/* SPDX-License-Identifier: BSD-3-Clause
 * Copyright(c) 2020 Intel Corporation
 */
#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include <errno.h>
#include <inttypes.h>
#include <sys/queue.h>
#include <arpa/inet.h>

#include <rte_common.h>
#include <rte_prefetch.h>
#include <rte_byteorder.h>

#include "rte_swx_pipeline.h"
#include "rte_swx_ctl.h"

#define CHECK(condition, err_code)                                             \
do {                                                                           \
        if (!(condition))                                                      \
                return -(err_code);                                            \
} while (0)

#define CHECK_NAME(name, err_code)                                             \
        CHECK((name) &&                                                        \
              (name)[0] &&                                                     \
              (strnlen((name), RTE_SWX_NAME_SIZE) < RTE_SWX_NAME_SIZE),        \
              err_code)

#define CHECK_INSTRUCTION(instr, err_code)                                     \
        CHECK((instr) &&                                                       \
              (instr)[0] &&                                                    \
              (strnlen((instr), RTE_SWX_INSTRUCTION_SIZE) <                    \
               RTE_SWX_INSTRUCTION_SIZE),                                      \
              err_code)

#ifndef TRACE_LEVEL
#define TRACE_LEVEL 0
#endif

#if TRACE_LEVEL
#define TRACE(...) printf(__VA_ARGS__)
#else
#define TRACE(...)
#endif

#define ntoh64(x) rte_be_to_cpu_64(x)
#define hton64(x) rte_cpu_to_be_64(x)

/*
 * Struct.
 */
struct field {
        char name[RTE_SWX_NAME_SIZE];
        uint32_t n_bits;
        uint32_t offset;
};

struct struct_type {
        TAILQ_ENTRY(struct_type) node;
        char name[RTE_SWX_NAME_SIZE];
        struct field *fields;
        uint32_t n_fields;
        uint32_t n_bits;
};

TAILQ_HEAD(struct_type_tailq, struct_type);

/*
 * Input port.
 */
struct port_in_type {
        TAILQ_ENTRY(port_in_type) node;
        char name[RTE_SWX_NAME_SIZE];
        struct rte_swx_port_in_ops ops;
};

TAILQ_HEAD(port_in_type_tailq, port_in_type);

struct port_in {
        TAILQ_ENTRY(port_in) node;
        struct port_in_type *type;
        void *obj;
        uint32_t id;
};

TAILQ_HEAD(port_in_tailq, port_in);

struct port_in_runtime {
        rte_swx_port_in_pkt_rx_t pkt_rx;
        void *obj;
};

/*
 * Output port.
 */
struct port_out_type {
        TAILQ_ENTRY(port_out_type) node;
        char name[RTE_SWX_NAME_SIZE];
        struct rte_swx_port_out_ops ops;
};

TAILQ_HEAD(port_out_type_tailq, port_out_type);

struct port_out {
        TAILQ_ENTRY(port_out) node;
        struct port_out_type *type;
        void *obj;
        uint32_t id;
};

TAILQ_HEAD(port_out_tailq, port_out);

struct port_out_runtime {
        rte_swx_port_out_pkt_tx_t pkt_tx;
        rte_swx_port_out_flush_t flush;
        void *obj;
};

/*
 * Extern object.
 */
struct extern_type_member_func {
        TAILQ_ENTRY(extern_type_member_func) node;
        char name[RTE_SWX_NAME_SIZE];
        rte_swx_extern_type_member_func_t func;
        uint32_t id;
};

TAILQ_HEAD(extern_type_member_func_tailq, extern_type_member_func);

struct extern_type {
        TAILQ_ENTRY(extern_type) node;
        char name[RTE_SWX_NAME_SIZE];
        struct struct_type *mailbox_struct_type;
        rte_swx_extern_type_constructor_t constructor;
        rte_swx_extern_type_destructor_t destructor;
        struct extern_type_member_func_tailq funcs;
        uint32_t n_funcs;
};

TAILQ_HEAD(extern_type_tailq, extern_type);

struct extern_obj {
        TAILQ_ENTRY(extern_obj) node;
        char name[RTE_SWX_NAME_SIZE];
        struct extern_type *type;
        void *obj;
        uint32_t struct_id;
        uint32_t id;
};

TAILQ_HEAD(extern_obj_tailq, extern_obj);

#ifndef RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX
#define RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX 8
#endif

struct extern_obj_runtime {
        void *obj;
        uint8_t *mailbox;
        rte_swx_extern_type_member_func_t funcs[RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX];
};

/*
 * Extern function.
 */
struct extern_func {
        TAILQ_ENTRY(extern_func) node;
        char name[RTE_SWX_NAME_SIZE];
        struct struct_type *mailbox_struct_type;
        rte_swx_extern_func_t func;
        uint32_t struct_id;
        uint32_t id;
};

TAILQ_HEAD(extern_func_tailq, extern_func);

struct extern_func_runtime {
        uint8_t *mailbox;
        rte_swx_extern_func_t func;
};

/*
 * Header.
 */
struct header {
        TAILQ_ENTRY(header) node;
        char name[RTE_SWX_NAME_SIZE];
        struct struct_type *st;
        uint32_t struct_id;
        uint32_t id;
};

TAILQ_HEAD(header_tailq, header);

struct header_runtime {
        uint8_t *ptr0;
};

struct header_out_runtime {
        uint8_t *ptr0;
        uint8_t *ptr;
        uint32_t n_bytes;
};

/*
 * Instruction.
 */

/* Packet headers are always in Network Byte Order (NBO), i.e. big endian.
 * Packet meta-data fields are always assumed to be in Host Byte Order (HBO).
 * Table entry fields can be in either NBO or HBO; they are assumed to be in HBO
 * when transferred to packet meta-data and in NBO when transferred to packet
 * headers.
 */

/* Notation conventions:
 *    -Header field: H = h.header.field (dst/src)
 *    -Meta-data field: M = m.field (dst/src)
 *    -Extern object mailbox field: E = e.field (dst/src)
 *    -Extern function mailbox field: F = f.field (dst/src)
 *    -Table action data field: T = t.field (src only)
 *    -Immediate value: I = 32-bit unsigned value (src only)
 */

enum instruction_type {
        /* rx m.port_in */
        INSTR_RX,

        /* tx m.port_out */
        INSTR_TX,

        /* extract h.header */
        INSTR_HDR_EXTRACT,
        INSTR_HDR_EXTRACT2,
        INSTR_HDR_EXTRACT3,
        INSTR_HDR_EXTRACT4,
        INSTR_HDR_EXTRACT5,
        INSTR_HDR_EXTRACT6,
        INSTR_HDR_EXTRACT7,
        INSTR_HDR_EXTRACT8,

        /* emit h.header */
        INSTR_HDR_EMIT,
        INSTR_HDR_EMIT_TX,
        INSTR_HDR_EMIT2_TX,
        INSTR_HDR_EMIT3_TX,
        INSTR_HDR_EMIT4_TX,
        INSTR_HDR_EMIT5_TX,
        INSTR_HDR_EMIT6_TX,
        INSTR_HDR_EMIT7_TX,
        INSTR_HDR_EMIT8_TX,

        /* validate h.header */
        INSTR_HDR_VALIDATE,

        /* invalidate h.header */
        INSTR_HDR_INVALIDATE,

        /* mov dst src
         * dst = src
         * dst = HMEF, src = HMEFTI
         */
        INSTR_MOV,   /* dst = MEF, src = MEFT */
        INSTR_MOV_S, /* (dst, src) = (MEF, H) or (dst, src) = (H, MEFT) */
        INSTR_MOV_I, /* dst = HMEF, src = I */

        /* dma h.header t.field
         * memcpy(h.header, t.field, sizeof(h.header))
         */
        INSTR_DMA_HT,
        INSTR_DMA_HT2,
        INSTR_DMA_HT3,
        INSTR_DMA_HT4,
        INSTR_DMA_HT5,
        INSTR_DMA_HT6,
        INSTR_DMA_HT7,
        INSTR_DMA_HT8,

        /* add dst src
         * dst += src
         * dst = HMEF, src = HMEFTI
         */
        INSTR_ALU_ADD,    /* dst = MEF, src = MEF */
        INSTR_ALU_ADD_MH, /* dst = MEF, src = H */
        INSTR_ALU_ADD_HM, /* dst = H, src = MEF */
        INSTR_ALU_ADD_HH, /* dst = H, src = H */
        INSTR_ALU_ADD_MI, /* dst = MEF, src = I */
        INSTR_ALU_ADD_HI, /* dst = H, src = I */

        /* sub dst src
         * dst -= src
         * dst = HMEF, src = HMEFTI
         */
        INSTR_ALU_SUB,    /* dst = MEF, src = MEF */
        INSTR_ALU_SUB_MH, /* dst = MEF, src = H */
        INSTR_ALU_SUB_HM, /* dst = H, src = MEF */
        INSTR_ALU_SUB_HH, /* dst = H, src = H */
        INSTR_ALU_SUB_MI, /* dst = MEF, src = I */
        INSTR_ALU_SUB_HI, /* dst = H, src = I */

        /* ckadd dst src
         * dst = dst '+ src[0:1] '+ src[2:3] + ...
         * dst = H, src = {H, h.header}
         */
        INSTR_ALU_CKADD_FIELD,    /* src = H */
        INSTR_ALU_CKADD_STRUCT20, /* src = h.header, with sizeof(header) = 20 */
        INSTR_ALU_CKADD_STRUCT,   /* src = h.hdeader, with any sizeof(header) */

        /* cksub dst src
         * dst = dst '- src
         * dst = H, src = H
         */
        INSTR_ALU_CKSUB_FIELD,

        /* and dst src
         * dst &= src
         * dst = HMEF, src = HMEFTI
         */
        INSTR_ALU_AND,   /* dst = MEF, src = MEFT */
        INSTR_ALU_AND_S, /* (dst, src) = (MEF, H) or (dst, src) = (H, MEFT) */
        INSTR_ALU_AND_I, /* dst = HMEF, src = I */

        /* or dst src
         * dst |= src
         * dst = HMEF, src = HMEFTI
         */
        INSTR_ALU_OR,   /* dst = MEF, src = MEFT */
        INSTR_ALU_OR_S, /* (dst, src) = (MEF, H) or (dst, src) = (H, MEFT) */
        INSTR_ALU_OR_I, /* dst = HMEF, src = I */

        /* xor dst src
         * dst ^= src
         * dst = HMEF, src = HMEFTI
         */
        INSTR_ALU_XOR,   /* dst = MEF, src = MEFT */
        INSTR_ALU_XOR_S, /* (dst, src) = (MEF, H) or (dst, src) = (H, MEFT) */
        INSTR_ALU_XOR_I, /* dst = HMEF, src = I */

        /* shl dst src
         * dst <<= src
         * dst = HMEF, src = HMEFTI
         */
        INSTR_ALU_SHL,    /* dst = MEF, src = MEF */
        INSTR_ALU_SHL_MH, /* dst = MEF, src = H */
        INSTR_ALU_SHL_HM, /* dst = H, src = MEF */
        INSTR_ALU_SHL_HH, /* dst = H, src = H */
        INSTR_ALU_SHL_MI, /* dst = MEF, src = I */
        INSTR_ALU_SHL_HI, /* dst = H, src = I */

        /* shr dst src
         * dst >>= src
         * dst = HMEF, src = HMEFTI
         */
        INSTR_ALU_SHR,    /* dst = MEF, src = MEF */
        INSTR_ALU_SHR_MH, /* dst = MEF, src = H */
        INSTR_ALU_SHR_HM, /* dst = H, src = MEF */
        INSTR_ALU_SHR_HH, /* dst = H, src = H */
        INSTR_ALU_SHR_MI, /* dst = MEF, src = I */
        INSTR_ALU_SHR_HI, /* dst = H, src = I */

        /* table TABLE */
        INSTR_TABLE,

        /* extern e.obj.func */
        INSTR_EXTERN_OBJ,

        /* extern f.func */
        INSTR_EXTERN_FUNC,

        /* jmp LABEL
         * Unconditional jump
         */
        INSTR_JMP,

        /* jmpv LABEL h.header
         * Jump if header is valid
         */
        INSTR_JMP_VALID,

        /* jmpnv LABEL h.header
         * Jump if header is invalid
         */
        INSTR_JMP_INVALID,

        /* jmph LABEL
         * Jump if table lookup hit
         */
        INSTR_JMP_HIT,

        /* jmpnh LABEL
         * Jump if table lookup miss
         */
        INSTR_JMP_MISS,

        /* jmpa LABEL ACTION
         * Jump if action run
         */
        INSTR_JMP_ACTION_HIT,

        /* jmpna LABEL ACTION
         * Jump if action not run
         */
        INSTR_JMP_ACTION_MISS,

        /* jmpeq LABEL a b
         * Jump is a is equal to b
         * a = HMEFT, b = HMEFTI
         */
        INSTR_JMP_EQ,   /* (a, b) = (MEFT, MEFT) or (a, b) = (H, H) */
        INSTR_JMP_EQ_S, /* (a, b) = (MEFT, H) or (a, b) = (H, MEFT) */
        INSTR_JMP_EQ_I, /* (a, b) = (MEFT, I) or (a, b) = (H, I) */

        /* jmpneq LABEL a b
         * Jump is a is not equal to b
         * a = HMEFT, b = HMEFTI
         */
        INSTR_JMP_NEQ,   /* (a, b) = (MEFT, MEFT) or (a, b) = (H, H) */
        INSTR_JMP_NEQ_S, /* (a, b) = (MEFT, H) or (a, b) = (H, MEFT) */
        INSTR_JMP_NEQ_I, /* (a, b) = (MEFT, I) or (a, b) = (H, I) */

        /* jmplt LABEL a b
         * Jump if a is less than b
         * a = HMEFT, b = HMEFTI
         */
        INSTR_JMP_LT,    /* a = MEF, b = MEF */
        INSTR_JMP_LT_MH, /* a = MEF, b = H */
        INSTR_JMP_LT_HM, /* a = H, b = MEF */
        INSTR_JMP_LT_HH, /* a = H, b = H */
        INSTR_JMP_LT_MI, /* a = MEF, b = I */
        INSTR_JMP_LT_HI, /* a = H, b = I */

        /* jmpgt LABEL a b
         * Jump if a is greater than b
         * a = HMEFT, b = HMEFTI
         */
        INSTR_JMP_GT,    /* a = MEF, b = MEF */
        INSTR_JMP_GT_MH, /* a = MEF, b = H */
        INSTR_JMP_GT_HM, /* a = H, b = MEF */
        INSTR_JMP_GT_HH, /* a = H, b = H */
        INSTR_JMP_GT_MI, /* a = MEF, b = I */
        INSTR_JMP_GT_HI, /* a = H, b = I */

        /* return
         * Return from action
         */
        INSTR_RETURN,
};

struct instr_operand {
        uint8_t struct_id;
        uint8_t n_bits;
        uint8_t offset;
        uint8_t pad;
};

struct instr_io {
        struct {
                uint8_t offset;
                uint8_t n_bits;
                uint8_t pad[2];
        } io;

        struct {
                uint8_t header_id[8];
                uint8_t struct_id[8];
                uint8_t n_bytes[8];
        } hdr;
};

struct instr_hdr_validity {
        uint8_t header_id;
};

struct instr_table {
        uint8_t table_id;
};

struct instr_extern_obj {
        uint8_t ext_obj_id;
        uint8_t func_id;
};

struct instr_extern_func {
        uint8_t ext_func_id;
};

struct instr_dst_src {
        struct instr_operand dst;
        union {
                struct instr_operand src;
                uint64_t src_val;
        };
};

struct instr_dma {
        struct {
                uint8_t header_id[8];
                uint8_t struct_id[8];
        } dst;

        struct {
                uint8_t offset[8];
        } src;

        uint16_t n_bytes[8];
};

struct instr_jmp {
        struct instruction *ip;

        union {
                struct instr_operand a;
                uint8_t header_id;
                uint8_t action_id;
        };

        union {
                struct instr_operand b;
                uint64_t b_val;
        };
};

struct instruction {
        enum instruction_type type;
        union {
                struct instr_io io;
                struct instr_hdr_validity valid;
                struct instr_dst_src mov;
                struct instr_dma dma;
                struct instr_dst_src alu;
                struct instr_table table;
                struct instr_extern_obj ext_obj;
                struct instr_extern_func ext_func;
                struct instr_jmp jmp;
        };
};

struct instruction_data {
        char label[RTE_SWX_NAME_SIZE];
        char jmp_label[RTE_SWX_NAME_SIZE];
        uint32_t n_users; /* user = jmp instruction to this instruction. */
        int invalid;
};

/*
 * Action.
 */
struct action {
        TAILQ_ENTRY(action) node;
        char name[RTE_SWX_NAME_SIZE];
        struct struct_type *st;
        struct instruction *instructions;
        uint32_t n_instructions;
        uint32_t id;
};

TAILQ_HEAD(action_tailq, action);

/*
 * Table.
 */
struct table_type {
        TAILQ_ENTRY(table_type) node;
        char name[RTE_SWX_NAME_SIZE];
        enum rte_swx_table_match_type match_type;
        struct rte_swx_table_ops ops;
};

TAILQ_HEAD(table_type_tailq, table_type);

struct match_field {
        enum rte_swx_table_match_type match_type;
        struct field *field;
};

struct table {
        TAILQ_ENTRY(table) node;
        char name[RTE_SWX_NAME_SIZE];
        char args[RTE_SWX_NAME_SIZE];
        struct table_type *type; /* NULL when n_fields == 0. */

        /* Match. */
        struct match_field *fields;
        uint32_t n_fields;
        int is_header; /* Only valid when n_fields > 0. */
        struct header *header; /* Only valid when n_fields > 0. */

        /* Action. */
        struct action **actions;
        struct action *default_action;
        uint8_t *default_action_data;
        uint32_t n_actions;
        int default_action_is_const;
        uint32_t action_data_size_max;

        uint32_t size;
        uint32_t id;
};

TAILQ_HEAD(table_tailq, table);

struct table_runtime {
        rte_swx_table_lookup_t func;
        void *mailbox;
        uint8_t **key;
};

/*
 * Pipeline.
 */
struct thread {
        /* Packet. */
        struct rte_swx_pkt pkt;
        uint8_t *ptr;

        /* Structures. */
        uint8_t **structs;

        /* Packet headers. */
        struct header_runtime *headers; /* Extracted or generated headers. */
        struct header_out_runtime *headers_out; /* Emitted headers. */
        uint8_t *header_storage;
        uint8_t *header_out_storage;
        uint64_t valid_headers;
        uint32_t n_headers_out;

        /* Packet meta-data. */
        uint8_t *metadata;

        /* Tables. */
        struct table_runtime *tables;
        struct rte_swx_table_state *table_state;
        uint64_t action_id;
        int hit; /* 0 = Miss, 1 = Hit. */

        /* Extern objects and functions. */
        struct extern_obj_runtime *extern_objs;
        struct extern_func_runtime *extern_funcs;

        /* Instructions. */
        struct instruction *ip;
        struct instruction *ret;
};

#define MASK64_BIT_GET(mask, pos) ((mask) & (1LLU << (pos)))
#define MASK64_BIT_SET(mask, pos) ((mask) | (1LLU << (pos)))
#define MASK64_BIT_CLR(mask, pos) ((mask) & ~(1LLU << (pos)))

#define HEADER_VALID(thread, header_id) \
        MASK64_BIT_GET((thread)->valid_headers, header_id)

#define ALU(thread, ip, operator)  \
{                                                                              \
        uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
        uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
        uint64_t dst64 = *dst64_ptr;                                           \
        uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
        uint64_t dst = dst64 & dst64_mask;                                     \
                                                                               \
        uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
        uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
        uint64_t src64 = *src64_ptr;                                           \
        uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits);       \
        uint64_t src = src64 & src64_mask;                                     \
                                                                               \
        uint64_t result = dst operator src;                                    \
                                                                               \
        *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask);            \
}

#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN

#define ALU_S(thread, ip, operator)  \
{                                                                              \
        uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
        uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
        uint64_t dst64 = *dst64_ptr;                                           \
        uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
        uint64_t dst = dst64 & dst64_mask;                                     \
                                                                               \
        uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
        uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
        uint64_t src64 = *src64_ptr;                                           \
        uint64_t src = ntoh64(src64) >> (64 - (ip)->alu.src.n_bits);           \
                                                                               \
        uint64_t result = dst operator src;                                    \
                                                                               \
        *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask);            \
}

#define ALU_MH ALU_S

#define ALU_HM(thread, ip, operator)  \
{                                                                              \
        uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
        uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
        uint64_t dst64 = *dst64_ptr;                                           \
        uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
        uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits);           \
                                                                               \
        uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
        uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
        uint64_t src64 = *src64_ptr;                                           \
        uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits);       \
        uint64_t src = src64 & src64_mask;                                     \
                                                                               \
        uint64_t result = dst operator src;                                    \
        result = hton64(result << (64 - (ip)->alu.dst.n_bits));                \
                                                                               \
        *dst64_ptr = (dst64 & ~dst64_mask) | result;                           \
}

#define ALU_HH(thread, ip, operator)  \
{                                                                              \
        uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
        uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
        uint64_t dst64 = *dst64_ptr;                                           \
        uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
        uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits);           \
                                                                               \
        uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id];      \
        uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset];   \
        uint64_t src64 = *src64_ptr;                                           \
        uint64_t src = ntoh64(src64) >> (64 - (ip)->alu.src.n_bits);           \
                                                                               \
        uint64_t result = dst operator src;                                    \
        result = hton64(result << (64 - (ip)->alu.dst.n_bits));                \
                                                                               \
        *dst64_ptr = (dst64 & ~dst64_mask) | result;                           \
}

#else

#define ALU_S ALU
#define ALU_MH ALU
#define ALU_HM ALU
#define ALU_HH ALU

#endif

#define ALU_I(thread, ip, operator)  \
{                                                                              \
        uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
        uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
        uint64_t dst64 = *dst64_ptr;                                           \
        uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
        uint64_t dst = dst64 & dst64_mask;                                     \
                                                                               \
        uint64_t src = (ip)->alu.src_val;                                      \
                                                                               \
        uint64_t result = dst operator src;                                    \
                                                                               \
        *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask);            \
}

#define ALU_MI ALU_I

#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN

#define ALU_HI(thread, ip, operator)  \
{                                                                              \
        uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id];      \
        uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset];   \
        uint64_t dst64 = *dst64_ptr;                                           \
        uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits);       \
        uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits);           \
                                                                               \
        uint64_t src = (ip)->alu.src_val;                                      \
                                                                               \
        uint64_t result = dst operator src;                                    \
        result = hton64(result << (64 - (ip)->alu.dst.n_bits));                \
                                                                               \
        *dst64_ptr = (dst64 & ~dst64_mask) | result;                           \
}

#else

#define ALU_HI ALU_I

#endif

#define MOV(thread, ip)  \
{                                                                              \
        uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
        uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
        uint64_t dst64 = *dst64_ptr;                                           \
        uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
                                                                               \
        uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id];      \
        uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset];   \
        uint64_t src64 = *src64_ptr;                                           \
        uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->mov.src.n_bits);       \
        uint64_t src = src64 & src64_mask;                                     \
                                                                               \
        *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);               \
}

#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN

#define MOV_S(thread, ip)  \
{                                                                              \
        uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
        uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
        uint64_t dst64 = *dst64_ptr;                                           \
        uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
                                                                               \
        uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id];      \
        uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset];   \
        uint64_t src64 = *src64_ptr;                                           \
        uint64_t src = ntoh64(src64) >> (64 - (ip)->mov.src.n_bits);           \
                                                                               \
        *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);               \
}

#else

#define MOV_S MOV

#endif

#define MOV_I(thread, ip)  \
{                                                                              \
        uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id];      \
        uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset];   \
        uint64_t dst64 = *dst64_ptr;                                           \
        uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits);       \
                                                                               \
        uint64_t src = (ip)->mov.src_val;                                      \
                                                                               \
        *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);               \
}

#define JMP_CMP(thread, ip, operator)  \
{                                                                              \
        uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
        uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
        uint64_t a64 = *a64_ptr;                                               \
        uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits);           \
        uint64_t a = a64 & a64_mask;                                           \
                                                                               \
        uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id];          \
        uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset];         \
        uint64_t b64 = *b64_ptr;                                               \
        uint64_t b64_mask = UINT64_MAX >> (64 - (ip)->jmp.b.n_bits);           \
        uint64_t b = b64 & b64_mask;                                           \
                                                                               \
        (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
}

#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN

#define JMP_CMP_S(thread, ip, operator)  \
{                                                                              \
        uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
        uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
        uint64_t a64 = *a64_ptr;                                               \
        uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits);           \
        uint64_t a = a64 & a64_mask;                                           \
                                                                               \
        uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id];          \
        uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset];         \
        uint64_t b64 = *b64_ptr;                                               \
        uint64_t b = ntoh64(b64) >> (64 - (ip)->jmp.b.n_bits);                 \
                                                                               \
        (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
}

#define JMP_CMP_MH JMP_CMP_S

#define JMP_CMP_HM(thread, ip, operator)  \
{                                                                              \
        uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
        uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
        uint64_t a64 = *a64_ptr;                                               \
        uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits);                 \
                                                                               \
        uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id];          \
        uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset];         \
        uint64_t b64 = *b64_ptr;                                               \
        uint64_t b64_mask = UINT64_MAX >> (64 - (ip)->jmp.b.n_bits);           \
        uint64_t b = b64 & b64_mask;                                           \
                                                                               \
        (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
}

#define JMP_CMP_HH(thread, ip, operator)  \
{                                                                              \
        uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
        uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
        uint64_t a64 = *a64_ptr;                                               \
        uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits);                 \
                                                                               \
        uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id];          \
        uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset];         \
        uint64_t b64 = *b64_ptr;                                               \
        uint64_t b = ntoh64(b64) >> (64 - (ip)->jmp.b.n_bits);                 \
                                                                               \
        (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
}

#else

#define JMP_CMP_S JMP_CMP
#define JMP_CMP_MH JMP_CMP
#define JMP_CMP_HM JMP_CMP
#define JMP_CMP_HH JMP_CMP

#endif

#define JMP_CMP_I(thread, ip, operator)  \
{                                                                              \
        uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
        uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
        uint64_t a64 = *a64_ptr;                                               \
        uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits);           \
        uint64_t a = a64 & a64_mask;                                           \
                                                                               \
        uint64_t b = (ip)->jmp.b_val;                                          \
                                                                               \
        (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
}

#define JMP_CMP_MI JMP_CMP_I

#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN

#define JMP_CMP_HI(thread, ip, operator)  \
{                                                                              \
        uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id];          \
        uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset];         \
        uint64_t a64 = *a64_ptr;                                               \
        uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits);                 \
                                                                               \
        uint64_t b = (ip)->jmp.b_val;                                          \
                                                                               \
        (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1);     \
}

#else

#define JMP_CMP_HI JMP_CMP_I

#endif

#define METADATA_READ(thread, offset, n_bits)                                  \
({                                                                             \
        uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset];           \
        uint64_t m64 = *m64_ptr;                                               \
        uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits));                     \
        (m64 & m64_mask);                                                      \
})

#define METADATA_WRITE(thread, offset, n_bits, value)                          \
{                                                                              \
        uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset];           \
        uint64_t m64 = *m64_ptr;                                               \
        uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits));                     \
                                                                               \
        uint64_t m_new = value;                                                \
                                                                               \
        *m64_ptr = (m64 & ~m64_mask) | (m_new & m64_mask);                     \
}

#ifndef RTE_SWX_PIPELINE_THREADS_MAX
#define RTE_SWX_PIPELINE_THREADS_MAX 16
#endif

struct rte_swx_pipeline {
        struct struct_type_tailq struct_types;
        struct port_in_type_tailq port_in_types;
        struct port_in_tailq ports_in;
        struct port_out_type_tailq port_out_types;
        struct port_out_tailq ports_out;
        struct extern_type_tailq extern_types;
        struct extern_obj_tailq extern_objs;
        struct extern_func_tailq extern_funcs;
        struct header_tailq headers;
        struct struct_type *metadata_st;
        uint32_t metadata_struct_id;
        struct action_tailq actions;
        struct table_type_tailq table_types;
        struct table_tailq tables;

        struct port_in_runtime *in;
        struct port_out_runtime *out;
        struct instruction **action_instructions;
        struct rte_swx_table_state *table_state;
        struct instruction *instructions;
        struct thread threads[RTE_SWX_PIPELINE_THREADS_MAX];

        uint32_t n_structs;
        uint32_t n_ports_in;
        uint32_t n_ports_out;
        uint32_t n_extern_objs;
        uint32_t n_extern_funcs;
        uint32_t n_actions;
        uint32_t n_tables;

        uint32_t n_headers;
        uint32_t thread_id;
        uint32_t port_id;
        uint32_t n_instructions;
        int build_done;
        int numa_node;
};

/*
 * Struct.
 */
static struct struct_type *
struct_type_find(struct rte_swx_pipeline *p, const char *name)
{
        struct struct_type *elem;

        TAILQ_FOREACH(elem, &p->struct_types, node)
                if (strcmp(elem->name, name) == 0)
                        return elem;

        return NULL;
}

static struct field *
struct_type_field_find(struct struct_type *st, const char *name)
{
        uint32_t i;

        for (i = 0; i < st->n_fields; i++) {
                struct field *f = &st->fields[i];

                if (strcmp(f->name, name) == 0)
                        return f;
        }

        return NULL;
}

int
rte_swx_pipeline_struct_type_register(struct rte_swx_pipeline *p,
                                      const char *name,
                                      struct rte_swx_field_params *fields,
                                      uint32_t n_fields)
{
        struct struct_type *st;
        uint32_t i;

        CHECK(p, EINVAL);
        CHECK_NAME(name, EINVAL);
        CHECK(fields, EINVAL);
        CHECK(n_fields, EINVAL);

        for (i = 0; i < n_fields; i++) {
                struct rte_swx_field_params *f = &fields[i];
                uint32_t j;

                CHECK_NAME(f->name, EINVAL);
                CHECK(f->n_bits, EINVAL);
                CHECK(f->n_bits <= 64, EINVAL);
                CHECK((f->n_bits & 7) == 0, EINVAL);

                for (j = 0; j < i; j++) {
                        struct rte_swx_field_params *f_prev = &fields[j];

                        CHECK(strcmp(f->name, f_prev->name), EINVAL);
                }
        }

        CHECK(!struct_type_find(p, name), EEXIST);

        /* Node allocation. */
        st = calloc(1, sizeof(struct struct_type));
        CHECK(st, ENOMEM);

        st->fields = calloc(n_fields, sizeof(struct field));
        if (!st->fields) {
                free(st);
                CHECK(0, ENOMEM);
        }

        /* Node initialization. */
        strcpy(st->name, name);
        for (i = 0; i < n_fields; i++) {
                struct field *dst = &st->fields[i];
                struct rte_swx_field_params *src = &fields[i];

                strcpy(dst->name, src->name);
                dst->n_bits = src->n_bits;
                dst->offset = st->n_bits;

                st->n_bits += src->n_bits;
        }
        st->n_fields = n_fields;

        /* Node add to tailq. */
        TAILQ_INSERT_TAIL(&p->struct_types, st, node);

        return 0;
}

static int
struct_build(struct rte_swx_pipeline *p)
{
        uint32_t i;

        for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
                struct thread *t = &p->threads[i];

                t->structs = calloc(p->n_structs, sizeof(uint8_t *));
                CHECK(t->structs, ENOMEM);
        }

        return 0;
}

static void
struct_build_free(struct rte_swx_pipeline *p)
{
        uint32_t i;

        for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
                struct thread *t = &p->threads[i];

                free(t->structs);
                t->structs = NULL;
        }
}

static void
struct_free(struct rte_swx_pipeline *p)
{
        struct_build_free(p);

        /* Struct types. */
        for ( ; ; ) {
                struct struct_type *elem;

                elem = TAILQ_FIRST(&p->struct_types);
                if (!elem)
                        break;

                TAILQ_REMOVE(&p->struct_types, elem, node);
                free(elem->fields);
                free(elem);
        }
}

/*
 * Input port.
 */
static struct port_in_type *
port_in_type_find(struct rte_swx_pipeline *p, const char *name)
{
        struct port_in_type *elem;

        if (!name)
                return NULL;

        TAILQ_FOREACH(elem, &p->port_in_types, node)
                if (strcmp(elem->name, name) == 0)
                        return elem;

        return NULL;
}

int
rte_swx_pipeline_port_in_type_register(struct rte_swx_pipeline *p,
                                       const char *name,
                                       struct rte_swx_port_in_ops *ops)
{
        struct port_in_type *elem;

        CHECK(p, EINVAL);
        CHECK_NAME(name, EINVAL);
        CHECK(ops, EINVAL);
        CHECK(ops->create, EINVAL);
        CHECK(ops->free, EINVAL);
        CHECK(ops->pkt_rx, EINVAL);
        CHECK(ops->stats_read, EINVAL);

        CHECK(!port_in_type_find(p, name), EEXIST);

        /* Node allocation. */
        elem = calloc(1, sizeof(struct port_in_type));
        CHECK(elem, ENOMEM);

        /* Node initialization. */
        strcpy(elem->name, name);
        memcpy(&elem->ops, ops, sizeof(*ops));

        /* Node add to tailq. */
        TAILQ_INSERT_TAIL(&p->port_in_types, elem, node);

        return 0;
}

static struct port_in *
port_in_find(struct rte_swx_pipeline *p, uint32_t port_id)
{
        struct port_in *port;

        TAILQ_FOREACH(port, &p->ports_in, node)
                if (port->id == port_id)
                        return port;

        return NULL;
}

int
rte_swx_pipeline_port_in_config(struct rte_swx_pipeline *p,
                                uint32_t port_id,
                                const char *port_type_name,
                                void *args)
{
        struct port_in_type *type = NULL;
        struct port_in *port = NULL;
        void *obj = NULL;

        CHECK(p, EINVAL);

        CHECK(!port_in_find(p, port_id), EINVAL);

        CHECK_NAME(port_type_name, EINVAL);
        type = port_in_type_find(p, port_type_name);
        CHECK(type, EINVAL);

        obj = type->ops.create(args);
        CHECK(obj, ENODEV);

        /* Node allocation. */
        port = calloc(1, sizeof(struct port_in));
        CHECK(port, ENOMEM);

        /* Node initialization. */
        port->type = type;
        port->obj = obj;
        port->id = port_id;

        /* Node add to tailq. */
        TAILQ_INSERT_TAIL(&p->ports_in, port, node);
        if (p->n_ports_in < port_id + 1)
                p->n_ports_in = port_id + 1;

        return 0;
}

static int
port_in_build(struct rte_swx_pipeline *p)
{
        struct port_in *port;
        uint32_t i;

        CHECK(p->n_ports_in, EINVAL);
        CHECK(rte_is_power_of_2(p->n_ports_in), EINVAL);

        for (i = 0; i < p->n_ports_in; i++)
                CHECK(port_in_find(p, i), EINVAL);

        p->in = calloc(p->n_ports_in, sizeof(struct port_in_runtime));
        CHECK(p->in, ENOMEM);

        TAILQ_FOREACH(port, &p->ports_in, node) {
                struct port_in_runtime *in = &p->in[port->id];

                in->pkt_rx = port->type->ops.pkt_rx;
                in->obj = port->obj;
        }

        return 0;
}

static void
port_in_build_free(struct rte_swx_pipeline *p)
{
        free(p->in);
        p->in = NULL;
}

static void
port_in_free(struct rte_swx_pipeline *p)
{
        port_in_build_free(p);

        /* Input ports. */
        for ( ; ; ) {
                struct port_in *port;

                port = TAILQ_FIRST(&p->ports_in);
                if (!port)
                        break;

                TAILQ_REMOVE(&p->ports_in, port, node);
                port->type->ops.free(port->obj);
                free(port);
        }

        /* Input port types. */
        for ( ; ; ) {
                struct port_in_type *elem;

                elem = TAILQ_FIRST(&p->port_in_types);
                if (!elem)
                        break;

                TAILQ_REMOVE(&p->port_in_types, elem, node);
                free(elem);
        }
}

/*
 * Output port.
 */
static struct port_out_type *
port_out_type_find(struct rte_swx_pipeline *p, const char *name)
{
        struct port_out_type *elem;

        if (!name)
                return NULL;

        TAILQ_FOREACH(elem, &p->port_out_types, node)
                if (!strcmp(elem->name, name))
                        return elem;

        return NULL;
}

int
rte_swx_pipeline_port_out_type_register(struct rte_swx_pipeline *p,
                                        const char *name,
                                        struct rte_swx_port_out_ops *ops)
{
        struct port_out_type *elem;

        CHECK(p, EINVAL);
        CHECK_NAME(name, EINVAL);
        CHECK(ops, EINVAL);
        CHECK(ops->create, EINVAL);
        CHECK(ops->free, EINVAL);
        CHECK(ops->pkt_tx, EINVAL);
        CHECK(ops->stats_read, EINVAL);

        CHECK(!port_out_type_find(p, name), EEXIST);

        /* Node allocation. */
        elem = calloc(1, sizeof(struct port_out_type));
        CHECK(elem, ENOMEM);

        /* Node initialization. */
        strcpy(elem->name, name);
        memcpy(&elem->ops, ops, sizeof(*ops));

        /* Node add to tailq. */
        TAILQ_INSERT_TAIL(&p->port_out_types, elem, node);

        return 0;
}

static struct port_out *
port_out_find(struct rte_swx_pipeline *p, uint32_t port_id)
{
        struct port_out *port;

        TAILQ_FOREACH(port, &p->ports_out, node)
                if (port->id == port_id)
                        return port;

        return NULL;
}

int
rte_swx_pipeline_port_out_config(struct rte_swx_pipeline *p,
                                 uint32_t port_id,
                                 const char *port_type_name,
                                 void *args)
{
        struct port_out_type *type = NULL;
        struct port_out *port = NULL;
        void *obj = NULL;

        CHECK(p, EINVAL);

        CHECK(!port_out_find(p, port_id), EINVAL);

        CHECK_NAME(port_type_name, EINVAL);
        type = port_out_type_find(p, port_type_name);
        CHECK(type, EINVAL);

        obj = type->ops.create(args);
        CHECK(obj, ENODEV);

        /* Node allocation. */
        port = calloc(1, sizeof(struct port_out));
        CHECK(port, ENOMEM);

        /* Node initialization. */
        port->type = type;
        port->obj = obj;
        port->id = port_id;

        /* Node add to tailq. */
        TAILQ_INSERT_TAIL(&p->ports_out, port, node);
        if (p->n_ports_out < port_id + 1)
                p->n_ports_out = port_id + 1;

        return 0;
}

static int
port_out_build(struct rte_swx_pipeline *p)
{
        struct port_out *port;
        uint32_t i;

        CHECK(p->n_ports_out, EINVAL);

        for (i = 0; i < p->n_ports_out; i++)
                CHECK(port_out_find(p, i), EINVAL);

        p->out = calloc(p->n_ports_out, sizeof(struct port_out_runtime));
        CHECK(p->out, ENOMEM);

        TAILQ_FOREACH(port, &p->ports_out, node) {
                struct port_out_runtime *out = &p->out[port->id];

                out->pkt_tx = port->type->ops.pkt_tx;
                out->flush = port->type->ops.flush;
                out->obj = port->obj;
        }

        return 0;
}

static void
port_out_build_free(struct rte_swx_pipeline *p)
{
        free(p->out);
        p->out = NULL;
}

static void
port_out_free(struct rte_swx_pipeline *p)
{
        port_out_build_free(p);

        /* Output ports. */
        for ( ; ; ) {
                struct port_out *port;

                port = TAILQ_FIRST(&p->ports_out);
                if (!port)
                        break;

                TAILQ_REMOVE(&p->ports_out, port, node);
                port->type->ops.free(port->obj);
                free(port);
        }

        /* Output port types. */
        for ( ; ; ) {
                struct port_out_type *elem;

                elem = TAILQ_FIRST(&p->port_out_types);
                if (!elem)
                        break;

                TAILQ_REMOVE(&p->port_out_types, elem, node);
                free(elem);
        }
}

/*
 * Extern object.
 */
static struct extern_type *
extern_type_find(struct rte_swx_pipeline *p, const char *name)
{
        struct extern_type *elem;

        TAILQ_FOREACH(elem, &p->extern_types, node)
                if (strcmp(elem->name, name) == 0)
                        return elem;

        return NULL;
}

static struct extern_type_member_func *
extern_type_member_func_find(struct extern_type *type, const char *name)
{
        struct extern_type_member_func *elem;

        TAILQ_FOREACH(elem, &type->funcs, node)
                if (strcmp(elem->name, name) == 0)
                        return elem;

        return NULL;
}

static struct extern_obj *
extern_obj_find(struct rte_swx_pipeline *p, const char *name)
{
        struct extern_obj *elem;

        TAILQ_FOREACH(elem, &p->extern_objs, node)
                if (strcmp(elem->name, name) == 0)
                        return elem;

        return NULL;
}

static struct extern_type_member_func *
extern_obj_member_func_parse(struct rte_swx_pipeline *p,
                             const char *name,
                             struct extern_obj **obj)
{
        struct extern_obj *object;
        struct extern_type_member_func *func;
        char *object_name, *func_name;

        if (name[0] != 'e' || name[1] != '.')
                return NULL;

        object_name = strdup(&name[2]);
        if (!object_name)
                return NULL;

        func_name = strchr(object_name, '.');
        if (!func_name) {
                free(object_name);
                return NULL;
        }

        *func_name = 0;
        func_name++;

        object = extern_obj_find(p, object_name);
        if (!object) {
                free(object_name);
                return NULL;
        }

        func = extern_type_member_func_find(object->type, func_name);
        if (!func) {
                free(object_name);
                return NULL;
        }

        if (obj)
                *obj = object;

        free(object_name);
        return func;
}

static struct field *
extern_obj_mailbox_field_parse(struct rte_swx_pipeline *p,
                               const char *name,
                               struct extern_obj **object)
{
        struct extern_obj *obj;
        struct field *f;
        char *obj_name, *field_name;

        if ((name[0] != 'e') || (name[1] != '.'))
                return NULL;

        obj_name = strdup(&name[2]);
        if (!obj_name)
                return NULL;

        field_name = strchr(obj_name, '.');
        if (!field_name) {
                free(obj_name);
                return NULL;
        }

        *field_name = 0;
        field_name++;

        obj = extern_obj_find(p, obj_name);
        if (!obj) {
                free(obj_name);
                return NULL;
        }

        f = struct_type_field_find(obj->type->mailbox_struct_type, field_name);
        if (!f) {
                free(obj_name);
                return NULL;
        }

        if (object)
                *object = obj;

        free(obj_name);
        return f;
}

int
rte_swx_pipeline_extern_type_register(struct rte_swx_pipeline *p,
        const char *name,
        const char *mailbox_struct_type_name,
        rte_swx_extern_type_constructor_t constructor,
        rte_swx_extern_type_destructor_t destructor)
{
        struct extern_type *elem;
        struct struct_type *mailbox_struct_type;

        CHECK(p, EINVAL);

        CHECK_NAME(name, EINVAL);
        CHECK(!extern_type_find(p, name), EEXIST);

        CHECK_NAME(mailbox_struct_type_name, EINVAL);
        mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
        CHECK(mailbox_struct_type, EINVAL);

        CHECK(constructor, EINVAL);
        CHECK(destructor, EINVAL);

        /* Node allocation. */
        elem = calloc(1, sizeof(struct extern_type));
        CHECK(elem, ENOMEM);

        /* Node initialization. */
        strcpy(elem->name, name);
        elem->mailbox_struct_type = mailbox_struct_type;
        elem->constructor = constructor;
        elem->destructor = destructor;
        TAILQ_INIT(&elem->funcs);

        /* Node add to tailq. */
        TAILQ_INSERT_TAIL(&p->extern_types, elem, node);

        return 0;
}

int
rte_swx_pipeline_extern_type_member_func_register(struct rte_swx_pipeline *p,
        const char *extern_type_name,
        const char *name,
        rte_swx_extern_type_member_func_t member_func)
{
        struct extern_type *type;
        struct extern_type_member_func *type_member;

        CHECK(p, EINVAL);

        CHECK_NAME(extern_type_name, EINVAL);
        type = extern_type_find(p, extern_type_name);
        CHECK(type, EINVAL);
        CHECK(type->n_funcs < RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX, ENOSPC);

        CHECK_NAME(name, EINVAL);
        CHECK(!extern_type_member_func_find(type, name), EEXIST);

        CHECK(member_func, EINVAL);

        /* Node allocation. */
        type_member = calloc(1, sizeof(struct extern_type_member_func));
        CHECK(type_member, ENOMEM);

        /* Node initialization. */
        strcpy(type_member->name, name);
        type_member->func = member_func;
        type_member->id = type->n_funcs;

        /* Node add to tailq. */
        TAILQ_INSERT_TAIL(&type->funcs, type_member, node);
        type->n_funcs++;

        return 0;
}

int
rte_swx_pipeline_extern_object_config(struct rte_swx_pipeline *p,
                                      const char *extern_type_name,
                                      const char *name,
                                      const char *args)
{
        struct extern_type *type;
        struct extern_obj *obj;
        void *obj_handle;

        CHECK(p, EINVAL);

        CHECK_NAME(extern_type_name, EINVAL);
        type = extern_type_find(p, extern_type_name);
        CHECK(type, EINVAL);

        CHECK_NAME(name, EINVAL);
        CHECK(!extern_obj_find(p, name), EEXIST);

        /* Node allocation. */
        obj = calloc(1, sizeof(struct extern_obj));
        CHECK(obj, ENOMEM);

        /* Object construction. */
        obj_handle = type->constructor(args);
        if (!obj_handle) {
                free(obj);
                CHECK(0, ENODEV);
        }

        /* Node initialization. */
        strcpy(obj->name, name);
        obj->type = type;
        obj->obj = obj_handle;
        obj->struct_id = p->n_structs;
        obj->id = p->n_extern_objs;

        /* Node add to tailq. */
        TAILQ_INSERT_TAIL(&p->extern_objs, obj, node);
        p->n_extern_objs++;
        p->n_structs++;

        return 0;
}

static int
extern_obj_build(struct rte_swx_pipeline *p)
{
        uint32_t i;

        for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
                struct thread *t = &p->threads[i];
                struct extern_obj *obj;

                t->extern_objs = calloc(p->n_extern_objs,
                                        sizeof(struct extern_obj_runtime));
                CHECK(t->extern_objs, ENOMEM);

                TAILQ_FOREACH(obj, &p->extern_objs, node) {
                        struct extern_obj_runtime *r =
                                &t->extern_objs[obj->id];
                        struct extern_type_member_func *func;
                        uint32_t mailbox_size =
                                obj->type->mailbox_struct_type->n_bits / 8;

                        r->obj = obj->obj;

                        r->mailbox = calloc(1, mailbox_size);
                        CHECK(r->mailbox, ENOMEM);

                        TAILQ_FOREACH(func, &obj->type->funcs, node)
                                r->funcs[func->id] = func->func;

                        t->structs[obj->struct_id] = r->mailbox;
                }
        }

        return 0;
}

static void
extern_obj_build_free(struct rte_swx_pipeline *p)
{
        uint32_t i;

        for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
                struct thread *t = &p->threads[i];
                uint32_t j;

                if (!t->extern_objs)
                        continue;

                for (j = 0; j < p->n_extern_objs; j++) {
                        struct extern_obj_runtime *r = &t->extern_objs[j];

                        free(r->mailbox);
                }

                free(t->extern_objs);
                t->extern_objs = NULL;
        }
}

static void
extern_obj_free(struct rte_swx_pipeline *p)
{
        extern_obj_build_free(p);

        /* Extern objects. */
        for ( ; ; ) {
                struct extern_obj *elem;

                elem = TAILQ_FIRST(&p->extern_objs);
                if (!elem)
                        break;

                TAILQ_REMOVE(&p->extern_objs, elem, node);
                if (elem->obj)
                        elem->type->destructor(elem->obj);
                free(elem);
        }

        /* Extern types. */
        for ( ; ; ) {
                struct extern_type *elem;

                elem = TAILQ_FIRST(&p->extern_types);
                if (!elem)
                        break;

                TAILQ_REMOVE(&p->extern_types, elem, node);

                for ( ; ; ) {
                        struct extern_type_member_func *func;

                        func = TAILQ_FIRST(&elem->funcs);
                        if (!func)
                                break;

                        TAILQ_REMOVE(&elem->funcs, func, node);
                        free(func);
                }

                free(elem);
        }
}

/*
 * Extern function.
 */
static struct extern_func *
extern_func_find(struct rte_swx_pipeline *p, const char *name)
{
        struct extern_func *elem;

        TAILQ_FOREACH(elem, &p->extern_funcs, node)
                if (strcmp(elem->name, name) == 0)
                        return elem;

        return NULL;
}

static struct extern_func *
extern_func_parse(struct rte_swx_pipeline *p,
                  const char *name)
{
        if (name[0] != 'f' || name[1] != '.')
                return NULL;

        return extern_func_find(p, &name[2]);
}

static struct field *
extern_func_mailbox_field_parse(struct rte_swx_pipeline *p,
                                const char *name,
                                struct extern_func **function)
{
        struct extern_func *func;
        struct field *f;
        char *func_name, *field_name;

        if ((name[0] != 'f') || (name[1] != '.'))
                return NULL;

        func_name = strdup(&name[2]);
        if (!func_name)
                return NULL;

        field_name = strchr(func_name, '.');
        if (!field_name) {
                free(func_name);
                return NULL;
        }

        *field_name = 0;
        field_name++;

        func = extern_func_find(p, func_name);
        if (!func) {
                free(func_name);
                return NULL;
        }

        f = struct_type_field_find(func->mailbox_struct_type, field_name);
        if (!f) {
                free(func_name);
                return NULL;
        }

        if (function)
                *function = func;

        free(func_name);
        return f;
}

int
rte_swx_pipeline_extern_func_register(struct rte_swx_pipeline *p,
                                      const char *name,
                                      const char *mailbox_struct_type_name,
                                      rte_swx_extern_func_t func)
{
        struct extern_func *f;
        struct struct_type *mailbox_struct_type;

        CHECK(p, EINVAL);

        CHECK_NAME(name, EINVAL);
        CHECK(!extern_func_find(p, name), EEXIST);

        CHECK_NAME(mailbox_struct_type_name, EINVAL);
        mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
        CHECK(mailbox_struct_type, EINVAL);

        CHECK(func, EINVAL);

        /* Node allocation. */
        f = calloc(1, sizeof(struct extern_func));
        CHECK(func, ENOMEM);

        /* Node initialization. */
        strcpy(f->name, name);
        f->mailbox_struct_type = mailbox_struct_type;
        f->func = func;
        f->struct_id = p->n_structs;
        f->id = p->n_extern_funcs;

        /* Node add to tailq. */
        TAILQ_INSERT_TAIL(&p->extern_funcs, f, node);
        p->n_extern_funcs++;
        p->n_structs++;

        return 0;
}

static int
extern_func_build(struct rte_swx_pipeline *p)
{
        uint32_t i;

        for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
                struct thread *t = &p->threads[i];
                struct extern_func *func;

                /* Memory allocation. */
                t->extern_funcs = calloc(p->n_extern_funcs,
                                         sizeof(struct extern_func_runtime));
                CHECK(t->extern_funcs, ENOMEM);

                /* Extern function. */
                TAILQ_FOREACH(func, &p->extern_funcs, node) {
                        struct extern_func_runtime *r =
                                &t->extern_funcs[func->id];
                        uint32_t mailbox_size =
                                func->mailbox_struct_type->n_bits / 8;

                        r->func = func->func;

                        r->mailbox = calloc(1, mailbox_size);
                        CHECK(r->mailbox, ENOMEM);

                        t->structs[func->struct_id] = r->mailbox;
                }
        }

        return 0;
}

static void
extern_func_build_free(struct rte_swx_pipeline *p)
{
        uint32_t i;

        for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
                struct thread *t = &p->threads[i];
                uint32_t j;

                if (!t->extern_funcs)
                        continue;

                for (j = 0; j < p->n_extern_funcs; j++) {
                        struct extern_func_runtime *r = &t->extern_funcs[j];

                        free(r->mailbox);
                }

                free(t->extern_funcs);
                t->extern_funcs = NULL;
        }
}

static void
extern_func_free(struct rte_swx_pipeline *p)
{
        extern_func_build_free(p);

        for ( ; ; ) {
                struct extern_func *elem;

                elem = TAILQ_FIRST(&p->extern_funcs);
                if (!elem)
                        break;

                TAILQ_REMOVE(&p->extern_funcs, elem, node);
                free(elem);
        }

}

/*
 * Header.
 */
static struct header *
header_find(struct rte_swx_pipeline *p, const char *name)
{
        struct header *elem;

        TAILQ_FOREACH(elem, &p->headers, node)
                if (strcmp(elem->name, name) == 0)
                        return elem;

        return NULL;
}

static struct header *
header_parse(struct rte_swx_pipeline *p,
             const char *name)
{
        if (name[0] != 'h' || name[1] != '.')
                return NULL;

        return header_find(p, &name[2]);
}

static struct field *
header_field_parse(struct rte_swx_pipeline *p,
                   const char *name,
                   struct header **header)
{
        struct header *h;
        struct field *f;
        char *header_name, *field_name;

        if ((name[0] != 'h') || (name[1] != '.'))
                return NULL;

        header_name = strdup(&name[2]);
        if (!header_name)
                return NULL;

        field_name = strchr(header_name, '.');
        if (!field_name) {
                free(header_name);
                return NULL;
        }

        *field_name = 0;
        field_name++;

        h = header_find(p, header_name);
        if (!h) {
                free(header_name);
                return NULL;
        }

        f = struct_type_field_find(h->st, field_name);
        if (!f) {
                free(header_name);
                return NULL;
        }

        if (header)
                *header = h;

        free(header_name);
        return f;
}

int
rte_swx_pipeline_packet_header_register(struct rte_swx_pipeline *p,
                                        const char *name,
                                        const char *struct_type_name)
{
        struct struct_type *st;
        struct header *h;
        size_t n_headers_max;

        CHECK(p, EINVAL);
        CHECK_NAME(name, EINVAL);
        CHECK_NAME(struct_type_name, EINVAL);

        CHECK(!header_find(p, name), EEXIST);

        st = struct_type_find(p, struct_type_name);
        CHECK(st, EINVAL);

        n_headers_max = RTE_SIZEOF_FIELD(struct thread, valid_headers) * 8;
        CHECK(p->n_headers < n_headers_max, ENOSPC);

        /* Node allocation. */
        h = calloc(1, sizeof(struct header));
        CHECK(h, ENOMEM);

        /* Node initialization. */
        strcpy(h->name, name);
        h->st = st;
        h->struct_id = p->n_structs;
        h->id = p->n_headers;

        /* Node add to tailq. */
        TAILQ_INSERT_TAIL(&p->headers, h, node);
        p->n_headers++;
        p->n_structs++;

        return 0;
}

static int
header_build(struct rte_swx_pipeline *p)
{
        struct header *h;
        uint32_t n_bytes = 0, i;

        TAILQ_FOREACH(h, &p->headers, node) {
                n_bytes += h->st->n_bits / 8;
        }

        for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
                struct thread *t = &p->threads[i];
                uint32_t offset = 0;

                t->headers = calloc(p->n_headers,
                                    sizeof(struct header_runtime));
                CHECK(t->headers, ENOMEM);

                t->headers_out = calloc(p->n_headers,
                                        sizeof(struct header_out_runtime));
                CHECK(t->headers_out, ENOMEM);

                t->header_storage = calloc(1, n_bytes);
                CHECK(t->header_storage, ENOMEM);

                t->header_out_storage = calloc(1, n_bytes);
                CHECK(t->header_out_storage, ENOMEM);

                TAILQ_FOREACH(h, &p->headers, node) {
                        uint8_t *header_storage;

                        header_storage = &t->header_storage[offset];
                        offset += h->st->n_bits / 8;

                        t->headers[h->id].ptr0 = header_storage;
                        t->structs[h->struct_id] = header_storage;
                }
        }

        return 0;
}

static void
header_build_free(struct rte_swx_pipeline *p)
{
        uint32_t i;

        for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
                struct thread *t = &p->threads[i];

                free(t->headers_out);
                t->headers_out = NULL;

                free(t->headers);
                t->headers = NULL;

                free(t->header_out_storage);
                t->header_out_storage = NULL;

                free(t->header_storage);
                t->header_storage = NULL;
        }
}

static void
header_free(struct rte_swx_pipeline *p)
{
        header_build_free(p);

        for ( ; ; ) {
                struct header *elem;

                elem = TAILQ_FIRST(&p->headers);
                if (!elem)
                        break;

                TAILQ_REMOVE(&p->headers, elem, node);
                free(elem);
        }
}

/*
 * Meta-data.
 */
static struct field *
metadata_field_parse(struct rte_swx_pipeline *p, const char *name)
{
        if (!p->metadata_st)
                return NULL;

        if (name[0] != 'm' || name[1] != '.')
                return NULL;

        return struct_type_field_find(p->metadata_st, &name[2]);
}

int
rte_swx_pipeline_packet_metadata_register(struct rte_swx_pipeline *p,
                                          const char *struct_type_name)
{
        struct struct_type *st = NULL;

        CHECK(p, EINVAL);

        CHECK_NAME(struct_type_name, EINVAL);
        st  = struct_type_find(p, struct_type_name);
        CHECK(st, EINVAL);
        CHECK(!p->metadata_st, EINVAL);

        p->metadata_st = st;
        p->metadata_struct_id = p->n_structs;

        p->n_structs++;

        return 0;
}

static int
metadata_build(struct rte_swx_pipeline *p)
{
        uint32_t n_bytes = p->metadata_st->n_bits / 8;
        uint32_t i;

        /* Thread-level initialization. */
        for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
                struct thread *t = &p->threads[i];
                uint8_t *metadata;

                metadata = calloc(1, n_bytes);
                CHECK(metadata, ENOMEM);

                t->metadata = metadata;
                t->structs[p->metadata_struct_id] = metadata;
        }

        return 0;
}

static void
metadata_build_free(struct rte_swx_pipeline *p)
{
        uint32_t i;

        for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
                struct thread *t = &p->threads[i];

                free(t->metadata);
                t->metadata = NULL;
        }
}

static void
metadata_free(struct rte_swx_pipeline *p)
{
        metadata_build_free(p);
}

/*
 * Instruction.
 */
static int
instruction_is_jmp(struct instruction *instr)
{
        switch (instr->type) {
        case INSTR_JMP:
        case INSTR_JMP_VALID:
        case INSTR_JMP_INVALID:
        case INSTR_JMP_HIT:
        case INSTR_JMP_MISS:
        case INSTR_JMP_ACTION_HIT:
        case INSTR_JMP_ACTION_MISS:
        case INSTR_JMP_EQ:
        case INSTR_JMP_EQ_S:
        case INSTR_JMP_EQ_I:
        case INSTR_JMP_NEQ:
        case INSTR_JMP_NEQ_S:
        case INSTR_JMP_NEQ_I:
        case INSTR_JMP_LT:
        case INSTR_JMP_LT_MH:
        case INSTR_JMP_LT_HM:
        case INSTR_JMP_LT_HH:
        case INSTR_JMP_LT_MI:
        case INSTR_JMP_LT_HI:
        case INSTR_JMP_GT:
        case INSTR_JMP_GT_MH:
        case INSTR_JMP_GT_HM:
        case INSTR_JMP_GT_HH:
        case INSTR_JMP_GT_MI:
        case INSTR_JMP_GT_HI:
                return 1;

        default:
                return 0;
        }
}

static struct field *
action_field_parse(struct action *action, const char *name);

static struct field *
struct_field_parse(struct rte_swx_pipeline *p,
                   struct action *action,
                   const char *name,
                   uint32_t *struct_id)
{
        struct field *f;

        switch (name[0]) {
        case 'h':
        {
                struct header *header;

                f = header_field_parse(p, name, &header);
                if (!f)
                        return NULL;

                *struct_id = header->struct_id;
                return f;
        }

        case 'm':
        {
                f = metadata_field_parse(p, name);
                if (!f)
                        return NULL;

                *struct_id = p->metadata_struct_id;
                return f;
        }

        case 't':
        {
                if (!action)
                        return NULL;

                f = action_field_parse(action, name);
                if (!f)
                        return NULL;

                *struct_id = 0;
                return f;
        }

        case 'e':
        {
                struct extern_obj *obj;

                f = extern_obj_mailbox_field_parse(p, name, &obj);
                if (!f)
                        return NULL;

                *struct_id = obj->struct_id;
                return f;
        }

        case 'f':
        {
                struct extern_func *func;

                f = extern_func_mailbox_field_parse(p, name, &func);
                if (!f)
                        return NULL;

                *struct_id = func->struct_id;
                return f;
        }

        default:
                return NULL;
        }
}

static inline void
pipeline_port_inc(struct rte_swx_pipeline *p)
{
        p->port_id = (p->port_id + 1) & (p->n_ports_in - 1);
}

static inline void
thread_ip_reset(struct rte_swx_pipeline *p, struct thread *t)
{
        t->ip = p->instructions;
}

static inline void
thread_ip_set(struct thread *t, struct instruction *ip)
{
        t->ip = ip;
}

static inline void
thread_ip_action_call(struct rte_swx_pipeline *p,
                      struct thread *t,
                      uint32_t action_id)
{
        t->ret = t->ip + 1;
        t->ip = p->action_instructions[action_id];
}

static inline void
thread_ip_inc(struct rte_swx_pipeline *p);

static inline void
thread_ip_inc(struct rte_swx_pipeline *p)
{
        struct thread *t = &p->threads[p->thread_id];

        t->ip++;
}

static inline void
thread_ip_inc_cond(struct thread *t, int cond)
{
        t->ip += cond;
}

static inline void
thread_yield(struct rte_swx_pipeline *p)
{
        p->thread_id = (p->thread_id + 1) & (RTE_SWX_PIPELINE_THREADS_MAX - 1);
}

static inline void
thread_yield_cond(struct rte_swx_pipeline *p, int cond)
{
        p->thread_id = (p->thread_id + cond) & (RTE_SWX_PIPELINE_THREADS_MAX - 1);
}

/*
 * rx.
 */
static int
instr_rx_translate(struct rte_swx_pipeline *p,
                   struct action *action,
                   char **tokens,
                   int n_tokens,
                   struct instruction *instr,
                   struct instruction_data *data __rte_unused)
{
        struct field *f;

        CHECK(!action, EINVAL);
        CHECK(n_tokens == 2, EINVAL);

        f = metadata_field_parse(p, tokens[1]);
        CHECK(f, EINVAL);

        instr->type = INSTR_RX;
        instr->io.io.offset = f->offset / 8;
        instr->io.io.n_bits = f->n_bits;
        return 0;
}

static inline void
instr_rx_exec(struct rte_swx_pipeline *p);

static inline void
instr_rx_exec(struct rte_swx_pipeline *p)
{
        struct thread *t = &p->threads[p->thread_id];
        struct instruction *ip = t->ip;
        struct port_in_runtime *port = &p->in[p->port_id];
        struct rte_swx_pkt *pkt = &t->pkt;
        int pkt_received;

        /* Packet. */
        pkt_received = port->pkt_rx(port->obj, pkt);
        t->ptr = &pkt->pkt[pkt->offset];
        rte_prefetch0(t->ptr);

        TRACE("[Thread %2u] rx %s from port %u\n",
              p->thread_id,
              pkt_received ? "1 pkt" : "0 pkts",
              p->port_id);

        /* Headers. */
        t->valid_headers = 0;
        t->n_headers_out = 0;

        /* Meta-data. */
        METADATA_WRITE(t, ip->io.io.offset, ip->io.io.n_bits, p->port_id);

        /* Tables. */
        t->table_state = p->table_state;

        /* Thread. */
        pipeline_port_inc(p);
        thread_ip_inc_cond(t, pkt_received);
        thread_yield(p);
}

/*
 * tx.
 */
static int
instr_tx_translate(struct rte_swx_pipeline *p,
                   struct action *action __rte_unused,
                   char **tokens,
                   int n_tokens,
                   struct instruction *instr,
                   struct instruction_data *data __rte_unused)
{
        struct field *f;

        CHECK(n_tokens == 2, EINVAL);

        f = metadata_field_parse(p, tokens[1]);
        CHECK(f, EINVAL);

        instr->type = INSTR_TX;
        instr->io.io.offset = f->offset / 8;
        instr->io.io.n_bits = f->n_bits;
        return 0;
}

static inline void
emit_handler(struct thread *t)
{
        struct header_out_runtime *h0 = &t->headers_out[0];
        struct header_out_runtime *h1 = &t->headers_out[1];
        uint32_t offset = 0, i;

        /* No header change or header decapsulation. */
        if ((t->n_headers_out == 1) &&
            (h0->ptr + h0->n_bytes == t->ptr)) {
                TRACE("Emit handler: no header change or header decap.\n");

                t->pkt.offset -= h0->n_bytes;
                t->pkt.length += h0->n_bytes;

                return;
        }

        /* Header encapsulation (optionally, with prior header decasulation). */
        if ((t->n_headers_out == 2) &&
            (h1->ptr + h1->n_bytes == t->ptr) &&
            (h0->ptr == h0->ptr0)) {
                uint32_t offset;

                TRACE("Emit handler: header encapsulation.\n");

                offset = h0->n_bytes + h1->n_bytes;
                memcpy(t->ptr - offset, h0->ptr, h0->n_bytes);
                t->pkt.offset -= offset;
                t->pkt.length += offset;

                return;
        }

        /* Header insertion. */
        /* TBD */

        /* Header extraction. */
        /* TBD */

        /* For any other case. */
        TRACE("Emit handler: complex case.\n");

        for (i = 0; i < t->n_headers_out; i++) {
                struct header_out_runtime *h = &t->headers_out[i];

                memcpy(&t->header_out_storage[offset], h->ptr, h->n_bytes);
                offset += h->n_bytes;
        }

        if (offset) {
                memcpy(t->ptr - offset, t->header_out_storage, offset);
                t->pkt.offset -= offset;
                t->pkt.length += offset;
        }
}

static inline void
instr_tx_exec(struct rte_swx_pipeline *p);

static inline void
instr_tx_exec(struct rte_swx_pipeline *p)
{
        struct thread *t = &p->threads[p->thread_id];
        struct instruction *ip = t->ip;
        uint64_t port_id = METADATA_READ(t, ip->io.io.offset, ip->io.io.n_bits);
        struct port_out_runtime *port = &p->out[port_id];
        struct rte_swx_pkt *pkt = &t->pkt;

        TRACE("[Thread %2u]: tx 1 pkt to port %u\n",
              p->thread_id,
              (uint32_t)port_id);

        /* Headers. */
        emit_handler(t);

        /* Packet. */
        port->pkt_tx(port->obj, pkt);

        /* Thread. */
        thread_ip_reset(p, t);
        instr_rx_exec(p);
}

/*
 * extract.
 */
static int
instr_hdr_extract_translate(struct rte_swx_pipeline *p,
                            struct action *action,
                            char **tokens,
                            int n_tokens,
                            struct instruction *instr,
                            struct instruction_data *data __rte_unused)
{
        struct header *h;

        CHECK(!action, EINVAL);
        CHECK(n_tokens == 2, EINVAL);

        h = header_parse(p, tokens[1]);
        CHECK(h, EINVAL);

        instr->type = INSTR_HDR_EXTRACT;
        instr->io.hdr.header_id[0] = h->id;
        instr->io.hdr.struct_id[0] = h->struct_id;
        instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
        return 0;
}

static inline void
__instr_hdr_extract_exec(struct rte_swx_pipeline *p, uint32_t n_extract);

static inline void
__instr_hdr_extract_exec(struct rte_swx_pipeline *p, uint32_t n_extract)
{
        struct thread *t = &p->threads[p->thread_id];
        struct instruction *ip = t->ip;
        uint64_t valid_headers = t->valid_headers;
        uint8_t *ptr = t->ptr;
        uint32_t offset = t->pkt.offset;
        uint32_t length = t->pkt.length;
        uint32_t i;

        for (i = 0; i < n_extract; i++) {
                uint32_t header_id = ip->io.hdr.header_id[i];
                uint32_t struct_id = ip->io.hdr.struct_id[i];
                uint32_t n_bytes = ip->io.hdr.n_bytes[i];

                TRACE("[Thread %2u]: extract header %u (%u bytes)\n",
                      p->thread_id,
                      header_id,
                      n_bytes);

                /* Headers. */
                t->structs[struct_id] = ptr;
                valid_headers = MASK64_BIT_SET(valid_headers, header_id);

                /* Packet. */
                offset += n_bytes;
                length -= n_bytes;
                ptr += n_bytes;
        }

        /* Headers. */
        t->valid_headers = valid_headers;

        /* Packet. */
        t->pkt.offset = offset;
        t->pkt.length = length;
        t->ptr = ptr;
}

static inline void
instr_hdr_extract_exec(struct rte_swx_pipeline *p)
{
        __instr_hdr_extract_exec(p, 1);

        /* Thread. */
        thread_ip_inc(p);
}

static inline void
instr_hdr_extract2_exec(struct rte_swx_pipeline *p)
{
        TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
              p->thread_id);

        __instr_hdr_extract_exec(p, 2);

        /* Thread. */
        thread_ip_inc(p);
}

static inline void
instr_hdr_extract3_exec(struct rte_swx_pipeline *p)
{
        TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
              p->thread_id);

        __instr_hdr_extract_exec(p, 3);

        /* Thread. */
        thread_ip_inc(p);
}

static inline void
instr_hdr_extract4_exec(struct rte_swx_pipeline *p)
{
        TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
              p->thread_id);

        __instr_hdr_extract_exec(p, 4);

        /* Thread. */
        thread_ip_inc(p);
}

static inline void
instr_hdr_extract5_exec(struct rte_swx_pipeline *p)
{
        TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
              p->thread_id);

        __instr_hdr_extract_exec(p, 5);

        /* Thread. */
        thread_ip_inc(p);
}

static inline void
instr_hdr_extract6_exec(struct rte_swx_pipeline *p)
{
        TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
              p->thread_id);

        __instr_hdr_extract_exec(p, 6);

        /* Thread. */
        thread_ip_inc(p);
}

static inline void
instr_hdr_extract7_exec(struct rte_swx_pipeline *p)
{
        TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
              p->thread_id);

        __instr_hdr_extract_exec(p, 7);

        /* Thread. */
        thread_ip_inc(p);
}

static inline void
instr_hdr_extract8_exec(struct rte_swx_pipeline *p)
{
        TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
              p->thread_id);

        __instr_hdr_extract_exec(p, 8);

        /* Thread. */
        thread_ip_inc(p);
}

/*
 * emit.
 */
static int
instr_hdr_emit_translate(struct rte_swx_pipeline *p,
                         struct action *action __rte_unused,
                         char **tokens,
                         int n_tokens,
                         struct instruction *instr,
                         struct instruction_data *data __rte_unused)
{
        struct header *h;

        CHECK(n_tokens == 2, EINVAL);

        h = header_parse(p, tokens[1]);
        CHECK(h, EINVAL);

        instr->type = INSTR_HDR_EMIT;
        instr->io.hdr.header_id[0] = h->id;
        instr->io.hdr.struct_id[0] = h->struct_id;
        instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
        return 0;
}

static inline void
__instr_hdr_emit_exec(struct rte_swx_pipeline *p, uint32_t n_emit);

static inline void
__instr_hdr_emit_exec(struct rte_swx_pipeline *p, uint32_t n_emit)
{
        struct thread *t = &p->threads[p->thread_id];
        struct instruction *ip = t->ip;
        uint32_t n_headers_out = t->n_headers_out;
        struct header_out_runtime *ho = &t->headers_out[n_headers_out - 1];
        uint8_t *ho_ptr = NULL;
        uint32_t ho_nbytes = 0, i;

        for (i = 0; i < n_emit; i++) {
                uint32_t header_id = ip->io.hdr.header_id[i];
                uint32_t struct_id = ip->io.hdr.struct_id[i];
                uint32_t n_bytes = ip->io.hdr.n_bytes[i];

                struct header_runtime *hi = &t->headers[header_id];
                uint8_t *hi_ptr = t->structs[struct_id];

                TRACE("[Thread %2u]: emit header %u\n",
                      p->thread_id,
                      header_id);

                /* Headers. */
                if (!i) {
                        if (!t->n_headers_out) {
                                ho = &t->headers_out[0];

                                ho->ptr0 = hi->ptr0;
                                ho->ptr = hi_ptr;

                                ho_ptr = hi_ptr;
                                ho_nbytes = n_bytes;

                                n_headers_out = 1;

                                continue;
                        } else {
                                ho_ptr = ho->ptr;
                                ho_nbytes = ho->n_bytes;
                        }
                }

                if (ho_ptr + ho_nbytes == hi_ptr) {
                        ho_nbytes += n_bytes;
                } else {
                        ho->n_bytes = ho_nbytes;

                        ho++;
                        ho->ptr0 = hi->ptr0;
                        ho->ptr = hi_ptr;

                        ho_ptr = hi_ptr;
                        ho_nbytes = n_bytes;

                        n_headers_out++;
                }
        }

        ho->n_bytes = ho_nbytes;
        t->n_headers_out = n_headers_out;
}

static inline void
instr_hdr_emit_exec(struct rte_swx_pipeline *p)
{
        __instr_hdr_emit_exec(p, 1);

        /* Thread. */
        thread_ip_inc(p);
}

static inline void
instr_hdr_emit_tx_exec(struct rte_swx_pipeline *p)
{
        TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
              p->thread_id);

        __instr_hdr_emit_exec(p, 1);
        instr_tx_exec(p);
}

static inline void
instr_hdr_emit2_tx_exec(struct rte_swx_pipeline *p)
{
        TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
              p->thread_id);

        __instr_hdr_emit_exec(p, 2);
        instr_tx_exec(p);
}

static inline void
instr_hdr_emit3_tx_exec(struct rte_swx_pipeline *p)
{
        TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
              p->thread_id);

        __instr_hdr_emit_exec(p, 3);
        instr_tx_exec(p);
}

static inline void
instr_hdr_emit4_tx_exec(struct rte_swx_pipeline *p)
{
        TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
              p->thread_id);

        __instr_hdr_emit_exec(p, 4);
        instr_tx_exec(p);
}

static inline void
instr_hdr_emit5_tx_exec(struct rte_swx_pipeline *p)
{
        TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
              p->thread_id);

        __instr_hdr_emit_exec(p, 5);
        instr_tx_exec(p);
}

static inline void
instr_hdr_emit6_tx_exec(struct rte_swx_pipeline *p)
{
        TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
              p->thread_id);

        __instr_hdr_emit_exec(p, 6);
        instr_tx_exec(p);
}

static inline void
instr_hdr_emit7_tx_exec(struct rte_swx_pipeline *p)
{
        TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
              p->thread_id);

        __instr_hdr_emit_exec(p, 7);
        instr_tx_exec(p);
}

static inline void
instr_hdr_emit8_tx_exec(struct rte_swx_pipeline *p)
{
        TRACE("[Thread %2u] *** The next 9 instructions are fused. ***\n",
              p->thread_id);

        __instr_hdr_emit_exec(p, 8);
        instr_tx_exec(p);
}

/*
 * validate.
 */
static int
instr_hdr_validate_translate(struct rte_swx_pipeline *p,
                             struct action *action __rte_unused,
                             char **tokens,
                             int n_tokens,
                             struct instruction *instr,
                             struct instruction_data *data __rte_unused)
{
        struct header *h;

        CHECK(n_tokens == 2, EINVAL);

        h = header_parse(p, tokens[1]);
        CHECK(h, EINVAL);

        instr->type = INSTR_HDR_VALIDATE;
        instr->valid.header_id = h->id;
        return 0;
}

static inline void
instr_hdr_validate_exec(struct rte_swx_pipeline *p)
{
        struct thread *t = &p->threads[p->thread_id];
        struct instruction *ip = t->ip;
        uint32_t header_id = ip->valid.header_id;

        TRACE("[Thread %2u] validate header %u\n", p->thread_id, header_id);

        /* Headers. */
        t->valid_headers = MASK64_BIT_SET(t->valid_headers, header_id);

        /* Thread. */
        thread_ip_inc(p);
}

/*
 * invalidate.
 */
static int
instr_hdr_invalidate_translate(struct rte_swx_pipeline *p,
                               struct action *action __rte_unused,
                               char **tokens,
                               int n_tokens,
                               struct instruction *instr,
                               struct instruction_data *data __rte_unused)
{