/* SPDX-License-Identifier: BSD-3-Clause
 * Copyright(c) 2020 Intel Corporation
 */
#include <stdlib.h>
#include <stdio.h>
#include <errno.h>
#include <dlfcn.h>

#include <rte_jhash.h>
#include <rte_hash_crc.h>

#include <rte_swx_port_ethdev.h>
#include <rte_swx_port_fd.h>
#include <rte_swx_port_ring.h>
#include "rte_swx_port_source_sink.h"

#include <rte_swx_table_em.h>
#include <rte_swx_table_wm.h>

#include "rte_swx_pipeline_internal.h"

#define CHECK(condition, err_code)                                             \
do {                                                                           \
        if (!(condition))                                                      \
                return -(err_code);                                            \
} while (0)

#define CHECK_NAME(name, err_code)                                             \
        CHECK((name) &&                                                        \
              (name)[0] &&                                                     \
              (strnlen((name), RTE_SWX_NAME_SIZE) < RTE_SWX_NAME_SIZE),        \
              err_code)

#define CHECK_INSTRUCTION(instr, err_code)                                     \
        CHECK((instr) &&                                                       \
              (instr)[0] &&                                                    \
              (strnlen((instr), RTE_SWX_INSTRUCTION_SIZE) <                    \
               RTE_SWX_INSTRUCTION_SIZE),                                      \
              err_code)

/*
 * Environment.
 */
#ifndef RTE_SWX_PIPELINE_HUGE_PAGES_DISABLE

#include <rte_malloc.h>

static void *
env_malloc(size_t size, size_t alignment, int numa_node)
{
        return rte_zmalloc_socket(NULL, size, alignment, numa_node);
}

static void
env_free(void *start, size_t size __rte_unused)
{
        rte_free(start);
}

#else

#include <numa.h>

static void *
env_malloc(size_t size, size_t alignment __rte_unused, int numa_node)
{
        void *start;

        if (numa_available() == -1)
                return NULL;

        start = numa_alloc_onnode(size, numa_node);
        if (!start)
                return NULL;

        memset(start, 0, size);
        return start;
}

static void
env_free(void *start, size_t size)
{
        if (numa_available() == -1)
                return;

        numa_free(start, size);
}

#endif

/*
 * Struct.
 */
static struct struct_type *
struct_type_find(struct rte_swx_pipeline *p, const char *name)
{
        struct struct_type *elem;

        TAILQ_FOREACH(elem, &p->struct_types, node)
                if (strcmp(elem->name, name) == 0)
                        return elem;

        return NULL;
}

static struct field *
struct_type_field_find(struct struct_type *st, const char *name)
{
        uint32_t i;

        for (i = 0; i < st->n_fields; i++) {
                struct field *f = &st->fields[i];

                if (strcmp(f->name, name) == 0)
                        return f;
        }

        return NULL;
}

int
rte_swx_pipeline_struct_type_register(struct rte_swx_pipeline *p,
                                      const char *name,
                                      struct rte_swx_field_params *fields,
                                      uint32_t n_fields,
                                      int last_field_has_variable_size)
{
        struct struct_type *st;
        uint32_t i;

        CHECK(p, EINVAL);
        CHECK_NAME(name, EINVAL);
        CHECK(fields, EINVAL);
        CHECK(n_fields, EINVAL);

        for (i = 0; i < n_fields; i++) {
                struct rte_swx_field_params *f = &fields[i];
                int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
                uint32_t j;

                CHECK_NAME(f->name, EINVAL);
                CHECK(f->n_bits, EINVAL);
                CHECK((f->n_bits <= 64) || var_size, EINVAL);
                CHECK((f->n_bits & 7) == 0, EINVAL);

                for (j = 0; j < i; j++) {
                        struct rte_swx_field_params *f_prev = &fields[j];

                        CHECK(strcmp(f->name, f_prev->name), EINVAL);
                }
        }

        CHECK(!struct_type_find(p, name), EEXIST);

        /* Node allocation. */
        st = calloc(1, sizeof(struct struct_type));
        CHECK(st, ENOMEM);

        st->fields = calloc(n_fields, sizeof(struct field));
        if (!st->fields) {
                free(st);
                CHECK(0, ENOMEM);
        }

        /* Node initialization. */
        strcpy(st->name, name);
        for (i = 0; i < n_fields; i++) {
                struct field *dst = &st->fields[i];
                struct rte_swx_field_params *src = &fields[i];
                int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;

                strcpy(dst->name, src->name);
                dst->n_bits = src->n_bits;
                dst->offset = st->n_bits;
                dst->var_size = var_size;

                st->n_bits += src->n_bits;
                st->n_bits_min += var_size ? 0 : src->n_bits;
        }
        st->n_fields = n_fields;
        st->var_size = last_field_has_variable_size;

        /* Node add to tailq. */
        TAILQ_INSERT_TAIL(&p->struct_types, st, node);

        return 0;
}

static int
struct_build(struct rte_swx_pipeline *p)
{
        uint32_t i;

        for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
                struct thread *t = &p->threads[i];

                t->structs = calloc(p->n_structs, sizeof(uint8_t *));
                CHECK(t->structs, ENOMEM);
        }

        return 0;
}

static void
struct_build_free(struct rte_swx_pipeline *p)
{
        uint32_t i;

        for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
                struct thread *t = &p->threads[i];

                free(t->structs);
                t->structs = NULL;
        }
}

static void
struct_free(struct rte_swx_pipeline *p)
{
        struct_build_free(p);

        /* Struct types. */
        for ( ; ; ) {
                struct struct_type *elem;

                elem = TAILQ_FIRST(&p->struct_types);
                if (!elem)
                        break;

                TAILQ_REMOVE(&p->struct_types, elem, node);
                free(elem->fields);
                free(elem);
        }
}

/*
 * Input port.
 */
static struct port_in_type *
port_in_type_find(struct rte_swx_pipeline *p, const char *name)
{
        struct port_in_type *elem;

        if (!name)
                return NULL;

        TAILQ_FOREACH(elem, &p->port_in_types, node)
                if (strcmp(elem->name, name) == 0)
                        return elem;

        return NULL;
}

int
rte_swx_pipeline_port_in_type_register(struct rte_swx_pipeline *p,
                                       const char *name,
                                       struct rte_swx_port_in_ops *ops)
{
        struct port_in_type *elem;

        CHECK(p, EINVAL);
        CHECK_NAME(name, EINVAL);
        CHECK(ops, EINVAL);
        CHECK(ops->create, EINVAL);
        CHECK(ops->free, EINVAL);
        CHECK(ops->pkt_rx, EINVAL);
        CHECK(ops->stats_read, EINVAL);

        CHECK(!port_in_type_find(p, name), EEXIST);

        /* Node allocation. */
        elem = calloc(1, sizeof(struct port_in_type));
        CHECK(elem, ENOMEM);

        /* Node initialization. */
        strcpy(elem->name, name);
        memcpy(&elem->ops, ops, sizeof(*ops));

        /* Node add to tailq. */
        TAILQ_INSERT_TAIL(&p->port_in_types, elem, node);

        return 0;
}

static struct port_in *
port_in_find(struct rte_swx_pipeline *p, uint32_t port_id)
{
        struct port_in *port;

        TAILQ_FOREACH(port, &p->ports_in, node)
                if (port->id == port_id)
                        return port;

        return NULL;
}

int
rte_swx_pipeline_port_in_config(struct rte_swx_pipeline *p,
                                uint32_t port_id,
                                const char *port_type_name,
                                void *args)
{
        struct port_in_type *type = NULL;
        struct port_in *port = NULL;
        void *obj = NULL;

        CHECK(p, EINVAL);

        CHECK(!port_in_find(p, port_id), EINVAL);

        CHECK_NAME(port_type_name, EINVAL);
        type = port_in_type_find(p, port_type_name);
        CHECK(type, EINVAL);

        obj = type->ops.create(args);
        CHECK(obj, ENODEV);

        /* Node allocation. */
        port = calloc(1, sizeof(struct port_in));
        CHECK(port, ENOMEM);

        /* Node initialization. */
        port->type = type;
        port->obj = obj;
        port->id = port_id;

        /* Node add to tailq. */
        TAILQ_INSERT_TAIL(&p->ports_in, port, node);
        if (p->n_ports_in < port_id + 1)
                p->n_ports_in = port_id + 1;

        return 0;
}

static int
port_in_build(struct rte_swx_pipeline *p)
{
        struct port_in *port;
        uint32_t i;

        CHECK(p->n_ports_in, EINVAL);
        CHECK(rte_is_power_of_2(p->n_ports_in), EINVAL);

        for (i = 0; i < p->n_ports_in; i++)
                CHECK(port_in_find(p, i), EINVAL);

        p->in = calloc(p->n_ports_in, sizeof(struct port_in_runtime));
        CHECK(p->in, ENOMEM);

        TAILQ_FOREACH(port, &p->ports_in, node) {
                struct port_in_runtime *in = &p->in[port->id];

                in->pkt_rx = port->type->ops.pkt_rx;
                in->obj = port->obj;
        }

        return 0;
}

static void
port_in_build_free(struct rte_swx_pipeline *p)
{
        free(p->in);
        p->in = NULL;
}

static void
port_in_free(struct rte_swx_pipeline *p)
{
        port_in_build_free(p);

        /* Input ports. */
        for ( ; ; ) {
                struct port_in *port;

                port = TAILQ_FIRST(&p->ports_in);
                if (!port)
                        break;

                TAILQ_REMOVE(&p->ports_in, port, node);
                port->type->ops.free(port->obj);
                free(port);
        }

        /* Input port types. */
        for ( ; ; ) {
                struct port_in_type *elem;

                elem = TAILQ_FIRST(&p->port_in_types);
                if (!elem)
                        break;

                TAILQ_REMOVE(&p->port_in_types, elem, node);
                free(elem);
        }
}

/*
 * Output port.
 */
static struct port_out_type *
port_out_type_find(struct rte_swx_pipeline *p, const char *name)
{
        struct port_out_type *elem;

        if (!name)
                return NULL;

        TAILQ_FOREACH(elem, &p->port_out_types, node)
                if (!strcmp(elem->name, name))
                        return elem;

        return NULL;
}

int
rte_swx_pipeline_port_out_type_register(struct rte_swx_pipeline *p,
                                        const char *name,
                                        struct rte_swx_port_out_ops *ops)
{
        struct port_out_type *elem;

        CHECK(p, EINVAL);
        CHECK_NAME(name, EINVAL);
        CHECK(ops, EINVAL);
        CHECK(ops->create, EINVAL);
        CHECK(ops->free, EINVAL);
        CHECK(ops->pkt_tx, EINVAL);
        CHECK(ops->pkt_fast_clone_tx, EINVAL);
        CHECK(ops->pkt_clone_tx, EINVAL);
        CHECK(ops->stats_read, EINVAL);

        CHECK(!port_out_type_find(p, name), EEXIST);

        /* Node allocation. */
        elem = calloc(1, sizeof(struct port_out_type));
        CHECK(elem, ENOMEM);

        /* Node initialization. */
        strcpy(elem->name, name);
        memcpy(&elem->ops, ops, sizeof(*ops));

        /* Node add to tailq. */
        TAILQ_INSERT_TAIL(&p->port_out_types, elem, node);

        return 0;
}

static struct port_out *
port_out_find(struct rte_swx_pipeline *p, uint32_t port_id)
{
        struct port_out *port;

        TAILQ_FOREACH(port, &p->ports_out, node)
                if (port->id == port_id)
                        return port;

        return NULL;
}

int
rte_swx_pipeline_port_out_config(struct rte_swx_pipeline *p,
                                 uint32_t port_id,
                                 const char *port_type_name,
                                 void *args)
{
        struct port_out_type *type = NULL;
        struct port_out *port = NULL;
        void *obj = NULL;

        CHECK(p, EINVAL);

        CHECK(!port_out_find(p, port_id), EINVAL);

        CHECK_NAME(port_type_name, EINVAL);
        type = port_out_type_find(p, port_type_name);
        CHECK(type, EINVAL);

        obj = type->ops.create(args);
        CHECK(obj, ENODEV);

        /* Node allocation. */
        port = calloc(1, sizeof(struct port_out));
        CHECK(port, ENOMEM);

        /* Node initialization. */
        port->type = type;
        port->obj = obj;
        port->id = port_id;

        /* Node add to tailq. */
        TAILQ_INSERT_TAIL(&p->ports_out, port, node);
        if (p->n_ports_out < port_id + 1)
                p->n_ports_out = port_id + 1;

        return 0;
}

static int
port_out_build(struct rte_swx_pipeline *p)
{
        struct port_out *port;
        uint32_t i;

        CHECK(p->n_ports_out, EINVAL);

        for (i = 0; i < p->n_ports_out; i++)
                CHECK(port_out_find(p, i), EINVAL);

        p->out = calloc(p->n_ports_out, sizeof(struct port_out_runtime));
        CHECK(p->out, ENOMEM);

        TAILQ_FOREACH(port, &p->ports_out, node) {
                struct port_out_runtime *out = &p->out[port->id];

                out->pkt_tx = port->type->ops.pkt_tx;
                out->pkt_fast_clone_tx = port->type->ops.pkt_fast_clone_tx;
                out->pkt_clone_tx = port->type->ops.pkt_clone_tx;
                out->flush = port->type->ops.flush;
                out->obj = port->obj;
        }

        return 0;
}

static void
port_out_build_free(struct rte_swx_pipeline *p)
{
        free(p->out);
        p->out = NULL;
}

static void
port_out_free(struct rte_swx_pipeline *p)
{
        port_out_build_free(p);

        /* Output ports. */
        for ( ; ; ) {
                struct port_out *port;

                port = TAILQ_FIRST(&p->ports_out);
                if (!port)
                        break;

                TAILQ_REMOVE(&p->ports_out, port, node);
                port->type->ops.free(port->obj);
                free(port);
        }

        /* Output port types. */
        for ( ; ; ) {
                struct port_out_type *elem;

                elem = TAILQ_FIRST(&p->port_out_types);
                if (!elem)
                        break;

                TAILQ_REMOVE(&p->port_out_types, elem, node);
                free(elem);
        }
}

/*
 * Packet mirroring.
 */
int
rte_swx_pipeline_mirroring_config(struct rte_swx_pipeline *p,
                                  struct rte_swx_pipeline_mirroring_params *params)
{
        CHECK(p, EINVAL);
        CHECK(params, EINVAL);
        CHECK(params->n_slots, EINVAL);
        CHECK(params->n_sessions, EINVAL);
        CHECK(!p->build_done, EEXIST);

        p->n_mirroring_slots = rte_align32pow2(params->n_slots);
        if (p->n_mirroring_slots > 64)
                p->n_mirroring_slots = 64;

        p->n_mirroring_sessions = rte_align32pow2(params->n_sessions);

        return 0;
}

static void
mirroring_build_free(struct rte_swx_pipeline *p)
{
        uint32_t i;

        for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
                struct thread *t = &p->threads[i];

                /* mirroring_slots. */
                free(t->mirroring_slots);
                t->mirroring_slots = NULL;
        }

        /* mirroring_sessions. */
        free(p->mirroring_sessions);
        p->mirroring_sessions = NULL;
}

static void
mirroring_free(struct rte_swx_pipeline *p)
{
        mirroring_build_free(p);
}

static int
mirroring_build(struct rte_swx_pipeline *p)
{
        uint32_t i;

        for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
                struct thread *t = &p->threads[i];

                /* mirroring_slots. */
                t->mirroring_slots = calloc(p->n_mirroring_slots, sizeof(uint32_t));
                if (!t->mirroring_slots)
                        goto error;
        }

        /* mirroring_sessions. */
        p->mirroring_sessions = calloc(p->n_mirroring_sessions, sizeof(struct mirroring_session));
        if (!p->mirroring_sessions)
                goto error;

        return 0;

error:
        mirroring_build_free(p);
        return -ENOMEM;
}

/*
 * Extern object.
 */
static struct extern_type *
extern_type_find(struct rte_swx_pipeline *p, const char *name)
{
        struct extern_type *elem;

        TAILQ_FOREACH(elem, &p->extern_types, node)
                if (strcmp(elem->name, name) == 0)
                        return elem;

        return NULL;
}

static struct extern_type_member_func *
extern_type_member_func_find(struct extern_type *type, const char *name)
{
        struct extern_type_member_func *elem;

        TAILQ_FOREACH(elem, &type->funcs, node)
                if (strcmp(elem->name, name) == 0)
                        return elem;

        return NULL;
}

static struct extern_obj *
extern_obj_find(struct rte_swx_pipeline *p, const char *name)
{
        struct extern_obj *elem;

        TAILQ_FOREACH(elem, &p->extern_objs, node)
                if (strcmp(elem->name, name) == 0)
                        return elem;

        return NULL;
}

static struct extern_type_member_func *
extern_obj_member_func_parse(struct rte_swx_pipeline *p,
                             const char *name,
                             struct extern_obj **obj)
{
        struct extern_obj *object;
        struct extern_type_member_func *func;
        char *object_name, *func_name;

        if (name[0] != 'e' || name[1] != '.')
                return NULL;

        object_name = strdup(&name[2]);
        if (!object_name)
                return NULL;

        func_name = strchr(object_name, '.');
        if (!func_name) {
                free(object_name);
                return NULL;
        }

        *func_name = 0;
        func_name++;

        object = extern_obj_find(p, object_name);
        if (!object) {
                free(object_name);
                return NULL;
        }

        func = extern_type_member_func_find(object->type, func_name);
        if (!func) {
                free(object_name);
                return NULL;
        }

        if (obj)
                *obj = object;

        free(object_name);
        return func;
}

static struct field *
extern_obj_mailbox_field_parse(struct rte_swx_pipeline *p,
                               const char *name,
                               struct extern_obj **object)
{
        struct extern_obj *obj;
        struct field *f;
        char *obj_name, *field_name;

        if ((name[0] != 'e') || (name[1] != '.'))
                return NULL;

        obj_name = strdup(&name[2]);
        if (!obj_name)
                return NULL;

        field_name = strchr(obj_name, '.');
        if (!field_name) {
                free(obj_name);
                return NULL;
        }

        *field_name = 0;
        field_name++;

        obj = extern_obj_find(p, obj_name);
        if (!obj) {
                free(obj_name);
                return NULL;
        }

        f = struct_type_field_find(obj->type->mailbox_struct_type, field_name);
        if (!f) {
                free(obj_name);
                return NULL;
        }

        if (object)
                *object = obj;

        free(obj_name);
        return f;
}

int
rte_swx_pipeline_extern_type_register(struct rte_swx_pipeline *p,
        const char *name,
        const char *mailbox_struct_type_name,
        rte_swx_extern_type_constructor_t constructor,
        rte_swx_extern_type_destructor_t destructor)
{
        struct extern_type *elem;
        struct struct_type *mailbox_struct_type;

        CHECK(p, EINVAL);

        CHECK_NAME(name, EINVAL);
        CHECK(!extern_type_find(p, name), EEXIST);

        CHECK_NAME(mailbox_struct_type_name, EINVAL);
        mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
        CHECK(mailbox_struct_type, EINVAL);
        CHECK(!mailbox_struct_type->var_size, EINVAL);

        CHECK(constructor, EINVAL);
        CHECK(destructor, EINVAL);

        /* Node allocation. */
        elem = calloc(1, sizeof(struct extern_type));
        CHECK(elem, ENOMEM);

        /* Node initialization. */
        strcpy(elem->name, name);
        elem->mailbox_struct_type = mailbox_struct_type;
        elem->constructor = constructor;
        elem->destructor = destructor;
        TAILQ_INIT(&elem->funcs);

        /* Node add to tailq. */
        TAILQ_INSERT_TAIL(&p->extern_types, elem, node);

        return 0;
}

int
rte_swx_pipeline_extern_type_member_func_register(struct rte_swx_pipeline *p,
        const char *extern_type_name,
        const char *name,
        rte_swx_extern_type_member_func_t member_func)
{
        struct extern_type *type;
        struct extern_type_member_func *type_member;

        CHECK(p, EINVAL);

        CHECK_NAME(extern_type_name, EINVAL);
        type = extern_type_find(p, extern_type_name);
        CHECK(type, EINVAL);
        CHECK(type->n_funcs < RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX, ENOSPC);

        CHECK_NAME(name, EINVAL);
        CHECK(!extern_type_member_func_find(type, name), EEXIST);

        CHECK(member_func, EINVAL);

        /* Node allocation. */
        type_member = calloc(1, sizeof(struct extern_type_member_func));
        CHECK(type_member, ENOMEM);

        /* Node initialization. */
        strcpy(type_member->name, name);
        type_member->func = member_func;
        type_member->id = type->n_funcs;

        /* Node add to tailq. */
        TAILQ_INSERT_TAIL(&type->funcs, type_member, node);
        type->n_funcs++;

        return 0;
}

int
rte_swx_pipeline_extern_object_config(struct rte_swx_pipeline *p,
                                      const char *extern_type_name,
                                      const char *name,
                                      const char *args)
{
        struct extern_type *type;
        struct extern_obj *obj;
        void *obj_handle;

        CHECK(p, EINVAL);

        CHECK_NAME(extern_type_name, EINVAL);
        type = extern_type_find(p, extern_type_name);
        CHECK(type, EINVAL);

        CHECK_NAME(name, EINVAL);
        CHECK(!extern_obj_find(p, name), EEXIST);

        /* Node allocation. */
        obj = calloc(1, sizeof(struct extern_obj));
        CHECK(obj, ENOMEM);

        /* Object construction. */
        obj_handle = type->constructor(args);
        if (!obj_handle) {
                free(obj);
                CHECK(0, ENODEV);
        }

        /* Node initialization. */
        strcpy(obj->name, name);
        obj->type = type;
        obj->obj = obj_handle;
        obj->struct_id = p->n_structs;
        obj->id = p->n_extern_objs;

        /* Node add to tailq. */
        TAILQ_INSERT_TAIL(&p->extern_objs, obj, node);
        p->n_extern_objs++;
        p->n_structs++;

        return 0;
}

static int
extern_obj_build(struct rte_swx_pipeline *p)
{
        uint32_t i;

        for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
                struct thread *t = &p->threads[i];
                struct extern_obj *obj;

                t->extern_objs = calloc(p->n_extern_objs,
                                        sizeof(struct extern_obj_runtime));
                CHECK(t->extern_objs, ENOMEM);

                TAILQ_FOREACH(obj, &p->extern_objs, node) {
                        struct extern_obj_runtime *r =
                                &t->extern_objs[obj->id];
                        struct extern_type_member_func *func;
                        uint32_t mailbox_size =
                                obj->type->mailbox_struct_type->n_bits / 8;

                        r->obj = obj->obj;

                        r->mailbox = calloc(1, mailbox_size);
                        CHECK(r->mailbox, ENOMEM);

                        TAILQ_FOREACH(func, &obj->type->funcs, node)
                                r->funcs[func->id] = func->func;

                        t->structs[obj->struct_id] = r->mailbox;
                }
        }

        return 0;
}

static void
extern_obj_build_free(struct rte_swx_pipeline *p)
{
        uint32_t i;

        for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
                struct thread *t = &p->threads[i];
                uint32_t j;

                if (!t->extern_objs)
                        continue;

                for (j = 0; j < p->n_extern_objs; j++) {
                        struct extern_obj_runtime *r = &t->extern_objs[j];

                        free(r->mailbox);
                }

                free(t->extern_objs);
                t->extern_objs = NULL;
        }
}

static void
extern_obj_free(struct rte_swx_pipeline *p)
{
        extern_obj_build_free(p);

        /* Extern objects. */
        for ( ; ; ) {
                struct extern_obj *elem;

                elem = TAILQ_FIRST(&p->extern_objs);
                if (!elem)
                        break;

                TAILQ_REMOVE(&p->extern_objs, elem, node);
                if (elem->obj)
                        elem->type->destructor(elem->obj);
                free(elem);
        }

        /* Extern types. */
        for ( ; ; ) {
                struct extern_type *elem;

                elem = TAILQ_FIRST(&p->extern_types);
                if (!elem)
                        break;

                TAILQ_REMOVE(&p->extern_types, elem, node);

                for ( ; ; ) {
                        struct extern_type_member_func *func;

                        func = TAILQ_FIRST(&elem->funcs);
                        if (!func)
                                break;

                        TAILQ_REMOVE(&elem->funcs, func, node);
                        free(func);
                }

                free(elem);
        }
}

/*
 * Extern function.
 */
static struct extern_func *
extern_func_find(struct rte_swx_pipeline *p, const char *name)
{
        struct extern_func *elem;

        TAILQ_FOREACH(elem, &p->extern_funcs, node)
                if (strcmp(elem->name, name) == 0)
                        return elem;

        return NULL;
}

static struct extern_func *
extern_func_parse(struct rte_swx_pipeline *p,
                  const char *name)
{
        if (name[0] != 'f' || name[1] != '.')
                return NULL;

        return extern_func_find(p, &name[2]);
}

static struct field *
extern_func_mailbox_field_parse(struct rte_swx_pipeline *p,
                                const char *name,
                                struct extern_func **function)
{
        struct extern_func *func;
        struct field *f;
        char *func_name, *field_name;

        if ((name[0] != 'f') || (name[1] != '.'))
                return NULL;

        func_name = strdup(&name[2]);
        if (!func_name)
                return NULL;

        field_name = strchr(func_name, '.');
        if (!field_name) {
                free(func_name);
                return NULL;
        }

        *field_name = 0;
        field_name++;

        func = extern_func_find(p, func_name);
        if (!func) {
                free(func_name);
                return NULL;
        }

        f = struct_type_field_find(func->mailbox_struct_type, field_name);
        if (!f) {
                free(func_name);
                return NULL;
        }

        if (function)
                *function = func;

        free(func_name);
        return f;
}

int
rte_swx_pipeline_extern_func_register(struct rte_swx_pipeline *p,
                                      const char *name,
                                      const char *mailbox_struct_type_name,
                                      rte_swx_extern_func_t func)
{
        struct extern_func *f;
        struct struct_type *mailbox_struct_type;

        CHECK(p, EINVAL);

        CHECK_NAME(name, EINVAL);
        CHECK(!extern_func_find(p, name), EEXIST);

        CHECK_NAME(mailbox_struct_type_name, EINVAL);
        mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
        CHECK(mailbox_struct_type, EINVAL);
        CHECK(!mailbox_struct_type->var_size, EINVAL);

        CHECK(func, EINVAL);

        /* Node allocation. */
        f = calloc(1, sizeof(struct extern_func));
        CHECK(func, ENOMEM);

        /* Node initialization. */
        strcpy(f->name, name);
        f->mailbox_struct_type = mailbox_struct_type;
        f->func = func;
        f->struct_id = p->n_structs;
        f->id = p->n_extern_funcs;

        /* Node add to tailq. */
        TAILQ_INSERT_TAIL(&p->extern_funcs, f, node);
        p->n_extern_funcs++;
        p->n_structs++;

        return 0;
}

static int
extern_func_build(struct rte_swx_pipeline *p)
{
        uint32_t i;

        for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
                struct thread *t = &p->threads[i];
                struct extern_func *func;

                /* Memory allocation. */
                t->extern_funcs = calloc(p->n_extern_funcs,
                                         sizeof(struct extern_func_runtime));
                CHECK(t->extern_funcs, ENOMEM);

                /* Extern function. */
                TAILQ_FOREACH(func, &p->extern_funcs, node) {
                        struct extern_func_runtime *r =
                                &t->extern_funcs[func->id];
                        uint32_t mailbox_size =
                                func->mailbox_struct_type->n_bits / 8;

                        r->func = func->func;

                        r->mailbox = calloc(1, mailbox_size);
                        CHECK(r->mailbox, ENOMEM);

                        t->structs[func->struct_id] = r->mailbox;
                }
        }

        return 0;
}

static void
extern_func_build_free(struct rte_swx_pipeline *p)
{
        uint32_t i;

        for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
                struct thread *t = &p->threads[i];
                uint32_t j;

                if (!t->extern_funcs)
                        continue;

                for (j = 0; j < p->n_extern_funcs; j++) {
                        struct extern_func_runtime *r = &t->extern_funcs[j];

                        free(r->mailbox);
                }

                free(t->extern_funcs);
                t->extern_funcs = NULL;
        }
}

static void
extern_func_free(struct rte_swx_pipeline *p)
{
        extern_func_build_free(p);

        for ( ; ; ) {
                struct extern_func *elem;

                elem = TAILQ_FIRST(&p->extern_funcs);
                if (!elem)
                        break;

                TAILQ_REMOVE(&p->extern_funcs, elem, node);
                free(elem);
        }
}

/*
 * Hash function.
 */
static struct hash_func *
hash_func_find(struct rte_swx_pipeline *p, const char *name)
{
        struct hash_func *elem;

        TAILQ_FOREACH(elem, &p->hash_funcs, node)
                if (strcmp(elem->name, name) == 0)
                        return elem;

        return NULL;
}

int
rte_swx_pipeline_hash_func_register(struct rte_swx_pipeline *p,
                                    const char *name,
                                    rte_swx_hash_func_t func)
{
        struct hash_func *f;

        CHECK(p, EINVAL);

        CHECK_NAME(name, EINVAL);
        CHECK(!hash_func_find(p, name), EEXIST);

        CHECK(func, EINVAL);

        /* Node allocation. */
        f = calloc(1, sizeof(struct hash_func));
        CHECK(func, ENOMEM);

        /* Node initialization. */
        strcpy(f->name, name);
        f->func = func;
        f->id = p->n_hash_funcs;

        /* Node add to tailq. */
        TAILQ_INSERT_TAIL(&p->hash_funcs, f, node);
        p->n_hash_funcs++;

        return 0;
}

static int
hash_func_build(struct rte_swx_pipeline *p)
{
        struct hash_func *func;

        /* Memory allocation. */
        p->hash_func_runtime = calloc(p->n_hash_funcs, sizeof(struct hash_func_runtime));
        CHECK(p->hash_func_runtime, ENOMEM);

        /* Hash function. */
        TAILQ_FOREACH(func, &p->hash_funcs, node) {
                struct hash_func_runtime *r = &p->hash_func_runtime[func->id];

                r->func = func->func;
        }

        return 0;
}

static void
hash_func_build_free(struct rte_swx_pipeline *p)
{
        free(p->hash_func_runtime);
        p->hash_func_runtime = NULL;
}

static void
hash_func_free(struct rte_swx_pipeline *p)
{
        hash_func_build_free(p);

        for ( ; ; ) {
                struct hash_func *elem;

                elem = TAILQ_FIRST(&p->hash_funcs);
                if (!elem)
                        break;

                TAILQ_REMOVE(&p->hash_funcs, elem, node);
                free(elem);
        }
}

/*
 * Header.
 */
static struct header *
header_find(struct rte_swx_pipeline *p, const char *name)
{
        struct header *elem;

        TAILQ_FOREACH(elem, &p->headers, node)
                if (strcmp(elem->name, name) == 0)
                        return elem;

        return NULL;
}

static struct header *
header_find_by_struct_id(struct rte_swx_pipeline *p, uint32_t struct_id)
{
        struct header *elem;

        TAILQ_FOREACH(elem, &p->headers, node)
                if (elem->struct_id == struct_id)
                        return elem;

        return NULL;
}

static struct header *
header_parse(struct rte_swx_pipeline *p,
             const char *name)
{
        if (name[0] != 'h' || name[1] != '.')
                return NULL;

        return header_find(p, &name[2]);
}

static struct field *
header_field_parse(struct rte_swx_pipeline *p,
                   const char *name,
                   struct header **header)
{
        struct header *h;
        struct field *f;
        char *header_name, *field_name;

        if ((name[0] != 'h') || (name[1] != '.'))
                return NULL;

        header_name = strdup(&name[2]);
        if (!header_name)
                return NULL;

        field_name = strchr(header_name, '.');
        if (!field_name) {
                free(header_name);
                return NULL;
        }

        *field_name = 0;
        field_name++;

        h = header_find(p, header_name);
        if (!h) {
                free(header_name);
                return NULL;
        }

        f = struct_type_field_find(h->st, field_name);
        if (!f) {
                free(header_name);
                return NULL;
        }

        if (header)
                *header = h;

        free(header_name);
        return f;
}

int
rte_swx_pipeline_packet_header_register(struct rte_swx_pipeline *p,
                                        const char *name,
                                        const char *struct_type_name)
{
        struct struct_type *st;
        struct header *h;
        size_t n_headers_max;

        CHECK(p, EINVAL);
        CHECK_NAME(name, EINVAL);
        CHECK_NAME(struct_type_name, EINVAL);

        CHECK(!header_find(p, name), EEXIST);

        st = struct_type_find(p, struct_type_name);
        CHECK(st, EINVAL);

        n_headers_max = RTE_SIZEOF_FIELD(struct thread, valid_headers) * 8;
        CHECK(p->n_headers < n_headers_max, ENOSPC);

        /* Node allocation. */
        h = calloc(1, sizeof(struct header));
        CHECK(h, ENOMEM);

        /* Node initialization. */
        strcpy(h->name, name);
        h->st = st;
        h->struct_id = p->n_structs;
        h->id = p->n_headers;

        /* Node add to tailq. */
        TAILQ_INSERT_TAIL(&p->headers, h, node);
        p->n_headers++;
        p->n_structs++;

        return 0;
}

static int
header_build(struct rte_swx_pipeline *p)
{
        struct header *h;
        uint32_t n_bytes = 0, i;

        TAILQ_FOREACH(h, &p->headers, node) {
                n_bytes += h->st->n_bits / 8;
        }

        for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
                struct thread *t = &p->threads[i];
                uint32_t offset = 0;

                t->headers = calloc(p->n_headers,
                                    sizeof(struct header_runtime));
                CHECK(t->headers, ENOMEM);

                t->headers_out = calloc(p->n_headers,
                                        sizeof(struct header_out_runtime));
                CHECK(t->headers_out, ENOMEM);

                t->header_storage = calloc(1, n_bytes);
                CHECK(t->header_storage, ENOMEM);

                t->header_out_storage = calloc(1, n_bytes);
                CHECK(t->header_out_storage, ENOMEM);

                TAILQ_FOREACH(h, &p->headers, node) {
                        uint8_t *header_storage;
                        uint32_t n_bytes =  h->st->n_bits / 8;

                        header_storage = &t->header_storage[offset];
                        offset += n_bytes;

                        t->headers[h->id].ptr0 = header_storage;
                        t->headers[h->id].n_bytes = n_bytes;

                        t->structs[h->struct_id] = header_storage;
                }
        }

        return 0;
}

static void
header_build_free(struct rte_swx_pipeline *p)
{
        uint32_t i;

        for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
                struct thread *t = &p->threads[i];

                free(t->headers_out);
                t->headers_out = NULL;

                free(t->headers);
                t->headers = NULL;

                free(t->header_out_storage);
                t->header_out_storage = NULL;

                free(t->header_storage);
                t->header_storage = NULL;
        }
}

static void
header_free(struct rte_swx_pipeline *p)
{
        header_build_free(p);

        for ( ; ; ) {
                struct header *elem;

                elem = TAILQ_FIRST(&p->headers);
                if (!elem)
                        break;

                TAILQ_REMOVE(&p->headers, elem, node);
                free(elem);
        }
}

/*
 * Meta-data.
 */
static struct field *
metadata_field_parse(struct rte_swx_pipeline *p, const char *name)
{
        if (!p->metadata_st)
                return NULL;

        if (name[0] != 'm' || name[1] != '.')
                return NULL;

        return struct_type_field_find(p->metadata_st, &name[2]);
}

int
rte_swx_pipeline_packet_metadata_register(struct rte_swx_pipeline *p,
                                          const char *struct_type_name)
{
        struct struct_type *st = NULL;

        CHECK(p, EINVAL);

        CHECK_NAME(struct_type_name, EINVAL);
        st  = struct_type_find(p, struct_type_name);
        CHECK(st, EINVAL);
        CHECK(!st->var_size, EINVAL);
        CHECK(!p->metadata_st, EINVAL);

        p->metadata_st = st;
        p->metadata_struct_id = p->n_structs;

        p->n_structs++;

        return 0;
}

static int
metadata_build(struct rte_swx_pipeline *p)
{
        uint32_t n_bytes = p->metadata_st->n_bits / 8;
        uint32_t i;

        /* Thread-level initialization. */
        for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
                struct thread *t = &p->threads[i];
                uint8_t *metadata;

                metadata = calloc(1, n_bytes);
                CHECK(metadata, ENOMEM);

                t->metadata = metadata;
                t->structs[p->metadata_struct_id] = metadata;
        }

        return 0;
}

static void
metadata_build_free(struct rte_swx_pipeline *p)
{
        uint32_t i;

        for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
                struct thread *t = &p->threads[i];

                free(t->metadata);
                t->metadata = NULL;
        }
}

static void
metadata_free(struct rte_swx_pipeline *p)
{
        metadata_build_free(p);
}

/*
 * Instruction.
 */
static int
instruction_is_tx(enum instruction_type type)
{
        switch (type) {
        case INSTR_TX:
        case INSTR_TX_I:
        case INSTR_DROP:
                return 1;

        default:
                return 0;
        }
}

static int
instruction_does_tx(struct instruction *instr)
{
        switch (instr->type) {
        case INSTR_TX:
        case INSTR_TX_I:
        case INSTR_DROP:
        case INSTR_HDR_EMIT_TX:
        case INSTR_HDR_EMIT2_TX:
        case INSTR_HDR_EMIT3_TX:
        case INSTR_HDR_EMIT4_TX:
        case INSTR_HDR_EMIT5_TX:
        case INSTR_HDR_EMIT6_TX:
        case INSTR_HDR_EMIT7_TX:
        case INSTR_HDR_EMIT8_TX:
                return 1;
        default:
                return 0;
        }
}

static int
instruction_is_jmp(struct instruction *instr)
{
        switch (instr->type) {
        case INSTR_JMP:
        case INSTR_JMP_VALID:
        case INSTR_JMP_INVALID:
        case INSTR_JMP_HIT:
        case INSTR_JMP_MISS:
        case INSTR_JMP_ACTION_HIT:
        case INSTR_JMP_ACTION_MISS:
        case INSTR_JMP_EQ:
        case INSTR_JMP_EQ_MH:
        case INSTR_JMP_EQ_HM:
        case INSTR_JMP_EQ_HH:
        case INSTR_JMP_EQ_I:
        case INSTR_JMP_NEQ:
        case INSTR_JMP_NEQ_MH:
        case INSTR_JMP_NEQ_HM:
        case INSTR_JMP_NEQ_HH:
        case INSTR_JMP_NEQ_I:
        case INSTR_JMP_LT:
        case INSTR_JMP_LT_MH:
        case INSTR_JMP_LT_HM:
        case INSTR_JMP_LT_HH:
        case INSTR_JMP_LT_MI:
        case INSTR_JMP_LT_HI:
        case INSTR_JMP_GT:
        case INSTR_JMP_GT_MH:
        case INSTR_JMP_GT_HM:
        case INSTR_JMP_GT_HH:
        case INSTR_JMP_GT_MI:
        case INSTR_JMP_GT_HI:
                return 1;

        default:
                return 0;
        }
}

static int
instruction_does_thread_yield(struct instruction *instr)
{
        switch (instr->type) {
        case INSTR_RX:
        case INSTR_TABLE:
        case INSTR_TABLE_AF:
        case INSTR_SELECTOR:
        case INSTR_LEARNER:
        case INSTR_LEARNER_AF:
        case INSTR_EXTERN_OBJ:
        case INSTR_EXTERN_FUNC:
                return 1;
        default:
                return 0;
        }
}

static struct field *
action_field_parse(struct action *action, const char *name);

static struct field *
struct_field_parse(struct rte_swx_pipeline *p,
                   struct action *action,
                   const char *name,
                   uint32_t *struct_id)
{
        struct field *f;

        switch (name[0]) {
        case 'h':
        {
                struct header *header;

                f = header_field_parse(p, name, &header);
                if (!f)
                        return NULL;

                *struct_id = header->struct_id;
                return f;
        }

        case 'm':
        {
                f = metadata_field_parse(p, name);
                if (!f)
                        return NULL;

                *struct_id = p->metadata_struct_id;
                return f;
        }

        case 't':
        {
                if (!action)
                        return NULL;

                f = action_field_parse(action, name);
                if (!f)
                        return NULL;

                *struct_id = 0;
                return f;
        }

        case 'e':
        {
                struct extern_obj *obj;

                f = extern_obj_mailbox_field_parse(p, name, &obj);
                if (!f)
                        return NULL;

                *struct_id = obj->struct_id;
                return f;
        }

        case 'f':
        {
                struct extern_func *func;

                f = extern_func_mailbox_field_parse(p, name, &func);
                if (!f)
                        return NULL;

                *struct_id = func->struct_id;
                return f;
        }

        default:
                return NULL;
        }
}

/*
 * rx.
 */
static int
instr_rx_translate(struct rte_swx_pipeline *p,
                   struct action *action,
                   char **tokens,
                   int n_tokens,
                   struct instruction *instr,
                   struct instruction_data *data __rte_unused)
{
        struct field *f;

        CHECK(!action, EINVAL);
        CHECK(n_tokens == 2, EINVAL);

        f = metadata_field_parse(p, tokens[1]);
        CHECK(f, EINVAL);

        instr->type = INSTR_RX;
        instr->io.io.offset = f->offset / 8;
        instr->io.io.n_bits = f->n_bits;
        return 0;
}

/*
 * tx.
 */
static int
instr_tx_translate(struct rte_swx_pipeline *p,
                   struct action *action __rte_unused,
                   char **tokens,
                   int n_tokens,
                   struct instruction *instr,
                   struct instruction_data *data __rte_unused)
{
        char *port = tokens[1];
        struct field *f;
        uint32_t port_val;

        CHECK(n_tokens == 2, EINVAL);

        f = metadata_field_parse(p, port);
        if (f) {
                instr->type = INSTR_TX;
                instr->io.io.offset = f->offset / 8;
                instr->io.io.n_bits = f->n_bits;
                return 0;
        }

        /* TX_I. */
        port_val = strtoul(port, &port, 0);
        CHECK(!port[0], EINVAL);

        instr->type = INSTR_TX_I;
        instr->io.io.val = port_val;
        return 0;
}

static int
instr_drop_translate(struct rte_swx_pipeline *p __rte_unused,
                     struct action *action __rte_unused,
                     char **tokens __rte_unused,
                     int n_tokens,
                     struct instruction *instr,
                     struct instruction_data *data __rte_unused)
{
        CHECK(n_tokens == 1, EINVAL);

        /* DROP. */
        instr->type = INSTR_DROP;
        return 0;
}

static inline void
instr_tx_exec(struct rte_swx_pipeline *p)
{
        struct thread *t = &p->threads[p->thread_id];
        struct instruction *ip = t->ip;

        __instr_tx_exec(p, t, ip);

        /* Thread. */
        thread_ip_reset(p, t);
        instr_rx_exec(p);
}

static inline void
instr_tx_i_exec(struct rte_swx_pipeline *p)
{
        struct thread *t = &p->threads[p->thread_id];
        struct instruction *ip = t->ip;

        __instr_tx_i_exec(p, t, ip);

        /* Thread. */
        thread_ip_reset(p, t);
        instr_rx_exec(p);
}

static inline void
instr_drop_exec(struct rte_swx_pipeline *p)
{
        struct thread *t = &p->threads[p->thread_id];
        struct instruction *ip = t->ip;

        __instr_drop_exec(p, t, ip);

        /* Thread. */
        thread_ip_reset(p, t);
        instr_rx_exec(p);
}

/*
 * mirror.
 */
static int
instr_mirror_translate(struct rte_swx_pipeline *p,
                       struct action *action,
                       char **tokens,
                       int n_tokens,
                       struct instruction *instr,
                       struct instruction_data *data __rte_unused)
{
        char *dst = tokens[1], *src = tokens[2];
        struct field *fdst, *fsrc;
        uint32_t dst_struct_id = 0, src_struct_id = 0;

        CHECK(n_tokens == 3, EINVAL);

        fdst = struct_field_parse(p, action, dst, &dst_struct_id);
        CHECK(fdst, EINVAL);
        CHECK(dst[0] != 'h', EINVAL);
        CHECK(!fdst->var_size, EINVAL);

        fsrc = struct_field_parse(p, action, src, &src_struct_id);
        CHECK(fsrc, EINVAL);
        CHECK(src[0] != 'h', EINVAL);
        CHECK(!fsrc->var_size, EINVAL);

        instr->type = INSTR_MIRROR;
        instr->mirror.dst.struct_id = (uint8_t)dst_struct_id;
        instr->mirror.dst.n_bits = fdst->n_bits;
        instr->mirror.dst.offset = fdst->offset / 8;
        instr->mirror.src.struct_id = (uint8_t)src_struct_id;
        instr->mirror.src.n_bits = fsrc->n_bits;
        instr->mirror.src.offset = fsrc->offset / 8;

        return 0;
}

static inline void
instr_mirror_exec(struct rte_swx_pipeline *p)
{
        struct thread *t = &p->threads[p->thread_id];
        struct instruction *ip = t->ip;

        __instr_mirror_exec(p, t, ip);

        /* Thread. */
        thread_ip_inc(p);
}

/*
 * recirculate.
 */
static int
instr_recirculate_translate(struct rte_swx_pipeline *p __rte_unused,
                            struct action *action __rte_unused,
                            char **tokens __rte_unused,
                            int n_tokens,
                            struct instruction *instr,
                            struct instruction_data *data __rte_unused)
{
        CHECK(n_tokens == 1, EINVAL);

        instr->type = INSTR_RECIRCULATE;
        return 0;
}

static int
instr_recircid_translate(struct rte_swx_pipeline *p,
                         struct action *action __rte_unused,
                         char **tokens,
                         int n_tokens,
                         struct instruction *instr,
                         struct instruction_data *data __rte_unused)
{
        struct field *f;

        CHECK(n_tokens == 2, EINVAL);

        f = metadata_field_parse(p, tokens[1]);
        CHECK(f, EINVAL);

        instr->type = INSTR_RECIRCID;
        instr->io.io.offset = f->offset / 8;
        instr->io.io.n_bits = f->n_bits;
        return 0;
}

static inline void
instr_recirculate_exec(struct rte_swx_pipeline *p)
{
        struct thread *t = &p->threads[p->thread_id];
        struct instruction *ip = t->ip;

        __instr_recirculate_exec(p, t, ip);

        /* Thread. */
        thread_ip_inc(p);
}

static inline void
instr_recircid_exec(struct rte_swx_pipeline *p)
{
        struct thread *t = &p->threads[p->thread_id];
        struct instruction *ip = t->ip;

        __instr_recircid_exec(p, t, ip);

        /* Thread. */
        thread_ip_inc(p);
}

/*
 * extract.
 */
static int
instr_hdr_extract_translate(struct rte_swx_pipeline *p,
                            struct action *action,
                            char **tokens,
                            int n_tokens,
                            struct instruction *instr,
                            struct instruction_data *data __rte_unused)
{
        struct header *h;

        CHECK(!action, EINVAL);
        CHECK((n_tokens == 2) || (n_tokens == 3), EINVAL);

        h = header_parse(p, tokens[1]);
        CHECK(h, EINVAL);

        if (n_tokens == 2) {
                CHECK(!h->st->var_size, EINVAL);

                instr->type = INSTR_HDR_EXTRACT;
                instr->io.hdr.header_id[0] = h->id;
                instr->io.hdr.struct_id[0] = h->struct_id;
                instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
        } else {
                struct field *mf;

                CHECK(h->st->var_size, EINVAL);

                mf = metadata_field_parse(p, tokens[2]);
                CHECK(mf, EINVAL);
                CHECK(!mf->var_size, EINVAL);

                instr->type = INSTR_HDR_EXTRACT_M;
                instr->io.io.offset = mf->offset / 8;
                instr->io.io.n_bits = mf->n_bits;
                instr->io.hdr.header_id[0] = h->id;
                instr->io.hdr.struct_id[0] = h->struct_id;
                instr->io.hdr.n_bytes[0] = h->st->n_bits_min / 8;
        }

        return 0;
}

static int
instr_hdr_lookahead_translate(struct rte_swx_pipeline *p,
                              struct action *action,
                              char **tokens,
                              int n_tokens,
                              struct instruction *instr,
                              struct instruction_data *data __rte_unused)
{
        struct header *h;

        CHECK(!action, EINVAL);
        CHECK(n_tokens == 2, EINVAL);

        h = header_parse(p, tokens[1]);
        CHECK(h, EINVAL);
        CHECK(!h->st->var_size, EINVAL);

        instr->type = INSTR_HDR_LOOKAHEAD;
        instr->io.hdr.header_id[0] = h->id;
        instr->io.hdr.struct_id[0] = h->struct_id;

        instr->io.hdr.n_bytes[0] = 0; /* Unused. */

        return 0;
}

static inline void
instr_hdr_extract_exec(struct rte_swx_pipeline *p)
{
        struct thread *t = &p->threads[p->thread_id];
        struct instruction *ip = t->ip;

        __instr_hdr_extract_exec(p, t, ip);

        /* Thread. */
        thread_ip_inc(p);
}

static inline void
instr_hdr_extract2_exec(struct rte_swx_pipeline *p)
{
        struct thread *t = &p->threads[p->thread_id];
        struct instruction *ip = t->ip;

        __instr_hdr_extract2_exec(p, t, ip);

        /* Thread. */
        thread_ip_inc(p);
}

static inline void
instr_hdr_extract3_exec(struct rte_swx_pipeline *p)
{
        struct thread *t = &p->threads[p->thread_id];
        struct instruction *ip = t->ip;

        __instr_hdr_extract3_exec(p, t, ip);

        /* Thread. */
        thread_ip_inc(p);
}

static inline void
instr_hdr_extract4_exec(struct rte_swx_pipeline *p)
{
        struct thread *t = &p->threads[p->thread_id];
        struct instruction *ip = t->ip;

        __instr_hdr_extract4_exec(p, t, ip);

        /* Thread. */
        thread_ip_inc(p);
}

static inline void
instr_hdr_extract5_exec(struct rte_swx_pipeline *p)
{
        struct thread *t = &p->threads[p->thread_id];
        struct instruction *ip = t->ip;

        __instr_hdr_extract5_exec(p, t, ip);

        /* Thread. */
        thread_ip_inc(p);
}

static inline void
instr_hdr_extract6_exec(struct rte_swx_pipeline *p)
{
        struct thread *t = &p->threads[p->thread_id];
        struct instruction *ip = t->ip;

        __instr_hdr_extract6_exec(p, t, ip);

        /* Thread. */
        thread_ip_inc(p);
}

static inline void
instr_hdr_extract7_exec(struct rte_swx_pipeline *p)
{
        struct thread *t = &p->threads[p->thread_id];
        struct instruction *ip = t->ip;

        __instr_hdr_extract7_exec(p, t, ip);

        /* Thread. */
        thread_ip_inc(p);
}

static inline void
instr_hdr_extract8_exec(struct rte_swx_pipeline *p)
{
        struct thread *t = &p->threads[p->thread_id];
        struct instruction *ip = t->ip;

        __instr_hdr_extract8_exec(p, t, ip);

        /* Thread. */
        thread_ip_inc(p);
}

static inline void
instr_hdr_extract_m_exec(struct rte_swx_pipeline *p)
{
        struct thread *t = &p->threads[p->thread_id];
        struct instruction *ip = t->ip;

        __instr_hdr_extract_m_exec(p, t, ip);

        /* Thread. */
        thread_ip_inc(p);
}

static inline void
instr_hdr_lookahead_exec(struct rte_swx_pipeline *p)
{
        struct thread *t = &p->threads[p->thread_id];
        struct instruction *ip = t->ip;

        __instr_hdr_lookahead_exec(p, t, ip);

        /* Thread. */
        thread_ip_inc(p);
}

/*
 * emit.
 */
static int
instr_hdr_emit_translate(struct rte_swx_pipeline *p,
                         struct action *action __rte_unused,
                         char **tokens,
                         int n_tokens,
                         struct instruction *instr,
                         struct instruction_data *data __rte_unused)
{
        struct header *h;

        CHECK(n_tokens == 2, EINVAL);

        h = header_parse(p, tokens[1]);
        CHECK(h, EINVAL);

        instr->type = INSTR_HDR_EMIT;
        instr->io.hdr.header_id[0] = h->id;
        instr->io.hdr.struct_id[0] = h->struct_id;
        instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
        return 0;
}

static inline void
instr_hdr_emit_exec(struct rte_swx_pipeline *p)
{
        struct thread *t = &p->threads[p->thread_id];
        struct instruction *ip = t->ip;

        __instr_hdr_emit_exec(p, t, ip);

        /* Thread. */
        thread_ip_inc(p);
}

static inline void
instr_hdr_emit_tx_exec(struct rte_swx_pipeline *p)
{
        struct thread *t = &p->threads[p->thread_id];
        struct instruction *ip = t->ip;

        __instr_hdr_emit_tx_exec(p, t, ip);

        /* Thread. */
        thread_ip_reset(p, t);
        instr_rx_exec(p);
}

static inline void
instr_hdr_emit2_tx_exec(struct rte_swx_pipeline *p)
{
        struct thread *t = &p->threads[p->thread_id];
        struct instruction *ip = t->ip;

        __instr_hdr_emit2_tx_exec(p, t, ip);

        /* Thread. */
        thread_ip_reset(p, t);
        instr_rx_exec(p);
}

static inline void
instr_hdr_emit3_tx_exec(struct rte_swx_pipeline *p)
{
        struct thread *t = &p->threads[p->thread_id];
        struct instruction *ip = t->ip;

        __instr_hdr_emit3_tx_exec(p, t, ip);

        /* Thread. */
        thread_ip_reset(p, t);
        instr_rx_exec(p);
}

static inline void
instr_hdr_emit4_tx_exec(struct rte_swx_pipeline *p)
{
        struct thread *t = &p->threads[p->thread_id];
        struct instruction *ip = t->ip;

        __instr_hdr_emit4_tx_exec(p, t, ip);

        /* Thread. */
        thread_ip_reset(p, t);
        instr_rx_exec(p);
}

static inline void
instr_hdr_emit5_tx_exec(struct rte_swx_pipeline *p)
{
        struct thread *t = &p->threads[p->thread_id];
        struct instruction *ip = t->ip;

        __instr_hdr_emit5_tx_exec(p, t, ip);

        /* Thread. */
        thread_ip_reset(p, t);
        instr_rx_exec(p);
}

static inline void
instr_hdr_emit6_tx_exec(struct rte_swx_pipeline *p)
{
        struct thread *t = &p->threads[p->thread_id];
        struct instruction *ip = t->ip;

        __instr_hdr_emit6_tx_exec(p, t, ip);

        /* Thread. */
        thread_ip_reset(p, t);
        instr_rx_exec(p);
}

static inline void
instr_hdr_emit7_tx_exec(struct rte_swx_pipeline *p)
{
        struct thread *t = &p->threads[p->thread_id];
        struct instruction *ip = t->ip;

        __instr_hdr_emit7_tx_exec(p, t, ip);

        /* Thread. */
        thread_ip_reset(p, t);
        instr_rx_exec(p);
}

static inline void
instr_hdr_emit8_tx_exec(struct rte_swx_pipeline *p)
{
        struct thread *t = &p->threads[p->thread_id];
        struct instruction *ip = t->ip;

        __instr_hdr_emit8_tx_exec(p, t, ip);

        /* Thread. */
        thread_ip_reset(p, t);
        instr_rx_exec(p);
}

/*
 * validate.
 */
static int
instr_hdr_validate_translate(struct rte_swx_pipeline *p,
                             struct action *action __rte_unused,
                             char **tokens,
                             int n_tokens,
                             struct instruction *instr,
                             struct instruction_data *data __rte_unused)
{
        struct header *h;

        CHECK(n_tokens == 2, EINVAL);

        h = header_parse(p, tokens[1]);
        CHECK(h, EINVAL);

        instr->type = INSTR_HDR_VALIDATE;
        instr->valid.header_id = h->id;
        instr->valid.struct_id = h->struct_id;
        return 0;
}

static inline void
instr_hdr_validate_exec(struct rte_swx_pipeline *p)
{
        struct thread *t = &p->threads[p->thread_id];
        struct instruction *ip = t->ip;

        __instr_hdr_validate_exec(p, t, ip);

        /* Thread. */
        thread_ip_inc(p);
}

/*
 * invalidate.
 */
static int
instr_hdr_invalidate_translate(struct rte_swx_pipeline *p,
                               struct action *action __rte_unused,
                               char **tokens,
                               int n_tokens,
                               struct instruction *instr,
                               struct instruction_data *data __rte_unused)
{
        struct header *h;

        CHECK(n_tokens == 2, EINVAL);

        h = header_parse(p, tokens[1]);
        CHECK(h, EINVAL);

        instr->type = INSTR_HDR_INVALIDATE;
        instr->valid.header_id = h->id;
        return 0;
}

static inline void
instr_hdr_invalidate_exec(struct rte_swx_pipeline *p)
{
        struct thread *t = &p->threads[p->thread_id];
        struct instruction *ip = t->ip;

        __instr_hdr_invalidate_exec(p, t, ip);

        /* Thread. */
        thread_ip_inc(p);
}

/*
 * table.
 */
static struct table *
table_find(struct rte_swx_pipeline *p, const char *name);

static struct selector *
selector_find(struct rte_swx_pipeline *p, const char *name);

static struct learner *
learner_find(struct rte_swx_pipeline *p, const char *name);

static int
instr_table_translate(struct rte_swx_pipeline *p,
                      struct action *action,
                      char **tokens,
                      int n_tokens,
                      struct instruction *instr,
                      struct instruction_data *data __rte_unused)
{
        struct table *t;
        struct selector *s;
        struct learner *l;

        CHECK(!action, EINVAL);
        CHECK(n_tokens == 2, EINVAL);

        t = table_find(p, tokens[1]);
        if (t) {
                instr->type = INSTR_TABLE;
                instr->table.table_id = t->id;
                return 0;
        }

        s = selector_find(p, tokens[1]);
        if (s) {
                instr->type = INSTR_SELECTOR;
                instr->table.table_id = s->id;
                return 0;
        }

        l = learner_find(p, tokens[1]);
        if (l) {
                instr->type = INSTR_LEARNER;
                instr->table.table_id = l->id;
                return 0;
        }

        CHECK(0, EINVAL);
}

static inline void
instr_table_exec(struct rte_swx_pipeline *p)
{
        struct thread *t = &p->threads[p->thread_id];
        struct instruction *ip = t->ip;
        uint32_t table_id = ip->table.table_id;
        struct rte_swx_table_state *ts = &t->table_state[table_id];
        struct table_runtime *table = &t->tables[table_id];
        struct table_statistics *stats = &p->table_stats[table_id];
        uint64_t action_id, n_pkts_hit, n_pkts_action;
        uint8_t *action_data;
        int done, hit;

        /* Table. */
        done = table->func(ts->obj,
                           table->mailbox,
                           table->key,
                           &action_id,
                           &action_data,
                           &hit);
        if (!done) {
                /* Thread. */
                TRACE("[Thread %2u] table %u (not finalized)\n",
                      p->thread_id,
                      table_id);

                thread_yield(p);
                return;
        }

        action_id = hit ? action_id : ts->default_action_id;
        action_data = hit ? action_data : ts->default_action_data;
        n_pkts_hit = stats->n_pkts_hit[hit];
        n_pkts_action = stats->n_pkts_action[action_id];

        TRACE("[Thread %2u] table %u (%s, action %u)\n",
              p->thread_id,
              table_id,
              hit ? "hit" : "miss",
              (uint32_t)action_id);

        t->action_id = action_id;
        t->structs[0] = action_data;
        t->hit = hit;
        stats->n_pkts_hit[hit] = n_pkts_hit + 1;
        stats->n_pkts_action[action_id] = n_pkts_action + 1;

        /* Thread. */
        thread_ip_action_call(p, t, action_id);
}

static inline void
instr_table_af_exec(struct rte_swx_pipeline *p)
{
        struct thread *t = &p->threads[p->thread_id];
        struct instruction *ip = t->ip;
        uint32_t table_id = ip->table.table_id;
        struct rte_swx_table_state *ts = &t->table_state[table_id];
        struct table_runtime *table = &t->tables[table_id];
        struct table_statistics *stats = &p->table_stats[table_id];
        uint64_t action_id, n_pkts_hit, n_pkts_action;
        uint8_t *action_data;
        action_func_t action_func;
        int done, hit;

        /* Table. */
        done = table->func(ts->obj,
                           table->mailbox,
                           table->key,
                           &action_id,
                           &action_data,
                           &hit);
        if (!done) {
                /* Thread. */
                TRACE("[Thread %2u] table %u (not finalized)\n",
                      p->thread_id,
                      table_id);

                thread_yield(p);
                return;
        }

        action_id = hit ? action_id : ts->default_action_id;
        action_data = hit ? action_data : ts->default_action_data;
        action_func = p->action_funcs[action_id];
        n_pkts_hit = stats->n_pkts_hit[hit];
        n_pkts_action = stats->n_pkts_action[action_id];

        TRACE("[Thread %2u] table %u (%s, action %u)\n",
              p->thread_id,
              table_id,
              hit ? "hit" : "miss",
              (uint32_t)action_id);

        t->action_id = action_id;
        t->structs[0] = action_data;
        t->hit = hit;
        stats->n_pkts_hit[hit] = n_pkts_hit + 1;
        stats->n_pkts_action[action_id] = n_pkts_action + 1;

        /* Thread. */
        thread_ip_inc(p);

        /* Action. */
        action_func(p);
}

static inline void
instr_selector_exec(struct rte_swx_pipeline *p)
{
        struct thread *t = &p->threads[p->thread_id];
        struct instruction *ip = t->ip;
        uint32_t selector_id = ip->table.table_id;
        struct rte_swx_table_state *ts = &t->table_state[p->n_tables + selector_id];
        struct selector_runtime *selector = &t->selectors[selector_id];
        struct selector_statistics *stats = &p->selector_stats[selector_id];
        uint64_t n_pkts = stats->n_pkts;
        int done;

        /* Table. */
        done = rte_swx_table_selector_select(ts->obj,
                           selector->mailbox,
                           selector->group_id_buffer,
                           selector->selector_buffer,
                           selector->member_id_buffer);
        if (!done) {
                /* Thread. */
                TRACE("[Thread %2u] selector %u (not finalized)\n",
                      p->thread_id,
                      selector_id);

                thread_yield(p);
                return;
        }


        TRACE("[Thread %2u] selector %u\n",
              p->thread_id,
              selector_id);

        stats->n_pkts = n_pkts + 1;

        /* Thread. */
        thread_ip_inc(p);
}

static inline void
instr_learner_exec(struct rte_swx_pipeline *p)
{
        struct thread *t = &p->threads[p->thread_id];
        struct instruction *ip = t->ip;
        uint32_t learner_id = ip->table.table_id;
        struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
                p->n_selectors + learner_id];
        struct learner_runtime *l = &t->learners[learner_id];
        struct learner_statistics *stats = &p->learner_stats[learner_id];
        uint64_t action_id, n_pkts_hit, n_pkts_action, time;
        uint8_t *action_data;
        int done, hit;

        /* Table. */
        time = rte_get_tsc_cycles();

        done = rte_swx_table_learner_lookup(ts->obj,
                                            l->mailbox,
                                            time,
                                            l->key,
                                            &action_id,
                                            &action_data,
                                            &hit);
        if (!done) {
                /* Thread. */
                TRACE("[Thread %2u] learner %u (not finalized)\n",
                      p->thread_id,
                      learner_id);

                thread_yield(p);
                return;
        }

        action_id = hit ? action_id : ts->default_action_id;
        action_data = hit ? action_data : ts->default_action_data;
        n_pkts_hit = stats->n_pkts_hit[hit];
        n_pkts_action = stats->n_pkts_action[action_id];

        TRACE("[Thread %2u] learner %u (%s, action %u)\n",
              p->thread_id,
              learner_id,
              hit ? "hit" : "miss",
              (uint32_t)action_id);

        t->action_id = action_id;
        t->structs[0] = action_data;
        t->hit = hit;
        t->learner_id = learner_id;
        t->time = time;
        stats->n_pkts_hit[hit] = n_pkts_hit + 1;
        stats->n_pkts_action[action_id] = n_pkts_action + 1;

        /* Thread. */
        thread_ip_action_call(p, t, action_id);
}

static inline void
instr_learner_af_exec(struct rte_swx_pipeline *p)
{
        struct thread *t = &p->threads[p->thread_id];
        struct instruction *ip = t->ip;
        uint32_t learner_id = ip->table.table_id;
        struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
                p->n_selectors + learner_id];
        struct learner_runtime *l = &t->learners[learner_id];
        struct learner_statistics *stats = &p->learner_stats[learner_id];
        uint64_t action_id, n_pkts_hit, n_pkts_action, time;
        uint8_t *action_data;
        action_func_t action_func;
        int done, hit;

        /* Table. */
        time = rte_get_tsc_cycles();

        done = rte_swx_table_learner_lookup(ts->obj,
                                            l->mailbox,
                                            time,
                                            l->key,
                                            &action_id,
                                            &action_data,
                                            &hit);
        if (!done) {
                /* Thread. */
                TRACE("[Thread %2u] learner %u (not finalized)\n",
                      p->thread_id,
                      learner_id);

                thread_yield(p);
                return;
        }

        action_id = hit ? action_id : ts->default_action_id;
        action_data = hit ? action_data : ts->default_action_data;
        action_func = p->action_funcs[action_id];
        n_pkts_hit = stats->n_pkts_hit[hit];
        n_pkts_action = stats->n_pkts_action[action_id];

        TRACE("[Thread %2u] learner %u (%s, action %u)\n",
              p->thread_id,
              learner_id,
              hit ? "hit" : "miss",
              (uint32_t)action_id);

        t->action_id = action_id;
        t->structs[0] = action_data;
        t->hit = hit;
        t->learner_id = learner_id;
        t->time = time;
        stats->n_pkts_hit[hit] = n_pkts_hit + 1;
        stats->n_pkts_action[action_id] = n_pkts_action + 1;

        /* Thread. */
        thread_ip_inc(p);

        /* Action */
        action_func(p);
}

/*
 * learn.
 */
static struct action *
action_find(struct rte_swx_pipeline *p, const char *name);

static int
action_has_nbo_args(struct action *a);

static int
learner_action_args_check(struct rte_swx_pipeline *p, struct action *a, const char *mf_name);

static int
instr_learn_translate(struct rte_swx_pipeline *p,
                      struct action *action,
                      char **tokens,
                      int n_tokens,
                      struct instruction *instr,
                      struct instruction_data *data __rte_unused)
{
        struct action *a;
        struct field *mf_first_arg = NULL, *mf_timeout_id = NULL;
        const char *mf_first_arg_name, *mf_timeout_id_name;

        CHECK(action, EINVAL);
        CHECK((n_tokens == 3) || (n_tokens == 4), EINVAL);

        /* Action. */
        a = action_find(p, tokens[1]);
        CHECK(a, EINVAL);
        CHECK(!action_has_nbo_args(a), EINVAL);

        /* Action first argument. */
        mf_first_arg_name = (n_tokens == 4) ? tokens[2] : NULL;
        CHECK(!learner_action_args_check(p, a, mf_first_arg_name), EINVAL);

        if (mf_first_arg_name) {
                mf_first_arg = metadata_field_parse(p, mf_first_arg_name);
                CHECK(mf_first_arg, EINVAL);
        }

        /* Timeout ID. */
        mf_timeout_id_name = (n_tokens == 4) ? tokens[3] : tokens[2];
        CHECK_NAME(mf_timeout_id_name, EINVAL);
        mf_timeout_id = metadata_field_parse(p, mf_timeout_id_name);
        CHECK(mf_timeout_id, EINVAL);

        /* Instruction. */
        instr->type = INSTR_LEARNER_LEARN;
        instr->learn.action_id = a->id;
        instr->learn.mf_first_arg_offset = mf_first_arg ? (mf_first_arg->offset / 8) : 0;
        instr->learn.mf_timeout_id_offset = mf_timeout_id->offset / 8;
        instr->learn.mf_timeout_id_n_bits = mf_timeout_id->n_bits;

        return 0;
}

static inline void
instr_learn_exec(struct rte_swx_pipeline *p)
{
        struct thread *t = &p->threads[p->thread_id];
        struct instruction *ip = t->ip;

        __instr_learn_exec(p, t, ip);

        /* Thread. */
        thread_ip_inc(p);
}

/*
 * rearm.
 */
static int
instr_rearm_translate(struct rte_swx_pipeline *p,
                      struct action *action,
                      char **tokens,
                      int n_tokens,
                      struct instruction *instr,
                      struct instruction_data *data __rte_unused)
{
        struct field *mf_timeout_id;
        const char *mf_timeout_id_name;

        CHECK(action, EINVAL);
        CHECK((n_tokens == 1) || (n_tokens == 2), EINVAL);

        /* INSTR_LEARNER_REARM. */
        if (n_tokens == 1) {
                instr->type = INSTR_LEARNER_REARM;
                return 0;
        }

        /* INSTR_LEARNER_REARM_NEW. */
        mf_timeout_id_name = tokens[1];
        CHECK_NAME(mf_timeout_id_name, EINVAL);
        mf_timeout_id = metadata_field_parse(p, mf_timeout_id_name);
        CHECK(mf_timeout_id, EINVAL);

        instr->type = INSTR_LEARNER_REARM_NEW;
        instr->learn.mf_timeout_id_offset = mf_timeout_id->offset / 8;
        instr->learn.mf_timeout_id_n_bits = mf_timeout_id->n_bits;

        return 0;
}

static inline void
instr_rearm_exec(struct rte_swx_pipeline *p)
{
        struct thread *t = &p->threads[p->thread_id];
        struct instruction *ip = t->ip;

        __instr_rearm_exec(p, t, ip);

        /* Thread. */
        thread_ip_inc(p);
}

static inline void
instr_rearm_new_exec(struct rte_swx_pipeline *p)
{
        struct thread *t = &p->threads[p->thread_id];
        struct instruction *ip = t->ip;

        __instr_rearm_new_exec(p, t, ip);

        /* Thread. */
        thread_ip_inc(p);
}

/*
 * forget.
 */
static int
instr_forget_translate(struct rte_swx_pipeline *p __rte_unused,
                       struct action *action,
                       char **tokens __rte_unused,
                       int n_tokens,
                       struct instruction *instr,
                       struct instruction_data *data __rte_unused)
{
        CHECK(action, EINVAL);
        CHECK(n_tokens == 1, EINVAL);

        instr->type = INSTR_LEARNER_FORGET;

        return 0;
}

static inline void
instr_forget_exec(struct rte_swx_pipeline *p)
{
        struct thread *t = &p->threads[p->thread_id];
        struct instruction *ip = t->ip;

        __instr_forget_exec(p, t, ip);

        /* Thread. */
        thread_ip_inc(p);
}

/*
 * extern.
 */
static int
instr_extern_translate(struct rte_swx_pipeline *p,
                       struct action *action __rte_unused,
                       char **tokens,
                       int n_tokens,
                       struct instruction *instr,
                       struct instruction_data *data __rte_unused)
{
        char *token = tokens[1];

        CHECK(n_tokens == 2, EINVAL);

        if (token[0] == 'e') {
                struct extern_obj *obj;
                struct extern_type_member_func *func;

                func = extern_obj_member_func_parse(p, token, &obj);
                CHECK(func, EINVAL);

                instr->type = INSTR_EXTERN_OBJ;
                instr->ext_obj.ext_obj_id = obj->id;
                instr->ext_obj.func_id = func->id;

                return 0;
        }

        if (token[0] == 'f') {
                struct extern_func *func;

                func = extern_func_parse(p, token);
                CHECK(func, EINVAL);

                instr->type = INSTR_EXTERN_FUNC;
                instr->ext_func.ext_func_id = func->id;

                return 0;
        }

        CHECK(0, EINVAL);
}

static inline void
instr_extern_obj_exec(struct rte_swx_pipeline *p)
{
        struct thread *t = &p->threads[p->thread_id];
        struct instruction *ip = t->ip;
        uint32_t done;

        /* Extern object member function execute. */
        done = __instr_extern_obj_exec(p, t, ip);

        /* Thread. */
        thread_ip_inc_cond(t, done);
        thread_yield_cond(p, done ^ 1);
}

static inline void
instr_extern_func_exec(struct rte_swx_pipeline *p)
{
        struct thread *t = &p->threads[p->thread_id];
        struct instruction *ip = t->ip;
        uint32_t done;

        /* Extern function execute. */
        done = __instr_extern_func_exec(p, t, ip);

        /* Thread. */
        thread_ip_inc_cond(t, done);
        thread_yield_cond(p, done ^ 1);
}

/*
 * hash.
 */
static int
instr_hash_translate(struct rte_swx_pipeline *p,
                     struct action *action,
                     char **tokens,
                     int n_tokens,
                     struct instruction *instr,
                     struct instruction_data *data __rte_unused)
{
        struct hash_func *func;
        struct field *dst, *src_first, *src_last;
        uint32_t src_struct_id_first = 0, src_struct_id_last = 0;

        CHECK(n_tokens == 5, EINVAL);

        func = hash_func_find(p, tokens[1]);
        CHECK(func, EINVAL);

        dst = metadata_field_parse(p, tokens[2]);
        CHECK(dst, EINVAL);

        src_first = struct_field_parse(p, action, tokens[3], &src_struct_id_first);
        CHECK(src_first, EINVAL);

        src_last = struct_field_parse(p, action, tokens[4], &src_struct_id_last);
        CHECK(src_last, EINVAL);
        CHECK(src_struct_id_first == src_struct_id_last, EINVAL);

        instr->type = INSTR_HASH_FUNC;
        instr->hash_func.hash_func_id = (uint8_t)func->id;
        instr->hash_func.dst.offset = (uint8_t)dst->offset / 8;
        instr->hash_func.dst.n_bits = (uint8_t)dst->n_bits;
        instr->hash_func.src.struct_id = (uint8_t)src_struct_id_first;
        instr->hash_func.src.offset = (uint16_t)src_first->offset / 8;
        instr->hash_func.src.n_bytes = (uint16_t)((src_last->offset + src_last->n_bits -
                src_first->offset) / 8);

        return 0;
}

static inline void
instr_hash_func_exec(struct rte_swx_pipeline *p)
{
        struct thread *t = &p->threads[p->thread_id];
        struct instruction *ip = t->ip;

        /* Extern function execute. */
        __instr_hash_func_exec(p, t, ip);

        /* Thread. */
        thread_ip_inc(p);
}

/*
 * mov.
 */
static int
instr_mov_translate(struct rte_swx_pipeline *p,
                    struct action *action,
                    char **tokens,
                    int n_tokens,
                    struct instruction *instr,
                    struct instruction_data *data __rte_unused)
{
        char *dst = tokens[1], *src = tokens[2];
        struct field *fdst, *fsrc;
        uint64_t src_val;
        uint32_t dst_struct_id = 0, src_struct_id = 0;

        CHECK(n_tokens == 3, EINVAL);

        fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
        CHECK(fdst, EINVAL);
        CHECK(!fdst->var_size, EINVAL);

        /* MOV, MOV_MH, MOV_HM or MOV_HH. */
        fsrc = struct_field_parse(p, action, src, &src_struct_id);
        if (fsrc) {
                CHECK(!fsrc->var_size, EINVAL);

                instr->type = INSTR_MOV;
                if (dst[0] != 'h' && src[0] == 'h')
                        instr->type = INSTR_MOV_MH;
                if (dst[0] == 'h' && src[0] != 'h')
                        instr->type = INSTR_MOV_HM;
                if (dst[0] == 'h' && src[0] == 'h')
                        instr->type = INSTR_MOV_HH;

                instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
                instr->mov.dst.n_bits = fdst->n_bits;
                instr->mov.dst.offset = fdst->offset / 8;
                instr->mov.src.struct_id = (uint8_t)src_struct_id;
                instr->mov.src.n_bits = fsrc->n_bits;
                instr->mov.src.offset = fsrc->offset / 8;
                return 0;
        }

        /* MOV_I. */
        src_val = strtoull(src, &src, 0);
        CHECK(!src[0], EINVAL);

        if (dst[0] == 'h')
                src_val = hton64(src_val) >> (64 - fdst->n_bits);

        instr->type = INSTR_MOV_I;
        instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
        instr->mov.dst.n_bits = fdst->n_bits;
        instr->mov.dst.offset = fdst->offset / 8;
        instr->mov.src_val = src_val;
        return 0;
}