qemu/contrib/rdmacm-mux/main.c
<<
>>
Prefs
   1/*
   2 * QEMU paravirtual RDMA - rdmacm-mux implementation
   3 *
   4 * Copyright (C) 2018 Oracle
   5 * Copyright (C) 2018 Red Hat Inc
   6 *
   7 * Authors:
   8 *     Yuval Shaia <yuval.shaia@oracle.com>
   9 *     Marcel Apfelbaum <marcel@redhat.com>
  10 *
  11 * This work is licensed under the terms of the GNU GPL, version 2 or later.
  12 * See the COPYING file in the top-level directory.
  13 *
  14 */
  15
  16#include "qemu/osdep.h"
  17#include <sys/poll.h>
  18#include <sys/ioctl.h>
  19#include <pthread.h>
  20#include <syslog.h>
  21
  22#include <infiniband/verbs.h>
  23#include <infiniband/umad.h>
  24#include <infiniband/umad_types.h>
  25#include <infiniband/umad_sa.h>
  26#include <infiniband/umad_cm.h>
  27
  28#include "rdmacm-mux.h"
  29
  30#define SCALE_US 1000
  31#define COMMID_TTL 2 /* How many SCALE_US a context of MAD session is saved */
  32#define SLEEP_SECS 5 /* This is used both in poll() and thread */
  33#define SERVER_LISTEN_BACKLOG 10
  34#define MAX_CLIENTS 4096
  35#define MAD_RMPP_VERSION 0
  36#define MAD_METHOD_MASK0 0x8
  37
  38#define IB_USER_MAD_LONGS_PER_METHOD_MASK (128 / (8 * sizeof(long)))
  39
  40#define CM_REQ_DGID_POS      80
  41#define CM_SIDR_REQ_DGID_POS 44
  42
  43/* The below can be override by command line parameter */
  44#define UNIX_SOCKET_PATH "/var/run/rdmacm-mux"
  45/* Has format %s-%s-%d" <path>-<rdma-dev--name>-<port> */
  46#define SOCKET_PATH_MAX (PATH_MAX - NAME_MAX - sizeof(int) - 2)
  47#define RDMA_PORT_NUM 1
  48
  49typedef struct RdmaCmServerArgs {
  50    char unix_socket_path[PATH_MAX];
  51    char rdma_dev_name[NAME_MAX];
  52    int rdma_port_num;
  53} RdmaCMServerArgs;
  54
  55typedef struct CommId2FdEntry {
  56    int fd;
  57    int ttl; /* Initialized to 2, decrement each timeout, entry delete when 0 */
  58    __be64 gid_ifid;
  59} CommId2FdEntry;
  60
  61typedef struct RdmaCmUMadAgent {
  62    int port_id;
  63    int agent_id;
  64    GHashTable *gid2fd; /* Used to find fd of a given gid */
  65    GHashTable *commid2fd; /* Used to find fd on of a given comm_id */
  66} RdmaCmUMadAgent;
  67
  68typedef struct RdmaCmServer {
  69    bool run;
  70    RdmaCMServerArgs args;
  71    struct pollfd fds[MAX_CLIENTS];
  72    int nfds;
  73    RdmaCmUMadAgent umad_agent;
  74    pthread_t umad_recv_thread;
  75    pthread_rwlock_t lock;
  76} RdmaCMServer;
  77
  78static RdmaCMServer server = {0};
  79
  80static void usage(const char *progname)
  81{
  82    printf("Usage: %s [OPTION]...\n"
  83           "Start a RDMA-CM multiplexer\n"
  84           "\n"
  85           "\t-h                    Show this help\n"
  86           "\t-d rdma-device-name   Name of RDMA device to register with\n"
  87           "\t-s unix-socket-path   Path to unix socket to listen on (default %s)\n"
  88           "\t-p rdma-device-port   Port number of RDMA device to register with (default %d)\n",
  89           progname, UNIX_SOCKET_PATH, RDMA_PORT_NUM);
  90}
  91
  92static void help(const char *progname)
  93{
  94    fprintf(stderr, "Try '%s -h' for more information.\n", progname);
  95}
  96
  97static void parse_args(int argc, char *argv[])
  98{
  99    int c;
 100    char unix_socket_path[SOCKET_PATH_MAX];
 101
 102    strcpy(server.args.rdma_dev_name, "");
 103    strcpy(unix_socket_path, UNIX_SOCKET_PATH);
 104    server.args.rdma_port_num = RDMA_PORT_NUM;
 105
 106    while ((c = getopt(argc, argv, "hs:d:p:")) != -1) {
 107        switch (c) {
 108        case 'h':
 109            usage(argv[0]);
 110            exit(0);
 111
 112        case 'd':
 113            strncpy(server.args.rdma_dev_name, optarg, NAME_MAX - 1);
 114            break;
 115
 116        case 's':
 117            /* This is temporary, final name will build below */
 118            strncpy(unix_socket_path, optarg, SOCKET_PATH_MAX - 1);
 119            break;
 120
 121        case 'p':
 122            server.args.rdma_port_num = atoi(optarg);
 123            break;
 124
 125        default:
 126            help(argv[0]);
 127            exit(1);
 128        }
 129    }
 130
 131    if (!strcmp(server.args.rdma_dev_name, "")) {
 132        fprintf(stderr, "Missing RDMA device name\n");
 133        help(argv[0]);
 134        exit(1);
 135    }
 136
 137    /* Build unique unix-socket file name */
 138    snprintf(server.args.unix_socket_path, PATH_MAX, "%s-%s-%d",
 139             unix_socket_path, server.args.rdma_dev_name,
 140             server.args.rdma_port_num);
 141
 142    syslog(LOG_INFO, "unix_socket_path=%s", server.args.unix_socket_path);
 143    syslog(LOG_INFO, "rdma-device-name=%s", server.args.rdma_dev_name);
 144    syslog(LOG_INFO, "rdma-device-port=%d", server.args.rdma_port_num);
 145}
 146
 147static void hash_tbl_alloc(void)
 148{
 149
 150    server.umad_agent.gid2fd = g_hash_table_new_full(g_int64_hash,
 151                                                     g_int64_equal,
 152                                                     g_free, g_free);
 153    server.umad_agent.commid2fd = g_hash_table_new_full(g_int_hash,
 154                                                        g_int_equal,
 155                                                        g_free, g_free);
 156}
 157
 158static void hash_tbl_free(void)
 159{
 160    if (server.umad_agent.commid2fd) {
 161        g_hash_table_destroy(server.umad_agent.commid2fd);
 162    }
 163    if (server.umad_agent.gid2fd) {
 164        g_hash_table_destroy(server.umad_agent.gid2fd);
 165    }
 166}
 167
 168
 169static int _hash_tbl_search_fd_by_ifid(__be64 *gid_ifid)
 170{
 171    int *fd;
 172
 173    fd = g_hash_table_lookup(server.umad_agent.gid2fd, gid_ifid);
 174    if (!fd) {
 175        /* Let's try IPv4 */
 176        *gid_ifid |= 0x00000000ffff0000;
 177        fd = g_hash_table_lookup(server.umad_agent.gid2fd, gid_ifid);
 178    }
 179
 180    return fd ? *fd : 0;
 181}
 182
 183static int hash_tbl_search_fd_by_ifid(int *fd, __be64 *gid_ifid)
 184{
 185    pthread_rwlock_rdlock(&server.lock);
 186    *fd = _hash_tbl_search_fd_by_ifid(gid_ifid);
 187    pthread_rwlock_unlock(&server.lock);
 188
 189    if (!fd) {
 190        syslog(LOG_WARNING, "Can't find matching for ifid 0x%llx\n", *gid_ifid);
 191        return -ENOENT;
 192    }
 193
 194    return 0;
 195}
 196
 197static int hash_tbl_search_fd_by_comm_id(uint32_t comm_id, int *fd,
 198                                         __be64 *gid_idid)
 199{
 200    CommId2FdEntry *fde;
 201
 202    pthread_rwlock_rdlock(&server.lock);
 203    fde = g_hash_table_lookup(server.umad_agent.commid2fd, &comm_id);
 204    pthread_rwlock_unlock(&server.lock);
 205
 206    if (!fde) {
 207        syslog(LOG_WARNING, "Can't find matching for comm_id 0x%x\n", comm_id);
 208        return -ENOENT;
 209    }
 210
 211    *fd = fde->fd;
 212    *gid_idid = fde->gid_ifid;
 213
 214    return 0;
 215}
 216
 217static RdmaCmMuxErrCode add_fd_ifid_pair(int fd, __be64 gid_ifid)
 218{
 219    int fd1;
 220
 221    pthread_rwlock_wrlock(&server.lock);
 222
 223    fd1 = _hash_tbl_search_fd_by_ifid(&gid_ifid);
 224    if (fd1) { /* record already exist - an error */
 225        pthread_rwlock_unlock(&server.lock);
 226        return fd == fd1 ? RDMACM_MUX_ERR_CODE_EEXIST :
 227                           RDMACM_MUX_ERR_CODE_EACCES;
 228    }
 229
 230    g_hash_table_insert(server.umad_agent.gid2fd, g_memdup(&gid_ifid,
 231                        sizeof(gid_ifid)), g_memdup(&fd, sizeof(fd)));
 232
 233    pthread_rwlock_unlock(&server.lock);
 234
 235    syslog(LOG_INFO, "0x%lx registered on socket %d",
 236           be64toh((uint64_t)gid_ifid), fd);
 237
 238    return RDMACM_MUX_ERR_CODE_OK;
 239}
 240
 241static RdmaCmMuxErrCode delete_fd_ifid_pair(int fd, __be64 gid_ifid)
 242{
 243    int fd1;
 244
 245    pthread_rwlock_wrlock(&server.lock);
 246
 247    fd1 = _hash_tbl_search_fd_by_ifid(&gid_ifid);
 248    if (!fd1) { /* record not exist - an error */
 249        pthread_rwlock_unlock(&server.lock);
 250        return RDMACM_MUX_ERR_CODE_ENOTFOUND;
 251    }
 252
 253    g_hash_table_remove(server.umad_agent.gid2fd, g_memdup(&gid_ifid,
 254                        sizeof(gid_ifid)));
 255    pthread_rwlock_unlock(&server.lock);
 256
 257    syslog(LOG_INFO, "0x%lx unregistered on socket %d",
 258           be64toh((uint64_t)gid_ifid), fd);
 259
 260    return RDMACM_MUX_ERR_CODE_OK;
 261}
 262
 263static void hash_tbl_save_fd_comm_id_pair(int fd, uint32_t comm_id,
 264                                          uint64_t gid_ifid)
 265{
 266    CommId2FdEntry fde = {fd, COMMID_TTL, gid_ifid};
 267
 268    pthread_rwlock_wrlock(&server.lock);
 269    g_hash_table_insert(server.umad_agent.commid2fd,
 270                        g_memdup(&comm_id, sizeof(comm_id)),
 271                        g_memdup(&fde, sizeof(fde)));
 272    pthread_rwlock_unlock(&server.lock);
 273}
 274
 275static gboolean remove_old_comm_ids(gpointer key, gpointer value,
 276                                    gpointer user_data)
 277{
 278    CommId2FdEntry *fde = (CommId2FdEntry *)value;
 279
 280    return !fde->ttl--;
 281}
 282
 283static gboolean remove_entry_from_gid2fd(gpointer key, gpointer value,
 284                                         gpointer user_data)
 285{
 286    if (*(int *)value == *(int *)user_data) {
 287        syslog(LOG_INFO, "0x%lx unregistered on socket %d",
 288               be64toh(*(uint64_t *)key), *(int *)value);
 289        return true;
 290    }
 291
 292    return false;
 293}
 294
 295static void hash_tbl_remove_fd_ifid_pair(int fd)
 296{
 297    pthread_rwlock_wrlock(&server.lock);
 298    g_hash_table_foreach_remove(server.umad_agent.gid2fd,
 299                                remove_entry_from_gid2fd, (gpointer)&fd);
 300    pthread_rwlock_unlock(&server.lock);
 301}
 302
 303static int get_fd(const char *mad, int umad_len, int *fd, __be64 *gid_ifid)
 304{
 305    struct umad_hdr *hdr = (struct umad_hdr *)mad;
 306    char *data = (char *)hdr + sizeof(*hdr);
 307    int32_t comm_id = 0;
 308    uint16_t attr_id = be16toh(hdr->attr_id);
 309    int rc = 0;
 310
 311    if (umad_len <= sizeof(*hdr)) {
 312        rc = -EINVAL;
 313        syslog(LOG_DEBUG, "Ignoring MAD packets with header only\n");
 314        goto out;
 315    }
 316
 317    switch (attr_id) {
 318    case UMAD_CM_ATTR_REQ:
 319        if (unlikely(umad_len < sizeof(*hdr) + CM_REQ_DGID_POS +
 320            sizeof(*gid_ifid))) {
 321            rc = -EINVAL;
 322            syslog(LOG_WARNING,
 323                   "Invalid MAD packet size (%d) for attr_id 0x%x\n", umad_len,
 324                    attr_id);
 325            goto out;
 326        }
 327        memcpy(gid_ifid, data + CM_REQ_DGID_POS, sizeof(*gid_ifid));
 328        rc = hash_tbl_search_fd_by_ifid(fd, gid_ifid);
 329        break;
 330
 331    case UMAD_CM_ATTR_SIDR_REQ:
 332        if (unlikely(umad_len < sizeof(*hdr) + CM_SIDR_REQ_DGID_POS +
 333            sizeof(*gid_ifid))) {
 334            rc = -EINVAL;
 335            syslog(LOG_WARNING,
 336                   "Invalid MAD packet size (%d) for attr_id 0x%x\n", umad_len,
 337                    attr_id);
 338            goto out;
 339        }
 340        memcpy(gid_ifid, data + CM_SIDR_REQ_DGID_POS, sizeof(*gid_ifid));
 341        rc = hash_tbl_search_fd_by_ifid(fd, gid_ifid);
 342        break;
 343
 344    case UMAD_CM_ATTR_REP:
 345        /* Fall through */
 346    case UMAD_CM_ATTR_REJ:
 347        /* Fall through */
 348    case UMAD_CM_ATTR_DREQ:
 349        /* Fall through */
 350    case UMAD_CM_ATTR_DREP:
 351        /* Fall through */
 352    case UMAD_CM_ATTR_RTU:
 353        data += sizeof(comm_id);
 354        /* Fall through */
 355    case UMAD_CM_ATTR_SIDR_REP:
 356        if (unlikely(umad_len < sizeof(*hdr) + sizeof(comm_id))) {
 357            rc = -EINVAL;
 358            syslog(LOG_WARNING,
 359                   "Invalid MAD packet size (%d) for attr_id 0x%x\n", umad_len,
 360                   attr_id);
 361            goto out;
 362        }
 363        memcpy(&comm_id, data, sizeof(comm_id));
 364        if (comm_id) {
 365            rc = hash_tbl_search_fd_by_comm_id(comm_id, fd, gid_ifid);
 366        }
 367        break;
 368
 369    default:
 370        rc = -EINVAL;
 371        syslog(LOG_WARNING, "Unsupported attr_id 0x%x\n", attr_id);
 372    }
 373
 374    syslog(LOG_DEBUG, "mad_to_vm: %d 0x%x 0x%x\n", *fd, attr_id, comm_id);
 375
 376out:
 377    return rc;
 378}
 379
 380static void *umad_recv_thread_func(void *args)
 381{
 382    int rc;
 383    RdmaCmMuxMsg msg = {};
 384    int fd = -2;
 385
 386    msg.hdr.msg_type = RDMACM_MUX_MSG_TYPE_REQ;
 387    msg.hdr.op_code = RDMACM_MUX_OP_CODE_MAD;
 388
 389    while (server.run) {
 390        do {
 391            msg.umad_len = sizeof(msg.umad.mad);
 392            rc = umad_recv(server.umad_agent.port_id, &msg.umad, &msg.umad_len,
 393                           SLEEP_SECS * SCALE_US);
 394            if ((rc == -EIO) || (rc == -EINVAL)) {
 395                syslog(LOG_CRIT, "Fatal error while trying to read MAD");
 396            }
 397
 398            if (rc == -ETIMEDOUT) {
 399                g_hash_table_foreach_remove(server.umad_agent.commid2fd,
 400                                            remove_old_comm_ids, NULL);
 401            }
 402        } while (rc && server.run);
 403
 404        if (server.run) {
 405            rc = get_fd(msg.umad.mad, msg.umad_len, &fd,
 406                        &msg.hdr.sgid.global.interface_id);
 407            if (rc) {
 408                continue;
 409            }
 410
 411            send(fd, &msg, sizeof(msg), 0);
 412        }
 413    }
 414
 415    return NULL;
 416}
 417
 418static int read_and_process(int fd)
 419{
 420    int rc;
 421    RdmaCmMuxMsg msg = {};
 422    struct umad_hdr *hdr;
 423    uint32_t *comm_id = 0;
 424    uint16_t attr_id;
 425
 426    rc = recv(fd, &msg, sizeof(msg), 0);
 427    syslog(LOG_DEBUG, "Socket %d, recv %d\n", fd, rc);
 428
 429    if (rc < 0 && errno != EWOULDBLOCK) {
 430        syslog(LOG_ERR, "Fail to read from socket %d\n", fd);
 431        return -EIO;
 432    }
 433
 434    if (!rc) {
 435        syslog(LOG_ERR, "Fail to read from socket %d\n", fd);
 436        return -EPIPE;
 437    }
 438
 439    if (msg.hdr.msg_type != RDMACM_MUX_MSG_TYPE_REQ) {
 440        syslog(LOG_WARNING, "Got non-request message (%d) from socket %d\n",
 441               msg.hdr.msg_type, fd);
 442        return -EPERM;
 443    }
 444
 445    switch (msg.hdr.op_code) {
 446    case RDMACM_MUX_OP_CODE_REG:
 447        rc = add_fd_ifid_pair(fd, msg.hdr.sgid.global.interface_id);
 448        break;
 449
 450    case RDMACM_MUX_OP_CODE_UNREG:
 451        rc = delete_fd_ifid_pair(fd, msg.hdr.sgid.global.interface_id);
 452        break;
 453
 454    case RDMACM_MUX_OP_CODE_MAD:
 455        /* If this is REQ or REP then store the pair comm_id,fd to be later
 456         * used for other messages where gid is unknown */
 457        hdr = (struct umad_hdr *)msg.umad.mad;
 458        attr_id = be16toh(hdr->attr_id);
 459        if ((attr_id == UMAD_CM_ATTR_REQ) || (attr_id == UMAD_CM_ATTR_DREQ) ||
 460            (attr_id == UMAD_CM_ATTR_SIDR_REQ) ||
 461            (attr_id == UMAD_CM_ATTR_REP) || (attr_id == UMAD_CM_ATTR_DREP)) {
 462            comm_id = (uint32_t *)(msg.umad.mad + sizeof(*hdr));
 463            hash_tbl_save_fd_comm_id_pair(fd, *comm_id,
 464                                          msg.hdr.sgid.global.interface_id);
 465        }
 466
 467        syslog(LOG_DEBUG, "vm_to_mad: %d 0x%x 0x%x\n", fd, attr_id,
 468               comm_id ? *comm_id : 0);
 469        rc = umad_send(server.umad_agent.port_id, server.umad_agent.agent_id,
 470                       &msg.umad, msg.umad_len, 1, 0);
 471        if (rc) {
 472            syslog(LOG_ERR,
 473                  "Fail to send MAD message (0x%x) from socket %d, err=%d",
 474                  attr_id, fd, rc);
 475        }
 476        break;
 477
 478    default:
 479        syslog(LOG_ERR, "Got invalid op_code (%d) from socket %d",
 480               msg.hdr.msg_type, fd);
 481        rc = RDMACM_MUX_ERR_CODE_EINVAL;
 482    }
 483
 484    msg.hdr.msg_type = RDMACM_MUX_MSG_TYPE_RESP;
 485    msg.hdr.err_code = rc;
 486    rc = send(fd, &msg, sizeof(msg), 0);
 487
 488    return rc == sizeof(msg) ? 0 : -EPIPE;
 489}
 490
 491static int accept_all(void)
 492{
 493    int fd, rc = 0;;
 494
 495    pthread_rwlock_wrlock(&server.lock);
 496
 497    do {
 498        if ((server.nfds + 1) > MAX_CLIENTS) {
 499            syslog(LOG_WARNING, "Too many clients (%d)", server.nfds);
 500            rc = -EIO;
 501            goto out;
 502        }
 503
 504        fd = accept(server.fds[0].fd, NULL, NULL);
 505        if (fd < 0) {
 506            if (errno != EWOULDBLOCK) {
 507                syslog(LOG_WARNING, "accept() failed");
 508                rc = -EIO;
 509                goto out;
 510            }
 511            break;
 512        }
 513
 514        syslog(LOG_INFO, "Client connected on socket %d\n", fd);
 515        server.fds[server.nfds].fd = fd;
 516        server.fds[server.nfds].events = POLLIN;
 517        server.nfds++;
 518    } while (fd != -1);
 519
 520out:
 521    pthread_rwlock_unlock(&server.lock);
 522    return rc;
 523}
 524
 525static void compress_fds(void)
 526{
 527    int i, j;
 528    int closed = 0;
 529
 530    pthread_rwlock_wrlock(&server.lock);
 531
 532    for (i = 1; i < server.nfds; i++) {
 533        if (!server.fds[i].fd) {
 534            closed++;
 535            for (j = i; j < server.nfds - 1; j++) {
 536                server.fds[j] = server.fds[j + 1];
 537            }
 538        }
 539    }
 540
 541    server.nfds -= closed;
 542
 543    pthread_rwlock_unlock(&server.lock);
 544}
 545
 546static void close_fd(int idx)
 547{
 548    close(server.fds[idx].fd);
 549    syslog(LOG_INFO, "Socket %d closed\n", server.fds[idx].fd);
 550    hash_tbl_remove_fd_ifid_pair(server.fds[idx].fd);
 551    server.fds[idx].fd = 0;
 552}
 553
 554static void run(void)
 555{
 556    int rc, nfds, i;
 557    bool compress = false;
 558
 559    syslog(LOG_INFO, "Service started");
 560
 561    while (server.run) {
 562        rc = poll(server.fds, server.nfds, SLEEP_SECS * SCALE_US);
 563        if (rc < 0) {
 564            if (errno != EINTR) {
 565                syslog(LOG_WARNING, "poll() failed");
 566            }
 567            continue;
 568        }
 569
 570        if (rc == 0) {
 571            continue;
 572        }
 573
 574        nfds = server.nfds;
 575        for (i = 0; i < nfds; i++) {
 576            syslog(LOG_DEBUG, "pollfd[%d]: revents 0x%x, events 0x%x\n", i,
 577                   server.fds[i].revents, server.fds[i].events);
 578            if (server.fds[i].revents == 0) {
 579                continue;
 580            }
 581
 582            if (server.fds[i].revents != POLLIN) {
 583                if (i == 0) {
 584                    syslog(LOG_NOTICE, "Unexpected poll() event (0x%x)\n",
 585                           server.fds[i].revents);
 586                } else {
 587                    close_fd(i);
 588                    compress = true;
 589                }
 590                continue;
 591            }
 592
 593            if (i == 0) {
 594                rc = accept_all();
 595                if (rc) {
 596                    continue;
 597                }
 598            } else {
 599                rc = read_and_process(server.fds[i].fd);
 600                if (rc) {
 601                    close_fd(i);
 602                    compress = true;
 603                }
 604            }
 605        }
 606
 607        if (compress) {
 608            compress = false;
 609            compress_fds();
 610        }
 611    }
 612}
 613
 614static void fini_listener(void)
 615{
 616    int i;
 617
 618    if (server.fds[0].fd <= 0) {
 619        return;
 620    }
 621
 622    for (i = server.nfds - 1; i >= 0; i--) {
 623        if (server.fds[i].fd) {
 624            close(server.fds[i].fd);
 625        }
 626    }
 627
 628    unlink(server.args.unix_socket_path);
 629}
 630
 631static void fini_umad(void)
 632{
 633    if (server.umad_agent.agent_id) {
 634        umad_unregister(server.umad_agent.port_id, server.umad_agent.agent_id);
 635    }
 636
 637    if (server.umad_agent.port_id) {
 638        umad_close_port(server.umad_agent.port_id);
 639    }
 640
 641    hash_tbl_free();
 642}
 643
 644static void fini(void)
 645{
 646    if (server.umad_recv_thread) {
 647        pthread_join(server.umad_recv_thread, NULL);
 648        server.umad_recv_thread = 0;
 649    }
 650    fini_umad();
 651    fini_listener();
 652    pthread_rwlock_destroy(&server.lock);
 653
 654    syslog(LOG_INFO, "Service going down");
 655}
 656
 657static int init_listener(void)
 658{
 659    struct sockaddr_un sun;
 660    int rc, on = 1;
 661
 662    server.fds[0].fd = socket(AF_UNIX, SOCK_STREAM, 0);
 663    if (server.fds[0].fd < 0) {
 664        syslog(LOG_ALERT, "socket() failed");
 665        return -EIO;
 666    }
 667
 668    rc = setsockopt(server.fds[0].fd, SOL_SOCKET, SO_REUSEADDR, (char *)&on,
 669                    sizeof(on));
 670    if (rc < 0) {
 671        syslog(LOG_ALERT, "setsockopt() failed");
 672        rc = -EIO;
 673        goto err;
 674    }
 675
 676    rc = ioctl(server.fds[0].fd, FIONBIO, (char *)&on);
 677    if (rc < 0) {
 678        syslog(LOG_ALERT, "ioctl() failed");
 679        rc = -EIO;
 680        goto err;
 681    }
 682
 683    if (strlen(server.args.unix_socket_path) >= sizeof(sun.sun_path)) {
 684        syslog(LOG_ALERT,
 685               "Invalid unix_socket_path, size must be less than %ld\n",
 686               sizeof(sun.sun_path));
 687        rc = -EINVAL;
 688        goto err;
 689    }
 690
 691    sun.sun_family = AF_UNIX;
 692    rc = snprintf(sun.sun_path, sizeof(sun.sun_path), "%s",
 693                  server.args.unix_socket_path);
 694    if (rc < 0 || rc >= sizeof(sun.sun_path)) {
 695        syslog(LOG_ALERT, "Could not copy unix socket path\n");
 696        rc = -EINVAL;
 697        goto err;
 698    }
 699
 700    rc = bind(server.fds[0].fd, (struct sockaddr *)&sun, sizeof(sun));
 701    if (rc < 0) {
 702        syslog(LOG_ALERT, "bind() failed");
 703        rc = -EIO;
 704        goto err;
 705    }
 706
 707    rc = listen(server.fds[0].fd, SERVER_LISTEN_BACKLOG);
 708    if (rc < 0) {
 709        syslog(LOG_ALERT, "listen() failed");
 710        rc = -EIO;
 711        goto err;
 712    }
 713
 714    server.fds[0].events = POLLIN;
 715    server.nfds = 1;
 716    server.run = true;
 717
 718    return 0;
 719
 720err:
 721    close(server.fds[0].fd);
 722    return rc;
 723}
 724
 725static int init_umad(void)
 726{
 727    long method_mask[IB_USER_MAD_LONGS_PER_METHOD_MASK];
 728
 729    server.umad_agent.port_id = umad_open_port(server.args.rdma_dev_name,
 730                                               server.args.rdma_port_num);
 731
 732    if (server.umad_agent.port_id < 0) {
 733        syslog(LOG_WARNING, "umad_open_port() failed");
 734        return -EIO;
 735    }
 736
 737    memset(&method_mask, 0, sizeof(method_mask));
 738    method_mask[0] = MAD_METHOD_MASK0;
 739    server.umad_agent.agent_id = umad_register(server.umad_agent.port_id,
 740                                               UMAD_CLASS_CM,
 741                                               UMAD_SA_CLASS_VERSION,
 742                                               MAD_RMPP_VERSION, method_mask);
 743    if (server.umad_agent.agent_id < 0) {
 744        syslog(LOG_WARNING, "umad_register() failed");
 745        return -EIO;
 746    }
 747
 748    hash_tbl_alloc();
 749
 750    return 0;
 751}
 752
 753static void signal_handler(int sig, siginfo_t *siginfo, void *context)
 754{
 755    static bool warned;
 756
 757    /* Prevent stop if clients are connected */
 758    if (server.nfds != 1) {
 759        if (!warned) {
 760            syslog(LOG_WARNING,
 761                   "Can't stop while active client exist, resend SIGINT to overid");
 762            warned = true;
 763            return;
 764        }
 765    }
 766
 767    if (sig == SIGINT) {
 768        server.run = false;
 769        fini();
 770    }
 771
 772    exit(0);
 773}
 774
 775static int init(void)
 776{
 777    int rc;
 778    struct sigaction sig = {};
 779
 780    rc = init_listener();
 781    if (rc) {
 782        return rc;
 783    }
 784
 785    rc = init_umad();
 786    if (rc) {
 787        return rc;
 788    }
 789
 790    pthread_rwlock_init(&server.lock, 0);
 791
 792    rc = pthread_create(&server.umad_recv_thread, NULL, umad_recv_thread_func,
 793                        NULL);
 794    if (rc) {
 795        syslog(LOG_ERR, "Fail to create UMAD receiver thread (%d)\n", rc);
 796        return rc;
 797    }
 798
 799    sig.sa_sigaction = &signal_handler;
 800    sig.sa_flags = SA_SIGINFO;
 801    rc = sigaction(SIGINT, &sig, NULL);
 802    if (rc < 0) {
 803        syslog(LOG_ERR, "Fail to install SIGINT handler (%d)\n", errno);
 804        return rc;
 805    }
 806
 807    return 0;
 808}
 809
 810int main(int argc, char *argv[])
 811{
 812    int rc;
 813
 814    memset(&server, 0, sizeof(server));
 815
 816    parse_args(argc, argv);
 817
 818    rc = init();
 819    if (rc) {
 820        syslog(LOG_ERR, "Fail to initialize server (%d)\n", rc);
 821        rc = -EAGAIN;
 822        goto out;
 823    }
 824
 825    run();
 826
 827out:
 828    fini();
 829
 830    return rc;
 831}
 832