LXR linux/drivers/misc/mic/scif/scif

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Intel MIC Platform Software Stack (MPSS)
   4 *
   5 * Copyright(c) 2015 Intel Corporation.
   6 *
   7 * Intel SCIF driver.
   8 */
   9
  10#include "scif_main.h"
  11
  12/**
  13 * scif_recv_mark: Handle SCIF_MARK request
  14 * @msg:        Interrupt message
  15 *
  16 * The peer has requested a mark.
  17 */
  18void scif_recv_mark(struct scif_dev *scifdev, struct scifmsg *msg)
  19{
  20        struct scif_endpt *ep = (struct scif_endpt *)msg->payload[0];
  21        int mark = 0;
  22        int err;
  23
  24        err = _scif_fence_mark(ep, &mark);
  25        if (err)
  26                msg->uop = SCIF_MARK_NACK;
  27        else
  28                msg->uop = SCIF_MARK_ACK;
  29        msg->payload[0] = ep->remote_ep;
  30        msg->payload[2] = mark;
  31        scif_nodeqp_send(ep->remote_dev, msg);
  32}
  33
  34/**
  35 * scif_recv_mark_resp: Handle SCIF_MARK_(N)ACK messages.
  36 * @msg:        Interrupt message
  37 *
  38 * The peer has responded to a SCIF_MARK message.
  39 */
  40void scif_recv_mark_resp(struct scif_dev *scifdev, struct scifmsg *msg)
  41{
  42        struct scif_endpt *ep = (struct scif_endpt *)msg->payload[0];
  43        struct scif_fence_info *fence_req =
  44                (struct scif_fence_info *)msg->payload[1];
  45
  46        mutex_lock(&ep->rma_info.rma_lock);
  47        if (msg->uop == SCIF_MARK_ACK) {
  48                fence_req->state = OP_COMPLETED;
  49                fence_req->dma_mark = (int)msg->payload[2];
  50        } else {
  51                fence_req->state = OP_FAILED;
  52        }
  53        mutex_unlock(&ep->rma_info.rma_lock);
  54        complete(&fence_req->comp);
  55}
  56
  57/**
  58 * scif_recv_wait: Handle SCIF_WAIT request
  59 * @msg:        Interrupt message
  60 *
  61 * The peer has requested waiting on a fence.
  62 */
  63void scif_recv_wait(struct scif_dev *scifdev, struct scifmsg *msg)
  64{
  65        struct scif_endpt *ep = (struct scif_endpt *)msg->payload[0];
  66        struct scif_remote_fence_info *fence;
  67
  68        /*
  69         * Allocate structure for remote fence information and
  70         * send a NACK if the allocation failed. The peer will
  71         * return ENOMEM upon receiving a NACK.
  72         */
  73        fence = kmalloc(sizeof(*fence), GFP_KERNEL);
  74        if (!fence) {
  75                msg->payload[0] = ep->remote_ep;
  76                msg->uop = SCIF_WAIT_NACK;
  77                scif_nodeqp_send(ep->remote_dev, msg);
  78                return;
  79        }
  80
  81        /* Prepare the fence request */
  82        memcpy(&fence->msg, msg, sizeof(struct scifmsg));
  83        INIT_LIST_HEAD(&fence->list);
  84
  85        /* Insert to the global remote fence request list */
  86        mutex_lock(&scif_info.fencelock);
  87        atomic_inc(&ep->rma_info.fence_refcount);
  88        list_add_tail(&fence->list, &scif_info.fence);
  89        mutex_unlock(&scif_info.fencelock);
  90
  91        schedule_work(&scif_info.misc_work);
  92}
  93
  94/**
  95 * scif_recv_wait_resp: Handle SCIF_WAIT_(N)ACK messages.
  96 * @msg:        Interrupt message
  97 *
  98 * The peer has responded to a SCIF_WAIT message.
  99 */
 100void scif_recv_wait_resp(struct scif_dev *scifdev, struct scifmsg *msg)
 101{
 102        struct scif_endpt *ep = (struct scif_endpt *)msg->payload[0];
 103        struct scif_fence_info *fence_req =
 104                (struct scif_fence_info *)msg->payload[1];
 105
 106        mutex_lock(&ep->rma_info.rma_lock);
 107        if (msg->uop == SCIF_WAIT_ACK)
 108                fence_req->state = OP_COMPLETED;
 109        else
 110                fence_req->state = OP_FAILED;
 111        mutex_unlock(&ep->rma_info.rma_lock);
 112        complete(&fence_req->comp);
 113}
 114
 115/**
 116 * scif_recv_sig_local: Handle SCIF_SIG_LOCAL request
 117 * @msg:        Interrupt message
 118 *
 119 * The peer has requested a signal on a local offset.
 120 */
 121void scif_recv_sig_local(struct scif_dev *scifdev, struct scifmsg *msg)
 122{
 123        struct scif_endpt *ep = (struct scif_endpt *)msg->payload[0];
 124        int err;
 125
 126        err = scif_prog_signal(ep, msg->payload[1], msg->payload[2],
 127                               SCIF_WINDOW_SELF);
 128        if (err)
 129                msg->uop = SCIF_SIG_NACK;
 130        else
 131                msg->uop = SCIF_SIG_ACK;
 132        msg->payload[0] = ep->remote_ep;
 133        scif_nodeqp_send(ep->remote_dev, msg);
 134}
 135
 136/**
 137 * scif_recv_sig_remote: Handle SCIF_SIGNAL_REMOTE request
 138 * @msg:        Interrupt message
 139 *
 140 * The peer has requested a signal on a remote offset.
 141 */
 142void scif_recv_sig_remote(struct scif_dev *scifdev, struct scifmsg *msg)
 143{
 144        struct scif_endpt *ep = (struct scif_endpt *)msg->payload[0];
 145        int err;
 146
 147        err = scif_prog_signal(ep, msg->payload[1], msg->payload[2],
 148                               SCIF_WINDOW_PEER);
 149        if (err)
 150                msg->uop = SCIF_SIG_NACK;
 151        else
 152                msg->uop = SCIF_SIG_ACK;
 153        msg->payload[0] = ep->remote_ep;
 154        scif_nodeqp_send(ep->remote_dev, msg);
 155}
 156
 157/**
 158 * scif_recv_sig_resp: Handle SCIF_SIG_(N)ACK messages.
 159 * @msg:        Interrupt message
 160 *
 161 * The peer has responded to a signal request.
 162 */
 163void scif_recv_sig_resp(struct scif_dev *scifdev, struct scifmsg *msg)
 164{
 165        struct scif_endpt *ep = (struct scif_endpt *)msg->payload[0];
 166        struct scif_fence_info *fence_req =
 167                (struct scif_fence_info *)msg->payload[3];
 168
 169        mutex_lock(&ep->rma_info.rma_lock);
 170        if (msg->uop == SCIF_SIG_ACK)
 171                fence_req->state = OP_COMPLETED;
 172        else
 173                fence_req->state = OP_FAILED;
 174        mutex_unlock(&ep->rma_info.rma_lock);
 175        complete(&fence_req->comp);
 176}
 177
 178static inline void *scif_get_local_va(off_t off, struct scif_window *window)
 179{
 180        struct page **pages = window->pinned_pages->pages;
 181        int page_nr = (off - window->offset) >> PAGE_SHIFT;
 182        off_t page_off = off & ~PAGE_MASK;
 183
 184        return page_address(pages[page_nr]) + page_off;
 185}
 186
 187static void scif_prog_signal_cb(void *arg)
 188{
 189        struct scif_cb_arg *cb_arg = arg;
 190
 191        dma_pool_free(cb_arg->ep->remote_dev->signal_pool, cb_arg->status,
 192                      cb_arg->src_dma_addr);
 193        kfree(cb_arg);
 194}
 195
 196static int _scif_prog_signal(scif_epd_t epd, dma_addr_t dst, u64 val)
 197{
 198        struct scif_endpt *ep = (struct scif_endpt *)epd;
 199        struct dma_chan *chan = ep->rma_info.dma_chan;
 200        struct dma_device *ddev = chan->device;
 201        bool x100 = !is_dma_copy_aligned(chan->device, 1, 1, 1);
 202        struct dma_async_tx_descriptor *tx;
 203        struct scif_status *status = NULL;
 204        struct scif_cb_arg *cb_arg = NULL;
 205        dma_addr_t src;
 206        dma_cookie_t cookie;
 207        int err;
 208
 209        tx = ddev->device_prep_dma_memcpy(chan, 0, 0, 0, DMA_PREP_FENCE);
 210        if (!tx) {
 211                err = -ENOMEM;
 212                dev_err(&ep->remote_dev->sdev->dev, "%s %d err %d\n",
 213                        __func__, __LINE__, err);
 214                goto alloc_fail;
 215        }
 216        cookie = tx->tx_submit(tx);
 217        if (dma_submit_error(cookie)) {
 218                err = (int)cookie;
 219                dev_err(&ep->remote_dev->sdev->dev, "%s %d err %d\n",
 220                        __func__, __LINE__, err);
 221                goto alloc_fail;
 222        }
 223        dma_async_issue_pending(chan);
 224        if (x100) {
 225                /*
 226                 * For X100 use the status descriptor to write the value to
 227                 * the destination.
 228                 */
 229                tx = ddev->device_prep_dma_imm_data(chan, dst, val, 0);
 230        } else {
 231                status = dma_pool_alloc(ep->remote_dev->signal_pool, GFP_KERNEL,
 232                                        &src);
 233                if (!status) {
 234                        err = -ENOMEM;
 235                        dev_err(&ep->remote_dev->sdev->dev, "%s %d err %d\n",
 236                                __func__, __LINE__, err);
 237                        goto alloc_fail;
 238                }
 239                status->val = val;
 240                status->src_dma_addr = src;
 241                status->ep = ep;
 242                src += offsetof(struct scif_status, val);
 243                tx = ddev->device_prep_dma_memcpy(chan, dst, src, sizeof(val),
 244                                                  DMA_PREP_INTERRUPT);
 245        }
 246        if (!tx) {
 247                err = -ENOMEM;
 248                dev_err(&ep->remote_dev->sdev->dev, "%s %d err %d\n",
 249                        __func__, __LINE__, err);
 250                goto dma_fail;
 251        }
 252        if (!x100) {
 253                cb_arg = kmalloc(sizeof(*cb_arg), GFP_KERNEL);
 254                if (!cb_arg) {
 255                        err = -ENOMEM;
 256                        goto dma_fail;
 257                }
 258                cb_arg->src_dma_addr = src;
 259                cb_arg->status = status;
 260                cb_arg->ep = ep;
 261                tx->callback = scif_prog_signal_cb;
 262                tx->callback_param = cb_arg;
 263        }
 264        cookie = tx->tx_submit(tx);
 265        if (dma_submit_error(cookie)) {
 266                err = -EIO;
 267                dev_err(&ep->remote_dev->sdev->dev, "%s %d err %d\n",
 268                        __func__, __LINE__, err);
 269                goto dma_fail;
 270        }
 271        dma_async_issue_pending(chan);
 272        return 0;
 273dma_fail:
 274        if (!x100) {
 275                dma_pool_free(ep->remote_dev->signal_pool, status,
 276                              src - offsetof(struct scif_status, val));
 277                kfree(cb_arg);
 278        }
 279alloc_fail:
 280        return err;
 281}
 282
 283/*
 284 * scif_prog_signal:
 285 * @epd - Endpoint Descriptor
 286 * @offset - registered address to write @val to
 287 * @val - Value to be written at @offset
 288 * @type - Type of the window.
 289 *
 290 * Arrange to write a value to the registered offset after ensuring that the
 291 * offset provided is indeed valid.
 292 */
 293int scif_prog_signal(scif_epd_t epd, off_t offset, u64 val,
 294                     enum scif_window_type type)
 295{
 296        struct scif_endpt *ep = (struct scif_endpt *)epd;
 297        struct scif_window *window = NULL;
 298        struct scif_rma_req req;
 299        dma_addr_t dst_dma_addr;
 300        int err;
 301
 302        mutex_lock(&ep->rma_info.rma_lock);
 303        req.out_window = &window;
 304        req.offset = offset;
 305        req.nr_bytes = sizeof(u64);
 306        req.prot = SCIF_PROT_WRITE;
 307        req.type = SCIF_WINDOW_SINGLE;
 308        if (type == SCIF_WINDOW_SELF)
 309                req.head = &ep->rma_info.reg_list;
 310        else
 311                req.head = &ep->rma_info.remote_reg_list;
 312        /* Does a valid window exist? */
 313        err = scif_query_window(&req);
 314        if (err) {
 315                dev_err(scif_info.mdev.this_device,
 316                        "%s %d err %d\n", __func__, __LINE__, err);
 317                goto unlock_ret;
 318        }
 319
 320        if (scif_is_mgmt_node() && scifdev_self(ep->remote_dev)) {
 321                u64 *dst_virt;
 322
 323                if (type == SCIF_WINDOW_SELF)
 324                        dst_virt = scif_get_local_va(offset, window);
 325                else
 326                        dst_virt =
 327                        scif_get_local_va(offset, (struct scif_window *)
 328                                          window->peer_window);
 329                *dst_virt = val;
 330        } else {
 331                dst_dma_addr = __scif_off_to_dma_addr(window, offset);
 332                err = _scif_prog_signal(epd, dst_dma_addr, val);
 333        }
 334unlock_ret:
 335        mutex_unlock(&ep->rma_info.rma_lock);
 336        return err;
 337}
 338
 339static int _scif_fence_wait(scif_epd_t epd, int mark)
 340{
 341        struct scif_endpt *ep = (struct scif_endpt *)epd;
 342        dma_cookie_t cookie = mark & ~SCIF_REMOTE_FENCE;
 343        int err;
 344
 345        /* Wait for DMA callback in scif_fence_mark_cb(..) */
 346        err = wait_event_interruptible_timeout(ep->rma_info.markwq,
 347                                               dma_async_is_tx_complete(
 348                                               ep->rma_info.dma_chan,
 349                                               cookie, NULL, NULL) ==
 350                                               DMA_COMPLETE,
 351                                               SCIF_NODE_ALIVE_TIMEOUT);
 352        if (!err)
 353                err = -ETIMEDOUT;
 354        else if (err > 0)
 355                err = 0;
 356        return err;
 357}
 358
 359/**
 360 * scif_rma_handle_remote_fences:
 361 *
 362 * This routine services remote fence requests.
 363 */
 364void scif_rma_handle_remote_fences(void)
 365{
 366        struct list_head *item, *tmp;
 367        struct scif_remote_fence_info *fence;
 368        struct scif_endpt *ep;
 369        int mark, err;
 370
 371        might_sleep();
 372        mutex_lock(&scif_info.fencelock);
 373        list_for_each_safe(item, tmp, &scif_info.fence) {
 374                fence = list_entry(item, struct scif_remote_fence_info,
 375                                   list);
 376                /* Remove fence from global list */
 377                list_del(&fence->list);
 378
 379                /* Initiate the fence operation */
 380                ep = (struct scif_endpt *)fence->msg.payload[0];
 381                mark = fence->msg.payload[2];
 382                err = _scif_fence_wait(ep, mark);
 383                if (err)
 384                        fence->msg.uop = SCIF_WAIT_NACK;
 385                else
 386                        fence->msg.uop = SCIF_WAIT_ACK;
 387                fence->msg.payload[0] = ep->remote_ep;
 388                scif_nodeqp_send(ep->remote_dev, &fence->msg);
 389                kfree(fence);
 390                if (!atomic_sub_return(1, &ep->rma_info.fence_refcount))
 391                        schedule_work(&scif_info.misc_work);
 392        }
 393        mutex_unlock(&scif_info.fencelock);
 394}
 395
 396static int _scif_send_fence(scif_epd_t epd, int uop, int mark, int *out_mark)
 397{
 398        int err;
 399        struct scifmsg msg;
 400        struct scif_fence_info *fence_req;
 401        struct scif_endpt *ep = (struct scif_endpt *)epd;
 402
 403        fence_req = kmalloc(sizeof(*fence_req), GFP_KERNEL);
 404        if (!fence_req) {
 405                err = -ENOMEM;
 406                goto error;
 407        }
 408
 409        fence_req->state = OP_IN_PROGRESS;
 410        init_completion(&fence_req->comp);
 411
 412        msg.src = ep->port;
 413        msg.uop = uop;
 414        msg.payload[0] = ep->remote_ep;
 415        msg.payload[1] = (u64)fence_req;
 416        if (uop == SCIF_WAIT)
 417                msg.payload[2] = mark;
 418        spin_lock(&ep->lock);
 419        if (ep->state == SCIFEP_CONNECTED)
 420                err = scif_nodeqp_send(ep->remote_dev, &msg);
 421        else
 422                err = -ENOTCONN;
 423        spin_unlock(&ep->lock);
 424        if (err)
 425                goto error_free;
 426retry:
 427        /* Wait for a SCIF_WAIT_(N)ACK message */
 428        err = wait_for_completion_timeout(&fence_req->comp,
 429                                          SCIF_NODE_ALIVE_TIMEOUT);
 430        if (!err && scifdev_alive(ep))
 431                goto retry;
 432        if (!err)
 433                err = -ENODEV;
 434        if (err > 0)
 435                err = 0;
 436        mutex_lock(&ep->rma_info.rma_lock);
 437        if (err < 0) {
 438                if (fence_req->state == OP_IN_PROGRESS)
 439                        fence_req->state = OP_FAILED;
 440        }
 441        if (fence_req->state == OP_FAILED && !err)
 442                err = -ENOMEM;
 443        if (uop == SCIF_MARK && fence_req->state == OP_COMPLETED)
 444                *out_mark = SCIF_REMOTE_FENCE | fence_req->dma_mark;
 445        mutex_unlock(&ep->rma_info.rma_lock);
 446error_free:
 447        kfree(fence_req);
 448error:
 449        return err;
 450}
 451
 452/**
 453 * scif_send_fence_mark:
 454 * @epd: end point descriptor.
 455 * @out_mark: Output DMA mark reported by peer.
 456 *
 457 * Send a remote fence mark request.
 458 */
 459static int scif_send_fence_mark(scif_epd_t epd, int *out_mark)
 460{
 461        return _scif_send_fence(epd, SCIF_MARK, 0, out_mark);
 462}
 463
 464/**
 465 * scif_send_fence_wait:
 466 * @epd: end point descriptor.
 467 * @mark: DMA mark to wait for.
 468 *
 469 * Send a remote fence wait request.
 470 */
 471static int scif_send_fence_wait(scif_epd_t epd, int mark)
 472{
 473        return _scif_send_fence(epd, SCIF_WAIT, mark, NULL);
 474}
 475
 476static int _scif_send_fence_signal_wait(struct scif_endpt *ep,
 477                                        struct scif_fence_info *fence_req)
 478{
 479        int err;
 480
 481retry:
 482        /* Wait for a SCIF_SIG_(N)ACK message */
 483        err = wait_for_completion_timeout(&fence_req->comp,
 484                                          SCIF_NODE_ALIVE_TIMEOUT);
 485        if (!err && scifdev_alive(ep))
 486                goto retry;
 487        if (!err)
 488                err = -ENODEV;
 489        if (err > 0)
 490                err = 0;
 491        if (err < 0) {
 492                mutex_lock(&ep->rma_info.rma_lock);
 493                if (fence_req->state == OP_IN_PROGRESS)
 494                        fence_req->state = OP_FAILED;
 495                mutex_unlock(&ep->rma_info.rma_lock);
 496        }
 497        if (fence_req->state == OP_FAILED && !err)
 498                err = -ENXIO;
 499        return err;
 500}
 501
 502/**
 503 * scif_send_fence_signal:
 504 * @epd - endpoint descriptor
 505 * @loff - local offset
 506 * @lval - local value to write to loffset
 507 * @roff - remote offset
 508 * @rval - remote value to write to roffset
 509 * @flags - flags
 510 *
 511 * Sends a remote fence signal request
 512 */
 513static int scif_send_fence_signal(scif_epd_t epd, off_t roff, u64 rval,
 514                                  off_t loff, u64 lval, int flags)
 515{
 516        int err = 0;
 517        struct scifmsg msg;
 518        struct scif_fence_info *fence_req;
 519        struct scif_endpt *ep = (struct scif_endpt *)epd;
 520
 521        fence_req = kmalloc(sizeof(*fence_req), GFP_KERNEL);
 522        if (!fence_req) {
 523                err = -ENOMEM;
 524                goto error;
 525        }
 526
 527        fence_req->state = OP_IN_PROGRESS;
 528        init_completion(&fence_req->comp);
 529        msg.src = ep->port;
 530        if (flags & SCIF_SIGNAL_LOCAL) {
 531                msg.uop = SCIF_SIG_LOCAL;
 532                msg.payload[0] = ep->remote_ep;
 533                msg.payload[1] = roff;
 534                msg.payload[2] = rval;
 535                msg.payload[3] = (u64)fence_req;
 536                spin_lock(&ep->lock);
 537                if (ep->state == SCIFEP_CONNECTED)
 538                        err = scif_nodeqp_send(ep->remote_dev, &msg);
 539                else
 540                        err = -ENOTCONN;
 541                spin_unlock(&ep->lock);
 542                if (err)
 543                        goto error_free;
 544                err = _scif_send_fence_signal_wait(ep, fence_req);
 545                if (err)
 546                        goto error_free;
 547        }
 548        fence_req->state = OP_IN_PROGRESS;
 549
 550        if (flags & SCIF_SIGNAL_REMOTE) {
 551                msg.uop = SCIF_SIG_REMOTE;
 552                msg.payload[0] = ep->remote_ep;
 553                msg.payload[1] = loff;
 554                msg.payload[2] = lval;
 555                msg.payload[3] = (u64)fence_req;
 556                spin_lock(&ep->lock);
 557                if (ep->state == SCIFEP_CONNECTED)
 558                        err = scif_nodeqp_send(ep->remote_dev, &msg);
 559                else
 560                        err = -ENOTCONN;
 561                spin_unlock(&ep->lock);
 562                if (err)
 563                        goto error_free;
 564                err = _scif_send_fence_signal_wait(ep, fence_req);
 565        }
 566error_free:
 567        kfree(fence_req);
 568error:
 569        return err;
 570}
 571
 572static void scif_fence_mark_cb(void *arg)
 573{
 574        struct scif_endpt *ep = (struct scif_endpt *)arg;
 575
 576        wake_up_interruptible(&ep->rma_info.markwq);
 577        atomic_dec(&ep->rma_info.fence_refcount);
 578}
 579
 580/*
 581 * _scif_fence_mark:
 582 *
 583 * @epd - endpoint descriptor
 584 * Set up a mark for this endpoint and return the value of the mark.
 585 */
 586int _scif_fence_mark(scif_epd_t epd, int *mark)
 587{
 588        struct scif_endpt *ep = (struct scif_endpt *)epd;
 589        struct dma_chan *chan = ep->rma_info.dma_chan;
 590        struct dma_device *ddev = chan->device;
 591        struct dma_async_tx_descriptor *tx;
 592        dma_cookie_t cookie;
 593        int err;
 594
 595        tx = ddev->device_prep_dma_memcpy(chan, 0, 0, 0, DMA_PREP_FENCE);
 596        if (!tx) {
 597                err = -ENOMEM;
 598                dev_err(&ep->remote_dev->sdev->dev, "%s %d err %d\n",
 599                        __func__, __LINE__, err);
 600                return err;
 601        }
 602        cookie = tx->tx_submit(tx);
 603        if (dma_submit_error(cookie)) {
 604                err = (int)cookie;
 605                dev_err(&ep->remote_dev->sdev->dev, "%s %d err %d\n",
 606                        __func__, __LINE__, err);
 607                return err;
 608        }
 609        dma_async_issue_pending(chan);
 610        tx = ddev->device_prep_dma_interrupt(chan, DMA_PREP_INTERRUPT);
 611        if (!tx) {
 612                err = -ENOMEM;
 613                dev_err(&ep->remote_dev->sdev->dev, "%s %d err %d\n",
 614                        __func__, __LINE__, err);
 615                return err;
 616        }
 617        tx->callback = scif_fence_mark_cb;
 618        tx->callback_param = ep;
 619        *mark = cookie = tx->tx_submit(tx);
 620        if (dma_submit_error(cookie)) {
 621                err = (int)cookie;
 622                dev_err(&ep->remote_dev->sdev->dev, "%s %d err %d\n",
 623                        __func__, __LINE__, err);
 624                return err;
 625        }
 626        atomic_inc(&ep->rma_info.fence_refcount);
 627        dma_async_issue_pending(chan);
 628        return 0;
 629}
 630
 631#define SCIF_LOOPB_MAGIC_MARK 0xdead
 632
 633int scif_fence_mark(scif_epd_t epd, int flags, int *mark)
 634{
 635        struct scif_endpt *ep = (struct scif_endpt *)epd;
 636        int err = 0;
 637
 638        dev_dbg(scif_info.mdev.this_device,
 639                "SCIFAPI fence_mark: ep %p flags 0x%x mark 0x%x\n",
 640                ep, flags, *mark);
 641        err = scif_verify_epd(ep);
 642        if (err)
 643                return err;
 644
 645        /* Invalid flags? */
 646        if (flags & ~(SCIF_FENCE_INIT_SELF | SCIF_FENCE_INIT_PEER))
 647                return -EINVAL;
 648
 649        /* At least one of init self or peer RMA should be set */
 650        if (!(flags & (SCIF_FENCE_INIT_SELF | SCIF_FENCE_INIT_PEER)))
 651                return -EINVAL;
 652
 653        /* Exactly one of init self or peer RMA should be set but not both */
 654        if ((flags & SCIF_FENCE_INIT_SELF) && (flags & SCIF_FENCE_INIT_PEER))
 655                return -EINVAL;
 656
 657        /*
 658         * Management node loopback does not need to use DMA.
 659         * Return a valid mark to be symmetric.
 660         */
 661        if (scifdev_self(ep->remote_dev) && scif_is_mgmt_node()) {
 662                *mark = SCIF_LOOPB_MAGIC_MARK;
 663                return 0;
 664        }
 665
 666        if (flags & SCIF_FENCE_INIT_SELF)
 667                err = _scif_fence_mark(epd, mark);
 668        else
 669                err = scif_send_fence_mark(ep, mark);
 670
 671        if (err)
 672                dev_err(scif_info.mdev.this_device,
 673                        "%s %d err %d\n", __func__, __LINE__, err);
 674        dev_dbg(scif_info.mdev.this_device,
 675                "SCIFAPI fence_mark: ep %p flags 0x%x mark 0x%x err %d\n",
 676                ep, flags, *mark, err);
 677        return err;
 678}
 679EXPORT_SYMBOL_GPL(scif_fence_mark);
 680
 681int scif_fence_wait(scif_epd_t epd, int mark)
 682{
 683        struct scif_endpt *ep = (struct scif_endpt *)epd;
 684        int err = 0;
 685
 686        dev_dbg(scif_info.mdev.this_device,
 687                "SCIFAPI fence_wait: ep %p mark 0x%x\n",
 688                ep, mark);
 689        err = scif_verify_epd(ep);
 690        if (err)
 691                return err;
 692        /*
 693         * Management node loopback does not need to use DMA.
 694         * The only valid mark provided is 0 so simply
 695         * return success if the mark is valid.
 696         */
 697        if (scifdev_self(ep->remote_dev) && scif_is_mgmt_node()) {
 698                if (mark == SCIF_LOOPB_MAGIC_MARK)
 699                        return 0;
 700                else
 701                        return -EINVAL;
 702        }
 703        if (mark & SCIF_REMOTE_FENCE)
 704                err = scif_send_fence_wait(epd, mark);
 705        else
 706                err = _scif_fence_wait(epd, mark);
 707        if (err < 0)
 708                dev_err(scif_info.mdev.this_device,
 709                        "%s %d err %d\n", __func__, __LINE__, err);
 710        return err;
 711}
 712EXPORT_SYMBOL_GPL(scif_fence_wait);
 713
 714int scif_fence_signal(scif_epd_t epd, off_t loff, u64 lval,
 715                      off_t roff, u64 rval, int flags)
 716{
 717        struct scif_endpt *ep = (struct scif_endpt *)epd;
 718        int err = 0;
 719
 720        dev_dbg(scif_info.mdev.this_device,
 721                "SCIFAPI fence_signal: ep %p loff 0x%lx lval 0x%llx roff 0x%lx rval 0x%llx flags 0x%x\n",
 722                ep, loff, lval, roff, rval, flags);
 723        err = scif_verify_epd(ep);
 724        if (err)
 725                return err;
 726
 727        /* Invalid flags? */
 728        if (flags & ~(SCIF_FENCE_INIT_SELF | SCIF_FENCE_INIT_PEER |
 729                        SCIF_SIGNAL_LOCAL | SCIF_SIGNAL_REMOTE))
 730                return -EINVAL;
 731
 732        /* At least one of init self or peer RMA should be set */
 733        if (!(flags & (SCIF_FENCE_INIT_SELF | SCIF_FENCE_INIT_PEER)))
 734                return -EINVAL;
 735
 736        /* Exactly one of init self or peer RMA should be set but not both */
 737        if ((flags & SCIF_FENCE_INIT_SELF) && (flags & SCIF_FENCE_INIT_PEER))
 738                return -EINVAL;
 739
 740        /* At least one of SCIF_SIGNAL_LOCAL or SCIF_SIGNAL_REMOTE required */
 741        if (!(flags & (SCIF_SIGNAL_LOCAL | SCIF_SIGNAL_REMOTE)))
 742                return -EINVAL;
 743
 744        /* Only Dword offsets allowed */
 745        if ((flags & SCIF_SIGNAL_LOCAL) && (loff & (sizeof(u32) - 1)))
 746                return -EINVAL;
 747
 748        /* Only Dword aligned offsets allowed */
 749        if ((flags & SCIF_SIGNAL_REMOTE) && (roff & (sizeof(u32) - 1)))
 750                return -EINVAL;
 751
 752        if (flags & SCIF_FENCE_INIT_PEER) {
 753                err = scif_send_fence_signal(epd, roff, rval, loff,
 754                                             lval, flags);
 755        } else {
 756                /* Local Signal in Local RAS */
 757                if (flags & SCIF_SIGNAL_LOCAL) {
 758                        err = scif_prog_signal(epd, loff, lval,
 759                                               SCIF_WINDOW_SELF);
 760                        if (err)
 761                                goto error_ret;
 762                }
 763
 764                /* Signal in Remote RAS */
 765                if (flags & SCIF_SIGNAL_REMOTE)
 766                        err = scif_prog_signal(epd, roff,
 767                                               rval, SCIF_WINDOW_PEER);
 768        }
 769error_ret:
 770        if (err)
 771                dev_err(scif_info.mdev.this_device,
 772                        "%s %d err %d\n", __func__, __LINE__, err);
 773        return err;
 774}
 775EXPORT_SYMBOL_GPL(scif_fence_signal);
 776