linux/drivers/infiniband/hw/hfi1/iowait.h
<<
>>
Prefs
   1#ifndef _HFI1_IOWAIT_H
   2#define _HFI1_IOWAIT_H
   3/*
   4 * Copyright(c) 2015 - 2018 Intel Corporation.
   5 *
   6 * This file is provided under a dual BSD/GPLv2 license.  When using or
   7 * redistributing this file, you may do so under either license.
   8 *
   9 * GPL LICENSE SUMMARY
  10 *
  11 * This program is free software; you can redistribute it and/or modify
  12 * it under the terms of version 2 of the GNU General Public License as
  13 * published by the Free Software Foundation.
  14 *
  15 * This program is distributed in the hope that it will be useful, but
  16 * WITHOUT ANY WARRANTY; without even the implied warranty of
  17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  18 * General Public License for more details.
  19 *
  20 * BSD LICENSE
  21 *
  22 * Redistribution and use in source and binary forms, with or without
  23 * modification, are permitted provided that the following conditions
  24 * are met:
  25 *
  26 *  - Redistributions of source code must retain the above copyright
  27 *    notice, this list of conditions and the following disclaimer.
  28 *  - Redistributions in binary form must reproduce the above copyright
  29 *    notice, this list of conditions and the following disclaimer in
  30 *    the documentation and/or other materials provided with the
  31 *    distribution.
  32 *  - Neither the name of Intel Corporation nor the names of its
  33 *    contributors may be used to endorse or promote products derived
  34 *    from this software without specific prior written permission.
  35 *
  36 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  37 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  38 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  39 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  40 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  41 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  42 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  43 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  44 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  45 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  46 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  47 *
  48 */
  49
  50#include <linux/list.h>
  51#include <linux/workqueue.h>
  52#include <linux/wait.h>
  53#include <linux/sched.h>
  54
  55#include "sdma_txreq.h"
  56
  57/*
  58 * typedef (*restart_t)() - restart callback
  59 * @work: pointer to work structure
  60 */
  61typedef void (*restart_t)(struct work_struct *work);
  62
  63#define IOWAIT_PENDING_IB  0x0
  64#define IOWAIT_PENDING_TID 0x1
  65
  66/*
  67 * A QP can have multiple Send Engines (SEs).
  68 *
  69 * The current use case is for supporting a TID RDMA
  70 * packet build/xmit mechanism independent from verbs.
  71 */
  72#define IOWAIT_SES 2
  73#define IOWAIT_IB_SE 0
  74#define IOWAIT_TID_SE 1
  75
  76struct sdma_txreq;
  77struct sdma_engine;
  78/**
  79 * @iowork: the work struct
  80 * @tx_head: list of prebuilt packets
  81 * @iow: the parent iowait structure
  82 *
  83 * This structure is the work item (process) specific
  84 * details associated with the each of the two SEs of the
  85 * QP.
  86 *
  87 * The workstruct and the queued TXs are unique to each
  88 * SE.
  89 */
  90struct iowait;
  91struct iowait_work {
  92        struct work_struct iowork;
  93        struct list_head tx_head;
  94        struct iowait *iow;
  95};
  96
  97/**
  98 * @list: used to add/insert into QP/PQ wait lists
  99 * @tx_head: overflow list of sdma_txreq's
 100 * @sleep: no space callback
 101 * @wakeup: space callback wakeup
 102 * @sdma_drained: sdma count drained
 103 * @init_priority: callback to manipulate priority
 104 * @lock: lock protected head of wait queue
 105 * @iowork: workqueue overhead
 106 * @wait_dma: wait for sdma_busy == 0
 107 * @wait_pio: wait for pio_busy == 0
 108 * @sdma_busy: # of packets in flight
 109 * @count: total number of descriptors in tx_head'ed list
 110 * @tx_limit: limit for overflow queuing
 111 * @tx_count: number of tx entry's in tx_head'ed list
 112 * @flags: wait flags (one per QP)
 113 * @wait: SE array for multiple legs
 114 *
 115 * This is to be embedded in user's state structure
 116 * (QP or PQ).
 117 *
 118 * The sleep and wakeup members are a
 119 * bit misnamed.   They do not strictly
 120 * speaking sleep or wake up, but they
 121 * are callbacks for the ULP to implement
 122 * what ever queuing/dequeuing of
 123 * the embedded iowait and its containing struct
 124 * when a resource shortage like SDMA ring space
 125 * or PIO credit space is seen.
 126 *
 127 * Both potentially have locks help
 128 * so sleeping is not allowed and it is not
 129 * supported to submit txreqs from the wakeup
 130 * call directly because of lock conflicts.
 131 *
 132 * The wait_dma member along with the iow
 133 *
 134 * The lock field is used by waiters to record
 135 * the seqlock_t that guards the list head.
 136 * Waiters explicity know that, but the destroy
 137 * code that unwaits QPs does not.
 138 */
 139struct iowait {
 140        struct list_head list;
 141        int (*sleep)(
 142                struct sdma_engine *sde,
 143                struct iowait_work *wait,
 144                struct sdma_txreq *tx,
 145                uint seq,
 146                bool pkts_sent
 147                );
 148        void (*wakeup)(struct iowait *wait, int reason);
 149        void (*sdma_drained)(struct iowait *wait);
 150        void (*init_priority)(struct iowait *wait);
 151        seqlock_t *lock;
 152        wait_queue_head_t wait_dma;
 153        wait_queue_head_t wait_pio;
 154        atomic_t sdma_busy;
 155        atomic_t pio_busy;
 156        u32 count;
 157        u32 tx_limit;
 158        u32 tx_count;
 159        u8 starved_cnt;
 160        u8 priority;
 161        unsigned long flags;
 162        struct iowait_work wait[IOWAIT_SES];
 163};
 164
 165#define SDMA_AVAIL_REASON 0
 166
 167void iowait_set_flag(struct iowait *wait, u32 flag);
 168bool iowait_flag_set(struct iowait *wait, u32 flag);
 169void iowait_clear_flag(struct iowait *wait, u32 flag);
 170
 171void iowait_init(struct iowait *wait, u32 tx_limit,
 172                 void (*func)(struct work_struct *work),
 173                 void (*tidfunc)(struct work_struct *work),
 174                 int (*sleep)(struct sdma_engine *sde,
 175                              struct iowait_work *wait,
 176                              struct sdma_txreq *tx,
 177                              uint seq,
 178                              bool pkts_sent),
 179                 void (*wakeup)(struct iowait *wait, int reason),
 180                 void (*sdma_drained)(struct iowait *wait),
 181                 void (*init_priority)(struct iowait *wait));
 182
 183/**
 184 * iowait_schedule() - schedule the default send engine work
 185 * @wait: wait struct to schedule
 186 * @wq: workqueue for schedule
 187 * @cpu: cpu
 188 */
 189static inline bool iowait_schedule(struct iowait *wait,
 190                                   struct workqueue_struct *wq, int cpu)
 191{
 192        return !!queue_work_on(cpu, wq, &wait->wait[IOWAIT_IB_SE].iowork);
 193}
 194
 195/**
 196 * iowait_tid_schedule - schedule the tid SE
 197 * @wait: the iowait structure
 198 * @wq: the work queue
 199 * @cpu: the cpu
 200 */
 201static inline bool iowait_tid_schedule(struct iowait *wait,
 202                                       struct workqueue_struct *wq, int cpu)
 203{
 204        return !!queue_work_on(cpu, wq, &wait->wait[IOWAIT_TID_SE].iowork);
 205}
 206
 207/**
 208 * iowait_sdma_drain() - wait for DMAs to drain
 209 *
 210 * @wait: iowait structure
 211 *
 212 * This will delay until the iowait sdmas have
 213 * completed.
 214 */
 215static inline void iowait_sdma_drain(struct iowait *wait)
 216{
 217        wait_event(wait->wait_dma, !atomic_read(&wait->sdma_busy));
 218}
 219
 220/**
 221 * iowait_sdma_pending() - return sdma pending count
 222 *
 223 * @wait: iowait structure
 224 *
 225 */
 226static inline int iowait_sdma_pending(struct iowait *wait)
 227{
 228        return atomic_read(&wait->sdma_busy);
 229}
 230
 231/**
 232 * iowait_sdma_inc - note sdma io pending
 233 * @wait: iowait structure
 234 */
 235static inline void iowait_sdma_inc(struct iowait *wait)
 236{
 237        atomic_inc(&wait->sdma_busy);
 238}
 239
 240/**
 241 * iowait_sdma_add - add count to pending
 242 * @wait: iowait structure
 243 */
 244static inline void iowait_sdma_add(struct iowait *wait, int count)
 245{
 246        atomic_add(count, &wait->sdma_busy);
 247}
 248
 249/**
 250 * iowait_sdma_dec - note sdma complete
 251 * @wait: iowait structure
 252 */
 253static inline int iowait_sdma_dec(struct iowait *wait)
 254{
 255        if (!wait)
 256                return 0;
 257        return atomic_dec_and_test(&wait->sdma_busy);
 258}
 259
 260/**
 261 * iowait_pio_drain() - wait for pios to drain
 262 *
 263 * @wait: iowait structure
 264 *
 265 * This will delay until the iowait pios have
 266 * completed.
 267 */
 268static inline void iowait_pio_drain(struct iowait *wait)
 269{
 270        wait_event_timeout(wait->wait_pio,
 271                           !atomic_read(&wait->pio_busy),
 272                           HZ);
 273}
 274
 275/**
 276 * iowait_pio_pending() - return pio pending count
 277 *
 278 * @wait: iowait structure
 279 *
 280 */
 281static inline int iowait_pio_pending(struct iowait *wait)
 282{
 283        return atomic_read(&wait->pio_busy);
 284}
 285
 286/**
 287 * iowait_pio_inc - note pio pending
 288 * @wait: iowait structure
 289 */
 290static inline void iowait_pio_inc(struct iowait *wait)
 291{
 292        atomic_inc(&wait->pio_busy);
 293}
 294
 295/**
 296 * iowait_pio_dec - note pio complete
 297 * @wait: iowait structure
 298 */
 299static inline int iowait_pio_dec(struct iowait *wait)
 300{
 301        if (!wait)
 302                return 0;
 303        return atomic_dec_and_test(&wait->pio_busy);
 304}
 305
 306/**
 307 * iowait_drain_wakeup() - trigger iowait_drain() waiter
 308 *
 309 * @wait: iowait structure
 310 *
 311 * This will trigger any waiters.
 312 */
 313static inline void iowait_drain_wakeup(struct iowait *wait)
 314{
 315        wake_up(&wait->wait_dma);
 316        wake_up(&wait->wait_pio);
 317        if (wait->sdma_drained)
 318                wait->sdma_drained(wait);
 319}
 320
 321/**
 322 * iowait_get_txhead() - get packet off of iowait list
 323 *
 324 * @wait iowait_work struture
 325 */
 326static inline struct sdma_txreq *iowait_get_txhead(struct iowait_work *wait)
 327{
 328        struct sdma_txreq *tx = NULL;
 329
 330        if (!list_empty(&wait->tx_head)) {
 331                tx = list_first_entry(
 332                        &wait->tx_head,
 333                        struct sdma_txreq,
 334                        list);
 335                list_del_init(&tx->list);
 336        }
 337        return tx;
 338}
 339
 340static inline u16 iowait_get_desc(struct iowait_work *w)
 341{
 342        u16 num_desc = 0;
 343        struct sdma_txreq *tx = NULL;
 344
 345        if (!list_empty(&w->tx_head)) {
 346                tx = list_first_entry(&w->tx_head, struct sdma_txreq,
 347                                      list);
 348                num_desc = tx->num_desc;
 349                if (tx->flags & SDMA_TXREQ_F_VIP)
 350                        w->iow->priority++;
 351        }
 352        return num_desc;
 353}
 354
 355static inline u32 iowait_get_all_desc(struct iowait *w)
 356{
 357        u32 num_desc = 0;
 358
 359        num_desc = iowait_get_desc(&w->wait[IOWAIT_IB_SE]);
 360        num_desc += iowait_get_desc(&w->wait[IOWAIT_TID_SE]);
 361        return num_desc;
 362}
 363
 364static inline void iowait_update_priority(struct iowait_work *w)
 365{
 366        struct sdma_txreq *tx = NULL;
 367
 368        if (!list_empty(&w->tx_head)) {
 369                tx = list_first_entry(&w->tx_head, struct sdma_txreq,
 370                                      list);
 371                if (tx->flags & SDMA_TXREQ_F_VIP)
 372                        w->iow->priority++;
 373        }
 374}
 375
 376static inline void iowait_update_all_priority(struct iowait *w)
 377{
 378        iowait_update_priority(&w->wait[IOWAIT_IB_SE]);
 379        iowait_update_priority(&w->wait[IOWAIT_TID_SE]);
 380}
 381
 382static inline void iowait_init_priority(struct iowait *w)
 383{
 384        w->priority = 0;
 385        if (w->init_priority)
 386                w->init_priority(w);
 387}
 388
 389static inline void iowait_get_priority(struct iowait *w)
 390{
 391        iowait_init_priority(w);
 392        iowait_update_all_priority(w);
 393}
 394
 395/**
 396 * iowait_queue - Put the iowait on a wait queue
 397 * @pkts_sent: have some packets been sent before queuing?
 398 * @w: the iowait struct
 399 * @wait_head: the wait queue
 400 *
 401 * This function is called to insert an iowait struct into a
 402 * wait queue after a resource (eg, sdma decriptor or pio
 403 * buffer) is run out.
 404 */
 405static inline void iowait_queue(bool pkts_sent, struct iowait *w,
 406                                struct list_head *wait_head)
 407{
 408        /*
 409         * To play fair, insert the iowait at the tail of the wait queue if it
 410         * has already sent some packets; Otherwise, put it at the head.
 411         * However, if it has priority packets to send, also put it at the
 412         * head.
 413         */
 414        if (pkts_sent)
 415                w->starved_cnt = 0;
 416        else
 417                w->starved_cnt++;
 418
 419        if (w->priority > 0 || !pkts_sent)
 420                list_add(&w->list, wait_head);
 421        else
 422                list_add_tail(&w->list, wait_head);
 423}
 424
 425/**
 426 * iowait_starve_clear - clear the wait queue's starve count
 427 * @pkts_sent: have some packets been sent?
 428 * @w: the iowait struct
 429 *
 430 * This function is called to clear the starve count. If no
 431 * packets have been sent, the starve count will not be cleared.
 432 */
 433static inline void iowait_starve_clear(bool pkts_sent, struct iowait *w)
 434{
 435        if (pkts_sent)
 436                w->starved_cnt = 0;
 437}
 438
 439/* Update the top priority index */
 440uint iowait_priority_update_top(struct iowait *w,
 441                                struct iowait *top,
 442                                uint idx, uint top_idx);
 443
 444/**
 445 * iowait_packet_queued() - determine if a packet is queued
 446 * @wait: the iowait_work structure
 447 */
 448static inline bool iowait_packet_queued(struct iowait_work *wait)
 449{
 450        return !list_empty(&wait->tx_head);
 451}
 452
 453/**
 454 * inc_wait_count - increment wait counts
 455 * @w: the log work struct
 456 * @n: the count
 457 */
 458static inline void iowait_inc_wait_count(struct iowait_work *w, u16 n)
 459{
 460        if (!w)
 461                return;
 462        w->iow->tx_count++;
 463        w->iow->count += n;
 464}
 465
 466/**
 467 * iowait_get_tid_work - return iowait_work for tid SE
 468 * @w: the iowait struct
 469 */
 470static inline struct iowait_work *iowait_get_tid_work(struct iowait *w)
 471{
 472        return &w->wait[IOWAIT_TID_SE];
 473}
 474
 475/**
 476 * iowait_get_ib_work - return iowait_work for ib SE
 477 * @w: the iowait struct
 478 */
 479static inline struct iowait_work *iowait_get_ib_work(struct iowait *w)
 480{
 481        return &w->wait[IOWAIT_IB_SE];
 482}
 483
 484/**
 485 * iowait_ioww_to_iow - return iowait given iowait_work
 486 * @w: the iowait_work struct
 487 */
 488static inline struct iowait *iowait_ioww_to_iow(struct iowait_work *w)
 489{
 490        if (likely(w))
 491                return w->iow;
 492        return NULL;
 493}
 494
 495void iowait_cancel_work(struct iowait *w);
 496int iowait_set_work_flag(struct iowait_work *w);
 497
 498#endif
 499