LXR linux/crypto/async_tx/async

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * core routines for the asynchronous memory transfer/transform api
   4 *
   5 * Copyright © 2006, Intel Corporation.
   6 *
   7 *      Dan Williams <dan.j.williams@intel.com>
   8 *
   9 *      with architecture considerations by:
  10 *      Neil Brown <neilb@suse.de>
  11 *      Jeff Garzik <jeff@garzik.org>
  12 */
  13#include <linux/rculist.h>
  14#include <linux/module.h>
  15#include <linux/kernel.h>
  16#include <linux/async_tx.h>
  17
  18#ifdef CONFIG_DMA_ENGINE
  19static int __init async_tx_init(void)
  20{
  21        async_dmaengine_get();
  22
  23        printk(KERN_INFO "async_tx: api initialized (async)\n");
  24
  25        return 0;
  26}
  27
  28static void __exit async_tx_exit(void)
  29{
  30        async_dmaengine_put();
  31}
  32
  33module_init(async_tx_init);
  34module_exit(async_tx_exit);
  35
  36/**
  37 * __async_tx_find_channel - find a channel to carry out the operation or let
  38 *      the transaction execute synchronously
  39 * @submit: transaction dependency and submission modifiers
  40 * @tx_type: transaction type
  41 */
  42struct dma_chan *
  43__async_tx_find_channel(struct async_submit_ctl *submit,
  44                        enum dma_transaction_type tx_type)
  45{
  46        struct dma_async_tx_descriptor *depend_tx = submit->depend_tx;
  47
  48        /* see if we can keep the chain on one channel */
  49        if (depend_tx &&
  50            dma_has_cap(tx_type, depend_tx->chan->device->cap_mask))
  51                return depend_tx->chan;
  52        return async_dma_find_channel(tx_type);
  53}
  54EXPORT_SYMBOL_GPL(__async_tx_find_channel);
  55#endif
  56
  57
  58/**
  59 * async_tx_channel_switch - queue an interrupt descriptor with a dependency
  60 *      pre-attached.
  61 * @depend_tx: the operation that must finish before the new operation runs
  62 * @tx: the new operation
  63 */
  64static void
  65async_tx_channel_switch(struct dma_async_tx_descriptor *depend_tx,
  66                        struct dma_async_tx_descriptor *tx)
  67{
  68        struct dma_chan *chan = depend_tx->chan;
  69        struct dma_device *device = chan->device;
  70        struct dma_async_tx_descriptor *intr_tx = (void *) ~0;
  71
  72        /* first check to see if we can still append to depend_tx */
  73        txd_lock(depend_tx);
  74        if (txd_parent(depend_tx) && depend_tx->chan == tx->chan) {
  75                txd_chain(depend_tx, tx);
  76                intr_tx = NULL;
  77        }
  78        txd_unlock(depend_tx);
  79
  80        /* attached dependency, flush the parent channel */
  81        if (!intr_tx) {
  82                device->device_issue_pending(chan);
  83                return;
  84        }
  85
  86        /* see if we can schedule an interrupt
  87         * otherwise poll for completion
  88         */
  89        if (dma_has_cap(DMA_INTERRUPT, device->cap_mask))
  90                intr_tx = device->device_prep_dma_interrupt(chan, 0);
  91        else
  92                intr_tx = NULL;
  93
  94        if (intr_tx) {
  95                intr_tx->callback = NULL;
  96                intr_tx->callback_param = NULL;
  97                /* safe to chain outside the lock since we know we are
  98                 * not submitted yet
  99                 */
 100                txd_chain(intr_tx, tx);
 101
 102                /* check if we need to append */
 103                txd_lock(depend_tx);
 104                if (txd_parent(depend_tx)) {
 105                        txd_chain(depend_tx, intr_tx);
 106                        async_tx_ack(intr_tx);
 107                        intr_tx = NULL;
 108                }
 109                txd_unlock(depend_tx);
 110
 111                if (intr_tx) {
 112                        txd_clear_parent(intr_tx);
 113                        intr_tx->tx_submit(intr_tx);
 114                        async_tx_ack(intr_tx);
 115                }
 116                device->device_issue_pending(chan);
 117        } else {
 118                if (dma_wait_for_async_tx(depend_tx) != DMA_COMPLETE)
 119                        panic("%s: DMA error waiting for depend_tx\n",
 120                              __func__);
 121                tx->tx_submit(tx);
 122        }
 123}
 124
 125
 126/**
 127 * submit_disposition - flags for routing an incoming operation
 128 * @ASYNC_TX_SUBMITTED: we were able to append the new operation under the lock
 129 * @ASYNC_TX_CHANNEL_SWITCH: when the lock is dropped schedule a channel switch
 130 * @ASYNC_TX_DIRECT_SUBMIT: when the lock is dropped submit directly
 131 *
 132 * while holding depend_tx->lock we must avoid submitting new operations
 133 * to prevent a circular locking dependency with drivers that already
 134 * hold a channel lock when calling async_tx_run_dependencies.
 135 */
 136enum submit_disposition {
 137        ASYNC_TX_SUBMITTED,
 138        ASYNC_TX_CHANNEL_SWITCH,
 139        ASYNC_TX_DIRECT_SUBMIT,
 140};
 141
 142void
 143async_tx_submit(struct dma_chan *chan, struct dma_async_tx_descriptor *tx,
 144                struct async_submit_ctl *submit)
 145{
 146        struct dma_async_tx_descriptor *depend_tx = submit->depend_tx;
 147
 148        tx->callback = submit->cb_fn;
 149        tx->callback_param = submit->cb_param;
 150
 151        if (depend_tx) {
 152                enum submit_disposition s;
 153
 154                /* sanity check the dependency chain:
 155                 * 1/ if ack is already set then we cannot be sure
 156                 * we are referring to the correct operation
 157                 * 2/ dependencies are 1:1 i.e. two transactions can
 158                 * not depend on the same parent
 159                 */
 160                BUG_ON(async_tx_test_ack(depend_tx) || txd_next(depend_tx) ||
 161                       txd_parent(tx));
 162
 163                /* the lock prevents async_tx_run_dependencies from missing
 164                 * the setting of ->next when ->parent != NULL
 165                 */
 166                txd_lock(depend_tx);
 167                if (txd_parent(depend_tx)) {
 168                        /* we have a parent so we can not submit directly
 169                         * if we are staying on the same channel: append
 170                         * else: channel switch
 171                         */
 172                        if (depend_tx->chan == chan) {
 173                                txd_chain(depend_tx, tx);
 174                                s = ASYNC_TX_SUBMITTED;
 175                        } else
 176                                s = ASYNC_TX_CHANNEL_SWITCH;
 177                } else {
 178                        /* we do not have a parent so we may be able to submit
 179                         * directly if we are staying on the same channel
 180                         */
 181                        if (depend_tx->chan == chan)
 182                                s = ASYNC_TX_DIRECT_SUBMIT;
 183                        else
 184                                s = ASYNC_TX_CHANNEL_SWITCH;
 185                }
 186                txd_unlock(depend_tx);
 187
 188                switch (s) {
 189                case ASYNC_TX_SUBMITTED:
 190                        break;
 191                case ASYNC_TX_CHANNEL_SWITCH:
 192                        async_tx_channel_switch(depend_tx, tx);
 193                        break;
 194                case ASYNC_TX_DIRECT_SUBMIT:
 195                        txd_clear_parent(tx);
 196                        tx->tx_submit(tx);
 197                        break;
 198                }
 199        } else {
 200                txd_clear_parent(tx);
 201                tx->tx_submit(tx);
 202        }
 203
 204        if (submit->flags & ASYNC_TX_ACK)
 205                async_tx_ack(tx);
 206
 207        if (depend_tx)
 208                async_tx_ack(depend_tx);
 209}
 210EXPORT_SYMBOL_GPL(async_tx_submit);
 211
 212/**
 213 * async_trigger_callback - schedules the callback function to be run
 214 * @submit: submission and completion parameters
 215 *
 216 * honored flags: ASYNC_TX_ACK
 217 *
 218 * The callback is run after any dependent operations have completed.
 219 */
 220struct dma_async_tx_descriptor *
 221async_trigger_callback(struct async_submit_ctl *submit)
 222{
 223        struct dma_chan *chan;
 224        struct dma_device *device;
 225        struct dma_async_tx_descriptor *tx;
 226        struct dma_async_tx_descriptor *depend_tx = submit->depend_tx;
 227
 228        if (depend_tx) {
 229                chan = depend_tx->chan;
 230                device = chan->device;
 231
 232                /* see if we can schedule an interrupt
 233                 * otherwise poll for completion
 234                 */
 235                if (device && !dma_has_cap(DMA_INTERRUPT, device->cap_mask))
 236                        device = NULL;
 237
 238                tx = device ? device->device_prep_dma_interrupt(chan, 0) : NULL;
 239        } else
 240                tx = NULL;
 241
 242        if (tx) {
 243                pr_debug("%s: (async)\n", __func__);
 244
 245                async_tx_submit(chan, tx, submit);
 246        } else {
 247                pr_debug("%s: (sync)\n", __func__);
 248
 249                /* wait for any prerequisite operations */
 250                async_tx_quiesce(&submit->depend_tx);
 251
 252                async_tx_sync_epilog(submit);
 253        }
 254
 255        return tx;
 256}
 257EXPORT_SYMBOL_GPL(async_trigger_callback);
 258
 259/**
 260 * async_tx_quiesce - ensure tx is complete and freeable upon return
 261 * @tx - transaction to quiesce
 262 */
 263void async_tx_quiesce(struct dma_async_tx_descriptor **tx)
 264{
 265        if (*tx) {
 266                /* if ack is already set then we cannot be sure
 267                 * we are referring to the correct operation
 268                 */
 269                BUG_ON(async_tx_test_ack(*tx));
 270                if (dma_wait_for_async_tx(*tx) != DMA_COMPLETE)
 271                        panic("%s: DMA error waiting for transaction\n",
 272                              __func__);
 273                async_tx_ack(*tx);
 274                *tx = NULL;
 275        }
 276}
 277EXPORT_SYMBOL_GPL(async_tx_quiesce);
 278
 279MODULE_AUTHOR("Intel Corporation");
 280MODULE_DESCRIPTION("Asynchronous Bulk Memory Transactions API");
 281MODULE_LICENSE("GPL");
 282