LXR linux/crypto/async_tx/async_raid6

   1/*
   2 * Asynchronous RAID-6 recovery calculations ASYNC_TX API.
   3 * Copyright(c) 2009 Intel Corporation
   4 *
   5 * based on raid6recov.c:
   6 *   Copyright 2002 H. Peter Anvin
   7 *
   8 * This program is free software; you can redistribute it and/or modify it
   9 * under the terms of the GNU General Public License as published by the Free
  10 * Software Foundation; either version 2 of the License, or (at your option)
  11 * any later version.
  12 *
  13 * This program is distributed in the hope that it will be useful, but WITHOUT
  14 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  15 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  16 * more details.
  17 *
  18 * You should have received a copy of the GNU General Public License along with
  19 * this program; if not, write to the Free Software Foundation, Inc., 51
  20 * Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
  21 *
  22 */
  23#include <linux/kernel.h>
  24#include <linux/interrupt.h>
  25#include <linux/module.h>
  26#include <linux/dma-mapping.h>
  27#include <linux/raid/pq.h>
  28#include <linux/async_tx.h>
  29#include <linux/dmaengine.h>
  30
  31static struct dma_async_tx_descriptor *
  32async_sum_product(struct page *dest, struct page **srcs, unsigned char *coef,
  33                  size_t len, struct async_submit_ctl *submit)
  34{
  35        struct dma_chan *chan = async_tx_find_channel(submit, DMA_PQ,
  36                                                      &dest, 1, srcs, 2, len);
  37        struct dma_device *dma = chan ? chan->device : NULL;
  38        struct dmaengine_unmap_data *unmap = NULL;
  39        const u8 *amul, *bmul;
  40        u8 ax, bx;
  41        u8 *a, *b, *c;
  42
  43        if (dma)
  44                unmap = dmaengine_get_unmap_data(dma->dev, 3, GFP_NOWAIT);
  45
  46        if (unmap) {
  47                struct device *dev = dma->dev;
  48                dma_addr_t pq[2];
  49                struct dma_async_tx_descriptor *tx;
  50                enum dma_ctrl_flags dma_flags = DMA_PREP_PQ_DISABLE_P;
  51
  52                if (submit->flags & ASYNC_TX_FENCE)
  53                        dma_flags |= DMA_PREP_FENCE;
  54                unmap->addr[0] = dma_map_page(dev, srcs[0], 0, len, DMA_TO_DEVICE);
  55                unmap->addr[1] = dma_map_page(dev, srcs[1], 0, len, DMA_TO_DEVICE);
  56                unmap->to_cnt = 2;
  57
  58                unmap->addr[2] = dma_map_page(dev, dest, 0, len, DMA_BIDIRECTIONAL);
  59                unmap->bidi_cnt = 1;
  60                /* engine only looks at Q, but expects it to follow P */
  61                pq[1] = unmap->addr[2];
  62
  63                unmap->len = len;
  64                tx = dma->device_prep_dma_pq(chan, pq, unmap->addr, 2, coef,
  65                                             len, dma_flags);
  66                if (tx) {
  67                        dma_set_unmap(tx, unmap);
  68                        async_tx_submit(chan, tx, submit);
  69                        dmaengine_unmap_put(unmap);
  70                        return tx;
  71                }
  72
  73                /* could not get a descriptor, unmap and fall through to
  74                 * the synchronous path
  75                 */
  76                dmaengine_unmap_put(unmap);
  77        }
  78
  79        /* run the operation synchronously */
  80        async_tx_quiesce(&submit->depend_tx);
  81        amul = raid6_gfmul[coef[0]];
  82        bmul = raid6_gfmul[coef[1]];
  83        a = page_address(srcs[0]);
  84        b = page_address(srcs[1]);
  85        c = page_address(dest);
  86
  87        while (len--) {
  88                ax    = amul[*a++];
  89                bx    = bmul[*b++];
  90                *c++ = ax ^ bx;
  91        }
  92
  93        return NULL;
  94}
  95
  96static struct dma_async_tx_descriptor *
  97async_mult(struct page *dest, struct page *src, u8 coef, size_t len,
  98           struct async_submit_ctl *submit)
  99{
 100        struct dma_chan *chan = async_tx_find_channel(submit, DMA_PQ,
 101                                                      &dest, 1, &src, 1, len);
 102        struct dma_device *dma = chan ? chan->device : NULL;
 103        struct dmaengine_unmap_data *unmap = NULL;
 104        const u8 *qmul; /* Q multiplier table */
 105        u8 *d, *s;
 106
 107        if (dma)
 108                unmap = dmaengine_get_unmap_data(dma->dev, 3, GFP_NOWAIT);
 109
 110        if (unmap) {
 111                dma_addr_t dma_dest[2];
 112                struct device *dev = dma->dev;
 113                struct dma_async_tx_descriptor *tx;
 114                enum dma_ctrl_flags dma_flags = DMA_PREP_PQ_DISABLE_P;
 115
 116                if (submit->flags & ASYNC_TX_FENCE)
 117                        dma_flags |= DMA_PREP_FENCE;
 118                unmap->addr[0] = dma_map_page(dev, src, 0, len, DMA_TO_DEVICE);
 119                unmap->to_cnt++;
 120                unmap->addr[1] = dma_map_page(dev, dest, 0, len, DMA_BIDIRECTIONAL);
 121                dma_dest[1] = unmap->addr[1];
 122                unmap->bidi_cnt++;
 123                unmap->len = len;
 124
 125                /* this looks funny, but the engine looks for Q at
 126                 * dma_dest[1] and ignores dma_dest[0] as a dest
 127                 * due to DMA_PREP_PQ_DISABLE_P
 128                 */
 129                tx = dma->device_prep_dma_pq(chan, dma_dest, unmap->addr,
 130                                             1, &coef, len, dma_flags);
 131
 132                if (tx) {
 133                        dma_set_unmap(tx, unmap);
 134                        dmaengine_unmap_put(unmap);
 135                        async_tx_submit(chan, tx, submit);
 136                        return tx;
 137                }
 138
 139                /* could not get a descriptor, unmap and fall through to
 140                 * the synchronous path
 141                 */
 142                dmaengine_unmap_put(unmap);
 143        }
 144
 145        /* no channel available, or failed to allocate a descriptor, so
 146         * perform the operation synchronously
 147         */
 148        async_tx_quiesce(&submit->depend_tx);
 149        qmul  = raid6_gfmul[coef];
 150        d = page_address(dest);
 151        s = page_address(src);
 152
 153        while (len--)
 154                *d++ = qmul[*s++];
 155
 156        return NULL;
 157}
 158
 159static struct dma_async_tx_descriptor *
 160__2data_recov_4(int disks, size_t bytes, int faila, int failb,
 161                struct page **blocks, struct async_submit_ctl *submit)
 162{
 163        struct dma_async_tx_descriptor *tx = NULL;
 164        struct page *p, *q, *a, *b;
 165        struct page *srcs[2];
 166        unsigned char coef[2];
 167        enum async_tx_flags flags = submit->flags;
 168        dma_async_tx_callback cb_fn = submit->cb_fn;
 169        void *cb_param = submit->cb_param;
 170        void *scribble = submit->scribble;
 171
 172        p = blocks[disks-2];
 173        q = blocks[disks-1];
 174
 175        a = blocks[faila];
 176        b = blocks[failb];
 177
 178        /* in the 4 disk case P + Pxy == P and Q + Qxy == Q */
 179        /* Dx = A*(P+Pxy) + B*(Q+Qxy) */
 180        srcs[0] = p;
 181        srcs[1] = q;
 182        coef[0] = raid6_gfexi[failb-faila];
 183        coef[1] = raid6_gfinv[raid6_gfexp[faila]^raid6_gfexp[failb]];
 184        init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
 185        tx = async_sum_product(b, srcs, coef, bytes, submit);
 186
 187        /* Dy = P+Pxy+Dx */
 188        srcs[0] = p;
 189        srcs[1] = b;
 190        init_async_submit(submit, flags | ASYNC_TX_XOR_ZERO_DST, tx, cb_fn,
 191                          cb_param, scribble);
 192        tx = async_xor(a, srcs, 0, 2, bytes, submit);
 193
 194        return tx;
 195
 196}
 197
 198static struct dma_async_tx_descriptor *
 199__2data_recov_5(int disks, size_t bytes, int faila, int failb,
 200                struct page **blocks, struct async_submit_ctl *submit)
 201{
 202        struct dma_async_tx_descriptor *tx = NULL;
 203        struct page *p, *q, *g, *dp, *dq;
 204        struct page *srcs[2];
 205        unsigned char coef[2];
 206        enum async_tx_flags flags = submit->flags;
 207        dma_async_tx_callback cb_fn = submit->cb_fn;
 208        void *cb_param = submit->cb_param;
 209        void *scribble = submit->scribble;
 210        int good_srcs, good, i;
 211
 212        good_srcs = 0;
 213        good = -1;
 214        for (i = 0; i < disks-2; i++) {
 215                if (blocks[i] == NULL)
 216                        continue;
 217                if (i == faila || i == failb)
 218                        continue;
 219                good = i;
 220                good_srcs++;
 221        }
 222        BUG_ON(good_srcs > 1);
 223
 224        p = blocks[disks-2];
 225        q = blocks[disks-1];
 226        g = blocks[good];
 227
 228        /* Compute syndrome with zero for the missing data pages
 229         * Use the dead data pages as temporary storage for delta p and
 230         * delta q
 231         */
 232        dp = blocks[faila];
 233        dq = blocks[failb];
 234
 235        init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
 236        tx = async_memcpy(dp, g, 0, 0, bytes, submit);
 237        init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
 238        tx = async_mult(dq, g, raid6_gfexp[good], bytes, submit);
 239
 240        /* compute P + Pxy */
 241        srcs[0] = dp;
 242        srcs[1] = p;
 243        init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
 244                          NULL, NULL, scribble);
 245        tx = async_xor(dp, srcs, 0, 2, bytes, submit);
 246
 247        /* compute Q + Qxy */
 248        srcs[0] = dq;
 249        srcs[1] = q;
 250        init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
 251                          NULL, NULL, scribble);
 252        tx = async_xor(dq, srcs, 0, 2, bytes, submit);
 253
 254        /* Dx = A*(P+Pxy) + B*(Q+Qxy) */
 255        srcs[0] = dp;
 256        srcs[1] = dq;
 257        coef[0] = raid6_gfexi[failb-faila];
 258        coef[1] = raid6_gfinv[raid6_gfexp[faila]^raid6_gfexp[failb]];
 259        init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
 260        tx = async_sum_product(dq, srcs, coef, bytes, submit);
 261
 262        /* Dy = P+Pxy+Dx */
 263        srcs[0] = dp;
 264        srcs[1] = dq;
 265        init_async_submit(submit, flags | ASYNC_TX_XOR_DROP_DST, tx, cb_fn,
 266                          cb_param, scribble);
 267        tx = async_xor(dp, srcs, 0, 2, bytes, submit);
 268
 269        return tx;
 270}
 271
 272static struct dma_async_tx_descriptor *
 273__2data_recov_n(int disks, size_t bytes, int faila, int failb,
 274              struct page **blocks, struct async_submit_ctl *submit)
 275{
 276        struct dma_async_tx_descriptor *tx = NULL;
 277        struct page *p, *q, *dp, *dq;
 278        struct page *srcs[2];
 279        unsigned char coef[2];
 280        enum async_tx_flags flags = submit->flags;
 281        dma_async_tx_callback cb_fn = submit->cb_fn;
 282        void *cb_param = submit->cb_param;
 283        void *scribble = submit->scribble;
 284
 285        p = blocks[disks-2];
 286        q = blocks[disks-1];
 287
 288        /* Compute syndrome with zero for the missing data pages
 289         * Use the dead data pages as temporary storage for
 290         * delta p and delta q
 291         */
 292        dp = blocks[faila];
 293        blocks[faila] = NULL;
 294        blocks[disks-2] = dp;
 295        dq = blocks[failb];
 296        blocks[failb] = NULL;
 297        blocks[disks-1] = dq;
 298
 299        init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
 300        tx = async_gen_syndrome(blocks, 0, disks, bytes, submit);
 301
 302        /* Restore pointer table */
 303        blocks[faila]   = dp;
 304        blocks[failb]   = dq;
 305        blocks[disks-2] = p;
 306        blocks[disks-1] = q;
 307
 308        /* compute P + Pxy */
 309        srcs[0] = dp;
 310        srcs[1] = p;
 311        init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
 312                          NULL, NULL, scribble);
 313        tx = async_xor(dp, srcs, 0, 2, bytes, submit);
 314
 315        /* compute Q + Qxy */
 316        srcs[0] = dq;
 317        srcs[1] = q;
 318        init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
 319                          NULL, NULL, scribble);
 320        tx = async_xor(dq, srcs, 0, 2, bytes, submit);
 321
 322        /* Dx = A*(P+Pxy) + B*(Q+Qxy) */
 323        srcs[0] = dp;
 324        srcs[1] = dq;
 325        coef[0] = raid6_gfexi[failb-faila];
 326        coef[1] = raid6_gfinv[raid6_gfexp[faila]^raid6_gfexp[failb]];
 327        init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
 328        tx = async_sum_product(dq, srcs, coef, bytes, submit);
 329
 330        /* Dy = P+Pxy+Dx */
 331        srcs[0] = dp;
 332        srcs[1] = dq;
 333        init_async_submit(submit, flags | ASYNC_TX_XOR_DROP_DST, tx, cb_fn,
 334                          cb_param, scribble);
 335        tx = async_xor(dp, srcs, 0, 2, bytes, submit);
 336
 337        return tx;
 338}
 339
 340/**
 341 * async_raid6_2data_recov - asynchronously calculate two missing data blocks
 342 * @disks: number of disks in the RAID-6 array
 343 * @bytes: block size
 344 * @faila: first failed drive index
 345 * @failb: second failed drive index
 346 * @blocks: array of source pointers where the last two entries are p and q
 347 * @submit: submission/completion modifiers
 348 */
 349struct dma_async_tx_descriptor *
 350async_raid6_2data_recov(int disks, size_t bytes, int faila, int failb,
 351                        struct page **blocks, struct async_submit_ctl *submit)
 352{
 353        void *scribble = submit->scribble;
 354        int non_zero_srcs, i;
 355
 356        BUG_ON(faila == failb);
 357        if (failb < faila)
 358                swap(faila, failb);
 359
 360        pr_debug("%s: disks: %d len: %zu\n", __func__, disks, bytes);
 361
 362        /* if a dma resource is not available or a scribble buffer is not
 363         * available punt to the synchronous path.  In the 'dma not
 364         * available' case be sure to use the scribble buffer to
 365         * preserve the content of 'blocks' as the caller intended.
 366         */
 367        if (!async_dma_find_channel(DMA_PQ) || !scribble) {
 368                void **ptrs = scribble ? scribble : (void **) blocks;
 369
 370                async_tx_quiesce(&submit->depend_tx);
 371                for (i = 0; i < disks; i++)
 372                        if (blocks[i] == NULL)
 373                                ptrs[i] = (void *) raid6_empty_zero_page;
 374                        else
 375                                ptrs[i] = page_address(blocks[i]);
 376
 377                raid6_2data_recov(disks, bytes, faila, failb, ptrs);
 378
 379                async_tx_sync_epilog(submit);
 380
 381                return NULL;
 382        }
 383
 384        non_zero_srcs = 0;
 385        for (i = 0; i < disks-2 && non_zero_srcs < 4; i++)
 386                if (blocks[i])
 387                        non_zero_srcs++;
 388        switch (non_zero_srcs) {
 389        case 0:
 390        case 1:
 391                /* There must be at least 2 sources - the failed devices. */
 392                BUG();
 393
 394        case 2:
 395                /* dma devices do not uniformly understand a zero source pq
 396                 * operation (in contrast to the synchronous case), so
 397                 * explicitly handle the special case of a 4 disk array with
 398                 * both data disks missing.
 399                 */
 400                return __2data_recov_4(disks, bytes, faila, failb, blocks, submit);
 401        case 3:
 402                /* dma devices do not uniformly understand a single
 403                 * source pq operation (in contrast to the synchronous
 404                 * case), so explicitly handle the special case of a 5 disk
 405                 * array with 2 of 3 data disks missing.
 406                 */
 407                return __2data_recov_5(disks, bytes, faila, failb, blocks, submit);
 408        default:
 409                return __2data_recov_n(disks, bytes, faila, failb, blocks, submit);
 410        }
 411}
 412EXPORT_SYMBOL_GPL(async_raid6_2data_recov);
 413
 414/**
 415 * async_raid6_datap_recov - asynchronously calculate a data and the 'p' block
 416 * @disks: number of disks in the RAID-6 array
 417 * @bytes: block size
 418 * @faila: failed drive index
 419 * @blocks: array of source pointers where the last two entries are p and q
 420 * @submit: submission/completion modifiers
 421 */
 422struct dma_async_tx_descriptor *
 423async_raid6_datap_recov(int disks, size_t bytes, int faila,
 424                        struct page **blocks, struct async_submit_ctl *submit)
 425{
 426        struct dma_async_tx_descriptor *tx = NULL;
 427        struct page *p, *q, *dq;
 428        u8 coef;
 429        enum async_tx_flags flags = submit->flags;
 430        dma_async_tx_callback cb_fn = submit->cb_fn;
 431        void *cb_param = submit->cb_param;
 432        void *scribble = submit->scribble;
 433        int good_srcs, good, i;
 434        struct page *srcs[2];
 435
 436        pr_debug("%s: disks: %d len: %zu\n", __func__, disks, bytes);
 437
 438        /* if a dma resource is not available or a scribble buffer is not
 439         * available punt to the synchronous path.  In the 'dma not
 440         * available' case be sure to use the scribble buffer to
 441         * preserve the content of 'blocks' as the caller intended.
 442         */
 443        if (!async_dma_find_channel(DMA_PQ) || !scribble) {
 444                void **ptrs = scribble ? scribble : (void **) blocks;
 445
 446                async_tx_quiesce(&submit->depend_tx);
 447                for (i = 0; i < disks; i++)
 448                        if (blocks[i] == NULL)
 449                                ptrs[i] = (void*)raid6_empty_zero_page;
 450                        else
 451                                ptrs[i] = page_address(blocks[i]);
 452
 453                raid6_datap_recov(disks, bytes, faila, ptrs);
 454
 455                async_tx_sync_epilog(submit);
 456
 457                return NULL;
 458        }
 459
 460        good_srcs = 0;
 461        good = -1;
 462        for (i = 0; i < disks-2; i++) {
 463                if (i == faila)
 464                        continue;
 465                if (blocks[i]) {
 466                        good = i;
 467                        good_srcs++;
 468                        if (good_srcs > 1)
 469                                break;
 470                }
 471        }
 472        BUG_ON(good_srcs == 0);
 473
 474        p = blocks[disks-2];
 475        q = blocks[disks-1];
 476
 477        /* Compute syndrome with zero for the missing data page
 478         * Use the dead data page as temporary storage for delta q
 479         */
 480        dq = blocks[faila];
 481        blocks[faila] = NULL;
 482        blocks[disks-1] = dq;
 483
 484        /* in the 4-disk case we only need to perform a single source
 485         * multiplication with the one good data block.
 486         */
 487        if (good_srcs == 1) {
 488                struct page *g = blocks[good];
 489
 490                init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL,
 491                                  scribble);
 492                tx = async_memcpy(p, g, 0, 0, bytes, submit);
 493
 494                init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL,
 495                                  scribble);
 496                tx = async_mult(dq, g, raid6_gfexp[good], bytes, submit);
 497        } else {
 498                init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL,
 499                                  scribble);
 500                tx = async_gen_syndrome(blocks, 0, disks, bytes, submit);
 501        }
 502
 503        /* Restore pointer table */
 504        blocks[faila]   = dq;
 505        blocks[disks-1] = q;
 506
 507        /* calculate g^{-faila} */
 508        coef = raid6_gfinv[raid6_gfexp[faila]];
 509
 510        srcs[0] = dq;
 511        srcs[1] = q;
 512        init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
 513                          NULL, NULL, scribble);
 514        tx = async_xor(dq, srcs, 0, 2, bytes, submit);
 515
 516        init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
 517        tx = async_mult(dq, dq, coef, bytes, submit);
 518
 519        srcs[0] = p;
 520        srcs[1] = dq;
 521        init_async_submit(submit, flags | ASYNC_TX_XOR_DROP_DST, tx, cb_fn,
 522                          cb_param, scribble);
 523        tx = async_xor(p, srcs, 0, 2, bytes, submit);
 524
 525        return tx;
 526}
 527EXPORT_SYMBOL_GPL(async_raid6_datap_recov);
 528
 529MODULE_AUTHOR("Dan Williams <dan.j.williams@intel.com>");
 530MODULE_DESCRIPTION("asynchronous RAID-6 recovery api");
 531MODULE_LICENSE("GPL");
 532