linux/drivers/dma/ppc4xx/adma.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0-or-later
   2/*
   3 * Copyright (C) 2006-2009 DENX Software Engineering.
   4 *
   5 * Author: Yuri Tikhonov <yur@emcraft.com>
   6 *
   7 * Further porting to arch/powerpc by
   8 *      Anatolij Gustschin <agust@denx.de>
   9 */
  10
  11/*
  12 * This driver supports the asynchrounous DMA copy and RAID engines available
  13 * on the AMCC PPC440SPe Processors.
  14 * Based on the Intel Xscale(R) family of I/O Processors (IOP 32x, 33x, 134x)
  15 * ADMA driver written by D.Williams.
  16 */
  17
  18#include <linux/init.h>
  19#include <linux/module.h>
  20#include <linux/async_tx.h>
  21#include <linux/delay.h>
  22#include <linux/dma-mapping.h>
  23#include <linux/spinlock.h>
  24#include <linux/interrupt.h>
  25#include <linux/slab.h>
  26#include <linux/uaccess.h>
  27#include <linux/proc_fs.h>
  28#include <linux/of.h>
  29#include <linux/of_address.h>
  30#include <linux/of_irq.h>
  31#include <linux/of_platform.h>
  32#include <asm/dcr.h>
  33#include <asm/dcr-regs.h>
  34#include "adma.h"
  35#include "../dmaengine.h"
  36
  37enum ppc_adma_init_code {
  38        PPC_ADMA_INIT_OK = 0,
  39        PPC_ADMA_INIT_MEMRES,
  40        PPC_ADMA_INIT_MEMREG,
  41        PPC_ADMA_INIT_ALLOC,
  42        PPC_ADMA_INIT_COHERENT,
  43        PPC_ADMA_INIT_CHANNEL,
  44        PPC_ADMA_INIT_IRQ1,
  45        PPC_ADMA_INIT_IRQ2,
  46        PPC_ADMA_INIT_REGISTER
  47};
  48
  49static char *ppc_adma_errors[] = {
  50        [PPC_ADMA_INIT_OK] = "ok",
  51        [PPC_ADMA_INIT_MEMRES] = "failed to get memory resource",
  52        [PPC_ADMA_INIT_MEMREG] = "failed to request memory region",
  53        [PPC_ADMA_INIT_ALLOC] = "failed to allocate memory for adev "
  54                                "structure",
  55        [PPC_ADMA_INIT_COHERENT] = "failed to allocate coherent memory for "
  56                                   "hardware descriptors",
  57        [PPC_ADMA_INIT_CHANNEL] = "failed to allocate memory for channel",
  58        [PPC_ADMA_INIT_IRQ1] = "failed to request first irq",
  59        [PPC_ADMA_INIT_IRQ2] = "failed to request second irq",
  60        [PPC_ADMA_INIT_REGISTER] = "failed to register dma async device",
  61};
  62
  63static enum ppc_adma_init_code
  64ppc440spe_adma_devices[PPC440SPE_ADMA_ENGINES_NUM];
  65
  66struct ppc_dma_chan_ref {
  67        struct dma_chan *chan;
  68        struct list_head node;
  69};
  70
  71/* The list of channels exported by ppc440spe ADMA */
  72static struct list_head
  73ppc440spe_adma_chan_list = LIST_HEAD_INIT(ppc440spe_adma_chan_list);
  74
  75/* This flag is set when want to refetch the xor chain in the interrupt
  76 * handler
  77 */
  78static u32 do_xor_refetch;
  79
  80/* Pointer to DMA0, DMA1 CP/CS FIFO */
  81static void *ppc440spe_dma_fifo_buf;
  82
  83/* Pointers to last submitted to DMA0, DMA1 CDBs */
  84static struct ppc440spe_adma_desc_slot *chan_last_sub[3];
  85static struct ppc440spe_adma_desc_slot *chan_first_cdb[3];
  86
  87/* Pointer to last linked and submitted xor CB */
  88static struct ppc440spe_adma_desc_slot *xor_last_linked;
  89static struct ppc440spe_adma_desc_slot *xor_last_submit;
  90
  91/* This array is used in data-check operations for storing a pattern */
  92static char ppc440spe_qword[16];
  93
  94static atomic_t ppc440spe_adma_err_irq_ref;
  95static dcr_host_t ppc440spe_mq_dcr_host;
  96static unsigned int ppc440spe_mq_dcr_len;
  97
  98/* Since RXOR operations use the common register (MQ0_CF2H) for setting-up
  99 * the block size in transactions, then we do not allow to activate more than
 100 * only one RXOR transactions simultaneously. So use this var to store
 101 * the information about is RXOR currently active (PPC440SPE_RXOR_RUN bit is
 102 * set) or not (PPC440SPE_RXOR_RUN is clear).
 103 */
 104static unsigned long ppc440spe_rxor_state;
 105
 106/* These are used in enable & check routines
 107 */
 108static u32 ppc440spe_r6_enabled;
 109static struct ppc440spe_adma_chan *ppc440spe_r6_tchan;
 110static struct completion ppc440spe_r6_test_comp;
 111
 112static int ppc440spe_adma_dma2rxor_prep_src(
 113                struct ppc440spe_adma_desc_slot *desc,
 114                struct ppc440spe_rxor *cursor, int index,
 115                int src_cnt, u32 addr);
 116static void ppc440spe_adma_dma2rxor_set_src(
 117                struct ppc440spe_adma_desc_slot *desc,
 118                int index, dma_addr_t addr);
 119static void ppc440spe_adma_dma2rxor_set_mult(
 120                struct ppc440spe_adma_desc_slot *desc,
 121                int index, u8 mult);
 122
 123#ifdef ADMA_LL_DEBUG
 124#define ADMA_LL_DBG(x) ({ if (1) x; 0; })
 125#else
 126#define ADMA_LL_DBG(x) ({ if (0) x; 0; })
 127#endif
 128
 129static void print_cb(struct ppc440spe_adma_chan *chan, void *block)
 130{
 131        struct dma_cdb *cdb;
 132        struct xor_cb *cb;
 133        int i;
 134
 135        switch (chan->device->id) {
 136        case 0:
 137        case 1:
 138                cdb = block;
 139
 140                pr_debug("CDB at %p [%d]:\n"
 141                        "\t attr 0x%02x opc 0x%02x cnt 0x%08x\n"
 142                        "\t sg1u 0x%08x sg1l 0x%08x\n"
 143                        "\t sg2u 0x%08x sg2l 0x%08x\n"
 144                        "\t sg3u 0x%08x sg3l 0x%08x\n",
 145                        cdb, chan->device->id,
 146                        cdb->attr, cdb->opc, le32_to_cpu(cdb->cnt),
 147                        le32_to_cpu(cdb->sg1u), le32_to_cpu(cdb->sg1l),
 148                        le32_to_cpu(cdb->sg2u), le32_to_cpu(cdb->sg2l),
 149                        le32_to_cpu(cdb->sg3u), le32_to_cpu(cdb->sg3l)
 150                );
 151                break;
 152        case 2:
 153                cb = block;
 154
 155                pr_debug("CB at %p [%d]:\n"
 156                        "\t cbc 0x%08x cbbc 0x%08x cbs 0x%08x\n"
 157                        "\t cbtah 0x%08x cbtal 0x%08x\n"
 158                        "\t cblah 0x%08x cblal 0x%08x\n",
 159                        cb, chan->device->id,
 160                        cb->cbc, cb->cbbc, cb->cbs,
 161                        cb->cbtah, cb->cbtal,
 162                        cb->cblah, cb->cblal);
 163                for (i = 0; i < 16; i++) {
 164                        if (i && !cb->ops[i].h && !cb->ops[i].l)
 165                                continue;
 166                        pr_debug("\t ops[%2d]: h 0x%08x l 0x%08x\n",
 167                                i, cb->ops[i].h, cb->ops[i].l);
 168                }
 169                break;
 170        }
 171}
 172
 173static void print_cb_list(struct ppc440spe_adma_chan *chan,
 174                          struct ppc440spe_adma_desc_slot *iter)
 175{
 176        for (; iter; iter = iter->hw_next)
 177                print_cb(chan, iter->hw_desc);
 178}
 179
 180static void prep_dma_xor_dbg(int id, dma_addr_t dst, dma_addr_t *src,
 181                             unsigned int src_cnt)
 182{
 183        int i;
 184
 185        pr_debug("\n%s(%d):\nsrc: ", __func__, id);
 186        for (i = 0; i < src_cnt; i++)
 187                pr_debug("\t0x%016llx ", src[i]);
 188        pr_debug("dst:\n\t0x%016llx\n", dst);
 189}
 190
 191static void prep_dma_pq_dbg(int id, dma_addr_t *dst, dma_addr_t *src,
 192                            unsigned int src_cnt)
 193{
 194        int i;
 195
 196        pr_debug("\n%s(%d):\nsrc: ", __func__, id);
 197        for (i = 0; i < src_cnt; i++)
 198                pr_debug("\t0x%016llx ", src[i]);
 199        pr_debug("dst: ");
 200        for (i = 0; i < 2; i++)
 201                pr_debug("\t0x%016llx ", dst[i]);
 202}
 203
 204static void prep_dma_pqzero_sum_dbg(int id, dma_addr_t *src,
 205                                    unsigned int src_cnt,
 206                                    const unsigned char *scf)
 207{
 208        int i;
 209
 210        pr_debug("\n%s(%d):\nsrc(coef): ", __func__, id);
 211        if (scf) {
 212                for (i = 0; i < src_cnt; i++)
 213                        pr_debug("\t0x%016llx(0x%02x) ", src[i], scf[i]);
 214        } else {
 215                for (i = 0; i < src_cnt; i++)
 216                        pr_debug("\t0x%016llx(no) ", src[i]);
 217        }
 218
 219        pr_debug("dst: ");
 220        for (i = 0; i < 2; i++)
 221                pr_debug("\t0x%016llx ", src[src_cnt + i]);
 222}
 223
 224/******************************************************************************
 225 * Command (Descriptor) Blocks low-level routines
 226 ******************************************************************************/
 227/**
 228 * ppc440spe_desc_init_interrupt - initialize the descriptor for INTERRUPT
 229 * pseudo operation
 230 */
 231static void ppc440spe_desc_init_interrupt(struct ppc440spe_adma_desc_slot *desc,
 232                                          struct ppc440spe_adma_chan *chan)
 233{
 234        struct xor_cb *p;
 235
 236        switch (chan->device->id) {
 237        case PPC440SPE_XOR_ID:
 238                p = desc->hw_desc;
 239                memset(desc->hw_desc, 0, sizeof(struct xor_cb));
 240                /* NOP with Command Block Complete Enable */
 241                p->cbc = XOR_CBCR_CBCE_BIT;
 242                break;
 243        case PPC440SPE_DMA0_ID:
 244        case PPC440SPE_DMA1_ID:
 245                memset(desc->hw_desc, 0, sizeof(struct dma_cdb));
 246                /* NOP with interrupt */
 247                set_bit(PPC440SPE_DESC_INT, &desc->flags);
 248                break;
 249        default:
 250                printk(KERN_ERR "Unsupported id %d in %s\n", chan->device->id,
 251                                __func__);
 252                break;
 253        }
 254}
 255
 256/**
 257 * ppc440spe_desc_init_null_xor - initialize the descriptor for NULL XOR
 258 * pseudo operation
 259 */
 260static void ppc440spe_desc_init_null_xor(struct ppc440spe_adma_desc_slot *desc)
 261{
 262        memset(desc->hw_desc, 0, sizeof(struct xor_cb));
 263        desc->hw_next = NULL;
 264        desc->src_cnt = 0;
 265        desc->dst_cnt = 1;
 266}
 267
 268/**
 269 * ppc440spe_desc_init_xor - initialize the descriptor for XOR operation
 270 */
 271static void ppc440spe_desc_init_xor(struct ppc440spe_adma_desc_slot *desc,
 272                                         int src_cnt, unsigned long flags)
 273{
 274        struct xor_cb *hw_desc = desc->hw_desc;
 275
 276        memset(desc->hw_desc, 0, sizeof(struct xor_cb));
 277        desc->hw_next = NULL;
 278        desc->src_cnt = src_cnt;
 279        desc->dst_cnt = 1;
 280
 281        hw_desc->cbc = XOR_CBCR_TGT_BIT | src_cnt;
 282        if (flags & DMA_PREP_INTERRUPT)
 283                /* Enable interrupt on completion */
 284                hw_desc->cbc |= XOR_CBCR_CBCE_BIT;
 285}
 286
 287/**
 288 * ppc440spe_desc_init_dma2pq - initialize the descriptor for PQ
 289 * operation in DMA2 controller
 290 */
 291static void ppc440spe_desc_init_dma2pq(struct ppc440spe_adma_desc_slot *desc,
 292                int dst_cnt, int src_cnt, unsigned long flags)
 293{
 294        struct xor_cb *hw_desc = desc->hw_desc;
 295
 296        memset(desc->hw_desc, 0, sizeof(struct xor_cb));
 297        desc->hw_next = NULL;
 298        desc->src_cnt = src_cnt;
 299        desc->dst_cnt = dst_cnt;
 300        memset(desc->reverse_flags, 0, sizeof(desc->reverse_flags));
 301        desc->descs_per_op = 0;
 302
 303        hw_desc->cbc = XOR_CBCR_TGT_BIT;
 304        if (flags & DMA_PREP_INTERRUPT)
 305                /* Enable interrupt on completion */
 306                hw_desc->cbc |= XOR_CBCR_CBCE_BIT;
 307}
 308
 309#define DMA_CTRL_FLAGS_LAST     DMA_PREP_FENCE
 310#define DMA_PREP_ZERO_P         (DMA_CTRL_FLAGS_LAST << 1)
 311#define DMA_PREP_ZERO_Q         (DMA_PREP_ZERO_P << 1)
 312
 313/**
 314 * ppc440spe_desc_init_dma01pq - initialize the descriptors for PQ operation
 315 * with DMA0/1
 316 */
 317static void ppc440spe_desc_init_dma01pq(struct ppc440spe_adma_desc_slot *desc,
 318                                int dst_cnt, int src_cnt, unsigned long flags,
 319                                unsigned long op)
 320{
 321        struct dma_cdb *hw_desc;
 322        struct ppc440spe_adma_desc_slot *iter;
 323        u8 dopc;
 324
 325        /* Common initialization of a PQ descriptors chain */
 326        set_bits(op, &desc->flags);
 327        desc->src_cnt = src_cnt;
 328        desc->dst_cnt = dst_cnt;
 329
 330        /* WXOR MULTICAST if both P and Q are being computed
 331         * MV_SG1_SG2 if Q only
 332         */
 333        dopc = (desc->dst_cnt == DMA_DEST_MAX_NUM) ?
 334                DMA_CDB_OPC_MULTICAST : DMA_CDB_OPC_MV_SG1_SG2;
 335
 336        list_for_each_entry(iter, &desc->group_list, chain_node) {
 337                hw_desc = iter->hw_desc;
 338                memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
 339
 340                if (likely(!list_is_last(&iter->chain_node,
 341                                &desc->group_list))) {
 342                        /* set 'next' pointer */
 343                        iter->hw_next = list_entry(iter->chain_node.next,
 344                                struct ppc440spe_adma_desc_slot, chain_node);
 345                        clear_bit(PPC440SPE_DESC_INT, &iter->flags);
 346                } else {
 347                        /* this is the last descriptor.
 348                         * this slot will be pasted from ADMA level
 349                         * each time it wants to configure parameters
 350                         * of the transaction (src, dst, ...)
 351                         */
 352                        iter->hw_next = NULL;
 353                        if (flags & DMA_PREP_INTERRUPT)
 354                                set_bit(PPC440SPE_DESC_INT, &iter->flags);
 355                        else
 356                                clear_bit(PPC440SPE_DESC_INT, &iter->flags);
 357                }
 358        }
 359
 360        /* Set OPS depending on WXOR/RXOR type of operation */
 361        if (!test_bit(PPC440SPE_DESC_RXOR, &desc->flags)) {
 362                /* This is a WXOR only chain:
 363                 * - first descriptors are for zeroing destinations
 364                 *   if PPC440SPE_ZERO_P/Q set;
 365                 * - descriptors remained are for GF-XOR operations.
 366                 */
 367                iter = list_first_entry(&desc->group_list,
 368                                        struct ppc440spe_adma_desc_slot,
 369                                        chain_node);
 370
 371                if (test_bit(PPC440SPE_ZERO_P, &desc->flags)) {
 372                        hw_desc = iter->hw_desc;
 373                        hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
 374                        iter = list_first_entry(&iter->chain_node,
 375                                        struct ppc440spe_adma_desc_slot,
 376                                        chain_node);
 377                }
 378
 379                if (test_bit(PPC440SPE_ZERO_Q, &desc->flags)) {
 380                        hw_desc = iter->hw_desc;
 381                        hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
 382                        iter = list_first_entry(&iter->chain_node,
 383                                        struct ppc440spe_adma_desc_slot,
 384                                        chain_node);
 385                }
 386
 387                list_for_each_entry_from(iter, &desc->group_list, chain_node) {
 388                        hw_desc = iter->hw_desc;
 389                        hw_desc->opc = dopc;
 390                }
 391        } else {
 392                /* This is either RXOR-only or mixed RXOR/WXOR */
 393
 394                /* The first 1 or 2 slots in chain are always RXOR,
 395                 * if need to calculate P & Q, then there are two
 396                 * RXOR slots; if only P or only Q, then there is one
 397                 */
 398                iter = list_first_entry(&desc->group_list,
 399                                        struct ppc440spe_adma_desc_slot,
 400                                        chain_node);
 401                hw_desc = iter->hw_desc;
 402                hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
 403
 404                if (desc->dst_cnt == DMA_DEST_MAX_NUM) {
 405                        iter = list_first_entry(&iter->chain_node,
 406                                                struct ppc440spe_adma_desc_slot,
 407                                                chain_node);
 408                        hw_desc = iter->hw_desc;
 409                        hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
 410                }
 411
 412                /* The remaining descs (if any) are WXORs */
 413                if (test_bit(PPC440SPE_DESC_WXOR, &desc->flags)) {
 414                        iter = list_first_entry(&iter->chain_node,
 415                                                struct ppc440spe_adma_desc_slot,
 416                                                chain_node);
 417                        list_for_each_entry_from(iter, &desc->group_list,
 418                                                chain_node) {
 419                                hw_desc = iter->hw_desc;
 420                                hw_desc->opc = dopc;
 421                        }
 422                }
 423        }
 424}
 425
 426/**
 427 * ppc440spe_desc_init_dma01pqzero_sum - initialize the descriptor
 428 * for PQ_ZERO_SUM operation
 429 */
 430static void ppc440spe_desc_init_dma01pqzero_sum(
 431                                struct ppc440spe_adma_desc_slot *desc,
 432                                int dst_cnt, int src_cnt)
 433{
 434        struct dma_cdb *hw_desc;
 435        struct ppc440spe_adma_desc_slot *iter;
 436        int i = 0;
 437        u8 dopc = (dst_cnt == 2) ? DMA_CDB_OPC_MULTICAST :
 438                                   DMA_CDB_OPC_MV_SG1_SG2;
 439        /*
 440         * Initialize starting from 2nd or 3rd descriptor dependent
 441         * on dst_cnt. First one or two slots are for cloning P
 442         * and/or Q to chan->pdest and/or chan->qdest as we have
 443         * to preserve original P/Q.
 444         */
 445        iter = list_first_entry(&desc->group_list,
 446                                struct ppc440spe_adma_desc_slot, chain_node);
 447        iter = list_entry(iter->chain_node.next,
 448                          struct ppc440spe_adma_desc_slot, chain_node);
 449
 450        if (dst_cnt > 1) {
 451                iter = list_entry(iter->chain_node.next,
 452                                  struct ppc440spe_adma_desc_slot, chain_node);
 453        }
 454        /* initialize each source descriptor in chain */
 455        list_for_each_entry_from(iter, &desc->group_list, chain_node) {
 456                hw_desc = iter->hw_desc;
 457                memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
 458                iter->src_cnt = 0;
 459                iter->dst_cnt = 0;
 460
 461                /* This is a ZERO_SUM operation:
 462                 * - <src_cnt> descriptors starting from 2nd or 3rd
 463                 *   descriptor are for GF-XOR operations;
 464                 * - remaining <dst_cnt> descriptors are for checking the result
 465                 */
 466                if (i++ < src_cnt)
 467                        /* MV_SG1_SG2 if only Q is being verified
 468                         * MULTICAST if both P and Q are being verified
 469                         */
 470                        hw_desc->opc = dopc;
 471                else
 472                        /* DMA_CDB_OPC_DCHECK128 operation */
 473                        hw_desc->opc = DMA_CDB_OPC_DCHECK128;
 474
 475                if (likely(!list_is_last(&iter->chain_node,
 476                                         &desc->group_list))) {
 477                        /* set 'next' pointer */
 478                        iter->hw_next = list_entry(iter->chain_node.next,
 479                                                struct ppc440spe_adma_desc_slot,
 480                                                chain_node);
 481                } else {
 482                        /* this is the last descriptor.
 483                         * this slot will be pasted from ADMA level
 484                         * each time it wants to configure parameters
 485                         * of the transaction (src, dst, ...)
 486                         */
 487                        iter->hw_next = NULL;
 488                        /* always enable interrupt generation since we get
 489                         * the status of pqzero from the handler
 490                         */
 491                        set_bit(PPC440SPE_DESC_INT, &iter->flags);
 492                }
 493        }
 494        desc->src_cnt = src_cnt;
 495        desc->dst_cnt = dst_cnt;
 496}
 497
 498/**
 499 * ppc440spe_desc_init_memcpy - initialize the descriptor for MEMCPY operation
 500 */
 501static void ppc440spe_desc_init_memcpy(struct ppc440spe_adma_desc_slot *desc,
 502                                        unsigned long flags)
 503{
 504        struct dma_cdb *hw_desc = desc->hw_desc;
 505
 506        memset(desc->hw_desc, 0, sizeof(struct dma_cdb));
 507        desc->hw_next = NULL;
 508        desc->src_cnt = 1;
 509        desc->dst_cnt = 1;
 510
 511        if (flags & DMA_PREP_INTERRUPT)
 512                set_bit(PPC440SPE_DESC_INT, &desc->flags);
 513        else
 514                clear_bit(PPC440SPE_DESC_INT, &desc->flags);
 515
 516        hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
 517}
 518
 519/**
 520 * ppc440spe_desc_set_src_addr - set source address into the descriptor
 521 */
 522static void ppc440spe_desc_set_src_addr(struct ppc440spe_adma_desc_slot *desc,
 523                                        struct ppc440spe_adma_chan *chan,
 524                                        int src_idx, dma_addr_t addrh,
 525                                        dma_addr_t addrl)
 526{
 527        struct dma_cdb *dma_hw_desc;
 528        struct xor_cb *xor_hw_desc;
 529        phys_addr_t addr64, tmplow, tmphi;
 530
 531        switch (chan->device->id) {
 532        case PPC440SPE_DMA0_ID:
 533        case PPC440SPE_DMA1_ID:
 534                if (!addrh) {
 535                        addr64 = addrl;
 536                        tmphi = (addr64 >> 32);
 537                        tmplow = (addr64 & 0xFFFFFFFF);
 538                } else {
 539                        tmphi = addrh;
 540                        tmplow = addrl;
 541                }
 542                dma_hw_desc = desc->hw_desc;
 543                dma_hw_desc->sg1l = cpu_to_le32((u32)tmplow);
 544                dma_hw_desc->sg1u |= cpu_to_le32((u32)tmphi);
 545                break;
 546        case PPC440SPE_XOR_ID:
 547                xor_hw_desc = desc->hw_desc;
 548                xor_hw_desc->ops[src_idx].l = addrl;
 549                xor_hw_desc->ops[src_idx].h |= addrh;
 550                break;
 551        }
 552}
 553
 554/**
 555 * ppc440spe_desc_set_src_mult - set source address mult into the descriptor
 556 */
 557static void ppc440spe_desc_set_src_mult(struct ppc440spe_adma_desc_slot *desc,
 558                        struct ppc440spe_adma_chan *chan, u32 mult_index,
 559                        int sg_index, unsigned char mult_value)
 560{
 561        struct dma_cdb *dma_hw_desc;
 562        u32 *psgu;
 563
 564        switch (chan->device->id) {
 565        case PPC440SPE_DMA0_ID:
 566        case PPC440SPE_DMA1_ID:
 567                dma_hw_desc = desc->hw_desc;
 568
 569                switch (sg_index) {
 570                /* for RXOR operations set multiplier
 571                 * into source cued address
 572                 */
 573                case DMA_CDB_SG_SRC:
 574                        psgu = &dma_hw_desc->sg1u;
 575                        break;
 576                /* for WXOR operations set multiplier
 577                 * into destination cued address(es)
 578                 */
 579                case DMA_CDB_SG_DST1:
 580                        psgu = &dma_hw_desc->sg2u;
 581                        break;
 582                case DMA_CDB_SG_DST2:
 583                        psgu = &dma_hw_desc->sg3u;
 584                        break;
 585                default:
 586                        BUG();
 587                }
 588
 589                *psgu |= cpu_to_le32(mult_value << mult_index);
 590                break;
 591        case PPC440SPE_XOR_ID:
 592                break;
 593        default:
 594                BUG();
 595        }
 596}
 597
 598/**
 599 * ppc440spe_desc_set_dest_addr - set destination address into the descriptor
 600 */
 601static void ppc440spe_desc_set_dest_addr(struct ppc440spe_adma_desc_slot *desc,
 602                                struct ppc440spe_adma_chan *chan,
 603                                dma_addr_t addrh, dma_addr_t addrl,
 604                                u32 dst_idx)
 605{
 606        struct dma_cdb *dma_hw_desc;
 607        struct xor_cb *xor_hw_desc;
 608        phys_addr_t addr64, tmphi, tmplow;
 609        u32 *psgu, *psgl;
 610
 611        switch (chan->device->id) {
 612        case PPC440SPE_DMA0_ID:
 613        case PPC440SPE_DMA1_ID:
 614                if (!addrh) {
 615                        addr64 = addrl;
 616                        tmphi = (addr64 >> 32);
 617                        tmplow = (addr64 & 0xFFFFFFFF);
 618                } else {
 619                        tmphi = addrh;
 620                        tmplow = addrl;
 621                }
 622                dma_hw_desc = desc->hw_desc;
 623
 624                psgu = dst_idx ? &dma_hw_desc->sg3u : &dma_hw_desc->sg2u;
 625                psgl = dst_idx ? &dma_hw_desc->sg3l : &dma_hw_desc->sg2l;
 626
 627                *psgl = cpu_to_le32((u32)tmplow);
 628                *psgu |= cpu_to_le32((u32)tmphi);
 629                break;
 630        case PPC440SPE_XOR_ID:
 631                xor_hw_desc = desc->hw_desc;
 632                xor_hw_desc->cbtal = addrl;
 633                xor_hw_desc->cbtah |= addrh;
 634                break;
 635        }
 636}
 637
 638/**
 639 * ppc440spe_desc_set_byte_count - set number of data bytes involved
 640 * into the operation
 641 */
 642static void ppc440spe_desc_set_byte_count(struct ppc440spe_adma_desc_slot *desc,
 643                                struct ppc440spe_adma_chan *chan,
 644                                u32 byte_count)
 645{
 646        struct dma_cdb *dma_hw_desc;
 647        struct xor_cb *xor_hw_desc;
 648
 649        switch (chan->device->id) {
 650        case PPC440SPE_DMA0_ID:
 651        case PPC440SPE_DMA1_ID:
 652                dma_hw_desc = desc->hw_desc;
 653                dma_hw_desc->cnt = cpu_to_le32(byte_count);
 654                break;
 655        case PPC440SPE_XOR_ID:
 656                xor_hw_desc = desc->hw_desc;
 657                xor_hw_desc->cbbc = byte_count;
 658                break;
 659        }
 660}
 661
 662/**
 663 * ppc440spe_desc_set_rxor_block_size - set RXOR block size
 664 */
 665static inline void ppc440spe_desc_set_rxor_block_size(u32 byte_count)
 666{
 667        /* assume that byte_count is aligned on the 512-boundary;
 668         * thus write it directly to the register (bits 23:31 are
 669         * reserved there).
 670         */
 671        dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_CF2H, byte_count);
 672}
 673
 674/**
 675 * ppc440spe_desc_set_dcheck - set CHECK pattern
 676 */
 677static void ppc440spe_desc_set_dcheck(struct ppc440spe_adma_desc_slot *desc,
 678                                struct ppc440spe_adma_chan *chan, u8 *qword)
 679{
 680        struct dma_cdb *dma_hw_desc;
 681
 682        switch (chan->device->id) {
 683        case PPC440SPE_DMA0_ID:
 684        case PPC440SPE_DMA1_ID:
 685                dma_hw_desc = desc->hw_desc;
 686                iowrite32(qword[0], &dma_hw_desc->sg3l);
 687                iowrite32(qword[4], &dma_hw_desc->sg3u);
 688                iowrite32(qword[8], &dma_hw_desc->sg2l);
 689                iowrite32(qword[12], &dma_hw_desc->sg2u);
 690                break;
 691        default:
 692                BUG();
 693        }
 694}
 695
 696/**
 697 * ppc440spe_xor_set_link - set link address in xor CB
 698 */
 699static void ppc440spe_xor_set_link(struct ppc440spe_adma_desc_slot *prev_desc,
 700                                struct ppc440spe_adma_desc_slot *next_desc)
 701{
 702        struct xor_cb *xor_hw_desc = prev_desc->hw_desc;
 703
 704        if (unlikely(!next_desc || !(next_desc->phys))) {
 705                printk(KERN_ERR "%s: next_desc=0x%p; next_desc->phys=0x%llx\n",
 706                        __func__, next_desc,
 707                        next_desc ? next_desc->phys : 0);
 708                BUG();
 709        }
 710
 711        xor_hw_desc->cbs = 0;
 712        xor_hw_desc->cblal = next_desc->phys;
 713        xor_hw_desc->cblah = 0;
 714        xor_hw_desc->cbc |= XOR_CBCR_LNK_BIT;
 715}
 716
 717/**
 718 * ppc440spe_desc_set_link - set the address of descriptor following this
 719 * descriptor in chain
 720 */
 721static void ppc440spe_desc_set_link(struct ppc440spe_adma_chan *chan,
 722                                struct ppc440spe_adma_desc_slot *prev_desc,
 723                                struct ppc440spe_adma_desc_slot *next_desc)
 724{
 725        unsigned long flags;
 726        struct ppc440spe_adma_desc_slot *tail = next_desc;
 727
 728        if (unlikely(!prev_desc || !next_desc ||
 729                (prev_desc->hw_next && prev_desc->hw_next != next_desc))) {
 730                /* If previous next is overwritten something is wrong.
 731                 * though we may refetch from append to initiate list
 732                 * processing; in this case - it's ok.
 733                 */
 734                printk(KERN_ERR "%s: prev_desc=0x%p; next_desc=0x%p; "
 735                        "prev->hw_next=0x%p\n", __func__, prev_desc,
 736                        next_desc, prev_desc ? prev_desc->hw_next : 0);
 737                BUG();
 738        }
 739
 740        local_irq_save(flags);
 741
 742        /* do s/w chaining both for DMA and XOR descriptors */
 743        prev_desc->hw_next = next_desc;
 744
 745        switch (chan->device->id) {
 746        case PPC440SPE_DMA0_ID:
 747        case PPC440SPE_DMA1_ID:
 748                break;
 749        case PPC440SPE_XOR_ID:
 750                /* bind descriptor to the chain */
 751                while (tail->hw_next)
 752                        tail = tail->hw_next;
 753                xor_last_linked = tail;
 754
 755                if (prev_desc == xor_last_submit)
 756                        /* do not link to the last submitted CB */
 757                        break;
 758                ppc440spe_xor_set_link(prev_desc, next_desc);
 759                break;
 760        }
 761
 762        local_irq_restore(flags);
 763}
 764
 765/**
 766 * ppc440spe_desc_get_link - get the address of the descriptor that
 767 * follows this one
 768 */
 769static inline u32 ppc440spe_desc_get_link(struct ppc440spe_adma_desc_slot *desc,
 770                                        struct ppc440spe_adma_chan *chan)
 771{
 772        if (!desc->hw_next)
 773                return 0;
 774
 775        return desc->hw_next->phys;
 776}
 777
 778/**
 779 * ppc440spe_desc_is_aligned - check alignment
 780 */
 781static inline int ppc440spe_desc_is_aligned(
 782        struct ppc440spe_adma_desc_slot *desc, int num_slots)
 783{
 784        return (desc->idx & (num_slots - 1)) ? 0 : 1;
 785}
 786
 787/**
 788 * ppc440spe_chan_xor_slot_count - get the number of slots necessary for
 789 * XOR operation
 790 */
 791static int ppc440spe_chan_xor_slot_count(size_t len, int src_cnt,
 792                        int *slots_per_op)
 793{
 794        int slot_cnt;
 795
 796        /* each XOR descriptor provides up to 16 source operands */
 797        slot_cnt = *slots_per_op = (src_cnt + XOR_MAX_OPS - 1)/XOR_MAX_OPS;
 798
 799        if (likely(len <= PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT))
 800                return slot_cnt;
 801
 802        printk(KERN_ERR "%s: len %d > max %d !!\n",
 803                __func__, len, PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT);
 804        BUG();
 805        return slot_cnt;
 806}
 807
 808/**
 809 * ppc440spe_dma2_pq_slot_count - get the number of slots necessary for
 810 * DMA2 PQ operation
 811 */
 812static int ppc440spe_dma2_pq_slot_count(dma_addr_t *srcs,
 813                int src_cnt, size_t len)
 814{
 815        signed long long order = 0;
 816        int state = 0;
 817        int addr_count = 0;
 818        int i;
 819        for (i = 1; i < src_cnt; i++) {
 820                dma_addr_t cur_addr = srcs[i];
 821                dma_addr_t old_addr = srcs[i-1];
 822                switch (state) {
 823                case 0:
 824                        if (cur_addr == old_addr + len) {
 825                                /* direct RXOR */
 826                                order = 1;
 827                                state = 1;
 828                                if (i == src_cnt-1)
 829                                        addr_count++;
 830                        } else if (old_addr == cur_addr + len) {
 831                                /* reverse RXOR */
 832                                order = -1;
 833                                state = 1;
 834                                if (i == src_cnt-1)
 835                                        addr_count++;
 836                        } else {
 837                                state = 3;
 838                        }
 839                        break;
 840                case 1:
 841                        if (i == src_cnt-2 || (order == -1
 842                                && cur_addr != old_addr - len)) {
 843                                order = 0;
 844                                state = 0;
 845                                addr_count++;
 846                        } else if (cur_addr == old_addr + len*order) {
 847                                state = 2;
 848                                if (i == src_cnt-1)
 849                                        addr_count++;
 850                        } else if (cur_addr == old_addr + 2*len) {
 851                                state = 2;
 852                                if (i == src_cnt-1)
 853                                        addr_count++;
 854                        } else if (cur_addr == old_addr + 3*len) {
 855                                state = 2;
 856                                if (i == src_cnt-1)
 857                                        addr_count++;
 858                        } else {
 859                                order = 0;
 860                                state = 0;
 861                                addr_count++;
 862                        }
 863                        break;
 864                case 2:
 865                        order = 0;
 866                        state = 0;
 867                        addr_count++;
 868                                break;
 869                }
 870                if (state == 3)
 871                        break;
 872        }
 873        if (src_cnt <= 1 || (state != 1 && state != 2)) {
 874                pr_err("%s: src_cnt=%d, state=%d, addr_count=%d, order=%lld\n",
 875                        __func__, src_cnt, state, addr_count, order);
 876                for (i = 0; i < src_cnt; i++)
 877                        pr_err("\t[%d] 0x%llx \n", i, srcs[i]);
 878                BUG();
 879        }
 880
 881        return (addr_count + XOR_MAX_OPS - 1) / XOR_MAX_OPS;
 882}
 883
 884
 885/******************************************************************************
 886 * ADMA channel low-level routines
 887 ******************************************************************************/
 888
 889static u32
 890ppc440spe_chan_get_current_descriptor(struct ppc440spe_adma_chan *chan);
 891static void ppc440spe_chan_append(struct ppc440spe_adma_chan *chan);
 892
 893/**
 894 * ppc440spe_adma_device_clear_eot_status - interrupt ack to XOR or DMA engine
 895 */
 896static void ppc440spe_adma_device_clear_eot_status(
 897                                        struct ppc440spe_adma_chan *chan)
 898{
 899        struct dma_regs *dma_reg;
 900        struct xor_regs *xor_reg;
 901        u8 *p = chan->device->dma_desc_pool_virt;
 902        struct dma_cdb *cdb;
 903        u32 rv, i;
 904
 905        switch (chan->device->id) {
 906        case PPC440SPE_DMA0_ID:
 907        case PPC440SPE_DMA1_ID:
 908                /* read FIFO to ack */
 909                dma_reg = chan->device->dma_reg;
 910                while ((rv = ioread32(&dma_reg->csfpl))) {
 911                        i = rv & DMA_CDB_ADDR_MSK;
 912                        cdb = (struct dma_cdb *)&p[i -
 913                            (u32)chan->device->dma_desc_pool];
 914
 915                        /* Clear opcode to ack. This is necessary for
 916                         * ZeroSum operations only
 917                         */
 918                        cdb->opc = 0;
 919
 920                        if (test_bit(PPC440SPE_RXOR_RUN,
 921                            &ppc440spe_rxor_state)) {
 922                                /* probably this is a completed RXOR op,
 923                                 * get pointer to CDB using the fact that
 924                                 * physical and virtual addresses of CDB
 925                                 * in pools have the same offsets
 926                                 */
 927                                if (le32_to_cpu(cdb->sg1u) &
 928                                    DMA_CUED_XOR_BASE) {
 929                                        /* this is a RXOR */
 930                                        clear_bit(PPC440SPE_RXOR_RUN,
 931                                                  &ppc440spe_rxor_state);
 932                                }
 933                        }
 934
 935                        if (rv & DMA_CDB_STATUS_MSK) {
 936                                /* ZeroSum check failed
 937                                 */
 938                                struct ppc440spe_adma_desc_slot *iter;
 939                                dma_addr_t phys = rv & ~DMA_CDB_MSK;
 940
 941                                /*
 942                                 * Update the status of corresponding
 943                                 * descriptor.
 944                                 */
 945                                list_for_each_entry(iter, &chan->chain,
 946                                    chain_node) {
 947                                        if (iter->phys == phys)
 948                                                break;
 949                                }
 950                                /*
 951                                 * if cannot find the corresponding
 952                                 * slot it's a bug
 953                                 */
 954                                BUG_ON(&iter->chain_node == &chan->chain);
 955
 956                                if (iter->xor_check_result) {
 957                                        if (test_bit(PPC440SPE_DESC_PCHECK,
 958                                                     &iter->flags)) {
 959                                                *iter->xor_check_result |=
 960                                                        SUM_CHECK_P_RESULT;
 961                                        } else
 962                                        if (test_bit(PPC440SPE_DESC_QCHECK,
 963                                                     &iter->flags)) {
 964                                                *iter->xor_check_result |=
 965                                                        SUM_CHECK_Q_RESULT;
 966                                        } else
 967                                                BUG();
 968                                }
 969                        }
 970                }
 971
 972                rv = ioread32(&dma_reg->dsts);
 973                if (rv) {
 974                        pr_err("DMA%d err status: 0x%x\n",
 975                               chan->device->id, rv);
 976                        /* write back to clear */
 977                        iowrite32(rv, &dma_reg->dsts);
 978                }
 979                break;
 980        case PPC440SPE_XOR_ID:
 981                /* reset status bits to ack */
 982                xor_reg = chan->device->xor_reg;
 983                rv = ioread32be(&xor_reg->sr);
 984                iowrite32be(rv, &xor_reg->sr);
 985
 986                if (rv & (XOR_IE_ICBIE_BIT|XOR_IE_ICIE_BIT|XOR_IE_RPTIE_BIT)) {
 987                        if (rv & XOR_IE_RPTIE_BIT) {
 988                                /* Read PLB Timeout Error.
 989                                 * Try to resubmit the CB
 990                                 */
 991                                u32 val = ioread32be(&xor_reg->ccbalr);
 992
 993                                iowrite32be(val, &xor_reg->cblalr);
 994
 995                                val = ioread32be(&xor_reg->crsr);
 996                                iowrite32be(val | XOR_CRSR_XAE_BIT,
 997                                            &xor_reg->crsr);
 998                        } else
 999                                pr_err("XOR ERR 0x%x status\n", rv);
1000                        break;
1001                }
1002
1003                /*  if the XORcore is idle, but there are unprocessed CBs
1004                 * then refetch the s/w chain here
1005                 */
1006                if (!(ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT) &&
1007                    do_xor_refetch)
1008                        ppc440spe_chan_append(chan);
1009                break;
1010        }
1011}
1012
1013/**
1014 * ppc440spe_chan_is_busy - get the channel status
1015 */
1016static int ppc440spe_chan_is_busy(struct ppc440spe_adma_chan *chan)
1017{
1018        struct dma_regs *dma_reg;
1019        struct xor_regs *xor_reg;
1020        int busy = 0;
1021
1022        switch (chan->device->id) {
1023        case PPC440SPE_DMA0_ID:
1024        case PPC440SPE_DMA1_ID:
1025                dma_reg = chan->device->dma_reg;
1026                /*  if command FIFO's head and tail pointers are equal and
1027                 * status tail is the same as command, then channel is free
1028                 */
1029                if (ioread16(&dma_reg->cpfhp) != ioread16(&dma_reg->cpftp) ||
1030                    ioread16(&dma_reg->cpftp) != ioread16(&dma_reg->csftp))
1031                        busy = 1;
1032                break;
1033        case PPC440SPE_XOR_ID:
1034                /* use the special status bit for the XORcore
1035                 */
1036                xor_reg = chan->device->xor_reg;
1037                busy = (ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT) ? 1 : 0;
1038                break;
1039        }
1040
1041        return busy;
1042}
1043
1044/**
1045 * ppc440spe_chan_set_first_xor_descriptor -  init XORcore chain
1046 */
1047static void ppc440spe_chan_set_first_xor_descriptor(
1048                                struct ppc440spe_adma_chan *chan,
1049                                struct ppc440spe_adma_desc_slot *next_desc)
1050{
1051        struct xor_regs *xor_reg = chan->device->xor_reg;
1052
1053        if (ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT)
1054                printk(KERN_INFO "%s: Warn: XORcore is running "
1055                        "when try to set the first CDB!\n",
1056                        __func__);
1057
1058        xor_last_submit = xor_last_linked = next_desc;
1059
1060        iowrite32be(XOR_CRSR_64BA_BIT, &xor_reg->crsr);
1061
1062        iowrite32be(next_desc->phys, &xor_reg->cblalr);
1063        iowrite32be(0, &xor_reg->cblahr);
1064        iowrite32be(ioread32be(&xor_reg->cbcr) | XOR_CBCR_LNK_BIT,
1065                    &xor_reg->cbcr);
1066
1067        chan->hw_chain_inited = 1;
1068}
1069
1070/**
1071 * ppc440spe_dma_put_desc - put DMA0,1 descriptor to FIFO.
1072 * called with irqs disabled
1073 */
1074static void ppc440spe_dma_put_desc(struct ppc440spe_adma_chan *chan,
1075                struct ppc440spe_adma_desc_slot *desc)
1076{
1077        u32 pcdb;
1078        struct dma_regs *dma_reg = chan->device->dma_reg;
1079
1080        pcdb = desc->phys;
1081        if (!test_bit(PPC440SPE_DESC_INT, &desc->flags))
1082                pcdb |= DMA_CDB_NO_INT;
1083
1084        chan_last_sub[chan->device->id] = desc;
1085
1086        ADMA_LL_DBG(print_cb(chan, desc->hw_desc));
1087
1088        iowrite32(pcdb, &dma_reg->cpfpl);
1089}
1090
1091/**
1092 * ppc440spe_chan_append - update the h/w chain in the channel
1093 */
1094static void ppc440spe_chan_append(struct ppc440spe_adma_chan *chan)
1095{
1096        struct xor_regs *xor_reg;
1097        struct ppc440spe_adma_desc_slot *iter;
1098        struct xor_cb *xcb;
1099        u32 cur_desc;
1100        unsigned long flags;
1101
1102        local_irq_save(flags);
1103
1104        switch (chan->device->id) {
1105        case PPC440SPE_DMA0_ID:
1106        case PPC440SPE_DMA1_ID:
1107                cur_desc = ppc440spe_chan_get_current_descriptor(chan);
1108
1109                if (likely(cur_desc)) {
1110                        iter = chan_last_sub[chan->device->id];
1111                        BUG_ON(!iter);
1112                } else {
1113                        /* first peer */
1114                        iter = chan_first_cdb[chan->device->id];
1115                        BUG_ON(!iter);
1116                        ppc440spe_dma_put_desc(chan, iter);
1117                        chan->hw_chain_inited = 1;
1118                }
1119
1120                /* is there something new to append */
1121                if (!iter->hw_next)
1122                        break;
1123
1124                /* flush descriptors from the s/w queue to fifo */
1125                list_for_each_entry_continue(iter, &chan->chain, chain_node) {
1126                        ppc440spe_dma_put_desc(chan, iter);
1127                        if (!iter->hw_next)
1128                                break;
1129                }
1130                break;
1131        case PPC440SPE_XOR_ID:
1132                /* update h/w links and refetch */
1133                if (!xor_last_submit->hw_next)
1134                        break;
1135
1136                xor_reg = chan->device->xor_reg;
1137                /* the last linked CDB has to generate an interrupt
1138                 * that we'd be able to append the next lists to h/w
1139                 * regardless of the XOR engine state at the moment of
1140                 * appending of these next lists
1141                 */
1142                xcb = xor_last_linked->hw_desc;
1143                xcb->cbc |= XOR_CBCR_CBCE_BIT;
1144
1145                if (!(ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT)) {
1146                        /* XORcore is idle. Refetch now */
1147                        do_xor_refetch = 0;
1148                        ppc440spe_xor_set_link(xor_last_submit,
1149                                xor_last_submit->hw_next);
1150
1151                        ADMA_LL_DBG(print_cb_list(chan,
1152                                xor_last_submit->hw_next));
1153
1154                        xor_last_submit = xor_last_linked;
1155                        iowrite32be(ioread32be(&xor_reg->crsr) |
1156                                    XOR_CRSR_RCBE_BIT | XOR_CRSR_64BA_BIT,
1157                                    &xor_reg->crsr);
1158                } else {
1159                        /* XORcore is running. Refetch later in the handler */
1160                        do_xor_refetch = 1;
1161                }
1162
1163                break;
1164        }
1165
1166        local_irq_restore(flags);
1167}
1168
1169/**
1170 * ppc440spe_chan_get_current_descriptor - get the currently executed descriptor
1171 */
1172static u32
1173ppc440spe_chan_get_current_descriptor(struct ppc440spe_adma_chan *chan)
1174{
1175        struct dma_regs *dma_reg;
1176        struct xor_regs *xor_reg;
1177
1178        if (unlikely(!chan->hw_chain_inited))
1179                /* h/w descriptor chain is not initialized yet */
1180                return 0;
1181
1182        switch (chan->device->id) {
1183        case PPC440SPE_DMA0_ID:
1184        case PPC440SPE_DMA1_ID:
1185                dma_reg = chan->device->dma_reg;
1186                return ioread32(&dma_reg->acpl) & (~DMA_CDB_MSK);
1187        case PPC440SPE_XOR_ID:
1188                xor_reg = chan->device->xor_reg;
1189                return ioread32be(&xor_reg->ccbalr);
1190        }
1191        return 0;
1192}
1193
1194/**
1195 * ppc440spe_chan_run - enable the channel
1196 */
1197static void ppc440spe_chan_run(struct ppc440spe_adma_chan *chan)
1198{
1199        struct xor_regs *xor_reg;
1200
1201        switch (chan->device->id) {
1202        case PPC440SPE_DMA0_ID:
1203        case PPC440SPE_DMA1_ID:
1204                /* DMAs are always enabled, do nothing */
1205                break;
1206        case PPC440SPE_XOR_ID:
1207                /* drain write buffer */
1208                xor_reg = chan->device->xor_reg;
1209
1210                /* fetch descriptor pointed to in <link> */
1211                iowrite32be(XOR_CRSR_64BA_BIT | XOR_CRSR_XAE_BIT,
1212                            &xor_reg->crsr);
1213                break;
1214        }
1215}
1216
1217/******************************************************************************
1218 * ADMA device level
1219 ******************************************************************************/
1220
1221static void ppc440spe_chan_start_null_xor(struct ppc440spe_adma_chan *chan);
1222static int ppc440spe_adma_alloc_chan_resources(struct dma_chan *chan);
1223
1224static dma_cookie_t
1225ppc440spe_adma_tx_submit(struct dma_async_tx_descriptor *tx);
1226
1227static void ppc440spe_adma_set_dest(struct ppc440spe_adma_desc_slot *tx,
1228                                    dma_addr_t addr, int index);
1229static void
1230ppc440spe_adma_memcpy_xor_set_src(struct ppc440spe_adma_desc_slot *tx,
1231                                  dma_addr_t addr, int index);
1232
1233static void
1234ppc440spe_adma_pq_set_dest(struct ppc440spe_adma_desc_slot *tx,
1235                           dma_addr_t *paddr, unsigned long flags);
1236static void
1237ppc440spe_adma_pq_set_src(struct ppc440spe_adma_desc_slot *tx,
1238                          dma_addr_t addr, int index);
1239static void
1240ppc440spe_adma_pq_set_src_mult(struct ppc440spe_adma_desc_slot *tx,
1241                               unsigned char mult, int index, int dst_pos);
1242static void
1243ppc440spe_adma_pqzero_sum_set_dest(struct ppc440spe_adma_desc_slot *tx,
1244                                   dma_addr_t paddr, dma_addr_t qaddr);
1245
1246static struct page *ppc440spe_rxor_srcs[32];
1247
1248/**
1249 * ppc440spe_can_rxor - check if the operands may be processed with RXOR
1250 */
1251static int ppc440spe_can_rxor(struct page **srcs, int src_cnt, size_t len)
1252{
1253        int i, order = 0, state = 0;
1254        int idx = 0;
1255
1256        if (unlikely(!(src_cnt > 1)))
1257                return 0;
1258
1259        BUG_ON(src_cnt > ARRAY_SIZE(ppc440spe_rxor_srcs));
1260
1261        /* Skip holes in the source list before checking */
1262        for (i = 0; i < src_cnt; i++) {
1263                if (!srcs[i])
1264                        continue;
1265                ppc440spe_rxor_srcs[idx++] = srcs[i];
1266        }
1267        src_cnt = idx;
1268
1269        for (i = 1; i < src_cnt; i++) {
1270                char *cur_addr = page_address(ppc440spe_rxor_srcs[i]);
1271                char *old_addr = page_address(ppc440spe_rxor_srcs[i - 1]);
1272
1273                switch (state) {
1274                case 0:
1275                        if (cur_addr == old_addr + len) {
1276                                /* direct RXOR */
1277                                order = 1;
1278                                state = 1;
1279                        } else if (old_addr == cur_addr + len) {
1280                                /* reverse RXOR */
1281                                order = -1;
1282                                state = 1;
1283                        } else
1284                                goto out;
1285                        break;
1286                case 1:
1287                        if ((i == src_cnt - 2) ||
1288                            (order == -1 && cur_addr != old_addr - len)) {
1289                                order = 0;
1290                                state = 0;
1291                        } else if ((cur_addr == old_addr + len * order) ||
1292                                   (cur_addr == old_addr + 2 * len) ||
1293                                   (cur_addr == old_addr + 3 * len)) {
1294                                state = 2;
1295                        } else {
1296                                order = 0;
1297                                state = 0;
1298                        }
1299                        break;
1300                case 2:
1301                        order = 0;
1302                        state = 0;
1303                        break;
1304                }
1305        }
1306
1307out:
1308        if (state == 1 || state == 2)
1309                return 1;
1310
1311        return 0;
1312}
1313
1314/**
1315 * ppc440spe_adma_device_estimate - estimate the efficiency of processing
1316 *      the operation given on this channel. It's assumed that 'chan' is
1317 *      capable to process 'cap' type of operation.
1318 * @chan: channel to use
1319 * @cap: type of transaction
1320 * @dst_lst: array of destination pointers
1321 * @dst_cnt: number of destination operands
1322 * @src_lst: array of source pointers
1323 * @src_cnt: number of source operands
1324 * @src_sz: size of each source operand
1325 */
1326static int ppc440spe_adma_estimate(struct dma_chan *chan,
1327        enum dma_transaction_type cap, struct page **dst_lst, int dst_cnt,
1328        struct page **src_lst, int src_cnt, size_t src_sz)
1329{
1330        int ef = 1;
1331
1332        if (cap == DMA_PQ || cap == DMA_PQ_VAL) {
1333                /* If RAID-6 capabilities were not activated don't try
1334                 * to use them
1335                 */
1336                if (unlikely(!ppc440spe_r6_enabled))
1337                        return -1;
1338        }
1339        /*  In the current implementation of ppc440spe ADMA driver it
1340         * makes sense to pick out only pq case, because it may be
1341         * processed:
1342         * (1) either using Biskup method on DMA2;
1343         * (2) or on DMA0/1.
1344         *  Thus we give a favour to (1) if the sources are suitable;
1345         * else let it be processed on one of the DMA0/1 engines.
1346         *  In the sum_product case where destination is also the
1347         * source process it on DMA0/1 only.
1348         */
1349        if (cap == DMA_PQ && chan->chan_id == PPC440SPE_XOR_ID) {
1350
1351                if (dst_cnt == 1 && src_cnt == 2 && dst_lst[0] == src_lst[1])
1352                        ef = 0; /* sum_product case, process on DMA0/1 */
1353                else if (ppc440spe_can_rxor(src_lst, src_cnt, src_sz))
1354                        ef = 3; /* override (DMA0/1 + idle) */
1355                else
1356                        ef = 0; /* can't process on DMA2 if !rxor */
1357        }
1358
1359        /* channel idleness increases the priority */
1360        if (likely(ef) &&
1361            !ppc440spe_chan_is_busy(to_ppc440spe_adma_chan(chan)))
1362                ef++;
1363
1364        return ef;
1365}
1366
1367struct dma_chan *
1368ppc440spe_async_tx_find_best_channel(enum dma_transaction_type cap,
1369        struct page **dst_lst, int dst_cnt, struct page **src_lst,
1370        int src_cnt, size_t src_sz)
1371{
1372        struct dma_chan *best_chan = NULL;
1373        struct ppc_dma_chan_ref *ref;
1374        int best_rank = -1;
1375
1376        if (unlikely(!src_sz))
1377                return NULL;
1378        if (src_sz > PAGE_SIZE) {
1379                /*
1380                 * should a user of the api ever pass > PAGE_SIZE requests
1381                 * we sort out cases where temporary page-sized buffers
1382                 * are used.
1383                 */
1384                switch (cap) {
1385                case DMA_PQ:
1386                        if (src_cnt == 1 && dst_lst[1] == src_lst[0])
1387                                return NULL;
1388                        if (src_cnt == 2 && dst_lst[1] == src_lst[1])
1389                                return NULL;
1390                        break;
1391                case DMA_PQ_VAL:
1392                case DMA_XOR_VAL:
1393                        return NULL;
1394                default:
1395                        break;
1396                }
1397        }
1398
1399        list_for_each_entry(ref, &ppc440spe_adma_chan_list, node) {
1400                if (dma_has_cap(cap, ref->chan->device->cap_mask)) {
1401                        int rank;
1402
1403                        rank = ppc440spe_adma_estimate(ref->chan, cap, dst_lst,
1404                                        dst_cnt, src_lst, src_cnt, src_sz);
1405                        if (rank > best_rank) {
1406                                best_rank = rank;
1407                                best_chan = ref->chan;
1408                        }
1409                }
1410        }
1411
1412        return best_chan;
1413}
1414EXPORT_SYMBOL_GPL(ppc440spe_async_tx_find_best_channel);
1415
1416/**
1417 * ppc440spe_get_group_entry - get group entry with index idx
1418 * @tdesc: is the last allocated slot in the group.
1419 */
1420static struct ppc440spe_adma_desc_slot *
1421ppc440spe_get_group_entry(struct ppc440spe_adma_desc_slot *tdesc, u32 entry_idx)
1422{
1423        struct ppc440spe_adma_desc_slot *iter = tdesc->group_head;
1424        int i = 0;
1425
1426        if (entry_idx < 0 || entry_idx >= (tdesc->src_cnt + tdesc->dst_cnt)) {
1427                printk("%s: entry_idx %d, src_cnt %d, dst_cnt %d\n",
1428                        __func__, entry_idx, tdesc->src_cnt, tdesc->dst_cnt);
1429                BUG();
1430        }
1431
1432        list_for_each_entry(iter, &tdesc->group_list, chain_node) {
1433                if (i++ == entry_idx)
1434                        break;
1435        }
1436        return iter;
1437}
1438
1439/**
1440 * ppc440spe_adma_free_slots - flags descriptor slots for reuse
1441 * @slot: Slot to free
1442 * Caller must hold &ppc440spe_chan->lock while calling this function
1443 */
1444static void ppc440spe_adma_free_slots(struct ppc440spe_adma_desc_slot *slot,
1445                                      struct ppc440spe_adma_chan *chan)
1446{
1447        int stride = slot->slots_per_op;
1448
1449        while (stride--) {
1450                slot->slots_per_op = 0;
1451                slot = list_entry(slot->slot_node.next,
1452                                struct ppc440spe_adma_desc_slot,
1453                                slot_node);
1454        }
1455}
1456
1457/**
1458 * ppc440spe_adma_run_tx_complete_actions - call functions to be called
1459 * upon completion
1460 */
1461static dma_cookie_t ppc440spe_adma_run_tx_complete_actions(
1462                struct ppc440spe_adma_desc_slot *desc,
1463                struct ppc440spe_adma_chan *chan,
1464                dma_cookie_t cookie)
1465{
1466        BUG_ON(desc->async_tx.cookie < 0);
1467        if (desc->async_tx.cookie > 0) {
1468                cookie = desc->async_tx.cookie;
1469                desc->async_tx.cookie = 0;
1470
1471                dma_descriptor_unmap(&desc->async_tx);
1472                /* call the callback (must not sleep or submit new
1473                 * operations to this channel)
1474                 */
1475                dmaengine_desc_get_callback_invoke(&desc->async_tx, NULL);
1476        }
1477
1478        /* run dependent operations */
1479        dma_run_dependencies(&desc->async_tx);
1480
1481        return cookie;
1482}
1483
1484/**
1485 * ppc440spe_adma_clean_slot - clean up CDB slot (if ack is set)
1486 */
1487static int ppc440spe_adma_clean_slot(struct ppc440spe_adma_desc_slot *desc,
1488                struct ppc440spe_adma_chan *chan)
1489{
1490        /* the client is allowed to attach dependent operations
1491         * until 'ack' is set
1492         */
1493        if (!async_tx_test_ack(&desc->async_tx))
1494                return 0;
1495
1496        /* leave the last descriptor in the chain
1497         * so we can append to it
1498         */
1499        if (list_is_last(&desc->chain_node, &chan->chain) ||
1500            desc->phys == ppc440spe_chan_get_current_descriptor(chan))
1501                return 1;
1502
1503        if (chan->device->id != PPC440SPE_XOR_ID) {
1504                /* our DMA interrupt handler clears opc field of
1505                 * each processed descriptor. For all types of
1506                 * operations except for ZeroSum we do not actually
1507                 * need ack from the interrupt handler. ZeroSum is a
1508                 * special case since the result of this operation
1509                 * is available from the handler only, so if we see
1510                 * such type of descriptor (which is unprocessed yet)
1511                 * then leave it in chain.
1512                 */
1513                struct dma_cdb *cdb = desc->hw_desc;
1514                if (cdb->opc == DMA_CDB_OPC_DCHECK128)
1515                        return 1;
1516        }
1517
1518        dev_dbg(chan->device->common.dev, "\tfree slot %llx: %d stride: %d\n",
1519                desc->phys, desc->idx, desc->slots_per_op);
1520
1521        list_del(&desc->chain_node);
1522        ppc440spe_adma_free_slots(desc, chan);
1523        return 0;
1524}
1525
1526/**
1527 * __ppc440spe_adma_slot_cleanup - this is the common clean-up routine
1528 *      which runs through the channel CDBs list until reach the descriptor
1529 *      currently processed. When routine determines that all CDBs of group
1530 *      are completed then corresponding callbacks (if any) are called and slots
1531 *      are freed.
1532 */
1533static void __ppc440spe_adma_slot_cleanup(struct ppc440spe_adma_chan *chan)
1534{
1535        struct ppc440spe_adma_desc_slot *iter, *_iter, *group_start = NULL;
1536        dma_cookie_t cookie = 0;
1537        u32 current_desc = ppc440spe_chan_get_current_descriptor(chan);
1538        int busy = ppc440spe_chan_is_busy(chan);
1539        int seen_current = 0, slot_cnt = 0, slots_per_op = 0;
1540
1541        dev_dbg(chan->device->common.dev, "ppc440spe adma%d: %s\n",
1542                chan->device->id, __func__);
1543
1544        if (!current_desc) {
1545                /*  There were no transactions yet, so
1546                 * nothing to clean
1547                 */
1548                return;
1549        }
1550
1551        /* free completed slots from the chain starting with
1552         * the oldest descriptor
1553         */
1554        list_for_each_entry_safe(iter, _iter, &chan->chain,
1555                                        chain_node) {
1556                dev_dbg(chan->device->common.dev, "\tcookie: %d slot: %d "
1557                    "busy: %d this_desc: %#llx next_desc: %#x "
1558                    "cur: %#x ack: %d\n",
1559                    iter->async_tx.cookie, iter->idx, busy, iter->phys,
1560                    ppc440spe_desc_get_link(iter, chan), current_desc,
1561                    async_tx_test_ack(&iter->async_tx));
1562                prefetch(_iter);
1563                prefetch(&_iter->async_tx);
1564
1565                /* do not advance past the current descriptor loaded into the
1566                 * hardware channel,subsequent descriptors are either in process
1567                 * or have not been submitted
1568                 */
1569                if (seen_current)
1570                        break;
1571
1572                /* stop the search if we reach the current descriptor and the
1573                 * channel is busy, or if it appears that the current descriptor
1574                 * needs to be re-read (i.e. has been appended to)
1575                 */
1576                if (iter->phys == current_desc) {
1577                        BUG_ON(seen_current++);
1578                        if (busy || ppc440spe_desc_get_link(iter, chan)) {
1579                                /* not all descriptors of the group have
1580                                 * been completed; exit.
1581                                 */
1582                                break;
1583                        }
1584                }
1585
1586                /* detect the start of a group transaction */
1587                if (!slot_cnt && !slots_per_op) {
1588                        slot_cnt = iter->slot_cnt;
1589                        slots_per_op = iter->slots_per_op;
1590                        if (slot_cnt <= slots_per_op) {
1591                                slot_cnt = 0;
1592                                slots_per_op = 0;
1593                        }
1594                }
1595
1596                if (slot_cnt) {
1597                        if (!group_start)
1598                                group_start = iter;
1599                        slot_cnt -= slots_per_op;
1600                }
1601
1602                /* all the members of a group are complete */
1603                if (slots_per_op != 0 && slot_cnt == 0) {
1604                        struct ppc440spe_adma_desc_slot *grp_iter, *_grp_iter;
1605                        int end_of_chain = 0;
1606
1607                        /* clean up the group */
1608                        slot_cnt = group_start->slot_cnt;
1609                        grp_iter = group_start;
1610                        list_for_each_entry_safe_from(grp_iter, _grp_iter,
1611                                &chan->chain, chain_node) {
1612
1613                                cookie = ppc440spe_adma_run_tx_complete_actions(
1614                                        grp_iter, chan, cookie);
1615
1616                                slot_cnt -= slots_per_op;
1617                                end_of_chain = ppc440spe_adma_clean_slot(
1618                                    grp_iter, chan);
1619                                if (end_of_chain && slot_cnt) {
1620                                        /* Should wait for ZeroSum completion */
1621                                        if (cookie > 0)
1622                                                chan->common.completed_cookie = cookie;
1623                                        return;
1624                                }
1625
1626                                if (slot_cnt == 0 || end_of_chain)
1627                                        break;
1628                        }
1629
1630                        /* the group should be complete at this point */
1631                        BUG_ON(slot_cnt);
1632
1633                        slots_per_op = 0;
1634                        group_start = NULL;
1635                        if (end_of_chain)
1636                                break;
1637                        else
1638                                continue;
1639                } else if (slots_per_op) /* wait for group completion */
1640                        continue;
1641
1642                cookie = ppc440spe_adma_run_tx_complete_actions(iter, chan,
1643                    cookie);
1644
1645                if (ppc440spe_adma_clean_slot(iter, chan))
1646                        break;
1647        }
1648
1649        BUG_ON(!seen_current);
1650
1651        if (cookie > 0) {
1652                chan->common.completed_cookie = cookie;
1653                pr_debug("\tcompleted cookie %d\n", cookie);
1654        }
1655
1656}
1657
1658/**
1659 * ppc440spe_adma_tasklet - clean up watch-dog initiator
1660 */
1661static void ppc440spe_adma_tasklet(struct tasklet_struct *t)
1662{
1663        struct ppc440spe_adma_chan *chan = from_tasklet(chan, t, irq_tasklet);
1664
1665        spin_lock_nested(&chan->lock, SINGLE_DEPTH_NESTING);
1666        __ppc440spe_adma_slot_cleanup(chan);
1667        spin_unlock(&chan->lock);
1668}
1669
1670/**
1671 * ppc440spe_adma_slot_cleanup - clean up scheduled initiator
1672 */
1673static void ppc440spe_adma_slot_cleanup(struct ppc440spe_adma_chan *chan)
1674{
1675        spin_lock_bh(&chan->lock);
1676        __ppc440spe_adma_slot_cleanup(chan);
1677        spin_unlock_bh(&chan->lock);
1678}
1679
1680/**
1681 * ppc440spe_adma_alloc_slots - allocate free slots (if any)
1682 */
1683static struct ppc440spe_adma_desc_slot *ppc440spe_adma_alloc_slots(
1684                struct ppc440spe_adma_chan *chan, int num_slots,
1685                int slots_per_op)
1686{
1687        struct ppc440spe_adma_desc_slot *iter = NULL, *_iter;
1688        struct ppc440spe_adma_desc_slot *alloc_start = NULL;
1689        int slots_found, retry = 0;
1690        LIST_HEAD(chain);
1691
1692
1693        BUG_ON(!num_slots || !slots_per_op);
1694        /* start search from the last allocated descrtiptor
1695         * if a contiguous allocation can not be found start searching
1696         * from the beginning of the list
1697         */
1698retry:
1699        slots_found = 0;
1700        if (retry == 0)
1701                iter = chan->last_used;
1702        else
1703                iter = list_entry(&chan->all_slots,
1704                                  struct ppc440spe_adma_desc_slot,
1705                                  slot_node);
1706        list_for_each_entry_safe_continue(iter, _iter, &chan->all_slots,
1707            slot_node) {
1708                prefetch(_iter);
1709                prefetch(&_iter->async_tx);
1710                if (iter->slots_per_op) {
1711                        slots_found = 0;
1712                        continue;
1713                }
1714
1715                /* start the allocation if the slot is correctly aligned */
1716                if (!slots_found++)
1717                        alloc_start = iter;
1718
1719                if (slots_found == num_slots) {
1720                        struct ppc440spe_adma_desc_slot *alloc_tail = NULL;
1721                        struct ppc440spe_adma_desc_slot *last_used = NULL;
1722
1723                        iter = alloc_start;
1724                        while (num_slots) {
1725                                int i;
1726                                /* pre-ack all but the last descriptor */
1727                                if (num_slots != slots_per_op)
1728                                        async_tx_ack(&iter->async_tx);
1729
1730                                list_add_tail(&iter->chain_node, &chain);
1731                                alloc_tail = iter;
1732                                iter->async_tx.cookie = 0;
1733                                iter->hw_next = NULL;
1734                                iter->flags = 0;
1735                                iter->slot_cnt = num_slots;
1736                                iter->xor_check_result = NULL;
1737                                for (i = 0; i < slots_per_op; i++) {
1738                                        iter->slots_per_op = slots_per_op - i;
1739                                        last_used = iter;
1740                                        iter = list_entry(iter->slot_node.next,
1741                                                struct ppc440spe_adma_desc_slot,
1742                                                slot_node);
1743                                }
1744                                num_slots -= slots_per_op;
1745                        }
1746                        alloc_tail->group_head = alloc_start;
1747                        alloc_tail->async_tx.cookie = -EBUSY;
1748                        list_splice(&chain, &alloc_tail->group_list);
1749                        chan->last_used = last_used;
1750                        return alloc_tail;
1751                }
1752        }
1753        if (!retry++)
1754                goto retry;
1755
1756        /* try to free some slots if the allocation fails */
1757        tasklet_schedule(&chan->irq_tasklet);
1758        return NULL;
1759}
1760
1761/**
1762 * ppc440spe_adma_alloc_chan_resources -  allocate pools for CDB slots
1763 */
1764static int ppc440spe_adma_alloc_chan_resources(struct dma_chan *chan)
1765{
1766        struct ppc440spe_adma_chan *ppc440spe_chan;
1767        struct ppc440spe_adma_desc_slot *slot = NULL;
1768        char *hw_desc;
1769        int i, db_sz;
1770        int init;
1771
1772        ppc440spe_chan = to_ppc440spe_adma_chan(chan);
1773        init = ppc440spe_chan->slots_allocated ? 0 : 1;
1774        chan->chan_id = ppc440spe_chan->device->id;
1775
1776        /* Allocate descriptor slots */
1777        i = ppc440spe_chan->slots_allocated;
1778        if (ppc440spe_chan->device->id != PPC440SPE_XOR_ID)
1779                db_sz = sizeof(struct dma_cdb);
1780        else
1781                db_sz = sizeof(struct xor_cb);
1782
1783        for (; i < (ppc440spe_chan->device->pool_size / db_sz); i++) {
1784                slot = kzalloc(sizeof(struct ppc440spe_adma_desc_slot),
1785                               GFP_KERNEL);
1786                if (!slot) {
1787                        printk(KERN_INFO "SPE ADMA Channel only initialized"
1788                                " %d descriptor slots", i--);
1789                        break;
1790                }
1791
1792                hw_desc = (char *) ppc440spe_chan->device->dma_desc_pool_virt;
1793                slot->hw_desc = (void *) &hw_desc[i * db_sz];
1794                dma_async_tx_descriptor_init(&slot->async_tx, chan);
1795                slot->async_tx.tx_submit = ppc440spe_adma_tx_submit;
1796                INIT_LIST_HEAD(&slot->chain_node);
1797                INIT_LIST_HEAD(&slot->slot_node);
1798                INIT_LIST_HEAD(&slot->group_list);
1799                slot->phys = ppc440spe_chan->device->dma_desc_pool + i * db_sz;
1800                slot->idx = i;
1801
1802                spin_lock_bh(&ppc440spe_chan->lock);
1803                ppc440spe_chan->slots_allocated++;
1804                list_add_tail(&slot->slot_node, &ppc440spe_chan->all_slots);
1805                spin_unlock_bh(&ppc440spe_chan->lock);
1806        }
1807
1808        if (i && !ppc440spe_chan->last_used) {
1809                ppc440spe_chan->last_used =
1810                        list_entry(ppc440spe_chan->all_slots.next,
1811                                struct ppc440spe_adma_desc_slot,
1812                                slot_node);
1813        }
1814
1815        dev_dbg(ppc440spe_chan->device->common.dev,
1816                "ppc440spe adma%d: allocated %d descriptor slots\n",
1817                ppc440spe_chan->device->id, i);
1818
1819        /* initialize the channel and the chain with a null operation */
1820        if (init) {
1821                switch (ppc440spe_chan->device->id) {
1822                case PPC440SPE_DMA0_ID:
1823                case PPC440SPE_DMA1_ID:
1824                        ppc440spe_chan->hw_chain_inited = 0;
1825                        /* Use WXOR for self-testing */
1826                        if (!ppc440spe_r6_tchan)
1827                                ppc440spe_r6_tchan = ppc440spe_chan;
1828                        break;
1829                case PPC440SPE_XOR_ID:
1830                        ppc440spe_chan_start_null_xor(ppc440spe_chan);
1831                        break;
1832                default:
1833                        BUG();
1834                }
1835                ppc440spe_chan->needs_unmap = 1;
1836        }
1837
1838        return (i > 0) ? i : -ENOMEM;
1839}
1840
1841/**
1842 * ppc440spe_rxor_set_region_data -
1843 */
1844static void ppc440spe_rxor_set_region(struct ppc440spe_adma_desc_slot *desc,
1845        u8 xor_arg_no, u32 mask)
1846{
1847        struct xor_cb *xcb = desc->hw_desc;
1848
1849        xcb->ops[xor_arg_no].h |= mask;
1850}
1851
1852/**
1853 * ppc440spe_rxor_set_src -
1854 */
1855static void ppc440spe_rxor_set_src(struct ppc440spe_adma_desc_slot *desc,
1856        u8 xor_arg_no, dma_addr_t addr)
1857{
1858        struct xor_cb *xcb = desc->hw_desc;
1859
1860        xcb->ops[xor_arg_no].h |= DMA_CUED_XOR_BASE;
1861        xcb->ops[xor_arg_no].l = addr;
1862}
1863
1864/**
1865 * ppc440spe_rxor_set_mult -
1866 */
1867static void ppc440spe_rxor_set_mult(struct ppc440spe_adma_desc_slot *desc,
1868        u8 xor_arg_no, u8 idx, u8 mult)
1869{
1870        struct xor_cb *xcb = desc->hw_desc;
1871
1872        xcb->ops[xor_arg_no].h |= mult << (DMA_CUED_MULT1_OFF + idx * 8);
1873}
1874
1875/**
1876 * ppc440spe_adma_check_threshold - append CDBs to h/w chain if threshold
1877 *      has been achieved
1878 */
1879static void ppc440spe_adma_check_threshold(struct ppc440spe_adma_chan *chan)
1880{
1881        dev_dbg(chan->device->common.dev, "ppc440spe adma%d: pending: %d\n",
1882                chan->device->id, chan->pending);
1883
1884        if (chan->pending >= PPC440SPE_ADMA_THRESHOLD) {
1885                chan->pending = 0;
1886                ppc440spe_chan_append(chan);
1887        }
1888}
1889
1890/**
1891 * ppc440spe_adma_tx_submit - submit new descriptor group to the channel
1892 *      (it's not necessary that descriptors will be submitted to the h/w
1893 *      chains too right now)
1894 */
1895static dma_cookie_t ppc440spe_adma_tx_submit(struct dma_async_tx_descriptor *tx)
1896{
1897        struct ppc440spe_adma_desc_slot *sw_desc;
1898        struct ppc440spe_adma_chan *chan = to_ppc440spe_adma_chan(tx->chan);
1899        struct ppc440spe_adma_desc_slot *group_start, *old_chain_tail;
1900        int slot_cnt;
1901        int slots_per_op;
1902        dma_cookie_t cookie;
1903
1904        sw_desc = tx_to_ppc440spe_adma_slot(tx);
1905
1906        group_start = sw_desc->group_head;
1907        slot_cnt = group_start->slot_cnt;
1908        slots_per_op = group_start->slots_per_op;
1909
1910        spin_lock_bh(&chan->lock);
1911        cookie = dma_cookie_assign(tx);
1912
1913        if (unlikely(list_empty(&chan->chain))) {
1914                /* first peer */
1915                list_splice_init(&sw_desc->group_list, &chan->chain);
1916                chan_first_cdb[chan->device->id] = group_start;
1917        } else {
1918                /* isn't first peer, bind CDBs to chain */
1919                old_chain_tail = list_entry(chan->chain.prev,
1920                                        struct ppc440spe_adma_desc_slot,
1921                                        chain_node);
1922                list_splice_init(&sw_desc->group_list,
1923                    &old_chain_tail->chain_node);
1924                /* fix up the hardware chain */
1925                ppc440spe_desc_set_link(chan, old_chain_tail, group_start);
1926        }
1927
1928        /* increment the pending count by the number of operations */
1929        chan->pending += slot_cnt / slots_per_op;
1930        ppc440spe_adma_check_threshold(chan);
1931        spin_unlock_bh(&chan->lock);
1932
1933        dev_dbg(chan->device->common.dev,
1934                "ppc440spe adma%d: %s cookie: %d slot: %d tx %p\n",
1935                chan->device->id, __func__,
1936                sw_desc->async_tx.cookie, sw_desc->idx, sw_desc);
1937
1938        return cookie;
1939}
1940
1941/**
1942 * ppc440spe_adma_prep_dma_interrupt - prepare CDB for a pseudo DMA operation
1943 */
1944static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_interrupt(
1945                struct dma_chan *chan, unsigned long flags)
1946{
1947        struct ppc440spe_adma_chan *ppc440spe_chan;
1948        struct ppc440spe_adma_desc_slot *sw_desc, *group_start;
1949        int slot_cnt, slots_per_op;
1950
1951        ppc440spe_chan = to_ppc440spe_adma_chan(chan);
1952
1953        dev_dbg(ppc440spe_chan->device->common.dev,
1954                "ppc440spe adma%d: %s\n", ppc440spe_chan->device->id,
1955                __func__);
1956
1957        spin_lock_bh(&ppc440spe_chan->lock);
1958        slot_cnt = slots_per_op = 1;
1959        sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt,
1960                        slots_per_op);
1961        if (sw_desc) {
1962                group_start = sw_desc->group_head;
1963                ppc440spe_desc_init_interrupt(group_start, ppc440spe_chan);
1964                group_start->unmap_len = 0;
1965                sw_desc->async_tx.flags = flags;
1966        }
1967        spin_unlock_bh(&ppc440spe_chan->lock);
1968
1969        return sw_desc ? &sw_desc->async_tx : NULL;
1970}
1971
1972/**
1973 * ppc440spe_adma_prep_dma_memcpy - prepare CDB for a MEMCPY operation
1974 */
1975static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_memcpy(
1976                struct dma_chan *chan, dma_addr_t dma_dest,
1977                dma_addr_t dma_src, size_t len, unsigned long flags)
1978{
1979        struct ppc440spe_adma_chan *ppc440spe_chan;
1980        struct ppc440spe_adma_desc_slot *sw_desc, *group_start;
1981        int slot_cnt, slots_per_op;
1982
1983        ppc440spe_chan = to_ppc440spe_adma_chan(chan);
1984
1985        if (unlikely(!len))
1986                return NULL;
1987
1988        BUG_ON(len > PPC440SPE_ADMA_DMA_MAX_BYTE_COUNT);
1989
1990        spin_lock_bh(&ppc440spe_chan->lock);
1991
1992        dev_dbg(ppc440spe_chan->device->common.dev,
1993                "ppc440spe adma%d: %s len: %u int_en %d\n",
1994                ppc440spe_chan->device->id, __func__, len,
1995                flags & DMA_PREP_INTERRUPT ? 1 : 0);
1996        slot_cnt = slots_per_op = 1;
1997        sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt,
1998                slots_per_op);
1999        if (sw_desc) {
2000                group_start = sw_desc->group_head;
2001                ppc440spe_desc_init_memcpy(group_start, flags);
2002                ppc440spe_adma_set_dest(group_start, dma_dest, 0);
2003                ppc440spe_adma_memcpy_xor_set_src(group_start, dma_src, 0);
2004                ppc440spe_desc_set_byte_count(group_start, ppc440spe_chan, len);
2005                sw_desc->unmap_len = len;
2006                sw_desc->async_tx.flags = flags;
2007        }
2008        spin_unlock_bh(&ppc440spe_chan->lock);
2009
2010        return sw_desc ? &sw_desc->async_tx : NULL;
2011}
2012
2013/**
2014 * ppc440spe_adma_prep_dma_xor - prepare CDB for a XOR operation
2015 */
2016static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_xor(
2017                struct dma_chan *chan, dma_addr_t dma_dest,
2018                dma_addr_t *dma_src, u32 src_cnt, size_t len,
2019                unsigned long flags)
2020{
2021        struct ppc440spe_adma_chan *ppc440spe_chan;
2022        struct ppc440spe_adma_desc_slot *sw_desc, *group_start;
2023        int slot_cnt, slots_per_op;
2024
2025        ppc440spe_chan = to_ppc440spe_adma_chan(chan);
2026
2027        ADMA_LL_DBG(prep_dma_xor_dbg(ppc440spe_chan->device->id,
2028                                     dma_dest, dma_src, src_cnt));
2029        if (unlikely(!len))
2030                return NULL;
2031        BUG_ON(len > PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT);
2032
2033        dev_dbg(ppc440spe_chan->device->common.dev,
2034                "ppc440spe adma%d: %s src_cnt: %d len: %u int_en: %d\n",
2035                ppc440spe_chan->device->id, __func__, src_cnt, len,
2036                flags & DMA_PREP_INTERRUPT ? 1 : 0);
2037
2038        spin_lock_bh(&ppc440spe_chan->lock);
2039        slot_cnt = ppc440spe_chan_xor_slot_count(len, src_cnt, &slots_per_op);
2040        sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt,
2041                        slots_per_op);
2042        if (sw_desc) {
2043                group_start = sw_desc->group_head;
2044                ppc440spe_desc_init_xor(group_start, src_cnt, flags);
2045                ppc440spe_adma_set_dest(group_start, dma_dest, 0);
2046                while (src_cnt--)
2047                        ppc440spe_adma_memcpy_xor_set_src(group_start,
2048                                dma_src[src_cnt], src_cnt);
2049                ppc440spe_desc_set_byte_count(group_start, ppc440spe_chan, len);
2050                sw_desc->unmap_len = len;
2051                sw_desc->async_tx.flags = flags;
2052        }
2053        spin_unlock_bh(&ppc440spe_chan->lock);
2054
2055        return sw_desc ? &sw_desc->async_tx : NULL;
2056}
2057
2058static inline void
2059ppc440spe_desc_set_xor_src_cnt(struct ppc440spe_adma_desc_slot *desc,
2060                                int src_cnt);
2061static void ppc440spe_init_rxor_cursor(struct ppc440spe_rxor *cursor);
2062
2063/**
2064 * ppc440spe_adma_init_dma2rxor_slot -
2065 */
2066static void ppc440spe_adma_init_dma2rxor_slot(
2067                struct ppc440spe_adma_desc_slot *desc,
2068                dma_addr_t *src, int src_cnt)
2069{
2070        int i;
2071
2072        /* initialize CDB */
2073        for (i = 0; i < src_cnt; i++) {
2074                ppc440spe_adma_dma2rxor_prep_src(desc, &desc->rxor_cursor, i,
2075                                                 desc->src_cnt, (u32)src[i]);
2076        }
2077}
2078
2079/**
2080 * ppc440spe_dma01_prep_mult -
2081 * for Q operation where destination is also the source
2082 */
2083static struct ppc440spe_adma_desc_slot *ppc440spe_dma01_prep_mult(
2084                struct ppc440spe_adma_chan *ppc440spe_chan,
2085                dma_addr_t *dst, int dst_cnt, dma_addr_t *src, int src_cnt,
2086                const unsigned char *scf, size_t len, unsigned long flags)
2087{
2088        struct ppc440spe_adma_desc_slot *sw_desc = NULL;
2089        unsigned long op = 0;
2090        int slot_cnt;
2091
2092        set_bit(PPC440SPE_DESC_WXOR, &op);
2093        slot_cnt = 2;
2094
2095        spin_lock_bh(&ppc440spe_chan->lock);
2096
2097        /* use WXOR, each descriptor occupies one slot */
2098        sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1);
2099        if (sw_desc) {
2100                struct ppc440spe_adma_chan *chan;
2101                struct ppc440spe_adma_desc_slot *iter;
2102                struct dma_cdb *hw_desc;
2103
2104                chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
2105                set_bits(op, &sw_desc->flags);
2106                sw_desc->src_cnt = src_cnt;
2107                sw_desc->dst_cnt = dst_cnt;
2108                /* First descriptor, zero data in the destination and copy it
2109                 * to q page using MULTICAST transfer.
2110                 */
2111                iter = list_first_entry(&sw_desc->group_list,
2112                                        struct ppc440spe_adma_desc_slot,
2113                                        chain_node);
2114                memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2115                /* set 'next' pointer */
2116                iter->hw_next = list_entry(iter->chain_node.next,
2117                                           struct ppc440spe_adma_desc_slot,
2118                                           chain_node);
2119                clear_bit(PPC440SPE_DESC_INT, &iter->flags);
2120                hw_desc = iter->hw_desc;
2121                hw_desc->opc = DMA_CDB_OPC_MULTICAST;
2122
2123                ppc440spe_desc_set_dest_addr(iter, chan,
2124                                             DMA_CUED_XOR_BASE, dst[0], 0);
2125                ppc440spe_desc_set_dest_addr(iter, chan, 0, dst[1], 1);
2126                ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB,
2127                                            src[0]);
2128                ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len);
2129                iter->unmap_len = len;
2130
2131                /*
2132                 * Second descriptor, multiply data from the q page
2133                 * and store the result in real destination.
2134                 */
2135                iter = list_first_entry(&iter->chain_node,
2136                                        struct ppc440spe_adma_desc_slot,
2137                                        chain_node);
2138                memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2139                iter->hw_next = NULL;
2140                if (flags & DMA_PREP_INTERRUPT)
2141                        set_bit(PPC440SPE_DESC_INT, &iter->flags);
2142                else
2143                        clear_bit(PPC440SPE_DESC_INT, &iter->flags);
2144
2145                hw_desc = iter->hw_desc;
2146                hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
2147                ppc440spe_desc_set_src_addr(iter, chan, 0,
2148                                            DMA_CUED_XOR_HB, dst[1]);
2149                ppc440spe_desc_set_dest_addr(iter, chan,
2150                                             DMA_CUED_XOR_BASE, dst[0], 0);
2151
2152                ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF,
2153                                            DMA_CDB_SG_DST1, scf[0]);
2154                ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len);
2155                iter->unmap_len = len;
2156                sw_desc->async_tx.flags = flags;
2157        }
2158
2159        spin_unlock_bh(&ppc440spe_chan->lock);
2160
2161        return sw_desc;
2162}
2163
2164/**
2165 * ppc440spe_dma01_prep_sum_product -
2166 * Dx = A*(P+Pxy) + B*(Q+Qxy) operation where destination is also
2167 * the source.
2168 */
2169static struct ppc440spe_adma_desc_slot *ppc440spe_dma01_prep_sum_product(
2170                struct ppc440spe_adma_chan *ppc440spe_chan,
2171                dma_addr_t *dst, dma_addr_t *src, int src_cnt,
2172                const unsigned char *scf, size_t len, unsigned long flags)
2173{
2174        struct ppc440spe_adma_desc_slot *sw_desc = NULL;
2175        unsigned long op = 0;
2176        int slot_cnt;
2177
2178        set_bit(PPC440SPE_DESC_WXOR, &op);
2179        slot_cnt = 3;
2180
2181        spin_lock_bh(&ppc440spe_chan->lock);
2182
2183        /* WXOR, each descriptor occupies one slot */
2184        sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1);
2185        if (sw_desc) {
2186                struct ppc440spe_adma_chan *chan;
2187                struct ppc440spe_adma_desc_slot *iter;
2188                struct dma_cdb *hw_desc;
2189
2190                chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
2191                set_bits(op, &sw_desc->flags);
2192                sw_desc->src_cnt = src_cnt;
2193                sw_desc->dst_cnt = 1;
2194                /* 1st descriptor, src[1] data to q page and zero destination */
2195                iter = list_first_entry(&sw_desc->group_list,
2196                                        struct ppc440spe_adma_desc_slot,
2197                                        chain_node);
2198                memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2199                iter->hw_next = list_entry(iter->chain_node.next,
2200                                           struct ppc440spe_adma_desc_slot,
2201                                           chain_node);
2202                clear_bit(PPC440SPE_DESC_INT, &iter->flags);
2203                hw_desc = iter->hw_desc;
2204                hw_desc->opc = DMA_CDB_OPC_MULTICAST;
2205
2206                ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE,
2207                                             *dst, 0);
2208                ppc440spe_desc_set_dest_addr(iter, chan, 0,
2209                                             ppc440spe_chan->qdest, 1);
2210                ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB,
2211                                            src[1]);
2212                ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len);
2213                iter->unmap_len = len;
2214
2215                /* 2nd descriptor, multiply src[1] data and store the
2216                 * result in destination */
2217                iter = list_first_entry(&iter->chain_node,
2218                                        struct ppc440spe_adma_desc_slot,
2219                                        chain_node);
2220                memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2221                /* set 'next' pointer */
2222                iter->hw_next = list_entry(iter->chain_node.next,
2223                                           struct ppc440spe_adma_desc_slot,
2224                                           chain_node);
2225                if (flags & DMA_PREP_INTERRUPT)
2226                        set_bit(PPC440SPE_DESC_INT, &iter->flags);
2227                else
2228                        clear_bit(PPC440SPE_DESC_INT, &iter->flags);
2229
2230                hw_desc = iter->hw_desc;
2231                hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
2232                ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB,
2233                                            ppc440spe_chan->qdest);
2234                ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE,
2235                                             *dst, 0);
2236                ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF,
2237                                            DMA_CDB_SG_DST1, scf[1]);
2238                ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len);
2239                iter->unmap_len = len;
2240
2241                /*
2242                 * 3rd descriptor, multiply src[0] data and xor it
2243                 * with destination
2244                 */
2245                iter = list_first_entry(&iter->chain_node,
2246                                        struct ppc440spe_adma_desc_slot,
2247                                        chain_node);
2248                memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2249                iter->hw_next = NULL;
2250                if (flags & DMA_PREP_INTERRUPT)
2251                        set_bit(PPC440SPE_DESC_INT, &iter->flags);
2252                else
2253                        clear_bit(PPC440SPE_DESC_INT, &iter->flags);
2254
2255                hw_desc = iter->hw_desc;
2256                hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
2257                ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB,
2258                                            src[0]);
2259                ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE,
2260                                             *dst, 0);
2261                ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF,
2262                                            DMA_CDB_SG_DST1, scf[0]);
2263                ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len);
2264                iter->unmap_len = len;
2265                sw_desc->async_tx.flags = flags;
2266        }
2267
2268        spin_unlock_bh(&ppc440spe_chan->lock);
2269
2270        return sw_desc;
2271}
2272
2273static struct ppc440spe_adma_desc_slot *ppc440spe_dma01_prep_pq(
2274                struct ppc440spe_adma_chan *ppc440spe_chan,
2275                dma_addr_t *dst, int dst_cnt, dma_addr_t *src, int src_cnt,
2276                const unsigned char *scf, size_t len, unsigned long flags)
2277{
2278        int slot_cnt;
2279        struct ppc440spe_adma_desc_slot *sw_desc = NULL, *iter;
2280        unsigned long op = 0;
2281        unsigned char mult = 1;
2282
2283        pr_debug("%s: dst_cnt %d, src_cnt %d, len %d\n",
2284                 __func__, dst_cnt, src_cnt, len);
2285        /*  select operations WXOR/RXOR depending on the
2286         * source addresses of operators and the number
2287         * of destinations (RXOR support only Q-parity calculations)
2288         */
2289        set_bit(PPC440SPE_DESC_WXOR, &op);
2290        if (!test_and_set_bit(PPC440SPE_RXOR_RUN, &ppc440spe_rxor_state)) {
2291                /* no active RXOR;
2292                 * do RXOR if:
2293                 * - there are more than 1 source,
2294                 * - len is aligned on 512-byte boundary,
2295                 * - source addresses fit to one of 4 possible regions.
2296                 */
2297                if (src_cnt > 1 &&
2298                    !(len & MQ0_CF2H_RXOR_BS_MASK) &&
2299                    (src[0] + len) == src[1]) {
2300                        /* may do RXOR R1 R2 */
2301                        set_bit(PPC440SPE_DESC_RXOR, &op);
2302                        if (src_cnt != 2) {
2303                                /* may try to enhance region of RXOR */
2304                                if ((src[1] + len) == src[2]) {
2305                                        /* do RXOR R1 R2 R3 */
2306                                        set_bit(PPC440SPE_DESC_RXOR123,
2307                                                &op);
2308                                } else if ((src[1] + len * 2) == src[2]) {
2309                                        /* do RXOR R1 R2 R4 */
2310                                        set_bit(PPC440SPE_DESC_RXOR124, &op);
2311                                } else if ((src[1] + len * 3) == src[2]) {
2312                                        /* do RXOR R1 R2 R5 */
2313                                        set_bit(PPC440SPE_DESC_RXOR125,
2314                                                &op);
2315                                } else {
2316                                        /* do RXOR R1 R2 */
2317                                        set_bit(PPC440SPE_DESC_RXOR12,
2318                                                &op);
2319                                }
2320                        } else {
2321                                /* do RXOR R1 R2 */
2322                                set_bit(PPC440SPE_DESC_RXOR12, &op);
2323                        }
2324                }
2325
2326                if (!test_bit(PPC440SPE_DESC_RXOR, &op)) {
2327                        /* can not do this operation with RXOR */
2328                        clear_bit(PPC440SPE_RXOR_RUN,
2329                                &ppc440spe_rxor_state);
2330                } else {
2331                        /* can do; set block size right now */
2332                        ppc440spe_desc_set_rxor_block_size(len);
2333                }
2334        }
2335
2336        /* Number of necessary slots depends on operation type selected */
2337        if (!test_bit(PPC440SPE_DESC_RXOR, &op)) {
2338                /*  This is a WXOR only chain. Need descriptors for each
2339                 * source to GF-XOR them with WXOR, and need descriptors
2340                 * for each destination to zero them with WXOR
2341                 */
2342                slot_cnt = src_cnt;
2343
2344                if (flags & DMA_PREP_ZERO_P) {
2345                        slot_cnt++;
2346                        set_bit(PPC440SPE_ZERO_P, &op);
2347                }
2348                if (flags & DMA_PREP_ZERO_Q) {
2349                        slot_cnt++;
2350                        set_bit(PPC440SPE_ZERO_Q, &op);
2351                }
2352        } else {
2353                /*  Need 1/2 descriptor for RXOR operation, and
2354                 * need (src_cnt - (2 or 3)) for WXOR of sources
2355                 * remained (if any)
2356                 */
2357                slot_cnt = dst_cnt;
2358
2359                if (flags & DMA_PREP_ZERO_P)
2360                        set_bit(PPC440SPE_ZERO_P, &op);
2361                if (flags & DMA_PREP_ZERO_Q)
2362                        set_bit(PPC440SPE_ZERO_Q, &op);
2363
2364                if (test_bit(PPC440SPE_DESC_RXOR12, &op))
2365                        slot_cnt += src_cnt - 2;
2366                else
2367                        slot_cnt += src_cnt - 3;
2368
2369                /*  Thus we have either RXOR only chain or
2370                 * mixed RXOR/WXOR
2371                 */
2372                if (slot_cnt == dst_cnt)
2373                        /* RXOR only chain */
2374                        clear_bit(PPC440SPE_DESC_WXOR, &op);
2375        }
2376
2377        spin_lock_bh(&ppc440spe_chan->lock);
2378        /* for both RXOR/WXOR each descriptor occupies one slot */
2379        sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1);
2380        if (sw_desc) {
2381                ppc440spe_desc_init_dma01pq(sw_desc, dst_cnt, src_cnt,
2382                                flags, op);
2383
2384                /* setup dst/src/mult */
2385                pr_debug("%s: set dst descriptor 0, 1: 0x%016llx, 0x%016llx\n",
2386                         __func__, dst[0], dst[1]);
2387                ppc440spe_adma_pq_set_dest(sw_desc, dst, flags);
2388                while (src_cnt--) {
2389                        ppc440spe_adma_pq_set_src(sw_desc, src[src_cnt],
2390                                                  src_cnt);
2391
2392                        /* NOTE: "Multi = 0 is equivalent to = 1" as it
2393                         * stated in 440SPSPe_RAID6_Addendum_UM_1_17.pdf
2394                         * doesn't work for RXOR with DMA0/1! Instead, multi=0
2395                         * leads to zeroing source data after RXOR.
2396                         * So, for P case set-up mult=1 explicitly.
2397                         */
2398                        if (!(flags & DMA_PREP_PQ_DISABLE_Q))
2399                                mult = scf[src_cnt];
2400                        ppc440spe_adma_pq_set_src_mult(sw_desc,
2401                                mult, src_cnt,  dst_cnt - 1);
2402                }
2403
2404                /* Setup byte count foreach slot just allocated */
2405                sw_desc->async_tx.flags = flags;
2406                list_for_each_entry(iter, &sw_desc->group_list,
2407                                chain_node) {
2408                        ppc440spe_desc_set_byte_count(iter,
2409                                ppc440spe_chan, len);
2410                        iter->unmap_len = len;
2411                }
2412        }
2413        spin_unlock_bh(&ppc440spe_chan->lock);
2414
2415        return sw_desc;
2416}
2417
2418static struct ppc440spe_adma_desc_slot *ppc440spe_dma2_prep_pq(
2419                struct ppc440spe_adma_chan *ppc440spe_chan,
2420                dma_addr_t *dst, int dst_cnt, dma_addr_t *src, int src_cnt,
2421                const unsigned char *scf, size_t len, unsigned long flags)
2422{
2423        int slot_cnt, descs_per_op;
2424        struct ppc440spe_adma_desc_slot *sw_desc = NULL, *iter;
2425        unsigned long op = 0;
2426        unsigned char mult = 1;
2427
2428        BUG_ON(!dst_cnt);
2429        /*pr_debug("%s: dst_cnt %d, src_cnt %d, len %d\n",
2430                 __func__, dst_cnt, src_cnt, len);*/
2431
2432        spin_lock_bh(&ppc440spe_chan->lock);
2433        descs_per_op = ppc440spe_dma2_pq_slot_count(src, src_cnt, len);
2434        if (descs_per_op < 0) {
2435                spin_unlock_bh(&ppc440spe_chan->lock);
2436                return NULL;
2437        }
2438
2439        /* depending on number of sources we have 1 or 2 RXOR chains */
2440        slot_cnt = descs_per_op * dst_cnt;
2441
2442        sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1);
2443        if (sw_desc) {
2444                op = slot_cnt;
2445                sw_desc->async_tx.flags = flags;
2446                list_for_each_entry(iter, &sw_desc->group_list, chain_node) {
2447                        ppc440spe_desc_init_dma2pq(iter, dst_cnt, src_cnt,
2448                                --op ? 0 : flags);
2449                        ppc440spe_desc_set_byte_count(iter, ppc440spe_chan,
2450                                len);
2451                        iter->unmap_len = len;
2452
2453                        ppc440spe_init_rxor_cursor(&(iter->rxor_cursor));
2454                        iter->rxor_cursor.len = len;
2455                        iter->descs_per_op = descs_per_op;
2456                }
2457                op = 0;
2458                list_for_each_entry(iter, &sw_desc->group_list, chain_node) {
2459                        op++;
2460                        if (op % descs_per_op == 0)
2461                                ppc440spe_adma_init_dma2rxor_slot(iter, src,
2462                                                                  src_cnt);
2463                        if (likely(!list_is_last(&iter->chain_node,
2464                                                 &sw_desc->group_list))) {
2465                                /* set 'next' pointer */
2466                                iter->hw_next =
2467                                        list_entry(iter->chain_node.next,
2468                                                struct ppc440spe_adma_desc_slot,
2469                                                chain_node);
2470                                ppc440spe_xor_set_link(iter, iter->hw_next);
2471                        } else {
2472                                /* this is the last descriptor. */
2473                                iter->hw_next = NULL;
2474                        }
2475                }
2476
2477                /* fixup head descriptor */
2478                sw_desc->dst_cnt = dst_cnt;
2479                if (flags & DMA_PREP_ZERO_P)
2480                        set_bit(PPC440SPE_ZERO_P, &sw_desc->flags);
2481                if (flags & DMA_PREP_ZERO_Q)
2482                        set_bit(PPC440SPE_ZERO_Q, &sw_desc->flags);
2483
2484                /* setup dst/src/mult */
2485                ppc440spe_adma_pq_set_dest(sw_desc, dst, flags);
2486
2487                while (src_cnt--) {
2488                        /* handle descriptors (if dst_cnt == 2) inside
2489                         * the ppc440spe_adma_pq_set_srcxxx() functions
2490                         */
2491                        ppc440spe_adma_pq_set_src(sw_desc, src[src_cnt],
2492                                                  src_cnt);
2493                        if (!(flags & DMA_PREP_PQ_DISABLE_Q))
2494                                mult = scf[src_cnt];
2495                        ppc440spe_adma_pq_set_src_mult(sw_desc,
2496                                        mult, src_cnt, dst_cnt - 1);
2497                }
2498        }
2499        spin_unlock_bh(&ppc440spe_chan->lock);
2500        ppc440spe_desc_set_rxor_block_size(len);
2501        return sw_desc;
2502}
2503
2504/**
2505 * ppc440spe_adma_prep_dma_pq - prepare CDB (group) for a GF-XOR operation
2506 */
2507static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_pq(
2508                struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src,
2509                unsigned int src_cnt, const unsigned char *scf,
2510                size_t len, unsigned long flags)
2511{
2512        struct ppc440spe_adma_chan *ppc440spe_chan;
2513        struct ppc440spe_adma_desc_slot *sw_desc = NULL;
2514        int dst_cnt = 0;
2515
2516        ppc440spe_chan = to_ppc440spe_adma_chan(chan);
2517
2518        ADMA_LL_DBG(prep_dma_pq_dbg(ppc440spe_chan->device->id,
2519                                    dst, src, src_cnt));
2520        BUG_ON(!len);
2521        BUG_ON(len > PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT);
2522        BUG_ON(!src_cnt);
2523
2524        if (src_cnt == 1 && dst[1] == src[0]) {
2525                dma_addr_t dest[2];
2526
2527                /* dst[1] is real destination (Q) */
2528                dest[0] = dst[1];
2529                /* this is the page to multicast source data to */
2530                dest[1] = ppc440spe_chan->qdest;
2531                sw_desc = ppc440spe_dma01_prep_mult(ppc440spe_chan,
2532                                dest, 2, src, src_cnt, scf, len, flags);
2533                return sw_desc ? &sw_desc->async_tx : NULL;
2534        }
2535
2536        if (src_cnt == 2 && dst[1] == src[1]) {
2537                sw_desc = ppc440spe_dma01_prep_sum_product(ppc440spe_chan,
2538                                        &dst[1], src, 2, scf, len, flags);
2539                return sw_desc ? &sw_desc->async_tx : NULL;
2540        }
2541
2542        if (!(flags & DMA_PREP_PQ_DISABLE_P)) {
2543                BUG_ON(!dst[0]);
2544                dst_cnt++;
2545                flags |= DMA_PREP_ZERO_P;
2546        }
2547
2548        if (!(flags & DMA_PREP_PQ_DISABLE_Q)) {
2549                BUG_ON(!dst[1]);
2550                dst_cnt++;
2551                flags |= DMA_PREP_ZERO_Q;
2552        }
2553
2554        BUG_ON(!dst_cnt);
2555
2556        dev_dbg(ppc440spe_chan->device->common.dev,
2557                "ppc440spe adma%d: %s src_cnt: %d len: %u int_en: %d\n",
2558                ppc440spe_chan->device->id, __func__, src_cnt, len,
2559                flags & DMA_PREP_INTERRUPT ? 1 : 0);
2560
2561        switch (ppc440spe_chan->device->id) {
2562        case PPC440SPE_DMA0_ID:
2563        case PPC440SPE_DMA1_ID:
2564                sw_desc = ppc440spe_dma01_prep_pq(ppc440spe_chan,
2565                                dst, dst_cnt, src, src_cnt, scf,
2566                                len, flags);
2567                break;
2568
2569        case PPC440SPE_XOR_ID:
2570                sw_desc = ppc440spe_dma2_prep_pq(ppc440spe_chan,
2571                                dst, dst_cnt, src, src_cnt, scf,
2572                                len, flags);
2573                break;
2574        }
2575
2576        return sw_desc ? &sw_desc->async_tx : NULL;
2577}
2578
2579/**
2580 * ppc440spe_adma_prep_dma_pqzero_sum - prepare CDB group for
2581 * a PQ_ZERO_SUM operation
2582 */
2583static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_pqzero_sum(
2584                struct dma_chan *chan, dma_addr_t *pq, dma_addr_t *src,
2585                unsigned int src_cnt, const unsigned char *scf, size_t len,
2586                enum sum_check_flags *pqres, unsigned long flags)
2587{
2588        struct ppc440spe_adma_chan *ppc440spe_chan;
2589        struct ppc440spe_adma_desc_slot *sw_desc, *iter;
2590        dma_addr_t pdest, qdest;
2591        int slot_cnt, slots_per_op, idst, dst_cnt;
2592
2593        ppc440spe_chan = to_ppc440spe_adma_chan(chan);
2594
2595        if (flags & DMA_PREP_PQ_DISABLE_P)
2596                pdest = 0;
2597        else
2598                pdest = pq[0];
2599
2600        if (flags & DMA_PREP_PQ_DISABLE_Q)
2601                qdest = 0;
2602        else
2603                qdest = pq[1];
2604
2605        ADMA_LL_DBG(prep_dma_pqzero_sum_dbg(ppc440spe_chan->device->id,
2606                                            src, src_cnt, scf));
2607
2608        /* Always use WXOR for P/Q calculations (two destinations).
2609         * Need 1 or 2 extra slots to verify results are zero.
2610         */
2611        idst = dst_cnt = (pdest && qdest) ? 2 : 1;
2612
2613        /* One additional slot per destination to clone P/Q
2614         * before calculation (we have to preserve destinations).
2615         */
2616        slot_cnt = src_cnt + dst_cnt * 2;
2617        slots_per_op = 1;
2618
2619        spin_lock_bh(&ppc440spe_chan->lock);
2620        sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt,
2621                                             slots_per_op);
2622        if (sw_desc) {
2623                ppc440spe_desc_init_dma01pqzero_sum(sw_desc, dst_cnt, src_cnt);
2624
2625                /* Setup byte count for each slot just allocated */
2626                sw_desc->async_tx.flags = flags;
2627                list_for_each_entry(iter, &sw_desc->group_list, chain_node) {
2628                        ppc440spe_desc_set_byte_count(iter, ppc440spe_chan,
2629                                                      len);
2630                        iter->unmap_len = len;
2631                }
2632
2633                if (pdest) {
2634                        struct dma_cdb *hw_desc;
2635                        struct ppc440spe_adma_chan *chan;
2636
2637                        iter = sw_desc->group_head;
2638                        chan = to_ppc440spe_adma_chan(iter->async_tx.chan);
2639                        memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2640                        iter->hw_next = list_entry(iter->chain_node.next,
2641                                                struct ppc440spe_adma_desc_slot,
2642                                                chain_node);
2643                        hw_desc = iter->hw_desc;
2644                        hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
2645                        iter->src_cnt = 0;
2646                        iter->dst_cnt = 0;
2647                        ppc440spe_desc_set_dest_addr(iter, chan, 0,
2648                                                     ppc440spe_chan->pdest, 0);
2649                        ppc440spe_desc_set_src_addr(iter, chan, 0, 0, pdest);
2650                        ppc440spe_desc_set_byte_count(iter, ppc440spe_chan,
2651                                                      len);
2652                        iter->unmap_len = 0;
2653                        /* override pdest to preserve original P */
2654                        pdest = ppc440spe_chan->pdest;
2655                }
2656                if (qdest) {
2657                        struct dma_cdb *hw_desc;
2658                        struct ppc440spe_adma_chan *chan;
2659
2660                        iter = list_first_entry(&sw_desc->group_list,
2661                                                struct ppc440spe_adma_desc_slot,
2662                                                chain_node);
2663                        chan = to_ppc440spe_adma_chan(iter->async_tx.chan);
2664
2665                        if (pdest) {
2666                                iter = list_entry(iter->chain_node.next,
2667                                                struct ppc440spe_adma_desc_slot,
2668                                                chain_node);
2669                        }
2670
2671                        memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2672                        iter->hw_next = list_entry(iter->chain_node.next,
2673                                                struct ppc440spe_adma_desc_slot,
2674                                                chain_node);
2675                        hw_desc = iter->hw_desc;
2676                        hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
2677                        iter->src_cnt = 0;
2678                        iter->dst_cnt = 0;
2679                        ppc440spe_desc_set_dest_addr(iter, chan, 0,
2680                                                     ppc440spe_chan->qdest, 0);
2681                        ppc440spe_desc_set_src_addr(iter, chan, 0, 0, qdest);
2682                        ppc440spe_desc_set_byte_count(iter, ppc440spe_chan,
2683                                                      len);
2684                        iter->unmap_len = 0;
2685                        /* override qdest to preserve original Q */
2686                        qdest = ppc440spe_chan->qdest;
2687                }
2688
2689                /* Setup destinations for P/Q ops */
2690                ppc440spe_adma_pqzero_sum_set_dest(sw_desc, pdest, qdest);
2691
2692                /* Setup zero QWORDs into DCHECK CDBs */
2693                idst = dst_cnt;
2694                list_for_each_entry_reverse(iter, &sw_desc->group_list,
2695                                            chain_node) {
2696                        /*
2697                         * The last CDB corresponds to Q-parity check,
2698                         * the one before last CDB corresponds
2699                         * P-parity check
2700                         */
2701                        if (idst == DMA_DEST_MAX_NUM) {
2702                                if (idst == dst_cnt) {
2703                                        set_bit(PPC440SPE_DESC_QCHECK,
2704                                                &iter->flags);
2705                                } else {
2706                                        set_bit(PPC440SPE_DESC_PCHECK,
2707                                                &iter->flags);
2708                                }
2709                        } else {
2710                                if (qdest) {
2711                                        set_bit(PPC440SPE_DESC_QCHECK,
2712                                                &iter->flags);
2713                                } else {
2714                                        set_bit(PPC440SPE_DESC_PCHECK,
2715                                                &iter->flags);
2716                                }
2717                        }
2718                        iter->xor_check_result = pqres;
2719
2720                        /*
2721                         * set it to zero, if check fail then result will
2722                         * be updated
2723                         */
2724                        *iter->xor_check_result = 0;
2725                        ppc440spe_desc_set_dcheck(iter, ppc440spe_chan,
2726                                ppc440spe_qword);
2727
2728                        if (!(--dst_cnt))
2729                                break;
2730                }
2731
2732                /* Setup sources and mults for P/Q ops */
2733                list_for_each_entry_continue_reverse(iter, &sw_desc->group_list,
2734                                                     chain_node) {
2735                        struct ppc440spe_adma_chan *chan;
2736                        u32 mult_dst;
2737
2738                        chan = to_ppc440spe_adma_chan(iter->async_tx.chan);
2739                        ppc440spe_desc_set_src_addr(iter, chan, 0,
2740                                                    DMA_CUED_XOR_HB,
2741                                                    src[src_cnt - 1]);
2742                        if (qdest) {
2743                                mult_dst = (dst_cnt - 1) ? DMA_CDB_SG_DST2 :
2744                                                           DMA_CDB_SG_DST1;
2745                                ppc440spe_desc_set_src_mult(iter, chan,
2746                                                            DMA_CUED_MULT1_OFF,
2747                                                            mult_dst,
2748                                                            scf[src_cnt - 1]);
2749                        }
2750                        if (!(--src_cnt))
2751                                break;
2752                }
2753        }
2754        spin_unlock_bh(&ppc440spe_chan->lock);
2755        return sw_desc ? &sw_desc->async_tx : NULL;
2756}
2757
2758/**
2759 * ppc440spe_adma_prep_dma_xor_zero_sum - prepare CDB group for
2760 * XOR ZERO_SUM operation
2761 */
2762static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_xor_zero_sum(
2763                struct dma_chan *chan, dma_addr_t *src, unsigned int src_cnt,
2764                size_t len, enum sum_check_flags *result, unsigned long flags)
2765{
2766        struct dma_async_tx_descriptor *tx;
2767        dma_addr_t pq[2];
2768
2769        /* validate P, disable Q */
2770        pq[0] = src[0];
2771        pq[1] = 0;
2772        flags |= DMA_PREP_PQ_DISABLE_Q;
2773
2774        tx = ppc440spe_adma_prep_dma_pqzero_sum(chan, pq, &src[1],
2775                                                src_cnt - 1, 0, len,
2776                                                result, flags);
2777        return tx;
2778}
2779
2780/**
2781 * ppc440spe_adma_set_dest - set destination address into descriptor
2782 */
2783static void ppc440spe_adma_set_dest(struct ppc440spe_adma_desc_slot *sw_desc,
2784                dma_addr_t addr, int index)
2785{
2786        struct ppc440spe_adma_chan *chan;
2787
2788        BUG_ON(index >= sw_desc->dst_cnt);
2789
2790        chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
2791
2792        switch (chan->device->id) {
2793        case PPC440SPE_DMA0_ID:
2794        case PPC440SPE_DMA1_ID:
2795                /* to do: support transfers lengths >
2796                 * PPC440SPE_ADMA_DMA/XOR_MAX_BYTE_COUNT
2797                 */
2798                ppc440spe_desc_set_dest_addr(sw_desc->group_head,
2799                        chan, 0, addr, index);
2800                break;
2801        case PPC440SPE_XOR_ID:
2802                sw_desc = ppc440spe_get_group_entry(sw_desc, index);
2803                ppc440spe_desc_set_dest_addr(sw_desc,
2804                        chan, 0, addr, index);
2805                break;
2806        }
2807}
2808
2809static void ppc440spe_adma_pq_zero_op(struct ppc440spe_adma_desc_slot *iter,
2810                struct ppc440spe_adma_chan *chan, dma_addr_t addr)
2811{
2812        /*  To clear destinations update the descriptor
2813         * (P or Q depending on index) as follows:
2814         * addr is destination (0 corresponds to SG2):
2815         */
2816        ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE, addr, 0);
2817
2818        /* ... and the addr is source: */
2819        ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB, addr);
2820
2821        /* addr is always SG2 then the mult is always DST1 */
2822        ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF,
2823                                    DMA_CDB_SG_DST1, 1);
2824}
2825
2826/**
2827 * ppc440spe_adma_pq_set_dest - set destination address into descriptor
2828 * for the PQXOR operation
2829 */
2830static void ppc440spe_adma_pq_set_dest(struct ppc440spe_adma_desc_slot *sw_desc,
2831                dma_addr_t *addrs, unsigned long flags)
2832{
2833        struct ppc440spe_adma_desc_slot *iter;
2834        struct ppc440spe_adma_chan *chan;
2835        dma_addr_t paddr, qaddr;
2836        dma_addr_t addr = 0, ppath, qpath;
2837        int index = 0, i;
2838
2839        chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
2840
2841        if (flags & DMA_PREP_PQ_DISABLE_P)
2842                paddr = 0;
2843        else
2844                paddr = addrs[0];
2845
2846        if (flags & DMA_PREP_PQ_DISABLE_Q)
2847                qaddr = 0;
2848        else
2849                qaddr = addrs[1];
2850
2851        if (!paddr || !qaddr)
2852                addr = paddr ? paddr : qaddr;
2853
2854        switch (chan->device->id) {
2855        case PPC440SPE_DMA0_ID:
2856        case PPC440SPE_DMA1_ID:
2857                /* walk through the WXOR source list and set P/Q-destinations
2858                 * for each slot:
2859                 */
2860                if (!test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags)) {
2861                        /* This is WXOR-only chain; may have 1/2 zero descs */
2862                        if (test_bit(PPC440SPE_ZERO_P, &sw_desc->flags))
2863                                index++;
2864                        if (test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags))
2865                                index++;
2866
2867                        iter = ppc440spe_get_group_entry(sw_desc, index);
2868                        if (addr) {
2869                                /* one destination */
2870                                list_for_each_entry_from(iter,
2871                                        &sw_desc->group_list, chain_node)
2872                                        ppc440spe_desc_set_dest_addr(iter, chan,
2873                                                DMA_CUED_XOR_BASE, addr, 0);
2874                        } else {
2875                                /* two destinations */
2876                                list_for_each_entry_from(iter,
2877                                        &sw_desc->group_list, chain_node) {
2878                                        ppc440spe_desc_set_dest_addr(iter, chan,
2879                                                DMA_CUED_XOR_BASE, paddr, 0);
2880                                        ppc440spe_desc_set_dest_addr(iter, chan,
2881                                                DMA_CUED_XOR_BASE, qaddr, 1);
2882                                }
2883                        }
2884
2885                        if (index) {
2886                                /*  To clear destinations update the descriptor
2887                                 * (1st,2nd, or both depending on flags)
2888                                 */
2889                                index = 0;
2890                                if (test_bit(PPC440SPE_ZERO_P,
2891                                                &sw_desc->flags)) {
2892                                        iter = ppc440spe_get_group_entry(
2893                                                        sw_desc, index++);
2894                                        ppc440spe_adma_pq_zero_op(iter, chan,
2895                                                        paddr);
2896                                }
2897
2898                                if (test_bit(PPC440SPE_ZERO_Q,
2899                                                &sw_desc->flags)) {
2900                                        iter = ppc440spe_get_group_entry(
2901                                                        sw_desc, index++);
2902                                        ppc440spe_adma_pq_zero_op(iter, chan,
2903                                                        qaddr);
2904                                }
2905
2906                                return;
2907                        }
2908                } else {
2909                        /* This is RXOR-only or RXOR/WXOR mixed chain */
2910
2911                        /* If we want to include destination into calculations,
2912                         * then make dest addresses cued with mult=1 (XOR).
2913                         */
2914                        ppath = test_bit(PPC440SPE_ZERO_P, &sw_desc->flags) ?
2915                                        DMA_CUED_XOR_HB :
2916                                        DMA_CUED_XOR_BASE |
2917                                                (1 << DMA_CUED_MULT1_OFF);
2918                        qpath = test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags) ?
2919                                        DMA_CUED_XOR_HB :
2920                                        DMA_CUED_XOR_BASE |
2921                                                (1 << DMA_CUED_MULT1_OFF);
2922
2923                        /* Setup destination(s) in RXOR slot(s) */
2924                        iter = ppc440spe_get_group_entry(sw_desc, index++);
2925                        ppc440spe_desc_set_dest_addr(iter, chan,
2926                                                paddr ? ppath : qpath,
2927                                                paddr ? paddr : qaddr, 0);
2928                        if (!addr) {
2929                                /* two destinations */
2930                                iter = ppc440spe_get_group_entry(sw_desc,
2931                                                                 index++);
2932                                ppc440spe_desc_set_dest_addr(iter, chan,
2933                                                qpath, qaddr, 0);
2934                        }
2935
2936                        if (test_bit(PPC440SPE_DESC_WXOR, &sw_desc->flags)) {
2937                                /* Setup destination(s) in remaining WXOR
2938                                 * slots
2939                                 */
2940                                iter = ppc440spe_get_group_entry(sw_desc,
2941                                                                 index);
2942                                if (addr) {
2943                                        /* one destination */
2944                                        list_for_each_entry_from(iter,
2945                                            &sw_desc->group_list,
2946                                            chain_node)
2947                                                ppc440spe_desc_set_dest_addr(
2948                                                        iter, chan,
2949                                                        DMA_CUED_XOR_BASE,
2950                                                        addr, 0);
2951
2952                                } else {
2953                                        /* two destinations */
2954                                        list_for_each_entry_from(iter,
2955                                            &sw_desc->group_list,
2956                                            chain_node) {
2957                                                ppc440spe_desc_set_dest_addr(
2958                                                        iter, chan,
2959                                                        DMA_CUED_XOR_BASE,
2960                                                        paddr, 0);
2961                                                ppc440spe_desc_set_dest_addr(
2962                                                        iter, chan,
2963                                                        DMA_CUED_XOR_BASE,
2964                                                        qaddr, 1);
2965                                        }
2966                                }
2967                        }
2968
2969                }
2970                break;
2971
2972        case PPC440SPE_XOR_ID:
2973                /* DMA2 descriptors have only 1 destination, so there are
2974                 * two chains - one for each dest.
2975                 * If we want to include destination into calculations,
2976                 * then make dest addresses cued with mult=1 (XOR).
2977                 */
2978                ppath = test_bit(PPC440SPE_ZERO_P, &sw_desc->flags) ?
2979                                DMA_CUED_XOR_HB :
2980                                DMA_CUED_XOR_BASE |
2981                                        (1 << DMA_CUED_MULT1_OFF);
2982
2983                qpath = test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags) ?
2984                                DMA_CUED_XOR_HB :
2985                                DMA_CUED_XOR_BASE |
2986                                        (1 << DMA_CUED_MULT1_OFF);
2987
2988                iter = ppc440spe_get_group_entry(sw_desc, 0);
2989                for (i = 0; i < sw_desc->descs_per_op; i++) {
2990                        ppc440spe_desc_set_dest_addr(iter, chan,
2991                                paddr ? ppath : qpath,
2992                                paddr ? paddr : qaddr, 0);
2993                        iter = list_entry(iter->chain_node.next,
2994                                          struct ppc440spe_adma_desc_slot,
2995                                          chain_node);
2996                }
2997
2998                if (!addr) {
2999                        /* Two destinations; setup Q here */
3000                        iter = ppc440spe_get_group_entry(sw_desc,
3001                                sw_desc->descs_per_op);
3002                        for (i = 0; i < sw_desc->descs_per_op; i++) {
3003                                ppc440spe_desc_set_dest_addr(iter,
3004                                        chan, qpath, qaddr, 0);
3005                                iter = list_entry(iter->chain_node.next,
3006                                                struct ppc440spe_adma_desc_slot,
3007                                                chain_node);
3008                        }
3009                }
3010
3011                break;
3012        }
3013}
3014
3015/**
3016 * ppc440spe_adma_pq_zero_sum_set_dest - set destination address into descriptor
3017 * for the PQ_ZERO_SUM operation
3018 */
3019static void ppc440spe_adma_pqzero_sum_set_dest(
3020                struct ppc440spe_adma_desc_slot *sw_desc,
3021                dma_addr_t paddr, dma_addr_t qaddr)
3022{
3023        struct ppc440spe_adma_desc_slot *iter, *end;
3024        struct ppc440spe_adma_chan *chan;
3025        dma_addr_t addr = 0;
3026        int idx;
3027
3028        chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
3029
3030        /* walk through the WXOR source list and set P/Q-destinations
3031         * for each slot
3032         */
3033        idx = (paddr && qaddr) ? 2 : 1;
3034        /* set end */
3035        list_for_each_entry_reverse(end, &sw_desc->group_list,
3036                                    chain_node) {
3037                if (!(--idx))
3038                        break;
3039        }
3040        /* set start */
3041        idx = (paddr && qaddr) ? 2 : 1;
3042        iter = ppc440spe_get_group_entry(sw_desc, idx);
3043
3044        if (paddr && qaddr) {
3045                /* two destinations */
3046                list_for_each_entry_from(iter, &sw_desc->group_list,
3047                                         chain_node) {
3048                        if (unlikely(iter == end))
3049                                break;
3050                        ppc440spe_desc_set_dest_addr(iter, chan,
3051                                                DMA_CUED_XOR_BASE, paddr, 0);
3052                        ppc440spe_desc_set_dest_addr(iter, chan,
3053                                                DMA_CUED_XOR_BASE, qaddr, 1);
3054                }
3055        } else {
3056                /* one destination */
3057                addr = paddr ? paddr : qaddr;
3058                list_for_each_entry_from(iter, &sw_desc->group_list,
3059                                         chain_node) {
3060                        if (unlikely(iter == end))
3061                                break;
3062                        ppc440spe_desc_set_dest_addr(iter, chan,
3063                                                DMA_CUED_XOR_BASE, addr, 0);
3064                }
3065        }
3066
3067        /*  The remaining descriptors are DATACHECK. These have no need in
3068         * destination. Actually, these destinations are used there
3069         * as sources for check operation. So, set addr as source.
3070         */
3071        ppc440spe_desc_set_src_addr(end, chan, 0, 0, addr ? addr : paddr);
3072
3073        if (!addr) {
3074                end = list_entry(end->chain_node.next,
3075                                 struct ppc440spe_adma_desc_slot, chain_node);
3076                ppc440spe_desc_set_src_addr(end, chan, 0, 0, qaddr);
3077        }
3078}
3079
3080/**
3081 * ppc440spe_desc_set_xor_src_cnt - set source count into descriptor
3082 */
3083static inline void ppc440spe_desc_set_xor_src_cnt(
3084                        struct ppc440spe_adma_desc_slot *desc,
3085                        int src_cnt)
3086{
3087        struct xor_cb *hw_desc = desc->hw_desc;
3088
3089        hw_desc->cbc &= ~XOR_CDCR_OAC_MSK;
3090        hw_desc->cbc |= src_cnt;
3091}
3092
3093/**
3094 * ppc440spe_adma_pq_set_src - set source address into descriptor
3095 */
3096static void ppc440spe_adma_pq_set_src(struct ppc440spe_adma_desc_slot *sw_desc,
3097                dma_addr_t addr, int index)
3098{
3099        struct ppc440spe_adma_chan *chan;
3100        dma_addr_t haddr = 0;
3101        struct ppc440spe_adma_desc_slot *iter = NULL;
3102
3103        chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
3104
3105        switch (chan->device->id) {
3106        case PPC440SPE_DMA0_ID:
3107        case PPC440SPE_DMA1_ID:
3108                /* DMA0,1 may do: WXOR, RXOR, RXOR+WXORs chain
3109                 */
3110                if (test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags)) {
3111                        /* RXOR-only or RXOR/WXOR operation */
3112                        int iskip = test_bit(PPC440SPE_DESC_RXOR12,
3113                                &sw_desc->flags) ?  2 : 3;
3114
3115                        if (index == 0) {
3116                                /* 1st slot (RXOR) */
3117                                /* setup sources region (R1-2-3, R1-2-4,
3118                                 * or R1-2-5)
3119                                 */
3120                                if (test_bit(PPC440SPE_DESC_RXOR12,
3121                                                &sw_desc->flags))
3122                                        haddr = DMA_RXOR12 <<
3123                                                DMA_CUED_REGION_OFF;
3124                                else if (test_bit(PPC440SPE_DESC_RXOR123,
3125                                    &sw_desc->flags))
3126                                        haddr = DMA_RXOR123 <<
3127                                                DMA_CUED_REGION_OFF;
3128                                else if (test_bit(PPC440SPE_DESC_RXOR124,
3129                                    &sw_desc->flags))
3130                                        haddr = DMA_RXOR124 <<
3131                                                DMA_CUED_REGION_OFF;
3132                                else if (test_bit(PPC440SPE_DESC_RXOR125,
3133                                    &sw_desc->flags))
3134                                        haddr = DMA_RXOR125 <<
3135                                                DMA_CUED_REGION_OFF;
3136                                else
3137                                        BUG();
3138                                haddr |= DMA_CUED_XOR_BASE;
3139                                iter = ppc440spe_get_group_entry(sw_desc, 0);
3140                        } else if (index < iskip) {
3141                                /* 1st slot (RXOR)
3142                                 * shall actually set source address only once
3143                                 * instead of first <iskip>
3144                                 */
3145                                iter = NULL;
3146                        } else {
3147                                /* 2nd/3d and next slots (WXOR);
3148                                 * skip first slot with RXOR
3149                                 */
3150                                haddr = DMA_CUED_XOR_HB;
3151                                iter = ppc440spe_get_group_entry(sw_desc,
3152                                    index - iskip + sw_desc->dst_cnt);
3153                        }
3154                } else {
3155                        int znum = 0;
3156
3157                        /* WXOR-only operation; skip first slots with
3158                         * zeroing destinations
3159                         */
3160                        if (test_bit(PPC440SPE_ZERO_P, &sw_desc->flags))
3161                                znum++;
3162                        if (test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags))
3163                                znum++;
3164
3165                        haddr = DMA_CUED_XOR_HB;
3166                        iter = ppc440spe_get_group_entry(sw_desc,
3167                                        index + znum);
3168                }
3169
3170                if (likely(iter)) {
3171                        ppc440spe_desc_set_src_addr(iter, chan, 0, haddr, addr);
3172
3173                        if (!index &&
3174                            test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags) &&
3175                            sw_desc->dst_cnt == 2) {
3176                                /* if we have two destinations for RXOR, then
3177                                 * setup source in the second descr too
3178                                 */
3179                                iter = ppc440spe_get_group_entry(sw_desc, 1);
3180                                ppc440spe_desc_set_src_addr(iter, chan, 0,
3181                                        haddr, addr);
3182                        }
3183                }
3184                break;
3185
3186        case PPC440SPE_XOR_ID:
3187                /* DMA2 may do Biskup */
3188                iter = sw_desc->group_head;
3189                if (iter->dst_cnt == 2) {
3190                        /* both P & Q calculations required; set P src here */
3191                        ppc440spe_adma_dma2rxor_set_src(iter, index, addr);
3192
3193                        /* this is for Q */
3194                        iter = ppc440spe_get_group_entry(sw_desc,
3195                                sw_desc->descs_per_op);
3196                }
3197                ppc440spe_adma_dma2rxor_set_src(iter, index, addr);
3198                break;
3199        }
3200}
3201
3202/**
3203 * ppc440spe_adma_memcpy_xor_set_src - set source address into descriptor
3204 */
3205static void ppc440spe_adma_memcpy_xor_set_src(
3206                struct ppc440spe_adma_desc_slot *sw_desc,
3207                dma_addr_t addr, int index)
3208{
3209        struct ppc440spe_adma_chan *chan;
3210
3211        chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
3212        sw_desc = sw_desc->group_head;
3213
3214        if (likely(sw_desc))
3215                ppc440spe_desc_set_src_addr(sw_desc, chan, index, 0, addr);
3216}
3217
3218/**
3219 * ppc440spe_adma_dma2rxor_inc_addr  -
3220 */
3221static void ppc440spe_adma_dma2rxor_inc_addr(
3222                struct ppc440spe_adma_desc_slot *desc,
3223                struct ppc440spe_rxor *cursor, int index, int src_cnt)
3224{
3225        cursor->addr_count++;
3226        if (index == src_cnt - 1) {
3227                ppc440spe_desc_set_xor_src_cnt(desc, cursor->addr_count);
3228        } else if (cursor->addr_count == XOR_MAX_OPS) {
3229                ppc440spe_desc_set_xor_src_cnt(desc, cursor->addr_count);
3230                cursor->addr_count = 0;
3231                cursor->desc_count++;
3232        }
3233}
3234
3235/**
3236 * ppc440spe_adma_dma2rxor_prep_src - setup RXOR types in DMA2 CDB
3237 */
3238static int ppc440spe_adma_dma2rxor_prep_src(
3239                struct ppc440spe_adma_desc_slot *hdesc,
3240                struct ppc440spe_rxor *cursor, int index,
3241                int src_cnt, u32 addr)
3242{
3243        u32 sign;
3244        struct ppc440spe_adma_desc_slot *desc = hdesc;
3245        int i;
3246
3247        for (i = 0; i < cursor->desc_count; i++) {
3248                desc = list_entry(hdesc->chain_node.next,
3249                                  struct ppc440spe_adma_desc_slot,
3250                                  chain_node);
3251        }
3252
3253        switch (cursor->state) {
3254        case 0:
3255                if (addr == cursor->addrl + cursor->len) {
3256                        /* direct RXOR */
3257                        cursor->state = 1;
3258                        cursor->xor_count++;
3259                        if (index == src_cnt-1) {
3260                                ppc440spe_rxor_set_region(desc,
3261                                        cursor->addr_count,
3262                                        DMA_RXOR12 << DMA_CUED_REGION_OFF);
3263                                ppc440spe_adma_dma2rxor_inc_addr(
3264                                        desc, cursor, index, src_cnt);
3265                        }
3266                } else if (cursor->addrl == addr + cursor->len) {
3267                        /* reverse RXOR */
3268                        cursor->state = 1;
3269                        cursor->xor_count++;
3270                        set_bit(cursor->addr_count, &desc->reverse_flags[0]);
3271                        if (index == src_cnt-1) {
3272                                ppc440spe_rxor_set_region(desc,
3273                                        cursor->addr_count,
3274                                        DMA_RXOR12 << DMA_CUED_REGION_OFF);
3275                                ppc440spe_adma_dma2rxor_inc_addr(
3276                                        desc, cursor, index, src_cnt);
3277                        }
3278                } else {
3279                        printk(KERN_ERR "Cannot build "
3280                                "DMA2 RXOR command block.\n");
3281                        BUG();
3282                }
3283                break;
3284        case 1:
3285                sign = test_bit(cursor->addr_count,
3286                                desc->reverse_flags)
3287                        ? -1 : 1;
3288                if (index == src_cnt-2 || (sign == -1
3289                        && addr != cursor->addrl - 2*cursor->len)) {
3290                        cursor->state = 0;
3291                        cursor->xor_count = 1;
3292                        cursor->addrl = addr;
3293                        ppc440spe_rxor_set_region(desc,
3294                                cursor->addr_count,
3295                                DMA_RXOR12 << DMA_CUED_REGION_OFF);
3296                        ppc440spe_adma_dma2rxor_inc_addr(
3297                                desc, cursor, index, src_cnt);
3298                } else if (addr == cursor->addrl + 2*sign*cursor->len) {
3299                        cursor->state = 2;
3300                        cursor->xor_count = 0;
3301                        ppc440spe_rxor_set_region(desc,
3302                                cursor->addr_count,
3303                                DMA_RXOR123 << DMA_CUED_REGION_OFF);
3304                        if (index == src_cnt-1) {
3305                                ppc440spe_adma_dma2rxor_inc_addr(
3306                                        desc, cursor, index, src_cnt);
3307                        }
3308                } else if (addr == cursor->addrl + 3*cursor->len) {
3309                        cursor->state = 2;
3310                        cursor->xor_count = 0;
3311                        ppc440spe_rxor_set_region(desc,
3312                                cursor->addr_count,
3313                                DMA_RXOR124 << DMA_CUED_REGION_OFF);
3314                        if (index == src_cnt-1) {
3315                                ppc440spe_adma_dma2rxor_inc_addr(
3316                                        desc, cursor, index, src_cnt);
3317                        }
3318                } else if (addr == cursor->addrl + 4*cursor->len) {
3319                        cursor->state = 2;
3320                        cursor->xor_count = 0;
3321                        ppc440spe_rxor_set_region(desc,
3322                                cursor->addr_count,
3323                                DMA_RXOR125 << DMA_CUED_REGION_OFF);
3324                        if (index == src_cnt-1) {
3325                                ppc440spe_adma_dma2rxor_inc_addr(
3326                                        desc, cursor, index, src_cnt);
3327                        }
3328                } else {
3329                        cursor->state = 0;
3330                        cursor->xor_count = 1;
3331                        cursor->addrl = addr;
3332                        ppc440spe_rxor_set_region(desc,
3333                                cursor->addr_count,
3334                                DMA_RXOR12 << DMA_CUED_REGION_OFF);
3335                        ppc440spe_adma_dma2rxor_inc_addr(
3336                                desc, cursor, index, src_cnt);
3337                }
3338                break;
3339        case 2:
3340                cursor->state = 0;
3341                cursor->addrl = addr;
3342                cursor->xor_count++;
3343                if (index) {
3344                        ppc440spe_adma_dma2rxor_inc_addr(
3345                                desc, cursor, index, src_cnt);
3346                }
3347                break;
3348        }
3349
3350        return 0;
3351}
3352
3353/**
3354 * ppc440spe_adma_dma2rxor_set_src - set RXOR source address; it's assumed that
3355 *      ppc440spe_adma_dma2rxor_prep_src() has already done prior this call
3356 */
3357static void ppc440spe_adma_dma2rxor_set_src(
3358                struct ppc440spe_adma_desc_slot *desc,
3359                int index, dma_addr_t addr)
3360{
3361        struct xor_cb *xcb = desc->hw_desc;
3362        int k = 0, op = 0, lop = 0;
3363
3364        /* get the RXOR operand which corresponds to index addr */
3365        while (op <= index) {
3366                lop = op;
3367                if (k == XOR_MAX_OPS) {
3368                        k = 0;
3369                        desc = list_entry(desc->chain_node.next,
3370                                struct ppc440spe_adma_desc_slot, chain_node);
3371                        xcb = desc->hw_desc;
3372
3373                }
3374                if ((xcb->ops[k++].h & (DMA_RXOR12 << DMA_CUED_REGION_OFF)) ==
3375                    (DMA_RXOR12 << DMA_CUED_REGION_OFF))
3376                        op += 2;
3377                else
3378                        op += 3;
3379        }
3380
3381        BUG_ON(k < 1);
3382
3383        if (test_bit(k-1, desc->reverse_flags)) {
3384                /* reverse operand order; put last op in RXOR group */
3385                if (index == op - 1)
3386                        ppc440spe_rxor_set_src(desc, k - 1, addr);
3387        } else {
3388                /* direct operand order; put first op in RXOR group */
3389                if (index == lop)
3390                        ppc440spe_rxor_set_src(desc, k - 1, addr);
3391        }
3392}
3393
3394/**
3395 * ppc440spe_adma_dma2rxor_set_mult - set RXOR multipliers; it's assumed that
3396 *      ppc440spe_adma_dma2rxor_prep_src() has already done prior this call
3397 */
3398static void ppc440spe_adma_dma2rxor_set_mult(
3399                struct ppc440spe_adma_desc_slot *desc,
3400                int index, u8 mult)
3401{
3402        struct xor_cb *xcb = desc->hw_desc;
3403        int k = 0, op = 0, lop = 0;
3404
3405        /* get the RXOR operand which corresponds to index mult */
3406        while (op <= index) {
3407                lop = op;
3408                if (k == XOR_MAX_OPS) {
3409                        k = 0;
3410                        desc = list_entry(desc->chain_node.next,
3411                                          struct ppc440spe_adma_desc_slot,
3412                                          chain_node);
3413                        xcb = desc->hw_desc;
3414
3415                }
3416                if ((xcb->ops[k++].h & (DMA_RXOR12 << DMA_CUED_REGION_OFF)) ==
3417                    (DMA_RXOR12 << DMA_CUED_REGION_OFF))
3418                        op += 2;
3419                else
3420                        op += 3;
3421        }
3422
3423        BUG_ON(k < 1);
3424        if (test_bit(k-1, desc->reverse_flags)) {
3425                /* reverse order */
3426                ppc440spe_rxor_set_mult(desc, k - 1, op - index - 1, mult);
3427        } else {
3428                /* direct order */
3429                ppc440spe_rxor_set_mult(desc, k - 1, index - lop, mult);
3430        }
3431}
3432
3433/**
3434 * ppc440spe_init_rxor_cursor -
3435 */
3436static void ppc440spe_init_rxor_cursor(struct ppc440spe_rxor *cursor)
3437{
3438        memset(cursor, 0, sizeof(struct ppc440spe_rxor));
3439        cursor->state = 2;
3440}
3441
3442/**
3443 * ppc440spe_adma_pq_set_src_mult - set multiplication coefficient into
3444 * descriptor for the PQXOR operation
3445 */
3446static void ppc440spe_adma_pq_set_src_mult(
3447                struct ppc440spe_adma_desc_slot *sw_desc,
3448                unsigned char mult, int index, int dst_pos)
3449{
3450        struct ppc440spe_adma_chan *chan;
3451        u32 mult_idx, mult_dst;
3452        struct ppc440spe_adma_desc_slot *iter = NULL, *iter1 = NULL;
3453
3454        chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
3455
3456        switch (chan->device->id) {
3457        case PPC440SPE_DMA0_ID:
3458        case PPC440SPE_DMA1_ID:
3459                if (test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags)) {
3460                        int region = test_bit(PPC440SPE_DESC_RXOR12,
3461                                        &sw_desc->flags) ? 2 : 3;
3462
3463                        if (index < region) {
3464                                /* RXOR multipliers */
3465                                iter = ppc440spe_get_group_entry(sw_desc,
3466                                        sw_desc->dst_cnt - 1);
3467                                if (sw_desc->dst_cnt == 2)
3468                                        iter1 = ppc440spe_get_group_entry(
3469                                                        sw_desc, 0);
3470
3471                                mult_idx = DMA_CUED_MULT1_OFF + (index << 3);
3472                                mult_dst = DMA_CDB_SG_SRC;
3473                        } else {
3474                                /* WXOR multiplier */
3475                                iter = ppc440spe_get_group_entry(sw_desc,
3476                                                        index - region +
3477                                                        sw_desc->dst_cnt);
3478                                mult_idx = DMA_CUED_MULT1_OFF;
3479                                mult_dst = dst_pos ? DMA_CDB_SG_DST2 :
3480                                                     DMA_CDB_SG_DST1;
3481                        }
3482                } else {
3483                        int znum = 0;
3484
3485                        /* WXOR-only;
3486                         * skip first slots with destinations (if ZERO_DST has
3487                         * place)
3488                         */
3489                        if (test_bit(PPC440SPE_ZERO_P, &sw_desc->flags))
3490                                znum++;
3491                        if (test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags))
3492                                znum++;
3493
3494                        iter = ppc440spe_get_group_entry(sw_desc, index + znum);
3495                        mult_idx = DMA_CUED_MULT1_OFF;
3496                        mult_dst = dst_pos ? DMA_CDB_SG_DST2 : DMA_CDB_SG_DST1;
3497                }
3498
3499                if (likely(iter)) {
3500                        ppc440spe_desc_set_src_mult(iter, chan,
3501                                mult_idx, mult_dst, mult);
3502
3503                        if (unlikely(iter1)) {
3504                                /* if we have two destinations for RXOR, then
3505                                 * we've just set Q mult. Set-up P now.
3506                                 */
3507                                ppc440spe_desc_set_src_mult(iter1, chan,
3508                                        mult_idx, mult_dst, 1);
3509                        }
3510
3511                }
3512                break;
3513
3514        case PPC440SPE_XOR_ID:
3515                iter = sw_desc->group_head;
3516                if (sw_desc->dst_cnt == 2) {
3517                        /* both P & Q calculations required; set P mult here */
3518                        ppc440spe_adma_dma2rxor_set_mult(iter, index, 1);
3519
3520                        /* and then set Q mult */
3521                        iter = ppc440spe_get_group_entry(sw_desc,
3522                               sw_desc->descs_per_op);
3523                }
3524                ppc440spe_adma_dma2rxor_set_mult(iter, index, mult);
3525                break;
3526        }
3527}
3528
3529/**
3530 * ppc440spe_adma_free_chan_resources - free the resources allocated
3531 */
3532static void ppc440spe_adma_free_chan_resources(struct dma_chan *chan)
3533{
3534        struct ppc440spe_adma_chan *ppc440spe_chan;
3535        struct ppc440spe_adma_desc_slot *iter, *_iter;
3536        int in_use_descs = 0;
3537
3538        ppc440spe_chan = to_ppc440spe_adma_chan(chan);
3539        ppc440spe_adma_slot_cleanup(ppc440spe_chan);
3540
3541        spin_lock_bh(&ppc440spe_chan->lock);
3542        list_for_each_entry_safe(iter, _iter, &ppc440spe_chan->chain,
3543                                        chain_node) {
3544                in_use_descs++;
3545                list_del(&iter->chain_node);
3546        }
3547        list_for_each_entry_safe_reverse(iter, _iter,
3548                        &ppc440spe_chan->all_slots, slot_node) {
3549                list_del(&iter->slot_node);
3550                kfree(iter);
3551                ppc440spe_chan->slots_allocated--;
3552        }
3553        ppc440spe_chan->last_used = NULL;
3554
3555        dev_dbg(ppc440spe_chan->device->common.dev,
3556                "ppc440spe adma%d %s slots_allocated %d\n",
3557                ppc440spe_chan->device->id,
3558                __func__, ppc440spe_chan->slots_allocated);
3559        spin_unlock_bh(&ppc440spe_chan->lock);
3560
3561        /* one is ok since we left it on there on purpose */
3562        if (in_use_descs > 1)
3563                printk(KERN_ERR "SPE: Freeing %d in use descriptors!\n",
3564                        in_use_descs - 1);
3565}
3566
3567/**
3568 * ppc440spe_adma_tx_status - poll the status of an ADMA transaction
3569 * @chan: ADMA channel handle
3570 * @cookie: ADMA transaction identifier
3571 * @txstate: a holder for the current state of the channel
3572 */
3573static enum dma_status ppc440spe_adma_tx_status(struct dma_chan *chan,
3574                        dma_cookie_t cookie, struct dma_tx_state *txstate)
3575{
3576        struct ppc440spe_adma_chan *ppc440spe_chan;
3577        enum dma_status ret;
3578
3579        ppc440spe_chan = to_ppc440spe_adma_chan(chan);
3580        ret = dma_cookie_status(chan, cookie, txstate);
3581        if (ret == DMA_COMPLETE)
3582                return ret;
3583
3584        ppc440spe_adma_slot_cleanup(ppc440spe_chan);
3585
3586        return dma_cookie_status(chan, cookie, txstate);
3587}
3588
3589/**
3590 * ppc440spe_adma_eot_handler - end of transfer interrupt handler
3591 */
3592static irqreturn_t ppc440spe_adma_eot_handler(int irq, void *data)
3593{
3594        struct ppc440spe_adma_chan *chan = data;
3595
3596        dev_dbg(chan->device->common.dev,
3597                "ppc440spe adma%d: %s\n", chan->device->id, __func__);
3598
3599        tasklet_schedule(&chan->irq_tasklet);
3600        ppc440spe_adma_device_clear_eot_status(chan);
3601
3602        return IRQ_HANDLED;
3603}
3604
3605/**
3606 * ppc440spe_adma_err_handler - DMA error interrupt handler;
3607 *      do the same things as a eot handler
3608 */
3609static irqreturn_t ppc440spe_adma_err_handler(int irq, void *data)
3610{
3611        struct ppc440spe_adma_chan *chan = data;
3612
3613        dev_dbg(chan->device->common.dev,
3614                "ppc440spe adma%d: %s\n", chan->device->id, __func__);
3615
3616        tasklet_schedule(&chan->irq_tasklet);
3617        ppc440spe_adma_device_clear_eot_status(chan);
3618
3619        return IRQ_HANDLED;
3620}
3621
3622/**
3623 * ppc440spe_test_callback - called when test operation has been done
3624 */
3625static void ppc440spe_test_callback(void *unused)
3626{
3627        complete(&ppc440spe_r6_test_comp);
3628}
3629
3630/**
3631 * ppc440spe_adma_issue_pending - flush all pending descriptors to h/w
3632 */
3633static void ppc440spe_adma_issue_pending(struct dma_chan *chan)
3634{
3635        struct ppc440spe_adma_chan *ppc440spe_chan;
3636
3637        ppc440spe_chan = to_ppc440spe_adma_chan(chan);
3638        dev_dbg(ppc440spe_chan->device->common.dev,
3639                "ppc440spe adma%d: %s %d \n", ppc440spe_chan->device->id,
3640                __func__, ppc440spe_chan->pending);
3641
3642        if (ppc440spe_chan->pending) {
3643                ppc440spe_chan->pending = 0;
3644                ppc440spe_chan_append(ppc440spe_chan);
3645        }
3646}
3647
3648/**
3649 * ppc440spe_chan_start_null_xor - initiate the first XOR operation (DMA engines
3650 *      use FIFOs (as opposite to chains used in XOR) so this is a XOR
3651 *      specific operation)
3652 */
3653static void ppc440spe_chan_start_null_xor(struct ppc440spe_adma_chan *chan)
3654{
3655        struct ppc440spe_adma_desc_slot *sw_desc, *group_start;
3656        dma_cookie_t cookie;
3657        int slot_cnt, slots_per_op;
3658
3659        dev_dbg(chan->device->common.dev,
3660                "ppc440spe adma%d: %s\n", chan->device->id, __func__);
3661
3662        spin_lock_bh(&chan->lock);
3663        slot_cnt = ppc440spe_chan_xor_slot_count(0, 2, &slots_per_op);
3664        sw_desc = ppc440spe_adma_alloc_slots(chan, slot_cnt, slots_per_op);
3665        if (sw_desc) {
3666                group_start = sw_desc->group_head;
3667                list_splice_init(&sw_desc->group_list, &chan->chain);
3668                async_tx_ack(&sw_desc->async_tx);
3669                ppc440spe_desc_init_null_xor(group_start);
3670
3671                cookie = dma_cookie_assign(&sw_desc->async_tx);
3672
3673                /* initialize the completed cookie to be less than
3674                 * the most recently used cookie
3675                 */
3676                chan->common.completed_cookie = cookie - 1;
3677
3678                /* channel should not be busy */
3679                BUG_ON(ppc440spe_chan_is_busy(chan));
3680
3681                /* set the descriptor address */
3682                ppc440spe_chan_set_first_xor_descriptor(chan, sw_desc);
3683
3684                /* run the descriptor */
3685                ppc440spe_chan_run(chan);
3686        } else
3687                printk(KERN_ERR "ppc440spe adma%d"
3688                        " failed to allocate null descriptor\n",
3689                        chan->device->id);
3690        spin_unlock_bh(&chan->lock);
3691}
3692
3693/**
3694 * ppc440spe_test_raid6 - test are RAID-6 capabilities enabled successfully.
3695 *      For this we just perform one WXOR operation with the same source
3696 *      and destination addresses, the GF-multiplier is 1; so if RAID-6
3697 *      capabilities are enabled then we'll get src/dst filled with zero.
3698 */
3699static int ppc440spe_test_raid6(struct ppc440spe_adma_chan *chan)
3700{
3701        struct ppc440spe_adma_desc_slot *sw_desc, *iter;
3702        struct page *pg;
3703        char *a;
3704        dma_addr_t dma_addr, addrs[2];
3705        unsigned long op = 0;
3706        int rval = 0;
3707
3708        set_bit(PPC440SPE_DESC_WXOR, &op);
3709
3710        pg = alloc_page(GFP_KERNEL);
3711        if (!pg)
3712                return -ENOMEM;
3713
3714        spin_lock_bh(&chan->lock);
3715        sw_desc = ppc440spe_adma_alloc_slots(chan, 1, 1);
3716        if (sw_desc) {
3717                /* 1 src, 1 dsr, int_ena, WXOR */
3718                ppc440spe_desc_init_dma01pq(sw_desc, 1, 1, 1, op);
3719                list_for_each_entry(iter, &sw_desc->group_list, chain_node) {
3720                        ppc440spe_desc_set_byte_count(iter, chan, PAGE_SIZE);
3721                        iter->unmap_len = PAGE_SIZE;
3722                }
3723        } else {
3724                rval = -EFAULT;
3725                spin_unlock_bh(&chan->lock);
3726                goto exit;
3727        }
3728        spin_unlock_bh(&chan->lock);
3729
3730        /* Fill the test page with ones */
3731        memset(page_address(pg), 0xFF, PAGE_SIZE);
3732        dma_addr = dma_map_page(chan->device->dev, pg, 0,
3733                                PAGE_SIZE, DMA_BIDIRECTIONAL);
3734
3735        /* Setup addresses */
3736        ppc440spe_adma_pq_set_src(sw_desc, dma_addr, 0);
3737        ppc440spe_adma_pq_set_src_mult(sw_desc, 1, 0, 0);
3738        addrs[0] = dma_addr;
3739        addrs[1] = 0;
3740        ppc440spe_adma_pq_set_dest(sw_desc, addrs, DMA_PREP_PQ_DISABLE_Q);
3741
3742        async_tx_ack(&sw_desc->async_tx);
3743        sw_desc->async_tx.callback = ppc440spe_test_callback;
3744        sw_desc->async_tx.callback_param = NULL;
3745
3746        init_completion(&ppc440spe_r6_test_comp);
3747
3748        ppc440spe_adma_tx_submit(&sw_desc->async_tx);
3749        ppc440spe_adma_issue_pending(&chan->common);
3750
3751        wait_for_completion(&ppc440spe_r6_test_comp);
3752
3753        /* Now check if the test page is zeroed */
3754        a = page_address(pg);
3755        if ((*(u32 *)a) == 0 && memcmp(a, a+4, PAGE_SIZE-4) == 0) {
3756                /* page is zero - RAID-6 enabled */
3757                rval = 0;
3758        } else {
3759                /* RAID-6 was not enabled */
3760                rval = -EINVAL;
3761        }
3762exit:
3763        __free_page(pg);
3764        return rval;
3765}
3766
3767static void ppc440spe_adma_init_capabilities(struct ppc440spe_adma_device *adev)
3768{
3769        switch (adev->id) {
3770        case PPC440SPE_DMA0_ID:
3771        case PPC440SPE_DMA1_ID:
3772                dma_cap_set(DMA_MEMCPY, adev->common.cap_mask);
3773                dma_cap_set(DMA_INTERRUPT, adev->common.cap_mask);
3774                dma_cap_set(DMA_PQ, adev->common.cap_mask);
3775                dma_cap_set(DMA_PQ_VAL, adev->common.cap_mask);
3776                dma_cap_set(DMA_XOR_VAL, adev->common.cap_mask);
3777                break;
3778        case PPC440SPE_XOR_ID:
3779                dma_cap_set(DMA_XOR, adev->common.cap_mask);
3780                dma_cap_set(DMA_PQ, adev->common.cap_mask);
3781                dma_cap_set(DMA_INTERRUPT, adev->common.cap_mask);
3782                adev->common.cap_mask = adev->common.cap_mask;
3783                break;
3784        }
3785
3786        /* Set base routines */
3787        adev->common.device_alloc_chan_resources =
3788                                ppc440spe_adma_alloc_chan_resources;
3789        adev->common.device_free_chan_resources =
3790                                ppc440spe_adma_free_chan_resources;
3791        adev->common.device_tx_status = ppc440spe_adma_tx_status;
3792        adev->common.device_issue_pending = ppc440spe_adma_issue_pending;
3793
3794        /* Set prep routines based on capability */
3795        if (dma_has_cap(DMA_MEMCPY, adev->common.cap_mask)) {
3796                adev->common.device_prep_dma_memcpy =
3797                        ppc440spe_adma_prep_dma_memcpy;
3798        }
3799        if (dma_has_cap(DMA_XOR, adev->common.cap_mask)) {
3800                adev->common.max_xor = XOR_MAX_OPS;
3801                adev->common.device_prep_dma_xor =
3802                        ppc440spe_adma_prep_dma_xor;
3803        }
3804        if (dma_has_cap(DMA_PQ, adev->common.cap_mask)) {
3805                switch (adev->id) {
3806                case PPC440SPE_DMA0_ID:
3807                        dma_set_maxpq(&adev->common,
3808                                DMA0_FIFO_SIZE / sizeof(struct dma_cdb), 0);
3809                        break;
3810                case PPC440SPE_DMA1_ID:
3811                        dma_set_maxpq(&adev->common,
3812                                DMA1_FIFO_SIZE / sizeof(struct dma_cdb), 0);
3813                        break;
3814                case PPC440SPE_XOR_ID:
3815                        adev->common.max_pq = XOR_MAX_OPS * 3;
3816                        break;
3817                }
3818                adev->common.device_prep_dma_pq =
3819                        ppc440spe_adma_prep_dma_pq;
3820        }
3821        if (dma_has_cap(DMA_PQ_VAL, adev->common.cap_mask)) {
3822                switch (adev->id) {
3823                case PPC440SPE_DMA0_ID:
3824                        adev->common.max_pq = DMA0_FIFO_SIZE /
3825                                                sizeof(struct dma_cdb);
3826                        break;
3827                case PPC440SPE_DMA1_ID:
3828                        adev->common.max_pq = DMA1_FIFO_SIZE /
3829                                                sizeof(struct dma_cdb);
3830                        break;
3831                }
3832                adev->common.device_prep_dma_pq_val =
3833                        ppc440spe_adma_prep_dma_pqzero_sum;
3834        }
3835        if (dma_has_cap(DMA_XOR_VAL, adev->common.cap_mask)) {
3836                switch (adev->id) {
3837                case PPC440SPE_DMA0_ID:
3838                        adev->common.max_xor = DMA0_FIFO_SIZE /
3839                                                sizeof(struct dma_cdb);
3840                        break;
3841                case PPC440SPE_DMA1_ID:
3842                        adev->common.max_xor = DMA1_FIFO_SIZE /
3843                                                sizeof(struct dma_cdb);
3844                        break;
3845                }
3846                adev->common.device_prep_dma_xor_val =
3847                        ppc440spe_adma_prep_dma_xor_zero_sum;
3848        }
3849        if (dma_has_cap(DMA_INTERRUPT, adev->common.cap_mask)) {
3850                adev->common.device_prep_dma_interrupt =
3851                        ppc440spe_adma_prep_dma_interrupt;
3852        }
3853        pr_info("%s: AMCC(R) PPC440SP(E) ADMA Engine: "
3854          "( %s%s%s%s%s%s)\n",
3855          dev_name(adev->dev),
3856          dma_has_cap(DMA_PQ, adev->common.cap_mask) ? "pq " : "",
3857          dma_has_cap(DMA_PQ_VAL, adev->common.cap_mask) ? "pq_val " : "",
3858          dma_has_cap(DMA_XOR, adev->common.cap_mask) ? "xor " : "",
3859          dma_has_cap(DMA_XOR_VAL, adev->common.cap_mask) ? "xor_val " : "",
3860          dma_has_cap(DMA_MEMCPY, adev->common.cap_mask) ? "memcpy " : "",
3861          dma_has_cap(DMA_INTERRUPT, adev->common.cap_mask) ? "intr " : "");
3862}
3863
3864static int ppc440spe_adma_setup_irqs(struct ppc440spe_adma_device *adev,
3865                                     struct ppc440spe_adma_chan *chan,
3866                                     int *initcode)
3867{
3868        struct platform_device *ofdev;
3869        struct device_node *np;
3870        int ret;
3871
3872        ofdev = container_of(adev->dev, struct platform_device, dev);
3873        np = ofdev->dev.of_node;
3874        if (adev->id != PPC440SPE_XOR_ID) {
3875                adev->err_irq = irq_of_parse_and_map(np, 1);
3876                if (!adev->err_irq) {
3877                        dev_warn(adev->dev, "no err irq resource?\n");
3878                        *initcode = PPC_ADMA_INIT_IRQ2;
3879                        adev->err_irq = -ENXIO;
3880                } else
3881                        atomic_inc(&ppc440spe_adma_err_irq_ref);
3882        } else {
3883                adev->err_irq = -ENXIO;
3884        }
3885
3886        adev->irq = irq_of_parse_and_map(np, 0);
3887        if (!adev->irq) {
3888                dev_err(adev->dev, "no irq resource\n");
3889                *initcode = PPC_ADMA_INIT_IRQ1;
3890                ret = -ENXIO;
3891                goto err_irq_map;
3892        }
3893        dev_dbg(adev->dev, "irq %d, err irq %d\n",
3894                adev->irq, adev->err_irq);
3895
3896        ret = request_irq(adev->irq, ppc440spe_adma_eot_handler,
3897                          0, dev_driver_string(adev->dev), chan);
3898        if (ret) {
3899                dev_err(adev->dev, "can't request irq %d\n",
3900                        adev->irq);
3901                *initcode = PPC_ADMA_INIT_IRQ1;
3902                ret = -EIO;
3903                goto err_req1;
3904        }
3905
3906        /* only DMA engines have a separate error IRQ
3907         * so it's Ok if err_irq < 0 in XOR engine case.
3908         */
3909        if (adev->err_irq > 0) {
3910                /* both DMA engines share common error IRQ */
3911                ret = request_irq(adev->err_irq,
3912                                  ppc440spe_adma_err_handler,
3913                                  IRQF_SHARED,
3914                                  dev_driver_string(adev->dev),
3915                                  chan);
3916                if (ret) {
3917                        dev_err(adev->dev, "can't request irq %d\n",
3918                                adev->err_irq);
3919                        *initcode = PPC_ADMA_INIT_IRQ2;
3920                        ret = -EIO;
3921                        goto err_req2;
3922                }
3923        }
3924
3925        if (adev->id == PPC440SPE_XOR_ID) {
3926                /* enable XOR engine interrupts */
3927                iowrite32be(XOR_IE_CBCIE_BIT | XOR_IE_ICBIE_BIT |
3928                            XOR_IE_ICIE_BIT | XOR_IE_RPTIE_BIT,
3929                            &adev->xor_reg->ier);
3930        } else {
3931                u32 mask, enable;
3932
3933                np = of_find_compatible_node(NULL, NULL, "ibm,i2o-440spe");
3934                if (!np) {
3935                        pr_err("%s: can't find I2O device tree node\n",
3936                                __func__);
3937                        ret = -ENODEV;
3938                        goto err_req2;
3939                }
3940                adev->i2o_reg = of_iomap(np, 0);
3941                if (!adev->i2o_reg) {
3942                        pr_err("%s: failed to map I2O registers\n", __func__);
3943                        of_node_put(np);
3944                        ret = -EINVAL;
3945                        goto err_req2;
3946                }
3947                of_node_put(np);
3948                /* Unmask 'CS FIFO Attention' interrupts and
3949                 * enable generating interrupts on errors
3950                 */
3951                enable = (adev->id == PPC440SPE_DMA0_ID) ?
3952                         ~(I2O_IOPIM_P0SNE | I2O_IOPIM_P0EM) :
3953                         ~(I2O_IOPIM_P1SNE | I2O_IOPIM_P1EM);
3954                mask = ioread32(&adev->i2o_reg->iopim) & enable;
3955                iowrite32(mask, &adev->i2o_reg->iopim);
3956        }
3957        return 0;
3958
3959err_req2:
3960        free_irq(adev->irq, chan);
3961err_req1:
3962        irq_dispose_mapping(adev->irq);
3963err_irq_map:
3964        if (adev->err_irq > 0) {
3965                if (atomic_dec_and_test(&ppc440spe_adma_err_irq_ref))
3966                        irq_dispose_mapping(adev->err_irq);
3967        }
3968        return ret;
3969}
3970
3971static void ppc440spe_adma_release_irqs(struct ppc440spe_adma_device *adev,
3972                                        struct ppc440spe_adma_chan *chan)
3973{
3974        u32 mask, disable;
3975
3976        if (adev->id == PPC440SPE_XOR_ID) {
3977                /* disable XOR engine interrupts */
3978                mask = ioread32be(&adev->xor_reg->ier);
3979                mask &= ~(XOR_IE_CBCIE_BIT | XOR_IE_ICBIE_BIT |
3980                          XOR_IE_ICIE_BIT | XOR_IE_RPTIE_BIT);
3981                iowrite32be(mask, &adev->xor_reg->ier);
3982        } else {
3983                /* disable DMAx engine interrupts */
3984                disable = (adev->id == PPC440SPE_DMA0_ID) ?
3985                          (I2O_IOPIM_P0SNE | I2O_IOPIM_P0EM) :
3986                          (I2O_IOPIM_P1SNE | I2O_IOPIM_P1EM);
3987                mask = ioread32(&adev->i2o_reg->iopim) | disable;
3988                iowrite32(mask, &adev->i2o_reg->iopim);
3989        }
3990        free_irq(adev->irq, chan);
3991        irq_dispose_mapping(adev->irq);
3992        if (adev->err_irq > 0) {
3993                free_irq(adev->err_irq, chan);
3994                if (atomic_dec_and_test(&ppc440spe_adma_err_irq_ref)) {
3995                        irq_dispose_mapping(adev->err_irq);
3996                        iounmap(adev->i2o_reg);
3997                }
3998        }
3999}
4000
4001/**
4002 * ppc440spe_adma_probe - probe the asynch device
4003 */
4004static int ppc440spe_adma_probe(struct platform_device *ofdev)
4005{
4006        struct device_node *np = ofdev->dev.of_node;
4007        struct resource res;
4008        struct ppc440spe_adma_device *adev;
4009        struct ppc440spe_adma_chan *chan;
4010        struct ppc_dma_chan_ref *ref, *_ref;
4011        int ret = 0, initcode = PPC_ADMA_INIT_OK;
4012        const u32 *idx;
4013        int len;
4014        void *regs;
4015        u32 id, pool_size;
4016
4017        if (of_device_is_compatible(np, "amcc,xor-accelerator")) {
4018                id = PPC440SPE_XOR_ID;
4019                /* As far as the XOR engine is concerned, it does not
4020                 * use FIFOs but uses linked list. So there is no dependency
4021                 * between pool size to allocate and the engine configuration.
4022                 */
4023                pool_size = PAGE_SIZE << 1;
4024        } else {
4025                /* it is DMA0 or DMA1 */
4026                idx = of_get_property(np, "cell-index", &len);
4027                if (!idx || (len != sizeof(u32))) {
4028                        dev_err(&ofdev->dev, "Device node %pOF has missing "
4029                                "or invalid cell-index property\n",
4030                                np);
4031                        return -EINVAL;
4032                }
4033                id = *idx;
4034                /* DMA0,1 engines use FIFO to maintain CDBs, so we
4035                 * should allocate the pool accordingly to size of this
4036                 * FIFO. Thus, the pool size depends on the FIFO depth:
4037                 * how much CDBs pointers the FIFO may contain then so
4038                 * much CDBs we should provide in the pool.
4039                 * That is
4040                 *   CDB size = 32B;
4041                 *   CDBs number = (DMA0_FIFO_SIZE >> 3);
4042                 *   Pool size = CDBs number * CDB size =
4043                 *      = (DMA0_FIFO_SIZE >> 3) << 5 = DMA0_FIFO_SIZE << 2.
4044                 */
4045                pool_size = (id == PPC440SPE_DMA0_ID) ?
4046                            DMA0_FIFO_SIZE : DMA1_FIFO_SIZE;
4047                pool_size <<= 2;
4048        }
4049
4050        if (of_address_to_resource(np, 0, &res)) {
4051                dev_err(&ofdev->dev, "failed to get memory resource\n");
4052                initcode = PPC_ADMA_INIT_MEMRES;
4053                ret = -ENODEV;
4054                goto out;
4055        }
4056
4057        if (!request_mem_region(res.start, resource_size(&res),
4058                                dev_driver_string(&ofdev->dev))) {
4059                dev_err(&ofdev->dev, "failed to request memory region %pR\n",
4060                        &res);
4061                initcode = PPC_ADMA_INIT_MEMREG;
4062                ret = -EBUSY;
4063                goto out;
4064        }
4065
4066        /* create a device */
4067        adev = kzalloc(sizeof(*adev), GFP_KERNEL);
4068        if (!adev) {
4069                initcode = PPC_ADMA_INIT_ALLOC;
4070                ret = -ENOMEM;
4071                goto err_adev_alloc;
4072        }
4073
4074        adev->id = id;
4075        adev->pool_size = pool_size;
4076        /* allocate coherent memory for hardware descriptors */
4077        adev->dma_desc_pool_virt = dma_alloc_coherent(&ofdev->dev,
4078                                        adev->pool_size, &adev->dma_desc_pool,
4079                                        GFP_KERNEL);
4080        if (adev->dma_desc_pool_virt == NULL) {
4081                dev_err(&ofdev->dev, "failed to allocate %d bytes of coherent "
4082                        "memory for hardware descriptors\n",
4083                        adev->pool_size);
4084                initcode = PPC_ADMA_INIT_COHERENT;
4085                ret = -ENOMEM;
4086                goto err_dma_alloc;
4087        }
4088        dev_dbg(&ofdev->dev, "allocated descriptor pool virt 0x%p phys 0x%llx\n",
4089                adev->dma_desc_pool_virt, (u64)adev->dma_desc_pool);
4090
4091        regs = ioremap(res.start, resource_size(&res));
4092        if (!regs) {
4093                dev_err(&ofdev->dev, "failed to ioremap regs!\n");
4094                ret = -ENOMEM;
4095                goto err_regs_alloc;
4096        }
4097
4098        if (adev->id == PPC440SPE_XOR_ID) {
4099                adev->xor_reg = regs;
4100                /* Reset XOR */
4101                iowrite32be(XOR_CRSR_XASR_BIT, &adev->xor_reg->crsr);
4102                iowrite32be(XOR_CRSR_64BA_BIT, &adev->xor_reg->crrr);
4103        } else {
4104                size_t fifo_size = (adev->id == PPC440SPE_DMA0_ID) ?
4105                                   DMA0_FIFO_SIZE : DMA1_FIFO_SIZE;
4106                adev->dma_reg = regs;
4107                /* DMAx_FIFO_SIZE is defined in bytes,
4108                 * <fsiz> - is defined in number of CDB pointers (8byte).
4109                 * DMA FIFO Length = CSlength + CPlength, where
4110                 * CSlength = CPlength = (fsiz + 1) * 8.
4111                 */
4112                iowrite32(DMA_FIFO_ENABLE | ((fifo_size >> 3) - 2),
4113                          &adev->dma_reg->fsiz);
4114                /* Configure DMA engine */
4115                iowrite32(DMA_CFG_DXEPR_HP | DMA_CFG_DFMPP_HP | DMA_CFG_FALGN,
4116                          &adev->dma_reg->cfg);
4117                /* Clear Status */
4118                iowrite32(~0, &adev->dma_reg->dsts);
4119        }
4120
4121        adev->dev = &ofdev->dev;
4122        adev->common.dev = &ofdev->dev;
4123        INIT_LIST_HEAD(&adev->common.channels);
4124        platform_set_drvdata(ofdev, adev);
4125
4126        /* create a channel */
4127        chan = kzalloc(sizeof(*chan), GFP_KERNEL);
4128        if (!chan) {
4129                initcode = PPC_ADMA_INIT_CHANNEL;
4130                ret = -ENOMEM;
4131                goto err_chan_alloc;
4132        }
4133
4134        spin_lock_init(&chan->lock);
4135        INIT_LIST_HEAD(&chan->chain);
4136        INIT_LIST_HEAD(&chan->all_slots);
4137        chan->device = adev;
4138        chan->common.device = &adev->common;
4139        dma_cookie_init(&chan->common);
4140        list_add_tail(&chan->common.device_node, &adev->common.channels);
4141        tasklet_setup(&chan->irq_tasklet, ppc440spe_adma_tasklet);
4142
4143        /* allocate and map helper pages for async validation or
4144         * async_mult/async_sum_product operations on DMA0/1.
4145         */
4146        if (adev->id != PPC440SPE_XOR_ID) {
4147                chan->pdest_page = alloc_page(GFP_KERNEL);
4148                chan->qdest_page = alloc_page(GFP_KERNEL);
4149                if (!chan->pdest_page ||
4150                    !chan->qdest_page) {
4151                        if (chan->pdest_page)
4152                                __free_page(chan->pdest_page);
4153                        if (chan->qdest_page)
4154                                __free_page(chan->qdest_page);
4155                        ret = -ENOMEM;
4156                        goto err_page_alloc;
4157                }
4158                chan->pdest = dma_map_page(&ofdev->dev, chan->pdest_page, 0,
4159                                           PAGE_SIZE, DMA_BIDIRECTIONAL);
4160                chan->qdest = dma_map_page(&ofdev->dev, chan->qdest_page, 0,
4161                                           PAGE_SIZE, DMA_BIDIRECTIONAL);
4162        }
4163
4164        ref = kmalloc(sizeof(*ref), GFP_KERNEL);
4165        if (ref) {
4166                ref->chan = &chan->common;
4167                INIT_LIST_HEAD(&ref->node);
4168                list_add_tail(&ref->node, &ppc440spe_adma_chan_list);
4169        } else {
4170                dev_err(&ofdev->dev, "failed to allocate channel reference!\n");
4171                ret = -ENOMEM;
4172                goto err_ref_alloc;
4173        }
4174
4175        ret = ppc440spe_adma_setup_irqs(adev, chan, &initcode);
4176        if (ret)
4177                goto err_irq;
4178
4179        ppc440spe_adma_init_capabilities(adev);
4180
4181        ret = dma_async_device_register(&adev->common);
4182        if (ret) {
4183                initcode = PPC_ADMA_INIT_REGISTER;
4184                dev_err(&ofdev->dev, "failed to register dma device\n");
4185                goto err_dev_reg;
4186        }
4187
4188        goto out;
4189
4190err_dev_reg:
4191        ppc440spe_adma_release_irqs(adev, chan);
4192err_irq:
4193        list_for_each_entry_safe(ref, _ref, &ppc440spe_adma_chan_list, node) {
4194                if (chan == to_ppc440spe_adma_chan(ref->chan)) {
4195                        list_del(&ref->node);
4196                        kfree(ref);
4197                }
4198        }
4199err_ref_alloc:
4200        if (adev->id != PPC440SPE_XOR_ID) {
4201                dma_unmap_page(&ofdev->dev, chan->pdest,
4202                               PAGE_SIZE, DMA_BIDIRECTIONAL);
4203                dma_unmap_page(&ofdev->dev, chan->qdest,
4204                               PAGE_SIZE, DMA_BIDIRECTIONAL);
4205                __free_page(chan->pdest_page);
4206                __free_page(chan->qdest_page);
4207        }
4208err_page_alloc:
4209        kfree(chan);
4210err_chan_alloc:
4211        if (adev->id == PPC440SPE_XOR_ID)
4212                iounmap(adev->xor_reg);
4213        else
4214                iounmap(adev->dma_reg);
4215err_regs_alloc:
4216        dma_free_coherent(adev->dev, adev->pool_size,
4217                          adev->dma_desc_pool_virt,
4218                          adev->dma_desc_pool);
4219err_dma_alloc:
4220        kfree(adev);
4221err_adev_alloc:
4222        release_mem_region(res.start, resource_size(&res));
4223out:
4224        if (id < PPC440SPE_ADMA_ENGINES_NUM)
4225                ppc440spe_adma_devices[id] = initcode;
4226
4227        return ret;
4228}
4229
4230/**
4231 * ppc440spe_adma_remove - remove the asynch device
4232 */
4233static int ppc440spe_adma_remove(struct platform_device *ofdev)
4234{
4235        struct ppc440spe_adma_device *adev = platform_get_drvdata(ofdev);
4236        struct device_node *np = ofdev->dev.of_node;
4237        struct resource res;
4238        struct dma_chan *chan, *_chan;
4239        struct ppc_dma_chan_ref *ref, *_ref;
4240        struct ppc440spe_adma_chan *ppc440spe_chan;
4241
4242        if (adev->id < PPC440SPE_ADMA_ENGINES_NUM)
4243                ppc440spe_adma_devices[adev->id] = -1;
4244
4245        dma_async_device_unregister(&adev->common);
4246
4247        list_for_each_entry_safe(chan, _chan, &adev->common.channels,
4248                                 device_node) {
4249                ppc440spe_chan = to_ppc440spe_adma_chan(chan);
4250                ppc440spe_adma_release_irqs(adev, ppc440spe_chan);
4251                tasklet_kill(&ppc440spe_chan->irq_tasklet);
4252                if (adev->id != PPC440SPE_XOR_ID) {
4253                        dma_unmap_page(&ofdev->dev, ppc440spe_chan->pdest,
4254                                        PAGE_SIZE, DMA_BIDIRECTIONAL);
4255                        dma_unmap_page(&ofdev->dev, ppc440spe_chan->qdest,
4256                                        PAGE_SIZE, DMA_BIDIRECTIONAL);
4257                        __free_page(ppc440spe_chan->pdest_page);
4258                        __free_page(ppc440spe_chan->qdest_page);
4259                }
4260                list_for_each_entry_safe(ref, _ref, &ppc440spe_adma_chan_list,
4261                                         node) {
4262                        if (ppc440spe_chan ==
4263                            to_ppc440spe_adma_chan(ref->chan)) {
4264                                list_del(&ref->node);
4265                                kfree(ref);
4266                        }
4267                }
4268                list_del(&chan->device_node);
4269                kfree(ppc440spe_chan);
4270        }
4271
4272        dma_free_coherent(adev->dev, adev->pool_size,
4273                          adev->dma_desc_pool_virt, adev->dma_desc_pool);
4274        if (adev->id == PPC440SPE_XOR_ID)
4275                iounmap(adev->xor_reg);
4276        else
4277                iounmap(adev->dma_reg);
4278        of_address_to_resource(np, 0, &res);
4279        release_mem_region(res.start, resource_size(&res));
4280        kfree(adev);
4281        return 0;
4282}
4283
4284/*
4285 * /sys driver interface to enable h/w RAID-6 capabilities
4286 * Files created in e.g. /sys/devices/plb.0/400100100.dma0/driver/
4287 * directory are "devices", "enable" and "poly".
4288 * "devices" shows available engines.
4289 * "enable" is used to enable RAID-6 capabilities or to check
4290 * whether these has been activated.
4291 * "poly" allows setting/checking used polynomial (for PPC440SPe only).
4292 */
4293
4294static ssize_t devices_show(struct device_driver *dev, char *buf)
4295{
4296        ssize_t size = 0;
4297        int i;
4298
4299        for (i = 0; i < PPC440SPE_ADMA_ENGINES_NUM; i++) {
4300                if (ppc440spe_adma_devices[i] == -1)
4301                        continue;
4302                size += scnprintf(buf + size, PAGE_SIZE - size,
4303                                 "PPC440SP(E)-ADMA.%d: %s\n", i,
4304                                 ppc_adma_errors[ppc440spe_adma_devices[i]]);
4305        }
4306        return size;
4307}
4308static DRIVER_ATTR_RO(devices);
4309
4310static ssize_t enable_show(struct device_driver *dev, char *buf)
4311{
4312        return snprintf(buf, PAGE_SIZE,
4313                        "PPC440SP(e) RAID-6 capabilities are %sABLED.\n",
4314                        ppc440spe_r6_enabled ? "EN" : "DIS");
4315}
4316
4317static ssize_t enable_store(struct device_driver *dev, const char *buf,
4318                            size_t count)
4319{
4320        unsigned long val;
4321        int err;
4322
4323        if (!count || count > 11)
4324                return -EINVAL;
4325
4326        if (!ppc440spe_r6_tchan)
4327                return -EFAULT;
4328
4329        /* Write a key */
4330        err = kstrtoul(buf, 16, &val);
4331        if (err)
4332                return err;
4333
4334        dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_XORBA, val);
4335        isync();
4336
4337        /* Verify whether it really works now */
4338        if (ppc440spe_test_raid6(ppc440spe_r6_tchan) == 0) {
4339                pr_info("PPC440SP(e) RAID-6 has been activated "
4340                        "successfully\n");
4341                ppc440spe_r6_enabled = 1;
4342        } else {
4343                pr_info("PPC440SP(e) RAID-6 hasn't been activated!"
4344                        " Error key ?\n");
4345                ppc440spe_r6_enabled = 0;
4346        }
4347        return count;
4348}
4349static DRIVER_ATTR_RW(enable);
4350
4351static ssize_t poly_show(struct device_driver *dev, char *buf)
4352{
4353        ssize_t size = 0;
4354        u32 reg;
4355
4356#ifdef CONFIG_440SP
4357        /* 440SP has fixed polynomial */
4358        reg = 0x4d;
4359#else
4360        reg = dcr_read(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL);
4361        reg >>= MQ0_CFBHL_POLY;
4362        reg &= 0xFF;
4363#endif
4364
4365        size = snprintf(buf, PAGE_SIZE, "PPC440SP(e) RAID-6 driver "
4366                        "uses 0x1%02x polynomial.\n", reg);
4367        return size;
4368}
4369
4370static ssize_t poly_store(struct device_driver *dev, const char *buf,
4371                          size_t count)
4372{
4373        unsigned long reg, val;
4374        int err;
4375#ifdef CONFIG_440SP
4376        /* 440SP uses default 0x14D polynomial only */
4377        return -EINVAL;
4378#endif
4379
4380        if (!count || count > 6)
4381                return -EINVAL;
4382
4383        /* e.g., 0x14D or 0x11D */
4384        err = kstrtoul(buf, 16, &val);
4385        if (err)
4386                return err;
4387
4388        if (val & ~0x1FF)
4389                return -EINVAL;
4390
4391        val &= 0xFF;
4392        reg = dcr_read(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL);
4393        reg &= ~(0xFF << MQ0_CFBHL_POLY);
4394        reg |= val << MQ0_CFBHL_POLY;
4395        dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL, reg);
4396
4397        return count;
4398}
4399static DRIVER_ATTR_RW(poly);
4400
4401/*
4402 * Common initialisation for RAID engines; allocate memory for
4403 * DMAx FIFOs, perform configuration common for all DMA engines.
4404 * Further DMA engine specific configuration is done at probe time.
4405 */
4406static int ppc440spe_configure_raid_devices(void)
4407{
4408        struct device_node *np;
4409        struct resource i2o_res;
4410        struct i2o_regs __iomem *i2o_reg;
4411        dcr_host_t i2o_dcr_host;
4412        unsigned int dcr_base, dcr_len;
4413        int i, ret;
4414
4415        np = of_find_compatible_node(NULL, NULL, "ibm,i2o-440spe");
4416        if (!np) {
4417                pr_err("%s: can't find I2O device tree node\n",
4418                        __func__);
4419                return -ENODEV;
4420        }
4421
4422        if (of_address_to_resource(np, 0, &i2o_res)) {
4423                of_node_put(np);
4424                return -EINVAL;
4425        }
4426
4427        i2o_reg = of_iomap(np, 0);
4428        if (!i2o_reg) {
4429                pr_err("%s: failed to map I2O registers\n", __func__);
4430                of_node_put(np);
4431                return -EINVAL;
4432        }
4433
4434        /* Get I2O DCRs base */
4435        dcr_base = dcr_resource_start(np, 0);
4436        dcr_len = dcr_resource_len(np, 0);
4437        if (!dcr_base && !dcr_len) {
4438                pr_err("%pOF: can't get DCR registers base/len!\n", np);
4439                of_node_put(np);
4440                iounmap(i2o_reg);
4441                return -ENODEV;
4442        }
4443
4444        i2o_dcr_host = dcr_map(np, dcr_base, dcr_len);
4445        if (!DCR_MAP_OK(i2o_dcr_host)) {
4446                pr_err("%pOF: failed to map DCRs!\n", np);
4447                of_node_put(np);
4448                iounmap(i2o_reg);
4449                return -ENODEV;
4450        }
4451        of_node_put(np);
4452
4453        /* Provide memory regions for DMA's FIFOs: I2O, DMA0 and DMA1 share
4454         * the base address of FIFO memory space.
4455         * Actually we need twice more physical memory than programmed in the
4456         * <fsiz> register (because there are two FIFOs for each DMA: CP and CS)
4457         */
4458        ppc440spe_dma_fifo_buf = kmalloc((DMA0_FIFO_SIZE + DMA1_FIFO_SIZE) << 1,
4459                                         GFP_KERNEL);
4460        if (!ppc440spe_dma_fifo_buf) {
4461                pr_err("%s: DMA FIFO buffer allocation failed.\n", __func__);
4462                iounmap(i2o_reg);
4463                dcr_unmap(i2o_dcr_host, dcr_len);
4464                return -ENOMEM;
4465        }
4466
4467        /*
4468         * Configure h/w
4469         */
4470        /* Reset I2O/DMA */
4471        mtdcri(SDR0, DCRN_SDR0_SRST, DCRN_SDR0_SRST_I2ODMA);
4472        mtdcri(SDR0, DCRN_SDR0_SRST, 0);
4473
4474        /* Setup the base address of mmaped registers */
4475        dcr_write(i2o_dcr_host, DCRN_I2O0_IBAH, (u32)(i2o_res.start >> 32));
4476        dcr_write(i2o_dcr_host, DCRN_I2O0_IBAL, (u32)(i2o_res.start) |
4477                                                I2O_REG_ENABLE);
4478        dcr_unmap(i2o_dcr_host, dcr_len);
4479
4480        /* Setup FIFO memory space base address */
4481        iowrite32(0, &i2o_reg->ifbah);
4482        iowrite32(((u32)__pa(ppc440spe_dma_fifo_buf)), &i2o_reg->ifbal);
4483
4484        /* set zero FIFO size for I2O, so the whole
4485         * ppc440spe_dma_fifo_buf is used by DMAs.
4486         * DMAx_FIFOs will be configured while probe.
4487         */
4488        iowrite32(0, &i2o_reg->ifsiz);
4489        iounmap(i2o_reg);
4490
4491        /* To prepare WXOR/RXOR functionality we need access to
4492         * Memory Queue Module DCRs (finally it will be enabled
4493         * via /sys interface of the ppc440spe ADMA driver).
4494         */
4495        np = of_find_compatible_node(NULL, NULL, "ibm,mq-440spe");
4496        if (!np) {
4497                pr_err("%s: can't find MQ device tree node\n",
4498                        __func__);
4499                ret = -ENODEV;
4500                goto out_free;
4501        }
4502
4503        /* Get MQ DCRs base */
4504        dcr_base = dcr_resource_start(np, 0);
4505        dcr_len = dcr_resource_len(np, 0);
4506        if (!dcr_base && !dcr_len) {
4507                pr_err("%pOF: can't get DCR registers base/len!\n", np);
4508                ret = -ENODEV;
4509                goto out_mq;
4510        }
4511
4512        ppc440spe_mq_dcr_host = dcr_map(np, dcr_base, dcr_len);
4513        if (!DCR_MAP_OK(ppc440spe_mq_dcr_host)) {
4514                pr_err("%pOF: failed to map DCRs!\n", np);
4515                ret = -ENODEV;
4516                goto out_mq;
4517        }
4518        of_node_put(np);
4519        ppc440spe_mq_dcr_len = dcr_len;
4520
4521        /* Set HB alias */
4522        dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_BAUH, DMA_CUED_XOR_HB);
4523
4524        /* Set:
4525         * - LL transaction passing limit to 1;
4526         * - Memory controller cycle limit to 1;
4527         * - Galois Polynomial to 0x14d (default)
4528         */
4529        dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL,
4530                  (1 << MQ0_CFBHL_TPLM) | (1 << MQ0_CFBHL_HBCL) |
4531                  (PPC440SPE_DEFAULT_POLY << MQ0_CFBHL_POLY));
4532
4533        atomic_set(&ppc440spe_adma_err_irq_ref, 0);
4534        for (i = 0; i < PPC440SPE_ADMA_ENGINES_NUM; i++)
4535                ppc440spe_adma_devices[i] = -1;
4536
4537        return 0;
4538
4539out_mq:
4540        of_node_put(np);
4541out_free:
4542        kfree(ppc440spe_dma_fifo_buf);
4543        return ret;
4544}
4545
4546static const struct of_device_id ppc440spe_adma_of_match[] = {
4547        { .compatible   = "ibm,dma-440spe", },
4548        { .compatible   = "amcc,xor-accelerator", },
4549        {},
4550};
4551MODULE_DEVICE_TABLE(of, ppc440spe_adma_of_match);
4552
4553static struct platform_driver ppc440spe_adma_driver = {
4554        .probe = ppc440spe_adma_probe,
4555        .remove = ppc440spe_adma_remove,
4556        .driver = {
4557                .name = "PPC440SP(E)-ADMA",
4558                .of_match_table = ppc440spe_adma_of_match,
4559        },
4560};
4561
4562static __init int ppc440spe_adma_init(void)
4563{
4564        int ret;
4565
4566        ret = ppc440spe_configure_raid_devices();
4567        if (ret)
4568                return ret;
4569
4570        ret = platform_driver_register(&ppc440spe_adma_driver);
4571        if (ret) {
4572                pr_err("%s: failed to register platform driver\n",
4573                        __func__);
4574                goto out_reg;
4575        }
4576
4577        /* Initialization status */
4578        ret = driver_create_file(&ppc440spe_adma_driver.driver,
4579                                 &driver_attr_devices);
4580        if (ret)
4581                goto out_dev;
4582
4583        /* RAID-6 h/w enable entry */
4584        ret = driver_create_file(&ppc440spe_adma_driver.driver,
4585                                 &driver_attr_enable);
4586        if (ret)
4587                goto out_en;
4588
4589        /* GF polynomial to use */
4590        ret = driver_create_file(&ppc440spe_adma_driver.driver,
4591                                 &driver_attr_poly);
4592        if (!ret)
4593                return ret;
4594
4595        driver_remove_file(&ppc440spe_adma_driver.driver,
4596                           &driver_attr_enable);
4597out_en:
4598        driver_remove_file(&ppc440spe_adma_driver.driver,
4599                           &driver_attr_devices);
4600out_dev:
4601        /* User will not be able to enable h/w RAID-6 */
4602        pr_err("%s: failed to create RAID-6 driver interface\n",
4603                __func__);
4604        platform_driver_unregister(&ppc440spe_adma_driver);
4605out_reg:
4606        dcr_unmap(ppc440spe_mq_dcr_host, ppc440spe_mq_dcr_len);
4607        kfree(ppc440spe_dma_fifo_buf);
4608        return ret;
4609}
4610
4611static void __exit ppc440spe_adma_exit(void)
4612{
4613        driver_remove_file(&ppc440spe_adma_driver.driver,
4614                           &driver_attr_poly);
4615        driver_remove_file(&ppc440spe_adma_driver.driver,
4616                           &driver_attr_enable);
4617        driver_remove_file(&ppc440spe_adma_driver.driver,
4618                           &driver_attr_devices);
4619        platform_driver_unregister(&ppc440spe_adma_driver);
4620        dcr_unmap(ppc440spe_mq_dcr_host, ppc440spe_mq_dcr_len);
4621        kfree(ppc440spe_dma_fifo_buf);
4622}
4623
4624arch_initcall(ppc440spe_adma_init);
4625module_exit(ppc440spe_adma_exit);
4626
4627MODULE_AUTHOR("Yuri Tikhonov <yur@emcraft.com>");
4628MODULE_DESCRIPTION("PPC440SPE ADMA Engine Driver");
4629MODULE_LICENSE("GPL");
4630