linux/drivers/mtd/nand/raw/meson_nand.c
<<
>>
Prefs
   1// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
   2/*
   3 * Amlogic Meson Nand Flash Controller Driver
   4 *
   5 * Copyright (c) 2018 Amlogic, inc.
   6 * Author: Liang Yang <liang.yang@amlogic.com>
   7 */
   8
   9#include <linux/platform_device.h>
  10#include <linux/dma-mapping.h>
  11#include <linux/interrupt.h>
  12#include <linux/clk.h>
  13#include <linux/mtd/rawnand.h>
  14#include <linux/mtd/mtd.h>
  15#include <linux/mfd/syscon.h>
  16#include <linux/regmap.h>
  17#include <linux/slab.h>
  18#include <linux/module.h>
  19#include <linux/iopoll.h>
  20#include <linux/of.h>
  21#include <linux/of_device.h>
  22#include <linux/sched/task_stack.h>
  23
  24#define NFC_REG_CMD             0x00
  25#define NFC_CMD_IDLE            (0xc << 14)
  26#define NFC_CMD_CLE             (0x5 << 14)
  27#define NFC_CMD_ALE             (0x6 << 14)
  28#define NFC_CMD_ADL             ((0 << 16) | (3 << 20))
  29#define NFC_CMD_ADH             ((1 << 16) | (3 << 20))
  30#define NFC_CMD_AIL             ((2 << 16) | (3 << 20))
  31#define NFC_CMD_AIH             ((3 << 16) | (3 << 20))
  32#define NFC_CMD_SEED            ((8 << 16) | (3 << 20))
  33#define NFC_CMD_M2N             ((0 << 17) | (2 << 20))
  34#define NFC_CMD_N2M             ((1 << 17) | (2 << 20))
  35#define NFC_CMD_RB              BIT(20)
  36#define NFC_CMD_SCRAMBLER_ENABLE        BIT(19)
  37#define NFC_CMD_SCRAMBLER_DISABLE       0
  38#define NFC_CMD_SHORTMODE_DISABLE       0
  39#define NFC_CMD_RB_INT          BIT(14)
  40
  41#define NFC_CMD_GET_SIZE(x)     (((x) >> 22) & GENMASK(4, 0))
  42
  43#define NFC_REG_CFG             0x04
  44#define NFC_REG_DADR            0x08
  45#define NFC_REG_IADR            0x0c
  46#define NFC_REG_BUF             0x10
  47#define NFC_REG_INFO            0x14
  48#define NFC_REG_DC              0x18
  49#define NFC_REG_ADR             0x1c
  50#define NFC_REG_DL              0x20
  51#define NFC_REG_DH              0x24
  52#define NFC_REG_CADR            0x28
  53#define NFC_REG_SADR            0x2c
  54#define NFC_REG_PINS            0x30
  55#define NFC_REG_VER             0x38
  56
  57#define NFC_RB_IRQ_EN           BIT(21)
  58
  59#define CMDRWGEN(cmd_dir, ran, bch, short_mode, page_size, pages)       \
  60        (                                                               \
  61                (cmd_dir)                       |                       \
  62                ((ran) << 19)                   |                       \
  63                ((bch) << 14)                   |                       \
  64                ((short_mode) << 13)            |                       \
  65                (((page_size) & 0x7f) << 6)     |                       \
  66                ((pages) & 0x3f)                                        \
  67        )
  68
  69#define GENCMDDADDRL(adl, addr)         ((adl) | ((addr) & 0xffff))
  70#define GENCMDDADDRH(adh, addr)         ((adh) | (((addr) >> 16) & 0xffff))
  71#define GENCMDIADDRL(ail, addr)         ((ail) | ((addr) & 0xffff))
  72#define GENCMDIADDRH(aih, addr)         ((aih) | (((addr) >> 16) & 0xffff))
  73
  74#define DMA_DIR(dir)            ((dir) ? NFC_CMD_N2M : NFC_CMD_M2N)
  75
  76#define ECC_CHECK_RETURN_FF     (-1)
  77
  78#define NAND_CE0                (0xe << 10)
  79#define NAND_CE1                (0xd << 10)
  80
  81#define DMA_BUSY_TIMEOUT        0x100000
  82#define CMD_FIFO_EMPTY_TIMEOUT  1000
  83
  84#define MAX_CE_NUM              2
  85
  86/* eMMC clock register, misc control */
  87#define CLK_SELECT_NAND         BIT(31)
  88
  89#define NFC_CLK_CYCLE           6
  90
  91/* nand flash controller delay 3 ns */
  92#define NFC_DEFAULT_DELAY       3000
  93
  94#define ROW_ADDER(page, index)  (((page) >> (8 * (index))) & 0xff)
  95#define MAX_CYCLE_ADDRS         5
  96#define DIRREAD                 1
  97#define DIRWRITE                0
  98
  99#define ECC_PARITY_BCH8_512B    14
 100#define ECC_COMPLETE            BIT(31)
 101#define ECC_ERR_CNT(x)          (((x) >> 24) & GENMASK(5, 0))
 102#define ECC_ZERO_CNT(x)         (((x) >> 16) & GENMASK(5, 0))
 103#define ECC_UNCORRECTABLE       0x3f
 104
 105#define PER_INFO_BYTE           8
 106
 107struct meson_nfc_nand_chip {
 108        struct list_head node;
 109        struct nand_chip nand;
 110        unsigned long clk_rate;
 111        unsigned long level1_divider;
 112        u32 bus_timing;
 113        u32 twb;
 114        u32 tadl;
 115        u32 tbers_max;
 116
 117        u32 bch_mode;
 118        u8 *data_buf;
 119        __le64 *info_buf;
 120        u32 nsels;
 121        u8 sels[];
 122};
 123
 124struct meson_nand_ecc {
 125        u32 bch;
 126        u32 strength;
 127};
 128
 129struct meson_nfc_data {
 130        const struct nand_ecc_caps *ecc_caps;
 131};
 132
 133struct meson_nfc_param {
 134        u32 chip_select;
 135        u32 rb_select;
 136};
 137
 138struct nand_rw_cmd {
 139        u32 cmd0;
 140        u32 addrs[MAX_CYCLE_ADDRS];
 141        u32 cmd1;
 142};
 143
 144struct nand_timing {
 145        u32 twb;
 146        u32 tadl;
 147        u32 tbers_max;
 148};
 149
 150struct meson_nfc {
 151        struct nand_controller controller;
 152        struct clk *core_clk;
 153        struct clk *device_clk;
 154        struct clk *phase_tx;
 155        struct clk *phase_rx;
 156
 157        unsigned long clk_rate;
 158        u32 bus_timing;
 159
 160        struct device *dev;
 161        void __iomem *reg_base;
 162        struct regmap *reg_clk;
 163        struct completion completion;
 164        struct list_head chips;
 165        const struct meson_nfc_data *data;
 166        struct meson_nfc_param param;
 167        struct nand_timing timing;
 168        union {
 169                int cmd[32];
 170                struct nand_rw_cmd rw;
 171        } cmdfifo;
 172
 173        dma_addr_t daddr;
 174        dma_addr_t iaddr;
 175
 176        unsigned long assigned_cs;
 177};
 178
 179enum {
 180        NFC_ECC_BCH8_1K         = 2,
 181        NFC_ECC_BCH24_1K,
 182        NFC_ECC_BCH30_1K,
 183        NFC_ECC_BCH40_1K,
 184        NFC_ECC_BCH50_1K,
 185        NFC_ECC_BCH60_1K,
 186};
 187
 188#define MESON_ECC_DATA(b, s)    { .bch = (b),   .strength = (s)}
 189
 190static struct meson_nand_ecc meson_ecc[] = {
 191        MESON_ECC_DATA(NFC_ECC_BCH8_1K, 8),
 192        MESON_ECC_DATA(NFC_ECC_BCH24_1K, 24),
 193        MESON_ECC_DATA(NFC_ECC_BCH30_1K, 30),
 194        MESON_ECC_DATA(NFC_ECC_BCH40_1K, 40),
 195        MESON_ECC_DATA(NFC_ECC_BCH50_1K, 50),
 196        MESON_ECC_DATA(NFC_ECC_BCH60_1K, 60),
 197};
 198
 199static int meson_nand_calc_ecc_bytes(int step_size, int strength)
 200{
 201        int ecc_bytes;
 202
 203        if (step_size == 512 && strength == 8)
 204                return ECC_PARITY_BCH8_512B;
 205
 206        ecc_bytes = DIV_ROUND_UP(strength * fls(step_size * 8), 8);
 207        ecc_bytes = ALIGN(ecc_bytes, 2);
 208
 209        return ecc_bytes;
 210}
 211
 212NAND_ECC_CAPS_SINGLE(meson_gxl_ecc_caps,
 213                     meson_nand_calc_ecc_bytes, 1024, 8, 24, 30, 40, 50, 60);
 214NAND_ECC_CAPS_SINGLE(meson_axg_ecc_caps,
 215                     meson_nand_calc_ecc_bytes, 1024, 8);
 216
 217static struct meson_nfc_nand_chip *to_meson_nand(struct nand_chip *nand)
 218{
 219        return container_of(nand, struct meson_nfc_nand_chip, nand);
 220}
 221
 222static void meson_nfc_select_chip(struct nand_chip *nand, int chip)
 223{
 224        struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
 225        struct meson_nfc *nfc = nand_get_controller_data(nand);
 226        int ret, value;
 227
 228        if (chip < 0 || WARN_ON_ONCE(chip >= meson_chip->nsels))
 229                return;
 230
 231        nfc->param.chip_select = meson_chip->sels[chip] ? NAND_CE1 : NAND_CE0;
 232        nfc->param.rb_select = nfc->param.chip_select;
 233        nfc->timing.twb = meson_chip->twb;
 234        nfc->timing.tadl = meson_chip->tadl;
 235        nfc->timing.tbers_max = meson_chip->tbers_max;
 236
 237        if (nfc->clk_rate != meson_chip->clk_rate) {
 238                ret = clk_set_rate(nfc->device_clk, meson_chip->clk_rate);
 239                if (ret) {
 240                        dev_err(nfc->dev, "failed to set clock rate\n");
 241                        return;
 242                }
 243                nfc->clk_rate = meson_chip->clk_rate;
 244        }
 245        if (nfc->bus_timing != meson_chip->bus_timing) {
 246                value = (NFC_CLK_CYCLE - 1) | (meson_chip->bus_timing << 5);
 247                writel(value, nfc->reg_base + NFC_REG_CFG);
 248                writel((1 << 31), nfc->reg_base + NFC_REG_CMD);
 249                nfc->bus_timing =  meson_chip->bus_timing;
 250        }
 251}
 252
 253static void meson_nfc_cmd_idle(struct meson_nfc *nfc, u32 time)
 254{
 255        writel(nfc->param.chip_select | NFC_CMD_IDLE | (time & 0x3ff),
 256               nfc->reg_base + NFC_REG_CMD);
 257}
 258
 259static void meson_nfc_cmd_seed(struct meson_nfc *nfc, u32 seed)
 260{
 261        writel(NFC_CMD_SEED | (0xc2 + (seed & 0x7fff)),
 262               nfc->reg_base + NFC_REG_CMD);
 263}
 264
 265static void meson_nfc_cmd_access(struct nand_chip *nand, int raw, bool dir,
 266                                 int scrambler)
 267{
 268        struct mtd_info *mtd = nand_to_mtd(nand);
 269        struct meson_nfc *nfc = nand_get_controller_data(mtd_to_nand(mtd));
 270        struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
 271        u32 bch = meson_chip->bch_mode, cmd;
 272        int len = mtd->writesize, pagesize, pages;
 273
 274        pagesize = nand->ecc.size;
 275
 276        if (raw) {
 277                len = mtd->writesize + mtd->oobsize;
 278                cmd = (len & GENMASK(5, 0)) | scrambler | DMA_DIR(dir);
 279                writel(cmd, nfc->reg_base + NFC_REG_CMD);
 280                return;
 281        }
 282
 283        pages = len / nand->ecc.size;
 284
 285        cmd = CMDRWGEN(DMA_DIR(dir), scrambler, bch,
 286                       NFC_CMD_SHORTMODE_DISABLE, pagesize, pages);
 287
 288        writel(cmd, nfc->reg_base + NFC_REG_CMD);
 289}
 290
 291static void meson_nfc_drain_cmd(struct meson_nfc *nfc)
 292{
 293        /*
 294         * Insert two commands to make sure all valid commands are finished.
 295         *
 296         * The Nand flash controller is designed as two stages pipleline -
 297         *  a) fetch and b) excute.
 298         * There might be cases when the driver see command queue is empty,
 299         * but the Nand flash controller still has two commands buffered,
 300         * one is fetched into NFC request queue (ready to run), and another
 301         * is actively executing. So pushing 2 "IDLE" commands guarantees that
 302         * the pipeline is emptied.
 303         */
 304        meson_nfc_cmd_idle(nfc, 0);
 305        meson_nfc_cmd_idle(nfc, 0);
 306}
 307
 308static int meson_nfc_wait_cmd_finish(struct meson_nfc *nfc,
 309                                     unsigned int timeout_ms)
 310{
 311        u32 cmd_size = 0;
 312        int ret;
 313
 314        /* wait cmd fifo is empty */
 315        ret = readl_relaxed_poll_timeout(nfc->reg_base + NFC_REG_CMD, cmd_size,
 316                                         !NFC_CMD_GET_SIZE(cmd_size),
 317                                         10, timeout_ms * 1000);
 318        if (ret)
 319                dev_err(nfc->dev, "wait for empty CMD FIFO time out\n");
 320
 321        return ret;
 322}
 323
 324static int meson_nfc_wait_dma_finish(struct meson_nfc *nfc)
 325{
 326        meson_nfc_drain_cmd(nfc);
 327
 328        return meson_nfc_wait_cmd_finish(nfc, DMA_BUSY_TIMEOUT);
 329}
 330
 331static u8 *meson_nfc_oob_ptr(struct nand_chip *nand, int i)
 332{
 333        struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
 334        int len;
 335
 336        len = nand->ecc.size * (i + 1) + (nand->ecc.bytes + 2) * i;
 337
 338        return meson_chip->data_buf + len;
 339}
 340
 341static u8 *meson_nfc_data_ptr(struct nand_chip *nand, int i)
 342{
 343        struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
 344        int len, temp;
 345
 346        temp = nand->ecc.size + nand->ecc.bytes;
 347        len = (temp + 2) * i;
 348
 349        return meson_chip->data_buf + len;
 350}
 351
 352static void meson_nfc_get_data_oob(struct nand_chip *nand,
 353                                   u8 *buf, u8 *oobbuf)
 354{
 355        int i, oob_len = 0;
 356        u8 *dsrc, *osrc;
 357
 358        oob_len = nand->ecc.bytes + 2;
 359        for (i = 0; i < nand->ecc.steps; i++) {
 360                if (buf) {
 361                        dsrc = meson_nfc_data_ptr(nand, i);
 362                        memcpy(buf, dsrc, nand->ecc.size);
 363                        buf += nand->ecc.size;
 364                }
 365                osrc = meson_nfc_oob_ptr(nand, i);
 366                memcpy(oobbuf, osrc, oob_len);
 367                oobbuf += oob_len;
 368        }
 369}
 370
 371static void meson_nfc_set_data_oob(struct nand_chip *nand,
 372                                   const u8 *buf, u8 *oobbuf)
 373{
 374        int i, oob_len = 0;
 375        u8 *dsrc, *osrc;
 376
 377        oob_len = nand->ecc.bytes + 2;
 378        for (i = 0; i < nand->ecc.steps; i++) {
 379                if (buf) {
 380                        dsrc = meson_nfc_data_ptr(nand, i);
 381                        memcpy(dsrc, buf, nand->ecc.size);
 382                        buf += nand->ecc.size;
 383                }
 384                osrc = meson_nfc_oob_ptr(nand, i);
 385                memcpy(osrc, oobbuf, oob_len);
 386                oobbuf += oob_len;
 387        }
 388}
 389
 390static int meson_nfc_queue_rb(struct meson_nfc *nfc, int timeout_ms)
 391{
 392        u32 cmd, cfg;
 393        int ret = 0;
 394
 395        meson_nfc_cmd_idle(nfc, nfc->timing.twb);
 396        meson_nfc_drain_cmd(nfc);
 397        meson_nfc_wait_cmd_finish(nfc, CMD_FIFO_EMPTY_TIMEOUT);
 398
 399        cfg = readl(nfc->reg_base + NFC_REG_CFG);
 400        cfg |= NFC_RB_IRQ_EN;
 401        writel(cfg, nfc->reg_base + NFC_REG_CFG);
 402
 403        reinit_completion(&nfc->completion);
 404
 405        /* use the max erase time as the maximum clock for waiting R/B */
 406        cmd = NFC_CMD_RB | NFC_CMD_RB_INT
 407                | nfc->param.chip_select | nfc->timing.tbers_max;
 408        writel(cmd, nfc->reg_base + NFC_REG_CMD);
 409
 410        ret = wait_for_completion_timeout(&nfc->completion,
 411                                          msecs_to_jiffies(timeout_ms));
 412        if (ret == 0)
 413                ret = -1;
 414
 415        return ret;
 416}
 417
 418static void meson_nfc_set_user_byte(struct nand_chip *nand, u8 *oob_buf)
 419{
 420        struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
 421        __le64 *info;
 422        int i, count;
 423
 424        for (i = 0, count = 0; i < nand->ecc.steps; i++, count += 2) {
 425                info = &meson_chip->info_buf[i];
 426                *info |= oob_buf[count];
 427                *info |= oob_buf[count + 1] << 8;
 428        }
 429}
 430
 431static void meson_nfc_get_user_byte(struct nand_chip *nand, u8 *oob_buf)
 432{
 433        struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
 434        __le64 *info;
 435        int i, count;
 436
 437        for (i = 0, count = 0; i < nand->ecc.steps; i++, count += 2) {
 438                info = &meson_chip->info_buf[i];
 439                oob_buf[count] = *info;
 440                oob_buf[count + 1] = *info >> 8;
 441        }
 442}
 443
 444static int meson_nfc_ecc_correct(struct nand_chip *nand, u32 *bitflips,
 445                                 u64 *correct_bitmap)
 446{
 447        struct mtd_info *mtd = nand_to_mtd(nand);
 448        struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
 449        __le64 *info;
 450        int ret = 0, i;
 451
 452        for (i = 0; i < nand->ecc.steps; i++) {
 453                info = &meson_chip->info_buf[i];
 454                if (ECC_ERR_CNT(*info) != ECC_UNCORRECTABLE) {
 455                        mtd->ecc_stats.corrected += ECC_ERR_CNT(*info);
 456                        *bitflips = max_t(u32, *bitflips, ECC_ERR_CNT(*info));
 457                        *correct_bitmap |= 1 >> i;
 458                        continue;
 459                }
 460                if ((nand->options & NAND_NEED_SCRAMBLING) &&
 461                    ECC_ZERO_CNT(*info) < nand->ecc.strength) {
 462                        mtd->ecc_stats.corrected += ECC_ZERO_CNT(*info);
 463                        *bitflips = max_t(u32, *bitflips,
 464                                          ECC_ZERO_CNT(*info));
 465                        ret = ECC_CHECK_RETURN_FF;
 466                } else {
 467                        ret = -EBADMSG;
 468                }
 469        }
 470        return ret;
 471}
 472
 473static int meson_nfc_dma_buffer_setup(struct nand_chip *nand, void *databuf,
 474                                      int datalen, void *infobuf, int infolen,
 475                                      enum dma_data_direction dir)
 476{
 477        struct meson_nfc *nfc = nand_get_controller_data(nand);
 478        u32 cmd;
 479        int ret = 0;
 480
 481        nfc->daddr = dma_map_single(nfc->dev, databuf, datalen, dir);
 482        ret = dma_mapping_error(nfc->dev, nfc->daddr);
 483        if (ret) {
 484                dev_err(nfc->dev, "DMA mapping error\n");
 485                return ret;
 486        }
 487        cmd = GENCMDDADDRL(NFC_CMD_ADL, nfc->daddr);
 488        writel(cmd, nfc->reg_base + NFC_REG_CMD);
 489
 490        cmd = GENCMDDADDRH(NFC_CMD_ADH, nfc->daddr);
 491        writel(cmd, nfc->reg_base + NFC_REG_CMD);
 492
 493        if (infobuf) {
 494                nfc->iaddr = dma_map_single(nfc->dev, infobuf, infolen, dir);
 495                ret = dma_mapping_error(nfc->dev, nfc->iaddr);
 496                if (ret) {
 497                        dev_err(nfc->dev, "DMA mapping error\n");
 498                        dma_unmap_single(nfc->dev,
 499                                         nfc->daddr, datalen, dir);
 500                        return ret;
 501                }
 502                cmd = GENCMDIADDRL(NFC_CMD_AIL, nfc->iaddr);
 503                writel(cmd, nfc->reg_base + NFC_REG_CMD);
 504
 505                cmd = GENCMDIADDRH(NFC_CMD_AIH, nfc->iaddr);
 506                writel(cmd, nfc->reg_base + NFC_REG_CMD);
 507        }
 508
 509        return ret;
 510}
 511
 512static void meson_nfc_dma_buffer_release(struct nand_chip *nand,
 513                                         int infolen, int datalen,
 514                                         enum dma_data_direction dir)
 515{
 516        struct meson_nfc *nfc = nand_get_controller_data(nand);
 517
 518        dma_unmap_single(nfc->dev, nfc->daddr, datalen, dir);
 519        if (infolen)
 520                dma_unmap_single(nfc->dev, nfc->iaddr, infolen, dir);
 521}
 522
 523static int meson_nfc_read_buf(struct nand_chip *nand, u8 *buf, int len)
 524{
 525        struct meson_nfc *nfc = nand_get_controller_data(nand);
 526        int ret = 0;
 527        u32 cmd;
 528        u8 *info;
 529
 530        info = kzalloc(PER_INFO_BYTE, GFP_KERNEL);
 531        if (!info)
 532                return -ENOMEM;
 533
 534        ret = meson_nfc_dma_buffer_setup(nand, buf, len, info,
 535                                         PER_INFO_BYTE, DMA_FROM_DEVICE);
 536        if (ret)
 537                goto out;
 538
 539        cmd = NFC_CMD_N2M | (len & GENMASK(5, 0));
 540        writel(cmd, nfc->reg_base + NFC_REG_CMD);
 541
 542        meson_nfc_drain_cmd(nfc);
 543        meson_nfc_wait_cmd_finish(nfc, 1000);
 544        meson_nfc_dma_buffer_release(nand, len, PER_INFO_BYTE, DMA_FROM_DEVICE);
 545
 546out:
 547        kfree(info);
 548
 549        return ret;
 550}
 551
 552static int meson_nfc_write_buf(struct nand_chip *nand, u8 *buf, int len)
 553{
 554        struct meson_nfc *nfc = nand_get_controller_data(nand);
 555        int ret = 0;
 556        u32 cmd;
 557
 558        ret = meson_nfc_dma_buffer_setup(nand, buf, len, NULL,
 559                                         0, DMA_TO_DEVICE);
 560        if (ret)
 561                return ret;
 562
 563        cmd = NFC_CMD_M2N | (len & GENMASK(5, 0));
 564        writel(cmd, nfc->reg_base + NFC_REG_CMD);
 565
 566        meson_nfc_drain_cmd(nfc);
 567        meson_nfc_wait_cmd_finish(nfc, 1000);
 568        meson_nfc_dma_buffer_release(nand, len, 0, DMA_TO_DEVICE);
 569
 570        return ret;
 571}
 572
 573static int meson_nfc_rw_cmd_prepare_and_execute(struct nand_chip *nand,
 574                                                int page, bool in)
 575{
 576        struct mtd_info *mtd = nand_to_mtd(nand);
 577        struct meson_nfc *nfc = nand_get_controller_data(nand);
 578        const struct nand_sdr_timings *sdr =
 579                nand_get_sdr_timings(&nand->data_interface);
 580        u32 *addrs = nfc->cmdfifo.rw.addrs;
 581        u32 cs = nfc->param.chip_select;
 582        u32 cmd0, cmd_num, row_start;
 583        int ret = 0, i;
 584
 585        cmd_num = sizeof(struct nand_rw_cmd) / sizeof(int);
 586
 587        cmd0 = in ? NAND_CMD_READ0 : NAND_CMD_SEQIN;
 588        nfc->cmdfifo.rw.cmd0 = cs | NFC_CMD_CLE | cmd0;
 589
 590        addrs[0] = cs | NFC_CMD_ALE | 0;
 591        if (mtd->writesize <= 512) {
 592                cmd_num--;
 593                row_start = 1;
 594        } else {
 595                addrs[1] = cs | NFC_CMD_ALE | 0;
 596                row_start = 2;
 597        }
 598
 599        addrs[row_start] = cs | NFC_CMD_ALE | ROW_ADDER(page, 0);
 600        addrs[row_start + 1] = cs | NFC_CMD_ALE | ROW_ADDER(page, 1);
 601
 602        if (nand->options & NAND_ROW_ADDR_3)
 603                addrs[row_start + 2] =
 604                        cs | NFC_CMD_ALE | ROW_ADDER(page, 2);
 605        else
 606                cmd_num--;
 607
 608        /* subtract cmd1 */
 609        cmd_num--;
 610
 611        for (i = 0; i < cmd_num; i++)
 612                writel_relaxed(nfc->cmdfifo.cmd[i],
 613                               nfc->reg_base + NFC_REG_CMD);
 614
 615        if (in) {
 616                nfc->cmdfifo.rw.cmd1 = cs | NFC_CMD_CLE | NAND_CMD_READSTART;
 617                writel(nfc->cmdfifo.rw.cmd1, nfc->reg_base + NFC_REG_CMD);
 618                meson_nfc_queue_rb(nfc, PSEC_TO_MSEC(sdr->tR_max));
 619        } else {
 620                meson_nfc_cmd_idle(nfc, nfc->timing.tadl);
 621        }
 622
 623        return ret;
 624}
 625
 626static int meson_nfc_write_page_sub(struct nand_chip *nand,
 627                                    int page, int raw)
 628{
 629        struct mtd_info *mtd = nand_to_mtd(nand);
 630        const struct nand_sdr_timings *sdr =
 631                nand_get_sdr_timings(&nand->data_interface);
 632        struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
 633        struct meson_nfc *nfc = nand_get_controller_data(nand);
 634        int data_len, info_len;
 635        u32 cmd;
 636        int ret;
 637
 638        meson_nfc_select_chip(nand, nand->cur_cs);
 639
 640        data_len =  mtd->writesize + mtd->oobsize;
 641        info_len = nand->ecc.steps * PER_INFO_BYTE;
 642
 643        ret = meson_nfc_rw_cmd_prepare_and_execute(nand, page, DIRWRITE);
 644        if (ret)
 645                return ret;
 646
 647        ret = meson_nfc_dma_buffer_setup(nand, meson_chip->data_buf,
 648                                         data_len, meson_chip->info_buf,
 649                                         info_len, DMA_TO_DEVICE);
 650        if (ret)
 651                return ret;
 652
 653        if (nand->options & NAND_NEED_SCRAMBLING) {
 654                meson_nfc_cmd_seed(nfc, page);
 655                meson_nfc_cmd_access(nand, raw, DIRWRITE,
 656                                     NFC_CMD_SCRAMBLER_ENABLE);
 657        } else {
 658                meson_nfc_cmd_access(nand, raw, DIRWRITE,
 659                                     NFC_CMD_SCRAMBLER_DISABLE);
 660        }
 661
 662        cmd = nfc->param.chip_select | NFC_CMD_CLE | NAND_CMD_PAGEPROG;
 663        writel(cmd, nfc->reg_base + NFC_REG_CMD);
 664        meson_nfc_queue_rb(nfc, PSEC_TO_MSEC(sdr->tPROG_max));
 665
 666        meson_nfc_dma_buffer_release(nand, data_len, info_len, DMA_TO_DEVICE);
 667
 668        return ret;
 669}
 670
 671static int meson_nfc_write_page_raw(struct nand_chip *nand, const u8 *buf,
 672                                    int oob_required, int page)
 673{
 674        u8 *oob_buf = nand->oob_poi;
 675
 676        meson_nfc_set_data_oob(nand, buf, oob_buf);
 677
 678        return meson_nfc_write_page_sub(nand, page, 1);
 679}
 680
 681static int meson_nfc_write_page_hwecc(struct nand_chip *nand,
 682                                      const u8 *buf, int oob_required, int page)
 683{
 684        struct mtd_info *mtd = nand_to_mtd(nand);
 685        struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
 686        u8 *oob_buf = nand->oob_poi;
 687
 688        memcpy(meson_chip->data_buf, buf, mtd->writesize);
 689        memset(meson_chip->info_buf, 0, nand->ecc.steps * PER_INFO_BYTE);
 690        meson_nfc_set_user_byte(nand, oob_buf);
 691
 692        return meson_nfc_write_page_sub(nand, page, 0);
 693}
 694
 695static void meson_nfc_check_ecc_pages_valid(struct meson_nfc *nfc,
 696                                            struct nand_chip *nand, int raw)
 697{
 698        struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
 699        __le64 *info;
 700        u32 neccpages;
 701        int ret;
 702
 703        neccpages = raw ? 1 : nand->ecc.steps;
 704        info = &meson_chip->info_buf[neccpages - 1];
 705        do {
 706                usleep_range(10, 15);
 707                /* info is updated by nfc dma engine*/
 708                smp_rmb();
 709                ret = *info & ECC_COMPLETE;
 710        } while (!ret);
 711}
 712
 713static int meson_nfc_read_page_sub(struct nand_chip *nand,
 714                                   int page, int raw)
 715{
 716        struct mtd_info *mtd = nand_to_mtd(nand);
 717        struct meson_nfc *nfc = nand_get_controller_data(nand);
 718        struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
 719        int data_len, info_len;
 720        int ret;
 721
 722        meson_nfc_select_chip(nand, nand->cur_cs);
 723
 724        data_len =  mtd->writesize + mtd->oobsize;
 725        info_len = nand->ecc.steps * PER_INFO_BYTE;
 726
 727        ret = meson_nfc_rw_cmd_prepare_and_execute(nand, page, DIRREAD);
 728        if (ret)
 729                return ret;
 730
 731        ret = meson_nfc_dma_buffer_setup(nand, meson_chip->data_buf,
 732                                         data_len, meson_chip->info_buf,
 733                                         info_len, DMA_FROM_DEVICE);
 734        if (ret)
 735                return ret;
 736
 737        if (nand->options & NAND_NEED_SCRAMBLING) {
 738                meson_nfc_cmd_seed(nfc, page);
 739                meson_nfc_cmd_access(nand, raw, DIRREAD,
 740                                     NFC_CMD_SCRAMBLER_ENABLE);
 741        } else {
 742                meson_nfc_cmd_access(nand, raw, DIRREAD,
 743                                     NFC_CMD_SCRAMBLER_DISABLE);
 744        }
 745
 746        ret = meson_nfc_wait_dma_finish(nfc);
 747        meson_nfc_check_ecc_pages_valid(nfc, nand, raw);
 748
 749        meson_nfc_dma_buffer_release(nand, data_len, info_len, DMA_FROM_DEVICE);
 750
 751        return ret;
 752}
 753
 754static int meson_nfc_read_page_raw(struct nand_chip *nand, u8 *buf,
 755                                   int oob_required, int page)
 756{
 757        u8 *oob_buf = nand->oob_poi;
 758        int ret;
 759
 760        ret = meson_nfc_read_page_sub(nand, page, 1);
 761        if (ret)
 762                return ret;
 763
 764        meson_nfc_get_data_oob(nand, buf, oob_buf);
 765
 766        return 0;
 767}
 768
 769static int meson_nfc_read_page_hwecc(struct nand_chip *nand, u8 *buf,
 770                                     int oob_required, int page)
 771{
 772        struct mtd_info *mtd = nand_to_mtd(nand);
 773        struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
 774        struct nand_ecc_ctrl *ecc = &nand->ecc;
 775        u64 correct_bitmap = 0;
 776        u32 bitflips = 0;
 777        u8 *oob_buf = nand->oob_poi;
 778        int ret, i;
 779
 780        ret = meson_nfc_read_page_sub(nand, page, 0);
 781        if (ret)
 782                return ret;
 783
 784        meson_nfc_get_user_byte(nand, oob_buf);
 785        ret = meson_nfc_ecc_correct(nand, &bitflips, &correct_bitmap);
 786        if (ret == ECC_CHECK_RETURN_FF) {
 787                if (buf)
 788                        memset(buf, 0xff, mtd->writesize);
 789                memset(oob_buf, 0xff, mtd->oobsize);
 790        } else if (ret < 0) {
 791                if ((nand->options & NAND_NEED_SCRAMBLING) || !buf) {
 792                        mtd->ecc_stats.failed++;
 793                        return bitflips;
 794                }
 795                ret  = meson_nfc_read_page_raw(nand, buf, 0, page);
 796                if (ret)
 797                        return ret;
 798
 799                for (i = 0; i < nand->ecc.steps ; i++) {
 800                        u8 *data = buf + i * ecc->size;
 801                        u8 *oob = nand->oob_poi + i * (ecc->bytes + 2);
 802
 803                        if (correct_bitmap & (1 << i))
 804                                continue;
 805                        ret = nand_check_erased_ecc_chunk(data, ecc->size,
 806                                                          oob, ecc->bytes + 2,
 807                                                          NULL, 0,
 808                                                          ecc->strength);
 809                        if (ret < 0) {
 810                                mtd->ecc_stats.failed++;
 811                        } else {
 812                                mtd->ecc_stats.corrected += ret;
 813                                bitflips =  max_t(u32, bitflips, ret);
 814                        }
 815                }
 816        } else if (buf && buf != meson_chip->data_buf) {
 817                memcpy(buf, meson_chip->data_buf, mtd->writesize);
 818        }
 819
 820        return bitflips;
 821}
 822
 823static int meson_nfc_read_oob_raw(struct nand_chip *nand, int page)
 824{
 825        return meson_nfc_read_page_raw(nand, NULL, 1, page);
 826}
 827
 828static int meson_nfc_read_oob(struct nand_chip *nand, int page)
 829{
 830        return meson_nfc_read_page_hwecc(nand, NULL, 1, page);
 831}
 832
 833static bool meson_nfc_is_buffer_dma_safe(const void *buffer)
 834{
 835        if (virt_addr_valid(buffer) && (!object_is_on_stack(buffer)))
 836                return true;
 837        return false;
 838}
 839
 840static void *
 841meson_nand_op_get_dma_safe_input_buf(const struct nand_op_instr *instr)
 842{
 843        if (WARN_ON(instr->type != NAND_OP_DATA_IN_INSTR))
 844                return NULL;
 845
 846        if (meson_nfc_is_buffer_dma_safe(instr->ctx.data.buf.in))
 847                return instr->ctx.data.buf.in;
 848
 849        return kzalloc(instr->ctx.data.len, GFP_KERNEL);
 850}
 851
 852static void
 853meson_nand_op_put_dma_safe_input_buf(const struct nand_op_instr *instr,
 854                                     void *buf)
 855{
 856        if (WARN_ON(instr->type != NAND_OP_DATA_IN_INSTR) ||
 857            WARN_ON(!buf))
 858                return;
 859
 860        if (buf == instr->ctx.data.buf.in)
 861                return;
 862
 863        memcpy(instr->ctx.data.buf.in, buf, instr->ctx.data.len);
 864        kfree(buf);
 865}
 866
 867static void *
 868meson_nand_op_get_dma_safe_output_buf(const struct nand_op_instr *instr)
 869{
 870        if (WARN_ON(instr->type != NAND_OP_DATA_OUT_INSTR))
 871                return NULL;
 872
 873        if (meson_nfc_is_buffer_dma_safe(instr->ctx.data.buf.out))
 874                return (void *)instr->ctx.data.buf.out;
 875
 876        return kmemdup(instr->ctx.data.buf.out,
 877                       instr->ctx.data.len, GFP_KERNEL);
 878}
 879
 880static void
 881meson_nand_op_put_dma_safe_output_buf(const struct nand_op_instr *instr,
 882                                      const void *buf)
 883{
 884        if (WARN_ON(instr->type != NAND_OP_DATA_OUT_INSTR) ||
 885            WARN_ON(!buf))
 886                return;
 887
 888        if (buf != instr->ctx.data.buf.out)
 889                kfree(buf);
 890}
 891
 892static int meson_nfc_exec_op(struct nand_chip *nand,
 893                             const struct nand_operation *op, bool check_only)
 894{
 895        struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
 896        struct meson_nfc *nfc = nand_get_controller_data(nand);
 897        const struct nand_op_instr *instr = NULL;
 898        void *buf;
 899        u32 op_id, delay_idle, cmd;
 900        int i;
 901
 902        meson_nfc_select_chip(nand, op->cs);
 903        for (op_id = 0; op_id < op->ninstrs; op_id++) {
 904                instr = &op->instrs[op_id];
 905                delay_idle = DIV_ROUND_UP(PSEC_TO_NSEC(instr->delay_ns),
 906                                          meson_chip->level1_divider *
 907                                          NFC_CLK_CYCLE);
 908                switch (instr->type) {
 909                case NAND_OP_CMD_INSTR:
 910                        cmd = nfc->param.chip_select | NFC_CMD_CLE;
 911                        cmd |= instr->ctx.cmd.opcode & 0xff;
 912                        writel(cmd, nfc->reg_base + NFC_REG_CMD);
 913                        meson_nfc_cmd_idle(nfc, delay_idle);
 914                        break;
 915
 916                case NAND_OP_ADDR_INSTR:
 917                        for (i = 0; i < instr->ctx.addr.naddrs; i++) {
 918                                cmd = nfc->param.chip_select | NFC_CMD_ALE;
 919                                cmd |= instr->ctx.addr.addrs[i] & 0xff;
 920                                writel(cmd, nfc->reg_base + NFC_REG_CMD);
 921                        }
 922                        meson_nfc_cmd_idle(nfc, delay_idle);
 923                        break;
 924
 925                case NAND_OP_DATA_IN_INSTR:
 926                        buf = meson_nand_op_get_dma_safe_input_buf(instr);
 927                        if (!buf)
 928                                return -ENOMEM;
 929                        meson_nfc_read_buf(nand, buf, instr->ctx.data.len);
 930                        meson_nand_op_put_dma_safe_input_buf(instr, buf);
 931                        break;
 932
 933                case NAND_OP_DATA_OUT_INSTR:
 934                        buf = meson_nand_op_get_dma_safe_output_buf(instr);
 935                        if (!buf)
 936                                return -ENOMEM;
 937                        meson_nfc_write_buf(nand, buf, instr->ctx.data.len);
 938                        meson_nand_op_put_dma_safe_output_buf(instr, buf);
 939                        break;
 940
 941                case NAND_OP_WAITRDY_INSTR:
 942                        meson_nfc_queue_rb(nfc, instr->ctx.waitrdy.timeout_ms);
 943                        if (instr->delay_ns)
 944                                meson_nfc_cmd_idle(nfc, delay_idle);
 945                        break;
 946                }
 947        }
 948        meson_nfc_wait_cmd_finish(nfc, 1000);
 949        return 0;
 950}
 951
 952static int meson_ooblayout_ecc(struct mtd_info *mtd, int section,
 953                               struct mtd_oob_region *oobregion)
 954{
 955        struct nand_chip *nand = mtd_to_nand(mtd);
 956
 957        if (section >= nand->ecc.steps)
 958                return -ERANGE;
 959
 960        oobregion->offset =  2 + (section * (2 + nand->ecc.bytes));
 961        oobregion->length = nand->ecc.bytes;
 962
 963        return 0;
 964}
 965
 966static int meson_ooblayout_free(struct mtd_info *mtd, int section,
 967                                struct mtd_oob_region *oobregion)
 968{
 969        struct nand_chip *nand = mtd_to_nand(mtd);
 970
 971        if (section >= nand->ecc.steps)
 972                return -ERANGE;
 973
 974        oobregion->offset = section * (2 + nand->ecc.bytes);
 975        oobregion->length = 2;
 976
 977        return 0;
 978}
 979
 980static const struct mtd_ooblayout_ops meson_ooblayout_ops = {
 981        .ecc = meson_ooblayout_ecc,
 982        .free = meson_ooblayout_free,
 983};
 984
 985static int meson_nfc_clk_init(struct meson_nfc *nfc)
 986{
 987        int ret;
 988
 989        /* request core clock */
 990        nfc->core_clk = devm_clk_get(nfc->dev, "core");
 991        if (IS_ERR(nfc->core_clk)) {
 992                dev_err(nfc->dev, "failed to get core clock\n");
 993                return PTR_ERR(nfc->core_clk);
 994        }
 995
 996        nfc->device_clk = devm_clk_get(nfc->dev, "device");
 997        if (IS_ERR(nfc->device_clk)) {
 998                dev_err(nfc->dev, "failed to get device clock\n");
 999                return PTR_ERR(nfc->device_clk);
1000        }
1001
1002        nfc->phase_tx = devm_clk_get(nfc->dev, "tx");
1003        if (IS_ERR(nfc->phase_tx)) {
1004                dev_err(nfc->dev, "failed to get TX clk\n");
1005                return PTR_ERR(nfc->phase_tx);
1006        }
1007
1008        nfc->phase_rx = devm_clk_get(nfc->dev, "rx");
1009        if (IS_ERR(nfc->phase_rx)) {
1010                dev_err(nfc->dev, "failed to get RX clk\n");
1011                return PTR_ERR(nfc->phase_rx);
1012        }
1013
1014        /* init SD_EMMC_CLOCK to sane defaults w/min clock rate */
1015        regmap_update_bits(nfc->reg_clk,
1016                           0, CLK_SELECT_NAND, CLK_SELECT_NAND);
1017
1018        ret = clk_prepare_enable(nfc->core_clk);
1019        if (ret) {
1020                dev_err(nfc->dev, "failed to enable core clock\n");
1021                return ret;
1022        }
1023
1024        ret = clk_prepare_enable(nfc->device_clk);
1025        if (ret) {
1026                dev_err(nfc->dev, "failed to enable device clock\n");
1027                goto err_device_clk;
1028        }
1029
1030        ret = clk_prepare_enable(nfc->phase_tx);
1031        if (ret) {
1032                dev_err(nfc->dev, "failed to enable TX clock\n");
1033                goto err_phase_tx;
1034        }
1035
1036        ret = clk_prepare_enable(nfc->phase_rx);
1037        if (ret) {
1038                dev_err(nfc->dev, "failed to enable RX clock\n");
1039                goto err_phase_rx;
1040        }
1041
1042        ret = clk_set_rate(nfc->device_clk, 24000000);
1043        if (ret)
1044                goto err_phase_rx;
1045
1046        return 0;
1047err_phase_rx:
1048        clk_disable_unprepare(nfc->phase_tx);
1049err_phase_tx:
1050        clk_disable_unprepare(nfc->device_clk);
1051err_device_clk:
1052        clk_disable_unprepare(nfc->core_clk);
1053        return ret;
1054}
1055
1056static void meson_nfc_disable_clk(struct meson_nfc *nfc)
1057{
1058        clk_disable_unprepare(nfc->phase_rx);
1059        clk_disable_unprepare(nfc->phase_tx);
1060        clk_disable_unprepare(nfc->device_clk);
1061        clk_disable_unprepare(nfc->core_clk);
1062}
1063
1064static void meson_nfc_free_buffer(struct nand_chip *nand)
1065{
1066        struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
1067
1068        kfree(meson_chip->info_buf);
1069        kfree(meson_chip->data_buf);
1070}
1071
1072static int meson_chip_buffer_init(struct nand_chip *nand)
1073{
1074        struct mtd_info *mtd = nand_to_mtd(nand);
1075        struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
1076        u32 page_bytes, info_bytes, nsectors;
1077
1078        nsectors = mtd->writesize / nand->ecc.size;
1079
1080        page_bytes =  mtd->writesize + mtd->oobsize;
1081        info_bytes = nsectors * PER_INFO_BYTE;
1082
1083        meson_chip->data_buf = kmalloc(page_bytes, GFP_KERNEL);
1084        if (!meson_chip->data_buf)
1085                return -ENOMEM;
1086
1087        meson_chip->info_buf = kmalloc(info_bytes, GFP_KERNEL);
1088        if (!meson_chip->info_buf) {
1089                kfree(meson_chip->data_buf);
1090                return -ENOMEM;
1091        }
1092
1093        return 0;
1094}
1095
1096static
1097int meson_nfc_setup_data_interface(struct nand_chip *nand, int csline,
1098                                   const struct nand_data_interface *conf)
1099{
1100        struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
1101        const struct nand_sdr_timings *timings;
1102        u32 div, bt_min, bt_max, tbers_clocks;
1103
1104        timings = nand_get_sdr_timings(conf);
1105        if (IS_ERR(timings))
1106                return -ENOTSUPP;
1107
1108        if (csline == NAND_DATA_IFACE_CHECK_ONLY)
1109                return 0;
1110
1111        div = DIV_ROUND_UP((timings->tRC_min / 1000), NFC_CLK_CYCLE);
1112        bt_min = (timings->tREA_max + NFC_DEFAULT_DELAY) / div;
1113        bt_max = (NFC_DEFAULT_DELAY + timings->tRHOH_min +
1114                  timings->tRC_min / 2) / div;
1115
1116        meson_chip->twb = DIV_ROUND_UP(PSEC_TO_NSEC(timings->tWB_max),
1117                                       div * NFC_CLK_CYCLE);
1118        meson_chip->tadl = DIV_ROUND_UP(PSEC_TO_NSEC(timings->tADL_min),
1119                                        div * NFC_CLK_CYCLE);
1120        tbers_clocks = DIV_ROUND_UP_ULL(PSEC_TO_NSEC(timings->tBERS_max),
1121                                        div * NFC_CLK_CYCLE);
1122        meson_chip->tbers_max = ilog2(tbers_clocks);
1123        if (!is_power_of_2(tbers_clocks))
1124                meson_chip->tbers_max++;
1125
1126        bt_min = DIV_ROUND_UP(bt_min, 1000);
1127        bt_max = DIV_ROUND_UP(bt_max, 1000);
1128
1129        if (bt_max < bt_min)
1130                return -EINVAL;
1131
1132        meson_chip->level1_divider = div;
1133        meson_chip->clk_rate = 1000000000 / meson_chip->level1_divider;
1134        meson_chip->bus_timing = (bt_min + bt_max) / 2 + 1;
1135
1136        return 0;
1137}
1138
1139static int meson_nand_bch_mode(struct nand_chip *nand)
1140{
1141        struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
1142        int i;
1143
1144        if (nand->ecc.strength > 60 || nand->ecc.strength < 8)
1145                return -EINVAL;
1146
1147        for (i = 0; i < ARRAY_SIZE(meson_ecc); i++) {
1148                if (meson_ecc[i].strength == nand->ecc.strength) {
1149                        meson_chip->bch_mode = meson_ecc[i].bch;
1150                        return 0;
1151                }
1152        }
1153
1154        return -EINVAL;
1155}
1156
1157static void meson_nand_detach_chip(struct nand_chip *nand)
1158{
1159        meson_nfc_free_buffer(nand);
1160}
1161
1162static int meson_nand_attach_chip(struct nand_chip *nand)
1163{
1164        struct meson_nfc *nfc = nand_get_controller_data(nand);
1165        struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
1166        struct mtd_info *mtd = nand_to_mtd(nand);
1167        int nsectors = mtd->writesize / 1024;
1168        int ret;
1169
1170        if (!mtd->name) {
1171                mtd->name = devm_kasprintf(nfc->dev, GFP_KERNEL,
1172                                           "%s:nand%d",
1173                                           dev_name(nfc->dev),
1174                                           meson_chip->sels[0]);
1175                if (!mtd->name)
1176                        return -ENOMEM;
1177        }
1178
1179        if (nand->bbt_options & NAND_BBT_USE_FLASH)
1180                nand->bbt_options |= NAND_BBT_NO_OOB;
1181
1182        nand->options |= NAND_NO_SUBPAGE_WRITE;
1183
1184        ret = nand_ecc_choose_conf(nand, nfc->data->ecc_caps,
1185                                   mtd->oobsize - 2 * nsectors);
1186        if (ret) {
1187                dev_err(nfc->dev, "failed to ECC init\n");
1188                return -EINVAL;
1189        }
1190
1191        mtd_set_ooblayout(mtd, &meson_ooblayout_ops);
1192
1193        ret = meson_nand_bch_mode(nand);
1194        if (ret)
1195                return -EINVAL;
1196
1197        nand->ecc.mode = NAND_ECC_HW;
1198        nand->ecc.write_page_raw = meson_nfc_write_page_raw;
1199        nand->ecc.write_page = meson_nfc_write_page_hwecc;
1200        nand->ecc.write_oob_raw = nand_write_oob_std;
1201        nand->ecc.write_oob = nand_write_oob_std;
1202
1203        nand->ecc.read_page_raw = meson_nfc_read_page_raw;
1204        nand->ecc.read_page = meson_nfc_read_page_hwecc;
1205        nand->ecc.read_oob_raw = meson_nfc_read_oob_raw;
1206        nand->ecc.read_oob = meson_nfc_read_oob;
1207
1208        if (nand->options & NAND_BUSWIDTH_16) {
1209                dev_err(nfc->dev, "16bits bus width not supported");
1210                return -EINVAL;
1211        }
1212        ret = meson_chip_buffer_init(nand);
1213        if (ret)
1214                return -ENOMEM;
1215
1216        return ret;
1217}
1218
1219static const struct nand_controller_ops meson_nand_controller_ops = {
1220        .attach_chip = meson_nand_attach_chip,
1221        .detach_chip = meson_nand_detach_chip,
1222        .setup_data_interface = meson_nfc_setup_data_interface,
1223        .exec_op = meson_nfc_exec_op,
1224};
1225
1226static int
1227meson_nfc_nand_chip_init(struct device *dev,
1228                         struct meson_nfc *nfc, struct device_node *np)
1229{
1230        struct meson_nfc_nand_chip *meson_chip;
1231        struct nand_chip *nand;
1232        struct mtd_info *mtd;
1233        int ret, i;
1234        u32 tmp, nsels;
1235
1236        nsels = of_property_count_elems_of_size(np, "reg", sizeof(u32));
1237        if (!nsels || nsels > MAX_CE_NUM) {
1238                dev_err(dev, "invalid register property size\n");
1239                return -EINVAL;
1240        }
1241
1242        meson_chip = devm_kzalloc(dev, struct_size(meson_chip, sels, nsels),
1243                                  GFP_KERNEL);
1244        if (!meson_chip)
1245                return -ENOMEM;
1246
1247        meson_chip->nsels = nsels;
1248
1249        for (i = 0; i < nsels; i++) {
1250                ret = of_property_read_u32_index(np, "reg", i, &tmp);
1251                if (ret) {
1252                        dev_err(dev, "could not retrieve register property: %d\n",
1253                                ret);
1254                        return ret;
1255                }
1256
1257                if (test_and_set_bit(tmp, &nfc->assigned_cs)) {
1258                        dev_err(dev, "CS %d already assigned\n", tmp);
1259                        return -EINVAL;
1260                }
1261        }
1262
1263        nand = &meson_chip->nand;
1264        nand->controller = &nfc->controller;
1265        nand->controller->ops = &meson_nand_controller_ops;
1266        nand_set_flash_node(nand, np);
1267        nand_set_controller_data(nand, nfc);
1268
1269        nand->options |= NAND_USE_BOUNCE_BUFFER;
1270        mtd = nand_to_mtd(nand);
1271        mtd->owner = THIS_MODULE;
1272        mtd->dev.parent = dev;
1273
1274        ret = nand_scan(nand, nsels);
1275        if (ret)
1276                return ret;
1277
1278        ret = mtd_device_register(mtd, NULL, 0);
1279        if (ret) {
1280                dev_err(dev, "failed to register MTD device: %d\n", ret);
1281                nand_cleanup(nand);
1282                return ret;
1283        }
1284
1285        list_add_tail(&meson_chip->node, &nfc->chips);
1286
1287        return 0;
1288}
1289
1290static int meson_nfc_nand_chip_cleanup(struct meson_nfc *nfc)
1291{
1292        struct meson_nfc_nand_chip *meson_chip;
1293        struct mtd_info *mtd;
1294        int ret;
1295
1296        while (!list_empty(&nfc->chips)) {
1297                meson_chip = list_first_entry(&nfc->chips,
1298                                              struct meson_nfc_nand_chip, node);
1299                mtd = nand_to_mtd(&meson_chip->nand);
1300                ret = mtd_device_unregister(mtd);
1301                if (ret)
1302                        return ret;
1303
1304                meson_nfc_free_buffer(&meson_chip->nand);
1305                nand_cleanup(&meson_chip->nand);
1306                list_del(&meson_chip->node);
1307        }
1308
1309        return 0;
1310}
1311
1312static int meson_nfc_nand_chips_init(struct device *dev,
1313                                     struct meson_nfc *nfc)
1314{
1315        struct device_node *np = dev->of_node;
1316        struct device_node *nand_np;
1317        int ret;
1318
1319        for_each_child_of_node(np, nand_np) {
1320                ret = meson_nfc_nand_chip_init(dev, nfc, nand_np);
1321                if (ret) {
1322                        meson_nfc_nand_chip_cleanup(nfc);
1323                        of_node_put(nand_np);
1324                        return ret;
1325                }
1326        }
1327
1328        return 0;
1329}
1330
1331static irqreturn_t meson_nfc_irq(int irq, void *id)
1332{
1333        struct meson_nfc *nfc = id;
1334        u32 cfg;
1335
1336        cfg = readl(nfc->reg_base + NFC_REG_CFG);
1337        if (!(cfg & NFC_RB_IRQ_EN))
1338                return IRQ_NONE;
1339
1340        cfg &= ~(NFC_RB_IRQ_EN);
1341        writel(cfg, nfc->reg_base + NFC_REG_CFG);
1342
1343        complete(&nfc->completion);
1344        return IRQ_HANDLED;
1345}
1346
1347static const struct meson_nfc_data meson_gxl_data = {
1348        .ecc_caps = &meson_gxl_ecc_caps,
1349};
1350
1351static const struct meson_nfc_data meson_axg_data = {
1352        .ecc_caps = &meson_axg_ecc_caps,
1353};
1354
1355static const struct of_device_id meson_nfc_id_table[] = {
1356        {
1357                .compatible = "amlogic,meson-gxl-nfc",
1358                .data = &meson_gxl_data,
1359        }, {
1360                .compatible = "amlogic,meson-axg-nfc",
1361                .data = &meson_axg_data,
1362        },
1363        {}
1364};
1365MODULE_DEVICE_TABLE(of, meson_nfc_id_table);
1366
1367static int meson_nfc_probe(struct platform_device *pdev)
1368{
1369        struct device *dev = &pdev->dev;
1370        struct meson_nfc *nfc;
1371        struct resource *res;
1372        int ret, irq;
1373
1374        nfc = devm_kzalloc(dev, sizeof(*nfc), GFP_KERNEL);
1375        if (!nfc)
1376                return -ENOMEM;
1377
1378        nfc->data = of_device_get_match_data(&pdev->dev);
1379        if (!nfc->data)
1380                return -ENODEV;
1381
1382        nand_controller_init(&nfc->controller);
1383        INIT_LIST_HEAD(&nfc->chips);
1384        init_completion(&nfc->completion);
1385
1386        nfc->dev = dev;
1387
1388        res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1389        nfc->reg_base = devm_ioremap_resource(dev, res);
1390        if (IS_ERR(nfc->reg_base))
1391                return PTR_ERR(nfc->reg_base);
1392
1393        nfc->reg_clk =
1394                syscon_regmap_lookup_by_phandle(dev->of_node,
1395                                                "amlogic,mmc-syscon");
1396        if (IS_ERR(nfc->reg_clk)) {
1397                dev_err(dev, "Failed to lookup clock base\n");
1398                return PTR_ERR(nfc->reg_clk);
1399        }
1400
1401        irq = platform_get_irq(pdev, 0);
1402        if (irq < 0)
1403                return -EINVAL;
1404
1405        ret = meson_nfc_clk_init(nfc);
1406        if (ret) {
1407                dev_err(dev, "failed to initialize NAND clock\n");
1408                return ret;
1409        }
1410
1411        writel(0, nfc->reg_base + NFC_REG_CFG);
1412        ret = devm_request_irq(dev, irq, meson_nfc_irq, 0, dev_name(dev), nfc);
1413        if (ret) {
1414                dev_err(dev, "failed to request NFC IRQ\n");
1415                ret = -EINVAL;
1416                goto err_clk;
1417        }
1418
1419        ret = dma_set_mask(dev, DMA_BIT_MASK(32));
1420        if (ret) {
1421                dev_err(dev, "failed to set DMA mask\n");
1422                goto err_clk;
1423        }
1424
1425        platform_set_drvdata(pdev, nfc);
1426
1427        ret = meson_nfc_nand_chips_init(dev, nfc);
1428        if (ret) {
1429                dev_err(dev, "failed to init NAND chips\n");
1430                goto err_clk;
1431        }
1432
1433        return 0;
1434err_clk:
1435        meson_nfc_disable_clk(nfc);
1436        return ret;
1437}
1438
1439static int meson_nfc_remove(struct platform_device *pdev)
1440{
1441        struct meson_nfc *nfc = platform_get_drvdata(pdev);
1442        int ret;
1443
1444        ret = meson_nfc_nand_chip_cleanup(nfc);
1445        if (ret)
1446                return ret;
1447
1448        meson_nfc_disable_clk(nfc);
1449
1450        platform_set_drvdata(pdev, NULL);
1451
1452        return 0;
1453}
1454
1455static struct platform_driver meson_nfc_driver = {
1456        .probe  = meson_nfc_probe,
1457        .remove = meson_nfc_remove,
1458        .driver = {
1459                .name  = "meson-nand",
1460                .of_match_table = meson_nfc_id_table,
1461        },
1462};
1463module_platform_driver(meson_nfc_driver);
1464
1465MODULE_LICENSE("Dual MIT/GPL");
1466MODULE_AUTHOR("Liang Yang <liang.yang@amlogic.com>");
1467MODULE_DESCRIPTION("Amlogic's Meson NAND Flash Controller driver");
1468