linux/drivers/iommu/mtk_iommu.c
<<
>>
Prefs
   1/*
   2 * Copyright (c) 2015-2016 MediaTek Inc.
   3 * Author: Yong Wu <yong.wu@mediatek.com>
   4 *
   5 * This program is free software; you can redistribute it and/or modify
   6 * it under the terms of the GNU General Public License version 2 as
   7 * published by the Free Software Foundation.
   8 *
   9 * This program is distributed in the hope that it will be useful,
  10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 * GNU General Public License for more details.
  13 */
  14#include <linux/memblock.h>
  15#include <linux/bug.h>
  16#include <linux/clk.h>
  17#include <linux/component.h>
  18#include <linux/device.h>
  19#include <linux/dma-iommu.h>
  20#include <linux/err.h>
  21#include <linux/interrupt.h>
  22#include <linux/io.h>
  23#include <linux/iommu.h>
  24#include <linux/iopoll.h>
  25#include <linux/list.h>
  26#include <linux/of_address.h>
  27#include <linux/of_iommu.h>
  28#include <linux/of_irq.h>
  29#include <linux/of_platform.h>
  30#include <linux/platform_device.h>
  31#include <linux/slab.h>
  32#include <linux/spinlock.h>
  33#include <asm/barrier.h>
  34#include <soc/mediatek/smi.h>
  35
  36#include "mtk_iommu.h"
  37
  38#define REG_MMU_PT_BASE_ADDR                    0x000
  39
  40#define REG_MMU_INVALIDATE                      0x020
  41#define F_ALL_INVLD                             0x2
  42#define F_MMU_INV_RANGE                         0x1
  43
  44#define REG_MMU_INVLD_START_A                   0x024
  45#define REG_MMU_INVLD_END_A                     0x028
  46
  47#define REG_MMU_INV_SEL                         0x038
  48#define F_INVLD_EN0                             BIT(0)
  49#define F_INVLD_EN1                             BIT(1)
  50
  51#define REG_MMU_STANDARD_AXI_MODE               0x048
  52#define REG_MMU_DCM_DIS                         0x050
  53
  54#define REG_MMU_CTRL_REG                        0x110
  55#define F_MMU_PREFETCH_RT_REPLACE_MOD           BIT(4)
  56#define F_MMU_TF_PROTECT_SEL_SHIFT(data) \
  57        ((data)->m4u_plat == M4U_MT2712 ? 4 : 5)
  58/* It's named by F_MMU_TF_PROT_SEL in mt2712. */
  59#define F_MMU_TF_PROTECT_SEL(prot, data) \
  60        (((prot) & 0x3) << F_MMU_TF_PROTECT_SEL_SHIFT(data))
  61
  62#define REG_MMU_IVRP_PADDR                      0x114
  63
  64#define REG_MMU_VLD_PA_RNG                      0x118
  65#define F_MMU_VLD_PA_RNG(EA, SA)                (((EA) << 8) | (SA))
  66
  67#define REG_MMU_INT_CONTROL0                    0x120
  68#define F_L2_MULIT_HIT_EN                       BIT(0)
  69#define F_TABLE_WALK_FAULT_INT_EN               BIT(1)
  70#define F_PREETCH_FIFO_OVERFLOW_INT_EN          BIT(2)
  71#define F_MISS_FIFO_OVERFLOW_INT_EN             BIT(3)
  72#define F_PREFETCH_FIFO_ERR_INT_EN              BIT(5)
  73#define F_MISS_FIFO_ERR_INT_EN                  BIT(6)
  74#define F_INT_CLR_BIT                           BIT(12)
  75
  76#define REG_MMU_INT_MAIN_CONTROL                0x124
  77#define F_INT_TRANSLATION_FAULT                 BIT(0)
  78#define F_INT_MAIN_MULTI_HIT_FAULT              BIT(1)
  79#define F_INT_INVALID_PA_FAULT                  BIT(2)
  80#define F_INT_ENTRY_REPLACEMENT_FAULT           BIT(3)
  81#define F_INT_TLB_MISS_FAULT                    BIT(4)
  82#define F_INT_MISS_TRANSACTION_FIFO_FAULT       BIT(5)
  83#define F_INT_PRETETCH_TRANSATION_FIFO_FAULT    BIT(6)
  84
  85#define REG_MMU_CPE_DONE                        0x12C
  86
  87#define REG_MMU_FAULT_ST1                       0x134
  88
  89#define REG_MMU_FAULT_VA                        0x13c
  90#define F_MMU_FAULT_VA_WRITE_BIT                BIT(1)
  91#define F_MMU_FAULT_VA_LAYER_BIT                BIT(0)
  92
  93#define REG_MMU_INVLD_PA                        0x140
  94#define REG_MMU_INT_ID                          0x150
  95#define F_MMU0_INT_ID_LARB_ID(a)                (((a) >> 7) & 0x7)
  96#define F_MMU0_INT_ID_PORT_ID(a)                (((a) >> 2) & 0x1f)
  97
  98#define MTK_PROTECT_PA_ALIGN                    128
  99
 100/*
 101 * Get the local arbiter ID and the portid within the larb arbiter
 102 * from mtk_m4u_id which is defined by MTK_M4U_ID.
 103 */
 104#define MTK_M4U_TO_LARB(id)             (((id) >> 5) & 0xf)
 105#define MTK_M4U_TO_PORT(id)             ((id) & 0x1f)
 106
 107struct mtk_iommu_domain {
 108        spinlock_t                      pgtlock; /* lock for page table */
 109
 110        struct io_pgtable_cfg           cfg;
 111        struct io_pgtable_ops           *iop;
 112
 113        struct iommu_domain             domain;
 114};
 115
 116static const struct iommu_ops mtk_iommu_ops;
 117
 118static LIST_HEAD(m4ulist);      /* List all the M4U HWs */
 119
 120#define for_each_m4u(data)      list_for_each_entry(data, &m4ulist, list)
 121
 122/*
 123 * There may be 1 or 2 M4U HWs, But we always expect they are in the same domain
 124 * for the performance.
 125 *
 126 * Here always return the mtk_iommu_data of the first probed M4U where the
 127 * iommu domain information is recorded.
 128 */
 129static struct mtk_iommu_data *mtk_iommu_get_m4u_data(void)
 130{
 131        struct mtk_iommu_data *data;
 132
 133        for_each_m4u(data)
 134                return data;
 135
 136        return NULL;
 137}
 138
 139static struct mtk_iommu_domain *to_mtk_domain(struct iommu_domain *dom)
 140{
 141        return container_of(dom, struct mtk_iommu_domain, domain);
 142}
 143
 144static void mtk_iommu_tlb_flush_all(void *cookie)
 145{
 146        struct mtk_iommu_data *data = cookie;
 147
 148        for_each_m4u(data) {
 149                writel_relaxed(F_INVLD_EN1 | F_INVLD_EN0,
 150                               data->base + REG_MMU_INV_SEL);
 151                writel_relaxed(F_ALL_INVLD, data->base + REG_MMU_INVALIDATE);
 152                wmb(); /* Make sure the tlb flush all done */
 153        }
 154}
 155
 156static void mtk_iommu_tlb_add_flush_nosync(unsigned long iova, size_t size,
 157                                           size_t granule, bool leaf,
 158                                           void *cookie)
 159{
 160        struct mtk_iommu_data *data = cookie;
 161
 162        for_each_m4u(data) {
 163                writel_relaxed(F_INVLD_EN1 | F_INVLD_EN0,
 164                               data->base + REG_MMU_INV_SEL);
 165
 166                writel_relaxed(iova, data->base + REG_MMU_INVLD_START_A);
 167                writel_relaxed(iova + size - 1,
 168                               data->base + REG_MMU_INVLD_END_A);
 169                writel_relaxed(F_MMU_INV_RANGE,
 170                               data->base + REG_MMU_INVALIDATE);
 171                data->tlb_flush_active = true;
 172        }
 173}
 174
 175static void mtk_iommu_tlb_sync(void *cookie)
 176{
 177        struct mtk_iommu_data *data = cookie;
 178        int ret;
 179        u32 tmp;
 180
 181        for_each_m4u(data) {
 182                /* Avoid timing out if there's nothing to wait for */
 183                if (!data->tlb_flush_active)
 184                        return;
 185
 186                ret = readl_poll_timeout_atomic(data->base + REG_MMU_CPE_DONE,
 187                                                tmp, tmp != 0, 10, 100000);
 188                if (ret) {
 189                        dev_warn(data->dev,
 190                                 "Partial TLB flush timed out, falling back to full flush\n");
 191                        mtk_iommu_tlb_flush_all(cookie);
 192                }
 193                /* Clear the CPE status */
 194                writel_relaxed(0, data->base + REG_MMU_CPE_DONE);
 195                data->tlb_flush_active = false;
 196        }
 197}
 198
 199static const struct iommu_gather_ops mtk_iommu_gather_ops = {
 200        .tlb_flush_all = mtk_iommu_tlb_flush_all,
 201        .tlb_add_flush = mtk_iommu_tlb_add_flush_nosync,
 202        .tlb_sync = mtk_iommu_tlb_sync,
 203};
 204
 205static irqreturn_t mtk_iommu_isr(int irq, void *dev_id)
 206{
 207        struct mtk_iommu_data *data = dev_id;
 208        struct mtk_iommu_domain *dom = data->m4u_dom;
 209        u32 int_state, regval, fault_iova, fault_pa;
 210        unsigned int fault_larb, fault_port;
 211        bool layer, write;
 212
 213        /* Read error info from registers */
 214        int_state = readl_relaxed(data->base + REG_MMU_FAULT_ST1);
 215        fault_iova = readl_relaxed(data->base + REG_MMU_FAULT_VA);
 216        layer = fault_iova & F_MMU_FAULT_VA_LAYER_BIT;
 217        write = fault_iova & F_MMU_FAULT_VA_WRITE_BIT;
 218        fault_pa = readl_relaxed(data->base + REG_MMU_INVLD_PA);
 219        regval = readl_relaxed(data->base + REG_MMU_INT_ID);
 220        fault_larb = F_MMU0_INT_ID_LARB_ID(regval);
 221        fault_port = F_MMU0_INT_ID_PORT_ID(regval);
 222
 223        if (report_iommu_fault(&dom->domain, data->dev, fault_iova,
 224                               write ? IOMMU_FAULT_WRITE : IOMMU_FAULT_READ)) {
 225                dev_err_ratelimited(
 226                        data->dev,
 227                        "fault type=0x%x iova=0x%x pa=0x%x larb=%d port=%d layer=%d %s\n",
 228                        int_state, fault_iova, fault_pa, fault_larb, fault_port,
 229                        layer, write ? "write" : "read");
 230        }
 231
 232        /* Interrupt clear */
 233        regval = readl_relaxed(data->base + REG_MMU_INT_CONTROL0);
 234        regval |= F_INT_CLR_BIT;
 235        writel_relaxed(regval, data->base + REG_MMU_INT_CONTROL0);
 236
 237        mtk_iommu_tlb_flush_all(data);
 238
 239        return IRQ_HANDLED;
 240}
 241
 242static void mtk_iommu_config(struct mtk_iommu_data *data,
 243                             struct device *dev, bool enable)
 244{
 245        struct mtk_smi_larb_iommu    *larb_mmu;
 246        unsigned int                 larbid, portid;
 247        struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
 248        int i;
 249
 250        for (i = 0; i < fwspec->num_ids; ++i) {
 251                larbid = MTK_M4U_TO_LARB(fwspec->ids[i]);
 252                portid = MTK_M4U_TO_PORT(fwspec->ids[i]);
 253                larb_mmu = &data->smi_imu.larb_imu[larbid];
 254
 255                dev_dbg(dev, "%s iommu port: %d\n",
 256                        enable ? "enable" : "disable", portid);
 257
 258                if (enable)
 259                        larb_mmu->mmu |= MTK_SMI_MMU_EN(portid);
 260                else
 261                        larb_mmu->mmu &= ~MTK_SMI_MMU_EN(portid);
 262        }
 263}
 264
 265static int mtk_iommu_domain_finalise(struct mtk_iommu_domain *dom)
 266{
 267        struct mtk_iommu_data *data = mtk_iommu_get_m4u_data();
 268
 269        spin_lock_init(&dom->pgtlock);
 270
 271        dom->cfg = (struct io_pgtable_cfg) {
 272                .quirks = IO_PGTABLE_QUIRK_ARM_NS |
 273                        IO_PGTABLE_QUIRK_NO_PERMS |
 274                        IO_PGTABLE_QUIRK_TLBI_ON_MAP,
 275                .pgsize_bitmap = mtk_iommu_ops.pgsize_bitmap,
 276                .ias = 32,
 277                .oas = 32,
 278                .tlb = &mtk_iommu_gather_ops,
 279                .iommu_dev = data->dev,
 280        };
 281
 282        if (data->enable_4GB)
 283                dom->cfg.quirks |= IO_PGTABLE_QUIRK_ARM_MTK_4GB;
 284
 285        dom->iop = alloc_io_pgtable_ops(ARM_V7S, &dom->cfg, data);
 286        if (!dom->iop) {
 287                dev_err(data->dev, "Failed to alloc io pgtable\n");
 288                return -EINVAL;
 289        }
 290
 291        /* Update our support page sizes bitmap */
 292        dom->domain.pgsize_bitmap = dom->cfg.pgsize_bitmap;
 293        return 0;
 294}
 295
 296static struct iommu_domain *mtk_iommu_domain_alloc(unsigned type)
 297{
 298        struct mtk_iommu_domain *dom;
 299
 300        if (type != IOMMU_DOMAIN_DMA)
 301                return NULL;
 302
 303        dom = kzalloc(sizeof(*dom), GFP_KERNEL);
 304        if (!dom)
 305                return NULL;
 306
 307        if (iommu_get_dma_cookie(&dom->domain))
 308                goto  free_dom;
 309
 310        if (mtk_iommu_domain_finalise(dom))
 311                goto  put_dma_cookie;
 312
 313        dom->domain.geometry.aperture_start = 0;
 314        dom->domain.geometry.aperture_end = DMA_BIT_MASK(32);
 315        dom->domain.geometry.force_aperture = true;
 316
 317        return &dom->domain;
 318
 319put_dma_cookie:
 320        iommu_put_dma_cookie(&dom->domain);
 321free_dom:
 322        kfree(dom);
 323        return NULL;
 324}
 325
 326static void mtk_iommu_domain_free(struct iommu_domain *domain)
 327{
 328        struct mtk_iommu_domain *dom = to_mtk_domain(domain);
 329
 330        free_io_pgtable_ops(dom->iop);
 331        iommu_put_dma_cookie(domain);
 332        kfree(to_mtk_domain(domain));
 333}
 334
 335static int mtk_iommu_attach_device(struct iommu_domain *domain,
 336                                   struct device *dev)
 337{
 338        struct mtk_iommu_domain *dom = to_mtk_domain(domain);
 339        struct mtk_iommu_data *data = dev_iommu_fwspec_get(dev)->iommu_priv;
 340
 341        if (!data)
 342                return -ENODEV;
 343
 344        /* Update the pgtable base address register of the M4U HW */
 345        if (!data->m4u_dom) {
 346                data->m4u_dom = dom;
 347                writel(dom->cfg.arm_v7s_cfg.ttbr[0],
 348                       data->base + REG_MMU_PT_BASE_ADDR);
 349        }
 350
 351        mtk_iommu_config(data, dev, true);
 352        return 0;
 353}
 354
 355static void mtk_iommu_detach_device(struct iommu_domain *domain,
 356                                    struct device *dev)
 357{
 358        struct mtk_iommu_data *data = dev_iommu_fwspec_get(dev)->iommu_priv;
 359
 360        if (!data)
 361                return;
 362
 363        mtk_iommu_config(data, dev, false);
 364}
 365
 366static int mtk_iommu_map(struct iommu_domain *domain, unsigned long iova,
 367                         phys_addr_t paddr, size_t size, int prot)
 368{
 369        struct mtk_iommu_domain *dom = to_mtk_domain(domain);
 370        unsigned long flags;
 371        int ret;
 372
 373        spin_lock_irqsave(&dom->pgtlock, flags);
 374        ret = dom->iop->map(dom->iop, iova, paddr & DMA_BIT_MASK(32),
 375                            size, prot);
 376        spin_unlock_irqrestore(&dom->pgtlock, flags);
 377
 378        return ret;
 379}
 380
 381static size_t mtk_iommu_unmap(struct iommu_domain *domain,
 382                              unsigned long iova, size_t size)
 383{
 384        struct mtk_iommu_domain *dom = to_mtk_domain(domain);
 385        unsigned long flags;
 386        size_t unmapsz;
 387
 388        spin_lock_irqsave(&dom->pgtlock, flags);
 389        unmapsz = dom->iop->unmap(dom->iop, iova, size);
 390        spin_unlock_irqrestore(&dom->pgtlock, flags);
 391
 392        return unmapsz;
 393}
 394
 395static void mtk_iommu_iotlb_sync(struct iommu_domain *domain)
 396{
 397        mtk_iommu_tlb_sync(mtk_iommu_get_m4u_data());
 398}
 399
 400static phys_addr_t mtk_iommu_iova_to_phys(struct iommu_domain *domain,
 401                                          dma_addr_t iova)
 402{
 403        struct mtk_iommu_domain *dom = to_mtk_domain(domain);
 404        struct mtk_iommu_data *data = mtk_iommu_get_m4u_data();
 405        unsigned long flags;
 406        phys_addr_t pa;
 407
 408        spin_lock_irqsave(&dom->pgtlock, flags);
 409        pa = dom->iop->iova_to_phys(dom->iop, iova);
 410        spin_unlock_irqrestore(&dom->pgtlock, flags);
 411
 412        if (data->enable_4GB)
 413                pa |= BIT_ULL(32);
 414
 415        return pa;
 416}
 417
 418static int mtk_iommu_add_device(struct device *dev)
 419{
 420        struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
 421        struct mtk_iommu_data *data;
 422        struct iommu_group *group;
 423
 424        if (!fwspec || fwspec->ops != &mtk_iommu_ops)
 425                return -ENODEV; /* Not a iommu client device */
 426
 427        data = fwspec->iommu_priv;
 428        iommu_device_link(&data->iommu, dev);
 429
 430        group = iommu_group_get_for_dev(dev);
 431        if (IS_ERR(group))
 432                return PTR_ERR(group);
 433
 434        iommu_group_put(group);
 435        return 0;
 436}
 437
 438static void mtk_iommu_remove_device(struct device *dev)
 439{
 440        struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
 441        struct mtk_iommu_data *data;
 442
 443        if (!fwspec || fwspec->ops != &mtk_iommu_ops)
 444                return;
 445
 446        data = fwspec->iommu_priv;
 447        iommu_device_unlink(&data->iommu, dev);
 448
 449        iommu_group_remove_device(dev);
 450        iommu_fwspec_free(dev);
 451}
 452
 453static struct iommu_group *mtk_iommu_device_group(struct device *dev)
 454{
 455        struct mtk_iommu_data *data = mtk_iommu_get_m4u_data();
 456
 457        if (!data)
 458                return ERR_PTR(-ENODEV);
 459
 460        /* All the client devices are in the same m4u iommu-group */
 461        if (!data->m4u_group) {
 462                data->m4u_group = iommu_group_alloc();
 463                if (IS_ERR(data->m4u_group))
 464                        dev_err(dev, "Failed to allocate M4U IOMMU group\n");
 465        } else {
 466                iommu_group_ref_get(data->m4u_group);
 467        }
 468        return data->m4u_group;
 469}
 470
 471static int mtk_iommu_of_xlate(struct device *dev, struct of_phandle_args *args)
 472{
 473        struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
 474        struct platform_device *m4updev;
 475
 476        if (args->args_count != 1) {
 477                dev_err(dev, "invalid #iommu-cells(%d) property for IOMMU\n",
 478                        args->args_count);
 479                return -EINVAL;
 480        }
 481
 482        if (!fwspec->iommu_priv) {
 483                /* Get the m4u device */
 484                m4updev = of_find_device_by_node(args->np);
 485                if (WARN_ON(!m4updev))
 486                        return -EINVAL;
 487
 488                fwspec->iommu_priv = platform_get_drvdata(m4updev);
 489        }
 490
 491        return iommu_fwspec_add_ids(dev, args->args, 1);
 492}
 493
 494static const struct iommu_ops mtk_iommu_ops = {
 495        .domain_alloc   = mtk_iommu_domain_alloc,
 496        .domain_free    = mtk_iommu_domain_free,
 497        .attach_dev     = mtk_iommu_attach_device,
 498        .detach_dev     = mtk_iommu_detach_device,
 499        .map            = mtk_iommu_map,
 500        .unmap          = mtk_iommu_unmap,
 501        .flush_iotlb_all = mtk_iommu_iotlb_sync,
 502        .iotlb_sync     = mtk_iommu_iotlb_sync,
 503        .iova_to_phys   = mtk_iommu_iova_to_phys,
 504        .add_device     = mtk_iommu_add_device,
 505        .remove_device  = mtk_iommu_remove_device,
 506        .device_group   = mtk_iommu_device_group,
 507        .of_xlate       = mtk_iommu_of_xlate,
 508        .pgsize_bitmap  = SZ_4K | SZ_64K | SZ_1M | SZ_16M,
 509};
 510
 511static int mtk_iommu_hw_init(const struct mtk_iommu_data *data)
 512{
 513        u32 regval;
 514        int ret;
 515
 516        ret = clk_prepare_enable(data->bclk);
 517        if (ret) {
 518                dev_err(data->dev, "Failed to enable iommu bclk(%d)\n", ret);
 519                return ret;
 520        }
 521
 522        regval = F_MMU_TF_PROTECT_SEL(2, data);
 523        if (data->m4u_plat == M4U_MT8173)
 524                regval |= F_MMU_PREFETCH_RT_REPLACE_MOD;
 525        writel_relaxed(regval, data->base + REG_MMU_CTRL_REG);
 526
 527        regval = F_L2_MULIT_HIT_EN |
 528                F_TABLE_WALK_FAULT_INT_EN |
 529                F_PREETCH_FIFO_OVERFLOW_INT_EN |
 530                F_MISS_FIFO_OVERFLOW_INT_EN |
 531                F_PREFETCH_FIFO_ERR_INT_EN |
 532                F_MISS_FIFO_ERR_INT_EN;
 533        writel_relaxed(regval, data->base + REG_MMU_INT_CONTROL0);
 534
 535        regval = F_INT_TRANSLATION_FAULT |
 536                F_INT_MAIN_MULTI_HIT_FAULT |
 537                F_INT_INVALID_PA_FAULT |
 538                F_INT_ENTRY_REPLACEMENT_FAULT |
 539                F_INT_TLB_MISS_FAULT |
 540                F_INT_MISS_TRANSACTION_FIFO_FAULT |
 541                F_INT_PRETETCH_TRANSATION_FIFO_FAULT;
 542        writel_relaxed(regval, data->base + REG_MMU_INT_MAIN_CONTROL);
 543
 544        if (data->m4u_plat == M4U_MT8173)
 545                regval = (data->protect_base >> 1) | (data->enable_4GB << 31);
 546        else
 547                regval = lower_32_bits(data->protect_base) |
 548                         upper_32_bits(data->protect_base);
 549        writel_relaxed(regval, data->base + REG_MMU_IVRP_PADDR);
 550
 551        if (data->enable_4GB && data->m4u_plat != M4U_MT8173) {
 552                /*
 553                 * If 4GB mode is enabled, the validate PA range is from
 554                 * 0x1_0000_0000 to 0x1_ffff_ffff. here record bit[32:30].
 555                 */
 556                regval = F_MMU_VLD_PA_RNG(7, 4);
 557                writel_relaxed(regval, data->base + REG_MMU_VLD_PA_RNG);
 558        }
 559        writel_relaxed(0, data->base + REG_MMU_DCM_DIS);
 560
 561        /* It's MISC control register whose default value is ok except mt8173.*/
 562        if (data->m4u_plat == M4U_MT8173)
 563                writel_relaxed(0, data->base + REG_MMU_STANDARD_AXI_MODE);
 564
 565        if (devm_request_irq(data->dev, data->irq, mtk_iommu_isr, 0,
 566                             dev_name(data->dev), (void *)data)) {
 567                writel_relaxed(0, data->base + REG_MMU_PT_BASE_ADDR);
 568                clk_disable_unprepare(data->bclk);
 569                dev_err(data->dev, "Failed @ IRQ-%d Request\n", data->irq);
 570                return -ENODEV;
 571        }
 572
 573        return 0;
 574}
 575
 576static const struct component_master_ops mtk_iommu_com_ops = {
 577        .bind           = mtk_iommu_bind,
 578        .unbind         = mtk_iommu_unbind,
 579};
 580
 581static int mtk_iommu_probe(struct platform_device *pdev)
 582{
 583        struct mtk_iommu_data   *data;
 584        struct device           *dev = &pdev->dev;
 585        struct resource         *res;
 586        resource_size_t         ioaddr;
 587        struct component_match  *match = NULL;
 588        void                    *protect;
 589        int                     i, larb_nr, ret;
 590
 591        data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL);
 592        if (!data)
 593                return -ENOMEM;
 594        data->dev = dev;
 595        data->m4u_plat = (enum mtk_iommu_plat)of_device_get_match_data(dev);
 596
 597        /* Protect memory. HW will access here while translation fault.*/
 598        protect = devm_kzalloc(dev, MTK_PROTECT_PA_ALIGN * 2, GFP_KERNEL);
 599        if (!protect)
 600                return -ENOMEM;
 601        data->protect_base = ALIGN(virt_to_phys(protect), MTK_PROTECT_PA_ALIGN);
 602
 603        /* Whether the current dram is over 4GB */
 604        data->enable_4GB = !!(max_pfn > (BIT_ULL(32) >> PAGE_SHIFT));
 605
 606        res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 607        data->base = devm_ioremap_resource(dev, res);
 608        if (IS_ERR(data->base))
 609                return PTR_ERR(data->base);
 610        ioaddr = res->start;
 611
 612        data->irq = platform_get_irq(pdev, 0);
 613        if (data->irq < 0)
 614                return data->irq;
 615
 616        data->bclk = devm_clk_get(dev, "bclk");
 617        if (IS_ERR(data->bclk))
 618                return PTR_ERR(data->bclk);
 619
 620        larb_nr = of_count_phandle_with_args(dev->of_node,
 621                                             "mediatek,larbs", NULL);
 622        if (larb_nr < 0)
 623                return larb_nr;
 624        data->smi_imu.larb_nr = larb_nr;
 625
 626        for (i = 0; i < larb_nr; i++) {
 627                struct device_node *larbnode;
 628                struct platform_device *plarbdev;
 629                u32 id;
 630
 631                larbnode = of_parse_phandle(dev->of_node, "mediatek,larbs", i);
 632                if (!larbnode)
 633                        return -EINVAL;
 634
 635                if (!of_device_is_available(larbnode))
 636                        continue;
 637
 638                ret = of_property_read_u32(larbnode, "mediatek,larb-id", &id);
 639                if (ret)/* The id is consecutive if there is no this property */
 640                        id = i;
 641
 642                plarbdev = of_find_device_by_node(larbnode);
 643                if (!plarbdev)
 644                        return -EPROBE_DEFER;
 645                data->smi_imu.larb_imu[id].dev = &plarbdev->dev;
 646
 647                component_match_add_release(dev, &match, release_of,
 648                                            compare_of, larbnode);
 649        }
 650
 651        platform_set_drvdata(pdev, data);
 652
 653        ret = mtk_iommu_hw_init(data);
 654        if (ret)
 655                return ret;
 656
 657        ret = iommu_device_sysfs_add(&data->iommu, dev, NULL,
 658                                     "mtk-iommu.%pa", &ioaddr);
 659        if (ret)
 660                return ret;
 661
 662        iommu_device_set_ops(&data->iommu, &mtk_iommu_ops);
 663        iommu_device_set_fwnode(&data->iommu, &pdev->dev.of_node->fwnode);
 664
 665        ret = iommu_device_register(&data->iommu);
 666        if (ret)
 667                return ret;
 668
 669        list_add_tail(&data->list, &m4ulist);
 670
 671        if (!iommu_present(&platform_bus_type))
 672                bus_set_iommu(&platform_bus_type, &mtk_iommu_ops);
 673
 674        return component_master_add_with_match(dev, &mtk_iommu_com_ops, match);
 675}
 676
 677static int mtk_iommu_remove(struct platform_device *pdev)
 678{
 679        struct mtk_iommu_data *data = platform_get_drvdata(pdev);
 680
 681        iommu_device_sysfs_remove(&data->iommu);
 682        iommu_device_unregister(&data->iommu);
 683
 684        if (iommu_present(&platform_bus_type))
 685                bus_set_iommu(&platform_bus_type, NULL);
 686
 687        clk_disable_unprepare(data->bclk);
 688        devm_free_irq(&pdev->dev, data->irq, data);
 689        component_master_del(&pdev->dev, &mtk_iommu_com_ops);
 690        return 0;
 691}
 692
 693static int __maybe_unused mtk_iommu_suspend(struct device *dev)
 694{
 695        struct mtk_iommu_data *data = dev_get_drvdata(dev);
 696        struct mtk_iommu_suspend_reg *reg = &data->reg;
 697        void __iomem *base = data->base;
 698
 699        reg->standard_axi_mode = readl_relaxed(base +
 700                                               REG_MMU_STANDARD_AXI_MODE);
 701        reg->dcm_dis = readl_relaxed(base + REG_MMU_DCM_DIS);
 702        reg->ctrl_reg = readl_relaxed(base + REG_MMU_CTRL_REG);
 703        reg->int_control0 = readl_relaxed(base + REG_MMU_INT_CONTROL0);
 704        reg->int_main_control = readl_relaxed(base + REG_MMU_INT_MAIN_CONTROL);
 705        reg->ivrp_paddr = readl_relaxed(base + REG_MMU_IVRP_PADDR);
 706        clk_disable_unprepare(data->bclk);
 707        return 0;
 708}
 709
 710static int __maybe_unused mtk_iommu_resume(struct device *dev)
 711{
 712        struct mtk_iommu_data *data = dev_get_drvdata(dev);
 713        struct mtk_iommu_suspend_reg *reg = &data->reg;
 714        void __iomem *base = data->base;
 715        int ret;
 716
 717        ret = clk_prepare_enable(data->bclk);
 718        if (ret) {
 719                dev_err(data->dev, "Failed to enable clk(%d) in resume\n", ret);
 720                return ret;
 721        }
 722        writel_relaxed(reg->standard_axi_mode,
 723                       base + REG_MMU_STANDARD_AXI_MODE);
 724        writel_relaxed(reg->dcm_dis, base + REG_MMU_DCM_DIS);
 725        writel_relaxed(reg->ctrl_reg, base + REG_MMU_CTRL_REG);
 726        writel_relaxed(reg->int_control0, base + REG_MMU_INT_CONTROL0);
 727        writel_relaxed(reg->int_main_control, base + REG_MMU_INT_MAIN_CONTROL);
 728        writel_relaxed(reg->ivrp_paddr, base + REG_MMU_IVRP_PADDR);
 729        if (data->m4u_dom)
 730                writel(data->m4u_dom->cfg.arm_v7s_cfg.ttbr[0],
 731                       base + REG_MMU_PT_BASE_ADDR);
 732        return 0;
 733}
 734
 735static const struct dev_pm_ops mtk_iommu_pm_ops = {
 736        SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(mtk_iommu_suspend, mtk_iommu_resume)
 737};
 738
 739static const struct of_device_id mtk_iommu_of_ids[] = {
 740        { .compatible = "mediatek,mt2712-m4u", .data = (void *)M4U_MT2712},
 741        { .compatible = "mediatek,mt8173-m4u", .data = (void *)M4U_MT8173},
 742        {}
 743};
 744
 745static struct platform_driver mtk_iommu_driver = {
 746        .probe  = mtk_iommu_probe,
 747        .remove = mtk_iommu_remove,
 748        .driver = {
 749                .name = "mtk-iommu",
 750                .of_match_table = of_match_ptr(mtk_iommu_of_ids),
 751                .pm = &mtk_iommu_pm_ops,
 752        }
 753};
 754
 755static int __init mtk_iommu_init(void)
 756{
 757        int ret;
 758
 759        ret = platform_driver_register(&mtk_iommu_driver);
 760        if (ret != 0)
 761                pr_err("Failed to register MTK IOMMU driver\n");
 762
 763        return ret;
 764}
 765
 766subsys_initcall(mtk_iommu_init)
 767