LXR linux/include/linux/io-pgtable.h

   1/* SPDX-License-Identifier: GPL-2.0 */
   2#ifndef __IO_PGTABLE_H
   3#define __IO_PGTABLE_H
   4
   5#include <linux/bitops.h>
   6#include <linux/iommu.h>
   7
   8/*
   9 * Public API for use by IOMMU drivers
  10 */
  11enum io_pgtable_fmt {
  12        ARM_32_LPAE_S1,
  13        ARM_32_LPAE_S2,
  14        ARM_64_LPAE_S1,
  15        ARM_64_LPAE_S2,
  16        ARM_V7S,
  17        ARM_MALI_LPAE,
  18        AMD_IOMMU_V1,
  19        APPLE_DART,
  20        IO_PGTABLE_NUM_FMTS,
  21};
  22
  23/**
  24 * struct iommu_flush_ops - IOMMU callbacks for TLB and page table management.
  25 *
  26 * @tlb_flush_all:  Synchronously invalidate the entire TLB context.
  27 * @tlb_flush_walk: Synchronously invalidate all intermediate TLB state
  28 *                  (sometimes referred to as the "walk cache") for a virtual
  29 *                  address range.
  30 * @tlb_add_page:   Optional callback to queue up leaf TLB invalidation for a
  31 *                  single page.  IOMMUs that cannot batch TLB invalidation
  32 *                  operations efficiently will typically issue them here, but
  33 *                  others may decide to update the iommu_iotlb_gather structure
  34 *                  and defer the invalidation until iommu_iotlb_sync() instead.
  35 *
  36 * Note that these can all be called in atomic context and must therefore
  37 * not block.
  38 */
  39struct iommu_flush_ops {
  40        void (*tlb_flush_all)(void *cookie);
  41        void (*tlb_flush_walk)(unsigned long iova, size_t size, size_t granule,
  42                               void *cookie);
  43        void (*tlb_add_page)(struct iommu_iotlb_gather *gather,
  44                             unsigned long iova, size_t granule, void *cookie);
  45};
  46
  47/**
  48 * struct io_pgtable_cfg - Configuration data for a set of page tables.
  49 *
  50 * @quirks:        A bitmap of hardware quirks that require some special
  51 *                 action by the low-level page table allocator.
  52 * @pgsize_bitmap: A bitmap of page sizes supported by this set of page
  53 *                 tables.
  54 * @ias:           Input address (iova) size, in bits.
  55 * @oas:           Output address (paddr) size, in bits.
  56 * @coherent_walk  A flag to indicate whether or not page table walks made
  57 *                 by the IOMMU are coherent with the CPU caches.
  58 * @tlb:           TLB management callbacks for this set of tables.
  59 * @iommu_dev:     The device representing the DMA configuration for the
  60 *                 page table walker.
  61 */
  62struct io_pgtable_cfg {
  63        /*
  64         * IO_PGTABLE_QUIRK_ARM_NS: (ARM formats) Set NS and NSTABLE bits in
  65         *      stage 1 PTEs, for hardware which insists on validating them
  66         *      even in non-secure state where they should normally be ignored.
  67         *
  68         * IO_PGTABLE_QUIRK_NO_PERMS: Ignore the IOMMU_READ, IOMMU_WRITE and
  69         *      IOMMU_NOEXEC flags and map everything with full access, for
  70         *      hardware which does not implement the permissions of a given
  71         *      format, and/or requires some format-specific default value.
  72         *
  73         * IO_PGTABLE_QUIRK_ARM_MTK_EXT: (ARM v7s format) MediaTek IOMMUs extend
  74         *      to support up to 35 bits PA where the bit32, bit33 and bit34 are
  75         *      encoded in the bit9, bit4 and bit5 of the PTE respectively.
  76         *
  77         * IO_PGTABLE_QUIRK_ARM_TTBR1: (ARM LPAE format) Configure the table
  78         *      for use in the upper half of a split address space.
  79         *
  80         * IO_PGTABLE_QUIRK_ARM_OUTER_WBWA: Override the outer-cacheability
  81         *      attributes set in the TCR for a non-coherent page-table walker.
  82         */
  83        #define IO_PGTABLE_QUIRK_ARM_NS         BIT(0)
  84        #define IO_PGTABLE_QUIRK_NO_PERMS       BIT(1)
  85        #define IO_PGTABLE_QUIRK_ARM_MTK_EXT    BIT(3)
  86        #define IO_PGTABLE_QUIRK_ARM_TTBR1      BIT(5)
  87        #define IO_PGTABLE_QUIRK_ARM_OUTER_WBWA BIT(6)
  88        unsigned long                   quirks;
  89        unsigned long                   pgsize_bitmap;
  90        unsigned int                    ias;
  91        unsigned int                    oas;
  92        bool                            coherent_walk;
  93        const struct iommu_flush_ops    *tlb;
  94        struct device                   *iommu_dev;
  95
  96        /* Low-level data specific to the table format */
  97        union {
  98                struct {
  99                        u64     ttbr;
 100                        struct {
 101                                u32     ips:3;
 102                                u32     tg:2;
 103                                u32     sh:2;
 104                                u32     orgn:2;
 105                                u32     irgn:2;
 106                                u32     tsz:6;
 107                        }       tcr;
 108                        u64     mair;
 109                } arm_lpae_s1_cfg;
 110
 111                struct {
 112                        u64     vttbr;
 113                        struct {
 114                                u32     ps:3;
 115                                u32     tg:2;
 116                                u32     sh:2;
 117                                u32     orgn:2;
 118                                u32     irgn:2;
 119                                u32     sl:2;
 120                                u32     tsz:6;
 121                        }       vtcr;
 122                } arm_lpae_s2_cfg;
 123
 124                struct {
 125                        u32     ttbr;
 126                        u32     tcr;
 127                        u32     nmrr;
 128                        u32     prrr;
 129                } arm_v7s_cfg;
 130
 131                struct {
 132                        u64     transtab;
 133                        u64     memattr;
 134                } arm_mali_lpae_cfg;
 135
 136                struct {
 137                        u64 ttbr[4];
 138                        u32 n_ttbrs;
 139                } apple_dart_cfg;
 140        };
 141};
 142
 143/**
 144 * struct io_pgtable_ops - Page table manipulation API for IOMMU drivers.
 145 *
 146 * @map:          Map a physically contiguous memory region.
 147 * @map_pages:    Map a physically contiguous range of pages of the same size.
 148 * @unmap:        Unmap a physically contiguous memory region.
 149 * @unmap_pages:  Unmap a range of virtually contiguous pages of the same size.
 150 * @iova_to_phys: Translate iova to physical address.
 151 *
 152 * These functions map directly onto the iommu_ops member functions with
 153 * the same names.
 154 */
 155struct io_pgtable_ops {
 156        int (*map)(struct io_pgtable_ops *ops, unsigned long iova,
 157                   phys_addr_t paddr, size_t size, int prot, gfp_t gfp);
 158        int (*map_pages)(struct io_pgtable_ops *ops, unsigned long iova,
 159                         phys_addr_t paddr, size_t pgsize, size_t pgcount,
 160                         int prot, gfp_t gfp, size_t *mapped);
 161        size_t (*unmap)(struct io_pgtable_ops *ops, unsigned long iova,
 162                        size_t size, struct iommu_iotlb_gather *gather);
 163        size_t (*unmap_pages)(struct io_pgtable_ops *ops, unsigned long iova,
 164                              size_t pgsize, size_t pgcount,
 165                              struct iommu_iotlb_gather *gather);
 166        phys_addr_t (*iova_to_phys)(struct io_pgtable_ops *ops,
 167                                    unsigned long iova);
 168};
 169
 170/**
 171 * alloc_io_pgtable_ops() - Allocate a page table allocator for use by an IOMMU.
 172 *
 173 * @fmt:    The page table format.
 174 * @cfg:    The page table configuration. This will be modified to represent
 175 *          the configuration actually provided by the allocator (e.g. the
 176 *          pgsize_bitmap may be restricted).
 177 * @cookie: An opaque token provided by the IOMMU driver and passed back to
 178 *          the callback routines in cfg->tlb.
 179 */
 180struct io_pgtable_ops *alloc_io_pgtable_ops(enum io_pgtable_fmt fmt,
 181                                            struct io_pgtable_cfg *cfg,
 182                                            void *cookie);
 183
 184/**
 185 * free_io_pgtable_ops() - Free an io_pgtable_ops structure. The caller
 186 *                         *must* ensure that the page table is no longer
 187 *                         live, but the TLB can be dirty.
 188 *
 189 * @ops: The ops returned from alloc_io_pgtable_ops.
 190 */
 191void free_io_pgtable_ops(struct io_pgtable_ops *ops);
 192
 193
 194/*
 195 * Internal structures for page table allocator implementations.
 196 */
 197
 198/**
 199 * struct io_pgtable - Internal structure describing a set of page tables.
 200 *
 201 * @fmt:    The page table format.
 202 * @cookie: An opaque token provided by the IOMMU driver and passed back to
 203 *          any callback routines.
 204 * @cfg:    A copy of the page table configuration.
 205 * @ops:    The page table operations in use for this set of page tables.
 206 */
 207struct io_pgtable {
 208        enum io_pgtable_fmt     fmt;
 209        void                    *cookie;
 210        struct io_pgtable_cfg   cfg;
 211        struct io_pgtable_ops   ops;
 212};
 213
 214#define io_pgtable_ops_to_pgtable(x) container_of((x), struct io_pgtable, ops)
 215
 216static inline void io_pgtable_tlb_flush_all(struct io_pgtable *iop)
 217{
 218        if (iop->cfg.tlb && iop->cfg.tlb->tlb_flush_all)
 219                iop->cfg.tlb->tlb_flush_all(iop->cookie);
 220}
 221
 222static inline void
 223io_pgtable_tlb_flush_walk(struct io_pgtable *iop, unsigned long iova,
 224                          size_t size, size_t granule)
 225{
 226        if (iop->cfg.tlb && iop->cfg.tlb->tlb_flush_walk)
 227                iop->cfg.tlb->tlb_flush_walk(iova, size, granule, iop->cookie);
 228}
 229
 230static inline void
 231io_pgtable_tlb_add_page(struct io_pgtable *iop,
 232                        struct iommu_iotlb_gather * gather, unsigned long iova,
 233                        size_t granule)
 234{
 235        if (iop->cfg.tlb && iop->cfg.tlb->tlb_add_page)
 236                iop->cfg.tlb->tlb_add_page(gather, iova, granule, iop->cookie);
 237}
 238
 239/**
 240 * struct io_pgtable_init_fns - Alloc/free a set of page tables for a
 241 *                              particular format.
 242 *
 243 * @alloc: Allocate a set of page tables described by cfg.
 244 * @free:  Free the page tables associated with iop.
 245 */
 246struct io_pgtable_init_fns {
 247        struct io_pgtable *(*alloc)(struct io_pgtable_cfg *cfg, void *cookie);
 248        void (*free)(struct io_pgtable *iop);
 249};
 250
 251extern struct io_pgtable_init_fns io_pgtable_arm_32_lpae_s1_init_fns;
 252extern struct io_pgtable_init_fns io_pgtable_arm_32_lpae_s2_init_fns;
 253extern struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s1_init_fns;
 254extern struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s2_init_fns;
 255extern struct io_pgtable_init_fns io_pgtable_arm_v7s_init_fns;
 256extern struct io_pgtable_init_fns io_pgtable_arm_mali_lpae_init_fns;
 257extern struct io_pgtable_init_fns io_pgtable_amd_iommu_v1_init_fns;
 258extern struct io_pgtable_init_fns io_pgtable_apple_dart_init_fns;
 259
 260#endif /* __IO_PGTABLE_H */
 261