LXR linux/include/linux/iova.h

   1/*
   2 * Copyright (c) 2006, Intel Corporation.
   3 *
   4 * This file is released under the GPLv2.
   5 *
   6 * Copyright (C) 2006-2008 Intel Corporation
   7 * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
   8 *
   9 */
  10
  11#ifndef _IOVA_H_
  12#define _IOVA_H_
  13
  14#include <linux/types.h>
  15#include <linux/kernel.h>
  16#include <linux/rbtree.h>
  17#include <linux/atomic.h>
  18#include <linux/dma-mapping.h>
  19
  20/* iova structure */
  21struct iova {
  22        struct rb_node  node;
  23        unsigned long   pfn_hi; /* Highest allocated pfn */
  24        unsigned long   pfn_lo; /* Lowest allocated pfn */
  25};
  26
  27struct iova_magazine;
  28struct iova_cpu_rcache;
  29
  30#define IOVA_RANGE_CACHE_MAX_SIZE 6     /* log of max cached IOVA range size (in pages) */
  31#define MAX_GLOBAL_MAGS 32      /* magazines per bin */
  32
  33struct iova_rcache {
  34        spinlock_t lock;
  35        unsigned long depot_size;
  36        struct iova_magazine *depot[MAX_GLOBAL_MAGS];
  37        struct iova_cpu_rcache __percpu *cpu_rcaches;
  38};
  39
  40struct iova_domain;
  41
  42/* Call-Back from IOVA code into IOMMU drivers */
  43typedef void (* iova_flush_cb)(struct iova_domain *domain);
  44
  45/* Destructor for per-entry data */
  46typedef void (* iova_entry_dtor)(unsigned long data);
  47
  48/* Number of entries per Flush Queue */
  49#define IOVA_FQ_SIZE    256
  50
  51/* Timeout (in ms) after which entries are flushed from the Flush-Queue */
  52#define IOVA_FQ_TIMEOUT 10
  53
  54/* Flush Queue entry for defered flushing */
  55struct iova_fq_entry {
  56        unsigned long iova_pfn;
  57        unsigned long pages;
  58        unsigned long data;
  59        u64 counter; /* Flush counter when this entrie was added */
  60};
  61
  62/* Per-CPU Flush Queue structure */
  63struct iova_fq {
  64        struct iova_fq_entry entries[IOVA_FQ_SIZE];
  65        unsigned head, tail;
  66        spinlock_t lock;
  67};
  68
  69/* holds all the iova translations for a domain */
  70struct iova_domain {
  71        spinlock_t      iova_rbtree_lock; /* Lock to protect update of rbtree */
  72        struct rb_root  rbroot;         /* iova domain rbtree root */
  73        struct rb_node  *cached32_node; /* Save last alloced node */
  74        unsigned long   granule;        /* pfn granularity for this domain */
  75        unsigned long   start_pfn;      /* Lower limit for this domain */
  76        unsigned long   dma_32bit_pfn;
  77        struct iova_rcache rcaches[IOVA_RANGE_CACHE_MAX_SIZE];  /* IOVA range caches */
  78
  79        iova_flush_cb   flush_cb;       /* Call-Back function to flush IOMMU
  80                                           TLBs */
  81
  82        iova_entry_dtor entry_dtor;     /* IOMMU driver specific destructor for
  83                                           iova entry */
  84
  85        struct iova_fq __percpu *fq;    /* Flush Queue */
  86
  87        atomic64_t      fq_flush_start_cnt;     /* Number of TLB flushes that
  88                                                   have been started */
  89
  90        atomic64_t      fq_flush_finish_cnt;    /* Number of TLB flushes that
  91                                                   have been finished */
  92
  93        struct timer_list fq_timer;             /* Timer to regularily empty the
  94                                                   flush-queues */
  95        atomic_t fq_timer_on;                   /* 1 when timer is active, 0
  96                                                   when not */
  97};
  98
  99static inline unsigned long iova_size(struct iova *iova)
 100{
 101        return iova->pfn_hi - iova->pfn_lo + 1;
 102}
 103
 104static inline unsigned long iova_shift(struct iova_domain *iovad)
 105{
 106        return __ffs(iovad->granule);
 107}
 108
 109static inline unsigned long iova_mask(struct iova_domain *iovad)
 110{
 111        return iovad->granule - 1;
 112}
 113
 114static inline size_t iova_offset(struct iova_domain *iovad, dma_addr_t iova)
 115{
 116        return iova & iova_mask(iovad);
 117}
 118
 119static inline size_t iova_align(struct iova_domain *iovad, size_t size)
 120{
 121        return ALIGN(size, iovad->granule);
 122}
 123
 124static inline dma_addr_t iova_dma_addr(struct iova_domain *iovad, struct iova *iova)
 125{
 126        return (dma_addr_t)iova->pfn_lo << iova_shift(iovad);
 127}
 128
 129static inline unsigned long iova_pfn(struct iova_domain *iovad, dma_addr_t iova)
 130{
 131        return iova >> iova_shift(iovad);
 132}
 133
 134#if IS_ENABLED(CONFIG_IOMMU_IOVA)
 135int iova_cache_get(void);
 136void iova_cache_put(void);
 137
 138struct iova *alloc_iova_mem(void);
 139void free_iova_mem(struct iova *iova);
 140void free_iova(struct iova_domain *iovad, unsigned long pfn);
 141void __free_iova(struct iova_domain *iovad, struct iova *iova);
 142struct iova *alloc_iova(struct iova_domain *iovad, unsigned long size,
 143        unsigned long limit_pfn,
 144        bool size_aligned);
 145void free_iova_fast(struct iova_domain *iovad, unsigned long pfn,
 146                    unsigned long size);
 147void queue_iova(struct iova_domain *iovad,
 148                unsigned long pfn, unsigned long pages,
 149                unsigned long data);
 150unsigned long alloc_iova_fast(struct iova_domain *iovad, unsigned long size,
 151                              unsigned long limit_pfn);
 152struct iova *reserve_iova(struct iova_domain *iovad, unsigned long pfn_lo,
 153        unsigned long pfn_hi);
 154void copy_reserved_iova(struct iova_domain *from, struct iova_domain *to);
 155void init_iova_domain(struct iova_domain *iovad, unsigned long granule,
 156        unsigned long start_pfn, unsigned long pfn_32bit);
 157int init_iova_flush_queue(struct iova_domain *iovad,
 158                          iova_flush_cb flush_cb, iova_entry_dtor entry_dtor);
 159struct iova *find_iova(struct iova_domain *iovad, unsigned long pfn);
 160void put_iova_domain(struct iova_domain *iovad);
 161struct iova *split_and_remove_iova(struct iova_domain *iovad,
 162        struct iova *iova, unsigned long pfn_lo, unsigned long pfn_hi);
 163void free_cpu_cached_iovas(unsigned int cpu, struct iova_domain *iovad);
 164#else
 165static inline int iova_cache_get(void)
 166{
 167        return -ENOTSUPP;
 168}
 169
 170static inline void iova_cache_put(void)
 171{
 172}
 173
 174static inline struct iova *alloc_iova_mem(void)
 175{
 176        return NULL;
 177}
 178
 179static inline void free_iova_mem(struct iova *iova)
 180{
 181}
 182
 183static inline void free_iova(struct iova_domain *iovad, unsigned long pfn)
 184{
 185}
 186
 187static inline void __free_iova(struct iova_domain *iovad, struct iova *iova)
 188{
 189}
 190
 191static inline struct iova *alloc_iova(struct iova_domain *iovad,
 192                                      unsigned long size,
 193                                      unsigned long limit_pfn,
 194                                      bool size_aligned)
 195{
 196        return NULL;
 197}
 198
 199static inline void free_iova_fast(struct iova_domain *iovad,
 200                                  unsigned long pfn,
 201                                  unsigned long size)
 202{
 203}
 204
 205static inline void queue_iova(struct iova_domain *iovad,
 206                              unsigned long pfn, unsigned long pages,
 207                              unsigned long data)
 208{
 209}
 210
 211static inline unsigned long alloc_iova_fast(struct iova_domain *iovad,
 212                                            unsigned long size,
 213                                            unsigned long limit_pfn)
 214{
 215        return 0;
 216}
 217
 218static inline struct iova *reserve_iova(struct iova_domain *iovad,
 219                                        unsigned long pfn_lo,
 220                                        unsigned long pfn_hi)
 221{
 222        return NULL;
 223}
 224
 225static inline void copy_reserved_iova(struct iova_domain *from,
 226                                      struct iova_domain *to)
 227{
 228}
 229
 230static inline void init_iova_domain(struct iova_domain *iovad,
 231                                    unsigned long granule,
 232                                    unsigned long start_pfn,
 233                                    unsigned long pfn_32bit)
 234{
 235}
 236
 237static inline int init_iova_flush_queue(struct iova_domain *iovad,
 238                                        iova_flush_cb flush_cb,
 239                                        iova_entry_dtor entry_dtor)
 240{
 241        return -ENODEV;
 242}
 243
 244static inline struct iova *find_iova(struct iova_domain *iovad,
 245                                     unsigned long pfn)
 246{
 247        return NULL;
 248}
 249
 250static inline void put_iova_domain(struct iova_domain *iovad)
 251{
 252}
 253
 254static inline struct iova *split_and_remove_iova(struct iova_domain *iovad,
 255                                                 struct iova *iova,
 256                                                 unsigned long pfn_lo,
 257                                                 unsigned long pfn_hi)
 258{
 259        return NULL;
 260}
 261
 262static inline void free_cpu_cached_iovas(unsigned int cpu,
 263                                         struct iova_domain *iovad)
 264{
 265}
 266#endif
 267
 268#endif
 269