LXR linux/drivers/dma/ioat/dca.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Intel I/OAT DMA Linux driver
   4 * Copyright(c) 2007 - 2009 Intel Corporation.
   5 */
   6
   7#include <linux/kernel.h>
   8#include <linux/pci.h>
   9#include <linux/smp.h>
  10#include <linux/interrupt.h>
  11#include <linux/dca.h>
  12
  13/* either a kernel change is needed, or we need something like this in kernel */
  14#ifndef CONFIG_SMP
  15#include <asm/smp.h>
  16#undef cpu_physical_id
  17#define cpu_physical_id(cpu) (cpuid_ebx(1) >> 24)
  18#endif
  19
  20#include "dma.h"
  21#include "registers.h"
  22
  23/*
  24 * Bit 7 of a tag map entry is the "valid" bit, if it is set then bits 0:6
  25 * contain the bit number of the APIC ID to map into the DCA tag.  If the valid
  26 * bit is not set, then the value must be 0 or 1 and defines the bit in the tag.
  27 */
  28#define DCA_TAG_MAP_VALID 0x80
  29
  30#define DCA3_TAG_MAP_BIT_TO_INV 0x80
  31#define DCA3_TAG_MAP_BIT_TO_SEL 0x40
  32#define DCA3_TAG_MAP_LITERAL_VAL 0x1
  33
  34#define DCA_TAG_MAP_MASK 0xDF
  35
  36/* expected tag map bytes for I/OAT ver.2 */
  37#define DCA2_TAG_MAP_BYTE0 0x80
  38#define DCA2_TAG_MAP_BYTE1 0x0
  39#define DCA2_TAG_MAP_BYTE2 0x81
  40#define DCA2_TAG_MAP_BYTE3 0x82
  41#define DCA2_TAG_MAP_BYTE4 0x82
  42
  43/*
  44 * "Legacy" DCA systems do not implement the DCA register set in the
  45 * I/OAT device.  Software needs direct support for their tag mappings.
  46 */
  47
  48#define APICID_BIT(x)           (DCA_TAG_MAP_VALID | (x))
  49#define IOAT_TAG_MAP_LEN        8
  50
  51/* pack PCI B/D/F into a u16 */
  52static inline u16 dcaid_from_pcidev(struct pci_dev *pci)
  53{
  54        return (pci->bus->number << 8) | pci->devfn;
  55}
  56
  57static int dca_enabled_in_bios(struct pci_dev *pdev)
  58{
  59        /* CPUID level 9 returns DCA configuration */
  60        /* Bit 0 indicates DCA enabled by the BIOS */
  61        unsigned long cpuid_level_9;
  62        int res;
  63
  64        cpuid_level_9 = cpuid_eax(9);
  65        res = test_bit(0, &cpuid_level_9);
  66        if (!res)
  67                dev_dbg(&pdev->dev, "DCA is disabled in BIOS\n");
  68
  69        return res;
  70}
  71
  72int system_has_dca_enabled(struct pci_dev *pdev)
  73{
  74        if (boot_cpu_has(X86_FEATURE_DCA))
  75                return dca_enabled_in_bios(pdev);
  76
  77        dev_dbg(&pdev->dev, "boot cpu doesn't have X86_FEATURE_DCA\n");
  78        return 0;
  79}
  80
  81struct ioat_dca_slot {
  82        struct pci_dev *pdev;   /* requester device */
  83        u16 rid;                /* requester id, as used by IOAT */
  84};
  85
  86#define IOAT_DCA_MAX_REQ 6
  87#define IOAT3_DCA_MAX_REQ 2
  88
  89struct ioat_dca_priv {
  90        void __iomem            *iobase;
  91        void __iomem            *dca_base;
  92        int                      max_requesters;
  93        int                      requester_count;
  94        u8                       tag_map[IOAT_TAG_MAP_LEN];
  95        struct ioat_dca_slot     req_slots[];
  96};
  97
  98static int ioat_dca_dev_managed(struct dca_provider *dca,
  99                                struct device *dev)
 100{
 101        struct ioat_dca_priv *ioatdca = dca_priv(dca);
 102        struct pci_dev *pdev;
 103        int i;
 104
 105        pdev = to_pci_dev(dev);
 106        for (i = 0; i < ioatdca->max_requesters; i++) {
 107                if (ioatdca->req_slots[i].pdev == pdev)
 108                        return 1;
 109        }
 110        return 0;
 111}
 112
 113static int ioat_dca_add_requester(struct dca_provider *dca, struct device *dev)
 114{
 115        struct ioat_dca_priv *ioatdca = dca_priv(dca);
 116        struct pci_dev *pdev;
 117        int i;
 118        u16 id;
 119        u16 global_req_table;
 120
 121        /* This implementation only supports PCI-Express */
 122        if (!dev_is_pci(dev))
 123                return -ENODEV;
 124        pdev = to_pci_dev(dev);
 125        id = dcaid_from_pcidev(pdev);
 126
 127        if (ioatdca->requester_count == ioatdca->max_requesters)
 128                return -ENODEV;
 129
 130        for (i = 0; i < ioatdca->max_requesters; i++) {
 131                if (ioatdca->req_slots[i].pdev == NULL) {
 132                        /* found an empty slot */
 133                        ioatdca->requester_count++;
 134                        ioatdca->req_slots[i].pdev = pdev;
 135                        ioatdca->req_slots[i].rid = id;
 136                        global_req_table =
 137                              readw(ioatdca->dca_base + IOAT3_DCA_GREQID_OFFSET);
 138                        writel(id | IOAT_DCA_GREQID_VALID,
 139                               ioatdca->iobase + global_req_table + (i * 4));
 140                        return i;
 141                }
 142        }
 143        /* Error, ioatdma->requester_count is out of whack */
 144        return -EFAULT;
 145}
 146
 147static int ioat_dca_remove_requester(struct dca_provider *dca,
 148                                      struct device *dev)
 149{
 150        struct ioat_dca_priv *ioatdca = dca_priv(dca);
 151        struct pci_dev *pdev;
 152        int i;
 153        u16 global_req_table;
 154
 155        /* This implementation only supports PCI-Express */
 156        if (!dev_is_pci(dev))
 157                return -ENODEV;
 158        pdev = to_pci_dev(dev);
 159
 160        for (i = 0; i < ioatdca->max_requesters; i++) {
 161                if (ioatdca->req_slots[i].pdev == pdev) {
 162                        global_req_table =
 163                              readw(ioatdca->dca_base + IOAT3_DCA_GREQID_OFFSET);
 164                        writel(0, ioatdca->iobase + global_req_table + (i * 4));
 165                        ioatdca->req_slots[i].pdev = NULL;
 166                        ioatdca->req_slots[i].rid = 0;
 167                        ioatdca->requester_count--;
 168                        return i;
 169                }
 170        }
 171        return -ENODEV;
 172}
 173
 174static u8 ioat_dca_get_tag(struct dca_provider *dca,
 175                            struct device *dev,
 176                            int cpu)
 177{
 178        u8 tag;
 179
 180        struct ioat_dca_priv *ioatdca = dca_priv(dca);
 181        int i, apic_id, bit, value;
 182        u8 entry;
 183
 184        tag = 0;
 185        apic_id = cpu_physical_id(cpu);
 186
 187        for (i = 0; i < IOAT_TAG_MAP_LEN; i++) {
 188                entry = ioatdca->tag_map[i];
 189                if (entry & DCA3_TAG_MAP_BIT_TO_SEL) {
 190                        bit = entry &
 191                                ~(DCA3_TAG_MAP_BIT_TO_SEL | DCA3_TAG_MAP_BIT_TO_INV);
 192                        value = (apic_id & (1 << bit)) ? 1 : 0;
 193                } else if (entry & DCA3_TAG_MAP_BIT_TO_INV) {
 194                        bit = entry & ~DCA3_TAG_MAP_BIT_TO_INV;
 195                        value = (apic_id & (1 << bit)) ? 0 : 1;
 196                } else {
 197                        value = (entry & DCA3_TAG_MAP_LITERAL_VAL) ? 1 : 0;
 198                }
 199                tag |= (value << i);
 200        }
 201
 202        return tag;
 203}
 204
 205static const struct dca_ops ioat_dca_ops = {
 206        .add_requester          = ioat_dca_add_requester,
 207        .remove_requester       = ioat_dca_remove_requester,
 208        .get_tag                = ioat_dca_get_tag,
 209        .dev_managed            = ioat_dca_dev_managed,
 210};
 211
 212static int ioat_dca_count_dca_slots(void *iobase, u16 dca_offset)
 213{
 214        int slots = 0;
 215        u32 req;
 216        u16 global_req_table;
 217
 218        global_req_table = readw(iobase + dca_offset + IOAT3_DCA_GREQID_OFFSET);
 219        if (global_req_table == 0)
 220                return 0;
 221
 222        do {
 223                req = readl(iobase + global_req_table + (slots * sizeof(u32)));
 224                slots++;
 225        } while ((req & IOAT_DCA_GREQID_LASTID) == 0);
 226
 227        return slots;
 228}
 229
 230static inline int dca3_tag_map_invalid(u8 *tag_map)
 231{
 232        /*
 233         * If the tag map is not programmed by the BIOS the default is:
 234         * 0x80 0x80 0x80 0x80 0x80 0x00 0x00 0x00
 235         *
 236         * This an invalid map and will result in only 2 possible tags
 237         * 0x1F and 0x00.  0x00 is an invalid DCA tag so we know that
 238         * this entire definition is invalid.
 239         */
 240        return ((tag_map[0] == DCA_TAG_MAP_VALID) &&
 241                (tag_map[1] == DCA_TAG_MAP_VALID) &&
 242                (tag_map[2] == DCA_TAG_MAP_VALID) &&
 243                (tag_map[3] == DCA_TAG_MAP_VALID) &&
 244                (tag_map[4] == DCA_TAG_MAP_VALID));
 245}
 246
 247struct dca_provider *ioat_dca_init(struct pci_dev *pdev, void __iomem *iobase)
 248{
 249        struct dca_provider *dca;
 250        struct ioat_dca_priv *ioatdca;
 251        int slots;
 252        int i;
 253        int err;
 254        u16 dca_offset;
 255        u16 csi_fsb_control;
 256        u16 pcie_control;
 257        u8 bit;
 258
 259        union {
 260                u64 full;
 261                struct {
 262                        u32 low;
 263                        u32 high;
 264                };
 265        } tag_map;
 266
 267        if (!system_has_dca_enabled(pdev))
 268                return NULL;
 269
 270        dca_offset = readw(iobase + IOAT_DCAOFFSET_OFFSET);
 271        if (dca_offset == 0)
 272                return NULL;
 273
 274        slots = ioat_dca_count_dca_slots(iobase, dca_offset);
 275        if (slots == 0)
 276                return NULL;
 277
 278        dca = alloc_dca_provider(&ioat_dca_ops,
 279                                 struct_size(ioatdca, req_slots, slots));
 280        if (!dca)
 281                return NULL;
 282
 283        ioatdca = dca_priv(dca);
 284        ioatdca->iobase = iobase;
 285        ioatdca->dca_base = iobase + dca_offset;
 286        ioatdca->max_requesters = slots;
 287
 288        /* some bios might not know to turn these on */
 289        csi_fsb_control = readw(ioatdca->dca_base + IOAT3_CSI_CONTROL_OFFSET);
 290        if ((csi_fsb_control & IOAT3_CSI_CONTROL_PREFETCH) == 0) {
 291                csi_fsb_control |= IOAT3_CSI_CONTROL_PREFETCH;
 292                writew(csi_fsb_control,
 293                       ioatdca->dca_base + IOAT3_CSI_CONTROL_OFFSET);
 294        }
 295        pcie_control = readw(ioatdca->dca_base + IOAT3_PCI_CONTROL_OFFSET);
 296        if ((pcie_control & IOAT3_PCI_CONTROL_MEMWR) == 0) {
 297                pcie_control |= IOAT3_PCI_CONTROL_MEMWR;
 298                writew(pcie_control,
 299                       ioatdca->dca_base + IOAT3_PCI_CONTROL_OFFSET);
 300        }
 301
 302
 303        /* TODO version, compatibility and configuration checks */
 304
 305        /* copy out the APIC to DCA tag map */
 306        tag_map.low =
 307                readl(ioatdca->dca_base + IOAT3_APICID_TAG_MAP_OFFSET_LOW);
 308        tag_map.high =
 309                readl(ioatdca->dca_base + IOAT3_APICID_TAG_MAP_OFFSET_HIGH);
 310        for (i = 0; i < 8; i++) {
 311                bit = tag_map.full >> (8 * i);
 312                ioatdca->tag_map[i] = bit & DCA_TAG_MAP_MASK;
 313        }
 314
 315        if (dca3_tag_map_invalid(ioatdca->tag_map)) {
 316                add_taint(TAINT_FIRMWARE_WORKAROUND, LOCKDEP_STILL_OK);
 317                pr_warn_once("%s %s: APICID_TAG_MAP set incorrectly by BIOS, disabling DCA\n",
 318                             dev_driver_string(&pdev->dev),
 319                             dev_name(&pdev->dev));
 320                free_dca_provider(dca);
 321                return NULL;
 322        }
 323
 324        err = register_dca_provider(dca, &pdev->dev);
 325        if (err) {
 326                free_dca_provider(dca);
 327                return NULL;
 328        }
 329
 330        return dca;
 331}
 332