LXR linux/drivers/dma/ioat/dca.c

   1/*
   2 * Intel I/OAT DMA Linux driver
   3 * Copyright(c) 2007 - 2009 Intel Corporation.
   4 *
   5 * This program is free software; you can redistribute it and/or modify it
   6 * under the terms and conditions of the GNU General Public License,
   7 * version 2, as published by the Free Software Foundation.
   8 *
   9 * This program is distributed in the hope that it will be useful, but WITHOUT
  10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  12 * more details.
  13 *
  14 * The full GNU General Public License is included in this distribution in
  15 * the file called "COPYING".
  16 *
  17 */
  18
  19#include <linux/kernel.h>
  20#include <linux/pci.h>
  21#include <linux/smp.h>
  22#include <linux/interrupt.h>
  23#include <linux/dca.h>
  24
  25/* either a kernel change is needed, or we need something like this in kernel */
  26#ifndef CONFIG_SMP
  27#include <asm/smp.h>
  28#undef cpu_physical_id
  29#define cpu_physical_id(cpu) (cpuid_ebx(1) >> 24)
  30#endif
  31
  32#include "dma.h"
  33#include "registers.h"
  34
  35/*
  36 * Bit 7 of a tag map entry is the "valid" bit, if it is set then bits 0:6
  37 * contain the bit number of the APIC ID to map into the DCA tag.  If the valid
  38 * bit is not set, then the value must be 0 or 1 and defines the bit in the tag.
  39 */
  40#define DCA_TAG_MAP_VALID 0x80
  41
  42#define DCA3_TAG_MAP_BIT_TO_INV 0x80
  43#define DCA3_TAG_MAP_BIT_TO_SEL 0x40
  44#define DCA3_TAG_MAP_LITERAL_VAL 0x1
  45
  46#define DCA_TAG_MAP_MASK 0xDF
  47
  48/* expected tag map bytes for I/OAT ver.2 */
  49#define DCA2_TAG_MAP_BYTE0 0x80
  50#define DCA2_TAG_MAP_BYTE1 0x0
  51#define DCA2_TAG_MAP_BYTE2 0x81
  52#define DCA2_TAG_MAP_BYTE3 0x82
  53#define DCA2_TAG_MAP_BYTE4 0x82
  54
  55/* verify if tag map matches expected values */
  56static inline int dca2_tag_map_valid(u8 *tag_map)
  57{
  58        return ((tag_map[0] == DCA2_TAG_MAP_BYTE0) &&
  59                (tag_map[1] == DCA2_TAG_MAP_BYTE1) &&
  60                (tag_map[2] == DCA2_TAG_MAP_BYTE2) &&
  61                (tag_map[3] == DCA2_TAG_MAP_BYTE3) &&
  62                (tag_map[4] == DCA2_TAG_MAP_BYTE4));
  63}
  64
  65/*
  66 * "Legacy" DCA systems do not implement the DCA register set in the
  67 * I/OAT device.  Software needs direct support for their tag mappings.
  68 */
  69
  70#define APICID_BIT(x)           (DCA_TAG_MAP_VALID | (x))
  71#define IOAT_TAG_MAP_LEN        8
  72
  73/* pack PCI B/D/F into a u16 */
  74static inline u16 dcaid_from_pcidev(struct pci_dev *pci)
  75{
  76        return (pci->bus->number << 8) | pci->devfn;
  77}
  78
  79static int dca_enabled_in_bios(struct pci_dev *pdev)
  80{
  81        /* CPUID level 9 returns DCA configuration */
  82        /* Bit 0 indicates DCA enabled by the BIOS */
  83        unsigned long cpuid_level_9;
  84        int res;
  85
  86        cpuid_level_9 = cpuid_eax(9);
  87        res = test_bit(0, &cpuid_level_9);
  88        if (!res)
  89                dev_dbg(&pdev->dev, "DCA is disabled in BIOS\n");
  90
  91        return res;
  92}
  93
  94int system_has_dca_enabled(struct pci_dev *pdev)
  95{
  96        if (boot_cpu_has(X86_FEATURE_DCA))
  97                return dca_enabled_in_bios(pdev);
  98
  99        dev_dbg(&pdev->dev, "boot cpu doesn't have X86_FEATURE_DCA\n");
 100        return 0;
 101}
 102
 103struct ioat_dca_slot {
 104        struct pci_dev *pdev;   /* requester device */
 105        u16 rid;                /* requester id, as used by IOAT */
 106};
 107
 108#define IOAT_DCA_MAX_REQ 6
 109#define IOAT3_DCA_MAX_REQ 2
 110
 111struct ioat_dca_priv {
 112        void __iomem            *iobase;
 113        void __iomem            *dca_base;
 114        int                      max_requesters;
 115        int                      requester_count;
 116        u8                       tag_map[IOAT_TAG_MAP_LEN];
 117        struct ioat_dca_slot     req_slots[0];
 118};
 119
 120static int ioat_dca_dev_managed(struct dca_provider *dca,
 121                                struct device *dev)
 122{
 123        struct ioat_dca_priv *ioatdca = dca_priv(dca);
 124        struct pci_dev *pdev;
 125        int i;
 126
 127        pdev = to_pci_dev(dev);
 128        for (i = 0; i < ioatdca->max_requesters; i++) {
 129                if (ioatdca->req_slots[i].pdev == pdev)
 130                        return 1;
 131        }
 132        return 0;
 133}
 134
 135static int ioat_dca_add_requester(struct dca_provider *dca, struct device *dev)
 136{
 137        struct ioat_dca_priv *ioatdca = dca_priv(dca);
 138        struct pci_dev *pdev;
 139        int i;
 140        u16 id;
 141        u16 global_req_table;
 142
 143        /* This implementation only supports PCI-Express */
 144        if (!dev_is_pci(dev))
 145                return -ENODEV;
 146        pdev = to_pci_dev(dev);
 147        id = dcaid_from_pcidev(pdev);
 148
 149        if (ioatdca->requester_count == ioatdca->max_requesters)
 150                return -ENODEV;
 151
 152        for (i = 0; i < ioatdca->max_requesters; i++) {
 153                if (ioatdca->req_slots[i].pdev == NULL) {
 154                        /* found an empty slot */
 155                        ioatdca->requester_count++;
 156                        ioatdca->req_slots[i].pdev = pdev;
 157                        ioatdca->req_slots[i].rid = id;
 158                        global_req_table =
 159                              readw(ioatdca->dca_base + IOAT3_DCA_GREQID_OFFSET);
 160                        writel(id | IOAT_DCA_GREQID_VALID,
 161                               ioatdca->iobase + global_req_table + (i * 4));
 162                        return i;
 163                }
 164        }
 165        /* Error, ioatdma->requester_count is out of whack */
 166        return -EFAULT;
 167}
 168
 169static int ioat_dca_remove_requester(struct dca_provider *dca,
 170                                      struct device *dev)
 171{
 172        struct ioat_dca_priv *ioatdca = dca_priv(dca);
 173        struct pci_dev *pdev;
 174        int i;
 175        u16 global_req_table;
 176
 177        /* This implementation only supports PCI-Express */
 178        if (!dev_is_pci(dev))
 179                return -ENODEV;
 180        pdev = to_pci_dev(dev);
 181
 182        for (i = 0; i < ioatdca->max_requesters; i++) {
 183                if (ioatdca->req_slots[i].pdev == pdev) {
 184                        global_req_table =
 185                              readw(ioatdca->dca_base + IOAT3_DCA_GREQID_OFFSET);
 186                        writel(0, ioatdca->iobase + global_req_table + (i * 4));
 187                        ioatdca->req_slots[i].pdev = NULL;
 188                        ioatdca->req_slots[i].rid = 0;
 189                        ioatdca->requester_count--;
 190                        return i;
 191                }
 192        }
 193        return -ENODEV;
 194}
 195
 196static u8 ioat_dca_get_tag(struct dca_provider *dca,
 197                            struct device *dev,
 198                            int cpu)
 199{
 200        u8 tag;
 201
 202        struct ioat_dca_priv *ioatdca = dca_priv(dca);
 203        int i, apic_id, bit, value;
 204        u8 entry;
 205
 206        tag = 0;
 207        apic_id = cpu_physical_id(cpu);
 208
 209        for (i = 0; i < IOAT_TAG_MAP_LEN; i++) {
 210                entry = ioatdca->tag_map[i];
 211                if (entry & DCA3_TAG_MAP_BIT_TO_SEL) {
 212                        bit = entry &
 213                                ~(DCA3_TAG_MAP_BIT_TO_SEL | DCA3_TAG_MAP_BIT_TO_INV);
 214                        value = (apic_id & (1 << bit)) ? 1 : 0;
 215                } else if (entry & DCA3_TAG_MAP_BIT_TO_INV) {
 216                        bit = entry & ~DCA3_TAG_MAP_BIT_TO_INV;
 217                        value = (apic_id & (1 << bit)) ? 0 : 1;
 218                } else {
 219                        value = (entry & DCA3_TAG_MAP_LITERAL_VAL) ? 1 : 0;
 220                }
 221                tag |= (value << i);
 222        }
 223
 224        return tag;
 225}
 226
 227static struct dca_ops ioat_dca_ops = {
 228        .add_requester          = ioat_dca_add_requester,
 229        .remove_requester       = ioat_dca_remove_requester,
 230        .get_tag                = ioat_dca_get_tag,
 231        .dev_managed            = ioat_dca_dev_managed,
 232};
 233
 234static int ioat_dca_count_dca_slots(void *iobase, u16 dca_offset)
 235{
 236        int slots = 0;
 237        u32 req;
 238        u16 global_req_table;
 239
 240        global_req_table = readw(iobase + dca_offset + IOAT3_DCA_GREQID_OFFSET);
 241        if (global_req_table == 0)
 242                return 0;
 243
 244        do {
 245                req = readl(iobase + global_req_table + (slots * sizeof(u32)));
 246                slots++;
 247        } while ((req & IOAT_DCA_GREQID_LASTID) == 0);
 248
 249        return slots;
 250}
 251
 252static inline int dca3_tag_map_invalid(u8 *tag_map)
 253{
 254        /*
 255         * If the tag map is not programmed by the BIOS the default is:
 256         * 0x80 0x80 0x80 0x80 0x80 0x00 0x00 0x00
 257         *
 258         * This an invalid map and will result in only 2 possible tags
 259         * 0x1F and 0x00.  0x00 is an invalid DCA tag so we know that
 260         * this entire definition is invalid.
 261         */
 262        return ((tag_map[0] == DCA_TAG_MAP_VALID) &&
 263                (tag_map[1] == DCA_TAG_MAP_VALID) &&
 264                (tag_map[2] == DCA_TAG_MAP_VALID) &&
 265                (tag_map[3] == DCA_TAG_MAP_VALID) &&
 266                (tag_map[4] == DCA_TAG_MAP_VALID));
 267}
 268
 269struct dca_provider *ioat_dca_init(struct pci_dev *pdev, void __iomem *iobase)
 270{
 271        struct dca_provider *dca;
 272        struct ioat_dca_priv *ioatdca;
 273        int slots;
 274        int i;
 275        int err;
 276        u16 dca_offset;
 277        u16 csi_fsb_control;
 278        u16 pcie_control;
 279        u8 bit;
 280
 281        union {
 282                u64 full;
 283                struct {
 284                        u32 low;
 285                        u32 high;
 286                };
 287        } tag_map;
 288
 289        if (!system_has_dca_enabled(pdev))
 290                return NULL;
 291
 292        dca_offset = readw(iobase + IOAT_DCAOFFSET_OFFSET);
 293        if (dca_offset == 0)
 294                return NULL;
 295
 296        slots = ioat_dca_count_dca_slots(iobase, dca_offset);
 297        if (slots == 0)
 298                return NULL;
 299
 300        dca = alloc_dca_provider(&ioat_dca_ops,
 301                                 sizeof(*ioatdca)
 302                                      + (sizeof(struct ioat_dca_slot) * slots));
 303        if (!dca)
 304                return NULL;
 305
 306        ioatdca = dca_priv(dca);
 307        ioatdca->iobase = iobase;
 308        ioatdca->dca_base = iobase + dca_offset;
 309        ioatdca->max_requesters = slots;
 310
 311        /* some bios might not know to turn these on */
 312        csi_fsb_control = readw(ioatdca->dca_base + IOAT3_CSI_CONTROL_OFFSET);
 313        if ((csi_fsb_control & IOAT3_CSI_CONTROL_PREFETCH) == 0) {
 314                csi_fsb_control |= IOAT3_CSI_CONTROL_PREFETCH;
 315                writew(csi_fsb_control,
 316                       ioatdca->dca_base + IOAT3_CSI_CONTROL_OFFSET);
 317        }
 318        pcie_control = readw(ioatdca->dca_base + IOAT3_PCI_CONTROL_OFFSET);
 319        if ((pcie_control & IOAT3_PCI_CONTROL_MEMWR) == 0) {
 320                pcie_control |= IOAT3_PCI_CONTROL_MEMWR;
 321                writew(pcie_control,
 322                       ioatdca->dca_base + IOAT3_PCI_CONTROL_OFFSET);
 323        }
 324
 325
 326        /* TODO version, compatibility and configuration checks */
 327
 328        /* copy out the APIC to DCA tag map */
 329        tag_map.low =
 330                readl(ioatdca->dca_base + IOAT3_APICID_TAG_MAP_OFFSET_LOW);
 331        tag_map.high =
 332                readl(ioatdca->dca_base + IOAT3_APICID_TAG_MAP_OFFSET_HIGH);
 333        for (i = 0; i < 8; i++) {
 334                bit = tag_map.full >> (8 * i);
 335                ioatdca->tag_map[i] = bit & DCA_TAG_MAP_MASK;
 336        }
 337
 338        if (dca3_tag_map_invalid(ioatdca->tag_map)) {
 339                WARN_TAINT_ONCE(1, TAINT_FIRMWARE_WORKAROUND,
 340                                "%s %s: APICID_TAG_MAP set incorrectly by BIOS, disabling DCA\n",
 341                                dev_driver_string(&pdev->dev),
 342                                dev_name(&pdev->dev));
 343                free_dca_provider(dca);
 344                return NULL;
 345        }
 346
 347        err = register_dca_provider(dca, &pdev->dev);
 348        if (err) {
 349                free_dca_provider(dca);
 350                return NULL;
 351        }
 352
 353        return dca;
 354}
 355