LXR linux/drivers/dma/ioat

   1/*
   2 * Intel I/OAT DMA Linux driver
   3 * Copyright(c) 2007 Intel Corporation.
   4 *
   5 * This program is free software; you can redistribute it and/or modify it
   6 * under the terms and conditions of the GNU General Public License,
   7 * version 2, as published by the Free Software Foundation.
   8 *
   9 * This program is distributed in the hope that it will be useful, but WITHOUT
  10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  12 * more details.
  13 *
  14 * You should have received a copy of the GNU General Public License along with
  15 * this program; if not, write to the Free Software Foundation, Inc.,
  16 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
  17 *
  18 * The full GNU General Public License is included in this distribution in
  19 * the file called "COPYING".
  20 *
  21 */
  22
  23#include <linux/kernel.h>
  24#include <linux/pci.h>
  25#include <linux/smp.h>
  26#include <linux/interrupt.h>
  27#include <linux/dca.h>
  28
  29/* either a kernel change is needed, or we need something like this in kernel */
  30#ifndef CONFIG_SMP
  31#include <asm/smp.h>
  32#undef cpu_physical_id
  33#define cpu_physical_id(cpu) (cpuid_ebx(1) >> 24)
  34#endif
  35
  36#include "ioatdma.h"
  37#include "ioatdma_registers.h"
  38
  39/*
  40 * Bit 16 of a tag map entry is the "valid" bit, if it is set then bits 0:15
  41 * contain the bit number of the APIC ID to map into the DCA tag.  If the valid
  42 * bit is not set, then the value must be 0 or 1 and defines the bit in the tag.
  43 */
  44#define DCA_TAG_MAP_VALID 0x80
  45
  46/*
  47 * "Legacy" DCA systems do not implement the DCA register set in the
  48 * I/OAT device.  Software needs direct support for their tag mappings.
  49 */
  50
  51#define APICID_BIT(x)           (DCA_TAG_MAP_VALID | (x))
  52#define IOAT_TAG_MAP_LEN        8
  53
  54static u8 ioat_tag_map_BNB[IOAT_TAG_MAP_LEN] = {
  55        1, APICID_BIT(1), APICID_BIT(2), APICID_BIT(2), };
  56static u8 ioat_tag_map_SCNB[IOAT_TAG_MAP_LEN] = {
  57        1, APICID_BIT(1), APICID_BIT(2), APICID_BIT(2), };
  58static u8 ioat_tag_map_CNB[IOAT_TAG_MAP_LEN] = {
  59        1, APICID_BIT(1), APICID_BIT(3), APICID_BIT(4), APICID_BIT(2), };
  60static u8 ioat_tag_map_UNISYS[IOAT_TAG_MAP_LEN] = { 0 };
  61
  62/* pack PCI B/D/F into a u16 */
  63static inline u16 dcaid_from_pcidev(struct pci_dev *pci)
  64{
  65        return (pci->bus->number << 8) | pci->devfn;
  66}
  67
  68static int dca_enabled_in_bios(struct pci_dev *pdev)
  69{
  70        /* CPUID level 9 returns DCA configuration */
  71        /* Bit 0 indicates DCA enabled by the BIOS */
  72        unsigned long cpuid_level_9;
  73        int res;
  74
  75        cpuid_level_9 = cpuid_eax(9);
  76        res = test_bit(0, &cpuid_level_9);
  77        if (!res)
  78                dev_err(&pdev->dev, "DCA is disabled in BIOS\n");
  79
  80        return res;
  81}
  82
  83static int system_has_dca_enabled(struct pci_dev *pdev)
  84{
  85        if (boot_cpu_has(X86_FEATURE_DCA))
  86                return dca_enabled_in_bios(pdev);
  87
  88        dev_err(&pdev->dev, "boot cpu doesn't have X86_FEATURE_DCA\n");
  89        return 0;
  90}
  91
  92struct ioat_dca_slot {
  93        struct pci_dev *pdev;   /* requester device */
  94        u16 rid;                /* requester id, as used by IOAT */
  95};
  96
  97#define IOAT_DCA_MAX_REQ 6
  98
  99struct ioat_dca_priv {
 100        void __iomem            *iobase;
 101        void                    *dca_base;
 102        int                      max_requesters;
 103        int                      requester_count;
 104        u8                       tag_map[IOAT_TAG_MAP_LEN];
 105        struct ioat_dca_slot     req_slots[0];
 106};
 107
 108/* 5000 series chipset DCA Port Requester ID Table Entry Format
 109 * [15:8]       PCI-Express Bus Number
 110 * [7:3]        PCI-Express Device Number
 111 * [2:0]        PCI-Express Function Number
 112 *
 113 * 5000 series chipset DCA control register format
 114 * [7:1]        Reserved (0)
 115 * [0]          Ignore Function Number
 116 */
 117
 118static int ioat_dca_add_requester(struct dca_provider *dca, struct device *dev)
 119{
 120        struct ioat_dca_priv *ioatdca = dca_priv(dca);
 121        struct pci_dev *pdev;
 122        int i;
 123        u16 id;
 124
 125        /* This implementation only supports PCI-Express */
 126        if (dev->bus != &pci_bus_type)
 127                return -ENODEV;
 128        pdev = to_pci_dev(dev);
 129        id = dcaid_from_pcidev(pdev);
 130
 131        if (ioatdca->requester_count == ioatdca->max_requesters)
 132                return -ENODEV;
 133
 134        for (i = 0; i < ioatdca->max_requesters; i++) {
 135                if (ioatdca->req_slots[i].pdev == NULL) {
 136                        /* found an empty slot */
 137                        ioatdca->requester_count++;
 138                        ioatdca->req_slots[i].pdev = pdev;
 139                        ioatdca->req_slots[i].rid = id;
 140                        writew(id, ioatdca->dca_base + (i * 4));
 141                        /* make sure the ignore function bit is off */
 142                        writeb(0, ioatdca->dca_base + (i * 4) + 2);
 143                        return i;
 144                }
 145        }
 146        /* Error, ioatdma->requester_count is out of whack */
 147        return -EFAULT;
 148}
 149
 150static int ioat_dca_remove_requester(struct dca_provider *dca,
 151                                     struct device *dev)
 152{
 153        struct ioat_dca_priv *ioatdca = dca_priv(dca);
 154        struct pci_dev *pdev;
 155        int i;
 156
 157        /* This implementation only supports PCI-Express */
 158        if (dev->bus != &pci_bus_type)
 159                return -ENODEV;
 160        pdev = to_pci_dev(dev);
 161
 162        for (i = 0; i < ioatdca->max_requesters; i++) {
 163                if (ioatdca->req_slots[i].pdev == pdev) {
 164                        writew(0, ioatdca->dca_base + (i * 4));
 165                        ioatdca->req_slots[i].pdev = NULL;
 166                        ioatdca->req_slots[i].rid = 0;
 167                        ioatdca->requester_count--;
 168                        return i;
 169                }
 170        }
 171        return -ENODEV;
 172}
 173
 174static u8 ioat_dca_get_tag(struct dca_provider *dca, int cpu)
 175{
 176        struct ioat_dca_priv *ioatdca = dca_priv(dca);
 177        int i, apic_id, bit, value;
 178        u8 entry, tag;
 179
 180        tag = 0;
 181        apic_id = cpu_physical_id(cpu);
 182
 183        for (i = 0; i < IOAT_TAG_MAP_LEN; i++) {
 184                entry = ioatdca->tag_map[i];
 185                if (entry & DCA_TAG_MAP_VALID) {
 186                        bit = entry & ~DCA_TAG_MAP_VALID;
 187                        value = (apic_id & (1 << bit)) ? 1 : 0;
 188                } else {
 189                        value = entry ? 1 : 0;
 190                }
 191                tag |= (value << i);
 192        }
 193        return tag;
 194}
 195
 196static struct dca_ops ioat_dca_ops = {
 197        .add_requester          = ioat_dca_add_requester,
 198        .remove_requester       = ioat_dca_remove_requester,
 199        .get_tag                = ioat_dca_get_tag,
 200};
 201
 202
 203struct dca_provider *ioat_dca_init(struct pci_dev *pdev, void __iomem *iobase)
 204{
 205        struct dca_provider *dca;
 206        struct ioat_dca_priv *ioatdca;
 207        u8 *tag_map = NULL;
 208        int i;
 209        int err;
 210
 211        if (!system_has_dca_enabled(pdev))
 212                return NULL;
 213
 214        /* I/OAT v1 systems must have a known tag_map to support DCA */
 215        switch (pdev->vendor) {
 216        case PCI_VENDOR_ID_INTEL:
 217                switch (pdev->device) {
 218                case PCI_DEVICE_ID_INTEL_IOAT:
 219                        tag_map = ioat_tag_map_BNB;
 220                        break;
 221                case PCI_DEVICE_ID_INTEL_IOAT_CNB:
 222                        tag_map = ioat_tag_map_CNB;
 223                        break;
 224                case PCI_DEVICE_ID_INTEL_IOAT_SCNB:
 225                        tag_map = ioat_tag_map_SCNB;
 226                        break;
 227                }
 228                break;
 229        case PCI_VENDOR_ID_UNISYS:
 230                switch (pdev->device) {
 231                case PCI_DEVICE_ID_UNISYS_DMA_DIRECTOR:
 232                        tag_map = ioat_tag_map_UNISYS;
 233                        break;
 234                }
 235                break;
 236        }
 237        if (tag_map == NULL)
 238                return NULL;
 239
 240        dca = alloc_dca_provider(&ioat_dca_ops,
 241                        sizeof(*ioatdca) +
 242                        (sizeof(struct ioat_dca_slot) * IOAT_DCA_MAX_REQ));
 243        if (!dca)
 244                return NULL;
 245
 246        ioatdca = dca_priv(dca);
 247        ioatdca->max_requesters = IOAT_DCA_MAX_REQ;
 248
 249        ioatdca->dca_base = iobase + 0x54;
 250
 251        /* copy over the APIC ID to DCA tag mapping */
 252        for (i = 0; i < IOAT_TAG_MAP_LEN; i++)
 253                ioatdca->tag_map[i] = tag_map[i];
 254
 255        err = register_dca_provider(dca, &pdev->dev);
 256        if (err) {
 257                free_dca_provider(dca);
 258                return NULL;
 259        }
 260
 261        return dca;
 262}
 263
 264
 265static int ioat2_dca_add_requester(struct dca_provider *dca, struct device *dev)
 266{
 267        struct ioat_dca_priv *ioatdca = dca_priv(dca);
 268        struct pci_dev *pdev;
 269        int i;
 270        u16 id;
 271        u16 global_req_table;
 272
 273        /* This implementation only supports PCI-Express */
 274        if (dev->bus != &pci_bus_type)
 275                return -ENODEV;
 276        pdev = to_pci_dev(dev);
 277        id = dcaid_from_pcidev(pdev);
 278
 279        if (ioatdca->requester_count == ioatdca->max_requesters)
 280                return -ENODEV;
 281
 282        for (i = 0; i < ioatdca->max_requesters; i++) {
 283                if (ioatdca->req_slots[i].pdev == NULL) {
 284                        /* found an empty slot */
 285                        ioatdca->requester_count++;
 286                        ioatdca->req_slots[i].pdev = pdev;
 287                        ioatdca->req_slots[i].rid = id;
 288                        global_req_table =
 289                              readw(ioatdca->dca_base + IOAT_DCA_GREQID_OFFSET);
 290                        writel(id | IOAT_DCA_GREQID_VALID,
 291                               ioatdca->iobase + global_req_table + (i * 4));
 292                        return i;
 293                }
 294        }
 295        /* Error, ioatdma->requester_count is out of whack */
 296        return -EFAULT;
 297}
 298
 299static int ioat2_dca_remove_requester(struct dca_provider *dca,
 300                                      struct device *dev)
 301{
 302        struct ioat_dca_priv *ioatdca = dca_priv(dca);
 303        struct pci_dev *pdev;
 304        int i;
 305        u16 global_req_table;
 306
 307        /* This implementation only supports PCI-Express */
 308        if (dev->bus != &pci_bus_type)
 309                return -ENODEV;
 310        pdev = to_pci_dev(dev);
 311
 312        for (i = 0; i < ioatdca->max_requesters; i++) {
 313                if (ioatdca->req_slots[i].pdev == pdev) {
 314                        global_req_table =
 315                              readw(ioatdca->dca_base + IOAT_DCA_GREQID_OFFSET);
 316                        writel(0, ioatdca->iobase + global_req_table + (i * 4));
 317                        ioatdca->req_slots[i].pdev = NULL;
 318                        ioatdca->req_slots[i].rid = 0;
 319                        ioatdca->requester_count--;
 320                        return i;
 321                }
 322        }
 323        return -ENODEV;
 324}
 325
 326static u8 ioat2_dca_get_tag(struct dca_provider *dca, int cpu)
 327{
 328        u8 tag;
 329
 330        tag = ioat_dca_get_tag(dca, cpu);
 331        tag = (~tag) & 0x1F;
 332        return tag;
 333}
 334
 335static struct dca_ops ioat2_dca_ops = {
 336        .add_requester          = ioat2_dca_add_requester,
 337        .remove_requester       = ioat2_dca_remove_requester,
 338        .get_tag                = ioat2_dca_get_tag,
 339};
 340
 341static int ioat2_dca_count_dca_slots(void *iobase, u16 dca_offset)
 342{
 343        int slots = 0;
 344        u32 req;
 345        u16 global_req_table;
 346
 347        global_req_table = readw(iobase + dca_offset + IOAT_DCA_GREQID_OFFSET);
 348        if (global_req_table == 0)
 349                return 0;
 350        do {
 351                req = readl(iobase + global_req_table + (slots * sizeof(u32)));
 352                slots++;
 353        } while ((req & IOAT_DCA_GREQID_LASTID) == 0);
 354
 355        return slots;
 356}
 357
 358struct dca_provider *ioat2_dca_init(struct pci_dev *pdev, void __iomem *iobase)
 359{
 360        struct dca_provider *dca;
 361        struct ioat_dca_priv *ioatdca;
 362        int slots;
 363        int i;
 364        int err;
 365        u32 tag_map;
 366        u16 dca_offset;
 367        u16 csi_fsb_control;
 368        u16 pcie_control;
 369        u8 bit;
 370
 371        if (!system_has_dca_enabled(pdev))
 372                return NULL;
 373
 374        dca_offset = readw(iobase + IOAT_DCAOFFSET_OFFSET);
 375        if (dca_offset == 0)
 376                return NULL;
 377
 378        slots = ioat2_dca_count_dca_slots(iobase, dca_offset);
 379        if (slots == 0)
 380                return NULL;
 381
 382        dca = alloc_dca_provider(&ioat2_dca_ops,
 383                                 sizeof(*ioatdca)
 384                                      + (sizeof(struct ioat_dca_slot) * slots));
 385        if (!dca)
 386                return NULL;
 387
 388        ioatdca = dca_priv(dca);
 389        ioatdca->iobase = iobase;
 390        ioatdca->dca_base = iobase + dca_offset;
 391        ioatdca->max_requesters = slots;
 392
 393        /* some bios might not know to turn these on */
 394        csi_fsb_control = readw(ioatdca->dca_base + IOAT_FSB_CAP_ENABLE_OFFSET);
 395        if ((csi_fsb_control & IOAT_FSB_CAP_ENABLE_PREFETCH) == 0) {
 396                csi_fsb_control |= IOAT_FSB_CAP_ENABLE_PREFETCH;
 397                writew(csi_fsb_control,
 398                       ioatdca->dca_base + IOAT_FSB_CAP_ENABLE_OFFSET);
 399        }
 400        pcie_control = readw(ioatdca->dca_base + IOAT_PCI_CAP_ENABLE_OFFSET);
 401        if ((pcie_control & IOAT_PCI_CAP_ENABLE_MEMWR) == 0) {
 402                pcie_control |= IOAT_PCI_CAP_ENABLE_MEMWR;
 403                writew(pcie_control,
 404                       ioatdca->dca_base + IOAT_PCI_CAP_ENABLE_OFFSET);
 405        }
 406
 407
 408        /* TODO version, compatibility and configuration checks */
 409
 410        /* copy out the APIC to DCA tag map */
 411        tag_map = readl(ioatdca->dca_base + IOAT_APICID_TAG_MAP_OFFSET);
 412        for (i = 0; i < 5; i++) {
 413                bit = (tag_map >> (4 * i)) & 0x0f;
 414                if (bit < 8)
 415                        ioatdca->tag_map[i] = bit | DCA_TAG_MAP_VALID;
 416                else
 417                        ioatdca->tag_map[i] = 0;
 418        }
 419
 420        err = register_dca_provider(dca, &pdev->dev);
 421        if (err) {
 422                free_dca_provider(dca);
 423                return NULL;
 424        }
 425
 426        return dca;
 427}
 428