LXR linux/drivers/char/agp/amd-k7-agp.c

   1/*
   2 * AMD K7 AGPGART routines.
   3 */
   4
   5#include <linux/module.h>
   6#include <linux/pci.h>
   7#include <linux/init.h>
   8#include <linux/agp_backend.h>
   9#include <linux/page-flags.h>
  10#include <linux/mm.h>
  11#include <linux/slab.h>
  12#include <asm/set_memory.h>
  13#include "agp.h"
  14
  15#define AMD_MMBASE_BAR  1
  16#define AMD_APSIZE      0xac
  17#define AMD_MODECNTL    0xb0
  18#define AMD_MODECNTL2   0xb2
  19#define AMD_GARTENABLE  0x02    /* In mmio region (16-bit register) */
  20#define AMD_ATTBASE     0x04    /* In mmio region (32-bit register) */
  21#define AMD_TLBFLUSH    0x0c    /* In mmio region (32-bit register) */
  22#define AMD_CACHEENTRY  0x10    /* In mmio region (32-bit register) */
  23
  24static const struct pci_device_id agp_amdk7_pci_table[];
  25
  26struct amd_page_map {
  27        unsigned long *real;
  28        unsigned long __iomem *remapped;
  29};
  30
  31static struct _amd_irongate_private {
  32        volatile u8 __iomem *registers;
  33        struct amd_page_map **gatt_pages;
  34        int num_tables;
  35} amd_irongate_private;
  36
  37static int amd_create_page_map(struct amd_page_map *page_map)
  38{
  39        int i;
  40
  41        page_map->real = (unsigned long *) __get_free_page(GFP_KERNEL);
  42        if (page_map->real == NULL)
  43                return -ENOMEM;
  44
  45        set_memory_uc((unsigned long)page_map->real, 1);
  46        page_map->remapped = page_map->real;
  47
  48        for (i = 0; i < PAGE_SIZE / sizeof(unsigned long); i++) {
  49                writel(agp_bridge->scratch_page, page_map->remapped+i);
  50                readl(page_map->remapped+i);    /* PCI Posting. */
  51        }
  52
  53        return 0;
  54}
  55
  56static void amd_free_page_map(struct amd_page_map *page_map)
  57{
  58        set_memory_wb((unsigned long)page_map->real, 1);
  59        free_page((unsigned long) page_map->real);
  60}
  61
  62static void amd_free_gatt_pages(void)
  63{
  64        int i;
  65        struct amd_page_map **tables;
  66        struct amd_page_map *entry;
  67
  68        tables = amd_irongate_private.gatt_pages;
  69        for (i = 0; i < amd_irongate_private.num_tables; i++) {
  70                entry = tables[i];
  71                if (entry != NULL) {
  72                        if (entry->real != NULL)
  73                                amd_free_page_map(entry);
  74                        kfree(entry);
  75                }
  76        }
  77        kfree(tables);
  78        amd_irongate_private.gatt_pages = NULL;
  79}
  80
  81static int amd_create_gatt_pages(int nr_tables)
  82{
  83        struct amd_page_map **tables;
  84        struct amd_page_map *entry;
  85        int retval = 0;
  86        int i;
  87
  88        tables = kzalloc((nr_tables + 1) * sizeof(struct amd_page_map *),GFP_KERNEL);
  89        if (tables == NULL)
  90                return -ENOMEM;
  91
  92        for (i = 0; i < nr_tables; i++) {
  93                entry = kzalloc(sizeof(struct amd_page_map), GFP_KERNEL);
  94                tables[i] = entry;
  95                if (entry == NULL) {
  96                        retval = -ENOMEM;
  97                        break;
  98                }
  99                retval = amd_create_page_map(entry);
 100                if (retval != 0)
 101                        break;
 102        }
 103        amd_irongate_private.num_tables = i;
 104        amd_irongate_private.gatt_pages = tables;
 105
 106        if (retval != 0)
 107                amd_free_gatt_pages();
 108
 109        return retval;
 110}
 111
 112/* Since we don't need contiguous memory we just try
 113 * to get the gatt table once
 114 */
 115
 116#define GET_PAGE_DIR_OFF(addr) (addr >> 22)
 117#define GET_PAGE_DIR_IDX(addr) (GET_PAGE_DIR_OFF(addr) - \
 118        GET_PAGE_DIR_OFF(agp_bridge->gart_bus_addr))
 119#define GET_GATT_OFF(addr) ((addr & 0x003ff000) >> 12)
 120#define GET_GATT(addr) (amd_irongate_private.gatt_pages[\
 121        GET_PAGE_DIR_IDX(addr)]->remapped)
 122
 123static int amd_create_gatt_table(struct agp_bridge_data *bridge)
 124{
 125        struct aper_size_info_lvl2 *value;
 126        struct amd_page_map page_dir;
 127        unsigned long __iomem *cur_gatt;
 128        unsigned long addr;
 129        int retval;
 130        int i;
 131
 132        value = A_SIZE_LVL2(agp_bridge->current_size);
 133        retval = amd_create_page_map(&page_dir);
 134        if (retval != 0)
 135                return retval;
 136
 137        retval = amd_create_gatt_pages(value->num_entries / 1024);
 138        if (retval != 0) {
 139                amd_free_page_map(&page_dir);
 140                return retval;
 141        }
 142
 143        agp_bridge->gatt_table_real = (u32 *)page_dir.real;
 144        agp_bridge->gatt_table = (u32 __iomem *)page_dir.remapped;
 145        agp_bridge->gatt_bus_addr = virt_to_phys(page_dir.real);
 146
 147        /* Get the address for the gart region.
 148         * This is a bus address even on the alpha, b/c its
 149         * used to program the agp master not the cpu
 150         */
 151
 152        addr = pci_bus_address(agp_bridge->dev, AGP_APERTURE_BAR);
 153        agp_bridge->gart_bus_addr = addr;
 154
 155        /* Calculate the agp offset */
 156        for (i = 0; i < value->num_entries / 1024; i++, addr += 0x00400000) {
 157                writel(virt_to_phys(amd_irongate_private.gatt_pages[i]->real) | 1,
 158                        page_dir.remapped+GET_PAGE_DIR_OFF(addr));
 159                readl(page_dir.remapped+GET_PAGE_DIR_OFF(addr));        /* PCI Posting. */
 160        }
 161
 162        for (i = 0; i < value->num_entries; i++) {
 163                addr = (i * PAGE_SIZE) + agp_bridge->gart_bus_addr;
 164                cur_gatt = GET_GATT(addr);
 165                writel(agp_bridge->scratch_page, cur_gatt+GET_GATT_OFF(addr));
 166                readl(cur_gatt+GET_GATT_OFF(addr));     /* PCI Posting. */
 167        }
 168
 169        return 0;
 170}
 171
 172static int amd_free_gatt_table(struct agp_bridge_data *bridge)
 173{
 174        struct amd_page_map page_dir;
 175
 176        page_dir.real = (unsigned long *)agp_bridge->gatt_table_real;
 177        page_dir.remapped = (unsigned long __iomem *)agp_bridge->gatt_table;
 178
 179        amd_free_gatt_pages();
 180        amd_free_page_map(&page_dir);
 181        return 0;
 182}
 183
 184static int amd_irongate_fetch_size(void)
 185{
 186        int i;
 187        u32 temp;
 188        struct aper_size_info_lvl2 *values;
 189
 190        pci_read_config_dword(agp_bridge->dev, AMD_APSIZE, &temp);
 191        temp = (temp & 0x0000000e);
 192        values = A_SIZE_LVL2(agp_bridge->driver->aperture_sizes);
 193        for (i = 0; i < agp_bridge->driver->num_aperture_sizes; i++) {
 194                if (temp == values[i].size_value) {
 195                        agp_bridge->previous_size =
 196                            agp_bridge->current_size = (void *) (values + i);
 197
 198                        agp_bridge->aperture_size_idx = i;
 199                        return values[i].size;
 200                }
 201        }
 202
 203        return 0;
 204}
 205
 206static int amd_irongate_configure(void)
 207{
 208        struct aper_size_info_lvl2 *current_size;
 209        phys_addr_t reg;
 210        u32 temp;
 211        u16 enable_reg;
 212
 213        current_size = A_SIZE_LVL2(agp_bridge->current_size);
 214
 215        if (!amd_irongate_private.registers) {
 216                /* Get the memory mapped registers */
 217                reg = pci_resource_start(agp_bridge->dev, AMD_MMBASE_BAR);
 218                amd_irongate_private.registers = (volatile u8 __iomem *) ioremap(reg, 4096);
 219                if (!amd_irongate_private.registers)
 220                        return -ENOMEM;
 221        }
 222
 223        /* Write out the address of the gatt table */
 224        writel(agp_bridge->gatt_bus_addr, amd_irongate_private.registers+AMD_ATTBASE);
 225        readl(amd_irongate_private.registers+AMD_ATTBASE);      /* PCI Posting. */
 226
 227        /* Write the Sync register */
 228        pci_write_config_byte(agp_bridge->dev, AMD_MODECNTL, 0x80);
 229
 230        /* Set indexing mode */
 231        pci_write_config_byte(agp_bridge->dev, AMD_MODECNTL2, 0x00);
 232
 233        /* Write the enable register */
 234        enable_reg = readw(amd_irongate_private.registers+AMD_GARTENABLE);
 235        enable_reg = (enable_reg | 0x0004);
 236        writew(enable_reg, amd_irongate_private.registers+AMD_GARTENABLE);
 237        readw(amd_irongate_private.registers+AMD_GARTENABLE);   /* PCI Posting. */
 238
 239        /* Write out the size register */
 240        pci_read_config_dword(agp_bridge->dev, AMD_APSIZE, &temp);
 241        temp = (((temp & ~(0x0000000e)) | current_size->size_value) | 1);
 242        pci_write_config_dword(agp_bridge->dev, AMD_APSIZE, temp);
 243
 244        /* Flush the tlb */
 245        writel(1, amd_irongate_private.registers+AMD_TLBFLUSH);
 246        readl(amd_irongate_private.registers+AMD_TLBFLUSH);     /* PCI Posting.*/
 247        return 0;
 248}
 249
 250static void amd_irongate_cleanup(void)
 251{
 252        struct aper_size_info_lvl2 *previous_size;
 253        u32 temp;
 254        u16 enable_reg;
 255
 256        previous_size = A_SIZE_LVL2(agp_bridge->previous_size);
 257
 258        enable_reg = readw(amd_irongate_private.registers+AMD_GARTENABLE);
 259        enable_reg = (enable_reg & ~(0x0004));
 260        writew(enable_reg, amd_irongate_private.registers+AMD_GARTENABLE);
 261        readw(amd_irongate_private.registers+AMD_GARTENABLE);   /* PCI Posting. */
 262
 263        /* Write back the previous size and disable gart translation */
 264        pci_read_config_dword(agp_bridge->dev, AMD_APSIZE, &temp);
 265        temp = ((temp & ~(0x0000000f)) | previous_size->size_value);
 266        pci_write_config_dword(agp_bridge->dev, AMD_APSIZE, temp);
 267        iounmap((void __iomem *) amd_irongate_private.registers);
 268}
 269
 270/*
 271 * This routine could be implemented by taking the addresses
 272 * written to the GATT, and flushing them individually.  However
 273 * currently it just flushes the whole table.  Which is probably
 274 * more efficient, since agp_memory blocks can be a large number of
 275 * entries.
 276 */
 277
 278static void amd_irongate_tlbflush(struct agp_memory *temp)
 279{
 280        writel(1, amd_irongate_private.registers+AMD_TLBFLUSH);
 281        readl(amd_irongate_private.registers+AMD_TLBFLUSH);     /* PCI Posting. */
 282}
 283
 284static int amd_insert_memory(struct agp_memory *mem, off_t pg_start, int type)
 285{
 286        int i, j, num_entries;
 287        unsigned long __iomem *cur_gatt;
 288        unsigned long addr;
 289
 290        num_entries = A_SIZE_LVL2(agp_bridge->current_size)->num_entries;
 291
 292        if (type != mem->type ||
 293            agp_bridge->driver->agp_type_to_mask_type(agp_bridge, type))
 294                return -EINVAL;
 295
 296        if ((pg_start + mem->page_count) > num_entries)
 297                return -EINVAL;
 298
 299        j = pg_start;
 300        while (j < (pg_start + mem->page_count)) {
 301                addr = (j * PAGE_SIZE) + agp_bridge->gart_bus_addr;
 302                cur_gatt = GET_GATT(addr);
 303                if (!PGE_EMPTY(agp_bridge, readl(cur_gatt+GET_GATT_OFF(addr))))
 304                        return -EBUSY;
 305                j++;
 306        }
 307
 308        if (!mem->is_flushed) {
 309                global_cache_flush();
 310                mem->is_flushed = true;
 311        }
 312
 313        for (i = 0, j = pg_start; i < mem->page_count; i++, j++) {
 314                addr = (j * PAGE_SIZE) + agp_bridge->gart_bus_addr;
 315                cur_gatt = GET_GATT(addr);
 316                writel(agp_generic_mask_memory(agp_bridge,
 317                                               page_to_phys(mem->pages[i]),
 318                                               mem->type),
 319                       cur_gatt+GET_GATT_OFF(addr));
 320                readl(cur_gatt+GET_GATT_OFF(addr));     /* PCI Posting. */
 321        }
 322        amd_irongate_tlbflush(mem);
 323        return 0;
 324}
 325
 326static int amd_remove_memory(struct agp_memory *mem, off_t pg_start, int type)
 327{
 328        int i;
 329        unsigned long __iomem *cur_gatt;
 330        unsigned long addr;
 331
 332        if (type != mem->type ||
 333            agp_bridge->driver->agp_type_to_mask_type(agp_bridge, type))
 334                return -EINVAL;
 335
 336        for (i = pg_start; i < (mem->page_count + pg_start); i++) {
 337                addr = (i * PAGE_SIZE) + agp_bridge->gart_bus_addr;
 338                cur_gatt = GET_GATT(addr);
 339                writel(agp_bridge->scratch_page, cur_gatt+GET_GATT_OFF(addr));
 340                readl(cur_gatt+GET_GATT_OFF(addr));     /* PCI Posting. */
 341        }
 342
 343        amd_irongate_tlbflush(mem);
 344        return 0;
 345}
 346
 347static const struct aper_size_info_lvl2 amd_irongate_sizes[7] =
 348{
 349        {2048, 524288, 0x0000000c},
 350        {1024, 262144, 0x0000000a},
 351        {512, 131072, 0x00000008},
 352        {256, 65536, 0x00000006},
 353        {128, 32768, 0x00000004},
 354        {64, 16384, 0x00000002},
 355        {32, 8192, 0x00000000}
 356};
 357
 358static const struct gatt_mask amd_irongate_masks[] =
 359{
 360        {.mask = 1, .type = 0}
 361};
 362
 363static const struct agp_bridge_driver amd_irongate_driver = {
 364        .owner                  = THIS_MODULE,
 365        .aperture_sizes         = amd_irongate_sizes,
 366        .size_type              = LVL2_APER_SIZE,
 367        .num_aperture_sizes     = 7,
 368        .needs_scratch_page     = true,
 369        .configure              = amd_irongate_configure,
 370        .fetch_size             = amd_irongate_fetch_size,
 371        .cleanup                = amd_irongate_cleanup,
 372        .tlb_flush              = amd_irongate_tlbflush,
 373        .mask_memory            = agp_generic_mask_memory,
 374        .masks                  = amd_irongate_masks,
 375        .agp_enable             = agp_generic_enable,
 376        .cache_flush            = global_cache_flush,
 377        .create_gatt_table      = amd_create_gatt_table,
 378        .free_gatt_table        = amd_free_gatt_table,
 379        .insert_memory          = amd_insert_memory,
 380        .remove_memory          = amd_remove_memory,
 381        .alloc_by_type          = agp_generic_alloc_by_type,
 382        .free_by_type           = agp_generic_free_by_type,
 383        .agp_alloc_page         = agp_generic_alloc_page,
 384        .agp_alloc_pages        = agp_generic_alloc_pages,
 385        .agp_destroy_page       = agp_generic_destroy_page,
 386        .agp_destroy_pages      = agp_generic_destroy_pages,
 387        .agp_type_to_mask_type  = agp_generic_type_to_mask_type,
 388};
 389
 390static struct agp_device_ids amd_agp_device_ids[] =
 391{
 392        {
 393                .device_id      = PCI_DEVICE_ID_AMD_FE_GATE_7006,
 394                .chipset_name   = "Irongate",
 395        },
 396        {
 397                .device_id      = PCI_DEVICE_ID_AMD_FE_GATE_700E,
 398                .chipset_name   = "761",
 399        },
 400        {
 401                .device_id      = PCI_DEVICE_ID_AMD_FE_GATE_700C,
 402                .chipset_name   = "760MP",
 403        },
 404        { }, /* dummy final entry, always present */
 405};
 406
 407static int agp_amdk7_probe(struct pci_dev *pdev,
 408                           const struct pci_device_id *ent)
 409{
 410        struct agp_bridge_data *bridge;
 411        u8 cap_ptr;
 412        int j;
 413
 414        cap_ptr = pci_find_capability(pdev, PCI_CAP_ID_AGP);
 415        if (!cap_ptr)
 416                return -ENODEV;
 417
 418        j = ent - agp_amdk7_pci_table;
 419        dev_info(&pdev->dev, "AMD %s chipset\n",
 420                 amd_agp_device_ids[j].chipset_name);
 421
 422        bridge = agp_alloc_bridge();
 423        if (!bridge)
 424                return -ENOMEM;
 425
 426        bridge->driver = &amd_irongate_driver;
 427        bridge->dev_private_data = &amd_irongate_private,
 428        bridge->dev = pdev;
 429        bridge->capndx = cap_ptr;
 430
 431        /* 751 Errata (22564_B-1.PDF)
 432           erratum 20: strobe glitch with Nvidia NV10 GeForce cards.
 433           system controller may experience noise due to strong drive strengths
 434         */
 435        if (agp_bridge->dev->device == PCI_DEVICE_ID_AMD_FE_GATE_7006) {
 436                struct pci_dev *gfxcard=NULL;
 437
 438                cap_ptr = 0;
 439                while (!cap_ptr) {
 440                        gfxcard = pci_get_class(PCI_CLASS_DISPLAY_VGA<<8, gfxcard);
 441                        if (!gfxcard) {
 442                                dev_info(&pdev->dev, "no AGP VGA controller\n");
 443                                return -ENODEV;
 444                        }
 445                        cap_ptr = pci_find_capability(gfxcard, PCI_CAP_ID_AGP);
 446                }
 447
 448                /* With so many variants of NVidia cards, it's simpler just
 449                   to blacklist them all, and then whitelist them as needed
 450                   (if necessary at all). */
 451                if (gfxcard->vendor == PCI_VENDOR_ID_NVIDIA) {
 452                        agp_bridge->flags |= AGP_ERRATA_1X;
 453                        dev_info(&pdev->dev, "AMD 751 chipset with NVidia GeForce; forcing 1X due to errata\n");
 454                }
 455                pci_dev_put(gfxcard);
 456        }
 457
 458        /* 761 Errata (23613_F.pdf)
 459         * Revisions B0/B1 were a disaster.
 460         * erratum 44: SYSCLK/AGPCLK skew causes 2X failures -- Force mode to 1X
 461         * erratum 45: Timing problem prevents fast writes -- Disable fast write.
 462         * erratum 46: Setup violation on AGP SBA pins - Disable side band addressing.
 463         * With this lot disabled, we should prevent lockups. */
 464        if (agp_bridge->dev->device == PCI_DEVICE_ID_AMD_FE_GATE_700E) {
 465                if (pdev->revision == 0x10 || pdev->revision == 0x11) {
 466                        agp_bridge->flags = AGP_ERRATA_FASTWRITES;
 467                        agp_bridge->flags |= AGP_ERRATA_SBA;
 468                        agp_bridge->flags |= AGP_ERRATA_1X;
 469                        dev_info(&pdev->dev, "AMD 761 chipset with errata; disabling AGP fast writes & SBA and forcing to 1X\n");
 470                }
 471        }
 472
 473        /* Fill in the mode register */
 474        pci_read_config_dword(pdev,
 475                        bridge->capndx+PCI_AGP_STATUS,
 476                        &bridge->mode);
 477
 478        pci_set_drvdata(pdev, bridge);
 479        return agp_add_bridge(bridge);
 480}
 481
 482static void agp_amdk7_remove(struct pci_dev *pdev)
 483{
 484        struct agp_bridge_data *bridge = pci_get_drvdata(pdev);
 485
 486        agp_remove_bridge(bridge);
 487        agp_put_bridge(bridge);
 488}
 489
 490#ifdef CONFIG_PM
 491
 492static int agp_amdk7_suspend(struct pci_dev *pdev, pm_message_t state)
 493{
 494        pci_save_state(pdev);
 495        pci_set_power_state(pdev, pci_choose_state(pdev, state));
 496
 497        return 0;
 498}
 499
 500static int agp_amdk7_resume(struct pci_dev *pdev)
 501{
 502        pci_set_power_state(pdev, PCI_D0);
 503        pci_restore_state(pdev);
 504
 505        return amd_irongate_driver.configure();
 506}
 507
 508#endif /* CONFIG_PM */
 509
 510/* must be the same order as name table above */
 511static const struct pci_device_id agp_amdk7_pci_table[] = {
 512        {
 513        .class          = (PCI_CLASS_BRIDGE_HOST << 8),
 514        .class_mask     = ~0,
 515        .vendor         = PCI_VENDOR_ID_AMD,
 516        .device         = PCI_DEVICE_ID_AMD_FE_GATE_7006,
 517        .subvendor      = PCI_ANY_ID,
 518        .subdevice      = PCI_ANY_ID,
 519        },
 520        {
 521        .class          = (PCI_CLASS_BRIDGE_HOST << 8),
 522        .class_mask     = ~0,
 523        .vendor         = PCI_VENDOR_ID_AMD,
 524        .device         = PCI_DEVICE_ID_AMD_FE_GATE_700E,
 525        .subvendor      = PCI_ANY_ID,
 526        .subdevice      = PCI_ANY_ID,
 527        },
 528        {
 529        .class          = (PCI_CLASS_BRIDGE_HOST << 8),
 530        .class_mask     = ~0,
 531        .vendor         = PCI_VENDOR_ID_AMD,
 532        .device         = PCI_DEVICE_ID_AMD_FE_GATE_700C,
 533        .subvendor      = PCI_ANY_ID,
 534        .subdevice      = PCI_ANY_ID,
 535        },
 536        { }
 537};
 538
 539MODULE_DEVICE_TABLE(pci, agp_amdk7_pci_table);
 540
 541static struct pci_driver agp_amdk7_pci_driver = {
 542        .name           = "agpgart-amdk7",
 543        .id_table       = agp_amdk7_pci_table,
 544        .probe          = agp_amdk7_probe,
 545        .remove         = agp_amdk7_remove,
 546#ifdef CONFIG_PM
 547        .suspend        = agp_amdk7_suspend,
 548        .resume         = agp_amdk7_resume,
 549#endif
 550};
 551
 552static int __init agp_amdk7_init(void)
 553{
 554        if (agp_off)
 555                return -EINVAL;
 556        return pci_register_driver(&agp_amdk7_pci_driver);
 557}
 558
 559static void __exit agp_amdk7_cleanup(void)
 560{
 561        pci_unregister_driver(&agp_amdk7_pci_driver);
 562}
 563
 564module_init(agp_amdk7_init);
 565module_exit(agp_amdk7_cleanup);
 566
 567MODULE_LICENSE("GPL and additional rights");
 568