LXR linux/drivers/pci/pcie/aer/aerdrv

   1/*
   2 * drivers/pci/pcie/aer/aerdrv_core.c
   3 *
   4 * This file is subject to the terms and conditions of the GNU General Public
   5 * License.  See the file "COPYING" in the main directory of this archive
   6 * for more details.
   7 *
   8 * This file implements the core part of PCI-Express AER. When an pci-express
   9 * error is delivered, an error message will be collected and printed to
  10 * console, then, an error recovery procedure will be executed by following
  11 * the pci error recovery rules.
  12 *
  13 * Copyright (C) 2006 Intel Corp.
  14 *      Tom Long Nguyen (tom.l.nguyen@intel.com)
  15 *      Zhang Yanmin (yanmin.zhang@intel.com)
  16 *
  17 */
  18
  19#include <linux/module.h>
  20#include <linux/pci.h>
  21#include <linux/kernel.h>
  22#include <linux/errno.h>
  23#include <linux/pm.h>
  24#include <linux/suspend.h>
  25#include <linux/delay.h>
  26#include "aerdrv.h"
  27
  28static int forceload;
  29static int nosourceid;
  30module_param(forceload, bool, 0);
  31module_param(nosourceid, bool, 0);
  32
  33int pci_enable_pcie_error_reporting(struct pci_dev *dev)
  34{
  35        u16 reg16 = 0;
  36        int pos;
  37
  38        pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ERR);
  39        if (!pos)
  40                return -EIO;
  41
  42        pos = pci_find_capability(dev, PCI_CAP_ID_EXP);
  43        if (!pos)
  44                return -EIO;
  45
  46        pci_read_config_word(dev, pos+PCI_EXP_DEVCTL, &reg16);
  47        reg16 = reg16 |
  48                PCI_EXP_DEVCTL_CERE |
  49                PCI_EXP_DEVCTL_NFERE |
  50                PCI_EXP_DEVCTL_FERE |
  51                PCI_EXP_DEVCTL_URRE;
  52        pci_write_config_word(dev, pos+PCI_EXP_DEVCTL, reg16);
  53
  54        return 0;
  55}
  56EXPORT_SYMBOL_GPL(pci_enable_pcie_error_reporting);
  57
  58int pci_disable_pcie_error_reporting(struct pci_dev *dev)
  59{
  60        u16 reg16 = 0;
  61        int pos;
  62
  63        pos = pci_find_capability(dev, PCI_CAP_ID_EXP);
  64        if (!pos)
  65                return -EIO;
  66
  67        pci_read_config_word(dev, pos+PCI_EXP_DEVCTL, &reg16);
  68        reg16 = reg16 & ~(PCI_EXP_DEVCTL_CERE |
  69                        PCI_EXP_DEVCTL_NFERE |
  70                        PCI_EXP_DEVCTL_FERE |
  71                        PCI_EXP_DEVCTL_URRE);
  72        pci_write_config_word(dev, pos+PCI_EXP_DEVCTL, reg16);
  73
  74        return 0;
  75}
  76EXPORT_SYMBOL_GPL(pci_disable_pcie_error_reporting);
  77
  78int pci_cleanup_aer_uncorrect_error_status(struct pci_dev *dev)
  79{
  80        int pos;
  81        u32 status, mask;
  82
  83        pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ERR);
  84        if (!pos)
  85                return -EIO;
  86
  87        pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS, &status);
  88        pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_SEVER, &mask);
  89        if (dev->error_state == pci_channel_io_normal)
  90                status &= ~mask; /* Clear corresponding nonfatal bits */
  91        else
  92                status &= mask; /* Clear corresponding fatal bits */
  93        pci_write_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS, status);
  94
  95        return 0;
  96}
  97EXPORT_SYMBOL_GPL(pci_cleanup_aer_uncorrect_error_status);
  98
  99#if 0
 100int pci_cleanup_aer_correct_error_status(struct pci_dev *dev)
 101{
 102        int pos;
 103        u32 status;
 104
 105        pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ERR);
 106        if (!pos)
 107                return -EIO;
 108
 109        pci_read_config_dword(dev, pos + PCI_ERR_COR_STATUS, &status);
 110        pci_write_config_dword(dev, pos + PCI_ERR_COR_STATUS, status);
 111
 112        return 0;
 113}
 114#endif  /*  0  */
 115
 116static int set_device_error_reporting(struct pci_dev *dev, void *data)
 117{
 118        bool enable = *((bool *)data);
 119
 120        if (dev->pcie_type == PCIE_RC_PORT ||
 121            dev->pcie_type == PCIE_SW_UPSTREAM_PORT ||
 122            dev->pcie_type == PCIE_SW_DOWNSTREAM_PORT) {
 123                if (enable)
 124                        pci_enable_pcie_error_reporting(dev);
 125                else
 126                        pci_disable_pcie_error_reporting(dev);
 127        }
 128
 129        if (enable)
 130                pcie_set_ecrc_checking(dev);
 131
 132        return 0;
 133}
 134
 135/**
 136 * set_downstream_devices_error_reporting - enable/disable the error reporting  bits on the root port and its downstream ports.
 137 * @dev: pointer to root port's pci_dev data structure
 138 * @enable: true = enable error reporting, false = disable error reporting.
 139 */
 140static void set_downstream_devices_error_reporting(struct pci_dev *dev,
 141                                                   bool enable)
 142{
 143        set_device_error_reporting(dev, &enable);
 144
 145        if (!dev->subordinate)
 146                return;
 147        pci_walk_bus(dev->subordinate, set_device_error_reporting, &enable);
 148}
 149
 150static inline int compare_device_id(struct pci_dev *dev,
 151                        struct aer_err_info *e_info)
 152{
 153        if (e_info->id == ((dev->bus->number << 8) | dev->devfn)) {
 154                /*
 155                 * Device ID match
 156                 */
 157                return 1;
 158        }
 159
 160        return 0;
 161}
 162
 163static int add_error_device(struct aer_err_info *e_info, struct pci_dev *dev)
 164{
 165        if (e_info->error_dev_num < AER_MAX_MULTI_ERR_DEVICES) {
 166                e_info->dev[e_info->error_dev_num] = dev;
 167                e_info->error_dev_num++;
 168                return 1;
 169        }
 170
 171        return 0;
 172}
 173
 174
 175#define PCI_BUS(x)      (((x) >> 8) & 0xff)
 176
 177static int find_device_iter(struct pci_dev *dev, void *data)
 178{
 179        int pos;
 180        u32 status;
 181        u32 mask;
 182        u16 reg16;
 183        int result;
 184        struct aer_err_info *e_info = (struct aer_err_info *)data;
 185
 186        /*
 187         * When bus id is equal to 0, it might be a bad id
 188         * reported by root port.
 189         */
 190        if (!nosourceid && (PCI_BUS(e_info->id) != 0)) {
 191                result = compare_device_id(dev, e_info);
 192                if (result)
 193                        add_error_device(e_info, dev);
 194
 195                /*
 196                 * If there is no multiple error, we stop
 197                 * or continue based on the id comparing.
 198                 */
 199                if (!e_info->multi_error_valid)
 200                        return result;
 201
 202                /*
 203                 * If there are multiple errors and id does match,
 204                 * We need continue to search other devices under
 205                 * the root port. Return 0 means that.
 206                 */
 207                if (result)
 208                        return 0;
 209        }
 210
 211        /*
 212         * When either
 213         *      1) nosourceid==y;
 214         *      2) bus id is equal to 0. Some ports might lose the bus
 215         *              id of error source id;
 216         *      3) There are multiple errors and prior id comparing fails;
 217         * We check AER status registers to find the initial reporter.
 218         */
 219        if (atomic_read(&dev->enable_cnt) == 0)
 220                return 0;
 221        pos = pci_find_capability(dev, PCI_CAP_ID_EXP);
 222        if (!pos)
 223                return 0;
 224        /* Check if AER is enabled */
 225        pci_read_config_word(dev, pos+PCI_EXP_DEVCTL, &reg16);
 226        if (!(reg16 & (
 227                PCI_EXP_DEVCTL_CERE |
 228                PCI_EXP_DEVCTL_NFERE |
 229                PCI_EXP_DEVCTL_FERE |
 230                PCI_EXP_DEVCTL_URRE)))
 231                return 0;
 232        pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ERR);
 233        if (!pos)
 234                return 0;
 235
 236        status = 0;
 237        mask = 0;
 238        if (e_info->severity == AER_CORRECTABLE) {
 239                pci_read_config_dword(dev, pos + PCI_ERR_COR_STATUS, &status);
 240                pci_read_config_dword(dev, pos + PCI_ERR_COR_MASK, &mask);
 241                if (status & ~mask) {
 242                        add_error_device(e_info, dev);
 243                        goto added;
 244                }
 245        } else {
 246                pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS, &status);
 247                pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_MASK, &mask);
 248                if (status & ~mask) {
 249                        add_error_device(e_info, dev);
 250                        goto added;
 251                }
 252        }
 253
 254        return 0;
 255
 256added:
 257        if (e_info->multi_error_valid)
 258                return 0;
 259        else
 260                return 1;
 261}
 262
 263/**
 264 * find_source_device - search through device hierarchy for source device
 265 * @parent: pointer to Root Port pci_dev data structure
 266 * @err_info: including detailed error information such like id
 267 *
 268 * Invoked when error is detected at the Root Port.
 269 */
 270static void find_source_device(struct pci_dev *parent,
 271                struct aer_err_info *e_info)
 272{
 273        struct pci_dev *dev = parent;
 274        int result;
 275
 276        /* Is Root Port an agent that sends error message? */
 277        result = find_device_iter(dev, e_info);
 278        if (result)
 279                return;
 280
 281        pci_walk_bus(parent->subordinate, find_device_iter, e_info);
 282}
 283
 284static int report_error_detected(struct pci_dev *dev, void *data)
 285{
 286        pci_ers_result_t vote;
 287        struct pci_error_handlers *err_handler;
 288        struct aer_broadcast_data *result_data;
 289        result_data = (struct aer_broadcast_data *) data;
 290
 291        dev->error_state = result_data->state;
 292
 293        if (!dev->driver ||
 294                !dev->driver->err_handler ||
 295                !dev->driver->err_handler->error_detected) {
 296                if (result_data->state == pci_channel_io_frozen &&
 297                        !(dev->hdr_type & PCI_HEADER_TYPE_BRIDGE)) {
 298                        /*
 299                         * In case of fatal recovery, if one of down-
 300                         * stream device has no driver. We might be
 301                         * unable to recover because a later insmod
 302                         * of a driver for this device is unaware of
 303                         * its hw state.
 304                         */
 305                        dev_printk(KERN_DEBUG, &dev->dev, "device has %s\n",
 306                                   dev->driver ?
 307                                   "no AER-aware driver" : "no driver");
 308                }
 309                return 0;
 310        }
 311
 312        err_handler = dev->driver->err_handler;
 313        vote = err_handler->error_detected(dev, result_data->state);
 314        result_data->result = merge_result(result_data->result, vote);
 315        return 0;
 316}
 317
 318static int report_mmio_enabled(struct pci_dev *dev, void *data)
 319{
 320        pci_ers_result_t vote;
 321        struct pci_error_handlers *err_handler;
 322        struct aer_broadcast_data *result_data;
 323        result_data = (struct aer_broadcast_data *) data;
 324
 325        if (!dev->driver ||
 326                !dev->driver->err_handler ||
 327                !dev->driver->err_handler->mmio_enabled)
 328                return 0;
 329
 330        err_handler = dev->driver->err_handler;
 331        vote = err_handler->mmio_enabled(dev);
 332        result_data->result = merge_result(result_data->result, vote);
 333        return 0;
 334}
 335
 336static int report_slot_reset(struct pci_dev *dev, void *data)
 337{
 338        pci_ers_result_t vote;
 339        struct pci_error_handlers *err_handler;
 340        struct aer_broadcast_data *result_data;
 341        result_data = (struct aer_broadcast_data *) data;
 342
 343        if (!dev->driver ||
 344                !dev->driver->err_handler ||
 345                !dev->driver->err_handler->slot_reset)
 346                return 0;
 347
 348        err_handler = dev->driver->err_handler;
 349        vote = err_handler->slot_reset(dev);
 350        result_data->result = merge_result(result_data->result, vote);
 351        return 0;
 352}
 353
 354static int report_resume(struct pci_dev *dev, void *data)
 355{
 356        struct pci_error_handlers *err_handler;
 357
 358        dev->error_state = pci_channel_io_normal;
 359
 360        if (!dev->driver ||
 361                !dev->driver->err_handler ||
 362                !dev->driver->err_handler->resume)
 363                return 0;
 364
 365        err_handler = dev->driver->err_handler;
 366        err_handler->resume(dev);
 367        return 0;
 368}
 369
 370/**
 371 * broadcast_error_message - handle message broadcast to downstream drivers
 372 * @dev: pointer to from where in a hierarchy message is broadcasted down
 373 * @state: error state
 374 * @error_mesg: message to print
 375 * @cb: callback to be broadcasted
 376 *
 377 * Invoked during error recovery process. Once being invoked, the content
 378 * of error severity will be broadcasted to all downstream drivers in a
 379 * hierarchy in question.
 380 */
 381static pci_ers_result_t broadcast_error_message(struct pci_dev *dev,
 382        enum pci_channel_state state,
 383        char *error_mesg,
 384        int (*cb)(struct pci_dev *, void *))
 385{
 386        struct aer_broadcast_data result_data;
 387
 388        dev_printk(KERN_DEBUG, &dev->dev, "broadcast %s message\n", error_mesg);
 389        result_data.state = state;
 390        if (cb == report_error_detected)
 391                result_data.result = PCI_ERS_RESULT_CAN_RECOVER;
 392        else
 393                result_data.result = PCI_ERS_RESULT_RECOVERED;
 394
 395        if (dev->hdr_type & PCI_HEADER_TYPE_BRIDGE) {
 396                /*
 397                 * If the error is reported by a bridge, we think this error
 398                 * is related to the downstream link of the bridge, so we
 399                 * do error recovery on all subordinates of the bridge instead
 400                 * of the bridge and clear the error status of the bridge.
 401                 */
 402                if (cb == report_error_detected)
 403                        dev->error_state = state;
 404                pci_walk_bus(dev->subordinate, cb, &result_data);
 405                if (cb == report_resume) {
 406                        pci_cleanup_aer_uncorrect_error_status(dev);
 407                        dev->error_state = pci_channel_io_normal;
 408                }
 409        } else {
 410                /*
 411                 * If the error is reported by an end point, we think this
 412                 * error is related to the upstream link of the end point.
 413                 */
 414                pci_walk_bus(dev->bus, cb, &result_data);
 415        }
 416
 417        return result_data.result;
 418}
 419
 420struct find_aer_service_data {
 421        struct pcie_port_service_driver *aer_driver;
 422        int is_downstream;
 423};
 424
 425static int find_aer_service_iter(struct device *device, void *data)
 426{
 427        struct device_driver *driver;
 428        struct pcie_port_service_driver *service_driver;
 429        struct find_aer_service_data *result;
 430
 431        result = (struct find_aer_service_data *) data;
 432
 433        if (device->bus == &pcie_port_bus_type) {
 434                struct pcie_port_data *port_data;
 435
 436                port_data = pci_get_drvdata(to_pcie_device(device)->port);
 437                if (port_data->port_type == PCIE_SW_DOWNSTREAM_PORT)
 438                        result->is_downstream = 1;
 439
 440                driver = device->driver;
 441                if (driver) {
 442                        service_driver = to_service_driver(driver);
 443                        if (service_driver->service == PCIE_PORT_SERVICE_AER) {
 444                                result->aer_driver = service_driver;
 445                                return 1;
 446                        }
 447                }
 448        }
 449
 450        return 0;
 451}
 452
 453static void find_aer_service(struct pci_dev *dev,
 454                struct find_aer_service_data *data)
 455{
 456        int retval;
 457        retval = device_for_each_child(&dev->dev, data, find_aer_service_iter);
 458}
 459
 460static pci_ers_result_t reset_link(struct pcie_device *aerdev,
 461                struct pci_dev *dev)
 462{
 463        struct pci_dev *udev;
 464        pci_ers_result_t status;
 465        struct find_aer_service_data data;
 466
 467        if (dev->hdr_type & PCI_HEADER_TYPE_BRIDGE)
 468                udev = dev;
 469        else
 470                udev = dev->bus->self;
 471
 472        data.is_downstream = 0;
 473        data.aer_driver = NULL;
 474        find_aer_service(udev, &data);
 475
 476        /*
 477         * Use the aer driver of the error agent firstly.
 478         * If it hasn't the aer driver, use the root port's
 479         */
 480        if (!data.aer_driver || !data.aer_driver->reset_link) {
 481                if (data.is_downstream &&
 482                        aerdev->device.driver &&
 483                        to_service_driver(aerdev->device.driver)->reset_link) {
 484                        data.aer_driver =
 485                                to_service_driver(aerdev->device.driver);
 486                } else {
 487                        dev_printk(KERN_DEBUG, &dev->dev, "no link-reset "
 488                                   "support\n");
 489                        return PCI_ERS_RESULT_DISCONNECT;
 490                }
 491        }
 492
 493        status = data.aer_driver->reset_link(udev);
 494        if (status != PCI_ERS_RESULT_RECOVERED) {
 495                dev_printk(KERN_DEBUG, &dev->dev, "link reset at upstream "
 496                           "device %s failed\n", pci_name(udev));
 497                return PCI_ERS_RESULT_DISCONNECT;
 498        }
 499
 500        return status;
 501}
 502
 503/**
 504 * do_recovery - handle nonfatal/fatal error recovery process
 505 * @aerdev: pointer to a pcie_device data structure of root port
 506 * @dev: pointer to a pci_dev data structure of agent detecting an error
 507 * @severity: error severity type
 508 *
 509 * Invoked when an error is nonfatal/fatal. Once being invoked, broadcast
 510 * error detected message to all downstream drivers within a hierarchy in
 511 * question and return the returned code.
 512 */
 513static pci_ers_result_t do_recovery(struct pcie_device *aerdev,
 514                struct pci_dev *dev,
 515                int severity)
 516{
 517        pci_ers_result_t status, result = PCI_ERS_RESULT_RECOVERED;
 518        enum pci_channel_state state;
 519
 520        if (severity == AER_FATAL)
 521                state = pci_channel_io_frozen;
 522        else
 523                state = pci_channel_io_normal;
 524
 525        status = broadcast_error_message(dev,
 526                        state,
 527                        "error_detected",
 528                        report_error_detected);
 529
 530        if (severity == AER_FATAL) {
 531                result = reset_link(aerdev, dev);
 532                if (result != PCI_ERS_RESULT_RECOVERED) {
 533                        /* TODO: Should panic here? */
 534                        return result;
 535                }
 536        }
 537
 538        if (status == PCI_ERS_RESULT_CAN_RECOVER)
 539                status = broadcast_error_message(dev,
 540                                state,
 541                                "mmio_enabled",
 542                                report_mmio_enabled);
 543
 544        if (status == PCI_ERS_RESULT_NEED_RESET) {
 545                /*
 546                 * TODO: Should call platform-specific
 547                 * functions to reset slot before calling
 548                 * drivers' slot_reset callbacks?
 549                 */
 550                status = broadcast_error_message(dev,
 551                                state,
 552                                "slot_reset",
 553                                report_slot_reset);
 554        }
 555
 556        if (status == PCI_ERS_RESULT_RECOVERED)
 557                broadcast_error_message(dev,
 558                                state,
 559                                "resume",
 560                                report_resume);
 561
 562        return status;
 563}
 564
 565/**
 566 * handle_error_source - handle logging error into an event log
 567 * @aerdev: pointer to pcie_device data structure of the root port
 568 * @dev: pointer to pci_dev data structure of error source device
 569 * @info: comprehensive error information
 570 *
 571 * Invoked when an error being detected by Root Port.
 572 */
 573static void handle_error_source(struct pcie_device *aerdev,
 574        struct pci_dev *dev,
 575        struct aer_err_info *info)
 576{
 577        pci_ers_result_t status = 0;
 578        int pos;
 579
 580        if (info->severity == AER_CORRECTABLE) {
 581                /*
 582                 * Correctable error does not need software intevention.
 583                 * No need to go through error recovery process.
 584                 */
 585                pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ERR);
 586                if (pos)
 587                        pci_write_config_dword(dev, pos + PCI_ERR_COR_STATUS,
 588                                        info->status);
 589        } else {
 590                status = do_recovery(aerdev, dev, info->severity);
 591                if (status == PCI_ERS_RESULT_RECOVERED) {
 592                        dev_printk(KERN_DEBUG, &dev->dev, "AER driver "
 593                                   "successfully recovered\n");
 594                } else {
 595                        /* TODO: Should kernel panic here? */
 596                        dev_printk(KERN_DEBUG, &dev->dev, "AER driver didn't "
 597                                   "recover\n");
 598                }
 599        }
 600}
 601
 602/**
 603 * aer_enable_rootport - enable Root Port's interrupts when receiving messages
 604 * @rpc: pointer to a Root Port data structure
 605 *
 606 * Invoked when PCIE bus loads AER service driver.
 607 */
 608void aer_enable_rootport(struct aer_rpc *rpc)
 609{
 610        struct pci_dev *pdev = rpc->rpd->port;
 611        int pos, aer_pos;
 612        u16 reg16;
 613        u32 reg32;
 614
 615        pos = pci_find_capability(pdev, PCI_CAP_ID_EXP);
 616        /* Clear PCIE Capability's Device Status */
 617        pci_read_config_word(pdev, pos+PCI_EXP_DEVSTA, &reg16);
 618        pci_write_config_word(pdev, pos+PCI_EXP_DEVSTA, reg16);
 619
 620        /* Disable system error generation in response to error messages */
 621        pci_read_config_word(pdev, pos + PCI_EXP_RTCTL, &reg16);
 622        reg16 &= ~(SYSTEM_ERROR_INTR_ON_MESG_MASK);
 623        pci_write_config_word(pdev, pos + PCI_EXP_RTCTL, reg16);
 624
 625        aer_pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ERR);
 626        /* Clear error status */
 627        pci_read_config_dword(pdev, aer_pos + PCI_ERR_ROOT_STATUS, &reg32);
 628        pci_write_config_dword(pdev, aer_pos + PCI_ERR_ROOT_STATUS, reg32);
 629        pci_read_config_dword(pdev, aer_pos + PCI_ERR_COR_STATUS, &reg32);
 630        pci_write_config_dword(pdev, aer_pos + PCI_ERR_COR_STATUS, reg32);
 631        pci_read_config_dword(pdev, aer_pos + PCI_ERR_UNCOR_STATUS, &reg32);
 632        pci_write_config_dword(pdev, aer_pos + PCI_ERR_UNCOR_STATUS, reg32);
 633
 634        /*
 635         * Enable error reporting for the root port device and downstream port
 636         * devices.
 637         */
 638        set_downstream_devices_error_reporting(pdev, true);
 639
 640        /* Enable Root Port's interrupt in response to error messages */
 641        pci_write_config_dword(pdev,
 642                aer_pos + PCI_ERR_ROOT_COMMAND,
 643                ROOT_PORT_INTR_ON_MESG_MASK);
 644}
 645
 646/**
 647 * disable_root_aer - disable Root Port's interrupts when receiving messages
 648 * @rpc: pointer to a Root Port data structure
 649 *
 650 * Invoked when PCIE bus unloads AER service driver.
 651 */
 652static void disable_root_aer(struct aer_rpc *rpc)
 653{
 654        struct pci_dev *pdev = rpc->rpd->port;
 655        u32 reg32;
 656        int pos;
 657
 658        /*
 659         * Disable error reporting for the root port device and downstream port
 660         * devices.
 661         */
 662        set_downstream_devices_error_reporting(pdev, false);
 663
 664        pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ERR);
 665        /* Disable Root's interrupt in response to error messages */
 666        pci_write_config_dword(pdev, pos + PCI_ERR_ROOT_COMMAND, 0);
 667
 668        /* Clear Root's error status reg */
 669        pci_read_config_dword(pdev, pos + PCI_ERR_ROOT_STATUS, &reg32);
 670        pci_write_config_dword(pdev, pos + PCI_ERR_ROOT_STATUS, reg32);
 671}
 672
 673/**
 674 * get_e_source - retrieve an error source
 675 * @rpc: pointer to the root port which holds an error
 676 *
 677 * Invoked by DPC handler to consume an error.
 678 */
 679static struct aer_err_source *get_e_source(struct aer_rpc *rpc)
 680{
 681        struct aer_err_source *e_source;
 682        unsigned long flags;
 683
 684        /* Lock access to Root error producer/consumer index */
 685        spin_lock_irqsave(&rpc->e_lock, flags);
 686        if (rpc->prod_idx == rpc->cons_idx) {
 687                spin_unlock_irqrestore(&rpc->e_lock, flags);
 688                return NULL;
 689        }
 690        e_source = &rpc->e_sources[rpc->cons_idx];
 691        rpc->cons_idx++;
 692        if (rpc->cons_idx == AER_ERROR_SOURCES_MAX)
 693                rpc->cons_idx = 0;
 694        spin_unlock_irqrestore(&rpc->e_lock, flags);
 695
 696        return e_source;
 697}
 698
 699/**
 700 * get_device_error_info - read error status from dev and store it to info
 701 * @dev: pointer to the device expected to have a error record
 702 * @info: pointer to structure to store the error record
 703 *
 704 * Return 1 on success, 0 on error.
 705 */
 706static int get_device_error_info(struct pci_dev *dev, struct aer_err_info *info)
 707{
 708        int pos, temp;
 709
 710        info->status = 0;
 711        info->tlp_header_valid = 0;
 712
 713        pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ERR);
 714
 715        /* The device might not support AER */
 716        if (!pos)
 717                return 1;
 718
 719        if (info->severity == AER_CORRECTABLE) {
 720                pci_read_config_dword(dev, pos + PCI_ERR_COR_STATUS,
 721                        &info->status);
 722                pci_read_config_dword(dev, pos + PCI_ERR_COR_MASK,
 723                        &info->mask);
 724                if (!(info->status & ~info->mask))
 725                        return 0;
 726        } else if (dev->hdr_type & PCI_HEADER_TYPE_BRIDGE ||
 727                info->severity == AER_NONFATAL) {
 728
 729                /* Link is still healthy for IO reads */
 730                pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS,
 731                        &info->status);
 732                pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_MASK,
 733                        &info->mask);
 734                if (!(info->status & ~info->mask))
 735                        return 0;
 736
 737                /* Get First Error Pointer */
 738                pci_read_config_dword(dev, pos + PCI_ERR_CAP, &temp);
 739                info->first_error = PCI_ERR_CAP_FEP(temp);
 740
 741                if (info->status & AER_LOG_TLP_MASKS) {
 742                        info->tlp_header_valid = 1;
 743                        pci_read_config_dword(dev,
 744                                pos + PCI_ERR_HEADER_LOG, &info->tlp.dw0);
 745                        pci_read_config_dword(dev,
 746                                pos + PCI_ERR_HEADER_LOG + 4, &info->tlp.dw1);
 747                        pci_read_config_dword(dev,
 748                                pos + PCI_ERR_HEADER_LOG + 8, &info->tlp.dw2);
 749                        pci_read_config_dword(dev,
 750                                pos + PCI_ERR_HEADER_LOG + 12, &info->tlp.dw3);
 751                }
 752        }
 753
 754        return 1;
 755}
 756
 757static inline void aer_process_err_devices(struct pcie_device *p_device,
 758                        struct aer_err_info *e_info)
 759{
 760        int i;
 761
 762        if (!e_info->dev[0]) {
 763                dev_printk(KERN_DEBUG, &p_device->port->dev,
 764                                "can't find device of ID%04x\n",
 765                                e_info->id);
 766        }
 767
 768        /* Report all before handle them, not to lost records by reset etc. */
 769        for (i = 0; i < e_info->error_dev_num && e_info->dev[i]; i++) {
 770                if (get_device_error_info(e_info->dev[i], e_info))
 771                        aer_print_error(e_info->dev[i], e_info);
 772        }
 773        for (i = 0; i < e_info->error_dev_num && e_info->dev[i]; i++) {
 774                if (get_device_error_info(e_info->dev[i], e_info))
 775                        handle_error_source(p_device, e_info->dev[i], e_info);
 776        }
 777}
 778
 779/**
 780 * aer_isr_one_error - consume an error detected by root port
 781 * @p_device: pointer to error root port service device
 782 * @e_src: pointer to an error source
 783 */
 784static void aer_isr_one_error(struct pcie_device *p_device,
 785                struct aer_err_source *e_src)
 786{
 787        struct aer_err_info *e_info;
 788        int i;
 789
 790        /* struct aer_err_info might be big, so we allocate it with slab */
 791        e_info = kmalloc(sizeof(struct aer_err_info), GFP_KERNEL);
 792        if (e_info == NULL) {
 793                dev_printk(KERN_DEBUG, &p_device->port->dev,
 794                        "Can't allocate mem when processing AER errors\n");
 795                return;
 796        }
 797
 798        /*
 799         * There is a possibility that both correctable error and
 800         * uncorrectable error being logged. Report correctable error first.
 801         */
 802        for (i = 1; i & ROOT_ERR_STATUS_MASKS ; i <<= 2) {
 803                if (i > 4)
 804                        break;
 805                if (!(e_src->status & i))
 806                        continue;
 807
 808                memset(e_info, 0, sizeof(struct aer_err_info));
 809
 810                /* Init comprehensive error information */
 811                if (i & PCI_ERR_ROOT_COR_RCV) {
 812                        e_info->id = ERR_COR_ID(e_src->id);
 813                        e_info->severity = AER_CORRECTABLE;
 814                } else {
 815                        e_info->id = ERR_UNCOR_ID(e_src->id);
 816                        e_info->severity = ((e_src->status >> 6) & 1);
 817                }
 818                if (e_src->status &
 819                        (PCI_ERR_ROOT_MULTI_COR_RCV |
 820                         PCI_ERR_ROOT_MULTI_UNCOR_RCV))
 821                        e_info->multi_error_valid = 1;
 822
 823                aer_print_port_info(p_device->port, e_info);
 824
 825                find_source_device(p_device->port, e_info);
 826                aer_process_err_devices(p_device, e_info);
 827        }
 828
 829        kfree(e_info);
 830}
 831
 832/**
 833 * aer_isr - consume errors detected by root port
 834 * @work: definition of this work item
 835 *
 836 * Invoked, as DPC, when root port records new detected error
 837 */
 838void aer_isr(struct work_struct *work)
 839{
 840        struct aer_rpc *rpc = container_of(work, struct aer_rpc, dpc_handler);
 841        struct pcie_device *p_device = rpc->rpd;
 842        struct aer_err_source *e_src;
 843
 844        mutex_lock(&rpc->rpc_mutex);
 845        e_src = get_e_source(rpc);
 846        while (e_src) {
 847                aer_isr_one_error(p_device, e_src);
 848                e_src = get_e_source(rpc);
 849        }
 850        mutex_unlock(&rpc->rpc_mutex);
 851
 852        wake_up(&rpc->wait_release);
 853}
 854
 855/**
 856 * aer_delete_rootport - disable root port aer and delete service data
 857 * @rpc: pointer to a root port device being deleted
 858 *
 859 * Invoked when AER service unloaded on a specific Root Port
 860 */
 861void aer_delete_rootport(struct aer_rpc *rpc)
 862{
 863        /* Disable root port AER itself */
 864        disable_root_aer(rpc);
 865
 866        kfree(rpc);
 867}
 868
 869/**
 870 * aer_init - provide AER initialization
 871 * @dev: pointer to AER pcie device
 872 *
 873 * Invoked when AER service driver is loaded.
 874 */
 875int aer_init(struct pcie_device *dev)
 876{
 877        if (aer_osc_setup(dev) && !forceload)
 878                return -ENXIO;
 879
 880        return 0;
 881}
 882