/*
 * Core routines and tables shareable across OS platforms.
 *
 * Copyright (c) 1994-2002 Justin T. Gibbs.
 * Copyright (c) 2000-2002 Adaptec Inc.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions, and the following disclaimer,
 *    without modification.
 * 2. Redistributions in binary form must reproduce at minimum a disclaimer
 *    substantially similar to the "NO WARRANTY" disclaimer below
 *    ("Disclaimer") and any redistribution must be conditioned upon
 *    including a substantially similar Disclaimer requirement for further
 *    binary redistribution.
 * 3. Neither the names of the above-listed copyright holders nor the names
 *    of any contributors may be used to endorse or promote products derived
 *    from this software without specific prior written permission.
 *
 * Alternatively, this software may be distributed under the terms of the
 * GNU General Public License ("GPL") version 2 as published by the Free
 * Software Foundation.
 *
 * NO WARRANTY
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 * HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
 * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGES.
 *
 * $Id: //depot/aic7xxx/aic7xxx/aic7xxx.c#155 $
 */

#include "aic7xxx_osm.h"
#include "aic7xxx_inline.h"
#include "aicasm/aicasm_insformat.h"

/***************************** Lookup Tables **********************************/
static const char *const ahc_chip_names[] = {
        "NONE",
        "aic7770",
        "aic7850",
        "aic7855",
        "aic7859",
        "aic7860",
        "aic7870",
        "aic7880",
        "aic7895",
        "aic7895C",
        "aic7890/91",
        "aic7896/97",
        "aic7892",
        "aic7899"
};

/*
 * Hardware error codes.
 */
struct ahc_hard_error_entry {
        uint8_t errno;
        const char *errmesg;
};

static const struct ahc_hard_error_entry ahc_hard_errors[] = {
        { ILLHADDR,     "Illegal Host Access" },
        { ILLSADDR,     "Illegal Sequencer Address referenced" },
        { ILLOPCODE,    "Illegal Opcode in sequencer program" },
        { SQPARERR,     "Sequencer Parity Error" },
        { DPARERR,      "Data-path Parity Error" },
        { MPARERR,      "Scratch or SCB Memory Parity Error" },
        { PCIERRSTAT,   "PCI Error detected" },
        { CIOPARERR,    "CIOBUS Parity Error" },
};
static const u_int num_errors = ARRAY_SIZE(ahc_hard_errors);

static const struct ahc_phase_table_entry ahc_phase_table[] =
{
        { P_DATAOUT,    NOP,                    "in Data-out phase"     },
        { P_DATAIN,     INITIATOR_ERROR,        "in Data-in phase"      },
        { P_DATAOUT_DT, NOP,                    "in DT Data-out phase"  },
        { P_DATAIN_DT,  INITIATOR_ERROR,        "in DT Data-in phase"   },
        { P_COMMAND,    NOP,                    "in Command phase"      },
        { P_MESGOUT,    NOP,                    "in Message-out phase"  },
        { P_STATUS,     INITIATOR_ERROR,        "in Status phase"       },
        { P_MESGIN,     MSG_PARITY_ERROR,       "in Message-in phase"   },
        { P_BUSFREE,    NOP,                    "while idle"            },
        { 0,            NOP,                    "in unknown phase"      }
};

/*
 * In most cases we only wish to itterate over real phases, so
 * exclude the last element from the count.
 */
static const u_int num_phases = ARRAY_SIZE(ahc_phase_table) - 1;

/*
 * Valid SCSIRATE values.  (p. 3-17)
 * Provides a mapping of tranfer periods in ns to the proper value to
 * stick in the scsixfer reg.
 */
static const struct ahc_syncrate ahc_syncrates[] =
{
      /* ultra2    fast/ultra  period     rate */
        { 0x42,      0x000,      9,      "80.0" },
        { 0x03,      0x000,     10,      "40.0" },
        { 0x04,      0x000,     11,      "33.0" },
        { 0x05,      0x100,     12,      "20.0" },
        { 0x06,      0x110,     15,      "16.0" },
        { 0x07,      0x120,     18,      "13.4" },
        { 0x08,      0x000,     25,      "10.0" },
        { 0x19,      0x010,     31,      "8.0"  },
        { 0x1a,      0x020,     37,      "6.67" },
        { 0x1b,      0x030,     43,      "5.7"  },
        { 0x1c,      0x040,     50,      "5.0"  },
        { 0x00,      0x050,     56,      "4.4"  },
        { 0x00,      0x060,     62,      "4.0"  },
        { 0x00,      0x070,     68,      "3.6"  },
        { 0x00,      0x000,      0,      NULL   }
};

/* Our Sequencer Program */
#include "aic7xxx_seq.h"

/**************************** Function Declarations ***************************/
static void             ahc_force_renegotiation(struct ahc_softc *ahc,
                                                struct ahc_devinfo *devinfo);
static struct ahc_tmode_tstate*
                        ahc_alloc_tstate(struct ahc_softc *ahc,
                                         u_int scsi_id, char channel);
#ifdef AHC_TARGET_MODE
static void             ahc_free_tstate(struct ahc_softc *ahc,
                                        u_int scsi_id, char channel, int force);
#endif
static const struct ahc_syncrate*
                        ahc_devlimited_syncrate(struct ahc_softc *ahc,
                                                struct ahc_initiator_tinfo *,
                                                u_int *period,
                                                u_int *ppr_options,
                                                role_t role);
static void             ahc_update_pending_scbs(struct ahc_softc *ahc);
static void             ahc_fetch_devinfo(struct ahc_softc *ahc,
                                          struct ahc_devinfo *devinfo);
static void             ahc_scb_devinfo(struct ahc_softc *ahc,
                                        struct ahc_devinfo *devinfo,
                                        struct scb *scb);
static void             ahc_assert_atn(struct ahc_softc *ahc);
static void             ahc_setup_initiator_msgout(struct ahc_softc *ahc,
                                                   struct ahc_devinfo *devinfo,
                                                   struct scb *scb);
static void             ahc_build_transfer_msg(struct ahc_softc *ahc,
                                               struct ahc_devinfo *devinfo);
static void             ahc_construct_sdtr(struct ahc_softc *ahc,
                                           struct ahc_devinfo *devinfo,
                                           u_int period, u_int offset);
static void             ahc_construct_wdtr(struct ahc_softc *ahc,
                                           struct ahc_devinfo *devinfo,
                                           u_int bus_width);
static void             ahc_construct_ppr(struct ahc_softc *ahc,
                                          struct ahc_devinfo *devinfo,
                                          u_int period, u_int offset,
                                          u_int bus_width, u_int ppr_options);
static void             ahc_clear_msg_state(struct ahc_softc *ahc);
static void             ahc_handle_proto_violation(struct ahc_softc *ahc);
static void             ahc_handle_message_phase(struct ahc_softc *ahc);
typedef enum {
        AHCMSG_1B,
        AHCMSG_2B,
        AHCMSG_EXT
} ahc_msgtype;
static int              ahc_sent_msg(struct ahc_softc *ahc, ahc_msgtype type,
                                     u_int msgval, int full);
static int              ahc_parse_msg(struct ahc_softc *ahc,
                                      struct ahc_devinfo *devinfo);
static int              ahc_handle_msg_reject(struct ahc_softc *ahc,
                                              struct ahc_devinfo *devinfo);
static void             ahc_handle_ign_wide_residue(struct ahc_softc *ahc,
                                                struct ahc_devinfo *devinfo);
static void             ahc_reinitialize_dataptrs(struct ahc_softc *ahc);
static void             ahc_handle_devreset(struct ahc_softc *ahc,
                                            struct ahc_devinfo *devinfo,
                                            cam_status status, char *message,
                                            int verbose_level);
#ifdef AHC_TARGET_MODE
static void             ahc_setup_target_msgin(struct ahc_softc *ahc,
                                               struct ahc_devinfo *devinfo,
                                               struct scb *scb);
#endif

static bus_dmamap_callback_t    ahc_dmamap_cb;
static void             ahc_build_free_scb_list(struct ahc_softc *ahc);
static int              ahc_init_scbdata(struct ahc_softc *ahc);
static void             ahc_fini_scbdata(struct ahc_softc *ahc);
static void             ahc_qinfifo_requeue(struct ahc_softc *ahc,
                                            struct scb *prev_scb,
                                            struct scb *scb);
static int              ahc_qinfifo_count(struct ahc_softc *ahc);
static u_int            ahc_rem_scb_from_disc_list(struct ahc_softc *ahc,
                                                   u_int prev, u_int scbptr);
static void             ahc_add_curscb_to_free_list(struct ahc_softc *ahc);
static u_int            ahc_rem_wscb(struct ahc_softc *ahc,
                                     u_int scbpos, u_int prev);
static void             ahc_reset_current_bus(struct ahc_softc *ahc);
#ifdef AHC_DUMP_SEQ
static void             ahc_dumpseq(struct ahc_softc *ahc);
#endif
static int              ahc_loadseq(struct ahc_softc *ahc);
static int              ahc_check_patch(struct ahc_softc *ahc,
                                        const struct patch **start_patch,
                                        u_int start_instr, u_int *skip_addr);
static void             ahc_download_instr(struct ahc_softc *ahc,
                                           u_int instrptr, uint8_t *dconsts);
#ifdef AHC_TARGET_MODE
static void             ahc_queue_lstate_event(struct ahc_softc *ahc,
                                               struct ahc_tmode_lstate *lstate,
                                               u_int initiator_id,
                                               u_int event_type,
                                               u_int event_arg);
static void             ahc_update_scsiid(struct ahc_softc *ahc,
                                          u_int targid_mask);
static int              ahc_handle_target_cmd(struct ahc_softc *ahc,
                                              struct target_cmd *cmd);
#endif

static u_int            ahc_index_busy_tcl(struct ahc_softc *ahc, u_int tcl);
static void             ahc_unbusy_tcl(struct ahc_softc *ahc, u_int tcl);
static void             ahc_busy_tcl(struct ahc_softc *ahc,
                                     u_int tcl, u_int busyid);

/************************** SCB and SCB queue management **********************/
static void             ahc_run_untagged_queues(struct ahc_softc *ahc);
static void             ahc_run_untagged_queue(struct ahc_softc *ahc,
                                               struct scb_tailq *queue);

/****************************** Initialization ********************************/
static void              ahc_alloc_scbs(struct ahc_softc *ahc);
static void              ahc_shutdown(void *arg);

/*************************** Interrupt Services *******************************/
static void             ahc_clear_intstat(struct ahc_softc *ahc);
static void             ahc_run_qoutfifo(struct ahc_softc *ahc);
#ifdef AHC_TARGET_MODE
static void             ahc_run_tqinfifo(struct ahc_softc *ahc, int paused);
#endif
static void             ahc_handle_brkadrint(struct ahc_softc *ahc);
static void             ahc_handle_seqint(struct ahc_softc *ahc, u_int intstat);
static void             ahc_handle_scsiint(struct ahc_softc *ahc,
                                           u_int intstat);
static void             ahc_clear_critical_section(struct ahc_softc *ahc);

/***************************** Error Recovery *********************************/
static void             ahc_freeze_devq(struct ahc_softc *ahc, struct scb *scb);
static int              ahc_abort_scbs(struct ahc_softc *ahc, int target,
                                       char channel, int lun, u_int tag,
                                       role_t role, uint32_t status);
static void             ahc_calc_residual(struct ahc_softc *ahc,
                                          struct scb *scb);

/*********************** Untagged Transaction Routines ************************/
static inline void      ahc_freeze_untagged_queues(struct ahc_softc *ahc);
static inline void      ahc_release_untagged_queues(struct ahc_softc *ahc);

/*
 * Block our completion routine from starting the next untagged
 * transaction for this target or target lun.
 */
static inline void
ahc_freeze_untagged_queues(struct ahc_softc *ahc)
{
        if ((ahc->flags & AHC_SCB_BTT) == 0)
                ahc->untagged_queue_lock++;
}

/*
 * Allow the next untagged transaction for this target or target lun
 * to be executed.  We use a counting semaphore to allow the lock
 * to be acquired recursively.  Once the count drops to zero, the
 * transaction queues will be run.
 */
static inline void
ahc_release_untagged_queues(struct ahc_softc *ahc)
{
        if ((ahc->flags & AHC_SCB_BTT) == 0) {
                ahc->untagged_queue_lock--;
                if (ahc->untagged_queue_lock == 0)
                        ahc_run_untagged_queues(ahc);
        }
}

/************************* Sequencer Execution Control ************************/
/*
 * Work around any chip bugs related to halting sequencer execution.
 * On Ultra2 controllers, we must clear the CIOBUS stretch signal by
 * reading a register that will set this signal and deassert it.
 * Without this workaround, if the chip is paused, by an interrupt or
 * manual pause while accessing scb ram, accesses to certain registers
 * will hang the system (infinite pci retries).
 */
static void
ahc_pause_bug_fix(struct ahc_softc *ahc)
{
        if ((ahc->features & AHC_ULTRA2) != 0)
                (void)ahc_inb(ahc, CCSCBCTL);
}

/*
 * Determine whether the sequencer has halted code execution.
 * Returns non-zero status if the sequencer is stopped.
 */
int
ahc_is_paused(struct ahc_softc *ahc)
{
        return ((ahc_inb(ahc, HCNTRL) & PAUSE) != 0);
}

/*
 * Request that the sequencer stop and wait, indefinitely, for it
 * to stop.  The sequencer will only acknowledge that it is paused
 * once it has reached an instruction boundary and PAUSEDIS is
 * cleared in the SEQCTL register.  The sequencer may use PAUSEDIS
 * for critical sections.
 */
void
ahc_pause(struct ahc_softc *ahc)
{
        ahc_outb(ahc, HCNTRL, ahc->pause);

        /*
         * Since the sequencer can disable pausing in a critical section, we
         * must loop until it actually stops.
         */
        while (ahc_is_paused(ahc) == 0)
                ;

        ahc_pause_bug_fix(ahc);
}

/*
 * Allow the sequencer to continue program execution.
 * We check here to ensure that no additional interrupt
 * sources that would cause the sequencer to halt have been
 * asserted.  If, for example, a SCSI bus reset is detected
 * while we are fielding a different, pausing, interrupt type,
 * we don't want to release the sequencer before going back
 * into our interrupt handler and dealing with this new
 * condition.
 */
void
ahc_unpause(struct ahc_softc *ahc)
{
        if ((ahc_inb(ahc, INTSTAT) & (SCSIINT | SEQINT | BRKADRINT)) == 0)
                ahc_outb(ahc, HCNTRL, ahc->unpause);
}

/************************** Memory mapping routines ***************************/
static struct ahc_dma_seg *
ahc_sg_bus_to_virt(struct scb *scb, uint32_t sg_busaddr)
{
        int sg_index;

        sg_index = (sg_busaddr - scb->sg_list_phys)/sizeof(struct ahc_dma_seg);
        /* sg_list_phys points to entry 1, not 0 */
        sg_index++;

        return (&scb->sg_list[sg_index]);
}

static uint32_t
ahc_sg_virt_to_bus(struct scb *scb, struct ahc_dma_seg *sg)
{
        int sg_index;

        /* sg_list_phys points to entry 1, not 0 */
        sg_index = sg - &scb->sg_list[1];

        return (scb->sg_list_phys + (sg_index * sizeof(*scb->sg_list)));
}

static uint32_t
ahc_hscb_busaddr(struct ahc_softc *ahc, u_int index)
{
        return (ahc->scb_data->hscb_busaddr
                + (sizeof(struct hardware_scb) * index));
}

static void
ahc_sync_scb(struct ahc_softc *ahc, struct scb *scb, int op)
{
        ahc_dmamap_sync(ahc, ahc->scb_data->hscb_dmat,
                        ahc->scb_data->hscb_dmamap,
                        /*offset*/(scb->hscb - ahc->hscbs) * sizeof(*scb->hscb),
                        /*len*/sizeof(*scb->hscb), op);
}

void
ahc_sync_sglist(struct ahc_softc *ahc, struct scb *scb, int op)
{
        if (scb->sg_count == 0)
                return;

        ahc_dmamap_sync(ahc, ahc->scb_data->sg_dmat, scb->sg_map->sg_dmamap,
                        /*offset*/(scb->sg_list - scb->sg_map->sg_vaddr)
                                * sizeof(struct ahc_dma_seg),
                        /*len*/sizeof(struct ahc_dma_seg) * scb->sg_count, op);
}

#ifdef AHC_TARGET_MODE
static uint32_t
ahc_targetcmd_offset(struct ahc_softc *ahc, u_int index)
{
        return (((uint8_t *)&ahc->targetcmds[index]) - ahc->qoutfifo);
}
#endif

/*********************** Miscellaneous Support Functions ***********************/
/*
 * Determine whether the sequencer reported a residual
 * for this SCB/transaction.
 */
static void
ahc_update_residual(struct ahc_softc *ahc, struct scb *scb)
{
        uint32_t sgptr;

        sgptr = ahc_le32toh(scb->hscb->sgptr);
        if ((sgptr & SG_RESID_VALID) != 0)
                ahc_calc_residual(ahc, scb);
}

/*
 * Return pointers to the transfer negotiation information
 * for the specified our_id/remote_id pair.
 */
struct ahc_initiator_tinfo *
ahc_fetch_transinfo(struct ahc_softc *ahc, char channel, u_int our_id,
                    u_int remote_id, struct ahc_tmode_tstate **tstate)
{
        /*
         * Transfer data structures are stored from the perspective
         * of the target role.  Since the parameters for a connection
         * in the initiator role to a given target are the same as
         * when the roles are reversed, we pretend we are the target.
         */
        if (channel == 'B')
                our_id += 8;
        *tstate = ahc->enabled_targets[our_id];
        return (&(*tstate)->transinfo[remote_id]);
}

uint16_t
ahc_inw(struct ahc_softc *ahc, u_int port)
{
        uint16_t r = ahc_inb(ahc, port+1) << 8;
        return r | ahc_inb(ahc, port);
}

void
ahc_outw(struct ahc_softc *ahc, u_int port, u_int value)
{
        ahc_outb(ahc, port, value & 0xFF);
        ahc_outb(ahc, port+1, (value >> 8) & 0xFF);
}

uint32_t
ahc_inl(struct ahc_softc *ahc, u_int port)
{
        return ((ahc_inb(ahc, port))
              | (ahc_inb(ahc, port+1) << 8)
              | (ahc_inb(ahc, port+2) << 16)
              | (ahc_inb(ahc, port+3) << 24));
}

void
ahc_outl(struct ahc_softc *ahc, u_int port, uint32_t value)
{
        ahc_outb(ahc, port, (value) & 0xFF);
        ahc_outb(ahc, port+1, ((value) >> 8) & 0xFF);
        ahc_outb(ahc, port+2, ((value) >> 16) & 0xFF);
        ahc_outb(ahc, port+3, ((value) >> 24) & 0xFF);
}

uint64_t
ahc_inq(struct ahc_softc *ahc, u_int port)
{
        return ((ahc_inb(ahc, port))
              | (ahc_inb(ahc, port+1) << 8)
              | (ahc_inb(ahc, port+2) << 16)
              | (((uint64_t)ahc_inb(ahc, port+3)) << 24)
              | (((uint64_t)ahc_inb(ahc, port+4)) << 32)
              | (((uint64_t)ahc_inb(ahc, port+5)) << 40)
              | (((uint64_t)ahc_inb(ahc, port+6)) << 48)
              | (((uint64_t)ahc_inb(ahc, port+7)) << 56));
}

void
ahc_outq(struct ahc_softc *ahc, u_int port, uint64_t value)
{
        ahc_outb(ahc, port, value & 0xFF);
        ahc_outb(ahc, port+1, (value >> 8) & 0xFF);
        ahc_outb(ahc, port+2, (value >> 16) & 0xFF);
        ahc_outb(ahc, port+3, (value >> 24) & 0xFF);
        ahc_outb(ahc, port+4, (value >> 32) & 0xFF);
        ahc_outb(ahc, port+5, (value >> 40) & 0xFF);
        ahc_outb(ahc, port+6, (value >> 48) & 0xFF);
        ahc_outb(ahc, port+7, (value >> 56) & 0xFF);
}

/*
 * Get a free scb. If there are none, see if we can allocate a new SCB.
 */
struct scb *
ahc_get_scb(struct ahc_softc *ahc)
{
        struct scb *scb;

        if ((scb = SLIST_FIRST(&ahc->scb_data->free_scbs)) == NULL) {
                ahc_alloc_scbs(ahc);
                scb = SLIST_FIRST(&ahc->scb_data->free_scbs);
                if (scb == NULL)
                        return (NULL);
        }
        SLIST_REMOVE_HEAD(&ahc->scb_data->free_scbs, links.sle);
        return (scb);
}

/*
 * Return an SCB resource to the free list.
 */
void
ahc_free_scb(struct ahc_softc *ahc, struct scb *scb)
{
        struct hardware_scb *hscb;

        hscb = scb->hscb;
        /* Clean up for the next user */
        ahc->scb_data->scbindex[hscb->tag] = NULL;
        scb->flags = SCB_FREE;
        hscb->control = 0;

        SLIST_INSERT_HEAD(&ahc->scb_data->free_scbs, scb, links.sle);

        /* Notify the OSM that a resource is now available. */
        ahc_platform_scb_free(ahc, scb);
}

struct scb *
ahc_lookup_scb(struct ahc_softc *ahc, u_int tag)
{
        struct scb* scb;

        scb = ahc->scb_data->scbindex[tag];
        if (scb != NULL)
                ahc_sync_scb(ahc, scb,
                             BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE);
        return (scb);
}

static void
ahc_swap_with_next_hscb(struct ahc_softc *ahc, struct scb *scb)
{
        struct hardware_scb *q_hscb;
        u_int  saved_tag;

        /*
         * Our queuing method is a bit tricky.  The card
         * knows in advance which HSCB to download, and we
         * can't disappoint it.  To achieve this, the next
         * SCB to download is saved off in ahc->next_queued_scb.
         * When we are called to queue "an arbitrary scb",
         * we copy the contents of the incoming HSCB to the one
         * the sequencer knows about, swap HSCB pointers and
         * finally assign the SCB to the tag indexed location
         * in the scb_array.  This makes sure that we can still
         * locate the correct SCB by SCB_TAG.
         */
        q_hscb = ahc->next_queued_scb->hscb;
        saved_tag = q_hscb->tag;
        memcpy(q_hscb, scb->hscb, sizeof(*scb->hscb));
        if ((scb->flags & SCB_CDB32_PTR) != 0) {
                q_hscb->shared_data.cdb_ptr =
                    ahc_htole32(ahc_hscb_busaddr(ahc, q_hscb->tag)
                              + offsetof(struct hardware_scb, cdb32));
        }
        q_hscb->tag = saved_tag;
        q_hscb->next = scb->hscb->tag;

        /* Now swap HSCB pointers. */
        ahc->next_queued_scb->hscb = scb->hscb;
        scb->hscb = q_hscb;

        /* Now define the mapping from tag to SCB in the scbindex */
        ahc->scb_data->scbindex[scb->hscb->tag] = scb;
}

/*
 * Tell the sequencer about a new transaction to execute.
 */
void
ahc_queue_scb(struct ahc_softc *ahc, struct scb *scb)
{
        ahc_swap_with_next_hscb(ahc, scb);

        if (scb->hscb->tag == SCB_LIST_NULL
         || scb->hscb->next == SCB_LIST_NULL)
                panic("Attempt to queue invalid SCB tag %x:%x\n",
                      scb->hscb->tag, scb->hscb->next);

        /*
         * Setup data "oddness".
         */
        scb->hscb->lun &= LID;
        if (ahc_get_transfer_length(scb) & 0x1)
                scb->hscb->lun |= SCB_XFERLEN_ODD;

        /*
         * Keep a history of SCBs we've downloaded in the qinfifo.
         */
        ahc->qinfifo[ahc->qinfifonext++] = scb->hscb->tag;

        /*
         * Make sure our data is consistent from the
         * perspective of the adapter.
         */
        ahc_sync_scb(ahc, scb, BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);

        /* Tell the adapter about the newly queued SCB */
        if ((ahc->features & AHC_QUEUE_REGS) != 0) {
                ahc_outb(ahc, HNSCB_QOFF, ahc->qinfifonext);
        } else {
                if ((ahc->features & AHC_AUTOPAUSE) == 0)
                        ahc_pause(ahc);
                ahc_outb(ahc, KERNEL_QINPOS, ahc->qinfifonext);
                if ((ahc->features & AHC_AUTOPAUSE) == 0)
                        ahc_unpause(ahc);
        }
}

struct scsi_sense_data *
ahc_get_sense_buf(struct ahc_softc *ahc, struct scb *scb)
{
        int offset;

        offset = scb - ahc->scb_data->scbarray;
        return (&ahc->scb_data->sense[offset]);
}

static uint32_t
ahc_get_sense_bufaddr(struct ahc_softc *ahc, struct scb *scb)
{
        int offset;

        offset = scb - ahc->scb_data->scbarray;
        return (ahc->scb_data->sense_busaddr
              + (offset * sizeof(struct scsi_sense_data)));
}

/************************** Interrupt Processing ******************************/
static void
ahc_sync_qoutfifo(struct ahc_softc *ahc, int op)
{
        ahc_dmamap_sync(ahc, ahc->shared_data_dmat, ahc->shared_data_dmamap,
                        /*offset*/0, /*len*/256, op);
}

static void
ahc_sync_tqinfifo(struct ahc_softc *ahc, int op)
{
#ifdef AHC_TARGET_MODE
        if ((ahc->flags & AHC_TARGETROLE) != 0) {
                ahc_dmamap_sync(ahc, ahc->shared_data_dmat,
                                ahc->shared_data_dmamap,
                                ahc_targetcmd_offset(ahc, 0),
                                sizeof(struct target_cmd) * AHC_TMODE_CMDS,
                                op);
        }
#endif
}

/*
 * See if the firmware has posted any completed commands
 * into our in-core command complete fifos.
 */
#define AHC_RUN_QOUTFIFO 0x1
#define AHC_RUN_TQINFIFO 0x2
static u_int
ahc_check_cmdcmpltqueues(struct ahc_softc *ahc)
{
        u_int retval;

        retval = 0;
        ahc_dmamap_sync(ahc, ahc->shared_data_dmat, ahc->shared_data_dmamap,
                        /*offset*/ahc->qoutfifonext, /*len*/1,
                        BUS_DMASYNC_POSTREAD);
        if (ahc->qoutfifo[ahc->qoutfifonext] != SCB_LIST_NULL)
                retval |= AHC_RUN_QOUTFIFO;
#ifdef AHC_TARGET_MODE
        if ((ahc->flags & AHC_TARGETROLE) != 0
         && (ahc->flags & AHC_TQINFIFO_BLOCKED) == 0) {
                ahc_dmamap_sync(ahc, ahc->shared_data_dmat,
                                ahc->shared_data_dmamap,
                                ahc_targetcmd_offset(ahc, ahc->tqinfifofnext),
                                /*len*/sizeof(struct target_cmd),
                                BUS_DMASYNC_POSTREAD);
                if (ahc->targetcmds[ahc->tqinfifonext].cmd_valid != 0)
                        retval |= AHC_RUN_TQINFIFO;
        }
#endif
        return (retval);
}

/*
 * Catch an interrupt from the adapter
 */
int
ahc_intr(struct ahc_softc *ahc)
{
        u_int   intstat;

        if ((ahc->pause & INTEN) == 0) {
                /*
                 * Our interrupt is not enabled on the chip
                 * and may be disabled for re-entrancy reasons,
                 * so just return.  This is likely just a shared
                 * interrupt.
                 */
                return (0);
        }
        /*
         * Instead of directly reading the interrupt status register,
         * infer the cause of the interrupt by checking our in-core
         * completion queues.  This avoids a costly PCI bus read in
         * most cases.
         */
        if ((ahc->flags & (AHC_ALL_INTERRUPTS|AHC_EDGE_INTERRUPT)) == 0
         && (ahc_check_cmdcmpltqueues(ahc) != 0))
                intstat = CMDCMPLT;
        else {
                intstat = ahc_inb(ahc, INTSTAT);
        }

        if ((intstat & INT_PEND) == 0) {
#if AHC_PCI_CONFIG > 0
                if (ahc->unsolicited_ints > 500) {
                        ahc->unsolicited_ints = 0;
                        if ((ahc->chip & AHC_PCI) != 0
                         && (ahc_inb(ahc, ERROR) & PCIERRSTAT) != 0)
                                ahc->bus_intr(ahc);
                }
#endif
                ahc->unsolicited_ints++;
                return (0);
        }
        ahc->unsolicited_ints = 0;

        if (intstat & CMDCMPLT) {
                ahc_outb(ahc, CLRINT, CLRCMDINT);

                /*
                 * Ensure that the chip sees that we've cleared
                 * this interrupt before we walk the output fifo.
                 * Otherwise, we may, due to posted bus writes,
                 * clear the interrupt after we finish the scan,
                 * and after the sequencer has added new entries
                 * and asserted the interrupt again.
                 */
                ahc_flush_device_writes(ahc);
                ahc_run_qoutfifo(ahc);
#ifdef AHC_TARGET_MODE
                if ((ahc->flags & AHC_TARGETROLE) != 0)
                        ahc_run_tqinfifo(ahc, /*paused*/FALSE);
#endif
        }

        /*
         * Handle statuses that may invalidate our cached
         * copy of INTSTAT separately.
         */
        if (intstat == 0xFF && (ahc->features & AHC_REMOVABLE) != 0) {
                /* Hot eject.  Do nothing */
        } else if (intstat & BRKADRINT) {
                ahc_handle_brkadrint(ahc);
        } else if ((intstat & (SEQINT|SCSIINT)) != 0) {

                ahc_pause_bug_fix(ahc);

                if ((intstat & SEQINT) != 0)
                        ahc_handle_seqint(ahc, intstat);

                if ((intstat & SCSIINT) != 0)
                        ahc_handle_scsiint(ahc, intstat);
        }
        return (1);
}

/************************* Sequencer Execution Control ************************/
/*
 * Restart the sequencer program from address zero
 */
static void
ahc_restart(struct ahc_softc *ahc)
{
        uint8_t sblkctl;

        ahc_pause(ahc);

        /* No more pending messages. */
        ahc_clear_msg_state(ahc);

        ahc_outb(ahc, SCSISIGO, 0);             /* De-assert BSY */
        ahc_outb(ahc, MSG_OUT, NOP);    /* No message to send */
        ahc_outb(ahc, SXFRCTL1, ahc_inb(ahc, SXFRCTL1) & ~BITBUCKET);
        ahc_outb(ahc, LASTPHASE, P_BUSFREE);
        ahc_outb(ahc, SAVED_SCSIID, 0xFF);
        ahc_outb(ahc, SAVED_LUN, 0xFF);

        /*
         * Ensure that the sequencer's idea of TQINPOS
         * matches our own.  The sequencer increments TQINPOS
         * only after it sees a DMA complete and a reset could
         * occur before the increment leaving the kernel to believe
         * the command arrived but the sequencer to not.
         */
        ahc_outb(ahc, TQINPOS, ahc->tqinfifonext);

        /* Always allow reselection */
        ahc_outb(ahc, SCSISEQ,
                 ahc_inb(ahc, SCSISEQ_TEMPLATE) & (ENSELI|ENRSELI|ENAUTOATNP));
        if ((ahc->features & AHC_CMD_CHAN) != 0) {
                /* Ensure that no DMA operations are in progress */
                ahc_outb(ahc, CCSCBCNT, 0);
                ahc_outb(ahc, CCSGCTL, 0);
                ahc_outb(ahc, CCSCBCTL, 0);
        }
        /*
         * If we were in the process of DMA'ing SCB data into
         * an SCB, replace that SCB on the free list.  This prevents
         * an SCB leak.
         */
        if ((ahc_inb(ahc, SEQ_FLAGS2) & SCB_DMA) != 0) {
                ahc_add_curscb_to_free_list(ahc);
                ahc_outb(ahc, SEQ_FLAGS2,
                         ahc_inb(ahc, SEQ_FLAGS2) & ~SCB_DMA);
        }

        /*
         * Clear any pending sequencer interrupt.  It is no
         * longer relevant since we're resetting the Program
         * Counter.
         */
        ahc_outb(ahc, CLRINT, CLRSEQINT);

        ahc_outb(ahc, MWI_RESIDUAL, 0);
        ahc_outb(ahc, SEQCTL, ahc->seqctl);
        ahc_outb(ahc, SEQADDR0, 0);
        ahc_outb(ahc, SEQADDR1, 0);

        /*
         * Take the LED out of diagnostic mode on PM resume, too
         */
        sblkctl = ahc_inb(ahc, SBLKCTL);
        ahc_outb(ahc, SBLKCTL, (sblkctl & ~(DIAGLEDEN|DIAGLEDON)));

        ahc_unpause(ahc);
}

/************************* Input/Output Queues ********************************/
static void
ahc_run_qoutfifo(struct ahc_softc *ahc)
{
        struct scb *scb;
        u_int  scb_index;

        ahc_sync_qoutfifo(ahc, BUS_DMASYNC_POSTREAD);
        while (ahc->qoutfifo[ahc->qoutfifonext] != SCB_LIST_NULL) {

                scb_index = ahc->qoutfifo[ahc->qoutfifonext];
                if ((ahc->qoutfifonext & 0x03) == 0x03) {
                        u_int modnext;

                        /*
                         * Clear 32bits of QOUTFIFO at a time
                         * so that we don't clobber an incoming
                         * byte DMA to the array on architectures
                         * that only support 32bit load and store
                         * operations.
                         */
                        modnext = ahc->qoutfifonext & ~0x3;
                        *((uint32_t *)(&ahc->qoutfifo[modnext])) = 0xFFFFFFFFUL;
                        ahc_dmamap_sync(ahc, ahc->shared_data_dmat,
                                        ahc->shared_data_dmamap,
                                        /*offset*/modnext, /*len*/4,
                                        BUS_DMASYNC_PREREAD);
                }
                ahc->qoutfifonext++;

                scb = ahc_lookup_scb(ahc, scb_index);
                if (scb == NULL) {
                        printk("%s: WARNING no command for scb %d "
                               "(cmdcmplt)\nQOUTPOS = %d\n",
                               ahc_name(ahc), scb_index,
                               (ahc->qoutfifonext - 1) & 0xFF);
                        continue;
                }

                /*
                 * Save off the residual
                 * if there is one.
                 */
                ahc_update_residual(ahc, scb);
                ahc_done(ahc, scb);
        }
}

static void
ahc_run_untagged_queues(struct ahc_softc *ahc)
{
        int i;

        for (i = 0; i < 16; i++)
                ahc_run_untagged_queue(ahc, &ahc->untagged_queues[i]);
}

static void
ahc_run_untagged_queue(struct ahc_softc *ahc, struct scb_tailq *queue)
{
        struct scb *scb;

        if (ahc->untagged_queue_lock != 0)
                return;

        if ((scb = TAILQ_FIRST(queue)) != NULL
         && (scb->flags & SCB_ACTIVE) == 0) {
                scb->flags |= SCB_ACTIVE;
                ahc_queue_scb(ahc, scb);
        }
}

/************************* Interrupt Handling *********************************/
static void
ahc_handle_brkadrint(struct ahc_softc *ahc)
{
        /*
         * We upset the sequencer :-(
         * Lookup the error message
         */
        int i;
        int error;

        error = ahc_inb(ahc, ERROR);
        for (i = 0; error != 1 && i < num_errors; i++)
                error >>= 1;
        printk("%s: brkadrint, %s at seqaddr = 0x%x\n",
               ahc_name(ahc), ahc_hard_errors[i].errmesg,
               ahc_inb(ahc, SEQADDR0) |
               (ahc_inb(ahc, SEQADDR1) << 8));

        ahc_dump_card_state(ahc);

        /* Tell everyone that this HBA is no longer available */
        ahc_abort_scbs(ahc, CAM_TARGET_WILDCARD, ALL_CHANNELS,
                       CAM_LUN_WILDCARD, SCB_LIST_NULL, ROLE_UNKNOWN,
                       CAM_NO_HBA);

        /* Disable all interrupt sources by resetting the controller */
        ahc_shutdown(ahc);
}

static void
ahc_handle_seqint(struct ahc_softc *ahc, u_int intstat)
{
        struct scb *scb;
        struct ahc_devinfo devinfo;

        ahc_fetch_devinfo(ahc, &devinfo);

        /*
         * Clear the upper byte that holds SEQINT status
         * codes and clear the SEQINT bit. We will unpause
         * the sequencer, if appropriate, after servicing
         * the request.
         */
        ahc_outb(ahc, CLRINT, CLRSEQINT);
        switch (intstat & SEQINT_MASK) {
        case BAD_STATUS:
        {
                u_int  scb_index;
                struct hardware_scb *hscb;

                /*
                 * Set the default return value to 0 (don't
                 * send sense).  The sense code will change
                 * this if needed.

                 */
                ahc_outb(ahc, RETURN_1, 0);

                /*
                 * The sequencer will notify us when a command
                 * has an error that would be of interest to
                 * the kernel.  This allows us to leave the sequencer
                 * running in the common case of command completes
                 * without error.  The sequencer will already have
                 * dma'd the SCB back up to us, so we can reference
                 * the in kernel copy directly.
                 */
                scb_index = ahc_inb(ahc, SCB_TAG);
                scb = ahc_lookup_scb(ahc, scb_index);
                if (scb == NULL) {
                        ahc_print_devinfo(ahc, &devinfo);
                        printk("ahc_intr - referenced scb "
                               "not valid during seqint 0x%x scb(%d)\n",
                               intstat, scb_index);
                        ahc_dump_card_state(ahc);
                        panic("for safety");
                        goto unpause;
                }

                hscb = scb->hscb;

                /* Don't want to clobber the original sense code */
                if ((scb->flags & SCB_SENSE) != 0) {
                        /*
                         * Clear the SCB_SENSE Flag and have
                         * the sequencer do a normal command
                         * complete.
                         */
                        scb->flags &= ~SCB_SENSE;
                        ahc_set_transaction_status(scb, CAM_AUTOSENSE_FAIL);
                        break;
                }
                ahc_set_transaction_status(scb, CAM_SCSI_STATUS_ERROR);
                /* Freeze the queue until the client sees the error. */
                ahc_freeze_devq(ahc, scb);
                ahc_freeze_scb(scb);
                ahc_set_scsi_status(scb, hscb->shared_data.status.scsi_status);
                switch (hscb->shared_data.status.scsi_status) {
                case SAM_STAT_GOOD:
                        printk("%s: Interrupted for status of 0???\n",
                               ahc_name(ahc));
                        break;
                case SAM_STAT_COMMAND_TERMINATED:
                case SAM_STAT_CHECK_CONDITION:
                {
                        struct ahc_dma_seg *sg;
                        struct scsi_sense *sc;
                        struct ahc_initiator_tinfo *targ_info;
                        struct ahc_tmode_tstate *tstate;
                        struct ahc_transinfo *tinfo;
#ifdef AHC_DEBUG
                        if (ahc_debug & AHC_SHOW_SENSE) {
                                ahc_print_path(ahc, scb);
                                printk("SCB %d: requests Check Status\n",
                                       scb->hscb->tag);
                        }
#endif

                        if (ahc_perform_autosense(scb) == 0)
                                break;

                        targ_info = ahc_fetch_transinfo(ahc,
                                                        devinfo.channel,
                                                        devinfo.our_scsiid,
                                                        devinfo.target,
                                                        &tstate);
                        tinfo = &targ_info->curr;
                        sg = scb->sg_list;
                        sc = (struct scsi_sense *)(&hscb->shared_data.cdb);
                        /*
                         * Save off the residual if there is one.
                         */
                        ahc_update_residual(ahc, scb);
#ifdef AHC_DEBUG
                        if (ahc_debug & AHC_SHOW_SENSE) {
                                ahc_print_path(ahc, scb);
                                printk("Sending Sense\n");
                        }
#endif
                        sg->addr = ahc_get_sense_bufaddr(ahc, scb);
                        sg->len = ahc_get_sense_bufsize(ahc, scb);
                        sg->len |= AHC_DMA_LAST_SEG;

                        /* Fixup byte order */
                        sg->addr = ahc_htole32(sg->addr);
                        sg->len = ahc_htole32(sg->len);

                        sc->opcode = REQUEST_SENSE;
                        sc->byte2 = 0;
                        if (tinfo->protocol_version <= SCSI_REV_2
                         && SCB_GET_LUN(scb) < 8)
                                sc->byte2 = SCB_GET_LUN(scb) << 5;
                        sc->unused[0] = 0;
                        sc->unused[1] = 0;
                        sc->length = sg->len;
                        sc->control = 0;

                        /*
                         * We can't allow the target to disconnect.
                         * This will be an untagged transaction and
                         * having the target disconnect will make this
                         * transaction indestinguishable from outstanding
                         * tagged transactions.
                         */
                        hscb->control = 0;

                        /*
                         * This request sense could be because the
                         * the device lost power or in some other
                         * way has lost our transfer negotiations.
                         * Renegotiate if appropriate.  Unit attention
                         * errors will be reported before any data
                         * phases occur.
                         */
                        if (ahc_get_residual(scb)
                            == ahc_get_transfer_length(scb)) {
                                ahc_update_neg_request(ahc, &devinfo,
                                                       tstate, targ_info,
                                                       AHC_NEG_IF_NON_ASYNC);
                        }
                        if (tstate->auto_negotiate & devinfo.target_mask) {
                                hscb->control |= MK_MESSAGE;
                                scb->flags &= ~SCB_NEGOTIATE;
                                scb->flags |= SCB_AUTO_NEGOTIATE;
                        }
                        hscb->cdb_len = sizeof(*sc);
                        hscb->dataptr = sg->addr;
                        hscb->datacnt = sg->len;
                        hscb->sgptr = scb->sg_list_phys | SG_FULL_RESID;
                        hscb->sgptr = ahc_htole32(hscb->sgptr);
                        scb->sg_count = 1;
                        scb->flags |= SCB_SENSE;
                        ahc_qinfifo_requeue_tail(ahc, scb);
                        ahc_outb(ahc, RETURN_1, SEND_SENSE);
                        /*
                         * Ensure we have enough time to actually
                         * retrieve the sense.
                         */
                        ahc_scb_timer_reset(scb, 5 * 1000000);
                        break;
                }
                default:
                        break;
                }
                break;
        }
        case NO_MATCH:
        {
                /* Ensure we don't leave the selection hardware on */
                ahc_outb(ahc, SCSISEQ,
                         ahc_inb(ahc, SCSISEQ) & (ENSELI|ENRSELI|ENAUTOATNP));

                printk("%s:%c:%d: no active SCB for reconnecting "
                       "target - issuing BUS DEVICE RESET\n",
                       ahc_name(ahc), devinfo.channel, devinfo.target);
                printk("SAVED_SCSIID == 0x%x, SAVED_LUN == 0x%x, "
                       "ARG_1 == 0x%x ACCUM = 0x%x\n",
                       ahc_inb(ahc, SAVED_SCSIID), ahc_inb(ahc, SAVED_LUN),
                       ahc_inb(ahc, ARG_1), ahc_inb(ahc, ACCUM));
                printk("SEQ_FLAGS == 0x%x, SCBPTR == 0x%x, BTT == 0x%x, "
                       "SINDEX == 0x%x\n",
                       ahc_inb(ahc, SEQ_FLAGS), ahc_inb(ahc, SCBPTR),
                       ahc_index_busy_tcl(ahc,
                            BUILD_TCL(ahc_inb(ahc, SAVED_SCSIID),
                                      ahc_inb(ahc, SAVED_LUN))),
                       ahc_inb(ahc, SINDEX));
                printk("SCSIID == 0x%x, SCB_SCSIID == 0x%x, SCB_LUN == 0x%x, "
                       "SCB_TAG == 0x%x, SCB_CONTROL == 0x%x\n",
                       ahc_inb(ahc, SCSIID), ahc_inb(ahc, SCB_SCSIID),
                       ahc_inb(ahc, SCB_LUN), ahc_inb(ahc, SCB_TAG),
                       ahc_inb(ahc, SCB_CONTROL));
                printk("SCSIBUSL == 0x%x, SCSISIGI == 0x%x\n",
                       ahc_inb(ahc, SCSIBUSL), ahc_inb(ahc, SCSISIGI));
                printk("SXFRCTL0 == 0x%x\n", ahc_inb(ahc, SXFRCTL0));
                printk("SEQCTL == 0x%x\n", ahc_inb(ahc, SEQCTL));
                ahc_dump_card_state(ahc);
                ahc->msgout_buf[0] = TARGET_RESET;
                ahc->msgout_len = 1;
                ahc->msgout_index = 0;
                ahc->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
                ahc_outb(ahc, MSG_OUT, HOST_MSG);
                ahc_assert_atn(ahc);
                break;
        }
        case SEND_REJECT:
        {
                u_int rejbyte = ahc_inb(ahc, ACCUM);
                printk("%s:%c:%d: Warning - unknown message received from "
                       "target (0x%x).  Rejecting\n",
                       ahc_name(ahc), devinfo.channel, devinfo.target, rejbyte);
                break;
        }
        case PROTO_VIOLATION:
        {
                ahc_handle_proto_violation(ahc);
                break;
        }
        case IGN_WIDE_RES:
                ahc_handle_ign_wide_residue(ahc, &devinfo);
                break;
        case PDATA_REINIT:
                ahc_reinitialize_dataptrs(ahc);
                break;
        case BAD_PHASE:
        {
                u_int lastphase;

                lastphase = ahc_inb(ahc, LASTPHASE);
                printk("%s:%c:%d: unknown scsi bus phase %x, "
                       "lastphase = 0x%x.  Attempting to continue\n",
                       ahc_name(ahc), devinfo.channel, devinfo.target,
                       lastphase, ahc_inb(ahc, SCSISIGI));
                break;
        }
        case MISSED_BUSFREE:
        {
                u_int lastphase;

                lastphase = ahc_inb(ahc, LASTPHASE);
                printk("%s:%c:%d: Missed busfree. "
                       "Lastphase = 0x%x, Curphase = 0x%x\n",
                       ahc_name(ahc), devinfo.channel, devinfo.target,
                       lastphase, ahc_inb(ahc, SCSISIGI));
                ahc_restart(ahc);
                return;
        }
        case HOST_MSG_LOOP:
        {
                /*
                 * The sequencer has encountered a message phase
                 * that requires host assistance for completion.
                 * While handling the message phase(s), we will be
                 * notified by the sequencer after each byte is
                 * transferred so we can track bus phase changes.
                 *
                 * If this is the first time we've seen a HOST_MSG_LOOP
                 * interrupt, initialize the state of the host message
                 * loop.
                 */
                if (ahc->msg_type == MSG_TYPE_NONE) {
                        struct scb *scb;
                        u_int scb_index;
                        u_int bus_phase;

                        bus_phase = ahc_inb(ahc, SCSISIGI) & PHASE_MASK;
                        if (bus_phase != P_MESGIN
                         && bus_phase != P_MESGOUT) {
                                printk("ahc_intr: HOST_MSG_LOOP bad "
                                       "phase 0x%x\n",
                                      bus_phase);
                                /*
                                 * Probably transitioned to bus free before
                                 * we got here.  Just punt the message.
                                 */
                                ahc_clear_intstat(ahc);
                                ahc_restart(ahc);
                                return;
                        }

                        scb_index = ahc_inb(ahc, SCB_TAG);
                        scb = ahc_lookup_scb(ahc, scb_index);
                        if (devinfo.role == ROLE_INITIATOR) {
                                if (bus_phase == P_MESGOUT) {
                                        if (scb == NULL)
                                                panic("HOST_MSG_LOOP with "
                                                      "invalid SCB %x\n",
                                                      scb_index);

                                        ahc_setup_initiator_msgout(ahc,
                                                                   &devinfo,
                                                                   scb);
                                } else {
                                        ahc->msg_type =
                                            MSG_TYPE_INITIATOR_MSGIN;
                                        ahc->msgin_index = 0;
                                }
                        }
#ifdef AHC_TARGET_MODE
                        else {
                                if (bus_phase == P_MESGOUT) {
                                        ahc->msg_type =
                                            MSG_TYPE_TARGET_MSGOUT;
                                        ahc->msgin_index = 0;
                                } else
                                        ahc_setup_target_msgin(ahc,
                                                               &devinfo,
                                                               scb);
                        }
#endif
                }

                ahc_handle_message_phase(ahc);
                break;
        }
        case PERR_DETECTED:
        {
                /*
                 * If we've cleared the parity error interrupt
                 * but the sequencer still believes that SCSIPERR
                 * is true, it must be that the parity error is
                 * for the currently presented byte on the bus,
                 * and we are not in a phase (data-in) where we will
                 * eventually ack this byte.  Ack the byte and
                 * throw it away in the hope that the target will
                 * take us to message out to deliver the appropriate
                 * error message.
                 */
                if ((intstat & SCSIINT) == 0
                 && (ahc_inb(ahc, SSTAT1) & SCSIPERR) != 0) {

                        if ((ahc->features & AHC_DT) == 0) {
                                u_int curphase;

                                /*
                                 * The hardware will only let you ack bytes
                                 * if the expected phase in SCSISIGO matches
                                 * the current phase.  Make sure this is
                                 * currently the case.
                                 */
                                curphase = ahc_inb(ahc, SCSISIGI) & PHASE_MASK;
                                ahc_outb(ahc, LASTPHASE, curphase);
                                ahc_outb(ahc, SCSISIGO, curphase);
                        }
                        if ((ahc_inb(ahc, SCSISIGI) & (CDI|MSGI)) == 0) {
                                int wait;

                                /*
                                 * In a data phase.  Faster to bitbucket
                                 * the data than to individually ack each
                                 * byte.  This is also the only strategy
                                 * that will work with AUTOACK enabled.
                                 */
                                ahc_outb(ahc, SXFRCTL1,
                                         ahc_inb(ahc, SXFRCTL1) | BITBUCKET);
                                wait = 5000;
                                while (--wait != 0) {
                                        if ((ahc_inb(ahc, SCSISIGI)
                                          & (CDI|MSGI)) != 0)
                                                break;
                                        ahc_delay(100);
                                }
                                ahc_outb(ahc, SXFRCTL1,
                                         ahc_inb(ahc, SXFRCTL1) & ~BITBUCKET);
                                if (wait == 0) {
                                        struct  scb *scb;
                                        u_int   scb_index;

                                        ahc_print_devinfo(ahc, &devinfo);
                                        printk("Unable to clear parity error.  "
                                               "Resetting bus.\n");
                                        scb_index = ahc_inb(ahc, SCB_TAG);
                                        scb = ahc_lookup_scb(ahc, scb_index);
                                        if (scb != NULL)
                                                ahc_set_transaction_status(scb,
                                                    CAM_UNCOR_PARITY);
                                        ahc_reset_channel(ahc, devinfo.channel,
                                                          /*init reset*/TRUE);
                                }
                        } else {
                                ahc_inb(ahc, SCSIDATL);
                        }
                }
                break;
        }
        case DATA_OVERRUN:
        {
                /*
                 * When the sequencer detects an overrun, it
                 * places the controller in "BITBUCKET" mode
                 * and allows the target to complete its transfer.
                 * Unfortunately, none of the counters get updated
                 * when the controller is in this mode, so we have
                 * no way of knowing how large the overrun was.
                 */
                u_int scbindex = ahc_inb(ahc, SCB_TAG);
                u_int lastphase = ahc_inb(ahc, LASTPHASE);
                u_int i;

                scb = ahc_lookup_scb(ahc, scbindex);
                for (i = 0; i < num_phases; i++) {
                        if (lastphase == ahc_phase_table[i].phase)
                                break;
                }
                ahc_print_path(ahc, scb);
                printk("data overrun detected %s."
                       "  Tag == 0x%x.\n",
                       ahc_phase_table[i].phasemsg,
                       scb->hscb->tag);
                ahc_print_path(ahc, scb);
                printk("%s seen Data Phase.  Length = %ld.  NumSGs = %d.\n",
                       ahc_inb(ahc, SEQ_FLAGS) & DPHASE ? "Have" : "Haven't",
                       ahc_get_transfer_length(scb), scb->sg_count);
                if (scb->sg_count > 0) {
                        for (i = 0; i < scb->sg_count; i++) {

                                printk("sg[%d] - Addr 0x%x%x : Length %d\n",
                                       i,
                                       (ahc_le32toh(scb->sg_list[i].len) >> 24
                                        & SG_HIGH_ADDR_BITS),
                                       ahc_le32toh(scb->sg_list[i].addr),
                                       ahc_le32toh(scb->sg_list[i].len)
                                       & AHC_SG_LEN_MASK);
                        }
                }
                /*
                 * Set this and it will take effect when the
                 * target does a command complete.
                 */
                ahc_freeze_devq(ahc, scb);
                if ((scb->flags & SCB_SENSE) == 0) {
                        ahc_set_transaction_status(scb, CAM_DATA_RUN_ERR);
                } else {
                        scb->flags &= ~SCB_SENSE;
                        ahc_set_transaction_status(scb, CAM_AUTOSENSE_FAIL);
                }
                ahc_freeze_scb(scb);

                if ((ahc->features & AHC_ULTRA2) != 0) {
                        /*
                         * Clear the channel in case we return
                         * to data phase later.
                         */
                        ahc_outb(ahc, SXFRCTL0,
                                 ahc_inb(ahc, SXFRCTL0) | CLRSTCNT|CLRCHN);
                        ahc_outb(ahc, SXFRCTL0,
                                 ahc_inb(ahc, SXFRCTL0) | CLRSTCNT|CLRCHN);
                }
                if ((ahc->flags & AHC_39BIT_ADDRESSING) != 0) {
                        u_int dscommand1;

                        /* Ensure HHADDR is 0 for future DMA operations. */
                        dscommand1 = ahc_inb(ahc, DSCOMMAND1);
                        ahc_outb(ahc, DSCOMMAND1, dscommand1 | HADDLDSEL0);
                        ahc_outb(ahc, HADDR, 0);
                        ahc_outb(ahc, DSCOMMAND1, dscommand1);
                }
                break;
        }
        case MKMSG_FAILED:
        {
                u_int scbindex;

                printk("%s:%c:%d:%d: Attempt to issue message failed\n",
                       ahc_name(ahc), devinfo.channel, devinfo.target,
                       devinfo.lun);
                scbindex = ahc_inb(ahc, SCB_TAG);
                scb = ahc_lookup_scb(ahc, scbindex);
                if (scb != NULL
                 && (scb->flags & SCB_RECOVERY_SCB) != 0)
                        /*
                         * Ensure that we didn't put a second instance of this
                         * SCB into the QINFIFO.
                         */
                        ahc_search_qinfifo(ahc, SCB_GET_TARGET(ahc, scb),
                                           SCB_GET_CHANNEL(ahc, scb),
                                           SCB_GET_LUN(scb), scb->hscb->tag,
                                           ROLE_INITIATOR, /*status*/0,
                                           SEARCH_REMOVE);
                break;
        }
        case NO_FREE_SCB:
        {
                printk("%s: No free or disconnected SCBs\n", ahc_name(ahc));
                ahc_dump_card_state(ahc);
                panic("for safety");
                break;
        }
        case SCB_MISMATCH:
        {
                u_int scbptr;

                scbptr = ahc_inb(ahc, SCBPTR);
                printk("Bogus TAG after DMA.  SCBPTR %d, tag %d, our tag %d\n",
                       scbptr, ahc_inb(ahc, ARG_1),
                       ahc->scb_data->hscbs[scbptr].tag);
                ahc_dump_card_state(ahc);
                panic("for safety");
                break;
        }
        case OUT_OF_RANGE:
        {
                printk("%s: BTT calculation out of range\n", ahc_name(ahc));
                printk("SAVED_SCSIID == 0x%x, SAVED_LUN == 0x%x, "
                       "ARG_1 == 0x%x ACCUM = 0x%x\n",
                       ahc_inb(ahc, SAVED_SCSIID), ahc_inb(ahc, SAVED_LUN),
                       ahc_inb(ahc, ARG_1), ahc_inb(ahc, ACCUM));
                printk("SEQ_FLAGS == 0x%x, SCBPTR == 0x%x, BTT == 0x%x, "
                       "SINDEX == 0x%x\n, A == 0x%x\n",
                       ahc_inb(ahc, SEQ_FLAGS), ahc_inb(ahc, SCBPTR),
                       ahc_index_busy_tcl(ahc,
                            BUILD_TCL(ahc_inb(ahc, SAVED_SCSIID),
                                      ahc_inb(ahc, SAVED_LUN))),
                       ahc_inb(ahc, SINDEX),
                       ahc_inb(ahc, ACCUM));
                printk("SCSIID == 0x%x, SCB_SCSIID == 0x%x, SCB_LUN == 0x%x, "
                       "SCB_TAG == 0x%x, SCB_CONTROL == 0x%x\n",
                       ahc_inb(ahc, SCSIID), ahc_inb(ahc, SCB_SCSIID),
                       ahc_inb(ahc, SCB_LUN), ahc_inb(ahc, SCB_TAG),
                       ahc_inb(ahc, SCB_CONTROL));
                printk("SCSIBUSL == 0x%x, SCSISIGI == 0x%x\n",
                       ahc_inb(ahc, SCSIBUSL), ahc_inb(ahc, SCSISIGI));
                ahc_dump_card_state(ahc);
                panic("for safety");
                break;
        }
        default:
                printk("ahc_intr: seqint, "
                       "intstat == 0x%x, scsisigi = 0x%x\n",
                       intstat, ahc_inb(ahc, SCSISIGI));
                break;
        }
unpause:
        /*
         *  The sequencer is paused immediately on
         *  a SEQINT, so we should restart it when
         *  we're done.
         */
        ahc_unpause(ahc);
}

static void
ahc_handle_scsiint(struct ahc_softc *ahc, u_int intstat)
{
        u_int   scb_index;
        u_int   status0;
        u_int   status;
        struct  scb *scb;
        char    cur_channel;
        char    intr_channel;

        if ((ahc->features & AHC_TWIN) != 0
         && ((ahc_inb(ahc, SBLKCTL) & SELBUSB) != 0))
                cur_channel = 'B';
        else
                cur_channel = 'A';
        intr_channel = cur_channel;

        if ((ahc->features & AHC_ULTRA2) != 0)
                status0 = ahc_inb(ahc, SSTAT0) & IOERR;
        else
                status0 = 0;
        status = ahc_inb(ahc, SSTAT1) & (SELTO|SCSIRSTI|BUSFREE|SCSIPERR);
        if (status == 0 && status0 == 0) {
                if ((ahc->features & AHC_TWIN) != 0) {
                        /* Try the other channel */
                        ahc_outb(ahc, SBLKCTL, ahc_inb(ahc, SBLKCTL) ^ SELBUSB);
                        status = ahc_inb(ahc, SSTAT1)
                               & (SELTO|SCSIRSTI|BUSFREE|SCSIPERR);
                        intr_channel = (cur_channel == 'A') ? 'B' : 'A';
                }
                if (status == 0) {
                        printk("%s: Spurious SCSI interrupt\n", ahc_name(ahc));
                        ahc_outb(ahc, CLRINT, CLRSCSIINT);
                        ahc_unpause(ahc);
                        return;
                }
        }

        /* Make sure the sequencer is in a safe location. */
        ahc_clear_critical_section(ahc);

        scb_index = ahc_inb(ahc, SCB_TAG);
        scb = ahc_lookup_scb(ahc, scb_index);
        if (scb != NULL
         && (ahc_inb(ahc, SEQ_FLAGS) & NOT_IDENTIFIED) != 0)
                scb = NULL;

        if ((ahc->features & AHC_ULTRA2) != 0
         && (status0 & IOERR) != 0) {
                int now_lvd;

                now_lvd = ahc_inb(ahc, SBLKCTL) & ENAB40;
                printk("%s: Transceiver State Has Changed to %s mode\n",
                       ahc_name(ahc), now_lvd ? "LVD" : "SE");
                ahc_outb(ahc, CLRSINT0, CLRIOERR);
                /*
                 * When transitioning to SE mode, the reset line
                 * glitches, triggering an arbitration bug in some
                 * Ultra2 controllers.  This bug is cleared when we
                 * assert the reset line.  Since a reset glitch has
                 * already occurred with this transition and a
                 * transceiver state change is handled just like
                 * a bus reset anyway, asserting the reset line
                 * ourselves is safe.
                 */
                ahc_reset_channel(ahc, intr_channel,
                                 /*Initiate Reset*/now_lvd == 0);
        } else if ((status & SCSIRSTI) != 0) {
                printk("%s: Someone reset channel %c\n",
                        ahc_name(ahc), intr_channel);
                if (intr_channel != cur_channel)
                        ahc_outb(ahc, SBLKCTL, ahc_inb(ahc, SBLKCTL) ^ SELBUSB);
                ahc_reset_channel(ahc, intr_channel, /*Initiate Reset*/FALSE);
        } else if ((status & SCSIPERR) != 0) {
                /*
                 * Determine the bus phase and queue an appropriate message.
                 * SCSIPERR is latched true as soon as a parity error
                 * occurs.  If the sequencer acked the transfer that
                 * caused the parity error and the currently presented
                 * transfer on the bus has correct parity, SCSIPERR will
                 * be cleared by CLRSCSIPERR.  Use this to determine if
                 * we should look at the last phase the sequencer recorded,
                 * or the current phase presented on the bus.
                 */
                struct  ahc_devinfo devinfo;
                u_int   mesg_out;
                u_int   curphase;
                u_int   errorphase;
                u_int   lastphase;
                u_int   scsirate;
                u_int   i;
                u_int   sstat2;
                int     silent;

                lastphase = ahc_inb(ahc, LASTPHASE);
                curphase = ahc_inb(ahc, SCSISIGI) & PHASE_MASK;
                sstat2 = ahc_inb(ahc, SSTAT2);
                ahc_outb(ahc, CLRSINT1, CLRSCSIPERR);
                /*
                 * For all phases save DATA, the sequencer won't
                 * automatically ack a byte that has a parity error
                 * in it.  So the only way that the current phase
                 * could be 'data-in' is if the parity error is for
                 * an already acked byte in the data phase.  During
                 * synchronous data-in transfers, we may actually
                 * ack bytes before latching the current phase in
                 * LASTPHASE, leading to the discrepancy between
                 * curphase and lastphase.
                 */
                if ((ahc_inb(ahc, SSTAT1) & SCSIPERR) != 0
                 || curphase == P_DATAIN || curphase == P_DATAIN_DT)
                        errorphase = curphase;
                else
                        errorphase = lastphase;

                for (i = 0; i < num_phases; i++) {
                        if (errorphase == ahc_phase_table[i].phase)
                                break;
                }
                mesg_out = ahc_phase_table[i].mesg_out;
                silent = FALSE;
                if (scb != NULL) {
                        if (SCB_IS_SILENT(scb))
                                silent = TRUE;
                        else
                                ahc_print_path(ahc, scb);
                        scb->flags |= SCB_TRANSMISSION_ERROR;
                } else
                        printk("%s:%c:%d: ", ahc_name(ahc), intr_channel,
                               SCSIID_TARGET(ahc, ahc_inb(ahc, SAVED_SCSIID)));
                scsirate = ahc_inb(ahc, SCSIRATE);
                if (silent == FALSE) {
                        printk("parity error detected %s. "
                               "SEQADDR(0x%x) SCSIRATE(0x%x)\n",
                               ahc_phase_table[i].phasemsg,
                               ahc_inw(ahc, SEQADDR0),
                               scsirate);
                        if ((ahc->features & AHC_DT) != 0) {
                                if ((sstat2 & CRCVALERR) != 0)
                                        printk("\tCRC Value Mismatch\n");
                                if ((sstat2 & CRCENDERR) != 0)
                                        printk("\tNo terminal CRC packet "
                                               "received\n");
                                if ((sstat2 & CRCREQERR) != 0)
                                        printk("\tIllegal CRC packet "
                                               "request\n");
                                if ((sstat2 & DUAL_EDGE_ERR) != 0)
                                        printk("\tUnexpected %sDT Data Phase\n",
                                               (scsirate & SINGLE_EDGE)
                                             ? "" : "non-");
                        }
                }

                if ((ahc->features & AHC_DT) != 0
                 && (sstat2 & DUAL_EDGE_ERR) != 0) {
                        /*
                         * This error applies regardless of
                         * data direction, so ignore the value
                         * in the phase table.
                         */
                        mesg_out = INITIATOR_ERROR;
                }

                /*
                 * We've set the hardware to assert ATN if we
                 * get a parity error on "in" phases, so all we
                 * need to do is stuff the message buffer with
                 * the appropriate message.  "In" phases have set
                 * mesg_out to something other than MSG_NOP.
                 */
                if (mesg_out != NOP) {
                        if (ahc->msg_type != MSG_TYPE_NONE)
                                ahc->send_msg_perror = TRUE;
                        else
                                ahc_outb(ahc, MSG_OUT, mesg_out);
                }
                /*
                 * Force a renegotiation with this target just in
                 * case we are out of sync for some external reason
                 * unknown (or unreported) by the target.
                 */
                ahc_fetch_devinfo(ahc, &devinfo);
                ahc_force_renegotiation(ahc, &devinfo);

                ahc_outb(ahc, CLRINT, CLRSCSIINT);
                ahc_unpause(ahc);
        } else if ((status & SELTO) != 0) {
                u_int   scbptr;

                /* Stop the selection */
                ahc_outb(ahc, SCSISEQ, 0);

                /* No more pending messages */
                ahc_clear_msg_state(ahc);

                /* Clear interrupt state */
                ahc_outb(ahc, SIMODE1, ahc_inb(ahc, SIMODE1) & ~ENBUSFREE);
                ahc_outb(ahc, CLRSINT1, CLRSELTIMEO|CLRBUSFREE|CLRSCSIPERR);

                /*
                 * Although the driver does not care about the
                 * 'Selection in Progress' status bit, the busy
                 * LED does.  SELINGO is only cleared by a successful
                 * selection, so we must manually clear it to insure
                 * the LED turns off just incase no future successful
                 * selections occur (e.g. no devices on the bus).
                 */
                ahc_outb(ahc, CLRSINT0, CLRSELINGO);

                scbptr = ahc_inb(ahc, WAITING_SCBH);
                ahc_outb(ahc, SCBPTR, scbptr);
                scb_index = ahc_inb(ahc, SCB_TAG);

                scb = ahc_lookup_scb(ahc, scb_index);
                if (scb == NULL) {
                        printk("%s: ahc_intr - referenced scb not "
                               "valid during SELTO scb(%d, %d)\n",
                               ahc_name(ahc), scbptr, scb_index);
                        ahc_dump_card_state(ahc);
                } else {
                        struct ahc_devinfo devinfo;
#ifdef AHC_DEBUG
                        if ((ahc_debug & AHC_SHOW_SELTO) != 0) {
                                ahc_print_path(ahc, scb);
                                printk("Saw Selection Timeout for SCB 0x%x\n",
                                       scb_index);
                        }
#endif
                        ahc_scb_devinfo(ahc, &devinfo, scb);
                        ahc_set_transaction_status(scb, CAM_SEL_TIMEOUT);
                        ahc_freeze_devq(ahc, scb);

                        /*
                         * Cancel any pending transactions on the device
                         * now that it seems to be missing.  This will
                         * also revert us to async/narrow transfers until
                         * we can renegotiate with the device.
                         */
                        ahc_handle_devreset(ahc, &devinfo,
                                            CAM_SEL_TIMEOUT,
                                            "Selection Timeout",
                                            /*verbose_level*/1);
                }
                ahc_outb(ahc, CLRINT, CLRSCSIINT);
                ahc_restart(ahc);
        } else if ((status & BUSFREE) != 0
                && (ahc_inb(ahc, SIMODE1) & ENBUSFREE) != 0) {
                struct  ahc_devinfo devinfo;
                u_int   lastphase;
                u_int   saved_scsiid;
                u_int   saved_lun;
                u_int   target;
                u_int   initiator_role_id;
                char    channel;
                int     printerror;

                /*
                 * Clear our selection hardware as soon as possible.
                 * We may have an entry in the waiting Q for this target,
                 * that is affected by this busfree and we don't want to
                 * go about selecting the target while we handle the event.
                 */
                ahc_outb(ahc, SCSISEQ,
                         ahc_inb(ahc, SCSISEQ) & (ENSELI|ENRSELI|ENAUTOATNP));

                /*
                 * Disable busfree interrupts and clear the busfree
                 * interrupt status.  We do this here so that several
                 * bus transactions occur prior to clearing the SCSIINT
                 * latch.  It can take a bit for the clearing to take effect.
                 */
                ahc_outb(ahc, SIMODE1, ahc_inb(ahc, SIMODE1) & ~ENBUSFREE);
                ahc_outb(ahc, CLRSINT1, CLRBUSFREE|CLRSCSIPERR);

                /*
                 * Look at what phase we were last in.
                 * If its message out, chances are pretty good
                 * that the busfree was in response to one of
                 * our abort requests.
                 */
                lastphase = ahc_inb(ahc, LASTPHASE);
                saved_scsiid = ahc_inb(ahc, SAVED_SCSIID);
                saved_lun = ahc_inb(ahc, SAVED_LUN);
                target = SCSIID_TARGET(ahc, saved_scsiid);
                initiator_role_id = SCSIID_OUR_ID(saved_scsiid);
                channel = SCSIID_CHANNEL(ahc, saved_scsiid);
                ahc_compile_devinfo(&devinfo, initiator_role_id,
                                    target, saved_lun, channel, ROLE_INITIATOR);
                printerror = 1;

                if (lastphase == P_MESGOUT) {
                        u_int tag;

                        tag = SCB_LIST_NULL;
                        if (ahc_sent_msg(ahc, AHCMSG_1B, ABORT_TASK, TRUE)
                         || ahc_sent_msg(ahc, AHCMSG_1B, ABORT_TASK_SET, TRUE)) {
                                if (ahc->msgout_buf[ahc->msgout_index - 1]
                                 == ABORT_TASK)
                                        tag = scb->hscb->tag;
                                ahc_print_path(ahc, scb);
                                printk("SCB %d - Abort%s Completed.\n",
                                       scb->hscb->tag, tag == SCB_LIST_NULL ?
                                       "" : " Tag");
                                ahc_abort_scbs(ahc, target, channel,
                                               saved_lun, tag,
                                               ROLE_INITIATOR,
                                               CAM_REQ_ABORTED);
                                printerror = 0;
                        } else if (ahc_sent_msg(ahc, AHCMSG_1B,
                                                TARGET_RESET, TRUE)) {
                                ahc_compile_devinfo(&devinfo,
                                                    initiator_role_id,
                                                    target,
                                                    CAM_LUN_WILDCARD,
                                                    channel,
                                                    ROLE_INITIATOR);
                                ahc_handle_devreset(ahc, &devinfo,
                                                    CAM_BDR_SENT,
                                                    "Bus Device Reset",
                                                    /*verbose_level*/0);
                                printerror = 0;
                        } else if (ahc_sent_msg(ahc, AHCMSG_EXT,
                                                EXTENDED_PPR, FALSE)) {
                                struct ahc_initiator_tinfo *tinfo;
                                struct ahc_tmode_tstate *tstate;

                                /*
                                 * PPR Rejected.  Try non-ppr negotiation
                                 * and retry command.
                                 */
                                tinfo = ahc_fetch_transinfo(ahc,
                                                            devinfo.channel,
                                                            devinfo.our_scsiid,
                                                            devinfo.target,
                                                            &tstate);
                                tinfo->curr.transport_version = 2;
                                tinfo->goal.transport_version = 2;
                                tinfo->goal.ppr_options = 0;
                                ahc_qinfifo_requeue_tail(ahc, scb);
                                printerror = 0;
                        } else if (ahc_sent_msg(ahc, AHCMSG_EXT,
                                                EXTENDED_WDTR, FALSE)) {
                                /*
                                 * Negotiation Rejected.  Go-narrow and
                                 * retry command.
                                 */
                                ahc_set_width(ahc, &devinfo,
                                              MSG_EXT_WDTR_BUS_8_BIT,
                                              AHC_TRANS_CUR|AHC_TRANS_GOAL,
                                              /*paused*/TRUE);
                                ahc_qinfifo_requeue_tail(ahc, scb);
                                printerror = 0;
                        } else if (ahc_sent_msg(ahc, AHCMSG_EXT,
                                                EXTENDED_SDTR, FALSE)) {
                                /*
                                 * Negotiation Rejected.  Go-async and
                                 * retry command.
                                 */
                                ahc_set_syncrate(ahc, &devinfo,
                                                /*syncrate*/NULL,
                                                /*period*/0, /*offset*/0,
                                                /*ppr_options*/0,
                                                AHC_TRANS_CUR|AHC_TRANS_GOAL,
                                                /*paused*/TRUE);
                                ahc_qinfifo_requeue_tail(ahc, scb);
                                printerror = 0;
                        }
                }
                if (printerror != 0) {
                        u_int i;

                        if (scb != NULL) {
                                u_int tag;

                                if ((scb->hscb->control & TAG_ENB) != 0)
                                        tag = scb->hscb->tag;
                                else
                                        tag = SCB_LIST_NULL;
                                ahc_print_path(ahc, scb);
                                ahc_abort_scbs(ahc, target, channel,
                                               SCB_GET_LUN(scb), tag,
                                               ROLE_INITIATOR,
                                               CAM_UNEXP_BUSFREE);
                        } else {
                                /*
                                 * We had not fully identified this connection,
                                 * so we cannot abort anything.
                                 */
                                printk("%s: ", ahc_name(ahc));
                        }
                        for (i = 0; i < num_phases; i++) {
                                if (lastphase == ahc_phase_table[i].phase)
                                        break;
                        }
                        if (lastphase != P_BUSFREE) {
                                /*
                                 * Renegotiate with this device at the
                                 * next opportunity just in case this busfree
                                 * is due to a negotiation mismatch with the
                                 * device.
                                 */
                                ahc_force_renegotiation(ahc, &devinfo);
                        }
                        printk("Unexpected busfree %s\n"
                               "SEQADDR == 0x%x\n",
                               ahc_phase_table[i].phasemsg,
                               ahc_inb(ahc, SEQADDR0)
                                | (ahc_inb(ahc, SEQADDR1) << 8));
                }
                ahc_outb(ahc, CLRINT, CLRSCSIINT);
                ahc_restart(ahc);
        } else {
                printk("%s: Missing case in ahc_handle_scsiint. status = %x\n",
                       ahc_name(ahc), status);
                ahc_outb(ahc, CLRINT, CLRSCSIINT);
        }
}

/*
 * Force renegotiation to occur the next time we initiate
 * a command to the current device.
 */
static void
ahc_force_renegotiation(struct ahc_softc *ahc, struct ahc_devinfo *devinfo)
{
        struct  ahc_initiator_tinfo *targ_info;
        struct  ahc_tmode_tstate *tstate;

        targ_info = ahc_fetch_transinfo(ahc,
                                        devinfo->channel,
                                        devinfo->our_scsiid,
                                        devinfo->target,
                                        &tstate);
        ahc_update_neg_request(ahc, devinfo, tstate,
                               targ_info, AHC_NEG_IF_NON_ASYNC);
}

#define AHC_MAX_STEPS 2000
static void
ahc_clear_critical_section(struct ahc_softc *ahc)
{
        int     stepping;
        int     steps;
        u_int   simode0;
        u_int   simode1;

        if (ahc->num_critical_sections == 0)
                return;

        stepping = FALSE;
        steps = 0;
        simode0 = 0;
        simode1 = 0;
        for (;;) {
                struct  cs *cs;
                u_int   seqaddr;
                u_int   i;

                seqaddr = ahc_inb(ahc, SEQADDR0)
                        | (ahc_inb(ahc, SEQADDR1) << 8);

                /*
                 * Seqaddr represents the next instruction to execute,
                 * so we are really executing the instruction just
                 * before it.
                 */
                if (seqaddr != 0)
                        seqaddr -= 1;
                cs = ahc->critical_sections;
                for (i = 0; i < ahc->num_critical_sections; i++, cs++) {
                        if (cs->begin < seqaddr && cs->end >= seqaddr)
                                break;
                }

                if (i == ahc->num_critical_sections)

                        break;

                if (steps > AHC_MAX_STEPS) {
                        printk("%s: Infinite loop in critical section\n",
                               ahc_name(ahc));
                        ahc_dump_card_state(ahc);
                        panic("critical section loop");
                }

                steps++;
                if (stepping == FALSE) {

                        /*
                         * Disable all interrupt sources so that the
                         * sequencer will not be stuck by a pausing
                         * interrupt condition while we attempt to
                         * leave a critical section.
                         */
                        simode0 = ahc_inb(ahc, SIMODE0);
                        ahc_outb(ahc, SIMODE0, 0);
                        simode1 = ahc_inb(ahc, SIMODE1);
                        if ((ahc->features & AHC_DT) != 0)
                                /*
                                 * On DT class controllers, we
                                 * use the enhanced busfree logic.
                                 * Unfortunately we cannot re-enable
                                 * busfree detection within the
                                 * current connection, so we must
                                 * leave it on while single stepping.
                                 */
                                ahc_outb(ahc, SIMODE1, simode1 & ENBUSFREE);
                        else
                                ahc_outb(ahc, SIMODE1, 0);
                        ahc_outb(ahc, CLRINT, CLRSCSIINT);
                        ahc_outb(ahc, SEQCTL, ahc->seqctl | STEP);
                        stepping = TRUE;
                }
                if ((ahc->features & AHC_DT) != 0) {
                        ahc_outb(ahc, CLRSINT1, CLRBUSFREE);
                        ahc_outb(ahc, CLRINT, CLRSCSIINT);
                }
                ahc_outb(ahc, HCNTRL, ahc->unpause);
                while (!ahc_is_paused(ahc))
                        ahc_delay(200);
        }
        if (stepping) {
                ahc_outb(ahc, SIMODE0, simode0);
                ahc_outb(ahc, SIMODE1, simode1);
                ahc_outb(ahc, SEQCTL, ahc->seqctl);
        }
}

/*
 * Clear any pending interrupt status.
 */
static void
ahc_clear_intstat(struct ahc_softc *ahc)
{
        /* Clear any interrupt conditions this may have caused */
        ahc_outb(ahc, CLRSINT1, CLRSELTIMEO|CLRATNO|CLRSCSIRSTI
                                |CLRBUSFREE|CLRSCSIPERR|CLRPHASECHG|
                                CLRREQINIT);
        ahc_flush_device_writes(ahc);
        ahc_outb(ahc, CLRSINT0, CLRSELDO|CLRSELDI|CLRSELINGO);
        ahc_flush_device_writes(ahc);
        ahc_outb(ahc, CLRINT, CLRSCSIINT);
        ahc_flush_device_writes(ahc);
}

/**************************** Debugging Routines ******************************/
#ifdef AHC_DEBUG
uint32_t ahc_debug = AHC_DEBUG_OPTS;
#endif

#if 0 /* unused */
static void
ahc_print_scb(struct scb *scb)
{
        int i;

        struct hardware_scb *hscb = scb->hscb;

        printk("scb:%p control:0x%x scsiid:0x%x lun:%d cdb_len:%d\n",
               (void *)scb,
               hscb->control,
               hscb->scsiid,
               hscb->lun,
               hscb->cdb_len);
        printk("Shared Data: ");
        for (i = 0; i < sizeof(hscb->shared_data.cdb); i++)
                printk("%#02x", hscb->shared_data.cdb[i]);
        printk("        dataptr:%#x datacnt:%#x sgptr:%#x tag:%#x\n",
                ahc_le32toh(hscb->dataptr),
                ahc_le32toh(hscb->datacnt),
                ahc_le32toh(hscb->sgptr),
                hscb->tag);
        if (scb->sg_count > 0) {
                for (i = 0; i < scb->sg_count; i++) {
                        printk("sg[%d] - Addr 0x%x%x : Length %d\n",
                               i,
                               (ahc_le32toh(scb->sg_list[i].len) >> 24
                                & SG_HIGH_ADDR_BITS),
                               ahc_le32toh(scb->sg_list[i].addr),
                               ahc_le32toh(scb->sg_list[i].len));
                }
        }
}
#endif

/************************* Transfer Negotiation *******************************/
/*
 * Allocate per target mode instance (ID we respond to as a target)
 * transfer negotiation data structures.
 */
static struct ahc_tmode_tstate *
ahc_alloc_tstate(struct ahc_softc *ahc, u_int scsi_id, char channel)
{
        struct ahc_tmode_tstate *master_tstate;
        struct ahc_tmode_tstate *tstate;
        int i;

        master_tstate = ahc->enabled_targets[ahc->our_id];
        if (channel == 'B') {
                scsi_id += 8;
                master_tstate = ahc->enabled_targets[ahc->our_id_b + 8];
        }
        if (ahc->enabled_targets[scsi_id] != NULL
         && ahc->enabled_targets[scsi_id] != master_tstate)
                panic("%s: ahc_alloc_tstate - Target already allocated",
                      ahc_name(ahc));
        tstate = kmalloc(sizeof(*tstate), GFP_ATOMIC);
        if (tstate == NULL)
                return (NULL);

        /*
         * If we have allocated a master tstate, copy user settings from
         * the master tstate (taken from SRAM or the EEPROM) for this
         * channel, but reset our current and goal settings to async/narrow
         * until an initiator talks to us.
         */
        if (master_tstate != NULL) {
                memcpy(tstate, master_tstate, sizeof(*tstate));
                memset(tstate->enabled_luns, 0, sizeof(tstate->enabled_luns));
                tstate->ultraenb = 0;
                for (i = 0; i < AHC_NUM_TARGETS; i++) {
                        memset(&tstate->transinfo[i].curr, 0,
                              sizeof(tstate->transinfo[i].curr));
                        memset(&tstate->transinfo[i].goal, 0,
                              sizeof(tstate->transinfo[i].goal));
                }
        } else
                memset(tstate, 0, sizeof(*tstate));
        ahc->enabled_targets[scsi_id] = tstate;
        return (tstate);
}

#ifdef AHC_TARGET_MODE
/*
 * Free per target mode instance (ID we respond to as a target)
 * transfer negotiation data structures.
 */
static void
ahc_free_tstate(struct ahc_softc *ahc, u_int scsi_id, char channel, int force)
{
        struct ahc_tmode_tstate *tstate;

        /*
         * Don't clean up our "master" tstate.
         * It has our default user settings.
         */
        if (((channel == 'B' && scsi_id == ahc->our_id_b)
          || (channel == 'A' && scsi_id == ahc->our_id))
         && force == FALSE)
                return;

        if (channel == 'B')
                scsi_id += 8;
        tstate = ahc->enabled_targets[scsi_id];
        kfree(tstate);
        ahc->enabled_targets[scsi_id] = NULL;
}
#endif

/*
 * Called when we have an active connection to a target on the bus,
 * this function finds the nearest syncrate to the input period limited
 * by the capabilities of the bus connectivity of and sync settings for
 * the target.
 */
static const struct ahc_syncrate *
ahc_devlimited_syncrate(struct ahc_softc *ahc,
                        struct ahc_initiator_tinfo *tinfo,
                        u_int *period, u_int *ppr_options, role_t role)
{
        struct  ahc_transinfo *transinfo;
        u_int   maxsync;

        if ((ahc->features & AHC_ULTRA2) != 0) {
                if ((ahc_inb(ahc, SBLKCTL) & ENAB40) != 0
                 && (ahc_inb(ahc, SSTAT2) & EXP_ACTIVE) == 0) {
                        maxsync = AHC_SYNCRATE_DT;
                } else {
                        maxsync = AHC_SYNCRATE_ULTRA;
                        /* Can't do DT on an SE bus */
                        *ppr_options &= ~MSG_EXT_PPR_DT_REQ;
                }
        } else if ((ahc->features & AHC_ULTRA) != 0) {
                maxsync = AHC_SYNCRATE_ULTRA;
        } else {
                maxsync = AHC_SYNCRATE_FAST;
        }
        /*
         * Never allow a value higher than our current goal
         * period otherwise we may allow a target initiated
         * negotiation to go above the limit as set by the
         * user.  In the case of an initiator initiated
         * sync negotiation, we limit based on the user
         * setting.  This allows the system to still accept
         * incoming negotiations even if target initiated
         * negotiation is not performed.
         */
        if (role == ROLE_TARGET)
                transinfo = &tinfo->user;
        else
                transinfo = &tinfo->goal;
        *ppr_options &= transinfo->ppr_options;
        if (transinfo->width == MSG_EXT_WDTR_BUS_8_BIT) {
                maxsync = max(maxsync, (u_int)AHC_SYNCRATE_ULTRA2);
                *ppr_options &= ~MSG_EXT_PPR_DT_REQ;
        }
        if (transinfo->period == 0) {
                *period = 0;
                *ppr_options = 0;
                return (NULL);
        }
        *period = max(*period, (u_int)transinfo->period);
        return (ahc_find_syncrate(ahc, period, ppr_options, maxsync));
}

/*
 * Look up the valid period to SCSIRATE conversion in our table.
 * Return the period and offset that should be sent to the target
 * if this was the beginning of an SDTR.
 */
const struct ahc_syncrate *
ahc_find_syncrate(struct ahc_softc *ahc, u_int *period,
                  u_int *ppr_options, u_int maxsync)
{
        const struct ahc_syncrate *syncrate;

        if ((ahc->features & AHC_DT) == 0)
                *ppr_options &= ~MSG_EXT_PPR_DT_REQ;

        /* Skip all DT only entries if DT is not available */
        if ((*ppr_options & MSG_EXT_PPR_DT_REQ) == 0
         && maxsync < AHC_SYNCRATE_ULTRA2)
                maxsync = AHC_SYNCRATE_ULTRA2;

        /* Now set the maxsync based on the card capabilities
         * DT is already done above */
        if ((ahc->features & (AHC_DT | AHC_ULTRA2)) == 0
            && maxsync < AHC_SYNCRATE_ULTRA)
                maxsync = AHC_SYNCRATE_ULTRA;
        if ((ahc->features & (AHC_DT | AHC_ULTRA2 | AHC_ULTRA)) == 0
            && maxsync < AHC_SYNCRATE_FAST)
                maxsync = AHC_SYNCRATE_FAST;

        for (syncrate = &ahc_syncrates[maxsync];
             syncrate->rate != NULL;
             syncrate++) {

                /*
                 * The Ultra2 table doesn't go as low
                 * as for the Fast/Ultra cards.
                 */
                if ((ahc->features & AHC_ULTRA2) != 0
                 && (syncrate->sxfr_u2 == 0))
                        break;

                if (*period <= syncrate->period) {
                        /*
                         * When responding to a target that requests
                         * sync, the requested rate may fall between
                         * two rates that we can output, but still be
                         * a rate that we can receive.  Because of this,
                         * we want to respond to the target with
                         * the same rate that it sent to us even
                         * if the period we use to send data to it
                         * is lower.  Only lower the response period
                         * if we must.
                         */
                        if (syncrate == &ahc_syncrates[maxsync])
                                *period = syncrate->period;

                        /*
                         * At some speeds, we only support
                         * ST transfers.
                         */
                        if ((syncrate->sxfr_u2 & ST_SXFR) != 0)
                                *ppr_options &= ~MSG_EXT_PPR_DT_REQ;
                        break;
                }
        }

        if ((*period == 0)
         || (syncrate->rate == NULL)
         || ((ahc->features & AHC_ULTRA2) != 0
          && (syncrate->sxfr_u2 == 0))) {
                /* Use asynchronous transfers. */
                *period = 0;
                syncrate = NULL;
                *ppr_options &= ~MSG_EXT_PPR_DT_REQ;
        }
        return (syncrate);
}

/*
 * Convert from an entry in our syncrate table to the SCSI equivalent
 * sync "period" factor.
 */
u_int
ahc_find_period(struct ahc_softc *ahc, u_int scsirate, u_int maxsync)
{
        const struct ahc_syncrate *syncrate;

        if ((ahc->features & AHC_ULTRA2) != 0)
                scsirate &= SXFR_ULTRA2;
        else
                scsirate &= SXFR;

        /* now set maxsync based on card capabilities */
        if ((ahc->features & AHC_DT) == 0 && maxsync < AHC_SYNCRATE_ULTRA2)
                maxsync = AHC_SYNCRATE_ULTRA2;
        if ((ahc->features & (AHC_DT | AHC_ULTRA2)) == 0
            && maxsync < AHC_SYNCRATE_ULTRA)
                maxsync = AHC_SYNCRATE_ULTRA;
        if ((ahc->features & (AHC_DT | AHC_ULTRA2 | AHC_ULTRA)) == 0
            && maxsync < AHC_SYNCRATE_FAST)
                maxsync = AHC_SYNCRATE_FAST;


        syncrate = &ahc_syncrates[maxsync];
        while (syncrate->rate != NULL) {

                if ((ahc->features & AHC_ULTRA2) != 0) {
                        if (syncrate->sxfr_u2 == 0)
                                break;
                        else if (scsirate == (syncrate->sxfr_u2 & SXFR_ULTRA2))
                                return (syncrate->period);
                } else if (scsirate == (syncrate->sxfr & SXFR)) {
                                return (syncrate->period);
                }
                syncrate++;
        }
        return (0); /* async */
}

/*
 * Truncate the given synchronous offset to a value the
 * current adapter type and syncrate are capable of.
 */
static void
ahc_validate_offset(struct ahc_softc *ahc,
                    struct ahc_initiator_tinfo *tinfo,
                    const struct ahc_syncrate *syncrate,
                    u_int *offset, int wide, role_t role)
{
        u_int maxoffset;

        /* Limit offset to what we can do */
        if (syncrate == NULL) {
                maxoffset = 0;
        } else if ((ahc->features & AHC_ULTRA2) != 0) {
                maxoffset = MAX_OFFSET_ULTRA2;
        } else {
                if (wide)
                        maxoffset = MAX_OFFSET_16BIT;
                else
                        maxoffset = MAX_OFFSET_8BIT;
        }
        *offset = min(*offset, maxoffset);
        if (tinfo != NULL) {
                if (role == ROLE_TARGET)
                        *offset = min(*offset, (u_int)tinfo->user.offset);
                else
                        *offset = min(*offset, (u_int)tinfo->goal.offset);
        }
}

/*
 * Truncate the given transfer width parameter to a value the
 * current adapter type is capable of.
 */
static void
ahc_validate_width(struct ahc_softc *ahc, struct ahc_initiator_tinfo *tinfo,
                   u_int *bus_width, role_t role)
{
        switch (*bus_width) {
        default:
                if (ahc->features & AHC_WIDE) {
                        /* Respond Wide */
                        *bus_width = MSG_EXT_WDTR_BUS_16_BIT;
                        break;
                }
                fallthrough;
        case MSG_EXT_WDTR_BUS_8_BIT:
                *bus_width = MSG_EXT_WDTR_BUS_8_BIT;
                break;
        }
        if (tinfo != NULL) {
                if (role == ROLE_TARGET)
                        *bus_width = min((u_int)tinfo->user.width, *bus_width);
                else
                        *bus_width = min((u_int)tinfo->goal.width, *bus_width);
        }
}

/*
 * Update the bitmask of targets for which the controller should
 * negotiate with at the next convenient opportunity.  This currently
 * means the next time we send the initial identify messages for
 * a new transaction.
 */
int
ahc_update_neg_request(struct ahc_softc *ahc, struct ahc_devinfo *devinfo,
                       struct ahc_tmode_tstate *tstate,
                       struct ahc_initiator_tinfo *tinfo, ahc_neg_type neg_type)
{
        u_int auto_negotiate_orig;

        auto_negotiate_orig = tstate->auto_negotiate;
        if (neg_type == AHC_NEG_ALWAYS) {
                /*
                 * Force our "current" settings to be
                 * unknown so that unless a bus reset
                 * occurs the need to renegotiate is
                 * recorded persistently.
                 */
                if ((ahc->features & AHC_WIDE) != 0)
                        tinfo->curr.width = AHC_WIDTH_UNKNOWN;
                tinfo->curr.period = AHC_PERIOD_UNKNOWN;
                tinfo->curr.offset = AHC_OFFSET_UNKNOWN;
        }
        if (tinfo->curr.period != tinfo->goal.period
         || tinfo->curr.width != tinfo->goal.width
         || tinfo->curr.offset != tinfo->goal.offset
         || tinfo->curr.ppr_options != tinfo->goal.ppr_options
         || (neg_type == AHC_NEG_IF_NON_ASYNC
          && (tinfo->goal.offset != 0
           || tinfo->goal.width != MSG_EXT_WDTR_BUS_8_BIT
           || tinfo->goal.ppr_options != 0)))
                tstate->auto_negotiate |= devinfo->target_mask;
        else
                tstate->auto_negotiate &= ~devinfo->target_mask;

        return (auto_negotiate_orig != tstate->auto_negotiate);
}

/*
 * Update the user/goal/curr tables of synchronous negotiation
 * parameters as well as, in the case of a current or active update,
 * any data structures on the host controller.  In the case of an
 * active update, the specified target is currently talking to us on
 * the bus, so the transfer parameter update must take effect
 * immediately.
 */
void
ahc_set_syncrate(struct ahc_softc *ahc, struct ahc_devinfo *devinfo,
                 const struct ahc_syncrate *syncrate, u_int period,
                 u_int offset, u_int ppr_options, u_int type, int paused)
{
        struct  ahc_initiator_tinfo *tinfo;
        struct  ahc_tmode_tstate *tstate;
        u_int   old_period;
        u_int   old_offset;
        u_int   old_ppr;
        int     active;
        int     update_needed;

        active = (type & AHC_TRANS_ACTIVE) == AHC_TRANS_ACTIVE;
        update_needed = 0;

        if (syncrate == NULL) {
                period = 0;
                offset = 0;
        }

        tinfo = ahc_fetch_transinfo(ahc, devinfo->channel, devinfo->our_scsiid,
                                    devinfo->target, &tstate);

        if ((type & AHC_TRANS_USER) != 0) {
                tinfo->user.period = period;
                tinfo->user.offset = offset;
                tinfo->user.ppr_options = ppr_options;
        }

        if ((type & AHC_TRANS_GOAL) != 0) {
                tinfo->goal.period = period;
                tinfo->goal.offset = offset;
                tinfo->goal.ppr_options = ppr_options;
        }

        old_period = tinfo->curr.period;
        old_offset = tinfo->curr.offset;
        old_ppr    = tinfo->curr.ppr_options;

        if ((type & AHC_TRANS_CUR) != 0
         && (old_period != period
          || old_offset != offset
          || old_ppr != ppr_options)) {
                u_int   scsirate;

                update_needed++;
                scsirate = tinfo->scsirate;
                if ((ahc->features & AHC_ULTRA2) != 0) {

                        scsirate &= ~(SXFR_ULTRA2|SINGLE_EDGE|ENABLE_CRC);
                        if (syncrate != NULL) {
                                scsirate |= syncrate->sxfr_u2;
                                if ((ppr_options & MSG_EXT_PPR_DT_REQ) != 0)
                                        scsirate |= ENABLE_CRC;
                                else
                                        scsirate |= SINGLE_EDGE;
                        }
                } else {

                        scsirate &= ~(SXFR|SOFS);
                        /*
                         * Ensure Ultra mode is set properly for
                         * this target.
                         */
                        tstate->ultraenb &= ~devinfo->target_mask;
                        if (syncrate != NULL) {
                                if (syncrate->sxfr & ULTRA_SXFR) {
                                        tstate->ultraenb |=
                                                devinfo->target_mask;
                                }
                                scsirate |= syncrate->sxfr & SXFR;
                                scsirate |= offset & SOFS;
                        }
                        if (active) {
                                u_int sxfrctl0;

                                sxfrctl0 = ahc_inb(ahc, SXFRCTL0);
                                sxfrctl0 &= ~FAST20;
                                if (tstate->ultraenb & devinfo->target_mask)
                                        sxfrctl0 |= FAST20;
                                ahc_outb(ahc, SXFRCTL0, sxfrctl0);
                        }
                }
                if (active) {
                        ahc_outb(ahc, SCSIRATE, scsirate);
                        if ((ahc->features & AHC_ULTRA2) != 0)
                                ahc_outb(ahc, SCSIOFFSET, offset);
                }

                tinfo->scsirate = scsirate;
                tinfo->curr.period = period;
                tinfo->curr.offset = offset;
                tinfo->curr.ppr_options = ppr_options;

                ahc_send_async(ahc, devinfo->channel, devinfo->target,
                               CAM_LUN_WILDCARD, AC_TRANSFER_NEG);
                if (bootverbose) {
                        if (offset != 0) {
                                printk("%s: target %d synchronous at %sMHz%s, "
                                       "offset = 0x%x\n", ahc_name(ahc),
                                       devinfo->target, syncrate->rate,
                                       (ppr_options & MSG_EXT_PPR_DT_REQ)
                                       ? " DT" : "", offset);
                        } else {
                                printk("%s: target %d using "
                                       "asynchronous transfers\n",
                                       ahc_name(ahc), devinfo->target);
                        }
                }
        }

        update_needed += ahc_update_neg_request(ahc, devinfo, tstate,
                                                tinfo, AHC_NEG_TO_GOAL);

        if (update_needed)
                ahc_update_pending_scbs(ahc);
}

/*
 * Update the user/goal/curr tables of wide negotiation
 * parameters as well as, in the case of a current or active update,
 * any data structures on the host controller.  In the case of an
 * active update, the specified target is currently talking to us on
 * the bus, so the transfer parameter update must take effect
 * immediately.
 */
void
ahc_set_width(struct ahc_softc *ahc, struct ahc_devinfo *devinfo,
              u_int width, u_int type, int paused)
{
        struct  ahc_initiator_tinfo *tinfo;
        struct  ahc_tmode_tstate *tstate;
        u_int   oldwidth;
        int     active;
        int     update_needed;

        active = (type & AHC_TRANS_ACTIVE) == AHC_TRANS_ACTIVE;
        update_needed = 0;
        tinfo = ahc_fetch_transinfo(ahc, devinfo->channel, devinfo->our_scsiid,
                                    devinfo->target, &tstate);

        if ((type & AHC_TRANS_USER) != 0)
                tinfo->user.width = width;

        if ((type & AHC_TRANS_GOAL) != 0)
                tinfo->goal.width = width;

        oldwidth = tinfo->curr.width;
        if ((type & AHC_TRANS_CUR) != 0 && oldwidth != width) {
                u_int   scsirate;

                update_needed++;
                scsirate =  tinfo->scsirate;
                scsirate &= ~WIDEXFER;
                if (width == MSG_EXT_WDTR_BUS_16_BIT)
                        scsirate |= WIDEXFER;

                tinfo->scsirate = scsirate;

                if (active)
                        ahc_outb(ahc, SCSIRATE, scsirate);

                tinfo->curr.width = width;

                ahc_send_async(ahc, devinfo->channel, devinfo->target,
                               CAM_LUN_WILDCARD, AC_TRANSFER_NEG);
                if (bootverbose) {
                        printk("%s: target %d using %dbit transfers\n",
                               ahc_name(ahc), devinfo->target,
                               8 * (0x01 << width));
                }
        }

        update_needed += ahc_update_neg_request(ahc, devinfo, tstate,
                                                tinfo, AHC_NEG_TO_GOAL);
        if (update_needed)
                ahc_update_pending_scbs(ahc);
}

/*
 * Update the current state of tagged queuing for a given target.
 */
static void
ahc_set_tags(struct ahc_softc *ahc, struct scsi_cmnd *cmd,
             struct ahc_devinfo *devinfo, ahc_queue_alg alg)
{
        struct scsi_device *sdev = cmd->device;

        ahc_platform_set_tags(ahc, sdev, devinfo, alg);
        ahc_send_async(ahc, devinfo->channel, devinfo->target,
                       devinfo->lun, AC_TRANSFER_NEG);
}

/*
 * When the transfer settings for a connection change, update any
 * in-transit SCBs to contain the new data so the hardware will
 * be set correctly during future (re)selections.
 */
static void
ahc_update_pending_scbs(struct ahc_softc *ahc)
{
        struct  scb *pending_scb;
        int     pending_scb_count;
        int     i;
        int     paused;
        u_int   saved_scbptr;

        /*
         * Traverse the pending SCB list and ensure that all of the
         * SCBs there have the proper settings.
         */
        pending_scb_count = 0;
        LIST_FOREACH(pending_scb, &ahc->pending_scbs, pending_links) {
                struct ahc_devinfo devinfo;
                struct hardware_scb *pending_hscb;
                struct ahc_initiator_tinfo *tinfo;
                struct ahc_tmode_tstate *tstate;

                ahc_scb_devinfo(ahc, &devinfo, pending_scb);
                tinfo = ahc_fetch_transinfo(ahc, devinfo.channel,
                                            devinfo.our_scsiid,
                                            devinfo.target, &tstate);
                pending_hscb = pending_scb->hscb;
                pending_hscb->control &= ~ULTRAENB;
                if ((tstate->ultraenb & devinfo.target_mask) != 0)
                        pending_hscb->control |= ULTRAENB;
                pending_hscb->scsirate = tinfo->scsirate;
                pending_hscb->scsioffset = tinfo->curr.offset;
                if ((tstate->auto_negotiate & devinfo.target_mask) == 0
                 && (pending_scb->flags & SCB_AUTO_NEGOTIATE) != 0) {
                        pending_scb->flags &= ~SCB_AUTO_NEGOTIATE;
                        pending_hscb->control &= ~MK_MESSAGE;
                }
                ahc_sync_scb(ahc, pending_scb,
                             BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);
                pending_scb_count++;
        }

        if (pending_scb_count == 0)
                return;

        if (ahc_is_paused(ahc)) {
                paused = 1;
        } else {
                paused = 0;
                ahc_pause(ahc);
        }

        saved_scbptr = ahc_inb(ahc, SCBPTR);
        /* Ensure that the hscbs down on the card match the new information */
        for (i = 0; i < ahc->scb_data->maxhscbs; i++) {
                struct  hardware_scb *pending_hscb;
                u_int   control;
                u_int   scb_tag;

                ahc_outb(ahc, SCBPTR, i);
                scb_tag = ahc_inb(ahc, SCB_TAG);
                pending_scb = ahc_lookup_scb(ahc, scb_tag);
                if (pending_scb == NULL)
                        continue;

                pending_hscb = pending_scb->hscb;
                control = ahc_inb(ahc, SCB_CONTROL);
                control &= ~(ULTRAENB|MK_MESSAGE);
                control |= pending_hscb->control & (ULTRAENB|MK_MESSAGE);
                ahc_outb(ahc, SCB_CONTROL, control);
                ahc_outb(ahc, SCB_SCSIRATE, pending_hscb->scsirate);
                ahc_outb(ahc, SCB_SCSIOFFSET, pending_hscb->scsioffset);
        }
        ahc_outb(ahc, SCBPTR, saved_scbptr);

        if (paused == 0)
                ahc_unpause(ahc);
}

/**************************** Pathing Information *****************************/
static void
ahc_fetch_devinfo(struct ahc_softc *ahc, struct ahc_devinfo *devinfo)
{
        u_int   saved_scsiid;
        role_t  role;
        int     our_id;

        if (ahc_inb(ahc, SSTAT0) & TARGET)
                role = ROLE_TARGET;
        else
                role = ROLE_INITIATOR;

        if (role == ROLE_TARGET
            && (ahc->features & AHC_MULTI_TID) != 0
            && (ahc_inb(ahc, SEQ_FLAGS)
             & (CMDPHASE_PENDING|TARG_CMD_PENDING|NO_DISCONNECT)) != 0) {
                /* We were selected, so pull our id from TARGIDIN */
                our_id = ahc_inb(ahc, TARGIDIN) & OID;
        } else if ((ahc->features & AHC_ULTRA2) != 0)
                our_id = ahc_inb(ahc, SCSIID_ULTRA2) & OID;
        else
                our_id = ahc_inb(ahc, SCSIID) & OID;

        saved_scsiid = ahc_inb(ahc, SAVED_SCSIID);
        ahc_compile_devinfo(devinfo,
                            our_id,
                            SCSIID_TARGET(ahc, saved_scsiid),
                            ahc_inb(ahc, SAVED_LUN),
                            SCSIID_CHANNEL(ahc, saved_scsiid),
                            role);
}

static const struct ahc_phase_table_entry*
ahc_lookup_phase_entry(int phase)
{
        const struct ahc_phase_table_entry *entry;
        const struct ahc_phase_table_entry *last_entry;

        /*
         * num_phases doesn't include the default entry which
         * will be returned if the phase doesn't match.
         */
        last_entry = &ahc_phase_table[num_phases];
        for (entry = ahc_phase_table; entry < last_entry; entry++) {
                if (phase == entry->phase)
                        break;
        }
        return (entry);
}

void
ahc_compile_devinfo(struct ahc_devinfo *devinfo, u_int our_id, u_int target,
                    u_int lun, char channel, role_t role)
{
        devinfo->our_scsiid = our_id;
        devinfo->target = target;
        devinfo->lun = lun;
        devinfo->target_offset = target;
        devinfo->channel = channel;
        devinfo->role = role;
        if (channel == 'B')
                devinfo->target_offset += 8;
        devinfo->target_mask = (0x01 << devinfo->target_offset);
}

void
ahc_print_devinfo(struct ahc_softc *ahc, struct ahc_devinfo *devinfo)
{
        printk("%s:%c:%d:%d: ", ahc_name(ahc), devinfo->channel,
               devinfo->target, devinfo->lun);
}

static void
ahc_scb_devinfo(struct ahc_softc *ahc, struct ahc_devinfo *devinfo,
                struct scb *scb)
{
        role_t  role;
        int     our_id;

        our_id = SCSIID_OUR_ID(scb->hscb->scsiid);
        role = ROLE_INITIATOR;
        if ((scb->flags & SCB_TARGET_SCB) != 0)
                role = ROLE_TARGET;
        ahc_compile_devinfo(devinfo, our_id, SCB_GET_TARGET(ahc, scb),
                            SCB_GET_LUN(scb), SCB_GET_CHANNEL(ahc, scb), role);
}


/************************ Message Phase Processing ****************************/
static void
ahc_assert_atn(struct ahc_softc *ahc)
{
        u_int scsisigo;

        scsisigo = ATNO;
        if ((ahc->features & AHC_DT) == 0)
                scsisigo |= ahc_inb(ahc, SCSISIGI);
        ahc_outb(ahc, SCSISIGO, scsisigo);
}

/*
 * When an initiator transaction with the MK_MESSAGE flag either reconnects
 * or enters the initial message out phase, we are interrupted.  Fill our
 * outgoing message buffer with the appropriate message and beging handing
 * the message phase(s) manually.
 */
static void
ahc_setup_initiator_msgout(struct ahc_softc *ahc, struct ahc_devinfo *devinfo,
                           struct scb *scb)
{
        /*
         * To facilitate adding multiple messages together,
         * each routine should increment the index and len
         * variables instead of setting them explicitly.
         */
        ahc->msgout_index = 0;
        ahc->msgout_len = 0;

        if ((scb->flags & SCB_DEVICE_RESET) == 0
         && ahc_inb(ahc, MSG_OUT) == MSG_IDENTIFYFLAG) {
                u_int identify_msg;

                identify_msg = MSG_IDENTIFYFLAG | SCB_GET_LUN(scb);
                if ((scb->hscb->control & DISCENB) != 0)
                        identify_msg |= MSG_IDENTIFY_DISCFLAG;
                ahc->msgout_buf[ahc->msgout_index++] = identify_msg;
                ahc->msgout_len++;

                if ((scb->hscb->control & TAG_ENB) != 0) {
                        ahc->msgout_buf[ahc->msgout_index++] =
                            scb->hscb->control & (TAG_ENB|SCB_TAG_TYPE);
                        ahc->msgout_buf[ahc->msgout_index++] = scb->hscb->tag;
                        ahc->msgout_len += 2;
                }
        }

        if (scb->flags & SCB_DEVICE_RESET) {
                ahc->msgout_buf[ahc->msgout_index++] = TARGET_RESET;
                ahc->msgout_len++;
                ahc_print_path(ahc, scb);
                printk("Bus Device Reset Message Sent\n");
                /*
                 * Clear our selection hardware in advance of
                 * the busfree.  We may have an entry in the waiting
                 * Q for this target, and we don't want to go about
                 * selecting while we handle the busfree and blow it
                 * away.
                 */
                ahc_outb(ahc, SCSISEQ, (ahc_inb(ahc, SCSISEQ) & ~ENSELO));
        } else if ((scb->flags & SCB_ABORT) != 0) {
                if ((scb->hscb->control & TAG_ENB) != 0)
                        ahc->msgout_buf[ahc->msgout_index++] = ABORT_TASK;
                else
                        ahc->msgout_buf[ahc->msgout_index++] = ABORT_TASK_SET;
                ahc->msgout_len++;
                ahc_print_path(ahc, scb);
                printk("Abort%s Message Sent\n",
                       (scb->hscb->control & TAG_ENB) != 0 ? " Tag" : "");
                /*
                 * Clear our selection hardware in advance of
                 * the busfree.  We may have an entry in the waiting
                 * Q for this target, and we don't want to go about
                 * selecting while we handle the busfree and blow it
                 * away.
                 */
                ahc_outb(ahc, SCSISEQ, (ahc_inb(ahc, SCSISEQ) & ~ENSELO));
        } else if ((scb->flags & (SCB_AUTO_NEGOTIATE|SCB_NEGOTIATE)) != 0) {
                ahc_build_transfer_msg(ahc, devinfo);
        } else {
                printk("ahc_intr: AWAITING_MSG for an SCB that "
                       "does not have a waiting message\n");
                printk("SCSIID = %x, target_mask = %x\n", scb->hscb->scsiid,
                       devinfo->target_mask);
                panic("SCB = %d, SCB Control = %x, MSG_OUT = %x "
                      "SCB flags = %x", scb->hscb->tag, scb->hscb->control,
                      ahc_inb(ahc, MSG_OUT), scb->flags);
        }

        /*
         * Clear the MK_MESSAGE flag from the SCB so we aren't
         * asked to send this message again.
         */
        ahc_outb(ahc, SCB_CONTROL, ahc_inb(ahc, SCB_CONTROL) & ~MK_MESSAGE);
        scb->hscb->control &= ~MK_MESSAGE;
        ahc->msgout_index = 0;
        ahc->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
}

/*
 * Build an appropriate transfer negotiation message for the
 * currently active target.
 */
static void
ahc_build_transfer_msg(struct ahc_softc *ahc, struct ahc_devinfo *devinfo)
{
        /*
         * We need to initiate transfer negotiations.
         * If our current and goal settings are identical,
         * we want to renegotiate due to a check condition.
         */
        struct  ahc_initiator_tinfo *tinfo;
        struct  ahc_tmode_tstate *tstate;
        const struct ahc_syncrate *rate;
        int     dowide;
        int     dosync;
        int     doppr;
        u_int   period;
        u_int   ppr_options;
        u_int   offset;

        tinfo = ahc_fetch_transinfo(ahc, devinfo->channel, devinfo->our_scsiid,
                                    devinfo->target, &tstate);
        /*
         * Filter our period based on the current connection.
         * If we can't perform DT transfers on this segment (not in LVD
         * mode for instance), then our decision to issue a PPR message
         * may change.
         */
        period = tinfo->goal.period;
        offset = tinfo->goal.offset;
        ppr_options = tinfo->goal.ppr_options;
        /* Target initiated PPR is not allowed in the SCSI spec */
        if (devinfo->role == ROLE_TARGET)
                ppr_options = 0;
        rate = ahc_devlimited_syncrate(ahc, tinfo, &period,
                                       &ppr_options, devinfo->role);
        dowide = tinfo->curr.width != tinfo->goal.width;
        dosync = tinfo->curr.offset != offset || tinfo->curr.period != period;
        /*
         * Only use PPR if we have options that need it, even if the device
         * claims to support it.  There might be an expander in the way
         * that doesn't.
         */
        doppr = ppr_options != 0;

        if (!dowide && !dosync && !doppr) {
                dowide = tinfo->goal.width != MSG_EXT_WDTR_BUS_8_BIT;
                dosync = tinfo->goal.offset != 0;
        }

        if (!dowide && !dosync && !doppr) {
                /*
                 * Force async with a WDTR message if we have a wide bus,
                 * or just issue an SDTR with a 0 offset.
                 */
                if ((ahc->features & AHC_WIDE) != 0)
                        dowide = 1;
                else
                        dosync = 1;

                if (bootverbose) {
                        ahc_print_devinfo(ahc, devinfo);
                        printk("Ensuring async\n");
                }
        }

        /* Target initiated PPR is not allowed in the SCSI spec */