xref: /linux/drivers/scsi/aic7xxx/aic79xx_core.c (revision af50e4ba34f4c45e92535364133d4deb5931c1c5)
1 /*
2  * Core routines and tables shareable across OS platforms.
3  *
4  * Copyright (c) 1994-2002 Justin T. Gibbs.
5  * Copyright (c) 2000-2003 Adaptec Inc.
6  * All rights reserved.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  * 1. Redistributions of source code must retain the above copyright
12  *    notice, this list of conditions, and the following disclaimer,
13  *    without modification.
14  * 2. Redistributions in binary form must reproduce at minimum a disclaimer
15  *    substantially similar to the "NO WARRANTY" disclaimer below
16  *    ("Disclaimer") and any redistribution must be conditioned upon
17  *    including a substantially similar Disclaimer requirement for further
18  *    binary redistribution.
19  * 3. Neither the names of the above-listed copyright holders nor the names
20  *    of any contributors may be used to endorse or promote products derived
21  *    from this software without specific prior written permission.
22  *
23  * Alternatively, this software may be distributed under the terms of the
24  * GNU General Public License ("GPL") version 2 as published by the Free
25  * Software Foundation.
26  *
27  * NO WARRANTY
28  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
29  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
30  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR
31  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
32  * HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
33  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
34  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
35  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
36  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
37  * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
38  * POSSIBILITY OF SUCH DAMAGES.
39  *
40  * $Id: //depot/aic7xxx/aic7xxx/aic79xx.c#250 $
41  */
42 
43 #ifdef __linux__
44 #include "aic79xx_osm.h"
45 #include "aic79xx_inline.h"
46 #include "aicasm/aicasm_insformat.h"
47 #else
48 #include <dev/aic7xxx/aic79xx_osm.h>
49 #include <dev/aic7xxx/aic79xx_inline.h>
50 #include <dev/aic7xxx/aicasm/aicasm_insformat.h>
51 #endif
52 
53 
54 /***************************** Lookup Tables **********************************/
55 static const char *const ahd_chip_names[] =
56 {
57 	"NONE",
58 	"aic7901",
59 	"aic7902",
60 	"aic7901A"
61 };
62 static const u_int num_chip_names = ARRAY_SIZE(ahd_chip_names);
63 
64 /*
65  * Hardware error codes.
66  */
67 struct ahd_hard_error_entry {
68         uint8_t errno;
69 	const char *errmesg;
70 };
71 
72 static const struct ahd_hard_error_entry ahd_hard_errors[] = {
73 	{ DSCTMOUT,	"Discard Timer has timed out" },
74 	{ ILLOPCODE,	"Illegal Opcode in sequencer program" },
75 	{ SQPARERR,	"Sequencer Parity Error" },
76 	{ DPARERR,	"Data-path Parity Error" },
77 	{ MPARERR,	"Scratch or SCB Memory Parity Error" },
78 	{ CIOPARERR,	"CIOBUS Parity Error" },
79 };
80 static const u_int num_errors = ARRAY_SIZE(ahd_hard_errors);
81 
82 static const struct ahd_phase_table_entry ahd_phase_table[] =
83 {
84 	{ P_DATAOUT,	MSG_NOOP,		"in Data-out phase"	},
85 	{ P_DATAIN,	MSG_INITIATOR_DET_ERR,	"in Data-in phase"	},
86 	{ P_DATAOUT_DT,	MSG_NOOP,		"in DT Data-out phase"	},
87 	{ P_DATAIN_DT,	MSG_INITIATOR_DET_ERR,	"in DT Data-in phase"	},
88 	{ P_COMMAND,	MSG_NOOP,		"in Command phase"	},
89 	{ P_MESGOUT,	MSG_NOOP,		"in Message-out phase"	},
90 	{ P_STATUS,	MSG_INITIATOR_DET_ERR,	"in Status phase"	},
91 	{ P_MESGIN,	MSG_PARITY_ERROR,	"in Message-in phase"	},
92 	{ P_BUSFREE,	MSG_NOOP,		"while idle"		},
93 	{ 0,		MSG_NOOP,		"in unknown phase"	}
94 };
95 
96 /*
97  * In most cases we only wish to itterate over real phases, so
98  * exclude the last element from the count.
99  */
100 static const u_int num_phases = ARRAY_SIZE(ahd_phase_table) - 1;
101 
102 /* Our Sequencer Program */
103 #include "aic79xx_seq.h"
104 
105 /**************************** Function Declarations ***************************/
106 static void		ahd_handle_transmission_error(struct ahd_softc *ahd);
107 static void		ahd_handle_lqiphase_error(struct ahd_softc *ahd,
108 						  u_int lqistat1);
109 static int		ahd_handle_pkt_busfree(struct ahd_softc *ahd,
110 					       u_int busfreetime);
111 static int		ahd_handle_nonpkt_busfree(struct ahd_softc *ahd);
112 static void		ahd_handle_proto_violation(struct ahd_softc *ahd);
113 static void		ahd_force_renegotiation(struct ahd_softc *ahd,
114 						struct ahd_devinfo *devinfo);
115 
116 static struct ahd_tmode_tstate*
117 			ahd_alloc_tstate(struct ahd_softc *ahd,
118 					 u_int scsi_id, char channel);
119 #ifdef AHD_TARGET_MODE
120 static void		ahd_free_tstate(struct ahd_softc *ahd,
121 					u_int scsi_id, char channel, int force);
122 #endif
123 static void		ahd_devlimited_syncrate(struct ahd_softc *ahd,
124 					        struct ahd_initiator_tinfo *,
125 						u_int *period,
126 						u_int *ppr_options,
127 						role_t role);
128 static void		ahd_update_neg_table(struct ahd_softc *ahd,
129 					     struct ahd_devinfo *devinfo,
130 					     struct ahd_transinfo *tinfo);
131 static void		ahd_update_pending_scbs(struct ahd_softc *ahd);
132 static void		ahd_fetch_devinfo(struct ahd_softc *ahd,
133 					  struct ahd_devinfo *devinfo);
134 static void		ahd_scb_devinfo(struct ahd_softc *ahd,
135 					struct ahd_devinfo *devinfo,
136 					struct scb *scb);
137 static void		ahd_setup_initiator_msgout(struct ahd_softc *ahd,
138 						   struct ahd_devinfo *devinfo,
139 						   struct scb *scb);
140 static void		ahd_build_transfer_msg(struct ahd_softc *ahd,
141 					       struct ahd_devinfo *devinfo);
142 static void		ahd_construct_sdtr(struct ahd_softc *ahd,
143 					   struct ahd_devinfo *devinfo,
144 					   u_int period, u_int offset);
145 static void		ahd_construct_wdtr(struct ahd_softc *ahd,
146 					   struct ahd_devinfo *devinfo,
147 					   u_int bus_width);
148 static void		ahd_construct_ppr(struct ahd_softc *ahd,
149 					  struct ahd_devinfo *devinfo,
150 					  u_int period, u_int offset,
151 					  u_int bus_width, u_int ppr_options);
152 static void		ahd_clear_msg_state(struct ahd_softc *ahd);
153 static void		ahd_handle_message_phase(struct ahd_softc *ahd);
154 typedef enum {
155 	AHDMSG_1B,
156 	AHDMSG_2B,
157 	AHDMSG_EXT
158 } ahd_msgtype;
159 static int		ahd_sent_msg(struct ahd_softc *ahd, ahd_msgtype type,
160 				     u_int msgval, int full);
161 static int		ahd_parse_msg(struct ahd_softc *ahd,
162 				      struct ahd_devinfo *devinfo);
163 static int		ahd_handle_msg_reject(struct ahd_softc *ahd,
164 					      struct ahd_devinfo *devinfo);
165 static void		ahd_handle_ign_wide_residue(struct ahd_softc *ahd,
166 						struct ahd_devinfo *devinfo);
167 static void		ahd_reinitialize_dataptrs(struct ahd_softc *ahd);
168 static void		ahd_handle_devreset(struct ahd_softc *ahd,
169 					    struct ahd_devinfo *devinfo,
170 					    u_int lun, cam_status status,
171 					    char *message, int verbose_level);
172 #ifdef AHD_TARGET_MODE
173 static void		ahd_setup_target_msgin(struct ahd_softc *ahd,
174 					       struct ahd_devinfo *devinfo,
175 					       struct scb *scb);
176 #endif
177 
178 static u_int		ahd_sglist_size(struct ahd_softc *ahd);
179 static u_int		ahd_sglist_allocsize(struct ahd_softc *ahd);
180 static bus_dmamap_callback_t
181 			ahd_dmamap_cb;
182 static void		ahd_initialize_hscbs(struct ahd_softc *ahd);
183 static int		ahd_init_scbdata(struct ahd_softc *ahd);
184 static void		ahd_fini_scbdata(struct ahd_softc *ahd);
185 static void		ahd_setup_iocell_workaround(struct ahd_softc *ahd);
186 static void		ahd_iocell_first_selection(struct ahd_softc *ahd);
187 static void		ahd_add_col_list(struct ahd_softc *ahd,
188 					 struct scb *scb, u_int col_idx);
189 static void		ahd_rem_col_list(struct ahd_softc *ahd,
190 					 struct scb *scb);
191 static void		ahd_chip_init(struct ahd_softc *ahd);
192 static void		ahd_qinfifo_requeue(struct ahd_softc *ahd,
193 					    struct scb *prev_scb,
194 					    struct scb *scb);
195 static int		ahd_qinfifo_count(struct ahd_softc *ahd);
196 static int		ahd_search_scb_list(struct ahd_softc *ahd, int target,
197 					    char channel, int lun, u_int tag,
198 					    role_t role, uint32_t status,
199 					    ahd_search_action action,
200 					    u_int *list_head, u_int *list_tail,
201 					    u_int tid);
202 static void		ahd_stitch_tid_list(struct ahd_softc *ahd,
203 					    u_int tid_prev, u_int tid_cur,
204 					    u_int tid_next);
205 static void		ahd_add_scb_to_free_list(struct ahd_softc *ahd,
206 						 u_int scbid);
207 static u_int		ahd_rem_wscb(struct ahd_softc *ahd, u_int scbid,
208 				     u_int prev, u_int next, u_int tid);
209 static void		ahd_reset_current_bus(struct ahd_softc *ahd);
210 static void		ahd_stat_timer(struct timer_list *t);
211 #ifdef AHD_DUMP_SEQ
212 static void		ahd_dumpseq(struct ahd_softc *ahd);
213 #endif
214 static void		ahd_loadseq(struct ahd_softc *ahd);
215 static int		ahd_check_patch(struct ahd_softc *ahd,
216 					const struct patch **start_patch,
217 					u_int start_instr, u_int *skip_addr);
218 static u_int		ahd_resolve_seqaddr(struct ahd_softc *ahd,
219 					    u_int address);
220 static void		ahd_download_instr(struct ahd_softc *ahd,
221 					   u_int instrptr, uint8_t *dconsts);
222 static int		ahd_probe_stack_size(struct ahd_softc *ahd);
223 static int		ahd_scb_active_in_fifo(struct ahd_softc *ahd,
224 					       struct scb *scb);
225 static void		ahd_run_data_fifo(struct ahd_softc *ahd,
226 					  struct scb *scb);
227 
228 #ifdef AHD_TARGET_MODE
229 static void		ahd_queue_lstate_event(struct ahd_softc *ahd,
230 					       struct ahd_tmode_lstate *lstate,
231 					       u_int initiator_id,
232 					       u_int event_type,
233 					       u_int event_arg);
234 static void		ahd_update_scsiid(struct ahd_softc *ahd,
235 					  u_int targid_mask);
236 static int		ahd_handle_target_cmd(struct ahd_softc *ahd,
237 					      struct target_cmd *cmd);
238 #endif
239 
240 static int		ahd_abort_scbs(struct ahd_softc *ahd, int target,
241 				       char channel, int lun, u_int tag,
242 				       role_t role, uint32_t status);
243 static void		ahd_alloc_scbs(struct ahd_softc *ahd);
244 static void		ahd_busy_tcl(struct ahd_softc *ahd, u_int tcl,
245 				     u_int scbid);
246 static void		ahd_calc_residual(struct ahd_softc *ahd,
247 					  struct scb *scb);
248 static void		ahd_clear_critical_section(struct ahd_softc *ahd);
249 static void		ahd_clear_intstat(struct ahd_softc *ahd);
250 static void		ahd_enable_coalescing(struct ahd_softc *ahd,
251 					      int enable);
252 static u_int		ahd_find_busy_tcl(struct ahd_softc *ahd, u_int tcl);
253 static void		ahd_freeze_devq(struct ahd_softc *ahd,
254 					struct scb *scb);
255 static void		ahd_handle_scb_status(struct ahd_softc *ahd,
256 					      struct scb *scb);
257 static const struct ahd_phase_table_entry* ahd_lookup_phase_entry(int phase);
258 static void		ahd_shutdown(void *arg);
259 static void		ahd_update_coalescing_values(struct ahd_softc *ahd,
260 						     u_int timer,
261 						     u_int maxcmds,
262 						     u_int mincmds);
263 static int		ahd_verify_vpd_cksum(struct vpd_config *vpd);
264 static int		ahd_wait_seeprom(struct ahd_softc *ahd);
265 static int		ahd_match_scb(struct ahd_softc *ahd, struct scb *scb,
266 				      int target, char channel, int lun,
267 				      u_int tag, role_t role);
268 
269 static void		ahd_reset_cmds_pending(struct ahd_softc *ahd);
270 
271 /*************************** Interrupt Services *******************************/
272 static void		ahd_run_qoutfifo(struct ahd_softc *ahd);
273 #ifdef AHD_TARGET_MODE
274 static void		ahd_run_tqinfifo(struct ahd_softc *ahd, int paused);
275 #endif
276 static void		ahd_handle_hwerrint(struct ahd_softc *ahd);
277 static void		ahd_handle_seqint(struct ahd_softc *ahd, u_int intstat);
278 static void		ahd_handle_scsiint(struct ahd_softc *ahd,
279 				           u_int intstat);
280 
281 /************************ Sequencer Execution Control *************************/
282 void
283 ahd_set_modes(struct ahd_softc *ahd, ahd_mode src, ahd_mode dst)
284 {
285 	if (ahd->src_mode == src && ahd->dst_mode == dst)
286 		return;
287 #ifdef AHD_DEBUG
288 	if (ahd->src_mode == AHD_MODE_UNKNOWN
289 	 || ahd->dst_mode == AHD_MODE_UNKNOWN)
290 		panic("Setting mode prior to saving it.\n");
291 	if ((ahd_debug & AHD_SHOW_MODEPTR) != 0)
292 		printk("%s: Setting mode 0x%x\n", ahd_name(ahd),
293 		       ahd_build_mode_state(ahd, src, dst));
294 #endif
295 	ahd_outb(ahd, MODE_PTR, ahd_build_mode_state(ahd, src, dst));
296 	ahd->src_mode = src;
297 	ahd->dst_mode = dst;
298 }
299 
300 static void
301 ahd_update_modes(struct ahd_softc *ahd)
302 {
303 	ahd_mode_state mode_ptr;
304 	ahd_mode src;
305 	ahd_mode dst;
306 
307 	mode_ptr = ahd_inb(ahd, MODE_PTR);
308 #ifdef AHD_DEBUG
309 	if ((ahd_debug & AHD_SHOW_MODEPTR) != 0)
310 		printk("Reading mode 0x%x\n", mode_ptr);
311 #endif
312 	ahd_extract_mode_state(ahd, mode_ptr, &src, &dst);
313 	ahd_known_modes(ahd, src, dst);
314 }
315 
316 static void
317 ahd_assert_modes(struct ahd_softc *ahd, ahd_mode srcmode,
318 		 ahd_mode dstmode, const char *file, int line)
319 {
320 #ifdef AHD_DEBUG
321 	if ((srcmode & AHD_MK_MSK(ahd->src_mode)) == 0
322 	 || (dstmode & AHD_MK_MSK(ahd->dst_mode)) == 0) {
323 		panic("%s:%s:%d: Mode assertion failed.\n",
324 		       ahd_name(ahd), file, line);
325 	}
326 #endif
327 }
328 
329 #define AHD_ASSERT_MODES(ahd, source, dest) \
330 	ahd_assert_modes(ahd, source, dest, __FILE__, __LINE__);
331 
332 ahd_mode_state
333 ahd_save_modes(struct ahd_softc *ahd)
334 {
335 	if (ahd->src_mode == AHD_MODE_UNKNOWN
336 	 || ahd->dst_mode == AHD_MODE_UNKNOWN)
337 		ahd_update_modes(ahd);
338 
339 	return (ahd_build_mode_state(ahd, ahd->src_mode, ahd->dst_mode));
340 }
341 
342 void
343 ahd_restore_modes(struct ahd_softc *ahd, ahd_mode_state state)
344 {
345 	ahd_mode src;
346 	ahd_mode dst;
347 
348 	ahd_extract_mode_state(ahd, state, &src, &dst);
349 	ahd_set_modes(ahd, src, dst);
350 }
351 
352 /*
353  * Determine whether the sequencer has halted code execution.
354  * Returns non-zero status if the sequencer is stopped.
355  */
356 int
357 ahd_is_paused(struct ahd_softc *ahd)
358 {
359 	return ((ahd_inb(ahd, HCNTRL) & PAUSE) != 0);
360 }
361 
362 /*
363  * Request that the sequencer stop and wait, indefinitely, for it
364  * to stop.  The sequencer will only acknowledge that it is paused
365  * once it has reached an instruction boundary and PAUSEDIS is
366  * cleared in the SEQCTL register.  The sequencer may use PAUSEDIS
367  * for critical sections.
368  */
369 void
370 ahd_pause(struct ahd_softc *ahd)
371 {
372 	ahd_outb(ahd, HCNTRL, ahd->pause);
373 
374 	/*
375 	 * Since the sequencer can disable pausing in a critical section, we
376 	 * must loop until it actually stops.
377 	 */
378 	while (ahd_is_paused(ahd) == 0)
379 		;
380 }
381 
382 /*
383  * Allow the sequencer to continue program execution.
384  * We check here to ensure that no additional interrupt
385  * sources that would cause the sequencer to halt have been
386  * asserted.  If, for example, a SCSI bus reset is detected
387  * while we are fielding a different, pausing, interrupt type,
388  * we don't want to release the sequencer before going back
389  * into our interrupt handler and dealing with this new
390  * condition.
391  */
392 void
393 ahd_unpause(struct ahd_softc *ahd)
394 {
395 	/*
396 	 * Automatically restore our modes to those saved
397 	 * prior to the first change of the mode.
398 	 */
399 	if (ahd->saved_src_mode != AHD_MODE_UNKNOWN
400 	 && ahd->saved_dst_mode != AHD_MODE_UNKNOWN) {
401 		if ((ahd->flags & AHD_UPDATE_PEND_CMDS) != 0)
402 			ahd_reset_cmds_pending(ahd);
403 		ahd_set_modes(ahd, ahd->saved_src_mode, ahd->saved_dst_mode);
404 	}
405 
406 	if ((ahd_inb(ahd, INTSTAT) & ~CMDCMPLT) == 0)
407 		ahd_outb(ahd, HCNTRL, ahd->unpause);
408 
409 	ahd_known_modes(ahd, AHD_MODE_UNKNOWN, AHD_MODE_UNKNOWN);
410 }
411 
412 /*********************** Scatter Gather List Handling *************************/
413 void *
414 ahd_sg_setup(struct ahd_softc *ahd, struct scb *scb,
415 	     void *sgptr, dma_addr_t addr, bus_size_t len, int last)
416 {
417 	scb->sg_count++;
418 	if (sizeof(dma_addr_t) > 4
419 	 && (ahd->flags & AHD_64BIT_ADDRESSING) != 0) {
420 		struct ahd_dma64_seg *sg;
421 
422 		sg = (struct ahd_dma64_seg *)sgptr;
423 		sg->addr = ahd_htole64(addr);
424 		sg->len = ahd_htole32(len | (last ? AHD_DMA_LAST_SEG : 0));
425 		return (sg + 1);
426 	} else {
427 		struct ahd_dma_seg *sg;
428 
429 		sg = (struct ahd_dma_seg *)sgptr;
430 		sg->addr = ahd_htole32(addr & 0xFFFFFFFF);
431 		sg->len = ahd_htole32(len | ((addr >> 8) & 0x7F000000)
432 				    | (last ? AHD_DMA_LAST_SEG : 0));
433 		return (sg + 1);
434 	}
435 }
436 
437 static void
438 ahd_setup_scb_common(struct ahd_softc *ahd, struct scb *scb)
439 {
440 	/* XXX Handle target mode SCBs. */
441 	scb->crc_retry_count = 0;
442 	if ((scb->flags & SCB_PACKETIZED) != 0) {
443 		/* XXX what about ACA??  It is type 4, but TAG_TYPE == 0x3. */
444 		scb->hscb->task_attribute = scb->hscb->control & SCB_TAG_TYPE;
445 	} else {
446 		if (ahd_get_transfer_length(scb) & 0x01)
447 			scb->hscb->task_attribute = SCB_XFERLEN_ODD;
448 		else
449 			scb->hscb->task_attribute = 0;
450 	}
451 
452 	if (scb->hscb->cdb_len <= MAX_CDB_LEN_WITH_SENSE_ADDR
453 	 || (scb->hscb->cdb_len & SCB_CDB_LEN_PTR) != 0)
454 		scb->hscb->shared_data.idata.cdb_plus_saddr.sense_addr =
455 		    ahd_htole32(scb->sense_busaddr);
456 }
457 
458 static void
459 ahd_setup_data_scb(struct ahd_softc *ahd, struct scb *scb)
460 {
461 	/*
462 	 * Copy the first SG into the "current" data ponter area.
463 	 */
464 	if ((ahd->flags & AHD_64BIT_ADDRESSING) != 0) {
465 		struct ahd_dma64_seg *sg;
466 
467 		sg = (struct ahd_dma64_seg *)scb->sg_list;
468 		scb->hscb->dataptr = sg->addr;
469 		scb->hscb->datacnt = sg->len;
470 	} else {
471 		struct ahd_dma_seg *sg;
472 		uint32_t *dataptr_words;
473 
474 		sg = (struct ahd_dma_seg *)scb->sg_list;
475 		dataptr_words = (uint32_t*)&scb->hscb->dataptr;
476 		dataptr_words[0] = sg->addr;
477 		dataptr_words[1] = 0;
478 		if ((ahd->flags & AHD_39BIT_ADDRESSING) != 0) {
479 			uint64_t high_addr;
480 
481 			high_addr = ahd_le32toh(sg->len) & 0x7F000000;
482 			scb->hscb->dataptr |= ahd_htole64(high_addr << 8);
483 		}
484 		scb->hscb->datacnt = sg->len;
485 	}
486 	/*
487 	 * Note where to find the SG entries in bus space.
488 	 * We also set the full residual flag which the
489 	 * sequencer will clear as soon as a data transfer
490 	 * occurs.
491 	 */
492 	scb->hscb->sgptr = ahd_htole32(scb->sg_list_busaddr|SG_FULL_RESID);
493 }
494 
495 static void
496 ahd_setup_noxfer_scb(struct ahd_softc *ahd, struct scb *scb)
497 {
498 	scb->hscb->sgptr = ahd_htole32(SG_LIST_NULL);
499 	scb->hscb->dataptr = 0;
500 	scb->hscb->datacnt = 0;
501 }
502 
503 /************************** Memory mapping routines ***************************/
504 static void *
505 ahd_sg_bus_to_virt(struct ahd_softc *ahd, struct scb *scb, uint32_t sg_busaddr)
506 {
507 	dma_addr_t sg_offset;
508 
509 	/* sg_list_phys points to entry 1, not 0 */
510 	sg_offset = sg_busaddr - (scb->sg_list_busaddr - ahd_sg_size(ahd));
511 	return ((uint8_t *)scb->sg_list + sg_offset);
512 }
513 
514 static uint32_t
515 ahd_sg_virt_to_bus(struct ahd_softc *ahd, struct scb *scb, void *sg)
516 {
517 	dma_addr_t sg_offset;
518 
519 	/* sg_list_phys points to entry 1, not 0 */
520 	sg_offset = ((uint8_t *)sg - (uint8_t *)scb->sg_list)
521 		  - ahd_sg_size(ahd);
522 
523 	return (scb->sg_list_busaddr + sg_offset);
524 }
525 
526 static void
527 ahd_sync_scb(struct ahd_softc *ahd, struct scb *scb, int op)
528 {
529 	ahd_dmamap_sync(ahd, ahd->scb_data.hscb_dmat,
530 			scb->hscb_map->dmamap,
531 			/*offset*/(uint8_t*)scb->hscb - scb->hscb_map->vaddr,
532 			/*len*/sizeof(*scb->hscb), op);
533 }
534 
535 void
536 ahd_sync_sglist(struct ahd_softc *ahd, struct scb *scb, int op)
537 {
538 	if (scb->sg_count == 0)
539 		return;
540 
541 	ahd_dmamap_sync(ahd, ahd->scb_data.sg_dmat,
542 			scb->sg_map->dmamap,
543 			/*offset*/scb->sg_list_busaddr - ahd_sg_size(ahd),
544 			/*len*/ahd_sg_size(ahd) * scb->sg_count, op);
545 }
546 
547 static void
548 ahd_sync_sense(struct ahd_softc *ahd, struct scb *scb, int op)
549 {
550 	ahd_dmamap_sync(ahd, ahd->scb_data.sense_dmat,
551 			scb->sense_map->dmamap,
552 			/*offset*/scb->sense_busaddr,
553 			/*len*/AHD_SENSE_BUFSIZE, op);
554 }
555 
556 #ifdef AHD_TARGET_MODE
557 static uint32_t
558 ahd_targetcmd_offset(struct ahd_softc *ahd, u_int index)
559 {
560 	return (((uint8_t *)&ahd->targetcmds[index])
561 	       - (uint8_t *)ahd->qoutfifo);
562 }
563 #endif
564 
565 /*********************** Miscellaneous Support Functions ***********************/
566 /*
567  * Return pointers to the transfer negotiation information
568  * for the specified our_id/remote_id pair.
569  */
570 struct ahd_initiator_tinfo *
571 ahd_fetch_transinfo(struct ahd_softc *ahd, char channel, u_int our_id,
572 		    u_int remote_id, struct ahd_tmode_tstate **tstate)
573 {
574 	/*
575 	 * Transfer data structures are stored from the perspective
576 	 * of the target role.  Since the parameters for a connection
577 	 * in the initiator role to a given target are the same as
578 	 * when the roles are reversed, we pretend we are the target.
579 	 */
580 	if (channel == 'B')
581 		our_id += 8;
582 	*tstate = ahd->enabled_targets[our_id];
583 	return (&(*tstate)->transinfo[remote_id]);
584 }
585 
586 uint16_t
587 ahd_inw(struct ahd_softc *ahd, u_int port)
588 {
589 	/*
590 	 * Read high byte first as some registers increment
591 	 * or have other side effects when the low byte is
592 	 * read.
593 	 */
594 	uint16_t r = ahd_inb(ahd, port+1) << 8;
595 	return r | ahd_inb(ahd, port);
596 }
597 
598 void
599 ahd_outw(struct ahd_softc *ahd, u_int port, u_int value)
600 {
601 	/*
602 	 * Write low byte first to accommodate registers
603 	 * such as PRGMCNT where the order maters.
604 	 */
605 	ahd_outb(ahd, port, value & 0xFF);
606 	ahd_outb(ahd, port+1, (value >> 8) & 0xFF);
607 }
608 
609 uint32_t
610 ahd_inl(struct ahd_softc *ahd, u_int port)
611 {
612 	return ((ahd_inb(ahd, port))
613 	      | (ahd_inb(ahd, port+1) << 8)
614 	      | (ahd_inb(ahd, port+2) << 16)
615 	      | (ahd_inb(ahd, port+3) << 24));
616 }
617 
618 void
619 ahd_outl(struct ahd_softc *ahd, u_int port, uint32_t value)
620 {
621 	ahd_outb(ahd, port, (value) & 0xFF);
622 	ahd_outb(ahd, port+1, ((value) >> 8) & 0xFF);
623 	ahd_outb(ahd, port+2, ((value) >> 16) & 0xFF);
624 	ahd_outb(ahd, port+3, ((value) >> 24) & 0xFF);
625 }
626 
627 uint64_t
628 ahd_inq(struct ahd_softc *ahd, u_int port)
629 {
630 	return ((ahd_inb(ahd, port))
631 	      | (ahd_inb(ahd, port+1) << 8)
632 	      | (ahd_inb(ahd, port+2) << 16)
633 	      | (ahd_inb(ahd, port+3) << 24)
634 	      | (((uint64_t)ahd_inb(ahd, port+4)) << 32)
635 	      | (((uint64_t)ahd_inb(ahd, port+5)) << 40)
636 	      | (((uint64_t)ahd_inb(ahd, port+6)) << 48)
637 	      | (((uint64_t)ahd_inb(ahd, port+7)) << 56));
638 }
639 
640 void
641 ahd_outq(struct ahd_softc *ahd, u_int port, uint64_t value)
642 {
643 	ahd_outb(ahd, port, value & 0xFF);
644 	ahd_outb(ahd, port+1, (value >> 8) & 0xFF);
645 	ahd_outb(ahd, port+2, (value >> 16) & 0xFF);
646 	ahd_outb(ahd, port+3, (value >> 24) & 0xFF);
647 	ahd_outb(ahd, port+4, (value >> 32) & 0xFF);
648 	ahd_outb(ahd, port+5, (value >> 40) & 0xFF);
649 	ahd_outb(ahd, port+6, (value >> 48) & 0xFF);
650 	ahd_outb(ahd, port+7, (value >> 56) & 0xFF);
651 }
652 
653 u_int
654 ahd_get_scbptr(struct ahd_softc *ahd)
655 {
656 	AHD_ASSERT_MODES(ahd, ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK),
657 			 ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK));
658 	return (ahd_inb(ahd, SCBPTR) | (ahd_inb(ahd, SCBPTR + 1) << 8));
659 }
660 
661 void
662 ahd_set_scbptr(struct ahd_softc *ahd, u_int scbptr)
663 {
664 	AHD_ASSERT_MODES(ahd, ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK),
665 			 ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK));
666 	ahd_outb(ahd, SCBPTR, scbptr & 0xFF);
667 	ahd_outb(ahd, SCBPTR+1, (scbptr >> 8) & 0xFF);
668 }
669 
670 #if 0 /* unused */
671 static u_int
672 ahd_get_hnscb_qoff(struct ahd_softc *ahd)
673 {
674 	return (ahd_inw_atomic(ahd, HNSCB_QOFF));
675 }
676 #endif
677 
678 static void
679 ahd_set_hnscb_qoff(struct ahd_softc *ahd, u_int value)
680 {
681 	ahd_outw_atomic(ahd, HNSCB_QOFF, value);
682 }
683 
684 #if 0 /* unused */
685 static u_int
686 ahd_get_hescb_qoff(struct ahd_softc *ahd)
687 {
688 	return (ahd_inb(ahd, HESCB_QOFF));
689 }
690 #endif
691 
692 static void
693 ahd_set_hescb_qoff(struct ahd_softc *ahd, u_int value)
694 {
695 	ahd_outb(ahd, HESCB_QOFF, value);
696 }
697 
698 static u_int
699 ahd_get_snscb_qoff(struct ahd_softc *ahd)
700 {
701 	u_int oldvalue;
702 
703 	AHD_ASSERT_MODES(ahd, AHD_MODE_CCHAN_MSK, AHD_MODE_CCHAN_MSK);
704 	oldvalue = ahd_inw(ahd, SNSCB_QOFF);
705 	ahd_outw(ahd, SNSCB_QOFF, oldvalue);
706 	return (oldvalue);
707 }
708 
709 static void
710 ahd_set_snscb_qoff(struct ahd_softc *ahd, u_int value)
711 {
712 	AHD_ASSERT_MODES(ahd, AHD_MODE_CCHAN_MSK, AHD_MODE_CCHAN_MSK);
713 	ahd_outw(ahd, SNSCB_QOFF, value);
714 }
715 
716 #if 0 /* unused */
717 static u_int
718 ahd_get_sescb_qoff(struct ahd_softc *ahd)
719 {
720 	AHD_ASSERT_MODES(ahd, AHD_MODE_CCHAN_MSK, AHD_MODE_CCHAN_MSK);
721 	return (ahd_inb(ahd, SESCB_QOFF));
722 }
723 #endif
724 
725 static void
726 ahd_set_sescb_qoff(struct ahd_softc *ahd, u_int value)
727 {
728 	AHD_ASSERT_MODES(ahd, AHD_MODE_CCHAN_MSK, AHD_MODE_CCHAN_MSK);
729 	ahd_outb(ahd, SESCB_QOFF, value);
730 }
731 
732 #if 0 /* unused */
733 static u_int
734 ahd_get_sdscb_qoff(struct ahd_softc *ahd)
735 {
736 	AHD_ASSERT_MODES(ahd, AHD_MODE_CCHAN_MSK, AHD_MODE_CCHAN_MSK);
737 	return (ahd_inb(ahd, SDSCB_QOFF) | (ahd_inb(ahd, SDSCB_QOFF + 1) << 8));
738 }
739 #endif
740 
741 static void
742 ahd_set_sdscb_qoff(struct ahd_softc *ahd, u_int value)
743 {
744 	AHD_ASSERT_MODES(ahd, AHD_MODE_CCHAN_MSK, AHD_MODE_CCHAN_MSK);
745 	ahd_outb(ahd, SDSCB_QOFF, value & 0xFF);
746 	ahd_outb(ahd, SDSCB_QOFF+1, (value >> 8) & 0xFF);
747 }
748 
749 u_int
750 ahd_inb_scbram(struct ahd_softc *ahd, u_int offset)
751 {
752 	u_int value;
753 
754 	/*
755 	 * Workaround PCI-X Rev A. hardware bug.
756 	 * After a host read of SCB memory, the chip
757 	 * may become confused into thinking prefetch
758 	 * was required.  This starts the discard timer
759 	 * running and can cause an unexpected discard
760 	 * timer interrupt.  The work around is to read
761 	 * a normal register prior to the exhaustion of
762 	 * the discard timer.  The mode pointer register
763 	 * has no side effects and so serves well for
764 	 * this purpose.
765 	 *
766 	 * Razor #528
767 	 */
768 	value = ahd_inb(ahd, offset);
769 	if ((ahd->bugs & AHD_PCIX_SCBRAM_RD_BUG) != 0)
770 		ahd_inb(ahd, MODE_PTR);
771 	return (value);
772 }
773 
774 u_int
775 ahd_inw_scbram(struct ahd_softc *ahd, u_int offset)
776 {
777 	return (ahd_inb_scbram(ahd, offset)
778 	      | (ahd_inb_scbram(ahd, offset+1) << 8));
779 }
780 
781 static uint32_t
782 ahd_inl_scbram(struct ahd_softc *ahd, u_int offset)
783 {
784 	return (ahd_inw_scbram(ahd, offset)
785 	      | (ahd_inw_scbram(ahd, offset+2) << 16));
786 }
787 
788 static uint64_t
789 ahd_inq_scbram(struct ahd_softc *ahd, u_int offset)
790 {
791 	return (ahd_inl_scbram(ahd, offset)
792 	      | ((uint64_t)ahd_inl_scbram(ahd, offset+4)) << 32);
793 }
794 
795 struct scb *
796 ahd_lookup_scb(struct ahd_softc *ahd, u_int tag)
797 {
798 	struct scb* scb;
799 
800 	if (tag >= AHD_SCB_MAX)
801 		return (NULL);
802 	scb = ahd->scb_data.scbindex[tag];
803 	if (scb != NULL)
804 		ahd_sync_scb(ahd, scb,
805 			     BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE);
806 	return (scb);
807 }
808 
809 static void
810 ahd_swap_with_next_hscb(struct ahd_softc *ahd, struct scb *scb)
811 {
812 	struct	 hardware_scb *q_hscb;
813 	struct	 map_node *q_hscb_map;
814 	uint32_t saved_hscb_busaddr;
815 
816 	/*
817 	 * Our queuing method is a bit tricky.  The card
818 	 * knows in advance which HSCB (by address) to download,
819 	 * and we can't disappoint it.  To achieve this, the next
820 	 * HSCB to download is saved off in ahd->next_queued_hscb.
821 	 * When we are called to queue "an arbitrary scb",
822 	 * we copy the contents of the incoming HSCB to the one
823 	 * the sequencer knows about, swap HSCB pointers and
824 	 * finally assign the SCB to the tag indexed location
825 	 * in the scb_array.  This makes sure that we can still
826 	 * locate the correct SCB by SCB_TAG.
827 	 */
828 	q_hscb = ahd->next_queued_hscb;
829 	q_hscb_map = ahd->next_queued_hscb_map;
830 	saved_hscb_busaddr = q_hscb->hscb_busaddr;
831 	memcpy(q_hscb, scb->hscb, sizeof(*scb->hscb));
832 	q_hscb->hscb_busaddr = saved_hscb_busaddr;
833 	q_hscb->next_hscb_busaddr = scb->hscb->hscb_busaddr;
834 
835 	/* Now swap HSCB pointers. */
836 	ahd->next_queued_hscb = scb->hscb;
837 	ahd->next_queued_hscb_map = scb->hscb_map;
838 	scb->hscb = q_hscb;
839 	scb->hscb_map = q_hscb_map;
840 
841 	/* Now define the mapping from tag to SCB in the scbindex */
842 	ahd->scb_data.scbindex[SCB_GET_TAG(scb)] = scb;
843 }
844 
845 /*
846  * Tell the sequencer about a new transaction to execute.
847  */
848 void
849 ahd_queue_scb(struct ahd_softc *ahd, struct scb *scb)
850 {
851 	ahd_swap_with_next_hscb(ahd, scb);
852 
853 	if (SCBID_IS_NULL(SCB_GET_TAG(scb)))
854 		panic("Attempt to queue invalid SCB tag %x\n",
855 		      SCB_GET_TAG(scb));
856 
857 	/*
858 	 * Keep a history of SCBs we've downloaded in the qinfifo.
859 	 */
860 	ahd->qinfifo[AHD_QIN_WRAP(ahd->qinfifonext)] = SCB_GET_TAG(scb);
861 	ahd->qinfifonext++;
862 
863 	if (scb->sg_count != 0)
864 		ahd_setup_data_scb(ahd, scb);
865 	else
866 		ahd_setup_noxfer_scb(ahd, scb);
867 	ahd_setup_scb_common(ahd, scb);
868 
869 	/*
870 	 * Make sure our data is consistent from the
871 	 * perspective of the adapter.
872 	 */
873 	ahd_sync_scb(ahd, scb, BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);
874 
875 #ifdef AHD_DEBUG
876 	if ((ahd_debug & AHD_SHOW_QUEUE) != 0) {
877 		uint64_t host_dataptr;
878 
879 		host_dataptr = ahd_le64toh(scb->hscb->dataptr);
880 		printk("%s: Queueing SCB %d:0x%x bus addr 0x%x - 0x%x%x/0x%x\n",
881 		       ahd_name(ahd),
882 		       SCB_GET_TAG(scb), scb->hscb->scsiid,
883 		       ahd_le32toh(scb->hscb->hscb_busaddr),
884 		       (u_int)((host_dataptr >> 32) & 0xFFFFFFFF),
885 		       (u_int)(host_dataptr & 0xFFFFFFFF),
886 		       ahd_le32toh(scb->hscb->datacnt));
887 	}
888 #endif
889 	/* Tell the adapter about the newly queued SCB */
890 	ahd_set_hnscb_qoff(ahd, ahd->qinfifonext);
891 }
892 
893 /************************** Interrupt Processing ******************************/
894 static void
895 ahd_sync_qoutfifo(struct ahd_softc *ahd, int op)
896 {
897 	ahd_dmamap_sync(ahd, ahd->shared_data_dmat, ahd->shared_data_map.dmamap,
898 			/*offset*/0,
899 			/*len*/AHD_SCB_MAX * sizeof(struct ahd_completion), op);
900 }
901 
902 static void
903 ahd_sync_tqinfifo(struct ahd_softc *ahd, int op)
904 {
905 #ifdef AHD_TARGET_MODE
906 	if ((ahd->flags & AHD_TARGETROLE) != 0) {
907 		ahd_dmamap_sync(ahd, ahd->shared_data_dmat,
908 				ahd->shared_data_map.dmamap,
909 				ahd_targetcmd_offset(ahd, 0),
910 				sizeof(struct target_cmd) * AHD_TMODE_CMDS,
911 				op);
912 	}
913 #endif
914 }
915 
916 /*
917  * See if the firmware has posted any completed commands
918  * into our in-core command complete fifos.
919  */
920 #define AHD_RUN_QOUTFIFO 0x1
921 #define AHD_RUN_TQINFIFO 0x2
922 static u_int
923 ahd_check_cmdcmpltqueues(struct ahd_softc *ahd)
924 {
925 	u_int retval;
926 
927 	retval = 0;
928 	ahd_dmamap_sync(ahd, ahd->shared_data_dmat, ahd->shared_data_map.dmamap,
929 			/*offset*/ahd->qoutfifonext * sizeof(*ahd->qoutfifo),
930 			/*len*/sizeof(*ahd->qoutfifo), BUS_DMASYNC_POSTREAD);
931 	if (ahd->qoutfifo[ahd->qoutfifonext].valid_tag
932 	  == ahd->qoutfifonext_valid_tag)
933 		retval |= AHD_RUN_QOUTFIFO;
934 #ifdef AHD_TARGET_MODE
935 	if ((ahd->flags & AHD_TARGETROLE) != 0
936 	 && (ahd->flags & AHD_TQINFIFO_BLOCKED) == 0) {
937 		ahd_dmamap_sync(ahd, ahd->shared_data_dmat,
938 				ahd->shared_data_map.dmamap,
939 				ahd_targetcmd_offset(ahd, ahd->tqinfifofnext),
940 				/*len*/sizeof(struct target_cmd),
941 				BUS_DMASYNC_POSTREAD);
942 		if (ahd->targetcmds[ahd->tqinfifonext].cmd_valid != 0)
943 			retval |= AHD_RUN_TQINFIFO;
944 	}
945 #endif
946 	return (retval);
947 }
948 
949 /*
950  * Catch an interrupt from the adapter
951  */
952 int
953 ahd_intr(struct ahd_softc *ahd)
954 {
955 	u_int	intstat;
956 
957 	if ((ahd->pause & INTEN) == 0) {
958 		/*
959 		 * Our interrupt is not enabled on the chip
960 		 * and may be disabled for re-entrancy reasons,
961 		 * so just return.  This is likely just a shared
962 		 * interrupt.
963 		 */
964 		return (0);
965 	}
966 
967 	/*
968 	 * Instead of directly reading the interrupt status register,
969 	 * infer the cause of the interrupt by checking our in-core
970 	 * completion queues.  This avoids a costly PCI bus read in
971 	 * most cases.
972 	 */
973 	if ((ahd->flags & AHD_ALL_INTERRUPTS) == 0
974 	 && (ahd_check_cmdcmpltqueues(ahd) != 0))
975 		intstat = CMDCMPLT;
976 	else
977 		intstat = ahd_inb(ahd, INTSTAT);
978 
979 	if ((intstat & INT_PEND) == 0)
980 		return (0);
981 
982 	if (intstat & CMDCMPLT) {
983 		ahd_outb(ahd, CLRINT, CLRCMDINT);
984 
985 		/*
986 		 * Ensure that the chip sees that we've cleared
987 		 * this interrupt before we walk the output fifo.
988 		 * Otherwise, we may, due to posted bus writes,
989 		 * clear the interrupt after we finish the scan,
990 		 * and after the sequencer has added new entries
991 		 * and asserted the interrupt again.
992 		 */
993 		if ((ahd->bugs & AHD_INTCOLLISION_BUG) != 0) {
994 			if (ahd_is_paused(ahd)) {
995 				/*
996 				 * Potentially lost SEQINT.
997 				 * If SEQINTCODE is non-zero,
998 				 * simulate the SEQINT.
999 				 */
1000 				if (ahd_inb(ahd, SEQINTCODE) != NO_SEQINT)
1001 					intstat |= SEQINT;
1002 			}
1003 		} else {
1004 			ahd_flush_device_writes(ahd);
1005 		}
1006 		ahd_run_qoutfifo(ahd);
1007 		ahd->cmdcmplt_counts[ahd->cmdcmplt_bucket]++;
1008 		ahd->cmdcmplt_total++;
1009 #ifdef AHD_TARGET_MODE
1010 		if ((ahd->flags & AHD_TARGETROLE) != 0)
1011 			ahd_run_tqinfifo(ahd, /*paused*/FALSE);
1012 #endif
1013 	}
1014 
1015 	/*
1016 	 * Handle statuses that may invalidate our cached
1017 	 * copy of INTSTAT separately.
1018 	 */
1019 	if (intstat == 0xFF && (ahd->features & AHD_REMOVABLE) != 0) {
1020 		/* Hot eject.  Do nothing */
1021 	} else if (intstat & HWERRINT) {
1022 		ahd_handle_hwerrint(ahd);
1023 	} else if ((intstat & (PCIINT|SPLTINT)) != 0) {
1024 		ahd->bus_intr(ahd);
1025 	} else {
1026 
1027 		if ((intstat & SEQINT) != 0)
1028 			ahd_handle_seqint(ahd, intstat);
1029 
1030 		if ((intstat & SCSIINT) != 0)
1031 			ahd_handle_scsiint(ahd, intstat);
1032 	}
1033 	return (1);
1034 }
1035 
1036 /******************************** Private Inlines *****************************/
1037 static inline void
1038 ahd_assert_atn(struct ahd_softc *ahd)
1039 {
1040 	ahd_outb(ahd, SCSISIGO, ATNO);
1041 }
1042 
1043 /*
1044  * Determine if the current connection has a packetized
1045  * agreement.  This does not necessarily mean that we
1046  * are currently in a packetized transfer.  We could
1047  * just as easily be sending or receiving a message.
1048  */
1049 static int
1050 ahd_currently_packetized(struct ahd_softc *ahd)
1051 {
1052 	ahd_mode_state	 saved_modes;
1053 	int		 packetized;
1054 
1055 	saved_modes = ahd_save_modes(ahd);
1056 	if ((ahd->bugs & AHD_PKTIZED_STATUS_BUG) != 0) {
1057 		/*
1058 		 * The packetized bit refers to the last
1059 		 * connection, not the current one.  Check
1060 		 * for non-zero LQISTATE instead.
1061 		 */
1062 		ahd_set_modes(ahd, AHD_MODE_CFG, AHD_MODE_CFG);
1063 		packetized = ahd_inb(ahd, LQISTATE) != 0;
1064 	} else {
1065 		ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
1066 		packetized = ahd_inb(ahd, LQISTAT2) & PACKETIZED;
1067 	}
1068 	ahd_restore_modes(ahd, saved_modes);
1069 	return (packetized);
1070 }
1071 
1072 static inline int
1073 ahd_set_active_fifo(struct ahd_softc *ahd)
1074 {
1075 	u_int active_fifo;
1076 
1077 	AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
1078 	active_fifo = ahd_inb(ahd, DFFSTAT) & CURRFIFO;
1079 	switch (active_fifo) {
1080 	case 0:
1081 	case 1:
1082 		ahd_set_modes(ahd, active_fifo, active_fifo);
1083 		return (1);
1084 	default:
1085 		return (0);
1086 	}
1087 }
1088 
1089 static inline void
1090 ahd_unbusy_tcl(struct ahd_softc *ahd, u_int tcl)
1091 {
1092 	ahd_busy_tcl(ahd, tcl, SCB_LIST_NULL);
1093 }
1094 
1095 /*
1096  * Determine whether the sequencer reported a residual
1097  * for this SCB/transaction.
1098  */
1099 static inline void
1100 ahd_update_residual(struct ahd_softc *ahd, struct scb *scb)
1101 {
1102 	uint32_t sgptr;
1103 
1104 	sgptr = ahd_le32toh(scb->hscb->sgptr);
1105 	if ((sgptr & SG_STATUS_VALID) != 0)
1106 		ahd_calc_residual(ahd, scb);
1107 }
1108 
1109 static inline void
1110 ahd_complete_scb(struct ahd_softc *ahd, struct scb *scb)
1111 {
1112 	uint32_t sgptr;
1113 
1114 	sgptr = ahd_le32toh(scb->hscb->sgptr);
1115 	if ((sgptr & SG_STATUS_VALID) != 0)
1116 		ahd_handle_scb_status(ahd, scb);
1117 	else
1118 		ahd_done(ahd, scb);
1119 }
1120 
1121 
1122 /************************* Sequencer Execution Control ************************/
1123 /*
1124  * Restart the sequencer program from address zero
1125  */
1126 static void
1127 ahd_restart(struct ahd_softc *ahd)
1128 {
1129 
1130 	ahd_pause(ahd);
1131 
1132 	ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
1133 
1134 	/* No more pending messages */
1135 	ahd_clear_msg_state(ahd);
1136 	ahd_outb(ahd, SCSISIGO, 0);		/* De-assert BSY */
1137 	ahd_outb(ahd, MSG_OUT, MSG_NOOP);	/* No message to send */
1138 	ahd_outb(ahd, SXFRCTL1, ahd_inb(ahd, SXFRCTL1) & ~BITBUCKET);
1139 	ahd_outb(ahd, SEQINTCTL, 0);
1140 	ahd_outb(ahd, LASTPHASE, P_BUSFREE);
1141 	ahd_outb(ahd, SEQ_FLAGS, 0);
1142 	ahd_outb(ahd, SAVED_SCSIID, 0xFF);
1143 	ahd_outb(ahd, SAVED_LUN, 0xFF);
1144 
1145 	/*
1146 	 * Ensure that the sequencer's idea of TQINPOS
1147 	 * matches our own.  The sequencer increments TQINPOS
1148 	 * only after it sees a DMA complete and a reset could
1149 	 * occur before the increment leaving the kernel to believe
1150 	 * the command arrived but the sequencer to not.
1151 	 */
1152 	ahd_outb(ahd, TQINPOS, ahd->tqinfifonext);
1153 
1154 	/* Always allow reselection */
1155 	ahd_outb(ahd, SCSISEQ1,
1156 		 ahd_inb(ahd, SCSISEQ_TEMPLATE) & (ENSELI|ENRSELI|ENAUTOATNP));
1157 	ahd_set_modes(ahd, AHD_MODE_CCHAN, AHD_MODE_CCHAN);
1158 
1159 	/*
1160 	 * Clear any pending sequencer interrupt.  It is no
1161 	 * longer relevant since we're resetting the Program
1162 	 * Counter.
1163 	 */
1164 	ahd_outb(ahd, CLRINT, CLRSEQINT);
1165 
1166 	ahd_outb(ahd, SEQCTL0, FASTMODE|SEQRESET);
1167 	ahd_unpause(ahd);
1168 }
1169 
1170 static void
1171 ahd_clear_fifo(struct ahd_softc *ahd, u_int fifo)
1172 {
1173 	ahd_mode_state	 saved_modes;
1174 
1175 #ifdef AHD_DEBUG
1176 	if ((ahd_debug & AHD_SHOW_FIFOS) != 0)
1177 		printk("%s: Clearing FIFO %d\n", ahd_name(ahd), fifo);
1178 #endif
1179 	saved_modes = ahd_save_modes(ahd);
1180 	ahd_set_modes(ahd, fifo, fifo);
1181 	ahd_outb(ahd, DFFSXFRCTL, RSTCHN|CLRSHCNT);
1182 	if ((ahd_inb(ahd, SG_STATE) & FETCH_INPROG) != 0)
1183 		ahd_outb(ahd, CCSGCTL, CCSGRESET);
1184 	ahd_outb(ahd, LONGJMP_ADDR + 1, INVALID_ADDR);
1185 	ahd_outb(ahd, SG_STATE, 0);
1186 	ahd_restore_modes(ahd, saved_modes);
1187 }
1188 
1189 /************************* Input/Output Queues ********************************/
1190 /*
1191  * Flush and completed commands that are sitting in the command
1192  * complete queues down on the chip but have yet to be dma'ed back up.
1193  */
1194 static void
1195 ahd_flush_qoutfifo(struct ahd_softc *ahd)
1196 {
1197 	struct		scb *scb;
1198 	ahd_mode_state	saved_modes;
1199 	u_int		saved_scbptr;
1200 	u_int		ccscbctl;
1201 	u_int		scbid;
1202 	u_int		next_scbid;
1203 
1204 	saved_modes = ahd_save_modes(ahd);
1205 
1206 	/*
1207 	 * Flush the good status FIFO for completed packetized commands.
1208 	 */
1209 	ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
1210 	saved_scbptr = ahd_get_scbptr(ahd);
1211 	while ((ahd_inb(ahd, LQISTAT2) & LQIGSAVAIL) != 0) {
1212 		u_int fifo_mode;
1213 		u_int i;
1214 
1215 		scbid = ahd_inw(ahd, GSFIFO);
1216 		scb = ahd_lookup_scb(ahd, scbid);
1217 		if (scb == NULL) {
1218 			printk("%s: Warning - GSFIFO SCB %d invalid\n",
1219 			       ahd_name(ahd), scbid);
1220 			continue;
1221 		}
1222 		/*
1223 		 * Determine if this transaction is still active in
1224 		 * any FIFO.  If it is, we must flush that FIFO to
1225 		 * the host before completing the  command.
1226 		 */
1227 		fifo_mode = 0;
1228 rescan_fifos:
1229 		for (i = 0; i < 2; i++) {
1230 			/* Toggle to the other mode. */
1231 			fifo_mode ^= 1;
1232 			ahd_set_modes(ahd, fifo_mode, fifo_mode);
1233 
1234 			if (ahd_scb_active_in_fifo(ahd, scb) == 0)
1235 				continue;
1236 
1237 			ahd_run_data_fifo(ahd, scb);
1238 
1239 			/*
1240 			 * Running this FIFO may cause a CFG4DATA for
1241 			 * this same transaction to assert in the other
1242 			 * FIFO or a new snapshot SAVEPTRS interrupt
1243 			 * in this FIFO.  Even running a FIFO may not
1244 			 * clear the transaction if we are still waiting
1245 			 * for data to drain to the host. We must loop
1246 			 * until the transaction is not active in either
1247 			 * FIFO just to be sure.  Reset our loop counter
1248 			 * so we will visit both FIFOs again before
1249 			 * declaring this transaction finished.  We
1250 			 * also delay a bit so that status has a chance
1251 			 * to change before we look at this FIFO again.
1252 			 */
1253 			ahd_delay(200);
1254 			goto rescan_fifos;
1255 		}
1256 		ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
1257 		ahd_set_scbptr(ahd, scbid);
1258 		if ((ahd_inb_scbram(ahd, SCB_SGPTR) & SG_LIST_NULL) == 0
1259 		 && ((ahd_inb_scbram(ahd, SCB_SGPTR) & SG_FULL_RESID) != 0
1260 		  || (ahd_inb_scbram(ahd, SCB_RESIDUAL_SGPTR)
1261 		      & SG_LIST_NULL) != 0)) {
1262 			u_int comp_head;
1263 
1264 			/*
1265 			 * The transfer completed with a residual.
1266 			 * Place this SCB on the complete DMA list
1267 			 * so that we update our in-core copy of the
1268 			 * SCB before completing the command.
1269 			 */
1270 			ahd_outb(ahd, SCB_SCSI_STATUS, 0);
1271 			ahd_outb(ahd, SCB_SGPTR,
1272 				 ahd_inb_scbram(ahd, SCB_SGPTR)
1273 				 | SG_STATUS_VALID);
1274 			ahd_outw(ahd, SCB_TAG, scbid);
1275 			ahd_outw(ahd, SCB_NEXT_COMPLETE, SCB_LIST_NULL);
1276 			comp_head = ahd_inw(ahd, COMPLETE_DMA_SCB_HEAD);
1277 			if (SCBID_IS_NULL(comp_head)) {
1278 				ahd_outw(ahd, COMPLETE_DMA_SCB_HEAD, scbid);
1279 				ahd_outw(ahd, COMPLETE_DMA_SCB_TAIL, scbid);
1280 			} else {
1281 				u_int tail;
1282 
1283 				tail = ahd_inw(ahd, COMPLETE_DMA_SCB_TAIL);
1284 				ahd_set_scbptr(ahd, tail);
1285 				ahd_outw(ahd, SCB_NEXT_COMPLETE, scbid);
1286 				ahd_outw(ahd, COMPLETE_DMA_SCB_TAIL, scbid);
1287 				ahd_set_scbptr(ahd, scbid);
1288 			}
1289 		} else
1290 			ahd_complete_scb(ahd, scb);
1291 	}
1292 	ahd_set_scbptr(ahd, saved_scbptr);
1293 
1294 	/*
1295 	 * Setup for command channel portion of flush.
1296 	 */
1297 	ahd_set_modes(ahd, AHD_MODE_CCHAN, AHD_MODE_CCHAN);
1298 
1299 	/*
1300 	 * Wait for any inprogress DMA to complete and clear DMA state
1301 	 * if this is for an SCB in the qinfifo.
1302 	 */
1303 	while (((ccscbctl = ahd_inb(ahd, CCSCBCTL)) & (CCARREN|CCSCBEN)) != 0) {
1304 
1305 		if ((ccscbctl & (CCSCBDIR|CCARREN)) == (CCSCBDIR|CCARREN)) {
1306 			if ((ccscbctl & ARRDONE) != 0)
1307 				break;
1308 		} else if ((ccscbctl & CCSCBDONE) != 0)
1309 			break;
1310 		ahd_delay(200);
1311 	}
1312 	/*
1313 	 * We leave the sequencer to cleanup in the case of DMA's to
1314 	 * update the qoutfifo.  In all other cases (DMA's to the
1315 	 * chip or a push of an SCB from the COMPLETE_DMA_SCB list),
1316 	 * we disable the DMA engine so that the sequencer will not
1317 	 * attempt to handle the DMA completion.
1318 	 */
1319 	if ((ccscbctl & CCSCBDIR) != 0 || (ccscbctl & ARRDONE) != 0)
1320 		ahd_outb(ahd, CCSCBCTL, ccscbctl & ~(CCARREN|CCSCBEN));
1321 
1322 	/*
1323 	 * Complete any SCBs that just finished
1324 	 * being DMA'ed into the qoutfifo.
1325 	 */
1326 	ahd_run_qoutfifo(ahd);
1327 
1328 	saved_scbptr = ahd_get_scbptr(ahd);
1329 	/*
1330 	 * Manually update/complete any completed SCBs that are waiting to be
1331 	 * DMA'ed back up to the host.
1332 	 */
1333 	scbid = ahd_inw(ahd, COMPLETE_DMA_SCB_HEAD);
1334 	while (!SCBID_IS_NULL(scbid)) {
1335 		uint8_t *hscb_ptr;
1336 		u_int	 i;
1337 
1338 		ahd_set_scbptr(ahd, scbid);
1339 		next_scbid = ahd_inw_scbram(ahd, SCB_NEXT_COMPLETE);
1340 		scb = ahd_lookup_scb(ahd, scbid);
1341 		if (scb == NULL) {
1342 			printk("%s: Warning - DMA-up and complete "
1343 			       "SCB %d invalid\n", ahd_name(ahd), scbid);
1344 			continue;
1345 		}
1346 		hscb_ptr = (uint8_t *)scb->hscb;
1347 		for (i = 0; i < sizeof(struct hardware_scb); i++)
1348 			*hscb_ptr++ = ahd_inb_scbram(ahd, SCB_BASE + i);
1349 
1350 		ahd_complete_scb(ahd, scb);
1351 		scbid = next_scbid;
1352 	}
1353 	ahd_outw(ahd, COMPLETE_DMA_SCB_HEAD, SCB_LIST_NULL);
1354 	ahd_outw(ahd, COMPLETE_DMA_SCB_TAIL, SCB_LIST_NULL);
1355 
1356 	scbid = ahd_inw(ahd, COMPLETE_ON_QFREEZE_HEAD);
1357 	while (!SCBID_IS_NULL(scbid)) {
1358 
1359 		ahd_set_scbptr(ahd, scbid);
1360 		next_scbid = ahd_inw_scbram(ahd, SCB_NEXT_COMPLETE);
1361 		scb = ahd_lookup_scb(ahd, scbid);
1362 		if (scb == NULL) {
1363 			printk("%s: Warning - Complete Qfrz SCB %d invalid\n",
1364 			       ahd_name(ahd), scbid);
1365 			continue;
1366 		}
1367 
1368 		ahd_complete_scb(ahd, scb);
1369 		scbid = next_scbid;
1370 	}
1371 	ahd_outw(ahd, COMPLETE_ON_QFREEZE_HEAD, SCB_LIST_NULL);
1372 
1373 	scbid = ahd_inw(ahd, COMPLETE_SCB_HEAD);
1374 	while (!SCBID_IS_NULL(scbid)) {
1375 
1376 		ahd_set_scbptr(ahd, scbid);
1377 		next_scbid = ahd_inw_scbram(ahd, SCB_NEXT_COMPLETE);
1378 		scb = ahd_lookup_scb(ahd, scbid);
1379 		if (scb == NULL) {
1380 			printk("%s: Warning - Complete SCB %d invalid\n",
1381 			       ahd_name(ahd), scbid);
1382 			continue;
1383 		}
1384 
1385 		ahd_complete_scb(ahd, scb);
1386 		scbid = next_scbid;
1387 	}
1388 	ahd_outw(ahd, COMPLETE_SCB_HEAD, SCB_LIST_NULL);
1389 
1390 	/*
1391 	 * Restore state.
1392 	 */
1393 	ahd_set_scbptr(ahd, saved_scbptr);
1394 	ahd_restore_modes(ahd, saved_modes);
1395 	ahd->flags |= AHD_UPDATE_PEND_CMDS;
1396 }
1397 
1398 /*
1399  * Determine if an SCB for a packetized transaction
1400  * is active in a FIFO.
1401  */
1402 static int
1403 ahd_scb_active_in_fifo(struct ahd_softc *ahd, struct scb *scb)
1404 {
1405 
1406 	/*
1407 	 * The FIFO is only active for our transaction if
1408 	 * the SCBPTR matches the SCB's ID and the firmware
1409 	 * has installed a handler for the FIFO or we have
1410 	 * a pending SAVEPTRS or CFG4DATA interrupt.
1411 	 */
1412 	if (ahd_get_scbptr(ahd) != SCB_GET_TAG(scb)
1413 	 || ((ahd_inb(ahd, LONGJMP_ADDR+1) & INVALID_ADDR) != 0
1414 	  && (ahd_inb(ahd, SEQINTSRC) & (CFG4DATA|SAVEPTRS)) == 0))
1415 		return (0);
1416 
1417 	return (1);
1418 }
1419 
1420 /*
1421  * Run a data fifo to completion for a transaction we know
1422  * has completed across the SCSI bus (good status has been
1423  * received).  We are already set to the correct FIFO mode
1424  * on entry to this routine.
1425  *
1426  * This function attempts to operate exactly as the firmware
1427  * would when running this FIFO.  Care must be taken to update
1428  * this routine any time the firmware's FIFO algorithm is
1429  * changed.
1430  */
1431 static void
1432 ahd_run_data_fifo(struct ahd_softc *ahd, struct scb *scb)
1433 {
1434 	u_int seqintsrc;
1435 
1436 	seqintsrc = ahd_inb(ahd, SEQINTSRC);
1437 	if ((seqintsrc & CFG4DATA) != 0) {
1438 		uint32_t datacnt;
1439 		uint32_t sgptr;
1440 
1441 		/*
1442 		 * Clear full residual flag.
1443 		 */
1444 		sgptr = ahd_inl_scbram(ahd, SCB_SGPTR) & ~SG_FULL_RESID;
1445 		ahd_outb(ahd, SCB_SGPTR, sgptr);
1446 
1447 		/*
1448 		 * Load datacnt and address.
1449 		 */
1450 		datacnt = ahd_inl_scbram(ahd, SCB_DATACNT);
1451 		if ((datacnt & AHD_DMA_LAST_SEG) != 0) {
1452 			sgptr |= LAST_SEG;
1453 			ahd_outb(ahd, SG_STATE, 0);
1454 		} else
1455 			ahd_outb(ahd, SG_STATE, LOADING_NEEDED);
1456 		ahd_outq(ahd, HADDR, ahd_inq_scbram(ahd, SCB_DATAPTR));
1457 		ahd_outl(ahd, HCNT, datacnt & AHD_SG_LEN_MASK);
1458 		ahd_outb(ahd, SG_CACHE_PRE, sgptr);
1459 		ahd_outb(ahd, DFCNTRL, PRELOADEN|SCSIEN|HDMAEN);
1460 
1461 		/*
1462 		 * Initialize Residual Fields.
1463 		 */
1464 		ahd_outb(ahd, SCB_RESIDUAL_DATACNT+3, datacnt >> 24);
1465 		ahd_outl(ahd, SCB_RESIDUAL_SGPTR, sgptr & SG_PTR_MASK);
1466 
1467 		/*
1468 		 * Mark the SCB as having a FIFO in use.
1469 		 */
1470 		ahd_outb(ahd, SCB_FIFO_USE_COUNT,
1471 			 ahd_inb_scbram(ahd, SCB_FIFO_USE_COUNT) + 1);
1472 
1473 		/*
1474 		 * Install a "fake" handler for this FIFO.
1475 		 */
1476 		ahd_outw(ahd, LONGJMP_ADDR, 0);
1477 
1478 		/*
1479 		 * Notify the hardware that we have satisfied
1480 		 * this sequencer interrupt.
1481 		 */
1482 		ahd_outb(ahd, CLRSEQINTSRC, CLRCFG4DATA);
1483 	} else if ((seqintsrc & SAVEPTRS) != 0) {
1484 		uint32_t sgptr;
1485 		uint32_t resid;
1486 
1487 		if ((ahd_inb(ahd, LONGJMP_ADDR+1)&INVALID_ADDR) != 0) {
1488 			/*
1489 			 * Snapshot Save Pointers.  All that
1490 			 * is necessary to clear the snapshot
1491 			 * is a CLRCHN.
1492 			 */
1493 			goto clrchn;
1494 		}
1495 
1496 		/*
1497 		 * Disable S/G fetch so the DMA engine
1498 		 * is available to future users.
1499 		 */
1500 		if ((ahd_inb(ahd, SG_STATE) & FETCH_INPROG) != 0)
1501 			ahd_outb(ahd, CCSGCTL, 0);
1502 		ahd_outb(ahd, SG_STATE, 0);
1503 
1504 		/*
1505 		 * Flush the data FIFO.  Strickly only
1506 		 * necessary for Rev A parts.
1507 		 */
1508 		ahd_outb(ahd, DFCNTRL, ahd_inb(ahd, DFCNTRL) | FIFOFLUSH);
1509 
1510 		/*
1511 		 * Calculate residual.
1512 		 */
1513 		sgptr = ahd_inl_scbram(ahd, SCB_RESIDUAL_SGPTR);
1514 		resid = ahd_inl(ahd, SHCNT);
1515 		resid |= ahd_inb_scbram(ahd, SCB_RESIDUAL_DATACNT+3) << 24;
1516 		ahd_outl(ahd, SCB_RESIDUAL_DATACNT, resid);
1517 		if ((ahd_inb(ahd, SG_CACHE_SHADOW) & LAST_SEG) == 0) {
1518 			/*
1519 			 * Must back up to the correct S/G element.
1520 			 * Typically this just means resetting our
1521 			 * low byte to the offset in the SG_CACHE,
1522 			 * but if we wrapped, we have to correct
1523 			 * the other bytes of the sgptr too.
1524 			 */
1525 			if ((ahd_inb(ahd, SG_CACHE_SHADOW) & 0x80) != 0
1526 			 && (sgptr & 0x80) == 0)
1527 				sgptr -= 0x100;
1528 			sgptr &= ~0xFF;
1529 			sgptr |= ahd_inb(ahd, SG_CACHE_SHADOW)
1530 			       & SG_ADDR_MASK;
1531 			ahd_outl(ahd, SCB_RESIDUAL_SGPTR, sgptr);
1532 			ahd_outb(ahd, SCB_RESIDUAL_DATACNT + 3, 0);
1533 		} else if ((resid & AHD_SG_LEN_MASK) == 0) {
1534 			ahd_outb(ahd, SCB_RESIDUAL_SGPTR,
1535 				 sgptr | SG_LIST_NULL);
1536 		}
1537 		/*
1538 		 * Save Pointers.
1539 		 */
1540 		ahd_outq(ahd, SCB_DATAPTR, ahd_inq(ahd, SHADDR));
1541 		ahd_outl(ahd, SCB_DATACNT, resid);
1542 		ahd_outl(ahd, SCB_SGPTR, sgptr);
1543 		ahd_outb(ahd, CLRSEQINTSRC, CLRSAVEPTRS);
1544 		ahd_outb(ahd, SEQIMODE,
1545 			 ahd_inb(ahd, SEQIMODE) | ENSAVEPTRS);
1546 		/*
1547 		 * If the data is to the SCSI bus, we are
1548 		 * done, otherwise wait for FIFOEMP.
1549 		 */
1550 		if ((ahd_inb(ahd, DFCNTRL) & DIRECTION) != 0)
1551 			goto clrchn;
1552 	} else if ((ahd_inb(ahd, SG_STATE) & LOADING_NEEDED) != 0) {
1553 		uint32_t sgptr;
1554 		uint64_t data_addr;
1555 		uint32_t data_len;
1556 		u_int	 dfcntrl;
1557 
1558 		/*
1559 		 * Disable S/G fetch so the DMA engine
1560 		 * is available to future users.  We won't
1561 		 * be using the DMA engine to load segments.
1562 		 */
1563 		if ((ahd_inb(ahd, SG_STATE) & FETCH_INPROG) != 0) {
1564 			ahd_outb(ahd, CCSGCTL, 0);
1565 			ahd_outb(ahd, SG_STATE, LOADING_NEEDED);
1566 		}
1567 
1568 		/*
1569 		 * Wait for the DMA engine to notice that the
1570 		 * host transfer is enabled and that there is
1571 		 * space in the S/G FIFO for new segments before
1572 		 * loading more segments.
1573 		 */
1574 		if ((ahd_inb(ahd, DFSTATUS) & PRELOAD_AVAIL) != 0
1575 		 && (ahd_inb(ahd, DFCNTRL) & HDMAENACK) != 0) {
1576 
1577 			/*
1578 			 * Determine the offset of the next S/G
1579 			 * element to load.
1580 			 */
1581 			sgptr = ahd_inl_scbram(ahd, SCB_RESIDUAL_SGPTR);
1582 			sgptr &= SG_PTR_MASK;
1583 			if ((ahd->flags & AHD_64BIT_ADDRESSING) != 0) {
1584 				struct ahd_dma64_seg *sg;
1585 
1586 				sg = ahd_sg_bus_to_virt(ahd, scb, sgptr);
1587 				data_addr = sg->addr;
1588 				data_len = sg->len;
1589 				sgptr += sizeof(*sg);
1590 			} else {
1591 				struct	ahd_dma_seg *sg;
1592 
1593 				sg = ahd_sg_bus_to_virt(ahd, scb, sgptr);
1594 				data_addr = sg->len & AHD_SG_HIGH_ADDR_MASK;
1595 				data_addr <<= 8;
1596 				data_addr |= sg->addr;
1597 				data_len = sg->len;
1598 				sgptr += sizeof(*sg);
1599 			}
1600 
1601 			/*
1602 			 * Update residual information.
1603 			 */
1604 			ahd_outb(ahd, SCB_RESIDUAL_DATACNT+3, data_len >> 24);
1605 			ahd_outl(ahd, SCB_RESIDUAL_SGPTR, sgptr);
1606 
1607 			/*
1608 			 * Load the S/G.
1609 			 */
1610 			if (data_len & AHD_DMA_LAST_SEG) {
1611 				sgptr |= LAST_SEG;
1612 				ahd_outb(ahd, SG_STATE, 0);
1613 			}
1614 			ahd_outq(ahd, HADDR, data_addr);
1615 			ahd_outl(ahd, HCNT, data_len & AHD_SG_LEN_MASK);
1616 			ahd_outb(ahd, SG_CACHE_PRE, sgptr & 0xFF);
1617 
1618 			/*
1619 			 * Advertise the segment to the hardware.
1620 			 */
1621 			dfcntrl = ahd_inb(ahd, DFCNTRL)|PRELOADEN|HDMAEN;
1622 			if ((ahd->features & AHD_NEW_DFCNTRL_OPTS) != 0) {
1623 				/*
1624 				 * Use SCSIENWRDIS so that SCSIEN
1625 				 * is never modified by this
1626 				 * operation.
1627 				 */
1628 				dfcntrl |= SCSIENWRDIS;
1629 			}
1630 			ahd_outb(ahd, DFCNTRL, dfcntrl);
1631 		}
1632 	} else if ((ahd_inb(ahd, SG_CACHE_SHADOW) & LAST_SEG_DONE) != 0) {
1633 
1634 		/*
1635 		 * Transfer completed to the end of SG list
1636 		 * and has flushed to the host.
1637 		 */
1638 		ahd_outb(ahd, SCB_SGPTR,
1639 			 ahd_inb_scbram(ahd, SCB_SGPTR) | SG_LIST_NULL);
1640 		goto clrchn;
1641 	} else if ((ahd_inb(ahd, DFSTATUS) & FIFOEMP) != 0) {
1642 clrchn:
1643 		/*
1644 		 * Clear any handler for this FIFO, decrement
1645 		 * the FIFO use count for the SCB, and release
1646 		 * the FIFO.
1647 		 */
1648 		ahd_outb(ahd, LONGJMP_ADDR + 1, INVALID_ADDR);
1649 		ahd_outb(ahd, SCB_FIFO_USE_COUNT,
1650 			 ahd_inb_scbram(ahd, SCB_FIFO_USE_COUNT) - 1);
1651 		ahd_outb(ahd, DFFSXFRCTL, CLRCHN);
1652 	}
1653 }
1654 
1655 /*
1656  * Look for entries in the QoutFIFO that have completed.
1657  * The valid_tag completion field indicates the validity
1658  * of the entry - the valid value toggles each time through
1659  * the queue. We use the sg_status field in the completion
1660  * entry to avoid referencing the hscb if the completion
1661  * occurred with no errors and no residual.  sg_status is
1662  * a copy of the first byte (little endian) of the sgptr
1663  * hscb field.
1664  */
1665 static void
1666 ahd_run_qoutfifo(struct ahd_softc *ahd)
1667 {
1668 	struct ahd_completion *completion;
1669 	struct scb *scb;
1670 	u_int  scb_index;
1671 
1672 	if ((ahd->flags & AHD_RUNNING_QOUTFIFO) != 0)
1673 		panic("ahd_run_qoutfifo recursion");
1674 	ahd->flags |= AHD_RUNNING_QOUTFIFO;
1675 	ahd_sync_qoutfifo(ahd, BUS_DMASYNC_POSTREAD);
1676 	for (;;) {
1677 		completion = &ahd->qoutfifo[ahd->qoutfifonext];
1678 
1679 		if (completion->valid_tag != ahd->qoutfifonext_valid_tag)
1680 			break;
1681 
1682 		scb_index = ahd_le16toh(completion->tag);
1683 		scb = ahd_lookup_scb(ahd, scb_index);
1684 		if (scb == NULL) {
1685 			printk("%s: WARNING no command for scb %d "
1686 			       "(cmdcmplt)\nQOUTPOS = %d\n",
1687 			       ahd_name(ahd), scb_index,
1688 			       ahd->qoutfifonext);
1689 			ahd_dump_card_state(ahd);
1690 		} else if ((completion->sg_status & SG_STATUS_VALID) != 0) {
1691 			ahd_handle_scb_status(ahd, scb);
1692 		} else {
1693 			ahd_done(ahd, scb);
1694 		}
1695 
1696 		ahd->qoutfifonext = (ahd->qoutfifonext+1) & (AHD_QOUT_SIZE-1);
1697 		if (ahd->qoutfifonext == 0)
1698 			ahd->qoutfifonext_valid_tag ^= QOUTFIFO_ENTRY_VALID;
1699 	}
1700 	ahd->flags &= ~AHD_RUNNING_QOUTFIFO;
1701 }
1702 
1703 /************************* Interrupt Handling *********************************/
1704 static void
1705 ahd_handle_hwerrint(struct ahd_softc *ahd)
1706 {
1707 	/*
1708 	 * Some catastrophic hardware error has occurred.
1709 	 * Print it for the user and disable the controller.
1710 	 */
1711 	int i;
1712 	int error;
1713 
1714 	error = ahd_inb(ahd, ERROR);
1715 	for (i = 0; i < num_errors; i++) {
1716 		if ((error & ahd_hard_errors[i].errno) != 0)
1717 			printk("%s: hwerrint, %s\n",
1718 			       ahd_name(ahd), ahd_hard_errors[i].errmesg);
1719 	}
1720 
1721 	ahd_dump_card_state(ahd);
1722 	panic("BRKADRINT");
1723 
1724 	/* Tell everyone that this HBA is no longer available */
1725 	ahd_abort_scbs(ahd, CAM_TARGET_WILDCARD, ALL_CHANNELS,
1726 		       CAM_LUN_WILDCARD, SCB_LIST_NULL, ROLE_UNKNOWN,
1727 		       CAM_NO_HBA);
1728 
1729 	/* Tell the system that this controller has gone away. */
1730 	ahd_free(ahd);
1731 }
1732 
1733 #ifdef AHD_DEBUG
1734 static void
1735 ahd_dump_sglist(struct scb *scb)
1736 {
1737 	int i;
1738 
1739 	if (scb->sg_count > 0) {
1740 		if ((scb->ahd_softc->flags & AHD_64BIT_ADDRESSING) != 0) {
1741 			struct ahd_dma64_seg *sg_list;
1742 
1743 			sg_list = (struct ahd_dma64_seg*)scb->sg_list;
1744 			for (i = 0; i < scb->sg_count; i++) {
1745 				uint64_t addr;
1746 				uint32_t len;
1747 
1748 				addr = ahd_le64toh(sg_list[i].addr);
1749 				len = ahd_le32toh(sg_list[i].len);
1750 				printk("sg[%d] - Addr 0x%x%x : Length %d%s\n",
1751 				       i,
1752 				       (uint32_t)((addr >> 32) & 0xFFFFFFFF),
1753 				       (uint32_t)(addr & 0xFFFFFFFF),
1754 				       sg_list[i].len & AHD_SG_LEN_MASK,
1755 				       (sg_list[i].len & AHD_DMA_LAST_SEG)
1756 				     ? " Last" : "");
1757 			}
1758 		} else {
1759 			struct ahd_dma_seg *sg_list;
1760 
1761 			sg_list = (struct ahd_dma_seg*)scb->sg_list;
1762 			for (i = 0; i < scb->sg_count; i++) {
1763 				uint32_t len;
1764 
1765 				len = ahd_le32toh(sg_list[i].len);
1766 				printk("sg[%d] - Addr 0x%x%x : Length %d%s\n",
1767 				       i,
1768 				       (len & AHD_SG_HIGH_ADDR_MASK) >> 24,
1769 				       ahd_le32toh(sg_list[i].addr),
1770 				       len & AHD_SG_LEN_MASK,
1771 				       len & AHD_DMA_LAST_SEG ? " Last" : "");
1772 			}
1773 		}
1774 	}
1775 }
1776 #endif  /*  AHD_DEBUG  */
1777 
1778 static void
1779 ahd_handle_seqint(struct ahd_softc *ahd, u_int intstat)
1780 {
1781 	u_int seqintcode;
1782 
1783 	/*
1784 	 * Save the sequencer interrupt code and clear the SEQINT
1785 	 * bit. We will unpause the sequencer, if appropriate,
1786 	 * after servicing the request.
1787 	 */
1788 	seqintcode = ahd_inb(ahd, SEQINTCODE);
1789 	ahd_outb(ahd, CLRINT, CLRSEQINT);
1790 	if ((ahd->bugs & AHD_INTCOLLISION_BUG) != 0) {
1791 		/*
1792 		 * Unpause the sequencer and let it clear
1793 		 * SEQINT by writing NO_SEQINT to it.  This
1794 		 * will cause the sequencer to be paused again,
1795 		 * which is the expected state of this routine.
1796 		 */
1797 		ahd_unpause(ahd);
1798 		while (!ahd_is_paused(ahd))
1799 			;
1800 		ahd_outb(ahd, CLRINT, CLRSEQINT);
1801 	}
1802 	ahd_update_modes(ahd);
1803 #ifdef AHD_DEBUG
1804 	if ((ahd_debug & AHD_SHOW_MISC) != 0)
1805 		printk("%s: Handle Seqint Called for code %d\n",
1806 		       ahd_name(ahd), seqintcode);
1807 #endif
1808 	switch (seqintcode) {
1809 	case ENTERING_NONPACK:
1810 	{
1811 		struct	scb *scb;
1812 		u_int	scbid;
1813 
1814 		AHD_ASSERT_MODES(ahd, ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK),
1815 				 ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK));
1816 		scbid = ahd_get_scbptr(ahd);
1817 		scb = ahd_lookup_scb(ahd, scbid);
1818 		if (scb == NULL) {
1819 			/*
1820 			 * Somehow need to know if this
1821 			 * is from a selection or reselection.
1822 			 * From that, we can determine target
1823 			 * ID so we at least have an I_T nexus.
1824 			 */
1825 		} else {
1826 			ahd_outb(ahd, SAVED_SCSIID, scb->hscb->scsiid);
1827 			ahd_outb(ahd, SAVED_LUN, scb->hscb->lun);
1828 			ahd_outb(ahd, SEQ_FLAGS, 0x0);
1829 		}
1830 		if ((ahd_inb(ahd, LQISTAT2) & LQIPHASE_OUTPKT) != 0
1831 		 && (ahd_inb(ahd, SCSISIGO) & ATNO) != 0) {
1832 			/*
1833 			 * Phase change after read stream with
1834 			 * CRC error with P0 asserted on last
1835 			 * packet.
1836 			 */
1837 #ifdef AHD_DEBUG
1838 			if ((ahd_debug & AHD_SHOW_RECOVERY) != 0)
1839 				printk("%s: Assuming LQIPHASE_NLQ with "
1840 				       "P0 assertion\n", ahd_name(ahd));
1841 #endif
1842 		}
1843 #ifdef AHD_DEBUG
1844 		if ((ahd_debug & AHD_SHOW_RECOVERY) != 0)
1845 			printk("%s: Entering NONPACK\n", ahd_name(ahd));
1846 #endif
1847 		break;
1848 	}
1849 	case INVALID_SEQINT:
1850 		printk("%s: Invalid Sequencer interrupt occurred, "
1851 		       "resetting channel.\n",
1852 		       ahd_name(ahd));
1853 #ifdef AHD_DEBUG
1854 		if ((ahd_debug & AHD_SHOW_RECOVERY) != 0)
1855 			ahd_dump_card_state(ahd);
1856 #endif
1857 		ahd_reset_channel(ahd, 'A', /*Initiate Reset*/TRUE);
1858 		break;
1859 	case STATUS_OVERRUN:
1860 	{
1861 		struct	scb *scb;
1862 		u_int	scbid;
1863 
1864 		scbid = ahd_get_scbptr(ahd);
1865 		scb = ahd_lookup_scb(ahd, scbid);
1866 		if (scb != NULL)
1867 			ahd_print_path(ahd, scb);
1868 		else
1869 			printk("%s: ", ahd_name(ahd));
1870 		printk("SCB %d Packetized Status Overrun", scbid);
1871 		ahd_dump_card_state(ahd);
1872 		ahd_reset_channel(ahd, 'A', /*Initiate Reset*/TRUE);
1873 		break;
1874 	}
1875 	case CFG4ISTAT_INTR:
1876 	{
1877 		struct	scb *scb;
1878 		u_int	scbid;
1879 
1880 		scbid = ahd_get_scbptr(ahd);
1881 		scb = ahd_lookup_scb(ahd, scbid);
1882 		if (scb == NULL) {
1883 			ahd_dump_card_state(ahd);
1884 			printk("CFG4ISTAT: Free SCB %d referenced", scbid);
1885 			panic("For safety");
1886 		}
1887 		ahd_outq(ahd, HADDR, scb->sense_busaddr);
1888 		ahd_outw(ahd, HCNT, AHD_SENSE_BUFSIZE);
1889 		ahd_outb(ahd, HCNT + 2, 0);
1890 		ahd_outb(ahd, SG_CACHE_PRE, SG_LAST_SEG);
1891 		ahd_outb(ahd, DFCNTRL, PRELOADEN|SCSIEN|HDMAEN);
1892 		break;
1893 	}
1894 	case ILLEGAL_PHASE:
1895 	{
1896 		u_int bus_phase;
1897 
1898 		bus_phase = ahd_inb(ahd, SCSISIGI) & PHASE_MASK;
1899 		printk("%s: ILLEGAL_PHASE 0x%x\n",
1900 		       ahd_name(ahd), bus_phase);
1901 
1902 		switch (bus_phase) {
1903 		case P_DATAOUT:
1904 		case P_DATAIN:
1905 		case P_DATAOUT_DT:
1906 		case P_DATAIN_DT:
1907 		case P_MESGOUT:
1908 		case P_STATUS:
1909 		case P_MESGIN:
1910 			ahd_reset_channel(ahd, 'A', /*Initiate Reset*/TRUE);
1911 			printk("%s: Issued Bus Reset.\n", ahd_name(ahd));
1912 			break;
1913 		case P_COMMAND:
1914 		{
1915 			struct	ahd_devinfo devinfo;
1916 			struct	scb *scb;
1917 			struct	ahd_initiator_tinfo *targ_info;
1918 			struct	ahd_tmode_tstate *tstate;
1919 			struct	ahd_transinfo *tinfo;
1920 			u_int	scbid;
1921 
1922 			/*
1923 			 * If a target takes us into the command phase
1924 			 * assume that it has been externally reset and
1925 			 * has thus lost our previous packetized negotiation
1926 			 * agreement.  Since we have not sent an identify
1927 			 * message and may not have fully qualified the
1928 			 * connection, we change our command to TUR, assert
1929 			 * ATN and ABORT the task when we go to message in
1930 			 * phase.  The OSM will see the REQUEUE_REQUEST
1931 			 * status and retry the command.
1932 			 */
1933 			scbid = ahd_get_scbptr(ahd);
1934 			scb = ahd_lookup_scb(ahd, scbid);
1935 			if (scb == NULL) {
1936 				printk("Invalid phase with no valid SCB.  "
1937 				       "Resetting bus.\n");
1938 				ahd_reset_channel(ahd, 'A',
1939 						  /*Initiate Reset*/TRUE);
1940 				break;
1941 			}
1942 			ahd_compile_devinfo(&devinfo, SCB_GET_OUR_ID(scb),
1943 					    SCB_GET_TARGET(ahd, scb),
1944 					    SCB_GET_LUN(scb),
1945 					    SCB_GET_CHANNEL(ahd, scb),
1946 					    ROLE_INITIATOR);
1947 			targ_info = ahd_fetch_transinfo(ahd,
1948 							devinfo.channel,
1949 							devinfo.our_scsiid,
1950 							devinfo.target,
1951 							&tstate);
1952 			tinfo = &targ_info->curr;
1953 			ahd_set_width(ahd, &devinfo, MSG_EXT_WDTR_BUS_8_BIT,
1954 				      AHD_TRANS_ACTIVE, /*paused*/TRUE);
1955 			ahd_set_syncrate(ahd, &devinfo, /*period*/0,
1956 					 /*offset*/0, /*ppr_options*/0,
1957 					 AHD_TRANS_ACTIVE, /*paused*/TRUE);
1958 			/* Hand-craft TUR command */
1959 			ahd_outb(ahd, SCB_CDB_STORE, 0);
1960 			ahd_outb(ahd, SCB_CDB_STORE+1, 0);
1961 			ahd_outb(ahd, SCB_CDB_STORE+2, 0);
1962 			ahd_outb(ahd, SCB_CDB_STORE+3, 0);
1963 			ahd_outb(ahd, SCB_CDB_STORE+4, 0);
1964 			ahd_outb(ahd, SCB_CDB_STORE+5, 0);
1965 			ahd_outb(ahd, SCB_CDB_LEN, 6);
1966 			scb->hscb->control &= ~(TAG_ENB|SCB_TAG_TYPE);
1967 			scb->hscb->control |= MK_MESSAGE;
1968 			ahd_outb(ahd, SCB_CONTROL, scb->hscb->control);
1969 			ahd_outb(ahd, MSG_OUT, HOST_MSG);
1970 			ahd_outb(ahd, SAVED_SCSIID, scb->hscb->scsiid);
1971 			/*
1972 			 * The lun is 0, regardless of the SCB's lun
1973 			 * as we have not sent an identify message.
1974 			 */
1975 			ahd_outb(ahd, SAVED_LUN, 0);
1976 			ahd_outb(ahd, SEQ_FLAGS, 0);
1977 			ahd_assert_atn(ahd);
1978 			scb->flags &= ~SCB_PACKETIZED;
1979 			scb->flags |= SCB_ABORT|SCB_EXTERNAL_RESET;
1980 			ahd_freeze_devq(ahd, scb);
1981 			ahd_set_transaction_status(scb, CAM_REQUEUE_REQ);
1982 			ahd_freeze_scb(scb);
1983 
1984 			/* Notify XPT */
1985 			ahd_send_async(ahd, devinfo.channel, devinfo.target,
1986 				       CAM_LUN_WILDCARD, AC_SENT_BDR);
1987 
1988 			/*
1989 			 * Allow the sequencer to continue with
1990 			 * non-pack processing.
1991 			 */
1992 			ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
1993 			ahd_outb(ahd, CLRLQOINT1, CLRLQOPHACHGINPKT);
1994 			if ((ahd->bugs & AHD_CLRLQO_AUTOCLR_BUG) != 0) {
1995 				ahd_outb(ahd, CLRLQOINT1, 0);
1996 			}
1997 #ifdef AHD_DEBUG
1998 			if ((ahd_debug & AHD_SHOW_RECOVERY) != 0) {
1999 				ahd_print_path(ahd, scb);
2000 				printk("Unexpected command phase from "
2001 				       "packetized target\n");
2002 			}
2003 #endif
2004 			break;
2005 		}
2006 		}
2007 		break;
2008 	}
2009 	case CFG4OVERRUN:
2010 	{
2011 		struct	scb *scb;
2012 		u_int	scb_index;
2013 
2014 #ifdef AHD_DEBUG
2015 		if ((ahd_debug & AHD_SHOW_RECOVERY) != 0) {
2016 			printk("%s: CFG4OVERRUN mode = %x\n", ahd_name(ahd),
2017 			       ahd_inb(ahd, MODE_PTR));
2018 		}
2019 #endif
2020 		scb_index = ahd_get_scbptr(ahd);
2021 		scb = ahd_lookup_scb(ahd, scb_index);
2022 		if (scb == NULL) {
2023 			/*
2024 			 * Attempt to transfer to an SCB that is
2025 			 * not outstanding.
2026 			 */
2027 			ahd_assert_atn(ahd);
2028 			ahd_outb(ahd, MSG_OUT, HOST_MSG);
2029 			ahd->msgout_buf[0] = MSG_ABORT_TASK;
2030 			ahd->msgout_len = 1;
2031 			ahd->msgout_index = 0;
2032 			ahd->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
2033 			/*
2034 			 * Clear status received flag to prevent any
2035 			 * attempt to complete this bogus SCB.
2036 			 */
2037 			ahd_outb(ahd, SCB_CONTROL,
2038 				 ahd_inb_scbram(ahd, SCB_CONTROL)
2039 				 & ~STATUS_RCVD);
2040 		}
2041 		break;
2042 	}
2043 	case DUMP_CARD_STATE:
2044 	{
2045 		ahd_dump_card_state(ahd);
2046 		break;
2047 	}
2048 	case PDATA_REINIT:
2049 	{
2050 #ifdef AHD_DEBUG
2051 		if ((ahd_debug & AHD_SHOW_RECOVERY) != 0) {
2052 			printk("%s: PDATA_REINIT - DFCNTRL = 0x%x "
2053 			       "SG_CACHE_SHADOW = 0x%x\n",
2054 			       ahd_name(ahd), ahd_inb(ahd, DFCNTRL),
2055 			       ahd_inb(ahd, SG_CACHE_SHADOW));
2056 		}
2057 #endif
2058 		ahd_reinitialize_dataptrs(ahd);
2059 		break;
2060 	}
2061 	case HOST_MSG_LOOP:
2062 	{
2063 		struct ahd_devinfo devinfo;
2064 
2065 		/*
2066 		 * The sequencer has encountered a message phase
2067 		 * that requires host assistance for completion.
2068 		 * While handling the message phase(s), we will be
2069 		 * notified by the sequencer after each byte is
2070 		 * transferred so we can track bus phase changes.
2071 		 *
2072 		 * If this is the first time we've seen a HOST_MSG_LOOP
2073 		 * interrupt, initialize the state of the host message
2074 		 * loop.
2075 		 */
2076 		ahd_fetch_devinfo(ahd, &devinfo);
2077 		if (ahd->msg_type == MSG_TYPE_NONE) {
2078 			struct scb *scb;
2079 			u_int scb_index;
2080 			u_int bus_phase;
2081 
2082 			bus_phase = ahd_inb(ahd, SCSISIGI) & PHASE_MASK;
2083 			if (bus_phase != P_MESGIN
2084 			 && bus_phase != P_MESGOUT) {
2085 				printk("ahd_intr: HOST_MSG_LOOP bad "
2086 				       "phase 0x%x\n", bus_phase);
2087 				/*
2088 				 * Probably transitioned to bus free before
2089 				 * we got here.  Just punt the message.
2090 				 */
2091 				ahd_dump_card_state(ahd);
2092 				ahd_clear_intstat(ahd);
2093 				ahd_restart(ahd);
2094 				return;
2095 			}
2096 
2097 			scb_index = ahd_get_scbptr(ahd);
2098 			scb = ahd_lookup_scb(ahd, scb_index);
2099 			if (devinfo.role == ROLE_INITIATOR) {
2100 				if (bus_phase == P_MESGOUT)
2101 					ahd_setup_initiator_msgout(ahd,
2102 								   &devinfo,
2103 								   scb);
2104 				else {
2105 					ahd->msg_type =
2106 					    MSG_TYPE_INITIATOR_MSGIN;
2107 					ahd->msgin_index = 0;
2108 				}
2109 			}
2110 #ifdef AHD_TARGET_MODE
2111 			else {
2112 				if (bus_phase == P_MESGOUT) {
2113 					ahd->msg_type =
2114 					    MSG_TYPE_TARGET_MSGOUT;
2115 					ahd->msgin_index = 0;
2116 				}
2117 				else
2118 					ahd_setup_target_msgin(ahd,
2119 							       &devinfo,
2120 							       scb);
2121 			}
2122 #endif
2123 		}
2124 
2125 		ahd_handle_message_phase(ahd);
2126 		break;
2127 	}
2128 	case NO_MATCH:
2129 	{
2130 		/* Ensure we don't leave the selection hardware on */
2131 		AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
2132 		ahd_outb(ahd, SCSISEQ0, ahd_inb(ahd, SCSISEQ0) & ~ENSELO);
2133 
2134 		printk("%s:%c:%d: no active SCB for reconnecting "
2135 		       "target - issuing BUS DEVICE RESET\n",
2136 		       ahd_name(ahd), 'A', ahd_inb(ahd, SELID) >> 4);
2137 		printk("SAVED_SCSIID == 0x%x, SAVED_LUN == 0x%x, "
2138 		       "REG0 == 0x%x ACCUM = 0x%x\n",
2139 		       ahd_inb(ahd, SAVED_SCSIID), ahd_inb(ahd, SAVED_LUN),
2140 		       ahd_inw(ahd, REG0), ahd_inb(ahd, ACCUM));
2141 		printk("SEQ_FLAGS == 0x%x, SCBPTR == 0x%x, BTT == 0x%x, "
2142 		       "SINDEX == 0x%x\n",
2143 		       ahd_inb(ahd, SEQ_FLAGS), ahd_get_scbptr(ahd),
2144 		       ahd_find_busy_tcl(ahd,
2145 					 BUILD_TCL(ahd_inb(ahd, SAVED_SCSIID),
2146 						   ahd_inb(ahd, SAVED_LUN))),
2147 		       ahd_inw(ahd, SINDEX));
2148 		printk("SELID == 0x%x, SCB_SCSIID == 0x%x, SCB_LUN == 0x%x, "
2149 		       "SCB_CONTROL == 0x%x\n",
2150 		       ahd_inb(ahd, SELID), ahd_inb_scbram(ahd, SCB_SCSIID),
2151 		       ahd_inb_scbram(ahd, SCB_LUN),
2152 		       ahd_inb_scbram(ahd, SCB_CONTROL));
2153 		printk("SCSIBUS[0] == 0x%x, SCSISIGI == 0x%x\n",
2154 		       ahd_inb(ahd, SCSIBUS), ahd_inb(ahd, SCSISIGI));
2155 		printk("SXFRCTL0 == 0x%x\n", ahd_inb(ahd, SXFRCTL0));
2156 		printk("SEQCTL0 == 0x%x\n", ahd_inb(ahd, SEQCTL0));
2157 		ahd_dump_card_state(ahd);
2158 		ahd->msgout_buf[0] = MSG_BUS_DEV_RESET;
2159 		ahd->msgout_len = 1;
2160 		ahd->msgout_index = 0;
2161 		ahd->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
2162 		ahd_outb(ahd, MSG_OUT, HOST_MSG);
2163 		ahd_assert_atn(ahd);
2164 		break;
2165 	}
2166 	case PROTO_VIOLATION:
2167 	{
2168 		ahd_handle_proto_violation(ahd);
2169 		break;
2170 	}
2171 	case IGN_WIDE_RES:
2172 	{
2173 		struct ahd_devinfo devinfo;
2174 
2175 		ahd_fetch_devinfo(ahd, &devinfo);
2176 		ahd_handle_ign_wide_residue(ahd, &devinfo);
2177 		break;
2178 	}
2179 	case BAD_PHASE:
2180 	{
2181 		u_int lastphase;
2182 
2183 		lastphase = ahd_inb(ahd, LASTPHASE);
2184 		printk("%s:%c:%d: unknown scsi bus phase %x, "
2185 		       "lastphase = 0x%x.  Attempting to continue\n",
2186 		       ahd_name(ahd), 'A',
2187 		       SCSIID_TARGET(ahd, ahd_inb(ahd, SAVED_SCSIID)),
2188 		       lastphase, ahd_inb(ahd, SCSISIGI));
2189 		break;
2190 	}
2191 	case MISSED_BUSFREE:
2192 	{
2193 		u_int lastphase;
2194 
2195 		lastphase = ahd_inb(ahd, LASTPHASE);
2196 		printk("%s:%c:%d: Missed busfree. "
2197 		       "Lastphase = 0x%x, Curphase = 0x%x\n",
2198 		       ahd_name(ahd), 'A',
2199 		       SCSIID_TARGET(ahd, ahd_inb(ahd, SAVED_SCSIID)),
2200 		       lastphase, ahd_inb(ahd, SCSISIGI));
2201 		ahd_restart(ahd);
2202 		return;
2203 	}
2204 	case DATA_OVERRUN:
2205 	{
2206 		/*
2207 		 * When the sequencer detects an overrun, it
2208 		 * places the controller in "BITBUCKET" mode
2209 		 * and allows the target to complete its transfer.
2210 		 * Unfortunately, none of the counters get updated
2211 		 * when the controller is in this mode, so we have
2212 		 * no way of knowing how large the overrun was.
2213 		 */
2214 		struct	scb *scb;
2215 		u_int	scbindex;
2216 #ifdef AHD_DEBUG
2217 		u_int	lastphase;
2218 #endif
2219 
2220 		scbindex = ahd_get_scbptr(ahd);
2221 		scb = ahd_lookup_scb(ahd, scbindex);
2222 #ifdef AHD_DEBUG
2223 		lastphase = ahd_inb(ahd, LASTPHASE);
2224 		if ((ahd_debug & AHD_SHOW_RECOVERY) != 0) {
2225 			ahd_print_path(ahd, scb);
2226 			printk("data overrun detected %s.  Tag == 0x%x.\n",
2227 			       ahd_lookup_phase_entry(lastphase)->phasemsg,
2228 			       SCB_GET_TAG(scb));
2229 			ahd_print_path(ahd, scb);
2230 			printk("%s seen Data Phase.  Length = %ld.  "
2231 			       "NumSGs = %d.\n",
2232 			       ahd_inb(ahd, SEQ_FLAGS) & DPHASE
2233 			       ? "Have" : "Haven't",
2234 			       ahd_get_transfer_length(scb), scb->sg_count);
2235 			ahd_dump_sglist(scb);
2236 		}
2237 #endif
2238 
2239 		/*
2240 		 * Set this and it will take effect when the
2241 		 * target does a command complete.
2242 		 */
2243 		ahd_freeze_devq(ahd, scb);
2244 		ahd_set_transaction_status(scb, CAM_DATA_RUN_ERR);
2245 		ahd_freeze_scb(scb);
2246 		break;
2247 	}
2248 	case MKMSG_FAILED:
2249 	{
2250 		struct ahd_devinfo devinfo;
2251 		struct scb *scb;
2252 		u_int scbid;
2253 
2254 		ahd_fetch_devinfo(ahd, &devinfo);
2255 		printk("%s:%c:%d:%d: Attempt to issue message failed\n",
2256 		       ahd_name(ahd), devinfo.channel, devinfo.target,
2257 		       devinfo.lun);
2258 		scbid = ahd_get_scbptr(ahd);
2259 		scb = ahd_lookup_scb(ahd, scbid);
2260 		if (scb != NULL
2261 		 && (scb->flags & SCB_RECOVERY_SCB) != 0)
2262 			/*
2263 			 * Ensure that we didn't put a second instance of this
2264 			 * SCB into the QINFIFO.
2265 			 */
2266 			ahd_search_qinfifo(ahd, SCB_GET_TARGET(ahd, scb),
2267 					   SCB_GET_CHANNEL(ahd, scb),
2268 					   SCB_GET_LUN(scb), SCB_GET_TAG(scb),
2269 					   ROLE_INITIATOR, /*status*/0,
2270 					   SEARCH_REMOVE);
2271 		ahd_outb(ahd, SCB_CONTROL,
2272 			 ahd_inb_scbram(ahd, SCB_CONTROL) & ~MK_MESSAGE);
2273 		break;
2274 	}
2275 	case TASKMGMT_FUNC_COMPLETE:
2276 	{
2277 		u_int	scbid;
2278 		struct	scb *scb;
2279 
2280 		scbid = ahd_get_scbptr(ahd);
2281 		scb = ahd_lookup_scb(ahd, scbid);
2282 		if (scb != NULL) {
2283 			u_int	   lun;
2284 			u_int	   tag;
2285 			cam_status error;
2286 
2287 			ahd_print_path(ahd, scb);
2288 			printk("Task Management Func 0x%x Complete\n",
2289 			       scb->hscb->task_management);
2290 			lun = CAM_LUN_WILDCARD;
2291 			tag = SCB_LIST_NULL;
2292 
2293 			switch (scb->hscb->task_management) {
2294 			case SIU_TASKMGMT_ABORT_TASK:
2295 				tag = SCB_GET_TAG(scb);
2296 			case SIU_TASKMGMT_ABORT_TASK_SET:
2297 			case SIU_TASKMGMT_CLEAR_TASK_SET:
2298 				lun = scb->hscb->lun;
2299 				error = CAM_REQ_ABORTED;
2300 				ahd_abort_scbs(ahd, SCB_GET_TARGET(ahd, scb),
2301 					       'A', lun, tag, ROLE_INITIATOR,
2302 					       error);
2303 				break;
2304 			case SIU_TASKMGMT_LUN_RESET:
2305 				lun = scb->hscb->lun;
2306 			case SIU_TASKMGMT_TARGET_RESET:
2307 			{
2308 				struct ahd_devinfo devinfo;
2309 
2310 				ahd_scb_devinfo(ahd, &devinfo, scb);
2311 				error = CAM_BDR_SENT;
2312 				ahd_handle_devreset(ahd, &devinfo, lun,
2313 						    CAM_BDR_SENT,
2314 						    lun != CAM_LUN_WILDCARD
2315 						    ? "Lun Reset"
2316 						    : "Target Reset",
2317 						    /*verbose_level*/0);
2318 				break;
2319 			}
2320 			default:
2321 				panic("Unexpected TaskMgmt Func\n");
2322 				break;
2323 			}
2324 		}
2325 		break;
2326 	}
2327 	case TASKMGMT_CMD_CMPLT_OKAY:
2328 	{
2329 		u_int	scbid;
2330 		struct	scb *scb;
2331 
2332 		/*
2333 		 * An ABORT TASK TMF failed to be delivered before
2334 		 * the targeted command completed normally.
2335 		 */
2336 		scbid = ahd_get_scbptr(ahd);
2337 		scb = ahd_lookup_scb(ahd, scbid);
2338 		if (scb != NULL) {
2339 			/*
2340 			 * Remove the second instance of this SCB from
2341 			 * the QINFIFO if it is still there.
2342                          */
2343 			ahd_print_path(ahd, scb);
2344 			printk("SCB completes before TMF\n");
2345 			/*
2346 			 * Handle losing the race.  Wait until any
2347 			 * current selection completes.  We will then
2348 			 * set the TMF back to zero in this SCB so that
2349 			 * the sequencer doesn't bother to issue another
2350 			 * sequencer interrupt for its completion.
2351 			 */
2352 			while ((ahd_inb(ahd, SCSISEQ0) & ENSELO) != 0
2353 			    && (ahd_inb(ahd, SSTAT0) & SELDO) == 0
2354 			    && (ahd_inb(ahd, SSTAT1) & SELTO) == 0)
2355 				;
2356 			ahd_outb(ahd, SCB_TASK_MANAGEMENT, 0);
2357 			ahd_search_qinfifo(ahd, SCB_GET_TARGET(ahd, scb),
2358 					   SCB_GET_CHANNEL(ahd, scb),
2359 					   SCB_GET_LUN(scb), SCB_GET_TAG(scb),
2360 					   ROLE_INITIATOR, /*status*/0,
2361 					   SEARCH_REMOVE);
2362 		}
2363 		break;
2364 	}
2365 	case TRACEPOINT0:
2366 	case TRACEPOINT1:
2367 	case TRACEPOINT2:
2368 	case TRACEPOINT3:
2369 		printk("%s: Tracepoint %d\n", ahd_name(ahd),
2370 		       seqintcode - TRACEPOINT0);
2371 		break;
2372 	case NO_SEQINT:
2373 		break;
2374 	case SAW_HWERR:
2375 		ahd_handle_hwerrint(ahd);
2376 		break;
2377 	default:
2378 		printk("%s: Unexpected SEQINTCODE %d\n", ahd_name(ahd),
2379 		       seqintcode);
2380 		break;
2381 	}
2382 	/*
2383 	 *  The sequencer is paused immediately on
2384 	 *  a SEQINT, so we should restart it when
2385 	 *  we're done.
2386 	 */
2387 	ahd_unpause(ahd);
2388 }
2389 
2390 static void
2391 ahd_handle_scsiint(struct ahd_softc *ahd, u_int intstat)
2392 {
2393 	struct scb	*scb;
2394 	u_int		 status0;
2395 	u_int		 status3;
2396 	u_int		 status;
2397 	u_int		 lqistat1;
2398 	u_int		 lqostat0;
2399 	u_int		 scbid;
2400 	u_int		 busfreetime;
2401 
2402 	ahd_update_modes(ahd);
2403 	ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
2404 
2405 	status3 = ahd_inb(ahd, SSTAT3) & (NTRAMPERR|OSRAMPERR);
2406 	status0 = ahd_inb(ahd, SSTAT0) & (IOERR|OVERRUN|SELDI|SELDO);
2407 	status = ahd_inb(ahd, SSTAT1) & (SELTO|SCSIRSTI|BUSFREE|SCSIPERR);
2408 	lqistat1 = ahd_inb(ahd, LQISTAT1);
2409 	lqostat0 = ahd_inb(ahd, LQOSTAT0);
2410 	busfreetime = ahd_inb(ahd, SSTAT2) & BUSFREETIME;
2411 
2412 	/*
2413 	 * Ignore external resets after a bus reset.
2414 	 */
2415 	if (((status & SCSIRSTI) != 0) && (ahd->flags & AHD_BUS_RESET_ACTIVE)) {
2416 		ahd_outb(ahd, CLRSINT1, CLRSCSIRSTI);
2417 		return;
2418 	}
2419 
2420 	/*
2421 	 * Clear bus reset flag
2422 	 */
2423 	ahd->flags &= ~AHD_BUS_RESET_ACTIVE;
2424 
2425 	if ((status0 & (SELDI|SELDO)) != 0) {
2426 		u_int simode0;
2427 
2428 		ahd_set_modes(ahd, AHD_MODE_CFG, AHD_MODE_CFG);
2429 		simode0 = ahd_inb(ahd, SIMODE0);
2430 		status0 &= simode0 & (IOERR|OVERRUN|SELDI|SELDO);
2431 		ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
2432 	}
2433 	scbid = ahd_get_scbptr(ahd);
2434 	scb = ahd_lookup_scb(ahd, scbid);
2435 	if (scb != NULL
2436 	 && (ahd_inb(ahd, SEQ_FLAGS) & NOT_IDENTIFIED) != 0)
2437 		scb = NULL;
2438 
2439 	if ((status0 & IOERR) != 0) {
2440 		u_int now_lvd;
2441 
2442 		now_lvd = ahd_inb(ahd, SBLKCTL) & ENAB40;
2443 		printk("%s: Transceiver State Has Changed to %s mode\n",
2444 		       ahd_name(ahd), now_lvd ? "LVD" : "SE");
2445 		ahd_outb(ahd, CLRSINT0, CLRIOERR);
2446 		/*
2447 		 * A change in I/O mode is equivalent to a bus reset.
2448 		 */
2449 		ahd_reset_channel(ahd, 'A', /*Initiate Reset*/TRUE);
2450 		ahd_pause(ahd);
2451 		ahd_setup_iocell_workaround(ahd);
2452 		ahd_unpause(ahd);
2453 	} else if ((status0 & OVERRUN) != 0) {
2454 
2455 		printk("%s: SCSI offset overrun detected.  Resetting bus.\n",
2456 		       ahd_name(ahd));
2457 		ahd_reset_channel(ahd, 'A', /*Initiate Reset*/TRUE);
2458 	} else if ((status & SCSIRSTI) != 0) {
2459 
2460 		printk("%s: Someone reset channel A\n", ahd_name(ahd));
2461 		ahd_reset_channel(ahd, 'A', /*Initiate Reset*/FALSE);
2462 	} else if ((status & SCSIPERR) != 0) {
2463 
2464 		/* Make sure the sequencer is in a safe location. */
2465 		ahd_clear_critical_section(ahd);
2466 
2467 		ahd_handle_transmission_error(ahd);
2468 	} else if (lqostat0 != 0) {
2469 
2470 		printk("%s: lqostat0 == 0x%x!\n", ahd_name(ahd), lqostat0);
2471 		ahd_outb(ahd, CLRLQOINT0, lqostat0);
2472 		if ((ahd->bugs & AHD_CLRLQO_AUTOCLR_BUG) != 0)
2473 			ahd_outb(ahd, CLRLQOINT1, 0);
2474 	} else if ((status & SELTO) != 0) {
2475 		/* Stop the selection */
2476 		ahd_outb(ahd, SCSISEQ0, 0);
2477 
2478 		/* Make sure the sequencer is in a safe location. */
2479 		ahd_clear_critical_section(ahd);
2480 
2481 		/* No more pending messages */
2482 		ahd_clear_msg_state(ahd);
2483 
2484 		/* Clear interrupt state */
2485 		ahd_outb(ahd, CLRSINT1, CLRSELTIMEO|CLRBUSFREE|CLRSCSIPERR);
2486 
2487 		/*
2488 		 * Although the driver does not care about the
2489 		 * 'Selection in Progress' status bit, the busy
2490 		 * LED does.  SELINGO is only cleared by a successful
2491 		 * selection, so we must manually clear it to insure
2492 		 * the LED turns off just incase no future successful
2493 		 * selections occur (e.g. no devices on the bus).
2494 		 */
2495 		ahd_outb(ahd, CLRSINT0, CLRSELINGO);
2496 
2497 		scbid = ahd_inw(ahd, WAITING_TID_HEAD);
2498 		scb = ahd_lookup_scb(ahd, scbid);
2499 		if (scb == NULL) {
2500 			printk("%s: ahd_intr - referenced scb not "
2501 			       "valid during SELTO scb(0x%x)\n",
2502 			       ahd_name(ahd), scbid);
2503 			ahd_dump_card_state(ahd);
2504 		} else {
2505 			struct ahd_devinfo devinfo;
2506 #ifdef AHD_DEBUG
2507 			if ((ahd_debug & AHD_SHOW_SELTO) != 0) {
2508 				ahd_print_path(ahd, scb);
2509 				printk("Saw Selection Timeout for SCB 0x%x\n",
2510 				       scbid);
2511 			}
2512 #endif
2513 			ahd_scb_devinfo(ahd, &devinfo, scb);
2514 			ahd_set_transaction_status(scb, CAM_SEL_TIMEOUT);
2515 			ahd_freeze_devq(ahd, scb);
2516 
2517 			/*
2518 			 * Cancel any pending transactions on the device
2519 			 * now that it seems to be missing.  This will
2520 			 * also revert us to async/narrow transfers until
2521 			 * we can renegotiate with the device.
2522 			 */
2523 			ahd_handle_devreset(ahd, &devinfo,
2524 					    CAM_LUN_WILDCARD,
2525 					    CAM_SEL_TIMEOUT,
2526 					    "Selection Timeout",
2527 					    /*verbose_level*/1);
2528 		}
2529 		ahd_outb(ahd, CLRINT, CLRSCSIINT);
2530 		ahd_iocell_first_selection(ahd);
2531 		ahd_unpause(ahd);
2532 	} else if ((status0 & (SELDI|SELDO)) != 0) {
2533 
2534 		ahd_iocell_first_selection(ahd);
2535 		ahd_unpause(ahd);
2536 	} else if (status3 != 0) {
2537 		printk("%s: SCSI Cell parity error SSTAT3 == 0x%x\n",
2538 		       ahd_name(ahd), status3);
2539 		ahd_outb(ahd, CLRSINT3, status3);
2540 	} else if ((lqistat1 & (LQIPHASE_LQ|LQIPHASE_NLQ)) != 0) {
2541 
2542 		/* Make sure the sequencer is in a safe location. */
2543 		ahd_clear_critical_section(ahd);
2544 
2545 		ahd_handle_lqiphase_error(ahd, lqistat1);
2546 	} else if ((lqistat1 & LQICRCI_NLQ) != 0) {
2547 		/*
2548 		 * This status can be delayed during some
2549 		 * streaming operations.  The SCSIPHASE
2550 		 * handler has already dealt with this case
2551 		 * so just clear the error.
2552 		 */
2553 		ahd_outb(ahd, CLRLQIINT1, CLRLQICRCI_NLQ);
2554 	} else if ((status & BUSFREE) != 0
2555 		|| (lqistat1 & LQOBUSFREE) != 0) {
2556 		u_int lqostat1;
2557 		int   restart;
2558 		int   clear_fifo;
2559 		int   packetized;
2560 		u_int mode;
2561 
2562 		/*
2563 		 * Clear our selection hardware as soon as possible.
2564 		 * We may have an entry in the waiting Q for this target,
2565 		 * that is affected by this busfree and we don't want to
2566 		 * go about selecting the target while we handle the event.
2567 		 */
2568 		ahd_outb(ahd, SCSISEQ0, 0);
2569 
2570 		/* Make sure the sequencer is in a safe location. */
2571 		ahd_clear_critical_section(ahd);
2572 
2573 		/*
2574 		 * Determine what we were up to at the time of
2575 		 * the busfree.
2576 		 */
2577 		mode = AHD_MODE_SCSI;
2578 		busfreetime = ahd_inb(ahd, SSTAT2) & BUSFREETIME;
2579 		lqostat1 = ahd_inb(ahd, LQOSTAT1);
2580 		switch (busfreetime) {
2581 		case BUSFREE_DFF0:
2582 		case BUSFREE_DFF1:
2583 		{
2584 			mode = busfreetime == BUSFREE_DFF0
2585 			     ? AHD_MODE_DFF0 : AHD_MODE_DFF1;
2586 			ahd_set_modes(ahd, mode, mode);
2587 			scbid = ahd_get_scbptr(ahd);
2588 			scb = ahd_lookup_scb(ahd, scbid);
2589 			if (scb == NULL) {
2590 				printk("%s: Invalid SCB %d in DFF%d "
2591 				       "during unexpected busfree\n",
2592 				       ahd_name(ahd), scbid, mode);
2593 				packetized = 0;
2594 			} else
2595 				packetized = (scb->flags & SCB_PACKETIZED) != 0;
2596 			clear_fifo = 1;
2597 			break;
2598 		}
2599 		case BUSFREE_LQO:
2600 			clear_fifo = 0;
2601 			packetized = 1;
2602 			break;
2603 		default:
2604 			clear_fifo = 0;
2605 			packetized =  (lqostat1 & LQOBUSFREE) != 0;
2606 			if (!packetized
2607 			 && ahd_inb(ahd, LASTPHASE) == P_BUSFREE
2608 			 && (ahd_inb(ahd, SSTAT0) & SELDI) == 0
2609 			 && ((ahd_inb(ahd, SSTAT0) & SELDO) == 0
2610 			  || (ahd_inb(ahd, SCSISEQ0) & ENSELO) == 0))
2611 				/*
2612 				 * Assume packetized if we are not
2613 				 * on the bus in a non-packetized
2614 				 * capacity and any pending selection
2615 				 * was a packetized selection.
2616 				 */
2617 				packetized = 1;
2618 			break;
2619 		}
2620 
2621 #ifdef AHD_DEBUG
2622 		if ((ahd_debug & AHD_SHOW_MISC) != 0)
2623 			printk("Saw Busfree.  Busfreetime = 0x%x.\n",
2624 			       busfreetime);
2625 #endif
2626 		/*
2627 		 * Busfrees that occur in non-packetized phases are
2628 		 * handled by the nonpkt_busfree handler.
2629 		 */
2630 		if (packetized && ahd_inb(ahd, LASTPHASE) == P_BUSFREE) {
2631 			restart = ahd_handle_pkt_busfree(ahd, busfreetime);
2632 		} else {
2633 			packetized = 0;
2634 			restart = ahd_handle_nonpkt_busfree(ahd);
2635 		}
2636 		/*
2637 		 * Clear the busfree interrupt status.  The setting of
2638 		 * the interrupt is a pulse, so in a perfect world, we
2639 		 * would not need to muck with the ENBUSFREE logic.  This
2640 		 * would ensure that if the bus moves on to another
2641 		 * connection, busfree protection is still in force.  If
2642 		 * BUSFREEREV is broken, however, we must manually clear
2643 		 * the ENBUSFREE if the busfree occurred during a non-pack
2644 		 * connection so that we don't get false positives during
2645 		 * future, packetized, connections.
2646 		 */
2647 		ahd_outb(ahd, CLRSINT1, CLRBUSFREE);
2648 		if (packetized == 0
2649 		 && (ahd->bugs & AHD_BUSFREEREV_BUG) != 0)
2650 			ahd_outb(ahd, SIMODE1,
2651 				 ahd_inb(ahd, SIMODE1) & ~ENBUSFREE);
2652 
2653 		if (clear_fifo)
2654 			ahd_clear_fifo(ahd, mode);
2655 
2656 		ahd_clear_msg_state(ahd);
2657 		ahd_outb(ahd, CLRINT, CLRSCSIINT);
2658 		if (restart) {
2659 			ahd_restart(ahd);
2660 		} else {
2661 			ahd_unpause(ahd);
2662 		}
2663 	} else {
2664 		printk("%s: Missing case in ahd_handle_scsiint. status = %x\n",
2665 		       ahd_name(ahd), status);
2666 		ahd_dump_card_state(ahd);
2667 		ahd_clear_intstat(ahd);
2668 		ahd_unpause(ahd);
2669 	}
2670 }
2671 
2672 static void
2673 ahd_handle_transmission_error(struct ahd_softc *ahd)
2674 {
2675 	struct	scb *scb;
2676 	u_int	scbid;
2677 	u_int	lqistat1;
2678 	u_int	lqistat2;
2679 	u_int	msg_out;
2680 	u_int	curphase;
2681 	u_int	lastphase;
2682 	u_int	perrdiag;
2683 	u_int	cur_col;
2684 	int	silent;
2685 
2686 	scb = NULL;
2687 	ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
2688 	lqistat1 = ahd_inb(ahd, LQISTAT1) & ~(LQIPHASE_LQ|LQIPHASE_NLQ);
2689 	lqistat2 = ahd_inb(ahd, LQISTAT2);
2690 	if ((lqistat1 & (LQICRCI_NLQ|LQICRCI_LQ)) == 0
2691 	 && (ahd->bugs & AHD_NLQICRC_DELAYED_BUG) != 0) {
2692 		u_int lqistate;
2693 
2694 		ahd_set_modes(ahd, AHD_MODE_CFG, AHD_MODE_CFG);
2695 		lqistate = ahd_inb(ahd, LQISTATE);
2696 		if ((lqistate >= 0x1E && lqistate <= 0x24)
2697 		 || (lqistate == 0x29)) {
2698 #ifdef AHD_DEBUG
2699 			if ((ahd_debug & AHD_SHOW_RECOVERY) != 0) {
2700 				printk("%s: NLQCRC found via LQISTATE\n",
2701 				       ahd_name(ahd));
2702 			}
2703 #endif
2704 			lqistat1 |= LQICRCI_NLQ;
2705 		}
2706 		ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
2707 	}
2708 
2709 	ahd_outb(ahd, CLRLQIINT1, lqistat1);
2710 	lastphase = ahd_inb(ahd, LASTPHASE);
2711 	curphase = ahd_inb(ahd, SCSISIGI) & PHASE_MASK;
2712 	perrdiag = ahd_inb(ahd, PERRDIAG);
2713 	msg_out = MSG_INITIATOR_DET_ERR;
2714 	ahd_outb(ahd, CLRSINT1, CLRSCSIPERR);
2715 
2716 	/*
2717 	 * Try to find the SCB associated with this error.
2718 	 */
2719 	silent = FALSE;
2720 	if (lqistat1 == 0
2721 	 || (lqistat1 & LQICRCI_NLQ) != 0) {
2722 	 	if ((lqistat1 & (LQICRCI_NLQ|LQIOVERI_NLQ)) != 0)
2723 			ahd_set_active_fifo(ahd);
2724 		scbid = ahd_get_scbptr(ahd);
2725 		scb = ahd_lookup_scb(ahd, scbid);
2726 		if (scb != NULL && SCB_IS_SILENT(scb))
2727 			silent = TRUE;
2728 	}
2729 
2730 	cur_col = 0;
2731 	if (silent == FALSE) {
2732 		printk("%s: Transmission error detected\n", ahd_name(ahd));
2733 		ahd_lqistat1_print(lqistat1, &cur_col, 50);
2734 		ahd_lastphase_print(lastphase, &cur_col, 50);
2735 		ahd_scsisigi_print(curphase, &cur_col, 50);
2736 		ahd_perrdiag_print(perrdiag, &cur_col, 50);
2737 		printk("\n");
2738 		ahd_dump_card_state(ahd);
2739 	}
2740 
2741 	if ((lqistat1 & (LQIOVERI_LQ|LQIOVERI_NLQ)) != 0) {
2742 		if (silent == FALSE) {
2743 			printk("%s: Gross protocol error during incoming "
2744 			       "packet.  lqistat1 == 0x%x.  Resetting bus.\n",
2745 			       ahd_name(ahd), lqistat1);
2746 		}
2747 		ahd_reset_channel(ahd, 'A', /*Initiate Reset*/TRUE);
2748 		return;
2749 	} else if ((lqistat1 & LQICRCI_LQ) != 0) {
2750 		/*
2751 		 * A CRC error has been detected on an incoming LQ.
2752 		 * The bus is currently hung on the last ACK.
2753 		 * Hit LQIRETRY to release the last ack, and
2754 		 * wait for the sequencer to determine that ATNO
2755 		 * is asserted while in message out to take us
2756 		 * to our host message loop.  No NONPACKREQ or
2757 		 * LQIPHASE type errors will occur in this
2758 		 * scenario.  After this first LQIRETRY, the LQI
2759 		 * manager will be in ISELO where it will
2760 		 * happily sit until another packet phase begins.
2761 		 * Unexpected bus free detection is enabled
2762 		 * through any phases that occur after we release
2763 		 * this last ack until the LQI manager sees a
2764 		 * packet phase.  This implies we may have to
2765 		 * ignore a perfectly valid "unexected busfree"
2766 		 * after our "initiator detected error" message is
2767 		 * sent.  A busfree is the expected response after
2768 		 * we tell the target that it's L_Q was corrupted.
2769 		 * (SPI4R09 10.7.3.3.3)
2770 		 */
2771 		ahd_outb(ahd, LQCTL2, LQIRETRY);
2772 		printk("LQIRetry for LQICRCI_LQ to release ACK\n");
2773 	} else if ((lqistat1 & LQICRCI_NLQ) != 0) {
2774 		/*
2775 		 * We detected a CRC error in a NON-LQ packet.
2776 		 * The hardware has varying behavior in this situation
2777 		 * depending on whether this packet was part of a
2778 		 * stream or not.
2779 		 *
2780 		 * PKT by PKT mode:
2781 		 * The hardware has already acked the complete packet.
2782 		 * If the target honors our outstanding ATN condition,
2783 		 * we should be (or soon will be) in MSGOUT phase.
2784 		 * This will trigger the LQIPHASE_LQ status bit as the
2785 		 * hardware was expecting another LQ.  Unexpected
2786 		 * busfree detection is enabled.  Once LQIPHASE_LQ is
2787 		 * true (first entry into host message loop is much
2788 		 * the same), we must clear LQIPHASE_LQ and hit
2789 		 * LQIRETRY so the hardware is ready to handle
2790 		 * a future LQ.  NONPACKREQ will not be asserted again
2791 		 * once we hit LQIRETRY until another packet is
2792 		 * processed.  The target may either go busfree
2793 		 * or start another packet in response to our message.
2794 		 *
2795 		 * Read Streaming P0 asserted:
2796 		 * If we raise ATN and the target completes the entire
2797 		 * stream (P0 asserted during the last packet), the
2798 		 * hardware will ack all data and return to the ISTART
2799 		 * state.  When the target reponds to our ATN condition,
2800 		 * LQIPHASE_LQ will be asserted.  We should respond to
2801 		 * this with an LQIRETRY to prepare for any future
2802 		 * packets.  NONPACKREQ will not be asserted again
2803 		 * once we hit LQIRETRY until another packet is
2804 		 * processed.  The target may either go busfree or
2805 		 * start another packet in response to our message.
2806 		 * Busfree detection is enabled.
2807 		 *
2808 		 * Read Streaming P0 not asserted:
2809 		 * If we raise ATN and the target transitions to
2810 		 * MSGOUT in or after a packet where P0 is not
2811 		 * asserted, the hardware will assert LQIPHASE_NLQ.
2812 		 * We should respond to the LQIPHASE_NLQ with an
2813 		 * LQIRETRY.  Should the target stay in a non-pkt
2814 		 * phase after we send our message, the hardware
2815 		 * will assert LQIPHASE_LQ.  Recovery is then just as
2816 		 * listed above for the read streaming with P0 asserted.
2817 		 * Busfree detection is enabled.
2818 		 */
2819 		if (silent == FALSE)
2820 			printk("LQICRC_NLQ\n");
2821 		if (scb == NULL) {
2822 			printk("%s: No SCB valid for LQICRC_NLQ.  "
2823 			       "Resetting bus\n", ahd_name(ahd));
2824 			ahd_reset_channel(ahd, 'A', /*Initiate Reset*/TRUE);
2825 			return;
2826 		}
2827 	} else if ((lqistat1 & LQIBADLQI) != 0) {
2828 		printk("Need to handle BADLQI!\n");
2829 		ahd_reset_channel(ahd, 'A', /*Initiate Reset*/TRUE);
2830 		return;
2831 	} else if ((perrdiag & (PARITYERR|PREVPHASE)) == PARITYERR) {
2832 		if ((curphase & ~P_DATAIN_DT) != 0) {
2833 			/* Ack the byte.  So we can continue. */
2834 			if (silent == FALSE)
2835 				printk("Acking %s to clear perror\n",
2836 				    ahd_lookup_phase_entry(curphase)->phasemsg);
2837 			ahd_inb(ahd, SCSIDAT);
2838 		}
2839 
2840 		if (curphase == P_MESGIN)
2841 			msg_out = MSG_PARITY_ERROR;
2842 	}
2843 
2844 	/*
2845 	 * We've set the hardware to assert ATN if we
2846 	 * get a parity error on "in" phases, so all we
2847 	 * need to do is stuff the message buffer with
2848 	 * the appropriate message.  "In" phases have set
2849 	 * mesg_out to something other than MSG_NOP.
2850 	 */
2851 	ahd->send_msg_perror = msg_out;
2852 	if (scb != NULL && msg_out == MSG_INITIATOR_DET_ERR)
2853 		scb->flags |= SCB_TRANSMISSION_ERROR;
2854 	ahd_outb(ahd, MSG_OUT, HOST_MSG);
2855 	ahd_outb(ahd, CLRINT, CLRSCSIINT);
2856 	ahd_unpause(ahd);
2857 }
2858 
2859 static void
2860 ahd_handle_lqiphase_error(struct ahd_softc *ahd, u_int lqistat1)
2861 {
2862 	/*
2863 	 * Clear the sources of the interrupts.
2864 	 */
2865 	ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
2866 	ahd_outb(ahd, CLRLQIINT1, lqistat1);
2867 
2868 	/*
2869 	 * If the "illegal" phase changes were in response
2870 	 * to our ATN to flag a CRC error, AND we ended up
2871 	 * on packet boundaries, clear the error, restart the
2872 	 * LQI manager as appropriate, and go on our merry
2873 	 * way toward sending the message.  Otherwise, reset
2874 	 * the bus to clear the error.
2875 	 */
2876 	ahd_set_active_fifo(ahd);
2877 	if ((ahd_inb(ahd, SCSISIGO) & ATNO) != 0
2878 	 && (ahd_inb(ahd, MDFFSTAT) & DLZERO) != 0) {
2879 		if ((lqistat1 & LQIPHASE_LQ) != 0) {
2880 			printk("LQIRETRY for LQIPHASE_LQ\n");
2881 			ahd_outb(ahd, LQCTL2, LQIRETRY);
2882 		} else if ((lqistat1 & LQIPHASE_NLQ) != 0) {
2883 			printk("LQIRETRY for LQIPHASE_NLQ\n");
2884 			ahd_outb(ahd, LQCTL2, LQIRETRY);
2885 		} else
2886 			panic("ahd_handle_lqiphase_error: No phase errors\n");
2887 		ahd_dump_card_state(ahd);
2888 		ahd_outb(ahd, CLRINT, CLRSCSIINT);
2889 		ahd_unpause(ahd);
2890 	} else {
2891 		printk("Resetting Channel for LQI Phase error\n");
2892 		ahd_dump_card_state(ahd);
2893 		ahd_reset_channel(ahd, 'A', /*Initiate Reset*/TRUE);
2894 	}
2895 }
2896 
2897 /*
2898  * Packetized unexpected or expected busfree.
2899  * Entered in mode based on busfreetime.
2900  */
2901 static int
2902 ahd_handle_pkt_busfree(struct ahd_softc *ahd, u_int busfreetime)
2903 {
2904 	u_int lqostat1;
2905 
2906 	AHD_ASSERT_MODES(ahd, ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK),
2907 			 ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK));
2908 	lqostat1 = ahd_inb(ahd, LQOSTAT1);
2909 	if ((lqostat1 & LQOBUSFREE) != 0) {
2910 		struct scb *scb;
2911 		u_int scbid;
2912 		u_int saved_scbptr;
2913 		u_int waiting_h;
2914 		u_int waiting_t;
2915 		u_int next;
2916 
2917 		/*
2918 		 * The LQO manager detected an unexpected busfree
2919 		 * either:
2920 		 *
2921 		 * 1) During an outgoing LQ.
2922 		 * 2) After an outgoing LQ but before the first
2923 		 *    REQ of the command packet.
2924 		 * 3) During an outgoing command packet.
2925 		 *
2926 		 * In all cases, CURRSCB is pointing to the
2927 		 * SCB that encountered the failure.  Clean
2928 		 * up the queue, clear SELDO and LQOBUSFREE,
2929 		 * and allow the sequencer to restart the select
2930 		 * out at its lesure.
2931 		 */
2932 		ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
2933 		scbid = ahd_inw(ahd, CURRSCB);
2934 		scb = ahd_lookup_scb(ahd, scbid);
2935 		if (scb == NULL)
2936 		       panic("SCB not valid during LQOBUSFREE");
2937 		/*
2938 		 * Clear the status.
2939 		 */
2940 		ahd_outb(ahd, CLRLQOINT1, CLRLQOBUSFREE);
2941 		if ((ahd->bugs & AHD_CLRLQO_AUTOCLR_BUG) != 0)
2942 			ahd_outb(ahd, CLRLQOINT1, 0);
2943 		ahd_outb(ahd, SCSISEQ0, ahd_inb(ahd, SCSISEQ0) & ~ENSELO);
2944 		ahd_flush_device_writes(ahd);
2945 		ahd_outb(ahd, CLRSINT0, CLRSELDO);
2946 
2947 		/*
2948 		 * Return the LQO manager to its idle loop.  It will
2949 		 * not do this automatically if the busfree occurs
2950 		 * after the first REQ of either the LQ or command
2951 		 * packet or between the LQ and command packet.
2952 		 */
2953 		ahd_outb(ahd, LQCTL2, ahd_inb(ahd, LQCTL2) | LQOTOIDLE);
2954 
2955 		/*
2956 		 * Update the waiting for selection queue so
2957 		 * we restart on the correct SCB.
2958 		 */
2959 		waiting_h = ahd_inw(ahd, WAITING_TID_HEAD);
2960 		saved_scbptr = ahd_get_scbptr(ahd);
2961 		if (waiting_h != scbid) {
2962 
2963 			ahd_outw(ahd, WAITING_TID_HEAD, scbid);
2964 			waiting_t = ahd_inw(ahd, WAITING_TID_TAIL);
2965 			if (waiting_t == waiting_h) {
2966 				ahd_outw(ahd, WAITING_TID_TAIL, scbid);
2967 				next = SCB_LIST_NULL;
2968 			} else {
2969 				ahd_set_scbptr(ahd, waiting_h);
2970 				next = ahd_inw_scbram(ahd, SCB_NEXT2);
2971 			}
2972 			ahd_set_scbptr(ahd, scbid);
2973 			ahd_outw(ahd, SCB_NEXT2, next);
2974 		}
2975 		ahd_set_scbptr(ahd, saved_scbptr);
2976 		if (scb->crc_retry_count < AHD_MAX_LQ_CRC_ERRORS) {
2977 			if (SCB_IS_SILENT(scb) == FALSE) {
2978 				ahd_print_path(ahd, scb);
2979 				printk("Probable outgoing LQ CRC error.  "
2980 				       "Retrying command\n");
2981 			}
2982 			scb->crc_retry_count++;
2983 		} else {
2984 			ahd_set_transaction_status(scb, CAM_UNCOR_PARITY);
2985 			ahd_freeze_scb(scb);
2986 			ahd_freeze_devq(ahd, scb);
2987 		}
2988 		/* Return unpausing the sequencer. */
2989 		return (0);
2990 	} else if ((ahd_inb(ahd, PERRDIAG) & PARITYERR) != 0) {
2991 		/*
2992 		 * Ignore what are really parity errors that
2993 		 * occur on the last REQ of a free running
2994 		 * clock prior to going busfree.  Some drives
2995 		 * do not properly active negate just before
2996 		 * going busfree resulting in a parity glitch.
2997 		 */
2998 		ahd_outb(ahd, CLRSINT1, CLRSCSIPERR|CLRBUSFREE);
2999 #ifdef AHD_DEBUG
3000 		if ((ahd_debug & AHD_SHOW_MASKED_ERRORS) != 0)
3001 			printk("%s: Parity on last REQ detected "
3002 			       "during busfree phase.\n",
3003 			       ahd_name(ahd));
3004 #endif
3005 		/* Return unpausing the sequencer. */
3006 		return (0);
3007 	}
3008 	if (ahd->src_mode != AHD_MODE_SCSI) {
3009 		u_int	scbid;
3010 		struct	scb *scb;
3011 
3012 		scbid = ahd_get_scbptr(ahd);
3013 		scb = ahd_lookup_scb(ahd, scbid);
3014 		ahd_print_path(ahd, scb);
3015 		printk("Unexpected PKT busfree condition\n");
3016 		ahd_dump_card_state(ahd);
3017 		ahd_abort_scbs(ahd, SCB_GET_TARGET(ahd, scb), 'A',
3018 			       SCB_GET_LUN(scb), SCB_GET_TAG(scb),
3019 			       ROLE_INITIATOR, CAM_UNEXP_BUSFREE);
3020 
3021 		/* Return restarting the sequencer. */
3022 		return (1);
3023 	}
3024 	printk("%s: Unexpected PKT busfree condition\n", ahd_name(ahd));
3025 	ahd_dump_card_state(ahd);
3026 	/* Restart the sequencer. */
3027 	return (1);
3028 }
3029 
3030 /*
3031  * Non-packetized unexpected or expected busfree.
3032  */
3033 static int
3034 ahd_handle_nonpkt_busfree(struct ahd_softc *ahd)
3035 {
3036 	struct	ahd_devinfo devinfo;
3037 	struct	scb *scb;
3038 	u_int	lastphase;
3039 	u_int	saved_scsiid;
3040 	u_int	saved_lun;
3041 	u_int	target;
3042 	u_int	initiator_role_id;
3043 	u_int	scbid;
3044 	u_int	ppr_busfree;
3045 	int	printerror;
3046 
3047 	/*
3048 	 * Look at what phase we were last in.  If its message out,
3049 	 * chances are pretty good that the busfree was in response
3050 	 * to one of our abort requests.
3051 	 */
3052 	lastphase = ahd_inb(ahd, LASTPHASE);
3053 	saved_scsiid = ahd_inb(ahd, SAVED_SCSIID);
3054 	saved_lun = ahd_inb(ahd, SAVED_LUN);
3055 	target = SCSIID_TARGET(ahd, saved_scsiid);
3056 	initiator_role_id = SCSIID_OUR_ID(saved_scsiid);
3057 	ahd_compile_devinfo(&devinfo, initiator_role_id,
3058 			    target, saved_lun, 'A', ROLE_INITIATOR);
3059 	printerror = 1;
3060 
3061 	scbid = ahd_get_scbptr(ahd);
3062 	scb = ahd_lookup_scb(ahd, scbid);
3063 	if (scb != NULL
3064 	 && (ahd_inb(ahd, SEQ_FLAGS) & NOT_IDENTIFIED) != 0)
3065 		scb = NULL;
3066 
3067 	ppr_busfree = (ahd->msg_flags & MSG_FLAG_EXPECT_PPR_BUSFREE) != 0;
3068 	if (lastphase == P_MESGOUT) {
3069 		u_int tag;
3070 
3071 		tag = SCB_LIST_NULL;
3072 		if (ahd_sent_msg(ahd, AHDMSG_1B, MSG_ABORT_TAG, TRUE)
3073 		 || ahd_sent_msg(ahd, AHDMSG_1B, MSG_ABORT, TRUE)) {
3074 			int found;
3075 			int sent_msg;
3076 
3077 			if (scb == NULL) {
3078 				ahd_print_devinfo(ahd, &devinfo);
3079 				printk("Abort for unidentified "
3080 				       "connection completed.\n");
3081 				/* restart the sequencer. */
3082 				return (1);
3083 			}
3084 			sent_msg = ahd->msgout_buf[ahd->msgout_index - 1];
3085 			ahd_print_path(ahd, scb);
3086 			printk("SCB %d - Abort%s Completed.\n",
3087 			       SCB_GET_TAG(scb),
3088 			       sent_msg == MSG_ABORT_TAG ? "" : " Tag");
3089 
3090 			if (sent_msg == MSG_ABORT_TAG)
3091 				tag = SCB_GET_TAG(scb);
3092 
3093 			if ((scb->flags & SCB_EXTERNAL_RESET) != 0) {
3094 				/*
3095 				 * This abort is in response to an
3096 				 * unexpected switch to command phase
3097 				 * for a packetized connection.  Since
3098 				 * the identify message was never sent,
3099 				 * "saved lun" is 0.  We really want to
3100 				 * abort only the SCB that encountered
3101 				 * this error, which could have a different
3102 				 * lun.  The SCB will be retried so the OS
3103 				 * will see the UA after renegotiating to
3104 				 * packetized.
3105 				 */
3106 				tag = SCB_GET_TAG(scb);
3107 				saved_lun = scb->hscb->lun;
3108 			}
3109 			found = ahd_abort_scbs(ahd, target, 'A', saved_lun,
3110 					       tag, ROLE_INITIATOR,
3111 					       CAM_REQ_ABORTED);
3112 			printk("found == 0x%x\n", found);
3113 			printerror = 0;
3114 		} else if (ahd_sent_msg(ahd, AHDMSG_1B,
3115 					MSG_BUS_DEV_RESET, TRUE)) {
3116 #ifdef __FreeBSD__
3117 			/*
3118 			 * Don't mark the user's request for this BDR
3119 			 * as completing with CAM_BDR_SENT.  CAM3
3120 			 * specifies CAM_REQ_CMP.
3121 			 */
3122 			if (scb != NULL
3123 			 && scb->io_ctx->ccb_h.func_code== XPT_RESET_DEV
3124 			 && ahd_match_scb(ahd, scb, target, 'A',
3125 					  CAM_LUN_WILDCARD, SCB_LIST_NULL,
3126 					  ROLE_INITIATOR))
3127 				ahd_set_transaction_status(scb, CAM_REQ_CMP);
3128 #endif
3129 			ahd_handle_devreset(ahd, &devinfo, CAM_LUN_WILDCARD,
3130 					    CAM_BDR_SENT, "Bus Device Reset",
3131 					    /*verbose_level*/0);
3132 			printerror = 0;
3133 		} else if (ahd_sent_msg(ahd, AHDMSG_EXT, MSG_EXT_PPR, FALSE)
3134 			&& ppr_busfree == 0) {
3135 			struct ahd_initiator_tinfo *tinfo;
3136 			struct ahd_tmode_tstate *tstate;
3137 
3138 			/*
3139 			 * PPR Rejected.
3140 			 *
3141 			 * If the previous negotiation was packetized,
3142 			 * this could be because the device has been
3143 			 * reset without our knowledge.  Force our
3144 			 * current negotiation to async and retry the
3145 			 * negotiation.  Otherwise retry the command
3146 			 * with non-ppr negotiation.
3147 			 */
3148 #ifdef AHD_DEBUG
3149 			if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
3150 				printk("PPR negotiation rejected busfree.\n");
3151 #endif
3152 			tinfo = ahd_fetch_transinfo(ahd, devinfo.channel,
3153 						    devinfo.our_scsiid,
3154 						    devinfo.target, &tstate);
3155 			if ((tinfo->curr.ppr_options & MSG_EXT_PPR_IU_REQ)!=0) {
3156 				ahd_set_width(ahd, &devinfo,
3157 					      MSG_EXT_WDTR_BUS_8_BIT,
3158 					      AHD_TRANS_CUR,
3159 					      /*paused*/TRUE);
3160 				ahd_set_syncrate(ahd, &devinfo,
3161 						/*period*/0, /*offset*/0,
3162 						/*ppr_options*/0,
3163 						AHD_TRANS_CUR,
3164 						/*paused*/TRUE);
3165 				/*
3166 				 * The expect PPR busfree handler below
3167 				 * will effect the retry and necessary
3168 				 * abort.
3169 				 */
3170 			} else {
3171 				tinfo->curr.transport_version = 2;
3172 				tinfo->goal.transport_version = 2;
3173 				tinfo->goal.ppr_options = 0;
3174 				if (scb != NULL) {
3175 					/*
3176 					 * Remove any SCBs in the waiting
3177 					 * for selection queue that may
3178 					 * also be for this target so that
3179 					 * command ordering is preserved.
3180 					 */
3181 					ahd_freeze_devq(ahd, scb);
3182 					ahd_qinfifo_requeue_tail(ahd, scb);
3183 				}
3184 				printerror = 0;
3185 			}
3186 		} else if (ahd_sent_msg(ahd, AHDMSG_EXT, MSG_EXT_WDTR, FALSE)
3187 			&& ppr_busfree == 0) {
3188 			/*
3189 			 * Negotiation Rejected.  Go-narrow and
3190 			 * retry command.
3191 			 */
3192 #ifdef AHD_DEBUG
3193 			if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
3194 				printk("WDTR negotiation rejected busfree.\n");
3195 #endif
3196 			ahd_set_width(ahd, &devinfo,
3197 				      MSG_EXT_WDTR_BUS_8_BIT,
3198 				      AHD_TRANS_CUR|AHD_TRANS_GOAL,
3199 				      /*paused*/TRUE);
3200 			if (scb != NULL) {
3201 				/*
3202 				 * Remove any SCBs in the waiting for
3203 				 * selection queue that may also be for
3204 				 * this target so that command ordering
3205 				 * is preserved.
3206 				 */
3207 				ahd_freeze_devq(ahd, scb);
3208 				ahd_qinfifo_requeue_tail(ahd, scb);
3209 			}
3210 			printerror = 0;
3211 		} else if (ahd_sent_msg(ahd, AHDMSG_EXT, MSG_EXT_SDTR, FALSE)
3212 			&& ppr_busfree == 0) {
3213 			/*
3214 			 * Negotiation Rejected.  Go-async and
3215 			 * retry command.
3216 			 */
3217 #ifdef AHD_DEBUG
3218 			if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
3219 				printk("SDTR negotiation rejected busfree.\n");
3220 #endif
3221 			ahd_set_syncrate(ahd, &devinfo,
3222 					/*period*/0, /*offset*/0,
3223 					/*ppr_options*/0,
3224 					AHD_TRANS_CUR|AHD_TRANS_GOAL,
3225 					/*paused*/TRUE);
3226 			if (scb != NULL) {
3227 				/*
3228 				 * Remove any SCBs in the waiting for
3229 				 * selection queue that may also be for
3230 				 * this target so that command ordering
3231 				 * is preserved.
3232 				 */
3233 				ahd_freeze_devq(ahd, scb);
3234 				ahd_qinfifo_requeue_tail(ahd, scb);
3235 			}
3236 			printerror = 0;
3237 		} else if ((ahd->msg_flags & MSG_FLAG_EXPECT_IDE_BUSFREE) != 0
3238 			&& ahd_sent_msg(ahd, AHDMSG_1B,
3239 					 MSG_INITIATOR_DET_ERR, TRUE)) {
3240 
3241 #ifdef AHD_DEBUG
3242 			if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
3243 				printk("Expected IDE Busfree\n");
3244 #endif
3245 			printerror = 0;
3246 		} else if ((ahd->msg_flags & MSG_FLAG_EXPECT_QASREJ_BUSFREE)
3247 			&& ahd_sent_msg(ahd, AHDMSG_1B,
3248 					MSG_MESSAGE_REJECT, TRUE)) {
3249 
3250 #ifdef AHD_DEBUG
3251 			if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
3252 				printk("Expected QAS Reject Busfree\n");
3253 #endif
3254 			printerror = 0;
3255 		}
3256 	}
3257 
3258 	/*
3259 	 * The busfree required flag is honored at the end of
3260 	 * the message phases.  We check it last in case we
3261 	 * had to send some other message that caused a busfree.
3262 	 */
3263 	if (scb != NULL && printerror != 0
3264 	 && (lastphase == P_MESGIN || lastphase == P_MESGOUT)
3265 	 && ((ahd->msg_flags & MSG_FLAG_EXPECT_PPR_BUSFREE) != 0)) {
3266 
3267 		ahd_freeze_devq(ahd, scb);
3268 		ahd_set_transaction_status(scb, CAM_REQUEUE_REQ);
3269 		ahd_freeze_scb(scb);
3270 		if ((ahd->msg_flags & MSG_FLAG_IU_REQ_CHANGED) != 0) {
3271 			ahd_abort_scbs(ahd, SCB_GET_TARGET(ahd, scb),
3272 				       SCB_GET_CHANNEL(ahd, scb),
3273 				       SCB_GET_LUN(scb), SCB_LIST_NULL,
3274 				       ROLE_INITIATOR, CAM_REQ_ABORTED);
3275 		} else {
3276 #ifdef AHD_DEBUG
3277 			if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
3278 				printk("PPR Negotiation Busfree.\n");
3279 #endif
3280 			ahd_done(ahd, scb);
3281 		}
3282 		printerror = 0;
3283 	}
3284 	if (printerror != 0) {
3285 		int aborted;
3286 
3287 		aborted = 0;
3288 		if (scb != NULL) {
3289 			u_int tag;
3290 
3291 			if ((scb->hscb->control & TAG_ENB) != 0)
3292 				tag = SCB_GET_TAG(scb);
3293 			else
3294 				tag = SCB_LIST_NULL;
3295 			ahd_print_path(ahd, scb);
3296 			aborted = ahd_abort_scbs(ahd, target, 'A',
3297 				       SCB_GET_LUN(scb), tag,
3298 				       ROLE_INITIATOR,
3299 				       CAM_UNEXP_BUSFREE);
3300 		} else {
3301 			/*
3302 			 * We had not fully identified this connection,
3303 			 * so we cannot abort anything.
3304 			 */
3305 			printk("%s: ", ahd_name(ahd));
3306 		}
3307 		printk("Unexpected busfree %s, %d SCBs aborted, "
3308 		       "PRGMCNT == 0x%x\n",
3309 		       ahd_lookup_phase_entry(lastphase)->phasemsg,
3310 		       aborted,
3311 		       ahd_inw(ahd, PRGMCNT));
3312 		ahd_dump_card_state(ahd);
3313 		if (lastphase != P_BUSFREE)
3314 			ahd_force_renegotiation(ahd, &devinfo);
3315 	}
3316 	/* Always restart the sequencer. */
3317 	return (1);
3318 }
3319 
3320 static void
3321 ahd_handle_proto_violation(struct ahd_softc *ahd)
3322 {
3323 	struct	ahd_devinfo devinfo;
3324 	struct	scb *scb;
3325 	u_int	scbid;
3326 	u_int	seq_flags;
3327 	u_int	curphase;
3328 	u_int	lastphase;
3329 	int	found;
3330 
3331 	ahd_fetch_devinfo(ahd, &devinfo);
3332 	scbid = ahd_get_scbptr(ahd);
3333 	scb = ahd_lookup_scb(ahd, scbid);
3334 	seq_flags = ahd_inb(ahd, SEQ_FLAGS);
3335 	curphase = ahd_inb(ahd, SCSISIGI) & PHASE_MASK;
3336 	lastphase = ahd_inb(ahd, LASTPHASE);
3337 	if ((seq_flags & NOT_IDENTIFIED) != 0) {
3338 
3339 		/*
3340 		 * The reconnecting target either did not send an
3341 		 * identify message, or did, but we didn't find an SCB
3342 		 * to match.
3343 		 */
3344 		ahd_print_devinfo(ahd, &devinfo);
3345 		printk("Target did not send an IDENTIFY message. "
3346 		       "LASTPHASE = 0x%x.\n", lastphase);
3347 		scb = NULL;
3348 	} else if (scb == NULL) {
3349 		/*
3350 		 * We don't seem to have an SCB active for this
3351 		 * transaction.  Print an error and reset the bus.
3352 		 */
3353 		ahd_print_devinfo(ahd, &devinfo);
3354 		printk("No SCB found during protocol violation\n");
3355 		goto proto_violation_reset;
3356 	} else {
3357 		ahd_set_transaction_status(scb, CAM_SEQUENCE_FAIL);
3358 		if ((seq_flags & NO_CDB_SENT) != 0) {
3359 			ahd_print_path(ahd, scb);
3360 			printk("No or incomplete CDB sent to device.\n");
3361 		} else if ((ahd_inb_scbram(ahd, SCB_CONTROL)
3362 			  & STATUS_RCVD) == 0) {
3363 			/*
3364 			 * The target never bothered to provide status to
3365 			 * us prior to completing the command.  Since we don't
3366 			 * know the disposition of this command, we must attempt
3367 			 * to abort it.  Assert ATN and prepare to send an abort
3368 			 * message.
3369 			 */
3370 			ahd_print_path(ahd, scb);
3371 			printk("Completed command without status.\n");
3372 		} else {
3373 			ahd_print_path(ahd, scb);
3374 			printk("Unknown protocol violation.\n");
3375 			ahd_dump_card_state(ahd);
3376 		}
3377 	}
3378 	if ((lastphase & ~P_DATAIN_DT) == 0
3379 	 || lastphase == P_COMMAND) {
3380 proto_violation_reset:
3381 		/*
3382 		 * Target either went directly to data
3383 		 * phase or didn't respond to our ATN.
3384 		 * The only safe thing to do is to blow
3385 		 * it away with a bus reset.
3386 		 */
3387 		found = ahd_reset_channel(ahd, 'A', TRUE);
3388 		printk("%s: Issued Channel %c Bus Reset. "
3389 		       "%d SCBs aborted\n", ahd_name(ahd), 'A', found);
3390 	} else {
3391 		/*
3392 		 * Leave the selection hardware off in case
3393 		 * this abort attempt will affect yet to
3394 		 * be sent commands.
3395 		 */
3396 		ahd_outb(ahd, SCSISEQ0,
3397 			 ahd_inb(ahd, SCSISEQ0) & ~ENSELO);
3398 		ahd_assert_atn(ahd);
3399 		ahd_outb(ahd, MSG_OUT, HOST_MSG);
3400 		if (scb == NULL) {
3401 			ahd_print_devinfo(ahd, &devinfo);
3402 			ahd->msgout_buf[0] = MSG_ABORT_TASK;
3403 			ahd->msgout_len = 1;
3404 			ahd->msgout_index = 0;
3405 			ahd->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
3406 		} else {
3407 			ahd_print_path(ahd, scb);
3408 			scb->flags |= SCB_ABORT;
3409 		}
3410 		printk("Protocol violation %s.  Attempting to abort.\n",
3411 		       ahd_lookup_phase_entry(curphase)->phasemsg);
3412 	}
3413 }
3414 
3415 /*
3416  * Force renegotiation to occur the next time we initiate
3417  * a command to the current device.
3418  */
3419 static void
3420 ahd_force_renegotiation(struct ahd_softc *ahd, struct ahd_devinfo *devinfo)
3421 {
3422 	struct	ahd_initiator_tinfo *targ_info;
3423 	struct	ahd_tmode_tstate *tstate;
3424 
3425 #ifdef AHD_DEBUG
3426 	if ((ahd_debug & AHD_SHOW_MESSAGES) != 0) {
3427 		ahd_print_devinfo(ahd, devinfo);
3428 		printk("Forcing renegotiation\n");
3429 	}
3430 #endif
3431 	targ_info = ahd_fetch_transinfo(ahd,
3432 					devinfo->channel,
3433 					devinfo->our_scsiid,
3434 					devinfo->target,
3435 					&tstate);
3436 	ahd_update_neg_request(ahd, devinfo, tstate,
3437 			       targ_info, AHD_NEG_IF_NON_ASYNC);
3438 }
3439 
3440 #define AHD_MAX_STEPS 2000
3441 static void
3442 ahd_clear_critical_section(struct ahd_softc *ahd)
3443 {
3444 	ahd_mode_state	saved_modes;
3445 	int		stepping;
3446 	int		steps;
3447 	int		first_instr;
3448 	u_int		simode0;
3449 	u_int		simode1;
3450 	u_int		simode3;
3451 	u_int		lqimode0;
3452 	u_int		lqimode1;
3453 	u_int		lqomode0;
3454 	u_int		lqomode1;
3455 
3456 	if (ahd->num_critical_sections == 0)
3457 		return;
3458 
3459 	stepping = FALSE;
3460 	steps = 0;
3461 	first_instr = 0;
3462 	simode0 = 0;
3463 	simode1 = 0;
3464 	simode3 = 0;
3465 	lqimode0 = 0;
3466 	lqimode1 = 0;
3467 	lqomode0 = 0;
3468 	lqomode1 = 0;
3469 	saved_modes = ahd_save_modes(ahd);
3470 	for (;;) {
3471 		struct	cs *cs;
3472 		u_int	seqaddr;
3473 		u_int	i;
3474 
3475 		ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
3476 		seqaddr = ahd_inw(ahd, CURADDR);
3477 
3478 		cs = ahd->critical_sections;
3479 		for (i = 0; i < ahd->num_critical_sections; i++, cs++) {
3480 
3481 			if (cs->begin < seqaddr && cs->end >= seqaddr)
3482 				break;
3483 		}
3484 
3485 		if (i == ahd->num_critical_sections)
3486 			break;
3487 
3488 		if (steps > AHD_MAX_STEPS) {
3489 			printk("%s: Infinite loop in critical section\n"
3490 			       "%s: First Instruction 0x%x now 0x%x\n",
3491 			       ahd_name(ahd), ahd_name(ahd), first_instr,
3492 			       seqaddr);
3493 			ahd_dump_card_state(ahd);
3494 			panic("critical section loop");
3495 		}
3496 
3497 		steps++;
3498 #ifdef AHD_DEBUG
3499 		if ((ahd_debug & AHD_SHOW_MISC) != 0)
3500 			printk("%s: Single stepping at 0x%x\n", ahd_name(ahd),
3501 			       seqaddr);
3502 #endif
3503 		if (stepping == FALSE) {
3504 
3505 			first_instr = seqaddr;
3506   			ahd_set_modes(ahd, AHD_MODE_CFG, AHD_MODE_CFG);
3507   			simode0 = ahd_inb(ahd, SIMODE0);
3508 			simode3 = ahd_inb(ahd, SIMODE3);
3509 			lqimode0 = ahd_inb(ahd, LQIMODE0);
3510 			lqimode1 = ahd_inb(ahd, LQIMODE1);
3511 			lqomode0 = ahd_inb(ahd, LQOMODE0);
3512 			lqomode1 = ahd_inb(ahd, LQOMODE1);
3513 			ahd_outb(ahd, SIMODE0, 0);
3514 			ahd_outb(ahd, SIMODE3, 0);
3515 			ahd_outb(ahd, LQIMODE0, 0);
3516 			ahd_outb(ahd, LQIMODE1, 0);
3517 			ahd_outb(ahd, LQOMODE0, 0);
3518 			ahd_outb(ahd, LQOMODE1, 0);
3519 			ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
3520 			simode1 = ahd_inb(ahd, SIMODE1);
3521 			/*
3522 			 * We don't clear ENBUSFREE.  Unfortunately
3523 			 * we cannot re-enable busfree detection within
3524 			 * the current connection, so we must leave it
3525 			 * on while single stepping.
3526 			 */
3527 			ahd_outb(ahd, SIMODE1, simode1 & ENBUSFREE);
3528 			ahd_outb(ahd, SEQCTL0, ahd_inb(ahd, SEQCTL0) | STEP);
3529 			stepping = TRUE;
3530 		}
3531 		ahd_outb(ahd, CLRSINT1, CLRBUSFREE);
3532 		ahd_outb(ahd, CLRINT, CLRSCSIINT);
3533 		ahd_set_modes(ahd, ahd->saved_src_mode, ahd->saved_dst_mode);
3534 		ahd_outb(ahd, HCNTRL, ahd->unpause);
3535 		while (!ahd_is_paused(ahd))
3536 			ahd_delay(200);
3537 		ahd_update_modes(ahd);
3538 	}
3539 	if (stepping) {
3540 		ahd_set_modes(ahd, AHD_MODE_CFG, AHD_MODE_CFG);
3541 		ahd_outb(ahd, SIMODE0, simode0);
3542 		ahd_outb(ahd, SIMODE3, simode3);
3543 		ahd_outb(ahd, LQIMODE0, lqimode0);
3544 		ahd_outb(ahd, LQIMODE1, lqimode1);
3545 		ahd_outb(ahd, LQOMODE0, lqomode0);
3546 		ahd_outb(ahd, LQOMODE1, lqomode1);
3547 		ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
3548 		ahd_outb(ahd, SEQCTL0, ahd_inb(ahd, SEQCTL0) & ~STEP);
3549   		ahd_outb(ahd, SIMODE1, simode1);
3550 		/*
3551 		 * SCSIINT seems to glitch occasionally when
3552 		 * the interrupt masks are restored.  Clear SCSIINT
3553 		 * one more time so that only persistent errors
3554 		 * are seen as a real interrupt.
3555 		 */
3556 		ahd_outb(ahd, CLRINT, CLRSCSIINT);
3557 	}
3558 	ahd_restore_modes(ahd, saved_modes);
3559 }
3560 
3561 /*
3562  * Clear any pending interrupt status.
3563  */
3564 static void
3565 ahd_clear_intstat(struct ahd_softc *ahd)
3566 {
3567 	AHD_ASSERT_MODES(ahd, ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK),
3568 			 ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK));
3569 	/* Clear any interrupt conditions this may have caused */
3570 	ahd_outb(ahd, CLRLQIINT0, CLRLQIATNQAS|CLRLQICRCT1|CLRLQICRCT2
3571 				 |CLRLQIBADLQT|CLRLQIATNLQ|CLRLQIATNCMD);
3572 	ahd_outb(ahd, CLRLQIINT1, CLRLQIPHASE_LQ|CLRLQIPHASE_NLQ|CLRLIQABORT
3573 				 |CLRLQICRCI_LQ|CLRLQICRCI_NLQ|CLRLQIBADLQI
3574 				 |CLRLQIOVERI_LQ|CLRLQIOVERI_NLQ|CLRNONPACKREQ);
3575 	ahd_outb(ahd, CLRLQOINT0, CLRLQOTARGSCBPERR|CLRLQOSTOPT2|CLRLQOATNLQ
3576 				 |CLRLQOATNPKT|CLRLQOTCRC);
3577 	ahd_outb(ahd, CLRLQOINT1, CLRLQOINITSCBPERR|CLRLQOSTOPI2|CLRLQOBADQAS
3578 				 |CLRLQOBUSFREE|CLRLQOPHACHGINPKT);
3579 	if ((ahd->bugs & AHD_CLRLQO_AUTOCLR_BUG) != 0) {
3580 		ahd_outb(ahd, CLRLQOINT0, 0);
3581 		ahd_outb(ahd, CLRLQOINT1, 0);
3582 	}
3583 	ahd_outb(ahd, CLRSINT3, CLRNTRAMPERR|CLROSRAMPERR);
3584 	ahd_outb(ahd, CLRSINT1, CLRSELTIMEO|CLRATNO|CLRSCSIRSTI
3585 				|CLRBUSFREE|CLRSCSIPERR|CLRREQINIT);
3586 	ahd_outb(ahd, CLRSINT0, CLRSELDO|CLRSELDI|CLRSELINGO
3587 			        |CLRIOERR|CLROVERRUN);
3588 	ahd_outb(ahd, CLRINT, CLRSCSIINT);
3589 }
3590 
3591 /**************************** Debugging Routines ******************************/
3592 #ifdef AHD_DEBUG
3593 uint32_t ahd_debug = AHD_DEBUG_OPTS;
3594 #endif
3595 
3596 #if 0
3597 void
3598 ahd_print_scb(struct scb *scb)
3599 {
3600 	struct hardware_scb *hscb;
3601 	int i;
3602 
3603 	hscb = scb->hscb;
3604 	printk("scb:%p control:0x%x scsiid:0x%x lun:%d cdb_len:%d\n",
3605 	       (void *)scb,
3606 	       hscb->control,
3607 	       hscb->scsiid,
3608 	       hscb->lun,
3609 	       hscb->cdb_len);
3610 	printk("Shared Data: ");
3611 	for (i = 0; i < sizeof(hscb->shared_data.idata.cdb); i++)
3612 		printk("%#02x", hscb->shared_data.idata.cdb[i]);
3613 	printk("        dataptr:%#x%x datacnt:%#x sgptr:%#x tag:%#x\n",
3614 	       (uint32_t)((ahd_le64toh(hscb->dataptr) >> 32) & 0xFFFFFFFF),
3615 	       (uint32_t)(ahd_le64toh(hscb->dataptr) & 0xFFFFFFFF),
3616 	       ahd_le32toh(hscb->datacnt),
3617 	       ahd_le32toh(hscb->sgptr),
3618 	       SCB_GET_TAG(scb));
3619 	ahd_dump_sglist(scb);
3620 }
3621 #endif  /*  0  */
3622 
3623 /************************* Transfer Negotiation *******************************/
3624 /*
3625  * Allocate per target mode instance (ID we respond to as a target)
3626  * transfer negotiation data structures.
3627  */
3628 static struct ahd_tmode_tstate *
3629 ahd_alloc_tstate(struct ahd_softc *ahd, u_int scsi_id, char channel)
3630 {
3631 	struct ahd_tmode_tstate *master_tstate;
3632 	struct ahd_tmode_tstate *tstate;
3633 	int i;
3634 
3635 	master_tstate = ahd->enabled_targets[ahd->our_id];
3636 	if (ahd->enabled_targets[scsi_id] != NULL
3637 	 && ahd->enabled_targets[scsi_id] != master_tstate)
3638 		panic("%s: ahd_alloc_tstate - Target already allocated",
3639 		      ahd_name(ahd));
3640 	tstate = kmalloc(sizeof(*tstate), GFP_ATOMIC);
3641 	if (tstate == NULL)
3642 		return (NULL);
3643 
3644 	/*
3645 	 * If we have allocated a master tstate, copy user settings from
3646 	 * the master tstate (taken from SRAM or the EEPROM) for this
3647 	 * channel, but reset our current and goal settings to async/narrow
3648 	 * until an initiator talks to us.
3649 	 */
3650 	if (master_tstate != NULL) {
3651 		memcpy(tstate, master_tstate, sizeof(*tstate));
3652 		memset(tstate->enabled_luns, 0, sizeof(tstate->enabled_luns));
3653 		for (i = 0; i < 16; i++) {
3654 			memset(&tstate->transinfo[i].curr, 0,
3655 			      sizeof(tstate->transinfo[i].curr));
3656 			memset(&tstate->transinfo[i].goal, 0,
3657 			      sizeof(tstate->transinfo[i].goal));
3658 		}
3659 	} else
3660 		memset(tstate, 0, sizeof(*tstate));
3661 	ahd->enabled_targets[scsi_id] = tstate;
3662 	return (tstate);
3663 }
3664 
3665 #ifdef AHD_TARGET_MODE
3666 /*
3667  * Free per target mode instance (ID we respond to as a target)
3668  * transfer negotiation data structures.
3669  */
3670 static void
3671 ahd_free_tstate(struct ahd_softc *ahd, u_int scsi_id, char channel, int force)
3672 {
3673 	struct ahd_tmode_tstate *tstate;
3674 
3675 	/*
3676 	 * Don't clean up our "master" tstate.
3677 	 * It has our default user settings.
3678 	 */
3679 	if (scsi_id == ahd->our_id
3680 	 && force == FALSE)
3681 		return;
3682 
3683 	tstate = ahd->enabled_targets[scsi_id];
3684 	if (tstate != NULL)
3685 		kfree(tstate);
3686 	ahd->enabled_targets[scsi_id] = NULL;
3687 }
3688 #endif
3689 
3690 /*
3691  * Called when we have an active connection to a target on the bus,
3692  * this function finds the nearest period to the input period limited
3693  * by the capabilities of the bus connectivity of and sync settings for
3694  * the target.
3695  */
3696 static void
3697 ahd_devlimited_syncrate(struct ahd_softc *ahd,
3698 			struct ahd_initiator_tinfo *tinfo,
3699 			u_int *period, u_int *ppr_options, role_t role)
3700 {
3701 	struct	ahd_transinfo *transinfo;
3702 	u_int	maxsync;
3703 
3704 	if ((ahd_inb(ahd, SBLKCTL) & ENAB40) != 0
3705 	 && (ahd_inb(ahd, SSTAT2) & EXP_ACTIVE) == 0) {
3706 		maxsync = AHD_SYNCRATE_PACED;
3707 	} else {
3708 		maxsync = AHD_SYNCRATE_ULTRA;
3709 		/* Can't do DT related options on an SE bus */
3710 		*ppr_options &= MSG_EXT_PPR_QAS_REQ;
3711 	}
3712 	/*
3713 	 * Never allow a value higher than our current goal
3714 	 * period otherwise we may allow a target initiated
3715 	 * negotiation to go above the limit as set by the
3716 	 * user.  In the case of an initiator initiated
3717 	 * sync negotiation, we limit based on the user
3718 	 * setting.  This allows the system to still accept
3719 	 * incoming negotiations even if target initiated
3720 	 * negotiation is not performed.
3721 	 */
3722 	if (role == ROLE_TARGET)
3723 		transinfo = &tinfo->user;
3724 	else
3725 		transinfo = &tinfo->goal;
3726 	*ppr_options &= (transinfo->ppr_options|MSG_EXT_PPR_PCOMP_EN);
3727 	if (transinfo->width == MSG_EXT_WDTR_BUS_8_BIT) {
3728 		maxsync = max(maxsync, (u_int)AHD_SYNCRATE_ULTRA2);
3729 		*ppr_options &= ~MSG_EXT_PPR_DT_REQ;
3730 	}
3731 	if (transinfo->period == 0) {
3732 		*period = 0;
3733 		*ppr_options = 0;
3734 	} else {
3735 		*period = max(*period, (u_int)transinfo->period);
3736 		ahd_find_syncrate(ahd, period, ppr_options, maxsync);
3737 	}
3738 }
3739 
3740 /*
3741  * Look up the valid period to SCSIRATE conversion in our table.
3742  * Return the period and offset that should be sent to the target
3743  * if this was the beginning of an SDTR.
3744  */
3745 void
3746 ahd_find_syncrate(struct ahd_softc *ahd, u_int *period,
3747 		  u_int *ppr_options, u_int maxsync)
3748 {
3749 	if (*period < maxsync)
3750 		*period = maxsync;
3751 
3752 	if ((*ppr_options & MSG_EXT_PPR_DT_REQ) != 0
3753 	 && *period > AHD_SYNCRATE_MIN_DT)
3754 		*ppr_options &= ~MSG_EXT_PPR_DT_REQ;
3755 
3756 	if (*period > AHD_SYNCRATE_MIN)
3757 		*period = 0;
3758 
3759 	/* Honor PPR option conformance rules. */
3760 	if (*period > AHD_SYNCRATE_PACED)
3761 		*ppr_options &= ~MSG_EXT_PPR_RTI;
3762 
3763 	if ((*ppr_options & MSG_EXT_PPR_IU_REQ) == 0)
3764 		*ppr_options &= (MSG_EXT_PPR_DT_REQ|MSG_EXT_PPR_QAS_REQ);
3765 
3766 	if ((*ppr_options & MSG_EXT_PPR_DT_REQ) == 0)
3767 		*ppr_options &= MSG_EXT_PPR_QAS_REQ;
3768 
3769 	/* Skip all PACED only entries if IU is not available */
3770 	if ((*ppr_options & MSG_EXT_PPR_IU_REQ) == 0
3771 	 && *period < AHD_SYNCRATE_DT)
3772 		*period = AHD_SYNCRATE_DT;
3773 
3774 	/* Skip all DT only entries if DT is not available */
3775 	if ((*ppr_options & MSG_EXT_PPR_DT_REQ) == 0
3776 	 && *period < AHD_SYNCRATE_ULTRA2)
3777 		*period = AHD_SYNCRATE_ULTRA2;
3778 }
3779 
3780 /*
3781  * Truncate the given synchronous offset to a value the
3782  * current adapter type and syncrate are capable of.
3783  */
3784 static void
3785 ahd_validate_offset(struct ahd_softc *ahd,
3786 		    struct ahd_initiator_tinfo *tinfo,
3787 		    u_int period, u_int *offset, int wide,
3788 		    role_t role)
3789 {
3790 	u_int maxoffset;
3791 
3792 	/* Limit offset to what we can do */
3793 	if (period == 0)
3794 		maxoffset = 0;
3795 	else if (period <= AHD_SYNCRATE_PACED) {
3796 		if ((ahd->bugs & AHD_PACED_NEGTABLE_BUG) != 0)
3797 			maxoffset = MAX_OFFSET_PACED_BUG;
3798 		else
3799 			maxoffset = MAX_OFFSET_PACED;
3800 	} else
3801 		maxoffset = MAX_OFFSET_NON_PACED;
3802 	*offset = min(*offset, maxoffset);
3803 	if (tinfo != NULL) {
3804 		if (role == ROLE_TARGET)
3805 			*offset = min(*offset, (u_int)tinfo->user.offset);
3806 		else
3807 			*offset = min(*offset, (u_int)tinfo->goal.offset);
3808 	}
3809 }
3810 
3811 /*
3812  * Truncate the given transfer width parameter to a value the
3813  * current adapter type is capable of.
3814  */
3815 static void
3816 ahd_validate_width(struct ahd_softc *ahd, struct ahd_initiator_tinfo *tinfo,
3817 		   u_int *bus_width, role_t role)
3818 {
3819 	switch (*bus_width) {
3820 	default:
3821 		if (ahd->features & AHD_WIDE) {
3822 			/* Respond Wide */
3823 			*bus_width = MSG_EXT_WDTR_BUS_16_BIT;
3824 			break;
3825 		}
3826 		/* FALLTHROUGH */
3827 	case MSG_EXT_WDTR_BUS_8_BIT:
3828 		*bus_width = MSG_EXT_WDTR_BUS_8_BIT;
3829 		break;
3830 	}
3831 	if (tinfo != NULL) {
3832 		if (role == ROLE_TARGET)
3833 			*bus_width = min((u_int)tinfo->user.width, *bus_width);
3834 		else
3835 			*bus_width = min((u_int)tinfo->goal.width, *bus_width);
3836 	}
3837 }
3838 
3839 /*
3840  * Update the bitmask of targets for which the controller should
3841  * negotiate with at the next convenient opportunity.  This currently
3842  * means the next time we send the initial identify messages for
3843  * a new transaction.
3844  */
3845 int
3846 ahd_update_neg_request(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
3847 		       struct ahd_tmode_tstate *tstate,
3848 		       struct ahd_initiator_tinfo *tinfo, ahd_neg_type neg_type)
3849 {
3850 	u_int auto_negotiate_orig;
3851 
3852 	auto_negotiate_orig = tstate->auto_negotiate;
3853 	if (neg_type == AHD_NEG_ALWAYS) {
3854 		/*
3855 		 * Force our "current" settings to be
3856 		 * unknown so that unless a bus reset
3857 		 * occurs the need to renegotiate is
3858 		 * recorded persistently.
3859 		 */
3860 		if ((ahd->features & AHD_WIDE) != 0)
3861 			tinfo->curr.width = AHD_WIDTH_UNKNOWN;
3862 		tinfo->curr.period = AHD_PERIOD_UNKNOWN;
3863 		tinfo->curr.offset = AHD_OFFSET_UNKNOWN;
3864 	}
3865 	if (tinfo->curr.period != tinfo->goal.period
3866 	 || tinfo->curr.width != tinfo->goal.width
3867 	 || tinfo->curr.offset != tinfo->goal.offset
3868 	 || tinfo->curr.ppr_options != tinfo->goal.ppr_options
3869 	 || (neg_type == AHD_NEG_IF_NON_ASYNC
3870 	  && (tinfo->goal.offset != 0
3871 	   || tinfo->goal.width != MSG_EXT_WDTR_BUS_8_BIT
3872 	   || tinfo->goal.ppr_options != 0)))
3873 		tstate->auto_negotiate |= devinfo->target_mask;
3874 	else
3875 		tstate->auto_negotiate &= ~devinfo->target_mask;
3876 
3877 	return (auto_negotiate_orig != tstate->auto_negotiate);
3878 }
3879 
3880 /*
3881  * Update the user/goal/curr tables of synchronous negotiation
3882  * parameters as well as, in the case of a current or active update,
3883  * any data structures on the host controller.  In the case of an
3884  * active update, the specified target is currently talking to us on
3885  * the bus, so the transfer parameter update must take effect
3886  * immediately.
3887  */
3888 void
3889 ahd_set_syncrate(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
3890 		 u_int period, u_int offset, u_int ppr_options,
3891 		 u_int type, int paused)
3892 {
3893 	struct	ahd_initiator_tinfo *tinfo;
3894 	struct	ahd_tmode_tstate *tstate;
3895 	u_int	old_period;
3896 	u_int	old_offset;
3897 	u_int	old_ppr;
3898 	int	active;
3899 	int	update_needed;
3900 
3901 	active = (type & AHD_TRANS_ACTIVE) == AHD_TRANS_ACTIVE;
3902 	update_needed = 0;
3903 
3904 	if (period == 0 || offset == 0) {
3905 		period = 0;
3906 		offset = 0;
3907 	}
3908 
3909 	tinfo = ahd_fetch_transinfo(ahd, devinfo->channel, devinfo->our_scsiid,
3910 				    devinfo->target, &tstate);
3911 
3912 	if ((type & AHD_TRANS_USER) != 0) {
3913 		tinfo->user.period = period;
3914 		tinfo->user.offset = offset;
3915 		tinfo->user.ppr_options = ppr_options;
3916 	}
3917 
3918 	if ((type & AHD_TRANS_GOAL) != 0) {
3919 		tinfo->goal.period = period;
3920 		tinfo->goal.offset = offset;
3921 		tinfo->goal.ppr_options = ppr_options;
3922 	}
3923 
3924 	old_period = tinfo->curr.period;
3925 	old_offset = tinfo->curr.offset;
3926 	old_ppr	   = tinfo->curr.ppr_options;
3927 
3928 	if ((type & AHD_TRANS_CUR) != 0
3929 	 && (old_period != period
3930 	  || old_offset != offset
3931 	  || old_ppr != ppr_options)) {
3932 
3933 		update_needed++;
3934 
3935 		tinfo->curr.period = period;
3936 		tinfo->curr.offset = offset;
3937 		tinfo->curr.ppr_options = ppr_options;
3938 
3939 		ahd_send_async(ahd, devinfo->channel, devinfo->target,
3940 			       CAM_LUN_WILDCARD, AC_TRANSFER_NEG);
3941 		if (bootverbose) {
3942 			if (offset != 0) {
3943 				int options;
3944 
3945 				printk("%s: target %d synchronous with "
3946 				       "period = 0x%x, offset = 0x%x",
3947 				       ahd_name(ahd), devinfo->target,
3948 				       period, offset);
3949 				options = 0;
3950 				if ((ppr_options & MSG_EXT_PPR_RD_STRM) != 0) {
3951 					printk("(RDSTRM");
3952 					options++;
3953 				}
3954 				if ((ppr_options & MSG_EXT_PPR_DT_REQ) != 0) {
3955 					printk("%s", options ? "|DT" : "(DT");
3956 					options++;
3957 				}
3958 				if ((ppr_options & MSG_EXT_PPR_IU_REQ) != 0) {
3959 					printk("%s", options ? "|IU" : "(IU");
3960 					options++;
3961 				}
3962 				if ((ppr_options & MSG_EXT_PPR_RTI) != 0) {
3963 					printk("%s", options ? "|RTI" : "(RTI");
3964 					options++;
3965 				}
3966 				if ((ppr_options & MSG_EXT_PPR_QAS_REQ) != 0) {
3967 					printk("%s", options ? "|QAS" : "(QAS");
3968 					options++;
3969 				}
3970 				if (options != 0)
3971 					printk(")\n");
3972 				else
3973 					printk("\n");
3974 			} else {
3975 				printk("%s: target %d using "
3976 				       "asynchronous transfers%s\n",
3977 				       ahd_name(ahd), devinfo->target,
3978 				       (ppr_options & MSG_EXT_PPR_QAS_REQ) != 0
3979 				     ?  "(QAS)" : "");
3980 			}
3981 		}
3982 	}
3983 	/*
3984 	 * Always refresh the neg-table to handle the case of the
3985 	 * sequencer setting the ENATNO bit for a MK_MESSAGE request.
3986 	 * We will always renegotiate in that case if this is a
3987 	 * packetized request.  Also manage the busfree expected flag
3988 	 * from this common routine so that we catch changes due to
3989 	 * WDTR or SDTR messages.
3990 	 */
3991 	if ((type & AHD_TRANS_CUR) != 0) {
3992 		if (!paused)
3993 			ahd_pause(ahd);
3994 		ahd_update_neg_table(ahd, devinfo, &tinfo->curr);
3995 		if (!paused)
3996 			ahd_unpause(ahd);
3997 		if (ahd->msg_type != MSG_TYPE_NONE) {
3998 			if ((old_ppr & MSG_EXT_PPR_IU_REQ)
3999 			 != (ppr_options & MSG_EXT_PPR_IU_REQ)) {
4000 #ifdef AHD_DEBUG
4001 				if ((ahd_debug & AHD_SHOW_MESSAGES) != 0) {
4002 					ahd_print_devinfo(ahd, devinfo);
4003 					printk("Expecting IU Change busfree\n");
4004 				}
4005 #endif
4006 				ahd->msg_flags |= MSG_FLAG_EXPECT_PPR_BUSFREE
4007 					       |  MSG_FLAG_IU_REQ_CHANGED;
4008 			}
4009 			if ((old_ppr & MSG_EXT_PPR_IU_REQ) != 0) {
4010 #ifdef AHD_DEBUG
4011 				if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
4012 					printk("PPR with IU_REQ outstanding\n");
4013 #endif
4014 				ahd->msg_flags |= MSG_FLAG_EXPECT_PPR_BUSFREE;
4015 			}
4016 		}
4017 	}
4018 
4019 	update_needed += ahd_update_neg_request(ahd, devinfo, tstate,
4020 						tinfo, AHD_NEG_TO_GOAL);
4021 
4022 	if (update_needed && active)
4023 		ahd_update_pending_scbs(ahd);
4024 }
4025 
4026 /*
4027  * Update the user/goal/curr tables of wide negotiation
4028  * parameters as well as, in the case of a current or active update,
4029  * any data structures on the host controller.  In the case of an
4030  * active update, the specified target is currently talking to us on
4031  * the bus, so the transfer parameter update must take effect
4032  * immediately.
4033  */
4034 void
4035 ahd_set_width(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
4036 	      u_int width, u_int type, int paused)
4037 {
4038 	struct	ahd_initiator_tinfo *tinfo;
4039 	struct	ahd_tmode_tstate *tstate;
4040 	u_int	oldwidth;
4041 	int	active;
4042 	int	update_needed;
4043 
4044 	active = (type & AHD_TRANS_ACTIVE) == AHD_TRANS_ACTIVE;
4045 	update_needed = 0;
4046 	tinfo = ahd_fetch_transinfo(ahd, devinfo->channel, devinfo->our_scsiid,
4047 				    devinfo->target, &tstate);
4048 
4049 	if ((type & AHD_TRANS_USER) != 0)
4050 		tinfo->user.width = width;
4051 
4052 	if ((type & AHD_TRANS_GOAL) != 0)
4053 		tinfo->goal.width = width;
4054 
4055 	oldwidth = tinfo->curr.width;
4056 	if ((type & AHD_TRANS_CUR) != 0 && oldwidth != width) {
4057 
4058 		update_needed++;
4059 
4060 		tinfo->curr.width = width;
4061 		ahd_send_async(ahd, devinfo->channel, devinfo->target,
4062 			       CAM_LUN_WILDCARD, AC_TRANSFER_NEG);
4063 		if (bootverbose) {
4064 			printk("%s: target %d using %dbit transfers\n",
4065 			       ahd_name(ahd), devinfo->target,
4066 			       8 * (0x01 << width));
4067 		}
4068 	}
4069 
4070 	if ((type & AHD_TRANS_CUR) != 0) {
4071 		if (!paused)
4072 			ahd_pause(ahd);
4073 		ahd_update_neg_table(ahd, devinfo, &tinfo->curr);
4074 		if (!paused)
4075 			ahd_unpause(ahd);
4076 	}
4077 
4078 	update_needed += ahd_update_neg_request(ahd, devinfo, tstate,
4079 						tinfo, AHD_NEG_TO_GOAL);
4080 	if (update_needed && active)
4081 		ahd_update_pending_scbs(ahd);
4082 
4083 }
4084 
4085 /*
4086  * Update the current state of tagged queuing for a given target.
4087  */
4088 static void
4089 ahd_set_tags(struct ahd_softc *ahd, struct scsi_cmnd *cmd,
4090 	     struct ahd_devinfo *devinfo, ahd_queue_alg alg)
4091 {
4092 	struct scsi_device *sdev = cmd->device;
4093 
4094 	ahd_platform_set_tags(ahd, sdev, devinfo, alg);
4095 	ahd_send_async(ahd, devinfo->channel, devinfo->target,
4096 		       devinfo->lun, AC_TRANSFER_NEG);
4097 }
4098 
4099 static void
4100 ahd_update_neg_table(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
4101 		     struct ahd_transinfo *tinfo)
4102 {
4103 	ahd_mode_state	saved_modes;
4104 	u_int		period;
4105 	u_int		ppr_opts;
4106 	u_int		con_opts;
4107 	u_int		offset;
4108 	u_int		saved_negoaddr;
4109 	uint8_t		iocell_opts[sizeof(ahd->iocell_opts)];
4110 
4111 	saved_modes = ahd_save_modes(ahd);
4112 	ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
4113 
4114 	saved_negoaddr = ahd_inb(ahd, NEGOADDR);
4115 	ahd_outb(ahd, NEGOADDR, devinfo->target);
4116 	period = tinfo->period;
4117 	offset = tinfo->offset;
4118 	memcpy(iocell_opts, ahd->iocell_opts, sizeof(ahd->iocell_opts));
4119 	ppr_opts = tinfo->ppr_options & (MSG_EXT_PPR_QAS_REQ|MSG_EXT_PPR_DT_REQ
4120 					|MSG_EXT_PPR_IU_REQ|MSG_EXT_PPR_RTI);
4121 	con_opts = 0;
4122 	if (period == 0)
4123 		period = AHD_SYNCRATE_ASYNC;
4124 	if (period == AHD_SYNCRATE_160) {
4125 
4126 		if ((ahd->bugs & AHD_PACED_NEGTABLE_BUG) != 0) {
4127 			/*
4128 			 * When the SPI4 spec was finalized, PACE transfers
4129 			 * was not made a configurable option in the PPR
4130 			 * message.  Instead it is assumed to be enabled for
4131 			 * any syncrate faster than 80MHz.  Nevertheless,
4132 			 * Harpoon2A4 allows this to be configurable.
4133 			 *
4134 			 * Harpoon2A4 also assumes at most 2 data bytes per
4135 			 * negotiated REQ/ACK offset.  Paced transfers take
4136 			 * 4, so we must adjust our offset.
4137 			 */
4138 			ppr_opts |= PPROPT_PACE;
4139 			offset *= 2;
4140 
4141 			/*
4142 			 * Harpoon2A assumed that there would be a
4143 			 * fallback rate between 160MHz and 80MHz,
4144 			 * so 7 is used as the period factor rather
4145 			 * than 8 for 160MHz.
4146 			 */
4147 			period = AHD_SYNCRATE_REVA_160;
4148 		}
4149 		if ((tinfo->ppr_options & MSG_EXT_PPR_PCOMP_EN) == 0)
4150 			iocell_opts[AHD_PRECOMP_SLEW_INDEX] &=
4151 			    ~AHD_PRECOMP_MASK;
4152 	} else {
4153 		/*
4154 		 * Precomp should be disabled for non-paced transfers.
4155 		 */
4156 		iocell_opts[AHD_PRECOMP_SLEW_INDEX] &= ~AHD_PRECOMP_MASK;
4157 
4158 		if ((ahd->features & AHD_NEW_IOCELL_OPTS) != 0
4159 		 && (ppr_opts & MSG_EXT_PPR_DT_REQ) != 0
4160 		 && (ppr_opts & MSG_EXT_PPR_IU_REQ) == 0) {
4161 			/*
4162 			 * Slow down our CRC interval to be
4163 			 * compatible with non-packetized
4164 			 * U160 devices that can't handle a
4165 			 * CRC at full speed.
4166 			 */
4167 			con_opts |= ENSLOWCRC;
4168 		}
4169 
4170 		if ((ahd->bugs & AHD_PACED_NEGTABLE_BUG) != 0) {
4171 			/*
4172 			 * On H2A4, revert to a slower slewrate
4173 			 * on non-paced transfers.
4174 			 */
4175 			iocell_opts[AHD_PRECOMP_SLEW_INDEX] &=
4176 			    ~AHD_SLEWRATE_MASK;
4177 		}
4178 	}
4179 
4180 	ahd_outb(ahd, ANNEXCOL, AHD_ANNEXCOL_PRECOMP_SLEW);
4181 	ahd_outb(ahd, ANNEXDAT, iocell_opts[AHD_PRECOMP_SLEW_INDEX]);
4182 	ahd_outb(ahd, ANNEXCOL, AHD_ANNEXCOL_AMPLITUDE);
4183 	ahd_outb(ahd, ANNEXDAT, iocell_opts[AHD_AMPLITUDE_INDEX]);
4184 
4185 	ahd_outb(ahd, NEGPERIOD, period);
4186 	ahd_outb(ahd, NEGPPROPTS, ppr_opts);
4187 	ahd_outb(ahd, NEGOFFSET, offset);
4188 
4189 	if (tinfo->width == MSG_EXT_WDTR_BUS_16_BIT)
4190 		con_opts |= WIDEXFER;
4191 
4192 	/*
4193 	 * Slow down our CRC interval to be
4194 	 * compatible with packetized U320 devices
4195 	 * that can't handle a CRC at full speed
4196 	 */
4197 	if (ahd->features & AHD_AIC79XXB_SLOWCRC) {
4198 		con_opts |= ENSLOWCRC;
4199 	}
4200 
4201 	/*
4202 	 * During packetized transfers, the target will
4203 	 * give us the opportunity to send command packets
4204 	 * without us asserting attention.
4205 	 */
4206 	if ((tinfo->ppr_options & MSG_EXT_PPR_IU_REQ) == 0)
4207 		con_opts |= ENAUTOATNO;
4208 	ahd_outb(ahd, NEGCONOPTS, con_opts);
4209 	ahd_outb(ahd, NEGOADDR, saved_negoaddr);
4210 	ahd_restore_modes(ahd, saved_modes);
4211 }
4212 
4213 /*
4214  * When the transfer settings for a connection change, setup for
4215  * negotiation in pending SCBs to effect the change as quickly as
4216  * possible.  We also cancel any negotiations that are scheduled
4217  * for inflight SCBs that have not been started yet.
4218  */
4219 static void
4220 ahd_update_pending_scbs(struct ahd_softc *ahd)
4221 {
4222 	struct		scb *pending_scb;
4223 	int		pending_scb_count;
4224 	int		paused;
4225 	u_int		saved_scbptr;
4226 	ahd_mode_state	saved_modes;
4227 
4228 	/*
4229 	 * Traverse the pending SCB list and ensure that all of the
4230 	 * SCBs there have the proper settings.  We can only safely
4231 	 * clear the negotiation required flag (setting requires the
4232 	 * execution queue to be modified) and this is only possible
4233 	 * if we are not already attempting to select out for this
4234 	 * SCB.  For this reason, all callers only call this routine
4235 	 * if we are changing the negotiation settings for the currently
4236 	 * active transaction on the bus.
4237 	 */
4238 	pending_scb_count = 0;
4239 	LIST_FOREACH(pending_scb, &ahd->pending_scbs, pending_links) {
4240 		struct ahd_devinfo devinfo;
4241 		struct ahd_initiator_tinfo *tinfo;
4242 		struct ahd_tmode_tstate *tstate;
4243 
4244 		ahd_scb_devinfo(ahd, &devinfo, pending_scb);
4245 		tinfo = ahd_fetch_transinfo(ahd, devinfo.channel,
4246 					    devinfo.our_scsiid,
4247 					    devinfo.target, &tstate);
4248 		if ((tstate->auto_negotiate & devinfo.target_mask) == 0
4249 		 && (pending_scb->flags & SCB_AUTO_NEGOTIATE) != 0) {
4250 			pending_scb->flags &= ~SCB_AUTO_NEGOTIATE;
4251 			pending_scb->hscb->control &= ~MK_MESSAGE;
4252 		}
4253 		ahd_sync_scb(ahd, pending_scb,
4254 			     BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);
4255 		pending_scb_count++;
4256 	}
4257 
4258 	if (pending_scb_count == 0)
4259 		return;
4260 
4261 	if (ahd_is_paused(ahd)) {
4262 		paused = 1;
4263 	} else {
4264 		paused = 0;
4265 		ahd_pause(ahd);
4266 	}
4267 
4268 	/*
4269 	 * Force the sequencer to reinitialize the selection for
4270 	 * the command at the head of the execution queue if it
4271 	 * has already been setup.  The negotiation changes may
4272 	 * effect whether we select-out with ATN.  It is only
4273 	 * safe to clear ENSELO when the bus is not free and no
4274 	 * selection is in progres or completed.
4275 	 */
4276 	saved_modes = ahd_save_modes(ahd);
4277 	ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
4278 	if ((ahd_inb(ahd, SCSISIGI) & BSYI) != 0
4279 	 && (ahd_inb(ahd, SSTAT0) & (SELDO|SELINGO)) == 0)
4280 		ahd_outb(ahd, SCSISEQ0, ahd_inb(ahd, SCSISEQ0) & ~ENSELO);
4281 	saved_scbptr = ahd_get_scbptr(ahd);
4282 	/* Ensure that the hscbs down on the card match the new information */
4283 	LIST_FOREACH(pending_scb, &ahd->pending_scbs, pending_links) {
4284 		u_int	scb_tag;
4285 		u_int	control;
4286 
4287 		scb_tag = SCB_GET_TAG(pending_scb);
4288 		ahd_set_scbptr(ahd, scb_tag);
4289 		control = ahd_inb_scbram(ahd, SCB_CONTROL);
4290 		control &= ~MK_MESSAGE;
4291 		control |= pending_scb->hscb->control & MK_MESSAGE;
4292 		ahd_outb(ahd, SCB_CONTROL, control);
4293 	}
4294 	ahd_set_scbptr(ahd, saved_scbptr);
4295 	ahd_restore_modes(ahd, saved_modes);
4296 
4297 	if (paused == 0)
4298 		ahd_unpause(ahd);
4299 }
4300 
4301 /**************************** Pathing Information *****************************/
4302 static void
4303 ahd_fetch_devinfo(struct ahd_softc *ahd, struct ahd_devinfo *devinfo)
4304 {
4305 	ahd_mode_state	saved_modes;
4306 	u_int		saved_scsiid;
4307 	role_t		role;
4308 	int		our_id;
4309 
4310 	saved_modes = ahd_save_modes(ahd);
4311 	ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
4312 
4313 	if (ahd_inb(ahd, SSTAT0) & TARGET)
4314 		role = ROLE_TARGET;
4315 	else
4316 		role = ROLE_INITIATOR;
4317 
4318 	if (role == ROLE_TARGET
4319 	 && (ahd_inb(ahd, SEQ_FLAGS) & CMDPHASE_PENDING) != 0) {
4320 		/* We were selected, so pull our id from TARGIDIN */
4321 		our_id = ahd_inb(ahd, TARGIDIN) & OID;
4322 	} else if (role == ROLE_TARGET)
4323 		our_id = ahd_inb(ahd, TOWNID);
4324 	else
4325 		our_id = ahd_inb(ahd, IOWNID);
4326 
4327 	saved_scsiid = ahd_inb(ahd, SAVED_SCSIID);
4328 	ahd_compile_devinfo(devinfo,
4329 			    our_id,
4330 			    SCSIID_TARGET(ahd, saved_scsiid),
4331 			    ahd_inb(ahd, SAVED_LUN),
4332 			    SCSIID_CHANNEL(ahd, saved_scsiid),
4333 			    role);
4334 	ahd_restore_modes(ahd, saved_modes);
4335 }
4336 
4337 void
4338 ahd_print_devinfo(struct ahd_softc *ahd, struct ahd_devinfo *devinfo)
4339 {
4340 	printk("%s:%c:%d:%d: ", ahd_name(ahd), 'A',
4341 	       devinfo->target, devinfo->lun);
4342 }
4343 
4344 static const struct ahd_phase_table_entry*
4345 ahd_lookup_phase_entry(int phase)
4346 {
4347 	const struct ahd_phase_table_entry *entry;
4348 	const struct ahd_phase_table_entry *last_entry;
4349 
4350 	/*
4351 	 * num_phases doesn't include the default entry which
4352 	 * will be returned if the phase doesn't match.
4353 	 */
4354 	last_entry = &ahd_phase_table[num_phases];
4355 	for (entry = ahd_phase_table; entry < last_entry; entry++) {
4356 		if (phase == entry->phase)
4357 			break;
4358 	}
4359 	return (entry);
4360 }
4361 
4362 void
4363 ahd_compile_devinfo(struct ahd_devinfo *devinfo, u_int our_id, u_int target,
4364 		    u_int lun, char channel, role_t role)
4365 {
4366 	devinfo->our_scsiid = our_id;
4367 	devinfo->target = target;
4368 	devinfo->lun = lun;
4369 	devinfo->target_offset = target;
4370 	devinfo->channel = channel;
4371 	devinfo->role = role;
4372 	if (channel == 'B')
4373 		devinfo->target_offset += 8;
4374 	devinfo->target_mask = (0x01 << devinfo->target_offset);
4375 }
4376 
4377 static void
4378 ahd_scb_devinfo(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
4379 		struct scb *scb)
4380 {
4381 	role_t	role;
4382 	int	our_id;
4383 
4384 	our_id = SCSIID_OUR_ID(scb->hscb->scsiid);
4385 	role = ROLE_INITIATOR;
4386 	if ((scb->hscb->control & TARGET_SCB) != 0)
4387 		role = ROLE_TARGET;
4388 	ahd_compile_devinfo(devinfo, our_id, SCB_GET_TARGET(ahd, scb),
4389 			    SCB_GET_LUN(scb), SCB_GET_CHANNEL(ahd, scb), role);
4390 }
4391 
4392 
4393 /************************ Message Phase Processing ****************************/
4394 /*
4395  * When an initiator transaction with the MK_MESSAGE flag either reconnects
4396  * or enters the initial message out phase, we are interrupted.  Fill our
4397  * outgoing message buffer with the appropriate message and beging handing
4398  * the message phase(s) manually.
4399  */
4400 static void
4401 ahd_setup_initiator_msgout(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
4402 			   struct scb *scb)
4403 {
4404 	/*
4405 	 * To facilitate adding multiple messages together,
4406 	 * each routine should increment the index and len
4407 	 * variables instead of setting them explicitly.
4408 	 */
4409 	ahd->msgout_index = 0;
4410 	ahd->msgout_len = 0;
4411 
4412 	if (ahd_currently_packetized(ahd))
4413 		ahd->msg_flags |= MSG_FLAG_PACKETIZED;
4414 
4415 	if (ahd->send_msg_perror
4416 	 && ahd_inb(ahd, MSG_OUT) == HOST_MSG) {
4417 		ahd->msgout_buf[ahd->msgout_index++] = ahd->send_msg_perror;
4418 		ahd->msgout_len++;
4419 		ahd->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
4420 #ifdef AHD_DEBUG
4421 		if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
4422 			printk("Setting up for Parity Error delivery\n");
4423 #endif
4424 		return;
4425 	} else if (scb == NULL) {
4426 		printk("%s: WARNING. No pending message for "
4427 		       "I_T msgin.  Issuing NO-OP\n", ahd_name(ahd));
4428 		ahd->msgout_buf[ahd->msgout_index++] = MSG_NOOP;
4429 		ahd->msgout_len++;
4430 		ahd->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
4431 		return;
4432 	}
4433 
4434 	if ((scb->flags & SCB_DEVICE_RESET) == 0
4435 	 && (scb->flags & SCB_PACKETIZED) == 0
4436 	 && ahd_inb(ahd, MSG_OUT) == MSG_IDENTIFYFLAG) {
4437 		u_int identify_msg;
4438 
4439 		identify_msg = MSG_IDENTIFYFLAG | SCB_GET_LUN(scb);
4440 		if ((scb->hscb->control & DISCENB) != 0)
4441 			identify_msg |= MSG_IDENTIFY_DISCFLAG;
4442 		ahd->msgout_buf[ahd->msgout_index++] = identify_msg;
4443 		ahd->msgout_len++;
4444 
4445 		if ((scb->hscb->control & TAG_ENB) != 0) {
4446 			ahd->msgout_buf[ahd->msgout_index++] =
4447 			    scb->hscb->control & (TAG_ENB|SCB_TAG_TYPE);
4448 			ahd->msgout_buf[ahd->msgout_index++] = SCB_GET_TAG(scb);
4449 			ahd->msgout_len += 2;
4450 		}
4451 	}
4452 
4453 	if (scb->flags & SCB_DEVICE_RESET) {
4454 		ahd->msgout_buf[ahd->msgout_index++] = MSG_BUS_DEV_RESET;
4455 		ahd->msgout_len++;
4456 		ahd_print_path(ahd, scb);
4457 		printk("Bus Device Reset Message Sent\n");
4458 		/*
4459 		 * Clear our selection hardware in advance of
4460 		 * the busfree.  We may have an entry in the waiting
4461 		 * Q for this target, and we don't want to go about
4462 		 * selecting while we handle the busfree and blow it
4463 		 * away.
4464 		 */
4465 		ahd_outb(ahd, SCSISEQ0, 0);
4466 	} else if ((scb->flags & SCB_ABORT) != 0) {
4467 
4468 		if ((scb->hscb->control & TAG_ENB) != 0) {
4469 			ahd->msgout_buf[ahd->msgout_index++] = MSG_ABORT_TAG;
4470 		} else {
4471 			ahd->msgout_buf[ahd->msgout_index++] = MSG_ABORT;
4472 		}
4473 		ahd->msgout_len++;
4474 		ahd_print_path(ahd, scb);
4475 		printk("Abort%s Message Sent\n",
4476 		       (scb->hscb->control & TAG_ENB) != 0 ? " Tag" : "");
4477 		/*
4478 		 * Clear our selection hardware in advance of
4479 		 * the busfree.  We may have an entry in the waiting
4480 		 * Q for this target, and we don't want to go about
4481 		 * selecting while we handle the busfree and blow it
4482 		 * away.
4483 		 */
4484 		ahd_outb(ahd, SCSISEQ0, 0);
4485 	} else if ((scb->flags & (SCB_AUTO_NEGOTIATE|SCB_NEGOTIATE)) != 0) {
4486 		ahd_build_transfer_msg(ahd, devinfo);
4487 		/*
4488 		 * Clear our selection hardware in advance of potential
4489 		 * PPR IU status change busfree.  We may have an entry in
4490 		 * the waiting Q for this target, and we don't want to go
4491 		 * about selecting while we handle the busfree and blow
4492 		 * it away.
4493 		 */
4494 		ahd_outb(ahd, SCSISEQ0, 0);
4495 	} else {
4496 		printk("ahd_intr: AWAITING_MSG for an SCB that "
4497 		       "does not have a waiting message\n");
4498 		printk("SCSIID = %x, target_mask = %x\n", scb->hscb->scsiid,
4499 		       devinfo->target_mask);
4500 		panic("SCB = %d, SCB Control = %x:%x, MSG_OUT = %x "
4501 		      "SCB flags = %x", SCB_GET_TAG(scb), scb->hscb->control,
4502 		      ahd_inb_scbram(ahd, SCB_CONTROL), ahd_inb(ahd, MSG_OUT),
4503 		      scb->flags);
4504 	}
4505 
4506 	/*
4507 	 * Clear the MK_MESSAGE flag from the SCB so we aren't
4508 	 * asked to send this message again.
4509 	 */
4510 	ahd_outb(ahd, SCB_CONTROL,
4511 		 ahd_inb_scbram(ahd, SCB_CONTROL) & ~MK_MESSAGE);
4512 	scb->hscb->control &= ~MK_MESSAGE;
4513 	ahd->msgout_index = 0;
4514 	ahd->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
4515 }
4516 
4517 /*
4518  * Build an appropriate transfer negotiation message for the
4519  * currently active target.
4520  */
4521 static void
4522 ahd_build_transfer_msg(struct ahd_softc *ahd, struct ahd_devinfo *devinfo)
4523 {
4524 	/*
4525 	 * We need to initiate transfer negotiations.
4526 	 * If our current and goal settings are identical,
4527 	 * we want to renegotiate due to a check condition.
4528 	 */
4529 	struct	ahd_initiator_tinfo *tinfo;
4530 	struct	ahd_tmode_tstate *tstate;
4531 	int	dowide;
4532 	int	dosync;
4533 	int	doppr;
4534 	u_int	period;
4535 	u_int	ppr_options;
4536 	u_int	offset;
4537 
4538 	tinfo = ahd_fetch_transinfo(ahd, devinfo->channel, devinfo->our_scsiid,
4539 				    devinfo->target, &tstate);
4540 	/*
4541 	 * Filter our period based on the current connection.
4542 	 * If we can't perform DT transfers on this segment (not in LVD
4543 	 * mode for instance), then our decision to issue a PPR message
4544 	 * may change.
4545 	 */
4546 	period = tinfo->goal.period;
4547 	offset = tinfo->goal.offset;
4548 	ppr_options = tinfo->goal.ppr_options;
4549 	/* Target initiated PPR is not allowed in the SCSI spec */
4550 	if (devinfo->role == ROLE_TARGET)
4551 		ppr_options = 0;
4552 	ahd_devlimited_syncrate(ahd, tinfo, &period,
4553 				&ppr_options, devinfo->role);
4554 	dowide = tinfo->curr.width != tinfo->goal.width;
4555 	dosync = tinfo->curr.offset != offset || tinfo->curr.period != period;
4556 	/*
4557 	 * Only use PPR if we have options that need it, even if the device
4558 	 * claims to support it.  There might be an expander in the way
4559 	 * that doesn't.
4560 	 */
4561 	doppr = ppr_options != 0;
4562 
4563 	if (!dowide && !dosync && !doppr) {
4564 		dowide = tinfo->goal.width != MSG_EXT_WDTR_BUS_8_BIT;
4565 		dosync = tinfo->goal.offset != 0;
4566 	}
4567 
4568 	if (!dowide && !dosync && !doppr) {
4569 		/*
4570 		 * Force async with a WDTR message if we have a wide bus,
4571 		 * or just issue an SDTR with a 0 offset.
4572 		 */
4573 		if ((ahd->features & AHD_WIDE) != 0)
4574 			dowide = 1;
4575 		else
4576 			dosync = 1;
4577 
4578 		if (bootverbose) {
4579 			ahd_print_devinfo(ahd, devinfo);
4580 			printk("Ensuring async\n");
4581 		}
4582 	}
4583 	/* Target initiated PPR is not allowed in the SCSI spec */
4584 	if (devinfo->role == ROLE_TARGET)
4585 		doppr = 0;
4586 
4587 	/*
4588 	 * Both the PPR message and SDTR message require the
4589 	 * goal syncrate to be limited to what the target device
4590 	 * is capable of handling (based on whether an LVD->SE
4591 	 * expander is on the bus), so combine these two cases.
4592 	 * Regardless, guarantee that if we are using WDTR and SDTR
4593 	 * messages that WDTR comes first.
4594 	 */
4595 	if (doppr || (dosync && !dowide)) {
4596 
4597 		offset = tinfo->goal.offset;
4598 		ahd_validate_offset(ahd, tinfo, period, &offset,
4599 				    doppr ? tinfo->goal.width
4600 					  : tinfo->curr.width,
4601 				    devinfo->role);
4602 		if (doppr) {
4603 			ahd_construct_ppr(ahd, devinfo, period, offset,
4604 					  tinfo->goal.width, ppr_options);
4605 		} else {
4606 			ahd_construct_sdtr(ahd, devinfo, period, offset);
4607 		}
4608 	} else {
4609 		ahd_construct_wdtr(ahd, devinfo, tinfo->goal.width);
4610 	}
4611 }
4612 
4613 /*
4614  * Build a synchronous negotiation message in our message
4615  * buffer based on the input parameters.
4616  */
4617 static void
4618 ahd_construct_sdtr(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
4619 		   u_int period, u_int offset)
4620 {
4621 	if (offset == 0)
4622 		period = AHD_ASYNC_XFER_PERIOD;
4623 	ahd->msgout_index += spi_populate_sync_msg(
4624 			ahd->msgout_buf + ahd->msgout_index, period, offset);
4625 	ahd->msgout_len += 5;
4626 	if (bootverbose) {
4627 		printk("(%s:%c:%d:%d): Sending SDTR period %x, offset %x\n",
4628 		       ahd_name(ahd), devinfo->channel, devinfo->target,
4629 		       devinfo->lun, period, offset);
4630 	}
4631 }
4632 
4633 /*
4634  * Build a wide negotiateion message in our message
4635  * buffer based on the input parameters.
4636  */
4637 static void
4638 ahd_construct_wdtr(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
4639 		   u_int bus_width)
4640 {
4641 	ahd->msgout_index += spi_populate_width_msg(
4642 			ahd->msgout_buf + ahd->msgout_index, bus_width);
4643 	ahd->msgout_len += 4;
4644 	if (bootverbose) {
4645 		printk("(%s:%c:%d:%d): Sending WDTR %x\n",
4646 		       ahd_name(ahd), devinfo->channel, devinfo->target,
4647 		       devinfo->lun, bus_width);
4648 	}
4649 }
4650 
4651 /*
4652  * Build a parallel protocol request message in our message
4653  * buffer based on the input parameters.
4654  */
4655 static void
4656 ahd_construct_ppr(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
4657 		  u_int period, u_int offset, u_int bus_width,
4658 		  u_int ppr_options)
4659 {
4660 	/*
4661 	 * Always request precompensation from
4662 	 * the other target if we are running
4663 	 * at paced syncrates.
4664 	 */
4665 	if (period <= AHD_SYNCRATE_PACED)
4666 		ppr_options |= MSG_EXT_PPR_PCOMP_EN;
4667 	if (offset == 0)
4668 		period = AHD_ASYNC_XFER_PERIOD;
4669 	ahd->msgout_index += spi_populate_ppr_msg(
4670 			ahd->msgout_buf + ahd->msgout_index, period, offset,
4671 			bus_width, ppr_options);
4672 	ahd->msgout_len += 8;
4673 	if (bootverbose) {
4674 		printk("(%s:%c:%d:%d): Sending PPR bus_width %x, period %x, "
4675 		       "offset %x, ppr_options %x\n", ahd_name(ahd),
4676 		       devinfo->channel, devinfo->target, devinfo->lun,
4677 		       bus_width, period, offset, ppr_options);
4678 	}
4679 }
4680 
4681 /*
4682  * Clear any active message state.
4683  */
4684 static void
4685 ahd_clear_msg_state(struct ahd_softc *ahd)
4686 {
4687 	ahd_mode_state saved_modes;
4688 
4689 	saved_modes = ahd_save_modes(ahd);
4690 	ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
4691 	ahd->send_msg_perror = 0;
4692 	ahd->msg_flags = MSG_FLAG_NONE;
4693 	ahd->msgout_len = 0;
4694 	ahd->msgin_index = 0;
4695 	ahd->msg_type = MSG_TYPE_NONE;
4696 	if ((ahd_inb(ahd, SCSISIGO) & ATNO) != 0) {
4697 		/*
4698 		 * The target didn't care to respond to our
4699 		 * message request, so clear ATN.
4700 		 */
4701 		ahd_outb(ahd, CLRSINT1, CLRATNO);
4702 	}
4703 	ahd_outb(ahd, MSG_OUT, MSG_NOOP);
4704 	ahd_outb(ahd, SEQ_FLAGS2,
4705 		 ahd_inb(ahd, SEQ_FLAGS2) & ~TARGET_MSG_PENDING);
4706 	ahd_restore_modes(ahd, saved_modes);
4707 }
4708 
4709 /*
4710  * Manual message loop handler.
4711  */
4712 static void
4713 ahd_handle_message_phase(struct ahd_softc *ahd)
4714 {
4715 	struct	ahd_devinfo devinfo;
4716 	u_int	bus_phase;
4717 	int	end_session;
4718 
4719 	ahd_fetch_devinfo(ahd, &devinfo);
4720 	end_session = FALSE;
4721 	bus_phase = ahd_inb(ahd, LASTPHASE);
4722 
4723 	if ((ahd_inb(ahd, LQISTAT2) & LQIPHASE_OUTPKT) != 0) {
4724 		printk("LQIRETRY for LQIPHASE_OUTPKT\n");
4725 		ahd_outb(ahd, LQCTL2, LQIRETRY);
4726 	}
4727 reswitch:
4728 	switch (ahd->msg_type) {
4729 	case MSG_TYPE_INITIATOR_MSGOUT:
4730 	{
4731 		int lastbyte;
4732 		int phasemis;
4733 		int msgdone;
4734 
4735 		if (ahd->msgout_len == 0 && ahd->send_msg_perror == 0)
4736 			panic("HOST_MSG_LOOP interrupt with no active message");
4737 
4738 #ifdef AHD_DEBUG
4739 		if ((ahd_debug & AHD_SHOW_MESSAGES) != 0) {
4740 			ahd_print_devinfo(ahd, &devinfo);
4741 			printk("INITIATOR_MSG_OUT");
4742 		}
4743 #endif
4744 		phasemis = bus_phase != P_MESGOUT;
4745 		if (phasemis) {
4746 #ifdef AHD_DEBUG
4747 			if ((ahd_debug & AHD_SHOW_MESSAGES) != 0) {
4748 				printk(" PHASEMIS %s\n",
4749 				       ahd_lookup_phase_entry(bus_phase)
4750 							     ->phasemsg);
4751 			}
4752 #endif
4753 			if (bus_phase == P_MESGIN) {
4754 				/*
4755 				 * Change gears and see if
4756 				 * this messages is of interest to
4757 				 * us or should be passed back to
4758 				 * the sequencer.
4759 				 */
4760 				ahd_outb(ahd, CLRSINT1, CLRATNO);
4761 				ahd->send_msg_perror = 0;
4762 				ahd->msg_type = MSG_TYPE_INITIATOR_MSGIN;
4763 				ahd->msgin_index = 0;
4764 				goto reswitch;
4765 			}
4766 			end_session = TRUE;
4767 			break;
4768 		}
4769 
4770 		if (ahd->send_msg_perror) {
4771 			ahd_outb(ahd, CLRSINT1, CLRATNO);
4772 			ahd_outb(ahd, CLRSINT1, CLRREQINIT);
4773 #ifdef AHD_DEBUG
4774 			if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
4775 				printk(" byte 0x%x\n", ahd->send_msg_perror);
4776 #endif
4777 			/*
4778 			 * If we are notifying the target of a CRC error
4779 			 * during packetized operations, the target is
4780 			 * within its rights to acknowledge our message
4781 			 * with a busfree.
4782 			 */
4783 			if ((ahd->msg_flags & MSG_FLAG_PACKETIZED) != 0
4784 			 && ahd->send_msg_perror == MSG_INITIATOR_DET_ERR)
4785 				ahd->msg_flags |= MSG_FLAG_EXPECT_IDE_BUSFREE;
4786 
4787 			ahd_outb(ahd, RETURN_2, ahd->send_msg_perror);
4788 			ahd_outb(ahd, RETURN_1, CONT_MSG_LOOP_WRITE);
4789 			break;
4790 		}
4791 
4792 		msgdone	= ahd->msgout_index == ahd->msgout_len;
4793 		if (msgdone) {
4794 			/*
4795 			 * The target has requested a retry.
4796 			 * Re-assert ATN, reset our message index to
4797 			 * 0, and try again.
4798 			 */
4799 			ahd->msgout_index = 0;
4800 			ahd_assert_atn(ahd);
4801 		}
4802 
4803 		lastbyte = ahd->msgout_index == (ahd->msgout_len - 1);
4804 		if (lastbyte) {
4805 			/* Last byte is signified by dropping ATN */
4806 			ahd_outb(ahd, CLRSINT1, CLRATNO);
4807 		}
4808 
4809 		/*
4810 		 * Clear our interrupt status and present
4811 		 * the next byte on the bus.
4812 		 */
4813 		ahd_outb(ahd, CLRSINT1, CLRREQINIT);
4814 #ifdef AHD_DEBUG
4815 		if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
4816 			printk(" byte 0x%x\n",
4817 			       ahd->msgout_buf[ahd->msgout_index]);
4818 #endif
4819 		ahd_outb(ahd, RETURN_2, ahd->msgout_buf[ahd->msgout_index++]);
4820 		ahd_outb(ahd, RETURN_1, CONT_MSG_LOOP_WRITE);
4821 		break;
4822 	}
4823 	case MSG_TYPE_INITIATOR_MSGIN:
4824 	{
4825 		int phasemis;
4826 		int message_done;
4827 
4828 #ifdef AHD_DEBUG
4829 		if ((ahd_debug & AHD_SHOW_MESSAGES) != 0) {
4830 			ahd_print_devinfo(ahd, &devinfo);
4831 			printk("INITIATOR_MSG_IN");
4832 		}
4833 #endif
4834 		phasemis = bus_phase != P_MESGIN;
4835 		if (phasemis) {
4836 #ifdef AHD_DEBUG
4837 			if ((ahd_debug & AHD_SHOW_MESSAGES) != 0) {
4838 				printk(" PHASEMIS %s\n",
4839 				       ahd_lookup_phase_entry(bus_phase)
4840 							     ->phasemsg);
4841 			}
4842 #endif
4843 			ahd->msgin_index = 0;
4844 			if (bus_phase == P_MESGOUT
4845 			 && (ahd->send_msg_perror != 0
4846 			  || (ahd->msgout_len != 0
4847 			   && ahd->msgout_index == 0))) {
4848 				ahd->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
4849 				goto reswitch;
4850 			}
4851 			end_session = TRUE;
4852 			break;
4853 		}
4854 
4855 		/* Pull the byte in without acking it */
4856 		ahd->msgin_buf[ahd->msgin_index] = ahd_inb(ahd, SCSIBUS);
4857 #ifdef AHD_DEBUG
4858 		if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
4859 			printk(" byte 0x%x\n",
4860 			       ahd->msgin_buf[ahd->msgin_index]);
4861 #endif
4862 
4863 		message_done = ahd_parse_msg(ahd, &devinfo);
4864 
4865 		if (message_done) {
4866 			/*
4867 			 * Clear our incoming message buffer in case there
4868 			 * is another message following this one.
4869 			 */
4870 			ahd->msgin_index = 0;
4871 
4872 			/*
4873 			 * If this message illicited a response,
4874 			 * assert ATN so the target takes us to the
4875 			 * message out phase.
4876 			 */
4877 			if (ahd->msgout_len != 0) {
4878 #ifdef AHD_DEBUG
4879 				if ((ahd_debug & AHD_SHOW_MESSAGES) != 0) {
4880 					ahd_print_devinfo(ahd, &devinfo);
4881 					printk("Asserting ATN for response\n");
4882 				}
4883 #endif
4884 				ahd_assert_atn(ahd);
4885 			}
4886 		} else
4887 			ahd->msgin_index++;
4888 
4889 		if (message_done == MSGLOOP_TERMINATED) {
4890 			end_session = TRUE;
4891 		} else {
4892 			/* Ack the byte */
4893 			ahd_outb(ahd, CLRSINT1, CLRREQINIT);
4894 			ahd_outb(ahd, RETURN_1, CONT_MSG_LOOP_READ);
4895 		}
4896 		break;
4897 	}
4898 	case MSG_TYPE_TARGET_MSGIN:
4899 	{
4900 		int msgdone;
4901 		int msgout_request;
4902 
4903 		/*
4904 		 * By default, the message loop will continue.
4905 		 */
4906 		ahd_outb(ahd, RETURN_1, CONT_MSG_LOOP_TARG);
4907 
4908 		if (ahd->msgout_len == 0)
4909 			panic("Target MSGIN with no active message");
4910 
4911 		/*
4912 		 * If we interrupted a mesgout session, the initiator
4913 		 * will not know this until our first REQ.  So, we
4914 		 * only honor mesgout requests after we've sent our
4915 		 * first byte.
4916 		 */
4917 		if ((ahd_inb(ahd, SCSISIGI) & ATNI) != 0
4918 		 && ahd->msgout_index > 0)
4919 			msgout_request = TRUE;
4920 		else
4921 			msgout_request = FALSE;
4922 
4923 		if (msgout_request) {
4924 
4925 			/*
4926 			 * Change gears and see if
4927 			 * this messages is of interest to
4928 			 * us or should be passed back to
4929 			 * the sequencer.
4930 			 */
4931 			ahd->msg_type = MSG_TYPE_TARGET_MSGOUT;
4932 			ahd_outb(ahd, SCSISIGO, P_MESGOUT | BSYO);
4933 			ahd->msgin_index = 0;
4934 			/* Dummy read to REQ for first byte */
4935 			ahd_inb(ahd, SCSIDAT);
4936 			ahd_outb(ahd, SXFRCTL0,
4937 				 ahd_inb(ahd, SXFRCTL0) | SPIOEN);
4938 			break;
4939 		}
4940 
4941 		msgdone = ahd->msgout_index == ahd->msgout_len;
4942 		if (msgdone) {
4943 			ahd_outb(ahd, SXFRCTL0,
4944 				 ahd_inb(ahd, SXFRCTL0) & ~SPIOEN);
4945 			end_session = TRUE;
4946 			break;
4947 		}
4948 
4949 		/*
4950 		 * Present the next byte on the bus.
4951 		 */
4952 		ahd_outb(ahd, SXFRCTL0, ahd_inb(ahd, SXFRCTL0) | SPIOEN);
4953 		ahd_outb(ahd, SCSIDAT, ahd->msgout_buf[ahd->msgout_index++]);
4954 		break;
4955 	}
4956 	case MSG_TYPE_TARGET_MSGOUT:
4957 	{
4958 		int lastbyte;
4959 		int msgdone;
4960 
4961 		/*
4962 		 * By default, the message loop will continue.
4963 		 */
4964 		ahd_outb(ahd, RETURN_1, CONT_MSG_LOOP_TARG);
4965 
4966 		/*
4967 		 * The initiator signals that this is
4968 		 * the last byte by dropping ATN.
4969 		 */
4970 		lastbyte = (ahd_inb(ahd, SCSISIGI) & ATNI) == 0;
4971 
4972 		/*
4973 		 * Read the latched byte, but turn off SPIOEN first
4974 		 * so that we don't inadvertently cause a REQ for the
4975 		 * next byte.
4976 		 */
4977 		ahd_outb(ahd, SXFRCTL0, ahd_inb(ahd, SXFRCTL0) & ~SPIOEN);
4978 		ahd->msgin_buf[ahd->msgin_index] = ahd_inb(ahd, SCSIDAT);
4979 		msgdone = ahd_parse_msg(ahd, &devinfo);
4980 		if (msgdone == MSGLOOP_TERMINATED) {
4981 			/*
4982 			 * The message is *really* done in that it caused
4983 			 * us to go to bus free.  The sequencer has already
4984 			 * been reset at this point, so pull the ejection
4985 			 * handle.
4986 			 */
4987 			return;
4988 		}
4989 
4990 		ahd->msgin_index++;
4991 
4992 		/*
4993 		 * XXX Read spec about initiator dropping ATN too soon
4994 		 *     and use msgdone to detect it.
4995 		 */
4996 		if (msgdone == MSGLOOP_MSGCOMPLETE) {
4997 			ahd->msgin_index = 0;
4998 
4999 			/*
5000 			 * If this message illicited a response, transition
5001 			 * to the Message in phase and send it.
5002 			 */
5003 			if (ahd->msgout_len != 0) {
5004 				ahd_outb(ahd, SCSISIGO, P_MESGIN | BSYO);
5005 				ahd_outb(ahd, SXFRCTL0,
5006 					 ahd_inb(ahd, SXFRCTL0) | SPIOEN);
5007 				ahd->msg_type = MSG_TYPE_TARGET_MSGIN;
5008 				ahd->msgin_index = 0;
5009 				break;
5010 			}
5011 		}
5012 
5013 		if (lastbyte)
5014 			end_session = TRUE;
5015 		else {
5016 			/* Ask for the next byte. */
5017 			ahd_outb(ahd, SXFRCTL0,
5018 				 ahd_inb(ahd, SXFRCTL0) | SPIOEN);
5019 		}
5020 
5021 		break;
5022 	}
5023 	default:
5024 		panic("Unknown REQINIT message type");
5025 	}
5026 
5027 	if (end_session) {
5028 		if ((ahd->msg_flags & MSG_FLAG_PACKETIZED) != 0) {
5029 			printk("%s: Returning to Idle Loop\n",
5030 			       ahd_name(ahd));
5031 			ahd_clear_msg_state(ahd);
5032 
5033 			/*
5034 			 * Perform the equivalent of a clear_target_state.
5035 			 */
5036 			ahd_outb(ahd, LASTPHASE, P_BUSFREE);
5037 			ahd_outb(ahd, SEQ_FLAGS, NOT_IDENTIFIED|NO_CDB_SENT);
5038 			ahd_outb(ahd, SEQCTL0, FASTMODE|SEQRESET);
5039 		} else {
5040 			ahd_clear_msg_state(ahd);
5041 			ahd_outb(ahd, RETURN_1, EXIT_MSG_LOOP);
5042 		}
5043 	}
5044 }
5045 
5046 /*
5047  * See if we sent a particular extended message to the target.
5048  * If "full" is true, return true only if the target saw the full
5049  * message.  If "full" is false, return true if the target saw at
5050  * least the first byte of the message.
5051  */
5052 static int
5053 ahd_sent_msg(struct ahd_softc *ahd, ahd_msgtype type, u_int msgval, int full)
5054 {
5055 	int found;
5056 	u_int index;
5057 
5058 	found = FALSE;
5059 	index = 0;
5060 
5061 	while (index < ahd->msgout_len) {
5062 		if (ahd->msgout_buf[index] == MSG_EXTENDED) {
5063 			u_int end_index;
5064 
5065 			end_index = index + 1 + ahd->msgout_buf[index + 1];
5066 			if (ahd->msgout_buf[index+2] == msgval
5067 			 && type == AHDMSG_EXT) {
5068 
5069 				if (full) {
5070 					if (ahd->msgout_index > end_index)
5071 						found = TRUE;
5072 				} else if (ahd->msgout_index > index)
5073 					found = TRUE;
5074 			}
5075 			index = end_index;
5076 		} else if (ahd->msgout_buf[index] >= MSG_SIMPLE_TASK
5077 			&& ahd->msgout_buf[index] <= MSG_IGN_WIDE_RESIDUE) {
5078 
5079 			/* Skip tag type and tag id or residue param*/
5080 			index += 2;
5081 		} else {
5082 			/* Single byte message */
5083 			if (type == AHDMSG_1B
5084 			 && ahd->msgout_index > index
5085 			 && (ahd->msgout_buf[index] == msgval
5086 			  || ((ahd->msgout_buf[index] & MSG_IDENTIFYFLAG) != 0
5087 			   && msgval == MSG_IDENTIFYFLAG)))
5088 				found = TRUE;
5089 			index++;
5090 		}
5091 
5092 		if (found)
5093 			break;
5094 	}
5095 	return (found);
5096 }
5097 
5098 /*
5099  * Wait for a complete incoming message, parse it, and respond accordingly.
5100  */
5101 static int
5102 ahd_parse_msg(struct ahd_softc *ahd, struct ahd_devinfo *devinfo)
5103 {
5104 	struct	ahd_initiator_tinfo *tinfo;
5105 	struct	ahd_tmode_tstate *tstate;
5106 	int	reject;
5107 	int	done;
5108 	int	response;
5109 
5110 	done = MSGLOOP_IN_PROG;
5111 	response = FALSE;
5112 	reject = FALSE;
5113 	tinfo = ahd_fetch_transinfo(ahd, devinfo->channel, devinfo->our_scsiid,
5114 				    devinfo->target, &tstate);
5115 
5116 	/*
5117 	 * Parse as much of the message as is available,
5118 	 * rejecting it if we don't support it.  When
5119 	 * the entire message is available and has been
5120 	 * handled, return MSGLOOP_MSGCOMPLETE, indicating
5121 	 * that we have parsed an entire message.
5122 	 *
5123 	 * In the case of extended messages, we accept the length
5124 	 * byte outright and perform more checking once we know the
5125 	 * extended message type.
5126 	 */
5127 	switch (ahd->msgin_buf[0]) {
5128 	case MSG_DISCONNECT:
5129 	case MSG_SAVEDATAPOINTER:
5130 	case MSG_CMDCOMPLETE:
5131 	case MSG_RESTOREPOINTERS:
5132 	case MSG_IGN_WIDE_RESIDUE:
5133 		/*
5134 		 * End our message loop as these are messages
5135 		 * the sequencer handles on its own.
5136 		 */
5137 		done = MSGLOOP_TERMINATED;
5138 		break;
5139 	case MSG_MESSAGE_REJECT:
5140 		response = ahd_handle_msg_reject(ahd, devinfo);
5141 		/* FALLTHROUGH */
5142 	case MSG_NOOP:
5143 		done = MSGLOOP_MSGCOMPLETE;
5144 		break;
5145 	case MSG_EXTENDED:
5146 	{
5147 		/* Wait for enough of the message to begin validation */
5148 		if (ahd->msgin_index < 2)
5149 			break;
5150 		switch (ahd->msgin_buf[2]) {
5151 		case MSG_EXT_SDTR:
5152 		{
5153 			u_int	 period;
5154 			u_int	 ppr_options;
5155 			u_int	 offset;
5156 			u_int	 saved_offset;
5157 
5158 			if (ahd->msgin_buf[1] != MSG_EXT_SDTR_LEN) {
5159 				reject = TRUE;
5160 				break;
5161 			}
5162 
5163 			/*
5164 			 * Wait until we have both args before validating
5165 			 * and acting on this message.
5166 			 *
5167 			 * Add one to MSG_EXT_SDTR_LEN to account for
5168 			 * the extended message preamble.
5169 			 */
5170 			if (ahd->msgin_index < (MSG_EXT_SDTR_LEN + 1))
5171 				break;
5172 
5173 			period = ahd->msgin_buf[3];
5174 			ppr_options = 0;
5175 			saved_offset = offset = ahd->msgin_buf[4];
5176 			ahd_devlimited_syncrate(ahd, tinfo, &period,
5177 						&ppr_options, devinfo->role);
5178 			ahd_validate_offset(ahd, tinfo, period, &offset,
5179 					    tinfo->curr.width, devinfo->role);
5180 			if (bootverbose) {
5181 				printk("(%s:%c:%d:%d): Received "
5182 				       "SDTR period %x, offset %x\n\t"
5183 				       "Filtered to period %x, offset %x\n",
5184 				       ahd_name(ahd), devinfo->channel,
5185 				       devinfo->target, devinfo->lun,
5186 				       ahd->msgin_buf[3], saved_offset,
5187 				       period, offset);
5188 			}
5189 			ahd_set_syncrate(ahd, devinfo, period,
5190 					 offset, ppr_options,
5191 					 AHD_TRANS_ACTIVE|AHD_TRANS_GOAL,
5192 					 /*paused*/TRUE);
5193 
5194 			/*
5195 			 * See if we initiated Sync Negotiation
5196 			 * and didn't have to fall down to async
5197 			 * transfers.
5198 			 */
5199 			if (ahd_sent_msg(ahd, AHDMSG_EXT, MSG_EXT_SDTR, TRUE)) {
5200 				/* We started it */
5201 				if (saved_offset != offset) {
5202 					/* Went too low - force async */
5203 					reject = TRUE;
5204 				}
5205 			} else {
5206 				/*
5207 				 * Send our own SDTR in reply
5208 				 */
5209 				if (bootverbose
5210 				 && devinfo->role == ROLE_INITIATOR) {
5211 					printk("(%s:%c:%d:%d): Target "
5212 					       "Initiated SDTR\n",
5213 					       ahd_name(ahd), devinfo->channel,
5214 					       devinfo->target, devinfo->lun);
5215 				}
5216 				ahd->msgout_index = 0;
5217 				ahd->msgout_len = 0;
5218 				ahd_construct_sdtr(ahd, devinfo,
5219 						   period, offset);
5220 				ahd->msgout_index = 0;
5221 				response = TRUE;
5222 			}
5223 			done = MSGLOOP_MSGCOMPLETE;
5224 			break;
5225 		}
5226 		case MSG_EXT_WDTR:
5227 		{
5228 			u_int bus_width;
5229 			u_int saved_width;
5230 			u_int sending_reply;
5231 
5232 			sending_reply = FALSE;
5233 			if (ahd->msgin_buf[1] != MSG_EXT_WDTR_LEN) {
5234 				reject = TRUE;
5235 				break;
5236 			}
5237 
5238 			/*
5239 			 * Wait until we have our arg before validating
5240 			 * and acting on this message.
5241 			 *
5242 			 * Add one to MSG_EXT_WDTR_LEN to account for
5243 			 * the extended message preamble.
5244 			 */
5245 			if (ahd->msgin_index < (MSG_EXT_WDTR_LEN + 1))
5246 				break;
5247 
5248 			bus_width = ahd->msgin_buf[3];
5249 			saved_width = bus_width;
5250 			ahd_validate_width(ahd, tinfo, &bus_width,
5251 					   devinfo->role);
5252 			if (bootverbose) {
5253 				printk("(%s:%c:%d:%d): Received WDTR "
5254 				       "%x filtered to %x\n",
5255 				       ahd_name(ahd), devinfo->channel,
5256 				       devinfo->target, devinfo->lun,
5257 				       saved_width, bus_width);
5258 			}
5259 
5260 			if (ahd_sent_msg(ahd, AHDMSG_EXT, MSG_EXT_WDTR, TRUE)) {
5261 				/*
5262 				 * Don't send a WDTR back to the
5263 				 * target, since we asked first.
5264 				 * If the width went higher than our
5265 				 * request, reject it.
5266 				 */
5267 				if (saved_width > bus_width) {
5268 					reject = TRUE;
5269 					printk("(%s:%c:%d:%d): requested %dBit "
5270 					       "transfers.  Rejecting...\n",
5271 					       ahd_name(ahd), devinfo->channel,
5272 					       devinfo->target, devinfo->lun,
5273 					       8 * (0x01 << bus_width));
5274 					bus_width = 0;
5275 				}
5276 			} else {
5277 				/*
5278 				 * Send our own WDTR in reply
5279 				 */
5280 				if (bootverbose
5281 				 && devinfo->role == ROLE_INITIATOR) {
5282 					printk("(%s:%c:%d:%d): Target "
5283 					       "Initiated WDTR\n",
5284 					       ahd_name(ahd), devinfo->channel,
5285 					       devinfo->target, devinfo->lun);
5286 				}
5287 				ahd->msgout_index = 0;
5288 				ahd->msgout_len = 0;
5289 				ahd_construct_wdtr(ahd, devinfo, bus_width);
5290 				ahd->msgout_index = 0;
5291 				response = TRUE;
5292 				sending_reply = TRUE;
5293 			}
5294 			/*
5295 			 * After a wide message, we are async, but
5296 			 * some devices don't seem to honor this portion
5297 			 * of the spec.  Force a renegotiation of the
5298 			 * sync component of our transfer agreement even
5299 			 * if our goal is async.  By updating our width
5300 			 * after forcing the negotiation, we avoid
5301 			 * renegotiating for width.
5302 			 */
5303 			ahd_update_neg_request(ahd, devinfo, tstate,
5304 					       tinfo, AHD_NEG_ALWAYS);
5305 			ahd_set_width(ahd, devinfo, bus_width,
5306 				      AHD_TRANS_ACTIVE|AHD_TRANS_GOAL,
5307 				      /*paused*/TRUE);
5308 			if (sending_reply == FALSE && reject == FALSE) {
5309 
5310 				/*
5311 				 * We will always have an SDTR to send.
5312 				 */
5313 				ahd->msgout_index = 0;
5314 				ahd->msgout_len = 0;
5315 				ahd_build_transfer_msg(ahd, devinfo);
5316 				ahd->msgout_index = 0;
5317 				response = TRUE;
5318 			}
5319 			done = MSGLOOP_MSGCOMPLETE;
5320 			break;
5321 		}
5322 		case MSG_EXT_PPR:
5323 		{
5324 			u_int	period;
5325 			u_int	offset;
5326 			u_int	bus_width;
5327 			u_int	ppr_options;
5328 			u_int	saved_width;
5329 			u_int	saved_offset;
5330 			u_int	saved_ppr_options;
5331 
5332 			if (ahd->msgin_buf[1] != MSG_EXT_PPR_LEN) {
5333 				reject = TRUE;
5334 				break;
5335 			}
5336 
5337 			/*
5338 			 * Wait until we have all args before validating
5339 			 * and acting on this message.
5340 			 *
5341 			 * Add one to MSG_EXT_PPR_LEN to account for
5342 			 * the extended message preamble.
5343 			 */
5344 			if (ahd->msgin_index < (MSG_EXT_PPR_LEN + 1))
5345 				break;
5346 
5347 			period = ahd->msgin_buf[3];
5348 			offset = ahd->msgin_buf[5];
5349 			bus_width = ahd->msgin_buf[6];
5350 			saved_width = bus_width;
5351 			ppr_options = ahd->msgin_buf[7];
5352 			/*
5353 			 * According to the spec, a DT only
5354 			 * period factor with no DT option
5355 			 * set implies async.
5356 			 */
5357 			if ((ppr_options & MSG_EXT_PPR_DT_REQ) == 0
5358 			 && period <= 9)
5359 				offset = 0;
5360 			saved_ppr_options = ppr_options;
5361 			saved_offset = offset;
5362 
5363 			/*
5364 			 * Transfer options are only available if we
5365 			 * are negotiating wide.
5366 			 */
5367 			if (bus_width == 0)
5368 				ppr_options &= MSG_EXT_PPR_QAS_REQ;
5369 
5370 			ahd_validate_width(ahd, tinfo, &bus_width,
5371 					   devinfo->role);
5372 			ahd_devlimited_syncrate(ahd, tinfo, &period,
5373 						&ppr_options, devinfo->role);
5374 			ahd_validate_offset(ahd, tinfo, period, &offset,
5375 					    bus_width, devinfo->role);
5376 
5377 			if (ahd_sent_msg(ahd, AHDMSG_EXT, MSG_EXT_PPR, TRUE)) {
5378 				/*
5379 				 * If we are unable to do any of the
5380 				 * requested options (we went too low),
5381 				 * then we'll have to reject the message.
5382 				 */
5383 				if (saved_width > bus_width
5384 				 || saved_offset != offset
5385 				 || saved_ppr_options != ppr_options) {
5386 					reject = TRUE;
5387 					period = 0;
5388 					offset = 0;
5389 					bus_width = 0;
5390 					ppr_options = 0;
5391 				}
5392 			} else {
5393 				if (devinfo->role != ROLE_TARGET)
5394 					printk("(%s:%c:%d:%d): Target "
5395 					       "Initiated PPR\n",
5396 					       ahd_name(ahd), devinfo->channel,
5397 					       devinfo->target, devinfo->lun);
5398 				else
5399 					printk("(%s:%c:%d:%d): Initiator "
5400 					       "Initiated PPR\n",
5401 					       ahd_name(ahd), devinfo->channel,
5402 					       devinfo->target, devinfo->lun);
5403 				ahd->msgout_index = 0;
5404 				ahd->msgout_len = 0;
5405 				ahd_construct_ppr(ahd, devinfo, period, offset,
5406 						  bus_width, ppr_options);
5407 				ahd->msgout_index = 0;
5408 				response = TRUE;
5409 			}
5410 			if (bootverbose) {
5411 				printk("(%s:%c:%d:%d): Received PPR width %x, "
5412 				       "period %x, offset %x,options %x\n"
5413 				       "\tFiltered to width %x, period %x, "
5414 				       "offset %x, options %x\n",
5415 				       ahd_name(ahd), devinfo->channel,
5416 				       devinfo->target, devinfo->lun,
5417 				       saved_width, ahd->msgin_buf[3],
5418 				       saved_offset, saved_ppr_options,
5419 				       bus_width, period, offset, ppr_options);
5420 			}
5421 			ahd_set_width(ahd, devinfo, bus_width,
5422 				      AHD_TRANS_ACTIVE|AHD_TRANS_GOAL,
5423 				      /*paused*/TRUE);
5424 			ahd_set_syncrate(ahd, devinfo, period,
5425 					 offset, ppr_options,
5426 					 AHD_TRANS_ACTIVE|AHD_TRANS_GOAL,
5427 					 /*paused*/TRUE);
5428 
5429 			done = MSGLOOP_MSGCOMPLETE;
5430 			break;
5431 		}
5432 		default:
5433 			/* Unknown extended message.  Reject it. */
5434 			reject = TRUE;
5435 			break;
5436 		}
5437 		break;
5438 	}
5439 #ifdef AHD_TARGET_MODE
5440 	case MSG_BUS_DEV_RESET:
5441 		ahd_handle_devreset(ahd, devinfo, CAM_LUN_WILDCARD,
5442 				    CAM_BDR_SENT,
5443 				    "Bus Device Reset Received",
5444 				    /*verbose_level*/0);
5445 		ahd_restart(ahd);
5446 		done = MSGLOOP_TERMINATED;
5447 		break;
5448 	case MSG_ABORT_TAG:
5449 	case MSG_ABORT:
5450 	case MSG_CLEAR_QUEUE:
5451 	{
5452 		int tag;
5453 
5454 		/* Target mode messages */
5455 		if (devinfo->role != ROLE_TARGET) {
5456 			reject = TRUE;
5457 			break;
5458 		}
5459 		tag = SCB_LIST_NULL;
5460 		if (ahd->msgin_buf[0] == MSG_ABORT_TAG)
5461 			tag = ahd_inb(ahd, INITIATOR_TAG);
5462 		ahd_abort_scbs(ahd, devinfo->target, devinfo->channel,
5463 			       devinfo->lun, tag, ROLE_TARGET,
5464 			       CAM_REQ_ABORTED);
5465 
5466 		tstate = ahd->enabled_targets[devinfo->our_scsiid];
5467 		if (tstate != NULL) {
5468 			struct ahd_tmode_lstate* lstate;
5469 
5470 			lstate = tstate->enabled_luns[devinfo->lun];
5471 			if (lstate != NULL) {
5472 				ahd_queue_lstate_event(ahd, lstate,
5473 						       devinfo->our_scsiid,
5474 						       ahd->msgin_buf[0],
5475 						       /*arg*/tag);
5476 				ahd_send_lstate_events(ahd, lstate);
5477 			}
5478 		}
5479 		ahd_restart(ahd);
5480 		done = MSGLOOP_TERMINATED;
5481 		break;
5482 	}
5483 #endif
5484 	case MSG_QAS_REQUEST:
5485 #ifdef AHD_DEBUG
5486 		if ((ahd_debug & AHD_SHOW_MESSAGES) != 0)
5487 			printk("%s: QAS request.  SCSISIGI == 0x%x\n",
5488 			       ahd_name(ahd), ahd_inb(ahd, SCSISIGI));
5489 #endif
5490 		ahd->msg_flags |= MSG_FLAG_EXPECT_QASREJ_BUSFREE;
5491 		/* FALLTHROUGH */
5492 	case MSG_TERM_IO_PROC:
5493 	default:
5494 		reject = TRUE;
5495 		break;
5496 	}
5497 
5498 	if (reject) {
5499 		/*
5500 		 * Setup to reject the message.
5501 		 */
5502 		ahd->msgout_index = 0;
5503 		ahd->msgout_len = 1;
5504 		ahd->msgout_buf[0] = MSG_MESSAGE_REJECT;
5505 		done = MSGLOOP_MSGCOMPLETE;
5506 		response = TRUE;
5507 	}
5508 
5509 	if (done != MSGLOOP_IN_PROG && !response)
5510 		/* Clear the outgoing message buffer */
5511 		ahd->msgout_len = 0;
5512 
5513 	return (done);
5514 }
5515 
5516 /*
5517  * Process a message reject message.
5518  */
5519 static int
5520 ahd_handle_msg_reject(struct ahd_softc *ahd, struct ahd_devinfo *devinfo)
5521 {
5522 	/*
5523 	 * What we care about here is if we had an
5524 	 * outstanding SDTR or WDTR message for this
5525 	 * target.  If we did, this is a signal that
5526 	 * the target is refusing negotiation.
5527 	 */
5528 	struct scb *scb;
5529 	struct ahd_initiator_tinfo *tinfo;
5530 	struct ahd_tmode_tstate *tstate;
5531 	u_int scb_index;
5532 	u_int last_msg;
5533 	int   response = 0;
5534 
5535 	scb_index = ahd_get_scbptr(ahd);
5536 	scb = ahd_lookup_scb(ahd, scb_index);
5537 	tinfo = ahd_fetch_transinfo(ahd, devinfo->channel,
5538 				    devinfo->our_scsiid,
5539 				    devinfo->target, &tstate);
5540 	/* Might be necessary */
5541 	last_msg = ahd_inb(ahd, LAST_MSG);
5542 
5543 	if (ahd_sent_msg(ahd, AHDMSG_EXT, MSG_EXT_PPR, /*full*/FALSE)) {
5544 		if (ahd_sent_msg(ahd, AHDMSG_EXT, MSG_EXT_PPR, /*full*/TRUE)
5545 		 && tinfo->goal.period <= AHD_SYNCRATE_PACED) {
5546 			/*
5547 			 * Target may not like our SPI-4 PPR Options.
5548 			 * Attempt to negotiate 80MHz which will turn
5549 			 * off these options.
5550 			 */
5551 			if (bootverbose) {
5552 				printk("(%s:%c:%d:%d): PPR Rejected. "
5553 				       "Trying simple U160 PPR\n",
5554 				       ahd_name(ahd), devinfo->channel,
5555 				       devinfo->target, devinfo->lun);
5556 			}
5557 			tinfo->goal.period = AHD_SYNCRATE_DT;
5558 			tinfo->goal.ppr_options &= MSG_EXT_PPR_IU_REQ
5559 						|  MSG_EXT_PPR_QAS_REQ
5560 						|  MSG_EXT_PPR_DT_REQ;
5561 		} else {
5562 			/*
5563 			 * Target does not support the PPR message.
5564 			 * Attempt to negotiate SPI-2 style.
5565 			 */
5566 			if (bootverbose) {
5567 				printk("(%s:%c:%d:%d): PPR Rejected. "
5568 				       "Trying WDTR/SDTR\n",
5569 				       ahd_name(ahd), devinfo->channel,
5570 				       devinfo->target, devinfo->lun);
5571 			}
5572 			tinfo->goal.ppr_options = 0;
5573 			tinfo->curr.transport_version = 2;
5574 			tinfo->goal.transport_version = 2;
5575 		}
5576 		ahd->msgout_index = 0;
5577 		ahd->msgout_len = 0;
5578 		ahd_build_transfer_msg(ahd, devinfo);
5579 		ahd->msgout_index = 0;
5580 		response = 1;
5581 	} else if (ahd_sent_msg(ahd, AHDMSG_EXT, MSG_EXT_WDTR, /*full*/FALSE)) {
5582 
5583 		/* note 8bit xfers */
5584 		printk("(%s:%c:%d:%d): refuses WIDE negotiation.  Using "
5585 		       "8bit transfers\n", ahd_name(ahd),
5586 		       devinfo->channel, devinfo->target, devinfo->lun);
5587 		ahd_set_width(ahd, devinfo, MSG_EXT_WDTR_BUS_8_BIT,
5588 			      AHD_TRANS_ACTIVE|AHD_TRANS_GOAL,
5589 			      /*paused*/TRUE);
5590 		/*
5591 		 * No need to clear the sync rate.  If the target
5592 		 * did not accept the command, our syncrate is
5593 		 * unaffected.  If the target started the negotiation,
5594 		 * but rejected our response, we already cleared the
5595 		 * sync rate before sending our WDTR.
5596 		 */
5597 		if (tinfo->goal.offset != tinfo->curr.offset) {
5598 
5599 			/* Start the sync negotiation */
5600 			ahd->msgout_index = 0;
5601 			ahd->msgout_len = 0;
5602 			ahd_build_transfer_msg(ahd, devinfo);
5603 			ahd->msgout_index = 0;
5604 			response = 1;
5605 		}
5606 	} else if (ahd_sent_msg(ahd, AHDMSG_EXT, MSG_EXT_SDTR, /*full*/FALSE)) {
5607 		/* note asynch xfers and clear flag */
5608 		ahd_set_syncrate(ahd, devinfo, /*period*/0,
5609 				 /*offset*/0, /*ppr_options*/0,
5610 				 AHD_TRANS_ACTIVE|AHD_TRANS_GOAL,
5611 				 /*paused*/TRUE);
5612 		printk("(%s:%c:%d:%d): refuses synchronous negotiation. "
5613 		       "Using asynchronous transfers\n",
5614 		       ahd_name(ahd), devinfo->channel,
5615 		       devinfo->target, devinfo->lun);
5616 	} else if ((scb->hscb->control & MSG_SIMPLE_TASK) != 0) {
5617 		int tag_type;
5618 		int mask;
5619 
5620 		tag_type = (scb->hscb->control & MSG_SIMPLE_TASK);
5621 
5622 		if (tag_type == MSG_SIMPLE_TASK) {
5623 			printk("(%s:%c:%d:%d): refuses tagged commands.  "
5624 			       "Performing non-tagged I/O\n", ahd_name(ahd),
5625 			       devinfo->channel, devinfo->target, devinfo->lun);
5626 			ahd_set_tags(ahd, scb->io_ctx, devinfo, AHD_QUEUE_NONE);
5627 			mask = ~0x23;
5628 		} else {
5629 			printk("(%s:%c:%d:%d): refuses %s tagged commands.  "
5630 			       "Performing simple queue tagged I/O only\n",
5631 			       ahd_name(ahd), devinfo->channel, devinfo->target,
5632 			       devinfo->lun, tag_type == MSG_ORDERED_TASK
5633 			       ? "ordered" : "head of queue");
5634 			ahd_set_tags(ahd, scb->io_ctx, devinfo, AHD_QUEUE_BASIC);
5635 			mask = ~0x03;
5636 		}
5637 
5638 		/*
5639 		 * Resend the identify for this CCB as the target
5640 		 * may believe that the selection is invalid otherwise.
5641 		 */
5642 		ahd_outb(ahd, SCB_CONTROL,
5643 			 ahd_inb_scbram(ahd, SCB_CONTROL) & mask);
5644 	 	scb->hscb->control &= mask;
5645 		ahd_set_transaction_tag(scb, /*enabled*/FALSE,
5646 					/*type*/MSG_SIMPLE_TASK);
5647 		ahd_outb(ahd, MSG_OUT, MSG_IDENTIFYFLAG);
5648 		ahd_assert_atn(ahd);
5649 		ahd_busy_tcl(ahd, BUILD_TCL(scb->hscb->scsiid, devinfo->lun),
5650 			     SCB_GET_TAG(scb));
5651 
5652 		/*
5653 		 * Requeue all tagged commands for this target
5654 		 * currently in our possession so they can be
5655 		 * converted to untagged commands.
5656 		 */
5657 		ahd_search_qinfifo(ahd, SCB_GET_TARGET(ahd, scb),
5658 				   SCB_GET_CHANNEL(ahd, scb),
5659 				   SCB_GET_LUN(scb), /*tag*/SCB_LIST_NULL,
5660 				   ROLE_INITIATOR, CAM_REQUEUE_REQ,
5661 				   SEARCH_COMPLETE);
5662 	} else if (ahd_sent_msg(ahd, AHDMSG_1B, MSG_IDENTIFYFLAG, TRUE)) {
5663 		/*
5664 		 * Most likely the device believes that we had
5665 		 * previously negotiated packetized.
5666 		 */
5667 		ahd->msg_flags |= MSG_FLAG_EXPECT_PPR_BUSFREE
5668 			       |  MSG_FLAG_IU_REQ_CHANGED;
5669 
5670 		ahd_force_renegotiation(ahd, devinfo);
5671 		ahd->msgout_index = 0;
5672 		ahd->msgout_len = 0;
5673 		ahd_build_transfer_msg(ahd, devinfo);
5674 		ahd->msgout_index = 0;
5675 		response = 1;
5676 	} else {
5677 		/*
5678 		 * Otherwise, we ignore it.
5679 		 */
5680 		printk("%s:%c:%d: Message reject for %x -- ignored\n",
5681 		       ahd_name(ahd), devinfo->channel, devinfo->target,
5682 		       last_msg);
5683 	}
5684 	return (response);
5685 }
5686 
5687 /*
5688  * Process an ingnore wide residue message.
5689  */
5690 static void
5691 ahd_handle_ign_wide_residue(struct ahd_softc *ahd, struct ahd_devinfo *devinfo)
5692 {
5693 	u_int scb_index;
5694 	struct scb *scb;
5695 
5696 	scb_index = ahd_get_scbptr(ahd);
5697 	scb = ahd_lookup_scb(ahd, scb_index);
5698 	/*
5699 	 * XXX Actually check data direction in the sequencer?
5700 	 * Perhaps add datadir to some spare bits in the hscb?
5701 	 */
5702 	if ((ahd_inb(ahd, SEQ_FLAGS) & DPHASE) == 0
5703 	 || ahd_get_transfer_dir(scb) != CAM_DIR_IN) {
5704 		/*
5705 		 * Ignore the message if we haven't
5706 		 * seen an appropriate data phase yet.
5707 		 */
5708 	} else {
5709 		/*
5710 		 * If the residual occurred on the last
5711 		 * transfer and the transfer request was
5712 		 * expected to end on an odd count, do
5713 		 * nothing.  Otherwise, subtract a byte
5714 		 * and update the residual count accordingly.
5715 		 */
5716 		uint32_t sgptr;
5717 
5718 		sgptr = ahd_inb_scbram(ahd, SCB_RESIDUAL_SGPTR);
5719 		if ((sgptr & SG_LIST_NULL) != 0
5720 		 && (ahd_inb_scbram(ahd, SCB_TASK_ATTRIBUTE)
5721 		     & SCB_XFERLEN_ODD) != 0) {
5722 			/*
5723 			 * If the residual occurred on the last
5724 			 * transfer and the transfer request was
5725 			 * expected to end on an odd count, do
5726 			 * nothing.
5727 			 */
5728 		} else {
5729 			uint32_t data_cnt;
5730 			uint64_t data_addr;
5731 			uint32_t sglen;
5732 
5733 			/* Pull in the rest of the sgptr */
5734 			sgptr = ahd_inl_scbram(ahd, SCB_RESIDUAL_SGPTR);
5735 			data_cnt = ahd_inl_scbram(ahd, SCB_RESIDUAL_DATACNT);
5736 			if ((sgptr & SG_LIST_NULL) != 0) {
5737 				/*
5738 				 * The residual data count is not updated
5739 				 * for the command run to completion case.
5740 				 * Explicitly zero the count.
5741 				 */
5742 				data_cnt &= ~AHD_SG_LEN_MASK;
5743 			}
5744 			data_addr = ahd_inq(ahd, SHADDR);
5745 			data_cnt += 1;
5746 			data_addr -= 1;
5747 			sgptr &= SG_PTR_MASK;
5748 			if ((ahd->flags & AHD_64BIT_ADDRESSING) != 0) {
5749 				struct ahd_dma64_seg *sg;
5750 
5751 				sg = ahd_sg_bus_to_virt(ahd, scb, sgptr);
5752 
5753 				/*
5754 				 * The residual sg ptr points to the next S/G
5755 				 * to load so we must go back one.
5756 				 */
5757 				sg--;
5758 				sglen = ahd_le32toh(sg->len) & AHD_SG_LEN_MASK;
5759 				if (sg != scb->sg_list
5760 				 && sglen < (data_cnt & AHD_SG_LEN_MASK)) {
5761 
5762 					sg--;
5763 					sglen = ahd_le32toh(sg->len);
5764 					/*
5765 					 * Preserve High Address and SG_LIST
5766 					 * bits while setting the count to 1.
5767 					 */
5768 					data_cnt = 1|(sglen&(~AHD_SG_LEN_MASK));
5769 					data_addr = ahd_le64toh(sg->addr)
5770 						  + (sglen & AHD_SG_LEN_MASK)
5771 						  - 1;
5772 
5773 					/*
5774 					 * Increment sg so it points to the
5775 					 * "next" sg.
5776 					 */
5777 					sg++;
5778 					sgptr = ahd_sg_virt_to_bus(ahd, scb,
5779 								   sg);
5780 				}
5781 			} else {
5782 				struct ahd_dma_seg *sg;
5783 
5784 				sg = ahd_sg_bus_to_virt(ahd, scb, sgptr);
5785 
5786 				/*
5787 				 * The residual sg ptr points to the next S/G
5788 				 * to load so we must go back one.
5789 				 */
5790 				sg--;
5791 				sglen = ahd_le32toh(sg->len) & AHD_SG_LEN_MASK;
5792 				if (sg != scb->sg_list
5793 				 && sglen < (data_cnt & AHD_SG_LEN_MASK)) {
5794 
5795 					sg--;
5796 					sglen = ahd_le32toh(sg->len);
5797 					/*
5798 					 * Preserve High Address and SG_LIST
5799 					 * bits while setting the count to 1.
5800 					 */
5801 					data_cnt = 1|(sglen&(~AHD_SG_LEN_MASK));
5802 					data_addr = ahd_le32toh(sg->addr)
5803 						  + (sglen & AHD_SG_LEN_MASK)
5804 						  - 1;
5805 
5806 					/*
5807 					 * Increment sg so it points to the
5808 					 * "next" sg.
5809 					 */
5810 					sg++;
5811 					sgptr = ahd_sg_virt_to_bus(ahd, scb,
5812 								  sg);
5813 				}
5814 			}
5815 			/*
5816 			 * Toggle the "oddness" of the transfer length
5817 			 * to handle this mid-transfer ignore wide
5818 			 * residue.  This ensures that the oddness is
5819 			 * correct for subsequent data transfers.
5820 			 */
5821 			ahd_outb(ahd, SCB_TASK_ATTRIBUTE,
5822 			    ahd_inb_scbram(ahd, SCB_TASK_ATTRIBUTE)
5823 			    ^ SCB_XFERLEN_ODD);
5824 
5825 			ahd_outl(ahd, SCB_RESIDUAL_SGPTR, sgptr);
5826 			ahd_outl(ahd, SCB_RESIDUAL_DATACNT, data_cnt);
5827 			/*
5828 			 * The FIFO's pointers will be updated if/when the
5829 			 * sequencer re-enters a data phase.
5830 			 */
5831 		}
5832 	}
5833 }
5834 
5835 
5836 /*
5837  * Reinitialize the data pointers for the active transfer
5838  * based on its current residual.
5839  */
5840 static void
5841 ahd_reinitialize_dataptrs(struct ahd_softc *ahd)
5842 {
5843 	struct		 scb *scb;
5844 	ahd_mode_state	 saved_modes;
5845 	u_int		 scb_index;
5846 	u_int		 wait;
5847 	uint32_t	 sgptr;
5848 	uint32_t	 resid;
5849 	uint64_t	 dataptr;
5850 
5851 	AHD_ASSERT_MODES(ahd, AHD_MODE_DFF0_MSK|AHD_MODE_DFF1_MSK,
5852 			 AHD_MODE_DFF0_MSK|AHD_MODE_DFF1_MSK);
5853 
5854 	scb_index = ahd_get_scbptr(ahd);
5855 	scb = ahd_lookup_scb(ahd, scb_index);
5856 
5857 	/*
5858 	 * Release and reacquire the FIFO so we
5859 	 * have a clean slate.
5860 	 */
5861 	ahd_outb(ahd, DFFSXFRCTL, CLRCHN);
5862 	wait = 1000;
5863 	while (--wait && !(ahd_inb(ahd, MDFFSTAT) & FIFOFREE))
5864 		ahd_delay(100);
5865 	if (wait == 0) {
5866 		ahd_print_path(ahd, scb);
5867 		printk("ahd_reinitialize_dataptrs: Forcing FIFO free.\n");
5868 		ahd_outb(ahd, DFFSXFRCTL, RSTCHN|CLRSHCNT);
5869 	}
5870 	saved_modes = ahd_save_modes(ahd);
5871 	ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
5872 	ahd_outb(ahd, DFFSTAT,
5873 		 ahd_inb(ahd, DFFSTAT)
5874 		| (saved_modes == 0x11 ? CURRFIFO_1 : CURRFIFO_0));
5875 
5876 	/*
5877 	 * Determine initial values for data_addr and data_cnt
5878 	 * for resuming the data phase.
5879 	 */
5880 	sgptr = ahd_inl_scbram(ahd, SCB_RESIDUAL_SGPTR);
5881 	sgptr &= SG_PTR_MASK;
5882 
5883 	resid = (ahd_inb_scbram(ahd, SCB_RESIDUAL_DATACNT + 2) << 16)
5884 	      | (ahd_inb_scbram(ahd, SCB_RESIDUAL_DATACNT + 1) << 8)
5885 	      | ahd_inb_scbram(ahd, SCB_RESIDUAL_DATACNT);
5886 
5887 	if ((ahd->flags & AHD_64BIT_ADDRESSING) != 0) {
5888 		struct ahd_dma64_seg *sg;
5889 
5890 		sg = ahd_sg_bus_to_virt(ahd, scb, sgptr);
5891 
5892 		/* The residual sg_ptr always points to the next sg */
5893 		sg--;
5894 
5895 		dataptr = ahd_le64toh(sg->addr)
5896 			+ (ahd_le32toh(sg->len) & AHD_SG_LEN_MASK)
5897 			- resid;
5898 		ahd_outl(ahd, HADDR + 4, dataptr >> 32);
5899 	} else {
5900 		struct	 ahd_dma_seg *sg;
5901 
5902 		sg = ahd_sg_bus_to_virt(ahd, scb, sgptr);
5903 
5904 		/* The residual sg_ptr always points to the next sg */
5905 		sg--;
5906 
5907 		dataptr = ahd_le32toh(sg->addr)
5908 			+ (ahd_le32toh(sg->len) & AHD_SG_LEN_MASK)
5909 			- resid;
5910 		ahd_outb(ahd, HADDR + 4,
5911 			 (ahd_le32toh(sg->len) & ~AHD_SG_LEN_MASK) >> 24);
5912 	}
5913 	ahd_outl(ahd, HADDR, dataptr);
5914 	ahd_outb(ahd, HCNT + 2, resid >> 16);
5915 	ahd_outb(ahd, HCNT + 1, resid >> 8);
5916 	ahd_outb(ahd, HCNT, resid);
5917 }
5918 
5919 /*
5920  * Handle the effects of issuing a bus device reset message.
5921  */
5922 static void
5923 ahd_handle_devreset(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
5924 		    u_int lun, cam_status status, char *message,
5925 		    int verbose_level)
5926 {
5927 #ifdef AHD_TARGET_MODE
5928 	struct ahd_tmode_tstate* tstate;
5929 #endif
5930 	int found;
5931 
5932 	found = ahd_abort_scbs(ahd, devinfo->target, devinfo->channel,
5933 			       lun, SCB_LIST_NULL, devinfo->role,
5934 			       status);
5935 
5936 #ifdef AHD_TARGET_MODE
5937 	/*
5938 	 * Send an immediate notify ccb to all target mord peripheral
5939 	 * drivers affected by this action.
5940 	 */
5941 	tstate = ahd->enabled_targets[devinfo->our_scsiid];
5942 	if (tstate != NULL) {
5943 		u_int cur_lun;
5944 		u_int max_lun;
5945 
5946 		if (lun != CAM_LUN_WILDCARD) {
5947 			cur_lun = 0;
5948 			max_lun = AHD_NUM_LUNS - 1;
5949 		} else {
5950 			cur_lun = lun;
5951 			max_lun = lun;
5952 		}
5953 		for (;cur_lun <= max_lun; cur_lun++) {
5954 			struct ahd_tmode_lstate* lstate;
5955 
5956 			lstate = tstate->enabled_luns[cur_lun];
5957 			if (lstate == NULL)
5958 				continue;
5959 
5960 			ahd_queue_lstate_event(ahd, lstate, devinfo->our_scsiid,
5961 					       MSG_BUS_DEV_RESET, /*arg*/0);
5962 			ahd_send_lstate_events(ahd, lstate);
5963 		}
5964 	}
5965 #endif
5966 
5967 	/*
5968 	 * Go back to async/narrow transfers and renegotiate.
5969 	 */
5970 	ahd_set_width(ahd, devinfo, MSG_EXT_WDTR_BUS_8_BIT,
5971 		      AHD_TRANS_CUR, /*paused*/TRUE);
5972 	ahd_set_syncrate(ahd, devinfo, /*period*/0, /*offset*/0,
5973 			 /*ppr_options*/0, AHD_TRANS_CUR,
5974 			 /*paused*/TRUE);
5975 
5976 	if (status != CAM_SEL_TIMEOUT)
5977 		ahd_send_async(ahd, devinfo->channel, devinfo->target,
5978 			       CAM_LUN_WILDCARD, AC_SENT_BDR);
5979 
5980 	if (message != NULL && bootverbose)
5981 		printk("%s: %s on %c:%d. %d SCBs aborted\n", ahd_name(ahd),
5982 		       message, devinfo->channel, devinfo->target, found);
5983 }
5984 
5985 #ifdef AHD_TARGET_MODE
5986 static void
5987 ahd_setup_target_msgin(struct ahd_softc *ahd, struct ahd_devinfo *devinfo,
5988 		       struct scb *scb)
5989 {
5990 
5991 	/*
5992 	 * To facilitate adding multiple messages together,
5993 	 * each routine should increment the index and len
5994 	 * variables instead of setting them explicitly.
5995 	 */
5996 	ahd->msgout_index = 0;
5997 	ahd->msgout_len = 0;
5998 
5999 	if (scb != NULL && (scb->flags & SCB_AUTO_NEGOTIATE) != 0)
6000 		ahd_build_transfer_msg(ahd, devinfo);
6001 	else
6002 		panic("ahd_intr: AWAITING target message with no message");
6003 
6004 	ahd->msgout_index = 0;
6005 	ahd->msg_type = MSG_TYPE_TARGET_MSGIN;
6006 }
6007 #endif
6008 /**************************** Initialization **********************************/
6009 static u_int
6010 ahd_sglist_size(struct ahd_softc *ahd)
6011 {
6012 	bus_size_t list_size;
6013 
6014 	list_size = sizeof(struct ahd_dma_seg) * AHD_NSEG;
6015 	if ((ahd->flags & AHD_64BIT_ADDRESSING) != 0)
6016 		list_size = sizeof(struct ahd_dma64_seg) * AHD_NSEG;
6017 	return (list_size);
6018 }
6019 
6020 /*
6021  * Calculate the optimum S/G List allocation size.  S/G elements used
6022  * for a given transaction must be physically contiguous.  Assume the
6023  * OS will allocate full pages to us, so it doesn't make sense to request
6024  * less than a page.
6025  */
6026 static u_int
6027 ahd_sglist_allocsize(struct ahd_softc *ahd)
6028 {
6029 	bus_size_t sg_list_increment;
6030 	bus_size_t sg_list_size;
6031 	bus_size_t max_list_size;
6032 	bus_size_t best_list_size;
6033 
6034 	/* Start out with the minimum required for AHD_NSEG. */
6035 	sg_list_increment = ahd_sglist_size(ahd);
6036 	sg_list_size = sg_list_increment;
6037 
6038 	/* Get us as close as possible to a page in size. */
6039 	while ((sg_list_size + sg_list_increment) <= PAGE_SIZE)
6040 		sg_list_size += sg_list_increment;
6041 
6042 	/*
6043 	 * Try to reduce the amount of wastage by allocating
6044 	 * multiple pages.
6045 	 */
6046 	best_list_size = sg_list_size;
6047 	max_list_size = roundup(sg_list_increment, PAGE_SIZE);
6048 	if (max_list_size < 4 * PAGE_SIZE)
6049 		max_list_size = 4 * PAGE_SIZE;
6050 	if (max_list_size > (AHD_SCB_MAX_ALLOC * sg_list_increment))
6051 		max_list_size = (AHD_SCB_MAX_ALLOC * sg_list_increment);
6052 	while ((sg_list_size + sg_list_increment) <= max_list_size
6053 	   &&  (sg_list_size % PAGE_SIZE) != 0) {
6054 		bus_size_t new_mod;
6055 		bus_size_t best_mod;
6056 
6057 		sg_list_size += sg_list_increment;
6058 		new_mod = sg_list_size % PAGE_SIZE;
6059 		best_mod = best_list_size % PAGE_SIZE;
6060 		if (new_mod > best_mod || new_mod == 0) {
6061 			best_list_size = sg_list_size;
6062 		}
6063 	}
6064 	return (best_list_size);
6065 }
6066 
6067 /*
6068  * Allocate a controller structure for a new device
6069  * and perform initial initializion.
6070  */
6071 struct ahd_softc *
6072 ahd_alloc(void *platform_arg, char *name)
6073 {
6074 	struct  ahd_softc *ahd;
6075 
6076 #ifndef	__FreeBSD__
6077 	ahd = kmalloc(sizeof(*ahd), GFP_ATOMIC);
6078 	if (!ahd) {
6079 		printk("aic7xxx: cannot malloc softc!\n");
6080 		kfree(name);
6081 		return NULL;
6082 	}
6083 #else
6084 	ahd = device_get_softc((device_t)platform_arg);
6085 #endif
6086 	memset(ahd, 0, sizeof(*ahd));
6087 	ahd->seep_config = kmalloc(sizeof(*ahd->seep_config), GFP_ATOMIC);
6088 	if (ahd->seep_config == NULL) {
6089 #ifndef	__FreeBSD__
6090 		kfree(ahd);
6091 #endif
6092 		kfree(name);
6093 		return (NULL);
6094 	}
6095 	LIST_INIT(&ahd->pending_scbs);
6096 	/* We don't know our unit number until the OSM sets it */
6097 	ahd->name = name;
6098 	ahd->unit = -1;
6099 	ahd->description = NULL;
6100 	ahd->bus_description = NULL;
6101 	ahd->channel = 'A';
6102 	ahd->chip = AHD_NONE;
6103 	ahd->features = AHD_FENONE;
6104 	ahd->bugs = AHD_BUGNONE;
6105 	ahd->flags = AHD_SPCHK_ENB_A|AHD_RESET_BUS_A|AHD_TERM_ENB_A
6106 		   | AHD_EXTENDED_TRANS_A|AHD_STPWLEVEL_A;
6107 	timer_setup(&ahd->stat_timer, ahd_stat_timer, 0);
6108 	ahd->int_coalescing_timer = AHD_INT_COALESCING_TIMER_DEFAULT;
6109 	ahd->int_coalescing_maxcmds = AHD_INT_COALESCING_MAXCMDS_DEFAULT;
6110 	ahd->int_coalescing_mincmds = AHD_INT_COALESCING_MINCMDS_DEFAULT;
6111 	ahd->int_coalescing_threshold = AHD_INT_COALESCING_THRESHOLD_DEFAULT;
6112 	ahd->int_coalescing_stop_threshold =
6113 	    AHD_INT_COALESCING_STOP_THRESHOLD_DEFAULT;
6114 
6115 	if (ahd_platform_alloc(ahd, platform_arg) != 0) {
6116 		ahd_free(ahd);
6117 		ahd = NULL;
6118 	}
6119 #ifdef AHD_DEBUG
6120 	if ((ahd_debug & AHD_SHOW_MEMORY) != 0) {
6121 		printk("%s: scb size = 0x%x, hscb size = 0x%x\n",
6122 		       ahd_name(ahd), (u_int)sizeof(struct scb),
6123 		       (u_int)sizeof(struct hardware_scb));
6124 	}
6125 #endif
6126 	return (ahd);
6127 }
6128 
6129 int
6130 ahd_softc_init(struct ahd_softc *ahd)
6131 {
6132 
6133 	ahd->unpause = 0;
6134 	ahd->pause = PAUSE;
6135 	return (0);
6136 }
6137 
6138 void
6139 ahd_set_unit(struct ahd_softc *ahd, int unit)
6140 {
6141 	ahd->unit = unit;
6142 }
6143 
6144 void
6145 ahd_set_name(struct ahd_softc *ahd, char *name)
6146 {
6147 	if (ahd->name != NULL)
6148 		kfree(ahd->name);
6149 	ahd->name = name;
6150 }
6151 
6152 void
6153 ahd_free(struct ahd_softc *ahd)
6154 {
6155 	int i;
6156 
6157 	switch (ahd->init_level) {
6158 	default:
6159 	case 5:
6160 		ahd_shutdown(ahd);
6161 		/* FALLTHROUGH */
6162 	case 4:
6163 		ahd_dmamap_unload(ahd, ahd->shared_data_dmat,
6164 				  ahd->shared_data_map.dmamap);
6165 		/* FALLTHROUGH */
6166 	case 3:
6167 		ahd_dmamem_free(ahd, ahd->shared_data_dmat, ahd->qoutfifo,
6168 				ahd->shared_data_map.dmamap);
6169 		ahd_dmamap_destroy(ahd, ahd->shared_data_dmat,
6170 				   ahd->shared_data_map.dmamap);
6171 		/* FALLTHROUGH */
6172 	case 2:
6173 		ahd_dma_tag_destroy(ahd, ahd->shared_data_dmat);
6174 	case 1:
6175 #ifndef __linux__
6176 		ahd_dma_tag_destroy(ahd, ahd->buffer_dmat);
6177 #endif
6178 		break;
6179 	case 0:
6180 		break;
6181 	}
6182 
6183 #ifndef __linux__
6184 	ahd_dma_tag_destroy(ahd, ahd->parent_dmat);
6185 #endif
6186 	ahd_platform_free(ahd);
6187 	ahd_fini_scbdata(ahd);
6188 	for (i = 0; i < AHD_NUM_TARGETS; i++) {
6189 		struct ahd_tmode_tstate *tstate;
6190 
6191 		tstate = ahd->enabled_targets[i];
6192 		if (tstate != NULL) {
6193 #ifdef AHD_TARGET_MODE
6194 			int j;
6195 
6196 			for (j = 0; j < AHD_NUM_LUNS; j++) {
6197 				struct ahd_tmode_lstate *lstate;
6198 
6199 				lstate = tstate->enabled_luns[j];
6200 				if (lstate != NULL) {
6201 					xpt_free_path(lstate->path);
6202 					kfree(lstate);
6203 				}
6204 			}
6205 #endif
6206 			kfree(tstate);
6207 		}
6208 	}
6209 #ifdef AHD_TARGET_MODE
6210 	if (ahd->black_hole != NULL) {
6211 		xpt_free_path(ahd->black_hole->path);
6212 		kfree(ahd->black_hole);
6213 	}
6214 #endif
6215 	if (ahd->name != NULL)
6216 		kfree(ahd->name);
6217 	if (ahd->seep_config != NULL)
6218 		kfree(ahd->seep_config);
6219 	if (ahd->saved_stack != NULL)
6220 		kfree(ahd->saved_stack);
6221 #ifndef __FreeBSD__
6222 	kfree(ahd);
6223 #endif
6224 	return;
6225 }
6226 
6227 static void
6228 ahd_shutdown(void *arg)
6229 {
6230 	struct	ahd_softc *ahd;
6231 
6232 	ahd = (struct ahd_softc *)arg;
6233 
6234 	/*
6235 	 * Stop periodic timer callbacks.
6236 	 */
6237 	del_timer_sync(&ahd->stat_timer);
6238 
6239 	/* This will reset most registers to 0, but not all */
6240 	ahd_reset(ahd, /*reinit*/FALSE);
6241 }
6242 
6243 /*
6244  * Reset the controller and record some information about it
6245  * that is only available just after a reset.  If "reinit" is
6246  * non-zero, this reset occurred after initial configuration
6247  * and the caller requests that the chip be fully reinitialized
6248  * to a runable state.  Chip interrupts are *not* enabled after
6249  * a reinitialization.  The caller must enable interrupts via
6250  * ahd_intr_enable().
6251  */
6252 int
6253 ahd_reset(struct ahd_softc *ahd, int reinit)
6254 {
6255 	u_int	 sxfrctl1;
6256 	int	 wait;
6257 	uint32_t cmd;
6258 
6259 	/*
6260 	 * Preserve the value of the SXFRCTL1 register for all channels.
6261 	 * It contains settings that affect termination and we don't want
6262 	 * to disturb the integrity of the bus.
6263 	 */
6264 	ahd_pause(ahd);
6265 	ahd_update_modes(ahd);
6266 	ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
6267 	sxfrctl1 = ahd_inb(ahd, SXFRCTL1);
6268 
6269 	cmd = ahd_pci_read_config(ahd->dev_softc, PCIR_COMMAND, /*bytes*/2);
6270 	if ((ahd->bugs & AHD_PCIX_CHIPRST_BUG) != 0) {
6271 		uint32_t mod_cmd;
6272 
6273 		/*
6274 		 * A4 Razor #632
6275 		 * During the assertion of CHIPRST, the chip
6276 		 * does not disable its parity logic prior to
6277 		 * the start of the reset.  This may cause a
6278 		 * parity error to be detected and thus a
6279 		 * spurious SERR or PERR assertion.  Disable
6280 		 * PERR and SERR responses during the CHIPRST.
6281 		 */
6282 		mod_cmd = cmd & ~(PCIM_CMD_PERRESPEN|PCIM_CMD_SERRESPEN);
6283 		ahd_pci_write_config(ahd->dev_softc, PCIR_COMMAND,
6284 				     mod_cmd, /*bytes*/2);
6285 	}
6286 	ahd_outb(ahd, HCNTRL, CHIPRST | ahd->pause);
6287 
6288 	/*
6289 	 * Ensure that the reset has finished.  We delay 1000us
6290 	 * prior to reading the register to make sure the chip
6291 	 * has sufficiently completed its reset to handle register
6292 	 * accesses.
6293 	 */
6294 	wait = 1000;
6295 	do {
6296 		ahd_delay(1000);
6297 	} while (--wait && !(ahd_inb(ahd, HCNTRL) & CHIPRSTACK));
6298 
6299 	if (wait == 0) {
6300 		printk("%s: WARNING - Failed chip reset!  "
6301 		       "Trying to initialize anyway.\n", ahd_name(ahd));
6302 	}
6303 	ahd_outb(ahd, HCNTRL, ahd->pause);
6304 
6305 	if ((ahd->bugs & AHD_PCIX_CHIPRST_BUG) != 0) {
6306 		/*
6307 		 * Clear any latched PCI error status and restore
6308 		 * previous SERR and PERR response enables.
6309 		 */
6310 		ahd_pci_write_config(ahd->dev_softc, PCIR_STATUS + 1,
6311 				     0xFF, /*bytes*/1);
6312 		ahd_pci_write_config(ahd->dev_softc, PCIR_COMMAND,
6313 				     cmd, /*bytes*/2);
6314 	}
6315 
6316 	/*
6317 	 * Mode should be SCSI after a chip reset, but lets
6318 	 * set it just to be safe.  We touch the MODE_PTR
6319 	 * register directly so as to bypass the lazy update
6320 	 * code in ahd_set_modes().
6321 	 */
6322 	ahd_known_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
6323 	ahd_outb(ahd, MODE_PTR,
6324 		 ahd_build_mode_state(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI));
6325 
6326 	/*
6327 	 * Restore SXFRCTL1.
6328 	 *
6329 	 * We must always initialize STPWEN to 1 before we
6330 	 * restore the saved values.  STPWEN is initialized
6331 	 * to a tri-state condition which can only be cleared
6332 	 * by turning it on.
6333 	 */
6334 	ahd_outb(ahd, SXFRCTL1, sxfrctl1|STPWEN);
6335 	ahd_outb(ahd, SXFRCTL1, sxfrctl1);
6336 
6337 	/* Determine chip configuration */
6338 	ahd->features &= ~AHD_WIDE;
6339 	if ((ahd_inb(ahd, SBLKCTL) & SELWIDE) != 0)
6340 		ahd->features |= AHD_WIDE;
6341 
6342 	/*
6343 	 * If a recovery action has forced a chip reset,
6344 	 * re-initialize the chip to our liking.
6345 	 */
6346 	if (reinit != 0)
6347 		ahd_chip_init(ahd);
6348 
6349 	return (0);
6350 }
6351 
6352 /*
6353  * Determine the number of SCBs available on the controller
6354  */
6355 static int
6356 ahd_probe_scbs(struct ahd_softc *ahd) {
6357 	int i;
6358 
6359 	AHD_ASSERT_MODES(ahd, ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK),
6360 			 ~(AHD_MODE_UNKNOWN_MSK|AHD_MODE_CFG_MSK));
6361 	for (i = 0; i < AHD_SCB_MAX; i++) {
6362 		int j;
6363 
6364 		ahd_set_scbptr(ahd, i);
6365 		ahd_outw(ahd, SCB_BASE, i);
6366 		for (j = 2; j < 64; j++)
6367 			ahd_outb(ahd, SCB_BASE+j, 0);
6368 		/* Start out life as unallocated (needing an abort) */
6369 		ahd_outb(ahd, SCB_CONTROL, MK_MESSAGE);
6370 		if (ahd_inw_scbram(ahd, SCB_BASE) != i)
6371 			break;
6372 		ahd_set_scbptr(ahd, 0);
6373 		if (ahd_inw_scbram(ahd, SCB_BASE) != 0)
6374 			break;
6375 	}
6376 	return (i);
6377 }
6378 
6379 static void
6380 ahd_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
6381 {
6382 	dma_addr_t *baddr;
6383 
6384 	baddr = (dma_addr_t *)arg;
6385 	*baddr = segs->ds_addr;
6386 }
6387 
6388 static void
6389 ahd_initialize_hscbs(struct ahd_softc *ahd)
6390 {
6391 	int i;
6392 
6393 	for (i = 0; i < ahd->scb_data.maxhscbs; i++) {
6394 		ahd_set_scbptr(ahd, i);
6395 
6396 		/* Clear the control byte. */
6397 		ahd_outb(ahd, SCB_CONTROL, 0);
6398 
6399 		/* Set the next pointer */
6400 		ahd_outw(ahd, SCB_NEXT, SCB_LIST_NULL);
6401 	}
6402 }
6403 
6404 static int
6405 ahd_init_scbdata(struct ahd_softc *ahd)
6406 {
6407 	struct	scb_data *scb_data;
6408 	int	i;
6409 
6410 	scb_data = &ahd->scb_data;
6411 	TAILQ_INIT(&scb_data->free_scbs);
6412 	for (i = 0; i < AHD_NUM_TARGETS * AHD_NUM_LUNS_NONPKT; i++)
6413 		LIST_INIT(&scb_data->free_scb_lists[i]);
6414 	LIST_INIT(&scb_data->any_dev_free_scb_list);
6415 	SLIST_INIT(&scb_data->hscb_maps);
6416 	SLIST_INIT(&scb_data->sg_maps);
6417 	SLIST_INIT(&scb_data->sense_maps);
6418 
6419 	/* Determine the number of hardware SCBs and initialize them */
6420 	scb_data->maxhscbs = ahd_probe_scbs(ahd);
6421 	if (scb_data->maxhscbs == 0) {
6422 		printk("%s: No SCB space found\n", ahd_name(ahd));
6423 		return (ENXIO);
6424 	}
6425 
6426 	ahd_initialize_hscbs(ahd);
6427 
6428 	/*
6429 	 * Create our DMA tags.  These tags define the kinds of device
6430 	 * accessible memory allocations and memory mappings we will
6431 	 * need to perform during normal operation.
6432 	 *
6433 	 * Unless we need to further restrict the allocation, we rely
6434 	 * on the restrictions of the parent dmat, hence the common
6435 	 * use of MAXADDR and MAXSIZE.
6436 	 */
6437 
6438 	/* DMA tag for our hardware scb structures */
6439 	if (ahd_dma_tag_create(ahd, ahd->parent_dmat, /*alignment*/1,
6440 			       /*boundary*/BUS_SPACE_MAXADDR_32BIT + 1,
6441 			       /*lowaddr*/BUS_SPACE_MAXADDR_32BIT,
6442 			       /*highaddr*/BUS_SPACE_MAXADDR,
6443 			       /*filter*/NULL, /*filterarg*/NULL,
6444 			       PAGE_SIZE, /*nsegments*/1,
6445 			       /*maxsegsz*/BUS_SPACE_MAXSIZE_32BIT,
6446 			       /*flags*/0, &scb_data->hscb_dmat) != 0) {
6447 		goto error_exit;
6448 	}
6449 
6450 	scb_data->init_level++;
6451 
6452 	/* DMA tag for our S/G structures. */
6453 	if (ahd_dma_tag_create(ahd, ahd->parent_dmat, /*alignment*/8,
6454 			       /*boundary*/BUS_SPACE_MAXADDR_32BIT + 1,
6455 			       /*lowaddr*/BUS_SPACE_MAXADDR_32BIT,
6456 			       /*highaddr*/BUS_SPACE_MAXADDR,
6457 			       /*filter*/NULL, /*filterarg*/NULL,
6458 			       ahd_sglist_allocsize(ahd), /*nsegments*/1,
6459 			       /*maxsegsz*/BUS_SPACE_MAXSIZE_32BIT,
6460 			       /*flags*/0, &scb_data->sg_dmat) != 0) {
6461 		goto error_exit;
6462 	}
6463 #ifdef AHD_DEBUG
6464 	if ((ahd_debug & AHD_SHOW_MEMORY) != 0)
6465 		printk("%s: ahd_sglist_allocsize = 0x%x\n", ahd_name(ahd),
6466 		       ahd_sglist_allocsize(ahd));
6467 #endif
6468 
6469 	scb_data->init_level++;
6470 
6471 	/* DMA tag for our sense buffers.  We allocate in page sized chunks */
6472 	if (ahd_dma_tag_create(ahd, ahd->parent_dmat, /*alignment*/1,
6473 			       /*boundary*/BUS_SPACE_MAXADDR_32BIT + 1,
6474 			       /*lowaddr*/BUS_SPACE_MAXADDR_32BIT,
6475 			       /*highaddr*/BUS_SPACE_MAXADDR,
6476 			       /*filter*/NULL, /*filterarg*/NULL,
6477 			       PAGE_SIZE, /*nsegments*/1,
6478 			       /*maxsegsz*/BUS_SPACE_MAXSIZE_32BIT,
6479 			       /*flags*/0, &scb_data->sense_dmat) != 0) {
6480 		goto error_exit;
6481 	}
6482 
6483 	scb_data->init_level++;
6484 
6485 	/* Perform initial CCB allocation */
6486 	ahd_alloc_scbs(ahd);
6487 
6488 	if (scb_data->numscbs == 0) {
6489 		printk("%s: ahd_init_scbdata - "
6490 		       "Unable to allocate initial scbs\n",
6491 		       ahd_name(ahd));
6492 		goto error_exit;
6493 	}
6494 
6495 	/*
6496 	 * Note that we were successful
6497 	 */
6498 	return (0);
6499 
6500 error_exit:
6501 
6502 	return (ENOMEM);
6503 }
6504 
6505 static struct scb *
6506 ahd_find_scb_by_tag(struct ahd_softc *ahd, u_int tag)
6507 {
6508 	struct scb *scb;
6509 
6510 	/*
6511 	 * Look on the pending list.
6512 	 */
6513 	LIST_FOREACH(scb, &ahd->pending_scbs, pending_links) {
6514 		if (SCB_GET_TAG(scb) == tag)
6515 			return (scb);
6516 	}
6517 
6518 	/*
6519 	 * Then on all of the collision free lists.
6520 	 */
6521 	TAILQ_FOREACH(scb, &ahd->scb_data.free_scbs, links.tqe) {
6522 		struct scb *list_scb;
6523 
6524 		list_scb = scb;
6525 		do {
6526 			if (SCB_GET_TAG(list_scb) == tag)
6527 				return (list_scb);
6528 			list_scb = LIST_NEXT(list_scb, collision_links);
6529 		} while (list_scb);
6530 	}
6531 
6532 	/*
6533 	 * And finally on the generic free list.
6534 	 */
6535 	LIST_FOREACH(scb, &ahd->scb_data.any_dev_free_scb_list, links.le) {
6536 		if (SCB_GET_TAG(scb) == tag)
6537 			return (scb);
6538 	}
6539 
6540 	return (NULL);
6541 }
6542 
6543 static void
6544 ahd_fini_scbdata(struct ahd_softc *ahd)
6545 {
6546 	struct scb_data *scb_data;
6547 
6548 	scb_data = &ahd->scb_data;
6549 	if (scb_data == NULL)
6550 		return;
6551 
6552 	switch (scb_data->init_level) {
6553 	default:
6554 	case 7:
6555 	{
6556 		struct map_node *sns_map;
6557 
6558 		while ((sns_map = SLIST_FIRST(&scb_data->sense_maps)) != NULL) {
6559 			SLIST_REMOVE_HEAD(&scb_data->sense_maps, links);
6560 			ahd_dmamap_unload(ahd, scb_data->sense_dmat,
6561 					  sns_map->dmamap);
6562 			ahd_dmamem_free(ahd, scb_data->sense_dmat,
6563 					sns_map->vaddr, sns_map->dmamap);
6564 			kfree(sns_map);
6565 		}
6566 		ahd_dma_tag_destroy(ahd, scb_data->sense_dmat);
6567 		/* FALLTHROUGH */
6568 	}
6569 	case 6:
6570 	{
6571 		struct map_node *sg_map;
6572 
6573 		while ((sg_map = SLIST_FIRST(&scb_data->sg_maps)) != NULL) {
6574 			SLIST_REMOVE_HEAD(&scb_data->sg_maps, links);
6575 			ahd_dmamap_unload(ahd, scb_data->sg_dmat,
6576 					  sg_map->dmamap);
6577 			ahd_dmamem_free(ahd, scb_data->sg_dmat,
6578 					sg_map->vaddr, sg_map->dmamap);
6579 			kfree(sg_map);
6580 		}
6581 		ahd_dma_tag_destroy(ahd, scb_data->sg_dmat);
6582 		/* FALLTHROUGH */
6583 	}
6584 	case 5:
6585 	{
6586 		struct map_node *hscb_map;
6587 
6588 		while ((hscb_map = SLIST_FIRST(&scb_data->hscb_maps)) != NULL) {
6589 			SLIST_REMOVE_HEAD(&scb_data->hscb_maps, links);
6590 			ahd_dmamap_unload(ahd, scb_data->hscb_dmat,
6591 					  hscb_map->dmamap);
6592 			ahd_dmamem_free(ahd, scb_data->hscb_dmat,
6593 					hscb_map->vaddr, hscb_map->dmamap);
6594 			kfree(hscb_map);
6595 		}
6596 		ahd_dma_tag_destroy(ahd, scb_data->hscb_dmat);
6597 		/* FALLTHROUGH */
6598 	}
6599 	case 4:
6600 	case 3:
6601 	case 2:
6602 	case 1:
6603 	case 0:
6604 		break;
6605 	}
6606 }
6607 
6608 /*
6609  * DSP filter Bypass must be enabled until the first selection
6610  * after a change in bus mode (Razor #491 and #493).
6611  */
6612 static void
6613 ahd_setup_iocell_workaround(struct ahd_softc *ahd)
6614 {
6615 	ahd_mode_state saved_modes;
6616 
6617 	saved_modes = ahd_save_modes(ahd);
6618 	ahd_set_modes(ahd, AHD_MODE_CFG, AHD_MODE_CFG);
6619 	ahd_outb(ahd, DSPDATACTL, ahd_inb(ahd, DSPDATACTL)
6620 	       | BYPASSENAB | RCVROFFSTDIS | XMITOFFSTDIS);
6621 	ahd_outb(ahd, SIMODE0, ahd_inb(ahd, SIMODE0) | (ENSELDO|ENSELDI));
6622 #ifdef AHD_DEBUG
6623 	if ((ahd_debug & AHD_SHOW_MISC) != 0)
6624 		printk("%s: Setting up iocell workaround\n", ahd_name(ahd));
6625 #endif
6626 	ahd_restore_modes(ahd, saved_modes);
6627 	ahd->flags &= ~AHD_HAD_FIRST_SEL;
6628 }
6629 
6630 static void
6631 ahd_iocell_first_selection(struct ahd_softc *ahd)
6632 {
6633 	ahd_mode_state	saved_modes;
6634 	u_int		sblkctl;
6635 
6636 	if ((ahd->flags & AHD_HAD_FIRST_SEL) != 0)
6637 		return;
6638 	saved_modes = ahd_save_modes(ahd);
6639 	ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
6640 	sblkctl = ahd_inb(ahd, SBLKCTL);
6641 	ahd_set_modes(ahd, AHD_MODE_CFG, AHD_MODE_CFG);
6642 #ifdef AHD_DEBUG
6643 	if ((ahd_debug & AHD_SHOW_MISC) != 0)
6644 		printk("%s: iocell first selection\n", ahd_name(ahd));
6645 #endif
6646 	if ((sblkctl & ENAB40) != 0) {
6647 		ahd_outb(ahd, DSPDATACTL,
6648 			 ahd_inb(ahd, DSPDATACTL) & ~BYPASSENAB);
6649 #ifdef AHD_DEBUG
6650 		if ((ahd_debug & AHD_SHOW_MISC) != 0)
6651 			printk("%s: BYPASS now disabled\n", ahd_name(ahd));
6652 #endif
6653 	}
6654 	ahd_outb(ahd, SIMODE0, ahd_inb(ahd, SIMODE0) & ~(ENSELDO|ENSELDI));
6655 	ahd_outb(ahd, CLRINT, CLRSCSIINT);
6656 	ahd_restore_modes(ahd, saved_modes);
6657 	ahd->flags |= AHD_HAD_FIRST_SEL;
6658 }
6659 
6660 /*************************** SCB Management ***********************************/
6661 static void
6662 ahd_add_col_list(struct ahd_softc *ahd, struct scb *scb, u_int col_idx)
6663 {
6664 	struct	scb_list *free_list;
6665 	struct	scb_tailq *free_tailq;
6666 	struct	scb *first_scb;
6667 
6668 	scb->flags |= SCB_ON_COL_LIST;
6669 	AHD_SET_SCB_COL_IDX(scb, col_idx);
6670 	free_list = &ahd->scb_data.free_scb_lists[col_idx];
6671 	free_tailq = &ahd->scb_data.free_scbs;
6672 	first_scb = LIST_FIRST(free_list);
6673 	if (first_scb != NULL) {
6674 		LIST_INSERT_AFTER(first_scb, scb, collision_links);
6675 	} else {
6676 		LIST_INSERT_HEAD(free_list, scb, collision_links);
6677 		TAILQ_INSERT_TAIL(free_tailq, scb, links.tqe);
6678 	}
6679 }
6680 
6681 static void
6682 ahd_rem_col_list(struct ahd_softc *ahd, struct scb *scb)
6683 {
6684 	struct	scb_list *free_list;
6685 	struct	scb_tailq *free_tailq;
6686 	struct	scb *first_scb;
6687 	u_int	col_idx;
6688 
6689 	scb->flags &= ~SCB_ON_COL_LIST;
6690 	col_idx = AHD_GET_SCB_COL_IDX(ahd, scb);
6691 	free_list = &ahd->scb_data.free_scb_lists[col_idx];
6692 	free_tailq = &ahd->scb_data.free_scbs;
6693 	first_scb = LIST_FIRST(free_list);
6694 	if (first_scb == scb) {
6695 		struct scb *next_scb;
6696 
6697 		/*
6698 		 * Maintain order in the collision free
6699 		 * lists for fairness if this device has
6700 		 * other colliding tags active.
6701 		 */
6702 		next_scb = LIST_NEXT(scb, collision_links);
6703 		if (next_scb != NULL) {
6704 			TAILQ_INSERT_AFTER(free_tailq, scb,
6705 					   next_scb, links.tqe);
6706 		}
6707 		TAILQ_REMOVE(free_tailq, scb, links.tqe);
6708 	}
6709 	LIST_REMOVE(scb, collision_links);
6710 }
6711 
6712 /*
6713  * Get a free scb. If there are none, see if we can allocate a new SCB.
6714  */
6715 struct scb *
6716 ahd_get_scb(struct ahd_softc *ahd, u_int col_idx)
6717 {
6718 	struct scb *scb;
6719 	int tries;
6720 
6721 	tries = 0;
6722 look_again:
6723 	TAILQ_FOREACH(scb, &ahd->scb_data.free_scbs, links.tqe) {
6724 		if (AHD_GET_SCB_COL_IDX(ahd, scb) != col_idx) {
6725 			ahd_rem_col_list(ahd, scb);
6726 			goto found;
6727 		}
6728 	}
6729 	if ((scb = LIST_FIRST(&ahd->scb_data.any_dev_free_scb_list)) == NULL) {
6730 
6731 		if (tries++ != 0)
6732 			return (NULL);
6733 		ahd_alloc_scbs(ahd);
6734 		goto look_again;
6735 	}
6736 	LIST_REMOVE(scb, links.le);
6737 	if (col_idx != AHD_NEVER_COL_IDX
6738 	 && (scb->col_scb != NULL)
6739 	 && (scb->col_scb->flags & SCB_ACTIVE) == 0) {
6740 		LIST_REMOVE(scb->col_scb, links.le);
6741 		ahd_add_col_list(ahd, scb->col_scb, col_idx);
6742 	}
6743 found:
6744 	scb->flags |= SCB_ACTIVE;
6745 	return (scb);
6746 }
6747 
6748 /*
6749  * Return an SCB resource to the free list.
6750  */
6751 void
6752 ahd_free_scb(struct ahd_softc *ahd, struct scb *scb)
6753 {
6754 	/* Clean up for the next user */
6755 	scb->flags = SCB_FLAG_NONE;
6756 	scb->hscb->control = 0;
6757 	ahd->scb_data.scbindex[SCB_GET_TAG(scb)] = NULL;
6758 
6759 	if (scb->col_scb == NULL) {
6760 
6761 		/*
6762 		 * No collision possible.  Just free normally.
6763 		 */
6764 		LIST_INSERT_HEAD(&ahd->scb_data.any_dev_free_scb_list,
6765 				 scb, links.le);
6766 	} else if ((scb->col_scb->flags & SCB_ON_COL_LIST) != 0) {
6767 
6768 		/*
6769 		 * The SCB we might have collided with is on
6770 		 * a free collision list.  Put both SCBs on
6771 		 * the generic list.
6772 		 */
6773 		ahd_rem_col_list(ahd, scb->col_scb);
6774 		LIST_INSERT_HEAD(&ahd->scb_data.any_dev_free_scb_list,
6775 				 scb, links.le);
6776 		LIST_INSERT_HEAD(&ahd->scb_data.any_dev_free_scb_list,
6777 				 scb->col_scb, links.le);
6778 	} else if ((scb->col_scb->flags
6779 		  & (SCB_PACKETIZED|SCB_ACTIVE)) == SCB_ACTIVE
6780 		&& (scb->col_scb->hscb->control & TAG_ENB) != 0) {
6781 
6782 		/*
6783 		 * The SCB we might collide with on the next allocation
6784 		 * is still active in a non-packetized, tagged, context.
6785 		 * Put us on the SCB collision list.
6786 		 */
6787 		ahd_add_col_list(ahd, scb,
6788 				 AHD_GET_SCB_COL_IDX(ahd, scb->col_scb));
6789 	} else {
6790 		/*
6791 		 * The SCB we might collide with on the next allocation
6792 		 * is either active in a packetized context, or free.
6793 		 * Since we can't collide, put this SCB on the generic
6794 		 * free list.
6795 		 */
6796 		LIST_INSERT_HEAD(&ahd->scb_data.any_dev_free_scb_list,
6797 				 scb, links.le);
6798 	}
6799 
6800 	ahd_platform_scb_free(ahd, scb);
6801 }
6802 
6803 static void
6804 ahd_alloc_scbs(struct ahd_softc *ahd)
6805 {
6806 	struct scb_data *scb_data;
6807 	struct scb	*next_scb;
6808 	struct hardware_scb *hscb;
6809 	struct map_node *hscb_map;
6810 	struct map_node *sg_map;
6811 	struct map_node *sense_map;
6812 	uint8_t		*segs;
6813 	uint8_t		*sense_data;
6814 	dma_addr_t	 hscb_busaddr;
6815 	dma_addr_t	 sg_busaddr;
6816 	dma_addr_t	 sense_busaddr;
6817 	int		 newcount;
6818 	int		 i;
6819 
6820 	scb_data = &ahd->scb_data;
6821 	if (scb_data->numscbs >= AHD_SCB_MAX_ALLOC)
6822 		/* Can't allocate any more */
6823 		return;
6824 
6825 	if (scb_data->scbs_left != 0) {
6826 		int offset;
6827 
6828 		offset = (PAGE_SIZE / sizeof(*hscb)) - scb_data->scbs_left;
6829 		hscb_map = SLIST_FIRST(&scb_data->hscb_maps);
6830 		hscb = &((struct hardware_scb *)hscb_map->vaddr)[offset];
6831 		hscb_busaddr = hscb_map->physaddr + (offset * sizeof(*hscb));
6832 	} else {
6833 		hscb_map = kmalloc(sizeof(*hscb_map), GFP_ATOMIC);
6834 
6835 		if (hscb_map == NULL)
6836 			return;
6837 
6838 		/* Allocate the next batch of hardware SCBs */
6839 		if (ahd_dmamem_alloc(ahd, scb_data->hscb_dmat,
6840 				     (void **)&hscb_map->vaddr,
6841 				     BUS_DMA_NOWAIT, &hscb_map->dmamap) != 0) {
6842 			kfree(hscb_map);
6843 			return;
6844 		}
6845 
6846 		SLIST_INSERT_HEAD(&scb_data->hscb_maps, hscb_map, links);
6847 
6848 		ahd_dmamap_load(ahd, scb_data->hscb_dmat, hscb_map->dmamap,
6849 				hscb_map->vaddr, PAGE_SIZE, ahd_dmamap_cb,
6850 				&hscb_map->physaddr, /*flags*/0);
6851 
6852 		hscb = (struct hardware_scb *)hscb_map->vaddr;
6853 		hscb_busaddr = hscb_map->physaddr;
6854 		scb_data->scbs_left = PAGE_SIZE / sizeof(*hscb);
6855 	}
6856 
6857 	if (scb_data->sgs_left != 0) {
6858 		int offset;
6859 
6860 		offset = ((ahd_sglist_allocsize(ahd) / ahd_sglist_size(ahd))
6861 		       - scb_data->sgs_left) * ahd_sglist_size(ahd);
6862 		sg_map = SLIST_FIRST(&scb_data->sg_maps);
6863 		segs = sg_map->vaddr + offset;
6864 		sg_busaddr = sg_map->physaddr + offset;
6865 	} else {
6866 		sg_map = kmalloc(sizeof(*sg_map), GFP_ATOMIC);
6867 
6868 		if (sg_map == NULL)
6869 			return;
6870 
6871 		/* Allocate the next batch of S/G lists */
6872 		if (ahd_dmamem_alloc(ahd, scb_data->sg_dmat,
6873 				     (void **)&sg_map->vaddr,
6874 				     BUS_DMA_NOWAIT, &sg_map->dmamap) != 0) {
6875 			kfree(sg_map);
6876 			return;
6877 		}
6878 
6879 		SLIST_INSERT_HEAD(&scb_data->sg_maps, sg_map, links);
6880 
6881 		ahd_dmamap_load(ahd, scb_data->sg_dmat, sg_map->dmamap,
6882 				sg_map->vaddr, ahd_sglist_allocsize(ahd),
6883 				ahd_dmamap_cb, &sg_map->physaddr, /*flags*/0);
6884 
6885 		segs = sg_map->vaddr;
6886 		sg_busaddr = sg_map->physaddr;
6887 		scb_data->sgs_left =
6888 		    ahd_sglist_allocsize(ahd) / ahd_sglist_size(ahd);
6889 #ifdef AHD_DEBUG
6890 		if (ahd_debug & AHD_SHOW_MEMORY)
6891 			printk("Mapped SG data\n");
6892 #endif
6893 	}
6894 
6895 	if (scb_data->sense_left != 0) {
6896 		int offset;
6897 
6898 		offset = PAGE_SIZE - (AHD_SENSE_BUFSIZE * scb_data->sense_left);
6899 		sense_map = SLIST_FIRST(&scb_data->sense_maps);
6900 		sense_data = sense_map->vaddr + offset;
6901 		sense_busaddr = sense_map->physaddr + offset;
6902 	} else {
6903 		sense_map = kmalloc(sizeof(*sense_map), GFP_ATOMIC);
6904 
6905 		if (sense_map == NULL)
6906 			return;
6907 
6908 		/* Allocate the next batch of sense buffers */
6909 		if (ahd_dmamem_alloc(ahd, scb_data->sense_dmat,
6910 				     (void **)&sense_map->vaddr,
6911 				     BUS_DMA_NOWAIT, &sense_map->dmamap) != 0) {
6912 			kfree(sense_map);
6913 			return;
6914 		}
6915 
6916 		SLIST_INSERT_HEAD(&scb_data->sense_maps, sense_map, links);
6917 
6918 		ahd_dmamap_load(ahd, scb_data->sense_dmat, sense_map->dmamap,
6919 				sense_map->vaddr, PAGE_SIZE, ahd_dmamap_cb,
6920 				&sense_map->physaddr, /*flags*/0);
6921 
6922 		sense_data = sense_map->vaddr;
6923 		sense_busaddr = sense_map->physaddr;
6924 		scb_data->sense_left = PAGE_SIZE / AHD_SENSE_BUFSIZE;
6925 #ifdef AHD_DEBUG
6926 		if (ahd_debug & AHD_SHOW_MEMORY)
6927 			printk("Mapped sense data\n");
6928 #endif
6929 	}
6930 
6931 	newcount = min(scb_data->sense_left, scb_data->scbs_left);
6932 	newcount = min(newcount, scb_data->sgs_left);
6933 	newcount = min(newcount, (AHD_SCB_MAX_ALLOC - scb_data->numscbs));
6934 	for (i = 0; i < newcount; i++) {
6935 		struct scb_platform_data *pdata;
6936 		u_int col_tag;
6937 #ifndef __linux__
6938 		int error;
6939 #endif
6940 
6941 		next_scb = kmalloc(sizeof(*next_scb), GFP_ATOMIC);
6942 		if (next_scb == NULL)
6943 			break;
6944 
6945 		pdata = kmalloc(sizeof(*pdata), GFP_ATOMIC);
6946 		if (pdata == NULL) {
6947 			kfree(next_scb);
6948 			break;
6949 		}
6950 		next_scb->platform_data = pdata;
6951 		next_scb->hscb_map = hscb_map;
6952 		next_scb->sg_map = sg_map;
6953 		next_scb->sense_map = sense_map;
6954 		next_scb->sg_list = segs;
6955 		next_scb->sense_data = sense_data;
6956 		next_scb->sense_busaddr = sense_busaddr;
6957 		memset(hscb, 0, sizeof(*hscb));
6958 		next_scb->hscb = hscb;
6959 		hscb->hscb_busaddr = ahd_htole32(hscb_busaddr);
6960 
6961 		/*
6962 		 * The sequencer always starts with the second entry.
6963 		 * The first entry is embedded in the scb.
6964 		 */
6965 		next_scb->sg_list_busaddr = sg_busaddr;
6966 		if ((ahd->flags & AHD_64BIT_ADDRESSING) != 0)
6967 			next_scb->sg_list_busaddr
6968 			    += sizeof(struct ahd_dma64_seg);
6969 		else
6970 			next_scb->sg_list_busaddr += sizeof(struct ahd_dma_seg);
6971 		next_scb->ahd_softc = ahd;
6972 		next_scb->flags = SCB_FLAG_NONE;
6973 #ifndef __linux__
6974 		error = ahd_dmamap_create(ahd, ahd->buffer_dmat, /*flags*/0,
6975 					  &next_scb->dmamap);
6976 		if (error != 0) {
6977 			kfree(next_scb);
6978 			kfree(pdata);
6979 			break;
6980 		}
6981 #endif
6982 		next_scb->hscb->tag = ahd_htole16(scb_data->numscbs);
6983 		col_tag = scb_data->numscbs ^ 0x100;
6984 		next_scb->col_scb = ahd_find_scb_by_tag(ahd, col_tag);
6985 		if (next_scb->col_scb != NULL)
6986 			next_scb->col_scb->col_scb = next_scb;
6987 		ahd_free_scb(ahd, next_scb);
6988 		hscb++;
6989 		hscb_busaddr += sizeof(*hscb);
6990 		segs += ahd_sglist_size(ahd);
6991 		sg_busaddr += ahd_sglist_size(ahd);
6992 		sense_data += AHD_SENSE_BUFSIZE;
6993 		sense_busaddr += AHD_SENSE_BUFSIZE;
6994 		scb_data->numscbs++;
6995 		scb_data->sense_left--;
6996 		scb_data->scbs_left--;
6997 		scb_data->sgs_left--;
6998 	}
6999 }
7000 
7001 void
7002 ahd_controller_info(struct ahd_softc *ahd, char *buf)
7003 {
7004 	const char *speed;
7005 	const char *type;
7006 	int len;
7007 
7008 	len = sprintf(buf, "%s: ", ahd_chip_names[ahd->chip & AHD_CHIPID_MASK]);
7009 	buf += len;
7010 
7011 	speed = "Ultra320 ";
7012 	if ((ahd->features & AHD_WIDE) != 0) {
7013 		type = "Wide ";
7014 	} else {
7015 		type = "Single ";
7016 	}
7017 	len = sprintf(buf, "%s%sChannel %c, SCSI Id=%d, ",
7018 		      speed, type, ahd->channel, ahd->our_id);
7019 	buf += len;
7020 
7021 	sprintf(buf, "%s, %d SCBs", ahd->bus_description,
7022 		ahd->scb_data.maxhscbs);
7023 }
7024 
7025 static const char *channel_strings[] = {
7026 	"Primary Low",
7027 	"Primary High",
7028 	"Secondary Low",
7029 	"Secondary High"
7030 };
7031 
7032 static const char *termstat_strings[] = {
7033 	"Terminated Correctly",
7034 	"Over Terminated",
7035 	"Under Terminated",
7036 	"Not Configured"
7037 };
7038 
7039 /***************************** Timer Facilities *******************************/
7040 static void
7041 ahd_timer_reset(struct timer_list *timer, int usec)
7042 {
7043 	del_timer(timer);
7044 	timer->expires = jiffies + (usec * HZ)/1000000;
7045 	add_timer(timer);
7046 }
7047 
7048 /*
7049  * Start the board, ready for normal operation
7050  */
7051 int
7052 ahd_init(struct ahd_softc *ahd)
7053 {
7054 	uint8_t		*next_vaddr;
7055 	dma_addr_t	 next_baddr;
7056 	size_t		 driver_data_size;
7057 	int		 i;
7058 	int		 error;
7059 	u_int		 warn_user;
7060 	uint8_t		 current_sensing;
7061 	uint8_t		 fstat;
7062 
7063 	AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
7064 
7065 	ahd->stack_size = ahd_probe_stack_size(ahd);
7066 	ahd->saved_stack = kmalloc(ahd->stack_size * sizeof(uint16_t), GFP_ATOMIC);
7067 	if (ahd->saved_stack == NULL)
7068 		return (ENOMEM);
7069 
7070 	/*
7071 	 * Verify that the compiler hasn't over-aggressively
7072 	 * padded important structures.
7073 	 */
7074 	if (sizeof(struct hardware_scb) != 64)
7075 		panic("Hardware SCB size is incorrect");
7076 
7077 #ifdef AHD_DEBUG
7078 	if ((ahd_debug & AHD_DEBUG_SEQUENCER) != 0)
7079 		ahd->flags |= AHD_SEQUENCER_DEBUG;
7080 #endif
7081 
7082 	/*
7083 	 * Default to allowing initiator operations.
7084 	 */
7085 	ahd->flags |= AHD_INITIATORROLE;
7086 
7087 	/*
7088 	 * Only allow target mode features if this unit has them enabled.
7089 	 */
7090 	if ((AHD_TMODE_ENABLE & (0x1 << ahd->unit)) == 0)
7091 		ahd->features &= ~AHD_TARGETMODE;
7092 
7093 #ifndef __linux__
7094 	/* DMA tag for mapping buffers into device visible space. */
7095 	if (ahd_dma_tag_create(ahd, ahd->parent_dmat, /*alignment*/1,
7096 			       /*boundary*/BUS_SPACE_MAXADDR_32BIT + 1,
7097 			       /*lowaddr*/ahd->flags & AHD_39BIT_ADDRESSING
7098 					? (dma_addr_t)0x7FFFFFFFFFULL
7099 					: BUS_SPACE_MAXADDR_32BIT,
7100 			       /*highaddr*/BUS_SPACE_MAXADDR,
7101 			       /*filter*/NULL, /*filterarg*/NULL,
7102 			       /*maxsize*/(AHD_NSEG - 1) * PAGE_SIZE,
7103 			       /*nsegments*/AHD_NSEG,
7104 			       /*maxsegsz*/AHD_MAXTRANSFER_SIZE,
7105 			       /*flags*/BUS_DMA_ALLOCNOW,
7106 			       &ahd->buffer_dmat) != 0) {
7107 		return (ENOMEM);
7108 	}
7109 #endif
7110 
7111 	ahd->init_level++;
7112 
7113 	/*
7114 	 * DMA tag for our command fifos and other data in system memory
7115 	 * the card's sequencer must be able to access.  For initiator
7116 	 * roles, we need to allocate space for the qoutfifo.  When providing
7117 	 * for the target mode role, we must additionally provide space for
7118 	 * the incoming target command fifo.
7119 	 */
7120 	driver_data_size = AHD_SCB_MAX * sizeof(*ahd->qoutfifo)
7121 			 + sizeof(struct hardware_scb);
7122 	if ((ahd->features & AHD_TARGETMODE) != 0)
7123 		driver_data_size += AHD_TMODE_CMDS * sizeof(struct target_cmd);
7124 	if ((ahd->bugs & AHD_PKT_BITBUCKET_BUG) != 0)
7125 		driver_data_size += PKT_OVERRUN_BUFSIZE;
7126 	if (ahd_dma_tag_create(ahd, ahd->parent_dmat, /*alignment*/1,
7127 			       /*boundary*/BUS_SPACE_MAXADDR_32BIT + 1,
7128 			       /*lowaddr*/BUS_SPACE_MAXADDR_32BIT,
7129 			       /*highaddr*/BUS_SPACE_MAXADDR,
7130 			       /*filter*/NULL, /*filterarg*/NULL,
7131 			       driver_data_size,
7132 			       /*nsegments*/1,
7133 			       /*maxsegsz*/BUS_SPACE_MAXSIZE_32BIT,
7134 			       /*flags*/0, &ahd->shared_data_dmat) != 0) {
7135 		return (ENOMEM);
7136 	}
7137 
7138 	ahd->init_level++;
7139 
7140 	/* Allocation of driver data */
7141 	if (ahd_dmamem_alloc(ahd, ahd->shared_data_dmat,
7142 			     (void **)&ahd->shared_data_map.vaddr,
7143 			     BUS_DMA_NOWAIT,
7144 			     &ahd->shared_data_map.dmamap) != 0) {
7145 		return (ENOMEM);
7146 	}
7147 
7148 	ahd->init_level++;
7149 
7150 	/* And permanently map it in */
7151 	ahd_dmamap_load(ahd, ahd->shared_data_dmat, ahd->shared_data_map.dmamap,
7152 			ahd->shared_data_map.vaddr, driver_data_size,
7153 			ahd_dmamap_cb, &ahd->shared_data_map.physaddr,
7154 			/*flags*/0);
7155 	ahd->qoutfifo = (struct ahd_completion *)ahd->shared_data_map.vaddr;
7156 	next_vaddr = (uint8_t *)&ahd->qoutfifo[AHD_QOUT_SIZE];
7157 	next_baddr = ahd->shared_data_map.physaddr
7158 		   + AHD_QOUT_SIZE*sizeof(struct ahd_completion);
7159 	if ((ahd->features & AHD_TARGETMODE) != 0) {
7160 		ahd->targetcmds = (struct target_cmd *)next_vaddr;
7161 		next_vaddr += AHD_TMODE_CMDS * sizeof(struct target_cmd);
7162 		next_baddr += AHD_TMODE_CMDS * sizeof(struct target_cmd);
7163 	}
7164 
7165 	if ((ahd->bugs & AHD_PKT_BITBUCKET_BUG) != 0) {
7166 		ahd->overrun_buf = next_vaddr;
7167 		next_vaddr += PKT_OVERRUN_BUFSIZE;
7168 		next_baddr += PKT_OVERRUN_BUFSIZE;
7169 	}
7170 
7171 	/*
7172 	 * We need one SCB to serve as the "next SCB".  Since the
7173 	 * tag identifier in this SCB will never be used, there is
7174 	 * no point in using a valid HSCB tag from an SCB pulled from
7175 	 * the standard free pool.  So, we allocate this "sentinel"
7176 	 * specially from the DMA safe memory chunk used for the QOUTFIFO.
7177 	 */
7178 	ahd->next_queued_hscb = (struct hardware_scb *)next_vaddr;
7179 	ahd->next_queued_hscb_map = &ahd->shared_data_map;
7180 	ahd->next_queued_hscb->hscb_busaddr = ahd_htole32(next_baddr);
7181 
7182 	ahd->init_level++;
7183 
7184 	/* Allocate SCB data now that buffer_dmat is initialized */
7185 	if (ahd_init_scbdata(ahd) != 0)
7186 		return (ENOMEM);
7187 
7188 	if ((ahd->flags & AHD_INITIATORROLE) == 0)
7189 		ahd->flags &= ~AHD_RESET_BUS_A;
7190 
7191 	/*
7192 	 * Before committing these settings to the chip, give
7193 	 * the OSM one last chance to modify our configuration.
7194 	 */
7195 	ahd_platform_init(ahd);
7196 
7197 	/* Bring up the chip. */
7198 	ahd_chip_init(ahd);
7199 
7200 	AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
7201 
7202 	if ((ahd->flags & AHD_CURRENT_SENSING) == 0)
7203 		goto init_done;
7204 
7205 	/*
7206 	 * Verify termination based on current draw and
7207 	 * warn user if the bus is over/under terminated.
7208 	 */
7209 	error = ahd_write_flexport(ahd, FLXADDR_ROMSTAT_CURSENSECTL,
7210 				   CURSENSE_ENB);
7211 	if (error != 0) {
7212 		printk("%s: current sensing timeout 1\n", ahd_name(ahd));
7213 		goto init_done;
7214 	}
7215 	for (i = 20, fstat = FLX_FSTAT_BUSY;
7216 	     (fstat & FLX_FSTAT_BUSY) != 0 && i; i--) {
7217 		error = ahd_read_flexport(ahd, FLXADDR_FLEXSTAT, &fstat);
7218 		if (error != 0) {
7219 			printk("%s: current sensing timeout 2\n",
7220 			       ahd_name(ahd));
7221 			goto init_done;
7222 		}
7223 	}
7224 	if (i == 0) {
7225 		printk("%s: Timedout during current-sensing test\n",
7226 		       ahd_name(ahd));
7227 		goto init_done;
7228 	}
7229 
7230 	/* Latch Current Sensing status. */
7231 	error = ahd_read_flexport(ahd, FLXADDR_CURRENT_STAT, &current_sensing);
7232 	if (error != 0) {
7233 		printk("%s: current sensing timeout 3\n", ahd_name(ahd));
7234 		goto init_done;
7235 	}
7236 
7237 	/* Diable current sensing. */
7238 	ahd_write_flexport(ahd, FLXADDR_ROMSTAT_CURSENSECTL, 0);
7239 
7240 #ifdef AHD_DEBUG
7241 	if ((ahd_debug & AHD_SHOW_TERMCTL) != 0) {
7242 		printk("%s: current_sensing == 0x%x\n",
7243 		       ahd_name(ahd), current_sensing);
7244 	}
7245 #endif
7246 	warn_user = 0;
7247 	for (i = 0; i < 4; i++, current_sensing >>= FLX_CSTAT_SHIFT) {
7248 		u_int term_stat;
7249 
7250 		term_stat = (current_sensing & FLX_CSTAT_MASK);
7251 		switch (term_stat) {
7252 		case FLX_CSTAT_OVER:
7253 		case FLX_CSTAT_UNDER:
7254 			warn_user++;
7255 		case FLX_CSTAT_INVALID:
7256 		case FLX_CSTAT_OKAY:
7257 			if (warn_user == 0 && bootverbose == 0)
7258 				break;
7259 			printk("%s: %s Channel %s\n", ahd_name(ahd),
7260 			       channel_strings[i], termstat_strings[term_stat]);
7261 			break;
7262 		}
7263 	}
7264 	if (warn_user) {
7265 		printk("%s: WARNING. Termination is not configured correctly.\n"
7266 		       "%s: WARNING. SCSI bus operations may FAIL.\n",
7267 		       ahd_name(ahd), ahd_name(ahd));
7268 	}
7269 init_done:
7270 	ahd_restart(ahd);
7271 	ahd_timer_reset(&ahd->stat_timer, AHD_STAT_UPDATE_US);
7272 	return (0);
7273 }
7274 
7275 /*
7276  * (Re)initialize chip state after a chip reset.
7277  */
7278 static void
7279 ahd_chip_init(struct ahd_softc *ahd)
7280 {
7281 	uint32_t busaddr;
7282 	u_int	 sxfrctl1;
7283 	u_int	 scsiseq_template;
7284 	u_int	 wait;
7285 	u_int	 i;
7286 	u_int	 target;
7287 
7288 	ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
7289 	/*
7290 	 * Take the LED out of diagnostic mode
7291 	 */
7292 	ahd_outb(ahd, SBLKCTL, ahd_inb(ahd, SBLKCTL) & ~(DIAGLEDEN|DIAGLEDON));
7293 
7294 	/*
7295 	 * Return HS_MAILBOX to its default value.
7296 	 */
7297 	ahd->hs_mailbox = 0;
7298 	ahd_outb(ahd, HS_MAILBOX, 0);
7299 
7300 	/* Set the SCSI Id, SXFRCTL0, SXFRCTL1, and SIMODE1. */
7301 	ahd_outb(ahd, IOWNID, ahd->our_id);
7302 	ahd_outb(ahd, TOWNID, ahd->our_id);
7303 	sxfrctl1 = (ahd->flags & AHD_TERM_ENB_A) != 0 ? STPWEN : 0;
7304 	sxfrctl1 |= (ahd->flags & AHD_SPCHK_ENB_A) != 0 ? ENSPCHK : 0;
7305 	if ((ahd->bugs & AHD_LONG_SETIMO_BUG)
7306 	 && (ahd->seltime != STIMESEL_MIN)) {
7307 		/*
7308 		 * The selection timer duration is twice as long
7309 		 * as it should be.  Halve it by adding "1" to
7310 		 * the user specified setting.
7311 		 */
7312 		sxfrctl1 |= ahd->seltime + STIMESEL_BUG_ADJ;
7313 	} else {
7314 		sxfrctl1 |= ahd->seltime;
7315 	}
7316 
7317 	ahd_outb(ahd, SXFRCTL0, DFON);
7318 	ahd_outb(ahd, SXFRCTL1, sxfrctl1|ahd->seltime|ENSTIMER|ACTNEGEN);
7319 	ahd_outb(ahd, SIMODE1, ENSELTIMO|ENSCSIRST|ENSCSIPERR);
7320 
7321 	/*
7322 	 * Now that termination is set, wait for up
7323 	 * to 500ms for our transceivers to settle.  If
7324 	 * the adapter does not have a cable attached,
7325 	 * the transceivers may never settle, so don't
7326 	 * complain if we fail here.
7327 	 */
7328 	for (wait = 10000;
7329 	     (ahd_inb(ahd, SBLKCTL) & (ENAB40|ENAB20)) == 0 && wait;
7330 	     wait--)
7331 		ahd_delay(100);
7332 
7333 	/* Clear any false bus resets due to the transceivers settling */
7334 	ahd_outb(ahd, CLRSINT1, CLRSCSIRSTI);
7335 	ahd_outb(ahd, CLRINT, CLRSCSIINT);
7336 
7337 	/* Initialize mode specific S/G state. */
7338 	for (i = 0; i < 2; i++) {
7339 		ahd_set_modes(ahd, AHD_MODE_DFF0 + i, AHD_MODE_DFF0 + i);
7340 		ahd_outb(ahd, LONGJMP_ADDR + 1, INVALID_ADDR);
7341 		ahd_outb(ahd, SG_STATE, 0);
7342 		ahd_outb(ahd, CLRSEQINTSRC, 0xFF);
7343 		ahd_outb(ahd, SEQIMODE,
7344 			 ENSAVEPTRS|ENCFG4DATA|ENCFG4ISTAT
7345 			|ENCFG4TSTAT|ENCFG4ICMD|ENCFG4TCMD);
7346 	}
7347 
7348 	ahd_set_modes(ahd, AHD_MODE_CFG, AHD_MODE_CFG);
7349 	ahd_outb(ahd, DSCOMMAND0, ahd_inb(ahd, DSCOMMAND0)|MPARCKEN|CACHETHEN);
7350 	ahd_outb(ahd, DFF_THRSH, RD_DFTHRSH_75|WR_DFTHRSH_75);
7351 	ahd_outb(ahd, SIMODE0, ENIOERR|ENOVERRUN);
7352 	ahd_outb(ahd, SIMODE3, ENNTRAMPERR|ENOSRAMPERR);
7353 	if ((ahd->bugs & AHD_BUSFREEREV_BUG) != 0) {
7354 		ahd_outb(ahd, OPTIONMODE, AUTOACKEN|AUTO_MSGOUT_DE);
7355 	} else {
7356 		ahd_outb(ahd, OPTIONMODE, AUTOACKEN|BUSFREEREV|AUTO_MSGOUT_DE);
7357 	}
7358 	ahd_outb(ahd, SCSCHKN, CURRFIFODEF|WIDERESEN|SHVALIDSTDIS);
7359 	if ((ahd->chip & AHD_BUS_MASK) == AHD_PCIX)
7360 		/*
7361 		 * Do not issue a target abort when a split completion
7362 		 * error occurs.  Let our PCIX interrupt handler deal
7363 		 * with it instead. H2A4 Razor #625
7364 		 */
7365 		ahd_outb(ahd, PCIXCTL, ahd_inb(ahd, PCIXCTL) | SPLTSTADIS);
7366 
7367 	if ((ahd->bugs & AHD_LQOOVERRUN_BUG) != 0)
7368 		ahd_outb(ahd, LQOSCSCTL, LQONOCHKOVER);
7369 
7370 	/*
7371 	 * Tweak IOCELL settings.
7372 	 */
7373 	if ((ahd->flags & AHD_HP_BOARD) != 0) {
7374 		for (i = 0; i < NUMDSPS; i++) {
7375 			ahd_outb(ahd, DSPSELECT, i);
7376 			ahd_outb(ahd, WRTBIASCTL, WRTBIASCTL_HP_DEFAULT);
7377 		}
7378 #ifdef AHD_DEBUG
7379 		if ((ahd_debug & AHD_SHOW_MISC) != 0)
7380 			printk("%s: WRTBIASCTL now 0x%x\n", ahd_name(ahd),
7381 			       WRTBIASCTL_HP_DEFAULT);
7382 #endif
7383 	}
7384 	ahd_setup_iocell_workaround(ahd);
7385 
7386 	/*
7387 	 * Enable LQI Manager interrupts.
7388 	 */
7389 	ahd_outb(ahd, LQIMODE1, ENLQIPHASE_LQ|ENLQIPHASE_NLQ|ENLIQABORT
7390 			      | ENLQICRCI_LQ|ENLQICRCI_NLQ|ENLQIBADLQI
7391 			      | ENLQIOVERI_LQ|ENLQIOVERI_NLQ);
7392 	ahd_outb(ahd, LQOMODE0, ENLQOATNLQ|ENLQOATNPKT|ENLQOTCRC);
7393 	/*
7394 	 * We choose to have the sequencer catch LQOPHCHGINPKT errors
7395 	 * manually for the command phase at the start of a packetized
7396 	 * selection case.  ENLQOBUSFREE should be made redundant by
7397 	 * the BUSFREE interrupt, but it seems that some LQOBUSFREE
7398 	 * events fail to assert the BUSFREE interrupt so we must
7399 	 * also enable LQOBUSFREE interrupts.
7400 	 */
7401 	ahd_outb(ahd, LQOMODE1, ENLQOBUSFREE);
7402 
7403 	/*
7404 	 * Setup sequencer interrupt handlers.
7405 	 */
7406 	ahd_outw(ahd, INTVEC1_ADDR, ahd_resolve_seqaddr(ahd, LABEL_seq_isr));
7407 	ahd_outw(ahd, INTVEC2_ADDR, ahd_resolve_seqaddr(ahd, LABEL_timer_isr));
7408 
7409 	/*
7410 	 * Setup SCB Offset registers.
7411 	 */
7412 	if ((ahd->bugs & AHD_PKT_LUN_BUG) != 0) {
7413 		ahd_outb(ahd, LUNPTR, offsetof(struct hardware_scb,
7414 			 pkt_long_lun));
7415 	} else {
7416 		ahd_outb(ahd, LUNPTR, offsetof(struct hardware_scb, lun));
7417 	}
7418 	ahd_outb(ahd, CMDLENPTR, offsetof(struct hardware_scb, cdb_len));
7419 	ahd_outb(ahd, ATTRPTR, offsetof(struct hardware_scb, task_attribute));
7420 	ahd_outb(ahd, FLAGPTR, offsetof(struct hardware_scb, task_management));
7421 	ahd_outb(ahd, CMDPTR, offsetof(struct hardware_scb,
7422 				       shared_data.idata.cdb));
7423 	ahd_outb(ahd, QNEXTPTR,
7424 		 offsetof(struct hardware_scb, next_hscb_busaddr));
7425 	ahd_outb(ahd, ABRTBITPTR, MK_MESSAGE_BIT_OFFSET);
7426 	ahd_outb(ahd, ABRTBYTEPTR, offsetof(struct hardware_scb, control));
7427 	if ((ahd->bugs & AHD_PKT_LUN_BUG) != 0) {
7428 		ahd_outb(ahd, LUNLEN,
7429 			 sizeof(ahd->next_queued_hscb->pkt_long_lun) - 1);
7430 	} else {
7431 		ahd_outb(ahd, LUNLEN, LUNLEN_SINGLE_LEVEL_LUN);
7432 	}
7433 	ahd_outb(ahd, CDBLIMIT, SCB_CDB_LEN_PTR - 1);
7434 	ahd_outb(ahd, MAXCMD, 0xFF);
7435 	ahd_outb(ahd, SCBAUTOPTR,
7436 		 AUSCBPTR_EN | offsetof(struct hardware_scb, tag));
7437 
7438 	/* We haven't been enabled for target mode yet. */
7439 	ahd_outb(ahd, MULTARGID, 0);
7440 	ahd_outb(ahd, MULTARGID + 1, 0);
7441 
7442 	ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
7443 	/* Initialize the negotiation table. */
7444 	if ((ahd->features & AHD_NEW_IOCELL_OPTS) == 0) {
7445 		/*
7446 		 * Clear the spare bytes in the neg table to avoid
7447 		 * spurious parity errors.
7448 		 */
7449 		for (target = 0; target < AHD_NUM_TARGETS; target++) {
7450 			ahd_outb(ahd, NEGOADDR, target);
7451 			ahd_outb(ahd, ANNEXCOL, AHD_ANNEXCOL_PER_DEV0);
7452 			for (i = 0; i < AHD_NUM_PER_DEV_ANNEXCOLS; i++)
7453 				ahd_outb(ahd, ANNEXDAT, 0);
7454 		}
7455 	}
7456 	for (target = 0; target < AHD_NUM_TARGETS; target++) {
7457 		struct	 ahd_devinfo devinfo;
7458 		struct	 ahd_initiator_tinfo *tinfo;
7459 		struct	 ahd_tmode_tstate *tstate;
7460 
7461 		tinfo = ahd_fetch_transinfo(ahd, 'A', ahd->our_id,
7462 					    target, &tstate);
7463 		ahd_compile_devinfo(&devinfo, ahd->our_id,
7464 				    target, CAM_LUN_WILDCARD,
7465 				    'A', ROLE_INITIATOR);
7466 		ahd_update_neg_table(ahd, &devinfo, &tinfo->curr);
7467 	}
7468 
7469 	ahd_outb(ahd, CLRSINT3, NTRAMPERR|OSRAMPERR);
7470 	ahd_outb(ahd, CLRINT, CLRSCSIINT);
7471 
7472 #ifdef NEEDS_MORE_TESTING
7473 	/*
7474 	 * Always enable abort on incoming L_Qs if this feature is
7475 	 * supported.  We use this to catch invalid SCB references.
7476 	 */
7477 	if ((ahd->bugs & AHD_ABORT_LQI_BUG) == 0)
7478 		ahd_outb(ahd, LQCTL1, ABORTPENDING);
7479 	else
7480 #endif
7481 		ahd_outb(ahd, LQCTL1, 0);
7482 
7483 	/* All of our queues are empty */
7484 	ahd->qoutfifonext = 0;
7485 	ahd->qoutfifonext_valid_tag = QOUTFIFO_ENTRY_VALID;
7486 	ahd_outb(ahd, QOUTFIFO_ENTRY_VALID_TAG, QOUTFIFO_ENTRY_VALID);
7487 	for (i = 0; i < AHD_QOUT_SIZE; i++)
7488 		ahd->qoutfifo[i].valid_tag = 0;
7489 	ahd_sync_qoutfifo(ahd, BUS_DMASYNC_PREREAD);
7490 
7491 	ahd->qinfifonext = 0;
7492 	for (i = 0; i < AHD_QIN_SIZE; i++)
7493 		ahd->qinfifo[i] = SCB_LIST_NULL;
7494 
7495 	if ((ahd->features & AHD_TARGETMODE) != 0) {
7496 		/* All target command blocks start out invalid. */
7497 		for (i = 0; i < AHD_TMODE_CMDS; i++)
7498 			ahd->targetcmds[i].cmd_valid = 0;
7499 		ahd_sync_tqinfifo(ahd, BUS_DMASYNC_PREREAD);
7500 		ahd->tqinfifonext = 1;
7501 		ahd_outb(ahd, KERNEL_TQINPOS, ahd->tqinfifonext - 1);
7502 		ahd_outb(ahd, TQINPOS, ahd->tqinfifonext);
7503 	}
7504 
7505 	/* Initialize Scratch Ram. */
7506 	ahd_outb(ahd, SEQ_FLAGS, 0);
7507 	ahd_outb(ahd, SEQ_FLAGS2, 0);
7508 
7509 	/* We don't have any waiting selections */
7510 	ahd_outw(ahd, WAITING_TID_HEAD, SCB_LIST_NULL);
7511 	ahd_outw(ahd, WAITING_TID_TAIL, SCB_LIST_NULL);
7512 	ahd_outw(ahd, MK_MESSAGE_SCB, SCB_LIST_NULL);
7513 	ahd_outw(ahd, MK_MESSAGE_SCSIID, 0xFF);
7514 	for (i = 0; i < AHD_NUM_TARGETS; i++)
7515 		ahd_outw(ahd, WAITING_SCB_TAILS + (2 * i), SCB_LIST_NULL);
7516 
7517 	/*
7518 	 * Nobody is waiting to be DMAed into the QOUTFIFO.
7519 	 */
7520 	ahd_outw(ahd, COMPLETE_SCB_HEAD, SCB_LIST_NULL);
7521 	ahd_outw(ahd, COMPLETE_SCB_DMAINPROG_HEAD, SCB_LIST_NULL);
7522 	ahd_outw(ahd, COMPLETE_DMA_SCB_HEAD, SCB_LIST_NULL);
7523 	ahd_outw(ahd, COMPLETE_DMA_SCB_TAIL, SCB_LIST_NULL);
7524 	ahd_outw(ahd, COMPLETE_ON_QFREEZE_HEAD, SCB_LIST_NULL);
7525 
7526 	/*
7527 	 * The Freeze Count is 0.
7528 	 */
7529 	ahd->qfreeze_cnt = 0;
7530 	ahd_outw(ahd, QFREEZE_COUNT, 0);
7531 	ahd_outw(ahd, KERNEL_QFREEZE_COUNT, 0);
7532 
7533 	/*
7534 	 * Tell the sequencer where it can find our arrays in memory.
7535 	 */
7536 	busaddr = ahd->shared_data_map.physaddr;
7537 	ahd_outl(ahd, SHARED_DATA_ADDR, busaddr);
7538 	ahd_outl(ahd, QOUTFIFO_NEXT_ADDR, busaddr);
7539 
7540 	/*
7541 	 * Setup the allowed SCSI Sequences based on operational mode.
7542 	 * If we are a target, we'll enable select in operations once
7543 	 * we've had a lun enabled.
7544 	 */
7545 	scsiseq_template = ENAUTOATNP;
7546 	if ((ahd->flags & AHD_INITIATORROLE) != 0)
7547 		scsiseq_template |= ENRSELI;
7548 	ahd_outb(ahd, SCSISEQ_TEMPLATE, scsiseq_template);
7549 
7550 	/* There are no busy SCBs yet. */
7551 	for (target = 0; target < AHD_NUM_TARGETS; target++) {
7552 		int lun;
7553 
7554 		for (lun = 0; lun < AHD_NUM_LUNS_NONPKT; lun++)
7555 			ahd_unbusy_tcl(ahd, BUILD_TCL_RAW(target, 'A', lun));
7556 	}
7557 
7558 	/*
7559 	 * Initialize the group code to command length table.
7560 	 * Vendor Unique codes are set to 0 so we only capture
7561 	 * the first byte of the cdb.  These can be overridden
7562 	 * when target mode is enabled.
7563 	 */
7564 	ahd_outb(ahd, CMDSIZE_TABLE, 5);
7565 	ahd_outb(ahd, CMDSIZE_TABLE + 1, 9);
7566 	ahd_outb(ahd, CMDSIZE_TABLE + 2, 9);
7567 	ahd_outb(ahd, CMDSIZE_TABLE + 3, 0);
7568 	ahd_outb(ahd, CMDSIZE_TABLE + 4, 15);
7569 	ahd_outb(ahd, CMDSIZE_TABLE + 5, 11);
7570 	ahd_outb(ahd, CMDSIZE_TABLE + 6, 0);
7571 	ahd_outb(ahd, CMDSIZE_TABLE + 7, 0);
7572 
7573 	/* Tell the sequencer of our initial queue positions */
7574 	ahd_set_modes(ahd, AHD_MODE_CCHAN, AHD_MODE_CCHAN);
7575 	ahd_outb(ahd, QOFF_CTLSTA, SCB_QSIZE_512);
7576 	ahd->qinfifonext = 0;
7577 	ahd_set_hnscb_qoff(ahd, ahd->qinfifonext);
7578 	ahd_set_hescb_qoff(ahd, 0);
7579 	ahd_set_snscb_qoff(ahd, 0);
7580 	ahd_set_sescb_qoff(ahd, 0);
7581 	ahd_set_sdscb_qoff(ahd, 0);
7582 
7583 	/*
7584 	 * Tell the sequencer which SCB will be the next one it receives.
7585 	 */
7586 	busaddr = ahd_le32toh(ahd->next_queued_hscb->hscb_busaddr);
7587 	ahd_outl(ahd, NEXT_QUEUED_SCB_ADDR, busaddr);
7588 
7589 	/*
7590 	 * Default to coalescing disabled.
7591 	 */
7592 	ahd_outw(ahd, INT_COALESCING_CMDCOUNT, 0);
7593 	ahd_outw(ahd, CMDS_PENDING, 0);
7594 	ahd_update_coalescing_values(ahd, ahd->int_coalescing_timer,
7595 				     ahd->int_coalescing_maxcmds,
7596 				     ahd->int_coalescing_mincmds);
7597 	ahd_enable_coalescing(ahd, FALSE);
7598 
7599 	ahd_loadseq(ahd);
7600 	ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
7601 
7602 	if (ahd->features & AHD_AIC79XXB_SLOWCRC) {
7603 		u_int negodat3 = ahd_inb(ahd, NEGCONOPTS);
7604 
7605 		negodat3 |= ENSLOWCRC;
7606 		ahd_outb(ahd, NEGCONOPTS, negodat3);
7607 		negodat3 = ahd_inb(ahd, NEGCONOPTS);
7608 		if (!(negodat3 & ENSLOWCRC))
7609 			printk("aic79xx: failed to set the SLOWCRC bit\n");
7610 		else
7611 			printk("aic79xx: SLOWCRC bit set\n");
7612 	}
7613 }
7614 
7615 /*
7616  * Setup default device and controller settings.
7617  * This should only be called if our probe has
7618  * determined that no configuration data is available.
7619  */
7620 int
7621 ahd_default_config(struct ahd_softc *ahd)
7622 {
7623 	int	targ;
7624 
7625 	ahd->our_id = 7;
7626 
7627 	/*
7628 	 * Allocate a tstate to house information for our
7629 	 * initiator presence on the bus as well as the user
7630 	 * data for any target mode initiator.
7631 	 */
7632 	if (ahd_alloc_tstate(ahd, ahd->our_id, 'A') == NULL) {
7633 		printk("%s: unable to allocate ahd_tmode_tstate.  "
7634 		       "Failing attach\n", ahd_name(ahd));
7635 		return (ENOMEM);
7636 	}
7637 
7638 	for (targ = 0; targ < AHD_NUM_TARGETS; targ++) {
7639 		struct	 ahd_devinfo devinfo;
7640 		struct	 ahd_initiator_tinfo *tinfo;
7641 		struct	 ahd_tmode_tstate *tstate;
7642 		uint16_t target_mask;
7643 
7644 		tinfo = ahd_fetch_transinfo(ahd, 'A', ahd->our_id,
7645 					    targ, &tstate);
7646 		/*
7647 		 * We support SPC2 and SPI4.
7648 		 */
7649 		tinfo->user.protocol_version = 4;
7650 		tinfo->user.transport_version = 4;
7651 
7652 		target_mask = 0x01 << targ;
7653 		ahd->user_discenable |= target_mask;
7654 		tstate->discenable |= target_mask;
7655 		ahd->user_tagenable |= target_mask;
7656 #ifdef AHD_FORCE_160
7657 		tinfo->user.period = AHD_SYNCRATE_DT;
7658 #else
7659 		tinfo->user.period = AHD_SYNCRATE_160;
7660 #endif
7661 		tinfo->user.offset = MAX_OFFSET;
7662 		tinfo->user.ppr_options = MSG_EXT_PPR_RD_STRM
7663 					| MSG_EXT_PPR_WR_FLOW
7664 					| MSG_EXT_PPR_HOLD_MCS
7665 					| MSG_EXT_PPR_IU_REQ
7666 					| MSG_EXT_PPR_QAS_REQ
7667 					| MSG_EXT_PPR_DT_REQ;
7668 		if ((ahd->features & AHD_RTI) != 0)
7669 			tinfo->user.ppr_options |= MSG_EXT_PPR_RTI;
7670 
7671 		tinfo->user.width = MSG_EXT_WDTR_BUS_16_BIT;
7672 
7673 		/*
7674 		 * Start out Async/Narrow/Untagged and with
7675 		 * conservative protocol support.
7676 		 */
7677 		tinfo->goal.protocol_version = 2;
7678 		tinfo->goal.transport_version = 2;
7679 		tinfo->curr.protocol_version = 2;
7680 		tinfo->curr.transport_version = 2;
7681 		ahd_compile_devinfo(&devinfo, ahd->our_id,
7682 				    targ, CAM_LUN_WILDCARD,
7683 				    'A', ROLE_INITIATOR);
7684 		tstate->tagenable &= ~target_mask;
7685 		ahd_set_width(ahd, &devinfo, MSG_EXT_WDTR_BUS_8_BIT,
7686 			      AHD_TRANS_CUR|AHD_TRANS_GOAL, /*paused*/TRUE);
7687 		ahd_set_syncrate(ahd, &devinfo, /*period*/0, /*offset*/0,
7688 				 /*ppr_options*/0, AHD_TRANS_CUR|AHD_TRANS_GOAL,
7689 				 /*paused*/TRUE);
7690 	}
7691 	return (0);
7692 }
7693 
7694 /*
7695  * Parse device configuration information.
7696  */
7697 int
7698 ahd_parse_cfgdata(struct ahd_softc *ahd, struct seeprom_config *sc)
7699 {
7700 	int targ;
7701 	int max_targ;
7702 
7703 	max_targ = sc->max_targets & CFMAXTARG;
7704 	ahd->our_id = sc->brtime_id & CFSCSIID;
7705 
7706 	/*
7707 	 * Allocate a tstate to house information for our
7708 	 * initiator presence on the bus as well as the user
7709 	 * data for any target mode initiator.
7710 	 */
7711 	if (ahd_alloc_tstate(ahd, ahd->our_id, 'A') == NULL) {
7712 		printk("%s: unable to allocate ahd_tmode_tstate.  "
7713 		       "Failing attach\n", ahd_name(ahd));
7714 		return (ENOMEM);
7715 	}
7716 
7717 	for (targ = 0; targ < max_targ; targ++) {
7718 		struct	 ahd_devinfo devinfo;
7719 		struct	 ahd_initiator_tinfo *tinfo;
7720 		struct	 ahd_transinfo *user_tinfo;
7721 		struct	 ahd_tmode_tstate *tstate;
7722 		uint16_t target_mask;
7723 
7724 		tinfo = ahd_fetch_transinfo(ahd, 'A', ahd->our_id,
7725 					    targ, &tstate);
7726 		user_tinfo = &tinfo->user;
7727 
7728 		/*
7729 		 * We support SPC2 and SPI4.
7730 		 */
7731 		tinfo->user.protocol_version = 4;
7732 		tinfo->user.transport_version = 4;
7733 
7734 		target_mask = 0x01 << targ;
7735 		ahd->user_discenable &= ~target_mask;
7736 		tstate->discenable &= ~target_mask;
7737 		ahd->user_tagenable &= ~target_mask;
7738 		if (sc->device_flags[targ] & CFDISC) {
7739 			tstate->discenable |= target_mask;
7740 			ahd->user_discenable |= target_mask;
7741 			ahd->user_tagenable |= target_mask;
7742 		} else {
7743 			/*
7744 			 * Cannot be packetized without disconnection.
7745 			 */
7746 			sc->device_flags[targ] &= ~CFPACKETIZED;
7747 		}
7748 
7749 		user_tinfo->ppr_options = 0;
7750 		user_tinfo->period = (sc->device_flags[targ] & CFXFER);
7751 		if (user_tinfo->period < CFXFER_ASYNC) {
7752 			if (user_tinfo->period <= AHD_PERIOD_10MHz)
7753 				user_tinfo->ppr_options |= MSG_EXT_PPR_DT_REQ;
7754 			user_tinfo->offset = MAX_OFFSET;
7755 		} else  {
7756 			user_tinfo->offset = 0;
7757 			user_tinfo->period = AHD_ASYNC_XFER_PERIOD;
7758 		}
7759 #ifdef AHD_FORCE_160
7760 		if (user_tinfo->period <= AHD_SYNCRATE_160)
7761 			user_tinfo->period = AHD_SYNCRATE_DT;
7762 #endif
7763 
7764 		if ((sc->device_flags[targ] & CFPACKETIZED) != 0) {
7765 			user_tinfo->ppr_options |= MSG_EXT_PPR_RD_STRM
7766 						|  MSG_EXT_PPR_WR_FLOW
7767 						|  MSG_EXT_PPR_HOLD_MCS
7768 						|  MSG_EXT_PPR_IU_REQ;
7769 			if ((ahd->features & AHD_RTI) != 0)
7770 				user_tinfo->ppr_options |= MSG_EXT_PPR_RTI;
7771 		}
7772 
7773 		if ((sc->device_flags[targ] & CFQAS) != 0)
7774 			user_tinfo->ppr_options |= MSG_EXT_PPR_QAS_REQ;
7775 
7776 		if ((sc->device_flags[targ] & CFWIDEB) != 0)
7777 			user_tinfo->width = MSG_EXT_WDTR_BUS_16_BIT;
7778 		else
7779 			user_tinfo->width = MSG_EXT_WDTR_BUS_8_BIT;
7780 #ifdef AHD_DEBUG
7781 		if ((ahd_debug & AHD_SHOW_MISC) != 0)
7782 			printk("(%d): %x:%x:%x:%x\n", targ, user_tinfo->width,
7783 			       user_tinfo->period, user_tinfo->offset,
7784 			       user_tinfo->ppr_options);
7785 #endif
7786 		/*
7787 		 * Start out Async/Narrow/Untagged and with
7788 		 * conservative protocol support.
7789 		 */
7790 		tstate->tagenable &= ~target_mask;
7791 		tinfo->goal.protocol_version = 2;
7792 		tinfo->goal.transport_version = 2;
7793 		tinfo->curr.protocol_version = 2;
7794 		tinfo->curr.transport_version = 2;
7795 		ahd_compile_devinfo(&devinfo, ahd->our_id,
7796 				    targ, CAM_LUN_WILDCARD,
7797 				    'A', ROLE_INITIATOR);
7798 		ahd_set_width(ahd, &devinfo, MSG_EXT_WDTR_BUS_8_BIT,
7799 			      AHD_TRANS_CUR|AHD_TRANS_GOAL, /*paused*/TRUE);
7800 		ahd_set_syncrate(ahd, &devinfo, /*period*/0, /*offset*/0,
7801 				 /*ppr_options*/0, AHD_TRANS_CUR|AHD_TRANS_GOAL,
7802 				 /*paused*/TRUE);
7803 	}
7804 
7805 	ahd->flags &= ~AHD_SPCHK_ENB_A;
7806 	if (sc->bios_control & CFSPARITY)
7807 		ahd->flags |= AHD_SPCHK_ENB_A;
7808 
7809 	ahd->flags &= ~AHD_RESET_BUS_A;
7810 	if (sc->bios_control & CFRESETB)
7811 		ahd->flags |= AHD_RESET_BUS_A;
7812 
7813 	ahd->flags &= ~AHD_EXTENDED_TRANS_A;
7814 	if (sc->bios_control & CFEXTEND)
7815 		ahd->flags |= AHD_EXTENDED_TRANS_A;
7816 
7817 	ahd->flags &= ~AHD_BIOS_ENABLED;
7818 	if ((sc->bios_control & CFBIOSSTATE) == CFBS_ENABLED)
7819 		ahd->flags |= AHD_BIOS_ENABLED;
7820 
7821 	ahd->flags &= ~AHD_STPWLEVEL_A;
7822 	if ((sc->adapter_control & CFSTPWLEVEL) != 0)
7823 		ahd->flags |= AHD_STPWLEVEL_A;
7824 
7825 	return (0);
7826 }
7827 
7828 /*
7829  * Parse device configuration information.
7830  */
7831 int
7832 ahd_parse_vpddata(struct ahd_softc *ahd, struct vpd_config *vpd)
7833 {
7834 	int error;
7835 
7836 	error = ahd_verify_vpd_cksum(vpd);
7837 	if (error == 0)
7838 		return (EINVAL);
7839 	if ((vpd->bios_flags & VPDBOOTHOST) != 0)
7840 		ahd->flags |= AHD_BOOT_CHANNEL;
7841 	return (0);
7842 }
7843 
7844 void
7845 ahd_intr_enable(struct ahd_softc *ahd, int enable)
7846 {
7847 	u_int hcntrl;
7848 
7849 	hcntrl = ahd_inb(ahd, HCNTRL);
7850 	hcntrl &= ~INTEN;
7851 	ahd->pause &= ~INTEN;
7852 	ahd->unpause &= ~INTEN;
7853 	if (enable) {
7854 		hcntrl |= INTEN;
7855 		ahd->pause |= INTEN;
7856 		ahd->unpause |= INTEN;
7857 	}
7858 	ahd_outb(ahd, HCNTRL, hcntrl);
7859 }
7860 
7861 static void
7862 ahd_update_coalescing_values(struct ahd_softc *ahd, u_int timer, u_int maxcmds,
7863 			     u_int mincmds)
7864 {
7865 	if (timer > AHD_TIMER_MAX_US)
7866 		timer = AHD_TIMER_MAX_US;
7867 	ahd->int_coalescing_timer = timer;
7868 
7869 	if (maxcmds > AHD_INT_COALESCING_MAXCMDS_MAX)
7870 		maxcmds = AHD_INT_COALESCING_MAXCMDS_MAX;
7871 	if (mincmds > AHD_INT_COALESCING_MINCMDS_MAX)
7872 		mincmds = AHD_INT_COALESCING_MINCMDS_MAX;
7873 	ahd->int_coalescing_maxcmds = maxcmds;
7874 	ahd_outw(ahd, INT_COALESCING_TIMER, timer / AHD_TIMER_US_PER_TICK);
7875 	ahd_outb(ahd, INT_COALESCING_MAXCMDS, -maxcmds);
7876 	ahd_outb(ahd, INT_COALESCING_MINCMDS, -mincmds);
7877 }
7878 
7879 static void
7880 ahd_enable_coalescing(struct ahd_softc *ahd, int enable)
7881 {
7882 
7883 	ahd->hs_mailbox &= ~ENINT_COALESCE;
7884 	if (enable)
7885 		ahd->hs_mailbox |= ENINT_COALESCE;
7886 	ahd_outb(ahd, HS_MAILBOX, ahd->hs_mailbox);
7887 	ahd_flush_device_writes(ahd);
7888 	ahd_run_qoutfifo(ahd);
7889 }
7890 
7891 /*
7892  * Ensure that the card is paused in a location
7893  * outside of all critical sections and that all
7894  * pending work is completed prior to returning.
7895  * This routine should only be called from outside
7896  * an interrupt context.
7897  */
7898 void
7899 ahd_pause_and_flushwork(struct ahd_softc *ahd)
7900 {
7901 	u_int intstat;
7902 	u_int maxloops;
7903 
7904 	maxloops = 1000;
7905 	ahd->flags |= AHD_ALL_INTERRUPTS;
7906 	ahd_pause(ahd);
7907 	/*
7908 	 * Freeze the outgoing selections.  We do this only
7909 	 * until we are safely paused without further selections
7910 	 * pending.
7911 	 */
7912 	ahd->qfreeze_cnt--;
7913 	ahd_outw(ahd, KERNEL_QFREEZE_COUNT, ahd->qfreeze_cnt);
7914 	ahd_outb(ahd, SEQ_FLAGS2, ahd_inb(ahd, SEQ_FLAGS2) | SELECTOUT_QFROZEN);
7915 	do {
7916 
7917 		ahd_unpause(ahd);
7918 		/*
7919 		 * Give the sequencer some time to service
7920 		 * any active selections.
7921 		 */
7922 		ahd_delay(500);
7923 
7924 		ahd_intr(ahd);
7925 		ahd_pause(ahd);
7926 		intstat = ahd_inb(ahd, INTSTAT);
7927 		if ((intstat & INT_PEND) == 0) {
7928 			ahd_clear_critical_section(ahd);
7929 			intstat = ahd_inb(ahd, INTSTAT);
7930 		}
7931 	} while (--maxloops
7932 	      && (intstat != 0xFF || (ahd->features & AHD_REMOVABLE) == 0)
7933 	      && ((intstat & INT_PEND) != 0
7934 	       || (ahd_inb(ahd, SCSISEQ0) & ENSELO) != 0
7935 	       || (ahd_inb(ahd, SSTAT0) & (SELDO|SELINGO)) != 0));
7936 
7937 	if (maxloops == 0) {
7938 		printk("Infinite interrupt loop, INTSTAT = %x",
7939 		      ahd_inb(ahd, INTSTAT));
7940 	}
7941 	ahd->qfreeze_cnt++;
7942 	ahd_outw(ahd, KERNEL_QFREEZE_COUNT, ahd->qfreeze_cnt);
7943 
7944 	ahd_flush_qoutfifo(ahd);
7945 
7946 	ahd->flags &= ~AHD_ALL_INTERRUPTS;
7947 }
7948 
7949 #ifdef CONFIG_PM
7950 int
7951 ahd_suspend(struct ahd_softc *ahd)
7952 {
7953 
7954 	ahd_pause_and_flushwork(ahd);
7955 
7956 	if (LIST_FIRST(&ahd->pending_scbs) != NULL) {
7957 		ahd_unpause(ahd);
7958 		return (EBUSY);
7959 	}
7960 	ahd_shutdown(ahd);
7961 	return (0);
7962 }
7963 
7964 void
7965 ahd_resume(struct ahd_softc *ahd)
7966 {
7967 
7968 	ahd_reset(ahd, /*reinit*/TRUE);
7969 	ahd_intr_enable(ahd, TRUE);
7970 	ahd_restart(ahd);
7971 }
7972 #endif
7973 
7974 /************************** Busy Target Table *********************************/
7975 /*
7976  * Set SCBPTR to the SCB that contains the busy
7977  * table entry for TCL.  Return the offset into
7978  * the SCB that contains the entry for TCL.
7979  * saved_scbid is dereferenced and set to the
7980  * scbid that should be restored once manipualtion
7981  * of the TCL entry is complete.
7982  */
7983 static inline u_int
7984 ahd_index_busy_tcl(struct ahd_softc *ahd, u_int *saved_scbid, u_int tcl)
7985 {
7986 	/*
7987 	 * Index to the SCB that contains the busy entry.
7988 	 */
7989 	AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
7990 	*saved_scbid = ahd_get_scbptr(ahd);
7991 	ahd_set_scbptr(ahd, TCL_LUN(tcl)
7992 		     | ((TCL_TARGET_OFFSET(tcl) & 0xC) << 4));
7993 
7994 	/*
7995 	 * And now calculate the SCB offset to the entry.
7996 	 * Each entry is 2 bytes wide, hence the
7997 	 * multiplication by 2.
7998 	 */
7999 	return (((TCL_TARGET_OFFSET(tcl) & 0x3) << 1) + SCB_DISCONNECTED_LISTS);
8000 }
8001 
8002 /*
8003  * Return the untagged transaction id for a given target/channel lun.
8004  */
8005 static u_int
8006 ahd_find_busy_tcl(struct ahd_softc *ahd, u_int tcl)
8007 {
8008 	u_int scbid;
8009 	u_int scb_offset;
8010 	u_int saved_scbptr;
8011 
8012 	scb_offset = ahd_index_busy_tcl(ahd, &saved_scbptr, tcl);
8013 	scbid = ahd_inw_scbram(ahd, scb_offset);
8014 	ahd_set_scbptr(ahd, saved_scbptr);
8015 	return (scbid);
8016 }
8017 
8018 static void
8019 ahd_busy_tcl(struct ahd_softc *ahd, u_int tcl, u_int scbid)
8020 {
8021 	u_int scb_offset;
8022 	u_int saved_scbptr;
8023 
8024 	scb_offset = ahd_index_busy_tcl(ahd, &saved_scbptr, tcl);
8025 	ahd_outw(ahd, scb_offset, scbid);
8026 	ahd_set_scbptr(ahd, saved_scbptr);
8027 }
8028 
8029 /************************** SCB and SCB queue management **********************/
8030 static int
8031 ahd_match_scb(struct ahd_softc *ahd, struct scb *scb, int target,
8032 	      char channel, int lun, u_int tag, role_t role)
8033 {
8034 	int targ = SCB_GET_TARGET(ahd, scb);
8035 	char chan = SCB_GET_CHANNEL(ahd, scb);
8036 	int slun = SCB_GET_LUN(scb);
8037 	int match;
8038 
8039 	match = ((chan == channel) || (channel == ALL_CHANNELS));
8040 	if (match != 0)
8041 		match = ((targ == target) || (target == CAM_TARGET_WILDCARD));
8042 	if (match != 0)
8043 		match = ((lun == slun) || (lun == CAM_LUN_WILDCARD));
8044 	if (match != 0) {
8045 #ifdef AHD_TARGET_MODE
8046 		int group;
8047 
8048 		group = XPT_FC_GROUP(scb->io_ctx->ccb_h.func_code);
8049 		if (role == ROLE_INITIATOR) {
8050 			match = (group != XPT_FC_GROUP_TMODE)
8051 			      && ((tag == SCB_GET_TAG(scb))
8052 			       || (tag == SCB_LIST_NULL));
8053 		} else if (role == ROLE_TARGET) {
8054 			match = (group == XPT_FC_GROUP_TMODE)
8055 			      && ((tag == scb->io_ctx->csio.tag_id)
8056 			       || (tag == SCB_LIST_NULL));
8057 		}
8058 #else /* !AHD_TARGET_MODE */
8059 		match = ((tag == SCB_GET_TAG(scb)) || (tag == SCB_LIST_NULL));
8060 #endif /* AHD_TARGET_MODE */
8061 	}
8062 
8063 	return match;
8064 }
8065 
8066 static void
8067 ahd_freeze_devq(struct ahd_softc *ahd, struct scb *scb)
8068 {
8069 	int	target;
8070 	char	channel;
8071 	int	lun;
8072 
8073 	target = SCB_GET_TARGET(ahd, scb);
8074 	lun = SCB_GET_LUN(scb);
8075 	channel = SCB_GET_CHANNEL(ahd, scb);
8076 
8077 	ahd_search_qinfifo(ahd, target, channel, lun,
8078 			   /*tag*/SCB_LIST_NULL, ROLE_UNKNOWN,
8079 			   CAM_REQUEUE_REQ, SEARCH_COMPLETE);
8080 
8081 	ahd_platform_freeze_devq(ahd, scb);
8082 }
8083 
8084 void
8085 ahd_qinfifo_requeue_tail(struct ahd_softc *ahd, struct scb *scb)
8086 {
8087 	struct scb	*prev_scb;
8088 	ahd_mode_state	 saved_modes;
8089 
8090 	saved_modes = ahd_save_modes(ahd);
8091 	ahd_set_modes(ahd, AHD_MODE_CCHAN, AHD_MODE_CCHAN);
8092 	prev_scb = NULL;
8093 	if (ahd_qinfifo_count(ahd) != 0) {
8094 		u_int prev_tag;
8095 		u_int prev_pos;
8096 
8097 		prev_pos = AHD_QIN_WRAP(ahd->qinfifonext - 1);
8098 		prev_tag = ahd->qinfifo[prev_pos];
8099 		prev_scb = ahd_lookup_scb(ahd, prev_tag);
8100 	}
8101 	ahd_qinfifo_requeue(ahd, prev_scb, scb);
8102 	ahd_set_hnscb_qoff(ahd, ahd->qinfifonext);
8103 	ahd_restore_modes(ahd, saved_modes);
8104 }
8105 
8106 static void
8107 ahd_qinfifo_requeue(struct ahd_softc *ahd, struct scb *prev_scb,
8108 		    struct scb *scb)
8109 {
8110 	if (prev_scb == NULL) {
8111 		uint32_t busaddr;
8112 
8113 		busaddr = ahd_le32toh(scb->hscb->hscb_busaddr);
8114 		ahd_outl(ahd, NEXT_QUEUED_SCB_ADDR, busaddr);
8115 	} else {
8116 		prev_scb->hscb->next_hscb_busaddr = scb->hscb->hscb_busaddr;
8117 		ahd_sync_scb(ahd, prev_scb,
8118 			     BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);
8119 	}
8120 	ahd->qinfifo[AHD_QIN_WRAP(ahd->qinfifonext)] = SCB_GET_TAG(scb);
8121 	ahd->qinfifonext++;
8122 	scb->hscb->next_hscb_busaddr = ahd->next_queued_hscb->hscb_busaddr;
8123 	ahd_sync_scb(ahd, scb, BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);
8124 }
8125 
8126 static int
8127 ahd_qinfifo_count(struct ahd_softc *ahd)
8128 {
8129 	u_int qinpos;
8130 	u_int wrap_qinpos;
8131 	u_int wrap_qinfifonext;
8132 
8133 	AHD_ASSERT_MODES(ahd, AHD_MODE_CCHAN_MSK, AHD_MODE_CCHAN_MSK);
8134 	qinpos = ahd_get_snscb_qoff(ahd);
8135 	wrap_qinpos = AHD_QIN_WRAP(qinpos);
8136 	wrap_qinfifonext = AHD_QIN_WRAP(ahd->qinfifonext);
8137 	if (wrap_qinfifonext >= wrap_qinpos)
8138 		return (wrap_qinfifonext - wrap_qinpos);
8139 	else
8140 		return (wrap_qinfifonext
8141 		      + ARRAY_SIZE(ahd->qinfifo) - wrap_qinpos);
8142 }
8143 
8144 static void
8145 ahd_reset_cmds_pending(struct ahd_softc *ahd)
8146 {
8147 	struct		scb *scb;
8148 	ahd_mode_state	saved_modes;
8149 	u_int		pending_cmds;
8150 
8151 	saved_modes = ahd_save_modes(ahd);
8152 	ahd_set_modes(ahd, AHD_MODE_CCHAN, AHD_MODE_CCHAN);
8153 
8154 	/*
8155 	 * Don't count any commands as outstanding that the
8156 	 * sequencer has already marked for completion.
8157 	 */
8158 	ahd_flush_qoutfifo(ahd);
8159 
8160 	pending_cmds = 0;
8161 	LIST_FOREACH(scb, &ahd->pending_scbs, pending_links) {
8162 		pending_cmds++;
8163 	}
8164 	ahd_outw(ahd, CMDS_PENDING, pending_cmds - ahd_qinfifo_count(ahd));
8165 	ahd_restore_modes(ahd, saved_modes);
8166 	ahd->flags &= ~AHD_UPDATE_PEND_CMDS;
8167 }
8168 
8169 static void
8170 ahd_done_with_status(struct ahd_softc *ahd, struct scb *scb, uint32_t status)
8171 {
8172 	cam_status ostat;
8173 	cam_status cstat;
8174 
8175 	ostat = ahd_get_transaction_status(scb);
8176 	if (ostat == CAM_REQ_INPROG)
8177 		ahd_set_transaction_status(scb, status);
8178 	cstat = ahd_get_transaction_status(scb);
8179 	if (cstat != CAM_REQ_CMP)
8180 		ahd_freeze_scb(scb);
8181 	ahd_done(ahd, scb);
8182 }
8183 
8184 int
8185 ahd_search_qinfifo(struct ahd_softc *ahd, int target, char channel,
8186 		   int lun, u_int tag, role_t role, uint32_t status,
8187 		   ahd_search_action action)
8188 {
8189 	struct scb	*scb;
8190 	struct scb	*mk_msg_scb;
8191 	struct scb	*prev_scb;
8192 	ahd_mode_state	 saved_modes;
8193 	u_int		 qinstart;
8194 	u_int		 qinpos;
8195 	u_int		 qintail;
8196 	u_int		 tid_next;
8197 	u_int		 tid_prev;
8198 	u_int		 scbid;
8199 	u_int		 seq_flags2;
8200 	u_int		 savedscbptr;
8201 	uint32_t	 busaddr;
8202 	int		 found;
8203 	int		 targets;
8204 
8205 	/* Must be in CCHAN mode */
8206 	saved_modes = ahd_save_modes(ahd);
8207 	ahd_set_modes(ahd, AHD_MODE_CCHAN, AHD_MODE_CCHAN);
8208 
8209 	/*
8210 	 * Halt any pending SCB DMA.  The sequencer will reinitiate
8211 	 * this dma if the qinfifo is not empty once we unpause.
8212 	 */
8213 	if ((ahd_inb(ahd, CCSCBCTL) & (CCARREN|CCSCBEN|CCSCBDIR))
8214 	 == (CCARREN|CCSCBEN|CCSCBDIR)) {
8215 		ahd_outb(ahd, CCSCBCTL,
8216 			 ahd_inb(ahd, CCSCBCTL) & ~(CCARREN|CCSCBEN));
8217 		while ((ahd_inb(ahd, CCSCBCTL) & (CCARREN|CCSCBEN)) != 0)
8218 			;
8219 	}
8220 	/* Determine sequencer's position in the qinfifo. */
8221 	qintail = AHD_QIN_WRAP(ahd->qinfifonext);
8222 	qinstart = ahd_get_snscb_qoff(ahd);
8223 	qinpos = AHD_QIN_WRAP(qinstart);
8224 	found = 0;
8225 	prev_scb = NULL;
8226 
8227 	if (action == SEARCH_PRINT) {
8228 		printk("qinstart = %d qinfifonext = %d\nQINFIFO:",
8229 		       qinstart, ahd->qinfifonext);
8230 	}
8231 
8232 	/*
8233 	 * Start with an empty queue.  Entries that are not chosen
8234 	 * for removal will be re-added to the queue as we go.
8235 	 */
8236 	ahd->qinfifonext = qinstart;
8237 	busaddr = ahd_le32toh(ahd->next_queued_hscb->hscb_busaddr);
8238 	ahd_outl(ahd, NEXT_QUEUED_SCB_ADDR, busaddr);
8239 
8240 	while (qinpos != qintail) {
8241 		scb = ahd_lookup_scb(ahd, ahd->qinfifo[qinpos]);
8242 		if (scb == NULL) {
8243 			printk("qinpos = %d, SCB index = %d\n",
8244 				qinpos, ahd->qinfifo[qinpos]);
8245 			panic("Loop 1\n");
8246 		}
8247 
8248 		if (ahd_match_scb(ahd, scb, target, channel, lun, tag, role)) {
8249 			/*
8250 			 * We found an scb that needs to be acted on.
8251 			 */
8252 			found++;
8253 			switch (action) {
8254 			case SEARCH_COMPLETE:
8255 				if ((scb->flags & SCB_ACTIVE) == 0)
8256 					printk("Inactive SCB in qinfifo\n");
8257 				ahd_done_with_status(ahd, scb, status);
8258 				/* FALLTHROUGH */
8259 			case SEARCH_REMOVE:
8260 				break;
8261 			case SEARCH_PRINT:
8262 				printk(" 0x%x", ahd->qinfifo[qinpos]);
8263 				/* FALLTHROUGH */
8264 			case SEARCH_COUNT:
8265 				ahd_qinfifo_requeue(ahd, prev_scb, scb);
8266 				prev_scb = scb;
8267 				break;
8268 			}
8269 		} else {
8270 			ahd_qinfifo_requeue(ahd, prev_scb, scb);
8271 			prev_scb = scb;
8272 		}
8273 		qinpos = AHD_QIN_WRAP(qinpos+1);
8274 	}
8275 
8276 	ahd_set_hnscb_qoff(ahd, ahd->qinfifonext);
8277 
8278 	if (action == SEARCH_PRINT)
8279 		printk("\nWAITING_TID_QUEUES:\n");
8280 
8281 	/*
8282 	 * Search waiting for selection lists.  We traverse the
8283 	 * list of "their ids" waiting for selection and, if
8284 	 * appropriate, traverse the SCBs of each "their id"
8285 	 * looking for matches.
8286 	 */
8287 	ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
8288 	seq_flags2 = ahd_inb(ahd, SEQ_FLAGS2);
8289 	if ((seq_flags2 & PENDING_MK_MESSAGE) != 0) {
8290 		scbid = ahd_inw(ahd, MK_MESSAGE_SCB);
8291 		mk_msg_scb = ahd_lookup_scb(ahd, scbid);
8292 	} else
8293 		mk_msg_scb = NULL;
8294 	savedscbptr = ahd_get_scbptr(ahd);
8295 	tid_next = ahd_inw(ahd, WAITING_TID_HEAD);
8296 	tid_prev = SCB_LIST_NULL;
8297 	targets = 0;
8298 	for (scbid = tid_next; !SCBID_IS_NULL(scbid); scbid = tid_next) {
8299 		u_int tid_head;
8300 		u_int tid_tail;
8301 
8302 		targets++;
8303 		if (targets > AHD_NUM_TARGETS)
8304 			panic("TID LIST LOOP");
8305 
8306 		if (scbid >= ahd->scb_data.numscbs) {
8307 			printk("%s: Waiting TID List inconsistency. "
8308 			       "SCB index == 0x%x, yet numscbs == 0x%x.",
8309 			       ahd_name(ahd), scbid, ahd->scb_data.numscbs);
8310 			ahd_dump_card_state(ahd);
8311 			panic("for safety");
8312 		}
8313 		scb = ahd_lookup_scb(ahd, scbid);
8314 		if (scb == NULL) {
8315 			printk("%s: SCB = 0x%x Not Active!\n",
8316 			       ahd_name(ahd), scbid);
8317 			panic("Waiting TID List traversal\n");
8318 		}
8319 		ahd_set_scbptr(ahd, scbid);
8320 		tid_next = ahd_inw_scbram(ahd, SCB_NEXT2);
8321 		if (ahd_match_scb(ahd, scb, target, channel, CAM_LUN_WILDCARD,
8322 				  SCB_LIST_NULL, ROLE_UNKNOWN) == 0) {
8323 			tid_prev = scbid;
8324 			continue;
8325 		}
8326 
8327 		/*
8328 		 * We found a list of scbs that needs to be searched.
8329 		 */
8330 		if (action == SEARCH_PRINT)
8331 			printk("       %d ( ", SCB_GET_TARGET(ahd, scb));
8332 		tid_head = scbid;
8333 		found += ahd_search_scb_list(ahd, target, channel,
8334 					     lun, tag, role, status,
8335 					     action, &tid_head, &tid_tail,
8336 					     SCB_GET_TARGET(ahd, scb));
8337 		/*
8338 		 * Check any MK_MESSAGE SCB that is still waiting to
8339 		 * enter this target's waiting for selection queue.
8340 		 */
8341 		if (mk_msg_scb != NULL
8342 		 && ahd_match_scb(ahd, mk_msg_scb, target, channel,
8343 				  lun, tag, role)) {
8344 
8345 			/*
8346 			 * We found an scb that needs to be acted on.
8347 			 */
8348 			found++;
8349 			switch (action) {
8350 			case SEARCH_COMPLETE:
8351 				if ((mk_msg_scb->flags & SCB_ACTIVE) == 0)
8352 					printk("Inactive SCB pending MK_MSG\n");
8353 				ahd_done_with_status(ahd, mk_msg_scb, status);
8354 				/* FALLTHROUGH */
8355 			case SEARCH_REMOVE:
8356 			{
8357 				u_int tail_offset;
8358 
8359 				printk("Removing MK_MSG scb\n");
8360 
8361 				/*
8362 				 * Reset our tail to the tail of the
8363 				 * main per-target list.
8364 				 */
8365 				tail_offset = WAITING_SCB_TAILS
8366 				    + (2 * SCB_GET_TARGET(ahd, mk_msg_scb));
8367 				ahd_outw(ahd, tail_offset, tid_tail);
8368 
8369 				seq_flags2 &= ~PENDING_MK_MESSAGE;
8370 				ahd_outb(ahd, SEQ_FLAGS2, seq_flags2);
8371 				ahd_outw(ahd, CMDS_PENDING,
8372 					 ahd_inw(ahd, CMDS_PENDING)-1);
8373 				mk_msg_scb = NULL;
8374 				break;
8375 			}
8376 			case SEARCH_PRINT:
8377 				printk(" 0x%x", SCB_GET_TAG(scb));
8378 				/* FALLTHROUGH */
8379 			case SEARCH_COUNT:
8380 				break;
8381 			}
8382 		}
8383 
8384 		if (mk_msg_scb != NULL
8385 		 && SCBID_IS_NULL(tid_head)
8386 		 && ahd_match_scb(ahd, scb, target, channel, CAM_LUN_WILDCARD,
8387 				  SCB_LIST_NULL, ROLE_UNKNOWN)) {
8388 
8389 			/*
8390 			 * When removing the last SCB for a target
8391 			 * queue with a pending MK_MESSAGE scb, we
8392 			 * must queue the MK_MESSAGE scb.
8393 			 */
8394 			printk("Queueing mk_msg_scb\n");
8395 			tid_head = ahd_inw(ahd, MK_MESSAGE_SCB);
8396 			seq_flags2 &= ~PENDING_MK_MESSAGE;
8397 			ahd_outb(ahd, SEQ_FLAGS2, seq_flags2);
8398 			mk_msg_scb = NULL;
8399 		}
8400 		if (tid_head != scbid)
8401 			ahd_stitch_tid_list(ahd, tid_prev, tid_head, tid_next);
8402 		if (!SCBID_IS_NULL(tid_head))
8403 			tid_prev = tid_head;
8404 		if (action == SEARCH_PRINT)
8405 			printk(")\n");
8406 	}
8407 
8408 	/* Restore saved state. */
8409 	ahd_set_scbptr(ahd, savedscbptr);
8410 	ahd_restore_modes(ahd, saved_modes);
8411 	return (found);
8412 }
8413 
8414 static int
8415 ahd_search_scb_list(struct ahd_softc *ahd, int target, char channel,
8416 		    int lun, u_int tag, role_t role, uint32_t status,
8417 		    ahd_search_action action, u_int *list_head,
8418 		    u_int *list_tail, u_int tid)
8419 {
8420 	struct	scb *scb;
8421 	u_int	scbid;
8422 	u_int	next;
8423 	u_int	prev;
8424 	int	found;
8425 
8426 	AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
8427 	found = 0;
8428 	prev = SCB_LIST_NULL;
8429 	next = *list_head;
8430 	*list_tail = SCB_LIST_NULL;
8431 	for (scbid = next; !SCBID_IS_NULL(scbid); scbid = next) {
8432 		if (scbid >= ahd->scb_data.numscbs) {
8433 			printk("%s:SCB List inconsistency. "
8434 			       "SCB == 0x%x, yet numscbs == 0x%x.",
8435 			       ahd_name(ahd), scbid, ahd->scb_data.numscbs);
8436 			ahd_dump_card_state(ahd);
8437 			panic("for safety");
8438 		}
8439 		scb = ahd_lookup_scb(ahd, scbid);
8440 		if (scb == NULL) {
8441 			printk("%s: SCB = %d Not Active!\n",
8442 			       ahd_name(ahd), scbid);
8443 			panic("Waiting List traversal\n");
8444 		}
8445 		ahd_set_scbptr(ahd, scbid);
8446 		*list_tail = scbid;
8447 		next = ahd_inw_scbram(ahd, SCB_NEXT);
8448 		if (ahd_match_scb(ahd, scb, target, channel,
8449 				  lun, SCB_LIST_NULL, role) == 0) {
8450 			prev = scbid;
8451 			continue;
8452 		}
8453 		found++;
8454 		switch (action) {
8455 		case SEARCH_COMPLETE:
8456 			if ((scb->flags & SCB_ACTIVE) == 0)
8457 				printk("Inactive SCB in Waiting List\n");
8458 			ahd_done_with_status(ahd, scb, status);
8459 			/* FALLTHROUGH */
8460 		case SEARCH_REMOVE:
8461 			ahd_rem_wscb(ahd, scbid, prev, next, tid);
8462 			*list_tail = prev;
8463 			if (SCBID_IS_NULL(prev))
8464 				*list_head = next;
8465 			break;
8466 		case SEARCH_PRINT:
8467 			printk("0x%x ", scbid);
8468 		case SEARCH_COUNT:
8469 			prev = scbid;
8470 			break;
8471 		}
8472 		if (found > AHD_SCB_MAX)
8473 			panic("SCB LIST LOOP");
8474 	}
8475 	if (action == SEARCH_COMPLETE
8476 	 || action == SEARCH_REMOVE)
8477 		ahd_outw(ahd, CMDS_PENDING, ahd_inw(ahd, CMDS_PENDING) - found);
8478 	return (found);
8479 }
8480 
8481 static void
8482 ahd_stitch_tid_list(struct ahd_softc *ahd, u_int tid_prev,
8483 		    u_int tid_cur, u_int tid_next)
8484 {
8485 	AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
8486 
8487 	if (SCBID_IS_NULL(tid_cur)) {
8488 
8489 		/* Bypass current TID list */
8490 		if (SCBID_IS_NULL(tid_prev)) {
8491 			ahd_outw(ahd, WAITING_TID_HEAD, tid_next);
8492 		} else {
8493 			ahd_set_scbptr(ahd, tid_prev);
8494 			ahd_outw(ahd, SCB_NEXT2, tid_next);
8495 		}
8496 		if (SCBID_IS_NULL(tid_next))
8497 			ahd_outw(ahd, WAITING_TID_TAIL, tid_prev);
8498 	} else {
8499 
8500 		/* Stitch through tid_cur */
8501 		if (SCBID_IS_NULL(tid_prev)) {
8502 			ahd_outw(ahd, WAITING_TID_HEAD, tid_cur);
8503 		} else {
8504 			ahd_set_scbptr(ahd, tid_prev);
8505 			ahd_outw(ahd, SCB_NEXT2, tid_cur);
8506 		}
8507 		ahd_set_scbptr(ahd, tid_cur);
8508 		ahd_outw(ahd, SCB_NEXT2, tid_next);
8509 
8510 		if (SCBID_IS_NULL(tid_next))
8511 			ahd_outw(ahd, WAITING_TID_TAIL, tid_cur);
8512 	}
8513 }
8514 
8515 /*
8516  * Manipulate the waiting for selection list and return the
8517  * scb that follows the one that we remove.
8518  */
8519 static u_int
8520 ahd_rem_wscb(struct ahd_softc *ahd, u_int scbid,
8521 	     u_int prev, u_int next, u_int tid)
8522 {
8523 	u_int tail_offset;
8524 
8525 	AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
8526 	if (!SCBID_IS_NULL(prev)) {
8527 		ahd_set_scbptr(ahd, prev);
8528 		ahd_outw(ahd, SCB_NEXT, next);
8529 	}
8530 
8531 	/*
8532 	 * SCBs that have MK_MESSAGE set in them may
8533 	 * cause the tail pointer to be updated without
8534 	 * setting the next pointer of the previous tail.
8535 	 * Only clear the tail if the removed SCB was
8536 	 * the tail.
8537 	 */
8538 	tail_offset = WAITING_SCB_TAILS + (2 * tid);
8539 	if (SCBID_IS_NULL(next)
8540 	 && ahd_inw(ahd, tail_offset) == scbid)
8541 		ahd_outw(ahd, tail_offset, prev);
8542 
8543 	ahd_add_scb_to_free_list(ahd, scbid);
8544 	return (next);
8545 }
8546 
8547 /*
8548  * Add the SCB as selected by SCBPTR onto the on chip list of
8549  * free hardware SCBs.  This list is empty/unused if we are not
8550  * performing SCB paging.
8551  */
8552 static void
8553 ahd_add_scb_to_free_list(struct ahd_softc *ahd, u_int scbid)
8554 {
8555 /* XXX Need some other mechanism to designate "free". */
8556 	/*
8557 	 * Invalidate the tag so that our abort
8558 	 * routines don't think it's active.
8559 	ahd_outb(ahd, SCB_TAG, SCB_LIST_NULL);
8560 	 */
8561 }
8562 
8563 /******************************** Error Handling ******************************/
8564 /*
8565  * Abort all SCBs that match the given description (target/channel/lun/tag),
8566  * setting their status to the passed in status if the status has not already
8567  * been modified from CAM_REQ_INPROG.  This routine assumes that the sequencer
8568  * is paused before it is called.
8569  */
8570 static int
8571 ahd_abort_scbs(struct ahd_softc *ahd, int target, char channel,
8572 	       int lun, u_int tag, role_t role, uint32_t status)
8573 {
8574 	struct		scb *scbp;
8575 	struct		scb *scbp_next;
8576 	u_int		i, j;
8577 	u_int		maxtarget;
8578 	u_int		minlun;
8579 	u_int		maxlun;
8580 	int		found;
8581 	ahd_mode_state	saved_modes;
8582 
8583 	/* restore this when we're done */
8584 	saved_modes = ahd_save_modes(ahd);
8585 	ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
8586 
8587 	found = ahd_search_qinfifo(ahd, target, channel, lun, SCB_LIST_NULL,
8588 				   role, CAM_REQUEUE_REQ, SEARCH_COMPLETE);
8589 
8590 	/*
8591 	 * Clean out the busy target table for any untagged commands.
8592 	 */
8593 	i = 0;
8594 	maxtarget = 16;
8595 	if (target != CAM_TARGET_WILDCARD) {
8596 		i = target;
8597 		if (channel == 'B')
8598 			i += 8;
8599 		maxtarget = i + 1;
8600 	}
8601 
8602 	if (lun == CAM_LUN_WILDCARD) {
8603 		minlun = 0;
8604 		maxlun = AHD_NUM_LUNS_NONPKT;
8605 	} else if (lun >= AHD_NUM_LUNS_NONPKT) {
8606 		minlun = maxlun = 0;
8607 	} else {
8608 		minlun = lun;
8609 		maxlun = lun + 1;
8610 	}
8611 
8612 	if (role != ROLE_TARGET) {
8613 		for (;i < maxtarget; i++) {
8614 			for (j = minlun;j < maxlun; j++) {
8615 				u_int scbid;
8616 				u_int tcl;
8617 
8618 				tcl = BUILD_TCL_RAW(i, 'A', j);
8619 				scbid = ahd_find_busy_tcl(ahd, tcl);
8620 				scbp = ahd_lookup_scb(ahd, scbid);
8621 				if (scbp == NULL
8622 				 || ahd_match_scb(ahd, scbp, target, channel,
8623 						  lun, tag, role) == 0)
8624 					continue;
8625 				ahd_unbusy_tcl(ahd, BUILD_TCL_RAW(i, 'A', j));
8626 			}
8627 		}
8628 	}
8629 
8630 	/*
8631 	 * Don't abort commands that have already completed,
8632 	 * but haven't quite made it up to the host yet.
8633 	 */
8634 	ahd_flush_qoutfifo(ahd);
8635 
8636 	/*
8637 	 * Go through the pending CCB list and look for
8638 	 * commands for this target that are still active.
8639 	 * These are other tagged commands that were
8640 	 * disconnected when the reset occurred.
8641 	 */
8642 	scbp_next = LIST_FIRST(&ahd->pending_scbs);
8643 	while (scbp_next != NULL) {
8644 		scbp = scbp_next;
8645 		scbp_next = LIST_NEXT(scbp, pending_links);
8646 		if (ahd_match_scb(ahd, scbp, target, channel, lun, tag, role)) {
8647 			cam_status ostat;
8648 
8649 			ostat = ahd_get_transaction_status(scbp);
8650 			if (ostat == CAM_REQ_INPROG)
8651 				ahd_set_transaction_status(scbp, status);
8652 			if (ahd_get_transaction_status(scbp) != CAM_REQ_CMP)
8653 				ahd_freeze_scb(scbp);
8654 			if ((scbp->flags & SCB_ACTIVE) == 0)
8655 				printk("Inactive SCB on pending list\n");
8656 			ahd_done(ahd, scbp);
8657 			found++;
8658 		}
8659 	}
8660 	ahd_restore_modes(ahd, saved_modes);
8661 	ahd_platform_abort_scbs(ahd, target, channel, lun, tag, role, status);
8662 	ahd->flags |= AHD_UPDATE_PEND_CMDS;
8663 	return found;
8664 }
8665 
8666 static void
8667 ahd_reset_current_bus(struct ahd_softc *ahd)
8668 {
8669 	uint8_t scsiseq;
8670 
8671 	AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
8672 	ahd_outb(ahd, SIMODE1, ahd_inb(ahd, SIMODE1) & ~ENSCSIRST);
8673 	scsiseq = ahd_inb(ahd, SCSISEQ0) & ~(ENSELO|ENARBO|SCSIRSTO);
8674 	ahd_outb(ahd, SCSISEQ0, scsiseq | SCSIRSTO);
8675 	ahd_flush_device_writes(ahd);
8676 	ahd_delay(AHD_BUSRESET_DELAY);
8677 	/* Turn off the bus reset */
8678 	ahd_outb(ahd, SCSISEQ0, scsiseq);
8679 	ahd_flush_device_writes(ahd);
8680 	ahd_delay(AHD_BUSRESET_DELAY);
8681 	if ((ahd->bugs & AHD_SCSIRST_BUG) != 0) {
8682 		/*
8683 		 * 2A Razor #474
8684 		 * Certain chip state is not cleared for
8685 		 * SCSI bus resets that we initiate, so
8686 		 * we must reset the chip.
8687 		 */
8688 		ahd_reset(ahd, /*reinit*/TRUE);
8689 		ahd_intr_enable(ahd, /*enable*/TRUE);
8690 		AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
8691 	}
8692 
8693 	ahd_clear_intstat(ahd);
8694 }
8695 
8696 int
8697 ahd_reset_channel(struct ahd_softc *ahd, char channel, int initiate_reset)
8698 {
8699 	struct	ahd_devinfo caminfo;
8700 	u_int	initiator;
8701 	u_int	target;
8702 	u_int	max_scsiid;
8703 	int	found;
8704 	u_int	fifo;
8705 	u_int	next_fifo;
8706 	uint8_t scsiseq;
8707 
8708 	/*
8709 	 * Check if the last bus reset is cleared
8710 	 */
8711 	if (ahd->flags & AHD_BUS_RESET_ACTIVE) {
8712 		printk("%s: bus reset still active\n",
8713 		       ahd_name(ahd));
8714 		return 0;
8715 	}
8716 	ahd->flags |= AHD_BUS_RESET_ACTIVE;
8717 
8718 	ahd->pending_device = NULL;
8719 
8720 	ahd_compile_devinfo(&caminfo,
8721 			    CAM_TARGET_WILDCARD,
8722 			    CAM_TARGET_WILDCARD,
8723 			    CAM_LUN_WILDCARD,
8724 			    channel, ROLE_UNKNOWN);
8725 	ahd_pause(ahd);
8726 
8727 	/* Make sure the sequencer is in a safe location. */
8728 	ahd_clear_critical_section(ahd);
8729 
8730 	/*
8731 	 * Run our command complete fifos to ensure that we perform
8732 	 * completion processing on any commands that 'completed'
8733 	 * before the reset occurred.
8734 	 */
8735 	ahd_run_qoutfifo(ahd);
8736 #ifdef AHD_TARGET_MODE
8737 	if ((ahd->flags & AHD_TARGETROLE) != 0) {
8738 		ahd_run_tqinfifo(ahd, /*paused*/TRUE);
8739 	}
8740 #endif
8741 	ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
8742 
8743 	/*
8744 	 * Disable selections so no automatic hardware
8745 	 * functions will modify chip state.
8746 	 */
8747 	ahd_outb(ahd, SCSISEQ0, 0);
8748 	ahd_outb(ahd, SCSISEQ1, 0);
8749 
8750 	/*
8751 	 * Safely shut down our DMA engines.  Always start with
8752 	 * the FIFO that is not currently active (if any are
8753 	 * actively connected).
8754 	 */
8755 	next_fifo = fifo = ahd_inb(ahd, DFFSTAT) & CURRFIFO;
8756 	if (next_fifo > CURRFIFO_1)
8757 		/* If disconneced, arbitrarily start with FIFO1. */
8758 		next_fifo = fifo = 0;
8759 	do {
8760 		next_fifo ^= CURRFIFO_1;
8761 		ahd_set_modes(ahd, next_fifo, next_fifo);
8762 		ahd_outb(ahd, DFCNTRL,
8763 			 ahd_inb(ahd, DFCNTRL) & ~(SCSIEN|HDMAEN));
8764 		while ((ahd_inb(ahd, DFCNTRL) & HDMAENACK) != 0)
8765 			ahd_delay(10);
8766 		/*
8767 		 * Set CURRFIFO to the now inactive channel.
8768 		 */
8769 		ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
8770 		ahd_outb(ahd, DFFSTAT, next_fifo);
8771 	} while (next_fifo != fifo);
8772 
8773 	/*
8774 	 * Reset the bus if we are initiating this reset
8775 	 */
8776 	ahd_clear_msg_state(ahd);
8777 	ahd_outb(ahd, SIMODE1,
8778 		 ahd_inb(ahd, SIMODE1) & ~(ENBUSFREE|ENSCSIRST));
8779 
8780 	if (initiate_reset)
8781 		ahd_reset_current_bus(ahd);
8782 
8783 	ahd_clear_intstat(ahd);
8784 
8785 	/*
8786 	 * Clean up all the state information for the
8787 	 * pending transactions on this bus.
8788 	 */
8789 	found = ahd_abort_scbs(ahd, CAM_TARGET_WILDCARD, channel,
8790 			       CAM_LUN_WILDCARD, SCB_LIST_NULL,
8791 			       ROLE_UNKNOWN, CAM_SCSI_BUS_RESET);
8792 
8793 	/*
8794 	 * Cleanup anything left in the FIFOs.
8795 	 */
8796 	ahd_clear_fifo(ahd, 0);
8797 	ahd_clear_fifo(ahd, 1);
8798 
8799 	/*
8800 	 * Clear SCSI interrupt status
8801 	 */
8802 	ahd_outb(ahd, CLRSINT1, CLRSCSIRSTI);
8803 
8804 	/*
8805 	 * Reenable selections
8806 	 */
8807 	ahd_outb(ahd, SIMODE1, ahd_inb(ahd, SIMODE1) | ENSCSIRST);
8808 	scsiseq = ahd_inb(ahd, SCSISEQ_TEMPLATE);
8809 	ahd_outb(ahd, SCSISEQ1, scsiseq & (ENSELI|ENRSELI|ENAUTOATNP));
8810 
8811 	max_scsiid = (ahd->features & AHD_WIDE) ? 15 : 7;
8812 #ifdef AHD_TARGET_MODE
8813 	/*
8814 	 * Send an immediate notify ccb to all target more peripheral
8815 	 * drivers affected by this action.
8816 	 */
8817 	for (target = 0; target <= max_scsiid; target++) {
8818 		struct ahd_tmode_tstate* tstate;
8819 		u_int lun;
8820 
8821 		tstate = ahd->enabled_targets[target];
8822 		if (tstate == NULL)
8823 			continue;
8824 		for (lun = 0; lun < AHD_NUM_LUNS; lun++) {
8825 			struct ahd_tmode_lstate* lstate;
8826 
8827 			lstate = tstate->enabled_luns[lun];
8828 			if (lstate == NULL)
8829 				continue;
8830 
8831 			ahd_queue_lstate_event(ahd, lstate, CAM_TARGET_WILDCARD,
8832 					       EVENT_TYPE_BUS_RESET, /*arg*/0);
8833 			ahd_send_lstate_events(ahd, lstate);
8834 		}
8835 	}
8836 #endif
8837 	/*
8838 	 * Revert to async/narrow transfers until we renegotiate.
8839 	 */
8840 	for (target = 0; target <= max_scsiid; target++) {
8841 
8842 		if (ahd->enabled_targets[target] == NULL)
8843 			continue;
8844 		for (initiator = 0; initiator <= max_scsiid; initiator++) {
8845 			struct ahd_devinfo devinfo;
8846 
8847 			ahd_compile_devinfo(&devinfo, target, initiator,
8848 					    CAM_LUN_WILDCARD,
8849 					    'A', ROLE_UNKNOWN);
8850 			ahd_set_width(ahd, &devinfo, MSG_EXT_WDTR_BUS_8_BIT,
8851 				      AHD_TRANS_CUR, /*paused*/TRUE);
8852 			ahd_set_syncrate(ahd, &devinfo, /*period*/0,
8853 					 /*offset*/0, /*ppr_options*/0,
8854 					 AHD_TRANS_CUR, /*paused*/TRUE);
8855 		}
8856 	}
8857 
8858 	/* Notify the XPT that a bus reset occurred */
8859 	ahd_send_async(ahd, caminfo.channel, CAM_TARGET_WILDCARD,
8860 		       CAM_LUN_WILDCARD, AC_BUS_RESET);
8861 
8862 	ahd_restart(ahd);
8863 
8864 	return (found);
8865 }
8866 
8867 /**************************** Statistics Processing ***************************/
8868 static void
8869 ahd_stat_timer(struct timer_list *t)
8870 {
8871 	struct	ahd_softc *ahd = from_timer(ahd, t, stat_timer);
8872 	u_long	s;
8873 	int	enint_coal;
8874 
8875 	ahd_lock(ahd, &s);
8876 
8877 	enint_coal = ahd->hs_mailbox & ENINT_COALESCE;
8878 	if (ahd->cmdcmplt_total > ahd->int_coalescing_threshold)
8879 		enint_coal |= ENINT_COALESCE;
8880 	else if (ahd->cmdcmplt_total < ahd->int_coalescing_stop_threshold)
8881 		enint_coal &= ~ENINT_COALESCE;
8882 
8883 	if (enint_coal != (ahd->hs_mailbox & ENINT_COALESCE)) {
8884 		ahd_enable_coalescing(ahd, enint_coal);
8885 #ifdef AHD_DEBUG
8886 		if ((ahd_debug & AHD_SHOW_INT_COALESCING) != 0)
8887 			printk("%s: Interrupt coalescing "
8888 			       "now %sabled. Cmds %d\n",
8889 			       ahd_name(ahd),
8890 			       (enint_coal & ENINT_COALESCE) ? "en" : "dis",
8891 			       ahd->cmdcmplt_total);
8892 #endif
8893 	}
8894 
8895 	ahd->cmdcmplt_bucket = (ahd->cmdcmplt_bucket+1) & (AHD_STAT_BUCKETS-1);
8896 	ahd->cmdcmplt_total -= ahd->cmdcmplt_counts[ahd->cmdcmplt_bucket];
8897 	ahd->cmdcmplt_counts[ahd->cmdcmplt_bucket] = 0;
8898 	ahd_timer_reset(&ahd->stat_timer, AHD_STAT_UPDATE_US);
8899 	ahd_unlock(ahd, &s);
8900 }
8901 
8902 /****************************** Status Processing *****************************/
8903 
8904 static void
8905 ahd_handle_scsi_status(struct ahd_softc *ahd, struct scb *scb)
8906 {
8907 	struct	hardware_scb *hscb;
8908 	int	paused;
8909 
8910 	/*
8911 	 * The sequencer freezes its select-out queue
8912 	 * anytime a SCSI status error occurs.  We must
8913 	 * handle the error and increment our qfreeze count
8914 	 * to allow the sequencer to continue.  We don't
8915 	 * bother clearing critical sections here since all
8916 	 * operations are on data structures that the sequencer
8917 	 * is not touching once the queue is frozen.
8918 	 */
8919 	hscb = scb->hscb;
8920 
8921 	if (ahd_is_paused(ahd)) {
8922 		paused = 1;
8923 	} else {
8924 		paused = 0;
8925 		ahd_pause(ahd);
8926 	}
8927 
8928 	/* Freeze the queue until the client sees the error. */
8929 	ahd_freeze_devq(ahd, scb);
8930 	ahd_freeze_scb(scb);
8931 	ahd->qfreeze_cnt++;
8932 	ahd_outw(ahd, KERNEL_QFREEZE_COUNT, ahd->qfreeze_cnt);
8933 
8934 	if (paused == 0)
8935 		ahd_unpause(ahd);
8936 
8937 	/* Don't want to clobber the original sense code */
8938 	if ((scb->flags & SCB_SENSE) != 0) {
8939 		/*
8940 		 * Clear the SCB_SENSE Flag and perform
8941 		 * a normal command completion.
8942 		 */
8943 		scb->flags &= ~SCB_SENSE;
8944 		ahd_set_transaction_status(scb, CAM_AUTOSENSE_FAIL);
8945 		ahd_done(ahd, scb);
8946 		return;
8947 	}
8948 	ahd_set_transaction_status(scb, CAM_SCSI_STATUS_ERROR);
8949 	ahd_set_scsi_status(scb, hscb->shared_data.istatus.scsi_status);
8950 	switch (hscb->shared_data.istatus.scsi_status) {
8951 	case STATUS_PKT_SENSE:
8952 	{
8953 		struct scsi_status_iu_header *siu;
8954 
8955 		ahd_sync_sense(ahd, scb, BUS_DMASYNC_POSTREAD);
8956 		siu = (struct scsi_status_iu_header *)scb->sense_data;
8957 		ahd_set_scsi_status(scb, siu->status);
8958 #ifdef AHD_DEBUG
8959 		if ((ahd_debug & AHD_SHOW_SENSE) != 0) {
8960 			ahd_print_path(ahd, scb);
8961 			printk("SCB 0x%x Received PKT Status of 0x%x\n",
8962 			       SCB_GET_TAG(scb), siu->status);
8963 			printk("\tflags = 0x%x, sense len = 0x%x, "
8964 			       "pktfail = 0x%x\n",
8965 			       siu->flags, scsi_4btoul(siu->sense_length),
8966 			       scsi_4btoul(siu->pkt_failures_length));
8967 		}
8968 #endif
8969 		if ((siu->flags & SIU_RSPVALID) != 0) {
8970 			ahd_print_path(ahd, scb);
8971 			if (scsi_4btoul(siu->pkt_failures_length) < 4) {
8972 				printk("Unable to parse pkt_failures\n");
8973 			} else {
8974 
8975 				switch (SIU_PKTFAIL_CODE(siu)) {
8976 				case SIU_PFC_NONE:
8977 					printk("No packet failure found\n");
8978 					break;
8979 				case SIU_PFC_CIU_FIELDS_INVALID:
8980 					printk("Invalid Command IU Field\n");
8981 					break;
8982 				case SIU_PFC_TMF_NOT_SUPPORTED:
8983 					printk("TMF not supported\n");
8984 					break;
8985 				case SIU_PFC_TMF_FAILED:
8986 					printk("TMF failed\n");
8987 					break;
8988 				case SIU_PFC_INVALID_TYPE_CODE:
8989 					printk("Invalid L_Q Type code\n");
8990 					break;
8991 				case SIU_PFC_ILLEGAL_REQUEST:
8992 					printk("Illegal request\n");
8993 				default:
8994 					break;
8995 				}
8996 			}
8997 			if (siu->status == SCSI_STATUS_OK)
8998 				ahd_set_transaction_status(scb,
8999 							   CAM_REQ_CMP_ERR);
9000 		}
9001 		if ((siu->flags & SIU_SNSVALID) != 0) {
9002 			scb->flags |= SCB_PKT_SENSE;
9003 #ifdef AHD_DEBUG
9004 			if ((ahd_debug & AHD_SHOW_SENSE) != 0)
9005 				printk("Sense data available\n");
9006 #endif
9007 		}
9008 		ahd_done(ahd, scb);
9009 		break;
9010 	}
9011 	case SCSI_STATUS_CMD_TERMINATED:
9012 	case SCSI_STATUS_CHECK_COND:
9013 	{
9014 		struct ahd_devinfo devinfo;
9015 		struct ahd_dma_seg *sg;
9016 		struct scsi_sense *sc;
9017 		struct ahd_initiator_tinfo *targ_info;
9018 		struct ahd_tmode_tstate *tstate;
9019 		struct ahd_transinfo *tinfo;
9020 #ifdef AHD_DEBUG
9021 		if (ahd_debug & AHD_SHOW_SENSE) {
9022 			ahd_print_path(ahd, scb);
9023 			printk("SCB %d: requests Check Status\n",
9024 			       SCB_GET_TAG(scb));
9025 		}
9026 #endif
9027 
9028 		if (ahd_perform_autosense(scb) == 0)
9029 			break;
9030 
9031 		ahd_compile_devinfo(&devinfo, SCB_GET_OUR_ID(scb),
9032 				    SCB_GET_TARGET(ahd, scb),
9033 				    SCB_GET_LUN(scb),
9034 				    SCB_GET_CHANNEL(ahd, scb),
9035 				    ROLE_INITIATOR);
9036 		targ_info = ahd_fetch_transinfo(ahd,
9037 						devinfo.channel,
9038 						devinfo.our_scsiid,
9039 						devinfo.target,
9040 						&tstate);
9041 		tinfo = &targ_info->curr;
9042 		sg = scb->sg_list;
9043 		sc = (struct scsi_sense *)hscb->shared_data.idata.cdb;
9044 		/*
9045 		 * Save off the residual if there is one.
9046 		 */
9047 		ahd_update_residual(ahd, scb);
9048 #ifdef AHD_DEBUG
9049 		if (ahd_debug & AHD_SHOW_SENSE) {
9050 			ahd_print_path(ahd, scb);
9051 			printk("Sending Sense\n");
9052 		}
9053 #endif
9054 		scb->sg_count = 0;
9055 		sg = ahd_sg_setup(ahd, scb, sg, ahd_get_sense_bufaddr(ahd, scb),
9056 				  ahd_get_sense_bufsize(ahd, scb),
9057 				  /*last*/TRUE);
9058 		sc->opcode = REQUEST_SENSE;
9059 		sc->byte2 = 0;
9060 		if (tinfo->protocol_version <= SCSI_REV_2
9061 		 && SCB_GET_LUN(scb) < 8)
9062 			sc->byte2 = SCB_GET_LUN(scb) << 5;
9063 		sc->unused[0] = 0;
9064 		sc->unused[1] = 0;
9065 		sc->length = ahd_get_sense_bufsize(ahd, scb);
9066 		sc->control = 0;
9067 
9068 		/*
9069 		 * We can't allow the target to disconnect.
9070 		 * This will be an untagged transaction and
9071 		 * having the target disconnect will make this
9072 		 * transaction indestinguishable from outstanding
9073 		 * tagged transactions.
9074 		 */
9075 		hscb->control = 0;
9076 
9077 		/*
9078 		 * This request sense could be because the
9079 		 * the device lost power or in some other
9080 		 * way has lost our transfer negotiations.
9081 		 * Renegotiate if appropriate.  Unit attention
9082 		 * errors will be reported before any data
9083 		 * phases occur.
9084 		 */
9085 		if (ahd_get_residual(scb) == ahd_get_transfer_length(scb)) {
9086 			ahd_update_neg_request(ahd, &devinfo,
9087 					       tstate, targ_info,
9088 					       AHD_NEG_IF_NON_ASYNC);
9089 		}
9090 		if (tstate->auto_negotiate & devinfo.target_mask) {
9091 			hscb->control |= MK_MESSAGE;
9092 			scb->flags &=
9093 			    ~(SCB_NEGOTIATE|SCB_ABORT|SCB_DEVICE_RESET);
9094 			scb->flags |= SCB_AUTO_NEGOTIATE;
9095 		}
9096 		hscb->cdb_len = sizeof(*sc);
9097 		ahd_setup_data_scb(ahd, scb);
9098 		scb->flags |= SCB_SENSE;
9099 		ahd_queue_scb(ahd, scb);
9100 		break;
9101 	}
9102 	case SCSI_STATUS_OK:
9103 		printk("%s: Interrupted for status of 0???\n",
9104 		       ahd_name(ahd));
9105 		/* FALLTHROUGH */
9106 	default:
9107 		ahd_done(ahd, scb);
9108 		break;
9109 	}
9110 }
9111 
9112 static void
9113 ahd_handle_scb_status(struct ahd_softc *ahd, struct scb *scb)
9114 {
9115 	if (scb->hscb->shared_data.istatus.scsi_status != 0) {
9116 		ahd_handle_scsi_status(ahd, scb);
9117 	} else {
9118 		ahd_calc_residual(ahd, scb);
9119 		ahd_done(ahd, scb);
9120 	}
9121 }
9122 
9123 /*
9124  * Calculate the residual for a just completed SCB.
9125  */
9126 static void
9127 ahd_calc_residual(struct ahd_softc *ahd, struct scb *scb)
9128 {
9129 	struct hardware_scb *hscb;
9130 	struct initiator_status *spkt;
9131 	uint32_t sgptr;
9132 	uint32_t resid_sgptr;
9133 	uint32_t resid;
9134 
9135 	/*
9136 	 * 5 cases.
9137 	 * 1) No residual.
9138 	 *    SG_STATUS_VALID clear in sgptr.
9139 	 * 2) Transferless command
9140 	 * 3) Never performed any transfers.
9141 	 *    sgptr has SG_FULL_RESID set.
9142 	 * 4) No residual but target did not
9143 	 *    save data pointers after the
9144 	 *    last transfer, so sgptr was
9145 	 *    never updated.
9146 	 * 5) We have a partial residual.
9147 	 *    Use residual_sgptr to determine
9148 	 *    where we are.
9149 	 */
9150 
9151 	hscb = scb->hscb;
9152 	sgptr = ahd_le32toh(hscb->sgptr);
9153 	if ((sgptr & SG_STATUS_VALID) == 0)
9154 		/* Case 1 */
9155 		return;
9156 	sgptr &= ~SG_STATUS_VALID;
9157 
9158 	if ((sgptr & SG_LIST_NULL) != 0)
9159 		/* Case 2 */
9160 		return;
9161 
9162 	/*
9163 	 * Residual fields are the same in both
9164 	 * target and initiator status packets,
9165 	 * so we can always use the initiator fields
9166 	 * regardless of the role for this SCB.
9167 	 */
9168 	spkt = &hscb->shared_data.istatus;
9169 	resid_sgptr = ahd_le32toh(spkt->residual_sgptr);
9170 	if ((sgptr & SG_FULL_RESID) != 0) {
9171 		/* Case 3 */
9172 		resid = ahd_get_transfer_length(scb);
9173 	} else if ((resid_sgptr & SG_LIST_NULL) != 0) {
9174 		/* Case 4 */
9175 		return;
9176 	} else if ((resid_sgptr & SG_OVERRUN_RESID) != 0) {
9177 		ahd_print_path(ahd, scb);
9178 		printk("data overrun detected Tag == 0x%x.\n",
9179 		       SCB_GET_TAG(scb));
9180 		ahd_freeze_devq(ahd, scb);
9181 		ahd_set_transaction_status(scb, CAM_DATA_RUN_ERR);
9182 		ahd_freeze_scb(scb);
9183 		return;
9184 	} else if ((resid_sgptr & ~SG_PTR_MASK) != 0) {
9185 		panic("Bogus resid sgptr value 0x%x\n", resid_sgptr);
9186 		/* NOTREACHED */
9187 	} else {
9188 		struct ahd_dma_seg *sg;
9189 
9190 		/*
9191 		 * Remainder of the SG where the transfer
9192 		 * stopped.
9193 		 */
9194 		resid = ahd_le32toh(spkt->residual_datacnt) & AHD_SG_LEN_MASK;
9195 		sg = ahd_sg_bus_to_virt(ahd, scb, resid_sgptr & SG_PTR_MASK);
9196 
9197 		/* The residual sg_ptr always points to the next sg */
9198 		sg--;
9199 
9200 		/*
9201 		 * Add up the contents of all residual
9202 		 * SG segments that are after the SG where
9203 		 * the transfer stopped.
9204 		 */
9205 		while ((ahd_le32toh(sg->len) & AHD_DMA_LAST_SEG) == 0) {
9206 			sg++;
9207 			resid += ahd_le32toh(sg->len) & AHD_SG_LEN_MASK;
9208 		}
9209 	}
9210 	if ((scb->flags & SCB_SENSE) == 0)
9211 		ahd_set_residual(scb, resid);
9212 	else
9213 		ahd_set_sense_residual(scb, resid);
9214 
9215 #ifdef AHD_DEBUG
9216 	if ((ahd_debug & AHD_SHOW_MISC) != 0) {
9217 		ahd_print_path(ahd, scb);
9218 		printk("Handled %sResidual of %d bytes\n",
9219 		       (scb->flags & SCB_SENSE) ? "Sense " : "", resid);
9220 	}
9221 #endif
9222 }
9223 
9224 /******************************* Target Mode **********************************/
9225 #ifdef AHD_TARGET_MODE
9226 /*
9227  * Add a target mode event to this lun's queue
9228  */
9229 static void
9230 ahd_queue_lstate_event(struct ahd_softc *ahd, struct ahd_tmode_lstate *lstate,
9231 		       u_int initiator_id, u_int event_type, u_int event_arg)
9232 {
9233 	struct ahd_tmode_event *event;
9234 	int pending;
9235 
9236 	xpt_freeze_devq(lstate->path, /*count*/1);
9237 	if (lstate->event_w_idx >= lstate->event_r_idx)
9238 		pending = lstate->event_w_idx - lstate->event_r_idx;
9239 	else
9240 		pending = AHD_TMODE_EVENT_BUFFER_SIZE + 1
9241 			- (lstate->event_r_idx - lstate->event_w_idx);
9242 
9243 	if (event_type == EVENT_TYPE_BUS_RESET
9244 	 || event_type == MSG_BUS_DEV_RESET) {
9245 		/*
9246 		 * Any earlier events are irrelevant, so reset our buffer.
9247 		 * This has the effect of allowing us to deal with reset
9248 		 * floods (an external device holding down the reset line)
9249 		 * without losing the event that is really interesting.
9250 		 */
9251 		lstate->event_r_idx = 0;
9252 		lstate->event_w_idx = 0;
9253 		xpt_release_devq(lstate->path, pending, /*runqueue*/FALSE);
9254 	}
9255 
9256 	if (pending == AHD_TMODE_EVENT_BUFFER_SIZE) {
9257 		xpt_print_path(lstate->path);
9258 		printk("immediate event %x:%x lost\n",
9259 		       lstate->event_buffer[lstate->event_r_idx].event_type,
9260 		       lstate->event_buffer[lstate->event_r_idx].event_arg);
9261 		lstate->event_r_idx++;
9262 		if (lstate->event_r_idx == AHD_TMODE_EVENT_BUFFER_SIZE)
9263 			lstate->event_r_idx = 0;
9264 		xpt_release_devq(lstate->path, /*count*/1, /*runqueue*/FALSE);
9265 	}
9266 
9267 	event = &lstate->event_buffer[lstate->event_w_idx];
9268 	event->initiator_id = initiator_id;
9269 	event->event_type = event_type;
9270 	event->event_arg = event_arg;
9271 	lstate->event_w_idx++;
9272 	if (lstate->event_w_idx == AHD_TMODE_EVENT_BUFFER_SIZE)
9273 		lstate->event_w_idx = 0;
9274 }
9275 
9276 /*
9277  * Send any target mode events queued up waiting
9278  * for immediate notify resources.
9279  */
9280 void
9281 ahd_send_lstate_events(struct ahd_softc *ahd, struct ahd_tmode_lstate *lstate)
9282 {
9283 	struct ccb_hdr *ccbh;
9284 	struct ccb_immed_notify *inot;
9285 
9286 	while (lstate->event_r_idx != lstate->event_w_idx
9287 	    && (ccbh = SLIST_FIRST(&lstate->immed_notifies)) != NULL) {
9288 		struct ahd_tmode_event *event;
9289 
9290 		event = &lstate->event_buffer[lstate->event_r_idx];
9291 		SLIST_REMOVE_HEAD(&lstate->immed_notifies, sim_links.sle);
9292 		inot = (struct ccb_immed_notify *)ccbh;
9293 		switch (event->event_type) {
9294 		case EVENT_TYPE_BUS_RESET:
9295 			ccbh->status = CAM_SCSI_BUS_RESET|CAM_DEV_QFRZN;
9296 			break;
9297 		default:
9298 			ccbh->status = CAM_MESSAGE_RECV|CAM_DEV_QFRZN;
9299 			inot->message_args[0] = event->event_type;
9300 			inot->message_args[1] = event->event_arg;
9301 			break;
9302 		}
9303 		inot->initiator_id = event->initiator_id;
9304 		inot->sense_len = 0;
9305 		xpt_done((union ccb *)inot);
9306 		lstate->event_r_idx++;
9307 		if (lstate->event_r_idx == AHD_TMODE_EVENT_BUFFER_SIZE)
9308 			lstate->event_r_idx = 0;
9309 	}
9310 }
9311 #endif
9312 
9313 /******************** Sequencer Program Patching/Download *********************/
9314 
9315 #ifdef AHD_DUMP_SEQ
9316 void
9317 ahd_dumpseq(struct ahd_softc* ahd)
9318 {
9319 	int i;
9320 	int max_prog;
9321 
9322 	max_prog = 2048;
9323 
9324 	ahd_outb(ahd, SEQCTL0, PERRORDIS|FAILDIS|FASTMODE|LOADRAM);
9325 	ahd_outw(ahd, PRGMCNT, 0);
9326 	for (i = 0; i < max_prog; i++) {
9327 		uint8_t ins_bytes[4];
9328 
9329 		ahd_insb(ahd, SEQRAM, ins_bytes, 4);
9330 		printk("0x%08x\n", ins_bytes[0] << 24
9331 				 | ins_bytes[1] << 16
9332 				 | ins_bytes[2] << 8
9333 				 | ins_bytes[3]);
9334 	}
9335 }
9336 #endif
9337 
9338 static void
9339 ahd_loadseq(struct ahd_softc *ahd)
9340 {
9341 	struct	cs cs_table[NUM_CRITICAL_SECTIONS];
9342 	u_int	begin_set[NUM_CRITICAL_SECTIONS];
9343 	u_int	end_set[NUM_CRITICAL_SECTIONS];
9344 	const struct patch *cur_patch;
9345 	u_int	cs_count;
9346 	u_int	cur_cs;
9347 	u_int	i;
9348 	int	downloaded;
9349 	u_int	skip_addr;
9350 	u_int	sg_prefetch_cnt;
9351 	u_int	sg_prefetch_cnt_limit;
9352 	u_int	sg_prefetch_align;
9353 	u_int	sg_size;
9354 	u_int	cacheline_mask;
9355 	uint8_t	download_consts[DOWNLOAD_CONST_COUNT];
9356 
9357 	if (bootverbose)
9358 		printk("%s: Downloading Sequencer Program...",
9359 		       ahd_name(ahd));
9360 
9361 #if DOWNLOAD_CONST_COUNT != 8
9362 #error "Download Const Mismatch"
9363 #endif
9364 	/*
9365 	 * Start out with 0 critical sections
9366 	 * that apply to this firmware load.
9367 	 */
9368 	cs_count = 0;
9369 	cur_cs = 0;
9370 	memset(begin_set, 0, sizeof(begin_set));
9371 	memset(end_set, 0, sizeof(end_set));
9372 
9373 	/*
9374 	 * Setup downloadable constant table.
9375 	 *
9376 	 * The computation for the S/G prefetch variables is
9377 	 * a bit complicated.  We would like to always fetch
9378 	 * in terms of cachelined sized increments.  However,
9379 	 * if the cacheline is not an even multiple of the
9380 	 * SG element size or is larger than our SG RAM, using
9381 	 * just the cache size might leave us with only a portion
9382 	 * of an SG element at the tail of a prefetch.  If the
9383 	 * cacheline is larger than our S/G prefetch buffer less
9384 	 * the size of an SG element, we may round down to a cacheline
9385 	 * that doesn't contain any or all of the S/G of interest
9386 	 * within the bounds of our S/G ram.  Provide variables to
9387 	 * the sequencer that will allow it to handle these edge
9388 	 * cases.
9389 	 */
9390 	/* Start by aligning to the nearest cacheline. */
9391 	sg_prefetch_align = ahd->pci_cachesize;
9392 	if (sg_prefetch_align == 0)
9393 		sg_prefetch_align = 8;
9394 	/* Round down to the nearest power of 2. */
9395 	while (powerof2(sg_prefetch_align) == 0)
9396 		sg_prefetch_align--;
9397 
9398 	cacheline_mask = sg_prefetch_align - 1;
9399 
9400 	/*
9401 	 * If the cacheline boundary is greater than half our prefetch RAM
9402 	 * we risk not being able to fetch even a single complete S/G
9403 	 * segment if we align to that boundary.
9404 	 */
9405 	if (sg_prefetch_align > CCSGADDR_MAX/2)
9406 		sg_prefetch_align = CCSGADDR_MAX/2;
9407 	/* Start by fetching a single cacheline. */
9408 	sg_prefetch_cnt = sg_prefetch_align;
9409 	/*
9410 	 * Increment the prefetch count by cachelines until
9411 	 * at least one S/G element will fit.
9412 	 */
9413 	sg_size = sizeof(struct ahd_dma_seg);
9414 	if ((ahd->flags & AHD_64BIT_ADDRESSING) != 0)
9415 		sg_size = sizeof(struct ahd_dma64_seg);
9416 	while (sg_prefetch_cnt < sg_size)
9417 		sg_prefetch_cnt += sg_prefetch_align;
9418 	/*
9419 	 * If the cacheline is not an even multiple of
9420 	 * the S/G size, we may only get a partial S/G when
9421 	 * we align. Add a cacheline if this is the case.
9422 	 */
9423 	if ((sg_prefetch_align % sg_size) != 0
9424 	 && (sg_prefetch_cnt < CCSGADDR_MAX))
9425 		sg_prefetch_cnt += sg_prefetch_align;
9426 	/*
9427 	 * Lastly, compute a value that the sequencer can use
9428 	 * to determine if the remainder of the CCSGRAM buffer
9429 	 * has a full S/G element in it.
9430 	 */
9431 	sg_prefetch_cnt_limit = -(sg_prefetch_cnt - sg_size + 1);
9432 	download_consts[SG_PREFETCH_CNT] = sg_prefetch_cnt;
9433 	download_consts[SG_PREFETCH_CNT_LIMIT] = sg_prefetch_cnt_limit;
9434 	download_consts[SG_PREFETCH_ALIGN_MASK] = ~(sg_prefetch_align - 1);
9435 	download_consts[SG_PREFETCH_ADDR_MASK] = (sg_prefetch_align - 1);
9436 	download_consts[SG_SIZEOF] = sg_size;
9437 	download_consts[PKT_OVERRUN_BUFOFFSET] =
9438 		(ahd->overrun_buf - (uint8_t *)ahd->qoutfifo) / 256;
9439 	download_consts[SCB_TRANSFER_SIZE] = SCB_TRANSFER_SIZE_1BYTE_LUN;
9440 	download_consts[CACHELINE_MASK] = cacheline_mask;
9441 	cur_patch = patches;
9442 	downloaded = 0;
9443 	skip_addr = 0;
9444 	ahd_outb(ahd, SEQCTL0, PERRORDIS|FAILDIS|FASTMODE|LOADRAM);
9445 	ahd_outw(ahd, PRGMCNT, 0);
9446 
9447 	for (i = 0; i < sizeof(seqprog)/4; i++) {
9448 		if (ahd_check_patch(ahd, &cur_patch, i, &skip_addr) == 0) {
9449 			/*
9450 			 * Don't download this instruction as it
9451 			 * is in a patch that was removed.
9452 			 */
9453 			continue;
9454 		}
9455 		/*
9456 		 * Move through the CS table until we find a CS
9457 		 * that might apply to this instruction.
9458 		 */
9459 		for (; cur_cs < NUM_CRITICAL_SECTIONS; cur_cs++) {
9460 			if (critical_sections[cur_cs].end <= i) {
9461 				if (begin_set[cs_count] == TRUE
9462 				 && end_set[cs_count] == FALSE) {
9463 					cs_table[cs_count].end = downloaded;
9464 				 	end_set[cs_count] = TRUE;
9465 					cs_count++;
9466 				}
9467 				continue;
9468 			}
9469 			if (critical_sections[cur_cs].begin <= i
9470 			 && begin_set[cs_count] == FALSE) {
9471 				cs_table[cs_count].begin = downloaded;
9472 				begin_set[cs_count] = TRUE;
9473 			}
9474 			break;
9475 		}
9476 		ahd_download_instr(ahd, i, download_consts);
9477 		downloaded++;
9478 	}
9479 
9480 	ahd->num_critical_sections = cs_count;
9481 	if (cs_count != 0) {
9482 
9483 		cs_count *= sizeof(struct cs);
9484 		ahd->critical_sections = kmalloc(cs_count, GFP_ATOMIC);
9485 		if (ahd->critical_sections == NULL)
9486 			panic("ahd_loadseq: Could not malloc");
9487 		memcpy(ahd->critical_sections, cs_table, cs_count);
9488 	}
9489 	ahd_outb(ahd, SEQCTL0, PERRORDIS|FAILDIS|FASTMODE);
9490 
9491 	if (bootverbose) {
9492 		printk(" %d instructions downloaded\n", downloaded);
9493 		printk("%s: Features 0x%x, Bugs 0x%x, Flags 0x%x\n",
9494 		       ahd_name(ahd), ahd->features, ahd->bugs, ahd->flags);
9495 	}
9496 }
9497 
9498 static int
9499 ahd_check_patch(struct ahd_softc *ahd, const struct patch **start_patch,
9500 		u_int start_instr, u_int *skip_addr)
9501 {
9502 	const struct patch *cur_patch;
9503 	const struct patch *last_patch;
9504 	u_int	num_patches;
9505 
9506 	num_patches = ARRAY_SIZE(patches);
9507 	last_patch = &patches[num_patches];
9508 	cur_patch = *start_patch;
9509 
9510 	while (cur_patch < last_patch && start_instr == cur_patch->begin) {
9511 
9512 		if (cur_patch->patch_func(ahd) == 0) {
9513 
9514 			/* Start rejecting code */
9515 			*skip_addr = start_instr + cur_patch->skip_instr;
9516 			cur_patch += cur_patch->skip_patch;
9517 		} else {
9518 			/* Accepted this patch.  Advance to the next
9519 			 * one and wait for our intruction pointer to
9520 			 * hit this point.
9521 			 */
9522 			cur_patch++;
9523 		}
9524 	}
9525 
9526 	*start_patch = cur_patch;
9527 	if (start_instr < *skip_addr)
9528 		/* Still skipping */
9529 		return (0);
9530 
9531 	return (1);
9532 }
9533 
9534 static u_int
9535 ahd_resolve_seqaddr(struct ahd_softc *ahd, u_int address)
9536 {
9537 	const struct patch *cur_patch;
9538 	int address_offset;
9539 	u_int skip_addr;
9540 	u_int i;
9541 
9542 	address_offset = 0;
9543 	cur_patch = patches;
9544 	skip_addr = 0;
9545 
9546 	for (i = 0; i < address;) {
9547 
9548 		ahd_check_patch(ahd, &cur_patch, i, &skip_addr);
9549 
9550 		if (skip_addr > i) {
9551 			int end_addr;
9552 
9553 			end_addr = min(address, skip_addr);
9554 			address_offset += end_addr - i;
9555 			i = skip_addr;
9556 		} else {
9557 			i++;
9558 		}
9559 	}
9560 	return (address - address_offset);
9561 }
9562 
9563 static void
9564 ahd_download_instr(struct ahd_softc *ahd, u_int instrptr, uint8_t *dconsts)
9565 {
9566 	union	ins_formats instr;
9567 	struct	ins_format1 *fmt1_ins;
9568 	struct	ins_format3 *fmt3_ins;
9569 	u_int	opcode;
9570 
9571 	/*
9572 	 * The firmware is always compiled into a little endian format.
9573 	 */
9574 	instr.integer = ahd_le32toh(*(uint32_t*)&seqprog[instrptr * 4]);
9575 
9576 	fmt1_ins = &instr.format1;
9577 	fmt3_ins = NULL;
9578 
9579 	/* Pull the opcode */
9580 	opcode = instr.format1.opcode;
9581 	switch (opcode) {
9582 	case AIC_OP_JMP:
9583 	case AIC_OP_JC:
9584 	case AIC_OP_JNC:
9585 	case AIC_OP_CALL:
9586 	case AIC_OP_JNE:
9587 	case AIC_OP_JNZ:
9588 	case AIC_OP_JE:
9589 	case AIC_OP_JZ:
9590 	{
9591 		fmt3_ins = &instr.format3;
9592 		fmt3_ins->address = ahd_resolve_seqaddr(ahd, fmt3_ins->address);
9593 		/* FALLTHROUGH */
9594 	}
9595 	case AIC_OP_OR:
9596 	case AIC_OP_AND:
9597 	case AIC_OP_XOR:
9598 	case AIC_OP_ADD:
9599 	case AIC_OP_ADC:
9600 	case AIC_OP_BMOV:
9601 		if (fmt1_ins->parity != 0) {
9602 			fmt1_ins->immediate = dconsts[fmt1_ins->immediate];
9603 		}
9604 		fmt1_ins->parity = 0;
9605 		/* FALLTHROUGH */
9606 	case AIC_OP_ROL:
9607 	{
9608 		int i, count;
9609 
9610 		/* Calculate odd parity for the instruction */
9611 		for (i = 0, count = 0; i < 31; i++) {
9612 			uint32_t mask;
9613 
9614 			mask = 0x01 << i;
9615 			if ((instr.integer & mask) != 0)
9616 				count++;
9617 		}
9618 		if ((count & 0x01) == 0)
9619 			instr.format1.parity = 1;
9620 
9621 		/* The sequencer is a little endian cpu */
9622 		instr.integer = ahd_htole32(instr.integer);
9623 		ahd_outsb(ahd, SEQRAM, instr.bytes, 4);
9624 		break;
9625 	}
9626 	default:
9627 		panic("Unknown opcode encountered in seq program");
9628 		break;
9629 	}
9630 }
9631 
9632 static int
9633 ahd_probe_stack_size(struct ahd_softc *ahd)
9634 {
9635 	int last_probe;
9636 
9637 	last_probe = 0;
9638 	while (1) {
9639 		int i;
9640 
9641 		/*
9642 		 * We avoid using 0 as a pattern to avoid
9643 		 * confusion if the stack implementation
9644 		 * "back-fills" with zeros when "poping'
9645 		 * entries.
9646 		 */
9647 		for (i = 1; i <= last_probe+1; i++) {
9648 		       ahd_outb(ahd, STACK, i & 0xFF);
9649 		       ahd_outb(ahd, STACK, (i >> 8) & 0xFF);
9650 		}
9651 
9652 		/* Verify */
9653 		for (i = last_probe+1; i > 0; i--) {
9654 			u_int stack_entry;
9655 
9656 			stack_entry = ahd_inb(ahd, STACK)
9657 				    |(ahd_inb(ahd, STACK) << 8);
9658 			if (stack_entry != i)
9659 				goto sized;
9660 		}
9661 		last_probe++;
9662 	}
9663 sized:
9664 	return (last_probe);
9665 }
9666 
9667 int
9668 ahd_print_register(const ahd_reg_parse_entry_t *table, u_int num_entries,
9669 		   const char *name, u_int address, u_int value,
9670 		   u_int *cur_column, u_int wrap_point)
9671 {
9672 	int	printed;
9673 	u_int	printed_mask;
9674 
9675 	if (cur_column != NULL && *cur_column >= wrap_point) {
9676 		printk("\n");
9677 		*cur_column = 0;
9678 	}
9679 	printed = printk("%s[0x%x]", name, value);
9680 	if (table == NULL) {
9681 		printed += printk(" ");
9682 		*cur_column += printed;
9683 		return (printed);
9684 	}
9685 	printed_mask = 0;
9686 	while (printed_mask != 0xFF) {
9687 		int entry;
9688 
9689 		for (entry = 0; entry < num_entries; entry++) {
9690 			if (((value & table[entry].mask)
9691 			  != table[entry].value)
9692 			 || ((printed_mask & table[entry].mask)
9693 			  == table[entry].mask))
9694 				continue;
9695 
9696 			printed += printk("%s%s",
9697 					  printed_mask == 0 ? ":(" : "|",
9698 					  table[entry].name);
9699 			printed_mask |= table[entry].mask;
9700 
9701 			break;
9702 		}
9703 		if (entry >= num_entries)
9704 			break;
9705 	}
9706 	if (printed_mask != 0)
9707 		printed += printk(") ");
9708 	else
9709 		printed += printk(" ");
9710 	if (cur_column != NULL)
9711 		*cur_column += printed;
9712 	return (printed);
9713 }
9714 
9715 void
9716 ahd_dump_card_state(struct ahd_softc *ahd)
9717 {
9718 	struct scb	*scb;
9719 	ahd_mode_state	 saved_modes;
9720 	u_int		 dffstat;
9721 	int		 paused;
9722 	u_int		 scb_index;
9723 	u_int		 saved_scb_index;
9724 	u_int		 cur_col;
9725 	int		 i;
9726 
9727 	if (ahd_is_paused(ahd)) {
9728 		paused = 1;
9729 	} else {
9730 		paused = 0;
9731 		ahd_pause(ahd);
9732 	}
9733 	saved_modes = ahd_save_modes(ahd);
9734 	ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
9735 	printk(">>>>>>>>>>>>>>>>>> Dump Card State Begins <<<<<<<<<<<<<<<<<\n"
9736 	       "%s: Dumping Card State at program address 0x%x Mode 0x%x\n",
9737 	       ahd_name(ahd),
9738 	       ahd_inw(ahd, CURADDR),
9739 	       ahd_build_mode_state(ahd, ahd->saved_src_mode,
9740 				    ahd->saved_dst_mode));
9741 	if (paused)
9742 		printk("Card was paused\n");
9743 
9744 	if (ahd_check_cmdcmpltqueues(ahd))
9745 		printk("Completions are pending\n");
9746 
9747 	/*
9748 	 * Mode independent registers.
9749 	 */
9750 	cur_col = 0;
9751 	ahd_intstat_print(ahd_inb(ahd, INTSTAT), &cur_col, 50);
9752 	ahd_seloid_print(ahd_inb(ahd, SELOID), &cur_col, 50);
9753 	ahd_selid_print(ahd_inb(ahd, SELID), &cur_col, 50);
9754 	ahd_hs_mailbox_print(ahd_inb(ahd, LOCAL_HS_MAILBOX), &cur_col, 50);
9755 	ahd_intctl_print(ahd_inb(ahd, INTCTL), &cur_col, 50);
9756 	ahd_seqintstat_print(ahd_inb(ahd, SEQINTSTAT), &cur_col, 50);
9757 	ahd_saved_mode_print(ahd_inb(ahd, SAVED_MODE), &cur_col, 50);
9758 	ahd_dffstat_print(ahd_inb(ahd, DFFSTAT), &cur_col, 50);
9759 	ahd_scsisigi_print(ahd_inb(ahd, SCSISIGI), &cur_col, 50);
9760 	ahd_scsiphase_print(ahd_inb(ahd, SCSIPHASE), &cur_col, 50);
9761 	ahd_scsibus_print(ahd_inb(ahd, SCSIBUS), &cur_col, 50);
9762 	ahd_lastphase_print(ahd_inb(ahd, LASTPHASE), &cur_col, 50);
9763 	ahd_scsiseq0_print(ahd_inb(ahd, SCSISEQ0), &cur_col, 50);
9764 	ahd_scsiseq1_print(ahd_inb(ahd, SCSISEQ1), &cur_col, 50);
9765 	ahd_seqctl0_print(ahd_inb(ahd, SEQCTL0), &cur_col, 50);
9766 	ahd_seqintctl_print(ahd_inb(ahd, SEQINTCTL), &cur_col, 50);
9767 	ahd_seq_flags_print(ahd_inb(ahd, SEQ_FLAGS), &cur_col, 50);
9768 	ahd_seq_flags2_print(ahd_inb(ahd, SEQ_FLAGS2), &cur_col, 50);
9769 	ahd_qfreeze_count_print(ahd_inw(ahd, QFREEZE_COUNT), &cur_col, 50);
9770 	ahd_kernel_qfreeze_count_print(ahd_inw(ahd, KERNEL_QFREEZE_COUNT),
9771 				       &cur_col, 50);
9772 	ahd_mk_message_scb_print(ahd_inw(ahd, MK_MESSAGE_SCB), &cur_col, 50);
9773 	ahd_mk_message_scsiid_print(ahd_inb(ahd, MK_MESSAGE_SCSIID),
9774 				    &cur_col, 50);
9775 	ahd_sstat0_print(ahd_inb(ahd, SSTAT0), &cur_col, 50);
9776 	ahd_sstat1_print(ahd_inb(ahd, SSTAT1), &cur_col, 50);
9777 	ahd_sstat2_print(ahd_inb(ahd, SSTAT2), &cur_col, 50);
9778 	ahd_sstat3_print(ahd_inb(ahd, SSTAT3), &cur_col, 50);
9779 	ahd_perrdiag_print(ahd_inb(ahd, PERRDIAG), &cur_col, 50);
9780 	ahd_simode1_print(ahd_inb(ahd, SIMODE1), &cur_col, 50);
9781 	ahd_lqistat0_print(ahd_inb(ahd, LQISTAT0), &cur_col, 50);
9782 	ahd_lqistat1_print(ahd_inb(ahd, LQISTAT1), &cur_col, 50);
9783 	ahd_lqistat2_print(ahd_inb(ahd, LQISTAT2), &cur_col, 50);
9784 	ahd_lqostat0_print(ahd_inb(ahd, LQOSTAT0), &cur_col, 50);
9785 	ahd_lqostat1_print(ahd_inb(ahd, LQOSTAT1), &cur_col, 50);
9786 	ahd_lqostat2_print(ahd_inb(ahd, LQOSTAT2), &cur_col, 50);
9787 	printk("\n");
9788 	printk("\nSCB Count = %d CMDS_PENDING = %d LASTSCB 0x%x "
9789 	       "CURRSCB 0x%x NEXTSCB 0x%x\n",
9790 	       ahd->scb_data.numscbs, ahd_inw(ahd, CMDS_PENDING),
9791 	       ahd_inw(ahd, LASTSCB), ahd_inw(ahd, CURRSCB),
9792 	       ahd_inw(ahd, NEXTSCB));
9793 	cur_col = 0;
9794 	/* QINFIFO */
9795 	ahd_search_qinfifo(ahd, CAM_TARGET_WILDCARD, ALL_CHANNELS,
9796 			   CAM_LUN_WILDCARD, SCB_LIST_NULL,
9797 			   ROLE_UNKNOWN, /*status*/0, SEARCH_PRINT);
9798 	saved_scb_index = ahd_get_scbptr(ahd);
9799 	printk("Pending list:");
9800 	i = 0;
9801 	LIST_FOREACH(scb, &ahd->pending_scbs, pending_links) {
9802 		if (i++ > AHD_SCB_MAX)
9803 			break;
9804 		cur_col = printk("\n%3d FIFO_USE[0x%x] ", SCB_GET_TAG(scb),
9805 				 ahd_inb_scbram(ahd, SCB_FIFO_USE_COUNT));
9806 		ahd_set_scbptr(ahd, SCB_GET_TAG(scb));
9807 		ahd_scb_control_print(ahd_inb_scbram(ahd, SCB_CONTROL),
9808 				      &cur_col, 60);
9809 		ahd_scb_scsiid_print(ahd_inb_scbram(ahd, SCB_SCSIID),
9810 				     &cur_col, 60);
9811 	}
9812 	printk("\nTotal %d\n", i);
9813 
9814 	printk("Kernel Free SCB list: ");
9815 	i = 0;
9816 	TAILQ_FOREACH(scb, &ahd->scb_data.free_scbs, links.tqe) {
9817 		struct scb *list_scb;
9818 
9819 		list_scb = scb;
9820 		do {
9821 			printk("%d ", SCB_GET_TAG(list_scb));
9822 			list_scb = LIST_NEXT(list_scb, collision_links);
9823 		} while (list_scb && i++ < AHD_SCB_MAX);
9824 	}
9825 
9826 	LIST_FOREACH(scb, &ahd->scb_data.any_dev_free_scb_list, links.le) {
9827 		if (i++ > AHD_SCB_MAX)
9828 			break;
9829 		printk("%d ", SCB_GET_TAG(scb));
9830 	}
9831 	printk("\n");
9832 
9833 	printk("Sequencer Complete DMA-inprog list: ");
9834 	scb_index = ahd_inw(ahd, COMPLETE_SCB_DMAINPROG_HEAD);
9835 	i = 0;
9836 	while (!SCBID_IS_NULL(scb_index) && i++ < AHD_SCB_MAX) {
9837 		ahd_set_scbptr(ahd, scb_index);
9838 		printk("%d ", scb_index);
9839 		scb_index = ahd_inw_scbram(ahd, SCB_NEXT_COMPLETE);
9840 	}
9841 	printk("\n");
9842 
9843 	printk("Sequencer Complete list: ");
9844 	scb_index = ahd_inw(ahd, COMPLETE_SCB_HEAD);
9845 	i = 0;
9846 	while (!SCBID_IS_NULL(scb_index) && i++ < AHD_SCB_MAX) {
9847 		ahd_set_scbptr(ahd, scb_index);
9848 		printk("%d ", scb_index);
9849 		scb_index = ahd_inw_scbram(ahd, SCB_NEXT_COMPLETE);
9850 	}
9851 	printk("\n");
9852 
9853 
9854 	printk("Sequencer DMA-Up and Complete list: ");
9855 	scb_index = ahd_inw(ahd, COMPLETE_DMA_SCB_HEAD);
9856 	i = 0;
9857 	while (!SCBID_IS_NULL(scb_index) && i++ < AHD_SCB_MAX) {
9858 		ahd_set_scbptr(ahd, scb_index);
9859 		printk("%d ", scb_index);
9860 		scb_index = ahd_inw_scbram(ahd, SCB_NEXT_COMPLETE);
9861 	}
9862 	printk("\n");
9863 	printk("Sequencer On QFreeze and Complete list: ");
9864 	scb_index = ahd_inw(ahd, COMPLETE_ON_QFREEZE_HEAD);
9865 	i = 0;
9866 	while (!SCBID_IS_NULL(scb_index) && i++ < AHD_SCB_MAX) {
9867 		ahd_set_scbptr(ahd, scb_index);
9868 		printk("%d ", scb_index);
9869 		scb_index = ahd_inw_scbram(ahd, SCB_NEXT_COMPLETE);
9870 	}
9871 	printk("\n");
9872 	ahd_set_scbptr(ahd, saved_scb_index);
9873 	dffstat = ahd_inb(ahd, DFFSTAT);
9874 	for (i = 0; i < 2; i++) {
9875 #ifdef AHD_DEBUG
9876 		struct scb *fifo_scb;
9877 #endif
9878 		u_int	    fifo_scbptr;
9879 
9880 		ahd_set_modes(ahd, AHD_MODE_DFF0 + i, AHD_MODE_DFF0 + i);
9881 		fifo_scbptr = ahd_get_scbptr(ahd);
9882 		printk("\n\n%s: FIFO%d %s, LONGJMP == 0x%x, SCB 0x%x\n",
9883 		       ahd_name(ahd), i,
9884 		       (dffstat & (FIFO0FREE << i)) ? "Free" : "Active",
9885 		       ahd_inw(ahd, LONGJMP_ADDR), fifo_scbptr);
9886 		cur_col = 0;
9887 		ahd_seqimode_print(ahd_inb(ahd, SEQIMODE), &cur_col, 50);
9888 		ahd_seqintsrc_print(ahd_inb(ahd, SEQINTSRC), &cur_col, 50);
9889 		ahd_dfcntrl_print(ahd_inb(ahd, DFCNTRL), &cur_col, 50);
9890 		ahd_dfstatus_print(ahd_inb(ahd, DFSTATUS), &cur_col, 50);
9891 		ahd_sg_cache_shadow_print(ahd_inb(ahd, SG_CACHE_SHADOW),
9892 					  &cur_col, 50);
9893 		ahd_sg_state_print(ahd_inb(ahd, SG_STATE), &cur_col, 50);
9894 		ahd_dffsxfrctl_print(ahd_inb(ahd, DFFSXFRCTL), &cur_col, 50);
9895 		ahd_soffcnt_print(ahd_inb(ahd, SOFFCNT), &cur_col, 50);
9896 		ahd_mdffstat_print(ahd_inb(ahd, MDFFSTAT), &cur_col, 50);
9897 		if (cur_col > 50) {
9898 			printk("\n");
9899 			cur_col = 0;
9900 		}
9901 		cur_col += printk("SHADDR = 0x%x%x, SHCNT = 0x%x ",
9902 				  ahd_inl(ahd, SHADDR+4),
9903 				  ahd_inl(ahd, SHADDR),
9904 				  (ahd_inb(ahd, SHCNT)
9905 				| (ahd_inb(ahd, SHCNT + 1) << 8)
9906 				| (ahd_inb(ahd, SHCNT + 2) << 16)));
9907 		if (cur_col > 50) {
9908 			printk("\n");
9909 			cur_col = 0;
9910 		}
9911 		cur_col += printk("HADDR = 0x%x%x, HCNT = 0x%x ",
9912 				  ahd_inl(ahd, HADDR+4),
9913 				  ahd_inl(ahd, HADDR),
9914 				  (ahd_inb(ahd, HCNT)
9915 				| (ahd_inb(ahd, HCNT + 1) << 8)
9916 				| (ahd_inb(ahd, HCNT + 2) << 16)));
9917 		ahd_ccsgctl_print(ahd_inb(ahd, CCSGCTL), &cur_col, 50);
9918 #ifdef AHD_DEBUG
9919 		if ((ahd_debug & AHD_SHOW_SG) != 0) {
9920 			fifo_scb = ahd_lookup_scb(ahd, fifo_scbptr);
9921 			if (fifo_scb != NULL)
9922 				ahd_dump_sglist(fifo_scb);
9923 		}
9924 #endif
9925 	}
9926 	printk("\nLQIN: ");
9927 	for (i = 0; i < 20; i++)
9928 		printk("0x%x ", ahd_inb(ahd, LQIN + i));
9929 	printk("\n");
9930 	ahd_set_modes(ahd, AHD_MODE_CFG, AHD_MODE_CFG);
9931 	printk("%s: LQISTATE = 0x%x, LQOSTATE = 0x%x, OPTIONMODE = 0x%x\n",
9932 	       ahd_name(ahd), ahd_inb(ahd, LQISTATE), ahd_inb(ahd, LQOSTATE),
9933 	       ahd_inb(ahd, OPTIONMODE));
9934 	printk("%s: OS_SPACE_CNT = 0x%x MAXCMDCNT = 0x%x\n",
9935 	       ahd_name(ahd), ahd_inb(ahd, OS_SPACE_CNT),
9936 	       ahd_inb(ahd, MAXCMDCNT));
9937 	printk("%s: SAVED_SCSIID = 0x%x SAVED_LUN = 0x%x\n",
9938 	       ahd_name(ahd), ahd_inb(ahd, SAVED_SCSIID),
9939 	       ahd_inb(ahd, SAVED_LUN));
9940 	ahd_simode0_print(ahd_inb(ahd, SIMODE0), &cur_col, 50);
9941 	printk("\n");
9942 	ahd_set_modes(ahd, AHD_MODE_CCHAN, AHD_MODE_CCHAN);
9943 	cur_col = 0;
9944 	ahd_ccscbctl_print(ahd_inb(ahd, CCSCBCTL), &cur_col, 50);
9945 	printk("\n");
9946 	ahd_set_modes(ahd, ahd->saved_src_mode, ahd->saved_dst_mode);
9947 	printk("%s: REG0 == 0x%x, SINDEX = 0x%x, DINDEX = 0x%x\n",
9948 	       ahd_name(ahd), ahd_inw(ahd, REG0), ahd_inw(ahd, SINDEX),
9949 	       ahd_inw(ahd, DINDEX));
9950 	printk("%s: SCBPTR == 0x%x, SCB_NEXT == 0x%x, SCB_NEXT2 == 0x%x\n",
9951 	       ahd_name(ahd), ahd_get_scbptr(ahd),
9952 	       ahd_inw_scbram(ahd, SCB_NEXT),
9953 	       ahd_inw_scbram(ahd, SCB_NEXT2));
9954 	printk("CDB %x %x %x %x %x %x\n",
9955 	       ahd_inb_scbram(ahd, SCB_CDB_STORE),
9956 	       ahd_inb_scbram(ahd, SCB_CDB_STORE+1),
9957 	       ahd_inb_scbram(ahd, SCB_CDB_STORE+2),
9958 	       ahd_inb_scbram(ahd, SCB_CDB_STORE+3),
9959 	       ahd_inb_scbram(ahd, SCB_CDB_STORE+4),
9960 	       ahd_inb_scbram(ahd, SCB_CDB_STORE+5));
9961 	printk("STACK:");
9962 	for (i = 0; i < ahd->stack_size; i++) {
9963 		ahd->saved_stack[i] =
9964 		    ahd_inb(ahd, STACK)|(ahd_inb(ahd, STACK) << 8);
9965 		printk(" 0x%x", ahd->saved_stack[i]);
9966 	}
9967 	for (i = ahd->stack_size-1; i >= 0; i--) {
9968 		ahd_outb(ahd, STACK, ahd->saved_stack[i] & 0xFF);
9969 		ahd_outb(ahd, STACK, (ahd->saved_stack[i] >> 8) & 0xFF);
9970 	}
9971 	printk("\n<<<<<<<<<<<<<<<<< Dump Card State Ends >>>>>>>>>>>>>>>>>>\n");
9972 	ahd_restore_modes(ahd, saved_modes);
9973 	if (paused == 0)
9974 		ahd_unpause(ahd);
9975 }
9976 
9977 #if 0
9978 void
9979 ahd_dump_scbs(struct ahd_softc *ahd)
9980 {
9981 	ahd_mode_state saved_modes;
9982 	u_int	       saved_scb_index;
9983 	int	       i;
9984 
9985 	saved_modes = ahd_save_modes(ahd);
9986 	ahd_set_modes(ahd, AHD_MODE_SCSI, AHD_MODE_SCSI);
9987 	saved_scb_index = ahd_get_scbptr(ahd);
9988 	for (i = 0; i < AHD_SCB_MAX; i++) {
9989 		ahd_set_scbptr(ahd, i);
9990 		printk("%3d", i);
9991 		printk("(CTRL 0x%x ID 0x%x N 0x%x N2 0x%x SG 0x%x, RSG 0x%x)\n",
9992 		       ahd_inb_scbram(ahd, SCB_CONTROL),
9993 		       ahd_inb_scbram(ahd, SCB_SCSIID),
9994 		       ahd_inw_scbram(ahd, SCB_NEXT),
9995 		       ahd_inw_scbram(ahd, SCB_NEXT2),
9996 		       ahd_inl_scbram(ahd, SCB_SGPTR),
9997 		       ahd_inl_scbram(ahd, SCB_RESIDUAL_SGPTR));
9998 	}
9999 	printk("\n");
10000 	ahd_set_scbptr(ahd, saved_scb_index);
10001 	ahd_restore_modes(ahd, saved_modes);
10002 }
10003 #endif  /*  0  */
10004 
10005 /**************************** Flexport Logic **********************************/
10006 /*
10007  * Read count 16bit words from 16bit word address start_addr from the
10008  * SEEPROM attached to the controller, into buf, using the controller's
10009  * SEEPROM reading state machine.  Optionally treat the data as a byte
10010  * stream in terms of byte order.
10011  */
10012 int
10013 ahd_read_seeprom(struct ahd_softc *ahd, uint16_t *buf,
10014 		 u_int start_addr, u_int count, int bytestream)
10015 {
10016 	u_int cur_addr;
10017 	u_int end_addr;
10018 	int   error;
10019 
10020 	/*
10021 	 * If we never make it through the loop even once,
10022 	 * we were passed invalid arguments.
10023 	 */
10024 	error = EINVAL;
10025 	AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
10026 	end_addr = start_addr + count;
10027 	for (cur_addr = start_addr; cur_addr < end_addr; cur_addr++) {
10028 
10029 		ahd_outb(ahd, SEEADR, cur_addr);
10030 		ahd_outb(ahd, SEECTL, SEEOP_READ | SEESTART);
10031 
10032 		error = ahd_wait_seeprom(ahd);
10033 		if (error)
10034 			break;
10035 		if (bytestream != 0) {
10036 			uint8_t *bytestream_ptr;
10037 
10038 			bytestream_ptr = (uint8_t *)buf;
10039 			*bytestream_ptr++ = ahd_inb(ahd, SEEDAT);
10040 			*bytestream_ptr = ahd_inb(ahd, SEEDAT+1);
10041 		} else {
10042 			/*
10043 			 * ahd_inw() already handles machine byte order.
10044 			 */
10045 			*buf = ahd_inw(ahd, SEEDAT);
10046 		}
10047 		buf++;
10048 	}
10049 	return (error);
10050 }
10051 
10052 /*
10053  * Write count 16bit words from buf, into SEEPROM attache to the
10054  * controller starting at 16bit word address start_addr, using the
10055  * controller's SEEPROM writing state machine.
10056  */
10057 int
10058 ahd_write_seeprom(struct ahd_softc *ahd, uint16_t *buf,
10059 		  u_int start_addr, u_int count)
10060 {
10061 	u_int cur_addr;
10062 	u_int end_addr;
10063 	int   error;
10064 	int   retval;
10065 
10066 	AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
10067 	error = ENOENT;
10068 
10069 	/* Place the chip into write-enable mode */
10070 	ahd_outb(ahd, SEEADR, SEEOP_EWEN_ADDR);
10071 	ahd_outb(ahd, SEECTL, SEEOP_EWEN | SEESTART);
10072 	error = ahd_wait_seeprom(ahd);
10073 	if (error)
10074 		return (error);
10075 
10076 	/*
10077 	 * Write the data.  If we don't get through the loop at
10078 	 * least once, the arguments were invalid.
10079 	 */
10080 	retval = EINVAL;
10081 	end_addr = start_addr + count;
10082 	for (cur_addr = start_addr; cur_addr < end_addr; cur_addr++) {
10083 		ahd_outw(ahd, SEEDAT, *buf++);
10084 		ahd_outb(ahd, SEEADR, cur_addr);
10085 		ahd_outb(ahd, SEECTL, SEEOP_WRITE | SEESTART);
10086 
10087 		retval = ahd_wait_seeprom(ahd);
10088 		if (retval)
10089 			break;
10090 	}
10091 
10092 	/*
10093 	 * Disable writes.
10094 	 */
10095 	ahd_outb(ahd, SEEADR, SEEOP_EWDS_ADDR);
10096 	ahd_outb(ahd, SEECTL, SEEOP_EWDS | SEESTART);
10097 	error = ahd_wait_seeprom(ahd);
10098 	if (error)
10099 		return (error);
10100 	return (retval);
10101 }
10102 
10103 /*
10104  * Wait ~100us for the serial eeprom to satisfy our request.
10105  */
10106 static int
10107 ahd_wait_seeprom(struct ahd_softc *ahd)
10108 {
10109 	int cnt;
10110 
10111 	cnt = 5000;
10112 	while ((ahd_inb(ahd, SEESTAT) & (SEEARBACK|SEEBUSY)) != 0 && --cnt)
10113 		ahd_delay(5);
10114 
10115 	if (cnt == 0)
10116 		return (ETIMEDOUT);
10117 	return (0);
10118 }
10119 
10120 /*
10121  * Validate the two checksums in the per_channel
10122  * vital product data struct.
10123  */
10124 static int
10125 ahd_verify_vpd_cksum(struct vpd_config *vpd)
10126 {
10127 	int i;
10128 	int maxaddr;
10129 	uint32_t checksum;
10130 	uint8_t *vpdarray;
10131 
10132 	vpdarray = (uint8_t *)vpd;
10133 	maxaddr = offsetof(struct vpd_config, vpd_checksum);
10134 	checksum = 0;
10135 	for (i = offsetof(struct vpd_config, resource_type); i < maxaddr; i++)
10136 		checksum = checksum + vpdarray[i];
10137 	if (checksum == 0
10138 	 || (-checksum & 0xFF) != vpd->vpd_checksum)
10139 		return (0);
10140 
10141 	checksum = 0;
10142 	maxaddr = offsetof(struct vpd_config, checksum);
10143 	for (i = offsetof(struct vpd_config, default_target_flags);
10144 	     i < maxaddr; i++)
10145 		checksum = checksum + vpdarray[i];
10146 	if (checksum == 0
10147 	 || (-checksum & 0xFF) != vpd->checksum)
10148 		return (0);
10149 	return (1);
10150 }
10151 
10152 int
10153 ahd_verify_cksum(struct seeprom_config *sc)
10154 {
10155 	int i;
10156 	int maxaddr;
10157 	uint32_t checksum;
10158 	uint16_t *scarray;
10159 
10160 	maxaddr = (sizeof(*sc)/2) - 1;
10161 	checksum = 0;
10162 	scarray = (uint16_t *)sc;
10163 
10164 	for (i = 0; i < maxaddr; i++)
10165 		checksum = checksum + scarray[i];
10166 	if (checksum == 0
10167 	 || (checksum & 0xFFFF) != sc->checksum) {
10168 		return (0);
10169 	} else {
10170 		return (1);
10171 	}
10172 }
10173 
10174 int
10175 ahd_acquire_seeprom(struct ahd_softc *ahd)
10176 {
10177 	/*
10178 	 * We should be able to determine the SEEPROM type
10179 	 * from the flexport logic, but unfortunately not
10180 	 * all implementations have this logic and there is
10181 	 * no programatic method for determining if the logic
10182 	 * is present.
10183 	 */
10184 	return (1);
10185 #if 0
10186 	uint8_t	seetype;
10187 	int	error;
10188 
10189 	error = ahd_read_flexport(ahd, FLXADDR_ROMSTAT_CURSENSECTL, &seetype);
10190 	if (error != 0
10191          || ((seetype & FLX_ROMSTAT_SEECFG) == FLX_ROMSTAT_SEE_NONE))
10192 		return (0);
10193 	return (1);
10194 #endif
10195 }
10196 
10197 void
10198 ahd_release_seeprom(struct ahd_softc *ahd)
10199 {
10200 	/* Currently a no-op */
10201 }
10202 
10203 /*
10204  * Wait at most 2 seconds for flexport arbitration to succeed.
10205  */
10206 static int
10207 ahd_wait_flexport(struct ahd_softc *ahd)
10208 {
10209 	int cnt;
10210 
10211 	AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
10212 	cnt = 1000000 * 2 / 5;
10213 	while ((ahd_inb(ahd, BRDCTL) & FLXARBACK) == 0 && --cnt)
10214 		ahd_delay(5);
10215 
10216 	if (cnt == 0)
10217 		return (ETIMEDOUT);
10218 	return (0);
10219 }
10220 
10221 int
10222 ahd_write_flexport(struct ahd_softc *ahd, u_int addr, u_int value)
10223 {
10224 	int error;
10225 
10226 	AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
10227 	if (addr > 7)
10228 		panic("ahd_write_flexport: address out of range");
10229 	ahd_outb(ahd, BRDCTL, BRDEN|(addr << 3));
10230 	error = ahd_wait_flexport(ahd);
10231 	if (error != 0)
10232 		return (error);
10233 	ahd_outb(ahd, BRDDAT, value);
10234 	ahd_flush_device_writes(ahd);
10235 	ahd_outb(ahd, BRDCTL, BRDSTB|BRDEN|(addr << 3));
10236 	ahd_flush_device_writes(ahd);
10237 	ahd_outb(ahd, BRDCTL, BRDEN|(addr << 3));
10238 	ahd_flush_device_writes(ahd);
10239 	ahd_outb(ahd, BRDCTL, 0);
10240 	ahd_flush_device_writes(ahd);
10241 	return (0);
10242 }
10243 
10244 int
10245 ahd_read_flexport(struct ahd_softc *ahd, u_int addr, uint8_t *value)
10246 {
10247 	int	error;
10248 
10249 	AHD_ASSERT_MODES(ahd, AHD_MODE_SCSI_MSK, AHD_MODE_SCSI_MSK);
10250 	if (addr > 7)
10251 		panic("ahd_read_flexport: address out of range");
10252 	ahd_outb(ahd, BRDCTL, BRDRW|BRDEN|(addr << 3));
10253 	error = ahd_wait_flexport(ahd);
10254 	if (error != 0)
10255 		return (error);
10256 	*value = ahd_inb(ahd, BRDDAT);
10257 	ahd_outb(ahd, BRDCTL, 0);
10258 	ahd_flush_device_writes(ahd);
10259 	return (0);
10260 }
10261 
10262 /************************* Target Mode ****************************************/
10263 #ifdef AHD_TARGET_MODE
10264 cam_status
10265 ahd_find_tmode_devs(struct ahd_softc *ahd, struct cam_sim *sim, union ccb *ccb,
10266 		    struct ahd_tmode_tstate **tstate,
10267 		    struct ahd_tmode_lstate **lstate,
10268 		    int notfound_failure)
10269 {
10270 
10271 	if ((ahd->features & AHD_TARGETMODE) == 0)
10272 		return (CAM_REQ_INVALID);
10273 
10274 	/*
10275 	 * Handle the 'black hole' device that sucks up
10276 	 * requests to unattached luns on enabled targets.
10277 	 */
10278 	if (ccb->ccb_h.target_id == CAM_TARGET_WILDCARD
10279 	 && ccb->ccb_h.target_lun == CAM_LUN_WILDCARD) {
10280 		*tstate = NULL;
10281 		*lstate = ahd->black_hole;
10282 	} else {
10283 		u_int max_id;
10284 
10285 		max_id = (ahd->features & AHD_WIDE) ? 16 : 8;
10286 		if (ccb->ccb_h.target_id >= max_id)
10287 			return (CAM_TID_INVALID);
10288 
10289 		if (ccb->ccb_h.target_lun >= AHD_NUM_LUNS)
10290 			return (CAM_LUN_INVALID);
10291 
10292 		*tstate = ahd->enabled_targets[ccb->ccb_h.target_id];
10293 		*lstate = NULL;
10294 		if (*tstate != NULL)
10295 			*lstate =
10296 			    (*tstate)->enabled_luns[ccb->ccb_h.target_lun];
10297 	}
10298 
10299 	if (notfound_failure != 0 && *lstate == NULL)
10300 		return (CAM_PATH_INVALID);
10301 
10302 	return (CAM_REQ_CMP);
10303 }
10304 
10305 void
10306 ahd_handle_en_lun(struct ahd_softc *ahd, struct cam_sim *sim, union ccb *ccb)
10307 {
10308 #if NOT_YET
10309 	struct	   ahd_tmode_tstate *tstate;
10310 	struct	   ahd_tmode_lstate *lstate;
10311 	struct	   ccb_en_lun *cel;
10312 	cam_status status;
10313 	u_int	   target;
10314 	u_int	   lun;
10315 	u_int	   target_mask;
10316 	u_long	   s;
10317 	char	   channel;
10318 
10319 	status = ahd_find_tmode_devs(ahd, sim, ccb, &tstate, &lstate,
10320 				     /*notfound_failure*/FALSE);
10321 
10322 	if (status != CAM_REQ_CMP) {
10323 		ccb->ccb_h.status = status;
10324 		return;
10325 	}
10326 
10327 	if ((ahd->features & AHD_MULTIROLE) != 0) {
10328 		u_int	   our_id;
10329 
10330 		our_id = ahd->our_id;
10331 		if (ccb->ccb_h.target_id != our_id) {
10332 			if ((ahd->features & AHD_MULTI_TID) != 0
10333 		   	 && (ahd->flags & AHD_INITIATORROLE) != 0) {
10334 				/*
10335 				 * Only allow additional targets if
10336 				 * the initiator role is disabled.
10337 				 * The hardware cannot handle a re-select-in
10338 				 * on the initiator id during a re-select-out
10339 				 * on a different target id.
10340 				 */
10341 				status = CAM_TID_INVALID;
10342 			} else if ((ahd->flags & AHD_INITIATORROLE) != 0
10343 				|| ahd->enabled_luns > 0) {
10344 				/*
10345 				 * Only allow our target id to change
10346 				 * if the initiator role is not configured
10347 				 * and there are no enabled luns which
10348 				 * are attached to the currently registered
10349 				 * scsi id.
10350 				 */
10351 				status = CAM_TID_INVALID;
10352 			}
10353 		}
10354 	}
10355 
10356 	if (status != CAM_REQ_CMP) {
10357 		ccb->ccb_h.status = status;
10358 		return;
10359 	}
10360 
10361 	/*
10362 	 * We now have an id that is valid.
10363 	 * If we aren't in target mode, switch modes.
10364 	 */
10365 	if ((ahd->flags & AHD_TARGETROLE) == 0
10366 	 && ccb->ccb_h.target_id != CAM_TARGET_WILDCARD) {
10367 		u_long	s;
10368 
10369 		printk("Configuring Target Mode\n");
10370 		ahd_lock(ahd, &s);
10371 		if (LIST_FIRST(&ahd->pending_scbs) != NULL) {
10372 			ccb->ccb_h.status = CAM_BUSY;
10373 			ahd_unlock(ahd, &s);
10374 			return;
10375 		}
10376 		ahd->flags |= AHD_TARGETROLE;
10377 		if ((ahd->features & AHD_MULTIROLE) == 0)
10378 			ahd->flags &= ~AHD_INITIATORROLE;
10379 		ahd_pause(ahd);
10380 		ahd_loadseq(ahd);
10381 		ahd_restart(ahd);
10382 		ahd_unlock(ahd, &s);
10383 	}
10384 	cel = &ccb->cel;
10385 	target = ccb->ccb_h.target_id;
10386 	lun = ccb->ccb_h.target_lun;
10387 	channel = SIM_CHANNEL(ahd, sim);
10388 	target_mask = 0x01 << target;
10389 	if (channel == 'B')
10390 		target_mask <<= 8;
10391 
10392 	if (cel->enable != 0) {
10393 		u_int scsiseq1;
10394 
10395 		/* Are we already enabled?? */
10396 		if (lstate != NULL) {
10397 			xpt_print_path(ccb->ccb_h.path);
10398 			printk("Lun already enabled\n");
10399 			ccb->ccb_h.status = CAM_LUN_ALRDY_ENA;
10400 			return;
10401 		}
10402 
10403 		if (cel->grp6_len != 0
10404 		 || cel->grp7_len != 0) {
10405 			/*
10406 			 * Don't (yet?) support vendor
10407 			 * specific commands.
10408 			 */
10409 			ccb->ccb_h.status = CAM_REQ_INVALID;
10410 			printk("Non-zero Group Codes\n");
10411 			return;
10412 		}
10413 
10414 		/*
10415 		 * Seems to be okay.
10416 		 * Setup our data structures.
10417 		 */
10418 		if (target != CAM_TARGET_WILDCARD && tstate == NULL) {
10419 			tstate = ahd_alloc_tstate(ahd, target, channel);
10420 			if (tstate == NULL) {
10421 				xpt_print_path(ccb->ccb_h.path);
10422 				printk("Couldn't allocate tstate\n");
10423 				ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
10424 				return;
10425 			}
10426 		}
10427 		lstate = kzalloc(sizeof(*lstate), GFP_ATOMIC);
10428 		if (lstate == NULL) {
10429 			xpt_print_path(ccb->ccb_h.path);
10430 			printk("Couldn't allocate lstate\n");
10431 			ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
10432 			return;
10433 		}
10434 		status = xpt_create_path(&lstate->path, /*periph*/NULL,
10435 					 xpt_path_path_id(ccb->ccb_h.path),
10436 					 xpt_path_target_id(ccb->ccb_h.path),
10437 					 xpt_path_lun_id(ccb->ccb_h.path));
10438 		if (status != CAM_REQ_CMP) {
10439 			kfree(lstate);
10440 			xpt_print_path(ccb->ccb_h.path);
10441 			printk("Couldn't allocate path\n");
10442 			ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
10443 			return;
10444 		}
10445 		SLIST_INIT(&lstate->accept_tios);
10446 		SLIST_INIT(&lstate->immed_notifies);
10447 		ahd_lock(ahd, &s);
10448 		ahd_pause(ahd);
10449 		if (target != CAM_TARGET_WILDCARD) {
10450 			tstate->enabled_luns[lun] = lstate;
10451 			ahd->enabled_luns++;
10452 
10453 			if ((ahd->features & AHD_MULTI_TID) != 0) {
10454 				u_int targid_mask;
10455 
10456 				targid_mask = ahd_inw(ahd, TARGID);
10457 				targid_mask |= target_mask;
10458 				ahd_outw(ahd, TARGID, targid_mask);
10459 				ahd_update_scsiid(ahd, targid_mask);
10460 			} else {
10461 				u_int our_id;
10462 				char  channel;
10463 
10464 				channel = SIM_CHANNEL(ahd, sim);
10465 				our_id = SIM_SCSI_ID(ahd, sim);
10466 
10467 				/*
10468 				 * This can only happen if selections
10469 				 * are not enabled
10470 				 */
10471 				if (target != our_id) {
10472 					u_int sblkctl;
10473 					char  cur_channel;
10474 					int   swap;
10475 
10476 					sblkctl = ahd_inb(ahd, SBLKCTL);
10477 					cur_channel = (sblkctl & SELBUSB)
10478 						    ? 'B' : 'A';
10479 					if ((ahd->features & AHD_TWIN) == 0)
10480 						cur_channel = 'A';
10481 					swap = cur_channel != channel;
10482 					ahd->our_id = target;
10483 
10484 					if (swap)
10485 						ahd_outb(ahd, SBLKCTL,
10486 							 sblkctl ^ SELBUSB);
10487 
10488 					ahd_outb(ahd, SCSIID, target);
10489 
10490 					if (swap)
10491 						ahd_outb(ahd, SBLKCTL, sblkctl);
10492 				}
10493 			}
10494 		} else
10495 			ahd->black_hole = lstate;
10496 		/* Allow select-in operations */
10497 		if (ahd->black_hole != NULL && ahd->enabled_luns > 0) {
10498 			scsiseq1 = ahd_inb(ahd, SCSISEQ_TEMPLATE);
10499 			scsiseq1 |= ENSELI;
10500 			ahd_outb(ahd, SCSISEQ_TEMPLATE, scsiseq1);
10501 			scsiseq1 = ahd_inb(ahd, SCSISEQ1);
10502 			scsiseq1 |= ENSELI;
10503 			ahd_outb(ahd, SCSISEQ1, scsiseq1);
10504 		}
10505 		ahd_unpause(ahd);
10506 		ahd_unlock(ahd, &s);
10507 		ccb->ccb_h.status = CAM_REQ_CMP;
10508 		xpt_print_path(ccb->ccb_h.path);
10509 		printk("Lun now enabled for target mode\n");
10510 	} else {
10511 		struct scb *scb;
10512 		int i, empty;
10513 
10514 		if (lstate == NULL) {
10515 			ccb->ccb_h.status = CAM_LUN_INVALID;
10516 			return;
10517 		}
10518 
10519 		ahd_lock(ahd, &s);
10520 
10521 		ccb->ccb_h.status = CAM_REQ_CMP;
10522 		LIST_FOREACH(scb, &ahd->pending_scbs, pending_links) {
10523 			struct ccb_hdr *ccbh;
10524 
10525 			ccbh = &scb->io_ctx->ccb_h;
10526 			if (ccbh->func_code == XPT_CONT_TARGET_IO
10527 			 && !xpt_path_comp(ccbh->path, ccb->ccb_h.path)){
10528 				printk("CTIO pending\n");
10529 				ccb->ccb_h.status = CAM_REQ_INVALID;
10530 				ahd_unlock(ahd, &s);
10531 				return;
10532 			}
10533 		}
10534 
10535 		if (SLIST_FIRST(&lstate->accept_tios) != NULL) {
10536 			printk("ATIOs pending\n");
10537 			ccb->ccb_h.status = CAM_REQ_INVALID;
10538 		}
10539 
10540 		if (SLIST_FIRST(&lstate->immed_notifies) != NULL) {
10541 			printk("INOTs pending\n");
10542 			ccb->ccb_h.status = CAM_REQ_INVALID;
10543 		}
10544 
10545 		if (ccb->ccb_h.status != CAM_REQ_CMP) {
10546 			ahd_unlock(ahd, &s);
10547 			return;
10548 		}
10549 
10550 		xpt_print_path(ccb->ccb_h.path);
10551 		printk("Target mode disabled\n");
10552 		xpt_free_path(lstate->path);
10553 		kfree(lstate);
10554 
10555 		ahd_pause(ahd);
10556 		/* Can we clean up the target too? */
10557 		if (target != CAM_TARGET_WILDCARD) {
10558 			tstate->enabled_luns[lun] = NULL;
10559 			ahd->enabled_luns--;
10560 			for (empty = 1, i = 0; i < 8; i++)
10561 				if (tstate->enabled_luns[i] != NULL) {
10562 					empty = 0;
10563 					break;
10564 				}
10565 
10566 			if (empty) {
10567 				ahd_free_tstate(ahd, target, channel,
10568 						/*force*/FALSE);
10569 				if (ahd->features & AHD_MULTI_TID) {
10570 					u_int targid_mask;
10571 
10572 					targid_mask = ahd_inw(ahd, TARGID);
10573 					targid_mask &= ~target_mask;
10574 					ahd_outw(ahd, TARGID, targid_mask);
10575 					ahd_update_scsiid(ahd, targid_mask);
10576 				}
10577 			}
10578 		} else {
10579 
10580 			ahd->black_hole = NULL;
10581 
10582 			/*
10583 			 * We can't allow selections without
10584 			 * our black hole device.
10585 			 */
10586 			empty = TRUE;
10587 		}
10588 		if (ahd->enabled_luns == 0) {
10589 			/* Disallow select-in */
10590 			u_int scsiseq1;
10591 
10592 			scsiseq1 = ahd_inb(ahd, SCSISEQ_TEMPLATE);
10593 			scsiseq1 &= ~ENSELI;
10594 			ahd_outb(ahd, SCSISEQ_TEMPLATE, scsiseq1);
10595 			scsiseq1 = ahd_inb(ahd, SCSISEQ1);
10596 			scsiseq1 &= ~ENSELI;
10597 			ahd_outb(ahd, SCSISEQ1, scsiseq1);
10598 
10599 			if ((ahd->features & AHD_MULTIROLE) == 0) {
10600 				printk("Configuring Initiator Mode\n");
10601 				ahd->flags &= ~AHD_TARGETROLE;
10602 				ahd->flags |= AHD_INITIATORROLE;
10603 				ahd_pause(ahd);
10604 				ahd_loadseq(ahd);
10605 				ahd_restart(ahd);
10606 				/*
10607 				 * Unpaused.  The extra unpause
10608 				 * that follows is harmless.
10609 				 */
10610 			}
10611 		}
10612 		ahd_unpause(ahd);
10613 		ahd_unlock(ahd, &s);
10614 	}
10615 #endif
10616 }
10617 
10618 static void
10619 ahd_update_scsiid(struct ahd_softc *ahd, u_int targid_mask)
10620 {
10621 #if NOT_YET
10622 	u_int scsiid_mask;
10623 	u_int scsiid;
10624 
10625 	if ((ahd->features & AHD_MULTI_TID) == 0)
10626 		panic("ahd_update_scsiid called on non-multitid unit\n");
10627 
10628 	/*
10629 	 * Since we will rely on the TARGID mask
10630 	 * for selection enables, ensure that OID
10631 	 * in SCSIID is not set to some other ID
10632 	 * that we don't want to allow selections on.
10633 	 */
10634 	if ((ahd->features & AHD_ULTRA2) != 0)
10635 		scsiid = ahd_inb(ahd, SCSIID_ULTRA2);
10636 	else
10637 		scsiid = ahd_inb(ahd, SCSIID);
10638 	scsiid_mask = 0x1 << (scsiid & OID);
10639 	if ((targid_mask & scsiid_mask) == 0) {
10640 		u_int our_id;
10641 
10642 		/* ffs counts from 1 */
10643 		our_id = ffs(targid_mask);
10644 		if (our_id == 0)
10645 			our_id = ahd->our_id;
10646 		else
10647 			our_id--;
10648 		scsiid &= TID;
10649 		scsiid |= our_id;
10650 	}
10651 	if ((ahd->features & AHD_ULTRA2) != 0)
10652 		ahd_outb(ahd, SCSIID_ULTRA2, scsiid);
10653 	else
10654 		ahd_outb(ahd, SCSIID, scsiid);
10655 #endif
10656 }
10657 
10658 static void
10659 ahd_run_tqinfifo(struct ahd_softc *ahd, int paused)
10660 {
10661 	struct target_cmd *cmd;
10662 
10663 	ahd_sync_tqinfifo(ahd, BUS_DMASYNC_POSTREAD);
10664 	while ((cmd = &ahd->targetcmds[ahd->tqinfifonext])->cmd_valid != 0) {
10665 
10666 		/*
10667 		 * Only advance through the queue if we
10668 		 * have the resources to process the command.
10669 		 */
10670 		if (ahd_handle_target_cmd(ahd, cmd) != 0)
10671 			break;
10672 
10673 		cmd->cmd_valid = 0;
10674 		ahd_dmamap_sync(ahd, ahd->shared_data_dmat,
10675 				ahd->shared_data_map.dmamap,
10676 				ahd_targetcmd_offset(ahd, ahd->tqinfifonext),
10677 				sizeof(struct target_cmd),
10678 				BUS_DMASYNC_PREREAD);
10679 		ahd->tqinfifonext++;
10680 
10681 		/*
10682 		 * Lazily update our position in the target mode incoming
10683 		 * command queue as seen by the sequencer.
10684 		 */
10685 		if ((ahd->tqinfifonext & (HOST_TQINPOS - 1)) == 1) {
10686 			u_int hs_mailbox;
10687 
10688 			hs_mailbox = ahd_inb(ahd, HS_MAILBOX);
10689 			hs_mailbox &= ~HOST_TQINPOS;
10690 			hs_mailbox |= ahd->tqinfifonext & HOST_TQINPOS;
10691 			ahd_outb(ahd, HS_MAILBOX, hs_mailbox);
10692 		}
10693 	}
10694 }
10695 
10696 static int
10697 ahd_handle_target_cmd(struct ahd_softc *ahd, struct target_cmd *cmd)
10698 {
10699 	struct	  ahd_tmode_tstate *tstate;
10700 	struct	  ahd_tmode_lstate *lstate;
10701 	struct	  ccb_accept_tio *atio;
10702 	uint8_t *byte;
10703 	int	  initiator;
10704 	int	  target;
10705 	int	  lun;
10706 
10707 	initiator = SCSIID_TARGET(ahd, cmd->scsiid);
10708 	target = SCSIID_OUR_ID(cmd->scsiid);
10709 	lun    = (cmd->identify & MSG_IDENTIFY_LUNMASK);
10710 
10711 	byte = cmd->bytes;
10712 	tstate = ahd->enabled_targets[target];
10713 	lstate = NULL;
10714 	if (tstate != NULL)
10715 		lstate = tstate->enabled_luns[lun];
10716 
10717 	/*
10718 	 * Commands for disabled luns go to the black hole driver.
10719 	 */
10720 	if (lstate == NULL)
10721 		lstate = ahd->black_hole;
10722 
10723 	atio = (struct ccb_accept_tio*)SLIST_FIRST(&lstate->accept_tios);
10724 	if (atio == NULL) {
10725 		ahd->flags |= AHD_TQINFIFO_BLOCKED;
10726 		/*
10727 		 * Wait for more ATIOs from the peripheral driver for this lun.
10728 		 */
10729 		return (1);
10730 	} else
10731 		ahd->flags &= ~AHD_TQINFIFO_BLOCKED;
10732 #ifdef AHD_DEBUG
10733 	if ((ahd_debug & AHD_SHOW_TQIN) != 0)
10734 		printk("Incoming command from %d for %d:%d%s\n",
10735 		       initiator, target, lun,
10736 		       lstate == ahd->black_hole ? "(Black Holed)" : "");
10737 #endif
10738 	SLIST_REMOVE_HEAD(&lstate->accept_tios, sim_links.sle);
10739 
10740 	if (lstate == ahd->black_hole) {
10741 		/* Fill in the wildcards */
10742 		atio->ccb_h.target_id = target;
10743 		atio->ccb_h.target_lun = lun;
10744 	}
10745 
10746 	/*
10747 	 * Package it up and send it off to
10748 	 * whomever has this lun enabled.
10749 	 */
10750 	atio->sense_len = 0;
10751 	atio->init_id = initiator;
10752 	if (byte[0] != 0xFF) {
10753 		/* Tag was included */
10754 		atio->tag_action = *byte++;
10755 		atio->tag_id = *byte++;
10756 		atio->ccb_h.flags = CAM_TAG_ACTION_VALID;
10757 	} else {
10758 		atio->ccb_h.flags = 0;
10759 	}
10760 	byte++;
10761 
10762 	/* Okay.  Now determine the cdb size based on the command code */
10763 	switch (*byte >> CMD_GROUP_CODE_SHIFT) {
10764 	case 0:
10765 		atio->cdb_len = 6;
10766 		break;
10767 	case 1:
10768 	case 2:
10769 		atio->cdb_len = 10;
10770 		break;
10771 	case 4:
10772 		atio->cdb_len = 16;
10773 		break;
10774 	case 5:
10775 		atio->cdb_len = 12;
10776 		break;
10777 	case 3:
10778 	default:
10779 		/* Only copy the opcode. */
10780 		atio->cdb_len = 1;
10781 		printk("Reserved or VU command code type encountered\n");
10782 		break;
10783 	}
10784 
10785 	memcpy(atio->cdb_io.cdb_bytes, byte, atio->cdb_len);
10786 
10787 	atio->ccb_h.status |= CAM_CDB_RECVD;
10788 
10789 	if ((cmd->identify & MSG_IDENTIFY_DISCFLAG) == 0) {
10790 		/*
10791 		 * We weren't allowed to disconnect.
10792 		 * We're hanging on the bus until a
10793 		 * continue target I/O comes in response
10794 		 * to this accept tio.
10795 		 */
10796 #ifdef AHD_DEBUG
10797 		if ((ahd_debug & AHD_SHOW_TQIN) != 0)
10798 			printk("Received Immediate Command %d:%d:%d - %p\n",
10799 			       initiator, target, lun, ahd->pending_device);
10800 #endif
10801 		ahd->pending_device = lstate;
10802 		ahd_freeze_ccb((union ccb *)atio);
10803 		atio->ccb_h.flags |= CAM_DIS_DISCONNECT;
10804 	}
10805 	xpt_done((union ccb*)atio);
10806 	return (0);
10807 }
10808 
10809 #endif
10810