xref: /freebsd/sys/dev/tws/tws.c (revision a3cbca537ef1d8ac03a693cd51d98fb8087acc8d)
1 /*
2  * Copyright (c) 2010, LSI Corp.
3  * All rights reserved.
4  * Author : Manjunath Ranganathaiah
5  * Support: freebsdraid@lsi.com
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  *
11  * 1. Redistributions of source code must retain the above copyright
12  *    notice, this list of conditions and the following disclaimer.
13  * 2. Redistributions in binary form must reproduce the above copyright
14  *    notice, this list of conditions and the following disclaimer in
15  *    the documentation and/or other materials provided with the
16  *    distribution.
17  * 3. Neither the name of the <ORGANIZATION> nor the names of its
18  *    contributors may be used to endorse or promote products derived
19  *    from this software without specific prior written permission.
20  *
21  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
24  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
25  * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
26  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
27  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
28  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
29  * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
30  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
31  * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
32  * POSSIBILITY OF SUCH DAMAGE.
33  */
34 
35 #include <sys/cdefs.h>
36 __FBSDID("$FreeBSD$");
37 
38 #include <dev/tws/tws.h>
39 #include <dev/tws/tws_services.h>
40 #include <dev/tws/tws_hdm.h>
41 
42 #include <cam/cam.h>
43 #include <cam/cam_ccb.h>
44 
45 MALLOC_DEFINE(M_TWS, "twsbuf", "buffers used by tws driver");
46 int tws_queue_depth = TWS_MAX_REQS;
47 int tws_enable_msi = 0;
48 int tws_enable_msix = 0;
49 
50 
51 
52 /* externs */
53 extern int tws_cam_attach(struct tws_softc *sc);
54 extern void tws_cam_detach(struct tws_softc *sc);
55 extern int tws_init_ctlr(struct tws_softc *sc);
56 extern boolean tws_ctlr_ready(struct tws_softc *sc);
57 extern void tws_turn_off_interrupts(struct tws_softc *sc);
58 extern void tws_q_insert_tail(struct tws_softc *sc, struct tws_request *req,
59                                 u_int8_t q_type );
60 extern struct tws_request *tws_q_remove_request(struct tws_softc *sc,
61                                    struct tws_request *req, u_int8_t q_type );
62 extern struct tws_request *tws_q_remove_head(struct tws_softc *sc,
63                                                        u_int8_t q_type );
64 extern boolean tws_get_response(struct tws_softc *sc, u_int16_t *req_id);
65 extern boolean tws_ctlr_reset(struct tws_softc *sc);
66 extern void tws_intr(void *arg);
67 extern int tws_use_32bit_sgls;
68 
69 
70 struct tws_request *tws_get_request(struct tws_softc *sc, u_int16_t type);
71 int tws_init_connect(struct tws_softc *sc, u_int16_t mc);
72 void tws_send_event(struct tws_softc *sc, u_int8_t event);
73 uint8_t tws_get_state(struct tws_softc *sc);
74 void tws_release_request(struct tws_request *req);
75 
76 
77 
78 /* Function prototypes */
79 static d_open_t     tws_open;
80 static d_close_t    tws_close;
81 static d_read_t     tws_read;
82 static d_write_t    tws_write;
83 extern d_ioctl_t    tws_ioctl;
84 
85 static int tws_init(struct tws_softc *sc);
86 static void tws_dmamap_cmds_load_cbfn(void *arg, bus_dma_segment_t *segs,
87                            int nseg, int error);
88 
89 static int tws_init_reqs(struct tws_softc *sc, u_int32_t dma_mem_size);
90 static int tws_init_aen_q(struct tws_softc *sc);
91 static int tws_init_trace_q(struct tws_softc *sc);
92 static int tws_setup_irq(struct tws_softc *sc);
93 int tws_setup_intr(struct tws_softc *sc, int irqs);
94 int tws_teardown_intr(struct tws_softc *sc);
95 
96 
97 /* Character device entry points */
98 
99 static struct cdevsw tws_cdevsw = {
100     .d_version =    D_VERSION,
101     .d_open =   tws_open,
102     .d_close =  tws_close,
103     .d_read =   tws_read,
104     .d_write =  tws_write,
105     .d_ioctl =  tws_ioctl,
106     .d_name =   "tws",
107 };
108 
109 /*
110  * In the cdevsw routines, we find our softc by using the si_drv1 member
111  * of struct cdev.  We set this variable to point to our softc in our
112  * attach routine when we create the /dev entry.
113  */
114 
115 int
116 tws_open(struct cdev *dev, int oflags, int devtype, d_thread_t *td)
117 {
118     struct tws_softc *sc = dev->si_drv1;
119 
120     if ( sc )
121         TWS_TRACE_DEBUG(sc, "entry", dev, oflags);
122     return (0);
123 }
124 
125 int
126 tws_close(struct cdev *dev, int fflag, int devtype, d_thread_t *td)
127 {
128     struct tws_softc *sc = dev->si_drv1;
129 
130     if ( sc )
131         TWS_TRACE_DEBUG(sc, "entry", dev, fflag);
132     return (0);
133 }
134 
135 int
136 tws_read(struct cdev *dev, struct uio *uio, int ioflag)
137 {
138     struct tws_softc *sc = dev->si_drv1;
139 
140     if ( sc )
141         TWS_TRACE_DEBUG(sc, "entry", dev, ioflag);
142     return (0);
143 }
144 
145 int
146 tws_write(struct cdev *dev, struct uio *uio, int ioflag)
147 {
148     struct tws_softc *sc = dev->si_drv1;
149 
150     if ( sc )
151         TWS_TRACE_DEBUG(sc, "entry", dev, ioflag);
152     return (0);
153 }
154 
155 /* PCI Support Functions */
156 
157 /*
158  * Compare the device ID of this device against the IDs that this driver
159  * supports.  If there is a match, set the description and return success.
160  */
161 static int
162 tws_probe(device_t dev)
163 {
164     static u_int8_t first_ctlr = 1;
165 
166     if ((pci_get_vendor(dev) == TWS_VENDOR_ID) &&
167         (pci_get_device(dev) == TWS_DEVICE_ID)) {
168         device_set_desc(dev, "LSI 3ware SAS/SATA Storage Controller");
169         if (first_ctlr) {
170             printf("LSI 3ware device driver for SAS/SATA storage "
171                     "controllers, version: %s\n", TWS_DRIVER_VERSION_STRING);
172             first_ctlr = 0;
173         }
174 
175         return(BUS_PROBE_DEFAULT);
176     }
177     return (ENXIO);
178 }
179 
180 /* Attach function is only called if the probe is successful. */
181 
182 static int
183 tws_attach(device_t dev)
184 {
185     struct tws_softc *sc = device_get_softc(dev);
186     u_int32_t bar;
187     int error=0,i;
188 
189     /* no tracing yet */
190     /* Look up our softc and initialize its fields. */
191     sc->tws_dev = dev;
192     sc->device_id = pci_get_device(dev);
193     sc->subvendor_id = pci_get_subvendor(dev);
194     sc->subdevice_id = pci_get_subdevice(dev);
195 
196     /* Intialize mutexes */
197     mtx_init( &sc->q_lock, "tws_q_lock", NULL, MTX_DEF);
198     mtx_init( &sc->sim_lock,  "tws_sim_lock", NULL, MTX_DEF);
199     mtx_init( &sc->gen_lock,  "tws_gen_lock", NULL, MTX_DEF);
200     mtx_init( &sc->io_lock,  "tws_io_lock", NULL, MTX_DEF | MTX_RECURSE);
201 
202     if ( tws_init_trace_q(sc) == FAILURE )
203         printf("trace init failure\n");
204     /* send init event */
205     mtx_lock(&sc->gen_lock);
206     tws_send_event(sc, TWS_INIT_START);
207     mtx_unlock(&sc->gen_lock);
208 
209 
210 #if _BYTE_ORDER == _BIG_ENDIAN
211     TWS_TRACE(sc, "BIG endian", 0, 0);
212 #endif
213     /* sysctl context setup */
214     sysctl_ctx_init(&sc->tws_clist);
215     sc->tws_oidp = SYSCTL_ADD_NODE(&sc->tws_clist,
216                                    SYSCTL_STATIC_CHILDREN(_hw), OID_AUTO,
217                                    device_get_nameunit(dev),
218                                    CTLFLAG_RD, 0, "");
219     if ( sc->tws_oidp == NULL ) {
220         tws_log(sc, SYSCTL_TREE_NODE_ADD);
221         goto attach_fail_1;
222     }
223     SYSCTL_ADD_STRING(&sc->tws_clist, SYSCTL_CHILDREN(sc->tws_oidp),
224                       OID_AUTO, "driver_version", CTLFLAG_RD,
225                       TWS_DRIVER_VERSION_STRING, 0, "TWS driver version");
226 
227     pci_enable_busmaster(dev);
228 
229     bar = pci_read_config(dev, TWS_PCI_BAR0, 4);
230     TWS_TRACE_DEBUG(sc, "bar0 ", bar, 0);
231     bar = pci_read_config(dev, TWS_PCI_BAR1, 4);
232     bar = bar & ~TWS_BIT2;
233     TWS_TRACE_DEBUG(sc, "bar1 ", bar, 0);
234 
235     /* MFA base address is BAR2 register used for
236      * push mode. Firmware will evatualy move to
237      * pull mode during witch this needs to change
238      */
239 #ifndef TWS_PULL_MODE_ENABLE
240     sc->mfa_base = (u_int64_t)pci_read_config(dev, TWS_PCI_BAR2, 4);
241     sc->mfa_base = sc->mfa_base & ~TWS_BIT2;
242     TWS_TRACE_DEBUG(sc, "bar2 ", sc->mfa_base, 0);
243 #endif
244 
245     /* allocate MMIO register space */
246     sc->reg_res_id = TWS_PCI_BAR1; /* BAR1 offset */
247     if ((sc->reg_res = bus_alloc_resource(dev, SYS_RES_MEMORY,
248                                 &(sc->reg_res_id), 0, ~0, 1, RF_ACTIVE))
249                                 == NULL) {
250         tws_log(sc, ALLOC_MEMORY_RES);
251         goto attach_fail_1;
252     }
253     sc->bus_tag = rman_get_bustag(sc->reg_res);
254     sc->bus_handle = rman_get_bushandle(sc->reg_res);
255 
256 #ifndef TWS_PULL_MODE_ENABLE
257     /* Allocate bus space for inbound mfa */
258     sc->mfa_res_id = TWS_PCI_BAR2; /* BAR2 offset */
259     if ((sc->mfa_res = bus_alloc_resource(dev, SYS_RES_MEMORY,
260                           &(sc->mfa_res_id), 0, ~0, 0x100000, RF_ACTIVE))
261                                 == NULL) {
262         tws_log(sc, ALLOC_MEMORY_RES);
263         goto attach_fail_2;
264     }
265     sc->bus_mfa_tag = rman_get_bustag(sc->mfa_res);
266     sc->bus_mfa_handle = rman_get_bushandle(sc->mfa_res);
267 #endif
268 
269     /* Allocate and register our interrupt. */
270     sc->intr_type = TWS_INTx; /* default */
271 
272     if ( tws_enable_msi )
273         sc->intr_type = TWS_MSI;
274     if ( tws_setup_irq(sc) == FAILURE ) {
275         tws_log(sc, ALLOC_MEMORY_RES);
276         goto attach_fail_3;
277     }
278 
279     /*
280      * Create a /dev entry for this device.  The kernel will assign us
281      * a major number automatically.  We use the unit number of this
282      * device as the minor number and name the character device
283      * "tws<unit>".
284      */
285     sc->tws_cdev = make_dev(&tws_cdevsw, device_get_unit(dev),
286         UID_ROOT, GID_OPERATOR, S_IRUSR | S_IWUSR, "tws%u",
287         device_get_unit(dev));
288     sc->tws_cdev->si_drv1 = sc;
289 
290     if ( tws_init(sc) == FAILURE ) {
291         tws_log(sc, TWS_INIT_FAILURE);
292         goto attach_fail_4;
293     }
294     if ( tws_init_ctlr(sc) == FAILURE ) {
295         tws_log(sc, TWS_CTLR_INIT_FAILURE);
296         goto attach_fail_4;
297     }
298     if ((error = tws_cam_attach(sc))) {
299         tws_log(sc, TWS_CAM_ATTACH);
300         goto attach_fail_4;
301     }
302     /* send init complete event */
303     mtx_lock(&sc->gen_lock);
304     tws_send_event(sc, TWS_INIT_COMPLETE);
305     mtx_unlock(&sc->gen_lock);
306 
307     TWS_TRACE_DEBUG(sc, "attached successfully", 0, sc->device_id);
308     return(0);
309 
310 attach_fail_4:
311     tws_teardown_intr(sc);
312     destroy_dev(sc->tws_cdev);
313     if (sc->dma_mem_phys)
314 	    bus_dmamap_unload(sc->cmd_tag, sc->cmd_map);
315     if (sc->dma_mem)
316 	    bus_dmamem_free(sc->cmd_tag, sc->dma_mem, sc->cmd_map);
317     if (sc->cmd_tag)
318 	    bus_dma_tag_destroy(sc->cmd_tag);
319 attach_fail_3:
320     for(i=0;i<sc->irqs;i++) {
321         if ( sc->irq_res[i] ){
322             if (bus_release_resource(sc->tws_dev,
323                  SYS_RES_IRQ, sc->irq_res_id[i], sc->irq_res[i]))
324                 TWS_TRACE(sc, "bus irq res", 0, 0);
325         }
326     }
327 #ifndef TWS_PULL_MODE_ENABLE
328 attach_fail_2:
329 #endif
330     if ( sc->mfa_res ){
331         if (bus_release_resource(sc->tws_dev,
332                  SYS_RES_MEMORY, sc->mfa_res_id, sc->mfa_res))
333             TWS_TRACE(sc, "bus release ", 0, sc->mfa_res_id);
334     }
335     if ( sc->reg_res ){
336         if (bus_release_resource(sc->tws_dev,
337                  SYS_RES_MEMORY, sc->reg_res_id, sc->reg_res))
338             TWS_TRACE(sc, "bus release2 ", 0, sc->reg_res_id);
339     }
340 attach_fail_1:
341     mtx_destroy(&sc->q_lock);
342     mtx_destroy(&sc->sim_lock);
343     mtx_destroy(&sc->gen_lock);
344     mtx_destroy(&sc->io_lock);
345     sysctl_ctx_free(&sc->tws_clist);
346     return (ENXIO);
347 }
348 
349 /* Detach device. */
350 
351 static int
352 tws_detach(device_t dev)
353 {
354     struct tws_softc *sc = device_get_softc(dev);
355     int i;
356     u_int32_t reg;
357 
358     TWS_TRACE_DEBUG(sc, "entry", 0, 0);
359 
360     mtx_lock(&sc->gen_lock);
361     tws_send_event(sc, TWS_UNINIT_START);
362     mtx_unlock(&sc->gen_lock);
363 
364     /* needs to disable interrupt before detaching from cam */
365     tws_turn_off_interrupts(sc);
366     /* clear door bell */
367     tws_write_reg(sc, TWS_I2O0_HOBDBC, ~0, 4);
368     reg = tws_read_reg(sc, TWS_I2O0_HIMASK, 4);
369     TWS_TRACE_DEBUG(sc, "turn-off-intr", reg, 0);
370     sc->obfl_q_overrun = false;
371     tws_init_connect(sc, 1);
372 
373     /* Teardown the state in our softc created in our attach routine. */
374     /* Disconnect the interrupt handler. */
375     tws_teardown_intr(sc);
376 
377     /* Release irq resource */
378     for(i=0;i<sc->irqs;i++) {
379         if ( sc->irq_res[i] ){
380             if (bus_release_resource(sc->tws_dev,
381                      SYS_RES_IRQ, sc->irq_res_id[i], sc->irq_res[i]))
382                 TWS_TRACE(sc, "bus release irq resource",
383                                        i, sc->irq_res_id[i]);
384         }
385     }
386     if ( sc->intr_type == TWS_MSI ) {
387         pci_release_msi(sc->tws_dev);
388     }
389 
390     tws_cam_detach(sc);
391 
392     if (sc->dma_mem_phys)
393 	    bus_dmamap_unload(sc->cmd_tag, sc->cmd_map);
394     if (sc->dma_mem)
395 	    bus_dmamem_free(sc->cmd_tag, sc->dma_mem, sc->cmd_map);
396     if (sc->cmd_tag)
397 	    bus_dma_tag_destroy(sc->cmd_tag);
398 
399     /* Release memory resource */
400     if ( sc->mfa_res ){
401         if (bus_release_resource(sc->tws_dev,
402                  SYS_RES_MEMORY, sc->mfa_res_id, sc->mfa_res))
403             TWS_TRACE(sc, "bus release mem resource", 0, sc->mfa_res_id);
404     }
405     if ( sc->reg_res ){
406         if (bus_release_resource(sc->tws_dev,
407                  SYS_RES_MEMORY, sc->reg_res_id, sc->reg_res))
408             TWS_TRACE(sc, "bus release mem resource", 0, sc->reg_res_id);
409     }
410 
411     free(sc->reqs, M_TWS);
412     free(sc->sense_bufs, M_TWS);
413     free(sc->scan_ccb, M_TWS);
414     if (sc->ioctl_data_mem)
415             bus_dmamem_free(sc->data_tag, sc->ioctl_data_mem, sc->ioctl_data_map);
416     free(sc->aen_q.q, M_TWS);
417     free(sc->trace_q.q, M_TWS);
418     mtx_destroy(&sc->q_lock);
419     mtx_destroy(&sc->sim_lock);
420     mtx_destroy(&sc->gen_lock);
421     mtx_destroy(&sc->io_lock);
422     destroy_dev(sc->tws_cdev);
423     sysctl_ctx_free(&sc->tws_clist);
424     return (0);
425 }
426 
427 int
428 tws_setup_intr(struct tws_softc *sc, int irqs)
429 {
430     int i, error;
431 
432     for(i=0;i<irqs;i++) {
433         if (!(sc->intr_handle[i])) {
434             if ((error = bus_setup_intr(sc->tws_dev, sc->irq_res[i],
435                                     INTR_TYPE_CAM | INTR_MPSAFE,
436 #if (__FreeBSD_version >= 700000)
437                                     NULL,
438 #endif
439                                     tws_intr, sc, &sc->intr_handle[i]))) {
440                 tws_log(sc, SETUP_INTR_RES);
441                 return(FAILURE);
442             }
443         }
444     }
445     return(SUCCESS);
446 
447 }
448 
449 
450 int
451 tws_teardown_intr(struct tws_softc *sc)
452 {
453     int i, error;
454 
455     for(i=0;i<sc->irqs;i++) {
456         if (sc->intr_handle[i]) {
457             error = bus_teardown_intr(sc->tws_dev,
458                                       sc->irq_res[i], sc->intr_handle[i]);
459             sc->intr_handle[i] = NULL;
460         }
461     }
462     return(SUCCESS);
463 }
464 
465 
466 static int
467 tws_setup_irq(struct tws_softc *sc)
468 {
469     int messages;
470 
471     switch(sc->intr_type) {
472         case TWS_INTx :
473             sc->irqs = 1;
474             sc->irq_res_id[0] = 0;
475             sc->irq_res[0] = bus_alloc_resource_any(sc->tws_dev, SYS_RES_IRQ,
476                             &sc->irq_res_id[0], RF_SHAREABLE | RF_ACTIVE);
477             if ( ! sc->irq_res[0] )
478                 return(FAILURE);
479             if ( tws_setup_intr(sc, sc->irqs) == FAILURE )
480                 return(FAILURE);
481             device_printf(sc->tws_dev, "Using legacy INTx\n");
482             break;
483         case TWS_MSI :
484             sc->irqs = 1;
485             sc->irq_res_id[0] = 1;
486             messages = 1;
487             if (pci_alloc_msi(sc->tws_dev, &messages) != 0 ) {
488                 TWS_TRACE(sc, "pci alloc msi fail", 0, messages);
489                 return(FAILURE);
490             }
491             sc->irq_res[0] = bus_alloc_resource_any(sc->tws_dev, SYS_RES_IRQ,
492                               &sc->irq_res_id[0], RF_SHAREABLE | RF_ACTIVE);
493 
494             if ( !sc->irq_res[0]  )
495                 return(FAILURE);
496             if ( tws_setup_intr(sc, sc->irqs) == FAILURE )
497                 return(FAILURE);
498             device_printf(sc->tws_dev, "Using MSI\n");
499             break;
500 
501     }
502 
503     return(SUCCESS);
504 }
505 
506 static int
507 tws_init(struct tws_softc *sc)
508 {
509 
510     u_int32_t max_sg_elements;
511     u_int32_t dma_mem_size;
512     int error;
513     u_int32_t reg;
514 
515     sc->seq_id = 0;
516     if ( tws_queue_depth > TWS_MAX_REQS )
517         tws_queue_depth = TWS_MAX_REQS;
518     if (tws_queue_depth < TWS_RESERVED_REQS+1)
519         tws_queue_depth = TWS_RESERVED_REQS+1;
520     sc->is64bit = (sizeof(bus_addr_t) == 8) ? true : false;
521     max_sg_elements = (sc->is64bit && !tws_use_32bit_sgls) ?
522                                  TWS_MAX_64BIT_SG_ELEMENTS :
523                                  TWS_MAX_32BIT_SG_ELEMENTS;
524     dma_mem_size = (sizeof(struct tws_command_packet) * tws_queue_depth) +
525                              (TWS_SECTOR_SIZE) ;
526     if ( bus_dma_tag_create(bus_get_dma_tag(sc->tws_dev), /* PCI parent */
527                             TWS_ALIGNMENT,           /* alignment */
528                             0,                       /* boundary */
529                             BUS_SPACE_MAXADDR_32BIT, /* lowaddr */
530                             BUS_SPACE_MAXADDR,       /* highaddr */
531                             NULL, NULL,              /* filter, filterarg */
532                             BUS_SPACE_MAXSIZE,       /* maxsize */
533                             max_sg_elements,         /* numsegs */
534                             BUS_SPACE_MAXSIZE,       /* maxsegsize */
535                             0,                       /* flags */
536                             NULL, NULL,              /* lockfunc, lockfuncarg */
537                             &sc->parent_tag          /* tag */
538                            )) {
539         TWS_TRACE_DEBUG(sc, "DMA parent tag Create fail", max_sg_elements,
540                                                     sc->is64bit);
541         return(ENOMEM);
542     }
543     /* In bound message frame requires 16byte alignment.
544      * Outbound MF's can live with 4byte alignment - for now just
545      * use 16 for both.
546      */
547     if ( bus_dma_tag_create(sc->parent_tag,       /* parent */
548                             TWS_IN_MF_ALIGNMENT,  /* alignment */
549                             0,                    /* boundary */
550                             BUS_SPACE_MAXADDR_32BIT, /* lowaddr */
551                             BUS_SPACE_MAXADDR,    /* highaddr */
552                             NULL, NULL,           /* filter, filterarg */
553                             dma_mem_size,         /* maxsize */
554                             1,                    /* numsegs */
555                             BUS_SPACE_MAXSIZE,    /* maxsegsize */
556                             0,                    /* flags */
557                             NULL, NULL,           /* lockfunc, lockfuncarg */
558                             &sc->cmd_tag          /* tag */
559                            )) {
560         TWS_TRACE_DEBUG(sc, "DMA cmd tag Create fail", max_sg_elements, sc->is64bit);
561         return(ENOMEM);
562     }
563 
564     if (bus_dmamem_alloc(sc->cmd_tag, &sc->dma_mem,
565                     BUS_DMA_NOWAIT, &sc->cmd_map)) {
566         TWS_TRACE_DEBUG(sc, "DMA mem alloc fail", max_sg_elements, sc->is64bit);
567         return(ENOMEM);
568     }
569 
570     /* if bus_dmamem_alloc succeeds then bus_dmamap_load will succeed */
571     sc->dma_mem_phys=0;
572     error = bus_dmamap_load(sc->cmd_tag, sc->cmd_map, sc->dma_mem,
573                     dma_mem_size, tws_dmamap_cmds_load_cbfn,
574                     &sc->dma_mem_phys, 0);
575 
576    /*
577     * Create a dma tag for data buffers; size will be the maximum
578     * possible I/O size (128kB).
579     */
580     if (bus_dma_tag_create(sc->parent_tag,         /* parent */
581                            TWS_ALIGNMENT,          /* alignment */
582                            0,                      /* boundary */
583                            BUS_SPACE_MAXADDR_32BIT,/* lowaddr */
584                            BUS_SPACE_MAXADDR,      /* highaddr */
585                            NULL, NULL,             /* filter, filterarg */
586                            TWS_MAX_IO_SIZE,        /* maxsize */
587                            max_sg_elements,        /* nsegments */
588                            TWS_MAX_IO_SIZE,        /* maxsegsize */
589                            BUS_DMA_ALLOCNOW,       /* flags */
590                            busdma_lock_mutex,      /* lockfunc */
591                            &sc->io_lock,           /* lockfuncarg */
592                            &sc->data_tag           /* tag */)) {
593         TWS_TRACE_DEBUG(sc, "DMA cmd tag Create fail", max_sg_elements, sc->is64bit);
594         return(ENOMEM);
595     }
596 
597     sc->reqs = malloc(sizeof(struct tws_request) * tws_queue_depth, M_TWS,
598                       M_WAITOK | M_ZERO);
599     if ( sc->reqs == NULL ) {
600         TWS_TRACE_DEBUG(sc, "malloc failed", 0, sc->is64bit);
601         return(ENOMEM);
602     }
603     sc->sense_bufs = malloc(sizeof(struct tws_sense) * tws_queue_depth, M_TWS,
604                       M_WAITOK | M_ZERO);
605     if ( sc->sense_bufs == NULL ) {
606         TWS_TRACE_DEBUG(sc, "sense malloc failed", 0, sc->is64bit);
607         return(ENOMEM);
608     }
609     sc->scan_ccb = malloc(sizeof(union ccb), M_TWS, M_WAITOK | M_ZERO);
610     if ( sc->scan_ccb == NULL ) {
611         TWS_TRACE_DEBUG(sc, "ccb malloc failed", 0, sc->is64bit);
612         return(ENOMEM);
613     }
614     if (bus_dmamem_alloc(sc->data_tag, (void **)&sc->ioctl_data_mem,
615             (BUS_DMA_NOWAIT | BUS_DMA_ZERO), &sc->ioctl_data_map)) {
616         device_printf(sc->tws_dev, "Cannot allocate ioctl data mem\n");
617         return(ENOMEM);
618     }
619 
620     if ( !tws_ctlr_ready(sc) )
621         if( !tws_ctlr_reset(sc) )
622             return(FAILURE);
623 
624     bzero(&sc->stats, sizeof(struct tws_stats));
625     tws_init_qs(sc);
626     tws_turn_off_interrupts(sc);
627 
628     /*
629      * enable pull mode by setting bit1 .
630      * setting bit0 to 1 will enable interrupt coalesing
631      * will revisit.
632      */
633 
634 #ifdef TWS_PULL_MODE_ENABLE
635 
636     reg = tws_read_reg(sc, TWS_I2O0_CTL, 4);
637     TWS_TRACE_DEBUG(sc, "i20 ctl", reg, TWS_I2O0_CTL);
638     tws_write_reg(sc, TWS_I2O0_CTL, reg | TWS_BIT1, 4);
639 
640 #endif
641 
642     TWS_TRACE_DEBUG(sc, "dma_mem_phys", sc->dma_mem_phys, TWS_I2O0_CTL);
643     if ( tws_init_reqs(sc, dma_mem_size) == FAILURE )
644         return(FAILURE);
645     if ( tws_init_aen_q(sc) == FAILURE )
646         return(FAILURE);
647 
648     return(SUCCESS);
649 
650 }
651 
652 static int
653 tws_init_aen_q(struct tws_softc *sc)
654 {
655     sc->aen_q.head=0;
656     sc->aen_q.tail=0;
657     sc->aen_q.depth=256;
658     sc->aen_q.overflow=0;
659     sc->aen_q.q = malloc(sizeof(struct tws_event_packet)*sc->aen_q.depth,
660                               M_TWS, M_WAITOK | M_ZERO);
661     if ( ! sc->aen_q.q )
662         return(FAILURE);
663     return(SUCCESS);
664 }
665 
666 static int
667 tws_init_trace_q(struct tws_softc *sc)
668 {
669     sc->trace_q.head=0;
670     sc->trace_q.tail=0;
671     sc->trace_q.depth=256;
672     sc->trace_q.overflow=0;
673     sc->trace_q.q = malloc(sizeof(struct tws_trace_rec)*sc->trace_q.depth,
674                               M_TWS, M_WAITOK | M_ZERO);
675     if ( ! sc->trace_q.q )
676         return(FAILURE);
677     return(SUCCESS);
678 }
679 
680 static int
681 tws_init_reqs(struct tws_softc *sc, u_int32_t dma_mem_size)
682 {
683 
684     struct tws_command_packet *cmd_buf;
685     cmd_buf = (struct tws_command_packet *)sc->dma_mem;
686     int i;
687 
688     bzero(cmd_buf, dma_mem_size);
689     TWS_TRACE_DEBUG(sc, "phy cmd", sc->dma_mem_phys, 0);
690     mtx_lock(&sc->q_lock);
691     for ( i=0; i< tws_queue_depth; i++)
692     {
693         if (bus_dmamap_create(sc->data_tag, 0, &sc->reqs[i].dma_map)) {
694             /* log a ENOMEM failure msg here */
695             mtx_unlock(&sc->q_lock);
696             return(FAILURE);
697         }
698         sc->reqs[i].cmd_pkt =  &cmd_buf[i];
699 
700         sc->sense_bufs[i].hdr = &cmd_buf[i].hdr ;
701         sc->sense_bufs[i].hdr_pkt_phy = sc->dma_mem_phys +
702                               (i * sizeof(struct tws_command_packet));
703 
704         sc->reqs[i].cmd_pkt_phy = sc->dma_mem_phys +
705                               sizeof(struct tws_command_header) +
706                               (i * sizeof(struct tws_command_packet));
707         sc->reqs[i].request_id = i;
708         sc->reqs[i].sc = sc;
709 
710         sc->reqs[i].cmd_pkt->hdr.header_desc.size_header = 128;
711 
712 	callout_handle_init(&sc->reqs[i].thandle);
713         sc->reqs[i].state = TWS_REQ_STATE_FREE;
714         if ( i >= TWS_RESERVED_REQS )
715             tws_q_insert_tail(sc, &sc->reqs[i], TWS_FREE_Q);
716     }
717     mtx_unlock(&sc->q_lock);
718     return(SUCCESS);
719 }
720 
721 static void
722 tws_dmamap_cmds_load_cbfn(void *arg, bus_dma_segment_t *segs,
723                            int nseg, int error)
724 {
725 
726     /* printf("command load done \n"); */
727 
728     *((bus_addr_t *)arg) = segs[0].ds_addr;
729 }
730 
731 void
732 tws_send_event(struct tws_softc *sc, u_int8_t event)
733 {
734     mtx_assert(&sc->gen_lock, MA_OWNED);
735     TWS_TRACE_DEBUG(sc, "received event ", 0, event);
736     switch (event) {
737 
738         case TWS_INIT_START:
739             sc->tws_state = TWS_INIT;
740             break;
741 
742         case TWS_INIT_COMPLETE:
743             if (sc->tws_state != TWS_INIT) {
744                 device_printf(sc->tws_dev, "invalid state transition %d => TWS_ONLINE\n", sc->tws_state);
745             } else {
746                 sc->tws_state = TWS_ONLINE;
747             }
748             break;
749 
750         case TWS_RESET_START:
751             /* We can transition to reset state from any state except reset*/
752             if (sc->tws_state != TWS_RESET) {
753                 sc->tws_prev_state = sc->tws_state;
754                 sc->tws_state = TWS_RESET;
755             }
756             break;
757 
758         case TWS_RESET_COMPLETE:
759             if (sc->tws_state != TWS_RESET) {
760                 device_printf(sc->tws_dev, "invalid state transition %d => %d (previous state)\n", sc->tws_state, sc->tws_prev_state);
761             } else {
762                 sc->tws_state = sc->tws_prev_state;
763             }
764             break;
765 
766         case TWS_SCAN_FAILURE:
767             if (sc->tws_state != TWS_ONLINE) {
768                 device_printf(sc->tws_dev, "invalid state transition %d => TWS_OFFLINE\n", sc->tws_state);
769             } else {
770                 sc->tws_state = TWS_OFFLINE;
771             }
772             break;
773 
774         case TWS_UNINIT_START:
775             if ((sc->tws_state != TWS_ONLINE) && (sc->tws_state != TWS_OFFLINE)) {
776                 device_printf(sc->tws_dev, "invalid state transition %d => TWS_UNINIT\n", sc->tws_state);
777             } else {
778                 sc->tws_state = TWS_UNINIT;
779             }
780             break;
781     }
782 
783 }
784 
785 uint8_t
786 tws_get_state(struct tws_softc *sc)
787 {
788 
789     return((u_int8_t)sc->tws_state);
790 
791 }
792 
793 /* Called during system shutdown after sync. */
794 
795 static int
796 tws_shutdown(device_t dev)
797 {
798 
799     struct tws_softc *sc = device_get_softc(dev);
800 
801     TWS_TRACE_DEBUG(sc, "entry", 0, 0);
802 
803     tws_turn_off_interrupts(sc);
804     tws_init_connect(sc, 1);
805 
806     return (0);
807 }
808 
809 /*
810  * Device suspend routine.
811  */
812 static int
813 tws_suspend(device_t dev)
814 {
815     struct tws_softc *sc = device_get_softc(dev);
816 
817     if ( sc )
818         TWS_TRACE_DEBUG(sc, "entry", 0, 0);
819     return (0);
820 }
821 
822 /*
823  * Device resume routine.
824  */
825 static int
826 tws_resume(device_t dev)
827 {
828 
829     struct tws_softc *sc = device_get_softc(dev);
830 
831     if ( sc )
832         TWS_TRACE_DEBUG(sc, "entry", 0, 0);
833     return (0);
834 }
835 
836 
837 struct tws_request *
838 tws_get_request(struct tws_softc *sc, u_int16_t type)
839 {
840     struct mtx *my_mutex = ((type == TWS_REQ_TYPE_SCSI_IO) ? &sc->q_lock : &sc->gen_lock);
841     struct tws_request *r = NULL;
842 
843     mtx_lock(my_mutex);
844 
845     if (type == TWS_REQ_TYPE_SCSI_IO) {
846         r = tws_q_remove_head(sc, TWS_FREE_Q);
847     } else {
848         if ( sc->reqs[type].state == TWS_REQ_STATE_FREE ) {
849             r = &sc->reqs[type];
850         }
851     }
852 
853     if ( r ) {
854         bzero(&r->cmd_pkt->cmd, sizeof(struct tws_command_apache));
855         r->data = NULL;
856         r->length = 0;
857         r->type = type;
858         r->flags = TWS_DIR_UNKNOWN;
859         r->error_code = TWS_REQ_RET_INVALID;
860         r->cb = NULL;
861         r->ccb_ptr = NULL;
862         r->thandle.callout = NULL;
863         r->next = r->prev = NULL;
864 
865         r->state = ((type == TWS_REQ_TYPE_SCSI_IO) ? TWS_REQ_STATE_TRAN : TWS_REQ_STATE_BUSY);
866     }
867 
868     mtx_unlock(my_mutex);
869 
870     return(r);
871 }
872 
873 void
874 tws_release_request(struct tws_request *req)
875 {
876 
877     struct tws_softc *sc = req->sc;
878 
879     TWS_TRACE_DEBUG(sc, "entry", sc, 0);
880     mtx_lock(&sc->q_lock);
881     tws_q_insert_tail(sc, req, TWS_FREE_Q);
882     mtx_unlock(&sc->q_lock);
883 }
884 
885 static device_method_t tws_methods[] = {
886     /* Device interface */
887     DEVMETHOD(device_probe,     tws_probe),
888     DEVMETHOD(device_attach,    tws_attach),
889     DEVMETHOD(device_detach,    tws_detach),
890     DEVMETHOD(device_shutdown,  tws_shutdown),
891     DEVMETHOD(device_suspend,   tws_suspend),
892     DEVMETHOD(device_resume,    tws_resume),
893 
894     DEVMETHOD_END
895 };
896 
897 static driver_t tws_driver = {
898         "tws",
899         tws_methods,
900         sizeof(struct tws_softc)
901 };
902 
903 
904 static devclass_t tws_devclass;
905 
906 /* DEFINE_CLASS_0(tws, tws_driver, tws_methods, sizeof(struct tws_softc)); */
907 DRIVER_MODULE(tws, pci, tws_driver, tws_devclass, 0, 0);
908 MODULE_DEPEND(tws, cam, 1, 1, 1);
909 MODULE_DEPEND(tws, pci, 1, 1, 1);
910 
911 TUNABLE_INT("hw.tws.queue_depth", &tws_queue_depth);
912 TUNABLE_INT("hw.tws.enable_msi", &tws_enable_msi);
913