xref: /freebsd/sys/powerpc/pseries/phyp_vscsi.c (revision 63d1fd5970ec814904aa0f4580b10a0d302d08b2)
1 /*-
2  * Copyright 2013 Nathan Whitehorn
3  * All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  * 1. Redistributions of source code must retain the above copyright
9  *    notice, this list of conditions and the following disclaimer.
10  * 2. Redistributions in binary form must reproduce the above copyright
11  *    notice, this list of conditions and the following disclaimer in the
12  *    documentation and/or other materials provided with the distribution.
13  *
14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24  * SUCH DAMAGE.
25  */
26 
27 #include <sys/cdefs.h>
28 __FBSDID("$FreeBSD$");
29 
30 #include <sys/param.h>
31 #include <sys/systm.h>
32 #include <sys/kernel.h>
33 #include <sys/malloc.h>
34 #include <sys/module.h>
35 #include <sys/selinfo.h>
36 #include <sys/bus.h>
37 #include <sys/conf.h>
38 #include <sys/eventhandler.h>
39 #include <sys/rman.h>
40 #include <sys/bus_dma.h>
41 #include <sys/bio.h>
42 #include <sys/ioccom.h>
43 #include <sys/uio.h>
44 #include <sys/proc.h>
45 #include <sys/signalvar.h>
46 #include <sys/sysctl.h>
47 #include <sys/endian.h>
48 #include <sys/vmem.h>
49 
50 #include <cam/cam.h>
51 #include <cam/cam_ccb.h>
52 #include <cam/cam_debug.h>
53 #include <cam/cam_periph.h>
54 #include <cam/cam_sim.h>
55 #include <cam/cam_xpt_periph.h>
56 #include <cam/cam_xpt_sim.h>
57 #include <cam/scsi/scsi_all.h>
58 #include <cam/scsi/scsi_message.h>
59 
60 #include <dev/ofw/openfirm.h>
61 #include <dev/ofw/ofw_bus.h>
62 #include <dev/ofw/ofw_bus_subr.h>
63 
64 #include <machine/bus.h>
65 #include <machine/resource.h>
66 
67 #include <powerpc/pseries/phyp-hvcall.h>
68 
69 struct vscsi_softc;
70 
71 /* VSCSI CRQ format from table 260 of PAPR spec 2.4 (page 760) */
72 struct vscsi_crq {
73 	uint8_t valid;
74 	uint8_t format;
75 	uint8_t reserved;
76 	uint8_t status;
77 	uint16_t timeout;
78 	uint16_t iu_length;
79 	uint64_t iu_data;
80 };
81 
82 struct vscsi_xfer {
83         TAILQ_ENTRY(vscsi_xfer) queue;
84         struct vscsi_softc *sc;
85         union ccb *ccb;
86         bus_dmamap_t dmamap;
87         uint64_t tag;
88 
89 	vmem_addr_t srp_iu_offset;
90 	vmem_size_t srp_iu_size;
91 };
92 
93 TAILQ_HEAD(vscsi_xferq, vscsi_xfer);
94 
95 struct vscsi_softc {
96 	device_t	dev;
97 	struct cam_devq *devq;
98 	struct cam_sim	*sim;
99 	struct cam_path	*path;
100 	struct mtx io_lock;
101 
102 	cell_t		unit;
103 	int		bus_initialized;
104 	int		bus_logged_in;
105 	int		max_transactions;
106 
107 	int		irqid;
108 	struct resource	*irq;
109 	void		*irq_cookie;
110 
111 	bus_dma_tag_t	crq_tag;
112 	struct vscsi_crq *crq_queue;
113 	int		n_crqs, cur_crq;
114 	bus_dmamap_t	crq_map;
115 	bus_addr_t	crq_phys;
116 
117 	vmem_t		*srp_iu_arena;
118 	void		*srp_iu_queue;
119 	bus_addr_t	srp_iu_phys;
120 
121 	bus_dma_tag_t	data_tag;
122 
123 	struct vscsi_xfer loginxp;
124 	struct vscsi_xfer *xfer;
125 	struct vscsi_xferq active_xferq;
126 	struct vscsi_xferq free_xferq;
127 };
128 
129 struct srp_login {
130 	uint8_t type;
131 	uint8_t reserved[7];
132 	uint64_t tag;
133 	uint64_t max_cmd_length;
134 	uint32_t reserved2;
135 	uint16_t buffer_formats;
136 	uint8_t flags;
137 	uint8_t reserved3[5];
138 	uint8_t initiator_port_id[16];
139 	uint8_t target_port_id[16];
140 } __packed;
141 
142 struct srp_login_rsp {
143 	uint8_t type;
144 	uint8_t reserved[3];
145 	uint32_t request_limit_delta;
146 	uint8_t tag;
147 	uint32_t max_i_to_t_len;
148 	uint32_t max_t_to_i_len;
149 	uint16_t buffer_formats;
150 	uint8_t flags;
151 	/* Some reserved bits follow */
152 } __packed;
153 
154 struct srp_cmd {
155 	uint8_t type;
156 	uint8_t flags1;
157 	uint8_t reserved[3];
158 	uint8_t formats;
159 	uint8_t out_buffer_count;
160 	uint8_t in_buffer_count;
161 	uint64_t tag;
162 	uint32_t reserved2;
163 	uint64_t lun;
164 	uint8_t reserved3[3];
165 	uint8_t additional_cdb;
166 	uint8_t cdb[16];
167 	uint8_t data_payload[0];
168 } __packed;
169 
170 struct srp_rsp {
171 	uint8_t type;
172 	uint8_t reserved[3];
173 	uint32_t request_limit_delta;
174 	uint64_t tag;
175 	uint16_t reserved2;
176 	uint8_t flags;
177 	uint8_t status;
178 	uint32_t data_out_resid;
179 	uint32_t data_in_resid;
180 	uint32_t sense_data_len;
181 	uint32_t response_data_len;
182 	uint8_t data_payload[0];
183 } __packed;
184 
185 struct srp_tsk_mgmt {
186 	uint8_t type;
187 	uint8_t reserved[7];
188 	uint64_t tag;
189 	uint32_t reserved2;
190 	uint64_t lun;
191 	uint8_t reserved3[2];
192 	uint8_t function;
193 	uint8_t reserved4;
194 	uint64_t manage_tag;
195 	uint64_t reserved5;
196 } __packed;
197 
198 /* Message code type */
199 #define SRP_LOGIN_REQ	0x00
200 #define SRP_TSK_MGMT	0x01
201 #define SRP_CMD		0x02
202 #define SRP_I_LOGOUT	0x03
203 
204 #define SRP_LOGIN_RSP	0xC0
205 #define SRP_RSP		0xC1
206 #define SRP_LOGIN_REJ	0xC2
207 
208 #define SRP_T_LOGOUT	0x80
209 #define SRP_CRED_REQ	0x81
210 #define SRP_AER_REQ	0x82
211 
212 #define SRP_CRED_RSP	0x41
213 #define SRP_AER_RSP	0x41
214 
215 /* Flags for srp_rsp flags field */
216 #define SRP_RSPVALID	0x01
217 #define SRP_SNSVALID	0x02
218 #define SRP_DOOVER	0x04
219 #define SRP_DOUNDER	0x08
220 #define SRP_DIOVER	0x10
221 #define SRP_DIUNDER	0x20
222 
223 #define	MAD_SUCESS			0x00
224 #define	MAD_NOT_SUPPORTED		0xf1
225 #define	MAD_FAILED			0xf7
226 
227 #define	MAD_EMPTY_IU			0x01
228 #define	MAD_ERROR_LOGGING_REQUEST	0x02
229 #define	MAD_ADAPTER_INFO_REQUEST	0x03
230 #define	MAD_CAPABILITIES_EXCHANGE	0x05
231 #define	MAD_PHYS_ADAP_INFO_REQUEST	0x06
232 #define	MAD_TAPE_PASSTHROUGH_REQUEST	0x07
233 #define	MAD_ENABLE_FAST_FAIL		0x08
234 
235 static int	vscsi_probe(device_t);
236 static int	vscsi_attach(device_t);
237 static int	vscsi_detach(device_t);
238 static void	vscsi_cam_action(struct cam_sim *, union ccb *);
239 static void	vscsi_cam_poll(struct cam_sim *);
240 static void	vscsi_intr(void *arg);
241 static void	vscsi_check_response_queue(struct vscsi_softc *sc);
242 static void	vscsi_setup_bus(struct vscsi_softc *sc);
243 
244 static void	vscsi_srp_login(struct vscsi_softc *sc);
245 static void	vscsi_crq_load_cb(void *, bus_dma_segment_t *, int, int);
246 static void	vscsi_scsi_command(void *xxp, bus_dma_segment_t *segs,
247 		    int nsegs, int err);
248 static void	vscsi_task_management(struct vscsi_softc *sc, union ccb *ccb);
249 static void	vscsi_srp_response(struct vscsi_xfer *, struct vscsi_crq *);
250 
251 static devclass_t	vscsi_devclass;
252 static device_method_t	vscsi_methods[] = {
253 	DEVMETHOD(device_probe,		vscsi_probe),
254 	DEVMETHOD(device_attach,	vscsi_attach),
255 	DEVMETHOD(device_detach,	vscsi_detach),
256 
257 	DEVMETHOD_END
258 };
259 static driver_t vscsi_driver = {
260 	"vscsi",
261 	vscsi_methods,
262 	sizeof(struct vscsi_softc)
263 };
264 DRIVER_MODULE(vscsi, vdevice, vscsi_driver, vscsi_devclass, 0, 0);
265 MALLOC_DEFINE(M_VSCSI, "vscsi", "CAM device queue for VSCSI");
266 
267 static int
268 vscsi_probe(device_t dev)
269 {
270 
271 	if (!ofw_bus_is_compatible(dev, "IBM,v-scsi"))
272 		return (ENXIO);
273 
274 	device_set_desc(dev, "POWER Hypervisor Virtual SCSI Bus");
275 	return (0);
276 }
277 
278 static int
279 vscsi_attach(device_t dev)
280 {
281 	struct vscsi_softc *sc;
282 	struct vscsi_xfer *xp;
283 	int error, i;
284 
285 	sc = device_get_softc(dev);
286 	if (sc == NULL)
287 		return (EINVAL);
288 
289 	sc->dev = dev;
290 	mtx_init(&sc->io_lock, "vscsi", NULL, MTX_DEF);
291 
292 	/* Get properties */
293 	OF_getencprop(ofw_bus_get_node(dev), "reg", &sc->unit,
294 	    sizeof(sc->unit));
295 
296 	/* Setup interrupt */
297 	sc->irqid = 0;
298 	sc->irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &sc->irqid,
299 	    RF_ACTIVE);
300 
301 	if (!sc->irq) {
302 		device_printf(dev, "Could not allocate IRQ\n");
303 		mtx_destroy(&sc->io_lock);
304 		return (ENXIO);
305 	}
306 
307 	bus_setup_intr(dev, sc->irq, INTR_TYPE_CAM | INTR_MPSAFE |
308 	    INTR_ENTROPY, NULL, vscsi_intr, sc, &sc->irq_cookie);
309 
310 	/* Data DMA */
311 	error = bus_dma_tag_create(bus_get_dma_tag(dev), 1, 0,
312 	    BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL, BUS_SPACE_MAXSIZE,
313 	    256, BUS_SPACE_MAXSIZE_32BIT, 0, busdma_lock_mutex, &sc->io_lock,
314 	    &sc->data_tag);
315 
316 	TAILQ_INIT(&sc->active_xferq);
317 	TAILQ_INIT(&sc->free_xferq);
318 
319 	/* First XFER for login data */
320 	sc->loginxp.sc = sc;
321 	bus_dmamap_create(sc->data_tag, 0, &sc->loginxp.dmamap);
322 	TAILQ_INSERT_TAIL(&sc->free_xferq, &sc->loginxp, queue);
323 
324 	/* CRQ area */
325 	error = bus_dma_tag_create(bus_get_dma_tag(dev), PAGE_SIZE, 0,
326 	    BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL, 8*PAGE_SIZE,
327 	    1, BUS_SPACE_MAXSIZE, 0, NULL, NULL, &sc->crq_tag);
328 	error = bus_dmamem_alloc(sc->crq_tag, (void **)&sc->crq_queue,
329 	    BUS_DMA_WAITOK | BUS_DMA_ZERO, &sc->crq_map);
330 	sc->crq_phys = 0;
331 	sc->n_crqs = 0;
332 	error = bus_dmamap_load(sc->crq_tag, sc->crq_map, sc->crq_queue,
333 	    8*PAGE_SIZE, vscsi_crq_load_cb, sc, 0);
334 
335 	mtx_lock(&sc->io_lock);
336 	vscsi_setup_bus(sc);
337 	sc->xfer = malloc(sizeof(sc->xfer[0])*sc->max_transactions, M_VSCSI,
338 	    M_NOWAIT);
339 	for (i = 0; i < sc->max_transactions; i++) {
340 		xp = &sc->xfer[i];
341 		xp->sc = sc;
342 
343 		error = bus_dmamap_create(sc->data_tag, 0, &xp->dmamap);
344 		if (error) {
345 			device_printf(dev, "Could not create DMA map (%d)\n",
346 			    error);
347 			break;
348 		}
349 
350 		TAILQ_INSERT_TAIL(&sc->free_xferq, xp, queue);
351 	}
352 	mtx_unlock(&sc->io_lock);
353 
354 	/* Allocate CAM bits */
355 	if ((sc->devq = cam_simq_alloc(sc->max_transactions)) == NULL)
356 		return (ENOMEM);
357 
358 	sc->sim = cam_sim_alloc(vscsi_cam_action, vscsi_cam_poll, "vscsi", sc,
359 				device_get_unit(dev), &sc->io_lock,
360 				sc->max_transactions, sc->max_transactions,
361 				sc->devq);
362 	if (sc->sim == NULL) {
363 		cam_simq_free(sc->devq);
364 		sc->devq = NULL;
365 		device_printf(dev, "CAM SIM attach failed\n");
366 		return (EINVAL);
367 	}
368 
369 
370 	mtx_lock(&sc->io_lock);
371 	if (xpt_bus_register(sc->sim, dev, 0) != 0) {
372 		device_printf(dev, "XPT bus registration failed\n");
373 		cam_sim_free(sc->sim, FALSE);
374 		sc->sim = NULL;
375 		cam_simq_free(sc->devq);
376 		sc->devq = NULL;
377 		mtx_unlock(&sc->io_lock);
378 		return (EINVAL);
379 	}
380 	mtx_unlock(&sc->io_lock);
381 
382 	return (0);
383 }
384 
385 static int
386 vscsi_detach(device_t dev)
387 {
388 	struct vscsi_softc *sc;
389 
390 	sc = device_get_softc(dev);
391 	if (sc == NULL)
392 		return (EINVAL);
393 
394 	if (sc->sim != NULL) {
395 		mtx_lock(&sc->io_lock);
396 		xpt_bus_deregister(cam_sim_path(sc->sim));
397 		cam_sim_free(sc->sim, FALSE);
398 		sc->sim = NULL;
399 		mtx_unlock(&sc->io_lock);
400 	}
401 
402 	if (sc->devq != NULL) {
403 		cam_simq_free(sc->devq);
404 		sc->devq = NULL;
405 	}
406 
407 	mtx_destroy(&sc->io_lock);
408 
409 	return (0);
410 }
411 
412 static void
413 vscsi_cam_action(struct cam_sim *sim, union ccb *ccb)
414 {
415 	struct vscsi_softc *sc = cam_sim_softc(sim);
416 
417 	mtx_assert(&sc->io_lock, MA_OWNED);
418 
419 	switch (ccb->ccb_h.func_code) {
420 	case XPT_PATH_INQ:
421 	{
422 		struct ccb_pathinq *cpi = &ccb->cpi;
423 
424 		cpi->version_num = 1;
425 		cpi->hba_inquiry = PI_TAG_ABLE;
426 		cpi->hba_misc = PIM_EXTLUNS;
427 		cpi->target_sprt = 0;
428 		cpi->hba_eng_cnt = 0;
429 		cpi->max_target = 0;
430 		cpi->max_lun = 0;
431 		cpi->initiator_id = ~0;
432 		strlcpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN);
433 		strlcpy(cpi->hba_vid, "IBM", HBA_IDLEN);
434 		strlcpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN);
435 		cpi->unit_number = cam_sim_unit(sim);
436 		cpi->bus_id = cam_sim_bus(sim);
437 		cpi->base_transfer_speed = 150000;
438 		cpi->transport = XPORT_SRP;
439 		cpi->transport_version = 0;
440 		cpi->protocol = PROTO_SCSI;
441 		cpi->protocol_version = SCSI_REV_SPC4;
442 		cpi->ccb_h.status = CAM_REQ_CMP;
443 		break;
444 	}
445 	case XPT_RESET_BUS:
446 		ccb->ccb_h.status = CAM_REQ_CMP;
447 		break;
448 	case XPT_RESET_DEV:
449 		ccb->ccb_h.status = CAM_REQ_INPROG;
450 		vscsi_task_management(sc, ccb);
451 		return;
452 	case XPT_GET_TRAN_SETTINGS:
453 		ccb->cts.protocol = PROTO_SCSI;
454 		ccb->cts.protocol_version = SCSI_REV_SPC4;
455 		ccb->cts.transport = XPORT_SRP;
456 		ccb->cts.transport_version = 0;
457 		ccb->cts.proto_specific.valid = 0;
458 		ccb->cts.xport_specific.valid = 0;
459 		ccb->ccb_h.status = CAM_REQ_CMP;
460 		break;
461 	case XPT_SET_TRAN_SETTINGS:
462 		ccb->ccb_h.status = CAM_FUNC_NOTAVAIL;
463 		break;
464 	case XPT_SCSI_IO:
465 	{
466 		struct vscsi_xfer *xp;
467 
468 		ccb->ccb_h.status = CAM_REQ_INPROG;
469 
470 		xp = TAILQ_FIRST(&sc->free_xferq);
471 		if (xp == NULL)
472 			panic("SCSI queue flooded");
473 		xp->ccb = ccb;
474 		TAILQ_REMOVE(&sc->free_xferq, xp, queue);
475 		TAILQ_INSERT_TAIL(&sc->active_xferq, xp, queue);
476 		bus_dmamap_load_ccb(sc->data_tag, xp->dmamap,
477 		    ccb, vscsi_scsi_command, xp, 0);
478 
479 		return;
480 	}
481 	default:
482 		ccb->ccb_h.status = CAM_REQ_INVALID;
483 		break;
484 	}
485 
486 	xpt_done(ccb);
487 	return;
488 }
489 
490 static void
491 vscsi_srp_login(struct vscsi_softc *sc)
492 {
493 	struct vscsi_xfer *xp;
494 	struct srp_login *login;
495 	struct vscsi_crq crq;
496 	int err;
497 
498 	mtx_assert(&sc->io_lock, MA_OWNED);
499 
500 	xp = TAILQ_FIRST(&sc->free_xferq);
501 	if (xp == NULL)
502 		panic("SCSI queue flooded");
503 	xp->ccb = NULL;
504 	TAILQ_REMOVE(&sc->free_xferq, xp, queue);
505 	TAILQ_INSERT_TAIL(&sc->active_xferq, xp, queue);
506 
507 	/* Set up command */
508 	xp->srp_iu_size = crq.iu_length = 64;
509 	err = vmem_alloc(xp->sc->srp_iu_arena, xp->srp_iu_size,
510 	    M_BESTFIT | M_NOWAIT, &xp->srp_iu_offset);
511 	if (err)
512 		panic("Error during VMEM allocation (%d)", err);
513 
514 	login = (struct srp_login *)((uint8_t *)xp->sc->srp_iu_queue +
515 	    (uintptr_t)xp->srp_iu_offset);
516 	bzero(login, xp->srp_iu_size);
517 	login->type = SRP_LOGIN_REQ;
518 	login->tag = (uint64_t)(xp);
519 	login->max_cmd_length = htobe64(256);
520 	login->buffer_formats = htobe16(0x1 | 0x2); /* Direct and indirect */
521 	login->flags = 0;
522 
523 	/* Create CRQ entry */
524 	crq.valid = 0x80;
525 	crq.format = 0x01;
526 	crq.iu_data = xp->sc->srp_iu_phys + xp->srp_iu_offset;
527 	bus_dmamap_sync(sc->crq_tag, sc->crq_map, BUS_DMASYNC_PREWRITE);
528 
529 	err = phyp_hcall(H_SEND_CRQ, xp->sc->unit, ((uint64_t *)(&crq))[0],
530 	    ((uint64_t *)(&crq))[1]);
531 	if (err != 0)
532 		panic("CRQ send failure (%d)", err);
533 }
534 
535 static void
536 vscsi_task_management(struct vscsi_softc *sc, union ccb *ccb)
537 {
538 	struct srp_tsk_mgmt *cmd;
539 	struct vscsi_xfer *xp;
540 	struct vscsi_crq crq;
541 	int err;
542 
543 	mtx_assert(&sc->io_lock, MA_OWNED);
544 
545 	xp = TAILQ_FIRST(&sc->free_xferq);
546 	if (xp == NULL)
547 		panic("SCSI queue flooded");
548 	xp->ccb = ccb;
549 	TAILQ_REMOVE(&sc->free_xferq, xp, queue);
550 	TAILQ_INSERT_TAIL(&sc->active_xferq, xp, queue);
551 
552 	xp->srp_iu_size = crq.iu_length = sizeof(*cmd);
553 	err = vmem_alloc(xp->sc->srp_iu_arena, xp->srp_iu_size,
554 	    M_BESTFIT | M_NOWAIT, &xp->srp_iu_offset);
555 	if (err)
556 		panic("Error during VMEM allocation (%d)", err);
557 
558 	cmd = (struct srp_tsk_mgmt *)((uint8_t *)xp->sc->srp_iu_queue +
559 	    (uintptr_t)xp->srp_iu_offset);
560 	bzero(cmd, xp->srp_iu_size);
561 	cmd->type = SRP_TSK_MGMT;
562 	cmd->tag = (uint64_t)xp;
563 	cmd->lun = htobe64(CAM_EXTLUN_BYTE_SWIZZLE(ccb->ccb_h.target_lun));
564 
565 	switch (ccb->ccb_h.func_code) {
566 	case XPT_RESET_DEV:
567 		cmd->function = 0x08;
568 		break;
569 	default:
570 		panic("Unimplemented code %d", ccb->ccb_h.func_code);
571 		break;
572 	}
573 
574 	bus_dmamap_sync(xp->sc->crq_tag, xp->sc->crq_map, BUS_DMASYNC_PREWRITE);
575 
576 	/* Create CRQ entry */
577 	crq.valid = 0x80;
578 	crq.format = 0x01;
579 	crq.iu_data = xp->sc->srp_iu_phys + xp->srp_iu_offset;
580 
581 	err = phyp_hcall(H_SEND_CRQ, xp->sc->unit, ((uint64_t *)(&crq))[0],
582 	    ((uint64_t *)(&crq))[1]);
583 	if (err != 0)
584 		panic("CRQ send failure (%d)", err);
585 }
586 
587 static void
588 vscsi_scsi_command(void *xxp, bus_dma_segment_t *segs, int nsegs, int err)
589 {
590 	struct vscsi_xfer *xp = xxp;
591 	uint8_t *cdb;
592 	union ccb *ccb = xp->ccb;
593 	struct srp_cmd *cmd;
594 	uint64_t chunk_addr;
595 	uint32_t chunk_size;
596 	int desc_start, i;
597 	struct vscsi_crq crq;
598 
599 	KASSERT(err == 0, ("DMA error %d\n", err));
600 
601 	mtx_assert(&xp->sc->io_lock, MA_OWNED);
602 
603 	cdb = (ccb->ccb_h.flags & CAM_CDB_POINTER) ?
604 	    ccb->csio.cdb_io.cdb_ptr : ccb->csio.cdb_io.cdb_bytes;
605 
606 	/* Command format from Table 20, page 37 of SRP spec */
607 	crq.iu_length = 48 + ((nsegs > 1) ? 20 : 16) +
608 	    ((ccb->csio.cdb_len > 16) ? (ccb->csio.cdb_len - 16) : 0);
609 	xp->srp_iu_size = crq.iu_length;
610 	if (nsegs > 1)
611 		xp->srp_iu_size += nsegs*16;
612 	xp->srp_iu_size = roundup(xp->srp_iu_size, 16);
613 	err = vmem_alloc(xp->sc->srp_iu_arena, xp->srp_iu_size,
614 	    M_BESTFIT | M_NOWAIT, &xp->srp_iu_offset);
615 	if (err)
616 		panic("Error during VMEM allocation (%d)", err);
617 
618 	cmd = (struct srp_cmd *)((uint8_t *)xp->sc->srp_iu_queue +
619 	    (uintptr_t)xp->srp_iu_offset);
620 	bzero(cmd, xp->srp_iu_size);
621 	cmd->type = SRP_CMD;
622 	if (ccb->csio.cdb_len > 16)
623 		cmd->additional_cdb = (ccb->csio.cdb_len - 16) << 2;
624 	memcpy(cmd->cdb, cdb, ccb->csio.cdb_len);
625 
626 	cmd->tag = (uint64_t)(xp); /* Let the responder find this again */
627 	cmd->lun = htobe64(CAM_EXTLUN_BYTE_SWIZZLE(ccb->ccb_h.target_lun));
628 
629 	if (nsegs > 1) {
630 		/* Use indirect descriptors */
631 		switch (ccb->ccb_h.flags & CAM_DIR_MASK) {
632 		case CAM_DIR_OUT:
633 			cmd->formats = (2 << 4);
634 			break;
635 		case CAM_DIR_IN:
636 			cmd->formats = 2;
637 			break;
638 		default:
639 			panic("Does not support bidirectional commands (%d)",
640 			    ccb->ccb_h.flags & CAM_DIR_MASK);
641 			break;
642 		}
643 
644 		desc_start = ((ccb->csio.cdb_len > 16) ?
645 		    ccb->csio.cdb_len - 16 : 0);
646 		chunk_addr = xp->sc->srp_iu_phys + xp->srp_iu_offset + 20 +
647 		    desc_start + sizeof(*cmd);
648 		chunk_size = 16*nsegs;
649 		memcpy(&cmd->data_payload[desc_start], &chunk_addr, 8);
650 		memcpy(&cmd->data_payload[desc_start+12], &chunk_size, 4);
651 		chunk_size = 0;
652 		for (i = 0; i < nsegs; i++)
653 			chunk_size += segs[i].ds_len;
654 		memcpy(&cmd->data_payload[desc_start+16], &chunk_size, 4);
655 		desc_start += 20;
656 		for (i = 0; i < nsegs; i++) {
657 			chunk_addr = segs[i].ds_addr;
658 			chunk_size = segs[i].ds_len;
659 
660 			memcpy(&cmd->data_payload[desc_start + 16*i],
661 			    &chunk_addr, 8);
662 			/* Set handle tag to 0 */
663 			memcpy(&cmd->data_payload[desc_start + 16*i + 12],
664 			    &chunk_size, 4);
665 		}
666 	} else if (nsegs == 1) {
667 		switch (ccb->ccb_h.flags & CAM_DIR_MASK) {
668 		case CAM_DIR_OUT:
669 			cmd->formats = (1 << 4);
670 			break;
671 		case CAM_DIR_IN:
672 			cmd->formats = 1;
673 			break;
674 		default:
675 			panic("Does not support bidirectional commands (%d)",
676 			    ccb->ccb_h.flags & CAM_DIR_MASK);
677 			break;
678 		}
679 
680 		/*
681 		 * Memory descriptor:
682 		 * 8 byte address
683 		 * 4 byte handle
684 		 * 4 byte length
685 		 */
686 
687 		chunk_addr = segs[0].ds_addr;
688 		chunk_size = segs[0].ds_len;
689 		desc_start = ((ccb->csio.cdb_len > 16) ?
690 		    ccb->csio.cdb_len - 16 : 0);
691 
692 		memcpy(&cmd->data_payload[desc_start], &chunk_addr, 8);
693 		/* Set handle tag to 0 */
694 		memcpy(&cmd->data_payload[desc_start+12], &chunk_size, 4);
695 		KASSERT(xp->srp_iu_size >= 48 + ((ccb->csio.cdb_len > 16) ?
696 		    ccb->csio.cdb_len : 16), ("SRP IU command length"));
697 	} else {
698 		cmd->formats = 0;
699 	}
700 	bus_dmamap_sync(xp->sc->crq_tag, xp->sc->crq_map, BUS_DMASYNC_PREWRITE);
701 
702 	/* Create CRQ entry */
703 	crq.valid = 0x80;
704 	crq.format = 0x01;
705 	crq.iu_data = xp->sc->srp_iu_phys + xp->srp_iu_offset;
706 
707 	err = phyp_hcall(H_SEND_CRQ, xp->sc->unit, ((uint64_t *)(&crq))[0],
708 	    ((uint64_t *)(&crq))[1]);
709 	if (err != 0)
710 		panic("CRQ send failure (%d)", err);
711 }
712 
713 static void
714 vscsi_crq_load_cb(void *xsc, bus_dma_segment_t *segs, int nsegs, int err)
715 {
716 	struct vscsi_softc *sc = xsc;
717 
718 	sc->crq_phys = segs[0].ds_addr;
719 	sc->n_crqs = PAGE_SIZE/sizeof(struct vscsi_crq);
720 
721 	sc->srp_iu_queue = (uint8_t *)(sc->crq_queue);
722 	sc->srp_iu_phys = segs[0].ds_addr;
723 	sc->srp_iu_arena = vmem_create("VSCSI SRP IU", PAGE_SIZE,
724 	    segs[0].ds_len - PAGE_SIZE, 16, 0, M_BESTFIT | M_NOWAIT);
725 }
726 
727 static void
728 vscsi_setup_bus(struct vscsi_softc *sc)
729 {
730 	struct vscsi_crq crq;
731 	struct vscsi_xfer *xp;
732 	int error;
733 
734 	struct {
735 		uint32_t type;
736 		uint16_t status;
737 		uint16_t length;
738 		uint64_t tag;
739 		uint64_t buffer;
740 		struct {
741 			char srp_version[8];
742 			char partition_name[96];
743 			uint32_t partition_number;
744 			uint32_t mad_version;
745 			uint32_t os_type;
746 			uint32_t port_max_txu[8];
747 		} payload;
748 	} mad_adapter_info;
749 
750 	bzero(&crq, sizeof(crq));
751 
752 	/* Init message */
753 	crq.valid = 0xc0;
754 	crq.format = 0x01;
755 
756 	do {
757 		error = phyp_hcall(H_FREE_CRQ, sc->unit);
758 	} while (error == H_BUSY);
759 
760 	/* See initialization sequence page 757 */
761 	bzero(sc->crq_queue, sc->n_crqs*sizeof(sc->crq_queue[0]));
762 	sc->cur_crq = 0;
763 	sc->bus_initialized = 0;
764 	sc->bus_logged_in = 0;
765 	bus_dmamap_sync(sc->crq_tag, sc->crq_map, BUS_DMASYNC_PREWRITE);
766 	error = phyp_hcall(H_REG_CRQ, sc->unit, sc->crq_phys,
767 	    sc->n_crqs*sizeof(sc->crq_queue[0]));
768 	KASSERT(error == 0, ("CRQ registration success"));
769 
770 	error = phyp_hcall(H_SEND_CRQ, sc->unit, ((uint64_t *)(&crq))[0],
771 	    ((uint64_t *)(&crq))[1]);
772 	if (error != 0)
773 		panic("CRQ setup failure (%d)", error);
774 
775 	while (sc->bus_initialized == 0)
776 		vscsi_check_response_queue(sc);
777 
778 	/* Send MAD adapter info */
779 	mad_adapter_info.type = MAD_ADAPTER_INFO_REQUEST;
780 	mad_adapter_info.status = 0;
781 	mad_adapter_info.length = sizeof(mad_adapter_info.payload);
782 
783 	strcpy(mad_adapter_info.payload.srp_version, "16.a");
784 	strcpy(mad_adapter_info.payload.partition_name, "UNKNOWN");
785 	mad_adapter_info.payload.partition_number = -1;
786 	mad_adapter_info.payload.mad_version = 1;
787 	mad_adapter_info.payload.os_type = 2; /* Claim we are Linux */
788 	mad_adapter_info.payload.port_max_txu[0] = 0;
789 	/* If this fails, we get the defaults above */
790 	OF_getprop(OF_finddevice("/"), "ibm,partition-name",
791 	    mad_adapter_info.payload.partition_name,
792 	    sizeof(mad_adapter_info.payload.partition_name));
793 	OF_getprop(OF_finddevice("/"), "ibm,partition-no",
794 	    &mad_adapter_info.payload.partition_number,
795 	    sizeof(mad_adapter_info.payload.partition_number));
796 
797 	xp = TAILQ_FIRST(&sc->free_xferq);
798 	xp->ccb = NULL;
799 	TAILQ_REMOVE(&sc->free_xferq, xp, queue);
800 	TAILQ_INSERT_TAIL(&sc->active_xferq, xp, queue);
801 	xp->srp_iu_size = crq.iu_length = sizeof(mad_adapter_info);
802 	vmem_alloc(xp->sc->srp_iu_arena, xp->srp_iu_size,
803 	    M_BESTFIT | M_NOWAIT, &xp->srp_iu_offset);
804 	mad_adapter_info.buffer = xp->sc->srp_iu_phys + xp->srp_iu_offset + 24;
805 	mad_adapter_info.tag = (uint64_t)xp;
806 	memcpy((uint8_t *)xp->sc->srp_iu_queue + (uintptr_t)xp->srp_iu_offset,
807 		&mad_adapter_info, sizeof(mad_adapter_info));
808 	crq.valid = 0x80;
809 	crq.format = 0x02;
810 	crq.iu_data = xp->sc->srp_iu_phys + xp->srp_iu_offset;
811 	bus_dmamap_sync(sc->crq_tag, sc->crq_map, BUS_DMASYNC_PREWRITE);
812 	phyp_hcall(H_SEND_CRQ, xp->sc->unit, ((uint64_t *)(&crq))[0],
813 	    ((uint64_t *)(&crq))[1]);
814 
815 	while (TAILQ_EMPTY(&sc->free_xferq))
816 		vscsi_check_response_queue(sc);
817 
818 	/* Send SRP login */
819 	vscsi_srp_login(sc);
820 	while (sc->bus_logged_in == 0)
821 		vscsi_check_response_queue(sc);
822 
823 	error = phyp_hcall(H_VIO_SIGNAL, sc->unit, 1); /* Enable interrupts */
824 }
825 
826 
827 static void
828 vscsi_intr(void *xsc)
829 {
830 	struct vscsi_softc *sc = xsc;
831 
832 	mtx_lock(&sc->io_lock);
833 	vscsi_check_response_queue(sc);
834 	mtx_unlock(&sc->io_lock);
835 }
836 
837 static void
838 vscsi_srp_response(struct vscsi_xfer *xp, struct vscsi_crq *crq)
839 {
840 	union ccb *ccb = xp->ccb;
841 	struct vscsi_softc *sc = xp->sc;
842 	struct srp_rsp *rsp;
843 	uint32_t sense_len;
844 
845 	/* SRP response packet in original request */
846 	rsp = (struct srp_rsp *)((uint8_t *)sc->srp_iu_queue +
847 	    (uintptr_t)xp->srp_iu_offset);
848 	ccb->csio.scsi_status = rsp->status;
849 	if (ccb->csio.scsi_status == SCSI_STATUS_OK)
850 		ccb->ccb_h.status = CAM_REQ_CMP;
851 	else
852 		ccb->ccb_h.status = CAM_SCSI_STATUS_ERROR;
853 #ifdef NOTYET
854 	/* Collect fast fail codes */
855 	if (crq->status != 0)
856 		ccb->ccb_h.status = CAM_REQ_CMP_ERR;
857 #endif
858 
859 	if (ccb->ccb_h.status != CAM_REQ_CMP) {
860 		ccb->ccb_h.status |= CAM_DEV_QFRZN;
861 		xpt_freeze_devq(ccb->ccb_h.path, /*count*/ 1);
862 	}
863 
864 	if (!(rsp->flags & SRP_RSPVALID))
865 		rsp->response_data_len = 0;
866 	if (!(rsp->flags & SRP_SNSVALID))
867 		rsp->sense_data_len = 0;
868 	if (!(rsp->flags & (SRP_DOOVER | SRP_DOUNDER)))
869 		rsp->data_out_resid = 0;
870 	if (!(rsp->flags & (SRP_DIOVER | SRP_DIUNDER)))
871 		rsp->data_in_resid = 0;
872 
873 	if (rsp->flags & SRP_SNSVALID) {
874 		bzero(&ccb->csio.sense_data, sizeof(struct scsi_sense_data));
875 		ccb->ccb_h.status |= CAM_AUTOSNS_VALID;
876 		sense_len = min(be32toh(rsp->sense_data_len),
877 		    ccb->csio.sense_len);
878 		memcpy(&ccb->csio.sense_data,
879 		    &rsp->data_payload[be32toh(rsp->response_data_len)],
880 		    sense_len);
881 		ccb->csio.sense_resid = ccb->csio.sense_len -
882 		    be32toh(rsp->sense_data_len);
883 	}
884 
885 	switch (ccb->ccb_h.flags & CAM_DIR_MASK) {
886 	case CAM_DIR_OUT:
887 		ccb->csio.resid = rsp->data_out_resid;
888 		break;
889 	case CAM_DIR_IN:
890 		ccb->csio.resid = rsp->data_in_resid;
891 		break;
892 	}
893 
894 	bus_dmamap_sync(sc->data_tag, xp->dmamap, BUS_DMASYNC_POSTREAD);
895 	bus_dmamap_unload(sc->data_tag, xp->dmamap);
896 	xpt_done(ccb);
897 	xp->ccb = NULL;
898 }
899 
900 static void
901 vscsi_login_response(struct vscsi_xfer *xp, struct vscsi_crq *crq)
902 {
903 	struct vscsi_softc *sc = xp->sc;
904 	struct srp_login_rsp *rsp;
905 
906 	/* SRP response packet in original request */
907 	rsp = (struct srp_login_rsp *)((uint8_t *)sc->srp_iu_queue +
908 	    (uintptr_t)xp->srp_iu_offset);
909 	KASSERT(be16toh(rsp->buffer_formats) & 0x3, ("Both direct and indirect "
910 	    "buffers supported"));
911 
912 	sc->max_transactions = be32toh(rsp->request_limit_delta);
913 	device_printf(sc->dev, "Queue depth %d commands\n",
914 	    sc->max_transactions);
915 	sc->bus_logged_in = 1;
916 }
917 
918 static void
919 vscsi_cam_poll(struct cam_sim *sim)
920 {
921 	struct vscsi_softc *sc = cam_sim_softc(sim);
922 
923 	vscsi_check_response_queue(sc);
924 }
925 
926 static void
927 vscsi_check_response_queue(struct vscsi_softc *sc)
928 {
929 	struct vscsi_crq *crq;
930 	struct vscsi_xfer *xp;
931 	int code;
932 
933 	mtx_assert(&sc->io_lock, MA_OWNED);
934 
935 	while (sc->crq_queue[sc->cur_crq].valid != 0) {
936 		/* The hypercalls at both ends of this are not optimal */
937 		phyp_hcall(H_VIO_SIGNAL, sc->unit, 0);
938 		bus_dmamap_sync(sc->crq_tag, sc->crq_map, BUS_DMASYNC_POSTREAD);
939 
940 		crq = &sc->crq_queue[sc->cur_crq];
941 
942 		switch (crq->valid) {
943 		case 0xc0:
944 			if (crq->format == 0x02)
945 				sc->bus_initialized = 1;
946 			break;
947 		case 0x80:
948 			/* IU data is set to tag pointer (the XP) */
949 			xp = (struct vscsi_xfer *)crq->iu_data;
950 
951 			switch (crq->format) {
952 			case 0x01:
953 				code = *((uint8_t *)sc->srp_iu_queue +
954 	    			    (uintptr_t)xp->srp_iu_offset);
955 				switch (code) {
956 				case SRP_RSP:
957 					vscsi_srp_response(xp, crq);
958 					break;
959 				case SRP_LOGIN_RSP:
960 					vscsi_login_response(xp, crq);
961 					break;
962 				default:
963 					device_printf(sc->dev, "Unknown SRP "
964 					    "response code %d\n", code);
965 					break;
966 				}
967 				break;
968 			case 0x02:
969 				/* Ignore management datagrams */
970 				break;
971 			default:
972 				panic("Unknown CRQ format %d\n", crq->format);
973 				break;
974 			}
975 			vmem_free(sc->srp_iu_arena, xp->srp_iu_offset,
976 			    xp->srp_iu_size);
977 			TAILQ_REMOVE(&sc->active_xferq, xp, queue);
978 			TAILQ_INSERT_TAIL(&sc->free_xferq, xp, queue);
979 			break;
980 		default:
981 			device_printf(sc->dev,
982 			    "Unknown CRQ message type %d\n", crq->valid);
983 			break;
984 		}
985 
986 		crq->valid = 0;
987 		sc->cur_crq = (sc->cur_crq + 1) % sc->n_crqs;
988 
989 		bus_dmamap_sync(sc->crq_tag, sc->crq_map, BUS_DMASYNC_PREWRITE);
990 		phyp_hcall(H_VIO_SIGNAL, sc->unit, 1);
991 	}
992 }
993 
994