xref: /linux/drivers/scsi/cxlflash/superpipe.c (revision 7ec462100ef9142344ddbf86f2c3008b97acddbe)
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * CXL Flash Device Driver
4  *
5  * Written by: Manoj N. Kumar <manoj@linux.vnet.ibm.com>, IBM Corporation
6  *             Matthew R. Ochs <mrochs@linux.vnet.ibm.com>, IBM Corporation
7  *
8  * Copyright (C) 2015 IBM Corporation
9  */
10 
11 #include <linux/delay.h>
12 #include <linux/file.h>
13 #include <linux/interrupt.h>
14 #include <linux/pci.h>
15 #include <linux/syscalls.h>
16 #include <linux/unaligned.h>
17 
18 #include <scsi/scsi.h>
19 #include <scsi/scsi_host.h>
20 #include <scsi/scsi_cmnd.h>
21 #include <scsi/scsi_eh.h>
22 #include <uapi/scsi/cxlflash_ioctl.h>
23 
24 #include "sislite.h"
25 #include "common.h"
26 #include "vlun.h"
27 #include "superpipe.h"
28 
29 struct cxlflash_global global;
30 
31 /**
32  * marshal_rele_to_resize() - translate release to resize structure
33  * @release:	Source structure from which to translate/copy.
34  * @resize:	Destination structure for the translate/copy.
35  */
marshal_rele_to_resize(struct dk_cxlflash_release * release,struct dk_cxlflash_resize * resize)36 static void marshal_rele_to_resize(struct dk_cxlflash_release *release,
37 				   struct dk_cxlflash_resize *resize)
38 {
39 	resize->hdr = release->hdr;
40 	resize->context_id = release->context_id;
41 	resize->rsrc_handle = release->rsrc_handle;
42 }
43 
44 /**
45  * marshal_det_to_rele() - translate detach to release structure
46  * @detach:	Destination structure for the translate/copy.
47  * @release:	Source structure from which to translate/copy.
48  */
marshal_det_to_rele(struct dk_cxlflash_detach * detach,struct dk_cxlflash_release * release)49 static void marshal_det_to_rele(struct dk_cxlflash_detach *detach,
50 				struct dk_cxlflash_release *release)
51 {
52 	release->hdr = detach->hdr;
53 	release->context_id = detach->context_id;
54 }
55 
56 /**
57  * marshal_udir_to_rele() - translate udirect to release structure
58  * @udirect:	Source structure from which to translate/copy.
59  * @release:	Destination structure for the translate/copy.
60  */
marshal_udir_to_rele(struct dk_cxlflash_udirect * udirect,struct dk_cxlflash_release * release)61 static void marshal_udir_to_rele(struct dk_cxlflash_udirect *udirect,
62 				 struct dk_cxlflash_release *release)
63 {
64 	release->hdr = udirect->hdr;
65 	release->context_id = udirect->context_id;
66 	release->rsrc_handle = udirect->rsrc_handle;
67 }
68 
69 /**
70  * cxlflash_free_errpage() - frees resources associated with global error page
71  */
cxlflash_free_errpage(void)72 void cxlflash_free_errpage(void)
73 {
74 
75 	mutex_lock(&global.mutex);
76 	if (global.err_page) {
77 		__free_page(global.err_page);
78 		global.err_page = NULL;
79 	}
80 	mutex_unlock(&global.mutex);
81 }
82 
83 /**
84  * cxlflash_stop_term_user_contexts() - stops/terminates known user contexts
85  * @cfg:	Internal structure associated with the host.
86  *
87  * When the host needs to go down, all users must be quiesced and their
88  * memory freed. This is accomplished by putting the contexts in error
89  * state which will notify the user and let them 'drive' the tear down.
90  * Meanwhile, this routine camps until all user contexts have been removed.
91  *
92  * Note that the main loop in this routine will always execute at least once
93  * to flush the reset_waitq.
94  */
cxlflash_stop_term_user_contexts(struct cxlflash_cfg * cfg)95 void cxlflash_stop_term_user_contexts(struct cxlflash_cfg *cfg)
96 {
97 	struct device *dev = &cfg->dev->dev;
98 	int i, found = true;
99 
100 	cxlflash_mark_contexts_error(cfg);
101 
102 	while (true) {
103 		for (i = 0; i < MAX_CONTEXT; i++)
104 			if (cfg->ctx_tbl[i]) {
105 				found = true;
106 				break;
107 			}
108 
109 		if (!found && list_empty(&cfg->ctx_err_recovery))
110 			return;
111 
112 		dev_dbg(dev, "%s: Wait for user contexts to quiesce...\n",
113 			__func__);
114 		wake_up_all(&cfg->reset_waitq);
115 		ssleep(1);
116 		found = false;
117 	}
118 }
119 
120 /**
121  * find_error_context() - locates a context by cookie on the error recovery list
122  * @cfg:	Internal structure associated with the host.
123  * @rctxid:	Desired context by id.
124  * @file:	Desired context by file.
125  *
126  * Return: Found context on success, NULL on failure
127  */
find_error_context(struct cxlflash_cfg * cfg,u64 rctxid,struct file * file)128 static struct ctx_info *find_error_context(struct cxlflash_cfg *cfg, u64 rctxid,
129 					   struct file *file)
130 {
131 	struct ctx_info *ctxi;
132 
133 	list_for_each_entry(ctxi, &cfg->ctx_err_recovery, list)
134 		if ((ctxi->ctxid == rctxid) || (ctxi->file == file))
135 			return ctxi;
136 
137 	return NULL;
138 }
139 
140 /**
141  * get_context() - obtains a validated and locked context reference
142  * @cfg:	Internal structure associated with the host.
143  * @rctxid:	Desired context (raw, un-decoded format).
144  * @arg:	LUN information or file associated with request.
145  * @ctx_ctrl:	Control information to 'steer' desired lookup.
146  *
147  * NOTE: despite the name pid, in linux, current->pid actually refers
148  * to the lightweight process id (tid) and can change if the process is
149  * multi threaded. The tgid remains constant for the process and only changes
150  * when the process of fork. For all intents and purposes, think of tgid
151  * as a pid in the traditional sense.
152  *
153  * Return: Validated context on success, NULL on failure
154  */
get_context(struct cxlflash_cfg * cfg,u64 rctxid,void * arg,enum ctx_ctrl ctx_ctrl)155 struct ctx_info *get_context(struct cxlflash_cfg *cfg, u64 rctxid,
156 			     void *arg, enum ctx_ctrl ctx_ctrl)
157 {
158 	struct device *dev = &cfg->dev->dev;
159 	struct ctx_info *ctxi = NULL;
160 	struct lun_access *lun_access = NULL;
161 	struct file *file = NULL;
162 	struct llun_info *lli = arg;
163 	u64 ctxid = DECODE_CTXID(rctxid);
164 	int rc;
165 	pid_t pid = task_tgid_nr(current), ctxpid = 0;
166 
167 	if (ctx_ctrl & CTX_CTRL_FILE) {
168 		lli = NULL;
169 		file = (struct file *)arg;
170 	}
171 
172 	if (ctx_ctrl & CTX_CTRL_CLONE)
173 		pid = task_ppid_nr(current);
174 
175 	if (likely(ctxid < MAX_CONTEXT)) {
176 		while (true) {
177 			mutex_lock(&cfg->ctx_tbl_list_mutex);
178 			ctxi = cfg->ctx_tbl[ctxid];
179 			if (ctxi)
180 				if ((file && (ctxi->file != file)) ||
181 				    (!file && (ctxi->ctxid != rctxid)))
182 					ctxi = NULL;
183 
184 			if ((ctx_ctrl & CTX_CTRL_ERR) ||
185 			    (!ctxi && (ctx_ctrl & CTX_CTRL_ERR_FALLBACK)))
186 				ctxi = find_error_context(cfg, rctxid, file);
187 			if (!ctxi) {
188 				mutex_unlock(&cfg->ctx_tbl_list_mutex);
189 				goto out;
190 			}
191 
192 			/*
193 			 * Need to acquire ownership of the context while still
194 			 * under the table/list lock to serialize with a remove
195 			 * thread. Use the 'try' to avoid stalling the
196 			 * table/list lock for a single context.
197 			 *
198 			 * Note that the lock order is:
199 			 *
200 			 *	cfg->ctx_tbl_list_mutex -> ctxi->mutex
201 			 *
202 			 * Therefore release ctx_tbl_list_mutex before retrying.
203 			 */
204 			rc = mutex_trylock(&ctxi->mutex);
205 			mutex_unlock(&cfg->ctx_tbl_list_mutex);
206 			if (rc)
207 				break; /* got the context's lock! */
208 		}
209 
210 		if (ctxi->unavail)
211 			goto denied;
212 
213 		ctxpid = ctxi->pid;
214 		if (likely(!(ctx_ctrl & CTX_CTRL_NOPID)))
215 			if (pid != ctxpid)
216 				goto denied;
217 
218 		if (lli) {
219 			list_for_each_entry(lun_access, &ctxi->luns, list)
220 				if (lun_access->lli == lli)
221 					goto out;
222 			goto denied;
223 		}
224 	}
225 
226 out:
227 	dev_dbg(dev, "%s: rctxid=%016llx ctxinfo=%p ctxpid=%u pid=%u "
228 		"ctx_ctrl=%u\n", __func__, rctxid, ctxi, ctxpid, pid,
229 		ctx_ctrl);
230 
231 	return ctxi;
232 
233 denied:
234 	mutex_unlock(&ctxi->mutex);
235 	ctxi = NULL;
236 	goto out;
237 }
238 
239 /**
240  * put_context() - release a context that was retrieved from get_context()
241  * @ctxi:	Context to release.
242  *
243  * For now, releasing the context equates to unlocking it's mutex.
244  */
put_context(struct ctx_info * ctxi)245 void put_context(struct ctx_info *ctxi)
246 {
247 	mutex_unlock(&ctxi->mutex);
248 }
249 
250 /**
251  * afu_attach() - attach a context to the AFU
252  * @cfg:	Internal structure associated with the host.
253  * @ctxi:	Context to attach.
254  *
255  * Upon setting the context capabilities, they must be confirmed with
256  * a read back operation as the context might have been closed since
257  * the mailbox was unlocked. When this occurs, registration is failed.
258  *
259  * Return: 0 on success, -errno on failure
260  */
afu_attach(struct cxlflash_cfg * cfg,struct ctx_info * ctxi)261 static int afu_attach(struct cxlflash_cfg *cfg, struct ctx_info *ctxi)
262 {
263 	struct device *dev = &cfg->dev->dev;
264 	struct afu *afu = cfg->afu;
265 	struct sisl_ctrl_map __iomem *ctrl_map = ctxi->ctrl_map;
266 	int rc = 0;
267 	struct hwq *hwq = get_hwq(afu, PRIMARY_HWQ);
268 	u64 val;
269 	int i;
270 
271 	/* Unlock cap and restrict user to read/write cmds in translated mode */
272 	readq_be(&ctrl_map->mbox_r);
273 	val = (SISL_CTX_CAP_READ_CMD | SISL_CTX_CAP_WRITE_CMD);
274 	writeq_be(val, &ctrl_map->ctx_cap);
275 	val = readq_be(&ctrl_map->ctx_cap);
276 	if (val != (SISL_CTX_CAP_READ_CMD | SISL_CTX_CAP_WRITE_CMD)) {
277 		dev_err(dev, "%s: ctx may be closed val=%016llx\n",
278 			__func__, val);
279 		rc = -EAGAIN;
280 		goto out;
281 	}
282 
283 	if (afu_is_ocxl_lisn(afu)) {
284 		/* Set up the LISN effective address for each interrupt */
285 		for (i = 0; i < ctxi->irqs; i++) {
286 			val = cfg->ops->get_irq_objhndl(ctxi->ctx, i);
287 			writeq_be(val, &ctrl_map->lisn_ea[i]);
288 		}
289 
290 		/* Use primary HWQ PASID as identifier for all interrupts */
291 		val = hwq->ctx_hndl;
292 		writeq_be(SISL_LISN_PASID(val, val), &ctrl_map->lisn_pasid[0]);
293 		writeq_be(SISL_LISN_PASID(0UL, val), &ctrl_map->lisn_pasid[1]);
294 	}
295 
296 	/* Set up MMIO registers pointing to the RHT */
297 	writeq_be((u64)ctxi->rht_start, &ctrl_map->rht_start);
298 	val = SISL_RHT_CNT_ID((u64)MAX_RHT_PER_CONTEXT, (u64)(hwq->ctx_hndl));
299 	writeq_be(val, &ctrl_map->rht_cnt_id);
300 out:
301 	dev_dbg(dev, "%s: returning rc=%d\n", __func__, rc);
302 	return rc;
303 }
304 
305 /**
306  * read_cap16() - issues a SCSI READ_CAP16 command
307  * @sdev:	SCSI device associated with LUN.
308  * @lli:	LUN destined for capacity request.
309  *
310  * The READ_CAP16 can take quite a while to complete. Should an EEH occur while
311  * in scsi_execute_cmd(), the EEH handler will attempt to recover. As part of
312  * the recovery, the handler drains all currently running ioctls, waiting until
313  * they have completed before proceeding with a reset. As this routine is used
314  * on the ioctl path, this can create a condition where the EEH handler becomes
315  * stuck, infinitely waiting for this ioctl thread. To avoid this behavior,
316  * temporarily unmark this thread as an ioctl thread by releasing the ioctl
317  * read semaphore. This will allow the EEH handler to proceed with a recovery
318  * while this thread is still running. Once the scsi_execute_cmd() returns,
319  * reacquire the ioctl read semaphore and check the adapter state in case it
320  * changed while inside of scsi_execute_cmd(). The state check will wait if the
321  * adapter is still being recovered or return a failure if the recovery failed.
322  * In the event that the adapter reset failed, simply return the failure as the
323  * ioctl would be unable to continue.
324  *
325  * Note that the above puts a requirement on this routine to only be called on
326  * an ioctl thread.
327  *
328  * Return: 0 on success, -errno on failure
329  */
read_cap16(struct scsi_device * sdev,struct llun_info * lli)330 static int read_cap16(struct scsi_device *sdev, struct llun_info *lli)
331 {
332 	struct cxlflash_cfg *cfg = shost_priv(sdev->host);
333 	struct device *dev = &cfg->dev->dev;
334 	struct glun_info *gli = lli->parent;
335 	struct scsi_sense_hdr sshdr;
336 	const struct scsi_exec_args exec_args = {
337 		.sshdr = &sshdr,
338 	};
339 	u8 *cmd_buf = NULL;
340 	u8 *scsi_cmd = NULL;
341 	int rc = 0;
342 	int result = 0;
343 	int retry_cnt = 0;
344 	u32 to = CMD_TIMEOUT * HZ;
345 
346 retry:
347 	cmd_buf = kzalloc(CMD_BUFSIZE, GFP_KERNEL);
348 	scsi_cmd = kzalloc(MAX_COMMAND_SIZE, GFP_KERNEL);
349 	if (unlikely(!cmd_buf || !scsi_cmd)) {
350 		rc = -ENOMEM;
351 		goto out;
352 	}
353 
354 	scsi_cmd[0] = SERVICE_ACTION_IN_16;	/* read cap(16) */
355 	scsi_cmd[1] = SAI_READ_CAPACITY_16;	/* service action */
356 	put_unaligned_be32(CMD_BUFSIZE, &scsi_cmd[10]);
357 
358 	dev_dbg(dev, "%s: %ssending cmd(%02x)\n", __func__,
359 		retry_cnt ? "re" : "", scsi_cmd[0]);
360 
361 	/* Drop the ioctl read semaphore across lengthy call */
362 	up_read(&cfg->ioctl_rwsem);
363 	result = scsi_execute_cmd(sdev, scsi_cmd, REQ_OP_DRV_IN, cmd_buf,
364 				  CMD_BUFSIZE, to, CMD_RETRIES, &exec_args);
365 	down_read(&cfg->ioctl_rwsem);
366 	rc = check_state(cfg);
367 	if (rc) {
368 		dev_err(dev, "%s: Failed state result=%08x\n",
369 			__func__, result);
370 		rc = -ENODEV;
371 		goto out;
372 	}
373 
374 	if (result > 0 && scsi_sense_valid(&sshdr)) {
375 		if (result & SAM_STAT_CHECK_CONDITION) {
376 			switch (sshdr.sense_key) {
377 			case NO_SENSE:
378 			case RECOVERED_ERROR:
379 			case NOT_READY:
380 				result &= ~SAM_STAT_CHECK_CONDITION;
381 				break;
382 			case UNIT_ATTENTION:
383 				switch (sshdr.asc) {
384 				case 0x29: /* Power on Reset or Device Reset */
385 					fallthrough;
386 				case 0x2A: /* Device capacity changed */
387 				case 0x3F: /* Report LUNs changed */
388 					/* Retry the command once more */
389 					if (retry_cnt++ < 1) {
390 						kfree(cmd_buf);
391 						kfree(scsi_cmd);
392 						goto retry;
393 					}
394 				}
395 				break;
396 			default:
397 				break;
398 			}
399 		}
400 	}
401 
402 	if (result) {
403 		dev_err(dev, "%s: command failed, result=%08x\n",
404 			__func__, result);
405 		rc = -EIO;
406 		goto out;
407 	}
408 
409 	/*
410 	 * Read cap was successful, grab values from the buffer;
411 	 * note that we don't need to worry about unaligned access
412 	 * as the buffer is allocated on an aligned boundary.
413 	 */
414 	mutex_lock(&gli->mutex);
415 	gli->max_lba = be64_to_cpu(*((__be64 *)&cmd_buf[0]));
416 	gli->blk_len = be32_to_cpu(*((__be32 *)&cmd_buf[8]));
417 	mutex_unlock(&gli->mutex);
418 
419 out:
420 	kfree(cmd_buf);
421 	kfree(scsi_cmd);
422 
423 	dev_dbg(dev, "%s: maxlba=%lld blklen=%d rc=%d\n",
424 		__func__, gli->max_lba, gli->blk_len, rc);
425 	return rc;
426 }
427 
428 /**
429  * get_rhte() - obtains validated resource handle table entry reference
430  * @ctxi:	Context owning the resource handle.
431  * @rhndl:	Resource handle associated with entry.
432  * @lli:	LUN associated with request.
433  *
434  * Return: Validated RHTE on success, NULL on failure
435  */
get_rhte(struct ctx_info * ctxi,res_hndl_t rhndl,struct llun_info * lli)436 struct sisl_rht_entry *get_rhte(struct ctx_info *ctxi, res_hndl_t rhndl,
437 				struct llun_info *lli)
438 {
439 	struct cxlflash_cfg *cfg = ctxi->cfg;
440 	struct device *dev = &cfg->dev->dev;
441 	struct sisl_rht_entry *rhte = NULL;
442 
443 	if (unlikely(!ctxi->rht_start)) {
444 		dev_dbg(dev, "%s: Context does not have allocated RHT\n",
445 			 __func__);
446 		goto out;
447 	}
448 
449 	if (unlikely(rhndl >= MAX_RHT_PER_CONTEXT)) {
450 		dev_dbg(dev, "%s: Bad resource handle rhndl=%d\n",
451 			__func__, rhndl);
452 		goto out;
453 	}
454 
455 	if (unlikely(ctxi->rht_lun[rhndl] != lli)) {
456 		dev_dbg(dev, "%s: Bad resource handle LUN rhndl=%d\n",
457 			__func__, rhndl);
458 		goto out;
459 	}
460 
461 	rhte = &ctxi->rht_start[rhndl];
462 	if (unlikely(rhte->nmask == 0)) {
463 		dev_dbg(dev, "%s: Unopened resource handle rhndl=%d\n",
464 			__func__, rhndl);
465 		rhte = NULL;
466 		goto out;
467 	}
468 
469 out:
470 	return rhte;
471 }
472 
473 /**
474  * rhte_checkout() - obtains free/empty resource handle table entry
475  * @ctxi:	Context owning the resource handle.
476  * @lli:	LUN associated with request.
477  *
478  * Return: Free RHTE on success, NULL on failure
479  */
rhte_checkout(struct ctx_info * ctxi,struct llun_info * lli)480 struct sisl_rht_entry *rhte_checkout(struct ctx_info *ctxi,
481 				     struct llun_info *lli)
482 {
483 	struct cxlflash_cfg *cfg = ctxi->cfg;
484 	struct device *dev = &cfg->dev->dev;
485 	struct sisl_rht_entry *rhte = NULL;
486 	int i;
487 
488 	/* Find a free RHT entry */
489 	for (i = 0; i < MAX_RHT_PER_CONTEXT; i++)
490 		if (ctxi->rht_start[i].nmask == 0) {
491 			rhte = &ctxi->rht_start[i];
492 			ctxi->rht_out++;
493 			break;
494 		}
495 
496 	if (likely(rhte))
497 		ctxi->rht_lun[i] = lli;
498 
499 	dev_dbg(dev, "%s: returning rhte=%p index=%d\n", __func__, rhte, i);
500 	return rhte;
501 }
502 
503 /**
504  * rhte_checkin() - releases a resource handle table entry
505  * @ctxi:	Context owning the resource handle.
506  * @rhte:	RHTE to release.
507  */
rhte_checkin(struct ctx_info * ctxi,struct sisl_rht_entry * rhte)508 void rhte_checkin(struct ctx_info *ctxi,
509 		  struct sisl_rht_entry *rhte)
510 {
511 	u32 rsrc_handle = rhte - ctxi->rht_start;
512 
513 	rhte->nmask = 0;
514 	rhte->fp = 0;
515 	ctxi->rht_out--;
516 	ctxi->rht_lun[rsrc_handle] = NULL;
517 	ctxi->rht_needs_ws[rsrc_handle] = false;
518 }
519 
520 /**
521  * rht_format1() - populates a RHTE for format 1
522  * @rhte:	RHTE to populate.
523  * @lun_id:	LUN ID of LUN associated with RHTE.
524  * @perm:	Desired permissions for RHTE.
525  * @port_sel:	Port selection mask
526  */
rht_format1(struct sisl_rht_entry * rhte,u64 lun_id,u32 perm,u32 port_sel)527 static void rht_format1(struct sisl_rht_entry *rhte, u64 lun_id, u32 perm,
528 			u32 port_sel)
529 {
530 	/*
531 	 * Populate the Format 1 RHT entry for direct access (physical
532 	 * LUN) using the synchronization sequence defined in the
533 	 * SISLite specification.
534 	 */
535 	struct sisl_rht_entry_f1 dummy = { 0 };
536 	struct sisl_rht_entry_f1 *rhte_f1 = (struct sisl_rht_entry_f1 *)rhte;
537 
538 	memset(rhte_f1, 0, sizeof(*rhte_f1));
539 	rhte_f1->fp = SISL_RHT_FP(1U, 0);
540 	dma_wmb(); /* Make setting of format bit visible */
541 
542 	rhte_f1->lun_id = lun_id;
543 	dma_wmb(); /* Make setting of LUN id visible */
544 
545 	/*
546 	 * Use a dummy RHT Format 1 entry to build the second dword
547 	 * of the entry that must be populated in a single write when
548 	 * enabled (valid bit set to TRUE).
549 	 */
550 	dummy.valid = 0x80;
551 	dummy.fp = SISL_RHT_FP(1U, perm);
552 	dummy.port_sel = port_sel;
553 	rhte_f1->dw = dummy.dw;
554 
555 	dma_wmb(); /* Make remaining RHT entry fields visible */
556 }
557 
558 /**
559  * cxlflash_lun_attach() - attaches a user to a LUN and manages the LUN's mode
560  * @gli:	LUN to attach.
561  * @mode:	Desired mode of the LUN.
562  * @locked:	Mutex status on current thread.
563  *
564  * Return: 0 on success, -errno on failure
565  */
cxlflash_lun_attach(struct glun_info * gli,enum lun_mode mode,bool locked)566 int cxlflash_lun_attach(struct glun_info *gli, enum lun_mode mode, bool locked)
567 {
568 	int rc = 0;
569 
570 	if (!locked)
571 		mutex_lock(&gli->mutex);
572 
573 	if (gli->mode == MODE_NONE)
574 		gli->mode = mode;
575 	else if (gli->mode != mode) {
576 		pr_debug("%s: gli_mode=%d requested_mode=%d\n",
577 			 __func__, gli->mode, mode);
578 		rc = -EINVAL;
579 		goto out;
580 	}
581 
582 	gli->users++;
583 	WARN_ON(gli->users <= 0);
584 out:
585 	pr_debug("%s: Returning rc=%d gli->mode=%u gli->users=%u\n",
586 		 __func__, rc, gli->mode, gli->users);
587 	if (!locked)
588 		mutex_unlock(&gli->mutex);
589 	return rc;
590 }
591 
592 /**
593  * cxlflash_lun_detach() - detaches a user from a LUN and resets the LUN's mode
594  * @gli:	LUN to detach.
595  *
596  * When resetting the mode, terminate block allocation resources as they
597  * are no longer required (service is safe to call even when block allocation
598  * resources were not present - such as when transitioning from physical mode).
599  * These resources will be reallocated when needed (subsequent transition to
600  * virtual mode).
601  */
cxlflash_lun_detach(struct glun_info * gli)602 void cxlflash_lun_detach(struct glun_info *gli)
603 {
604 	mutex_lock(&gli->mutex);
605 	WARN_ON(gli->mode == MODE_NONE);
606 	if (--gli->users == 0) {
607 		gli->mode = MODE_NONE;
608 		cxlflash_ba_terminate(&gli->blka.ba_lun);
609 	}
610 	pr_debug("%s: gli->users=%u\n", __func__, gli->users);
611 	WARN_ON(gli->users < 0);
612 	mutex_unlock(&gli->mutex);
613 }
614 
615 /**
616  * _cxlflash_disk_release() - releases the specified resource entry
617  * @sdev:	SCSI device associated with LUN.
618  * @ctxi:	Context owning resources.
619  * @release:	Release ioctl data structure.
620  *
621  * For LUNs in virtual mode, the virtual LUN associated with the specified
622  * resource handle is resized to 0 prior to releasing the RHTE. Note that the
623  * AFU sync should _not_ be performed when the context is sitting on the error
624  * recovery list. A context on the error recovery list is not known to the AFU
625  * due to reset. When the context is recovered, it will be reattached and made
626  * known again to the AFU.
627  *
628  * Return: 0 on success, -errno on failure
629  */
_cxlflash_disk_release(struct scsi_device * sdev,struct ctx_info * ctxi,struct dk_cxlflash_release * release)630 int _cxlflash_disk_release(struct scsi_device *sdev,
631 			   struct ctx_info *ctxi,
632 			   struct dk_cxlflash_release *release)
633 {
634 	struct cxlflash_cfg *cfg = shost_priv(sdev->host);
635 	struct device *dev = &cfg->dev->dev;
636 	struct llun_info *lli = sdev->hostdata;
637 	struct glun_info *gli = lli->parent;
638 	struct afu *afu = cfg->afu;
639 	bool put_ctx = false;
640 
641 	struct dk_cxlflash_resize size;
642 	res_hndl_t rhndl = release->rsrc_handle;
643 
644 	int rc = 0;
645 	int rcr = 0;
646 	u64 ctxid = DECODE_CTXID(release->context_id),
647 	    rctxid = release->context_id;
648 
649 	struct sisl_rht_entry *rhte;
650 	struct sisl_rht_entry_f1 *rhte_f1;
651 
652 	dev_dbg(dev, "%s: ctxid=%llu rhndl=%llu gli->mode=%u gli->users=%u\n",
653 		__func__, ctxid, release->rsrc_handle, gli->mode, gli->users);
654 
655 	if (!ctxi) {
656 		ctxi = get_context(cfg, rctxid, lli, CTX_CTRL_ERR_FALLBACK);
657 		if (unlikely(!ctxi)) {
658 			dev_dbg(dev, "%s: Bad context ctxid=%llu\n",
659 				__func__, ctxid);
660 			rc = -EINVAL;
661 			goto out;
662 		}
663 
664 		put_ctx = true;
665 	}
666 
667 	rhte = get_rhte(ctxi, rhndl, lli);
668 	if (unlikely(!rhte)) {
669 		dev_dbg(dev, "%s: Bad resource handle rhndl=%d\n",
670 			__func__, rhndl);
671 		rc = -EINVAL;
672 		goto out;
673 	}
674 
675 	/*
676 	 * Resize to 0 for virtual LUNS by setting the size
677 	 * to 0. This will clear LXT_START and LXT_CNT fields
678 	 * in the RHT entry and properly sync with the AFU.
679 	 *
680 	 * Afterwards we clear the remaining fields.
681 	 */
682 	switch (gli->mode) {
683 	case MODE_VIRTUAL:
684 		marshal_rele_to_resize(release, &size);
685 		size.req_size = 0;
686 		rc = _cxlflash_vlun_resize(sdev, ctxi, &size);
687 		if (rc) {
688 			dev_dbg(dev, "%s: resize failed rc %d\n", __func__, rc);
689 			goto out;
690 		}
691 
692 		break;
693 	case MODE_PHYSICAL:
694 		/*
695 		 * Clear the Format 1 RHT entry for direct access
696 		 * (physical LUN) using the synchronization sequence
697 		 * defined in the SISLite specification.
698 		 */
699 		rhte_f1 = (struct sisl_rht_entry_f1 *)rhte;
700 
701 		rhte_f1->valid = 0;
702 		dma_wmb(); /* Make revocation of RHT entry visible */
703 
704 		rhte_f1->lun_id = 0;
705 		dma_wmb(); /* Make clearing of LUN id visible */
706 
707 		rhte_f1->dw = 0;
708 		dma_wmb(); /* Make RHT entry bottom-half clearing visible */
709 
710 		if (!ctxi->err_recovery_active) {
711 			rcr = cxlflash_afu_sync(afu, ctxid, rhndl, AFU_HW_SYNC);
712 			if (unlikely(rcr))
713 				dev_dbg(dev, "%s: AFU sync failed rc=%d\n",
714 					__func__, rcr);
715 		}
716 		break;
717 	default:
718 		WARN(1, "Unsupported LUN mode!");
719 		goto out;
720 	}
721 
722 	rhte_checkin(ctxi, rhte);
723 	cxlflash_lun_detach(gli);
724 
725 out:
726 	if (put_ctx)
727 		put_context(ctxi);
728 	dev_dbg(dev, "%s: returning rc=%d\n", __func__, rc);
729 	return rc;
730 }
731 
cxlflash_disk_release(struct scsi_device * sdev,void * release)732 int cxlflash_disk_release(struct scsi_device *sdev, void *release)
733 {
734 	return _cxlflash_disk_release(sdev, NULL, release);
735 }
736 
737 /**
738  * destroy_context() - releases a context
739  * @cfg:	Internal structure associated with the host.
740  * @ctxi:	Context to release.
741  *
742  * This routine is safe to be called with a a non-initialized context.
743  * Also note that the routine conditionally checks for the existence
744  * of the context control map before clearing the RHT registers and
745  * context capabilities because it is possible to destroy a context
746  * while the context is in the error state (previous mapping was
747  * removed [so there is no need to worry about clearing] and context
748  * is waiting for a new mapping).
749  */
destroy_context(struct cxlflash_cfg * cfg,struct ctx_info * ctxi)750 static void destroy_context(struct cxlflash_cfg *cfg,
751 			    struct ctx_info *ctxi)
752 {
753 	struct afu *afu = cfg->afu;
754 
755 	if (ctxi->initialized) {
756 		WARN_ON(!list_empty(&ctxi->luns));
757 
758 		/* Clear RHT registers and drop all capabilities for context */
759 		if (afu->afu_map && ctxi->ctrl_map) {
760 			writeq_be(0, &ctxi->ctrl_map->rht_start);
761 			writeq_be(0, &ctxi->ctrl_map->rht_cnt_id);
762 			writeq_be(0, &ctxi->ctrl_map->ctx_cap);
763 		}
764 	}
765 
766 	/* Free memory associated with context */
767 	free_page((ulong)ctxi->rht_start);
768 	kfree(ctxi->rht_needs_ws);
769 	kfree(ctxi->rht_lun);
770 	kfree(ctxi);
771 }
772 
773 /**
774  * create_context() - allocates and initializes a context
775  * @cfg:	Internal structure associated with the host.
776  *
777  * Return: Allocated context on success, NULL on failure
778  */
create_context(struct cxlflash_cfg * cfg)779 static struct ctx_info *create_context(struct cxlflash_cfg *cfg)
780 {
781 	struct device *dev = &cfg->dev->dev;
782 	struct ctx_info *ctxi = NULL;
783 	struct llun_info **lli = NULL;
784 	u8 *ws = NULL;
785 	struct sisl_rht_entry *rhte;
786 
787 	ctxi = kzalloc(sizeof(*ctxi), GFP_KERNEL);
788 	lli = kzalloc((MAX_RHT_PER_CONTEXT * sizeof(*lli)), GFP_KERNEL);
789 	ws = kzalloc((MAX_RHT_PER_CONTEXT * sizeof(*ws)), GFP_KERNEL);
790 	if (unlikely(!ctxi || !lli || !ws)) {
791 		dev_err(dev, "%s: Unable to allocate context\n", __func__);
792 		goto err;
793 	}
794 
795 	rhte = (struct sisl_rht_entry *)get_zeroed_page(GFP_KERNEL);
796 	if (unlikely(!rhte)) {
797 		dev_err(dev, "%s: Unable to allocate RHT\n", __func__);
798 		goto err;
799 	}
800 
801 	ctxi->rht_lun = lli;
802 	ctxi->rht_needs_ws = ws;
803 	ctxi->rht_start = rhte;
804 out:
805 	return ctxi;
806 
807 err:
808 	kfree(ws);
809 	kfree(lli);
810 	kfree(ctxi);
811 	ctxi = NULL;
812 	goto out;
813 }
814 
815 /**
816  * init_context() - initializes a previously allocated context
817  * @ctxi:	Previously allocated context
818  * @cfg:	Internal structure associated with the host.
819  * @ctx:	Previously obtained context cookie.
820  * @ctxid:	Previously obtained process element associated with CXL context.
821  * @file:	Previously obtained file associated with CXL context.
822  * @perms:	User-specified permissions.
823  * @irqs:	User-specified number of interrupts.
824  */
init_context(struct ctx_info * ctxi,struct cxlflash_cfg * cfg,void * ctx,int ctxid,struct file * file,u32 perms,u64 irqs)825 static void init_context(struct ctx_info *ctxi, struct cxlflash_cfg *cfg,
826 			 void *ctx, int ctxid, struct file *file, u32 perms,
827 			 u64 irqs)
828 {
829 	struct afu *afu = cfg->afu;
830 
831 	ctxi->rht_perms = perms;
832 	ctxi->ctrl_map = &afu->afu_map->ctrls[ctxid].ctrl;
833 	ctxi->ctxid = ENCODE_CTXID(ctxi, ctxid);
834 	ctxi->irqs = irqs;
835 	ctxi->pid = task_tgid_nr(current); /* tgid = pid */
836 	ctxi->ctx = ctx;
837 	ctxi->cfg = cfg;
838 	ctxi->file = file;
839 	ctxi->initialized = true;
840 	mutex_init(&ctxi->mutex);
841 	kref_init(&ctxi->kref);
842 	INIT_LIST_HEAD(&ctxi->luns);
843 	INIT_LIST_HEAD(&ctxi->list); /* initialize for list_empty() */
844 }
845 
846 /**
847  * remove_context() - context kref release handler
848  * @kref:	Kernel reference associated with context to be removed.
849  *
850  * When a context no longer has any references it can safely be removed
851  * from global access and destroyed. Note that it is assumed the thread
852  * relinquishing access to the context holds its mutex.
853  */
remove_context(struct kref * kref)854 static void remove_context(struct kref *kref)
855 {
856 	struct ctx_info *ctxi = container_of(kref, struct ctx_info, kref);
857 	struct cxlflash_cfg *cfg = ctxi->cfg;
858 	u64 ctxid = DECODE_CTXID(ctxi->ctxid);
859 
860 	/* Remove context from table/error list */
861 	WARN_ON(!mutex_is_locked(&ctxi->mutex));
862 	ctxi->unavail = true;
863 	mutex_unlock(&ctxi->mutex);
864 	mutex_lock(&cfg->ctx_tbl_list_mutex);
865 	mutex_lock(&ctxi->mutex);
866 
867 	if (!list_empty(&ctxi->list))
868 		list_del(&ctxi->list);
869 	cfg->ctx_tbl[ctxid] = NULL;
870 	mutex_unlock(&cfg->ctx_tbl_list_mutex);
871 	mutex_unlock(&ctxi->mutex);
872 
873 	/* Context now completely uncoupled/unreachable */
874 	destroy_context(cfg, ctxi);
875 }
876 
877 /**
878  * _cxlflash_disk_detach() - detaches a LUN from a context
879  * @sdev:	SCSI device associated with LUN.
880  * @ctxi:	Context owning resources.
881  * @detach:	Detach ioctl data structure.
882  *
883  * As part of the detach, all per-context resources associated with the LUN
884  * are cleaned up. When detaching the last LUN for a context, the context
885  * itself is cleaned up and released.
886  *
887  * Return: 0 on success, -errno on failure
888  */
_cxlflash_disk_detach(struct scsi_device * sdev,struct ctx_info * ctxi,struct dk_cxlflash_detach * detach)889 static int _cxlflash_disk_detach(struct scsi_device *sdev,
890 				 struct ctx_info *ctxi,
891 				 struct dk_cxlflash_detach *detach)
892 {
893 	struct cxlflash_cfg *cfg = shost_priv(sdev->host);
894 	struct device *dev = &cfg->dev->dev;
895 	struct llun_info *lli = sdev->hostdata;
896 	struct lun_access *lun_access, *t;
897 	struct dk_cxlflash_release rel;
898 	bool put_ctx = false;
899 
900 	int i;
901 	int rc = 0;
902 	u64 ctxid = DECODE_CTXID(detach->context_id),
903 	    rctxid = detach->context_id;
904 
905 	dev_dbg(dev, "%s: ctxid=%llu\n", __func__, ctxid);
906 
907 	if (!ctxi) {
908 		ctxi = get_context(cfg, rctxid, lli, CTX_CTRL_ERR_FALLBACK);
909 		if (unlikely(!ctxi)) {
910 			dev_dbg(dev, "%s: Bad context ctxid=%llu\n",
911 				__func__, ctxid);
912 			rc = -EINVAL;
913 			goto out;
914 		}
915 
916 		put_ctx = true;
917 	}
918 
919 	/* Cleanup outstanding resources tied to this LUN */
920 	if (ctxi->rht_out) {
921 		marshal_det_to_rele(detach, &rel);
922 		for (i = 0; i < MAX_RHT_PER_CONTEXT; i++) {
923 			if (ctxi->rht_lun[i] == lli) {
924 				rel.rsrc_handle = i;
925 				_cxlflash_disk_release(sdev, ctxi, &rel);
926 			}
927 
928 			/* No need to loop further if we're done */
929 			if (ctxi->rht_out == 0)
930 				break;
931 		}
932 	}
933 
934 	/* Take our LUN out of context, free the node */
935 	list_for_each_entry_safe(lun_access, t, &ctxi->luns, list)
936 		if (lun_access->lli == lli) {
937 			list_del(&lun_access->list);
938 			kfree(lun_access);
939 			lun_access = NULL;
940 			break;
941 		}
942 
943 	/*
944 	 * Release the context reference and the sdev reference that
945 	 * bound this LUN to the context.
946 	 */
947 	if (kref_put(&ctxi->kref, remove_context))
948 		put_ctx = false;
949 	scsi_device_put(sdev);
950 out:
951 	if (put_ctx)
952 		put_context(ctxi);
953 	dev_dbg(dev, "%s: returning rc=%d\n", __func__, rc);
954 	return rc;
955 }
956 
cxlflash_disk_detach(struct scsi_device * sdev,void * detach)957 static int cxlflash_disk_detach(struct scsi_device *sdev, void *detach)
958 {
959 	return _cxlflash_disk_detach(sdev, NULL, detach);
960 }
961 
962 /**
963  * cxlflash_cxl_release() - release handler for adapter file descriptor
964  * @inode:	File-system inode associated with fd.
965  * @file:	File installed with adapter file descriptor.
966  *
967  * This routine is the release handler for the fops registered with
968  * the CXL services on an initial attach for a context. It is called
969  * when a close (explicity by the user or as part of a process tear
970  * down) is performed on the adapter file descriptor returned to the
971  * user. The user should be aware that explicitly performing a close
972  * considered catastrophic and subsequent usage of the superpipe API
973  * with previously saved off tokens will fail.
974  *
975  * This routine derives the context reference and calls detach for
976  * each LUN associated with the context.The final detach operation
977  * causes the context itself to be freed. With exception to when the
978  * CXL process element (context id) lookup fails (a case that should
979  * theoretically never occur), every call into this routine results
980  * in a complete freeing of a context.
981  *
982  * Detaching the LUN is typically an ioctl() operation and the underlying
983  * code assumes that ioctl_rwsem has been acquired as a reader. To support
984  * that design point, the semaphore is acquired and released around detach.
985  *
986  * Return: 0 on success
987  */
cxlflash_cxl_release(struct inode * inode,struct file * file)988 static int cxlflash_cxl_release(struct inode *inode, struct file *file)
989 {
990 	struct cxlflash_cfg *cfg = container_of(file->f_op, struct cxlflash_cfg,
991 						cxl_fops);
992 	void *ctx = cfg->ops->fops_get_context(file);
993 	struct device *dev = &cfg->dev->dev;
994 	struct ctx_info *ctxi = NULL;
995 	struct dk_cxlflash_detach detach = { { 0 }, 0 };
996 	struct lun_access *lun_access, *t;
997 	enum ctx_ctrl ctrl = CTX_CTRL_ERR_FALLBACK | CTX_CTRL_FILE;
998 	int ctxid;
999 
1000 	ctxid = cfg->ops->process_element(ctx);
1001 	if (unlikely(ctxid < 0)) {
1002 		dev_err(dev, "%s: Context %p was closed ctxid=%d\n",
1003 			__func__, ctx, ctxid);
1004 		goto out;
1005 	}
1006 
1007 	ctxi = get_context(cfg, ctxid, file, ctrl);
1008 	if (unlikely(!ctxi)) {
1009 		ctxi = get_context(cfg, ctxid, file, ctrl | CTX_CTRL_CLONE);
1010 		if (!ctxi) {
1011 			dev_dbg(dev, "%s: ctxid=%d already free\n",
1012 				__func__, ctxid);
1013 			goto out_release;
1014 		}
1015 
1016 		dev_dbg(dev, "%s: Another process owns ctxid=%d\n",
1017 			__func__, ctxid);
1018 		put_context(ctxi);
1019 		goto out;
1020 	}
1021 
1022 	dev_dbg(dev, "%s: close for ctxid=%d\n", __func__, ctxid);
1023 
1024 	down_read(&cfg->ioctl_rwsem);
1025 	detach.context_id = ctxi->ctxid;
1026 	list_for_each_entry_safe(lun_access, t, &ctxi->luns, list)
1027 		_cxlflash_disk_detach(lun_access->sdev, ctxi, &detach);
1028 	up_read(&cfg->ioctl_rwsem);
1029 out_release:
1030 	cfg->ops->fd_release(inode, file);
1031 out:
1032 	dev_dbg(dev, "%s: returning\n", __func__);
1033 	return 0;
1034 }
1035 
1036 /**
1037  * unmap_context() - clears a previously established mapping
1038  * @ctxi:	Context owning the mapping.
1039  *
1040  * This routine is used to switch between the error notification page
1041  * (dummy page of all 1's) and the real mapping (established by the CXL
1042  * fault handler).
1043  */
unmap_context(struct ctx_info * ctxi)1044 static void unmap_context(struct ctx_info *ctxi)
1045 {
1046 	unmap_mapping_range(ctxi->file->f_mapping, 0, 0, 1);
1047 }
1048 
1049 /**
1050  * get_err_page() - obtains and allocates the error notification page
1051  * @cfg:	Internal structure associated with the host.
1052  *
1053  * Return: error notification page on success, NULL on failure
1054  */
get_err_page(struct cxlflash_cfg * cfg)1055 static struct page *get_err_page(struct cxlflash_cfg *cfg)
1056 {
1057 	struct page *err_page = global.err_page;
1058 	struct device *dev = &cfg->dev->dev;
1059 
1060 	if (unlikely(!err_page)) {
1061 		err_page = alloc_page(GFP_KERNEL);
1062 		if (unlikely(!err_page)) {
1063 			dev_err(dev, "%s: Unable to allocate err_page\n",
1064 				__func__);
1065 			goto out;
1066 		}
1067 
1068 		memset(page_address(err_page), -1, PAGE_SIZE);
1069 
1070 		/* Serialize update w/ other threads to avoid a leak */
1071 		mutex_lock(&global.mutex);
1072 		if (likely(!global.err_page))
1073 			global.err_page = err_page;
1074 		else {
1075 			__free_page(err_page);
1076 			err_page = global.err_page;
1077 		}
1078 		mutex_unlock(&global.mutex);
1079 	}
1080 
1081 out:
1082 	dev_dbg(dev, "%s: returning err_page=%p\n", __func__, err_page);
1083 	return err_page;
1084 }
1085 
1086 /**
1087  * cxlflash_mmap_fault() - mmap fault handler for adapter file descriptor
1088  * @vmf:	VM fault associated with current fault.
1089  *
1090  * To support error notification via MMIO, faults are 'caught' by this routine
1091  * that was inserted before passing back the adapter file descriptor on attach.
1092  * When a fault occurs, this routine evaluates if error recovery is active and
1093  * if so, installs the error page to 'notify' the user about the error state.
1094  * During normal operation, the fault is simply handled by the original fault
1095  * handler that was installed by CXL services as part of initializing the
1096  * adapter file descriptor. The VMA's page protection bits are toggled to
1097  * indicate cached/not-cached depending on the memory backing the fault.
1098  *
1099  * Return: 0 on success, VM_FAULT_SIGBUS on failure
1100  */
cxlflash_mmap_fault(struct vm_fault * vmf)1101 static vm_fault_t cxlflash_mmap_fault(struct vm_fault *vmf)
1102 {
1103 	struct vm_area_struct *vma = vmf->vma;
1104 	struct file *file = vma->vm_file;
1105 	struct cxlflash_cfg *cfg = container_of(file->f_op, struct cxlflash_cfg,
1106 						cxl_fops);
1107 	void *ctx = cfg->ops->fops_get_context(file);
1108 	struct device *dev = &cfg->dev->dev;
1109 	struct ctx_info *ctxi = NULL;
1110 	struct page *err_page = NULL;
1111 	enum ctx_ctrl ctrl = CTX_CTRL_ERR_FALLBACK | CTX_CTRL_FILE;
1112 	vm_fault_t rc = 0;
1113 	int ctxid;
1114 
1115 	ctxid = cfg->ops->process_element(ctx);
1116 	if (unlikely(ctxid < 0)) {
1117 		dev_err(dev, "%s: Context %p was closed ctxid=%d\n",
1118 			__func__, ctx, ctxid);
1119 		goto err;
1120 	}
1121 
1122 	ctxi = get_context(cfg, ctxid, file, ctrl);
1123 	if (unlikely(!ctxi)) {
1124 		dev_dbg(dev, "%s: Bad context ctxid=%d\n", __func__, ctxid);
1125 		goto err;
1126 	}
1127 
1128 	dev_dbg(dev, "%s: fault for context %d\n", __func__, ctxid);
1129 
1130 	if (likely(!ctxi->err_recovery_active)) {
1131 		vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
1132 		rc = ctxi->cxl_mmap_vmops->fault(vmf);
1133 	} else {
1134 		dev_dbg(dev, "%s: err recovery active, use err_page\n",
1135 			__func__);
1136 
1137 		err_page = get_err_page(cfg);
1138 		if (unlikely(!err_page)) {
1139 			dev_err(dev, "%s: Could not get err_page\n", __func__);
1140 			rc = VM_FAULT_RETRY;
1141 			goto out;
1142 		}
1143 
1144 		get_page(err_page);
1145 		vmf->page = err_page;
1146 		vma->vm_page_prot = pgprot_cached(vma->vm_page_prot);
1147 	}
1148 
1149 out:
1150 	if (likely(ctxi))
1151 		put_context(ctxi);
1152 	dev_dbg(dev, "%s: returning rc=%x\n", __func__, rc);
1153 	return rc;
1154 
1155 err:
1156 	rc = VM_FAULT_SIGBUS;
1157 	goto out;
1158 }
1159 
1160 /*
1161  * Local MMAP vmops to 'catch' faults
1162  */
1163 static const struct vm_operations_struct cxlflash_mmap_vmops = {
1164 	.fault = cxlflash_mmap_fault,
1165 };
1166 
1167 /**
1168  * cxlflash_cxl_mmap() - mmap handler for adapter file descriptor
1169  * @file:	File installed with adapter file descriptor.
1170  * @vma:	VM area associated with mapping.
1171  *
1172  * Installs local mmap vmops to 'catch' faults for error notification support.
1173  *
1174  * Return: 0 on success, -errno on failure
1175  */
cxlflash_cxl_mmap(struct file * file,struct vm_area_struct * vma)1176 static int cxlflash_cxl_mmap(struct file *file, struct vm_area_struct *vma)
1177 {
1178 	struct cxlflash_cfg *cfg = container_of(file->f_op, struct cxlflash_cfg,
1179 						cxl_fops);
1180 	void *ctx = cfg->ops->fops_get_context(file);
1181 	struct device *dev = &cfg->dev->dev;
1182 	struct ctx_info *ctxi = NULL;
1183 	enum ctx_ctrl ctrl = CTX_CTRL_ERR_FALLBACK | CTX_CTRL_FILE;
1184 	int ctxid;
1185 	int rc = 0;
1186 
1187 	ctxid = cfg->ops->process_element(ctx);
1188 	if (unlikely(ctxid < 0)) {
1189 		dev_err(dev, "%s: Context %p was closed ctxid=%d\n",
1190 			__func__, ctx, ctxid);
1191 		rc = -EIO;
1192 		goto out;
1193 	}
1194 
1195 	ctxi = get_context(cfg, ctxid, file, ctrl);
1196 	if (unlikely(!ctxi)) {
1197 		dev_dbg(dev, "%s: Bad context ctxid=%d\n", __func__, ctxid);
1198 		rc = -EIO;
1199 		goto out;
1200 	}
1201 
1202 	dev_dbg(dev, "%s: mmap for context %d\n", __func__, ctxid);
1203 
1204 	rc = cfg->ops->fd_mmap(file, vma);
1205 	if (likely(!rc)) {
1206 		/* Insert ourself in the mmap fault handler path */
1207 		ctxi->cxl_mmap_vmops = vma->vm_ops;
1208 		vma->vm_ops = &cxlflash_mmap_vmops;
1209 	}
1210 
1211 out:
1212 	if (likely(ctxi))
1213 		put_context(ctxi);
1214 	return rc;
1215 }
1216 
1217 const struct file_operations cxlflash_cxl_fops = {
1218 	.owner = THIS_MODULE,
1219 	.mmap = cxlflash_cxl_mmap,
1220 	.release = cxlflash_cxl_release,
1221 };
1222 
1223 /**
1224  * cxlflash_mark_contexts_error() - move contexts to error state and list
1225  * @cfg:	Internal structure associated with the host.
1226  *
1227  * A context is only moved over to the error list when there are no outstanding
1228  * references to it. This ensures that a running operation has completed.
1229  *
1230  * Return: 0 on success, -errno on failure
1231  */
cxlflash_mark_contexts_error(struct cxlflash_cfg * cfg)1232 int cxlflash_mark_contexts_error(struct cxlflash_cfg *cfg)
1233 {
1234 	int i, rc = 0;
1235 	struct ctx_info *ctxi = NULL;
1236 
1237 	mutex_lock(&cfg->ctx_tbl_list_mutex);
1238 
1239 	for (i = 0; i < MAX_CONTEXT; i++) {
1240 		ctxi = cfg->ctx_tbl[i];
1241 		if (ctxi) {
1242 			mutex_lock(&ctxi->mutex);
1243 			cfg->ctx_tbl[i] = NULL;
1244 			list_add(&ctxi->list, &cfg->ctx_err_recovery);
1245 			ctxi->err_recovery_active = true;
1246 			ctxi->ctrl_map = NULL;
1247 			unmap_context(ctxi);
1248 			mutex_unlock(&ctxi->mutex);
1249 		}
1250 	}
1251 
1252 	mutex_unlock(&cfg->ctx_tbl_list_mutex);
1253 	return rc;
1254 }
1255 
1256 /*
1257  * Dummy NULL fops
1258  */
1259 static const struct file_operations null_fops = {
1260 	.owner = THIS_MODULE,
1261 };
1262 
1263 /**
1264  * check_state() - checks and responds to the current adapter state
1265  * @cfg:	Internal structure associated with the host.
1266  *
1267  * This routine can block and should only be used on process context.
1268  * It assumes that the caller is an ioctl thread and holding the ioctl
1269  * read semaphore. This is temporarily let up across the wait to allow
1270  * for draining actively running ioctls. Also note that when waking up
1271  * from waiting in reset, the state is unknown and must be checked again
1272  * before proceeding.
1273  *
1274  * Return: 0 on success, -errno on failure
1275  */
check_state(struct cxlflash_cfg * cfg)1276 int check_state(struct cxlflash_cfg *cfg)
1277 {
1278 	struct device *dev = &cfg->dev->dev;
1279 	int rc = 0;
1280 
1281 retry:
1282 	switch (cfg->state) {
1283 	case STATE_RESET:
1284 		dev_dbg(dev, "%s: Reset state, going to wait...\n", __func__);
1285 		up_read(&cfg->ioctl_rwsem);
1286 		rc = wait_event_interruptible(cfg->reset_waitq,
1287 					      cfg->state != STATE_RESET);
1288 		down_read(&cfg->ioctl_rwsem);
1289 		if (unlikely(rc))
1290 			break;
1291 		goto retry;
1292 	case STATE_FAILTERM:
1293 		dev_dbg(dev, "%s: Failed/Terminating\n", __func__);
1294 		rc = -ENODEV;
1295 		break;
1296 	default:
1297 		break;
1298 	}
1299 
1300 	return rc;
1301 }
1302 
1303 /**
1304  * cxlflash_disk_attach() - attach a LUN to a context
1305  * @sdev:	SCSI device associated with LUN.
1306  * @arg:	Attach ioctl data structure.
1307  *
1308  * Creates a context and attaches LUN to it. A LUN can only be attached
1309  * one time to a context (subsequent attaches for the same context/LUN pair
1310  * are not supported). Additional LUNs can be attached to a context by
1311  * specifying the 'reuse' flag defined in the cxlflash_ioctl.h header.
1312  *
1313  * Return: 0 on success, -errno on failure
1314  */
cxlflash_disk_attach(struct scsi_device * sdev,void * arg)1315 static int cxlflash_disk_attach(struct scsi_device *sdev, void *arg)
1316 {
1317 	struct dk_cxlflash_attach *attach = arg;
1318 	struct cxlflash_cfg *cfg = shost_priv(sdev->host);
1319 	struct device *dev = &cfg->dev->dev;
1320 	struct afu *afu = cfg->afu;
1321 	struct llun_info *lli = sdev->hostdata;
1322 	struct glun_info *gli = lli->parent;
1323 	struct ctx_info *ctxi = NULL;
1324 	struct lun_access *lun_access = NULL;
1325 	int rc = 0;
1326 	u32 perms;
1327 	int ctxid = -1;
1328 	u64 irqs = attach->num_interrupts;
1329 	u64 flags = 0UL;
1330 	u64 rctxid = 0UL;
1331 	struct file *file = NULL;
1332 
1333 	void *ctx = NULL;
1334 
1335 	int fd = -1;
1336 
1337 	if (irqs > 4) {
1338 		dev_dbg(dev, "%s: Cannot support this many interrupts %llu\n",
1339 			__func__, irqs);
1340 		rc = -EINVAL;
1341 		goto out;
1342 	}
1343 
1344 	if (gli->max_lba == 0) {
1345 		dev_dbg(dev, "%s: No capacity info for LUN=%016llx\n",
1346 			__func__, lli->lun_id[sdev->channel]);
1347 		rc = read_cap16(sdev, lli);
1348 		if (rc) {
1349 			dev_err(dev, "%s: Invalid device rc=%d\n",
1350 				__func__, rc);
1351 			rc = -ENODEV;
1352 			goto out;
1353 		}
1354 		dev_dbg(dev, "%s: LBA = %016llx\n", __func__, gli->max_lba);
1355 		dev_dbg(dev, "%s: BLK_LEN = %08x\n", __func__, gli->blk_len);
1356 	}
1357 
1358 	if (attach->hdr.flags & DK_CXLFLASH_ATTACH_REUSE_CONTEXT) {
1359 		rctxid = attach->context_id;
1360 		ctxi = get_context(cfg, rctxid, NULL, 0);
1361 		if (!ctxi) {
1362 			dev_dbg(dev, "%s: Bad context rctxid=%016llx\n",
1363 				__func__, rctxid);
1364 			rc = -EINVAL;
1365 			goto out;
1366 		}
1367 
1368 		list_for_each_entry(lun_access, &ctxi->luns, list)
1369 			if (lun_access->lli == lli) {
1370 				dev_dbg(dev, "%s: Already attached\n",
1371 					__func__);
1372 				rc = -EINVAL;
1373 				goto out;
1374 			}
1375 	}
1376 
1377 	rc = scsi_device_get(sdev);
1378 	if (unlikely(rc)) {
1379 		dev_err(dev, "%s: Unable to get sdev reference\n", __func__);
1380 		goto out;
1381 	}
1382 
1383 	lun_access = kzalloc(sizeof(*lun_access), GFP_KERNEL);
1384 	if (unlikely(!lun_access)) {
1385 		dev_err(dev, "%s: Unable to allocate lun_access\n", __func__);
1386 		rc = -ENOMEM;
1387 		goto err;
1388 	}
1389 
1390 	lun_access->lli = lli;
1391 	lun_access->sdev = sdev;
1392 
1393 	/* Non-NULL context indicates reuse (another context reference) */
1394 	if (ctxi) {
1395 		dev_dbg(dev, "%s: Reusing context for LUN rctxid=%016llx\n",
1396 			__func__, rctxid);
1397 		kref_get(&ctxi->kref);
1398 		list_add(&lun_access->list, &ctxi->luns);
1399 		goto out_attach;
1400 	}
1401 
1402 	ctxi = create_context(cfg);
1403 	if (unlikely(!ctxi)) {
1404 		dev_err(dev, "%s: Failed to create context ctxid=%d\n",
1405 			__func__, ctxid);
1406 		rc = -ENOMEM;
1407 		goto err;
1408 	}
1409 
1410 	ctx = cfg->ops->dev_context_init(cfg->dev, cfg->afu_cookie);
1411 	if (IS_ERR_OR_NULL(ctx)) {
1412 		dev_err(dev, "%s: Could not initialize context %p\n",
1413 			__func__, ctx);
1414 		rc = -ENODEV;
1415 		goto err;
1416 	}
1417 
1418 	rc = cfg->ops->start_work(ctx, irqs);
1419 	if (unlikely(rc)) {
1420 		dev_dbg(dev, "%s: Could not start context rc=%d\n",
1421 			__func__, rc);
1422 		goto err;
1423 	}
1424 
1425 	ctxid = cfg->ops->process_element(ctx);
1426 	if (unlikely((ctxid >= MAX_CONTEXT) || (ctxid < 0))) {
1427 		dev_err(dev, "%s: ctxid=%d invalid\n", __func__, ctxid);
1428 		rc = -EPERM;
1429 		goto err;
1430 	}
1431 
1432 	file = cfg->ops->get_fd(ctx, &cfg->cxl_fops, &fd);
1433 	if (unlikely(fd < 0)) {
1434 		rc = -ENODEV;
1435 		dev_err(dev, "%s: Could not get file descriptor\n", __func__);
1436 		goto err;
1437 	}
1438 
1439 	/* Translate read/write O_* flags from fcntl.h to AFU permission bits */
1440 	perms = SISL_RHT_PERM(attach->hdr.flags + 1);
1441 
1442 	/* Context mutex is locked upon return */
1443 	init_context(ctxi, cfg, ctx, ctxid, file, perms, irqs);
1444 
1445 	rc = afu_attach(cfg, ctxi);
1446 	if (unlikely(rc)) {
1447 		dev_err(dev, "%s: Could not attach AFU rc %d\n", __func__, rc);
1448 		goto err;
1449 	}
1450 
1451 	/*
1452 	 * No error paths after this point. Once the fd is installed it's
1453 	 * visible to user space and can't be undone safely on this thread.
1454 	 * There is no need to worry about a deadlock here because no one
1455 	 * knows about us yet; we can be the only one holding our mutex.
1456 	 */
1457 	list_add(&lun_access->list, &ctxi->luns);
1458 	mutex_lock(&cfg->ctx_tbl_list_mutex);
1459 	mutex_lock(&ctxi->mutex);
1460 	cfg->ctx_tbl[ctxid] = ctxi;
1461 	mutex_unlock(&cfg->ctx_tbl_list_mutex);
1462 	fd_install(fd, file);
1463 
1464 out_attach:
1465 	if (fd != -1)
1466 		flags |= DK_CXLFLASH_APP_CLOSE_ADAP_FD;
1467 	if (afu_is_sq_cmd_mode(afu))
1468 		flags |= DK_CXLFLASH_CONTEXT_SQ_CMD_MODE;
1469 
1470 	attach->hdr.return_flags = flags;
1471 	attach->context_id = ctxi->ctxid;
1472 	attach->block_size = gli->blk_len;
1473 	attach->mmio_size = sizeof(afu->afu_map->hosts[0].harea);
1474 	attach->last_lba = gli->max_lba;
1475 	attach->max_xfer = sdev->host->max_sectors * MAX_SECTOR_UNIT;
1476 	attach->max_xfer /= gli->blk_len;
1477 
1478 out:
1479 	attach->adap_fd = fd;
1480 
1481 	if (ctxi)
1482 		put_context(ctxi);
1483 
1484 	dev_dbg(dev, "%s: returning ctxid=%d fd=%d bs=%lld rc=%d llba=%lld\n",
1485 		__func__, ctxid, fd, attach->block_size, rc, attach->last_lba);
1486 	return rc;
1487 
1488 err:
1489 	/* Cleanup CXL context; okay to 'stop' even if it was not started */
1490 	if (!IS_ERR_OR_NULL(ctx)) {
1491 		cfg->ops->stop_context(ctx);
1492 		cfg->ops->release_context(ctx);
1493 		ctx = NULL;
1494 	}
1495 
1496 	/*
1497 	 * Here, we're overriding the fops with a dummy all-NULL fops because
1498 	 * fput() calls the release fop, which will cause us to mistakenly
1499 	 * call into the CXL code. Rather than try to add yet more complexity
1500 	 * to that routine (cxlflash_cxl_release) we should try to fix the
1501 	 * issue here.
1502 	 */
1503 	if (fd > 0) {
1504 		file->f_op = &null_fops;
1505 		fput(file);
1506 		put_unused_fd(fd);
1507 		fd = -1;
1508 		file = NULL;
1509 	}
1510 
1511 	/* Cleanup our context */
1512 	if (ctxi) {
1513 		destroy_context(cfg, ctxi);
1514 		ctxi = NULL;
1515 	}
1516 
1517 	kfree(lun_access);
1518 	scsi_device_put(sdev);
1519 	goto out;
1520 }
1521 
1522 /**
1523  * recover_context() - recovers a context in error
1524  * @cfg:	Internal structure associated with the host.
1525  * @ctxi:	Context to release.
1526  * @adap_fd:	Adapter file descriptor associated with new/recovered context.
1527  *
1528  * Restablishes the state for a context-in-error.
1529  *
1530  * Return: 0 on success, -errno on failure
1531  */
recover_context(struct cxlflash_cfg * cfg,struct ctx_info * ctxi,int * adap_fd)1532 static int recover_context(struct cxlflash_cfg *cfg,
1533 			   struct ctx_info *ctxi,
1534 			   int *adap_fd)
1535 {
1536 	struct device *dev = &cfg->dev->dev;
1537 	int rc = 0;
1538 	int fd = -1;
1539 	int ctxid = -1;
1540 	struct file *file;
1541 	void *ctx;
1542 	struct afu *afu = cfg->afu;
1543 
1544 	ctx = cfg->ops->dev_context_init(cfg->dev, cfg->afu_cookie);
1545 	if (IS_ERR_OR_NULL(ctx)) {
1546 		dev_err(dev, "%s: Could not initialize context %p\n",
1547 			__func__, ctx);
1548 		rc = -ENODEV;
1549 		goto out;
1550 	}
1551 
1552 	rc = cfg->ops->start_work(ctx, ctxi->irqs);
1553 	if (unlikely(rc)) {
1554 		dev_dbg(dev, "%s: Could not start context rc=%d\n",
1555 			__func__, rc);
1556 		goto err1;
1557 	}
1558 
1559 	ctxid = cfg->ops->process_element(ctx);
1560 	if (unlikely((ctxid >= MAX_CONTEXT) || (ctxid < 0))) {
1561 		dev_err(dev, "%s: ctxid=%d invalid\n", __func__, ctxid);
1562 		rc = -EPERM;
1563 		goto err2;
1564 	}
1565 
1566 	file = cfg->ops->get_fd(ctx, &cfg->cxl_fops, &fd);
1567 	if (unlikely(fd < 0)) {
1568 		rc = -ENODEV;
1569 		dev_err(dev, "%s: Could not get file descriptor\n", __func__);
1570 		goto err2;
1571 	}
1572 
1573 	/* Update with new MMIO area based on updated context id */
1574 	ctxi->ctrl_map = &afu->afu_map->ctrls[ctxid].ctrl;
1575 
1576 	rc = afu_attach(cfg, ctxi);
1577 	if (rc) {
1578 		dev_err(dev, "%s: Could not attach AFU rc %d\n", __func__, rc);
1579 		goto err3;
1580 	}
1581 
1582 	/*
1583 	 * No error paths after this point. Once the fd is installed it's
1584 	 * visible to user space and can't be undone safely on this thread.
1585 	 */
1586 	ctxi->ctxid = ENCODE_CTXID(ctxi, ctxid);
1587 	ctxi->ctx = ctx;
1588 	ctxi->file = file;
1589 
1590 	/*
1591 	 * Put context back in table (note the reinit of the context list);
1592 	 * we must first drop the context's mutex and then acquire it in
1593 	 * order with the table/list mutex to avoid a deadlock - safe to do
1594 	 * here because no one can find us at this moment in time.
1595 	 */
1596 	mutex_unlock(&ctxi->mutex);
1597 	mutex_lock(&cfg->ctx_tbl_list_mutex);
1598 	mutex_lock(&ctxi->mutex);
1599 	list_del_init(&ctxi->list);
1600 	cfg->ctx_tbl[ctxid] = ctxi;
1601 	mutex_unlock(&cfg->ctx_tbl_list_mutex);
1602 	fd_install(fd, file);
1603 	*adap_fd = fd;
1604 out:
1605 	dev_dbg(dev, "%s: returning ctxid=%d fd=%d rc=%d\n",
1606 		__func__, ctxid, fd, rc);
1607 	return rc;
1608 
1609 err3:
1610 	fput(file);
1611 	put_unused_fd(fd);
1612 err2:
1613 	cfg->ops->stop_context(ctx);
1614 err1:
1615 	cfg->ops->release_context(ctx);
1616 	goto out;
1617 }
1618 
1619 /**
1620  * cxlflash_afu_recover() - initiates AFU recovery
1621  * @sdev:	SCSI device associated with LUN.
1622  * @arg:	Recover ioctl data structure.
1623  *
1624  * Only a single recovery is allowed at a time to avoid exhausting CXL
1625  * resources (leading to recovery failure) in the event that we're up
1626  * against the maximum number of contexts limit. For similar reasons,
1627  * a context recovery is retried if there are multiple recoveries taking
1628  * place at the same time and the failure was due to CXL services being
1629  * unable to keep up.
1630  *
1631  * As this routine is called on ioctl context, it holds the ioctl r/w
1632  * semaphore that is used to drain ioctls in recovery scenarios. The
1633  * implementation to achieve the pacing described above (a local mutex)
1634  * requires that the ioctl r/w semaphore be dropped and reacquired to
1635  * avoid a 3-way deadlock when multiple process recoveries operate in
1636  * parallel.
1637  *
1638  * Because a user can detect an error condition before the kernel, it is
1639  * quite possible for this routine to act as the kernel's EEH detection
1640  * source (MMIO read of mbox_r). Because of this, there is a window of
1641  * time where an EEH might have been detected but not yet 'serviced'
1642  * (callback invoked, causing the device to enter reset state). To avoid
1643  * looping in this routine during that window, a 1 second sleep is in place
1644  * between the time the MMIO failure is detected and the time a wait on the
1645  * reset wait queue is attempted via check_state().
1646  *
1647  * Return: 0 on success, -errno on failure
1648  */
cxlflash_afu_recover(struct scsi_device * sdev,void * arg)1649 static int cxlflash_afu_recover(struct scsi_device *sdev, void *arg)
1650 {
1651 	struct dk_cxlflash_recover_afu *recover = arg;
1652 	struct cxlflash_cfg *cfg = shost_priv(sdev->host);
1653 	struct device *dev = &cfg->dev->dev;
1654 	struct llun_info *lli = sdev->hostdata;
1655 	struct afu *afu = cfg->afu;
1656 	struct ctx_info *ctxi = NULL;
1657 	struct mutex *mutex = &cfg->ctx_recovery_mutex;
1658 	struct hwq *hwq = get_hwq(afu, PRIMARY_HWQ);
1659 	u64 flags;
1660 	u64 ctxid = DECODE_CTXID(recover->context_id),
1661 	    rctxid = recover->context_id;
1662 	long reg;
1663 	bool locked = true;
1664 	int lretry = 20; /* up to 2 seconds */
1665 	int new_adap_fd = -1;
1666 	int rc = 0;
1667 
1668 	atomic_inc(&cfg->recovery_threads);
1669 	up_read(&cfg->ioctl_rwsem);
1670 	rc = mutex_lock_interruptible(mutex);
1671 	down_read(&cfg->ioctl_rwsem);
1672 	if (rc) {
1673 		locked = false;
1674 		goto out;
1675 	}
1676 
1677 	rc = check_state(cfg);
1678 	if (rc) {
1679 		dev_err(dev, "%s: Failed state rc=%d\n", __func__, rc);
1680 		rc = -ENODEV;
1681 		goto out;
1682 	}
1683 
1684 	dev_dbg(dev, "%s: reason=%016llx rctxid=%016llx\n",
1685 		__func__, recover->reason, rctxid);
1686 
1687 retry:
1688 	/* Ensure that this process is attached to the context */
1689 	ctxi = get_context(cfg, rctxid, lli, CTX_CTRL_ERR_FALLBACK);
1690 	if (unlikely(!ctxi)) {
1691 		dev_dbg(dev, "%s: Bad context ctxid=%llu\n", __func__, ctxid);
1692 		rc = -EINVAL;
1693 		goto out;
1694 	}
1695 
1696 	if (ctxi->err_recovery_active) {
1697 retry_recover:
1698 		rc = recover_context(cfg, ctxi, &new_adap_fd);
1699 		if (unlikely(rc)) {
1700 			dev_err(dev, "%s: Recovery failed ctxid=%llu rc=%d\n",
1701 				__func__, ctxid, rc);
1702 			if ((rc == -ENODEV) &&
1703 			    ((atomic_read(&cfg->recovery_threads) > 1) ||
1704 			     (lretry--))) {
1705 				dev_dbg(dev, "%s: Going to try again\n",
1706 					__func__);
1707 				mutex_unlock(mutex);
1708 				msleep(100);
1709 				rc = mutex_lock_interruptible(mutex);
1710 				if (rc) {
1711 					locked = false;
1712 					goto out;
1713 				}
1714 				goto retry_recover;
1715 			}
1716 
1717 			goto out;
1718 		}
1719 
1720 		ctxi->err_recovery_active = false;
1721 
1722 		flags = DK_CXLFLASH_APP_CLOSE_ADAP_FD |
1723 			DK_CXLFLASH_RECOVER_AFU_CONTEXT_RESET;
1724 		if (afu_is_sq_cmd_mode(afu))
1725 			flags |= DK_CXLFLASH_CONTEXT_SQ_CMD_MODE;
1726 
1727 		recover->hdr.return_flags = flags;
1728 		recover->context_id = ctxi->ctxid;
1729 		recover->adap_fd = new_adap_fd;
1730 		recover->mmio_size = sizeof(afu->afu_map->hosts[0].harea);
1731 		goto out;
1732 	}
1733 
1734 	/* Test if in error state */
1735 	reg = readq_be(&hwq->ctrl_map->mbox_r);
1736 	if (reg == -1) {
1737 		dev_dbg(dev, "%s: MMIO fail, wait for recovery.\n", __func__);
1738 
1739 		/*
1740 		 * Before checking the state, put back the context obtained with
1741 		 * get_context() as it is no longer needed and sleep for a short
1742 		 * period of time (see prolog notes).
1743 		 */
1744 		put_context(ctxi);
1745 		ctxi = NULL;
1746 		ssleep(1);
1747 		rc = check_state(cfg);
1748 		if (unlikely(rc))
1749 			goto out;
1750 		goto retry;
1751 	}
1752 
1753 	dev_dbg(dev, "%s: MMIO working, no recovery required\n", __func__);
1754 out:
1755 	if (likely(ctxi))
1756 		put_context(ctxi);
1757 	if (locked)
1758 		mutex_unlock(mutex);
1759 	atomic_dec_if_positive(&cfg->recovery_threads);
1760 	return rc;
1761 }
1762 
1763 /**
1764  * process_sense() - evaluates and processes sense data
1765  * @sdev:	SCSI device associated with LUN.
1766  * @verify:	Verify ioctl data structure.
1767  *
1768  * Return: 0 on success, -errno on failure
1769  */
process_sense(struct scsi_device * sdev,struct dk_cxlflash_verify * verify)1770 static int process_sense(struct scsi_device *sdev,
1771 			 struct dk_cxlflash_verify *verify)
1772 {
1773 	struct cxlflash_cfg *cfg = shost_priv(sdev->host);
1774 	struct device *dev = &cfg->dev->dev;
1775 	struct llun_info *lli = sdev->hostdata;
1776 	struct glun_info *gli = lli->parent;
1777 	u64 prev_lba = gli->max_lba;
1778 	struct scsi_sense_hdr sshdr = { 0 };
1779 	int rc = 0;
1780 
1781 	rc = scsi_normalize_sense((const u8 *)&verify->sense_data,
1782 				  DK_CXLFLASH_VERIFY_SENSE_LEN, &sshdr);
1783 	if (!rc) {
1784 		dev_err(dev, "%s: Failed to normalize sense data\n", __func__);
1785 		rc = -EINVAL;
1786 		goto out;
1787 	}
1788 
1789 	switch (sshdr.sense_key) {
1790 	case NO_SENSE:
1791 	case RECOVERED_ERROR:
1792 	case NOT_READY:
1793 		break;
1794 	case UNIT_ATTENTION:
1795 		switch (sshdr.asc) {
1796 		case 0x29: /* Power on Reset or Device Reset */
1797 			fallthrough;
1798 		case 0x2A: /* Device settings/capacity changed */
1799 			rc = read_cap16(sdev, lli);
1800 			if (rc) {
1801 				rc = -ENODEV;
1802 				break;
1803 			}
1804 			if (prev_lba != gli->max_lba)
1805 				dev_dbg(dev, "%s: Capacity changed old=%lld "
1806 					"new=%lld\n", __func__, prev_lba,
1807 					gli->max_lba);
1808 			break;
1809 		case 0x3F: /* Report LUNs changed, Rescan. */
1810 			scsi_scan_host(cfg->host);
1811 			break;
1812 		default:
1813 			rc = -EIO;
1814 			break;
1815 		}
1816 		break;
1817 	default:
1818 		rc = -EIO;
1819 		break;
1820 	}
1821 out:
1822 	dev_dbg(dev, "%s: sense_key %x asc %x ascq %x rc %d\n", __func__,
1823 		sshdr.sense_key, sshdr.asc, sshdr.ascq, rc);
1824 	return rc;
1825 }
1826 
1827 /**
1828  * cxlflash_disk_verify() - verifies a LUN is the same and handle size changes
1829  * @sdev:	SCSI device associated with LUN.
1830  * @arg:	Verify ioctl data structure.
1831  *
1832  * Return: 0 on success, -errno on failure
1833  */
cxlflash_disk_verify(struct scsi_device * sdev,void * arg)1834 static int cxlflash_disk_verify(struct scsi_device *sdev, void *arg)
1835 {
1836 	struct dk_cxlflash_verify *verify = arg;
1837 	int rc = 0;
1838 	struct ctx_info *ctxi = NULL;
1839 	struct cxlflash_cfg *cfg = shost_priv(sdev->host);
1840 	struct device *dev = &cfg->dev->dev;
1841 	struct llun_info *lli = sdev->hostdata;
1842 	struct glun_info *gli = lli->parent;
1843 	struct sisl_rht_entry *rhte = NULL;
1844 	res_hndl_t rhndl = verify->rsrc_handle;
1845 	u64 ctxid = DECODE_CTXID(verify->context_id),
1846 	    rctxid = verify->context_id;
1847 	u64 last_lba = 0;
1848 
1849 	dev_dbg(dev, "%s: ctxid=%llu rhndl=%016llx, hint=%016llx, "
1850 		"flags=%016llx\n", __func__, ctxid, verify->rsrc_handle,
1851 		verify->hint, verify->hdr.flags);
1852 
1853 	ctxi = get_context(cfg, rctxid, lli, 0);
1854 	if (unlikely(!ctxi)) {
1855 		dev_dbg(dev, "%s: Bad context ctxid=%llu\n", __func__, ctxid);
1856 		rc = -EINVAL;
1857 		goto out;
1858 	}
1859 
1860 	rhte = get_rhte(ctxi, rhndl, lli);
1861 	if (unlikely(!rhte)) {
1862 		dev_dbg(dev, "%s: Bad resource handle rhndl=%d\n",
1863 			__func__, rhndl);
1864 		rc = -EINVAL;
1865 		goto out;
1866 	}
1867 
1868 	/*
1869 	 * Look at the hint/sense to see if it requires us to redrive
1870 	 * inquiry (i.e. the Unit attention is due to the WWN changing).
1871 	 */
1872 	if (verify->hint & DK_CXLFLASH_VERIFY_HINT_SENSE) {
1873 		/* Can't hold mutex across process_sense/read_cap16,
1874 		 * since we could have an intervening EEH event.
1875 		 */
1876 		ctxi->unavail = true;
1877 		mutex_unlock(&ctxi->mutex);
1878 		rc = process_sense(sdev, verify);
1879 		if (unlikely(rc)) {
1880 			dev_err(dev, "%s: Failed to validate sense data (%d)\n",
1881 				__func__, rc);
1882 			mutex_lock(&ctxi->mutex);
1883 			ctxi->unavail = false;
1884 			goto out;
1885 		}
1886 		mutex_lock(&ctxi->mutex);
1887 		ctxi->unavail = false;
1888 	}
1889 
1890 	switch (gli->mode) {
1891 	case MODE_PHYSICAL:
1892 		last_lba = gli->max_lba;
1893 		break;
1894 	case MODE_VIRTUAL:
1895 		/* Cast lxt_cnt to u64 for multiply to be treated as 64bit op */
1896 		last_lba = ((u64)rhte->lxt_cnt * MC_CHUNK_SIZE * gli->blk_len);
1897 		last_lba /= CXLFLASH_BLOCK_SIZE;
1898 		last_lba--;
1899 		break;
1900 	default:
1901 		WARN(1, "Unsupported LUN mode!");
1902 	}
1903 
1904 	verify->last_lba = last_lba;
1905 
1906 out:
1907 	if (likely(ctxi))
1908 		put_context(ctxi);
1909 	dev_dbg(dev, "%s: returning rc=%d llba=%llx\n",
1910 		__func__, rc, verify->last_lba);
1911 	return rc;
1912 }
1913 
1914 /**
1915  * decode_ioctl() - translates an encoded ioctl to an easily identifiable string
1916  * @cmd:	The ioctl command to decode.
1917  *
1918  * Return: A string identifying the decoded ioctl.
1919  */
decode_ioctl(unsigned int cmd)1920 static char *decode_ioctl(unsigned int cmd)
1921 {
1922 	switch (cmd) {
1923 	case DK_CXLFLASH_ATTACH:
1924 		return __stringify_1(DK_CXLFLASH_ATTACH);
1925 	case DK_CXLFLASH_USER_DIRECT:
1926 		return __stringify_1(DK_CXLFLASH_USER_DIRECT);
1927 	case DK_CXLFLASH_USER_VIRTUAL:
1928 		return __stringify_1(DK_CXLFLASH_USER_VIRTUAL);
1929 	case DK_CXLFLASH_VLUN_RESIZE:
1930 		return __stringify_1(DK_CXLFLASH_VLUN_RESIZE);
1931 	case DK_CXLFLASH_RELEASE:
1932 		return __stringify_1(DK_CXLFLASH_RELEASE);
1933 	case DK_CXLFLASH_DETACH:
1934 		return __stringify_1(DK_CXLFLASH_DETACH);
1935 	case DK_CXLFLASH_VERIFY:
1936 		return __stringify_1(DK_CXLFLASH_VERIFY);
1937 	case DK_CXLFLASH_VLUN_CLONE:
1938 		return __stringify_1(DK_CXLFLASH_VLUN_CLONE);
1939 	case DK_CXLFLASH_RECOVER_AFU:
1940 		return __stringify_1(DK_CXLFLASH_RECOVER_AFU);
1941 	case DK_CXLFLASH_MANAGE_LUN:
1942 		return __stringify_1(DK_CXLFLASH_MANAGE_LUN);
1943 	}
1944 
1945 	return "UNKNOWN";
1946 }
1947 
1948 /**
1949  * cxlflash_disk_direct_open() - opens a direct (physical) disk
1950  * @sdev:	SCSI device associated with LUN.
1951  * @arg:	UDirect ioctl data structure.
1952  *
1953  * On successful return, the user is informed of the resource handle
1954  * to be used to identify the direct lun and the size (in blocks) of
1955  * the direct lun in last LBA format.
1956  *
1957  * Return: 0 on success, -errno on failure
1958  */
cxlflash_disk_direct_open(struct scsi_device * sdev,void * arg)1959 static int cxlflash_disk_direct_open(struct scsi_device *sdev, void *arg)
1960 {
1961 	struct cxlflash_cfg *cfg = shost_priv(sdev->host);
1962 	struct device *dev = &cfg->dev->dev;
1963 	struct afu *afu = cfg->afu;
1964 	struct llun_info *lli = sdev->hostdata;
1965 	struct glun_info *gli = lli->parent;
1966 	struct dk_cxlflash_release rel = { { 0 }, 0 };
1967 
1968 	struct dk_cxlflash_udirect *pphys = (struct dk_cxlflash_udirect *)arg;
1969 
1970 	u64 ctxid = DECODE_CTXID(pphys->context_id),
1971 	    rctxid = pphys->context_id;
1972 	u64 lun_size = 0;
1973 	u64 last_lba = 0;
1974 	u64 rsrc_handle = -1;
1975 	u32 port = CHAN2PORTMASK(sdev->channel);
1976 
1977 	int rc = 0;
1978 
1979 	struct ctx_info *ctxi = NULL;
1980 	struct sisl_rht_entry *rhte = NULL;
1981 
1982 	dev_dbg(dev, "%s: ctxid=%llu ls=%llu\n", __func__, ctxid, lun_size);
1983 
1984 	rc = cxlflash_lun_attach(gli, MODE_PHYSICAL, false);
1985 	if (unlikely(rc)) {
1986 		dev_dbg(dev, "%s: Failed attach to LUN (PHYSICAL)\n", __func__);
1987 		goto out;
1988 	}
1989 
1990 	ctxi = get_context(cfg, rctxid, lli, 0);
1991 	if (unlikely(!ctxi)) {
1992 		dev_dbg(dev, "%s: Bad context ctxid=%llu\n", __func__, ctxid);
1993 		rc = -EINVAL;
1994 		goto err1;
1995 	}
1996 
1997 	rhte = rhte_checkout(ctxi, lli);
1998 	if (unlikely(!rhte)) {
1999 		dev_dbg(dev, "%s: Too many opens ctxid=%lld\n",
2000 			__func__, ctxid);
2001 		rc = -EMFILE;	/* too many opens  */
2002 		goto err1;
2003 	}
2004 
2005 	rsrc_handle = (rhte - ctxi->rht_start);
2006 
2007 	rht_format1(rhte, lli->lun_id[sdev->channel], ctxi->rht_perms, port);
2008 
2009 	last_lba = gli->max_lba;
2010 	pphys->hdr.return_flags = 0;
2011 	pphys->last_lba = last_lba;
2012 	pphys->rsrc_handle = rsrc_handle;
2013 
2014 	rc = cxlflash_afu_sync(afu, ctxid, rsrc_handle, AFU_LW_SYNC);
2015 	if (unlikely(rc)) {
2016 		dev_dbg(dev, "%s: AFU sync failed rc=%d\n", __func__, rc);
2017 		goto err2;
2018 	}
2019 
2020 out:
2021 	if (likely(ctxi))
2022 		put_context(ctxi);
2023 	dev_dbg(dev, "%s: returning handle=%llu rc=%d llba=%llu\n",
2024 		__func__, rsrc_handle, rc, last_lba);
2025 	return rc;
2026 
2027 err2:
2028 	marshal_udir_to_rele(pphys, &rel);
2029 	_cxlflash_disk_release(sdev, ctxi, &rel);
2030 	goto out;
2031 err1:
2032 	cxlflash_lun_detach(gli);
2033 	goto out;
2034 }
2035 
2036 /**
2037  * ioctl_common() - common IOCTL handler for driver
2038  * @sdev:	SCSI device associated with LUN.
2039  * @cmd:	IOCTL command.
2040  *
2041  * Handles common fencing operations that are valid for multiple ioctls. Always
2042  * allow through ioctls that are cleanup oriented in nature, even when operating
2043  * in a failed/terminating state.
2044  *
2045  * Return: 0 on success, -errno on failure
2046  */
ioctl_common(struct scsi_device * sdev,unsigned int cmd)2047 static int ioctl_common(struct scsi_device *sdev, unsigned int cmd)
2048 {
2049 	struct cxlflash_cfg *cfg = shost_priv(sdev->host);
2050 	struct device *dev = &cfg->dev->dev;
2051 	struct llun_info *lli = sdev->hostdata;
2052 	int rc = 0;
2053 
2054 	if (unlikely(!lli)) {
2055 		dev_dbg(dev, "%s: Unknown LUN\n", __func__);
2056 		rc = -EINVAL;
2057 		goto out;
2058 	}
2059 
2060 	rc = check_state(cfg);
2061 	if (unlikely(rc) && (cfg->state == STATE_FAILTERM)) {
2062 		switch (cmd) {
2063 		case DK_CXLFLASH_VLUN_RESIZE:
2064 		case DK_CXLFLASH_RELEASE:
2065 		case DK_CXLFLASH_DETACH:
2066 			dev_dbg(dev, "%s: Command override rc=%d\n",
2067 				__func__, rc);
2068 			rc = 0;
2069 			break;
2070 		}
2071 	}
2072 out:
2073 	return rc;
2074 }
2075 
2076 /**
2077  * cxlflash_ioctl() - IOCTL handler for driver
2078  * @sdev:	SCSI device associated with LUN.
2079  * @cmd:	IOCTL command.
2080  * @arg:	Userspace ioctl data structure.
2081  *
2082  * A read/write semaphore is used to implement a 'drain' of currently
2083  * running ioctls. The read semaphore is taken at the beginning of each
2084  * ioctl thread and released upon concluding execution. Additionally the
2085  * semaphore should be released and then reacquired in any ioctl execution
2086  * path which will wait for an event to occur that is outside the scope of
2087  * the ioctl (i.e. an adapter reset). To drain the ioctls currently running,
2088  * a thread simply needs to acquire the write semaphore.
2089  *
2090  * Return: 0 on success, -errno on failure
2091  */
cxlflash_ioctl(struct scsi_device * sdev,unsigned int cmd,void __user * arg)2092 int cxlflash_ioctl(struct scsi_device *sdev, unsigned int cmd, void __user *arg)
2093 {
2094 	typedef int (*sioctl) (struct scsi_device *, void *);
2095 
2096 	struct cxlflash_cfg *cfg = shost_priv(sdev->host);
2097 	struct device *dev = &cfg->dev->dev;
2098 	struct afu *afu = cfg->afu;
2099 	struct dk_cxlflash_hdr *hdr;
2100 	char buf[sizeof(union cxlflash_ioctls)];
2101 	size_t size = 0;
2102 	bool known_ioctl = false;
2103 	int idx;
2104 	int rc = 0;
2105 	struct Scsi_Host *shost = sdev->host;
2106 	sioctl do_ioctl = NULL;
2107 
2108 	static const struct {
2109 		size_t size;
2110 		sioctl ioctl;
2111 	} ioctl_tbl[] = {	/* NOTE: order matters here */
2112 	{sizeof(struct dk_cxlflash_attach), cxlflash_disk_attach},
2113 	{sizeof(struct dk_cxlflash_udirect), cxlflash_disk_direct_open},
2114 	{sizeof(struct dk_cxlflash_release), cxlflash_disk_release},
2115 	{sizeof(struct dk_cxlflash_detach), cxlflash_disk_detach},
2116 	{sizeof(struct dk_cxlflash_verify), cxlflash_disk_verify},
2117 	{sizeof(struct dk_cxlflash_recover_afu), cxlflash_afu_recover},
2118 	{sizeof(struct dk_cxlflash_manage_lun), cxlflash_manage_lun},
2119 	{sizeof(struct dk_cxlflash_uvirtual), cxlflash_disk_virtual_open},
2120 	{sizeof(struct dk_cxlflash_resize), cxlflash_vlun_resize},
2121 	{sizeof(struct dk_cxlflash_clone), cxlflash_disk_clone},
2122 	};
2123 
2124 	/* Hold read semaphore so we can drain if needed */
2125 	down_read(&cfg->ioctl_rwsem);
2126 
2127 	/* Restrict command set to physical support only for internal LUN */
2128 	if (afu->internal_lun)
2129 		switch (cmd) {
2130 		case DK_CXLFLASH_RELEASE:
2131 		case DK_CXLFLASH_USER_VIRTUAL:
2132 		case DK_CXLFLASH_VLUN_RESIZE:
2133 		case DK_CXLFLASH_VLUN_CLONE:
2134 			dev_dbg(dev, "%s: %s not supported for lun_mode=%d\n",
2135 				__func__, decode_ioctl(cmd), afu->internal_lun);
2136 			rc = -EINVAL;
2137 			goto cxlflash_ioctl_exit;
2138 		}
2139 
2140 	switch (cmd) {
2141 	case DK_CXLFLASH_ATTACH:
2142 	case DK_CXLFLASH_USER_DIRECT:
2143 	case DK_CXLFLASH_RELEASE:
2144 	case DK_CXLFLASH_DETACH:
2145 	case DK_CXLFLASH_VERIFY:
2146 	case DK_CXLFLASH_RECOVER_AFU:
2147 	case DK_CXLFLASH_USER_VIRTUAL:
2148 	case DK_CXLFLASH_VLUN_RESIZE:
2149 	case DK_CXLFLASH_VLUN_CLONE:
2150 		dev_dbg(dev, "%s: %s (%08X) on dev(%d/%d/%d/%llu)\n",
2151 			__func__, decode_ioctl(cmd), cmd, shost->host_no,
2152 			sdev->channel, sdev->id, sdev->lun);
2153 		rc = ioctl_common(sdev, cmd);
2154 		if (unlikely(rc))
2155 			goto cxlflash_ioctl_exit;
2156 
2157 		fallthrough;
2158 
2159 	case DK_CXLFLASH_MANAGE_LUN:
2160 		known_ioctl = true;
2161 		idx = _IOC_NR(cmd) - _IOC_NR(DK_CXLFLASH_ATTACH);
2162 		size = ioctl_tbl[idx].size;
2163 		do_ioctl = ioctl_tbl[idx].ioctl;
2164 
2165 		if (likely(do_ioctl))
2166 			break;
2167 
2168 		fallthrough;
2169 	default:
2170 		rc = -EINVAL;
2171 		goto cxlflash_ioctl_exit;
2172 	}
2173 
2174 	if (unlikely(copy_from_user(&buf, arg, size))) {
2175 		dev_err(dev, "%s: copy_from_user() fail size=%lu cmd=%u (%s) arg=%p\n",
2176 			__func__, size, cmd, decode_ioctl(cmd), arg);
2177 		rc = -EFAULT;
2178 		goto cxlflash_ioctl_exit;
2179 	}
2180 
2181 	hdr = (struct dk_cxlflash_hdr *)&buf;
2182 	if (hdr->version != DK_CXLFLASH_VERSION_0) {
2183 		dev_dbg(dev, "%s: Version %u not supported for %s\n",
2184 			__func__, hdr->version, decode_ioctl(cmd));
2185 		rc = -EINVAL;
2186 		goto cxlflash_ioctl_exit;
2187 	}
2188 
2189 	if (hdr->rsvd[0] || hdr->rsvd[1] || hdr->rsvd[2] || hdr->return_flags) {
2190 		dev_dbg(dev, "%s: Reserved/rflags populated\n", __func__);
2191 		rc = -EINVAL;
2192 		goto cxlflash_ioctl_exit;
2193 	}
2194 
2195 	rc = do_ioctl(sdev, (void *)&buf);
2196 	if (likely(!rc))
2197 		if (unlikely(copy_to_user(arg, &buf, size))) {
2198 			dev_err(dev, "%s: copy_to_user() fail size=%lu cmd=%u (%s) arg=%p\n",
2199 				__func__, size, cmd, decode_ioctl(cmd), arg);
2200 			rc = -EFAULT;
2201 		}
2202 
2203 	/* fall through to exit */
2204 
2205 cxlflash_ioctl_exit:
2206 	up_read(&cfg->ioctl_rwsem);
2207 	if (unlikely(rc && known_ioctl))
2208 		dev_err(dev, "%s: ioctl %s (%08X) on dev(%d/%d/%d/%llu) "
2209 			"returned rc %d\n", __func__,
2210 			decode_ioctl(cmd), cmd, shost->host_no,
2211 			sdev->channel, sdev->id, sdev->lun, rc);
2212 	else
2213 		dev_dbg(dev, "%s: ioctl %s (%08X) on dev(%d/%d/%d/%llu) "
2214 			"returned rc %d\n", __func__, decode_ioctl(cmd),
2215 			cmd, shost->host_no, sdev->channel, sdev->id,
2216 			sdev->lun, rc);
2217 	return rc;
2218 }
2219