xref: /linux/drivers/misc/bcm-vk/bcm_vk_msg.c (revision 0ea5c948cb64bab5bc7a5516774eb8536f05aa0d)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright 2018-2020 Broadcom.
4  */
5 
6 #include <linux/delay.h>
7 #include <linux/fs.h>
8 #include <linux/hash.h>
9 #include <linux/interrupt.h>
10 #include <linux/list.h>
11 #include <linux/module.h>
12 #include <linux/poll.h>
13 #include <linux/sizes.h>
14 #include <linux/spinlock.h>
15 #include <linux/timer.h>
16 
17 #include "bcm_vk.h"
18 #include "bcm_vk_msg.h"
19 #include "bcm_vk_sg.h"
20 
21 /* functions to manipulate the transport id in msg block */
22 #define BCM_VK_MSG_Q_SHIFT	 4
23 #define BCM_VK_MSG_Q_MASK	 0xF
24 #define BCM_VK_MSG_ID_MASK	 0xFFF
25 
26 #define BCM_VK_DMA_DRAIN_MAX_MS	  2000
27 
28 /* number x q_size will be the max number of msg processed per loop */
29 #define BCM_VK_MSG_PROC_MAX_LOOP 2
30 
31 /* module parameter */
32 static bool hb_mon = true;
33 module_param(hb_mon, bool, 0444);
34 MODULE_PARM_DESC(hb_mon, "Monitoring heartbeat continuously.\n");
35 static int batch_log = 1;
36 module_param(batch_log, int, 0444);
37 MODULE_PARM_DESC(batch_log, "Max num of logs per batch operation.\n");
38 
hb_mon_is_on(void)39 static bool hb_mon_is_on(void)
40 {
41 	return hb_mon;
42 }
43 
get_q_num(const struct vk_msg_blk * msg)44 static u32 get_q_num(const struct vk_msg_blk *msg)
45 {
46 	u32 q_num = msg->trans_id & BCM_VK_MSG_Q_MASK;
47 
48 	if (q_num >= VK_MSGQ_PER_CHAN_MAX)
49 		q_num = VK_MSGQ_NUM_DEFAULT;
50 	return q_num;
51 }
52 
set_q_num(struct vk_msg_blk * msg,u32 q_num)53 static void set_q_num(struct vk_msg_blk *msg, u32 q_num)
54 {
55 	u32 trans_q;
56 
57 	if (q_num >= VK_MSGQ_PER_CHAN_MAX)
58 		trans_q = VK_MSGQ_NUM_DEFAULT;
59 	else
60 		trans_q = q_num;
61 
62 	msg->trans_id = (msg->trans_id & ~BCM_VK_MSG_Q_MASK) | trans_q;
63 }
64 
get_msg_id(const struct vk_msg_blk * msg)65 static u32 get_msg_id(const struct vk_msg_blk *msg)
66 {
67 	return ((msg->trans_id >> BCM_VK_MSG_Q_SHIFT) & BCM_VK_MSG_ID_MASK);
68 }
69 
set_msg_id(struct vk_msg_blk * msg,u32 val)70 static void set_msg_id(struct vk_msg_blk *msg, u32 val)
71 {
72 	msg->trans_id = (val << BCM_VK_MSG_Q_SHIFT) | get_q_num(msg);
73 }
74 
msgq_inc(const struct bcm_vk_sync_qinfo * qinfo,u32 idx,u32 inc)75 static u32 msgq_inc(const struct bcm_vk_sync_qinfo *qinfo, u32 idx, u32 inc)
76 {
77 	return ((idx + inc) & qinfo->q_mask);
78 }
79 
80 static
msgq_blk_addr(const struct bcm_vk_sync_qinfo * qinfo,u32 idx)81 struct vk_msg_blk __iomem *msgq_blk_addr(const struct bcm_vk_sync_qinfo *qinfo,
82 					 u32 idx)
83 {
84 	return qinfo->q_start + (VK_MSGQ_BLK_SIZE * idx);
85 }
86 
msgq_occupied(const struct bcm_vk_msgq __iomem * msgq,const struct bcm_vk_sync_qinfo * qinfo)87 static u32 msgq_occupied(const struct bcm_vk_msgq __iomem *msgq,
88 			 const struct bcm_vk_sync_qinfo *qinfo)
89 {
90 	u32 wr_idx, rd_idx;
91 
92 	wr_idx = readl_relaxed(&msgq->wr_idx);
93 	rd_idx = readl_relaxed(&msgq->rd_idx);
94 
95 	return ((wr_idx - rd_idx) & qinfo->q_mask);
96 }
97 
98 static
msgq_avail_space(const struct bcm_vk_msgq __iomem * msgq,const struct bcm_vk_sync_qinfo * qinfo)99 u32 msgq_avail_space(const struct bcm_vk_msgq __iomem *msgq,
100 		     const struct bcm_vk_sync_qinfo *qinfo)
101 {
102 	return (qinfo->q_size - msgq_occupied(msgq, qinfo) - 1);
103 }
104 
105 /* number of retries when enqueue message fails before returning EAGAIN */
106 #define BCM_VK_H2VK_ENQ_RETRY 10
107 #define BCM_VK_H2VK_ENQ_RETRY_DELAY_MS 50
108 
bcm_vk_drv_access_ok(struct bcm_vk * vk)109 bool bcm_vk_drv_access_ok(struct bcm_vk *vk)
110 {
111 	return (!!atomic_read(&vk->msgq_inited));
112 }
113 
bcm_vk_set_host_alert(struct bcm_vk * vk,u32 bit_mask)114 void bcm_vk_set_host_alert(struct bcm_vk *vk, u32 bit_mask)
115 {
116 	struct bcm_vk_alert *alert = &vk->host_alert;
117 	unsigned long flags;
118 
119 	/* use irqsave version as this maybe called inside timer interrupt */
120 	spin_lock_irqsave(&vk->host_alert_lock, flags);
121 	alert->notfs |= bit_mask;
122 	spin_unlock_irqrestore(&vk->host_alert_lock, flags);
123 
124 	if (test_and_set_bit(BCM_VK_WQ_NOTF_PEND, vk->wq_offload) == 0)
125 		queue_work(vk->wq_thread, &vk->wq_work);
126 }
127 
128 /*
129  * Heartbeat related defines
130  * The heartbeat from host is a last resort.  If stuck condition happens
131  * on the card, firmware is supposed to detect it.  Therefore, the heartbeat
132  * values used will be more relaxed on the driver, which need to be bigger
133  * than the watchdog timeout on the card.  The watchdog timeout on the card
134  * is 20s, with a jitter of 2s => 22s.  We use a value of 27s here.
135  */
136 #define BCM_VK_HB_TIMER_S 3
137 #define BCM_VK_HB_TIMER_VALUE (BCM_VK_HB_TIMER_S * HZ)
138 #define BCM_VK_HB_LOST_MAX (27 / BCM_VK_HB_TIMER_S)
139 
bcm_vk_hb_poll(struct work_struct * work)140 static void bcm_vk_hb_poll(struct work_struct *work)
141 {
142 	u32 uptime_s;
143 	struct bcm_vk_hb_ctrl *hb = container_of(to_delayed_work(work), struct bcm_vk_hb_ctrl,
144 						 work);
145 	struct bcm_vk *vk = container_of(hb, struct bcm_vk, hb_ctrl);
146 
147 	if (bcm_vk_drv_access_ok(vk) && hb_mon_is_on()) {
148 		/* read uptime from register and compare */
149 		uptime_s = vkread32(vk, BAR_0, BAR_OS_UPTIME);
150 
151 		if (uptime_s == hb->last_uptime)
152 			hb->lost_cnt++;
153 		else /* reset to avoid accumulation */
154 			hb->lost_cnt = 0;
155 
156 		dev_dbg(&vk->pdev->dev, "Last uptime %d current %d, lost %d\n",
157 			hb->last_uptime, uptime_s, hb->lost_cnt);
158 
159 		/*
160 		 * if the interface goes down without any activity, a value
161 		 * of 0xFFFFFFFF will be continuously read, and the detection
162 		 * will be happened eventually.
163 		 */
164 		hb->last_uptime = uptime_s;
165 	} else {
166 		/* reset heart beat lost cnt */
167 		hb->lost_cnt = 0;
168 	}
169 
170 	/* next, check if heartbeat exceeds limit */
171 	if (hb->lost_cnt > BCM_VK_HB_LOST_MAX) {
172 		dev_err(&vk->pdev->dev, "Heartbeat Misses %d times, %d s!\n",
173 			BCM_VK_HB_LOST_MAX,
174 			BCM_VK_HB_LOST_MAX * BCM_VK_HB_TIMER_S);
175 
176 		bcm_vk_blk_drv_access(vk);
177 		bcm_vk_set_host_alert(vk, ERR_LOG_HOST_HB_FAIL);
178 	}
179 	/* re-arm timer */
180 	schedule_delayed_work(&hb->work, BCM_VK_HB_TIMER_VALUE);
181 }
182 
bcm_vk_hb_init(struct bcm_vk * vk)183 void bcm_vk_hb_init(struct bcm_vk *vk)
184 {
185 	struct bcm_vk_hb_ctrl *hb = &vk->hb_ctrl;
186 
187 	INIT_DELAYED_WORK(&hb->work, bcm_vk_hb_poll);
188 	schedule_delayed_work(&hb->work, BCM_VK_HB_TIMER_VALUE);
189 }
190 
bcm_vk_hb_deinit(struct bcm_vk * vk)191 void bcm_vk_hb_deinit(struct bcm_vk *vk)
192 {
193 	struct bcm_vk_hb_ctrl *hb = &vk->hb_ctrl;
194 
195 	cancel_delayed_work_sync(&hb->work);
196 }
197 
bcm_vk_msgid_bitmap_clear(struct bcm_vk * vk,unsigned int start,unsigned int nbits)198 static void bcm_vk_msgid_bitmap_clear(struct bcm_vk *vk,
199 				      unsigned int start,
200 				      unsigned int nbits)
201 {
202 	spin_lock(&vk->msg_id_lock);
203 	bitmap_clear(vk->bmap, start, nbits);
204 	spin_unlock(&vk->msg_id_lock);
205 }
206 
207 /*
208  * allocate a ctx per file struct
209  */
bcm_vk_get_ctx(struct bcm_vk * vk,const pid_t pid)210 static struct bcm_vk_ctx *bcm_vk_get_ctx(struct bcm_vk *vk, const pid_t pid)
211 {
212 	u32 i;
213 	struct bcm_vk_ctx *ctx = NULL;
214 	u32 hash_idx = hash_32(pid, VK_PID_HT_SHIFT_BIT);
215 
216 	spin_lock(&vk->ctx_lock);
217 
218 	/* check if it is in reset, if so, don't allow */
219 	if (vk->reset_pid) {
220 		dev_err(&vk->pdev->dev,
221 			"No context allowed during reset by pid %d\n",
222 			vk->reset_pid);
223 
224 		goto in_reset_exit;
225 	}
226 
227 	for (i = 0; i < ARRAY_SIZE(vk->ctx); i++) {
228 		if (!vk->ctx[i].in_use) {
229 			vk->ctx[i].in_use = true;
230 			ctx = &vk->ctx[i];
231 			break;
232 		}
233 	}
234 
235 	if (!ctx) {
236 		dev_err(&vk->pdev->dev, "All context in use\n");
237 
238 		goto all_in_use_exit;
239 	}
240 
241 	/* set the pid and insert it to hash table */
242 	ctx->pid = pid;
243 	ctx->hash_idx = hash_idx;
244 	list_add_tail(&ctx->node, &vk->pid_ht[hash_idx].head);
245 
246 	/* increase kref */
247 	kref_get(&vk->kref);
248 
249 	/* clear counter */
250 	atomic_set(&ctx->pend_cnt, 0);
251 	atomic_set(&ctx->dma_cnt, 0);
252 	init_waitqueue_head(&ctx->rd_wq);
253 
254 all_in_use_exit:
255 in_reset_exit:
256 	spin_unlock(&vk->ctx_lock);
257 
258 	return ctx;
259 }
260 
bcm_vk_get_msg_id(struct bcm_vk * vk)261 static u16 bcm_vk_get_msg_id(struct bcm_vk *vk)
262 {
263 	u16 rc = VK_MSG_ID_OVERFLOW;
264 	u16 test_bit_count = 0;
265 
266 	spin_lock(&vk->msg_id_lock);
267 	while (test_bit_count < (VK_MSG_ID_BITMAP_SIZE - 1)) {
268 		/*
269 		 * first time come in this loop, msg_id will be 0
270 		 * and the first one tested will be 1.  We skip
271 		 * VK_SIMPLEX_MSG_ID (0) for one way host2vk
272 		 * communication
273 		 */
274 		vk->msg_id++;
275 		if (vk->msg_id == VK_MSG_ID_BITMAP_SIZE)
276 			vk->msg_id = 1;
277 
278 		if (test_bit(vk->msg_id, vk->bmap)) {
279 			test_bit_count++;
280 			continue;
281 		}
282 		rc = vk->msg_id;
283 		bitmap_set(vk->bmap, vk->msg_id, 1);
284 		break;
285 	}
286 	spin_unlock(&vk->msg_id_lock);
287 
288 	return rc;
289 }
290 
bcm_vk_free_ctx(struct bcm_vk * vk,struct bcm_vk_ctx * ctx)291 static int bcm_vk_free_ctx(struct bcm_vk *vk, struct bcm_vk_ctx *ctx)
292 {
293 	u32 idx;
294 	u32 hash_idx;
295 	pid_t pid;
296 	struct bcm_vk_ctx *entry;
297 	int count = 0;
298 
299 	if (!ctx) {
300 		dev_err(&vk->pdev->dev, "NULL context detected\n");
301 		return -EINVAL;
302 	}
303 	idx = ctx->idx;
304 	pid = ctx->pid;
305 
306 	spin_lock(&vk->ctx_lock);
307 
308 	if (!vk->ctx[idx].in_use) {
309 		dev_err(&vk->pdev->dev, "context[%d] not in use!\n", idx);
310 	} else {
311 		vk->ctx[idx].in_use = false;
312 		vk->ctx[idx].miscdev = NULL;
313 
314 		/* Remove it from hash list and see if it is the last one. */
315 		list_del(&ctx->node);
316 		hash_idx = ctx->hash_idx;
317 		list_for_each_entry(entry, &vk->pid_ht[hash_idx].head, node) {
318 			if (entry->pid == pid)
319 				count++;
320 		}
321 	}
322 
323 	spin_unlock(&vk->ctx_lock);
324 
325 	return count;
326 }
327 
bcm_vk_free_wkent(struct device * dev,struct bcm_vk_wkent * entry)328 static void bcm_vk_free_wkent(struct device *dev, struct bcm_vk_wkent *entry)
329 {
330 	int proc_cnt;
331 
332 	bcm_vk_sg_free(dev, entry->dma, VK_DMA_MAX_ADDRS, &proc_cnt);
333 	if (proc_cnt)
334 		atomic_dec(&entry->ctx->dma_cnt);
335 
336 	kfree(entry->to_h_msg);
337 	kfree(entry);
338 }
339 
bcm_vk_drain_all_pend(struct device * dev,struct bcm_vk_msg_chan * chan,struct bcm_vk_ctx * ctx)340 static void bcm_vk_drain_all_pend(struct device *dev,
341 				  struct bcm_vk_msg_chan *chan,
342 				  struct bcm_vk_ctx *ctx)
343 {
344 	u32 num;
345 	struct bcm_vk_wkent *entry, *tmp;
346 	struct bcm_vk *vk;
347 	struct list_head del_q;
348 
349 	if (ctx)
350 		vk = container_of(ctx->miscdev, struct bcm_vk, miscdev);
351 
352 	INIT_LIST_HEAD(&del_q);
353 	spin_lock(&chan->pendq_lock);
354 	for (num = 0; num < chan->q_nr; num++) {
355 		list_for_each_entry_safe(entry, tmp, &chan->pendq[num], node) {
356 			if ((!ctx) || (entry->ctx->idx == ctx->idx)) {
357 				list_move_tail(&entry->node, &del_q);
358 			}
359 		}
360 	}
361 	spin_unlock(&chan->pendq_lock);
362 
363 	/* batch clean up */
364 	num = 0;
365 	list_for_each_entry_safe(entry, tmp, &del_q, node) {
366 		list_del(&entry->node);
367 		num++;
368 		if (ctx) {
369 			struct vk_msg_blk *msg;
370 			int bit_set;
371 			bool responded;
372 			u32 msg_id;
373 
374 			/* if it is specific ctx, log for any stuck */
375 			msg = entry->to_v_msg;
376 			msg_id = get_msg_id(msg);
377 			bit_set = test_bit(msg_id, vk->bmap);
378 			responded = entry->to_h_msg ? true : false;
379 			if (num <= batch_log)
380 				dev_info(dev,
381 					 "Drained: fid %u size %u msg 0x%x(seq-%x) ctx 0x%x[fd-%d] args:[0x%x 0x%x] resp %s, bmap %d\n",
382 					 msg->function_id, msg->size,
383 					 msg_id, entry->seq_num,
384 					 msg->context_id, entry->ctx->idx,
385 					 msg->cmd, msg->arg,
386 					 responded ? "T" : "F", bit_set);
387 			if (responded)
388 				atomic_dec(&ctx->pend_cnt);
389 			else if (bit_set)
390 				bcm_vk_msgid_bitmap_clear(vk, msg_id, 1);
391 		}
392 		bcm_vk_free_wkent(dev, entry);
393 	}
394 	if (num && ctx)
395 		dev_info(dev, "Total drained items %d [fd-%d]\n",
396 			 num, ctx->idx);
397 }
398 
bcm_vk_drain_msg_on_reset(struct bcm_vk * vk)399 void bcm_vk_drain_msg_on_reset(struct bcm_vk *vk)
400 {
401 	bcm_vk_drain_all_pend(&vk->pdev->dev, &vk->to_v_msg_chan, NULL);
402 	bcm_vk_drain_all_pend(&vk->pdev->dev, &vk->to_h_msg_chan, NULL);
403 }
404 
405 /*
406  * Function to sync up the messages queue info that is provided by BAR1
407  */
bcm_vk_sync_msgq(struct bcm_vk * vk,bool force_sync)408 int bcm_vk_sync_msgq(struct bcm_vk *vk, bool force_sync)
409 {
410 	struct bcm_vk_msgq __iomem *msgq;
411 	struct device *dev = &vk->pdev->dev;
412 	u32 msgq_off;
413 	u32 num_q;
414 	struct bcm_vk_msg_chan *chan_list[] = {&vk->to_v_msg_chan,
415 					       &vk->to_h_msg_chan};
416 	struct bcm_vk_msg_chan *chan;
417 	int i, j;
418 	int ret = 0;
419 
420 	/*
421 	 * If the driver is loaded at startup where vk OS is not up yet,
422 	 * the msgq-info may not be available until a later time.  In
423 	 * this case, we skip and the sync function is supposed to be
424 	 * called again.
425 	 */
426 	if (!bcm_vk_msgq_marker_valid(vk)) {
427 		dev_info(dev, "BAR1 msgq marker not initialized.\n");
428 		return -EAGAIN;
429 	}
430 
431 	msgq_off = vkread32(vk, BAR_1, VK_BAR1_MSGQ_CTRL_OFF);
432 
433 	/* each side is always half the total  */
434 	num_q = vkread32(vk, BAR_1, VK_BAR1_MSGQ_NR) / 2;
435 	if (!num_q || (num_q > VK_MSGQ_PER_CHAN_MAX)) {
436 		dev_err(dev,
437 			"Advertised msgq %d error - max %d allowed\n",
438 			num_q, VK_MSGQ_PER_CHAN_MAX);
439 		return -EINVAL;
440 	}
441 
442 	vk->to_v_msg_chan.q_nr = num_q;
443 	vk->to_h_msg_chan.q_nr = num_q;
444 
445 	/* first msgq location */
446 	msgq = vk->bar[BAR_1] + msgq_off;
447 
448 	/*
449 	 * if this function is called when it is already inited,
450 	 * something is wrong
451 	 */
452 	if (bcm_vk_drv_access_ok(vk) && !force_sync) {
453 		dev_err(dev, "Msgq info already in sync\n");
454 		return -EPERM;
455 	}
456 
457 	for (i = 0; i < ARRAY_SIZE(chan_list); i++) {
458 		chan = chan_list[i];
459 		memset(chan->sync_qinfo, 0, sizeof(chan->sync_qinfo));
460 
461 		for (j = 0; j < num_q; j++) {
462 			struct bcm_vk_sync_qinfo *qinfo;
463 			u32 msgq_start;
464 			u32 msgq_size;
465 			u32 msgq_nxt;
466 			u32 msgq_db_offset, q_db_offset;
467 
468 			chan->msgq[j] = msgq;
469 			msgq_start = readl_relaxed(&msgq->start);
470 			msgq_size = readl_relaxed(&msgq->size);
471 			msgq_nxt = readl_relaxed(&msgq->nxt);
472 			msgq_db_offset = readl_relaxed(&msgq->db_offset);
473 			q_db_offset = (msgq_db_offset & ((1 << DB_SHIFT) - 1));
474 			if (q_db_offset  == (~msgq_db_offset >> DB_SHIFT))
475 				msgq_db_offset = q_db_offset;
476 			else
477 				/* fall back to default */
478 				msgq_db_offset = VK_BAR0_Q_DB_BASE(j);
479 
480 			dev_info(dev,
481 				 "MsgQ[%d] type %d num %d, @ 0x%x, db_offset 0x%x rd_idx %d wr_idx %d, size %d, nxt 0x%x\n",
482 				 j,
483 				 readw_relaxed(&msgq->type),
484 				 readw_relaxed(&msgq->num),
485 				 msgq_start,
486 				 msgq_db_offset,
487 				 readl_relaxed(&msgq->rd_idx),
488 				 readl_relaxed(&msgq->wr_idx),
489 				 msgq_size,
490 				 msgq_nxt);
491 
492 			qinfo = &chan->sync_qinfo[j];
493 			/* formulate and record static info */
494 			qinfo->q_start = vk->bar[BAR_1] + msgq_start;
495 			qinfo->q_size = msgq_size;
496 			/* set low threshold as 50% or 1/2 */
497 			qinfo->q_low = qinfo->q_size >> 1;
498 			qinfo->q_mask = qinfo->q_size - 1;
499 			qinfo->q_db_offset = msgq_db_offset;
500 
501 			msgq++;
502 		}
503 	}
504 	atomic_set(&vk->msgq_inited, 1);
505 
506 	return ret;
507 }
508 
bcm_vk_msg_chan_init(struct bcm_vk_msg_chan * chan)509 static int bcm_vk_msg_chan_init(struct bcm_vk_msg_chan *chan)
510 {
511 	u32 i;
512 
513 	mutex_init(&chan->msgq_mutex);
514 	spin_lock_init(&chan->pendq_lock);
515 	for (i = 0; i < VK_MSGQ_MAX_NR; i++)
516 		INIT_LIST_HEAD(&chan->pendq[i]);
517 
518 	return 0;
519 }
520 
bcm_vk_append_pendq(struct bcm_vk_msg_chan * chan,u16 q_num,struct bcm_vk_wkent * entry)521 static void bcm_vk_append_pendq(struct bcm_vk_msg_chan *chan, u16 q_num,
522 				struct bcm_vk_wkent *entry)
523 {
524 	struct bcm_vk_ctx *ctx;
525 
526 	spin_lock(&chan->pendq_lock);
527 	list_add_tail(&entry->node, &chan->pendq[q_num]);
528 	if (entry->to_h_msg) {
529 		ctx = entry->ctx;
530 		atomic_inc(&ctx->pend_cnt);
531 		wake_up_interruptible(&ctx->rd_wq);
532 	}
533 	spin_unlock(&chan->pendq_lock);
534 }
535 
bcm_vk_append_ib_sgl(struct bcm_vk * vk,struct bcm_vk_wkent * entry,struct _vk_data * data,unsigned int num_planes)536 static u32 bcm_vk_append_ib_sgl(struct bcm_vk *vk,
537 				struct bcm_vk_wkent *entry,
538 				struct _vk_data *data,
539 				unsigned int num_planes)
540 {
541 	unsigned int i;
542 	unsigned int item_cnt = 0;
543 	struct device *dev = &vk->pdev->dev;
544 	struct bcm_vk_msg_chan *chan = &vk->to_v_msg_chan;
545 	struct vk_msg_blk *msg = &entry->to_v_msg[0];
546 	struct bcm_vk_msgq __iomem *msgq;
547 	struct bcm_vk_sync_qinfo *qinfo;
548 	u32 ib_sgl_size = 0;
549 	u8 *buf = (u8 *)&entry->to_v_msg[entry->to_v_blks];
550 	u32 avail;
551 	u32 q_num;
552 
553 	/* check if high watermark is hit, and if so, skip */
554 	q_num = get_q_num(msg);
555 	msgq = chan->msgq[q_num];
556 	qinfo = &chan->sync_qinfo[q_num];
557 	avail = msgq_avail_space(msgq, qinfo);
558 	if (avail < qinfo->q_low) {
559 		dev_dbg(dev, "Skip inserting inband SGL, [0x%x/0x%x]\n",
560 			avail, qinfo->q_size);
561 		return 0;
562 	}
563 
564 	for (i = 0; i < num_planes; i++) {
565 		if (data[i].address &&
566 		    (ib_sgl_size + data[i].size) <= vk->ib_sgl_size) {
567 			item_cnt++;
568 			memcpy(buf, entry->dma[i].sglist, data[i].size);
569 			ib_sgl_size += data[i].size;
570 			buf += data[i].size;
571 		}
572 	}
573 
574 	dev_dbg(dev, "Num %u sgl items appended, size 0x%x, room 0x%x\n",
575 		item_cnt, ib_sgl_size, vk->ib_sgl_size);
576 
577 	/* round up size */
578 	ib_sgl_size = (ib_sgl_size + VK_MSGQ_BLK_SIZE - 1)
579 		       >> VK_MSGQ_BLK_SZ_SHIFT;
580 
581 	return ib_sgl_size;
582 }
583 
bcm_to_v_q_doorbell(struct bcm_vk * vk,u32 q_num,u32 db_val)584 void bcm_to_v_q_doorbell(struct bcm_vk *vk, u32 q_num, u32 db_val)
585 {
586 	struct bcm_vk_msg_chan *chan = &vk->to_v_msg_chan;
587 	struct bcm_vk_sync_qinfo *qinfo = &chan->sync_qinfo[q_num];
588 
589 	vkwrite32(vk, db_val, BAR_0, qinfo->q_db_offset);
590 }
591 
bcm_to_v_msg_enqueue(struct bcm_vk * vk,struct bcm_vk_wkent * entry)592 static int bcm_to_v_msg_enqueue(struct bcm_vk *vk, struct bcm_vk_wkent *entry)
593 {
594 	static u32 seq_num;
595 	struct bcm_vk_msg_chan *chan = &vk->to_v_msg_chan;
596 	struct device *dev = &vk->pdev->dev;
597 	struct vk_msg_blk *src = &entry->to_v_msg[0];
598 
599 	struct vk_msg_blk __iomem *dst;
600 	struct bcm_vk_msgq __iomem *msgq;
601 	struct bcm_vk_sync_qinfo *qinfo;
602 	u32 q_num = get_q_num(src);
603 	u32 wr_idx; /* local copy */
604 	u32 i;
605 	u32 avail;
606 	u32 retry;
607 
608 	if (entry->to_v_blks != src->size + 1) {
609 		dev_err(dev, "number of blks %d not matching %d MsgId[0x%x]: func %d ctx 0x%x\n",
610 			entry->to_v_blks,
611 			src->size + 1,
612 			get_msg_id(src),
613 			src->function_id,
614 			src->context_id);
615 		return -EMSGSIZE;
616 	}
617 
618 	msgq = chan->msgq[q_num];
619 	qinfo = &chan->sync_qinfo[q_num];
620 
621 	mutex_lock(&chan->msgq_mutex);
622 
623 	avail = msgq_avail_space(msgq, qinfo);
624 
625 	/* if not enough space, return EAGAIN and let app handles it */
626 	retry = 0;
627 	while ((avail < entry->to_v_blks) &&
628 	       (retry++ < BCM_VK_H2VK_ENQ_RETRY)) {
629 		mutex_unlock(&chan->msgq_mutex);
630 
631 		msleep(BCM_VK_H2VK_ENQ_RETRY_DELAY_MS);
632 		mutex_lock(&chan->msgq_mutex);
633 		avail = msgq_avail_space(msgq, qinfo);
634 	}
635 	if (retry > BCM_VK_H2VK_ENQ_RETRY) {
636 		mutex_unlock(&chan->msgq_mutex);
637 		return -EAGAIN;
638 	}
639 
640 	/* at this point, mutex is taken and there is enough space */
641 	entry->seq_num = seq_num++; /* update debug seq number */
642 	wr_idx = readl_relaxed(&msgq->wr_idx);
643 
644 	if (wr_idx >= qinfo->q_size) {
645 		dev_crit(dev, "Invalid wr_idx 0x%x => max 0x%x!",
646 			 wr_idx, qinfo->q_size);
647 		bcm_vk_blk_drv_access(vk);
648 		bcm_vk_set_host_alert(vk, ERR_LOG_HOST_PCIE_DWN);
649 		goto idx_err;
650 	}
651 
652 	dst = msgq_blk_addr(qinfo, wr_idx);
653 	for (i = 0; i < entry->to_v_blks; i++) {
654 		memcpy_toio(dst, src, sizeof(*dst));
655 
656 		src++;
657 		wr_idx = msgq_inc(qinfo, wr_idx, 1);
658 		dst = msgq_blk_addr(qinfo, wr_idx);
659 	}
660 
661 	/* flush the write pointer */
662 	writel(wr_idx, &msgq->wr_idx);
663 
664 	/* log new info for debugging */
665 	dev_dbg(dev,
666 		"MsgQ[%d] [Rd Wr] = [%d %d] blks inserted %d - Q = [u-%d a-%d]/%d\n",
667 		readl_relaxed(&msgq->num),
668 		readl_relaxed(&msgq->rd_idx),
669 		wr_idx,
670 		entry->to_v_blks,
671 		msgq_occupied(msgq, qinfo),
672 		msgq_avail_space(msgq, qinfo),
673 		readl_relaxed(&msgq->size));
674 	/*
675 	 * press door bell based on queue number. 1 is added to the wr_idx
676 	 * to avoid the value of 0 appearing on the VK side to distinguish
677 	 * from initial value.
678 	 */
679 	bcm_to_v_q_doorbell(vk, q_num, wr_idx + 1);
680 idx_err:
681 	mutex_unlock(&chan->msgq_mutex);
682 	return 0;
683 }
684 
bcm_vk_send_shutdown_msg(struct bcm_vk * vk,u32 shut_type,const pid_t pid,const u32 q_num)685 int bcm_vk_send_shutdown_msg(struct bcm_vk *vk, u32 shut_type,
686 			     const pid_t pid, const u32 q_num)
687 {
688 	int rc = 0;
689 	struct bcm_vk_wkent *entry;
690 	struct device *dev = &vk->pdev->dev;
691 
692 	/*
693 	 * check if the marker is still good.  Sometimes, the PCIe interface may
694 	 * have gone done, and if so and we ship down thing based on broken
695 	 * values, kernel may panic.
696 	 */
697 	if (!bcm_vk_msgq_marker_valid(vk)) {
698 		dev_info(dev, "PCIe comm chan - invalid marker (0x%x)!\n",
699 			 vkread32(vk, BAR_1, VK_BAR1_MSGQ_DEF_RDY));
700 		return -EINVAL;
701 	}
702 
703 	entry = kzalloc(struct_size(entry, to_v_msg, 1), GFP_KERNEL);
704 	if (!entry)
705 		return -ENOMEM;
706 	entry->to_v_blks = 1;	/* always 1 block */
707 
708 	/* fill up necessary data */
709 	entry->to_v_msg[0].function_id = VK_FID_SHUTDOWN;
710 	set_q_num(&entry->to_v_msg[0], q_num);
711 	set_msg_id(&entry->to_v_msg[0], VK_SIMPLEX_MSG_ID);
712 
713 	entry->to_v_msg[0].cmd = shut_type;
714 	entry->to_v_msg[0].arg = pid;
715 
716 	rc = bcm_to_v_msg_enqueue(vk, entry);
717 	if (rc)
718 		dev_err(dev,
719 			"Sending shutdown message to q %d for pid %d fails.\n",
720 			get_q_num(&entry->to_v_msg[0]), pid);
721 
722 	kfree(entry);
723 
724 	return rc;
725 }
726 
bcm_vk_handle_last_sess(struct bcm_vk * vk,const pid_t pid,const u32 q_num)727 static int bcm_vk_handle_last_sess(struct bcm_vk *vk, const pid_t pid,
728 				   const u32 q_num)
729 {
730 	int rc = 0;
731 	struct device *dev = &vk->pdev->dev;
732 
733 	/*
734 	 * don't send down or do anything if message queue is not initialized
735 	 * and if it is the reset session, clear it.
736 	 */
737 	if (!bcm_vk_drv_access_ok(vk)) {
738 		if (vk->reset_pid == pid)
739 			vk->reset_pid = 0;
740 		return -EPERM;
741 	}
742 
743 	dev_dbg(dev, "No more sessions, shut down pid %d\n", pid);
744 
745 	/* only need to do it if it is not the reset process */
746 	if (vk->reset_pid != pid)
747 		rc = bcm_vk_send_shutdown_msg(vk, VK_SHUTDOWN_PID, pid, q_num);
748 	else
749 		/* put reset_pid to 0 if it is exiting last session */
750 		vk->reset_pid = 0;
751 
752 	return rc;
753 }
754 
bcm_vk_dequeue_pending(struct bcm_vk * vk,struct bcm_vk_msg_chan * chan,u16 q_num,u16 msg_id)755 static struct bcm_vk_wkent *bcm_vk_dequeue_pending(struct bcm_vk *vk,
756 						   struct bcm_vk_msg_chan *chan,
757 						   u16 q_num,
758 						   u16 msg_id)
759 {
760 	struct bcm_vk_wkent *entry = NULL, *iter;
761 
762 	spin_lock(&chan->pendq_lock);
763 	list_for_each_entry(iter, &chan->pendq[q_num], node) {
764 		if (get_msg_id(&iter->to_v_msg[0]) == msg_id) {
765 			list_del(&iter->node);
766 			entry = iter;
767 			bcm_vk_msgid_bitmap_clear(vk, msg_id, 1);
768 			break;
769 		}
770 	}
771 	spin_unlock(&chan->pendq_lock);
772 	return entry;
773 }
774 
bcm_to_h_msg_dequeue(struct bcm_vk * vk)775 s32 bcm_to_h_msg_dequeue(struct bcm_vk *vk)
776 {
777 	struct device *dev = &vk->pdev->dev;
778 	struct bcm_vk_msg_chan *chan = &vk->to_h_msg_chan;
779 	struct vk_msg_blk *data;
780 	struct vk_msg_blk __iomem *src;
781 	struct vk_msg_blk *dst;
782 	struct bcm_vk_msgq __iomem *msgq;
783 	struct bcm_vk_sync_qinfo *qinfo;
784 	struct bcm_vk_wkent *entry;
785 	u32 rd_idx, wr_idx;
786 	u32 q_num, msg_id, j;
787 	u32 num_blks;
788 	s32 total = 0;
789 	int cnt = 0;
790 	int msg_processed = 0;
791 	int max_msg_to_process;
792 	bool exit_loop;
793 
794 	/*
795 	 * drain all the messages from the queues, and find its pending
796 	 * entry in the to_v queue, based on msg_id & q_num, and move the
797 	 * entry to the to_h pending queue, waiting for user space
798 	 * program to extract
799 	 */
800 	mutex_lock(&chan->msgq_mutex);
801 
802 	for (q_num = 0; q_num < chan->q_nr; q_num++) {
803 		msgq = chan->msgq[q_num];
804 		qinfo = &chan->sync_qinfo[q_num];
805 		max_msg_to_process = BCM_VK_MSG_PROC_MAX_LOOP * qinfo->q_size;
806 
807 		rd_idx = readl_relaxed(&msgq->rd_idx);
808 		wr_idx = readl_relaxed(&msgq->wr_idx);
809 		msg_processed = 0;
810 		exit_loop = false;
811 		while ((rd_idx != wr_idx) && !exit_loop) {
812 			u8 src_size;
813 
814 			/*
815 			 * Make a local copy and get pointer to src blk
816 			 * The rd_idx is masked before getting the pointer to
817 			 * avoid out of bound access in case the interface goes
818 			 * down.  It will end up pointing to the last block in
819 			 * the buffer, but subsequent src->size check would be
820 			 * able to catch this.
821 			 */
822 			src = msgq_blk_addr(qinfo, rd_idx & qinfo->q_mask);
823 			src_size = readb(&src->size);
824 
825 			if ((rd_idx >= qinfo->q_size) ||
826 			    (src_size > (qinfo->q_size - 1))) {
827 				dev_crit(dev,
828 					 "Invalid rd_idx 0x%x or size 0x%x => max 0x%x!",
829 					 rd_idx, src_size, qinfo->q_size);
830 				bcm_vk_blk_drv_access(vk);
831 				bcm_vk_set_host_alert(vk,
832 						      ERR_LOG_HOST_PCIE_DWN);
833 				goto idx_err;
834 			}
835 
836 			num_blks = src_size + 1;
837 			data = kzalloc(num_blks * VK_MSGQ_BLK_SIZE, GFP_KERNEL);
838 			if (data) {
839 				/* copy messages and linearize it */
840 				dst = data;
841 				for (j = 0; j < num_blks; j++) {
842 					memcpy_fromio(dst, src, sizeof(*dst));
843 
844 					dst++;
845 					rd_idx = msgq_inc(qinfo, rd_idx, 1);
846 					src = msgq_blk_addr(qinfo, rd_idx);
847 				}
848 				total++;
849 			} else {
850 				/*
851 				 * if we could not allocate memory in kernel,
852 				 * that is fatal.
853 				 */
854 				dev_crit(dev, "Kernel mem allocation failure.\n");
855 				total = -ENOMEM;
856 				goto idx_err;
857 			}
858 
859 			/* flush rd pointer after a message is dequeued */
860 			writel(rd_idx, &msgq->rd_idx);
861 
862 			/* log new info for debugging */
863 			dev_dbg(dev,
864 				"MsgQ[%d] [Rd Wr] = [%d %d] blks extracted %d - Q = [u-%d a-%d]/%d\n",
865 				readl_relaxed(&msgq->num),
866 				rd_idx,
867 				wr_idx,
868 				num_blks,
869 				msgq_occupied(msgq, qinfo),
870 				msgq_avail_space(msgq, qinfo),
871 				readl_relaxed(&msgq->size));
872 
873 			/*
874 			 * No need to search if it is an autonomous one-way
875 			 * message from driver, as these messages do not bear
876 			 * a to_v pending item. Currently, only the shutdown
877 			 * message falls into this category.
878 			 */
879 			if (data->function_id == VK_FID_SHUTDOWN) {
880 				kfree(data);
881 				continue;
882 			}
883 
884 			msg_id = get_msg_id(data);
885 			/* lookup original message in to_v direction */
886 			entry = bcm_vk_dequeue_pending(vk,
887 						       &vk->to_v_msg_chan,
888 						       q_num,
889 						       msg_id);
890 
891 			/*
892 			 * if there is message to does not have prior send,
893 			 * this is the location to add here
894 			 */
895 			if (entry) {
896 				entry->to_h_blks = num_blks;
897 				entry->to_h_msg = data;
898 				bcm_vk_append_pendq(&vk->to_h_msg_chan,
899 						    q_num, entry);
900 
901 			} else {
902 				if (cnt++ < batch_log)
903 					dev_info(dev,
904 						 "Could not find MsgId[0x%x] for resp func %d bmap %d\n",
905 						 msg_id, data->function_id,
906 						 test_bit(msg_id, vk->bmap));
907 				kfree(data);
908 			}
909 			/* Fetch wr_idx to handle more back-to-back events */
910 			wr_idx = readl(&msgq->wr_idx);
911 
912 			/*
913 			 * cap the max so that even we try to handle more back-to-back events,
914 			 * so that it won't hold CPU too long or in case rd/wr idexes are
915 			 * corrupted which triggers infinite looping.
916 			 */
917 			if (++msg_processed >= max_msg_to_process) {
918 				dev_warn(dev, "Q[%d] Per loop processing exceeds %d\n",
919 					 q_num, max_msg_to_process);
920 				exit_loop = true;
921 			}
922 		}
923 	}
924 idx_err:
925 	mutex_unlock(&chan->msgq_mutex);
926 	dev_dbg(dev, "total %d drained from queues\n", total);
927 
928 	return total;
929 }
930 
931 /*
932  * init routine for all required data structures
933  */
bcm_vk_data_init(struct bcm_vk * vk)934 static int bcm_vk_data_init(struct bcm_vk *vk)
935 {
936 	int i;
937 
938 	spin_lock_init(&vk->ctx_lock);
939 	for (i = 0; i < ARRAY_SIZE(vk->ctx); i++) {
940 		vk->ctx[i].in_use = false;
941 		vk->ctx[i].idx = i;	/* self identity */
942 		vk->ctx[i].miscdev = NULL;
943 	}
944 	spin_lock_init(&vk->msg_id_lock);
945 	spin_lock_init(&vk->host_alert_lock);
946 	vk->msg_id = 0;
947 
948 	/* initialize hash table */
949 	for (i = 0; i < VK_PID_HT_SZ; i++)
950 		INIT_LIST_HEAD(&vk->pid_ht[i].head);
951 
952 	return 0;
953 }
954 
bcm_vk_msgq_irqhandler(int irq,void * dev_id)955 irqreturn_t bcm_vk_msgq_irqhandler(int irq, void *dev_id)
956 {
957 	struct bcm_vk *vk = dev_id;
958 
959 	if (!bcm_vk_drv_access_ok(vk)) {
960 		dev_err(&vk->pdev->dev,
961 			"Interrupt %d received when msgq not inited\n", irq);
962 		goto skip_schedule_work;
963 	}
964 
965 	queue_work(vk->wq_thread, &vk->wq_work);
966 
967 skip_schedule_work:
968 	return IRQ_HANDLED;
969 }
970 
bcm_vk_open(struct inode * inode,struct file * p_file)971 int bcm_vk_open(struct inode *inode, struct file *p_file)
972 {
973 	struct bcm_vk_ctx *ctx;
974 	struct miscdevice *miscdev = (struct miscdevice *)p_file->private_data;
975 	struct bcm_vk *vk = container_of(miscdev, struct bcm_vk, miscdev);
976 	struct device *dev = &vk->pdev->dev;
977 	int rc = 0;
978 
979 	/* get a context and set it up for file */
980 	ctx = bcm_vk_get_ctx(vk, task_tgid_nr(current));
981 	if (!ctx) {
982 		dev_err(dev, "Error allocating context\n");
983 		rc = -ENOMEM;
984 	} else {
985 		/*
986 		 * set up context and replace private data with context for
987 		 * other methods to use.  Reason for the context is because
988 		 * it is allowed for multiple sessions to open the sysfs, and
989 		 * for each file open, when upper layer query the response,
990 		 * only those that are tied to a specific open should be
991 		 * returned.  The context->idx will be used for such binding
992 		 */
993 		ctx->miscdev = miscdev;
994 		p_file->private_data = ctx;
995 		dev_dbg(dev, "ctx_returned with idx %d, pid %d\n",
996 			ctx->idx, ctx->pid);
997 	}
998 	return rc;
999 }
1000 
bcm_vk_read(struct file * p_file,char __user * buf,size_t count,loff_t * f_pos)1001 ssize_t bcm_vk_read(struct file *p_file,
1002 		    char __user *buf,
1003 		    size_t count,
1004 		    loff_t *f_pos)
1005 {
1006 	ssize_t rc = -ENOMSG;
1007 	struct bcm_vk_ctx *ctx = p_file->private_data;
1008 	struct bcm_vk *vk = container_of(ctx->miscdev, struct bcm_vk,
1009 					 miscdev);
1010 	struct device *dev = &vk->pdev->dev;
1011 	struct bcm_vk_msg_chan *chan = &vk->to_h_msg_chan;
1012 	struct bcm_vk_wkent *entry = NULL, *iter;
1013 	u32 q_num;
1014 	u32 rsp_length;
1015 
1016 	if (!bcm_vk_drv_access_ok(vk))
1017 		return -EPERM;
1018 
1019 	dev_dbg(dev, "Buf count %zu\n", count);
1020 
1021 	/*
1022 	 * search through the pendq on the to_h chan, and return only those
1023 	 * that belongs to the same context.  Search is always from the high to
1024 	 * the low priority queues
1025 	 */
1026 	spin_lock(&chan->pendq_lock);
1027 	for (q_num = 0; q_num < chan->q_nr; q_num++) {
1028 		list_for_each_entry(iter, &chan->pendq[q_num], node) {
1029 			if (iter->ctx->idx == ctx->idx) {
1030 				if (count >=
1031 				    (iter->to_h_blks * VK_MSGQ_BLK_SIZE)) {
1032 					list_del(&iter->node);
1033 					atomic_dec(&ctx->pend_cnt);
1034 					entry = iter;
1035 				} else {
1036 					/* buffer not big enough */
1037 					rc = -EMSGSIZE;
1038 				}
1039 				goto read_loop_exit;
1040 			}
1041 		}
1042 	}
1043 read_loop_exit:
1044 	spin_unlock(&chan->pendq_lock);
1045 
1046 	if (entry) {
1047 		/* retrieve the passed down msg_id */
1048 		set_msg_id(&entry->to_h_msg[0], entry->usr_msg_id);
1049 		rsp_length = entry->to_h_blks * VK_MSGQ_BLK_SIZE;
1050 		if (copy_to_user(buf, entry->to_h_msg, rsp_length) == 0)
1051 			rc = rsp_length;
1052 
1053 		bcm_vk_free_wkent(dev, entry);
1054 	} else if (rc == -EMSGSIZE) {
1055 		struct vk_msg_blk tmp_msg = entry->to_h_msg[0];
1056 
1057 		/*
1058 		 * in this case, return just the first block, so
1059 		 * that app knows what size it is looking for.
1060 		 */
1061 		set_msg_id(&tmp_msg, entry->usr_msg_id);
1062 		tmp_msg.size = entry->to_h_blks - 1;
1063 		if (copy_to_user(buf, &tmp_msg, VK_MSGQ_BLK_SIZE) != 0) {
1064 			dev_err(dev, "Error return 1st block in -EMSGSIZE\n");
1065 			rc = -EFAULT;
1066 		}
1067 	}
1068 	return rc;
1069 }
1070 
bcm_vk_write(struct file * p_file,const char __user * buf,size_t count,loff_t * f_pos)1071 ssize_t bcm_vk_write(struct file *p_file,
1072 		     const char __user *buf,
1073 		     size_t count,
1074 		     loff_t *f_pos)
1075 {
1076 	ssize_t rc;
1077 	struct bcm_vk_ctx *ctx = p_file->private_data;
1078 	struct bcm_vk *vk = container_of(ctx->miscdev, struct bcm_vk,
1079 					 miscdev);
1080 	struct bcm_vk_msgq __iomem *msgq;
1081 	struct device *dev = &vk->pdev->dev;
1082 	struct bcm_vk_wkent *entry;
1083 	u32 sgl_extra_blks;
1084 	u32 q_num;
1085 	u32 msg_size;
1086 	u32 msgq_size;
1087 
1088 	if (!bcm_vk_drv_access_ok(vk))
1089 		return -EPERM;
1090 
1091 	dev_dbg(dev, "Msg count %zu\n", count);
1092 
1093 	/* first, do sanity check where count should be multiple of basic blk */
1094 	if (count & (VK_MSGQ_BLK_SIZE - 1)) {
1095 		dev_err(dev, "Failure with size %zu not multiple of %zu\n",
1096 			count, VK_MSGQ_BLK_SIZE);
1097 		rc = -EINVAL;
1098 		goto write_err;
1099 	}
1100 
1101 	/* allocate the work entry + buffer for size count and inband sgl */
1102 	entry = kzalloc(sizeof(*entry) + count + vk->ib_sgl_size,
1103 			GFP_KERNEL);
1104 	if (!entry) {
1105 		rc = -ENOMEM;
1106 		goto write_err;
1107 	}
1108 
1109 	/* now copy msg from user space, and then formulate the work entry */
1110 	if (copy_from_user(&entry->to_v_msg[0], buf, count)) {
1111 		rc = -EFAULT;
1112 		goto write_free_ent;
1113 	}
1114 
1115 	entry->to_v_blks = count >> VK_MSGQ_BLK_SZ_SHIFT;
1116 	entry->ctx = ctx;
1117 
1118 	/* do a check on the blk size which could not exceed queue space */
1119 	q_num = get_q_num(&entry->to_v_msg[0]);
1120 	msgq = vk->to_v_msg_chan.msgq[q_num];
1121 	msgq_size = readl_relaxed(&msgq->size);
1122 	if (entry->to_v_blks + (vk->ib_sgl_size >> VK_MSGQ_BLK_SZ_SHIFT)
1123 	    > (msgq_size - 1)) {
1124 		dev_err(dev, "Blk size %d exceed max queue size allowed %d\n",
1125 			entry->to_v_blks, msgq_size - 1);
1126 		rc = -EINVAL;
1127 		goto write_free_ent;
1128 	}
1129 
1130 	/* Use internal message id */
1131 	entry->usr_msg_id = get_msg_id(&entry->to_v_msg[0]);
1132 	rc = bcm_vk_get_msg_id(vk);
1133 	if (rc == VK_MSG_ID_OVERFLOW) {
1134 		dev_err(dev, "msg_id overflow\n");
1135 		rc = -EOVERFLOW;
1136 		goto write_free_ent;
1137 	}
1138 	set_msg_id(&entry->to_v_msg[0], rc);
1139 	ctx->q_num = q_num;
1140 
1141 	dev_dbg(dev,
1142 		"[Q-%d]Message ctx id %d, usr_msg_id 0x%x sent msg_id 0x%x\n",
1143 		ctx->q_num, ctx->idx, entry->usr_msg_id,
1144 		get_msg_id(&entry->to_v_msg[0]));
1145 
1146 	if (entry->to_v_msg[0].function_id == VK_FID_TRANS_BUF) {
1147 		/* Convert any pointers to sg list */
1148 		unsigned int num_planes;
1149 		int dir;
1150 		struct _vk_data *data;
1151 
1152 		/*
1153 		 * check if we are in reset, if so, no buffer transfer is
1154 		 * allowed and return error.
1155 		 */
1156 		if (vk->reset_pid) {
1157 			dev_dbg(dev, "No Transfer allowed during reset, pid %d.\n",
1158 				ctx->pid);
1159 			rc = -EACCES;
1160 			goto write_free_msgid;
1161 		}
1162 
1163 		num_planes = entry->to_v_msg[0].cmd & VK_CMD_PLANES_MASK;
1164 		if ((entry->to_v_msg[0].cmd & VK_CMD_MASK) == VK_CMD_DOWNLOAD)
1165 			dir = DMA_FROM_DEVICE;
1166 		else
1167 			dir = DMA_TO_DEVICE;
1168 
1169 		/* Calculate vk_data location */
1170 		/* Go to end of the message */
1171 		msg_size = entry->to_v_msg[0].size;
1172 		if (msg_size > entry->to_v_blks) {
1173 			rc = -EMSGSIZE;
1174 			goto write_free_msgid;
1175 		}
1176 
1177 		data = (struct _vk_data *)&entry->to_v_msg[msg_size + 1];
1178 
1179 		/* Now back up to the start of the pointers */
1180 		data -= num_planes;
1181 
1182 		/* Convert user addresses to DMA SG List */
1183 		rc = bcm_vk_sg_alloc(dev, entry->dma, dir, data, num_planes);
1184 		if (rc)
1185 			goto write_free_msgid;
1186 
1187 		atomic_inc(&ctx->dma_cnt);
1188 		/* try to embed inband sgl */
1189 		sgl_extra_blks = bcm_vk_append_ib_sgl(vk, entry, data,
1190 						      num_planes);
1191 		entry->to_v_blks += sgl_extra_blks;
1192 		entry->to_v_msg[0].size += sgl_extra_blks;
1193 	} else if (entry->to_v_msg[0].function_id == VK_FID_INIT &&
1194 		   entry->to_v_msg[0].context_id == VK_NEW_CTX) {
1195 		/*
1196 		 * Init happens in 2 stages, only the first stage contains the
1197 		 * pid that needs translating.
1198 		 */
1199 		pid_t org_pid, pid;
1200 
1201 		/*
1202 		 * translate the pid into the unique host space as user
1203 		 * may run sessions inside containers or process
1204 		 * namespaces.
1205 		 */
1206 #define VK_MSG_PID_MASK 0xffffff00
1207 #define VK_MSG_PID_SH   8
1208 		org_pid = (entry->to_v_msg[0].arg & VK_MSG_PID_MASK)
1209 			   >> VK_MSG_PID_SH;
1210 
1211 		pid = task_tgid_nr(current);
1212 		entry->to_v_msg[0].arg =
1213 			(entry->to_v_msg[0].arg & ~VK_MSG_PID_MASK) |
1214 			(pid << VK_MSG_PID_SH);
1215 		if (org_pid != pid)
1216 			dev_dbg(dev, "In PID 0x%x(%d), converted PID 0x%x(%d)\n",
1217 				org_pid, org_pid, pid, pid);
1218 	}
1219 
1220 	/*
1221 	 * store work entry to pending queue until a response is received.
1222 	 * This needs to be done before enqueuing the message
1223 	 */
1224 	bcm_vk_append_pendq(&vk->to_v_msg_chan, q_num, entry);
1225 
1226 	rc = bcm_to_v_msg_enqueue(vk, entry);
1227 	if (rc) {
1228 		dev_err(dev, "Fail to enqueue msg to to_v queue\n");
1229 
1230 		/* remove message from pending list */
1231 		entry = bcm_vk_dequeue_pending
1232 			       (vk,
1233 				&vk->to_v_msg_chan,
1234 				q_num,
1235 				get_msg_id(&entry->to_v_msg[0]));
1236 		goto write_free_ent;
1237 	}
1238 
1239 	return count;
1240 
1241 write_free_msgid:
1242 	bcm_vk_msgid_bitmap_clear(vk, get_msg_id(&entry->to_v_msg[0]), 1);
1243 write_free_ent:
1244 	kfree(entry);
1245 write_err:
1246 	return rc;
1247 }
1248 
bcm_vk_poll(struct file * p_file,struct poll_table_struct * wait)1249 __poll_t bcm_vk_poll(struct file *p_file, struct poll_table_struct *wait)
1250 {
1251 	__poll_t ret = 0;
1252 	int cnt;
1253 	struct bcm_vk_ctx *ctx = p_file->private_data;
1254 	struct bcm_vk *vk = container_of(ctx->miscdev, struct bcm_vk, miscdev);
1255 	struct device *dev = &vk->pdev->dev;
1256 
1257 	poll_wait(p_file, &ctx->rd_wq, wait);
1258 
1259 	cnt = atomic_read(&ctx->pend_cnt);
1260 	if (cnt) {
1261 		ret = (__force __poll_t)(POLLIN | POLLRDNORM);
1262 		if (cnt < 0) {
1263 			dev_err(dev, "Error cnt %d, setting back to 0", cnt);
1264 			atomic_set(&ctx->pend_cnt, 0);
1265 		}
1266 	}
1267 
1268 	return ret;
1269 }
1270 
bcm_vk_release(struct inode * inode,struct file * p_file)1271 int bcm_vk_release(struct inode *inode, struct file *p_file)
1272 {
1273 	int ret;
1274 	struct bcm_vk_ctx *ctx = p_file->private_data;
1275 	struct bcm_vk *vk = container_of(ctx->miscdev, struct bcm_vk, miscdev);
1276 	struct device *dev = &vk->pdev->dev;
1277 	pid_t pid = ctx->pid;
1278 	int dma_cnt;
1279 	unsigned long timeout, start_time;
1280 
1281 	/*
1282 	 * if there are outstanding DMA transactions, need to delay long enough
1283 	 * to ensure that the card side would have stopped touching the host buffer
1284 	 * and its SGL list.  A race condition could happen if the host app is killed
1285 	 * abruptly, eg kill -9, while some DMA transfer orders are still inflight.
1286 	 * Nothing could be done except for a delay as host side is running in a
1287 	 * completely async fashion.
1288 	 */
1289 	start_time = jiffies;
1290 	timeout = start_time + msecs_to_jiffies(BCM_VK_DMA_DRAIN_MAX_MS);
1291 	do {
1292 		if (time_after(jiffies, timeout)) {
1293 			dev_warn(dev, "%d dma still pending for [fd-%d] pid %d\n",
1294 				 dma_cnt, ctx->idx, pid);
1295 			break;
1296 		}
1297 		dma_cnt = atomic_read(&ctx->dma_cnt);
1298 		cpu_relax();
1299 		cond_resched();
1300 	} while (dma_cnt);
1301 	dev_dbg(dev, "Draining for [fd-%d] pid %d - delay %d ms\n",
1302 		ctx->idx, pid, jiffies_to_msecs(jiffies - start_time));
1303 
1304 	bcm_vk_drain_all_pend(&vk->pdev->dev, &vk->to_v_msg_chan, ctx);
1305 	bcm_vk_drain_all_pend(&vk->pdev->dev, &vk->to_h_msg_chan, ctx);
1306 
1307 	ret = bcm_vk_free_ctx(vk, ctx);
1308 	if (ret == 0)
1309 		ret = bcm_vk_handle_last_sess(vk, pid, ctx->q_num);
1310 	else
1311 		ret = 0;
1312 
1313 	kref_put(&vk->kref, bcm_vk_release_data);
1314 
1315 	return ret;
1316 }
1317 
bcm_vk_msg_init(struct bcm_vk * vk)1318 int bcm_vk_msg_init(struct bcm_vk *vk)
1319 {
1320 	struct device *dev = &vk->pdev->dev;
1321 	int ret;
1322 
1323 	if (bcm_vk_data_init(vk)) {
1324 		dev_err(dev, "Error initializing internal data structures\n");
1325 		return -EINVAL;
1326 	}
1327 
1328 	if (bcm_vk_msg_chan_init(&vk->to_v_msg_chan) ||
1329 	    bcm_vk_msg_chan_init(&vk->to_h_msg_chan)) {
1330 		dev_err(dev, "Error initializing communication channel\n");
1331 		return -EIO;
1332 	}
1333 
1334 	/* read msgq info if ready */
1335 	ret = bcm_vk_sync_msgq(vk, false);
1336 	if (ret && (ret != -EAGAIN)) {
1337 		dev_err(dev, "Error reading comm msg Q info\n");
1338 		return -EIO;
1339 	}
1340 
1341 	return 0;
1342 }
1343 
bcm_vk_msg_remove(struct bcm_vk * vk)1344 void bcm_vk_msg_remove(struct bcm_vk *vk)
1345 {
1346 	bcm_vk_blk_drv_access(vk);
1347 
1348 	/* drain all pending items */
1349 	bcm_vk_drain_all_pend(&vk->pdev->dev, &vk->to_v_msg_chan, NULL);
1350 	bcm_vk_drain_all_pend(&vk->pdev->dev, &vk->to_h_msg_chan, NULL);
1351 }
1352 
1353