xref: /linux/drivers/misc/fastrpc.c (revision 3a39d672e7f48b8d6b91a09afa4b55352773b4b5)
1 // SPDX-License-Identifier: GPL-2.0
2 // Copyright (c) 2011-2018, The Linux Foundation. All rights reserved.
3 // Copyright (c) 2018, Linaro Limited
4 
5 #include <linux/completion.h>
6 #include <linux/device.h>
7 #include <linux/dma-buf.h>
8 #include <linux/dma-mapping.h>
9 #include <linux/dma-resv.h>
10 #include <linux/idr.h>
11 #include <linux/list.h>
12 #include <linux/miscdevice.h>
13 #include <linux/module.h>
14 #include <linux/of_address.h>
15 #include <linux/of.h>
16 #include <linux/platform_device.h>
17 #include <linux/sort.h>
18 #include <linux/of_platform.h>
19 #include <linux/rpmsg.h>
20 #include <linux/scatterlist.h>
21 #include <linux/slab.h>
22 #include <linux/firmware/qcom/qcom_scm.h>
23 #include <uapi/misc/fastrpc.h>
24 #include <linux/of_reserved_mem.h>
25 
26 #define ADSP_DOMAIN_ID (0)
27 #define MDSP_DOMAIN_ID (1)
28 #define SDSP_DOMAIN_ID (2)
29 #define CDSP_DOMAIN_ID (3)
30 #define CDSP1_DOMAIN_ID (4)
31 #define FASTRPC_DEV_MAX		5 /* adsp, mdsp, slpi, cdsp, cdsp1 */
32 #define FASTRPC_MAX_SESSIONS	14
33 #define FASTRPC_MAX_VMIDS	16
34 #define FASTRPC_ALIGN		128
35 #define FASTRPC_MAX_FDLIST	16
36 #define FASTRPC_MAX_CRCLIST	64
37 #define FASTRPC_PHYS(p)	((p) & 0xffffffff)
38 #define FASTRPC_CTX_MAX (256)
39 #define FASTRPC_INIT_HANDLE	1
40 #define FASTRPC_DSP_UTILITIES_HANDLE	2
41 #define FASTRPC_CTXID_MASK (0xFF0)
42 #define INIT_FILELEN_MAX (2 * 1024 * 1024)
43 #define INIT_FILE_NAMELEN_MAX (128)
44 #define FASTRPC_DEVICE_NAME	"fastrpc"
45 
46 /* Add memory to static PD pool, protection thru XPU */
47 #define ADSP_MMAP_HEAP_ADDR  4
48 /* MAP static DMA buffer on DSP User PD */
49 #define ADSP_MMAP_DMA_BUFFER  6
50 /* Add memory to static PD pool protection thru hypervisor */
51 #define ADSP_MMAP_REMOTE_HEAP_ADDR  8
52 /* Add memory to userPD pool, for user heap */
53 #define ADSP_MMAP_ADD_PAGES 0x1000
54 /* Add memory to userPD pool, for LLC heap */
55 #define ADSP_MMAP_ADD_PAGES_LLC 0x3000,
56 
57 #define DSP_UNSUPPORTED_API (0x80000414)
58 /* MAX NUMBER of DSP ATTRIBUTES SUPPORTED */
59 #define FASTRPC_MAX_DSP_ATTRIBUTES (256)
60 #define FASTRPC_MAX_DSP_ATTRIBUTES_LEN (sizeof(u32) * FASTRPC_MAX_DSP_ATTRIBUTES)
61 
62 /* Retrives number of input buffers from the scalars parameter */
63 #define REMOTE_SCALARS_INBUFS(sc)	(((sc) >> 16) & 0x0ff)
64 
65 /* Retrives number of output buffers from the scalars parameter */
66 #define REMOTE_SCALARS_OUTBUFS(sc)	(((sc) >> 8) & 0x0ff)
67 
68 /* Retrives number of input handles from the scalars parameter */
69 #define REMOTE_SCALARS_INHANDLES(sc)	(((sc) >> 4) & 0x0f)
70 
71 /* Retrives number of output handles from the scalars parameter */
72 #define REMOTE_SCALARS_OUTHANDLES(sc)	((sc) & 0x0f)
73 
74 #define REMOTE_SCALARS_LENGTH(sc)	(REMOTE_SCALARS_INBUFS(sc) +   \
75 					 REMOTE_SCALARS_OUTBUFS(sc) +  \
76 					 REMOTE_SCALARS_INHANDLES(sc)+ \
77 					 REMOTE_SCALARS_OUTHANDLES(sc))
78 #define FASTRPC_BUILD_SCALARS(attr, method, in, out, oin, oout)  \
79 				(((attr & 0x07) << 29) |		\
80 				((method & 0x1f) << 24) |	\
81 				((in & 0xff) << 16) |		\
82 				((out & 0xff) <<  8) |		\
83 				((oin & 0x0f) <<  4) |		\
84 				(oout & 0x0f))
85 
86 #define FASTRPC_SCALARS(method, in, out) \
87 		FASTRPC_BUILD_SCALARS(0, method, in, out, 0, 0)
88 
89 #define FASTRPC_CREATE_PROCESS_NARGS	6
90 #define FASTRPC_CREATE_STATIC_PROCESS_NARGS	3
91 /* Remote Method id table */
92 #define FASTRPC_RMID_INIT_ATTACH	0
93 #define FASTRPC_RMID_INIT_RELEASE	1
94 #define FASTRPC_RMID_INIT_MMAP		4
95 #define FASTRPC_RMID_INIT_MUNMAP	5
96 #define FASTRPC_RMID_INIT_CREATE	6
97 #define FASTRPC_RMID_INIT_CREATE_ATTR	7
98 #define FASTRPC_RMID_INIT_CREATE_STATIC	8
99 #define FASTRPC_RMID_INIT_MEM_MAP      10
100 #define FASTRPC_RMID_INIT_MEM_UNMAP    11
101 
102 /* Protection Domain(PD) ids */
103 #define ROOT_PD		(0)
104 #define USER_PD		(1)
105 #define SENSORS_PD	(2)
106 
107 #define miscdev_to_fdevice(d) container_of(d, struct fastrpc_device, miscdev)
108 
109 static const char *domains[FASTRPC_DEV_MAX] = { "adsp", "mdsp",
110 						"sdsp", "cdsp", "cdsp1" };
111 struct fastrpc_phy_page {
112 	u64 addr;		/* physical address */
113 	u64 size;		/* size of contiguous region */
114 };
115 
116 struct fastrpc_invoke_buf {
117 	u32 num;		/* number of contiguous regions */
118 	u32 pgidx;		/* index to start of contiguous region */
119 };
120 
121 struct fastrpc_remote_dmahandle {
122 	s32 fd;		/* dma handle fd */
123 	u32 offset;	/* dma handle offset */
124 	u32 len;	/* dma handle length */
125 };
126 
127 struct fastrpc_remote_buf {
128 	u64 pv;		/* buffer pointer */
129 	u64 len;	/* length of buffer */
130 };
131 
132 union fastrpc_remote_arg {
133 	struct fastrpc_remote_buf buf;
134 	struct fastrpc_remote_dmahandle dma;
135 };
136 
137 struct fastrpc_mmap_rsp_msg {
138 	u64 vaddr;
139 };
140 
141 struct fastrpc_mmap_req_msg {
142 	s32 pgid;
143 	u32 flags;
144 	u64 vaddr;
145 	s32 num;
146 };
147 
148 struct fastrpc_mem_map_req_msg {
149 	s32 pgid;
150 	s32 fd;
151 	s32 offset;
152 	u32 flags;
153 	u64 vaddrin;
154 	s32 num;
155 	s32 data_len;
156 };
157 
158 struct fastrpc_munmap_req_msg {
159 	s32 pgid;
160 	u64 vaddr;
161 	u64 size;
162 };
163 
164 struct fastrpc_mem_unmap_req_msg {
165 	s32 pgid;
166 	s32 fd;
167 	u64 vaddrin;
168 	u64 len;
169 };
170 
171 struct fastrpc_msg {
172 	int pid;		/* process group id */
173 	int tid;		/* thread id */
174 	u64 ctx;		/* invoke caller context */
175 	u32 handle;	/* handle to invoke */
176 	u32 sc;		/* scalars structure describing the data */
177 	u64 addr;		/* physical address */
178 	u64 size;		/* size of contiguous region */
179 };
180 
181 struct fastrpc_invoke_rsp {
182 	u64 ctx;		/* invoke caller context */
183 	int retval;		/* invoke return value */
184 };
185 
186 struct fastrpc_buf_overlap {
187 	u64 start;
188 	u64 end;
189 	int raix;
190 	u64 mstart;
191 	u64 mend;
192 	u64 offset;
193 };
194 
195 struct fastrpc_buf {
196 	struct fastrpc_user *fl;
197 	struct dma_buf *dmabuf;
198 	struct device *dev;
199 	void *virt;
200 	u64 phys;
201 	u64 size;
202 	/* Lock for dma buf attachments */
203 	struct mutex lock;
204 	struct list_head attachments;
205 	/* mmap support */
206 	struct list_head node; /* list of user requested mmaps */
207 	uintptr_t raddr;
208 };
209 
210 struct fastrpc_dma_buf_attachment {
211 	struct device *dev;
212 	struct sg_table sgt;
213 	struct list_head node;
214 };
215 
216 struct fastrpc_map {
217 	struct list_head node;
218 	struct fastrpc_user *fl;
219 	int fd;
220 	struct dma_buf *buf;
221 	struct sg_table *table;
222 	struct dma_buf_attachment *attach;
223 	u64 phys;
224 	u64 size;
225 	void *va;
226 	u64 len;
227 	u64 raddr;
228 	u32 attr;
229 	struct kref refcount;
230 };
231 
232 struct fastrpc_invoke_ctx {
233 	int nscalars;
234 	int nbufs;
235 	int retval;
236 	int pid;
237 	int tgid;
238 	u32 sc;
239 	u32 *crc;
240 	u64 ctxid;
241 	u64 msg_sz;
242 	struct kref refcount;
243 	struct list_head node; /* list of ctxs */
244 	struct completion work;
245 	struct work_struct put_work;
246 	struct fastrpc_msg msg;
247 	struct fastrpc_user *fl;
248 	union fastrpc_remote_arg *rpra;
249 	struct fastrpc_map **maps;
250 	struct fastrpc_buf *buf;
251 	struct fastrpc_invoke_args *args;
252 	struct fastrpc_buf_overlap *olaps;
253 	struct fastrpc_channel_ctx *cctx;
254 };
255 
256 struct fastrpc_session_ctx {
257 	struct device *dev;
258 	int sid;
259 	bool used;
260 	bool valid;
261 };
262 
263 struct fastrpc_channel_ctx {
264 	int domain_id;
265 	int sesscount;
266 	int vmcount;
267 	struct qcom_scm_vmperm vmperms[FASTRPC_MAX_VMIDS];
268 	struct rpmsg_device *rpdev;
269 	struct fastrpc_session_ctx session[FASTRPC_MAX_SESSIONS];
270 	spinlock_t lock;
271 	struct idr ctx_idr;
272 	struct list_head users;
273 	struct kref refcount;
274 	/* Flag if dsp attributes are cached */
275 	bool valid_attributes;
276 	u32 dsp_attributes[FASTRPC_MAX_DSP_ATTRIBUTES];
277 	struct fastrpc_device *secure_fdevice;
278 	struct fastrpc_device *fdevice;
279 	struct fastrpc_buf *remote_heap;
280 	struct list_head invoke_interrupted_mmaps;
281 	bool secure;
282 	bool unsigned_support;
283 	u64 dma_mask;
284 };
285 
286 struct fastrpc_device {
287 	struct fastrpc_channel_ctx *cctx;
288 	struct miscdevice miscdev;
289 	bool secure;
290 };
291 
292 struct fastrpc_user {
293 	struct list_head user;
294 	struct list_head maps;
295 	struct list_head pending;
296 	struct list_head mmaps;
297 
298 	struct fastrpc_channel_ctx *cctx;
299 	struct fastrpc_session_ctx *sctx;
300 	struct fastrpc_buf *init_mem;
301 
302 	int tgid;
303 	int pd;
304 	bool is_secure_dev;
305 	/* Lock for lists */
306 	spinlock_t lock;
307 	/* lock for allocations */
308 	struct mutex mutex;
309 };
310 
fastrpc_free_map(struct kref * ref)311 static void fastrpc_free_map(struct kref *ref)
312 {
313 	struct fastrpc_map *map;
314 
315 	map = container_of(ref, struct fastrpc_map, refcount);
316 
317 	if (map->table) {
318 		if (map->attr & FASTRPC_ATTR_SECUREMAP) {
319 			struct qcom_scm_vmperm perm;
320 			int vmid = map->fl->cctx->vmperms[0].vmid;
321 			u64 src_perms = BIT(QCOM_SCM_VMID_HLOS) | BIT(vmid);
322 			int err = 0;
323 
324 			perm.vmid = QCOM_SCM_VMID_HLOS;
325 			perm.perm = QCOM_SCM_PERM_RWX;
326 			err = qcom_scm_assign_mem(map->phys, map->size,
327 				&src_perms, &perm, 1);
328 			if (err) {
329 				dev_err(map->fl->sctx->dev, "Failed to assign memory phys 0x%llx size 0x%llx err %d\n",
330 						map->phys, map->size, err);
331 				return;
332 			}
333 		}
334 		dma_buf_unmap_attachment_unlocked(map->attach, map->table,
335 						  DMA_BIDIRECTIONAL);
336 		dma_buf_detach(map->buf, map->attach);
337 		dma_buf_put(map->buf);
338 	}
339 
340 	if (map->fl) {
341 		spin_lock(&map->fl->lock);
342 		list_del(&map->node);
343 		spin_unlock(&map->fl->lock);
344 		map->fl = NULL;
345 	}
346 
347 	kfree(map);
348 }
349 
fastrpc_map_put(struct fastrpc_map * map)350 static void fastrpc_map_put(struct fastrpc_map *map)
351 {
352 	if (map)
353 		kref_put(&map->refcount, fastrpc_free_map);
354 }
355 
fastrpc_map_get(struct fastrpc_map * map)356 static int fastrpc_map_get(struct fastrpc_map *map)
357 {
358 	if (!map)
359 		return -ENOENT;
360 
361 	return kref_get_unless_zero(&map->refcount) ? 0 : -ENOENT;
362 }
363 
364 
fastrpc_map_lookup(struct fastrpc_user * fl,int fd,struct fastrpc_map ** ppmap,bool take_ref)365 static int fastrpc_map_lookup(struct fastrpc_user *fl, int fd,
366 			    struct fastrpc_map **ppmap, bool take_ref)
367 {
368 	struct fastrpc_session_ctx *sess = fl->sctx;
369 	struct fastrpc_map *map = NULL;
370 	int ret = -ENOENT;
371 
372 	spin_lock(&fl->lock);
373 	list_for_each_entry(map, &fl->maps, node) {
374 		if (map->fd != fd)
375 			continue;
376 
377 		if (take_ref) {
378 			ret = fastrpc_map_get(map);
379 			if (ret) {
380 				dev_dbg(sess->dev, "%s: Failed to get map fd=%d ret=%d\n",
381 					__func__, fd, ret);
382 				break;
383 			}
384 		}
385 
386 		*ppmap = map;
387 		ret = 0;
388 		break;
389 	}
390 	spin_unlock(&fl->lock);
391 
392 	return ret;
393 }
394 
fastrpc_buf_free(struct fastrpc_buf * buf)395 static void fastrpc_buf_free(struct fastrpc_buf *buf)
396 {
397 	dma_free_coherent(buf->dev, buf->size, buf->virt,
398 			  FASTRPC_PHYS(buf->phys));
399 	kfree(buf);
400 }
401 
__fastrpc_buf_alloc(struct fastrpc_user * fl,struct device * dev,u64 size,struct fastrpc_buf ** obuf)402 static int __fastrpc_buf_alloc(struct fastrpc_user *fl, struct device *dev,
403 			     u64 size, struct fastrpc_buf **obuf)
404 {
405 	struct fastrpc_buf *buf;
406 
407 	buf = kzalloc(sizeof(*buf), GFP_KERNEL);
408 	if (!buf)
409 		return -ENOMEM;
410 
411 	INIT_LIST_HEAD(&buf->attachments);
412 	INIT_LIST_HEAD(&buf->node);
413 	mutex_init(&buf->lock);
414 
415 	buf->fl = fl;
416 	buf->virt = NULL;
417 	buf->phys = 0;
418 	buf->size = size;
419 	buf->dev = dev;
420 	buf->raddr = 0;
421 
422 	buf->virt = dma_alloc_coherent(dev, buf->size, (dma_addr_t *)&buf->phys,
423 				       GFP_KERNEL);
424 	if (!buf->virt) {
425 		mutex_destroy(&buf->lock);
426 		kfree(buf);
427 		return -ENOMEM;
428 	}
429 
430 	*obuf = buf;
431 
432 	return 0;
433 }
434 
fastrpc_buf_alloc(struct fastrpc_user * fl,struct device * dev,u64 size,struct fastrpc_buf ** obuf)435 static int fastrpc_buf_alloc(struct fastrpc_user *fl, struct device *dev,
436 			     u64 size, struct fastrpc_buf **obuf)
437 {
438 	int ret;
439 	struct fastrpc_buf *buf;
440 
441 	ret = __fastrpc_buf_alloc(fl, dev, size, obuf);
442 	if (ret)
443 		return ret;
444 
445 	buf = *obuf;
446 
447 	if (fl->sctx && fl->sctx->sid)
448 		buf->phys += ((u64)fl->sctx->sid << 32);
449 
450 	return 0;
451 }
452 
fastrpc_remote_heap_alloc(struct fastrpc_user * fl,struct device * dev,u64 size,struct fastrpc_buf ** obuf)453 static int fastrpc_remote_heap_alloc(struct fastrpc_user *fl, struct device *dev,
454 				     u64 size, struct fastrpc_buf **obuf)
455 {
456 	struct device *rdev = &fl->cctx->rpdev->dev;
457 
458 	return  __fastrpc_buf_alloc(fl, rdev, size, obuf);
459 }
460 
fastrpc_channel_ctx_free(struct kref * ref)461 static void fastrpc_channel_ctx_free(struct kref *ref)
462 {
463 	struct fastrpc_channel_ctx *cctx;
464 
465 	cctx = container_of(ref, struct fastrpc_channel_ctx, refcount);
466 
467 	kfree(cctx);
468 }
469 
fastrpc_channel_ctx_get(struct fastrpc_channel_ctx * cctx)470 static void fastrpc_channel_ctx_get(struct fastrpc_channel_ctx *cctx)
471 {
472 	kref_get(&cctx->refcount);
473 }
474 
fastrpc_channel_ctx_put(struct fastrpc_channel_ctx * cctx)475 static void fastrpc_channel_ctx_put(struct fastrpc_channel_ctx *cctx)
476 {
477 	kref_put(&cctx->refcount, fastrpc_channel_ctx_free);
478 }
479 
fastrpc_context_free(struct kref * ref)480 static void fastrpc_context_free(struct kref *ref)
481 {
482 	struct fastrpc_invoke_ctx *ctx;
483 	struct fastrpc_channel_ctx *cctx;
484 	unsigned long flags;
485 	int i;
486 
487 	ctx = container_of(ref, struct fastrpc_invoke_ctx, refcount);
488 	cctx = ctx->cctx;
489 
490 	for (i = 0; i < ctx->nbufs; i++)
491 		fastrpc_map_put(ctx->maps[i]);
492 
493 	if (ctx->buf)
494 		fastrpc_buf_free(ctx->buf);
495 
496 	spin_lock_irqsave(&cctx->lock, flags);
497 	idr_remove(&cctx->ctx_idr, ctx->ctxid >> 4);
498 	spin_unlock_irqrestore(&cctx->lock, flags);
499 
500 	kfree(ctx->maps);
501 	kfree(ctx->olaps);
502 	kfree(ctx);
503 
504 	fastrpc_channel_ctx_put(cctx);
505 }
506 
fastrpc_context_get(struct fastrpc_invoke_ctx * ctx)507 static void fastrpc_context_get(struct fastrpc_invoke_ctx *ctx)
508 {
509 	kref_get(&ctx->refcount);
510 }
511 
fastrpc_context_put(struct fastrpc_invoke_ctx * ctx)512 static void fastrpc_context_put(struct fastrpc_invoke_ctx *ctx)
513 {
514 	kref_put(&ctx->refcount, fastrpc_context_free);
515 }
516 
fastrpc_context_put_wq(struct work_struct * work)517 static void fastrpc_context_put_wq(struct work_struct *work)
518 {
519 	struct fastrpc_invoke_ctx *ctx =
520 			container_of(work, struct fastrpc_invoke_ctx, put_work);
521 
522 	fastrpc_context_put(ctx);
523 }
524 
525 #define CMP(aa, bb) ((aa) == (bb) ? 0 : (aa) < (bb) ? -1 : 1)
olaps_cmp(const void * a,const void * b)526 static int olaps_cmp(const void *a, const void *b)
527 {
528 	struct fastrpc_buf_overlap *pa = (struct fastrpc_buf_overlap *)a;
529 	struct fastrpc_buf_overlap *pb = (struct fastrpc_buf_overlap *)b;
530 	/* sort with lowest starting buffer first */
531 	int st = CMP(pa->start, pb->start);
532 	/* sort with highest ending buffer first */
533 	int ed = CMP(pb->end, pa->end);
534 
535 	return st == 0 ? ed : st;
536 }
537 
fastrpc_get_buff_overlaps(struct fastrpc_invoke_ctx * ctx)538 static void fastrpc_get_buff_overlaps(struct fastrpc_invoke_ctx *ctx)
539 {
540 	u64 max_end = 0;
541 	int i;
542 
543 	for (i = 0; i < ctx->nbufs; ++i) {
544 		ctx->olaps[i].start = ctx->args[i].ptr;
545 		ctx->olaps[i].end = ctx->olaps[i].start + ctx->args[i].length;
546 		ctx->olaps[i].raix = i;
547 	}
548 
549 	sort(ctx->olaps, ctx->nbufs, sizeof(*ctx->olaps), olaps_cmp, NULL);
550 
551 	for (i = 0; i < ctx->nbufs; ++i) {
552 		/* Falling inside previous range */
553 		if (ctx->olaps[i].start < max_end) {
554 			ctx->olaps[i].mstart = max_end;
555 			ctx->olaps[i].mend = ctx->olaps[i].end;
556 			ctx->olaps[i].offset = max_end - ctx->olaps[i].start;
557 
558 			if (ctx->olaps[i].end > max_end) {
559 				max_end = ctx->olaps[i].end;
560 			} else {
561 				ctx->olaps[i].mend = 0;
562 				ctx->olaps[i].mstart = 0;
563 			}
564 
565 		} else  {
566 			ctx->olaps[i].mend = ctx->olaps[i].end;
567 			ctx->olaps[i].mstart = ctx->olaps[i].start;
568 			ctx->olaps[i].offset = 0;
569 			max_end = ctx->olaps[i].end;
570 		}
571 	}
572 }
573 
fastrpc_context_alloc(struct fastrpc_user * user,u32 kernel,u32 sc,struct fastrpc_invoke_args * args)574 static struct fastrpc_invoke_ctx *fastrpc_context_alloc(
575 			struct fastrpc_user *user, u32 kernel, u32 sc,
576 			struct fastrpc_invoke_args *args)
577 {
578 	struct fastrpc_channel_ctx *cctx = user->cctx;
579 	struct fastrpc_invoke_ctx *ctx = NULL;
580 	unsigned long flags;
581 	int ret;
582 
583 	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
584 	if (!ctx)
585 		return ERR_PTR(-ENOMEM);
586 
587 	INIT_LIST_HEAD(&ctx->node);
588 	ctx->fl = user;
589 	ctx->nscalars = REMOTE_SCALARS_LENGTH(sc);
590 	ctx->nbufs = REMOTE_SCALARS_INBUFS(sc) +
591 		     REMOTE_SCALARS_OUTBUFS(sc);
592 
593 	if (ctx->nscalars) {
594 		ctx->maps = kcalloc(ctx->nscalars,
595 				    sizeof(*ctx->maps), GFP_KERNEL);
596 		if (!ctx->maps) {
597 			kfree(ctx);
598 			return ERR_PTR(-ENOMEM);
599 		}
600 		ctx->olaps = kcalloc(ctx->nscalars,
601 				    sizeof(*ctx->olaps), GFP_KERNEL);
602 		if (!ctx->olaps) {
603 			kfree(ctx->maps);
604 			kfree(ctx);
605 			return ERR_PTR(-ENOMEM);
606 		}
607 		ctx->args = args;
608 		fastrpc_get_buff_overlaps(ctx);
609 	}
610 
611 	/* Released in fastrpc_context_put() */
612 	fastrpc_channel_ctx_get(cctx);
613 
614 	ctx->sc = sc;
615 	ctx->retval = -1;
616 	ctx->pid = current->pid;
617 	ctx->tgid = user->tgid;
618 	ctx->cctx = cctx;
619 	init_completion(&ctx->work);
620 	INIT_WORK(&ctx->put_work, fastrpc_context_put_wq);
621 
622 	spin_lock(&user->lock);
623 	list_add_tail(&ctx->node, &user->pending);
624 	spin_unlock(&user->lock);
625 
626 	spin_lock_irqsave(&cctx->lock, flags);
627 	ret = idr_alloc_cyclic(&cctx->ctx_idr, ctx, 1,
628 			       FASTRPC_CTX_MAX, GFP_ATOMIC);
629 	if (ret < 0) {
630 		spin_unlock_irqrestore(&cctx->lock, flags);
631 		goto err_idr;
632 	}
633 	ctx->ctxid = ret << 4;
634 	spin_unlock_irqrestore(&cctx->lock, flags);
635 
636 	kref_init(&ctx->refcount);
637 
638 	return ctx;
639 err_idr:
640 	spin_lock(&user->lock);
641 	list_del(&ctx->node);
642 	spin_unlock(&user->lock);
643 	fastrpc_channel_ctx_put(cctx);
644 	kfree(ctx->maps);
645 	kfree(ctx->olaps);
646 	kfree(ctx);
647 
648 	return ERR_PTR(ret);
649 }
650 
651 static struct sg_table *
fastrpc_map_dma_buf(struct dma_buf_attachment * attachment,enum dma_data_direction dir)652 fastrpc_map_dma_buf(struct dma_buf_attachment *attachment,
653 		    enum dma_data_direction dir)
654 {
655 	struct fastrpc_dma_buf_attachment *a = attachment->priv;
656 	struct sg_table *table;
657 	int ret;
658 
659 	table = &a->sgt;
660 
661 	ret = dma_map_sgtable(attachment->dev, table, dir, 0);
662 	if (ret)
663 		table = ERR_PTR(ret);
664 	return table;
665 }
666 
fastrpc_unmap_dma_buf(struct dma_buf_attachment * attach,struct sg_table * table,enum dma_data_direction dir)667 static void fastrpc_unmap_dma_buf(struct dma_buf_attachment *attach,
668 				  struct sg_table *table,
669 				  enum dma_data_direction dir)
670 {
671 	dma_unmap_sgtable(attach->dev, table, dir, 0);
672 }
673 
fastrpc_release(struct dma_buf * dmabuf)674 static void fastrpc_release(struct dma_buf *dmabuf)
675 {
676 	struct fastrpc_buf *buffer = dmabuf->priv;
677 
678 	fastrpc_buf_free(buffer);
679 }
680 
fastrpc_dma_buf_attach(struct dma_buf * dmabuf,struct dma_buf_attachment * attachment)681 static int fastrpc_dma_buf_attach(struct dma_buf *dmabuf,
682 				  struct dma_buf_attachment *attachment)
683 {
684 	struct fastrpc_dma_buf_attachment *a;
685 	struct fastrpc_buf *buffer = dmabuf->priv;
686 	int ret;
687 
688 	a = kzalloc(sizeof(*a), GFP_KERNEL);
689 	if (!a)
690 		return -ENOMEM;
691 
692 	ret = dma_get_sgtable(buffer->dev, &a->sgt, buffer->virt,
693 			      FASTRPC_PHYS(buffer->phys), buffer->size);
694 	if (ret < 0) {
695 		dev_err(buffer->dev, "failed to get scatterlist from DMA API\n");
696 		kfree(a);
697 		return -EINVAL;
698 	}
699 
700 	a->dev = attachment->dev;
701 	INIT_LIST_HEAD(&a->node);
702 	attachment->priv = a;
703 
704 	mutex_lock(&buffer->lock);
705 	list_add(&a->node, &buffer->attachments);
706 	mutex_unlock(&buffer->lock);
707 
708 	return 0;
709 }
710 
fastrpc_dma_buf_detatch(struct dma_buf * dmabuf,struct dma_buf_attachment * attachment)711 static void fastrpc_dma_buf_detatch(struct dma_buf *dmabuf,
712 				    struct dma_buf_attachment *attachment)
713 {
714 	struct fastrpc_dma_buf_attachment *a = attachment->priv;
715 	struct fastrpc_buf *buffer = dmabuf->priv;
716 
717 	mutex_lock(&buffer->lock);
718 	list_del(&a->node);
719 	mutex_unlock(&buffer->lock);
720 	sg_free_table(&a->sgt);
721 	kfree(a);
722 }
723 
fastrpc_vmap(struct dma_buf * dmabuf,struct iosys_map * map)724 static int fastrpc_vmap(struct dma_buf *dmabuf, struct iosys_map *map)
725 {
726 	struct fastrpc_buf *buf = dmabuf->priv;
727 
728 	iosys_map_set_vaddr(map, buf->virt);
729 
730 	return 0;
731 }
732 
fastrpc_mmap(struct dma_buf * dmabuf,struct vm_area_struct * vma)733 static int fastrpc_mmap(struct dma_buf *dmabuf,
734 			struct vm_area_struct *vma)
735 {
736 	struct fastrpc_buf *buf = dmabuf->priv;
737 	size_t size = vma->vm_end - vma->vm_start;
738 
739 	dma_resv_assert_held(dmabuf->resv);
740 
741 	return dma_mmap_coherent(buf->dev, vma, buf->virt,
742 				 FASTRPC_PHYS(buf->phys), size);
743 }
744 
745 static const struct dma_buf_ops fastrpc_dma_buf_ops = {
746 	.attach = fastrpc_dma_buf_attach,
747 	.detach = fastrpc_dma_buf_detatch,
748 	.map_dma_buf = fastrpc_map_dma_buf,
749 	.unmap_dma_buf = fastrpc_unmap_dma_buf,
750 	.mmap = fastrpc_mmap,
751 	.vmap = fastrpc_vmap,
752 	.release = fastrpc_release,
753 };
754 
fastrpc_map_create(struct fastrpc_user * fl,int fd,u64 len,u32 attr,struct fastrpc_map ** ppmap)755 static int fastrpc_map_create(struct fastrpc_user *fl, int fd,
756 			      u64 len, u32 attr, struct fastrpc_map **ppmap)
757 {
758 	struct fastrpc_session_ctx *sess = fl->sctx;
759 	struct fastrpc_map *map = NULL;
760 	struct sg_table *table;
761 	int err = 0;
762 
763 	if (!fastrpc_map_lookup(fl, fd, ppmap, true))
764 		return 0;
765 
766 	map = kzalloc(sizeof(*map), GFP_KERNEL);
767 	if (!map)
768 		return -ENOMEM;
769 
770 	INIT_LIST_HEAD(&map->node);
771 	kref_init(&map->refcount);
772 
773 	map->fl = fl;
774 	map->fd = fd;
775 	map->buf = dma_buf_get(fd);
776 	if (IS_ERR(map->buf)) {
777 		err = PTR_ERR(map->buf);
778 		goto get_err;
779 	}
780 
781 	map->attach = dma_buf_attach(map->buf, sess->dev);
782 	if (IS_ERR(map->attach)) {
783 		dev_err(sess->dev, "Failed to attach dmabuf\n");
784 		err = PTR_ERR(map->attach);
785 		goto attach_err;
786 	}
787 
788 	table = dma_buf_map_attachment_unlocked(map->attach, DMA_BIDIRECTIONAL);
789 	if (IS_ERR(table)) {
790 		err = PTR_ERR(table);
791 		goto map_err;
792 	}
793 	map->table = table;
794 
795 	if (attr & FASTRPC_ATTR_SECUREMAP) {
796 		map->phys = sg_phys(map->table->sgl);
797 	} else {
798 		map->phys = sg_dma_address(map->table->sgl);
799 		map->phys += ((u64)fl->sctx->sid << 32);
800 	}
801 	map->size = len;
802 	map->va = sg_virt(map->table->sgl);
803 	map->len = len;
804 
805 	if (attr & FASTRPC_ATTR_SECUREMAP) {
806 		/*
807 		 * If subsystem VMIDs are defined in DTSI, then do
808 		 * hyp_assign from HLOS to those VM(s)
809 		 */
810 		u64 src_perms = BIT(QCOM_SCM_VMID_HLOS);
811 		struct qcom_scm_vmperm dst_perms[2] = {0};
812 
813 		dst_perms[0].vmid = QCOM_SCM_VMID_HLOS;
814 		dst_perms[0].perm = QCOM_SCM_PERM_RW;
815 		dst_perms[1].vmid = fl->cctx->vmperms[0].vmid;
816 		dst_perms[1].perm = QCOM_SCM_PERM_RWX;
817 		map->attr = attr;
818 		err = qcom_scm_assign_mem(map->phys, (u64)map->size, &src_perms, dst_perms, 2);
819 		if (err) {
820 			dev_err(sess->dev, "Failed to assign memory with phys 0x%llx size 0x%llx err %d\n",
821 					map->phys, map->size, err);
822 			goto map_err;
823 		}
824 	}
825 	spin_lock(&fl->lock);
826 	list_add_tail(&map->node, &fl->maps);
827 	spin_unlock(&fl->lock);
828 	*ppmap = map;
829 
830 	return 0;
831 
832 map_err:
833 	dma_buf_detach(map->buf, map->attach);
834 attach_err:
835 	dma_buf_put(map->buf);
836 get_err:
837 	fastrpc_map_put(map);
838 
839 	return err;
840 }
841 
842 /*
843  * Fastrpc payload buffer with metadata looks like:
844  *
845  * >>>>>>  START of METADATA <<<<<<<<<
846  * +---------------------------------+
847  * |           Arguments             |
848  * | type:(union fastrpc_remote_arg)|
849  * |             (0 - N)             |
850  * +---------------------------------+
851  * |         Invoke Buffer list      |
852  * | type:(struct fastrpc_invoke_buf)|
853  * |           (0 - N)               |
854  * +---------------------------------+
855  * |         Page info list          |
856  * | type:(struct fastrpc_phy_page)  |
857  * |             (0 - N)             |
858  * +---------------------------------+
859  * |         Optional info           |
860  * |(can be specific to SoC/Firmware)|
861  * +---------------------------------+
862  * >>>>>>>>  END of METADATA <<<<<<<<<
863  * +---------------------------------+
864  * |         Inline ARGS             |
865  * |            (0-N)                |
866  * +---------------------------------+
867  */
868 
fastrpc_get_meta_size(struct fastrpc_invoke_ctx * ctx)869 static int fastrpc_get_meta_size(struct fastrpc_invoke_ctx *ctx)
870 {
871 	int size = 0;
872 
873 	size = (sizeof(struct fastrpc_remote_buf) +
874 		sizeof(struct fastrpc_invoke_buf) +
875 		sizeof(struct fastrpc_phy_page)) * ctx->nscalars +
876 		sizeof(u64) * FASTRPC_MAX_FDLIST +
877 		sizeof(u32) * FASTRPC_MAX_CRCLIST;
878 
879 	return size;
880 }
881 
fastrpc_get_payload_size(struct fastrpc_invoke_ctx * ctx,int metalen)882 static u64 fastrpc_get_payload_size(struct fastrpc_invoke_ctx *ctx, int metalen)
883 {
884 	u64 size = 0;
885 	int oix;
886 
887 	size = ALIGN(metalen, FASTRPC_ALIGN);
888 	for (oix = 0; oix < ctx->nbufs; oix++) {
889 		int i = ctx->olaps[oix].raix;
890 
891 		if (ctx->args[i].fd == 0 || ctx->args[i].fd == -1) {
892 
893 			if (ctx->olaps[oix].offset == 0)
894 				size = ALIGN(size, FASTRPC_ALIGN);
895 
896 			size += (ctx->olaps[oix].mend - ctx->olaps[oix].mstart);
897 		}
898 	}
899 
900 	return size;
901 }
902 
fastrpc_create_maps(struct fastrpc_invoke_ctx * ctx)903 static int fastrpc_create_maps(struct fastrpc_invoke_ctx *ctx)
904 {
905 	struct device *dev = ctx->fl->sctx->dev;
906 	int i, err;
907 
908 	for (i = 0; i < ctx->nscalars; ++i) {
909 
910 		if (ctx->args[i].fd == 0 || ctx->args[i].fd == -1 ||
911 		    ctx->args[i].length == 0)
912 			continue;
913 
914 		err = fastrpc_map_create(ctx->fl, ctx->args[i].fd,
915 			 ctx->args[i].length, ctx->args[i].attr, &ctx->maps[i]);
916 		if (err) {
917 			dev_err(dev, "Error Creating map %d\n", err);
918 			return -EINVAL;
919 		}
920 
921 	}
922 	return 0;
923 }
924 
fastrpc_invoke_buf_start(union fastrpc_remote_arg * pra,int len)925 static struct fastrpc_invoke_buf *fastrpc_invoke_buf_start(union fastrpc_remote_arg *pra, int len)
926 {
927 	return (struct fastrpc_invoke_buf *)(&pra[len]);
928 }
929 
fastrpc_phy_page_start(struct fastrpc_invoke_buf * buf,int len)930 static struct fastrpc_phy_page *fastrpc_phy_page_start(struct fastrpc_invoke_buf *buf, int len)
931 {
932 	return (struct fastrpc_phy_page *)(&buf[len]);
933 }
934 
fastrpc_get_args(u32 kernel,struct fastrpc_invoke_ctx * ctx)935 static int fastrpc_get_args(u32 kernel, struct fastrpc_invoke_ctx *ctx)
936 {
937 	struct device *dev = ctx->fl->sctx->dev;
938 	union fastrpc_remote_arg *rpra;
939 	struct fastrpc_invoke_buf *list;
940 	struct fastrpc_phy_page *pages;
941 	int inbufs, i, oix, err = 0;
942 	u64 len, rlen, pkt_size;
943 	u64 pg_start, pg_end;
944 	uintptr_t args;
945 	int metalen;
946 
947 	inbufs = REMOTE_SCALARS_INBUFS(ctx->sc);
948 	metalen = fastrpc_get_meta_size(ctx);
949 	pkt_size = fastrpc_get_payload_size(ctx, metalen);
950 
951 	err = fastrpc_create_maps(ctx);
952 	if (err)
953 		return err;
954 
955 	ctx->msg_sz = pkt_size;
956 
957 	if (ctx->fl->sctx->sid)
958 		err = fastrpc_buf_alloc(ctx->fl, dev, pkt_size, &ctx->buf);
959 	else
960 		err = fastrpc_remote_heap_alloc(ctx->fl, dev, pkt_size, &ctx->buf);
961 	if (err)
962 		return err;
963 
964 	memset(ctx->buf->virt, 0, pkt_size);
965 	rpra = ctx->buf->virt;
966 	list = fastrpc_invoke_buf_start(rpra, ctx->nscalars);
967 	pages = fastrpc_phy_page_start(list, ctx->nscalars);
968 	args = (uintptr_t)ctx->buf->virt + metalen;
969 	rlen = pkt_size - metalen;
970 	ctx->rpra = rpra;
971 
972 	for (oix = 0; oix < ctx->nbufs; ++oix) {
973 		int mlen;
974 
975 		i = ctx->olaps[oix].raix;
976 		len = ctx->args[i].length;
977 
978 		rpra[i].buf.pv = 0;
979 		rpra[i].buf.len = len;
980 		list[i].num = len ? 1 : 0;
981 		list[i].pgidx = i;
982 
983 		if (!len)
984 			continue;
985 
986 		if (ctx->maps[i]) {
987 			struct vm_area_struct *vma = NULL;
988 
989 			rpra[i].buf.pv = (u64) ctx->args[i].ptr;
990 			pages[i].addr = ctx->maps[i]->phys;
991 
992 			mmap_read_lock(current->mm);
993 			vma = find_vma(current->mm, ctx->args[i].ptr);
994 			if (vma)
995 				pages[i].addr += ctx->args[i].ptr -
996 						 vma->vm_start;
997 			mmap_read_unlock(current->mm);
998 
999 			pg_start = (ctx->args[i].ptr & PAGE_MASK) >> PAGE_SHIFT;
1000 			pg_end = ((ctx->args[i].ptr + len - 1) & PAGE_MASK) >>
1001 				  PAGE_SHIFT;
1002 			pages[i].size = (pg_end - pg_start + 1) * PAGE_SIZE;
1003 
1004 		} else {
1005 
1006 			if (ctx->olaps[oix].offset == 0) {
1007 				rlen -= ALIGN(args, FASTRPC_ALIGN) - args;
1008 				args = ALIGN(args, FASTRPC_ALIGN);
1009 			}
1010 
1011 			mlen = ctx->olaps[oix].mend - ctx->olaps[oix].mstart;
1012 
1013 			if (rlen < mlen)
1014 				goto bail;
1015 
1016 			rpra[i].buf.pv = args - ctx->olaps[oix].offset;
1017 			pages[i].addr = ctx->buf->phys -
1018 					ctx->olaps[oix].offset +
1019 					(pkt_size - rlen);
1020 			pages[i].addr = pages[i].addr &	PAGE_MASK;
1021 
1022 			pg_start = (args & PAGE_MASK) >> PAGE_SHIFT;
1023 			pg_end = ((args + len - 1) & PAGE_MASK) >> PAGE_SHIFT;
1024 			pages[i].size = (pg_end - pg_start + 1) * PAGE_SIZE;
1025 			args = args + mlen;
1026 			rlen -= mlen;
1027 		}
1028 
1029 		if (i < inbufs && !ctx->maps[i]) {
1030 			void *dst = (void *)(uintptr_t)rpra[i].buf.pv;
1031 			void *src = (void *)(uintptr_t)ctx->args[i].ptr;
1032 
1033 			if (!kernel) {
1034 				if (copy_from_user(dst, (void __user *)src,
1035 						   len)) {
1036 					err = -EFAULT;
1037 					goto bail;
1038 				}
1039 			} else {
1040 				memcpy(dst, src, len);
1041 			}
1042 		}
1043 	}
1044 
1045 	for (i = ctx->nbufs; i < ctx->nscalars; ++i) {
1046 		list[i].num = ctx->args[i].length ? 1 : 0;
1047 		list[i].pgidx = i;
1048 		if (ctx->maps[i]) {
1049 			pages[i].addr = ctx->maps[i]->phys;
1050 			pages[i].size = ctx->maps[i]->size;
1051 		}
1052 		rpra[i].dma.fd = ctx->args[i].fd;
1053 		rpra[i].dma.len = ctx->args[i].length;
1054 		rpra[i].dma.offset = (u64) ctx->args[i].ptr;
1055 	}
1056 
1057 bail:
1058 	if (err)
1059 		dev_err(dev, "Error: get invoke args failed:%d\n", err);
1060 
1061 	return err;
1062 }
1063 
fastrpc_put_args(struct fastrpc_invoke_ctx * ctx,u32 kernel)1064 static int fastrpc_put_args(struct fastrpc_invoke_ctx *ctx,
1065 			    u32 kernel)
1066 {
1067 	union fastrpc_remote_arg *rpra = ctx->rpra;
1068 	struct fastrpc_user *fl = ctx->fl;
1069 	struct fastrpc_map *mmap = NULL;
1070 	struct fastrpc_invoke_buf *list;
1071 	struct fastrpc_phy_page *pages;
1072 	u64 *fdlist;
1073 	int i, inbufs, outbufs, handles;
1074 
1075 	inbufs = REMOTE_SCALARS_INBUFS(ctx->sc);
1076 	outbufs = REMOTE_SCALARS_OUTBUFS(ctx->sc);
1077 	handles = REMOTE_SCALARS_INHANDLES(ctx->sc) + REMOTE_SCALARS_OUTHANDLES(ctx->sc);
1078 	list = fastrpc_invoke_buf_start(rpra, ctx->nscalars);
1079 	pages = fastrpc_phy_page_start(list, ctx->nscalars);
1080 	fdlist = (uint64_t *)(pages + inbufs + outbufs + handles);
1081 
1082 	for (i = inbufs; i < ctx->nbufs; ++i) {
1083 		if (!ctx->maps[i]) {
1084 			void *src = (void *)(uintptr_t)rpra[i].buf.pv;
1085 			void *dst = (void *)(uintptr_t)ctx->args[i].ptr;
1086 			u64 len = rpra[i].buf.len;
1087 
1088 			if (!kernel) {
1089 				if (copy_to_user((void __user *)dst, src, len))
1090 					return -EFAULT;
1091 			} else {
1092 				memcpy(dst, src, len);
1093 			}
1094 		}
1095 	}
1096 
1097 	/* Clean up fdlist which is updated by DSP */
1098 	for (i = 0; i < FASTRPC_MAX_FDLIST; i++) {
1099 		if (!fdlist[i])
1100 			break;
1101 		if (!fastrpc_map_lookup(fl, (int)fdlist[i], &mmap, false))
1102 			fastrpc_map_put(mmap);
1103 	}
1104 
1105 	return 0;
1106 }
1107 
fastrpc_invoke_send(struct fastrpc_session_ctx * sctx,struct fastrpc_invoke_ctx * ctx,u32 kernel,uint32_t handle)1108 static int fastrpc_invoke_send(struct fastrpc_session_ctx *sctx,
1109 			       struct fastrpc_invoke_ctx *ctx,
1110 			       u32 kernel, uint32_t handle)
1111 {
1112 	struct fastrpc_channel_ctx *cctx;
1113 	struct fastrpc_user *fl = ctx->fl;
1114 	struct fastrpc_msg *msg = &ctx->msg;
1115 	int ret;
1116 
1117 	cctx = fl->cctx;
1118 	msg->pid = fl->tgid;
1119 	msg->tid = current->pid;
1120 
1121 	if (kernel)
1122 		msg->pid = 0;
1123 
1124 	msg->ctx = ctx->ctxid | fl->pd;
1125 	msg->handle = handle;
1126 	msg->sc = ctx->sc;
1127 	msg->addr = ctx->buf ? ctx->buf->phys : 0;
1128 	msg->size = roundup(ctx->msg_sz, PAGE_SIZE);
1129 	fastrpc_context_get(ctx);
1130 
1131 	ret = rpmsg_send(cctx->rpdev->ept, (void *)msg, sizeof(*msg));
1132 
1133 	if (ret)
1134 		fastrpc_context_put(ctx);
1135 
1136 	return ret;
1137 
1138 }
1139 
fastrpc_internal_invoke(struct fastrpc_user * fl,u32 kernel,u32 handle,u32 sc,struct fastrpc_invoke_args * args)1140 static int fastrpc_internal_invoke(struct fastrpc_user *fl,  u32 kernel,
1141 				   u32 handle, u32 sc,
1142 				   struct fastrpc_invoke_args *args)
1143 {
1144 	struct fastrpc_invoke_ctx *ctx = NULL;
1145 	struct fastrpc_buf *buf, *b;
1146 
1147 	int err = 0;
1148 
1149 	if (!fl->sctx)
1150 		return -EINVAL;
1151 
1152 	if (!fl->cctx->rpdev)
1153 		return -EPIPE;
1154 
1155 	if (handle == FASTRPC_INIT_HANDLE && !kernel) {
1156 		dev_warn_ratelimited(fl->sctx->dev, "user app trying to send a kernel RPC message (%d)\n",  handle);
1157 		return -EPERM;
1158 	}
1159 
1160 	ctx = fastrpc_context_alloc(fl, kernel, sc, args);
1161 	if (IS_ERR(ctx))
1162 		return PTR_ERR(ctx);
1163 
1164 	err = fastrpc_get_args(kernel, ctx);
1165 	if (err)
1166 		goto bail;
1167 
1168 	/* make sure that all CPU memory writes are seen by DSP */
1169 	dma_wmb();
1170 	/* Send invoke buffer to remote dsp */
1171 	err = fastrpc_invoke_send(fl->sctx, ctx, kernel, handle);
1172 	if (err)
1173 		goto bail;
1174 
1175 	if (kernel) {
1176 		if (!wait_for_completion_timeout(&ctx->work, 10 * HZ))
1177 			err = -ETIMEDOUT;
1178 	} else {
1179 		err = wait_for_completion_interruptible(&ctx->work);
1180 	}
1181 
1182 	if (err)
1183 		goto bail;
1184 
1185 	/* make sure that all memory writes by DSP are seen by CPU */
1186 	dma_rmb();
1187 	/* populate all the output buffers with results */
1188 	err = fastrpc_put_args(ctx, kernel);
1189 	if (err)
1190 		goto bail;
1191 
1192 	/* Check the response from remote dsp */
1193 	err = ctx->retval;
1194 	if (err)
1195 		goto bail;
1196 
1197 bail:
1198 	if (err != -ERESTARTSYS && err != -ETIMEDOUT) {
1199 		/* We are done with this compute context */
1200 		spin_lock(&fl->lock);
1201 		list_del(&ctx->node);
1202 		spin_unlock(&fl->lock);
1203 		fastrpc_context_put(ctx);
1204 	}
1205 
1206 	if (err == -ERESTARTSYS) {
1207 		list_for_each_entry_safe(buf, b, &fl->mmaps, node) {
1208 			list_del(&buf->node);
1209 			list_add_tail(&buf->node, &fl->cctx->invoke_interrupted_mmaps);
1210 		}
1211 	}
1212 
1213 	if (err)
1214 		dev_dbg(fl->sctx->dev, "Error: Invoke Failed %d\n", err);
1215 
1216 	return err;
1217 }
1218 
is_session_rejected(struct fastrpc_user * fl,bool unsigned_pd_request)1219 static bool is_session_rejected(struct fastrpc_user *fl, bool unsigned_pd_request)
1220 {
1221 	/* Check if the device node is non-secure and channel is secure*/
1222 	if (!fl->is_secure_dev && fl->cctx->secure) {
1223 		/*
1224 		 * Allow untrusted applications to offload only to Unsigned PD when
1225 		 * channel is configured as secure and block untrusted apps on channel
1226 		 * that does not support unsigned PD offload
1227 		 */
1228 		if (!fl->cctx->unsigned_support || !unsigned_pd_request) {
1229 			dev_err(&fl->cctx->rpdev->dev, "Error: Untrusted application trying to offload to signed PD\n");
1230 			return true;
1231 		}
1232 	}
1233 
1234 	return false;
1235 }
1236 
fastrpc_init_create_static_process(struct fastrpc_user * fl,char __user * argp)1237 static int fastrpc_init_create_static_process(struct fastrpc_user *fl,
1238 					      char __user *argp)
1239 {
1240 	struct fastrpc_init_create_static init;
1241 	struct fastrpc_invoke_args *args;
1242 	struct fastrpc_phy_page pages[1];
1243 	char *name;
1244 	int err;
1245 	bool scm_done = false;
1246 	struct {
1247 		int pgid;
1248 		u32 namelen;
1249 		u32 pageslen;
1250 	} inbuf;
1251 	u32 sc;
1252 
1253 	args = kcalloc(FASTRPC_CREATE_STATIC_PROCESS_NARGS, sizeof(*args), GFP_KERNEL);
1254 	if (!args)
1255 		return -ENOMEM;
1256 
1257 	if (copy_from_user(&init, argp, sizeof(init))) {
1258 		err = -EFAULT;
1259 		goto err;
1260 	}
1261 
1262 	if (init.namelen > INIT_FILE_NAMELEN_MAX) {
1263 		err = -EINVAL;
1264 		goto err;
1265 	}
1266 
1267 	name = memdup_user(u64_to_user_ptr(init.name), init.namelen);
1268 	if (IS_ERR(name)) {
1269 		err = PTR_ERR(name);
1270 		goto err;
1271 	}
1272 
1273 	if (!fl->cctx->remote_heap) {
1274 		err = fastrpc_remote_heap_alloc(fl, fl->sctx->dev, init.memlen,
1275 						&fl->cctx->remote_heap);
1276 		if (err)
1277 			goto err_name;
1278 
1279 		/* Map if we have any heap VMIDs associated with this ADSP Static Process. */
1280 		if (fl->cctx->vmcount) {
1281 			u64 src_perms = BIT(QCOM_SCM_VMID_HLOS);
1282 
1283 			err = qcom_scm_assign_mem(fl->cctx->remote_heap->phys,
1284 							(u64)fl->cctx->remote_heap->size,
1285 							&src_perms,
1286 							fl->cctx->vmperms, fl->cctx->vmcount);
1287 			if (err) {
1288 				dev_err(fl->sctx->dev, "Failed to assign memory with phys 0x%llx size 0x%llx err %d\n",
1289 					fl->cctx->remote_heap->phys, fl->cctx->remote_heap->size, err);
1290 				goto err_map;
1291 			}
1292 			scm_done = true;
1293 		}
1294 	}
1295 
1296 	inbuf.pgid = fl->tgid;
1297 	inbuf.namelen = init.namelen;
1298 	inbuf.pageslen = 0;
1299 	fl->pd = USER_PD;
1300 
1301 	args[0].ptr = (u64)(uintptr_t)&inbuf;
1302 	args[0].length = sizeof(inbuf);
1303 	args[0].fd = -1;
1304 
1305 	args[1].ptr = (u64)(uintptr_t)name;
1306 	args[1].length = inbuf.namelen;
1307 	args[1].fd = -1;
1308 
1309 	pages[0].addr = fl->cctx->remote_heap->phys;
1310 	pages[0].size = fl->cctx->remote_heap->size;
1311 
1312 	args[2].ptr = (u64)(uintptr_t) pages;
1313 	args[2].length = sizeof(*pages);
1314 	args[2].fd = -1;
1315 
1316 	sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_CREATE_STATIC, 3, 0);
1317 
1318 	err = fastrpc_internal_invoke(fl, true, FASTRPC_INIT_HANDLE,
1319 				      sc, args);
1320 	if (err)
1321 		goto err_invoke;
1322 
1323 	kfree(args);
1324 	kfree(name);
1325 
1326 	return 0;
1327 err_invoke:
1328 	if (fl->cctx->vmcount && scm_done) {
1329 		u64 src_perms = 0;
1330 		struct qcom_scm_vmperm dst_perms;
1331 		u32 i;
1332 
1333 		for (i = 0; i < fl->cctx->vmcount; i++)
1334 			src_perms |= BIT(fl->cctx->vmperms[i].vmid);
1335 
1336 		dst_perms.vmid = QCOM_SCM_VMID_HLOS;
1337 		dst_perms.perm = QCOM_SCM_PERM_RWX;
1338 		err = qcom_scm_assign_mem(fl->cctx->remote_heap->phys,
1339 						(u64)fl->cctx->remote_heap->size,
1340 						&src_perms, &dst_perms, 1);
1341 		if (err)
1342 			dev_err(fl->sctx->dev, "Failed to assign memory phys 0x%llx size 0x%llx err %d\n",
1343 				fl->cctx->remote_heap->phys, fl->cctx->remote_heap->size, err);
1344 	}
1345 err_map:
1346 	fastrpc_buf_free(fl->cctx->remote_heap);
1347 err_name:
1348 	kfree(name);
1349 err:
1350 	kfree(args);
1351 
1352 	return err;
1353 }
1354 
fastrpc_init_create_process(struct fastrpc_user * fl,char __user * argp)1355 static int fastrpc_init_create_process(struct fastrpc_user *fl,
1356 					char __user *argp)
1357 {
1358 	struct fastrpc_init_create init;
1359 	struct fastrpc_invoke_args *args;
1360 	struct fastrpc_phy_page pages[1];
1361 	struct fastrpc_map *map = NULL;
1362 	struct fastrpc_buf *imem = NULL;
1363 	int memlen;
1364 	int err;
1365 	struct {
1366 		int pgid;
1367 		u32 namelen;
1368 		u32 filelen;
1369 		u32 pageslen;
1370 		u32 attrs;
1371 		u32 siglen;
1372 	} inbuf;
1373 	u32 sc;
1374 	bool unsigned_module = false;
1375 
1376 	args = kcalloc(FASTRPC_CREATE_PROCESS_NARGS, sizeof(*args), GFP_KERNEL);
1377 	if (!args)
1378 		return -ENOMEM;
1379 
1380 	if (copy_from_user(&init, argp, sizeof(init))) {
1381 		err = -EFAULT;
1382 		goto err;
1383 	}
1384 
1385 	if (init.attrs & FASTRPC_MODE_UNSIGNED_MODULE)
1386 		unsigned_module = true;
1387 
1388 	if (is_session_rejected(fl, unsigned_module)) {
1389 		err = -ECONNREFUSED;
1390 		goto err;
1391 	}
1392 
1393 	if (init.filelen > INIT_FILELEN_MAX) {
1394 		err = -EINVAL;
1395 		goto err;
1396 	}
1397 
1398 	inbuf.pgid = fl->tgid;
1399 	inbuf.namelen = strlen(current->comm) + 1;
1400 	inbuf.filelen = init.filelen;
1401 	inbuf.pageslen = 1;
1402 	inbuf.attrs = init.attrs;
1403 	inbuf.siglen = init.siglen;
1404 	fl->pd = USER_PD;
1405 
1406 	if (init.filelen && init.filefd) {
1407 		err = fastrpc_map_create(fl, init.filefd, init.filelen, 0, &map);
1408 		if (err)
1409 			goto err;
1410 	}
1411 
1412 	memlen = ALIGN(max(INIT_FILELEN_MAX, (int)init.filelen * 4),
1413 		       1024 * 1024);
1414 	err = fastrpc_buf_alloc(fl, fl->sctx->dev, memlen,
1415 				&imem);
1416 	if (err)
1417 		goto err_alloc;
1418 
1419 	fl->init_mem = imem;
1420 	args[0].ptr = (u64)(uintptr_t)&inbuf;
1421 	args[0].length = sizeof(inbuf);
1422 	args[0].fd = -1;
1423 
1424 	args[1].ptr = (u64)(uintptr_t)current->comm;
1425 	args[1].length = inbuf.namelen;
1426 	args[1].fd = -1;
1427 
1428 	args[2].ptr = (u64) init.file;
1429 	args[2].length = inbuf.filelen;
1430 	args[2].fd = init.filefd;
1431 
1432 	pages[0].addr = imem->phys;
1433 	pages[0].size = imem->size;
1434 
1435 	args[3].ptr = (u64)(uintptr_t) pages;
1436 	args[3].length = 1 * sizeof(*pages);
1437 	args[3].fd = -1;
1438 
1439 	args[4].ptr = (u64)(uintptr_t)&inbuf.attrs;
1440 	args[4].length = sizeof(inbuf.attrs);
1441 	args[4].fd = -1;
1442 
1443 	args[5].ptr = (u64)(uintptr_t) &inbuf.siglen;
1444 	args[5].length = sizeof(inbuf.siglen);
1445 	args[5].fd = -1;
1446 
1447 	sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_CREATE, 4, 0);
1448 	if (init.attrs)
1449 		sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_CREATE_ATTR, 4, 0);
1450 
1451 	err = fastrpc_internal_invoke(fl, true, FASTRPC_INIT_HANDLE,
1452 				      sc, args);
1453 	if (err)
1454 		goto err_invoke;
1455 
1456 	kfree(args);
1457 
1458 	return 0;
1459 
1460 err_invoke:
1461 	fl->init_mem = NULL;
1462 	fastrpc_buf_free(imem);
1463 err_alloc:
1464 	fastrpc_map_put(map);
1465 err:
1466 	kfree(args);
1467 
1468 	return err;
1469 }
1470 
fastrpc_session_alloc(struct fastrpc_channel_ctx * cctx)1471 static struct fastrpc_session_ctx *fastrpc_session_alloc(
1472 					struct fastrpc_channel_ctx *cctx)
1473 {
1474 	struct fastrpc_session_ctx *session = NULL;
1475 	unsigned long flags;
1476 	int i;
1477 
1478 	spin_lock_irqsave(&cctx->lock, flags);
1479 	for (i = 0; i < cctx->sesscount; i++) {
1480 		if (!cctx->session[i].used && cctx->session[i].valid) {
1481 			cctx->session[i].used = true;
1482 			session = &cctx->session[i];
1483 			break;
1484 		}
1485 	}
1486 	spin_unlock_irqrestore(&cctx->lock, flags);
1487 
1488 	return session;
1489 }
1490 
fastrpc_session_free(struct fastrpc_channel_ctx * cctx,struct fastrpc_session_ctx * session)1491 static void fastrpc_session_free(struct fastrpc_channel_ctx *cctx,
1492 				 struct fastrpc_session_ctx *session)
1493 {
1494 	unsigned long flags;
1495 
1496 	spin_lock_irqsave(&cctx->lock, flags);
1497 	session->used = false;
1498 	spin_unlock_irqrestore(&cctx->lock, flags);
1499 }
1500 
fastrpc_release_current_dsp_process(struct fastrpc_user * fl)1501 static int fastrpc_release_current_dsp_process(struct fastrpc_user *fl)
1502 {
1503 	struct fastrpc_invoke_args args[1];
1504 	int tgid = 0;
1505 	u32 sc;
1506 
1507 	tgid = fl->tgid;
1508 	args[0].ptr = (u64)(uintptr_t) &tgid;
1509 	args[0].length = sizeof(tgid);
1510 	args[0].fd = -1;
1511 	sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_RELEASE, 1, 0);
1512 
1513 	return fastrpc_internal_invoke(fl, true, FASTRPC_INIT_HANDLE,
1514 				       sc, &args[0]);
1515 }
1516 
fastrpc_device_release(struct inode * inode,struct file * file)1517 static int fastrpc_device_release(struct inode *inode, struct file *file)
1518 {
1519 	struct fastrpc_user *fl = (struct fastrpc_user *)file->private_data;
1520 	struct fastrpc_channel_ctx *cctx = fl->cctx;
1521 	struct fastrpc_invoke_ctx *ctx, *n;
1522 	struct fastrpc_map *map, *m;
1523 	struct fastrpc_buf *buf, *b;
1524 	unsigned long flags;
1525 
1526 	fastrpc_release_current_dsp_process(fl);
1527 
1528 	spin_lock_irqsave(&cctx->lock, flags);
1529 	list_del(&fl->user);
1530 	spin_unlock_irqrestore(&cctx->lock, flags);
1531 
1532 	if (fl->init_mem)
1533 		fastrpc_buf_free(fl->init_mem);
1534 
1535 	list_for_each_entry_safe(ctx, n, &fl->pending, node) {
1536 		list_del(&ctx->node);
1537 		fastrpc_context_put(ctx);
1538 	}
1539 
1540 	list_for_each_entry_safe(map, m, &fl->maps, node)
1541 		fastrpc_map_put(map);
1542 
1543 	list_for_each_entry_safe(buf, b, &fl->mmaps, node) {
1544 		list_del(&buf->node);
1545 		fastrpc_buf_free(buf);
1546 	}
1547 
1548 	fastrpc_session_free(cctx, fl->sctx);
1549 	fastrpc_channel_ctx_put(cctx);
1550 
1551 	mutex_destroy(&fl->mutex);
1552 	kfree(fl);
1553 	file->private_data = NULL;
1554 
1555 	return 0;
1556 }
1557 
fastrpc_device_open(struct inode * inode,struct file * filp)1558 static int fastrpc_device_open(struct inode *inode, struct file *filp)
1559 {
1560 	struct fastrpc_channel_ctx *cctx;
1561 	struct fastrpc_device *fdevice;
1562 	struct fastrpc_user *fl = NULL;
1563 	unsigned long flags;
1564 
1565 	fdevice = miscdev_to_fdevice(filp->private_data);
1566 	cctx = fdevice->cctx;
1567 
1568 	fl = kzalloc(sizeof(*fl), GFP_KERNEL);
1569 	if (!fl)
1570 		return -ENOMEM;
1571 
1572 	/* Released in fastrpc_device_release() */
1573 	fastrpc_channel_ctx_get(cctx);
1574 
1575 	filp->private_data = fl;
1576 	spin_lock_init(&fl->lock);
1577 	mutex_init(&fl->mutex);
1578 	INIT_LIST_HEAD(&fl->pending);
1579 	INIT_LIST_HEAD(&fl->maps);
1580 	INIT_LIST_HEAD(&fl->mmaps);
1581 	INIT_LIST_HEAD(&fl->user);
1582 	fl->tgid = current->tgid;
1583 	fl->cctx = cctx;
1584 	fl->is_secure_dev = fdevice->secure;
1585 
1586 	fl->sctx = fastrpc_session_alloc(cctx);
1587 	if (!fl->sctx) {
1588 		dev_err(&cctx->rpdev->dev, "No session available\n");
1589 		mutex_destroy(&fl->mutex);
1590 		kfree(fl);
1591 
1592 		return -EBUSY;
1593 	}
1594 
1595 	spin_lock_irqsave(&cctx->lock, flags);
1596 	list_add_tail(&fl->user, &cctx->users);
1597 	spin_unlock_irqrestore(&cctx->lock, flags);
1598 
1599 	return 0;
1600 }
1601 
fastrpc_dmabuf_alloc(struct fastrpc_user * fl,char __user * argp)1602 static int fastrpc_dmabuf_alloc(struct fastrpc_user *fl, char __user *argp)
1603 {
1604 	struct fastrpc_alloc_dma_buf bp;
1605 	DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
1606 	struct fastrpc_buf *buf = NULL;
1607 	int err;
1608 
1609 	if (copy_from_user(&bp, argp, sizeof(bp)))
1610 		return -EFAULT;
1611 
1612 	err = fastrpc_buf_alloc(fl, fl->sctx->dev, bp.size, &buf);
1613 	if (err)
1614 		return err;
1615 	exp_info.ops = &fastrpc_dma_buf_ops;
1616 	exp_info.size = bp.size;
1617 	exp_info.flags = O_RDWR;
1618 	exp_info.priv = buf;
1619 	buf->dmabuf = dma_buf_export(&exp_info);
1620 	if (IS_ERR(buf->dmabuf)) {
1621 		err = PTR_ERR(buf->dmabuf);
1622 		fastrpc_buf_free(buf);
1623 		return err;
1624 	}
1625 
1626 	bp.fd = dma_buf_fd(buf->dmabuf, O_ACCMODE);
1627 	if (bp.fd < 0) {
1628 		dma_buf_put(buf->dmabuf);
1629 		return -EINVAL;
1630 	}
1631 
1632 	if (copy_to_user(argp, &bp, sizeof(bp))) {
1633 		/*
1634 		 * The usercopy failed, but we can't do much about it, as
1635 		 * dma_buf_fd() already called fd_install() and made the
1636 		 * file descriptor accessible for the current process. It
1637 		 * might already be closed and dmabuf no longer valid when
1638 		 * we reach this point. Therefore "leak" the fd and rely on
1639 		 * the process exit path to do any required cleanup.
1640 		 */
1641 		return -EFAULT;
1642 	}
1643 
1644 	return 0;
1645 }
1646 
fastrpc_init_attach(struct fastrpc_user * fl,int pd)1647 static int fastrpc_init_attach(struct fastrpc_user *fl, int pd)
1648 {
1649 	struct fastrpc_invoke_args args[1];
1650 	int tgid = fl->tgid;
1651 	u32 sc;
1652 
1653 	args[0].ptr = (u64)(uintptr_t) &tgid;
1654 	args[0].length = sizeof(tgid);
1655 	args[0].fd = -1;
1656 	sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_ATTACH, 1, 0);
1657 	fl->pd = pd;
1658 
1659 	return fastrpc_internal_invoke(fl, true, FASTRPC_INIT_HANDLE,
1660 				       sc, &args[0]);
1661 }
1662 
fastrpc_invoke(struct fastrpc_user * fl,char __user * argp)1663 static int fastrpc_invoke(struct fastrpc_user *fl, char __user *argp)
1664 {
1665 	struct fastrpc_invoke_args *args = NULL;
1666 	struct fastrpc_invoke inv;
1667 	u32 nscalars;
1668 	int err;
1669 
1670 	if (copy_from_user(&inv, argp, sizeof(inv)))
1671 		return -EFAULT;
1672 
1673 	/* nscalars is truncated here to max supported value */
1674 	nscalars = REMOTE_SCALARS_LENGTH(inv.sc);
1675 	if (nscalars) {
1676 		args = kcalloc(nscalars, sizeof(*args), GFP_KERNEL);
1677 		if (!args)
1678 			return -ENOMEM;
1679 
1680 		if (copy_from_user(args, (void __user *)(uintptr_t)inv.args,
1681 				   nscalars * sizeof(*args))) {
1682 			kfree(args);
1683 			return -EFAULT;
1684 		}
1685 	}
1686 
1687 	err = fastrpc_internal_invoke(fl, false, inv.handle, inv.sc, args);
1688 	kfree(args);
1689 
1690 	return err;
1691 }
1692 
fastrpc_get_info_from_dsp(struct fastrpc_user * fl,uint32_t * dsp_attr_buf,uint32_t dsp_attr_buf_len)1693 static int fastrpc_get_info_from_dsp(struct fastrpc_user *fl, uint32_t *dsp_attr_buf,
1694 				     uint32_t dsp_attr_buf_len)
1695 {
1696 	struct fastrpc_invoke_args args[2] = { 0 };
1697 
1698 	/*
1699 	 * Capability filled in userspace. This carries the information
1700 	 * about the remoteproc support which is fetched from the remoteproc
1701 	 * sysfs node by userspace.
1702 	 */
1703 	dsp_attr_buf[0] = 0;
1704 	dsp_attr_buf_len -= 1;
1705 
1706 	args[0].ptr = (u64)(uintptr_t)&dsp_attr_buf_len;
1707 	args[0].length = sizeof(dsp_attr_buf_len);
1708 	args[0].fd = -1;
1709 	args[1].ptr = (u64)(uintptr_t)&dsp_attr_buf[1];
1710 	args[1].length = dsp_attr_buf_len * sizeof(u32);
1711 	args[1].fd = -1;
1712 
1713 	return fastrpc_internal_invoke(fl, true, FASTRPC_DSP_UTILITIES_HANDLE,
1714 				       FASTRPC_SCALARS(0, 1, 1), args);
1715 }
1716 
fastrpc_get_info_from_kernel(struct fastrpc_ioctl_capability * cap,struct fastrpc_user * fl)1717 static int fastrpc_get_info_from_kernel(struct fastrpc_ioctl_capability *cap,
1718 					struct fastrpc_user *fl)
1719 {
1720 	struct fastrpc_channel_ctx *cctx = fl->cctx;
1721 	uint32_t attribute_id = cap->attribute_id;
1722 	uint32_t *dsp_attributes;
1723 	unsigned long flags;
1724 	uint32_t domain = cap->domain;
1725 	int err;
1726 
1727 	spin_lock_irqsave(&cctx->lock, flags);
1728 	/* check if we already have queried dsp for attributes */
1729 	if (cctx->valid_attributes) {
1730 		spin_unlock_irqrestore(&cctx->lock, flags);
1731 		goto done;
1732 	}
1733 	spin_unlock_irqrestore(&cctx->lock, flags);
1734 
1735 	dsp_attributes = kzalloc(FASTRPC_MAX_DSP_ATTRIBUTES_LEN, GFP_KERNEL);
1736 	if (!dsp_attributes)
1737 		return -ENOMEM;
1738 
1739 	err = fastrpc_get_info_from_dsp(fl, dsp_attributes, FASTRPC_MAX_DSP_ATTRIBUTES);
1740 	if (err == DSP_UNSUPPORTED_API) {
1741 		dev_info(&cctx->rpdev->dev,
1742 			 "Warning: DSP capabilities not supported on domain: %d\n", domain);
1743 		kfree(dsp_attributes);
1744 		return -EOPNOTSUPP;
1745 	} else if (err) {
1746 		dev_err(&cctx->rpdev->dev, "Error: dsp information is incorrect err: %d\n", err);
1747 		kfree(dsp_attributes);
1748 		return err;
1749 	}
1750 
1751 	spin_lock_irqsave(&cctx->lock, flags);
1752 	memcpy(cctx->dsp_attributes, dsp_attributes, FASTRPC_MAX_DSP_ATTRIBUTES_LEN);
1753 	cctx->valid_attributes = true;
1754 	spin_unlock_irqrestore(&cctx->lock, flags);
1755 	kfree(dsp_attributes);
1756 done:
1757 	cap->capability = cctx->dsp_attributes[attribute_id];
1758 	return 0;
1759 }
1760 
fastrpc_get_dsp_info(struct fastrpc_user * fl,char __user * argp)1761 static int fastrpc_get_dsp_info(struct fastrpc_user *fl, char __user *argp)
1762 {
1763 	struct fastrpc_ioctl_capability cap = {0};
1764 	int err = 0;
1765 
1766 	if (copy_from_user(&cap, argp, sizeof(cap)))
1767 		return  -EFAULT;
1768 
1769 	cap.capability = 0;
1770 	if (cap.domain >= FASTRPC_DEV_MAX) {
1771 		dev_err(&fl->cctx->rpdev->dev, "Error: Invalid domain id:%d, err:%d\n",
1772 			cap.domain, err);
1773 		return -ECHRNG;
1774 	}
1775 
1776 	/* Fastrpc Capablities does not support modem domain */
1777 	if (cap.domain == MDSP_DOMAIN_ID) {
1778 		dev_err(&fl->cctx->rpdev->dev, "Error: modem not supported %d\n", err);
1779 		return -ECHRNG;
1780 	}
1781 
1782 	if (cap.attribute_id >= FASTRPC_MAX_DSP_ATTRIBUTES) {
1783 		dev_err(&fl->cctx->rpdev->dev, "Error: invalid attribute: %d, err: %d\n",
1784 			cap.attribute_id, err);
1785 		return -EOVERFLOW;
1786 	}
1787 
1788 	err = fastrpc_get_info_from_kernel(&cap, fl);
1789 	if (err)
1790 		return err;
1791 
1792 	if (copy_to_user(argp, &cap, sizeof(cap)))
1793 		return -EFAULT;
1794 
1795 	return 0;
1796 }
1797 
fastrpc_req_munmap_impl(struct fastrpc_user * fl,struct fastrpc_buf * buf)1798 static int fastrpc_req_munmap_impl(struct fastrpc_user *fl, struct fastrpc_buf *buf)
1799 {
1800 	struct fastrpc_invoke_args args[1] = { [0] = { 0 } };
1801 	struct fastrpc_munmap_req_msg req_msg;
1802 	struct device *dev = fl->sctx->dev;
1803 	int err;
1804 	u32 sc;
1805 
1806 	req_msg.pgid = fl->tgid;
1807 	req_msg.size = buf->size;
1808 	req_msg.vaddr = buf->raddr;
1809 
1810 	args[0].ptr = (u64) (uintptr_t) &req_msg;
1811 	args[0].length = sizeof(req_msg);
1812 
1813 	sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_MUNMAP, 1, 0);
1814 	err = fastrpc_internal_invoke(fl, true, FASTRPC_INIT_HANDLE, sc,
1815 				      &args[0]);
1816 	if (!err) {
1817 		dev_dbg(dev, "unmmap\tpt 0x%09lx OK\n", buf->raddr);
1818 		spin_lock(&fl->lock);
1819 		list_del(&buf->node);
1820 		spin_unlock(&fl->lock);
1821 		fastrpc_buf_free(buf);
1822 	} else {
1823 		dev_err(dev, "unmmap\tpt 0x%09lx ERROR\n", buf->raddr);
1824 	}
1825 
1826 	return err;
1827 }
1828 
fastrpc_req_munmap(struct fastrpc_user * fl,char __user * argp)1829 static int fastrpc_req_munmap(struct fastrpc_user *fl, char __user *argp)
1830 {
1831 	struct fastrpc_buf *buf = NULL, *iter, *b;
1832 	struct fastrpc_req_munmap req;
1833 	struct device *dev = fl->sctx->dev;
1834 
1835 	if (copy_from_user(&req, argp, sizeof(req)))
1836 		return -EFAULT;
1837 
1838 	spin_lock(&fl->lock);
1839 	list_for_each_entry_safe(iter, b, &fl->mmaps, node) {
1840 		if ((iter->raddr == req.vaddrout) && (iter->size == req.size)) {
1841 			buf = iter;
1842 			break;
1843 		}
1844 	}
1845 	spin_unlock(&fl->lock);
1846 
1847 	if (!buf) {
1848 		dev_err(dev, "mmap\t\tpt 0x%09llx [len 0x%08llx] not in list\n",
1849 			req.vaddrout, req.size);
1850 		return -EINVAL;
1851 	}
1852 
1853 	return fastrpc_req_munmap_impl(fl, buf);
1854 }
1855 
fastrpc_req_mmap(struct fastrpc_user * fl,char __user * argp)1856 static int fastrpc_req_mmap(struct fastrpc_user *fl, char __user *argp)
1857 {
1858 	struct fastrpc_invoke_args args[3] = { [0 ... 2] = { 0 } };
1859 	struct fastrpc_buf *buf = NULL;
1860 	struct fastrpc_mmap_req_msg req_msg;
1861 	struct fastrpc_mmap_rsp_msg rsp_msg;
1862 	struct fastrpc_phy_page pages;
1863 	struct fastrpc_req_mmap req;
1864 	struct device *dev = fl->sctx->dev;
1865 	int err;
1866 	u32 sc;
1867 
1868 	if (copy_from_user(&req, argp, sizeof(req)))
1869 		return -EFAULT;
1870 
1871 	if (req.flags != ADSP_MMAP_ADD_PAGES && req.flags != ADSP_MMAP_REMOTE_HEAP_ADDR) {
1872 		dev_err(dev, "flag not supported 0x%x\n", req.flags);
1873 
1874 		return -EINVAL;
1875 	}
1876 
1877 	if (req.vaddrin) {
1878 		dev_err(dev, "adding user allocated pages is not supported\n");
1879 		return -EINVAL;
1880 	}
1881 
1882 	if (req.flags == ADSP_MMAP_REMOTE_HEAP_ADDR)
1883 		err = fastrpc_remote_heap_alloc(fl, dev, req.size, &buf);
1884 	else
1885 		err = fastrpc_buf_alloc(fl, dev, req.size, &buf);
1886 
1887 	if (err) {
1888 		dev_err(dev, "failed to allocate buffer\n");
1889 		return err;
1890 	}
1891 
1892 	req_msg.pgid = fl->tgid;
1893 	req_msg.flags = req.flags;
1894 	req_msg.vaddr = req.vaddrin;
1895 	req_msg.num = sizeof(pages);
1896 
1897 	args[0].ptr = (u64) (uintptr_t) &req_msg;
1898 	args[0].length = sizeof(req_msg);
1899 
1900 	pages.addr = buf->phys;
1901 	pages.size = buf->size;
1902 
1903 	args[1].ptr = (u64) (uintptr_t) &pages;
1904 	args[1].length = sizeof(pages);
1905 
1906 	args[2].ptr = (u64) (uintptr_t) &rsp_msg;
1907 	args[2].length = sizeof(rsp_msg);
1908 
1909 	sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_MMAP, 2, 1);
1910 	err = fastrpc_internal_invoke(fl, true, FASTRPC_INIT_HANDLE, sc,
1911 				      &args[0]);
1912 	if (err) {
1913 		dev_err(dev, "mmap error (len 0x%08llx)\n", buf->size);
1914 		fastrpc_buf_free(buf);
1915 		return err;
1916 	}
1917 
1918 	/* update the buffer to be able to deallocate the memory on the DSP */
1919 	buf->raddr = (uintptr_t) rsp_msg.vaddr;
1920 
1921 	/* let the client know the address to use */
1922 	req.vaddrout = rsp_msg.vaddr;
1923 
1924 	/* Add memory to static PD pool, protection thru hypervisor */
1925 	if (req.flags == ADSP_MMAP_REMOTE_HEAP_ADDR && fl->cctx->vmcount) {
1926 		u64 src_perms = BIT(QCOM_SCM_VMID_HLOS);
1927 
1928 		err = qcom_scm_assign_mem(buf->phys, (u64)buf->size,
1929 			&src_perms, fl->cctx->vmperms, fl->cctx->vmcount);
1930 		if (err) {
1931 			dev_err(fl->sctx->dev, "Failed to assign memory phys 0x%llx size 0x%llx err %d",
1932 					buf->phys, buf->size, err);
1933 			goto err_assign;
1934 		}
1935 	}
1936 
1937 	spin_lock(&fl->lock);
1938 	list_add_tail(&buf->node, &fl->mmaps);
1939 	spin_unlock(&fl->lock);
1940 
1941 	if (copy_to_user((void __user *)argp, &req, sizeof(req))) {
1942 		err = -EFAULT;
1943 		goto err_assign;
1944 	}
1945 
1946 	dev_dbg(dev, "mmap\t\tpt 0x%09lx OK [len 0x%08llx]\n",
1947 		buf->raddr, buf->size);
1948 
1949 	return 0;
1950 
1951 err_assign:
1952 	fastrpc_req_munmap_impl(fl, buf);
1953 
1954 	return err;
1955 }
1956 
fastrpc_req_mem_unmap_impl(struct fastrpc_user * fl,struct fastrpc_mem_unmap * req)1957 static int fastrpc_req_mem_unmap_impl(struct fastrpc_user *fl, struct fastrpc_mem_unmap *req)
1958 {
1959 	struct fastrpc_invoke_args args[1] = { [0] = { 0 } };
1960 	struct fastrpc_map *map = NULL, *iter, *m;
1961 	struct fastrpc_mem_unmap_req_msg req_msg = { 0 };
1962 	int err = 0;
1963 	u32 sc;
1964 	struct device *dev = fl->sctx->dev;
1965 
1966 	spin_lock(&fl->lock);
1967 	list_for_each_entry_safe(iter, m, &fl->maps, node) {
1968 		if ((req->fd < 0 || iter->fd == req->fd) && (iter->raddr == req->vaddr)) {
1969 			map = iter;
1970 			break;
1971 		}
1972 	}
1973 
1974 	spin_unlock(&fl->lock);
1975 
1976 	if (!map) {
1977 		dev_err(dev, "map not in list\n");
1978 		return -EINVAL;
1979 	}
1980 
1981 	req_msg.pgid = fl->tgid;
1982 	req_msg.len = map->len;
1983 	req_msg.vaddrin = map->raddr;
1984 	req_msg.fd = map->fd;
1985 
1986 	args[0].ptr = (u64) (uintptr_t) &req_msg;
1987 	args[0].length = sizeof(req_msg);
1988 
1989 	sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_MEM_UNMAP, 1, 0);
1990 	err = fastrpc_internal_invoke(fl, true, FASTRPC_INIT_HANDLE, sc,
1991 				      &args[0]);
1992 	if (err) {
1993 		dev_err(dev, "unmmap\tpt fd = %d, 0x%09llx error\n",  map->fd, map->raddr);
1994 		return err;
1995 	}
1996 	fastrpc_map_put(map);
1997 
1998 	return 0;
1999 }
2000 
fastrpc_req_mem_unmap(struct fastrpc_user * fl,char __user * argp)2001 static int fastrpc_req_mem_unmap(struct fastrpc_user *fl, char __user *argp)
2002 {
2003 	struct fastrpc_mem_unmap req;
2004 
2005 	if (copy_from_user(&req, argp, sizeof(req)))
2006 		return -EFAULT;
2007 
2008 	return fastrpc_req_mem_unmap_impl(fl, &req);
2009 }
2010 
fastrpc_req_mem_map(struct fastrpc_user * fl,char __user * argp)2011 static int fastrpc_req_mem_map(struct fastrpc_user *fl, char __user *argp)
2012 {
2013 	struct fastrpc_invoke_args args[4] = { [0 ... 3] = { 0 } };
2014 	struct fastrpc_mem_map_req_msg req_msg = { 0 };
2015 	struct fastrpc_mmap_rsp_msg rsp_msg = { 0 };
2016 	struct fastrpc_mem_unmap req_unmap = { 0 };
2017 	struct fastrpc_phy_page pages = { 0 };
2018 	struct fastrpc_mem_map req;
2019 	struct device *dev = fl->sctx->dev;
2020 	struct fastrpc_map *map = NULL;
2021 	int err;
2022 	u32 sc;
2023 
2024 	if (copy_from_user(&req, argp, sizeof(req)))
2025 		return -EFAULT;
2026 
2027 	/* create SMMU mapping */
2028 	err = fastrpc_map_create(fl, req.fd, req.length, 0, &map);
2029 	if (err) {
2030 		dev_err(dev, "failed to map buffer, fd = %d\n", req.fd);
2031 		return err;
2032 	}
2033 
2034 	req_msg.pgid = fl->tgid;
2035 	req_msg.fd = req.fd;
2036 	req_msg.offset = req.offset;
2037 	req_msg.vaddrin = req.vaddrin;
2038 	map->va = (void *) (uintptr_t) req.vaddrin;
2039 	req_msg.flags = req.flags;
2040 	req_msg.num = sizeof(pages);
2041 	req_msg.data_len = 0;
2042 
2043 	args[0].ptr = (u64) (uintptr_t) &req_msg;
2044 	args[0].length = sizeof(req_msg);
2045 
2046 	pages.addr = map->phys;
2047 	pages.size = map->size;
2048 
2049 	args[1].ptr = (u64) (uintptr_t) &pages;
2050 	args[1].length = sizeof(pages);
2051 
2052 	args[2].ptr = (u64) (uintptr_t) &pages;
2053 	args[2].length = 0;
2054 
2055 	args[3].ptr = (u64) (uintptr_t) &rsp_msg;
2056 	args[3].length = sizeof(rsp_msg);
2057 
2058 	sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_MEM_MAP, 3, 1);
2059 	err = fastrpc_internal_invoke(fl, true, FASTRPC_INIT_HANDLE, sc, &args[0]);
2060 	if (err) {
2061 		dev_err(dev, "mem mmap error, fd %d, vaddr %llx, size %lld\n",
2062 			req.fd, req.vaddrin, map->size);
2063 		goto err_invoke;
2064 	}
2065 
2066 	/* update the buffer to be able to deallocate the memory on the DSP */
2067 	map->raddr = rsp_msg.vaddr;
2068 
2069 	/* let the client know the address to use */
2070 	req.vaddrout = rsp_msg.vaddr;
2071 
2072 	if (copy_to_user((void __user *)argp, &req, sizeof(req))) {
2073 		/* unmap the memory and release the buffer */
2074 		req_unmap.vaddr = (uintptr_t) rsp_msg.vaddr;
2075 		req_unmap.length = map->size;
2076 		fastrpc_req_mem_unmap_impl(fl, &req_unmap);
2077 		return -EFAULT;
2078 	}
2079 
2080 	return 0;
2081 
2082 err_invoke:
2083 	fastrpc_map_put(map);
2084 
2085 	return err;
2086 }
2087 
fastrpc_device_ioctl(struct file * file,unsigned int cmd,unsigned long arg)2088 static long fastrpc_device_ioctl(struct file *file, unsigned int cmd,
2089 				 unsigned long arg)
2090 {
2091 	struct fastrpc_user *fl = (struct fastrpc_user *)file->private_data;
2092 	char __user *argp = (char __user *)arg;
2093 	int err;
2094 
2095 	switch (cmd) {
2096 	case FASTRPC_IOCTL_INVOKE:
2097 		err = fastrpc_invoke(fl, argp);
2098 		break;
2099 	case FASTRPC_IOCTL_INIT_ATTACH:
2100 		err = fastrpc_init_attach(fl, ROOT_PD);
2101 		break;
2102 	case FASTRPC_IOCTL_INIT_ATTACH_SNS:
2103 		err = fastrpc_init_attach(fl, SENSORS_PD);
2104 		break;
2105 	case FASTRPC_IOCTL_INIT_CREATE_STATIC:
2106 		err = fastrpc_init_create_static_process(fl, argp);
2107 		break;
2108 	case FASTRPC_IOCTL_INIT_CREATE:
2109 		err = fastrpc_init_create_process(fl, argp);
2110 		break;
2111 	case FASTRPC_IOCTL_ALLOC_DMA_BUFF:
2112 		err = fastrpc_dmabuf_alloc(fl, argp);
2113 		break;
2114 	case FASTRPC_IOCTL_MMAP:
2115 		err = fastrpc_req_mmap(fl, argp);
2116 		break;
2117 	case FASTRPC_IOCTL_MUNMAP:
2118 		err = fastrpc_req_munmap(fl, argp);
2119 		break;
2120 	case FASTRPC_IOCTL_MEM_MAP:
2121 		err = fastrpc_req_mem_map(fl, argp);
2122 		break;
2123 	case FASTRPC_IOCTL_MEM_UNMAP:
2124 		err = fastrpc_req_mem_unmap(fl, argp);
2125 		break;
2126 	case FASTRPC_IOCTL_GET_DSP_INFO:
2127 		err = fastrpc_get_dsp_info(fl, argp);
2128 		break;
2129 	default:
2130 		err = -ENOTTY;
2131 		break;
2132 	}
2133 
2134 	return err;
2135 }
2136 
2137 static const struct file_operations fastrpc_fops = {
2138 	.open = fastrpc_device_open,
2139 	.release = fastrpc_device_release,
2140 	.unlocked_ioctl = fastrpc_device_ioctl,
2141 	.compat_ioctl = fastrpc_device_ioctl,
2142 };
2143 
fastrpc_cb_probe(struct platform_device * pdev)2144 static int fastrpc_cb_probe(struct platform_device *pdev)
2145 {
2146 	struct fastrpc_channel_ctx *cctx;
2147 	struct fastrpc_session_ctx *sess;
2148 	struct device *dev = &pdev->dev;
2149 	int i, sessions = 0;
2150 	unsigned long flags;
2151 	int rc;
2152 
2153 	cctx = dev_get_drvdata(dev->parent);
2154 	if (!cctx)
2155 		return -EINVAL;
2156 
2157 	of_property_read_u32(dev->of_node, "qcom,nsessions", &sessions);
2158 
2159 	spin_lock_irqsave(&cctx->lock, flags);
2160 	if (cctx->sesscount >= FASTRPC_MAX_SESSIONS) {
2161 		dev_err(&pdev->dev, "too many sessions\n");
2162 		spin_unlock_irqrestore(&cctx->lock, flags);
2163 		return -ENOSPC;
2164 	}
2165 	sess = &cctx->session[cctx->sesscount++];
2166 	sess->used = false;
2167 	sess->valid = true;
2168 	sess->dev = dev;
2169 	dev_set_drvdata(dev, sess);
2170 
2171 	if (of_property_read_u32(dev->of_node, "reg", &sess->sid))
2172 		dev_info(dev, "FastRPC Session ID not specified in DT\n");
2173 
2174 	if (sessions > 0) {
2175 		struct fastrpc_session_ctx *dup_sess;
2176 
2177 		for (i = 1; i < sessions; i++) {
2178 			if (cctx->sesscount >= FASTRPC_MAX_SESSIONS)
2179 				break;
2180 			dup_sess = &cctx->session[cctx->sesscount++];
2181 			memcpy(dup_sess, sess, sizeof(*dup_sess));
2182 		}
2183 	}
2184 	spin_unlock_irqrestore(&cctx->lock, flags);
2185 	rc = dma_set_mask(dev, DMA_BIT_MASK(32));
2186 	if (rc) {
2187 		dev_err(dev, "32-bit DMA enable failed\n");
2188 		return rc;
2189 	}
2190 
2191 	return 0;
2192 }
2193 
fastrpc_cb_remove(struct platform_device * pdev)2194 static void fastrpc_cb_remove(struct platform_device *pdev)
2195 {
2196 	struct fastrpc_channel_ctx *cctx = dev_get_drvdata(pdev->dev.parent);
2197 	struct fastrpc_session_ctx *sess = dev_get_drvdata(&pdev->dev);
2198 	unsigned long flags;
2199 	int i;
2200 
2201 	spin_lock_irqsave(&cctx->lock, flags);
2202 	for (i = 0; i < FASTRPC_MAX_SESSIONS; i++) {
2203 		if (cctx->session[i].sid == sess->sid) {
2204 			cctx->session[i].valid = false;
2205 			cctx->sesscount--;
2206 		}
2207 	}
2208 	spin_unlock_irqrestore(&cctx->lock, flags);
2209 }
2210 
2211 static const struct of_device_id fastrpc_match_table[] = {
2212 	{ .compatible = "qcom,fastrpc-compute-cb", },
2213 	{}
2214 };
2215 
2216 static struct platform_driver fastrpc_cb_driver = {
2217 	.probe = fastrpc_cb_probe,
2218 	.remove_new = fastrpc_cb_remove,
2219 	.driver = {
2220 		.name = "qcom,fastrpc-cb",
2221 		.of_match_table = fastrpc_match_table,
2222 		.suppress_bind_attrs = true,
2223 	},
2224 };
2225 
fastrpc_device_register(struct device * dev,struct fastrpc_channel_ctx * cctx,bool is_secured,const char * domain)2226 static int fastrpc_device_register(struct device *dev, struct fastrpc_channel_ctx *cctx,
2227 				   bool is_secured, const char *domain)
2228 {
2229 	struct fastrpc_device *fdev;
2230 	int err;
2231 
2232 	fdev = devm_kzalloc(dev, sizeof(*fdev), GFP_KERNEL);
2233 	if (!fdev)
2234 		return -ENOMEM;
2235 
2236 	fdev->secure = is_secured;
2237 	fdev->cctx = cctx;
2238 	fdev->miscdev.minor = MISC_DYNAMIC_MINOR;
2239 	fdev->miscdev.fops = &fastrpc_fops;
2240 	fdev->miscdev.name = devm_kasprintf(dev, GFP_KERNEL, "fastrpc-%s%s",
2241 					    domain, is_secured ? "-secure" : "");
2242 	if (!fdev->miscdev.name)
2243 		return -ENOMEM;
2244 
2245 	err = misc_register(&fdev->miscdev);
2246 	if (!err) {
2247 		if (is_secured)
2248 			cctx->secure_fdevice = fdev;
2249 		else
2250 			cctx->fdevice = fdev;
2251 	}
2252 
2253 	return err;
2254 }
2255 
fastrpc_rpmsg_probe(struct rpmsg_device * rpdev)2256 static int fastrpc_rpmsg_probe(struct rpmsg_device *rpdev)
2257 {
2258 	struct device *rdev = &rpdev->dev;
2259 	struct fastrpc_channel_ctx *data;
2260 	int i, err, domain_id = -1, vmcount;
2261 	const char *domain;
2262 	bool secure_dsp;
2263 	struct device_node *rmem_node;
2264 	struct reserved_mem *rmem;
2265 	unsigned int vmids[FASTRPC_MAX_VMIDS];
2266 
2267 	err = of_property_read_string(rdev->of_node, "label", &domain);
2268 	if (err) {
2269 		dev_info(rdev, "FastRPC Domain not specified in DT\n");
2270 		return err;
2271 	}
2272 
2273 	for (i = 0; i < FASTRPC_DEV_MAX; i++) {
2274 		if (!strcmp(domains[i], domain)) {
2275 			domain_id = i;
2276 			break;
2277 		}
2278 	}
2279 
2280 	if (domain_id < 0) {
2281 		dev_info(rdev, "FastRPC Invalid Domain ID %d\n", domain_id);
2282 		return -EINVAL;
2283 	}
2284 
2285 	if (of_reserved_mem_device_init_by_idx(rdev, rdev->of_node, 0))
2286 		dev_info(rdev, "no reserved DMA memory for FASTRPC\n");
2287 
2288 	vmcount = of_property_read_variable_u32_array(rdev->of_node,
2289 				"qcom,vmids", &vmids[0], 0, FASTRPC_MAX_VMIDS);
2290 	if (vmcount < 0)
2291 		vmcount = 0;
2292 	else if (!qcom_scm_is_available())
2293 		return -EPROBE_DEFER;
2294 
2295 	data = kzalloc(sizeof(*data), GFP_KERNEL);
2296 	if (!data)
2297 		return -ENOMEM;
2298 
2299 	if (vmcount) {
2300 		data->vmcount = vmcount;
2301 		for (i = 0; i < data->vmcount; i++) {
2302 			data->vmperms[i].vmid = vmids[i];
2303 			data->vmperms[i].perm = QCOM_SCM_PERM_RWX;
2304 		}
2305 	}
2306 
2307 	rmem_node = of_parse_phandle(rdev->of_node, "memory-region", 0);
2308 	if (domain_id == SDSP_DOMAIN_ID && rmem_node) {
2309 		u64 src_perms;
2310 
2311 		rmem = of_reserved_mem_lookup(rmem_node);
2312 		if (!rmem) {
2313 			err = -EINVAL;
2314 			goto fdev_error;
2315 		}
2316 
2317 		src_perms = BIT(QCOM_SCM_VMID_HLOS);
2318 
2319 		qcom_scm_assign_mem(rmem->base, rmem->size, &src_perms,
2320 				    data->vmperms, data->vmcount);
2321 
2322 	}
2323 
2324 	secure_dsp = !(of_property_read_bool(rdev->of_node, "qcom,non-secure-domain"));
2325 	data->secure = secure_dsp;
2326 
2327 	switch (domain_id) {
2328 	case ADSP_DOMAIN_ID:
2329 	case MDSP_DOMAIN_ID:
2330 	case SDSP_DOMAIN_ID:
2331 		/* Unsigned PD offloading is only supported on CDSP and CDSP1 */
2332 		data->unsigned_support = false;
2333 		err = fastrpc_device_register(rdev, data, secure_dsp, domains[domain_id]);
2334 		if (err)
2335 			goto fdev_error;
2336 		break;
2337 	case CDSP_DOMAIN_ID:
2338 	case CDSP1_DOMAIN_ID:
2339 		data->unsigned_support = true;
2340 		/* Create both device nodes so that we can allow both Signed and Unsigned PD */
2341 		err = fastrpc_device_register(rdev, data, true, domains[domain_id]);
2342 		if (err)
2343 			goto fdev_error;
2344 
2345 		err = fastrpc_device_register(rdev, data, false, domains[domain_id]);
2346 		if (err)
2347 			goto fdev_error;
2348 		break;
2349 	default:
2350 		err = -EINVAL;
2351 		goto fdev_error;
2352 	}
2353 
2354 	kref_init(&data->refcount);
2355 
2356 	dev_set_drvdata(&rpdev->dev, data);
2357 	rdev->dma_mask = &data->dma_mask;
2358 	dma_set_mask_and_coherent(rdev, DMA_BIT_MASK(32));
2359 	INIT_LIST_HEAD(&data->users);
2360 	INIT_LIST_HEAD(&data->invoke_interrupted_mmaps);
2361 	spin_lock_init(&data->lock);
2362 	idr_init(&data->ctx_idr);
2363 	data->domain_id = domain_id;
2364 	data->rpdev = rpdev;
2365 
2366 	err = of_platform_populate(rdev->of_node, NULL, NULL, rdev);
2367 	if (err)
2368 		goto populate_error;
2369 
2370 	return 0;
2371 
2372 populate_error:
2373 	if (data->fdevice)
2374 		misc_deregister(&data->fdevice->miscdev);
2375 	if (data->secure_fdevice)
2376 		misc_deregister(&data->secure_fdevice->miscdev);
2377 
2378 fdev_error:
2379 	kfree(data);
2380 	return err;
2381 }
2382 
fastrpc_notify_users(struct fastrpc_user * user)2383 static void fastrpc_notify_users(struct fastrpc_user *user)
2384 {
2385 	struct fastrpc_invoke_ctx *ctx;
2386 
2387 	spin_lock(&user->lock);
2388 	list_for_each_entry(ctx, &user->pending, node) {
2389 		ctx->retval = -EPIPE;
2390 		complete(&ctx->work);
2391 	}
2392 	spin_unlock(&user->lock);
2393 }
2394 
fastrpc_rpmsg_remove(struct rpmsg_device * rpdev)2395 static void fastrpc_rpmsg_remove(struct rpmsg_device *rpdev)
2396 {
2397 	struct fastrpc_channel_ctx *cctx = dev_get_drvdata(&rpdev->dev);
2398 	struct fastrpc_buf *buf, *b;
2399 	struct fastrpc_user *user;
2400 	unsigned long flags;
2401 
2402 	/* No invocations past this point */
2403 	spin_lock_irqsave(&cctx->lock, flags);
2404 	cctx->rpdev = NULL;
2405 	list_for_each_entry(user, &cctx->users, user)
2406 		fastrpc_notify_users(user);
2407 	spin_unlock_irqrestore(&cctx->lock, flags);
2408 
2409 	if (cctx->fdevice)
2410 		misc_deregister(&cctx->fdevice->miscdev);
2411 
2412 	if (cctx->secure_fdevice)
2413 		misc_deregister(&cctx->secure_fdevice->miscdev);
2414 
2415 	list_for_each_entry_safe(buf, b, &cctx->invoke_interrupted_mmaps, node)
2416 		list_del(&buf->node);
2417 
2418 	if (cctx->remote_heap)
2419 		fastrpc_buf_free(cctx->remote_heap);
2420 
2421 	of_platform_depopulate(&rpdev->dev);
2422 
2423 	fastrpc_channel_ctx_put(cctx);
2424 }
2425 
fastrpc_rpmsg_callback(struct rpmsg_device * rpdev,void * data,int len,void * priv,u32 addr)2426 static int fastrpc_rpmsg_callback(struct rpmsg_device *rpdev, void *data,
2427 				  int len, void *priv, u32 addr)
2428 {
2429 	struct fastrpc_channel_ctx *cctx = dev_get_drvdata(&rpdev->dev);
2430 	struct fastrpc_invoke_rsp *rsp = data;
2431 	struct fastrpc_invoke_ctx *ctx;
2432 	unsigned long flags;
2433 	unsigned long ctxid;
2434 
2435 	if (len < sizeof(*rsp))
2436 		return -EINVAL;
2437 
2438 	ctxid = ((rsp->ctx & FASTRPC_CTXID_MASK) >> 4);
2439 
2440 	spin_lock_irqsave(&cctx->lock, flags);
2441 	ctx = idr_find(&cctx->ctx_idr, ctxid);
2442 	spin_unlock_irqrestore(&cctx->lock, flags);
2443 
2444 	if (!ctx) {
2445 		dev_err(&rpdev->dev, "No context ID matches response\n");
2446 		return -ENOENT;
2447 	}
2448 
2449 	ctx->retval = rsp->retval;
2450 	complete(&ctx->work);
2451 
2452 	/*
2453 	 * The DMA buffer associated with the context cannot be freed in
2454 	 * interrupt context so schedule it through a worker thread to
2455 	 * avoid a kernel BUG.
2456 	 */
2457 	schedule_work(&ctx->put_work);
2458 
2459 	return 0;
2460 }
2461 
2462 static const struct of_device_id fastrpc_rpmsg_of_match[] = {
2463 	{ .compatible = "qcom,fastrpc" },
2464 	{ },
2465 };
2466 MODULE_DEVICE_TABLE(of, fastrpc_rpmsg_of_match);
2467 
2468 static struct rpmsg_driver fastrpc_driver = {
2469 	.probe = fastrpc_rpmsg_probe,
2470 	.remove = fastrpc_rpmsg_remove,
2471 	.callback = fastrpc_rpmsg_callback,
2472 	.drv = {
2473 		.name = "qcom,fastrpc",
2474 		.of_match_table = fastrpc_rpmsg_of_match,
2475 	},
2476 };
2477 
fastrpc_init(void)2478 static int fastrpc_init(void)
2479 {
2480 	int ret;
2481 
2482 	ret = platform_driver_register(&fastrpc_cb_driver);
2483 	if (ret < 0) {
2484 		pr_err("fastrpc: failed to register cb driver\n");
2485 		return ret;
2486 	}
2487 
2488 	ret = register_rpmsg_driver(&fastrpc_driver);
2489 	if (ret < 0) {
2490 		pr_err("fastrpc: failed to register rpmsg driver\n");
2491 		platform_driver_unregister(&fastrpc_cb_driver);
2492 		return ret;
2493 	}
2494 
2495 	return 0;
2496 }
2497 module_init(fastrpc_init);
2498 
fastrpc_exit(void)2499 static void fastrpc_exit(void)
2500 {
2501 	platform_driver_unregister(&fastrpc_cb_driver);
2502 	unregister_rpmsg_driver(&fastrpc_driver);
2503 }
2504 module_exit(fastrpc_exit);
2505 
2506 MODULE_DESCRIPTION("Qualcomm FastRPC");
2507 MODULE_LICENSE("GPL v2");
2508 MODULE_IMPORT_NS(DMA_BUF);
2509