fastrpc.c (7f1f481263c3ce5387d4fd5ad63ddaa8a295aab2) fastrpc.c (e90d911906196bf987492c94e38f10ca611dfd7b)
1// SPDX-License-Identifier: GPL-2.0
2// Copyright (c) 2011-2018, The Linux Foundation. All rights reserved.
3// Copyright (c) 2018, Linaro Limited
4
5#include <linux/completion.h>
6#include <linux/device.h>
7#include <linux/dma-buf.h>
8#include <linux/dma-mapping.h>
9#include <linux/idr.h>
10#include <linux/list.h>
11#include <linux/miscdevice.h>
12#include <linux/module.h>
13#include <linux/of_address.h>
14#include <linux/of.h>
15#include <linux/sort.h>
16#include <linux/of_platform.h>
17#include <linux/rpmsg.h>
18#include <linux/scatterlist.h>
19#include <linux/slab.h>
1// SPDX-License-Identifier: GPL-2.0
2// Copyright (c) 2011-2018, The Linux Foundation. All rights reserved.
3// Copyright (c) 2018, Linaro Limited
4
5#include <linux/completion.h>
6#include <linux/device.h>
7#include <linux/dma-buf.h>
8#include <linux/dma-mapping.h>
9#include <linux/idr.h>
10#include <linux/list.h>
11#include <linux/miscdevice.h>
12#include <linux/module.h>
13#include <linux/of_address.h>
14#include <linux/of.h>
15#include <linux/sort.h>
16#include <linux/of_platform.h>
17#include <linux/rpmsg.h>
18#include <linux/scatterlist.h>
19#include <linux/slab.h>
20#include <linux/qcom_scm.h>
20#include <uapi/misc/fastrpc.h>
21
22#define ADSP_DOMAIN_ID (0)
23#define MDSP_DOMAIN_ID (1)
24#define SDSP_DOMAIN_ID (2)
25#define CDSP_DOMAIN_ID (3)
26#define FASTRPC_DEV_MAX 4 /* adsp, mdsp, slpi, cdsp*/
27#define FASTRPC_MAX_SESSIONS 13 /*12 compute, 1 cpz*/
21#include <uapi/misc/fastrpc.h>
22
23#define ADSP_DOMAIN_ID (0)
24#define MDSP_DOMAIN_ID (1)
25#define SDSP_DOMAIN_ID (2)
26#define CDSP_DOMAIN_ID (3)
27#define FASTRPC_DEV_MAX 4 /* adsp, mdsp, slpi, cdsp*/
28#define FASTRPC_MAX_SESSIONS 13 /*12 compute, 1 cpz*/
29#define FASTRPC_MAX_VMIDS 16
28#define FASTRPC_ALIGN 128
29#define FASTRPC_MAX_FDLIST 16
30#define FASTRPC_MAX_CRCLIST 64
31#define FASTRPC_PHYS(p) ((p) & 0xffffffff)
32#define FASTRPC_CTX_MAX (256)
33#define FASTRPC_INIT_HANDLE 1
34#define FASTRPC_DSP_UTILITIES_HANDLE 2
35#define FASTRPC_CTXID_MASK (0xFF0)

--- 154 unchanged lines hidden (view full) ---

190 struct dma_buf *buf;
191 struct sg_table *table;
192 struct dma_buf_attachment *attach;
193 u64 phys;
194 u64 size;
195 void *va;
196 u64 len;
197 u64 raddr;
30#define FASTRPC_ALIGN 128
31#define FASTRPC_MAX_FDLIST 16
32#define FASTRPC_MAX_CRCLIST 64
33#define FASTRPC_PHYS(p) ((p) & 0xffffffff)
34#define FASTRPC_CTX_MAX (256)
35#define FASTRPC_INIT_HANDLE 1
36#define FASTRPC_DSP_UTILITIES_HANDLE 2
37#define FASTRPC_CTXID_MASK (0xFF0)

--- 154 unchanged lines hidden (view full) ---

192 struct dma_buf *buf;
193 struct sg_table *table;
194 struct dma_buf_attachment *attach;
195 u64 phys;
196 u64 size;
197 void *va;
198 u64 len;
199 u64 raddr;
200 u32 attr;
198 struct kref refcount;
199};
200
201struct fastrpc_invoke_ctx {
202 int nscalars;
203 int nbufs;
204 int retval;
205 int pid;

--- 21 unchanged lines hidden (view full) ---

227 int sid;
228 bool used;
229 bool valid;
230};
231
232struct fastrpc_channel_ctx {
233 int domain_id;
234 int sesscount;
201 struct kref refcount;
202};
203
204struct fastrpc_invoke_ctx {
205 int nscalars;
206 int nbufs;
207 int retval;
208 int pid;

--- 21 unchanged lines hidden (view full) ---

230 int sid;
231 bool used;
232 bool valid;
233};
234
235struct fastrpc_channel_ctx {
236 int domain_id;
237 int sesscount;
238 int vmcount;
239 u32 perms;
240 struct qcom_scm_vmperm vmperms[FASTRPC_MAX_VMIDS];
235 struct rpmsg_device *rpdev;
236 struct fastrpc_session_ctx session[FASTRPC_MAX_SESSIONS];
237 spinlock_t lock;
238 struct idr ctx_idr;
239 struct list_head users;
240 struct kref refcount;
241 /* Flag if dsp attributes are cached */
242 bool valid_attributes;

--- 31 unchanged lines hidden (view full) ---

274
275static void fastrpc_free_map(struct kref *ref)
276{
277 struct fastrpc_map *map;
278
279 map = container_of(ref, struct fastrpc_map, refcount);
280
281 if (map->table) {
241 struct rpmsg_device *rpdev;
242 struct fastrpc_session_ctx session[FASTRPC_MAX_SESSIONS];
243 spinlock_t lock;
244 struct idr ctx_idr;
245 struct list_head users;
246 struct kref refcount;
247 /* Flag if dsp attributes are cached */
248 bool valid_attributes;

--- 31 unchanged lines hidden (view full) ---

280
281static void fastrpc_free_map(struct kref *ref)
282{
283 struct fastrpc_map *map;
284
285 map = container_of(ref, struct fastrpc_map, refcount);
286
287 if (map->table) {
288 if (map->attr & FASTRPC_ATTR_SECUREMAP) {
289 struct qcom_scm_vmperm perm;
290 int err = 0;
291
292 perm.vmid = QCOM_SCM_VMID_HLOS;
293 perm.perm = QCOM_SCM_PERM_RWX;
294 err = qcom_scm_assign_mem(map->phys, map->size,
295 &(map->fl->cctx->vmperms[0].vmid), &perm, 1);
296 if (err) {
297 dev_err(map->fl->sctx->dev, "Failed to assign memory phys 0x%llx size 0x%llx err %d",
298 map->phys, map->size, err);
299 return;
300 }
301 }
282 dma_buf_unmap_attachment(map->attach, map->table,
283 DMA_BIDIRECTIONAL);
284 dma_buf_detach(map->buf, map->attach);
285 dma_buf_put(map->buf);
286 }
287
288 kfree(map);
289}

--- 360 unchanged lines hidden (view full) ---

650 .map_dma_buf = fastrpc_map_dma_buf,
651 .unmap_dma_buf = fastrpc_unmap_dma_buf,
652 .mmap = fastrpc_mmap,
653 .vmap = fastrpc_vmap,
654 .release = fastrpc_release,
655};
656
657static int fastrpc_map_create(struct fastrpc_user *fl, int fd,
302 dma_buf_unmap_attachment(map->attach, map->table,
303 DMA_BIDIRECTIONAL);
304 dma_buf_detach(map->buf, map->attach);
305 dma_buf_put(map->buf);
306 }
307
308 kfree(map);
309}

--- 360 unchanged lines hidden (view full) ---

670 .map_dma_buf = fastrpc_map_dma_buf,
671 .unmap_dma_buf = fastrpc_unmap_dma_buf,
672 .mmap = fastrpc_mmap,
673 .vmap = fastrpc_vmap,
674 .release = fastrpc_release,
675};
676
677static int fastrpc_map_create(struct fastrpc_user *fl, int fd,
658 u64 len, struct fastrpc_map **ppmap)
678 u64 len, u32 attr, struct fastrpc_map **ppmap)
659{
660 struct fastrpc_session_ctx *sess = fl->sctx;
661 struct fastrpc_map *map = NULL;
662 int err = 0;
663
664 if (!fastrpc_map_find(fl, fd, ppmap))
665 return 0;
666

--- 25 unchanged lines hidden (view full) ---

692
693 map->phys = sg_dma_address(map->table->sgl);
694 map->phys += ((u64)fl->sctx->sid << 32);
695 map->size = len;
696 map->va = sg_virt(map->table->sgl);
697 map->len = len;
698 kref_init(&map->refcount);
699
679{
680 struct fastrpc_session_ctx *sess = fl->sctx;
681 struct fastrpc_map *map = NULL;
682 int err = 0;
683
684 if (!fastrpc_map_find(fl, fd, ppmap))
685 return 0;
686

--- 25 unchanged lines hidden (view full) ---

712
713 map->phys = sg_dma_address(map->table->sgl);
714 map->phys += ((u64)fl->sctx->sid << 32);
715 map->size = len;
716 map->va = sg_virt(map->table->sgl);
717 map->len = len;
718 kref_init(&map->refcount);
719
720 if (attr & FASTRPC_ATTR_SECUREMAP) {
721 /*
722 * If subsystem VMIDs are defined in DTSI, then do
723 * hyp_assign from HLOS to those VM(s)
724 */
725 unsigned int perms = BIT(QCOM_SCM_VMID_HLOS);
726
727 map->attr = attr;
728 err = qcom_scm_assign_mem(map->phys, (u64)map->size, &perms,
729 fl->cctx->vmperms, fl->cctx->vmcount);
730 if (err) {
731 dev_err(sess->dev, "Failed to assign memory with phys 0x%llx size 0x%llx err %d",
732 map->phys, map->size, err);
733 goto map_err;
734 }
735 }
700 spin_lock(&fl->lock);
701 list_add_tail(&map->node, &fl->maps);
702 spin_unlock(&fl->lock);
703 *ppmap = map;
704
705 return 0;
706
707map_err:

--- 68 unchanged lines hidden (view full) ---

776}
777
778static int fastrpc_create_maps(struct fastrpc_invoke_ctx *ctx)
779{
780 struct device *dev = ctx->fl->sctx->dev;
781 int i, err;
782
783 for (i = 0; i < ctx->nscalars; ++i) {
736 spin_lock(&fl->lock);
737 list_add_tail(&map->node, &fl->maps);
738 spin_unlock(&fl->lock);
739 *ppmap = map;
740
741 return 0;
742
743map_err:

--- 68 unchanged lines hidden (view full) ---

812}
813
814static int fastrpc_create_maps(struct fastrpc_invoke_ctx *ctx)
815{
816 struct device *dev = ctx->fl->sctx->dev;
817 int i, err;
818
819 for (i = 0; i < ctx->nscalars; ++i) {
784 /* Make sure reserved field is set to 0 */
785 if (ctx->args[i].reserved)
786 return -EINVAL;
787
788 if (ctx->args[i].fd == 0 || ctx->args[i].fd == -1 ||
789 ctx->args[i].length == 0)
790 continue;
791
792 err = fastrpc_map_create(ctx->fl, ctx->args[i].fd,
820
821 if (ctx->args[i].fd == 0 || ctx->args[i].fd == -1 ||
822 ctx->args[i].length == 0)
823 continue;
824
825 err = fastrpc_map_create(ctx->fl, ctx->args[i].fd,
793 ctx->args[i].length, &ctx->maps[i]);
826 ctx->args[i].length, ctx->args[i].attr, &ctx->maps[i]);
794 if (err) {
795 dev_err(dev, "Error Creating map %d\n", err);
796 return -EINVAL;
797 }
798
799 }
800 return 0;
801}

--- 317 unchanged lines hidden (view full) ---

1119 inbuf.namelen = strlen(current->comm) + 1;
1120 inbuf.filelen = init.filelen;
1121 inbuf.pageslen = 1;
1122 inbuf.attrs = init.attrs;
1123 inbuf.siglen = init.siglen;
1124 fl->pd = USER_PD;
1125
1126 if (init.filelen && init.filefd) {
827 if (err) {
828 dev_err(dev, "Error Creating map %d\n", err);
829 return -EINVAL;
830 }
831
832 }
833 return 0;
834}

--- 317 unchanged lines hidden (view full) ---

1152 inbuf.namelen = strlen(current->comm) + 1;
1153 inbuf.filelen = init.filelen;
1154 inbuf.pageslen = 1;
1155 inbuf.attrs = init.attrs;
1156 inbuf.siglen = init.siglen;
1157 fl->pd = USER_PD;
1158
1159 if (init.filelen && init.filefd) {
1127 err = fastrpc_map_create(fl, init.filefd, init.filelen, &map);
1160 err = fastrpc_map_create(fl, init.filefd, init.filelen, 0, &map);
1128 if (err)
1129 goto err;
1130 }
1131
1132 memlen = ALIGN(max(INIT_FILELEN_MAX, (int)init.filelen * 4),
1133 1024 * 1024);
1134 err = fastrpc_buf_alloc(fl, fl->sctx->dev, memlen,
1135 &imem);

--- 92 unchanged lines hidden (view full) ---

1228 struct fastrpc_invoke_args args[1];
1229 int tgid = 0;
1230 u32 sc;
1231
1232 tgid = fl->tgid;
1233 args[0].ptr = (u64)(uintptr_t) &tgid;
1234 args[0].length = sizeof(tgid);
1235 args[0].fd = -1;
1161 if (err)
1162 goto err;
1163 }
1164
1165 memlen = ALIGN(max(INIT_FILELEN_MAX, (int)init.filelen * 4),
1166 1024 * 1024);
1167 err = fastrpc_buf_alloc(fl, fl->sctx->dev, memlen,
1168 &imem);

--- 92 unchanged lines hidden (view full) ---

1261 struct fastrpc_invoke_args args[1];
1262 int tgid = 0;
1263 u32 sc;
1264
1265 tgid = fl->tgid;
1266 args[0].ptr = (u64)(uintptr_t) &tgid;
1267 args[0].length = sizeof(tgid);
1268 args[0].fd = -1;
1236 args[0].reserved = 0;
1237 sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_RELEASE, 1, 0);
1238
1239 return fastrpc_internal_invoke(fl, true, FASTRPC_INIT_HANDLE,
1240 sc, &args[0]);
1241}
1242
1243static int fastrpc_device_release(struct inode *inode, struct file *file)
1244{

--- 131 unchanged lines hidden (view full) ---

1376{
1377 struct fastrpc_invoke_args args[1];
1378 int tgid = fl->tgid;
1379 u32 sc;
1380
1381 args[0].ptr = (u64)(uintptr_t) &tgid;
1382 args[0].length = sizeof(tgid);
1383 args[0].fd = -1;
1269 sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_RELEASE, 1, 0);
1270
1271 return fastrpc_internal_invoke(fl, true, FASTRPC_INIT_HANDLE,
1272 sc, &args[0]);
1273}
1274
1275static int fastrpc_device_release(struct inode *inode, struct file *file)
1276{

--- 131 unchanged lines hidden (view full) ---

1408{
1409 struct fastrpc_invoke_args args[1];
1410 int tgid = fl->tgid;
1411 u32 sc;
1412
1413 args[0].ptr = (u64)(uintptr_t) &tgid;
1414 args[0].length = sizeof(tgid);
1415 args[0].fd = -1;
1384 args[0].reserved = 0;
1385 sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_ATTACH, 1, 0);
1386 fl->pd = pd;
1387
1388 return fastrpc_internal_invoke(fl, true, FASTRPC_INIT_HANDLE,
1389 sc, &args[0]);
1390}
1391
1392static int fastrpc_invoke(struct fastrpc_user *fl, char __user *argp)

--- 556 unchanged lines hidden (view full) ---

1949
1950 return err;
1951}
1952
1953static int fastrpc_rpmsg_probe(struct rpmsg_device *rpdev)
1954{
1955 struct device *rdev = &rpdev->dev;
1956 struct fastrpc_channel_ctx *data;
1416 sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_ATTACH, 1, 0);
1417 fl->pd = pd;
1418
1419 return fastrpc_internal_invoke(fl, true, FASTRPC_INIT_HANDLE,
1420 sc, &args[0]);
1421}
1422
1423static int fastrpc_invoke(struct fastrpc_user *fl, char __user *argp)

--- 556 unchanged lines hidden (view full) ---

1980
1981 return err;
1982}
1983
1984static int fastrpc_rpmsg_probe(struct rpmsg_device *rpdev)
1985{
1986 struct device *rdev = &rpdev->dev;
1987 struct fastrpc_channel_ctx *data;
1957 int i, err, domain_id = -1;
1988 int i, err, domain_id = -1, vmcount;
1958 const char *domain;
1959 bool secure_dsp;
1989 const char *domain;
1990 bool secure_dsp;
1991 unsigned int vmids[FASTRPC_MAX_VMIDS];
1960
1961 err = of_property_read_string(rdev->of_node, "label", &domain);
1962 if (err) {
1963 dev_info(rdev, "FastRPC Domain not specified in DT\n");
1964 return err;
1965 }
1966
1967 for (i = 0; i <= CDSP_DOMAIN_ID; i++) {
1968 if (!strcmp(domains[i], domain)) {
1969 domain_id = i;
1970 break;
1971 }
1972 }
1973
1974 if (domain_id < 0) {
1975 dev_info(rdev, "FastRPC Invalid Domain ID %d\n", domain_id);
1976 return -EINVAL;
1977 }
1978
1992
1993 err = of_property_read_string(rdev->of_node, "label", &domain);
1994 if (err) {
1995 dev_info(rdev, "FastRPC Domain not specified in DT\n");
1996 return err;
1997 }
1998
1999 for (i = 0; i <= CDSP_DOMAIN_ID; i++) {
2000 if (!strcmp(domains[i], domain)) {
2001 domain_id = i;
2002 break;
2003 }
2004 }
2005
2006 if (domain_id < 0) {
2007 dev_info(rdev, "FastRPC Invalid Domain ID %d\n", domain_id);
2008 return -EINVAL;
2009 }
2010
2011 vmcount = of_property_read_variable_u32_array(rdev->of_node,
2012 "qcom,vmids", &vmids[0], 0, FASTRPC_MAX_VMIDS);
2013 if (vmcount < 0)
2014 vmcount = 0;
2015 else if (!qcom_scm_is_available())
2016 return -EPROBE_DEFER;
2017
1979 data = kzalloc(sizeof(*data), GFP_KERNEL);
1980 if (!data)
1981 return -ENOMEM;
1982
2018 data = kzalloc(sizeof(*data), GFP_KERNEL);
2019 if (!data)
2020 return -ENOMEM;
2021
2022 if (vmcount) {
2023 data->vmcount = vmcount;
2024 data->perms = BIT(QCOM_SCM_VMID_HLOS);
2025 for (i = 0; i < data->vmcount; i++) {
2026 data->vmperms[i].vmid = vmids[i];
2027 data->vmperms[i].perm = QCOM_SCM_PERM_RWX;
2028 }
2029 }
1983
1984 secure_dsp = !(of_property_read_bool(rdev->of_node, "qcom,non-secure-domain"));
1985 data->secure = secure_dsp;
1986
1987 switch (domain_id) {
1988 case ADSP_DOMAIN_ID:
1989 case MDSP_DOMAIN_ID:
1990 case SDSP_DOMAIN_ID:

--- 153 unchanged lines hidden --- 		2030
2031 secure_dsp = !(of_property_read_bool(rdev->of_node, "qcom,non-secure-domain"));
2032 data->secure = secure_dsp;
2033
2034 switch (domain_id) {
2035 case ADSP_DOMAIN_ID:
2036 case MDSP_DOMAIN_ID:
2037 case SDSP_DOMAIN_ID:

--- 153 unchanged lines hidden ---