1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Arm Firmware Framework for ARMv8-A(FFA) interface driver 4 * 5 * The Arm FFA specification[1] describes a software architecture to 6 * leverages the virtualization extension to isolate software images 7 * provided by an ecosystem of vendors from each other and describes 8 * interfaces that standardize communication between the various software 9 * images including communication between images in the Secure world and 10 * Normal world. Any Hypervisor could use the FFA interfaces to enable 11 * communication between VMs it manages. 12 * 13 * The Hypervisor a.k.a Partition managers in FFA terminology can assign 14 * system resources(Memory regions, Devices, CPU cycles) to the partitions 15 * and manage isolation amongst them. 16 * 17 * [1] https://developer.arm.com/docs/den0077/latest 18 * 19 * Copyright (C) 2021 ARM Ltd. 20 */ 21 22 #define DRIVER_NAME "ARM FF-A" 23 #define pr_fmt(fmt) DRIVER_NAME ": " fmt 24 25 #include <linux/acpi.h> 26 #include <linux/arm_ffa.h> 27 #include <linux/bitfield.h> 28 #include <linux/cpuhotplug.h> 29 #include <linux/delay.h> 30 #include <linux/device.h> 31 #include <linux/hashtable.h> 32 #include <linux/interrupt.h> 33 #include <linux/io.h> 34 #include <linux/kernel.h> 35 #include <linux/module.h> 36 #include <linux/mm.h> 37 #include <linux/mutex.h> 38 #include <linux/of_irq.h> 39 #include <linux/platform_device.h> 40 #include <linux/scatterlist.h> 41 #include <linux/slab.h> 42 #include <linux/smp.h> 43 #include <linux/uuid.h> 44 #include <linux/xarray.h> 45 46 #include <asm/virt.h> 47 48 #include "common.h" 49 50 #define FFA_DRIVER_VERSION FFA_VERSION_1_2 51 #define FFA_MIN_VERSION FFA_VERSION_1_0 52 #define FFA_PLATFORM_NAME "arm-ffa" 53 54 #define SENDER_ID_MASK GENMASK(31, 16) 55 #define RECEIVER_ID_MASK GENMASK(15, 0) 56 #define SENDER_ID(x) ((u16)(FIELD_GET(SENDER_ID_MASK, (x)))) 57 #define RECEIVER_ID(x) ((u16)(FIELD_GET(RECEIVER_ID_MASK, (x)))) 58 #define PACK_TARGET_INFO(s, r) \ 59 (FIELD_PREP(SENDER_ID_MASK, (s)) | FIELD_PREP(RECEIVER_ID_MASK, (r))) 60 61 #define RXTX_MAP_MIN_BUFSZ_MASK GENMASK(1, 0) 62 #define RXTX_MAP_MIN_BUFSZ(x) ((x) & RXTX_MAP_MIN_BUFSZ_MASK) 63 64 #define FFA_MAX_NOTIFICATIONS 64 65 66 static ffa_fn *invoke_ffa_fn; 67 68 static const int ffa_linux_errmap[] = { 69 /* better than switch case as long as return value is continuous */ 70 0, /* FFA_RET_SUCCESS */ 71 -EOPNOTSUPP, /* FFA_RET_NOT_SUPPORTED */ 72 -EINVAL, /* FFA_RET_INVALID_PARAMETERS */ 73 -ENOMEM, /* FFA_RET_NO_MEMORY */ 74 -EBUSY, /* FFA_RET_BUSY */ 75 -EINTR, /* FFA_RET_INTERRUPTED */ 76 -EACCES, /* FFA_RET_DENIED */ 77 -EAGAIN, /* FFA_RET_RETRY */ 78 -ECANCELED, /* FFA_RET_ABORTED */ 79 -ENODATA, /* FFA_RET_NO_DATA */ 80 -EAGAIN, /* FFA_RET_NOT_READY */ 81 }; 82 83 static inline int ffa_to_linux_errno(int errno) 84 { 85 int err_idx = -errno; 86 87 if (err_idx >= 0 && err_idx < ARRAY_SIZE(ffa_linux_errmap)) 88 return ffa_linux_errmap[err_idx]; 89 return -EINVAL; 90 } 91 92 struct ffa_pcpu_irq { 93 struct ffa_drv_info *info; 94 struct work_struct notif_pcpu_work; 95 }; 96 97 struct ffa_drv_info { 98 u32 version; 99 u16 vm_id; 100 struct mutex rx_lock; /* lock to protect Rx buffer */ 101 struct mutex tx_lock; /* lock to protect Tx buffer */ 102 void *rx_buffer; 103 void *tx_buffer; 104 size_t rxtx_bufsz; 105 bool mem_ops_native; 106 bool msg_direct_req2_supp; 107 bool bitmap_created; 108 bool bus_notifier_registered; 109 bool notif_enabled; 110 unsigned int sched_recv_irq; 111 unsigned int notif_pend_irq; 112 unsigned int cpuhp_state; 113 struct ffa_pcpu_irq __percpu *irq_pcpu; 114 struct workqueue_struct *notif_pcpu_wq; 115 struct work_struct sched_recv_irq_work; 116 struct xarray partition_info; 117 DECLARE_HASHTABLE(notifier_hash, ilog2(FFA_MAX_NOTIFICATIONS)); 118 rwlock_t notify_lock; /* lock to protect notifier hashtable */ 119 }; 120 121 static struct ffa_drv_info *drv_info; 122 static struct platform_device *ffa_pdev; 123 124 /* 125 * The driver must be able to support all the versions from the earliest 126 * supported FFA_MIN_VERSION to the latest supported FFA_DRIVER_VERSION. 127 * The specification states that if firmware supports a FFA implementation 128 * that is incompatible with and at a greater version number than specified 129 * by the caller(FFA_DRIVER_VERSION passed as parameter to FFA_VERSION), 130 * it must return the NOT_SUPPORTED error code. 131 */ 132 static u32 ffa_compatible_version_find(u32 version) 133 { 134 u16 major = FFA_MAJOR_VERSION(version), minor = FFA_MINOR_VERSION(version); 135 u16 drv_major = FFA_MAJOR_VERSION(FFA_DRIVER_VERSION); 136 u16 drv_minor = FFA_MINOR_VERSION(FFA_DRIVER_VERSION); 137 138 if ((major < drv_major) || (major == drv_major && minor <= drv_minor)) 139 return version; 140 141 pr_info("Firmware version higher than driver version, downgrading\n"); 142 return FFA_DRIVER_VERSION; 143 } 144 145 static int ffa_version_check(u32 *version) 146 { 147 ffa_value_t ver; 148 149 invoke_ffa_fn((ffa_value_t){ 150 .a0 = FFA_VERSION, .a1 = FFA_DRIVER_VERSION, 151 }, &ver); 152 153 if ((s32)ver.a0 == FFA_RET_NOT_SUPPORTED) { 154 pr_info("FFA_VERSION returned not supported\n"); 155 return -EOPNOTSUPP; 156 } 157 158 if (FFA_MAJOR_VERSION(ver.a0) > FFA_MAJOR_VERSION(FFA_DRIVER_VERSION)) { 159 pr_err("Incompatible v%d.%d! Latest supported v%d.%d\n", 160 FFA_MAJOR_VERSION(ver.a0), FFA_MINOR_VERSION(ver.a0), 161 FFA_MAJOR_VERSION(FFA_DRIVER_VERSION), 162 FFA_MINOR_VERSION(FFA_DRIVER_VERSION)); 163 return -EINVAL; 164 } 165 166 if (ver.a0 < FFA_MIN_VERSION) { 167 pr_err("Incompatible v%d.%d! Earliest supported v%d.%d\n", 168 FFA_MAJOR_VERSION(ver.a0), FFA_MINOR_VERSION(ver.a0), 169 FFA_MAJOR_VERSION(FFA_MIN_VERSION), 170 FFA_MINOR_VERSION(FFA_MIN_VERSION)); 171 return -EINVAL; 172 } 173 174 pr_info("Driver version %d.%d\n", FFA_MAJOR_VERSION(FFA_DRIVER_VERSION), 175 FFA_MINOR_VERSION(FFA_DRIVER_VERSION)); 176 pr_info("Firmware version %d.%d found\n", FFA_MAJOR_VERSION(ver.a0), 177 FFA_MINOR_VERSION(ver.a0)); 178 *version = ffa_compatible_version_find(ver.a0); 179 180 return 0; 181 } 182 183 static int ffa_rx_release(void) 184 { 185 ffa_value_t ret; 186 187 invoke_ffa_fn((ffa_value_t){ 188 .a0 = FFA_RX_RELEASE, 189 }, &ret); 190 191 if (ret.a0 == FFA_ERROR) 192 return ffa_to_linux_errno((int)ret.a2); 193 194 /* check for ret.a0 == FFA_RX_RELEASE ? */ 195 196 return 0; 197 } 198 199 static int ffa_rxtx_map(phys_addr_t tx_buf, phys_addr_t rx_buf, u32 pg_cnt) 200 { 201 ffa_value_t ret; 202 203 invoke_ffa_fn((ffa_value_t){ 204 .a0 = FFA_FN_NATIVE(RXTX_MAP), 205 .a1 = tx_buf, .a2 = rx_buf, .a3 = pg_cnt, 206 }, &ret); 207 208 if (ret.a0 == FFA_ERROR) 209 return ffa_to_linux_errno((int)ret.a2); 210 211 return 0; 212 } 213 214 static int ffa_rxtx_unmap(void) 215 { 216 ffa_value_t ret; 217 218 invoke_ffa_fn((ffa_value_t){ 219 .a0 = FFA_RXTX_UNMAP, 220 }, &ret); 221 222 if (ret.a0 == FFA_ERROR) 223 return ffa_to_linux_errno((int)ret.a2); 224 225 return 0; 226 } 227 228 static int ffa_features(u32 func_feat_id, u32 input_props, 229 u32 *if_props_1, u32 *if_props_2) 230 { 231 ffa_value_t id; 232 233 if (!ARM_SMCCC_IS_FAST_CALL(func_feat_id) && input_props) { 234 pr_err("%s: Invalid Parameters: %x, %x", __func__, 235 func_feat_id, input_props); 236 return ffa_to_linux_errno(FFA_RET_INVALID_PARAMETERS); 237 } 238 239 invoke_ffa_fn((ffa_value_t){ 240 .a0 = FFA_FEATURES, .a1 = func_feat_id, .a2 = input_props, 241 }, &id); 242 243 if (id.a0 == FFA_ERROR) 244 return ffa_to_linux_errno((int)id.a2); 245 246 if (if_props_1) 247 *if_props_1 = id.a2; 248 if (if_props_2) 249 *if_props_2 = id.a3; 250 251 return 0; 252 } 253 254 #define PARTITION_INFO_GET_RETURN_COUNT_ONLY BIT(0) 255 #define FFA_SUPPORTS_GET_COUNT_ONLY(version) ((version) > FFA_VERSION_1_0) 256 #define FFA_PART_INFO_HAS_SIZE_IN_RESP(version) ((version) > FFA_VERSION_1_0) 257 #define FFA_PART_INFO_HAS_UUID_IN_RESP(version) ((version) > FFA_VERSION_1_0) 258 #define FFA_PART_INFO_HAS_EXEC_STATE_IN_RESP(version) \ 259 ((version) > FFA_VERSION_1_0) 260 261 /* buffer must be sizeof(struct ffa_partition_info) * num_partitions */ 262 static int 263 __ffa_partition_info_get(u32 uuid0, u32 uuid1, u32 uuid2, u32 uuid3, 264 struct ffa_partition_info *buffer, int num_partitions) 265 { 266 int idx, count, flags = 0, sz, buf_sz; 267 ffa_value_t partition_info; 268 269 if (FFA_SUPPORTS_GET_COUNT_ONLY(drv_info->version) && 270 (!buffer || !num_partitions)) /* Just get the count for now */ 271 flags = PARTITION_INFO_GET_RETURN_COUNT_ONLY; 272 273 mutex_lock(&drv_info->rx_lock); 274 invoke_ffa_fn((ffa_value_t){ 275 .a0 = FFA_PARTITION_INFO_GET, 276 .a1 = uuid0, .a2 = uuid1, .a3 = uuid2, .a4 = uuid3, 277 .a5 = flags, 278 }, &partition_info); 279 280 if (partition_info.a0 == FFA_ERROR) { 281 mutex_unlock(&drv_info->rx_lock); 282 return ffa_to_linux_errno((int)partition_info.a2); 283 } 284 285 count = partition_info.a2; 286 287 if (FFA_PART_INFO_HAS_SIZE_IN_RESP(drv_info->version)) { 288 buf_sz = sz = partition_info.a3; 289 if (sz > sizeof(*buffer)) 290 buf_sz = sizeof(*buffer); 291 } else { 292 buf_sz = sz = 8; 293 } 294 295 if (buffer && count <= num_partitions) 296 for (idx = 0; idx < count; idx++) { 297 struct ffa_partition_info_le { 298 __le16 id; 299 __le16 exec_ctxt; 300 __le32 properties; 301 uuid_t uuid; 302 } *rx_buf = drv_info->rx_buffer + idx * sz; 303 struct ffa_partition_info *buf = buffer + idx; 304 305 buf->id = le16_to_cpu(rx_buf->id); 306 buf->exec_ctxt = le16_to_cpu(rx_buf->exec_ctxt); 307 buf->properties = le32_to_cpu(rx_buf->properties); 308 if (buf_sz > 8) 309 import_uuid(&buf->uuid, (u8 *)&rx_buf->uuid); 310 } 311 312 if (!(flags & PARTITION_INFO_GET_RETURN_COUNT_ONLY)) 313 ffa_rx_release(); 314 315 mutex_unlock(&drv_info->rx_lock); 316 317 return count; 318 } 319 320 #define LAST_INDEX_MASK GENMASK(15, 0) 321 #define CURRENT_INDEX_MASK GENMASK(31, 16) 322 #define UUID_INFO_TAG_MASK GENMASK(47, 32) 323 #define PARTITION_INFO_SZ_MASK GENMASK(63, 48) 324 #define PARTITION_COUNT(x) ((u16)(FIELD_GET(LAST_INDEX_MASK, (x))) + 1) 325 #define CURRENT_INDEX(x) ((u16)(FIELD_GET(CURRENT_INDEX_MASK, (x)))) 326 #define UUID_INFO_TAG(x) ((u16)(FIELD_GET(UUID_INFO_TAG_MASK, (x)))) 327 #define PARTITION_INFO_SZ(x) ((u16)(FIELD_GET(PARTITION_INFO_SZ_MASK, (x)))) 328 #define PART_INFO_ID_MASK GENMASK(15, 0) 329 #define PART_INFO_EXEC_CXT_MASK GENMASK(31, 16) 330 #define PART_INFO_PROPS_MASK GENMASK(63, 32) 331 #define FFA_PART_INFO_GET_REGS_FIRST_REG 3 332 #define FFA_PART_INFO_GET_REGS_MIN_REGS_PER_DESC 3 333 #define FFA_PART_INFO_GET_REGS_NUM_REGS \ 334 (sizeof(ffa_value_t) / sizeof_field(ffa_value_t, a0)) 335 #define PART_INFO_ID(x) ((u16)(FIELD_GET(PART_INFO_ID_MASK, (x)))) 336 #define PART_INFO_EXEC_CXT(x) ((u16)(FIELD_GET(PART_INFO_EXEC_CXT_MASK, (x)))) 337 #define PART_INFO_PROPERTIES(x) ((u32)(FIELD_GET(PART_INFO_PROPS_MASK, (x)))) 338 static int 339 __ffa_partition_info_get_regs(u32 uuid0, u32 uuid1, u32 uuid2, u32 uuid3, 340 struct ffa_partition_info *buffer, int num_parts) 341 { 342 u16 buf_sz, start_idx = 0, cur_idx, count = 0, tag = 0; 343 struct ffa_partition_info *buf = buffer; 344 ffa_value_t partition_info; 345 346 do { 347 __le64 *regs; 348 int idx, nr_desc, buf_idx, regs_per_desc, max_desc; 349 350 invoke_ffa_fn((ffa_value_t){ 351 .a0 = FFA_PARTITION_INFO_GET_REGS, 352 .a1 = (u64)uuid1 << 32 | uuid0, 353 .a2 = (u64)uuid3 << 32 | uuid2, 354 .a3 = start_idx | tag << 16, 355 }, &partition_info); 356 357 if (partition_info.a0 == FFA_ERROR) 358 return ffa_to_linux_errno((int)partition_info.a2); 359 360 if (!count) 361 count = PARTITION_COUNT(partition_info.a2); 362 if (!buffer || !num_parts) /* count only */ 363 return count; 364 if (count > num_parts) 365 return -EINVAL; 366 367 cur_idx = CURRENT_INDEX(partition_info.a2); 368 if (cur_idx < start_idx || cur_idx >= count) 369 return -EINVAL; 370 371 buf_sz = PARTITION_INFO_SZ(partition_info.a2); 372 if (buf_sz % sizeof(*regs)) 373 return -EINVAL; 374 375 regs_per_desc = buf_sz / sizeof(*regs); 376 if (regs_per_desc < FFA_PART_INFO_GET_REGS_MIN_REGS_PER_DESC) 377 return -EINVAL; 378 379 nr_desc = cur_idx - start_idx + 1; 380 max_desc = (FFA_PART_INFO_GET_REGS_NUM_REGS - 381 FFA_PART_INFO_GET_REGS_FIRST_REG) / regs_per_desc; 382 if (nr_desc > max_desc) 383 return -EINVAL; 384 385 buf_idx = buf - buffer; 386 if (buf_idx + nr_desc > num_parts) 387 return -EINVAL; 388 389 tag = UUID_INFO_TAG(partition_info.a2); 390 391 regs = (void *)&partition_info.a3; 392 for (idx = 0; idx < nr_desc; idx++, buf++) { 393 union { 394 uuid_t uuid; 395 u64 regs[2]; 396 } uuid_regs = { 397 .regs = { 398 le64_to_cpu(*(regs + 1)), 399 le64_to_cpu(*(regs + 2)), 400 } 401 }; 402 u64 val = *(u64 *)regs; 403 404 buf->id = PART_INFO_ID(val); 405 buf->exec_ctxt = PART_INFO_EXEC_CXT(val); 406 buf->properties = PART_INFO_PROPERTIES(val); 407 uuid_copy(&buf->uuid, &uuid_regs.uuid); 408 regs += regs_per_desc; 409 } 410 start_idx = cur_idx + 1; 411 412 } while (cur_idx < (count - 1)); 413 414 return count; 415 } 416 417 /* buffer is allocated and caller must free the same if returned count > 0 */ 418 static int 419 ffa_partition_probe(const uuid_t *uuid, struct ffa_partition_info **buffer) 420 { 421 int count; 422 u32 uuid0_4[4]; 423 bool reg_mode = false; 424 struct ffa_partition_info *pbuf; 425 426 if (!ffa_features(FFA_PARTITION_INFO_GET_REGS, 0, NULL, NULL)) 427 reg_mode = true; 428 429 export_uuid((u8 *)uuid0_4, uuid); 430 if (reg_mode) 431 count = __ffa_partition_info_get_regs(uuid0_4[0], uuid0_4[1], 432 uuid0_4[2], uuid0_4[3], 433 NULL, 0); 434 else 435 count = __ffa_partition_info_get(uuid0_4[0], uuid0_4[1], 436 uuid0_4[2], uuid0_4[3], 437 NULL, 0); 438 if (count <= 0) 439 return count; 440 441 pbuf = kzalloc_objs(*pbuf, count); 442 if (!pbuf) 443 return -ENOMEM; 444 445 if (reg_mode) 446 count = __ffa_partition_info_get_regs(uuid0_4[0], uuid0_4[1], 447 uuid0_4[2], uuid0_4[3], 448 pbuf, count); 449 else 450 count = __ffa_partition_info_get(uuid0_4[0], uuid0_4[1], 451 uuid0_4[2], uuid0_4[3], 452 pbuf, count); 453 if (count <= 0) 454 kfree(pbuf); 455 else 456 *buffer = pbuf; 457 458 return count; 459 } 460 461 #define VM_ID_MASK GENMASK(15, 0) 462 static int ffa_id_get(u16 *vm_id) 463 { 464 ffa_value_t id; 465 466 invoke_ffa_fn((ffa_value_t){ 467 .a0 = FFA_ID_GET, 468 }, &id); 469 470 if (id.a0 == FFA_ERROR) 471 return ffa_to_linux_errno((int)id.a2); 472 473 *vm_id = FIELD_GET(VM_ID_MASK, (id.a2)); 474 475 return 0; 476 } 477 478 static inline void ffa_msg_send_wait_for_completion(ffa_value_t *ret) 479 { 480 while (ret->a0 == FFA_INTERRUPT || ret->a0 == FFA_YIELD) { 481 if (ret->a0 == FFA_YIELD) 482 fsleep(1000); 483 484 invoke_ffa_fn((ffa_value_t){ 485 .a0 = FFA_RUN, .a1 = ret->a1, 486 }, ret); 487 } 488 } 489 490 static int ffa_msg_send_direct_req(u16 src_id, u16 dst_id, bool mode_32bit, 491 struct ffa_send_direct_data *data) 492 { 493 u32 req_id, resp_id, src_dst_ids = PACK_TARGET_INFO(src_id, dst_id); 494 ffa_value_t ret; 495 496 if (mode_32bit) { 497 req_id = FFA_MSG_SEND_DIRECT_REQ; 498 resp_id = FFA_MSG_SEND_DIRECT_RESP; 499 } else { 500 req_id = FFA_FN_NATIVE(MSG_SEND_DIRECT_REQ); 501 resp_id = FFA_FN_NATIVE(MSG_SEND_DIRECT_RESP); 502 } 503 504 invoke_ffa_fn((ffa_value_t){ 505 .a0 = req_id, .a1 = src_dst_ids, .a2 = 0, 506 .a3 = data->data0, .a4 = data->data1, .a5 = data->data2, 507 .a6 = data->data3, .a7 = data->data4, 508 }, &ret); 509 510 ffa_msg_send_wait_for_completion(&ret); 511 512 if (ret.a0 == FFA_ERROR) 513 return ffa_to_linux_errno((int)ret.a2); 514 515 if (ret.a0 == resp_id) { 516 data->data0 = ret.a3; 517 data->data1 = ret.a4; 518 data->data2 = ret.a5; 519 data->data3 = ret.a6; 520 data->data4 = ret.a7; 521 return 0; 522 } 523 524 return -EINVAL; 525 } 526 527 static int ffa_msg_send2(struct ffa_device *dev, u16 src_id, void *buf, size_t sz) 528 { 529 u32 src_dst_ids = PACK_TARGET_INFO(src_id, dev->vm_id); 530 struct ffa_indirect_msg_hdr *msg; 531 ffa_value_t ret; 532 int retval = 0; 533 534 if (sz > (drv_info->rxtx_bufsz - sizeof(*msg))) 535 return -ERANGE; 536 537 mutex_lock(&drv_info->tx_lock); 538 539 msg = drv_info->tx_buffer; 540 msg->flags = 0; 541 msg->res0 = 0; 542 msg->offset = sizeof(*msg); 543 msg->send_recv_id = src_dst_ids; 544 msg->size = sz; 545 uuid_copy(&msg->uuid, &dev->uuid); 546 memcpy((u8 *)msg + msg->offset, buf, sz); 547 548 /* flags = 0, sender VMID = 0 works for both physical/virtual NS */ 549 invoke_ffa_fn((ffa_value_t){ 550 .a0 = FFA_MSG_SEND2, .a1 = 0, .a2 = 0 551 }, &ret); 552 553 if (ret.a0 == FFA_ERROR) 554 retval = ffa_to_linux_errno((int)ret.a2); 555 556 mutex_unlock(&drv_info->tx_lock); 557 return retval; 558 } 559 560 static int ffa_msg_send_direct_req2(u16 src_id, u16 dst_id, const uuid_t *uuid, 561 struct ffa_send_direct_data2 *data) 562 { 563 u32 src_dst_ids = PACK_TARGET_INFO(src_id, dst_id); 564 union { 565 uuid_t uuid; 566 __le64 regs[2]; 567 } uuid_regs = { .uuid = *uuid }; 568 ffa_value_t ret, args = { 569 .a0 = FFA_MSG_SEND_DIRECT_REQ2, 570 .a1 = src_dst_ids, 571 .a2 = le64_to_cpu(uuid_regs.regs[0]), 572 .a3 = le64_to_cpu(uuid_regs.regs[1]), 573 }; 574 memcpy((void *)&args + offsetof(ffa_value_t, a4), data, sizeof(*data)); 575 576 invoke_ffa_fn(args, &ret); 577 578 ffa_msg_send_wait_for_completion(&ret); 579 580 if (ret.a0 == FFA_ERROR) 581 return ffa_to_linux_errno((int)ret.a2); 582 583 if (ret.a0 == FFA_MSG_SEND_DIRECT_RESP2) { 584 memcpy(data, (void *)&ret + offsetof(ffa_value_t, a4), sizeof(*data)); 585 return 0; 586 } 587 588 return -EINVAL; 589 } 590 591 static int ffa_mem_first_frag(u32 func_id, phys_addr_t buf, u32 buf_sz, 592 u32 frag_len, u32 len, u64 *handle) 593 { 594 ffa_value_t ret; 595 596 invoke_ffa_fn((ffa_value_t){ 597 .a0 = func_id, .a1 = len, .a2 = frag_len, 598 .a3 = buf, .a4 = buf_sz, 599 }, &ret); 600 601 while (ret.a0 == FFA_MEM_OP_PAUSE) 602 invoke_ffa_fn((ffa_value_t){ 603 .a0 = FFA_MEM_OP_RESUME, 604 .a1 = ret.a1, .a2 = ret.a2, 605 }, &ret); 606 607 if (ret.a0 == FFA_ERROR) 608 return ffa_to_linux_errno((int)ret.a2); 609 610 if (ret.a0 == FFA_SUCCESS) { 611 if (handle) 612 *handle = PACK_HANDLE(ret.a2, ret.a3); 613 } else if (ret.a0 == FFA_MEM_FRAG_RX) { 614 if (handle) 615 *handle = PACK_HANDLE(ret.a1, ret.a2); 616 } else { 617 return -EOPNOTSUPP; 618 } 619 620 return frag_len; 621 } 622 623 static int ffa_mem_next_frag(u64 handle, u32 frag_len) 624 { 625 ffa_value_t ret; 626 627 invoke_ffa_fn((ffa_value_t){ 628 .a0 = FFA_MEM_FRAG_TX, 629 .a1 = HANDLE_LOW(handle), .a2 = HANDLE_HIGH(handle), 630 .a3 = frag_len, 631 }, &ret); 632 633 while (ret.a0 == FFA_MEM_OP_PAUSE) 634 invoke_ffa_fn((ffa_value_t){ 635 .a0 = FFA_MEM_OP_RESUME, 636 .a1 = ret.a1, .a2 = ret.a2, 637 }, &ret); 638 639 if (ret.a0 == FFA_ERROR) 640 return ffa_to_linux_errno((int)ret.a2); 641 642 if (ret.a0 == FFA_MEM_FRAG_RX) 643 return ret.a3; 644 else if (ret.a0 == FFA_SUCCESS) 645 return 0; 646 647 return -EOPNOTSUPP; 648 } 649 650 static int 651 ffa_transmit_fragment(u32 func_id, phys_addr_t buf, u32 buf_sz, u32 frag_len, 652 u32 len, u64 *handle, bool first) 653 { 654 if (!first) 655 return ffa_mem_next_frag(*handle, frag_len); 656 657 return ffa_mem_first_frag(func_id, buf, buf_sz, frag_len, len, handle); 658 } 659 660 static u32 ffa_get_num_pages_sg(struct scatterlist *sg) 661 { 662 u32 num_pages = 0; 663 664 do { 665 num_pages += sg->length / FFA_PAGE_SIZE; 666 } while ((sg = sg_next(sg))); 667 668 return num_pages; 669 } 670 671 static u16 ffa_memory_attributes_get(u32 func_id) 672 { 673 /* 674 * For the memory lend or donate operation, if the receiver is a PE or 675 * a proxy endpoint, the owner/sender must not specify the attributes 676 */ 677 if (func_id == FFA_FN_NATIVE(MEM_LEND) || 678 func_id == FFA_MEM_LEND) 679 return 0; 680 681 return FFA_MEM_NORMAL | FFA_MEM_WRITE_BACK | FFA_MEM_INNER_SHAREABLE; 682 } 683 684 static void ffa_emad_impdef_value_init(u32 version, void *dst, void *src) 685 { 686 struct ffa_mem_region_attributes *ep_mem_access; 687 688 if (FFA_EMAD_HAS_IMPDEF_FIELD(version)) 689 memcpy(dst, src, sizeof(ep_mem_access->impdef_val)); 690 } 691 692 static void 693 ffa_mem_region_additional_setup(u32 version, struct ffa_mem_region *mem_region) 694 { 695 if (!FFA_MEM_REGION_HAS_EP_MEM_OFFSET(version)) { 696 mem_region->ep_mem_size = 0; 697 } else { 698 mem_region->ep_mem_size = ffa_emad_size_get(version); 699 mem_region->ep_mem_offset = sizeof(*mem_region); 700 memset(mem_region->reserved, 0, 12); 701 } 702 } 703 704 static int 705 ffa_setup_and_transmit(u32 func_id, void *buffer, u32 max_fragsize, 706 struct ffa_mem_ops_args *args) 707 { 708 int rc = 0; 709 bool first = true; 710 u32 composite_offset; 711 phys_addr_t addr = 0; 712 struct ffa_mem_region *mem_region = buffer; 713 struct ffa_composite_mem_region *composite; 714 struct ffa_mem_region_addr_range *constituents; 715 struct ffa_mem_region_attributes *ep_mem_access; 716 u32 idx, frag_len, length, buf_sz = 0, num_entries = sg_nents(args->sg); 717 718 mem_region->tag = args->tag; 719 mem_region->flags = args->flags; 720 mem_region->sender_id = drv_info->vm_id; 721 mem_region->attributes = ffa_memory_attributes_get(func_id); 722 composite_offset = ffa_mem_desc_offset(buffer, args->nattrs, 723 drv_info->version); 724 725 for (idx = 0; idx < args->nattrs; idx++) { 726 ep_mem_access = buffer + 727 ffa_mem_desc_offset(buffer, idx, drv_info->version); 728 ep_mem_access->receiver = args->attrs[idx].receiver; 729 ep_mem_access->attrs = args->attrs[idx].attrs; 730 ep_mem_access->composite_off = composite_offset; 731 ep_mem_access->flag = 0; 732 ep_mem_access->reserved = 0; 733 ffa_emad_impdef_value_init(drv_info->version, 734 ep_mem_access->impdef_val, 735 args->attrs[idx].impdef_val); 736 } 737 mem_region->handle = 0; 738 mem_region->ep_count = args->nattrs; 739 ffa_mem_region_additional_setup(drv_info->version, mem_region); 740 741 composite = buffer + composite_offset; 742 composite->total_pg_cnt = ffa_get_num_pages_sg(args->sg); 743 composite->addr_range_cnt = num_entries; 744 composite->reserved = 0; 745 746 length = composite_offset + CONSTITUENTS_OFFSET(num_entries); 747 frag_len = composite_offset + CONSTITUENTS_OFFSET(0); 748 if (frag_len > max_fragsize) 749 return -ENXIO; 750 751 if (!args->use_txbuf) { 752 addr = virt_to_phys(buffer); 753 buf_sz = max_fragsize / FFA_PAGE_SIZE; 754 } 755 756 constituents = buffer + frag_len; 757 idx = 0; 758 do { 759 if (frag_len == max_fragsize) { 760 rc = ffa_transmit_fragment(func_id, addr, buf_sz, 761 frag_len, length, 762 &args->g_handle, first); 763 if (rc < 0) 764 return -ENXIO; 765 766 first = false; 767 idx = 0; 768 frag_len = 0; 769 constituents = buffer; 770 } 771 772 if ((void *)constituents - buffer > max_fragsize) { 773 pr_err("Memory Region Fragment > Tx Buffer size\n"); 774 return -EFAULT; 775 } 776 777 constituents->address = sg_phys(args->sg); 778 constituents->pg_cnt = args->sg->length / FFA_PAGE_SIZE; 779 constituents->reserved = 0; 780 constituents++; 781 frag_len += sizeof(struct ffa_mem_region_addr_range); 782 } while ((args->sg = sg_next(args->sg))); 783 784 return ffa_transmit_fragment(func_id, addr, buf_sz, frag_len, 785 length, &args->g_handle, first); 786 } 787 788 static int ffa_memory_ops(u32 func_id, struct ffa_mem_ops_args *args) 789 { 790 int ret; 791 void *buffer; 792 size_t rxtx_bufsz = drv_info->rxtx_bufsz; 793 794 if (!args->use_txbuf) { 795 buffer = alloc_pages_exact(rxtx_bufsz, GFP_KERNEL); 796 if (!buffer) 797 return -ENOMEM; 798 } else { 799 buffer = drv_info->tx_buffer; 800 mutex_lock(&drv_info->tx_lock); 801 } 802 803 ret = ffa_setup_and_transmit(func_id, buffer, rxtx_bufsz, args); 804 805 if (args->use_txbuf) 806 mutex_unlock(&drv_info->tx_lock); 807 else 808 free_pages_exact(buffer, rxtx_bufsz); 809 810 return ret < 0 ? ret : 0; 811 } 812 813 static int ffa_memory_reclaim(u64 g_handle, u32 flags) 814 { 815 ffa_value_t ret; 816 817 invoke_ffa_fn((ffa_value_t){ 818 .a0 = FFA_MEM_RECLAIM, 819 .a1 = HANDLE_LOW(g_handle), .a2 = HANDLE_HIGH(g_handle), 820 .a3 = flags, 821 }, &ret); 822 823 if (ret.a0 == FFA_ERROR) 824 return ffa_to_linux_errno((int)ret.a2); 825 826 return 0; 827 } 828 829 static int ffa_notification_bitmap_create(void) 830 { 831 ffa_value_t ret; 832 u16 vcpu_count = nr_cpu_ids; 833 834 invoke_ffa_fn((ffa_value_t){ 835 .a0 = FFA_NOTIFICATION_BITMAP_CREATE, 836 .a1 = drv_info->vm_id, .a2 = vcpu_count, 837 }, &ret); 838 839 if (ret.a0 == FFA_ERROR) 840 return ffa_to_linux_errno((int)ret.a2); 841 842 return 0; 843 } 844 845 static int ffa_notification_bitmap_destroy(void) 846 { 847 ffa_value_t ret; 848 849 invoke_ffa_fn((ffa_value_t){ 850 .a0 = FFA_NOTIFICATION_BITMAP_DESTROY, 851 .a1 = drv_info->vm_id, 852 }, &ret); 853 854 if (ret.a0 == FFA_ERROR) 855 return ffa_to_linux_errno((int)ret.a2); 856 857 return 0; 858 } 859 860 enum notify_type { 861 SECURE_PARTITION, 862 NON_SECURE_VM, 863 SPM_FRAMEWORK, 864 NS_HYP_FRAMEWORK, 865 }; 866 867 #define NOTIFICATION_LOW_MASK GENMASK(31, 0) 868 #define NOTIFICATION_HIGH_MASK GENMASK(63, 32) 869 #define NOTIFICATION_BITMAP_HIGH(x) \ 870 ((u32)(FIELD_GET(NOTIFICATION_HIGH_MASK, (x)))) 871 #define NOTIFICATION_BITMAP_LOW(x) \ 872 ((u32)(FIELD_GET(NOTIFICATION_LOW_MASK, (x)))) 873 #define PACK_NOTIFICATION_BITMAP(low, high) \ 874 (FIELD_PREP(NOTIFICATION_LOW_MASK, (low)) | \ 875 FIELD_PREP(NOTIFICATION_HIGH_MASK, (high))) 876 877 #define RECEIVER_VCPU_MASK GENMASK(31, 16) 878 #define PACK_NOTIFICATION_GET_RECEIVER_INFO(vcpu_r, r) \ 879 (FIELD_PREP(RECEIVER_VCPU_MASK, (vcpu_r)) | \ 880 FIELD_PREP(RECEIVER_ID_MASK, (r))) 881 882 #define NOTIFICATION_INFO_GET_MORE_PEND_MASK BIT(0) 883 #define NOTIFICATION_INFO_GET_ID_COUNT GENMASK(11, 7) 884 #define ID_LIST_MASK_64 GENMASK(51, 12) 885 #define ID_LIST_MASK_32 GENMASK(31, 12) 886 #define MAX_IDS_64 20 887 #define MAX_IDS_32 10 888 889 #define PER_VCPU_NOTIFICATION_FLAG BIT(0) 890 #define SECURE_PARTITION_BITMAP_ENABLE BIT(SECURE_PARTITION) 891 #define NON_SECURE_VM_BITMAP_ENABLE BIT(NON_SECURE_VM) 892 #define SPM_FRAMEWORK_BITMAP_ENABLE BIT(SPM_FRAMEWORK) 893 #define NS_HYP_FRAMEWORK_BITMAP_ENABLE BIT(NS_HYP_FRAMEWORK) 894 #define FFA_BITMAP_SECURE_ENABLE_MASK \ 895 (SECURE_PARTITION_BITMAP_ENABLE | SPM_FRAMEWORK_BITMAP_ENABLE) 896 #define FFA_BITMAP_NS_ENABLE_MASK \ 897 (NON_SECURE_VM_BITMAP_ENABLE | NS_HYP_FRAMEWORK_BITMAP_ENABLE) 898 #define FFA_BITMAP_ALL_ENABLE_MASK \ 899 (FFA_BITMAP_SECURE_ENABLE_MASK | FFA_BITMAP_NS_ENABLE_MASK) 900 901 #define FFA_SECURE_PARTITION_ID_FLAG BIT(15) 902 903 #define SPM_FRAMEWORK_BITMAP(x) NOTIFICATION_BITMAP_LOW(x) 904 #define NS_HYP_FRAMEWORK_BITMAP(x) NOTIFICATION_BITMAP_HIGH(x) 905 #define FRAMEWORK_NOTIFY_RX_BUFFER_FULL BIT(0) 906 907 static int ffa_notification_bind_common(u16 dst_id, u64 bitmap, 908 u32 flags, bool is_bind) 909 { 910 ffa_value_t ret; 911 u32 func, src_dst_ids = PACK_TARGET_INFO(dst_id, drv_info->vm_id); 912 913 func = is_bind ? FFA_NOTIFICATION_BIND : FFA_NOTIFICATION_UNBIND; 914 915 invoke_ffa_fn((ffa_value_t){ 916 .a0 = func, .a1 = src_dst_ids, .a2 = flags, 917 .a3 = NOTIFICATION_BITMAP_LOW(bitmap), 918 .a4 = NOTIFICATION_BITMAP_HIGH(bitmap), 919 }, &ret); 920 921 if (ret.a0 == FFA_ERROR) 922 return ffa_to_linux_errno((int)ret.a2); 923 else if (ret.a0 != FFA_SUCCESS) 924 return -EINVAL; 925 926 return 0; 927 } 928 929 static 930 int ffa_notification_set(u16 src_id, u16 dst_id, u32 flags, u64 bitmap) 931 { 932 ffa_value_t ret; 933 u32 src_dst_ids = PACK_TARGET_INFO(dst_id, src_id); 934 935 invoke_ffa_fn((ffa_value_t) { 936 .a0 = FFA_NOTIFICATION_SET, .a1 = src_dst_ids, .a2 = flags, 937 .a3 = NOTIFICATION_BITMAP_LOW(bitmap), 938 .a4 = NOTIFICATION_BITMAP_HIGH(bitmap), 939 }, &ret); 940 941 if (ret.a0 == FFA_ERROR) 942 return ffa_to_linux_errno((int)ret.a2); 943 else if (ret.a0 != FFA_SUCCESS) 944 return -EINVAL; 945 946 return 0; 947 } 948 949 struct ffa_notify_bitmaps { 950 u64 sp_map; 951 u64 vm_map; 952 u64 arch_map; 953 }; 954 955 static int ffa_notification_get(u32 flags, struct ffa_notify_bitmaps *notify) 956 { 957 ffa_value_t ret; 958 u16 src_id = drv_info->vm_id; 959 u16 cpu_id = smp_processor_id(); 960 u32 rec_vcpu_ids = PACK_NOTIFICATION_GET_RECEIVER_INFO(cpu_id, src_id); 961 962 invoke_ffa_fn((ffa_value_t){ 963 .a0 = FFA_NOTIFICATION_GET, .a1 = rec_vcpu_ids, .a2 = flags, 964 }, &ret); 965 966 if (ret.a0 == FFA_ERROR) 967 return ffa_to_linux_errno((int)ret.a2); 968 else if (ret.a0 != FFA_SUCCESS) 969 return -EINVAL; /* Something else went wrong. */ 970 971 if (flags & SECURE_PARTITION_BITMAP_ENABLE) 972 notify->sp_map = PACK_NOTIFICATION_BITMAP(ret.a2, ret.a3); 973 if (flags & NON_SECURE_VM_BITMAP_ENABLE) 974 notify->vm_map = PACK_NOTIFICATION_BITMAP(ret.a4, ret.a5); 975 if (flags & SPM_FRAMEWORK_BITMAP_ENABLE) 976 notify->arch_map = SPM_FRAMEWORK_BITMAP(ret.a6); 977 if (flags & NS_HYP_FRAMEWORK_BITMAP_ENABLE) 978 notify->arch_map = PACK_NOTIFICATION_BITMAP(notify->arch_map, 979 ret.a7); 980 981 return 0; 982 } 983 984 struct ffa_dev_part_info { 985 ffa_sched_recv_cb callback; 986 void *cb_data; 987 rwlock_t rw_lock; 988 struct ffa_device *dev; 989 struct list_head node; 990 }; 991 992 static void __do_sched_recv_cb(u16 part_id, u16 vcpu, bool is_per_vcpu) 993 { 994 struct ffa_dev_part_info *partition = NULL, *tmp; 995 ffa_sched_recv_cb callback; 996 struct list_head *phead; 997 void *cb_data; 998 999 phead = xa_load(&drv_info->partition_info, part_id); 1000 if (!phead) { 1001 pr_err("%s: Invalid partition ID 0x%x\n", __func__, part_id); 1002 return; 1003 } 1004 1005 list_for_each_entry_safe(partition, tmp, phead, node) { 1006 read_lock(&partition->rw_lock); 1007 callback = partition->callback; 1008 cb_data = partition->cb_data; 1009 read_unlock(&partition->rw_lock); 1010 1011 if (callback) 1012 callback(vcpu, is_per_vcpu, cb_data); 1013 } 1014 } 1015 1016 /* 1017 * Map logical ID index to the u16 index within the packed ID list. 1018 * 1019 * For native responses (FF-A width == kernel word size), IDs are 1020 * tightly packed: idx -> idx. 1021 * 1022 * For 32-bit responses on a 64-bit kernel, each 64-bit register 1023 * contributes 4 x u16 values but only the lower 2 are defined; the 1024 * upper 2 are garbage. This mapping skips those upper halves: 1025 * 0,1,2,3,4,5,... -> 0,1,4,5,8,9,... 1026 */ 1027 static int list_idx_to_u16_idx(int idx, bool is_native_resp) 1028 { 1029 return is_native_resp ? idx : idx + 2 * (idx >> 1); 1030 } 1031 1032 static void ffa_notification_info_get(void) 1033 { 1034 int ids_processed, ids_count[MAX_IDS_64]; 1035 int idx, list, max_ids, lists_cnt; 1036 bool is_64b_resp, is_native_resp; 1037 ffa_value_t ret; 1038 u64 id_list; 1039 1040 do { 1041 invoke_ffa_fn((ffa_value_t){ 1042 .a0 = FFA_FN_NATIVE(NOTIFICATION_INFO_GET), 1043 }, &ret); 1044 1045 if (ret.a0 != FFA_FN_NATIVE(SUCCESS) && ret.a0 != FFA_SUCCESS) { 1046 if ((s32)ret.a2 != FFA_RET_NO_DATA) 1047 pr_err("Notification Info fetch failed: 0x%lx (0x%lx)", 1048 ret.a0, ret.a2); 1049 return; 1050 } 1051 1052 is_64b_resp = (ret.a0 == FFA_FN64_SUCCESS); 1053 is_native_resp = (ret.a0 == FFA_FN_NATIVE(SUCCESS)); 1054 1055 ids_processed = 0; 1056 lists_cnt = FIELD_GET(NOTIFICATION_INFO_GET_ID_COUNT, ret.a2); 1057 if (is_64b_resp) { 1058 max_ids = MAX_IDS_64; 1059 id_list = FIELD_GET(ID_LIST_MASK_64, ret.a2); 1060 } else { 1061 max_ids = MAX_IDS_32; 1062 id_list = FIELD_GET(ID_LIST_MASK_32, ret.a2); 1063 } 1064 1065 for (idx = 0; idx < lists_cnt; idx++, id_list >>= 2) 1066 ids_count[idx] = (id_list & 0x3) + 1; 1067 1068 /* Process IDs */ 1069 for (list = 0; list < lists_cnt; list++) { 1070 int u16_idx; 1071 u16 vcpu_id, part_id, *packed_id_list = (u16 *)&ret.a3; 1072 1073 if (ids_processed >= max_ids - 1) 1074 break; 1075 1076 u16_idx = list_idx_to_u16_idx(ids_processed, 1077 is_native_resp); 1078 part_id = packed_id_list[u16_idx]; 1079 ids_processed++; 1080 1081 if (ids_count[list] == 1) { /* Global Notification */ 1082 __do_sched_recv_cb(part_id, 0, false); 1083 continue; 1084 } 1085 1086 /* Per vCPU Notification */ 1087 for (idx = 1; idx < ids_count[list]; idx++) { 1088 if (ids_processed >= max_ids - 1) 1089 break; 1090 1091 u16_idx = list_idx_to_u16_idx(ids_processed, 1092 is_native_resp); 1093 vcpu_id = packed_id_list[u16_idx]; 1094 ids_processed++; 1095 1096 __do_sched_recv_cb(part_id, vcpu_id, true); 1097 } 1098 } 1099 } while (ret.a2 & NOTIFICATION_INFO_GET_MORE_PEND_MASK); 1100 } 1101 1102 static int ffa_run(struct ffa_device *dev, u16 vcpu) 1103 { 1104 ffa_value_t ret; 1105 u32 target = dev->vm_id << 16 | vcpu; 1106 1107 invoke_ffa_fn((ffa_value_t){ .a0 = FFA_RUN, .a1 = target, }, &ret); 1108 1109 while (ret.a0 == FFA_INTERRUPT) 1110 invoke_ffa_fn((ffa_value_t){ .a0 = FFA_RUN, .a1 = ret.a1, }, 1111 &ret); 1112 1113 if (ret.a0 == FFA_ERROR) 1114 return ffa_to_linux_errno((int)ret.a2); 1115 1116 return 0; 1117 } 1118 1119 static void ffa_drvinfo_flags_init(void) 1120 { 1121 if (!ffa_features(FFA_FN_NATIVE(MEM_LEND), 0, NULL, NULL) || 1122 !ffa_features(FFA_FN_NATIVE(MEM_SHARE), 0, NULL, NULL)) 1123 drv_info->mem_ops_native = true; 1124 1125 if (!ffa_features(FFA_MSG_SEND_DIRECT_REQ2, 0, NULL, NULL) || 1126 !ffa_features(FFA_MSG_SEND_DIRECT_RESP2, 0, NULL, NULL)) 1127 drv_info->msg_direct_req2_supp = true; 1128 } 1129 1130 static u32 ffa_api_version_get(void) 1131 { 1132 return drv_info->version; 1133 } 1134 1135 static int ffa_partition_info_get(const char *uuid_str, 1136 struct ffa_partition_info *buffer) 1137 { 1138 int count; 1139 uuid_t uuid; 1140 struct ffa_partition_info *pbuf; 1141 1142 if (uuid_parse(uuid_str, &uuid)) { 1143 pr_err("invalid uuid (%s)\n", uuid_str); 1144 return -ENODEV; 1145 } 1146 1147 count = ffa_partition_probe(&uuid, &pbuf); 1148 if (count <= 0) 1149 return -ENOENT; 1150 1151 memcpy(buffer, pbuf, sizeof(*pbuf) * count); 1152 kfree(pbuf); 1153 return 0; 1154 } 1155 1156 static void ffa_mode_32bit_set(struct ffa_device *dev) 1157 { 1158 dev->mode_32bit = true; 1159 } 1160 1161 static int ffa_sync_send_receive(struct ffa_device *dev, 1162 struct ffa_send_direct_data *data) 1163 { 1164 return ffa_msg_send_direct_req(drv_info->vm_id, dev->vm_id, 1165 dev->mode_32bit, data); 1166 } 1167 1168 static int ffa_indirect_msg_send(struct ffa_device *dev, void *buf, size_t sz) 1169 { 1170 return ffa_msg_send2(dev, drv_info->vm_id, buf, sz); 1171 } 1172 1173 static int ffa_sync_send_receive2(struct ffa_device *dev, 1174 struct ffa_send_direct_data2 *data) 1175 { 1176 if (!drv_info->msg_direct_req2_supp) 1177 return -EOPNOTSUPP; 1178 1179 return ffa_msg_send_direct_req2(drv_info->vm_id, dev->vm_id, 1180 &dev->uuid, data); 1181 } 1182 1183 static int ffa_memory_share(struct ffa_mem_ops_args *args) 1184 { 1185 if (drv_info->mem_ops_native) 1186 return ffa_memory_ops(FFA_FN_NATIVE(MEM_SHARE), args); 1187 1188 return ffa_memory_ops(FFA_MEM_SHARE, args); 1189 } 1190 1191 static int ffa_memory_lend(struct ffa_mem_ops_args *args) 1192 { 1193 /* Note that upon a successful MEM_LEND request the caller 1194 * must ensure that the memory region specified is not accessed 1195 * until a successful MEM_RECALIM call has been made. 1196 * On systems with a hypervisor present this will been enforced, 1197 * however on systems without a hypervisor the responsibility 1198 * falls to the calling kernel driver to prevent access. 1199 */ 1200 if (drv_info->mem_ops_native) 1201 return ffa_memory_ops(FFA_FN_NATIVE(MEM_LEND), args); 1202 1203 return ffa_memory_ops(FFA_MEM_LEND, args); 1204 } 1205 1206 #define ffa_notifications_disabled() (!drv_info->notif_enabled) 1207 1208 struct notifier_cb_info { 1209 struct hlist_node hnode; 1210 struct ffa_device *dev; 1211 ffa_fwk_notifier_cb fwk_cb; 1212 ffa_notifier_cb cb; 1213 void *cb_data; 1214 }; 1215 1216 static int 1217 ffa_sched_recv_cb_update(struct ffa_device *dev, ffa_sched_recv_cb callback, 1218 void *cb_data, bool is_registration) 1219 { 1220 struct ffa_dev_part_info *partition = NULL; 1221 struct list_head *phead; 1222 bool cb_valid; 1223 1224 if (ffa_notifications_disabled()) 1225 return -EOPNOTSUPP; 1226 1227 phead = xa_load(&drv_info->partition_info, dev->vm_id); 1228 if (!phead) { 1229 pr_err("%s: Invalid partition ID 0x%x\n", __func__, dev->vm_id); 1230 return -EINVAL; 1231 } 1232 1233 list_for_each_entry(partition, phead, node) 1234 if (partition->dev == dev) 1235 break; 1236 1237 if (&partition->node == phead) { 1238 pr_err("%s: No such partition ID 0x%x\n", __func__, dev->vm_id); 1239 return -EINVAL; 1240 } 1241 1242 write_lock(&partition->rw_lock); 1243 1244 cb_valid = !!partition->callback; 1245 if (!(is_registration ^ cb_valid)) { 1246 write_unlock(&partition->rw_lock); 1247 return -EINVAL; 1248 } 1249 1250 partition->callback = callback; 1251 partition->cb_data = cb_data; 1252 1253 write_unlock(&partition->rw_lock); 1254 return 0; 1255 } 1256 1257 static int ffa_sched_recv_cb_register(struct ffa_device *dev, 1258 ffa_sched_recv_cb cb, void *cb_data) 1259 { 1260 return ffa_sched_recv_cb_update(dev, cb, cb_data, true); 1261 } 1262 1263 static int ffa_sched_recv_cb_unregister(struct ffa_device *dev) 1264 { 1265 return ffa_sched_recv_cb_update(dev, NULL, NULL, false); 1266 } 1267 1268 static int ffa_notification_bind(u16 dst_id, u64 bitmap, u32 flags) 1269 { 1270 return ffa_notification_bind_common(dst_id, bitmap, flags, true); 1271 } 1272 1273 static int ffa_notification_unbind(u16 dst_id, u64 bitmap) 1274 { 1275 return ffa_notification_bind_common(dst_id, bitmap, 0, false); 1276 } 1277 1278 static enum notify_type ffa_notify_type_get(u16 vm_id) 1279 { 1280 if (vm_id & FFA_SECURE_PARTITION_ID_FLAG) 1281 return SECURE_PARTITION; 1282 else 1283 return NON_SECURE_VM; 1284 } 1285 1286 /* notifier_hnode_get* should be called with notify_lock held */ 1287 static struct notifier_cb_info * 1288 notifier_hnode_get_by_vmid(u16 notify_id, int vmid) 1289 { 1290 struct notifier_cb_info *node; 1291 1292 hash_for_each_possible(drv_info->notifier_hash, node, hnode, notify_id) 1293 if (node->fwk_cb && vmid == node->dev->vm_id) 1294 return node; 1295 1296 return NULL; 1297 } 1298 1299 static struct notifier_cb_info * 1300 notifier_hnode_get_by_vmid_uuid(u16 notify_id, int vmid, const uuid_t *uuid) 1301 { 1302 struct notifier_cb_info *node; 1303 1304 if (uuid_is_null(uuid)) 1305 return notifier_hnode_get_by_vmid(notify_id, vmid); 1306 1307 hash_for_each_possible(drv_info->notifier_hash, node, hnode, notify_id) 1308 if (node->fwk_cb && vmid == node->dev->vm_id && 1309 uuid_equal(&node->dev->uuid, uuid)) 1310 return node; 1311 1312 return NULL; 1313 } 1314 1315 static struct notifier_cb_info * 1316 notifier_hnode_get_by_type(u16 notify_id, enum notify_type type) 1317 { 1318 struct notifier_cb_info *node; 1319 1320 hash_for_each_possible(drv_info->notifier_hash, node, hnode, notify_id) 1321 if (node->cb && type == ffa_notify_type_get(node->dev->vm_id)) 1322 return node; 1323 1324 return NULL; 1325 } 1326 1327 static int update_notifier_cb(struct ffa_device *dev, int notify_id, 1328 struct notifier_cb_info *cb, bool is_framework) 1329 { 1330 struct notifier_cb_info *cb_info = NULL; 1331 enum notify_type type = ffa_notify_type_get(dev->vm_id); 1332 bool cb_found, is_registration = !!cb; 1333 1334 if (is_framework) 1335 cb_info = notifier_hnode_get_by_vmid_uuid(notify_id, dev->vm_id, 1336 &dev->uuid); 1337 else 1338 cb_info = notifier_hnode_get_by_type(notify_id, type); 1339 1340 cb_found = !!cb_info; 1341 1342 if (!(is_registration ^ cb_found)) 1343 return -EINVAL; 1344 1345 if (is_registration) { 1346 hash_add(drv_info->notifier_hash, &cb->hnode, notify_id); 1347 } else { 1348 hash_del(&cb_info->hnode); 1349 kfree(cb_info); 1350 } 1351 1352 return 0; 1353 } 1354 1355 static int __ffa_notify_relinquish(struct ffa_device *dev, int notify_id, 1356 bool is_framework) 1357 { 1358 int rc; 1359 1360 if (ffa_notifications_disabled()) 1361 return -EOPNOTSUPP; 1362 1363 if (notify_id >= FFA_MAX_NOTIFICATIONS) 1364 return -EINVAL; 1365 1366 write_lock(&drv_info->notify_lock); 1367 1368 rc = update_notifier_cb(dev, notify_id, NULL, is_framework); 1369 if (rc) { 1370 pr_err("Could not unregister notification callback\n"); 1371 write_unlock(&drv_info->notify_lock); 1372 return rc; 1373 } 1374 1375 if (!is_framework) 1376 rc = ffa_notification_unbind(dev->vm_id, BIT(notify_id)); 1377 1378 write_unlock(&drv_info->notify_lock); 1379 1380 return rc; 1381 } 1382 1383 static int ffa_notify_relinquish(struct ffa_device *dev, int notify_id) 1384 { 1385 return __ffa_notify_relinquish(dev, notify_id, false); 1386 } 1387 1388 static int ffa_fwk_notify_relinquish(struct ffa_device *dev, int notify_id) 1389 { 1390 return __ffa_notify_relinquish(dev, notify_id, true); 1391 } 1392 1393 static int __ffa_notify_request(struct ffa_device *dev, bool is_per_vcpu, 1394 void *cb, void *cb_data, 1395 int notify_id, bool is_framework) 1396 { 1397 int rc; 1398 u32 flags = 0; 1399 struct notifier_cb_info *cb_info = NULL; 1400 1401 if (ffa_notifications_disabled()) 1402 return -EOPNOTSUPP; 1403 1404 if (notify_id >= FFA_MAX_NOTIFICATIONS) 1405 return -EINVAL; 1406 1407 cb_info = kzalloc_obj(*cb_info); 1408 if (!cb_info) 1409 return -ENOMEM; 1410 1411 cb_info->dev = dev; 1412 cb_info->cb_data = cb_data; 1413 if (is_framework) 1414 cb_info->fwk_cb = cb; 1415 else 1416 cb_info->cb = cb; 1417 1418 write_lock(&drv_info->notify_lock); 1419 1420 if (!is_framework) { 1421 if (is_per_vcpu) 1422 flags = PER_VCPU_NOTIFICATION_FLAG; 1423 1424 rc = ffa_notification_bind(dev->vm_id, BIT(notify_id), flags); 1425 if (rc) 1426 goto out_unlock_free; 1427 } 1428 1429 rc = update_notifier_cb(dev, notify_id, cb_info, is_framework); 1430 if (rc) { 1431 pr_err("Failed to register callback for %d - %d\n", 1432 notify_id, rc); 1433 if (!is_framework) 1434 ffa_notification_unbind(dev->vm_id, BIT(notify_id)); 1435 } 1436 1437 out_unlock_free: 1438 write_unlock(&drv_info->notify_lock); 1439 if (rc) 1440 kfree(cb_info); 1441 1442 return rc; 1443 } 1444 1445 static int ffa_notify_request(struct ffa_device *dev, bool is_per_vcpu, 1446 ffa_notifier_cb cb, void *cb_data, int notify_id) 1447 { 1448 return __ffa_notify_request(dev, is_per_vcpu, cb, cb_data, notify_id, 1449 false); 1450 } 1451 1452 static int 1453 ffa_fwk_notify_request(struct ffa_device *dev, ffa_fwk_notifier_cb cb, 1454 void *cb_data, int notify_id) 1455 { 1456 return __ffa_notify_request(dev, false, cb, cb_data, notify_id, true); 1457 } 1458 1459 static int ffa_notify_send(struct ffa_device *dev, int notify_id, 1460 bool is_per_vcpu, u16 vcpu) 1461 { 1462 u32 flags = 0; 1463 1464 if (ffa_notifications_disabled()) 1465 return -EOPNOTSUPP; 1466 1467 if (is_per_vcpu) 1468 flags |= (PER_VCPU_NOTIFICATION_FLAG | vcpu << 16); 1469 1470 return ffa_notification_set(dev->vm_id, drv_info->vm_id, flags, 1471 BIT(notify_id)); 1472 } 1473 1474 static void handle_notif_callbacks(u64 bitmap, enum notify_type type) 1475 { 1476 ffa_notifier_cb cb; 1477 void *cb_data; 1478 int notify_id; 1479 1480 for (notify_id = 0; notify_id <= FFA_MAX_NOTIFICATIONS && bitmap; 1481 notify_id++, bitmap >>= 1) { 1482 if (!(bitmap & 1)) 1483 continue; 1484 1485 scoped_guard(read_lock, &drv_info->notify_lock) { 1486 struct notifier_cb_info *cb_info; 1487 1488 cb_info = notifier_hnode_get_by_type(notify_id, type); 1489 cb = cb_info ? cb_info->cb : NULL; 1490 cb_data = cb_info ? cb_info->cb_data : NULL; 1491 } 1492 1493 if (cb) 1494 cb(notify_id, cb_data); 1495 } 1496 } 1497 1498 static void handle_fwk_notif_callbacks(u32 bitmap) 1499 { 1500 void *buf; 1501 uuid_t uuid; 1502 void *fwk_cb_data; 1503 int notify_id = 0, target; 1504 ffa_fwk_notifier_cb fwk_cb; 1505 struct ffa_indirect_msg_hdr *msg; 1506 size_t min_offset = offsetof(struct ffa_indirect_msg_hdr, uuid); 1507 1508 /* Only one framework notification defined and supported for now */ 1509 if (!(bitmap & FRAMEWORK_NOTIFY_RX_BUFFER_FULL)) 1510 return; 1511 1512 scoped_guard(mutex, &drv_info->rx_lock) { 1513 u32 offset, size; 1514 1515 msg = drv_info->rx_buffer; 1516 offset = msg->offset; 1517 size = msg->size; 1518 1519 if (!size || (offset != min_offset && offset < sizeof(*msg)) || 1520 offset > drv_info->rxtx_bufsz || 1521 size > drv_info->rxtx_bufsz - offset) { 1522 pr_err("invalid framework notification message\n"); 1523 ffa_rx_release(); 1524 return; 1525 } 1526 1527 buf = kmemdup((void *)msg + offset, size, GFP_KERNEL); 1528 if (!buf) { 1529 ffa_rx_release(); 1530 return; 1531 } 1532 1533 target = SENDER_ID(msg->send_recv_id); 1534 if (offset >= sizeof(*msg)) 1535 uuid_copy(&uuid, &msg->uuid); 1536 else 1537 uuid_copy(&uuid, &uuid_null); 1538 ffa_rx_release(); 1539 } 1540 1541 scoped_guard(read_lock, &drv_info->notify_lock) { 1542 struct notifier_cb_info *cb_info; 1543 1544 cb_info = notifier_hnode_get_by_vmid_uuid(notify_id, target, 1545 &uuid); 1546 fwk_cb = cb_info ? cb_info->fwk_cb : NULL; 1547 fwk_cb_data = cb_info ? cb_info->cb_data : NULL; 1548 } 1549 1550 if (fwk_cb) 1551 fwk_cb(notify_id, fwk_cb_data, buf); 1552 kfree(buf); 1553 } 1554 1555 static void notif_get_and_handle(void *cb_data) 1556 { 1557 int rc; 1558 u32 flags; 1559 struct ffa_drv_info *info = cb_data; 1560 struct ffa_notify_bitmaps bitmaps = { 0 }; 1561 1562 if (info->vm_id == 0) /* Non secure physical instance */ 1563 flags = FFA_BITMAP_SECURE_ENABLE_MASK; 1564 else 1565 flags = FFA_BITMAP_ALL_ENABLE_MASK; 1566 1567 rc = ffa_notification_get(flags, &bitmaps); 1568 if (rc) { 1569 pr_err("Failed to retrieve notifications with %d!\n", rc); 1570 return; 1571 } 1572 1573 handle_fwk_notif_callbacks(SPM_FRAMEWORK_BITMAP(bitmaps.arch_map)); 1574 handle_fwk_notif_callbacks(NS_HYP_FRAMEWORK_BITMAP(bitmaps.arch_map)); 1575 handle_notif_callbacks(bitmaps.vm_map, NON_SECURE_VM); 1576 handle_notif_callbacks(bitmaps.sp_map, SECURE_PARTITION); 1577 } 1578 1579 static void 1580 ffa_self_notif_handle(u16 vcpu, bool is_per_vcpu, void *cb_data) 1581 { 1582 struct ffa_drv_info *info = cb_data; 1583 1584 if (!is_per_vcpu) 1585 notif_get_and_handle(info); 1586 else 1587 smp_call_function_single(vcpu, notif_get_and_handle, info, 0); 1588 } 1589 1590 static void notif_pcpu_irq_work_fn(struct work_struct *work) 1591 { 1592 struct ffa_pcpu_irq *pcpu = container_of(work, struct ffa_pcpu_irq, 1593 notif_pcpu_work); 1594 struct ffa_drv_info *info = pcpu->info; 1595 1596 notif_get_and_handle(info); 1597 } 1598 1599 static const struct ffa_info_ops ffa_drv_info_ops = { 1600 .api_version_get = ffa_api_version_get, 1601 .partition_info_get = ffa_partition_info_get, 1602 }; 1603 1604 static const struct ffa_msg_ops ffa_drv_msg_ops = { 1605 .mode_32bit_set = ffa_mode_32bit_set, 1606 .sync_send_receive = ffa_sync_send_receive, 1607 .indirect_send = ffa_indirect_msg_send, 1608 .sync_send_receive2 = ffa_sync_send_receive2, 1609 }; 1610 1611 static const struct ffa_mem_ops ffa_drv_mem_ops = { 1612 .memory_reclaim = ffa_memory_reclaim, 1613 .memory_share = ffa_memory_share, 1614 .memory_lend = ffa_memory_lend, 1615 }; 1616 1617 static const struct ffa_cpu_ops ffa_drv_cpu_ops = { 1618 .run = ffa_run, 1619 }; 1620 1621 static const struct ffa_notifier_ops ffa_drv_notifier_ops = { 1622 .sched_recv_cb_register = ffa_sched_recv_cb_register, 1623 .sched_recv_cb_unregister = ffa_sched_recv_cb_unregister, 1624 .notify_request = ffa_notify_request, 1625 .notify_relinquish = ffa_notify_relinquish, 1626 .fwk_notify_request = ffa_fwk_notify_request, 1627 .fwk_notify_relinquish = ffa_fwk_notify_relinquish, 1628 .notify_send = ffa_notify_send, 1629 }; 1630 1631 static const struct ffa_ops ffa_drv_ops = { 1632 .info_ops = &ffa_drv_info_ops, 1633 .msg_ops = &ffa_drv_msg_ops, 1634 .mem_ops = &ffa_drv_mem_ops, 1635 .cpu_ops = &ffa_drv_cpu_ops, 1636 .notifier_ops = &ffa_drv_notifier_ops, 1637 }; 1638 1639 void ffa_device_match_uuid(struct ffa_device *ffa_dev, const uuid_t *uuid) 1640 { 1641 int count, idx; 1642 struct ffa_partition_info *pbuf, *tpbuf; 1643 1644 count = ffa_partition_probe(uuid, &pbuf); 1645 if (count <= 0) 1646 return; 1647 1648 for (idx = 0, tpbuf = pbuf; idx < count; idx++, tpbuf++) 1649 if (tpbuf->id == ffa_dev->vm_id) 1650 uuid_copy(&ffa_dev->uuid, uuid); 1651 kfree(pbuf); 1652 } 1653 1654 static int 1655 ffa_bus_notifier(struct notifier_block *nb, unsigned long action, void *data) 1656 { 1657 struct device *dev = data; 1658 struct ffa_device *fdev = to_ffa_dev(dev); 1659 1660 if (action == BUS_NOTIFY_BIND_DRIVER) { 1661 struct ffa_driver *ffa_drv = to_ffa_driver(dev->driver); 1662 const struct ffa_device_id *id_table = ffa_drv->id_table; 1663 1664 /* 1665 * FF-A v1.1 provides UUID for each partition as part of the 1666 * discovery API, the discovered UUID must be populated in the 1667 * device's UUID and there is no need to workaround by copying 1668 * the same from the driver table. 1669 */ 1670 if (uuid_is_null(&fdev->uuid)) 1671 ffa_device_match_uuid(fdev, &id_table->uuid); 1672 1673 return NOTIFY_OK; 1674 } 1675 1676 return NOTIFY_DONE; 1677 } 1678 1679 static struct notifier_block ffa_bus_nb = { 1680 .notifier_call = ffa_bus_notifier, 1681 }; 1682 1683 static void ffa_bus_notifier_unregister(void) 1684 { 1685 if (!drv_info->bus_notifier_registered) 1686 return; 1687 1688 bus_unregister_notifier(&ffa_bus_type, &ffa_bus_nb); 1689 drv_info->bus_notifier_registered = false; 1690 } 1691 1692 static int ffa_xa_add_partition_info(struct ffa_device *dev) 1693 { 1694 struct ffa_dev_part_info *info; 1695 struct list_head *head, *phead; 1696 int ret = -ENOMEM; 1697 1698 phead = xa_load(&drv_info->partition_info, dev->vm_id); 1699 if (phead) { 1700 head = phead; 1701 list_for_each_entry(info, head, node) { 1702 if (info->dev == dev) { 1703 pr_err("%s: duplicate dev %p part ID 0x%x\n", 1704 __func__, dev, dev->vm_id); 1705 return -EEXIST; 1706 } 1707 } 1708 } 1709 1710 info = kzalloc_obj(*info); 1711 if (!info) 1712 return ret; 1713 1714 rwlock_init(&info->rw_lock); 1715 info->dev = dev; 1716 1717 if (!phead) { 1718 phead = kzalloc_obj(*phead); 1719 if (!phead) 1720 goto free_out; 1721 1722 INIT_LIST_HEAD(phead); 1723 1724 ret = xa_insert(&drv_info->partition_info, dev->vm_id, phead, 1725 GFP_KERNEL); 1726 if (ret) { 1727 pr_err("%s: failed to save part ID 0x%x Ret:%d\n", 1728 __func__, dev->vm_id, ret); 1729 goto free_out; 1730 } 1731 } 1732 list_add(&info->node, phead); 1733 return 0; 1734 1735 free_out: 1736 kfree(phead); 1737 kfree(info); 1738 return ret; 1739 } 1740 1741 static int ffa_setup_host_partition(int vm_id) 1742 { 1743 struct ffa_partition_info buf = { 0 }; 1744 struct ffa_device *ffa_dev; 1745 int ret; 1746 1747 buf.id = vm_id; 1748 ffa_dev = ffa_device_register(&buf, &ffa_drv_ops, &ffa_pdev->dev); 1749 if (!ffa_dev) { 1750 pr_err("%s: failed to register host partition ID 0x%x\n", 1751 __func__, vm_id); 1752 return -EINVAL; 1753 } 1754 1755 ret = ffa_xa_add_partition_info(ffa_dev); 1756 if (ret) 1757 return ret; 1758 1759 if (ffa_notifications_disabled()) 1760 return 0; 1761 1762 ret = ffa_sched_recv_cb_update(ffa_dev, ffa_self_notif_handle, 1763 drv_info, true); 1764 if (ret) 1765 pr_info("Failed to register driver sched callback %d\n", ret); 1766 1767 return ret; 1768 } 1769 1770 static void ffa_partitions_cleanup(void) 1771 { 1772 struct list_head *phead; 1773 unsigned long idx; 1774 1775 ffa_bus_notifier_unregister(); 1776 1777 /* Clean up/free all registered devices */ 1778 ffa_devices_unregister(); 1779 1780 xa_for_each(&drv_info->partition_info, idx, phead) { 1781 struct ffa_dev_part_info *info, *tmp; 1782 1783 xa_erase(&drv_info->partition_info, idx); 1784 list_for_each_entry_safe(info, tmp, phead, node) { 1785 list_del(&info->node); 1786 kfree(info); 1787 } 1788 kfree(phead); 1789 } 1790 1791 xa_destroy(&drv_info->partition_info); 1792 } 1793 1794 static int ffa_setup_partitions(void) 1795 { 1796 int count, idx, ret; 1797 struct ffa_device *ffa_dev; 1798 struct ffa_partition_info *pbuf, *tpbuf; 1799 1800 if (!FFA_PART_INFO_HAS_UUID_IN_RESP(drv_info->version)) { 1801 ret = bus_register_notifier(&ffa_bus_type, &ffa_bus_nb); 1802 if (ret) 1803 pr_err("Failed to register FF-A bus notifiers\n"); 1804 else 1805 drv_info->bus_notifier_registered = true; 1806 } 1807 1808 count = ffa_partition_probe(&uuid_null, &pbuf); 1809 if (count <= 0) { 1810 pr_info("%s: No partitions found, error %d\n", __func__, count); 1811 ffa_bus_notifier_unregister(); 1812 return -EINVAL; 1813 } 1814 1815 xa_init(&drv_info->partition_info); 1816 for (idx = 0, tpbuf = pbuf; idx < count; idx++, tpbuf++) { 1817 /* Note that if the UUID will be uuid_null, that will require 1818 * ffa_bus_notifier() to find the UUID of this partition id 1819 * with help of ffa_device_match_uuid(). FF-A v1.1 and above 1820 * provides UUID here for each partition as part of the 1821 * discovery API and the same is passed. 1822 */ 1823 ffa_dev = ffa_device_register(tpbuf, &ffa_drv_ops, 1824 &ffa_pdev->dev); 1825 if (!ffa_dev) { 1826 pr_err("%s: failed to register partition ID 0x%x\n", 1827 __func__, tpbuf->id); 1828 continue; 1829 } 1830 1831 if (FFA_PART_INFO_HAS_EXEC_STATE_IN_RESP(drv_info->version) && 1832 !(tpbuf->properties & FFA_PARTITION_AARCH64_EXEC)) 1833 ffa_mode_32bit_set(ffa_dev); 1834 1835 if (ffa_xa_add_partition_info(ffa_dev)) { 1836 ffa_device_unregister(ffa_dev); 1837 continue; 1838 } 1839 } 1840 1841 kfree(pbuf); 1842 1843 /* 1844 * Check if the host is already added as part of partition info 1845 * No multiple UUID possible for the host, so just checking if 1846 * there is an entry will suffice 1847 */ 1848 if (xa_load(&drv_info->partition_info, drv_info->vm_id)) 1849 return 0; 1850 1851 /* Allocate for the host */ 1852 ret = ffa_setup_host_partition(drv_info->vm_id); 1853 if (ret) 1854 ffa_partitions_cleanup(); 1855 1856 return ret; 1857 } 1858 1859 /* FFA FEATURE IDs */ 1860 #define FFA_FEAT_NOTIFICATION_PENDING_INT (1) 1861 #define FFA_FEAT_SCHEDULE_RECEIVER_INT (2) 1862 #define FFA_FEAT_MANAGED_EXIT_INT (3) 1863 1864 static irqreturn_t ffa_sched_recv_irq_handler(int irq, void *irq_data) 1865 { 1866 struct ffa_pcpu_irq *pcpu = irq_data; 1867 struct ffa_drv_info *info = pcpu->info; 1868 1869 queue_work(info->notif_pcpu_wq, &info->sched_recv_irq_work); 1870 1871 return IRQ_HANDLED; 1872 } 1873 1874 static irqreturn_t notif_pend_irq_handler(int irq, void *irq_data) 1875 { 1876 struct ffa_pcpu_irq *pcpu = irq_data; 1877 struct ffa_drv_info *info = pcpu->info; 1878 1879 queue_work_on(smp_processor_id(), info->notif_pcpu_wq, 1880 &pcpu->notif_pcpu_work); 1881 1882 return IRQ_HANDLED; 1883 } 1884 1885 static void ffa_sched_recv_irq_work_fn(struct work_struct *work) 1886 { 1887 ffa_notification_info_get(); 1888 } 1889 1890 static int ffa_irq_map(u32 id) 1891 { 1892 char *err_str; 1893 int ret, irq, intid; 1894 1895 if (id == FFA_FEAT_NOTIFICATION_PENDING_INT) 1896 err_str = "Notification Pending Interrupt"; 1897 else if (id == FFA_FEAT_SCHEDULE_RECEIVER_INT) 1898 err_str = "Schedule Receiver Interrupt"; 1899 else 1900 err_str = "Unknown ID"; 1901 1902 /* The returned intid is assumed to be SGI donated to NS world */ 1903 ret = ffa_features(id, 0, &intid, NULL); 1904 if (ret < 0) { 1905 if (ret != -EOPNOTSUPP) 1906 pr_err("Failed to retrieve FF-A %s %u\n", err_str, id); 1907 return ret; 1908 } 1909 1910 if (acpi_disabled) { 1911 struct of_phandle_args oirq = {}; 1912 struct device_node *gic; 1913 1914 /* Only GICv3 supported currently with the device tree */ 1915 gic = of_find_compatible_node(NULL, NULL, "arm,gic-v3"); 1916 if (!gic) 1917 return -ENXIO; 1918 1919 oirq.np = gic; 1920 oirq.args_count = 1; 1921 oirq.args[0] = intid; 1922 irq = irq_create_of_mapping(&oirq); 1923 of_node_put(gic); 1924 #ifdef CONFIG_ACPI 1925 } else { 1926 irq = acpi_register_gsi(NULL, intid, ACPI_EDGE_SENSITIVE, 1927 ACPI_ACTIVE_HIGH); 1928 #endif 1929 } 1930 1931 if (irq <= 0) { 1932 pr_err("Failed to create IRQ mapping!\n"); 1933 return -ENODATA; 1934 } 1935 1936 return irq; 1937 } 1938 1939 static void ffa_irq_unmap(unsigned int irq) 1940 { 1941 if (!irq) 1942 return; 1943 irq_dispose_mapping(irq); 1944 } 1945 1946 static int ffa_cpuhp_pcpu_irq_enable(unsigned int cpu) 1947 { 1948 if (drv_info->sched_recv_irq) 1949 enable_percpu_irq(drv_info->sched_recv_irq, IRQ_TYPE_NONE); 1950 if (drv_info->notif_pend_irq) 1951 enable_percpu_irq(drv_info->notif_pend_irq, IRQ_TYPE_NONE); 1952 return 0; 1953 } 1954 1955 static int ffa_cpuhp_pcpu_irq_disable(unsigned int cpu) 1956 { 1957 if (drv_info->sched_recv_irq) 1958 disable_percpu_irq(drv_info->sched_recv_irq); 1959 if (drv_info->notif_pend_irq) 1960 disable_percpu_irq(drv_info->notif_pend_irq); 1961 return 0; 1962 } 1963 1964 static void ffa_uninit_pcpu_irq(void) 1965 { 1966 if (drv_info->cpuhp_state) { 1967 cpuhp_remove_state(drv_info->cpuhp_state); 1968 drv_info->cpuhp_state = 0; 1969 } 1970 1971 if (drv_info->notif_pcpu_wq) { 1972 destroy_workqueue(drv_info->notif_pcpu_wq); 1973 drv_info->notif_pcpu_wq = NULL; 1974 } 1975 1976 if (drv_info->sched_recv_irq) 1977 free_percpu_irq(drv_info->sched_recv_irq, drv_info->irq_pcpu); 1978 1979 if (drv_info->notif_pend_irq) 1980 free_percpu_irq(drv_info->notif_pend_irq, drv_info->irq_pcpu); 1981 1982 if (drv_info->irq_pcpu) { 1983 free_percpu(drv_info->irq_pcpu); 1984 drv_info->irq_pcpu = NULL; 1985 } 1986 } 1987 1988 static int ffa_init_pcpu_irq(void) 1989 { 1990 struct ffa_pcpu_irq __percpu *irq_pcpu; 1991 int ret, cpu; 1992 1993 irq_pcpu = alloc_percpu(struct ffa_pcpu_irq); 1994 if (!irq_pcpu) 1995 return -ENOMEM; 1996 1997 for_each_present_cpu(cpu) { 1998 per_cpu_ptr(irq_pcpu, cpu)->info = drv_info; 1999 INIT_WORK(&per_cpu_ptr(irq_pcpu, cpu)->notif_pcpu_work, 2000 notif_pcpu_irq_work_fn); 2001 } 2002 2003 drv_info->irq_pcpu = irq_pcpu; 2004 2005 if (drv_info->sched_recv_irq) { 2006 ret = request_percpu_irq(drv_info->sched_recv_irq, 2007 ffa_sched_recv_irq_handler, 2008 "ARM-FFA-SRI", irq_pcpu); 2009 if (ret) { 2010 pr_err("Error registering percpu SRI nIRQ %d : %d\n", 2011 drv_info->sched_recv_irq, ret); 2012 drv_info->sched_recv_irq = 0; 2013 return ret; 2014 } 2015 } 2016 2017 if (drv_info->notif_pend_irq) { 2018 ret = request_percpu_irq(drv_info->notif_pend_irq, 2019 notif_pend_irq_handler, 2020 "ARM-FFA-NPI", irq_pcpu); 2021 if (ret) { 2022 pr_err("Error registering percpu NPI nIRQ %d : %d\n", 2023 drv_info->notif_pend_irq, ret); 2024 drv_info->notif_pend_irq = 0; 2025 return ret; 2026 } 2027 } 2028 2029 INIT_WORK(&drv_info->sched_recv_irq_work, ffa_sched_recv_irq_work_fn); 2030 drv_info->notif_pcpu_wq = create_workqueue("ffa_pcpu_irq_notification"); 2031 if (!drv_info->notif_pcpu_wq) 2032 return -EINVAL; 2033 2034 ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "ffa/pcpu-irq:starting", 2035 ffa_cpuhp_pcpu_irq_enable, 2036 ffa_cpuhp_pcpu_irq_disable); 2037 2038 if (ret < 0) 2039 return ret; 2040 2041 drv_info->cpuhp_state = ret; 2042 return 0; 2043 } 2044 2045 static void ffa_notifications_cleanup(void) 2046 { 2047 ffa_uninit_pcpu_irq(); 2048 ffa_irq_unmap(drv_info->sched_recv_irq); 2049 drv_info->sched_recv_irq = 0; 2050 ffa_irq_unmap(drv_info->notif_pend_irq); 2051 drv_info->notif_pend_irq = 0; 2052 2053 if (drv_info->bitmap_created) { 2054 ffa_notification_bitmap_destroy(); 2055 drv_info->bitmap_created = false; 2056 } 2057 drv_info->notif_enabled = false; 2058 } 2059 2060 static void ffa_notifications_setup(void) 2061 { 2062 int ret; 2063 2064 ret = ffa_features(FFA_NOTIFICATION_BITMAP_CREATE, 0, NULL, NULL); 2065 if (!ret) { 2066 ret = ffa_notification_bitmap_create(); 2067 if (ret) { 2068 pr_err("Notification bitmap create error %d\n", ret); 2069 return; 2070 } 2071 2072 drv_info->bitmap_created = true; 2073 } 2074 2075 ret = ffa_irq_map(FFA_FEAT_SCHEDULE_RECEIVER_INT); 2076 if (ret > 0) 2077 drv_info->sched_recv_irq = ret; 2078 2079 ret = ffa_irq_map(FFA_FEAT_NOTIFICATION_PENDING_INT); 2080 if (ret > 0) 2081 drv_info->notif_pend_irq = ret; 2082 2083 if (!drv_info->sched_recv_irq && !drv_info->notif_pend_irq) 2084 goto cleanup; 2085 2086 ret = ffa_init_pcpu_irq(); 2087 if (ret) 2088 goto cleanup; 2089 2090 hash_init(drv_info->notifier_hash); 2091 rwlock_init(&drv_info->notify_lock); 2092 2093 drv_info->notif_enabled = true; 2094 return; 2095 cleanup: 2096 pr_info("Notification setup failed %d, not enabled\n", ret); 2097 ffa_notifications_cleanup(); 2098 } 2099 2100 static int ffa_probe(struct platform_device *pdev) 2101 { 2102 int ret; 2103 u32 buf_sz; 2104 size_t rxtx_bufsz = SZ_4K; 2105 2106 if (IS_BUILTIN(CONFIG_ARM_FFA_TRANSPORT) && 2107 is_protected_kvm_enabled() && !is_pkvm_initialized()) 2108 return -EPROBE_DEFER; 2109 2110 ret = ffa_transport_init(&invoke_ffa_fn); 2111 if (ret) 2112 return ret == -EOPNOTSUPP ? -ENODEV : ret; 2113 2114 drv_info = kzalloc_obj(*drv_info); 2115 if (!drv_info) 2116 return -ENOMEM; 2117 platform_set_drvdata(pdev, drv_info); 2118 2119 ret = ffa_version_check(&drv_info->version); 2120 if (ret) { 2121 if (ret == -EOPNOTSUPP) 2122 ret = -ENODEV; 2123 goto free_drv_info; 2124 } 2125 2126 if (ffa_id_get(&drv_info->vm_id)) { 2127 pr_err("failed to obtain VM id for self\n"); 2128 ret = -ENODEV; 2129 goto free_drv_info; 2130 } 2131 2132 ret = ffa_features(FFA_FN_NATIVE(RXTX_MAP), 0, &buf_sz, NULL); 2133 if (!ret) { 2134 if (RXTX_MAP_MIN_BUFSZ(buf_sz) == 1) 2135 rxtx_bufsz = SZ_64K; 2136 else if (RXTX_MAP_MIN_BUFSZ(buf_sz) == 2) 2137 rxtx_bufsz = SZ_16K; 2138 else 2139 rxtx_bufsz = SZ_4K; 2140 } 2141 2142 rxtx_bufsz = PAGE_ALIGN(rxtx_bufsz); 2143 drv_info->rxtx_bufsz = rxtx_bufsz; 2144 drv_info->rx_buffer = alloc_pages_exact(rxtx_bufsz, GFP_KERNEL); 2145 if (!drv_info->rx_buffer) { 2146 ret = -ENOMEM; 2147 goto free_drv_info; 2148 } 2149 2150 drv_info->tx_buffer = alloc_pages_exact(rxtx_bufsz, GFP_KERNEL); 2151 if (!drv_info->tx_buffer) { 2152 ret = -ENOMEM; 2153 goto free_pages; 2154 } 2155 2156 ret = ffa_rxtx_map(virt_to_phys(drv_info->tx_buffer), 2157 virt_to_phys(drv_info->rx_buffer), 2158 rxtx_bufsz / FFA_PAGE_SIZE); 2159 if (ret) { 2160 pr_err("failed to register FFA RxTx buffers\n"); 2161 goto free_pages; 2162 } 2163 2164 mutex_init(&drv_info->rx_lock); 2165 mutex_init(&drv_info->tx_lock); 2166 2167 ffa_drvinfo_flags_init(); 2168 2169 ffa_notifications_setup(); 2170 2171 ret = ffa_setup_partitions(); 2172 if (!ret) 2173 return ret; 2174 2175 pr_err("failed to setup partitions\n"); 2176 ffa_notifications_cleanup(); 2177 ffa_rxtx_unmap(); 2178 free_pages: 2179 if (drv_info->tx_buffer) 2180 free_pages_exact(drv_info->tx_buffer, rxtx_bufsz); 2181 free_pages_exact(drv_info->rx_buffer, rxtx_bufsz); 2182 free_drv_info: 2183 platform_set_drvdata(pdev, NULL); 2184 kfree(drv_info); 2185 drv_info = NULL; 2186 return ret; 2187 } 2188 2189 static void ffa_remove(struct platform_device *pdev) 2190 { 2191 struct ffa_drv_info *info = platform_get_drvdata(pdev); 2192 2193 ffa_notifications_cleanup(); 2194 ffa_partitions_cleanup(); 2195 ffa_rxtx_unmap(); 2196 free_pages_exact(info->tx_buffer, info->rxtx_bufsz); 2197 free_pages_exact(info->rx_buffer, info->rxtx_bufsz); 2198 kfree(info); 2199 platform_set_drvdata(pdev, NULL); 2200 drv_info = NULL; 2201 } 2202 2203 static struct platform_driver ffa_driver = { 2204 .probe = ffa_probe, 2205 .remove = ffa_remove, 2206 .driver = { 2207 .name = FFA_PLATFORM_NAME, 2208 }, 2209 }; 2210 2211 static int __init ffa_init(void) 2212 { 2213 int ret; 2214 2215 ffa_pdev = platform_device_register_simple(FFA_PLATFORM_NAME, 2216 PLATFORM_DEVID_NONE, 2217 NULL, 0); 2218 if (IS_ERR(ffa_pdev)) 2219 return PTR_ERR(ffa_pdev); 2220 2221 ret = platform_driver_register(&ffa_driver); 2222 if (ret) 2223 platform_device_unregister(ffa_pdev); 2224 2225 return ret; 2226 } 2227 module_init(ffa_init); 2228 2229 static void __exit ffa_exit(void) 2230 { 2231 platform_device_unregister(ffa_pdev); 2232 platform_driver_unregister(&ffa_driver); 2233 } 2234 module_exit(ffa_exit); 2235 2236 MODULE_ALIAS("arm-ffa"); 2237 MODULE_AUTHOR("Sudeep Holla <sudeep.holla@arm.com>"); 2238 MODULE_DESCRIPTION("Arm FF-A interface driver"); 2239 MODULE_LICENSE("GPL v2"); 2240