1 // SPDX-License-Identifier: GPL-2.0-only OR MIT
2 /*
3 * Apple RTKit IPC library
4 * Copyright (C) The Asahi Linux Contributors
5 */
6
7 #include "rtkit-internal.h"
8
9 enum {
10 APPLE_RTKIT_PWR_STATE_OFF = 0x00, /* power off, cannot be restarted */
11 APPLE_RTKIT_PWR_STATE_SLEEP = 0x01, /* sleeping, can be restarted */
12 APPLE_RTKIT_PWR_STATE_IDLE = 0x201, /* sleeping, retain state */
13 APPLE_RTKIT_PWR_STATE_QUIESCED = 0x10, /* running but no communication */
14 APPLE_RTKIT_PWR_STATE_ON = 0x20, /* normal operating state */
15 APPLE_RTKIT_PWR_STATE_INIT = 0x220, /* init after starting the coproc */
16 };
17
18 enum {
19 APPLE_RTKIT_EP_MGMT = 0,
20 APPLE_RTKIT_EP_CRASHLOG = 1,
21 APPLE_RTKIT_EP_SYSLOG = 2,
22 APPLE_RTKIT_EP_DEBUG = 3,
23 APPLE_RTKIT_EP_IOREPORT = 4,
24 APPLE_RTKIT_EP_OSLOG = 8,
25 };
26
27 #define APPLE_RTKIT_MGMT_TYPE GENMASK_ULL(59, 52)
28
29 enum {
30 APPLE_RTKIT_MGMT_HELLO = 1,
31 APPLE_RTKIT_MGMT_HELLO_REPLY = 2,
32 APPLE_RTKIT_MGMT_STARTEP = 5,
33 APPLE_RTKIT_MGMT_SET_IOP_PWR_STATE = 6,
34 APPLE_RTKIT_MGMT_SET_IOP_PWR_STATE_ACK = 7,
35 APPLE_RTKIT_MGMT_EPMAP = 8,
36 APPLE_RTKIT_MGMT_EPMAP_REPLY = 8,
37 APPLE_RTKIT_MGMT_SET_AP_PWR_STATE = 0xb,
38 APPLE_RTKIT_MGMT_SET_AP_PWR_STATE_ACK = 0xb,
39 };
40
41 #define APPLE_RTKIT_MGMT_HELLO_MINVER GENMASK_ULL(15, 0)
42 #define APPLE_RTKIT_MGMT_HELLO_MAXVER GENMASK_ULL(31, 16)
43
44 #define APPLE_RTKIT_MGMT_EPMAP_LAST BIT_ULL(51)
45 #define APPLE_RTKIT_MGMT_EPMAP_BASE GENMASK_ULL(34, 32)
46 #define APPLE_RTKIT_MGMT_EPMAP_BITMAP GENMASK_ULL(31, 0)
47
48 #define APPLE_RTKIT_MGMT_EPMAP_REPLY_MORE BIT_ULL(0)
49
50 #define APPLE_RTKIT_MGMT_STARTEP_EP GENMASK_ULL(39, 32)
51 #define APPLE_RTKIT_MGMT_STARTEP_FLAG BIT_ULL(1)
52
53 #define APPLE_RTKIT_MGMT_PWR_STATE GENMASK_ULL(15, 0)
54
55 #define APPLE_RTKIT_CRASHLOG_CRASH 1
56
57 #define APPLE_RTKIT_BUFFER_REQUEST 1
58 #define APPLE_RTKIT_BUFFER_REQUEST_SIZE GENMASK_ULL(51, 44)
59 #define APPLE_RTKIT_BUFFER_REQUEST_IOVA GENMASK_ULL(43, 0)
60
61 #define APPLE_RTKIT_SYSLOG_TYPE GENMASK_ULL(59, 52)
62
63 #define APPLE_RTKIT_SYSLOG_LOG 5
64
65 #define APPLE_RTKIT_SYSLOG_INIT 8
66 #define APPLE_RTKIT_SYSLOG_N_ENTRIES GENMASK_ULL(7, 0)
67 #define APPLE_RTKIT_SYSLOG_MSG_SIZE GENMASK_ULL(31, 24)
68
69 #define APPLE_RTKIT_OSLOG_TYPE GENMASK_ULL(63, 56)
70 #define APPLE_RTKIT_OSLOG_BUFFER_REQUEST 1
71 #define APPLE_RTKIT_OSLOG_SIZE GENMASK_ULL(55, 36)
72 #define APPLE_RTKIT_OSLOG_IOVA GENMASK_ULL(35, 0)
73
74 #define APPLE_RTKIT_MIN_SUPPORTED_VERSION 11
75 #define APPLE_RTKIT_MAX_SUPPORTED_VERSION 12
76
77 struct apple_rtkit_rx_work {
78 struct apple_rtkit *rtk;
79 u8 ep;
80 u64 msg;
81 struct work_struct work;
82 };
83
apple_rtkit_is_running(struct apple_rtkit * rtk)84 bool apple_rtkit_is_running(struct apple_rtkit *rtk)
85 {
86 if (rtk->crashed)
87 return false;
88 if ((rtk->iop_power_state & 0xff) != APPLE_RTKIT_PWR_STATE_ON)
89 return false;
90 if ((rtk->ap_power_state & 0xff) != APPLE_RTKIT_PWR_STATE_ON)
91 return false;
92 return true;
93 }
94 EXPORT_SYMBOL_GPL(apple_rtkit_is_running);
95
apple_rtkit_is_crashed(struct apple_rtkit * rtk)96 bool apple_rtkit_is_crashed(struct apple_rtkit *rtk)
97 {
98 return rtk->crashed;
99 }
100 EXPORT_SYMBOL_GPL(apple_rtkit_is_crashed);
101
apple_rtkit_management_send(struct apple_rtkit * rtk,u8 type,u64 msg)102 static int apple_rtkit_management_send(struct apple_rtkit *rtk, u8 type,
103 u64 msg)
104 {
105 int ret;
106
107 msg &= ~APPLE_RTKIT_MGMT_TYPE;
108 msg |= FIELD_PREP(APPLE_RTKIT_MGMT_TYPE, type);
109 ret = apple_rtkit_send_message(rtk, APPLE_RTKIT_EP_MGMT, msg, NULL, false);
110
111 if (ret)
112 dev_err(rtk->dev, "RTKit: Failed to send management message: %d\n", ret);
113
114 return ret;
115 }
116
apple_rtkit_management_rx_hello(struct apple_rtkit * rtk,u64 msg)117 static void apple_rtkit_management_rx_hello(struct apple_rtkit *rtk, u64 msg)
118 {
119 u64 reply;
120
121 int min_ver = FIELD_GET(APPLE_RTKIT_MGMT_HELLO_MINVER, msg);
122 int max_ver = FIELD_GET(APPLE_RTKIT_MGMT_HELLO_MAXVER, msg);
123 int want_ver = min(APPLE_RTKIT_MAX_SUPPORTED_VERSION, max_ver);
124
125 dev_dbg(rtk->dev, "RTKit: Min ver %d, max ver %d\n", min_ver, max_ver);
126
127 if (min_ver > APPLE_RTKIT_MAX_SUPPORTED_VERSION) {
128 dev_err(rtk->dev, "RTKit: Firmware min version %d is too new\n",
129 min_ver);
130 goto abort_boot;
131 }
132
133 if (max_ver < APPLE_RTKIT_MIN_SUPPORTED_VERSION) {
134 dev_err(rtk->dev, "RTKit: Firmware max version %d is too old\n",
135 max_ver);
136 goto abort_boot;
137 }
138
139 dev_info(rtk->dev, "RTKit: Initializing (protocol version %d)\n",
140 want_ver);
141 rtk->version = want_ver;
142
143 reply = FIELD_PREP(APPLE_RTKIT_MGMT_HELLO_MINVER, want_ver);
144 reply |= FIELD_PREP(APPLE_RTKIT_MGMT_HELLO_MAXVER, want_ver);
145 apple_rtkit_management_send(rtk, APPLE_RTKIT_MGMT_HELLO_REPLY, reply);
146
147 return;
148
149 abort_boot:
150 rtk->boot_result = -EINVAL;
151 complete_all(&rtk->epmap_completion);
152 }
153
apple_rtkit_management_rx_epmap(struct apple_rtkit * rtk,u64 msg)154 static void apple_rtkit_management_rx_epmap(struct apple_rtkit *rtk, u64 msg)
155 {
156 int i, ep;
157 u64 reply;
158 unsigned long bitmap = FIELD_GET(APPLE_RTKIT_MGMT_EPMAP_BITMAP, msg);
159 u32 base = FIELD_GET(APPLE_RTKIT_MGMT_EPMAP_BASE, msg);
160
161 dev_dbg(rtk->dev,
162 "RTKit: received endpoint bitmap 0x%lx with base 0x%x\n",
163 bitmap, base);
164
165 for_each_set_bit(i, &bitmap, 32) {
166 ep = 32 * base + i;
167 dev_dbg(rtk->dev, "RTKit: Discovered endpoint 0x%02x\n", ep);
168 set_bit(ep, rtk->endpoints);
169 }
170
171 reply = FIELD_PREP(APPLE_RTKIT_MGMT_EPMAP_BASE, base);
172 if (msg & APPLE_RTKIT_MGMT_EPMAP_LAST)
173 reply |= APPLE_RTKIT_MGMT_EPMAP_LAST;
174 else
175 reply |= APPLE_RTKIT_MGMT_EPMAP_REPLY_MORE;
176
177 apple_rtkit_management_send(rtk, APPLE_RTKIT_MGMT_EPMAP_REPLY, reply);
178
179 if (!(msg & APPLE_RTKIT_MGMT_EPMAP_LAST))
180 return;
181
182 for_each_set_bit(ep, rtk->endpoints, APPLE_RTKIT_APP_ENDPOINT_START) {
183 switch (ep) {
184 /* the management endpoint is started by default */
185 case APPLE_RTKIT_EP_MGMT:
186 break;
187
188 /* without starting these RTKit refuses to boot */
189 case APPLE_RTKIT_EP_SYSLOG:
190 case APPLE_RTKIT_EP_CRASHLOG:
191 case APPLE_RTKIT_EP_DEBUG:
192 case APPLE_RTKIT_EP_IOREPORT:
193 case APPLE_RTKIT_EP_OSLOG:
194 dev_dbg(rtk->dev,
195 "RTKit: Starting system endpoint 0x%02x\n", ep);
196 apple_rtkit_start_ep(rtk, ep);
197 break;
198
199 default:
200 dev_warn(rtk->dev,
201 "RTKit: Unknown system endpoint: 0x%02x\n",
202 ep);
203 }
204 }
205
206 rtk->boot_result = 0;
207 complete_all(&rtk->epmap_completion);
208 }
209
apple_rtkit_management_rx_iop_pwr_ack(struct apple_rtkit * rtk,u64 msg)210 static void apple_rtkit_management_rx_iop_pwr_ack(struct apple_rtkit *rtk,
211 u64 msg)
212 {
213 unsigned int new_state = FIELD_GET(APPLE_RTKIT_MGMT_PWR_STATE, msg);
214
215 dev_dbg(rtk->dev, "RTKit: IOP power state transition: 0x%x -> 0x%x\n",
216 rtk->iop_power_state, new_state);
217 rtk->iop_power_state = new_state;
218
219 complete_all(&rtk->iop_pwr_ack_completion);
220 }
221
apple_rtkit_management_rx_ap_pwr_ack(struct apple_rtkit * rtk,u64 msg)222 static void apple_rtkit_management_rx_ap_pwr_ack(struct apple_rtkit *rtk,
223 u64 msg)
224 {
225 unsigned int new_state = FIELD_GET(APPLE_RTKIT_MGMT_PWR_STATE, msg);
226
227 dev_dbg(rtk->dev, "RTKit: AP power state transition: 0x%x -> 0x%x\n",
228 rtk->ap_power_state, new_state);
229 rtk->ap_power_state = new_state;
230
231 complete_all(&rtk->ap_pwr_ack_completion);
232 }
233
apple_rtkit_management_rx(struct apple_rtkit * rtk,u64 msg)234 static void apple_rtkit_management_rx(struct apple_rtkit *rtk, u64 msg)
235 {
236 u8 type = FIELD_GET(APPLE_RTKIT_MGMT_TYPE, msg);
237
238 switch (type) {
239 case APPLE_RTKIT_MGMT_HELLO:
240 apple_rtkit_management_rx_hello(rtk, msg);
241 break;
242 case APPLE_RTKIT_MGMT_EPMAP:
243 apple_rtkit_management_rx_epmap(rtk, msg);
244 break;
245 case APPLE_RTKIT_MGMT_SET_IOP_PWR_STATE_ACK:
246 apple_rtkit_management_rx_iop_pwr_ack(rtk, msg);
247 break;
248 case APPLE_RTKIT_MGMT_SET_AP_PWR_STATE_ACK:
249 apple_rtkit_management_rx_ap_pwr_ack(rtk, msg);
250 break;
251 default:
252 dev_warn(
253 rtk->dev,
254 "RTKit: unknown management message: 0x%llx (type: 0x%02x)\n",
255 msg, type);
256 }
257 }
258
apple_rtkit_common_rx_get_buffer(struct apple_rtkit * rtk,struct apple_rtkit_shmem * buffer,u8 ep,u64 msg)259 static int apple_rtkit_common_rx_get_buffer(struct apple_rtkit *rtk,
260 struct apple_rtkit_shmem *buffer,
261 u8 ep, u64 msg)
262 {
263 u64 reply;
264 int err;
265
266 /* The different size vs. IOVA shifts look odd but are indeed correct this way */
267 if (ep == APPLE_RTKIT_EP_OSLOG) {
268 buffer->size = FIELD_GET(APPLE_RTKIT_OSLOG_SIZE, msg);
269 buffer->iova = FIELD_GET(APPLE_RTKIT_OSLOG_IOVA, msg) << 12;
270 } else {
271 buffer->size = FIELD_GET(APPLE_RTKIT_BUFFER_REQUEST_SIZE, msg) << 12;
272 buffer->iova = FIELD_GET(APPLE_RTKIT_BUFFER_REQUEST_IOVA, msg);
273 }
274
275 buffer->buffer = NULL;
276 buffer->iomem = NULL;
277 buffer->is_mapped = false;
278
279 dev_dbg(rtk->dev, "RTKit: buffer request for 0x%zx bytes at %pad\n",
280 buffer->size, &buffer->iova);
281
282 if (buffer->iova &&
283 (!rtk->ops->shmem_setup || !rtk->ops->shmem_destroy)) {
284 err = -EINVAL;
285 goto error;
286 }
287
288 if (rtk->ops->shmem_setup) {
289 err = rtk->ops->shmem_setup(rtk->cookie, buffer);
290 if (err)
291 goto error;
292 } else {
293 buffer->buffer = dma_alloc_coherent(rtk->dev, buffer->size,
294 &buffer->iova, GFP_KERNEL);
295 if (!buffer->buffer) {
296 err = -ENOMEM;
297 goto error;
298 }
299 }
300
301 if (!buffer->is_mapped) {
302 /* oslog uses different fields and needs a shifted IOVA instead of size */
303 if (ep == APPLE_RTKIT_EP_OSLOG) {
304 reply = FIELD_PREP(APPLE_RTKIT_OSLOG_TYPE,
305 APPLE_RTKIT_OSLOG_BUFFER_REQUEST);
306 reply |= FIELD_PREP(APPLE_RTKIT_OSLOG_SIZE, buffer->size);
307 reply |= FIELD_PREP(APPLE_RTKIT_OSLOG_IOVA,
308 buffer->iova >> 12);
309 } else {
310 reply = FIELD_PREP(APPLE_RTKIT_SYSLOG_TYPE,
311 APPLE_RTKIT_BUFFER_REQUEST);
312 reply |= FIELD_PREP(APPLE_RTKIT_BUFFER_REQUEST_SIZE,
313 buffer->size >> 12);
314 reply |= FIELD_PREP(APPLE_RTKIT_BUFFER_REQUEST_IOVA,
315 buffer->iova);
316 }
317 apple_rtkit_send_message(rtk, ep, reply, NULL, false);
318 }
319
320 return 0;
321
322 error:
323 dev_err(rtk->dev, "RTKit: failed buffer request for 0x%zx bytes (%d)\n",
324 buffer->size, err);
325
326 buffer->buffer = NULL;
327 buffer->iomem = NULL;
328 buffer->iova = 0;
329 buffer->size = 0;
330 buffer->is_mapped = false;
331 return err;
332 }
333
apple_rtkit_free_buffer(struct apple_rtkit * rtk,struct apple_rtkit_shmem * bfr)334 static void apple_rtkit_free_buffer(struct apple_rtkit *rtk,
335 struct apple_rtkit_shmem *bfr)
336 {
337 if (bfr->size == 0)
338 return;
339
340 if (rtk->ops->shmem_destroy)
341 rtk->ops->shmem_destroy(rtk->cookie, bfr);
342 else if (bfr->buffer)
343 dma_free_coherent(rtk->dev, bfr->size, bfr->buffer, bfr->iova);
344
345 bfr->buffer = NULL;
346 bfr->iomem = NULL;
347 bfr->iova = 0;
348 bfr->size = 0;
349 bfr->is_mapped = false;
350 }
351
apple_rtkit_memcpy(struct apple_rtkit * rtk,void * dst,struct apple_rtkit_shmem * bfr,size_t offset,size_t len)352 static void apple_rtkit_memcpy(struct apple_rtkit *rtk, void *dst,
353 struct apple_rtkit_shmem *bfr, size_t offset,
354 size_t len)
355 {
356 if (bfr->iomem)
357 memcpy_fromio(dst, bfr->iomem + offset, len);
358 else
359 memcpy(dst, bfr->buffer + offset, len);
360 }
361
apple_rtkit_crashlog_rx(struct apple_rtkit * rtk,u64 msg)362 static void apple_rtkit_crashlog_rx(struct apple_rtkit *rtk, u64 msg)
363 {
364 u8 type = FIELD_GET(APPLE_RTKIT_SYSLOG_TYPE, msg);
365 u8 *bfr;
366
367 if (type != APPLE_RTKIT_CRASHLOG_CRASH) {
368 dev_warn(rtk->dev, "RTKit: Unknown crashlog message: %llx\n",
369 msg);
370 return;
371 }
372
373 if (!rtk->crashlog_buffer.size) {
374 apple_rtkit_common_rx_get_buffer(rtk, &rtk->crashlog_buffer,
375 APPLE_RTKIT_EP_CRASHLOG, msg);
376 return;
377 }
378
379 dev_err(rtk->dev, "RTKit: co-processor has crashed\n");
380
381 /*
382 * create a shadow copy here to make sure the co-processor isn't able
383 * to change the log while we're dumping it. this also ensures
384 * the buffer is in normal memory and not iomem for e.g. the SMC
385 */
386 bfr = kzalloc(rtk->crashlog_buffer.size, GFP_KERNEL);
387 if (bfr) {
388 apple_rtkit_memcpy(rtk, bfr, &rtk->crashlog_buffer, 0,
389 rtk->crashlog_buffer.size);
390 apple_rtkit_crashlog_dump(rtk, bfr, rtk->crashlog_buffer.size);
391 } else {
392 dev_err(rtk->dev,
393 "RTKit: Couldn't allocate crashlog shadow buffer\n");
394 }
395
396 rtk->crashed = true;
397 if (rtk->ops->crashed)
398 rtk->ops->crashed(rtk->cookie, bfr, rtk->crashlog_buffer.size);
399
400 kfree(bfr);
401 }
402
apple_rtkit_ioreport_rx(struct apple_rtkit * rtk,u64 msg)403 static void apple_rtkit_ioreport_rx(struct apple_rtkit *rtk, u64 msg)
404 {
405 u8 type = FIELD_GET(APPLE_RTKIT_SYSLOG_TYPE, msg);
406
407 switch (type) {
408 case APPLE_RTKIT_BUFFER_REQUEST:
409 apple_rtkit_common_rx_get_buffer(rtk, &rtk->ioreport_buffer,
410 APPLE_RTKIT_EP_IOREPORT, msg);
411 break;
412 /* unknown, must be ACKed or the co-processor will hang */
413 case 0x8:
414 case 0xc:
415 apple_rtkit_send_message(rtk, APPLE_RTKIT_EP_IOREPORT, msg,
416 NULL, false);
417 break;
418 default:
419 dev_warn(rtk->dev, "RTKit: Unknown ioreport message: %llx\n",
420 msg);
421 }
422 }
423
apple_rtkit_syslog_rx_init(struct apple_rtkit * rtk,u64 msg)424 static void apple_rtkit_syslog_rx_init(struct apple_rtkit *rtk, u64 msg)
425 {
426 rtk->syslog_n_entries = FIELD_GET(APPLE_RTKIT_SYSLOG_N_ENTRIES, msg);
427 rtk->syslog_msg_size = FIELD_GET(APPLE_RTKIT_SYSLOG_MSG_SIZE, msg);
428
429 rtk->syslog_msg_buffer = kzalloc(rtk->syslog_msg_size, GFP_KERNEL);
430
431 dev_dbg(rtk->dev,
432 "RTKit: syslog initialized: entries: %zd, msg_size: %zd\n",
433 rtk->syslog_n_entries, rtk->syslog_msg_size);
434 }
435
should_crop_syslog_char(char c)436 static bool should_crop_syslog_char(char c)
437 {
438 return c == '\n' || c == '\r' || c == ' ' || c == '\0';
439 }
440
apple_rtkit_syslog_rx_log(struct apple_rtkit * rtk,u64 msg)441 static void apple_rtkit_syslog_rx_log(struct apple_rtkit *rtk, u64 msg)
442 {
443 u8 idx = msg & 0xff;
444 char log_context[24];
445 size_t entry_size = 0x20 + rtk->syslog_msg_size;
446 int msglen;
447
448 if (!rtk->syslog_msg_buffer) {
449 dev_warn(
450 rtk->dev,
451 "RTKit: received syslog message but no syslog_msg_buffer\n");
452 goto done;
453 }
454 if (!rtk->syslog_buffer.size) {
455 dev_warn(
456 rtk->dev,
457 "RTKit: received syslog message but syslog_buffer.size is zero\n");
458 goto done;
459 }
460 if (!rtk->syslog_buffer.buffer && !rtk->syslog_buffer.iomem) {
461 dev_warn(
462 rtk->dev,
463 "RTKit: received syslog message but no syslog_buffer.buffer or syslog_buffer.iomem\n");
464 goto done;
465 }
466 if (idx > rtk->syslog_n_entries) {
467 dev_warn(rtk->dev, "RTKit: syslog index %d out of range\n",
468 idx);
469 goto done;
470 }
471
472 apple_rtkit_memcpy(rtk, log_context, &rtk->syslog_buffer,
473 idx * entry_size + 8, sizeof(log_context));
474 apple_rtkit_memcpy(rtk, rtk->syslog_msg_buffer, &rtk->syslog_buffer,
475 idx * entry_size + 8 + sizeof(log_context),
476 rtk->syslog_msg_size);
477
478 log_context[sizeof(log_context) - 1] = 0;
479
480 msglen = strnlen(rtk->syslog_msg_buffer, rtk->syslog_msg_size - 1);
481 while (msglen > 0 &&
482 should_crop_syslog_char(rtk->syslog_msg_buffer[msglen - 1]))
483 msglen--;
484
485 rtk->syslog_msg_buffer[msglen] = 0;
486 dev_info(rtk->dev, "RTKit: syslog message: %s: %s\n", log_context,
487 rtk->syslog_msg_buffer);
488
489 done:
490 apple_rtkit_send_message(rtk, APPLE_RTKIT_EP_SYSLOG, msg, NULL, false);
491 }
492
apple_rtkit_syslog_rx(struct apple_rtkit * rtk,u64 msg)493 static void apple_rtkit_syslog_rx(struct apple_rtkit *rtk, u64 msg)
494 {
495 u8 type = FIELD_GET(APPLE_RTKIT_SYSLOG_TYPE, msg);
496
497 switch (type) {
498 case APPLE_RTKIT_BUFFER_REQUEST:
499 apple_rtkit_common_rx_get_buffer(rtk, &rtk->syslog_buffer,
500 APPLE_RTKIT_EP_SYSLOG, msg);
501 break;
502 case APPLE_RTKIT_SYSLOG_INIT:
503 apple_rtkit_syslog_rx_init(rtk, msg);
504 break;
505 case APPLE_RTKIT_SYSLOG_LOG:
506 apple_rtkit_syslog_rx_log(rtk, msg);
507 break;
508 default:
509 dev_warn(rtk->dev, "RTKit: Unknown syslog message: %llx\n",
510 msg);
511 }
512 }
513
apple_rtkit_oslog_rx(struct apple_rtkit * rtk,u64 msg)514 static void apple_rtkit_oslog_rx(struct apple_rtkit *rtk, u64 msg)
515 {
516 u8 type = FIELD_GET(APPLE_RTKIT_OSLOG_TYPE, msg);
517
518 switch (type) {
519 case APPLE_RTKIT_OSLOG_BUFFER_REQUEST:
520 apple_rtkit_common_rx_get_buffer(rtk, &rtk->oslog_buffer,
521 APPLE_RTKIT_EP_OSLOG, msg);
522 break;
523 default:
524 dev_warn(rtk->dev, "RTKit: Unknown oslog message: %llx\n",
525 msg);
526 }
527 }
528
apple_rtkit_rx_work(struct work_struct * work)529 static void apple_rtkit_rx_work(struct work_struct *work)
530 {
531 struct apple_rtkit_rx_work *rtk_work =
532 container_of(work, struct apple_rtkit_rx_work, work);
533 struct apple_rtkit *rtk = rtk_work->rtk;
534
535 switch (rtk_work->ep) {
536 case APPLE_RTKIT_EP_MGMT:
537 apple_rtkit_management_rx(rtk, rtk_work->msg);
538 break;
539 case APPLE_RTKIT_EP_CRASHLOG:
540 apple_rtkit_crashlog_rx(rtk, rtk_work->msg);
541 break;
542 case APPLE_RTKIT_EP_SYSLOG:
543 apple_rtkit_syslog_rx(rtk, rtk_work->msg);
544 break;
545 case APPLE_RTKIT_EP_IOREPORT:
546 apple_rtkit_ioreport_rx(rtk, rtk_work->msg);
547 break;
548 case APPLE_RTKIT_EP_OSLOG:
549 apple_rtkit_oslog_rx(rtk, rtk_work->msg);
550 break;
551 case APPLE_RTKIT_APP_ENDPOINT_START ... 0xff:
552 if (rtk->ops->recv_message)
553 rtk->ops->recv_message(rtk->cookie, rtk_work->ep,
554 rtk_work->msg);
555 else
556 dev_warn(
557 rtk->dev,
558 "Received unexpected message to EP%02d: %llx\n",
559 rtk_work->ep, rtk_work->msg);
560 break;
561 default:
562 dev_warn(rtk->dev,
563 "RTKit: message to unknown endpoint %02x: %llx\n",
564 rtk_work->ep, rtk_work->msg);
565 }
566
567 kfree(rtk_work);
568 }
569
apple_rtkit_rx(struct apple_mbox * mbox,struct apple_mbox_msg msg,void * cookie)570 static void apple_rtkit_rx(struct apple_mbox *mbox, struct apple_mbox_msg msg,
571 void *cookie)
572 {
573 struct apple_rtkit *rtk = cookie;
574 struct apple_rtkit_rx_work *work;
575 u8 ep = msg.msg1;
576
577 /*
578 * The message was read from a MMIO FIFO and we have to make
579 * sure all reads from buffers sent with that message happen
580 * afterwards.
581 */
582 dma_rmb();
583
584 if (!test_bit(ep, rtk->endpoints))
585 dev_warn(rtk->dev,
586 "RTKit: Message to undiscovered endpoint 0x%02x\n",
587 ep);
588
589 if (ep >= APPLE_RTKIT_APP_ENDPOINT_START &&
590 rtk->ops->recv_message_early &&
591 rtk->ops->recv_message_early(rtk->cookie, ep, msg.msg0))
592 return;
593
594 work = kzalloc(sizeof(*work), GFP_ATOMIC);
595 if (!work)
596 return;
597
598 work->rtk = rtk;
599 work->ep = ep;
600 work->msg = msg.msg0;
601 INIT_WORK(&work->work, apple_rtkit_rx_work);
602 queue_work(rtk->wq, &work->work);
603 }
604
apple_rtkit_send_message(struct apple_rtkit * rtk,u8 ep,u64 message,struct completion * completion,bool atomic)605 int apple_rtkit_send_message(struct apple_rtkit *rtk, u8 ep, u64 message,
606 struct completion *completion, bool atomic)
607 {
608 struct apple_mbox_msg msg = {
609 .msg0 = message,
610 .msg1 = ep,
611 };
612
613 if (rtk->crashed) {
614 dev_warn(rtk->dev,
615 "RTKit: Device is crashed, cannot send message\n");
616 return -EINVAL;
617 }
618
619 if (ep >= APPLE_RTKIT_APP_ENDPOINT_START &&
620 !apple_rtkit_is_running(rtk)) {
621 dev_warn(rtk->dev,
622 "RTKit: Endpoint 0x%02x is not running, cannot send message\n", ep);
623 return -EINVAL;
624 }
625
626 /*
627 * The message will be sent with a MMIO write. We need the barrier
628 * here to ensure any previous writes to buffers are visible to the
629 * device before that MMIO write happens.
630 */
631 dma_wmb();
632
633 return apple_mbox_send(rtk->mbox, msg, atomic);
634 }
635 EXPORT_SYMBOL_GPL(apple_rtkit_send_message);
636
apple_rtkit_poll(struct apple_rtkit * rtk)637 int apple_rtkit_poll(struct apple_rtkit *rtk)
638 {
639 return apple_mbox_poll(rtk->mbox);
640 }
641 EXPORT_SYMBOL_GPL(apple_rtkit_poll);
642
apple_rtkit_start_ep(struct apple_rtkit * rtk,u8 endpoint)643 int apple_rtkit_start_ep(struct apple_rtkit *rtk, u8 endpoint)
644 {
645 u64 msg;
646
647 if (!test_bit(endpoint, rtk->endpoints))
648 return -EINVAL;
649 if (endpoint >= APPLE_RTKIT_APP_ENDPOINT_START &&
650 !apple_rtkit_is_running(rtk))
651 return -EINVAL;
652
653 msg = FIELD_PREP(APPLE_RTKIT_MGMT_STARTEP_EP, endpoint);
654 msg |= APPLE_RTKIT_MGMT_STARTEP_FLAG;
655 apple_rtkit_management_send(rtk, APPLE_RTKIT_MGMT_STARTEP, msg);
656
657 return 0;
658 }
659 EXPORT_SYMBOL_GPL(apple_rtkit_start_ep);
660
apple_rtkit_init(struct device * dev,void * cookie,const char * mbox_name,int mbox_idx,const struct apple_rtkit_ops * ops)661 struct apple_rtkit *apple_rtkit_init(struct device *dev, void *cookie,
662 const char *mbox_name, int mbox_idx,
663 const struct apple_rtkit_ops *ops)
664 {
665 struct apple_rtkit *rtk;
666 int ret;
667
668 if (!ops)
669 return ERR_PTR(-EINVAL);
670
671 rtk = kzalloc(sizeof(*rtk), GFP_KERNEL);
672 if (!rtk)
673 return ERR_PTR(-ENOMEM);
674
675 rtk->dev = dev;
676 rtk->cookie = cookie;
677 rtk->ops = ops;
678
679 init_completion(&rtk->epmap_completion);
680 init_completion(&rtk->iop_pwr_ack_completion);
681 init_completion(&rtk->ap_pwr_ack_completion);
682
683 bitmap_zero(rtk->endpoints, APPLE_RTKIT_MAX_ENDPOINTS);
684 set_bit(APPLE_RTKIT_EP_MGMT, rtk->endpoints);
685
686 if (mbox_name)
687 rtk->mbox = apple_mbox_get_byname(dev, mbox_name);
688 else
689 rtk->mbox = apple_mbox_get(dev, mbox_idx);
690
691 if (IS_ERR(rtk->mbox)) {
692 ret = PTR_ERR(rtk->mbox);
693 goto free_rtk;
694 }
695
696 rtk->mbox->rx = apple_rtkit_rx;
697 rtk->mbox->cookie = rtk;
698
699 rtk->wq = alloc_ordered_workqueue("rtkit-%s", WQ_HIGHPRI | WQ_MEM_RECLAIM,
700 dev_name(rtk->dev));
701 if (!rtk->wq) {
702 ret = -ENOMEM;
703 goto free_rtk;
704 }
705
706 ret = apple_mbox_start(rtk->mbox);
707 if (ret)
708 goto destroy_wq;
709
710 return rtk;
711
712 destroy_wq:
713 destroy_workqueue(rtk->wq);
714 free_rtk:
715 kfree(rtk);
716 return ERR_PTR(ret);
717 }
718 EXPORT_SYMBOL_GPL(apple_rtkit_init);
719
apple_rtkit_wait_for_completion(struct completion * c)720 static int apple_rtkit_wait_for_completion(struct completion *c)
721 {
722 long t;
723
724 t = wait_for_completion_interruptible_timeout(c,
725 msecs_to_jiffies(1000));
726 if (t < 0)
727 return t;
728 else if (t == 0)
729 return -ETIME;
730 else
731 return 0;
732 }
733
apple_rtkit_reinit(struct apple_rtkit * rtk)734 int apple_rtkit_reinit(struct apple_rtkit *rtk)
735 {
736 /* make sure we don't handle any messages while reinitializing */
737 apple_mbox_stop(rtk->mbox);
738 flush_workqueue(rtk->wq);
739
740 apple_rtkit_free_buffer(rtk, &rtk->ioreport_buffer);
741 apple_rtkit_free_buffer(rtk, &rtk->crashlog_buffer);
742 apple_rtkit_free_buffer(rtk, &rtk->oslog_buffer);
743 apple_rtkit_free_buffer(rtk, &rtk->syslog_buffer);
744
745 kfree(rtk->syslog_msg_buffer);
746
747 rtk->syslog_msg_buffer = NULL;
748 rtk->syslog_n_entries = 0;
749 rtk->syslog_msg_size = 0;
750
751 bitmap_zero(rtk->endpoints, APPLE_RTKIT_MAX_ENDPOINTS);
752 set_bit(APPLE_RTKIT_EP_MGMT, rtk->endpoints);
753
754 reinit_completion(&rtk->epmap_completion);
755 reinit_completion(&rtk->iop_pwr_ack_completion);
756 reinit_completion(&rtk->ap_pwr_ack_completion);
757
758 rtk->crashed = false;
759 rtk->iop_power_state = APPLE_RTKIT_PWR_STATE_OFF;
760 rtk->ap_power_state = APPLE_RTKIT_PWR_STATE_OFF;
761
762 return apple_mbox_start(rtk->mbox);
763 }
764 EXPORT_SYMBOL_GPL(apple_rtkit_reinit);
765
apple_rtkit_set_ap_power_state(struct apple_rtkit * rtk,unsigned int state)766 static int apple_rtkit_set_ap_power_state(struct apple_rtkit *rtk,
767 unsigned int state)
768 {
769 u64 msg;
770 int ret;
771
772 reinit_completion(&rtk->ap_pwr_ack_completion);
773
774 msg = FIELD_PREP(APPLE_RTKIT_MGMT_PWR_STATE, state);
775 ret = apple_rtkit_management_send(rtk, APPLE_RTKIT_MGMT_SET_AP_PWR_STATE,
776 msg);
777 if (ret)
778 return ret;
779
780 ret = apple_rtkit_wait_for_completion(&rtk->ap_pwr_ack_completion);
781 if (ret)
782 return ret;
783
784 if (rtk->ap_power_state != state)
785 return -EINVAL;
786 return 0;
787 }
788
apple_rtkit_set_iop_power_state(struct apple_rtkit * rtk,unsigned int state)789 static int apple_rtkit_set_iop_power_state(struct apple_rtkit *rtk,
790 unsigned int state)
791 {
792 u64 msg;
793 int ret;
794
795 reinit_completion(&rtk->iop_pwr_ack_completion);
796
797 msg = FIELD_PREP(APPLE_RTKIT_MGMT_PWR_STATE, state);
798 ret = apple_rtkit_management_send(rtk, APPLE_RTKIT_MGMT_SET_IOP_PWR_STATE,
799 msg);
800 if (ret)
801 return ret;
802
803 ret = apple_rtkit_wait_for_completion(&rtk->iop_pwr_ack_completion);
804 if (ret)
805 return ret;
806
807 if (rtk->iop_power_state != state)
808 return -EINVAL;
809 return 0;
810 }
811
apple_rtkit_boot(struct apple_rtkit * rtk)812 int apple_rtkit_boot(struct apple_rtkit *rtk)
813 {
814 int ret;
815
816 if (apple_rtkit_is_running(rtk))
817 return 0;
818 if (rtk->crashed)
819 return -EINVAL;
820
821 dev_dbg(rtk->dev, "RTKit: waiting for boot to finish\n");
822 ret = apple_rtkit_wait_for_completion(&rtk->epmap_completion);
823 if (ret)
824 return ret;
825 if (rtk->boot_result)
826 return rtk->boot_result;
827
828 dev_dbg(rtk->dev, "RTKit: waiting for IOP power state ACK\n");
829 ret = apple_rtkit_wait_for_completion(&rtk->iop_pwr_ack_completion);
830 if (ret)
831 return ret;
832
833 return apple_rtkit_set_ap_power_state(rtk, APPLE_RTKIT_PWR_STATE_ON);
834 }
835 EXPORT_SYMBOL_GPL(apple_rtkit_boot);
836
apple_rtkit_shutdown(struct apple_rtkit * rtk)837 int apple_rtkit_shutdown(struct apple_rtkit *rtk)
838 {
839 int ret;
840
841 /* if OFF is used here the co-processor will not wake up again */
842 ret = apple_rtkit_set_ap_power_state(rtk,
843 APPLE_RTKIT_PWR_STATE_QUIESCED);
844 if (ret)
845 return ret;
846
847 ret = apple_rtkit_set_iop_power_state(rtk, APPLE_RTKIT_PWR_STATE_SLEEP);
848 if (ret)
849 return ret;
850
851 return apple_rtkit_reinit(rtk);
852 }
853 EXPORT_SYMBOL_GPL(apple_rtkit_shutdown);
854
apple_rtkit_idle(struct apple_rtkit * rtk)855 int apple_rtkit_idle(struct apple_rtkit *rtk)
856 {
857 int ret;
858
859 /* if OFF is used here the co-processor will not wake up again */
860 ret = apple_rtkit_set_ap_power_state(rtk,
861 APPLE_RTKIT_PWR_STATE_IDLE);
862 if (ret)
863 return ret;
864
865 ret = apple_rtkit_set_iop_power_state(rtk, APPLE_RTKIT_PWR_STATE_IDLE);
866 if (ret)
867 return ret;
868
869 rtk->iop_power_state = APPLE_RTKIT_PWR_STATE_IDLE;
870 rtk->ap_power_state = APPLE_RTKIT_PWR_STATE_IDLE;
871 return 0;
872 }
873 EXPORT_SYMBOL_GPL(apple_rtkit_idle);
874
apple_rtkit_quiesce(struct apple_rtkit * rtk)875 int apple_rtkit_quiesce(struct apple_rtkit *rtk)
876 {
877 int ret;
878
879 ret = apple_rtkit_set_ap_power_state(rtk,
880 APPLE_RTKIT_PWR_STATE_QUIESCED);
881 if (ret)
882 return ret;
883
884 ret = apple_rtkit_set_iop_power_state(rtk,
885 APPLE_RTKIT_PWR_STATE_QUIESCED);
886 if (ret)
887 return ret;
888
889 ret = apple_rtkit_reinit(rtk);
890 if (ret)
891 return ret;
892
893 rtk->iop_power_state = APPLE_RTKIT_PWR_STATE_QUIESCED;
894 rtk->ap_power_state = APPLE_RTKIT_PWR_STATE_QUIESCED;
895 return 0;
896 }
897 EXPORT_SYMBOL_GPL(apple_rtkit_quiesce);
898
apple_rtkit_wake(struct apple_rtkit * rtk)899 int apple_rtkit_wake(struct apple_rtkit *rtk)
900 {
901 u64 msg;
902 int ret;
903
904 if (apple_rtkit_is_running(rtk))
905 return -EINVAL;
906
907 reinit_completion(&rtk->iop_pwr_ack_completion);
908
909 /*
910 * Use open-coded apple_rtkit_set_iop_power_state since apple_rtkit_boot
911 * will wait for the completion anyway.
912 */
913 msg = FIELD_PREP(APPLE_RTKIT_MGMT_PWR_STATE, APPLE_RTKIT_PWR_STATE_INIT);
914 ret = apple_rtkit_management_send(rtk, APPLE_RTKIT_MGMT_SET_IOP_PWR_STATE,
915 msg);
916 if (ret)
917 return ret;
918
919 return apple_rtkit_boot(rtk);
920 }
921 EXPORT_SYMBOL_GPL(apple_rtkit_wake);
922
apple_rtkit_free(struct apple_rtkit * rtk)923 void apple_rtkit_free(struct apple_rtkit *rtk)
924 {
925 apple_mbox_stop(rtk->mbox);
926 destroy_workqueue(rtk->wq);
927
928 apple_rtkit_free_buffer(rtk, &rtk->ioreport_buffer);
929 apple_rtkit_free_buffer(rtk, &rtk->crashlog_buffer);
930 apple_rtkit_free_buffer(rtk, &rtk->oslog_buffer);
931 apple_rtkit_free_buffer(rtk, &rtk->syslog_buffer);
932
933 kfree(rtk->syslog_msg_buffer);
934 kfree(rtk);
935 }
936 EXPORT_SYMBOL_GPL(apple_rtkit_free);
937
apple_rtkit_free_wrapper(void * data)938 static void apple_rtkit_free_wrapper(void *data)
939 {
940 apple_rtkit_free(data);
941 }
942
devm_apple_rtkit_init(struct device * dev,void * cookie,const char * mbox_name,int mbox_idx,const struct apple_rtkit_ops * ops)943 struct apple_rtkit *devm_apple_rtkit_init(struct device *dev, void *cookie,
944 const char *mbox_name, int mbox_idx,
945 const struct apple_rtkit_ops *ops)
946 {
947 struct apple_rtkit *rtk;
948 int ret;
949
950 rtk = apple_rtkit_init(dev, cookie, mbox_name, mbox_idx, ops);
951 if (IS_ERR(rtk))
952 return rtk;
953
954 ret = devm_add_action_or_reset(dev, apple_rtkit_free_wrapper, rtk);
955 if (ret)
956 return ERR_PTR(ret);
957
958 return rtk;
959 }
960 EXPORT_SYMBOL_GPL(devm_apple_rtkit_init);
961
962 MODULE_LICENSE("Dual MIT/GPL");
963 MODULE_AUTHOR("Sven Peter <sven@svenpeter.dev>");
964 MODULE_DESCRIPTION("Apple RTKit driver");
965