1 // SPDX-License-Identifier: GPL-2.0-only OR MIT
2 /*
3 * Apple RTKit IPC library
4 * Copyright (C) The Asahi Linux Contributors
5 */
6
7 #include "rtkit-internal.h"
8
9 enum {
10 APPLE_RTKIT_PWR_STATE_OFF = 0x00, /* power off, cannot be restarted */
11 APPLE_RTKIT_PWR_STATE_SLEEP = 0x01, /* sleeping, can be restarted */
12 APPLE_RTKIT_PWR_STATE_IDLE = 0x201, /* sleeping, retain state */
13 APPLE_RTKIT_PWR_STATE_QUIESCED = 0x10, /* running but no communication */
14 APPLE_RTKIT_PWR_STATE_ON = 0x20, /* normal operating state */
15 APPLE_RTKIT_PWR_STATE_INIT = 0x220, /* init after starting the coproc */
16 };
17
18 enum {
19 APPLE_RTKIT_EP_MGMT = 0,
20 APPLE_RTKIT_EP_CRASHLOG = 1,
21 APPLE_RTKIT_EP_SYSLOG = 2,
22 APPLE_RTKIT_EP_DEBUG = 3,
23 APPLE_RTKIT_EP_IOREPORT = 4,
24 APPLE_RTKIT_EP_OSLOG = 8,
25 };
26
27 #define APPLE_RTKIT_MGMT_TYPE GENMASK_ULL(59, 52)
28
29 enum {
30 APPLE_RTKIT_MGMT_HELLO = 1,
31 APPLE_RTKIT_MGMT_HELLO_REPLY = 2,
32 APPLE_RTKIT_MGMT_STARTEP = 5,
33 APPLE_RTKIT_MGMT_SET_IOP_PWR_STATE = 6,
34 APPLE_RTKIT_MGMT_SET_IOP_PWR_STATE_ACK = 7,
35 APPLE_RTKIT_MGMT_EPMAP = 8,
36 APPLE_RTKIT_MGMT_EPMAP_REPLY = 8,
37 APPLE_RTKIT_MGMT_SET_AP_PWR_STATE = 0xb,
38 APPLE_RTKIT_MGMT_SET_AP_PWR_STATE_ACK = 0xb,
39 };
40
41 #define APPLE_RTKIT_MGMT_HELLO_MINVER GENMASK_ULL(15, 0)
42 #define APPLE_RTKIT_MGMT_HELLO_MAXVER GENMASK_ULL(31, 16)
43
44 #define APPLE_RTKIT_MGMT_EPMAP_LAST BIT_ULL(51)
45 #define APPLE_RTKIT_MGMT_EPMAP_BASE GENMASK_ULL(34, 32)
46 #define APPLE_RTKIT_MGMT_EPMAP_BITMAP GENMASK_ULL(31, 0)
47
48 #define APPLE_RTKIT_MGMT_EPMAP_REPLY_MORE BIT_ULL(0)
49
50 #define APPLE_RTKIT_MGMT_STARTEP_EP GENMASK_ULL(39, 32)
51 #define APPLE_RTKIT_MGMT_STARTEP_FLAG BIT_ULL(1)
52
53 #define APPLE_RTKIT_MGMT_PWR_STATE GENMASK_ULL(15, 0)
54
55 #define APPLE_RTKIT_CRASHLOG_CRASH 1
56
57 #define APPLE_RTKIT_BUFFER_REQUEST 1
58 #define APPLE_RTKIT_BUFFER_REQUEST_SIZE GENMASK_ULL(51, 44)
59 #define APPLE_RTKIT_BUFFER_REQUEST_IOVA GENMASK_ULL(43, 0)
60
61 #define APPLE_RTKIT_SYSLOG_TYPE GENMASK_ULL(59, 52)
62
63 #define APPLE_RTKIT_SYSLOG_LOG 5
64
65 #define APPLE_RTKIT_SYSLOG_INIT 8
66 #define APPLE_RTKIT_SYSLOG_N_ENTRIES GENMASK_ULL(7, 0)
67 #define APPLE_RTKIT_SYSLOG_MSG_SIZE GENMASK_ULL(31, 24)
68
69 #define APPLE_RTKIT_OSLOG_TYPE GENMASK_ULL(63, 56)
70 #define APPLE_RTKIT_OSLOG_BUFFER_REQUEST 1
71 #define APPLE_RTKIT_OSLOG_SIZE GENMASK_ULL(55, 36)
72 #define APPLE_RTKIT_OSLOG_IOVA GENMASK_ULL(35, 0)
73
74 #define APPLE_RTKIT_MIN_SUPPORTED_VERSION 11
75 #define APPLE_RTKIT_MAX_SUPPORTED_VERSION 12
76
77 struct apple_rtkit_rx_work {
78 struct apple_rtkit *rtk;
79 u8 ep;
80 u64 msg;
81 struct work_struct work;
82 };
83
apple_rtkit_is_running(struct apple_rtkit * rtk)84 bool apple_rtkit_is_running(struct apple_rtkit *rtk)
85 {
86 if (rtk->crashed)
87 return false;
88 if ((rtk->iop_power_state & 0xff) != APPLE_RTKIT_PWR_STATE_ON)
89 return false;
90 if ((rtk->ap_power_state & 0xff) != APPLE_RTKIT_PWR_STATE_ON)
91 return false;
92 return true;
93 }
94 EXPORT_SYMBOL_GPL(apple_rtkit_is_running);
95
apple_rtkit_is_crashed(struct apple_rtkit * rtk)96 bool apple_rtkit_is_crashed(struct apple_rtkit *rtk)
97 {
98 return rtk->crashed;
99 }
100 EXPORT_SYMBOL_GPL(apple_rtkit_is_crashed);
101
apple_rtkit_management_send(struct apple_rtkit * rtk,u8 type,u64 msg)102 static int apple_rtkit_management_send(struct apple_rtkit *rtk, u8 type,
103 u64 msg)
104 {
105 int ret;
106
107 msg &= ~APPLE_RTKIT_MGMT_TYPE;
108 msg |= FIELD_PREP(APPLE_RTKIT_MGMT_TYPE, type);
109 ret = apple_rtkit_send_message(rtk, APPLE_RTKIT_EP_MGMT, msg, NULL, false);
110
111 if (ret)
112 dev_err(rtk->dev, "RTKit: Failed to send management message: %d\n", ret);
113
114 return ret;
115 }
116
apple_rtkit_management_rx_hello(struct apple_rtkit * rtk,u64 msg)117 static void apple_rtkit_management_rx_hello(struct apple_rtkit *rtk, u64 msg)
118 {
119 u64 reply;
120
121 int min_ver = FIELD_GET(APPLE_RTKIT_MGMT_HELLO_MINVER, msg);
122 int max_ver = FIELD_GET(APPLE_RTKIT_MGMT_HELLO_MAXVER, msg);
123 int want_ver = min(APPLE_RTKIT_MAX_SUPPORTED_VERSION, max_ver);
124
125 dev_dbg(rtk->dev, "RTKit: Min ver %d, max ver %d\n", min_ver, max_ver);
126
127 if (min_ver > APPLE_RTKIT_MAX_SUPPORTED_VERSION) {
128 dev_err(rtk->dev, "RTKit: Firmware min version %d is too new\n",
129 min_ver);
130 goto abort_boot;
131 }
132
133 if (max_ver < APPLE_RTKIT_MIN_SUPPORTED_VERSION) {
134 dev_err(rtk->dev, "RTKit: Firmware max version %d is too old\n",
135 max_ver);
136 goto abort_boot;
137 }
138
139 dev_info(rtk->dev, "RTKit: Initializing (protocol version %d)\n",
140 want_ver);
141 rtk->version = want_ver;
142
143 reply = FIELD_PREP(APPLE_RTKIT_MGMT_HELLO_MINVER, want_ver);
144 reply |= FIELD_PREP(APPLE_RTKIT_MGMT_HELLO_MAXVER, want_ver);
145 apple_rtkit_management_send(rtk, APPLE_RTKIT_MGMT_HELLO_REPLY, reply);
146
147 return;
148
149 abort_boot:
150 rtk->boot_result = -EINVAL;
151 complete_all(&rtk->epmap_completion);
152 }
153
apple_rtkit_management_rx_epmap(struct apple_rtkit * rtk,u64 msg)154 static void apple_rtkit_management_rx_epmap(struct apple_rtkit *rtk, u64 msg)
155 {
156 int i, ep;
157 u64 reply;
158 unsigned long bitmap = FIELD_GET(APPLE_RTKIT_MGMT_EPMAP_BITMAP, msg);
159 u32 base = FIELD_GET(APPLE_RTKIT_MGMT_EPMAP_BASE, msg);
160
161 dev_dbg(rtk->dev,
162 "RTKit: received endpoint bitmap 0x%lx with base 0x%x\n",
163 bitmap, base);
164
165 for_each_set_bit(i, &bitmap, 32) {
166 ep = 32 * base + i;
167 dev_dbg(rtk->dev, "RTKit: Discovered endpoint 0x%02x\n", ep);
168 set_bit(ep, rtk->endpoints);
169 }
170
171 reply = FIELD_PREP(APPLE_RTKIT_MGMT_EPMAP_BASE, base);
172 if (msg & APPLE_RTKIT_MGMT_EPMAP_LAST)
173 reply |= APPLE_RTKIT_MGMT_EPMAP_LAST;
174 else
175 reply |= APPLE_RTKIT_MGMT_EPMAP_REPLY_MORE;
176
177 apple_rtkit_management_send(rtk, APPLE_RTKIT_MGMT_EPMAP_REPLY, reply);
178
179 if (!(msg & APPLE_RTKIT_MGMT_EPMAP_LAST))
180 return;
181
182 for_each_set_bit(ep, rtk->endpoints, APPLE_RTKIT_APP_ENDPOINT_START) {
183 switch (ep) {
184 /* the management endpoint is started by default */
185 case APPLE_RTKIT_EP_MGMT:
186 break;
187
188 /* without starting these RTKit refuses to boot */
189 case APPLE_RTKIT_EP_SYSLOG:
190 case APPLE_RTKIT_EP_CRASHLOG:
191 case APPLE_RTKIT_EP_DEBUG:
192 case APPLE_RTKIT_EP_IOREPORT:
193 case APPLE_RTKIT_EP_OSLOG:
194 dev_dbg(rtk->dev,
195 "RTKit: Starting system endpoint 0x%02x\n", ep);
196 apple_rtkit_start_ep(rtk, ep);
197 break;
198
199 default:
200 dev_warn(rtk->dev,
201 "RTKit: Unknown system endpoint: 0x%02x\n",
202 ep);
203 }
204 }
205
206 rtk->boot_result = 0;
207 complete_all(&rtk->epmap_completion);
208 }
209
apple_rtkit_management_rx_iop_pwr_ack(struct apple_rtkit * rtk,u64 msg)210 static void apple_rtkit_management_rx_iop_pwr_ack(struct apple_rtkit *rtk,
211 u64 msg)
212 {
213 unsigned int new_state = FIELD_GET(APPLE_RTKIT_MGMT_PWR_STATE, msg);
214
215 dev_dbg(rtk->dev, "RTKit: IOP power state transition: 0x%x -> 0x%x\n",
216 rtk->iop_power_state, new_state);
217 rtk->iop_power_state = new_state;
218
219 complete_all(&rtk->iop_pwr_ack_completion);
220 }
221
apple_rtkit_management_rx_ap_pwr_ack(struct apple_rtkit * rtk,u64 msg)222 static void apple_rtkit_management_rx_ap_pwr_ack(struct apple_rtkit *rtk,
223 u64 msg)
224 {
225 unsigned int new_state = FIELD_GET(APPLE_RTKIT_MGMT_PWR_STATE, msg);
226
227 dev_dbg(rtk->dev, "RTKit: AP power state transition: 0x%x -> 0x%x\n",
228 rtk->ap_power_state, new_state);
229 rtk->ap_power_state = new_state;
230
231 complete_all(&rtk->ap_pwr_ack_completion);
232 }
233
apple_rtkit_management_rx(struct apple_rtkit * rtk,u64 msg)234 static void apple_rtkit_management_rx(struct apple_rtkit *rtk, u64 msg)
235 {
236 u8 type = FIELD_GET(APPLE_RTKIT_MGMT_TYPE, msg);
237
238 switch (type) {
239 case APPLE_RTKIT_MGMT_HELLO:
240 apple_rtkit_management_rx_hello(rtk, msg);
241 break;
242 case APPLE_RTKIT_MGMT_EPMAP:
243 apple_rtkit_management_rx_epmap(rtk, msg);
244 break;
245 case APPLE_RTKIT_MGMT_SET_IOP_PWR_STATE_ACK:
246 apple_rtkit_management_rx_iop_pwr_ack(rtk, msg);
247 break;
248 case APPLE_RTKIT_MGMT_SET_AP_PWR_STATE_ACK:
249 apple_rtkit_management_rx_ap_pwr_ack(rtk, msg);
250 break;
251 default:
252 dev_warn(
253 rtk->dev,
254 "RTKit: unknown management message: 0x%llx (type: 0x%02x)\n",
255 msg, type);
256 }
257 }
258
apple_rtkit_common_rx_get_buffer(struct apple_rtkit * rtk,struct apple_rtkit_shmem * buffer,u8 ep,u64 msg)259 static int apple_rtkit_common_rx_get_buffer(struct apple_rtkit *rtk,
260 struct apple_rtkit_shmem *buffer,
261 u8 ep, u64 msg)
262 {
263 u64 reply;
264 int err;
265
266 /* The different size vs. IOVA shifts look odd but are indeed correct this way */
267 if (ep == APPLE_RTKIT_EP_OSLOG) {
268 buffer->size = FIELD_GET(APPLE_RTKIT_OSLOG_SIZE, msg);
269 buffer->iova = FIELD_GET(APPLE_RTKIT_OSLOG_IOVA, msg) << 12;
270 } else {
271 buffer->size = FIELD_GET(APPLE_RTKIT_BUFFER_REQUEST_SIZE, msg) << 12;
272 buffer->iova = FIELD_GET(APPLE_RTKIT_BUFFER_REQUEST_IOVA, msg);
273 }
274
275 buffer->buffer = NULL;
276 buffer->iomem = NULL;
277 buffer->is_mapped = false;
278
279 dev_dbg(rtk->dev, "RTKit: buffer request for 0x%zx bytes at %pad\n",
280 buffer->size, &buffer->iova);
281
282 if (buffer->iova && !rtk->ops->shmem_setup) {
283 err = -EINVAL;
284 goto error;
285 }
286
287 if (rtk->ops->shmem_setup) {
288 err = rtk->ops->shmem_setup(rtk->cookie, buffer);
289 if (err)
290 goto error;
291 } else {
292 buffer->buffer = dma_alloc_coherent(rtk->dev, buffer->size,
293 &buffer->iova, GFP_KERNEL);
294 if (!buffer->buffer) {
295 err = -ENOMEM;
296 goto error;
297 }
298 }
299
300 if (!buffer->is_mapped) {
301 /* oslog uses different fields and needs a shifted IOVA instead of size */
302 if (ep == APPLE_RTKIT_EP_OSLOG) {
303 reply = FIELD_PREP(APPLE_RTKIT_OSLOG_TYPE,
304 APPLE_RTKIT_OSLOG_BUFFER_REQUEST);
305 reply |= FIELD_PREP(APPLE_RTKIT_OSLOG_SIZE, buffer->size);
306 reply |= FIELD_PREP(APPLE_RTKIT_OSLOG_IOVA,
307 buffer->iova >> 12);
308 } else {
309 reply = FIELD_PREP(APPLE_RTKIT_SYSLOG_TYPE,
310 APPLE_RTKIT_BUFFER_REQUEST);
311 reply |= FIELD_PREP(APPLE_RTKIT_BUFFER_REQUEST_SIZE,
312 buffer->size >> 12);
313 reply |= FIELD_PREP(APPLE_RTKIT_BUFFER_REQUEST_IOVA,
314 buffer->iova);
315 }
316 apple_rtkit_send_message(rtk, ep, reply, NULL, false);
317 }
318
319 return 0;
320
321 error:
322 dev_err(rtk->dev, "RTKit: failed buffer request for 0x%zx bytes (%d)\n",
323 buffer->size, err);
324
325 buffer->buffer = NULL;
326 buffer->iomem = NULL;
327 buffer->iova = 0;
328 buffer->size = 0;
329 buffer->is_mapped = false;
330 return err;
331 }
332
apple_rtkit_free_buffer(struct apple_rtkit * rtk,struct apple_rtkit_shmem * bfr)333 static void apple_rtkit_free_buffer(struct apple_rtkit *rtk,
334 struct apple_rtkit_shmem *bfr)
335 {
336 if (bfr->size == 0)
337 return;
338
339 if (rtk->ops->shmem_destroy)
340 rtk->ops->shmem_destroy(rtk->cookie, bfr);
341 else if (bfr->buffer)
342 dma_free_coherent(rtk->dev, bfr->size, bfr->buffer, bfr->iova);
343
344 bfr->buffer = NULL;
345 bfr->iomem = NULL;
346 bfr->iova = 0;
347 bfr->size = 0;
348 bfr->is_mapped = false;
349 }
350
apple_rtkit_memcpy(struct apple_rtkit * rtk,void * dst,struct apple_rtkit_shmem * bfr,size_t offset,size_t len)351 static void apple_rtkit_memcpy(struct apple_rtkit *rtk, void *dst,
352 struct apple_rtkit_shmem *bfr, size_t offset,
353 size_t len)
354 {
355 if (bfr->iomem)
356 memcpy_fromio(dst, bfr->iomem + offset, len);
357 else
358 memcpy(dst, bfr->buffer + offset, len);
359 }
360
apple_rtkit_crashlog_rx(struct apple_rtkit * rtk,u64 msg)361 static void apple_rtkit_crashlog_rx(struct apple_rtkit *rtk, u64 msg)
362 {
363 u8 type = FIELD_GET(APPLE_RTKIT_SYSLOG_TYPE, msg);
364 u8 *bfr;
365
366 if (type != APPLE_RTKIT_CRASHLOG_CRASH) {
367 dev_warn(rtk->dev, "RTKit: Unknown crashlog message: %llx\n",
368 msg);
369 return;
370 }
371
372 if (!rtk->crashlog_buffer.size) {
373 apple_rtkit_common_rx_get_buffer(rtk, &rtk->crashlog_buffer,
374 APPLE_RTKIT_EP_CRASHLOG, msg);
375 return;
376 }
377
378 dev_err(rtk->dev, "RTKit: co-processor has crashed\n");
379
380 /*
381 * create a shadow copy here to make sure the co-processor isn't able
382 * to change the log while we're dumping it. this also ensures
383 * the buffer is in normal memory and not iomem for e.g. the SMC
384 */
385 bfr = kzalloc(rtk->crashlog_buffer.size, GFP_KERNEL);
386 if (bfr) {
387 apple_rtkit_memcpy(rtk, bfr, &rtk->crashlog_buffer, 0,
388 rtk->crashlog_buffer.size);
389 apple_rtkit_crashlog_dump(rtk, bfr, rtk->crashlog_buffer.size);
390 } else {
391 dev_err(rtk->dev,
392 "RTKit: Couldn't allocate crashlog shadow buffer\n");
393 }
394
395 rtk->crashed = true;
396 if (rtk->ops->crashed)
397 rtk->ops->crashed(rtk->cookie, bfr, rtk->crashlog_buffer.size);
398
399 kfree(bfr);
400 }
401
apple_rtkit_ioreport_rx(struct apple_rtkit * rtk,u64 msg)402 static void apple_rtkit_ioreport_rx(struct apple_rtkit *rtk, u64 msg)
403 {
404 u8 type = FIELD_GET(APPLE_RTKIT_SYSLOG_TYPE, msg);
405
406 switch (type) {
407 case APPLE_RTKIT_BUFFER_REQUEST:
408 apple_rtkit_common_rx_get_buffer(rtk, &rtk->ioreport_buffer,
409 APPLE_RTKIT_EP_IOREPORT, msg);
410 break;
411 /* unknown, must be ACKed or the co-processor will hang */
412 case 0x8:
413 case 0xc:
414 apple_rtkit_send_message(rtk, APPLE_RTKIT_EP_IOREPORT, msg,
415 NULL, false);
416 break;
417 default:
418 dev_warn(rtk->dev, "RTKit: Unknown ioreport message: %llx\n",
419 msg);
420 }
421 }
422
apple_rtkit_syslog_rx_init(struct apple_rtkit * rtk,u64 msg)423 static void apple_rtkit_syslog_rx_init(struct apple_rtkit *rtk, u64 msg)
424 {
425 rtk->syslog_n_entries = FIELD_GET(APPLE_RTKIT_SYSLOG_N_ENTRIES, msg);
426 rtk->syslog_msg_size = FIELD_GET(APPLE_RTKIT_SYSLOG_MSG_SIZE, msg);
427
428 rtk->syslog_msg_buffer = kzalloc(rtk->syslog_msg_size, GFP_KERNEL);
429
430 dev_dbg(rtk->dev,
431 "RTKit: syslog initialized: entries: %zd, msg_size: %zd\n",
432 rtk->syslog_n_entries, rtk->syslog_msg_size);
433 }
434
should_crop_syslog_char(char c)435 static bool should_crop_syslog_char(char c)
436 {
437 return c == '\n' || c == '\r' || c == ' ' || c == '\0';
438 }
439
apple_rtkit_syslog_rx_log(struct apple_rtkit * rtk,u64 msg)440 static void apple_rtkit_syslog_rx_log(struct apple_rtkit *rtk, u64 msg)
441 {
442 u8 idx = msg & 0xff;
443 char log_context[24];
444 size_t entry_size = 0x20 + rtk->syslog_msg_size;
445 int msglen;
446
447 if (!rtk->syslog_msg_buffer) {
448 dev_warn(
449 rtk->dev,
450 "RTKit: received syslog message but no syslog_msg_buffer\n");
451 goto done;
452 }
453 if (!rtk->syslog_buffer.size) {
454 dev_warn(
455 rtk->dev,
456 "RTKit: received syslog message but syslog_buffer.size is zero\n");
457 goto done;
458 }
459 if (!rtk->syslog_buffer.buffer && !rtk->syslog_buffer.iomem) {
460 dev_warn(
461 rtk->dev,
462 "RTKit: received syslog message but no syslog_buffer.buffer or syslog_buffer.iomem\n");
463 goto done;
464 }
465 if (idx > rtk->syslog_n_entries) {
466 dev_warn(rtk->dev, "RTKit: syslog index %d out of range\n",
467 idx);
468 goto done;
469 }
470
471 apple_rtkit_memcpy(rtk, log_context, &rtk->syslog_buffer,
472 idx * entry_size + 8, sizeof(log_context));
473 apple_rtkit_memcpy(rtk, rtk->syslog_msg_buffer, &rtk->syslog_buffer,
474 idx * entry_size + 8 + sizeof(log_context),
475 rtk->syslog_msg_size);
476
477 log_context[sizeof(log_context) - 1] = 0;
478
479 msglen = strnlen(rtk->syslog_msg_buffer, rtk->syslog_msg_size - 1);
480 while (msglen > 0 &&
481 should_crop_syslog_char(rtk->syslog_msg_buffer[msglen - 1]))
482 msglen--;
483
484 rtk->syslog_msg_buffer[msglen] = 0;
485 dev_info(rtk->dev, "RTKit: syslog message: %s: %s\n", log_context,
486 rtk->syslog_msg_buffer);
487
488 done:
489 apple_rtkit_send_message(rtk, APPLE_RTKIT_EP_SYSLOG, msg, NULL, false);
490 }
491
apple_rtkit_syslog_rx(struct apple_rtkit * rtk,u64 msg)492 static void apple_rtkit_syslog_rx(struct apple_rtkit *rtk, u64 msg)
493 {
494 u8 type = FIELD_GET(APPLE_RTKIT_SYSLOG_TYPE, msg);
495
496 switch (type) {
497 case APPLE_RTKIT_BUFFER_REQUEST:
498 apple_rtkit_common_rx_get_buffer(rtk, &rtk->syslog_buffer,
499 APPLE_RTKIT_EP_SYSLOG, msg);
500 break;
501 case APPLE_RTKIT_SYSLOG_INIT:
502 apple_rtkit_syslog_rx_init(rtk, msg);
503 break;
504 case APPLE_RTKIT_SYSLOG_LOG:
505 apple_rtkit_syslog_rx_log(rtk, msg);
506 break;
507 default:
508 dev_warn(rtk->dev, "RTKit: Unknown syslog message: %llx\n",
509 msg);
510 }
511 }
512
apple_rtkit_oslog_rx(struct apple_rtkit * rtk,u64 msg)513 static void apple_rtkit_oslog_rx(struct apple_rtkit *rtk, u64 msg)
514 {
515 u8 type = FIELD_GET(APPLE_RTKIT_OSLOG_TYPE, msg);
516
517 switch (type) {
518 case APPLE_RTKIT_OSLOG_BUFFER_REQUEST:
519 apple_rtkit_common_rx_get_buffer(rtk, &rtk->oslog_buffer,
520 APPLE_RTKIT_EP_OSLOG, msg);
521 break;
522 default:
523 dev_warn(rtk->dev, "RTKit: Unknown oslog message: %llx\n",
524 msg);
525 }
526 }
527
apple_rtkit_rx_work(struct work_struct * work)528 static void apple_rtkit_rx_work(struct work_struct *work)
529 {
530 struct apple_rtkit_rx_work *rtk_work =
531 container_of(work, struct apple_rtkit_rx_work, work);
532 struct apple_rtkit *rtk = rtk_work->rtk;
533
534 switch (rtk_work->ep) {
535 case APPLE_RTKIT_EP_MGMT:
536 apple_rtkit_management_rx(rtk, rtk_work->msg);
537 break;
538 case APPLE_RTKIT_EP_CRASHLOG:
539 apple_rtkit_crashlog_rx(rtk, rtk_work->msg);
540 break;
541 case APPLE_RTKIT_EP_SYSLOG:
542 apple_rtkit_syslog_rx(rtk, rtk_work->msg);
543 break;
544 case APPLE_RTKIT_EP_IOREPORT:
545 apple_rtkit_ioreport_rx(rtk, rtk_work->msg);
546 break;
547 case APPLE_RTKIT_EP_OSLOG:
548 apple_rtkit_oslog_rx(rtk, rtk_work->msg);
549 break;
550 case APPLE_RTKIT_APP_ENDPOINT_START ... 0xff:
551 if (rtk->ops->recv_message)
552 rtk->ops->recv_message(rtk->cookie, rtk_work->ep,
553 rtk_work->msg);
554 else
555 dev_warn(
556 rtk->dev,
557 "Received unexpected message to EP%02d: %llx\n",
558 rtk_work->ep, rtk_work->msg);
559 break;
560 default:
561 dev_warn(rtk->dev,
562 "RTKit: message to unknown endpoint %02x: %llx\n",
563 rtk_work->ep, rtk_work->msg);
564 }
565
566 kfree(rtk_work);
567 }
568
apple_rtkit_rx(struct apple_mbox * mbox,struct apple_mbox_msg msg,void * cookie)569 static void apple_rtkit_rx(struct apple_mbox *mbox, struct apple_mbox_msg msg,
570 void *cookie)
571 {
572 struct apple_rtkit *rtk = cookie;
573 struct apple_rtkit_rx_work *work;
574 u8 ep = msg.msg1;
575
576 /*
577 * The message was read from a MMIO FIFO and we have to make
578 * sure all reads from buffers sent with that message happen
579 * afterwards.
580 */
581 dma_rmb();
582
583 if (!test_bit(ep, rtk->endpoints))
584 dev_warn(rtk->dev,
585 "RTKit: Message to undiscovered endpoint 0x%02x\n",
586 ep);
587
588 if (ep >= APPLE_RTKIT_APP_ENDPOINT_START &&
589 rtk->ops->recv_message_early &&
590 rtk->ops->recv_message_early(rtk->cookie, ep, msg.msg0))
591 return;
592
593 work = kzalloc(sizeof(*work), GFP_ATOMIC);
594 if (!work)
595 return;
596
597 work->rtk = rtk;
598 work->ep = ep;
599 work->msg = msg.msg0;
600 INIT_WORK(&work->work, apple_rtkit_rx_work);
601 queue_work(rtk->wq, &work->work);
602 }
603
apple_rtkit_send_message(struct apple_rtkit * rtk,u8 ep,u64 message,struct completion * completion,bool atomic)604 int apple_rtkit_send_message(struct apple_rtkit *rtk, u8 ep, u64 message,
605 struct completion *completion, bool atomic)
606 {
607 struct apple_mbox_msg msg = {
608 .msg0 = message,
609 .msg1 = ep,
610 };
611
612 if (rtk->crashed) {
613 dev_warn(rtk->dev,
614 "RTKit: Device is crashed, cannot send message\n");
615 return -EINVAL;
616 }
617
618 if (ep >= APPLE_RTKIT_APP_ENDPOINT_START &&
619 !apple_rtkit_is_running(rtk)) {
620 dev_warn(rtk->dev,
621 "RTKit: Endpoint 0x%02x is not running, cannot send message\n", ep);
622 return -EINVAL;
623 }
624
625 /*
626 * The message will be sent with a MMIO write. We need the barrier
627 * here to ensure any previous writes to buffers are visible to the
628 * device before that MMIO write happens.
629 */
630 dma_wmb();
631
632 return apple_mbox_send(rtk->mbox, msg, atomic);
633 }
634 EXPORT_SYMBOL_GPL(apple_rtkit_send_message);
635
apple_rtkit_poll(struct apple_rtkit * rtk)636 int apple_rtkit_poll(struct apple_rtkit *rtk)
637 {
638 return apple_mbox_poll(rtk->mbox);
639 }
640 EXPORT_SYMBOL_GPL(apple_rtkit_poll);
641
apple_rtkit_start_ep(struct apple_rtkit * rtk,u8 endpoint)642 int apple_rtkit_start_ep(struct apple_rtkit *rtk, u8 endpoint)
643 {
644 u64 msg;
645
646 if (!test_bit(endpoint, rtk->endpoints))
647 return -EINVAL;
648 if (endpoint >= APPLE_RTKIT_APP_ENDPOINT_START &&
649 !apple_rtkit_is_running(rtk))
650 return -EINVAL;
651
652 msg = FIELD_PREP(APPLE_RTKIT_MGMT_STARTEP_EP, endpoint);
653 msg |= APPLE_RTKIT_MGMT_STARTEP_FLAG;
654 apple_rtkit_management_send(rtk, APPLE_RTKIT_MGMT_STARTEP, msg);
655
656 return 0;
657 }
658 EXPORT_SYMBOL_GPL(apple_rtkit_start_ep);
659
apple_rtkit_init(struct device * dev,void * cookie,const char * mbox_name,int mbox_idx,const struct apple_rtkit_ops * ops)660 struct apple_rtkit *apple_rtkit_init(struct device *dev, void *cookie,
661 const char *mbox_name, int mbox_idx,
662 const struct apple_rtkit_ops *ops)
663 {
664 struct apple_rtkit *rtk;
665 int ret;
666
667 if (!ops)
668 return ERR_PTR(-EINVAL);
669
670 rtk = kzalloc(sizeof(*rtk), GFP_KERNEL);
671 if (!rtk)
672 return ERR_PTR(-ENOMEM);
673
674 rtk->dev = dev;
675 rtk->cookie = cookie;
676 rtk->ops = ops;
677
678 init_completion(&rtk->epmap_completion);
679 init_completion(&rtk->iop_pwr_ack_completion);
680 init_completion(&rtk->ap_pwr_ack_completion);
681
682 bitmap_zero(rtk->endpoints, APPLE_RTKIT_MAX_ENDPOINTS);
683 set_bit(APPLE_RTKIT_EP_MGMT, rtk->endpoints);
684
685 if (mbox_name)
686 rtk->mbox = apple_mbox_get_byname(dev, mbox_name);
687 else
688 rtk->mbox = apple_mbox_get(dev, mbox_idx);
689
690 if (IS_ERR(rtk->mbox)) {
691 ret = PTR_ERR(rtk->mbox);
692 goto free_rtk;
693 }
694
695 rtk->mbox->rx = apple_rtkit_rx;
696 rtk->mbox->cookie = rtk;
697
698 rtk->wq = alloc_ordered_workqueue("rtkit-%s", WQ_HIGHPRI | WQ_MEM_RECLAIM,
699 dev_name(rtk->dev));
700 if (!rtk->wq) {
701 ret = -ENOMEM;
702 goto free_rtk;
703 }
704
705 ret = apple_mbox_start(rtk->mbox);
706 if (ret)
707 goto destroy_wq;
708
709 return rtk;
710
711 destroy_wq:
712 destroy_workqueue(rtk->wq);
713 free_rtk:
714 kfree(rtk);
715 return ERR_PTR(ret);
716 }
717 EXPORT_SYMBOL_GPL(apple_rtkit_init);
718
apple_rtkit_wait_for_completion(struct completion * c)719 static int apple_rtkit_wait_for_completion(struct completion *c)
720 {
721 long t;
722
723 t = wait_for_completion_interruptible_timeout(c,
724 msecs_to_jiffies(1000));
725 if (t < 0)
726 return t;
727 else if (t == 0)
728 return -ETIME;
729 else
730 return 0;
731 }
732
apple_rtkit_reinit(struct apple_rtkit * rtk)733 int apple_rtkit_reinit(struct apple_rtkit *rtk)
734 {
735 /* make sure we don't handle any messages while reinitializing */
736 apple_mbox_stop(rtk->mbox);
737 flush_workqueue(rtk->wq);
738
739 apple_rtkit_free_buffer(rtk, &rtk->ioreport_buffer);
740 apple_rtkit_free_buffer(rtk, &rtk->crashlog_buffer);
741 apple_rtkit_free_buffer(rtk, &rtk->oslog_buffer);
742 apple_rtkit_free_buffer(rtk, &rtk->syslog_buffer);
743
744 kfree(rtk->syslog_msg_buffer);
745
746 rtk->syslog_msg_buffer = NULL;
747 rtk->syslog_n_entries = 0;
748 rtk->syslog_msg_size = 0;
749
750 bitmap_zero(rtk->endpoints, APPLE_RTKIT_MAX_ENDPOINTS);
751 set_bit(APPLE_RTKIT_EP_MGMT, rtk->endpoints);
752
753 reinit_completion(&rtk->epmap_completion);
754 reinit_completion(&rtk->iop_pwr_ack_completion);
755 reinit_completion(&rtk->ap_pwr_ack_completion);
756
757 rtk->crashed = false;
758 rtk->iop_power_state = APPLE_RTKIT_PWR_STATE_OFF;
759 rtk->ap_power_state = APPLE_RTKIT_PWR_STATE_OFF;
760
761 return apple_mbox_start(rtk->mbox);
762 }
763 EXPORT_SYMBOL_GPL(apple_rtkit_reinit);
764
apple_rtkit_set_ap_power_state(struct apple_rtkit * rtk,unsigned int state)765 static int apple_rtkit_set_ap_power_state(struct apple_rtkit *rtk,
766 unsigned int state)
767 {
768 u64 msg;
769 int ret;
770
771 reinit_completion(&rtk->ap_pwr_ack_completion);
772
773 msg = FIELD_PREP(APPLE_RTKIT_MGMT_PWR_STATE, state);
774 ret = apple_rtkit_management_send(rtk, APPLE_RTKIT_MGMT_SET_AP_PWR_STATE,
775 msg);
776 if (ret)
777 return ret;
778
779 ret = apple_rtkit_wait_for_completion(&rtk->ap_pwr_ack_completion);
780 if (ret)
781 return ret;
782
783 if (rtk->ap_power_state != state)
784 return -EINVAL;
785 return 0;
786 }
787
apple_rtkit_set_iop_power_state(struct apple_rtkit * rtk,unsigned int state)788 static int apple_rtkit_set_iop_power_state(struct apple_rtkit *rtk,
789 unsigned int state)
790 {
791 u64 msg;
792 int ret;
793
794 reinit_completion(&rtk->iop_pwr_ack_completion);
795
796 msg = FIELD_PREP(APPLE_RTKIT_MGMT_PWR_STATE, state);
797 ret = apple_rtkit_management_send(rtk, APPLE_RTKIT_MGMT_SET_IOP_PWR_STATE,
798 msg);
799 if (ret)
800 return ret;
801
802 ret = apple_rtkit_wait_for_completion(&rtk->iop_pwr_ack_completion);
803 if (ret)
804 return ret;
805
806 if (rtk->iop_power_state != state)
807 return -EINVAL;
808 return 0;
809 }
810
apple_rtkit_boot(struct apple_rtkit * rtk)811 int apple_rtkit_boot(struct apple_rtkit *rtk)
812 {
813 int ret;
814
815 if (apple_rtkit_is_running(rtk))
816 return 0;
817 if (rtk->crashed)
818 return -EINVAL;
819
820 dev_dbg(rtk->dev, "RTKit: waiting for boot to finish\n");
821 ret = apple_rtkit_wait_for_completion(&rtk->epmap_completion);
822 if (ret)
823 return ret;
824 if (rtk->boot_result)
825 return rtk->boot_result;
826
827 dev_dbg(rtk->dev, "RTKit: waiting for IOP power state ACK\n");
828 ret = apple_rtkit_wait_for_completion(&rtk->iop_pwr_ack_completion);
829 if (ret)
830 return ret;
831
832 return apple_rtkit_set_ap_power_state(rtk, APPLE_RTKIT_PWR_STATE_ON);
833 }
834 EXPORT_SYMBOL_GPL(apple_rtkit_boot);
835
apple_rtkit_shutdown(struct apple_rtkit * rtk)836 int apple_rtkit_shutdown(struct apple_rtkit *rtk)
837 {
838 int ret;
839
840 /* if OFF is used here the co-processor will not wake up again */
841 ret = apple_rtkit_set_ap_power_state(rtk,
842 APPLE_RTKIT_PWR_STATE_QUIESCED);
843 if (ret)
844 return ret;
845
846 ret = apple_rtkit_set_iop_power_state(rtk, APPLE_RTKIT_PWR_STATE_SLEEP);
847 if (ret)
848 return ret;
849
850 return apple_rtkit_reinit(rtk);
851 }
852 EXPORT_SYMBOL_GPL(apple_rtkit_shutdown);
853
apple_rtkit_idle(struct apple_rtkit * rtk)854 int apple_rtkit_idle(struct apple_rtkit *rtk)
855 {
856 int ret;
857
858 /* if OFF is used here the co-processor will not wake up again */
859 ret = apple_rtkit_set_ap_power_state(rtk,
860 APPLE_RTKIT_PWR_STATE_IDLE);
861 if (ret)
862 return ret;
863
864 ret = apple_rtkit_set_iop_power_state(rtk, APPLE_RTKIT_PWR_STATE_IDLE);
865 if (ret)
866 return ret;
867
868 rtk->iop_power_state = APPLE_RTKIT_PWR_STATE_IDLE;
869 rtk->ap_power_state = APPLE_RTKIT_PWR_STATE_IDLE;
870 return 0;
871 }
872 EXPORT_SYMBOL_GPL(apple_rtkit_idle);
873
apple_rtkit_quiesce(struct apple_rtkit * rtk)874 int apple_rtkit_quiesce(struct apple_rtkit *rtk)
875 {
876 int ret;
877
878 ret = apple_rtkit_set_ap_power_state(rtk,
879 APPLE_RTKIT_PWR_STATE_QUIESCED);
880 if (ret)
881 return ret;
882
883 ret = apple_rtkit_set_iop_power_state(rtk,
884 APPLE_RTKIT_PWR_STATE_QUIESCED);
885 if (ret)
886 return ret;
887
888 ret = apple_rtkit_reinit(rtk);
889 if (ret)
890 return ret;
891
892 rtk->iop_power_state = APPLE_RTKIT_PWR_STATE_QUIESCED;
893 rtk->ap_power_state = APPLE_RTKIT_PWR_STATE_QUIESCED;
894 return 0;
895 }
896 EXPORT_SYMBOL_GPL(apple_rtkit_quiesce);
897
apple_rtkit_wake(struct apple_rtkit * rtk)898 int apple_rtkit_wake(struct apple_rtkit *rtk)
899 {
900 u64 msg;
901 int ret;
902
903 if (apple_rtkit_is_running(rtk))
904 return -EINVAL;
905
906 reinit_completion(&rtk->iop_pwr_ack_completion);
907
908 /*
909 * Use open-coded apple_rtkit_set_iop_power_state since apple_rtkit_boot
910 * will wait for the completion anyway.
911 */
912 msg = FIELD_PREP(APPLE_RTKIT_MGMT_PWR_STATE, APPLE_RTKIT_PWR_STATE_INIT);
913 ret = apple_rtkit_management_send(rtk, APPLE_RTKIT_MGMT_SET_IOP_PWR_STATE,
914 msg);
915 if (ret)
916 return ret;
917
918 return apple_rtkit_boot(rtk);
919 }
920 EXPORT_SYMBOL_GPL(apple_rtkit_wake);
921
apple_rtkit_free(struct apple_rtkit * rtk)922 void apple_rtkit_free(struct apple_rtkit *rtk)
923 {
924 apple_mbox_stop(rtk->mbox);
925 destroy_workqueue(rtk->wq);
926
927 apple_rtkit_free_buffer(rtk, &rtk->ioreport_buffer);
928 apple_rtkit_free_buffer(rtk, &rtk->crashlog_buffer);
929 apple_rtkit_free_buffer(rtk, &rtk->oslog_buffer);
930 apple_rtkit_free_buffer(rtk, &rtk->syslog_buffer);
931
932 kfree(rtk->syslog_msg_buffer);
933 kfree(rtk);
934 }
935 EXPORT_SYMBOL_GPL(apple_rtkit_free);
936
apple_rtkit_free_wrapper(void * data)937 static void apple_rtkit_free_wrapper(void *data)
938 {
939 apple_rtkit_free(data);
940 }
941
devm_apple_rtkit_init(struct device * dev,void * cookie,const char * mbox_name,int mbox_idx,const struct apple_rtkit_ops * ops)942 struct apple_rtkit *devm_apple_rtkit_init(struct device *dev, void *cookie,
943 const char *mbox_name, int mbox_idx,
944 const struct apple_rtkit_ops *ops)
945 {
946 struct apple_rtkit *rtk;
947 int ret;
948
949 rtk = apple_rtkit_init(dev, cookie, mbox_name, mbox_idx, ops);
950 if (IS_ERR(rtk))
951 return rtk;
952
953 ret = devm_add_action_or_reset(dev, apple_rtkit_free_wrapper, rtk);
954 if (ret)
955 return ERR_PTR(ret);
956
957 return rtk;
958 }
959 EXPORT_SYMBOL_GPL(devm_apple_rtkit_init);
960
961 MODULE_LICENSE("Dual MIT/GPL");
962 MODULE_AUTHOR("Sven Peter <sven@svenpeter.dev>");
963 MODULE_DESCRIPTION("Apple RTKit driver");
964