1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2026 Intel Corporation
4 */
5
6 #include <linux/bitfield.h>
7 #include <linux/cleanup.h>
8 #include <linux/minmax.h>
9 #include <linux/slab.h>
10 #include <linux/string.h>
11
12 #include "regs/xe_sysctrl_regs.h"
13 #include "xe_device.h"
14 #include "xe_mmio.h"
15 #include "xe_pm.h"
16 #include "xe_printk.h"
17 #include "xe_sysctrl.h"
18 #include "xe_sysctrl_mailbox.h"
19 #include "xe_sysctrl_mailbox_types.h"
20
21 struct xe_sysctrl_mailbox_msg_hdr {
22 u32 data;
23 };
24
25 #define XE_SYSCTRL_HDR_GROUP_ID(hdr) \
26 FIELD_GET(SYSCTRL_HDR_GROUP_ID_MASK, (hdr)->data)
27
28 #define XE_SYSCTRL_HDR_COMMAND(hdr) \
29 FIELD_GET(SYSCTRL_HDR_COMMAND_MASK, (hdr)->data)
30
31 #define XE_SYSCTRL_HDR_IS_RESPONSE(hdr) \
32 FIELD_GET(SYSCTRL_HDR_IS_RESPONSE, (hdr)->data)
33
34 #define XE_SYSCTRL_HDR_RESULT(hdr) \
35 FIELD_GET(SYSCTRL_HDR_RESULT_MASK, (hdr)->data)
36
sysctrl_wait_bit_clear(struct xe_sysctrl * sc,u32 bit_mask,unsigned int timeout_ms)37 static bool sysctrl_wait_bit_clear(struct xe_sysctrl *sc, u32 bit_mask,
38 unsigned int timeout_ms)
39 {
40 int ret;
41
42 ret = xe_mmio_wait32_not(sc->mmio, SYSCTRL_MB_CTRL, bit_mask, bit_mask,
43 timeout_ms * 1000, NULL, false);
44
45 return ret == 0;
46 }
47
sysctrl_wait_bit_set(struct xe_sysctrl * sc,u32 bit_mask,unsigned int timeout_ms)48 static bool sysctrl_wait_bit_set(struct xe_sysctrl *sc, u32 bit_mask,
49 unsigned int timeout_ms)
50 {
51 int ret;
52
53 ret = xe_mmio_wait32(sc->mmio, SYSCTRL_MB_CTRL, bit_mask, bit_mask,
54 timeout_ms * 1000, NULL, false);
55
56 return ret == 0;
57 }
58
sysctrl_write_frame(struct xe_sysctrl * sc,const void * frame,size_t len)59 static int sysctrl_write_frame(struct xe_sysctrl *sc, const void *frame,
60 size_t len)
61 {
62 static const struct xe_reg regs[] = {
63 SYSCTRL_MB_DATA0, SYSCTRL_MB_DATA1, SYSCTRL_MB_DATA2, SYSCTRL_MB_DATA3
64 };
65 struct xe_device *xe = sc_to_xe(sc);
66 u32 val[XE_SYSCTRL_MB_FRAME_SIZE / sizeof(u32)] = {0};
67 u32 dw = DIV_ROUND_UP(len, sizeof(u32));
68 u32 i;
69
70 xe_assert(xe, len > 0 && len <= XE_SYSCTRL_MB_FRAME_SIZE);
71
72 memcpy(val, frame, len);
73
74 for (i = 0; i < dw; i++)
75 xe_mmio_write32(sc->mmio, regs[i], val[i]);
76
77 return 0;
78 }
79
sysctrl_read_frame(struct xe_sysctrl * sc,void * frame,size_t len)80 static int sysctrl_read_frame(struct xe_sysctrl *sc, void *frame,
81 size_t len)
82 {
83 static const struct xe_reg regs[] = {
84 SYSCTRL_MB_DATA0, SYSCTRL_MB_DATA1, SYSCTRL_MB_DATA2, SYSCTRL_MB_DATA3
85 };
86 struct xe_device *xe = sc_to_xe(sc);
87 u32 val[XE_SYSCTRL_MB_FRAME_SIZE / sizeof(u32)] = {0};
88 u32 dw = DIV_ROUND_UP(len, sizeof(u32));
89 u32 i;
90
91 xe_assert(xe, len > 0 && len <= XE_SYSCTRL_MB_FRAME_SIZE);
92
93 for (i = 0; i < dw; i++)
94 val[i] = xe_mmio_read32(sc->mmio, regs[i]);
95
96 memcpy(frame, val, len);
97
98 return 0;
99 }
100
sysctrl_clear_response(struct xe_sysctrl * sc)101 static void sysctrl_clear_response(struct xe_sysctrl *sc)
102 {
103 xe_mmio_rmw32(sc->mmio, SYSCTRL_MB_CTRL, SYSCTRL_MB_CTRL_RUN_BUSY_OUT, 0);
104 }
105
sysctrl_prepare_command(struct xe_device * xe,u8 group_id,u8 command,const void * data_in,size_t data_in_len,u8 ** mbox_cmd,size_t * cmd_size)106 static int sysctrl_prepare_command(struct xe_device *xe,
107 u8 group_id, u8 command,
108 const void *data_in, size_t data_in_len,
109 u8 **mbox_cmd, size_t *cmd_size)
110 {
111 struct xe_sysctrl_mailbox_msg_hdr *hdr;
112 size_t size;
113 u8 *buffer;
114
115 xe_assert(xe, command <= SYSCTRL_HDR_COMMAND_MAX);
116
117 if (data_in_len > XE_SYSCTRL_MB_MAX_MESSAGE_SIZE - sizeof(*hdr)) {
118 xe_err(xe, "sysctrl: Input data too large: %zu bytes\n", data_in_len);
119 return -EINVAL;
120 }
121
122 size = sizeof(*hdr) + data_in_len;
123
124 buffer = kmalloc(size, GFP_KERNEL);
125 if (!buffer)
126 return -ENOMEM;
127
128 hdr = (struct xe_sysctrl_mailbox_msg_hdr *)buffer;
129 hdr->data = FIELD_PREP(SYSCTRL_HDR_GROUP_ID_MASK, group_id) |
130 FIELD_PREP(SYSCTRL_HDR_COMMAND_MASK, command);
131
132 if (data_in && data_in_len)
133 memcpy(buffer + sizeof(*hdr), data_in, data_in_len);
134
135 *mbox_cmd = buffer;
136 *cmd_size = size;
137
138 return 0;
139 }
140
sysctrl_send_frames(struct xe_sysctrl * sc,const u8 * mbox_cmd,size_t cmd_size,unsigned int timeout_ms)141 static int sysctrl_send_frames(struct xe_sysctrl *sc,
142 const u8 *mbox_cmd,
143 size_t cmd_size, unsigned int timeout_ms)
144 {
145 struct xe_device *xe = sc_to_xe(sc);
146 u32 ctrl_reg, total_frames, frame;
147 size_t bytes_sent, frame_size;
148 bool phase;
149
150 total_frames = DIV_ROUND_UP(cmd_size, XE_SYSCTRL_MB_FRAME_SIZE);
151
152 if (!sysctrl_wait_bit_clear(sc, SYSCTRL_MB_CTRL_RUN_BUSY, timeout_ms)) {
153 xe_err(xe, "sysctrl: Mailbox busy\n");
154 return -EBUSY;
155 }
156
157 ctrl_reg = xe_mmio_read32(sc->mmio, SYSCTRL_MB_CTRL);
158 phase = !(ctrl_reg & SYSCTRL_FRAME_PHASE);
159 bytes_sent = 0;
160
161 for (frame = 0; frame < total_frames; frame++) {
162 frame_size = min_t(size_t, cmd_size - bytes_sent, XE_SYSCTRL_MB_FRAME_SIZE);
163
164 if (sysctrl_write_frame(sc, mbox_cmd + bytes_sent, frame_size)) {
165 xe_err(xe, "sysctrl: Failed to write frame %u\n", frame);
166 return -EIO;
167 }
168
169 ctrl_reg = SYSCTRL_MB_CTRL_RUN_BUSY |
170 REG_FIELD_PREP(SYSCTRL_FRAME_CURRENT_MASK, frame) |
171 REG_FIELD_PREP(SYSCTRL_FRAME_TOTAL_MASK, total_frames - 1) |
172 SYSCTRL_MB_CTRL_CMD |
173 (phase ? SYSCTRL_FRAME_PHASE : 0);
174
175 xe_mmio_write32(sc->mmio, SYSCTRL_MB_CTRL, ctrl_reg);
176
177 if (!sysctrl_wait_bit_clear(sc, SYSCTRL_MB_CTRL_RUN_BUSY, timeout_ms)) {
178 xe_err(xe, "sysctrl: Frame %u acknowledgment timeout\n", frame);
179 return -ETIMEDOUT;
180 }
181
182 bytes_sent += frame_size;
183 }
184
185 return 0;
186 }
187
sysctrl_process_frame(struct xe_sysctrl * sc,void * out,size_t frame_size,unsigned int timeout_ms,bool * done)188 static int sysctrl_process_frame(struct xe_sysctrl *sc, void *out,
189 size_t frame_size, unsigned int timeout_ms,
190 bool *done)
191 {
192 u32 curr_frame, total_frames, ctrl_reg;
193 struct xe_device *xe = sc_to_xe(sc);
194 int ret;
195
196 if (!sysctrl_wait_bit_set(sc, SYSCTRL_MB_CTRL_RUN_BUSY_OUT, timeout_ms)) {
197 xe_err(xe, "sysctrl: Response frame timeout\n");
198 return -ETIMEDOUT;
199 }
200
201 ctrl_reg = xe_mmio_read32(sc->mmio, SYSCTRL_MB_CTRL);
202 total_frames = FIELD_GET(SYSCTRL_FRAME_TOTAL_MASK, ctrl_reg);
203 curr_frame = FIELD_GET(SYSCTRL_FRAME_CURRENT_MASK, ctrl_reg);
204
205 ret = sysctrl_read_frame(sc, out, frame_size);
206 if (ret)
207 return ret;
208
209 sysctrl_clear_response(sc);
210
211 if (curr_frame == total_frames)
212 *done = true;
213
214 return 0;
215 }
216
sysctrl_receive_frames(struct xe_sysctrl * sc,const struct xe_sysctrl_mailbox_msg_hdr * req,void * data_out,size_t data_out_len,size_t * rdata_len,unsigned int timeout_ms)217 static int sysctrl_receive_frames(struct xe_sysctrl *sc,
218 const struct xe_sysctrl_mailbox_msg_hdr *req,
219 void *data_out, size_t data_out_len,
220 size_t *rdata_len, unsigned int timeout_ms)
221 {
222 struct xe_sysctrl_mailbox_msg_hdr *hdr;
223 struct xe_device *xe = sc_to_xe(sc);
224 size_t remain = sizeof(*hdr) + data_out_len;
225 u8 *buffer __free(kfree) = kzalloc(remain, GFP_KERNEL);
226 size_t frame_size;
227 bool done = false;
228 int ret = 0;
229 u8 *out;
230
231 if (!buffer)
232 return -ENOMEM;
233
234 out = buffer;
235 while (!done && remain) {
236 frame_size = min_t(size_t, remain, XE_SYSCTRL_MB_FRAME_SIZE);
237
238 ret = sysctrl_process_frame(sc, out, frame_size, timeout_ms,
239 &done);
240 if (ret)
241 return ret;
242
243 remain -= frame_size;
244 out += frame_size;
245 }
246
247 hdr = (struct xe_sysctrl_mailbox_msg_hdr *)buffer;
248
249 if (!XE_SYSCTRL_HDR_IS_RESPONSE(hdr) ||
250 XE_SYSCTRL_HDR_GROUP_ID(hdr) != XE_SYSCTRL_HDR_GROUP_ID(req) ||
251 XE_SYSCTRL_HDR_COMMAND(hdr) != XE_SYSCTRL_HDR_COMMAND(req)) {
252 xe_err(xe, "sysctrl: Response header mismatch\n");
253 return -EPROTO;
254 }
255
256 if (XE_SYSCTRL_HDR_RESULT(hdr) != 0) {
257 xe_err(xe, "sysctrl: Firmware error: 0x%02lx\n",
258 XE_SYSCTRL_HDR_RESULT(hdr));
259 return -EIO;
260 }
261
262 memcpy(data_out, hdr + 1, data_out_len);
263 *rdata_len = out - buffer - sizeof(*hdr);
264
265 return 0;
266 }
267
sysctrl_send_command(struct xe_sysctrl * sc,const u8 * mbox_cmd,size_t cmd_size,void * data_out,size_t data_out_len,size_t * rdata_len,unsigned int timeout_ms)268 static int sysctrl_send_command(struct xe_sysctrl *sc,
269 const u8 *mbox_cmd, size_t cmd_size,
270 void *data_out, size_t data_out_len,
271 size_t *rdata_len, unsigned int timeout_ms)
272 {
273 const struct xe_sysctrl_mailbox_msg_hdr *hdr;
274 size_t received;
275 int ret;
276
277 ret = sysctrl_send_frames(sc, mbox_cmd, cmd_size, timeout_ms);
278 if (ret)
279 return ret;
280
281 if (!data_out || !rdata_len)
282 return 0;
283
284 hdr = (const struct xe_sysctrl_mailbox_msg_hdr *)mbox_cmd;
285
286 ret = sysctrl_receive_frames(sc, hdr, data_out, data_out_len,
287 &received, timeout_ms);
288 if (ret)
289 return ret;
290
291 *rdata_len = received;
292
293 return 0;
294 }
295
296 /**
297 * xe_sysctrl_create_command() - Create system controller command
298 * @command: Sysctrl command structure
299 * @group_id: Command group ID
300 * @cmd_id: Command ID
301 * @request: Pointer to request buffer (can be NULL)
302 * @request_len: Size of request buffer
303 * @response: Pointer to response buffer
304 * @response_len: Size of response buffer
305 *
306 * Helper function to create sysctrl command to be sent via %xe_sysctrl_send_command()
307 */
xe_sysctrl_create_command(struct xe_sysctrl_mailbox_command * command,u8 group_id,u8 cmd_id,void * request,size_t request_len,void * response,size_t response_len)308 void xe_sysctrl_create_command(struct xe_sysctrl_mailbox_command *command, u8 group_id, u8 cmd_id,
309 void *request, size_t request_len, void *response,
310 size_t response_len)
311 {
312 struct xe_sysctrl_app_msg_hdr header = {0};
313
314 header.data = FIELD_PREP(APP_HDR_GROUP_ID_MASK, group_id) |
315 FIELD_PREP(APP_HDR_COMMAND_MASK, cmd_id);
316
317 command->header = header;
318 command->data_in = request;
319 command->data_in_len = request_len;
320 command->data_out = response;
321 command->data_out_len = response_len;
322 }
323
324 /**
325 * xe_sysctrl_send_command() - Send mailbox command to System Controller
326 * @sc: System Controller instance
327 * @cmd: Command descriptor containing request header and payload buffers
328 * @rdata_len: Pointer to store actual response data length
329 *
330 * Sends a mailbox command to System Controller firmware using
331 * System Controller mailbox and waits for a response.
332 *
333 * Request payload is provided via @cmd->data_in and @cmd->data_in_len.
334 * If a response is expected, @cmd->data_out must point to a buffer of
335 * size @cmd->data_out_len supplied by caller.
336 *
337 * On success, @rdata_len is updated with number of valid response bytes
338 * returned by firmware, bounded by @cmd->data_out_len.
339 *
340 * Return: 0 on success, or negative errno on failure.
341 */
xe_sysctrl_send_command(struct xe_sysctrl * sc,struct xe_sysctrl_mailbox_command * cmd,size_t * rdata_len)342 int xe_sysctrl_send_command(struct xe_sysctrl *sc,
343 struct xe_sysctrl_mailbox_command *cmd,
344 size_t *rdata_len)
345 {
346 struct xe_device *xe = sc_to_xe(sc);
347 u8 group_id, command_code;
348 u8 *mbox_cmd = NULL;
349 size_t cmd_size = 0;
350 int ret;
351
352 guard(xe_pm_runtime_noresume)(xe);
353
354 if (!xe->info.has_sysctrl)
355 return -ENODEV;
356
357 xe_assert(xe, cmd->data_in || cmd->data_out);
358 xe_assert(xe, !cmd->data_in || cmd->data_in_len);
359 xe_assert(xe, !cmd->data_out || cmd->data_out_len);
360
361 group_id = XE_SYSCTRL_APP_HDR_GROUP_ID(&cmd->header);
362 command_code = XE_SYSCTRL_APP_HDR_COMMAND(&cmd->header);
363
364 might_sleep();
365
366 ret = sysctrl_prepare_command(xe, group_id, command_code,
367 cmd->data_in, cmd->data_in_len,
368 &mbox_cmd, &cmd_size);
369 if (ret) {
370 xe_err(xe, "sysctrl: Failed to prepare command: %pe\n", ERR_PTR(ret));
371 return ret;
372 }
373
374 guard(mutex)(&sc->cmd_lock);
375
376 ret = sysctrl_send_command(sc, mbox_cmd, cmd_size,
377 cmd->data_out, cmd->data_out_len, rdata_len,
378 XE_SYSCTRL_MB_DEFAULT_TIMEOUT_MS);
379 if (ret)
380 xe_err(xe, "sysctrl: Mailbox command failed: %pe\n", ERR_PTR(ret));
381
382 kfree(mbox_cmd);
383
384 return ret;
385 }
386