1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /* Copyright (C) 2025 Intel Corporation */
3
4 #ifndef __LIBIE_CONTROLQ_H
5 #define __LIBIE_CONTROLQ_H
6
7 #include <linux/dmapool.h>
8 #include <net/libeth/rx.h>
9
10 #include <linux/net/intel/libie/pci.h>
11 #include <linux/net/intel/virtchnl2.h>
12
13 /* Default mailbox control queue */
14 #define LIBIE_CTLQ_MBX_ID -1
15 #define LIBIE_CTLQ_MAX_BUF_LEN SZ_4K
16
17 /**
18 * enum libie_ctlq_type - control queue type
19 * @LIBIE_CTLQ_TYPE_TX: basic Tx control queue
20 * @LIBIE_CTLQ_TYPE_RX: basic Rx control queue
21 */
22 enum libie_ctlq_type {
23 LIBIE_CTLQ_TYPE_TX = 0,
24 LIBIE_CTLQ_TYPE_RX = 1,
25 };
26
27 /* Opcode used to send controlq message to the control plane */
28 #define LIBIE_CTLQ_SEND_MSG_TO_CP 0x801
29 #define LIBIE_CTLQ_SEND_MSG_TO_PEER 0x804
30
31 #define LIBIE_CP_TX_COPYBREAK 128
32
33 /**
34 * struct libie_ctlq_ctx - contains controlq info and MMIO region info
35 * @mmio_info: MMIO region info structure
36 * @ctlqs: list that stores all the control queues
37 * @ctlqs_lock: lock for control queue list
38 */
39 struct libie_ctlq_ctx {
40 struct libie_mmio_info mmio_info;
41 struct list_head ctlqs;
42 spinlock_t ctlqs_lock; /* protects the ctlqs list */
43 };
44
45 /**
46 * struct libie_ctlq_reg - structure representing virtual addresses of the
47 * controlq registers and masks
48 * @head: controlq head register address
49 * @tail: controlq tail register address
50 * @len: register address to write controlq length and enable bit
51 * @addr_high: register address to write the upper 32b of ring physical address
52 * @addr_low: register address to write the lower 32b of ring physical address
53 * @len_mask: mask to read the controlq length
54 * @len_ena_mask: mask to write the controlq enable bit
55 * @head_mask: mask to read the head value
56 */
57 struct libie_ctlq_reg {
58 void __iomem *head;
59 void __iomem *tail;
60 void __iomem *len;
61 void __iomem *addr_high;
62 void __iomem *addr_low;
63 u32 len_mask;
64 u32 len_ena_mask;
65 u32 head_mask;
66 };
67
68 /**
69 * struct libie_cp_dma_mem - structure for DMA memory
70 * @va: virtual address
71 * @pa: physical address
72 * @size: memory size
73 * @direction: memory to device or device to memory
74 */
75 struct libie_cp_dma_mem {
76 void *va;
77 dma_addr_t pa;
78 size_t size;
79 int direction;
80 };
81
82 /**
83 * struct libie_ctlq_msg - control queue message data
84 * @flags: refer to 'Flags sub-structure' definitions
85 * @opcode: infrastructure message opcode
86 * @data_len: size of the payload
87 * @func_id: queue id for mailbox selection, 0 for default mailbox (Tx)
88 * @hw_retval: execution status from the HW (Rx)
89 * @chnl_opcode: virtchnl message opcode
90 * @chnl_retval: virtchnl return value
91 * @param0: indirect message raw parameter0
92 * @sw_cookie: used to verify the response of the sent virtchnl message
93 * @virt_flags: virtchnl capability flags
94 * @addr_param: additional parameters in place of the address, given no buffer
95 * @recv_mem: virtual address and size of the buffer that contains
96 * the indirect response
97 * @send_mem: physical and virtual address of the DMA buffer,
98 * used for sending
99 */
100 struct libie_ctlq_msg {
101 u16 flags;
102 u16 opcode;
103 u16 data_len;
104 union {
105 u16 func_id;
106 u16 hw_retval;
107 };
108 u32 chnl_opcode;
109 u32 chnl_retval;
110 u32 param0;
111 u16 sw_cookie;
112 u16 virt_flags;
113 u64 addr_param;
114 union {
115 struct kvec recv_mem;
116 struct libie_cp_dma_mem send_mem;
117 };
118 };
119
120 /**
121 * struct libie_ctlq_create_info - control queue create information
122 * @type: control queue type (Rx or Tx)
123 * @id: queue offset passed as input, -1 for default mailbox
124 * @reg: registers accessed by control queue
125 * @len: controlq length
126 */
127 struct libie_ctlq_create_info {
128 enum libie_ctlq_type type;
129 int id;
130 struct libie_ctlq_reg reg;
131 u16 len;
132 };
133
134 /**
135 * struct libie_ctlq_info - control queue information
136 * @list: used to add a controlq to the list of queues in libie_ctlq_ctx
137 * @type: control queue type
138 * @qid: queue identifier
139 * @lock: control queue lock
140 * @ring_mem: descriptor ring DMA memory
141 * @descs: array of descriptors
142 * @rx_fqes: array of controlq Rx buffers
143 * @tx_msg: Tx messages sent to hardware
144 * @reg: registers used by control queue
145 * @dev: device that owns this control queue
146 * @pp: page pool for controlq Rx buffers
147 * @truesize: size to allocate per buffer
148 * @next_to_clean: next descriptor to be cleaned
149 * @next_to_use: next available slot to send buffer (Tx queue)
150 * @next_to_post: next available slot to post buffers to (Rx queue)
151 * @ring_len: length of the descriptor ring
152 */
153 struct libie_ctlq_info {
154 struct list_head list;
155 enum libie_ctlq_type type;
156 int qid;
157 spinlock_t lock; /* for concurrent processing */
158 struct libie_cp_dma_mem ring_mem;
159 struct libie_ctlq_desc *descs;
160 union {
161 struct libeth_fqe *rx_fqes;
162 struct libie_ctlq_msg **tx_msg;
163 };
164 struct libie_ctlq_reg reg;
165 struct device *dev;
166 struct page_pool *pp;
167 u32 truesize;
168 u32 next_to_clean;
169 union {
170 u32 next_to_use;
171 u32 next_to_post;
172 };
173 u32 ring_len;
174 };
175
176 #define LIBIE_CTLQ_MBX_ATQ_LEN GENMASK(9, 0)
177
178 /* libie controlq descriptor qword0 details */
179
180 /* Flags sub-structure
181 * |0 |1 |2 |3 |4 |5 |6 |7 |8 |9 |10 |11 |12 |13 |14 |15 |
182 * |DD |CMP|ERR| * RSV * |FTYPE | *RSV* |RD |VFC|BUF| HOST_ID |
183 */
184 #define LIBIE_CTLQ_DESC_FLAG_DD BIT(0)
185 #define LIBIE_CTLQ_DESC_FLAG_CMP BIT(1)
186 #define LIBIE_CTLQ_DESC_FLAG_ERR BIT(2)
187 #define LIBIE_CTLQ_DESC_FLAG_FTYPE_VM BIT(6)
188 #define LIBIE_CTLQ_DESC_FLAG_FTYPE_PF BIT(7)
189 #define LIBIE_CTLQ_DESC_FLAG_FTYPE GENMASK(7, 6)
190 #define LIBIE_CTLQ_DESC_FLAG_RD BIT(10)
191 #define LIBIE_CTLQ_DESC_FLAG_VFC BIT(11)
192 #define LIBIE_CTLQ_DESC_FLAG_BUF BIT(12)
193 #define LIBIE_CTLQ_DESC_FLAG_HOST_ID GENMASK(15, 13)
194
195 #define LIBIE_CTLQ_DESC_FLAGS GENMASK(15, 0)
196 #define LIBIE_CTLQ_DESC_INFRA_OPCODE GENMASK_ULL(31, 16)
197 #define LIBIE_CTLQ_DESC_DATA_LEN GENMASK_ULL(47, 32)
198 #define LIBIE_CTLQ_DESC_HW_RETVAL GENMASK_ULL(63, 48)
199
200 #define LIBIE_CTLQ_DESC_PFID_VFID GENMASK_ULL(63, 48)
201
202 /* libie controlq descriptor qword1 details */
203 #define LIBIE_CTLQ_DESC_VIRTCHNL_OPCODE GENMASK(27, 0)
204 #define LIBIE_CTLQ_DESC_VIRTCHNL_DESC_TYPE GENMASK_ULL(31, 28)
205 #define LIBIE_CTLQ_DESC_VIRTCHNL_MSG_RET_VAL GENMASK_ULL(63, 32)
206
207 /* libie controlq descriptor qword2 details */
208 #define LIBIE_CTLQ_DESC_MSG_PARAM0 GENMASK_ULL(31, 0)
209 #define LIBIE_CTLQ_DESC_SW_COOKIE GENMASK_ULL(47, 32)
210 #define LIBIE_CTLQ_DESC_VIRTCHNL_FLAGS GENMASK_ULL(63, 48)
211
212 /* libie controlq descriptor qword3 details */
213 #define LIBIE_CTLQ_DESC_DATA_ADDR_HIGH GENMASK_ULL(31, 0)
214 #define LIBIE_CTLQ_DESC_DATA_ADDR_LOW GENMASK_ULL(63, 32)
215
216 /**
217 * struct libie_ctlq_desc - control queue descriptor format
218 * @qword0: flags, message opcode, data length etc
219 * @qword1: virtchnl opcode, descriptor type and return value
220 * @qword2: indirect message parameters
221 * @qword3: indirect message buffer address
222 */
223 struct libie_ctlq_desc {
224 __le64 qword0;
225 __le64 qword1;
226 __le64 qword2;
227 __le64 qword3;
228 };
229
230 /**
231 * struct libie_ctlq_clean_params - cleaning parameters for Tx messages
232 * @rel_dma_mem: non-sleeping callback to put the DMA buffer after send
233 * @rel_ctx: additional context for release callback
234 * @ctlq: control queue information
235 * @num_msgs: number of messages to be cleaned
236 * @force: clean even if DD is not yet set, use only for final cleanup
237 */
238 struct libie_ctlq_clean_params {
239 void (*rel_dma_mem)(const void *ctx, struct libie_cp_dma_mem *dma_mem);
240 const void *rel_ctx;
241 struct libie_ctlq_info *ctlq;
242 u16 num_msgs;
243 bool force;
244 };
245
246 /**
247 * libie_ctlq_release_rx_buf - Release Rx buffer for a specific control queue
248 * @rx_buf: Rx buffer to be freed
249 *
250 * Driver uses this function to post back the Rx buffer after the usage.
251 */
libie_ctlq_release_rx_buf(struct kvec * rx_buf)252 static inline void libie_ctlq_release_rx_buf(struct kvec *rx_buf)
253 {
254 netmem_ref netmem;
255
256 if (!rx_buf->iov_base)
257 return;
258
259 netmem = virt_to_netmem(rx_buf->iov_base);
260 page_pool_put_full_netmem(netmem_get_pp(netmem), netmem, false);
261 }
262
263 int libie_ctlq_init(struct libie_ctlq_ctx *ctx,
264 const struct libie_ctlq_create_info *qinfo, u32 numq);
265 void libie_ctlq_deinit(struct libie_ctlq_ctx *ctx);
266
267 struct libie_ctlq_info *libie_find_ctlq(struct libie_ctlq_ctx *ctx,
268 enum libie_ctlq_type type,
269 int id);
270
271 u32 libie_ctlq_send_desc_avail(const struct libie_ctlq_info *ctlq);
272 void libie_ctlq_send(struct libie_ctlq_info *ctlq, u32 num_q_msg);
273 u32 libie_ctlq_send_clean(const struct libie_ctlq_clean_params *params);
274 u32 libie_ctlq_recv(struct libie_ctlq_info *ctlq, struct libie_ctlq_msg *msg,
275 u32 num_q_msg);
276
277 int libie_ctlq_post_rx_buffs(struct libie_ctlq_info *ctlq);
278
279 /* Only 8 bits are available in descriptor for Xn index */
280 #define LIBIE_CTLQ_MAX_XN_ENTRIES 256
281 #define LIBIE_CTLQ_XN_COOKIE_M GENMASK(15, 8)
282 #define LIBIE_CTLQ_XN_INDEX_M GENMASK(7, 0)
283
284 /**
285 * enum libie_ctlq_xn_state - Transaction state of a virtchnl message
286 * @LIBIE_CTLQ_XN_IDLE: transaction is available to use
287 * @LIBIE_CTLQ_XN_WAITING: waiting for transaction to complete
288 * @LIBIE_CTLQ_XN_COMPLETED_SUCCESS: transaction completed with success
289 * @LIBIE_CTLQ_XN_COMPLETED_FAILED: transaction completed with failure
290 * @LIBIE_CTLQ_XN_ASYNC: asynchronous virtchnl message transaction type
291 * @LIBIE_CTLQ_XN_SHUTDOWN: transaction cannot be used anymore
292 */
293 enum libie_ctlq_xn_state {
294 LIBIE_CTLQ_XN_IDLE = 0,
295 LIBIE_CTLQ_XN_WAITING,
296 LIBIE_CTLQ_XN_COMPLETED_SUCCESS,
297 LIBIE_CTLQ_XN_COMPLETED_FAILED,
298 LIBIE_CTLQ_XN_ASYNC,
299 LIBIE_CTLQ_XN_SHUTDOWN,
300 };
301
302 /**
303 * struct libie_ctlq_xn - structure representing a virtchnl transaction entry
304 * @resp_cb: non-sleeping callback to handle the response to an async message
305 * @xn_lock: lock to protect the transaction entry state
306 * @cmd_completion_event: wait until reply is received or xn is terminated
307 * @small_dma_mem: DMA memory for copying small send buffers from stack,
308 * is recycled when response is received or on timeout
309 * @send_dma_mem: DMA memory of send buffer
310 * @recv_mem: receive buffer
311 * @send_ctx: context for callback function
312 * @timeout_ms: Xn transaction timeout in msecs
313 * @timestamp: timestamp to record the Xn send
314 * @tx_msg: control queue Tx message slot to track small DMA usage
315 * @virtchnl_opcode: virtchnl command opcode used for Xn transaction
316 * @state: transaction state of a virtchnl message
317 * @cookie: unique message identifier, incremented every time the slot is used
318 * @index: index of the transaction entry
319 */
320 struct libie_ctlq_xn {
321 void (*resp_cb)(void *ctx, struct kvec *mem, int status);
322 spinlock_t xn_lock; /* protects state */
323 struct completion cmd_completion_event;
324 struct libie_cp_dma_mem small_dma_mem;
325 struct libie_cp_dma_mem send_dma_mem;
326 struct kvec recv_mem;
327 void *send_ctx;
328 u64 timeout_ms;
329 ktime_t timestamp;
330 struct libie_ctlq_msg *tx_msg;
331 u32 virtchnl_opcode;
332 enum libie_ctlq_xn_state state;
333 u8 cookie;
334 u8 index;
335 };
336
337 /**
338 * struct libie_ctlq_xn_manager - structure representing the array of virtchnl
339 * transaction entries
340 * @ctx: pointer to controlq context structure
341 * @free_xns_bm_lock: lock to protect the free Xn entries bit map
342 * @free_xns_bm: bitmap that represents the free Xn entries
343 * @ring: array of Xn entries
344 * @small_buff_pool: DMA pool for small send buffers
345 * @can_destroy: completion, triggered by the last released transaction
346 * @shutdown: shutdown process has been started, no new transactions allowed
347 */
348 struct libie_ctlq_xn_manager {
349 struct libie_ctlq_ctx *ctx;
350 spinlock_t free_xns_bm_lock; /* get/check entries */
351 DECLARE_BITMAP(free_xns_bm, LIBIE_CTLQ_MAX_XN_ENTRIES);
352 struct libie_ctlq_xn ring[LIBIE_CTLQ_MAX_XN_ENTRIES];
353 struct dma_pool *small_buff_pool;
354 struct completion can_destroy;
355 bool shutdown;
356 };
357
358 /**
359 * struct libie_ctlq_xn_send_params - structure representing send Xn entry
360 * @resp_cb: non-sleeping callback to handle the response to an async message
361 * @rel_tx_buf: non-sleeping callback for freeing the send buffer
362 * @xnm: Xn manager to process Xn entries
363 * @ctlq: send control queue information
364 * @ctlq_msg: control queue message information
365 * @send_buf: buffer that carries outgoing message data, buffers larger than
366 * LIBIE_CP_TX_COPYBREAK bytes will always be consumed
367 * @recv_mem: receive buffer
368 * @send_ctx: context for callback function
369 * @timeout_ms: virtchnl transaction timeout in msecs
370 * @chnl_opcode: virtchnl message opcode
371 */
372 struct libie_ctlq_xn_send_params {
373 void (*resp_cb)(void *ctx, struct kvec *mem, int status);
374 void (*rel_tx_buf)(const void *buf_va);
375 struct libie_ctlq_xn_manager *xnm;
376 struct libie_ctlq_info *ctlq;
377 struct libie_ctlq_msg *ctlq_msg;
378 struct kvec send_buf;
379 struct kvec recv_mem;
380 void *send_ctx;
381 u64 timeout_ms;
382 u32 chnl_opcode;
383 };
384
385 /**
386 * libie_cp_can_send_onstack - can a message be sent using a stack variable
387 * @size: ctlq data buffer size
388 *
389 * Return: %true if the message size is small enough for caller to pass
390 * an on-stack buffer, %false if kmalloc is needed
391 */
libie_cp_can_send_onstack(u32 size)392 static inline bool libie_cp_can_send_onstack(u32 size)
393 {
394 return size <= LIBIE_CP_TX_COPYBREAK;
395 }
396
397 /**
398 * struct libie_ctlq_xn_recv_params - request to receive xn responses
399 * @ctlq_msg_handler: handler for Rx messages with no matching xn (mandatory)
400 * @xnm: Xn manager to process Xn entries
401 * @ctlq: control queue information
402 * @budget: maximum number of messages to process
403 */
404 struct libie_ctlq_xn_recv_params {
405 void (*ctlq_msg_handler)(struct libie_ctlq_ctx *ctx,
406 struct libie_ctlq_msg *msg);
407 struct libie_ctlq_xn_manager *xnm;
408 struct libie_ctlq_info *ctlq;
409 u32 budget;
410 };
411
412 /**
413 * struct libie_ctlq_xn_init_params - xn transaction manager parameters
414 * @cctlq_info: control queue information
415 * @ctx: pointer to controlq context structure
416 * @xnm: Xn manager to process Xn entries
417 * @num_qs: number of control queues to be initialized
418 */
419 struct libie_ctlq_xn_init_params {
420 struct libie_ctlq_create_info *cctlq_info;
421 struct libie_ctlq_ctx *ctx;
422 struct libie_ctlq_xn_manager *xnm;
423 u32 num_qs;
424 };
425
426 int libie_ctlq_xn_init(struct libie_ctlq_xn_init_params *params);
427 void libie_ctlq_xn_deinit(struct libie_ctlq_xn_manager *xnm,
428 struct libie_ctlq_ctx *ctx);
429 void libie_ctlq_xn_shutdown(struct libie_ctlq_xn_manager *xnm);
430 int libie_ctlq_xn_send(struct libie_ctlq_xn_send_params *params);
431 u32 libie_ctlq_xn_recv(struct libie_ctlq_xn_recv_params *params);
432 u32 libie_ctlq_xn_send_clean(struct libie_ctlq_info *ctlq,
433 void (*rel_tx_buf)(const void *buf_va),
434 bool force);
435
436 #endif /* __LIBIE_CONTROLQ_H */
437