xref: /linux/drivers/dma/idxd/registers.h (revision 621cde16e49b3ecf7d59a8106a20aaebfb4a59a9)
1 /* SPDX-License-Identifier: GPL-2.0 */
2 /* Copyright(c) 2019 Intel Corporation. All rights rsvd. */
3 #ifndef _IDXD_REGISTERS_H_
4 #define _IDXD_REGISTERS_H_
5 
6 #include <uapi/linux/idxd.h>
7 
8 /* PCI Config */
9 #define DEVICE_VERSION_1		0x100
10 #define DEVICE_VERSION_2		0x200
11 
12 #define IDXD_MMIO_BAR		0
13 #define IDXD_WQ_BAR		2
14 #define IDXD_PORTAL_SIZE	PAGE_SIZE
15 
16 /* MMIO Device BAR0 Registers */
17 #define IDXD_VER_OFFSET			0x00
18 #define IDXD_VER_MAJOR_MASK		0xf0
19 #define IDXD_VER_MINOR_MASK		0x0f
20 #define GET_IDXD_VER_MAJOR(x)		(((x) & IDXD_VER_MAJOR_MASK) >> 4)
21 #define GET_IDXD_VER_MINOR(x)		((x) & IDXD_VER_MINOR_MASK)
22 
23 union gen_cap_reg {
24 	struct {
25 		u64 block_on_fault:1;
26 		u64 overlap_copy:1;
27 		u64 cache_control_mem:1;
28 		u64 cache_control_cache:1;
29 		u64 cmd_cap:1;
30 		u64 rsvd:3;
31 		u64 dest_readback:1;
32 		u64 drain_readback:1;
33 		u64 rsvd2:3;
34 		u64 evl_support:2;
35 		u64 batch_continuation:1;
36 		u64 max_xfer_shift:5;
37 		u64 max_batch_shift:4;
38 		u64 max_ims_mult:6;
39 		u64 config_en:1;
40 		u64 rsvd3:32;
41 	};
42 	u64 bits;
43 } __packed;
44 #define IDXD_GENCAP_OFFSET		0x10
45 
46 union wq_cap_reg {
47 	struct {
48 		u64 total_wq_size:16;
49 		u64 num_wqs:8;
50 		u64 wqcfg_size:4;
51 		u64 rsvd:20;
52 		u64 shared_mode:1;
53 		u64 dedicated_mode:1;
54 		u64 wq_ats_support:1;
55 		u64 priority:1;
56 		u64 occupancy:1;
57 		u64 occupancy_int:1;
58 		u64 op_config:1;
59 		u64 wq_prs_support:1;
60 		u64 rsvd4:8;
61 	};
62 	u64 bits;
63 } __packed;
64 #define IDXD_WQCAP_OFFSET		0x20
65 #define IDXD_WQCFG_MIN			5
66 
67 union group_cap_reg {
68 	struct {
69 		u64 num_groups:8;
70 		u64 total_rdbufs:8;	/* formerly total_tokens */
71 		u64 rdbuf_ctrl:1;	/* formerly token_en */
72 		u64 rdbuf_limit:1;	/* formerly token_limit */
73 		u64 progress_limit:1;	/* descriptor and batch descriptor */
74 		u64 rsvd:45;
75 	};
76 	u64 bits;
77 } __packed;
78 #define IDXD_GRPCAP_OFFSET		0x30
79 
80 union engine_cap_reg {
81 	struct {
82 		u64 num_engines:8;
83 		u64 rsvd:56;
84 	};
85 	u64 bits;
86 } __packed;
87 
88 #define IDXD_ENGCAP_OFFSET		0x38
89 
90 #define IDXD_OPCAP_NOOP			0x0001
91 #define IDXD_OPCAP_BATCH			0x0002
92 #define IDXD_OPCAP_MEMMOVE		0x0008
93 struct opcap {
94 	u64 bits[4];
95 };
96 
97 #define IDXD_MAX_OPCAP_BITS		256U
98 
99 #define IDXD_OPCAP_OFFSET		0x40
100 
101 #define IDXD_TABLE_OFFSET		0x60
102 union offsets_reg {
103 	struct {
104 		u64 grpcfg:16;
105 		u64 wqcfg:16;
106 		u64 msix_perm:16;
107 		u64 ims:16;
108 		u64 perfmon:16;
109 		u64 rsvd:48;
110 	};
111 	u64 bits[2];
112 } __packed;
113 
114 #define IDXD_TABLE_MULT			0x100
115 
116 #define IDXD_GENCFG_OFFSET		0x80
117 union gencfg_reg {
118 	struct {
119 		u32 rdbuf_limit:8;
120 		u32 rsvd:4;
121 		u32 user_int_en:1;
122 		u32 evl_en:1;
123 		u32 rsvd2:18;
124 	};
125 	u32 bits;
126 } __packed;
127 
128 #define IDXD_GENCTRL_OFFSET		0x88
129 union genctrl_reg {
130 	struct {
131 		u32 softerr_int_en:1;
132 		u32 halt_int_en:1;
133 		u32 evl_int_en:1;
134 		u32 rsvd:29;
135 	};
136 	u32 bits;
137 } __packed;
138 
139 #define IDXD_GENSTATS_OFFSET		0x90
140 union gensts_reg {
141 	struct {
142 		u32 state:2;
143 		u32 reset_type:2;
144 		u32 rsvd:28;
145 	};
146 	u32 bits;
147 } __packed;
148 
149 enum idxd_device_status_state {
150 	IDXD_DEVICE_STATE_DISABLED = 0,
151 	IDXD_DEVICE_STATE_ENABLED,
152 	IDXD_DEVICE_STATE_DRAIN,
153 	IDXD_DEVICE_STATE_HALT,
154 };
155 
156 enum idxd_device_reset_type {
157 	IDXD_DEVICE_RESET_SOFTWARE = 0,
158 	IDXD_DEVICE_RESET_FLR,
159 	IDXD_DEVICE_RESET_WARM,
160 	IDXD_DEVICE_RESET_COLD,
161 };
162 
163 #define IDXD_INTCAUSE_OFFSET		0x98
164 #define IDXD_INTC_ERR			0x01
165 #define IDXD_INTC_CMD			0x02
166 #define IDXD_INTC_OCCUPY			0x04
167 #define IDXD_INTC_PERFMON_OVFL		0x08
168 #define IDXD_INTC_HALT_STATE		0x10
169 #define IDXD_INTC_EVL			0x20
170 #define IDXD_INTC_INT_HANDLE_REVOKED	0x80000000
171 
172 #define IDXD_CMD_OFFSET			0xa0
173 union idxd_command_reg {
174 	struct {
175 		u32 operand:20;
176 		u32 cmd:5;
177 		u32 rsvd:6;
178 		u32 int_req:1;
179 	};
180 	u32 bits;
181 } __packed;
182 
183 enum idxd_cmd {
184 	IDXD_CMD_ENABLE_DEVICE = 1,
185 	IDXD_CMD_DISABLE_DEVICE,
186 	IDXD_CMD_DRAIN_ALL,
187 	IDXD_CMD_ABORT_ALL,
188 	IDXD_CMD_RESET_DEVICE,
189 	IDXD_CMD_ENABLE_WQ,
190 	IDXD_CMD_DISABLE_WQ,
191 	IDXD_CMD_DRAIN_WQ,
192 	IDXD_CMD_ABORT_WQ,
193 	IDXD_CMD_RESET_WQ,
194 	IDXD_CMD_DRAIN_PASID,
195 	IDXD_CMD_ABORT_PASID,
196 	IDXD_CMD_REQUEST_INT_HANDLE,
197 	IDXD_CMD_RELEASE_INT_HANDLE,
198 };
199 
200 #define CMD_INT_HANDLE_IMS		0x10000
201 
202 #define IDXD_CMDSTS_OFFSET		0xa8
203 union cmdsts_reg {
204 	struct {
205 		u8 err;
206 		u16 result;
207 		u8 rsvd:7;
208 		u8 active:1;
209 	};
210 	u32 bits;
211 } __packed;
212 #define IDXD_CMDSTS_ACTIVE		0x80000000
213 #define IDXD_CMDSTS_ERR_MASK		0xff
214 #define IDXD_CMDSTS_RES_SHIFT		8
215 
216 enum idxd_cmdsts_err {
217 	IDXD_CMDSTS_SUCCESS = 0,
218 	IDXD_CMDSTS_INVAL_CMD,
219 	IDXD_CMDSTS_INVAL_WQIDX,
220 	IDXD_CMDSTS_HW_ERR,
221 	/* enable device errors */
222 	IDXD_CMDSTS_ERR_DEV_ENABLED = 0x10,
223 	IDXD_CMDSTS_ERR_CONFIG,
224 	IDXD_CMDSTS_ERR_BUSMASTER_EN,
225 	IDXD_CMDSTS_ERR_PASID_INVAL,
226 	IDXD_CMDSTS_ERR_WQ_SIZE_ERANGE,
227 	IDXD_CMDSTS_ERR_GRP_CONFIG,
228 	IDXD_CMDSTS_ERR_GRP_CONFIG2,
229 	IDXD_CMDSTS_ERR_GRP_CONFIG3,
230 	IDXD_CMDSTS_ERR_GRP_CONFIG4,
231 	/* enable wq errors */
232 	IDXD_CMDSTS_ERR_DEV_NOTEN = 0x20,
233 	IDXD_CMDSTS_ERR_WQ_ENABLED,
234 	IDXD_CMDSTS_ERR_WQ_SIZE,
235 	IDXD_CMDSTS_ERR_WQ_PRIOR,
236 	IDXD_CMDSTS_ERR_WQ_MODE,
237 	IDXD_CMDSTS_ERR_BOF_EN,
238 	IDXD_CMDSTS_ERR_PASID_EN,
239 	IDXD_CMDSTS_ERR_MAX_BATCH_SIZE,
240 	IDXD_CMDSTS_ERR_MAX_XFER_SIZE,
241 	/* disable device errors */
242 	IDXD_CMDSTS_ERR_DIS_DEV_EN = 0x31,
243 	/* disable WQ, drain WQ, abort WQ, reset WQ */
244 	IDXD_CMDSTS_ERR_DEV_NOT_EN,
245 	/* request interrupt handle */
246 	IDXD_CMDSTS_ERR_INVAL_INT_IDX = 0x41,
247 	IDXD_CMDSTS_ERR_NO_HANDLE,
248 };
249 
250 #define IDXD_CMDCAP_OFFSET		0xb0
251 
252 #define IDXD_SWERR_OFFSET		0xc0
253 #define IDXD_SWERR_VALID		0x00000001
254 #define IDXD_SWERR_OVERFLOW		0x00000002
255 #define IDXD_SWERR_ACK			(IDXD_SWERR_VALID | IDXD_SWERR_OVERFLOW)
256 union sw_err_reg {
257 	struct {
258 		u64 valid:1;
259 		u64 overflow:1;
260 		u64 desc_valid:1;
261 		u64 wq_idx_valid:1;
262 		u64 batch:1;
263 		u64 fault_rw:1;
264 		u64 priv:1;
265 		u64 rsvd:1;
266 		u64 error:8;
267 		u64 wq_idx:8;
268 		u64 rsvd2:8;
269 		u64 operation:8;
270 		u64 pasid:20;
271 		u64 rsvd3:4;
272 
273 		u64 batch_idx:16;
274 		u64 rsvd4:16;
275 		u64 invalid_flags:32;
276 
277 		u64 fault_addr;
278 
279 		u64 rsvd5;
280 	};
281 	u64 bits[4];
282 } __packed;
283 
284 union iaa_cap_reg {
285 	struct {
286 		u64 dec_aecs_format_ver:1;
287 		u64 drop_init_bits:1;
288 		u64 chaining:1;
289 		u64 force_array_output_mod:1;
290 		u64 load_part_aecs:1;
291 		u64 comp_early_abort:1;
292 		u64 nested_comp:1;
293 		u64 diction_comp:1;
294 		u64 header_gen:1;
295 		u64 crypto_gcm:1;
296 		u64 crypto_cfb:1;
297 		u64 crypto_xts:1;
298 		u64 rsvd:52;
299 	};
300 	u64 bits;
301 } __packed;
302 
303 #define IDXD_IAACAP_OFFSET	0x180
304 
305 #define IDXD_EVLCFG_OFFSET	0xe0
306 union evlcfg_reg {
307 	struct {
308 		u64 pasid_en:1;
309 		u64 priv:1;
310 		u64 rsvd:10;
311 		u64 base_addr:52;
312 
313 		u64 size:16;
314 		u64 pasid:20;
315 		u64 rsvd2:28;
316 	};
317 	u64 bits[2];
318 } __packed;
319 
320 #define IDXD_EVL_SIZE_MIN	0x0040
321 #define IDXD_EVL_SIZE_MAX	0xffff
322 
323 union msix_perm {
324 	struct {
325 		u32 rsvd:2;
326 		u32 ignore:1;
327 		u32 pasid_en:1;
328 		u32 rsvd2:8;
329 		u32 pasid:20;
330 	};
331 	u32 bits;
332 } __packed;
333 
334 union group_flags {
335 	struct {
336 		u64 tc_a:3;
337 		u64 tc_b:3;
338 		u64 rsvd:1;
339 		u64 use_rdbuf_limit:1;
340 		u64 rdbufs_reserved:8;
341 		u64 rsvd2:4;
342 		u64 rdbufs_allowed:8;
343 		u64 rsvd3:4;
344 		u64 desc_progress_limit:2;
345 		u64 rsvd4:2;
346 		u64 batch_progress_limit:2;
347 		u64 rsvd5:26;
348 	};
349 	u64 bits;
350 } __packed;
351 
352 struct grpcfg {
353 	u64 wqs[4];
354 	u64 engines;
355 	union group_flags flags;
356 } __packed;
357 
358 union wqcfg {
359 	struct {
360 		/* bytes 0-3 */
361 		u16 wq_size;
362 		u16 rsvd;
363 
364 		/* bytes 4-7 */
365 		u16 wq_thresh;
366 		u16 rsvd1;
367 
368 		/* bytes 8-11 */
369 		u32 mode:1;	/* shared or dedicated */
370 		u32 bof:1;	/* block on fault */
371 		u32 wq_ats_disable:1;
372 		u32 wq_prs_disable:1;
373 		u32 priority:4;
374 		u32 pasid:20;
375 		u32 pasid_en:1;
376 		u32 priv:1;
377 		u32 rsvd3:2;
378 
379 		/* bytes 12-15 */
380 		u32 max_xfer_shift:5;
381 		u32 max_batch_shift:4;
382 		u32 rsvd4:23;
383 
384 		/* bytes 16-19 */
385 		u16 occupancy_inth;
386 		u16 occupancy_table_sel:1;
387 		u16 rsvd5:15;
388 
389 		/* bytes 20-23 */
390 		u16 occupancy_limit;
391 		u16 occupancy_int_en:1;
392 		u16 rsvd6:15;
393 
394 		/* bytes 24-27 */
395 		u16 occupancy;
396 		u16 occupancy_int:1;
397 		u16 rsvd7:12;
398 		u16 mode_support:1;
399 		u16 wq_state:2;
400 
401 		/* bytes 28-31 */
402 		u32 rsvd8;
403 
404 		/* bytes 32-63 */
405 		u64 op_config[4];
406 	};
407 	u32 bits[16];
408 } __packed;
409 
410 #define WQCFG_PASID_IDX                2
411 #define WQCFG_PRIVL_IDX		2
412 #define WQCFG_OCCUP_IDX		6
413 
414 #define WQCFG_OCCUP_MASK	0xffff
415 
416 /*
417  * This macro calculates the offset into the WQCFG register
418  * idxd - struct idxd *
419  * n - wq id
420  * ofs - the index of the 32b dword for the config register
421  *
422  * The WQCFG register block is divided into groups per each wq. The n index
423  * allows us to move to the register group that's for that particular wq.
424  * Each register is 32bits. The ofs gives us the number of register to access.
425  */
426 #define WQCFG_OFFSET(_idxd_dev, n, ofs) \
427 ({\
428 	typeof(_idxd_dev) __idxd_dev = (_idxd_dev);	\
429 	(__idxd_dev)->wqcfg_offset + (n) * (__idxd_dev)->wqcfg_size + sizeof(u32) * (ofs);	\
430 })
431 
432 #define WQCFG_STRIDES(_idxd_dev) ((_idxd_dev)->wqcfg_size / sizeof(u32))
433 
434 #define GRPCFG_SIZE		64
435 #define GRPWQCFG_STRIDES	4
436 
437 /*
438  * This macro calculates the offset into the GRPCFG register
439  * idxd - struct idxd *
440  * n - group id
441  * ofs - the index of the 64b qword for the config register
442  *
443  * The GRPCFG register block is divided into three sub-registers, which
444  * are GRPWQCFG, GRPENGCFG and GRPFLGCFG. The n index allows us to move
445  * to the register block that contains the three sub-registers.
446  * Each register block is 64bits. And the ofs gives us the offset
447  * within the GRPWQCFG register to access.
448  */
449 #define GRPWQCFG_OFFSET(idxd_dev, n, ofs) ((idxd_dev)->grpcfg_offset +\
450 					   (n) * GRPCFG_SIZE + sizeof(u64) * (ofs))
451 #define GRPENGCFG_OFFSET(idxd_dev, n) ((idxd_dev)->grpcfg_offset + (n) * GRPCFG_SIZE + 32)
452 #define GRPFLGCFG_OFFSET(idxd_dev, n) ((idxd_dev)->grpcfg_offset + (n) * GRPCFG_SIZE + 40)
453 
454 /* Following is performance monitor registers */
455 #define IDXD_PERFCAP_OFFSET		0x0
456 union idxd_perfcap {
457 	struct {
458 		u64 num_perf_counter:6;
459 		u64 rsvd1:2;
460 		u64 counter_width:8;
461 		u64 num_event_category:4;
462 		u64 global_event_category:16;
463 		u64 filter:8;
464 		u64 rsvd2:8;
465 		u64 cap_per_counter:1;
466 		u64 writeable_counter:1;
467 		u64 counter_freeze:1;
468 		u64 overflow_interrupt:1;
469 		u64 rsvd3:8;
470 	};
471 	u64 bits;
472 } __packed;
473 
474 #define IDXD_EVNTCAP_OFFSET		0x80
475 union idxd_evntcap {
476 	struct {
477 		u64 events:28;
478 		u64 rsvd:36;
479 	};
480 	u64 bits;
481 } __packed;
482 
483 struct idxd_event {
484 	union {
485 		struct {
486 			u32 event_category:4;
487 			u32 events:28;
488 		};
489 		u32 val;
490 	};
491 } __packed;
492 
493 #define IDXD_CNTRCAP_OFFSET		0x800
494 struct idxd_cntrcap {
495 	union {
496 		struct {
497 			u32 counter_width:8;
498 			u32 rsvd:20;
499 			u32 num_events:4;
500 		};
501 		u32 val;
502 	};
503 	struct idxd_event events[];
504 } __packed;
505 
506 #define IDXD_PERFRST_OFFSET		0x10
507 union idxd_perfrst {
508 	struct {
509 		u32 perfrst_config:1;
510 		u32 perfrst_counter:1;
511 		u32 rsvd:30;
512 	};
513 	u32 val;
514 } __packed;
515 
516 #define IDXD_OVFSTATUS_OFFSET		0x30
517 #define IDXD_PERFFRZ_OFFSET		0x20
518 #define IDXD_CNTRCFG_OFFSET		0x100
519 union idxd_cntrcfg {
520 	struct {
521 		u64 enable:1;
522 		u64 interrupt_ovf:1;
523 		u64 global_freeze_ovf:1;
524 		u64 rsvd1:5;
525 		u64 event_category:4;
526 		u64 rsvd2:20;
527 		u64 events:28;
528 		u64 rsvd3:4;
529 	};
530 	u64 val;
531 } __packed;
532 
533 #define IDXD_FLTCFG_OFFSET		0x300
534 
535 #define IDXD_CNTRDATA_OFFSET		0x200
536 union idxd_cntrdata {
537 	struct {
538 		u64 event_count_value;
539 	};
540 	u64 val;
541 } __packed;
542 
543 union event_cfg {
544 	struct {
545 		u64 event_cat:4;
546 		u64 event_enc:28;
547 	};
548 	u64 val;
549 } __packed;
550 
551 union filter_cfg {
552 	struct {
553 		u64 wq:32;
554 		u64 tc:8;
555 		u64 pg_sz:4;
556 		u64 xfer_sz:8;
557 		u64 eng:8;
558 	};
559 	u64 val;
560 } __packed;
561 
562 #define IDXD_EVLSTATUS_OFFSET		0xf0
563 
564 union evl_status_reg {
565 	struct {
566 		u32 head:16;
567 		u32 rsvd:16;
568 		u32 tail:16;
569 		u32 rsvd2:14;
570 		u32 int_pending:1;
571 		u32 rsvd3:1;
572 	};
573 	struct {
574 		u32 bits_lower32;
575 		u32 bits_upper32;
576 	};
577 	u64 bits;
578 } __packed;
579 
580 #define IDXD_MAX_BATCH_IDENT	256
581 
582 struct __evl_entry {
583 	u64 rsvd:2;
584 	u64 desc_valid:1;
585 	u64 wq_idx_valid:1;
586 	u64 batch:1;
587 	u64 fault_rw:1;
588 	u64 priv:1;
589 	u64 err_info_valid:1;
590 	u64 error:8;
591 	u64 wq_idx:8;
592 	u64 batch_id:8;
593 	u64 operation:8;
594 	u64 pasid:20;
595 	u64 rsvd2:4;
596 
597 	u16 batch_idx;
598 	u16 rsvd3;
599 	union {
600 		/* Invalid Flags 0x11 */
601 		u32 invalid_flags;
602 		/* Invalid Int Handle 0x19 */
603 		/* Page fault 0x1a */
604 		/* Page fault 0x06, 0x1f, only operand_id */
605 		/* Page fault before drain or in batch, 0x26, 0x27 */
606 		struct {
607 			u16 int_handle;
608 			u16 rci:1;
609 			u16 ims:1;
610 			u16 rcr:1;
611 			u16 first_err_in_batch:1;
612 			u16 rsvd4_2:9;
613 			u16 operand_id:3;
614 		};
615 	};
616 	u64 fault_addr;
617 	u64 rsvd5;
618 } __packed;
619 
620 struct dsa_evl_entry {
621 	struct __evl_entry e;
622 	struct dsa_completion_record cr;
623 } __packed;
624 
625 struct iax_evl_entry {
626 	struct __evl_entry e;
627 	u64 rsvd[4];
628 	struct iax_completion_record cr;
629 } __packed;
630 
631 #endif
632