xref: /linux/drivers/accel/habanalabs/common/habanalabs.h (revision a1ff5a7d78a036d6c2178ee5acd6ba4946243800)
1  /* SPDX-License-Identifier: GPL-2.0
2   *
3   * Copyright 2016-2023 HabanaLabs, Ltd.
4   * All Rights Reserved.
5   *
6   */
7  
8  #ifndef HABANALABSP_H_
9  #define HABANALABSP_H_
10  
11  #include <linux/habanalabs/cpucp_if.h>
12  #include "../include/common/qman_if.h"
13  #include "../include/hw_ip/mmu/mmu_general.h"
14  #include <uapi/drm/habanalabs_accel.h>
15  
16  #include <linux/cdev.h>
17  #include <linux/iopoll.h>
18  #include <linux/irqreturn.h>
19  #include <linux/dma-direction.h>
20  #include <linux/scatterlist.h>
21  #include <linux/hashtable.h>
22  #include <linux/debugfs.h>
23  #include <linux/rwsem.h>
24  #include <linux/eventfd.h>
25  #include <linux/bitfield.h>
26  #include <linux/genalloc.h>
27  #include <linux/sched/signal.h>
28  #include <linux/io-64-nonatomic-lo-hi.h>
29  #include <linux/coresight.h>
30  #include <linux/dma-buf.h>
31  
32  #include <drm/drm_device.h>
33  #include <drm/drm_file.h>
34  
35  #include "security.h"
36  
37  #define HL_NAME				"habanalabs"
38  
39  struct hl_device;
40  struct hl_fpriv;
41  
42  #define PCI_VENDOR_ID_HABANALABS	0x1da3
43  
44  /* Use upper bits of mmap offset to store habana driver specific information.
45   * bits[63:59] - Encode mmap type
46   * bits[45:0]  - mmap offset value
47   *
48   * NOTE: struct vm_area_struct.vm_pgoff uses offset in pages. Hence, these
49   *  defines are w.r.t to PAGE_SIZE
50   */
51  #define HL_MMAP_TYPE_SHIFT		(59 - PAGE_SHIFT)
52  #define HL_MMAP_TYPE_MASK		(0x1full << HL_MMAP_TYPE_SHIFT)
53  #define HL_MMAP_TYPE_TS_BUFF		(0x10ull << HL_MMAP_TYPE_SHIFT)
54  #define HL_MMAP_TYPE_BLOCK		(0x4ull << HL_MMAP_TYPE_SHIFT)
55  #define HL_MMAP_TYPE_CB			(0x2ull << HL_MMAP_TYPE_SHIFT)
56  
57  #define HL_MMAP_OFFSET_VALUE_MASK	(0x1FFFFFFFFFFFull >> PAGE_SHIFT)
58  #define HL_MMAP_OFFSET_VALUE_GET(off)	(off & HL_MMAP_OFFSET_VALUE_MASK)
59  
60  #define HL_PENDING_RESET_PER_SEC		10
61  #define HL_PENDING_RESET_MAX_TRIALS		60 /* 10 minutes */
62  #define HL_PENDING_RESET_LONG_SEC		60
63  /*
64   * In device fini, wait 10 minutes for user processes to be terminated after we kill them.
65   * This is needed to prevent situation of clearing resources while user processes are still alive.
66   */
67  #define HL_WAIT_PROCESS_KILL_ON_DEVICE_FINI	600
68  
69  #define HL_HARD_RESET_MAX_TIMEOUT	120
70  #define HL_PLDM_HARD_RESET_MAX_TIMEOUT	(HL_HARD_RESET_MAX_TIMEOUT * 3)
71  
72  #define HL_DEVICE_TIMEOUT_USEC		1000000 /* 1 s */
73  
74  #define HL_HEARTBEAT_PER_USEC		10000000 /* 10 s */
75  
76  #define HL_PLL_LOW_JOB_FREQ_USEC	5000000 /* 5 s */
77  
78  #define HL_CPUCP_INFO_TIMEOUT_USEC	10000000 /* 10s */
79  #define HL_CPUCP_EEPROM_TIMEOUT_USEC	10000000 /* 10s */
80  #define HL_CPUCP_MON_DUMP_TIMEOUT_USEC	10000000 /* 10s */
81  #define HL_CPUCP_SEC_ATTEST_INFO_TINEOUT_USEC 10000000 /* 10s */
82  
83  #define HL_FW_STATUS_POLL_INTERVAL_USEC		10000 /* 10ms */
84  #define HL_FW_COMMS_STATUS_PLDM_POLL_INTERVAL_USEC	1000000 /* 1s */
85  
86  #define HL_PCI_ELBI_TIMEOUT_MSEC	10 /* 10ms */
87  
88  #define HL_INVALID_QUEUE		UINT_MAX
89  
90  #define HL_COMMON_USER_CQ_INTERRUPT_ID	0xFFF
91  #define HL_COMMON_DEC_INTERRUPT_ID	0xFFE
92  
93  #define HL_STATE_DUMP_HIST_LEN		5
94  
95  /* Default value for device reset trigger , an invalid value */
96  #define HL_RESET_TRIGGER_DEFAULT	0xFF
97  
98  #define OBJ_NAMES_HASH_TABLE_BITS	7 /* 1 << 7 buckets */
99  #define SYNC_TO_ENGINE_HASH_TABLE_BITS	7 /* 1 << 7 buckets */
100  
101  /* Memory */
102  #define MEM_HASH_TABLE_BITS		7 /* 1 << 7 buckets */
103  
104  /* MMU */
105  #define MMU_HASH_TABLE_BITS		7 /* 1 << 7 buckets */
106  
107  #define TIMESTAMP_FREE_NODES_NUM	512
108  
109  /**
110   * enum hl_mmu_page_table_location - mmu page table location
111   * @MMU_DR_PGT: page-table is located on device DRAM.
112   * @MMU_HR_PGT: page-table is located on host memory.
113   * @MMU_NUM_PGT_LOCATIONS: number of page-table locations currently supported.
114   */
115  enum hl_mmu_page_table_location {
116  	MMU_DR_PGT = 0,		/* device-dram-resident MMU PGT */
117  	MMU_HR_PGT,		/* host resident MMU PGT */
118  	MMU_NUM_PGT_LOCATIONS	/* num of PGT locations */
119  };
120  
121  /*
122   * HL_RSVD_SOBS 'sync stream' reserved sync objects per QMAN stream
123   * HL_RSVD_MONS 'sync stream' reserved monitors per QMAN stream
124   */
125  #define HL_RSVD_SOBS			2
126  #define HL_RSVD_MONS			1
127  
128  /*
129   * HL_COLLECTIVE_RSVD_MSTR_MONS 'collective' reserved monitors per QMAN stream
130   */
131  #define HL_COLLECTIVE_RSVD_MSTR_MONS	2
132  
133  #define HL_MAX_SOB_VAL			(1 << 15)
134  
135  #define IS_POWER_OF_2(n)		(n != 0 && ((n & (n - 1)) == 0))
136  #define IS_MAX_PENDING_CS_VALID(n)	(IS_POWER_OF_2(n) && (n > 1))
137  
138  #define HL_PCI_NUM_BARS			6
139  
140  /* Completion queue entry relates to completed job */
141  #define HL_COMPLETION_MODE_JOB		0
142  /* Completion queue entry relates to completed command submission */
143  #define HL_COMPLETION_MODE_CS		1
144  
145  #define HL_MAX_DCORES			8
146  
147  /* DMA alloc/free wrappers */
148  #define hl_asic_dma_alloc_coherent(hdev, size, dma_handle, flags) \
149  	hl_asic_dma_alloc_coherent_caller(hdev, size, dma_handle, flags, __func__)
150  
151  #define hl_asic_dma_pool_zalloc(hdev, size, mem_flags, dma_handle) \
152  	hl_asic_dma_pool_zalloc_caller(hdev, size, mem_flags, dma_handle, __func__)
153  
154  #define hl_asic_dma_free_coherent(hdev, size, cpu_addr, dma_handle) \
155  	hl_asic_dma_free_coherent_caller(hdev, size, cpu_addr, dma_handle, __func__)
156  
157  #define hl_asic_dma_pool_free(hdev, vaddr, dma_addr) \
158  	hl_asic_dma_pool_free_caller(hdev, vaddr, dma_addr, __func__)
159  
160  #define hl_dma_map_sgtable(hdev, sgt, dir) \
161  	hl_dma_map_sgtable_caller(hdev, sgt, dir, __func__)
162  #define hl_dma_unmap_sgtable(hdev, sgt, dir) \
163  	hl_dma_unmap_sgtable_caller(hdev, sgt, dir, __func__)
164  
165  /*
166   * Reset Flags
167   *
168   * - HL_DRV_RESET_HARD
169   *       If set do hard reset to all engines. If not set reset just
170   *       compute/DMA engines.
171   *
172   * - HL_DRV_RESET_FROM_RESET_THR
173   *       Set if the caller is the hard-reset thread
174   *
175   * - HL_DRV_RESET_HEARTBEAT
176   *       Set if reset is due to heartbeat
177   *
178   * - HL_DRV_RESET_TDR
179   *       Set if reset is due to TDR
180   *
181   * - HL_DRV_RESET_DEV_RELEASE
182   *       Set if reset is due to device release
183   *
184   * - HL_DRV_RESET_BYPASS_REQ_TO_FW
185   *       F/W will perform the reset. No need to ask it to reset the device. This is relevant
186   *       only when running with secured f/w
187   *
188   * - HL_DRV_RESET_FW_FATAL_ERR
189   *       Set if reset is due to a fatal error from FW
190   *
191   * - HL_DRV_RESET_DELAY
192   *       Set if a delay should be added before the reset
193   *
194   * - HL_DRV_RESET_FROM_WD_THR
195   *       Set if the caller is the device release watchdog thread
196   */
197  
198  #define HL_DRV_RESET_HARD		(1 << 0)
199  #define HL_DRV_RESET_FROM_RESET_THR	(1 << 1)
200  #define HL_DRV_RESET_HEARTBEAT		(1 << 2)
201  #define HL_DRV_RESET_TDR		(1 << 3)
202  #define HL_DRV_RESET_DEV_RELEASE	(1 << 4)
203  #define HL_DRV_RESET_BYPASS_REQ_TO_FW	(1 << 5)
204  #define HL_DRV_RESET_FW_FATAL_ERR	(1 << 6)
205  #define HL_DRV_RESET_DELAY		(1 << 7)
206  #define HL_DRV_RESET_FROM_WD_THR	(1 << 8)
207  
208  /*
209   * Security
210   */
211  
212  #define HL_PB_SHARED		1
213  #define HL_PB_NA		0
214  #define HL_PB_SINGLE_INSTANCE	1
215  #define HL_BLOCK_SIZE		0x1000
216  #define HL_BLOCK_GLBL_ERR_MASK	0xF40
217  #define HL_BLOCK_GLBL_ERR_ADDR	0xF44
218  #define HL_BLOCK_GLBL_ERR_CAUSE	0xF48
219  #define HL_BLOCK_GLBL_SEC_OFFS	0xF80
220  #define HL_BLOCK_GLBL_SEC_SIZE	(HL_BLOCK_SIZE - HL_BLOCK_GLBL_SEC_OFFS)
221  #define HL_BLOCK_GLBL_SEC_LEN	(HL_BLOCK_GLBL_SEC_SIZE / sizeof(u32))
222  #define UNSET_GLBL_SEC_BIT(array, b) ((array)[((b) / 32)] |= (1 << ((b) % 32)))
223  
224  enum hl_protection_levels {
225  	SECURED_LVL,
226  	PRIVILEGED_LVL,
227  	NON_SECURED_LVL
228  };
229  
230  /**
231   * struct iterate_module_ctx - HW module iterator
232   * @fn: function to apply to each HW module instance
233   * @data: optional internal data to the function iterator
234   * @rc: return code for optional use of iterator/iterator-caller
235   */
236  struct iterate_module_ctx {
237  	/*
238  	 * callback for the HW module iterator
239  	 * @hdev: pointer to the habanalabs device structure
240  	 * @block: block (ASIC specific definition can be dcore/hdcore)
241  	 * @inst: HW module instance within the block
242  	 * @offset: current HW module instance offset from the 1-st HW module instance
243  	 *          in the 1-st block
244  	 * @ctx: the iterator context.
245  	 */
246  	void (*fn)(struct hl_device *hdev, int block, int inst, u32 offset,
247  			struct iterate_module_ctx *ctx);
248  	void *data;
249  	int rc;
250  };
251  
252  struct hl_block_glbl_sec {
253  	u32 sec_array[HL_BLOCK_GLBL_SEC_LEN];
254  };
255  
256  #define HL_MAX_SOBS_PER_MONITOR	8
257  
258  /**
259   * struct hl_gen_wait_properties - properties for generating a wait CB
260   * @data: command buffer
261   * @q_idx: queue id is used to extract fence register address
262   * @size: offset in command buffer
263   * @sob_base: SOB base to use in this wait CB
264   * @sob_val: SOB value to wait for
265   * @mon_id: monitor to use in this wait CB
266   * @sob_mask: each bit represents a SOB offset from sob_base to be used
267   */
268  struct hl_gen_wait_properties {
269  	void	*data;
270  	u32	q_idx;
271  	u32	size;
272  	u16	sob_base;
273  	u16	sob_val;
274  	u16	mon_id;
275  	u8	sob_mask;
276  };
277  
278  /**
279   * struct pgt_info - MMU hop page info.
280   * @node: hash linked-list node for the pgts on host (shadow pgts for device resident MMU and
281   *        actual pgts for host resident MMU).
282   * @phys_addr: physical address of the pgt.
283   * @virt_addr: host virtual address of the pgt (see above device/host resident).
284   * @shadow_addr: shadow hop in the host for device resident MMU.
285   * @ctx: pointer to the owner ctx.
286   * @num_of_ptes: indicates how many ptes are used in the pgt. used only for dynamically
287   *               allocated HOPs (all HOPs but HOP0)
288   *
289   * The MMU page tables hierarchy can be placed either on the device's DRAM (in which case shadow
290   * pgts will be stored on host memory) or on host memory (in which case no shadow is required).
291   *
292   * When a new level (hop) is needed during mapping this structure will be used to describe
293   * the newly allocated hop as well as to track number of PTEs in it.
294   * During unmapping, if no valid PTEs remained in the page of a newly allocated hop, it is
295   * freed with its pgt_info structure.
296   */
297  struct pgt_info {
298  	struct hlist_node	node;
299  	u64			phys_addr;
300  	u64			virt_addr;
301  	u64			shadow_addr;
302  	struct hl_ctx		*ctx;
303  	int			num_of_ptes;
304  };
305  
306  /**
307   * enum hl_pci_match_mode - pci match mode per region
308   * @PCI_ADDRESS_MATCH_MODE: address match mode
309   * @PCI_BAR_MATCH_MODE: bar match mode
310   */
311  enum hl_pci_match_mode {
312  	PCI_ADDRESS_MATCH_MODE,
313  	PCI_BAR_MATCH_MODE
314  };
315  
316  /**
317   * enum hl_fw_component - F/W components to read version through registers.
318   * @FW_COMP_BOOT_FIT: boot fit.
319   * @FW_COMP_PREBOOT: preboot.
320   * @FW_COMP_LINUX: linux.
321   */
322  enum hl_fw_component {
323  	FW_COMP_BOOT_FIT,
324  	FW_COMP_PREBOOT,
325  	FW_COMP_LINUX,
326  };
327  
328  /**
329   * enum hl_fw_types - F/W types present in the system
330   * @FW_TYPE_NONE: no FW component indication
331   * @FW_TYPE_LINUX: Linux image for device CPU
332   * @FW_TYPE_BOOT_CPU: Boot image for device CPU
333   * @FW_TYPE_PREBOOT_CPU: Indicates pre-loaded CPUs are present in the system
334   *                       (preboot, ppboot etc...)
335   * @FW_TYPE_ALL_TYPES: Mask for all types
336   */
337  enum hl_fw_types {
338  	FW_TYPE_NONE = 0x0,
339  	FW_TYPE_LINUX = 0x1,
340  	FW_TYPE_BOOT_CPU = 0x2,
341  	FW_TYPE_PREBOOT_CPU = 0x4,
342  	FW_TYPE_ALL_TYPES =
343  		(FW_TYPE_LINUX | FW_TYPE_BOOT_CPU | FW_TYPE_PREBOOT_CPU)
344  };
345  
346  /**
347   * enum hl_queue_type - Supported QUEUE types.
348   * @QUEUE_TYPE_NA: queue is not available.
349   * @QUEUE_TYPE_EXT: external queue which is a DMA channel that may access the
350   *                  host.
351   * @QUEUE_TYPE_INT: internal queue that performs DMA inside the device's
352   *			memories and/or operates the compute engines.
353   * @QUEUE_TYPE_CPU: S/W queue for communication with the device's CPU.
354   * @QUEUE_TYPE_HW: queue of DMA and compute engines jobs, for which completion
355   *                 notifications are sent by H/W.
356   */
357  enum hl_queue_type {
358  	QUEUE_TYPE_NA,
359  	QUEUE_TYPE_EXT,
360  	QUEUE_TYPE_INT,
361  	QUEUE_TYPE_CPU,
362  	QUEUE_TYPE_HW
363  };
364  
365  enum hl_cs_type {
366  	CS_TYPE_DEFAULT,
367  	CS_TYPE_SIGNAL,
368  	CS_TYPE_WAIT,
369  	CS_TYPE_COLLECTIVE_WAIT,
370  	CS_RESERVE_SIGNALS,
371  	CS_UNRESERVE_SIGNALS,
372  	CS_TYPE_ENGINE_CORE,
373  	CS_TYPE_ENGINES,
374  	CS_TYPE_FLUSH_PCI_HBW_WRITES,
375  };
376  
377  /*
378   * struct hl_inbound_pci_region - inbound region descriptor
379   * @mode: pci match mode for this region
380   * @addr: region target address
381   * @size: region size in bytes
382   * @offset_in_bar: offset within bar (address match mode)
383   * @bar: bar id
384   */
385  struct hl_inbound_pci_region {
386  	enum hl_pci_match_mode	mode;
387  	u64			addr;
388  	u64			size;
389  	u64			offset_in_bar;
390  	u8			bar;
391  };
392  
393  /*
394   * struct hl_outbound_pci_region - outbound region descriptor
395   * @addr: region target address
396   * @size: region size in bytes
397   */
398  struct hl_outbound_pci_region {
399  	u64	addr;
400  	u64	size;
401  };
402  
403  /*
404   * enum queue_cb_alloc_flags - Indicates queue support for CBs that
405   * allocated by Kernel or by User
406   * @CB_ALLOC_KERNEL: support only CBs that allocated by Kernel
407   * @CB_ALLOC_USER: support only CBs that allocated by User
408   */
409  enum queue_cb_alloc_flags {
410  	CB_ALLOC_KERNEL = 0x1,
411  	CB_ALLOC_USER   = 0x2
412  };
413  
414  /*
415   * struct hl_hw_sob - H/W SOB info.
416   * @hdev: habanalabs device structure.
417   * @kref: refcount of this SOB. The SOB will reset once the refcount is zero.
418   * @sob_id: id of this SOB.
419   * @sob_addr: the sob offset from the base address.
420   * @q_idx: the H/W queue that uses this SOB.
421   * @need_reset: reset indication set when switching to the other sob.
422   */
423  struct hl_hw_sob {
424  	struct hl_device	*hdev;
425  	struct kref		kref;
426  	u32			sob_id;
427  	u32			sob_addr;
428  	u32			q_idx;
429  	bool			need_reset;
430  };
431  
432  enum hl_collective_mode {
433  	HL_COLLECTIVE_NOT_SUPPORTED = 0x0,
434  	HL_COLLECTIVE_MASTER = 0x1,
435  	HL_COLLECTIVE_SLAVE = 0x2
436  };
437  
438  /**
439   * struct hw_queue_properties - queue information.
440   * @type: queue type.
441   * @cb_alloc_flags: bitmap which indicates if the hw queue supports CB
442   *                  that allocated by the Kernel driver and therefore,
443   *                  a CB handle can be provided for jobs on this queue.
444   *                  Otherwise, a CB address must be provided.
445   * @collective_mode: collective mode of current queue
446   * @q_dram_bd_address: PQ dram address, used when PQ need to reside in DRAM.
447   * @driver_only: true if only the driver is allowed to send a job to this queue,
448   *               false otherwise.
449   * @binned: True if the queue is binned out and should not be used
450   * @supports_sync_stream: True if queue supports sync stream
451   * @dram_bd: True if the bd should be copied to dram, needed for PQ which has been allocated on dram
452   */
453  struct hw_queue_properties {
454  	enum hl_queue_type		type;
455  	enum queue_cb_alloc_flags	cb_alloc_flags;
456  	enum hl_collective_mode		collective_mode;
457  	u64				q_dram_bd_address;
458  	u8				driver_only;
459  	u8				binned;
460  	u8				supports_sync_stream;
461  	u8				dram_bd;
462  };
463  
464  /**
465   * enum vm_type - virtual memory mapping request information.
466   * @VM_TYPE_USERPTR: mapping of user memory to device virtual address.
467   * @VM_TYPE_PHYS_PACK: mapping of DRAM memory to device virtual address.
468   */
469  enum vm_type {
470  	VM_TYPE_USERPTR = 0x1,
471  	VM_TYPE_PHYS_PACK = 0x2
472  };
473  
474  /**
475   * enum mmu_op_flags - mmu operation relevant information.
476   * @MMU_OP_USERPTR: operation on user memory (host resident).
477   * @MMU_OP_PHYS_PACK: operation on DRAM (device resident).
478   * @MMU_OP_CLEAR_MEMCACHE: operation has to clear memcache.
479   * @MMU_OP_SKIP_LOW_CACHE_INV: operation is allowed to skip parts of cache invalidation.
480   */
481  enum mmu_op_flags {
482  	MMU_OP_USERPTR = 0x1,
483  	MMU_OP_PHYS_PACK = 0x2,
484  	MMU_OP_CLEAR_MEMCACHE = 0x4,
485  	MMU_OP_SKIP_LOW_CACHE_INV = 0x8,
486  };
487  
488  
489  /**
490   * enum hl_device_hw_state - H/W device state. use this to understand whether
491   *                           to do reset before hw_init or not
492   * @HL_DEVICE_HW_STATE_CLEAN: H/W state is clean. i.e. after hard reset
493   * @HL_DEVICE_HW_STATE_DIRTY: H/W state is dirty. i.e. we started to execute
494   *                            hw_init
495   */
496  enum hl_device_hw_state {
497  	HL_DEVICE_HW_STATE_CLEAN = 0,
498  	HL_DEVICE_HW_STATE_DIRTY
499  };
500  
501  #define HL_MMU_VA_ALIGNMENT_NOT_NEEDED 0
502  
503  /**
504   * struct hl_mmu_properties - ASIC specific MMU address translation properties.
505   * @start_addr: virtual start address of the memory region.
506   * @end_addr: virtual end address of the memory region.
507   * @hop_shifts: array holds HOPs shifts.
508   * @hop_masks: array holds HOPs masks.
509   * @last_mask: mask to get the bit indicating this is the last hop.
510   * @pgt_size: size for page tables.
511   * @supported_pages_mask: bitmask for supported page size (relevant only for MMUs
512   *                        supporting multiple page size).
513   * @page_size: default page size used to allocate memory.
514   * @num_hops: The amount of hops supported by the translation table.
515   * @hop_table_size: HOP table size.
516   * @hop0_tables_total_size: total size for all HOP0 tables.
517   * @host_resident: Should the MMU page table reside in host memory or in the
518   *                 device DRAM.
519   */
520  struct hl_mmu_properties {
521  	u64	start_addr;
522  	u64	end_addr;
523  	u64	hop_shifts[MMU_HOP_MAX];
524  	u64	hop_masks[MMU_HOP_MAX];
525  	u64	last_mask;
526  	u64	pgt_size;
527  	u64	supported_pages_mask;
528  	u32	page_size;
529  	u32	num_hops;
530  	u32	hop_table_size;
531  	u32	hop0_tables_total_size;
532  	u8	host_resident;
533  };
534  
535  /**
536   * struct hl_hints_range - hint addresses reserved va range.
537   * @start_addr: start address of the va range.
538   * @end_addr: end address of the va range.
539   */
540  struct hl_hints_range {
541  	u64 start_addr;
542  	u64 end_addr;
543  };
544  
545  /**
546   * struct asic_fixed_properties - ASIC specific immutable properties.
547   * @hw_queues_props: H/W queues properties.
548   * @special_blocks: points to an array containing special blocks info.
549   * @skip_special_blocks_cfg: special blocks skip configs.
550   * @cpucp_info: received various information from CPU-CP regarding the H/W, e.g.
551   *		available sensors.
552   * @uboot_ver: F/W U-boot version.
553   * @preboot_ver: F/W Preboot version.
554   * @dmmu: DRAM MMU address translation properties.
555   * @pmmu: PCI (host) MMU address translation properties.
556   * @pmmu_huge: PCI (host) MMU address translation properties for memory
557   *              allocated with huge pages.
558   * @hints_dram_reserved_va_range: dram hint addresses reserved range.
559   * @hints_host_reserved_va_range: host hint addresses reserved range.
560   * @hints_host_hpage_reserved_va_range: host huge page hint addresses reserved range.
561   * @sram_base_address: SRAM physical start address.
562   * @sram_end_address: SRAM physical end address.
563   * @sram_user_base_address - SRAM physical start address for user access.
564   * @dram_base_address: DRAM physical start address.
565   * @dram_end_address: DRAM physical end address.
566   * @dram_user_base_address: DRAM physical start address for user access.
567   * @dram_size: DRAM total size.
568   * @dram_pci_bar_size: size of PCI bar towards DRAM.
569   * @max_power_default: max power of the device after reset.
570   * @dc_power_default: power consumed by the device in mode idle.
571   * @dram_size_for_default_page_mapping: DRAM size needed to map to avoid page
572   *                                      fault.
573   * @pcie_dbi_base_address: Base address of the PCIE_DBI block.
574   * @pcie_aux_dbi_reg_addr: Address of the PCIE_AUX DBI register.
575   * @mmu_pgt_addr: base physical address in DRAM of MMU page tables.
576   * @mmu_dram_default_page_addr: DRAM default page physical address.
577   * @tpc_enabled_mask: which TPCs are enabled.
578   * @tpc_binning_mask: which TPCs are binned. 0 means usable and 1 means binned.
579   * @dram_enabled_mask: which DRAMs are enabled.
580   * @dram_binning_mask: which DRAMs are binned. 0 means usable, 1 means binned.
581   * @dram_hints_align_mask: dram va hint addresses alignment mask which is used
582   *                  for hints validity check.
583   * @cfg_base_address: config space base address.
584   * @mmu_cache_mng_addr: address of the MMU cache.
585   * @mmu_cache_mng_size: size of the MMU cache.
586   * @device_dma_offset_for_host_access: the offset to add to host DMA addresses
587   *                                     to enable the device to access them.
588   * @host_base_address: host physical start address for host DMA from device
589   * @host_end_address: host physical end address for host DMA from device
590   * @max_freq_value: current max clk frequency.
591   * @engine_core_interrupt_reg_addr: interrupt register address for engine core to use
592   *                                  in order to raise events toward FW.
593   * @clk_pll_index: clock PLL index that specify which PLL determines the clock
594   *                 we display to the user
595   * @mmu_pgt_size: MMU page tables total size.
596   * @mmu_pte_size: PTE size in MMU page tables.
597   * @dram_page_size: The DRAM physical page size.
598   * @cfg_size: configuration space size on SRAM.
599   * @sram_size: total size of SRAM.
600   * @max_asid: maximum number of open contexts (ASIDs).
601   * @num_of_events: number of possible internal H/W IRQs.
602   * @psoc_pci_pll_nr: PCI PLL NR value.
603   * @psoc_pci_pll_nf: PCI PLL NF value.
604   * @psoc_pci_pll_od: PCI PLL OD value.
605   * @psoc_pci_pll_div_factor: PCI PLL DIV FACTOR 1 value.
606   * @psoc_timestamp_frequency: frequency of the psoc timestamp clock.
607   * @high_pll: high PLL frequency used by the device.
608   * @cb_pool_cb_cnt: number of CBs in the CB pool.
609   * @cb_pool_cb_size: size of each CB in the CB pool.
610   * @decoder_enabled_mask: which decoders are enabled.
611   * @decoder_binning_mask: which decoders are binned, 0 means usable and 1 means binned.
612   * @rotator_enabled_mask: which rotators are enabled.
613   * @edma_enabled_mask: which EDMAs are enabled.
614   * @edma_binning_mask: which EDMAs are binned, 0 means usable and 1 means
615   *                     binned (at most one binned DMA).
616   * @max_pending_cs: maximum of concurrent pending command submissions
617   * @max_queues: maximum amount of queues in the system
618   * @fw_preboot_cpu_boot_dev_sts0: bitmap representation of preboot cpu
619   *                                capabilities reported by FW, bit description
620   *                                can be found in CPU_BOOT_DEV_STS0
621   * @fw_preboot_cpu_boot_dev_sts1: bitmap representation of preboot cpu
622   *                                capabilities reported by FW, bit description
623   *                                can be found in CPU_BOOT_DEV_STS1
624   * @fw_bootfit_cpu_boot_dev_sts0: bitmap representation of boot cpu security
625   *                                status reported by FW, bit description can be
626   *                                found in CPU_BOOT_DEV_STS0
627   * @fw_bootfit_cpu_boot_dev_sts1: bitmap representation of boot cpu security
628   *                                status reported by FW, bit description can be
629   *                                found in CPU_BOOT_DEV_STS1
630   * @fw_app_cpu_boot_dev_sts0: bitmap representation of application security
631   *                            status reported by FW, bit description can be
632   *                            found in CPU_BOOT_DEV_STS0
633   * @fw_app_cpu_boot_dev_sts1: bitmap representation of application security
634   *                            status reported by FW, bit description can be
635   *                            found in CPU_BOOT_DEV_STS1
636   * @max_dec: maximum number of decoders
637   * @hmmu_hif_enabled_mask: mask of HMMUs/HIFs that are not isolated (enabled)
638   *                         1- enabled, 0- isolated.
639   * @faulty_dram_cluster_map: mask of faulty DRAM cluster.
640   *                         1- faulty cluster, 0- good cluster.
641   * @xbar_edge_enabled_mask: mask of XBAR_EDGEs that are not isolated (enabled)
642   *                          1- enabled, 0- isolated.
643   * @device_mem_alloc_default_page_size: may be different than dram_page_size only for ASICs for
644   *                                      which the property supports_user_set_page_size is true
645   *                                      (i.e. the DRAM supports multiple page sizes), otherwise
646   *                                      it will shall  be equal to dram_page_size.
647   * @num_engine_cores: number of engine cpu cores.
648   * @max_num_of_engines: maximum number of all engines in the ASIC.
649   * @num_of_special_blocks: special_blocks array size.
650   * @glbl_err_max_cause_num: global err max cause number.
651   * @hbw_flush_reg: register to read to generate HBW flush. value of 0 means HBW flush is
652   *                 not supported.
653   * @reserved_fw_mem_size: size of dram memory reserved for FW.
654   * @fw_event_queue_size: queue size for events from CPU-CP.
655   *                       A value of 0 means using the default HL_EQ_SIZE_IN_BYTES value.
656   * @collective_first_sob: first sync object available for collective use
657   * @collective_first_mon: first monitor available for collective use
658   * @sync_stream_first_sob: first sync object available for sync stream use
659   * @sync_stream_first_mon: first monitor available for sync stream use
660   * @first_available_user_sob: first sob available for the user
661   * @first_available_user_mon: first monitor available for the user
662   * @first_available_user_interrupt: first available interrupt reserved for the user
663   * @first_available_cq: first available CQ for the user.
664   * @user_interrupt_count: number of user interrupts.
665   * @user_dec_intr_count: number of decoder interrupts exposed to user.
666   * @tpc_interrupt_id: interrupt id for TPC to use in order to raise events towards the host.
667   * @eq_interrupt_id: interrupt id for EQ, uses to synchronize EQ interrupts in hard-reset.
668   * @cache_line_size: device cache line size.
669   * @server_type: Server type that the ASIC is currently installed in.
670   *               The value is according to enum hl_server_type in uapi file.
671   * @completion_queues_count: number of completion queues.
672   * @completion_mode: 0 - job based completion, 1 - cs based completion
673   * @mme_master_slave_mode: 0 - Each MME works independently, 1 - MME works
674   *                         in Master/Slave mode
675   * @fw_security_enabled: true if security measures are enabled in firmware,
676   *                       false otherwise
677   * @fw_cpu_boot_dev_sts0_valid: status bits are valid and can be fetched from
678   *                              BOOT_DEV_STS0
679   * @fw_cpu_boot_dev_sts1_valid: status bits are valid and can be fetched from
680   *                              BOOT_DEV_STS1
681   * @dram_supports_virtual_memory: is there an MMU towards the DRAM
682   * @hard_reset_done_by_fw: true if firmware is handling hard reset flow
683   * @num_functional_hbms: number of functional HBMs in each DCORE.
684   * @hints_range_reservation: device support hint addresses range reservation.
685   * @iatu_done_by_fw: true if iATU configuration is being done by FW.
686   * @dynamic_fw_load: is dynamic FW load is supported.
687   * @gic_interrupts_enable: true if FW is not blocking GIC controller,
688   *                         false otherwise.
689   * @use_get_power_for_reset_history: To support backward compatibility for Goya
690   *                                   and Gaudi
691   * @supports_compute_reset: is a reset which is not a hard-reset supported by this asic.
692   * @allow_inference_soft_reset: true if the ASIC supports soft reset that is
693   *                              initiated by user or TDR. This is only true
694   *                              in inference ASICs, as there is no real-world
695   *                              use-case of doing soft-reset in training (due
696   *                              to the fact that training runs on multiple
697   *                              devices)
698   * @configurable_stop_on_err: is stop-on-error option configurable via debugfs.
699   * @set_max_power_on_device_init: true if need to set max power in F/W on device init.
700   * @supports_user_set_page_size: true if user can set the allocation page size.
701   * @dma_mask: the dma mask to be set for this device.
702   * @supports_advanced_cpucp_rc: true if new cpucp opcodes are supported.
703   * @supports_engine_modes: true if changing engines/engine_cores modes is supported.
704   * @support_dynamic_resereved_fw_size: true if we support dynamic reserved size for fw.
705   */
706  struct asic_fixed_properties {
707  	struct hw_queue_properties	*hw_queues_props;
708  	struct hl_special_block_info	*special_blocks;
709  	struct hl_skip_blocks_cfg	skip_special_blocks_cfg;
710  	struct cpucp_info		cpucp_info;
711  	char				uboot_ver[VERSION_MAX_LEN];
712  	char				preboot_ver[VERSION_MAX_LEN];
713  	struct hl_mmu_properties	dmmu;
714  	struct hl_mmu_properties	pmmu;
715  	struct hl_mmu_properties	pmmu_huge;
716  	struct hl_hints_range		hints_dram_reserved_va_range;
717  	struct hl_hints_range		hints_host_reserved_va_range;
718  	struct hl_hints_range		hints_host_hpage_reserved_va_range;
719  	u64				sram_base_address;
720  	u64				sram_end_address;
721  	u64				sram_user_base_address;
722  	u64				dram_base_address;
723  	u64				dram_end_address;
724  	u64				dram_user_base_address;
725  	u64				dram_size;
726  	u64				dram_pci_bar_size;
727  	u64				max_power_default;
728  	u64				dc_power_default;
729  	u64				dram_size_for_default_page_mapping;
730  	u64				pcie_dbi_base_address;
731  	u64				pcie_aux_dbi_reg_addr;
732  	u64				mmu_pgt_addr;
733  	u64				mmu_dram_default_page_addr;
734  	u64				tpc_enabled_mask;
735  	u64				tpc_binning_mask;
736  	u64				dram_enabled_mask;
737  	u64				dram_binning_mask;
738  	u64				dram_hints_align_mask;
739  	u64				cfg_base_address;
740  	u64				mmu_cache_mng_addr;
741  	u64				mmu_cache_mng_size;
742  	u64				device_dma_offset_for_host_access;
743  	u64				host_base_address;
744  	u64				host_end_address;
745  	u64				max_freq_value;
746  	u64				engine_core_interrupt_reg_addr;
747  	u32				clk_pll_index;
748  	u32				mmu_pgt_size;
749  	u32				mmu_pte_size;
750  	u32				dram_page_size;
751  	u32				cfg_size;
752  	u32				sram_size;
753  	u32				max_asid;
754  	u32				num_of_events;
755  	u32				psoc_pci_pll_nr;
756  	u32				psoc_pci_pll_nf;
757  	u32				psoc_pci_pll_od;
758  	u32				psoc_pci_pll_div_factor;
759  	u32				psoc_timestamp_frequency;
760  	u32				high_pll;
761  	u32				cb_pool_cb_cnt;
762  	u32				cb_pool_cb_size;
763  	u32				decoder_enabled_mask;
764  	u32				decoder_binning_mask;
765  	u32				rotator_enabled_mask;
766  	u32				edma_enabled_mask;
767  	u32				edma_binning_mask;
768  	u32				max_pending_cs;
769  	u32				max_queues;
770  	u32				fw_preboot_cpu_boot_dev_sts0;
771  	u32				fw_preboot_cpu_boot_dev_sts1;
772  	u32				fw_bootfit_cpu_boot_dev_sts0;
773  	u32				fw_bootfit_cpu_boot_dev_sts1;
774  	u32				fw_app_cpu_boot_dev_sts0;
775  	u32				fw_app_cpu_boot_dev_sts1;
776  	u32				max_dec;
777  	u32				hmmu_hif_enabled_mask;
778  	u32				faulty_dram_cluster_map;
779  	u32				xbar_edge_enabled_mask;
780  	u32				device_mem_alloc_default_page_size;
781  	u32				num_engine_cores;
782  	u32				max_num_of_engines;
783  	u32				num_of_special_blocks;
784  	u32				glbl_err_max_cause_num;
785  	u32				hbw_flush_reg;
786  	u32				reserved_fw_mem_size;
787  	u32				fw_event_queue_size;
788  	u16				collective_first_sob;
789  	u16				collective_first_mon;
790  	u16				sync_stream_first_sob;
791  	u16				sync_stream_first_mon;
792  	u16				first_available_user_sob[HL_MAX_DCORES];
793  	u16				first_available_user_mon[HL_MAX_DCORES];
794  	u16				first_available_user_interrupt;
795  	u16				first_available_cq[HL_MAX_DCORES];
796  	u16				user_interrupt_count;
797  	u16				user_dec_intr_count;
798  	u16				tpc_interrupt_id;
799  	u16				eq_interrupt_id;
800  	u16				cache_line_size;
801  	u16				server_type;
802  	u8				completion_queues_count;
803  	u8				completion_mode;
804  	u8				mme_master_slave_mode;
805  	u8				fw_security_enabled;
806  	u8				fw_cpu_boot_dev_sts0_valid;
807  	u8				fw_cpu_boot_dev_sts1_valid;
808  	u8				dram_supports_virtual_memory;
809  	u8				hard_reset_done_by_fw;
810  	u8				num_functional_hbms;
811  	u8				hints_range_reservation;
812  	u8				iatu_done_by_fw;
813  	u8				dynamic_fw_load;
814  	u8				gic_interrupts_enable;
815  	u8				use_get_power_for_reset_history;
816  	u8				supports_compute_reset;
817  	u8				allow_inference_soft_reset;
818  	u8				configurable_stop_on_err;
819  	u8				set_max_power_on_device_init;
820  	u8				supports_user_set_page_size;
821  	u8				dma_mask;
822  	u8				supports_advanced_cpucp_rc;
823  	u8				supports_engine_modes;
824  	u8				support_dynamic_resereved_fw_size;
825  };
826  
827  /**
828   * struct hl_fence - software synchronization primitive
829   * @completion: fence is implemented using completion
830   * @refcount: refcount for this fence
831   * @cs_sequence: sequence of the corresponding command submission
832   * @stream_master_qid_map: streams masters QID bitmap to represent all streams
833   *                         masters QIDs that multi cs is waiting on
834   * @error: mark this fence with error
835   * @timestamp: timestamp upon completion
836   * @mcs_handling_done: indicates that corresponding command submission has
837   *                     finished msc handling, this does not mean it was part
838   *                     of the mcs
839   */
840  struct hl_fence {
841  	struct completion	completion;
842  	struct kref		refcount;
843  	u64			cs_sequence;
844  	u32			stream_master_qid_map;
845  	int			error;
846  	ktime_t			timestamp;
847  	u8			mcs_handling_done;
848  };
849  
850  /**
851   * struct hl_cs_compl - command submission completion object.
852   * @base_fence: hl fence object.
853   * @lock: spinlock to protect fence.
854   * @hdev: habanalabs device structure.
855   * @hw_sob: the H/W SOB used in this signal/wait CS.
856   * @encaps_sig_hdl: encaps signals handler.
857   * @cs_seq: command submission sequence number.
858   * @type: type of the CS - signal/wait.
859   * @sob_val: the SOB value that is used in this signal/wait CS.
860   * @sob_group: the SOB group that is used in this collective wait CS.
861   * @encaps_signals: indication whether it's a completion object of cs with
862   * encaps signals or not.
863   */
864  struct hl_cs_compl {
865  	struct hl_fence		base_fence;
866  	spinlock_t		lock;
867  	struct hl_device	*hdev;
868  	struct hl_hw_sob	*hw_sob;
869  	struct hl_cs_encaps_sig_handle *encaps_sig_hdl;
870  	u64			cs_seq;
871  	enum hl_cs_type		type;
872  	u16			sob_val;
873  	u16			sob_group;
874  	bool			encaps_signals;
875  };
876  
877  /*
878   * Command Buffers
879   */
880  
881  /**
882   * struct hl_ts_buff - describes a timestamp buffer.
883   * @kernel_buff_address: Holds the internal buffer's kernel virtual address.
884   * @user_buff_address: Holds the user buffer's kernel virtual address.
885   * @kernel_buff_size: Holds the internal kernel buffer size.
886   */
887  struct hl_ts_buff {
888  	void			*kernel_buff_address;
889  	void			*user_buff_address;
890  	u32			kernel_buff_size;
891  };
892  
893  struct hl_mmap_mem_buf;
894  
895  /**
896   * struct hl_mem_mgr - describes unified memory manager for mappable memory chunks.
897   * @dev: back pointer to the owning device
898   * @lock: protects handles
899   * @handles: an idr holding all active handles to the memory buffers in the system.
900   */
901  struct hl_mem_mgr {
902  	struct device *dev;
903  	spinlock_t lock;
904  	struct idr handles;
905  };
906  
907  /**
908   * struct hl_mem_mgr_fini_stats - describes statistics returned during memory manager teardown.
909   * @n_busy_cb: the amount of CB handles that could not be removed
910   * @n_busy_ts: the amount of TS handles that could not be removed
911   * @n_busy_other: the amount of any other type of handles that could not be removed
912   */
913  struct hl_mem_mgr_fini_stats {
914  	u32 n_busy_cb;
915  	u32 n_busy_ts;
916  	u32 n_busy_other;
917  };
918  
919  /**
920   * struct hl_mmap_mem_buf_behavior - describes unified memory manager buffer behavior
921   * @topic: string identifier used for logging
922   * @mem_id: memory type identifier, embedded in the handle and used to identify
923   *          the memory type by handle.
924   * @alloc: callback executed on buffer allocation, shall allocate the memory,
925   *         set it under buffer private, and set mappable size.
926   * @mmap: callback executed on mmap, must map the buffer to vma
927   * @release: callback executed on release, must free the resources used by the buffer
928   */
929  struct hl_mmap_mem_buf_behavior {
930  	const char *topic;
931  	u64 mem_id;
932  
933  	int (*alloc)(struct hl_mmap_mem_buf *buf, gfp_t gfp, void *args);
934  	int (*mmap)(struct hl_mmap_mem_buf *buf, struct vm_area_struct *vma, void *args);
935  	void (*release)(struct hl_mmap_mem_buf *buf);
936  };
937  
938  /**
939   * struct hl_mmap_mem_buf - describes a single unified memory buffer
940   * @behavior: buffer behavior
941   * @mmg: back pointer to the unified memory manager
942   * @refcount: reference counter for buffer users
943   * @private: pointer to buffer behavior private data
944   * @mmap: atomic boolean indicating whether or not the buffer is mapped right now
945   * @real_mapped_size: the actual size of buffer mapped, after part of it may be released,
946   *                   may change at runtime.
947   * @mappable_size: the original mappable size of the buffer, does not change after
948   *                 the allocation.
949   * @handle: the buffer id in mmg handles store
950   */
951  struct hl_mmap_mem_buf {
952  	struct hl_mmap_mem_buf_behavior *behavior;
953  	struct hl_mem_mgr *mmg;
954  	struct kref refcount;
955  	void *private;
956  	atomic_t mmap;
957  	u64 real_mapped_size;
958  	u64 mappable_size;
959  	u64 handle;
960  };
961  
962  /**
963   * struct hl_cb - describes a Command Buffer.
964   * @hdev: pointer to device this CB belongs to.
965   * @ctx: pointer to the CB owner's context.
966   * @buf: back pointer to the parent mappable memory buffer
967   * @debugfs_list: node in debugfs list of command buffers.
968   * @pool_list: node in pool list of command buffers.
969   * @kernel_address: Holds the CB's kernel virtual address.
970   * @virtual_addr: Holds the CB's virtual address.
971   * @bus_address: Holds the CB's DMA address.
972   * @size: holds the CB's size.
973   * @roundup_size: holds the cb size after roundup to page size.
974   * @cs_cnt: holds number of CS that this CB participates in.
975   * @is_handle_destroyed: atomic boolean indicating whether or not the CB handle was destroyed.
976   * @is_pool: true if CB was acquired from the pool, false otherwise.
977   * @is_internal: internally allocated
978   * @is_mmu_mapped: true if the CB is mapped to the device's MMU.
979   */
980  struct hl_cb {
981  	struct hl_device	*hdev;
982  	struct hl_ctx		*ctx;
983  	struct hl_mmap_mem_buf	*buf;
984  	struct list_head	debugfs_list;
985  	struct list_head	pool_list;
986  	void			*kernel_address;
987  	u64			virtual_addr;
988  	dma_addr_t		bus_address;
989  	u32			size;
990  	u32			roundup_size;
991  	atomic_t		cs_cnt;
992  	atomic_t		is_handle_destroyed;
993  	u8			is_pool;
994  	u8			is_internal;
995  	u8			is_mmu_mapped;
996  };
997  
998  
999  /*
1000   * QUEUES
1001   */
1002  
1003  struct hl_cs_job;
1004  
1005  /* Queue length of external and HW queues */
1006  #define HL_QUEUE_LENGTH			4096
1007  #define HL_QUEUE_SIZE_IN_BYTES		(HL_QUEUE_LENGTH * HL_BD_SIZE)
1008  
1009  #if (HL_MAX_JOBS_PER_CS > HL_QUEUE_LENGTH)
1010  #error "HL_QUEUE_LENGTH must be greater than HL_MAX_JOBS_PER_CS"
1011  #endif
1012  
1013  /* HL_CQ_LENGTH is in units of struct hl_cq_entry */
1014  #define HL_CQ_LENGTH			HL_QUEUE_LENGTH
1015  #define HL_CQ_SIZE_IN_BYTES		(HL_CQ_LENGTH * HL_CQ_ENTRY_SIZE)
1016  
1017  /* Must be power of 2 */
1018  #define HL_EQ_LENGTH			64
1019  #define HL_EQ_SIZE_IN_BYTES		(HL_EQ_LENGTH * HL_EQ_ENTRY_SIZE)
1020  
1021  /* Host <-> CPU-CP shared memory size */
1022  #define HL_CPU_ACCESSIBLE_MEM_SIZE	SZ_2M
1023  
1024  /**
1025   * struct hl_sync_stream_properties -
1026   *     describes a H/W queue sync stream properties
1027   * @hw_sob: array of the used H/W SOBs by this H/W queue.
1028   * @next_sob_val: the next value to use for the currently used SOB.
1029   * @base_sob_id: the base SOB id of the SOBs used by this queue.
1030   * @base_mon_id: the base MON id of the MONs used by this queue.
1031   * @collective_mstr_mon_id: the MON ids of the MONs used by this master queue
1032   *                          in order to sync with all slave queues.
1033   * @collective_slave_mon_id: the MON id used by this slave queue in order to
1034   *                           sync with its master queue.
1035   * @collective_sob_id: current SOB id used by this collective slave queue
1036   *                     to signal its collective master queue upon completion.
1037   * @curr_sob_offset: the id offset to the currently used SOB from the
1038   *                   HL_RSVD_SOBS that are being used by this queue.
1039   */
1040  struct hl_sync_stream_properties {
1041  	struct hl_hw_sob hw_sob[HL_RSVD_SOBS];
1042  	u16		next_sob_val;
1043  	u16		base_sob_id;
1044  	u16		base_mon_id;
1045  	u16		collective_mstr_mon_id[HL_COLLECTIVE_RSVD_MSTR_MONS];
1046  	u16		collective_slave_mon_id;
1047  	u16		collective_sob_id;
1048  	u8		curr_sob_offset;
1049  };
1050  
1051  /**
1052   * struct hl_encaps_signals_mgr - describes sync stream encapsulated signals
1053   * handlers manager
1054   * @lock: protects handles.
1055   * @handles: an idr to hold all encapsulated signals handles.
1056   */
1057  struct hl_encaps_signals_mgr {
1058  	spinlock_t		lock;
1059  	struct idr		handles;
1060  };
1061  
1062  /**
1063   * struct hl_hw_queue - describes a H/W transport queue.
1064   * @shadow_queue: pointer to a shadow queue that holds pointers to jobs.
1065   * @sync_stream_prop: sync stream queue properties
1066   * @queue_type: type of queue.
1067   * @collective_mode: collective mode of current queue
1068   * @kernel_address: holds the queue's kernel virtual address.
1069   * @bus_address: holds the queue's DMA address.
1070   * @pq_dram_address: hold the dram address when the PQ is allocated, used when dram_bd is true in
1071   *                   queue properites.
1072   * @pi: holds the queue's pi value.
1073   * @ci: holds the queue's ci value, AS CALCULATED BY THE DRIVER (not real ci).
1074   * @hw_queue_id: the id of the H/W queue.
1075   * @cq_id: the id for the corresponding CQ for this H/W queue.
1076   * @msi_vec: the IRQ number of the H/W queue.
1077   * @int_queue_len: length of internal queue (number of entries).
1078   * @valid: is the queue valid (we have array of 32 queues, not all of them
1079   *         exist).
1080   * @supports_sync_stream: True if queue supports sync stream
1081   * @dram_bd: True if the bd should be copied to dram, needed for PQ which has been allocated on dram
1082   */
1083  struct hl_hw_queue {
1084  	struct hl_cs_job			**shadow_queue;
1085  	struct hl_sync_stream_properties	sync_stream_prop;
1086  	enum hl_queue_type			queue_type;
1087  	enum hl_collective_mode			collective_mode;
1088  	void					*kernel_address;
1089  	dma_addr_t				bus_address;
1090  	u64					pq_dram_address;
1091  	u32					pi;
1092  	atomic_t				ci;
1093  	u32					hw_queue_id;
1094  	u32					cq_id;
1095  	u32					msi_vec;
1096  	u16					int_queue_len;
1097  	u8					valid;
1098  	u8					supports_sync_stream;
1099  	u8					dram_bd;
1100  };
1101  
1102  /**
1103   * struct hl_cq - describes a completion queue
1104   * @hdev: pointer to the device structure
1105   * @kernel_address: holds the queue's kernel virtual address
1106   * @bus_address: holds the queue's DMA address
1107   * @cq_idx: completion queue index in array
1108   * @hw_queue_id: the id of the matching H/W queue
1109   * @ci: ci inside the queue
1110   * @pi: pi inside the queue
1111   * @free_slots_cnt: counter of free slots in queue
1112   */
1113  struct hl_cq {
1114  	struct hl_device	*hdev;
1115  	void			*kernel_address;
1116  	dma_addr_t		bus_address;
1117  	u32			cq_idx;
1118  	u32			hw_queue_id;
1119  	u32			ci;
1120  	u32			pi;
1121  	atomic_t		free_slots_cnt;
1122  };
1123  
1124  enum hl_user_interrupt_type {
1125  	HL_USR_INTERRUPT_CQ = 0,
1126  	HL_USR_INTERRUPT_DECODER,
1127  	HL_USR_INTERRUPT_TPC,
1128  	HL_USR_INTERRUPT_UNEXPECTED
1129  };
1130  
1131  /**
1132   * struct hl_ts_free_jobs - holds user interrupt ts free nodes related data
1133   * @free_nodes_pool: pool of nodes to be used for free timestamp jobs
1134   * @free_nodes_length: number of nodes in free_nodes_pool
1135   * @next_avail_free_node_idx: index of the next free node in the pool
1136   *
1137   * the free nodes pool must be protected by the user interrupt lock
1138   * to avoid race between different interrupts which are using the same
1139   * ts buffer with different offsets.
1140   */
1141  struct hl_ts_free_jobs {
1142  	struct timestamp_reg_free_node *free_nodes_pool;
1143  	u32				free_nodes_length;
1144  	u32				next_avail_free_node_idx;
1145  };
1146  
1147  /**
1148   * struct hl_user_interrupt - holds user interrupt information
1149   * @hdev: pointer to the device structure
1150   * @ts_free_jobs_data: timestamp free jobs related data
1151   * @type: user interrupt type
1152   * @wait_list_head: head to the list of user threads pending on this interrupt
1153   * @ts_list_head: head to the list of timestamp records
1154   * @wait_list_lock: protects wait_list_head
1155   * @ts_list_lock: protects ts_list_head
1156   * @timestamp: last timestamp taken upon interrupt
1157   * @interrupt_id: msix interrupt id
1158   */
1159  struct hl_user_interrupt {
1160  	struct hl_device		*hdev;
1161  	struct hl_ts_free_jobs		ts_free_jobs_data;
1162  	enum hl_user_interrupt_type	type;
1163  	struct list_head		wait_list_head;
1164  	struct list_head		ts_list_head;
1165  	spinlock_t			wait_list_lock;
1166  	spinlock_t			ts_list_lock;
1167  	ktime_t				timestamp;
1168  	u32				interrupt_id;
1169  };
1170  
1171  /**
1172   * struct timestamp_reg_free_node - holds the timestamp registration free objects node
1173   * @free_objects_node: node in the list free_obj_jobs
1174   * @cq_cb: pointer to cq command buffer to be freed
1175   * @buf: pointer to timestamp buffer to be freed
1176   * @in_use: indicates whether the node still in use in workqueue thread.
1177   * @dynamic_alloc: indicates whether the node was allocated dynamically in the interrupt handler
1178   */
1179  struct timestamp_reg_free_node {
1180  	struct list_head	free_objects_node;
1181  	struct hl_cb		*cq_cb;
1182  	struct hl_mmap_mem_buf	*buf;
1183  	atomic_t		in_use;
1184  	u8			dynamic_alloc;
1185  };
1186  
1187  /* struct timestamp_reg_work_obj - holds the timestamp registration free objects job
1188   * the job will be to pass over the free_obj_jobs list and put refcount to objects
1189   * in each node of the list
1190   * @free_obj: workqueue object to free timestamp registration node objects
1191   * @hdev: pointer to the device structure
1192   * @free_obj_head: list of free jobs nodes (node type timestamp_reg_free_node)
1193   * @dynamic_alloc_free_obj_head: list of free jobs nodes which were dynamically allocated in the
1194   *                               interrupt handler.
1195   */
1196  struct timestamp_reg_work_obj {
1197  	struct work_struct	free_obj;
1198  	struct hl_device	*hdev;
1199  	struct list_head	*free_obj_head;
1200  	struct list_head	*dynamic_alloc_free_obj_head;
1201  };
1202  
1203  /* struct timestamp_reg_info - holds the timestamp registration related data.
1204   * @buf: pointer to the timestamp buffer which include both user/kernel buffers.
1205   *       relevant only when doing timestamps records registration.
1206   * @cq_cb: pointer to CQ counter CB.
1207   * @interrupt: interrupt that the node hanged on it's wait list.
1208   * @timestamp_kernel_addr: timestamp handle address, where to set timestamp
1209   *                         relevant only when doing timestamps records
1210   *                         registration.
1211   * @in_use: indicates if the node already in use. relevant only when doing
1212   *          timestamps records registration, since in this case the driver
1213   *          will have it's own buffer which serve as a records pool instead of
1214   *          allocating records dynamically.
1215   */
1216  struct timestamp_reg_info {
1217  	struct hl_mmap_mem_buf		*buf;
1218  	struct hl_cb			*cq_cb;
1219  	struct hl_user_interrupt	*interrupt;
1220  	u64				*timestamp_kernel_addr;
1221  	bool				in_use;
1222  };
1223  
1224  /**
1225   * struct hl_user_pending_interrupt - holds a context to a user thread
1226   *                                    pending on an interrupt
1227   * @ts_reg_info: holds the timestamps registration nodes info
1228   * @list_node: node in the list of user threads pending on an interrupt or timestamp
1229   * @fence: hl fence object for interrupt completion
1230   * @cq_target_value: CQ target value
1231   * @cq_kernel_addr: CQ kernel address, to be used in the cq interrupt
1232   *                  handler for target value comparison
1233   */
1234  struct hl_user_pending_interrupt {
1235  	struct timestamp_reg_info	ts_reg_info;
1236  	struct list_head		list_node;
1237  	struct hl_fence			fence;
1238  	u64				cq_target_value;
1239  	u64				*cq_kernel_addr;
1240  };
1241  
1242  /**
1243   * struct hl_eq - describes the event queue (single one per device)
1244   * @hdev: pointer to the device structure
1245   * @kernel_address: holds the queue's kernel virtual address
1246   * @bus_address: holds the queue's DMA address
1247   * @size: the event queue size
1248   * @ci: ci inside the queue
1249   * @prev_eqe_index: the index of the previous event queue entry. The index of
1250   *                  the current entry's index must be +1 of the previous one.
1251   * @check_eqe_index: do we need to check the index of the current entry vs. the
1252   *                   previous one. This is for backward compatibility with older
1253   *                   firmwares
1254   */
1255  struct hl_eq {
1256  	struct hl_device	*hdev;
1257  	void			*kernel_address;
1258  	dma_addr_t		bus_address;
1259  	u32			size;
1260  	u32			ci;
1261  	u32			prev_eqe_index;
1262  	bool			check_eqe_index;
1263  };
1264  
1265  /**
1266   * struct hl_dec - describes a decoder sw instance.
1267   * @hdev: pointer to the device structure.
1268   * @abnrm_intr_work: workqueue work item to run when decoder generates an error interrupt.
1269   * @core_id: ID of the decoder.
1270   * @base_addr: base address of the decoder.
1271   */
1272  struct hl_dec {
1273  	struct hl_device	*hdev;
1274  	struct work_struct	abnrm_intr_work;
1275  	u32			core_id;
1276  	u32			base_addr;
1277  };
1278  
1279  /**
1280   * enum hl_asic_type - supported ASIC types.
1281   * @ASIC_INVALID: Invalid ASIC type.
1282   * @ASIC_GOYA: Goya device (HL-1000).
1283   * @ASIC_GAUDI: Gaudi device (HL-2000).
1284   * @ASIC_GAUDI_SEC: Gaudi secured device (HL-2000).
1285   * @ASIC_GAUDI2: Gaudi2 device.
1286   * @ASIC_GAUDI2B: Gaudi2B device.
1287   * @ASIC_GAUDI2C: Gaudi2C device.
1288   * @ASIC_GAUDI2D: Gaudi2D device.
1289   */
1290  enum hl_asic_type {
1291  	ASIC_INVALID,
1292  
1293  	ASIC_GOYA,
1294  	ASIC_GAUDI,
1295  	ASIC_GAUDI_SEC,
1296  	ASIC_GAUDI2,
1297  	ASIC_GAUDI2B,
1298  	ASIC_GAUDI2C,
1299  	ASIC_GAUDI2D,
1300  };
1301  
1302  struct hl_cs_parser;
1303  
1304  /**
1305   * enum hl_pm_mng_profile - power management profile.
1306   * @PM_AUTO: internal clock is set by the Linux driver.
1307   * @PM_MANUAL: internal clock is set by the user.
1308   * @PM_LAST: last power management type.
1309   */
1310  enum hl_pm_mng_profile {
1311  	PM_AUTO = 1,
1312  	PM_MANUAL,
1313  	PM_LAST
1314  };
1315  
1316  /**
1317   * enum hl_pll_frequency - PLL frequency.
1318   * @PLL_HIGH: high frequency.
1319   * @PLL_LOW: low frequency.
1320   * @PLL_LAST: last frequency values that were configured by the user.
1321   */
1322  enum hl_pll_frequency {
1323  	PLL_HIGH = 1,
1324  	PLL_LOW,
1325  	PLL_LAST
1326  };
1327  
1328  #define PLL_REF_CLK 50
1329  
1330  enum div_select_defs {
1331  	DIV_SEL_REF_CLK = 0,
1332  	DIV_SEL_PLL_CLK = 1,
1333  	DIV_SEL_DIVIDED_REF = 2,
1334  	DIV_SEL_DIVIDED_PLL = 3,
1335  };
1336  
1337  enum debugfs_access_type {
1338  	DEBUGFS_READ8,
1339  	DEBUGFS_WRITE8,
1340  	DEBUGFS_READ32,
1341  	DEBUGFS_WRITE32,
1342  	DEBUGFS_READ64,
1343  	DEBUGFS_WRITE64,
1344  };
1345  
1346  enum pci_region {
1347  	PCI_REGION_CFG,
1348  	PCI_REGION_SRAM,
1349  	PCI_REGION_DRAM,
1350  	PCI_REGION_SP_SRAM,
1351  	PCI_REGION_NUMBER,
1352  };
1353  
1354  /**
1355   * struct pci_mem_region - describe memory region in a PCI bar
1356   * @region_base: region base address
1357   * @region_size: region size
1358   * @bar_size: size of the BAR
1359   * @offset_in_bar: region offset into the bar
1360   * @bar_id: bar ID of the region
1361   * @used: if used 1, otherwise 0
1362   */
1363  struct pci_mem_region {
1364  	u64 region_base;
1365  	u64 region_size;
1366  	u64 bar_size;
1367  	u64 offset_in_bar;
1368  	u8 bar_id;
1369  	u8 used;
1370  };
1371  
1372  /**
1373   * struct static_fw_load_mgr - static FW load manager
1374   * @preboot_version_max_off: max offset to preboot version
1375   * @boot_fit_version_max_off: max offset to boot fit version
1376   * @kmd_msg_to_cpu_reg: register address for KDM->CPU messages
1377   * @cpu_cmd_status_to_host_reg: register address for CPU command status response
1378   * @cpu_boot_status_reg: boot status register
1379   * @cpu_boot_dev_status0_reg: boot device status register 0
1380   * @cpu_boot_dev_status1_reg: boot device status register 1
1381   * @boot_err0_reg: boot error register 0
1382   * @boot_err1_reg: boot error register 1
1383   * @preboot_version_offset_reg: SRAM offset to preboot version register
1384   * @boot_fit_version_offset_reg: SRAM offset to boot fit version register
1385   * @sram_offset_mask: mask for getting offset into the SRAM
1386   * @cpu_reset_wait_msec: used when setting WFE via kmd_msg_to_cpu_reg
1387   */
1388  struct static_fw_load_mgr {
1389  	u64 preboot_version_max_off;
1390  	u64 boot_fit_version_max_off;
1391  	u32 kmd_msg_to_cpu_reg;
1392  	u32 cpu_cmd_status_to_host_reg;
1393  	u32 cpu_boot_status_reg;
1394  	u32 cpu_boot_dev_status0_reg;
1395  	u32 cpu_boot_dev_status1_reg;
1396  	u32 boot_err0_reg;
1397  	u32 boot_err1_reg;
1398  	u32 preboot_version_offset_reg;
1399  	u32 boot_fit_version_offset_reg;
1400  	u32 sram_offset_mask;
1401  	u32 cpu_reset_wait_msec;
1402  };
1403  
1404  /**
1405   * struct fw_response - FW response to LKD command
1406   * @ram_offset: descriptor offset into the RAM
1407   * @ram_type: RAM type containing the descriptor (SRAM/DRAM)
1408   * @status: command status
1409   */
1410  struct fw_response {
1411  	u32 ram_offset;
1412  	u8 ram_type;
1413  	u8 status;
1414  };
1415  
1416  /**
1417   * struct dynamic_fw_load_mgr - dynamic FW load manager
1418   * @response: FW to LKD response
1419   * @comm_desc: the communication descriptor with FW
1420   * @image_region: region to copy the FW image to
1421   * @fw_image_size: size of FW image to load
1422   * @wait_for_bl_timeout: timeout for waiting for boot loader to respond
1423   * @fw_desc_valid: true if FW descriptor has been validated and hence the data can be used
1424   */
1425  struct dynamic_fw_load_mgr {
1426  	struct fw_response response;
1427  	struct lkd_fw_comms_desc comm_desc;
1428  	struct pci_mem_region *image_region;
1429  	size_t fw_image_size;
1430  	u32 wait_for_bl_timeout;
1431  	bool fw_desc_valid;
1432  };
1433  
1434  /**
1435   * struct pre_fw_load_props - needed properties for pre-FW load
1436   * @cpu_boot_status_reg: cpu_boot_status register address
1437   * @sts_boot_dev_sts0_reg: sts_boot_dev_sts0 register address
1438   * @sts_boot_dev_sts1_reg: sts_boot_dev_sts1 register address
1439   * @boot_err0_reg: boot_err0 register address
1440   * @boot_err1_reg: boot_err1 register address
1441   * @wait_for_preboot_timeout: timeout to poll for preboot ready
1442   * @wait_for_preboot_extended_timeout: timeout to pull for preboot ready in case where we know
1443   *		preboot needs longer time.
1444   */
1445  struct pre_fw_load_props {
1446  	u32 cpu_boot_status_reg;
1447  	u32 sts_boot_dev_sts0_reg;
1448  	u32 sts_boot_dev_sts1_reg;
1449  	u32 boot_err0_reg;
1450  	u32 boot_err1_reg;
1451  	u32 wait_for_preboot_timeout;
1452  	u32 wait_for_preboot_extended_timeout;
1453  };
1454  
1455  /**
1456   * struct fw_image_props - properties of FW image
1457   * @image_name: name of the image
1458   * @src_off: offset in src FW to copy from
1459   * @copy_size: amount of bytes to copy (0 to copy the whole binary)
1460   */
1461  struct fw_image_props {
1462  	char *image_name;
1463  	u32 src_off;
1464  	u32 copy_size;
1465  };
1466  
1467  /**
1468   * struct fw_load_mgr - manager FW loading process
1469   * @dynamic_loader: specific structure for dynamic load
1470   * @static_loader: specific structure for static load
1471   * @pre_fw_load_props: parameter for pre FW load
1472   * @boot_fit_img: boot fit image properties
1473   * @linux_img: linux image properties
1474   * @cpu_timeout: CPU response timeout in usec
1475   * @boot_fit_timeout: Boot fit load timeout in usec
1476   * @skip_bmc: should BMC be skipped
1477   * @sram_bar_id: SRAM bar ID
1478   * @dram_bar_id: DRAM bar ID
1479   * @fw_comp_loaded: bitmask of loaded FW components. set bit meaning loaded
1480   *                  component. values are set according to enum hl_fw_types.
1481   */
1482  struct fw_load_mgr {
1483  	union {
1484  		struct dynamic_fw_load_mgr dynamic_loader;
1485  		struct static_fw_load_mgr static_loader;
1486  	};
1487  	struct pre_fw_load_props pre_fw_load;
1488  	struct fw_image_props boot_fit_img;
1489  	struct fw_image_props linux_img;
1490  	u32 cpu_timeout;
1491  	u32 boot_fit_timeout;
1492  	u8 skip_bmc;
1493  	u8 sram_bar_id;
1494  	u8 dram_bar_id;
1495  	u8 fw_comp_loaded;
1496  };
1497  
1498  struct hl_cs;
1499  
1500  /**
1501   * struct engines_data - asic engines data
1502   * @buf: buffer for engines data in ascii
1503   * @actual_size: actual size of data that was written by the driver to the allocated buffer
1504   * @allocated_buf_size: total size of allocated buffer
1505   */
1506  struct engines_data {
1507  	char *buf;
1508  	int actual_size;
1509  	u32 allocated_buf_size;
1510  };
1511  
1512  /**
1513   * struct hl_asic_funcs - ASIC specific functions that are can be called from
1514   *                        common code.
1515   * @early_init: sets up early driver state (pre sw_init), doesn't configure H/W.
1516   * @early_fini: tears down what was done in early_init.
1517   * @late_init: sets up late driver/hw state (post hw_init) - Optional.
1518   * @late_fini: tears down what was done in late_init (pre hw_fini) - Optional.
1519   * @sw_init: sets up driver state, does not configure H/W.
1520   * @sw_fini: tears down driver state, does not configure H/W.
1521   * @hw_init: sets up the H/W state.
1522   * @hw_fini: tears down the H/W state.
1523   * @halt_engines: halt engines, needed for reset sequence. This also disables
1524   *                interrupts from the device. Should be called before
1525   *                hw_fini and before CS rollback.
1526   * @suspend: handles IP specific H/W or SW changes for suspend.
1527   * @resume: handles IP specific H/W or SW changes for resume.
1528   * @mmap: maps a memory.
1529   * @ring_doorbell: increment PI on a given QMAN.
1530   * @pqe_write: Write the PQ entry to the PQ. This is ASIC-specific
1531   *             function because the PQs are located in different memory areas
1532   *             per ASIC (SRAM, DRAM, Host memory) and therefore, the method of
1533   *             writing the PQE must match the destination memory area
1534   *             properties.
1535   * @asic_dma_alloc_coherent: Allocate coherent DMA memory by calling
1536   *                           dma_alloc_coherent(). This is ASIC function because
1537   *                           its implementation is not trivial when the driver
1538   *                           is loaded in simulation mode (not upstreamed).
1539   * @asic_dma_free_coherent:  Free coherent DMA memory by calling
1540   *                           dma_free_coherent(). This is ASIC function because
1541   *                           its implementation is not trivial when the driver
1542   *                           is loaded in simulation mode (not upstreamed).
1543   * @scrub_device_mem: Scrub the entire SRAM and DRAM.
1544   * @scrub_device_dram: Scrub the dram memory of the device.
1545   * @get_int_queue_base: get the internal queue base address.
1546   * @test_queues: run simple test on all queues for sanity check.
1547   * @asic_dma_pool_zalloc: small DMA allocation of coherent memory from DMA pool.
1548   *                        size of allocation is HL_DMA_POOL_BLK_SIZE.
1549   * @asic_dma_pool_free: free small DMA allocation from pool.
1550   * @cpu_accessible_dma_pool_alloc: allocate CPU PQ packet from DMA pool.
1551   * @cpu_accessible_dma_pool_free: free CPU PQ packet from DMA pool.
1552   * @dma_unmap_sgtable: DMA unmap scatter-gather table.
1553   * @dma_map_sgtable: DMA map scatter-gather table.
1554   * @cs_parser: parse Command Submission.
1555   * @add_end_of_cb_packets: Add packets to the end of CB, if device requires it.
1556   * @update_eq_ci: update event queue CI.
1557   * @context_switch: called upon ASID context switch.
1558   * @restore_phase_topology: clear all SOBs amd MONs.
1559   * @debugfs_read_dma: debug interface for reading up to 2MB from the device's
1560   *                    internal memory via DMA engine.
1561   * @add_device_attr: add ASIC specific device attributes.
1562   * @handle_eqe: handle event queue entry (IRQ) from CPU-CP.
1563   * @get_events_stat: retrieve event queue entries histogram.
1564   * @read_pte: read MMU page table entry from DRAM.
1565   * @write_pte: write MMU page table entry to DRAM.
1566   * @mmu_invalidate_cache: flush MMU STLB host/DRAM cache, either with soft
1567   *                        (L1 only) or hard (L0 & L1) flush.
1568   * @mmu_invalidate_cache_range: flush specific MMU STLB cache lines with ASID-VA-size mask.
1569   * @mmu_prefetch_cache_range: pre-fetch specific MMU STLB cache lines with ASID-VA-size mask.
1570   * @send_heartbeat: send is-alive packet to CPU-CP and verify response.
1571   * @debug_coresight: perform certain actions on Coresight for debugging.
1572   * @is_device_idle: return true if device is idle, false otherwise.
1573   * @compute_reset_late_init: perform certain actions needed after a compute reset
1574   * @hw_queues_lock: acquire H/W queues lock.
1575   * @hw_queues_unlock: release H/W queues lock.
1576   * @get_pci_id: retrieve PCI ID.
1577   * @get_eeprom_data: retrieve EEPROM data from F/W.
1578   * @get_monitor_dump: retrieve monitor registers dump from F/W.
1579   * @send_cpu_message: send message to F/W. If the message is timedout, the
1580   *                    driver will eventually reset the device. The timeout can
1581   *                    be determined by the calling function or it can be 0 and
1582   *                    then the timeout is the default timeout for the specific
1583   *                    ASIC
1584   * @get_hw_state: retrieve the H/W state
1585   * @pci_bars_map: Map PCI BARs.
1586   * @init_iatu: Initialize the iATU unit inside the PCI controller.
1587   * @rreg: Read a register. Needed for simulator support.
1588   * @wreg: Write a register. Needed for simulator support.
1589   * @halt_coresight: stop the ETF and ETR traces.
1590   * @ctx_init: context dependent initialization.
1591   * @ctx_fini: context dependent cleanup.
1592   * @pre_schedule_cs: Perform pre-CS-scheduling operations.
1593   * @get_queue_id_for_cq: Get the H/W queue id related to the given CQ index.
1594   * @load_firmware_to_device: load the firmware to the device's memory
1595   * @load_boot_fit_to_device: load boot fit to device's memory
1596   * @get_signal_cb_size: Get signal CB size.
1597   * @get_wait_cb_size: Get wait CB size.
1598   * @gen_signal_cb: Generate a signal CB.
1599   * @gen_wait_cb: Generate a wait CB.
1600   * @reset_sob: Reset a SOB.
1601   * @reset_sob_group: Reset SOB group
1602   * @get_device_time: Get the device time.
1603   * @pb_print_security_errors: print security errors according block and cause
1604   * @collective_wait_init_cs: Generate collective master/slave packets
1605   *                           and place them in the relevant cs jobs
1606   * @collective_wait_create_jobs: allocate collective wait cs jobs
1607   * @get_dec_base_addr: get the base address of a given decoder.
1608   * @scramble_addr: Routine to scramble the address prior of mapping it
1609   *                 in the MMU.
1610   * @descramble_addr: Routine to de-scramble the address prior of
1611   *                   showing it to users.
1612   * @ack_protection_bits_errors: ack and dump all security violations
1613   * @get_hw_block_id: retrieve a HW block id to be used by the user to mmap it.
1614   *                   also returns the size of the block if caller supplies
1615   *                   a valid pointer for it
1616   * @hw_block_mmap: mmap a HW block with a given id.
1617   * @enable_events_from_fw: send interrupt to firmware to notify them the
1618   *                         driver is ready to receive asynchronous events. This
1619   *                         function should be called during the first init and
1620   *                         after every hard-reset of the device
1621   * @ack_mmu_errors: check and ack mmu errors, page fault, access violation.
1622   * @get_msi_info: Retrieve asic-specific MSI ID of the f/w async event
1623   * @map_pll_idx_to_fw_idx: convert driver specific per asic PLL index to
1624   *                         generic f/w compatible PLL Indexes
1625   * @init_firmware_preload_params: initialize pre FW-load parameters.
1626   * @init_firmware_loader: initialize data for FW loader.
1627   * @init_cpu_scrambler_dram: Enable CPU specific DRAM scrambling
1628   * @state_dump_init: initialize constants required for state dump
1629   * @get_sob_addr: get SOB base address offset.
1630   * @set_pci_memory_regions: setting properties of PCI memory regions
1631   * @get_stream_master_qid_arr: get pointer to stream masters QID array
1632   * @check_if_razwi_happened: check if there was a razwi due to RR violation.
1633   * @access_dev_mem: access device memory
1634   * @set_dram_bar_base: set the base of the DRAM BAR
1635   * @set_engine_cores: set a config command to engine cores
1636   * @set_engines: set a config command to user engines
1637   * @send_device_activity: indication to FW about device availability
1638   * @set_dram_properties: set DRAM related properties.
1639   * @set_binning_masks: set binning/enable masks for all relevant components.
1640   */
1641  struct hl_asic_funcs {
1642  	int (*early_init)(struct hl_device *hdev);
1643  	int (*early_fini)(struct hl_device *hdev);
1644  	int (*late_init)(struct hl_device *hdev);
1645  	void (*late_fini)(struct hl_device *hdev);
1646  	int (*sw_init)(struct hl_device *hdev);
1647  	int (*sw_fini)(struct hl_device *hdev);
1648  	int (*hw_init)(struct hl_device *hdev);
1649  	int (*hw_fini)(struct hl_device *hdev, bool hard_reset, bool fw_reset);
1650  	void (*halt_engines)(struct hl_device *hdev, bool hard_reset, bool fw_reset);
1651  	int (*suspend)(struct hl_device *hdev);
1652  	int (*resume)(struct hl_device *hdev);
1653  	int (*mmap)(struct hl_device *hdev, struct vm_area_struct *vma,
1654  			void *cpu_addr, dma_addr_t dma_addr, size_t size);
1655  	void (*ring_doorbell)(struct hl_device *hdev, u32 hw_queue_id, u32 pi);
1656  	void (*pqe_write)(struct hl_device *hdev, __le64 *pqe,
1657  			struct hl_bd *bd);
1658  	void* (*asic_dma_alloc_coherent)(struct hl_device *hdev, size_t size,
1659  					dma_addr_t *dma_handle, gfp_t flag);
1660  	void (*asic_dma_free_coherent)(struct hl_device *hdev, size_t size,
1661  					void *cpu_addr, dma_addr_t dma_handle);
1662  	int (*scrub_device_mem)(struct hl_device *hdev);
1663  	int (*scrub_device_dram)(struct hl_device *hdev, u64 val);
1664  	void* (*get_int_queue_base)(struct hl_device *hdev, u32 queue_id,
1665  				dma_addr_t *dma_handle, u16 *queue_len);
1666  	int (*test_queues)(struct hl_device *hdev);
1667  	void* (*asic_dma_pool_zalloc)(struct hl_device *hdev, size_t size,
1668  				gfp_t mem_flags, dma_addr_t *dma_handle);
1669  	void (*asic_dma_pool_free)(struct hl_device *hdev, void *vaddr,
1670  				dma_addr_t dma_addr);
1671  	void* (*cpu_accessible_dma_pool_alloc)(struct hl_device *hdev,
1672  				size_t size, dma_addr_t *dma_handle);
1673  	void (*cpu_accessible_dma_pool_free)(struct hl_device *hdev,
1674  				size_t size, void *vaddr);
1675  	void (*dma_unmap_sgtable)(struct hl_device *hdev, struct sg_table *sgt,
1676  				enum dma_data_direction dir);
1677  	int (*dma_map_sgtable)(struct hl_device *hdev, struct sg_table *sgt,
1678  				enum dma_data_direction dir);
1679  	int (*cs_parser)(struct hl_device *hdev, struct hl_cs_parser *parser);
1680  	void (*add_end_of_cb_packets)(struct hl_device *hdev,
1681  					void *kernel_address, u32 len,
1682  					u32 original_len,
1683  					u64 cq_addr, u32 cq_val, u32 msix_num,
1684  					bool eb);
1685  	void (*update_eq_ci)(struct hl_device *hdev, u32 val);
1686  	int (*context_switch)(struct hl_device *hdev, u32 asid);
1687  	void (*restore_phase_topology)(struct hl_device *hdev);
1688  	int (*debugfs_read_dma)(struct hl_device *hdev, u64 addr, u32 size,
1689  				void *blob_addr);
1690  	void (*add_device_attr)(struct hl_device *hdev, struct attribute_group *dev_clk_attr_grp,
1691  				struct attribute_group *dev_vrm_attr_grp);
1692  	void (*handle_eqe)(struct hl_device *hdev,
1693  				struct hl_eq_entry *eq_entry);
1694  	void* (*get_events_stat)(struct hl_device *hdev, bool aggregate,
1695  				u32 *size);
1696  	u64 (*read_pte)(struct hl_device *hdev, u64 addr);
1697  	void (*write_pte)(struct hl_device *hdev, u64 addr, u64 val);
1698  	int (*mmu_invalidate_cache)(struct hl_device *hdev, bool is_hard,
1699  					u32 flags);
1700  	int (*mmu_invalidate_cache_range)(struct hl_device *hdev, bool is_hard,
1701  				u32 flags, u32 asid, u64 va, u64 size);
1702  	int (*mmu_prefetch_cache_range)(struct hl_ctx *ctx, u32 flags, u32 asid, u64 va, u64 size);
1703  	int (*send_heartbeat)(struct hl_device *hdev);
1704  	int (*debug_coresight)(struct hl_device *hdev, struct hl_ctx *ctx, void *data);
1705  	bool (*is_device_idle)(struct hl_device *hdev, u64 *mask_arr, u8 mask_len,
1706  				struct engines_data *e);
1707  	int (*compute_reset_late_init)(struct hl_device *hdev);
1708  	void (*hw_queues_lock)(struct hl_device *hdev);
1709  	void (*hw_queues_unlock)(struct hl_device *hdev);
1710  	u32 (*get_pci_id)(struct hl_device *hdev);
1711  	int (*get_eeprom_data)(struct hl_device *hdev, void *data, size_t max_size);
1712  	int (*get_monitor_dump)(struct hl_device *hdev, void *data);
1713  	int (*send_cpu_message)(struct hl_device *hdev, u32 *msg,
1714  				u16 len, u32 timeout, u64 *result);
1715  	int (*pci_bars_map)(struct hl_device *hdev);
1716  	int (*init_iatu)(struct hl_device *hdev);
1717  	u32 (*rreg)(struct hl_device *hdev, u32 reg);
1718  	void (*wreg)(struct hl_device *hdev, u32 reg, u32 val);
1719  	void (*halt_coresight)(struct hl_device *hdev, struct hl_ctx *ctx);
1720  	int (*ctx_init)(struct hl_ctx *ctx);
1721  	void (*ctx_fini)(struct hl_ctx *ctx);
1722  	int (*pre_schedule_cs)(struct hl_cs *cs);
1723  	u32 (*get_queue_id_for_cq)(struct hl_device *hdev, u32 cq_idx);
1724  	int (*load_firmware_to_device)(struct hl_device *hdev);
1725  	int (*load_boot_fit_to_device)(struct hl_device *hdev);
1726  	u32 (*get_signal_cb_size)(struct hl_device *hdev);
1727  	u32 (*get_wait_cb_size)(struct hl_device *hdev);
1728  	u32 (*gen_signal_cb)(struct hl_device *hdev, void *data, u16 sob_id,
1729  			u32 size, bool eb);
1730  	u32 (*gen_wait_cb)(struct hl_device *hdev,
1731  			struct hl_gen_wait_properties *prop);
1732  	void (*reset_sob)(struct hl_device *hdev, void *data);
1733  	void (*reset_sob_group)(struct hl_device *hdev, u16 sob_group);
1734  	u64 (*get_device_time)(struct hl_device *hdev);
1735  	void (*pb_print_security_errors)(struct hl_device *hdev,
1736  			u32 block_addr, u32 cause, u32 offended_addr);
1737  	int (*collective_wait_init_cs)(struct hl_cs *cs);
1738  	int (*collective_wait_create_jobs)(struct hl_device *hdev,
1739  			struct hl_ctx *ctx, struct hl_cs *cs,
1740  			u32 wait_queue_id, u32 collective_engine_id,
1741  			u32 encaps_signal_offset);
1742  	u32 (*get_dec_base_addr)(struct hl_device *hdev, u32 core_id);
1743  	u64 (*scramble_addr)(struct hl_device *hdev, u64 addr);
1744  	u64 (*descramble_addr)(struct hl_device *hdev, u64 addr);
1745  	void (*ack_protection_bits_errors)(struct hl_device *hdev);
1746  	int (*get_hw_block_id)(struct hl_device *hdev, u64 block_addr,
1747  				u32 *block_size, u32 *block_id);
1748  	int (*hw_block_mmap)(struct hl_device *hdev, struct vm_area_struct *vma,
1749  			u32 block_id, u32 block_size);
1750  	void (*enable_events_from_fw)(struct hl_device *hdev);
1751  	int (*ack_mmu_errors)(struct hl_device *hdev, u64 mmu_cap_mask);
1752  	void (*get_msi_info)(__le32 *table);
1753  	int (*map_pll_idx_to_fw_idx)(u32 pll_idx);
1754  	void (*init_firmware_preload_params)(struct hl_device *hdev);
1755  	void (*init_firmware_loader)(struct hl_device *hdev);
1756  	void (*init_cpu_scrambler_dram)(struct hl_device *hdev);
1757  	void (*state_dump_init)(struct hl_device *hdev);
1758  	u32 (*get_sob_addr)(struct hl_device *hdev, u32 sob_id);
1759  	void (*set_pci_memory_regions)(struct hl_device *hdev);
1760  	u32* (*get_stream_master_qid_arr)(void);
1761  	void (*check_if_razwi_happened)(struct hl_device *hdev);
1762  	int (*mmu_get_real_page_size)(struct hl_device *hdev, struct hl_mmu_properties *mmu_prop,
1763  					u32 page_size, u32 *real_page_size, bool is_dram_addr);
1764  	int (*access_dev_mem)(struct hl_device *hdev, enum pci_region region_type,
1765  				u64 addr, u64 *val, enum debugfs_access_type acc_type);
1766  	u64 (*set_dram_bar_base)(struct hl_device *hdev, u64 addr);
1767  	int (*set_engine_cores)(struct hl_device *hdev, u32 *core_ids,
1768  					u32 num_cores, u32 core_command);
1769  	int (*set_engines)(struct hl_device *hdev, u32 *engine_ids,
1770  					u32 num_engines, u32 engine_command);
1771  	int (*send_device_activity)(struct hl_device *hdev, bool open);
1772  	int (*set_dram_properties)(struct hl_device *hdev);
1773  	int (*set_binning_masks)(struct hl_device *hdev);
1774  };
1775  
1776  
1777  /*
1778   * CONTEXTS
1779   */
1780  
1781  #define HL_KERNEL_ASID_ID	0
1782  
1783  /**
1784   * enum hl_va_range_type - virtual address range type.
1785   * @HL_VA_RANGE_TYPE_HOST: range type of host pages
1786   * @HL_VA_RANGE_TYPE_HOST_HUGE: range type of host huge pages
1787   * @HL_VA_RANGE_TYPE_DRAM: range type of dram pages
1788   */
1789  enum hl_va_range_type {
1790  	HL_VA_RANGE_TYPE_HOST,
1791  	HL_VA_RANGE_TYPE_HOST_HUGE,
1792  	HL_VA_RANGE_TYPE_DRAM,
1793  	HL_VA_RANGE_TYPE_MAX
1794  };
1795  
1796  /**
1797   * struct hl_va_range - virtual addresses range.
1798   * @lock: protects the virtual addresses list.
1799   * @list: list of virtual addresses blocks available for mappings.
1800   * @start_addr: range start address.
1801   * @end_addr: range end address.
1802   * @page_size: page size of this va range.
1803   */
1804  struct hl_va_range {
1805  	struct mutex		lock;
1806  	struct list_head	list;
1807  	u64			start_addr;
1808  	u64			end_addr;
1809  	u32			page_size;
1810  };
1811  
1812  /**
1813   * struct hl_cs_counters_atomic - command submission counters
1814   * @out_of_mem_drop_cnt: dropped due to memory allocation issue
1815   * @parsing_drop_cnt: dropped due to error in packet parsing
1816   * @queue_full_drop_cnt: dropped due to queue full
1817   * @device_in_reset_drop_cnt: dropped due to device in reset
1818   * @max_cs_in_flight_drop_cnt: dropped due to maximum CS in-flight
1819   * @validation_drop_cnt: dropped due to error in validation
1820   */
1821  struct hl_cs_counters_atomic {
1822  	atomic64_t out_of_mem_drop_cnt;
1823  	atomic64_t parsing_drop_cnt;
1824  	atomic64_t queue_full_drop_cnt;
1825  	atomic64_t device_in_reset_drop_cnt;
1826  	atomic64_t max_cs_in_flight_drop_cnt;
1827  	atomic64_t validation_drop_cnt;
1828  };
1829  
1830  /**
1831   * struct hl_dmabuf_priv - a dma-buf private object.
1832   * @dmabuf: pointer to dma-buf object.
1833   * @ctx: pointer to the dma-buf owner's context.
1834   * @phys_pg_pack: pointer to physical page pack if the dma-buf was exported
1835   *                where virtual memory is supported.
1836   * @memhash_hnode: pointer to the memhash node. this object holds the export count.
1837   * @offset: the offset into the buffer from which the memory is exported.
1838   *          Relevant only if virtual memory is supported and phys_pg_pack is being used.
1839   * device_phys_addr: physical address of the device's memory. Relevant only
1840   *                   if phys_pg_pack is NULL (dma-buf was exported from address).
1841   *                   The total size can be taken from the dmabuf object.
1842   */
1843  struct hl_dmabuf_priv {
1844  	struct dma_buf			*dmabuf;
1845  	struct hl_ctx			*ctx;
1846  	struct hl_vm_phys_pg_pack	*phys_pg_pack;
1847  	struct hl_vm_hash_node		*memhash_hnode;
1848  	u64				offset;
1849  	u64				device_phys_addr;
1850  };
1851  
1852  #define HL_CS_OUTCOME_HISTORY_LEN 256
1853  
1854  /**
1855   * struct hl_cs_outcome - represents a single completed CS outcome
1856   * @list_link: link to either container's used list or free list
1857   * @map_link: list to the container hash map
1858   * @ts: completion ts
1859   * @seq: the original cs sequence
1860   * @error: error code cs completed with, if any
1861   */
1862  struct hl_cs_outcome {
1863  	struct list_head list_link;
1864  	struct hlist_node map_link;
1865  	ktime_t ts;
1866  	u64 seq;
1867  	int error;
1868  };
1869  
1870  /**
1871   * struct hl_cs_outcome_store - represents a limited store of completed CS outcomes
1872   * @outcome_map: index of completed CS searchable by sequence number
1873   * @used_list: list of outcome objects currently in use
1874   * @free_list: list of outcome objects currently not in use
1875   * @nodes_pool: a static pool of pre-allocated outcome objects
1876   * @db_lock: any operation on the store must take this lock
1877   */
1878  struct hl_cs_outcome_store {
1879  	DECLARE_HASHTABLE(outcome_map, 8);
1880  	struct list_head used_list;
1881  	struct list_head free_list;
1882  	struct hl_cs_outcome nodes_pool[HL_CS_OUTCOME_HISTORY_LEN];
1883  	spinlock_t db_lock;
1884  };
1885  
1886  /**
1887   * struct hl_ctx - user/kernel context.
1888   * @mem_hash: holds mapping from virtual address to virtual memory area
1889   *		descriptor (hl_vm_phys_pg_list or hl_userptr).
1890   * @mmu_shadow_hash: holds a mapping from shadow address to pgt_info structure.
1891   * @hr_mmu_phys_hash: if host-resident MMU is used, holds a mapping from
1892   *                    MMU-hop-page physical address to its host-resident
1893   *                    pgt_info structure.
1894   * @hpriv: pointer to the private (Kernel Driver) data of the process (fd).
1895   * @hdev: pointer to the device structure.
1896   * @refcount: reference counter for the context. Context is released only when
1897   *		this hits 0. It is incremented on CS and CS_WAIT.
1898   * @cs_pending: array of hl fence objects representing pending CS.
1899   * @outcome_store: storage data structure used to remember outcomes of completed
1900   *                 command submissions for a long time after CS id wraparound.
1901   * @va_range: holds available virtual addresses for host and dram mappings.
1902   * @mem_hash_lock: protects the mem_hash.
1903   * @hw_block_list_lock: protects the HW block memory list.
1904   * @ts_reg_lock: timestamp registration ioctls lock.
1905   * @debugfs_list: node in debugfs list of contexts.
1906   * @hw_block_mem_list: list of HW block virtual mapped addresses.
1907   * @cs_counters: context command submission counters.
1908   * @cb_va_pool: device VA pool for command buffers which are mapped to the
1909   *              device's MMU.
1910   * @sig_mgr: encaps signals handle manager.
1911   * @cb_va_pool_base: the base address for the device VA pool
1912   * @cs_sequence: sequence number for CS. Value is assigned to a CS and passed
1913   *			to user so user could inquire about CS. It is used as
1914   *			index to cs_pending array.
1915   * @dram_default_hops: array that holds all hops addresses needed for default
1916   *                     DRAM mapping.
1917   * @cs_lock: spinlock to protect cs_sequence.
1918   * @dram_phys_mem: amount of used physical DRAM memory by this context.
1919   * @thread_ctx_switch_token: token to prevent multiple threads of the same
1920   *				context	from running the context switch phase.
1921   *				Only a single thread should run it.
1922   * @thread_ctx_switch_wait_token: token to prevent the threads that didn't run
1923   *				the context switch phase from moving to their
1924   *				execution phase before the context switch phase
1925   *				has finished.
1926   * @asid: context's unique address space ID in the device's MMU.
1927   * @handle: context's opaque handle for user
1928   */
1929  struct hl_ctx {
1930  	DECLARE_HASHTABLE(mem_hash, MEM_HASH_TABLE_BITS);
1931  	DECLARE_HASHTABLE(mmu_shadow_hash, MMU_HASH_TABLE_BITS);
1932  	DECLARE_HASHTABLE(hr_mmu_phys_hash, MMU_HASH_TABLE_BITS);
1933  	struct hl_fpriv			*hpriv;
1934  	struct hl_device		*hdev;
1935  	struct kref			refcount;
1936  	struct hl_fence			**cs_pending;
1937  	struct hl_cs_outcome_store	outcome_store;
1938  	struct hl_va_range		*va_range[HL_VA_RANGE_TYPE_MAX];
1939  	struct mutex			mem_hash_lock;
1940  	struct mutex			hw_block_list_lock;
1941  	struct mutex			ts_reg_lock;
1942  	struct list_head		debugfs_list;
1943  	struct list_head		hw_block_mem_list;
1944  	struct hl_cs_counters_atomic	cs_counters;
1945  	struct gen_pool			*cb_va_pool;
1946  	struct hl_encaps_signals_mgr	sig_mgr;
1947  	u64				cb_va_pool_base;
1948  	u64				cs_sequence;
1949  	u64				*dram_default_hops;
1950  	spinlock_t			cs_lock;
1951  	atomic64_t			dram_phys_mem;
1952  	atomic_t			thread_ctx_switch_token;
1953  	u32				thread_ctx_switch_wait_token;
1954  	u32				asid;
1955  	u32				handle;
1956  };
1957  
1958  /**
1959   * struct hl_ctx_mgr - for handling multiple contexts.
1960   * @lock: protects ctx_handles.
1961   * @handles: idr to hold all ctx handles.
1962   */
1963  struct hl_ctx_mgr {
1964  	struct mutex	lock;
1965  	struct idr	handles;
1966  };
1967  
1968  
1969  /*
1970   * COMMAND SUBMISSIONS
1971   */
1972  
1973  /**
1974   * struct hl_userptr - memory mapping chunk information
1975   * @vm_type: type of the VM.
1976   * @job_node: linked-list node for hanging the object on the Job's list.
1977   * @pages: pointer to struct page array
1978   * @npages: size of @pages array
1979   * @sgt: pointer to the scatter-gather table that holds the pages.
1980   * @dir: for DMA unmapping, the direction must be supplied, so save it.
1981   * @debugfs_list: node in debugfs list of command submissions.
1982   * @pid: the pid of the user process owning the memory
1983   * @addr: user-space virtual address of the start of the memory area.
1984   * @size: size of the memory area to pin & map.
1985   * @dma_mapped: true if the SG was mapped to DMA addresses, false otherwise.
1986   */
1987  struct hl_userptr {
1988  	enum vm_type			vm_type; /* must be first */
1989  	struct list_head		job_node;
1990  	struct page			**pages;
1991  	unsigned int			npages;
1992  	struct sg_table			*sgt;
1993  	enum dma_data_direction		dir;
1994  	struct list_head		debugfs_list;
1995  	pid_t				pid;
1996  	u64				addr;
1997  	u64				size;
1998  	u8				dma_mapped;
1999  };
2000  
2001  /**
2002   * struct hl_cs - command submission.
2003   * @jobs_in_queue_cnt: per each queue, maintain counter of submitted jobs.
2004   * @ctx: the context this CS belongs to.
2005   * @job_list: list of the CS's jobs in the various queues.
2006   * @job_lock: spinlock for the CS's jobs list. Needed for free_job.
2007   * @refcount: reference counter for usage of the CS.
2008   * @fence: pointer to the fence object of this CS.
2009   * @signal_fence: pointer to the fence object of the signal CS (used by wait
2010   *                CS only).
2011   * @finish_work: workqueue object to run when CS is completed by H/W.
2012   * @work_tdr: delayed work node for TDR.
2013   * @mirror_node : node in device mirror list of command submissions.
2014   * @staged_cs_node: node in the staged cs list.
2015   * @debugfs_list: node in debugfs list of command submissions.
2016   * @encaps_sig_hdl: holds the encaps signals handle.
2017   * @sequence: the sequence number of this CS.
2018   * @staged_sequence: the sequence of the staged submission this CS is part of,
2019   *                   relevant only if staged_cs is set.
2020   * @timeout_jiffies: cs timeout in jiffies.
2021   * @submission_time_jiffies: submission time of the cs
2022   * @type: CS_TYPE_*.
2023   * @jobs_cnt: counter of submitted jobs on all queues.
2024   * @encaps_sig_hdl_id: encaps signals handle id, set for the first staged cs.
2025   * @completion_timestamp: timestamp of the last completed cs job.
2026   * @sob_addr_offset: sob offset from the configuration base address.
2027   * @initial_sob_count: count of completed signals in SOB before current submission of signal or
2028   *                     cs with encaps signals.
2029   * @submitted: true if CS was submitted to H/W.
2030   * @completed: true if CS was completed by device.
2031   * @timedout : true if CS was timedout.
2032   * @tdr_active: true if TDR was activated for this CS (to prevent
2033   *		double TDR activation).
2034   * @aborted: true if CS was aborted due to some device error.
2035   * @timestamp: true if a timestamp must be captured upon completion.
2036   * @staged_last: true if this is the last staged CS and needs completion.
2037   * @staged_first: true if this is the first staged CS and we need to receive
2038   *                timeout for this CS.
2039   * @staged_cs: true if this CS is part of a staged submission.
2040   * @skip_reset_on_timeout: true if we shall not reset the device in case
2041   *                         timeout occurs (debug scenario).
2042   * @encaps_signals: true if this CS has encaps reserved signals.
2043   */
2044  struct hl_cs {
2045  	u16			*jobs_in_queue_cnt;
2046  	struct hl_ctx		*ctx;
2047  	struct list_head	job_list;
2048  	spinlock_t		job_lock;
2049  	struct kref		refcount;
2050  	struct hl_fence		*fence;
2051  	struct hl_fence		*signal_fence;
2052  	struct work_struct	finish_work;
2053  	struct delayed_work	work_tdr;
2054  	struct list_head	mirror_node;
2055  	struct list_head	staged_cs_node;
2056  	struct list_head	debugfs_list;
2057  	struct hl_cs_encaps_sig_handle *encaps_sig_hdl;
2058  	ktime_t			completion_timestamp;
2059  	u64			sequence;
2060  	u64			staged_sequence;
2061  	u64			timeout_jiffies;
2062  	u64			submission_time_jiffies;
2063  	enum hl_cs_type		type;
2064  	u32			jobs_cnt;
2065  	u32			encaps_sig_hdl_id;
2066  	u32			sob_addr_offset;
2067  	u16			initial_sob_count;
2068  	u8			submitted;
2069  	u8			completed;
2070  	u8			timedout;
2071  	u8			tdr_active;
2072  	u8			aborted;
2073  	u8			timestamp;
2074  	u8			staged_last;
2075  	u8			staged_first;
2076  	u8			staged_cs;
2077  	u8			skip_reset_on_timeout;
2078  	u8			encaps_signals;
2079  };
2080  
2081  /**
2082   * struct hl_cs_job - command submission job.
2083   * @cs_node: the node to hang on the CS jobs list.
2084   * @cs: the CS this job belongs to.
2085   * @user_cb: the CB we got from the user.
2086   * @patched_cb: in case of patching, this is internal CB which is submitted on
2087   *		the queue instead of the CB we got from the IOCTL.
2088   * @finish_work: workqueue object to run when job is completed.
2089   * @userptr_list: linked-list of userptr mappings that belong to this job and
2090   *			wait for completion.
2091   * @debugfs_list: node in debugfs list of command submission jobs.
2092   * @refcount: reference counter for usage of the CS job.
2093   * @queue_type: the type of the H/W queue this job is submitted to.
2094   * @timestamp: timestamp upon job completion
2095   * @id: the id of this job inside a CS.
2096   * @hw_queue_id: the id of the H/W queue this job is submitted to.
2097   * @user_cb_size: the actual size of the CB we got from the user.
2098   * @job_cb_size: the actual size of the CB that we put on the queue.
2099   * @encaps_sig_wait_offset: encapsulated signals offset, which allow user
2100   *                          to wait on part of the reserved signals.
2101   * @is_kernel_allocated_cb: true if the CB handle we got from the user holds a
2102   *                          handle to a kernel-allocated CB object, false
2103   *                          otherwise (SRAM/DRAM/host address).
2104   * @contains_dma_pkt: whether the JOB contains at least one DMA packet. This
2105   *                    info is needed later, when adding the 2xMSG_PROT at the
2106   *                    end of the JOB, to know which barriers to put in the
2107   *                    MSG_PROT packets. Relevant only for GAUDI as GOYA doesn't
2108   *                    have streams so the engine can't be busy by another
2109   *                    stream.
2110   */
2111  struct hl_cs_job {
2112  	struct list_head	cs_node;
2113  	struct hl_cs		*cs;
2114  	struct hl_cb		*user_cb;
2115  	struct hl_cb		*patched_cb;
2116  	struct work_struct	finish_work;
2117  	struct list_head	userptr_list;
2118  	struct list_head	debugfs_list;
2119  	struct kref		refcount;
2120  	enum hl_queue_type	queue_type;
2121  	ktime_t			timestamp;
2122  	u32			id;
2123  	u32			hw_queue_id;
2124  	u32			user_cb_size;
2125  	u32			job_cb_size;
2126  	u32			encaps_sig_wait_offset;
2127  	u8			is_kernel_allocated_cb;
2128  	u8			contains_dma_pkt;
2129  };
2130  
2131  /**
2132   * struct hl_cs_parser - command submission parser properties.
2133   * @user_cb: the CB we got from the user.
2134   * @patched_cb: in case of patching, this is internal CB which is submitted on
2135   *		the queue instead of the CB we got from the IOCTL.
2136   * @job_userptr_list: linked-list of userptr mappings that belong to the related
2137   *			job and wait for completion.
2138   * @cs_sequence: the sequence number of the related CS.
2139   * @queue_type: the type of the H/W queue this job is submitted to.
2140   * @ctx_id: the ID of the context the related CS belongs to.
2141   * @hw_queue_id: the id of the H/W queue this job is submitted to.
2142   * @user_cb_size: the actual size of the CB we got from the user.
2143   * @patched_cb_size: the size of the CB after parsing.
2144   * @job_id: the id of the related job inside the related CS.
2145   * @is_kernel_allocated_cb: true if the CB handle we got from the user holds a
2146   *                          handle to a kernel-allocated CB object, false
2147   *                          otherwise (SRAM/DRAM/host address).
2148   * @contains_dma_pkt: whether the JOB contains at least one DMA packet. This
2149   *                    info is needed later, when adding the 2xMSG_PROT at the
2150   *                    end of the JOB, to know which barriers to put in the
2151   *                    MSG_PROT packets. Relevant only for GAUDI as GOYA doesn't
2152   *                    have streams so the engine can't be busy by another
2153   *                    stream.
2154   * @completion: true if we need completion for this CS.
2155   */
2156  struct hl_cs_parser {
2157  	struct hl_cb		*user_cb;
2158  	struct hl_cb		*patched_cb;
2159  	struct list_head	*job_userptr_list;
2160  	u64			cs_sequence;
2161  	enum hl_queue_type	queue_type;
2162  	u32			ctx_id;
2163  	u32			hw_queue_id;
2164  	u32			user_cb_size;
2165  	u32			patched_cb_size;
2166  	u8			job_id;
2167  	u8			is_kernel_allocated_cb;
2168  	u8			contains_dma_pkt;
2169  	u8			completion;
2170  };
2171  
2172  /*
2173   * MEMORY STRUCTURE
2174   */
2175  
2176  /**
2177   * struct hl_vm_hash_node - hash element from virtual address to virtual
2178   *				memory area descriptor (hl_vm_phys_pg_list or
2179   *				hl_userptr).
2180   * @node: node to hang on the hash table in context object.
2181   * @vaddr: key virtual address.
2182   * @handle: memory handle for device memory allocation.
2183   * @ptr: value pointer (hl_vm_phys_pg_list or hl_userptr).
2184   * @export_cnt: number of exports from within the VA block.
2185   */
2186  struct hl_vm_hash_node {
2187  	struct hlist_node	node;
2188  	u64			vaddr;
2189  	u64			handle;
2190  	void			*ptr;
2191  	int			export_cnt;
2192  };
2193  
2194  /**
2195   * struct hl_vm_hw_block_list_node - list element from user virtual address to
2196   *				HW block id.
2197   * @node: node to hang on the list in context object.
2198   * @ctx: the context this node belongs to.
2199   * @vaddr: virtual address of the HW block.
2200   * @block_size: size of the block.
2201   * @mapped_size: size of the block which is mapped. May change if partial un-mappings are done.
2202   * @id: HW block id (handle).
2203   */
2204  struct hl_vm_hw_block_list_node {
2205  	struct list_head	node;
2206  	struct hl_ctx		*ctx;
2207  	unsigned long		vaddr;
2208  	u32			block_size;
2209  	u32			mapped_size;
2210  	u32			id;
2211  };
2212  
2213  /**
2214   * struct hl_vm_phys_pg_pack - physical page pack.
2215   * @vm_type: describes the type of the virtual area descriptor.
2216   * @pages: the physical page array.
2217   * @npages: num physical pages in the pack.
2218   * @total_size: total size of all the pages in this list.
2219   * @node: used to attach to deletion list that is used when all the allocations are cleared
2220   *        at the teardown of the context.
2221   * @mapping_cnt: number of shared mappings.
2222   * @asid: the context related to this list.
2223   * @page_size: size of each page in the pack.
2224   * @flags: HL_MEM_* flags related to this list.
2225   * @handle: the provided handle related to this list.
2226   * @offset: offset from the first page.
2227   * @contiguous: is contiguous physical memory.
2228   * @created_from_userptr: is product of host virtual address.
2229   */
2230  struct hl_vm_phys_pg_pack {
2231  	enum vm_type		vm_type; /* must be first */
2232  	u64			*pages;
2233  	u64			npages;
2234  	u64			total_size;
2235  	struct list_head	node;
2236  	atomic_t		mapping_cnt;
2237  	u32			asid;
2238  	u32			page_size;
2239  	u32			flags;
2240  	u32			handle;
2241  	u32			offset;
2242  	u8			contiguous;
2243  	u8			created_from_userptr;
2244  };
2245  
2246  /**
2247   * struct hl_vm_va_block - virtual range block information.
2248   * @node: node to hang on the virtual range list in context object.
2249   * @start: virtual range start address.
2250   * @end: virtual range end address.
2251   * @size: virtual range size.
2252   */
2253  struct hl_vm_va_block {
2254  	struct list_head	node;
2255  	u64			start;
2256  	u64			end;
2257  	u64			size;
2258  };
2259  
2260  /**
2261   * struct hl_vm - virtual memory manager for MMU.
2262   * @dram_pg_pool: pool for DRAM physical pages of 2MB.
2263   * @dram_pg_pool_refcount: reference counter for the pool usage.
2264   * @idr_lock: protects the phys_pg_list_handles.
2265   * @phys_pg_pack_handles: idr to hold all device allocations handles.
2266   * @init_done: whether initialization was done. We need this because VM
2267   *		initialization might be skipped during device initialization.
2268   */
2269  struct hl_vm {
2270  	struct gen_pool		*dram_pg_pool;
2271  	struct kref		dram_pg_pool_refcount;
2272  	spinlock_t		idr_lock;
2273  	struct idr		phys_pg_pack_handles;
2274  	u8			init_done;
2275  };
2276  
2277  
2278  /*
2279   * DEBUG, PROFILING STRUCTURE
2280   */
2281  
2282  /**
2283   * struct hl_debug_params - Coresight debug parameters.
2284   * @input: pointer to component specific input parameters.
2285   * @output: pointer to component specific output parameters.
2286   * @output_size: size of output buffer.
2287   * @reg_idx: relevant register ID.
2288   * @op: component operation to execute.
2289   * @enable: true if to enable component debugging, false otherwise.
2290   */
2291  struct hl_debug_params {
2292  	void *input;
2293  	void *output;
2294  	u32 output_size;
2295  	u32 reg_idx;
2296  	u32 op;
2297  	bool enable;
2298  };
2299  
2300  /**
2301   * struct hl_notifier_event - holds the notifier data structure
2302   * @eventfd: the event file descriptor to raise the notifications
2303   * @lock: mutex lock to protect the notifier data flows
2304   * @events_mask: indicates the bitmap events
2305   */
2306  struct hl_notifier_event {
2307  	struct eventfd_ctx	*eventfd;
2308  	struct mutex		lock;
2309  	u64			events_mask;
2310  };
2311  
2312  /*
2313   * FILE PRIVATE STRUCTURE
2314   */
2315  
2316  /**
2317   * struct hl_fpriv - process information stored in FD private data.
2318   * @hdev: habanalabs device structure.
2319   * @file_priv: pointer to the DRM file private data structure.
2320   * @taskpid: current process ID.
2321   * @ctx: current executing context. TODO: remove for multiple ctx per process
2322   * @ctx_mgr: context manager to handle multiple context for this FD.
2323   * @mem_mgr: manager descriptor for memory exportable via mmap
2324   * @notifier_event: notifier eventfd towards user process
2325   * @debugfs_list: list of relevant ASIC debugfs.
2326   * @dev_node: node in the device list of file private data
2327   * @refcount: number of related contexts.
2328   * @restore_phase_mutex: lock for context switch and restore phase.
2329   * @ctx_lock: protects the pointer to current executing context pointer. TODO: remove for multiple
2330   *            ctx per process.
2331   */
2332  struct hl_fpriv {
2333  	struct hl_device		*hdev;
2334  	struct drm_file			*file_priv;
2335  	struct pid			*taskpid;
2336  	struct hl_ctx			*ctx;
2337  	struct hl_ctx_mgr		ctx_mgr;
2338  	struct hl_mem_mgr		mem_mgr;
2339  	struct hl_notifier_event	notifier_event;
2340  	struct list_head		debugfs_list;
2341  	struct list_head		dev_node;
2342  	struct kref			refcount;
2343  	struct mutex			restore_phase_mutex;
2344  	struct mutex			ctx_lock;
2345  };
2346  
2347  
2348  /*
2349   * DebugFS
2350   */
2351  
2352  /**
2353   * struct hl_info_list - debugfs file ops.
2354   * @name: file name.
2355   * @show: function to output information.
2356   * @write: function to write to the file.
2357   */
2358  struct hl_info_list {
2359  	const char	*name;
2360  	int		(*show)(struct seq_file *s, void *data);
2361  	ssize_t		(*write)(struct file *file, const char __user *buf,
2362  				size_t count, loff_t *f_pos);
2363  };
2364  
2365  /**
2366   * struct hl_debugfs_entry - debugfs dentry wrapper.
2367   * @info_ent: dentry related ops.
2368   * @dev_entry: ASIC specific debugfs manager.
2369   */
2370  struct hl_debugfs_entry {
2371  	const struct hl_info_list	*info_ent;
2372  	struct hl_dbg_device_entry	*dev_entry;
2373  };
2374  
2375  /**
2376   * struct hl_dbg_device_entry - ASIC specific debugfs manager.
2377   * @root: root dentry.
2378   * @hdev: habanalabs device structure.
2379   * @entry_arr: array of available hl_debugfs_entry.
2380   * @file_list: list of available debugfs files.
2381   * @file_mutex: protects file_list.
2382   * @cb_list: list of available CBs.
2383   * @cb_spinlock: protects cb_list.
2384   * @cs_list: list of available CSs.
2385   * @cs_spinlock: protects cs_list.
2386   * @cs_job_list: list of available CB jobs.
2387   * @cs_job_spinlock: protects cs_job_list.
2388   * @userptr_list: list of available userptrs (virtual memory chunk descriptor).
2389   * @userptr_spinlock: protects userptr_list.
2390   * @ctx_mem_hash_list: list of available contexts with MMU mappings.
2391   * @ctx_mem_hash_mutex: protects list of available contexts with MMU mappings.
2392   * @data_dma_blob_desc: data DMA descriptor of blob.
2393   * @mon_dump_blob_desc: monitor dump descriptor of blob.
2394   * @state_dump: data of the system states in case of a bad cs.
2395   * @state_dump_sem: protects state_dump.
2396   * @addr: next address to read/write from/to in read/write32.
2397   * @mmu_addr: next virtual address to translate to physical address in mmu_show.
2398   * @mmu_cap_mask: mmu hw capability mask, to be used in mmu_ack_error.
2399   * @userptr_lookup: the target user ptr to look up for on demand.
2400   * @mmu_asid: ASID to use while translating in mmu_show.
2401   * @state_dump_head: index of the latest state dump
2402   * @i2c_bus: generic u8 debugfs file for bus value to use in i2c_data_read.
2403   * @i2c_addr: generic u8 debugfs file for address value to use in i2c_data_read.
2404   * @i2c_reg: generic u8 debugfs file for register value to use in i2c_data_read.
2405   * @i2c_len: generic u8 debugfs file for length value to use in i2c_data_read.
2406   */
2407  struct hl_dbg_device_entry {
2408  	struct dentry			*root;
2409  	struct hl_device		*hdev;
2410  	struct hl_debugfs_entry		*entry_arr;
2411  	struct list_head		file_list;
2412  	struct mutex			file_mutex;
2413  	struct list_head		cb_list;
2414  	spinlock_t			cb_spinlock;
2415  	struct list_head		cs_list;
2416  	spinlock_t			cs_spinlock;
2417  	struct list_head		cs_job_list;
2418  	spinlock_t			cs_job_spinlock;
2419  	struct list_head		userptr_list;
2420  	spinlock_t			userptr_spinlock;
2421  	struct list_head		ctx_mem_hash_list;
2422  	struct mutex			ctx_mem_hash_mutex;
2423  	struct debugfs_blob_wrapper	data_dma_blob_desc;
2424  	struct debugfs_blob_wrapper	mon_dump_blob_desc;
2425  	char				*state_dump[HL_STATE_DUMP_HIST_LEN];
2426  	struct rw_semaphore		state_dump_sem;
2427  	u64				addr;
2428  	u64				mmu_addr;
2429  	u64				mmu_cap_mask;
2430  	u64				userptr_lookup;
2431  	u32				mmu_asid;
2432  	u32				state_dump_head;
2433  	u8				i2c_bus;
2434  	u8				i2c_addr;
2435  	u8				i2c_reg;
2436  	u8				i2c_len;
2437  };
2438  
2439  /**
2440   * struct hl_hw_obj_name_entry - single hw object name, member of
2441   * hl_state_dump_specs
2442   * @node: link to the containing hash table
2443   * @name: hw object name
2444   * @id: object identifier
2445   */
2446  struct hl_hw_obj_name_entry {
2447  	struct hlist_node	node;
2448  	const char		*name;
2449  	u32			id;
2450  };
2451  
2452  enum hl_state_dump_specs_props {
2453  	SP_SYNC_OBJ_BASE_ADDR,
2454  	SP_NEXT_SYNC_OBJ_ADDR,
2455  	SP_SYNC_OBJ_AMOUNT,
2456  	SP_MON_OBJ_WR_ADDR_LOW,
2457  	SP_MON_OBJ_WR_ADDR_HIGH,
2458  	SP_MON_OBJ_WR_DATA,
2459  	SP_MON_OBJ_ARM_DATA,
2460  	SP_MON_OBJ_STATUS,
2461  	SP_MONITORS_AMOUNT,
2462  	SP_TPC0_CMDQ,
2463  	SP_TPC0_CFG_SO,
2464  	SP_NEXT_TPC,
2465  	SP_MME_CMDQ,
2466  	SP_MME_CFG_SO,
2467  	SP_NEXT_MME,
2468  	SP_DMA_CMDQ,
2469  	SP_DMA_CFG_SO,
2470  	SP_DMA_QUEUES_OFFSET,
2471  	SP_NUM_OF_MME_ENGINES,
2472  	SP_SUB_MME_ENG_NUM,
2473  	SP_NUM_OF_DMA_ENGINES,
2474  	SP_NUM_OF_TPC_ENGINES,
2475  	SP_ENGINE_NUM_OF_QUEUES,
2476  	SP_ENGINE_NUM_OF_STREAMS,
2477  	SP_ENGINE_NUM_OF_FENCES,
2478  	SP_FENCE0_CNT_OFFSET,
2479  	SP_FENCE0_RDATA_OFFSET,
2480  	SP_CP_STS_OFFSET,
2481  	SP_NUM_CORES,
2482  
2483  	SP_MAX
2484  };
2485  
2486  enum hl_sync_engine_type {
2487  	ENGINE_TPC,
2488  	ENGINE_DMA,
2489  	ENGINE_MME,
2490  };
2491  
2492  /**
2493   * struct hl_mon_state_dump - represents a state dump of a single monitor
2494   * @id: monitor id
2495   * @wr_addr_low: address monitor will write to, low bits
2496   * @wr_addr_high: address monitor will write to, high bits
2497   * @wr_data: data monitor will write
2498   * @arm_data: register value containing monitor configuration
2499   * @status: monitor status
2500   */
2501  struct hl_mon_state_dump {
2502  	u32		id;
2503  	u32		wr_addr_low;
2504  	u32		wr_addr_high;
2505  	u32		wr_data;
2506  	u32		arm_data;
2507  	u32		status;
2508  };
2509  
2510  /**
2511   * struct hl_sync_to_engine_map_entry - sync object id to engine mapping entry
2512   * @engine_type: type of the engine
2513   * @engine_id: id of the engine
2514   * @sync_id: id of the sync object
2515   */
2516  struct hl_sync_to_engine_map_entry {
2517  	struct hlist_node		node;
2518  	enum hl_sync_engine_type	engine_type;
2519  	u32				engine_id;
2520  	u32				sync_id;
2521  };
2522  
2523  /**
2524   * struct hl_sync_to_engine_map - maps sync object id to associated engine id
2525   * @tb: hash table containing the mapping, each element is of type
2526   *      struct hl_sync_to_engine_map_entry
2527   */
2528  struct hl_sync_to_engine_map {
2529  	DECLARE_HASHTABLE(tb, SYNC_TO_ENGINE_HASH_TABLE_BITS);
2530  };
2531  
2532  /**
2533   * struct hl_state_dump_specs_funcs - virtual functions used by the state dump
2534   * @gen_sync_to_engine_map: generate a hash map from sync obj id to its engine
2535   * @print_single_monitor: format monitor data as string
2536   * @monitor_valid: return true if given monitor dump is valid
2537   * @print_fences_single_engine: format fences data as string
2538   */
2539  struct hl_state_dump_specs_funcs {
2540  	int (*gen_sync_to_engine_map)(struct hl_device *hdev,
2541  				struct hl_sync_to_engine_map *map);
2542  	int (*print_single_monitor)(char **buf, size_t *size, size_t *offset,
2543  				    struct hl_device *hdev,
2544  				    struct hl_mon_state_dump *mon);
2545  	int (*monitor_valid)(struct hl_mon_state_dump *mon);
2546  	int (*print_fences_single_engine)(struct hl_device *hdev,
2547  					u64 base_offset,
2548  					u64 status_base_offset,
2549  					enum hl_sync_engine_type engine_type,
2550  					u32 engine_id, char **buf,
2551  					size_t *size, size_t *offset);
2552  };
2553  
2554  /**
2555   * struct hl_state_dump_specs - defines ASIC known hw objects names
2556   * @so_id_to_str_tb: sync objects names index table
2557   * @monitor_id_to_str_tb: monitors names index table
2558   * @funcs: virtual functions used for state dump
2559   * @sync_namager_names: readable names for sync manager if available (ex: N_E)
2560   * @props: pointer to a per asic const props array required for state dump
2561   */
2562  struct hl_state_dump_specs {
2563  	DECLARE_HASHTABLE(so_id_to_str_tb, OBJ_NAMES_HASH_TABLE_BITS);
2564  	DECLARE_HASHTABLE(monitor_id_to_str_tb, OBJ_NAMES_HASH_TABLE_BITS);
2565  	struct hl_state_dump_specs_funcs	funcs;
2566  	const char * const			*sync_namager_names;
2567  	s64					*props;
2568  };
2569  
2570  
2571  /*
2572   * DEVICES
2573   */
2574  
2575  #define HL_STR_MAX	64
2576  
2577  #define HL_DEV_STS_MAX (HL_DEVICE_STATUS_LAST + 1)
2578  
2579  /* Theoretical limit only. A single host can only contain up to 4 or 8 PCIe
2580   * x16 cards. In extreme cases, there are hosts that can accommodate 16 cards.
2581   */
2582  #define HL_MAX_MINORS	256
2583  
2584  /*
2585   * Registers read & write functions.
2586   */
2587  
2588  u32 hl_rreg(struct hl_device *hdev, u32 reg);
2589  void hl_wreg(struct hl_device *hdev, u32 reg, u32 val);
2590  
2591  #define RREG32(reg) hdev->asic_funcs->rreg(hdev, (reg))
2592  #define WREG32(reg, v) hdev->asic_funcs->wreg(hdev, (reg), (v))
2593  #define DREG32(reg) pr_info("REGISTER: " #reg " : 0x%08X\n",	\
2594  			hdev->asic_funcs->rreg(hdev, (reg)))
2595  
2596  #define WREG32_P(reg, val, mask)				\
2597  	do {							\
2598  		u32 tmp_ = RREG32(reg);				\
2599  		tmp_ &= (mask);					\
2600  		tmp_ |= ((val) & ~(mask));			\
2601  		WREG32(reg, tmp_);				\
2602  	} while (0)
2603  #define WREG32_AND(reg, and) WREG32_P(reg, 0, and)
2604  #define WREG32_OR(reg, or) WREG32_P(reg, or, ~(or))
2605  
2606  #define RMWREG32_SHIFTED(reg, val, mask) WREG32_P(reg, val, ~(mask))
2607  
2608  #define RMWREG32(reg, val, mask) RMWREG32_SHIFTED(reg, (val) << __ffs(mask), mask)
2609  
2610  #define RREG32_MASK(reg, mask) ((RREG32(reg) & mask) >> __ffs(mask))
2611  
2612  #define REG_FIELD_SHIFT(reg, field) reg##_##field##_SHIFT
2613  #define REG_FIELD_MASK(reg, field) reg##_##field##_MASK
2614  #define WREG32_FIELD(reg, offset, field, val)	\
2615  	WREG32(mm##reg + offset, (RREG32(mm##reg + offset) & \
2616  				~REG_FIELD_MASK(reg, field)) | \
2617  				(val) << REG_FIELD_SHIFT(reg, field))
2618  
2619  /* Timeout should be longer when working with simulator but cap the
2620   * increased timeout to some maximum
2621   */
2622  #define hl_poll_timeout_common(hdev, addr, val, cond, sleep_us, timeout_us, elbi) \
2623  ({ \
2624  	ktime_t __timeout; \
2625  	u32 __elbi_read; \
2626  	int __rc = 0; \
2627  	__timeout = ktime_add_us(ktime_get(), timeout_us); \
2628  	might_sleep_if(sleep_us); \
2629  	for (;;) { \
2630  		if (elbi) { \
2631  			__rc = hl_pci_elbi_read(hdev, addr, &__elbi_read); \
2632  			if (__rc) \
2633  				break; \
2634  			(val) = __elbi_read; \
2635  		} else {\
2636  			(val) = RREG32(lower_32_bits(addr)); \
2637  		} \
2638  		if (cond) \
2639  			break; \
2640  		if (timeout_us && ktime_compare(ktime_get(), __timeout) > 0) { \
2641  			if (elbi) { \
2642  				__rc = hl_pci_elbi_read(hdev, addr, &__elbi_read); \
2643  				if (__rc) \
2644  					break; \
2645  				(val) = __elbi_read; \
2646  			} else {\
2647  				(val) = RREG32(lower_32_bits(addr)); \
2648  			} \
2649  			break; \
2650  		} \
2651  		if (sleep_us) \
2652  			usleep_range((sleep_us >> 2) + 1, sleep_us); \
2653  	} \
2654  	__rc ? __rc : ((cond) ? 0 : -ETIMEDOUT); \
2655  })
2656  
2657  #define hl_poll_timeout(hdev, addr, val, cond, sleep_us, timeout_us) \
2658  		hl_poll_timeout_common(hdev, addr, val, cond, sleep_us, timeout_us, false)
2659  
2660  #define hl_poll_timeout_elbi(hdev, addr, val, cond, sleep_us, timeout_us) \
2661  		hl_poll_timeout_common(hdev, addr, val, cond, sleep_us, timeout_us, true)
2662  
2663  /*
2664   * poll array of register addresses.
2665   * condition is satisfied if all registers values match the expected value.
2666   * once some register in the array satisfies the condition it will not be polled again,
2667   * this is done both for efficiency and due to some registers are "clear on read".
2668   * TODO: use read from PCI bar in other places in the code (SW-91406)
2669   */
2670  #define hl_poll_reg_array_timeout_common(hdev, addr_arr, arr_size, expected_val, sleep_us, \
2671  						timeout_us, elbi) \
2672  ({ \
2673  	ktime_t __timeout; \
2674  	u64 __elem_bitmask; \
2675  	u32 __read_val;	\
2676  	u8 __arr_idx;	\
2677  	int __rc = 0; \
2678  	\
2679  	__timeout = ktime_add_us(ktime_get(), timeout_us); \
2680  	might_sleep_if(sleep_us); \
2681  	if (arr_size >= 64) \
2682  		__rc = -EINVAL; \
2683  	else \
2684  		__elem_bitmask = BIT_ULL(arr_size) - 1; \
2685  	for (;;) { \
2686  		if (__rc) \
2687  			break; \
2688  		for (__arr_idx = 0; __arr_idx < (arr_size); __arr_idx++) {	\
2689  			if (!(__elem_bitmask & BIT_ULL(__arr_idx)))	\
2690  				continue;	\
2691  			if (elbi) { \
2692  				__rc = hl_pci_elbi_read(hdev, (addr_arr)[__arr_idx], &__read_val); \
2693  				if (__rc) \
2694  					break; \
2695  			} else { \
2696  				__read_val = RREG32(lower_32_bits(addr_arr[__arr_idx])); \
2697  			} \
2698  			if (__read_val == (expected_val))	\
2699  				__elem_bitmask &= ~BIT_ULL(__arr_idx);	\
2700  		}	\
2701  		if (__rc || (__elem_bitmask == 0)) \
2702  			break; \
2703  		if (timeout_us && ktime_compare(ktime_get(), __timeout) > 0) \
2704  			break; \
2705  		if (sleep_us) \
2706  			usleep_range((sleep_us >> 2) + 1, sleep_us); \
2707  	} \
2708  	__rc ? __rc : ((__elem_bitmask == 0) ? 0 : -ETIMEDOUT); \
2709  })
2710  
2711  #define hl_poll_reg_array_timeout(hdev, addr_arr, arr_size, expected_val, sleep_us, \
2712  					timeout_us) \
2713  	hl_poll_reg_array_timeout_common(hdev, addr_arr, arr_size, expected_val, sleep_us, \
2714  						timeout_us, false)
2715  
2716  #define hl_poll_reg_array_timeout_elbi(hdev, addr_arr, arr_size, expected_val, sleep_us, \
2717  					timeout_us) \
2718  	hl_poll_reg_array_timeout_common(hdev, addr_arr, arr_size, expected_val, sleep_us, \
2719  						timeout_us, true)
2720  
2721  /*
2722   * address in this macro points always to a memory location in the
2723   * host's (server's) memory. That location is updated asynchronously
2724   * either by the direct access of the device or by another core.
2725   *
2726   * To work both in LE and BE architectures, we need to distinguish between the
2727   * two states (device or another core updates the memory location). Therefore,
2728   * if mem_written_by_device is true, the host memory being polled will be
2729   * updated directly by the device. If false, the host memory being polled will
2730   * be updated by host CPU. Required so host knows whether or not the memory
2731   * might need to be byte-swapped before returning value to caller.
2732   *
2733   * On the first 4 polling iterations the macro goes to sleep for short period of
2734   * time that gradually increases and reaches sleep_us on the fifth iteration.
2735   */
2736  #define hl_poll_timeout_memory(hdev, addr, val, cond, sleep_us, timeout_us, \
2737  				mem_written_by_device) \
2738  ({ \
2739  	u64 __sleep_step_us; \
2740  	ktime_t __timeout; \
2741  	u8 __step = 8; \
2742  	\
2743  	__timeout = ktime_add_us(ktime_get(), timeout_us); \
2744  	might_sleep_if(sleep_us); \
2745  	for (;;) { \
2746  		/* Verify we read updates done by other cores or by device */ \
2747  		mb(); \
2748  		(val) = *((u32 *)(addr)); \
2749  		if (mem_written_by_device) \
2750  			(val) = le32_to_cpu(*(__le32 *) &(val)); \
2751  		if (cond) \
2752  			break; \
2753  		if (timeout_us && ktime_compare(ktime_get(), __timeout) > 0) { \
2754  			(val) = *((u32 *)(addr)); \
2755  			if (mem_written_by_device) \
2756  				(val) = le32_to_cpu(*(__le32 *) &(val)); \
2757  			break; \
2758  		} \
2759  		__sleep_step_us = sleep_us >> __step; \
2760  		if (__sleep_step_us) \
2761  			usleep_range((__sleep_step_us >> 2) + 1, __sleep_step_us); \
2762  		__step >>= 1; \
2763  	} \
2764  	(cond) ? 0 : -ETIMEDOUT; \
2765  })
2766  
2767  #define HL_USR_MAPPED_BLK_INIT(blk, base, sz) \
2768  ({ \
2769  	struct user_mapped_block *p = blk; \
2770  \
2771  	p->address = base; \
2772  	p->size = sz; \
2773  })
2774  
2775  #define HL_USR_INTR_STRUCT_INIT(usr_intr, hdev, intr_id, intr_type) \
2776  ({ \
2777  	usr_intr.hdev = hdev; \
2778  	usr_intr.interrupt_id = intr_id; \
2779  	usr_intr.type = intr_type; \
2780  	INIT_LIST_HEAD(&usr_intr.wait_list_head); \
2781  	spin_lock_init(&usr_intr.wait_list_lock); \
2782  	INIT_LIST_HEAD(&usr_intr.ts_list_head); \
2783  	spin_lock_init(&usr_intr.ts_list_lock); \
2784  })
2785  
2786  struct hwmon_chip_info;
2787  
2788  /**
2789   * struct hl_device_reset_work - reset work wrapper.
2790   * @reset_work: reset work to be done.
2791   * @hdev: habanalabs device structure.
2792   * @flags: reset flags.
2793   */
2794  struct hl_device_reset_work {
2795  	struct delayed_work	reset_work;
2796  	struct hl_device	*hdev;
2797  	u32			flags;
2798  };
2799  
2800  /**
2801   * struct hl_mmu_hr_pgt_priv - used for holding per-device mmu host-resident
2802   * page-table internal information.
2803   * @mmu_pgt_pool: pool of page tables used by a host-resident MMU for
2804   *                allocating hops.
2805   * @mmu_asid_hop0: per-ASID array of host-resident hop0 tables.
2806   */
2807  struct hl_mmu_hr_priv {
2808  	struct gen_pool	*mmu_pgt_pool;
2809  	struct pgt_info	*mmu_asid_hop0;
2810  };
2811  
2812  /**
2813   * struct hl_mmu_dr_pgt_priv - used for holding per-device mmu device-resident
2814   * page-table internal information.
2815   * @mmu_pgt_pool: pool of page tables used by MMU for allocating hops.
2816   * @mmu_shadow_hop0: shadow array of hop0 tables.
2817   */
2818  struct hl_mmu_dr_priv {
2819  	struct gen_pool *mmu_pgt_pool;
2820  	void *mmu_shadow_hop0;
2821  };
2822  
2823  /**
2824   * struct hl_mmu_priv - used for holding per-device mmu internal information.
2825   * @dr: information on the device-resident MMU, when exists.
2826   * @hr: information on the host-resident MMU, when exists.
2827   */
2828  struct hl_mmu_priv {
2829  	struct hl_mmu_dr_priv dr;
2830  	struct hl_mmu_hr_priv hr;
2831  };
2832  
2833  /**
2834   * struct hl_mmu_per_hop_info - A structure describing one TLB HOP and its entry
2835   *                that was created in order to translate a virtual address to a
2836   *                physical one.
2837   * @hop_addr: The address of the hop.
2838   * @hop_pte_addr: The address of the hop entry.
2839   * @hop_pte_val: The value in the hop entry.
2840   */
2841  struct hl_mmu_per_hop_info {
2842  	u64 hop_addr;
2843  	u64 hop_pte_addr;
2844  	u64 hop_pte_val;
2845  };
2846  
2847  /**
2848   * struct hl_mmu_hop_info - A structure describing the TLB hops and their
2849   * hop-entries that were created in order to translate a virtual address to a
2850   * physical one.
2851   * @scrambled_vaddr: The value of the virtual address after scrambling. This
2852   *                   address replaces the original virtual-address when mapped
2853   *                   in the MMU tables.
2854   * @unscrambled_paddr: The un-scrambled physical address.
2855   * @hop_info: Array holding the per-hop information used for the translation.
2856   * @used_hops: The number of hops used for the translation.
2857   * @range_type: virtual address range type.
2858   */
2859  struct hl_mmu_hop_info {
2860  	u64 scrambled_vaddr;
2861  	u64 unscrambled_paddr;
2862  	struct hl_mmu_per_hop_info hop_info[MMU_ARCH_6_HOPS];
2863  	u32 used_hops;
2864  	enum hl_va_range_type range_type;
2865  };
2866  
2867  /**
2868   * struct hl_hr_mmu_funcs - Device related host resident MMU functions.
2869   * @get_hop0_pgt_info: get page table info structure for HOP0.
2870   * @get_pgt_info: get page table info structure for HOP other than HOP0.
2871   * @add_pgt_info: add page table info structure to hash.
2872   * @get_tlb_mapping_params: get mapping parameters needed for getting TLB info for specific mapping.
2873   */
2874  struct hl_hr_mmu_funcs {
2875  	struct pgt_info *(*get_hop0_pgt_info)(struct hl_ctx *ctx);
2876  	struct pgt_info *(*get_pgt_info)(struct hl_ctx *ctx, u64 phys_hop_addr);
2877  	void (*add_pgt_info)(struct hl_ctx *ctx, struct pgt_info *pgt_info, dma_addr_t phys_addr);
2878  	int (*get_tlb_mapping_params)(struct hl_device *hdev, struct hl_mmu_properties **mmu_prop,
2879  								struct hl_mmu_hop_info *hops,
2880  								u64 virt_addr, bool *is_huge);
2881  };
2882  
2883  /**
2884   * struct hl_mmu_funcs - Device related MMU functions.
2885   * @init: initialize the MMU module.
2886   * @fini: release the MMU module.
2887   * @ctx_init: Initialize a context for using the MMU module.
2888   * @ctx_fini: disable a ctx from using the mmu module.
2889   * @map: maps a virtual address to physical address for a context.
2890   * @unmap: unmap a virtual address of a context.
2891   * @flush: flush all writes from all cores to reach device MMU.
2892   * @swap_out: marks all mapping of the given context as swapped out.
2893   * @swap_in: marks all mapping of the given context as swapped in.
2894   * @get_tlb_info: returns the list of hops and hop-entries used that were
2895   *                created in order to translate the giver virtual address to a
2896   *                physical one.
2897   * @hr_funcs: functions specific to host resident MMU.
2898   */
2899  struct hl_mmu_funcs {
2900  	int (*init)(struct hl_device *hdev);
2901  	void (*fini)(struct hl_device *hdev);
2902  	int (*ctx_init)(struct hl_ctx *ctx);
2903  	void (*ctx_fini)(struct hl_ctx *ctx);
2904  	int (*map)(struct hl_ctx *ctx, u64 virt_addr, u64 phys_addr, u32 page_size,
2905  				bool is_dram_addr);
2906  	int (*unmap)(struct hl_ctx *ctx, u64 virt_addr, bool is_dram_addr);
2907  	void (*flush)(struct hl_ctx *ctx);
2908  	void (*swap_out)(struct hl_ctx *ctx);
2909  	void (*swap_in)(struct hl_ctx *ctx);
2910  	int (*get_tlb_info)(struct hl_ctx *ctx, u64 virt_addr, struct hl_mmu_hop_info *hops);
2911  	struct hl_hr_mmu_funcs hr_funcs;
2912  };
2913  
2914  /**
2915   * struct hl_prefetch_work - prefetch work structure handler
2916   * @prefetch_work: actual work struct.
2917   * @ctx: compute context.
2918   * @va: virtual address to pre-fetch.
2919   * @size: pre-fetch size.
2920   * @flags: operation flags.
2921   * @asid: ASID for maintenance operation.
2922   */
2923  struct hl_prefetch_work {
2924  	struct work_struct	prefetch_work;
2925  	struct hl_ctx		*ctx;
2926  	u64			va;
2927  	u64			size;
2928  	u32			flags;
2929  	u32			asid;
2930  };
2931  
2932  /*
2933   * number of user contexts allowed to call wait_for_multi_cs ioctl in
2934   * parallel
2935   */
2936  #define MULTI_CS_MAX_USER_CTX	2
2937  
2938  /**
2939   * struct multi_cs_completion - multi CS wait completion.
2940   * @completion: completion of any of the CS in the list
2941   * @lock: spinlock for the completion structure
2942   * @timestamp: timestamp for the multi-CS completion
2943   * @stream_master_qid_map: bitmap of all stream masters on which the multi-CS
2944   *                        is waiting
2945   * @used: 1 if in use, otherwise 0
2946   */
2947  struct multi_cs_completion {
2948  	struct completion	completion;
2949  	spinlock_t		lock;
2950  	s64			timestamp;
2951  	u32			stream_master_qid_map;
2952  	u8			used;
2953  };
2954  
2955  /**
2956   * struct multi_cs_data - internal data for multi CS call
2957   * @ctx: pointer to the context structure
2958   * @fence_arr: array of fences of all CSs
2959   * @seq_arr: array of CS sequence numbers
2960   * @timeout_jiffies: timeout in jiffies for waiting for CS to complete
2961   * @timestamp: timestamp of first completed CS
2962   * @wait_status: wait for CS status
2963   * @completion_bitmap: bitmap of completed CSs (1- completed, otherwise 0)
2964   * @arr_len: fence_arr and seq_arr array length
2965   * @gone_cs: indication of gone CS (1- there was gone CS, otherwise 0)
2966   * @update_ts: update timestamp. 1- update the timestamp, otherwise 0.
2967   */
2968  struct multi_cs_data {
2969  	struct hl_ctx	*ctx;
2970  	struct hl_fence	**fence_arr;
2971  	u64		*seq_arr;
2972  	s64		timeout_jiffies;
2973  	s64		timestamp;
2974  	long		wait_status;
2975  	u32		completion_bitmap;
2976  	u8		arr_len;
2977  	u8		gone_cs;
2978  	u8		update_ts;
2979  };
2980  
2981  /**
2982   * struct hl_clk_throttle_timestamp - current/last clock throttling timestamp
2983   * @start: timestamp taken when 'start' event is received in driver
2984   * @end: timestamp taken when 'end' event is received in driver
2985   */
2986  struct hl_clk_throttle_timestamp {
2987  	ktime_t		start;
2988  	ktime_t		end;
2989  };
2990  
2991  /**
2992   * struct hl_clk_throttle - keeps current/last clock throttling timestamps
2993   * @timestamp: timestamp taken by driver and firmware, index 0 refers to POWER
2994   *             index 1 refers to THERMAL
2995   * @lock: protects this structure as it can be accessed from both event queue
2996   *        context and info_ioctl context
2997   * @current_reason: bitmask represents the current clk throttling reasons
2998   * @aggregated_reason: bitmask represents aggregated clk throttling reasons since driver load
2999   */
3000  struct hl_clk_throttle {
3001  	struct hl_clk_throttle_timestamp timestamp[HL_CLK_THROTTLE_TYPE_MAX];
3002  	struct mutex	lock;
3003  	u32		current_reason;
3004  	u32		aggregated_reason;
3005  };
3006  
3007  /**
3008   * struct user_mapped_block - describes a hw block allowed to be mmapped by user
3009   * @address: physical HW block address
3010   * @size: allowed size for mmap
3011   */
3012  struct user_mapped_block {
3013  	u32 address;
3014  	u32 size;
3015  };
3016  
3017  /**
3018   * struct cs_timeout_info - info of last CS timeout occurred.
3019   * @timestamp: CS timeout timestamp.
3020   * @write_enable: if set writing to CS parameters in the structure is enabled. otherwise - disabled,
3021   *                so the first (root cause) CS timeout will not be overwritten.
3022   * @seq: CS timeout sequence number.
3023   */
3024  struct cs_timeout_info {
3025  	ktime_t		timestamp;
3026  	atomic_t	write_enable;
3027  	u64		seq;
3028  };
3029  
3030  #define MAX_QMAN_STREAMS_INFO		4
3031  #define OPCODE_INFO_MAX_ADDR_SIZE	8
3032  /**
3033   * struct undefined_opcode_info - info about last undefined opcode error
3034   * @timestamp: timestamp of the undefined opcode error
3035   * @cb_addr_streams: CB addresses (per stream) that are currently exists in the PQ
3036   *                   entries. In case all streams array entries are
3037   *                   filled with values, it means the execution was in Lower-CP.
3038   * @cq_addr: the address of the current handled command buffer
3039   * @cq_size: the size of the current handled command buffer
3040   * @cb_addr_streams_len: num of streams - actual len of cb_addr_streams array.
3041   *                       should be equal to 1 in case of undefined opcode
3042   *                       in Upper-CP (specific stream) and equal to 4 in case
3043   *                       of undefined opcode in Lower-CP.
3044   * @engine_id: engine-id that the error occurred on
3045   * @stream_id: the stream id the error occurred on. In case the stream equals to
3046   *             MAX_QMAN_STREAMS_INFO it means the error occurred on a Lower-CP.
3047   * @write_enable: if set, writing to undefined opcode parameters in the structure
3048   *                 is enable so the first (root cause) undefined opcode will not be
3049   *                 overwritten.
3050   */
3051  struct undefined_opcode_info {
3052  	ktime_t timestamp;
3053  	u64 cb_addr_streams[MAX_QMAN_STREAMS_INFO][OPCODE_INFO_MAX_ADDR_SIZE];
3054  	u64 cq_addr;
3055  	u32 cq_size;
3056  	u32 cb_addr_streams_len;
3057  	u32 engine_id;
3058  	u32 stream_id;
3059  	bool write_enable;
3060  };
3061  
3062  /**
3063   * struct page_fault_info - page fault information.
3064   * @page_fault: holds information collected during a page fault.
3065   * @user_mappings: buffer containing user mappings.
3066   * @num_of_user_mappings: number of user mappings.
3067   * @page_fault_detected: if set as 1, then a page-fault was discovered for the
3068   *                       first time after the driver has finished booting-up.
3069   *                       Since we're looking for the page-fault's root cause,
3070   *                       we don't care of the others that might follow it-
3071   *                       so once changed to 1, it will remain that way.
3072   * @page_fault_info_available: indicates that a page fault info is now available.
3073   */
3074  struct page_fault_info {
3075  	struct hl_page_fault_info	page_fault;
3076  	struct hl_user_mapping		*user_mappings;
3077  	u64				num_of_user_mappings;
3078  	atomic_t			page_fault_detected;
3079  	bool				page_fault_info_available;
3080  };
3081  
3082  /**
3083   * struct razwi_info - RAZWI information.
3084   * @razwi: holds information collected during a RAZWI
3085   * @razwi_detected: if set as 1, then a RAZWI was discovered for the
3086   *                  first time after the driver has finished booting-up.
3087   *                  Since we're looking for the RAZWI's root cause,
3088   *                  we don't care of the others that might follow it-
3089   *                  so once changed to 1, it will remain that way.
3090   * @razwi_info_available: indicates that a RAZWI info is now available.
3091   */
3092  struct razwi_info {
3093  	struct hl_info_razwi_event	razwi;
3094  	atomic_t			razwi_detected;
3095  	bool				razwi_info_available;
3096  };
3097  
3098  /**
3099   * struct hw_err_info - HW error information.
3100   * @event: holds information on the event.
3101   * @event_detected: if set as 1, then a HW event was discovered for the
3102   *                  first time after the driver has finished booting-up.
3103   *                  currently we assume that only fatal events (that require hard-reset) are
3104   *                  reported so we don't care of the others that might follow it.
3105   *                  so once changed to 1, it will remain that way.
3106   *                  TODO: support multiple events.
3107   * @event_info_available: indicates that a HW event info is now available.
3108   */
3109  struct hw_err_info {
3110  	struct hl_info_hw_err_event	event;
3111  	atomic_t			event_detected;
3112  	bool				event_info_available;
3113  };
3114  
3115  /**
3116   * struct fw_err_info - FW error information.
3117   * @event: holds information on the event.
3118   * @event_detected: if set as 1, then a FW event was discovered for the
3119   *                  first time after the driver has finished booting-up.
3120   *                  currently we assume that only fatal events (that require hard-reset) are
3121   *                  reported so we don't care of the others that might follow it.
3122   *                  so once changed to 1, it will remain that way.
3123   *                  TODO: support multiple events.
3124   * @event_info_available: indicates that a HW event info is now available.
3125   */
3126  struct fw_err_info {
3127  	struct hl_info_fw_err_event	event;
3128  	atomic_t			event_detected;
3129  	bool				event_info_available;
3130  };
3131  
3132  /**
3133   * struct engine_err_info - engine error information.
3134   * @event: holds information on the event.
3135   * @event_detected: if set as 1, then an engine event was discovered for the
3136   *                  first time after the driver has finished booting-up.
3137   * @event_info_available: indicates that an engine event info is now available.
3138   */
3139  struct engine_err_info {
3140  	struct hl_info_engine_err_event	event;
3141  	atomic_t			event_detected;
3142  	bool				event_info_available;
3143  };
3144  
3145  
3146  /**
3147   * struct hl_error_info - holds information collected during an error.
3148   * @cs_timeout: CS timeout error information.
3149   * @razwi_info: RAZWI information.
3150   * @undef_opcode: undefined opcode information.
3151   * @page_fault_info: page fault information.
3152   * @hw_err: (fatal) hardware error information.
3153   * @fw_err: firmware error information.
3154   * @engine_err: engine error information.
3155   */
3156  struct hl_error_info {
3157  	struct cs_timeout_info		cs_timeout;
3158  	struct razwi_info		razwi_info;
3159  	struct undefined_opcode_info	undef_opcode;
3160  	struct page_fault_info		page_fault_info;
3161  	struct hw_err_info		hw_err;
3162  	struct fw_err_info		fw_err;
3163  	struct engine_err_info		engine_err;
3164  };
3165  
3166  /**
3167   * struct hl_reset_info - holds current device reset information.
3168   * @lock: lock to protect critical reset flows.
3169   * @compute_reset_cnt: number of compute resets since the driver was loaded.
3170   * @hard_reset_cnt: number of hard resets since the driver was loaded.
3171   * @hard_reset_schedule_flags: hard reset is scheduled to after current compute reset,
3172   *                             here we hold the hard reset flags.
3173   * @in_reset: is device in reset flow.
3174   * @in_compute_reset: Device is currently in reset but not in hard-reset.
3175   * @needs_reset: true if reset_on_lockup is false and device should be reset
3176   *               due to lockup.
3177   * @hard_reset_pending: is there a hard reset work pending.
3178   * @curr_reset_cause: saves an enumerated reset cause when a hard reset is
3179   *                    triggered, and cleared after it is shared with preboot.
3180   * @prev_reset_trigger: saves the previous trigger which caused a reset, overridden
3181   *                      with a new value on next reset
3182   * @reset_trigger_repeated: set if device reset is triggered more than once with
3183   *                          same cause.
3184   * @skip_reset_on_timeout: Skip device reset if CS has timed out, wait for it to
3185   *                         complete instead.
3186   * @watchdog_active: true if a device release watchdog work is scheduled.
3187   */
3188  struct hl_reset_info {
3189  	spinlock_t	lock;
3190  	u32		compute_reset_cnt;
3191  	u32		hard_reset_cnt;
3192  	u32		hard_reset_schedule_flags;
3193  	u8		in_reset;
3194  	u8		in_compute_reset;
3195  	u8		needs_reset;
3196  	u8		hard_reset_pending;
3197  	u8		curr_reset_cause;
3198  	u8		prev_reset_trigger;
3199  	u8		reset_trigger_repeated;
3200  	u8		skip_reset_on_timeout;
3201  	u8		watchdog_active;
3202  };
3203  
3204  /**
3205   * struct eq_heartbeat_debug_info - stores debug info to be used upon heartbeat failure.
3206   * @last_pq_heartbeat_ts: timestamp of the last test packet that was sent to FW.
3207   *                        This packet is the trigger in FW to send the EQ heartbeat event.
3208   * @last_eq_heartbeat_ts: timestamp of the last EQ heartbeat event that was received from FW.
3209   * @heartbeat_event_counter: number of heartbeat events received.
3210   * @cpu_queue_id: used to read the queue pi/ci
3211   */
3212  struct eq_heartbeat_debug_info {
3213  	time64_t last_pq_heartbeat_ts;
3214  	time64_t last_eq_heartbeat_ts;
3215  	u32 heartbeat_event_counter;
3216  	u32 cpu_queue_id;
3217  };
3218  
3219  /**
3220   * struct hl_device - habanalabs device structure.
3221   * @pdev: pointer to PCI device, can be NULL in case of simulator device.
3222   * @pcie_bar_phys: array of available PCIe bars physical addresses.
3223   *		   (required only for PCI address match mode)
3224   * @pcie_bar: array of available PCIe bars virtual addresses.
3225   * @rmmio: configuration area address on SRAM.
3226   * @drm: related DRM device.
3227   * @cdev_ctrl: char device for control operations only (INFO IOCTL)
3228   * @dev: related kernel basic device structure.
3229   * @dev_ctrl: related kernel device structure for the control device
3230   * @work_heartbeat: delayed work for CPU-CP is-alive check.
3231   * @device_reset_work: delayed work which performs hard reset
3232   * @device_release_watchdog_work: watchdog work that performs hard reset if user doesn't release
3233   *                                device upon certain error cases.
3234   * @asic_name: ASIC specific name.
3235   * @asic_type: ASIC specific type.
3236   * @completion_queue: array of hl_cq.
3237   * @user_interrupt: array of hl_user_interrupt. upon the corresponding user
3238   *                  interrupt, driver will monitor the list of fences
3239   *                  registered to this interrupt.
3240   * @tpc_interrupt: single TPC interrupt for all TPCs.
3241   * @unexpected_error_interrupt: single interrupt for unexpected user error indication.
3242   * @common_user_cq_interrupt: common user CQ interrupt for all user CQ interrupts.
3243   *                         upon any user CQ interrupt, driver will monitor the
3244   *                         list of fences registered to this common structure.
3245   * @common_decoder_interrupt: common decoder interrupt for all user decoder interrupts.
3246   * @shadow_cs_queue: pointer to a shadow queue that holds pointers to
3247   *                   outstanding command submissions.
3248   * @cq_wq: work queues of completion queues for executing work in process
3249   *         context.
3250   * @eq_wq: work queue of event queue for executing work in process context.
3251   * @cs_cmplt_wq: work queue of CS completions for executing work in process
3252   *               context.
3253   * @ts_free_obj_wq: work queue for timestamp registration objects release.
3254   * @prefetch_wq: work queue for MMU pre-fetch operations.
3255   * @reset_wq: work queue for device reset procedure.
3256   * @kernel_ctx: Kernel driver context structure.
3257   * @kernel_queues: array of hl_hw_queue.
3258   * @cs_mirror_list: CS mirror list for TDR.
3259   * @cs_mirror_lock: protects cs_mirror_list.
3260   * @kernel_mem_mgr: memory manager for memory buffers with lifespan of driver.
3261   * @event_queue: event queue for IRQ from CPU-CP.
3262   * @dma_pool: DMA pool for small allocations.
3263   * @cpu_accessible_dma_mem: Host <-> CPU-CP shared memory CPU address.
3264   * @cpu_accessible_dma_address: Host <-> CPU-CP shared memory DMA address.
3265   * @cpu_accessible_dma_pool: Host <-> CPU-CP shared memory pool.
3266   * @asid_bitmap: holds used/available ASIDs.
3267   * @asid_mutex: protects asid_bitmap.
3268   * @send_cpu_message_lock: enforces only one message in Host <-> CPU-CP queue.
3269   * @debug_lock: protects critical section of setting debug mode for device
3270   * @mmu_lock: protects the MMU page tables and invalidation h/w. Although the
3271   *            page tables are per context, the invalidation h/w is per MMU.
3272   *            Therefore, we can't allow multiple contexts (we only have two,
3273   *            user and kernel) to access the invalidation h/w at the same time.
3274   *            In addition, any change to the PGT, modifying the MMU hash or
3275   *            walking the PGT requires talking this lock.
3276   * @asic_prop: ASIC specific immutable properties.
3277   * @asic_funcs: ASIC specific functions.
3278   * @asic_specific: ASIC specific information to use only from ASIC files.
3279   * @vm: virtual memory manager for MMU.
3280   * @hwmon_dev: H/W monitor device.
3281   * @hl_chip_info: ASIC's sensors information.
3282   * @device_status_description: device status description.
3283   * @hl_debugfs: device's debugfs manager.
3284   * @cb_pool: list of pre allocated CBs.
3285   * @cb_pool_lock: protects the CB pool.
3286   * @internal_cb_pool_virt_addr: internal command buffer pool virtual address.
3287   * @internal_cb_pool_dma_addr: internal command buffer pool dma address.
3288   * @internal_cb_pool: internal command buffer memory pool.
3289   * @internal_cb_va_base: internal cb pool mmu virtual address base
3290   * @fpriv_list: list of file private data structures. Each structure is created
3291   *              when a user opens the device
3292   * @fpriv_ctrl_list: list of file private data structures. Each structure is created
3293   *              when a user opens the control device
3294   * @fpriv_list_lock: protects the fpriv_list
3295   * @fpriv_ctrl_list_lock: protects the fpriv_ctrl_list
3296   * @aggregated_cs_counters: aggregated cs counters among all contexts
3297   * @mmu_priv: device-specific MMU data.
3298   * @mmu_func: device-related MMU functions.
3299   * @dec: list of decoder sw instance
3300   * @fw_loader: FW loader manager.
3301   * @pci_mem_region: array of memory regions in the PCI
3302   * @state_dump_specs: constants and dictionaries needed to dump system state.
3303   * @multi_cs_completion: array of multi-CS completion.
3304   * @clk_throttling: holds information about current/previous clock throttling events
3305   * @captured_err_info: holds information about errors.
3306   * @reset_info: holds current device reset information.
3307   * @heartbeat_debug_info: counters used to debug heartbeat failures.
3308   * @irq_affinity_mask: mask of available CPU cores for user and decoder interrupt handling.
3309   * @stream_master_qid_arr: pointer to array with QIDs of master streams.
3310   * @fw_inner_major_ver: the major of current loaded preboot inner version.
3311   * @fw_inner_minor_ver: the minor of current loaded preboot inner version.
3312   * @fw_sw_major_ver: the major of current loaded preboot SW version.
3313   * @fw_sw_minor_ver: the minor of current loaded preboot SW version.
3314   * @fw_sw_sub_minor_ver: the sub-minor of current loaded preboot SW version.
3315   * @dram_used_mem: current DRAM memory consumption.
3316   * @memory_scrub_val: the value to which the dram will be scrubbed to using cb scrub_device_dram
3317   * @timeout_jiffies: device CS timeout value.
3318   * @max_power: the max power of the device, as configured by the sysadmin. This
3319   *             value is saved so in case of hard-reset, the driver will restore
3320   *             this value and update the F/W after the re-initialization
3321   * @boot_error_status_mask: contains a mask of the device boot error status.
3322   *                          Each bit represents a different error, according to
3323   *                          the defines in hl_boot_if.h. If the bit is cleared,
3324   *                          the error will be ignored by the driver during
3325   *                          device initialization. Mainly used to debug and
3326   *                          workaround firmware bugs
3327   * @dram_pci_bar_start: start bus address of PCIe bar towards DRAM.
3328   * @last_successful_open_ktime: timestamp (ktime) of the last successful device open.
3329   * @last_successful_open_jif: timestamp (jiffies) of the last successful
3330   *                            device open.
3331   * @last_open_session_duration_jif: duration (jiffies) of the last device open
3332   *                                  session.
3333   * @open_counter: number of successful device open operations.
3334   * @fw_poll_interval_usec: FW status poll interval in usec.
3335   *                         used for CPU boot status
3336   * @fw_comms_poll_interval_usec: FW comms/protocol poll interval in usec.
3337   *                                  used for COMMs protocols cmds(COMMS_STS_*)
3338   * @dram_binning: contains mask of drams that is received from the f/w which indicates which
3339   *                drams are binned-out
3340   * @tpc_binning: contains mask of tpc engines that is received from the f/w which indicates which
3341   *               tpc engines are binned-out
3342   * @dmabuf_export_cnt: number of dma-buf exporting.
3343   * @card_type: Various ASICs have several card types. This indicates the card
3344   *             type of the current device.
3345   * @major: habanalabs kernel driver major.
3346   * @high_pll: high PLL profile frequency.
3347   * @decoder_binning: contains mask of decoder engines that is received from the f/w which
3348   *                   indicates which decoder engines are binned-out
3349   * @edma_binning: contains mask of edma engines that is received from the f/w which
3350   *                   indicates which edma engines are binned-out
3351   * @device_release_watchdog_timeout_sec: device release watchdog timeout value in seconds.
3352   * @rotator_binning: contains mask of rotators engines that is received from the f/w
3353   *			which indicates which rotator engines are binned-out(Gaudi3 and above).
3354   * @id: device minor.
3355   * @cdev_idx: char device index.
3356   * @cpu_pci_msb_addr: 50-bit extension bits for the device CPU's 40-bit
3357   *                    addresses.
3358   * @is_in_dram_scrub: true if dram scrub operation is on going.
3359   * @disabled: is device disabled.
3360   * @late_init_done: is late init stage was done during initialization.
3361   * @hwmon_initialized: is H/W monitor sensors was initialized.
3362   * @reset_on_lockup: true if a reset should be done in case of stuck CS, false
3363   *                   otherwise.
3364   * @dram_default_page_mapping: is DRAM default page mapping enabled.
3365   * @memory_scrub: true to perform device memory scrub in various locations,
3366   *                such as context-switch, context close, page free, etc.
3367   * @pmmu_huge_range: is a different virtual addresses range used for PMMU with
3368   *                   huge pages.
3369   * @init_done: is the initialization of the device done.
3370   * @device_cpu_disabled: is the device CPU disabled (due to timeouts)
3371   * @in_debug: whether the device is in a state where the profiling/tracing infrastructure
3372   *            can be used. This indication is needed because in some ASICs we need to do
3373   *            specific operations to enable that infrastructure.
3374   * @cdev_sysfs_debugfs_created: were char devices and sysfs/debugfs files created.
3375   * @stop_on_err: true if engines should stop on error.
3376   * @supports_sync_stream: is sync stream supported.
3377   * @sync_stream_queue_idx: helper index for sync stream queues initialization.
3378   * @collective_mon_idx: helper index for collective initialization
3379   * @supports_coresight: is CoreSight supported.
3380   * @supports_cb_mapping: is mapping a CB to the device's MMU supported.
3381   * @process_kill_trial_cnt: number of trials reset thread tried killing
3382   *                          user processes
3383   * @device_fini_pending: true if device_fini was called and might be
3384   *                       waiting for the reset thread to finish
3385   * @supports_staged_submission: true if staged submissions are supported
3386   * @device_cpu_is_halted: Flag to indicate whether the device CPU was already
3387   *                        halted. We can't halt it again because the COMMS
3388   *                        protocol will throw an error. Relevant only for
3389   *                        cases where Linux was not loaded to device CPU
3390   * @supports_wait_for_multi_cs: true if wait for multi CS is supported
3391   * @is_compute_ctx_active: Whether there is an active compute context executing.
3392   * @compute_ctx_in_release: true if the current compute context is being released.
3393   * @supports_mmu_prefetch: true if prefetch is supported, otherwise false.
3394   * @reset_upon_device_release: reset the device when the user closes the file descriptor of the
3395   *                             device.
3396   * @supports_ctx_switch: true if a ctx switch is required upon first submission.
3397   * @support_preboot_binning: true if we support read binning info from preboot.
3398   * @eq_heartbeat_received: indication that eq heartbeat event has received from FW.
3399   * @nic_ports_mask: Controls which NIC ports are enabled. Used only for testing.
3400   * @fw_components: Controls which f/w components to load to the device. There are multiple f/w
3401   *                 stages and sometimes we want to stop at a certain stage. Used only for testing.
3402   * @mmu_disable: Disable the device MMU(s). Used only for testing.
3403   * @cpu_queues_enable: Whether to enable queues communication vs. the f/w. Used only for testing.
3404   * @pldm: Whether we are running in Palladium environment. Used only for testing.
3405   * @hard_reset_on_fw_events: Whether to do device hard-reset when a fatal event is received from
3406   *                           the f/w. Used only for testing.
3407   * @bmc_enable: Whether we are running in a box with BMC. Used only for testing.
3408   * @reset_on_preboot_fail: Whether to reset the device if preboot f/w fails to load.
3409   *                         Used only for testing.
3410   * @heartbeat: Controls if we want to enable the heartbeat mechanism vs. the f/w, which verifies
3411   *             that the f/w is always alive. Used only for testing.
3412   */
3413  struct hl_device {
3414  	struct pci_dev			*pdev;
3415  	u64				pcie_bar_phys[HL_PCI_NUM_BARS];
3416  	void __iomem			*pcie_bar[HL_PCI_NUM_BARS];
3417  	void __iomem			*rmmio;
3418  	struct drm_device		drm;
3419  	struct cdev			cdev_ctrl;
3420  	struct device			*dev;
3421  	struct device			*dev_ctrl;
3422  	struct delayed_work		work_heartbeat;
3423  	struct hl_device_reset_work	device_reset_work;
3424  	struct hl_device_reset_work	device_release_watchdog_work;
3425  	char				asic_name[HL_STR_MAX];
3426  	char				status[HL_DEV_STS_MAX][HL_STR_MAX];
3427  	enum hl_asic_type		asic_type;
3428  	struct hl_cq			*completion_queue;
3429  	struct hl_user_interrupt	*user_interrupt;
3430  	struct hl_user_interrupt	tpc_interrupt;
3431  	struct hl_user_interrupt	unexpected_error_interrupt;
3432  	struct hl_user_interrupt	common_user_cq_interrupt;
3433  	struct hl_user_interrupt	common_decoder_interrupt;
3434  	struct hl_cs			**shadow_cs_queue;
3435  	struct workqueue_struct		**cq_wq;
3436  	struct workqueue_struct		*eq_wq;
3437  	struct workqueue_struct		*cs_cmplt_wq;
3438  	struct workqueue_struct		*ts_free_obj_wq;
3439  	struct workqueue_struct		*prefetch_wq;
3440  	struct workqueue_struct		*reset_wq;
3441  	struct hl_ctx			*kernel_ctx;
3442  	struct hl_hw_queue		*kernel_queues;
3443  	struct list_head		cs_mirror_list;
3444  	spinlock_t			cs_mirror_lock;
3445  	struct hl_mem_mgr		kernel_mem_mgr;
3446  	struct hl_eq			event_queue;
3447  	struct dma_pool			*dma_pool;
3448  	void				*cpu_accessible_dma_mem;
3449  	dma_addr_t			cpu_accessible_dma_address;
3450  	struct gen_pool			*cpu_accessible_dma_pool;
3451  	unsigned long			*asid_bitmap;
3452  	struct mutex			asid_mutex;
3453  	struct mutex			send_cpu_message_lock;
3454  	struct mutex			debug_lock;
3455  	struct mutex			mmu_lock;
3456  	struct asic_fixed_properties	asic_prop;
3457  	const struct hl_asic_funcs	*asic_funcs;
3458  	void				*asic_specific;
3459  	struct hl_vm			vm;
3460  	struct device			*hwmon_dev;
3461  	struct hwmon_chip_info		*hl_chip_info;
3462  
3463  	struct hl_dbg_device_entry	hl_debugfs;
3464  
3465  	struct list_head		cb_pool;
3466  	spinlock_t			cb_pool_lock;
3467  
3468  	void				*internal_cb_pool_virt_addr;
3469  	dma_addr_t			internal_cb_pool_dma_addr;
3470  	struct gen_pool			*internal_cb_pool;
3471  	u64				internal_cb_va_base;
3472  
3473  	struct list_head		fpriv_list;
3474  	struct list_head		fpriv_ctrl_list;
3475  	struct mutex			fpriv_list_lock;
3476  	struct mutex			fpriv_ctrl_list_lock;
3477  
3478  	struct hl_cs_counters_atomic	aggregated_cs_counters;
3479  
3480  	struct hl_mmu_priv		mmu_priv;
3481  	struct hl_mmu_funcs		mmu_func[MMU_NUM_PGT_LOCATIONS];
3482  
3483  	struct hl_dec			*dec;
3484  
3485  	struct fw_load_mgr		fw_loader;
3486  
3487  	struct pci_mem_region		pci_mem_region[PCI_REGION_NUMBER];
3488  
3489  	struct hl_state_dump_specs	state_dump_specs;
3490  
3491  	struct multi_cs_completion	multi_cs_completion[
3492  							MULTI_CS_MAX_USER_CTX];
3493  	struct hl_clk_throttle		clk_throttling;
3494  	struct hl_error_info		captured_err_info;
3495  
3496  	struct hl_reset_info		reset_info;
3497  
3498  	struct eq_heartbeat_debug_info	heartbeat_debug_info;
3499  
3500  	cpumask_t			irq_affinity_mask;
3501  
3502  	u32				*stream_master_qid_arr;
3503  	u32				fw_inner_major_ver;
3504  	u32				fw_inner_minor_ver;
3505  	u32				fw_sw_major_ver;
3506  	u32				fw_sw_minor_ver;
3507  	u32				fw_sw_sub_minor_ver;
3508  	atomic64_t			dram_used_mem;
3509  	u64				memory_scrub_val;
3510  	u64				timeout_jiffies;
3511  	u64				max_power;
3512  	u64				boot_error_status_mask;
3513  	u64				dram_pci_bar_start;
3514  	u64				last_successful_open_jif;
3515  	u64				last_open_session_duration_jif;
3516  	u64				open_counter;
3517  	u64				fw_poll_interval_usec;
3518  	ktime_t				last_successful_open_ktime;
3519  	u64				fw_comms_poll_interval_usec;
3520  	u64				dram_binning;
3521  	u64				tpc_binning;
3522  	atomic_t			dmabuf_export_cnt;
3523  	enum cpucp_card_types		card_type;
3524  	u32				major;
3525  	u32				high_pll;
3526  	u32				decoder_binning;
3527  	u32				edma_binning;
3528  	u32				device_release_watchdog_timeout_sec;
3529  	u32				rotator_binning;
3530  	u16				id;
3531  	u16				cdev_idx;
3532  	u16				cpu_pci_msb_addr;
3533  	u8				is_in_dram_scrub;
3534  	u8				disabled;
3535  	u8				late_init_done;
3536  	u8				hwmon_initialized;
3537  	u8				reset_on_lockup;
3538  	u8				dram_default_page_mapping;
3539  	u8				memory_scrub;
3540  	u8				pmmu_huge_range;
3541  	u8				init_done;
3542  	u8				device_cpu_disabled;
3543  	u8				in_debug;
3544  	u8				cdev_sysfs_debugfs_created;
3545  	u8				stop_on_err;
3546  	u8				supports_sync_stream;
3547  	u8				sync_stream_queue_idx;
3548  	u8				collective_mon_idx;
3549  	u8				supports_coresight;
3550  	u8				supports_cb_mapping;
3551  	u8				process_kill_trial_cnt;
3552  	u8				device_fini_pending;
3553  	u8				supports_staged_submission;
3554  	u8				device_cpu_is_halted;
3555  	u8				supports_wait_for_multi_cs;
3556  	u8				stream_master_qid_arr_size;
3557  	u8				is_compute_ctx_active;
3558  	u8				compute_ctx_in_release;
3559  	u8				supports_mmu_prefetch;
3560  	u8				reset_upon_device_release;
3561  	u8				supports_ctx_switch;
3562  	u8				support_preboot_binning;
3563  	u8				eq_heartbeat_received;
3564  
3565  	/* Parameters for bring-up to be upstreamed */
3566  	u64				nic_ports_mask;
3567  	u64				fw_components;
3568  	u8				mmu_disable;
3569  	u8				cpu_queues_enable;
3570  	u8				pldm;
3571  	u8				hard_reset_on_fw_events;
3572  	u8				bmc_enable;
3573  	u8				reset_on_preboot_fail;
3574  	u8				heartbeat;
3575  };
3576  
3577  /* Retrieve PCI device name in case of a PCI device or dev name in simulator */
3578  #define HL_DEV_NAME(hdev)	\
3579  		((hdev)->pdev ? dev_name(&(hdev)->pdev->dev) : "NA-DEVICE")
3580  
3581  /**
3582   * struct hl_cs_encaps_sig_handle - encapsulated signals handle structure
3583   * @refcount: refcount used to protect removing this id when several
3584   *            wait cs are used to wait of the reserved encaps signals.
3585   * @hdev: pointer to habanalabs device structure.
3586   * @hw_sob: pointer to  H/W SOB used in the reservation.
3587   * @ctx: pointer to the user's context data structure
3588   * @cs_seq: staged cs sequence which contains encapsulated signals
3589   * @id: idr handler id to be used to fetch the handler info
3590   * @q_idx: stream queue index
3591   * @pre_sob_val: current SOB value before reservation
3592   * @count: signals number
3593   */
3594  struct hl_cs_encaps_sig_handle {
3595  	struct kref refcount;
3596  	struct hl_device *hdev;
3597  	struct hl_hw_sob *hw_sob;
3598  	struct hl_ctx *ctx;
3599  	u64  cs_seq;
3600  	u32  id;
3601  	u32  q_idx;
3602  	u32  pre_sob_val;
3603  	u32  count;
3604  };
3605  
3606  /**
3607   * struct hl_info_fw_err_info - firmware error information structure
3608   * @err_type: The type of error detected (or reported).
3609   * @event_mask: Pointer to the event mask to be modified with the detected error flag
3610   *              (can be NULL)
3611   * @event_id: The id of the event that reported the error
3612   *            (applicable when err_type is HL_INFO_FW_REPORTED_ERR).
3613   */
3614  struct hl_info_fw_err_info {
3615  	enum hl_info_fw_err_type err_type;
3616  	u64 *event_mask;
3617  	u16 event_id;
3618  };
3619  
3620  /*
3621   * IOCTLs
3622   */
3623  
3624  /**
3625   * typedef hl_ioctl_t - typedef for ioctl function in the driver
3626   * @hpriv: pointer to the FD's private data, which contains state of
3627   *		user process
3628   * @data: pointer to the input/output arguments structure of the IOCTL
3629   *
3630   * Return: 0 for success, negative value for error
3631   */
3632  typedef int hl_ioctl_t(struct hl_fpriv *hpriv, void *data);
3633  
3634  /**
3635   * struct hl_ioctl_desc - describes an IOCTL entry of the driver.
3636   * @cmd: the IOCTL code as created by the kernel macros.
3637   * @func: pointer to the driver's function that should be called for this IOCTL.
3638   */
3639  struct hl_ioctl_desc {
3640  	unsigned int cmd;
3641  	hl_ioctl_t *func;
3642  };
3643  
3644  /*
3645   * Kernel module functions that can be accessed by entire module
3646   */
3647  
3648  /**
3649   * hl_get_sg_info() - get number of pages and the DMA address from SG list.
3650   * @sg: the SG list.
3651   * @dma_addr: pointer to DMA address to return.
3652   *
3653   * Calculate the number of consecutive pages described by the SG list. Take the
3654   * offset of the address in the first page, add to it the length and round it up
3655   * to the number of needed pages.
3656   */
hl_get_sg_info(struct scatterlist * sg,dma_addr_t * dma_addr)3657  static inline u32 hl_get_sg_info(struct scatterlist *sg, dma_addr_t *dma_addr)
3658  {
3659  	*dma_addr = sg_dma_address(sg);
3660  
3661  	return ((((*dma_addr) & (PAGE_SIZE - 1)) + sg_dma_len(sg)) +
3662  			(PAGE_SIZE - 1)) >> PAGE_SHIFT;
3663  }
3664  
3665  /**
3666   * hl_mem_area_inside_range() - Checks whether address+size are inside a range.
3667   * @address: The start address of the area we want to validate.
3668   * @size: The size in bytes of the area we want to validate.
3669   * @range_start_address: The start address of the valid range.
3670   * @range_end_address: The end address of the valid range.
3671   *
3672   * Return: true if the area is inside the valid range, false otherwise.
3673   */
hl_mem_area_inside_range(u64 address,u64 size,u64 range_start_address,u64 range_end_address)3674  static inline bool hl_mem_area_inside_range(u64 address, u64 size,
3675  				u64 range_start_address, u64 range_end_address)
3676  {
3677  	u64 end_address = address + size;
3678  
3679  	if ((address >= range_start_address) &&
3680  			(end_address <= range_end_address) &&
3681  			(end_address > address))
3682  		return true;
3683  
3684  	return false;
3685  }
3686  
to_hl_device(struct drm_device * ddev)3687  static inline struct hl_device *to_hl_device(struct drm_device *ddev)
3688  {
3689  	return container_of(ddev, struct hl_device, drm);
3690  }
3691  
3692  /**
3693   * hl_mem_area_crosses_range() - Checks whether address+size crossing a range.
3694   * @address: The start address of the area we want to validate.
3695   * @size: The size in bytes of the area we want to validate.
3696   * @range_start_address: The start address of the valid range.
3697   * @range_end_address: The end address of the valid range.
3698   *
3699   * Return: true if the area overlaps part or all of the valid range,
3700   *		false otherwise.
3701   */
hl_mem_area_crosses_range(u64 address,u32 size,u64 range_start_address,u64 range_end_address)3702  static inline bool hl_mem_area_crosses_range(u64 address, u32 size,
3703  				u64 range_start_address, u64 range_end_address)
3704  {
3705  	u64 end_address = address + size - 1;
3706  
3707  	return ((address <= range_end_address) && (range_start_address <= end_address));
3708  }
3709  
3710  uint64_t hl_set_dram_bar_default(struct hl_device *hdev, u64 addr);
3711  void *hl_cpu_accessible_dma_pool_alloc(struct hl_device *hdev, size_t size, dma_addr_t *dma_handle);
3712  void hl_cpu_accessible_dma_pool_free(struct hl_device *hdev, size_t size, void *vaddr);
3713  void *hl_asic_dma_alloc_coherent_caller(struct hl_device *hdev, size_t size, dma_addr_t *dma_handle,
3714  					gfp_t flag, const char *caller);
3715  void hl_asic_dma_free_coherent_caller(struct hl_device *hdev, size_t size, void *cpu_addr,
3716  					dma_addr_t dma_handle, const char *caller);
3717  void *hl_asic_dma_pool_zalloc_caller(struct hl_device *hdev, size_t size, gfp_t mem_flags,
3718  					dma_addr_t *dma_handle, const char *caller);
3719  void hl_asic_dma_pool_free_caller(struct hl_device *hdev, void *vaddr, dma_addr_t dma_addr,
3720  					const char *caller);
3721  int hl_dma_map_sgtable_caller(struct hl_device *hdev, struct sg_table *sgt,
3722  				enum dma_data_direction dir, const char *caller);
3723  void hl_dma_unmap_sgtable_caller(struct hl_device *hdev, struct sg_table *sgt,
3724  					enum dma_data_direction dir, const char *caller);
3725  int hl_asic_dma_map_sgtable(struct hl_device *hdev, struct sg_table *sgt,
3726  				enum dma_data_direction dir);
3727  void hl_asic_dma_unmap_sgtable(struct hl_device *hdev, struct sg_table *sgt,
3728  				enum dma_data_direction dir);
3729  int hl_access_sram_dram_region(struct hl_device *hdev, u64 addr, u64 *val,
3730  	enum debugfs_access_type acc_type, enum pci_region region_type, bool set_dram_bar);
3731  int hl_access_cfg_region(struct hl_device *hdev, u64 addr, u64 *val,
3732  	enum debugfs_access_type acc_type);
3733  int hl_access_dev_mem(struct hl_device *hdev, enum pci_region region_type,
3734  			u64 addr, u64 *val, enum debugfs_access_type acc_type);
3735  
3736  int hl_mmap(struct file *filp, struct vm_area_struct *vma);
3737  
3738  int hl_device_open(struct drm_device *drm, struct drm_file *file_priv);
3739  void hl_device_release(struct drm_device *ddev, struct drm_file *file_priv);
3740  
3741  int hl_device_open_ctrl(struct inode *inode, struct file *filp);
3742  bool hl_device_operational(struct hl_device *hdev,
3743  		enum hl_device_status *status);
3744  bool hl_ctrl_device_operational(struct hl_device *hdev,
3745  		enum hl_device_status *status);
3746  enum hl_device_status hl_device_status(struct hl_device *hdev);
3747  int hl_device_set_debug_mode(struct hl_device *hdev, struct hl_ctx *ctx, bool enable);
3748  int hl_hw_queues_create(struct hl_device *hdev);
3749  void hl_hw_queues_destroy(struct hl_device *hdev);
3750  int hl_hw_queue_send_cb_no_cmpl(struct hl_device *hdev, u32 hw_queue_id,
3751  		u32 cb_size, u64 cb_ptr);
3752  void hl_hw_queue_submit_bd(struct hl_device *hdev, struct hl_hw_queue *q,
3753  		u32 ctl, u32 len, u64 ptr);
3754  int hl_hw_queue_schedule_cs(struct hl_cs *cs);
3755  u32 hl_hw_queue_add_ptr(u32 ptr, u16 val);
3756  void hl_hw_queue_inc_ci_kernel(struct hl_device *hdev, u32 hw_queue_id);
3757  void hl_hw_queue_update_ci(struct hl_cs *cs);
3758  void hl_hw_queue_reset(struct hl_device *hdev, bool hard_reset);
3759  
3760  #define hl_queue_inc_ptr(p)		hl_hw_queue_add_ptr(p, 1)
3761  #define hl_pi_2_offset(pi)		((pi) & (HL_QUEUE_LENGTH - 1))
3762  
3763  int hl_cq_init(struct hl_device *hdev, struct hl_cq *q, u32 hw_queue_id);
3764  void hl_cq_fini(struct hl_device *hdev, struct hl_cq *q);
3765  int hl_eq_init(struct hl_device *hdev, struct hl_eq *q);
3766  void hl_eq_fini(struct hl_device *hdev, struct hl_eq *q);
3767  void hl_cq_reset(struct hl_device *hdev, struct hl_cq *q);
3768  void hl_eq_reset(struct hl_device *hdev, struct hl_eq *q);
3769  void hl_eq_dump(struct hl_device *hdev, struct hl_eq *q);
3770  irqreturn_t hl_irq_handler_cq(int irq, void *arg);
3771  irqreturn_t hl_irq_handler_eq(int irq, void *arg);
3772  irqreturn_t hl_irq_handler_dec_abnrm(int irq, void *arg);
3773  irqreturn_t hl_irq_user_interrupt_handler(int irq, void *arg);
3774  irqreturn_t hl_irq_user_interrupt_thread_handler(int irq, void *arg);
3775  irqreturn_t hl_irq_eq_error_interrupt_thread_handler(int irq, void *arg);
3776  u32 hl_cq_inc_ptr(u32 ptr);
3777  
3778  int hl_asid_init(struct hl_device *hdev);
3779  void hl_asid_fini(struct hl_device *hdev);
3780  unsigned long hl_asid_alloc(struct hl_device *hdev);
3781  void hl_asid_free(struct hl_device *hdev, unsigned long asid);
3782  
3783  int hl_ctx_create(struct hl_device *hdev, struct hl_fpriv *hpriv);
3784  void hl_ctx_free(struct hl_device *hdev, struct hl_ctx *ctx);
3785  int hl_ctx_init(struct hl_device *hdev, struct hl_ctx *ctx, bool is_kernel_ctx);
3786  void hl_ctx_do_release(struct kref *ref);
3787  void hl_ctx_get(struct hl_ctx *ctx);
3788  int hl_ctx_put(struct hl_ctx *ctx);
3789  struct hl_ctx *hl_get_compute_ctx(struct hl_device *hdev);
3790  struct hl_fence *hl_ctx_get_fence(struct hl_ctx *ctx, u64 seq);
3791  int hl_ctx_get_fences(struct hl_ctx *ctx, u64 *seq_arr,
3792  				struct hl_fence **fence, u32 arr_len);
3793  void hl_ctx_mgr_init(struct hl_ctx_mgr *mgr);
3794  void hl_ctx_mgr_fini(struct hl_device *hdev, struct hl_ctx_mgr *mgr);
3795  
3796  int hl_device_init(struct hl_device *hdev);
3797  void hl_device_fini(struct hl_device *hdev);
3798  int hl_device_suspend(struct hl_device *hdev);
3799  int hl_device_resume(struct hl_device *hdev);
3800  int hl_device_reset(struct hl_device *hdev, u32 flags);
3801  int hl_device_cond_reset(struct hl_device *hdev, u32 flags, u64 event_mask);
3802  void hl_hpriv_get(struct hl_fpriv *hpriv);
3803  int hl_hpriv_put(struct hl_fpriv *hpriv);
3804  int hl_device_utilization(struct hl_device *hdev, u32 *utilization);
3805  
3806  int hl_build_hwmon_channel_info(struct hl_device *hdev,
3807  		struct cpucp_sensor *sensors_arr);
3808  
3809  void hl_notifier_event_send_all(struct hl_device *hdev, u64 event_mask);
3810  
3811  int hl_sysfs_init(struct hl_device *hdev);
3812  void hl_sysfs_fini(struct hl_device *hdev);
3813  
3814  int hl_hwmon_init(struct hl_device *hdev);
3815  void hl_hwmon_fini(struct hl_device *hdev);
3816  void hl_hwmon_release_resources(struct hl_device *hdev);
3817  
3818  int hl_cb_create(struct hl_device *hdev, struct hl_mem_mgr *mmg,
3819  			struct hl_ctx *ctx, u32 cb_size, bool internal_cb,
3820  			bool map_cb, u64 *handle);
3821  int hl_cb_destroy(struct hl_mem_mgr *mmg, u64 cb_handle);
3822  int hl_hw_block_mmap(struct hl_fpriv *hpriv, struct vm_area_struct *vma);
3823  struct hl_cb *hl_cb_get(struct hl_mem_mgr *mmg, u64 handle);
3824  void hl_cb_put(struct hl_cb *cb);
3825  struct hl_cb *hl_cb_kernel_create(struct hl_device *hdev, u32 cb_size,
3826  					bool internal_cb);
3827  int hl_cb_pool_init(struct hl_device *hdev);
3828  int hl_cb_pool_fini(struct hl_device *hdev);
3829  int hl_cb_va_pool_init(struct hl_ctx *ctx);
3830  void hl_cb_va_pool_fini(struct hl_ctx *ctx);
3831  
3832  void hl_cs_rollback_all(struct hl_device *hdev, bool skip_wq_flush);
3833  struct hl_cs_job *hl_cs_allocate_job(struct hl_device *hdev,
3834  		enum hl_queue_type queue_type, bool is_kernel_allocated_cb);
3835  void hl_sob_reset_error(struct kref *ref);
3836  int hl_gen_sob_mask(u16 sob_base, u8 sob_mask, u8 *mask);
3837  void hl_fence_put(struct hl_fence *fence);
3838  void hl_fences_put(struct hl_fence **fence, int len);
3839  void hl_fence_get(struct hl_fence *fence);
3840  void cs_get(struct hl_cs *cs);
3841  bool cs_needs_completion(struct hl_cs *cs);
3842  bool cs_needs_timeout(struct hl_cs *cs);
3843  bool is_staged_cs_last_exists(struct hl_device *hdev, struct hl_cs *cs);
3844  struct hl_cs *hl_staged_cs_find_first(struct hl_device *hdev, u64 cs_seq);
3845  void hl_multi_cs_completion_init(struct hl_device *hdev);
3846  u32 hl_get_active_cs_num(struct hl_device *hdev);
3847  
3848  void goya_set_asic_funcs(struct hl_device *hdev);
3849  void gaudi_set_asic_funcs(struct hl_device *hdev);
3850  void gaudi2_set_asic_funcs(struct hl_device *hdev);
3851  
3852  int hl_vm_ctx_init(struct hl_ctx *ctx);
3853  void hl_vm_ctx_fini(struct hl_ctx *ctx);
3854  
3855  int hl_vm_init(struct hl_device *hdev);
3856  void hl_vm_fini(struct hl_device *hdev);
3857  
3858  void hl_hw_block_mem_init(struct hl_ctx *ctx);
3859  void hl_hw_block_mem_fini(struct hl_ctx *ctx);
3860  
3861  u64 hl_reserve_va_block(struct hl_device *hdev, struct hl_ctx *ctx,
3862  		enum hl_va_range_type type, u64 size, u32 alignment);
3863  int hl_unreserve_va_block(struct hl_device *hdev, struct hl_ctx *ctx,
3864  		u64 start_addr, u64 size);
3865  int hl_pin_host_memory(struct hl_device *hdev, u64 addr, u64 size,
3866  			struct hl_userptr *userptr);
3867  void hl_unpin_host_memory(struct hl_device *hdev, struct hl_userptr *userptr);
3868  void hl_userptr_delete_list(struct hl_device *hdev,
3869  				struct list_head *userptr_list);
3870  bool hl_userptr_is_pinned(struct hl_device *hdev, u64 addr, u32 size,
3871  				struct list_head *userptr_list,
3872  				struct hl_userptr **userptr);
3873  
3874  int hl_mmu_init(struct hl_device *hdev);
3875  void hl_mmu_fini(struct hl_device *hdev);
3876  int hl_mmu_ctx_init(struct hl_ctx *ctx);
3877  void hl_mmu_ctx_fini(struct hl_ctx *ctx);
3878  int hl_mmu_map_page(struct hl_ctx *ctx, u64 virt_addr, u64 phys_addr,
3879  		u32 page_size, bool flush_pte);
3880  int hl_mmu_get_real_page_size(struct hl_device *hdev, struct hl_mmu_properties *mmu_prop,
3881  				u32 page_size, u32 *real_page_size, bool is_dram_addr);
3882  int hl_mmu_unmap_page(struct hl_ctx *ctx, u64 virt_addr, u32 page_size,
3883  		bool flush_pte);
3884  int hl_mmu_map_contiguous(struct hl_ctx *ctx, u64 virt_addr,
3885  					u64 phys_addr, u32 size);
3886  int hl_mmu_unmap_contiguous(struct hl_ctx *ctx, u64 virt_addr, u32 size);
3887  int hl_mmu_invalidate_cache(struct hl_device *hdev, bool is_hard, u32 flags);
3888  int hl_mmu_invalidate_cache_range(struct hl_device *hdev, bool is_hard,
3889  					u32 flags, u32 asid, u64 va, u64 size);
3890  int hl_mmu_prefetch_cache_range(struct hl_ctx *ctx, u32 flags, u32 asid, u64 va, u64 size);
3891  u64 hl_mmu_get_next_hop_addr(struct hl_ctx *ctx, u64 curr_pte);
3892  u64 hl_mmu_get_hop_pte_phys_addr(struct hl_ctx *ctx, struct hl_mmu_properties *mmu_prop,
3893  					u8 hop_idx, u64 hop_addr, u64 virt_addr);
3894  void hl_mmu_hr_flush(struct hl_ctx *ctx);
3895  int hl_mmu_hr_init(struct hl_device *hdev, struct hl_mmu_hr_priv *hr_priv, u32 hop_table_size,
3896  			u64 pgt_size);
3897  void hl_mmu_hr_fini(struct hl_device *hdev, struct hl_mmu_hr_priv *hr_priv, u32 hop_table_size);
3898  void hl_mmu_hr_free_hop_remove_pgt(struct pgt_info *pgt_info, struct hl_mmu_hr_priv *hr_priv,
3899  				u32 hop_table_size);
3900  u64 hl_mmu_hr_pte_phys_to_virt(struct hl_ctx *ctx, struct pgt_info *pgt, u64 phys_pte_addr,
3901  							u32 hop_table_size);
3902  void hl_mmu_hr_write_pte(struct hl_ctx *ctx, struct pgt_info *pgt_info, u64 phys_pte_addr,
3903  							u64 val, u32 hop_table_size);
3904  void hl_mmu_hr_clear_pte(struct hl_ctx *ctx, struct pgt_info *pgt_info, u64 phys_pte_addr,
3905  							u32 hop_table_size);
3906  int hl_mmu_hr_put_pte(struct hl_ctx *ctx, struct pgt_info *pgt_info, struct hl_mmu_hr_priv *hr_priv,
3907  							u32 hop_table_size);
3908  void hl_mmu_hr_get_pte(struct hl_ctx *ctx, struct hl_hr_mmu_funcs *hr_func, u64 phys_hop_addr);
3909  struct pgt_info *hl_mmu_hr_get_next_hop_pgt_info(struct hl_ctx *ctx,
3910  							struct hl_hr_mmu_funcs *hr_func,
3911  							u64 curr_pte);
3912  struct pgt_info *hl_mmu_hr_alloc_hop(struct hl_ctx *ctx, struct hl_mmu_hr_priv *hr_priv,
3913  							struct hl_hr_mmu_funcs *hr_func,
3914  							struct hl_mmu_properties *mmu_prop);
3915  struct pgt_info *hl_mmu_hr_get_alloc_next_hop(struct hl_ctx *ctx,
3916  							struct hl_mmu_hr_priv *hr_priv,
3917  							struct hl_hr_mmu_funcs *hr_func,
3918  							struct hl_mmu_properties *mmu_prop,
3919  							u64 curr_pte, bool *is_new_hop);
3920  int hl_mmu_hr_get_tlb_info(struct hl_ctx *ctx, u64 virt_addr, struct hl_mmu_hop_info *hops,
3921  							struct hl_hr_mmu_funcs *hr_func);
3922  int hl_mmu_if_set_funcs(struct hl_device *hdev);
3923  void hl_mmu_v1_set_funcs(struct hl_device *hdev, struct hl_mmu_funcs *mmu);
3924  void hl_mmu_v2_set_funcs(struct hl_device *hdev, struct hl_mmu_funcs *mmu);
3925  void hl_mmu_v2_hr_set_funcs(struct hl_device *hdev, struct hl_mmu_funcs *mmu);
3926  int hl_mmu_va_to_pa(struct hl_ctx *ctx, u64 virt_addr, u64 *phys_addr);
3927  int hl_mmu_get_tlb_info(struct hl_ctx *ctx, u64 virt_addr,
3928  			struct hl_mmu_hop_info *hops);
3929  u64 hl_mmu_scramble_addr(struct hl_device *hdev, u64 addr);
3930  u64 hl_mmu_descramble_addr(struct hl_device *hdev, u64 addr);
3931  bool hl_is_dram_va(struct hl_device *hdev, u64 virt_addr);
3932  struct pgt_info *hl_mmu_dr_get_pgt_info(struct hl_ctx *ctx, u64 hop_addr);
3933  void hl_mmu_dr_free_hop(struct hl_ctx *ctx, u64 hop_addr);
3934  void hl_mmu_dr_free_pgt_node(struct hl_ctx *ctx, struct pgt_info *pgt_info);
3935  u64 hl_mmu_dr_get_phys_hop0_addr(struct hl_ctx *ctx);
3936  u64 hl_mmu_dr_get_hop0_addr(struct hl_ctx *ctx);
3937  void hl_mmu_dr_write_pte(struct hl_ctx *ctx, u64 shadow_pte_addr, u64 val);
3938  void hl_mmu_dr_write_final_pte(struct hl_ctx *ctx, u64 shadow_pte_addr, u64 val);
3939  void hl_mmu_dr_clear_pte(struct hl_ctx *ctx, u64 pte_addr);
3940  u64 hl_mmu_dr_get_phys_addr(struct hl_ctx *ctx, u64 shadow_addr);
3941  void hl_mmu_dr_get_pte(struct hl_ctx *ctx, u64 hop_addr);
3942  int hl_mmu_dr_put_pte(struct hl_ctx *ctx, u64 hop_addr);
3943  u64 hl_mmu_dr_get_alloc_next_hop_addr(struct hl_ctx *ctx, u64 curr_pte, bool *is_new_hop);
3944  u64 hl_mmu_dr_alloc_hop(struct hl_ctx *ctx);
3945  void hl_mmu_dr_flush(struct hl_ctx *ctx);
3946  int hl_mmu_dr_init(struct hl_device *hdev);
3947  void hl_mmu_dr_fini(struct hl_device *hdev);
3948  
3949  int hl_fw_version_cmp(struct hl_device *hdev, u32 major, u32 minor, u32 subminor);
3950  int hl_fw_load_fw_to_device(struct hl_device *hdev, const char *fw_name,
3951  				void __iomem *dst, u32 src_offset, u32 size);
3952  int hl_fw_send_pci_access_msg(struct hl_device *hdev, u32 opcode, u64 value);
3953  int hl_fw_send_cpu_message(struct hl_device *hdev, u32 hw_queue_id, u32 *msg,
3954  				u16 len, u32 timeout, u64 *result);
3955  int hl_fw_unmask_irq(struct hl_device *hdev, u16 event_type);
3956  int hl_fw_unmask_irq_arr(struct hl_device *hdev, const u32 *irq_arr,
3957  		size_t irq_arr_size);
3958  int hl_fw_test_cpu_queue(struct hl_device *hdev);
3959  void *hl_fw_cpu_accessible_dma_pool_alloc(struct hl_device *hdev, size_t size,
3960  						dma_addr_t *dma_handle);
3961  void hl_fw_cpu_accessible_dma_pool_free(struct hl_device *hdev, size_t size,
3962  					void *vaddr);
3963  int hl_fw_send_heartbeat(struct hl_device *hdev);
3964  int hl_fw_cpucp_info_get(struct hl_device *hdev,
3965  				u32 sts_boot_dev_sts0_reg,
3966  				u32 sts_boot_dev_sts1_reg, u32 boot_err0_reg,
3967  				u32 boot_err1_reg);
3968  int hl_fw_cpucp_handshake(struct hl_device *hdev,
3969  				u32 sts_boot_dev_sts0_reg,
3970  				u32 sts_boot_dev_sts1_reg, u32 boot_err0_reg,
3971  				u32 boot_err1_reg);
3972  int hl_fw_get_eeprom_data(struct hl_device *hdev, void *data, size_t max_size);
3973  int hl_fw_get_monitor_dump(struct hl_device *hdev, void *data);
3974  int hl_fw_cpucp_pci_counters_get(struct hl_device *hdev,
3975  		struct hl_info_pci_counters *counters);
3976  int hl_fw_cpucp_total_energy_get(struct hl_device *hdev,
3977  			u64 *total_energy);
3978  int get_used_pll_index(struct hl_device *hdev, u32 input_pll_index,
3979  						enum pll_index *pll_index);
3980  int hl_fw_cpucp_pll_info_get(struct hl_device *hdev, u32 pll_index,
3981  		u16 *pll_freq_arr);
3982  int hl_fw_cpucp_power_get(struct hl_device *hdev, u64 *power);
3983  void hl_fw_ask_hard_reset_without_linux(struct hl_device *hdev);
3984  void hl_fw_ask_halt_machine_without_linux(struct hl_device *hdev);
3985  int hl_fw_init_cpu(struct hl_device *hdev);
3986  int hl_fw_wait_preboot_ready(struct hl_device *hdev);
3987  int hl_fw_read_preboot_status(struct hl_device *hdev);
3988  int hl_fw_dynamic_send_protocol_cmd(struct hl_device *hdev,
3989  				struct fw_load_mgr *fw_loader,
3990  				enum comms_cmd cmd, unsigned int size,
3991  				bool wait_ok, u32 timeout);
3992  int hl_fw_dram_replaced_row_get(struct hl_device *hdev,
3993  				struct cpucp_hbm_row_info *info);
3994  int hl_fw_dram_pending_row_get(struct hl_device *hdev, u32 *pend_rows_num);
3995  int hl_fw_cpucp_engine_core_asid_set(struct hl_device *hdev, u32 asid);
3996  int hl_fw_send_device_activity(struct hl_device *hdev, bool open);
3997  int hl_fw_send_soft_reset(struct hl_device *hdev);
3998  int hl_pci_bars_map(struct hl_device *hdev, const char * const name[3],
3999  			bool is_wc[3]);
4000  int hl_pci_elbi_read(struct hl_device *hdev, u64 addr, u32 *data);
4001  int hl_pci_iatu_write(struct hl_device *hdev, u32 addr, u32 data);
4002  int hl_pci_set_inbound_region(struct hl_device *hdev, u8 region,
4003  		struct hl_inbound_pci_region *pci_region);
4004  int hl_pci_set_outbound_region(struct hl_device *hdev,
4005  		struct hl_outbound_pci_region *pci_region);
4006  enum pci_region hl_get_pci_memory_region(struct hl_device *hdev, u64 addr);
4007  int hl_pci_init(struct hl_device *hdev);
4008  void hl_pci_fini(struct hl_device *hdev);
4009  
4010  long hl_fw_get_frequency(struct hl_device *hdev, u32 pll_index, bool curr);
4011  void hl_fw_set_frequency(struct hl_device *hdev, u32 pll_index, u64 freq);
4012  int hl_get_temperature(struct hl_device *hdev, int sensor_index, u32 attr, long *value);
4013  int hl_set_temperature(struct hl_device *hdev, int sensor_index, u32 attr, long value);
4014  int hl_get_voltage(struct hl_device *hdev, int sensor_index, u32 attr, long *value);
4015  int hl_get_current(struct hl_device *hdev, int sensor_index, u32 attr, long *value);
4016  int hl_get_fan_speed(struct hl_device *hdev, int sensor_index, u32 attr, long *value);
4017  int hl_get_pwm_info(struct hl_device *hdev, int sensor_index, u32 attr, long *value);
4018  void hl_set_pwm_info(struct hl_device *hdev, int sensor_index, u32 attr, long value);
4019  long hl_fw_get_max_power(struct hl_device *hdev);
4020  void hl_fw_set_max_power(struct hl_device *hdev);
4021  int hl_fw_get_sec_attest_info(struct hl_device *hdev, struct cpucp_sec_attest_info *sec_attest_info,
4022  				u32 nonce);
4023  int hl_fw_get_dev_info_signed(struct hl_device *hdev,
4024  			      struct cpucp_dev_info_signed *dev_info_signed, u32 nonce);
4025  int hl_set_voltage(struct hl_device *hdev, int sensor_index, u32 attr, long value);
4026  int hl_set_current(struct hl_device *hdev, int sensor_index, u32 attr, long value);
4027  int hl_set_power(struct hl_device *hdev, int sensor_index, u32 attr, long value);
4028  int hl_get_power(struct hl_device *hdev, int sensor_index, u32 attr, long *value);
4029  int hl_fw_get_clk_rate(struct hl_device *hdev, u32 *cur_clk, u32 *max_clk);
4030  void hl_fw_set_pll_profile(struct hl_device *hdev);
4031  void hl_sysfs_add_dev_clk_attr(struct hl_device *hdev, struct attribute_group *dev_clk_attr_grp);
4032  void hl_sysfs_add_dev_vrm_attr(struct hl_device *hdev, struct attribute_group *dev_vrm_attr_grp);
4033  int hl_fw_send_generic_request(struct hl_device *hdev, enum hl_passthrough_type sub_opcode,
4034  						dma_addr_t buff, u32 *size);
4035  
4036  void hw_sob_get(struct hl_hw_sob *hw_sob);
4037  void hw_sob_put(struct hl_hw_sob *hw_sob);
4038  void hl_encaps_release_handle_and_put_ctx(struct kref *ref);
4039  void hl_encaps_release_handle_and_put_sob_ctx(struct kref *ref);
4040  void hl_hw_queue_encaps_sig_set_sob_info(struct hl_device *hdev,
4041  			struct hl_cs *cs, struct hl_cs_job *job,
4042  			struct hl_cs_compl *cs_cmpl);
4043  
4044  int hl_dec_init(struct hl_device *hdev);
4045  void hl_dec_fini(struct hl_device *hdev);
4046  void hl_dec_ctx_fini(struct hl_ctx *ctx);
4047  
4048  void hl_release_pending_user_interrupts(struct hl_device *hdev);
4049  void hl_abort_waiting_for_cs_completions(struct hl_device *hdev);
4050  int hl_cs_signal_sob_wraparound_handler(struct hl_device *hdev, u32 q_idx,
4051  			struct hl_hw_sob **hw_sob, u32 count, bool encaps_sig);
4052  
4053  int hl_state_dump(struct hl_device *hdev);
4054  const char *hl_state_dump_get_sync_name(struct hl_device *hdev, u32 sync_id);
4055  const char *hl_state_dump_get_monitor_name(struct hl_device *hdev,
4056  					struct hl_mon_state_dump *mon);
4057  void hl_state_dump_free_sync_to_engine_map(struct hl_sync_to_engine_map *map);
4058  __printf(4, 5) int hl_snprintf_resize(char **buf, size_t *size, size_t *offset,
4059  					const char *format, ...);
4060  char *hl_format_as_binary(char *buf, size_t buf_len, u32 n);
4061  const char *hl_sync_engine_to_string(enum hl_sync_engine_type engine_type);
4062  
4063  void hl_mem_mgr_init(struct device *dev, struct hl_mem_mgr *mmg);
4064  void hl_mem_mgr_fini(struct hl_mem_mgr *mmg, struct hl_mem_mgr_fini_stats *stats);
4065  void hl_mem_mgr_idr_destroy(struct hl_mem_mgr *mmg);
4066  int hl_mem_mgr_mmap(struct hl_mem_mgr *mmg, struct vm_area_struct *vma,
4067  		    void *args);
4068  struct hl_mmap_mem_buf *hl_mmap_mem_buf_get(struct hl_mem_mgr *mmg,
4069  						   u64 handle);
4070  int hl_mmap_mem_buf_put_handle(struct hl_mem_mgr *mmg, u64 handle);
4071  int hl_mmap_mem_buf_put(struct hl_mmap_mem_buf *buf);
4072  struct hl_mmap_mem_buf *
4073  hl_mmap_mem_buf_alloc(struct hl_mem_mgr *mmg,
4074  		      struct hl_mmap_mem_buf_behavior *behavior, gfp_t gfp,
4075  		      void *args);
4076  __printf(2, 3) void hl_engine_data_sprintf(struct engines_data *e, const char *fmt, ...);
4077  void hl_capture_razwi(struct hl_device *hdev, u64 addr, u16 *engine_id, u16 num_of_engines,
4078  			u8 flags);
4079  void hl_handle_razwi(struct hl_device *hdev, u64 addr, u16 *engine_id, u16 num_of_engines,
4080  			u8 flags, u64 *event_mask);
4081  void hl_capture_page_fault(struct hl_device *hdev, u64 addr, u16 eng_id, bool is_pmmu);
4082  void hl_handle_page_fault(struct hl_device *hdev, u64 addr, u16 eng_id, bool is_pmmu,
4083  				u64 *event_mask);
4084  void hl_handle_critical_hw_err(struct hl_device *hdev, u16 event_id, u64 *event_mask);
4085  void hl_handle_fw_err(struct hl_device *hdev, struct hl_info_fw_err_info *info);
4086  void hl_capture_engine_err(struct hl_device *hdev, u16 engine_id, u16 error_count);
4087  void hl_enable_err_info_capture(struct hl_error_info *captured_err_info);
4088  void hl_init_cpu_for_irq(struct hl_device *hdev);
4089  void hl_set_irq_affinity(struct hl_device *hdev, int irq);
4090  void hl_eq_heartbeat_event_handle(struct hl_device *hdev);
4091  void hl_handle_clk_change_event(struct hl_device *hdev, u16 event_type, u64 *event_mask);
4092  
4093  #ifdef CONFIG_DEBUG_FS
4094  
4095  int hl_debugfs_device_init(struct hl_device *hdev);
4096  void hl_debugfs_device_fini(struct hl_device *hdev);
4097  void hl_debugfs_add_device(struct hl_device *hdev);
4098  void hl_debugfs_add_file(struct hl_fpriv *hpriv);
4099  void hl_debugfs_remove_file(struct hl_fpriv *hpriv);
4100  void hl_debugfs_add_cb(struct hl_cb *cb);
4101  void hl_debugfs_remove_cb(struct hl_cb *cb);
4102  void hl_debugfs_add_cs(struct hl_cs *cs);
4103  void hl_debugfs_remove_cs(struct hl_cs *cs);
4104  void hl_debugfs_add_job(struct hl_device *hdev, struct hl_cs_job *job);
4105  void hl_debugfs_remove_job(struct hl_device *hdev, struct hl_cs_job *job);
4106  void hl_debugfs_add_userptr(struct hl_device *hdev, struct hl_userptr *userptr);
4107  void hl_debugfs_remove_userptr(struct hl_device *hdev,
4108  				struct hl_userptr *userptr);
4109  void hl_debugfs_add_ctx_mem_hash(struct hl_device *hdev, struct hl_ctx *ctx);
4110  void hl_debugfs_remove_ctx_mem_hash(struct hl_device *hdev, struct hl_ctx *ctx);
4111  void hl_debugfs_set_state_dump(struct hl_device *hdev, char *data,
4112  					unsigned long length);
4113  
4114  #else
4115  
hl_debugfs_device_init(struct hl_device * hdev)4116  static inline int hl_debugfs_device_init(struct hl_device *hdev)
4117  {
4118  	return 0;
4119  }
4120  
hl_debugfs_device_fini(struct hl_device * hdev)4121  static inline void hl_debugfs_device_fini(struct hl_device *hdev)
4122  {
4123  }
4124  
hl_debugfs_add_device(struct hl_device * hdev)4125  static inline void hl_debugfs_add_device(struct hl_device *hdev)
4126  {
4127  }
4128  
hl_debugfs_add_file(struct hl_fpriv * hpriv)4129  static inline void hl_debugfs_add_file(struct hl_fpriv *hpriv)
4130  {
4131  }
4132  
hl_debugfs_remove_file(struct hl_fpriv * hpriv)4133  static inline void hl_debugfs_remove_file(struct hl_fpriv *hpriv)
4134  {
4135  }
4136  
hl_debugfs_add_cb(struct hl_cb * cb)4137  static inline void hl_debugfs_add_cb(struct hl_cb *cb)
4138  {
4139  }
4140  
hl_debugfs_remove_cb(struct hl_cb * cb)4141  static inline void hl_debugfs_remove_cb(struct hl_cb *cb)
4142  {
4143  }
4144  
hl_debugfs_add_cs(struct hl_cs * cs)4145  static inline void hl_debugfs_add_cs(struct hl_cs *cs)
4146  {
4147  }
4148  
hl_debugfs_remove_cs(struct hl_cs * cs)4149  static inline void hl_debugfs_remove_cs(struct hl_cs *cs)
4150  {
4151  }
4152  
hl_debugfs_add_job(struct hl_device * hdev,struct hl_cs_job * job)4153  static inline void hl_debugfs_add_job(struct hl_device *hdev,
4154  					struct hl_cs_job *job)
4155  {
4156  }
4157  
hl_debugfs_remove_job(struct hl_device * hdev,struct hl_cs_job * job)4158  static inline void hl_debugfs_remove_job(struct hl_device *hdev,
4159  					struct hl_cs_job *job)
4160  {
4161  }
4162  
hl_debugfs_add_userptr(struct hl_device * hdev,struct hl_userptr * userptr)4163  static inline void hl_debugfs_add_userptr(struct hl_device *hdev,
4164  					struct hl_userptr *userptr)
4165  {
4166  }
4167  
hl_debugfs_remove_userptr(struct hl_device * hdev,struct hl_userptr * userptr)4168  static inline void hl_debugfs_remove_userptr(struct hl_device *hdev,
4169  					struct hl_userptr *userptr)
4170  {
4171  }
4172  
hl_debugfs_add_ctx_mem_hash(struct hl_device * hdev,struct hl_ctx * ctx)4173  static inline void hl_debugfs_add_ctx_mem_hash(struct hl_device *hdev,
4174  					struct hl_ctx *ctx)
4175  {
4176  }
4177  
hl_debugfs_remove_ctx_mem_hash(struct hl_device * hdev,struct hl_ctx * ctx)4178  static inline void hl_debugfs_remove_ctx_mem_hash(struct hl_device *hdev,
4179  					struct hl_ctx *ctx)
4180  {
4181  }
4182  
hl_debugfs_set_state_dump(struct hl_device * hdev,char * data,unsigned long length)4183  static inline void hl_debugfs_set_state_dump(struct hl_device *hdev,
4184  					char *data, unsigned long length)
4185  {
4186  }
4187  
4188  #endif
4189  
4190  /* Security */
4191  int hl_unsecure_register(struct hl_device *hdev, u32 mm_reg_addr, int offset,
4192  		const u32 pb_blocks[], struct hl_block_glbl_sec sgs_array[],
4193  		int array_size);
4194  int hl_unsecure_registers(struct hl_device *hdev, const u32 mm_reg_array[],
4195  		int mm_array_size, int offset, const u32 pb_blocks[],
4196  		struct hl_block_glbl_sec sgs_array[], int blocks_array_size);
4197  void hl_config_glbl_sec(struct hl_device *hdev, const u32 pb_blocks[],
4198  		struct hl_block_glbl_sec sgs_array[], u32 block_offset,
4199  		int array_size);
4200  void hl_secure_block(struct hl_device *hdev,
4201  		struct hl_block_glbl_sec sgs_array[], int array_size);
4202  int hl_init_pb_with_mask(struct hl_device *hdev, u32 num_dcores,
4203  		u32 dcore_offset, u32 num_instances, u32 instance_offset,
4204  		const u32 pb_blocks[], u32 blocks_array_size,
4205  		const u32 *regs_array, u32 regs_array_size, u64 mask);
4206  int hl_init_pb(struct hl_device *hdev, u32 num_dcores, u32 dcore_offset,
4207  		u32 num_instances, u32 instance_offset,
4208  		const u32 pb_blocks[], u32 blocks_array_size,
4209  		const u32 *regs_array, u32 regs_array_size);
4210  int hl_init_pb_ranges_with_mask(struct hl_device *hdev, u32 num_dcores,
4211  		u32 dcore_offset, u32 num_instances, u32 instance_offset,
4212  		const u32 pb_blocks[], u32 blocks_array_size,
4213  		const struct range *regs_range_array, u32 regs_range_array_size,
4214  		u64 mask);
4215  int hl_init_pb_ranges(struct hl_device *hdev, u32 num_dcores,
4216  		u32 dcore_offset, u32 num_instances, u32 instance_offset,
4217  		const u32 pb_blocks[], u32 blocks_array_size,
4218  		const struct range *regs_range_array,
4219  		u32 regs_range_array_size);
4220  int hl_init_pb_single_dcore(struct hl_device *hdev, u32 dcore_offset,
4221  		u32 num_instances, u32 instance_offset,
4222  		const u32 pb_blocks[], u32 blocks_array_size,
4223  		const u32 *regs_array, u32 regs_array_size);
4224  int hl_init_pb_ranges_single_dcore(struct hl_device *hdev, u32 dcore_offset,
4225  		u32 num_instances, u32 instance_offset,
4226  		const u32 pb_blocks[], u32 blocks_array_size,
4227  		const struct range *regs_range_array,
4228  		u32 regs_range_array_size);
4229  void hl_ack_pb(struct hl_device *hdev, u32 num_dcores, u32 dcore_offset,
4230  		u32 num_instances, u32 instance_offset,
4231  		const u32 pb_blocks[], u32 blocks_array_size);
4232  void hl_ack_pb_with_mask(struct hl_device *hdev, u32 num_dcores,
4233  		u32 dcore_offset, u32 num_instances, u32 instance_offset,
4234  		const u32 pb_blocks[], u32 blocks_array_size, u64 mask);
4235  void hl_ack_pb_single_dcore(struct hl_device *hdev, u32 dcore_offset,
4236  		u32 num_instances, u32 instance_offset,
4237  		const u32 pb_blocks[], u32 blocks_array_size);
4238  
4239  /* IOCTLs */
4240  long hl_ioctl_control(struct file *filep, unsigned int cmd, unsigned long arg);
4241  int hl_info_ioctl(struct drm_device *ddev, void *data, struct drm_file *file_priv);
4242  int hl_cb_ioctl(struct drm_device *ddev, void *data, struct drm_file *file_priv);
4243  int hl_cs_ioctl(struct drm_device *ddev, void *data, struct drm_file *file_priv);
4244  int hl_wait_ioctl(struct drm_device *ddev, void *data, struct drm_file *file_priv);
4245  int hl_mem_ioctl(struct drm_device *ddev, void *data, struct drm_file *file_priv);
4246  int hl_debug_ioctl(struct drm_device *ddev, void *data, struct drm_file *file_priv);
4247  
4248  #endif /* HABANALABSP_H_ */
4249