1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 3 /* Copyright (c) 2019-2021, The Linux Foundation. All rights reserved. */ 4 /* Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ 5 6 #ifndef _QAIC_H_ 7 #define _QAIC_H_ 8 9 #include <linux/interrupt.h> 10 #include <linux/kref.h> 11 #include <linux/mhi.h> 12 #include <linux/mutex.h> 13 #include <linux/pci.h> 14 #include <linux/spinlock.h> 15 #include <linux/srcu.h> 16 #include <linux/wait.h> 17 #include <linux/workqueue.h> 18 #include <drm/drm_device.h> 19 #include <drm/drm_gem.h> 20 21 #define QAIC_DBC_BASE SZ_128K 22 #define QAIC_DBC_SIZE SZ_4K 23 #define QAIC_SSR_DBC_SENTINEL U32_MAX /* No ongoing SSR sentinel */ 24 25 #define QAIC_NO_PARTITION -1 26 27 #define QAIC_DBC_OFF(i) ((i) * QAIC_DBC_SIZE + QAIC_DBC_BASE) 28 29 #define to_qaic_bo(obj) container_of(obj, struct qaic_bo, base) 30 #define to_qaic_drm_device(dev) container_of(dev, struct qaic_drm_device, drm) 31 #define to_drm(qddev) (&(qddev)->drm) 32 #define to_accel_kdev(qddev) (to_drm(qddev)->accel->kdev) /* Return Linux device of accel node */ 33 #define to_qaic_device(dev) (to_qaic_drm_device((dev))->qdev) 34 35 enum aic_families { 36 FAMILY_AIC100, 37 FAMILY_AIC200, 38 FAMILY_MAX, 39 }; 40 41 enum __packed dev_states { 42 /* Device is offline or will be very soon */ 43 QAIC_OFFLINE, 44 /* Device is booting, not clear if it's in a usable state */ 45 QAIC_BOOT, 46 /* Device is fully operational */ 47 QAIC_ONLINE, 48 }; 49 50 enum dbc_states { 51 /* DBC is free and can be activated */ 52 DBC_STATE_IDLE, 53 /* DBC is activated and a workload is running on device */ 54 DBC_STATE_ASSIGNED, 55 /* Sub-system associated with this workload has crashed and it will shutdown soon */ 56 DBC_STATE_BEFORE_SHUTDOWN, 57 /* Sub-system associated with this workload has crashed and it has shutdown */ 58 DBC_STATE_AFTER_SHUTDOWN, 59 /* Sub-system associated with this workload is shutdown and it will be powered up soon */ 60 DBC_STATE_BEFORE_POWER_UP, 61 /* Sub-system associated with this workload is now powered up */ 62 DBC_STATE_AFTER_POWER_UP, 63 DBC_STATE_MAX, 64 }; 65 66 extern bool datapath_polling; 67 68 struct qaic_user { 69 /* Uniquely identifies this user for the device */ 70 int handle; 71 struct kref ref_count; 72 /* Char device opened by this user */ 73 struct qaic_drm_device *qddev; 74 /* Node in list of users that opened this drm device */ 75 struct list_head node; 76 /* SRCU used to synchronize this user during cleanup */ 77 struct srcu_struct qddev_lock; 78 atomic_t chunk_id; 79 }; 80 81 struct dma_bridge_chan { 82 /* Pointer to device strcut maintained by driver */ 83 struct qaic_device *qdev; 84 /* ID of this DMA bridge channel(DBC) */ 85 unsigned int id; 86 /* Synchronizes access to xfer_list */ 87 spinlock_t xfer_lock; 88 /* Base address of request queue */ 89 void *req_q_base; 90 /* Base address of response queue */ 91 void *rsp_q_base; 92 /* 93 * Base bus address of request queue. Response queue bus address can be 94 * calculated by adding request queue size to this variable 95 */ 96 dma_addr_t dma_addr; 97 /* Total size of request and response queue in byte */ 98 u32 total_size; 99 /* Capacity of request/response queue */ 100 u32 nelem; 101 /* The user that opened this DBC */ 102 struct qaic_user *usr; 103 /* 104 * Request ID of next memory handle that goes in request queue. One 105 * memory handle can enqueue more than one request elements, all 106 * this requests that belong to same memory handle have same request ID 107 */ 108 u16 next_req_id; 109 /* true: DBC is in use; false: DBC not in use */ 110 bool in_use; 111 /* 112 * Base address of device registers. Used to read/write request and 113 * response queue's head and tail pointer of this DBC. 114 */ 115 void __iomem *dbc_base; 116 /* Synchronizes access to Request queue's head and tail pointer */ 117 struct mutex req_lock; 118 /* Head of list where each node is a memory handle queued in request queue */ 119 struct list_head xfer_list; 120 /* Synchronizes DBC readers during cleanup */ 121 struct srcu_struct ch_lock; 122 /* 123 * When this DBC is released, any thread waiting on this wait queue is 124 * woken up 125 */ 126 wait_queue_head_t dbc_release; 127 /* Head of list where each node is a bo associated with this DBC */ 128 struct list_head bo_lists; 129 /* The irq line for this DBC. Used for polling */ 130 unsigned int irq; 131 /* Polling work item to simulate interrupts */ 132 struct work_struct poll_work; 133 /* Represents various states of this DBC from enum dbc_states */ 134 unsigned int state; 135 }; 136 137 struct qaic_device { 138 /* Pointer to base PCI device struct of our physical device */ 139 struct pci_dev *pdev; 140 /* Req. ID of request that will be queued next in MHI control device */ 141 u32 next_seq_num; 142 /* Base address of the MHI bar */ 143 void __iomem *bar_mhi; 144 /* Base address of the DBCs bar */ 145 void __iomem *bar_dbc; 146 /* Controller structure for MHI devices */ 147 struct mhi_controller *mhi_cntrl; 148 /* MHI control channel device */ 149 struct mhi_device *cntl_ch; 150 /* List of requests queued in MHI control device */ 151 struct list_head cntl_xfer_list; 152 /* Synchronizes MHI control device transactions and its xfer list */ 153 struct mutex cntl_mutex; 154 /* Work queue for tasks related to MHI control device */ 155 struct workqueue_struct *cntl_wq; 156 /* Synchronizes all the users of device during cleanup */ 157 struct srcu_struct dev_lock; 158 /* Track the state of the device during resets */ 159 enum dev_states dev_state; 160 /* true: single MSI is used to operate device */ 161 bool single_msi; 162 /* 163 * true: A tx MHI transaction has failed and a rx buffer is still queued 164 * in control device. Such a buffer is considered lost rx buffer 165 * false: No rx buffer is lost in control device 166 */ 167 bool cntl_lost_buf; 168 /* Maximum number of DBC supported by this device */ 169 u32 num_dbc; 170 /* Reference to the drm_device for this device when it is created */ 171 struct qaic_drm_device *qddev; 172 /* Generate the CRC of a control message */ 173 u32 (*gen_crc)(void *msg); 174 /* Validate the CRC of a control message */ 175 bool (*valid_crc)(void *msg); 176 /* MHI "QAIC_TIMESYNC" channel device */ 177 struct mhi_device *qts_ch; 178 /* Work queue for tasks related to MHI "QAIC_TIMESYNC" channel */ 179 struct workqueue_struct *qts_wq; 180 /* MHI "QAIC_TIMESYNC_PERIODIC" channel device */ 181 struct mhi_device *mqts_ch; 182 /* Head of list of page allocated by MHI bootlog device */ 183 struct list_head bootlog; 184 /* MHI bootlog channel device */ 185 struct mhi_device *bootlog_ch; 186 /* Work queue for tasks related to MHI bootlog device */ 187 struct workqueue_struct *bootlog_wq; 188 /* Synchronizes access of pages in MHI bootlog device */ 189 struct mutex bootlog_mutex; 190 /* MHI RAS channel device */ 191 struct mhi_device *ras_ch; 192 /* Correctable error count */ 193 unsigned int ce_count; 194 /* Un-correctable error count */ 195 unsigned int ue_count; 196 /* Un-correctable non-fatal error count */ 197 unsigned int ue_nf_count; 198 /* MHI SSR channel device */ 199 struct mhi_device *ssr_ch; 200 /* Work queue for tasks related to MHI SSR device */ 201 struct workqueue_struct *ssr_wq; 202 /* Buffer to collect SSR crashdump via SSR MHI channel */ 203 void *ssr_mhi_buf; 204 /* DBC which is under SSR. Sentinel U32_MAX would mean that no SSR in progress */ 205 u32 ssr_dbc; 206 /* Array of DBC struct of this device */ 207 struct dma_bridge_chan dbc[] __counted_by(num_dbc); 208 }; 209 210 struct qaic_drm_device { 211 /* The drm device struct of this drm device */ 212 struct drm_device drm; 213 /* Pointer to the root device struct driven by this driver */ 214 struct qaic_device *qdev; 215 /* 216 * The physical device can be partition in number of logical devices. 217 * And each logical device is given a partition id. This member stores 218 * that id. QAIC_NO_PARTITION is a sentinel used to mark that this drm 219 * device is the actual physical device 220 */ 221 s32 partition_id; 222 /* Head in list of users who have opened this drm device */ 223 struct list_head users; 224 /* Synchronizes access to users list */ 225 struct mutex users_mutex; 226 /* Pointer to array of DBC sysfs attributes */ 227 void *sysfs_attrs; 228 }; 229 230 struct qaic_bo { 231 struct drm_gem_object base; 232 /* Scatter/gather table for allocate/imported BO */ 233 struct sg_table *sgt; 234 /* Head in list of slices of this BO */ 235 struct list_head slices; 236 /* Total nents, for all slices of this BO */ 237 int total_slice_nents; 238 /* 239 * Direction of transfer. It can assume only two value DMA_TO_DEVICE and 240 * DMA_FROM_DEVICE. 241 */ 242 int dir; 243 /* The pointer of the DBC which operates on this BO */ 244 struct dma_bridge_chan *dbc; 245 /* Number of slice that belongs to this buffer */ 246 u32 nr_slice; 247 /* Number of slice that have been transferred by DMA engine */ 248 u32 nr_slice_xfer_done; 249 /* 250 * If true then user has attached slicing information to this BO by 251 * calling DRM_IOCTL_QAIC_ATTACH_SLICE_BO ioctl. 252 */ 253 bool sliced; 254 /* Request ID of this BO if it is queued for execution */ 255 u16 req_id; 256 /* Wait on this for completion of DMA transfer of this BO */ 257 struct completion xfer_done; 258 /* 259 * Node in linked list where head is dbc->xfer_list. 260 * This link list contain BO's that are queued for DMA transfer. 261 */ 262 struct list_head xfer_list; 263 /* 264 * Node in linked list where head is dbc->bo_lists. 265 * This link list contain BO's that are associated with the DBC it is 266 * linked to. 267 */ 268 struct list_head bo_list; 269 struct { 270 /* 271 * Latest timestamp(ns) at which kernel received a request to 272 * execute this BO 273 */ 274 u64 req_received_ts; 275 /* 276 * Latest timestamp(ns) at which kernel enqueued requests of 277 * this BO for execution in DMA queue 278 */ 279 u64 req_submit_ts; 280 /* 281 * Latest timestamp(ns) at which kernel received a completion 282 * interrupt for requests of this BO 283 */ 284 u64 req_processed_ts; 285 /* 286 * Number of elements already enqueued in DMA queue before 287 * enqueuing requests of this BO 288 */ 289 u32 queue_level_before; 290 } perf_stats; 291 /* Synchronizes BO operations */ 292 struct mutex lock; 293 }; 294 295 struct bo_slice { 296 /* Mapped pages */ 297 struct sg_table *sgt; 298 /* Number of requests required to queue in DMA queue */ 299 int nents; 300 /* See enum dma_data_direction */ 301 int dir; 302 /* Actual requests that will be copied in DMA queue */ 303 struct dbc_req *reqs; 304 struct kref ref_count; 305 /* true: No DMA transfer required */ 306 bool no_xfer; 307 /* Pointer to the parent BO handle */ 308 struct qaic_bo *bo; 309 /* Node in list of slices maintained by parent BO */ 310 struct list_head slice; 311 /* Size of this slice in bytes */ 312 u64 size; 313 /* Offset of this slice in buffer */ 314 u64 offset; 315 }; 316 317 int get_dbc_req_elem_size(void); 318 int get_dbc_rsp_elem_size(void); 319 int get_cntl_version(struct qaic_device *qdev, struct qaic_user *usr, u16 *major, u16 *minor); 320 int qaic_manage_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); 321 void qaic_mhi_ul_xfer_cb(struct mhi_device *mhi_dev, struct mhi_result *mhi_result); 322 323 void qaic_mhi_dl_xfer_cb(struct mhi_device *mhi_dev, struct mhi_result *mhi_result); 324 325 int qaic_control_open(struct qaic_device *qdev); 326 void qaic_control_close(struct qaic_device *qdev); 327 void qaic_release_usr(struct qaic_device *qdev, struct qaic_user *usr); 328 329 irqreturn_t dbc_irq_threaded_fn(int irq, void *data); 330 irqreturn_t dbc_irq_handler(int irq, void *data); 331 int disable_dbc(struct qaic_device *qdev, u32 dbc_id, struct qaic_user *usr); 332 void enable_dbc(struct qaic_device *qdev, u32 dbc_id, struct qaic_user *usr); 333 void wakeup_dbc(struct qaic_device *qdev, u32 dbc_id); 334 void release_dbc(struct qaic_device *qdev, u32 dbc_id); 335 void qaic_data_get_fifo_info(struct dma_bridge_chan *dbc, u32 *head, u32 *tail); 336 337 void wake_all_cntl(struct qaic_device *qdev); 338 void qaic_dev_reset_clean_local_state(struct qaic_device *qdev); 339 340 struct drm_gem_object *qaic_gem_prime_import(struct drm_device *dev, struct dma_buf *dma_buf); 341 342 int qaic_create_bo_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); 343 int qaic_mmap_bo_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); 344 int qaic_attach_slice_bo_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); 345 int qaic_execute_bo_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); 346 int qaic_partial_execute_bo_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); 347 int qaic_wait_bo_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); 348 int qaic_perf_stats_bo_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); 349 int qaic_detach_slice_bo_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); 350 void qaic_irq_polling_work(struct work_struct *work); 351 void qaic_dbc_enter_ssr(struct qaic_device *qdev, u32 dbc_id); 352 void qaic_dbc_exit_ssr(struct qaic_device *qdev); 353 354 /* qaic_sysfs.c */ 355 int qaic_sysfs_init(struct qaic_drm_device *qddev); 356 void qaic_sysfs_remove(struct qaic_drm_device *qddev); 357 void set_dbc_state(struct qaic_device *qdev, u32 dbc_id, unsigned int state); 358 359 #endif /* _QAIC_H_ */ 360