1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * Copyright (C) 2023-2026, Advanced Micro Devices, Inc. 4 */ 5 6 #ifndef _AIE2_PCI_H_ 7 #define _AIE2_PCI_H_ 8 9 #include <drm/amdxdna_accel.h> 10 #include <linux/limits.h> 11 #include <linux/semaphore.h> 12 13 #include "aie.h" 14 #include "aie2_msg_priv.h" 15 #include "amdxdna_mailbox.h" 16 17 /* Firmware determines device memory base address and size */ 18 #define AIE2_DEVM_BASE 0x4000000 19 #define AIE2_DEVM_SIZE SZ_64M 20 #define AIE2_DEVM_MAX_SIZE SZ_512M 21 22 #define NDEV2PDEV(ndev) (to_pci_dev((ndev)->aie.xdna->ddev.dev)) 23 24 #define AIE2_SRAM_OFF(ndev, addr) ((addr) - (ndev)->priv->sram_dev_addr) 25 #define AIE2_MBOX_OFF(ndev, addr) ((addr) - (ndev)->priv->mbox_dev_addr) 26 27 #define SRAM_REG_OFF(ndev, idx) ((ndev)->priv->sram_offs[(idx)].offset) 28 29 #define SRAM_GET_ADDR(ndev, idx) \ 30 ({ \ 31 typeof(ndev) _ndev = ndev; \ 32 ((_ndev)->sram_base + SRAM_REG_OFF((_ndev), (idx))); \ 33 }) 34 35 #define CHAN_SLOT_SZ SZ_8K 36 #define MBOX_SIZE(ndev) \ 37 ({ \ 38 typeof(ndev) _ndev = (ndev); \ 39 ((_ndev)->priv->mbox_size) ? (_ndev)->priv->mbox_size : \ 40 pci_resource_len(NDEV2PDEV(_ndev), (_ndev)->aie.xdna->dev_info->mbox_bar); \ 41 }) 42 43 #if IS_ENABLED(CONFIG_AMD_PMF) 44 #define AIE2_GET_PMF_NPU_METRICS(metrics) amd_pmf_get_npu_data(metrics) 45 #define AIE2_GET_PMF_NPU_DATA(field, val) \ 46 ({ \ 47 struct amd_pmf_npu_metrics _npu_metrics; \ 48 int _ret; \ 49 \ 50 _ret = amd_pmf_get_npu_data(&_npu_metrics); \ 51 val = _ret ? U32_MAX : _npu_metrics.field; \ 52 (_ret); \ 53 }) 54 #else 55 #define AIE2_GET_PMF_NPU_METRICS(metrics) \ 56 ({ \ 57 typeof(metrics) _m = metrics; \ 58 memset(_m, 0xff, sizeof(*_m)); \ 59 (-EOPNOTSUPP); \ 60 }) 61 62 #define SENSOR_DEFAULT_npu_power U32_MAX 63 #define AIE2_GET_PMF_NPU_DATA(field, val) \ 64 ({ \ 65 val = SENSOR_DEFAULT_##field; \ 66 (-EOPNOTSUPP); \ 67 }) 68 #endif 69 70 enum aie2_sram_reg_idx { 71 MBOX_CHANN_OFF = 0, 72 FW_ALIVE_OFF, 73 SRAM_MAX_INDEX /* Keep this at the end */ 74 }; 75 76 struct amdxdna_client; 77 struct amdxdna_fw_ver; 78 struct amdxdna_hwctx; 79 struct amdxdna_sched_job; 80 81 enum rt_config_category { 82 AIE2_RT_CFG_INIT, 83 AIE2_RT_CFG_CLK_GATING, 84 AIE2_RT_CFG_FORCE_PREEMPT, 85 AIE2_RT_CFG_FRAME_BOUNDARY_PREEMPT, 86 }; 87 88 struct rt_config { 89 u32 type; 90 u32 value; 91 u32 category; 92 unsigned long feature_mask; 93 }; 94 95 struct dpm_clk_freq { 96 u32 npuclk; 97 u32 hclk; 98 }; 99 100 /* 101 * Define the maximum number of pending commands in a hardware context. 102 * Must be power of 2! 103 */ 104 #define HWCTX_MAX_CMDS 4 105 #define get_job_idx(seq) ((seq) & (HWCTX_MAX_CMDS - 1)) 106 struct amdxdna_hwctx_priv { 107 struct amdxdna_gem_obj *heap; 108 void *mbox_chann; 109 110 struct drm_gpu_scheduler *sched; 111 struct drm_sched_entity entity; 112 113 struct mutex io_lock; /* protect seq and cmd order */ 114 struct wait_queue_head job_free_wq; 115 u32 num_pending; 116 u64 seq; 117 struct semaphore job_sem; 118 bool job_done; 119 120 /* Completed job counter */ 121 u64 completed; 122 123 struct amdxdna_gem_obj *cmd_buf[HWCTX_MAX_CMDS]; 124 struct drm_syncobj *syncobj; 125 }; 126 127 enum aie2_dev_status { 128 AIE2_DEV_UNINIT, 129 AIE2_DEV_INIT, 130 AIE2_DEV_START, 131 }; 132 133 struct aie2_exec_msg_ops { 134 int (*init_cu_req)(struct amdxdna_gem_obj *cmd_bo, void *req, 135 size_t *size, u32 *msg_op); 136 int (*init_dpu_req)(struct amdxdna_gem_obj *cmd_bo, void *req, 137 size_t *size, u32 *msg_op); 138 void (*init_chain_req)(void *req, u64 slot_addr, size_t size, u32 cmd_cnt); 139 int (*fill_cf_slot)(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *size); 140 int (*fill_dpu_slot)(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *size); 141 int (*fill_preempt_slot)(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *size); 142 int (*fill_elf_slot)(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *size); 143 u32 (*get_chain_msg_op)(u32 cmd_op); 144 }; 145 146 struct amdxdna_dev_hdl { 147 struct aie_device aie; 148 const struct amdxdna_dev_priv *priv; 149 void __iomem *sram_base; 150 void __iomem *mbox_base; 151 152 u32 total_col; 153 struct amdxdna_drm_query_aie_version version; 154 struct aie2_exec_msg_ops *exec_msg_ops; 155 struct drm_gpu_scheduler *hwctx_sched; 156 struct ida hwctx_sched_ida; 157 158 /* power management and clock*/ 159 enum amdxdna_power_mode_type pw_mode; 160 u32 dpm_level; 161 u32 dft_dpm_level; 162 u32 max_dpm_level; 163 u32 clk_gating; 164 u32 npuclk_freq; 165 u32 hclk_freq; 166 u32 max_tops; 167 u32 curr_tops; 168 u32 force_preempt_enabled; 169 u32 frame_boundary_preempt; 170 171 /* Mailbox and the management channel */ 172 struct mailbox *mbox; 173 struct async_events *async_events; 174 175 enum aie2_dev_status dev_status; 176 u32 hwctx_num; 177 178 struct amdxdna_async_error last_async_err; 179 unsigned long last_signal_ts; 180 }; 181 182 struct aie2_hw_ops { 183 int (*set_dpm)(struct amdxdna_dev_hdl *ndev, u32 dpm_level); 184 int (*update_counters)(struct amdxdna_dev_hdl *ndev); 185 }; 186 187 #define aie2_update_counters(ndev) \ 188 ({ \ 189 typeof(ndev) _ndev = ndev; \ 190 if (_ndev->priv->hw_ops->update_counters) \ 191 _ndev->priv->hw_ops->update_counters(_ndev); \ 192 }) 193 194 enum aie2_fw_feature { 195 AIE2_NPU_COMMAND, 196 AIE2_PREEMPT, 197 AIE2_TEMPORAL_ONLY, 198 AIE2_APP_HEALTH, 199 AIE2_ADD_HOST_BUFFER, 200 AIE2_UPDATE_PROPERTY, 201 AIE2_GET_DEV_REVISION, 202 AIE2_FEATURE_MAX 203 }; 204 205 #define AIE2_ALL_FEATURES GENMASK_ULL(AIE2_FEATURE_MAX - 1, AIE2_NPU_COMMAND) 206 207 struct amdxdna_dev_priv { 208 const char *fw_path; 209 const struct rt_config *rt_config; 210 const struct dpm_clk_freq *dpm_clk_tbl; 211 212 #define COL_ALIGN_NONE 0 213 #define COL_ALIGN_NATURE 1 214 u32 col_align; 215 u32 col_opc; 216 u32 mbox_dev_addr; 217 /* If mbox_size is 0, use BAR size. See MBOX_SIZE macro */ 218 u32 mbox_size; 219 u32 hwctx_limit; 220 u32 sram_dev_addr; 221 struct aie_bar_off_pair sram_offs[SRAM_MAX_INDEX]; 222 struct aie_bar_off_pair psp_regs_off[PSP_MAX_REGS]; 223 struct aie_bar_off_pair smu_regs_off[SMU_MAX_REGS]; 224 const struct aie2_hw_ops *hw_ops; 225 }; 226 227 extern const struct amdxdna_dev_ops aie2_ops; 228 229 int aie2_runtime_cfg(struct amdxdna_dev_hdl *ndev, 230 enum rt_config_category category, u32 *val); 231 232 /* aie2 npu hw config */ 233 extern const struct dpm_clk_freq npu1_dpm_clk_table[]; 234 extern const struct dpm_clk_freq npu4_dpm_clk_table[]; 235 extern const struct rt_config npu1_default_rt_cfg[]; 236 extern const struct rt_config npu4_default_rt_cfg[]; 237 extern const struct amdxdna_fw_feature_tbl npu4_fw_feature_table[]; 238 extern const struct amdxdna_rev_vbnv npu4_rev_vbnv_tbl[]; 239 extern const struct aie2_hw_ops npu4_hw_ops; 240 241 /* aie2_pm.c */ 242 int aie2_pm_init(struct amdxdna_dev_hdl *ndev); 243 int aie2_pm_set_mode(struct amdxdna_dev_hdl *ndev, enum amdxdna_power_mode_type target); 244 int aie2_pm_set_dpm(struct amdxdna_dev_hdl *ndev, u32 dpm_level); 245 246 /* aie2_error.c */ 247 int aie2_error_async_events_alloc(struct amdxdna_dev_hdl *ndev, u32 num_events); 248 void aie2_error_async_events_free(struct amdxdna_dev_hdl *ndev); 249 int aie2_error_async_msg_thread(void *data); 250 int aie2_get_array_async_error(struct amdxdna_dev_hdl *ndev, 251 struct amdxdna_drm_get_array *args); 252 253 /* aie2_message.c */ 254 void aie2_msg_init(struct amdxdna_dev_hdl *ndev); 255 void aie2_destroy_mgmt_chann(struct amdxdna_dev_hdl *ndev); 256 int aie2_suspend_fw(struct amdxdna_dev_hdl *ndev); 257 int aie2_resume_fw(struct amdxdna_dev_hdl *ndev); 258 int aie2_set_runtime_cfg(struct amdxdna_dev_hdl *ndev, u32 type, u64 value); 259 int aie2_get_runtime_cfg(struct amdxdna_dev_hdl *ndev, u32 type, u64 *value); 260 int aie2_assign_mgmt_pasid(struct amdxdna_dev_hdl *ndev, u16 pasid); 261 int aie2_query_aie_version(struct amdxdna_dev_hdl *ndev, 262 struct amdxdna_drm_query_aie_version *version); 263 int aie2_query_aie_metadata(struct amdxdna_dev_hdl *ndev, 264 struct amdxdna_drm_query_aie_metadata *metadata); 265 int aie2_query_firmware_version(struct amdxdna_dev_hdl *ndev, 266 struct amdxdna_fw_ver *fw_ver); 267 int aie2_query_app_health(struct amdxdna_dev_hdl *ndev, u32 context_id, 268 struct app_health_report *report); 269 int aie2_get_dev_revision(struct amdxdna_dev_hdl *ndev, enum aie2_dev_revision *rev); 270 int aie2_create_context(struct amdxdna_dev_hdl *ndev, struct amdxdna_hwctx *hwctx); 271 int aie2_destroy_context(struct amdxdna_dev_hdl *ndev, struct amdxdna_hwctx *hwctx); 272 int aie2_map_host_buf(struct amdxdna_dev_hdl *ndev, u32 context_id, u64 addr, u64 size); 273 int aie2_add_host_buf(struct amdxdna_dev_hdl *ndev, u32 context_id, u64 addr, u64 size); 274 int aie2_query_status(struct amdxdna_dev_hdl *ndev, char __user *buf, u32 size, u32 *cols_filled); 275 int aie2_query_telemetry(struct amdxdna_dev_hdl *ndev, 276 char __user *buf, u32 size, 277 struct amdxdna_drm_query_telemetry_header *header); 278 int aie2_register_asyn_event_msg(struct amdxdna_dev_hdl *ndev, dma_addr_t addr, u32 size, 279 void *handle, int (*cb)(void*, void __iomem *, size_t)); 280 int aie2_config_cu(struct amdxdna_hwctx *hwctx, 281 int (*notify_cb)(void *, void __iomem *, size_t)); 282 int aie2_execbuf(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job, 283 int (*notify_cb)(void *, void __iomem *, size_t)); 284 int aie2_cmdlist_single_execbuf(struct amdxdna_hwctx *hwctx, 285 struct amdxdna_sched_job *job, 286 int (*notify_cb)(void *, void __iomem *, size_t)); 287 int aie2_cmdlist_multi_execbuf(struct amdxdna_hwctx *hwctx, 288 struct amdxdna_sched_job *job, 289 int (*notify_cb)(void *, void __iomem *, size_t)); 290 int aie2_sync_bo(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job, 291 int (*notify_cb)(void *, void __iomem *, size_t)); 292 int aie2_config_debug_bo(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job, 293 int (*notify_cb)(void *, void __iomem *, size_t)); 294 int aie2_update_prop_time_quota(struct amdxdna_dev_hdl *ndev, u32 us); 295 296 /* aie2_hwctx.c */ 297 int aie2_hwctx_init(struct amdxdna_hwctx *hwctx); 298 void aie2_hwctx_fini(struct amdxdna_hwctx *hwctx); 299 int aie2_hwctx_sched_init(struct amdxdna_dev_hdl *ndev); 300 void aie2_hwctx_sched_fini(struct amdxdna_dev_hdl *ndev); 301 int aie2_hwctx_config(struct amdxdna_hwctx *hwctx, u32 type, u64 value, void *buf, u32 size); 302 int aie2_hwctx_sync_debug_bo(struct amdxdna_hwctx *hwctx, u32 debug_bo_hdl); 303 void aie2_hwctx_suspend(struct amdxdna_client *client); 304 int aie2_hwctx_resume(struct amdxdna_client *client); 305 int aie2_cmd_submit(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job, u64 *seq); 306 void aie2_hmm_invalidate(struct amdxdna_gem_obj *abo, unsigned long cur_seq); 307 int aie2_hwctx_heap_expand(struct amdxdna_hwctx *hwctx, struct amdxdna_gem_obj *heap); 308 309 #endif /* _AIE2_PCI_H_ */ 310