1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * Copyright (C) 2023-2026, Advanced Micro Devices, Inc. 4 */ 5 6 #ifndef _AIE2_PCI_H_ 7 #define _AIE2_PCI_H_ 8 9 #include <drm/amdxdna_accel.h> 10 #include <linux/limits.h> 11 #include <linux/semaphore.h> 12 13 #include "aie.h" 14 #include "aie2_msg_priv.h" 15 #include "amdxdna_mailbox.h" 16 17 /* Firmware determines device memory base address and size */ 18 #define AIE2_DEVM_BASE 0x4000000 19 #define AIE2_DEVM_SIZE SZ_64M 20 #define AIE2_DEVM_MAX_SIZE SZ_512M 21 22 #define NDEV2PDEV(ndev) (to_pci_dev((ndev)->aie.xdna->ddev.dev)) 23 24 #define AIE2_SRAM_OFF(ndev, addr) ((addr) - (ndev)->priv->sram_dev_addr) 25 #define AIE2_MBOX_OFF(ndev, addr) ((addr) - (ndev)->priv->mbox_dev_addr) 26 27 #define SRAM_REG_OFF(ndev, idx) ((ndev)->priv->sram_offs[(idx)].offset) 28 29 #define SRAM_GET_ADDR(ndev, idx) \ 30 ({ \ 31 typeof(ndev) _ndev = ndev; \ 32 ((_ndev)->sram_base + SRAM_REG_OFF((_ndev), (idx))); \ 33 }) 34 35 #define CHAN_SLOT_SZ SZ_8K 36 #define MBOX_SIZE(ndev) \ 37 ({ \ 38 typeof(ndev) _ndev = (ndev); \ 39 ((_ndev)->priv->mbox_size) ? (_ndev)->priv->mbox_size : \ 40 pci_resource_len(NDEV2PDEV(_ndev), (_ndev)->aie.xdna->dev_info->mbox_bar); \ 41 }) 42 43 #if IS_ENABLED(CONFIG_AMD_PMF) 44 #define AIE2_GET_PMF_NPU_METRICS(metrics) amd_pmf_get_npu_data(metrics) 45 #define AIE2_GET_PMF_NPU_DATA(field, val) \ 46 ({ \ 47 struct amd_pmf_npu_metrics _npu_metrics; \ 48 int _ret; \ 49 \ 50 _ret = amd_pmf_get_npu_data(&_npu_metrics); \ 51 val = _ret ? U32_MAX : _npu_metrics.field; \ 52 (_ret); \ 53 }) 54 #else 55 #define AIE2_GET_PMF_NPU_METRICS(metrics) \ 56 ({ \ 57 typeof(metrics) _m = metrics; \ 58 memset(_m, 0xff, sizeof(*_m)); \ 59 (-EOPNOTSUPP); \ 60 }) 61 62 #define SENSOR_DEFAULT_npu_power U32_MAX 63 #define AIE2_GET_PMF_NPU_DATA(field, val) \ 64 ({ \ 65 val = SENSOR_DEFAULT_##field; \ 66 (-EOPNOTSUPP); \ 67 }) 68 #endif 69 70 enum aie2_sram_reg_idx { 71 MBOX_CHANN_OFF = 0, 72 FW_ALIVE_OFF, 73 SRAM_MAX_INDEX /* Keep this at the end */ 74 }; 75 76 struct amdxdna_client; 77 struct amdxdna_fw_ver; 78 struct amdxdna_hwctx; 79 struct amdxdna_sched_job; 80 81 enum rt_config_category { 82 AIE2_RT_CFG_INIT, 83 AIE2_RT_CFG_CLK_GATING, 84 AIE2_RT_CFG_FORCE_PREEMPT, 85 AIE2_RT_CFG_FRAME_BOUNDARY_PREEMPT, 86 }; 87 88 struct rt_config { 89 u32 type; 90 u32 value; 91 u32 category; 92 unsigned long feature_mask; 93 }; 94 95 struct dpm_clk_freq { 96 u32 npuclk; 97 u32 hclk; 98 }; 99 100 /* 101 * Define the maximum number of pending commands in a hardware context. 102 * Must be power of 2! 103 */ 104 #define HWCTX_MAX_CMDS 4 105 #define get_job_idx(seq) ((seq) & (HWCTX_MAX_CMDS - 1)) 106 struct amdxdna_hwctx_priv { 107 struct amdxdna_gem_obj *heap; 108 void *mbox_chann; 109 110 struct drm_gpu_scheduler sched; 111 struct drm_sched_entity entity; 112 113 struct mutex io_lock; /* protect seq and cmd order */ 114 struct wait_queue_head job_free_wq; 115 u32 num_pending; 116 u64 seq; 117 struct semaphore job_sem; 118 bool job_done; 119 120 /* Completed job counter */ 121 u64 completed; 122 123 struct amdxdna_gem_obj *cmd_buf[HWCTX_MAX_CMDS]; 124 struct drm_syncobj *syncobj; 125 }; 126 127 enum aie2_dev_status { 128 AIE2_DEV_UNINIT, 129 AIE2_DEV_INIT, 130 AIE2_DEV_START, 131 }; 132 133 struct aie2_exec_msg_ops { 134 int (*init_cu_req)(struct amdxdna_gem_obj *cmd_bo, void *req, 135 size_t *size, u32 *msg_op); 136 int (*init_dpu_req)(struct amdxdna_gem_obj *cmd_bo, void *req, 137 size_t *size, u32 *msg_op); 138 void (*init_chain_req)(void *req, u64 slot_addr, size_t size, u32 cmd_cnt); 139 int (*fill_cf_slot)(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *size); 140 int (*fill_dpu_slot)(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *size); 141 int (*fill_preempt_slot)(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *size); 142 int (*fill_elf_slot)(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *size); 143 u32 (*get_chain_msg_op)(u32 cmd_op); 144 }; 145 146 enum aie2_tdr_status { 147 AIE2_TDR_WAIT, 148 AIE2_TDR_SIGNALED, 149 }; 150 151 struct amdxdna_dev_hdl { 152 struct aie_device aie; 153 const struct amdxdna_dev_priv *priv; 154 void __iomem *sram_base; 155 void __iomem *mbox_base; 156 157 u32 total_col; 158 struct amdxdna_drm_query_aie_version version; 159 struct aie2_exec_msg_ops *exec_msg_ops; 160 161 /* power management and clock*/ 162 enum amdxdna_power_mode_type pw_mode; 163 u32 dpm_level; 164 u32 dft_dpm_level; 165 u32 max_dpm_level; 166 u32 clk_gating; 167 u32 npuclk_freq; 168 u32 hclk_freq; 169 u32 max_tops; 170 u32 curr_tops; 171 u32 force_preempt_enabled; 172 u32 frame_boundary_preempt; 173 174 /* Mailbox and the management channel */ 175 struct mailbox *mbox; 176 struct async_events *async_events; 177 178 enum aie2_dev_status dev_status; 179 u32 hwctx_num; 180 181 struct amdxdna_async_error last_async_err; 182 enum aie2_tdr_status tdr_status; 183 }; 184 185 struct aie2_hw_ops { 186 int (*set_dpm)(struct amdxdna_dev_hdl *ndev, u32 dpm_level); 187 int (*update_counters)(struct amdxdna_dev_hdl *ndev); 188 }; 189 190 #define aie2_update_counters(ndev) \ 191 ({ \ 192 typeof(ndev) _ndev = ndev; \ 193 if (_ndev->priv->hw_ops->update_counters) \ 194 _ndev->priv->hw_ops->update_counters(_ndev); \ 195 }) 196 197 enum aie2_fw_feature { 198 AIE2_NPU_COMMAND, 199 AIE2_PREEMPT, 200 AIE2_TEMPORAL_ONLY, 201 AIE2_APP_HEALTH, 202 AIE2_ADD_HOST_BUFFER, 203 AIE2_UPDATE_PROPERTY, 204 AIE2_GET_DEV_REVISION, 205 AIE2_FEATURE_MAX 206 }; 207 208 #define AIE2_ALL_FEATURES GENMASK_ULL(AIE2_FEATURE_MAX - 1, AIE2_NPU_COMMAND) 209 210 struct amdxdna_dev_priv { 211 const char *fw_path; 212 const struct rt_config *rt_config; 213 const struct dpm_clk_freq *dpm_clk_tbl; 214 215 #define COL_ALIGN_NONE 0 216 #define COL_ALIGN_NATURE 1 217 u32 col_align; 218 u32 col_opc; 219 u32 mbox_dev_addr; 220 /* If mbox_size is 0, use BAR size. See MBOX_SIZE macro */ 221 u32 mbox_size; 222 u32 hwctx_limit; 223 u32 sram_dev_addr; 224 struct aie_bar_off_pair sram_offs[SRAM_MAX_INDEX]; 225 struct aie_bar_off_pair psp_regs_off[PSP_MAX_REGS]; 226 struct aie_bar_off_pair smu_regs_off[SMU_MAX_REGS]; 227 const struct aie2_hw_ops *hw_ops; 228 }; 229 230 extern const struct amdxdna_dev_ops aie2_ops; 231 232 int aie2_runtime_cfg(struct amdxdna_dev_hdl *ndev, 233 enum rt_config_category category, u32 *val); 234 235 /* aie2 npu hw config */ 236 extern const struct dpm_clk_freq npu1_dpm_clk_table[]; 237 extern const struct dpm_clk_freq npu4_dpm_clk_table[]; 238 extern const struct rt_config npu1_default_rt_cfg[]; 239 extern const struct rt_config npu4_default_rt_cfg[]; 240 extern const struct amdxdna_fw_feature_tbl npu4_fw_feature_table[]; 241 extern const struct amdxdna_rev_vbnv npu4_rev_vbnv_tbl[]; 242 extern const struct aie2_hw_ops npu4_hw_ops; 243 244 /* aie2_pm.c */ 245 int aie2_pm_init(struct amdxdna_dev_hdl *ndev); 246 int aie2_pm_set_mode(struct amdxdna_dev_hdl *ndev, enum amdxdna_power_mode_type target); 247 int aie2_pm_set_dpm(struct amdxdna_dev_hdl *ndev, u32 dpm_level); 248 249 /* aie2_error.c */ 250 int aie2_error_async_events_alloc(struct amdxdna_dev_hdl *ndev); 251 void aie2_error_async_events_free(struct amdxdna_dev_hdl *ndev); 252 int aie2_error_async_msg_thread(void *data); 253 int aie2_get_array_async_error(struct amdxdna_dev_hdl *ndev, 254 struct amdxdna_drm_get_array *args); 255 256 /* aie2_message.c */ 257 void aie2_msg_init(struct amdxdna_dev_hdl *ndev); 258 void aie2_destroy_mgmt_chann(struct amdxdna_dev_hdl *ndev); 259 int aie2_suspend_fw(struct amdxdna_dev_hdl *ndev); 260 int aie2_resume_fw(struct amdxdna_dev_hdl *ndev); 261 int aie2_set_runtime_cfg(struct amdxdna_dev_hdl *ndev, u32 type, u64 value); 262 int aie2_get_runtime_cfg(struct amdxdna_dev_hdl *ndev, u32 type, u64 *value); 263 int aie2_assign_mgmt_pasid(struct amdxdna_dev_hdl *ndev, u16 pasid); 264 int aie2_query_aie_version(struct amdxdna_dev_hdl *ndev, 265 struct amdxdna_drm_query_aie_version *version); 266 int aie2_query_aie_metadata(struct amdxdna_dev_hdl *ndev, 267 struct amdxdna_drm_query_aie_metadata *metadata); 268 int aie2_query_firmware_version(struct amdxdna_dev_hdl *ndev, 269 struct amdxdna_fw_ver *fw_ver); 270 int aie2_query_app_health(struct amdxdna_dev_hdl *ndev, u32 context_id, 271 struct app_health_report *report); 272 int aie2_get_dev_revision(struct amdxdna_dev_hdl *ndev, enum aie2_dev_revision *rev); 273 int aie2_create_context(struct amdxdna_dev_hdl *ndev, struct amdxdna_hwctx *hwctx); 274 int aie2_destroy_context(struct amdxdna_dev_hdl *ndev, struct amdxdna_hwctx *hwctx); 275 int aie2_map_host_buf(struct amdxdna_dev_hdl *ndev, u32 context_id, u64 addr, u64 size); 276 int aie2_add_host_buf(struct amdxdna_dev_hdl *ndev, u32 context_id, u64 addr, u64 size); 277 int aie2_query_status(struct amdxdna_dev_hdl *ndev, char __user *buf, u32 size, u32 *cols_filled); 278 int aie2_query_telemetry(struct amdxdna_dev_hdl *ndev, 279 char __user *buf, u32 size, 280 struct amdxdna_drm_query_telemetry_header *header); 281 int aie2_register_asyn_event_msg(struct amdxdna_dev_hdl *ndev, dma_addr_t addr, u32 size, 282 void *handle, int (*cb)(void*, void __iomem *, size_t)); 283 int aie2_config_cu(struct amdxdna_hwctx *hwctx, 284 int (*notify_cb)(void *, void __iomem *, size_t)); 285 int aie2_execbuf(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job, 286 int (*notify_cb)(void *, void __iomem *, size_t)); 287 int aie2_cmdlist_single_execbuf(struct amdxdna_hwctx *hwctx, 288 struct amdxdna_sched_job *job, 289 int (*notify_cb)(void *, void __iomem *, size_t)); 290 int aie2_cmdlist_multi_execbuf(struct amdxdna_hwctx *hwctx, 291 struct amdxdna_sched_job *job, 292 int (*notify_cb)(void *, void __iomem *, size_t)); 293 int aie2_sync_bo(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job, 294 int (*notify_cb)(void *, void __iomem *, size_t)); 295 int aie2_config_debug_bo(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job, 296 int (*notify_cb)(void *, void __iomem *, size_t)); 297 int aie2_update_prop_time_quota(struct amdxdna_dev_hdl *ndev, u32 us); 298 299 /* aie2_hwctx.c */ 300 int aie2_hwctx_init(struct amdxdna_hwctx *hwctx); 301 void aie2_hwctx_fini(struct amdxdna_hwctx *hwctx); 302 int aie2_hwctx_config(struct amdxdna_hwctx *hwctx, u32 type, u64 value, void *buf, u32 size); 303 int aie2_hwctx_sync_debug_bo(struct amdxdna_hwctx *hwctx, u32 debug_bo_hdl); 304 void aie2_hwctx_suspend(struct amdxdna_client *client); 305 int aie2_hwctx_resume(struct amdxdna_client *client); 306 int aie2_cmd_submit(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job, u64 *seq); 307 void aie2_hmm_invalidate(struct amdxdna_gem_obj *abo, unsigned long cur_seq); 308 int aie2_hwctx_heap_expand(struct amdxdna_hwctx *hwctx, struct amdxdna_gem_obj *heap); 309 310 #endif /* _AIE2_PCI_H_ */ 311