1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * Copyright (C) 2023-2026, Advanced Micro Devices, Inc. 4 */ 5 6 #ifndef _AIE2_PCI_H_ 7 #define _AIE2_PCI_H_ 8 9 #include <drm/amdxdna_accel.h> 10 #include <linux/limits.h> 11 #include <linux/semaphore.h> 12 13 #include "aie.h" 14 #include "aie2_msg_priv.h" 15 #include "amdxdna_mailbox.h" 16 17 /* Firmware determines device memory base address and size */ 18 #define AIE2_DEVM_BASE 0x4000000 19 #define AIE2_DEVM_SIZE SZ_64M 20 #define AIE2_DEVM_MAX_SIZE SZ_512M 21 22 #define NDEV2PDEV(ndev) (to_pci_dev((ndev)->aie.xdna->ddev.dev)) 23 24 #define AIE2_SRAM_OFF(ndev, addr) ((addr) - (ndev)->priv->sram_dev_addr) 25 #define AIE2_MBOX_OFF(ndev, addr) ((addr) - (ndev)->priv->mbox_dev_addr) 26 27 #define SRAM_REG_OFF(ndev, idx) ((ndev)->priv->sram_offs[(idx)].offset) 28 29 #define SRAM_GET_ADDR(ndev, idx) \ 30 ({ \ 31 typeof(ndev) _ndev = ndev; \ 32 ((_ndev)->sram_base + SRAM_REG_OFF((_ndev), (idx))); \ 33 }) 34 35 #define CHAN_SLOT_SZ SZ_8K 36 #define MBOX_SIZE(ndev) \ 37 ({ \ 38 typeof(ndev) _ndev = (ndev); \ 39 ((_ndev)->priv->mbox_size) ? (_ndev)->priv->mbox_size : \ 40 pci_resource_len(NDEV2PDEV(_ndev), (_ndev)->aie.xdna->dev_info->mbox_bar); \ 41 }) 42 43 #if IS_ENABLED(CONFIG_AMD_PMF) 44 #define AIE2_GET_PMF_NPU_METRICS(metrics) amd_pmf_get_npu_data(metrics) 45 #define AIE2_GET_PMF_NPU_DATA(field, val) \ 46 ({ \ 47 struct amd_pmf_npu_metrics _npu_metrics; \ 48 int _ret; \ 49 \ 50 _ret = amd_pmf_get_npu_data(&_npu_metrics); \ 51 val = _ret ? U32_MAX : _npu_metrics.field; \ 52 (_ret); \ 53 }) 54 #else 55 #define AIE2_GET_PMF_NPU_METRICS(metrics) \ 56 ({ \ 57 typeof(metrics) _m = metrics; \ 58 memset(_m, 0xff, sizeof(*_m)); \ 59 (-EOPNOTSUPP); \ 60 }) 61 62 #define SENSOR_DEFAULT_npu_power U32_MAX 63 #define AIE2_GET_PMF_NPU_DATA(field, val) \ 64 ({ \ 65 val = SENSOR_DEFAULT_##field; \ 66 (-EOPNOTSUPP); \ 67 }) 68 #endif 69 70 enum aie2_sram_reg_idx { 71 MBOX_CHANN_OFF = 0, 72 FW_ALIVE_OFF, 73 SRAM_MAX_INDEX /* Keep this at the end */ 74 }; 75 76 struct amdxdna_client; 77 struct amdxdna_fw_ver; 78 struct amdxdna_hwctx; 79 struct amdxdna_sched_job; 80 81 enum rt_config_category { 82 AIE2_RT_CFG_INIT, 83 AIE2_RT_CFG_CLK_GATING, 84 AIE2_RT_CFG_FORCE_PREEMPT, 85 AIE2_RT_CFG_FRAME_BOUNDARY_PREEMPT, 86 }; 87 88 struct rt_config { 89 u32 type; 90 u32 value; 91 u32 category; 92 unsigned long feature_mask; 93 }; 94 95 struct dpm_clk_freq { 96 u32 npuclk; 97 u32 hclk; 98 }; 99 100 /* 101 * Define the maximum number of pending commands in a hardware context. 102 * Must be power of 2! 103 */ 104 #define HWCTX_MAX_CMDS 4 105 #define get_job_idx(seq) ((seq) & (HWCTX_MAX_CMDS - 1)) 106 struct amdxdna_hwctx_priv { 107 struct amdxdna_gem_obj *heap; 108 void *mbox_chann; 109 110 struct drm_gpu_scheduler sched; 111 struct drm_sched_entity entity; 112 113 struct mutex io_lock; /* protect seq and cmd order */ 114 struct wait_queue_head job_free_wq; 115 u32 num_pending; 116 u64 seq; 117 struct semaphore job_sem; 118 bool job_done; 119 120 /* Completed job counter */ 121 u64 completed; 122 123 struct amdxdna_gem_obj *cmd_buf[HWCTX_MAX_CMDS]; 124 struct drm_syncobj *syncobj; 125 }; 126 127 enum aie2_dev_status { 128 AIE2_DEV_UNINIT, 129 AIE2_DEV_INIT, 130 AIE2_DEV_START, 131 }; 132 133 struct aie2_exec_msg_ops { 134 int (*init_cu_req)(struct amdxdna_gem_obj *cmd_bo, void *req, 135 size_t *size, u32 *msg_op); 136 int (*init_dpu_req)(struct amdxdna_gem_obj *cmd_bo, void *req, 137 size_t *size, u32 *msg_op); 138 void (*init_chain_req)(void *req, u64 slot_addr, size_t size, u32 cmd_cnt); 139 int (*fill_cf_slot)(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *size); 140 int (*fill_dpu_slot)(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *size); 141 int (*fill_preempt_slot)(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *size); 142 int (*fill_elf_slot)(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *size); 143 u32 (*get_chain_msg_op)(u32 cmd_op); 144 }; 145 146 struct amdxdna_dev_hdl { 147 struct aie_device aie; 148 const struct amdxdna_dev_priv *priv; 149 void __iomem *sram_base; 150 void __iomem *mbox_base; 151 152 u32 total_col; 153 struct amdxdna_drm_query_aie_version version; 154 struct aie2_exec_msg_ops *exec_msg_ops; 155 156 /* power management and clock*/ 157 enum amdxdna_power_mode_type pw_mode; 158 u32 dpm_level; 159 u32 dft_dpm_level; 160 u32 max_dpm_level; 161 u32 clk_gating; 162 u32 npuclk_freq; 163 u32 hclk_freq; 164 u32 max_tops; 165 u32 curr_tops; 166 u32 force_preempt_enabled; 167 u32 frame_boundary_preempt; 168 169 /* Mailbox and the management channel */ 170 struct mailbox *mbox; 171 struct async_events *async_events; 172 173 enum aie2_dev_status dev_status; 174 u32 hwctx_num; 175 176 struct amdxdna_async_error last_async_err; 177 unsigned long last_signal_ts; 178 }; 179 180 struct aie2_hw_ops { 181 int (*set_dpm)(struct amdxdna_dev_hdl *ndev, u32 dpm_level); 182 int (*update_counters)(struct amdxdna_dev_hdl *ndev); 183 }; 184 185 #define aie2_update_counters(ndev) \ 186 ({ \ 187 typeof(ndev) _ndev = ndev; \ 188 if (_ndev->priv->hw_ops->update_counters) \ 189 _ndev->priv->hw_ops->update_counters(_ndev); \ 190 }) 191 192 enum aie2_fw_feature { 193 AIE2_NPU_COMMAND, 194 AIE2_PREEMPT, 195 AIE2_TEMPORAL_ONLY, 196 AIE2_APP_HEALTH, 197 AIE2_ADD_HOST_BUFFER, 198 AIE2_UPDATE_PROPERTY, 199 AIE2_GET_DEV_REVISION, 200 AIE2_FEATURE_MAX 201 }; 202 203 #define AIE2_ALL_FEATURES GENMASK_ULL(AIE2_FEATURE_MAX - 1, AIE2_NPU_COMMAND) 204 205 struct amdxdna_dev_priv { 206 const char *fw_path; 207 const struct rt_config *rt_config; 208 const struct dpm_clk_freq *dpm_clk_tbl; 209 210 #define COL_ALIGN_NONE 0 211 #define COL_ALIGN_NATURE 1 212 u32 col_align; 213 u32 col_opc; 214 u32 mbox_dev_addr; 215 /* If mbox_size is 0, use BAR size. See MBOX_SIZE macro */ 216 u32 mbox_size; 217 u32 hwctx_limit; 218 u32 sram_dev_addr; 219 struct aie_bar_off_pair sram_offs[SRAM_MAX_INDEX]; 220 struct aie_bar_off_pair psp_regs_off[PSP_MAX_REGS]; 221 struct aie_bar_off_pair smu_regs_off[SMU_MAX_REGS]; 222 const struct aie2_hw_ops *hw_ops; 223 }; 224 225 extern const struct amdxdna_dev_ops aie2_ops; 226 227 int aie2_runtime_cfg(struct amdxdna_dev_hdl *ndev, 228 enum rt_config_category category, u32 *val); 229 230 /* aie2 npu hw config */ 231 extern const struct dpm_clk_freq npu1_dpm_clk_table[]; 232 extern const struct dpm_clk_freq npu4_dpm_clk_table[]; 233 extern const struct rt_config npu1_default_rt_cfg[]; 234 extern const struct rt_config npu4_default_rt_cfg[]; 235 extern const struct amdxdna_fw_feature_tbl npu4_fw_feature_table[]; 236 extern const struct amdxdna_rev_vbnv npu4_rev_vbnv_tbl[]; 237 extern const struct aie2_hw_ops npu4_hw_ops; 238 239 /* aie2_pm.c */ 240 int aie2_pm_init(struct amdxdna_dev_hdl *ndev); 241 int aie2_pm_set_mode(struct amdxdna_dev_hdl *ndev, enum amdxdna_power_mode_type target); 242 int aie2_pm_set_dpm(struct amdxdna_dev_hdl *ndev, u32 dpm_level); 243 244 /* aie2_error.c */ 245 int aie2_error_async_events_alloc(struct amdxdna_dev_hdl *ndev); 246 void aie2_error_async_events_free(struct amdxdna_dev_hdl *ndev); 247 int aie2_error_async_msg_thread(void *data); 248 int aie2_get_array_async_error(struct amdxdna_dev_hdl *ndev, 249 struct amdxdna_drm_get_array *args); 250 251 /* aie2_message.c */ 252 void aie2_msg_init(struct amdxdna_dev_hdl *ndev); 253 void aie2_destroy_mgmt_chann(struct amdxdna_dev_hdl *ndev); 254 int aie2_suspend_fw(struct amdxdna_dev_hdl *ndev); 255 int aie2_resume_fw(struct amdxdna_dev_hdl *ndev); 256 int aie2_set_runtime_cfg(struct amdxdna_dev_hdl *ndev, u32 type, u64 value); 257 int aie2_get_runtime_cfg(struct amdxdna_dev_hdl *ndev, u32 type, u64 *value); 258 int aie2_assign_mgmt_pasid(struct amdxdna_dev_hdl *ndev, u16 pasid); 259 int aie2_query_aie_version(struct amdxdna_dev_hdl *ndev, 260 struct amdxdna_drm_query_aie_version *version); 261 int aie2_query_aie_metadata(struct amdxdna_dev_hdl *ndev, 262 struct amdxdna_drm_query_aie_metadata *metadata); 263 int aie2_query_firmware_version(struct amdxdna_dev_hdl *ndev, 264 struct amdxdna_fw_ver *fw_ver); 265 int aie2_query_app_health(struct amdxdna_dev_hdl *ndev, u32 context_id, 266 struct app_health_report *report); 267 int aie2_get_dev_revision(struct amdxdna_dev_hdl *ndev, enum aie2_dev_revision *rev); 268 int aie2_create_context(struct amdxdna_dev_hdl *ndev, struct amdxdna_hwctx *hwctx); 269 int aie2_destroy_context(struct amdxdna_dev_hdl *ndev, struct amdxdna_hwctx *hwctx); 270 int aie2_map_host_buf(struct amdxdna_dev_hdl *ndev, u32 context_id, u64 addr, u64 size); 271 int aie2_add_host_buf(struct amdxdna_dev_hdl *ndev, u32 context_id, u64 addr, u64 size); 272 int aie2_query_status(struct amdxdna_dev_hdl *ndev, char __user *buf, u32 size, u32 *cols_filled); 273 int aie2_query_telemetry(struct amdxdna_dev_hdl *ndev, 274 char __user *buf, u32 size, 275 struct amdxdna_drm_query_telemetry_header *header); 276 int aie2_register_asyn_event_msg(struct amdxdna_dev_hdl *ndev, dma_addr_t addr, u32 size, 277 void *handle, int (*cb)(void*, void __iomem *, size_t)); 278 int aie2_config_cu(struct amdxdna_hwctx *hwctx, 279 int (*notify_cb)(void *, void __iomem *, size_t)); 280 int aie2_execbuf(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job, 281 int (*notify_cb)(void *, void __iomem *, size_t)); 282 int aie2_cmdlist_single_execbuf(struct amdxdna_hwctx *hwctx, 283 struct amdxdna_sched_job *job, 284 int (*notify_cb)(void *, void __iomem *, size_t)); 285 int aie2_cmdlist_multi_execbuf(struct amdxdna_hwctx *hwctx, 286 struct amdxdna_sched_job *job, 287 int (*notify_cb)(void *, void __iomem *, size_t)); 288 int aie2_sync_bo(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job, 289 int (*notify_cb)(void *, void __iomem *, size_t)); 290 int aie2_config_debug_bo(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job, 291 int (*notify_cb)(void *, void __iomem *, size_t)); 292 int aie2_update_prop_time_quota(struct amdxdna_dev_hdl *ndev, u32 us); 293 294 /* aie2_hwctx.c */ 295 int aie2_hwctx_init(struct amdxdna_hwctx *hwctx); 296 void aie2_hwctx_fini(struct amdxdna_hwctx *hwctx); 297 int aie2_hwctx_config(struct amdxdna_hwctx *hwctx, u32 type, u64 value, void *buf, u32 size); 298 int aie2_hwctx_sync_debug_bo(struct amdxdna_hwctx *hwctx, u32 debug_bo_hdl); 299 void aie2_hwctx_suspend(struct amdxdna_client *client); 300 int aie2_hwctx_resume(struct amdxdna_client *client); 301 int aie2_cmd_submit(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job, u64 *seq); 302 void aie2_hmm_invalidate(struct amdxdna_gem_obj *abo, unsigned long cur_seq); 303 int aie2_hwctx_heap_expand(struct amdxdna_hwctx *hwctx, struct amdxdna_gem_obj *heap); 304 305 #endif /* _AIE2_PCI_H_ */ 306