1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright(c) 2023 Intel Corporation */ 3 4 #include <linux/dev_printk.h> 5 #include <linux/dma-mapping.h> 6 #include <linux/export.h> 7 #include <linux/kernel.h> 8 #include <linux/kstrtox.h> 9 #include <linux/overflow.h> 10 #include <linux/string.h> 11 #include <linux/slab.h> 12 #include <linux/types.h> 13 #include <asm/errno.h> 14 #include "adf_accel_devices.h" 15 #include "adf_admin.h" 16 #include "adf_cfg.h" 17 #include "adf_cfg_strings.h" 18 #include "adf_clock.h" 19 #include "adf_common_drv.h" 20 #include "adf_heartbeat.h" 21 #include "adf_transport_internal.h" 22 #include "icp_qat_fw_init_admin.h" 23 24 #define ADF_HB_EMPTY_SIG 0xA5A5A5A5 25 26 /* Heartbeat counter pair */ 27 struct hb_cnt_pair { 28 __u16 resp_heartbeat_cnt; 29 __u16 req_heartbeat_cnt; 30 }; 31 32 static int adf_hb_check_polling_freq(struct adf_accel_dev *accel_dev) 33 { 34 u64 curr_time = adf_clock_get_current_time(); 35 u64 polling_time = curr_time - accel_dev->heartbeat->last_hb_check_time; 36 37 if (polling_time < accel_dev->heartbeat->hb_timer) { 38 dev_warn(&GET_DEV(accel_dev), 39 "HB polling too frequent. Configured HB timer %d ms\n", 40 accel_dev->heartbeat->hb_timer); 41 return -EINVAL; 42 } 43 44 accel_dev->heartbeat->last_hb_check_time = curr_time; 45 return 0; 46 } 47 48 /** 49 * validate_hb_ctrs_cnt() - checks if the number of heartbeat counters should 50 * be updated by one to support the currently loaded firmware. 51 * @accel_dev: Pointer to acceleration device. 52 * 53 * Return: 54 * * true - hb_ctrs must increased by ADF_NUM_PKE_STRAND 55 * * false - no changes needed 56 */ 57 static bool validate_hb_ctrs_cnt(struct adf_accel_dev *accel_dev) 58 { 59 const size_t hb_ctrs = accel_dev->hw_device->num_hb_ctrs; 60 const size_t max_aes = accel_dev->hw_device->num_engines; 61 const size_t hb_struct_size = sizeof(struct hb_cnt_pair); 62 const size_t exp_diff_size = array3_size(ADF_NUM_PKE_STRAND, max_aes, 63 hb_struct_size); 64 const size_t dev_ctrs = size_mul(max_aes, hb_ctrs); 65 const size_t stats_size = size_mul(dev_ctrs, hb_struct_size); 66 const u32 exp_diff_cnt = exp_diff_size / sizeof(u32); 67 const u32 stats_el_cnt = stats_size / sizeof(u32); 68 struct hb_cnt_pair *hb_stats = accel_dev->heartbeat->dma.virt_addr; 69 const u32 *mem_to_chk = (u32 *)(hb_stats + dev_ctrs); 70 u32 el_diff_cnt = 0; 71 int i; 72 73 /* count how many bytes are different from pattern */ 74 for (i = 0; i < stats_el_cnt; i++) { 75 if (mem_to_chk[i] == ADF_HB_EMPTY_SIG) 76 break; 77 78 el_diff_cnt++; 79 } 80 81 return el_diff_cnt && el_diff_cnt == exp_diff_cnt; 82 } 83 84 void adf_heartbeat_check_ctrs(struct adf_accel_dev *accel_dev) 85 { 86 struct hb_cnt_pair *hb_stats = accel_dev->heartbeat->dma.virt_addr; 87 const size_t hb_ctrs = accel_dev->hw_device->num_hb_ctrs; 88 const size_t max_aes = accel_dev->hw_device->num_engines; 89 const size_t dev_ctrs = size_mul(max_aes, hb_ctrs); 90 const size_t stats_size = size_mul(dev_ctrs, sizeof(struct hb_cnt_pair)); 91 const size_t mem_items_to_fill = size_mul(stats_size, 2) / sizeof(u32); 92 93 /* fill hb stats memory with pattern */ 94 memset32((uint32_t *)hb_stats, ADF_HB_EMPTY_SIG, mem_items_to_fill); 95 accel_dev->heartbeat->ctrs_cnt_checked = false; 96 } 97 EXPORT_SYMBOL_GPL(adf_heartbeat_check_ctrs); 98 99 static int get_timer_ticks(struct adf_accel_dev *accel_dev, unsigned int *value) 100 { 101 char timer_str[ADF_CFG_MAX_VAL_LEN_IN_BYTES] = { }; 102 u32 timer_ms = ADF_CFG_HB_TIMER_DEFAULT_MS; 103 int cfg_read_status; 104 u32 ticks; 105 int ret; 106 107 cfg_read_status = adf_cfg_get_param_value(accel_dev, ADF_GENERAL_SEC, 108 ADF_HEARTBEAT_TIMER, timer_str); 109 if (cfg_read_status == 0) { 110 if (kstrtouint(timer_str, 10, &timer_ms)) 111 dev_dbg(&GET_DEV(accel_dev), 112 "kstrtouint failed to parse the %s, param value", 113 ADF_HEARTBEAT_TIMER); 114 } 115 116 if (timer_ms < ADF_CFG_HB_TIMER_MIN_MS) { 117 dev_err(&GET_DEV(accel_dev), "Timer cannot be less than %u\n", 118 ADF_CFG_HB_TIMER_MIN_MS); 119 return -EINVAL; 120 } 121 122 /* 123 * On 4xxx devices adf_timer is responsible for HB updates and 124 * its period is fixed to 200ms 125 */ 126 if (accel_dev->timer) 127 timer_ms = ADF_CFG_HB_TIMER_MIN_MS; 128 129 ret = adf_heartbeat_ms_to_ticks(accel_dev, timer_ms, &ticks); 130 if (ret) 131 return ret; 132 133 adf_heartbeat_save_cfg_param(accel_dev, timer_ms); 134 135 accel_dev->heartbeat->hb_timer = timer_ms; 136 *value = ticks; 137 138 return 0; 139 } 140 141 static int check_ae(struct hb_cnt_pair *curr, struct hb_cnt_pair *prev, 142 u16 *count, const size_t hb_ctrs) 143 { 144 size_t thr; 145 146 /* loop through all threads in AE */ 147 for (thr = 0; thr < hb_ctrs; thr++) { 148 u16 req = curr[thr].req_heartbeat_cnt; 149 u16 resp = curr[thr].resp_heartbeat_cnt; 150 u16 last = prev[thr].resp_heartbeat_cnt; 151 152 if ((thr == ADF_AE_ADMIN_THREAD || req != resp) && resp == last) { 153 u16 retry = ++count[thr]; 154 155 if (retry >= ADF_CFG_HB_COUNT_THRESHOLD) 156 return -EIO; 157 158 } else { 159 count[thr] = 0; 160 } 161 } 162 return 0; 163 } 164 165 static int adf_hb_get_status(struct adf_accel_dev *accel_dev) 166 { 167 struct adf_hw_device_data *hw_device = accel_dev->hw_device; 168 struct hb_cnt_pair *live_stats, *last_stats, *curr_stats; 169 const size_t hb_ctrs = hw_device->num_hb_ctrs; 170 const unsigned long ae_mask = hw_device->ae_mask; 171 const size_t max_aes = hw_device->num_engines; 172 const size_t dev_ctrs = size_mul(max_aes, hb_ctrs); 173 const size_t stats_size = size_mul(dev_ctrs, sizeof(*curr_stats)); 174 struct hb_cnt_pair *ae_curr_p, *ae_prev_p; 175 u16 *count_fails, *ae_count_p; 176 size_t ae_offset; 177 size_t ae = 0; 178 int ret = 0; 179 180 if (!accel_dev->heartbeat->ctrs_cnt_checked) { 181 if (validate_hb_ctrs_cnt(accel_dev)) 182 hw_device->num_hb_ctrs += ADF_NUM_PKE_STRAND; 183 184 accel_dev->heartbeat->ctrs_cnt_checked = true; 185 } 186 187 live_stats = accel_dev->heartbeat->dma.virt_addr; 188 last_stats = live_stats + dev_ctrs; 189 count_fails = (u16 *)(last_stats + dev_ctrs); 190 191 curr_stats = kmemdup(live_stats, stats_size, GFP_KERNEL); 192 if (!curr_stats) 193 return -ENOMEM; 194 195 /* loop through active AEs */ 196 for_each_set_bit(ae, &ae_mask, max_aes) { 197 ae_offset = size_mul(ae, hb_ctrs); 198 ae_curr_p = curr_stats + ae_offset; 199 ae_prev_p = last_stats + ae_offset; 200 ae_count_p = count_fails + ae_offset; 201 202 ret = check_ae(ae_curr_p, ae_prev_p, ae_count_p, hb_ctrs); 203 if (ret) 204 break; 205 } 206 207 /* Copy current stats for the next iteration */ 208 memcpy(last_stats, curr_stats, stats_size); 209 kfree(curr_stats); 210 211 return ret; 212 } 213 214 void adf_heartbeat_status(struct adf_accel_dev *accel_dev, 215 enum adf_device_heartbeat_status *hb_status) 216 { 217 struct adf_heartbeat *hb; 218 219 if (!adf_dev_started(accel_dev) || 220 test_bit(ADF_STATUS_RESTARTING, &accel_dev->status)) { 221 *hb_status = HB_DEV_UNRESPONSIVE; 222 return; 223 } 224 225 if (adf_hb_check_polling_freq(accel_dev) == -EINVAL) { 226 *hb_status = HB_DEV_UNSUPPORTED; 227 return; 228 } 229 230 hb = accel_dev->heartbeat; 231 hb->hb_sent_counter++; 232 233 if (adf_hb_get_status(accel_dev)) { 234 dev_err(&GET_DEV(accel_dev), 235 "Heartbeat ERROR: QAT is not responding.\n"); 236 *hb_status = HB_DEV_UNRESPONSIVE; 237 hb->hb_failed_counter++; 238 return; 239 } 240 241 *hb_status = HB_DEV_ALIVE; 242 } 243 244 int adf_heartbeat_ms_to_ticks(struct adf_accel_dev *accel_dev, unsigned int time_ms, 245 u32 *value) 246 { 247 struct adf_hw_device_data *hw_data = accel_dev->hw_device; 248 u32 clk_per_sec; 249 250 /* HB clock may be different than AE clock */ 251 if (!hw_data->get_hb_clock) 252 return -EINVAL; 253 254 clk_per_sec = hw_data->get_hb_clock(hw_data); 255 *value = time_ms * (clk_per_sec / MSEC_PER_SEC); 256 257 return 0; 258 } 259 260 int adf_heartbeat_save_cfg_param(struct adf_accel_dev *accel_dev, 261 unsigned int timer_ms) 262 { 263 char timer_str[ADF_CFG_MAX_VAL_LEN_IN_BYTES]; 264 265 snprintf(timer_str, sizeof(timer_str), "%u", timer_ms); 266 return adf_cfg_add_key_value_param(accel_dev, ADF_GENERAL_SEC, 267 ADF_HEARTBEAT_TIMER, timer_str, 268 ADF_STR); 269 } 270 EXPORT_SYMBOL_GPL(adf_heartbeat_save_cfg_param); 271 272 int adf_heartbeat_init(struct adf_accel_dev *accel_dev) 273 { 274 struct adf_heartbeat *hb; 275 276 hb = kzalloc(sizeof(*hb), GFP_KERNEL); 277 if (!hb) 278 goto err_ret; 279 280 hb->dma.virt_addr = dma_alloc_coherent(&GET_DEV(accel_dev), PAGE_SIZE, 281 &hb->dma.phy_addr, GFP_KERNEL); 282 if (!hb->dma.virt_addr) 283 goto err_free; 284 285 /* 286 * Default set this flag as true to avoid unnecessary checks, 287 * it will be reset on platforms that need such a check 288 */ 289 hb->ctrs_cnt_checked = true; 290 accel_dev->heartbeat = hb; 291 292 return 0; 293 294 err_free: 295 kfree(hb); 296 err_ret: 297 return -ENOMEM; 298 } 299 300 int adf_heartbeat_start(struct adf_accel_dev *accel_dev) 301 { 302 unsigned int timer_ticks; 303 int ret; 304 305 if (!accel_dev->heartbeat) { 306 dev_warn(&GET_DEV(accel_dev), "Heartbeat instance not found!"); 307 return -EFAULT; 308 } 309 310 if (accel_dev->hw_device->check_hb_ctrs) 311 accel_dev->hw_device->check_hb_ctrs(accel_dev); 312 313 ret = get_timer_ticks(accel_dev, &timer_ticks); 314 if (ret) 315 return ret; 316 317 ret = adf_send_admin_hb_timer(accel_dev, timer_ticks); 318 if (ret) 319 dev_warn(&GET_DEV(accel_dev), "Heartbeat not supported!"); 320 321 return ret; 322 } 323 324 void adf_heartbeat_shutdown(struct adf_accel_dev *accel_dev) 325 { 326 struct adf_heartbeat *hb = accel_dev->heartbeat; 327 328 if (!hb) 329 return; 330 331 if (hb->dma.virt_addr) 332 dma_free_coherent(&GET_DEV(accel_dev), PAGE_SIZE, 333 hb->dma.virt_addr, hb->dma.phy_addr); 334 335 kfree(hb); 336 accel_dev->heartbeat = NULL; 337 } 338