111 	struct perf_output_handle handle; /* AUX buffer output handle */
804 	/* Use AUX buffer. No need to allocate it by ourself */  in __hw_perf_event_init()
1179 	 * AUX buffer is used when in diagnostic sampling mode.  in hw_perf_event_update()
1255 static inline unsigned long aux_sdb_index(struct aux_buffer *aux,  in aux_sdb_index()  argument
1258 	return i % aux->sfb.num_sdb;  in aux_sdb_index()
1266 static inline unsigned long aux_sdb_num_alert(struct aux_buffer *aux)  in aux_sdb_num_alert()  argument
1268 	return aux_sdb_num(aux->head, aux->alert_mark);  in aux_sdb_num_alert()
1271 static inline unsigned long aux_sdb_num_empty(struct aux_buffer *aux)  in aux_sdb_num_empty()  argument
1273 	return aux_sdb_num(aux->head, aux->empty_mark);  in aux_sdb_num_empty()
1279 static struct hws_trailer_entry *aux_sdb_trailer(struct aux_buffer *aux,  in aux_sdb_trailer()  argument
1284 	index = aux_sdb_index(aux, index);  in aux_sdb_trailer()
1285 	sdb = aux->sdb_index[index];  in aux_sdb_trailer()
1291  * disabled. Collect the full SDBs in AUX buffer which have not reached
1301 	struct aux_buffer *aux;  in aux_output_end()  local
1304 	aux = perf_get_aux(handle);  in aux_output_end()
1305 	if (!aux)  in aux_output_end()
1308 	range_scan = aux_sdb_num_alert(aux);  in aux_output_end()
1309 	for (i = 0, idx = aux->head; i < range_scan; i++, idx++) {  in aux_output_end()
1310 		te = aux_sdb_trailer(aux, idx);  in aux_output_end()
1318 	te = aux_sdb_trailer(aux, aux->alert_mark);  in aux_output_end()
1332 			    struct aux_buffer *aux,  in aux_output_begin()  argument
1341 	aux->head = handle->head >> PAGE_SHIFT;  in aux_output_begin()
1347 	 * SDBs between aux->head and aux->empty_mark are already ready  in aux_output_begin()
1350 	if (range > aux_sdb_num_empty(aux)) {  in aux_output_begin()
1351 		range_scan = range - aux_sdb_num_empty(aux);  in aux_output_begin()
1352 		idx = aux->empty_mark + 1;  in aux_output_begin()
1354 			te = aux_sdb_trailer(aux, idx);  in aux_output_begin()
1360 		aux->empty_mark = aux->head + range - 1;  in aux_output_begin()
1364 	aux->alert_mark = aux->head + range/2 - 1;  in aux_output_begin()
1365 	te = aux_sdb_trailer(aux, aux->alert_mark);  in aux_output_begin()
1369 	head = aux_sdb_index(aux, aux->head);  in aux_output_begin()
1370 	base = aux->sdbt_index[head / CPUM_SF_SDB_PER_TABLE];  in aux_output_begin()
1373 	cpuhw->lsctl.dear = virt_to_phys((void *)aux->sdb_index[head]);  in aux_output_begin()
1384 static bool aux_set_alert(struct aux_buffer *aux, unsigned long alert_index,  in aux_set_alert()  argument
1390 	te = aux_sdb_trailer(aux, alert_index);  in aux_set_alert()
1411  * @aux:	The AUX buffer to set
1412  * @range:	The range of SDBs to scan started from aux->head
1415  * Set alert indicator on the SDB at index of aux->alert_mark. If this SDB is
1420  * Scan the SDBs in AUX buffer from behind aux->empty_mark. They are used
1423  * If aux->alert_mark fall in this area, just set it. Overflow count is
1426  * SDBs between aux->head and aux->empty_mark are already reset at last time.
1431 static bool aux_reset_buffer(struct aux_buffer *aux, unsigned long range,  in aux_reset_buffer()  argument
1439 	if (range <= aux_sdb_num_empty(aux))  in aux_reset_buffer()
1445 		return aux_set_alert(aux, aux->alert_mark, overflow);  in aux_reset_buffer()
1447 	if (aux->alert_mark <= aux->empty_mark)  in aux_reset_buffer()
1452 		if (!aux_set_alert(aux, aux->alert_mark, overflow))  in aux_reset_buffer()
1460 	range_scan = range - aux_sdb_num_empty(aux);  in aux_reset_buffer()
1461 	idx = aux->empty_mark + 1;  in aux_reset_buffer()
1463 		te = aux_sdb_trailer(aux, idx);  in aux_reset_buffer()
1470 			if (idx == aux->alert_mark)  in aux_reset_buffer()
1479 	aux->empty_mark = aux->head + range - 1;  in aux_reset_buffer()
1489 	struct aux_buffer *aux;  in hw_collect_aux()  local
1496 	aux = perf_get_aux(handle);  in hw_collect_aux()
1497 	if (!aux)  in hw_collect_aux()
1501 	size = aux_sdb_num_alert(aux) << PAGE_SHIFT;  in hw_collect_aux()
1506 	num_sdb = aux->sfb.num_sdb;  in hw_collect_aux()
1509 		aux = perf_aux_output_begin(handle, cpuhw->event);  in hw_collect_aux()
1511 			pr_err("The AUX buffer with %lu pages for the "  in hw_collect_aux()
1516 		if (!aux)  in hw_collect_aux()
1520 		aux->head = handle->head >> PAGE_SHIFT;  in hw_collect_aux()
1523 			aux->alert_mark = aux->head;  in hw_collect_aux()
1525 			aux->alert_mark = aux->head + range/2 - 1;  in hw_collect_aux()
1527 		if (aux_reset_buffer(aux, range, &overflow)) {  in hw_collect_aux()
1534 			pr_err("Sample data caused the AUX buffer with %lu "  in hw_collect_aux()
1535 			       "pages to overflow\n", aux->sfb.num_sdb);  in hw_collect_aux()
1537 			size = aux_sdb_num_alert(aux) << PAGE_SHIFT;  in hw_collect_aux()
1544  * Callback when freeing AUX buffers.
1548 	struct aux_buffer *aux = data;  in aux_buffer_free()  local
1551 	if (!aux)  in aux_buffer_free()
1555 	num_sdbt = aux->sfb.num_sdbt;  in aux_buffer_free()
1557 		free_page(aux->sdbt_index[i]);  in aux_buffer_free()
1559 	kfree(aux->sdbt_index);  in aux_buffer_free()
1560 	kfree(aux->sdb_index);  in aux_buffer_free()
1561 	kfree(aux);  in aux_buffer_free()
1576  * aux_buffer_setup() - Setup AUX buffer for diagnostic mode sampling
1582  * This is the callback when setup an event using AUX buffer. Perf tool can
1584  * for basic samples, AUX buffer belongs to the event. It is scheduled with
1587  * Return the private AUX buffer structure if success or NULL if fails.
1593 	struct aux_buffer *aux;  in aux_buffer_setup()  local
1601 		pr_err("AUX buffer size (%i pages) is larger than the "  in aux_buffer_setup()
1606 		pr_err("AUX buffer size (%i pages) is less than the "  in aux_buffer_setup()
1613 	aux = kzalloc(sizeof(struct aux_buffer), GFP_KERNEL);  in aux_buffer_setup()
1614 	if (!aux)  in aux_buffer_setup()
1616 	sfb = &aux->sfb;  in aux_buffer_setup()
1620 	aux->sdbt_index = kmalloc_array(n_sdbt, sizeof(void *), GFP_KERNEL);  in aux_buffer_setup()
1621 	if (!aux->sdbt_index)  in aux_buffer_setup()
1625 	aux->sdb_index = kmalloc_array(nr_pages, sizeof(void *), GFP_KERNEL);  in aux_buffer_setup()
1626 	if (!aux->sdb_index)  in aux_buffer_setup()
1634 	aux->sdbt_index[sfb->num_sdbt++] = (unsigned long)sfb->sdbt;  in aux_buffer_setup()
1638 	 * Link the provided pages of AUX buffer to SDBT.  in aux_buffer_setup()
1646 			aux->sdbt_index[sfb->num_sdbt++] = (unsigned long)new;  in aux_buffer_setup()
1653 		aux->sdb_index[i] = (unsigned long)pages[i];  in aux_buffer_setup()
1667 	aux->empty_mark = sfb->num_sdb - 1;  in aux_buffer_setup()
1669 	return aux;  in aux_buffer_setup()
1672 	/* SDBs (AUX buffer pages) are freed by caller */  in aux_buffer_setup()
1674 		free_page(aux->sdbt_index[i]);  in aux_buffer_setup()
1675 	kfree(aux->sdb_index);  in aux_buffer_setup()
1677 	kfree(aux->sdbt_index);  in aux_buffer_setup()
1679 	kfree(aux);  in aux_buffer_setup()
1766 	struct aux_buffer *aux;  in cpumsf_pmu_add()  local
1800 		aux = perf_aux_output_begin(&cpuhw->handle, event);  in cpumsf_pmu_add()
1801 		if (!aux) {  in cpumsf_pmu_add()
1805 		err = aux_output_begin(&cpuhw->handle, aux, cpuhw);  in cpumsf_pmu_add()