453 ice_ptp_is_tx_tracker_up(struct ice_ptp_tx *tx)  in ice_ptp_is_tx_tracker_up()  argument
455 	lockdep_assert_held(&tx->lock);  in ice_ptp_is_tx_tracker_up()
457 	return tx->init && !tx->calibrating;  in ice_ptp_is_tx_tracker_up()
465 void ice_ptp_req_tx_single_tstamp(struct ice_ptp_tx *tx, u8 idx)  in ice_ptp_req_tx_single_tstamp()  argument
473 	if (!tx->init)  in ice_ptp_req_tx_single_tstamp()
476 	ptp_port = container_of(tx, struct ice_ptp_port, tx);  in ice_ptp_req_tx_single_tstamp()
481 	if (time_is_before_jiffies(tx->tstamps[idx].start + 2 * HZ)) {  in ice_ptp_req_tx_single_tstamp()
485 		skb = tx->tstamps[idx].skb;  in ice_ptp_req_tx_single_tstamp()
486 		tx->tstamps[idx].skb = NULL;  in ice_ptp_req_tx_single_tstamp()
487 		clear_bit(idx, tx->in_use);  in ice_ptp_req_tx_single_tstamp()
493 	ice_trace(tx_tstamp_fw_req, tx->tstamps[idx].skb, idx);  in ice_ptp_req_tx_single_tstamp()
503 	tx->last_ll_ts_idx_read = idx;  in ice_ptp_req_tx_single_tstamp()
512 void ice_ptp_complete_tx_single_tstamp(struct ice_ptp_tx *tx)  in ice_ptp_complete_tx_single_tstamp()  argument
515 	u8 idx = tx->last_ll_ts_idx_read;  in ice_ptp_complete_tx_single_tstamp()
526 	if (!tx->init || tx->last_ll_ts_idx_read < 0)  in ice_ptp_complete_tx_single_tstamp()
529 	ptp_port = container_of(tx, struct ice_ptp_port, tx);  in ice_ptp_complete_tx_single_tstamp()
534 	ice_trace(tx_tstamp_fw_done, tx->tstamps[idx].skb, idx);  in ice_ptp_complete_tx_single_tstamp()
566 	if (raw_tstamp == tx->tstamps[idx].cached_tstamp)  in ice_ptp_complete_tx_single_tstamp()
569 	tx->tstamps[idx].cached_tstamp = raw_tstamp;  in ice_ptp_complete_tx_single_tstamp()
570 	clear_bit(idx, tx->in_use);  in ice_ptp_complete_tx_single_tstamp()
571 	skb = tx->tstamps[idx].skb;  in ice_ptp_complete_tx_single_tstamp()
572 	tx->tstamps[idx].skb = NULL;  in ice_ptp_complete_tx_single_tstamp()
573 	if (test_and_clear_bit(idx, tx->stale))  in ice_ptp_complete_tx_single_tstamp()
643 static void ice_ptp_process_tx_tstamp(struct ice_ptp_tx *tx)  in ice_ptp_process_tx_tstamp()  argument
654 	ptp_port = container_of(tx, struct ice_ptp_port, tx);  in ice_ptp_process_tx_tstamp()
659 	if (tx->has_ready_bitmap) {  in ice_ptp_process_tx_tstamp()
660 		err = ice_get_phy_tx_tstamp_ready(hw, tx->block, &tstamp_ready);  in ice_ptp_process_tx_tstamp()
668 	for_each_set_bit(idx, tx->in_use, tx->len) {  in ice_ptp_process_tx_tstamp()
670 		u8 phy_idx = idx + tx->offset;  in ice_ptp_process_tx_tstamp()
676 		if (time_is_before_jiffies(tx->tstamps[idx].start + 2 * HZ)) {  in ice_ptp_process_tx_tstamp()
691 		if (tx->has_ready_bitmap &&  in ice_ptp_process_tx_tstamp()
699 		ice_trace(tx_tstamp_fw_req, tx->tstamps[idx].skb, idx);  in ice_ptp_process_tx_tstamp()
701 		err = ice_read_phy_tstamp(hw, tx->block, phy_idx, &raw_tstamp);  in ice_ptp_process_tx_tstamp()
705 		ice_trace(tx_tstamp_fw_done, tx->tstamps[idx].skb, idx);  in ice_ptp_process_tx_tstamp()
712 		if (!drop_ts && !tx->has_ready_bitmap &&  in ice_ptp_process_tx_tstamp()
713 		    raw_tstamp == tx->tstamps[idx].cached_tstamp)  in ice_ptp_process_tx_tstamp()
721 		spin_lock_irqsave(&tx->lock, flags);  in ice_ptp_process_tx_tstamp()
722 		if (!tx->has_ready_bitmap && raw_tstamp)  in ice_ptp_process_tx_tstamp()
723 			tx->tstamps[idx].cached_tstamp = raw_tstamp;  in ice_ptp_process_tx_tstamp()
724 		clear_bit(idx, tx->in_use);  in ice_ptp_process_tx_tstamp()
725 		skb = tx->tstamps[idx].skb;  in ice_ptp_process_tx_tstamp()
726 		tx->tstamps[idx].skb = NULL;  in ice_ptp_process_tx_tstamp()
727 		if (test_and_clear_bit(idx, tx->stale))  in ice_ptp_process_tx_tstamp()
729 		spin_unlock_irqrestore(&tx->lock, flags);  in ice_ptp_process_tx_tstamp()
765 		struct ice_ptp_tx *tx = &port->tx;  in ice_ptp_tx_tstamp_owner()  local
767 		if (!tx || !tx->init)  in ice_ptp_tx_tstamp_owner()
770 		ice_ptp_process_tx_tstamp(tx);  in ice_ptp_tx_tstamp_owner()
796 static enum ice_tx_tstamp_work ice_ptp_tx_tstamp(struct ice_ptp_tx *tx)  in ice_ptp_tx_tstamp()  argument
801 	if (!tx->init)  in ice_ptp_tx_tstamp()
805 	ice_ptp_process_tx_tstamp(tx);  in ice_ptp_tx_tstamp()
808 	spin_lock_irqsave(&tx->lock, flags);  in ice_ptp_tx_tstamp()
809 	more_timestamps = tx->init && !bitmap_empty(tx->in_use, tx->len);  in ice_ptp_tx_tstamp()
810 	spin_unlock_irqrestore(&tx->lock, flags);  in ice_ptp_tx_tstamp()
826 ice_ptp_alloc_tx_tracker(struct ice_ptp_tx *tx)  in ice_ptp_alloc_tx_tracker()  argument
831 	tstamps = kcalloc(tx->len, sizeof(*tstamps), GFP_KERNEL);  in ice_ptp_alloc_tx_tracker()
832 	in_use = bitmap_zalloc(tx->len, GFP_KERNEL);  in ice_ptp_alloc_tx_tracker()
833 	stale = bitmap_zalloc(tx->len, GFP_KERNEL);  in ice_ptp_alloc_tx_tracker()
843 	tx->tstamps = tstamps;  in ice_ptp_alloc_tx_tracker()
844 	tx->in_use = in_use;  in ice_ptp_alloc_tx_tracker()
845 	tx->stale = stale;  in ice_ptp_alloc_tx_tracker()
846 	tx->init = 1;  in ice_ptp_alloc_tx_tracker()
847 	tx->last_ll_ts_idx_read = -1;  in ice_ptp_alloc_tx_tracker()
849 	spin_lock_init(&tx->lock);  in ice_ptp_alloc_tx_tracker()
862 ice_ptp_flush_tx_tracker(struct ice_pf *pf, struct ice_ptp_tx *tx)  in ice_ptp_flush_tx_tracker()  argument
870 	err = ice_get_phy_tx_tstamp_ready(hw, tx->block, &tstamp_ready);  in ice_ptp_flush_tx_tracker()
873 			tx->block, err);  in ice_ptp_flush_tx_tracker()
881 	for_each_set_bit(idx, tx->in_use, tx->len) {  in ice_ptp_flush_tx_tracker()
882 		u8 phy_idx = idx + tx->offset;  in ice_ptp_flush_tx_tracker()
887 			ice_clear_phy_tstamp(hw, tx->block, phy_idx);  in ice_ptp_flush_tx_tracker()
889 		spin_lock_irqsave(&tx->lock, flags);  in ice_ptp_flush_tx_tracker()
890 		skb = tx->tstamps[idx].skb;  in ice_ptp_flush_tx_tracker()
891 		tx->tstamps[idx].skb = NULL;  in ice_ptp_flush_tx_tracker()
892 		clear_bit(idx, tx->in_use);  in ice_ptp_flush_tx_tracker()
893 		clear_bit(idx, tx->stale);  in ice_ptp_flush_tx_tracker()
894 		spin_unlock_irqrestore(&tx->lock, flags);  in ice_ptp_flush_tx_tracker()
916 ice_ptp_mark_tx_tracker_stale(struct ice_ptp_tx *tx)  in ice_ptp_mark_tx_tracker_stale()  argument
920 	spin_lock_irqsave(&tx->lock, flags);  in ice_ptp_mark_tx_tracker_stale()
921 	bitmap_or(tx->stale, tx->stale, tx->in_use, tx->len);  in ice_ptp_mark_tx_tracker_stale()
922 	spin_unlock_irqrestore(&tx->lock, flags);  in ice_ptp_mark_tx_tracker_stale()
938 		ice_ptp_flush_tx_tracker(ptp_port_to_pf(port), &port->tx);  in ice_ptp_flush_all_tx_tracker()
949 ice_ptp_release_tx_tracker(struct ice_pf *pf, struct ice_ptp_tx *tx)  in ice_ptp_release_tx_tracker()  argument
953 	spin_lock_irqsave(&tx->lock, flags);  in ice_ptp_release_tx_tracker()
954 	tx->init = 0;  in ice_ptp_release_tx_tracker()
955 	spin_unlock_irqrestore(&tx->lock, flags);  in ice_ptp_release_tx_tracker()
960 	ice_ptp_flush_tx_tracker(pf, tx);  in ice_ptp_release_tx_tracker()
962 	kfree(tx->tstamps);  in ice_ptp_release_tx_tracker()
963 	tx->tstamps = NULL;  in ice_ptp_release_tx_tracker()
965 	bitmap_free(tx->in_use);  in ice_ptp_release_tx_tracker()
966 	tx->in_use = NULL;  in ice_ptp_release_tx_tracker()
968 	bitmap_free(tx->stale);  in ice_ptp_release_tx_tracker()
969 	tx->stale = NULL;  in ice_ptp_release_tx_tracker()
971 	tx->len = 0;  in ice_ptp_release_tx_tracker()
985 static int ice_ptp_init_tx_eth56g(struct ice_pf *pf, struct ice_ptp_tx *tx,  in ice_ptp_init_tx_eth56g()  argument
988 	tx->block = port;  in ice_ptp_init_tx_eth56g()
989 	tx->offset = 0;  in ice_ptp_init_tx_eth56g()
990 	tx->len = INDEX_PER_PORT_ETH56G;  in ice_ptp_init_tx_eth56g()
991 	tx->has_ready_bitmap = 1;  in ice_ptp_init_tx_eth56g()
993 	return ice_ptp_alloc_tx_tracker(tx);  in ice_ptp_init_tx_eth56g()
1008 ice_ptp_init_tx_e82x(struct ice_pf *pf, struct ice_ptp_tx *tx, u8 port)  in ice_ptp_init_tx_e82x()  argument
1010 	tx->block = ICE_GET_QUAD_NUM(port);  in ice_ptp_init_tx_e82x()
1011 	tx->offset = (port % ICE_PORTS_PER_QUAD) * INDEX_PER_PORT_E82X;  in ice_ptp_init_tx_e82x()
1012 	tx->len = INDEX_PER_PORT_E82X;  in ice_ptp_init_tx_e82x()
1013 	tx->has_ready_bitmap = 1;  in ice_ptp_init_tx_e82x()
1015 	return ice_ptp_alloc_tx_tracker(tx);  in ice_ptp_init_tx_e82x()
1027 ice_ptp_init_tx_e810(struct ice_pf *pf, struct ice_ptp_tx *tx)  in ice_ptp_init_tx_e810()  argument
1029 	tx->block = pf->hw.port_info->lport;  in ice_ptp_init_tx_e810()
1030 	tx->offset = 0;  in ice_ptp_init_tx_e810()
1031 	tx->len = INDEX_PER_PORT_E810;  in ice_ptp_init_tx_e810()
1036 	tx->has_ready_bitmap = 0;  in ice_ptp_init_tx_e810()
1038 	return ice_ptp_alloc_tx_tracker(tx);  in ice_ptp_init_tx_e810()
1146 	ice_ptp_mark_tx_tracker_stale(&pf->ptp.port.tx);  in ice_ptp_reset_cached_phctime()
1383 		spin_lock_irqsave(&ptp_port->tx.lock, flags);  in ice_ptp_port_phy_restart()
1384 		ptp_port->tx.calibrating = true;  in ice_ptp_port_phy_restart()
1385 		spin_unlock_irqrestore(&ptp_port->tx.lock, flags);  in ice_ptp_port_phy_restart()
1394 		spin_lock_irqsave(&ptp_port->tx.lock, flags);  in ice_ptp_port_phy_restart()
1395 		ptp_port->tx.calibrating = false;  in ice_ptp_port_phy_restart()
1396 		spin_unlock_irqrestore(&ptp_port->tx.lock, flags);  in ice_ptp_port_phy_restart()
2691 s8 ice_ptp_request_ts(struct ice_ptp_tx *tx, struct sk_buff *skb)  in ice_ptp_request_ts()  argument
2696 	spin_lock_irqsave(&tx->lock, flags);  in ice_ptp_request_ts()
2699 	if (!ice_ptp_is_tx_tracker_up(tx)) {  in ice_ptp_request_ts()
2700 		spin_unlock_irqrestore(&tx->lock, flags);  in ice_ptp_request_ts()
2705 	idx = find_next_zero_bit(tx->in_use, tx->len,  in ice_ptp_request_ts()
2706 				 tx->last_ll_ts_idx_read + 1);  in ice_ptp_request_ts()
2707 	if (idx == tx->len)  in ice_ptp_request_ts()
2708 		idx = find_first_zero_bit(tx->in_use, tx->len);  in ice_ptp_request_ts()
2710 	if (idx < tx->len) {  in ice_ptp_request_ts()
2715 		set_bit(idx, tx->in_use);  in ice_ptp_request_ts()
2716 		clear_bit(idx, tx->stale);  in ice_ptp_request_ts()
2717 		tx->tstamps[idx].start = jiffies;  in ice_ptp_request_ts()
2718 		tx->tstamps[idx].skb = skb_get(skb);  in ice_ptp_request_ts()
2723 	spin_unlock_irqrestore(&tx->lock, flags);  in ice_ptp_request_ts()
2728 	if (idx >= tx->len)  in ice_ptp_request_ts()
2731 		return idx + tx->offset;  in ice_ptp_request_ts()
2749 		return ice_ptp_tx_tstamp(&pf->ptp.port.tx);  in ice_ptp_process_ts()
2849 	ice_ptp_release_tx_tracker(pf, &pf->ptp.port.tx);  in ice_ptp_prepare_for_reset()
3139 		return ice_ptp_init_tx_eth56g(pf, &ptp_port->tx,  in ice_ptp_init_port()
3142 		return ice_ptp_init_tx_e810(pf, &ptp_port->tx);  in ice_ptp_init_port()
3147 		return ice_ptp_init_tx_e82x(pf, &ptp_port->tx,  in ice_ptp_init_port()
3276 	ice_ptp_release_tx_tracker(pf, &pf->ptp.port.tx);  in ice_ptp_release()