1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef __NET_PKT_SCHED_H 3 #define __NET_PKT_SCHED_H 4 5 #include <linux/jiffies.h> 6 #include <linux/ktime.h> 7 #include <linux/if_vlan.h> 8 #include <linux/netdevice.h> 9 #include <net/sch_generic.h> 10 #include <net/net_namespace.h> 11 #include <uapi/linux/pkt_sched.h> 12 13 #define DEFAULT_TX_QUEUE_LEN 1000 14 #define STAB_SIZE_LOG_MAX 30 15 #define QDISC_PKT_LEN_MAX (1 << 20) /* 1 MiB */ 16 17 struct qdisc_walker { 18 int stop; 19 int skip; 20 int count; 21 int (*fn)(struct Qdisc *, unsigned long cl, struct qdisc_walker *); 22 }; 23 24 #define qdisc_priv(q) \ 25 _Generic(q, \ 26 const struct Qdisc * : (const void *)&q->privdata, \ 27 struct Qdisc * : (void *)&q->privdata) 28 29 /* 30 Timer resolution MUST BE < 10% of min_schedulable_packet_size/bandwidth 31 32 Normal IP packet size ~ 512byte, hence: 33 34 0.5Kbyte/1Mbyte/sec = 0.5msec, so that we need 50usec timer for 35 10Mbit ethernet. 36 37 10msec resolution -> <50Kbit/sec. 38 39 The result: [34]86 is not good choice for QoS router :-( 40 41 The things are not so bad, because we may use artificial 42 clock evaluated by integration of network data flow 43 in the most critical places. 44 */ 45 46 typedef u64 psched_time_t; 47 48 /* Avoid doing 64 bit divide */ 49 #define PSCHED_SHIFT 6 50 #define PSCHED_TICKS2NS(x) ((s64)(x) << PSCHED_SHIFT) 51 #define PSCHED_NS2TICKS(x) ((x) >> PSCHED_SHIFT) 52 53 #define PSCHED_TICKS_PER_SEC PSCHED_NS2TICKS(NSEC_PER_SEC) 54 #define PSCHED_PASTPERFECT 0 55 56 static inline psched_time_t psched_get_time(void) 57 { 58 return PSCHED_NS2TICKS(ktime_get_ns()); 59 } 60 61 struct qdisc_watchdog { 62 struct hrtimer timer; 63 struct Qdisc *qdisc; 64 }; 65 66 void qdisc_watchdog_init_clockid(struct qdisc_watchdog *wd, struct Qdisc *qdisc, 67 clockid_t clockid); 68 void qdisc_watchdog_init(struct qdisc_watchdog *wd, struct Qdisc *qdisc); 69 70 void qdisc_watchdog_schedule_range_ns(struct qdisc_watchdog *wd, u64 expires, 71 u64 delta_ns); 72 73 static inline void qdisc_watchdog_schedule_ns(struct qdisc_watchdog *wd, 74 u64 expires) 75 { 76 return qdisc_watchdog_schedule_range_ns(wd, expires, 0ULL); 77 } 78 79 static inline void qdisc_watchdog_schedule(struct qdisc_watchdog *wd, 80 psched_time_t expires) 81 { 82 qdisc_watchdog_schedule_ns(wd, PSCHED_TICKS2NS(expires)); 83 } 84 85 void qdisc_watchdog_cancel(struct qdisc_watchdog *wd); 86 87 extern struct Qdisc_ops pfifo_qdisc_ops; 88 extern struct Qdisc_ops bfifo_qdisc_ops; 89 extern struct Qdisc_ops pfifo_head_drop_qdisc_ops; 90 91 int fifo_set_limit(struct Qdisc *q, unsigned int limit); 92 struct Qdisc *fifo_create_dflt(struct Qdisc *sch, struct Qdisc_ops *ops, 93 unsigned int limit, 94 struct netlink_ext_ack *extack); 95 96 int register_qdisc(struct Qdisc_ops *qops); 97 void unregister_qdisc(struct Qdisc_ops *qops); 98 #define NET_SCH_ALIAS_PREFIX "net-sch-" 99 #define MODULE_ALIAS_NET_SCH(id) MODULE_ALIAS(NET_SCH_ALIAS_PREFIX id) 100 void qdisc_get_default(char *id, size_t len); 101 int qdisc_set_default(const char *id); 102 103 void qdisc_hash_add(struct Qdisc *q, bool invisible); 104 void qdisc_hash_del(struct Qdisc *q); 105 struct Qdisc *qdisc_lookup(struct net_device *dev, u32 handle); 106 struct Qdisc *qdisc_lookup_rcu(struct net_device *dev, u32 handle); 107 struct qdisc_rate_table *qdisc_get_rtab(struct tc_ratespec *r, 108 struct nlattr *tab, 109 struct netlink_ext_ack *extack); 110 void qdisc_put_rtab(struct qdisc_rate_table *tab); 111 void qdisc_put_stab(struct qdisc_size_table *tab); 112 bool sch_direct_xmit(struct sk_buff *skb, struct Qdisc *q, 113 struct net_device *dev, struct netdev_queue *txq, 114 spinlock_t *root_lock, bool validate); 115 116 void __qdisc_run(struct Qdisc *q); 117 118 static inline struct sk_buff *qdisc_run(struct Qdisc *q) 119 { 120 if (qdisc_run_begin(q)) { 121 __qdisc_run(q); 122 return qdisc_run_end(q); 123 } 124 return NULL; 125 } 126 127 extern const struct nla_policy rtm_tca_policy[TCA_MAX + 1]; 128 129 /* Calculate maximal size of packet seen by hard_start_xmit 130 routine of this device. 131 */ 132 static inline unsigned int psched_mtu(const struct net_device *dev) 133 { 134 return READ_ONCE(dev->mtu) + dev->hard_header_len; 135 } 136 137 static inline struct net *qdisc_net(struct Qdisc *q) 138 { 139 return dev_net(q->dev_queue->dev); 140 } 141 142 struct tc_query_caps_base { 143 enum tc_setup_type type; 144 void *caps; 145 }; 146 147 struct tc_cbs_qopt_offload { 148 u8 enable; 149 s32 queue; 150 s32 hicredit; 151 s32 locredit; 152 s32 idleslope; 153 s32 sendslope; 154 }; 155 156 struct tc_etf_qopt_offload { 157 u8 enable; 158 s32 queue; 159 }; 160 161 struct tc_mqprio_caps { 162 bool validate_queue_counts:1; 163 }; 164 165 struct tc_mqprio_qopt_offload { 166 /* struct tc_mqprio_qopt must always be the first element */ 167 struct tc_mqprio_qopt qopt; 168 struct netlink_ext_ack *extack; 169 u16 mode; 170 u16 shaper; 171 u32 flags; 172 u64 min_rate[TC_QOPT_MAX_QUEUE]; 173 u64 max_rate[TC_QOPT_MAX_QUEUE]; 174 unsigned long preemptible_tcs; 175 }; 176 177 struct tc_taprio_caps { 178 bool supports_queue_max_sdu:1; 179 bool gate_mask_per_txq:1; 180 /* Device expects lower TXQ numbers to have higher priority over higher 181 * TXQs, regardless of their TC mapping. DO NOT USE FOR NEW DRIVERS, 182 * INSTEAD ENFORCE A PROPER TC:TXQ MAPPING COMING FROM USER SPACE. 183 */ 184 bool broken_mqprio:1; 185 }; 186 187 enum tc_taprio_qopt_cmd { 188 TAPRIO_CMD_REPLACE, 189 TAPRIO_CMD_DESTROY, 190 TAPRIO_CMD_STATS, 191 TAPRIO_CMD_QUEUE_STATS, 192 }; 193 194 /** 195 * struct tc_taprio_qopt_stats - IEEE 802.1Qbv statistics 196 * @window_drops: Frames that were dropped because they were too large to be 197 * transmitted in any of the allotted time windows (open gates) for their 198 * traffic class. 199 * @tx_overruns: Frames still being transmitted by the MAC after the 200 * transmission gate associated with their traffic class has closed. 201 * Equivalent to `12.29.1.1.2 TransmissionOverrun` from 802.1Q-2018. 202 */ 203 struct tc_taprio_qopt_stats { 204 u64 window_drops; 205 u64 tx_overruns; 206 }; 207 208 struct tc_taprio_qopt_queue_stats { 209 int queue; 210 struct tc_taprio_qopt_stats stats; 211 }; 212 213 struct tc_taprio_sched_entry { 214 u8 command; /* TC_TAPRIO_CMD_* */ 215 216 /* The gate_mask in the offloading side refers to traffic classes */ 217 u32 gate_mask; 218 u32 interval; 219 }; 220 221 struct tc_taprio_qopt_offload { 222 enum tc_taprio_qopt_cmd cmd; 223 224 union { 225 /* TAPRIO_CMD_STATS */ 226 struct tc_taprio_qopt_stats stats; 227 /* TAPRIO_CMD_QUEUE_STATS */ 228 struct tc_taprio_qopt_queue_stats queue_stats; 229 /* TAPRIO_CMD_REPLACE */ 230 struct { 231 struct tc_mqprio_qopt_offload mqprio; 232 struct netlink_ext_ack *extack; 233 ktime_t base_time; 234 u64 cycle_time; 235 u64 cycle_time_extension; 236 u32 max_sdu[TC_MAX_QUEUE]; 237 238 size_t num_entries; 239 struct tc_taprio_sched_entry entries[]; 240 }; 241 }; 242 }; 243 244 #if IS_ENABLED(CONFIG_NET_SCH_TAPRIO) 245 246 /* Reference counting */ 247 struct tc_taprio_qopt_offload *taprio_offload_get(struct tc_taprio_qopt_offload 248 *offload); 249 void taprio_offload_free(struct tc_taprio_qopt_offload *offload); 250 251 #else 252 253 /* Reference counting */ 254 static inline struct tc_taprio_qopt_offload * 255 taprio_offload_get(struct tc_taprio_qopt_offload *offload) 256 { 257 return NULL; 258 } 259 260 static inline void taprio_offload_free(struct tc_taprio_qopt_offload *offload) 261 { 262 } 263 264 #endif 265 266 /* Ensure skb_mstamp_ns, which might have been populated with the txtime, is 267 * not mistaken for a software timestamp, because this will otherwise prevent 268 * the dispatch of hardware timestamps to the socket. 269 */ 270 static inline void skb_txtime_consumed(struct sk_buff *skb) 271 { 272 skb->tstamp = ktime_set(0, 0); 273 } 274 275 static inline bool tc_qdisc_stats_dump(struct Qdisc *sch, 276 unsigned long cl, 277 struct qdisc_walker *arg) 278 { 279 if (arg->count >= arg->skip && arg->fn(sch, cl, arg) < 0) { 280 arg->stop = 1; 281 return false; 282 } 283 284 arg->count++; 285 return true; 286 } 287 288 static inline void qdisc_warn_nonwc(const char *txt, struct Qdisc *qdisc) 289 { 290 if (!(qdisc->flags & TCQ_F_WARN_NONWC)) { 291 pr_warn("%s: %s qdisc %X: is non-work-conserving?\n", 292 txt, qdisc->ops->id, qdisc->handle >> 16); 293 qdisc->flags |= TCQ_F_WARN_NONWC; 294 } 295 } 296 297 static inline unsigned int qdisc_peek_len(struct Qdisc *sch) 298 { 299 struct sk_buff *skb; 300 unsigned int len; 301 302 skb = sch->ops->peek(sch); 303 if (unlikely(skb == NULL)) { 304 qdisc_warn_nonwc("qdisc_peek_len", sch); 305 return 0; 306 } 307 len = qdisc_pkt_len(skb); 308 309 return len; 310 } 311 312 static inline void qdisc_lock_init(struct Qdisc *sch, 313 const struct Qdisc_ops *ops) 314 { 315 spin_lock_init(&sch->q.lock); 316 317 /* Skip dynamic keys if nesting is not possible */ 318 if (ops->static_flags & TCQ_F_INGRESS || 319 ops == &noqueue_qdisc_ops) 320 return; 321 322 lockdep_register_key(&sch->root_lock_key); 323 lockdep_set_class(&sch->q.lock, &sch->root_lock_key); 324 } 325 326 static inline void qdisc_lock_uninit(struct Qdisc *sch, 327 const struct Qdisc_ops *ops) 328 { 329 if (ops->static_flags & TCQ_F_INGRESS || 330 ops == &noqueue_qdisc_ops) 331 return; 332 333 lockdep_unregister_key(&sch->root_lock_key); 334 } 335 336 #endif 337