1#!/usr/bin/env python3 2# SPDX-License-Identifier: GPL-2.0 3 4"""Test suite for PSP capable drivers.""" 5 6import errno 7import fcntl 8import os 9import socket 10import struct 11import termios 12import time 13 14from contextlib import contextmanager 15 16from lib.py import defer 17from lib.py import ksft_run, ksft_exit, ksft_pr 18from lib.py import ksft_true, ksft_eq, ksft_ne, ksft_gt, ksft_raises 19from lib.py import ksft_not_none 20from lib.py import ksft_variants, KsftNamedVariant 21from lib.py import KsftSkipEx, KsftFailEx 22from lib.py import NetDrvEpEnv, NetDrvContEnv 23from lib.py import Netlink, NlError, PSPFamily, RtnlFamily 24from lib.py import NetNSEnter 25from lib.py import bkg, rand_port, wait_port_listen 26from lib.py import ip 27 28TCP_ULP = 31 29 30 31def _get_outq(s): 32 one = b'\0' * 4 33 outq = fcntl.ioctl(s.fileno(), termios.TIOCOUTQ, one) 34 return struct.unpack("I", outq)[0] 35 36 37def _send_with_ack(cfg, msg): 38 cfg.comm_sock.send(msg) 39 response = cfg.comm_sock.recv(4) 40 if response != b'ack\0': 41 raise RuntimeError("Unexpected server response", response) 42 43 44def _remote_read_len(cfg): 45 cfg.comm_sock.send(b'read len\0') 46 return int(cfg.comm_sock.recv(1024)[:-1].decode('utf-8')) 47 48 49def _make_clr_conn(cfg, ipver=None): 50 _send_with_ack(cfg, b'conn clr\0') 51 remote_addr = cfg.remote_addr_v[ipver] if ipver else cfg.remote_addr 52 s = socket.create_connection((remote_addr, cfg.comm_port), ) 53 return s 54 55 56def _make_psp_conn(cfg, version=0, ipver=None): 57 _send_with_ack(cfg, b'conn psp\0' + struct.pack('BB', version, version)) 58 remote_addr = cfg.remote_addr_v[ipver] if ipver else cfg.remote_addr 59 s = socket.create_connection((remote_addr, cfg.comm_port), ) 60 return s 61 62 63@contextmanager 64def _make_lo_conn(): 65 # After tx-assoc, the client's egress is dropped, since lo has no 66 # psp_dev, so its FIN never reaches the server. Closing the server 67 # resets the unaccepted child, and the client accepts the cleartext 68 # RST because it hasn't received any PSP traffic yet. 69 with socket.create_server(("localhost", 0)) as srv, \ 70 socket.create_connection(srv.getsockname()[:2]) as s: 71 yield s 72 73 74def _close_conn(cfg, s): 75 _send_with_ack(cfg, b'data close\0') 76 s.close() 77 78 79def _close_psp_conn(cfg, s): 80 _close_conn(cfg, s) 81 82 83def _spi_xchg(s, rx): 84 s.send(struct.pack('I', rx['spi']) + rx['key']) 85 tx = s.recv(4 + len(rx['key'])) 86 return { 87 'spi': struct.unpack('I', tx[:4])[0], 88 'key': tx[4:] 89 } 90 91 92def _send_careful(cfg, s, rounds): 93 data = b'0123456789' * 200 94 for i in range(rounds): 95 n = 0 96 for _ in range(10): # allow 10 retries 97 try: 98 n += s.send(data[n:], socket.MSG_DONTWAIT) 99 if n == len(data): 100 break 101 except BlockingIOError: 102 time.sleep(0.05) 103 else: 104 rlen = _remote_read_len(cfg) 105 outq = _get_outq(s) 106 report = f'sent: {i * len(data) + n} remote len: {rlen} outq: {outq}' 107 raise RuntimeError(report) 108 109 return len(data) * rounds 110 111 112def _check_data_rx(cfg, exp_len): 113 read_len = -1 114 for _ in range(30): 115 cfg.comm_sock.send(b'read len\0') 116 read_len = int(cfg.comm_sock.recv(1024)[:-1].decode('utf-8')) 117 if read_len == exp_len: 118 break 119 time.sleep(0.01) 120 ksft_eq(read_len, exp_len) 121 122 123def _check_data_outq(s, exp_len, force_wait=False): 124 outq = 0 125 for _ in range(10): 126 outq = _get_outq(s) 127 if not force_wait and outq == exp_len: 128 break 129 time.sleep(0.01) 130 ksft_eq(outq, exp_len) 131 132 133def _get_stat(cfg, key): 134 return cfg.pspnl.get_stats({'dev-id': cfg.psp_dev_id})[key] 135 136# 137# Test case boiler plate 138# 139 140def _init_psp_dev(cfg, use_psp_ifindex=False): 141 if not hasattr(cfg, 'psp_dev_id'): 142 # Figure out which local device we are testing against 143 # For NetDrvContEnv: use psp_ifindex instead of ifindex 144 target_ifindex = cfg.psp_ifindex if use_psp_ifindex else cfg.ifindex 145 for dev in cfg.pspnl.dev_get({}, dump=True): 146 if dev['ifindex'] == target_ifindex: 147 cfg.psp_info = dev 148 cfg.psp_dev_id = cfg.psp_info['id'] 149 break 150 else: 151 raise KsftSkipEx("No PSP devices found") 152 153 # Enable PSP if necessary 154 cap = cfg.psp_info['psp-versions-cap'] 155 ena = cfg.psp_info['psp-versions-ena'] 156 if cap != ena: 157 cfg.pspnl.dev_set({'id': cfg.psp_dev_id, 'psp-versions-ena': cap}) 158 defer(cfg.pspnl.dev_set, {'id': cfg.psp_dev_id, 159 'psp-versions-ena': ena }) 160 161# 162# Test cases 163# 164 165def dev_list_devices(cfg): 166 """ Dump all devices """ 167 _init_psp_dev(cfg) 168 169 devices = cfg.pspnl.dev_get({}, dump=True) 170 171 found = False 172 for dev in devices: 173 found |= dev['id'] == cfg.psp_dev_id 174 ksft_true(found) 175 176 177def dev_get_device(cfg): 178 """ Get the device we intend to use """ 179 _init_psp_dev(cfg) 180 181 dev = cfg.pspnl.dev_get({'id': cfg.psp_dev_id}) 182 ksft_eq(dev['id'], cfg.psp_dev_id) 183 184 185def dev_get_device_bad(cfg): 186 """ Test getting device which doesn't exist """ 187 raised = False 188 try: 189 cfg.pspnl.dev_get({'id': 1234567}) 190 except NlError as e: 191 ksft_eq(e.nl_msg.error, -errno.ENODEV) 192 raised = True 193 ksft_true(raised) 194 195 196def dev_rotate(cfg): 197 """ Test key rotation """ 198 _init_psp_dev(cfg) 199 200 prev_rotations = _get_stat(cfg, 'key-rotations') 201 202 rot = cfg.pspnl.key_rotate({"id": cfg.psp_dev_id}) 203 ksft_eq(rot['id'], cfg.psp_dev_id) 204 rot = cfg.pspnl.key_rotate({"id": cfg.psp_dev_id}) 205 ksft_eq(rot['id'], cfg.psp_dev_id) 206 207 cur_rotations = _get_stat(cfg, 'key-rotations') 208 ksft_eq(cur_rotations, prev_rotations + 2) 209 210 211def dev_rotate_spi(cfg): 212 """ Test key rotation and SPI check """ 213 _init_psp_dev(cfg) 214 215 top_a = top_b = 0 216 with _make_lo_conn() as s: 217 assoc_a = cfg.pspnl.rx_assoc({"version": 0, 218 "dev-id": cfg.psp_dev_id, 219 "sock-fd": s.fileno()}) 220 top_a = assoc_a['rx-key']['spi'] >> 31 221 rot = cfg.pspnl.key_rotate({"id": cfg.psp_dev_id}) 222 with _make_lo_conn() as s: 223 ksft_eq(rot['id'], cfg.psp_dev_id) 224 assoc_b = cfg.pspnl.rx_assoc({"version": 0, 225 "dev-id": cfg.psp_dev_id, 226 "sock-fd": s.fileno()}) 227 top_b = assoc_b['rx-key']['spi'] >> 31 228 ksft_ne(top_a, top_b) 229 230 231def assoc_basic(cfg): 232 """ Test creating associations """ 233 _init_psp_dev(cfg) 234 235 with _make_lo_conn() as s: 236 assoc = cfg.pspnl.rx_assoc({"version": 0, 237 "dev-id": cfg.psp_dev_id, 238 "sock-fd": s.fileno()}) 239 ksft_eq(assoc['dev-id'], cfg.psp_dev_id) 240 ksft_gt(assoc['rx-key']['spi'], 0) 241 ksft_eq(len(assoc['rx-key']['key']), 16) 242 243 assoc = cfg.pspnl.tx_assoc({"dev-id": cfg.psp_dev_id, 244 "version": 0, 245 "tx-key": assoc['rx-key'], 246 "sock-fd": s.fileno()}) 247 ksft_eq(len(assoc), 0) 248 249 250def assoc_bad_dev(cfg): 251 """ Test creating associations with bad device ID """ 252 _init_psp_dev(cfg) 253 254 with socket.socket(socket.AF_INET6, socket.SOCK_STREAM) as s: 255 with ksft_raises(NlError) as cm: 256 cfg.pspnl.rx_assoc({"version": 0, 257 "dev-id": cfg.psp_dev_id + 1234567, 258 "sock-fd": s.fileno()}) 259 ksft_eq(cm.exception.nl_msg.error, -errno.ENODEV) 260 261 262def assoc_sk_only_conn(cfg): 263 """ Test creating associations based on socket """ 264 _init_psp_dev(cfg) 265 266 with _make_clr_conn(cfg) as s: 267 assoc = cfg.pspnl.rx_assoc({"version": 0, 268 "sock-fd": s.fileno()}) 269 ksft_eq(assoc['dev-id'], cfg.psp_dev_id) 270 cfg.pspnl.tx_assoc({"version": 0, 271 "tx-key": assoc['rx-key'], 272 "sock-fd": s.fileno()}) 273 _close_conn(cfg, s) 274 275 276def assoc_sk_only_mismatch(cfg): 277 """ Test creating associations based on socket (dev mismatch) """ 278 _init_psp_dev(cfg) 279 280 with _make_clr_conn(cfg) as s: 281 with ksft_raises(NlError) as cm: 282 cfg.pspnl.rx_assoc({"version": 0, 283 "dev-id": cfg.psp_dev_id + 1234567, 284 "sock-fd": s.fileno()}) 285 the_exception = cm.exception 286 ksft_eq(the_exception.nl_msg.extack['bad-attr'], ".dev-id") 287 ksft_eq(the_exception.nl_msg.error, -errno.EINVAL) 288 _close_conn(cfg, s) 289 290 291def assoc_sk_only_mismatch_tx(cfg): 292 """ Test creating associations based on socket (dev mismatch) """ 293 _init_psp_dev(cfg) 294 295 with _make_clr_conn(cfg) as s: 296 with ksft_raises(NlError) as cm: 297 assoc = cfg.pspnl.rx_assoc({"version": 0, 298 "sock-fd": s.fileno()}) 299 cfg.pspnl.tx_assoc({"version": 0, 300 "tx-key": assoc['rx-key'], 301 "dev-id": cfg.psp_dev_id + 1234567, 302 "sock-fd": s.fileno()}) 303 the_exception = cm.exception 304 ksft_eq(the_exception.nl_msg.extack['bad-attr'], ".dev-id") 305 ksft_eq(the_exception.nl_msg.error, -errno.EINVAL) 306 _close_conn(cfg, s) 307 308 309def assoc_sk_only_unconn(cfg): 310 """ Test creating associations based on socket (unconnected, should fail) """ 311 _init_psp_dev(cfg) 312 313 with socket.socket(socket.AF_INET6, socket.SOCK_STREAM) as s: 314 with ksft_raises(NlError) as cm: 315 cfg.pspnl.rx_assoc({"version": 0, 316 "sock-fd": s.fileno()}) 317 the_exception = cm.exception 318 ksft_eq(the_exception.nl_msg.extack['miss-type'], "dev-id") 319 ksft_eq(the_exception.nl_msg.error, -errno.EINVAL) 320 321 322def assoc_rx_unconnected(cfg): 323 """ Test that an Rx assoc is rejected on an unconnected socket """ 324 _init_psp_dev(cfg) 325 326 with socket.socket(socket.AF_INET6, socket.SOCK_STREAM) as s: 327 with ksft_raises(NlError) as cm: 328 cfg.pspnl.rx_assoc({"version": 0, 329 "dev-id": cfg.psp_dev_id, 330 "sock-fd": s.fileno()}) 331 ksft_eq(cm.exception.nl_msg.error, -errno.ENOTCONN) 332 ksft_eq(cm.exception.nl_msg.extack['bad-attr'], ".sock-fd") 333 334 335def assoc_rx_listener(cfg): 336 """ Test that an Rx assoc is rejected on a listening socket """ 337 _init_psp_dev(cfg) 338 339 with socket.create_server(("localhost", 0)) as s: 340 with ksft_raises(NlError) as cm: 341 cfg.pspnl.rx_assoc({"version": 0, 342 "dev-id": cfg.psp_dev_id, 343 "sock-fd": s.fileno()}) 344 ksft_eq(cm.exception.nl_msg.error, -errno.ENOTCONN) 345 ksft_eq(cm.exception.nl_msg.extack['bad-attr'], ".sock-fd") 346 347 348def assoc_version_mismatch(cfg): 349 """ Test creating associations where Rx and Tx PSP versions do not match """ 350 _init_psp_dev(cfg) 351 352 versions = list(cfg.psp_info['psp-versions-cap']) 353 if len(versions) < 2: 354 raise KsftSkipEx("Not enough PSP versions supported by the device for the test") 355 356 # Translate versions to integers 357 versions = [cfg.pspnl.consts["version"].entries[v].value for v in versions] 358 359 with _make_lo_conn() as s: 360 rx = cfg.pspnl.rx_assoc({"version": versions[0], 361 "dev-id": cfg.psp_dev_id, 362 "sock-fd": s.fileno()}) 363 364 for version in versions[1:]: 365 with ksft_raises(NlError) as cm: 366 cfg.pspnl.tx_assoc({"dev-id": cfg.psp_dev_id, 367 "version": version, 368 "tx-key": rx['rx-key'], 369 "sock-fd": s.fileno()}) 370 the_exception = cm.exception 371 ksft_eq(the_exception.nl_msg.error, -errno.EINVAL) 372 373 374def _require_tls_ulp(): 375 with socket.create_server(("localhost", 0)) as srv, \ 376 socket.create_connection(srv.getsockname()) as s: 377 try: 378 s.setsockopt(socket.SOL_TCP, TCP_ULP, b"tls") 379 except OSError as exc: 380 raise KsftSkipEx("kTLS not available") from exc 381 382 383def assoc_psp_ulp_exclusive(cfg): 384 """ Test that a TCP ULP cannot be attached to a PSP socket """ 385 _init_psp_dev(cfg) 386 _require_tls_ulp() 387 388 with _make_clr_conn(cfg) as s: 389 try: 390 cfg.pspnl.rx_assoc({"version": 0, 391 "dev-id": cfg.psp_dev_id, 392 "sock-fd": s.fileno()}) 393 with ksft_raises(OSError) as cm: 394 s.setsockopt(socket.SOL_TCP, TCP_ULP, b"tls") 395 ksft_eq(cm.exception.errno, errno.EINVAL) 396 finally: 397 _close_conn(cfg, s) 398 399 400def assoc_ulp_psp_exclusive(cfg): 401 """ Test that a PSP assoc cannot be added to a socket with a TCP ULP """ 402 _init_psp_dev(cfg) 403 _require_tls_ulp() 404 405 with _make_clr_conn(cfg) as s: 406 try: 407 s.setsockopt(socket.SOL_TCP, TCP_ULP, b"tls") 408 with ksft_raises(NlError) as cm: 409 cfg.pspnl.rx_assoc({"version": 0, 410 "dev-id": cfg.psp_dev_id, 411 "sock-fd": s.fileno()}) 412 ksft_eq(cm.exception.nl_msg.error, -errno.EINVAL) 413 ksft_eq(cm.exception.nl_msg.extack['bad-attr'], ".sock-fd") 414 finally: 415 _close_conn(cfg, s) 416 417 418def assoc_twice(cfg): 419 """ Test reusing Tx assoc for two sockets """ 420 _init_psp_dev(cfg) 421 422 def rx_assoc_check(s): 423 assoc = cfg.pspnl.rx_assoc({"version": 0, 424 "dev-id": cfg.psp_dev_id, 425 "sock-fd": s.fileno()}) 426 ksft_eq(assoc['dev-id'], cfg.psp_dev_id) 427 ksft_gt(assoc['rx-key']['spi'], 0) 428 ksft_eq(len(assoc['rx-key']['key']), 16) 429 430 return assoc 431 432 with _make_lo_conn() as s: 433 assoc = rx_assoc_check(s) 434 tx = cfg.pspnl.tx_assoc({"dev-id": cfg.psp_dev_id, 435 "version": 0, 436 "tx-key": assoc['rx-key'], 437 "sock-fd": s.fileno()}) 438 ksft_eq(len(tx), 0) 439 440 # Use the same Tx assoc second time 441 with _make_lo_conn() as s2: 442 rx_assoc_check(s2) 443 tx = cfg.pspnl.tx_assoc({"dev-id": cfg.psp_dev_id, 444 "version": 0, 445 "tx-key": assoc['rx-key'], 446 "sock-fd": s2.fileno()}) 447 ksft_eq(len(tx), 0) 448 449 450def _data_basic_send(cfg, version, ipver): 451 """ Test basic data send """ 452 _init_psp_dev(cfg) 453 454 # Version 0 is required by spec, don't let it skip 455 if version: 456 name = cfg.pspnl.consts["version"].entries_by_val[version].name 457 if name not in cfg.psp_info['psp-versions-cap']: 458 with socket.socket(socket.AF_INET6, socket.SOCK_STREAM) as s: 459 with ksft_raises(NlError) as cm: 460 cfg.pspnl.rx_assoc({"version": version, 461 "dev-id": cfg.psp_dev_id, 462 "sock-fd": s.fileno()}) 463 ksft_eq(cm.exception.nl_msg.error, -errno.EOPNOTSUPP) 464 raise KsftSkipEx("PSP version not supported", name) 465 466 s = _make_psp_conn(cfg, version, ipver) 467 468 rx_assoc = cfg.pspnl.rx_assoc({"version": version, 469 "dev-id": cfg.psp_dev_id, 470 "sock-fd": s.fileno()}) 471 rx = rx_assoc['rx-key'] 472 tx = _spi_xchg(s, rx) 473 474 cfg.pspnl.tx_assoc({"dev-id": cfg.psp_dev_id, 475 "version": version, 476 "tx-key": tx, 477 "sock-fd": s.fileno()}) 478 479 data_len = _send_careful(cfg, s, 100) 480 _check_data_rx(cfg, data_len) 481 _close_psp_conn(cfg, s) 482 483 484def __bad_xfer_do(cfg, s, tx, version='hdr0-aes-gcm-128'): 485 # Make sure we accept the ACK for the SPI before we seal with the bad assoc 486 _check_data_outq(s, 0) 487 488 cfg.pspnl.tx_assoc({"dev-id": cfg.psp_dev_id, 489 "version": version, 490 "tx-key": tx, 491 "sock-fd": s.fileno()}) 492 493 data_len = _send_careful(cfg, s, 20) 494 _check_data_outq(s, data_len, force_wait=True) 495 _check_data_rx(cfg, 0) 496 _close_psp_conn(cfg, s) 497 498 499def data_send_bad_key(cfg): 500 """ Test send data with bad key """ 501 _init_psp_dev(cfg) 502 503 s = _make_psp_conn(cfg) 504 505 rx_assoc = cfg.pspnl.rx_assoc({"version": 0, 506 "dev-id": cfg.psp_dev_id, 507 "sock-fd": s.fileno()}) 508 rx = rx_assoc['rx-key'] 509 tx = _spi_xchg(s, rx) 510 tx['key'] = (tx['key'][0] ^ 0xff).to_bytes(1, 'little') + tx['key'][1:] 511 __bad_xfer_do(cfg, s, tx) 512 513 514def data_send_disconnect(cfg): 515 """ Test socket close after sending data """ 516 _init_psp_dev(cfg) 517 518 with _make_psp_conn(cfg) as s: 519 assoc = cfg.pspnl.rx_assoc({"version": 0, 520 "sock-fd": s.fileno()}) 521 tx = _spi_xchg(s, assoc['rx-key']) 522 cfg.pspnl.tx_assoc({"version": 0, 523 "tx-key": tx, 524 "sock-fd": s.fileno()}) 525 526 data_len = _send_careful(cfg, s, 100) 527 _check_data_rx(cfg, data_len) 528 529 s.shutdown(socket.SHUT_RDWR) 530 s.close() 531 532 533def _data_mss_adjust(cfg, ipver): 534 _init_psp_dev(cfg) 535 536 # First figure out what the MSS would be without any adjustments 537 s = _make_clr_conn(cfg, ipver) 538 s.send(b"0123456789abcdef" * 1024) 539 _check_data_rx(cfg, 16 * 1024) 540 mss = s.getsockopt(socket.IPPROTO_TCP, socket.TCP_MAXSEG) 541 _close_conn(cfg, s) 542 543 s = _make_psp_conn(cfg, 0, ipver) 544 try: 545 rx_assoc = cfg.pspnl.rx_assoc({"version": 0, 546 "dev-id": cfg.psp_dev_id, 547 "sock-fd": s.fileno()}) 548 rx = rx_assoc['rx-key'] 549 tx = _spi_xchg(s, rx) 550 551 rxmss = s.getsockopt(socket.IPPROTO_TCP, socket.TCP_MAXSEG) 552 ksft_eq(mss, rxmss) 553 554 cfg.pspnl.tx_assoc({"dev-id": cfg.psp_dev_id, 555 "version": 0, 556 "tx-key": tx, 557 "sock-fd": s.fileno()}) 558 559 txmss = s.getsockopt(socket.IPPROTO_TCP, socket.TCP_MAXSEG) 560 ksft_eq(mss, txmss + 40) 561 562 data_len = _send_careful(cfg, s, 100) 563 _check_data_rx(cfg, data_len) 564 _check_data_outq(s, 0) 565 566 txmss = s.getsockopt(socket.IPPROTO_TCP, socket.TCP_MAXSEG) 567 ksft_eq(mss, txmss + 40) 568 finally: 569 _close_psp_conn(cfg, s) 570 571 572def data_stale_key(cfg): 573 """ Test send on a double-rotated key """ 574 _init_psp_dev(cfg) 575 576 prev_stale = _get_stat(cfg, 'stale-events') 577 s = _make_psp_conn(cfg) 578 try: 579 rx_assoc = cfg.pspnl.rx_assoc({"version": 0, 580 "dev-id": cfg.psp_dev_id, 581 "sock-fd": s.fileno()}) 582 rx = rx_assoc['rx-key'] 583 tx = _spi_xchg(s, rx) 584 585 cfg.pspnl.tx_assoc({"dev-id": cfg.psp_dev_id, 586 "version": 0, 587 "tx-key": tx, 588 "sock-fd": s.fileno()}) 589 590 data_len = _send_careful(cfg, s, 100) 591 _check_data_rx(cfg, data_len) 592 _check_data_outq(s, 0) 593 594 cfg.pspnl.key_rotate({"id": cfg.psp_dev_id}) 595 cfg.pspnl.key_rotate({"id": cfg.psp_dev_id}) 596 597 cur_stale = _get_stat(cfg, 'stale-events') 598 ksft_gt(cur_stale, prev_stale) 599 600 s.send(b'0123456789' * 200) 601 _check_data_outq(s, 2000, force_wait=True) 602 finally: 603 _close_psp_conn(cfg, s) 604 605 606def __nsim_psp_rereg(cfg): 607 # The PSP dev ID will change, remember what was there before 608 before = {x['id'] for x in cfg.pspnl.dev_get({}, dump=True)} 609 610 cfg._ns.nsims[0].dfs_write('psp_rereg', '1') 611 612 after = {x['id'] for x in cfg.pspnl.dev_get({}, dump=True)} 613 614 new_devs = list(after - before) 615 ksft_eq(len(new_devs), 1) 616 cfg.psp_dev_id = list(after - before)[0] 617 618 619def removal_device_rx(cfg): 620 """ Test removing a netdev / PSD with active Rx assoc """ 621 622 # We could technically devlink reload real devices, too 623 # but that kills the control socket. So test this on 624 # netdevsim only for now 625 cfg.require_nsim() 626 627 s = _make_clr_conn(cfg) 628 try: 629 rx_assoc = cfg.pspnl.rx_assoc({"version": 0, 630 "dev-id": cfg.psp_dev_id, 631 "sock-fd": s.fileno()}) 632 ksft_not_none(rx_assoc) 633 634 __nsim_psp_rereg(cfg) 635 finally: 636 _close_conn(cfg, s) 637 638 639def removal_device_bi(cfg): 640 """ Test removing a netdev / PSD with active Rx/Tx assoc """ 641 642 # We could technically devlink reload real devices, too 643 # but that kills the control socket. So test this on 644 # netdevsim only for now 645 cfg.require_nsim() 646 647 s = _make_clr_conn(cfg) 648 try: 649 rx_assoc = cfg.pspnl.rx_assoc({"version": 0, 650 "dev-id": cfg.psp_dev_id, 651 "sock-fd": s.fileno()}) 652 cfg.pspnl.tx_assoc({"dev-id": cfg.psp_dev_id, 653 "version": 0, 654 "tx-key": rx_assoc['rx-key'], 655 "sock-fd": s.fileno()}) 656 __nsim_psp_rereg(cfg) 657 finally: 658 _close_conn(cfg, s) 659 660 661def _get_psp_ver_ip_variants(): 662 for ver in range(4): 663 for ipv in ("4", "6"): 664 yield KsftNamedVariant(f"v{ver}_ip{ipv}", ver, ipv) 665 666 667def _get_ip_variants(): 668 for ipv in ("4", "6"): 669 yield KsftNamedVariant(f"ip{ipv}", ipv) 670 671 672@ksft_variants(_get_psp_ver_ip_variants()) 673def data_basic_send(cfg, version, ipver): 674 """Test basic PSP data send.""" 675 cfg.require_ipver(ipver) 676 _data_basic_send(cfg, version, ipver) 677 678 679@ksft_variants(_get_ip_variants()) 680def data_mss_adjust(cfg, ipver): 681 """Test MSS adjustment with PSP.""" 682 cfg.require_ipver(ipver) 683 _data_mss_adjust(cfg, ipver) 684 685 686def _check_assoc_list(cfg, psp_dev_id, ifindex, nsid=None): 687 """Verify assoc-list contains device with given ifindex, no duplicates.""" 688 dev_info = cfg.pspnl.dev_get({'id': psp_dev_id}) 689 690 ksft_true('assoc-list' in dev_info, 691 "No assoc-list in dev_get() response after association") 692 found = False 693 for assoc in dev_info['assoc-list']: 694 if assoc['ifindex'] != ifindex: 695 continue 696 if nsid is not None and assoc['nsid'] != nsid: 697 continue 698 ksft_eq(found, False, "Duplicate assoc entry found") 699 found = True 700 ksft_eq(found, True, 701 "Associated device not found in dev_get() response") 702 703 704def _data_basic_send_netkit_psp_assoc(cfg, version, ipver): 705 """ 706 Test basic data send with netkit interface associated with PSP dev. 707 """ 708 _assoc_nk_guest(cfg) 709 710 # Enter guest namespace (netns) to run PSP test 711 with NetNSEnter(cfg.netns.name): 712 cfg.pspnl = PSPFamily() 713 714 sock = _make_psp_conn(cfg, version, ipver) 715 716 rx_assoc = cfg.pspnl.rx_assoc({"version": version, 717 "dev-id": cfg.psp_dev_id, 718 "sock-fd": sock.fileno()}) 719 rx_key = rx_assoc['rx-key'] 720 tx_key = _spi_xchg(sock, rx_key) 721 722 cfg.pspnl.tx_assoc({"dev-id": cfg.psp_dev_id, 723 "version": version, 724 "tx-key": tx_key, 725 "sock-fd": sock.fileno()}) 726 727 data_len = _send_careful(cfg, sock, 100) 728 _check_data_rx(cfg, data_len) 729 _close_psp_conn(cfg, sock) 730 731 732def _assoc_check_list(cfg): 733 """Test that assoc-list is correctly populated after dev-assoc.""" 734 _assoc_nk_guest(cfg) 735 _check_assoc_list(cfg, cfg.psp_dev_id, cfg.nk_guest_ifindex, 736 cfg.psp_dev_peer_nsid) 737 738 739def _get_psp_ver_ip6_variants(): 740 for ver in range(4): 741 yield KsftNamedVariant(f"v{ver}_ip6", ver, "6") 742 743 744@ksft_variants(_get_psp_ver_ip6_variants()) 745def data_basic_send_netkit_psp_assoc(cfg, version, ipver): 746 """Test PSP data send via netkit with dev-assoc.""" 747 cfg.require_ipver(ipver) 748 _data_basic_send_netkit_psp_assoc(cfg, version, ipver) 749 750 751def _key_rotation_notify_multi_ns_netkit(cfg): 752 """ Test key rotation notifications across multiple namespaces using netkit """ 753 _assoc_nk_guest(cfg) 754 755 # Create listener in guest namespace; socket stays bound to that ns 756 with NetNSEnter(cfg.netns.name): 757 peer_pspnl = PSPFamily() 758 peer_pspnl.ntf_subscribe('use') 759 760 # Create listener in main namespace 761 main_pspnl = PSPFamily() 762 main_pspnl.ntf_subscribe('use') 763 764 # Trigger key rotation on the PSP device 765 cfg.pspnl.key_rotate({"id": cfg.psp_dev_id}) 766 767 # Poll both sockets from main thread 768 for pspnl, label in [(main_pspnl, "main"), (peer_pspnl, "guest")]: 769 for ntf in pspnl.poll_ntf(duration=10): 770 if ntf['msg'].get('id') == cfg.psp_dev_id: 771 break 772 else: 773 raise KsftFailEx( 774 f"No key rotation notification received" 775 f" in {label} namespace") 776 777 778def _dev_change_notify_multi_ns_netkit(cfg): 779 """ Test dev_change notifications across multiple namespaces using netkit """ 780 _assoc_nk_guest(cfg) 781 782 # Create listener in guest namespace; socket stays bound to that ns 783 with NetNSEnter(cfg.netns.name): 784 peer_pspnl = PSPFamily() 785 peer_pspnl.ntf_subscribe('mgmt') 786 787 # Create listener in main namespace 788 main_pspnl = PSPFamily() 789 main_pspnl.ntf_subscribe('mgmt') 790 791 # Trigger dev_change by calling dev_set (notification is always sent) 792 cfg.pspnl.dev_set({'id': cfg.psp_dev_id, 793 'psp-versions-ena': cfg.psp_info['psp-versions-cap']}) 794 795 # Poll both sockets from main thread 796 for pspnl, label in [(main_pspnl, "main"), (peer_pspnl, "guest")]: 797 for ntf in pspnl.poll_ntf(duration=10): 798 if ntf['msg'].get('id') == cfg.psp_dev_id: 799 break 800 else: 801 raise KsftFailEx( 802 f"No dev_change notification received" 803 f" in {label} namespace") 804 805 806def _subscribe_mgmt(cfg): 807 """Listen on the mgmt group in the guest and in the main namespace.""" 808 # Listener in the guest namespace; socket stays bound to that ns 809 with NetNSEnter(cfg.netns.name): 810 peer_pspnl = PSPFamily() 811 peer_pspnl.ntf_subscribe('mgmt') 812 813 main_pspnl = PSPFamily() 814 main_pspnl.ntf_subscribe('mgmt') 815 816 return main_pspnl, peer_pspnl 817 818 819def _get_dev_ntf(cfg, pspnl, label): 820 """Wait for the next notification about the PSP device under test.""" 821 for ntf in pspnl.poll_ntf(duration=10): 822 if ntf['msg'].get('id') == cfg.psp_dev_id: 823 return ntf 824 raise KsftFailEx(f"No notification received in the {label} namespace") 825 826 827def _check_disassoc_ntf(cfg, main_pspnl, peer_pspnl, ifindex): 828 """Check the notifications for a netns losing its last association.""" 829 ntf = _get_dev_ntf(cfg, main_pspnl, "main") 830 ksft_eq(ntf['name'], 'dev-change-ntf') 831 for assoc in ntf['msg'].get('assoc-list', []): 832 if assoc['nsid'] != cfg.psp_dev_peer_nsid: 833 continue 834 ksft_ne(assoc['ifindex'], ifindex, 835 "Disassociated device still listed in the notification") 836 837 # The device is gone as far as the disassociated namespace is concerned 838 ntf = _get_dev_ntf(cfg, peer_pspnl, "guest") 839 ksft_eq(ntf['name'], 'dev-del-ntf') 840 ksft_true('ifindex' not in ntf['msg'], 841 "ifindex reported to an associated namespace") 842 843 844def _dev_disassoc_notify_multi_ns_netkit(cfg): 845 """ Test the notifications dev-disassoc generates in both namespaces """ 846 _init_psp_dev(cfg, True) 847 defer(delattr, cfg, 'psp_dev_id') 848 defer(delattr, cfg, 'psp_info') 849 850 cfg.pspnl.dev_assoc({'id': cfg.psp_dev_id, 851 'ifindex': cfg.nk_guest_ifindex, 852 'nsid': cfg.psp_dev_peer_nsid}) 853 defer(_try_disassoc, cfg, cfg.psp_dev_id, cfg.nk_guest_ifindex, 854 cfg.psp_dev_peer_nsid) 855 856 main_pspnl, peer_pspnl = _subscribe_mgmt(cfg) 857 858 cfg.pspnl.dev_disassoc({'id': cfg.psp_dev_id, 859 'ifindex': cfg.nk_guest_ifindex, 860 'nsid': cfg.psp_dev_peer_nsid}) 861 862 _check_disassoc_ntf(cfg, main_pspnl, peer_pspnl, cfg.nk_guest_ifindex) 863 864 865def _dev_disassoc_notify_one_of_two_netkit(cfg): 866 """Test the notifications with two netkits associated in one netns. 867 868 Disassociating the first netkit leaves the PSP device visible in the 869 guest namespace, generates a dev-change-ntf. 870 Disassociating the second one takes the device out of its view, 871 generates 'dev-del-ntf'. 872 """ 873 _init_psp_dev(cfg, True) 874 defer(delattr, cfg, 'psp_dev_id') 875 defer(delattr, cfg, 'psp_info') 876 877 tmp_ifindex, _ = _add_netkit_guest(cfg, "tmp_nk_host", "tmp_nk_guest") 878 879 for ifindex in [cfg.nk_guest_ifindex, tmp_ifindex]: 880 cfg.pspnl.dev_assoc({'id': cfg.psp_dev_id, 'ifindex': ifindex, 881 'nsid': cfg.psp_dev_peer_nsid}) 882 defer(_try_disassoc, cfg, cfg.psp_dev_id, ifindex, 883 cfg.psp_dev_peer_nsid) 884 885 main_pspnl, peer_pspnl = _subscribe_mgmt(cfg) 886 887 # One of the two goes away, the device stays visible in the guest netns 888 cfg.pspnl.dev_disassoc({'id': cfg.psp_dev_id, 'ifindex': tmp_ifindex, 889 'nsid': cfg.psp_dev_peer_nsid}) 890 891 ntf = _get_dev_ntf(cfg, main_pspnl, "main") 892 ksft_eq(ntf['name'], 'dev-change-ntf') 893 894 ntf = _get_dev_ntf(cfg, peer_pspnl, "guest") 895 ksft_eq(ntf['name'], 'dev-change-ntf') 896 found = False 897 for assoc in ntf['msg'].get('assoc-list', []): 898 ksft_ne(assoc['ifindex'], tmp_ifindex, 899 "Disassociated device still listed in the notification") 900 found |= assoc['ifindex'] == cfg.nk_guest_ifindex 901 ksft_true(found, "Remaining association missing from the notification") 902 903 # And now the last one, the device disappears from the guest netns 904 cfg.pspnl.dev_disassoc({'id': cfg.psp_dev_id, 905 'ifindex': cfg.nk_guest_ifindex, 906 'nsid': cfg.psp_dev_peer_nsid}) 907 908 _check_disassoc_ntf(cfg, main_pspnl, peer_pspnl, cfg.nk_guest_ifindex) 909 910 911def _psp_dev_get_check_netkit_psp_assoc(cfg): 912 """ Check psp dev-get output with netkit interface associated with PSP dev """ 913 _assoc_nk_guest(cfg) 914 915 # Check 1: In default netns, verify dev-get has correct ifindex and assoc-list 916 dev_info = cfg.pspnl.dev_get({'id': cfg.psp_dev_id}) 917 ksft_eq(dev_info['ifindex'], cfg.psp_ifindex) 918 _check_assoc_list(cfg, cfg.psp_dev_id, cfg.nk_guest_ifindex, 919 cfg.psp_dev_peer_nsid) 920 921 # Check 2: In guest netns, verify dev-get has assoc-list with nk_guest device 922 with NetNSEnter(cfg.netns.name): 923 peer_pspnl = PSPFamily() 924 925 # Dump all devices in the guest namespace 926 peer_devices = peer_pspnl.dev_get({}, dump=True) 927 928 # Find the device with by-association flag 929 peer_dev = None 930 for dev in peer_devices: 931 if dev.get('by-association'): 932 peer_dev = dev 933 break 934 935 ksft_not_none(peer_dev, "No PSP device found with by-association flag in guest netns") 936 937 # ifindex of the main netdevice means nothing in this namespace 938 ksft_true('ifindex' not in peer_dev, 939 "ifindex reported to an associated namespace") 940 941 # Verify assoc-list contains the nk_guest device 942 ksft_true('assoc-list' in peer_dev and len(peer_dev['assoc-list']) > 0, 943 "Guest device should have assoc-list with local devices") 944 945 # Verify the assoc-list contains nk_guest ifindex with nsid=-1 (same namespace) 946 found = False 947 for assoc in peer_dev['assoc-list']: 948 if assoc['ifindex'] == cfg.nk_guest_ifindex: 949 ksft_eq(assoc['nsid'], -1, 950 "nsid should be -1 (NETNSA_NSID_NOT_ASSIGNED) for same-namespace device") 951 found = True 952 break 953 ksft_true(found, "nk_guest ifindex not found in assoc-list") 954 955 956def _dev_assoc_no_nsid(cfg): 957 """ Test dev-assoc and dev-disassoc without nsid attribute """ 958 _init_psp_dev(cfg, True) 959 960 # Associate without nsid - should look up ifindex in caller's netns 961 cfg.pspnl.dev_assoc({'id': cfg.psp_dev_id, 962 'ifindex': cfg.nk_host_ifindex}) 963 defer(_try_disassoc, cfg, 964 cfg.psp_dev_id, cfg.nk_host_ifindex) 965 defer(delattr, cfg, 'psp_dev_id') 966 defer(delattr, cfg, 'psp_info') 967 968 # Verify assoc-list contains the device (match by ifindex only) 969 _check_assoc_list(cfg, cfg.psp_dev_id, cfg.nk_host_ifindex) 970 971 # Disassociate without nsid - should also use caller's netns 972 cfg.pspnl.dev_disassoc({'id': cfg.psp_dev_id, 973 'ifindex': cfg.nk_host_ifindex}) 974 975 # Verify assoc-list no longer contains the device 976 dev_info = cfg.pspnl.dev_get({'id': cfg.psp_dev_id}) 977 found = False 978 if 'assoc-list' in dev_info: 979 for assoc in dev_info['assoc-list']: 980 if assoc['ifindex'] == cfg.nk_host_ifindex: 981 found = True 982 break 983 ksft_true(not found, "Device should not be in assoc-list after disassociation") 984 985 986def _add_netkit_guest(cfg, host_name, guest_name): 987 """Create a netkit pair and move its peer into the test namespace. 988 989 Returns the peer's ifindex there and the defer() deleting the pair. 990 """ 991 existing = {link['ifindex'] for link in ip("-d link show", json=True) 992 if link.get('linkinfo', {}).get('info_kind') == 'netkit'} 993 994 rtnl = RtnlFamily() 995 rtnl.newlink( 996 { 997 "ifname": host_name, 998 "linkinfo": { 999 "kind": "netkit", 1000 "data": { 1001 "mode": "l2", 1002 "policy": "forward", 1003 "peer-policy": "forward", 1004 }, 1005 }, 1006 }, 1007 flags=[Netlink.NLM_F_CREATE, Netlink.NLM_F_EXCL], 1008 ) 1009 cleanup = defer(ip, f"link del {host_name}") 1010 1011 # Find the peer by diffing against existing netkit ifindexes 1012 all_links = ip("-d link show", json=True) 1013 peer = [link for link in all_links 1014 if link.get('linkinfo', {}).get('info_kind') == 'netkit' 1015 and link['ifindex'] not in existing 1016 and link['ifname'] != host_name] 1017 ksft_eq(len(peer), 1, "Failed to find the new netkit peer") 1018 1019 # Rename and move guest end into the test namespace 1020 ip(f"link set dev {peer[0]['ifname']} name {guest_name}") 1021 ip(f"link set dev {guest_name} netns {cfg.netns.name}") 1022 guest_dev = ip(f"link show dev {guest_name}", json=True, ns=cfg.netns)[0] 1023 ip(f"link set dev {guest_name} up", ns=cfg.netns) 1024 1025 return guest_dev['ifindex'], cleanup 1026 1027 1028def _psp_dev_assoc_cleanup_on_netkit_del(cfg): 1029 """Test that assoc-list is cleared when associated netkit is deleted. 1030 1031 Creates a disposable netkit pair for this test to avoid destroying 1032 the shared environment. 1033 """ 1034 _init_psp_dev(cfg, True) 1035 defer(delattr, cfg, 'psp_dev_id') 1036 defer(delattr, cfg, 'psp_info') 1037 1038 tmp_host_name = "tmp_nk_host" 1039 tmp_guest_ifindex, cleanup_netkit = _add_netkit_guest(cfg, tmp_host_name, 1040 "tmp_nk_guest") 1041 1042 # Associate PSP device with the temporary guest interface 1043 cfg.pspnl.dev_assoc({'id': cfg.psp_dev_id, 1044 'ifindex': tmp_guest_ifindex, 1045 'nsid': cfg.psp_dev_peer_nsid}) 1046 1047 # Verify assoc-list contains the temporary device 1048 _check_assoc_list(cfg, cfg.psp_dev_id, tmp_guest_ifindex, 1049 cfg.psp_dev_peer_nsid) 1050 1051 # Removing the netdevice is notified like a disassociation 1052 main_pspnl, peer_pspnl = _subscribe_mgmt(cfg) 1053 1054 # Delete the temporary netkit pair (deleting one end removes both) 1055 ip(f"link del {tmp_host_name}") 1056 cleanup_netkit.cancel() 1057 1058 # Verify assoc-list is cleared after netkit deletion 1059 dev_info = cfg.pspnl.dev_get({'id': cfg.psp_dev_id}) 1060 ksft_true('assoc-list' not in dev_info 1061 or len(dev_info['assoc-list']) == 0, 1062 "assoc-list should be empty after netkit deletion") 1063 1064 _check_disassoc_ntf(cfg, main_pspnl, peer_pspnl, tmp_guest_ifindex) 1065 1066 1067def _try_disassoc(cfg, psp_dev_id, ifindex, nsid=None): 1068 """Best-effort disassociate, ignoring errors if already removed.""" 1069 try: 1070 params = {'id': psp_dev_id, 'ifindex': ifindex} 1071 if nsid is not None: 1072 params['nsid'] = nsid 1073 cfg.pspnl.dev_disassoc(params) 1074 except NlError: 1075 pass 1076 1077 1078def _assoc_nk_guest(cfg): 1079 """Associate nk_guest with PSP device and register cleanup via defer().""" 1080 _init_psp_dev(cfg, True) 1081 1082 cfg.pspnl.dev_assoc({'id': cfg.psp_dev_id, 1083 'ifindex': cfg.nk_guest_ifindex, 1084 'nsid': cfg.psp_dev_peer_nsid}) 1085 defer(_disassoc_nk_guest, cfg, 1086 cfg.psp_dev_id, cfg.nk_guest_ifindex) 1087 1088 1089def _disassoc_nk_guest(cfg, psp_dev_id, nk_guest_ifindex): 1090 """Disassociate nk_guest and reset cfg PSP state.""" 1091 pspnl = PSPFamily() 1092 pspnl.dev_disassoc({'id': psp_dev_id, 'ifindex': nk_guest_ifindex, 1093 'nsid': cfg.psp_dev_peer_nsid}) 1094 cfg.pspnl = pspnl 1095 del cfg.psp_dev_id 1096 del cfg.psp_info 1097 1098 1099def _get_nsid(ns_name): 1100 """Get the nsid for a namespace.""" 1101 for entry in ip("netns list-id", json=True): 1102 if entry.get("name") == str(ns_name): 1103 return entry["nsid"] 1104 raise KsftSkipEx(f"nsid not found for namespace {ns_name}") 1105 1106 1107def _setup_psp_attributes(cfg): 1108 # pylint: disable=protected-access 1109 """ 1110 Set up PSP-specific attributes on the environment. 1111 1112 This sets attributes needed for PSP tests based on whether we're using 1113 netdevsim or a real NIC. 1114 """ 1115 if cfg._ns is not None: 1116 # netdevsim case: PSP device is the local dev (in host namespace) 1117 cfg.psp_dev = cfg._ns.nsims[0].dev 1118 cfg.psp_ifname = cfg.psp_dev['ifname'] 1119 cfg.psp_ifindex = cfg.psp_dev['ifindex'] 1120 1121 # PSP peer device is the remote dev (in _netns, where psp_responder runs) 1122 cfg.psp_dev_peer = cfg._ns_peer.nsims[0].dev 1123 cfg.psp_dev_peer_ifname = cfg.psp_dev_peer['ifname'] 1124 cfg.psp_dev_peer_ifindex = cfg.psp_dev_peer['ifindex'] 1125 else: 1126 # Real NIC case: PSP device is the local interface 1127 cfg.psp_dev = cfg.dev 1128 cfg.psp_ifname = cfg.ifname 1129 cfg.psp_ifindex = cfg.ifindex 1130 1131 # PSP peer device is the remote interface 1132 cfg.psp_dev_peer = cfg.remote_dev 1133 cfg.psp_dev_peer_ifname = cfg.remote_ifname 1134 cfg.psp_dev_peer_ifindex = cfg.remote_ifindex 1135 1136 # Get nsid for the guest namespace (netns) where nk_guest is 1137 cfg.psp_dev_peer_nsid = _get_nsid(cfg.netns.name) 1138 1139 1140 1141def main() -> None: 1142 """ Ksft boiler plate main """ 1143 1144 # Make sure LOCAL_PREFIX_V6 is set 1145 if "LOCAL_PREFIX_V6" not in os.environ: 1146 os.environ["LOCAL_PREFIX_V6"] = "2001:db8:2::" 1147 1148 try: 1149 env = NetDrvContEnv(__file__, primary_rx_redirect=True) 1150 has_cont = True 1151 except KsftSkipEx: 1152 env = NetDrvEpEnv(__file__) 1153 has_cont = False 1154 1155 with env as cfg: 1156 cfg.pspnl = PSPFamily() 1157 1158 if has_cont: 1159 _setup_psp_attributes(cfg) 1160 1161 # Set up responder and communication sock 1162 # psp_responder runs in _netns (remote namespace with psp_dev_peer) 1163 responder = cfg.remote.deploy("psp_responder") 1164 1165 cfg.comm_port = rand_port() 1166 srv = None 1167 try: 1168 with bkg(responder + f" -p {cfg.comm_port} -i {cfg.remote_ifindex}", 1169 host=cfg.remote, exit_wait=True) as srv: 1170 wait_port_listen(cfg.comm_port, host=cfg.remote) 1171 1172 cfg.comm_sock = socket.create_connection((cfg.remote_addr, 1173 cfg.comm_port), 1174 timeout=1) 1175 1176 cases = [data_basic_send, data_mss_adjust] 1177 1178 if has_cont: 1179 cases += [ 1180 _assoc_check_list, 1181 data_basic_send_netkit_psp_assoc, 1182 _key_rotation_notify_multi_ns_netkit, 1183 _dev_change_notify_multi_ns_netkit, 1184 _dev_disassoc_notify_multi_ns_netkit, 1185 _dev_disassoc_notify_one_of_two_netkit, 1186 _psp_dev_get_check_netkit_psp_assoc, 1187 _dev_assoc_no_nsid, 1188 _psp_dev_assoc_cleanup_on_netkit_del, 1189 ] 1190 1191 ksft_run(cases=cases, globs=globals(), 1192 case_pfx={"dev_", "data_", "assoc_", "removal_"}, 1193 args=(cfg, )) 1194 1195 cfg.comm_sock.send(b"exit\0") 1196 cfg.comm_sock.close() 1197 finally: 1198 if srv and (srv.stdout or srv.stderr): 1199 ksft_pr("") 1200 ksft_pr(f"Responder logs ({srv.ret}):") 1201 if srv and srv.stdout: 1202 ksft_pr("STDOUT:\n# " + srv.stdout.strip().replace("\n", "\n# ")) 1203 if srv and srv.stderr: 1204 ksft_pr("STDERR:\n# " + srv.stderr.strip().replace("\n", "\n# ")) 1205 ksft_exit() 1206 1207 1208if __name__ == "__main__": 1209 main() 1210