1#!/usr/local/bin/python3 2import copy 3import ipaddress 4import os 5import re 6import socket 7import sys 8import time 9from multiprocessing import connection 10from multiprocessing import Pipe 11from multiprocessing import Process 12from typing import Dict 13from typing import List 14from typing import NamedTuple 15 16from atf_python.sys.net.tools import ToolsHelper 17from atf_python.utils import BaseTest 18from atf_python.utils import libc 19 20 21def run_cmd(cmd: str, verbose=True) -> str: 22 if verbose: 23 print("run: '{}'".format(cmd)) 24 return os.popen(cmd).read() 25 26 27def get_topology_id(test_id: str) -> str: 28 """ 29 Gets a unique topology id based on the pytest test_id. 30 "test_ip6_output.py::TestIP6Output::test_output6_pktinfo[ipandif]" -> 31 "TestIP6Output:test_output6_pktinfo[ipandif]" 32 """ 33 return ":".join(test_id.split("::")[-2:]) 34 35 36def convert_test_name(test_name: str) -> str: 37 """Convert test name to a string that can be used in the file/jail names""" 38 ret = "" 39 for char in test_name: 40 if char.isalnum() or char in ("_", "-", ":"): 41 ret += char 42 elif char in ("["): 43 ret += "_" 44 return ret 45 46 47class VnetInterface(object): 48 # defines from net/if_types.h 49 IFT_LOOP = 0x18 50 IFT_ETHER = 0x06 51 52 def __init__(self, iface_alias: str, iface_name: str): 53 self.name = iface_name 54 self.alias = iface_alias 55 self.vnet_name = "" 56 self.jailed = False 57 self.addr_map: Dict[str, Dict] = {"inet6": {}, "inet": {}} 58 self.prefixes4: List[List[str]] = [] 59 self.prefixes6: List[List[str]] = [] 60 self.fib: int 61 if iface_name.startswith("lo"): 62 self.iftype = self.IFT_LOOP 63 else: 64 self.iftype = self.IFT_ETHER 65 self.ether = ToolsHelper.get_output("/sbin/ifconfig %s ether | awk '/ether/ { print $2; }'" % iface_name).rstrip() 66 67 @property 68 def ifindex(self): 69 return socket.if_nametoindex(self.name) 70 71 @property 72 def first_ipv6(self): 73 d = self.addr_map["inet6"] 74 return d[next(iter(d))] 75 76 @property 77 def first_ipv4(self): 78 d = self.addr_map["inet"] 79 return d[next(iter(d))] 80 81 def set_vnet(self, vnet_name: str): 82 self.vnet_name = vnet_name 83 84 def set_jailed(self, jailed: bool): 85 self.jailed = jailed 86 87 def run_cmd(self, cmd, verbose=False): 88 if self.vnet_name and not self.jailed: 89 cmd = "/usr/sbin/jexec {} {}".format(self.vnet_name, cmd) 90 return run_cmd(cmd, verbose) 91 92 @classmethod 93 def setup_loopback(cls, vnet_name: str): 94 lo = VnetInterface("", "lo0") 95 lo.set_vnet(vnet_name) 96 lo.setup_addr("127.0.0.1/8") 97 lo.turn_up() 98 99 @classmethod 100 def create_iface(cls, alias_name: str, iface_name: str) -> List["VnetInterface"]: 101 name = run_cmd("/sbin/ifconfig {} create".format(iface_name)).rstrip() 102 if not name: 103 raise Exception("Unable to create iface {}".format(iface_name)) 104 if1 = cls(alias_name, name) 105 ret = [if1] 106 if name.startswith("epair"): 107 run_cmd("/sbin/ifconfig {} -txcsum -txcsum6".format(name)) 108 if2 = cls(alias_name, name[:-1] + "b") 109 if1.epairb = if2 110 ret.append(if2) 111 return ret 112 113 def set_mtu(self, mtu): 114 run_cmd("/sbin/ifconfig {} mtu {}".format(self.name, mtu)) 115 116 def setup_addr(self, _addr: str): 117 addr = ipaddress.ip_interface(_addr) 118 if addr.version == 6: 119 family = "inet6" 120 cmd = "/sbin/ifconfig {} {} {}".format(self.name, family, addr) 121 else: 122 family = "inet" 123 if self.addr_map[family]: 124 cmd = "/sbin/ifconfig {} alias {}".format(self.name, addr) 125 else: 126 cmd = "/sbin/ifconfig {} {} {}".format(self.name, family, addr) 127 self.run_cmd(cmd) 128 self.addr_map[family][str(addr.ip)] = addr 129 130 def delete_addr(self, _addr: str): 131 addr = ipaddress.ip_address(_addr) 132 if addr.version == 6: 133 family = "inet6" 134 cmd = "/sbin/ifconfig {} inet6 {} delete".format(self.name, addr) 135 else: 136 family = "inet" 137 cmd = "/sbin/ifconfig {} -alias {}".format(self.name, addr) 138 self.run_cmd(cmd) 139 del self.addr_map[family][str(addr)] 140 141 def turn_up(self): 142 cmd = "/sbin/ifconfig {} up".format(self.name) 143 self.run_cmd(cmd) 144 145 def setfib(self, fib: int): 146 cmd = "/sbin/ifconfig {} fib {}".format(self.name, fib) 147 self.run_cmd(cmd) 148 149 def enable_ipv6(self): 150 cmd = "/usr/sbin/ndp -i {} -- -disabled".format(self.name) 151 self.run_cmd(cmd) 152 153 def has_tentative(self) -> bool: 154 """True if an interface has some addresses in tenative state""" 155 cmd = "/sbin/ifconfig {} inet6".format(self.name) 156 out = self.run_cmd(cmd, verbose=False) 157 for line in out.splitlines(): 158 if "tentative" in line: 159 return True 160 return False 161 162 163class IfaceFactory(object): 164 INTERFACES_FNAME = "created_ifaces.lst" 165 AUTODELETE_TYPES = ("epair", "gif", "gre", "lo", "tap", "tun") 166 167 def __init__(self): 168 self.file_name = self.INTERFACES_FNAME 169 170 def _register_iface(self, iface_name: str): 171 with open(self.file_name, "a") as f: 172 f.write(iface_name + "\n") 173 174 def _list_ifaces(self) -> List[str]: 175 ret: List[str] = [] 176 try: 177 with open(self.file_name, "r") as f: 178 for line in f: 179 ret.append(line.strip()) 180 except OSError: 181 pass 182 return ret 183 184 def create_iface(self, alias_name: str, iface_name: str) -> List[VnetInterface]: 185 ifaces = VnetInterface.create_iface(alias_name, iface_name) 186 for iface in ifaces: 187 if not self.is_autodeleted(iface.name): 188 self._register_iface(iface.name) 189 return ifaces 190 191 @staticmethod 192 def is_autodeleted(iface_name: str) -> bool: 193 if iface_name == "lo0": 194 return False 195 iface_type = re.split(r"\d+", iface_name)[0] 196 return iface_type in IfaceFactory.AUTODELETE_TYPES 197 198 def cleanup_vnet_interfaces(self, vnet_name: str) -> List[str]: 199 """Destroys""" 200 ifaces_lst = ToolsHelper.get_output( 201 "/usr/sbin/jexec {} /sbin/ifconfig -l".format(vnet_name) 202 ) 203 for iface_name in ifaces_lst.split(): 204 if not self.is_autodeleted(iface_name): 205 if iface_name not in self._list_ifaces(): 206 print("Skipping interface {}:{}".format(vnet_name, iface_name)) 207 continue 208 run_cmd( 209 "/usr/sbin/jexec {} /sbin/ifconfig {} destroy".format(vnet_name, iface_name) 210 ) 211 212 def cleanup(self): 213 try: 214 os.unlink(self.INTERFACES_FNAME) 215 except OSError: 216 pass 217 218 219class VnetInstance(object): 220 def __init__( 221 self, vnet_alias: str, vnet_name: str, jid: int, ifaces: List[VnetInterface] 222 ): 223 self.name = vnet_name 224 self.alias = vnet_alias # reference in the test topology 225 self.jid = jid 226 self.ifaces = ifaces 227 self.iface_alias_map = {} # iface.alias: iface 228 self.iface_map = {} # iface.name: iface 229 for iface in ifaces: 230 iface.set_vnet(vnet_name) 231 iface.set_jailed(True) 232 self.iface_alias_map[iface.alias] = iface 233 self.iface_map[iface.name] = iface 234 # Allow reference to interfce aliases as attributes 235 setattr(self, iface.alias, iface) 236 self.need_dad = False # Disable duplicate address detection by default 237 self.attached = False 238 self.pipe = None 239 self.subprocess = None 240 241 def run_vnet_cmd(self, cmd, verbose=True): 242 if not self.attached: 243 cmd = "/usr/sbin/jexec {} {}".format(self.name, cmd) 244 return run_cmd(cmd, verbose) 245 246 def disable_dad(self): 247 self.run_vnet_cmd("/sbin/sysctl net.inet6.ip6.dad_count=0") 248 249 def set_pipe(self, pipe): 250 self.pipe = pipe 251 252 def set_subprocess(self, p): 253 self.subprocess = p 254 255 @staticmethod 256 def attach_jid(jid: int): 257 error_code = libc.jail_attach(jid) 258 if error_code != 0: 259 raise Exception("jail_attach() failed: errno {}".format(error_code)) 260 261 def attach(self): 262 self.attach_jid(self.jid) 263 self.attached = True 264 265 266class VnetFactory(object): 267 JAILS_FNAME = "created_jails.lst" 268 269 def __init__(self, topology_id: str): 270 self.topology_id = topology_id 271 self.file_name = self.JAILS_FNAME 272 self._vnets: List[str] = [] 273 274 def _register_vnet(self, vnet_name: str): 275 self._vnets.append(vnet_name) 276 with open(self.file_name, "a") as f: 277 f.write(vnet_name + "\n") 278 279 @staticmethod 280 def _wait_interfaces(vnet_name: str, ifaces: List[str]) -> List[str]: 281 cmd = "/usr/sbin/jexec {} /sbin/ifconfig -l".format(vnet_name) 282 not_matched: List[str] = [] 283 for i in range(50): 284 vnet_ifaces = run_cmd(cmd).strip().split(" ") 285 not_matched = [] 286 for iface_name in ifaces: 287 if iface_name not in vnet_ifaces: 288 not_matched.append(iface_name) 289 if len(not_matched) == 0: 290 return [] 291 time.sleep(0.1) 292 return not_matched 293 294 def create_vnet(self, vnet_alias: str, ifaces: List[VnetInterface], opts: List[str]): 295 vnet_name = "pytest:{}".format(convert_test_name(self.topology_id)) 296 if self._vnets: 297 # add number to distinguish jails 298 vnet_name = "{}_{}".format(vnet_name, len(self._vnets) + 1) 299 iface_cmds = " ".join(["vnet.interface={}".format(i.name) for i in ifaces]) 300 opt_cmds = " ".join(["{}".format(i) for i in opts]) 301 cmd = "/usr/sbin/jail -i -c name={} persist vnet {} {}".format( 302 vnet_name, iface_cmds, opt_cmds 303 ) 304 jid = 0 305 try: 306 jid_str = run_cmd(cmd) 307 jid = int(jid_str) 308 except ValueError: 309 print("Jail creation failed, output: {}".format(jid_str)) 310 raise 311 self._register_vnet(vnet_name) 312 313 # Run expedited version of routing 314 VnetInterface.setup_loopback(vnet_name) 315 316 not_found = self._wait_interfaces(vnet_name, [i.name for i in ifaces]) 317 if not_found: 318 raise Exception( 319 "Interfaces {} has not appeared in vnet {}".format(not_found, vnet_name) 320 ) 321 return VnetInstance(vnet_alias, vnet_name, jid, ifaces) 322 323 def cleanup(self): 324 iface_factory = IfaceFactory() 325 try: 326 with open(self.file_name) as f: 327 for line in f: 328 vnet_name = line.strip() 329 iface_factory.cleanup_vnet_interfaces(vnet_name) 330 run_cmd("/usr/sbin/jail -r {}".format(vnet_name)) 331 os.unlink(self.JAILS_FNAME) 332 except OSError: 333 pass 334 335 336class SingleInterfaceMap(NamedTuple): 337 ifaces: List[VnetInterface] 338 vnet_aliases: List[str] 339 340 341class ObjectsMap(NamedTuple): 342 iface_map: Dict[str, SingleInterfaceMap] # keyed by ifX 343 vnet_map: Dict[str, VnetInstance] # keyed by vnetX 344 topo_map: Dict # self.TOPOLOGY 345 346 347class VnetTestTemplate(BaseTest): 348 NEED_ROOT: bool = True 349 TOPOLOGY = {} 350 351 def _require_default_modules(self): 352 libc.kldload("if_epair.ko") 353 self.require_module("if_epair") 354 355 def _get_vnet_handler(self, vnet_alias: str): 356 handler_name = "{}_handler".format(vnet_alias) 357 return getattr(self, handler_name, None) 358 359 def _setup_vnet(self, vnet: VnetInstance, obj_map: Dict, pipe): 360 """Base Handler to setup given VNET. 361 Can be run in a subprocess. If so, passes control to the special 362 vnetX_handler() after setting up interface addresses 363 """ 364 vnet.attach() 365 os.chdir(os.getenv("HOME")) 366 print("# setup_vnet({})".format(vnet.name)) 367 if pipe is not None: 368 vnet.set_pipe(pipe) 369 370 topo = obj_map.topo_map 371 ipv6_ifaces = [] 372 # Disable DAD 373 if not vnet.need_dad: 374 vnet.disable_dad() 375 for iface in vnet.ifaces: 376 # check index of vnet within an interface 377 # as we have prefixes for both ends of the interface 378 iface_map = obj_map.iface_map[iface.alias] 379 idx = iface_map.vnet_aliases.index(vnet.alias) 380 prefixes6 = topo[iface.alias].get("prefixes6", []) 381 prefixes4 = topo[iface.alias].get("prefixes4", []) 382 mtu = topo[iface.alias].get("mtu", 0) 383 if "fib" in topo[iface.alias]: 384 fib = topo[iface.alias]["fib"] 385 iface.setfib(fib[idx]) 386 if prefixes6 or prefixes4: 387 ipv6_ifaces.append(iface) 388 iface.turn_up() 389 if prefixes6: 390 iface.enable_ipv6() 391 for prefix in prefixes6 + prefixes4: 392 if prefix[idx]: 393 iface.setup_addr(prefix[idx]) 394 if mtu != 0: 395 iface.set_mtu(mtu) 396 for iface in ipv6_ifaces: 397 while iface.has_tentative(): 398 time.sleep(0.1) 399 400 # Run actual handler 401 handler = self._get_vnet_handler(vnet.alias) 402 if handler: 403 # Do unbuffered stdout for children 404 # so the logs are present if the child hangs 405 sys.stdout.reconfigure(line_buffering=True) 406 self.drop_privileges() 407 handler(vnet) 408 409 def _get_topo_ifmap(self, topo: Dict): 410 iface_factory = IfaceFactory() 411 iface_map: Dict[str, SingleInterfaceMap] = {} 412 iface_aliases = set() 413 for obj_name, obj_data in topo.items(): 414 if obj_name.startswith("vnet"): 415 for iface_alias in obj_data["ifaces"]: 416 iface_aliases.add(iface_alias) 417 for iface_alias in iface_aliases: 418 print("Creating {}".format(iface_alias)) 419 iface_data = topo[iface_alias] 420 iface_type = iface_data.get("type", "epair") 421 ifaces = iface_factory.create_iface(iface_alias, iface_type) 422 smap = SingleInterfaceMap(ifaces, []) 423 iface_map[iface_alias] = smap 424 return iface_map 425 426 def setup_topology(self, topo: Dict, topology_id: str): 427 """Creates jails & interfaces for the provided topology""" 428 vnet_map = {} 429 vnet_factory = VnetFactory(topology_id) 430 iface_map = self._get_topo_ifmap(topo) 431 for obj_name, obj_data in topo.items(): 432 if obj_name.startswith("vnet"): 433 vnet_ifaces = [] 434 maxfib = 0 435 for iface_alias in obj_data["ifaces"]: 436 # epair creates 2 interfaces, grab first _available_ 437 # and map it to the VNET being created 438 idx = len(iface_map[iface_alias].vnet_aliases) 439 iface_map[iface_alias].vnet_aliases.append(obj_name) 440 vnet_ifaces.append(iface_map[iface_alias].ifaces[idx]) 441 fib = topo[iface_alias].get("fib", (0, 0)) 442 maxfib = max(maxfib, fib[idx]) 443 opts = [] 444 if "opts" in obj_data: 445 opts = obj_data["opts"] 446 vnet = vnet_factory.create_vnet(obj_name, vnet_ifaces, opts) 447 if maxfib != 0: 448 # Make sure the VNET has enough FIBs. 449 vnet.run_vnet_cmd("/sbin/sysctl net.fibs={}".format(maxfib + 1)) 450 vnet_map[obj_name] = vnet 451 # Allow reference to VNETs as attributes 452 setattr(self, obj_name, vnet) 453 # Debug output 454 print("============= TEST TOPOLOGY =============") 455 for vnet_alias, vnet in vnet_map.items(): 456 print("# vnet {} -> {}".format(vnet.alias, vnet.name), end="") 457 handler = self._get_vnet_handler(vnet.alias) 458 if handler: 459 print(" handler: {}".format(handler.__name__), end="") 460 print() 461 for iface_alias, iface_data in iface_map.items(): 462 vnets = iface_data.vnet_aliases 463 ifaces: List[VnetInterface] = iface_data.ifaces 464 if len(vnets) == 1 and len(ifaces) == 2: 465 print( 466 "# iface {}: {}::{} -> main::{}".format( 467 iface_alias, vnets[0], ifaces[0].name, ifaces[1].name 468 ) 469 ) 470 elif len(vnets) == 2 and len(ifaces) == 2: 471 print( 472 "# iface {}: {}::{} -> {}::{}".format( 473 iface_alias, vnets[0], ifaces[0].name, vnets[1], ifaces[1].name 474 ) 475 ) 476 else: 477 print( 478 "# iface {}: ifaces: {} vnets: {}".format( 479 iface_alias, vnets, [i.name for i in ifaces] 480 ) 481 ) 482 print() 483 return ObjectsMap(iface_map, vnet_map, topo) 484 485 def setup_method(self, _method): 486 """Sets up all the required topology and handlers for the given test""" 487 super().setup_method(_method) 488 self._require_default_modules() 489 490 # TestIP6Output.test_output6_pktinfo[ipandif] 491 topology_id = get_topology_id(self.test_id) 492 topology = self.TOPOLOGY 493 # First, setup kernel objects - interfaces & vnets 494 obj_map = self.setup_topology(topology, topology_id) 495 main_vnet = None # one without subprocess handler 496 for vnet_alias, vnet in obj_map.vnet_map.items(): 497 if self._get_vnet_handler(vnet_alias): 498 # Need subprocess to run 499 parent_pipe, child_pipe = Pipe() 500 p = Process( 501 target=self._setup_vnet, 502 args=( 503 vnet, 504 obj_map, 505 child_pipe, 506 ), 507 ) 508 vnet.set_pipe(parent_pipe) 509 vnet.set_subprocess(p) 510 p.start() 511 else: 512 if main_vnet is not None: 513 raise Exception("there can be only 1 VNET w/o handler") 514 main_vnet = vnet 515 # Main vnet needs to be the last, so all the other subprocesses 516 # are started & their pipe handles collected 517 self.vnet = main_vnet 518 self._setup_vnet(main_vnet, obj_map, None) 519 # Save state for the main handler 520 self.iface_map = obj_map.iface_map 521 self.vnet_map = obj_map.vnet_map 522 self.drop_privileges() 523 524 def cleanup(self, test_id: str): 525 # pytest test id: file::class::test_name 526 topology_id = get_topology_id(self.test_id) 527 528 print("============= vnet cleanup =============") 529 print("# topology_id: '{}'".format(topology_id)) 530 VnetFactory(topology_id).cleanup() 531 IfaceFactory().cleanup() 532 533 def wait_object(self, pipe, timeout=5): 534 if pipe.poll(timeout): 535 return pipe.recv() 536 raise TimeoutError 537 538 def wait_objects_any(self, pipe_list, timeout=5): 539 objects = connection.wait(pipe_list, timeout) 540 if objects: 541 return objects[0].recv() 542 raise TimeoutError 543 544 def send_object(self, pipe, obj): 545 pipe.send(obj) 546 547 def wait(self): 548 while True: 549 time.sleep(1) 550 551 @property 552 def curvnet(self): 553 pass 554 555 556class SingleVnetTestTemplate(VnetTestTemplate): 557 IPV6_PREFIXES: List[str] = [] 558 IPV4_PREFIXES: List[str] = [] 559 IFTYPE = "epair" 560 561 def _setup_default_topology(self): 562 topology = copy.deepcopy( 563 { 564 "vnet1": {"ifaces": ["if1"]}, 565 "if1": {"type": self.IFTYPE, "prefixes4": [], "prefixes6": []}, 566 } 567 ) 568 for prefix in self.IPV6_PREFIXES: 569 topology["if1"]["prefixes6"].append((prefix,)) 570 for prefix in self.IPV4_PREFIXES: 571 topology["if1"]["prefixes4"].append((prefix,)) 572 return topology 573 574 def setup_method(self, method): 575 if not getattr(self, "TOPOLOGY", None): 576 self.TOPOLOGY = self._setup_default_topology() 577 else: 578 names = self.TOPOLOGY.keys() 579 assert len([n for n in names if n.startswith("vnet")]) == 1 580 super().setup_method(method) 581