1#!/usr/local/bin/python3 2import copy 3import ipaddress 4import os 5import re 6import socket 7import sys 8import time 9from multiprocessing import connection 10from multiprocessing import Pipe 11from multiprocessing import Process 12from typing import Dict 13from typing import List 14from typing import NamedTuple 15 16from atf_python.sys.net.tools import ToolsHelper 17from atf_python.utils import BaseTest 18from atf_python.utils import libc 19 20 21def run_cmd(cmd: str, verbose=True) -> str: 22 if verbose: 23 print("run: '{}'".format(cmd)) 24 return os.popen(cmd).read() 25 26 27def get_topology_id(test_id: str) -> str: 28 """ 29 Gets a unique topology id based on the pytest test_id. 30 "test_ip6_output.py::TestIP6Output::test_output6_pktinfo[ipandif]" -> 31 "TestIP6Output:test_output6_pktinfo[ipandif]" 32 """ 33 return ":".join(test_id.split("::")[-2:]) 34 35 36def convert_test_name(test_name: str) -> str: 37 """Convert test name to a string that can be used in the file/jail names""" 38 ret = "" 39 for char in test_name: 40 if char.isalnum() or char in ("_", "-", ":"): 41 ret += char 42 elif char in ("["): 43 ret += "_" 44 return ret 45 46 47class VnetInterface(object): 48 # defines from net/if_types.h 49 IFT_LOOP = 0x18 50 IFT_ETHER = 0x06 51 52 def __init__(self, iface_alias: str, iface_name: str): 53 self.name = iface_name 54 self.alias = iface_alias 55 self.vnet_name = "" 56 self.jailed = False 57 self.addr_map: Dict[str, Dict] = {"inet6": {}, "inet": {}} 58 self.prefixes4: List[List[str]] = [] 59 self.prefixes6: List[List[str]] = [] 60 if iface_name.startswith("lo"): 61 self.iftype = self.IFT_LOOP 62 else: 63 self.iftype = self.IFT_ETHER 64 self.ether = ToolsHelper.get_output("/sbin/ifconfig %s ether | awk '/ether/ { print $2; }'" % iface_name).rstrip() 65 66 @property 67 def ifindex(self): 68 return socket.if_nametoindex(self.name) 69 70 @property 71 def first_ipv6(self): 72 d = self.addr_map["inet6"] 73 return d[next(iter(d))] 74 75 @property 76 def first_ipv4(self): 77 d = self.addr_map["inet"] 78 return d[next(iter(d))] 79 80 def set_vnet(self, vnet_name: str): 81 self.vnet_name = vnet_name 82 83 def set_jailed(self, jailed: bool): 84 self.jailed = jailed 85 86 def run_cmd(self, cmd, verbose=False): 87 if self.vnet_name and not self.jailed: 88 cmd = "/usr/sbin/jexec {} {}".format(self.vnet_name, cmd) 89 return run_cmd(cmd, verbose) 90 91 @classmethod 92 def setup_loopback(cls, vnet_name: str): 93 lo = VnetInterface("", "lo0") 94 lo.set_vnet(vnet_name) 95 lo.setup_addr("127.0.0.1/8") 96 lo.turn_up() 97 98 @classmethod 99 def create_iface(cls, alias_name: str, iface_name: str) -> List["VnetInterface"]: 100 name = run_cmd("/sbin/ifconfig {} create".format(iface_name)).rstrip() 101 if not name: 102 raise Exception("Unable to create iface {}".format(iface_name)) 103 if1 = cls(alias_name, name) 104 ret = [if1] 105 if name.startswith("epair"): 106 run_cmd("/sbin/ifconfig {} -txcsum -txcsum6".format(name)) 107 if2 = cls(alias_name, name[:-1] + "b") 108 if1.epairb = if2 109 ret.append(if2); 110 return ret 111 112 def set_mtu(self, mtu): 113 run_cmd("/sbin/ifconfig {} mtu {}".format(self.name, mtu)) 114 115 def setup_addr(self, _addr: str): 116 addr = ipaddress.ip_interface(_addr) 117 if addr.version == 6: 118 family = "inet6" 119 cmd = "/sbin/ifconfig {} {} {}".format(self.name, family, addr) 120 else: 121 family = "inet" 122 if self.addr_map[family]: 123 cmd = "/sbin/ifconfig {} alias {}".format(self.name, addr) 124 else: 125 cmd = "/sbin/ifconfig {} {} {}".format(self.name, family, addr) 126 self.run_cmd(cmd) 127 self.addr_map[family][str(addr.ip)] = addr 128 129 def delete_addr(self, _addr: str): 130 addr = ipaddress.ip_address(_addr) 131 if addr.version == 6: 132 family = "inet6" 133 cmd = "/sbin/ifconfig {} inet6 {} delete".format(self.name, addr) 134 else: 135 family = "inet" 136 cmd = "/sbin/ifconfig {} -alias {}".format(self.name, addr) 137 self.run_cmd(cmd) 138 del self.addr_map[family][str(addr)] 139 140 def turn_up(self): 141 cmd = "/sbin/ifconfig {} up".format(self.name) 142 self.run_cmd(cmd) 143 144 def enable_ipv6(self): 145 cmd = "/usr/sbin/ndp -i {} -- -disabled".format(self.name) 146 self.run_cmd(cmd) 147 148 def has_tentative(self) -> bool: 149 """True if an interface has some addresses in tenative state""" 150 cmd = "/sbin/ifconfig {} inet6".format(self.name) 151 out = self.run_cmd(cmd, verbose=False) 152 for line in out.splitlines(): 153 if "tentative" in line: 154 return True 155 return False 156 157 158class IfaceFactory(object): 159 INTERFACES_FNAME = "created_ifaces.lst" 160 AUTODELETE_TYPES = ("epair", "gif", "gre", "lo", "tap", "tun") 161 162 def __init__(self): 163 self.file_name = self.INTERFACES_FNAME 164 165 def _register_iface(self, iface_name: str): 166 with open(self.file_name, "a") as f: 167 f.write(iface_name + "\n") 168 169 def _list_ifaces(self) -> List[str]: 170 ret: List[str] = [] 171 try: 172 with open(self.file_name, "r") as f: 173 for line in f: 174 ret.append(line.strip()) 175 except OSError: 176 pass 177 return ret 178 179 def create_iface(self, alias_name: str, iface_name: str) -> List[VnetInterface]: 180 ifaces = VnetInterface.create_iface(alias_name, iface_name) 181 for iface in ifaces: 182 if not self.is_autodeleted(iface.name): 183 self._register_iface(iface.name) 184 return ifaces 185 186 @staticmethod 187 def is_autodeleted(iface_name: str) -> bool: 188 if iface_name == "lo0": 189 return False 190 iface_type = re.split(r"\d+", iface_name)[0] 191 return iface_type in IfaceFactory.AUTODELETE_TYPES 192 193 def cleanup_vnet_interfaces(self, vnet_name: str) -> List[str]: 194 """Destroys""" 195 ifaces_lst = ToolsHelper.get_output( 196 "/usr/sbin/jexec {} /sbin/ifconfig -l".format(vnet_name) 197 ) 198 for iface_name in ifaces_lst.split(): 199 if not self.is_autodeleted(iface_name): 200 if iface_name not in self._list_ifaces(): 201 print("Skipping interface {}:{}".format(vnet_name, iface_name)) 202 continue 203 run_cmd( 204 "/usr/sbin/jexec {} /sbin/ifconfig {} destroy".format(vnet_name, iface_name) 205 ) 206 207 def cleanup(self): 208 try: 209 os.unlink(self.INTERFACES_FNAME) 210 except OSError: 211 pass 212 213 214class VnetInstance(object): 215 def __init__( 216 self, vnet_alias: str, vnet_name: str, jid: int, ifaces: List[VnetInterface] 217 ): 218 self.name = vnet_name 219 self.alias = vnet_alias # reference in the test topology 220 self.jid = jid 221 self.ifaces = ifaces 222 self.iface_alias_map = {} # iface.alias: iface 223 self.iface_map = {} # iface.name: iface 224 for iface in ifaces: 225 iface.set_vnet(vnet_name) 226 iface.set_jailed(True) 227 self.iface_alias_map[iface.alias] = iface 228 self.iface_map[iface.name] = iface 229 # Allow reference to interfce aliases as attributes 230 setattr(self, iface.alias, iface) 231 self.need_dad = False # Disable duplicate address detection by default 232 self.attached = False 233 self.pipe = None 234 self.subprocess = None 235 236 def run_vnet_cmd(self, cmd, verbose=True): 237 if not self.attached: 238 cmd = "/usr/sbin/jexec {} {}".format(self.name, cmd) 239 return run_cmd(cmd, verbose) 240 241 def disable_dad(self): 242 self.run_vnet_cmd("/sbin/sysctl net.inet6.ip6.dad_count=0") 243 244 def set_pipe(self, pipe): 245 self.pipe = pipe 246 247 def set_subprocess(self, p): 248 self.subprocess = p 249 250 @staticmethod 251 def attach_jid(jid: int): 252 error_code = libc.jail_attach(jid) 253 if error_code != 0: 254 raise Exception("jail_attach() failed: errno {}".format(error_code)) 255 256 def attach(self): 257 self.attach_jid(self.jid) 258 self.attached = True 259 260 261class VnetFactory(object): 262 JAILS_FNAME = "created_jails.lst" 263 264 def __init__(self, topology_id: str): 265 self.topology_id = topology_id 266 self.file_name = self.JAILS_FNAME 267 self._vnets: List[str] = [] 268 269 def _register_vnet(self, vnet_name: str): 270 self._vnets.append(vnet_name) 271 with open(self.file_name, "a") as f: 272 f.write(vnet_name + "\n") 273 274 @staticmethod 275 def _wait_interfaces(vnet_name: str, ifaces: List[str]) -> List[str]: 276 cmd = "/usr/sbin/jexec {} /sbin/ifconfig -l".format(vnet_name) 277 not_matched: List[str] = [] 278 for i in range(50): 279 vnet_ifaces = run_cmd(cmd).strip().split(" ") 280 not_matched = [] 281 for iface_name in ifaces: 282 if iface_name not in vnet_ifaces: 283 not_matched.append(iface_name) 284 if len(not_matched) == 0: 285 return [] 286 time.sleep(0.1) 287 return not_matched 288 289 def create_vnet(self, vnet_alias: str, ifaces: List[VnetInterface], opts: List[str]): 290 vnet_name = "pytest:{}".format(convert_test_name(self.topology_id)) 291 if self._vnets: 292 # add number to distinguish jails 293 vnet_name = "{}_{}".format(vnet_name, len(self._vnets) + 1) 294 iface_cmds = " ".join(["vnet.interface={}".format(i.name) for i in ifaces]) 295 opt_cmds = " ".join(["{}".format(i) for i in opts]) 296 cmd = "/usr/sbin/jail -i -c name={} persist vnet {} {}".format( 297 vnet_name, iface_cmds, opt_cmds 298 ) 299 jid = 0 300 try: 301 jid_str = run_cmd(cmd) 302 jid = int(jid_str) 303 except ValueError: 304 print("Jail creation failed, output: {}".format(jid_str)) 305 raise 306 self._register_vnet(vnet_name) 307 308 # Run expedited version of routing 309 VnetInterface.setup_loopback(vnet_name) 310 311 not_found = self._wait_interfaces(vnet_name, [i.name for i in ifaces]) 312 if not_found: 313 raise Exception( 314 "Interfaces {} has not appeared in vnet {}".format(not_found, vnet_name) 315 ) 316 return VnetInstance(vnet_alias, vnet_name, jid, ifaces) 317 318 def cleanup(self): 319 iface_factory = IfaceFactory() 320 try: 321 with open(self.file_name) as f: 322 for line in f: 323 vnet_name = line.strip() 324 iface_factory.cleanup_vnet_interfaces(vnet_name) 325 run_cmd("/usr/sbin/jail -r {}".format(vnet_name)) 326 os.unlink(self.JAILS_FNAME) 327 except OSError: 328 pass 329 330 331class SingleInterfaceMap(NamedTuple): 332 ifaces: List[VnetInterface] 333 vnet_aliases: List[str] 334 335 336class ObjectsMap(NamedTuple): 337 iface_map: Dict[str, SingleInterfaceMap] # keyed by ifX 338 vnet_map: Dict[str, VnetInstance] # keyed by vnetX 339 topo_map: Dict # self.TOPOLOGY 340 341 342class VnetTestTemplate(BaseTest): 343 NEED_ROOT: bool = True 344 TOPOLOGY = {} 345 346 def _require_default_modules(self): 347 libc.kldload("if_epair.ko") 348 self.require_module("if_epair") 349 350 def _get_vnet_handler(self, vnet_alias: str): 351 handler_name = "{}_handler".format(vnet_alias) 352 return getattr(self, handler_name, None) 353 354 def _setup_vnet(self, vnet: VnetInstance, obj_map: Dict, pipe): 355 """Base Handler to setup given VNET. 356 Can be run in a subprocess. If so, passes control to the special 357 vnetX_handler() after setting up interface addresses 358 """ 359 vnet.attach() 360 os.chdir(os.getenv("HOME")) 361 print("# setup_vnet({})".format(vnet.name)) 362 if pipe is not None: 363 vnet.set_pipe(pipe) 364 365 topo = obj_map.topo_map 366 ipv6_ifaces = [] 367 # Disable DAD 368 if not vnet.need_dad: 369 vnet.disable_dad() 370 for iface in vnet.ifaces: 371 # check index of vnet within an interface 372 # as we have prefixes for both ends of the interface 373 iface_map = obj_map.iface_map[iface.alias] 374 idx = iface_map.vnet_aliases.index(vnet.alias) 375 prefixes6 = topo[iface.alias].get("prefixes6", []) 376 prefixes4 = topo[iface.alias].get("prefixes4", []) 377 mtu = topo[iface.alias].get("mtu", 0) 378 if prefixes6 or prefixes4: 379 ipv6_ifaces.append(iface) 380 iface.turn_up() 381 if prefixes6: 382 iface.enable_ipv6() 383 for prefix in prefixes6 + prefixes4: 384 if prefix[idx]: 385 iface.setup_addr(prefix[idx]) 386 if mtu != 0: 387 iface.set_mtu(mtu) 388 for iface in ipv6_ifaces: 389 while iface.has_tentative(): 390 time.sleep(0.1) 391 392 # Run actual handler 393 handler = self._get_vnet_handler(vnet.alias) 394 if handler: 395 # Do unbuffered stdout for children 396 # so the logs are present if the child hangs 397 sys.stdout.reconfigure(line_buffering=True) 398 self.drop_privileges() 399 handler(vnet) 400 401 def _get_topo_ifmap(self, topo: Dict): 402 iface_factory = IfaceFactory() 403 iface_map: Dict[str, SingleInterfaceMap] = {} 404 iface_aliases = set() 405 for obj_name, obj_data in topo.items(): 406 if obj_name.startswith("vnet"): 407 for iface_alias in obj_data["ifaces"]: 408 iface_aliases.add(iface_alias) 409 for iface_alias in iface_aliases: 410 print("Creating {}".format(iface_alias)) 411 iface_data = topo[iface_alias] 412 iface_type = iface_data.get("type", "epair") 413 ifaces = iface_factory.create_iface(iface_alias, iface_type) 414 smap = SingleInterfaceMap(ifaces, []) 415 iface_map[iface_alias] = smap 416 return iface_map 417 418 def setup_topology(self, topo: Dict, topology_id: str): 419 """Creates jails & interfaces for the provided topology""" 420 vnet_map = {} 421 vnet_factory = VnetFactory(topology_id) 422 iface_map = self._get_topo_ifmap(topo) 423 for obj_name, obj_data in topo.items(): 424 if obj_name.startswith("vnet"): 425 vnet_ifaces = [] 426 for iface_alias in obj_data["ifaces"]: 427 # epair creates 2 interfaces, grab first _available_ 428 # and map it to the VNET being created 429 idx = len(iface_map[iface_alias].vnet_aliases) 430 iface_map[iface_alias].vnet_aliases.append(obj_name) 431 vnet_ifaces.append(iface_map[iface_alias].ifaces[idx]) 432 opts = [] 433 if "opts" in obj_data: 434 opts = obj_data["opts"] 435 vnet = vnet_factory.create_vnet(obj_name, vnet_ifaces, opts) 436 vnet_map[obj_name] = vnet 437 # Allow reference to VNETs as attributes 438 setattr(self, obj_name, vnet) 439 # Debug output 440 print("============= TEST TOPOLOGY =============") 441 for vnet_alias, vnet in vnet_map.items(): 442 print("# vnet {} -> {}".format(vnet.alias, vnet.name), end="") 443 handler = self._get_vnet_handler(vnet.alias) 444 if handler: 445 print(" handler: {}".format(handler.__name__), end="") 446 print() 447 for iface_alias, iface_data in iface_map.items(): 448 vnets = iface_data.vnet_aliases 449 ifaces: List[VnetInterface] = iface_data.ifaces 450 if len(vnets) == 1 and len(ifaces) == 2: 451 print( 452 "# iface {}: {}::{} -> main::{}".format( 453 iface_alias, vnets[0], ifaces[0].name, ifaces[1].name 454 ) 455 ) 456 elif len(vnets) == 2 and len(ifaces) == 2: 457 print( 458 "# iface {}: {}::{} -> {}::{}".format( 459 iface_alias, vnets[0], ifaces[0].name, vnets[1], ifaces[1].name 460 ) 461 ) 462 else: 463 print( 464 "# iface {}: ifaces: {} vnets: {}".format( 465 iface_alias, vnets, [i.name for i in ifaces] 466 ) 467 ) 468 print() 469 return ObjectsMap(iface_map, vnet_map, topo) 470 471 def setup_method(self, _method): 472 """Sets up all the required topology and handlers for the given test""" 473 super().setup_method(_method) 474 self._require_default_modules() 475 476 # TestIP6Output.test_output6_pktinfo[ipandif] 477 topology_id = get_topology_id(self.test_id) 478 topology = self.TOPOLOGY 479 # First, setup kernel objects - interfaces & vnets 480 obj_map = self.setup_topology(topology, topology_id) 481 main_vnet = None # one without subprocess handler 482 for vnet_alias, vnet in obj_map.vnet_map.items(): 483 if self._get_vnet_handler(vnet_alias): 484 # Need subprocess to run 485 parent_pipe, child_pipe = Pipe() 486 p = Process( 487 target=self._setup_vnet, 488 args=( 489 vnet, 490 obj_map, 491 child_pipe, 492 ), 493 ) 494 vnet.set_pipe(parent_pipe) 495 vnet.set_subprocess(p) 496 p.start() 497 else: 498 if main_vnet is not None: 499 raise Exception("there can be only 1 VNET w/o handler") 500 main_vnet = vnet 501 # Main vnet needs to be the last, so all the other subprocesses 502 # are started & their pipe handles collected 503 self.vnet = main_vnet 504 self._setup_vnet(main_vnet, obj_map, None) 505 # Save state for the main handler 506 self.iface_map = obj_map.iface_map 507 self.vnet_map = obj_map.vnet_map 508 self.drop_privileges() 509 510 def cleanup(self, test_id: str): 511 # pytest test id: file::class::test_name 512 topology_id = get_topology_id(self.test_id) 513 514 print("============= vnet cleanup =============") 515 print("# topology_id: '{}'".format(topology_id)) 516 VnetFactory(topology_id).cleanup() 517 IfaceFactory().cleanup() 518 519 def wait_object(self, pipe, timeout=5): 520 if pipe.poll(timeout): 521 return pipe.recv() 522 raise TimeoutError 523 524 def wait_objects_any(self, pipe_list, timeout=5): 525 objects = connection.wait(pipe_list, timeout) 526 if objects: 527 return objects[0].recv() 528 raise TimeoutError 529 530 def send_object(self, pipe, obj): 531 pipe.send(obj) 532 533 def wait(self): 534 while True: 535 time.sleep(1) 536 537 @property 538 def curvnet(self): 539 pass 540 541 542class SingleVnetTestTemplate(VnetTestTemplate): 543 IPV6_PREFIXES: List[str] = [] 544 IPV4_PREFIXES: List[str] = [] 545 IFTYPE = "epair" 546 547 def _setup_default_topology(self): 548 topology = copy.deepcopy( 549 { 550 "vnet1": {"ifaces": ["if1"]}, 551 "if1": {"type": self.IFTYPE, "prefixes4": [], "prefixes6": []}, 552 } 553 ) 554 for prefix in self.IPV6_PREFIXES: 555 topology["if1"]["prefixes6"].append((prefix,)) 556 for prefix in self.IPV4_PREFIXES: 557 topology["if1"]["prefixes4"].append((prefix,)) 558 return topology 559 560 def setup_method(self, method): 561 if not getattr(self, "TOPOLOGY", None): 562 self.TOPOLOGY = self._setup_default_topology() 563 else: 564 names = self.TOPOLOGY.keys() 565 assert len([n for n in names if n.startswith("vnet")]) == 1 566 super().setup_method(method) 567