1#!/usr/local/bin/python3 2import copy 3import ipaddress 4import os 5import re 6import socket 7import sys 8import time 9from multiprocessing import connection 10from multiprocessing import Pipe 11from multiprocessing import Process 12from typing import Dict 13from typing import List 14from typing import NamedTuple 15 16from atf_python.sys.net.tools import ToolsHelper 17from atf_python.utils import BaseTest 18from atf_python.utils import libc 19 20 21def run_cmd(cmd: str, verbose=True) -> str: 22 if verbose: 23 print("run: '{}'".format(cmd)) 24 return os.popen(cmd).read() 25 26 27def get_topology_id(test_id: str) -> str: 28 """ 29 Gets a unique topology id based on the pytest test_id. 30 "test_ip6_output.py::TestIP6Output::test_output6_pktinfo[ipandif]" -> 31 "TestIP6Output:test_output6_pktinfo[ipandif]" 32 """ 33 return ":".join(test_id.split("::")[-2:]) 34 35 36def convert_test_name(test_name: str) -> str: 37 """Convert test name to a string that can be used in the file/jail names""" 38 ret = "" 39 for char in test_name: 40 if char.isalnum() or char in ("_", "-", ":"): 41 ret += char 42 elif char in ("["): 43 ret += "_" 44 return ret 45 46 47class VnetInterface(object): 48 # defines from net/if_types.h 49 IFT_LOOP = 0x18 50 IFT_ETHER = 0x06 51 52 def __init__(self, iface_alias: str, iface_name: str): 53 self.name = iface_name 54 self.alias = iface_alias 55 self.vnet_name = "" 56 self.jailed = False 57 self.addr_map: Dict[str, Dict] = {"inet6": {}, "inet": {}} 58 self.prefixes4: List[List[str]] = [] 59 self.prefixes6: List[List[str]] = [] 60 if iface_name.startswith("lo"): 61 self.iftype = self.IFT_LOOP 62 else: 63 self.iftype = self.IFT_ETHER 64 self.ether = ToolsHelper.get_output("/sbin/ifconfig %s ether | awk '/ether/ { print $2; }'" % iface_name).rstrip() 65 66 @property 67 def ifindex(self): 68 return socket.if_nametoindex(self.name) 69 70 @property 71 def first_ipv6(self): 72 d = self.addr_map["inet6"] 73 return d[next(iter(d))] 74 75 @property 76 def first_ipv4(self): 77 d = self.addr_map["inet"] 78 return d[next(iter(d))] 79 80 def set_vnet(self, vnet_name: str): 81 self.vnet_name = vnet_name 82 83 def set_jailed(self, jailed: bool): 84 self.jailed = jailed 85 86 def run_cmd(self, cmd, verbose=False): 87 if self.vnet_name and not self.jailed: 88 cmd = "/usr/sbin/jexec {} {}".format(self.vnet_name, cmd) 89 return run_cmd(cmd, verbose) 90 91 @classmethod 92 def setup_loopback(cls, vnet_name: str): 93 lo = VnetInterface("", "lo0") 94 lo.set_vnet(vnet_name) 95 lo.setup_addr("127.0.0.1/8") 96 lo.turn_up() 97 98 @classmethod 99 def create_iface(cls, alias_name: str, iface_name: str) -> List["VnetInterface"]: 100 name = run_cmd("/sbin/ifconfig {} create".format(iface_name)).rstrip() 101 if not name: 102 raise Exception("Unable to create iface {}".format(iface_name)) 103 if1 = cls(alias_name, name) 104 ret = [if1] 105 if name.startswith("epair"): 106 run_cmd("/sbin/ifconfig {} -txcsum -txcsum6".format(name)) 107 if2 = cls(alias_name, name[:-1] + "b") 108 if1.epairb = if2 109 ret.append(if2); 110 return ret 111 112 def setup_addr(self, _addr: str): 113 addr = ipaddress.ip_interface(_addr) 114 if addr.version == 6: 115 family = "inet6" 116 cmd = "/sbin/ifconfig {} {} {}".format(self.name, family, addr) 117 else: 118 family = "inet" 119 if self.addr_map[family]: 120 cmd = "/sbin/ifconfig {} alias {}".format(self.name, addr) 121 else: 122 cmd = "/sbin/ifconfig {} {} {}".format(self.name, family, addr) 123 self.run_cmd(cmd) 124 self.addr_map[family][str(addr.ip)] = addr 125 126 def delete_addr(self, _addr: str): 127 addr = ipaddress.ip_address(_addr) 128 if addr.version == 6: 129 family = "inet6" 130 cmd = "/sbin/ifconfig {} inet6 {} delete".format(self.name, addr) 131 else: 132 family = "inet" 133 cmd = "/sbin/ifconfig {} -alias {}".format(self.name, addr) 134 self.run_cmd(cmd) 135 del self.addr_map[family][str(addr)] 136 137 def turn_up(self): 138 cmd = "/sbin/ifconfig {} up".format(self.name) 139 self.run_cmd(cmd) 140 141 def enable_ipv6(self): 142 cmd = "/usr/sbin/ndp -i {} -- -disabled".format(self.name) 143 self.run_cmd(cmd) 144 145 def has_tentative(self) -> bool: 146 """True if an interface has some addresses in tenative state""" 147 cmd = "/sbin/ifconfig {} inet6".format(self.name) 148 out = self.run_cmd(cmd, verbose=False) 149 for line in out.splitlines(): 150 if "tentative" in line: 151 return True 152 return False 153 154 155class IfaceFactory(object): 156 INTERFACES_FNAME = "created_ifaces.lst" 157 AUTODELETE_TYPES = ("epair", "gif", "gre", "lo", "tap", "tun") 158 159 def __init__(self): 160 self.file_name = self.INTERFACES_FNAME 161 162 def _register_iface(self, iface_name: str): 163 with open(self.file_name, "a") as f: 164 f.write(iface_name + "\n") 165 166 def _list_ifaces(self) -> List[str]: 167 ret: List[str] = [] 168 try: 169 with open(self.file_name, "r") as f: 170 for line in f: 171 ret.append(line.strip()) 172 except OSError: 173 pass 174 return ret 175 176 def create_iface(self, alias_name: str, iface_name: str) -> List[VnetInterface]: 177 ifaces = VnetInterface.create_iface(alias_name, iface_name) 178 for iface in ifaces: 179 if not self.is_autodeleted(iface.name): 180 self._register_iface(iface.name) 181 return ifaces 182 183 @staticmethod 184 def is_autodeleted(iface_name: str) -> bool: 185 if iface_name == "lo0": 186 return False 187 iface_type = re.split(r"\d+", iface_name)[0] 188 return iface_type in IfaceFactory.AUTODELETE_TYPES 189 190 def cleanup_vnet_interfaces(self, vnet_name: str) -> List[str]: 191 """Destroys""" 192 ifaces_lst = ToolsHelper.get_output( 193 "/usr/sbin/jexec {} /sbin/ifconfig -l".format(vnet_name) 194 ) 195 for iface_name in ifaces_lst.split(): 196 if not self.is_autodeleted(iface_name): 197 if iface_name not in self._list_ifaces(): 198 print("Skipping interface {}:{}".format(vnet_name, iface_name)) 199 continue 200 run_cmd( 201 "/usr/sbin/jexec {} /sbin/ifconfig {} destroy".format(vnet_name, iface_name) 202 ) 203 204 def cleanup(self): 205 try: 206 os.unlink(self.INTERFACES_FNAME) 207 except OSError: 208 pass 209 210 211class VnetInstance(object): 212 def __init__( 213 self, vnet_alias: str, vnet_name: str, jid: int, ifaces: List[VnetInterface] 214 ): 215 self.name = vnet_name 216 self.alias = vnet_alias # reference in the test topology 217 self.jid = jid 218 self.ifaces = ifaces 219 self.iface_alias_map = {} # iface.alias: iface 220 self.iface_map = {} # iface.name: iface 221 for iface in ifaces: 222 iface.set_vnet(vnet_name) 223 iface.set_jailed(True) 224 self.iface_alias_map[iface.alias] = iface 225 self.iface_map[iface.name] = iface 226 # Allow reference to interfce aliases as attributes 227 setattr(self, iface.alias, iface) 228 self.need_dad = False # Disable duplicate address detection by default 229 self.attached = False 230 self.pipe = None 231 self.subprocess = None 232 233 def run_vnet_cmd(self, cmd, verbose=True): 234 if not self.attached: 235 cmd = "/usr/sbin/jexec {} {}".format(self.name, cmd) 236 return run_cmd(cmd, verbose) 237 238 def disable_dad(self): 239 self.run_vnet_cmd("/sbin/sysctl net.inet6.ip6.dad_count=0") 240 241 def set_pipe(self, pipe): 242 self.pipe = pipe 243 244 def set_subprocess(self, p): 245 self.subprocess = p 246 247 @staticmethod 248 def attach_jid(jid: int): 249 error_code = libc.jail_attach(jid) 250 if error_code != 0: 251 raise Exception("jail_attach() failed: errno {}".format(error_code)) 252 253 def attach(self): 254 self.attach_jid(self.jid) 255 self.attached = True 256 257 258class VnetFactory(object): 259 JAILS_FNAME = "created_jails.lst" 260 261 def __init__(self, topology_id: str): 262 self.topology_id = topology_id 263 self.file_name = self.JAILS_FNAME 264 self._vnets: List[str] = [] 265 266 def _register_vnet(self, vnet_name: str): 267 self._vnets.append(vnet_name) 268 with open(self.file_name, "a") as f: 269 f.write(vnet_name + "\n") 270 271 @staticmethod 272 def _wait_interfaces(vnet_name: str, ifaces: List[str]) -> List[str]: 273 cmd = "/usr/sbin/jexec {} /sbin/ifconfig -l".format(vnet_name) 274 not_matched: List[str] = [] 275 for i in range(50): 276 vnet_ifaces = run_cmd(cmd).strip().split(" ") 277 not_matched = [] 278 for iface_name in ifaces: 279 if iface_name not in vnet_ifaces: 280 not_matched.append(iface_name) 281 if len(not_matched) == 0: 282 return [] 283 time.sleep(0.1) 284 return not_matched 285 286 def create_vnet(self, vnet_alias: str, ifaces: List[VnetInterface], opts: List[str]): 287 vnet_name = "pytest:{}".format(convert_test_name(self.topology_id)) 288 if self._vnets: 289 # add number to distinguish jails 290 vnet_name = "{}_{}".format(vnet_name, len(self._vnets) + 1) 291 iface_cmds = " ".join(["vnet.interface={}".format(i.name) for i in ifaces]) 292 opt_cmds = " ".join(["{}".format(i) for i in opts]) 293 cmd = "/usr/sbin/jail -i -c name={} persist vnet {} {}".format( 294 vnet_name, iface_cmds, opt_cmds 295 ) 296 jid = 0 297 try: 298 jid_str = run_cmd(cmd) 299 jid = int(jid_str) 300 except ValueError: 301 print("Jail creation failed, output: {}".format(jid_str)) 302 raise 303 self._register_vnet(vnet_name) 304 305 # Run expedited version of routing 306 VnetInterface.setup_loopback(vnet_name) 307 308 not_found = self._wait_interfaces(vnet_name, [i.name for i in ifaces]) 309 if not_found: 310 raise Exception( 311 "Interfaces {} has not appeared in vnet {}".format(not_found, vnet_name) 312 ) 313 return VnetInstance(vnet_alias, vnet_name, jid, ifaces) 314 315 def cleanup(self): 316 iface_factory = IfaceFactory() 317 try: 318 with open(self.file_name) as f: 319 for line in f: 320 vnet_name = line.strip() 321 iface_factory.cleanup_vnet_interfaces(vnet_name) 322 run_cmd("/usr/sbin/jail -r {}".format(vnet_name)) 323 os.unlink(self.JAILS_FNAME) 324 except OSError: 325 pass 326 327 328class SingleInterfaceMap(NamedTuple): 329 ifaces: List[VnetInterface] 330 vnet_aliases: List[str] 331 332 333class ObjectsMap(NamedTuple): 334 iface_map: Dict[str, SingleInterfaceMap] # keyed by ifX 335 vnet_map: Dict[str, VnetInstance] # keyed by vnetX 336 topo_map: Dict # self.TOPOLOGY 337 338 339class VnetTestTemplate(BaseTest): 340 NEED_ROOT: bool = True 341 TOPOLOGY = {} 342 343 def _require_default_modules(self): 344 libc.kldload("if_epair.ko") 345 self.require_module("if_epair") 346 347 def _get_vnet_handler(self, vnet_alias: str): 348 handler_name = "{}_handler".format(vnet_alias) 349 return getattr(self, handler_name, None) 350 351 def _setup_vnet(self, vnet: VnetInstance, obj_map: Dict, pipe): 352 """Base Handler to setup given VNET. 353 Can be run in a subprocess. If so, passes control to the special 354 vnetX_handler() after setting up interface addresses 355 """ 356 vnet.attach() 357 print("# setup_vnet({})".format(vnet.name)) 358 if pipe is not None: 359 vnet.set_pipe(pipe) 360 361 topo = obj_map.topo_map 362 ipv6_ifaces = [] 363 # Disable DAD 364 if not vnet.need_dad: 365 vnet.disable_dad() 366 for iface in vnet.ifaces: 367 # check index of vnet within an interface 368 # as we have prefixes for both ends of the interface 369 iface_map = obj_map.iface_map[iface.alias] 370 idx = iface_map.vnet_aliases.index(vnet.alias) 371 prefixes6 = topo[iface.alias].get("prefixes6", []) 372 prefixes4 = topo[iface.alias].get("prefixes4", []) 373 if prefixes6 or prefixes4: 374 ipv6_ifaces.append(iface) 375 iface.turn_up() 376 if prefixes6: 377 iface.enable_ipv6() 378 for prefix in prefixes6 + prefixes4: 379 if prefix[idx]: 380 iface.setup_addr(prefix[idx]) 381 for iface in ipv6_ifaces: 382 while iface.has_tentative(): 383 time.sleep(0.1) 384 385 # Run actual handler 386 handler = self._get_vnet_handler(vnet.alias) 387 if handler: 388 # Do unbuffered stdout for children 389 # so the logs are present if the child hangs 390 sys.stdout.reconfigure(line_buffering=True) 391 self.drop_privileges() 392 handler(vnet) 393 394 def _get_topo_ifmap(self, topo: Dict): 395 iface_factory = IfaceFactory() 396 iface_map: Dict[str, SingleInterfaceMap] = {} 397 iface_aliases = set() 398 for obj_name, obj_data in topo.items(): 399 if obj_name.startswith("vnet"): 400 for iface_alias in obj_data["ifaces"]: 401 iface_aliases.add(iface_alias) 402 for iface_alias in iface_aliases: 403 print("Creating {}".format(iface_alias)) 404 iface_data = topo[iface_alias] 405 iface_type = iface_data.get("type", "epair") 406 ifaces = iface_factory.create_iface(iface_alias, iface_type) 407 smap = SingleInterfaceMap(ifaces, []) 408 iface_map[iface_alias] = smap 409 return iface_map 410 411 def setup_topology(self, topo: Dict, topology_id: str): 412 """Creates jails & interfaces for the provided topology""" 413 vnet_map = {} 414 vnet_factory = VnetFactory(topology_id) 415 iface_map = self._get_topo_ifmap(topo) 416 for obj_name, obj_data in topo.items(): 417 if obj_name.startswith("vnet"): 418 vnet_ifaces = [] 419 for iface_alias in obj_data["ifaces"]: 420 # epair creates 2 interfaces, grab first _available_ 421 # and map it to the VNET being created 422 idx = len(iface_map[iface_alias].vnet_aliases) 423 iface_map[iface_alias].vnet_aliases.append(obj_name) 424 vnet_ifaces.append(iface_map[iface_alias].ifaces[idx]) 425 opts = [] 426 if "opts" in obj_data: 427 opts = obj_data["opts"] 428 vnet = vnet_factory.create_vnet(obj_name, vnet_ifaces, opts) 429 vnet_map[obj_name] = vnet 430 # Allow reference to VNETs as attributes 431 setattr(self, obj_name, vnet) 432 # Debug output 433 print("============= TEST TOPOLOGY =============") 434 for vnet_alias, vnet in vnet_map.items(): 435 print("# vnet {} -> {}".format(vnet.alias, vnet.name), end="") 436 handler = self._get_vnet_handler(vnet.alias) 437 if handler: 438 print(" handler: {}".format(handler.__name__), end="") 439 print() 440 for iface_alias, iface_data in iface_map.items(): 441 vnets = iface_data.vnet_aliases 442 ifaces: List[VnetInterface] = iface_data.ifaces 443 if len(vnets) == 1 and len(ifaces) == 2: 444 print( 445 "# iface {}: {}::{} -> main::{}".format( 446 iface_alias, vnets[0], ifaces[0].name, ifaces[1].name 447 ) 448 ) 449 elif len(vnets) == 2 and len(ifaces) == 2: 450 print( 451 "# iface {}: {}::{} -> {}::{}".format( 452 iface_alias, vnets[0], ifaces[0].name, vnets[1], ifaces[1].name 453 ) 454 ) 455 else: 456 print( 457 "# iface {}: ifaces: {} vnets: {}".format( 458 iface_alias, vnets, [i.name for i in ifaces] 459 ) 460 ) 461 print() 462 return ObjectsMap(iface_map, vnet_map, topo) 463 464 def setup_method(self, _method): 465 """Sets up all the required topology and handlers for the given test""" 466 super().setup_method(_method) 467 self._require_default_modules() 468 469 # TestIP6Output.test_output6_pktinfo[ipandif] 470 topology_id = get_topology_id(self.test_id) 471 topology = self.TOPOLOGY 472 # First, setup kernel objects - interfaces & vnets 473 obj_map = self.setup_topology(topology, topology_id) 474 main_vnet = None # one without subprocess handler 475 for vnet_alias, vnet in obj_map.vnet_map.items(): 476 if self._get_vnet_handler(vnet_alias): 477 # Need subprocess to run 478 parent_pipe, child_pipe = Pipe() 479 p = Process( 480 target=self._setup_vnet, 481 args=( 482 vnet, 483 obj_map, 484 child_pipe, 485 ), 486 ) 487 vnet.set_pipe(parent_pipe) 488 vnet.set_subprocess(p) 489 p.start() 490 else: 491 if main_vnet is not None: 492 raise Exception("there can be only 1 VNET w/o handler") 493 main_vnet = vnet 494 # Main vnet needs to be the last, so all the other subprocesses 495 # are started & their pipe handles collected 496 self.vnet = main_vnet 497 self._setup_vnet(main_vnet, obj_map, None) 498 # Save state for the main handler 499 self.iface_map = obj_map.iface_map 500 self.vnet_map = obj_map.vnet_map 501 self.drop_privileges() 502 503 def cleanup(self, test_id: str): 504 # pytest test id: file::class::test_name 505 topology_id = get_topology_id(self.test_id) 506 507 print("============= vnet cleanup =============") 508 print("# topology_id: '{}'".format(topology_id)) 509 VnetFactory(topology_id).cleanup() 510 IfaceFactory().cleanup() 511 512 def wait_object(self, pipe, timeout=5): 513 if pipe.poll(timeout): 514 return pipe.recv() 515 raise TimeoutError 516 517 def wait_objects_any(self, pipe_list, timeout=5): 518 objects = connection.wait(pipe_list, timeout) 519 if objects: 520 return objects[0].recv() 521 raise TimeoutError 522 523 def send_object(self, pipe, obj): 524 pipe.send(obj) 525 526 def wait(self): 527 while True: 528 time.sleep(1) 529 530 @property 531 def curvnet(self): 532 pass 533 534 535class SingleVnetTestTemplate(VnetTestTemplate): 536 IPV6_PREFIXES: List[str] = [] 537 IPV4_PREFIXES: List[str] = [] 538 IFTYPE = "epair" 539 540 def _setup_default_topology(self): 541 topology = copy.deepcopy( 542 { 543 "vnet1": {"ifaces": ["if1"]}, 544 "if1": {"type": self.IFTYPE, "prefixes4": [], "prefixes6": []}, 545 } 546 ) 547 for prefix in self.IPV6_PREFIXES: 548 topology["if1"]["prefixes6"].append((prefix,)) 549 for prefix in self.IPV4_PREFIXES: 550 topology["if1"]["prefixes4"].append((prefix,)) 551 return topology 552 553 def setup_method(self, method): 554 if not getattr(self, "TOPOLOGY", None): 555 self.TOPOLOGY = self._setup_default_topology() 556 else: 557 names = self.TOPOLOGY.keys() 558 assert len([n for n in names if n.startswith("vnet")]) == 1 559 super().setup_method(method) 560