1#!/usr/local/bin/python3 2import copy 3import ipaddress 4import os 5import re 6import socket 7import sys 8import time 9from multiprocessing import connection 10from multiprocessing import Pipe 11from multiprocessing import Process 12from typing import Dict 13from typing import List 14from typing import NamedTuple 15 16from atf_python.sys.net.tools import ToolsHelper 17from atf_python.utils import BaseTest 18from atf_python.utils import libc 19 20 21def run_cmd(cmd: str, verbose=True) -> str: 22 if verbose: 23 print("run: '{}'".format(cmd)) 24 return os.popen(cmd).read() 25 26 27def get_topology_id(test_id: str) -> str: 28 """ 29 Gets a unique topology id based on the pytest test_id. 30 "test_ip6_output.py::TestIP6Output::test_output6_pktinfo[ipandif]" -> 31 "TestIP6Output:test_output6_pktinfo[ipandif]" 32 """ 33 return ":".join(test_id.split("::")[-2:]) 34 35 36def convert_test_name(test_name: str) -> str: 37 """Convert test name to a string that can be used in the file/jail names""" 38 ret = "" 39 for char in test_name: 40 if char.isalnum() or char in ("_", "-", ":"): 41 ret += char 42 elif char in ("["): 43 ret += "_" 44 return ret 45 46 47class VnetInterface(object): 48 # defines from net/if_types.h 49 IFT_LOOP = 0x18 50 IFT_ETHER = 0x06 51 52 def __init__(self, iface_alias: str, iface_name: str): 53 self.name = iface_name 54 self.alias = iface_alias 55 self.vnet_name = "" 56 self.jailed = False 57 self.addr_map: Dict[str, Dict] = {"inet6": {}, "inet": {}} 58 self.prefixes4: List[List[str]] = [] 59 self.prefixes6: List[List[str]] = [] 60 if iface_name.startswith("lo"): 61 self.iftype = self.IFT_LOOP 62 else: 63 self.iftype = self.IFT_ETHER 64 self.ether = ToolsHelper.get_output("/sbin/ifconfig %s ether | awk '/ether/ { print $2; }'" % iface_name).rstrip() 65 66 @property 67 def ifindex(self): 68 return socket.if_nametoindex(self.name) 69 70 @property 71 def first_ipv6(self): 72 d = self.addr_map["inet6"] 73 return d[next(iter(d))] 74 75 @property 76 def first_ipv4(self): 77 d = self.addr_map["inet"] 78 return d[next(iter(d))] 79 80 def set_vnet(self, vnet_name: str): 81 self.vnet_name = vnet_name 82 83 def set_jailed(self, jailed: bool): 84 self.jailed = jailed 85 86 def run_cmd(self, cmd, verbose=False): 87 if self.vnet_name and not self.jailed: 88 cmd = "/usr/sbin/jexec {} {}".format(self.vnet_name, cmd) 89 return run_cmd(cmd, verbose) 90 91 @classmethod 92 def setup_loopback(cls, vnet_name: str): 93 lo = VnetInterface("", "lo0") 94 lo.set_vnet(vnet_name) 95 lo.setup_addr("127.0.0.1/8") 96 lo.turn_up() 97 98 @classmethod 99 def create_iface(cls, alias_name: str, iface_name: str) -> List["VnetInterface"]: 100 name = run_cmd("/sbin/ifconfig {} create".format(iface_name)).rstrip() 101 if not name: 102 raise Exception("Unable to create iface {}".format(iface_name)) 103 if1 = cls(alias_name, name) 104 ret = [if1] 105 if name.startswith("epair"): 106 run_cmd("/sbin/ifconfig {} -txcsum -txcsum6".format(name)) 107 if2 = cls(alias_name, name[:-1] + "b") 108 if1.epairb = if2 109 ret.append(if2); 110 return ret 111 112 def set_mtu(self, mtu): 113 run_cmd("/sbin/ifconfig {} mtu {}".format(self.name, mtu)) 114 115 def setup_addr(self, _addr: str): 116 addr = ipaddress.ip_interface(_addr) 117 if addr.version == 6: 118 family = "inet6" 119 cmd = "/sbin/ifconfig {} {} {}".format(self.name, family, addr) 120 else: 121 family = "inet" 122 if self.addr_map[family]: 123 cmd = "/sbin/ifconfig {} alias {}".format(self.name, addr) 124 else: 125 cmd = "/sbin/ifconfig {} {} {}".format(self.name, family, addr) 126 self.run_cmd(cmd) 127 self.addr_map[family][str(addr.ip)] = addr 128 129 def delete_addr(self, _addr: str): 130 addr = ipaddress.ip_address(_addr) 131 if addr.version == 6: 132 family = "inet6" 133 cmd = "/sbin/ifconfig {} inet6 {} delete".format(self.name, addr) 134 else: 135 family = "inet" 136 cmd = "/sbin/ifconfig {} -alias {}".format(self.name, addr) 137 self.run_cmd(cmd) 138 del self.addr_map[family][str(addr)] 139 140 def turn_up(self): 141 cmd = "/sbin/ifconfig {} up".format(self.name) 142 self.run_cmd(cmd) 143 144 def enable_ipv6(self): 145 cmd = "/usr/sbin/ndp -i {} -- -disabled".format(self.name) 146 self.run_cmd(cmd) 147 148 def has_tentative(self) -> bool: 149 """True if an interface has some addresses in tenative state""" 150 cmd = "/sbin/ifconfig {} inet6".format(self.name) 151 out = self.run_cmd(cmd, verbose=False) 152 for line in out.splitlines(): 153 if "tentative" in line: 154 return True 155 return False 156 157 158class IfaceFactory(object): 159 INTERFACES_FNAME = "created_ifaces.lst" 160 AUTODELETE_TYPES = ("epair", "gif", "gre", "lo", "tap", "tun") 161 162 def __init__(self): 163 self.file_name = self.INTERFACES_FNAME 164 165 def _register_iface(self, iface_name: str): 166 with open(self.file_name, "a") as f: 167 f.write(iface_name + "\n") 168 169 def _list_ifaces(self) -> List[str]: 170 ret: List[str] = [] 171 try: 172 with open(self.file_name, "r") as f: 173 for line in f: 174 ret.append(line.strip()) 175 except OSError: 176 pass 177 return ret 178 179 def create_iface(self, alias_name: str, iface_name: str) -> List[VnetInterface]: 180 ifaces = VnetInterface.create_iface(alias_name, iface_name) 181 for iface in ifaces: 182 if not self.is_autodeleted(iface.name): 183 self._register_iface(iface.name) 184 return ifaces 185 186 @staticmethod 187 def is_autodeleted(iface_name: str) -> bool: 188 if iface_name == "lo0": 189 return False 190 iface_type = re.split(r"\d+", iface_name)[0] 191 return iface_type in IfaceFactory.AUTODELETE_TYPES 192 193 def cleanup_vnet_interfaces(self, vnet_name: str) -> List[str]: 194 """Destroys""" 195 ifaces_lst = ToolsHelper.get_output( 196 "/usr/sbin/jexec {} /sbin/ifconfig -l".format(vnet_name) 197 ) 198 for iface_name in ifaces_lst.split(): 199 if not self.is_autodeleted(iface_name): 200 if iface_name not in self._list_ifaces(): 201 print("Skipping interface {}:{}".format(vnet_name, iface_name)) 202 continue 203 run_cmd( 204 "/usr/sbin/jexec {} /sbin/ifconfig {} destroy".format(vnet_name, iface_name) 205 ) 206 207 def cleanup(self): 208 try: 209 os.unlink(self.INTERFACES_FNAME) 210 except OSError: 211 pass 212 213 214class VnetInstance(object): 215 def __init__( 216 self, vnet_alias: str, vnet_name: str, jid: int, ifaces: List[VnetInterface] 217 ): 218 self.name = vnet_name 219 self.alias = vnet_alias # reference in the test topology 220 self.jid = jid 221 self.ifaces = ifaces 222 self.iface_alias_map = {} # iface.alias: iface 223 self.iface_map = {} # iface.name: iface 224 for iface in ifaces: 225 iface.set_vnet(vnet_name) 226 iface.set_jailed(True) 227 self.iface_alias_map[iface.alias] = iface 228 self.iface_map[iface.name] = iface 229 # Allow reference to interfce aliases as attributes 230 setattr(self, iface.alias, iface) 231 self.need_dad = False # Disable duplicate address detection by default 232 self.attached = False 233 self.pipe = None 234 self.subprocess = None 235 236 def run_vnet_cmd(self, cmd, verbose=True): 237 if not self.attached: 238 cmd = "/usr/sbin/jexec {} {}".format(self.name, cmd) 239 return run_cmd(cmd, verbose) 240 241 def disable_dad(self): 242 self.run_vnet_cmd("/sbin/sysctl net.inet6.ip6.dad_count=0") 243 244 def set_pipe(self, pipe): 245 self.pipe = pipe 246 247 def set_subprocess(self, p): 248 self.subprocess = p 249 250 @staticmethod 251 def attach_jid(jid: int): 252 error_code = libc.jail_attach(jid) 253 if error_code != 0: 254 raise Exception("jail_attach() failed: errno {}".format(error_code)) 255 256 def attach(self): 257 self.attach_jid(self.jid) 258 self.attached = True 259 260 261class VnetFactory(object): 262 JAILS_FNAME = "created_jails.lst" 263 264 def __init__(self, topology_id: str): 265 self.topology_id = topology_id 266 self.file_name = self.JAILS_FNAME 267 self._vnets: List[str] = [] 268 269 def _register_vnet(self, vnet_name: str): 270 self._vnets.append(vnet_name) 271 with open(self.file_name, "a") as f: 272 f.write(vnet_name + "\n") 273 274 @staticmethod 275 def _wait_interfaces(vnet_name: str, ifaces: List[str]) -> List[str]: 276 cmd = "/usr/sbin/jexec {} /sbin/ifconfig -l".format(vnet_name) 277 not_matched: List[str] = [] 278 for i in range(50): 279 vnet_ifaces = run_cmd(cmd).strip().split(" ") 280 not_matched = [] 281 for iface_name in ifaces: 282 if iface_name not in vnet_ifaces: 283 not_matched.append(iface_name) 284 if len(not_matched) == 0: 285 return [] 286 time.sleep(0.1) 287 return not_matched 288 289 def create_vnet(self, vnet_alias: str, ifaces: List[VnetInterface], opts: List[str]): 290 vnet_name = "pytest:{}".format(convert_test_name(self.topology_id)) 291 if self._vnets: 292 # add number to distinguish jails 293 vnet_name = "{}_{}".format(vnet_name, len(self._vnets) + 1) 294 iface_cmds = " ".join(["vnet.interface={}".format(i.name) for i in ifaces]) 295 opt_cmds = " ".join(["{}".format(i) for i in opts]) 296 cmd = "/usr/sbin/jail -i -c name={} persist vnet {} {}".format( 297 vnet_name, iface_cmds, opt_cmds 298 ) 299 jid = 0 300 try: 301 jid_str = run_cmd(cmd) 302 jid = int(jid_str) 303 except ValueError: 304 print("Jail creation failed, output: {}".format(jid_str)) 305 raise 306 self._register_vnet(vnet_name) 307 308 # Run expedited version of routing 309 VnetInterface.setup_loopback(vnet_name) 310 311 not_found = self._wait_interfaces(vnet_name, [i.name for i in ifaces]) 312 if not_found: 313 raise Exception( 314 "Interfaces {} has not appeared in vnet {}".format(not_found, vnet_name) 315 ) 316 return VnetInstance(vnet_alias, vnet_name, jid, ifaces) 317 318 def cleanup(self): 319 iface_factory = IfaceFactory() 320 try: 321 with open(self.file_name) as f: 322 for line in f: 323 vnet_name = line.strip() 324 iface_factory.cleanup_vnet_interfaces(vnet_name) 325 run_cmd("/usr/sbin/jail -r {}".format(vnet_name)) 326 os.unlink(self.JAILS_FNAME) 327 except OSError: 328 pass 329 330 331class SingleInterfaceMap(NamedTuple): 332 ifaces: List[VnetInterface] 333 vnet_aliases: List[str] 334 335 336class ObjectsMap(NamedTuple): 337 iface_map: Dict[str, SingleInterfaceMap] # keyed by ifX 338 vnet_map: Dict[str, VnetInstance] # keyed by vnetX 339 topo_map: Dict # self.TOPOLOGY 340 341 342class VnetTestTemplate(BaseTest): 343 NEED_ROOT: bool = True 344 TOPOLOGY = {} 345 346 def _require_default_modules(self): 347 libc.kldload("if_epair.ko") 348 self.require_module("if_epair") 349 350 def _get_vnet_handler(self, vnet_alias: str): 351 handler_name = "{}_handler".format(vnet_alias) 352 return getattr(self, handler_name, None) 353 354 def _setup_vnet(self, vnet: VnetInstance, obj_map: Dict, pipe): 355 """Base Handler to setup given VNET. 356 Can be run in a subprocess. If so, passes control to the special 357 vnetX_handler() after setting up interface addresses 358 """ 359 vnet.attach() 360 print("# setup_vnet({})".format(vnet.name)) 361 if pipe is not None: 362 vnet.set_pipe(pipe) 363 364 topo = obj_map.topo_map 365 ipv6_ifaces = [] 366 # Disable DAD 367 if not vnet.need_dad: 368 vnet.disable_dad() 369 for iface in vnet.ifaces: 370 # check index of vnet within an interface 371 # as we have prefixes for both ends of the interface 372 iface_map = obj_map.iface_map[iface.alias] 373 idx = iface_map.vnet_aliases.index(vnet.alias) 374 prefixes6 = topo[iface.alias].get("prefixes6", []) 375 prefixes4 = topo[iface.alias].get("prefixes4", []) 376 mtu = topo[iface.alias].get("mtu", 0) 377 if prefixes6 or prefixes4: 378 ipv6_ifaces.append(iface) 379 iface.turn_up() 380 if prefixes6: 381 iface.enable_ipv6() 382 for prefix in prefixes6 + prefixes4: 383 if prefix[idx]: 384 iface.setup_addr(prefix[idx]) 385 if mtu != 0: 386 iface.set_mtu(mtu) 387 for iface in ipv6_ifaces: 388 while iface.has_tentative(): 389 time.sleep(0.1) 390 391 # Run actual handler 392 handler = self._get_vnet_handler(vnet.alias) 393 if handler: 394 # Do unbuffered stdout for children 395 # so the logs are present if the child hangs 396 sys.stdout.reconfigure(line_buffering=True) 397 self.drop_privileges() 398 handler(vnet) 399 400 def _get_topo_ifmap(self, topo: Dict): 401 iface_factory = IfaceFactory() 402 iface_map: Dict[str, SingleInterfaceMap] = {} 403 iface_aliases = set() 404 for obj_name, obj_data in topo.items(): 405 if obj_name.startswith("vnet"): 406 for iface_alias in obj_data["ifaces"]: 407 iface_aliases.add(iface_alias) 408 for iface_alias in iface_aliases: 409 print("Creating {}".format(iface_alias)) 410 iface_data = topo[iface_alias] 411 iface_type = iface_data.get("type", "epair") 412 ifaces = iface_factory.create_iface(iface_alias, iface_type) 413 smap = SingleInterfaceMap(ifaces, []) 414 iface_map[iface_alias] = smap 415 return iface_map 416 417 def setup_topology(self, topo: Dict, topology_id: str): 418 """Creates jails & interfaces for the provided topology""" 419 vnet_map = {} 420 vnet_factory = VnetFactory(topology_id) 421 iface_map = self._get_topo_ifmap(topo) 422 for obj_name, obj_data in topo.items(): 423 if obj_name.startswith("vnet"): 424 vnet_ifaces = [] 425 for iface_alias in obj_data["ifaces"]: 426 # epair creates 2 interfaces, grab first _available_ 427 # and map it to the VNET being created 428 idx = len(iface_map[iface_alias].vnet_aliases) 429 iface_map[iface_alias].vnet_aliases.append(obj_name) 430 vnet_ifaces.append(iface_map[iface_alias].ifaces[idx]) 431 opts = [] 432 if "opts" in obj_data: 433 opts = obj_data["opts"] 434 vnet = vnet_factory.create_vnet(obj_name, vnet_ifaces, opts) 435 vnet_map[obj_name] = vnet 436 # Allow reference to VNETs as attributes 437 setattr(self, obj_name, vnet) 438 # Debug output 439 print("============= TEST TOPOLOGY =============") 440 for vnet_alias, vnet in vnet_map.items(): 441 print("# vnet {} -> {}".format(vnet.alias, vnet.name), end="") 442 handler = self._get_vnet_handler(vnet.alias) 443 if handler: 444 print(" handler: {}".format(handler.__name__), end="") 445 print() 446 for iface_alias, iface_data in iface_map.items(): 447 vnets = iface_data.vnet_aliases 448 ifaces: List[VnetInterface] = iface_data.ifaces 449 if len(vnets) == 1 and len(ifaces) == 2: 450 print( 451 "# iface {}: {}::{} -> main::{}".format( 452 iface_alias, vnets[0], ifaces[0].name, ifaces[1].name 453 ) 454 ) 455 elif len(vnets) == 2 and len(ifaces) == 2: 456 print( 457 "# iface {}: {}::{} -> {}::{}".format( 458 iface_alias, vnets[0], ifaces[0].name, vnets[1], ifaces[1].name 459 ) 460 ) 461 else: 462 print( 463 "# iface {}: ifaces: {} vnets: {}".format( 464 iface_alias, vnets, [i.name for i in ifaces] 465 ) 466 ) 467 print() 468 return ObjectsMap(iface_map, vnet_map, topo) 469 470 def setup_method(self, _method): 471 """Sets up all the required topology and handlers for the given test""" 472 super().setup_method(_method) 473 self._require_default_modules() 474 475 # TestIP6Output.test_output6_pktinfo[ipandif] 476 topology_id = get_topology_id(self.test_id) 477 topology = self.TOPOLOGY 478 # First, setup kernel objects - interfaces & vnets 479 obj_map = self.setup_topology(topology, topology_id) 480 main_vnet = None # one without subprocess handler 481 for vnet_alias, vnet in obj_map.vnet_map.items(): 482 if self._get_vnet_handler(vnet_alias): 483 # Need subprocess to run 484 parent_pipe, child_pipe = Pipe() 485 p = Process( 486 target=self._setup_vnet, 487 args=( 488 vnet, 489 obj_map, 490 child_pipe, 491 ), 492 ) 493 vnet.set_pipe(parent_pipe) 494 vnet.set_subprocess(p) 495 p.start() 496 else: 497 if main_vnet is not None: 498 raise Exception("there can be only 1 VNET w/o handler") 499 main_vnet = vnet 500 # Main vnet needs to be the last, so all the other subprocesses 501 # are started & their pipe handles collected 502 self.vnet = main_vnet 503 self._setup_vnet(main_vnet, obj_map, None) 504 # Save state for the main handler 505 self.iface_map = obj_map.iface_map 506 self.vnet_map = obj_map.vnet_map 507 self.drop_privileges() 508 509 def cleanup(self, test_id: str): 510 # pytest test id: file::class::test_name 511 topology_id = get_topology_id(self.test_id) 512 513 print("============= vnet cleanup =============") 514 print("# topology_id: '{}'".format(topology_id)) 515 VnetFactory(topology_id).cleanup() 516 IfaceFactory().cleanup() 517 518 def wait_object(self, pipe, timeout=5): 519 if pipe.poll(timeout): 520 return pipe.recv() 521 raise TimeoutError 522 523 def wait_objects_any(self, pipe_list, timeout=5): 524 objects = connection.wait(pipe_list, timeout) 525 if objects: 526 return objects[0].recv() 527 raise TimeoutError 528 529 def send_object(self, pipe, obj): 530 pipe.send(obj) 531 532 def wait(self): 533 while True: 534 time.sleep(1) 535 536 @property 537 def curvnet(self): 538 pass 539 540 541class SingleVnetTestTemplate(VnetTestTemplate): 542 IPV6_PREFIXES: List[str] = [] 543 IPV4_PREFIXES: List[str] = [] 544 IFTYPE = "epair" 545 546 def _setup_default_topology(self): 547 topology = copy.deepcopy( 548 { 549 "vnet1": {"ifaces": ["if1"]}, 550 "if1": {"type": self.IFTYPE, "prefixes4": [], "prefixes6": []}, 551 } 552 ) 553 for prefix in self.IPV6_PREFIXES: 554 topology["if1"]["prefixes6"].append((prefix,)) 555 for prefix in self.IPV4_PREFIXES: 556 topology["if1"]["prefixes4"].append((prefix,)) 557 return topology 558 559 def setup_method(self, method): 560 if not getattr(self, "TOPOLOGY", None): 561 self.TOPOLOGY = self._setup_default_topology() 562 else: 563 names = self.TOPOLOGY.keys() 564 assert len([n for n in names if n.startswith("vnet")]) == 1 565 super().setup_method(method) 566