1#!/usr/local/bin/python3 2import copy 3import ipaddress 4import os 5import re 6import socket 7import sys 8import time 9from multiprocessing import connection 10from multiprocessing import Pipe 11from multiprocessing import Process 12from typing import Dict 13from typing import List 14from typing import NamedTuple 15 16from atf_python.sys.net.tools import ToolsHelper 17from atf_python.utils import BaseTest 18from atf_python.utils import libc 19 20 21def run_cmd(cmd: str, verbose=True) -> str: 22 if verbose: 23 print("run: '{}'".format(cmd)) 24 return os.popen(cmd).read() 25 26 27def get_topology_id(test_id: str) -> str: 28 """ 29 Gets a unique topology id based on the pytest test_id. 30 "test_ip6_output.py::TestIP6Output::test_output6_pktinfo[ipandif]" -> 31 "TestIP6Output:test_output6_pktinfo[ipandif]" 32 """ 33 return ":".join(test_id.split("::")[-2:]) 34 35 36def convert_test_name(test_name: str) -> str: 37 """Convert test name to a string that can be used in the file/jail names""" 38 ret = "" 39 for char in test_name: 40 if char.isalnum() or char in ("_", "-", ":"): 41 ret += char 42 elif char in ("["): 43 ret += "_" 44 return ret 45 46 47class VnetInterface(object): 48 # defines from net/if_types.h 49 IFT_LOOP = 0x18 50 IFT_ETHER = 0x06 51 52 def __init__(self, iface_alias: str, iface_name: str): 53 self.name = iface_name 54 self.alias = iface_alias 55 self.vnet_name = "" 56 self.jailed = False 57 self.addr_map: Dict[str, Dict] = {"inet6": {}, "inet": {}} 58 self.prefixes4: List[List[str]] = [] 59 self.prefixes6: List[List[str]] = [] 60 if iface_name.startswith("lo"): 61 self.iftype = self.IFT_LOOP 62 else: 63 self.iftype = self.IFT_ETHER 64 self.ether = ToolsHelper.get_output("/sbin/ifconfig %s ether | awk '/ether/ { print $2; }'" % iface_name).rstrip() 65 66 @property 67 def ifindex(self): 68 return socket.if_nametoindex(self.name) 69 70 @property 71 def first_ipv6(self): 72 d = self.addr_map["inet6"] 73 return d[next(iter(d))] 74 75 @property 76 def first_ipv4(self): 77 d = self.addr_map["inet"] 78 return d[next(iter(d))] 79 80 def set_vnet(self, vnet_name: str): 81 self.vnet_name = vnet_name 82 83 def set_jailed(self, jailed: bool): 84 self.jailed = jailed 85 86 def run_cmd(self, cmd, verbose=False): 87 if self.vnet_name and not self.jailed: 88 cmd = "/usr/sbin/jexec {} {}".format(self.vnet_name, cmd) 89 return run_cmd(cmd, verbose) 90 91 @classmethod 92 def setup_loopback(cls, vnet_name: str): 93 lo = VnetInterface("", "lo0") 94 lo.set_vnet(vnet_name) 95 lo.setup_addr("127.0.0.1/8") 96 lo.turn_up() 97 98 @classmethod 99 def create_iface(cls, alias_name: str, iface_name: str) -> List["VnetInterface"]: 100 name = run_cmd("/sbin/ifconfig {} create".format(iface_name)).rstrip() 101 if not name: 102 raise Exception("Unable to create iface {}".format(iface_name)) 103 if1 = cls(alias_name, name) 104 ret = [if1] 105 if name.startswith("epair"): 106 run_cmd("/sbin/ifconfig {} -txcsum -txcsum6".format(name)) 107 if2 = cls(alias_name, name[:-1] + "b") 108 if1.epairb = if2 109 ret.append(if2); 110 return ret 111 112 def setup_addr(self, _addr: str): 113 addr = ipaddress.ip_interface(_addr) 114 if addr.version == 6: 115 family = "inet6" 116 cmd = "/sbin/ifconfig {} {} {}".format(self.name, family, addr) 117 else: 118 family = "inet" 119 if self.addr_map[family]: 120 cmd = "/sbin/ifconfig {} alias {}".format(self.name, addr) 121 else: 122 cmd = "/sbin/ifconfig {} {} {}".format(self.name, family, addr) 123 self.run_cmd(cmd) 124 self.addr_map[family][str(addr.ip)] = addr 125 126 def delete_addr(self, _addr: str): 127 addr = ipaddress.ip_address(_addr) 128 if addr.version == 6: 129 family = "inet6" 130 cmd = "/sbin/ifconfig {} inet6 {} delete".format(self.name, addr) 131 else: 132 family = "inet" 133 cmd = "/sbin/ifconfig {} -alias {}".format(self.name, addr) 134 self.run_cmd(cmd) 135 del self.addr_map[family][str(addr)] 136 137 def turn_up(self): 138 cmd = "/sbin/ifconfig {} up".format(self.name) 139 self.run_cmd(cmd) 140 141 def enable_ipv6(self): 142 cmd = "/usr/sbin/ndp -i {} -- -disabled".format(self.name) 143 self.run_cmd(cmd) 144 145 def has_tentative(self) -> bool: 146 """True if an interface has some addresses in tenative state""" 147 cmd = "/sbin/ifconfig {} inet6".format(self.name) 148 out = self.run_cmd(cmd, verbose=False) 149 for line in out.splitlines(): 150 if "tentative" in line: 151 return True 152 return False 153 154 155class IfaceFactory(object): 156 INTERFACES_FNAME = "created_ifaces.lst" 157 AUTODELETE_TYPES = ("epair", "gif", "gre", "lo", "tap", "tun") 158 159 def __init__(self): 160 self.file_name = self.INTERFACES_FNAME 161 162 def _register_iface(self, iface_name: str): 163 with open(self.file_name, "a") as f: 164 f.write(iface_name + "\n") 165 166 def _list_ifaces(self) -> List[str]: 167 ret: List[str] = [] 168 try: 169 with open(self.file_name, "r") as f: 170 for line in f: 171 ret.append(line.strip()) 172 except OSError: 173 pass 174 return ret 175 176 def create_iface(self, alias_name: str, iface_name: str) -> List[VnetInterface]: 177 ifaces = VnetInterface.create_iface(alias_name, iface_name) 178 for iface in ifaces: 179 if not self.is_autodeleted(iface.name): 180 self._register_iface(iface.name) 181 return ifaces 182 183 @staticmethod 184 def is_autodeleted(iface_name: str) -> bool: 185 if iface_name == "lo0": 186 return False 187 iface_type = re.split(r"\d+", iface_name)[0] 188 return iface_type in IfaceFactory.AUTODELETE_TYPES 189 190 def cleanup_vnet_interfaces(self, vnet_name: str) -> List[str]: 191 """Destroys""" 192 ifaces_lst = ToolsHelper.get_output( 193 "/usr/sbin/jexec {} /sbin/ifconfig -l".format(vnet_name) 194 ) 195 for iface_name in ifaces_lst.split(): 196 if not self.is_autodeleted(iface_name): 197 if iface_name not in self._list_ifaces(): 198 print("Skipping interface {}:{}".format(vnet_name, iface_name)) 199 continue 200 run_cmd( 201 "/usr/sbin/jexec {} /sbin/ifconfig {} destroy".format(vnet_name, iface_name) 202 ) 203 204 def cleanup(self): 205 try: 206 os.unlink(self.INTERFACES_FNAME) 207 except OSError: 208 pass 209 210 211class VnetInstance(object): 212 def __init__( 213 self, vnet_alias: str, vnet_name: str, jid: int, ifaces: List[VnetInterface] 214 ): 215 self.name = vnet_name 216 self.alias = vnet_alias # reference in the test topology 217 self.jid = jid 218 self.ifaces = ifaces 219 self.iface_alias_map = {} # iface.alias: iface 220 self.iface_map = {} # iface.name: iface 221 for iface in ifaces: 222 iface.set_vnet(vnet_name) 223 iface.set_jailed(True) 224 self.iface_alias_map[iface.alias] = iface 225 self.iface_map[iface.name] = iface 226 # Allow reference to interfce aliases as attributes 227 setattr(self, iface.alias, iface) 228 self.need_dad = False # Disable duplicate address detection by default 229 self.attached = False 230 self.pipe = None 231 self.subprocess = None 232 233 def run_vnet_cmd(self, cmd, verbose=True): 234 if not self.attached: 235 cmd = "/usr/sbin/jexec {} {}".format(self.name, cmd) 236 return run_cmd(cmd, verbose) 237 238 def disable_dad(self): 239 self.run_vnet_cmd("/sbin/sysctl net.inet6.ip6.dad_count=0") 240 241 def set_pipe(self, pipe): 242 self.pipe = pipe 243 244 def set_subprocess(self, p): 245 self.subprocess = p 246 247 @staticmethod 248 def attach_jid(jid: int): 249 error_code = libc.jail_attach(jid) 250 if error_code != 0: 251 raise Exception("jail_attach() failed: errno {}".format(error_code)) 252 253 def attach(self): 254 self.attach_jid(self.jid) 255 self.attached = True 256 257 258class VnetFactory(object): 259 JAILS_FNAME = "created_jails.lst" 260 261 def __init__(self, topology_id: str): 262 self.topology_id = topology_id 263 self.file_name = self.JAILS_FNAME 264 self._vnets: List[str] = [] 265 266 def _register_vnet(self, vnet_name: str): 267 self._vnets.append(vnet_name) 268 with open(self.file_name, "a") as f: 269 f.write(vnet_name + "\n") 270 271 @staticmethod 272 def _wait_interfaces(vnet_name: str, ifaces: List[str]) -> List[str]: 273 cmd = "/usr/sbin/jexec {} /sbin/ifconfig -l".format(vnet_name) 274 not_matched: List[str] = [] 275 for i in range(50): 276 vnet_ifaces = run_cmd(cmd).strip().split(" ") 277 not_matched = [] 278 for iface_name in ifaces: 279 if iface_name not in vnet_ifaces: 280 not_matched.append(iface_name) 281 if len(not_matched) == 0: 282 return [] 283 time.sleep(0.1) 284 return not_matched 285 286 def create_vnet(self, vnet_alias: str, ifaces: List[VnetInterface]): 287 vnet_name = "pytest:{}".format(convert_test_name(self.topology_id)) 288 if self._vnets: 289 # add number to distinguish jails 290 vnet_name = "{}_{}".format(vnet_name, len(self._vnets) + 1) 291 iface_cmds = " ".join(["vnet.interface={}".format(i.name) for i in ifaces]) 292 cmd = "/usr/sbin/jail -i -c name={} persist vnet {}".format( 293 vnet_name, iface_cmds 294 ) 295 jid = 0 296 try: 297 jid_str = run_cmd(cmd) 298 jid = int(jid_str) 299 except ValueError: 300 print("Jail creation failed, output: {}".format(jid_str)) 301 raise 302 self._register_vnet(vnet_name) 303 304 # Run expedited version of routing 305 VnetInterface.setup_loopback(vnet_name) 306 307 not_found = self._wait_interfaces(vnet_name, [i.name for i in ifaces]) 308 if not_found: 309 raise Exception( 310 "Interfaces {} has not appeared in vnet {}".format(not_found, vnet_name) 311 ) 312 return VnetInstance(vnet_alias, vnet_name, jid, ifaces) 313 314 def cleanup(self): 315 iface_factory = IfaceFactory() 316 try: 317 with open(self.file_name) as f: 318 for line in f: 319 vnet_name = line.strip() 320 iface_factory.cleanup_vnet_interfaces(vnet_name) 321 run_cmd("/usr/sbin/jail -r {}".format(vnet_name)) 322 os.unlink(self.JAILS_FNAME) 323 except OSError: 324 pass 325 326 327class SingleInterfaceMap(NamedTuple): 328 ifaces: List[VnetInterface] 329 vnet_aliases: List[str] 330 331 332class ObjectsMap(NamedTuple): 333 iface_map: Dict[str, SingleInterfaceMap] # keyed by ifX 334 vnet_map: Dict[str, VnetInstance] # keyed by vnetX 335 topo_map: Dict # self.TOPOLOGY 336 337 338class VnetTestTemplate(BaseTest): 339 NEED_ROOT: bool = True 340 TOPOLOGY = {} 341 342 def _require_default_modules(self): 343 libc.kldload("if_epair.ko") 344 self.require_module("if_epair") 345 346 def _get_vnet_handler(self, vnet_alias: str): 347 handler_name = "{}_handler".format(vnet_alias) 348 return getattr(self, handler_name, None) 349 350 def _setup_vnet(self, vnet: VnetInstance, obj_map: Dict, pipe): 351 """Base Handler to setup given VNET. 352 Can be run in a subprocess. If so, passes control to the special 353 vnetX_handler() after setting up interface addresses 354 """ 355 vnet.attach() 356 print("# setup_vnet({})".format(vnet.name)) 357 if pipe is not None: 358 vnet.set_pipe(pipe) 359 360 topo = obj_map.topo_map 361 ipv6_ifaces = [] 362 # Disable DAD 363 if not vnet.need_dad: 364 vnet.disable_dad() 365 for iface in vnet.ifaces: 366 # check index of vnet within an interface 367 # as we have prefixes for both ends of the interface 368 iface_map = obj_map.iface_map[iface.alias] 369 idx = iface_map.vnet_aliases.index(vnet.alias) 370 prefixes6 = topo[iface.alias].get("prefixes6", []) 371 prefixes4 = topo[iface.alias].get("prefixes4", []) 372 if prefixes6 or prefixes4: 373 ipv6_ifaces.append(iface) 374 iface.turn_up() 375 if prefixes6: 376 iface.enable_ipv6() 377 for prefix in prefixes6 + prefixes4: 378 if prefix[idx]: 379 iface.setup_addr(prefix[idx]) 380 for iface in ipv6_ifaces: 381 while iface.has_tentative(): 382 time.sleep(0.1) 383 384 # Run actual handler 385 handler = self._get_vnet_handler(vnet.alias) 386 if handler: 387 # Do unbuffered stdout for children 388 # so the logs are present if the child hangs 389 sys.stdout.reconfigure(line_buffering=True) 390 self.drop_privileges() 391 handler(vnet) 392 393 def _get_topo_ifmap(self, topo: Dict): 394 iface_factory = IfaceFactory() 395 iface_map: Dict[str, SingleInterfaceMap] = {} 396 iface_aliases = set() 397 for obj_name, obj_data in topo.items(): 398 if obj_name.startswith("vnet"): 399 for iface_alias in obj_data["ifaces"]: 400 iface_aliases.add(iface_alias) 401 for iface_alias in iface_aliases: 402 print("Creating {}".format(iface_alias)) 403 iface_data = topo[iface_alias] 404 iface_type = iface_data.get("type", "epair") 405 ifaces = iface_factory.create_iface(iface_alias, iface_type) 406 smap = SingleInterfaceMap(ifaces, []) 407 iface_map[iface_alias] = smap 408 return iface_map 409 410 def setup_topology(self, topo: Dict, topology_id: str): 411 """Creates jails & interfaces for the provided topology""" 412 vnet_map = {} 413 vnet_factory = VnetFactory(topology_id) 414 iface_map = self._get_topo_ifmap(topo) 415 for obj_name, obj_data in topo.items(): 416 if obj_name.startswith("vnet"): 417 vnet_ifaces = [] 418 for iface_alias in obj_data["ifaces"]: 419 # epair creates 2 interfaces, grab first _available_ 420 # and map it to the VNET being created 421 idx = len(iface_map[iface_alias].vnet_aliases) 422 iface_map[iface_alias].vnet_aliases.append(obj_name) 423 vnet_ifaces.append(iface_map[iface_alias].ifaces[idx]) 424 vnet = vnet_factory.create_vnet(obj_name, vnet_ifaces) 425 vnet_map[obj_name] = vnet 426 # Allow reference to VNETs as attributes 427 setattr(self, obj_name, vnet) 428 # Debug output 429 print("============= TEST TOPOLOGY =============") 430 for vnet_alias, vnet in vnet_map.items(): 431 print("# vnet {} -> {}".format(vnet.alias, vnet.name), end="") 432 handler = self._get_vnet_handler(vnet.alias) 433 if handler: 434 print(" handler: {}".format(handler.__name__), end="") 435 print() 436 for iface_alias, iface_data in iface_map.items(): 437 vnets = iface_data.vnet_aliases 438 ifaces: List[VnetInterface] = iface_data.ifaces 439 if len(vnets) == 1 and len(ifaces) == 2: 440 print( 441 "# iface {}: {}::{} -> main::{}".format( 442 iface_alias, vnets[0], ifaces[0].name, ifaces[1].name 443 ) 444 ) 445 elif len(vnets) == 2 and len(ifaces) == 2: 446 print( 447 "# iface {}: {}::{} -> {}::{}".format( 448 iface_alias, vnets[0], ifaces[0].name, vnets[1], ifaces[1].name 449 ) 450 ) 451 else: 452 print( 453 "# iface {}: ifaces: {} vnets: {}".format( 454 iface_alias, vnets, [i.name for i in ifaces] 455 ) 456 ) 457 print() 458 return ObjectsMap(iface_map, vnet_map, topo) 459 460 def setup_method(self, _method): 461 """Sets up all the required topology and handlers for the given test""" 462 super().setup_method(_method) 463 self._require_default_modules() 464 465 # TestIP6Output.test_output6_pktinfo[ipandif] 466 topology_id = get_topology_id(self.test_id) 467 topology = self.TOPOLOGY 468 # First, setup kernel objects - interfaces & vnets 469 obj_map = self.setup_topology(topology, topology_id) 470 main_vnet = None # one without subprocess handler 471 for vnet_alias, vnet in obj_map.vnet_map.items(): 472 if self._get_vnet_handler(vnet_alias): 473 # Need subprocess to run 474 parent_pipe, child_pipe = Pipe() 475 p = Process( 476 target=self._setup_vnet, 477 args=( 478 vnet, 479 obj_map, 480 child_pipe, 481 ), 482 ) 483 vnet.set_pipe(parent_pipe) 484 vnet.set_subprocess(p) 485 p.start() 486 else: 487 if main_vnet is not None: 488 raise Exception("there can be only 1 VNET w/o handler") 489 main_vnet = vnet 490 # Main vnet needs to be the last, so all the other subprocesses 491 # are started & their pipe handles collected 492 self.vnet = main_vnet 493 self._setup_vnet(main_vnet, obj_map, None) 494 # Save state for the main handler 495 self.iface_map = obj_map.iface_map 496 self.vnet_map = obj_map.vnet_map 497 self.drop_privileges() 498 499 def cleanup(self, test_id: str): 500 # pytest test id: file::class::test_name 501 topology_id = get_topology_id(self.test_id) 502 503 print("============= vnet cleanup =============") 504 print("# topology_id: '{}'".format(topology_id)) 505 VnetFactory(topology_id).cleanup() 506 IfaceFactory().cleanup() 507 508 def wait_object(self, pipe, timeout=5): 509 if pipe.poll(timeout): 510 return pipe.recv() 511 raise TimeoutError 512 513 def wait_objects_any(self, pipe_list, timeout=5): 514 objects = connection.wait(pipe_list, timeout) 515 if objects: 516 return objects[0].recv() 517 raise TimeoutError 518 519 def send_object(self, pipe, obj): 520 pipe.send(obj) 521 522 def wait(self): 523 while True: 524 time.sleep(1) 525 526 @property 527 def curvnet(self): 528 pass 529 530 531class SingleVnetTestTemplate(VnetTestTemplate): 532 IPV6_PREFIXES: List[str] = [] 533 IPV4_PREFIXES: List[str] = [] 534 IFTYPE = "epair" 535 536 def _setup_default_topology(self): 537 topology = copy.deepcopy( 538 { 539 "vnet1": {"ifaces": ["if1"]}, 540 "if1": {"type": self.IFTYPE, "prefixes4": [], "prefixes6": []}, 541 } 542 ) 543 for prefix in self.IPV6_PREFIXES: 544 topology["if1"]["prefixes6"].append((prefix,)) 545 for prefix in self.IPV4_PREFIXES: 546 topology["if1"]["prefixes4"].append((prefix,)) 547 return topology 548 549 def setup_method(self, method): 550 if not getattr(self, "TOPOLOGY", None): 551 self.TOPOLOGY = self._setup_default_topology() 552 else: 553 names = self.TOPOLOGY.keys() 554 assert len([n for n in names if n.startswith("vnet")]) == 1 555 super().setup_method(method) 556