1#!/usr/local/bin/python3 2import copy 3import ipaddress 4import os 5import re 6import socket 7import sys 8import time 9from multiprocessing import connection 10from multiprocessing import Pipe 11from multiprocessing import Process 12from typing import Dict 13from typing import List 14from typing import NamedTuple 15 16from atf_python.sys.net.tools import ToolsHelper 17from atf_python.utils import BaseTest 18from atf_python.utils import libc 19 20 21def run_cmd(cmd: str, verbose=True) -> str: 22 if verbose: 23 print("run: '{}'".format(cmd)) 24 return os.popen(cmd).read() 25 26 27def get_topology_id(test_id: str) -> str: 28 """ 29 Gets a unique topology id based on the pytest test_id. 30 "test_ip6_output.py::TestIP6Output::test_output6_pktinfo[ipandif]" -> 31 "TestIP6Output:test_output6_pktinfo[ipandif]" 32 """ 33 return ":".join(test_id.split("::")[-2:]) 34 35 36def convert_test_name(test_name: str) -> str: 37 """Convert test name to a string that can be used in the file/jail names""" 38 ret = "" 39 for char in test_name: 40 if char.isalnum() or char in ("_", "-", ":"): 41 ret += char 42 elif char in ("["): 43 ret += "_" 44 return ret 45 46 47class VnetInterface(object): 48 # defines from net/if_types.h 49 IFT_LOOP = 0x18 50 IFT_ETHER = 0x06 51 52 def __init__(self, iface_alias: str, iface_name: str): 53 self.name = iface_name 54 self.alias = iface_alias 55 self.vnet_name = "" 56 self.jailed = False 57 self.addr_map: Dict[str, Dict] = {"inet6": {}, "inet": {}} 58 self.prefixes4: List[List[str]] = [] 59 self.prefixes6: List[List[str]] = [] 60 if iface_name.startswith("lo"): 61 self.iftype = self.IFT_LOOP 62 else: 63 self.iftype = self.IFT_ETHER 64 65 @property 66 def ifindex(self): 67 return socket.if_nametoindex(self.name) 68 69 @property 70 def first_ipv6(self): 71 d = self.addr_map["inet6"] 72 return d[next(iter(d))] 73 74 @property 75 def first_ipv4(self): 76 d = self.addr_map["inet"] 77 return d[next(iter(d))] 78 79 def set_vnet(self, vnet_name: str): 80 self.vnet_name = vnet_name 81 82 def set_jailed(self, jailed: bool): 83 self.jailed = jailed 84 85 def run_cmd(self, cmd, verbose=False): 86 if self.vnet_name and not self.jailed: 87 cmd = "/usr/sbin/jexec {} {}".format(self.vnet_name, cmd) 88 return run_cmd(cmd, verbose) 89 90 @classmethod 91 def setup_loopback(cls, vnet_name: str): 92 lo = VnetInterface("", "lo0") 93 lo.set_vnet(vnet_name) 94 lo.setup_addr("127.0.0.1/8") 95 lo.turn_up() 96 97 @classmethod 98 def create_iface(cls, alias_name: str, iface_name: str) -> List["VnetInterface"]: 99 name = run_cmd("/sbin/ifconfig {} create".format(iface_name)).rstrip() 100 if not name: 101 raise Exception("Unable to create iface {}".format(iface_name)) 102 ret = [cls(alias_name, name)] 103 if name.startswith("epair"): 104 ret.append(cls(alias_name, name[:-1] + "b")) 105 return ret 106 107 def setup_addr(self, _addr: str): 108 addr = ipaddress.ip_interface(_addr) 109 if addr.version == 6: 110 family = "inet6" 111 cmd = "/sbin/ifconfig {} {} {}".format(self.name, family, addr) 112 else: 113 family = "inet" 114 if self.addr_map[family]: 115 cmd = "/sbin/ifconfig {} alias {}".format(self.name, addr) 116 else: 117 cmd = "/sbin/ifconfig {} {} {}".format(self.name, family, addr) 118 self.run_cmd(cmd) 119 self.addr_map[family][str(addr.ip)] = addr 120 121 def delete_addr(self, _addr: str): 122 addr = ipaddress.ip_address(_addr) 123 if addr.version == 6: 124 family = "inet6" 125 cmd = "/sbin/ifconfig {} inet6 {} delete".format(self.name, addr) 126 else: 127 family = "inet" 128 cmd = "/sbin/ifconfig {} -alias {}".format(self.name, addr) 129 self.run_cmd(cmd) 130 del self.addr_map[family][str(addr)] 131 132 def turn_up(self): 133 cmd = "/sbin/ifconfig {} up".format(self.name) 134 self.run_cmd(cmd) 135 136 def enable_ipv6(self): 137 cmd = "/usr/sbin/ndp -i {} -disabled".format(self.name) 138 self.run_cmd(cmd) 139 140 def has_tentative(self) -> bool: 141 """True if an interface has some addresses in tenative state""" 142 cmd = "/sbin/ifconfig {} inet6".format(self.name) 143 out = self.run_cmd(cmd, verbose=False) 144 for line in out.splitlines(): 145 if "tentative" in line: 146 return True 147 return False 148 149 150class IfaceFactory(object): 151 INTERFACES_FNAME = "created_ifaces.lst" 152 AUTODELETE_TYPES = ("epair", "gif", "gre", "lo", "tap", "tun") 153 154 def __init__(self): 155 self.file_name = self.INTERFACES_FNAME 156 157 def _register_iface(self, iface_name: str): 158 with open(self.file_name, "a") as f: 159 f.write(iface_name + "\n") 160 161 def _list_ifaces(self) -> List[str]: 162 ret: List[str] = [] 163 try: 164 with open(self.file_name, "r") as f: 165 for line in f: 166 ret.append(line.strip()) 167 except OSError: 168 pass 169 return ret 170 171 def create_iface(self, alias_name: str, iface_name: str) -> List[VnetInterface]: 172 ifaces = VnetInterface.create_iface(alias_name, iface_name) 173 for iface in ifaces: 174 if not self.is_autodeleted(iface.name): 175 self._register_iface(iface.name) 176 return ifaces 177 178 @staticmethod 179 def is_autodeleted(iface_name: str) -> bool: 180 if iface_name == "lo0": 181 return False 182 iface_type = re.split(r"\d+", iface_name)[0] 183 return iface_type in IfaceFactory.AUTODELETE_TYPES 184 185 def cleanup_vnet_interfaces(self, vnet_name: str) -> List[str]: 186 """Destroys""" 187 ifaces_lst = ToolsHelper.get_output( 188 "/usr/sbin/jexec {} /sbin/ifconfig -l".format(vnet_name) 189 ) 190 for iface_name in ifaces_lst.split(): 191 if not self.is_autodeleted(iface_name): 192 if iface_name not in self._list_ifaces(): 193 print("Skipping interface {}:{}".format(vnet_name, iface_name)) 194 continue 195 run_cmd( 196 "/usr/sbin/jexec {} /sbin/ifconfig {} destroy".format(vnet_name, iface_name) 197 ) 198 199 def cleanup(self): 200 try: 201 os.unlink(self.INTERFACES_FNAME) 202 except OSError: 203 pass 204 205 206class VnetInstance(object): 207 def __init__( 208 self, vnet_alias: str, vnet_name: str, jid: int, ifaces: List[VnetInterface] 209 ): 210 self.name = vnet_name 211 self.alias = vnet_alias # reference in the test topology 212 self.jid = jid 213 self.ifaces = ifaces 214 self.iface_alias_map = {} # iface.alias: iface 215 self.iface_map = {} # iface.name: iface 216 for iface in ifaces: 217 iface.set_vnet(vnet_name) 218 iface.set_jailed(True) 219 self.iface_alias_map[iface.alias] = iface 220 self.iface_map[iface.name] = iface 221 # Allow reference to interfce aliases as attributes 222 setattr(self, iface.alias, iface) 223 self.need_dad = False # Disable duplicate address detection by default 224 self.attached = False 225 self.pipe = None 226 self.subprocess = None 227 228 def run_vnet_cmd(self, cmd, verbose=True): 229 if not self.attached: 230 cmd = "/usr/sbin/jexec {} {}".format(self.name, cmd) 231 return run_cmd(cmd, verbose) 232 233 def disable_dad(self): 234 self.run_vnet_cmd("/sbin/sysctl net.inet6.ip6.dad_count=0") 235 236 def set_pipe(self, pipe): 237 self.pipe = pipe 238 239 def set_subprocess(self, p): 240 self.subprocess = p 241 242 @staticmethod 243 def attach_jid(jid: int): 244 error_code = libc.jail_attach(jid) 245 if error_code != 0: 246 raise Exception("jail_attach() failed: errno {}".format(error_code)) 247 248 def attach(self): 249 self.attach_jid(self.jid) 250 self.attached = True 251 252 253class VnetFactory(object): 254 JAILS_FNAME = "created_jails.lst" 255 256 def __init__(self, topology_id: str): 257 self.topology_id = topology_id 258 self.file_name = self.JAILS_FNAME 259 self._vnets: List[str] = [] 260 261 def _register_vnet(self, vnet_name: str): 262 self._vnets.append(vnet_name) 263 with open(self.file_name, "a") as f: 264 f.write(vnet_name + "\n") 265 266 @staticmethod 267 def _wait_interfaces(vnet_name: str, ifaces: List[str]) -> List[str]: 268 cmd = "/usr/sbin/jexec {} /sbin/ifconfig -l".format(vnet_name) 269 not_matched: List[str] = [] 270 for i in range(50): 271 vnet_ifaces = run_cmd(cmd).strip().split(" ") 272 not_matched = [] 273 for iface_name in ifaces: 274 if iface_name not in vnet_ifaces: 275 not_matched.append(iface_name) 276 if len(not_matched) == 0: 277 return [] 278 time.sleep(0.1) 279 return not_matched 280 281 def create_vnet(self, vnet_alias: str, ifaces: List[VnetInterface]): 282 vnet_name = "pytest:{}".format(convert_test_name(self.topology_id)) 283 if self._vnets: 284 # add number to distinguish jails 285 vnet_name = "{}_{}".format(vnet_name, len(self._vnets) + 1) 286 iface_cmds = " ".join(["vnet.interface={}".format(i.name) for i in ifaces]) 287 cmd = "/usr/sbin/jail -i -c name={} persist vnet {}".format( 288 vnet_name, iface_cmds 289 ) 290 jid = 0 291 try: 292 jid_str = run_cmd(cmd) 293 jid = int(jid_str) 294 except ValueError: 295 print("Jail creation failed, output: {}".format(jid_str)) 296 raise 297 self._register_vnet(vnet_name) 298 299 # Run expedited version of routing 300 VnetInterface.setup_loopback(vnet_name) 301 302 not_found = self._wait_interfaces(vnet_name, [i.name for i in ifaces]) 303 if not_found: 304 raise Exception( 305 "Interfaces {} has not appeared in vnet {}".format(not_found, vnet_name) 306 ) 307 return VnetInstance(vnet_alias, vnet_name, jid, ifaces) 308 309 def cleanup(self): 310 iface_factory = IfaceFactory() 311 try: 312 with open(self.file_name) as f: 313 for line in f: 314 vnet_name = line.strip() 315 iface_factory.cleanup_vnet_interfaces(vnet_name) 316 run_cmd("/usr/sbin/jail -r {}".format(vnet_name)) 317 os.unlink(self.JAILS_FNAME) 318 except OSError: 319 pass 320 321 322class SingleInterfaceMap(NamedTuple): 323 ifaces: List[VnetInterface] 324 vnet_aliases: List[str] 325 326 327class ObjectsMap(NamedTuple): 328 iface_map: Dict[str, SingleInterfaceMap] # keyed by ifX 329 vnet_map: Dict[str, VnetInstance] # keyed by vnetX 330 topo_map: Dict # self.TOPOLOGY 331 332 333class VnetTestTemplate(BaseTest): 334 NEED_ROOT: bool = True 335 TOPOLOGY = {} 336 337 def _require_default_modules(self): 338 libc.kldload("if_epair.ko") 339 self.require_module("if_epair") 340 341 def _get_vnet_handler(self, vnet_alias: str): 342 handler_name = "{}_handler".format(vnet_alias) 343 return getattr(self, handler_name, None) 344 345 def _setup_vnet(self, vnet: VnetInstance, obj_map: Dict, pipe): 346 """Base Handler to setup given VNET. 347 Can be run in a subprocess. If so, passes control to the special 348 vnetX_handler() after setting up interface addresses 349 """ 350 vnet.attach() 351 print("# setup_vnet({})".format(vnet.name)) 352 if pipe is not None: 353 vnet.set_pipe(pipe) 354 355 topo = obj_map.topo_map 356 ipv6_ifaces = [] 357 # Disable DAD 358 if not vnet.need_dad: 359 vnet.disable_dad() 360 for iface in vnet.ifaces: 361 # check index of vnet within an interface 362 # as we have prefixes for both ends of the interface 363 iface_map = obj_map.iface_map[iface.alias] 364 idx = iface_map.vnet_aliases.index(vnet.alias) 365 prefixes6 = topo[iface.alias].get("prefixes6", []) 366 prefixes4 = topo[iface.alias].get("prefixes4", []) 367 if prefixes6 or prefixes4: 368 ipv6_ifaces.append(iface) 369 iface.turn_up() 370 if prefixes6: 371 iface.enable_ipv6() 372 for prefix in prefixes6 + prefixes4: 373 if prefix[idx]: 374 iface.setup_addr(prefix[idx]) 375 for iface in ipv6_ifaces: 376 while iface.has_tentative(): 377 time.sleep(0.1) 378 379 # Run actual handler 380 handler = self._get_vnet_handler(vnet.alias) 381 if handler: 382 # Do unbuffered stdout for children 383 # so the logs are present if the child hangs 384 sys.stdout.reconfigure(line_buffering=True) 385 self.drop_privileges() 386 handler(vnet) 387 388 def _get_topo_ifmap(self, topo: Dict): 389 iface_factory = IfaceFactory() 390 iface_map: Dict[str, SingleInterfaceMap] = {} 391 iface_aliases = set() 392 for obj_name, obj_data in topo.items(): 393 if obj_name.startswith("vnet"): 394 for iface_alias in obj_data["ifaces"]: 395 iface_aliases.add(iface_alias) 396 for iface_alias in iface_aliases: 397 print("Creating {}".format(iface_alias)) 398 iface_data = topo[iface_alias] 399 iface_type = iface_data.get("type", "epair") 400 ifaces = iface_factory.create_iface(iface_alias, iface_type) 401 smap = SingleInterfaceMap(ifaces, []) 402 iface_map[iface_alias] = smap 403 return iface_map 404 405 def setup_topology(self, topo: Dict, topology_id: str): 406 """Creates jails & interfaces for the provided topology""" 407 vnet_map = {} 408 vnet_factory = VnetFactory(topology_id) 409 iface_map = self._get_topo_ifmap(topo) 410 for obj_name, obj_data in topo.items(): 411 if obj_name.startswith("vnet"): 412 vnet_ifaces = [] 413 for iface_alias in obj_data["ifaces"]: 414 # epair creates 2 interfaces, grab first _available_ 415 # and map it to the VNET being created 416 idx = len(iface_map[iface_alias].vnet_aliases) 417 iface_map[iface_alias].vnet_aliases.append(obj_name) 418 vnet_ifaces.append(iface_map[iface_alias].ifaces[idx]) 419 vnet = vnet_factory.create_vnet(obj_name, vnet_ifaces) 420 vnet_map[obj_name] = vnet 421 # Allow reference to VNETs as attributes 422 setattr(self, obj_name, vnet) 423 # Debug output 424 print("============= TEST TOPOLOGY =============") 425 for vnet_alias, vnet in vnet_map.items(): 426 print("# vnet {} -> {}".format(vnet.alias, vnet.name), end="") 427 handler = self._get_vnet_handler(vnet.alias) 428 if handler: 429 print(" handler: {}".format(handler.__name__), end="") 430 print() 431 for iface_alias, iface_data in iface_map.items(): 432 vnets = iface_data.vnet_aliases 433 ifaces: List[VnetInterface] = iface_data.ifaces 434 if len(vnets) == 1 and len(ifaces) == 2: 435 print( 436 "# iface {}: {}::{} -> main::{}".format( 437 iface_alias, vnets[0], ifaces[0].name, ifaces[1].name 438 ) 439 ) 440 elif len(vnets) == 2 and len(ifaces) == 2: 441 print( 442 "# iface {}: {}::{} -> {}::{}".format( 443 iface_alias, vnets[0], ifaces[0].name, vnets[1], ifaces[1].name 444 ) 445 ) 446 else: 447 print( 448 "# iface {}: ifaces: {} vnets: {}".format( 449 iface_alias, vnets, [i.name for i in ifaces] 450 ) 451 ) 452 print() 453 return ObjectsMap(iface_map, vnet_map, topo) 454 455 def setup_method(self, _method): 456 """Sets up all the required topology and handlers for the given test""" 457 super().setup_method(_method) 458 self._require_default_modules() 459 460 # TestIP6Output.test_output6_pktinfo[ipandif] 461 topology_id = get_topology_id(self.test_id) 462 topology = self.TOPOLOGY 463 # First, setup kernel objects - interfaces & vnets 464 obj_map = self.setup_topology(topology, topology_id) 465 main_vnet = None # one without subprocess handler 466 for vnet_alias, vnet in obj_map.vnet_map.items(): 467 if self._get_vnet_handler(vnet_alias): 468 # Need subprocess to run 469 parent_pipe, child_pipe = Pipe() 470 p = Process( 471 target=self._setup_vnet, 472 args=( 473 vnet, 474 obj_map, 475 child_pipe, 476 ), 477 ) 478 vnet.set_pipe(parent_pipe) 479 vnet.set_subprocess(p) 480 p.start() 481 else: 482 if main_vnet is not None: 483 raise Exception("there can be only 1 VNET w/o handler") 484 main_vnet = vnet 485 # Main vnet needs to be the last, so all the other subprocesses 486 # are started & their pipe handles collected 487 self.vnet = main_vnet 488 self._setup_vnet(main_vnet, obj_map, None) 489 # Save state for the main handler 490 self.iface_map = obj_map.iface_map 491 self.vnet_map = obj_map.vnet_map 492 self.drop_privileges() 493 494 def cleanup(self, test_id: str): 495 # pytest test id: file::class::test_name 496 topology_id = get_topology_id(self.test_id) 497 498 print("============= vnet cleanup =============") 499 print("# topology_id: '{}'".format(topology_id)) 500 VnetFactory(topology_id).cleanup() 501 IfaceFactory().cleanup() 502 503 def wait_object(self, pipe, timeout=5): 504 if pipe.poll(timeout): 505 return pipe.recv() 506 raise TimeoutError 507 508 def wait_objects_any(self, pipe_list, timeout=5): 509 objects = connection.wait(pipe_list, timeout) 510 if objects: 511 return objects[0].recv() 512 raise TimeoutError 513 514 def send_object(self, pipe, obj): 515 pipe.send(obj) 516 517 def wait(self): 518 while True: 519 time.sleep(1) 520 521 @property 522 def curvnet(self): 523 pass 524 525 526class SingleVnetTestTemplate(VnetTestTemplate): 527 IPV6_PREFIXES: List[str] = [] 528 IPV4_PREFIXES: List[str] = [] 529 IFTYPE = "epair" 530 531 def _setup_default_topology(self): 532 topology = copy.deepcopy( 533 { 534 "vnet1": {"ifaces": ["if1"]}, 535 "if1": {"type": self.IFTYPE, "prefixes4": [], "prefixes6": []}, 536 } 537 ) 538 for prefix in self.IPV6_PREFIXES: 539 topology["if1"]["prefixes6"].append((prefix,)) 540 for prefix in self.IPV4_PREFIXES: 541 topology["if1"]["prefixes4"].append((prefix,)) 542 return topology 543 544 def setup_method(self, method): 545 if not getattr(self, "TOPOLOGY", None): 546 self.TOPOLOGY = self._setup_default_topology() 547 else: 548 names = self.TOPOLOGY.keys() 549 assert len([n for n in names if n.startswith("vnet")]) == 1 550 super().setup_method(method) 551