1#!/usr/local/bin/python3 2import copy 3import ipaddress 4import os 5import re 6import socket 7import sys 8import time 9from multiprocessing import connection 10from multiprocessing import Pipe 11from multiprocessing import Process 12from typing import Dict 13from typing import List 14from typing import NamedTuple 15 16from atf_python.sys.net.tools import ToolsHelper 17from atf_python.utils import BaseTest 18from atf_python.utils import libc 19 20 21def run_cmd(cmd: str, verbose=True) -> str: 22 if verbose: 23 print("run: '{}'".format(cmd)) 24 return os.popen(cmd).read() 25 26 27def get_topology_id(test_id: str) -> str: 28 """ 29 Gets a unique topology id based on the pytest test_id. 30 "test_ip6_output.py::TestIP6Output::test_output6_pktinfo[ipandif]" -> 31 "TestIP6Output:test_output6_pktinfo[ipandif]" 32 """ 33 return ":".join(test_id.split("::")[-2:]) 34 35 36def convert_test_name(test_name: str) -> str: 37 """Convert test name to a string that can be used in the file/jail names""" 38 ret = "" 39 for char in test_name: 40 if char.isalnum() or char in ("_", "-", ":"): 41 ret += char 42 elif char in ("["): 43 ret += "_" 44 return ret 45 46 47class VnetInterface(object): 48 # defines from net/if_types.h 49 IFT_LOOP = 0x18 50 IFT_ETHER = 0x06 51 52 def __init__(self, iface_alias: str, iface_name: str): 53 self.name = iface_name 54 self.alias = iface_alias 55 self.vnet_name = "" 56 self.jailed = False 57 self.addr_map: Dict[str, Dict] = {"inet6": {}, "inet": {}} 58 self.prefixes4: List[List[str]] = [] 59 self.prefixes6: List[List[str]] = [] 60 if iface_name.startswith("lo"): 61 self.iftype = self.IFT_LOOP 62 else: 63 self.iftype = self.IFT_ETHER 64 self.ether = ToolsHelper.get_output("/sbin/ifconfig %s ether | awk '/ether/ { print $2; }'" % iface_name).rstrip() 65 66 @property 67 def ifindex(self): 68 return socket.if_nametoindex(self.name) 69 70 @property 71 def first_ipv6(self): 72 d = self.addr_map["inet6"] 73 return d[next(iter(d))] 74 75 @property 76 def first_ipv4(self): 77 d = self.addr_map["inet"] 78 return d[next(iter(d))] 79 80 def set_vnet(self, vnet_name: str): 81 self.vnet_name = vnet_name 82 83 def set_jailed(self, jailed: bool): 84 self.jailed = jailed 85 86 def run_cmd(self, cmd, verbose=False): 87 if self.vnet_name and not self.jailed: 88 cmd = "/usr/sbin/jexec {} {}".format(self.vnet_name, cmd) 89 return run_cmd(cmd, verbose) 90 91 @classmethod 92 def setup_loopback(cls, vnet_name: str): 93 lo = VnetInterface("", "lo0") 94 lo.set_vnet(vnet_name) 95 lo.setup_addr("127.0.0.1/8") 96 lo.turn_up() 97 98 @classmethod 99 def create_iface(cls, alias_name: str, iface_name: str) -> List["VnetInterface"]: 100 name = run_cmd("/sbin/ifconfig {} create".format(iface_name)).rstrip() 101 if not name: 102 raise Exception("Unable to create iface {}".format(iface_name)) 103 if1 = cls(alias_name, name) 104 ret = [if1] 105 if name.startswith("epair"): 106 if2 = cls(alias_name, name[:-1] + "b") 107 if1.epairb = if2 108 ret.append(if2); 109 return ret 110 111 def setup_addr(self, _addr: str): 112 addr = ipaddress.ip_interface(_addr) 113 if addr.version == 6: 114 family = "inet6" 115 cmd = "/sbin/ifconfig {} {} {}".format(self.name, family, addr) 116 else: 117 family = "inet" 118 if self.addr_map[family]: 119 cmd = "/sbin/ifconfig {} alias {}".format(self.name, addr) 120 else: 121 cmd = "/sbin/ifconfig {} {} {}".format(self.name, family, addr) 122 self.run_cmd(cmd) 123 self.addr_map[family][str(addr.ip)] = addr 124 125 def delete_addr(self, _addr: str): 126 addr = ipaddress.ip_address(_addr) 127 if addr.version == 6: 128 family = "inet6" 129 cmd = "/sbin/ifconfig {} inet6 {} delete".format(self.name, addr) 130 else: 131 family = "inet" 132 cmd = "/sbin/ifconfig {} -alias {}".format(self.name, addr) 133 self.run_cmd(cmd) 134 del self.addr_map[family][str(addr)] 135 136 def turn_up(self): 137 cmd = "/sbin/ifconfig {} up".format(self.name) 138 self.run_cmd(cmd) 139 140 def enable_ipv6(self): 141 cmd = "/usr/sbin/ndp -i {} -- -disabled".format(self.name) 142 self.run_cmd(cmd) 143 144 def has_tentative(self) -> bool: 145 """True if an interface has some addresses in tenative state""" 146 cmd = "/sbin/ifconfig {} inet6".format(self.name) 147 out = self.run_cmd(cmd, verbose=False) 148 for line in out.splitlines(): 149 if "tentative" in line: 150 return True 151 return False 152 153 154class IfaceFactory(object): 155 INTERFACES_FNAME = "created_ifaces.lst" 156 AUTODELETE_TYPES = ("epair", "gif", "gre", "lo", "tap", "tun") 157 158 def __init__(self): 159 self.file_name = self.INTERFACES_FNAME 160 161 def _register_iface(self, iface_name: str): 162 with open(self.file_name, "a") as f: 163 f.write(iface_name + "\n") 164 165 def _list_ifaces(self) -> List[str]: 166 ret: List[str] = [] 167 try: 168 with open(self.file_name, "r") as f: 169 for line in f: 170 ret.append(line.strip()) 171 except OSError: 172 pass 173 return ret 174 175 def create_iface(self, alias_name: str, iface_name: str) -> List[VnetInterface]: 176 ifaces = VnetInterface.create_iface(alias_name, iface_name) 177 for iface in ifaces: 178 if not self.is_autodeleted(iface.name): 179 self._register_iface(iface.name) 180 return ifaces 181 182 @staticmethod 183 def is_autodeleted(iface_name: str) -> bool: 184 if iface_name == "lo0": 185 return False 186 iface_type = re.split(r"\d+", iface_name)[0] 187 return iface_type in IfaceFactory.AUTODELETE_TYPES 188 189 def cleanup_vnet_interfaces(self, vnet_name: str) -> List[str]: 190 """Destroys""" 191 ifaces_lst = ToolsHelper.get_output( 192 "/usr/sbin/jexec {} /sbin/ifconfig -l".format(vnet_name) 193 ) 194 for iface_name in ifaces_lst.split(): 195 if not self.is_autodeleted(iface_name): 196 if iface_name not in self._list_ifaces(): 197 print("Skipping interface {}:{}".format(vnet_name, iface_name)) 198 continue 199 run_cmd( 200 "/usr/sbin/jexec {} /sbin/ifconfig {} destroy".format(vnet_name, iface_name) 201 ) 202 203 def cleanup(self): 204 try: 205 os.unlink(self.INTERFACES_FNAME) 206 except OSError: 207 pass 208 209 210class VnetInstance(object): 211 def __init__( 212 self, vnet_alias: str, vnet_name: str, jid: int, ifaces: List[VnetInterface] 213 ): 214 self.name = vnet_name 215 self.alias = vnet_alias # reference in the test topology 216 self.jid = jid 217 self.ifaces = ifaces 218 self.iface_alias_map = {} # iface.alias: iface 219 self.iface_map = {} # iface.name: iface 220 for iface in ifaces: 221 iface.set_vnet(vnet_name) 222 iface.set_jailed(True) 223 self.iface_alias_map[iface.alias] = iface 224 self.iface_map[iface.name] = iface 225 # Allow reference to interfce aliases as attributes 226 setattr(self, iface.alias, iface) 227 self.need_dad = False # Disable duplicate address detection by default 228 self.attached = False 229 self.pipe = None 230 self.subprocess = None 231 232 def run_vnet_cmd(self, cmd, verbose=True): 233 if not self.attached: 234 cmd = "/usr/sbin/jexec {} {}".format(self.name, cmd) 235 return run_cmd(cmd, verbose) 236 237 def disable_dad(self): 238 self.run_vnet_cmd("/sbin/sysctl net.inet6.ip6.dad_count=0") 239 240 def set_pipe(self, pipe): 241 self.pipe = pipe 242 243 def set_subprocess(self, p): 244 self.subprocess = p 245 246 @staticmethod 247 def attach_jid(jid: int): 248 error_code = libc.jail_attach(jid) 249 if error_code != 0: 250 raise Exception("jail_attach() failed: errno {}".format(error_code)) 251 252 def attach(self): 253 self.attach_jid(self.jid) 254 self.attached = True 255 256 257class VnetFactory(object): 258 JAILS_FNAME = "created_jails.lst" 259 260 def __init__(self, topology_id: str): 261 self.topology_id = topology_id 262 self.file_name = self.JAILS_FNAME 263 self._vnets: List[str] = [] 264 265 def _register_vnet(self, vnet_name: str): 266 self._vnets.append(vnet_name) 267 with open(self.file_name, "a") as f: 268 f.write(vnet_name + "\n") 269 270 @staticmethod 271 def _wait_interfaces(vnet_name: str, ifaces: List[str]) -> List[str]: 272 cmd = "/usr/sbin/jexec {} /sbin/ifconfig -l".format(vnet_name) 273 not_matched: List[str] = [] 274 for i in range(50): 275 vnet_ifaces = run_cmd(cmd).strip().split(" ") 276 not_matched = [] 277 for iface_name in ifaces: 278 if iface_name not in vnet_ifaces: 279 not_matched.append(iface_name) 280 if len(not_matched) == 0: 281 return [] 282 time.sleep(0.1) 283 return not_matched 284 285 def create_vnet(self, vnet_alias: str, ifaces: List[VnetInterface]): 286 vnet_name = "pytest:{}".format(convert_test_name(self.topology_id)) 287 if self._vnets: 288 # add number to distinguish jails 289 vnet_name = "{}_{}".format(vnet_name, len(self._vnets) + 1) 290 iface_cmds = " ".join(["vnet.interface={}".format(i.name) for i in ifaces]) 291 cmd = "/usr/sbin/jail -i -c name={} persist vnet {}".format( 292 vnet_name, iface_cmds 293 ) 294 jid = 0 295 try: 296 jid_str = run_cmd(cmd) 297 jid = int(jid_str) 298 except ValueError: 299 print("Jail creation failed, output: {}".format(jid_str)) 300 raise 301 self._register_vnet(vnet_name) 302 303 # Run expedited version of routing 304 VnetInterface.setup_loopback(vnet_name) 305 306 not_found = self._wait_interfaces(vnet_name, [i.name for i in ifaces]) 307 if not_found: 308 raise Exception( 309 "Interfaces {} has not appeared in vnet {}".format(not_found, vnet_name) 310 ) 311 return VnetInstance(vnet_alias, vnet_name, jid, ifaces) 312 313 def cleanup(self): 314 iface_factory = IfaceFactory() 315 try: 316 with open(self.file_name) as f: 317 for line in f: 318 vnet_name = line.strip() 319 iface_factory.cleanup_vnet_interfaces(vnet_name) 320 run_cmd("/usr/sbin/jail -r {}".format(vnet_name)) 321 os.unlink(self.JAILS_FNAME) 322 except OSError: 323 pass 324 325 326class SingleInterfaceMap(NamedTuple): 327 ifaces: List[VnetInterface] 328 vnet_aliases: List[str] 329 330 331class ObjectsMap(NamedTuple): 332 iface_map: Dict[str, SingleInterfaceMap] # keyed by ifX 333 vnet_map: Dict[str, VnetInstance] # keyed by vnetX 334 topo_map: Dict # self.TOPOLOGY 335 336 337class VnetTestTemplate(BaseTest): 338 NEED_ROOT: bool = True 339 TOPOLOGY = {} 340 341 def _require_default_modules(self): 342 libc.kldload("if_epair.ko") 343 self.require_module("if_epair") 344 345 def _get_vnet_handler(self, vnet_alias: str): 346 handler_name = "{}_handler".format(vnet_alias) 347 return getattr(self, handler_name, None) 348 349 def _setup_vnet(self, vnet: VnetInstance, obj_map: Dict, pipe): 350 """Base Handler to setup given VNET. 351 Can be run in a subprocess. If so, passes control to the special 352 vnetX_handler() after setting up interface addresses 353 """ 354 vnet.attach() 355 print("# setup_vnet({})".format(vnet.name)) 356 if pipe is not None: 357 vnet.set_pipe(pipe) 358 359 topo = obj_map.topo_map 360 ipv6_ifaces = [] 361 # Disable DAD 362 if not vnet.need_dad: 363 vnet.disable_dad() 364 for iface in vnet.ifaces: 365 # check index of vnet within an interface 366 # as we have prefixes for both ends of the interface 367 iface_map = obj_map.iface_map[iface.alias] 368 idx = iface_map.vnet_aliases.index(vnet.alias) 369 prefixes6 = topo[iface.alias].get("prefixes6", []) 370 prefixes4 = topo[iface.alias].get("prefixes4", []) 371 if prefixes6 or prefixes4: 372 ipv6_ifaces.append(iface) 373 iface.turn_up() 374 if prefixes6: 375 iface.enable_ipv6() 376 for prefix in prefixes6 + prefixes4: 377 if prefix[idx]: 378 iface.setup_addr(prefix[idx]) 379 for iface in ipv6_ifaces: 380 while iface.has_tentative(): 381 time.sleep(0.1) 382 383 # Run actual handler 384 handler = self._get_vnet_handler(vnet.alias) 385 if handler: 386 # Do unbuffered stdout for children 387 # so the logs are present if the child hangs 388 sys.stdout.reconfigure(line_buffering=True) 389 self.drop_privileges() 390 handler(vnet) 391 392 def _get_topo_ifmap(self, topo: Dict): 393 iface_factory = IfaceFactory() 394 iface_map: Dict[str, SingleInterfaceMap] = {} 395 iface_aliases = set() 396 for obj_name, obj_data in topo.items(): 397 if obj_name.startswith("vnet"): 398 for iface_alias in obj_data["ifaces"]: 399 iface_aliases.add(iface_alias) 400 for iface_alias in iface_aliases: 401 print("Creating {}".format(iface_alias)) 402 iface_data = topo[iface_alias] 403 iface_type = iface_data.get("type", "epair") 404 ifaces = iface_factory.create_iface(iface_alias, iface_type) 405 smap = SingleInterfaceMap(ifaces, []) 406 iface_map[iface_alias] = smap 407 return iface_map 408 409 def setup_topology(self, topo: Dict, topology_id: str): 410 """Creates jails & interfaces for the provided topology""" 411 vnet_map = {} 412 vnet_factory = VnetFactory(topology_id) 413 iface_map = self._get_topo_ifmap(topo) 414 for obj_name, obj_data in topo.items(): 415 if obj_name.startswith("vnet"): 416 vnet_ifaces = [] 417 for iface_alias in obj_data["ifaces"]: 418 # epair creates 2 interfaces, grab first _available_ 419 # and map it to the VNET being created 420 idx = len(iface_map[iface_alias].vnet_aliases) 421 iface_map[iface_alias].vnet_aliases.append(obj_name) 422 vnet_ifaces.append(iface_map[iface_alias].ifaces[idx]) 423 vnet = vnet_factory.create_vnet(obj_name, vnet_ifaces) 424 vnet_map[obj_name] = vnet 425 # Allow reference to VNETs as attributes 426 setattr(self, obj_name, vnet) 427 # Debug output 428 print("============= TEST TOPOLOGY =============") 429 for vnet_alias, vnet in vnet_map.items(): 430 print("# vnet {} -> {}".format(vnet.alias, vnet.name), end="") 431 handler = self._get_vnet_handler(vnet.alias) 432 if handler: 433 print(" handler: {}".format(handler.__name__), end="") 434 print() 435 for iface_alias, iface_data in iface_map.items(): 436 vnets = iface_data.vnet_aliases 437 ifaces: List[VnetInterface] = iface_data.ifaces 438 if len(vnets) == 1 and len(ifaces) == 2: 439 print( 440 "# iface {}: {}::{} -> main::{}".format( 441 iface_alias, vnets[0], ifaces[0].name, ifaces[1].name 442 ) 443 ) 444 elif len(vnets) == 2 and len(ifaces) == 2: 445 print( 446 "# iface {}: {}::{} -> {}::{}".format( 447 iface_alias, vnets[0], ifaces[0].name, vnets[1], ifaces[1].name 448 ) 449 ) 450 else: 451 print( 452 "# iface {}: ifaces: {} vnets: {}".format( 453 iface_alias, vnets, [i.name for i in ifaces] 454 ) 455 ) 456 print() 457 return ObjectsMap(iface_map, vnet_map, topo) 458 459 def setup_method(self, _method): 460 """Sets up all the required topology and handlers for the given test""" 461 super().setup_method(_method) 462 self._require_default_modules() 463 464 # TestIP6Output.test_output6_pktinfo[ipandif] 465 topology_id = get_topology_id(self.test_id) 466 topology = self.TOPOLOGY 467 # First, setup kernel objects - interfaces & vnets 468 obj_map = self.setup_topology(topology, topology_id) 469 main_vnet = None # one without subprocess handler 470 for vnet_alias, vnet in obj_map.vnet_map.items(): 471 if self._get_vnet_handler(vnet_alias): 472 # Need subprocess to run 473 parent_pipe, child_pipe = Pipe() 474 p = Process( 475 target=self._setup_vnet, 476 args=( 477 vnet, 478 obj_map, 479 child_pipe, 480 ), 481 ) 482 vnet.set_pipe(parent_pipe) 483 vnet.set_subprocess(p) 484 p.start() 485 else: 486 if main_vnet is not None: 487 raise Exception("there can be only 1 VNET w/o handler") 488 main_vnet = vnet 489 # Main vnet needs to be the last, so all the other subprocesses 490 # are started & their pipe handles collected 491 self.vnet = main_vnet 492 self._setup_vnet(main_vnet, obj_map, None) 493 # Save state for the main handler 494 self.iface_map = obj_map.iface_map 495 self.vnet_map = obj_map.vnet_map 496 self.drop_privileges() 497 498 def cleanup(self, test_id: str): 499 # pytest test id: file::class::test_name 500 topology_id = get_topology_id(self.test_id) 501 502 print("============= vnet cleanup =============") 503 print("# topology_id: '{}'".format(topology_id)) 504 VnetFactory(topology_id).cleanup() 505 IfaceFactory().cleanup() 506 507 def wait_object(self, pipe, timeout=5): 508 if pipe.poll(timeout): 509 return pipe.recv() 510 raise TimeoutError 511 512 def wait_objects_any(self, pipe_list, timeout=5): 513 objects = connection.wait(pipe_list, timeout) 514 if objects: 515 return objects[0].recv() 516 raise TimeoutError 517 518 def send_object(self, pipe, obj): 519 pipe.send(obj) 520 521 def wait(self): 522 while True: 523 time.sleep(1) 524 525 @property 526 def curvnet(self): 527 pass 528 529 530class SingleVnetTestTemplate(VnetTestTemplate): 531 IPV6_PREFIXES: List[str] = [] 532 IPV4_PREFIXES: List[str] = [] 533 IFTYPE = "epair" 534 535 def _setup_default_topology(self): 536 topology = copy.deepcopy( 537 { 538 "vnet1": {"ifaces": ["if1"]}, 539 "if1": {"type": self.IFTYPE, "prefixes4": [], "prefixes6": []}, 540 } 541 ) 542 for prefix in self.IPV6_PREFIXES: 543 topology["if1"]["prefixes6"].append((prefix,)) 544 for prefix in self.IPV4_PREFIXES: 545 topology["if1"]["prefixes4"].append((prefix,)) 546 return topology 547 548 def setup_method(self, method): 549 if not getattr(self, "TOPOLOGY", None): 550 self.TOPOLOGY = self._setup_default_topology() 551 else: 552 names = self.TOPOLOGY.keys() 553 assert len([n for n in names if n.startswith("vnet")]) == 1 554 super().setup_method(method) 555