1#!/usr/local/bin/python3 2import copy 3import ipaddress 4import os 5import re 6import socket 7import sys 8import time 9from multiprocessing import connection 10from multiprocessing import Pipe 11from multiprocessing import Process 12from typing import Dict 13from typing import List 14from typing import NamedTuple 15 16from atf_python.sys.net.tools import ToolsHelper 17from atf_python.utils import BaseTest 18from atf_python.utils import libc 19 20 21def run_cmd(cmd: str, verbose=True) -> str: 22 if verbose: 23 print("run: '{}'".format(cmd)) 24 return os.popen(cmd).read() 25 26 27def get_topology_id(test_id: str) -> str: 28 """ 29 Gets a unique topology id based on the pytest test_id. 30 "test_ip6_output.py::TestIP6Output::test_output6_pktinfo[ipandif]" -> 31 "TestIP6Output:test_output6_pktinfo[ipandif]" 32 """ 33 return ":".join(test_id.split("::")[-2:]) 34 35 36def convert_test_name(test_name: str) -> str: 37 """Convert test name to a string that can be used in the file/jail names""" 38 ret = "" 39 for char in test_name: 40 if char.isalnum() or char in ("_", "-", ":"): 41 ret += char 42 elif char in ("["): 43 ret += "_" 44 return ret 45 46 47class VnetInterface(object): 48 # defines from net/if_types.h 49 IFT_LOOP = 0x18 50 IFT_ETHER = 0x06 51 52 def __init__(self, iface_alias: str, iface_name: str): 53 self.name = iface_name 54 self.alias = iface_alias 55 self.vnet_name = "" 56 self.jailed = False 57 self.addr_map: Dict[str, Dict] = {"inet6": {}, "inet": {}} 58 self.prefixes4: List[List[str]] = [] 59 self.prefixes6: List[List[str]] = [] 60 if iface_name.startswith("lo"): 61 self.iftype = self.IFT_LOOP 62 else: 63 self.iftype = self.IFT_ETHER 64 65 @property 66 def ifindex(self): 67 return socket.if_nametoindex(self.name) 68 69 @property 70 def first_ipv6(self): 71 d = self.addr_map["inet6"] 72 return d[next(iter(d))] 73 74 @property 75 def first_ipv4(self): 76 d = self.addr_map["inet"] 77 return d[next(iter(d))] 78 79 def set_vnet(self, vnet_name: str): 80 self.vnet_name = vnet_name 81 82 def set_jailed(self, jailed: bool): 83 self.jailed = jailed 84 85 def run_cmd(self, cmd, verbose=False): 86 if self.vnet_name and not self.jailed: 87 cmd = "/usr/sbin/jexec {} {}".format(self.vnet_name, cmd) 88 return run_cmd(cmd, verbose) 89 90 @classmethod 91 def setup_loopback(cls, vnet_name: str): 92 lo = VnetInterface("", "lo0") 93 lo.set_vnet(vnet_name) 94 lo.setup_addr("127.0.0.1/8") 95 lo.turn_up() 96 97 @classmethod 98 def create_iface(cls, alias_name: str, iface_name: str) -> List["VnetInterface"]: 99 name = run_cmd("/sbin/ifconfig {} create".format(iface_name)).rstrip() 100 if not name: 101 raise Exception("Unable to create iface {}".format(iface_name)) 102 ret = [cls(alias_name, name)] 103 if name.startswith("epair"): 104 ret.append(cls(alias_name, name[:-1] + "b")) 105 return ret 106 107 def setup_addr(self, _addr: str): 108 addr = ipaddress.ip_interface(_addr) 109 if addr.version == 6: 110 family = "inet6" 111 cmd = "/sbin/ifconfig {} {} {}".format(self.name, family, addr) 112 else: 113 family = "inet" 114 if self.addr_map[family]: 115 cmd = "/sbin/ifconfig {} alias {}".format(self.name, addr) 116 else: 117 cmd = "/sbin/ifconfig {} {} {}".format(self.name, family, addr) 118 self.run_cmd(cmd) 119 self.addr_map[family][str(addr.ip)] = addr 120 121 def delete_addr(self, _addr: str): 122 addr = ipaddress.ip_address(_addr) 123 if addr.version == 6: 124 family = "inet6" 125 cmd = "/sbin/ifconfig {} inet6 {} delete".format(self.name, addr) 126 else: 127 family = "inet" 128 cmd = "/sbin/ifconfig {} -alias {}".format(self.name, addr) 129 self.run_cmd(cmd) 130 del self.addr_map[family][str(addr)] 131 132 def turn_up(self): 133 cmd = "/sbin/ifconfig {} up".format(self.name) 134 self.run_cmd(cmd) 135 136 def enable_ipv6(self): 137 cmd = "/usr/sbin/ndp -i {} -disabled".format(self.name) 138 self.run_cmd(cmd) 139 140 def has_tentative(self) -> bool: 141 """True if an interface has some addresses in tenative state""" 142 cmd = "/sbin/ifconfig {} inet6".format(self.name) 143 out = self.run_cmd(cmd, verbose=False) 144 for line in out.splitlines(): 145 if "tentative" in line: 146 return True 147 return False 148 149 150class IfaceFactory(object): 151 INTERFACES_FNAME = "created_ifaces.lst" 152 AUTODELETE_TYPES = ("epair", "gif", "gre", "lo", "tap", "tun") 153 154 def __init__(self): 155 self.file_name = self.INTERFACES_FNAME 156 157 def _register_iface(self, iface_name: str): 158 with open(self.file_name, "a") as f: 159 f.write(iface_name + "\n") 160 161 def _list_ifaces(self) -> List[str]: 162 ret: List[str] = [] 163 try: 164 with open(self.file_name, "r") as f: 165 for line in f: 166 ret.append(line.strip()) 167 except OSError: 168 pass 169 return ret 170 171 def create_iface(self, alias_name: str, iface_name: str) -> List[VnetInterface]: 172 ifaces = VnetInterface.create_iface(alias_name, iface_name) 173 for iface in ifaces: 174 if not self.is_autodeleted(iface.name): 175 self._register_iface(iface.name) 176 return ifaces 177 178 @staticmethod 179 def is_autodeleted(iface_name: str) -> bool: 180 if iface_name == "lo0": 181 return False 182 iface_type = re.split(r"\d+", iface_name)[0] 183 return iface_type in IfaceFactory.AUTODELETE_TYPES 184 185 def cleanup_vnet_interfaces(self, vnet_name: str) -> List[str]: 186 """Destroys""" 187 ifaces_lst = ToolsHelper.get_output( 188 "/usr/sbin/jexec {} /sbin/ifconfig -l".format(vnet_name) 189 ) 190 for iface_name in ifaces_lst.split(): 191 if not self.is_autodeleted(iface_name): 192 if iface_name not in self._list_ifaces(): 193 print("Skipping interface {}:{}".format(vnet_name, iface_name)) 194 continue 195 run_cmd( 196 "/usr/sbin/jexec {} /sbin/ifconfig {} destroy".format(vnet_name, iface_name) 197 ) 198 199 def cleanup(self): 200 try: 201 os.unlink(self.INTERFACES_FNAME) 202 except OSError: 203 pass 204 205 206class VnetInstance(object): 207 def __init__( 208 self, vnet_alias: str, vnet_name: str, jid: int, ifaces: List[VnetInterface] 209 ): 210 self.name = vnet_name 211 self.alias = vnet_alias # reference in the test topology 212 self.jid = jid 213 self.ifaces = ifaces 214 self.iface_alias_map = {} # iface.alias: iface 215 self.iface_map = {} # iface.name: iface 216 for iface in ifaces: 217 iface.set_vnet(vnet_name) 218 iface.set_jailed(True) 219 self.iface_alias_map[iface.alias] = iface 220 self.iface_map[iface.name] = iface 221 # Allow reference to interfce aliases as attributes 222 setattr(self, iface.alias, iface) 223 self.need_dad = False # Disable duplicate address detection by default 224 self.attached = False 225 self.pipe = None 226 self.subprocess = None 227 228 def run_vnet_cmd(self, cmd, verbose=True): 229 if not self.attached: 230 cmd = "/usr/sbin/jexec {} {}".format(self.name, cmd) 231 return run_cmd(cmd, verbose) 232 233 def disable_dad(self): 234 self.run_vnet_cmd("/sbin/sysctl net.inet6.ip6.dad_count=0") 235 236 def set_pipe(self, pipe): 237 self.pipe = pipe 238 239 def set_subprocess(self, p): 240 self.subprocess = p 241 242 @staticmethod 243 def attach_jid(jid: int): 244 error_code = libc.jail_attach(jid) 245 if error_code != 0: 246 raise Exception("jail_attach() failed: errno {}".format(error_code)) 247 248 def attach(self): 249 self.attach_jid(self.jid) 250 self.attached = True 251 252 253class VnetFactory(object): 254 JAILS_FNAME = "created_jails.lst" 255 256 def __init__(self, topology_id: str): 257 self.topology_id = topology_id 258 self.file_name = self.JAILS_FNAME 259 self._vnets: List[str] = [] 260 261 def _register_vnet(self, vnet_name: str): 262 self._vnets.append(vnet_name) 263 with open(self.file_name, "a") as f: 264 f.write(vnet_name + "\n") 265 266 @staticmethod 267 def _wait_interfaces(vnet_name: str, ifaces: List[str]) -> List[str]: 268 cmd = "/usr/sbin/jexec {} /sbin/ifconfig -l".format(vnet_name) 269 not_matched: List[str] = [] 270 for i in range(50): 271 vnet_ifaces = run_cmd(cmd).strip().split(" ") 272 not_matched = [] 273 for iface_name in ifaces: 274 if iface_name not in vnet_ifaces: 275 not_matched.append(iface_name) 276 if len(not_matched) == 0: 277 return [] 278 time.sleep(0.1) 279 return not_matched 280 281 def create_vnet(self, vnet_alias: str, ifaces: List[VnetInterface]): 282 vnet_name = "pytest:{}".format(convert_test_name(self.topology_id)) 283 if self._vnets: 284 # add number to distinguish jails 285 vnet_name = "{}_{}".format(vnet_name, len(self._vnets) + 1) 286 iface_cmds = " ".join(["vnet.interface={}".format(i.name) for i in ifaces]) 287 cmd = "/usr/sbin/jail -i -c name={} persist vnet {}".format( 288 vnet_name, iface_cmds 289 ) 290 jid = 0 291 try: 292 jid_str = run_cmd(cmd) 293 jid = int(jid_str) 294 except ValueError: 295 print("Jail creation failed, output: {}".format(jid_str)) 296 raise 297 self._register_vnet(vnet_name) 298 299 # Run expedited version of routing 300 VnetInterface.setup_loopback(vnet_name) 301 302 not_found = self._wait_interfaces(vnet_name, [i.name for i in ifaces]) 303 if not_found: 304 raise Exception( 305 "Interfaces {} has not appeared in vnet {}".format(not_found, vnet_name) 306 ) 307 return VnetInstance(vnet_alias, vnet_name, jid, ifaces) 308 309 def cleanup(self): 310 iface_factory = IfaceFactory() 311 try: 312 with open(self.file_name) as f: 313 for line in f: 314 vnet_name = line.strip() 315 iface_factory.cleanup_vnet_interfaces(vnet_name) 316 run_cmd("/usr/sbin/jail -r {}".format(vnet_name)) 317 os.unlink(self.JAILS_FNAME) 318 except OSError: 319 pass 320 321 322class SingleInterfaceMap(NamedTuple): 323 ifaces: List[VnetInterface] 324 vnet_aliases: List[str] 325 326 327class ObjectsMap(NamedTuple): 328 iface_map: Dict[str, SingleInterfaceMap] # keyed by ifX 329 vnet_map: Dict[str, VnetInstance] # keyed by vnetX 330 topo_map: Dict # self.TOPOLOGY 331 332 333class VnetTestTemplate(BaseTest): 334 NEED_ROOT: bool = True 335 TOPOLOGY = {} 336 337 def _get_vnet_handler(self, vnet_alias: str): 338 handler_name = "{}_handler".format(vnet_alias) 339 return getattr(self, handler_name, None) 340 341 def _setup_vnet(self, vnet: VnetInstance, obj_map: Dict, pipe): 342 """Base Handler to setup given VNET. 343 Can be run in a subprocess. If so, passes control to the special 344 vnetX_handler() after setting up interface addresses 345 """ 346 vnet.attach() 347 print("# setup_vnet({})".format(vnet.name)) 348 if pipe is not None: 349 vnet.set_pipe(pipe) 350 351 topo = obj_map.topo_map 352 ipv6_ifaces = [] 353 # Disable DAD 354 if not vnet.need_dad: 355 vnet.disable_dad() 356 for iface in vnet.ifaces: 357 # check index of vnet within an interface 358 # as we have prefixes for both ends of the interface 359 iface_map = obj_map.iface_map[iface.alias] 360 idx = iface_map.vnet_aliases.index(vnet.alias) 361 prefixes6 = topo[iface.alias].get("prefixes6", []) 362 prefixes4 = topo[iface.alias].get("prefixes4", []) 363 if prefixes6 or prefixes4: 364 ipv6_ifaces.append(iface) 365 iface.turn_up() 366 if prefixes6: 367 iface.enable_ipv6() 368 for prefix in prefixes6 + prefixes4: 369 if prefix[idx]: 370 iface.setup_addr(prefix[idx]) 371 for iface in ipv6_ifaces: 372 while iface.has_tentative(): 373 time.sleep(0.1) 374 375 # Run actual handler 376 handler = self._get_vnet_handler(vnet.alias) 377 if handler: 378 # Do unbuffered stdout for children 379 # so the logs are present if the child hangs 380 sys.stdout.reconfigure(line_buffering=True) 381 self.drop_privileges() 382 handler(vnet) 383 384 def _get_topo_ifmap(self, topo: Dict): 385 iface_factory = IfaceFactory() 386 iface_map: Dict[str, SingleInterfaceMap] = {} 387 iface_aliases = set() 388 for obj_name, obj_data in topo.items(): 389 if obj_name.startswith("vnet"): 390 for iface_alias in obj_data["ifaces"]: 391 iface_aliases.add(iface_alias) 392 for iface_alias in iface_aliases: 393 print("Creating {}".format(iface_alias)) 394 iface_data = topo[iface_alias] 395 iface_type = iface_data.get("type", "epair") 396 ifaces = iface_factory.create_iface(iface_alias, iface_type) 397 smap = SingleInterfaceMap(ifaces, []) 398 iface_map[iface_alias] = smap 399 return iface_map 400 401 def setup_topology(self, topo: Dict, topology_id: str): 402 """Creates jails & interfaces for the provided topology""" 403 vnet_map = {} 404 vnet_factory = VnetFactory(topology_id) 405 iface_map = self._get_topo_ifmap(topo) 406 for obj_name, obj_data in topo.items(): 407 if obj_name.startswith("vnet"): 408 vnet_ifaces = [] 409 for iface_alias in obj_data["ifaces"]: 410 # epair creates 2 interfaces, grab first _available_ 411 # and map it to the VNET being created 412 idx = len(iface_map[iface_alias].vnet_aliases) 413 iface_map[iface_alias].vnet_aliases.append(obj_name) 414 vnet_ifaces.append(iface_map[iface_alias].ifaces[idx]) 415 vnet = vnet_factory.create_vnet(obj_name, vnet_ifaces) 416 vnet_map[obj_name] = vnet 417 # Allow reference to VNETs as attributes 418 setattr(self, obj_name, vnet) 419 # Debug output 420 print("============= TEST TOPOLOGY =============") 421 for vnet_alias, vnet in vnet_map.items(): 422 print("# vnet {} -> {}".format(vnet.alias, vnet.name), end="") 423 handler = self._get_vnet_handler(vnet.alias) 424 if handler: 425 print(" handler: {}".format(handler.__name__), end="") 426 print() 427 for iface_alias, iface_data in iface_map.items(): 428 vnets = iface_data.vnet_aliases 429 ifaces: List[VnetInterface] = iface_data.ifaces 430 if len(vnets) == 1 and len(ifaces) == 2: 431 print( 432 "# iface {}: {}::{} -> main::{}".format( 433 iface_alias, vnets[0], ifaces[0].name, ifaces[1].name 434 ) 435 ) 436 elif len(vnets) == 2 and len(ifaces) == 2: 437 print( 438 "# iface {}: {}::{} -> {}::{}".format( 439 iface_alias, vnets[0], ifaces[0].name, vnets[1], ifaces[1].name 440 ) 441 ) 442 else: 443 print( 444 "# iface {}: ifaces: {} vnets: {}".format( 445 iface_alias, vnets, [i.name for i in ifaces] 446 ) 447 ) 448 print() 449 return ObjectsMap(iface_map, vnet_map, topo) 450 451 def setup_method(self, _method): 452 """Sets up all the required topology and handlers for the given test""" 453 super().setup_method(_method) 454 # TestIP6Output.test_output6_pktinfo[ipandif] 455 topology_id = get_topology_id(self.test_id) 456 topology = self.TOPOLOGY 457 # First, setup kernel objects - interfaces & vnets 458 obj_map = self.setup_topology(topology, topology_id) 459 main_vnet = None # one without subprocess handler 460 for vnet_alias, vnet in obj_map.vnet_map.items(): 461 if self._get_vnet_handler(vnet_alias): 462 # Need subprocess to run 463 parent_pipe, child_pipe = Pipe() 464 p = Process( 465 target=self._setup_vnet, 466 args=( 467 vnet, 468 obj_map, 469 child_pipe, 470 ), 471 ) 472 vnet.set_pipe(parent_pipe) 473 vnet.set_subprocess(p) 474 p.start() 475 else: 476 if main_vnet is not None: 477 raise Exception("there can be only 1 VNET w/o handler") 478 main_vnet = vnet 479 # Main vnet needs to be the last, so all the other subprocesses 480 # are started & their pipe handles collected 481 self.vnet = main_vnet 482 self._setup_vnet(main_vnet, obj_map, None) 483 # Save state for the main handler 484 self.iface_map = obj_map.iface_map 485 self.vnet_map = obj_map.vnet_map 486 self.drop_privileges() 487 488 def cleanup(self, test_id: str): 489 # pytest test id: file::class::test_name 490 topology_id = get_topology_id(self.test_id) 491 492 print("============= vnet cleanup =============") 493 print("# topology_id: '{}'".format(topology_id)) 494 VnetFactory(topology_id).cleanup() 495 IfaceFactory().cleanup() 496 497 def wait_object(self, pipe, timeout=5): 498 if pipe.poll(timeout): 499 return pipe.recv() 500 raise TimeoutError 501 502 def wait_objects_any(self, pipe_list, timeout=5): 503 objects = connection.wait(pipe_list, timeout) 504 if objects: 505 return objects[0].recv() 506 raise TimeoutError 507 508 def send_object(self, pipe, obj): 509 pipe.send(obj) 510 511 def wait(self): 512 while True: 513 time.sleep(1) 514 515 @property 516 def curvnet(self): 517 pass 518 519 520class SingleVnetTestTemplate(VnetTestTemplate): 521 IPV6_PREFIXES: List[str] = [] 522 IPV4_PREFIXES: List[str] = [] 523 IFTYPE = "epair" 524 525 def _setup_default_topology(self): 526 topology = copy.deepcopy( 527 { 528 "vnet1": {"ifaces": ["if1"]}, 529 "if1": {"type": self.IFTYPE, "prefixes4": [], "prefixes6": []}, 530 } 531 ) 532 for prefix in self.IPV6_PREFIXES: 533 topology["if1"]["prefixes6"].append((prefix,)) 534 for prefix in self.IPV4_PREFIXES: 535 topology["if1"]["prefixes4"].append((prefix,)) 536 return topology 537 538 def setup_method(self, method): 539 if not getattr(self, "TOPOLOGY", None): 540 self.TOPOLOGY = self._setup_default_topology() 541 else: 542 names = self.TOPOLOGY.keys() 543 assert len([n for n in names if n.startswith("vnet")]) == 1 544 super().setup_method(method) 545