1#!/usr/local/bin/python3 2import copy 3import ipaddress 4import os 5import re 6import socket 7import sys 8import time 9from multiprocessing import connection 10from multiprocessing import Pipe 11from multiprocessing import Process 12from typing import Dict 13from typing import List 14from typing import NamedTuple 15 16from atf_python.sys.net.tools import ToolsHelper 17from atf_python.utils import BaseTest 18from atf_python.utils import libc 19 20 21def run_cmd(cmd: str, verbose=True) -> str: 22 print("run: '{}'".format(cmd)) 23 return os.popen(cmd).read() 24 25 26def get_topology_id(test_id: str) -> str: 27 """ 28 Gets a unique topology id based on the pytest test_id. 29 "test_ip6_output.py::TestIP6Output::test_output6_pktinfo[ipandif]" -> 30 "TestIP6Output:test_output6_pktinfo[ipandif]" 31 """ 32 return ":".join(test_id.split("::")[-2:]) 33 34 35def convert_test_name(test_name: str) -> str: 36 """Convert test name to a string that can be used in the file/jail names""" 37 ret = "" 38 for char in test_name: 39 if char.isalnum() or char in ("_", "-", ":"): 40 ret += char 41 elif char in ("["): 42 ret += "_" 43 return ret 44 45 46class VnetInterface(object): 47 # defines from net/if_types.h 48 IFT_LOOP = 0x18 49 IFT_ETHER = 0x06 50 51 def __init__(self, iface_alias: str, iface_name: str): 52 self.name = iface_name 53 self.alias = iface_alias 54 self.vnet_name = "" 55 self.jailed = False 56 self.addr_map: Dict[str, Dict] = {"inet6": {}, "inet": {}} 57 self.prefixes4: List[List[str]] = [] 58 self.prefixes6: List[List[str]] = [] 59 if iface_name.startswith("lo"): 60 self.iftype = self.IFT_LOOP 61 else: 62 self.iftype = self.IFT_ETHER 63 64 @property 65 def ifindex(self): 66 return socket.if_nametoindex(self.name) 67 68 @property 69 def first_ipv6(self): 70 d = self.addr_map["inet6"] 71 return d[next(iter(d))] 72 73 @property 74 def first_ipv4(self): 75 d = self.addr_map["inet"] 76 return d[next(iter(d))] 77 78 def set_vnet(self, vnet_name: str): 79 self.vnet_name = vnet_name 80 81 def set_jailed(self, jailed: bool): 82 self.jailed = jailed 83 84 def run_cmd( 85 self, 86 cmd, 87 verbose=False, 88 ): 89 if self.vnet_name and not self.jailed: 90 cmd = "jexec {} {}".format(self.vnet_name, cmd) 91 return run_cmd(cmd, verbose) 92 93 @classmethod 94 def setup_loopback(cls, vnet_name: str): 95 lo = VnetInterface("", "lo0") 96 lo.set_vnet(vnet_name) 97 lo.setup_addr("127.0.0.1/8") 98 lo.turn_up() 99 100 @classmethod 101 def create_iface(cls, alias_name: str, iface_name: str) -> List["VnetInterface"]: 102 name = run_cmd("/sbin/ifconfig {} create".format(iface_name)).rstrip() 103 if not name: 104 raise Exception("Unable to create iface {}".format(iface_name)) 105 ret = [cls(alias_name, name)] 106 if name.startswith("epair"): 107 ret.append(cls(alias_name, name[:-1] + "b")) 108 return ret 109 110 def setup_addr(self, _addr: str): 111 addr = ipaddress.ip_interface(_addr) 112 if addr.version == 6: 113 family = "inet6" 114 cmd = "/sbin/ifconfig {} {} {}".format(self.name, family, addr) 115 else: 116 family = "inet" 117 if self.addr_map[family]: 118 cmd = "/sbin/ifconfig {} alias {}".format(self.name, addr) 119 else: 120 cmd = "/sbin/ifconfig {} {} {}".format(self.name, family, addr) 121 self.run_cmd(cmd) 122 self.addr_map[family][str(addr.ip)] = addr 123 124 def delete_addr(self, _addr: str): 125 addr = ipaddress.ip_address(_addr) 126 if addr.version == 6: 127 family = "inet6" 128 cmd = "/sbin/ifconfig {} inet6 {} delete".format(self.name, addr) 129 else: 130 family = "inet" 131 cmd = "/sbin/ifconfig {} -alias {}".format(self.name, addr) 132 self.run_cmd(cmd) 133 del self.addr_map[family][str(addr)] 134 135 def turn_up(self): 136 cmd = "/sbin/ifconfig {} up".format(self.name) 137 self.run_cmd(cmd) 138 139 def enable_ipv6(self): 140 cmd = "/usr/sbin/ndp -i {} -disabled".format(self.name) 141 self.run_cmd(cmd) 142 143 def has_tentative(self) -> bool: 144 """True if an interface has some addresses in tenative state""" 145 cmd = "/sbin/ifconfig {} inet6".format(self.name) 146 out = self.run_cmd(cmd, verbose=False) 147 for line in out.splitlines(): 148 if "tentative" in line: 149 return True 150 return False 151 152 153class IfaceFactory(object): 154 INTERFACES_FNAME = "created_ifaces.lst" 155 AUTODELETE_TYPES = ("epair", "gif", "gre", "lo", "tap", "tun") 156 157 def __init__(self): 158 self.file_name = self.INTERFACES_FNAME 159 160 def _register_iface(self, iface_name: str): 161 with open(self.file_name, "a") as f: 162 f.write(iface_name + "\n") 163 164 def _list_ifaces(self) -> List[str]: 165 ret: List[str] = [] 166 try: 167 with open(self.file_name, "r") as f: 168 for line in f: 169 ret.append(line.strip()) 170 except OSError: 171 pass 172 return ret 173 174 def create_iface(self, alias_name: str, iface_name: str) -> List[VnetInterface]: 175 ifaces = VnetInterface.create_iface(alias_name, iface_name) 176 for iface in ifaces: 177 if not self.is_autodeleted(iface.name): 178 self._register_iface(iface.name) 179 return ifaces 180 181 @staticmethod 182 def is_autodeleted(iface_name: str) -> bool: 183 if iface_name == "lo0": 184 return False 185 iface_type = re.split(r"\d+", iface_name)[0] 186 return iface_type in IfaceFactory.AUTODELETE_TYPES 187 188 def cleanup_vnet_interfaces(self, vnet_name: str) -> List[str]: 189 """Destroys""" 190 ifaces_lst = ToolsHelper.get_output( 191 "/usr/sbin/jexec {} ifconfig -l".format(vnet_name) 192 ) 193 for iface_name in ifaces_lst.split(): 194 if not self.is_autodeleted(iface_name): 195 if iface_name not in self._list_ifaces(): 196 print("Skipping interface {}:{}".format(vnet_name, iface_name)) 197 continue 198 run_cmd( 199 "/usr/sbin/jexec {} ifconfig {} destroy".format(vnet_name, iface_name) 200 ) 201 202 def cleanup(self): 203 try: 204 os.unlink(self.INTERFACES_FNAME) 205 except OSError: 206 pass 207 208 209class VnetInstance(object): 210 def __init__( 211 self, vnet_alias: str, vnet_name: str, jid: int, ifaces: List[VnetInterface] 212 ): 213 self.name = vnet_name 214 self.alias = vnet_alias # reference in the test topology 215 self.jid = jid 216 self.ifaces = ifaces 217 self.iface_alias_map = {} # iface.alias: iface 218 self.iface_map = {} # iface.name: iface 219 for iface in ifaces: 220 iface.set_vnet(vnet_name) 221 iface.set_jailed(True) 222 self.iface_alias_map[iface.alias] = iface 223 self.iface_map[iface.name] = iface 224 # Allow reference to interfce aliases as attributes 225 setattr(self, iface.alias, iface) 226 self.need_dad = False # Disable duplicate address detection by default 227 self.attached = False 228 self.pipe = None 229 self.subprocess = None 230 231 def run_vnet_cmd(self, cmd): 232 if not self.attached: 233 cmd = "jexec {} {}".format(self.name, cmd) 234 return run_cmd(cmd) 235 236 def disable_dad(self): 237 self.run_vnet_cmd("/sbin/sysctl net.inet6.ip6.dad_count=0") 238 239 def set_pipe(self, pipe): 240 self.pipe = pipe 241 242 def set_subprocess(self, p): 243 self.subprocess = p 244 245 @staticmethod 246 def attach_jid(jid: int): 247 error_code = libc.jail_attach(jid) 248 if error_code != 0: 249 raise Exception("jail_attach() failed: errno {}".format(error_code)) 250 251 def attach(self): 252 self.attach_jid(self.jid) 253 self.attached = True 254 255 256class VnetFactory(object): 257 JAILS_FNAME = "created_jails.lst" 258 259 def __init__(self, topology_id: str): 260 self.topology_id = topology_id 261 self.file_name = self.JAILS_FNAME 262 self._vnets: List[str] = [] 263 264 def _register_vnet(self, vnet_name: str): 265 self._vnets.append(vnet_name) 266 with open(self.file_name, "a") as f: 267 f.write(vnet_name + "\n") 268 269 @staticmethod 270 def _wait_interfaces(vnet_name: str, ifaces: List[str]) -> List[str]: 271 cmd = "jexec {} /sbin/ifconfig -l".format(vnet_name) 272 not_matched: List[str] = [] 273 for i in range(50): 274 vnet_ifaces = run_cmd(cmd).strip().split(" ") 275 not_matched = [] 276 for iface_name in ifaces: 277 if iface_name not in vnet_ifaces: 278 not_matched.append(iface_name) 279 if len(not_matched) == 0: 280 return [] 281 time.sleep(0.1) 282 return not_matched 283 284 def create_vnet(self, vnet_alias: str, ifaces: List[VnetInterface]): 285 vnet_name = "pytest:{}".format(convert_test_name(self.topology_id)) 286 if self._vnets: 287 # add number to distinguish jails 288 vnet_name = "{}_{}".format(vnet_name, len(self._vnets) + 1) 289 iface_cmds = " ".join(["vnet.interface={}".format(i.name) for i in ifaces]) 290 cmd = "/usr/sbin/jail -i -c name={} persist vnet {}".format( 291 vnet_name, iface_cmds 292 ) 293 jid = 0 294 try: 295 jid_str = run_cmd(cmd) 296 jid = int(jid_str) 297 except ValueError: 298 print("Jail creation failed, output: {}".format(jid_str)) 299 raise 300 self._register_vnet(vnet_name) 301 302 # Run expedited version of routing 303 VnetInterface.setup_loopback(vnet_name) 304 305 not_found = self._wait_interfaces(vnet_name, [i.name for i in ifaces]) 306 if not_found: 307 raise Exception( 308 "Interfaces {} has not appeared in vnet {}".format(not_found, vnet_name) 309 ) 310 return VnetInstance(vnet_alias, vnet_name, jid, ifaces) 311 312 def cleanup(self): 313 iface_factory = IfaceFactory() 314 try: 315 with open(self.file_name) as f: 316 for line in f: 317 vnet_name = line.strip() 318 iface_factory.cleanup_vnet_interfaces(vnet_name) 319 run_cmd("/usr/sbin/jail -r {}".format(vnet_name)) 320 os.unlink(self.JAILS_FNAME) 321 except OSError: 322 pass 323 324 325class SingleInterfaceMap(NamedTuple): 326 ifaces: List[VnetInterface] 327 vnet_aliases: List[str] 328 329 330class ObjectsMap(NamedTuple): 331 iface_map: Dict[str, SingleInterfaceMap] # keyed by ifX 332 vnet_map: Dict[str, VnetInstance] # keyed by vnetX 333 topo_map: Dict # self.TOPOLOGY 334 335 336class VnetTestTemplate(BaseTest): 337 NEED_ROOT: bool = True 338 TOPOLOGY = {} 339 340 def _get_vnet_handler(self, vnet_alias: str): 341 handler_name = "{}_handler".format(vnet_alias) 342 return getattr(self, handler_name, None) 343 344 def _setup_vnet(self, vnet: VnetInstance, obj_map: Dict, pipe): 345 """Base Handler to setup given VNET. 346 Can be run in a subprocess. If so, passes control to the special 347 vnetX_handler() after setting up interface addresses 348 """ 349 vnet.attach() 350 print("# setup_vnet({})".format(vnet.name)) 351 if pipe is not None: 352 vnet.set_pipe(pipe) 353 354 topo = obj_map.topo_map 355 ipv6_ifaces = [] 356 # Disable DAD 357 if not vnet.need_dad: 358 vnet.disable_dad() 359 for iface in vnet.ifaces: 360 # check index of vnet within an interface 361 # as we have prefixes for both ends of the interface 362 iface_map = obj_map.iface_map[iface.alias] 363 idx = iface_map.vnet_aliases.index(vnet.alias) 364 prefixes6 = topo[iface.alias].get("prefixes6", []) 365 prefixes4 = topo[iface.alias].get("prefixes4", []) 366 if prefixes6 or prefixes4: 367 ipv6_ifaces.append(iface) 368 iface.turn_up() 369 if prefixes6: 370 iface.enable_ipv6() 371 for prefix in prefixes6 + prefixes4: 372 if prefix[idx]: 373 iface.setup_addr(prefix[idx]) 374 for iface in ipv6_ifaces: 375 while iface.has_tentative(): 376 time.sleep(0.1) 377 378 # Run actual handler 379 handler = self._get_vnet_handler(vnet.alias) 380 if handler: 381 # Do unbuffered stdout for children 382 # so the logs are present if the child hangs 383 sys.stdout.reconfigure(line_buffering=True) 384 self.drop_privileges() 385 handler(vnet) 386 387 def _get_topo_ifmap(self, topo: Dict): 388 iface_factory = IfaceFactory() 389 iface_map: Dict[str, SingleInterfaceMap] = {} 390 iface_aliases = set() 391 for obj_name, obj_data in topo.items(): 392 if obj_name.startswith("vnet"): 393 for iface_alias in obj_data["ifaces"]: 394 iface_aliases.add(iface_alias) 395 for iface_alias in iface_aliases: 396 print("Creating {}".format(iface_alias)) 397 iface_data = topo[iface_alias] 398 iface_type = iface_data.get("type", "epair") 399 ifaces = iface_factory.create_iface(iface_alias, iface_type) 400 smap = SingleInterfaceMap(ifaces, []) 401 iface_map[iface_alias] = smap 402 return iface_map 403 404 def setup_topology(self, topo: Dict, topology_id: str): 405 """Creates jails & interfaces for the provided topology""" 406 vnet_map = {} 407 vnet_factory = VnetFactory(topology_id) 408 iface_map = self._get_topo_ifmap(topo) 409 for obj_name, obj_data in topo.items(): 410 if obj_name.startswith("vnet"): 411 vnet_ifaces = [] 412 for iface_alias in obj_data["ifaces"]: 413 # epair creates 2 interfaces, grab first _available_ 414 # and map it to the VNET being created 415 idx = len(iface_map[iface_alias].vnet_aliases) 416 iface_map[iface_alias].vnet_aliases.append(obj_name) 417 vnet_ifaces.append(iface_map[iface_alias].ifaces[idx]) 418 vnet = vnet_factory.create_vnet(obj_name, vnet_ifaces) 419 vnet_map[obj_name] = vnet 420 # Allow reference to VNETs as attributes 421 setattr(self, obj_name, vnet) 422 # Debug output 423 print("============= TEST TOPOLOGY =============") 424 for vnet_alias, vnet in vnet_map.items(): 425 print("# vnet {} -> {}".format(vnet.alias, vnet.name), end="") 426 handler = self._get_vnet_handler(vnet.alias) 427 if handler: 428 print(" handler: {}".format(handler.__name__), end="") 429 print() 430 for iface_alias, iface_data in iface_map.items(): 431 vnets = iface_data.vnet_aliases 432 ifaces: List[VnetInterface] = iface_data.ifaces 433 if len(vnets) == 1 and len(ifaces) == 2: 434 print( 435 "# iface {}: {}::{} -> main::{}".format( 436 iface_alias, vnets[0], ifaces[0].name, ifaces[1].name 437 ) 438 ) 439 elif len(vnets) == 2 and len(ifaces) == 2: 440 print( 441 "# iface {}: {}::{} -> {}::{}".format( 442 iface_alias, vnets[0], ifaces[0].name, vnets[1], ifaces[1].name 443 ) 444 ) 445 else: 446 print( 447 "# iface {}: ifaces: {} vnets: {}".format( 448 iface_alias, vnets, [i.name for i in ifaces] 449 ) 450 ) 451 print() 452 return ObjectsMap(iface_map, vnet_map, topo) 453 454 def setup_method(self, _method): 455 """Sets up all the required topology and handlers for the given test""" 456 super().setup_method(_method) 457 # TestIP6Output.test_output6_pktinfo[ipandif] 458 topology_id = get_topology_id(self.test_id) 459 topology = self.TOPOLOGY 460 # First, setup kernel objects - interfaces & vnets 461 obj_map = self.setup_topology(topology, topology_id) 462 main_vnet = None # one without subprocess handler 463 for vnet_alias, vnet in obj_map.vnet_map.items(): 464 if self._get_vnet_handler(vnet_alias): 465 # Need subprocess to run 466 parent_pipe, child_pipe = Pipe() 467 p = Process( 468 target=self._setup_vnet, 469 args=( 470 vnet, 471 obj_map, 472 child_pipe, 473 ), 474 ) 475 vnet.set_pipe(parent_pipe) 476 vnet.set_subprocess(p) 477 p.start() 478 else: 479 if main_vnet is not None: 480 raise Exception("there can be only 1 VNET w/o handler") 481 main_vnet = vnet 482 # Main vnet needs to be the last, so all the other subprocesses 483 # are started & their pipe handles collected 484 self.vnet = main_vnet 485 self._setup_vnet(main_vnet, obj_map, None) 486 # Save state for the main handler 487 self.iface_map = obj_map.iface_map 488 self.vnet_map = obj_map.vnet_map 489 self.drop_privileges() 490 491 def cleanup(self, test_id: str): 492 # pytest test id: file::class::test_name 493 topology_id = get_topology_id(self.test_id) 494 495 print("==== vnet cleanup ===") 496 print("# topology_id: '{}'".format(topology_id)) 497 VnetFactory(topology_id).cleanup() 498 IfaceFactory().cleanup() 499 500 def wait_object(self, pipe, timeout=5): 501 if pipe.poll(timeout): 502 return pipe.recv() 503 raise TimeoutError 504 505 def wait_objects_any(self, pipe_list, timeout=5): 506 objects = connection.wait(pipe_list, timeout) 507 if objects: 508 return objects[0].recv() 509 raise TimeoutError 510 511 def send_object(self, pipe, obj): 512 pipe.send(obj) 513 514 def wait(self): 515 while True: 516 time.sleep(1) 517 518 @property 519 def curvnet(self): 520 pass 521 522 523class SingleVnetTestTemplate(VnetTestTemplate): 524 IPV6_PREFIXES: List[str] = [] 525 IPV4_PREFIXES: List[str] = [] 526 IFTYPE = "epair" 527 528 def _setup_default_topology(self): 529 topology = copy.deepcopy( 530 { 531 "vnet1": {"ifaces": ["if1"]}, 532 "if1": {"type": self.IFTYPE, "prefixes4": [], "prefixes6": []}, 533 } 534 ) 535 for prefix in self.IPV6_PREFIXES: 536 topology["if1"]["prefixes6"].append((prefix,)) 537 for prefix in self.IPV4_PREFIXES: 538 topology["if1"]["prefixes4"].append((prefix,)) 539 return topology 540 541 def setup_method(self, method): 542 if not getattr(self, "TOPOLOGY", None): 543 self.TOPOLOGY = self._setup_default_topology() 544 else: 545 names = self.TOPOLOGY.keys() 546 assert len([n for n in names if n.startswith("vnet")]) == 1 547 super().setup_method(method) 548