1#!/usr/local/bin/python3 2import copy 3import ipaddress 4import os 5import re 6import socket 7import sys 8import time 9from multiprocessing import connection 10from multiprocessing import Pipe 11from multiprocessing import Process 12from typing import Dict 13from typing import List 14from typing import NamedTuple 15 16from atf_python.sys.net.tools import ToolsHelper 17from atf_python.utils import BaseTest 18from atf_python.utils import libc 19 20 21def run_cmd(cmd: str, verbose=True) -> str: 22 if verbose: 23 print("run: '{}'".format(cmd)) 24 return os.popen(cmd).read() 25 26 27def get_topology_id(test_id: str) -> str: 28 """ 29 Gets a unique topology id based on the pytest test_id. 30 "test_ip6_output.py::TestIP6Output::test_output6_pktinfo[ipandif]" -> 31 "TestIP6Output:test_output6_pktinfo[ipandif]" 32 """ 33 return ":".join(test_id.split("::")[-2:]) 34 35 36def convert_test_name(test_name: str) -> str: 37 """Convert test name to a string that can be used in the file/jail names""" 38 ret = "" 39 for char in test_name: 40 if char.isalnum() or char in ("_", "-", ":"): 41 ret += char 42 elif char in ("["): 43 ret += "_" 44 return ret 45 46 47class VnetInterface(object): 48 # defines from net/if_types.h 49 IFT_LOOP = 0x18 50 IFT_ETHER = 0x06 51 52 def __init__(self, iface_alias: str, iface_name: str): 53 self.name = iface_name 54 self.alias = iface_alias 55 self.vnet_name = "" 56 self.jailed = False 57 self.addr_map: Dict[str, Dict] = {"inet6": {}, "inet": {}} 58 self.prefixes4: List[List[str]] = [] 59 self.prefixes6: List[List[str]] = [] 60 if iface_name.startswith("lo"): 61 self.iftype = self.IFT_LOOP 62 else: 63 self.iftype = self.IFT_ETHER 64 65 @property 66 def ifindex(self): 67 return socket.if_nametoindex(self.name) 68 69 @property 70 def first_ipv6(self): 71 d = self.addr_map["inet6"] 72 return d[next(iter(d))] 73 74 @property 75 def first_ipv4(self): 76 d = self.addr_map["inet"] 77 return d[next(iter(d))] 78 79 def set_vnet(self, vnet_name: str): 80 self.vnet_name = vnet_name 81 82 def set_jailed(self, jailed: bool): 83 self.jailed = jailed 84 85 def run_cmd( 86 self, 87 cmd, 88 verbose=False, 89 ): 90 if self.vnet_name and not self.jailed: 91 cmd = "jexec {} {}".format(self.vnet_name, cmd) 92 return run_cmd(cmd, verbose) 93 94 @classmethod 95 def setup_loopback(cls, vnet_name: str): 96 lo = VnetInterface("", "lo0") 97 lo.set_vnet(vnet_name) 98 lo.setup_addr("127.0.0.1/8") 99 lo.turn_up() 100 101 @classmethod 102 def create_iface(cls, alias_name: str, iface_name: str) -> List["VnetInterface"]: 103 name = run_cmd("/sbin/ifconfig {} create".format(iface_name)).rstrip() 104 if not name: 105 raise Exception("Unable to create iface {}".format(iface_name)) 106 ret = [cls(alias_name, name)] 107 if name.startswith("epair"): 108 ret.append(cls(alias_name, name[:-1] + "b")) 109 return ret 110 111 def setup_addr(self, _addr: str): 112 addr = ipaddress.ip_interface(_addr) 113 if addr.version == 6: 114 family = "inet6" 115 cmd = "/sbin/ifconfig {} {} {}".format(self.name, family, addr) 116 else: 117 family = "inet" 118 if self.addr_map[family]: 119 cmd = "/sbin/ifconfig {} alias {}".format(self.name, addr) 120 else: 121 cmd = "/sbin/ifconfig {} {} {}".format(self.name, family, addr) 122 self.run_cmd(cmd) 123 self.addr_map[family][str(addr.ip)] = addr 124 125 def delete_addr(self, _addr: str): 126 addr = ipaddress.ip_address(_addr) 127 if addr.version == 6: 128 family = "inet6" 129 cmd = "/sbin/ifconfig {} inet6 {} delete".format(self.name, addr) 130 else: 131 family = "inet" 132 cmd = "/sbin/ifconfig {} -alias {}".format(self.name, addr) 133 self.run_cmd(cmd) 134 del self.addr_map[family][str(addr)] 135 136 def turn_up(self): 137 cmd = "/sbin/ifconfig {} up".format(self.name) 138 self.run_cmd(cmd) 139 140 def enable_ipv6(self): 141 cmd = "/usr/sbin/ndp -i {} -disabled".format(self.name) 142 self.run_cmd(cmd) 143 144 def has_tentative(self) -> bool: 145 """True if an interface has some addresses in tenative state""" 146 cmd = "/sbin/ifconfig {} inet6".format(self.name) 147 out = self.run_cmd(cmd, verbose=False) 148 for line in out.splitlines(): 149 if "tentative" in line: 150 return True 151 return False 152 153 154class IfaceFactory(object): 155 INTERFACES_FNAME = "created_ifaces.lst" 156 AUTODELETE_TYPES = ("epair", "gif", "gre", "lo", "tap", "tun") 157 158 def __init__(self): 159 self.file_name = self.INTERFACES_FNAME 160 161 def _register_iface(self, iface_name: str): 162 with open(self.file_name, "a") as f: 163 f.write(iface_name + "\n") 164 165 def _list_ifaces(self) -> List[str]: 166 ret: List[str] = [] 167 try: 168 with open(self.file_name, "r") as f: 169 for line in f: 170 ret.append(line.strip()) 171 except OSError: 172 pass 173 return ret 174 175 def create_iface(self, alias_name: str, iface_name: str) -> List[VnetInterface]: 176 ifaces = VnetInterface.create_iface(alias_name, iface_name) 177 for iface in ifaces: 178 if not self.is_autodeleted(iface.name): 179 self._register_iface(iface.name) 180 return ifaces 181 182 @staticmethod 183 def is_autodeleted(iface_name: str) -> bool: 184 if iface_name == "lo0": 185 return False 186 iface_type = re.split(r"\d+", iface_name)[0] 187 return iface_type in IfaceFactory.AUTODELETE_TYPES 188 189 def cleanup_vnet_interfaces(self, vnet_name: str) -> List[str]: 190 """Destroys""" 191 ifaces_lst = ToolsHelper.get_output( 192 "/usr/sbin/jexec {} ifconfig -l".format(vnet_name) 193 ) 194 for iface_name in ifaces_lst.split(): 195 if not self.is_autodeleted(iface_name): 196 if iface_name not in self._list_ifaces(): 197 print("Skipping interface {}:{}".format(vnet_name, iface_name)) 198 continue 199 run_cmd( 200 "/usr/sbin/jexec {} ifconfig {} destroy".format(vnet_name, iface_name) 201 ) 202 203 def cleanup(self): 204 try: 205 os.unlink(self.INTERFACES_FNAME) 206 except OSError: 207 pass 208 209 210class VnetInstance(object): 211 def __init__( 212 self, vnet_alias: str, vnet_name: str, jid: int, ifaces: List[VnetInterface] 213 ): 214 self.name = vnet_name 215 self.alias = vnet_alias # reference in the test topology 216 self.jid = jid 217 self.ifaces = ifaces 218 self.iface_alias_map = {} # iface.alias: iface 219 self.iface_map = {} # iface.name: iface 220 for iface in ifaces: 221 iface.set_vnet(vnet_name) 222 iface.set_jailed(True) 223 self.iface_alias_map[iface.alias] = iface 224 self.iface_map[iface.name] = iface 225 # Allow reference to interfce aliases as attributes 226 setattr(self, iface.alias, iface) 227 self.need_dad = False # Disable duplicate address detection by default 228 self.attached = False 229 self.pipe = None 230 self.subprocess = None 231 232 def run_vnet_cmd(self, cmd): 233 if not self.attached: 234 cmd = "jexec {} {}".format(self.name, cmd) 235 return run_cmd(cmd) 236 237 def disable_dad(self): 238 self.run_vnet_cmd("/sbin/sysctl net.inet6.ip6.dad_count=0") 239 240 def set_pipe(self, pipe): 241 self.pipe = pipe 242 243 def set_subprocess(self, p): 244 self.subprocess = p 245 246 @staticmethod 247 def attach_jid(jid: int): 248 error_code = libc.jail_attach(jid) 249 if error_code != 0: 250 raise Exception("jail_attach() failed: errno {}".format(error_code)) 251 252 def attach(self): 253 self.attach_jid(self.jid) 254 self.attached = True 255 256 257class VnetFactory(object): 258 JAILS_FNAME = "created_jails.lst" 259 260 def __init__(self, topology_id: str): 261 self.topology_id = topology_id 262 self.file_name = self.JAILS_FNAME 263 self._vnets: List[str] = [] 264 265 def _register_vnet(self, vnet_name: str): 266 self._vnets.append(vnet_name) 267 with open(self.file_name, "a") as f: 268 f.write(vnet_name + "\n") 269 270 @staticmethod 271 def _wait_interfaces(vnet_name: str, ifaces: List[str]) -> List[str]: 272 cmd = "jexec {} /sbin/ifconfig -l".format(vnet_name) 273 not_matched: List[str] = [] 274 for i in range(50): 275 vnet_ifaces = run_cmd(cmd).strip().split(" ") 276 not_matched = [] 277 for iface_name in ifaces: 278 if iface_name not in vnet_ifaces: 279 not_matched.append(iface_name) 280 if len(not_matched) == 0: 281 return [] 282 time.sleep(0.1) 283 return not_matched 284 285 def create_vnet(self, vnet_alias: str, ifaces: List[VnetInterface]): 286 vnet_name = "pytest:{}".format(convert_test_name(self.topology_id)) 287 if self._vnets: 288 # add number to distinguish jails 289 vnet_name = "{}_{}".format(vnet_name, len(self._vnets) + 1) 290 iface_cmds = " ".join(["vnet.interface={}".format(i.name) for i in ifaces]) 291 cmd = "/usr/sbin/jail -i -c name={} persist vnet {}".format( 292 vnet_name, iface_cmds 293 ) 294 jid = 0 295 try: 296 jid_str = run_cmd(cmd) 297 jid = int(jid_str) 298 except ValueError: 299 print("Jail creation failed, output: {}".format(jid_str)) 300 raise 301 self._register_vnet(vnet_name) 302 303 # Run expedited version of routing 304 VnetInterface.setup_loopback(vnet_name) 305 306 not_found = self._wait_interfaces(vnet_name, [i.name for i in ifaces]) 307 if not_found: 308 raise Exception( 309 "Interfaces {} has not appeared in vnet {}".format(not_found, vnet_name) 310 ) 311 return VnetInstance(vnet_alias, vnet_name, jid, ifaces) 312 313 def cleanup(self): 314 iface_factory = IfaceFactory() 315 try: 316 with open(self.file_name) as f: 317 for line in f: 318 vnet_name = line.strip() 319 iface_factory.cleanup_vnet_interfaces(vnet_name) 320 run_cmd("/usr/sbin/jail -r {}".format(vnet_name)) 321 os.unlink(self.JAILS_FNAME) 322 except OSError: 323 pass 324 325 326class SingleInterfaceMap(NamedTuple): 327 ifaces: List[VnetInterface] 328 vnet_aliases: List[str] 329 330 331class ObjectsMap(NamedTuple): 332 iface_map: Dict[str, SingleInterfaceMap] # keyed by ifX 333 vnet_map: Dict[str, VnetInstance] # keyed by vnetX 334 topo_map: Dict # self.TOPOLOGY 335 336 337class VnetTestTemplate(BaseTest): 338 NEED_ROOT: bool = True 339 TOPOLOGY = {} 340 341 def _get_vnet_handler(self, vnet_alias: str): 342 handler_name = "{}_handler".format(vnet_alias) 343 return getattr(self, handler_name, None) 344 345 def _setup_vnet(self, vnet: VnetInstance, obj_map: Dict, pipe): 346 """Base Handler to setup given VNET. 347 Can be run in a subprocess. If so, passes control to the special 348 vnetX_handler() after setting up interface addresses 349 """ 350 vnet.attach() 351 print("# setup_vnet({})".format(vnet.name)) 352 if pipe is not None: 353 vnet.set_pipe(pipe) 354 355 topo = obj_map.topo_map 356 ipv6_ifaces = [] 357 # Disable DAD 358 if not vnet.need_dad: 359 vnet.disable_dad() 360 for iface in vnet.ifaces: 361 # check index of vnet within an interface 362 # as we have prefixes for both ends of the interface 363 iface_map = obj_map.iface_map[iface.alias] 364 idx = iface_map.vnet_aliases.index(vnet.alias) 365 prefixes6 = topo[iface.alias].get("prefixes6", []) 366 prefixes4 = topo[iface.alias].get("prefixes4", []) 367 if prefixes6 or prefixes4: 368 ipv6_ifaces.append(iface) 369 iface.turn_up() 370 if prefixes6: 371 iface.enable_ipv6() 372 for prefix in prefixes6 + prefixes4: 373 if prefix[idx]: 374 iface.setup_addr(prefix[idx]) 375 for iface in ipv6_ifaces: 376 while iface.has_tentative(): 377 time.sleep(0.1) 378 379 # Run actual handler 380 handler = self._get_vnet_handler(vnet.alias) 381 if handler: 382 # Do unbuffered stdout for children 383 # so the logs are present if the child hangs 384 sys.stdout.reconfigure(line_buffering=True) 385 self.drop_privileges() 386 handler(vnet) 387 388 def _get_topo_ifmap(self, topo: Dict): 389 iface_factory = IfaceFactory() 390 iface_map: Dict[str, SingleInterfaceMap] = {} 391 iface_aliases = set() 392 for obj_name, obj_data in topo.items(): 393 if obj_name.startswith("vnet"): 394 for iface_alias in obj_data["ifaces"]: 395 iface_aliases.add(iface_alias) 396 for iface_alias in iface_aliases: 397 print("Creating {}".format(iface_alias)) 398 iface_data = topo[iface_alias] 399 iface_type = iface_data.get("type", "epair") 400 ifaces = iface_factory.create_iface(iface_alias, iface_type) 401 smap = SingleInterfaceMap(ifaces, []) 402 iface_map[iface_alias] = smap 403 return iface_map 404 405 def setup_topology(self, topo: Dict, topology_id: str): 406 """Creates jails & interfaces for the provided topology""" 407 vnet_map = {} 408 vnet_factory = VnetFactory(topology_id) 409 iface_map = self._get_topo_ifmap(topo) 410 for obj_name, obj_data in topo.items(): 411 if obj_name.startswith("vnet"): 412 vnet_ifaces = [] 413 for iface_alias in obj_data["ifaces"]: 414 # epair creates 2 interfaces, grab first _available_ 415 # and map it to the VNET being created 416 idx = len(iface_map[iface_alias].vnet_aliases) 417 iface_map[iface_alias].vnet_aliases.append(obj_name) 418 vnet_ifaces.append(iface_map[iface_alias].ifaces[idx]) 419 vnet = vnet_factory.create_vnet(obj_name, vnet_ifaces) 420 vnet_map[obj_name] = vnet 421 # Allow reference to VNETs as attributes 422 setattr(self, obj_name, vnet) 423 # Debug output 424 print("============= TEST TOPOLOGY =============") 425 for vnet_alias, vnet in vnet_map.items(): 426 print("# vnet {} -> {}".format(vnet.alias, vnet.name), end="") 427 handler = self._get_vnet_handler(vnet.alias) 428 if handler: 429 print(" handler: {}".format(handler.__name__), end="") 430 print() 431 for iface_alias, iface_data in iface_map.items(): 432 vnets = iface_data.vnet_aliases 433 ifaces: List[VnetInterface] = iface_data.ifaces 434 if len(vnets) == 1 and len(ifaces) == 2: 435 print( 436 "# iface {}: {}::{} -> main::{}".format( 437 iface_alias, vnets[0], ifaces[0].name, ifaces[1].name 438 ) 439 ) 440 elif len(vnets) == 2 and len(ifaces) == 2: 441 print( 442 "# iface {}: {}::{} -> {}::{}".format( 443 iface_alias, vnets[0], ifaces[0].name, vnets[1], ifaces[1].name 444 ) 445 ) 446 else: 447 print( 448 "# iface {}: ifaces: {} vnets: {}".format( 449 iface_alias, vnets, [i.name for i in ifaces] 450 ) 451 ) 452 print() 453 return ObjectsMap(iface_map, vnet_map, topo) 454 455 def setup_method(self, _method): 456 """Sets up all the required topology and handlers for the given test""" 457 super().setup_method(_method) 458 # TestIP6Output.test_output6_pktinfo[ipandif] 459 topology_id = get_topology_id(self.test_id) 460 topology = self.TOPOLOGY 461 # First, setup kernel objects - interfaces & vnets 462 obj_map = self.setup_topology(topology, topology_id) 463 main_vnet = None # one without subprocess handler 464 for vnet_alias, vnet in obj_map.vnet_map.items(): 465 if self._get_vnet_handler(vnet_alias): 466 # Need subprocess to run 467 parent_pipe, child_pipe = Pipe() 468 p = Process( 469 target=self._setup_vnet, 470 args=( 471 vnet, 472 obj_map, 473 child_pipe, 474 ), 475 ) 476 vnet.set_pipe(parent_pipe) 477 vnet.set_subprocess(p) 478 p.start() 479 else: 480 if main_vnet is not None: 481 raise Exception("there can be only 1 VNET w/o handler") 482 main_vnet = vnet 483 # Main vnet needs to be the last, so all the other subprocesses 484 # are started & their pipe handles collected 485 self.vnet = main_vnet 486 self._setup_vnet(main_vnet, obj_map, None) 487 # Save state for the main handler 488 self.iface_map = obj_map.iface_map 489 self.vnet_map = obj_map.vnet_map 490 self.drop_privileges() 491 492 def cleanup(self, test_id: str): 493 # pytest test id: file::class::test_name 494 topology_id = get_topology_id(self.test_id) 495 496 print("==== vnet cleanup ===") 497 print("# topology_id: '{}'".format(topology_id)) 498 VnetFactory(topology_id).cleanup() 499 IfaceFactory().cleanup() 500 501 def wait_object(self, pipe, timeout=5): 502 if pipe.poll(timeout): 503 return pipe.recv() 504 raise TimeoutError 505 506 def wait_objects_any(self, pipe_list, timeout=5): 507 objects = connection.wait(pipe_list, timeout) 508 if objects: 509 return objects[0].recv() 510 raise TimeoutError 511 512 def send_object(self, pipe, obj): 513 pipe.send(obj) 514 515 def wait(self): 516 while True: 517 time.sleep(1) 518 519 @property 520 def curvnet(self): 521 pass 522 523 524class SingleVnetTestTemplate(VnetTestTemplate): 525 IPV6_PREFIXES: List[str] = [] 526 IPV4_PREFIXES: List[str] = [] 527 IFTYPE = "epair" 528 529 def _setup_default_topology(self): 530 topology = copy.deepcopy( 531 { 532 "vnet1": {"ifaces": ["if1"]}, 533 "if1": {"type": self.IFTYPE, "prefixes4": [], "prefixes6": []}, 534 } 535 ) 536 for prefix in self.IPV6_PREFIXES: 537 topology["if1"]["prefixes6"].append((prefix,)) 538 for prefix in self.IPV4_PREFIXES: 539 topology["if1"]["prefixes4"].append((prefix,)) 540 return topology 541 542 def setup_method(self, method): 543 if not getattr(self, "TOPOLOGY", None): 544 self.TOPOLOGY = self._setup_default_topology() 545 else: 546 names = self.TOPOLOGY.keys() 547 assert len([n for n in names if n.startswith("vnet")]) == 1 548 super().setup_method(method) 549