1#!/usr/local/bin/python3 2import copy 3import ipaddress 4import os 5import socket 6import sys 7import time 8from multiprocessing import Pipe 9from multiprocessing import Process 10from typing import Dict 11from typing import List 12from typing import NamedTuple 13 14from atf_python.sys.net.tools import ToolsHelper 15from atf_python.utils import BaseTest 16from atf_python.utils import libc 17 18 19def run_cmd(cmd: str, verbose=True) -> str: 20 print("run: '{}'".format(cmd)) 21 return os.popen(cmd).read() 22 23 24def get_topology_id(test_id: str) -> str: 25 """ 26 Gets a unique topology id based on the pytest test_id. 27 "test_ip6_output.py::TestIP6Output::test_output6_pktinfo[ipandif]" -> 28 "TestIP6Output:test_output6_pktinfo[ipandif]" 29 """ 30 return ":".join(test_id.split("::")[-2:]) 31 32 33def convert_test_name(test_name: str) -> str: 34 """Convert test name to a string that can be used in the file/jail names""" 35 ret = "" 36 for char in test_name: 37 if char.isalnum() or char in ("_", "-", ":"): 38 ret += char 39 elif char in ("["): 40 ret += "_" 41 return ret 42 43 44class VnetInterface(object): 45 # defines from net/if_types.h 46 IFT_LOOP = 0x18 47 IFT_ETHER = 0x06 48 49 def __init__(self, iface_alias: str, iface_name: str): 50 self.name = iface_name 51 self.alias = iface_alias 52 self.vnet_name = "" 53 self.jailed = False 54 self.addr_map: Dict[str, Dict] = {"inet6": {}, "inet": {}} 55 self.prefixes4: List[List[str]] = [] 56 self.prefixes6: List[List[str]] = [] 57 if iface_name.startswith("lo"): 58 self.iftype = self.IFT_LOOP 59 else: 60 self.iftype = self.IFT_ETHER 61 62 @property 63 def ifindex(self): 64 return socket.if_nametoindex(self.name) 65 66 @property 67 def first_ipv6(self): 68 d = self.addr_map["inet6"] 69 return d[next(iter(d))] 70 71 @property 72 def first_ipv4(self): 73 d = self.addr_map["inet"] 74 return d[next(iter(d))] 75 76 def set_vnet(self, vnet_name: str): 77 self.vnet_name = vnet_name 78 79 def set_jailed(self, jailed: bool): 80 self.jailed = jailed 81 82 def run_cmd( 83 self, 84 cmd, 85 verbose=False, 86 ): 87 if self.vnet_name and not self.jailed: 88 cmd = "jexec {} {}".format(self.vnet_name, cmd) 89 return run_cmd(cmd, verbose) 90 91 @classmethod 92 def setup_loopback(cls, vnet_name: str): 93 lo = VnetInterface("", "lo0") 94 lo.set_vnet(vnet_name) 95 lo.setup_addr("127.0.0.1/8") 96 lo.turn_up() 97 98 @classmethod 99 def create_iface(cls, alias_name: str, iface_name: str) -> List["VnetInterface"]: 100 name = run_cmd("/sbin/ifconfig {} create".format(iface_name)).rstrip() 101 if not name: 102 raise Exception("Unable to create iface {}".format(iface_name)) 103 ret = [cls(alias_name, name)] 104 if name.startswith("epair"): 105 ret.append(cls(alias_name, name[:-1] + "b")) 106 return ret 107 108 def setup_addr(self, _addr: str): 109 addr = ipaddress.ip_interface(_addr) 110 if addr.version == 6: 111 family = "inet6" 112 cmd = "/sbin/ifconfig {} {} {}".format(self.name, family, addr) 113 else: 114 family = "inet" 115 if self.addr_map[family]: 116 cmd = "/sbin/ifconfig {} alias {}".format(self.name, addr) 117 else: 118 cmd = "/sbin/ifconfig {} {} {}".format(self.name, family, addr) 119 self.run_cmd(cmd) 120 self.addr_map[family][str(addr.ip)] = addr 121 122 def delete_addr(self, _addr: str): 123 addr = ipaddress.ip_address(_addr) 124 if addr.version == 6: 125 family = "inet6" 126 cmd = "/sbin/ifconfig {} inet6 {} delete".format(self.name, addr) 127 else: 128 family = "inet" 129 cmd = "/sbin/ifconfig {} -alias {}".format(self.name, addr) 130 self.run_cmd(cmd) 131 del self.addr_map[family][str(addr)] 132 133 def turn_up(self): 134 cmd = "/sbin/ifconfig {} up".format(self.name) 135 self.run_cmd(cmd) 136 137 def enable_ipv6(self): 138 cmd = "/usr/sbin/ndp -i {} -disabled".format(self.name) 139 self.run_cmd(cmd) 140 141 def has_tentative(self) -> bool: 142 """True if an interface has some addresses in tenative state""" 143 cmd = "/sbin/ifconfig {} inet6".format(self.name) 144 out = self.run_cmd(cmd, verbose=False) 145 for line in out.splitlines(): 146 if "tentative" in line: 147 return True 148 return False 149 150 151class IfaceFactory(object): 152 INTERFACES_FNAME = "created_ifaces.lst" 153 154 def __init__(self): 155 self.file_name = self.INTERFACES_FNAME 156 157 def _register_iface(self, iface_name: str): 158 with open(self.file_name, "a") as f: 159 f.write(iface_name + "\n") 160 161 def create_iface(self, alias_name: str, iface_name: str) -> List[VnetInterface]: 162 ifaces = VnetInterface.create_iface(alias_name, iface_name) 163 for iface in ifaces: 164 self._register_iface(iface.name) 165 return ifaces 166 167 def cleanup(self): 168 try: 169 with open(self.file_name, "r") as f: 170 for line in f: 171 run_cmd("/sbin/ifconfig {} destroy".format(line.strip())) 172 os.unlink(self.INTERFACES_FNAME) 173 except Exception: 174 pass 175 176 177class VnetInstance(object): 178 def __init__( 179 self, vnet_alias: str, vnet_name: str, jid: int, ifaces: List[VnetInterface] 180 ): 181 self.name = vnet_name 182 self.alias = vnet_alias # reference in the test topology 183 self.jid = jid 184 self.ifaces = ifaces 185 self.iface_alias_map = {} # iface.alias: iface 186 self.iface_map = {} # iface.name: iface 187 for iface in ifaces: 188 iface.set_vnet(vnet_name) 189 iface.set_jailed(True) 190 self.iface_alias_map[iface.alias] = iface 191 self.iface_map[iface.name] = iface 192 self.need_dad = False # Disable duplicate address detection by default 193 self.attached = False 194 self.pipe = None 195 self.subprocess = None 196 197 def run_vnet_cmd(self, cmd): 198 if not self.attached: 199 cmd = "jexec {} {}".format(self.name, cmd) 200 return run_cmd(cmd) 201 202 def disable_dad(self): 203 self.run_vnet_cmd("/sbin/sysctl net.inet6.ip6.dad_count=0") 204 205 def set_pipe(self, pipe): 206 self.pipe = pipe 207 208 def set_subprocess(self, p): 209 self.subprocess = p 210 211 @staticmethod 212 def attach_jid(jid: int): 213 error_code = libc.jail_attach(jid) 214 if error_code != 0: 215 raise Exception("jail_attach() failed: errno {}".format(error_code)) 216 217 def attach(self): 218 self.attach_jid(self.jid) 219 self.attached = True 220 221 222class VnetFactory(object): 223 JAILS_FNAME = "created_jails.lst" 224 225 def __init__(self, topology_id: str): 226 self.topology_id = topology_id 227 self.file_name = self.JAILS_FNAME 228 self._vnets: List[str] = [] 229 230 def _register_vnet(self, vnet_name: str): 231 self._vnets.append(vnet_name) 232 with open(self.file_name, "a") as f: 233 f.write(vnet_name + "\n") 234 235 @staticmethod 236 def _wait_interfaces(vnet_name: str, ifaces: List[str]) -> List[str]: 237 cmd = "jexec {} /sbin/ifconfig -l".format(vnet_name) 238 not_matched: List[str] = [] 239 for i in range(50): 240 vnet_ifaces = run_cmd(cmd).strip().split(" ") 241 not_matched = [] 242 for iface_name in ifaces: 243 if iface_name not in vnet_ifaces: 244 not_matched.append(iface_name) 245 if len(not_matched) == 0: 246 return [] 247 time.sleep(0.1) 248 return not_matched 249 250 def create_vnet(self, vnet_alias: str, ifaces: List[VnetInterface]): 251 vnet_name = "pytest:{}".format(convert_test_name(self.topology_id)) 252 if self._vnets: 253 # add number to distinguish jails 254 vnet_name = "{}_{}".format(vnet_name, len(self._vnets) + 1) 255 iface_cmds = " ".join(["vnet.interface={}".format(i.name) for i in ifaces]) 256 cmd = "/usr/sbin/jail -i -c name={} persist vnet {}".format( 257 vnet_name, iface_cmds 258 ) 259 jid = 0 260 try: 261 jid_str = run_cmd(cmd) 262 jid = int(jid_str) 263 except ValueError as e: 264 print("Jail creation failed, output: {}".format(jid_str)) 265 raise 266 self._register_vnet(vnet_name) 267 268 # Run expedited version of routing 269 VnetInterface.setup_loopback(vnet_name) 270 271 not_found = self._wait_interfaces(vnet_name, [i.name for i in ifaces]) 272 if not_found: 273 raise Exception( 274 "Interfaces {} has not appeared in vnet {}".format(not_found, vnet_name) 275 ) 276 return VnetInstance(vnet_alias, vnet_name, jid, ifaces) 277 278 def cleanup(self): 279 try: 280 with open(self.file_name) as f: 281 for line in f: 282 vnet_name = line.strip() 283 ToolsHelper.print_output( 284 "/usr/sbin/jexec {} ifconfig -l".format(vnet_name) 285 ) 286 run_cmd("/usr/sbin/jail -r {}".format(vnet_name)) 287 os.unlink(self.JAILS_FNAME) 288 except OSError: 289 pass 290 291 292class SingleInterfaceMap(NamedTuple): 293 ifaces: List[VnetInterface] 294 vnet_aliases: List[str] 295 296 297class ObjectsMap(NamedTuple): 298 iface_map: Dict[str, SingleInterfaceMap] # keyed by ifX 299 vnet_map: Dict[str, VnetInstance] # keyed by vnetX 300 topo_map: Dict # self.TOPOLOGY 301 302 303class VnetTestTemplate(BaseTest): 304 TOPOLOGY = {} 305 306 def _get_vnet_handler(self, vnet_alias: str): 307 handler_name = "{}_handler".format(vnet_alias) 308 return getattr(self, handler_name, None) 309 310 def _setup_vnet(self, vnet: VnetInstance, obj_map: Dict, pipe): 311 """Base Handler to setup given VNET. 312 Can be run in a subprocess. If so, passes control to the special 313 vnetX_handler() after setting up interface addresses 314 """ 315 vnet.attach() 316 print("# setup_vnet({})".format(vnet.name)) 317 if pipe is not None: 318 vnet.set_pipe(pipe) 319 320 topo = obj_map.topo_map 321 ipv6_ifaces = [] 322 # Disable DAD 323 if not vnet.need_dad: 324 vnet.disable_dad() 325 for iface in vnet.ifaces: 326 # check index of vnet within an interface 327 # as we have prefixes for both ends of the interface 328 iface_map = obj_map.iface_map[iface.alias] 329 idx = iface_map.vnet_aliases.index(vnet.alias) 330 prefixes6 = topo[iface.alias].get("prefixes6", []) 331 prefixes4 = topo[iface.alias].get("prefixes4", []) 332 if prefixes6 or prefixes4: 333 ipv6_ifaces.append(iface) 334 iface.turn_up() 335 if prefixes6: 336 iface.enable_ipv6() 337 for prefix in prefixes6 + prefixes4: 338 iface.setup_addr(prefix[idx]) 339 for iface in ipv6_ifaces: 340 while iface.has_tentative(): 341 time.sleep(0.1) 342 343 # Run actual handler 344 handler = self._get_vnet_handler(vnet.alias) 345 if handler: 346 # Do unbuffered stdout for children 347 # so the logs are present if the child hangs 348 sys.stdout.reconfigure(line_buffering=True) 349 handler(vnet) 350 351 def setup_topology(self, topo: Dict, topology_id: str): 352 """Creates jails & interfaces for the provided topology""" 353 iface_map: Dict[str, SingleInterfaceMap] = {} 354 vnet_map = {} 355 iface_factory = IfaceFactory() 356 vnet_factory = VnetFactory(topology_id) 357 for obj_name, obj_data in topo.items(): 358 if obj_name.startswith("if"): 359 epair_ifaces = iface_factory.create_iface(obj_name, "epair") 360 smap = SingleInterfaceMap(epair_ifaces, []) 361 iface_map[obj_name] = smap 362 for obj_name, obj_data in topo.items(): 363 if obj_name.startswith("vnet"): 364 vnet_ifaces = [] 365 for iface_alias in obj_data["ifaces"]: 366 # epair creates 2 interfaces, grab first _available_ 367 # and map it to the VNET being created 368 idx = len(iface_map[iface_alias].vnet_aliases) 369 iface_map[iface_alias].vnet_aliases.append(obj_name) 370 vnet_ifaces.append(iface_map[iface_alias].ifaces[idx]) 371 vnet = vnet_factory.create_vnet(obj_name, vnet_ifaces) 372 vnet_map[obj_name] = vnet 373 # Debug output 374 print("============= TEST TOPOLOGY =============") 375 for vnet_alias, vnet in vnet_map.items(): 376 print("# vnet {} -> {}".format(vnet.alias, vnet.name), end="") 377 handler = self._get_vnet_handler(vnet.alias) 378 if handler: 379 print(" handler: {}".format(handler.__name__), end="") 380 print() 381 for iface_alias, iface_data in iface_map.items(): 382 vnets = iface_data.vnet_aliases 383 ifaces: List[VnetInterface] = iface_data.ifaces 384 if len(vnets) == 1 and len(ifaces) == 2: 385 print( 386 "# iface {}: {}::{} -> main::{}".format( 387 iface_alias, vnets[0], ifaces[0].name, ifaces[1].name 388 ) 389 ) 390 elif len(vnets) == 2 and len(ifaces) == 2: 391 print( 392 "# iface {}: {}::{} -> {}::{}".format( 393 iface_alias, vnets[0], ifaces[0].name, vnets[1], ifaces[1].name 394 ) 395 ) 396 else: 397 print( 398 "# iface {}: ifaces: {} vnets: {}".format( 399 iface_alias, vnets, [i.name for i in ifaces] 400 ) 401 ) 402 print() 403 return ObjectsMap(iface_map, vnet_map, topo) 404 405 def setup_method(self, _method): 406 """Sets up all the required topology and handlers for the given test""" 407 super().setup_method(_method) 408 # TestIP6Output.test_output6_pktinfo[ipandif] 409 topology_id = get_topology_id(self.test_id) 410 topology = self.TOPOLOGY 411 # First, setup kernel objects - interfaces & vnets 412 obj_map = self.setup_topology(topology, topology_id) 413 main_vnet = None # one without subprocess handler 414 for vnet_alias, vnet in obj_map.vnet_map.items(): 415 if self._get_vnet_handler(vnet_alias): 416 # Need subprocess to run 417 parent_pipe, child_pipe = Pipe() 418 p = Process( 419 target=self._setup_vnet, 420 args=( 421 vnet, 422 obj_map, 423 child_pipe, 424 ), 425 ) 426 vnet.set_pipe(parent_pipe) 427 vnet.set_subprocess(p) 428 p.start() 429 else: 430 if main_vnet is not None: 431 raise Exception("there can be only 1 VNET w/o handler") 432 main_vnet = vnet 433 # Main vnet needs to be the last, so all the other subprocesses 434 # are started & their pipe handles collected 435 self.vnet = main_vnet 436 self._setup_vnet(main_vnet, obj_map, None) 437 # Save state for the main handler 438 self.iface_map = obj_map.iface_map 439 self.vnet_map = obj_map.vnet_map 440 441 def cleanup(self, test_id: str): 442 # pytest test id: file::class::test_name 443 topology_id = get_topology_id(self.test_id) 444 445 print("==== vnet cleanup ===") 446 print("# topology_id: '{}'".format(topology_id)) 447 VnetFactory(topology_id).cleanup() 448 IfaceFactory().cleanup() 449 450 def wait_object(self, pipe, timeout=5): 451 if pipe.poll(timeout): 452 return pipe.recv() 453 raise TimeoutError 454 455 def send_object(self, pipe, obj): 456 pipe.send(obj) 457 458 @property 459 def curvnet(self): 460 pass 461 462 463class SingleVnetTestTemplate(VnetTestTemplate): 464 IPV6_PREFIXES: List[str] = [] 465 IPV4_PREFIXES: List[str] = [] 466 467 def setup_method(self, method): 468 topology = copy.deepcopy( 469 { 470 "vnet1": {"ifaces": ["if1"]}, 471 "if1": {"prefixes4": [], "prefixes6": []}, 472 } 473 ) 474 for prefix in self.IPV6_PREFIXES: 475 topology["if1"]["prefixes6"].append((prefix,)) 476 for prefix in self.IPV4_PREFIXES: 477 topology["if1"]["prefixes4"].append((prefix,)) 478 self.TOPOLOGY = topology 479 super().setup_method(method) 480