1#!/usr/local/bin/python3 2import copy 3import ipaddress 4import os 5import socket 6import sys 7import time 8from ctypes import cdll 9from ctypes import get_errno 10from ctypes.util import find_library 11from multiprocessing import Pipe 12from multiprocessing import Process 13from typing import Dict 14from typing import List 15from typing import NamedTuple 16from typing import Optional 17 18from atf_python.sys.net.tools import ToolsHelper 19 20 21def run_cmd(cmd: str, verbose=True) -> str: 22 print("run: '{}'".format(cmd)) 23 return os.popen(cmd).read() 24 25 26def convert_test_name(test_name: str) -> str: 27 """Convert test name to a string that can be used in the file/jail names""" 28 ret = "" 29 for char in test_name: 30 if char.isalnum() or char in ("_", "-"): 31 ret += char 32 elif char in ("["): 33 ret += "_" 34 return ret 35 36 37class VnetInterface(object): 38 # defines from net/if_types.h 39 IFT_LOOP = 0x18 40 IFT_ETHER = 0x06 41 42 def __init__(self, iface_alias: str, iface_name: str): 43 self.name = iface_name 44 self.alias = iface_alias 45 self.vnet_name = "" 46 self.jailed = False 47 self.addr_map: Dict[str, Dict] = {"inet6": {}, "inet": {}} 48 self.prefixes4: List[List[str]] = [] 49 self.prefixes6: List[List[str]] = [] 50 if iface_name.startswith("lo"): 51 self.iftype = self.IFT_LOOP 52 else: 53 self.iftype = self.IFT_ETHER 54 55 @property 56 def ifindex(self): 57 return socket.if_nametoindex(self.name) 58 59 @property 60 def first_ipv6(self): 61 d = self.addr_map["inet6"] 62 return d[next(iter(d))] 63 64 @property 65 def first_ipv4(self): 66 d = self.addr_map["inet"] 67 return d[next(iter(d))] 68 69 def set_vnet(self, vnet_name: str): 70 self.vnet_name = vnet_name 71 72 def set_jailed(self, jailed: bool): 73 self.jailed = jailed 74 75 def run_cmd( 76 self, 77 cmd, 78 verbose=False, 79 ): 80 if self.vnet_name and not self.jailed: 81 cmd = "jexec {} {}".format(self.vnet_name, cmd) 82 return run_cmd(cmd, verbose) 83 84 @classmethod 85 def setup_loopback(cls, vnet_name: str): 86 lo = VnetInterface("", "lo0") 87 lo.set_vnet(vnet_name) 88 lo.turn_up() 89 90 @classmethod 91 def create_iface(cls, alias_name: str, iface_name: str) -> List["VnetInterface"]: 92 name = run_cmd("/sbin/ifconfig {} create".format(iface_name)).rstrip() 93 if not name: 94 raise Exception("Unable to create iface {}".format(iface_name)) 95 ret = [cls(alias_name, name)] 96 if name.startswith("epair"): 97 ret.append(cls(alias_name, name[:-1] + "b")) 98 return ret 99 100 def setup_addr(self, _addr: str): 101 addr = ipaddress.ip_interface(_addr) 102 if addr.version == 6: 103 family = "inet6" 104 cmd = "/sbin/ifconfig {} {} {}".format(self.name, family, addr) 105 else: 106 family = "inet" 107 if self.addr_map[family]: 108 cmd = "/sbin/ifconfig {} alias {}".format(self.name, addr) 109 else: 110 cmd = "/sbin/ifconfig {} {} {}".format(self.name, family, addr) 111 self.run_cmd(cmd) 112 self.addr_map[family][str(addr.ip)] = addr 113 114 def delete_addr(self, _addr: str): 115 addr = ipaddress.ip_address(_addr) 116 if addr.version == 6: 117 family = "inet6" 118 cmd = "/sbin/ifconfig {} inet6 {} delete".format(self.name, addr) 119 else: 120 family = "inet" 121 cmd = "/sbin/ifconfig {} -alias {}".format(self.name, addr) 122 self.run_cmd(cmd) 123 del self.addr_map[family][str(addr)] 124 125 def turn_up(self): 126 cmd = "/sbin/ifconfig {} up".format(self.name) 127 self.run_cmd(cmd) 128 129 def enable_ipv6(self): 130 cmd = "/usr/sbin/ndp -i {} -disabled".format(self.name) 131 self.run_cmd(cmd) 132 133 def has_tentative(self) -> bool: 134 """True if an interface has some addresses in tenative state""" 135 cmd = "/sbin/ifconfig {} inet6".format(self.name) 136 out = self.run_cmd(cmd, verbose=False) 137 for line in out.splitlines(): 138 if "tentative" in line: 139 return True 140 return False 141 142 143class IfaceFactory(object): 144 INTERFACES_FNAME = "created_ifaces.lst" 145 146 def __init__(self, test_name: str): 147 self.test_name = test_name 148 test_id = convert_test_name(test_name) 149 self.file_name = self.INTERFACES_FNAME 150 151 def _register_iface(self, iface_name: str): 152 with open(self.file_name, "a") as f: 153 f.write(iface_name + "\n") 154 155 def create_iface(self, alias_name: str, iface_name: str) -> List[VnetInterface]: 156 ifaces = VnetInterface.create_iface(alias_name, iface_name) 157 for iface in ifaces: 158 self._register_iface(iface.name) 159 return ifaces 160 161 def cleanup(self): 162 try: 163 with open(self.file_name, "r") as f: 164 for line in f: 165 run_cmd("/sbin/ifconfig {} destroy".format(line.strip())) 166 os.unlink(self.INTERFACES_FNAME) 167 except Exception: 168 pass 169 170 171class VnetInstance(object): 172 def __init__( 173 self, vnet_alias: str, vnet_name: str, jid: int, ifaces: List[VnetInterface] 174 ): 175 self.name = vnet_name 176 self.alias = vnet_alias # reference in the test topology 177 self.jid = jid 178 self.ifaces = ifaces 179 self.iface_alias_map = {} # iface.alias: iface 180 self.iface_map = {} # iface.name: iface 181 for iface in ifaces: 182 iface.set_vnet(vnet_name) 183 iface.set_jailed(True) 184 self.iface_alias_map[iface.alias] = iface 185 self.iface_map[iface.name] = iface 186 self.need_dad = False # Disable duplicate address detection by default 187 self.attached = False 188 self.pipe = None 189 self.subprocess = None 190 191 def run_vnet_cmd(self, cmd): 192 if not self.attached: 193 cmd = "jexec {} {}".format(self.name, cmd) 194 return run_cmd(cmd) 195 196 def disable_dad(self): 197 self.run_vnet_cmd("/sbin/sysctl net.inet6.ip6.dad_count=0") 198 199 def set_pipe(self, pipe): 200 self.pipe = pipe 201 202 def set_subprocess(self, p): 203 self.subprocess = p 204 205 @staticmethod 206 def attach_jid(jid: int): 207 _path: Optional[str] = find_library("c") 208 if _path is None: 209 raise Exception("libc not found") 210 path: str = _path 211 libc = cdll.LoadLibrary(path) 212 if libc.jail_attach(jid) != 0: 213 raise Exception("jail_attach() failed: errno {}".format(get_errno())) 214 215 def attach(self): 216 self.attach_jid(self.jid) 217 self.attached = True 218 219 220class VnetFactory(object): 221 JAILS_FNAME = "created_jails.lst" 222 223 def __init__(self, test_name: str): 224 self.test_name = test_name 225 self.test_id = convert_test_name(test_name) 226 self.file_name = self.JAILS_FNAME 227 self._vnets: List[str] = [] 228 229 def _register_vnet(self, vnet_name: str): 230 self._vnets.append(vnet_name) 231 with open(self.file_name, "a") as f: 232 f.write(vnet_name + "\n") 233 234 @staticmethod 235 def _wait_interfaces(vnet_name: str, ifaces: List[str]) -> List[str]: 236 cmd = "jexec {} /sbin/ifconfig -l".format(vnet_name) 237 not_matched: List[str] = [] 238 for i in range(50): 239 vnet_ifaces = run_cmd(cmd).strip().split(" ") 240 not_matched = [] 241 for iface_name in ifaces: 242 if iface_name not in vnet_ifaces: 243 not_matched.append(iface_name) 244 if len(not_matched) == 0: 245 return [] 246 time.sleep(0.1) 247 return not_matched 248 249 def create_vnet(self, vnet_alias: str, ifaces: List[VnetInterface]): 250 vnet_name = "jail_{}".format(self.test_id) 251 if self._vnets: 252 # add number to distinguish jails 253 vnet_name = "{}_{}".format(vnet_name, len(self._vnets) + 1) 254 iface_cmds = " ".join(["vnet.interface={}".format(i.name) for i in ifaces]) 255 cmd = "/usr/sbin/jail -i -c name={} persist vnet {}".format( 256 vnet_name, iface_cmds 257 ) 258 jid_str = run_cmd(cmd) 259 jid = int(jid_str) 260 if jid <= 0: 261 raise Exception("Jail creation failed, output: {}".format(jid)) 262 self._register_vnet(vnet_name) 263 264 # Run expedited version of routing 265 VnetInterface.setup_loopback(vnet_name) 266 267 not_found = self._wait_interfaces(vnet_name, [i.name for i in ifaces]) 268 if not_found: 269 raise Exception( 270 "Interfaces {} has not appeared in vnet {}".format(not_found, vnet_name) 271 ) 272 return VnetInstance(vnet_alias, vnet_name, jid, ifaces) 273 274 def cleanup(self): 275 try: 276 with open(self.file_name) as f: 277 for line in f: 278 jail_name = line.strip() 279 ToolsHelper.print_output( 280 "/usr/sbin/jexec {} ifconfig -l".format(jail_name) 281 ) 282 run_cmd("/usr/sbin/jail -r {}".format(line.strip())) 283 os.unlink(self.JAILS_FNAME) 284 except OSError: 285 pass 286 287 288class SingleInterfaceMap(NamedTuple): 289 ifaces: List[VnetInterface] 290 vnet_aliases: List[str] 291 292 293class VnetTestTemplate(object): 294 TOPOLOGY = {} 295 296 def _get_vnet_handler(self, vnet_alias: str): 297 handler_name = "{}_handler".format(vnet_alias) 298 return getattr(self, handler_name, None) 299 300 def _setup_vnet(self, vnet: VnetInstance, obj_map: Dict, pipe): 301 """Base Handler to setup given VNET. 302 Can be run in a subprocess. If so, passes control to the special 303 vnetX_handler() after setting up interface addresses 304 """ 305 vnet.attach() 306 print("# setup_vnet({})".format(vnet.name)) 307 308 topo = obj_map["topo_map"] 309 ipv6_ifaces = [] 310 # Disable DAD 311 if not vnet.need_dad: 312 vnet.disable_dad() 313 for iface in vnet.ifaces: 314 # check index of vnet within an interface 315 # as we have prefixes for both ends of the interface 316 iface_map = obj_map["iface_map"][iface.alias] 317 idx = iface_map.vnet_aliases.index(vnet.alias) 318 prefixes6 = topo[iface.alias].get("prefixes6", []) 319 prefixes4 = topo[iface.alias].get("prefixes4", []) 320 if prefixes6 or prefixes4: 321 ipv6_ifaces.append(iface) 322 iface.turn_up() 323 if prefixes6: 324 iface.enable_ipv6() 325 for prefix in prefixes6 + prefixes4: 326 iface.setup_addr(prefix[idx]) 327 for iface in ipv6_ifaces: 328 while iface.has_tentative(): 329 time.sleep(0.1) 330 331 # Run actual handler 332 handler = self._get_vnet_handler(vnet.alias) 333 if handler: 334 # Do unbuffered stdout for children 335 # so the logs are present if the child hangs 336 sys.stdout.reconfigure(line_buffering=True) 337 handler(vnet, obj_map, pipe) 338 339 def setup_topology(self, topo: Dict, test_name: str): 340 """Creates jails & interfaces for the provided topology""" 341 iface_map: Dict[str, SingleInterfaceMap] = {} 342 vnet_map = {} 343 iface_factory = IfaceFactory(test_name) 344 vnet_factory = VnetFactory(test_name) 345 for obj_name, obj_data in topo.items(): 346 if obj_name.startswith("if"): 347 epair_ifaces = iface_factory.create_iface(obj_name, "epair") 348 smap = SingleInterfaceMap(epair_ifaces, []) 349 iface_map[obj_name] = smap 350 for obj_name, obj_data in topo.items(): 351 if obj_name.startswith("vnet"): 352 vnet_ifaces = [] 353 for iface_alias in obj_data["ifaces"]: 354 # epair creates 2 interfaces, grab first _available_ 355 # and map it to the VNET being created 356 idx = len(iface_map[iface_alias].vnet_aliases) 357 iface_map[iface_alias].vnet_aliases.append(obj_name) 358 vnet_ifaces.append(iface_map[iface_alias].ifaces[idx]) 359 vnet = vnet_factory.create_vnet(obj_name, vnet_ifaces) 360 vnet_map[obj_name] = vnet 361 # Debug output 362 print("============= TEST TOPOLOGY =============") 363 for vnet_alias, vnet in vnet_map.items(): 364 print("# vnet {} -> {}".format(vnet.alias, vnet.name), end="") 365 handler = self._get_vnet_handler(vnet.alias) 366 if handler: 367 print(" handler: {}".format(handler.__name__), end="") 368 print() 369 for iface_alias, iface_data in iface_map.items(): 370 vnets = iface_data.vnet_aliases 371 ifaces: List[VnetInterface] = iface_data.ifaces 372 if len(vnets) == 1 and len(ifaces) == 2: 373 print( 374 "# iface {}: {}::{} -> main::{}".format( 375 iface_alias, vnets[0], ifaces[0].name, ifaces[1].name 376 ) 377 ) 378 elif len(vnets) == 2 and len(ifaces) == 2: 379 print( 380 "# iface {}: {}::{} -> {}::{}".format( 381 iface_alias, vnets[0], ifaces[0].name, vnets[1], ifaces[1].name 382 ) 383 ) 384 else: 385 print( 386 "# iface {}: ifaces: {} vnets: {}".format( 387 iface_alias, vnets, [i.name for i in ifaces] 388 ) 389 ) 390 print() 391 return {"iface_map": iface_map, "vnet_map": vnet_map, "topo_map": topo} 392 393 def setup_method(self, method): 394 """Sets up all the required topology and handlers for the given test""" 395 # 'test_ip6_output.py::TestIP6Output::test_output6_pktinfo[ipandif] (setup)' 396 test_id = os.environ.get("PYTEST_CURRENT_TEST").split(" ")[0] 397 test_name = test_id.split("::")[-1] 398 topology = self.TOPOLOGY 399 # First, setup kernel objects - interfaces & vnets 400 obj_map = self.setup_topology(topology, test_name) 401 main_vnet = None # one without subprocess handler 402 for vnet_alias, vnet in obj_map["vnet_map"].items(): 403 if self._get_vnet_handler(vnet_alias): 404 # Need subprocess to run 405 parent_pipe, child_pipe = Pipe() 406 p = Process( 407 target=self._setup_vnet, 408 args=( 409 vnet, 410 obj_map, 411 child_pipe, 412 ), 413 ) 414 vnet.set_pipe(parent_pipe) 415 vnet.set_subprocess(p) 416 p.start() 417 else: 418 if main_vnet is not None: 419 raise Exception("there can be only 1 VNET w/o handler") 420 main_vnet = vnet 421 # Main vnet needs to be the last, so all the other subprocesses 422 # are started & their pipe handles collected 423 self.vnet = main_vnet 424 self._setup_vnet(main_vnet, obj_map, None) 425 # Save state for the main handler 426 self.iface_map = obj_map["iface_map"] 427 self.vnet_map = obj_map["vnet_map"] 428 429 def cleanup(self, test_id: str): 430 # pytest test id: file::class::test_name 431 test_name = test_id.split("::")[-1] 432 433 print("==== vnet cleanup ===") 434 print("# test_name: '{}'".format(test_name)) 435 VnetFactory(test_name).cleanup() 436 IfaceFactory(test_name).cleanup() 437 438 def wait_object(self, pipe, timeout=5): 439 if pipe.poll(timeout): 440 return pipe.recv() 441 raise TimeoutError 442 443 @property 444 def curvnet(self): 445 pass 446 447 448class SingleVnetTestTemplate(VnetTestTemplate): 449 IPV6_PREFIXES: List[str] = [] 450 IPV4_PREFIXES: List[str] = [] 451 452 def setup_method(self, method): 453 topology = copy.deepcopy( 454 { 455 "vnet1": {"ifaces": ["if1"]}, 456 "if1": {"prefixes4": [], "prefixes6": []}, 457 } 458 ) 459 for prefix in self.IPV6_PREFIXES: 460 topology["if1"]["prefixes6"].append((prefix,)) 461 for prefix in self.IPV4_PREFIXES: 462 topology["if1"]["prefixes4"].append((prefix,)) 463 self.TOPOLOGY = topology 464 super().setup_method(method) 465