1#!/usr/local/bin/python3 2import copy 3import ipaddress 4import os 5import socket 6import sys 7import time 8from multiprocessing import Pipe 9from multiprocessing import Process 10from typing import Dict 11from typing import List 12from typing import NamedTuple 13 14from atf_python.sys.net.tools import ToolsHelper 15from atf_python.utils import BaseTest 16from atf_python.utils import libc 17 18 19def run_cmd(cmd: str, verbose=True) -> str: 20 print("run: '{}'".format(cmd)) 21 return os.popen(cmd).read() 22 23 24def get_topology_id(test_id: str) -> str: 25 """ 26 Gets a unique topology id based on the pytest test_id. 27 "test_ip6_output.py::TestIP6Output::test_output6_pktinfo[ipandif]" -> 28 "TestIP6Output:test_output6_pktinfo[ipandif]" 29 """ 30 return ":".join(test_id.split("::")[-2:]) 31 32 33def convert_test_name(test_name: str) -> str: 34 """Convert test name to a string that can be used in the file/jail names""" 35 ret = "" 36 for char in test_name: 37 if char.isalnum() or char in ("_", "-", ":"): 38 ret += char 39 elif char in ("["): 40 ret += "_" 41 return ret 42 43 44class VnetInterface(object): 45 # defines from net/if_types.h 46 IFT_LOOP = 0x18 47 IFT_ETHER = 0x06 48 49 def __init__(self, iface_alias: str, iface_name: str): 50 self.name = iface_name 51 self.alias = iface_alias 52 self.vnet_name = "" 53 self.jailed = False 54 self.addr_map: Dict[str, Dict] = {"inet6": {}, "inet": {}} 55 self.prefixes4: List[List[str]] = [] 56 self.prefixes6: List[List[str]] = [] 57 if iface_name.startswith("lo"): 58 self.iftype = self.IFT_LOOP 59 else: 60 self.iftype = self.IFT_ETHER 61 62 @property 63 def ifindex(self): 64 return socket.if_nametoindex(self.name) 65 66 @property 67 def first_ipv6(self): 68 d = self.addr_map["inet6"] 69 return d[next(iter(d))] 70 71 @property 72 def first_ipv4(self): 73 d = self.addr_map["inet"] 74 return d[next(iter(d))] 75 76 def set_vnet(self, vnet_name: str): 77 self.vnet_name = vnet_name 78 79 def set_jailed(self, jailed: bool): 80 self.jailed = jailed 81 82 def run_cmd( 83 self, 84 cmd, 85 verbose=False, 86 ): 87 if self.vnet_name and not self.jailed: 88 cmd = "jexec {} {}".format(self.vnet_name, cmd) 89 return run_cmd(cmd, verbose) 90 91 @classmethod 92 def setup_loopback(cls, vnet_name: str): 93 lo = VnetInterface("", "lo0") 94 lo.set_vnet(vnet_name) 95 lo.turn_up() 96 97 @classmethod 98 def create_iface(cls, alias_name: str, iface_name: str) -> List["VnetInterface"]: 99 name = run_cmd("/sbin/ifconfig {} create".format(iface_name)).rstrip() 100 if not name: 101 raise Exception("Unable to create iface {}".format(iface_name)) 102 ret = [cls(alias_name, name)] 103 if name.startswith("epair"): 104 ret.append(cls(alias_name, name[:-1] + "b")) 105 return ret 106 107 def setup_addr(self, _addr: str): 108 addr = ipaddress.ip_interface(_addr) 109 if addr.version == 6: 110 family = "inet6" 111 cmd = "/sbin/ifconfig {} {} {}".format(self.name, family, addr) 112 else: 113 family = "inet" 114 if self.addr_map[family]: 115 cmd = "/sbin/ifconfig {} alias {}".format(self.name, addr) 116 else: 117 cmd = "/sbin/ifconfig {} {} {}".format(self.name, family, addr) 118 self.run_cmd(cmd) 119 self.addr_map[family][str(addr.ip)] = addr 120 121 def delete_addr(self, _addr: str): 122 addr = ipaddress.ip_address(_addr) 123 if addr.version == 6: 124 family = "inet6" 125 cmd = "/sbin/ifconfig {} inet6 {} delete".format(self.name, addr) 126 else: 127 family = "inet" 128 cmd = "/sbin/ifconfig {} -alias {}".format(self.name, addr) 129 self.run_cmd(cmd) 130 del self.addr_map[family][str(addr)] 131 132 def turn_up(self): 133 cmd = "/sbin/ifconfig {} up".format(self.name) 134 self.run_cmd(cmd) 135 136 def enable_ipv6(self): 137 cmd = "/usr/sbin/ndp -i {} -disabled".format(self.name) 138 self.run_cmd(cmd) 139 140 def has_tentative(self) -> bool: 141 """True if an interface has some addresses in tenative state""" 142 cmd = "/sbin/ifconfig {} inet6".format(self.name) 143 out = self.run_cmd(cmd, verbose=False) 144 for line in out.splitlines(): 145 if "tentative" in line: 146 return True 147 return False 148 149 150class IfaceFactory(object): 151 INTERFACES_FNAME = "created_ifaces.lst" 152 153 def __init__(self): 154 self.file_name = self.INTERFACES_FNAME 155 156 def _register_iface(self, iface_name: str): 157 with open(self.file_name, "a") as f: 158 f.write(iface_name + "\n") 159 160 def create_iface(self, alias_name: str, iface_name: str) -> List[VnetInterface]: 161 ifaces = VnetInterface.create_iface(alias_name, iface_name) 162 for iface in ifaces: 163 self._register_iface(iface.name) 164 return ifaces 165 166 def cleanup(self): 167 try: 168 with open(self.file_name, "r") as f: 169 for line in f: 170 run_cmd("/sbin/ifconfig {} destroy".format(line.strip())) 171 os.unlink(self.INTERFACES_FNAME) 172 except Exception: 173 pass 174 175 176class VnetInstance(object): 177 def __init__( 178 self, vnet_alias: str, vnet_name: str, jid: int, ifaces: List[VnetInterface] 179 ): 180 self.name = vnet_name 181 self.alias = vnet_alias # reference in the test topology 182 self.jid = jid 183 self.ifaces = ifaces 184 self.iface_alias_map = {} # iface.alias: iface 185 self.iface_map = {} # iface.name: iface 186 for iface in ifaces: 187 iface.set_vnet(vnet_name) 188 iface.set_jailed(True) 189 self.iface_alias_map[iface.alias] = iface 190 self.iface_map[iface.name] = iface 191 self.need_dad = False # Disable duplicate address detection by default 192 self.attached = False 193 self.pipe = None 194 self.subprocess = None 195 196 def run_vnet_cmd(self, cmd): 197 if not self.attached: 198 cmd = "jexec {} {}".format(self.name, cmd) 199 return run_cmd(cmd) 200 201 def disable_dad(self): 202 self.run_vnet_cmd("/sbin/sysctl net.inet6.ip6.dad_count=0") 203 204 def set_pipe(self, pipe): 205 self.pipe = pipe 206 207 def set_subprocess(self, p): 208 self.subprocess = p 209 210 @staticmethod 211 def attach_jid(jid: int): 212 error_code = libc.jail_attach(jid) 213 if error_code != 0: 214 raise Exception("jail_attach() failed: errno {}".format(error_code)) 215 216 def attach(self): 217 self.attach_jid(self.jid) 218 self.attached = True 219 220 221class VnetFactory(object): 222 JAILS_FNAME = "created_jails.lst" 223 224 def __init__(self, topology_id: str): 225 self.topology_id = topology_id 226 self.file_name = self.JAILS_FNAME 227 self._vnets: List[str] = [] 228 229 def _register_vnet(self, vnet_name: str): 230 self._vnets.append(vnet_name) 231 with open(self.file_name, "a") as f: 232 f.write(vnet_name + "\n") 233 234 @staticmethod 235 def _wait_interfaces(vnet_name: str, ifaces: List[str]) -> List[str]: 236 cmd = "jexec {} /sbin/ifconfig -l".format(vnet_name) 237 not_matched: List[str] = [] 238 for i in range(50): 239 vnet_ifaces = run_cmd(cmd).strip().split(" ") 240 not_matched = [] 241 for iface_name in ifaces: 242 if iface_name not in vnet_ifaces: 243 not_matched.append(iface_name) 244 if len(not_matched) == 0: 245 return [] 246 time.sleep(0.1) 247 return not_matched 248 249 def create_vnet(self, vnet_alias: str, ifaces: List[VnetInterface]): 250 vnet_name = "pytest:{}".format(convert_test_name(self.topology_id)) 251 if self._vnets: 252 # add number to distinguish jails 253 vnet_name = "{}_{}".format(vnet_name, len(self._vnets) + 1) 254 iface_cmds = " ".join(["vnet.interface={}".format(i.name) for i in ifaces]) 255 cmd = "/usr/sbin/jail -i -c name={} persist vnet {}".format( 256 vnet_name, iface_cmds 257 ) 258 jid = 0 259 try: 260 jid_str = run_cmd(cmd) 261 jid = int(jid_str) 262 except ValueError as e: 263 print("Jail creation failed, output: {}".format(jid_str)) 264 raise 265 self._register_vnet(vnet_name) 266 267 # Run expedited version of routing 268 VnetInterface.setup_loopback(vnet_name) 269 270 not_found = self._wait_interfaces(vnet_name, [i.name for i in ifaces]) 271 if not_found: 272 raise Exception( 273 "Interfaces {} has not appeared in vnet {}".format(not_found, vnet_name) 274 ) 275 return VnetInstance(vnet_alias, vnet_name, jid, ifaces) 276 277 def cleanup(self): 278 try: 279 with open(self.file_name) as f: 280 for line in f: 281 vnet_name = line.strip() 282 ToolsHelper.print_output( 283 "/usr/sbin/jexec {} ifconfig -l".format(vnet_name) 284 ) 285 run_cmd("/usr/sbin/jail -r {}".format(vnet_name)) 286 os.unlink(self.JAILS_FNAME) 287 except OSError: 288 pass 289 290 291class SingleInterfaceMap(NamedTuple): 292 ifaces: List[VnetInterface] 293 vnet_aliases: List[str] 294 295 296class ObjectsMap(NamedTuple): 297 iface_map: Dict[str, SingleInterfaceMap] # keyed by ifX 298 vnet_map: Dict[str, VnetInstance] # keyed by vnetX 299 topo_map: Dict # self.TOPOLOGY 300 301 302class VnetTestTemplate(BaseTest): 303 TOPOLOGY = {} 304 305 def _get_vnet_handler(self, vnet_alias: str): 306 handler_name = "{}_handler".format(vnet_alias) 307 return getattr(self, handler_name, None) 308 309 def _setup_vnet(self, vnet: VnetInstance, obj_map: Dict, pipe): 310 """Base Handler to setup given VNET. 311 Can be run in a subprocess. If so, passes control to the special 312 vnetX_handler() after setting up interface addresses 313 """ 314 vnet.attach() 315 print("# setup_vnet({})".format(vnet.name)) 316 if pipe is not None: 317 vnet.set_pipe(pipe) 318 319 topo = obj_map.topo_map 320 ipv6_ifaces = [] 321 # Disable DAD 322 if not vnet.need_dad: 323 vnet.disable_dad() 324 for iface in vnet.ifaces: 325 # check index of vnet within an interface 326 # as we have prefixes for both ends of the interface 327 iface_map = obj_map.iface_map[iface.alias] 328 idx = iface_map.vnet_aliases.index(vnet.alias) 329 prefixes6 = topo[iface.alias].get("prefixes6", []) 330 prefixes4 = topo[iface.alias].get("prefixes4", []) 331 if prefixes6 or prefixes4: 332 ipv6_ifaces.append(iface) 333 iface.turn_up() 334 if prefixes6: 335 iface.enable_ipv6() 336 for prefix in prefixes6 + prefixes4: 337 iface.setup_addr(prefix[idx]) 338 for iface in ipv6_ifaces: 339 while iface.has_tentative(): 340 time.sleep(0.1) 341 342 # Run actual handler 343 handler = self._get_vnet_handler(vnet.alias) 344 if handler: 345 # Do unbuffered stdout for children 346 # so the logs are present if the child hangs 347 sys.stdout.reconfigure(line_buffering=True) 348 handler(vnet) 349 350 def setup_topology(self, topo: Dict, topology_id: str): 351 """Creates jails & interfaces for the provided topology""" 352 iface_map: Dict[str, SingleInterfaceMap] = {} 353 vnet_map = {} 354 iface_factory = IfaceFactory() 355 vnet_factory = VnetFactory(topology_id) 356 for obj_name, obj_data in topo.items(): 357 if obj_name.startswith("if"): 358 epair_ifaces = iface_factory.create_iface(obj_name, "epair") 359 smap = SingleInterfaceMap(epair_ifaces, []) 360 iface_map[obj_name] = smap 361 for obj_name, obj_data in topo.items(): 362 if obj_name.startswith("vnet"): 363 vnet_ifaces = [] 364 for iface_alias in obj_data["ifaces"]: 365 # epair creates 2 interfaces, grab first _available_ 366 # and map it to the VNET being created 367 idx = len(iface_map[iface_alias].vnet_aliases) 368 iface_map[iface_alias].vnet_aliases.append(obj_name) 369 vnet_ifaces.append(iface_map[iface_alias].ifaces[idx]) 370 vnet = vnet_factory.create_vnet(obj_name, vnet_ifaces) 371 vnet_map[obj_name] = vnet 372 # Debug output 373 print("============= TEST TOPOLOGY =============") 374 for vnet_alias, vnet in vnet_map.items(): 375 print("# vnet {} -> {}".format(vnet.alias, vnet.name), end="") 376 handler = self._get_vnet_handler(vnet.alias) 377 if handler: 378 print(" handler: {}".format(handler.__name__), end="") 379 print() 380 for iface_alias, iface_data in iface_map.items(): 381 vnets = iface_data.vnet_aliases 382 ifaces: List[VnetInterface] = iface_data.ifaces 383 if len(vnets) == 1 and len(ifaces) == 2: 384 print( 385 "# iface {}: {}::{} -> main::{}".format( 386 iface_alias, vnets[0], ifaces[0].name, ifaces[1].name 387 ) 388 ) 389 elif len(vnets) == 2 and len(ifaces) == 2: 390 print( 391 "# iface {}: {}::{} -> {}::{}".format( 392 iface_alias, vnets[0], ifaces[0].name, vnets[1], ifaces[1].name 393 ) 394 ) 395 else: 396 print( 397 "# iface {}: ifaces: {} vnets: {}".format( 398 iface_alias, vnets, [i.name for i in ifaces] 399 ) 400 ) 401 print() 402 return ObjectsMap(iface_map, vnet_map, topo) 403 404 def setup_method(self, _method): 405 """Sets up all the required topology and handlers for the given test""" 406 super().setup_method(_method) 407 # TestIP6Output.test_output6_pktinfo[ipandif] 408 topology_id = get_topology_id(self.test_id) 409 topology = self.TOPOLOGY 410 # First, setup kernel objects - interfaces & vnets 411 obj_map = self.setup_topology(topology, topology_id) 412 main_vnet = None # one without subprocess handler 413 for vnet_alias, vnet in obj_map.vnet_map.items(): 414 if self._get_vnet_handler(vnet_alias): 415 # Need subprocess to run 416 parent_pipe, child_pipe = Pipe() 417 p = Process( 418 target=self._setup_vnet, 419 args=( 420 vnet, 421 obj_map, 422 child_pipe, 423 ), 424 ) 425 vnet.set_pipe(parent_pipe) 426 vnet.set_subprocess(p) 427 p.start() 428 else: 429 if main_vnet is not None: 430 raise Exception("there can be only 1 VNET w/o handler") 431 main_vnet = vnet 432 # Main vnet needs to be the last, so all the other subprocesses 433 # are started & their pipe handles collected 434 self.vnet = main_vnet 435 self._setup_vnet(main_vnet, obj_map, None) 436 # Save state for the main handler 437 self.iface_map = obj_map.iface_map 438 self.vnet_map = obj_map.vnet_map 439 440 def cleanup(self, test_id: str): 441 # pytest test id: file::class::test_name 442 topology_id = get_topology_id(self.test_id) 443 444 print("==== vnet cleanup ===") 445 print("# topology_id: '{}'".format(topology_id)) 446 VnetFactory(topology_id).cleanup() 447 IfaceFactory().cleanup() 448 449 def wait_object(self, pipe, timeout=5): 450 if pipe.poll(timeout): 451 return pipe.recv() 452 raise TimeoutError 453 454 def send_object(self, pipe, obj): 455 pipe.send(obj) 456 457 @property 458 def curvnet(self): 459 pass 460 461 462class SingleVnetTestTemplate(VnetTestTemplate): 463 IPV6_PREFIXES: List[str] = [] 464 IPV4_PREFIXES: List[str] = [] 465 466 def setup_method(self, method): 467 topology = copy.deepcopy( 468 { 469 "vnet1": {"ifaces": ["if1"]}, 470 "if1": {"prefixes4": [], "prefixes6": []}, 471 } 472 ) 473 for prefix in self.IPV6_PREFIXES: 474 topology["if1"]["prefixes6"].append((prefix,)) 475 for prefix in self.IPV4_PREFIXES: 476 topology["if1"]["prefixes4"].append((prefix,)) 477 self.TOPOLOGY = topology 478 super().setup_method(method) 479