1.. SPDX-License-Identifier: GPL-2.0 2 3====================== 4Multi-Level Interleave 5====================== 6This cxl-cli configuration dump shows the following host configuration: 7 8* A single socket system with one CXL root 9* CXL Root has Four (4) CXL Host Bridges 10* Two CXL Host Bridges have a two CXL Memory Expanders Attached each. 11* The CXL root is configured to interleave across the two host bridges. 12* Each host bridge with expanders interleaves across two endpoints. 13 14This output is generated by :code:`cxl list -v` and describes the relationships 15between objects exposed in :code:`/sys/bus/cxl/devices/`. 16 17:: 18 19 [ 20 { 21 "bus":"root0", 22 "provider":"ACPI.CXL", 23 "nr_dports":4, 24 "dports":[ 25 { 26 "dport":"pci0000:00", 27 "alias":"ACPI0016:01", 28 "id":0 29 }, 30 { 31 "dport":"pci0000:a8", 32 "alias":"ACPI0016:02", 33 "id":4 34 }, 35 { 36 "dport":"pci0000:2a", 37 "alias":"ACPI0016:03", 38 "id":1 39 }, 40 { 41 "dport":"pci0000:d2", 42 "alias":"ACPI0016:00", 43 "id":5 44 } 45 ], 46 47This chunk shows the CXL "bus" (root0) has 4 downstream ports attached to CXL 48Host Bridges. The `Root` can be considered the singular upstream port attached 49to the platform's memory controller - which routes memory requests to it. 50 51The `ports:root0` section lays out how each of these downstream ports are 52configured. If a port is not configured (id's 0 and 1), they are omitted. 53 54:: 55 56 "ports:root0":[ 57 { 58 "port":"port1", 59 "host":"pci0000:d2", 60 "depth":1, 61 "nr_dports":3, 62 "dports":[ 63 { 64 "dport":"0000:d2:01.1", 65 "alias":"device:02", 66 "id":0 67 }, 68 { 69 "dport":"0000:d2:01.3", 70 "alias":"device:05", 71 "id":2 72 }, 73 { 74 "dport":"0000:d2:07.1", 75 "alias":"device:0d", 76 "id":113 77 } 78 ], 79 80This chunk shows the available downstream ports associated with the CXL Host 81Bridge :code:`port1`. In this case, :code:`port1` has 3 available downstream 82ports: :code:`dport0`, :code:`dport2`, and :code:`dport113`. 83 84:: 85 86 "endpoints:port1":[ 87 { 88 "endpoint":"endpoint5", 89 "host":"mem0", 90 "parent_dport":"0000:d2:01.1", 91 "depth":2, 92 "memdev":{ 93 "memdev":"mem0", 94 "ram_size":137438953472, 95 "serial":0, 96 "numa_node":0, 97 "host":"0000:d3:00.0" 98 }, 99 "decoders:endpoint5":[ 100 { 101 "decoder":"decoder5.0", 102 "resource":825975898112, 103 "size":549755813888, 104 "interleave_ways":4, 105 "interleave_granularity":256, 106 "region":"region0", 107 "dpa_resource":0, 108 "dpa_size":137438953472, 109 "mode":"ram" 110 } 111 ] 112 }, 113 { 114 "endpoint":"endpoint6", 115 "host":"mem1", 116 "parent_dport":"0000:d2:01.3", 117 "depth":2, 118 "memdev":{ 119 "memdev":"mem1", 120 "ram_size":137438953472, 121 "serial":0, 122 "numa_node":0, 123 "host":"0000:d3:00.0" 124 }, 125 "decoders:endpoint6":[ 126 { 127 "decoder":"decoder6.0", 128 "resource":825975898112, 129 "size":549755813888, 130 "interleave_ways":4, 131 "interleave_granularity":256, 132 "region":"region0", 133 "dpa_resource":0, 134 "dpa_size":137438953472, 135 "mode":"ram" 136 } 137 ] 138 } 139 ], 140 141This chunk shows the endpoints attached to the host bridge :code:`port1`. 142 143:code:`endpoint5` contains a single configured decoder :code:`decoder5.0` 144which has the same interleave configuration as :code:`region0` (shown later). 145 146:code:`endpoint6` contains a single configured decoder :code:`decoder5.0` 147which has the same interleave configuration as :code:`region0` (shown later). 148 149Next we have the decoders belonging to the host bridge: 150 151:: 152 153 "decoders:port1":[ 154 { 155 "decoder":"decoder1.0", 156 "resource":825975898112, 157 "size":549755813888, 158 "interleave_ways":2, 159 "interleave_granularity":512, 160 "region":"region0", 161 "nr_targets":2, 162 "targets":[ 163 { 164 "target":"0000:d2:01.1", 165 "alias":"device:02", 166 "position":0, 167 "id":0 168 }, 169 { 170 "target":"0000:d2:01.3", 171 "alias":"device:05", 172 "position":2, 173 "id":0 174 } 175 ] 176 } 177 ] 178 }, 179 180Host Bridge :code:`port1` has a single decoder (:code:`decoder1.0`), whose 181targets are :code:`dport0` and :code:`dport2` - which are attached to 182:code:`endpoint5` and :code:`endpoint6` respectively. 183 184The following chunk shows a similar configuration for Host Bridge :code:`port3`, 185the second host bridge with a memory device attached. 186 187:: 188 189 { 190 "port":"port3", 191 "host":"pci0000:a8", 192 "depth":1, 193 "nr_dports":1, 194 "dports":[ 195 { 196 "dport":"0000:a8:01.1", 197 "alias":"device:c3", 198 "id":0 199 }, 200 { 201 "dport":"0000:a8:01.3", 202 "alias":"device:c5", 203 "id":0 204 } 205 ], 206 "endpoints:port3":[ 207 { 208 "endpoint":"endpoint7", 209 "host":"mem2", 210 "parent_dport":"0000:a8:01.1", 211 "depth":2, 212 "memdev":{ 213 "memdev":"mem2", 214 "ram_size":137438953472, 215 "serial":0, 216 "numa_node":0, 217 "host":"0000:a9:00.0" 218 }, 219 "decoders:endpoint7":[ 220 { 221 "decoder":"decoder7.0", 222 "resource":825975898112, 223 "size":549755813888, 224 "interleave_ways":4, 225 "interleave_granularity":256, 226 "region":"region0", 227 "dpa_resource":0, 228 "dpa_size":137438953472, 229 "mode":"ram" 230 } 231 ] 232 }, 233 { 234 "endpoint":"endpoint8", 235 "host":"mem3", 236 "parent_dport":"0000:a8:01.3", 237 "depth":2, 238 "memdev":{ 239 "memdev":"mem3", 240 "ram_size":137438953472, 241 "serial":0, 242 "numa_node":0, 243 "host":"0000:a9:00.0" 244 }, 245 "decoders:endpoint8":[ 246 { 247 "decoder":"decoder8.0", 248 "resource":825975898112, 249 "size":549755813888, 250 "interleave_ways":4, 251 "interleave_granularity":256, 252 "region":"region0", 253 "dpa_resource":0, 254 "dpa_size":137438953472, 255 "mode":"ram" 256 } 257 ] 258 } 259 ], 260 "decoders:port3":[ 261 { 262 "decoder":"decoder3.0", 263 "resource":825975898112, 264 "size":549755813888, 265 "interleave_ways":2, 266 "interleave_granularity":512, 267 "region":"region0", 268 "nr_targets":1, 269 "targets":[ 270 { 271 "target":"0000:a8:01.1", 272 "alias":"device:c3", 273 "position":1, 274 "id":0 275 }, 276 { 277 "target":"0000:a8:01.3", 278 "alias":"device:c5", 279 "position":3, 280 "id":0 281 } 282 ] 283 } 284 ] 285 }, 286 287 288The next chunk shows the two CXL host bridges without attached endpoints. 289 290:: 291 292 { 293 "port":"port2", 294 "host":"pci0000:00", 295 "depth":1, 296 "nr_dports":2, 297 "dports":[ 298 { 299 "dport":"0000:00:01.3", 300 "alias":"device:55", 301 "id":2 302 }, 303 { 304 "dport":"0000:00:07.1", 305 "alias":"device:5d", 306 "id":113 307 } 308 ] 309 }, 310 { 311 "port":"port4", 312 "host":"pci0000:2a", 313 "depth":1, 314 "nr_dports":1, 315 "dports":[ 316 { 317 "dport":"0000:2a:01.1", 318 "alias":"device:d0", 319 "id":0 320 } 321 ] 322 } 323 ], 324 325Next we have the `Root Decoders` belonging to :code:`root0`. This root decoder 326applies the interleave across the downstream ports :code:`port1` and 327:code:`port3` - with a granularity of 256 bytes. 328 329This information is generated by the CXL driver reading the ACPI CEDT CMFWS. 330 331:: 332 333 "decoders:root0":[ 334 { 335 "decoder":"decoder0.0", 336 "resource":825975898112, 337 "size":549755813888, 338 "interleave_ways":2, 339 "interleave_granularity":256, 340 "max_available_extent":0, 341 "volatile_capable":true, 342 "nr_targets":2, 343 "targets":[ 344 { 345 "target":"pci0000:a8", 346 "alias":"ACPI0016:02", 347 "position":1, 348 "id":4 349 }, 350 { 351 "target":"pci0000:d2", 352 "alias":"ACPI0016:00", 353 "position":0, 354 "id":5 355 } 356 ], 357 358Finally we have the `Memory Region` associated with the `Root Decoder` 359:code:`decoder0.0`. This region describes the overall interleave configuration 360of the interleave set. So we see there are a total of :code:`4` interleave 361targets across 4 endpoint decoders. 362 363:: 364 365 "regions:decoder0.0":[ 366 { 367 "region":"region0", 368 "resource":825975898112, 369 "size":549755813888, 370 "type":"ram", 371 "interleave_ways":4, 372 "interleave_granularity":256, 373 "decode_state":"commit", 374 "mappings":[ 375 { 376 "position":3, 377 "memdev":"mem3", 378 "decoder":"decoder8.0" 379 }, 380 { 381 "position":2, 382 "memdev":"mem1", 383 "decoder":"decoder6.0" 384 } 385 { 386 "position":1, 387 "memdev":"mem2", 388 "decoder":"decoder7.0" 389 }, 390 { 391 "position":0, 392 "memdev":"mem0", 393 "decoder":"decoder5.0" 394 } 395 ] 396 } 397 ] 398 } 399 ] 400 } 401 ] 402