1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /* Copyright(c) 2020 Intel Corporation. */
3
4 #ifndef __CXL_H__
5 #define __CXL_H__
6
7 #include <linux/libnvdimm.h>
8 #include <linux/bitfield.h>
9 #include <linux/notifier.h>
10 #include <linux/bitops.h>
11 #include <linux/log2.h>
12 #include <linux/node.h>
13 #include <linux/io.h>
14 #include <linux/range.h>
15 #include <cxl/cxl.h>
16
17 extern const struct nvdimm_security_ops *cxl_security_ops;
18
19 /**
20 * DOC: cxl objects
21 *
22 * The CXL core objects like ports, decoders, and regions are shared
23 * between the subsystem drivers cxl_acpi, cxl_pci, and core drivers
24 * (port-driver, region-driver, nvdimm object-drivers... etc).
25 */
26
27 /* CXL 2.0 8.2.4 CXL Component Register Layout and Definition */
28 #define CXL_COMPONENT_REG_BLOCK_SIZE SZ_64K
29
30 /* CXL 2.0 8.2.5 CXL.cache and CXL.mem Registers*/
31 #define CXL_CM_OFFSET 0x1000
32 #define CXL_CM_CAP_HDR_OFFSET 0x0
33 #define CXL_CM_CAP_HDR_ID_MASK GENMASK(15, 0)
34 #define CM_CAP_HDR_CAP_ID 1
35 #define CXL_CM_CAP_HDR_VERSION_MASK GENMASK(19, 16)
36 #define CM_CAP_HDR_CAP_VERSION 1
37 #define CXL_CM_CAP_HDR_CACHE_MEM_VERSION_MASK GENMASK(23, 20)
38 #define CM_CAP_HDR_CACHE_MEM_VERSION 1
39 #define CXL_CM_CAP_HDR_ARRAY_SIZE_MASK GENMASK(31, 24)
40 #define CXL_CM_CAP_PTR_MASK GENMASK(31, 20)
41
42 #define CXL_CM_CAP_CAP_ID_RAS 0x2
43 #define CXL_CM_CAP_CAP_ID_HDM 0x5
44 #define CXL_CM_CAP_CAP_HDM_VERSION 1
45
46 /* HDM decoders CXL 2.0 8.2.5.12 CXL HDM Decoder Capability Structure */
47 #define CXL_HDM_DECODER_CAP_OFFSET 0x0
48 #define CXL_HDM_DECODER_COUNT_MASK GENMASK(3, 0)
49 #define CXL_HDM_DECODER_TARGET_COUNT_MASK GENMASK(7, 4)
50 #define CXL_HDM_DECODER_INTERLEAVE_11_8 BIT(8)
51 #define CXL_HDM_DECODER_INTERLEAVE_14_12 BIT(9)
52 #define CXL_HDM_DECODER_INTERLEAVE_3_6_12_WAY BIT(11)
53 #define CXL_HDM_DECODER_INTERLEAVE_16_WAY BIT(12)
54 #define CXL_HDM_DECODER_CTRL_OFFSET 0x4
55 #define CXL_HDM_DECODER_ENABLE BIT(1)
56 #define CXL_HDM_DECODER0_BASE_LOW_OFFSET(i) (0x20 * (i) + 0x10)
57 #define CXL_HDM_DECODER0_BASE_HIGH_OFFSET(i) (0x20 * (i) + 0x14)
58 #define CXL_HDM_DECODER0_SIZE_LOW_OFFSET(i) (0x20 * (i) + 0x18)
59 #define CXL_HDM_DECODER0_SIZE_HIGH_OFFSET(i) (0x20 * (i) + 0x1c)
60 #define CXL_HDM_DECODER0_CTRL_OFFSET(i) (0x20 * (i) + 0x20)
61 #define CXL_HDM_DECODER0_CTRL_IG_MASK GENMASK(3, 0)
62 #define CXL_HDM_DECODER0_CTRL_IW_MASK GENMASK(7, 4)
63 #define CXL_HDM_DECODER0_CTRL_LOCK BIT(8)
64 #define CXL_HDM_DECODER0_CTRL_COMMIT BIT(9)
65 #define CXL_HDM_DECODER0_CTRL_COMMITTED BIT(10)
66 #define CXL_HDM_DECODER0_CTRL_COMMIT_ERROR BIT(11)
67 #define CXL_HDM_DECODER0_CTRL_HOSTONLY BIT(12)
68 #define CXL_HDM_DECODER0_TL_LOW(i) (0x20 * (i) + 0x24)
69 #define CXL_HDM_DECODER0_TL_HIGH(i) (0x20 * (i) + 0x28)
70 #define CXL_HDM_DECODER0_SKIP_LOW(i) CXL_HDM_DECODER0_TL_LOW(i)
71 #define CXL_HDM_DECODER0_SKIP_HIGH(i) CXL_HDM_DECODER0_TL_HIGH(i)
72
73 /* HDM decoder control register constants CXL 3.0 8.2.5.19.7 */
74 #define CXL_DECODER_MIN_GRANULARITY 256
75 #define CXL_DECODER_MAX_ENCODED_IG 6
76
cxl_hdm_decoder_count(u32 cap_hdr)77 static inline int cxl_hdm_decoder_count(u32 cap_hdr)
78 {
79 int val = FIELD_GET(CXL_HDM_DECODER_COUNT_MASK, cap_hdr);
80
81 switch (val) {
82 case 0:
83 return 1;
84 case 1 ... 8:
85 return val * 2;
86 case 9 ... 12:
87 return (val - 4) * 4;
88 default:
89 return -ENXIO;
90 }
91 }
92
93 /* Encode defined in CXL 2.0 8.2.5.12.7 HDM Decoder Control Register */
eig_to_granularity(u16 eig,int * granularity)94 static inline int eig_to_granularity(u16 eig, int *granularity)
95 {
96 if (eig > CXL_DECODER_MAX_ENCODED_IG)
97 return -EINVAL;
98 *granularity = CXL_DECODER_MIN_GRANULARITY << eig;
99 return 0;
100 }
101
102 /* Encode defined in CXL ECN "3, 6, 12 and 16-way memory Interleaving" */
eiw_to_ways(u8 eiw,int * ways)103 static inline int eiw_to_ways(u8 eiw, int *ways)
104 {
105 switch (eiw) {
106 case 0 ... 4:
107 *ways = 1 << eiw;
108 break;
109 case 8 ... 10:
110 *ways = 3 << (eiw - 8);
111 break;
112 default:
113 return -EINVAL;
114 }
115
116 return 0;
117 }
118
granularity_to_eig(int granularity,u16 * eig)119 static inline int granularity_to_eig(int granularity, u16 *eig)
120 {
121 *eig = 0;
122 if (granularity > SZ_16K || granularity < CXL_DECODER_MIN_GRANULARITY ||
123 !is_power_of_2(granularity))
124 return -EINVAL;
125 *eig = ilog2(granularity) - 8;
126 return 0;
127 }
128
ways_to_eiw(unsigned int ways,u8 * eiw)129 static inline int ways_to_eiw(unsigned int ways, u8 *eiw)
130 {
131 *eiw = 0;
132 if (ways > 16)
133 return -EINVAL;
134 if (is_power_of_2(ways)) {
135 *eiw = ilog2(ways);
136 return 0;
137 }
138 if (ways % 3)
139 return -EINVAL;
140 ways /= 3;
141 if (!is_power_of_2(ways))
142 return -EINVAL;
143 *eiw = ilog2(ways) + 8;
144 return 0;
145 }
146
147 /* RAS Registers CXL 2.0 8.2.5.9 CXL RAS Capability Structure */
148 #define CXL_RAS_UNCORRECTABLE_STATUS_OFFSET 0x0
149 #define CXL_RAS_UNCORRECTABLE_STATUS_MASK (GENMASK(16, 14) | GENMASK(11, 0))
150 #define CXL_RAS_UNCORRECTABLE_MASK_OFFSET 0x4
151 #define CXL_RAS_UNCORRECTABLE_MASK_MASK (GENMASK(16, 14) | GENMASK(11, 0))
152 #define CXL_RAS_UNCORRECTABLE_MASK_F256B_MASK BIT(8)
153 #define CXL_RAS_UNCORRECTABLE_SEVERITY_OFFSET 0x8
154 #define CXL_RAS_UNCORRECTABLE_SEVERITY_MASK (GENMASK(16, 14) | GENMASK(11, 0))
155 #define CXL_RAS_CORRECTABLE_STATUS_OFFSET 0xC
156 #define CXL_RAS_CORRECTABLE_STATUS_MASK GENMASK(6, 0)
157 #define CXL_RAS_CORRECTABLE_MASK_OFFSET 0x10
158 #define CXL_RAS_CORRECTABLE_MASK_MASK GENMASK(6, 0)
159 #define CXL_RAS_CAP_CONTROL_OFFSET 0x14
160 #define CXL_RAS_CAP_CONTROL_FE_MASK GENMASK(5, 0)
161 #define CXL_RAS_HEADER_LOG_OFFSET 0x18
162 #define CXL_RAS_CAPABILITY_LENGTH 0x58
163 #define CXL_HEADERLOG_SIZE SZ_64
164 #define CXL_HEADERLOG_SIZE_U32 (CXL_HEADERLOG_SIZE / sizeof(u32))
165
166 /*
167 * The RAS UCE trace event header array was originally sized at SZ_512/sizeof(u32)
168 * = 128 u32s due to a bug. Userspace tools (rasdaemon) have grown a dependency
169 * on that 512-byte layout. Keep the trace array at 128 u32s to preserve the
170 * ABI; only CXL_HEADERLOG_SIZE_U32 (16) dwords are valid hardware data, the
171 * remainder are zero-filled.
172 */
173 #define CXL_HEADERLOG_TRACE_SIZE SZ_512
174 #define CXL_HEADERLOG_TRACE_SIZE_U32 (CXL_HEADERLOG_TRACE_SIZE / sizeof(u32))
175
176 /* CXL 2.0 8.2.8.1 Device Capabilities Array Register */
177 #define CXLDEV_CAP_ARRAY_OFFSET 0x0
178 #define CXLDEV_CAP_ARRAY_CAP_ID 0
179 #define CXLDEV_CAP_ARRAY_ID_MASK GENMASK_ULL(15, 0)
180 #define CXLDEV_CAP_ARRAY_COUNT_MASK GENMASK_ULL(47, 32)
181 /* CXL 2.0 8.2.8.2 CXL Device Capability Header Register */
182 #define CXLDEV_CAP_HDR_CAP_ID_MASK GENMASK(15, 0)
183 /* CXL 2.0 8.2.8.2.1 CXL Device Capabilities */
184 #define CXLDEV_CAP_CAP_ID_DEVICE_STATUS 0x1
185 #define CXLDEV_CAP_CAP_ID_PRIMARY_MAILBOX 0x2
186 #define CXLDEV_CAP_CAP_ID_SECONDARY_MAILBOX 0x3
187 #define CXLDEV_CAP_CAP_ID_MEMDEV 0x4000
188
189 /* CXL 3.0 8.2.8.3.1 Event Status Register */
190 #define CXLDEV_DEV_EVENT_STATUS_OFFSET 0x00
191 #define CXLDEV_EVENT_STATUS_INFO BIT(0)
192 #define CXLDEV_EVENT_STATUS_WARN BIT(1)
193 #define CXLDEV_EVENT_STATUS_FAIL BIT(2)
194 #define CXLDEV_EVENT_STATUS_FATAL BIT(3)
195
196 #define CXLDEV_EVENT_STATUS_ALL (CXLDEV_EVENT_STATUS_INFO | \
197 CXLDEV_EVENT_STATUS_WARN | \
198 CXLDEV_EVENT_STATUS_FAIL | \
199 CXLDEV_EVENT_STATUS_FATAL)
200
201 /* CXL rev 3.0 section 8.2.9.2.4; Table 8-52 */
202 #define CXLDEV_EVENT_INT_MODE_MASK GENMASK(1, 0)
203 #define CXLDEV_EVENT_INT_MSGNUM_MASK GENMASK(7, 4)
204
205 /* CXL 2.0 8.2.8.4 Mailbox Registers */
206 #define CXLDEV_MBOX_CAPS_OFFSET 0x00
207 #define CXLDEV_MBOX_CAP_PAYLOAD_SIZE_MASK GENMASK(4, 0)
208 #define CXLDEV_MBOX_CAP_BG_CMD_IRQ BIT(6)
209 #define CXLDEV_MBOX_CAP_IRQ_MSGNUM_MASK GENMASK(10, 7)
210 #define CXLDEV_MBOX_CTRL_OFFSET 0x04
211 #define CXLDEV_MBOX_CTRL_DOORBELL BIT(0)
212 #define CXLDEV_MBOX_CTRL_BG_CMD_IRQ BIT(2)
213 #define CXLDEV_MBOX_CMD_OFFSET 0x08
214 #define CXLDEV_MBOX_CMD_COMMAND_OPCODE_MASK GENMASK_ULL(15, 0)
215 #define CXLDEV_MBOX_CMD_PAYLOAD_LENGTH_MASK GENMASK_ULL(36, 16)
216 #define CXLDEV_MBOX_STATUS_OFFSET 0x10
217 #define CXLDEV_MBOX_STATUS_BG_CMD BIT(0)
218 #define CXLDEV_MBOX_STATUS_RET_CODE_MASK GENMASK_ULL(47, 32)
219 #define CXLDEV_MBOX_BG_CMD_STATUS_OFFSET 0x18
220 #define CXLDEV_MBOX_BG_CMD_COMMAND_OPCODE_MASK GENMASK_ULL(15, 0)
221 #define CXLDEV_MBOX_BG_CMD_COMMAND_PCT_MASK GENMASK_ULL(22, 16)
222 #define CXLDEV_MBOX_BG_CMD_COMMAND_RC_MASK GENMASK_ULL(47, 32)
223 #define CXLDEV_MBOX_BG_CMD_COMMAND_VENDOR_MASK GENMASK_ULL(63, 48)
224 #define CXLDEV_MBOX_PAYLOAD_OFFSET 0x20
225
226 void cxl_probe_component_regs(struct device *dev, void __iomem *base,
227 struct cxl_component_reg_map *map);
228 void cxl_probe_device_regs(struct device *dev, void __iomem *base,
229 struct cxl_device_reg_map *map);
230 int cxl_map_component_regs(const struct cxl_register_map *map,
231 struct cxl_component_regs *regs,
232 unsigned long map_mask);
233 int cxl_map_device_regs(const struct cxl_register_map *map,
234 struct cxl_device_regs *regs);
235 int cxl_map_pmu_regs(struct cxl_register_map *map, struct cxl_pmu_regs *regs);
236
237 #define CXL_INSTANCES_COUNT -1
238 enum cxl_regloc_type;
239 int cxl_count_regblock(struct pci_dev *pdev, enum cxl_regloc_type type);
240 int cxl_find_regblock_instance(struct pci_dev *pdev, enum cxl_regloc_type type,
241 struct cxl_register_map *map, unsigned int index);
242 int cxl_find_regblock(struct pci_dev *pdev, enum cxl_regloc_type type,
243 struct cxl_register_map *map);
244 int cxl_setup_regs(struct cxl_register_map *map);
245 struct cxl_dport;
246 int cxl_dport_map_rcd_linkcap(struct pci_dev *pdev, struct cxl_dport *dport);
247
248 #define CXL_RESOURCE_NONE ((resource_size_t) -1)
249 #define CXL_TARGET_STRLEN 20
250
251 /*
252 * cxl_decoder flags that define the type of memory / devices this
253 * decoder supports as well as configuration lock status See "CXL 2.0
254 * 8.2.5.12.7 CXL HDM Decoder 0 Control Register" for details.
255 * Additionally indicate whether decoder settings were autodetected,
256 * user customized.
257 */
258 #define CXL_DECODER_F_RAM BIT(0)
259 #define CXL_DECODER_F_PMEM BIT(1)
260 #define CXL_DECODER_F_TYPE2 BIT(2)
261 #define CXL_DECODER_F_TYPE3 BIT(3)
262 #define CXL_DECODER_F_LOCK BIT(4)
263 #define CXL_DECODER_F_ENABLE BIT(5)
264 #define CXL_DECODER_F_NORMALIZED_ADDRESSING BIT(6)
265 #define CXL_DECODER_F_RESET_MASK (CXL_DECODER_F_ENABLE | CXL_DECODER_F_LOCK)
266
267 enum cxl_decoder_type {
268 CXL_DECODER_DEVMEM = 2,
269 CXL_DECODER_HOSTONLYMEM = 3,
270 };
271
272 /*
273 * Current specification goes up to 8, double that seems a reasonable
274 * software max for the foreseeable future
275 */
276 #define CXL_DECODER_MAX_INTERLEAVE 16
277
278 #define CXL_QOS_CLASS_INVALID -1
279
280 /**
281 * struct cxl_decoder - Common CXL HDM Decoder Attributes
282 * @dev: this decoder's device
283 * @id: kernel device name id
284 * @hpa_range: Host physical address range mapped by this decoder
285 * @interleave_ways: number of cxl_dports in this decode
286 * @interleave_granularity: data stride per dport
287 * @target_type: accelerator vs expander (type2 vs type3) selector
288 * @region: currently assigned region for this decoder
289 * @flags: memory type capabilities and locking
290 * @target_map: cached copy of hardware port-id list, available at init
291 * before all @dport objects have been instantiated. While
292 * dport id is 8bit, CFMWS interleave targets are 32bits.
293 * @commit: device/decoder-type specific callback to commit settings to hw
294 * @reset: device/decoder-type specific callback to reset hw settings
295 */
296 struct cxl_decoder {
297 struct device dev;
298 int id;
299 struct range hpa_range;
300 int interleave_ways;
301 int interleave_granularity;
302 enum cxl_decoder_type target_type;
303 struct cxl_region *region;
304 unsigned long flags;
305 u32 target_map[CXL_DECODER_MAX_INTERLEAVE];
306 int (*commit)(struct cxl_decoder *cxld);
307 void (*reset)(struct cxl_decoder *cxld);
308 };
309
310 /*
311 * Track whether this decoder is free for userspace provisioning, reserved for
312 * region autodiscovery, whether it is started connecting (awaiting other
313 * peers), or has completed auto assembly.
314 */
315 enum cxl_decoder_state {
316 CXL_DECODER_STATE_MANUAL,
317 CXL_DECODER_STATE_AUTO,
318 CXL_DECODER_STATE_AUTO_STAGED,
319 };
320
321 /**
322 * struct cxl_endpoint_decoder - Endpoint / SPA to DPA decoder
323 * @cxld: base cxl_decoder_object
324 * @dpa_res: actively claimed DPA span of this decoder
325 * @skip: offset into @dpa_res where @cxld.hpa_range maps
326 * @state: autodiscovery state
327 * @part: partition index this decoder maps
328 * @pos: interleave position in @cxld.region
329 */
330 struct cxl_endpoint_decoder {
331 struct cxl_decoder cxld;
332 struct resource *dpa_res;
333 resource_size_t skip;
334 enum cxl_decoder_state state;
335 int part;
336 int pos;
337 };
338
339 /**
340 * struct cxl_switch_decoder - Switch specific CXL HDM Decoder
341 * @cxld: base cxl_decoder object
342 * @nr_targets: number of elements in @target
343 * @target: active ordered target list in current decoder configuration
344 *
345 * The 'switch' decoder type represents the decoder instances of cxl_port's that
346 * route from the root of a CXL memory decode topology to the endpoints. They
347 * come in two flavors, root-level decoders, statically defined by platform
348 * firmware, and mid-level decoders, where interleave-granularity,
349 * interleave-width, and the target list are mutable.
350 */
351 struct cxl_switch_decoder {
352 struct cxl_decoder cxld;
353 int nr_targets;
354 struct cxl_dport *target[];
355 };
356
357 struct cxl_root_decoder;
358 /**
359 * struct cxl_rd_ops - CXL root decoder callback operations
360 * @hpa_to_spa: Convert host physical address to system physical address
361 * @spa_to_hpa: Convert system physical address to host physical address
362 */
363 struct cxl_rd_ops {
364 u64 (*hpa_to_spa)(struct cxl_root_decoder *cxlrd, u64 hpa);
365 u64 (*spa_to_hpa)(struct cxl_root_decoder *cxlrd, u64 spa);
366 };
367
368 /**
369 * struct cxl_root_decoder - Static platform CXL address decoder
370 * @res: host / parent resource for region allocations
371 * @cache_size: extended linear cache size if exists, otherwise zero.
372 * @region_id: region id for next region provisioning event
373 * @platform_data: platform specific configuration data
374 * @regions_lock: sync region discovery, construction, and deletion
375 * @regions: regions to remove at root decoder destruct time
376 * @dead: root decoder dead to region creation
377 * @qos_class: QoS performance class cookie
378 * @ops: CXL root decoder operations
379 * @cxlsd: base cxl switch decoder
380 */
381 struct cxl_root_decoder {
382 struct resource *res;
383 resource_size_t cache_size;
384 atomic_t region_id;
385 void *platform_data;
386 struct mutex regions_lock;
387 struct xarray regions;
388 bool dead;
389 int qos_class;
390 struct cxl_rd_ops ops;
391 struct cxl_switch_decoder cxlsd;
392 };
393
394 /*
395 * enum cxl_config_state - State machine for region configuration
396 * @CXL_CONFIG_IDLE: Any sysfs attribute can be written freely
397 * @CXL_CONFIG_INTERLEAVE_ACTIVE: region size has been set, no more
398 * changes to interleave_ways or interleave_granularity
399 * @CXL_CONFIG_ACTIVE: All targets have been added the region is now
400 * active
401 * @CXL_CONFIG_RESET_PENDING: see commit_store()
402 * @CXL_CONFIG_COMMIT: Soft-config has been committed to hardware
403 */
404 enum cxl_config_state {
405 CXL_CONFIG_IDLE,
406 CXL_CONFIG_INTERLEAVE_ACTIVE,
407 CXL_CONFIG_ACTIVE,
408 CXL_CONFIG_RESET_PENDING,
409 CXL_CONFIG_COMMIT,
410 };
411
412 /**
413 * struct cxl_region_params - region settings
414 * @state: allow the driver to lockdown further parameter changes
415 * @uuid: unique id for persistent regions
416 * @interleave_ways: number of endpoints in the region
417 * @interleave_granularity: capacity each endpoint contributes to a stripe
418 * @res: allocated iomem capacity for this region
419 * @targets: active ordered targets in current decoder configuration
420 * @nr_targets: number of targets
421 * @cache_size: extended linear cache size if exists, otherwise zero.
422 *
423 * State transitions are protected by cxl_rwsem.region
424 */
425 struct cxl_region_params {
426 enum cxl_config_state state;
427 uuid_t uuid;
428 int interleave_ways;
429 int interleave_granularity;
430 struct resource *res;
431 struct cxl_endpoint_decoder *targets[CXL_DECODER_MAX_INTERLEAVE];
432 int nr_targets;
433 resource_size_t cache_size;
434 };
435
436 /*
437 * Indicate whether this region has been assembled by autodetection or
438 * userspace assembly. Prevent endpoint decoders outside of automatic
439 * detection from being added to the region.
440 */
441 #define CXL_REGION_F_AUTO 0
442
443 /*
444 * Require that a committed region successfully complete a teardown once
445 * any of its associated decoders have been torn down. This maintains
446 * the commit state for the region since there are committed decoders,
447 * but blocks cxl_region_probe().
448 */
449 #define CXL_REGION_F_NEEDS_RESET 1
450
451 /*
452 * Indicate whether this region is locked due to 1 or more decoders that have
453 * been locked. The approach of all or nothing is taken with regard to the
454 * locked attribute. CXL_REGION_F_NEEDS_RESET should not be set if this flag is
455 * set.
456 */
457 #define CXL_REGION_F_LOCK 2
458
459 /*
460 * Indicate Normalized Addressing. Use it to disable SPA conversion if
461 * HPA != SPA and an address translation callback handler does not
462 * exist. Flag is needed by AMD Zen5 platforms.
463 */
464 #define CXL_REGION_F_NORMALIZED_ADDRESSING 3
465
466 /**
467 * struct cxl_region - CXL region
468 * @dev: This region's device
469 * @id: This region's id. Id is globally unique across all regions
470 * @cxlrd: Region's root decoder
471 * @hpa_range: Address range occupied by the region
472 * @mode: Operational mode of the mapped capacity
473 * @type: Endpoint decoder target type
474 * @cxl_nvb: nvdimm bridge for coordinating @cxlr_pmem setup / shutdown
475 * @cxlr_pmem: (for pmem regions) cached copy of the nvdimm bridge
476 * @flags: Region state flags
477 * @params: active + config params for the region
478 * @coord: QoS access coordinates for the region
479 * @node_notifier: notifier for setting the access coordinates to node
480 * @adist_notifier: notifier for calculating the abstract distance of node
481 * @mce_notifier: notifier for MCE
482 */
483 struct cxl_region {
484 struct device dev;
485 int id;
486 struct cxl_root_decoder *cxlrd;
487 struct range hpa_range;
488 enum cxl_partition_mode mode;
489 enum cxl_decoder_type type;
490 struct cxl_nvdimm_bridge *cxl_nvb;
491 struct cxl_pmem_region *cxlr_pmem;
492 unsigned long flags;
493 struct cxl_region_params params;
494 struct access_coordinate coord[ACCESS_COORDINATE_MAX];
495 struct notifier_block node_notifier;
496 struct notifier_block adist_notifier;
497 struct notifier_block mce_notifier;
498 };
499
500 struct cxl_nvdimm_bridge {
501 int id;
502 struct device dev;
503 struct cxl_port *port;
504 struct nvdimm_bus *nvdimm_bus;
505 struct nvdimm_bus_descriptor nd_desc;
506 };
507
508 /* Holds a u64 serial as a decimal string: up to 20 digits + NUL */
509 #define CXL_DEV_ID_LEN 21
510
511 enum {
512 CXL_NVD_F_INVALIDATED = 0,
513 };
514
515 struct cxl_nvdimm {
516 struct device dev;
517 struct cxl_memdev *cxlmd;
518 u8 dev_id[CXL_DEV_ID_LEN]; /* for nvdimm, string of 'serial' */
519 u64 dirty_shutdowns;
520 unsigned long flags;
521 };
522
523 struct cxl_pmem_region_mapping {
524 struct cxl_memdev *cxlmd;
525 struct cxl_nvdimm *cxl_nvd;
526 u64 start;
527 u64 size;
528 int position;
529 };
530
531 struct cxl_pmem_region {
532 struct device dev;
533 struct cxl_region *cxlr;
534 struct nd_region *nd_region;
535 struct range hpa_range;
536 int nr_mappings;
537 struct cxl_pmem_region_mapping mapping[];
538 };
539
540 struct cxl_dax_region {
541 struct device dev;
542 struct cxl_region *cxlr;
543 struct range hpa_range;
544 };
545
546 /**
547 * struct cxl_port - logical collection of upstream port devices and
548 * downstream port devices to construct a CXL memory
549 * decode hierarchy.
550 * @dev: this port's device
551 * @uport_dev: PCI or platform device implementing the upstream port capability
552 * @host_bridge: Shortcut to the platform attach point for this port
553 * @id: id for port device-name
554 * @dports: cxl_dport instances referenced by decoders
555 * @endpoints: cxl_ep instances, endpoints that are a descendant of this port
556 * @regions: cxl_region_ref instances, regions mapped by this port
557 * @parent_dport: dport that points to this port in the parent
558 * @decoder_ida: allocator for decoder ids
559 * @reg_map: component and ras register mapping parameters
560 * @regs: mapped component registers
561 * @nr_dports: number of entries in @dports
562 * @hdm_end: track last allocated HDM decoder instance for allocation ordering
563 * @commit_end: cursor to track highest committed decoder for commit ordering
564 * @dead: last ep has been removed, force port re-creation
565 * @depth: How deep this port is relative to the root. depth 0 is the root.
566 * @cdat: Cached CDAT data
567 * @cdat_available: Should a CDAT attribute be available in sysfs
568 * @pci_latency: Upstream latency in picoseconds
569 * @component_reg_phys: Physical address of component register
570 */
571 struct cxl_port {
572 struct device dev;
573 struct device *uport_dev;
574 struct device *host_bridge;
575 int id;
576 struct xarray dports;
577 struct xarray endpoints;
578 struct xarray regions;
579 struct cxl_dport *parent_dport;
580 struct ida decoder_ida;
581 struct cxl_register_map reg_map;
582 struct cxl_component_regs regs;
583 int nr_dports;
584 int hdm_end;
585 int commit_end;
586 bool dead;
587 unsigned int depth;
588 struct cxl_cdat {
589 void *table;
590 size_t length;
591 } cdat;
592 bool cdat_available;
593 long pci_latency;
594 resource_size_t component_reg_phys;
595 };
596
597 struct cxl_root;
598
599 struct cxl_root_ops {
600 int (*qos_class)(struct cxl_root *cxl_root,
601 struct access_coordinate *coord, int entries,
602 int *qos_class);
603 int (*translation_setup_root)(struct cxl_root *cxl_root, void *data);
604 };
605
606 /**
607 * struct cxl_root - logical collection of root cxl_port items
608 *
609 * @port: cxl_port member
610 * @ops: cxl root operations
611 */
612 struct cxl_root {
613 struct cxl_port port;
614 struct cxl_root_ops ops;
615 };
616
617 static inline struct cxl_root *
to_cxl_root(const struct cxl_port * port)618 to_cxl_root(const struct cxl_port *port)
619 {
620 return container_of(port, struct cxl_root, port);
621 }
622
623 static inline struct cxl_dport *
cxl_find_dport_by_dev(struct cxl_port * port,const struct device * dport_dev)624 cxl_find_dport_by_dev(struct cxl_port *port, const struct device *dport_dev)
625 {
626 return xa_load(&port->dports, (unsigned long)dport_dev);
627 }
628
629 struct cxl_rcrb_info {
630 resource_size_t base;
631 u16 aer_cap;
632 };
633
634 /**
635 * struct cxl_dport - CXL downstream port
636 * @dport_dev: PCI bridge or firmware device representing the downstream link
637 * @reg_map: component and ras register mapping parameters
638 * @port_id: unique hardware identifier for dport in decoder target list
639 * @rcrb: Data about the Root Complex Register Block layout
640 * @rch: Indicate whether this dport was enumerated in RCH or VH mode
641 * @port: reference to cxl_port that contains this downstream port
642 * @regs: Dport parsed register blocks
643 * @coord: access coordinates (bandwidth and latency performance attributes)
644 * @link_latency: calculated PCIe downstream latency
645 * @gpf_dvsec: Cached GPF port DVSEC
646 */
647 struct cxl_dport {
648 struct device *dport_dev;
649 struct cxl_register_map reg_map;
650 int port_id;
651 struct cxl_rcrb_info rcrb;
652 bool rch;
653 struct cxl_port *port;
654 struct cxl_regs regs;
655 struct access_coordinate coord[ACCESS_COORDINATE_MAX];
656 long link_latency;
657 int gpf_dvsec;
658 };
659
660 /**
661 * struct cxl_ep - track an endpoint's interest in a port
662 * @ep: device that hosts a generic CXL endpoint (expander or accelerator)
663 * @dport: which dport routes to this endpoint on @port
664 * @next: cxl switch port across the link attached to @dport NULL if
665 * attached to an endpoint
666 */
667 struct cxl_ep {
668 struct device *ep;
669 struct cxl_dport *dport;
670 struct cxl_port *next;
671 };
672
673 /**
674 * struct cxl_region_ref - track a region's interest in a port
675 * @port: point in topology to install this reference
676 * @decoder: decoder assigned for @region in @port
677 * @region: region for this reference
678 * @endpoints: cxl_ep references for region members beneath @port
679 * @nr_targets_set: track how many targets have been programmed during setup
680 * @nr_eps: number of endpoints beneath @port
681 * @nr_targets: number of distinct targets needed to reach @nr_eps
682 */
683 struct cxl_region_ref {
684 struct cxl_port *port;
685 struct cxl_decoder *decoder;
686 struct cxl_region *region;
687 struct xarray endpoints;
688 int nr_targets_set;
689 int nr_eps;
690 int nr_targets;
691 };
692
693 /*
694 * The platform firmware device hosting the root is also the top of the
695 * CXL port topology. All other CXL ports have another CXL port as their
696 * parent and their ->uport_dev / host device is out-of-line of the port
697 * ancestry.
698 */
is_cxl_root(struct cxl_port * port)699 static inline bool is_cxl_root(struct cxl_port *port)
700 {
701 return port->uport_dev == port->dev.parent;
702 }
703
704 /* Address translation functions exported to cxl_translate test module only */
705 int cxl_validate_translation_params(u8 eiw, u16 eig, int pos);
706 u64 cxl_calculate_hpa_offset(u64 dpa_offset, int pos, u8 eiw, u16 eig);
707 u64 cxl_calculate_dpa_offset(u64 hpa_offset, u8 eiw, u16 eig);
708 int cxl_calculate_position(u64 hpa_offset, u8 eiw, u16 eig);
709 struct cxl_cxims_data {
710 int nr_maps;
711 u64 xormaps[] __counted_by(nr_maps);
712 };
713
714 #if IS_ENABLED(CONFIG_CXL_ACPI)
715 u64 cxl_do_xormap_calc(struct cxl_cxims_data *cximsd, u64 addr, int hbiw);
716 #else
cxl_do_xormap_calc(struct cxl_cxims_data * cximsd,u64 addr,int hbiw)717 static inline u64 cxl_do_xormap_calc(struct cxl_cxims_data *cximsd, u64 addr, int hbiw)
718 {
719 return ULLONG_MAX;
720 }
721 #endif
722
723 int cxl_num_decoders_committed(struct cxl_port *port);
724 bool is_cxl_port(const struct device *dev);
725 struct cxl_port *to_cxl_port(const struct device *dev);
726 struct cxl_port *parent_port_of(struct cxl_port *port);
727 void cxl_port_commit_reap(struct cxl_decoder *cxld);
728 struct pci_bus;
729 int devm_cxl_register_pci_bus(struct device *host, struct device *uport_dev,
730 struct pci_bus *bus);
731 struct pci_bus *cxl_port_to_pci_bus(struct cxl_port *port);
732 struct cxl_port *devm_cxl_add_port(struct device *host,
733 struct device *uport_dev,
734 resource_size_t component_reg_phys,
735 struct cxl_dport *parent_dport);
736 struct cxl_root *devm_cxl_add_root(struct device *host);
737 int devm_cxl_add_endpoint(struct device *host, struct cxl_memdev *cxlmd,
738 struct cxl_dport *parent_dport);
739 struct cxl_root *find_cxl_root(struct cxl_port *port);
740
741 DEFINE_FREE(put_cxl_root, struct cxl_root *, if (_T) put_device(&_T->port.dev))
742 DEFINE_FREE(put_cxl_port, struct cxl_port *, if (!IS_ERR_OR_NULL(_T)) put_device(&_T->dev))
743 DEFINE_FREE(put_cxl_root_decoder, struct cxl_root_decoder *, if (!IS_ERR_OR_NULL(_T)) put_device(&_T->cxlsd.cxld.dev))
744 DEFINE_FREE(put_cxl_region, struct cxl_region *, if (!IS_ERR_OR_NULL(_T)) put_device(&_T->dev))
745 DEFINE_FREE(put_cxl_dax_region, struct cxl_dax_region *, if (!IS_ERR_OR_NULL(_T)) put_device(&_T->dev))
746
747 int devm_cxl_enumerate_ports(struct cxl_memdev *cxlmd);
748 void cxl_bus_rescan(void);
749 void cxl_bus_drain(void);
750 struct cxl_port *cxl_pci_find_port(struct pci_dev *pdev,
751 struct cxl_dport **dport);
752 struct cxl_port *cxl_mem_find_port(struct cxl_memdev *cxlmd,
753 struct cxl_dport **dport);
754 bool schedule_cxl_memdev_detach(struct cxl_memdev *cxlmd);
755
756 struct cxl_dport *devm_cxl_add_dport(struct cxl_port *port,
757 struct device *dport, int port_id,
758 resource_size_t component_reg_phys);
759 struct cxl_dport *devm_cxl_add_rch_dport(struct cxl_port *port,
760 struct device *dport_dev, int port_id,
761 resource_size_t rcrb);
762
763 #ifdef CONFIG_CXL_ATL
764 void cxl_setup_prm_address_translation(struct cxl_root *cxl_root);
765 #else
766 static inline
cxl_setup_prm_address_translation(struct cxl_root * cxl_root)767 void cxl_setup_prm_address_translation(struct cxl_root *cxl_root) {}
768 #endif
769
770 struct cxl_decoder *to_cxl_decoder(struct device *dev);
771 struct cxl_root_decoder *to_cxl_root_decoder(struct device *dev);
772 struct cxl_switch_decoder *to_cxl_switch_decoder(struct device *dev);
773 struct cxl_endpoint_decoder *to_cxl_endpoint_decoder(struct device *dev);
774 bool is_root_decoder(struct device *dev);
775 bool is_switch_decoder(struct device *dev);
776 bool is_endpoint_decoder(struct device *dev);
777 struct cxl_root_decoder *cxl_root_decoder_alloc(struct cxl_port *port,
778 unsigned int nr_targets);
779 struct cxl_switch_decoder *cxl_switch_decoder_alloc(struct cxl_port *port,
780 unsigned int nr_targets);
781 int cxl_decoder_add(struct cxl_decoder *cxld);
782 struct cxl_endpoint_decoder *cxl_endpoint_decoder_alloc(struct cxl_port *port);
783 int cxl_decoder_add_locked(struct cxl_decoder *cxld);
784 int cxl_decoder_autoremove(struct device *host, struct cxl_decoder *cxld);
cxl_root_decoder_autoremove(struct device * host,struct cxl_root_decoder * cxlrd)785 static inline int cxl_root_decoder_autoremove(struct device *host,
786 struct cxl_root_decoder *cxlrd)
787 {
788 return cxl_decoder_autoremove(host, &cxlrd->cxlsd.cxld);
789 }
790 int cxl_endpoint_autoremove(struct cxl_memdev *cxlmd, struct cxl_port *endpoint);
791
792 /**
793 * struct cxl_endpoint_dvsec_info - Cached DVSEC info
794 * @mem_enabled: cached value of mem_enabled in the DVSEC at init time
795 * @ranges: Number of active HDM ranges this device uses.
796 * @port: endpoint port associated with this info instance
797 * @dvsec_range: cached attributes of the ranges in the DVSEC, PCIE_DEVICE
798 */
799 struct cxl_endpoint_dvsec_info {
800 bool mem_enabled;
801 int ranges;
802 struct cxl_port *port;
803 struct range dvsec_range[2];
804 };
805
806 int devm_cxl_switch_port_decoders_setup(struct cxl_port *port);
807 int devm_cxl_endpoint_decoders_setup(struct cxl_port *port);
808 void cxl_port_update_decoder_targets(struct cxl_port *port,
809 struct cxl_dport *dport);
810 int cxl_port_setup_regs(struct cxl_port *port,
811 resource_size_t component_reg_phys);
812
813 struct cxl_dev_state;
814 int cxl_dvsec_rr_decode(struct cxl_dev_state *cxlds,
815 struct cxl_endpoint_dvsec_info *info);
816
817 bool is_cxl_region(struct device *dev);
818
819 extern const struct bus_type cxl_bus_type;
820
821 /*
822 * Note, add_dport() is expressly for the cxl_port driver. TODO: investigate a
823 * type-safe driver model where probe()/remove() take the type of object implied
824 * by @id and the add_dport() op only defined for the CXL_DEVICE_PORT driver
825 * template.
826 */
827 struct cxl_driver {
828 const char *name;
829 int (*probe)(struct device *dev);
830 void (*remove)(struct device *dev);
831 struct cxl_dport *(*add_dport)(struct cxl_port *port,
832 struct device *dport_dev);
833 struct device_driver drv;
834 int id;
835 };
836
837 #define to_cxl_drv(__drv) container_of_const(__drv, struct cxl_driver, drv)
838
839 int __cxl_driver_register(struct cxl_driver *cxl_drv, struct module *owner,
840 const char *modname);
841 #define cxl_driver_register(x) __cxl_driver_register(x, THIS_MODULE, KBUILD_MODNAME)
842 void cxl_driver_unregister(struct cxl_driver *cxl_drv);
843
844 #define module_cxl_driver(__cxl_driver) \
845 module_driver(__cxl_driver, cxl_driver_register, cxl_driver_unregister)
846
847 #define CXL_DEVICE_NVDIMM_BRIDGE 1
848 #define CXL_DEVICE_NVDIMM 2
849 #define CXL_DEVICE_PORT 3
850 #define CXL_DEVICE_ROOT 4
851 #define CXL_DEVICE_MEMORY_EXPANDER 5
852 #define CXL_DEVICE_REGION 6
853 #define CXL_DEVICE_PMEM_REGION 7
854 #define CXL_DEVICE_DAX_REGION 8
855 #define CXL_DEVICE_PMU 9
856
857 #define MODULE_ALIAS_CXL(type) MODULE_ALIAS("cxl:t" __stringify(type) "*")
858 #define CXL_MODALIAS_FMT "cxl:t%d"
859
860 struct cxl_nvdimm_bridge *to_cxl_nvdimm_bridge(struct device *dev);
861 struct cxl_nvdimm_bridge *devm_cxl_add_nvdimm_bridge(struct device *host,
862 struct cxl_port *port);
863 struct cxl_nvdimm_bridge *__devm_cxl_add_nvdimm_bridge(struct device *host,
864 struct cxl_port *port);
865 struct cxl_nvdimm *to_cxl_nvdimm(struct device *dev);
866 bool is_cxl_nvdimm(struct device *dev);
867 int devm_cxl_add_nvdimm(struct device *host, struct cxl_port *port,
868 struct cxl_memdev *cxlmd);
869 struct cxl_nvdimm_bridge *cxl_find_nvdimm_bridge(struct cxl_port *port);
870
871 #ifdef CONFIG_CXL_REGION
872 bool is_cxl_pmem_region(struct device *dev);
873 struct cxl_pmem_region *to_cxl_pmem_region(struct device *dev);
874 int cxl_add_to_region(struct cxl_endpoint_decoder *cxled);
875 struct cxl_dax_region *to_cxl_dax_region(struct device *dev);
876 bool cxl_region_contains_resource(const struct resource *res);
877 #else
is_cxl_pmem_region(struct device * dev)878 static inline bool is_cxl_pmem_region(struct device *dev)
879 {
880 return false;
881 }
to_cxl_pmem_region(struct device * dev)882 static inline struct cxl_pmem_region *to_cxl_pmem_region(struct device *dev)
883 {
884 return NULL;
885 }
cxl_add_to_region(struct cxl_endpoint_decoder * cxled)886 static inline int cxl_add_to_region(struct cxl_endpoint_decoder *cxled)
887 {
888 return 0;
889 }
to_cxl_dax_region(struct device * dev)890 static inline struct cxl_dax_region *to_cxl_dax_region(struct device *dev)
891 {
892 return NULL;
893 }
cxl_region_contains_resource(const struct resource * res)894 static inline bool cxl_region_contains_resource(const struct resource *res)
895 {
896 return false;
897 }
898 #endif
899
900 void cxl_endpoint_parse_cdat(struct cxl_port *port);
901 void cxl_switch_parse_cdat(struct cxl_dport *dport);
902
903 int cxl_endpoint_get_perf_coordinates(struct cxl_port *port,
904 struct access_coordinate *coord);
905 void cxl_region_perf_data_calculate(struct cxl_region *cxlr,
906 struct cxl_endpoint_decoder *cxled);
907 void cxl_region_shared_upstream_bandwidth_update(struct cxl_region *cxlr);
908
909 void cxl_memdev_update_perf(struct cxl_memdev *cxlmd);
910
911 void cxl_coordinates_combine(struct access_coordinate *out,
912 struct access_coordinate *c1,
913 struct access_coordinate *c2);
914
915 bool cxl_endpoint_decoder_reset_detected(struct cxl_port *port);
916 struct cxl_dport *devm_cxl_add_dport_by_dev(struct cxl_port *port,
917 struct device *dport_dev);
918
919 /*
920 * Unit test builds overrides this to __weak, find the 'strong' version
921 * of these symbols in tools/testing/cxl/.
922 */
923 #ifndef __mock
924 #define __mock static
925 #endif
926
927 u16 cxl_gpf_get_dvsec(struct device *dev);
928
929 #endif /* __CXL_H__ */
930