1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Author(s)......: Carsten Otte <cotte@de.ibm.com> 4 * Rob M van der Heij <rvdheij@nl.ibm.com> 5 * Steven Shultz <shultzss@us.ibm.com> 6 * Bugreports.to..: <Linux390@de.ibm.com> 7 * Copyright IBM Corp. 2002, 2004 8 */ 9 10 #define pr_fmt(fmt) "extmem: " fmt 11 12 #include <linux/kernel.h> 13 #include <linux/string.h> 14 #include <linux/spinlock.h> 15 #include <linux/list.h> 16 #include <linux/slab.h> 17 #include <linux/export.h> 18 #include <linux/memblock.h> 19 #include <linux/ctype.h> 20 #include <linux/ioport.h> 21 #include <linux/refcount.h> 22 #include <linux/pgtable.h> 23 #include <asm/machine.h> 24 #include <asm/diag.h> 25 #include <asm/page.h> 26 #include <asm/ebcdic.h> 27 #include <asm/errno.h> 28 #include <asm/extmem.h> 29 #include <asm/cpcmd.h> 30 #include <asm/setup.h> 31 #include <asm/asm.h> 32 33 #define DCSS_PURGESEG 0x08 34 #define DCSS_LOADSHRX 0x20 35 #define DCSS_LOADNSRX 0x24 36 #define DCSS_FINDSEGX 0x2c 37 #define DCSS_SEGEXTX 0x38 38 #define DCSS_FINDSEGA 0x0c 39 40 struct qrange { 41 unsigned long start; /* last byte type */ 42 unsigned long end; /* last byte reserved */ 43 }; 44 45 struct qout64 { 46 unsigned long segstart; 47 unsigned long segend; 48 int segcnt; 49 int segrcnt; 50 struct qrange range[6]; 51 }; 52 53 struct qin64 { 54 char qopcode; 55 char rsrv1[3]; 56 char qrcode; 57 char rsrv2[3]; 58 char qname[8]; 59 unsigned int qoutptr; 60 short int qoutlen; 61 }; 62 63 struct dcss_segment { 64 struct list_head list; 65 char dcss_name[8]; 66 char res_name[16]; 67 unsigned long start_addr; 68 unsigned long end; 69 refcount_t ref_count; 70 int do_nonshared; 71 unsigned int vm_segtype; 72 struct qrange range[6]; 73 int segcnt; 74 struct resource *res; 75 }; 76 77 static DEFINE_MUTEX(dcss_lock); 78 static LIST_HEAD(dcss_list); 79 static char *segtype_string[] = { "SW", "EW", "SR", "ER", "SN", "EN", "SC", 80 "EW/EN-MIXED" }; 81 static int loadshr_scode = DCSS_LOADSHRX; 82 static int loadnsr_scode = DCSS_LOADNSRX; 83 static int purgeseg_scode = DCSS_PURGESEG; 84 static int segext_scode = DCSS_SEGEXTX; 85 86 /* 87 * Create the 8 bytes, ebcdic VM segment name from 88 * an ascii name. 89 */ 90 static void 91 dcss_mkname(char *name, char *dcss_name) 92 { 93 int i; 94 95 for (i = 0; i < 8; i++) { 96 if (name[i] == '\0') 97 break; 98 dcss_name[i] = toupper(name[i]); 99 } 100 for (; i < 8; i++) 101 dcss_name[i] = ' '; 102 ASCEBC(dcss_name, 8); 103 } 104 105 106 /* 107 * search all segments in dcss_list, and return the one 108 * namend *name. If not found, return NULL. 109 */ 110 static struct dcss_segment * 111 segment_by_name (char *name) 112 { 113 char dcss_name[9]; 114 struct list_head *l; 115 struct dcss_segment *tmp, *retval = NULL; 116 117 BUG_ON(!mutex_is_locked(&dcss_lock)); 118 dcss_mkname (name, dcss_name); 119 list_for_each (l, &dcss_list) { 120 tmp = list_entry (l, struct dcss_segment, list); 121 if (memcmp(tmp->dcss_name, dcss_name, 8) == 0) { 122 retval = tmp; 123 break; 124 } 125 } 126 return retval; 127 } 128 129 130 /* 131 * Perform a function on a dcss segment. 132 */ 133 static inline int 134 dcss_diag(int *func, void *parameter, 135 unsigned long *ret1, unsigned long *ret2) 136 { 137 unsigned long rx, ry; 138 int cc; 139 140 rx = virt_to_phys(parameter); 141 ry = (unsigned long) *func; 142 143 diag_stat_inc(DIAG_STAT_X064); 144 asm volatile( 145 " diag %[rx],%[ry],0x64\n" 146 CC_IPM(cc) 147 : CC_OUT(cc, cc), [rx] "+d" (rx), [ry] "+d" (ry) 148 : 149 : CC_CLOBBER); 150 *ret1 = rx; 151 *ret2 = ry; 152 return CC_TRANSFORM(cc); 153 } 154 155 static inline int 156 dcss_diag_translate_rc (int vm_rc) { 157 if (vm_rc == 44) 158 return -ENOENT; 159 return -EIO; 160 } 161 162 163 /* do a diag to get info about a segment. 164 * fills start_address, end and vm_segtype fields 165 */ 166 static int 167 query_segment_type (struct dcss_segment *seg) 168 { 169 unsigned long dummy, vmrc; 170 int diag_cc, rc, i; 171 struct qout64 *qout; 172 struct qin64 *qin; 173 174 qin = kmalloc(sizeof(*qin), GFP_KERNEL | GFP_DMA); 175 qout = kmalloc(sizeof(*qout), GFP_KERNEL | GFP_DMA); 176 if ((qin == NULL) || (qout == NULL)) { 177 rc = -ENOMEM; 178 goto out_free; 179 } 180 181 /* initialize diag input parameters */ 182 qin->qopcode = DCSS_FINDSEGA; 183 qin->qoutptr = virt_to_phys(qout); 184 qin->qoutlen = sizeof(struct qout64); 185 memcpy (qin->qname, seg->dcss_name, 8); 186 187 diag_cc = dcss_diag(&segext_scode, qin, &dummy, &vmrc); 188 189 if (diag_cc < 0) { 190 rc = diag_cc; 191 goto out_free; 192 } 193 if (diag_cc > 1) { 194 pr_warn("Querying a DCSS type failed with rc=%ld\n", vmrc); 195 rc = dcss_diag_translate_rc (vmrc); 196 goto out_free; 197 } 198 199 if (qout->segcnt > 6) { 200 rc = -EOPNOTSUPP; 201 goto out_free; 202 } 203 204 if (qout->segcnt == 1) { 205 seg->vm_segtype = qout->range[0].start & 0xff; 206 } else { 207 /* multi-part segment. only one type supported here: 208 - all parts are contiguous 209 - all parts are either EW or EN type 210 - maximum 6 parts allowed */ 211 unsigned long start = qout->segstart >> PAGE_SHIFT; 212 for (i=0; i<qout->segcnt; i++) { 213 if (((qout->range[i].start & 0xff) != SEG_TYPE_EW) && 214 ((qout->range[i].start & 0xff) != SEG_TYPE_EN)) { 215 rc = -EOPNOTSUPP; 216 goto out_free; 217 } 218 if (start != qout->range[i].start >> PAGE_SHIFT) { 219 rc = -EOPNOTSUPP; 220 goto out_free; 221 } 222 start = (qout->range[i].end >> PAGE_SHIFT) + 1; 223 } 224 seg->vm_segtype = SEG_TYPE_EWEN; 225 } 226 227 /* analyze diag output and update seg */ 228 seg->start_addr = qout->segstart; 229 seg->end = qout->segend; 230 231 memcpy (seg->range, qout->range, 6*sizeof(struct qrange)); 232 seg->segcnt = qout->segcnt; 233 234 rc = 0; 235 236 out_free: 237 kfree(qin); 238 kfree(qout); 239 return rc; 240 } 241 242 /* 243 * get info about a segment 244 * possible return values: 245 * -ENOSYS : we are not running on VM 246 * -EIO : could not perform query diagnose 247 * -ENOENT : no such segment 248 * -EOPNOTSUPP: multi-part segment cannot be used with linux 249 * -ENOMEM : out of memory 250 * 0 .. 6 : type of segment as defined in include/asm-s390/extmem.h 251 */ 252 int 253 segment_type (char* name) 254 { 255 int rc; 256 struct dcss_segment seg; 257 258 if (!machine_is_vm()) 259 return -ENOSYS; 260 261 dcss_mkname(name, seg.dcss_name); 262 rc = query_segment_type (&seg); 263 if (rc < 0) 264 return rc; 265 return seg.vm_segtype; 266 } 267 268 /* 269 * check if segment collides with other segments that are currently loaded 270 * returns 1 if this is the case, 0 if no collision was found 271 */ 272 static int 273 segment_overlaps_others (struct dcss_segment *seg) 274 { 275 struct list_head *l; 276 struct dcss_segment *tmp; 277 278 BUG_ON(!mutex_is_locked(&dcss_lock)); 279 list_for_each(l, &dcss_list) { 280 tmp = list_entry(l, struct dcss_segment, list); 281 if ((tmp->start_addr >> 20) > (seg->end >> 20)) 282 continue; 283 if ((tmp->end >> 20) < (seg->start_addr >> 20)) 284 continue; 285 if (seg == tmp) 286 continue; 287 return 1; 288 } 289 return 0; 290 } 291 292 /* 293 * real segment loading function, called from segment_load 294 * Must return either an error code < 0, or the segment type code >= 0 295 */ 296 static int 297 __segment_load (char *name, int do_nonshared, unsigned long *addr, unsigned long *end) 298 { 299 unsigned long start_addr, end_addr, dummy; 300 struct dcss_segment *seg; 301 int rc, diag_cc, segtype; 302 303 start_addr = end_addr = 0; 304 segtype = -1; 305 seg = kmalloc(sizeof(*seg), GFP_KERNEL | GFP_DMA); 306 if (seg == NULL) { 307 rc = -ENOMEM; 308 goto out; 309 } 310 dcss_mkname (name, seg->dcss_name); 311 rc = query_segment_type (seg); 312 if (rc < 0) 313 goto out_free; 314 315 if (segment_overlaps_others(seg)) { 316 rc = -EBUSY; 317 goto out_free; 318 } 319 320 seg->res = kzalloc(sizeof(struct resource), GFP_KERNEL); 321 if (seg->res == NULL) { 322 rc = -ENOMEM; 323 goto out_free; 324 } 325 seg->res->flags = IORESOURCE_BUSY | IORESOURCE_MEM; 326 seg->res->start = seg->start_addr; 327 seg->res->end = seg->end; 328 memcpy(&seg->res_name, seg->dcss_name, 8); 329 EBCASC(seg->res_name, 8); 330 seg->res_name[8] = '\0'; 331 strlcat(seg->res_name, " (DCSS)", sizeof(seg->res_name)); 332 seg->res->name = seg->res_name; 333 segtype = seg->vm_segtype; 334 if (segtype == SEG_TYPE_SC || 335 ((segtype == SEG_TYPE_SR || segtype == SEG_TYPE_ER) && !do_nonshared)) 336 seg->res->flags |= IORESOURCE_READONLY; 337 338 /* Check for overlapping resources before adding the mapping. */ 339 if (request_resource(&iomem_resource, seg->res)) { 340 rc = -EBUSY; 341 goto out_free_resource; 342 } 343 344 rc = vmem_add_mapping(seg->start_addr, seg->end - seg->start_addr + 1); 345 if (rc) 346 goto out_resource; 347 348 if (do_nonshared) 349 diag_cc = dcss_diag(&loadnsr_scode, seg->dcss_name, 350 &start_addr, &end_addr); 351 else 352 diag_cc = dcss_diag(&loadshr_scode, seg->dcss_name, 353 &start_addr, &end_addr); 354 if (diag_cc < 0) { 355 dcss_diag(&purgeseg_scode, seg->dcss_name, 356 &dummy, &dummy); 357 rc = diag_cc; 358 goto out_mapping; 359 } 360 if (diag_cc > 1) { 361 pr_warn("Loading DCSS %s failed with rc=%ld\n", name, end_addr); 362 rc = dcss_diag_translate_rc(end_addr); 363 dcss_diag(&purgeseg_scode, seg->dcss_name, 364 &dummy, &dummy); 365 goto out_mapping; 366 } 367 seg->start_addr = start_addr; 368 seg->end = end_addr; 369 seg->do_nonshared = do_nonshared; 370 refcount_set(&seg->ref_count, 1); 371 list_add(&seg->list, &dcss_list); 372 *addr = seg->start_addr; 373 *end = seg->end; 374 if (do_nonshared) 375 pr_info("DCSS %s of range %px to %px and type %s loaded as " 376 "exclusive-writable\n", name, (void*) seg->start_addr, 377 (void*) seg->end, segtype_string[seg->vm_segtype]); 378 else { 379 pr_info("DCSS %s of range %px to %px and type %s loaded in " 380 "shared access mode\n", name, (void*) seg->start_addr, 381 (void*) seg->end, segtype_string[seg->vm_segtype]); 382 } 383 goto out; 384 out_mapping: 385 vmem_remove_mapping(seg->start_addr, seg->end - seg->start_addr + 1); 386 out_resource: 387 release_resource(seg->res); 388 out_free_resource: 389 kfree(seg->res); 390 out_free: 391 kfree(seg); 392 out: 393 return rc < 0 ? rc : segtype; 394 } 395 396 /* 397 * this function loads a DCSS segment 398 * name : name of the DCSS 399 * do_nonshared : 0 indicates that the dcss should be shared with other linux images 400 * 1 indicates that the dcss should be exclusive for this linux image 401 * addr : will be filled with start address of the segment 402 * end : will be filled with end address of the segment 403 * return values: 404 * -ENOSYS : we are not running on VM 405 * -EIO : could not perform query or load diagnose 406 * -ENOENT : no such segment 407 * -EOPNOTSUPP: multi-part segment cannot be used with linux 408 * -EBUSY : segment cannot be used (overlaps with dcss or storage) 409 * -ERANGE : segment cannot be used (exceeds kernel mapping range) 410 * -EPERM : segment is currently loaded with incompatible permissions 411 * -ENOMEM : out of memory 412 * 0 .. 6 : type of segment as defined in include/asm-s390/extmem.h 413 */ 414 int 415 segment_load (char *name, int do_nonshared, unsigned long *addr, 416 unsigned long *end) 417 { 418 struct dcss_segment *seg; 419 int rc; 420 421 if (!machine_is_vm()) 422 return -ENOSYS; 423 424 mutex_lock(&dcss_lock); 425 seg = segment_by_name (name); 426 if (seg == NULL) 427 rc = __segment_load (name, do_nonshared, addr, end); 428 else { 429 if (do_nonshared == seg->do_nonshared) { 430 refcount_inc(&seg->ref_count); 431 *addr = seg->start_addr; 432 *end = seg->end; 433 rc = seg->vm_segtype; 434 } else { 435 *addr = *end = 0; 436 rc = -EPERM; 437 } 438 } 439 mutex_unlock(&dcss_lock); 440 return rc; 441 } 442 443 /* 444 * this function modifies the shared state of a DCSS segment. note that 445 * name : name of the DCSS 446 * do_nonshared : 0 indicates that the dcss should be shared with other linux images 447 * 1 indicates that the dcss should be exclusive for this linux image 448 * return values: 449 * -EIO : could not perform load diagnose (segment gone!) 450 * -ENOENT : no such segment (segment gone!) 451 * -EAGAIN : segment is in use by other exploiters, try later 452 * -EINVAL : no segment with the given name is currently loaded - name invalid 453 * -EBUSY : segment can temporarily not be used (overlaps with dcss) 454 * 0 : operation succeeded 455 */ 456 int 457 segment_modify_shared (char *name, int do_nonshared) 458 { 459 struct dcss_segment *seg; 460 unsigned long start_addr, end_addr, dummy; 461 int rc, diag_cc; 462 463 start_addr = end_addr = 0; 464 mutex_lock(&dcss_lock); 465 seg = segment_by_name (name); 466 if (seg == NULL) { 467 rc = -EINVAL; 468 goto out_unlock; 469 } 470 if (do_nonshared == seg->do_nonshared) { 471 pr_info("DCSS %s is already in the requested access " 472 "mode\n", name); 473 rc = 0; 474 goto out_unlock; 475 } 476 if (refcount_read(&seg->ref_count) != 1) { 477 pr_warn("DCSS %s is in use and cannot be reloaded\n", name); 478 rc = -EAGAIN; 479 goto out_unlock; 480 } 481 release_resource(seg->res); 482 if (do_nonshared) 483 seg->res->flags &= ~IORESOURCE_READONLY; 484 else 485 if (seg->vm_segtype == SEG_TYPE_SR || 486 seg->vm_segtype == SEG_TYPE_ER) 487 seg->res->flags |= IORESOURCE_READONLY; 488 489 if (request_resource(&iomem_resource, seg->res)) { 490 pr_warn("DCSS %s overlaps with used memory resources and cannot be reloaded\n", 491 name); 492 rc = -EBUSY; 493 kfree(seg->res); 494 goto out_del_mem; 495 } 496 497 dcss_diag(&purgeseg_scode, seg->dcss_name, &dummy, &dummy); 498 if (do_nonshared) 499 diag_cc = dcss_diag(&loadnsr_scode, seg->dcss_name, 500 &start_addr, &end_addr); 501 else 502 diag_cc = dcss_diag(&loadshr_scode, seg->dcss_name, 503 &start_addr, &end_addr); 504 if (diag_cc < 0) { 505 rc = diag_cc; 506 goto out_del_res; 507 } 508 if (diag_cc > 1) { 509 pr_warn("Reloading DCSS %s failed with rc=%ld\n", 510 name, end_addr); 511 rc = dcss_diag_translate_rc(end_addr); 512 goto out_del_res; 513 } 514 seg->start_addr = start_addr; 515 seg->end = end_addr; 516 seg->do_nonshared = do_nonshared; 517 rc = 0; 518 goto out_unlock; 519 out_del_res: 520 release_resource(seg->res); 521 kfree(seg->res); 522 out_del_mem: 523 vmem_remove_mapping(seg->start_addr, seg->end - seg->start_addr + 1); 524 list_del(&seg->list); 525 dcss_diag(&purgeseg_scode, seg->dcss_name, &dummy, &dummy); 526 kfree(seg); 527 out_unlock: 528 mutex_unlock(&dcss_lock); 529 return rc; 530 } 531 532 static void __dcss_diag_purge_on_cpu_0(void *data) 533 { 534 struct dcss_segment *seg = (struct dcss_segment *)data; 535 unsigned long dummy; 536 537 dcss_diag(&purgeseg_scode, seg->dcss_name, &dummy, &dummy); 538 } 539 540 /* 541 * Decrease the use count of a DCSS segment and remove 542 * it from the address space if nobody is using it 543 * any longer. 544 */ 545 void 546 segment_unload(char *name) 547 { 548 struct dcss_segment *seg; 549 550 if (!machine_is_vm()) 551 return; 552 553 mutex_lock(&dcss_lock); 554 seg = segment_by_name (name); 555 if (seg == NULL) { 556 pr_err("Unloading unknown DCSS %s failed\n", name); 557 goto out_unlock; 558 } 559 if (!refcount_dec_and_test(&seg->ref_count)) 560 goto out_unlock; 561 release_resource(seg->res); 562 kfree(seg->res); 563 vmem_remove_mapping(seg->start_addr, seg->end - seg->start_addr + 1); 564 list_del(&seg->list); 565 /* 566 * Workaround for z/VM issue, where calling the DCSS unload diag on 567 * a non-IPL CPU would cause bogus sclp maximum memory detection on 568 * next IPL. 569 * IPL CPU 0 cannot be set offline, so the dcss_diag() call can 570 * directly be scheduled to that CPU. 571 */ 572 smp_call_function_single(0, __dcss_diag_purge_on_cpu_0, seg, 1); 573 kfree(seg); 574 out_unlock: 575 mutex_unlock(&dcss_lock); 576 } 577 578 /* 579 * save segment content permanently 580 */ 581 void 582 segment_save(char *name) 583 { 584 struct dcss_segment *seg; 585 char cmd1[160]; 586 char cmd2[80]; 587 int i, response; 588 589 if (!machine_is_vm()) 590 return; 591 592 mutex_lock(&dcss_lock); 593 seg = segment_by_name (name); 594 595 if (seg == NULL) { 596 pr_err("Saving unknown DCSS %s failed\n", name); 597 goto out; 598 } 599 600 snprintf(cmd1, sizeof(cmd1), "DEFSEG %s", name); 601 for (i=0; i<seg->segcnt; i++) { 602 size_t len = strlen(cmd1); 603 604 snprintf(cmd1 + len, sizeof(cmd1) - len, " %lX-%lX %s", 605 seg->range[i].start >> PAGE_SHIFT, 606 seg->range[i].end >> PAGE_SHIFT, 607 segtype_string[seg->range[i].start & 0xff]); 608 } 609 snprintf(cmd2, sizeof(cmd2), "SAVESEG %s", name); 610 response = 0; 611 cpcmd(cmd1, NULL, 0, &response); 612 if (response) { 613 pr_err("Saving a DCSS failed with DEFSEG response code " 614 "%i\n", response); 615 goto out; 616 } 617 cpcmd(cmd2, NULL, 0, &response); 618 if (response) { 619 pr_err("Saving a DCSS failed with SAVESEG response code " 620 "%i\n", response); 621 goto out; 622 } 623 out: 624 mutex_unlock(&dcss_lock); 625 } 626 627 /* 628 * print appropriate error message for segment_load()/segment_type() 629 * return code 630 */ 631 void segment_warning(int rc, char *seg_name) 632 { 633 switch (rc) { 634 case -ENOENT: 635 pr_err("DCSS %s cannot be loaded or queried\n", seg_name); 636 break; 637 case -ENOSYS: 638 pr_err("DCSS %s cannot be loaded or queried without " 639 "z/VM\n", seg_name); 640 break; 641 case -EIO: 642 pr_err("Loading or querying DCSS %s resulted in a " 643 "hardware error\n", seg_name); 644 break; 645 case -EOPNOTSUPP: 646 pr_err("DCSS %s has multiple page ranges and cannot be " 647 "loaded or queried\n", seg_name); 648 break; 649 case -EBUSY: 650 pr_err("%s needs used memory resources and cannot be " 651 "loaded or queried\n", seg_name); 652 break; 653 case -EPERM: 654 pr_err("DCSS %s is already loaded in a different access " 655 "mode\n", seg_name); 656 break; 657 case -ENOMEM: 658 pr_err("There is not enough memory to load or query " 659 "DCSS %s\n", seg_name); 660 break; 661 case -ERANGE: { 662 struct range mhp_range = arch_get_mappable_range(); 663 664 pr_err("DCSS %s exceeds the kernel mapping range (%llu) " 665 "and cannot be loaded\n", seg_name, mhp_range.end + 1); 666 break; 667 } 668 default: 669 break; 670 } 671 } 672 673 EXPORT_SYMBOL(segment_load); 674 EXPORT_SYMBOL(segment_unload); 675 EXPORT_SYMBOL(segment_save); 676 EXPORT_SYMBOL(segment_type); 677 EXPORT_SYMBOL(segment_modify_shared); 678 EXPORT_SYMBOL(segment_warning); 679