1 // SPDX-License-Identifier: GPL-2.0-or-later 2 3 /* 4 * Copyright 2020 IBM Corp. 5 * 6 * Author: Bulent Abali <abali@us.ibm.com> 7 * 8 */ 9 #include <stdio.h> 10 #include <stdlib.h> 11 #include <string.h> 12 #include <unistd.h> 13 #include <stdint.h> 14 #include <sys/types.h> 15 #include <sys/stat.h> 16 #include <sys/time.h> 17 #include <sys/fcntl.h> 18 #include <sys/mman.h> 19 #include <endian.h> 20 #include <bits/endian.h> 21 #include <sys/ioctl.h> 22 #include <assert.h> 23 #include <errno.h> 24 #include <signal.h> 25 #include "vas-api.h" 26 #include "nx.h" 27 #include "copy-paste.h" 28 #include "nxu.h" 29 #include "nx_dbg.h" 30 #include <sys/platform/ppc.h> 31 32 #define barrier() 33 #define hwsync() ({ asm volatile("sync" ::: "memory"); }) 34 35 #ifndef NX_NO_CPU_PRI 36 #define cpu_pri_default() ({ asm volatile ("or 2, 2, 2"); }) 37 #define cpu_pri_low() ({ asm volatile ("or 31, 31, 31"); }) 38 #else 39 #define cpu_pri_default() 40 #define cpu_pri_low() 41 #endif 42 43 void *nx_fault_storage_address; 44 45 struct nx_handle { 46 int fd; 47 int function; 48 void *paste_addr; 49 }; 50 51 static int open_device_nodes(char *devname, int pri, struct nx_handle *handle) 52 { 53 int rc, fd; 54 void *addr; 55 struct vas_tx_win_open_attr txattr; 56 57 fd = open(devname, O_RDWR); 58 if (fd < 0) { 59 fprintf(stderr, " open device name %s\n", devname); 60 return -errno; 61 } 62 63 memset(&txattr, 0, sizeof(txattr)); 64 txattr.version = 1; 65 txattr.vas_id = pri; 66 rc = ioctl(fd, VAS_TX_WIN_OPEN, (unsigned long)&txattr); 67 if (rc < 0) { 68 fprintf(stderr, "ioctl() n %d, error %d\n", rc, errno); 69 rc = -errno; 70 goto out; 71 } 72 73 addr = mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0ULL); 74 if (addr == MAP_FAILED) { 75 fprintf(stderr, "mmap() failed, errno %d\n", errno); 76 rc = -errno; 77 goto out; 78 } 79 handle->fd = fd; 80 handle->paste_addr = (void *)((char *)addr + 0x400); 81 82 rc = 0; 83 out: 84 close(fd); 85 return rc; 86 } 87 88 void *nx_function_begin(int function, int pri) 89 { 90 int rc; 91 char *devname = "/dev/crypto/nx-gzip"; 92 struct nx_handle *nxhandle; 93 94 if (function != NX_FUNC_COMP_GZIP) { 95 errno = EINVAL; 96 fprintf(stderr, " NX_FUNC_COMP_GZIP not found\n"); 97 return NULL; 98 } 99 100 101 nxhandle = malloc(sizeof(*nxhandle)); 102 if (!nxhandle) { 103 errno = ENOMEM; 104 fprintf(stderr, " No memory\n"); 105 return NULL; 106 } 107 108 nxhandle->function = function; 109 rc = open_device_nodes(devname, pri, nxhandle); 110 if (rc < 0) { 111 errno = -rc; 112 fprintf(stderr, " open_device_nodes failed\n"); 113 return NULL; 114 } 115 116 return nxhandle; 117 } 118 119 int nx_function_end(void *handle) 120 { 121 int rc = 0; 122 struct nx_handle *nxhandle = handle; 123 124 rc = munmap(nxhandle->paste_addr - 0x400, 4096); 125 if (rc < 0) { 126 fprintf(stderr, "munmap() failed, errno %d\n", errno); 127 return rc; 128 } 129 close(nxhandle->fd); 130 free(nxhandle); 131 132 return rc; 133 } 134 135 static int nx_wait_for_csb(struct nx_gzip_crb_cpb_t *cmdp) 136 { 137 long poll = 0; 138 uint64_t t; 139 140 /* Save power and let other threads use the h/w. top may show 141 * 100% but only because OS doesn't know we slowed the this 142 * h/w thread while polling. We're letting other threads have 143 * higher throughput on the core. 144 */ 145 cpu_pri_low(); 146 147 #define CSB_MAX_POLL 200000000UL 148 #define USLEEP_TH 300000UL 149 150 t = __ppc_get_timebase(); 151 152 while (getnn(cmdp->crb.csb, csb_v) == 0) { 153 ++poll; 154 hwsync(); 155 156 cpu_pri_low(); 157 158 /* usleep(0) takes around 29000 ticks ~60 us. 159 * 300000 is spinning for about 600 us then 160 * start sleeping. 161 */ 162 if ((__ppc_get_timebase() - t) > USLEEP_TH) { 163 cpu_pri_default(); 164 usleep(1); 165 } 166 167 if (poll > CSB_MAX_POLL) 168 break; 169 170 /* Fault address from signal handler */ 171 if (nx_fault_storage_address) { 172 cpu_pri_default(); 173 return -EAGAIN; 174 } 175 176 } 177 178 cpu_pri_default(); 179 180 /* hw has updated csb and output buffer */ 181 hwsync(); 182 183 /* Check CSB flags. */ 184 if (getnn(cmdp->crb.csb, csb_v) == 0) { 185 fprintf(stderr, "CSB still not valid after %d polls.\n", 186 (int) poll); 187 prt_err("CSB still not valid after %d polls, giving up.\n", 188 (int) poll); 189 return -ETIMEDOUT; 190 } 191 192 return 0; 193 } 194 195 static int nxu_run_job(struct nx_gzip_crb_cpb_t *cmdp, void *handle) 196 { 197 int i, ret, retries; 198 struct nx_handle *nxhandle = handle; 199 200 assert(handle != NULL); 201 i = 0; 202 retries = 5000; 203 while (i++ < retries) { 204 hwsync(); 205 vas_copy(&cmdp->crb, 0); 206 ret = vas_paste(nxhandle->paste_addr, 0); 207 hwsync(); 208 209 NXPRT(fprintf(stderr, "Paste attempt %d/%d returns 0x%x\n", 210 i, retries, ret)); 211 212 if ((ret == 2) || (ret == 3)) { 213 214 ret = nx_wait_for_csb(cmdp); 215 if (!ret) { 216 goto out; 217 } else if (ret == -EAGAIN) { 218 long x; 219 220 prt_err("Touching address %p, 0x%lx\n", 221 nx_fault_storage_address, 222 *(long *) nx_fault_storage_address); 223 x = *(long *) nx_fault_storage_address; 224 *(long *) nx_fault_storage_address = x; 225 nx_fault_storage_address = 0; 226 continue; 227 } else { 228 prt_err("wait_for_csb() returns %d\n", ret); 229 break; 230 } 231 } else { 232 if (i < 10) { 233 /* spin for few ticks */ 234 #define SPIN_TH 500UL 235 uint64_t fail_spin; 236 237 fail_spin = __ppc_get_timebase(); 238 while ((__ppc_get_timebase() - fail_spin) < 239 SPIN_TH) 240 ; 241 } else { 242 /* sleep */ 243 unsigned int pr = 0; 244 245 if (pr++ % 100 == 0) { 246 prt_err("Paste attempt %d/", i); 247 prt_err("%d, failed pid= %d\n", retries, 248 getpid()); 249 } 250 usleep(1); 251 } 252 continue; 253 } 254 } 255 256 out: 257 cpu_pri_default(); 258 259 return ret; 260 } 261 262 int nxu_submit_job(struct nx_gzip_crb_cpb_t *cmdp, void *handle) 263 { 264 int cc; 265 266 cc = nxu_run_job(cmdp, handle); 267 268 if (!cc) 269 cc = getnn(cmdp->crb.csb, csb_cc); /* CC Table 6-8 */ 270 271 return cc; 272 } 273 274 275 void nxu_sigsegv_handler(int sig, siginfo_t *info, void *ctx) 276 { 277 fprintf(stderr, "%d: Got signal %d si_code %d, si_addr %p\n", getpid(), 278 sig, info->si_code, info->si_addr); 279 280 nx_fault_storage_address = info->si_addr; 281 } 282 283 /* 284 * Fault in pages prior to NX job submission. wr=1 may be required to 285 * touch writeable pages. System zero pages do not fault-in the page as 286 * intended. Typically set wr=1 for NX target pages and set wr=0 for NX 287 * source pages. 288 */ 289 int nxu_touch_pages(void *buf, long buf_len, long page_len, int wr) 290 { 291 char *begin = buf; 292 char *end = (char *) buf + buf_len - 1; 293 volatile char t; 294 295 assert(buf_len >= 0 && !!buf); 296 297 NXPRT(fprintf(stderr, "touch %p %p len 0x%lx wr=%d\n", buf, 298 (buf + buf_len), buf_len, wr)); 299 300 if (buf_len <= 0 || buf == NULL) 301 return -1; 302 303 do { 304 t = *begin; 305 if (wr) 306 *begin = t; 307 begin = begin + page_len; 308 } while (begin < end); 309 310 /* When buf_sz is small or buf tail is in another page */ 311 t = *end; 312 if (wr) 313 *end = t; 314 315 return 0; 316 } 317