1 // SPDX-License-Identifier: GPL-2.0-or-later
2
3 /*
4 * Copyright 2020 IBM Corp.
5 *
6 * Author: Bulent Abali <abali@us.ibm.com>
7 *
8 */
9 #include <stdio.h>
10 #include <stdlib.h>
11 #include <string.h>
12 #include <unistd.h>
13 #include <stdint.h>
14 #include <sys/types.h>
15 #include <sys/stat.h>
16 #include <sys/time.h>
17 #include <sys/fcntl.h>
18 #include <sys/mman.h>
19 #include <endian.h>
20 #include <bits/endian.h>
21 #include <sys/ioctl.h>
22 #include <assert.h>
23 #include <errno.h>
24 #include <signal.h>
25 #include "vas-api.h"
26 #include "nx.h"
27 #include "copy-paste.h"
28 #include "nxu.h"
29 #include "nx_dbg.h"
30 #include <sys/platform/ppc.h>
31
32 #define barrier()
33 #define hwsync() ({ asm volatile("sync" ::: "memory"); })
34
35 #ifndef NX_NO_CPU_PRI
36 #define cpu_pri_default() ({ asm volatile ("or 2, 2, 2"); })
37 #define cpu_pri_low() ({ asm volatile ("or 31, 31, 31"); })
38 #else
39 #define cpu_pri_default()
40 #define cpu_pri_low()
41 #endif
42
43 void *nx_fault_storage_address;
44
45 struct nx_handle {
46 int fd;
47 int function;
48 void *paste_addr;
49 };
50
open_device_nodes(char * devname,int pri,struct nx_handle * handle)51 static int open_device_nodes(char *devname, int pri, struct nx_handle *handle)
52 {
53 int rc, fd;
54 void *addr;
55 struct vas_tx_win_open_attr txattr;
56
57 fd = open(devname, O_RDWR);
58 if (fd < 0) {
59 fprintf(stderr, " open device name %s\n", devname);
60 return -errno;
61 }
62
63 memset(&txattr, 0, sizeof(txattr));
64 txattr.version = 1;
65 txattr.vas_id = pri;
66 rc = ioctl(fd, VAS_TX_WIN_OPEN, (unsigned long)&txattr);
67 if (rc < 0) {
68 fprintf(stderr, "ioctl() n %d, error %d\n", rc, errno);
69 rc = -errno;
70 goto out;
71 }
72
73 addr = mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0ULL);
74 if (addr == MAP_FAILED) {
75 fprintf(stderr, "mmap() failed, errno %d\n", errno);
76 rc = -errno;
77 goto out;
78 }
79 handle->fd = fd;
80 handle->paste_addr = (void *)((char *)addr + 0x400);
81
82 rc = 0;
83 out:
84 close(fd);
85 return rc;
86 }
87
nx_function_begin(int function,int pri)88 void *nx_function_begin(int function, int pri)
89 {
90 int rc;
91 char *devname = "/dev/crypto/nx-gzip";
92 struct nx_handle *nxhandle;
93
94 if (function != NX_FUNC_COMP_GZIP) {
95 errno = EINVAL;
96 fprintf(stderr, " NX_FUNC_COMP_GZIP not found\n");
97 return NULL;
98 }
99
100
101 nxhandle = malloc(sizeof(*nxhandle));
102 if (!nxhandle) {
103 errno = ENOMEM;
104 fprintf(stderr, " No memory\n");
105 return NULL;
106 }
107
108 nxhandle->function = function;
109 rc = open_device_nodes(devname, pri, nxhandle);
110 if (rc < 0) {
111 errno = -rc;
112 fprintf(stderr, " open_device_nodes failed\n");
113 return NULL;
114 }
115
116 return nxhandle;
117 }
118
nx_function_end(void * handle)119 int nx_function_end(void *handle)
120 {
121 int rc = 0;
122 struct nx_handle *nxhandle = handle;
123
124 rc = munmap(nxhandle->paste_addr - 0x400, 4096);
125 if (rc < 0) {
126 fprintf(stderr, "munmap() failed, errno %d\n", errno);
127 return rc;
128 }
129 close(nxhandle->fd);
130 free(nxhandle);
131
132 return rc;
133 }
134
nx_wait_for_csb(struct nx_gzip_crb_cpb_t * cmdp)135 static int nx_wait_for_csb(struct nx_gzip_crb_cpb_t *cmdp)
136 {
137 long poll = 0;
138 uint64_t t;
139
140 /* Save power and let other threads use the h/w. top may show
141 * 100% but only because OS doesn't know we slowed the this
142 * h/w thread while polling. We're letting other threads have
143 * higher throughput on the core.
144 */
145 cpu_pri_low();
146
147 #define CSB_MAX_POLL 200000000UL
148 #define USLEEP_TH 300000UL
149
150 t = __ppc_get_timebase();
151
152 while (getnn(cmdp->crb.csb, csb_v) == 0) {
153 ++poll;
154 hwsync();
155
156 cpu_pri_low();
157
158 /* usleep(0) takes around 29000 ticks ~60 us.
159 * 300000 is spinning for about 600 us then
160 * start sleeping.
161 */
162 if ((__ppc_get_timebase() - t) > USLEEP_TH) {
163 cpu_pri_default();
164 usleep(1);
165 }
166
167 if (poll > CSB_MAX_POLL)
168 break;
169
170 /* Fault address from signal handler */
171 if (nx_fault_storage_address) {
172 cpu_pri_default();
173 return -EAGAIN;
174 }
175
176 }
177
178 cpu_pri_default();
179
180 /* hw has updated csb and output buffer */
181 hwsync();
182
183 /* Check CSB flags. */
184 if (getnn(cmdp->crb.csb, csb_v) == 0) {
185 fprintf(stderr, "CSB still not valid after %d polls.\n",
186 (int) poll);
187 prt_err("CSB still not valid after %d polls, giving up.\n",
188 (int) poll);
189 return -ETIMEDOUT;
190 }
191
192 return 0;
193 }
194
nxu_run_job(struct nx_gzip_crb_cpb_t * cmdp,void * handle)195 static int nxu_run_job(struct nx_gzip_crb_cpb_t *cmdp, void *handle)
196 {
197 int i, ret, retries;
198 struct nx_handle *nxhandle = handle;
199
200 assert(handle != NULL);
201 i = 0;
202 retries = 5000;
203 while (i++ < retries) {
204 hwsync();
205 vas_copy(&cmdp->crb, 0);
206 ret = vas_paste(nxhandle->paste_addr, 0);
207 hwsync();
208
209 NXPRT(fprintf(stderr, "Paste attempt %d/%d returns 0x%x\n",
210 i, retries, ret));
211
212 if ((ret == 2) || (ret == 3)) {
213
214 ret = nx_wait_for_csb(cmdp);
215 if (!ret) {
216 goto out;
217 } else if (ret == -EAGAIN) {
218 long x;
219
220 prt_err("Touching address %p, 0x%lx\n",
221 nx_fault_storage_address,
222 *(long *) nx_fault_storage_address);
223 x = *(long *) nx_fault_storage_address;
224 *(long *) nx_fault_storage_address = x;
225 nx_fault_storage_address = 0;
226 continue;
227 } else {
228 prt_err("wait_for_csb() returns %d\n", ret);
229 break;
230 }
231 } else {
232 if (i < 10) {
233 /* spin for few ticks */
234 #define SPIN_TH 500UL
235 uint64_t fail_spin;
236
237 fail_spin = __ppc_get_timebase();
238 while ((__ppc_get_timebase() - fail_spin) <
239 SPIN_TH)
240 ;
241 } else {
242 /* sleep */
243 unsigned int pr = 0;
244
245 if (pr++ % 100 == 0) {
246 prt_err("Paste attempt %d/", i);
247 prt_err("%d, failed pid= %d\n", retries,
248 getpid());
249 }
250 usleep(1);
251 }
252 continue;
253 }
254 }
255
256 out:
257 cpu_pri_default();
258
259 return ret;
260 }
261
nxu_submit_job(struct nx_gzip_crb_cpb_t * cmdp,void * handle)262 int nxu_submit_job(struct nx_gzip_crb_cpb_t *cmdp, void *handle)
263 {
264 int cc;
265
266 cc = nxu_run_job(cmdp, handle);
267
268 if (!cc)
269 cc = getnn(cmdp->crb.csb, csb_cc); /* CC Table 6-8 */
270
271 return cc;
272 }
273
274
nxu_sigsegv_handler(int sig,siginfo_t * info,void * ctx)275 void nxu_sigsegv_handler(int sig, siginfo_t *info, void *ctx)
276 {
277 fprintf(stderr, "%d: Got signal %d si_code %d, si_addr %p\n", getpid(),
278 sig, info->si_code, info->si_addr);
279
280 nx_fault_storage_address = info->si_addr;
281 }
282
283 /*
284 * Fault in pages prior to NX job submission. wr=1 may be required to
285 * touch writeable pages. System zero pages do not fault-in the page as
286 * intended. Typically set wr=1 for NX target pages and set wr=0 for NX
287 * source pages.
288 */
nxu_touch_pages(void * buf,long buf_len,long page_len,int wr)289 int nxu_touch_pages(void *buf, long buf_len, long page_len, int wr)
290 {
291 char *begin = buf;
292 char *end = (char *) buf + buf_len - 1;
293 volatile char t;
294
295 assert(buf_len >= 0 && !!buf);
296
297 NXPRT(fprintf(stderr, "touch %p %p len 0x%lx wr=%d\n", buf,
298 (buf + buf_len), buf_len, wr));
299
300 if (buf_len <= 0 || buf == NULL)
301 return -1;
302
303 do {
304 t = *begin;
305 if (wr)
306 *begin = t;
307 begin = begin + page_len;
308 } while (begin < end);
309
310 /* When buf_sz is small or buf tail is in another page */
311 t = *end;
312 if (wr)
313 *end = t;
314
315 return 0;
316 }
317