1 /*
2 * Copyright 2013 Red Hat Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Ben Skeggs
23 * Roy Spliet <rspliet@eclipso.eu>
24 */
25 #define gt215_ram(p) container_of((p), struct gt215_ram, base)
26 #include "ram.h"
27 #include "ramfuc.h"
28
29 #include <core/memory.h>
30 #include <core/option.h>
31 #include <subdev/bios.h>
32 #include <subdev/bios/M0205.h>
33 #include <subdev/bios/rammap.h>
34 #include <subdev/bios/timing.h>
35 #include <subdev/clk/gt215.h>
36 #include <subdev/gpio.h>
37
38 struct gt215_ramfuc {
39 struct ramfuc base;
40 struct ramfuc_reg r_0x001610;
41 struct ramfuc_reg r_0x001700;
42 struct ramfuc_reg r_0x002504;
43 struct ramfuc_reg r_0x004000;
44 struct ramfuc_reg r_0x004004;
45 struct ramfuc_reg r_0x004018;
46 struct ramfuc_reg r_0x004128;
47 struct ramfuc_reg r_0x004168;
48 struct ramfuc_reg r_0x100080;
49 struct ramfuc_reg r_0x100200;
50 struct ramfuc_reg r_0x100210;
51 struct ramfuc_reg r_0x100220[9];
52 struct ramfuc_reg r_0x100264;
53 struct ramfuc_reg r_0x1002d0;
54 struct ramfuc_reg r_0x1002d4;
55 struct ramfuc_reg r_0x1002dc;
56 struct ramfuc_reg r_0x10053c;
57 struct ramfuc_reg r_0x1005a0;
58 struct ramfuc_reg r_0x1005a4;
59 struct ramfuc_reg r_0x100700;
60 struct ramfuc_reg r_0x100714;
61 struct ramfuc_reg r_0x100718;
62 struct ramfuc_reg r_0x10071c;
63 struct ramfuc_reg r_0x100720;
64 struct ramfuc_reg r_0x100760;
65 struct ramfuc_reg r_0x1007a0;
66 struct ramfuc_reg r_0x1007e0;
67 struct ramfuc_reg r_0x100da0;
68 struct ramfuc_reg r_0x10f804;
69 struct ramfuc_reg r_0x1110e0;
70 struct ramfuc_reg r_0x111100;
71 struct ramfuc_reg r_0x111104;
72 struct ramfuc_reg r_0x1111e0;
73 struct ramfuc_reg r_0x111400;
74 struct ramfuc_reg r_0x611200;
75 struct ramfuc_reg r_mr[4];
76 struct ramfuc_reg r_gpio[4];
77 };
78
79 struct gt215_ltrain {
80 enum {
81 NVA3_TRAIN_UNKNOWN,
82 NVA3_TRAIN_UNSUPPORTED,
83 NVA3_TRAIN_ONCE,
84 NVA3_TRAIN_EXEC,
85 NVA3_TRAIN_DONE
86 } state;
87 u32 r_100720;
88 u32 r_1111e0;
89 u32 r_111400;
90 struct nvkm_memory *memory;
91 };
92
93 struct gt215_ram {
94 struct nvkm_ram base;
95 struct gt215_ramfuc fuc;
96 struct gt215_ltrain ltrain;
97 };
98
99 static void
gt215_link_train_calc(u32 * vals,struct gt215_ltrain * train)100 gt215_link_train_calc(u32 *vals, struct gt215_ltrain *train)
101 {
102 int i, lo, hi;
103 u8 median[8], bins[4] = {0, 0, 0, 0}, bin = 0, qty = 0;
104
105 for (i = 0; i < 8; i++) {
106 for (lo = 0; lo < 0x40; lo++) {
107 if (!(vals[lo] & 0x80000000))
108 continue;
109 if (vals[lo] & (0x101 << i))
110 break;
111 }
112
113 if (lo == 0x40)
114 return;
115
116 for (hi = lo + 1; hi < 0x40; hi++) {
117 if (!(vals[lo] & 0x80000000))
118 continue;
119 if (!(vals[hi] & (0x101 << i))) {
120 hi--;
121 break;
122 }
123 }
124
125 median[i] = ((hi - lo) >> 1) + lo;
126 bins[(median[i] & 0xf0) >> 4]++;
127 median[i] += 0x30;
128 }
129
130 /* Find the best value for 0x1111e0 */
131 for (i = 0; i < 4; i++) {
132 if (bins[i] > qty) {
133 bin = i + 3;
134 qty = bins[i];
135 }
136 }
137
138 train->r_100720 = 0;
139 for (i = 0; i < 8; i++) {
140 median[i] = max(median[i], (u8) (bin << 4));
141 median[i] = min(median[i], (u8) ((bin << 4) | 0xf));
142
143 train->r_100720 |= ((median[i] & 0x0f) << (i << 2));
144 }
145
146 train->r_1111e0 = 0x02000000 | (bin * 0x101);
147 train->r_111400 = 0x0;
148 }
149
150 /*
151 * Link training for (at least) DDR3
152 */
153 static int
gt215_link_train(struct gt215_ram * ram)154 gt215_link_train(struct gt215_ram *ram)
155 {
156 struct gt215_ltrain *train = &ram->ltrain;
157 struct gt215_ramfuc *fuc = &ram->fuc;
158 struct nvkm_subdev *subdev = &ram->base.fb->subdev;
159 struct nvkm_device *device = subdev->device;
160 struct nvkm_bios *bios = device->bios;
161 struct nvkm_clk *clk = device->clk;
162 u32 *result, r1700;
163 int ret, i;
164 struct nvbios_M0205T M0205T = { 0 };
165 u8 ver, hdr, cnt, len, snr, ssz;
166 unsigned int clk_current;
167 unsigned long flags;
168 unsigned long *f = &flags;
169
170 if (nvkm_boolopt(device->cfgopt, "NvMemExec", true) != true)
171 return -ENOSYS;
172
173 /* XXX: Multiple partitions? */
174 result = kmalloc_array(64, sizeof(u32), GFP_KERNEL);
175 if (!result)
176 return -ENOMEM;
177
178 train->state = NVA3_TRAIN_EXEC;
179
180 /* Clock speeds for training and back */
181 nvbios_M0205Tp(bios, &ver, &hdr, &cnt, &len, &snr, &ssz, &M0205T);
182 if (M0205T.freq == 0) {
183 kfree(result);
184 return -ENOENT;
185 }
186
187 clk_current = nvkm_clk_read(clk, nv_clk_src_mem);
188
189 ret = gt215_clk_pre(clk, f);
190 if (ret)
191 goto out;
192
193 /* First: clock up/down */
194 ret = ram->base.func->calc(&ram->base, (u32) M0205T.freq * 1000);
195 if (ret)
196 goto out;
197
198 /* Do this *after* calc, eliminates write in script */
199 nvkm_wr32(device, 0x111400, 0x00000000);
200 /* XXX: Magic writes that improve train reliability? */
201 nvkm_mask(device, 0x100674, 0x0000ffff, 0x00000000);
202 nvkm_mask(device, 0x1005e4, 0x0000ffff, 0x00000000);
203 nvkm_mask(device, 0x100b0c, 0x000000ff, 0x00000000);
204 nvkm_wr32(device, 0x100c04, 0x00000400);
205
206 /* Now the training script */
207 r1700 = ram_rd32(fuc, 0x001700);
208
209 ram_mask(fuc, 0x100200, 0x00000800, 0x00000000);
210 ram_wr32(fuc, 0x611200, 0x3300);
211 ram_wait_vblank(fuc);
212 ram_wait(fuc, 0x611200, 0x00000003, 0x00000000, 500000);
213 ram_mask(fuc, 0x001610, 0x00000083, 0x00000003);
214 ram_mask(fuc, 0x100080, 0x00000020, 0x00000000);
215 ram_mask(fuc, 0x10f804, 0x80000000, 0x00000000);
216 ram_wr32(fuc, 0x001700, 0x00000000);
217
218 ram_train(fuc);
219
220 /* Reset */
221 ram_mask(fuc, 0x10f804, 0x80000000, 0x80000000);
222 ram_wr32(fuc, 0x10053c, 0x0);
223 ram_wr32(fuc, 0x100720, train->r_100720);
224 ram_wr32(fuc, 0x1111e0, train->r_1111e0);
225 ram_wr32(fuc, 0x111400, train->r_111400);
226 ram_nuke(fuc, 0x100080);
227 ram_mask(fuc, 0x100080, 0x00000020, 0x00000020);
228 ram_nsec(fuc, 1000);
229
230 ram_wr32(fuc, 0x001700, r1700);
231 ram_mask(fuc, 0x001610, 0x00000083, 0x00000080);
232 ram_wr32(fuc, 0x611200, 0x3330);
233 ram_mask(fuc, 0x100200, 0x00000800, 0x00000800);
234
235 ram_exec(fuc, true);
236
237 ram->base.func->calc(&ram->base, clk_current);
238 ram_exec(fuc, true);
239
240 /* Post-processing, avoids flicker */
241 nvkm_mask(device, 0x616308, 0x10, 0x10);
242 nvkm_mask(device, 0x616b08, 0x10, 0x10);
243
244 gt215_clk_post(clk, f);
245
246 ram_train_result(ram->base.fb, result, 64);
247 for (i = 0; i < 64; i++)
248 nvkm_debug(subdev, "Train: %08x", result[i]);
249 gt215_link_train_calc(result, train);
250
251 nvkm_debug(subdev, "Train: %08x %08x %08x", train->r_100720,
252 train->r_1111e0, train->r_111400);
253
254 kfree(result);
255
256 train->state = NVA3_TRAIN_DONE;
257
258 return ret;
259
260 out:
261 if(ret == -EBUSY)
262 f = NULL;
263
264 train->state = NVA3_TRAIN_UNSUPPORTED;
265
266 gt215_clk_post(clk, f);
267 kfree(result);
268 return ret;
269 }
270
271 static int
gt215_link_train_init(struct gt215_ram * ram)272 gt215_link_train_init(struct gt215_ram *ram)
273 {
274 static const u32 pattern[16] = {
275 0xaaaaaaaa, 0xcccccccc, 0xdddddddd, 0xeeeeeeee,
276 0x00000000, 0x11111111, 0x44444444, 0xdddddddd,
277 0x33333333, 0x55555555, 0x77777777, 0x66666666,
278 0x99999999, 0x88888888, 0xeeeeeeee, 0xbbbbbbbb,
279 };
280 struct gt215_ltrain *train = &ram->ltrain;
281 struct nvkm_device *device = ram->base.fb->subdev.device;
282 struct nvkm_bios *bios = device->bios;
283 struct nvbios_M0205E M0205E;
284 u8 ver, hdr, cnt, len;
285 u32 r001700;
286 u64 addr;
287 int ret, i = 0;
288
289 train->state = NVA3_TRAIN_UNSUPPORTED;
290
291 /* We support type "5"
292 * XXX: training pattern table appears to be unused for this routine */
293 if (!nvbios_M0205Ep(bios, i, &ver, &hdr, &cnt, &len, &M0205E))
294 return -ENOENT;
295
296 if (M0205E.type != 5)
297 return 0;
298
299 train->state = NVA3_TRAIN_ONCE;
300
301 ret = nvkm_ram_get(device, NVKM_RAM_MM_NORMAL, 0x01, 16, 0x8000,
302 true, true, &ram->ltrain.memory);
303 if (ret)
304 return ret;
305
306 addr = nvkm_memory_addr(ram->ltrain.memory);
307
308 nvkm_wr32(device, 0x100538, 0x10000000 | (addr >> 16));
309 nvkm_wr32(device, 0x1005a8, 0x0000ffff);
310 nvkm_mask(device, 0x10f800, 0x00000001, 0x00000001);
311
312 for (i = 0; i < 0x30; i++) {
313 nvkm_wr32(device, 0x10f8c0, (i << 8) | i);
314 nvkm_wr32(device, 0x10f900, pattern[i % 16]);
315 }
316
317 for (i = 0; i < 0x30; i++) {
318 nvkm_wr32(device, 0x10f8e0, (i << 8) | i);
319 nvkm_wr32(device, 0x10f920, pattern[i % 16]);
320 }
321
322 /* And upload the pattern */
323 r001700 = nvkm_rd32(device, 0x1700);
324 nvkm_wr32(device, 0x1700, addr >> 16);
325 for (i = 0; i < 16; i++)
326 nvkm_wr32(device, 0x700000 + (i << 2), pattern[i]);
327 for (i = 0; i < 16; i++)
328 nvkm_wr32(device, 0x700100 + (i << 2), pattern[i]);
329 nvkm_wr32(device, 0x1700, r001700);
330
331 train->r_100720 = nvkm_rd32(device, 0x100720);
332 train->r_1111e0 = nvkm_rd32(device, 0x1111e0);
333 train->r_111400 = nvkm_rd32(device, 0x111400);
334 return 0;
335 }
336
337 static void
gt215_link_train_fini(struct gt215_ram * ram)338 gt215_link_train_fini(struct gt215_ram *ram)
339 {
340 nvkm_memory_unref(&ram->ltrain.memory);
341 }
342
343 /*
344 * RAM reclocking
345 */
346 #define T(t) cfg->timing_10_##t
347 static int
gt215_ram_timing_calc(struct gt215_ram * ram,u32 * timing)348 gt215_ram_timing_calc(struct gt215_ram *ram, u32 *timing)
349 {
350 struct nvbios_ramcfg *cfg = &ram->base.target.bios;
351 struct nvkm_subdev *subdev = &ram->base.fb->subdev;
352 struct nvkm_device *device = subdev->device;
353 int tUNK_base, tUNK_40_0, prevCL;
354 u32 cur2, cur3, cur7, cur8;
355
356 cur2 = nvkm_rd32(device, 0x100228);
357 cur3 = nvkm_rd32(device, 0x10022c);
358 cur7 = nvkm_rd32(device, 0x10023c);
359 cur8 = nvkm_rd32(device, 0x100240);
360
361
362 switch ((!T(CWL)) * ram->base.type) {
363 case NVKM_RAM_TYPE_DDR2:
364 T(CWL) = T(CL) - 1;
365 break;
366 case NVKM_RAM_TYPE_GDDR3:
367 T(CWL) = ((cur2 & 0xff000000) >> 24) + 1;
368 break;
369 }
370
371 prevCL = (cur3 & 0x000000ff) + 1;
372 tUNK_base = ((cur7 & 0x00ff0000) >> 16) - prevCL;
373
374 timing[0] = (T(RP) << 24 | T(RAS) << 16 | T(RFC) << 8 | T(RC));
375 timing[1] = (T(WR) + 1 + T(CWL)) << 24 |
376 max_t(u8,T(18), 1) << 16 |
377 (T(WTR) + 1 + T(CWL)) << 8 |
378 (5 + T(CL) - T(CWL));
379 timing[2] = (T(CWL) - 1) << 24 |
380 (T(RRD) << 16) |
381 (T(RCDWR) << 8) |
382 T(RCDRD);
383 timing[3] = (cur3 & 0x00ff0000) |
384 (0x30 + T(CL)) << 24 |
385 (0xb + T(CL)) << 8 |
386 (T(CL) - 1);
387 timing[4] = T(20) << 24 |
388 T(21) << 16 |
389 T(13) << 8 |
390 T(13);
391 timing[5] = T(RFC) << 24 |
392 max_t(u8,T(RCDRD), T(RCDWR)) << 16 |
393 max_t(u8, (T(CWL) + 6), (T(CL) + 2)) << 8 |
394 T(RP);
395 timing[6] = (0x5a + T(CL)) << 16 |
396 max_t(u8, 1, (6 - T(CL) + T(CWL))) << 8 |
397 (0x50 + T(CL) - T(CWL));
398 timing[7] = (cur7 & 0xff000000) |
399 ((tUNK_base + T(CL)) << 16) |
400 0x202;
401 timing[8] = cur8 & 0xffffff00;
402
403 switch (ram->base.type) {
404 case NVKM_RAM_TYPE_DDR2:
405 case NVKM_RAM_TYPE_GDDR3:
406 tUNK_40_0 = prevCL - (cur8 & 0xff);
407 if (tUNK_40_0 > 0)
408 timing[8] |= T(CL);
409 break;
410 default:
411 break;
412 }
413
414 nvkm_debug(subdev, "Entry: 220: %08x %08x %08x %08x\n",
415 timing[0], timing[1], timing[2], timing[3]);
416 nvkm_debug(subdev, " 230: %08x %08x %08x %08x\n",
417 timing[4], timing[5], timing[6], timing[7]);
418 nvkm_debug(subdev, " 240: %08x\n", timing[8]);
419 return 0;
420 }
421 #undef T
422
423 static void
nvkm_sddr2_dll_reset(struct gt215_ramfuc * fuc)424 nvkm_sddr2_dll_reset(struct gt215_ramfuc *fuc)
425 {
426 ram_mask(fuc, mr[0], 0x100, 0x100);
427 ram_nsec(fuc, 1000);
428 ram_mask(fuc, mr[0], 0x100, 0x000);
429 ram_nsec(fuc, 1000);
430 }
431
432 static void
nvkm_sddr3_dll_disable(struct gt215_ramfuc * fuc,u32 * mr)433 nvkm_sddr3_dll_disable(struct gt215_ramfuc *fuc, u32 *mr)
434 {
435 u32 mr1_old = ram_rd32(fuc, mr[1]);
436
437 if (!(mr1_old & 0x1)) {
438 ram_wr32(fuc, 0x1002d4, 0x00000001);
439 ram_wr32(fuc, mr[1], mr[1]);
440 ram_nsec(fuc, 1000);
441 }
442 }
443
444 static void
nvkm_gddr3_dll_disable(struct gt215_ramfuc * fuc,u32 * mr)445 nvkm_gddr3_dll_disable(struct gt215_ramfuc *fuc, u32 *mr)
446 {
447 u32 mr1_old = ram_rd32(fuc, mr[1]);
448
449 if (!(mr1_old & 0x40)) {
450 ram_wr32(fuc, mr[1], mr[1]);
451 ram_nsec(fuc, 1000);
452 }
453 }
454
455 static void
gt215_ram_lock_pll(struct gt215_ramfuc * fuc,struct gt215_clk_info * mclk)456 gt215_ram_lock_pll(struct gt215_ramfuc *fuc, struct gt215_clk_info *mclk)
457 {
458 ram_wr32(fuc, 0x004004, mclk->pll);
459 ram_mask(fuc, 0x004000, 0x00000001, 0x00000001);
460 ram_mask(fuc, 0x004000, 0x00000010, 0x00000000);
461 ram_wait(fuc, 0x004000, 0x00020000, 0x00020000, 64000);
462 ram_mask(fuc, 0x004000, 0x00000010, 0x00000010);
463 }
464
465 static void
gt215_ram_gpio(struct gt215_ramfuc * fuc,u8 tag,u32 val)466 gt215_ram_gpio(struct gt215_ramfuc *fuc, u8 tag, u32 val)
467 {
468 struct nvkm_gpio *gpio = fuc->base.fb->subdev.device->gpio;
469 struct dcb_gpio_func func;
470 u32 reg, sh, gpio_val;
471 int ret;
472
473 if (nvkm_gpio_get(gpio, 0, tag, DCB_GPIO_UNUSED) != val) {
474 ret = nvkm_gpio_find(gpio, 0, tag, DCB_GPIO_UNUSED, &func);
475 if (ret)
476 return;
477
478 reg = func.line >> 3;
479 sh = (func.line & 0x7) << 2;
480 gpio_val = ram_rd32(fuc, gpio[reg]);
481 if (gpio_val & (8 << sh))
482 val = !val;
483 if (!(func.log[1] & 1))
484 val = !val;
485
486 ram_mask(fuc, gpio[reg], (0x3 << sh), ((val | 0x2) << sh));
487 ram_nsec(fuc, 20000);
488 }
489 }
490
491 static int
gt215_ram_calc(struct nvkm_ram * base,u32 freq)492 gt215_ram_calc(struct nvkm_ram *base, u32 freq)
493 {
494 struct gt215_ram *ram = gt215_ram(base);
495 struct gt215_ramfuc *fuc = &ram->fuc;
496 struct gt215_ltrain *train = &ram->ltrain;
497 struct nvkm_subdev *subdev = &ram->base.fb->subdev;
498 struct nvkm_device *device = subdev->device;
499 struct nvkm_bios *bios = device->bios;
500 struct gt215_clk_info mclk;
501 struct nvkm_gpio *gpio = device->gpio;
502 struct nvkm_ram_data *next;
503 u8 ver, hdr, cnt, len, strap;
504 u32 data;
505 u32 r004018, r100760, r100da0, r111100, ctrl;
506 u32 unk714, unk718, unk71c;
507 int ret, i;
508 u32 timing[9];
509 bool pll2pll;
510
511 next = &ram->base.target;
512 next->freq = freq;
513 ram->base.next = next;
514
515 if (ram->ltrain.state == NVA3_TRAIN_ONCE)
516 gt215_link_train(ram);
517
518 /* lookup memory config data relevant to the target frequency */
519 data = nvbios_rammapEm(bios, freq / 1000, &ver, &hdr, &cnt, &len,
520 &next->bios);
521 if (!data || ver != 0x10 || hdr < 0x05) {
522 nvkm_error(subdev, "invalid/missing rammap entry\n");
523 return -EINVAL;
524 }
525
526 /* locate specific data set for the attached memory */
527 strap = nvbios_ramcfg_index(subdev);
528 if (strap >= cnt) {
529 nvkm_error(subdev, "invalid ramcfg strap\n");
530 return -EINVAL;
531 }
532
533 data = nvbios_rammapSp(bios, data, ver, hdr, cnt, len, strap,
534 &ver, &hdr, &next->bios);
535 if (!data || ver != 0x10 || hdr < 0x09) {
536 nvkm_error(subdev, "invalid/missing ramcfg entry\n");
537 return -EINVAL;
538 }
539
540 /* lookup memory timings, if bios says they're present */
541 if (next->bios.ramcfg_timing != 0xff) {
542 data = nvbios_timingEp(bios, next->bios.ramcfg_timing,
543 &ver, &hdr, &cnt, &len,
544 &next->bios);
545 if (!data || ver != 0x10 || hdr < 0x17) {
546 nvkm_error(subdev, "invalid/missing timing entry\n");
547 return -EINVAL;
548 }
549 }
550
551 ret = gt215_pll_info(device->clk, 0x12, 0x4000, freq, &mclk);
552 if (ret < 0) {
553 nvkm_error(subdev, "failed mclk calculation\n");
554 return ret;
555 }
556
557 gt215_ram_timing_calc(ram, timing);
558
559 ret = ram_init(fuc, ram->base.fb);
560 if (ret)
561 return ret;
562
563 /* Determine ram-specific MR values */
564 ram->base.mr[0] = ram_rd32(fuc, mr[0]);
565 ram->base.mr[1] = ram_rd32(fuc, mr[1]);
566 ram->base.mr[2] = ram_rd32(fuc, mr[2]);
567
568 switch (ram->base.type) {
569 case NVKM_RAM_TYPE_DDR2:
570 ret = nvkm_sddr2_calc(&ram->base);
571 break;
572 case NVKM_RAM_TYPE_DDR3:
573 ret = nvkm_sddr3_calc(&ram->base);
574 break;
575 case NVKM_RAM_TYPE_GDDR3:
576 ret = nvkm_gddr3_calc(&ram->base);
577 break;
578 default:
579 ret = -ENOSYS;
580 break;
581 }
582
583 if (ret)
584 return ret;
585
586 /* XXX: 750MHz seems rather arbitrary */
587 if (freq <= 750000) {
588 r004018 = 0x10000000;
589 r100760 = 0x22222222;
590 r100da0 = 0x00000010;
591 } else {
592 r004018 = 0x00000000;
593 r100760 = 0x00000000;
594 r100da0 = 0x00000000;
595 }
596
597 if (!next->bios.ramcfg_DLLoff)
598 r004018 |= 0x00004000;
599
600 /* pll2pll requires to switch to a safe clock first */
601 ctrl = ram_rd32(fuc, 0x004000);
602 pll2pll = (!(ctrl & 0x00000008)) && mclk.pll;
603
604 /* Pre, NVIDIA does this outside the script */
605 if (next->bios.ramcfg_10_02_10) {
606 ram_mask(fuc, 0x111104, 0x00000600, 0x00000000);
607 } else {
608 ram_mask(fuc, 0x111100, 0x40000000, 0x40000000);
609 ram_mask(fuc, 0x111104, 0x00000180, 0x00000000);
610 }
611 /* Always disable this bit during reclock */
612 ram_mask(fuc, 0x100200, 0x00000800, 0x00000000);
613
614 /* If switching from non-pll to pll, lock before disabling FB */
615 if (mclk.pll && !pll2pll) {
616 ram_mask(fuc, 0x004128, 0x003f3141, mclk.clk | 0x00000101);
617 gt215_ram_lock_pll(fuc, &mclk);
618 }
619
620 /* Start with disabling some CRTCs and PFIFO? */
621 ram_wait_vblank(fuc);
622 ram_wr32(fuc, 0x611200, 0x3300);
623 ram_mask(fuc, 0x002504, 0x1, 0x1);
624 ram_nsec(fuc, 10000);
625 ram_wait(fuc, 0x002504, 0x10, 0x10, 20000); /* XXX: or longer? */
626 ram_block(fuc);
627 ram_nsec(fuc, 2000);
628
629 if (!next->bios.ramcfg_10_02_10) {
630 if (ram->base.type == NVKM_RAM_TYPE_GDDR3)
631 ram_mask(fuc, 0x111100, 0x04020000, 0x00020000);
632 else
633 ram_mask(fuc, 0x111100, 0x04020000, 0x04020000);
634 }
635
636 /* If we're disabling the DLL, do it now */
637 switch (next->bios.ramcfg_DLLoff * ram->base.type) {
638 case NVKM_RAM_TYPE_DDR3:
639 nvkm_sddr3_dll_disable(fuc, ram->base.mr);
640 break;
641 case NVKM_RAM_TYPE_GDDR3:
642 nvkm_gddr3_dll_disable(fuc, ram->base.mr);
643 break;
644 }
645
646 if (next->bios.timing_10_ODT)
647 gt215_ram_gpio(fuc, 0x2e, 1);
648
649 /* Brace RAM for impact */
650 ram_wr32(fuc, 0x1002d4, 0x00000001);
651 ram_wr32(fuc, 0x1002d0, 0x00000001);
652 ram_wr32(fuc, 0x1002d0, 0x00000001);
653 ram_wr32(fuc, 0x100210, 0x00000000);
654 ram_wr32(fuc, 0x1002dc, 0x00000001);
655 ram_nsec(fuc, 2000);
656
657 if (device->chipset == 0xa3 && freq <= 500000)
658 ram_mask(fuc, 0x100700, 0x00000006, 0x00000006);
659
660 /* Alter FBVDD/Q, apparently must be done with PLL disabled, thus
661 * set it to bypass */
662 if (nvkm_gpio_get(gpio, 0, 0x18, DCB_GPIO_UNUSED) ==
663 next->bios.ramcfg_FBVDDQ) {
664 data = ram_rd32(fuc, 0x004000) & 0x9;
665
666 if (data == 0x1)
667 ram_mask(fuc, 0x004000, 0x8, 0x8);
668 if (data & 0x1)
669 ram_mask(fuc, 0x004000, 0x1, 0x0);
670
671 gt215_ram_gpio(fuc, 0x18, !next->bios.ramcfg_FBVDDQ);
672
673 if (data & 0x1)
674 ram_mask(fuc, 0x004000, 0x1, 0x1);
675 }
676
677 /* Fiddle with clocks */
678 /* There's 4 scenario's
679 * pll->pll: first switch to a 324MHz clock, set up new PLL, switch
680 * clk->pll: Set up new PLL, switch
681 * pll->clk: Set up clock, switch
682 * clk->clk: Overwrite ctrl and other bits, switch */
683
684 /* Switch to regular clock - 324MHz */
685 if (pll2pll) {
686 ram_mask(fuc, 0x004000, 0x00000004, 0x00000004);
687 ram_mask(fuc, 0x004168, 0x003f3141, 0x00083101);
688 ram_mask(fuc, 0x004000, 0x00000008, 0x00000008);
689 ram_mask(fuc, 0x1110e0, 0x00088000, 0x00088000);
690 ram_wr32(fuc, 0x004018, 0x00001000);
691 gt215_ram_lock_pll(fuc, &mclk);
692 }
693
694 if (mclk.pll) {
695 ram_mask(fuc, 0x004000, 0x00000105, 0x00000105);
696 ram_wr32(fuc, 0x004018, 0x00001000 | r004018);
697 ram_wr32(fuc, 0x100da0, r100da0);
698 } else {
699 ram_mask(fuc, 0x004168, 0x003f3141, mclk.clk | 0x00000101);
700 ram_mask(fuc, 0x004000, 0x00000108, 0x00000008);
701 ram_mask(fuc, 0x1110e0, 0x00088000, 0x00088000);
702 ram_wr32(fuc, 0x004018, 0x00009000 | r004018);
703 ram_wr32(fuc, 0x100da0, r100da0);
704 }
705 ram_nsec(fuc, 20000);
706
707 if (next->bios.rammap_10_04_08) {
708 ram_wr32(fuc, 0x1005a0, next->bios.ramcfg_10_06 << 16 |
709 next->bios.ramcfg_10_05 << 8 |
710 next->bios.ramcfg_10_05);
711 ram_wr32(fuc, 0x1005a4, next->bios.ramcfg_10_08 << 8 |
712 next->bios.ramcfg_10_07);
713 ram_wr32(fuc, 0x10f804, next->bios.ramcfg_10_09_f0 << 20 |
714 next->bios.ramcfg_10_03_0f << 16 |
715 next->bios.ramcfg_10_09_0f |
716 0x80000000);
717 ram_mask(fuc, 0x10053c, 0x00001000, 0x00000000);
718 } else {
719 if (train->state == NVA3_TRAIN_DONE) {
720 ram_wr32(fuc, 0x100080, 0x1020);
721 ram_mask(fuc, 0x111400, 0xffffffff, train->r_111400);
722 ram_mask(fuc, 0x1111e0, 0xffffffff, train->r_1111e0);
723 ram_mask(fuc, 0x100720, 0xffffffff, train->r_100720);
724 }
725 ram_mask(fuc, 0x10053c, 0x00001000, 0x00001000);
726 ram_mask(fuc, 0x10f804, 0x80000000, 0x00000000);
727 ram_mask(fuc, 0x100760, 0x22222222, r100760);
728 ram_mask(fuc, 0x1007a0, 0x22222222, r100760);
729 ram_mask(fuc, 0x1007e0, 0x22222222, r100760);
730 }
731
732 if (device->chipset == 0xa3 && freq > 500000) {
733 ram_mask(fuc, 0x100700, 0x00000006, 0x00000000);
734 }
735
736 /* Final switch */
737 if (mclk.pll) {
738 ram_mask(fuc, 0x1110e0, 0x00088000, 0x00011000);
739 ram_mask(fuc, 0x004000, 0x00000008, 0x00000000);
740 }
741
742 ram_wr32(fuc, 0x1002dc, 0x00000000);
743 ram_wr32(fuc, 0x1002d4, 0x00000001);
744 ram_wr32(fuc, 0x100210, 0x80000000);
745 ram_nsec(fuc, 2000);
746
747 /* Set RAM MR parameters and timings */
748 for (i = 2; i >= 0; i--) {
749 if (ram_rd32(fuc, mr[i]) != ram->base.mr[i]) {
750 ram_wr32(fuc, mr[i], ram->base.mr[i]);
751 ram_nsec(fuc, 1000);
752 }
753 }
754
755 ram_wr32(fuc, 0x100220[3], timing[3]);
756 ram_wr32(fuc, 0x100220[1], timing[1]);
757 ram_wr32(fuc, 0x100220[6], timing[6]);
758 ram_wr32(fuc, 0x100220[7], timing[7]);
759 ram_wr32(fuc, 0x100220[2], timing[2]);
760 ram_wr32(fuc, 0x100220[4], timing[4]);
761 ram_wr32(fuc, 0x100220[5], timing[5]);
762 ram_wr32(fuc, 0x100220[0], timing[0]);
763 ram_wr32(fuc, 0x100220[8], timing[8]);
764
765 /* Misc */
766 ram_mask(fuc, 0x100200, 0x00001000, !next->bios.ramcfg_10_02_08 << 12);
767
768 /* XXX: A lot of "chipset"/"ram type" specific stuff...? */
769 unk714 = ram_rd32(fuc, 0x100714) & ~0xf0000130;
770 unk718 = ram_rd32(fuc, 0x100718) & ~0x00000100;
771 unk71c = ram_rd32(fuc, 0x10071c) & ~0x00000100;
772 r111100 = ram_rd32(fuc, 0x111100) & ~0x3a800000;
773
774 /* NVA8 seems to skip various bits related to ramcfg_10_02_04 */
775 if (device->chipset == 0xa8) {
776 r111100 |= 0x08000000;
777 if (!next->bios.ramcfg_10_02_04)
778 unk714 |= 0x00000010;
779 } else {
780 if (next->bios.ramcfg_10_02_04) {
781 switch (ram->base.type) {
782 case NVKM_RAM_TYPE_DDR2:
783 case NVKM_RAM_TYPE_DDR3:
784 r111100 &= ~0x00000020;
785 if (next->bios.ramcfg_10_02_10)
786 r111100 |= 0x08000004;
787 else
788 r111100 |= 0x00000024;
789 break;
790 default:
791 break;
792 }
793 } else {
794 switch (ram->base.type) {
795 case NVKM_RAM_TYPE_DDR2:
796 case NVKM_RAM_TYPE_DDR3:
797 r111100 &= ~0x00000024;
798 r111100 |= 0x12800000;
799
800 if (next->bios.ramcfg_10_02_10)
801 r111100 |= 0x08000000;
802 unk714 |= 0x00000010;
803 break;
804 case NVKM_RAM_TYPE_GDDR3:
805 r111100 |= 0x30000000;
806 unk714 |= 0x00000020;
807 break;
808 default:
809 break;
810 }
811 }
812 }
813
814 unk714 |= (next->bios.ramcfg_10_04_01) << 8;
815
816 if (next->bios.ramcfg_10_02_20)
817 unk714 |= 0xf0000000;
818 if (next->bios.ramcfg_10_02_02)
819 unk718 |= 0x00000100;
820 if (next->bios.ramcfg_10_02_01)
821 unk71c |= 0x00000100;
822 if (next->bios.timing_10_24 != 0xff) {
823 unk718 &= ~0xf0000000;
824 unk718 |= next->bios.timing_10_24 << 28;
825 }
826 if (next->bios.ramcfg_10_02_10)
827 r111100 &= ~0x04020000;
828
829 ram_mask(fuc, 0x100714, 0xffffffff, unk714);
830 ram_mask(fuc, 0x10071c, 0xffffffff, unk71c);
831 ram_mask(fuc, 0x100718, 0xffffffff, unk718);
832 ram_mask(fuc, 0x111100, 0xffffffff, r111100);
833
834 if (!next->bios.timing_10_ODT)
835 gt215_ram_gpio(fuc, 0x2e, 0);
836
837 /* Reset DLL */
838 if (!next->bios.ramcfg_DLLoff)
839 nvkm_sddr2_dll_reset(fuc);
840
841 if (ram->base.type == NVKM_RAM_TYPE_GDDR3) {
842 ram_nsec(fuc, 31000);
843 } else {
844 ram_nsec(fuc, 14000);
845 }
846
847 if (ram->base.type == NVKM_RAM_TYPE_DDR3) {
848 ram_wr32(fuc, 0x100264, 0x1);
849 ram_nsec(fuc, 2000);
850 }
851
852 ram_nuke(fuc, 0x100700);
853 ram_mask(fuc, 0x100700, 0x01000000, 0x01000000);
854 ram_mask(fuc, 0x100700, 0x01000000, 0x00000000);
855
856 /* Re-enable FB */
857 ram_unblock(fuc);
858 ram_wr32(fuc, 0x611200, 0x3330);
859
860 /* Post fiddlings */
861 if (next->bios.rammap_10_04_02)
862 ram_mask(fuc, 0x100200, 0x00000800, 0x00000800);
863 if (next->bios.ramcfg_10_02_10) {
864 ram_mask(fuc, 0x111104, 0x00000180, 0x00000180);
865 ram_mask(fuc, 0x111100, 0x40000000, 0x00000000);
866 } else {
867 ram_mask(fuc, 0x111104, 0x00000600, 0x00000600);
868 }
869
870 if (mclk.pll) {
871 ram_mask(fuc, 0x004168, 0x00000001, 0x00000000);
872 ram_mask(fuc, 0x004168, 0x00000100, 0x00000000);
873 } else {
874 ram_mask(fuc, 0x004000, 0x00000001, 0x00000000);
875 ram_mask(fuc, 0x004128, 0x00000001, 0x00000000);
876 ram_mask(fuc, 0x004128, 0x00000100, 0x00000000);
877 }
878
879 return 0;
880 }
881
882 static int
gt215_ram_prog(struct nvkm_ram * base)883 gt215_ram_prog(struct nvkm_ram *base)
884 {
885 struct gt215_ram *ram = gt215_ram(base);
886 struct gt215_ramfuc *fuc = &ram->fuc;
887 struct nvkm_device *device = ram->base.fb->subdev.device;
888 bool exec = nvkm_boolopt(device->cfgopt, "NvMemExec", true);
889
890 if (exec) {
891 nvkm_mask(device, 0x001534, 0x2, 0x2);
892
893 ram_exec(fuc, true);
894
895 /* Post-processing, avoids flicker */
896 nvkm_mask(device, 0x002504, 0x1, 0x0);
897 nvkm_mask(device, 0x001534, 0x2, 0x0);
898
899 nvkm_mask(device, 0x616308, 0x10, 0x10);
900 nvkm_mask(device, 0x616b08, 0x10, 0x10);
901 } else {
902 ram_exec(fuc, false);
903 }
904 return 0;
905 }
906
907 static void
gt215_ram_tidy(struct nvkm_ram * base)908 gt215_ram_tidy(struct nvkm_ram *base)
909 {
910 struct gt215_ram *ram = gt215_ram(base);
911 ram_exec(&ram->fuc, false);
912 }
913
914 static int
gt215_ram_init(struct nvkm_ram * base)915 gt215_ram_init(struct nvkm_ram *base)
916 {
917 struct gt215_ram *ram = gt215_ram(base);
918 gt215_link_train_init(ram);
919 return 0;
920 }
921
922 static void *
gt215_ram_dtor(struct nvkm_ram * base)923 gt215_ram_dtor(struct nvkm_ram *base)
924 {
925 struct gt215_ram *ram = gt215_ram(base);
926 gt215_link_train_fini(ram);
927 return ram;
928 }
929
930 static const struct nvkm_ram_func
931 gt215_ram_func = {
932 .dtor = gt215_ram_dtor,
933 .init = gt215_ram_init,
934 .calc = gt215_ram_calc,
935 .prog = gt215_ram_prog,
936 .tidy = gt215_ram_tidy,
937 };
938
939 int
gt215_ram_new(struct nvkm_fb * fb,struct nvkm_ram ** pram)940 gt215_ram_new(struct nvkm_fb *fb, struct nvkm_ram **pram)
941 {
942 struct gt215_ram *ram;
943 int ret, i;
944
945 if (!(ram = kzalloc(sizeof(*ram), GFP_KERNEL)))
946 return -ENOMEM;
947 *pram = &ram->base;
948
949 ret = nv50_ram_ctor(>215_ram_func, fb, &ram->base);
950 if (ret)
951 return ret;
952
953 ram->fuc.r_0x001610 = ramfuc_reg(0x001610);
954 ram->fuc.r_0x001700 = ramfuc_reg(0x001700);
955 ram->fuc.r_0x002504 = ramfuc_reg(0x002504);
956 ram->fuc.r_0x004000 = ramfuc_reg(0x004000);
957 ram->fuc.r_0x004004 = ramfuc_reg(0x004004);
958 ram->fuc.r_0x004018 = ramfuc_reg(0x004018);
959 ram->fuc.r_0x004128 = ramfuc_reg(0x004128);
960 ram->fuc.r_0x004168 = ramfuc_reg(0x004168);
961 ram->fuc.r_0x100080 = ramfuc_reg(0x100080);
962 ram->fuc.r_0x100200 = ramfuc_reg(0x100200);
963 ram->fuc.r_0x100210 = ramfuc_reg(0x100210);
964 for (i = 0; i < 9; i++)
965 ram->fuc.r_0x100220[i] = ramfuc_reg(0x100220 + (i * 4));
966 ram->fuc.r_0x100264 = ramfuc_reg(0x100264);
967 ram->fuc.r_0x1002d0 = ramfuc_reg(0x1002d0);
968 ram->fuc.r_0x1002d4 = ramfuc_reg(0x1002d4);
969 ram->fuc.r_0x1002dc = ramfuc_reg(0x1002dc);
970 ram->fuc.r_0x10053c = ramfuc_reg(0x10053c);
971 ram->fuc.r_0x1005a0 = ramfuc_reg(0x1005a0);
972 ram->fuc.r_0x1005a4 = ramfuc_reg(0x1005a4);
973 ram->fuc.r_0x100700 = ramfuc_reg(0x100700);
974 ram->fuc.r_0x100714 = ramfuc_reg(0x100714);
975 ram->fuc.r_0x100718 = ramfuc_reg(0x100718);
976 ram->fuc.r_0x10071c = ramfuc_reg(0x10071c);
977 ram->fuc.r_0x100720 = ramfuc_reg(0x100720);
978 ram->fuc.r_0x100760 = ramfuc_stride(0x100760, 4, ram->base.part_mask);
979 ram->fuc.r_0x1007a0 = ramfuc_stride(0x1007a0, 4, ram->base.part_mask);
980 ram->fuc.r_0x1007e0 = ramfuc_stride(0x1007e0, 4, ram->base.part_mask);
981 ram->fuc.r_0x100da0 = ramfuc_stride(0x100da0, 4, ram->base.part_mask);
982 ram->fuc.r_0x10f804 = ramfuc_reg(0x10f804);
983 ram->fuc.r_0x1110e0 = ramfuc_stride(0x1110e0, 4, ram->base.part_mask);
984 ram->fuc.r_0x111100 = ramfuc_reg(0x111100);
985 ram->fuc.r_0x111104 = ramfuc_reg(0x111104);
986 ram->fuc.r_0x1111e0 = ramfuc_reg(0x1111e0);
987 ram->fuc.r_0x111400 = ramfuc_reg(0x111400);
988 ram->fuc.r_0x611200 = ramfuc_reg(0x611200);
989
990 if (ram->base.ranks > 1) {
991 ram->fuc.r_mr[0] = ramfuc_reg2(0x1002c0, 0x1002c8);
992 ram->fuc.r_mr[1] = ramfuc_reg2(0x1002c4, 0x1002cc);
993 ram->fuc.r_mr[2] = ramfuc_reg2(0x1002e0, 0x1002e8);
994 ram->fuc.r_mr[3] = ramfuc_reg2(0x1002e4, 0x1002ec);
995 } else {
996 ram->fuc.r_mr[0] = ramfuc_reg(0x1002c0);
997 ram->fuc.r_mr[1] = ramfuc_reg(0x1002c4);
998 ram->fuc.r_mr[2] = ramfuc_reg(0x1002e0);
999 ram->fuc.r_mr[3] = ramfuc_reg(0x1002e4);
1000 }
1001 ram->fuc.r_gpio[0] = ramfuc_reg(0x00e104);
1002 ram->fuc.r_gpio[1] = ramfuc_reg(0x00e108);
1003 ram->fuc.r_gpio[2] = ramfuc_reg(0x00e120);
1004 ram->fuc.r_gpio[3] = ramfuc_reg(0x00e124);
1005
1006 return 0;
1007 }
1008