44 	struct nvkm_device *device = fifo->engine.subdev.device;  in nv04_chan_stop()  local
45 	struct nvkm_memory *fctx = device->imem->ramfc;  in nv04_chan_stop()
53 	nvkm_wr32(device, NV03_PFIFO_CACHES, 0);  in nv04_chan_stop()
56 	chid = nvkm_rd32(device, NV03_PFIFO_CACHE1_PUSH1) & fifo->chid->mask;  in nv04_chan_stop()
58 		nvkm_mask(device, NV04_PFIFO_CACHE1_DMA_PUSH, 0x00000001, 0);  in nv04_chan_stop()
59 		nvkm_wr32(device, NV03_PFIFO_CACHE1_PUSH0, 0);  in nv04_chan_stop()
60 		nvkm_mask(device, NV04_PFIFO_CACHE1_PULL0, 0x00000001, 0);  in nv04_chan_stop()
67 			u32 rv = (nvkm_rd32(device, c->regp) &  rm) >> c->regs;  in nv04_chan_stop()
75 			nvkm_wr32(device, c->regp, 0x00000000);  in nv04_chan_stop()
78 		nvkm_wr32(device, NV03_PFIFO_CACHE1_GET, 0);  in nv04_chan_stop()
79 		nvkm_wr32(device, NV03_PFIFO_CACHE1_PUT, 0);  in nv04_chan_stop()
80 		nvkm_wr32(device, NV03_PFIFO_CACHE1_PUSH1, fifo->chid->mask);  in nv04_chan_stop()
81 		nvkm_wr32(device, NV03_PFIFO_CACHE1_PUSH0, 1);  in nv04_chan_stop()
82 		nvkm_wr32(device, NV04_PFIFO_CACHE1_PULL0, 1);  in nv04_chan_stop()
86 	nvkm_mask(device, NV04_PFIFO_MODE, BIT(chan->id), 0);  in nv04_chan_stop()
87 	nvkm_wr32(device, NV03_PFIFO_CACHES, 1);  in nv04_chan_stop()
98 	nvkm_mask(fifo->engine.subdev.device, NV04_PFIFO_MODE, BIT(chan->id), BIT(chan->id));  in nv04_chan_start()
105 	struct nvkm_memory *ramfc = chan->cgrp->runl->fifo->engine.subdev.device->imem->ramfc;  in nv04_chan_ramfc_clear()
118 	struct nvkm_memory *ramfc = chan->cgrp->runl->fifo->engine.subdev.device->imem->ramfc;  in nv04_chan_ramfc_write()
184 	struct nvkm_instmem *imem = fifo->engine.subdev.device->imem;  in nv04_eobj_ramht_del()
195 	struct nvkm_instmem *imem = fifo->engine.subdev.device->imem;  in nv04_eobj_ramht_add()
215 	struct nvkm_device *device = fifo->engine.subdev.device;  in nv04_fifo_pause()  local
221 	nvkm_wr32(device, NV03_PFIFO_CACHES, 0x00000000);  in nv04_fifo_pause()
222 	nvkm_mask(device, NV04_PFIFO_CACHE1_PULL0, 0x00000001, 0x00000000);  in nv04_fifo_pause()
233 	nvkm_msec(device, 2000,  in nv04_fifo_pause()
234 		u32 tmp = nvkm_rd32(device, NV04_PFIFO_CACHE1_PULL0);  in nv04_fifo_pause()
239 	if (nvkm_rd32(device, NV04_PFIFO_CACHE1_PULL0) &  in nv04_fifo_pause()
241 		nvkm_wr32(device, NV03_PFIFO_INTR_0, NV_PFIFO_INTR_CACHE_ERROR);  in nv04_fifo_pause()
243 	nvkm_wr32(device, NV04_PFIFO_CACHE1_HASH, 0x00000000);  in nv04_fifo_pause()
250 	struct nvkm_device *device = fifo->engine.subdev.device;  in nv04_fifo_start()  local
253 	nvkm_mask(device, NV04_PFIFO_CACHE1_PULL0, 0x00000001, 0x00000001);  in nv04_fifo_start()
254 	nvkm_wr32(device, NV03_PFIFO_CACHES, 0x00000001);  in nv04_fifo_start()
274 nv04_fifo_swmthd(struct nvkm_device *device, u32 chid, u32 addr, u32 data)  in nv04_fifo_swmthd()  argument
276 	struct nvkm_sw *sw = device->sw;  in nv04_fifo_swmthd()
280 	u32 engine = nvkm_rd32(device, 0x003280);  in nv04_fifo_swmthd()
285 		nvkm_wr32(device, 0x003280, (engine &= ~mask));  in nv04_fifo_swmthd()
288 		data = nvkm_rd32(device, 0x003258) & 0x0000ffff;  in nv04_fifo_swmthd()
306 	struct nvkm_device *device = subdev->device;  in nv04_fifo_intr_cache_error()  local
309 	u32 pull0 = nvkm_rd32(device, 0x003250);  in nv04_fifo_intr_cache_error()
320 	if (device->card_type < NV_40) {  in nv04_fifo_intr_cache_error()
321 		mthd = nvkm_rd32(device, NV04_PFIFO_CACHE1_METHOD(ptr));  in nv04_fifo_intr_cache_error()
322 		data = nvkm_rd32(device, NV04_PFIFO_CACHE1_DATA(ptr));  in nv04_fifo_intr_cache_error()
324 		mthd = nvkm_rd32(device, NV40_PFIFO_CACHE1_METHOD(ptr));  in nv04_fifo_intr_cache_error()
325 		data = nvkm_rd32(device, NV40_PFIFO_CACHE1_DATA(ptr));  in nv04_fifo_intr_cache_error()
329 	    !nv04_fifo_swmthd(device, chid, mthd, data)) {  in nv04_fifo_intr_cache_error()
338 	nvkm_wr32(device, NV04_PFIFO_CACHE1_DMA_PUSH, 0);  in nv04_fifo_intr_cache_error()
339 	nvkm_wr32(device, NV03_PFIFO_INTR_0, NV_PFIFO_INTR_CACHE_ERROR);  in nv04_fifo_intr_cache_error()
341 	nvkm_wr32(device, NV03_PFIFO_CACHE1_PUSH0,  in nv04_fifo_intr_cache_error()
342 		nvkm_rd32(device, NV03_PFIFO_CACHE1_PUSH0) & ~1);  in nv04_fifo_intr_cache_error()
343 	nvkm_wr32(device, NV03_PFIFO_CACHE1_GET, get + 4);  in nv04_fifo_intr_cache_error()
344 	nvkm_wr32(device, NV03_PFIFO_CACHE1_PUSH0,  in nv04_fifo_intr_cache_error()
345 		nvkm_rd32(device, NV03_PFIFO_CACHE1_PUSH0) | 1);  in nv04_fifo_intr_cache_error()
346 	nvkm_wr32(device, NV04_PFIFO_CACHE1_HASH, 0);  in nv04_fifo_intr_cache_error()
348 	nvkm_wr32(device, NV04_PFIFO_CACHE1_DMA_PUSH,  in nv04_fifo_intr_cache_error()
349 		nvkm_rd32(device, NV04_PFIFO_CACHE1_DMA_PUSH) | 1);  in nv04_fifo_intr_cache_error()
350 	nvkm_wr32(device, NV04_PFIFO_CACHE1_PULL0, 1);  in nv04_fifo_intr_cache_error()
357 	struct nvkm_device *device = subdev->device;  in nv04_fifo_intr_dma_pusher()  local
358 	u32 dma_get = nvkm_rd32(device, 0x003244);  in nv04_fifo_intr_dma_pusher()
359 	u32 dma_put = nvkm_rd32(device, 0x003240);  in nv04_fifo_intr_dma_pusher()
360 	u32 push = nvkm_rd32(device, 0x003220);  in nv04_fifo_intr_dma_pusher()
361 	u32 state = nvkm_rd32(device, 0x003228);  in nv04_fifo_intr_dma_pusher()
368 	if (device->card_type == NV_50) {  in nv04_fifo_intr_dma_pusher()
369 		u32 ho_get = nvkm_rd32(device, 0x003328);  in nv04_fifo_intr_dma_pusher()
370 		u32 ho_put = nvkm_rd32(device, 0x003320);  in nv04_fifo_intr_dma_pusher()
371 		u32 ib_get = nvkm_rd32(device, 0x003334);  in nv04_fifo_intr_dma_pusher()
372 		u32 ib_put = nvkm_rd32(device, 0x003330);  in nv04_fifo_intr_dma_pusher()
382 		nvkm_wr32(device, 0x003364, 0x00000000);  in nv04_fifo_intr_dma_pusher()
384 			nvkm_wr32(device, 0x003244, dma_put);  in nv04_fifo_intr_dma_pusher()
385 			nvkm_wr32(device, 0x003328, ho_put);  in nv04_fifo_intr_dma_pusher()
388 			nvkm_wr32(device, 0x003334, ib_put);  in nv04_fifo_intr_dma_pusher()
396 			nvkm_wr32(device, 0x003244, dma_put);  in nv04_fifo_intr_dma_pusher()
400 	nvkm_wr32(device, 0x003228, 0x00000000);  in nv04_fifo_intr_dma_pusher()
401 	nvkm_wr32(device, 0x003220, 0x00000001);  in nv04_fifo_intr_dma_pusher()
402 	nvkm_wr32(device, 0x002100, NV_PFIFO_INTR_DMA_PUSHER);  in nv04_fifo_intr_dma_pusher()
410 	struct nvkm_device *device = subdev->device;  in nv04_fifo_intr()  local
411 	u32 mask = nvkm_rd32(device, NV03_PFIFO_INTR_EN_0);  in nv04_fifo_intr()
412 	u32 stat = nvkm_rd32(device, NV03_PFIFO_INTR_0) & mask;  in nv04_fifo_intr()
415 	reassign = nvkm_rd32(device, NV03_PFIFO_CACHES) & 1;  in nv04_fifo_intr()
416 	nvkm_wr32(device, NV03_PFIFO_CACHES, 0);  in nv04_fifo_intr()
418 	chid = nvkm_rd32(device, NV03_PFIFO_CACHE1_PUSH1) & fifo->chid->mask;  in nv04_fifo_intr()
419 	get  = nvkm_rd32(device, NV03_PFIFO_CACHE1_GET);  in nv04_fifo_intr()
433 		nvkm_wr32(device, NV03_PFIFO_INTR_0, NV_PFIFO_INTR_SEMAPHORE);  in nv04_fifo_intr()
435 		sem = nvkm_rd32(device, NV10_PFIFO_CACHE1_SEMAPHORE);  in nv04_fifo_intr()
436 		nvkm_wr32(device, NV10_PFIFO_CACHE1_SEMAPHORE, sem | 0x1);  in nv04_fifo_intr()
438 		nvkm_wr32(device, NV03_PFIFO_CACHE1_GET, get + 4);  in nv04_fifo_intr()
439 		nvkm_wr32(device, NV04_PFIFO_CACHE1_PULL0, 1);  in nv04_fifo_intr()
442 	if (device->card_type == NV_50) {  in nv04_fifo_intr()
445 			nvkm_wr32(device, 0x002100, 0x00000010);  in nv04_fifo_intr()
449 			nvkm_wr32(device, 0x002100, 0x40000000);  in nv04_fifo_intr()
457 		nvkm_mask(device, NV03_PFIFO_INTR_EN_0, stat, 0x00000000);  in nv04_fifo_intr()
458 		nvkm_wr32(device, NV03_PFIFO_INTR_0, stat);  in nv04_fifo_intr()
461 	nvkm_wr32(device, NV03_PFIFO_CACHES, reassign);  in nv04_fifo_intr()
468 	struct nvkm_device *device = fifo->engine.subdev.device;  in nv04_fifo_init()  local
469 	struct nvkm_instmem *imem = device->imem;  in nv04_fifo_init()
474 	nvkm_wr32(device, NV04_PFIFO_DELAY_0, 0x000000ff);  in nv04_fifo_init()
475 	nvkm_wr32(device, NV04_PFIFO_DMA_TIMESLICE, 0x0101ffff);  in nv04_fifo_init()
477 	nvkm_wr32(device, NV03_PFIFO_RAMHT, (0x03 << 24) /* search 128 */ |  in nv04_fifo_init()
480 	nvkm_wr32(device, NV03_PFIFO_RAMRO, nvkm_memory_addr(ramro) >> 8);  in nv04_fifo_init()
481 	nvkm_wr32(device, NV03_PFIFO_RAMFC, nvkm_memory_addr(ramfc) >> 8);  in nv04_fifo_init()
483 	nvkm_wr32(device, NV03_PFIFO_CACHE1_PUSH1, fifo->chid->mask);  in nv04_fifo_init()
485 	nvkm_wr32(device, NV03_PFIFO_INTR_0, 0xffffffff);  in nv04_fifo_init()
486 	nvkm_wr32(device, NV03_PFIFO_INTR_EN_0, 0xffffffff);  in nv04_fifo_init()
488 	nvkm_wr32(device, NV03_PFIFO_CACHE1_PUSH0, 1);  in nv04_fifo_init()
489 	nvkm_wr32(device, NV04_PFIFO_CACHE1_PULL0, 1);  in nv04_fifo_init()
490 	nvkm_wr32(device, NV03_PFIFO_CACHES, 1);  in nv04_fifo_init()
539 nv04_fifo_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst,  in nv04_fifo_new()  argument
542 	return nvkm_fifo_new_(&nv04_fifo, device, type, inst, pfifo);  in nv04_fifo_new()