1 /* SPDX-License-Identifier: MIT */ 2 #ifndef __NVKM_FIFO_PRIV_H__ 3 #define __NVKM_FIFO_PRIV_H__ 4 #define nvkm_fifo(p) container_of((p), struct nvkm_fifo, engine) 5 #include <engine/fifo.h> 6 #include <core/enum.h> 7 struct nvkm_cctx; 8 struct nvkm_cgrp; 9 struct nvkm_engn; 10 struct nvkm_memory; 11 struct nvkm_runl; 12 struct nvkm_runq; 13 struct nvkm_vctx; 14 15 struct nvkm_fifo_func { 16 int (*chid_nr)(struct nvkm_fifo *); 17 int (*chid_ctor)(struct nvkm_fifo *, int nr); 18 int (*runq_nr)(struct nvkm_fifo *); 19 int (*runl_ctor)(struct nvkm_fifo *); 20 21 void (*init)(struct nvkm_fifo *); 22 void (*init_pbdmas)(struct nvkm_fifo *, u32 mask); 23 24 irqreturn_t (*intr)(struct nvkm_inth *); 25 void (*intr_mmu_fault_unit)(struct nvkm_fifo *, int unit); 26 void (*intr_ctxsw_timeout)(struct nvkm_fifo *, u32 engm); 27 28 const struct nvkm_fifo_func_mmu_fault { 29 void (*recover)(struct nvkm_fifo *, struct nvkm_fault_data *); 30 const struct nvkm_enum *access; 31 const struct nvkm_enum *engine; 32 const struct nvkm_enum *reason; 33 const struct nvkm_enum *hubclient; 34 const struct nvkm_enum *gpcclient; 35 } *mmu_fault; 36 37 void (*pause)(struct nvkm_fifo *, unsigned long *); 38 void (*start)(struct nvkm_fifo *, unsigned long *); 39 40 int (*nonstall_ctor)(struct nvkm_fifo *); 41 const struct nvkm_event_func *nonstall; 42 43 const struct nvkm_runl_func *runl; 44 const struct nvkm_runq_func *runq; 45 const struct nvkm_engn_func *engn; 46 const struct nvkm_engn_func *engn_sw; 47 const struct nvkm_engn_func *engn_ce; 48 49 struct nvkm_fifo_func_cgrp { 50 struct nvkm_sclass user; 51 const struct nvkm_cgrp_func *func; 52 bool force; 53 } cgrp; 54 55 struct nvkm_fifo_func_chan { 56 struct nvkm_sclass user; 57 const struct nvkm_chan_func *func; 58 } chan; 59 }; 60 61 int nvkm_fifo_new_(const struct nvkm_fifo_func *, struct nvkm_device *, enum nvkm_subdev_type, int, 62 struct nvkm_fifo **); 63 64 int nv04_fifo_chid_ctor(struct nvkm_fifo *, int); 65 int nv04_fifo_runl_ctor(struct nvkm_fifo *); 66 void nv04_fifo_init(struct nvkm_fifo *); 67 irqreturn_t nv04_fifo_intr(struct nvkm_inth *); 68 void nv04_fifo_pause(struct nvkm_fifo *, unsigned long *); 69 void nv04_fifo_start(struct nvkm_fifo *, unsigned long *); 70 extern const struct nvkm_runl_func nv04_runl; 71 extern const struct nvkm_engn_func nv04_engn; 72 extern const struct nvkm_cgrp_func nv04_cgrp; 73 extern const struct nvkm_chan_func_inst nv04_chan_inst; 74 extern const struct nvkm_chan_func_userd nv04_chan_userd; 75 void nv04_chan_ramfc_clear(struct nvkm_chan *); 76 void nv04_chan_start(struct nvkm_chan *); 77 void nv04_chan_stop(struct nvkm_chan *); 78 void nv04_eobj_ramht_del(struct nvkm_chan *, int); 79 80 int nv10_fifo_chid_nr(struct nvkm_fifo *); 81 82 int nv50_fifo_chid_nr(struct nvkm_fifo *); 83 int nv50_fifo_chid_ctor(struct nvkm_fifo *, int); 84 void nv50_fifo_init(struct nvkm_fifo *); 85 extern const struct nvkm_runl_func nv50_runl; 86 int nv50_runl_update(struct nvkm_runl *); 87 int nv50_runl_wait(struct nvkm_runl *); 88 extern const struct nvkm_engn_func nv50_engn_sw; 89 extern const struct nvkm_chan_func_inst nv50_chan_inst; 90 extern const struct nvkm_chan_func_userd nv50_chan_userd; 91 void nv50_chan_unbind(struct nvkm_chan *); 92 void nv50_chan_start(struct nvkm_chan *); 93 void nv50_chan_stop(struct nvkm_chan *); 94 void nv50_chan_preempt(struct nvkm_chan *); 95 int nv50_eobj_ramht_add(struct nvkm_engn *, struct nvkm_object *, struct nvkm_chan *); 96 void nv50_eobj_ramht_del(struct nvkm_chan *, int); 97 98 extern const struct nvkm_event_func g84_fifo_nonstall; 99 extern const struct nvkm_engn_func g84_engn; 100 extern const struct nvkm_chan_func g84_chan; 101 102 int gf100_fifo_chid_ctor(struct nvkm_fifo *, int); 103 int gf100_fifo_runq_nr(struct nvkm_fifo *); 104 bool gf100_fifo_intr_pbdma(struct nvkm_fifo *); 105 void gf100_fifo_intr_mmu_fault(struct nvkm_fifo *); 106 void gf100_fifo_intr_mmu_fault_unit(struct nvkm_fifo *, int); 107 void gf100_fifo_intr_sched(struct nvkm_fifo *); 108 void gf100_fifo_intr_ctxsw_timeout(struct nvkm_fifo *, u32); 109 void gf100_fifo_mmu_fault_recover(struct nvkm_fifo *, struct nvkm_fault_data *); 110 extern const struct nvkm_enum gf100_fifo_mmu_fault_access[]; 111 extern const struct nvkm_event_func gf100_fifo_nonstall; 112 bool gf100_runl_preempt_pending(struct nvkm_runl *); 113 void gf100_runq_init(struct nvkm_runq *); 114 bool gf100_runq_intr(struct nvkm_runq *, struct nvkm_runl *); 115 void gf100_engn_mmu_fault_trigger(struct nvkm_engn *); 116 bool gf100_engn_mmu_fault_triggered(struct nvkm_engn *); 117 extern const struct nvkm_engn_func gf100_engn_sw; 118 extern const struct nvkm_chan_func_inst gf100_chan_inst; 119 void gf100_chan_userd_clear(struct nvkm_chan *); 120 void gf100_chan_preempt(struct nvkm_chan *); 121 122 int gk104_fifo_chid_nr(struct nvkm_fifo *); 123 int gk104_fifo_runl_ctor(struct nvkm_fifo *); 124 void gk104_fifo_init(struct nvkm_fifo *); 125 void gk104_fifo_init_pbdmas(struct nvkm_fifo *, u32); 126 irqreturn_t gk104_fifo_intr(struct nvkm_inth *); 127 void gk104_fifo_intr_runlist(struct nvkm_fifo *); 128 void gk104_fifo_intr_chsw(struct nvkm_fifo *); 129 void gk104_fifo_intr_bind(struct nvkm_fifo *); 130 extern const struct nvkm_fifo_func_mmu_fault gk104_fifo_mmu_fault; 131 extern const struct nvkm_enum gk104_fifo_mmu_fault_reason[]; 132 extern const struct nvkm_enum gk104_fifo_mmu_fault_hubclient[]; 133 extern const struct nvkm_enum gk104_fifo_mmu_fault_gpcclient[]; 134 void gk104_runl_insert_chan(struct nvkm_chan *, struct nvkm_memory *, u64); 135 void gk104_runl_commit(struct nvkm_runl *, struct nvkm_memory *, u32, int); 136 bool gk104_runl_pending(struct nvkm_runl *); 137 void gk104_runl_block(struct nvkm_runl *, u32); 138 void gk104_runl_allow(struct nvkm_runl *, u32); 139 void gk104_runl_fault_clear(struct nvkm_runl *); 140 extern const struct nvkm_runq_func gk104_runq; 141 void gk104_runq_init(struct nvkm_runq *); 142 bool gk104_runq_intr(struct nvkm_runq *, struct nvkm_runl *); 143 extern const struct nvkm_bitfield gk104_runq_intr_0_names[]; 144 bool gk104_runq_idle(struct nvkm_runq *); 145 extern const struct nvkm_engn_func gk104_engn; 146 bool gk104_engn_chsw(struct nvkm_engn *); 147 int gk104_engn_cxid(struct nvkm_engn *, bool *cgid); 148 int gk104_ectx_ctor(struct nvkm_engn *, struct nvkm_vctx *); 149 extern const struct nvkm_engn_func gk104_engn_ce; 150 extern const struct nvkm_chan_func_userd gk104_chan_userd; 151 extern const struct nvkm_chan_func_ramfc gk104_chan_ramfc; 152 void gk104_chan_bind(struct nvkm_chan *); 153 void gk104_chan_bind_inst(struct nvkm_chan *); 154 void gk104_chan_unbind(struct nvkm_chan *); 155 void gk104_chan_start(struct nvkm_chan *); 156 void gk104_chan_stop(struct nvkm_chan *); 157 158 int gk110_fifo_chid_ctor(struct nvkm_fifo *, int); 159 extern const struct nvkm_runl_func gk110_runl; 160 extern const struct nvkm_cgrp_func gk110_cgrp; 161 void gk110_runl_insert_cgrp(struct nvkm_cgrp *, struct nvkm_memory *, u64); 162 extern const struct nvkm_chan_func gk110_chan; 163 void gk110_chan_preempt(struct nvkm_chan *); 164 165 extern const struct nvkm_runq_func gk208_runq; 166 void gk208_runq_init(struct nvkm_runq *); 167 168 void gm107_fifo_intr_mmu_fault_unit(struct nvkm_fifo *, int); 169 extern const struct nvkm_fifo_func_mmu_fault gm107_fifo_mmu_fault; 170 extern const struct nvkm_runl_func gm107_runl; 171 extern const struct nvkm_chan_func gm107_chan; 172 173 int gm200_fifo_chid_nr(struct nvkm_fifo *); 174 int gm200_fifo_runq_nr(struct nvkm_fifo *); 175 176 extern const struct nvkm_enum gv100_fifo_mmu_fault_access[]; 177 extern const struct nvkm_enum gv100_fifo_mmu_fault_reason[]; 178 extern const struct nvkm_enum gv100_fifo_mmu_fault_hubclient[]; 179 extern const struct nvkm_enum gv100_fifo_mmu_fault_gpcclient[]; 180 void gv100_runl_insert_cgrp(struct nvkm_cgrp *, struct nvkm_memory *, u64); 181 void gv100_runl_insert_chan(struct nvkm_chan *, struct nvkm_memory *, u64); 182 void gv100_runl_preempt(struct nvkm_runl *); 183 extern const struct nvkm_runq_func gv100_runq; 184 extern const struct nvkm_engn_func gv100_engn; 185 void gv100_ectx_bind(struct nvkm_engn *, struct nvkm_cctx *, struct nvkm_chan *); 186 extern const struct nvkm_engn_func gv100_engn_ce; 187 int gv100_ectx_ce_ctor(struct nvkm_engn *, struct nvkm_vctx *); 188 void gv100_ectx_ce_bind(struct nvkm_engn *, struct nvkm_cctx *, struct nvkm_chan *); 189 extern const struct nvkm_chan_func_userd gv100_chan_userd; 190 extern const struct nvkm_chan_func_ramfc gv100_chan_ramfc; 191 192 void tu102_fifo_intr_ctxsw_timeout_info(struct nvkm_engn *, u32 info); 193 extern const struct nvkm_fifo_func_mmu_fault tu102_fifo_mmu_fault; 194 195 int ga100_fifo_runl_ctor(struct nvkm_fifo *); 196 int ga100_fifo_nonstall_ctor(struct nvkm_fifo *); 197 extern const struct nvkm_event_func ga100_fifo_nonstall; 198 extern const struct nvkm_runl_func ga100_runl; 199 extern const struct nvkm_runq_func ga100_runq; 200 extern const struct nvkm_engn_func ga100_engn; 201 extern const struct nvkm_engn_func ga100_engn_ce; 202 extern const struct nvkm_cgrp_func ga100_cgrp; 203 extern const struct nvkm_chan_func ga100_chan; 204 205 int nvkm_uchan_new(struct nvkm_fifo *, struct nvkm_cgrp *, const struct nvkm_oclass *, 206 void *argv, u32 argc, struct nvkm_object **); 207 int nvkm_ucgrp_new(struct nvkm_fifo *, const struct nvkm_oclass *, void *argv, u32 argc, 208 struct nvkm_object **); 209 #endif 210