xref: /linux/drivers/infiniband/hw/mthca/mthca_main.c (revision 5e8d780d745c1619aba81fe7166c5a4b5cad2b84)
1 /*
2  * Copyright (c) 2004, 2005 Topspin Communications.  All rights reserved.
3  * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
4  * Copyright (c) 2005 Mellanox Technologies. All rights reserved.
5  *
6  * This software is available to you under a choice of one of two
7  * licenses.  You may choose to be licensed under the terms of the GNU
8  * General Public License (GPL) Version 2, available from the file
9  * COPYING in the main directory of this source tree, or the
10  * OpenIB.org BSD license below:
11  *
12  *     Redistribution and use in source and binary forms, with or
13  *     without modification, are permitted provided that the following
14  *     conditions are met:
15  *
16  *      - Redistributions of source code must retain the above
17  *        copyright notice, this list of conditions and the following
18  *        disclaimer.
19  *
20  *      - Redistributions in binary form must reproduce the above
21  *        copyright notice, this list of conditions and the following
22  *        disclaimer in the documentation and/or other materials
23  *        provided with the distribution.
24  *
25  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32  * SOFTWARE.
33  *
34  * $Id: mthca_main.c 1396 2004-12-28 04:10:27Z roland $
35  */
36 
37 #include <linux/config.h>
38 #include <linux/module.h>
39 #include <linux/init.h>
40 #include <linux/errno.h>
41 #include <linux/pci.h>
42 #include <linux/interrupt.h>
43 
44 #include "mthca_dev.h"
45 #include "mthca_config_reg.h"
46 #include "mthca_cmd.h"
47 #include "mthca_profile.h"
48 #include "mthca_memfree.h"
49 
50 MODULE_AUTHOR("Roland Dreier");
51 MODULE_DESCRIPTION("Mellanox InfiniBand HCA low-level driver");
52 MODULE_LICENSE("Dual BSD/GPL");
53 MODULE_VERSION(DRV_VERSION);
54 
55 #ifdef CONFIG_INFINIBAND_MTHCA_DEBUG
56 
57 int mthca_debug_level = 0;
58 module_param_named(debug_level, mthca_debug_level, int, 0644);
59 MODULE_PARM_DESC(debug_level, "Enable debug tracing if > 0");
60 
61 #endif /* CONFIG_INFINIBAND_MTHCA_DEBUG */
62 
63 #ifdef CONFIG_PCI_MSI
64 
65 static int msi_x = 0;
66 module_param(msi_x, int, 0444);
67 MODULE_PARM_DESC(msi_x, "attempt to use MSI-X if nonzero");
68 
69 static int msi = 0;
70 module_param(msi, int, 0444);
71 MODULE_PARM_DESC(msi, "attempt to use MSI if nonzero");
72 
73 #else /* CONFIG_PCI_MSI */
74 
75 #define msi_x (0)
76 #define msi   (0)
77 
78 #endif /* CONFIG_PCI_MSI */
79 
80 static int tune_pci = 0;
81 module_param(tune_pci, int, 0444);
82 MODULE_PARM_DESC(tune_pci, "increase PCI burst from the default set by BIOS if nonzero");
83 
84 static const char mthca_version[] __devinitdata =
85 	DRV_NAME ": Mellanox InfiniBand HCA driver v"
86 	DRV_VERSION " (" DRV_RELDATE ")\n";
87 
88 static struct mthca_profile default_profile = {
89 	.num_qp		   = 1 << 16,
90 	.rdb_per_qp	   = 4,
91 	.num_cq		   = 1 << 16,
92 	.num_mcg	   = 1 << 13,
93 	.num_mpt	   = 1 << 17,
94 	.num_mtt	   = 1 << 20,
95 	.num_udav	   = 1 << 15,	/* Tavor only */
96 	.fmr_reserved_mtts = 1 << 18,	/* Tavor only */
97 	.uarc_size	   = 1 << 18,	/* Arbel only */
98 };
99 
100 static int __devinit mthca_tune_pci(struct mthca_dev *mdev)
101 {
102 	int cap;
103 	u16 val;
104 
105 	if (!tune_pci)
106 		return 0;
107 
108 	/* First try to max out Read Byte Count */
109 	cap = pci_find_capability(mdev->pdev, PCI_CAP_ID_PCIX);
110 	if (cap) {
111 		if (pci_read_config_word(mdev->pdev, cap + PCI_X_CMD, &val)) {
112 			mthca_err(mdev, "Couldn't read PCI-X command register, "
113 				  "aborting.\n");
114 			return -ENODEV;
115 		}
116 		val = (val & ~PCI_X_CMD_MAX_READ) | (3 << 2);
117 		if (pci_write_config_word(mdev->pdev, cap + PCI_X_CMD, val)) {
118 			mthca_err(mdev, "Couldn't write PCI-X command register, "
119 				  "aborting.\n");
120 			return -ENODEV;
121 		}
122 	} else if (!(mdev->mthca_flags & MTHCA_FLAG_PCIE))
123 		mthca_info(mdev, "No PCI-X capability, not setting RBC.\n");
124 
125 	cap = pci_find_capability(mdev->pdev, PCI_CAP_ID_EXP);
126 	if (cap) {
127 		if (pci_read_config_word(mdev->pdev, cap + PCI_EXP_DEVCTL, &val)) {
128 			mthca_err(mdev, "Couldn't read PCI Express device control "
129 				  "register, aborting.\n");
130 			return -ENODEV;
131 		}
132 		val = (val & ~PCI_EXP_DEVCTL_READRQ) | (5 << 12);
133 		if (pci_write_config_word(mdev->pdev, cap + PCI_EXP_DEVCTL, val)) {
134 			mthca_err(mdev, "Couldn't write PCI Express device control "
135 				  "register, aborting.\n");
136 			return -ENODEV;
137 		}
138 	} else if (mdev->mthca_flags & MTHCA_FLAG_PCIE)
139 		mthca_info(mdev, "No PCI Express capability, "
140 			   "not setting Max Read Request Size.\n");
141 
142 	return 0;
143 }
144 
145 static int __devinit mthca_dev_lim(struct mthca_dev *mdev, struct mthca_dev_lim *dev_lim)
146 {
147 	int err;
148 	u8 status;
149 
150 	err = mthca_QUERY_DEV_LIM(mdev, dev_lim, &status);
151 	if (err) {
152 		mthca_err(mdev, "QUERY_DEV_LIM command failed, aborting.\n");
153 		return err;
154 	}
155 	if (status) {
156 		mthca_err(mdev, "QUERY_DEV_LIM returned status 0x%02x, "
157 			  "aborting.\n", status);
158 		return -EINVAL;
159 	}
160 	if (dev_lim->min_page_sz > PAGE_SIZE) {
161 		mthca_err(mdev, "HCA minimum page size of %d bigger than "
162 			  "kernel PAGE_SIZE of %ld, aborting.\n",
163 			  dev_lim->min_page_sz, PAGE_SIZE);
164 		return -ENODEV;
165 	}
166 	if (dev_lim->num_ports > MTHCA_MAX_PORTS) {
167 		mthca_err(mdev, "HCA has %d ports, but we only support %d, "
168 			  "aborting.\n",
169 			  dev_lim->num_ports, MTHCA_MAX_PORTS);
170 		return -ENODEV;
171 	}
172 
173 	if (dev_lim->uar_size > pci_resource_len(mdev->pdev, 2)) {
174 		mthca_err(mdev, "HCA reported UAR size of 0x%x bigger than "
175 			  "PCI resource 2 size of 0x%llx, aborting.\n",
176 			  dev_lim->uar_size,
177 			  (unsigned long long)pci_resource_len(mdev->pdev, 2));
178 		return -ENODEV;
179 	}
180 
181 	mdev->limits.num_ports      	= dev_lim->num_ports;
182 	mdev->limits.vl_cap             = dev_lim->max_vl;
183 	mdev->limits.mtu_cap            = dev_lim->max_mtu;
184 	mdev->limits.gid_table_len  	= dev_lim->max_gids;
185 	mdev->limits.pkey_table_len 	= dev_lim->max_pkeys;
186 	mdev->limits.local_ca_ack_delay = dev_lim->local_ca_ack_delay;
187 	mdev->limits.max_sg             = dev_lim->max_sg;
188 	mdev->limits.max_wqes           = dev_lim->max_qp_sz;
189 	mdev->limits.max_qp_init_rdma   = dev_lim->max_requester_per_qp;
190 	mdev->limits.reserved_qps       = dev_lim->reserved_qps;
191 	mdev->limits.max_srq_wqes       = dev_lim->max_srq_sz;
192 	mdev->limits.reserved_srqs      = dev_lim->reserved_srqs;
193 	mdev->limits.reserved_eecs      = dev_lim->reserved_eecs;
194 	mdev->limits.max_desc_sz        = dev_lim->max_desc_sz;
195 	mdev->limits.max_srq_sge	= mthca_max_srq_sge(mdev);
196 	/*
197 	 * Subtract 1 from the limit because we need to allocate a
198 	 * spare CQE so the HCA HW can tell the difference between an
199 	 * empty CQ and a full CQ.
200 	 */
201 	mdev->limits.max_cqes           = dev_lim->max_cq_sz - 1;
202 	mdev->limits.reserved_cqs       = dev_lim->reserved_cqs;
203 	mdev->limits.reserved_eqs       = dev_lim->reserved_eqs;
204 	mdev->limits.reserved_mtts      = dev_lim->reserved_mtts;
205 	mdev->limits.reserved_mrws      = dev_lim->reserved_mrws;
206 	mdev->limits.reserved_uars      = dev_lim->reserved_uars;
207 	mdev->limits.reserved_pds       = dev_lim->reserved_pds;
208 	mdev->limits.port_width_cap     = dev_lim->max_port_width;
209 	mdev->limits.page_size_cap      = ~(u32) (dev_lim->min_page_sz - 1);
210 	mdev->limits.flags              = dev_lim->flags;
211 	/*
212 	 * For old FW that doesn't return static rate support, use a
213 	 * value of 0x3 (only static rate values of 0 or 1 are handled),
214 	 * except on Sinai, where even old FW can handle static rate
215 	 * values of 2 and 3.
216 	 */
217 	if (dev_lim->stat_rate_support)
218 		mdev->limits.stat_rate_support = dev_lim->stat_rate_support;
219 	else if (mdev->mthca_flags & MTHCA_FLAG_SINAI_OPT)
220 		mdev->limits.stat_rate_support = 0xf;
221 	else
222 		mdev->limits.stat_rate_support = 0x3;
223 
224 	/* IB_DEVICE_RESIZE_MAX_WR not supported by driver.
225 	   May be doable since hardware supports it for SRQ.
226 
227 	   IB_DEVICE_N_NOTIFY_CQ is supported by hardware but not by driver.
228 
229 	   IB_DEVICE_SRQ_RESIZE is supported by hardware but SRQ is not
230 	   supported by driver. */
231 	mdev->device_cap_flags = IB_DEVICE_CHANGE_PHY_PORT |
232 		IB_DEVICE_PORT_ACTIVE_EVENT |
233 		IB_DEVICE_SYS_IMAGE_GUID |
234 		IB_DEVICE_RC_RNR_NAK_GEN;
235 
236 	if (dev_lim->flags & DEV_LIM_FLAG_BAD_PKEY_CNTR)
237 		mdev->device_cap_flags |= IB_DEVICE_BAD_PKEY_CNTR;
238 
239 	if (dev_lim->flags & DEV_LIM_FLAG_BAD_QKEY_CNTR)
240 		mdev->device_cap_flags |= IB_DEVICE_BAD_QKEY_CNTR;
241 
242 	if (dev_lim->flags & DEV_LIM_FLAG_RAW_MULTI)
243 		mdev->device_cap_flags |= IB_DEVICE_RAW_MULTI;
244 
245 	if (dev_lim->flags & DEV_LIM_FLAG_AUTO_PATH_MIG)
246 		mdev->device_cap_flags |= IB_DEVICE_AUTO_PATH_MIG;
247 
248 	if (dev_lim->flags & DEV_LIM_FLAG_UD_AV_PORT_ENFORCE)
249 		mdev->device_cap_flags |= IB_DEVICE_UD_AV_PORT_ENFORCE;
250 
251 	if (dev_lim->flags & DEV_LIM_FLAG_SRQ)
252 		mdev->mthca_flags |= MTHCA_FLAG_SRQ;
253 
254 	return 0;
255 }
256 
257 static int __devinit mthca_init_tavor(struct mthca_dev *mdev)
258 {
259 	u8 status;
260 	int err;
261 	struct mthca_dev_lim        dev_lim;
262 	struct mthca_profile        profile;
263 	struct mthca_init_hca_param init_hca;
264 
265 	err = mthca_SYS_EN(mdev, &status);
266 	if (err) {
267 		mthca_err(mdev, "SYS_EN command failed, aborting.\n");
268 		return err;
269 	}
270 	if (status) {
271 		mthca_err(mdev, "SYS_EN returned status 0x%02x, "
272 			  "aborting.\n", status);
273 		return -EINVAL;
274 	}
275 
276 	err = mthca_QUERY_FW(mdev, &status);
277 	if (err) {
278 		mthca_err(mdev, "QUERY_FW command failed, aborting.\n");
279 		goto err_disable;
280 	}
281 	if (status) {
282 		mthca_err(mdev, "QUERY_FW returned status 0x%02x, "
283 			  "aborting.\n", status);
284 		err = -EINVAL;
285 		goto err_disable;
286 	}
287 	err = mthca_QUERY_DDR(mdev, &status);
288 	if (err) {
289 		mthca_err(mdev, "QUERY_DDR command failed, aborting.\n");
290 		goto err_disable;
291 	}
292 	if (status) {
293 		mthca_err(mdev, "QUERY_DDR returned status 0x%02x, "
294 			  "aborting.\n", status);
295 		err = -EINVAL;
296 		goto err_disable;
297 	}
298 
299 	err = mthca_dev_lim(mdev, &dev_lim);
300 	if (err) {
301 		mthca_err(mdev, "QUERY_DEV_LIM command failed, aborting.\n");
302 		goto err_disable;
303 	}
304 
305 	profile = default_profile;
306 	profile.num_uar   = dev_lim.uar_size / PAGE_SIZE;
307 	profile.uarc_size = 0;
308 	if (mdev->mthca_flags & MTHCA_FLAG_SRQ)
309 		profile.num_srq = dev_lim.max_srqs;
310 
311 	err = mthca_make_profile(mdev, &profile, &dev_lim, &init_hca);
312 	if (err < 0)
313 		goto err_disable;
314 
315 	err = mthca_INIT_HCA(mdev, &init_hca, &status);
316 	if (err) {
317 		mthca_err(mdev, "INIT_HCA command failed, aborting.\n");
318 		goto err_disable;
319 	}
320 	if (status) {
321 		mthca_err(mdev, "INIT_HCA returned status 0x%02x, "
322 			  "aborting.\n", status);
323 		err = -EINVAL;
324 		goto err_disable;
325 	}
326 
327 	return 0;
328 
329 err_disable:
330 	mthca_SYS_DIS(mdev, &status);
331 
332 	return err;
333 }
334 
335 static int __devinit mthca_load_fw(struct mthca_dev *mdev)
336 {
337 	u8 status;
338 	int err;
339 
340 	/* FIXME: use HCA-attached memory for FW if present */
341 
342 	mdev->fw.arbel.fw_icm =
343 		mthca_alloc_icm(mdev, mdev->fw.arbel.fw_pages,
344 				GFP_HIGHUSER | __GFP_NOWARN);
345 	if (!mdev->fw.arbel.fw_icm) {
346 		mthca_err(mdev, "Couldn't allocate FW area, aborting.\n");
347 		return -ENOMEM;
348 	}
349 
350 	err = mthca_MAP_FA(mdev, mdev->fw.arbel.fw_icm, &status);
351 	if (err) {
352 		mthca_err(mdev, "MAP_FA command failed, aborting.\n");
353 		goto err_free;
354 	}
355 	if (status) {
356 		mthca_err(mdev, "MAP_FA returned status 0x%02x, aborting.\n", status);
357 		err = -EINVAL;
358 		goto err_free;
359 	}
360 	err = mthca_RUN_FW(mdev, &status);
361 	if (err) {
362 		mthca_err(mdev, "RUN_FW command failed, aborting.\n");
363 		goto err_unmap_fa;
364 	}
365 	if (status) {
366 		mthca_err(mdev, "RUN_FW returned status 0x%02x, aborting.\n", status);
367 		err = -EINVAL;
368 		goto err_unmap_fa;
369 	}
370 
371 	return 0;
372 
373 err_unmap_fa:
374 	mthca_UNMAP_FA(mdev, &status);
375 
376 err_free:
377 	mthca_free_icm(mdev, mdev->fw.arbel.fw_icm);
378 	return err;
379 }
380 
381 static int __devinit mthca_init_icm(struct mthca_dev *mdev,
382 				    struct mthca_dev_lim *dev_lim,
383 				    struct mthca_init_hca_param *init_hca,
384 				    u64 icm_size)
385 {
386 	u64 aux_pages;
387 	u8 status;
388 	int err;
389 
390 	err = mthca_SET_ICM_SIZE(mdev, icm_size, &aux_pages, &status);
391 	if (err) {
392 		mthca_err(mdev, "SET_ICM_SIZE command failed, aborting.\n");
393 		return err;
394 	}
395 	if (status) {
396 		mthca_err(mdev, "SET_ICM_SIZE returned status 0x%02x, "
397 			  "aborting.\n", status);
398 		return -EINVAL;
399 	}
400 
401 	mthca_dbg(mdev, "%lld KB of HCA context requires %lld KB aux memory.\n",
402 		  (unsigned long long) icm_size >> 10,
403 		  (unsigned long long) aux_pages << 2);
404 
405 	mdev->fw.arbel.aux_icm = mthca_alloc_icm(mdev, aux_pages,
406 						 GFP_HIGHUSER | __GFP_NOWARN);
407 	if (!mdev->fw.arbel.aux_icm) {
408 		mthca_err(mdev, "Couldn't allocate aux memory, aborting.\n");
409 		return -ENOMEM;
410 	}
411 
412 	err = mthca_MAP_ICM_AUX(mdev, mdev->fw.arbel.aux_icm, &status);
413 	if (err) {
414 		mthca_err(mdev, "MAP_ICM_AUX command failed, aborting.\n");
415 		goto err_free_aux;
416 	}
417 	if (status) {
418 		mthca_err(mdev, "MAP_ICM_AUX returned status 0x%02x, aborting.\n", status);
419 		err = -EINVAL;
420 		goto err_free_aux;
421 	}
422 
423 	err = mthca_map_eq_icm(mdev, init_hca->eqc_base);
424 	if (err) {
425 		mthca_err(mdev, "Failed to map EQ context memory, aborting.\n");
426 		goto err_unmap_aux;
427 	}
428 
429 	mdev->mr_table.mtt_table = mthca_alloc_icm_table(mdev, init_hca->mtt_base,
430 							 MTHCA_MTT_SEG_SIZE,
431 							 mdev->limits.num_mtt_segs,
432 							 mdev->limits.reserved_mtts, 1);
433 	if (!mdev->mr_table.mtt_table) {
434 		mthca_err(mdev, "Failed to map MTT context memory, aborting.\n");
435 		err = -ENOMEM;
436 		goto err_unmap_eq;
437 	}
438 
439 	mdev->mr_table.mpt_table = mthca_alloc_icm_table(mdev, init_hca->mpt_base,
440 							 dev_lim->mpt_entry_sz,
441 							 mdev->limits.num_mpts,
442 							 mdev->limits.reserved_mrws, 1);
443 	if (!mdev->mr_table.mpt_table) {
444 		mthca_err(mdev, "Failed to map MPT context memory, aborting.\n");
445 		err = -ENOMEM;
446 		goto err_unmap_mtt;
447 	}
448 
449 	mdev->qp_table.qp_table = mthca_alloc_icm_table(mdev, init_hca->qpc_base,
450 							dev_lim->qpc_entry_sz,
451 							mdev->limits.num_qps,
452 							mdev->limits.reserved_qps, 0);
453 	if (!mdev->qp_table.qp_table) {
454 		mthca_err(mdev, "Failed to map QP context memory, aborting.\n");
455 		err = -ENOMEM;
456 		goto err_unmap_mpt;
457 	}
458 
459 	mdev->qp_table.eqp_table = mthca_alloc_icm_table(mdev, init_hca->eqpc_base,
460 							 dev_lim->eqpc_entry_sz,
461 							 mdev->limits.num_qps,
462 							 mdev->limits.reserved_qps, 0);
463 	if (!mdev->qp_table.eqp_table) {
464 		mthca_err(mdev, "Failed to map EQP context memory, aborting.\n");
465 		err = -ENOMEM;
466 		goto err_unmap_qp;
467 	}
468 
469 	mdev->qp_table.rdb_table = mthca_alloc_icm_table(mdev, init_hca->rdb_base,
470 							 MTHCA_RDB_ENTRY_SIZE,
471 							 mdev->limits.num_qps <<
472 							 mdev->qp_table.rdb_shift,
473 							 0, 0);
474 	if (!mdev->qp_table.rdb_table) {
475 		mthca_err(mdev, "Failed to map RDB context memory, aborting\n");
476 		err = -ENOMEM;
477 		goto err_unmap_eqp;
478 	}
479 
480        mdev->cq_table.table = mthca_alloc_icm_table(mdev, init_hca->cqc_base,
481 						    dev_lim->cqc_entry_sz,
482 						    mdev->limits.num_cqs,
483 						    mdev->limits.reserved_cqs, 0);
484 	if (!mdev->cq_table.table) {
485 		mthca_err(mdev, "Failed to map CQ context memory, aborting.\n");
486 		err = -ENOMEM;
487 		goto err_unmap_rdb;
488 	}
489 
490 	if (mdev->mthca_flags & MTHCA_FLAG_SRQ) {
491 		mdev->srq_table.table =
492 			mthca_alloc_icm_table(mdev, init_hca->srqc_base,
493 					      dev_lim->srq_entry_sz,
494 					      mdev->limits.num_srqs,
495 					      mdev->limits.reserved_srqs, 0);
496 		if (!mdev->srq_table.table) {
497 			mthca_err(mdev, "Failed to map SRQ context memory, "
498 				  "aborting.\n");
499 			err = -ENOMEM;
500 			goto err_unmap_cq;
501 		}
502 	}
503 
504 	/*
505 	 * It's not strictly required, but for simplicity just map the
506 	 * whole multicast group table now.  The table isn't very big
507 	 * and it's a lot easier than trying to track ref counts.
508 	 */
509 	mdev->mcg_table.table = mthca_alloc_icm_table(mdev, init_hca->mc_base,
510 						      MTHCA_MGM_ENTRY_SIZE,
511 						      mdev->limits.num_mgms +
512 						      mdev->limits.num_amgms,
513 						      mdev->limits.num_mgms +
514 						      mdev->limits.num_amgms,
515 						      0);
516 	if (!mdev->mcg_table.table) {
517 		mthca_err(mdev, "Failed to map MCG context memory, aborting.\n");
518 		err = -ENOMEM;
519 		goto err_unmap_srq;
520 	}
521 
522 	return 0;
523 
524 err_unmap_srq:
525 	if (mdev->mthca_flags & MTHCA_FLAG_SRQ)
526 		mthca_free_icm_table(mdev, mdev->srq_table.table);
527 
528 err_unmap_cq:
529 	mthca_free_icm_table(mdev, mdev->cq_table.table);
530 
531 err_unmap_rdb:
532 	mthca_free_icm_table(mdev, mdev->qp_table.rdb_table);
533 
534 err_unmap_eqp:
535 	mthca_free_icm_table(mdev, mdev->qp_table.eqp_table);
536 
537 err_unmap_qp:
538 	mthca_free_icm_table(mdev, mdev->qp_table.qp_table);
539 
540 err_unmap_mpt:
541 	mthca_free_icm_table(mdev, mdev->mr_table.mpt_table);
542 
543 err_unmap_mtt:
544 	mthca_free_icm_table(mdev, mdev->mr_table.mtt_table);
545 
546 err_unmap_eq:
547 	mthca_unmap_eq_icm(mdev);
548 
549 err_unmap_aux:
550 	mthca_UNMAP_ICM_AUX(mdev, &status);
551 
552 err_free_aux:
553 	mthca_free_icm(mdev, mdev->fw.arbel.aux_icm);
554 
555 	return err;
556 }
557 
558 static void mthca_free_icms(struct mthca_dev *mdev)
559 {
560 	u8 status;
561 
562 	mthca_free_icm_table(mdev, mdev->mcg_table.table);
563 	if (mdev->mthca_flags & MTHCA_FLAG_SRQ)
564 		mthca_free_icm_table(mdev, mdev->srq_table.table);
565 	mthca_free_icm_table(mdev, mdev->cq_table.table);
566 	mthca_free_icm_table(mdev, mdev->qp_table.rdb_table);
567 	mthca_free_icm_table(mdev, mdev->qp_table.eqp_table);
568 	mthca_free_icm_table(mdev, mdev->qp_table.qp_table);
569 	mthca_free_icm_table(mdev, mdev->mr_table.mpt_table);
570 	mthca_free_icm_table(mdev, mdev->mr_table.mtt_table);
571 	mthca_unmap_eq_icm(mdev);
572 
573 	mthca_UNMAP_ICM_AUX(mdev, &status);
574 	mthca_free_icm(mdev, mdev->fw.arbel.aux_icm);
575 }
576 
577 static int __devinit mthca_init_arbel(struct mthca_dev *mdev)
578 {
579 	struct mthca_dev_lim        dev_lim;
580 	struct mthca_profile        profile;
581 	struct mthca_init_hca_param init_hca;
582 	u64 icm_size;
583 	u8 status;
584 	int err;
585 
586 	err = mthca_QUERY_FW(mdev, &status);
587 	if (err) {
588 		mthca_err(mdev, "QUERY_FW command failed, aborting.\n");
589 		return err;
590 	}
591 	if (status) {
592 		mthca_err(mdev, "QUERY_FW returned status 0x%02x, "
593 			  "aborting.\n", status);
594 		return -EINVAL;
595 	}
596 
597 	err = mthca_ENABLE_LAM(mdev, &status);
598 	if (err) {
599 		mthca_err(mdev, "ENABLE_LAM command failed, aborting.\n");
600 		return err;
601 	}
602 	if (status == MTHCA_CMD_STAT_LAM_NOT_PRE) {
603 		mthca_dbg(mdev, "No HCA-attached memory (running in MemFree mode)\n");
604 		mdev->mthca_flags |= MTHCA_FLAG_NO_LAM;
605 	} else if (status) {
606 		mthca_err(mdev, "ENABLE_LAM returned status 0x%02x, "
607 			  "aborting.\n", status);
608 		return -EINVAL;
609 	}
610 
611 	err = mthca_load_fw(mdev);
612 	if (err) {
613 		mthca_err(mdev, "Failed to start FW, aborting.\n");
614 		goto err_disable;
615 	}
616 
617 	err = mthca_dev_lim(mdev, &dev_lim);
618 	if (err) {
619 		mthca_err(mdev, "QUERY_DEV_LIM command failed, aborting.\n");
620 		goto err_stop_fw;
621 	}
622 
623 	profile = default_profile;
624 	profile.num_uar  = dev_lim.uar_size / PAGE_SIZE;
625 	profile.num_udav = 0;
626 	if (mdev->mthca_flags & MTHCA_FLAG_SRQ)
627 		profile.num_srq = dev_lim.max_srqs;
628 
629 	icm_size = mthca_make_profile(mdev, &profile, &dev_lim, &init_hca);
630 	if ((int) icm_size < 0) {
631 		err = icm_size;
632 		goto err_stop_fw;
633 	}
634 
635 	err = mthca_init_icm(mdev, &dev_lim, &init_hca, icm_size);
636 	if (err)
637 		goto err_stop_fw;
638 
639 	err = mthca_INIT_HCA(mdev, &init_hca, &status);
640 	if (err) {
641 		mthca_err(mdev, "INIT_HCA command failed, aborting.\n");
642 		goto err_free_icm;
643 	}
644 	if (status) {
645 		mthca_err(mdev, "INIT_HCA returned status 0x%02x, "
646 			  "aborting.\n", status);
647 		err = -EINVAL;
648 		goto err_free_icm;
649 	}
650 
651 	return 0;
652 
653 err_free_icm:
654 	mthca_free_icms(mdev);
655 
656 err_stop_fw:
657 	mthca_UNMAP_FA(mdev, &status);
658 	mthca_free_icm(mdev, mdev->fw.arbel.fw_icm);
659 
660 err_disable:
661 	if (!(mdev->mthca_flags & MTHCA_FLAG_NO_LAM))
662 		mthca_DISABLE_LAM(mdev, &status);
663 
664 	return err;
665 }
666 
667 static void mthca_close_hca(struct mthca_dev *mdev)
668 {
669 	u8 status;
670 
671 	mthca_CLOSE_HCA(mdev, 0, &status);
672 
673 	if (mthca_is_memfree(mdev)) {
674 		mthca_free_icms(mdev);
675 
676 		mthca_UNMAP_FA(mdev, &status);
677 		mthca_free_icm(mdev, mdev->fw.arbel.fw_icm);
678 
679 		if (!(mdev->mthca_flags & MTHCA_FLAG_NO_LAM))
680 			mthca_DISABLE_LAM(mdev, &status);
681 	} else
682 		mthca_SYS_DIS(mdev, &status);
683 }
684 
685 static int __devinit mthca_init_hca(struct mthca_dev *mdev)
686 {
687 	u8 status;
688 	int err;
689 	struct mthca_adapter adapter;
690 
691 	if (mthca_is_memfree(mdev))
692 		err = mthca_init_arbel(mdev);
693 	else
694 		err = mthca_init_tavor(mdev);
695 
696 	if (err)
697 		return err;
698 
699 	err = mthca_QUERY_ADAPTER(mdev, &adapter, &status);
700 	if (err) {
701 		mthca_err(mdev, "QUERY_ADAPTER command failed, aborting.\n");
702 		goto err_close;
703 	}
704 	if (status) {
705 		mthca_err(mdev, "QUERY_ADAPTER returned status 0x%02x, "
706 			  "aborting.\n", status);
707 		err = -EINVAL;
708 		goto err_close;
709 	}
710 
711 	mdev->eq_table.inta_pin = adapter.inta_pin;
712 	mdev->rev_id            = adapter.revision_id;
713 	memcpy(mdev->board_id, adapter.board_id, sizeof mdev->board_id);
714 
715 	return 0;
716 
717 err_close:
718 	mthca_close_hca(mdev);
719 	return err;
720 }
721 
722 static int __devinit mthca_setup_hca(struct mthca_dev *dev)
723 {
724 	int err;
725 	u8 status;
726 
727 	MTHCA_INIT_DOORBELL_LOCK(&dev->doorbell_lock);
728 
729 	err = mthca_init_uar_table(dev);
730 	if (err) {
731 		mthca_err(dev, "Failed to initialize "
732 			  "user access region table, aborting.\n");
733 		return err;
734 	}
735 
736 	err = mthca_uar_alloc(dev, &dev->driver_uar);
737 	if (err) {
738 		mthca_err(dev, "Failed to allocate driver access region, "
739 			  "aborting.\n");
740 		goto err_uar_table_free;
741 	}
742 
743 	dev->kar = ioremap(dev->driver_uar.pfn << PAGE_SHIFT, PAGE_SIZE);
744 	if (!dev->kar) {
745 		mthca_err(dev, "Couldn't map kernel access region, "
746 			  "aborting.\n");
747 		err = -ENOMEM;
748 		goto err_uar_free;
749 	}
750 
751 	err = mthca_init_pd_table(dev);
752 	if (err) {
753 		mthca_err(dev, "Failed to initialize "
754 			  "protection domain table, aborting.\n");
755 		goto err_kar_unmap;
756 	}
757 
758 	err = mthca_init_mr_table(dev);
759 	if (err) {
760 		mthca_err(dev, "Failed to initialize "
761 			  "memory region table, aborting.\n");
762 		goto err_pd_table_free;
763 	}
764 
765 	err = mthca_pd_alloc(dev, 1, &dev->driver_pd);
766 	if (err) {
767 		mthca_err(dev, "Failed to create driver PD, "
768 			  "aborting.\n");
769 		goto err_mr_table_free;
770 	}
771 
772 	err = mthca_init_eq_table(dev);
773 	if (err) {
774 		mthca_err(dev, "Failed to initialize "
775 			  "event queue table, aborting.\n");
776 		goto err_pd_free;
777 	}
778 
779 	err = mthca_cmd_use_events(dev);
780 	if (err) {
781 		mthca_err(dev, "Failed to switch to event-driven "
782 			  "firmware commands, aborting.\n");
783 		goto err_eq_table_free;
784 	}
785 
786 	err = mthca_NOP(dev, &status);
787 	if (err || status) {
788 		mthca_err(dev, "NOP command failed to generate interrupt (IRQ %d), aborting.\n",
789 			  dev->mthca_flags & MTHCA_FLAG_MSI_X ?
790 			  dev->eq_table.eq[MTHCA_EQ_CMD].msi_x_vector :
791 			  dev->pdev->irq);
792 		if (dev->mthca_flags & (MTHCA_FLAG_MSI | MTHCA_FLAG_MSI_X))
793 			mthca_err(dev, "Try again with MSI/MSI-X disabled.\n");
794 		else
795 			mthca_err(dev, "BIOS or ACPI interrupt routing problem?\n");
796 
797 		goto err_cmd_poll;
798 	}
799 
800 	mthca_dbg(dev, "NOP command IRQ test passed\n");
801 
802 	err = mthca_init_cq_table(dev);
803 	if (err) {
804 		mthca_err(dev, "Failed to initialize "
805 			  "completion queue table, aborting.\n");
806 		goto err_cmd_poll;
807 	}
808 
809 	err = mthca_init_srq_table(dev);
810 	if (err) {
811 		mthca_err(dev, "Failed to initialize "
812 			  "shared receive queue table, aborting.\n");
813 		goto err_cq_table_free;
814 	}
815 
816 	err = mthca_init_qp_table(dev);
817 	if (err) {
818 		mthca_err(dev, "Failed to initialize "
819 			  "queue pair table, aborting.\n");
820 		goto err_srq_table_free;
821 	}
822 
823 	err = mthca_init_av_table(dev);
824 	if (err) {
825 		mthca_err(dev, "Failed to initialize "
826 			  "address vector table, aborting.\n");
827 		goto err_qp_table_free;
828 	}
829 
830 	err = mthca_init_mcg_table(dev);
831 	if (err) {
832 		mthca_err(dev, "Failed to initialize "
833 			  "multicast group table, aborting.\n");
834 		goto err_av_table_free;
835 	}
836 
837 	return 0;
838 
839 err_av_table_free:
840 	mthca_cleanup_av_table(dev);
841 
842 err_qp_table_free:
843 	mthca_cleanup_qp_table(dev);
844 
845 err_srq_table_free:
846 	mthca_cleanup_srq_table(dev);
847 
848 err_cq_table_free:
849 	mthca_cleanup_cq_table(dev);
850 
851 err_cmd_poll:
852 	mthca_cmd_use_polling(dev);
853 
854 err_eq_table_free:
855 	mthca_cleanup_eq_table(dev);
856 
857 err_pd_free:
858 	mthca_pd_free(dev, &dev->driver_pd);
859 
860 err_mr_table_free:
861 	mthca_cleanup_mr_table(dev);
862 
863 err_pd_table_free:
864 	mthca_cleanup_pd_table(dev);
865 
866 err_kar_unmap:
867 	iounmap(dev->kar);
868 
869 err_uar_free:
870 	mthca_uar_free(dev, &dev->driver_uar);
871 
872 err_uar_table_free:
873 	mthca_cleanup_uar_table(dev);
874 	return err;
875 }
876 
877 static int __devinit mthca_request_regions(struct pci_dev *pdev,
878 					   int ddr_hidden)
879 {
880 	int err;
881 
882 	/*
883 	 * We can't just use pci_request_regions() because the MSI-X
884 	 * table is right in the middle of the first BAR.  If we did
885 	 * pci_request_region and grab all of the first BAR, then
886 	 * setting up MSI-X would fail, since the PCI core wants to do
887 	 * request_mem_region on the MSI-X vector table.
888 	 *
889 	 * So just request what we need right now, and request any
890 	 * other regions we need when setting up EQs.
891 	 */
892 	if (!request_mem_region(pci_resource_start(pdev, 0) + MTHCA_HCR_BASE,
893 				MTHCA_HCR_SIZE, DRV_NAME))
894 		return -EBUSY;
895 
896 	err = pci_request_region(pdev, 2, DRV_NAME);
897 	if (err)
898 		goto err_bar2_failed;
899 
900 	if (!ddr_hidden) {
901 		err = pci_request_region(pdev, 4, DRV_NAME);
902 		if (err)
903 			goto err_bar4_failed;
904 	}
905 
906 	return 0;
907 
908 err_bar4_failed:
909 	pci_release_region(pdev, 2);
910 
911 err_bar2_failed:
912 	release_mem_region(pci_resource_start(pdev, 0) + MTHCA_HCR_BASE,
913 			   MTHCA_HCR_SIZE);
914 
915 	return err;
916 }
917 
918 static void mthca_release_regions(struct pci_dev *pdev,
919 				  int ddr_hidden)
920 {
921 	if (!ddr_hidden)
922 		pci_release_region(pdev, 4);
923 
924 	pci_release_region(pdev, 2);
925 
926 	release_mem_region(pci_resource_start(pdev, 0) + MTHCA_HCR_BASE,
927 			   MTHCA_HCR_SIZE);
928 }
929 
930 static int __devinit mthca_enable_msi_x(struct mthca_dev *mdev)
931 {
932 	struct msix_entry entries[3];
933 	int err;
934 
935 	entries[0].entry = 0;
936 	entries[1].entry = 1;
937 	entries[2].entry = 2;
938 
939 	err = pci_enable_msix(mdev->pdev, entries, ARRAY_SIZE(entries));
940 	if (err) {
941 		if (err > 0)
942 			mthca_info(mdev, "Only %d MSI-X vectors available, "
943 				   "not using MSI-X\n", err);
944 		return err;
945 	}
946 
947 	mdev->eq_table.eq[MTHCA_EQ_COMP ].msi_x_vector = entries[0].vector;
948 	mdev->eq_table.eq[MTHCA_EQ_ASYNC].msi_x_vector = entries[1].vector;
949 	mdev->eq_table.eq[MTHCA_EQ_CMD  ].msi_x_vector = entries[2].vector;
950 
951 	return 0;
952 }
953 
954 /* Types of supported HCA */
955 enum {
956 	TAVOR,			/* MT23108                        */
957 	ARBEL_COMPAT,		/* MT25208 in Tavor compat mode   */
958 	ARBEL_NATIVE,		/* MT25208 with extended features */
959 	SINAI			/* MT25204 */
960 };
961 
962 #define MTHCA_FW_VER(major, minor, subminor) \
963 	(((u64) (major) << 32) | ((u64) (minor) << 16) | (u64) (subminor))
964 
965 static struct {
966 	u64 latest_fw;
967 	u32 flags;
968 } mthca_hca_table[] = {
969 	[TAVOR]        = { .latest_fw = MTHCA_FW_VER(3, 4, 0),
970 			   .flags     = 0 },
971 	[ARBEL_COMPAT] = { .latest_fw = MTHCA_FW_VER(4, 7, 400),
972 			   .flags     = MTHCA_FLAG_PCIE },
973 	[ARBEL_NATIVE] = { .latest_fw = MTHCA_FW_VER(5, 1, 0),
974 			   .flags     = MTHCA_FLAG_MEMFREE |
975 					MTHCA_FLAG_PCIE },
976 	[SINAI]        = { .latest_fw = MTHCA_FW_VER(1, 0, 800),
977 			   .flags     = MTHCA_FLAG_MEMFREE |
978 					MTHCA_FLAG_PCIE    |
979 					MTHCA_FLAG_SINAI_OPT }
980 };
981 
982 static int __devinit mthca_init_one(struct pci_dev *pdev,
983 				    const struct pci_device_id *id)
984 {
985 	static int mthca_version_printed = 0;
986 	int ddr_hidden = 0;
987 	int err;
988 	struct mthca_dev *mdev;
989 
990 	if (!mthca_version_printed) {
991 		printk(KERN_INFO "%s", mthca_version);
992 		++mthca_version_printed;
993 	}
994 
995 	printk(KERN_INFO PFX "Initializing %s\n",
996 	       pci_name(pdev));
997 
998 	if (id->driver_data >= ARRAY_SIZE(mthca_hca_table)) {
999 		printk(KERN_ERR PFX "%s has invalid driver data %lx\n",
1000 		       pci_name(pdev), id->driver_data);
1001 		return -ENODEV;
1002 	}
1003 
1004 	err = pci_enable_device(pdev);
1005 	if (err) {
1006 		dev_err(&pdev->dev, "Cannot enable PCI device, "
1007 			"aborting.\n");
1008 		return err;
1009 	}
1010 
1011 	/*
1012 	 * Check for BARs.  We expect 0: 1MB, 2: 8MB, 4: DDR (may not
1013 	 * be present)
1014 	 */
1015 	if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM) ||
1016 	    pci_resource_len(pdev, 0) != 1 << 20) {
1017 		dev_err(&pdev->dev, "Missing DCS, aborting.\n");
1018 		err = -ENODEV;
1019 		goto err_disable_pdev;
1020 	}
1021 	if (!(pci_resource_flags(pdev, 2) & IORESOURCE_MEM)) {
1022 		dev_err(&pdev->dev, "Missing UAR, aborting.\n");
1023 		err = -ENODEV;
1024 		goto err_disable_pdev;
1025 	}
1026 	if (!(pci_resource_flags(pdev, 4) & IORESOURCE_MEM))
1027 		ddr_hidden = 1;
1028 
1029 	err = mthca_request_regions(pdev, ddr_hidden);
1030 	if (err) {
1031 		dev_err(&pdev->dev, "Cannot obtain PCI resources, "
1032 			"aborting.\n");
1033 		goto err_disable_pdev;
1034 	}
1035 
1036 	pci_set_master(pdev);
1037 
1038 	err = pci_set_dma_mask(pdev, DMA_64BIT_MASK);
1039 	if (err) {
1040 		dev_warn(&pdev->dev, "Warning: couldn't set 64-bit PCI DMA mask.\n");
1041 		err = pci_set_dma_mask(pdev, DMA_32BIT_MASK);
1042 		if (err) {
1043 			dev_err(&pdev->dev, "Can't set PCI DMA mask, aborting.\n");
1044 			goto err_free_res;
1045 		}
1046 	}
1047 	err = pci_set_consistent_dma_mask(pdev, DMA_64BIT_MASK);
1048 	if (err) {
1049 		dev_warn(&pdev->dev, "Warning: couldn't set 64-bit "
1050 			 "consistent PCI DMA mask.\n");
1051 		err = pci_set_consistent_dma_mask(pdev, DMA_32BIT_MASK);
1052 		if (err) {
1053 			dev_err(&pdev->dev, "Can't set consistent PCI DMA mask, "
1054 				"aborting.\n");
1055 			goto err_free_res;
1056 		}
1057 	}
1058 
1059 	mdev = (struct mthca_dev *) ib_alloc_device(sizeof *mdev);
1060 	if (!mdev) {
1061 		dev_err(&pdev->dev, "Device struct alloc failed, "
1062 			"aborting.\n");
1063 		err = -ENOMEM;
1064 		goto err_free_res;
1065 	}
1066 
1067 	mdev->pdev = pdev;
1068 
1069 	mdev->mthca_flags = mthca_hca_table[id->driver_data].flags;
1070 	if (ddr_hidden)
1071 		mdev->mthca_flags |= MTHCA_FLAG_DDR_HIDDEN;
1072 
1073 	/*
1074 	 * Now reset the HCA before we touch the PCI capabilities or
1075 	 * attempt a firmware command, since a boot ROM may have left
1076 	 * the HCA in an undefined state.
1077 	 */
1078 	err = mthca_reset(mdev);
1079 	if (err) {
1080 		mthca_err(mdev, "Failed to reset HCA, aborting.\n");
1081 		goto err_free_dev;
1082 	}
1083 
1084 	if (msi_x && !mthca_enable_msi_x(mdev))
1085 		mdev->mthca_flags |= MTHCA_FLAG_MSI_X;
1086 	if (msi && !(mdev->mthca_flags & MTHCA_FLAG_MSI_X) &&
1087 	    !pci_enable_msi(pdev))
1088 		mdev->mthca_flags |= MTHCA_FLAG_MSI;
1089 
1090 	if (mthca_cmd_init(mdev)) {
1091 		mthca_err(mdev, "Failed to init command interface, aborting.\n");
1092 		goto err_free_dev;
1093 	}
1094 
1095 	err = mthca_tune_pci(mdev);
1096 	if (err)
1097 		goto err_cmd;
1098 
1099 	err = mthca_init_hca(mdev);
1100 	if (err)
1101 		goto err_cmd;
1102 
1103 	if (mdev->fw_ver < mthca_hca_table[id->driver_data].latest_fw) {
1104 		mthca_warn(mdev, "HCA FW version %d.%d.%d is old (%d.%d.%d is current).\n",
1105 			   (int) (mdev->fw_ver >> 32), (int) (mdev->fw_ver >> 16) & 0xffff,
1106 			   (int) (mdev->fw_ver & 0xffff),
1107 			   (int) (mthca_hca_table[id->driver_data].latest_fw >> 32),
1108 			   (int) (mthca_hca_table[id->driver_data].latest_fw >> 16) & 0xffff,
1109 			   (int) (mthca_hca_table[id->driver_data].latest_fw & 0xffff));
1110 		mthca_warn(mdev, "If you have problems, try updating your HCA FW.\n");
1111 	}
1112 
1113 	err = mthca_setup_hca(mdev);
1114 	if (err)
1115 		goto err_close;
1116 
1117 	err = mthca_register_device(mdev);
1118 	if (err)
1119 		goto err_cleanup;
1120 
1121 	err = mthca_create_agents(mdev);
1122 	if (err)
1123 		goto err_unregister;
1124 
1125 	pci_set_drvdata(pdev, mdev);
1126 
1127 	return 0;
1128 
1129 err_unregister:
1130 	mthca_unregister_device(mdev);
1131 
1132 err_cleanup:
1133 	mthca_cleanup_mcg_table(mdev);
1134 	mthca_cleanup_av_table(mdev);
1135 	mthca_cleanup_qp_table(mdev);
1136 	mthca_cleanup_srq_table(mdev);
1137 	mthca_cleanup_cq_table(mdev);
1138 	mthca_cmd_use_polling(mdev);
1139 	mthca_cleanup_eq_table(mdev);
1140 
1141 	mthca_pd_free(mdev, &mdev->driver_pd);
1142 
1143 	mthca_cleanup_mr_table(mdev);
1144 	mthca_cleanup_pd_table(mdev);
1145 	mthca_cleanup_uar_table(mdev);
1146 
1147 err_close:
1148 	mthca_close_hca(mdev);
1149 
1150 err_cmd:
1151 	mthca_cmd_cleanup(mdev);
1152 
1153 err_free_dev:
1154 	if (mdev->mthca_flags & MTHCA_FLAG_MSI_X)
1155 		pci_disable_msix(pdev);
1156 	if (mdev->mthca_flags & MTHCA_FLAG_MSI)
1157 		pci_disable_msi(pdev);
1158 
1159 	ib_dealloc_device(&mdev->ib_dev);
1160 
1161 err_free_res:
1162 	mthca_release_regions(pdev, ddr_hidden);
1163 
1164 err_disable_pdev:
1165 	pci_disable_device(pdev);
1166 	pci_set_drvdata(pdev, NULL);
1167 	return err;
1168 }
1169 
1170 static void __devexit mthca_remove_one(struct pci_dev *pdev)
1171 {
1172 	struct mthca_dev *mdev = pci_get_drvdata(pdev);
1173 	u8 status;
1174 	int p;
1175 
1176 	if (mdev) {
1177 		mthca_free_agents(mdev);
1178 		mthca_unregister_device(mdev);
1179 
1180 		for (p = 1; p <= mdev->limits.num_ports; ++p)
1181 			mthca_CLOSE_IB(mdev, p, &status);
1182 
1183 		mthca_cleanup_mcg_table(mdev);
1184 		mthca_cleanup_av_table(mdev);
1185 		mthca_cleanup_qp_table(mdev);
1186 		mthca_cleanup_srq_table(mdev);
1187 		mthca_cleanup_cq_table(mdev);
1188 		mthca_cmd_use_polling(mdev);
1189 		mthca_cleanup_eq_table(mdev);
1190 
1191 		mthca_pd_free(mdev, &mdev->driver_pd);
1192 
1193 		mthca_cleanup_mr_table(mdev);
1194 		mthca_cleanup_pd_table(mdev);
1195 
1196 		iounmap(mdev->kar);
1197 		mthca_uar_free(mdev, &mdev->driver_uar);
1198 		mthca_cleanup_uar_table(mdev);
1199 		mthca_close_hca(mdev);
1200 		mthca_cmd_cleanup(mdev);
1201 
1202 		if (mdev->mthca_flags & MTHCA_FLAG_MSI_X)
1203 			pci_disable_msix(pdev);
1204 		if (mdev->mthca_flags & MTHCA_FLAG_MSI)
1205 			pci_disable_msi(pdev);
1206 
1207 		ib_dealloc_device(&mdev->ib_dev);
1208 		mthca_release_regions(pdev, mdev->mthca_flags &
1209 				      MTHCA_FLAG_DDR_HIDDEN);
1210 		pci_disable_device(pdev);
1211 		pci_set_drvdata(pdev, NULL);
1212 	}
1213 }
1214 
1215 static struct pci_device_id mthca_pci_table[] = {
1216 	{ PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, PCI_DEVICE_ID_MELLANOX_TAVOR),
1217 	  .driver_data = TAVOR },
1218 	{ PCI_DEVICE(PCI_VENDOR_ID_TOPSPIN, PCI_DEVICE_ID_MELLANOX_TAVOR),
1219 	  .driver_data = TAVOR },
1220 	{ PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, PCI_DEVICE_ID_MELLANOX_ARBEL_COMPAT),
1221 	  .driver_data = ARBEL_COMPAT },
1222 	{ PCI_DEVICE(PCI_VENDOR_ID_TOPSPIN, PCI_DEVICE_ID_MELLANOX_ARBEL_COMPAT),
1223 	  .driver_data = ARBEL_COMPAT },
1224 	{ PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, PCI_DEVICE_ID_MELLANOX_ARBEL),
1225 	  .driver_data = ARBEL_NATIVE },
1226 	{ PCI_DEVICE(PCI_VENDOR_ID_TOPSPIN, PCI_DEVICE_ID_MELLANOX_ARBEL),
1227 	  .driver_data = ARBEL_NATIVE },
1228 	{ PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, PCI_DEVICE_ID_MELLANOX_SINAI),
1229 	  .driver_data = SINAI },
1230 	{ PCI_DEVICE(PCI_VENDOR_ID_TOPSPIN, PCI_DEVICE_ID_MELLANOX_SINAI),
1231 	  .driver_data = SINAI },
1232 	{ PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, PCI_DEVICE_ID_MELLANOX_SINAI_OLD),
1233 	  .driver_data = SINAI },
1234 	{ PCI_DEVICE(PCI_VENDOR_ID_TOPSPIN, PCI_DEVICE_ID_MELLANOX_SINAI_OLD),
1235 	  .driver_data = SINAI },
1236 	{ 0, }
1237 };
1238 
1239 MODULE_DEVICE_TABLE(pci, mthca_pci_table);
1240 
1241 static struct pci_driver mthca_driver = {
1242 	.name		= DRV_NAME,
1243 	.id_table	= mthca_pci_table,
1244 	.probe		= mthca_init_one,
1245 	.remove		= __devexit_p(mthca_remove_one)
1246 };
1247 
1248 static int __init mthca_init(void)
1249 {
1250 	int ret;
1251 
1252 	ret = pci_register_driver(&mthca_driver);
1253 	return ret < 0 ? ret : 0;
1254 }
1255 
1256 static void __exit mthca_cleanup(void)
1257 {
1258 	pci_unregister_driver(&mthca_driver);
1259 }
1260 
1261 module_init(mthca_init);
1262 module_exit(mthca_cleanup);
1263