xref: /linux/drivers/scsi/fnic/fip.c (revision 88e45067a30918ebb4942120892963e2311330af)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright 2008 Cisco Systems, Inc.  All rights reserved.
4  * Copyright 2007 Nuova Systems, Inc.  All rights reserved.
5  */
6 #include "fnic.h"
7 #include "fip.h"
8 #include <linux/etherdevice.h>
9 
10 #define FIP_FNIC_RESET_WAIT_COUNT 15
11 
12 /**
13  * fnic_fcoe_reset_vlans - Free up the list of discovered vlans
14  * @fnic: Handle to fnic driver instance
15  */
fnic_fcoe_reset_vlans(struct fnic * fnic)16 void fnic_fcoe_reset_vlans(struct fnic *fnic)
17 {
18 	unsigned long flags;
19 	struct fcoe_vlan *vlan, *next;
20 
21 	spin_lock_irqsave(&fnic->vlans_lock, flags);
22 	if (!list_empty(&fnic->vlan_list)) {
23 		list_for_each_entry_safe(vlan, next, &fnic->vlan_list, list) {
24 			list_del(&vlan->list);
25 			kfree(vlan);
26 		}
27 	}
28 
29 	spin_unlock_irqrestore(&fnic->vlans_lock, flags);
30 	FNIC_FIP_DBG(KERN_INFO, fnic->host, fnic->fnic_num,
31 		     "Reset vlan complete\n");
32 }
33 
34 /**
35  * fnic_fcoe_send_vlan_req - Send FIP vlan request to all FCFs MAC
36  * @fnic: Handle to fnic driver instance
37  */
fnic_fcoe_send_vlan_req(struct fnic * fnic)38 void fnic_fcoe_send_vlan_req(struct fnic *fnic)
39 {
40 	uint8_t *frame;
41 	struct fnic_iport_s *iport = &fnic->iport;
42 	struct fnic_stats *fnic_stats = &fnic->fnic_stats;
43 	u64 vlan_tov;
44 	struct fip_vlan_req *pvlan_req;
45 	uint16_t frame_size = sizeof(struct fip_vlan_req);
46 
47 	frame = fdls_alloc_frame(iport);
48 	if (frame == NULL) {
49 		FNIC_FIP_DBG(KERN_ERR, fnic->host, fnic->fnic_num,
50 		     "Failed to allocate frame to send VLAN req");
51 		return;
52 	}
53 
54 	fnic_fcoe_reset_vlans(fnic);
55 
56 	fnic->set_vlan(fnic, 0);
57 	FNIC_FIP_DBG(KERN_INFO, fnic->host, fnic->fnic_num,
58 		     "set vlan done\n");
59 
60 	FNIC_FIP_DBG(KERN_INFO, fnic->host, fnic->fnic_num,
61 		     "got MAC 0x%x:%x:%x:%x:%x:%x\n", iport->hwmac[0],
62 		     iport->hwmac[1], iport->hwmac[2], iport->hwmac[3],
63 		     iport->hwmac[4], iport->hwmac[5]);
64 
65 	pvlan_req = (struct fip_vlan_req *) frame;
66 	*pvlan_req = (struct fip_vlan_req) {
67 		.eth = {.h_dest = FCOE_ALL_FCFS_MAC,
68 			.h_proto = cpu_to_be16(ETH_P_FIP)},
69 		.fip = {.fip_ver = FIP_VER_ENCAPS(FIP_VER),
70 			.fip_op = cpu_to_be16(FIP_OP_VLAN),
71 			.fip_subcode = FIP_SC_REQ,
72 			.fip_dl_len = cpu_to_be16(FIP_VLAN_REQ_LEN)},
73 		.mac_desc = {.fd_desc = {.fip_dtype = FIP_DT_MAC,
74 						.fip_dlen = 2}}
75 	};
76 
77 	memcpy(pvlan_req->eth.h_source, iport->hwmac, ETH_ALEN);
78 	memcpy(pvlan_req->mac_desc.fd_mac, iport->hwmac, ETH_ALEN);
79 
80 	atomic64_inc(&fnic_stats->vlan_stats.vlan_disc_reqs);
81 
82 	iport->fip.state = FDLS_FIP_VLAN_DISCOVERY_STARTED;
83 
84 	FNIC_FIP_DBG(KERN_INFO, fnic->host, fnic->fnic_num,
85 		     "Send VLAN req\n");
86 	fnic_send_fip_frame(iport, frame, frame_size);
87 
88 	vlan_tov = jiffies + msecs_to_jiffies(FCOE_CTLR_FIPVLAN_TOV);
89 	mod_timer(&fnic->retry_fip_timer, round_jiffies(vlan_tov));
90 	FNIC_FIP_DBG(KERN_INFO, fnic->host, fnic->fnic_num,
91 		     "fip timer set\n");
92 }
93 
94 /**
95  * fnic_fcoe_process_vlan_resp - Processes the vlan response from one FCF and
96  * populates VLAN list.
97  * @fnic: Handle to fnic driver instance
98  * @fiph: Received FIP frame
99  *
100  * Will wait for responses from multiple FCFs until timeout.
101  */
fnic_fcoe_process_vlan_resp(struct fnic * fnic,struct fip_header * fiph)102 void fnic_fcoe_process_vlan_resp(struct fnic *fnic, struct fip_header *fiph)
103 {
104 	struct fip_vlan_notif *vlan_notif = (struct fip_vlan_notif *)fiph;
105 
106 	struct fnic_stats *fnic_stats = &fnic->fnic_stats;
107 	u16 vid;
108 	int num_vlan = 0;
109 	int cur_desc, desc_len;
110 	struct fcoe_vlan *vlan;
111 	struct fip_vlan_desc *vlan_desc;
112 	unsigned long flags;
113 
114 	FNIC_FIP_DBG(KERN_INFO, fnic->host, fnic->fnic_num,
115 		     "fnic 0x%p got vlan resp\n", fnic);
116 
117 	desc_len = be16_to_cpu(vlan_notif->fip.fip_dl_len);
118 	FNIC_FIP_DBG(KERN_INFO, fnic->host, fnic->fnic_num,
119 		     "desc_len %d\n", desc_len);
120 
121 	spin_lock_irqsave(&fnic->vlans_lock, flags);
122 
123 	cur_desc = 0;
124 	while (desc_len > 0) {
125 		vlan_desc =
126 		    (struct fip_vlan_desc *)(((char *)vlan_notif->vlans_desc)
127 					       + cur_desc * 4);
128 
129 		if (vlan_desc->fd_desc.fip_dtype == FIP_DT_VLAN) {
130 			if (vlan_desc->fd_desc.fip_dlen != 1) {
131 				FNIC_FIP_DBG(KERN_INFO, fnic->host,
132 					     fnic->fnic_num,
133 					     "Invalid descriptor length(%x) in VLan response\n",
134 					     vlan_desc->fd_desc.fip_dlen);
135 
136 			}
137 			num_vlan++;
138 			vid = be16_to_cpu(vlan_desc->fd_vlan);
139 			FNIC_FIP_DBG(KERN_INFO, fnic->host,
140 				     fnic->fnic_num,
141 				     "process_vlan_resp: FIP VLAN %d\n", vid);
142 			vlan = kzalloc(sizeof(*vlan), GFP_KERNEL);
143 
144 			if (!vlan) {
145 				/* retry from timer */
146 				FNIC_FIP_DBG(KERN_INFO, fnic->host,
147 					     fnic->fnic_num,
148 					     "Mem Alloc failure\n");
149 				spin_unlock_irqrestore(&fnic->vlans_lock,
150 						       flags);
151 				goto out;
152 			}
153 			vlan->vid = vid & 0x0fff;
154 			vlan->state = FIP_VLAN_AVAIL;
155 			list_add_tail(&vlan->list, &fnic->vlan_list);
156 			break;
157 		}
158 		FNIC_FIP_DBG(KERN_INFO, fnic->host,
159 			     fnic->fnic_num,
160 			     "Invalid descriptor type(%x) in VLan response\n",
161 			     vlan_desc->fd_desc.fip_dtype);
162 		/*
163 		 * Note : received a type=2 descriptor here i.e. FIP
164 		 * MAC Address Descriptor
165 		 */
166 		cur_desc += vlan_desc->fd_desc.fip_dlen;
167 		desc_len -= vlan_desc->fd_desc.fip_dlen;
168 	}
169 
170 	/* any VLAN descriptors present ? */
171 	if (num_vlan == 0) {
172 		atomic64_inc(&fnic_stats->vlan_stats.resp_withno_vlanID);
173 		FNIC_FIP_DBG(KERN_INFO, fnic->host, fnic->fnic_num,
174 			     "fnic 0x%p No VLAN descriptors in FIP VLAN response\n",
175 			     fnic);
176 	}
177 
178 	spin_unlock_irqrestore(&fnic->vlans_lock, flags);
179 
180  out:
181 	return;
182 }
183 
184 /**
185  * fnic_fcoe_start_fcf_discovery - Start FIP FCF discovery in a selected vlan
186  * @fnic: Handle to fnic driver instance
187  */
fnic_fcoe_start_fcf_discovery(struct fnic * fnic)188 void fnic_fcoe_start_fcf_discovery(struct fnic *fnic)
189 {
190 	uint8_t *frame;
191 	struct fnic_iport_s *iport = &fnic->iport;
192 	u64 fcs_tov;
193 	struct fip_discovery *pdisc_sol;
194 	uint16_t frame_size = sizeof(struct fip_discovery);
195 
196 	frame = fdls_alloc_frame(iport);
197 	if (frame == NULL) {
198 		FNIC_FIP_DBG(KERN_ERR, fnic->host, fnic->fnic_num,
199 		     "Failed to allocate frame to start FCF discovery");
200 		return;
201 	}
202 
203 	memset(iport->selected_fcf.fcf_mac, 0, ETH_ALEN);
204 
205 	pdisc_sol = (struct fip_discovery *) frame;
206 	*pdisc_sol = (struct fip_discovery) {
207 		.eth = {.h_dest = FCOE_ALL_FCFS_MAC,
208 			.h_proto = cpu_to_be16(ETH_P_FIP)},
209 		.fip = {
210 			.fip_ver = FIP_VER_ENCAPS(FIP_VER), .fip_op = cpu_to_be16(FIP_OP_DISC),
211 			.fip_subcode = FIP_SC_REQ, .fip_dl_len = cpu_to_be16(FIP_DISC_SOL_LEN),
212 			.fip_flags = cpu_to_be16(FIP_FL_FPMA)},
213 		.mac_desc = {.fd_desc = {.fip_dtype = FIP_DT_MAC, .fip_dlen = 2}},
214 		.name_desc = {.fd_desc = {.fip_dtype = FIP_DT_NAME, .fip_dlen = 3}},
215 		.fcoe_desc = {.fd_desc = {.fip_dtype = FIP_DT_FCOE_SIZE, .fip_dlen = 1},
216 			      .fd_size = cpu_to_be16(FCOE_MAX_SIZE)}
217 	};
218 
219 	memcpy(pdisc_sol->eth.h_source, iport->hwmac, ETH_ALEN);
220 	memcpy(pdisc_sol->mac_desc.fd_mac, iport->hwmac, ETH_ALEN);
221 	iport->selected_fcf.fcf_priority = 0xFF;
222 
223 	FNIC_STD_SET_NODE_NAME(&pdisc_sol->name_desc.fd_wwn, iport->wwnn);
224 
225 	FNIC_FIP_DBG(KERN_INFO, fnic->host, fnic->fnic_num,
226 		     "Start FCF discovery\n");
227 	fnic_send_fip_frame(iport, frame, frame_size);
228 
229 	iport->fip.state = FDLS_FIP_FCF_DISCOVERY_STARTED;
230 
231 	fcs_tov = jiffies + msecs_to_jiffies(FCOE_CTLR_FCS_TOV);
232 	mod_timer(&fnic->retry_fip_timer, round_jiffies(fcs_tov));
233 }
234 
235 /**
236  * fnic_fcoe_fip_discovery_resp - Processes FCF advertisements.
237  * @fnic: Handle to fnic driver instance
238  * @fiph: Received frame
239  *
240  * FCF advertisements can be:
241  * solicited - Sent in response of a discover FCF FIP request
242  * Store the information of the FCF with highest priority.
243  * Wait until timeout in case of multiple FCFs.
244  *
245  * unsolicited - Sent periodically by the FCF for keep alive.
246  * If FLOGI is in progress or completed and the advertisement is
247  * received by our selected FCF, refresh the keep alive timer.
248  */
fnic_fcoe_fip_discovery_resp(struct fnic * fnic,struct fip_header * fiph)249 void fnic_fcoe_fip_discovery_resp(struct fnic *fnic, struct fip_header *fiph)
250 {
251 	struct fnic_iport_s *iport = &fnic->iport;
252 	struct fip_disc_adv *disc_adv = (struct fip_disc_adv *)fiph;
253 	u64 fcs_ka_tov;
254 	u64 tov;
255 	int fka_has_changed;
256 
257 	switch (iport->fip.state) {
258 	case FDLS_FIP_FCF_DISCOVERY_STARTED:
259 		if (be16_to_cpu(disc_adv->fip.fip_flags) & FIP_FL_SOL) {
260 			FNIC_FIP_DBG(KERN_INFO, fnic->host,
261 				     fnic->fnic_num,
262 				     "fnic 0x%p Solicited adv\n", fnic);
263 
264 			if ((disc_adv->prio_desc.fd_pri <
265 			     iport->selected_fcf.fcf_priority)
266 			    && (be16_to_cpu(disc_adv->fip.fip_flags) & FIP_FL_AVAIL)) {
267 
268 				FNIC_FIP_DBG(KERN_INFO, fnic->host,
269 					     fnic->fnic_num,
270 					     "fnic 0x%p FCF Available\n", fnic);
271 				memcpy(iport->selected_fcf.fcf_mac,
272 				       disc_adv->mac_desc.fd_mac, ETH_ALEN);
273 				iport->selected_fcf.fcf_priority =
274 				    disc_adv->prio_desc.fd_pri;
275 				iport->selected_fcf.fka_adv_period =
276 				    be32_to_cpu(disc_adv->fka_adv_desc.fd_fka_period);
277 				FNIC_FIP_DBG(KERN_INFO, fnic->host,
278 					     fnic->fnic_num, "adv time %d",
279 					     iport->selected_fcf.fka_adv_period);
280 				iport->selected_fcf.ka_disabled =
281 				    (disc_adv->fka_adv_desc.fd_flags & 1);
282 			}
283 		}
284 		break;
285 	case FDLS_FIP_FLOGI_STARTED:
286 	case FDLS_FIP_FLOGI_COMPLETE:
287 		if (!(be16_to_cpu(disc_adv->fip.fip_flags) & FIP_FL_SOL)) {
288 			/* same fcf */
289 			if (memcmp
290 			    (iport->selected_fcf.fcf_mac,
291 			     disc_adv->mac_desc.fd_mac, ETH_ALEN) == 0) {
292 				if (iport->selected_fcf.fka_adv_period !=
293 				    be32_to_cpu(disc_adv->fka_adv_desc.fd_fka_period)) {
294 					iport->selected_fcf.fka_adv_period =
295 					    be32_to_cpu(disc_adv->fka_adv_desc.fd_fka_period);
296 					FNIC_FIP_DBG(KERN_INFO,
297 						     fnic->host,
298 						     fnic->fnic_num,
299 						     "change fka to %d",
300 						     iport->selected_fcf.fka_adv_period);
301 				}
302 
303 				fka_has_changed =
304 				    (iport->selected_fcf.ka_disabled == 1)
305 				    && ((disc_adv->fka_adv_desc.fd_flags & 1) ==
306 					0);
307 
308 				iport->selected_fcf.ka_disabled =
309 				    (disc_adv->fka_adv_desc.fd_flags & 1);
310 				if (!((iport->selected_fcf.ka_disabled)
311 				      || (iport->selected_fcf.fka_adv_period ==
312 					  0))) {
313 
314 					fcs_ka_tov = jiffies
315 					    + 3
316 					    *
317 					    msecs_to_jiffies(iport->selected_fcf.fka_adv_period);
318 					mod_timer(&fnic->fcs_ka_timer,
319 						  round_jiffies(fcs_ka_tov));
320 				} else {
321 					if (timer_pending(&fnic->fcs_ka_timer))
322 						del_timer_sync(&fnic->fcs_ka_timer);
323 				}
324 
325 				if (fka_has_changed) {
326 					if (iport->selected_fcf.fka_adv_period != 0) {
327 						tov =
328 						 jiffies +
329 						 msecs_to_jiffies(
330 							 iport->selected_fcf.fka_adv_period);
331 						mod_timer(&fnic->enode_ka_timer,
332 							  round_jiffies(tov));
333 
334 						tov =
335 						    jiffies +
336 						    msecs_to_jiffies
337 						    (FIP_VN_KA_PERIOD);
338 						mod_timer(&fnic->vn_ka_timer,
339 							  round_jiffies(tov));
340 					}
341 				}
342 			}
343 		}
344 		break;
345 	default:
346 		break;
347 	}			/* end switch */
348 }
349 
350 /**
351  * fnic_fcoe_start_flogi - Send FIP FLOGI to the selected FCF
352  * @fnic: Handle to fnic driver instance
353  */
fnic_fcoe_start_flogi(struct fnic * fnic)354 void fnic_fcoe_start_flogi(struct fnic *fnic)
355 {
356 	uint8_t *frame;
357 	struct fnic_iport_s *iport = &fnic->iport;
358 	struct fip_flogi *pflogi_req;
359 	u64 flogi_tov;
360 	uint16_t oxid;
361 	uint16_t frame_size = sizeof(struct fip_flogi);
362 
363 	frame = fdls_alloc_frame(iport);
364 	if (frame == NULL) {
365 		FNIC_FIP_DBG(KERN_ERR, fnic->host, fnic->fnic_num,
366 		     "Failed to allocate frame to start FIP FLOGI");
367 		return;
368 	}
369 
370 	pflogi_req = (struct fip_flogi *) frame;
371 	*pflogi_req = (struct fip_flogi) {
372 		.eth = {
373 			.h_proto = cpu_to_be16(ETH_P_FIP)},
374 		.fip = {
375 			.fip_ver = FIP_VER_ENCAPS(FIP_VER),
376 			.fip_op = cpu_to_be16(FIP_OP_LS),
377 			.fip_subcode = FIP_SC_REQ,
378 			.fip_dl_len = cpu_to_be16(FIP_FLOGI_LEN),
379 			.fip_flags = cpu_to_be16(FIP_FL_FPMA)},
380 		.flogi_desc = {
381 				.fd_desc = {.fip_dtype = FIP_DT_FLOGI, .fip_dlen = 36},
382 			       .flogi = {
383 					 .fchdr = {
384 						   .fh_r_ctl = FC_RCTL_ELS_REQ,
385 						   .fh_d_id = {0xFF, 0xFF, 0xFE},
386 						   .fh_type = FC_TYPE_ELS,
387 						   .fh_f_ctl = {FNIC_ELS_REQ_FCTL, 0, 0},
388 						   .fh_rx_id = cpu_to_be16(FNIC_UNASSIGNED_RXID)},
389 					 .els = {
390 						 .fl_cmd = ELS_FLOGI,
391 						 .fl_csp = {
392 							    .sp_hi_ver =
393 							    FNIC_FC_PH_VER_HI,
394 							    .sp_lo_ver =
395 							    FNIC_FC_PH_VER_LO,
396 							    .sp_bb_cred =
397 							    cpu_to_be16
398 							    (FNIC_FC_B2B_CREDIT),
399 							    .sp_bb_data =
400 							    cpu_to_be16
401 							    (FNIC_FC_B2B_RDF_SZ)},
402 						 .fl_cssp[2].cp_class =
403 						 cpu_to_be16(FC_CPC_VALID | FC_CPC_SEQ)
404 						},
405 					}
406 			},
407 		.mac_desc = {.fd_desc = {.fip_dtype = FIP_DT_MAC, .fip_dlen = 2}}
408 	};
409 
410 	memcpy(pflogi_req->eth.h_source, iport->hwmac, ETH_ALEN);
411 	if (iport->usefip)
412 		memcpy(pflogi_req->eth.h_dest, iport->selected_fcf.fcf_mac,
413 		       ETH_ALEN);
414 
415 	oxid = fdls_alloc_oxid(iport, FNIC_FRAME_TYPE_FABRIC_FLOGI,
416 		&iport->active_oxid_fabric_req);
417 	if (oxid == FNIC_UNASSIGNED_OXID) {
418 		FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num,
419 		     "Failed to allocate OXID to send FIP FLOGI");
420 		mempool_free(frame, fnic->frame_pool);
421 		return;
422 	}
423 	FNIC_STD_SET_OX_ID(pflogi_req->flogi_desc.flogi.fchdr, oxid);
424 
425 	FNIC_STD_SET_NPORT_NAME(&pflogi_req->flogi_desc.flogi.els.fl_wwpn,
426 			iport->wwpn);
427 	FNIC_STD_SET_NODE_NAME(&pflogi_req->flogi_desc.flogi.els.fl_wwnn,
428 			iport->wwnn);
429 
430 	FNIC_FIP_DBG(KERN_INFO, fnic->host, fnic->fnic_num,
431 		     "FIP start FLOGI\n");
432 	fnic_send_fip_frame(iport, frame, frame_size);
433 	iport->fip.flogi_retry++;
434 
435 	iport->fip.state = FDLS_FIP_FLOGI_STARTED;
436 	flogi_tov = jiffies + msecs_to_jiffies(fnic->config.flogi_timeout);
437 	mod_timer(&fnic->retry_fip_timer, round_jiffies(flogi_tov));
438 }
439 
440 /**
441  * fnic_fcoe_process_flogi_resp - Processes FLOGI response from FCF.
442  * @fnic: Handle to fnic driver instance
443  * @fiph: Received frame
444  *
445  * If successful save assigned fc_id and MAC, program firmware
446  * and start fdls discovery, else restart vlan discovery.
447  */
fnic_fcoe_process_flogi_resp(struct fnic * fnic,struct fip_header * fiph)448 void fnic_fcoe_process_flogi_resp(struct fnic *fnic, struct fip_header *fiph)
449 {
450 	struct fnic_iport_s *iport = &fnic->iport;
451 	struct fip_flogi_rsp *flogi_rsp = (struct fip_flogi_rsp *)fiph;
452 	int desc_len;
453 	uint32_t s_id;
454 	int frame_type;
455 	uint16_t oxid;
456 
457 	struct fnic_stats *fnic_stats = &fnic->fnic_stats;
458 	struct fc_frame_header *fchdr = &flogi_rsp->rsp_desc.flogi.fchdr;
459 
460 	FNIC_FIP_DBG(KERN_INFO, fnic->host, fnic->fnic_num,
461 		     "fnic 0x%p FIP FLOGI rsp\n", fnic);
462 	desc_len = be16_to_cpu(flogi_rsp->fip.fip_dl_len);
463 	if (desc_len != 38) {
464 		FNIC_FIP_DBG(KERN_INFO, fnic->host, fnic->fnic_num,
465 			     "Invalid Descriptor List len (%x). Dropping frame\n",
466 			     desc_len);
467 		return;
468 	}
469 
470 	if (!((flogi_rsp->rsp_desc.fd_desc.fip_dtype == 7)
471 	      && (flogi_rsp->rsp_desc.fd_desc.fip_dlen == 36))
472 	    || !((flogi_rsp->mac_desc.fd_desc.fip_dtype == 2)
473 		 && (flogi_rsp->mac_desc.fd_desc.fip_dlen == 2))) {
474 		FNIC_FIP_DBG(KERN_INFO, fnic->host, fnic->fnic_num,
475 			     "Dropping frame invalid type and len mix\n");
476 		return;
477 	}
478 
479 	frame_type = fnic_fdls_validate_and_get_frame_type(iport, fchdr);
480 
481 	s_id = ntoh24(fchdr->fh_s_id);
482 	if ((fchdr->fh_f_ctl[0] != 0x98)
483 	    || (fchdr->fh_r_ctl != 0x23)
484 	    || (s_id != FC_FID_FLOGI)
485 	    || (frame_type != FNIC_FABRIC_FLOGI_RSP)
486 	    || (fchdr->fh_type != 0x01)) {
487 		FNIC_FIP_DBG(KERN_INFO, fnic->host, fnic->fnic_num,
488 			     "Dropping invalid frame: s_id %x F %x R %x t %x OX_ID %x\n",
489 			     s_id, fchdr->fh_f_ctl[0], fchdr->fh_r_ctl,
490 			     fchdr->fh_type, FNIC_STD_GET_OX_ID(fchdr));
491 		return;
492 	}
493 
494 	if (iport->fip.state == FDLS_FIP_FLOGI_STARTED) {
495 		FNIC_FIP_DBG(KERN_INFO, fnic->host, fnic->fnic_num,
496 			     "fnic 0x%p rsp for pending FLOGI\n", fnic);
497 
498 		oxid = FNIC_STD_GET_OX_ID(fchdr);
499 		fdls_free_oxid(iport, oxid, &iport->active_oxid_fabric_req);
500 		del_timer_sync(&fnic->retry_fip_timer);
501 
502 		if ((be16_to_cpu(flogi_rsp->fip.fip_dl_len) == FIP_FLOGI_LEN)
503 		    && (flogi_rsp->rsp_desc.flogi.els.fl_cmd == ELS_LS_ACC)) {
504 
505 			FNIC_FIP_DBG(KERN_INFO, fnic->host,
506 				     fnic->fnic_num,
507 				     "fnic 0x%p FLOGI success\n", fnic);
508 			memcpy(iport->fpma, flogi_rsp->mac_desc.fd_mac, ETH_ALEN);
509 			iport->fcid =
510 			    ntoh24(flogi_rsp->rsp_desc.flogi.fchdr.fh_d_id);
511 
512 			iport->r_a_tov =
513 			    be32_to_cpu(flogi_rsp->rsp_desc.flogi.els.fl_csp.sp_r_a_tov);
514 			iport->e_d_tov =
515 			    be32_to_cpu(flogi_rsp->rsp_desc.flogi.els.fl_csp.sp_e_d_tov);
516 			memcpy(fnic->iport.fcfmac, iport->selected_fcf.fcf_mac,
517 			       ETH_ALEN);
518 			vnic_dev_add_addr(fnic->vdev, flogi_rsp->mac_desc.fd_mac);
519 
520 			if (fnic_fdls_register_portid(iport, iport->fcid, NULL)
521 			    != 0) {
522 				FNIC_FIP_DBG(KERN_INFO, fnic->host,
523 					     fnic->fnic_num,
524 					     "fnic 0x%p flogi registration failed\n",
525 					     fnic);
526 				return;
527 			}
528 
529 			iport->fip.state = FDLS_FIP_FLOGI_COMPLETE;
530 			iport->state = FNIC_IPORT_STATE_FABRIC_DISC;
531 			FNIC_FIP_DBG(KERN_INFO, fnic->host,
532 				     fnic->fnic_num, "iport->state:%d\n",
533 				     iport->state);
534 			fnic_fdls_disc_start(iport);
535 			if (!((iport->selected_fcf.ka_disabled)
536 			      || (iport->selected_fcf.fka_adv_period == 0))) {
537 				u64 tov;
538 
539 				tov = jiffies
540 				    +
541 				    msecs_to_jiffies(iport->selected_fcf.fka_adv_period);
542 				mod_timer(&fnic->enode_ka_timer,
543 					  round_jiffies(tov));
544 
545 				tov =
546 				    jiffies +
547 				    msecs_to_jiffies(FIP_VN_KA_PERIOD);
548 				mod_timer(&fnic->vn_ka_timer,
549 					  round_jiffies(tov));
550 
551 			}
552 		} else {
553 			/*
554 			 * If there's FLOGI rejects - clear all
555 			 * fcf's & restart from scratch
556 			 */
557 			atomic64_inc(&fnic_stats->vlan_stats.flogi_rejects);
558 			/* start FCoE VLAN discovery */
559 			fnic_fcoe_send_vlan_req(fnic);
560 
561 			iport->fip.state = FDLS_FIP_VLAN_DISCOVERY_STARTED;
562 		}
563 	}
564 }
565 
566 /**
567  * fnic_common_fip_cleanup - Clean up FCF info and timers in case of
568  * link down/CVL
569  * @fnic: Handle to fnic driver instance
570  */
fnic_common_fip_cleanup(struct fnic * fnic)571 void fnic_common_fip_cleanup(struct fnic *fnic)
572 {
573 
574 	struct fnic_iport_s *iport = &fnic->iport;
575 
576 	if (!iport->usefip)
577 		return;
578 	FNIC_FIP_DBG(KERN_INFO, fnic->host, fnic->fnic_num,
579 		     "fnic 0x%p fip cleanup\n", fnic);
580 
581 	iport->fip.state = FDLS_FIP_INIT;
582 
583 	del_timer_sync(&fnic->retry_fip_timer);
584 	del_timer_sync(&fnic->fcs_ka_timer);
585 	del_timer_sync(&fnic->enode_ka_timer);
586 	del_timer_sync(&fnic->vn_ka_timer);
587 
588 	if (!is_zero_ether_addr(iport->fpma))
589 		vnic_dev_del_addr(fnic->vdev, iport->fpma);
590 
591 	memset(iport->fpma, 0, ETH_ALEN);
592 	iport->fcid = 0;
593 	iport->r_a_tov = 0;
594 	iport->e_d_tov = 0;
595 	memset(fnic->iport.fcfmac, 0, ETH_ALEN);
596 	memset(iport->selected_fcf.fcf_mac, 0, ETH_ALEN);
597 	iport->selected_fcf.fcf_priority = 0;
598 	iport->selected_fcf.fka_adv_period = 0;
599 	iport->selected_fcf.ka_disabled = 0;
600 
601 	fnic_fcoe_reset_vlans(fnic);
602 }
603 
604 /**
605  * fnic_fcoe_process_cvl - Processes Clear Virtual Link from FCF.
606  * @fnic: Handle to fnic driver instance
607  * @fiph: Received frame
608  *
609  * Verify that cvl is received from our current FCF for our assigned MAC
610  * and clean up and restart the vlan discovery.
611  */
fnic_fcoe_process_cvl(struct fnic * fnic,struct fip_header * fiph)612 void fnic_fcoe_process_cvl(struct fnic *fnic, struct fip_header *fiph)
613 {
614 	struct fnic_iport_s *iport = &fnic->iport;
615 	struct fip_cvl *cvl_msg = (struct fip_cvl *)fiph;
616 	int i;
617 	int found = false;
618 	int max_count = 0;
619 
620 	FNIC_FIP_DBG(KERN_INFO, fnic->host, fnic->fnic_num,
621 		     "fnic 0x%p clear virtual link handler\n", fnic);
622 
623 	if (!((cvl_msg->fcf_mac_desc.fd_desc.fip_dtype == 2)
624 	      && (cvl_msg->fcf_mac_desc.fd_desc.fip_dlen == 2))
625 	    || !((cvl_msg->name_desc.fd_desc.fip_dtype == 4)
626 		 && (cvl_msg->name_desc.fd_desc.fip_dlen == 3))) {
627 
628 		FNIC_FIP_DBG(KERN_INFO, fnic->host, fnic->fnic_num,
629 			     "invalid mix: ft %x fl %x ndt %x ndl %x",
630 			     cvl_msg->fcf_mac_desc.fd_desc.fip_dtype,
631 			     cvl_msg->fcf_mac_desc.fd_desc.fip_dlen,
632 				 cvl_msg->name_desc.fd_desc.fip_dtype,
633 			     cvl_msg->name_desc.fd_desc.fip_dlen);
634 	}
635 
636 	if (memcmp
637 	    (iport->selected_fcf.fcf_mac, cvl_msg->fcf_mac_desc.fd_mac, ETH_ALEN)
638 	    == 0) {
639 		for (i = 0; i < ((be16_to_cpu(fiph->fip_dl_len) / 5) - 1); i++) {
640 			if (!((cvl_msg->vn_ports_desc[i].fd_desc.fip_dtype == 11)
641 			      && (cvl_msg->vn_ports_desc[i].fd_desc.fip_dlen == 5))) {
642 
643 				FNIC_FIP_DBG(KERN_INFO, fnic->host,
644 					     fnic->fnic_num,
645 					     "Invalid type and len mix type: %d len: %d\n",
646 					     cvl_msg->vn_ports_desc[i].fd_desc.fip_dtype,
647 					     cvl_msg->vn_ports_desc[i].fd_desc.fip_dlen);
648 			}
649 			if (memcmp
650 			    (iport->fpma, cvl_msg->vn_ports_desc[i].fd_mac,
651 			     ETH_ALEN) == 0) {
652 				found = true;
653 				break;
654 			}
655 		}
656 		if (!found)
657 			return;
658 		fnic_common_fip_cleanup(fnic);
659 
660 		while (fnic->reset_in_progress == IN_PROGRESS) {
661 			spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags);
662 			wait_for_completion_timeout(&fnic->reset_completion_wait,
663 							msecs_to_jiffies(5000));
664 			spin_lock_irqsave(&fnic->fnic_lock, fnic->lock_flags);
665 			max_count++;
666 			if (max_count >= FIP_FNIC_RESET_WAIT_COUNT) {
667 				FNIC_FIP_DBG(KERN_INFO, fnic->host, fnic->fnic_num,
668 					 "Rthr waited too long. Skipping handle link event %p\n",
669 					 fnic);
670 				return;
671 			}
672 			FNIC_FIP_DBG(KERN_INFO, fnic->host, fnic->fnic_num,
673 				 "fnic reset in progress. Link event needs to wait %p",
674 				 fnic);
675 		}
676 		fnic->reset_in_progress = IN_PROGRESS;
677 		fnic_fdls_link_down(iport);
678 		fnic->reset_in_progress = NOT_IN_PROGRESS;
679 		complete(&fnic->reset_completion_wait);
680 		fnic_fcoe_send_vlan_req(fnic);
681 	}
682 }
683 
684 /**
685  * fdls_fip_recv_frame - Demultiplexer for FIP frames
686  * @fnic: Handle to fnic driver instance
687  * @frame: Received ethernet frame
688  */
fdls_fip_recv_frame(struct fnic * fnic,void * frame)689 int fdls_fip_recv_frame(struct fnic *fnic, void *frame)
690 {
691 	struct ethhdr *eth = (struct ethhdr *)frame;
692 	struct fip_header *fiph;
693 	u16 op;
694 	u8 sub;
695 	int len = 2048;
696 
697 	if (be16_to_cpu(eth->h_proto) == ETH_P_FIP) {
698 		fiph = (struct fip_header *)(eth + 1);
699 		op = be16_to_cpu(fiph->fip_op);
700 		sub = fiph->fip_subcode;
701 
702 		fnic_debug_dump_fip_frame(fnic, eth, len, "Incoming");
703 
704 		if (op == FIP_OP_DISC && sub == FIP_SC_REP)
705 			fnic_fcoe_fip_discovery_resp(fnic, fiph);
706 		else if (op == FIP_OP_VLAN && sub == FIP_SC_REP)
707 			fnic_fcoe_process_vlan_resp(fnic, fiph);
708 		else if (op == FIP_OP_CTRL && sub == FIP_SC_REP)
709 			fnic_fcoe_process_cvl(fnic, fiph);
710 		else if (op == FIP_OP_LS && sub == FIP_SC_REP)
711 			fnic_fcoe_process_flogi_resp(fnic, fiph);
712 
713 		/* Return true if the frame was a FIP frame */
714 		return true;
715 	}
716 
717 	FNIC_FCS_DBG(KERN_INFO, fnic->host, fnic->fnic_num,
718 		"Not a FIP Frame");
719 	return false;
720 }
721 
fnic_work_on_fip_timer(struct work_struct * work)722 void fnic_work_on_fip_timer(struct work_struct *work)
723 {
724 	struct fnic *fnic = container_of(work, struct fnic, fip_timer_work);
725 	struct fnic_iport_s *iport = &fnic->iport;
726 
727 	FNIC_FIP_DBG(KERN_INFO, fnic->host, fnic->fnic_num,
728 		     "FIP timeout\n");
729 
730 	if (iport->fip.state == FDLS_FIP_VLAN_DISCOVERY_STARTED) {
731 		fnic_vlan_discovery_timeout(fnic);
732 	} else if (iport->fip.state == FDLS_FIP_FCF_DISCOVERY_STARTED) {
733 		u8 zmac[ETH_ALEN] = { 0, 0, 0, 0, 0, 0 };
734 
735 		FNIC_FIP_DBG(KERN_INFO, fnic->host, fnic->fnic_num,
736 			     "FCF Discovery timeout\n");
737 		if (memcmp(iport->selected_fcf.fcf_mac, zmac, ETH_ALEN) != 0) {
738 
739 			if (iport->flags & FNIC_FIRST_LINK_UP) {
740 				fnic_scsi_fcpio_reset(iport->fnic);
741 				iport->flags &= ~FNIC_FIRST_LINK_UP;
742 			}
743 
744 			fnic_fcoe_start_flogi(fnic);
745 			if (!((iport->selected_fcf.ka_disabled)
746 			      || (iport->selected_fcf.fka_adv_period == 0))) {
747 				u64 fcf_tov;
748 
749 				fcf_tov = jiffies
750 				    + 3
751 				    *
752 				    msecs_to_jiffies(iport->selected_fcf.fka_adv_period);
753 				mod_timer(&fnic->fcs_ka_timer,
754 					  round_jiffies(fcf_tov));
755 			}
756 		} else {
757 			FNIC_FIP_DBG(KERN_INFO, fnic->host,
758 				     fnic->fnic_num, "FCF Discovery timeout\n");
759 			fnic_vlan_discovery_timeout(fnic);
760 		}
761 	} else if (iport->fip.state == FDLS_FIP_FLOGI_STARTED) {
762 		fdls_schedule_oxid_free(iport, &iport->active_oxid_fabric_req);
763 		FNIC_FIP_DBG(KERN_INFO, fnic->host, fnic->fnic_num,
764 			     "FLOGI timeout\n");
765 		if (iport->fip.flogi_retry < fnic->config.flogi_retries)
766 			fnic_fcoe_start_flogi(fnic);
767 		else
768 			fnic_vlan_discovery_timeout(fnic);
769 	}
770 }
771 
772 /**
773  * fnic_handle_fip_timer - Timeout handler for FIP discover phase.
774  * @t: Handle to the timer list
775  *
776  * Based on the current state, start next phase or restart discovery.
777  */
fnic_handle_fip_timer(struct timer_list * t)778 void fnic_handle_fip_timer(struct timer_list *t)
779 {
780 	struct fnic *fnic = from_timer(fnic, t, retry_fip_timer);
781 
782 	INIT_WORK(&fnic->fip_timer_work, fnic_work_on_fip_timer);
783 	queue_work(fnic_fip_queue, &fnic->fip_timer_work);
784 }
785 
786 /**
787  * fnic_handle_enode_ka_timer - FIP node keep alive.
788  * @t: Handle to the timer list
789  */
fnic_handle_enode_ka_timer(struct timer_list * t)790 void fnic_handle_enode_ka_timer(struct timer_list *t)
791 {
792 	uint8_t *frame;
793 	struct fnic *fnic = from_timer(fnic, t, enode_ka_timer);
794 
795 	struct fnic_iport_s *iport = &fnic->iport;
796 	struct fip_enode_ka *penode_ka;
797 	u64 enode_ka_tov;
798 	uint16_t frame_size = sizeof(struct fip_enode_ka);
799 
800 	if (iport->fip.state != FDLS_FIP_FLOGI_COMPLETE)
801 		return;
802 
803 	if ((iport->selected_fcf.ka_disabled)
804 	    || (iport->selected_fcf.fka_adv_period == 0)) {
805 		return;
806 	}
807 
808 	frame = fdls_alloc_frame(iport);
809 	if (frame == NULL) {
810 		FNIC_FIP_DBG(KERN_ERR, fnic->host, fnic->fnic_num,
811 		     "Failed to allocate frame to send enode ka");
812 		return;
813 	}
814 
815 	penode_ka = (struct fip_enode_ka *) frame;
816 	*penode_ka = (struct fip_enode_ka) {
817 		.eth = {
818 			.h_proto = cpu_to_be16(ETH_P_FIP)},
819 		.fip = {
820 			.fip_ver = FIP_VER_ENCAPS(FIP_VER),
821 			.fip_op = cpu_to_be16(FIP_OP_CTRL),
822 			.fip_subcode = FIP_SC_REQ,
823 			.fip_dl_len = cpu_to_be16(FIP_ENODE_KA_LEN)},
824 		.mac_desc = {.fd_desc = {.fip_dtype = FIP_DT_MAC, .fip_dlen = 2}}
825 	};
826 
827 	memcpy(penode_ka->eth.h_source, iport->hwmac, ETH_ALEN);
828 	memcpy(penode_ka->eth.h_dest, iport->selected_fcf.fcf_mac, ETH_ALEN);
829 	memcpy(penode_ka->mac_desc.fd_mac, iport->hwmac, ETH_ALEN);
830 
831 	FNIC_FIP_DBG(KERN_DEBUG, fnic->host, fnic->fnic_num,
832 		     "Handle enode KA timer\n");
833 	fnic_send_fip_frame(iport, frame, frame_size);
834 	enode_ka_tov = jiffies
835 	    + msecs_to_jiffies(iport->selected_fcf.fka_adv_period);
836 	mod_timer(&fnic->enode_ka_timer, round_jiffies(enode_ka_tov));
837 }
838 
839 /**
840  * fnic_handle_vn_ka_timer - FIP virtual port keep alive.
841  * @t: Handle to the timer list
842  */
fnic_handle_vn_ka_timer(struct timer_list * t)843 void fnic_handle_vn_ka_timer(struct timer_list *t)
844 {
845 	uint8_t *frame;
846 	struct fnic *fnic = from_timer(fnic, t, vn_ka_timer);
847 
848 	struct fnic_iport_s *iport = &fnic->iport;
849 	struct fip_vn_port_ka *pvn_port_ka;
850 	u64 vn_ka_tov;
851 	uint8_t fcid[3];
852 	uint16_t frame_size = sizeof(struct fip_vn_port_ka);
853 
854 	if (iport->fip.state != FDLS_FIP_FLOGI_COMPLETE)
855 		return;
856 
857 	if ((iport->selected_fcf.ka_disabled)
858 	    || (iport->selected_fcf.fka_adv_period == 0)) {
859 		return;
860 	}
861 
862 	frame = fdls_alloc_frame(iport);
863 	if (frame == NULL) {
864 		FNIC_FIP_DBG(KERN_ERR, fnic->host, fnic->fnic_num,
865 		     "Failed to allocate frame to send vn ka");
866 		return;
867 	}
868 
869 	pvn_port_ka = (struct fip_vn_port_ka *) frame;
870 	*pvn_port_ka = (struct fip_vn_port_ka) {
871 		.eth = {
872 			.h_proto = cpu_to_be16(ETH_P_FIP)},
873 		.fip = {
874 			.fip_ver = FIP_VER_ENCAPS(FIP_VER),
875 			.fip_op = cpu_to_be16(FIP_OP_CTRL),
876 			.fip_subcode = FIP_SC_REQ,
877 			.fip_dl_len = cpu_to_be16(FIP_VN_KA_LEN)},
878 		.mac_desc = {.fd_desc = {.fip_dtype = FIP_DT_MAC, .fip_dlen = 2}},
879 		.vn_port_desc = {.fd_desc = {.fip_dtype = FIP_DT_VN_ID, .fip_dlen = 5}}
880 	};
881 
882 	memcpy(pvn_port_ka->eth.h_source, iport->fpma, ETH_ALEN);
883 	memcpy(pvn_port_ka->eth.h_dest, iport->selected_fcf.fcf_mac, ETH_ALEN);
884 	memcpy(pvn_port_ka->mac_desc.fd_mac, iport->hwmac, ETH_ALEN);
885 	memcpy(pvn_port_ka->vn_port_desc.fd_mac, iport->fpma, ETH_ALEN);
886 	hton24(fcid, iport->fcid);
887 	memcpy(pvn_port_ka->vn_port_desc.fd_fc_id, fcid, 3);
888 	FNIC_STD_SET_NPORT_NAME(&pvn_port_ka->vn_port_desc.fd_wwpn, iport->wwpn);
889 
890 	FNIC_FIP_DBG(KERN_DEBUG, fnic->host, fnic->fnic_num,
891 		     "Handle vnport KA timer\n");
892 	fnic_send_fip_frame(iport, frame, frame_size);
893 	vn_ka_tov = jiffies + msecs_to_jiffies(FIP_VN_KA_PERIOD);
894 	mod_timer(&fnic->vn_ka_timer, round_jiffies(vn_ka_tov));
895 }
896 
897 /**
898  * fnic_vlan_discovery_timeout - Handle vlan discovery timeout
899  * @fnic: Handle to fnic driver instance
900  *
901  * End of VLAN discovery or FCF discovery time window.
902  * Start the FCF discovery if VLAN was never used.
903  */
fnic_vlan_discovery_timeout(struct fnic * fnic)904 void fnic_vlan_discovery_timeout(struct fnic *fnic)
905 {
906 	struct fcoe_vlan *vlan;
907 	struct fnic_iport_s *iport = &fnic->iport;
908 	struct fnic_stats *fnic_stats = &fnic->fnic_stats;
909 	unsigned long flags;
910 
911 	spin_lock_irqsave(&fnic->fnic_lock, flags);
912 	if (fnic->stop_rx_link_events) {
913 		spin_unlock_irqrestore(&fnic->fnic_lock, flags);
914 		return;
915 	}
916 	spin_unlock_irqrestore(&fnic->fnic_lock, flags);
917 
918 	if (!iport->usefip)
919 		return;
920 
921 	spin_lock_irqsave(&fnic->vlans_lock, flags);
922 	if (list_empty(&fnic->vlan_list)) {
923 		/* no vlans available, try again */
924 		spin_unlock_irqrestore(&fnic->vlans_lock, flags);
925 		fnic_fcoe_send_vlan_req(fnic);
926 		return;
927 	}
928 
929 	vlan = list_first_entry(&fnic->vlan_list, struct fcoe_vlan, list);
930 
931 	if (vlan->state == FIP_VLAN_SENT) {
932 		if (vlan->sol_count >= FCOE_CTLR_MAX_SOL) {
933 			/*
934 			 * no response on this vlan, remove  from the list.
935 			 * Try the next vlan
936 			 */
937 			list_del(&vlan->list);
938 			kfree(vlan);
939 			vlan = NULL;
940 			if (list_empty(&fnic->vlan_list)) {
941 				/* we exhausted all vlans, restart vlan disc */
942 				spin_unlock_irqrestore(&fnic->vlans_lock,
943 						       flags);
944 				fnic_fcoe_send_vlan_req(fnic);
945 				return;
946 			}
947 			/* check the next vlan */
948 			vlan =
949 			    list_first_entry(&fnic->vlan_list, struct fcoe_vlan,
950 					     list);
951 
952 			fnic->set_vlan(fnic, vlan->vid);
953 			vlan->state = FIP_VLAN_SENT;	/* sent now */
954 
955 		}
956 		atomic64_inc(&fnic_stats->vlan_stats.sol_expiry_count);
957 
958 	} else {
959 		fnic->set_vlan(fnic, vlan->vid);
960 		vlan->state = FIP_VLAN_SENT;	/* sent now */
961 	}
962 	vlan->sol_count++;
963 	spin_unlock_irqrestore(&fnic->vlans_lock, flags);
964 	fnic_fcoe_start_fcf_discovery(fnic);
965 }
966 
967 /**
968  * fnic_work_on_fcs_ka_timer - Handle work on FCS keep alive timer.
969  * @work: the work queue to be serviced
970  *
971  * Finish handling fcs_ka_timer in process context.
972  * Clean up, bring the link down, and restart all FIP discovery.
973  */
fnic_work_on_fcs_ka_timer(struct work_struct * work)974 void fnic_work_on_fcs_ka_timer(struct work_struct *work)
975 {
976 	struct fnic
977 	*fnic = container_of(work, struct fnic, fip_timer_work);
978 	struct fnic_iport_s *iport = &fnic->iport;
979 
980 	FNIC_FIP_DBG(KERN_INFO, fnic->host, fnic->fnic_num,
981 		     "fnic 0x%p fcs ka timeout\n", fnic);
982 
983 	fnic_common_fip_cleanup(fnic);
984 	spin_lock_irqsave(&fnic->fnic_lock, fnic->lock_flags);
985 	fnic_fdls_link_down(iport);
986 	iport->state = FNIC_IPORT_STATE_FIP;
987 	spin_unlock_irqrestore(&fnic->fnic_lock, fnic->lock_flags);
988 
989 	fnic_fcoe_send_vlan_req(fnic);
990 }
991 
992 /**
993  * fnic_handle_fcs_ka_timer - Handle FCS keep alive timer.
994  * @t: Handle to the timer list
995  *
996  * No keep alives received from FCF. Clean up, bring the link down
997  * and restart all the FIP discovery.
998  */
fnic_handle_fcs_ka_timer(struct timer_list * t)999 void fnic_handle_fcs_ka_timer(struct timer_list *t)
1000 {
1001 	struct fnic *fnic = from_timer(fnic, t, fcs_ka_timer);
1002 
1003 	INIT_WORK(&fnic->fip_timer_work, fnic_work_on_fcs_ka_timer);
1004 	queue_work(fnic_fip_queue, &fnic->fip_timer_work);
1005 }
1006