Lines Matching +full:sml +full:- +full:size

1 // SPDX-License-Identifier: GPL-2.0
3 * driver for Microchip PQI-based storage controllers
4 * Copyright (c) 2019-2023 Microchip Technology Inc. and its subsidiaries
5 * Copyright (c) 2016-2018 Microsemi Corporation
6 * Copyright (c) 2016 PMC-Sierra, Inc.
35 #define DRIVER_VERSION "2.1.30-031"
50 #define PQI_NO_COMPLETION ((void *)-1)
193 "RAID-0",
194 "RAID-4",
195 "RAID-1(1+0)",
196 "RAID-5",
197 "RAID-5+1",
198 "RAID-6",
199 "RAID-1(Triple)",
228 sdev->no_write_same = 1; in pqi_disable_write_same()
238 return !device->is_physical_device; in pqi_is_logical_device()
248 return !ctrl_info->controller_online; in pqi_ctrl_offline()
253 if (ctrl_info->controller_online) in pqi_check_ctrl_health()
307 ctrl_info->scan_blocked = true; in pqi_ctrl_block_scan()
308 mutex_lock(&ctrl_info->scan_mutex); in pqi_ctrl_block_scan()
313 ctrl_info->scan_blocked = false; in pqi_ctrl_unblock_scan()
314 mutex_unlock(&ctrl_info->scan_mutex); in pqi_ctrl_unblock_scan()
319 return ctrl_info->scan_blocked; in pqi_ctrl_scan_blocked()
324 mutex_lock(&ctrl_info->lun_reset_mutex); in pqi_ctrl_block_device_reset()
329 mutex_unlock(&ctrl_info->lun_reset_mutex); in pqi_ctrl_unblock_device_reset()
338 shost = ctrl_info->scsi_host; in pqi_scsi_block_requests()
354 scsi_unblock_requests(ctrl_info->scsi_host); in pqi_scsi_unblock_requests()
359 atomic_inc(&ctrl_info->num_busy_threads); in pqi_ctrl_busy()
364 atomic_dec(&ctrl_info->num_busy_threads); in pqi_ctrl_unbusy()
369 return ctrl_info->block_requests; in pqi_ctrl_blocked()
374 ctrl_info->block_requests = true; in pqi_ctrl_block_requests()
379 ctrl_info->block_requests = false; in pqi_ctrl_unblock_requests()
380 wake_up_all(&ctrl_info->block_requests_wait); in pqi_ctrl_unblock_requests()
388 atomic_inc(&ctrl_info->num_blocked_threads); in pqi_wait_if_ctrl_blocked()
389 wait_event(ctrl_info->block_requests_wait, in pqi_wait_if_ctrl_blocked()
391 atomic_dec(&ctrl_info->num_blocked_threads); in pqi_wait_if_ctrl_blocked()
406 while (atomic_read(&ctrl_info->num_busy_threads) > in pqi_ctrl_wait_until_quiesced()
407 atomic_read(&ctrl_info->num_blocked_threads)) { in pqi_ctrl_wait_until_quiesced()
409 dev_warn(&ctrl_info->pci_dev->dev, in pqi_ctrl_wait_until_quiesced()
411 jiffies_to_msecs(jiffies - start_jiffies) / 1000); in pqi_ctrl_wait_until_quiesced()
419 dev_warn(&ctrl_info->pci_dev->dev, in pqi_ctrl_wait_until_quiesced()
421 jiffies_to_msecs(jiffies - start_jiffies) / 1000); in pqi_ctrl_wait_until_quiesced()
426 return device->device_offline; in pqi_device_offline()
431 mutex_lock(&ctrl_info->ofa_mutex); in pqi_ctrl_ofa_start()
436 mutex_unlock(&ctrl_info->ofa_mutex); in pqi_ctrl_ofa_done()
441 mutex_lock(&ctrl_info->ofa_mutex); in pqi_wait_until_ofa_finished()
442 mutex_unlock(&ctrl_info->ofa_mutex); in pqi_wait_until_ofa_finished()
447 return mutex_is_locked(&ctrl_info->ofa_mutex); in pqi_ofa_in_progress()
452 device->in_remove = true; in pqi_device_remove_start()
457 return device->in_remove; in pqi_device_in_remove()
462 device->in_reset[lun] = true; in pqi_device_reset_start()
467 device->in_reset[lun] = false; in pqi_device_reset_done()
472 return device->in_reset[lun]; in pqi_device_in_reset()
483 return -1; in pqi_event_type_to_event_index()
488 return pqi_event_type_to_event_index(event_type) != -1; in pqi_is_supported_event()
497 schedule_delayed_work(&ctrl_info->rescan_work, delay); in pqi_schedule_rescan_worker_with_delay()
514 cancel_delayed_work_sync(&ctrl_info->rescan_work); in pqi_cancel_rescan_worker()
519 if (!ctrl_info->heartbeat_counter) in pqi_read_heartbeat_counter()
522 return readl(ctrl_info->heartbeat_counter); in pqi_read_heartbeat_counter()
527 return readb(ctrl_info->soft_reset_status); in pqi_read_soft_reset_status()
536 writeb(status, ctrl_info->soft_reset_status); in pqi_clear_soft_reset_status()
546 if (device->ncq_prio_enable) { in pqi_is_io_high_priority()
551 switch (scmd->cmnd[0]) { in pqi_is_io_high_priority()
578 bus_address = dma_map_single(&pci_dev->dev, buffer, buffer_length, in pqi_map_single()
580 if (dma_mapping_error(&pci_dev->dev, bus_address)) in pqi_map_single()
581 return -ENOMEM; in pqi_map_single()
583 put_unaligned_le64((u64)bus_address, &sg_descriptor->address); in pqi_map_single()
584 put_unaligned_le32(buffer_length, &sg_descriptor->length); in pqi_map_single()
585 put_unaligned_le32(CISS_SG_LAST, &sg_descriptor->flags); in pqi_map_single()
600 dma_unmap_single(&pci_dev->dev, in pqi_pci_unmap()
616 request->header.iu_type = PQI_REQUEST_IU_RAID_PATH_IO; in pqi_build_raid_path_request()
618 sg_descriptors[1]) - PQI_REQUEST_HEADER_LENGTH, in pqi_build_raid_path_request()
619 &request->header.iu_length); in pqi_build_raid_path_request()
620 put_unaligned_le32(buffer_length, &request->buffer_length); in pqi_build_raid_path_request()
621 memcpy(request->lun_number, scsi3addr, sizeof(request->lun_number)); in pqi_build_raid_path_request()
622 request->task_attribute = SOP_TASK_ATTRIBUTE_SIMPLE; in pqi_build_raid_path_request()
623 request->additional_cdb_bytes_usage = SOP_ADDITIONAL_CDB_BYTES_0; in pqi_build_raid_path_request()
625 cdb = request->cdb; in pqi_build_raid_path_request()
629 request->data_direction = SOP_READ_FLAG; in pqi_build_raid_path_request()
639 request->data_direction = SOP_READ_FLAG; in pqi_build_raid_path_request()
642 if (ctrl_info->rpl_extended_format_4_5_supported) in pqi_build_raid_path_request()
647 cdb[1] = ctrl_info->ciss_report_log_flags; in pqi_build_raid_path_request()
652 request->data_direction = SOP_READ_FLAG; in pqi_build_raid_path_request()
658 request->header.driver_flags = PQI_DRIVER_NONBLOCKABLE_REQUEST; in pqi_build_raid_path_request()
659 request->data_direction = SOP_WRITE_FLAG; in pqi_build_raid_path_request()
671 request->data_direction = SOP_READ_FLAG; in pqi_build_raid_path_request()
680 request->data_direction = SOP_WRITE_FLAG; in pqi_build_raid_path_request()
686 request->data_direction = SOP_BIDIRECTIONAL; in pqi_build_raid_path_request()
693 dev_err(&ctrl_info->pci_dev->dev, "unknown command 0x%c\n", cmd); in pqi_build_raid_path_request()
697 switch (request->data_direction) { in pqi_build_raid_path_request()
712 return pqi_map_single(ctrl_info->pci_dev, &request->sg_descriptors[0], in pqi_build_raid_path_request()
718 io_request->scmd = NULL; in pqi_reinit_io_request()
719 io_request->status = 0; in pqi_reinit_io_request()
720 io_request->error_info = NULL; in pqi_reinit_io_request()
721 io_request->raid_bypass = false; in pqi_reinit_io_request()
729 if (scmd) { /* SML I/O request */ in pqi_alloc_io_request()
733 io_request = &ctrl_info->io_request_pool[i]; in pqi_alloc_io_request()
734 if (atomic_inc_return(&io_request->refcount) > 1) { in pqi_alloc_io_request()
735 atomic_dec(&io_request->refcount); in pqi_alloc_io_request()
740 * benignly racy - may have to wait for an open slot. in pqi_alloc_io_request()
741 * command slot range is scsi_ml_can_queue - in pqi_alloc_io_request()
742 * [scsi_ml_can_queue + (PQI_RESERVED_IO_SLOTS - 1)] in pqi_alloc_io_request()
746 io_request = &ctrl_info->io_request_pool[ctrl_info->scsi_ml_can_queue + i]; in pqi_alloc_io_request()
747 if (atomic_inc_return(&io_request->refcount) == 1) in pqi_alloc_io_request()
749 atomic_dec(&io_request->refcount); in pqi_alloc_io_request()
762 atomic_dec(&io_request->refcount); in pqi_free_io_request()
780 pqi_pci_unmap(ctrl_info->pci_dev, request.sg_descriptors, 1, dir); in pqi_send_scsi_raid_request()
839 bmic_device_index = CISS_GET_DRIVE_NUMBER(device->scsi3addr); in pqi_identify_physical_device()
845 pqi_pci_unmap(ctrl_info->pci_dev, request.sg_descriptors, 1, dir); in pqi_identify_physical_device()
878 max_write_raid_1_10_3drive) - \
890 return -ENOMEM; in pqi_get_advanced_raid_bypass_config()
902 pqi_pci_unmap(ctrl_info->pci_dev, request.sg_descriptors, 1, dir); in pqi_get_advanced_raid_bypass_config()
907 if (buffer->header.page_code != BMIC_SENSE_FEATURE_IO_PAGE || in pqi_get_advanced_raid_bypass_config()
908 buffer->header.subpage_code != in pqi_get_advanced_raid_bypass_config()
910 get_unaligned_le16(&buffer->header.buffer_length) < in pqi_get_advanced_raid_bypass_config()
912 buffer->aio_subpage.header.page_code != in pqi_get_advanced_raid_bypass_config()
914 buffer->aio_subpage.header.subpage_code != in pqi_get_advanced_raid_bypass_config()
916 get_unaligned_le16(&buffer->aio_subpage.header.page_length) < in pqi_get_advanced_raid_bypass_config()
921 ctrl_info->max_transfer_encrypted_sas_sata = in pqi_get_advanced_raid_bypass_config()
923 &buffer->aio_subpage.max_transfer_encrypted_sas_sata); in pqi_get_advanced_raid_bypass_config()
925 ctrl_info->max_transfer_encrypted_nvme = in pqi_get_advanced_raid_bypass_config()
927 &buffer->aio_subpage.max_transfer_encrypted_nvme); in pqi_get_advanced_raid_bypass_config()
929 ctrl_info->max_write_raid_5_6 = in pqi_get_advanced_raid_bypass_config()
931 &buffer->aio_subpage.max_write_raid_5_6); in pqi_get_advanced_raid_bypass_config()
933 ctrl_info->max_write_raid_1_10_2drive = in pqi_get_advanced_raid_bypass_config()
935 &buffer->aio_subpage.max_write_raid_1_10_2drive); in pqi_get_advanced_raid_bypass_config()
937 ctrl_info->max_write_raid_1_10_3drive = in pqi_get_advanced_raid_bypass_config()
939 &buffer->aio_subpage.max_write_raid_1_10_3drive); in pqi_get_advanced_raid_bypass_config()
955 return -ENOMEM; in pqi_flush_cache()
957 flush_cache->shutdown_event = shutdown_event; in pqi_flush_cache()
984 return -ENOMEM; in pqi_set_diag_rescan()
991 diag->options |= cpu_to_le32(PQI_FETCH_PTRAID_DATA); in pqi_set_diag_rescan()
1033 return -ENOMEM; in pqi_write_driver_version_to_host_wellness()
1035 buffer->start_tag[0] = '<'; in pqi_write_driver_version_to_host_wellness()
1036 buffer->start_tag[1] = 'H'; in pqi_write_driver_version_to_host_wellness()
1037 buffer->start_tag[2] = 'W'; in pqi_write_driver_version_to_host_wellness()
1038 buffer->start_tag[3] = '>'; in pqi_write_driver_version_to_host_wellness()
1039 buffer->driver_version_tag[0] = 'D'; in pqi_write_driver_version_to_host_wellness()
1040 buffer->driver_version_tag[1] = 'V'; in pqi_write_driver_version_to_host_wellness()
1041 put_unaligned_le16(sizeof(buffer->driver_version), in pqi_write_driver_version_to_host_wellness()
1042 &buffer->driver_version_length); in pqi_write_driver_version_to_host_wellness()
1043 strscpy(buffer->driver_version, "Linux " DRIVER_VERSION, in pqi_write_driver_version_to_host_wellness()
1044 sizeof(buffer->driver_version)); in pqi_write_driver_version_to_host_wellness()
1045 buffer->dont_write_tag[0] = 'D'; in pqi_write_driver_version_to_host_wellness()
1046 buffer->dont_write_tag[1] = 'W'; in pqi_write_driver_version_to_host_wellness()
1047 buffer->end_tag[0] = 'Z'; in pqi_write_driver_version_to_host_wellness()
1048 buffer->end_tag[1] = 'Z'; in pqi_write_driver_version_to_host_wellness()
1084 return -ENOMEM; in pqi_write_current_time_to_host_wellness()
1086 buffer->start_tag[0] = '<'; in pqi_write_current_time_to_host_wellness()
1087 buffer->start_tag[1] = 'H'; in pqi_write_current_time_to_host_wellness()
1088 buffer->start_tag[2] = 'W'; in pqi_write_current_time_to_host_wellness()
1089 buffer->start_tag[3] = '>'; in pqi_write_current_time_to_host_wellness()
1090 buffer->time_tag[0] = 'T'; in pqi_write_current_time_to_host_wellness()
1091 buffer->time_tag[1] = 'D'; in pqi_write_current_time_to_host_wellness()
1092 put_unaligned_le16(sizeof(buffer->time), in pqi_write_current_time_to_host_wellness()
1093 &buffer->time_length); in pqi_write_current_time_to_host_wellness()
1096 time64_to_tm(local_time, -sys_tz.tz_minuteswest * 60, &tm); in pqi_write_current_time_to_host_wellness()
1099 buffer->time[0] = bin2bcd(tm.tm_hour); in pqi_write_current_time_to_host_wellness()
1100 buffer->time[1] = bin2bcd(tm.tm_min); in pqi_write_current_time_to_host_wellness()
1101 buffer->time[2] = bin2bcd(tm.tm_sec); in pqi_write_current_time_to_host_wellness()
1102 buffer->time[3] = 0; in pqi_write_current_time_to_host_wellness()
1103 buffer->time[4] = bin2bcd(tm.tm_mon + 1); in pqi_write_current_time_to_host_wellness()
1104 buffer->time[5] = bin2bcd(tm.tm_mday); in pqi_write_current_time_to_host_wellness()
1105 buffer->time[6] = bin2bcd(year / 100); in pqi_write_current_time_to_host_wellness()
1106 buffer->time[7] = bin2bcd(year % 100); in pqi_write_current_time_to_host_wellness()
1108 buffer->dont_write_tag[0] = 'D'; in pqi_write_current_time_to_host_wellness()
1109 buffer->dont_write_tag[1] = 'W'; in pqi_write_current_time_to_host_wellness()
1110 buffer->end_tag[0] = 'Z'; in pqi_write_current_time_to_host_wellness()
1111 buffer->end_tag[1] = 'Z'; in pqi_write_current_time_to_host_wellness()
1132 dev_warn(&ctrl_info->pci_dev->dev, in pqi_update_time_worker()
1135 schedule_delayed_work(&ctrl_info->update_time_work, in pqi_update_time_worker()
1141 schedule_delayed_work(&ctrl_info->update_time_work, 0); in pqi_schedule_update_time_worker()
1146 cancel_delayed_work_sync(&ctrl_info->update_time_work); in pqi_cancel_update_time_worker()
1166 rc = -ENOMEM; in pqi_report_phys_logical_luns()
1174 lun_list_length = get_unaligned_be32(&report_lun_header->list_length); in pqi_report_phys_logical_luns()
1181 rc = -ENOMEM; in pqi_report_phys_logical_luns()
1195 get_unaligned_be32(&((struct report_lun_header *)lun_data)->list_length); in pqi_report_phys_logical_luns()
1231 if (ctrl_info->rpl_extended_format_4_5_supported) { in pqi_report_phys_luns()
1233 rpl_response_format = rpl_header->flags & CISS_REPORT_PHYS_FLAG_EXTENDED_FORMAT_MASK; in pqi_report_phys_luns()
1238 dev_err(&ctrl_info->pci_dev->dev, in pqi_report_phys_luns()
1241 return -EINVAL; in pqi_report_phys_luns()
1243 dev_warn(&ctrl_info->pci_dev->dev, in pqi_report_phys_luns()
1249 …physicals = get_unaligned_be32(&rpl_8byte_wwid_list->header.list_length) / sizeof(rpl_8byte_wwid_l… in pqi_report_phys_luns()
1254 return -ENOMEM; in pqi_report_phys_luns()
1257 &rpl_16byte_wwid_list->header.list_length); in pqi_report_phys_luns()
1258 rpl_16byte_wwid_list->header.flags = rpl_8byte_wwid_list->header.flags; in pqi_report_phys_luns()
1261 …memcpy(&rpl_16byte_wwid_list->lun_entries[i].lunid, &rpl_8byte_wwid_list->lun_entries[i].lunid, si… in pqi_report_phys_luns()
1262 …memcpy(&rpl_16byte_wwid_list->lun_entries[i].wwid[0], &rpl_8byte_wwid_list->lun_entries[i].wwid, s… in pqi_report_phys_luns()
1263 memset(&rpl_16byte_wwid_list->lun_entries[i].wwid[8], 0, 8); in pqi_report_phys_luns()
1264 …rpl_16byte_wwid_list->lun_entries[i].device_type = rpl_8byte_wwid_list->lun_entries[i].device_type; in pqi_report_phys_luns()
1265 …rpl_16byte_wwid_list->lun_entries[i].device_flags = rpl_8byte_wwid_list->lun_entries[i].device_fla… in pqi_report_phys_luns()
1266 rpl_16byte_wwid_list->lun_entries[i].lun_count = rpl_8byte_wwid_list->lun_entries[i].lun_count; in pqi_report_phys_luns()
1267 …rpl_16byte_wwid_list->lun_entries[i].redundant_paths = rpl_8byte_wwid_list->lun_entries[i].redunda… in pqi_report_phys_luns()
1268 rpl_16byte_wwid_list->lun_entries[i].aio_handle = rpl_8byte_wwid_list->lun_entries[i].aio_handle; in pqi_report_phys_luns()
1295 dev_err(&ctrl_info->pci_dev->dev, in pqi_get_device_lists()
1300 dev_err(&ctrl_info->pci_dev->dev, in pqi_get_device_lists()
1312 get_unaligned_be32(&logdev_data->header.list_length); in pqi_get_device_lists()
1328 return -ENOMEM; in pqi_get_device_lists()
1336 &internal_logdev_list->header.list_length); in pqi_get_device_lists()
1347 device->bus = bus; in pqi_set_bus_target_lun()
1348 device->target = target; in pqi_set_bus_target_lun()
1349 device->lun = lun; in pqi_set_bus_target_lun()
1360 scsi3addr = device->scsi3addr; in pqi_assign_bus_target_lun()
1366 device->target_lun_valid = true; in pqi_assign_bus_target_lun()
1371 if (device->is_external_raid_device) { in pqi_assign_bus_target_lun()
1381 device->target_lun_valid = true; in pqi_assign_bus_target_lun()
1386 * Defer target and LUN assignment for non-controller physical devices in pqi_assign_bus_target_lun()
1403 rc = pqi_scsi_inquiry(ctrl_info, device->scsi3addr, in pqi_get_raid_level()
1413 device->raid_level = raid_level; in pqi_get_raid_level()
1423 raid_map_size = get_unaligned_le32(&raid_map->structure_size); in pqi_validate_raid_map()
1430 if (device->raid_level == SA_RAID_1) { in pqi_validate_raid_map()
1431 if (get_unaligned_le16(&raid_map->layout_map_count) != 2) { in pqi_validate_raid_map()
1432 err_msg = "invalid RAID-1 map"; in pqi_validate_raid_map()
1435 } else if (device->raid_level == SA_RAID_TRIPLE) { in pqi_validate_raid_map()
1436 if (get_unaligned_le16(&raid_map->layout_map_count) != 3) { in pqi_validate_raid_map()
1437 err_msg = "invalid RAID-1(Triple) map"; in pqi_validate_raid_map()
1440 } else if ((device->raid_level == SA_RAID_5 || in pqi_validate_raid_map()
1441 device->raid_level == SA_RAID_6) && in pqi_validate_raid_map()
1442 get_unaligned_le16(&raid_map->layout_map_count) > 1) { in pqi_validate_raid_map()
1445 get_unaligned_le16(&raid_map->strip_size) * in pqi_validate_raid_map()
1446 get_unaligned_le16(&raid_map->data_disks_per_row); in pqi_validate_raid_map()
1448 err_msg = "invalid RAID-5 or RAID-6 map"; in pqi_validate_raid_map()
1456 dev_warn(&ctrl_info->pci_dev->dev, in pqi_validate_raid_map()
1458 *((u32 *)&device->scsi3addr), in pqi_validate_raid_map()
1459 *((u32 *)&device->scsi3addr[4]), err_msg); in pqi_validate_raid_map()
1461 return -EINVAL; in pqi_validate_raid_map()
1473 return -ENOMEM; in pqi_get_raid_map()
1476 device->scsi3addr, raid_map, sizeof(*raid_map), 0, NULL); in pqi_get_raid_map()
1480 raid_map_size = get_unaligned_le32(&raid_map->structure_size); in pqi_get_raid_map()
1488 return -ENOMEM; in pqi_get_raid_map()
1491 device->scsi3addr, raid_map, raid_map_size, 0, NULL); in pqi_get_raid_map()
1495 if (get_unaligned_le32(&raid_map->structure_size) in pqi_get_raid_map()
1497 dev_warn(&ctrl_info->pci_dev->dev, in pqi_get_raid_map()
1500 get_unaligned_le32(&raid_map->structure_size)); in pqi_get_raid_map()
1501 rc = -EINVAL; in pqi_get_raid_map()
1510 device->raid_io_stats = alloc_percpu(struct pqi_raid_io_stats); in pqi_get_raid_map()
1511 if (!device->raid_io_stats) { in pqi_get_raid_map()
1512 rc = -ENOMEM; in pqi_get_raid_map()
1516 device->raid_map = raid_map; in pqi_get_raid_map()
1529 if (!ctrl_info->lv_drive_type_mix_valid) { in pqi_set_max_transfer_encrypted()
1530 device->max_transfer_encrypted = ~0; in pqi_set_max_transfer_encrypted()
1534 switch (LV_GET_DRIVE_TYPE_MIX(device->scsi3addr)) { in pqi_set_max_transfer_encrypted()
1542 device->max_transfer_encrypted = in pqi_set_max_transfer_encrypted()
1543 ctrl_info->max_transfer_encrypted_sas_sata; in pqi_set_max_transfer_encrypted()
1546 device->max_transfer_encrypted = in pqi_set_max_transfer_encrypted()
1547 ctrl_info->max_transfer_encrypted_nvme; in pqi_set_max_transfer_encrypted()
1552 device->max_transfer_encrypted = in pqi_set_max_transfer_encrypted()
1553 min(ctrl_info->max_transfer_encrypted_sas_sata, in pqi_set_max_transfer_encrypted()
1554 ctrl_info->max_transfer_encrypted_nvme); in pqi_set_max_transfer_encrypted()
1570 rc = pqi_scsi_inquiry(ctrl_info, device->scsi3addr, in pqi_get_raid_bypass_status()
1580 device->raid_bypass_configured = in pqi_get_raid_bypass_status()
1582 if (device->raid_bypass_configured && in pqi_get_raid_bypass_status()
1585 device->raid_bypass_enabled = true; in pqi_get_raid_bypass_status()
1586 if (get_unaligned_le16(&device->raid_map->flags) & in pqi_get_raid_bypass_status()
1596 * Use vendor-specific VPD to determine online/offline status of a volume.
1613 rc = pqi_scsi_inquiry(ctrl_info, device->scsi3addr, in pqi_get_volume_status()
1618 if (vpd->page_code != CISS_VPD_LV_STATUS) in pqi_get_volume_status()
1622 volume_status) + vpd->page_length; in pqi_get_volume_status()
1626 volume_status = vpd->volume_status; in pqi_get_volume_status()
1627 volume_flags = get_unaligned_be32(&vpd->flags); in pqi_get_volume_status()
1633 device->volume_status = volume_status; in pqi_get_volume_status()
1634 device->volume_offline = volume_offline; in pqi_get_volume_status()
1652 device->queue_depth = PQI_PHYSICAL_DISK_DEFAULT_MAX_QUEUE_DEPTH; in pqi_get_physical_device_info()
1656 scsi_sanitize_inquiry_string(&id_phys->model[0], 8); in pqi_get_physical_device_info()
1657 scsi_sanitize_inquiry_string(&id_phys->model[8], 16); in pqi_get_physical_device_info()
1659 memcpy(device->vendor, &id_phys->model[0], sizeof(device->vendor)); in pqi_get_physical_device_info()
1660 memcpy(device->model, &id_phys->model[8], sizeof(device->model)); in pqi_get_physical_device_info()
1662 device->box_index = id_phys->box_index; in pqi_get_physical_device_info()
1663 device->phys_box_on_bus = id_phys->phys_box_on_bus; in pqi_get_physical_device_info()
1664 device->phy_connected_dev_type = id_phys->phy_connected_dev_type[0]; in pqi_get_physical_device_info()
1665 device->queue_depth = in pqi_get_physical_device_info()
1666 get_unaligned_le16(&id_phys->current_queue_depth_limit); in pqi_get_physical_device_info()
1667 device->active_path_index = id_phys->active_path_number; in pqi_get_physical_device_info()
1668 device->path_map = id_phys->redundant_path_present_map; in pqi_get_physical_device_info()
1669 memcpy(&device->box, in pqi_get_physical_device_info()
1670 &id_phys->alternate_paths_phys_box_on_port, in pqi_get_physical_device_info()
1671 sizeof(device->box)); in pqi_get_physical_device_info()
1672 memcpy(&device->phys_connector, in pqi_get_physical_device_info()
1673 &id_phys->alternate_paths_phys_connector, in pqi_get_physical_device_info()
1674 sizeof(device->phys_connector)); in pqi_get_physical_device_info()
1675 device->bay = id_phys->phys_bay_in_box; in pqi_get_physical_device_info()
1676 device->lun_count = id_phys->multi_lun_device_lun_count; in pqi_get_physical_device_info()
1677 if ((id_phys->even_more_flags & PQI_DEVICE_PHY_MAP_SUPPORTED) && in pqi_get_physical_device_info()
1678 id_phys->phy_count) in pqi_get_physical_device_info()
1679 device->phy_id = in pqi_get_physical_device_info()
1680 id_phys->phy_to_phy_map[device->active_path_index]; in pqi_get_physical_device_info()
1682 device->phy_id = 0xFF; in pqi_get_physical_device_info()
1684 device->ncq_prio_support = in pqi_get_physical_device_info()
1685 ((get_unaligned_le32(&id_phys->misc_drive_flags) >> 16) & in pqi_get_physical_device_info()
1688 …device->erase_in_progress = !!(get_unaligned_le16(&id_phys->extra_physical_drive_flags) & PQI_DEVI… in pqi_get_physical_device_info()
1701 return -ENOMEM; in pqi_get_logical_device_info()
1704 rc = pqi_scsi_inquiry(ctrl_info, device->scsi3addr, 0, buffer, 64); in pqi_get_logical_device_info()
1711 device->devtype = buffer[0] & 0x1f; in pqi_get_logical_device_info()
1712 memcpy(device->vendor, &buffer[8], sizeof(device->vendor)); in pqi_get_logical_device_info()
1713 memcpy(device->model, &buffer[16], sizeof(device->model)); in pqi_get_logical_device_info()
1715 if (device->devtype == TYPE_DISK) { in pqi_get_logical_device_info()
1716 if (device->is_external_raid_device) { in pqi_get_logical_device_info()
1717 device->raid_level = SA_RAID_UNKNOWN; in pqi_get_logical_device_info()
1718 device->volume_status = CISS_LV_OK; in pqi_get_logical_device_info()
1719 device->volume_offline = false; in pqi_get_logical_device_info()
1740 * Note: devices that have completed sanitize must be re-enabled
1745 return device->erase_in_progress; in pqi_keep_device_offline()
1754 if (device->is_expander_smp_device) in pqi_get_device_info_phys_logical()
1773 if (rc == 0 && device->lun_count == 0) in pqi_get_device_info()
1774 device->lun_count = 1; in pqi_get_device_info()
1787 switch (device->volume_status) { in pqi_show_volume_status()
1846 status = "Encrypted volume inaccessible - key not present"; in pqi_show_volume_status()
1852 status = "Volume undergoing encryption re-keying process"; in pqi_show_volume_status()
1871 unknown_state_str, device->volume_status); in pqi_show_volume_status()
1876 dev_info(&ctrl_info->pci_dev->dev, in pqi_show_volume_status()
1878 ctrl_info->scsi_host->host_no, in pqi_show_volume_status()
1879 device->bus, device->target, device->lun, status); in pqi_show_volume_status()
1898 rc = scsi_add_device(ctrl_info->scsi_host, device->bus, in pqi_add_device()
1899 device->target, device->lun); in pqi_add_device()
1901 rc = pqi_add_sas_device(ctrl_info->sas_host, device); in pqi_add_device()
1913 for (lun = 0; lun < device->lun_count; lun++) { in pqi_remove_device()
1917 dev_err(&ctrl_info->pci_dev->dev, in pqi_remove_device()
1919 ctrl_info->scsi_host->host_no, device->bus, in pqi_remove_device()
1920 device->target, lun, in pqi_remove_device()
1921 atomic_read(&device->scsi_cmds_outstanding[lun])); in pqi_remove_device()
1925 scsi_remove_device(device->sdev); in pqi_remove_device()
1939 list_for_each_entry(device, &ctrl_info->scsi_device_list, scsi_device_list_entry) in pqi_find_scsi_dev()
1940 if (device->bus == bus && device->target == target && device->lun == lun) in pqi_find_scsi_dev()
1948 if (dev1->is_physical_device != dev2->is_physical_device) in pqi_device_equal()
1951 if (dev1->is_physical_device) in pqi_device_equal()
1952 return memcmp(dev1->wwid, dev2->wwid, sizeof(dev1->wwid)) == 0; in pqi_device_equal()
1954 return memcmp(dev1->volume_id, dev2->volume_id, sizeof(dev1->volume_id)) == 0; in pqi_device_equal()
1968 list_for_each_entry(device, &ctrl_info->scsi_device_list, scsi_device_list_entry) { in pqi_scsi_find_entry()
1969 if (pqi_scsi3addr_equal(device_to_find->scsi3addr, device->scsi3addr)) { in pqi_scsi_find_entry()
1972 if (device_to_find->volume_offline) in pqi_scsi_find_entry()
1985 if (device->is_expander_smp_device) in pqi_device_type()
1988 return scsi_device_type(device->devtype); in pqi_device_type()
2000 "%d:%d:", ctrl_info->scsi_host->host_no, device->bus); in pqi_dev_info()
2002 if (device->target_lun_valid) in pqi_dev_info()
2004 PQI_DEV_INFO_BUFFER_LENGTH - count, in pqi_dev_info()
2006 device->target, in pqi_dev_info()
2007 device->lun); in pqi_dev_info()
2010 PQI_DEV_INFO_BUFFER_LENGTH - count, in pqi_dev_info()
2011 "-:-"); in pqi_dev_info()
2015 PQI_DEV_INFO_BUFFER_LENGTH - count, in pqi_dev_info()
2017 *((u32 *)&device->scsi3addr), in pqi_dev_info()
2018 *((u32 *)&device->scsi3addr[4])); in pqi_dev_info()
2021 PQI_DEV_INFO_BUFFER_LENGTH - count, in pqi_dev_info()
2023 get_unaligned_be64(&device->wwid[0]), in pqi_dev_info()
2024 get_unaligned_be64(&device->wwid[8])); in pqi_dev_info()
2026 count += scnprintf(buffer + count, PQI_DEV_INFO_BUFFER_LENGTH - count, in pqi_dev_info()
2029 device->vendor, in pqi_dev_info()
2030 device->model); in pqi_dev_info()
2033 if (device->devtype == TYPE_DISK) in pqi_dev_info()
2035 PQI_DEV_INFO_BUFFER_LENGTH - count, in pqi_dev_info()
2036 "SSDSmartPathCap%c En%c %-12s", in pqi_dev_info()
2037 device->raid_bypass_configured ? '+' : '-', in pqi_dev_info()
2038 device->raid_bypass_enabled ? '+' : '-', in pqi_dev_info()
2039 pqi_raid_level_to_string(device->raid_level)); in pqi_dev_info()
2042 PQI_DEV_INFO_BUFFER_LENGTH - count, in pqi_dev_info()
2043 "AIO%c", device->aio_enabled ? '+' : '-'); in pqi_dev_info()
2044 if (device->devtype == TYPE_DISK || in pqi_dev_info()
2045 device->devtype == TYPE_ZBC) in pqi_dev_info()
2047 PQI_DEV_INFO_BUFFER_LENGTH - count, in pqi_dev_info()
2048 " qd=%-6d", device->queue_depth); in pqi_dev_info()
2051 dev_info(&ctrl_info->pci_dev->dev, "%s %s\n", action, buffer); in pqi_dev_info()
2062 raid_map1_size = get_unaligned_le32(&raid_map1->structure_size); in pqi_raid_maps_equal()
2063 raid_map2_size = get_unaligned_le32(&raid_map2->structure_size); in pqi_raid_maps_equal()
2076 existing_device->device_type = new_device->device_type; in pqi_scsi_update_device()
2077 existing_device->bus = new_device->bus; in pqi_scsi_update_device()
2078 if (new_device->target_lun_valid) { in pqi_scsi_update_device()
2079 existing_device->target = new_device->target; in pqi_scsi_update_device()
2080 existing_device->lun = new_device->lun; in pqi_scsi_update_device()
2081 existing_device->target_lun_valid = true; in pqi_scsi_update_device()
2086 existing_device->is_physical_device = new_device->is_physical_device; in pqi_scsi_update_device()
2087 memcpy(existing_device->vendor, new_device->vendor, sizeof(existing_device->vendor)); in pqi_scsi_update_device()
2088 memcpy(existing_device->model, new_device->model, sizeof(existing_device->model)); in pqi_scsi_update_device()
2089 existing_device->sas_address = new_device->sas_address; in pqi_scsi_update_device()
2090 existing_device->queue_depth = new_device->queue_depth; in pqi_scsi_update_device()
2091 existing_device->device_offline = false; in pqi_scsi_update_device()
2092 existing_device->lun_count = new_device->lun_count; in pqi_scsi_update_device()
2095 existing_device->is_external_raid_device = new_device->is_external_raid_device; in pqi_scsi_update_device()
2097 if (existing_device->devtype == TYPE_DISK) { in pqi_scsi_update_device()
2098 existing_device->raid_level = new_device->raid_level; in pqi_scsi_update_device()
2099 existing_device->volume_status = new_device->volume_status; in pqi_scsi_update_device()
2100 memset(existing_device->next_bypass_group, 0, sizeof(existing_device->next_bypass_group)); in pqi_scsi_update_device()
2101 if (!pqi_raid_maps_equal(existing_device->raid_map, new_device->raid_map)) { in pqi_scsi_update_device()
2102 kfree(existing_device->raid_map); in pqi_scsi_update_device()
2103 existing_device->raid_map = new_device->raid_map; in pqi_scsi_update_device()
2105 new_device->raid_map = NULL; in pqi_scsi_update_device()
2107 if (new_device->raid_bypass_enabled && existing_device->raid_io_stats == NULL) { in pqi_scsi_update_device()
2108 existing_device->raid_io_stats = new_device->raid_io_stats; in pqi_scsi_update_device()
2109 new_device->raid_io_stats = NULL; in pqi_scsi_update_device()
2111 existing_device->raid_bypass_configured = new_device->raid_bypass_configured; in pqi_scsi_update_device()
2112 existing_device->raid_bypass_enabled = new_device->raid_bypass_enabled; in pqi_scsi_update_device()
2115 existing_device->aio_enabled = new_device->aio_enabled; in pqi_scsi_update_device()
2116 existing_device->aio_handle = new_device->aio_handle; in pqi_scsi_update_device()
2117 existing_device->is_expander_smp_device = new_device->is_expander_smp_device; in pqi_scsi_update_device()
2118 existing_device->active_path_index = new_device->active_path_index; in pqi_scsi_update_device()
2119 existing_device->phy_id = new_device->phy_id; in pqi_scsi_update_device()
2120 existing_device->path_map = new_device->path_map; in pqi_scsi_update_device()
2121 existing_device->bay = new_device->bay; in pqi_scsi_update_device()
2122 existing_device->box_index = new_device->box_index; in pqi_scsi_update_device()
2123 existing_device->phys_box_on_bus = new_device->phys_box_on_bus; in pqi_scsi_update_device()
2124 existing_device->phy_connected_dev_type = new_device->phy_connected_dev_type; in pqi_scsi_update_device()
2125 memcpy(existing_device->box, new_device->box, sizeof(existing_device->box)); in pqi_scsi_update_device()
2126 …memcpy(existing_device->phys_connector, new_device->phys_connector, sizeof(existing_device->phys_c… in pqi_scsi_update_device()
2133 free_percpu(device->raid_io_stats); in pqi_free_device()
2134 kfree(device->raid_map); in pqi_free_device()
2140 * Called when exposing a new device to the OS fails in order to re-adjust
2149 spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags); in pqi_fixup_botched_add()
2150 list_del(&device->scsi_device_list_entry); in pqi_fixup_botched_add()
2151 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags); in pqi_fixup_botched_add()
2154 device->keep_device = false; in pqi_fixup_botched_add()
2159 if (device->is_expander_smp_device) in pqi_is_device_added()
2160 return device->sas_port != NULL; in pqi_is_device_added()
2162 return device->sdev != NULL; in pqi_is_device_added()
2170 for (lun = 0, tmf_work = device->tmf_work; lun < PQI_MAX_LUNS_PER_DEVICE; lun++, tmf_work++) in pqi_init_device_tmf_work()
2171 INIT_WORK(&tmf_work->work_struct, pqi_tmf_worker); in pqi_init_device_tmf_work()
2179 if (device->sdev == NULL) in pqi_volume_rescan_needed()
2182 if (!scsi_device_online(device->sdev)) in pqi_volume_rescan_needed()
2185 return device->rescan; in pqi_volume_rescan_needed()
2208 spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags); in pqi_update_device_list()
2211 list_for_each_entry(device, &ctrl_info->scsi_device_list, scsi_device_list_entry) in pqi_update_device_list()
2212 device->device_gone = true; in pqi_update_device_list()
2226 device->new_device = false; in pqi_update_device_list()
2227 matching_device->device_gone = false; in pqi_update_device_list()
2235 device->new_device = true; in pqi_update_device_list()
2242 device->new_device = true; in pqi_update_device_list()
2248 list_for_each_entry_safe(device, next, &ctrl_info->scsi_device_list, in pqi_update_device_list()
2250 if (device->device_gone) { in pqi_update_device_list()
2251 list_del(&device->scsi_device_list_entry); in pqi_update_device_list()
2252 list_add_tail(&device->delete_list_entry, &delete_list); in pqi_update_device_list()
2259 if (!device->new_device) in pqi_update_device_list()
2261 if (device->volume_offline) in pqi_update_device_list()
2263 list_add_tail(&device->scsi_device_list_entry, in pqi_update_device_list()
2264 &ctrl_info->scsi_device_list); in pqi_update_device_list()
2265 list_add_tail(&device->add_list_entry, &add_list); in pqi_update_device_list()
2267 device->keep_device = true; in pqi_update_device_list()
2271 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags); in pqi_update_device_list()
2288 if (device->volume_offline) { in pqi_update_device_list()
2296 list_del(&device->delete_list_entry); in pqi_update_device_list()
2301 * Notify the SML of any existing device changes such as; in pqi_update_device_list()
2302 * queue depth, device size. in pqi_update_device_list()
2304 list_for_each_entry(device, &ctrl_info->scsi_device_list, scsi_device_list_entry) { in pqi_update_device_list()
2308 if (device->sdev && device->queue_depth != device->advertised_queue_depth) { in pqi_update_device_list()
2309 device->advertised_queue_depth = device->queue_depth; in pqi_update_device_list()
2310 scsi_change_queue_depth(device->sdev, device->advertised_queue_depth); in pqi_update_device_list()
2312 spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags); in pqi_update_device_list()
2314 * Check for changes in the device, such as size. in pqi_update_device_list()
2317 device->rescan = false; in pqi_update_device_list()
2318 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags); in pqi_update_device_list()
2319 scsi_rescan_device(device->sdev); in pqi_update_device_list()
2321 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags); in pqi_update_device_list()
2332 dev_warn(&ctrl_info->pci_dev->dev, in pqi_update_device_list()
2334 ctrl_info->scsi_host->host_no, in pqi_update_device_list()
2335 device->bus, device->target, in pqi_update_device_list()
2336 device->lun); in pqi_update_device_list()
2352 if (device->device_type == SA_DEVICE_TYPE_CONTROLLER && in pqi_is_supported_device()
2353 !pqi_is_hba_lunid(device->scsi3addr)) in pqi_is_supported_device()
2375 return !device->is_physical_device || !pqi_skip_device(device->scsi3addr); in pqi_expose_device()
2408 get_unaligned_be32(&physdev_list->header.list_length) in pqi_update_scsi_devices()
2409 / sizeof(physdev_list->lun_entries[0]); in pqi_update_scsi_devices()
2415 get_unaligned_be32(&logdev_list->header.list_length) in pqi_update_scsi_devices()
2416 / sizeof(logdev_list->lun_entries[0]); in pqi_update_scsi_devices()
2429 dev_warn(&ctrl_info->pci_dev->dev, "%s\n", in pqi_update_scsi_devices()
2431 rc = -ENOMEM; in pqi_update_scsi_devices()
2436 for (i = num_physicals - 1; i >= 0; i--) { in pqi_update_scsi_devices()
2437 phys_lun = &physdev_list->lun_entries[i]; in pqi_update_scsi_devices()
2438 if (CISS_GET_DRIVE_NUMBER(phys_lun->lunid) == PQI_VSEP_CISS_BTL) { in pqi_update_scsi_devices()
2439 pqi_mask_device(phys_lun->lunid); in pqi_update_scsi_devices()
2447 (logdev_list->header.flags & CISS_REPORT_LOG_FLAG_DRIVE_TYPE_MIX)) in pqi_update_scsi_devices()
2448 ctrl_info->lv_drive_type_mix_valid = true; in pqi_update_scsi_devices()
2456 dev_warn(&ctrl_info->pci_dev->dev, "%s\n", out_of_memory_msg); in pqi_update_scsi_devices()
2457 rc = -ENOMEM; in pqi_update_scsi_devices()
2464 dev_warn(&ctrl_info->pci_dev->dev, "%s\n", in pqi_update_scsi_devices()
2466 rc = -ENOMEM; in pqi_update_scsi_devices()
2469 list_add_tail(&device->new_device_list_entry, in pqi_update_scsi_devices()
2483 phys_lun = &physdev_list->lun_entries[physical_index++]; in pqi_update_scsi_devices()
2485 scsi3addr = phys_lun->lunid; in pqi_update_scsi_devices()
2489 log_lun = &logdev_list->lun_entries[logical_index++]; in pqi_update_scsi_devices()
2490 scsi3addr = log_lun->lunid; in pqi_update_scsi_devices()
2502 memcpy(device->scsi3addr, scsi3addr, sizeof(device->scsi3addr)); in pqi_update_scsi_devices()
2503 device->is_physical_device = is_physical_device; in pqi_update_scsi_devices()
2505 device->device_type = phys_lun->device_type; in pqi_update_scsi_devices()
2506 if (device->device_type == SA_DEVICE_TYPE_EXPANDER_SMP) in pqi_update_scsi_devices()
2507 device->is_expander_smp_device = true; in pqi_update_scsi_devices()
2509 device->is_external_raid_device = in pqi_update_scsi_devices()
2518 if (rc == -ENOMEM) { in pqi_update_scsi_devices()
2519 dev_warn(&ctrl_info->pci_dev->dev, "%s\n", in pqi_update_scsi_devices()
2524 if (device->is_physical_device) in pqi_update_scsi_devices()
2525 dev_warn(&ctrl_info->pci_dev->dev, in pqi_update_scsi_devices()
2527 get_unaligned_be64(&phys_lun->wwid[0]), in pqi_update_scsi_devices()
2528 get_unaligned_be64(&phys_lun->wwid[8])); in pqi_update_scsi_devices()
2530 dev_warn(&ctrl_info->pci_dev->dev, in pqi_update_scsi_devices()
2532 *((u32 *)&device->scsi3addr), in pqi_update_scsi_devices()
2533 *((u32 *)&device->scsi3addr[4])); in pqi_update_scsi_devices()
2544 if (device->is_physical_device) { in pqi_update_scsi_devices()
2545 memcpy(device->wwid, phys_lun->wwid, sizeof(device->wwid)); in pqi_update_scsi_devices()
2546 if ((phys_lun->device_flags & in pqi_update_scsi_devices()
2548 phys_lun->aio_handle) { in pqi_update_scsi_devices()
2549 device->aio_enabled = true; in pqi_update_scsi_devices()
2550 device->aio_handle = in pqi_update_scsi_devices()
2551 phys_lun->aio_handle; in pqi_update_scsi_devices()
2554 memcpy(device->volume_id, log_lun->volume_id, in pqi_update_scsi_devices()
2555 sizeof(device->volume_id)); in pqi_update_scsi_devices()
2558 device->sas_address = get_unaligned_be64(&device->wwid[0]); in pqi_update_scsi_devices()
2568 if (device->keep_device) in pqi_update_scsi_devices()
2570 list_del(&device->new_device_list_entry); in pqi_update_scsi_devices()
2588 return -ENXIO; in pqi_scan_scsi_devices()
2590 mutex_acquired = mutex_trylock(&ctrl_info->scan_mutex); in pqi_scan_scsi_devices()
2594 return -EBUSY; in pqi_scan_scsi_devices()
2596 return -EINPROGRESS; in pqi_scan_scsi_devices()
2603 mutex_unlock(&ctrl_info->scan_mutex); in pqi_scan_scsi_devices()
2626 return !mutex_is_locked(&ctrl_info->scan_mutex); in pqi_scan_finished()
2636 * If the block size is 512, the tweak value is equal to the LBA. in pqi_set_encryption_info()
2637 * For other block sizes, tweak value is (LBA * block size) / 512. in pqi_set_encryption_info()
2639 volume_blk_size = get_unaligned_le32(&raid_map->volume_blk_size); in pqi_set_encryption_info()
2643 encryption_info->data_encryption_key_index = in pqi_set_encryption_info()
2644 get_unaligned_le16(&raid_map->data_encryption_key_index); in pqi_set_encryption_info()
2645 encryption_info->encrypt_tweak_lower = lower_32_bits(first_block); in pqi_set_encryption_info()
2646 encryption_info->encrypt_tweak_upper = upper_32_bits(first_block); in pqi_set_encryption_info()
2658 switch (rmd->raid_level) { in pqi_aio_raid_level_supported()
2662 if (rmd->is_write && (!ctrl_info->enable_r1_writes || in pqi_aio_raid_level_supported()
2663 rmd->data_length > ctrl_info->max_write_raid_1_10_2drive)) in pqi_aio_raid_level_supported()
2667 if (rmd->is_write && (!ctrl_info->enable_r1_writes || in pqi_aio_raid_level_supported()
2668 rmd->data_length > ctrl_info->max_write_raid_1_10_3drive)) in pqi_aio_raid_level_supported()
2672 if (rmd->is_write && (!ctrl_info->enable_r5_writes || in pqi_aio_raid_level_supported()
2673 rmd->data_length > ctrl_info->max_write_raid_5_6)) in pqi_aio_raid_level_supported()
2677 if (rmd->is_write && (!ctrl_info->enable_r6_writes || in pqi_aio_raid_level_supported()
2678 rmd->data_length > ctrl_info->max_write_raid_5_6)) in pqi_aio_raid_level_supported()
2695 switch (scmd->cmnd[0]) { in pqi_get_aio_lba_and_block_count()
2697 rmd->is_write = true; in pqi_get_aio_lba_and_block_count()
2700 rmd->first_block = (u64)(((scmd->cmnd[1] & 0x1f) << 16) | in pqi_get_aio_lba_and_block_count()
2701 (scmd->cmnd[2] << 8) | scmd->cmnd[3]); in pqi_get_aio_lba_and_block_count()
2702 rmd->block_cnt = (u32)scmd->cmnd[4]; in pqi_get_aio_lba_and_block_count()
2703 if (rmd->block_cnt == 0) in pqi_get_aio_lba_and_block_count()
2704 rmd->block_cnt = 256; in pqi_get_aio_lba_and_block_count()
2707 rmd->is_write = true; in pqi_get_aio_lba_and_block_count()
2710 rmd->first_block = (u64)get_unaligned_be32(&scmd->cmnd[2]); in pqi_get_aio_lba_and_block_count()
2711 rmd->block_cnt = (u32)get_unaligned_be16(&scmd->cmnd[7]); in pqi_get_aio_lba_and_block_count()
2714 rmd->is_write = true; in pqi_get_aio_lba_and_block_count()
2717 rmd->first_block = (u64)get_unaligned_be32(&scmd->cmnd[2]); in pqi_get_aio_lba_and_block_count()
2718 rmd->block_cnt = get_unaligned_be32(&scmd->cmnd[6]); in pqi_get_aio_lba_and_block_count()
2721 rmd->is_write = true; in pqi_get_aio_lba_and_block_count()
2724 rmd->first_block = get_unaligned_be64(&scmd->cmnd[2]); in pqi_get_aio_lba_and_block_count()
2725 rmd->block_cnt = get_unaligned_be32(&scmd->cmnd[10]); in pqi_get_aio_lba_and_block_count()
2732 put_unaligned_le32(scsi_bufflen(scmd), &rmd->data_length); in pqi_get_aio_lba_and_block_count()
2744 rmd->last_block = rmd->first_block + rmd->block_cnt - 1; in pci_get_aio_common_raid_map_values()
2747 if (rmd->last_block >= in pci_get_aio_common_raid_map_values()
2748 get_unaligned_le64(&raid_map->volume_blk_cnt) || in pci_get_aio_common_raid_map_values()
2749 rmd->last_block < rmd->first_block) in pci_get_aio_common_raid_map_values()
2752 rmd->data_disks_per_row = in pci_get_aio_common_raid_map_values()
2753 get_unaligned_le16(&raid_map->data_disks_per_row); in pci_get_aio_common_raid_map_values()
2754 rmd->strip_size = get_unaligned_le16(&raid_map->strip_size); in pci_get_aio_common_raid_map_values()
2755 rmd->layout_map_count = get_unaligned_le16(&raid_map->layout_map_count); in pci_get_aio_common_raid_map_values()
2758 rmd->blocks_per_row = rmd->data_disks_per_row * rmd->strip_size; in pci_get_aio_common_raid_map_values()
2759 if (rmd->blocks_per_row == 0) /* Used as a divisor in many calculations */ in pci_get_aio_common_raid_map_values()
2762 tmpdiv = rmd->first_block; in pci_get_aio_common_raid_map_values()
2763 do_div(tmpdiv, rmd->blocks_per_row); in pci_get_aio_common_raid_map_values()
2764 rmd->first_row = tmpdiv; in pci_get_aio_common_raid_map_values()
2765 tmpdiv = rmd->last_block; in pci_get_aio_common_raid_map_values()
2766 do_div(tmpdiv, rmd->blocks_per_row); in pci_get_aio_common_raid_map_values()
2767 rmd->last_row = tmpdiv; in pci_get_aio_common_raid_map_values()
2768 rmd->first_row_offset = (u32)(rmd->first_block - (rmd->first_row * rmd->blocks_per_row)); in pci_get_aio_common_raid_map_values()
2769 rmd->last_row_offset = (u32)(rmd->last_block - (rmd->last_row * rmd->blocks_per_row)); in pci_get_aio_common_raid_map_values()
2770 tmpdiv = rmd->first_row_offset; in pci_get_aio_common_raid_map_values()
2771 do_div(tmpdiv, rmd->strip_size); in pci_get_aio_common_raid_map_values()
2772 rmd->first_column = tmpdiv; in pci_get_aio_common_raid_map_values()
2773 tmpdiv = rmd->last_row_offset; in pci_get_aio_common_raid_map_values()
2774 do_div(tmpdiv, rmd->strip_size); in pci_get_aio_common_raid_map_values()
2775 rmd->last_column = tmpdiv; in pci_get_aio_common_raid_map_values()
2777 rmd->first_row = rmd->first_block / rmd->blocks_per_row; in pci_get_aio_common_raid_map_values()
2778 rmd->last_row = rmd->last_block / rmd->blocks_per_row; in pci_get_aio_common_raid_map_values()
2779 rmd->first_row_offset = (u32)(rmd->first_block - in pci_get_aio_common_raid_map_values()
2780 (rmd->first_row * rmd->blocks_per_row)); in pci_get_aio_common_raid_map_values()
2781 rmd->last_row_offset = (u32)(rmd->last_block - (rmd->last_row * in pci_get_aio_common_raid_map_values()
2782 rmd->blocks_per_row)); in pci_get_aio_common_raid_map_values()
2783 rmd->first_column = rmd->first_row_offset / rmd->strip_size; in pci_get_aio_common_raid_map_values()
2784 rmd->last_column = rmd->last_row_offset / rmd->strip_size; in pci_get_aio_common_raid_map_values()
2788 if (rmd->first_row != rmd->last_row || in pci_get_aio_common_raid_map_values()
2789 rmd->first_column != rmd->last_column) in pci_get_aio_common_raid_map_values()
2793 rmd->total_disks_per_row = rmd->data_disks_per_row + in pci_get_aio_common_raid_map_values()
2794 get_unaligned_le16(&raid_map->metadata_disks_per_row); in pci_get_aio_common_raid_map_values()
2795 rmd->map_row = ((u32)(rmd->first_row >> in pci_get_aio_common_raid_map_values()
2796 raid_map->parity_rotation_shift)) % in pci_get_aio_common_raid_map_values()
2797 get_unaligned_le16(&raid_map->row_cnt); in pci_get_aio_common_raid_map_values()
2798 rmd->map_index = (rmd->map_row * rmd->total_disks_per_row) + in pci_get_aio_common_raid_map_values()
2799 rmd->first_column; in pci_get_aio_common_raid_map_values()
2811 if (rmd->blocks_per_row == 0) /* Used as a divisor in many calculations */ in pqi_calc_aio_r5_or_r6()
2816 rmd->stripesize = rmd->blocks_per_row * rmd->layout_map_count; in pqi_calc_aio_r5_or_r6()
2818 tmpdiv = rmd->first_block; in pqi_calc_aio_r5_or_r6()
2819 rmd->first_group = do_div(tmpdiv, rmd->stripesize); in pqi_calc_aio_r5_or_r6()
2820 tmpdiv = rmd->first_group; in pqi_calc_aio_r5_or_r6()
2821 do_div(tmpdiv, rmd->blocks_per_row); in pqi_calc_aio_r5_or_r6()
2822 rmd->first_group = tmpdiv; in pqi_calc_aio_r5_or_r6()
2823 tmpdiv = rmd->last_block; in pqi_calc_aio_r5_or_r6()
2824 rmd->last_group = do_div(tmpdiv, rmd->stripesize); in pqi_calc_aio_r5_or_r6()
2825 tmpdiv = rmd->last_group; in pqi_calc_aio_r5_or_r6()
2826 do_div(tmpdiv, rmd->blocks_per_row); in pqi_calc_aio_r5_or_r6()
2827 rmd->last_group = tmpdiv; in pqi_calc_aio_r5_or_r6()
2829 rmd->first_group = (rmd->first_block % rmd->stripesize) / rmd->blocks_per_row; in pqi_calc_aio_r5_or_r6()
2830 rmd->last_group = (rmd->last_block % rmd->stripesize) / rmd->blocks_per_row; in pqi_calc_aio_r5_or_r6()
2832 if (rmd->first_group != rmd->last_group) in pqi_calc_aio_r5_or_r6()
2837 tmpdiv = rmd->first_block; in pqi_calc_aio_r5_or_r6()
2838 do_div(tmpdiv, rmd->stripesize); in pqi_calc_aio_r5_or_r6()
2839 rmd->first_row = tmpdiv; in pqi_calc_aio_r5_or_r6()
2840 rmd->r5or6_first_row = tmpdiv; in pqi_calc_aio_r5_or_r6()
2841 tmpdiv = rmd->last_block; in pqi_calc_aio_r5_or_r6()
2842 do_div(tmpdiv, rmd->stripesize); in pqi_calc_aio_r5_or_r6()
2843 rmd->r5or6_last_row = tmpdiv; in pqi_calc_aio_r5_or_r6()
2845 rmd->first_row = rmd->r5or6_first_row = in pqi_calc_aio_r5_or_r6()
2846 rmd->first_block / rmd->stripesize; in pqi_calc_aio_r5_or_r6()
2847 rmd->r5or6_last_row = rmd->last_block / rmd->stripesize; in pqi_calc_aio_r5_or_r6()
2849 if (rmd->r5or6_first_row != rmd->r5or6_last_row) in pqi_calc_aio_r5_or_r6()
2854 tmpdiv = rmd->first_block; in pqi_calc_aio_r5_or_r6()
2855 rmd->first_row_offset = do_div(tmpdiv, rmd->stripesize); in pqi_calc_aio_r5_or_r6()
2856 tmpdiv = rmd->first_row_offset; in pqi_calc_aio_r5_or_r6()
2857 rmd->first_row_offset = (u32)do_div(tmpdiv, rmd->blocks_per_row); in pqi_calc_aio_r5_or_r6()
2858 rmd->r5or6_first_row_offset = rmd->first_row_offset; in pqi_calc_aio_r5_or_r6()
2859 tmpdiv = rmd->last_block; in pqi_calc_aio_r5_or_r6()
2860 rmd->r5or6_last_row_offset = do_div(tmpdiv, rmd->stripesize); in pqi_calc_aio_r5_or_r6()
2861 tmpdiv = rmd->r5or6_last_row_offset; in pqi_calc_aio_r5_or_r6()
2862 rmd->r5or6_last_row_offset = do_div(tmpdiv, rmd->blocks_per_row); in pqi_calc_aio_r5_or_r6()
2863 tmpdiv = rmd->r5or6_first_row_offset; in pqi_calc_aio_r5_or_r6()
2864 do_div(tmpdiv, rmd->strip_size); in pqi_calc_aio_r5_or_r6()
2865 rmd->first_column = rmd->r5or6_first_column = tmpdiv; in pqi_calc_aio_r5_or_r6()
2866 tmpdiv = rmd->r5or6_last_row_offset; in pqi_calc_aio_r5_or_r6()
2867 do_div(tmpdiv, rmd->strip_size); in pqi_calc_aio_r5_or_r6()
2868 rmd->r5or6_last_column = tmpdiv; in pqi_calc_aio_r5_or_r6()
2870 rmd->first_row_offset = rmd->r5or6_first_row_offset = in pqi_calc_aio_r5_or_r6()
2871 (u32)((rmd->first_block % rmd->stripesize) % in pqi_calc_aio_r5_or_r6()
2872 rmd->blocks_per_row); in pqi_calc_aio_r5_or_r6()
2874 rmd->r5or6_last_row_offset = in pqi_calc_aio_r5_or_r6()
2875 (u32)((rmd->last_block % rmd->stripesize) % in pqi_calc_aio_r5_or_r6()
2876 rmd->blocks_per_row); in pqi_calc_aio_r5_or_r6()
2878 rmd->first_column = in pqi_calc_aio_r5_or_r6()
2879 rmd->r5or6_first_row_offset / rmd->strip_size; in pqi_calc_aio_r5_or_r6()
2880 rmd->r5or6_first_column = rmd->first_column; in pqi_calc_aio_r5_or_r6()
2881 rmd->r5or6_last_column = rmd->r5or6_last_row_offset / rmd->strip_size; in pqi_calc_aio_r5_or_r6()
2883 if (rmd->r5or6_first_column != rmd->r5or6_last_column) in pqi_calc_aio_r5_or_r6()
2887 rmd->map_row = in pqi_calc_aio_r5_or_r6()
2888 ((u32)(rmd->first_row >> raid_map->parity_rotation_shift)) % in pqi_calc_aio_r5_or_r6()
2889 get_unaligned_le16(&raid_map->row_cnt); in pqi_calc_aio_r5_or_r6()
2891 rmd->map_index = (rmd->first_group * in pqi_calc_aio_r5_or_r6()
2892 (get_unaligned_le16(&raid_map->row_cnt) * in pqi_calc_aio_r5_or_r6()
2893 rmd->total_disks_per_row)) + in pqi_calc_aio_r5_or_r6()
2894 (rmd->map_row * rmd->total_disks_per_row) + rmd->first_column; in pqi_calc_aio_r5_or_r6()
2896 if (rmd->is_write) { in pqi_calc_aio_r5_or_r6()
2905 * The devices RAID map size is checked during device in pqi_calc_aio_r5_or_r6()
2908 index = DIV_ROUND_UP(rmd->map_index + 1, rmd->total_disks_per_row); in pqi_calc_aio_r5_or_r6()
2909 index *= rmd->total_disks_per_row; in pqi_calc_aio_r5_or_r6()
2910 index -= get_unaligned_le16(&raid_map->metadata_disks_per_row); in pqi_calc_aio_r5_or_r6()
2912 rmd->p_parity_it_nexus = raid_map->disk_data[index].aio_handle; in pqi_calc_aio_r5_or_r6()
2913 if (rmd->raid_level == SA_RAID_6) { in pqi_calc_aio_r5_or_r6()
2914 rmd->q_parity_it_nexus = raid_map->disk_data[index + 1].aio_handle; in pqi_calc_aio_r5_or_r6()
2915 rmd->xor_mult = raid_map->disk_data[rmd->map_index].xor_mult[1]; in pqi_calc_aio_r5_or_r6()
2918 tmpdiv = rmd->first_block; in pqi_calc_aio_r5_or_r6()
2919 do_div(tmpdiv, rmd->blocks_per_row); in pqi_calc_aio_r5_or_r6()
2920 rmd->row = tmpdiv; in pqi_calc_aio_r5_or_r6()
2922 rmd->row = rmd->first_block / rmd->blocks_per_row; in pqi_calc_aio_r5_or_r6()
2932 if (rmd->disk_block > 0xffffffff) { in pqi_set_aio_cdb()
2933 rmd->cdb[0] = rmd->is_write ? WRITE_16 : READ_16; in pqi_set_aio_cdb()
2934 rmd->cdb[1] = 0; in pqi_set_aio_cdb()
2935 put_unaligned_be64(rmd->disk_block, &rmd->cdb[2]); in pqi_set_aio_cdb()
2936 put_unaligned_be32(rmd->disk_block_cnt, &rmd->cdb[10]); in pqi_set_aio_cdb()
2937 rmd->cdb[14] = 0; in pqi_set_aio_cdb()
2938 rmd->cdb[15] = 0; in pqi_set_aio_cdb()
2939 rmd->cdb_length = 16; in pqi_set_aio_cdb()
2941 rmd->cdb[0] = rmd->is_write ? WRITE_10 : READ_10; in pqi_set_aio_cdb()
2942 rmd->cdb[1] = 0; in pqi_set_aio_cdb()
2943 put_unaligned_be32((u32)rmd->disk_block, &rmd->cdb[2]); in pqi_set_aio_cdb()
2944 rmd->cdb[6] = 0; in pqi_set_aio_cdb()
2945 put_unaligned_be16((u16)rmd->disk_block_cnt, &rmd->cdb[7]); in pqi_set_aio_cdb()
2946 rmd->cdb[9] = 0; in pqi_set_aio_cdb()
2947 rmd->cdb_length = 10; in pqi_set_aio_cdb()
2957 group = rmd->map_index / rmd->data_disks_per_row; in pqi_calc_aio_r1_nexus()
2959 index = rmd->map_index - (group * rmd->data_disks_per_row); in pqi_calc_aio_r1_nexus()
2960 rmd->it_nexus[0] = raid_map->disk_data[index].aio_handle; in pqi_calc_aio_r1_nexus()
2961 index += rmd->data_disks_per_row; in pqi_calc_aio_r1_nexus()
2962 rmd->it_nexus[1] = raid_map->disk_data[index].aio_handle; in pqi_calc_aio_r1_nexus()
2963 if (rmd->layout_map_count > 2) { in pqi_calc_aio_r1_nexus()
2964 index += rmd->data_disks_per_row; in pqi_calc_aio_r1_nexus()
2965 rmd->it_nexus[2] = raid_map->disk_data[index].aio_handle; in pqi_calc_aio_r1_nexus()
2968 rmd->num_it_nexus_entries = rmd->layout_map_count; in pqi_calc_aio_r1_nexus()
2987 rmd.raid_level = device->raid_level; in pqi_raid_bypass_submit_scsi_cmd()
2995 raid_map = device->raid_map; in pqi_raid_bypass_submit_scsi_cmd()
3001 if (device->raid_level == SA_RAID_1 || in pqi_raid_bypass_submit_scsi_cmd()
3002 device->raid_level == SA_RAID_TRIPLE) { in pqi_raid_bypass_submit_scsi_cmd()
3006 group = device->next_bypass_group[rmd.map_index]; in pqi_raid_bypass_submit_scsi_cmd()
3010 device->next_bypass_group[rmd.map_index] = next_bypass_group; in pqi_raid_bypass_submit_scsi_cmd()
3013 } else if ((device->raid_level == SA_RAID_5 || in pqi_raid_bypass_submit_scsi_cmd()
3014 device->raid_level == SA_RAID_6) && in pqi_raid_bypass_submit_scsi_cmd()
3024 rmd.aio_handle = raid_map->disk_data[rmd.map_index].aio_handle; in pqi_raid_bypass_submit_scsi_cmd()
3025 rmd.disk_block = get_unaligned_le64(&raid_map->disk_starting_blk) + in pqi_raid_bypass_submit_scsi_cmd()
3027 (rmd.first_row_offset - rmd.first_column * rmd.strip_size); in pqi_raid_bypass_submit_scsi_cmd()
3031 if (raid_map->phys_blk_shift) { in pqi_raid_bypass_submit_scsi_cmd()
3032 rmd.disk_block <<= raid_map->phys_blk_shift; in pqi_raid_bypass_submit_scsi_cmd()
3033 rmd.disk_block_cnt <<= raid_map->phys_blk_shift; in pqi_raid_bypass_submit_scsi_cmd()
3041 if (get_unaligned_le16(&raid_map->flags) & RAID_MAP_ENCRYPTION_ENABLED) { in pqi_raid_bypass_submit_scsi_cmd()
3042 if (rmd.data_length > device->max_transfer_encrypted) in pqi_raid_bypass_submit_scsi_cmd()
3051 switch (device->raid_level) { in pqi_raid_bypass_submit_scsi_cmd()
3089 pqi_registers = ctrl_info->pqi_registers; in pqi_wait_for_pqi_mode_ready()
3093 signature = readq(&pqi_registers->signature); in pqi_wait_for_pqi_mode_ready()
3098 dev_err(&ctrl_info->pci_dev->dev, in pqi_wait_for_pqi_mode_ready()
3100 return -ETIMEDOUT; in pqi_wait_for_pqi_mode_ready()
3106 status = readb(&pqi_registers->function_and_status_code); in pqi_wait_for_pqi_mode_ready()
3110 dev_err(&ctrl_info->pci_dev->dev, in pqi_wait_for_pqi_mode_ready()
3112 return -ETIMEDOUT; in pqi_wait_for_pqi_mode_ready()
3118 if (readl(&pqi_registers->device_status) == in pqi_wait_for_pqi_mode_ready()
3122 dev_err(&ctrl_info->pci_dev->dev, in pqi_wait_for_pqi_mode_ready()
3124 return -ETIMEDOUT; in pqi_wait_for_pqi_mode_ready()
3136 device = io_request->scmd->device->hostdata; in pqi_aio_path_disabled()
3137 device->raid_bypass_enabled = false; in pqi_aio_path_disabled()
3138 device->aio_enabled = false; in pqi_aio_path_disabled()
3146 device = sdev->hostdata; in pqi_take_device_offline()
3147 if (device->device_offline) in pqi_take_device_offline()
3150 device->device_offline = true; in pqi_take_device_offline()
3151 ctrl_info = shost_to_hba(sdev->host); in pqi_take_device_offline()
3153 dev_err(&ctrl_info->pci_dev->dev, "re-scanning %s scsi %d:%d:%d:%d\n", in pqi_take_device_offline()
3154 path, ctrl_info->scsi_host->host_no, device->bus, in pqi_take_device_offline()
3155 device->target, device->lun); in pqi_take_device_offline()
3169 scmd = io_request->scmd; in pqi_process_raid_io_error()
3173 error_info = io_request->error_info; in pqi_process_raid_io_error()
3174 scsi_status = error_info->status; in pqi_process_raid_io_error()
3177 switch (error_info->data_out_result) { in pqi_process_raid_io_error()
3182 get_unaligned_le32(&error_info->data_out_transferred); in pqi_process_raid_io_error()
3183 residual_count = scsi_bufflen(scmd) - xfer_count; in pqi_process_raid_io_error()
3185 if (xfer_count < scmd->underflow) in pqi_process_raid_io_error()
3216 sense_data_length = get_unaligned_le16(&error_info->sense_data_length); in pqi_process_raid_io_error()
3219 get_unaligned_le16(&error_info->response_data_length); in pqi_process_raid_io_error()
3221 if (sense_data_length > sizeof(error_info->data)) in pqi_process_raid_io_error()
3222 sense_data_length = sizeof(error_info->data); in pqi_process_raid_io_error()
3225 scsi_normalize_sense(error_info->data, in pqi_process_raid_io_error()
3229 struct pqi_ctrl_info *ctrl_info = shost_to_hba(scmd->device->host); in pqi_process_raid_io_error()
3230 struct pqi_scsi_dev *device = scmd->device->hostdata; in pqi_process_raid_io_error()
3236 ctrl_info->scsi_host->host_no, device->bus, device->target, device->lun); in pqi_process_raid_io_error()
3237 pqi_take_device_offline(scmd->device, "RAID"); in pqi_process_raid_io_error()
3241 default: /* See http://www.t10.org/lists/asc-num.htm#ASC_3E */ in pqi_process_raid_io_error()
3244 sshdr.ascq, ctrl_info->scsi_host->host_no, device->bus, device->target, device->lun); in pqi_process_raid_io_error()
3251 memcpy(scmd->sense_buffer, error_info->data, in pqi_process_raid_io_error()
3255 if (pqi_cmd_priv(scmd)->this_residual && in pqi_process_raid_io_error()
3256 !pqi_is_logical_device(scmd->device->hostdata) && in pqi_process_raid_io_error()
3260 scsi_normalize_sense(error_info->data, sense_data_length, &sshdr) && in pqi_process_raid_io_error()
3265 pqi_take_device_offline(scmd->device, "AIO"); in pqi_process_raid_io_error()
3266 scsi_build_sense_buffer(0, scmd->sense_buffer, HARDWARE_ERROR, 0x3e, 0x1); in pqi_process_raid_io_error()
3269 scmd->result = scsi_status; in pqi_process_raid_io_error()
3284 scmd = io_request->scmd; in pqi_process_aio_io_error()
3285 error_info = io_request->error_info; in pqi_process_aio_io_error()
3290 switch (error_info->service_response) { in pqi_process_aio_io_error()
3292 scsi_status = error_info->status; in pqi_process_aio_io_error()
3295 switch (error_info->status) { in pqi_process_aio_io_error()
3302 &error_info->residual_count); in pqi_process_aio_io_error()
3304 xfer_count = scsi_bufflen(scmd) - residual_count; in pqi_process_aio_io_error()
3305 if (xfer_count < scmd->underflow) in pqi_process_aio_io_error()
3314 io_request->status = -EAGAIN; in pqi_process_aio_io_error()
3318 if (!io_request->raid_bypass) { in pqi_process_aio_io_error()
3320 pqi_take_device_offline(scmd->device, "AIO"); in pqi_process_aio_io_error()
3342 if (error_info->data_present) { in pqi_process_aio_io_error()
3344 get_unaligned_le16(&error_info->data_length); in pqi_process_aio_io_error()
3346 if (sense_data_length > sizeof(error_info->data)) in pqi_process_aio_io_error()
3347 sense_data_length = sizeof(error_info->data); in pqi_process_aio_io_error()
3350 memcpy(scmd->sense_buffer, error_info->data, in pqi_process_aio_io_error()
3358 scmd->result = scsi_status; in pqi_process_aio_io_error()
3380 switch (response->response_code) { in pqi_interpret_task_management_response()
3386 rc = -EAGAIN; in pqi_interpret_task_management_response()
3389 rc = -ENODEV; in pqi_interpret_task_management_response()
3392 rc = -EIO; in pqi_interpret_task_management_response()
3397 dev_err(&ctrl_info->pci_dev->dev, in pqi_interpret_task_management_response()
3398 "Task Management Function error: %d (response code: %u)\n", rc, response->response_code); in pqi_interpret_task_management_response()
3419 oq_ci = queue_group->oq_ci_copy; in pqi_process_io_intr()
3422 oq_pi = readl(queue_group->oq_pi); in pqi_process_io_intr()
3423 if (oq_pi >= ctrl_info->num_elements_per_oq) { in pqi_process_io_intr()
3425 dev_err(&ctrl_info->pci_dev->dev, in pqi_process_io_intr()
3426 "I/O interrupt: producer index (%u) out of range (0-%u): consumer index: %u\n", in pqi_process_io_intr()
3427 oq_pi, ctrl_info->num_elements_per_oq - 1, oq_ci); in pqi_process_io_intr()
3428 return -1; in pqi_process_io_intr()
3434 response = queue_group->oq_element_array + in pqi_process_io_intr()
3437 request_id = get_unaligned_le16(&response->request_id); in pqi_process_io_intr()
3438 if (request_id >= ctrl_info->max_io_slots) { in pqi_process_io_intr()
3440 dev_err(&ctrl_info->pci_dev->dev, in pqi_process_io_intr()
3441 "request ID in response (%u) out of range (0-%u): producer index: %u consumer index: %u\n", in pqi_process_io_intr()
3442 request_id, ctrl_info->max_io_slots - 1, oq_pi, oq_ci); in pqi_process_io_intr()
3443 return -1; in pqi_process_io_intr()
3446 io_request = &ctrl_info->io_request_pool[request_id]; in pqi_process_io_intr()
3447 if (atomic_read(&io_request->refcount) == 0) { in pqi_process_io_intr()
3449 dev_err(&ctrl_info->pci_dev->dev, in pqi_process_io_intr()
3452 return -1; in pqi_process_io_intr()
3455 switch (response->header.iu_type) { in pqi_process_io_intr()
3458 if (io_request->scmd) in pqi_process_io_intr()
3459 io_request->scmd->result = 0; in pqi_process_io_intr()
3464 io_request->status = in pqi_process_io_intr()
3466 &((struct pqi_vendor_general_response *)response)->status); in pqi_process_io_intr()
3469 io_request->status = pqi_interpret_task_management_response(ctrl_info, in pqi_process_io_intr()
3474 io_request->status = -EAGAIN; in pqi_process_io_intr()
3478 io_request->error_info = ctrl_info->error_buffer + in pqi_process_io_intr()
3479 (get_unaligned_le16(&response->error_index) * in pqi_process_io_intr()
3481 pqi_process_io_error(response->header.iu_type, io_request); in pqi_process_io_intr()
3485 dev_err(&ctrl_info->pci_dev->dev, in pqi_process_io_intr()
3487 response->header.iu_type, oq_pi, oq_ci); in pqi_process_io_intr()
3488 return -1; in pqi_process_io_intr()
3491 io_request->io_complete_callback(io_request, io_request->context); in pqi_process_io_intr()
3497 oq_ci = (oq_ci + 1) % ctrl_info->num_elements_per_oq; in pqi_process_io_intr()
3501 queue_group->oq_ci_copy = oq_ci; in pqi_process_io_intr()
3502 writel(oq_ci, queue_group->oq_ci); in pqi_process_io_intr()
3514 num_elements_used = pi - ci; in pqi_num_elements_free()
3516 num_elements_used = elements_in_queue - ci + pi; in pqi_num_elements_free()
3518 return elements_in_queue - num_elements_used - 1; in pqi_num_elements_free()
3530 queue_group = &ctrl_info->queue_groups[PQI_DEFAULT_QUEUE_GROUP]; in pqi_send_event_ack()
3531 put_unaligned_le16(queue_group->oq_id, &iu->header.response_queue_id); in pqi_send_event_ack()
3534 spin_lock_irqsave(&queue_group->submit_lock[RAID_PATH], flags); in pqi_send_event_ack()
3536 iq_pi = queue_group->iq_pi_copy[RAID_PATH]; in pqi_send_event_ack()
3537 iq_ci = readl(queue_group->iq_ci[RAID_PATH]); in pqi_send_event_ack()
3540 ctrl_info->num_elements_per_iq)) in pqi_send_event_ack()
3544 &queue_group->submit_lock[RAID_PATH], flags); in pqi_send_event_ack()
3550 next_element = queue_group->iq_element_array[RAID_PATH] + in pqi_send_event_ack()
3555 iq_pi = (iq_pi + 1) % ctrl_info->num_elements_per_iq; in pqi_send_event_ack()
3556 queue_group->iq_pi_copy[RAID_PATH] = iq_pi; in pqi_send_event_ack()
3562 writel(iq_pi, queue_group->iq_pi[RAID_PATH]); in pqi_send_event_ack()
3564 spin_unlock_irqrestore(&queue_group->submit_lock[RAID_PATH], flags); in pqi_send_event_ack()
3575 put_unaligned_le16(sizeof(request) - PQI_REQUEST_HEADER_LENGTH, in pqi_acknowledge_event()
3577 request.event_type = event->event_type; in pqi_acknowledge_event()
3578 put_unaligned_le16(event->event_id, &request.event_id); in pqi_acknowledge_event()
3579 put_unaligned_le32(event->additional_event_id, &request.additional_event_id); in pqi_acknowledge_event()
3607 dev_warn(&ctrl_info->pci_dev->dev, in pqi_poll_for_soft_reset_status()
3622 if (ctrl_info->soft_reset_handshake_supported) in pqi_process_soft_reset()
3634 dev_info(&ctrl_info->pci_dev->dev, in pqi_process_soft_reset()
3639 ctrl_info->pqi_mode_enabled = false; in pqi_process_soft_reset()
3642 pqi_host_free_buffer(ctrl_info, &ctrl_info->ofa_memory); in pqi_process_soft_reset()
3644 dev_info(&ctrl_info->pci_dev->dev, in pqi_process_soft_reset()
3649 dev_info(&ctrl_info->pci_dev->dev, in pqi_process_soft_reset()
3651 if (ctrl_info->soft_reset_handshake_supported) in pqi_process_soft_reset()
3653 pqi_host_free_buffer(ctrl_info, &ctrl_info->ofa_memory); in pqi_process_soft_reset()
3660 dev_err(&ctrl_info->pci_dev->dev, in pqi_process_soft_reset()
3663 pqi_host_free_buffer(ctrl_info, &ctrl_info->ofa_memory); in pqi_process_soft_reset()
3678 …pqi_host_setup_buffer(ctrl_info, &ctrl_info->ofa_memory, ctrl_info->ofa_bytes_requested, ctrl_info in pqi_ofa_memory_alloc_worker()
3679 pqi_host_memory_update(ctrl_info, &ctrl_info->ofa_memory, PQI_VENDOR_GENERAL_OFA_MEMORY_UPDATE); in pqi_ofa_memory_alloc_worker()
3689 event = &ctrl_info->events[pqi_event_type_to_event_index(PQI_EVENT_TYPE_OFA)]; in pqi_ofa_quiesce_worker()
3703 switch (event->event_id) { in pqi_ofa_process_event()
3705 dev_info(&ctrl_info->pci_dev->dev, in pqi_ofa_process_event()
3707 schedule_work(&ctrl_info->ofa_memory_alloc_work); in pqi_ofa_process_event()
3710 dev_info(&ctrl_info->pci_dev->dev, in pqi_ofa_process_event()
3712 schedule_work(&ctrl_info->ofa_quiesce_work); in pqi_ofa_process_event()
3716 dev_info(&ctrl_info->pci_dev->dev, in pqi_ofa_process_event()
3718 ctrl_info->ofa_cancel_reason); in pqi_ofa_process_event()
3719 pqi_host_free_buffer(ctrl_info, &ctrl_info->ofa_memory); in pqi_ofa_process_event()
3723 dev_err(&ctrl_info->pci_dev->dev, in pqi_ofa_process_event()
3725 event->event_id); in pqi_ofa_process_event()
3737 spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags); in pqi_mark_volumes_for_rescan()
3739 list_for_each_entry(device, &ctrl_info->scsi_device_list, scsi_device_list_entry) { in pqi_mark_volumes_for_rescan()
3740 if (pqi_is_logical_device(device) && device->devtype == TYPE_DISK) in pqi_mark_volumes_for_rescan()
3741 device->rescan = true; in pqi_mark_volumes_for_rescan()
3744 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags); in pqi_mark_volumes_for_rescan()
3752 spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags); in pqi_disable_raid_bypass()
3754 list_for_each_entry(device, &ctrl_info->scsi_device_list, scsi_device_list_entry) in pqi_disable_raid_bypass()
3755 if (device->raid_bypass_enabled) in pqi_disable_raid_bypass()
3756 device->raid_bypass_enabled = false; in pqi_disable_raid_bypass()
3758 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags); in pqi_disable_raid_bypass()
3777 event = ctrl_info->events; in pqi_event_worker()
3779 if (event->pending) { in pqi_event_worker()
3780 event->pending = false; in pqi_event_worker()
3781 if (event->event_type == PQI_EVENT_TYPE_OFA) { in pqi_event_worker()
3786 if (event->event_type == PQI_EVENT_TYPE_LOGICAL_DEVICE) in pqi_event_worker()
3788 else if (event->event_type == PQI_EVENT_TYPE_AIO_STATE_CHANGE) in pqi_event_worker()
3819 num_interrupts = atomic_read(&ctrl_info->num_interrupts); in pqi_heartbeat_timer_handler()
3822 if (num_interrupts == ctrl_info->previous_num_interrupts) { in pqi_heartbeat_timer_handler()
3823 if (heartbeat_count == ctrl_info->previous_heartbeat_count) { in pqi_heartbeat_timer_handler()
3824 dev_err(&ctrl_info->pci_dev->dev, in pqi_heartbeat_timer_handler()
3825 "no heartbeat detected - last heartbeat count: %u\n", in pqi_heartbeat_timer_handler()
3831 ctrl_info->previous_num_interrupts = num_interrupts; in pqi_heartbeat_timer_handler()
3834 ctrl_info->previous_heartbeat_count = heartbeat_count; in pqi_heartbeat_timer_handler()
3835 mod_timer(&ctrl_info->heartbeat_timer, in pqi_heartbeat_timer_handler()
3841 if (!ctrl_info->heartbeat_counter) in pqi_start_heartbeat_timer()
3844 ctrl_info->previous_num_interrupts = in pqi_start_heartbeat_timer()
3845 atomic_read(&ctrl_info->num_interrupts); in pqi_start_heartbeat_timer()
3846 ctrl_info->previous_heartbeat_count = in pqi_start_heartbeat_timer()
3849 ctrl_info->heartbeat_timer.expires = in pqi_start_heartbeat_timer()
3851 add_timer(&ctrl_info->heartbeat_timer); in pqi_start_heartbeat_timer()
3856 del_timer_sync(&ctrl_info->heartbeat_timer); in pqi_stop_heartbeat_timer()
3862 switch (event->event_id) { in pqi_ofa_capture_event_payload()
3864 ctrl_info->ofa_bytes_requested = in pqi_ofa_capture_event_payload()
3865 get_unaligned_le32(&response->data.ofa_memory_allocation.bytes_requested); in pqi_ofa_capture_event_payload()
3868 ctrl_info->ofa_cancel_reason = in pqi_ofa_capture_event_payload()
3869 get_unaligned_le16(&response->data.ofa_cancelled.reason); in pqi_ofa_capture_event_payload()
3884 event_queue = &ctrl_info->event_queue; in pqi_process_event_intr()
3886 oq_ci = event_queue->oq_ci_copy; in pqi_process_event_intr()
3889 oq_pi = readl(event_queue->oq_pi); in pqi_process_event_intr()
3892 dev_err(&ctrl_info->pci_dev->dev, in pqi_process_event_intr()
3893 "event interrupt: producer index (%u) out of range (0-%u): consumer index: %u\n", in pqi_process_event_intr()
3894 oq_pi, PQI_NUM_EVENT_QUEUE_ELEMENTS - 1, oq_ci); in pqi_process_event_intr()
3895 return -1; in pqi_process_event_intr()
3902 response = event_queue->oq_element_array + (oq_ci * PQI_EVENT_OQ_ELEMENT_LENGTH); in pqi_process_event_intr()
3904 event_index = pqi_event_type_to_event_index(response->event_type); in pqi_process_event_intr()
3906 if (event_index >= 0 && response->request_acknowledge) { in pqi_process_event_intr()
3907 event = &ctrl_info->events[event_index]; in pqi_process_event_intr()
3908 event->pending = true; in pqi_process_event_intr()
3909 event->event_type = response->event_type; in pqi_process_event_intr()
3910 event->event_id = get_unaligned_le16(&response->event_id); in pqi_process_event_intr()
3911 event->additional_event_id = in pqi_process_event_intr()
3912 get_unaligned_le32(&response->additional_event_id); in pqi_process_event_intr()
3913 if (event->event_type == PQI_EVENT_TYPE_OFA) in pqi_process_event_intr()
3921 event_queue->oq_ci_copy = oq_ci; in pqi_process_event_intr()
3922 writel(oq_ci, event_queue->oq_ci); in pqi_process_event_intr()
3923 schedule_work(&ctrl_info->event_work); in pqi_process_event_intr()
3937 pqi_registers = ctrl_info->pqi_registers; in pqi_configure_legacy_intx()
3940 register_addr = &pqi_registers->legacy_intx_mask_clear; in pqi_configure_legacy_intx()
3942 register_addr = &pqi_registers->legacy_intx_mask_set; in pqi_configure_legacy_intx()
3952 switch (ctrl_info->irq_mode) { in pqi_change_irq_mode()
3993 ctrl_info->irq_mode = new_mode; in pqi_change_irq_mode()
4003 switch (ctrl_info->irq_mode) { in pqi_is_valid_irq()
4008 intx_status = readl(&ctrl_info->pqi_registers->legacy_intx_status); in pqi_is_valid_irq()
4031 ctrl_info = queue_group->ctrl_info; in pqi_irq_handler()
4040 if (irq == ctrl_info->event_irq) { in pqi_irq_handler()
4049 atomic_inc(&ctrl_info->num_interrupts); in pqi_irq_handler()
4060 struct pci_dev *pci_dev = ctrl_info->pci_dev; in pqi_request_irqs()
4064 ctrl_info->event_irq = pci_irq_vector(pci_dev, 0); in pqi_request_irqs()
4066 for (i = 0; i < ctrl_info->num_msix_vectors_enabled; i++) { in pqi_request_irqs()
4068 DRIVER_NAME_SHORT, &ctrl_info->queue_groups[i]); in pqi_request_irqs()
4070 dev_err(&pci_dev->dev, in pqi_request_irqs()
4075 ctrl_info->num_msix_vectors_initialized++; in pqi_request_irqs()
4085 for (i = 0; i < ctrl_info->num_msix_vectors_initialized; i++) in pqi_free_irqs()
4086 free_irq(pci_irq_vector(ctrl_info->pci_dev, i), in pqi_free_irqs()
4087 &ctrl_info->queue_groups[i]); in pqi_free_irqs()
4089 ctrl_info->num_msix_vectors_initialized = 0; in pqi_free_irqs()
4100 num_vectors_enabled = pci_alloc_irq_vectors(ctrl_info->pci_dev, in pqi_enable_msix_interrupts()
4101 PQI_MIN_MSIX_VECTORS, ctrl_info->num_queue_groups, in pqi_enable_msix_interrupts()
4104 dev_err(&ctrl_info->pci_dev->dev, in pqi_enable_msix_interrupts()
4105 "MSI-X init failed with error %d\n", in pqi_enable_msix_interrupts()
4110 ctrl_info->num_msix_vectors_enabled = num_vectors_enabled; in pqi_enable_msix_interrupts()
4111 ctrl_info->irq_mode = IRQ_MODE_MSIX; in pqi_enable_msix_interrupts()
4117 if (ctrl_info->num_msix_vectors_enabled) { in pqi_disable_msix_interrupts()
4118 pci_free_irq_vectors(ctrl_info->pci_dev); in pqi_disable_msix_interrupts()
4119 ctrl_info->num_msix_vectors_enabled = 0; in pqi_disable_msix_interrupts()
4139 ctrl_info->num_elements_per_iq; in pqi_alloc_operational_queues()
4142 ctrl_info->num_elements_per_oq; in pqi_alloc_operational_queues()
4143 num_inbound_queues = ctrl_info->num_queue_groups * 2; in pqi_alloc_operational_queues()
4144 num_outbound_queues = ctrl_info->num_queue_groups; in pqi_alloc_operational_queues()
4145 num_queue_indexes = (ctrl_info->num_queue_groups * 3) + 1; in pqi_alloc_operational_queues()
4177 ctrl_info->queue_memory_base = in pqi_alloc_operational_queues()
4178 dma_alloc_coherent(&ctrl_info->pci_dev->dev, alloc_length, in pqi_alloc_operational_queues()
4179 &ctrl_info->queue_memory_base_dma_handle, in pqi_alloc_operational_queues()
4182 if (!ctrl_info->queue_memory_base) in pqi_alloc_operational_queues()
4183 return -ENOMEM; in pqi_alloc_operational_queues()
4185 ctrl_info->queue_memory_length = alloc_length; in pqi_alloc_operational_queues()
4187 element_array = PTR_ALIGN(ctrl_info->queue_memory_base, in pqi_alloc_operational_queues()
4190 for (i = 0; i < ctrl_info->num_queue_groups; i++) { in pqi_alloc_operational_queues()
4191 queue_group = &ctrl_info->queue_groups[i]; in pqi_alloc_operational_queues()
4192 queue_group->iq_element_array[RAID_PATH] = element_array; in pqi_alloc_operational_queues()
4193 queue_group->iq_element_array_bus_addr[RAID_PATH] = in pqi_alloc_operational_queues()
4194 ctrl_info->queue_memory_base_dma_handle + in pqi_alloc_operational_queues()
4195 (element_array - ctrl_info->queue_memory_base); in pqi_alloc_operational_queues()
4199 queue_group->iq_element_array[AIO_PATH] = element_array; in pqi_alloc_operational_queues()
4200 queue_group->iq_element_array_bus_addr[AIO_PATH] = in pqi_alloc_operational_queues()
4201 ctrl_info->queue_memory_base_dma_handle + in pqi_alloc_operational_queues()
4202 (element_array - ctrl_info->queue_memory_base); in pqi_alloc_operational_queues()
4208 for (i = 0; i < ctrl_info->num_queue_groups; i++) { in pqi_alloc_operational_queues()
4209 queue_group = &ctrl_info->queue_groups[i]; in pqi_alloc_operational_queues()
4210 queue_group->oq_element_array = element_array; in pqi_alloc_operational_queues()
4211 queue_group->oq_element_array_bus_addr = in pqi_alloc_operational_queues()
4212 ctrl_info->queue_memory_base_dma_handle + in pqi_alloc_operational_queues()
4213 (element_array - ctrl_info->queue_memory_base); in pqi_alloc_operational_queues()
4219 ctrl_info->event_queue.oq_element_array = element_array; in pqi_alloc_operational_queues()
4220 ctrl_info->event_queue.oq_element_array_bus_addr = in pqi_alloc_operational_queues()
4221 ctrl_info->queue_memory_base_dma_handle + in pqi_alloc_operational_queues()
4222 (element_array - ctrl_info->queue_memory_base); in pqi_alloc_operational_queues()
4229 for (i = 0; i < ctrl_info->num_queue_groups; i++) { in pqi_alloc_operational_queues()
4230 queue_group = &ctrl_info->queue_groups[i]; in pqi_alloc_operational_queues()
4231 queue_group->iq_ci[RAID_PATH] = next_queue_index; in pqi_alloc_operational_queues()
4232 queue_group->iq_ci_bus_addr[RAID_PATH] = in pqi_alloc_operational_queues()
4233 ctrl_info->queue_memory_base_dma_handle + in pqi_alloc_operational_queues()
4234 (next_queue_index - in pqi_alloc_operational_queues()
4235 (void __iomem *)ctrl_info->queue_memory_base); in pqi_alloc_operational_queues()
4239 queue_group->iq_ci[AIO_PATH] = next_queue_index; in pqi_alloc_operational_queues()
4240 queue_group->iq_ci_bus_addr[AIO_PATH] = in pqi_alloc_operational_queues()
4241 ctrl_info->queue_memory_base_dma_handle + in pqi_alloc_operational_queues()
4242 (next_queue_index - in pqi_alloc_operational_queues()
4243 (void __iomem *)ctrl_info->queue_memory_base); in pqi_alloc_operational_queues()
4247 queue_group->oq_pi = next_queue_index; in pqi_alloc_operational_queues()
4248 queue_group->oq_pi_bus_addr = in pqi_alloc_operational_queues()
4249 ctrl_info->queue_memory_base_dma_handle + in pqi_alloc_operational_queues()
4250 (next_queue_index - in pqi_alloc_operational_queues()
4251 (void __iomem *)ctrl_info->queue_memory_base); in pqi_alloc_operational_queues()
4257 ctrl_info->event_queue.oq_pi = next_queue_index; in pqi_alloc_operational_queues()
4258 ctrl_info->event_queue.oq_pi_bus_addr = in pqi_alloc_operational_queues()
4259 ctrl_info->queue_memory_base_dma_handle + in pqi_alloc_operational_queues()
4260 (next_queue_index - in pqi_alloc_operational_queues()
4261 (void __iomem *)ctrl_info->queue_memory_base); in pqi_alloc_operational_queues()
4276 for (i = 0; i < ctrl_info->num_queue_groups; i++) in pqi_init_operational_queues()
4277 ctrl_info->queue_groups[i].ctrl_info = ctrl_info; in pqi_init_operational_queues()
4284 ctrl_info->event_queue.oq_id = next_oq_id++; in pqi_init_operational_queues()
4285 for (i = 0; i < ctrl_info->num_queue_groups; i++) { in pqi_init_operational_queues()
4286 ctrl_info->queue_groups[i].iq_id[RAID_PATH] = next_iq_id++; in pqi_init_operational_queues()
4287 ctrl_info->queue_groups[i].iq_id[AIO_PATH] = next_iq_id++; in pqi_init_operational_queues()
4288 ctrl_info->queue_groups[i].oq_id = next_oq_id++; in pqi_init_operational_queues()
4292 * Assign MSI-X table entry indexes to all queues. Note that the in pqi_init_operational_queues()
4295 ctrl_info->event_queue.int_msg_num = 0; in pqi_init_operational_queues()
4296 for (i = 0; i < ctrl_info->num_queue_groups; i++) in pqi_init_operational_queues()
4297 ctrl_info->queue_groups[i].int_msg_num = i; in pqi_init_operational_queues()
4299 for (i = 0; i < ctrl_info->num_queue_groups; i++) { in pqi_init_operational_queues()
4300 spin_lock_init(&ctrl_info->queue_groups[i].submit_lock[0]); in pqi_init_operational_queues()
4301 spin_lock_init(&ctrl_info->queue_groups[i].submit_lock[1]); in pqi_init_operational_queues()
4302 INIT_LIST_HEAD(&ctrl_info->queue_groups[i].request_list[0]); in pqi_init_operational_queues()
4303 INIT_LIST_HEAD(&ctrl_info->queue_groups[i].request_list[1]); in pqi_init_operational_queues()
4316 ctrl_info->admin_queue_memory_base = in pqi_alloc_admin_queues()
4317 dma_alloc_coherent(&ctrl_info->pci_dev->dev, alloc_length, in pqi_alloc_admin_queues()
4318 &ctrl_info->admin_queue_memory_base_dma_handle, in pqi_alloc_admin_queues()
4321 if (!ctrl_info->admin_queue_memory_base) in pqi_alloc_admin_queues()
4322 return -ENOMEM; in pqi_alloc_admin_queues()
4324 ctrl_info->admin_queue_memory_length = alloc_length; in pqi_alloc_admin_queues()
4326 admin_queues = &ctrl_info->admin_queues; in pqi_alloc_admin_queues()
4327 admin_queues_aligned = PTR_ALIGN(ctrl_info->admin_queue_memory_base, in pqi_alloc_admin_queues()
4329 admin_queues->iq_element_array = in pqi_alloc_admin_queues()
4330 &admin_queues_aligned->iq_element_array; in pqi_alloc_admin_queues()
4331 admin_queues->oq_element_array = in pqi_alloc_admin_queues()
4332 &admin_queues_aligned->oq_element_array; in pqi_alloc_admin_queues()
4333 admin_queues->iq_ci = in pqi_alloc_admin_queues()
4334 (pqi_index_t __iomem *)&admin_queues_aligned->iq_ci; in pqi_alloc_admin_queues()
4335 admin_queues->oq_pi = in pqi_alloc_admin_queues()
4336 (pqi_index_t __iomem *)&admin_queues_aligned->oq_pi; in pqi_alloc_admin_queues()
4338 admin_queues->iq_element_array_bus_addr = in pqi_alloc_admin_queues()
4339 ctrl_info->admin_queue_memory_base_dma_handle + in pqi_alloc_admin_queues()
4340 (admin_queues->iq_element_array - in pqi_alloc_admin_queues()
4341 ctrl_info->admin_queue_memory_base); in pqi_alloc_admin_queues()
4342 admin_queues->oq_element_array_bus_addr = in pqi_alloc_admin_queues()
4343 ctrl_info->admin_queue_memory_base_dma_handle + in pqi_alloc_admin_queues()
4344 (admin_queues->oq_element_array - in pqi_alloc_admin_queues()
4345 ctrl_info->admin_queue_memory_base); in pqi_alloc_admin_queues()
4346 admin_queues->iq_ci_bus_addr = in pqi_alloc_admin_queues()
4347 ctrl_info->admin_queue_memory_base_dma_handle + in pqi_alloc_admin_queues()
4348 ((void __iomem *)admin_queues->iq_ci - in pqi_alloc_admin_queues()
4349 (void __iomem *)ctrl_info->admin_queue_memory_base); in pqi_alloc_admin_queues()
4350 admin_queues->oq_pi_bus_addr = in pqi_alloc_admin_queues()
4351 ctrl_info->admin_queue_memory_base_dma_handle + in pqi_alloc_admin_queues()
4352 ((void __iomem *)admin_queues->oq_pi - in pqi_alloc_admin_queues()
4353 (void __iomem *)ctrl_info->admin_queue_memory_base); in pqi_alloc_admin_queues()
4369 pqi_registers = ctrl_info->pqi_registers; in pqi_create_admin_queues()
4370 admin_queues = &ctrl_info->admin_queues; in pqi_create_admin_queues()
4372 writeq((u64)admin_queues->iq_element_array_bus_addr, in pqi_create_admin_queues()
4373 &pqi_registers->admin_iq_element_array_addr); in pqi_create_admin_queues()
4374 writeq((u64)admin_queues->oq_element_array_bus_addr, in pqi_create_admin_queues()
4375 &pqi_registers->admin_oq_element_array_addr); in pqi_create_admin_queues()
4376 writeq((u64)admin_queues->iq_ci_bus_addr, in pqi_create_admin_queues()
4377 &pqi_registers->admin_iq_ci_addr); in pqi_create_admin_queues()
4378 writeq((u64)admin_queues->oq_pi_bus_addr, in pqi_create_admin_queues()
4379 &pqi_registers->admin_oq_pi_addr); in pqi_create_admin_queues()
4383 (admin_queues->int_msg_num << 16); in pqi_create_admin_queues()
4384 writel(reg, &pqi_registers->admin_iq_num_elements); in pqi_create_admin_queues()
4387 &pqi_registers->function_and_status_code); in pqi_create_admin_queues()
4392 status = readb(&pqi_registers->function_and_status_code); in pqi_create_admin_queues()
4396 return -ETIMEDOUT; in pqi_create_admin_queues()
4404 admin_queues->iq_pi = ctrl_info->iomem_base + in pqi_create_admin_queues()
4406 readq(&pqi_registers->admin_iq_pi_offset); in pqi_create_admin_queues()
4407 admin_queues->oq_ci = ctrl_info->iomem_base + in pqi_create_admin_queues()
4409 readq(&pqi_registers->admin_oq_ci_offset); in pqi_create_admin_queues()
4421 admin_queues = &ctrl_info->admin_queues; in pqi_submit_admin_request()
4422 iq_pi = admin_queues->iq_pi_copy; in pqi_submit_admin_request()
4424 next_element = admin_queues->iq_element_array + in pqi_submit_admin_request()
4430 admin_queues->iq_pi_copy = iq_pi; in pqi_submit_admin_request()
4436 writel(iq_pi, admin_queues->iq_pi); in pqi_submit_admin_request()
4449 admin_queues = &ctrl_info->admin_queues; in pqi_poll_for_admin_response()
4450 oq_ci = admin_queues->oq_ci_copy; in pqi_poll_for_admin_response()
4455 oq_pi = readl(admin_queues->oq_pi); in pqi_poll_for_admin_response()
4459 dev_err(&ctrl_info->pci_dev->dev, in pqi_poll_for_admin_response()
4461 return -ETIMEDOUT; in pqi_poll_for_admin_response()
4464 return -ENXIO; in pqi_poll_for_admin_response()
4468 memcpy(response, admin_queues->oq_element_array + in pqi_poll_for_admin_response()
4472 admin_queues->oq_ci_copy = oq_ci; in pqi_poll_for_admin_response()
4473 writel(oq_ci, admin_queues->oq_ci); in pqi_poll_for_admin_response()
4493 spin_lock_irqsave(&queue_group->submit_lock[path], flags); in pqi_start_io()
4496 io_request->queue_group = queue_group; in pqi_start_io()
4497 list_add_tail(&io_request->request_list_entry, in pqi_start_io()
4498 &queue_group->request_list[path]); in pqi_start_io()
4501 iq_pi = queue_group->iq_pi_copy[path]; in pqi_start_io()
4504 &queue_group->request_list[path], request_list_entry) { in pqi_start_io()
4506 request = io_request->iu; in pqi_start_io()
4508 iu_length = get_unaligned_le16(&request->iu_length) + in pqi_start_io()
4514 iq_ci = readl(queue_group->iq_ci[path]); in pqi_start_io()
4517 ctrl_info->num_elements_per_iq)) in pqi_start_io()
4520 put_unaligned_le16(queue_group->oq_id, in pqi_start_io()
4521 &request->response_queue_id); in pqi_start_io()
4523 next_element = queue_group->iq_element_array[path] + in pqi_start_io()
4527 ctrl_info->num_elements_per_iq - iq_pi; in pqi_start_io()
4535 memcpy(queue_group->iq_element_array[path], in pqi_start_io()
4537 iu_length - copy_count); in pqi_start_io()
4541 ctrl_info->num_elements_per_iq; in pqi_start_io()
4543 list_del(&io_request->request_list_entry); in pqi_start_io()
4546 if (iq_pi != queue_group->iq_pi_copy[path]) { in pqi_start_io()
4547 queue_group->iq_pi_copy[path] = iq_pi; in pqi_start_io()
4552 writel(iq_pi, queue_group->iq_pi[path]); in pqi_start_io()
4555 spin_unlock_irqrestore(&queue_group->submit_lock[path], flags); in pqi_start_io()
4574 rc = -ENXIO; in pqi_wait_for_completion_io()
4593 int rc = -EIO; in pqi_process_raid_io_error_synchronous()
4595 switch (error_info->data_out_result) { in pqi_process_raid_io_error_synchronous()
4597 if (error_info->status == SAM_STAT_GOOD) in pqi_process_raid_io_error_synchronous()
4601 if (error_info->status == SAM_STAT_GOOD || in pqi_process_raid_io_error_synchronous()
4602 error_info->status == SAM_STAT_CHECK_CONDITION) in pqi_process_raid_io_error_synchronous()
4615 return (request->driver_flags & PQI_DRIVER_NONBLOCKABLE_REQUEST) == 0; in pqi_is_blockable_request()
4628 if (down_interruptible(&ctrl_info->sync_request_sem)) in pqi_submit_raid_request_synchronous()
4629 return -ERESTARTSYS; in pqi_submit_raid_request_synchronous()
4631 down(&ctrl_info->sync_request_sem); in pqi_submit_raid_request_synchronous()
4643 rc = -ENXIO; in pqi_submit_raid_request_synchronous()
4649 put_unaligned_le16(io_request->index, in pqi_submit_raid_request_synchronous()
4650 &(((struct pqi_raid_path_request *)request)->request_id)); in pqi_submit_raid_request_synchronous()
4652 if (request->iu_type == PQI_REQUEST_IU_RAID_PATH_IO) in pqi_submit_raid_request_synchronous()
4653 ((struct pqi_raid_path_request *)request)->error_index = in pqi_submit_raid_request_synchronous()
4654 ((struct pqi_raid_path_request *)request)->request_id; in pqi_submit_raid_request_synchronous()
4656 iu_length = get_unaligned_le16(&request->iu_length) + in pqi_submit_raid_request_synchronous()
4658 memcpy(io_request->iu, request, iu_length); in pqi_submit_raid_request_synchronous()
4660 io_request->io_complete_callback = pqi_raid_synchronous_complete; in pqi_submit_raid_request_synchronous()
4661 io_request->context = &wait; in pqi_submit_raid_request_synchronous()
4663 pqi_start_io(ctrl_info, &ctrl_info->queue_groups[PQI_DEFAULT_QUEUE_GROUP], RAID_PATH, in pqi_submit_raid_request_synchronous()
4669 if (io_request->error_info) in pqi_submit_raid_request_synchronous()
4670 memcpy(error_info, io_request->error_info, sizeof(*error_info)); in pqi_submit_raid_request_synchronous()
4673 } else if (rc == 0 && io_request->error_info) { in pqi_submit_raid_request_synchronous()
4674 rc = pqi_process_raid_io_error_synchronous(io_request->error_info); in pqi_submit_raid_request_synchronous()
4681 up(&ctrl_info->sync_request_sem); in pqi_submit_raid_request_synchronous()
4689 if (response->header.iu_type != PQI_RESPONSE_IU_GENERAL_ADMIN) in pqi_validate_admin_response()
4690 return -EINVAL; in pqi_validate_admin_response()
4692 if (get_unaligned_le16(&response->header.iu_length) != in pqi_validate_admin_response()
4694 return -EINVAL; in pqi_validate_admin_response()
4696 if (response->function_code != expected_function_code) in pqi_validate_admin_response()
4697 return -EINVAL; in pqi_validate_admin_response()
4699 if (response->status != PQI_GENERAL_ADMIN_STATUS_SUCCESS) in pqi_validate_admin_response()
4700 return -EINVAL; in pqi_validate_admin_response()
4717 rc = pqi_validate_admin_response(response, request->function_code); in pqi_submit_admin_request_synchronous()
4732 return -ENOMEM; in pqi_report_device_capability()
4744 rc = pqi_map_single(ctrl_info->pci_dev, in pqi_report_device_capability()
4753 pqi_pci_unmap(ctrl_info->pci_dev, in pqi_report_device_capability()
4761 rc = -EIO; in pqi_report_device_capability()
4765 ctrl_info->max_inbound_queues = in pqi_report_device_capability()
4766 get_unaligned_le16(&capability->max_inbound_queues); in pqi_report_device_capability()
4767 ctrl_info->max_elements_per_iq = in pqi_report_device_capability()
4768 get_unaligned_le16(&capability->max_elements_per_iq); in pqi_report_device_capability()
4769 ctrl_info->max_iq_element_length = in pqi_report_device_capability()
4770 get_unaligned_le16(&capability->max_iq_element_length) in pqi_report_device_capability()
4772 ctrl_info->max_outbound_queues = in pqi_report_device_capability()
4773 get_unaligned_le16(&capability->max_outbound_queues); in pqi_report_device_capability()
4774 ctrl_info->max_elements_per_oq = in pqi_report_device_capability()
4775 get_unaligned_le16(&capability->max_elements_per_oq); in pqi_report_device_capability()
4776 ctrl_info->max_oq_element_length = in pqi_report_device_capability()
4777 get_unaligned_le16(&capability->max_oq_element_length) in pqi_report_device_capability()
4781 &capability->iu_layer_descriptors[PQI_PROTOCOL_SOP]; in pqi_report_device_capability()
4783 ctrl_info->max_inbound_iu_length_per_firmware = in pqi_report_device_capability()
4785 &sop_iu_layer_descriptor->max_inbound_iu_length); in pqi_report_device_capability()
4786 ctrl_info->inbound_spanning_supported = in pqi_report_device_capability()
4787 sop_iu_layer_descriptor->inbound_spanning_supported; in pqi_report_device_capability()
4788 ctrl_info->outbound_spanning_supported = in pqi_report_device_capability()
4789 sop_iu_layer_descriptor->outbound_spanning_supported; in pqi_report_device_capability()
4799 if (ctrl_info->max_iq_element_length < in pqi_validate_device_capability()
4801 dev_err(&ctrl_info->pci_dev->dev, in pqi_validate_device_capability()
4803 ctrl_info->max_iq_element_length, in pqi_validate_device_capability()
4805 return -EINVAL; in pqi_validate_device_capability()
4808 if (ctrl_info->max_oq_element_length < in pqi_validate_device_capability()
4810 dev_err(&ctrl_info->pci_dev->dev, in pqi_validate_device_capability()
4812 ctrl_info->max_oq_element_length, in pqi_validate_device_capability()
4814 return -EINVAL; in pqi_validate_device_capability()
4817 if (ctrl_info->max_inbound_iu_length_per_firmware < in pqi_validate_device_capability()
4819 dev_err(&ctrl_info->pci_dev->dev, in pqi_validate_device_capability()
4821 ctrl_info->max_inbound_iu_length_per_firmware, in pqi_validate_device_capability()
4823 return -EINVAL; in pqi_validate_device_capability()
4826 if (!ctrl_info->inbound_spanning_supported) { in pqi_validate_device_capability()
4827 dev_err(&ctrl_info->pci_dev->dev, in pqi_validate_device_capability()
4829 return -EINVAL; in pqi_validate_device_capability()
4832 if (ctrl_info->outbound_spanning_supported) { in pqi_validate_device_capability()
4833 dev_err(&ctrl_info->pci_dev->dev, in pqi_validate_device_capability()
4835 return -EINVAL; in pqi_validate_device_capability()
4848 event_queue = &ctrl_info->event_queue; in pqi_create_event_queue()
4851 * Create OQ (Outbound Queue - device to host queue) to dedicate in pqi_create_event_queue()
4859 put_unaligned_le16(event_queue->oq_id, in pqi_create_event_queue()
4861 put_unaligned_le64((u64)event_queue->oq_element_array_bus_addr, in pqi_create_event_queue()
4863 put_unaligned_le64((u64)event_queue->oq_pi_bus_addr, in pqi_create_event_queue()
4870 put_unaligned_le16(event_queue->int_msg_num, in pqi_create_event_queue()
4878 event_queue->oq_ci = ctrl_info->iomem_base + in pqi_create_event_queue()
4894 queue_group = &ctrl_info->queue_groups[group_number]; in pqi_create_queue_group()
4897 * Create IQ (Inbound Queue - host to device queue) for in pqi_create_queue_group()
4905 put_unaligned_le16(queue_group->iq_id[RAID_PATH], in pqi_create_queue_group()
4908 (u64)queue_group->iq_element_array_bus_addr[RAID_PATH], in pqi_create_queue_group()
4910 put_unaligned_le64((u64)queue_group->iq_ci_bus_addr[RAID_PATH], in pqi_create_queue_group()
4912 put_unaligned_le16(ctrl_info->num_elements_per_iq, in pqi_create_queue_group()
4921 dev_err(&ctrl_info->pci_dev->dev, in pqi_create_queue_group()
4926 queue_group->iq_pi[RAID_PATH] = ctrl_info->iomem_base + in pqi_create_queue_group()
4932 * Create IQ (Inbound Queue - host to device queue) for in pqi_create_queue_group()
4940 put_unaligned_le16(queue_group->iq_id[AIO_PATH], in pqi_create_queue_group()
4942 put_unaligned_le64((u64)queue_group-> in pqi_create_queue_group()
4945 put_unaligned_le64((u64)queue_group->iq_ci_bus_addr[AIO_PATH], in pqi_create_queue_group()
4947 put_unaligned_le16(ctrl_info->num_elements_per_iq, in pqi_create_queue_group()
4956 dev_err(&ctrl_info->pci_dev->dev, in pqi_create_queue_group()
4961 queue_group->iq_pi[AIO_PATH] = ctrl_info->iomem_base + in pqi_create_queue_group()
4976 put_unaligned_le16(queue_group->iq_id[AIO_PATH], in pqi_create_queue_group()
4984 dev_err(&ctrl_info->pci_dev->dev, in pqi_create_queue_group()
4990 * Create OQ (Outbound Queue - device to host queue). in pqi_create_queue_group()
4997 put_unaligned_le16(queue_group->oq_id, in pqi_create_queue_group()
4999 put_unaligned_le64((u64)queue_group->oq_element_array_bus_addr, in pqi_create_queue_group()
5001 put_unaligned_le64((u64)queue_group->oq_pi_bus_addr, in pqi_create_queue_group()
5003 put_unaligned_le16(ctrl_info->num_elements_per_oq, in pqi_create_queue_group()
5008 put_unaligned_le16(queue_group->int_msg_num, in pqi_create_queue_group()
5014 dev_err(&ctrl_info->pci_dev->dev, in pqi_create_queue_group()
5019 queue_group->oq_ci = ctrl_info->iomem_base + in pqi_create_queue_group()
5034 dev_err(&ctrl_info->pci_dev->dev, in pqi_create_queues()
5039 for (i = 0; i < ctrl_info->num_queue_groups; i++) { in pqi_create_queues()
5042 dev_err(&ctrl_info->pci_dev->dev, in pqi_create_queues()
5044 i, ctrl_info->num_queue_groups); in pqi_create_queues()
5067 return -ENOMEM; in pqi_configure_events()
5073 data.report_event_configuration.sg_descriptors[1]) - in pqi_configure_events()
5078 rc = pqi_map_single(ctrl_info->pci_dev, in pqi_configure_events()
5087 pqi_pci_unmap(ctrl_info->pci_dev, in pqi_configure_events()
5094 for (i = 0; i < event_config->num_event_descriptors; i++) { in pqi_configure_events()
5095 event_descriptor = &event_config->descriptors[i]; in pqi_configure_events()
5097 pqi_is_supported_event(event_descriptor->event_type)) in pqi_configure_events()
5098 put_unaligned_le16(ctrl_info->event_queue.oq_id, in pqi_configure_events()
5099 &event_descriptor->oq_id); in pqi_configure_events()
5101 put_unaligned_le16(0, &event_descriptor->oq_id); in pqi_configure_events()
5108 data.report_event_configuration.sg_descriptors[1]) - in pqi_configure_events()
5113 rc = pqi_map_single(ctrl_info->pci_dev, in pqi_configure_events()
5122 pqi_pci_unmap(ctrl_info->pci_dev, in pqi_configure_events()
5144 if (!ctrl_info->io_request_pool) in pqi_free_all_io_requests()
5147 dev = &ctrl_info->pci_dev->dev; in pqi_free_all_io_requests()
5148 sg_chain_buffer_length = ctrl_info->sg_chain_buffer_length; in pqi_free_all_io_requests()
5149 io_request = ctrl_info->io_request_pool; in pqi_free_all_io_requests()
5151 for (i = 0; i < ctrl_info->max_io_slots; i++) { in pqi_free_all_io_requests()
5152 kfree(io_request->iu); in pqi_free_all_io_requests()
5153 if (!io_request->sg_chain_buffer) in pqi_free_all_io_requests()
5156 io_request->sg_chain_buffer, in pqi_free_all_io_requests()
5157 io_request->sg_chain_buffer_dma_handle); in pqi_free_all_io_requests()
5161 kfree(ctrl_info->io_request_pool); in pqi_free_all_io_requests()
5162 ctrl_info->io_request_pool = NULL; in pqi_free_all_io_requests()
5167 ctrl_info->error_buffer = dma_alloc_coherent(&ctrl_info->pci_dev->dev, in pqi_alloc_error_buffer()
5168 ctrl_info->error_buffer_length, in pqi_alloc_error_buffer()
5169 &ctrl_info->error_buffer_dma_handle, in pqi_alloc_error_buffer()
5171 if (!ctrl_info->error_buffer) in pqi_alloc_error_buffer()
5172 return -ENOMEM; in pqi_alloc_error_buffer()
5186 ctrl_info->io_request_pool = kcalloc(ctrl_info->max_io_slots, in pqi_alloc_io_resources()
5187 sizeof(ctrl_info->io_request_pool[0]), GFP_KERNEL); in pqi_alloc_io_resources()
5189 if (!ctrl_info->io_request_pool) { in pqi_alloc_io_resources()
5190 dev_err(&ctrl_info->pci_dev->dev, in pqi_alloc_io_resources()
5195 dev = &ctrl_info->pci_dev->dev; in pqi_alloc_io_resources()
5196 sg_chain_buffer_length = ctrl_info->sg_chain_buffer_length; in pqi_alloc_io_resources()
5197 io_request = ctrl_info->io_request_pool; in pqi_alloc_io_resources()
5199 for (i = 0; i < ctrl_info->max_io_slots; i++) { in pqi_alloc_io_resources()
5200 io_request->iu = kmalloc(ctrl_info->max_inbound_iu_length, GFP_KERNEL); in pqi_alloc_io_resources()
5202 if (!io_request->iu) { in pqi_alloc_io_resources()
5203 dev_err(&ctrl_info->pci_dev->dev, in pqi_alloc_io_resources()
5213 dev_err(&ctrl_info->pci_dev->dev, in pqi_alloc_io_resources()
5214 "failed to allocate PQI scatter-gather chain buffers\n"); in pqi_alloc_io_resources()
5218 io_request->index = i; in pqi_alloc_io_resources()
5219 io_request->sg_chain_buffer = sg_chain_buffer; in pqi_alloc_io_resources()
5220 io_request->sg_chain_buffer_dma_handle = sg_chain_buffer_dma_handle; in pqi_alloc_io_resources()
5229 return -ENOMEM; in pqi_alloc_io_resources()
5234 * requests and max. transfer size.
5242 ctrl_info->scsi_ml_can_queue = in pqi_calculate_io_resources()
5243 ctrl_info->max_outstanding_requests - PQI_RESERVED_IO_SLOTS; in pqi_calculate_io_resources()
5244 ctrl_info->max_io_slots = ctrl_info->max_outstanding_requests; in pqi_calculate_io_resources()
5246 ctrl_info->error_buffer_length = in pqi_calculate_io_resources()
5247 ctrl_info->max_io_slots * PQI_ERROR_BUFFER_ELEMENT_LENGTH; in pqi_calculate_io_resources()
5250 max_transfer_size = min(ctrl_info->max_transfer_size, in pqi_calculate_io_resources()
5253 max_transfer_size = min(ctrl_info->max_transfer_size, in pqi_calculate_io_resources()
5258 /* +1 to cover when the buffer is not page-aligned. */ in pqi_calculate_io_resources()
5261 max_sg_entries = min(ctrl_info->max_sg_entries, max_sg_entries); in pqi_calculate_io_resources()
5263 max_transfer_size = (max_sg_entries - 1) * PAGE_SIZE; in pqi_calculate_io_resources()
5265 ctrl_info->sg_chain_buffer_length = in pqi_calculate_io_resources()
5268 ctrl_info->sg_tablesize = max_sg_entries; in pqi_calculate_io_resources()
5269 ctrl_info->max_sectors = max_transfer_size / 512; in pqi_calculate_io_resources()
5284 max_queue_groups = min(ctrl_info->max_inbound_queues / 2, in pqi_calculate_queue_resources()
5285 ctrl_info->max_outbound_queues - 1); in pqi_calculate_queue_resources()
5289 num_queue_groups = min(num_cpus, ctrl_info->max_msix_vectors); in pqi_calculate_queue_resources()
5293 ctrl_info->num_queue_groups = num_queue_groups; in pqi_calculate_queue_resources()
5299 ctrl_info->max_inbound_iu_length = in pqi_calculate_queue_resources()
5300 (ctrl_info->max_inbound_iu_length_per_firmware / in pqi_calculate_queue_resources()
5305 (ctrl_info->max_inbound_iu_length / in pqi_calculate_queue_resources()
5312 ctrl_info->max_elements_per_iq); in pqi_calculate_queue_resources()
5314 num_elements_per_oq = ((num_elements_per_iq - 1) * 2) + 1; in pqi_calculate_queue_resources()
5316 ctrl_info->max_elements_per_oq); in pqi_calculate_queue_resources()
5318 ctrl_info->num_elements_per_iq = num_elements_per_iq; in pqi_calculate_queue_resources()
5319 ctrl_info->num_elements_per_oq = num_elements_per_oq; in pqi_calculate_queue_resources()
5321 ctrl_info->max_sg_per_iu = in pqi_calculate_queue_resources()
5322 ((ctrl_info->max_inbound_iu_length - in pqi_calculate_queue_resources()
5327 ctrl_info->max_sg_per_r56_iu = in pqi_calculate_queue_resources()
5328 ((ctrl_info->max_inbound_iu_length - in pqi_calculate_queue_resources()
5340 put_unaligned_le64(address, &sg_descriptor->address); in pqi_set_sg_descriptor()
5341 put_unaligned_le32(length, &sg_descriptor->length); in pqi_set_sg_descriptor()
5342 put_unaligned_le32(0, &sg_descriptor->flags); in pqi_set_sg_descriptor()
5355 max_sg_per_iu--; /* Subtract 1 to leave room for chain marker. */ in pqi_build_sg_list()
5366 put_unaligned_le64((u64)io_request->sg_chain_buffer_dma_handle, in pqi_build_sg_list()
5367 &sg_descriptor->address); in pqi_build_sg_list()
5368 put_unaligned_le32((sg_count - num_sg_in_iu) * sizeof(*sg_descriptor), in pqi_build_sg_list()
5369 &sg_descriptor->length); in pqi_build_sg_list()
5370 put_unaligned_le32(CISS_SG_CHAIN, &sg_descriptor->flags); in pqi_build_sg_list()
5373 sg_descriptor = io_request->sg_chain_buffer; in pqi_build_sg_list()
5378 put_unaligned_le32(CISS_SG_LAST, &sg_descriptor->flags); in pqi_build_sg_list()
5398 iu_length = offsetof(struct pqi_raid_path_request, sg_descriptors) - in pqi_build_raid_sg_list()
5405 sg_descriptor = request->sg_descriptors; in pqi_build_raid_sg_list()
5408 ctrl_info->max_sg_per_iu, &chained); in pqi_build_raid_sg_list()
5410 request->partial = chained; in pqi_build_raid_sg_list()
5414 put_unaligned_le16(iu_length, &request->header.iu_length); in pqi_build_raid_sg_list()
5434 iu_length = offsetof(struct pqi_aio_r1_path_request, sg_descriptors) - in pqi_build_aio_r1_sg_list()
5442 sg_descriptor = request->sg_descriptors; in pqi_build_aio_r1_sg_list()
5445 ctrl_info->max_sg_per_iu, &chained); in pqi_build_aio_r1_sg_list()
5447 request->partial = chained; in pqi_build_aio_r1_sg_list()
5451 put_unaligned_le16(iu_length, &request->header.iu_length); in pqi_build_aio_r1_sg_list()
5452 request->num_sg_descriptors = num_sg_in_iu; in pqi_build_aio_r1_sg_list()
5472 iu_length = offsetof(struct pqi_aio_r56_path_request, sg_descriptors) - in pqi_build_aio_r56_sg_list()
5478 sg_descriptor = request->sg_descriptors; in pqi_build_aio_r56_sg_list()
5481 ctrl_info->max_sg_per_r56_iu, &chained); in pqi_build_aio_r56_sg_list()
5483 request->partial = chained; in pqi_build_aio_r56_sg_list()
5487 put_unaligned_le16(iu_length, &request->header.iu_length); in pqi_build_aio_r56_sg_list()
5488 request->num_sg_descriptors = num_sg_in_iu; in pqi_build_aio_r56_sg_list()
5508 iu_length = offsetof(struct pqi_aio_path_request, sg_descriptors) - in pqi_build_aio_sg_list()
5516 sg_descriptor = request->sg_descriptors; in pqi_build_aio_sg_list()
5519 ctrl_info->max_sg_per_iu, &chained); in pqi_build_aio_sg_list()
5521 request->partial = chained; in pqi_build_aio_sg_list()
5525 put_unaligned_le16(iu_length, &request->header.iu_length); in pqi_build_aio_sg_list()
5526 request->num_sg_descriptors = num_sg_in_iu; in pqi_build_aio_sg_list()
5536 scmd = io_request->scmd; in pqi_raid_io_complete()
5555 io_request->io_complete_callback = pqi_raid_io_complete; in pqi_raid_submit_io()
5556 io_request->scmd = scmd; in pqi_raid_submit_io()
5558 request = io_request->iu; in pqi_raid_submit_io()
5561 request->header.iu_type = PQI_REQUEST_IU_RAID_PATH_IO; in pqi_raid_submit_io()
5562 put_unaligned_le32(scsi_bufflen(scmd), &request->buffer_length); in pqi_raid_submit_io()
5563 request->task_attribute = SOP_TASK_ATTRIBUTE_SIMPLE; in pqi_raid_submit_io()
5564 request->command_priority = io_high_prio; in pqi_raid_submit_io()
5565 put_unaligned_le16(io_request->index, &request->request_id); in pqi_raid_submit_io()
5566 request->error_index = request->request_id; in pqi_raid_submit_io()
5567 memcpy(request->lun_number, device->scsi3addr, sizeof(request->lun_number)); in pqi_raid_submit_io()
5568 request->ml_device_lun_number = (u8)scmd->device->lun; in pqi_raid_submit_io()
5570 cdb_length = min_t(size_t, scmd->cmd_len, sizeof(request->cdb)); in pqi_raid_submit_io()
5571 memcpy(request->cdb, scmd->cmnd, cdb_length); in pqi_raid_submit_io()
5578 request->additional_cdb_bytes_usage = SOP_ADDITIONAL_CDB_BYTES_0; in pqi_raid_submit_io()
5581 request->additional_cdb_bytes_usage = SOP_ADDITIONAL_CDB_BYTES_4; in pqi_raid_submit_io()
5584 request->additional_cdb_bytes_usage = SOP_ADDITIONAL_CDB_BYTES_8; in pqi_raid_submit_io()
5587 request->additional_cdb_bytes_usage = SOP_ADDITIONAL_CDB_BYTES_12; in pqi_raid_submit_io()
5591 request->additional_cdb_bytes_usage = SOP_ADDITIONAL_CDB_BYTES_16; in pqi_raid_submit_io()
5595 switch (scmd->sc_data_direction) { in pqi_raid_submit_io()
5597 request->data_direction = SOP_READ_FLAG; in pqi_raid_submit_io()
5600 request->data_direction = SOP_WRITE_FLAG; in pqi_raid_submit_io()
5603 request->data_direction = SOP_NO_DIRECTION_FLAG; in pqi_raid_submit_io()
5606 request->data_direction = SOP_BIDIRECTIONAL; in pqi_raid_submit_io()
5609 dev_err(&ctrl_info->pci_dev->dev, in pqi_raid_submit_io()
5611 scmd->sc_data_direction); in pqi_raid_submit_io()
5643 if (!io_request->raid_bypass) in pqi_raid_bypass_retry_needed()
5646 scmd = io_request->scmd; in pqi_raid_bypass_retry_needed()
5647 if ((scmd->result & 0xff) == SAM_STAT_GOOD) in pqi_raid_bypass_retry_needed()
5649 if (host_byte(scmd->result) == DID_NO_CONNECT) in pqi_raid_bypass_retry_needed()
5652 device = scmd->device->hostdata; in pqi_raid_bypass_retry_needed()
5656 ctrl_info = shost_to_hba(scmd->device->host); in pqi_raid_bypass_retry_needed()
5668 scmd = io_request->scmd; in pqi_aio_io_complete()
5670 if (io_request->status == -EAGAIN || pqi_raid_bypass_retry_needed(io_request)) { in pqi_aio_io_complete()
5672 pqi_cmd_priv(scmd)->this_residual++; in pqi_aio_io_complete()
5687 return pqi_aio_submit_io(ctrl_info, scmd, device->aio_handle, in pqi_aio_submit_scsi_cmd()
5688 scmd->cmnd, scmd->cmd_len, queue_group, NULL, in pqi_aio_submit_scsi_cmd()
5706 io_request->io_complete_callback = pqi_aio_io_complete; in pqi_aio_submit_io()
5707 io_request->scmd = scmd; in pqi_aio_submit_io()
5708 io_request->raid_bypass = raid_bypass; in pqi_aio_submit_io()
5710 request = io_request->iu; in pqi_aio_submit_io()
5713 request->header.iu_type = PQI_REQUEST_IU_AIO_PATH_IO; in pqi_aio_submit_io()
5714 put_unaligned_le32(aio_handle, &request->nexus_id); in pqi_aio_submit_io()
5715 put_unaligned_le32(scsi_bufflen(scmd), &request->buffer_length); in pqi_aio_submit_io()
5716 request->task_attribute = SOP_TASK_ATTRIBUTE_SIMPLE; in pqi_aio_submit_io()
5717 request->command_priority = io_high_prio; in pqi_aio_submit_io()
5718 put_unaligned_le16(io_request->index, &request->request_id); in pqi_aio_submit_io()
5719 request->error_index = request->request_id; in pqi_aio_submit_io()
5720 if (!raid_bypass && ctrl_info->multi_lun_device_supported) in pqi_aio_submit_io()
5721 put_unaligned_le64(scmd->device->lun << 8, &request->lun_number); in pqi_aio_submit_io()
5722 if (cdb_length > sizeof(request->cdb)) in pqi_aio_submit_io()
5723 cdb_length = sizeof(request->cdb); in pqi_aio_submit_io()
5724 request->cdb_length = cdb_length; in pqi_aio_submit_io()
5725 memcpy(request->cdb, cdb, cdb_length); in pqi_aio_submit_io()
5727 switch (scmd->sc_data_direction) { in pqi_aio_submit_io()
5729 request->data_direction = SOP_READ_FLAG; in pqi_aio_submit_io()
5732 request->data_direction = SOP_WRITE_FLAG; in pqi_aio_submit_io()
5735 request->data_direction = SOP_NO_DIRECTION_FLAG; in pqi_aio_submit_io()
5738 request->data_direction = SOP_BIDIRECTIONAL; in pqi_aio_submit_io()
5741 dev_err(&ctrl_info->pci_dev->dev, in pqi_aio_submit_io()
5743 scmd->sc_data_direction); in pqi_aio_submit_io()
5748 request->encryption_enable = true; in pqi_aio_submit_io()
5749 put_unaligned_le16(encryption_info->data_encryption_key_index, in pqi_aio_submit_io()
5750 &request->data_encryption_key_index); in pqi_aio_submit_io()
5751 put_unaligned_le32(encryption_info->encrypt_tweak_lower, in pqi_aio_submit_io()
5752 &request->encrypt_tweak_lower); in pqi_aio_submit_io()
5753 put_unaligned_le32(encryption_info->encrypt_tweak_upper, in pqi_aio_submit_io()
5754 &request->encrypt_tweak_upper); in pqi_aio_submit_io()
5781 io_request->io_complete_callback = pqi_aio_io_complete; in pqi_aio_submit_r1_write_io()
5782 io_request->scmd = scmd; in pqi_aio_submit_r1_write_io()
5783 io_request->raid_bypass = true; in pqi_aio_submit_r1_write_io()
5785 r1_request = io_request->iu; in pqi_aio_submit_r1_write_io()
5788 r1_request->header.iu_type = PQI_REQUEST_IU_AIO_PATH_RAID1_IO; in pqi_aio_submit_r1_write_io()
5789 put_unaligned_le16(*(u16 *)device->scsi3addr & 0x3fff, &r1_request->volume_id); in pqi_aio_submit_r1_write_io()
5790 r1_request->num_drives = rmd->num_it_nexus_entries; in pqi_aio_submit_r1_write_io()
5791 put_unaligned_le32(rmd->it_nexus[0], &r1_request->it_nexus_1); in pqi_aio_submit_r1_write_io()
5792 put_unaligned_le32(rmd->it_nexus[1], &r1_request->it_nexus_2); in pqi_aio_submit_r1_write_io()
5793 if (rmd->num_it_nexus_entries == 3) in pqi_aio_submit_r1_write_io()
5794 put_unaligned_le32(rmd->it_nexus[2], &r1_request->it_nexus_3); in pqi_aio_submit_r1_write_io()
5796 put_unaligned_le32(scsi_bufflen(scmd), &r1_request->data_length); in pqi_aio_submit_r1_write_io()
5797 r1_request->task_attribute = SOP_TASK_ATTRIBUTE_SIMPLE; in pqi_aio_submit_r1_write_io()
5798 put_unaligned_le16(io_request->index, &r1_request->request_id); in pqi_aio_submit_r1_write_io()
5799 r1_request->error_index = r1_request->request_id; in pqi_aio_submit_r1_write_io()
5800 if (rmd->cdb_length > sizeof(r1_request->cdb)) in pqi_aio_submit_r1_write_io()
5801 rmd->cdb_length = sizeof(r1_request->cdb); in pqi_aio_submit_r1_write_io()
5802 r1_request->cdb_length = rmd->cdb_length; in pqi_aio_submit_r1_write_io()
5803 memcpy(r1_request->cdb, rmd->cdb, rmd->cdb_length); in pqi_aio_submit_r1_write_io()
5806 r1_request->data_direction = SOP_READ_FLAG; in pqi_aio_submit_r1_write_io()
5809 r1_request->encryption_enable = true; in pqi_aio_submit_r1_write_io()
5810 put_unaligned_le16(encryption_info->data_encryption_key_index, in pqi_aio_submit_r1_write_io()
5811 &r1_request->data_encryption_key_index); in pqi_aio_submit_r1_write_io()
5812 put_unaligned_le32(encryption_info->encrypt_tweak_lower, in pqi_aio_submit_r1_write_io()
5813 &r1_request->encrypt_tweak_lower); in pqi_aio_submit_r1_write_io()
5814 put_unaligned_le32(encryption_info->encrypt_tweak_upper, in pqi_aio_submit_r1_write_io()
5815 &r1_request->encrypt_tweak_upper); in pqi_aio_submit_r1_write_io()
5841 io_request->io_complete_callback = pqi_aio_io_complete; in pqi_aio_submit_r56_write_io()
5842 io_request->scmd = scmd; in pqi_aio_submit_r56_write_io()
5843 io_request->raid_bypass = true; in pqi_aio_submit_r56_write_io()
5845 r56_request = io_request->iu; in pqi_aio_submit_r56_write_io()
5848 if (device->raid_level == SA_RAID_5 || device->raid_level == SA_RAID_51) in pqi_aio_submit_r56_write_io()
5849 r56_request->header.iu_type = PQI_REQUEST_IU_AIO_PATH_RAID5_IO; in pqi_aio_submit_r56_write_io()
5851 r56_request->header.iu_type = PQI_REQUEST_IU_AIO_PATH_RAID6_IO; in pqi_aio_submit_r56_write_io()
5853 put_unaligned_le16(*(u16 *)device->scsi3addr & 0x3fff, &r56_request->volume_id); in pqi_aio_submit_r56_write_io()
5854 put_unaligned_le32(rmd->aio_handle, &r56_request->data_it_nexus); in pqi_aio_submit_r56_write_io()
5855 put_unaligned_le32(rmd->p_parity_it_nexus, &r56_request->p_parity_it_nexus); in pqi_aio_submit_r56_write_io()
5856 if (rmd->raid_level == SA_RAID_6) { in pqi_aio_submit_r56_write_io()
5857 put_unaligned_le32(rmd->q_parity_it_nexus, &r56_request->q_parity_it_nexus); in pqi_aio_submit_r56_write_io()
5858 r56_request->xor_multiplier = rmd->xor_mult; in pqi_aio_submit_r56_write_io()
5860 put_unaligned_le32(scsi_bufflen(scmd), &r56_request->data_length); in pqi_aio_submit_r56_write_io()
5861 r56_request->task_attribute = SOP_TASK_ATTRIBUTE_SIMPLE; in pqi_aio_submit_r56_write_io()
5862 put_unaligned_le64(rmd->row, &r56_request->row); in pqi_aio_submit_r56_write_io()
5864 put_unaligned_le16(io_request->index, &r56_request->request_id); in pqi_aio_submit_r56_write_io()
5865 r56_request->error_index = r56_request->request_id; in pqi_aio_submit_r56_write_io()
5867 if (rmd->cdb_length > sizeof(r56_request->cdb)) in pqi_aio_submit_r56_write_io()
5868 rmd->cdb_length = sizeof(r56_request->cdb); in pqi_aio_submit_r56_write_io()
5869 r56_request->cdb_length = rmd->cdb_length; in pqi_aio_submit_r56_write_io()
5870 memcpy(r56_request->cdb, rmd->cdb, rmd->cdb_length); in pqi_aio_submit_r56_write_io()
5873 r56_request->data_direction = SOP_READ_FLAG; in pqi_aio_submit_r56_write_io()
5876 r56_request->encryption_enable = true; in pqi_aio_submit_r56_write_io()
5877 put_unaligned_le16(encryption_info->data_encryption_key_index, in pqi_aio_submit_r56_write_io()
5878 &r56_request->data_encryption_key_index); in pqi_aio_submit_r56_write_io()
5879 put_unaligned_le32(encryption_info->encrypt_tweak_lower, in pqi_aio_submit_r56_write_io()
5880 &r56_request->encrypt_tweak_lower); in pqi_aio_submit_r56_write_io()
5881 put_unaligned_le32(encryption_info->encrypt_tweak_upper, in pqi_aio_submit_r56_write_io()
5882 &r56_request->encrypt_tweak_upper); in pqi_aio_submit_r56_write_io()
5910 return pqi_cmd_priv(scmd)->this_residual == 0; in pqi_is_bypass_eligible_request()
5915 * back to the SML.
5923 if (!scmd->device) { in pqi_prep_for_scsi_done()
5928 device = scmd->device->hostdata; in pqi_prep_for_scsi_done()
5934 atomic_dec(&device->scsi_cmds_outstanding[scmd->device->lun]); in pqi_prep_for_scsi_done()
5936 wait = (struct completion *)xchg(&scmd->host_scribble, NULL); in pqi_prep_for_scsi_done()
5952 if (!ctrl_info->enable_stream_detection) in pqi_is_parity_write_stream()
5963 device = scmd->device->hostdata; in pqi_is_parity_write_stream()
5966 if (device->raid_level != SA_RAID_5 && device->raid_level != SA_RAID_6) in pqi_is_parity_write_stream()
5971 * requests down non-AIO path. in pqi_is_parity_write_stream()
5973 if ((device->raid_level == SA_RAID_5 && !ctrl_info->enable_r5_writes) || in pqi_is_parity_write_stream()
5974 (device->raid_level == SA_RAID_6 && !ctrl_info->enable_r6_writes)) in pqi_is_parity_write_stream()
5980 pqi_stream_data = &device->stream_data[i]; in pqi_is_parity_write_stream()
5985 if ((pqi_stream_data->next_lba && in pqi_is_parity_write_stream()
5986 rmd.first_block >= pqi_stream_data->next_lba) && in pqi_is_parity_write_stream()
5987 rmd.first_block <= pqi_stream_data->next_lba + in pqi_is_parity_write_stream()
5989 pqi_stream_data->next_lba = rmd.first_block + in pqi_is_parity_write_stream()
5991 pqi_stream_data->last_accessed = jiffies; in pqi_is_parity_write_stream()
5992 per_cpu_ptr(device->raid_io_stats, smp_processor_id())->write_stream_cnt++; in pqi_is_parity_write_stream()
5997 if (pqi_stream_data->last_accessed == 0) { in pqi_is_parity_write_stream()
6003 if (pqi_stream_data->last_accessed <= oldest_jiffies) { in pqi_is_parity_write_stream()
6004 oldest_jiffies = pqi_stream_data->last_accessed; in pqi_is_parity_write_stream()
6010 pqi_stream_data = &device->stream_data[lru_index]; in pqi_is_parity_write_stream()
6011 pqi_stream_data->last_accessed = jiffies; in pqi_is_parity_write_stream()
6012 pqi_stream_data->next_lba = rmd.first_block + rmd.block_cnt; in pqi_is_parity_write_stream()
6027 scmd->host_scribble = PQI_NO_COMPLETION; in pqi_scsi_queue_command()
6029 device = scmd->device->hostdata; in pqi_scsi_queue_command()
6037 lun = (u8)scmd->device->lun; in pqi_scsi_queue_command()
6039 atomic_inc(&device->scsi_cmds_outstanding[lun]); in pqi_scsi_queue_command()
6055 * This is necessary because the SML doesn't zero out this field during in pqi_scsi_queue_command()
6058 scmd->result = 0; in pqi_scsi_queue_command()
6061 queue_group = &ctrl_info->queue_groups[hw_queue]; in pqi_scsi_queue_command()
6065 if (device->raid_bypass_enabled && in pqi_scsi_queue_command()
6071 per_cpu_ptr(device->raid_io_stats, smp_processor_id())->raid_bypass_cnt++; in pqi_scsi_queue_command()
6077 if (device->aio_enabled) in pqi_scsi_queue_command()
6085 scmd->host_scribble = NULL; in pqi_scsi_queue_command()
6086 atomic_dec(&device->scsi_cmds_outstanding[lun]); in pqi_scsi_queue_command()
6103 for (i = 0; i < ctrl_info->num_queue_groups; i++) { in pqi_queued_io_count()
6104 queue_group = &ctrl_info->queue_groups[i]; in pqi_queued_io_count()
6106 spin_lock_irqsave(&queue_group->submit_lock[path], flags); in pqi_queued_io_count()
6107 list_for_each_entry(io_request, &queue_group->request_list[path], request_list_entry) in pqi_queued_io_count()
6109 spin_unlock_irqrestore(&queue_group->submit_lock[path], flags); in pqi_queued_io_count()
6127 for (i = 0; i < ctrl_info->num_queue_groups; i++) { in pqi_nonempty_inbound_queue_count()
6128 queue_group = &ctrl_info->queue_groups[i]; in pqi_nonempty_inbound_queue_count()
6130 iq_pi = queue_group->iq_pi_copy[path]; in pqi_nonempty_inbound_queue_count()
6131 iq_ci = readl(queue_group->iq_ci[path]); in pqi_nonempty_inbound_queue_count()
6161 return -ENXIO; in pqi_wait_until_inbound_queues_empty()
6163 dev_warn(&ctrl_info->pci_dev->dev, in pqi_wait_until_inbound_queues_empty()
6164 …"waiting %u seconds for queued I/O to drain (queued I/O count: %u; non-empty inbound queue count: … in pqi_wait_until_inbound_queues_empty()
6165 … jiffies_to_msecs(jiffies - start_jiffies) / 1000, queued_io_count, nonempty_inbound_queue_count); in pqi_wait_until_inbound_queues_empty()
6173 dev_warn(&ctrl_info->pci_dev->dev, in pqi_wait_until_inbound_queues_empty()
6175 jiffies_to_msecs(jiffies - start_jiffies) / 1000); in pqi_wait_until_inbound_queues_empty()
6192 for (i = 0; i < ctrl_info->num_queue_groups; i++) { in pqi_fail_io_queued_for_device()
6193 queue_group = &ctrl_info->queue_groups[i]; in pqi_fail_io_queued_for_device()
6197 &queue_group->submit_lock[path], flags); in pqi_fail_io_queued_for_device()
6200 &queue_group->request_list[path], in pqi_fail_io_queued_for_device()
6203 scmd = io_request->scmd; in pqi_fail_io_queued_for_device()
6207 scsi_device = scmd->device->hostdata; in pqi_fail_io_queued_for_device()
6209 list_del(&io_request->request_list_entry); in pqi_fail_io_queued_for_device()
6210 if (scsi_device == device && (u8)scmd->device->lun == lun) in pqi_fail_io_queued_for_device()
6220 &queue_group->submit_lock[path], flags); in pqi_fail_io_queued_for_device()
6238 while ((cmds_outstanding = atomic_read(&device->scsi_cmds_outstanding[lun])) > 0) { in pqi_device_wait_for_pending_io()
6239 if (ctrl_info->ctrl_removal_state != PQI_CTRL_GRACEFUL_REMOVAL) { in pqi_device_wait_for_pending_io()
6242 return -ENXIO; in pqi_device_wait_for_pending_io()
6244 msecs_waiting = jiffies_to_msecs(jiffies - start_jiffies); in pqi_device_wait_for_pending_io()
6246 dev_err(&ctrl_info->pci_dev->dev, in pqi_device_wait_for_pending_io()
6248 ctrl_info->scsi_host->host_no, device->bus, device->target, in pqi_device_wait_for_pending_io()
6250 return -ETIMEDOUT; in pqi_device_wait_for_pending_io()
6253 dev_warn(&ctrl_info->pci_dev->dev, in pqi_device_wait_for_pending_io()
6255 ctrl_info->scsi_host->host_no, device->bus, device->target, in pqi_device_wait_for_pending_io()
6293 rc = -ENXIO; in pqi_wait_for_lun_reset_completion()
6298 cmds_outstanding = atomic_read(&device->scsi_cmds_outstanding[lun]); in pqi_wait_for_lun_reset_completion()
6299 dev_warn(&ctrl_info->pci_dev->dev, in pqi_wait_for_lun_reset_completion()
6301 ctrl_info->scsi_host->host_no, device->bus, device->target, lun, wait_secs, cmds_outstanding); in pqi_wait_for_lun_reset_completion()
6317 io_request->io_complete_callback = pqi_lun_reset_complete; in pqi_lun_reset()
6318 io_request->context = &wait; in pqi_lun_reset()
6320 request = io_request->iu; in pqi_lun_reset()
6323 request->header.iu_type = PQI_REQUEST_IU_TASK_MANAGEMENT; in pqi_lun_reset()
6324 put_unaligned_le16(sizeof(*request) - PQI_REQUEST_HEADER_LENGTH, in pqi_lun_reset()
6325 &request->header.iu_length); in pqi_lun_reset()
6326 put_unaligned_le16(io_request->index, &request->request_id); in pqi_lun_reset()
6327 memcpy(request->lun_number, device->scsi3addr, in pqi_lun_reset()
6328 sizeof(request->lun_number)); in pqi_lun_reset()
6329 if (!pqi_is_logical_device(device) && ctrl_info->multi_lun_device_supported) in pqi_lun_reset()
6330 request->ml_device_lun_number = lun; in pqi_lun_reset()
6331 request->task_management_function = SOP_TASK_MANAGEMENT_LUN_RESET; in pqi_lun_reset()
6332 if (ctrl_info->tmf_iu_timeout_supported) in pqi_lun_reset()
6333 put_unaligned_le16(PQI_LUN_RESET_FIRMWARE_TIMEOUT_SECS, &request->timeout); in pqi_lun_reset()
6335 pqi_start_io(ctrl_info, &ctrl_info->queue_groups[PQI_DEFAULT_QUEUE_GROUP], RAID_PATH, in pqi_lun_reset()
6340 rc = io_request->status; in pqi_lun_reset()
6361 …if (reset_rc == 0 || reset_rc == -ENODEV || reset_rc == -ENXIO || ++retries > PQI_LUN_RESET_RETRIE… in pqi_lun_reset_with_retries()
6399 mutex_lock(&ctrl_info->lun_reset_mutex); in pqi_device_reset_handler()
6401 dev_err(&ctrl_info->pci_dev->dev, in pqi_device_reset_handler()
6403 ctrl_info->scsi_host->host_no, device->bus, device->target, lun, scmd, scsi_opcode); in pqi_device_reset_handler()
6411 dev_err(&ctrl_info->pci_dev->dev, in pqi_device_reset_handler()
6413 ctrl_info->scsi_host->host_no, device->bus, device->target, lun, in pqi_device_reset_handler()
6416 mutex_unlock(&ctrl_info->lun_reset_mutex); in pqi_device_reset_handler()
6428 shost = scmd->device->host; in pqi_eh_device_reset_handler()
6430 device = scmd->device->hostdata; in pqi_eh_device_reset_handler()
6431 scsi_opcode = scmd->cmd_len > 0 ? scmd->cmnd[0] : 0xff; in pqi_eh_device_reset_handler()
6433 return pqi_device_reset_handler(ctrl_info, device, (u8)scmd->device->lun, scmd, scsi_opcode); in pqi_eh_device_reset_handler()
6442 scmd = (struct scsi_cmnd *)xchg(&tmf_work->scmd, NULL); in pqi_tmf_worker()
6444 …pqi_device_reset_handler(tmf_work->ctrl_info, tmf_work->device, tmf_work->lun, scmd, tmf_work->scs… in pqi_tmf_worker()
6455 shost = scmd->device->host; in pqi_eh_abort_handler()
6457 device = scmd->device->hostdata; in pqi_eh_abort_handler()
6459 dev_err(&ctrl_info->pci_dev->dev, in pqi_eh_abort_handler()
6461 shost->host_no, device->bus, device->target, (int)scmd->device->lun, scmd); in pqi_eh_abort_handler()
6463 if (cmpxchg(&scmd->host_scribble, PQI_NO_COMPLETION, (void *)&wait) == NULL) { in pqi_eh_abort_handler()
6464 dev_err(&ctrl_info->pci_dev->dev, in pqi_eh_abort_handler()
6466 shost->host_no, device->bus, device->target, (int)scmd->device->lun, scmd); in pqi_eh_abort_handler()
6467 scmd->result = DID_RESET << 16; in pqi_eh_abort_handler()
6471 tmf_work = &device->tmf_work[scmd->device->lun]; in pqi_eh_abort_handler()
6473 if (cmpxchg(&tmf_work->scmd, NULL, scmd) == NULL) { in pqi_eh_abort_handler()
6474 tmf_work->ctrl_info = ctrl_info; in pqi_eh_abort_handler()
6475 tmf_work->device = device; in pqi_eh_abort_handler()
6476 tmf_work->lun = (u8)scmd->device->lun; in pqi_eh_abort_handler()
6477 tmf_work->scsi_opcode = scmd->cmd_len > 0 ? scmd->cmnd[0] : 0xff; in pqi_eh_abort_handler()
6478 schedule_work(&tmf_work->work_struct); in pqi_eh_abort_handler()
6483 dev_err(&ctrl_info->pci_dev->dev, in pqi_eh_abort_handler()
6485 shost->host_no, device->bus, device->target, (int)scmd->device->lun, scmd); in pqi_eh_abort_handler()
6500 ctrl_info = shost_to_hba(sdev->host); in pqi_sdev_init()
6502 spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags); in pqi_sdev_init()
6509 if (device->target_lun_valid) { in pqi_sdev_init()
6510 device->ignore_device = true; in pqi_sdev_init()
6512 device->target = sdev_id(sdev); in pqi_sdev_init()
6513 device->lun = sdev->lun; in pqi_sdev_init()
6514 device->target_lun_valid = true; in pqi_sdev_init()
6519 sdev_id(sdev), sdev->lun); in pqi_sdev_init()
6523 sdev->hostdata = device; in pqi_sdev_init()
6524 device->sdev = sdev; in pqi_sdev_init()
6525 if (device->queue_depth) { in pqi_sdev_init()
6526 device->advertised_queue_depth = device->queue_depth; in pqi_sdev_init()
6528 device->advertised_queue_depth); in pqi_sdev_init()
6533 sdev->allow_restart = 1; in pqi_sdev_init()
6534 if (device->device_type == SA_DEVICE_TYPE_NVME) in pqi_sdev_init()
6539 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags); in pqi_sdev_init()
6548 if (!ctrl_info->disable_managed_interrupts) in pqi_map_queues()
6549 blk_mq_map_hw_queues(&shost->tag_set.map[HCTX_TYPE_DEFAULT], in pqi_map_queues()
6550 &ctrl_info->pci_dev->dev, 0); in pqi_map_queues()
6552 blk_mq_map_queues(&shost->tag_set.map[HCTX_TYPE_DEFAULT]); in pqi_map_queues()
6557 return device->devtype == TYPE_TAPE || device->devtype == TYPE_MEDIUM_CHANGER; in pqi_is_tape_changer_device()
6566 device = sdev->hostdata; in pqi_sdev_configure()
6567 device->devtype = sdev->type; in pqi_sdev_configure()
6569 if (pqi_is_tape_changer_device(device) && device->ignore_device) { in pqi_sdev_configure()
6570 rc = -ENXIO; in pqi_sdev_configure()
6571 device->ignore_device = false; in pqi_sdev_configure()
6584 ctrl_info = shost_to_hba(sdev->host); in pqi_sdev_destroy()
6586 mutex_acquired = mutex_trylock(&ctrl_info->scan_mutex); in pqi_sdev_destroy()
6590 device = sdev->hostdata; in pqi_sdev_destroy()
6592 mutex_unlock(&ctrl_info->scan_mutex); in pqi_sdev_destroy()
6596 device->lun_count--; in pqi_sdev_destroy()
6597 if (device->lun_count > 0) { in pqi_sdev_destroy()
6598 mutex_unlock(&ctrl_info->scan_mutex); in pqi_sdev_destroy()
6602 spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags); in pqi_sdev_destroy()
6603 list_del(&device->scsi_device_list_entry); in pqi_sdev_destroy()
6604 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags); in pqi_sdev_destroy()
6606 mutex_unlock(&ctrl_info->scan_mutex); in pqi_sdev_destroy()
6620 return -EINVAL; in pqi_getpciinfo_ioctl()
6622 pci_dev = ctrl_info->pci_dev; in pqi_getpciinfo_ioctl()
6624 pci_info.domain = pci_domain_nr(pci_dev->bus); in pqi_getpciinfo_ioctl()
6625 pci_info.bus = pci_dev->bus->number; in pqi_getpciinfo_ioctl()
6626 pci_info.dev_fn = pci_dev->devfn; in pqi_getpciinfo_ioctl()
6627 subsystem_vendor = pci_dev->subsystem_vendor; in pqi_getpciinfo_ioctl()
6628 subsystem_device = pci_dev->subsystem_device; in pqi_getpciinfo_ioctl()
6632 return -EFAULT; in pqi_getpciinfo_ioctl()
6642 return -EINVAL; in pqi_getdrivver_ioctl()
6648 return -EFAULT; in pqi_getdrivver_ioctl()
6665 switch (pqi_error_info->data_out_result) { in pqi_error_info_to_ciss()
6709 get_unaligned_le16(&pqi_error_info->sense_data_length); in pqi_error_info_to_ciss()
6712 get_unaligned_le16(&pqi_error_info->response_data_length); in pqi_error_info_to_ciss()
6714 if (sense_data_length > sizeof(pqi_error_info->data)) in pqi_error_info_to_ciss()
6715 sense_data_length = sizeof(pqi_error_info->data); in pqi_error_info_to_ciss()
6717 ciss_error_info->scsi_status = pqi_error_info->status; in pqi_error_info_to_ciss()
6718 ciss_error_info->command_status = ciss_cmd_status; in pqi_error_info_to_ciss()
6719 ciss_error_info->sense_data_length = sense_data_length; in pqi_error_info_to_ciss()
6734 return -ENXIO; in pqi_passthru_ioctl()
6736 return -EBUSY; in pqi_passthru_ioctl()
6738 return -EINVAL; in pqi_passthru_ioctl()
6740 return -EPERM; in pqi_passthru_ioctl()
6742 return -EFAULT; in pqi_passthru_ioctl()
6745 return -EINVAL; in pqi_passthru_ioctl()
6747 return -EINVAL; in pqi_passthru_ioctl()
6749 return -EINVAL; in pqi_passthru_ioctl()
6758 return -EINVAL; in pqi_passthru_ioctl()
6764 return -ENOMEM; in pqi_passthru_ioctl()
6768 rc = -EFAULT; in pqi_passthru_ioctl()
6779 iu_length = offsetof(struct pqi_raid_path_request, sg_descriptors) - in pqi_passthru_ioctl()
6806 rc = pqi_map_single(ctrl_info->pci_dev, in pqi_passthru_ioctl()
6817 if (ctrl_info->raid_iu_timeout_supported) in pqi_passthru_ioctl()
6824 pqi_pci_unmap(ctrl_info->pci_dev, request.sg_descriptors, 1, in pqi_passthru_ioctl()
6847 rc = -EFAULT; in pqi_passthru_ioctl()
6855 rc = -EFAULT; in pqi_passthru_ioctl()
6871 ctrl_info = shost_to_hba(sdev->host); in pqi_ioctl()
6889 rc = -EINVAL; in pqi_ioctl()
6905 return scnprintf(buffer, PAGE_SIZE, "%s\n", ctrl_info->firmware_version); in pqi_firmware_version_show()
6917 return scnprintf(buffer, PAGE_SIZE, "%s\n", ctrl_info->serial_number); in pqi_serial_number_show()
6929 return scnprintf(buffer, PAGE_SIZE, "%s\n", ctrl_info->model); in pqi_model_show()
6941 return scnprintf(buffer, PAGE_SIZE, "%s\n", ctrl_info->vendor); in pqi_vendor_show()
6962 count += scnprintf(buffer + count, PAGE_SIZE - count, in pqi_lockup_action_show()
6965 count += scnprintf(buffer + count, PAGE_SIZE - count, in pqi_lockup_action_show()
6969 count += scnprintf(buffer + count, PAGE_SIZE - count, "\n"); in pqi_lockup_action_show()
6991 return -EINVAL; in pqi_lockup_action_store()
7001 ctrl_info->enable_stream_detection); in pqi_host_enable_stream_detection_show()
7012 return -EINVAL; in pqi_host_enable_stream_detection_store()
7017 ctrl_info->enable_stream_detection = set_stream_detection; in pqi_host_enable_stream_detection_store()
7028 return scnprintf(buffer, 10, "%x\n", ctrl_info->enable_r5_writes); in pqi_host_enable_r5_writes_show()
7039 return -EINVAL; in pqi_host_enable_r5_writes_store()
7044 ctrl_info->enable_r5_writes = set_r5_writes; in pqi_host_enable_r5_writes_store()
7055 return scnprintf(buffer, 10, "%x\n", ctrl_info->enable_r6_writes); in pqi_host_enable_r6_writes_show()
7066 return -EINVAL; in pqi_host_enable_r6_writes_store()
7071 ctrl_info->enable_r6_writes = set_r6_writes; in pqi_host_enable_r6_writes_store()
7119 ctrl_info = shost_to_hba(sdev->host); in pqi_unique_id_show()
7122 return -ENODEV; in pqi_unique_id_show()
7124 spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags); in pqi_unique_id_show()
7126 device = sdev->hostdata; in pqi_unique_id_show()
7128 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags); in pqi_unique_id_show()
7129 return -ENODEV; in pqi_unique_id_show()
7132 if (device->is_physical_device) in pqi_unique_id_show()
7133 memcpy(unique_id, device->wwid, sizeof(device->wwid)); in pqi_unique_id_show()
7135 memcpy(unique_id, device->volume_id, sizeof(device->volume_id)); in pqi_unique_id_show()
7137 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags); in pqi_unique_id_show()
7158 ctrl_info = shost_to_hba(sdev->host); in pqi_lunid_show()
7161 return -ENODEV; in pqi_lunid_show()
7163 spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags); in pqi_lunid_show()
7165 device = sdev->hostdata; in pqi_lunid_show()
7167 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags); in pqi_lunid_show()
7168 return -ENODEV; in pqi_lunid_show()
7171 memcpy(lunid, device->scsi3addr, sizeof(lunid)); in pqi_lunid_show()
7173 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags); in pqi_lunid_show()
7196 ctrl_info = shost_to_hba(sdev->host); in pqi_path_info_show()
7199 return -ENODEV; in pqi_path_info_show()
7201 spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags); in pqi_path_info_show()
7203 device = sdev->hostdata; in pqi_path_info_show()
7205 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags); in pqi_path_info_show()
7206 return -ENODEV; in pqi_path_info_show()
7209 bay = device->bay; in pqi_path_info_show()
7212 if (i == device->active_path_index) in pqi_path_info_show()
7214 else if (device->path_map & path_map_index) in pqi_path_info_show()
7220 PAGE_SIZE - output_len, in pqi_path_info_show()
7222 ctrl_info->scsi_host->host_no, in pqi_path_info_show()
7223 device->bus, device->target, in pqi_path_info_show()
7224 device->lun, in pqi_path_info_show()
7225 scsi_device_type(device->devtype)); in pqi_path_info_show()
7227 if (device->devtype == TYPE_RAID || in pqi_path_info_show()
7231 memcpy(&phys_connector, &device->phys_connector[i], in pqi_path_info_show()
7239 PAGE_SIZE - output_len, in pqi_path_info_show()
7242 box = device->box[i]; in pqi_path_info_show()
7245 PAGE_SIZE - output_len, in pqi_path_info_show()
7248 if ((device->devtype == TYPE_DISK || in pqi_path_info_show()
7249 device->devtype == TYPE_ZBC) && in pqi_path_info_show()
7252 PAGE_SIZE - output_len, in pqi_path_info_show()
7257 PAGE_SIZE - output_len, in pqi_path_info_show()
7261 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags); in pqi_path_info_show()
7276 ctrl_info = shost_to_hba(sdev->host); in pqi_sas_address_show()
7279 return -ENODEV; in pqi_sas_address_show()
7281 spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags); in pqi_sas_address_show()
7283 device = sdev->hostdata; in pqi_sas_address_show()
7285 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags); in pqi_sas_address_show()
7286 return -ENODEV; in pqi_sas_address_show()
7289 sas_address = device->sas_address; in pqi_sas_address_show()
7291 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags); in pqi_sas_address_show()
7305 ctrl_info = shost_to_hba(sdev->host); in pqi_ssd_smart_path_enabled_show()
7308 return -ENODEV; in pqi_ssd_smart_path_enabled_show()
7310 spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags); in pqi_ssd_smart_path_enabled_show()
7312 device = sdev->hostdata; in pqi_ssd_smart_path_enabled_show()
7314 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags); in pqi_ssd_smart_path_enabled_show()
7315 return -ENODEV; in pqi_ssd_smart_path_enabled_show()
7318 buffer[0] = device->raid_bypass_enabled ? '1' : '0'; in pqi_ssd_smart_path_enabled_show()
7322 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags); in pqi_ssd_smart_path_enabled_show()
7337 ctrl_info = shost_to_hba(sdev->host); in pqi_raid_level_show()
7340 return -ENODEV; in pqi_raid_level_show()
7342 spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags); in pqi_raid_level_show()
7344 device = sdev->hostdata; in pqi_raid_level_show()
7346 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags); in pqi_raid_level_show()
7347 return -ENODEV; in pqi_raid_level_show()
7350 if (pqi_is_logical_device(device) && device->devtype == TYPE_DISK) in pqi_raid_level_show()
7351 raid_level = pqi_raid_level_to_string(device->raid_level); in pqi_raid_level_show()
7355 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags); in pqi_raid_level_show()
7371 ctrl_info = shost_to_hba(sdev->host); in pqi_raid_bypass_cnt_show()
7374 return -ENODEV; in pqi_raid_bypass_cnt_show()
7376 spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags); in pqi_raid_bypass_cnt_show()
7378 device = sdev->hostdata; in pqi_raid_bypass_cnt_show()
7380 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags); in pqi_raid_bypass_cnt_show()
7381 return -ENODEV; in pqi_raid_bypass_cnt_show()
7386 if (device->raid_io_stats) { in pqi_raid_bypass_cnt_show()
7388 raid_bypass_cnt += per_cpu_ptr(device->raid_io_stats, cpu)->raid_bypass_cnt; in pqi_raid_bypass_cnt_show()
7392 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags); in pqi_raid_bypass_cnt_show()
7407 ctrl_info = shost_to_hba(sdev->host); in pqi_sas_ncq_prio_enable_show()
7410 return -ENODEV; in pqi_sas_ncq_prio_enable_show()
7412 spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags); in pqi_sas_ncq_prio_enable_show()
7414 device = sdev->hostdata; in pqi_sas_ncq_prio_enable_show()
7416 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags); in pqi_sas_ncq_prio_enable_show()
7417 return -ENODEV; in pqi_sas_ncq_prio_enable_show()
7421 device->ncq_prio_enable); in pqi_sas_ncq_prio_enable_show()
7422 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags); in pqi_sas_ncq_prio_enable_show()
7438 return -EINVAL; in pqi_sas_ncq_prio_enable_store()
7441 ctrl_info = shost_to_hba(sdev->host); in pqi_sas_ncq_prio_enable_store()
7443 spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags); in pqi_sas_ncq_prio_enable_store()
7445 device = sdev->hostdata; in pqi_sas_ncq_prio_enable_store()
7448 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags); in pqi_sas_ncq_prio_enable_store()
7449 return -ENODEV; in pqi_sas_ncq_prio_enable_store()
7452 if (!device->ncq_prio_support) { in pqi_sas_ncq_prio_enable_store()
7453 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags); in pqi_sas_ncq_prio_enable_store()
7454 return -EINVAL; in pqi_sas_ncq_prio_enable_store()
7457 device->ncq_prio_enable = ncq_prio_enable; in pqi_sas_ncq_prio_enable_store()
7459 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags); in pqi_sas_ncq_prio_enable_store()
7471 ctrl_info = shost_to_hba(sdev->host); in pqi_numa_node_show()
7473 return scnprintf(buffer, PAGE_SIZE, "%d\n", ctrl_info->numa_node); in pqi_numa_node_show()
7487 ctrl_info = shost_to_hba(sdev->host); in pqi_write_stream_cnt_show()
7490 return -ENODEV; in pqi_write_stream_cnt_show()
7492 spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags); in pqi_write_stream_cnt_show()
7494 device = sdev->hostdata; in pqi_write_stream_cnt_show()
7496 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags); in pqi_write_stream_cnt_show()
7497 return -ENODEV; in pqi_write_stream_cnt_show()
7502 if (device->raid_io_stats) { in pqi_write_stream_cnt_show()
7504 write_stream_cnt += per_cpu_ptr(device->raid_io_stats, cpu)->write_stream_cnt; in pqi_write_stream_cnt_show()
7508 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags); in pqi_write_stream_cnt_show()
7548 .this_id = -1,
7568 dev_err(&ctrl_info->pci_dev->dev, "scsi_host_alloc failed\n"); in pqi_register_scsi()
7569 return -ENOMEM; in pqi_register_scsi()
7572 shost->io_port = 0; in pqi_register_scsi()
7573 shost->n_io_port = 0; in pqi_register_scsi()
7574 shost->this_id = -1; in pqi_register_scsi()
7575 shost->max_channel = PQI_MAX_BUS; in pqi_register_scsi()
7576 shost->max_cmd_len = MAX_COMMAND_SIZE; in pqi_register_scsi()
7577 shost->max_lun = PQI_MAX_LUNS_PER_DEVICE; in pqi_register_scsi()
7578 shost->max_id = ~0; in pqi_register_scsi()
7579 shost->max_sectors = ctrl_info->max_sectors; in pqi_register_scsi()
7580 shost->can_queue = ctrl_info->scsi_ml_can_queue; in pqi_register_scsi()
7581 shost->cmd_per_lun = shost->can_queue; in pqi_register_scsi()
7582 shost->sg_tablesize = ctrl_info->sg_tablesize; in pqi_register_scsi()
7583 shost->transportt = pqi_sas_transport_template; in pqi_register_scsi()
7584 shost->irq = pci_irq_vector(ctrl_info->pci_dev, 0); in pqi_register_scsi()
7585 shost->unique_id = shost->irq; in pqi_register_scsi()
7586 shost->nr_hw_queues = ctrl_info->num_queue_groups; in pqi_register_scsi()
7587 shost->host_tagset = 1; in pqi_register_scsi()
7588 shost->hostdata[0] = (unsigned long)ctrl_info; in pqi_register_scsi()
7590 rc = scsi_add_host(shost, &ctrl_info->pci_dev->dev); in pqi_register_scsi()
7592 dev_err(&ctrl_info->pci_dev->dev, "scsi_add_host failed\n"); in pqi_register_scsi()
7598 dev_err(&ctrl_info->pci_dev->dev, "add SAS host failed\n"); in pqi_register_scsi()
7602 ctrl_info->scsi_host = shost; in pqi_register_scsi()
7620 shost = ctrl_info->scsi_host; in pqi_unregister_scsi()
7636 pqi_registers = ctrl_info->pqi_registers; in pqi_wait_for_pqi_reset_completion()
7637 timeout_msecs = readw(&pqi_registers->max_reset_timeout) * 100; in pqi_wait_for_pqi_reset_completion()
7642 reset_reg.all_bits = readl(&pqi_registers->device_reset); in pqi_wait_for_pqi_reset_completion()
7646 rc = -ENXIO; in pqi_wait_for_pqi_reset_completion()
7650 rc = -ETIMEDOUT; in pqi_wait_for_pqi_reset_completion()
7663 if (ctrl_info->pqi_reset_quiesce_supported) { in pqi_reset()
7666 dev_err(&ctrl_info->pci_dev->dev, in pqi_reset()
7676 writel(reset_reg.all_bits, &ctrl_info->pqi_registers->device_reset); in pqi_reset()
7680 dev_err(&ctrl_info->pci_dev->dev, in pqi_reset()
7693 return -ENOMEM; in pqi_get_ctrl_serial_number()
7699 memcpy(ctrl_info->serial_number, sense_info->ctrl_serial_number, in pqi_get_ctrl_serial_number()
7700 sizeof(sense_info->ctrl_serial_number)); in pqi_get_ctrl_serial_number()
7701 ctrl_info->serial_number[sizeof(sense_info->ctrl_serial_number)] = '\0'; in pqi_get_ctrl_serial_number()
7716 return -ENOMEM; in pqi_get_ctrl_product_details()
7722 if (get_unaligned_le32(&identify->extra_controller_flags) & in pqi_get_ctrl_product_details()
7724 memcpy(ctrl_info->firmware_version, in pqi_get_ctrl_product_details()
7725 identify->firmware_version_long, in pqi_get_ctrl_product_details()
7726 sizeof(identify->firmware_version_long)); in pqi_get_ctrl_product_details()
7728 memcpy(ctrl_info->firmware_version, in pqi_get_ctrl_product_details()
7729 identify->firmware_version_short, in pqi_get_ctrl_product_details()
7730 sizeof(identify->firmware_version_short)); in pqi_get_ctrl_product_details()
7731 ctrl_info->firmware_version in pqi_get_ctrl_product_details()
7732 [sizeof(identify->firmware_version_short)] = '\0'; in pqi_get_ctrl_product_details()
7733 snprintf(ctrl_info->firmware_version + in pqi_get_ctrl_product_details()
7734 strlen(ctrl_info->firmware_version), in pqi_get_ctrl_product_details()
7735 sizeof(ctrl_info->firmware_version) - in pqi_get_ctrl_product_details()
7736 sizeof(identify->firmware_version_short), in pqi_get_ctrl_product_details()
7737 "-%u", in pqi_get_ctrl_product_details()
7738 get_unaligned_le16(&identify->firmware_build_number)); in pqi_get_ctrl_product_details()
7741 memcpy(ctrl_info->model, identify->product_id, in pqi_get_ctrl_product_details()
7742 sizeof(identify->product_id)); in pqi_get_ctrl_product_details()
7743 ctrl_info->model[sizeof(identify->product_id)] = '\0'; in pqi_get_ctrl_product_details()
7745 memcpy(ctrl_info->vendor, identify->vendor_id, in pqi_get_ctrl_product_details()
7746 sizeof(identify->vendor_id)); in pqi_get_ctrl_product_details()
7747 ctrl_info->vendor[sizeof(identify->vendor_id)] = '\0'; in pqi_get_ctrl_product_details()
7749 dev_info(&ctrl_info->pci_dev->dev, in pqi_get_ctrl_product_details()
7750 "Firmware version: %s\n", ctrl_info->firmware_version); in pqi_get_ctrl_product_details()
7773 if (byte_index >= le16_to_cpu(firmware_features->num_elements)) in pqi_is_firmware_feature_supported()
7776 return firmware_features->features_supported[byte_index] & in pqi_is_firmware_feature_supported()
7789 (le16_to_cpu(firmware_features->num_elements) * 2); in pqi_is_firmware_feature_enabled()
7806 le16_to_cpu(firmware_features->num_elements); in pqi_request_firmware_feature()
7808 firmware_features->features_supported[byte_index] |= in pqi_request_firmware_feature()
7820 put_unaligned_le16(sizeof(request) - PQI_REQUEST_HEADER_LENGTH, in pqi_config_table_update()
7840 features_requested = firmware_features->features_supported + in pqi_enable_firmware_features()
7841 le16_to_cpu(firmware_features->num_elements); in pqi_enable_firmware_features()
7844 (features_requested - (void *)firmware_features); in pqi_enable_firmware_features()
7847 le16_to_cpu(firmware_features->num_elements)); in pqi_enable_firmware_features()
7853 (le16_to_cpu(firmware_features->num_elements) * 2) + in pqi_enable_firmware_features()
7876 if (!firmware_feature->supported) { in pqi_firmware_feature_status()
7877 dev_info(&ctrl_info->pci_dev->dev, "%s not supported by controller\n", in pqi_firmware_feature_status()
7878 firmware_feature->feature_name); in pqi_firmware_feature_status()
7882 if (firmware_feature->enabled) { in pqi_firmware_feature_status()
7883 dev_info(&ctrl_info->pci_dev->dev, in pqi_firmware_feature_status()
7884 "%s enabled\n", firmware_feature->feature_name); in pqi_firmware_feature_status()
7888 dev_err(&ctrl_info->pci_dev->dev, "failed to enable %s\n", in pqi_firmware_feature_status()
7889 firmware_feature->feature_name); in pqi_firmware_feature_status()
7895 switch (firmware_feature->feature_bit) { in pqi_ctrl_update_feature_flags()
7897 ctrl_info->enable_r1_writes = firmware_feature->enabled; in pqi_ctrl_update_feature_flags()
7900 ctrl_info->enable_r5_writes = firmware_feature->enabled; in pqi_ctrl_update_feature_flags()
7903 ctrl_info->enable_r6_writes = firmware_feature->enabled; in pqi_ctrl_update_feature_flags()
7906 ctrl_info->soft_reset_handshake_supported = in pqi_ctrl_update_feature_flags()
7907 firmware_feature->enabled && in pqi_ctrl_update_feature_flags()
7911 ctrl_info->raid_iu_timeout_supported = firmware_feature->enabled; in pqi_ctrl_update_feature_flags()
7914 ctrl_info->tmf_iu_timeout_supported = firmware_feature->enabled; in pqi_ctrl_update_feature_flags()
7917 ctrl_info->firmware_triage_supported = firmware_feature->enabled; in pqi_ctrl_update_feature_flags()
7918 pqi_save_fw_triage_setting(ctrl_info, firmware_feature->enabled); in pqi_ctrl_update_feature_flags()
7921 ctrl_info->rpl_extended_format_4_5_supported = firmware_feature->enabled; in pqi_ctrl_update_feature_flags()
7924 ctrl_info->multi_lun_device_supported = firmware_feature->enabled; in pqi_ctrl_update_feature_flags()
7927 ctrl_info->ctrl_logging_supported = firmware_feature->enabled; in pqi_ctrl_update_feature_flags()
7937 if (firmware_feature->feature_status) in pqi_firmware_feature_update()
7938 firmware_feature->feature_status(ctrl_info, firmware_feature); in pqi_firmware_feature_update()
8030 .feature_name = "Multi-LUN Target",
8051 ctrl_info = section_info->ctrl_info; in pqi_process_firmware_features()
8052 firmware_features = section_info->section; in pqi_process_firmware_features()
8053 firmware_features_iomem_addr = section_info->section_iomem_addr; in pqi_process_firmware_features()
8080 dev_err(&ctrl_info->pci_dev->dev, in pqi_process_firmware_features()
8130 ctrl_info->heartbeat_counter = NULL; in pqi_ctrl_reset_config()
8131 ctrl_info->soft_reset_status = NULL; in pqi_ctrl_reset_config()
8132 ctrl_info->soft_reset_handshake_supported = false; in pqi_ctrl_reset_config()
8133 ctrl_info->enable_r1_writes = false; in pqi_ctrl_reset_config()
8134 ctrl_info->enable_r5_writes = false; in pqi_ctrl_reset_config()
8135 ctrl_info->enable_r6_writes = false; in pqi_ctrl_reset_config()
8136 ctrl_info->raid_iu_timeout_supported = false; in pqi_ctrl_reset_config()
8137 ctrl_info->tmf_iu_timeout_supported = false; in pqi_ctrl_reset_config()
8138 ctrl_info->firmware_triage_supported = false; in pqi_ctrl_reset_config()
8139 ctrl_info->rpl_extended_format_4_5_supported = false; in pqi_ctrl_reset_config()
8140 ctrl_info->multi_lun_device_supported = false; in pqi_ctrl_reset_config()
8141 ctrl_info->ctrl_logging_supported = false; in pqi_ctrl_reset_config()
8155 table_length = ctrl_info->config_table_length; in pqi_process_config_table()
8161 dev_err(&ctrl_info->pci_dev->dev, in pqi_process_config_table()
8163 return -ENOMEM; in pqi_process_config_table()
8170 table_iomem_addr = ctrl_info->iomem_base + ctrl_info->config_table_offset; in pqi_process_config_table()
8175 section_offset = get_unaligned_le32(&config_table->first_section_offset); in pqi_process_config_table()
8184 switch (get_unaligned_le16(&section->section_id)) { in pqi_process_config_table()
8191 dev_warn(&ctrl_info->pci_dev->dev, in pqi_process_config_table()
8194 ctrl_info->heartbeat_counter = in pqi_process_config_table()
8201 ctrl_info->soft_reset_status = in pqi_process_config_table()
8209 section_offset = get_unaligned_le16(&section->next_section_offset); in pqi_process_config_table()
8237 dev_err(&ctrl_info->pci_dev->dev, in pqi_revert_to_sis_mode()
8238 "re-enabling SIS mode failed with error %d\n", rc); in pqi_revert_to_sis_mode()
8254 return -ENXIO; in pqi_force_sis_mode()
8317 dev_err(&ctrl_info->pci_dev->dev, in pqi_ctrl_init()
8331 dev_err(&ctrl_info->pci_dev->dev, in pqi_ctrl_init()
8338 dev_err(&ctrl_info->pci_dev->dev, in pqi_ctrl_init()
8344 ctrl_info->product_id = (u8)product_id; in pqi_ctrl_init()
8345 ctrl_info->product_revision = (u8)(product_id >> 8); in pqi_ctrl_init()
8348 if (ctrl_info->max_outstanding_requests > in pqi_ctrl_init()
8350 ctrl_info->max_outstanding_requests = in pqi_ctrl_init()
8353 if (ctrl_info->max_outstanding_requests > in pqi_ctrl_init()
8355 ctrl_info->max_outstanding_requests = in pqi_ctrl_init()
8363 dev_err(&ctrl_info->pci_dev->dev, in pqi_ctrl_init()
8375 dev_err(&ctrl_info->pci_dev->dev, in pqi_ctrl_init()
8380 /* Wait for the controller to complete the SIS -> PQI transition. */ in pqi_ctrl_init()
8383 dev_err(&ctrl_info->pci_dev->dev, in pqi_ctrl_init()
8389 ctrl_info->pqi_mode_enabled = true; in pqi_ctrl_init()
8394 dev_err(&ctrl_info->pci_dev->dev, in pqi_ctrl_init()
8401 dev_err(&ctrl_info->pci_dev->dev, in pqi_ctrl_init()
8408 dev_err(&ctrl_info->pci_dev->dev, in pqi_ctrl_init()
8423 if (ctrl_info->num_msix_vectors_enabled < ctrl_info->num_queue_groups) { in pqi_ctrl_init()
8424 ctrl_info->max_msix_vectors = in pqi_ctrl_init()
8425 ctrl_info->num_msix_vectors_enabled; in pqi_ctrl_init()
8435 dev_err(&ctrl_info->pci_dev->dev, in pqi_ctrl_init()
8452 ctrl_info->controller_online = true; in pqi_ctrl_init()
8460 if (ctrl_info->enable_r5_writes || ctrl_info->enable_r6_writes) { in pqi_ctrl_init()
8463 dev_err(&ctrl_info->pci_dev->dev, in pqi_ctrl_init()
8467 ctrl_info->ciss_report_log_flags |= in pqi_ctrl_init()
8473 dev_err(&ctrl_info->pci_dev->dev, in pqi_ctrl_init()
8483 if (ctrl_info->ctrl_logging_supported && !reset_devices) { in pqi_ctrl_init()
8484 …pqi_host_setup_buffer(ctrl_info, &ctrl_info->ctrl_log_memory, PQI_CTRL_LOG_TOTAL_SIZE, PQI_CTRL_LO… in pqi_ctrl_init()
8485 …pqi_host_memory_update(ctrl_info, &ctrl_info->ctrl_log_memory, PQI_VENDOR_GENERAL_CTRL_LOG_MEMORY_… in pqi_ctrl_init()
8490 dev_err(&ctrl_info->pci_dev->dev, in pqi_ctrl_init()
8497 dev_err(&ctrl_info->pci_dev->dev, in pqi_ctrl_init()
8504 dev_err(&ctrl_info->pci_dev->dev, in pqi_ctrl_init()
8505 "error enabling multi-lun rescan\n"); in pqi_ctrl_init()
8511 dev_err(&ctrl_info->pci_dev->dev, in pqi_ctrl_init()
8529 admin_queues = &ctrl_info->admin_queues; in pqi_reinit_queues()
8530 admin_queues->iq_pi_copy = 0; in pqi_reinit_queues()
8531 admin_queues->oq_ci_copy = 0; in pqi_reinit_queues()
8532 writel(0, admin_queues->oq_pi); in pqi_reinit_queues()
8534 for (i = 0; i < ctrl_info->num_queue_groups; i++) { in pqi_reinit_queues()
8535 ctrl_info->queue_groups[i].iq_pi_copy[RAID_PATH] = 0; in pqi_reinit_queues()
8536 ctrl_info->queue_groups[i].iq_pi_copy[AIO_PATH] = 0; in pqi_reinit_queues()
8537 ctrl_info->queue_groups[i].oq_ci_copy = 0; in pqi_reinit_queues()
8539 writel(0, ctrl_info->queue_groups[i].iq_ci[RAID_PATH]); in pqi_reinit_queues()
8540 writel(0, ctrl_info->queue_groups[i].iq_ci[AIO_PATH]); in pqi_reinit_queues()
8541 writel(0, ctrl_info->queue_groups[i].oq_pi); in pqi_reinit_queues()
8544 event_queue = &ctrl_info->event_queue; in pqi_reinit_queues()
8545 writel(0, event_queue->oq_pi); in pqi_reinit_queues()
8546 event_queue->oq_ci_copy = 0; in pqi_reinit_queues()
8571 dev_err(&ctrl_info->pci_dev->dev, in pqi_ctrl_init_resume()
8578 dev_err(&ctrl_info->pci_dev->dev, in pqi_ctrl_init_resume()
8590 dev_err(&ctrl_info->pci_dev->dev, in pqi_ctrl_init_resume()
8595 /* Wait for the controller to complete the SIS -> PQI transition. */ in pqi_ctrl_init_resume()
8598 dev_err(&ctrl_info->pci_dev->dev, in pqi_ctrl_init_resume()
8604 ctrl_info->pqi_mode_enabled = true; in pqi_ctrl_init_resume()
8611 dev_err(&ctrl_info->pci_dev->dev, in pqi_ctrl_init_resume()
8622 ctrl_info->controller_online = true; in pqi_ctrl_init_resume()
8633 if (ctrl_info->enable_r5_writes || ctrl_info->enable_r6_writes) { in pqi_ctrl_init_resume()
8636 dev_err(&ctrl_info->pci_dev->dev, in pqi_ctrl_init_resume()
8640 ctrl_info->ciss_report_log_flags |= in pqi_ctrl_init_resume()
8646 dev_err(&ctrl_info->pci_dev->dev, in pqi_ctrl_init_resume()
8653 dev_err(&ctrl_info->pci_dev->dev, in pqi_ctrl_init_resume()
8660 dev_err(&ctrl_info->pci_dev->dev, in pqi_ctrl_init_resume()
8661 "error enabling multi-lun rescan\n"); in pqi_ctrl_init_resume()
8667 dev_err(&ctrl_info->pci_dev->dev, in pqi_ctrl_init_resume()
8674 if (ctrl_info->ctrl_logging_supported) { in pqi_ctrl_init_resume()
8675 if (!ctrl_info->ctrl_log_memory.host_memory) in pqi_ctrl_init_resume()
8677 &ctrl_info->ctrl_log_memory, in pqi_ctrl_init_resume()
8681 &ctrl_info->ctrl_log_memory, PQI_VENDOR_GENERAL_CTRL_LOG_MEMORY_UPDATE); in pqi_ctrl_init_resume()
8683 if (ctrl_info->ctrl_log_memory.host_memory) in pqi_ctrl_init_resume()
8685 &ctrl_info->ctrl_log_memory); in pqi_ctrl_init_resume()
8709 rc = pci_enable_device(ctrl_info->pci_dev); in pqi_pci_init()
8711 dev_err(&ctrl_info->pci_dev->dev, in pqi_pci_init()
8721 rc = dma_set_mask_and_coherent(&ctrl_info->pci_dev->dev, mask); in pqi_pci_init()
8723 dev_err(&ctrl_info->pci_dev->dev, "failed to set DMA mask\n"); in pqi_pci_init()
8727 rc = pci_request_regions(ctrl_info->pci_dev, DRIVER_NAME_SHORT); in pqi_pci_init()
8729 dev_err(&ctrl_info->pci_dev->dev, in pqi_pci_init()
8734 ctrl_info->iomem_base = ioremap(pci_resource_start( in pqi_pci_init()
8735 ctrl_info->pci_dev, 0), in pqi_pci_init()
8736 pci_resource_len(ctrl_info->pci_dev, 0)); in pqi_pci_init()
8737 if (!ctrl_info->iomem_base) { in pqi_pci_init()
8738 dev_err(&ctrl_info->pci_dev->dev, in pqi_pci_init()
8740 rc = -ENOMEM; in pqi_pci_init()
8747 rc = pqi_set_pcie_completion_timeout(ctrl_info->pci_dev, in pqi_pci_init()
8750 dev_err(&ctrl_info->pci_dev->dev, in pqi_pci_init()
8756 pci_set_master(ctrl_info->pci_dev); in pqi_pci_init()
8758 ctrl_info->registers = ctrl_info->iomem_base; in pqi_pci_init()
8759 ctrl_info->pqi_registers = &ctrl_info->registers->pqi_registers; in pqi_pci_init()
8761 pci_set_drvdata(ctrl_info->pci_dev, ctrl_info); in pqi_pci_init()
8766 pci_release_regions(ctrl_info->pci_dev); in pqi_pci_init()
8768 pci_disable_device(ctrl_info->pci_dev); in pqi_pci_init()
8775 iounmap(ctrl_info->iomem_base); in pqi_cleanup_pci_init()
8776 pci_release_regions(ctrl_info->pci_dev); in pqi_cleanup_pci_init()
8777 if (pci_is_enabled(ctrl_info->pci_dev)) in pqi_cleanup_pci_init()
8778 pci_disable_device(ctrl_info->pci_dev); in pqi_cleanup_pci_init()
8779 pci_set_drvdata(ctrl_info->pci_dev, NULL); in pqi_cleanup_pci_init()
8791 mutex_init(&ctrl_info->scan_mutex); in pqi_alloc_ctrl_info()
8792 mutex_init(&ctrl_info->lun_reset_mutex); in pqi_alloc_ctrl_info()
8793 mutex_init(&ctrl_info->ofa_mutex); in pqi_alloc_ctrl_info()
8795 INIT_LIST_HEAD(&ctrl_info->scsi_device_list); in pqi_alloc_ctrl_info()
8796 spin_lock_init(&ctrl_info->scsi_device_list_lock); in pqi_alloc_ctrl_info()
8798 INIT_WORK(&ctrl_info->event_work, pqi_event_worker); in pqi_alloc_ctrl_info()
8799 atomic_set(&ctrl_info->num_interrupts, 0); in pqi_alloc_ctrl_info()
8801 INIT_DELAYED_WORK(&ctrl_info->rescan_work, pqi_rescan_worker); in pqi_alloc_ctrl_info()
8802 INIT_DELAYED_WORK(&ctrl_info->update_time_work, pqi_update_time_worker); in pqi_alloc_ctrl_info()
8804 timer_setup(&ctrl_info->heartbeat_timer, pqi_heartbeat_timer_handler, 0); in pqi_alloc_ctrl_info()
8805 INIT_WORK(&ctrl_info->ctrl_offline_work, pqi_ctrl_offline_worker); in pqi_alloc_ctrl_info()
8807 INIT_WORK(&ctrl_info->ofa_memory_alloc_work, pqi_ofa_memory_alloc_worker); in pqi_alloc_ctrl_info()
8808 INIT_WORK(&ctrl_info->ofa_quiesce_work, pqi_ofa_quiesce_worker); in pqi_alloc_ctrl_info()
8810 sema_init(&ctrl_info->sync_request_sem, in pqi_alloc_ctrl_info()
8812 init_waitqueue_head(&ctrl_info->block_requests_wait); in pqi_alloc_ctrl_info()
8814 ctrl_info->ctrl_id = atomic_inc_return(&pqi_controller_count) - 1; in pqi_alloc_ctrl_info()
8815 ctrl_info->irq_mode = IRQ_MODE_NONE; in pqi_alloc_ctrl_info()
8816 ctrl_info->max_msix_vectors = PQI_MAX_MSIX_VECTORS; in pqi_alloc_ctrl_info()
8818 ctrl_info->ciss_report_log_flags = CISS_REPORT_LOG_FLAG_UNIQUE_LUN_ID; in pqi_alloc_ctrl_info()
8819 ctrl_info->max_transfer_encrypted_sas_sata = in pqi_alloc_ctrl_info()
8821 ctrl_info->max_transfer_encrypted_nvme = in pqi_alloc_ctrl_info()
8823 ctrl_info->max_write_raid_5_6 = PQI_DEFAULT_MAX_WRITE_RAID_5_6; in pqi_alloc_ctrl_info()
8824 ctrl_info->max_write_raid_1_10_2drive = ~0; in pqi_alloc_ctrl_info()
8825 ctrl_info->max_write_raid_1_10_3drive = ~0; in pqi_alloc_ctrl_info()
8826 ctrl_info->disable_managed_interrupts = pqi_disable_managed_interrupts; in pqi_alloc_ctrl_info()
8845 if (ctrl_info->queue_memory_base) in pqi_free_ctrl_resources()
8846 dma_free_coherent(&ctrl_info->pci_dev->dev, in pqi_free_ctrl_resources()
8847 ctrl_info->queue_memory_length, in pqi_free_ctrl_resources()
8848 ctrl_info->queue_memory_base, in pqi_free_ctrl_resources()
8849 ctrl_info->queue_memory_base_dma_handle); in pqi_free_ctrl_resources()
8850 if (ctrl_info->admin_queue_memory_base) in pqi_free_ctrl_resources()
8851 dma_free_coherent(&ctrl_info->pci_dev->dev, in pqi_free_ctrl_resources()
8852 ctrl_info->admin_queue_memory_length, in pqi_free_ctrl_resources()
8853 ctrl_info->admin_queue_memory_base, in pqi_free_ctrl_resources()
8854 ctrl_info->admin_queue_memory_base_dma_handle); in pqi_free_ctrl_resources()
8856 if (ctrl_info->error_buffer) in pqi_free_ctrl_resources()
8857 dma_free_coherent(&ctrl_info->pci_dev->dev, in pqi_free_ctrl_resources()
8858 ctrl_info->error_buffer_length, in pqi_free_ctrl_resources()
8859 ctrl_info->error_buffer, in pqi_free_ctrl_resources()
8860 ctrl_info->error_buffer_dma_handle); in pqi_free_ctrl_resources()
8861 if (ctrl_info->iomem_base) in pqi_free_ctrl_resources()
8868 ctrl_info->controller_online = false; in pqi_remove_ctrl()
8873 if (ctrl_info->ctrl_removal_state == PQI_CTRL_SURPRISE_REMOVAL) { in pqi_remove_ctrl()
8875 ctrl_info->pqi_mode_enabled = false; in pqi_remove_ctrl()
8877 pqi_host_free_buffer(ctrl_info, &ctrl_info->ctrl_log_memory); in pqi_remove_ctrl()
8879 if (ctrl_info->pqi_mode_enabled) in pqi_remove_ctrl()
8925 host_memory_descriptor->host_chunk_virt_address = kmalloc(sg_count * sizeof(void *), GFP_KERNEL); in pqi_host_alloc_mem()
8926 if (!host_memory_descriptor->host_chunk_virt_address) in pqi_host_alloc_mem()
8929 dev = &ctrl_info->pci_dev->dev; in pqi_host_alloc_mem()
8930 host_memory = host_memory_descriptor->host_memory; in pqi_host_alloc_mem()
8933 …host_memory_descriptor->host_chunk_virt_address[i] = dma_alloc_coherent(dev, chunk_size, &dma_hand… in pqi_host_alloc_mem()
8934 if (!host_memory_descriptor->host_chunk_virt_address[i]) in pqi_host_alloc_mem()
8936 mem_descriptor = &host_memory->sg_descriptor[i]; in pqi_host_alloc_mem()
8937 put_unaligned_le64((u64)dma_handle, &mem_descriptor->address); in pqi_host_alloc_mem()
8938 put_unaligned_le32(chunk_size, &mem_descriptor->length); in pqi_host_alloc_mem()
8941 put_unaligned_le32(CISS_SG_LAST, &mem_descriptor->flags); in pqi_host_alloc_mem()
8942 put_unaligned_le16(sg_count, &host_memory->num_memory_descriptors); in pqi_host_alloc_mem()
8943 put_unaligned_le32(sg_count * chunk_size, &host_memory->bytes_allocated); in pqi_host_alloc_mem()
8948 while (--i >= 0) { in pqi_host_alloc_mem()
8949 mem_descriptor = &host_memory->sg_descriptor[i]; in pqi_host_alloc_mem()
8951 host_memory_descriptor->host_chunk_virt_address[i], in pqi_host_alloc_mem()
8952 get_unaligned_le64(&mem_descriptor->address)); in pqi_host_alloc_mem()
8954 kfree(host_memory_descriptor->host_chunk_virt_address); in pqi_host_alloc_mem()
8956 return -ENOMEM; in pqi_host_alloc_mem()
8987 return -ENOMEM; in pqi_host_alloc_buffer()
8997 dev = &ctrl_info->pci_dev->dev; in pqi_host_setup_buffer()
9000 &host_memory_descriptor->host_memory_dma_handle, GFP_KERNEL); in pqi_host_setup_buffer()
9004 host_memory_descriptor->host_memory = host_memory; in pqi_host_setup_buffer()
9010 host_memory_descriptor->host_memory_dma_handle); in pqi_host_setup_buffer()
9011 host_memory_descriptor->host_memory = NULL; in pqi_host_setup_buffer()
9025 host_memory = host_memory_descriptor->host_memory; in pqi_host_free_buffer()
9029 dev = &ctrl_info->pci_dev->dev; in pqi_host_free_buffer()
9031 if (get_unaligned_le32(&host_memory->bytes_allocated) == 0) in pqi_host_free_buffer()
9034 mem_descriptor = host_memory->sg_descriptor; in pqi_host_free_buffer()
9035 num_memory_descriptors = get_unaligned_le16(&host_memory->num_memory_descriptors); in pqi_host_free_buffer()
9040 host_memory_descriptor->host_chunk_virt_address[i], in pqi_host_free_buffer()
9043 kfree(host_memory_descriptor->host_chunk_virt_address); in pqi_host_free_buffer()
9047 host_memory_descriptor->host_memory_dma_handle); in pqi_host_free_buffer()
9048 host_memory_descriptor->host_memory = NULL; in pqi_host_free_buffer()
9062 put_unaligned_le16(sizeof(request) - PQI_REQUEST_HEADER_LENGTH, &request.header.iu_length); in pqi_host_memory_update()
9065 host_memory = host_memory_descriptor->host_memory; in pqi_host_memory_update()
9068 …tof(struct pqi_host_memory, sg_descriptor) + get_unaligned_le16(&host_memory->num_memory_descripto… in pqi_host_memory_update()
9069 …put_unaligned_le64((u64)host_memory_descriptor->host_memory_dma_handle, &request.data.host_memory_… in pqi_host_memory_update()
9073 put_unaligned_le16(PQI_OFA_VERSION, &host_memory->version); in pqi_host_memory_update()
9074 memcpy(&host_memory->signature, PQI_OFA_SIGNATURE, sizeof(host_memory->signature)); in pqi_host_memory_update()
9076 put_unaligned_le16(PQI_CTRL_LOG_VERSION, &host_memory->version); in pqi_host_memory_update()
9077 memcpy(&host_memory->signature, PQI_CTRL_LOG_SIGNATURE, sizeof(host_memory->signature)); in pqi_host_memory_update()
9096 for (i = 0; i < ctrl_info->max_io_slots; i++) { in pqi_fail_all_outstanding_requests()
9097 io_request = &ctrl_info->io_request_pool[i]; in pqi_fail_all_outstanding_requests()
9098 if (atomic_read(&io_request->refcount) == 0) in pqi_fail_all_outstanding_requests()
9101 scmd = io_request->scmd; in pqi_fail_all_outstanding_requests()
9103 sdev = scmd->device; in pqi_fail_all_outstanding_requests()
9111 io_request->status = -ENXIO; in pqi_fail_all_outstanding_requests()
9112 io_request->error_info = in pqi_fail_all_outstanding_requests()
9116 io_request->io_complete_callback(io_request, in pqi_fail_all_outstanding_requests()
9117 io_request->context); in pqi_fail_all_outstanding_requests()
9190 if (!ctrl_info->controller_online) in pqi_take_ctrl_offline()
9193 ctrl_info->controller_online = false; in pqi_take_ctrl_offline()
9194 ctrl_info->pqi_mode_enabled = false; in pqi_take_ctrl_offline()
9198 pci_disable_device(ctrl_info->pci_dev); in pqi_take_ctrl_offline()
9199 dev_err(&ctrl_info->pci_dev->dev, in pqi_take_ctrl_offline()
9202 schedule_work(&ctrl_info->ctrl_offline_work); in pqi_take_ctrl_offline()
9210 if (id->driver_data) in pqi_print_ctrl_info()
9211 ctrl_description = (char *)id->driver_data; in pqi_print_ctrl_info()
9215 dev_info(&pci_dev->dev, "%s found\n", ctrl_description); in pqi_print_ctrl_info()
9228 id->subvendor == PCI_ANY_ID && in pqi_pci_probe()
9229 id->subdevice == PCI_ANY_ID) { in pqi_pci_probe()
9230 dev_warn(&pci_dev->dev, in pqi_pci_probe()
9232 return -ENODEV; in pqi_pci_probe()
9235 if (id->subvendor == PCI_ANY_ID || id->subdevice == PCI_ANY_ID) in pqi_pci_probe()
9236 dev_warn(&pci_dev->dev, in pqi_pci_probe()
9239 node = dev_to_node(&pci_dev->dev); in pqi_pci_probe()
9244 set_dev_node(&pci_dev->dev, node); in pqi_pci_probe()
9249 dev_err(&pci_dev->dev, in pqi_pci_probe()
9251 return -ENOMEM; in pqi_pci_probe()
9253 ctrl_info->numa_node = node; in pqi_pci_probe()
9255 ctrl_info->pci_dev = pci_dev; in pqi_pci_probe()
9283 pci_read_config_word(ctrl_info->pci_dev, PCI_SUBSYSTEM_VENDOR_ID, &vendor_id); in pqi_pci_remove()
9285 ctrl_info->ctrl_removal_state = PQI_CTRL_SURPRISE_REMOVAL; in pqi_pci_remove()
9287 ctrl_info->ctrl_removal_state = PQI_CTRL_GRACEFUL_REMOVAL; in pqi_pci_remove()
9289 if (ctrl_info->ctrl_removal_state == PQI_CTRL_GRACEFUL_REMOVAL) { in pqi_pci_remove()
9292 dev_err(&pci_dev->dev, in pqi_pci_remove()
9305 for (i = 0; i < ctrl_info->max_io_slots; i++) { in pqi_crash_if_pending_command()
9306 io_request = &ctrl_info->io_request_pool[i]; in pqi_crash_if_pending_command()
9307 if (atomic_read(&io_request->refcount) == 0) in pqi_crash_if_pending_command()
9309 scmd = io_request->scmd; in pqi_crash_if_pending_command()
9310 WARN_ON(scmd != NULL); /* IO command from SML */ in pqi_crash_if_pending_command()
9311 WARN_ON(scmd == NULL); /* Non-IO cmd or driver initiated*/ in pqi_crash_if_pending_command()
9323 dev_err(&pci_dev->dev, in pqi_shutdown()
9341 * Write all data in the controller's battery-backed cache to in pqi_shutdown()
9346 dev_err(&pci_dev->dev, in pqi_shutdown()
9368 pr_warn("%s: invalid lockup action setting \"%s\" - supported settings: none, reboot, panic\n", in pqi_process_lockup_action_param()
9381 …pr_warn("%s: ctrl_ready_timeout parm of %u second(s) is less than minimum timeout of %d seconds - in pqi_process_ctrl_ready_timeout_param()
9385 …dy_timeout parm of %u seconds is greater than maximum timeout of %d seconds - setting timeout to %… in pqi_process_ctrl_ready_timeout_param()
9403 if (pci_dev->subsystem_vendor == PCI_VENDOR_ID_ADAPTEC2 && pci_dev->subsystem_device == 0x1304) in pqi_get_flush_cache_shutdown_event()
9436 ctrl_info->controller_online = false; in pqi_suspend_or_freeze()
9437 ctrl_info->pqi_mode_enabled = false; in pqi_suspend_or_freeze()
9488 ctrl_info->controller_online = true; in pqi_thaw()
9489 ctrl_info->pqi_mode_enabled = true; in pqi_thaw()
10535 return -ENODEV; in pqi_init()
10728 data.create_operational_iq) != 64 - 11); in pqi_verify_structures()
10730 data.create_operational_oq) != 64 - 11); in pqi_verify_structures()
10732 data.delete_operational_queue) != 64 - 11); in pqi_verify_structures()