diff --git a/sys/dev/mpi3mr/mpi3mr.h b/sys/dev/mpi3mr/mpi3mr.h index 2226c747d3cf..f1a2cbc0fd4c 100644 --- a/sys/dev/mpi3mr/mpi3mr.h +++ b/sys/dev/mpi3mr/mpi3mr.h @@ -1,990 +1,989 @@ /* * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2020-2023, Broadcom Inc. All rights reserved. * Support: * * Authors: Sumit Saxena * Chandrakanth Patil * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are * met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation and/or other * materials provided with the distribution. * 3. Neither the name of the Broadcom Inc. nor the names of its contributors * may be used to endorse or promote products derived from this software without * specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. * * The views and conclusions contained in the software and documentation are * those of the authors and should not be interpreted as representing * official policies,either expressed or implied, of the FreeBSD Project. * * Mail to: Broadcom Inc 1320 Ridder Park Dr, San Jose, CA 95131 * * Broadcom Inc. (Broadcom) MPI3MR Adapter FreeBSD */ #ifndef _MPI3MRVAR_H #define _MPI3MRVAR_H #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "mpi/mpi30_api.h" #define MPI3MR_DRIVER_VERSION "8.6.0.2.0" #define MPI3MR_DRIVER_RELDATE "17th May 2023" #define MPI3MR_DRIVER_NAME "mpi3mr" #define MPI3MR_NAME_LENGTH 32 #define IOCNAME "%s: " #define SAS4116_CHIP_REV_A0 0 #define SAS4116_CHIP_REV_B0 1 #define MPI3MR_SG_DEPTH (MPI3MR_4K_PGSZ/sizeof(Mpi3SGESimple_t)) #define MPI3MR_MAX_SECTORS 2048 #define MPI3MR_MAX_CMDS_LUN 7 #define MPI3MR_MAX_CDB_LENGTH 16 #define MPI3MR_MAX_LUN 16895 #define MPI3MR_SATA_QDEPTH 32 #define MPI3MR_SAS_QDEPTH 64 #define MPI3MR_RAID_QDEPTH 128 #define MPI3MR_NVME_QDEPTH 128 #define MPI3MR_4K_PGSZ 4096 #define MPI3MR_AREQQ_SIZE (2 * MPI3MR_4K_PGSZ) #define MPI3MR_AREPQ_SIZE (4 * MPI3MR_4K_PGSZ) #define MPI3MR_AREQ_FRAME_SZ 128 #define MPI3MR_AREP_FRAME_SZ 16 #define MPI3MR_OPREQQ_SIZE (8 * MPI3MR_4K_PGSZ) #define MPI3MR_OPREPQ_SIZE (4 * MPI3MR_4K_PGSZ) /* Operational queue management definitions */ #define MPI3MR_OP_REQ_Q_QD 512 #define MPI3MR_OP_REP_Q_QD 1024 #define MPI3MR_OP_REP_Q_QD_A0 4096 #define MPI3MR_CHAINSGE_SIZE MPI3MR_4K_PGSZ #define MPI3MR_SGEFLAGS_SYSTEM_SIMPLE_END_OF_LIST \ (MPI3_SGE_FLAGS_ELEMENT_TYPE_SIMPLE | MPI3_SGE_FLAGS_DLAS_SYSTEM | \ MPI3_SGE_FLAGS_END_OF_LIST) #define MPI3MR_HOSTTAG_INVALID 0xFFFF #define MPI3MR_HOSTTAG_INITCMDS 1 #define MPI3MR_HOSTTAG_IOCTLCMDS 2 #define MPI3MR_HOSTTAG_PELABORT 3 #define MPI3MR_HOSTTAG_PELWAIT 4 #define MPI3MR_HOSTTAG_TMS 5 #define MAX_MGMT_ADAPTERS 8 #define MPI3MR_WAIT_BEFORE_CTRL_RESET 5 struct mpi3mr_mgmt_info { uint16_t count; struct mpi3mr_softc *sc_ptr[MAX_MGMT_ADAPTERS]; int max_index; }; extern char fmt_os_ver[16]; #define MPI3MR_OS_VERSION(raw_os_ver, fmt_os_ver) sprintf(raw_os_ver, "%d", __FreeBSD_version); \ sprintf(fmt_os_ver, "%c%c.%c%c.%c%c%c",\ raw_os_ver[0], raw_os_ver[1], raw_os_ver[2],\ raw_os_ver[3], raw_os_ver[4], raw_os_ver[5],\ raw_os_ver[6]); #define MPI3MR_NUM_DEVRMCMD 1 #define MPI3MR_HOSTTAG_DEVRMCMD_MIN (MPI3MR_HOSTTAG_TMS + 1) #define MPI3MR_HOSTTAG_DEVRMCMD_MAX (MPI3MR_HOSTTAG_DEVRMCMD_MIN + \ MPI3MR_NUM_DEVRMCMD - 1) #define MPI3MR_INTERNALCMDS_RESVD MPI3MR_HOSTTAG_DEVRMCMD_MAX #define MPI3MR_NUM_EVTACKCMD 4 #define MPI3MR_HOSTTAG_EVTACKCMD_MIN (MPI3MR_HOSTTAG_DEVRMCMD_MAX + 1) #define MPI3MR_HOSTTAG_EVTACKCMD_MAX (MPI3MR_HOSTTAG_EVTACKCMD_MIN + \ MPI3MR_NUM_EVTACKCMD - 1) /* command/controller interaction timeout definitions in seconds */ #define MPI3MR_INTADMCMD_TIMEOUT 60 #define MPI3MR_PORTENABLE_TIMEOUT 300 #define MPI3MR_ABORTTM_TIMEOUT 60 #define MPI3MR_RESETTM_TIMEOUT 60 #define MPI3MR_TSUPDATE_INTERVAL 900 #define MPI3MR_DEFAULT_SHUTDOWN_TIME 120 #define MPI3MR_RAID_ERRREC_RESET_TIMEOUT 180 #define MPI3MR_RESET_HOST_IOWAIT_TIMEOUT 5 #define MPI3MR_PREPARE_FOR_RESET_TIMEOUT 180 #define MPI3MR_RESET_ACK_TIMEOUT 30 #define MPI3MR_MUR_TIMEOUT 120 #define MPI3MR_CMD_NOTUSED 0x8000 #define MPI3MR_CMD_COMPLETE 0x0001 #define MPI3MR_CMD_PENDING 0x0002 #define MPI3MR_CMD_REPLYVALID 0x0004 #define MPI3MR_CMD_RESET 0x0008 #define MPI3MR_NUM_EVTREPLIES 64 #define MPI3MR_SENSEBUF_SZ 256 #define MPI3MR_SENSEBUF_FACTOR 3 #define MPI3MR_CHAINBUF_FACTOR 3 #define MPT3SAS_HOSTPGSZ_4KEXP 12 #define MPI3MR_INVALID_DEV_HANDLE 0xFFFF /* Controller Reset related definitions */ #define MPI3MR_HOSTDIAG_UNLOCK_RETRY_COUNT 5 #define MPI3MR_MAX_SHUTDOWN_RETRY_COUNT 2 /* ResponseCode values */ #define MPI3MR_RI_MASK_RESPCODE (0x000000FF) #define MPI3MR_RSP_TM_COMPLETE 0x00 #define MPI3MR_RSP_INVALID_FRAME 0x02 #define MPI3MR_RSP_TM_NOT_SUPPORTED 0x04 #define MPI3MR_RSP_TM_FAILED 0x05 #define MPI3MR_RSP_TM_SUCCEEDED 0x08 #define MPI3MR_RSP_TM_INVALID_LUN 0x09 #define MPI3MR_RSP_TM_OVERLAPPED_TAG 0x0A #define MPI3MR_RSP_IO_QUEUED_ON_IOC \ MPI3_SCSITASKMGMT_RSPCODE_IO_QUEUED_ON_IOC /* Definitions for the controller security status*/ #define MPI3MR_CTLR_SECURITY_STATUS_MASK 0x0C #define MPI3MR_CTLR_SECURE_DBG_STATUS_MASK 0x02 #define MPI3MR_INVALID_DEVICE 0x00 #define MPI3MR_CONFIG_SECURE_DEVICE 0x04 #define MPI3MR_HARD_SECURE_DEVICE 0x08 #define MPI3MR_TAMPERED_DEVICE 0x0C #define MPI3MR_DEFAULT_MDTS (128 * 1024) #define MPI3MR_DEFAULT_PGSZEXP (12) #define MPI3MR_MAX_IOCTL_TRANSFER_SIZE (1024 * 1024) #define MPI3MR_DEVRMHS_RETRYCOUNT 3 #define MPI3MR_PELCMDS_RETRYCOUNT 3 #define MPI3MR_PERIODIC_DELAY 1 /* 1 second heartbeat/watchdog check */ struct completion { unsigned int done; struct mtx lock; }; typedef union { volatile unsigned int val; unsigned int val_rdonly; } mpi3mr_atomic_t; #define mpi3mr_atomic_read(v) atomic_load_acq_int(&(v)->val) #define mpi3mr_atomic_set(v,i) atomic_store_rel_int(&(v)->val, i) #define mpi3mr_atomic_dec(v) atomic_subtract_int(&(v)->val, 1) #define mpi3mr_atomic_inc(v) atomic_add_int(&(v)->val, 1) #define mpi3mr_atomic_add(v, u) atomic_add_int(&(v)->val, u) #define mpi3mr_atomic_sub(v, u) atomic_subtract_int(&(v)->val, u) /* IOCTL data transfer sge*/ #define MPI3MR_NUM_IOCTL_SGE 256 #define MPI3MR_IOCTL_SGE_SIZE (8 * 1024) struct dma_memory_desc { U32 size; void *addr; bus_dma_tag_t tag; bus_dmamap_t dmamap; bus_addr_t dma_addr; }; enum mpi3mr_iocstate { MRIOC_STATE_READY = 1, MRIOC_STATE_RESET, MRIOC_STATE_FAULT, MRIOC_STATE_BECOMING_READY, MRIOC_STATE_RESET_REQUESTED, MRIOC_STATE_UNRECOVERABLE, MRIOC_STATE_COUNT, }; /* Init type definitions */ enum mpi3mr_init_type { MPI3MR_INIT_TYPE_INIT = 0, MPI3MR_INIT_TYPE_RESET, MPI3MR_INIT_TYPE_RESUME, }; /* Reset reason code definitions*/ enum mpi3mr_reset_reason { MPI3MR_RESET_FROM_BRINGUP = 1, MPI3MR_RESET_FROM_FAULT_WATCH = 2, MPI3MR_RESET_FROM_IOCTL = 3, MPI3MR_RESET_FROM_EH_HOS = 4, MPI3MR_RESET_FROM_TM_TIMEOUT = 5, MPI3MR_RESET_FROM_IOCTL_TIMEOUT = 6, MPI3MR_RESET_FROM_MUR_FAILURE = 7, MPI3MR_RESET_FROM_CTLR_CLEANUP = 8, MPI3MR_RESET_FROM_CIACTIV_FAULT = 9, MPI3MR_RESET_FROM_PE_TIMEOUT = 10, MPI3MR_RESET_FROM_TSU_TIMEOUT = 11, MPI3MR_RESET_FROM_DELREQQ_TIMEOUT = 12, MPI3MR_RESET_FROM_DELREPQ_TIMEOUT = 13, MPI3MR_RESET_FROM_CREATEREPQ_TIMEOUT = 14, MPI3MR_RESET_FROM_CREATEREQQ_TIMEOUT = 15, MPI3MR_RESET_FROM_IOCFACTS_TIMEOUT = 16, MPI3MR_RESET_FROM_IOCINIT_TIMEOUT = 17, MPI3MR_RESET_FROM_EVTNOTIFY_TIMEOUT = 18, MPI3MR_RESET_FROM_EVTACK_TIMEOUT = 19, MPI3MR_RESET_FROM_CIACTVRST_TIMER = 20, MPI3MR_RESET_FROM_GETPKGVER_TIMEOUT = 21, MPI3MR_RESET_FROM_PELABORT_TIMEOUT = 22, MPI3MR_RESET_FROM_SYSFS = 23, MPI3MR_RESET_FROM_SYSFS_TIMEOUT = 24, MPI3MR_RESET_FROM_DIAG_BUFFER_POST_TIMEOUT = 25, MPI3MR_RESET_FROM_SCSIIO_TIMEOUT = 26, MPI3MR_RESET_FROM_FIRMWARE = 27, MPI3MR_DEFAULT_RESET_REASON = 28, MPI3MR_RESET_REASON_COUNT, }; struct mpi3mr_compimg_ver { U16 build_num; U16 cust_id; U8 ph_minor; U8 ph_major; U8 gen_minor; U8 gen_major; }; struct mpi3mr_ioc_facts { U32 ioc_capabilities; struct mpi3mr_compimg_ver fw_ver; U32 mpi_version; U16 max_reqs; U16 product_id; U16 op_req_sz; U16 reply_sz; U16 exceptions; U16 max_perids; U16 max_pds; U16 max_sasexpanders; U16 max_sasinitiators; U16 max_enclosures; U16 max_pcieswitches; U16 max_nvme; U16 max_vds; U16 max_hpds; U16 max_advhpds; U16 max_raidpds; U16 min_devhandle; U16 max_devhandle; U16 max_op_req_q; U16 max_op_reply_q; U16 shutdown_timeout; U8 ioc_num; U8 who_init; U16 max_msix_vectors; U8 personality; U8 dma_mask; U8 protocol_flags; U8 sge_mod_mask; U8 sge_mod_value; U8 sge_mod_shift; U8 max_dev_per_tg; U16 max_io_throttle_group; U16 io_throttle_data_length; U16 io_throttle_low; U16 io_throttle_high; }; struct mpi3mr_op_req_queue { U16 ci; U16 pi; U16 num_reqs; U8 qid; U8 reply_qid; U32 qsz; void *q_base; bus_dma_tag_t q_base_tag; bus_dmamap_t q_base_dmamap; bus_addr_t q_base_phys; struct mtx q_lock; }; struct mpi3mr_op_reply_queue { U16 ci; U8 ephase; U8 qid; U16 num_replies; U32 qsz; bus_dma_tag_t q_base_tag; bus_dmamap_t q_base_dmamap; void *q_base; bus_addr_t q_base_phys; mpi3mr_atomic_t pend_ios; bool in_use; struct mtx q_lock; }; struct irq_info { MPI3_REPLY_DESCRIPTORS_UNION *post_queue; bus_dma_tag_t buffer_dmat; struct resource *irq; void *intrhand; int irq_rid; }; struct mpi3mr_irq_context { struct mpi3mr_softc *sc; U16 msix_index; struct mpi3mr_op_reply_queue *op_reply_q; char name[MPI3MR_NAME_LENGTH]; struct irq_info irq_info; }; MALLOC_DECLARE(M_MPI3MR); SYSCTL_DECL(_hw_mpi3mr); typedef struct mpi3mr_drvr_cmd DRVR_CMD; typedef void (*DRVR_CMD_CALLBACK)(struct mpi3mr_softc *mrioc, DRVR_CMD *drvrcmd); struct mpi3mr_drvr_cmd { struct mtx lock; struct completion completion; void *reply; U8 *sensebuf; U8 iou_rc; U16 state; U16 dev_handle; U16 ioc_status; U32 ioc_loginfo; U8 is_waiting; U8 is_senseprst; U8 retry_count; U16 host_tag; DRVR_CMD_CALLBACK callback; }; struct mpi3mr_cmd; typedef void mpi3mr_evt_callback_t(struct mpi3mr_softc *, uintptr_t, Mpi3EventNotificationReply_t *reply); typedef void mpi3mr_cmd_callback_t(struct mpi3mr_softc *, struct mpi3mr_cmd *cmd); #define MPI3MR_IOVEC_COUNT 2 enum mpi3mr_data_xfer_direction { MPI3MR_READ = 1, MPI3MR_WRITE, }; enum mpi3mr_cmd_state { MPI3MR_CMD_STATE_FREE = 1, MPI3MR_CMD_STATE_BUSY, MPI3MR_CMD_STATE_IN_QUEUE, MPI3MR_CMD_STATE_IN_TM, }; enum mpi3mr_target_state { MPI3MR_DEV_CREATED = 1, MPI3MR_DEV_REMOVE_HS_STARTED = 2, MPI3MR_DEV_DELETED = 3, }; struct mpi3mr_cmd { TAILQ_ENTRY(mpi3mr_cmd) next; struct mpi3mr_softc *sc; union ccb *ccb; void *data; u_int length; struct mpi3mr_target *targ; u_int data_dir; u_int state; bus_dmamap_t dmamap; struct scsi_sense_data *sense; struct callout callout; bool callout_owner; U16 hosttag; U8 req_qidx; Mpi3SCSIIORequest_t io_request; - int error_code; }; struct mpi3mr_chain { bus_dmamap_t buf_dmamap; void *buf; bus_addr_t buf_phys; }; struct mpi3mr_event_handle { TAILQ_ENTRY(mpi3mr_event_handle) eh_list; mpi3mr_evt_callback_t *callback; void *data; uint8_t mask[16]; }; struct mpi3mr_fw_event_work { U16 event; void *event_data; TAILQ_ENTRY(mpi3mr_fw_event_work) ev_link; U8 send_ack; U8 process_event; U32 event_context; U16 event_data_size; }; /** * struct delayed_dev_rmhs_node - Delayed device removal node * * @list: list head * @handle: Device handle * @iou_rc: IO Unit Control Reason Code */ struct delayed_dev_rmhs_node { TAILQ_ENTRY(delayed_dev_rmhs_node) list; U16 handle; U8 iou_rc; }; /** * struct delayed_evtack_node - Delayed event ack node * * @list: list head * @event: MPI3 event ID * @event_ctx: Event context */ struct delayed_evtack_node { TAILQ_ENTRY(delayed_evtack_node) list; U8 event; U32 event_ctx; }; /* Reset types */ enum reset_type { MPI3MR_NO_RESET, MPI3MR_TRIGGER_SOFT_RESET, }; struct mpi3mr_reset { u_int type; U32 reason; int status; bool ioctl_reset_snapdump; }; struct mpi3mr_softc { device_t mpi3mr_dev; struct cdev *mpi3mr_cdev; u_int mpi3mr_flags; #define MPI3MR_FLAGS_SHUTDOWN (1 << 0) #define MPI3MR_FLAGS_DIAGRESET (1 << 1) #define MPI3MR_FLAGS_ATTACH_DONE (1 << 2) #define MPI3MR_FLAGS_PORT_ENABLE_DONE (1 << 3) U8 id; int cpu_count; char name[MPI3MR_NAME_LENGTH]; char driver_name[MPI3MR_NAME_LENGTH]; int bars; int dma_mask; u_int mpi3mr_debug; struct mpi3mr_reset reset; int max_msix_vectors; int msix_count; bool msix_enable; int io_cmds_highwater; int max_chains; uint32_t chain_frame_size; struct sysctl_ctx_list sysctl_ctx; struct sysctl_oid *sysctl_tree; char fw_version[16]; char msg_version[8]; struct mpi3mr_chain *chains; struct callout periodic; struct callout device_check_callout; struct mpi3mr_cam_softc *cam_sc; struct mpi3mr_cmd **cmd_list; TAILQ_HEAD(, mpi3mr_cmd) cmd_list_head; struct mtx cmd_pool_lock; struct resource *mpi3mr_regs_resource; bus_space_handle_t mpi3mr_bhandle; bus_space_tag_t mpi3mr_btag; int mpi3mr_regs_rid; bus_dma_tag_t mpi3mr_parent_dmat; bus_dma_tag_t buffer_dmat; int num_reqs; int num_replies; int num_chains; TAILQ_HEAD(, mpi3mr_event_handle) event_list; struct mpi3mr_event_handle *mpi3mr_log_eh; struct intr_config_hook mpi3mr_ich; struct mtx mpi3mr_mtx; struct mtx io_lock; U8 intr_enabled; TAILQ_HEAD(, delayed_dev_rmhs_node) delayed_rmhs_list; TAILQ_HEAD(, delayed_evtack_node) delayed_evtack_cmds_list; U16 num_admin_reqs; U32 admin_req_q_sz; U16 admin_req_pi; U16 admin_req_ci; bus_dma_tag_t admin_req_tag; bus_dmamap_t admin_req_dmamap; bus_addr_t admin_req_phys; U8 *admin_req; struct mtx admin_req_lock; U16 num_admin_replies; U32 admin_reply_q_sz; U16 admin_reply_ci; U8 admin_reply_ephase; bus_dma_tag_t admin_reply_tag; bus_dmamap_t admin_reply_dmamap; bus_addr_t admin_reply_phys; U8 *admin_reply; struct mtx admin_reply_lock; bool admin_in_use; U32 num_reply_bufs; bus_dma_tag_t reply_buf_tag; bus_dmamap_t reply_buf_dmamap; bus_addr_t reply_buf_phys; U8 *reply_buf; bus_addr_t reply_buf_dma_max_address; bus_addr_t reply_buf_dma_min_address; U16 reply_free_q_sz; bus_dma_tag_t reply_free_q_tag; bus_dmamap_t reply_free_q_dmamap; bus_addr_t reply_free_q_phys; U64 *reply_free_q; struct mtx reply_free_q_lock; U32 reply_free_q_host_index; U32 num_sense_bufs; bus_dma_tag_t sense_buf_tag; bus_dmamap_t sense_buf_dmamap; bus_addr_t sense_buf_phys; U8 *sense_buf; U16 sense_buf_q_sz; bus_dma_tag_t sense_buf_q_tag; bus_dmamap_t sense_buf_q_dmamap; bus_addr_t sense_buf_q_phys; U64 *sense_buf_q; struct mtx sense_buf_q_lock; U32 sense_buf_q_host_index; void *nvme_encap_prp_list; bus_addr_t nvme_encap_prp_list_dma; bus_dma_tag_t nvme_encap_prp_list_dmatag; bus_dmamap_t nvme_encap_prp_list_dma_dmamap; U32 nvme_encap_prp_sz; U32 ready_timeout; struct mpi3mr_irq_context *irq_ctx; U16 num_queues; /* Number of request/reply queues */ struct mpi3mr_op_req_queue *op_req_q; struct mpi3mr_op_reply_queue *op_reply_q; U16 num_hosttag_op_req_q; struct mpi3mr_drvr_cmd init_cmds; struct mpi3mr_ioc_facts facts; U16 reply_sz; U16 op_reply_sz; U32 event_masks[MPI3_EVENT_NOTIFY_EVENTMASK_WORDS]; char fwevt_worker_name[MPI3MR_NAME_LENGTH]; struct workqueue_struct *fwevt_worker_thread; struct mtx fwevt_lock; struct mtx target_lock; U16 max_host_ios; bus_dma_tag_t chain_sgl_list_tag; struct mpi3mr_chain *chain_sgl_list; U16 chain_bitmap_sz; void *chain_bitmap; struct mtx chain_buf_lock; U16 chain_buf_count; struct mpi3mr_drvr_cmd ioctl_cmds; struct mpi3mr_drvr_cmd host_tm_cmds; struct mpi3mr_drvr_cmd dev_rmhs_cmds[MPI3MR_NUM_DEVRMCMD]; struct mpi3mr_drvr_cmd evtack_cmds[MPI3MR_NUM_EVTACKCMD]; U16 devrem_bitmap_sz; void *devrem_bitmap; U16 dev_handle_bitmap_sz; void *removepend_bitmap; U16 evtack_cmds_bitmap_sz; void *evtack_cmds_bitmap; U32 ts_update_counter; U8 reset_in_progress; U8 unrecoverable; U8 block_ioctls; U8 in_prep_ciactv_rst; U16 prep_ciactv_rst_counter; struct mtx reset_mutex; U8 prepare_for_reset; U16 prepare_for_reset_timeout_counter; U16 diagsave_timeout; int logging_level; U16 flush_io_count; Mpi3DriverInfoLayout_t driver_info; U16 change_count; U8 *log_data_buffer; U16 log_data_buffer_index; U16 log_data_entry_size; U8 pel_wait_pend; U8 pel_abort_requested; U8 pel_class; U16 pel_locale; struct mpi3mr_drvr_cmd pel_cmds; struct mpi3mr_drvr_cmd pel_abort_cmd; U32 newest_seqnum; void *pel_seq_number; bus_addr_t pel_seq_number_dma; bus_dma_tag_t pel_seq_num_dmatag; bus_dmamap_t pel_seq_num_dmamap; U32 pel_seq_number_sz; struct selinfo mpi3mr_select; U32 mpi3mr_poll_waiting; U32 mpi3mr_aen_triggered; U16 wait_for_port_enable; U16 track_mapping_events; U16 pending_map_events; mpi3mr_atomic_t fw_outstanding; mpi3mr_atomic_t pend_ioctls; struct proc *watchdog_thread; void *watchdog_chan; void *tm_chan; u_int8_t remove_in_progress; u_int8_t watchdog_thread_active; u_int8_t do_timedout_reset; bool allow_ios; bool secure_ctrl; mpi3mr_atomic_t pend_large_data_sz; u_int32_t io_throttle_data_length; u_int32_t io_throttle_high; u_int32_t io_throttle_low; u_int16_t num_io_throttle_group; u_int iot_enable; struct mpi3mr_throttle_group_info *throttle_groups; struct dma_memory_desc ioctl_sge[MPI3MR_NUM_IOCTL_SGE]; struct dma_memory_desc ioctl_chain_sge; struct dma_memory_desc ioctl_resp_sge; bool ioctl_sges_allocated; }; static __inline uint64_t mpi3mr_regread64(struct mpi3mr_softc *sc, uint32_t offset) { return bus_space_read_8(sc->mpi3mr_btag, sc->mpi3mr_bhandle, offset); } static __inline void mpi3mr_regwrite64(struct mpi3mr_softc *sc, uint32_t offset, uint64_t val) { bus_space_write_8(sc->mpi3mr_btag, sc->mpi3mr_bhandle, offset, val); } static __inline uint32_t mpi3mr_regread(struct mpi3mr_softc *sc, uint32_t offset) { return bus_space_read_4(sc->mpi3mr_btag, sc->mpi3mr_bhandle, offset); } static __inline void mpi3mr_regwrite(struct mpi3mr_softc *sc, uint32_t offset, uint32_t val) { bus_space_write_4(sc->mpi3mr_btag, sc->mpi3mr_bhandle, offset, val); } #define MPI3MR_INFO (1 << 0) /* Basic info */ #define MPI3MR_FAULT (1 << 1) /* Hardware faults */ #define MPI3MR_EVENT (1 << 2) /* Event data from the controller */ #define MPI3MR_LOG (1 << 3) /* Log data from the controller */ #define MPI3MR_RECOVERY (1 << 4) /* Command error recovery tracing */ #define MPI3MR_ERROR (1 << 5) /* Fatal driver/OS APIs failure */ #define MPI3MR_XINFO (1 << 6) /* Additional info logs*/ #define MPI3MR_TRACE (1 << 7) /* Trace functions */ #define MPI3MR_IOT (1 << 8) /* IO throttling related debugs */ #define MPI3MR_DEBUG_TM (1 << 9) /* Task management related debugs */ #define MPI3MR_DEBUG_IOCTL (1 << 10) /* IOCTL related debugs */ #define mpi3mr_printf(sc, args...) \ device_printf((sc)->mpi3mr_dev, ##args) #define mpi3mr_print_field(sc, msg, args...) \ printf("\t" msg, ##args) #define mpi3mr_vprintf(sc, args...) \ do { \ if (bootverbose) \ mpi3mr_printf(sc, ##args); \ } while (0) #define mpi3mr_dprint(sc, level, msg, args...) \ do { \ if ((sc)->mpi3mr_debug & (level)) \ device_printf((sc)->mpi3mr_dev, msg, ##args); \ } while (0) #define MPI3MR_PRINTFIELD_START(sc, tag...) \ mpi3mr_printf((sc), ##tag); \ mpi3mr_print_field((sc), ":\n") #define MPI3MR_PRINTFIELD_END(sc, tag) \ mpi3mr_printf((sc), tag "\n") #define MPI3MR_PRINTFIELD(sc, facts, attr, fmt) \ mpi3mr_print_field((sc), #attr ": " #fmt "\n", (facts)->attr) #define mpi3mr_kproc_create(func, farg, proc_ptr, flags, stackpgs, fmtstr, arg) \ kproc_create(func, farg, proc_ptr, flags, stackpgs, fmtstr, arg) #define mpi3mr_kproc_exit(arg) kproc_exit(arg) #if defined(CAM_PRIORITY_XPT) #define MPI3MR_PRIORITY_XPT CAM_PRIORITY_XPT #else #define MPI3MR_PRIORITY_XPT 5 #endif static __inline void mpi3mr_clear_bit(int b, volatile void *p) { atomic_clear_int(((volatile int *)p) + (b >> 5), 1 << (b & 0x1f)); } static __inline void mpi3mr_set_bit(int b, volatile void *p) { atomic_set_int(((volatile int *)p) + (b >> 5), 1 << (b & 0x1f)); } static __inline int mpi3mr_test_bit(int b, volatile void *p) { return ((volatile int *)p)[b >> 5] & (1 << (b & 0x1f)); } static __inline int mpi3mr_test_and_set_bit(int b, volatile void *p) { int ret = ((volatile int *)p)[b >> 5] & (1 << (b & 0x1f)); atomic_set_int(((volatile int *)p) + (b >> 5), 1 << (b & 0x1f)); return ret; } static __inline int mpi3mr_find_first_zero_bit(void *p, int bit_count) { int i, sz, j=0; U8 *loc; sz = bit_count % 8 ? (bit_count / 8 + 1) : (bit_count / 8); loc = malloc(sz, M_MPI3MR, M_NOWAIT | M_ZERO); memcpy(loc, p, sz); for (i = 0; i < sz; i++) { j = 0; while (j < 8) { if (!((loc[i] >> j) & 0x1)) goto out; j++; } } out: free(loc, M_MPI3MR); return (i + j); } #define MPI3MR_DIV_ROUND_UP(n,d) (((n) + (d) - 1) / (d)) void init_completion(struct completion *completion); void complete(struct completion *completion); void wait_for_completion_timeout(struct completion *completion, U32 timeout); void wait_for_completion_timeout_tm(struct completion *completion, U32 timeout, struct mpi3mr_softc *sc); void mpi3mr_add_sg_single(void *paddr, U8 flags, U32 length, bus_addr_t dma_addr); void mpi3mr_enable_interrupts(struct mpi3mr_softc *sc); void mpi3mr_disable_interrupts(struct mpi3mr_softc *sc); void mpi3mr_memaddr_cb(void *arg, bus_dma_segment_t *segs, int nsegs, int error); int mpi3mr_submit_admin_cmd(struct mpi3mr_softc *mrioc, void *admin_req, U16 admin_req_sz); int mpi3mr_submit_io(struct mpi3mr_softc *mrioc, struct mpi3mr_op_req_queue *op_req_q, U8 *req); int mpi3mr_alloc_interrupts(struct mpi3mr_softc *sc, U16 setup_one); void mpi3mr_cleanup_ioc(struct mpi3mr_softc *sc); int mpi3mr_initialize_ioc(struct mpi3mr_softc *sc, U8 reason); void mpi3mr_build_zero_len_sge(void *paddr); int mpi3mr_issue_event_notification(struct mpi3mr_softc *sc); int mpi3mr_register_events(struct mpi3mr_softc *sc); void mpi3mr_process_op_reply_desc(struct mpi3mr_softc *sc, Mpi3DefaultReplyDescriptor_t *reply_desc, U64 *reply_dma); struct mpi3mr_cmd * mpi3mr_get_command(struct mpi3mr_softc *sc); void mpi3mr_release_command(struct mpi3mr_cmd *cmd); int mpi3mr_complete_io_cmd(struct mpi3mr_softc *sc, struct mpi3mr_irq_context *irq_context); int mpi3mr_cam_detach(struct mpi3mr_softc *sc); int mpi3mr_cam_attach(struct mpi3mr_softc *sc); struct mpi3mr_target * mpi3mr_find_target_by_per_id(struct mpi3mr_cam_softc *cam_sc, uint16_t per_id); struct mpi3mr_target * mpi3mr_find_target_by_dev_handle(struct mpi3mr_cam_softc *cam_sc, uint16_t dev_handle); int mpi3mr_create_device(struct mpi3mr_softc *sc, Mpi3DevicePage0_t *dev_pg0); void mpi3mr_unmap_request(struct mpi3mr_softc *sc, struct mpi3mr_cmd *cmd); void init_completion(struct completion *completion); void complete(struct completion *completion); void wait_for_completion_timeout(struct completion *completion, U32 timeout); void poll_for_command_completion(struct mpi3mr_softc *sc, struct mpi3mr_drvr_cmd *cmd, U16 wait); int mpi3mr_alloc_requests(struct mpi3mr_softc *sc); void mpi3mr_watchdog(void *arg); int mpi3mr_issue_port_enable(struct mpi3mr_softc *mrioc, U8 async); void mpi3mr_isr(void *privdata); int mpi3mr_alloc_msix_queues(struct mpi3mr_softc *sc); void mpi3mr_destory_mtx(struct mpi3mr_softc *sc); void mpi3mr_free_mem(struct mpi3mr_softc *sc); void mpi3mr_cleanup_interrupts(struct mpi3mr_softc *sc); int mpi3mr_setup_irqs(struct mpi3mr_softc *sc); void mpi3mr_cleanup_event_taskq(struct mpi3mr_softc *sc); void mpi3mr_hexdump(void *buf, int sz, int format); int mpi3mr_soft_reset_handler(struct mpi3mr_softc *sc, U32 reset_reason, bool snapdump); void mpi3mrsas_release_simq_reinit(struct mpi3mr_cam_softc *cam_sc); void mpi3mr_watchdog_thread(void *arg); void mpi3mr_add_device(struct mpi3mr_softc *sc, U16 per_id); int mpi3mr_remove_device(struct mpi3mr_softc *sc, U16 handle); int mpi3mrsas_register_events(struct mpi3mr_softc *sc); int mpi3mr_process_event_ack(struct mpi3mr_softc *sc, U8 event, U32 event_ctx); int mpi3mr_remove_device_from_os(struct mpi3mr_softc *sc, U16 handle); void mpi3mr_remove_device_from_list(struct mpi3mr_softc *sc, struct mpi3mr_target *target, bool must_delete); void mpi3mr_update_device(struct mpi3mr_softc *mrioc, struct mpi3mr_target *tgtdev, Mpi3DevicePage0_t *dev_pg0, bool is_added); void mpi3mr_app_save_logdata(struct mpi3mr_softc *sc, char *event_data, U16 event_data_size); void mpi3mr_set_io_divert_for_all_vd_in_tg(struct mpi3mr_softc *sc, struct mpi3mr_throttle_group_info *tg, U8 divert_value); enum mpi3mr_iocstate mpi3mr_get_iocstate(struct mpi3mr_softc *sc); void mpi3mr_poll_pend_io_completions(struct mpi3mr_softc *sc); void int_to_lun(unsigned int lun, U8 *req_lun); void trigger_reset_from_watchdog(struct mpi3mr_softc *sc, U8 reset_type, U32 reset_reason); void mpi3mr_alloc_ioctl_dma_memory(struct mpi3mr_softc *sc); #endif /*MPI3MR_H_INCLUDED*/ diff --git a/sys/dev/mpi3mr/mpi3mr_cam.c b/sys/dev/mpi3mr/mpi3mr_cam.c index b6e47eac058a..15ef2732ec56 100644 --- a/sys/dev/mpi3mr/mpi3mr_cam.c +++ b/sys/dev/mpi3mr/mpi3mr_cam.c @@ -1,2218 +1,2234 @@ /* * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2020-2023, Broadcom Inc. All rights reserved. * Support: * * Authors: Sumit Saxena * Chandrakanth Patil * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are * met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation and/or other * materials provided with the distribution. * 3. Neither the name of the Broadcom Inc. nor the names of its contributors * may be used to endorse or promote products derived from this software without * specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. * * The views and conclusions contained in the software and documentation are * those of the authors and should not be interpreted as representing * official policies,either expressed or implied, of the FreeBSD Project. * * Mail to: Broadcom Inc 1320 Ridder Park Dr, San Jose, CA 95131 * * Broadcom Inc. (Broadcom) MPI3MR Adapter FreeBSD */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "mpi/mpi30_api.h" #include "mpi3mr_cam.h" #include "mpi3mr.h" #include /* XXX for pcpu.h */ #include /* XXX for PCPU_GET */ #define smp_processor_id() PCPU_GET(cpuid) -static int +static void +mpi3mr_enqueue_request(struct mpi3mr_softc *sc, struct mpi3mr_cmd *cm); +static void mpi3mr_map_request(struct mpi3mr_softc *sc, struct mpi3mr_cmd *cm); void mpi3mr_release_simq_reinit(struct mpi3mr_cam_softc *cam_sc); static void mpi3mr_freeup_events(struct mpi3mr_softc *sc); extern int mpi3mr_register_events(struct mpi3mr_softc *sc); extern void mpi3mr_add_sg_single(void *paddr, U8 flags, U32 length, bus_addr_t dma_addr); extern void mpi3mr_build_zero_len_sge(void *paddr); static U32 event_count; static void mpi3mr_prepare_sgls(void *arg, bus_dma_segment_t *segs, int nsegs, int error) { struct mpi3mr_softc *sc; struct mpi3mr_cmd *cm; u_int i; bus_addr_t chain_dma; void *chain; U8 *sg_local; U32 chain_length; int sges_left; U32 sges_in_segment; U8 simple_sgl_flags; U8 simple_sgl_flags_last; U8 last_chain_sgl_flags; struct mpi3mr_chain *chain_req; Mpi3SCSIIORequest_t *scsiio_req; + union ccb *ccb; cm = (struct mpi3mr_cmd *)arg; sc = cm->sc; scsiio_req = (Mpi3SCSIIORequest_t *) &cm->io_request; + ccb = cm->ccb; if (error) { - cm->error_code = error; device_printf(sc->mpi3mr_dev, "%s: error=%d\n",__func__, error); if (error == EFBIG) { - cm->ccb->ccb_h.status = CAM_REQ_TOO_BIG; - return; + mpi3mr_set_ccbstatus(ccb, CAM_REQ_TOO_BIG); + } else { + mpi3mr_set_ccbstatus(ccb, CAM_REQ_CMP_ERR); } + mpi3mr_release_command(cm); + xpt_done(ccb); + return; } if (cm->data_dir == MPI3MR_READ) bus_dmamap_sync(sc->buffer_dmat, cm->dmamap, BUS_DMASYNC_PREREAD); if (cm->data_dir == MPI3MR_WRITE) bus_dmamap_sync(sc->buffer_dmat, cm->dmamap, BUS_DMASYNC_PREWRITE); - if (nsegs > MPI3MR_SG_DEPTH) { - device_printf(sc->mpi3mr_dev, "SGE count is too large or 0.\n"); - return; - } + + KASSERT(nsegs <= MPI3MR_SG_DEPTH && nsegs > 0, + ("%s: bad SGE count: %d\n", device_get_nameunit(sc->mpi3mr_dev), nsegs)); simple_sgl_flags = MPI3_SGE_FLAGS_ELEMENT_TYPE_SIMPLE | MPI3_SGE_FLAGS_DLAS_SYSTEM; simple_sgl_flags_last = simple_sgl_flags | MPI3_SGE_FLAGS_END_OF_LIST; last_chain_sgl_flags = MPI3_SGE_FLAGS_ELEMENT_TYPE_LAST_CHAIN | MPI3_SGE_FLAGS_DLAS_SYSTEM; sg_local = (U8 *)&scsiio_req->SGL; - if (!scsiio_req->DataLength) { + if (scsiio_req->DataLength == 0) { + /* XXX we don't ever get here when DataLength == 0, right? cm->data is NULL */ + /* This whole if can likely be removed -- we handle it in mpi3mr_request_map */ mpi3mr_build_zero_len_sge(sg_local); - return; + goto enqueue; } sges_left = nsegs; - if (sges_left < 0) { - printf("scsi_dma_map failed: request for %d bytes!\n", - scsiio_req->DataLength); - return; - } - if (sges_left > MPI3MR_SG_DEPTH) { - printf("scsi_dma_map returned unsupported sge count %d!\n", - sges_left); - return; - } - sges_in_segment = (sc->facts.op_req_sz - offsetof(Mpi3SCSIIORequest_t, SGL))/sizeof(Mpi3SGESimple_t); i = 0; mpi3mr_dprint(sc, MPI3MR_TRACE, "SGE count: %d IO size: %d\n", nsegs, scsiio_req->DataLength); if (sges_left <= sges_in_segment) goto fill_in_last_segment; /* fill in main message segment when there is a chain following */ while (sges_in_segment > 1) { mpi3mr_add_sg_single(sg_local, simple_sgl_flags, segs[i].ds_len, segs[i].ds_addr); sg_local += sizeof(Mpi3SGESimple_t); sges_left--; sges_in_segment--; i++; } chain_req = &sc->chain_sgl_list[cm->hosttag]; chain = chain_req->buf; chain_dma = chain_req->buf_phys; memset(chain_req->buf, 0, PAGE_SIZE); sges_in_segment = sges_left; chain_length = sges_in_segment * sizeof(Mpi3SGESimple_t); mpi3mr_add_sg_single(sg_local, last_chain_sgl_flags, chain_length, chain_dma); sg_local = chain; fill_in_last_segment: while (sges_left > 0) { if (sges_left == 1) mpi3mr_add_sg_single(sg_local, simple_sgl_flags_last, segs[i].ds_len, segs[i].ds_addr); else mpi3mr_add_sg_single(sg_local, simple_sgl_flags, segs[i].ds_len, segs[i].ds_addr); sg_local += sizeof(Mpi3SGESimple_t); sges_left--; i++; } +enqueue: + /* + * Now that we've created the sgls, we send the request to the device. + * Unlike in Linux, dmaload isn't guaranteed to load every time, but + * this function is always called when the resources are available, so + * we can send the request to hardware here always. mpi3mr_map_request + * knows about this quirk and will only take evasive action when an + * error other than EINPROGRESS is returned from dmaload. + */ + mpi3mr_enqueue_request(sc, cm); + return; } -int +static void mpi3mr_map_request(struct mpi3mr_softc *sc, struct mpi3mr_cmd *cm) { u_int32_t retcode = 0; + union ccb *ccb; + ccb = cm->ccb; if (cm->data != NULL) { mtx_lock(&sc->io_lock); /* Map data buffer into bus space */ retcode = bus_dmamap_load_ccb(sc->buffer_dmat, cm->dmamap, - cm->ccb, mpi3mr_prepare_sgls, cm, 0); + ccb, mpi3mr_prepare_sgls, cm, 0); mtx_unlock(&sc->io_lock); - if (retcode) - device_printf(sc->mpi3mr_dev, "bus_dmamap_load(): retcode = %d\n", retcode); - if (retcode == EINPROGRESS) { - device_printf(sc->mpi3mr_dev, "request load in progress\n"); - xpt_freeze_simq(sc->cam_sc->sim, 1); + if (retcode != 0 && retcode != EINPROGRESS) { + device_printf(sc->mpi3mr_dev, + "bus_dmamap_load(): retcode = %d\n", retcode); + /* + * Any other error means prepare_sgls wasn't called, and + * will never be called, so we have to mop up. This error + * should never happen, though. + */ + mpi3mr_set_ccbstatus(ccb, CAM_REQ_CMP_ERR); + mpi3mr_release_command(cm); + xpt_done(ccb); } + } else { + /* + * No data, we enqueue it directly here. + */ + mpi3mr_enqueue_request(sc, cm); } - if (cm->error_code) - return cm->error_code; - if (retcode) - mpi3mr_set_ccbstatus(cm->ccb, CAM_REQ_INVALID); - - return (retcode); } void mpi3mr_unmap_request(struct mpi3mr_softc *sc, struct mpi3mr_cmd *cmd) { if (cmd->data != NULL) { if (cmd->data_dir == MPI3MR_READ) bus_dmamap_sync(sc->buffer_dmat, cmd->dmamap, BUS_DMASYNC_POSTREAD); if (cmd->data_dir == MPI3MR_WRITE) bus_dmamap_sync(sc->buffer_dmat, cmd->dmamap, BUS_DMASYNC_POSTWRITE); mtx_lock(&sc->io_lock); bus_dmamap_unload(sc->buffer_dmat, cmd->dmamap); mtx_unlock(&sc->io_lock); } } /** * mpi3mr_allow_unmap_to_fw - Whether an unmap is allowed to fw * @sc: Adapter instance reference * @ccb: SCSI Command reference * * The controller hardware cannot handle certain unmap commands * for NVMe drives, this routine checks those and return true * and completes the SCSI command with proper status and sense * data. * * Return: TRUE for allowed unmap, FALSE otherwise. */ static bool mpi3mr_allow_unmap_to_fw(struct mpi3mr_softc *sc, union ccb *ccb) { struct ccb_scsiio *csio; uint16_t param_list_len, block_desc_len, trunc_param_len = 0; csio = &ccb->csio; param_list_len = (uint16_t) ((scsiio_cdb_ptr(csio)[7] << 8) | scsiio_cdb_ptr(csio)[8]); switch(pci_get_revid(sc->mpi3mr_dev)) { case SAS4116_CHIP_REV_A0: if (!param_list_len) { mpi3mr_dprint(sc, MPI3MR_ERROR, "%s: CDB received with zero parameter length\n", __func__); mpi3mr_print_cdb(ccb); mpi3mr_set_ccbstatus(ccb, CAM_REQ_CMP); xpt_done(ccb); return false; } if (param_list_len < 24) { mpi3mr_dprint(sc, MPI3MR_ERROR, "%s: CDB received with invalid param_list_len: %d\n", __func__, param_list_len); mpi3mr_print_cdb(ccb); scsi_set_sense_data(&ccb->csio.sense_data, /*sense_format*/ SSD_TYPE_FIXED, /*current_error*/ 1, /*sense_key*/ SSD_KEY_ILLEGAL_REQUEST, /*asc*/ 0x1A, /*ascq*/ 0x00, /*extra args*/ SSD_ELEM_NONE); ccb->csio.scsi_status = SCSI_STATUS_CHECK_COND; ccb->ccb_h.status = CAM_SCSI_STATUS_ERROR | CAM_AUTOSNS_VALID; return false; } if (param_list_len != csio->dxfer_len) { mpi3mr_dprint(sc, MPI3MR_ERROR, "%s: CDB received with param_list_len: %d bufflen: %d\n", __func__, param_list_len, csio->dxfer_len); mpi3mr_print_cdb(ccb); scsi_set_sense_data(&ccb->csio.sense_data, /*sense_format*/ SSD_TYPE_FIXED, /*current_error*/ 1, /*sense_key*/ SSD_KEY_ILLEGAL_REQUEST, /*asc*/ 0x1A, /*ascq*/ 0x00, /*extra args*/ SSD_ELEM_NONE); ccb->csio.scsi_status = SCSI_STATUS_CHECK_COND; ccb->ccb_h.status = CAM_SCSI_STATUS_ERROR | CAM_AUTOSNS_VALID; xpt_done(ccb); return false; } block_desc_len = (uint16_t) (csio->data_ptr[2] << 8 | csio->data_ptr[3]); if (block_desc_len < 16) { mpi3mr_dprint(sc, MPI3MR_ERROR, "%s: Invalid descriptor length in param list: %d\n", __func__, block_desc_len); mpi3mr_print_cdb(ccb); scsi_set_sense_data(&ccb->csio.sense_data, /*sense_format*/ SSD_TYPE_FIXED, /*current_error*/ 1, /*sense_key*/ SSD_KEY_ILLEGAL_REQUEST, /*asc*/ 0x26, /*ascq*/ 0x00, /*extra args*/ SSD_ELEM_NONE); ccb->csio.scsi_status = SCSI_STATUS_CHECK_COND; ccb->ccb_h.status = CAM_SCSI_STATUS_ERROR | CAM_AUTOSNS_VALID; xpt_done(ccb); return false; } if (param_list_len > (block_desc_len + 8)) { mpi3mr_print_cdb(ccb); mpi3mr_dprint(sc, MPI3MR_INFO, "%s: Truncating param_list_len(%d) to block_desc_len+8(%d)\n", __func__, param_list_len, (block_desc_len + 8)); param_list_len = block_desc_len + 8; scsiio_cdb_ptr(csio)[7] = (param_list_len >> 8) | 0xff; scsiio_cdb_ptr(csio)[8] = param_list_len | 0xff; mpi3mr_print_cdb(ccb); } break; case SAS4116_CHIP_REV_B0: if ((param_list_len > 24) && ((param_list_len - 8) & 0xF)) { trunc_param_len -= (param_list_len - 8) & 0xF; mpi3mr_print_cdb(ccb); mpi3mr_dprint(sc, MPI3MR_INFO, "%s: Truncating param_list_len from (%d) to (%d)\n", __func__, param_list_len, trunc_param_len); scsiio_cdb_ptr(csio)[7] = (param_list_len >> 8) | 0xff; scsiio_cdb_ptr(csio)[8] = param_list_len | 0xff; mpi3mr_print_cdb(ccb); } break; } return true; } /** * mpi3mr_tm_response_name - get TM response as a string * @resp_code: TM response code * * Convert known task management response code as a readable * string. * * Return: response code string. */ static const char* mpi3mr_tm_response_name(U8 resp_code) { char *desc; switch (resp_code) { case MPI3_SCSITASKMGMT_RSPCODE_TM_COMPLETE: desc = "task management request completed"; break; case MPI3_SCSITASKMGMT_RSPCODE_INVALID_FRAME: desc = "invalid frame"; break; case MPI3_SCSITASKMGMT_RSPCODE_TM_FUNCTION_NOT_SUPPORTED: desc = "task management request not supported"; break; case MPI3_SCSITASKMGMT_RSPCODE_TM_FAILED: desc = "task management request failed"; break; case MPI3_SCSITASKMGMT_RSPCODE_TM_SUCCEEDED: desc = "task management request succeeded"; break; case MPI3_SCSITASKMGMT_RSPCODE_TM_INVALID_LUN: desc = "invalid LUN"; break; case MPI3_SCSITASKMGMT_RSPCODE_TM_OVERLAPPED_TAG: desc = "overlapped tag attempted"; break; case MPI3_SCSITASKMGMT_RSPCODE_IO_QUEUED_ON_IOC: desc = "task queued, however not sent to target"; break; case MPI3_SCSITASKMGMT_RSPCODE_TM_NVME_DENIED: desc = "task management request denied by NVMe device"; break; default: desc = "unknown"; break; } return desc; } void mpi3mr_poll_pend_io_completions(struct mpi3mr_softc *sc) { int i; int num_of_reply_queues = sc->num_queues; struct mpi3mr_irq_context *irq_ctx; for (i = 0; i < num_of_reply_queues; i++) { irq_ctx = &sc->irq_ctx[i]; mpi3mr_complete_io_cmd(sc, irq_ctx); } } void trigger_reset_from_watchdog(struct mpi3mr_softc *sc, U8 reset_type, U32 reset_reason) { if (sc->reset_in_progress) { mpi3mr_dprint(sc, MPI3MR_INFO, "Another reset is in progress, no need to trigger the reset\n"); return; } sc->reset.type = reset_type; sc->reset.reason = reset_reason; return; } /** * mpi3mr_issue_tm - Issue Task Management request * @sc: Adapter instance reference * @tm_type: Task Management type * @handle: Device handle * @lun: lun ID * @htag: Host tag of the TM request * @timeout: TM timeout value * @drv_cmd: Internal command tracker * @resp_code: Response code place holder * @cmd: Timed out command reference * * Issues a Task Management Request to the controller for a * specified target, lun and command and wait for its completion * and check TM response. Recover the TM if it timed out by * issuing controller reset. * * Return: 0 on success, non-zero on errors */ static int mpi3mr_issue_tm(struct mpi3mr_softc *sc, struct mpi3mr_cmd *cmd, U8 tm_type, unsigned long timeout) { int retval = 0; MPI3_SCSI_TASK_MGMT_REQUEST tm_req; MPI3_SCSI_TASK_MGMT_REPLY *tm_reply = NULL; struct mpi3mr_drvr_cmd *drv_cmd = NULL; struct mpi3mr_target *tgtdev = NULL; struct mpi3mr_op_req_queue *op_req_q = NULL; union ccb *ccb; U8 resp_code; if (sc->unrecoverable) { mpi3mr_dprint(sc, MPI3MR_INFO, "Controller is in unrecoverable state!! TM not required\n"); return retval; } if (sc->reset_in_progress) { mpi3mr_dprint(sc, MPI3MR_INFO, "controller reset in progress!! TM not required\n"); return retval; } if (!cmd->ccb) { mpi3mr_dprint(sc, MPI3MR_ERROR, "SCSIIO command timed-out with NULL ccb\n"); return retval; } ccb = cmd->ccb; tgtdev = cmd->targ; if (tgtdev == NULL) { mpi3mr_dprint(sc, MPI3MR_ERROR, "Device does not exist target ID:0x%x," "TM is not required\n", ccb->ccb_h.target_id); return retval; } if (tgtdev->dev_removed == 1) { mpi3mr_dprint(sc, MPI3MR_ERROR, "Device(0x%x) is removed, TM is not required\n", ccb->ccb_h.target_id); return retval; } drv_cmd = &sc->host_tm_cmds; mtx_lock(&drv_cmd->lock); memset(&tm_req, 0, sizeof(tm_req)); tm_req.DevHandle = htole16(tgtdev->dev_handle); tm_req.TaskType = tm_type; tm_req.HostTag = htole16(MPI3MR_HOSTTAG_TMS); int_to_lun(ccb->ccb_h.target_lun, tm_req.LUN); tm_req.Function = MPI3_FUNCTION_SCSI_TASK_MGMT; drv_cmd->state = MPI3MR_CMD_PENDING; drv_cmd->is_waiting = 1; drv_cmd->callback = NULL; if (ccb) { if (tm_type == MPI3_SCSITASKMGMT_TASKTYPE_ABORT_TASK) { op_req_q = &sc->op_req_q[cmd->req_qidx]; tm_req.TaskHostTag = htole16(cmd->hosttag); tm_req.TaskRequestQueueID = htole16(op_req_q->qid); } } if (tgtdev) mpi3mr_atomic_inc(&tgtdev->block_io); if (tgtdev && (tgtdev->dev_type == MPI3_DEVICE_DEVFORM_PCIE)) { if ((tm_type == MPI3_SCSITASKMGMT_TASKTYPE_ABORT_TASK) && tgtdev->dev_spec.pcie_inf.abort_to) timeout = tgtdev->dev_spec.pcie_inf.abort_to; else if ((tm_type == MPI3_SCSITASKMGMT_TASKTYPE_TARGET_RESET) && tgtdev->dev_spec.pcie_inf.reset_to) timeout = tgtdev->dev_spec.pcie_inf.reset_to; } sc->tm_chan = (void *)&drv_cmd; mpi3mr_dprint(sc, MPI3MR_DEBUG_TM, "posting task management request: type(%d), handle(0x%04x)\n", tm_type, tgtdev->dev_handle); init_completion(&drv_cmd->completion); retval = mpi3mr_submit_admin_cmd(sc, &tm_req, sizeof(tm_req)); if (retval) { mpi3mr_dprint(sc, MPI3MR_ERROR, "posting task management request is failed\n"); retval = -1; goto out_unlock; } wait_for_completion_timeout_tm(&drv_cmd->completion, timeout, sc); if (!(drv_cmd->state & MPI3MR_CMD_COMPLETE)) { drv_cmd->is_waiting = 0; retval = -1; if (!(drv_cmd->state & MPI3MR_CMD_RESET)) { mpi3mr_dprint(sc, MPI3MR_ERROR, "task management request timed out after %ld seconds\n", timeout); if (sc->mpi3mr_debug & MPI3MR_DEBUG_TM) { mpi3mr_dprint(sc, MPI3MR_INFO, "tm_request dump\n"); mpi3mr_hexdump(&tm_req, sizeof(tm_req), 8); } trigger_reset_from_watchdog(sc, MPI3MR_TRIGGER_SOFT_RESET, MPI3MR_RESET_FROM_TM_TIMEOUT); retval = ETIMEDOUT; } goto out_unlock; } if (!(drv_cmd->state & MPI3MR_CMD_REPLYVALID)) { mpi3mr_dprint(sc, MPI3MR_ERROR, "invalid task management reply message\n"); retval = -1; goto out_unlock; } tm_reply = (MPI3_SCSI_TASK_MGMT_REPLY *)drv_cmd->reply; switch (drv_cmd->ioc_status) { case MPI3_IOCSTATUS_SUCCESS: resp_code = tm_reply->ResponseData & MPI3MR_RI_MASK_RESPCODE; break; case MPI3_IOCSTATUS_SCSI_IOC_TERMINATED: resp_code = MPI3_SCSITASKMGMT_RSPCODE_TM_COMPLETE; break; default: mpi3mr_dprint(sc, MPI3MR_ERROR, "task management request to handle(0x%04x) is failed with ioc_status(0x%04x) log_info(0x%08x)\n", tgtdev->dev_handle, drv_cmd->ioc_status, drv_cmd->ioc_loginfo); retval = -1; goto out_unlock; } switch (resp_code) { case MPI3_SCSITASKMGMT_RSPCODE_TM_SUCCEEDED: case MPI3_SCSITASKMGMT_RSPCODE_TM_COMPLETE: break; case MPI3_SCSITASKMGMT_RSPCODE_IO_QUEUED_ON_IOC: if (tm_type != MPI3_SCSITASKMGMT_TASKTYPE_QUERY_TASK) retval = -1; break; default: retval = -1; break; } mpi3mr_dprint(sc, MPI3MR_DEBUG_TM, "task management request type(%d) completed for handle(0x%04x) with ioc_status(0x%04x), log_info(0x%08x)" "termination_count(%u), response:%s(0x%x)\n", tm_type, tgtdev->dev_handle, drv_cmd->ioc_status, drv_cmd->ioc_loginfo, tm_reply->TerminationCount, mpi3mr_tm_response_name(resp_code), resp_code); if (retval) goto out_unlock; mpi3mr_disable_interrupts(sc); mpi3mr_poll_pend_io_completions(sc); mpi3mr_enable_interrupts(sc); mpi3mr_poll_pend_io_completions(sc); switch (tm_type) { case MPI3_SCSITASKMGMT_TASKTYPE_ABORT_TASK: if (cmd->state == MPI3MR_CMD_STATE_IN_TM) { mpi3mr_dprint(sc, MPI3MR_ERROR, "%s: task abort returned success from firmware but corresponding CCB (%p) was not terminated" "marking task abort failed!\n", sc->name, cmd->ccb); retval = -1; } break; case MPI3_SCSITASKMGMT_TASKTYPE_TARGET_RESET: if (mpi3mr_atomic_read(&tgtdev->outstanding)) { mpi3mr_dprint(sc, MPI3MR_ERROR, "%s: target reset returned success from firmware but IOs are still pending on the target (%p)" "marking target reset failed!\n", sc->name, tgtdev); retval = -1; } break; default: break; } out_unlock: drv_cmd->state = MPI3MR_CMD_NOTUSED; mtx_unlock(&drv_cmd->lock); if (tgtdev && mpi3mr_atomic_read(&tgtdev->block_io) > 0) mpi3mr_atomic_dec(&tgtdev->block_io); return retval; } /** * mpi3mr_task_abort- Abort error handling callback * @cmd: Timed out command reference * * Issue Abort Task Management if the command is in LLD scope * and verify if it is aborted successfully and return status * accordingly. * * Return: SUCCESS of successful abort the SCSI command else FAILED */ static int mpi3mr_task_abort(struct mpi3mr_cmd *cmd) { int retval = 0; struct mpi3mr_softc *sc; union ccb *ccb; sc = cmd->sc; if (!cmd->ccb) { mpi3mr_dprint(sc, MPI3MR_ERROR, "SCSIIO command timed-out with NULL ccb\n"); return retval; } ccb = cmd->ccb; mpi3mr_dprint(sc, MPI3MR_INFO, "attempting abort task for ccb(%p)\n", ccb); mpi3mr_print_cdb(ccb); if (cmd->state != MPI3MR_CMD_STATE_BUSY) { mpi3mr_dprint(sc, MPI3MR_INFO, "%s: ccb is not in driver scope, abort task is not required\n", sc->name); return retval; } cmd->state = MPI3MR_CMD_STATE_IN_TM; retval = mpi3mr_issue_tm(sc, cmd, MPI3_SCSITASKMGMT_TASKTYPE_ABORT_TASK, MPI3MR_ABORTTM_TIMEOUT); mpi3mr_dprint(sc, MPI3MR_INFO, "abort task is %s for ccb(%p)\n", ((retval == 0) ? "SUCCESS" : "FAILED"), ccb); return retval; } /** * mpi3mr_target_reset - Target reset error handling callback * @cmd: Timed out command reference * * Issue Target reset Task Management and verify the SCSI commands are * terminated successfully and return status accordingly. * * Return: SUCCESS of successful termination of the SCSI commands else * FAILED */ static int mpi3mr_target_reset(struct mpi3mr_cmd *cmd) { int retval = 0; struct mpi3mr_softc *sc; struct mpi3mr_target *target; sc = cmd->sc; target = cmd->targ; if (target == NULL) { mpi3mr_dprint(sc, MPI3MR_XINFO, "Device does not exist for target:0x%p," "target reset is not required\n", target); return retval; } mpi3mr_dprint(sc, MPI3MR_INFO, "attempting target reset on target(%d)\n", target->per_id); if (mpi3mr_atomic_read(&target->outstanding)) { mpi3mr_dprint(sc, MPI3MR_INFO, "no outstanding IOs on the target(%d)," " target reset not required.\n", target->per_id); return retval; } retval = mpi3mr_issue_tm(sc, cmd, MPI3_SCSITASKMGMT_TASKTYPE_TARGET_RESET, MPI3MR_RESETTM_TIMEOUT); mpi3mr_dprint(sc, MPI3MR_INFO, "target reset is %s for target(%d)\n", ((retval == 0) ? "SUCCESS" : "FAILED"), target->per_id); return retval; } /** * mpi3mr_get_fw_pending_ios - Calculate pending I/O count * @sc: Adapter instance reference * * Calculate the pending I/Os for the controller and return. * * Return: Number of pending I/Os */ static inline int mpi3mr_get_fw_pending_ios(struct mpi3mr_softc *sc) { U16 i, pend_ios = 0; for (i = 0; i < sc->num_queues; i++) pend_ios += mpi3mr_atomic_read(&sc->op_reply_q[i].pend_ios); return pend_ios; } /** * mpi3mr_wait_for_host_io - block for I/Os to complete * @sc: Adapter instance reference * @timeout: time out in seconds * * Waits for pending I/Os for the given adapter to complete or * to hit the timeout. * * Return: Nothing */ static int mpi3mr_wait_for_host_io(struct mpi3mr_softc *sc, U32 timeout) { enum mpi3mr_iocstate iocstate; iocstate = mpi3mr_get_iocstate(sc); if (iocstate != MRIOC_STATE_READY) { mpi3mr_dprint(sc, MPI3MR_XINFO, "%s :Controller is in NON-READY state! Proceed with Reset\n", __func__); return -1; } if (!mpi3mr_get_fw_pending_ios(sc)) return 0; mpi3mr_dprint(sc, MPI3MR_INFO, "%s :Waiting for %d seconds prior to reset for %d pending I/Os to complete\n", __func__, timeout, mpi3mr_get_fw_pending_ios(sc)); int i; for (i = 0; i < timeout; i++) { if (!mpi3mr_get_fw_pending_ios(sc)) { mpi3mr_dprint(sc, MPI3MR_INFO, "%s :All pending I/Os got completed while waiting! Reset not required\n", __func__); return 0; } iocstate = mpi3mr_get_iocstate(sc); if (iocstate != MRIOC_STATE_READY) { mpi3mr_dprint(sc, MPI3MR_XINFO, "%s :Controller state becomes NON-READY while waiting! dont wait further" "Proceed with Reset\n", __func__); return -1; } DELAY(1000 * 1000); } mpi3mr_dprint(sc, MPI3MR_INFO, "%s :Pending I/Os after wait exaust is %d! Proceed with Reset\n", __func__, mpi3mr_get_fw_pending_ios(sc)); return -1; } static void mpi3mr_scsiio_timeout(void *data) { int retval = 0; struct mpi3mr_softc *sc; struct mpi3mr_cmd *cmd; struct mpi3mr_target *targ_dev = NULL; if (!data) return; cmd = (struct mpi3mr_cmd *)data; sc = cmd->sc; if (cmd->ccb == NULL) { mpi3mr_dprint(sc, MPI3MR_ERROR, "SCSIIO command timed-out with NULL ccb\n"); return; } /* * TMs are not supported for IO timeouts on VD/LD, so directly issue controller reset * with max timeout for outstanding IOs to complete is 180sec. */ targ_dev = cmd->targ; if (targ_dev && (targ_dev->dev_type == MPI3_DEVICE_DEVFORM_VD)) { if (mpi3mr_wait_for_host_io(sc, MPI3MR_RAID_ERRREC_RESET_TIMEOUT)) trigger_reset_from_watchdog(sc, MPI3MR_TRIGGER_SOFT_RESET, MPI3MR_RESET_FROM_SCSIIO_TIMEOUT); return; } /* Issue task abort to recover the timed out IO */ retval = mpi3mr_task_abort(cmd); if (!retval || (retval == ETIMEDOUT)) return; /* * task abort has failed to recover the timed out IO, * try with the target reset */ retval = mpi3mr_target_reset(cmd); if (!retval || (retval == ETIMEDOUT)) return; /* * task abort and target reset has failed. So issue Controller reset(soft reset) * through OCR thread context */ trigger_reset_from_watchdog(sc, MPI3MR_TRIGGER_SOFT_RESET, MPI3MR_RESET_FROM_SCSIIO_TIMEOUT); return; } void int_to_lun(unsigned int lun, U8 *req_lun) { int i; memset(req_lun, 0, sizeof(*req_lun)); for (i = 0; i < sizeof(lun); i += 2) { req_lun[i] = (lun >> 8) & 0xFF; req_lun[i+1] = lun & 0xFF; lun = lun >> 16; } } static U16 get_req_queue_index(struct mpi3mr_softc *sc) { U16 i = 0, reply_q_index = 0, reply_q_pend_ios = 0; reply_q_pend_ios = mpi3mr_atomic_read(&sc->op_reply_q[0].pend_ios); for (i = 0; i < sc->num_queues; i++) { if (reply_q_pend_ios > mpi3mr_atomic_read(&sc->op_reply_q[i].pend_ios)) { reply_q_pend_ios = mpi3mr_atomic_read(&sc->op_reply_q[i].pend_ios); reply_q_index = i; } } return reply_q_index; } static void mpi3mr_action_scsiio(struct mpi3mr_cam_softc *cam_sc, union ccb *ccb) { Mpi3SCSIIORequest_t *req = NULL; struct ccb_scsiio *csio; struct mpi3mr_softc *sc; struct mpi3mr_target *targ; struct mpi3mr_cmd *cm; uint8_t scsi_opcode, queue_idx; uint32_t mpi_control; - struct mpi3mr_op_req_queue *opreqq = NULL; - U32 data_len_blks = 0; - U32 tracked_io_sz = 0; - U32 ioc_pend_data_len = 0, tg_pend_data_len = 0; - struct mpi3mr_throttle_group_info *tg = NULL; - static int ratelimit; sc = cam_sc->sc; mtx_assert(&sc->mpi3mr_mtx, MA_OWNED); if (sc->unrecoverable) { mpi3mr_set_ccbstatus(ccb, CAM_DEV_NOT_THERE); xpt_done(ccb); return; } csio = &ccb->csio; KASSERT(csio->ccb_h.target_id < cam_sc->maxtargets, ("Target %d out of bounds in XPT_SCSI_IO\n", csio->ccb_h.target_id)); scsi_opcode = scsiio_cdb_ptr(csio)[0]; if ((sc->mpi3mr_flags & MPI3MR_FLAGS_SHUTDOWN) && !((scsi_opcode == SYNCHRONIZE_CACHE) || (scsi_opcode == START_STOP_UNIT))) { mpi3mr_set_ccbstatus(ccb, CAM_REQ_CMP); xpt_done(ccb); return; } targ = mpi3mr_find_target_by_per_id(cam_sc, csio->ccb_h.target_id); if (targ == NULL) { mpi3mr_dprint(sc, MPI3MR_XINFO, "Device with target ID: 0x%x does not exist\n", csio->ccb_h.target_id); mpi3mr_set_ccbstatus(ccb, CAM_DEV_NOT_THERE); xpt_done(ccb); return; } if (targ && targ->is_hidden) { mpi3mr_dprint(sc, MPI3MR_XINFO, "Device with target ID: 0x%x is hidden\n", csio->ccb_h.target_id); mpi3mr_set_ccbstatus(ccb, CAM_DEV_NOT_THERE); xpt_done(ccb); return; } if (targ->dev_removed == 1) { mpi3mr_dprint(sc, MPI3MR_XINFO, "Device with target ID: 0x%x is removed\n", csio->ccb_h.target_id); mpi3mr_set_ccbstatus(ccb, CAM_DEV_NOT_THERE); xpt_done(ccb); return; } if (targ->dev_handle == 0x0) { mpi3mr_dprint(sc, MPI3MR_ERROR, "%s NULL handle for target 0x%x\n", __func__, csio->ccb_h.target_id); mpi3mr_set_ccbstatus(ccb, CAM_DEV_NOT_THERE); xpt_done(ccb); return; } if (mpi3mr_atomic_read(&targ->block_io) || (sc->reset_in_progress == 1) || (sc->prepare_for_reset == 1)) { mpi3mr_dprint(sc, MPI3MR_TRACE, "%s target is busy target_id: 0x%x\n", __func__, csio->ccb_h.target_id); mpi3mr_set_ccbstatus(ccb, CAM_REQUEUE_REQ); xpt_done(ccb); return; } /* * Sometimes, it is possible to get a command that is not "In * Progress" and was actually aborted by the upper layer. Check for * this here and complete the command without error. */ if (mpi3mr_get_ccbstatus(ccb) != CAM_REQ_INPROG) { mpi3mr_dprint(sc, MPI3MR_TRACE, "%s Command is not in progress for " "target %u\n", __func__, csio->ccb_h.target_id); xpt_done(ccb); return; } /* * If devinfo is 0 this will be a volume. In that case don't tell CAM * that the volume has timed out. We want volumes to be enumerated * until they are deleted/removed, not just failed. */ if (targ->flags & MPI3MRSAS_TARGET_INREMOVAL) { if (targ->devinfo == 0) mpi3mr_set_ccbstatus(ccb, CAM_REQ_CMP); else mpi3mr_set_ccbstatus(ccb, CAM_SEL_TIMEOUT); xpt_done(ccb); return; } if ((scsi_opcode == UNMAP) && (pci_get_device(sc->mpi3mr_dev) == MPI3_MFGPAGE_DEVID_SAS4116) && (targ->dev_type == MPI3_DEVICE_DEVFORM_PCIE) && (mpi3mr_allow_unmap_to_fw(sc, ccb) == false)) return; cm = mpi3mr_get_command(sc); if (cm == NULL || (sc->mpi3mr_flags & MPI3MR_FLAGS_DIAGRESET)) { if (cm != NULL) { mpi3mr_release_command(cm); } if ((cam_sc->flags & MPI3MRSAS_QUEUE_FROZEN) == 0) { xpt_freeze_simq(cam_sc->sim, 1); cam_sc->flags |= MPI3MRSAS_QUEUE_FROZEN; } ccb->ccb_h.status &= ~CAM_SIM_QUEUED; mpi3mr_set_ccbstatus(ccb, CAM_REQUEUE_REQ); xpt_done(ccb); return; } switch (csio->ccb_h.flags & CAM_DIR_MASK) { case CAM_DIR_IN: mpi_control = MPI3_SCSIIO_FLAGS_DATADIRECTION_READ; cm->data_dir = MPI3MR_READ; break; case CAM_DIR_OUT: mpi_control = MPI3_SCSIIO_FLAGS_DATADIRECTION_WRITE; cm->data_dir = MPI3MR_WRITE; break; case CAM_DIR_NONE: default: mpi_control = MPI3_SCSIIO_FLAGS_DATADIRECTION_NO_DATA_TRANSFER; break; } if (csio->cdb_len > 16) mpi_control |= MPI3_SCSIIO_FLAGS_CDB_GREATER_THAN_16; req = (Mpi3SCSIIORequest_t *)&cm->io_request; bzero(req, sizeof(*req)); req->Function = MPI3_FUNCTION_SCSI_IO; req->HostTag = cm->hosttag; req->DataLength = htole32(csio->dxfer_len); req->DevHandle = htole16(targ->dev_handle); /* * It looks like the hardware doesn't require an explicit tag * number for each transaction. SAM Task Management not supported * at the moment. */ switch (csio->tag_action) { case MSG_HEAD_OF_Q_TAG: mpi_control |= MPI3_SCSIIO_FLAGS_TASKATTRIBUTE_HEADOFQ; break; case MSG_ORDERED_Q_TAG: mpi_control |= MPI3_SCSIIO_FLAGS_TASKATTRIBUTE_ORDEREDQ; break; case MSG_ACA_TASK: mpi_control |= MPI3_SCSIIO_FLAGS_TASKATTRIBUTE_ACAQ; break; case CAM_TAG_ACTION_NONE: case MSG_SIMPLE_Q_TAG: default: mpi_control |= MPI3_SCSIIO_FLAGS_TASKATTRIBUTE_SIMPLEQ; break; } req->Flags = htole32(mpi_control); if (csio->ccb_h.flags & CAM_CDB_POINTER) bcopy(csio->cdb_io.cdb_ptr, &req->CDB.CDB32[0], csio->cdb_len); else { KASSERT(csio->cdb_len <= IOCDBLEN, ("cdb_len %d is greater than IOCDBLEN but CAM_CDB_POINTER " "is not set", csio->cdb_len)); bcopy(csio->cdb_io.cdb_bytes, &req->CDB.CDB32[0],csio->cdb_len); } cm->length = csio->dxfer_len; cm->targ = targ; int_to_lun(csio->ccb_h.target_lun, req->LUN); cm->ccb = ccb; csio->ccb_h.qos.sim_data = sbinuptime(); queue_idx = get_req_queue_index(sc); cm->req_qidx = queue_idx; mpi3mr_dprint(sc, MPI3MR_TRACE, "[QID:%d]: func: %s line:%d CDB: 0x%x targetid: %x SMID: 0x%x\n", (queue_idx + 1), __func__, __LINE__, scsi_opcode, csio->ccb_h.target_id, cm->hosttag); switch ((ccb->ccb_h.flags & CAM_DATA_MASK)) { case CAM_DATA_PADDR: case CAM_DATA_SG_PADDR: device_printf(sc->mpi3mr_dev, "%s: physical addresses not supported\n", __func__); - mpi3mr_release_command(cm); mpi3mr_set_ccbstatus(ccb, CAM_REQ_INVALID); + mpi3mr_release_command(cm); xpt_done(ccb); return; case CAM_DATA_SG: device_printf(sc->mpi3mr_dev, "%s: scatter gather is not supported\n", __func__); - mpi3mr_release_command(cm); mpi3mr_set_ccbstatus(ccb, CAM_REQ_INVALID); + mpi3mr_release_command(cm); xpt_done(ccb); return; case CAM_DATA_VADDR: case CAM_DATA_BIO: if (csio->dxfer_len > (MPI3MR_SG_DEPTH * MPI3MR_4K_PGSZ)) { mpi3mr_set_ccbstatus(ccb, CAM_REQ_TOO_BIG); mpi3mr_release_command(cm); xpt_done(ccb); return; } ccb->ccb_h.status |= CAM_SIM_QUEUED; cm->length = csio->dxfer_len; if (cm->length) cm->data = csio->data_ptr; break; default: - mpi3mr_release_command(cm); mpi3mr_set_ccbstatus(ccb, CAM_REQ_INVALID); - xpt_done(ccb); - return; - } - - /* Prepare SGEs */ - if (mpi3mr_map_request(sc, cm)) { mpi3mr_release_command(cm); xpt_done(ccb); - printf("func: %s line: %d Build SGLs failed\n", __func__, __LINE__); return; } - - opreqq = &sc->op_req_q[queue_idx]; + + /* Prepare SGEs and queue to hardware */ + mpi3mr_map_request(sc, cm); +} + +static void +mpi3mr_enqueue_request(struct mpi3mr_softc *sc, struct mpi3mr_cmd *cm) +{ + static int ratelimit; + struct mpi3mr_op_req_queue *opreqq = &sc->op_req_q[cm->req_qidx]; + struct mpi3mr_throttle_group_info *tg = NULL; + uint32_t data_len_blks = 0; + uint32_t tracked_io_sz = 0; + uint32_t ioc_pend_data_len = 0, tg_pend_data_len = 0; + struct mpi3mr_target *targ = cm->targ; + union ccb *ccb = cm->ccb; + Mpi3SCSIIORequest_t *req = (Mpi3SCSIIORequest_t *)&cm->io_request; if (sc->iot_enable) { - data_len_blks = csio->dxfer_len >> 9; - + data_len_blks = ccb->csio.dxfer_len >> 9; + if ((data_len_blks >= sc->io_throttle_data_length) && targ->io_throttle_enabled) { + tracked_io_sz = data_len_blks; tg = targ->throttle_group; if (tg) { mpi3mr_atomic_add(&sc->pend_large_data_sz, data_len_blks); mpi3mr_atomic_add(&tg->pend_large_data_sz, data_len_blks); ioc_pend_data_len = mpi3mr_atomic_read(&sc->pend_large_data_sz); tg_pend_data_len = mpi3mr_atomic_read(&tg->pend_large_data_sz); if (ratelimit % 1000) { mpi3mr_dprint(sc, MPI3MR_IOT, "large vd_io persist_id(%d), handle(0x%04x), data_len(%d)," "ioc_pending(%d), tg_pending(%d), ioc_high(%d), tg_high(%d)\n", targ->per_id, targ->dev_handle, data_len_blks, ioc_pend_data_len, tg_pend_data_len, sc->io_throttle_high, tg->high); ratelimit++; } if (!tg->io_divert && ((ioc_pend_data_len >= sc->io_throttle_high) || (tg_pend_data_len >= tg->high))) { tg->io_divert = 1; mpi3mr_dprint(sc, MPI3MR_IOT, "VD: Setting divert flag for tg_id(%d), persist_id(%d)\n", tg->id, targ->per_id); if (sc->mpi3mr_debug | MPI3MR_IOT) mpi3mr_print_cdb(ccb); mpi3mr_set_io_divert_for_all_vd_in_tg(sc, tg, 1); } } else { mpi3mr_atomic_add(&sc->pend_large_data_sz, data_len_blks); ioc_pend_data_len = mpi3mr_atomic_read(&sc->pend_large_data_sz); if (ratelimit % 1000) { mpi3mr_dprint(sc, MPI3MR_IOT, "large pd_io persist_id(%d), handle(0x%04x), data_len(%d), ioc_pending(%d), ioc_high(%d)\n", targ->per_id, targ->dev_handle, data_len_blks, ioc_pend_data_len, sc->io_throttle_high); ratelimit++; } if (ioc_pend_data_len >= sc->io_throttle_high) { targ->io_divert = 1; mpi3mr_dprint(sc, MPI3MR_IOT, "PD: Setting divert flag for persist_id(%d)\n", targ->per_id); if (sc->mpi3mr_debug | MPI3MR_IOT) mpi3mr_print_cdb(ccb); } } } if (targ->io_divert) { req->MsgFlags |= MPI3_SCSIIO_MSGFLAGS_DIVERT_TO_FIRMWARE; - mpi_control |= MPI3_SCSIIO_FLAGS_DIVERT_REASON_IO_THROTTLING; + req->Flags = htole32(le32toh(req->Flags) | MPI3_SCSIIO_FLAGS_DIVERT_REASON_IO_THROTTLING); } } - req->Flags = htole32(mpi_control); if (mpi3mr_submit_io(sc, opreqq, (U8 *)&cm->io_request)) { - mpi3mr_release_command(cm); if (tracked_io_sz) { mpi3mr_atomic_sub(&sc->pend_large_data_sz, tracked_io_sz); if (tg) mpi3mr_atomic_sub(&tg->pend_large_data_sz, tracked_io_sz); } mpi3mr_set_ccbstatus(ccb, CAM_RESRC_UNAVAIL); + mpi3mr_release_command(cm); xpt_done(ccb); } else { callout_reset_sbt(&cm->callout, mstosbt(ccb->ccb_h.timeout), 0, mpi3mr_scsiio_timeout, cm, 0); cm->callout_owner = true; mpi3mr_atomic_inc(&sc->fw_outstanding); mpi3mr_atomic_inc(&targ->outstanding); if (mpi3mr_atomic_read(&sc->fw_outstanding) > sc->io_cmds_highwater) sc->io_cmds_highwater++; } return; } static void mpi3mr_cam_poll(struct cam_sim *sim) { struct mpi3mr_cam_softc *cam_sc; struct mpi3mr_irq_context *irq_ctx; struct mpi3mr_softc *sc; int i; cam_sc = cam_sim_softc(sim); sc = cam_sc->sc; mpi3mr_dprint(cam_sc->sc, MPI3MR_TRACE, "func: %s line: %d is called\n", __func__, __LINE__); for (i = 0; i < sc->num_queues; i++) { irq_ctx = sc->irq_ctx + i; if (irq_ctx->op_reply_q->qid) { mpi3mr_complete_io_cmd(sc, irq_ctx); } } } static void mpi3mr_cam_action(struct cam_sim *sim, union ccb *ccb) { struct mpi3mr_cam_softc *cam_sc; struct mpi3mr_target *targ; cam_sc = cam_sim_softc(sim); mpi3mr_dprint(cam_sc->sc, MPI3MR_TRACE, "ccb func_code 0x%x target id: 0x%x\n", ccb->ccb_h.func_code, ccb->ccb_h.target_id); mtx_assert(&cam_sc->sc->mpi3mr_mtx, MA_OWNED); switch (ccb->ccb_h.func_code) { case XPT_PATH_INQ: { struct ccb_pathinq *cpi = &ccb->cpi; cpi->version_num = 1; cpi->hba_inquiry = PI_SDTR_ABLE|PI_TAG_ABLE|PI_WIDE_16; cpi->target_sprt = 0; cpi->hba_misc = PIM_NOBUSRESET | PIM_UNMAPPED | PIM_NOSCAN; cpi->hba_eng_cnt = 0; cpi->max_target = cam_sc->maxtargets - 1; cpi->max_lun = 0; /* * initiator_id is set here to an ID outside the set of valid * target IDs (including volumes). */ cpi->initiator_id = cam_sc->maxtargets; strlcpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN); strlcpy(cpi->hba_vid, "Broadcom", HBA_IDLEN); strlcpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN); cpi->unit_number = cam_sim_unit(sim); cpi->bus_id = cam_sim_bus(sim); /* * XXXSLM-I think this needs to change based on config page or * something instead of hardcoded to 150000. */ cpi->base_transfer_speed = 150000; cpi->transport = XPORT_SAS; cpi->transport_version = 0; cpi->protocol = PROTO_SCSI; cpi->protocol_version = SCSI_REV_SPC; targ = mpi3mr_find_target_by_per_id(cam_sc, ccb->ccb_h.target_id); if (targ && (targ->dev_type == MPI3_DEVICE_DEVFORM_PCIE) && ((targ->dev_spec.pcie_inf.dev_info & MPI3_DEVICE0_PCIE_DEVICE_INFO_TYPE_MASK) == MPI3_DEVICE0_PCIE_DEVICE_INFO_TYPE_NVME_DEVICE)) { cpi->maxio = targ->dev_spec.pcie_inf.mdts; mpi3mr_dprint(cam_sc->sc, MPI3MR_XINFO, "PCI device target_id: %u max io size: %u\n", ccb->ccb_h.target_id, cpi->maxio); } else { cpi->maxio = PAGE_SIZE * (MPI3MR_SG_DEPTH - 1); } mpi3mr_set_ccbstatus(ccb, CAM_REQ_CMP); break; } case XPT_GET_TRAN_SETTINGS: { struct ccb_trans_settings *cts; struct ccb_trans_settings_sas *sas; struct ccb_trans_settings_scsi *scsi; cts = &ccb->cts; sas = &cts->xport_specific.sas; scsi = &cts->proto_specific.scsi; KASSERT(cts->ccb_h.target_id < cam_sc->maxtargets, ("Target %d out of bounds in XPT_GET_TRAN_SETTINGS\n", cts->ccb_h.target_id)); targ = mpi3mr_find_target_by_per_id(cam_sc, cts->ccb_h.target_id); if (targ == NULL) { mpi3mr_dprint(cam_sc->sc, MPI3MR_TRACE, "Device with target ID: 0x%x does not exist\n", cts->ccb_h.target_id); mpi3mr_set_ccbstatus(ccb, CAM_DEV_NOT_THERE); break; } if ((targ->dev_handle == 0x0) || (targ->dev_removed == 1)) { mpi3mr_set_ccbstatus(ccb, CAM_DEV_NOT_THERE); break; } cts->protocol_version = SCSI_REV_SPC2; cts->transport = XPORT_SAS; cts->transport_version = 0; sas->valid = CTS_SAS_VALID_SPEED; switch (targ->link_rate) { case 0x08: sas->bitrate = 150000; break; case 0x09: sas->bitrate = 300000; break; case 0x0a: sas->bitrate = 600000; break; case 0x0b: sas->bitrate = 1200000; break; default: sas->valid = 0; } cts->protocol = PROTO_SCSI; scsi->valid = CTS_SCSI_VALID_TQ; scsi->flags = CTS_SCSI_FLAGS_TAG_ENB; mpi3mr_set_ccbstatus(ccb, CAM_REQ_CMP); break; } case XPT_CALC_GEOMETRY: cam_calc_geometry(&ccb->ccg, /*extended*/1); mpi3mr_set_ccbstatus(ccb, CAM_REQ_CMP); break; case XPT_RESET_DEV: mpi3mr_dprint(cam_sc->sc, MPI3MR_INFO, "mpi3mr_action " "XPT_RESET_DEV\n"); return; case XPT_RESET_BUS: case XPT_ABORT: case XPT_TERM_IO: mpi3mr_dprint(cam_sc->sc, MPI3MR_INFO, "mpi3mr_action faking success " "for abort or reset\n"); mpi3mr_set_ccbstatus(ccb, CAM_REQ_CMP); break; case XPT_SCSI_IO: mpi3mr_action_scsiio(cam_sc, ccb); return; default: mpi3mr_set_ccbstatus(ccb, CAM_FUNC_NOTAVAIL); break; } xpt_done(ccb); } void mpi3mr_startup_increment(struct mpi3mr_cam_softc *cam_sc) { if ((cam_sc->flags & MPI3MRSAS_IN_STARTUP) != 0) { if (cam_sc->startup_refcount++ == 0) { /* just starting, freeze the simq */ mpi3mr_dprint(cam_sc->sc, MPI3MR_XINFO, "%s freezing simq\n", __func__); xpt_hold_boot(); } mpi3mr_dprint(cam_sc->sc, MPI3MR_XINFO, "%s refcount %u\n", __func__, cam_sc->startup_refcount); } } void mpi3mr_release_simq_reinit(struct mpi3mr_cam_softc *cam_sc) { if (cam_sc->flags & MPI3MRSAS_QUEUE_FROZEN) { cam_sc->flags &= ~MPI3MRSAS_QUEUE_FROZEN; xpt_release_simq(cam_sc->sim, 1); mpi3mr_dprint(cam_sc->sc, MPI3MR_INFO, "Unfreezing SIM queue\n"); } } void mpi3mr_rescan_target(struct mpi3mr_softc *sc, struct mpi3mr_target *targ) { struct mpi3mr_cam_softc *cam_sc = sc->cam_sc; path_id_t pathid; target_id_t targetid; union ccb *ccb; pathid = cam_sim_path(cam_sc->sim); if (targ == NULL) targetid = CAM_TARGET_WILDCARD; else targetid = targ->per_id; /* * Allocate a CCB and schedule a rescan. */ ccb = xpt_alloc_ccb_nowait(); if (ccb == NULL) { mpi3mr_dprint(sc, MPI3MR_ERROR, "unable to alloc CCB for rescan\n"); return; } if (xpt_create_path(&ccb->ccb_h.path, NULL, pathid, targetid, CAM_LUN_WILDCARD) != CAM_REQ_CMP) { mpi3mr_dprint(sc, MPI3MR_ERROR, "unable to create path for rescan\n"); xpt_free_ccb(ccb); return; } if (targetid == CAM_TARGET_WILDCARD) ccb->ccb_h.func_code = XPT_SCAN_BUS; else ccb->ccb_h.func_code = XPT_SCAN_TGT; mpi3mr_dprint(sc, MPI3MR_EVENT, "%s target id 0x%x\n", __func__, targetid); xpt_rescan(ccb); } void mpi3mr_startup_decrement(struct mpi3mr_cam_softc *cam_sc) { if ((cam_sc->flags & MPI3MRSAS_IN_STARTUP) != 0) { if (--cam_sc->startup_refcount == 0) { /* finished all discovery-related actions, release * the simq and rescan for the latest topology. */ mpi3mr_dprint(cam_sc->sc, MPI3MR_XINFO, "%s releasing simq\n", __func__); cam_sc->flags &= ~MPI3MRSAS_IN_STARTUP; xpt_release_simq(cam_sc->sim, 1); xpt_release_boot(); } mpi3mr_dprint(cam_sc->sc, MPI3MR_XINFO, "%s refcount %u\n", __func__, cam_sc->startup_refcount); } } static void mpi3mr_fw_event_free(struct mpi3mr_softc *sc, struct mpi3mr_fw_event_work *fw_event) { if (!fw_event) return; if (fw_event->event_data != NULL) { free(fw_event->event_data, M_MPI3MR); fw_event->event_data = NULL; } free(fw_event, M_MPI3MR); fw_event = NULL; } static void mpi3mr_freeup_events(struct mpi3mr_softc *sc) { struct mpi3mr_fw_event_work *fw_event = NULL; mtx_lock(&sc->mpi3mr_mtx); while ((fw_event = TAILQ_FIRST(&sc->cam_sc->ev_queue)) != NULL) { TAILQ_REMOVE(&sc->cam_sc->ev_queue, fw_event, ev_link); mpi3mr_fw_event_free(sc, fw_event); } mtx_unlock(&sc->mpi3mr_mtx); } static void mpi3mr_sastopochg_evt_debug(struct mpi3mr_softc *sc, Mpi3EventDataSasTopologyChangeList_t *event_data) { int i; U16 handle; U8 reason_code, phy_number; char *status_str = NULL; U8 link_rate, prev_link_rate; switch (event_data->ExpStatus) { case MPI3_EVENT_SAS_TOPO_ES_NOT_RESPONDING: status_str = "remove"; break; case MPI3_EVENT_SAS_TOPO_ES_RESPONDING: status_str = "responding"; break; case MPI3_EVENT_SAS_TOPO_ES_DELAY_NOT_RESPONDING: status_str = "remove delay"; break; case MPI3_EVENT_SAS_TOPO_ES_NO_EXPANDER: status_str = "direct attached"; break; default: status_str = "unknown status"; break; } mpi3mr_dprint(sc, MPI3MR_INFO, "%s :sas topology change: (%s)\n", __func__, status_str); mpi3mr_dprint(sc, MPI3MR_INFO, "%s :\texpander_handle(0x%04x), enclosure_handle(0x%04x) " "start_phy(%02d), num_entries(%d)\n", __func__, (event_data->ExpanderDevHandle), (event_data->EnclosureHandle), event_data->StartPhyNum, event_data->NumEntries); for (i = 0; i < event_data->NumEntries; i++) { handle = (event_data->PhyEntry[i].AttachedDevHandle); if (!handle) continue; phy_number = event_data->StartPhyNum + i; reason_code = event_data->PhyEntry[i].Status & MPI3_EVENT_SAS_TOPO_PHY_RC_MASK; switch (reason_code) { case MPI3_EVENT_SAS_TOPO_PHY_RC_TARG_NOT_RESPONDING: status_str = "target remove"; break; case MPI3_EVENT_SAS_TOPO_PHY_RC_DELAY_NOT_RESPONDING: status_str = "delay target remove"; break; case MPI3_EVENT_SAS_TOPO_PHY_RC_PHY_CHANGED: status_str = "link rate change"; break; case MPI3_EVENT_SAS_TOPO_PHY_RC_NO_CHANGE: status_str = "target responding"; break; default: status_str = "unknown"; break; } link_rate = event_data->PhyEntry[i].LinkRate >> 4; prev_link_rate = event_data->PhyEntry[i].LinkRate & 0xF; mpi3mr_dprint(sc, MPI3MR_INFO, "%s :\tphy(%02d), attached_handle(0x%04x): %s:" " link rate: new(0x%02x), old(0x%02x)\n", __func__, phy_number, handle, status_str, link_rate, prev_link_rate); } } static void mpi3mr_process_sastopochg_evt(struct mpi3mr_softc *sc, struct mpi3mr_fw_event_work *fwevt) { Mpi3EventDataSasTopologyChangeList_t *event_data = (Mpi3EventDataSasTopologyChangeList_t *)fwevt->event_data; int i; U16 handle; U8 reason_code, link_rate; struct mpi3mr_target *target = NULL; mpi3mr_sastopochg_evt_debug(sc, event_data); for (i = 0; i < event_data->NumEntries; i++) { handle = le16toh(event_data->PhyEntry[i].AttachedDevHandle); link_rate = event_data->PhyEntry[i].LinkRate >> 4; if (!handle) continue; target = mpi3mr_find_target_by_dev_handle(sc->cam_sc, handle); if (!target) continue; target->link_rate = link_rate; reason_code = event_data->PhyEntry[i].Status & MPI3_EVENT_SAS_TOPO_PHY_RC_MASK; switch (reason_code) { case MPI3_EVENT_SAS_TOPO_PHY_RC_TARG_NOT_RESPONDING: if (target->exposed_to_os) mpi3mr_remove_device_from_os(sc, target->dev_handle); mpi3mr_remove_device_from_list(sc, target, false); break; case MPI3_EVENT_SAS_TOPO_PHY_RC_PHY_CHANGED: break; default: break; } } /* * refcount was incremented for this event in * mpi3mr_evt_handler. Decrement it here because the event has * been processed. */ mpi3mr_startup_decrement(sc->cam_sc); return; } static inline void mpi3mr_logdata_evt_bh(struct mpi3mr_softc *sc, struct mpi3mr_fw_event_work *fwevt) { mpi3mr_app_save_logdata(sc, fwevt->event_data, fwevt->event_data_size); } static void mpi3mr_pcietopochg_evt_debug(struct mpi3mr_softc *sc, Mpi3EventDataPcieTopologyChangeList_t *event_data) { int i; U16 handle; U16 reason_code; U8 port_number; char *status_str = NULL; U8 link_rate, prev_link_rate; switch (event_data->SwitchStatus) { case MPI3_EVENT_PCIE_TOPO_SS_NOT_RESPONDING: status_str = "remove"; break; case MPI3_EVENT_PCIE_TOPO_SS_RESPONDING: status_str = "responding"; break; case MPI3_EVENT_PCIE_TOPO_SS_DELAY_NOT_RESPONDING: status_str = "remove delay"; break; case MPI3_EVENT_PCIE_TOPO_SS_NO_PCIE_SWITCH: status_str = "direct attached"; break; default: status_str = "unknown status"; break; } mpi3mr_dprint(sc, MPI3MR_INFO, "%s :pcie topology change: (%s)\n", __func__, status_str); mpi3mr_dprint(sc, MPI3MR_INFO, "%s :\tswitch_handle(0x%04x), enclosure_handle(0x%04x)" "start_port(%02d), num_entries(%d)\n", __func__, le16toh(event_data->SwitchDevHandle), le16toh(event_data->EnclosureHandle), event_data->StartPortNum, event_data->NumEntries); for (i = 0; i < event_data->NumEntries; i++) { handle = le16toh(event_data->PortEntry[i].AttachedDevHandle); if (!handle) continue; port_number = event_data->StartPortNum + i; reason_code = event_data->PortEntry[i].PortStatus; switch (reason_code) { case MPI3_EVENT_PCIE_TOPO_PS_NOT_RESPONDING: status_str = "target remove"; break; case MPI3_EVENT_PCIE_TOPO_PS_DELAY_NOT_RESPONDING: status_str = "delay target remove"; break; case MPI3_EVENT_PCIE_TOPO_PS_PORT_CHANGED: status_str = "link rate change"; break; case MPI3_EVENT_PCIE_TOPO_PS_NO_CHANGE: status_str = "target responding"; break; default: status_str = "unknown"; break; } link_rate = event_data->PortEntry[i].CurrentPortInfo & MPI3_EVENT_PCIE_TOPO_PI_RATE_MASK; prev_link_rate = event_data->PortEntry[i].PreviousPortInfo & MPI3_EVENT_PCIE_TOPO_PI_RATE_MASK; mpi3mr_dprint(sc, MPI3MR_INFO, "%s :\tport(%02d), attached_handle(0x%04x): %s:" " link rate: new(0x%02x), old(0x%02x)\n", __func__, port_number, handle, status_str, link_rate, prev_link_rate); } } static void mpi3mr_process_pcietopochg_evt(struct mpi3mr_softc *sc, struct mpi3mr_fw_event_work *fwevt) { Mpi3EventDataPcieTopologyChangeList_t *event_data = (Mpi3EventDataPcieTopologyChangeList_t *)fwevt->event_data; int i; U16 handle; U8 reason_code, link_rate; struct mpi3mr_target *target = NULL; mpi3mr_pcietopochg_evt_debug(sc, event_data); for (i = 0; i < event_data->NumEntries; i++) { handle = le16toh(event_data->PortEntry[i].AttachedDevHandle); if (!handle) continue; target = mpi3mr_find_target_by_dev_handle(sc->cam_sc, handle); if (!target) continue; link_rate = event_data->PortEntry[i].CurrentPortInfo & MPI3_EVENT_PCIE_TOPO_PI_RATE_MASK; target->link_rate = link_rate; reason_code = event_data->PortEntry[i].PortStatus; switch (reason_code) { case MPI3_EVENT_PCIE_TOPO_PS_NOT_RESPONDING: if (target->exposed_to_os) mpi3mr_remove_device_from_os(sc, target->dev_handle); mpi3mr_remove_device_from_list(sc, target, false); break; case MPI3_EVENT_PCIE_TOPO_PS_PORT_CHANGED: break; default: break; } } /* * refcount was incremented for this event in * mpi3mr_evt_handler. Decrement it here because the event has * been processed. */ mpi3mr_startup_decrement(sc->cam_sc); return; } void mpi3mr_add_device(struct mpi3mr_softc *sc, U16 per_id) { struct mpi3mr_target *target; mpi3mr_dprint(sc, MPI3MR_EVENT, "Adding device(persistent id: 0x%x)\n", per_id); mpi3mr_startup_increment(sc->cam_sc); target = mpi3mr_find_target_by_per_id(sc->cam_sc, per_id); if (!target) { mpi3mr_dprint(sc, MPI3MR_INFO, "Not available in driver's" "internal target list, persistent_id: %d\n", per_id); goto out; } if (target->is_hidden) { mpi3mr_dprint(sc, MPI3MR_EVENT, "Target is hidden, persistent_id: %d\n", per_id); goto out; } if (!target->exposed_to_os && !sc->reset_in_progress) { mpi3mr_rescan_target(sc, target); mpi3mr_dprint(sc, MPI3MR_INFO, "Added device persistent_id: %d dev_handle: %d\n", per_id, target->dev_handle); target->exposed_to_os = 1; } out: mpi3mr_startup_decrement(sc->cam_sc); } int mpi3mr_remove_device_from_os(struct mpi3mr_softc *sc, U16 handle) { U32 i = 0; int retval = 0; struct mpi3mr_target *target; mpi3mr_dprint(sc, MPI3MR_EVENT, "Removing Device (dev_handle: %d)\n", handle); target = mpi3mr_find_target_by_dev_handle(sc->cam_sc, handle); if (!target) { mpi3mr_dprint(sc, MPI3MR_INFO, "Device (persistent_id: %d dev_handle: %d) is already removed from driver's list\n", target->per_id, handle); mpi3mr_rescan_target(sc, NULL); retval = -1; goto out; } target->flags |= MPI3MRSAS_TARGET_INREMOVAL; while (mpi3mr_atomic_read(&target->outstanding) && (i < 30)) { i++; if (!(i % 2)) { mpi3mr_dprint(sc, MPI3MR_INFO, "[%2d]waiting for " "waiting for outstanding commands to complete on target: %d\n", i, target->per_id); } DELAY(1000 * 1000); } if (target->exposed_to_os && !sc->reset_in_progress) { mpi3mr_rescan_target(sc, target); mpi3mr_dprint(sc, MPI3MR_INFO, "Removed device(persistent_id: %d dev_handle: %d)\n", target->per_id, handle); target->exposed_to_os = 0; } target->flags &= ~MPI3MRSAS_TARGET_INREMOVAL; out: return retval; } void mpi3mr_remove_device_from_list(struct mpi3mr_softc *sc, struct mpi3mr_target *target, bool must_delete) { mtx_lock_spin(&sc->target_lock); if ((target->state == MPI3MR_DEV_REMOVE_HS_STARTED) || (must_delete == true)) { TAILQ_REMOVE(&sc->cam_sc->tgt_list, target, tgt_next); target->state = MPI3MR_DEV_DELETED; } mtx_unlock_spin(&sc->target_lock); if (target->state == MPI3MR_DEV_DELETED) { free(target, M_MPI3MR); target = NULL; } return; } /** * mpi3mr_devstatuschg_evt_bh - DevStatusChange evt bottomhalf * @sc: Adapter instance reference * @fwevt: Firmware event * * Process Device Status Change event and based on device's new * information, either expose the device to the upper layers, or * remove the device from upper layers. * * Return: Nothing. */ static void mpi3mr_devstatuschg_evt_bh(struct mpi3mr_softc *sc, struct mpi3mr_fw_event_work *fwevt) { U16 dev_handle = 0; U8 uhide = 0, delete = 0, cleanup = 0; struct mpi3mr_target *tgtdev = NULL; Mpi3EventDataDeviceStatusChange_t *evtdata = (Mpi3EventDataDeviceStatusChange_t *)fwevt->event_data; dev_handle = le16toh(evtdata->DevHandle); mpi3mr_dprint(sc, MPI3MR_INFO, "%s :device status change: handle(0x%04x): reason code(0x%x)\n", __func__, dev_handle, evtdata->ReasonCode); switch (evtdata->ReasonCode) { case MPI3_EVENT_DEV_STAT_RC_HIDDEN: delete = 1; break; case MPI3_EVENT_DEV_STAT_RC_NOT_HIDDEN: uhide = 1; break; case MPI3_EVENT_DEV_STAT_RC_VD_NOT_RESPONDING: delete = 1; cleanup = 1; break; default: mpi3mr_dprint(sc, MPI3MR_INFO, "%s :Unhandled reason code(0x%x)\n", __func__, evtdata->ReasonCode); break; } tgtdev = mpi3mr_find_target_by_dev_handle(sc->cam_sc, dev_handle); if (!tgtdev) return; if (uhide) { if (!tgtdev->exposed_to_os) mpi3mr_add_device(sc, tgtdev->per_id); } if (delete) mpi3mr_remove_device_from_os(sc, dev_handle); if (cleanup) mpi3mr_remove_device_from_list(sc, tgtdev, false); } /** * mpi3mr_devinfochg_evt_bh - DeviceInfoChange evt bottomhalf * @sc: Adapter instance reference * @dev_pg0: New device page0 * * Process Device Info Change event and based on device's new * information, either expose the device to the upper layers, or * remove the device from upper layers or update the details of * the device. * * Return: Nothing. */ static void mpi3mr_devinfochg_evt_bh(struct mpi3mr_softc *sc, Mpi3DevicePage0_t *dev_pg0) { struct mpi3mr_target *tgtdev = NULL; U16 dev_handle = 0, perst_id = 0; perst_id = le16toh(dev_pg0->PersistentID); dev_handle = le16toh(dev_pg0->DevHandle); mpi3mr_dprint(sc, MPI3MR_INFO, "%s :Device info change: handle(0x%04x): persist_id(0x%x)\n", __func__, dev_handle, perst_id); tgtdev = mpi3mr_find_target_by_dev_handle(sc->cam_sc, dev_handle); if (!tgtdev) return; mpi3mr_update_device(sc, tgtdev, dev_pg0, false); if (!tgtdev->is_hidden && !tgtdev->exposed_to_os) mpi3mr_add_device(sc, perst_id); if (tgtdev->is_hidden && tgtdev->exposed_to_os) mpi3mr_remove_device_from_os(sc, tgtdev->dev_handle); } static void mpi3mr_fw_work(struct mpi3mr_softc *sc, struct mpi3mr_fw_event_work *fw_event) { if (sc->mpi3mr_flags & MPI3MR_FLAGS_SHUTDOWN) goto out; if (!fw_event->process_event) goto evt_ack; mpi3mr_dprint(sc, MPI3MR_EVENT, "(%d)->(%s) Working on Event: [%x]\n", event_count++, __func__, fw_event->event); switch (fw_event->event) { case MPI3_EVENT_DEVICE_ADDED: { Mpi3DevicePage0_t *dev_pg0 = (Mpi3DevicePage0_t *) fw_event->event_data; mpi3mr_add_device(sc, dev_pg0->PersistentID); break; } case MPI3_EVENT_DEVICE_INFO_CHANGED: { mpi3mr_devinfochg_evt_bh(sc, (Mpi3DevicePage0_t *) fw_event->event_data); break; } case MPI3_EVENT_DEVICE_STATUS_CHANGE: { mpi3mr_devstatuschg_evt_bh(sc, fw_event); break; } case MPI3_EVENT_SAS_TOPOLOGY_CHANGE_LIST: { mpi3mr_process_sastopochg_evt(sc, fw_event); break; } case MPI3_EVENT_PCIE_TOPOLOGY_CHANGE_LIST: { mpi3mr_process_pcietopochg_evt(sc, fw_event); break; } case MPI3_EVENT_LOG_DATA: { mpi3mr_logdata_evt_bh(sc, fw_event); break; } default: mpi3mr_dprint(sc, MPI3MR_TRACE,"Unhandled event 0x%0X\n", fw_event->event); break; } evt_ack: if (fw_event->send_ack) { mpi3mr_dprint(sc, MPI3MR_EVENT,"Process event ACK for event 0x%0X\n", fw_event->event); mpi3mr_process_event_ack(sc, fw_event->event, fw_event->event_context); } out: mpi3mr_dprint(sc, MPI3MR_EVENT, "(%d)->(%s) Event Free: [%x]\n", event_count, __func__, fw_event->event); mpi3mr_fw_event_free(sc, fw_event); } void mpi3mr_firmware_event_work(void *arg, int pending) { struct mpi3mr_fw_event_work *fw_event; struct mpi3mr_softc *sc; sc = (struct mpi3mr_softc *)arg; mtx_lock(&sc->fwevt_lock); while ((fw_event = TAILQ_FIRST(&sc->cam_sc->ev_queue)) != NULL) { TAILQ_REMOVE(&sc->cam_sc->ev_queue, fw_event, ev_link); mtx_unlock(&sc->fwevt_lock); mpi3mr_fw_work(sc, fw_event); mtx_lock(&sc->fwevt_lock); } mtx_unlock(&sc->fwevt_lock); } /* * mpi3mr_cam_attach - CAM layer registration * @sc: Adapter reference * * This function does simq allocation, cam registration, xpt_bus registration, * event taskqueue initialization and async event handler registration. * * Return: 0 on success and proper error codes on failure */ int mpi3mr_cam_attach(struct mpi3mr_softc *sc) { struct mpi3mr_cam_softc *cam_sc; cam_status status; int unit, error = 0, reqs; mpi3mr_dprint(sc, MPI3MR_XINFO, "Starting CAM Attach\n"); cam_sc = malloc(sizeof(struct mpi3mr_cam_softc), M_MPI3MR, M_WAITOK|M_ZERO); if (!cam_sc) { mpi3mr_dprint(sc, MPI3MR_ERROR, "Failed to allocate memory for controller CAM instance\n"); return (ENOMEM); } cam_sc->maxtargets = sc->facts.max_perids + 1; TAILQ_INIT(&cam_sc->tgt_list); sc->cam_sc = cam_sc; cam_sc->sc = sc; reqs = sc->max_host_ios; if ((cam_sc->devq = cam_simq_alloc(reqs)) == NULL) { mpi3mr_dprint(sc, MPI3MR_ERROR, "Failed to allocate SIMQ\n"); error = ENOMEM; goto out; } unit = device_get_unit(sc->mpi3mr_dev); cam_sc->sim = cam_sim_alloc(mpi3mr_cam_action, mpi3mr_cam_poll, "mpi3mr", cam_sc, unit, &sc->mpi3mr_mtx, reqs, reqs, cam_sc->devq); if (cam_sc->sim == NULL) { mpi3mr_dprint(sc, MPI3MR_ERROR, "Failed to allocate SIM\n"); error = EINVAL; goto out; } TAILQ_INIT(&cam_sc->ev_queue); /* Initialize taskqueue for Event Handling */ TASK_INIT(&cam_sc->ev_task, 0, mpi3mr_firmware_event_work, sc); cam_sc->ev_tq = taskqueue_create("mpi3mr_taskq", M_NOWAIT | M_ZERO, taskqueue_thread_enqueue, &cam_sc->ev_tq); taskqueue_start_threads(&cam_sc->ev_tq, 1, PRIBIO, "%s taskq", device_get_nameunit(sc->mpi3mr_dev)); mtx_lock(&sc->mpi3mr_mtx); /* * XXX There should be a bus for every port on the adapter, but since * we're just going to fake the topology for now, we'll pretend that * everything is just a target on a single bus. */ if ((error = xpt_bus_register(cam_sc->sim, sc->mpi3mr_dev, 0)) != 0) { mpi3mr_dprint(sc, MPI3MR_ERROR, "Error 0x%x registering SCSI bus\n", error); mtx_unlock(&sc->mpi3mr_mtx); goto out; } /* * Assume that discovery events will start right away. * * Hold off boot until discovery is complete. */ cam_sc->flags |= MPI3MRSAS_IN_STARTUP | MPI3MRSAS_IN_DISCOVERY; sc->cam_sc->startup_refcount = 0; mpi3mr_startup_increment(cam_sc); callout_init(&cam_sc->discovery_callout, 1 /*mpsafe*/); /* * Register for async events so we can determine the EEDP * capabilities of devices. */ status = xpt_create_path(&cam_sc->path, /*periph*/NULL, cam_sim_path(sc->cam_sc->sim), CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD); if (status != CAM_REQ_CMP) { mpi3mr_dprint(sc, MPI3MR_ERROR, "Error 0x%x creating sim path\n", status); cam_sc->path = NULL; } if (status != CAM_REQ_CMP) { /* * EEDP use is the exception, not the rule. * Warn the user, but do not fail to attach. */ mpi3mr_dprint(sc, MPI3MR_INFO, "EEDP capabilities disabled.\n"); } mtx_unlock(&sc->mpi3mr_mtx); error = mpi3mr_register_events(sc); out: mpi3mr_dprint(sc, MPI3MR_XINFO, "%s Exiting CAM attach, error: 0x%x n", __func__, error); return (error); } int mpi3mr_cam_detach(struct mpi3mr_softc *sc) { struct mpi3mr_cam_softc *cam_sc; struct mpi3mr_target *target; mpi3mr_dprint(sc, MPI3MR_XINFO, "%s, Starting CAM detach\n", __func__); if (sc->cam_sc == NULL) return (0); cam_sc = sc->cam_sc; mpi3mr_freeup_events(sc); /* * Drain and free the event handling taskqueue with the lock * unheld so that any parallel processing tasks drain properly * without deadlocking. */ if (cam_sc->ev_tq != NULL) taskqueue_free(cam_sc->ev_tq); mtx_lock(&sc->mpi3mr_mtx); while (cam_sc->startup_refcount != 0) mpi3mr_startup_decrement(cam_sc); /* Deregister our async handler */ if (cam_sc->path != NULL) { xpt_free_path(cam_sc->path); cam_sc->path = NULL; } if (cam_sc->flags & MPI3MRSAS_IN_STARTUP) xpt_release_simq(cam_sc->sim, 1); if (cam_sc->sim != NULL) { xpt_bus_deregister(cam_sim_path(cam_sc->sim)); cam_sim_free(cam_sc->sim, FALSE); } mtx_unlock(&sc->mpi3mr_mtx); if (cam_sc->devq != NULL) cam_simq_free(cam_sc->devq); get_target: mtx_lock_spin(&sc->target_lock); TAILQ_FOREACH(target, &cam_sc->tgt_list, tgt_next) { TAILQ_REMOVE(&sc->cam_sc->tgt_list, target, tgt_next); mtx_unlock_spin(&sc->target_lock); goto out_tgt_free; } mtx_unlock_spin(&sc->target_lock); out_tgt_free: if (target) { free(target, M_MPI3MR); target = NULL; goto get_target; } free(cam_sc, M_MPI3MR); sc->cam_sc = NULL; mpi3mr_dprint(sc, MPI3MR_XINFO, "%s, Exiting CAM detach\n", __func__); return (0); }