D15143.1785235709.diff
No OneTemporary
Actions

Size

916 KB

Referenced Files

None

Subscribers

None

D15143.1785235709.diff
View Options

This file is larger than 256 KB, so syntax highlighting was skipped.

	Index: sys/dev/bnxt/bnxt.h
	===================================================================
	--- sys/dev/bnxt/bnxt.h
	+++ sys/dev/bnxt/bnxt.h
	@@ -93,6 +93,18 @@

	#define BNXT_MAX_MTU 9000

	+#define BNXT_RSS_HASH_TYPE_TCPV4 0
	+#define BNXT_RSS_HASH_TYPE_UDPV4 1
	+#define BNXT_RSS_HASH_TYPE_IPV4 2
	+#define BNXT_RSS_HASH_TYPE_TCPV6 3
	+#define BNXT_RSS_HASH_TYPE_UDPV6 4
	+#define BNXT_RSS_HASH_TYPE_IPV6 5
	+#define BNXT_GET_RSS_PROFILE_ID(rss_hash_type) ((rss_hash_type >> 1) & 0x1F)
	+
	+#define BNXT_NO_MORE_WOL_FILTERS 0xFFFF
	+#define bnxt_wol_supported(softc) (!((softc)->flags & BNXT_FLAG_VF) && \
	+ ((softc)->flags & BNXT_FLAG_WOL_CAP ))
	+
	/* Completion related defines */
	#define CMP_VALID(cmp, v_bit) \
	((!!(((struct cmpl_base *)(cmp))->info3_v & htole32(CMPL_BASE_V))) == !!(v_bit) )
	@@ -199,10 +211,33 @@
	#define BNXT_HWRM_LOCK_DESTROY(_softc) mtx_destroy(&(_softc)->hwrm_lock)
	#define BNXT_HWRM_LOCK_ASSERT(_softc) mtx_assert(&(_softc)->hwrm_lock, \
	MA_OWNED)
	+#define BNXT_IS_FLOW_CTRL_CHANGED(link_info) \
	+ ((link_info->last_flow_ctrl.tx != link_info->flow_ctrl.tx) \|\| \
	+ (link_info->last_flow_ctrl.rx != link_info->flow_ctrl.rx) \|\| \
	+ (link_info->last_flow_ctrl.autoneg != link_info->flow_ctrl.autoneg))

	/* Chip info */
	#define BNXT_TSO_SIZE UINT16_MAX

	+#define min_t(type, x, y) ({ \
	+ type __min1 = (x); \
	+ type __min2 = (y); \
	+ __min1 < __min2 ? __min1 : __min2; })
	+
	+#define max_t(type, x, y) ({ \
	+ type __max1 = (x); \
	+ type __max2 = (y); \
	+ __max1 > __max2 ? __max1 : __max2; })
	+
	+#define clamp_t(type, _x, min, max) min_t(type, max_t(type, _x, min), max)
	+
	+#define BNXT_IFMEDIA_ADD(supported, fw_speed, ifm_speed) do { \
	+ if ((supported) & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_ ## fw_speed) \
	+ ifmedia_add(softc->media, IFM_ETHER \| (ifm_speed), 0, NULL); \
	+} while(0)
	+
	+#define BNXT_MIN_FRAME_SIZE 52 /* Frames must be padded to this size for some A0 chips */
	+
	/* NVRAM access */
	enum bnxt_nvm_directory_type {
	BNX_DIR_TYPE_UNUSED = 0,
	@@ -264,6 +299,12 @@
	int rid;
	};

	+struct bnxt_flow_ctrl {
	+ bool rx;
	+ bool tx;
	+ bool autoneg;
	+};
	+
	struct bnxt_link_info {
	uint8_t media_type;
	uint8_t transceiver;
	@@ -275,10 +316,8 @@
	uint8_t last_link_up;
	uint8_t duplex;
	uint8_t last_duplex;
	- uint8_t pause;
	- uint8_t last_pause;
	- uint8_t auto_pause;
	- uint8_t force_pause;
	+ struct bnxt_flow_ctrl flow_ctrl;
	+ struct bnxt_flow_ctrl last_flow_ctrl;
	uint8_t duplex_setting;
	uint8_t auto_mode;
	#define PHY_VER_LEN 3
	@@ -296,7 +335,6 @@
	#define BNXT_AUTONEG_SPEED 1
	#define BNXT_AUTONEG_FLOW_CTRL 2
	uint8_t req_duplex;
	- uint8_t req_flow_ctrl;
	uint16_t req_link_speed;
	};

	@@ -370,7 +408,6 @@
	bus_addr_t hwrm_cmd_req_dma_addr;
	};

	-#define BNXT_FLAG_VF (1<<1)

	#define BNXT_PF(softc) (!((softc)->flags & BNXT_FLAG_VF))
	#define BNXT_VF(softc) ((softc)->flags & BNXT_FLAG_VF)
	@@ -427,6 +464,7 @@
	uint32_t ring_size; /* Must be a power of two */
	uint16_t id; /* Logical ID */
	uint16_t phys_id;
	+ struct bnxt_full_tpa_start *tpa_start;
	};

	struct bnxt_cp_ring {
	@@ -495,6 +533,21 @@
	struct sysctl_oid *nvm_oid;
	};

	+struct bnxt_func_qcfg {
	+ uint16_t alloc_completion_rings;
	+ uint16_t alloc_tx_rings;
	+ uint16_t alloc_rx_rings;
	+ uint16_t alloc_vnics;
	+};
	+
	+struct bnxt_hw_lro {
	+ uint16_t enable;
	+ uint16_t is_mode_gro;
	+ uint16_t max_agg_segs;
	+ uint16_t max_aggs;
	+ uint32_t min_agg_len;
	+};
	+
	struct bnxt_softc {
	device_t dev;
	if_ctx_t ctx;
	@@ -505,17 +558,22 @@
	struct bnxt_bar_info hwrm_bar;
	struct bnxt_bar_info doorbell_bar;
	struct bnxt_link_info link_info;
	-#define BNXT_FLAG_NPAR 1
	+#define BNXT_FLAG_VF 0x0001
	+#define BNXT_FLAG_NPAR 0x0002
	+#define BNXT_FLAG_WOL_CAP 0x0004
	+#define BNXT_FLAG_SHORT_CMD 0x0008
	uint32_t flags;
	uint32_t total_msix;

	struct bnxt_func_info func;
	+ struct bnxt_func_qcfg fn_qcfg;
	struct bnxt_pf_info pf;
	struct bnxt_vf_info vf;

	uint16_t hwrm_cmd_seq;
	uint32_t hwrm_cmd_timeo; /* milliseconds */
	struct iflib_dma_info hwrm_cmd_resp;
	+ struct iflib_dma_info hwrm_short_cmd_req_addr;
	/* Interrupt info for HWRM */
	struct if_irq irq;
	struct mtx hwrm_lock;
	@@ -525,6 +583,7 @@
	uint8_t max_tc;
	struct bnxt_cos_queue q_info[BNXT_MAX_QUEUE];

	+ uint64_t admin_ticks;
	struct iflib_dma_info hw_rx_port_stats;
	struct iflib_dma_info hw_tx_port_stats;
	struct rx_port_stats *rx_port_stats;
	@@ -551,10 +610,30 @@

	struct sysctl_ctx_list hw_stats;
	struct sysctl_oid *hw_stats_oid;
	+ struct sysctl_ctx_list hw_lro_ctx;
	+ struct sysctl_oid *hw_lro_oid;
	+ struct sysctl_ctx_list flow_ctrl_ctx;
	+ struct sysctl_oid *flow_ctrl_oid;

	- struct bnxt_full_tpa_start *tpa_start;
	struct bnxt_ver_info *ver_info;
	struct bnxt_nvram_info *nvm_info;
	+ bool wol;
	+ struct bnxt_hw_lro hw_lro;
	+ uint8_t wol_filter_id;
	+ uint16_t rx_coal_usecs;
	+ uint16_t rx_coal_usecs_irq;
	+ uint16_t rx_coal_frames;
	+ uint16_t rx_coal_frames_irq;
	+ uint16_t tx_coal_usecs;
	+ uint16_t tx_coal_usecs_irq;
	+ uint16_t tx_coal_frames;
	+ uint16_t tx_coal_frames_irq;
	+
	+#define BNXT_USEC_TO_COAL_TIMER(x) ((x) * 25 / 2)
	+#define BNXT_DEF_STATS_COAL_TICKS 1000000
	+#define BNXT_MIN_STATS_COAL_TICKS 250000
	+#define BNXT_MAX_STATS_COAL_TICKS 1000000
	+
	};

	struct bnxt_filter_info {
	Index: sys/dev/bnxt/bnxt_hwrm.h
	===================================================================
	--- sys/dev/bnxt/bnxt_hwrm.h
	+++ sys/dev/bnxt/bnxt_hwrm.h
	@@ -32,6 +32,12 @@
	#ifndef _BNXT_HWRM_H
	#define _BNXT_HWRM_H

	+#define BNXT_PAUSE_TX (HWRM_PORT_PHY_QCFG_OUTPUT_PAUSE_TX)
	+#define BNXT_PAUSE_RX (HWRM_PORT_PHY_QCFG_OUTPUT_PAUSE_RX)
	+#define BNXT_AUTO_PAUSE_AUTONEG_PAUSE \
	+ (HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_PAUSE_AUTONEG_PAUSE)
	+#define BNXT_HWRM_SHORT_REQ_LEN sizeof(struct hwrm_short_input)
	+
	/* HWRM Function Prototypes */
	int bnxt_alloc_hwrm_dma_mem(struct bnxt_softc *softc);
	void bnxt_free_hwrm_dma_mem(struct bnxt_softc *softc);
	@@ -43,14 +49,16 @@
	int bnxt_hwrm_func_drv_rgtr(struct bnxt_softc *softc);
	int bnxt_hwrm_func_drv_unrgtr(struct bnxt_softc *softc, bool shutdown);
	int bnxt_hwrm_func_qcaps(struct bnxt_softc *softc);
	+int bnxt_hwrm_func_qcfg(struct bnxt_softc *softc);
	int bnxt_hwrm_func_reset(struct bnxt_softc *softc);
	-int bnxt_hwrm_set_link_setting(struct bnxt_softc *, bool set_pause,
	- bool set_eee);
	+int bnxt_hwrm_set_link_setting(struct bnxt_softc *softc, bool set_pause,
	+ bool set_eee, bool set_link);
	int bnxt_hwrm_set_pause(struct bnxt_softc *softc);
	int bnxt_hwrm_vnic_ctx_alloc(struct bnxt_softc softc, uint16_t ctx_id);
	int bnxt_hwrm_vnic_cfg(struct bnxt_softc softc, struct bnxt_vnic_info vnic);
	int bnxt_hwrm_stat_ctx_alloc(struct bnxt_softc softc, struct bnxt_cp_ring cpr,
	uint64_t paddr);
	+int bnxt_hwrm_port_qstats(struct bnxt_softc *softc);
	int bnxt_hwrm_ring_grp_alloc(struct bnxt_softc *softc,
	struct bnxt_grp_info *grp);
	int bnxt_hwrm_vnic_alloc(struct bnxt_softc softc, struct bnxt_vnic_info vnic);
	@@ -59,9 +67,9 @@
	int bnxt_hwrm_set_filter(struct bnxt_softc softc, struct bnxt_vnic_info vnic);
	int bnxt_hwrm_rss_cfg(struct bnxt_softc softc, struct bnxt_vnic_info vnic,
	uint32_t hash_type);
	-int bnxt_hwrm_func_cfg(struct bnxt_softc *softc);
	-int bnxt_hwrm_vnic_tpa_cfg(struct bnxt_softc *softc,
	- struct bnxt_vnic_info *vnic, uint32_t flags);
	+int bnxt_cfg_async_cr(struct bnxt_softc *softc);
	+int bnxt_hwrm_vnic_tpa_cfg(struct bnxt_softc *softc);
	+void bnxt_validate_hw_lro_settings(struct bnxt_softc *softc);
	int bnxt_hwrm_nvm_find_dir_entry(struct bnxt_softc *softc, uint16_t type,
	uint16_t ordinal, uint16_t ext, uint16_t index, bool use_index,
	uint8_t search_opt, uint32_t data_length, uint32_t item_length,
	@@ -98,5 +106,10 @@
	uint8_t month, uint8_t day, uint8_t hour, uint8_t minute, uint8_t second,
	uint16_t millisecond, uint16_t zone);
	int bnxt_hwrm_port_phy_qcfg(struct bnxt_softc *softc);
	-
	+uint16_t bnxt_hwrm_get_wol_fltrs(struct bnxt_softc *softc, uint16_t handle);
	+int bnxt_hwrm_alloc_wol_fltr(struct bnxt_softc *softc);
	+int bnxt_hwrm_free_wol_fltr(struct bnxt_softc *softc);
	+int bnxt_hwrm_set_coal(struct bnxt_softc *softc);
	+int bnxt_hwrm_func_rgtr_async_events(struct bnxt_softc softc, unsigned long bmap,
	+ int bmap_size);
	#endif
	Index: sys/dev/bnxt/bnxt_hwrm.c
	===================================================================
	--- sys/dev/bnxt/bnxt_hwrm.c
	+++ sys/dev/bnxt/bnxt_hwrm.c
	@@ -30,6 +30,7 @@
	__FBSDID("$FreeBSD$");

	#include <sys/endian.h>
	+#include <sys/bitstring.h>

	#include "bnxt.h"
	#include "bnxt_hwrm.h"
	@@ -121,12 +122,37 @@
	uint16_t cp_ring_id;
	uint8_t *valid;
	uint16_t err;
	+ uint16_t max_req_len = HWRM_MAX_REQ_LEN;
	+ struct hwrm_short_input short_input = {0};

	/* TODO: DMASYNC in here. */
	req->seq_id = htole16(softc->hwrm_cmd_seq++);
	memset(resp, 0, PAGE_SIZE);
	cp_ring_id = le16toh(req->cmpl_ring);

	+ if (softc->flags & BNXT_FLAG_SHORT_CMD) {
	+ void *short_cmd_req = softc->hwrm_short_cmd_req_addr.idi_vaddr;
	+
	+ memcpy(short_cmd_req, req, msg_len);
	+ memset((uint8_t *) short_cmd_req + msg_len, 0, softc->hwrm_max_req_len-
	+ msg_len);
	+
	+ short_input.req_type = req->req_type;
	+ short_input.signature =
	+ htole16(HWRM_SHORT_INPUT_SIGNATURE_SHORT_CMD);
	+ short_input.size = htole16(msg_len);
	+ short_input.req_addr =
	+ htole64(softc->hwrm_short_cmd_req_addr.idi_paddr);
	+
	+ data = (uint32_t *)&short_input;
	+ msg_len = sizeof(short_input);
	+
	+ /* Sync memory write before updating doorbell */
	+ wmb();
	+
	+ max_req_len = BNXT_HWRM_SHORT_REQ_LEN;
	+ }
	+
	/* Write request msg to hwrm channel */
	for (i = 0; i < msg_len; i += 4) {
	bus_space_write_4(softc->hwrm_bar.tag,
	@@ -136,7 +162,7 @@
	}

	/* Clear to the end of the request buffer */
	- for (i = msg_len; i < HWRM_MAX_REQ_LEN; i += 4)
	+ for (i = msg_len; i < max_req_len; i += 4)
	bus_space_write_4(softc->hwrm_bar.tag, softc->hwrm_bar.handle,
	i, 0);

	@@ -247,6 +273,7 @@
	int rc;
	const char nastr[] = "<not installed>";
	const char naver[] = "<N/A>";
	+ uint32_t dev_caps_cfg;

	softc->hwrm_max_req_len = HWRM_MAX_REQ_LEN;
	softc->hwrm_cmd_timeo = 1000;
	@@ -322,6 +349,11 @@
	if (resp->def_req_timeout)
	softc->hwrm_cmd_timeo = le16toh(resp->def_req_timeout);

	+ dev_caps_cfg = le32toh(resp->dev_caps_cfg);
	+ if ((dev_caps_cfg & HWRM_VER_GET_OUTPUT_DEV_CAPS_CFG_SHORT_CMD_SUPPORTED) &&
	+ (dev_caps_cfg & HWRM_VER_GET_OUTPUT_DEV_CAPS_CFG_SHORT_CMD_REQUIRED))
	+ softc->flags \|= BNXT_FLAG_SHORT_CMD;
	+
	fail:
	BNXT_HWRM_UNLOCK(softc);
	return rc;
	@@ -398,6 +430,10 @@
	if (rc)
	goto fail;

	+ if (resp->flags &
	+ htole32(HWRM_FUNC_QCAPS_OUTPUT_FLAGS_WOL_MAGICPKT_SUPPORTED))
	+ softc->flags \|= BNXT_FLAG_WOL_CAP;
	+
	func->fw_fid = le16toh(resp->fid);
	memcpy(func->mac_addr, resp->mac_address, ETHER_ADDR_LEN);
	func->max_rsscos_ctxs = le16toh(resp->max_rsscos_ctx);
	@@ -433,6 +469,31 @@
	return rc;
	}

	+int
	+bnxt_hwrm_func_qcfg(struct bnxt_softc *softc)
	+{
	+ struct hwrm_func_qcfg_input req = {0};
	+ struct hwrm_func_qcfg_output *resp =
	+ (void *)softc->hwrm_cmd_resp.idi_vaddr;
	+ struct bnxt_func_qcfg *fn_qcfg = &softc->fn_qcfg;
	+ int rc;
	+
	+ bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_FUNC_QCFG);
	+ req.fid = htole16(0xffff);
	+ BNXT_HWRM_LOCK(softc);
	+ rc = _hwrm_send_message(softc, &req, sizeof(req));
	+ if (rc)
	+ goto fail;
	+
	+ fn_qcfg->alloc_completion_rings = le16toh(resp->alloc_cmpl_rings);
	+ fn_qcfg->alloc_tx_rings = le16toh(resp->alloc_tx_rings);
	+ fn_qcfg->alloc_rx_rings = le16toh(resp->alloc_rx_rings);
	+ fn_qcfg->alloc_vnics = le16toh(resp->alloc_vnics);
	+fail:
	+ BNXT_HWRM_UNLOCK(softc);
	+ return rc;
	+}
	+
	int
	bnxt_hwrm_func_reset(struct bnxt_softc *softc)
	{
	@@ -473,33 +534,28 @@
	bnxt_hwrm_set_pause_common(struct bnxt_softc *softc,
	struct hwrm_port_phy_cfg_input *req)
	{
	- if (softc->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) {
	+ struct bnxt_link_info *link_info = &softc->link_info;
	+
	+ if (link_info->flow_ctrl.autoneg) {
	req->auto_pause =
	HWRM_PORT_PHY_CFG_INPUT_AUTO_PAUSE_AUTONEG_PAUSE;
	- if (softc->link_info.req_flow_ctrl &
	- HWRM_PORT_PHY_QCFG_OUTPUT_PAUSE_RX)
	+ if (link_info->flow_ctrl.rx)
	req->auto_pause \|=
	HWRM_PORT_PHY_CFG_INPUT_AUTO_PAUSE_RX;
	- if (softc->link_info.req_flow_ctrl &
	- HWRM_PORT_PHY_QCFG_OUTPUT_PAUSE_TX)
	+ if (link_info->flow_ctrl.tx)
	req->auto_pause \|=
	- HWRM_PORT_PHY_CFG_INPUT_AUTO_PAUSE_RX;
	+ HWRM_PORT_PHY_CFG_INPUT_AUTO_PAUSE_TX;
	req->enables \|=
	htole32(HWRM_PORT_PHY_CFG_INPUT_ENABLES_AUTO_PAUSE);
	} else {
	- if (softc->link_info.req_flow_ctrl &
	- HWRM_PORT_PHY_QCFG_OUTPUT_PAUSE_RX)
	+ if (link_info->flow_ctrl.rx)
	req->force_pause \|=
	HWRM_PORT_PHY_CFG_INPUT_FORCE_PAUSE_RX;
	- if (softc->link_info.req_flow_ctrl &
	- HWRM_PORT_PHY_QCFG_OUTPUT_PAUSE_TX)
	+ if (link_info->flow_ctrl.tx)
	req->force_pause \|=
	HWRM_PORT_PHY_CFG_INPUT_FORCE_PAUSE_TX;
	req->enables \|=
	htole32(HWRM_PORT_PHY_CFG_INPUT_ENABLES_FORCE_PAUSE);
	- req->auto_pause = req->force_pause;
	- req->enables \|= htole32(
	- HWRM_PORT_PHY_CFG_INPUT_ENABLES_AUTO_PAUSE);
	}
	}

	@@ -533,49 +589,41 @@

	int
	bnxt_hwrm_set_link_setting(struct bnxt_softc *softc, bool set_pause,
	- bool set_eee)
	+ bool set_eee, bool set_link)
	{
	struct hwrm_port_phy_cfg_input req = {0};
	+ int rc;

	if (softc->flags & BNXT_FLAG_NPAR)
	return ENOTSUP;

	bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_PORT_PHY_CFG);
	- if (set_pause)
	+
	+ if (set_pause) {
	bnxt_hwrm_set_pause_common(softc, &req);

	- bnxt_hwrm_set_link_common(softc, &req);
	- if (set_eee)
	- bnxt_hwrm_set_eee(softc, &req);
	- return hwrm_send_message(softc, &req, sizeof(req));
	-}
	+ if (softc->link_info.flow_ctrl.autoneg)
	+ set_link = true;
	+ }

	-
	-int
	-bnxt_hwrm_set_pause(struct bnxt_softc *softc)
	-{
	- struct hwrm_port_phy_cfg_input req = {0};
	- int rc;
	-
	- if (softc->flags & BNXT_FLAG_NPAR)
	- return ENOTSUP;
	-
	- bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_PORT_PHY_CFG);
	- bnxt_hwrm_set_pause_common(softc, &req);
	-
	- if (softc->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL)
	+ if (set_link)
	bnxt_hwrm_set_link_common(softc, &req);
	-
	+
	+ if (set_eee)
	+ bnxt_hwrm_set_eee(softc, &req);
	+
	BNXT_HWRM_LOCK(softc);
	rc = _hwrm_send_message(softc, &req, sizeof(req));
	- if (!rc && !(softc->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL)) {
	- /* since changing of pause setting doesn't trigger any link
	- * change event, the driver needs to update the current pause
	- * result upon successfully return of the phy_cfg command */
	- softc->link_info.pause =
	- softc->link_info.force_pause = softc->link_info.req_flow_ctrl;
	- softc->link_info.auto_pause = 0;
	- bnxt_report_link(softc);
	+
	+ if (!rc) {
	+ if (set_pause) {
	+ /* since changing of 'force pause' setting doesn't
	+ * trigger any link change event, the driver needs to
	+ * update the current pause result upon successfully i
	+ * return of the phy_cfg command */
	+ if (!softc->link_info.flow_ctrl.autoneg)
	+ bnxt_report_link(softc);
	+ }
	}
	BNXT_HWRM_UNLOCK(softc);
	return rc;
	@@ -790,6 +838,25 @@
	}

	int
	+bnxt_hwrm_port_qstats(struct bnxt_softc *softc)
	+{
	+ struct hwrm_port_qstats_input req = {0};
	+ int rc = 0;
	+
	+ bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_PORT_QSTATS);
	+
	+ req.port_id = htole16(softc->pf.port_id);
	+ req.rx_stat_host_addr = htole64(softc->hw_rx_port_stats.idi_paddr);
	+ req.tx_stat_host_addr = htole64(softc->hw_tx_port_stats.idi_paddr);
	+
	+ BNXT_HWRM_LOCK(softc);
	+ rc = _hwrm_send_message(softc, &req, sizeof(req));
	+ BNXT_HWRM_UNLOCK(softc);
	+
	+ return rc;
	+}
	+
	+int
	bnxt_hwrm_cfa_l2_set_rx_mask(struct bnxt_softc *softc,
	struct bnxt_vnic_info *vnic)
	{
	@@ -900,44 +967,82 @@
	}

	int
	-bnxt_hwrm_func_cfg(struct bnxt_softc *softc)
	+bnxt_cfg_async_cr(struct bnxt_softc *softc)
	{
	- struct hwrm_func_cfg_input req = {0};
	+ int rc = 0;
	+
	+ if (BNXT_PF(softc)) {
	+ struct hwrm_func_cfg_input req = {0};

	- bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_FUNC_CFG);
	+ bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_FUNC_CFG);

	- req.fid = 0xffff;
	- req.enables = htole32(HWRM_FUNC_CFG_INPUT_ENABLES_ASYNC_EVENT_CR);
	+ req.fid = htole16(0xffff);
	+ req.enables = htole32(HWRM_FUNC_CFG_INPUT_ENABLES_ASYNC_EVENT_CR);
	+ req.async_event_cr = htole16(softc->def_cp_ring.ring.phys_id);

	- req.async_event_cr = softc->def_cp_ring.ring.phys_id;
	+ rc = hwrm_send_message(softc, &req, sizeof(req));
	+ }
	+ else {
	+ struct hwrm_func_vf_cfg_input req = {0};

	- return hwrm_send_message(softc, &req, sizeof(req));
	+ bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_FUNC_VF_CFG);
	+
	+ req.enables = htole32(HWRM_FUNC_VF_CFG_INPUT_ENABLES_ASYNC_EVENT_CR);
	+ req.async_event_cr = htole16(softc->def_cp_ring.ring.phys_id);
	+
	+ rc = hwrm_send_message(softc, &req, sizeof(req));
	+ }
	+ return rc;
	}

	+void
	+bnxt_validate_hw_lro_settings(struct bnxt_softc *softc)
	+{
	+ softc->hw_lro.enable = min(softc->hw_lro.enable, 1);
	+
	+ softc->hw_lro.is_mode_gro = min(softc->hw_lro.is_mode_gro, 1);
	+
	+ softc->hw_lro.max_agg_segs = min(softc->hw_lro.max_agg_segs,
	+ HWRM_VNIC_TPA_CFG_INPUT_MAX_AGG_SEGS_MAX);
	+
	+ softc->hw_lro.max_aggs = min(softc->hw_lro.max_aggs,
	+ HWRM_VNIC_TPA_CFG_INPUT_MAX_AGGS_MAX);
	+
	+ softc->hw_lro.min_agg_len = min(softc->hw_lro.min_agg_len, BNXT_MAX_MTU);
	+}
	+
	int
	-bnxt_hwrm_vnic_tpa_cfg(struct bnxt_softc softc, struct bnxt_vnic_info vnic,
	- uint32_t flags)
	+bnxt_hwrm_vnic_tpa_cfg(struct bnxt_softc *softc)
	{
	struct hwrm_vnic_tpa_cfg_input req = {0};
	+ uint32_t flags;

	bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_VNIC_TPA_CFG);

	- req.flags = htole32(flags);
	- req.vnic_id = htole16(vnic->id);
	- req.enables = htole32(HWRM_VNIC_TPA_CFG_INPUT_ENABLES_MAX_AGG_SEGS \|
	- HWRM_VNIC_TPA_CFG_INPUT_ENABLES_MAX_AGGS \|
	- /* HWRM_VNIC_TPA_CFG_INPUT_ENABLES_MAX_AGG_TIMER \| */
	- HWRM_VNIC_TPA_CFG_INPUT_ENABLES_MIN_AGG_LEN);
	- /* TODO: Calculate this based on ring size? */
	- req.max_agg_segs = htole16(3);
	- /* Base this in the allocated TPA start size... */
	- req.max_aggs = htole16(2);
	- /*
	- * TODO: max_agg_timer?
	- * req.mag_agg_timer = htole32(XXX);
	- */
	- req.min_agg_len = htole32(0);
	+ if (softc->hw_lro.enable) {
	+ flags = HWRM_VNIC_TPA_CFG_INPUT_FLAGS_TPA \|
	+ HWRM_VNIC_TPA_CFG_INPUT_FLAGS_ENCAP_TPA \|
	+ HWRM_VNIC_TPA_CFG_INPUT_FLAGS_AGG_WITH_ECN \|
	+ HWRM_VNIC_TPA_CFG_INPUT_FLAGS_AGG_WITH_SAME_GRE_SEQ;
	+
	+ if (softc->hw_lro.is_mode_gro)
	+ flags \|= HWRM_VNIC_TPA_CFG_INPUT_FLAGS_GRO;
	+ else
	+ flags \|= HWRM_VNIC_TPA_CFG_INPUT_FLAGS_RSC_WND_UPDATE;
	+
	+ req.flags = htole32(flags);

	+ req.enables = htole32(HWRM_VNIC_TPA_CFG_INPUT_ENABLES_MAX_AGG_SEGS \|
	+ HWRM_VNIC_TPA_CFG_INPUT_ENABLES_MAX_AGGS \|
	+ HWRM_VNIC_TPA_CFG_INPUT_ENABLES_MIN_AGG_LEN);
	+
	+ req.max_agg_segs = htole16(softc->hw_lro.max_agg_segs);
	+ req.max_aggs = htole16(softc->hw_lro.max_aggs);
	+ req.min_agg_len = htole32(softc->hw_lro.min_agg_len);
	+ }
	+
	+ req.vnic_id = htole16(softc->vnic_info.id);
	+
	return hwrm_send_message(softc, &req, sizeof(req));
	}

	@@ -1448,12 +1553,45 @@
	goto exit;

	link_info->phy_link_status = resp->link;
	- link_info->duplex = resp->duplex;
	- link_info->pause = resp->pause;
	+ link_info->duplex = resp->duplex_cfg;
	link_info->auto_mode = resp->auto_mode;
	- link_info->auto_pause = resp->auto_pause;
	- link_info->force_pause = resp->force_pause;
	- link_info->duplex_setting = resp->duplex;
	+
	+ /*
	+ * When AUTO_PAUSE_AUTONEG_PAUSE bit is set to 1,
	+ * the advertisement of pause is enabled.
	+ * 1. When the auto_mode is not set to none and this flag is set to 1,
	+ * then the auto_pause bits on this port are being advertised and
	+ * autoneg pause results are being interpreted.
	+ * 2. When the auto_mode is not set to none and this flag is set to 0,
	+ * the pause is forced as indicated in force_pause, and also
	+ * advertised as auto_pause bits, but the autoneg results are not
	+ * interpreted since the pause configuration is being forced.
	+ * 3. When the auto_mode is set to none and this flag is set to 1,
	+ * auto_pause bits should be ignored and should be set to 0.
	+ */
	+
	+ link_info->flow_ctrl.autoneg = false;
	+ link_info->flow_ctrl.tx = false;
	+ link_info->flow_ctrl.rx = false;
	+
	+ if ((resp->auto_mode) &&
	+ (resp->auto_pause & BNXT_AUTO_PAUSE_AUTONEG_PAUSE)) {
	+ link_info->flow_ctrl.autoneg = true;
	+ }
	+
	+ if (link_info->flow_ctrl.autoneg) {
	+ if (resp->auto_pause & BNXT_PAUSE_TX)
	+ link_info->flow_ctrl.tx = true;
	+ if (resp->auto_pause & BNXT_PAUSE_RX)
	+ link_info->flow_ctrl.rx = true;
	+ } else {
	+ if (resp->force_pause & BNXT_PAUSE_TX)
	+ link_info->flow_ctrl.tx = true;
	+ if (resp->force_pause & BNXT_PAUSE_RX)
	+ link_info->flow_ctrl.rx = true;
	+ }
	+
	+ link_info->duplex_setting = resp->duplex_cfg;
	if (link_info->phy_link_status == HWRM_PORT_PHY_QCFG_OUTPUT_LINK_LINK)
	link_info->link_speed = le16toh(resp->link_speed);
	else
	@@ -1482,4 +1620,191 @@
	exit:
	BNXT_HWRM_UNLOCK(softc);
	return rc;
	+}
	+
	+uint16_t
	+bnxt_hwrm_get_wol_fltrs(struct bnxt_softc *softc, uint16_t handle)
	+{
	+ struct hwrm_wol_filter_qcfg_input req = {0};
	+ struct hwrm_wol_filter_qcfg_output *resp =
	+ (void *)softc->hwrm_cmd_resp.idi_vaddr;
	+ uint16_t next_handle = 0;
	+ int rc;
	+
	+ bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_WOL_FILTER_QCFG);
	+ req.port_id = htole16(softc->pf.port_id);
	+ req.handle = htole16(handle);
	+ rc = hwrm_send_message(softc, &req, sizeof(req));
	+ if (!rc) {
	+ next_handle = le16toh(resp->next_handle);
	+ if (next_handle != 0) {
	+ if (resp->wol_type ==
	+ HWRM_WOL_FILTER_ALLOC_INPUT_WOL_TYPE_MAGICPKT) {
	+ softc->wol = 1;
	+ softc->wol_filter_id = resp->wol_filter_id;
	+ }
	+ }
	+ }
	+ return next_handle;
	+}
	+
	+int
	+bnxt_hwrm_alloc_wol_fltr(struct bnxt_softc *softc)
	+{
	+ struct hwrm_wol_filter_alloc_input req = {0};
	+ struct hwrm_wol_filter_alloc_output *resp =
	+ (void *)softc->hwrm_cmd_resp.idi_vaddr;
	+ int rc;
	+
	+ bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_WOL_FILTER_ALLOC);
	+ req.port_id = htole16(softc->pf.port_id);
	+ req.wol_type = HWRM_WOL_FILTER_ALLOC_INPUT_WOL_TYPE_MAGICPKT;
	+ req.enables =
	+ htole32(HWRM_WOL_FILTER_ALLOC_INPUT_ENABLES_MAC_ADDRESS);
	+ memcpy(req.mac_address, softc->func.mac_addr, ETHER_ADDR_LEN);
	+ rc = hwrm_send_message(softc, &req, sizeof(req));
	+ if (!rc)
	+ softc->wol_filter_id = resp->wol_filter_id;
	+
	+ return rc;
	+}
	+
	+int
	+bnxt_hwrm_free_wol_fltr(struct bnxt_softc *softc)
	+{
	+ struct hwrm_wol_filter_free_input req = {0};
	+
	+ bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_WOL_FILTER_FREE);
	+ req.port_id = htole16(softc->pf.port_id);
	+ req.enables =
	+ htole32(HWRM_WOL_FILTER_FREE_INPUT_ENABLES_WOL_FILTER_ID);
	+ req.wol_filter_id = softc->wol_filter_id;
	+ return hwrm_send_message(softc, &req, sizeof(req));
	+}
	+
	+static void bnxt_hwrm_set_coal_params(struct bnxt_softc *softc, uint32_t max_frames,
	+ uint32_t buf_tmrs, uint16_t flags,
	+ struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req)
	+{
	+ req->flags = htole16(flags);
	+ req->num_cmpl_dma_aggr = htole16((uint16_t)max_frames);
	+ req->num_cmpl_dma_aggr_during_int = htole16(max_frames >> 16);
	+ req->cmpl_aggr_dma_tmr = htole16((uint16_t)buf_tmrs);
	+ req->cmpl_aggr_dma_tmr_during_int = htole16(buf_tmrs >> 16);
	+ /* Minimum time between 2 interrupts set to buf_tmr x 2 */
	+ req->int_lat_tmr_min = htole16((uint16_t)buf_tmrs * 2);
	+ req->int_lat_tmr_max = htole16((uint16_t)buf_tmrs * 4);
	+ req->num_cmpl_aggr_int = htole16((uint16_t)max_frames * 4);
	+}
	+
	+
	+int bnxt_hwrm_set_coal(struct bnxt_softc *softc)
	+{
	+ int i, rc = 0;
	+ struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req_rx = {0},
	+ req_tx = {0}, *req;
	+ uint16_t max_buf, max_buf_irq;
	+ uint16_t buf_tmr, buf_tmr_irq;
	+ uint32_t flags;
	+
	+ bnxt_hwrm_cmd_hdr_init(softc, &req_rx,
	+ HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS);
	+ bnxt_hwrm_cmd_hdr_init(softc, &req_tx,
	+ HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS);
	+
	+ /* Each rx completion (2 records) should be DMAed immediately.
	+ * DMA 1/4 of the completion buffers at a time.
	+ */
	+ max_buf = min_t(uint16_t, softc->rx_coal_frames / 4, 2);
	+ /* max_buf must not be zero */
	+ max_buf = clamp_t(uint16_t, max_buf, 1, 63);
	+ max_buf_irq = clamp_t(uint16_t, softc->rx_coal_frames_irq, 1, 63);
	+ buf_tmr = BNXT_USEC_TO_COAL_TIMER(softc->rx_coal_usecs);
	+ /* buf timer set to 1/4 of interrupt timer */
	+ buf_tmr = max_t(uint16_t, buf_tmr / 4, 1);
	+ buf_tmr_irq = BNXT_USEC_TO_COAL_TIMER(softc->rx_coal_usecs_irq);
	+ buf_tmr_irq = max_t(uint16_t, buf_tmr_irq, 1);
	+
	+ flags = HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS_INPUT_FLAGS_TIMER_RESET;
	+
	+ /* RING_IDLE generates more IRQs for lower latency. Enable it only
	+ * if coal_usecs is less than 25 us.
	+ */
	+ if (softc->rx_coal_usecs < 25)
	+ flags \|= HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS_INPUT_FLAGS_RING_IDLE;
	+
	+ bnxt_hwrm_set_coal_params(softc, max_buf_irq << 16 \| max_buf,
	+ buf_tmr_irq << 16 \| buf_tmr, flags, &req_rx);
	+
	+ /* max_buf must not be zero */
	+ max_buf = clamp_t(uint16_t, softc->tx_coal_frames, 1, 63);
	+ max_buf_irq = clamp_t(uint16_t, softc->tx_coal_frames_irq, 1, 63);
	+ buf_tmr = BNXT_USEC_TO_COAL_TIMER(softc->tx_coal_usecs);
	+ /* buf timer set to 1/4 of interrupt timer */
	+ buf_tmr = max_t(uint16_t, buf_tmr / 4, 1);
	+ buf_tmr_irq = BNXT_USEC_TO_COAL_TIMER(softc->tx_coal_usecs_irq);
	+ buf_tmr_irq = max_t(uint16_t, buf_tmr_irq, 1);
	+ flags = HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS_INPUT_FLAGS_TIMER_RESET;
	+ bnxt_hwrm_set_coal_params(softc, max_buf_irq << 16 \| max_buf,
	+ buf_tmr_irq << 16 \| buf_tmr, flags, &req_tx);
	+
	+ for (i = 0; i < softc->nrxqsets; i++) {
	+
	+
	+ req = &req_rx;
	+ /*
	+ * TBD:
	+ * Check if Tx also needs to be done
	+ * So far, Tx processing has been done in softirq contest
	+ *
	+ * req = &req_tx;
	+ */
	+ req->ring_id = htole16(softc->grp_info[i].cp_ring_id);
	+
	+ rc = hwrm_send_message(softc, req, sizeof(*req));
	+ if (rc)
	+ break;
	+ }
	+ return rc;
	+}
	+
	+
	+
	+int bnxt_hwrm_func_rgtr_async_events(struct bnxt_softc softc, unsigned long bmap,
	+ int bmap_size)
	+{
	+ struct hwrm_func_drv_rgtr_input req = {0};
	+ bitstr_t *async_events_bmap;
	+ uint32_t *events;
	+ int i;
	+
	+ async_events_bmap = bit_alloc(256, M_DEVBUF, M_WAITOK\|M_ZERO);
	+ events = (uint32_t *)async_events_bmap;
	+
	+ bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_FUNC_DRV_RGTR);
	+
	+ req.enables =
	+ htole32(HWRM_FUNC_DRV_RGTR_INPUT_ENABLES_ASYNC_EVENT_FWD);
	+
	+ memset(async_events_bmap, 0, sizeof(256 / 8));
	+
	+ bit_set(async_events_bmap, HWRM_ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE);
	+ bit_set(async_events_bmap, HWRM_ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD);
	+ bit_set(async_events_bmap, HWRM_ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED);
	+ bit_set(async_events_bmap, HWRM_ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE);
	+ bit_set(async_events_bmap, HWRM_ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE);
	+
	+ if (bmap && bmap_size) {
	+ for (i = 0; i < bmap_size; i++) {
	+ if (bit_test(bmap, i))
	+ bit_set(async_events_bmap, i);
	+ }
	+ }
	+
	+ for (i = 0; i < 8; i++)
	+ req.async_event_fwd[i] \|= htole32(events[i]);
	+
	+ free(async_events_bmap, M_DEVBUF);
	+
	+ return hwrm_send_message(softc, &req, sizeof(req));
	}
	Index: sys/dev/bnxt/bnxt_sysctl.h
	===================================================================
	--- sys/dev/bnxt/bnxt_sysctl.h
	+++ sys/dev/bnxt/bnxt_sysctl.h
	@@ -33,9 +33,12 @@

	int bnxt_init_sysctl_ctx(struct bnxt_softc *softc);
	int bnxt_free_sysctl_ctx(struct bnxt_softc *softc);
	+int bnxt_create_port_stats_sysctls(struct bnxt_softc *softc);
	int bnxt_create_tx_sysctls(struct bnxt_softc *softc, int txr);
	int bnxt_create_rx_sysctls(struct bnxt_softc *softc, int rxr);
	int bnxt_create_ver_sysctls(struct bnxt_softc *softc);
	int bnxt_create_nvram_sysctls(struct bnxt_nvram_info *ni);
	int bnxt_create_config_sysctls_pre(struct bnxt_softc *softc);
	int bnxt_create_config_sysctls_post(struct bnxt_softc *softc);
	+int bnxt_create_hw_lro_sysctls(struct bnxt_softc *softc);
	+int bnxt_create_pause_fc_sysctls(struct bnxt_softc *softc);
	Index: sys/dev/bnxt/bnxt_sysctl.c
	===================================================================
	--- sys/dev/bnxt/bnxt_sysctl.c
	+++ sys/dev/bnxt/bnxt_sysctl.c
	@@ -74,16 +74,38 @@
	return ENOMEM;
	}

	- sysctl_ctx_init(&softc->nvm_info->nvm_ctx);
	+ if (BNXT_PF(softc)) {
	+ sysctl_ctx_init(&softc->nvm_info->nvm_ctx);
	+ ctx = device_get_sysctl_ctx(softc->dev);
	+ softc->nvm_info->nvm_oid = SYSCTL_ADD_NODE(ctx,
	+ SYSCTL_CHILDREN(device_get_sysctl_tree(softc->dev)), OID_AUTO,
	+ "nvram", CTLFLAG_RD, 0, "nvram information");
	+ if (!softc->nvm_info->nvm_oid) {
	+ sysctl_ctx_free(&softc->nvm_info->nvm_ctx);
	+ return ENOMEM;
	+ }
	+ }
	+
	+ sysctl_ctx_init(&softc->hw_lro_ctx);
	ctx = device_get_sysctl_ctx(softc->dev);
	- softc->nvm_info->nvm_oid = SYSCTL_ADD_NODE(ctx,
	+ softc->hw_lro_oid = SYSCTL_ADD_NODE(ctx,
	SYSCTL_CHILDREN(device_get_sysctl_tree(softc->dev)), OID_AUTO,
	- "nvram", CTLFLAG_RD, 0, "nvram information");
	- if (!softc->nvm_info->nvm_oid) {
	- sysctl_ctx_free(&softc->nvm_info->nvm_ctx);
	+ "hw_lro", CTLFLAG_RD, 0, "hardware lro");
	+ if (!softc->hw_lro_oid) {
	+ sysctl_ctx_free(&softc->hw_lro_ctx);
	return ENOMEM;
	}

	+ sysctl_ctx_init(&softc->flow_ctrl_ctx);
	+ ctx = device_get_sysctl_ctx(softc->dev);
	+ softc->flow_ctrl_oid = SYSCTL_ADD_NODE(ctx,
	+ SYSCTL_CHILDREN(device_get_sysctl_tree(softc->dev)), OID_AUTO,
	+ "fc", CTLFLAG_RD, 0, "flow ctrl");
	+ if (!softc->flow_ctrl_oid) {
	+ sysctl_ctx_free(&softc->flow_ctrl_ctx);
	+ return ENOMEM;
	+ }
	+
	return 0;
	}

	@@ -107,14 +129,29 @@
	else
	softc->ver_info->ver_oid = NULL;
	}
	- if (softc->nvm_info->nvm_oid != NULL) {
	+ if (BNXT_PF(softc) && softc->nvm_info->nvm_oid != NULL) {
	orc = sysctl_ctx_free(&softc->nvm_info->nvm_ctx);
	if (orc)
	rc = orc;
	else
	softc->nvm_info->nvm_oid = NULL;
	}
	+ if (softc->hw_lro_oid != NULL) {
	+ orc = sysctl_ctx_free(&softc->hw_lro_ctx);
	+ if (orc)
	+ rc = orc;
	+ else
	+ softc->hw_lro_oid = NULL;
	+ }

	+ if (softc->flow_ctrl_oid != NULL) {
	+ orc = sysctl_ctx_free(&softc->flow_ctrl_ctx);
	+ if (orc)
	+ rc = orc;
	+ else
	+ softc->flow_ctrl_oid = NULL;
	+ }
	+
	return rc;
	}

	@@ -164,6 +201,456 @@
	}

	int
	+bnxt_create_port_stats_sysctls(struct bnxt_softc *softc)
	+{
	+ struct sysctl_oid *oid;
	+ char name[32];
	+ char desc[64];
	+
	+ sprintf(name, "port_stats");
	+ sprintf(desc, "Port Stats");
	+ oid = SYSCTL_ADD_NODE(&softc->hw_stats,
	+ SYSCTL_CHILDREN(softc->hw_stats_oid), OID_AUTO, name, CTLFLAG_RD, 0,
	+ desc);
	+ if (!oid)
	+ return ENOMEM;
	+
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "tx_64b_frames", CTLFLAG_RD,
	+ &softc->tx_port_stats->tx_64b_frames, "Transmitted 64b frames");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "tx_65b_127b_frames", CTLFLAG_RD,
	+ &softc->tx_port_stats->tx_65b_127b_frames,
	+ "Transmitted 65b 127b frames");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "tx_128b_255b_frames", CTLFLAG_RD,
	+ &softc->tx_port_stats->tx_128b_255b_frames,
	+ "Transmitted 128b 255b frames");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "tx_256b_511b_frames", CTLFLAG_RD,
	+ &softc->tx_port_stats->tx_256b_511b_frames,
	+ "Transmitted 256b 511b frames");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "tx_512b_1023b_frames", CTLFLAG_RD,
	+ &softc->tx_port_stats->tx_512b_1023b_frames,
	+ "Transmitted 512b 1023b frames");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "tx_1024b_1518_frames", CTLFLAG_RD,
	+ &softc->tx_port_stats->tx_1024b_1518_frames,
	+ "Transmitted 1024b 1518 frames");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "tx_good_vlan_frames", CTLFLAG_RD,
	+ &softc->tx_port_stats->tx_good_vlan_frames,
	+ "Transmitted good vlan frames");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "tx_1519b_2047_frames", CTLFLAG_RD,
	+ &softc->tx_port_stats->tx_1519b_2047_frames,
	+ "Transmitted 1519b 2047 frames");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "tx_2048b_4095b_frames", CTLFLAG_RD,
	+ &softc->tx_port_stats->tx_2048b_4095b_frames,
	+ "Transmitted 2048b 4095b frames");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "tx_4096b_9216b_frames", CTLFLAG_RD,
	+ &softc->tx_port_stats->tx_4096b_9216b_frames,
	+ "Transmitted 4096b 9216b frames");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "tx_9217b_16383b_frames", CTLFLAG_RD,
	+ &softc->tx_port_stats->tx_9217b_16383b_frames,
	+ "Transmitted 9217b 16383b frames");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "tx_good_frames", CTLFLAG_RD,
	+ &softc->tx_port_stats->tx_good_frames, "Transmitted good frames");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "tx_total_frames", CTLFLAG_RD,
	+ &softc->tx_port_stats->tx_total_frames, "Transmitted total frames");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "tx_ucast_frames", CTLFLAG_RD,
	+ &softc->tx_port_stats->tx_ucast_frames, "Transmitted ucast frames");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "tx_mcast_frames", CTLFLAG_RD,
	+ &softc->tx_port_stats->tx_mcast_frames, "Transmitted mcast frames");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "tx_bcast_frames", CTLFLAG_RD,
	+ &softc->tx_port_stats->tx_bcast_frames, "Transmitted bcast frames");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "tx_pause_frames", CTLFLAG_RD,
	+ &softc->tx_port_stats->tx_pause_frames, "Transmitted pause frames");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "tx_pfc_frames", CTLFLAG_RD,
	+ &softc->tx_port_stats->tx_pfc_frames, "Transmitted pfc frames");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "tx_jabber_frames", CTLFLAG_RD,
	+ &softc->tx_port_stats->tx_jabber_frames, "Transmitted jabber frames");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "tx_fcs_err_frames", CTLFLAG_RD,
	+ &softc->tx_port_stats->tx_fcs_err_frames,
	+ "Transmitted fcs err frames");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "tx_control_frames", CTLFLAG_RD,
	+ &softc->tx_port_stats->tx_control_frames,
	+ "Transmitted control frames");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "tx_oversz_frames", CTLFLAG_RD,
	+ &softc->tx_port_stats->tx_oversz_frames, "Transmitted oversz frames");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "tx_single_dfrl_frames", CTLFLAG_RD,
	+ &softc->tx_port_stats->tx_single_dfrl_frames,
	+ "Transmitted single dfrl frames");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "tx_multi_dfrl_frames", CTLFLAG_RD,
	+ &softc->tx_port_stats->tx_multi_dfrl_frames,
	+ "Transmitted multi dfrl frames");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "tx_single_coll_frames", CTLFLAG_RD,
	+ &softc->tx_port_stats->tx_single_coll_frames,
	+ "Transmitted single coll frames");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "tx_multi_coll_frames", CTLFLAG_RD,
	+ &softc->tx_port_stats->tx_multi_coll_frames,
	+ "Transmitted multi coll frames");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "tx_late_coll_frames", CTLFLAG_RD,
	+ &softc->tx_port_stats->tx_late_coll_frames,
	+ "Transmitted late coll frames");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "tx_excessive_coll_frames", CTLFLAG_RD,
	+ &softc->tx_port_stats->tx_excessive_coll_frames,
	+ "Transmitted excessive coll frames");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "tx_frag_frames", CTLFLAG_RD,
	+ &softc->tx_port_stats->tx_frag_frames, "Transmitted frag frames");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "tx_err", CTLFLAG_RD,
	+ &softc->tx_port_stats->tx_err, "Transmitted err");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "tx_tagged_frames", CTLFLAG_RD,
	+ &softc->tx_port_stats->tx_tagged_frames, "Transmitted tagged frames");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "tx_dbl_tagged_frames", CTLFLAG_RD,
	+ &softc->tx_port_stats->tx_dbl_tagged_frames,
	+ "Transmitted dbl tagged frames");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "tx_runt_frames", CTLFLAG_RD,
	+ &softc->tx_port_stats->tx_runt_frames, "Transmitted runt frames");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "tx_fifo_underruns", CTLFLAG_RD,
	+ &softc->tx_port_stats->tx_fifo_underruns,
	+ "Transmitted fifo underruns");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "tx_pfc_ena_frames_pri0", CTLFLAG_RD,
	+ &softc->tx_port_stats->tx_pfc_ena_frames_pri0,
	+ "Transmitted pfc ena frames pri0");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "tx_pfc_ena_frames_pri1", CTLFLAG_RD,
	+ &softc->tx_port_stats->tx_pfc_ena_frames_pri1,
	+ "Transmitted pfc ena frames pri1");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "tx_pfc_ena_frames_pri2", CTLFLAG_RD,
	+ &softc->tx_port_stats->tx_pfc_ena_frames_pri2,
	+ "Transmitted pfc ena frames pri2");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "tx_pfc_ena_frames_pri3", CTLFLAG_RD,
	+ &softc->tx_port_stats->tx_pfc_ena_frames_pri3,
	+ "Transmitted pfc ena frames pri3");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "tx_pfc_ena_frames_pri4", CTLFLAG_RD,
	+ &softc->tx_port_stats->tx_pfc_ena_frames_pri4,
	+ "Transmitted pfc ena frames pri4");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "tx_pfc_ena_frames_pri5", CTLFLAG_RD,
	+ &softc->tx_port_stats->tx_pfc_ena_frames_pri5,
	+ "Transmitted pfc ena frames pri5");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "tx_pfc_ena_frames_pri6", CTLFLAG_RD,
	+ &softc->tx_port_stats->tx_pfc_ena_frames_pri6,
	+ "Transmitted pfc ena frames pri6");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "tx_pfc_ena_frames_pri7", CTLFLAG_RD,
	+ &softc->tx_port_stats->tx_pfc_ena_frames_pri7,
	+ "Transmitted pfc ena frames pri7");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "tx_eee_lpi_events", CTLFLAG_RD,
	+ &softc->tx_port_stats->tx_eee_lpi_events,
	+ "Transmitted eee lpi events");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "tx_eee_lpi_duration", CTLFLAG_RD,
	+ &softc->tx_port_stats->tx_eee_lpi_duration,
	+ "Transmitted eee lpi duration");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "tx_llfc_logical_msgs", CTLFLAG_RD,
	+ &softc->tx_port_stats->tx_llfc_logical_msgs,
	+ "Transmitted llfc logical msgs");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "tx_hcfc_msgs", CTLFLAG_RD,
	+ &softc->tx_port_stats->tx_hcfc_msgs, "Transmitted hcfc msgs");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "tx_total_collisions", CTLFLAG_RD,
	+ &softc->tx_port_stats->tx_total_collisions,
	+ "Transmitted total collisions");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "tx_bytes", CTLFLAG_RD,
	+ &softc->tx_port_stats->tx_bytes, "Transmitted bytes");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "tx_xthol_frames", CTLFLAG_RD,
	+ &softc->tx_port_stats->tx_xthol_frames, "Transmitted xthol frames");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "tx_stat_discard", CTLFLAG_RD,
	+ &softc->tx_port_stats->tx_stat_discard, "Transmitted stat discard");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "tx_stat_error", CTLFLAG_RD,
	+ &softc->tx_port_stats->tx_stat_error, "Transmitted stat error");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "rx_64b_frames", CTLFLAG_RD,
	+ &softc->rx_port_stats->rx_64b_frames, "Received 64b frames");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "rx_65b_127b_frames", CTLFLAG_RD,
	+ &softc->rx_port_stats->rx_65b_127b_frames, "Received 65b 127b frames");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "rx_128b_255b_frames", CTLFLAG_RD,
	+ &softc->rx_port_stats->rx_128b_255b_frames,
	+ "Received 128b 255b frames");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "rx_256b_511b_frames", CTLFLAG_RD,
	+ &softc->rx_port_stats->rx_256b_511b_frames,
	+ "Received 256b 511b frames");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "rx_512b_1023b_frames", CTLFLAG_RD,
	+ &softc->rx_port_stats->rx_512b_1023b_frames,
	+ "Received 512b 1023b frames");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "rx_1024b_1518_frames", CTLFLAG_RD,
	+ &softc->rx_port_stats->rx_1024b_1518_frames,
	+ "Received 1024b 1518 frames");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "rx_good_vlan_frames", CTLFLAG_RD,
	+ &softc->rx_port_stats->rx_good_vlan_frames,
	+ "Received good vlan frames");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "rx_1519b_2047b_frames", CTLFLAG_RD,
	+ &softc->rx_port_stats->rx_1519b_2047b_frames,
	+ "Received 1519b 2047b frames");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "rx_2048b_4095b_frames", CTLFLAG_RD,
	+ &softc->rx_port_stats->rx_2048b_4095b_frames,
	+ "Received 2048b 4095b frames");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "rx_4096b_9216b_frames", CTLFLAG_RD,
	+ &softc->rx_port_stats->rx_4096b_9216b_frames,
	+ "Received 4096b 9216b frames");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "rx_9217b_16383b_frames", CTLFLAG_RD,
	+ &softc->rx_port_stats->rx_9217b_16383b_frames,
	+ "Received 9217b 16383b frames");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "rx_total_frames", CTLFLAG_RD,
	+ &softc->rx_port_stats->rx_total_frames, "Received total frames");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "rx_ucast_frames", CTLFLAG_RD,
	+ &softc->rx_port_stats->rx_ucast_frames, "Received ucast frames");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "rx_mcast_frames", CTLFLAG_RD,
	+ &softc->rx_port_stats->rx_mcast_frames, "Received mcast frames");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "rx_bcast_frames", CTLFLAG_RD,
	+ &softc->rx_port_stats->rx_bcast_frames, "Received bcast frames");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "rx_fcs_err_frames", CTLFLAG_RD,
	+ &softc->rx_port_stats->rx_fcs_err_frames, "Received fcs err frames");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "rx_ctrl_frames", CTLFLAG_RD,
	+ &softc->rx_port_stats->rx_ctrl_frames, "Received ctrl frames");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "rx_pause_frames", CTLFLAG_RD,
	+ &softc->rx_port_stats->rx_pause_frames, "Received pause frames");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "rx_pfc_frames", CTLFLAG_RD,
	+ &softc->rx_port_stats->rx_pfc_frames, "Received pfc frames");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "rx_unsupported_opcode_frames", CTLFLAG_RD,
	+ &softc->rx_port_stats->rx_unsupported_opcode_frames,
	+ "Received unsupported opcode frames");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "rx_unsupported_da_pausepfc_frames", CTLFLAG_RD,
	+ &softc->rx_port_stats->rx_unsupported_da_pausepfc_frames,
	+ "Received unsupported da pausepfc frames");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "rx_wrong_sa_frames", CTLFLAG_RD,
	+ &softc->rx_port_stats->rx_wrong_sa_frames,
	+ "Received wrong sa frames");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "rx_align_err_frames", CTLFLAG_RD,
	+ &softc->rx_port_stats->rx_align_err_frames,
	+ "Received align err frames");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "rx_oor_len_frames", CTLFLAG_RD,
	+ &softc->rx_port_stats->rx_oor_len_frames,
	+ "Received oor len frames");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "rx_code_err_frames", CTLFLAG_RD,
	+ &softc->rx_port_stats->rx_code_err_frames,
	+ "Received code err frames");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "rx_false_carrier_frames", CTLFLAG_RD,
	+ &softc->rx_port_stats->rx_false_carrier_frames,
	+ "Received false carrier frames");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "rx_ovrsz_frames", CTLFLAG_RD,
	+ &softc->rx_port_stats->rx_ovrsz_frames,
	+ "Received ovrsz frames");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "rx_jbr_frames", CTLFLAG_RD,
	+ &softc->rx_port_stats->rx_jbr_frames,
	+ "Received jbr frames");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "rx_mtu_err_frames", CTLFLAG_RD,
	+ &softc->rx_port_stats->rx_mtu_err_frames,
	+ "Received mtu err frames");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "rx_match_crc_frames", CTLFLAG_RD,
	+ &softc->rx_port_stats->rx_match_crc_frames,
	+ "Received match crc frames");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "rx_promiscuous_frames", CTLFLAG_RD,
	+ &softc->rx_port_stats->rx_promiscuous_frames,
	+ "Received promiscuous frames");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "rx_tagged_frames", CTLFLAG_RD,
	+ &softc->rx_port_stats->rx_tagged_frames,
	+ "Received tagged frames");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "rx_double_tagged_frames", CTLFLAG_RD,
	+ &softc->rx_port_stats->rx_double_tagged_frames,
	+ "Received double tagged frames");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "rx_trunc_frames", CTLFLAG_RD,
	+ &softc->rx_port_stats->rx_trunc_frames,
	+ "Received trunc frames");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "rx_good_frames", CTLFLAG_RD,
	+ &softc->rx_port_stats->rx_good_frames,
	+ "Received good frames");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "rx_pfc_xon2xoff_frames_pri0", CTLFLAG_RD,
	+ &softc->rx_port_stats->rx_pfc_xon2xoff_frames_pri0,
	+ "Received pfc xon2xoff frames pri0");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "rx_pfc_xon2xoff_frames_pri1", CTLFLAG_RD,
	+ &softc->rx_port_stats->rx_pfc_xon2xoff_frames_pri1,
	+ "Received pfc xon2xoff frames pri1");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "rx_pfc_xon2xoff_frames_pri2", CTLFLAG_RD,
	+ &softc->rx_port_stats->rx_pfc_xon2xoff_frames_pri2,
	+ "Received pfc xon2xoff frames pri2");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "rx_pfc_xon2xoff_frames_pri3", CTLFLAG_RD,
	+ &softc->rx_port_stats->rx_pfc_xon2xoff_frames_pri3,
	+ "Received pfc xon2xoff frames pri3");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "rx_pfc_xon2xoff_frames_pri4", CTLFLAG_RD,
	+ &softc->rx_port_stats->rx_pfc_xon2xoff_frames_pri4,
	+ "Received pfc xon2xoff frames pri4");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "rx_pfc_xon2xoff_frames_pri5", CTLFLAG_RD,
	+ &softc->rx_port_stats->rx_pfc_xon2xoff_frames_pri5,
	+ "Received pfc xon2xoff frames pri5");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "rx_pfc_xon2xoff_frames_pri6", CTLFLAG_RD,
	+ &softc->rx_port_stats->rx_pfc_xon2xoff_frames_pri6,
	+ "Received pfc xon2xoff frames pri6");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "rx_pfc_xon2xoff_frames_pri7", CTLFLAG_RD,
	+ &softc->rx_port_stats->rx_pfc_xon2xoff_frames_pri7,
	+ "Received pfc xon2xoff frames pri7");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "rx_pfc_ena_frames_pri0", CTLFLAG_RD,
	+ &softc->rx_port_stats->rx_pfc_ena_frames_pri0,
	+ "Received pfc ena frames pri0");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "rx_pfc_ena_frames_pri1", CTLFLAG_RD,
	+ &softc->rx_port_stats->rx_pfc_ena_frames_pri1,
	+ "Received pfc ena frames pri1");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "rx_pfc_ena_frames_pri2", CTLFLAG_RD,
	+ &softc->rx_port_stats->rx_pfc_ena_frames_pri2,
	+ "Received pfc ena frames pri2");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "rx_pfc_ena_frames_pri3", CTLFLAG_RD,
	+ &softc->rx_port_stats->rx_pfc_ena_frames_pri3,
	+ "Received pfc ena frames pri3");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "rx_pfc_ena_frames_pri4", CTLFLAG_RD,
	+ &softc->rx_port_stats->rx_pfc_ena_frames_pri4,
	+ "Received pfc ena frames pri4");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "rx_pfc_ena_frames_pri5", CTLFLAG_RD,
	+ &softc->rx_port_stats->rx_pfc_ena_frames_pri5,
	+ "Received pfc ena frames pri5");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "rx_pfc_ena_frames_pri6", CTLFLAG_RD,
	+ &softc->rx_port_stats->rx_pfc_ena_frames_pri6,
	+ "Received pfc ena frames pri6");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "rx_pfc_ena_frames_pri7", CTLFLAG_RD,
	+ &softc->rx_port_stats->rx_pfc_ena_frames_pri7,
	+ "Received pfc ena frames pri7");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "rx_sch_crc_err_frames", CTLFLAG_RD,
	+ &softc->rx_port_stats->rx_sch_crc_err_frames,
	+ "Received sch crc err frames");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "rx_undrsz_frames", CTLFLAG_RD,
	+ &softc->rx_port_stats->rx_undrsz_frames, "Received undrsz frames");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "rx_frag_frames", CTLFLAG_RD,
	+ &softc->rx_port_stats->rx_frag_frames, "Received frag frames");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "rx_eee_lpi_events", CTLFLAG_RD,
	+ &softc->rx_port_stats->rx_eee_lpi_events, "Received eee lpi events");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "rx_eee_lpi_duration", CTLFLAG_RD,
	+ &softc->rx_port_stats->rx_eee_lpi_duration,
	+ "Received eee lpi duration");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "rx_llfc_physical_msgs", CTLFLAG_RD,
	+ &softc->rx_port_stats->rx_llfc_physical_msgs,
	+ "Received llfc physical msgs");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "rx_llfc_logical_msgs", CTLFLAG_RD,
	+ &softc->rx_port_stats->rx_llfc_logical_msgs,
	+ "Received llfc logical msgs");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "rx_llfc_msgs_with_crc_err", CTLFLAG_RD,
	+ &softc->rx_port_stats->rx_llfc_msgs_with_crc_err,
	+ "Received llfc msgs with crc err");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "rx_hcfc_msgs", CTLFLAG_RD,
	+ &softc->rx_port_stats->rx_hcfc_msgs, "Received hcfc msgs");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "rx_hcfc_msgs_with_crc_err", CTLFLAG_RD,
	+ &softc->rx_port_stats->rx_hcfc_msgs_with_crc_err,
	+ "Received hcfc msgs with crc err");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "rx_bytes", CTLFLAG_RD,
	+ &softc->rx_port_stats->rx_bytes, "Received bytes");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "rx_runt_bytes", CTLFLAG_RD,
	+ &softc->rx_port_stats->rx_runt_bytes, "Received runt bytes");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "rx_runt_frames", CTLFLAG_RD,
	+ &softc->rx_port_stats->rx_runt_frames, "Received runt frames");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "rx_stat_discard", CTLFLAG_RD,
	+ &softc->rx_port_stats->rx_stat_discard, "Received stat discard");
	+ SYSCTL_ADD_QUAD(&softc->hw_stats, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "rx_stat_err", CTLFLAG_RD,
	+ &softc->rx_port_stats->rx_stat_err, "Received stat err");
	+
	+ return 0;
	+}
	+
	+
	+int
	bnxt_create_rx_sysctls(struct bnxt_softc *softc, int rxr)
	{
	struct sysctl_oid *oid;
	@@ -549,6 +1036,166 @@
	return rc;
	}

	+static int
	+bnxt_set_coal_rx_usecs(SYSCTL_HANDLER_ARGS) {
	+ struct bnxt_softc *softc = arg1;
	+ int rc;
	+ int val;
	+
	+ if (softc == NULL)
	+ return EBUSY;
	+
	+ val = softc->rx_coal_usecs;
	+ rc = sysctl_handle_int(oidp, &val, 0, req);
	+ if (rc \|\| !req->newptr)
	+ return rc;
	+
	+ softc->rx_coal_usecs = val;
	+ rc = bnxt_hwrm_set_coal(softc);
	+
	+ return rc;
	+}
	+
	+static int
	+bnxt_set_coal_rx_frames(SYSCTL_HANDLER_ARGS) {
	+ struct bnxt_softc *softc = arg1;
	+ int rc;
	+ int val;
	+
	+ if (softc == NULL)
	+ return EBUSY;
	+
	+ val = softc->rx_coal_frames;
	+ rc = sysctl_handle_int(oidp, &val, 0, req);
	+ if (rc \|\| !req->newptr)
	+ return rc;
	+
	+ softc->rx_coal_frames = val;
	+ rc = bnxt_hwrm_set_coal(softc);
	+
	+ return rc;
	+}
	+
	+static int
	+bnxt_set_coal_rx_usecs_irq(SYSCTL_HANDLER_ARGS) {
	+ struct bnxt_softc *softc = arg1;
	+ int rc;
	+ int val;
	+
	+ if (softc == NULL)
	+ return EBUSY;
	+
	+ val = softc->rx_coal_usecs_irq;
	+ rc = sysctl_handle_int(oidp, &val, 0, req);
	+ if (rc \|\| !req->newptr)
	+ return rc;
	+
	+ softc->rx_coal_usecs_irq = val;
	+ rc = bnxt_hwrm_set_coal(softc);
	+
	+ return rc;
	+}
	+
	+static int
	+bnxt_set_coal_rx_frames_irq(SYSCTL_HANDLER_ARGS) {
	+ struct bnxt_softc *softc = arg1;
	+ int rc;
	+ int val;
	+
	+ if (softc == NULL)
	+ return EBUSY;
	+
	+ val = softc->rx_coal_frames_irq;
	+ rc = sysctl_handle_int(oidp, &val, 0, req);
	+ if (rc \|\| !req->newptr)
	+ return rc;
	+
	+ softc->rx_coal_frames_irq = val;
	+ rc = bnxt_hwrm_set_coal(softc);
	+
	+ return rc;
	+}
	+
	+static int
	+bnxt_set_coal_tx_usecs(SYSCTL_HANDLER_ARGS) {
	+ struct bnxt_softc *softc = arg1;
	+ int rc;
	+ int val;
	+
	+ if (softc == NULL)
	+ return EBUSY;
	+
	+ val = softc->tx_coal_usecs;
	+ rc = sysctl_handle_int(oidp, &val, 0, req);
	+ if (rc \|\| !req->newptr)
	+ return rc;
	+
	+ softc->tx_coal_usecs = val;
	+ rc = bnxt_hwrm_set_coal(softc);
	+
	+ return rc;
	+}
	+
	+static int
	+bnxt_set_coal_tx_frames(SYSCTL_HANDLER_ARGS) {
	+ struct bnxt_softc *softc = arg1;
	+ int rc;
	+ int val;
	+
	+ if (softc == NULL)
	+ return EBUSY;
	+
	+ val = softc->tx_coal_frames;
	+ rc = sysctl_handle_int(oidp, &val, 0, req);
	+ if (rc \|\| !req->newptr)
	+ return rc;
	+
	+ softc->tx_coal_frames = val;
	+ rc = bnxt_hwrm_set_coal(softc);
	+
	+ return rc;
	+}
	+
	+static int
	+bnxt_set_coal_tx_usecs_irq(SYSCTL_HANDLER_ARGS) {
	+ struct bnxt_softc *softc = arg1;
	+ int rc;
	+ int val;
	+
	+ if (softc == NULL)
	+ return EBUSY;
	+
	+ val = softc->tx_coal_usecs_irq;
	+ rc = sysctl_handle_int(oidp, &val, 0, req);
	+ if (rc \|\| !req->newptr)
	+ return rc;
	+
	+ softc->tx_coal_usecs_irq = val;
	+ rc = bnxt_hwrm_set_coal(softc);
	+
	+ return rc;
	+}
	+
	+static int
	+bnxt_set_coal_tx_frames_irq(SYSCTL_HANDLER_ARGS) {
	+ struct bnxt_softc *softc = arg1;
	+ int rc;
	+ int val;
	+
	+ if (softc == NULL)
	+ return EBUSY;
	+
	+ val = softc->tx_coal_frames_irq;
	+ rc = sysctl_handle_int(oidp, &val, 0, req);
	+ if (rc \|\| !req->newptr)
	+ return rc;
	+
	+ softc->tx_coal_frames_irq = val;
	+ rc = bnxt_hwrm_set_coal(softc);
	+
	+ return rc;
	+}
	+
	int
	bnxt_create_config_sysctls_pre(struct bnxt_softc *softc)
	{
	@@ -572,9 +1219,158 @@
	SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "if_name", CTLFLAG_RD,
	iflib_get_ifp(softc->ctx)->if_xname, 0, "interface name");

	+ SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "intr_coal_rx_usecs",
	+ CTLTYPE_INT\|CTLFLAG_RWTUN, softc, 0, bnxt_set_coal_rx_usecs,
	+ "I", "interrupt coalescing Rx Usecs");
	+ SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "intr_coal_rx_frames",
	+ CTLTYPE_INT\|CTLFLAG_RWTUN, softc, 0, bnxt_set_coal_rx_frames,
	+ "I", "interrupt coalescing Rx Frames");
	+ SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "intr_coal_rx_usecs_irq",
	+ CTLTYPE_INT\|CTLFLAG_RWTUN, softc, 0, bnxt_set_coal_rx_usecs_irq,
	+ "I", "interrupt coalescing Rx Usecs IRQ");
	+ SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "intr_coal_rx_frames_irq",
	+ CTLTYPE_INT\|CTLFLAG_RWTUN, softc, 0, bnxt_set_coal_rx_frames_irq,
	+ "I", "interrupt coalescing Rx Frames IRQ");
	+ SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "intr_coal_tx_usecs",
	+ CTLTYPE_INT\|CTLFLAG_RWTUN, softc, 0, bnxt_set_coal_tx_usecs,
	+ "I", "interrupt coalescing Tx Usces");
	+ SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "intr_coal_tx_frames",
	+ CTLTYPE_INT\|CTLFLAG_RWTUN, softc, 0, bnxt_set_coal_tx_frames,
	+ "I", "interrupt coalescing Tx Frames");
	+ SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "intr_coal_tx_usecs_irq",
	+ CTLTYPE_INT\|CTLFLAG_RWTUN, softc, 0, bnxt_set_coal_tx_usecs_irq,
	+ "I", "interrupt coalescing Tx Usecs IRQ");
	+ SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "intr_coal_tx_frames_irq",
	+ CTLTYPE_INT\|CTLFLAG_RWTUN, softc, 0, bnxt_set_coal_tx_frames_irq,
	+ "I", "interrupt coalescing Tx Frames IRQ");
	+
	return 0;
	}

	+#define BNXT_HW_LRO_FN(fn_name, arg) \
	+static int \
	+fn_name(SYSCTL_HANDLER_ARGS) { \
	+ struct bnxt_softc *softc = arg1; \
	+ int rc; \
	+ int val; \
	+ \
	+ if (softc == NULL) \
	+ return EBUSY; \
	+ \
	+ val = softc->hw_lro.arg; \
	+ rc = sysctl_handle_int(oidp, &val, 0, req); \
	+ if (rc \|\| !req->newptr) \
	+ return rc; \
	+ \
	+ if ((if_getdrvflags(iflib_get_ifp(softc->ctx)) & IFF_DRV_RUNNING)) \
	+ return EBUSY; \
	+ \
	+ softc->hw_lro.arg = val; \
	+ bnxt_validate_hw_lro_settings(softc); \
	+ rc = bnxt_hwrm_vnic_tpa_cfg(softc); \
	+ \
	+ return rc; \
	+}
	+
	+BNXT_HW_LRO_FN(bnxt_hw_lro_enable_disable, enable)
	+BNXT_HW_LRO_FN(bnxt_hw_lro_set_mode, is_mode_gro)
	+BNXT_HW_LRO_FN(bnxt_hw_lro_set_max_agg_segs, max_agg_segs)
	+BNXT_HW_LRO_FN(bnxt_hw_lro_set_max_aggs, max_aggs)
	+BNXT_HW_LRO_FN(bnxt_hw_lro_set_min_agg_len, min_agg_len)
	+
	+#define BNXT_FLOW_CTRL_FN(fn_name, arg) \
	+static int \
	+fn_name(SYSCTL_HANDLER_ARGS) { \
	+ struct bnxt_softc *softc = arg1; \
	+ int rc; \
	+ int val; \
	+ \
	+ if (softc == NULL) \
	+ return EBUSY; \
	+ \
	+ val = softc->link_info.flow_ctrl.arg; \
	+ rc = sysctl_handle_int(oidp, &val, 0, req); \
	+ if (rc \|\| !req->newptr) \
	+ return rc; \
	+ \
	+ if (val) \
	+ val = 1; \
	+ \
	+ if (softc->link_info.flow_ctrl.arg != val) { \
	+ softc->link_info.flow_ctrl.arg = val; \
	+ rc = bnxt_hwrm_set_link_setting(softc, true, false, false);\
	+ rc = bnxt_hwrm_port_phy_qcfg(softc); \
	+ } \
	+ \
	+ return rc; \
	+}
	+
	+BNXT_FLOW_CTRL_FN(bnxt_flow_ctrl_tx, tx)
	+BNXT_FLOW_CTRL_FN(bnxt_flow_ctrl_rx, rx)
	+BNXT_FLOW_CTRL_FN(bnxt_flow_ctrl_autoneg, autoneg)
	+int
	+bnxt_create_pause_fc_sysctls(struct bnxt_softc *softc)
	+{
	+ struct sysctl_oid *oid = softc->flow_ctrl_oid;
	+
	+ if (!oid)
	+ return ENOMEM;
	+
	+ SYSCTL_ADD_PROC(&softc->flow_ctrl_ctx, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "tx", CTLTYPE_INT\|CTLFLAG_RWTUN, softc, 0,
	+ bnxt_flow_ctrl_tx, "A",
	+ "Enable or Disable Tx Flow Ctrl: 0 / 1");
	+
	+ SYSCTL_ADD_PROC(&softc->flow_ctrl_ctx, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "rx", CTLTYPE_INT\|CTLFLAG_RWTUN, softc, 0,
	+ bnxt_flow_ctrl_rx, "A",
	+ "Enable or Disable Tx Flow Ctrl: 0 / 1");
	+
	+ SYSCTL_ADD_PROC(&softc->flow_ctrl_ctx, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "autoneg", CTLTYPE_INT\|CTLFLAG_RWTUN, softc, 0,
	+ bnxt_flow_ctrl_autoneg, "A",
	+ "Enable or Disable Autoneg Flow Ctrl: 0 / 1");
	+
	+ return 0;
	+}
	+
	+int
	+bnxt_create_hw_lro_sysctls(struct bnxt_softc *softc)
	+{
	+ struct sysctl_oid *oid = softc->hw_lro_oid;
	+
	+ if (!oid)
	+ return ENOMEM;
	+
	+ SYSCTL_ADD_PROC(&softc->hw_lro_ctx, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "enable", CTLTYPE_INT\|CTLFLAG_RWTUN, softc, 0,
	+ bnxt_hw_lro_enable_disable, "A",
	+ "Enable or Disable HW LRO: 0 / 1");
	+
	+ SYSCTL_ADD_PROC(&softc->hw_lro_ctx, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "gro_mode", CTLTYPE_INT\|CTLFLAG_RWTUN, softc, 0,
	+ bnxt_hw_lro_set_mode, "A",
	+ "Set mode: 1 = GRO mode, 0 = RSC mode");
	+
	+ SYSCTL_ADD_PROC(&softc->hw_lro_ctx, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "max_agg_segs", CTLTYPE_INT\|CTLFLAG_RWTUN, softc, 0,
	+ bnxt_hw_lro_set_max_agg_segs, "A",
	+ "Set Max Agg Seg Value (unit is Log2): "
	+ "0 (= 1 seg) / 1 (= 2 segs) / ... / 31 (= 2^31 segs)");
	+
	+ SYSCTL_ADD_PROC(&softc->hw_lro_ctx, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "max_aggs", CTLTYPE_INT\|CTLFLAG_RWTUN, softc, 0,
	+ bnxt_hw_lro_set_max_aggs, "A",
	+ "Set Max Aggs Value (unit is Log2): "
	+ "0 (= 1 agg) / 1 (= 2 aggs) / ... / 7 (= 2^7 segs)");
	+
	+ SYSCTL_ADD_PROC(&softc->hw_lro_ctx, SYSCTL_CHILDREN(oid), OID_AUTO,
	+ "min_agg_len", CTLTYPE_INT\|CTLFLAG_RWTUN, softc, 0,
	+ bnxt_hw_lro_set_min_agg_len, "A",
	+ "Min Agg Len: 1 to 9000");
	+
	+ return 0;
	+}
	static int
	bnxt_vlan_only_sysctl(SYSCTL_HANDLER_ARGS) {
	struct bnxt_softc *softc = arg1;
	Index: sys/dev/bnxt/bnxt_txrx.c
	===================================================================
	--- sys/dev/bnxt/bnxt_txrx.c
	+++ sys/dev/bnxt/bnxt_txrx.c
	@@ -48,17 +48,19 @@
	*/

	static int bnxt_isc_txd_encap(void *sc, if_pkt_info_t pi);
	-static void bnxt_isc_txd_flush(void *sc, uint16_t txqid, uint32_t pidx);
	-static int bnxt_isc_txd_credits_update(void *sc, uint16_t txqid, uint32_t cidx,
	- bool clear);
	+static void bnxt_isc_txd_flush(void *sc, uint16_t txqid, qidx_t pidx);
	+static int bnxt_isc_txd_credits_update(void *sc, uint16_t txqid, bool clear);

	-static void bnxt_isc_rxd_refill(void *sc, uint16_t rxqid, uint8_t flid,
	+static void bnxt_isc_rxd_refill(void *sc, if_rxd_update_t iru);
	+
	+/* uint16_t rxqid, uint8_t flid,
	uint32_t pidx, uint64_t paddrs, caddr_t vaddrs, uint16_t count,
	uint16_t buf_size);
	+*/
	static void bnxt_isc_rxd_flush(void *sc, uint16_t rxqid, uint8_t flid,
	- uint32_t pidx);
	-static int bnxt_isc_rxd_available(void *sc, uint16_t rxqid, uint32_t idx,
	- int budget);
	+ qidx_t pidx);
	+static int bnxt_isc_rxd_available(void *sc, uint16_t rxqid, qidx_t idx,
	+ qidx_t budget);
	static int bnxt_isc_rxd_pkt_get(void *sc, if_rxd_info_t ri);

	static int bnxt_intr(void *sc);
	@@ -172,7 +174,7 @@
	}

	static void
	-bnxt_isc_txd_flush(void *sc, uint16_t txqid, uint32_t pidx)
	+bnxt_isc_txd_flush(void *sc, uint16_t txqid, qidx_t pidx)
	{
	struct bnxt_softc softc = (struct bnxt_softc )sc;
	struct bnxt_ring *tx_ring = &softc->tx_rings[txqid];
	@@ -185,7 +187,7 @@
	}

	static int
	-bnxt_isc_txd_credits_update(void *sc, uint16_t txqid, uint32_t idx, bool clear)
	+bnxt_isc_txd_credits_update(void *sc, uint16_t txqid, bool clear)
	{
	struct bnxt_softc softc = (struct bnxt_softc )sc;
	struct bnxt_cp_ring *cpr = &softc->tx_cp_rings[txqid];
	@@ -249,16 +251,30 @@
	}

	static void
	-bnxt_isc_rxd_refill(void *sc, uint16_t rxqid, uint8_t flid,
	- uint32_t pidx, uint64_t *paddrs,
	- caddr_t *vaddrs, uint16_t count, uint16_t len)
	+bnxt_isc_rxd_refill(void *sc, if_rxd_update_t iru)
	{
	struct bnxt_softc softc = (struct bnxt_softc )sc;
	struct bnxt_ring *rx_ring;
	struct rx_prod_pkt_bd *rxbd;
	uint16_t type;
	uint16_t i;
	+ uint16_t rxqid;
	+ uint16_t count, len;
	+ uint32_t pidx;
	+ uint8_t flid;
	+ uint64_t *paddrs;
	+ caddr_t *vaddrs;
	+ qidx_t *frag_idxs;

	+ rxqid = iru->iru_qsidx;
	+ count = iru->iru_count;
	+ len = iru->iru_buf_size;
	+ pidx = iru->iru_pidx;
	+ flid = iru->iru_flidx;
	+ vaddrs = iru->iru_vaddrs;
	+ paddrs = iru->iru_paddrs;
	+ frag_idxs = iru->iru_idxs;
	+
	if (flid == 0) {
	rx_ring = &softc->rx_rings[rxqid];
	type = RX_PROD_PKT_BD_TYPE_RX_PROD_PKT;
	@@ -273,8 +289,8 @@
	rxbd[pidx].flags_type = htole16(type);
	rxbd[pidx].len = htole16(len);
	/* No need to byte-swap the opaque value */
	- rxbd[pidx].opaque = ((rxqid & 0xff) << 24) \| (flid << 16)
	- \| pidx;
	+ rxbd[pidx].opaque = (((rxqid & 0xff) << 24) \| (flid << 16)
	+ \| (frag_idxs[i]));
	rxbd[pidx].addr = htole64(paddrs[i]);
	if (++pidx == rx_ring->ring_size)
	pidx = 0;
	@@ -284,7 +300,7 @@

	static void
	bnxt_isc_rxd_flush(void *sc, uint16_t rxqid, uint8_t flid,
	- uint32_t pidx)
	+ qidx_t pidx)
	{
	struct bnxt_softc softc = (struct bnxt_softc )sc;
	struct bnxt_ring *rx_ring;
	@@ -310,12 +326,11 @@
	}

	static int
	-bnxt_isc_rxd_available(void *sc, uint16_t rxqid, uint32_t idx, int budget)
	+bnxt_isc_rxd_available(void *sc, uint16_t rxqid, qidx_t idx, qidx_t budget)
	{
	struct bnxt_softc softc = (struct bnxt_softc )sc;
	struct bnxt_cp_ring *cpr = &softc->rx_cp_rings[rxqid];
	struct rx_pkt_cmpl *rcp;
	- struct rx_tpa_start_cmpl *rtpa;
	struct rx_tpa_end_cmpl *rtpae;
	struct cmpl_base cmp = (struct cmpl_base )cpr->ring.vaddr;
	int avail = 0;
	@@ -324,7 +339,6 @@
	uint8_t ags;
	int i;
	uint16_t type;
	- uint8_t agg_id;

	for (;;) {
	NEXT_CP_CONS_V(&cpr->ring, cons, v_bit);
	@@ -374,18 +388,11 @@
	avail++;
	break;
	case CMPL_BASE_TYPE_RX_TPA_START:
	- rtpa = (void *)&cmp[cons];
	- agg_id = (rtpa->agg_id &
	- RX_TPA_START_CMPL_AGG_ID_MASK) >>
	- RX_TPA_START_CMPL_AGG_ID_SFT;
	- softc->tpa_start[agg_id].low = *rtpa;
	NEXT_CP_CONS_V(&cpr->ring, cons, v_bit);
	CMPL_PREFETCH_NEXT(cpr, cons);

	if (!CMP_VALID(&cmp[cons], v_bit))
	goto cmpl_invalid;
	- softc->tpa_start[agg_id].high =
	- ((struct rx_tpa_start_cmpl_hi *)cmp)[cons];
	break;
	case CMPL_BASE_TYPE_RX_AGG:
	break;
	@@ -412,6 +419,37 @@
	return avail;
	}

	+static void
	+bnxt_set_rsstype(if_rxd_info_t ri, uint8_t rss_hash_type)
	+{
	+ uint8_t rss_profile_id;
	+
	+ rss_profile_id = BNXT_GET_RSS_PROFILE_ID(rss_hash_type);
	+ switch (rss_profile_id) {
	+ case BNXT_RSS_HASH_TYPE_TCPV4:
	+ ri->iri_rsstype = M_HASHTYPE_RSS_TCP_IPV4;
	+ break;
	+ case BNXT_RSS_HASH_TYPE_UDPV4:
	+ ri->iri_rsstype = M_HASHTYPE_RSS_UDP_IPV4;
	+ break;
	+ case BNXT_RSS_HASH_TYPE_IPV4:
	+ ri->iri_rsstype = M_HASHTYPE_RSS_IPV4;
	+ break;
	+ case BNXT_RSS_HASH_TYPE_TCPV6:
	+ ri->iri_rsstype = M_HASHTYPE_RSS_TCP_IPV6;
	+ break;
	+ case BNXT_RSS_HASH_TYPE_UDPV6:
	+ ri->iri_rsstype = M_HASHTYPE_RSS_UDP_IPV6;
	+ break;
	+ case BNXT_RSS_HASH_TYPE_IPV6:
	+ ri->iri_rsstype = M_HASHTYPE_RSS_IPV6;
	+ break;
	+ default:
	+ ri->iri_rsstype = M_HASHTYPE_OPAQUE_HASH;
	+ break;
	+ }
	+}
	+
	static int
	bnxt_pkt_get_l2(struct bnxt_softc *softc, if_rxd_info_t ri,
	struct bnxt_cp_ring *cpr, uint16_t flags_type)
	@@ -429,13 +467,7 @@
	/* Extract from the first 16-byte BD */
	if (flags_type & RX_PKT_CMPL_FLAGS_RSS_VALID) {
	ri->iri_flowid = le32toh(rcp->rss_hash);
	- /*
	- * TODO: Extract something useful from rcp->rss_hash_type
	- * (undocumented)
	- * May be documented in the "LSI ES"
	- * also check the firmware code.
	- */
	- ri->iri_rsstype = M_HASHTYPE_OPAQUE;
	+ bnxt_set_rsstype(ri, rcp->rss_hash_type);
	}
	else {
	ri->iri_rsstype = M_HASHTYPE_NONE;
	@@ -469,9 +501,11 @@
	if (!(errors & RX_PKT_CMPL_ERRORS_IP_CS_ERROR))
	ri->iri_csum_flags \|= CSUM_IP_VALID;
	}
	- if (flags2 & RX_PKT_CMPL_FLAGS2_L4_CS_CALC) {
	+ if (flags2 & (RX_PKT_CMPL_FLAGS2_L4_CS_CALC \|
	+ RX_PKT_CMPL_FLAGS2_T_L4_CS_CALC)) {
	ri->iri_csum_flags \|= CSUM_L4_CALC;
	- if (!(errors & RX_PKT_CMPL_ERRORS_L4_CS_ERROR)) {
	+ if (!(errors & (RX_PKT_CMPL_ERRORS_L4_CS_ERROR \|
	+ RX_PKT_CMPL_ERRORS_T_L4_CS_ERROR))) {
	ri->iri_csum_flags \|= CSUM_L4_VALID;
	ri->iri_csum_data = 0xffff;
	}
	@@ -510,18 +544,12 @@
	/* Get the agg_id */
	agg_id = (agend->agg_id & RX_TPA_END_CMPL_AGG_ID_MASK) >>
	RX_TPA_END_CMPL_AGG_ID_SFT;
	- tpas = &softc->tpa_start[agg_id];
	+ tpas = &(softc->rx_rings[ri->iri_qsidx].tpa_start[agg_id]);

	/* Extract from the first 16-byte BD */
	if (le16toh(tpas->low.flags_type) & RX_TPA_START_CMPL_FLAGS_RSS_VALID) {
	ri->iri_flowid = le32toh(tpas->low.rss_hash);
	- /*
	- * TODO: Extract something useful from tpas->low.rss_hash_type
	- * (undocumented)
	- * May be documented in the "LSI ES"
	- * also check the firmware code.
	- */
	- ri->iri_rsstype = M_HASHTYPE_OPAQUE;
	+ bnxt_set_rsstype(ri, tpas->low.rss_hash_type);
	}
	else {
	ri->iri_rsstype = M_HASHTYPE_NONE;
	@@ -530,8 +558,8 @@
	RX_TPA_END_CMPL_AGG_BUFS_SFT;
	ri->iri_nfrags = ags + 1;
	/* No need to byte-swap the opaque value */
	- ri->iri_frags[0].irf_flid = (tpas->low.opaque >> 16) & 0xff;
	- ri->iri_frags[0].irf_idx = tpas->low.opaque & 0xffff;
	+ ri->iri_frags[0].irf_flid = ((tpas->low.opaque >> 16) & 0xff);
	+ ri->iri_frags[0].irf_idx = (tpas->low.opaque & 0xffff);
	ri->iri_frags[0].irf_len = le16toh(tpas->low.len);
	ri->iri_len = le16toh(tpas->low.len);

	@@ -567,8 +595,8 @@
	acp = &((struct rx_abuf_cmpl *)cpr->ring.vaddr)[cpr->cons];

	/* No need to byte-swap the opaque value */
	- ri->iri_frags[i].irf_flid = (acp->opaque >> 16) & 0xff;
	- ri->iri_frags[i].irf_idx = acp->opaque & 0xffff;
	+ ri->iri_frags[i].irf_flid = ((acp->opaque >> 16) & 0xff);
	+ ri->iri_frags[i].irf_idx = (acp->opaque & 0xffff);
	ri->iri_frags[i].irf_len = le16toh(acp->len);
	ri->iri_len += le16toh(acp->len);
	}
	@@ -576,8 +604,8 @@
	/* And finally, the empty BD at the end... */
	ri->iri_nfrags++;
	/* No need to byte-swap the opaque value */
	- ri->iri_frags[i].irf_flid = (agend->opaque >> 16) % 0xff;
	- ri->iri_frags[i].irf_idx = agend->opaque & 0xffff;
	+ ri->iri_frags[i].irf_flid = ((agend->opaque >> 16) & 0xff);
	+ ri->iri_frags[i].irf_idx = (agend->opaque & 0xffff);
	ri->iri_frags[i].irf_len = le16toh(agend->len);
	ri->iri_len += le16toh(agend->len);

	@@ -590,9 +618,12 @@
	{
	struct bnxt_softc softc = (struct bnxt_softc )sc;
	struct bnxt_cp_ring *cpr = &softc->rx_cp_rings[ri->iri_qsidx];
	+ struct cmpl_base cmp_q = (struct cmpl_base )cpr->ring.vaddr;
	struct cmpl_base *cmp;
	+ struct rx_tpa_start_cmpl *rtpa;
	uint16_t flags_type;
	uint16_t type;
	+ uint8_t agg_id;

	for (;;) {
	NEXT_CP_CONS_V(&cpr->ring, cpr->cons, cpr->v_bit);
	@@ -609,9 +640,18 @@
	case CMPL_BASE_TYPE_RX_TPA_END:
	return bnxt_pkt_get_tpa(softc, ri, cpr, flags_type);
	case CMPL_BASE_TYPE_RX_TPA_START:
	+ rtpa = (void *)&cmp_q[cpr->cons];
	+ agg_id = (rtpa->agg_id &
	+ RX_TPA_START_CMPL_AGG_ID_MASK) >>
	+ RX_TPA_START_CMPL_AGG_ID_SFT;
	+ softc->rx_rings[ri->iri_qsidx].tpa_start[agg_id].low = *rtpa;
	+
	NEXT_CP_CONS_V(&cpr->ring, cpr->cons, cpr->v_bit);
	ri->iri_cidx = RING_NEXT(&cpr->ring, ri->iri_cidx);
	CMPL_PREFETCH_NEXT(cpr, cpr->cons);
	+
	+ softc->rx_rings[ri->iri_qsidx].tpa_start[agg_id].high =
	+ ((struct rx_tpa_start_cmpl_hi *)cmp_q)[cpr->cons];
	break;
	default:
	device_printf(softc->dev,
	Index: sys/dev/bnxt/hsi_struct_def.h
	===================================================================
	--- sys/dev/bnxt/hsi_struct_def.h
	+++ sys/dev/bnxt/hsi_struct_def.h
	@@ -34,7 +34,7 @@
	*
	* Description: Definition of HSI data structures
	*
	- * Date: 07/26/16 21:30:37
	+ * Date: 08/31/17 17:55:46
	*
	* Note: This file is scripted generated by hsi_decode.py.
	* DO NOT modify this file manually !!!!
	@@ -42,35 +42,45 @@
	****************************************************************************/
	#ifndef _HSI_STRUCT_DEF_EXTERNAL_H_
	#define _HSI_STRUCT_DEF_EXTERNAL_H_
	+/* HSI and HWRM Specification 1.8.1 */
	+#define HWRM_VERSION_MAJOR 1
	+#define HWRM_VERSION_MINOR 8
	+#define HWRM_VERSION_UPDATE 1

	+#define HWRM_VERSION_RSVD 7
	+
	+#define HWRM_VERSION_STR "1.8.1.7"
	/*
	- * per-context HW statistics -- chip view
	- * Reference to stat_ctx_stat_xxx for
	+ * Following is the signature for HWRM message field that indicates not
	+ * applicable (All F's). Need to cast it the size of the field if needed.
	*/
	+#define HWRM_NA_SIGNATURE ((uint32_t)(-1))
	+#define HWRM_MAX_REQ_LEN (128) /* hwrm_func_buf_rgtr */
	+#define HWRM_MAX_RESP_LEN (272) /* hwrm_selftest_qlist */
	+#define HW_HASH_INDEX_SIZE 0x80 /* 7 bit indirection table index. */
	+#define HW_HASH_KEY_SIZE 40
	+#define HWRM_RESP_VALID_KEY 1 /* valid key for HWRM response */
	+#define ROCE_SP_HSI_VERSION_MAJOR 1
	+#define ROCE_SP_HSI_VERSION_MINOR 8
	+#define ROCE_SP_HSI_VERSION_UPDATE 1

	-struct ctx_hw_stats {
	- uint64_t rx_ucast_pkts;
	- uint64_t rx_mcast_pkts;
	- uint64_t rx_bcast_pkts;
	- uint64_t rx_discard_pkts;
	- uint64_t rx_drop_pkts;
	- uint64_t rx_ucast_bytes;
	- uint64_t rx_mcast_bytes;
	- uint64_t rx_bcast_bytes;
	- uint64_t tx_ucast_pkts;
	- uint64_t tx_mcast_pkts;
	- uint64_t tx_bcast_pkts;
	- uint64_t tx_discard_pkts;
	- uint64_t tx_drop_pkts;
	- uint64_t tx_ucast_bytes;
	- uint64_t tx_mcast_bytes;
	- uint64_t tx_bcast_bytes;
	- uint64_t tpa_pkts;
	- uint64_t tpa_bytes;
	- uint64_t tpa_events;
	- uint64_t tpa_aborts;
	-} __attribute__((packed));
	-
	+#define ROCE_SP_HSI_VERSION_STR "1.8.1"
	+/*
	+ * Following is the signature for ROCE_SP_HSI message field that indicates not
	+ * applicable (All F's). Need to cast it the size of the field if needed.
	+ */
	+#define ROCE_SP_HSI_NA_SIGNATURE ((uint32_t)(-1))
	+/*
	+ * Note: The Host Software Interface (HSI) and Hardware Resource Manager (HWRM)
	+ * specification describes the data structures used in Ethernet packet or RDMA
	+ * message data transfers as well as an abstract interface for managing Ethernet
	+ * NIC hardware resources.
	+ */
	+/* Ethernet Data path Host Structures */
	+/*
	+ * Description: The following three sections document the host structures used
	+ * between device and software drivers for communicating Ethernet packets.
	+ */
	/* BD Ring Structures */
	/*
	* Description: This structure is used to inform the NIC of a location for and
	@@ -845,6 +855,7 @@
	/* This bit is '1' if the RSS field in this completion is valid. */
	#define RX_PKT_CMPL_FLAGS_RSS_VALID UINT32_C(0x400)
	/* unused is 1 b */
	+ #define RX_PKT_CMPL_FLAGS_UNUSED UINT32_C(0x800)
	/*
	* This value indicates what the inner packet determined for the packet
	* was.
	@@ -931,7 +942,27 @@
	uint8_t rss_hash_type;
	/*
	* This is the RSS hash type for the packet. The value is packed
	- * {tuple_extrac_op[1:0],rss_profile_id[4:0],tuple_extrac_op[2]}.
	+ * {tuple_extrac_op[1:0],rss_profile_id[4:0],tuple_extrac_op[2]}. The
	+ * value of tuple_extrac_op provides the information about what fields
	+ * the hash was computed on. * 0: The RSS hash was computed over source
	+ * IP address, destination IP address, source port, and destination port
	+ * of inner IP and TCP or UDP headers. Note: For non-tunneled packets,
	+ * the packet headers are considered inner packet headers for the RSS
	+ * hash computation purpose. * 1: The RSS hash was computed over source
	+ * IP address and destination IP address of inner IP header. Note: For
	+ * non-tunneled packets, the packet headers are considered inner packet
	+ * headers for the RSS hash computation purpose. * 2: The RSS hash was
	+ * computed over source IP address, destination IP address, source port,
	+ * and destination port of IP and TCP or UDP headers of outer tunnel
	+ * headers. Note: For non-tunneled packets, this value is not
	+ * applicable. * 3: The RSS hash was computed over source IP address and
	+ * destination IP address of IP header of outer tunnel headers. Note:
	+ * For non-tunneled packets, this value is not applicable. Note that
	+ * 4-tuples values listed above are applicable for layer 4 protocols
	+ * supported and enabled for RSS in the hardware, HWRM firmware, and
	+ * drivers. For example, if RSS hash is supported and enabled for TCP
	+ * traffic only, then the values of tuple_extract_op corresponding to
	+ * 4-tuples are only valid for TCP traffic.
	*/
	uint8_t payload_offset;
	/*
	@@ -1234,6 +1265,7 @@
	/* This bit is '1' if the RSS field in this completion is valid. */
	#define RX_TPA_START_CMPL_FLAGS_RSS_VALID UINT32_C(0x400)
	/* unused is 1 b */
	+ #define RX_TPA_START_CMPL_FLAGS_UNUSED UINT32_C(0x800)
	/*
	* This value indicates what the inner packet determined for the packet
	* was.
	@@ -1267,7 +1299,27 @@
	uint8_t rss_hash_type;
	/*
	* This is the RSS hash type for the packet. The value is packed
	- * {tuple_extrac_op[1:0],rss_profile_id[4:0],tuple_extrac_op[2]}.
	+ * {tuple_extrac_op[1:0],rss_profile_id[4:0],tuple_extrac_op[2]}. The
	+ * value of tuple_extrac_op provides the information about what fields
	+ * the hash was computed on. * 0: The RSS hash was computed over source
	+ * IP address, destination IP address, source port, and destination port
	+ * of inner IP and TCP or UDP headers. Note: For non-tunneled packets,
	+ * the packet headers are considered inner packet headers for the RSS
	+ * hash computation purpose. * 1: The RSS hash was computed over source
	+ * IP address and destination IP address of inner IP header. Note: For
	+ * non-tunneled packets, the packet headers are considered inner packet
	+ * headers for the RSS hash computation purpose. * 2: The RSS hash was
	+ * computed over source IP address, destination IP address, source port,
	+ * and destination port of IP and TCP or UDP headers of outer tunnel
	+ * headers. Note: For non-tunneled packets, this value is not
	+ * applicable. * 3: The RSS hash was computed over source IP address and
	+ * destination IP address of IP header of outer tunnel headers. Note:
	+ * For non-tunneled packets, this value is not applicable. Note that
	+ * 4-tuples values listed above are applicable for layer 4 protocols
	+ * supported and enabled for RSS in the hardware, HWRM firmware, and
	+ * drivers. For example, if RSS hash is supported and enabled for TCP
	+ * traffic only, then the values of tuple_extract_op corresponding to
	+ * 4-tuples are only valid for TCP traffic.
	*/
	uint16_t agg_id;
	/*
	@@ -1456,6 +1508,8 @@
	#define RX_TPA_END_CMPL_FLAGS_PLACEMENT_GRO_HDS (UINT32_C(0x6) << 7)
	#define RX_TPA_END_CMPL_FLAGS_PLACEMENT_LAST RX_TPA_END_CMPL_FLAGS_PLACEMENT_GRO_HDS
	/* unused is 2 b */
	+ #define RX_TPA_END_CMPL_FLAGS_UNUSED_MASK UINT32_C(0xc00)
	+ #define RX_TPA_END_CMPL_FLAGS_UNUSED_SFT 10
	/*
	* This value indicates what the inner packet determined for the packet
	* was. - 2 TCP Packet Indicates that the packet was IP and TCP. This
	@@ -1794,6 +1848,7 @@
	((x) == 0x33 ? "VF_CFG_CHANGE": \
	((x) == 0x11 ? "FUNC_DRVR_LOAD": \
	((x) == 0x31 ? "VF_MAC_ADDR_CHANGE": \
	+ ((x) == 0x34 ? "LLFC_PFC_CHANGE": \
	((x) == 0x4 ? "PORT_CONN_NOT_ALLOWED": \
	((x) == 0x5 ? "LINK_SPEED_CFG_NOT_ALLOWED": \
	((x) == 0x6 ? "LINK_SPEED_CFG_CHANGE": \
	@@ -1804,7 +1859,7 @@
	((x) == 0x3 ? "DCB_CONFIG_CHANGE": \
	((x) == 0x12 ? "FUNC_FLR_PROC_CMPLT": \
	((x) == 0x21 ? "PF_DRVR_LOAD": \
	- "Unknown event_id"))))))))))))))))))
	+ "Unknown event_id")))))))))))))))))))

	/* HWRM Asynchronous Event Completion Record (16 bytes) */

	@@ -1857,6 +1912,8 @@
	#define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_PF_VF_COMM_STATUS_CHANGE UINT32_C(0x32)
	/* VF Configuration Change */
	#define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE UINT32_C(0x33)
	+ /* LLFC/PFC Configuration Change */
	+ #define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_LLFC_PFC_CHANGE UINT32_C(0x34)
	/* HWRM Error */
	#define HWRM_ASYNC_EVENT_CMPL_EVENT_ID_HWRM_ERROR UINT32_C(0xff)
	uint32_t event_data2;
	@@ -2075,6 +2132,12 @@
	#define HWRM_ASYNC_EVENT_CMPL_DCB_CONFIG_CHANGE_EVENT_ID_DCB_CONFIG_CHANGE UINT32_C(0x3)
	uint32_t event_data2;
	/* Event specific data */
	+ /* ETS configuration change */
	+ #define HWRM_ASYNC_EVENT_CMPL_DCB_CONFIG_CHANGE_EVENT_DATA2_ETS UINT32_C(0x1)
	+ /* PFC configuration change */
	+ #define HWRM_ASYNC_EVENT_CMPL_DCB_CONFIG_CHANGE_EVENT_DATA2_PFC UINT32_C(0x2)
	+ /* APP configuration change */
	+ #define HWRM_ASYNC_EVENT_CMPL_DCB_CONFIG_CHANGE_EVENT_DATA2_APP UINT32_C(0x4)
	uint8_t opaque_v;
	/* opaque is 7 b */
	/*
	@@ -2095,6 +2158,18 @@
	/* PORT ID */
	#define HWRM_ASYNC_EVENT_CMPL_DCB_CONFIG_CHANGE_EVENT_DATA1_PORT_ID_MASK UINT32_C(0xffff)
	#define HWRM_ASYNC_EVENT_CMPL_DCB_CONFIG_CHANGE_EVENT_DATA1_PORT_ID_SFT 0
	+ /* Priority recommended for RoCE traffic */
	+ #define HWRM_ASYNC_EVENT_CMPL_DCB_CONFIG_CHANGE_EVENT_DATA1_RECOMMEND_ROCE_PRIORITY_MASK UINT32_C(0xff0000)
	+ #define HWRM_ASYNC_EVENT_CMPL_DCB_CONFIG_CHANGE_EVENT_DATA1_RECOMMEND_ROCE_PRIORITY_SFT 16
	+ /* none is 255 */
	+ #define HWRM_ASYNC_EVENT_CMPL_DCB_CONFIG_CHANGE_EVENT_DATA1_RECOMMEND_ROCE_PRIORITY_NONE (UINT32_C(0xff) << 16)
	+ #define HWRM_ASYNC_EVENT_CMPL_DCB_CONFIG_CHANGE_EVENT_DATA1_RECOMMEND_ROCE_PRIORITY_LAST HWRM_ASYNC_EVENT_CMPL_DCB_CONFIG_CHANGE_EVENT_DATA1_RECOMMEND_ROCE_PRIORITY_NONE
	+ /* Priority recommended for L2 traffic */
	+ #define HWRM_ASYNC_EVENT_CMPL_DCB_CONFIG_CHANGE_EVENT_DATA1_RECOMMEND_L2_PRIORITY_MASK UINT32_C(0xff000000)
	+ #define HWRM_ASYNC_EVENT_CMPL_DCB_CONFIG_CHANGE_EVENT_DATA1_RECOMMEND_L2_PRIORITY_SFT 24
	+ /* none is 255 */
	+ #define HWRM_ASYNC_EVENT_CMPL_DCB_CONFIG_CHANGE_EVENT_DATA1_RECOMMEND_L2_PRIORITY_NONE (UINT32_C(0xff) << 24)
	+ #define HWRM_ASYNC_EVENT_CMPL_DCB_CONFIG_CHANGE_EVENT_DATA1_RECOMMEND_L2_PRIORITY_LAST HWRM_ASYNC_EVENT_CMPL_DCB_CONFIG_CHANGE_EVENT_DATA1_RECOMMEND_L2_PRIORITY_NONE
	} __attribute__((packed));

	/* HWRM Asynchronous Event Completion Record for port connection not allowed (16 bytes) */
	@@ -2733,6 +2808,60 @@
	#define HWRM_ASYNC_EVENT_CMPL_VF_CFG_CHANGE_EVENT_DATA1_DFLT_VLAN_CHANGE UINT32_C(0x8)
	} __attribute__((packed));

	+/* HWRM Asynchronous Event Completion Record for llfc pfc status change (16 bytes) */
	+
	+struct hwrm_async_event_cmpl_llfc_pfc_change {
	+ uint16_t type;
	+ /* unused1 is 10 b */
	+ /*
	+ * This field indicates the exact type of the completion. By convention,
	+ * the LSB identifies the length of the record in 16B units. Even values
	+ * indicate 16B records. Odd values indicate 32B records.
	+ */
	+ #define HWRM_ASYNC_EVENT_CMPL_LLFC_PFC_CHANGE_TYPE_MASK UINT32_C(0x3f)
	+ #define HWRM_ASYNC_EVENT_CMPL_LLFC_PFC_CHANGE_TYPE_SFT 0
	+ /* HWRM Asynchronous Event Information */
	+ #define HWRM_ASYNC_EVENT_CMPL_LLFC_PFC_CHANGE_TYPE_HWRM_ASYNC_EVENT UINT32_C(0x2e)
	+ /* unused1 is 10 b */
	+ uint16_t event_id;
	+ /* Identifiers of events. */
	+ /* LLFC/PFC Configuration Change */
	+ #define HWRM_ASYNC_EVENT_CMPL_LLFC_PFC_CHANGE_EVENT_ID_LLFC_PFC_CHANGE UINT32_C(0x34)
	+ uint32_t event_data2;
	+ /* Event specific data */
	+ uint8_t opaque_v;
	+ /* opaque is 7 b */
	+ /*
	+ * This value is written by the NIC such that it will be different for
	+ * each pass through the completion queue. The even passes will write 1.
	+ * The odd passes will write 0.
	+ */
	+ #define HWRM_ASYNC_EVENT_CMPL_LLFC_PFC_CHANGE_V UINT32_C(0x1)
	+ /* opaque is 7 b */
	+ #define HWRM_ASYNC_EVENT_CMPL_LLFC_PFC_CHANGE_OPAQUE_MASK UINT32_C(0xfe)
	+ #define HWRM_ASYNC_EVENT_CMPL_LLFC_PFC_CHANGE_OPAQUE_SFT 1
	+ uint8_t timestamp_lo;
	+ /* 8-lsb timestamp from POR (100-msec resolution) */
	+ uint16_t timestamp_hi;
	+ /* 16-lsb timestamp from POR (100-msec resolution) */
	+ uint32_t event_data1;
	+ /* Event specific data */
	+ /* Indicates llfc pfc status change */
	+ #define HWRM_ASYNC_EVENT_CMPL_LLFC_PFC_CHANGE_EVENT_DATA1_LLFC_PFC_MASK UINT32_C(0x3)
	+ #define HWRM_ASYNC_EVENT_CMPL_LLFC_PFC_CHANGE_EVENT_DATA1_LLFC_PFC_SFT 0
	+ /* If this field set to 1, then it indicates that llfc is enabled. */
	+ #define HWRM_ASYNC_EVENT_CMPL_LLFC_PFC_CHANGE_EVENT_DATA1_LLFC_PFC_LLFC (UINT32_C(0x1) << 0)
	+ /* If this field is set to 2, then it indicates that pfc is enabled. */
	+ #define HWRM_ASYNC_EVENT_CMPL_LLFC_PFC_CHANGE_EVENT_DATA1_LLFC_PFC_PFC (UINT32_C(0x2) << 0)
	+ #define HWRM_ASYNC_EVENT_CMPL_LLFC_PFC_CHANGE_EVENT_DATA1_LLFC_PFC_LAST HWRM_ASYNC_EVENT_CMPL_LLFC_PFC_CHANGE_EVENT_DATA1_LLFC_PFC_PFC
	+ /* Indicates the physical port this llfc pfc change occur */
	+ #define HWRM_ASYNC_EVENT_CMPL_LLFC_PFC_CHANGE_EVENT_DATA1_PORT_MASK UINT32_C(0x1c)
	+ #define HWRM_ASYNC_EVENT_CMPL_LLFC_PFC_CHANGE_EVENT_DATA1_PORT_SFT 2
	+ /* PORT ID */
	+ #define HWRM_ASYNC_EVENT_CMPL_LLFC_PFC_CHANGE_EVENT_DATA1_PORT_ID_MASK UINT32_C(0x1fffe0)
	+ #define HWRM_ASYNC_EVENT_CMPL_LLFC_PFC_CHANGE_EVENT_DATA1_PORT_ID_SFT 5
	+} __attribute__((packed));
	+
	/* HWRM Asynchronous Event Completion Record for HWRM Error (16 bytes) */

	struct hwrm_async_event_cmpl_hwrm_error {
	@@ -3205,758 +3334,2126 @@
	*/
	} __attribute__((packed));

	-/* HW Resource Manager Specification 1.4.0 */
	-#define HWRM_VERSION_MAJOR 1
	-#define HWRM_VERSION_MINOR 4
	-#define HWRM_VERSION_UPDATE 0
	-
	-#define HWRM_VERSION_STR "1.4.0"
	+/* Doorbell Structures */
	/*
	- * Following is the signature for HWRM message field that indicates not
	- * applicable (All F's). Need to cast it the size of the field if needed.
	+ * Description: This is the RoCE 32b Doorbell format. The host writes this
	+ * message format directly to byte offset 8 of the appropriate doorbell page.
	*/
	-#define HWRM_NA_SIGNATURE ((uint32_t)(-1))
	-#define HWRM_MAX_REQ_LEN (128) /* hwrm_func_buf_rgtr */
	-#define HWRM_MAX_RESP_LEN (176) /* hwrm_func_qstats */
	-#define HW_HASH_INDEX_SIZE 0x80 /* 7 bit indirection table index. */
	-#define HW_HASH_KEY_SIZE 40
	-#define HWRM_RESP_VALID_KEY 1 /* valid key for HWRM response */
	+/* 64b Doorbell Format (8 bytes) */
	+
	+struct dbr_dbr {
	+ uint32_t index;
	+ /*
	+ * This value is the index being written. For SQ, RQ, SRQ, this is the
	+ * producer index and should be the queue index of the last WQE written
	+ * plus 1. For CQ, this is the consumer index and should be the index of
	+ * the last CQE processed plus 1.
	+ */
	+ #define DBR_DBR_INDEX_MASK UINT32_C(0xfffff)
	+ #define DBR_DBR_INDEX_SFT 0
	+ #define DBR_DBR_RESERVED12_MASK UINT32_C(0xfff00000)
	+ #define DBR_DBR_RESERVED12_SFT 20
	+ uint32_t type_xid;
	+ /* This value identifies the type of doorbell being written. */
	+ /*
	+ * This value identifies the resource that the doorbell is intended to
	+ * notify. For SQ and RQ, this is the QPID. For SRQ, this is the SID.
	+ * For CQ, this is the CID. Bits [19:16] of this values must be zero for
	+ * a SID value.
	+ */
	+ #define DBR_DBR_XID_MASK UINT32_C(0xfffff)
	+ #define DBR_DBR_XID_SFT 0
	+ #define DBR_DBR_RESERVED8_MASK UINT32_C(0xff00000)
	+ #define DBR_DBR_RESERVED8_SFT 20
	+ /* This value identifies the type of doorbell being written. */
	+ #define DBR_DBR_TYPE_MASK UINT32_C(0xf0000000)
	+ #define DBR_DBR_TYPE_SFT 28
	+ /*
	+ * This is a SQ producer index update. It indicates one or more
	+ * new entries have been written to the SQ for the QPID
	+ * indicated on the xID field.
	+ */
	+ #define DBR_DBR_TYPE_SQ (UINT32_C(0x0) << 28)
	+ /*
	+ * This is a RQ producer index update. It indicates one or more
	+ * new entries have been written to the RQ for the QPID
	+ * indicated on the xID field.
	+ */
	+ #define DBR_DBR_TYPE_RQ (UINT32_C(0x1) << 28)
	+ /*
	+ * This is a SRQ producer index update. It indicates one or more
	+ * new entries have been written to the SRQ for the SID
	+ * indicated on the xID field.
	+ */
	+ #define DBR_DBR_TYPE_SRQ (UINT32_C(0x2) << 28)
	+ /*
	+ * This doorbell command arms the SRQ async event. The xID field
	+ * must identify the SID that is begin armed. The index field is
	+ * will set the arm threshold such that a notification will be
	+ * generated if less than that number or SRQ entries are posted.
	+ */
	+ #define DBR_DBR_TYPE_SRQ_ARM (UINT32_C(0x3) << 28)
	+ /*
	+ * This is a CQ consumer index update. It indicates one or more
	+ * entries have been processed off the CQ indicated on the xID
	+ * field.
	+ */
	+ #define DBR_DBR_TYPE_CQ (UINT32_C(0x4) << 28)
	+ /*
	+ * this is a CQ consumer index update that also arms the CQ for
	+ * solicited events.
	+ */
	+ #define DBR_DBR_TYPE_CQ_ARMSE (UINT32_C(0x5) << 28)
	+ /*
	+ * This is a CQ consumer index update that also arms the CQ for
	+ * any new CQE.
	+ */
	+ #define DBR_DBR_TYPE_CQ_ARMALL (UINT32_C(0x6) << 28)
	+ /*
	+ * This is a CQ arm enable message. This message must be sent
	+ * from the privileged driver before a new CQ_ARMSE or CQ_ARMALL
	+ * message will be accepted. This doorbell can only be sent from
	+ * the privileged (first) doorbell page of a function.
	+ */
	+ #define DBR_DBR_TYPE_CQ_ARMENA (UINT32_C(0x7) << 28)
	+ /*
	+ * This doorbell command enables the SRQ async event to be
	+ * armed. This message must be setn from the privileged driver
	+ * before a new SRQ_ARM message will be accepted. The xID field
	+ * must identify the SID that is begin enabled for arm. This
	+ * doorbell can only be sent from the privileged (first)
	+ * doorbell page of a function.
	+ */
	+ #define DBR_DBR_TYPE_SRQ_ARMENA (UINT32_C(0x8) << 28)
	+ /*
	+ * This doorbell command indicates that the cutoff CQE has been
	+ * processed and the driver is now processing completions from
	+ * the new CQ. The index field for this doorbell type must be
	+ * zero.
	+ */
	+ #define DBR_DBR_TYPE_CQ_CUTOFF_ACK (UINT32_C(0x9) << 28)
	+ /*
	+ * This doorbell command is used during doorbell moderation to
	+ * consume system BW and help prevent doorbell FIFO overflow.
	+ * All other fields should be zero for NULL doorbell.
	+ */
	+ #define DBR_DBR_TYPE_NULL (UINT32_C(0xf) << 28)
	+} __attribute__((packed));
	+
	+/* 32b Doorbell Format (4 bytes) */
	+
	+struct dbr_dbr32 {
	+ uint32_t type_abs_incr_xid;
	+ /* This value identifies the type of doorbell being written. */
	+ /*
	+ * This value identifies the resource that the doorbell is intended to
	+ * notify. For SQ and RQ, this is the QPID. For SRQ, this is the SID.
	+ * For CQ, this is the CID. Bits [19:16] of this values must be zero for
	+ * a SID value.
	+ */
	+ #define DBR_DBR32_XID_MASK UINT32_C(0xfffff)
	+ #define DBR_DBR32_XID_SFT 0
	+ #define DBR_DBR32_RESERVED4_MASK UINT32_C(0xf00000)
	+ #define DBR_DBR32_RESERVED4_SFT 20
	+ /*
	+ * When abs=0, this value is the value to add to the appropriate index
	+ * value. When abs=1, this value is the new value for the index.
	+ * Absolute value is used when the queue is being wrapped. When abs=1,
	+ * the incr value follows the same rules as the index value in the 64b
	+ * doorbell.
	+ */
	+ #define DBR_DBR32_INCR_MASK UINT32_C(0xf000000)
	+ #define DBR_DBR32_INCR_SFT 24
	+ /* This value defines how the incr value will be interpreted. */
	+ #define DBR_DBR32_ABS UINT32_C(0x10000000)
	+ /* This value identifies the type of doorbell being written. */
	+ #define DBR_DBR32_TYPE_MASK UINT32_C(0xe0000000)
	+ #define DBR_DBR32_TYPE_SFT 29
	+ /*
	+ * This is a SQ producer index update. It indicates one or more
	+ * new entries have been written to the SQ for the QPID
	+ * indicated on the xID field.
	+ */
	+ #define DBR_DBR32_TYPE_SQ (UINT32_C(0x0) << 29)
	+} __attribute__((packed));
	+
	+/* SQ WQE Structures */
	/*
	- * Description: Port Rx Statistics Formats. The HWRM shall return any
	- * unsupported counter with a value of 0xFFFFFFFF for 32-bit counters and
	- * 0xFFFFFFFFFFFFFFFF for 64-bit counters.
	+ * Description: This is the Bind WQE structure. This WQE can perform either: *
	+ * type1 "bind memory window", if mw_type==Type1 * type2 "post send bind memory
	+ * window", if mw_type==Type2
	*/
	-/*
	- * Note: The Hardware Resource Manager (HWRM) manages various hardware resources
	- * inside the chip. The HWRM is implemented in firmware, and runs on embedded
	- * processors inside the chip. This firmware service is vital part of the chip.
	- * The chip can not be used by a driver or HWRM client without the HWRM.
	- */
	-/* Input (16 bytes) */
	+/* Base SQ WQE (8 bytes) */

	-struct input {
	- uint16_t req_type;
	+struct sq_base {
	+ uint8_t wqe_type;
	+ /* This field defines the type of SQ WQE. */
	+ /* Send */
	+ #define SQ_BASE_WQE_TYPE_SEND UINT32_C(0x0)
	/*
	- * This value indicates what type of request this is. The format for the
	- * rest of the command is determined by this field.
	+ * Send with Immediate Allowed only on reliable connection (RC)
	+ * and unreliable datagram (UD) SQ's.
	*/
	- uint16_t cmpl_ring;
	+ #define SQ_BASE_WQE_TYPE_SEND_W_IMMEAD UINT32_C(0x1)
	/*
	- * This value indicates the what completion ring the request will be
	- * optionally completed on. If the value is -1, then no CR completion
	- * will be generated. Any other value must be a valid CR ring_id value
	- * for this function.
	+ * Send with Invalidate. Allowed only on reliable connection
	+ * (RC) SQ's.
	*/
	- uint16_t seq_id;
	- /* This value indicates the command sequence number. */
	- uint16_t target_id;
	+ #define SQ_BASE_WQE_TYPE_SEND_W_INVALID UINT32_C(0x2)
	+ /* RDMA Write. Allowed only on reliable connection (RC) SQ's. */
	+ #define SQ_BASE_WQE_TYPE_WRITE_WQE UINT32_C(0x4)
	/*
	- * Target ID of this command. 0x0 - 0xFFF8 - Used for function ids
	- * 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF - HWRM
	+ * RDMA Write with Immediate. Allowed only on reliable
	+ * connection (RC) SQ's.
	*/
	- uint64_t resp_addr;
	+ #define SQ_BASE_WQE_TYPE_WRITE_W_IMMEAD UINT32_C(0x5)
	+ /* RDMA Read. Allowed only on reliable connection (RC) SQ's. */
	+ #define SQ_BASE_WQE_TYPE_READ_WQE UINT32_C(0x6)
	/*
	- * This is the host address where the response will be written when the
	- * request is complete. This area must be 16B aligned and must be
	- * cleared to zero before the request is made.
	+ * Atomic Compare/Swap. Allowed only on reliable connection (RC)
	+ * SQ's.
	*/
	+ #define SQ_BASE_WQE_TYPE_ATOMIC_CS UINT32_C(0x8)
	+ /* Atomic Fetch/Add. Allowed only on reliable connection (RC) SQ's. */
	+ #define SQ_BASE_WQE_TYPE_ATOMIC_FA UINT32_C(0xb)
	+ /* Local Invalidate. Allowed only on reliable connection (RC) SQ's. */
	+ #define SQ_BASE_WQE_TYPE_LOCAL_INVALID UINT32_C(0xc)
	+ /*
	+ * FR-PMR (Fast Register Physical Memory Region) Allowed only on
	+ * reliable connection (RC) SQ's.
	+ */
	+ #define SQ_BASE_WQE_TYPE_FR_PMR UINT32_C(0xd)
	+ /* Memory Bind Allowed only on reliable connection (RC) SQ's. */
	+ #define SQ_BASE_WQE_TYPE_BIND UINT32_C(0xe)
	+ uint8_t unused_0[7];
	} __attribute__((packed));

	-/* Output (8 bytes) */
	+/* WQE SGE (16 bytes) */

	-struct output {
	- uint16_t error_code;
	+struct sq_sge {
	+ uint64_t va_or_pa;
	/*
	- * Pass/Fail or error type Note: receiver to verify the in parameters,
	- * and fail the call with an error when appropriate
	+ * The virtual address in local memory or a physical address when l_key
	+ * value is a reserved value of a physical address. Driver configures
	+ * this value in the chip and the chip compares l_key in SGEs with that
	+ * reserved value, if equal it access the physical address specified.
	+ * The chip however MUST verify that the QP allows the use reserved key.
	*/
	- uint16_t req_type;
	- /* This field returns the type of original request. */
	- uint16_t seq_id;
	- /* This field provides original sequence number of the command. */
	- uint16_t resp_len;
	+ uint32_t l_key;
	/*
	- * This field is the length of the response in bytes. The last byte of
	- * the response is a valid flag that will read as '1' when the command
	- * has been completely written to memory.
	+ * Local Key associated with this registered MR; The 24 msb of the key
	+ * used to index the MRW Table and the 8 lsb are compared with the 8
	+ * bits key part stored in the MRWC. The PBL in the MRW Context is used
	+ * to translate the above VA to physical address.
	*/
	+ uint32_t size;
	+ /*
	+ * Size of SGE in bytes; Based on page size of the system the chip knows
	+ * how many entries are in the PBL
	+ */
	} __attribute__((packed));

	-#define GET_HWRM_REQ_TYPE(x) \
	- ((x) == 0x98 ? "HWRM_CFA_ENCAP_RECORD_FREE": \
	- ((x) == 0x99 ? "HWRM_CFA_NTUPLE_FILTER_ALLOC": \
	- ((x) == 0x90 ? "HWRM_CFA_L2_FILTER_ALLOC": \
	- ((x) == 0x91 ? "HWRM_CFA_L2_FILTER_FREE": \
	- ((x) == 0x92 ? "HWRM_CFA_L2_FILTER_CFG": \
	- ((x) == 0x93 ? "HWRM_CFA_L2_SET_RX_MASK": \
	- ((x) == 0x94 ? "RESERVED3": \
	- ((x) == 0x95 ? "HWRM_CFA_TUNNEL_FILTER_ALLOC": \
	- ((x) == 0x96 ? "HWRM_CFA_TUNNEL_FILTER_FREE": \
	- ((x) == 0x97 ? "HWRM_CFA_ENCAP_RECORD_ALLOC": \
	- ((x) == 0x10 ? "RESERVED1": \
	- ((x) == 0x11 ? "HWRM_FUNC_RESET": \
	- ((x) == 0x12 ? "HWRM_FUNC_GETFID": \
	- ((x) == 0x13 ? "HWRM_FUNC_VF_ALLOC": \
	- ((x) == 0x14 ? "HWRM_FUNC_VF_FREE": \
	- ((x) == 0x15 ? "HWRM_FUNC_QCAPS": \
	- ((x) == 0x16 ? "HWRM_FUNC_QCFG": \
	- ((x) == 0x17 ? "HWRM_FUNC_CFG": \
	- ((x) == 0x18 ? "HWRM_FUNC_QSTATS": \
	- ((x) == 0x19 ? "HWRM_FUNC_CLR_STATS": \
	- ((x) == 0xe0 ? "HWRM_TEMP_MONITOR_QUERY": \
	- ((x) == 0xd3 ? "HWRM_FWD_ASYNC_EVENT_CMPL": \
	- ((x) == 0xd2 ? "HWRM_FWD_RESP": \
	- ((x) == 0x1a ? "HWRM_FUNC_DRV_UNRGTR": \
	- ((x) == 0x1b ? "HWRM_FUNC_VF_RESC_FREE": \
	- ((x) == 0x1c ? "HWRM_FUNC_VF_VNIC_IDS_QUERY": \
	- ((x) == 0x1d ? "HWRM_FUNC_DRV_RGTR": \
	- ((x) == 0x1e ? "HWRM_FUNC_DRV_QVER": \
	- ((x) == 0x1f ? "HWRM_FUNC_BUF_RGTR": \
	- ((x) == 0x9a ? "HWRM_CFA_NTUPLE_FILTER_FREE": \
	- ((x) == 0x9b ? "HWRM_CFA_NTUPLE_FILTER_CFG": \
	- ((x) == 0x9c ? "HWRM_CFA_EM_FLOW_ALLOC": \
	- ((x) == 0x9d ? "HWRM_CFA_EM_FLOW_FREE": \
	- ((x) == 0x9e ? "HWRM_CFA_EM_FLOW_CFG": \
	- ((x) == 0xd1 ? "HWRM_REJECT_FWD_RESP": \
	- ((x) == 0xd0 ? "HWRM_EXEC_FWD_RESP": \
	- ((x) == 0xc8 ? "HWRM_FW_SET_TIME": \
	- ((x) == 0xc9 ? "HWRM_FW_GET_TIME": \
	- ((x) == 0xc0 ? "HWRM_FW_RESET": \
	- ((x) == 0xc1 ? "HWRM_FW_QSTATUS": \
	- ((x) == 0x70 ? "HWRM_VNIC_RSS_COS_LB_CTX_ALLOC": \
	- ((x) == 0x71 ? "HWRM_VNIC_RSS_COS_LB_CTX_FREE": \
	- ((x) == 0xb1 ? "HWRM_STAT_CTX_FREE": \
	- ((x) == 0xb0 ? "HWRM_STAT_CTX_ALLOC": \
	- ((x) == 0xb3 ? "HWRM_STAT_CTX_CLR_STATS": \
	- ((x) == 0xb2 ? "HWRM_STAT_CTX_QUERY": \
	- ((x) == 0xfff6 ? "HWRM_NVM_GET_DEV_INFO": \
	- ((x) == 0x61 ? "HWRM_RING_GRP_FREE": \
	- ((x) == 0x60 ? "HWRM_RING_GRP_ALLOC": \
	- ((x) == 0xf1 ? "HWRM_WOL_FILTER_FREE": \
	- ((x) == 0xf0 ? "HWRM_WOL_FILTER_ALLOC": \
	- ((x) == 0xf3 ? "HWRM_WOL_REASON_QCFG": \
	- ((x) == 0xf2 ? "HWRM_WOL_FILTER_QCFG": \
	- ((x) == 0xa0 ? "HWRM_TUNNEL_DST_PORT_QUERY": \
	- ((x) == 0xa1 ? "HWRM_TUNNEL_DST_PORT_ALLOC": \
	- ((x) == 0xa2 ? "HWRM_TUNNEL_DST_PORT_FREE": \
	- ((x) == 0xfffc ? "HWRM_NVM_RAW_DUMP": \
	- ((x) == 0xfffb ? "HWRM_NVM_GET_DIR_INFO": \
	- ((x) == 0xfffa ? "HWRM_NVM_GET_DIR_ENTRIES": \
	- ((x) == 0xe ? "HWRM_FUNC_BUF_UNRGTR": \
	- ((x) == 0xf ? "HWRM_FUNC_VF_CFG": \
	- ((x) == 0xffff ? "HWRM_NVM_RAW_WRITE_BLK": \
	- ((x) == 0xfffe ? "HWRM_NVM_WRITE": \
	- ((x) == 0xfffd ? "HWRM_NVM_READ": \
	- ((x) == 0x50 ? "HWRM_RING_ALLOC": \
	- ((x) == 0x51 ? "HWRM_RING_FREE": \
	- ((x) == 0x52 ? "HWRM_RING_CMPL_RING_QAGGINT_PARAMS": \
	- ((x) == 0x53 ? "HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS": \
	- ((x) == 0x4a ? "HWRM_VNIC_QCAPS": \
	- ((x) == 0x49 ? "HWRM_VNIC_PLCMODES_QCFG": \
	- ((x) == 0x48 ? "HWRM_VNIC_PLCMODES_CFG": \
	- ((x) == 0x47 ? "HWRM_VNIC_RSS_QCFG": \
	- ((x) == 0x46 ? "HWRM_VNIC_RSS_CFG": \
	- ((x) == 0x45 ? "HWRM_VNIC_TPA_QCFG": \
	- ((x) == 0x44 ? "HWRM_VNIC_TPA_CFG": \
	- ((x) == 0x43 ? "HWRM_VNIC_QCFG": \
	- ((x) == 0x42 ? "HWRM_VNIC_CFG": \
	- ((x) == 0x41 ? "HWRM_VNIC_FREE": \
	- ((x) == 0x40 ? "HWRM_VNIC_ALLOC": \
	- ((x) == 0x0 ? "HWRM_VER_GET": \
	- ((x) == 0xfff9 ? "HWRM_NVM_FIND_DIR_ENTRY": \
	- ((x) == 0xfff8 ? "HWRM_NVM_MOD_DIR_ENTRY": \
	- ((x) == 0xfff7 ? "HWRM_NVM_ERASE_DIR_ENTRY": \
	- ((x) == 0x5e ? "HWRM_RING_RESET": \
	- ((x) == 0xfff5 ? "HWRM_NVM_VERIFY_UPDATE": \
	- ((x) == 0xfff4 ? "HWRM_NVM_MODIFY": \
	- ((x) == 0xfff3 ? "HWRM_NVM_INSTALL_UPDATE": \
	- ((x) == 0x2a ? "HWRM_PORT_PHY_QCAPS": \
	- ((x) == 0x2c ? "HWRM_PORT_PHY_I2C_READ": \
	- ((x) == 0x2b ? "HWRM_PORT_PHY_I2C_WRITE": \
	- ((x) == 0x38 ? "HWRM_QUEUE_PRI2COS_CFG": \
	- ((x) == 0x39 ? "HWRM_QUEUE_COS2BW_QCFG": \
	- ((x) == 0x32 ? "HWRM_QUEUE_CFG": \
	- ((x) == 0x33 ? "HWRM_QUEUE_BUFFERS_QCFG": \
	- ((x) == 0x30 ? "HWRM_QUEUE_QPORTCFG": \
	- ((x) == 0x31 ? "HWRM_QUEUE_QCFG": \
	- ((x) == 0x36 ? "HWRM_QUEUE_PFCENABLE_CFG": \
	- ((x) == 0x37 ? "HWRM_QUEUE_PRI2COS_QCFG": \
	- ((x) == 0x34 ? "HWRM_QUEUE_BUFFERS_CFG": \
	- ((x) == 0x35 ? "HWRM_QUEUE_PFCENABLE_QCFG": \
	- ((x) == 0xff14 ? "HWRM_DBG_DUMP": \
	- ((x) == 0xff12 ? "HWRM_DBG_WRITE_DIRECT": \
	- ((x) == 0xff13 ? "HWRM_DBG_WRITE_INDIRECT": \
	- ((x) == 0xff10 ? "HWRM_DBG_READ_DIRECT": \
	- ((x) == 0xff11 ? "HWRM_DBG_READ_INDIRECT": \
	- ((x) == 0x25 ? "HWRM_PORT_CLR_STATS": \
	- ((x) == 0x24 ? "HWRM_PORT_LPBK_QSTATS": \
	- ((x) == 0x27 ? "HWRM_PORT_PHY_QCFG": \
	- ((x) == 0x26 ? "HWRM_PORT_LPBK_CLR_STATS": \
	- ((x) == 0x21 ? "HWRM_PORT_MAC_CFG": \
	- ((x) == 0x20 ? "HWRM_PORT_PHY_CFG": \
	- ((x) == 0x23 ? "HWRM_PORT_QSTATS": \
	- ((x) == 0x22 ? "HWRM_PORT_TS_QUERY": \
	- ((x) == 0x29 ? "HWRM_PORT_BLINK_LED": \
	- ((x) == 0x28 ? "HWRM_PORT_MAC_QCFG": \
	- ((x) == 0x3a ? "HWRM_QUEUE_COS2BW_CFG": \
	- "Unknown req_type"))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))
	+/* PSN Search Structure (8 bytes) */

	-/* Command numbering (8 bytes) */
	+struct sq_psn_search {
	+ uint32_t opcode_start_psn;
	+ /* The opcodes are software defined. */
	+ /* Start PSN. */
	+ #define SQ_PSN_SEARCH_START_PSN_MASK UINT32_C(0xffffff)
	+ #define SQ_PSN_SEARCH_START_PSN_SFT 0
	+ /* The opcodes are software defined. */
	+ #define SQ_PSN_SEARCH_OPCODE_MASK UINT32_C(0xff000000)
	+ #define SQ_PSN_SEARCH_OPCODE_SFT 24
	+ uint32_t flags_next_psn;
	+ /* Opcode specific flags. */
	+ /* Next PSN. Equal to the start PSN of the next WQE. */
	+ #define SQ_PSN_SEARCH_NEXT_PSN_MASK UINT32_C(0xffffff)
	+ #define SQ_PSN_SEARCH_NEXT_PSN_SFT 0
	+ /* Opcode specific flags. */
	+ #define SQ_PSN_SEARCH_FLAGS_MASK UINT32_C(0xff000000)
	+ #define SQ_PSN_SEARCH_FLAGS_SFT 24
	+} __attribute__((packed));

	-struct cmd_nums {
	- uint16_t req_type;
	+/* Send SQ WQE (40 bytes) */
	+
	+struct sq_send {
	+ uint8_t wqe_type;
	+ /* This field defines the type of SQ WQE. */
	+ /* Send */
	+ #define SQ_SEND_WQE_TYPE_SEND UINT32_C(0x0)
	/*
	- * This version of the specification defines the commands listed in the
	- * table below. The following are general implementation requirements
	- * for these commands: # All commands listed below that are marked
	- * neither reserved nor experimental shall be implemented by the HWRM. #
	- * A HWRM client compliant to this specification should not use commands
	- * outside of the list below. # A HWRM client compliant to this
	- * specification should not use command numbers marked reserved below. #
	- * A command marked experimental below may not be implemented by the
	- * HWRM. # A command marked experimental may change in the future
	- * version of the HWRM specification. # A command not listed below may
	- * be implemented by the HWRM. The behavior of commands that are not
	- * listed below is outside the scope of this specification.
	+ * Send with Immediate Allowed only on reliable connection (RC)
	+ * and unreliable datagram (UD) SQ's.
	*/
	- #define HWRM_VER_GET (UINT32_C(0x0))
	- #define HWRM_FUNC_BUF_UNRGTR (UINT32_C(0xe))
	- /* Experimental */
	- #define HWRM_FUNC_VF_CFG (UINT32_C(0xf))
	- /* Reserved for future use */
	- #define RESERVED1 (UINT32_C(0x10))
	- #define HWRM_FUNC_RESET (UINT32_C(0x11))
	- #define HWRM_FUNC_GETFID (UINT32_C(0x12))
	- #define HWRM_FUNC_VF_ALLOC (UINT32_C(0x13))
	- #define HWRM_FUNC_VF_FREE (UINT32_C(0x14))
	- #define HWRM_FUNC_QCAPS (UINT32_C(0x15))
	- #define HWRM_FUNC_QCFG (UINT32_C(0x16))
	- #define HWRM_FUNC_CFG (UINT32_C(0x17))
	- #define HWRM_FUNC_QSTATS (UINT32_C(0x18))
	- #define HWRM_FUNC_CLR_STATS (UINT32_C(0x19))
	- #define HWRM_FUNC_DRV_UNRGTR (UINT32_C(0x1a))
	- #define HWRM_FUNC_VF_RESC_FREE (UINT32_C(0x1b))
	- #define HWRM_FUNC_VF_VNIC_IDS_QUERY (UINT32_C(0x1c))
	- #define HWRM_FUNC_DRV_RGTR (UINT32_C(0x1d))
	- #define HWRM_FUNC_DRV_QVER (UINT32_C(0x1e))
	- #define HWRM_FUNC_BUF_RGTR (UINT32_C(0x1f))
	- #define HWRM_PORT_PHY_CFG (UINT32_C(0x20))
	- #define HWRM_PORT_MAC_CFG (UINT32_C(0x21))
	- /* Experimental */
	- #define HWRM_PORT_TS_QUERY (UINT32_C(0x22))
	- #define HWRM_PORT_QSTATS (UINT32_C(0x23))
	- #define HWRM_PORT_LPBK_QSTATS (UINT32_C(0x24))
	- /* Experimental */
	- #define HWRM_PORT_CLR_STATS (UINT32_C(0x25))
	- /* Experimental */
	- #define HWRM_PORT_LPBK_CLR_STATS (UINT32_C(0x26))
	- #define HWRM_PORT_PHY_QCFG (UINT32_C(0x27))
	- /* Experimental */
	- #define HWRM_PORT_MAC_QCFG (UINT32_C(0x28))
	- /* Experimental */
	- #define HWRM_PORT_BLINK_LED (UINT32_C(0x29))
	- /* Experimental */
	- #define HWRM_PORT_PHY_QCAPS (UINT32_C(0x2a))
	- /* Experimental */
	- #define HWRM_PORT_PHY_I2C_WRITE (UINT32_C(0x2b))
	- /* Experimental */
	- #define HWRM_PORT_PHY_I2C_READ (UINT32_C(0x2c))
	- #define HWRM_QUEUE_QPORTCFG (UINT32_C(0x30))
	- #define HWRM_QUEUE_QCFG (UINT32_C(0x31))
	- #define HWRM_QUEUE_CFG (UINT32_C(0x32))
	- #define HWRM_QUEUE_BUFFERS_QCFG (UINT32_C(0x33))
	- #define HWRM_QUEUE_BUFFERS_CFG (UINT32_C(0x34))
	- /* Experimental */
	- #define HWRM_QUEUE_PFCENABLE_QCFG (UINT32_C(0x35))
	- /* Experimental */
	- #define HWRM_QUEUE_PFCENABLE_CFG (UINT32_C(0x36))
	- /* Experimental */
	- #define HWRM_QUEUE_PRI2COS_QCFG (UINT32_C(0x37))
	- /* Experimental */
	- #define HWRM_QUEUE_PRI2COS_CFG (UINT32_C(0x38))
	- /* Experimental */
	- #define HWRM_QUEUE_COS2BW_QCFG (UINT32_C(0x39))
	- /* Experimental */
	- #define HWRM_QUEUE_COS2BW_CFG (UINT32_C(0x3a))
	- #define HWRM_VNIC_ALLOC (UINT32_C(0x40))
	- #define HWRM_VNIC_FREE (UINT32_C(0x41))
	- #define HWRM_VNIC_CFG (UINT32_C(0x42))
	- /* Experimental */
	- #define HWRM_VNIC_QCFG (UINT32_C(0x43))
	- #define HWRM_VNIC_TPA_CFG (UINT32_C(0x44))
	- /* Experimental */
	- #define HWRM_VNIC_TPA_QCFG (UINT32_C(0x45))
	- #define HWRM_VNIC_RSS_CFG (UINT32_C(0x46))
	- #define HWRM_VNIC_RSS_QCFG (UINT32_C(0x47))
	- #define HWRM_VNIC_PLCMODES_CFG (UINT32_C(0x48))
	- #define HWRM_VNIC_PLCMODES_QCFG (UINT32_C(0x49))
	- /* Experimental */
	- #define HWRM_VNIC_QCAPS (UINT32_C(0x4a))
	- #define HWRM_RING_ALLOC (UINT32_C(0x50))
	- #define HWRM_RING_FREE (UINT32_C(0x51))
	- #define HWRM_RING_CMPL_RING_QAGGINT_PARAMS (UINT32_C(0x52))
	- #define HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS (UINT32_C(0x53))
	- #define HWRM_RING_RESET (UINT32_C(0x5e))
	- #define HWRM_RING_GRP_ALLOC (UINT32_C(0x60))
	- #define HWRM_RING_GRP_FREE (UINT32_C(0x61))
	- #define HWRM_VNIC_RSS_COS_LB_CTX_ALLOC (UINT32_C(0x70))
	- #define HWRM_VNIC_RSS_COS_LB_CTX_FREE (UINT32_C(0x71))
	- #define HWRM_CFA_L2_FILTER_ALLOC (UINT32_C(0x90))
	- #define HWRM_CFA_L2_FILTER_FREE (UINT32_C(0x91))
	- #define HWRM_CFA_L2_FILTER_CFG (UINT32_C(0x92))
	- #define HWRM_CFA_L2_SET_RX_MASK (UINT32_C(0x93))
	- /* Reserved for future use */
	- #define RESERVED3 (UINT32_C(0x94))
	- #define HWRM_CFA_TUNNEL_FILTER_ALLOC (UINT32_C(0x95))
	- #define HWRM_CFA_TUNNEL_FILTER_FREE (UINT32_C(0x96))
	- /* Experimental */
	- #define HWRM_CFA_ENCAP_RECORD_ALLOC (UINT32_C(0x97))
	- /* Experimental */
	- #define HWRM_CFA_ENCAP_RECORD_FREE (UINT32_C(0x98))
	- #define HWRM_CFA_NTUPLE_FILTER_ALLOC (UINT32_C(0x99))
	- #define HWRM_CFA_NTUPLE_FILTER_FREE (UINT32_C(0x9a))
	- #define HWRM_CFA_NTUPLE_FILTER_CFG (UINT32_C(0x9b))
	- /* Experimental */
	- #define HWRM_CFA_EM_FLOW_ALLOC (UINT32_C(0x9c))
	- /* Experimental */
	- #define HWRM_CFA_EM_FLOW_FREE (UINT32_C(0x9d))
	- /* Experimental */
	- #define HWRM_CFA_EM_FLOW_CFG (UINT32_C(0x9e))
	- #define HWRM_TUNNEL_DST_PORT_QUERY (UINT32_C(0xa0))
	- #define HWRM_TUNNEL_DST_PORT_ALLOC (UINT32_C(0xa1))
	- #define HWRM_TUNNEL_DST_PORT_FREE (UINT32_C(0xa2))
	- #define HWRM_STAT_CTX_ALLOC (UINT32_C(0xb0))
	- #define HWRM_STAT_CTX_FREE (UINT32_C(0xb1))
	- #define HWRM_STAT_CTX_QUERY (UINT32_C(0xb2))
	- #define HWRM_STAT_CTX_CLR_STATS (UINT32_C(0xb3))
	- #define HWRM_FW_RESET (UINT32_C(0xc0))
	- #define HWRM_FW_QSTATUS (UINT32_C(0xc1))
	- /* Experimental */
	- #define HWRM_FW_SET_TIME (UINT32_C(0xc8))
	- /* Experimental */
	- #define HWRM_FW_GET_TIME (UINT32_C(0xc9))
	- #define HWRM_EXEC_FWD_RESP (UINT32_C(0xd0))
	- #define HWRM_REJECT_FWD_RESP (UINT32_C(0xd1))
	- #define HWRM_FWD_RESP (UINT32_C(0xd2))
	- #define HWRM_FWD_ASYNC_EVENT_CMPL (UINT32_C(0xd3))
	- #define HWRM_TEMP_MONITOR_QUERY (UINT32_C(0xe0))
	- /* Experimental */
	- #define HWRM_WOL_FILTER_ALLOC (UINT32_C(0xf0))
	- /* Experimental */
	- #define HWRM_WOL_FILTER_FREE (UINT32_C(0xf1))
	- /* Experimental */
	- #define HWRM_WOL_FILTER_QCFG (UINT32_C(0xf2))
	- /* Experimental */
	- #define HWRM_WOL_REASON_QCFG (UINT32_C(0xf3))
	- /* Experimental */
	- #define HWRM_DBG_READ_DIRECT (UINT32_C(0xff10))
	- /* Experimental */
	- #define HWRM_DBG_READ_INDIRECT (UINT32_C(0xff11))
	- /* Experimental */
	- #define HWRM_DBG_WRITE_DIRECT (UINT32_C(0xff12))
	- /* Experimental */
	- #define HWRM_DBG_WRITE_INDIRECT (UINT32_C(0xff13))
	- #define HWRM_DBG_DUMP (UINT32_C(0xff14))
	- #define HWRM_NVM_INSTALL_UPDATE (UINT32_C(0xfff3))
	- #define HWRM_NVM_MODIFY (UINT32_C(0xfff4))
	- #define HWRM_NVM_VERIFY_UPDATE (UINT32_C(0xfff5))
	- #define HWRM_NVM_GET_DEV_INFO (UINT32_C(0xfff6))
	- #define HWRM_NVM_ERASE_DIR_ENTRY (UINT32_C(0xfff7))
	- #define HWRM_NVM_MOD_DIR_ENTRY (UINT32_C(0xfff8))
	- #define HWRM_NVM_FIND_DIR_ENTRY (UINT32_C(0xfff9))
	- #define HWRM_NVM_GET_DIR_ENTRIES (UINT32_C(0xfffa))
	- #define HWRM_NVM_GET_DIR_INFO (UINT32_C(0xfffb))
	- #define HWRM_NVM_RAW_DUMP (UINT32_C(0xfffc))
	- #define HWRM_NVM_READ (UINT32_C(0xfffd))
	- #define HWRM_NVM_WRITE (UINT32_C(0xfffe))
	- #define HWRM_NVM_RAW_WRITE_BLK (UINT32_C(0xffff))
	- uint16_t unused_0[3];
	+ #define SQ_SEND_WQE_TYPE_SEND_W_IMMEAD UINT32_C(0x1)
	+ /*
	+ * Send with Invalidate. Allowed only on reliable connection
	+ * (RC) SQ's.
	+ */
	+ #define SQ_SEND_WQE_TYPE_SEND_W_INVALID UINT32_C(0x2)
	+ uint8_t flags;
	+ /*
	+ * Set if completion signaling is requested. If this bit is 0, and the
	+ * SQ is configured to support Unsignaled completion the controller
	+ * should not generate a CQE unless there was an error. This refers to
	+ * the CQE on the sender side. (The se flag refers to the receiver
	+ * side).
	+ */
	+ #define SQ_SEND_FLAGS_SIGNAL_COMP UINT32_C(0x1)
	+ /*
	+ * Indication to complete all previous RDMA Read or Atomic WQEs on the
	+ * SQ before executing this WQE. This flag must be zero for a UD send.
	+ */
	+ #define SQ_SEND_FLAGS_RD_OR_ATOMIC_FENCE UINT32_C(0x2)
	+ /*
	+ * For local invalidate request. Indicate to complete all previous SQ's
	+ * WQEs before executing this WQE. This flag must be zero for a UD send.
	+ */
	+ #define SQ_SEND_FLAGS_UC_FENCE UINT32_C(0x4)
	+ /*
	+ * Solicit event flag. Indication sent in BTH header to the receiver to
	+ * generate a Completion Event Notification, i.e. CNQE. This bit should
	+ * be set only in the last (or only) packet of the message.
	+ */
	+ #define SQ_SEND_FLAGS_SE UINT32_C(0x8)
	+ /*
	+ * Indicate that inline data is posted to the SQ in the data area of
	+ * this WQE.
	+ */
	+ #define SQ_SEND_FLAGS_INLINE UINT32_C(0x10)
	+ uint8_t wqe_size;
	+ /*
	+ * The number of 16 bytes chunks of data including this first word of
	+ * the request that are a valid part of the request. The valid 16 bytes
	+ * units other than the WQE structure can be SGEs (Scatter Gather
	+ * Elements) OR inline data. While this field defines the valid WQE
	+ * size. The actual total WQE size is always 128B.
	+ */
	+ uint8_t reserved8_1;
	+ uint32_t inv_key_or_imm_data;
	+ /*
	+ * Either invalidate key (R_Key of the remote host) that will be send
	+ * with IETH (Invalidate ETH) if wqe_type is of Send with Invalidate, or
	+ * immediate value that will be sent with ImmDt header if wqe_type is
	+ * Send with Immediate.
	+ */
	+ uint32_t length;
	+ /* This field represents a 32-bit total data length, in bytes. */
	+ uint32_t q_key;
	+ /*
	+ * When in the SQ of a UD QP, indicates the q_key to be used in the
	+ * transmitted packet. However, if the most significant bit of this
	+ * field is set, then the q_key will be taken from QP context, rather
	+ * than from this field. When in the SQ of a non-UD QP, this field is
	+ * reserved and should be filled with zeros.
	+ */
	+ uint32_t dst_qp;
	+ /*
	+ * When in the SQ of a UD QP, indicates the destination QP to be used in
	+ * the transmitted packet. When in the SQ of a non-UD QP, this field is
	+ * reserved and should be filled with zeros.
	+ */
	+ #define SQ_SEND_DST_QP_MASK UINT32_C(0xffffff)
	+ #define SQ_SEND_DST_QP_SFT 0
	+ #define SQ_SEND_RESERVED8_2_MASK UINT32_C(0xff000000)
	+ #define SQ_SEND_RESERVED8_2_SFT 24
	+ uint32_t avid;
	+ /* This field is reserved for future expansion of the AVID. */
	+ /*
	+ * If the serv_type is 'UD', then this field supplies the AVID (Address
	+ * Vector ID).
	+ */
	+ #define SQ_SEND_AVID_MASK UINT32_C(0xfffff)
	+ #define SQ_SEND_AVID_SFT 0
	+ /* This field is reserved for future expansion of the AVID. */
	+ #define SQ_SEND_RESERVED_AVID_MASK UINT32_C(0xfff00000)
	+ #define SQ_SEND_RESERVED_AVID_SFT 20
	+ uint64_t reserved64;
	+ uint32_t data[24];
	+ /*
	+ * When inline=0, then this area is filled with from 1 to 6 SGEs based
	+ * on the wqe_size field. When inline=1, this area is filled with
	+ * payload data for the send based on the length_or_AVID field. Bits
	+ * [7:0] of word 0 hold the first byte to go out on the wire.
	+ */
	} __attribute__((packed));

	-#define GET_HWRM_ERROR_CODE(x) \
	- ((x) == 0xf ? "HWRM_ERROR": \
	- ((x) == 0xffff ? "CMD_NOT_SUPPORTED": \
	- ((x) == 0xfffe ? "UNKNOWN_ERR": \
	- ((x) == 0x4 ? "RESOURCE_ALLOC_ERROR": \
	- ((x) == 0x5 ? "INVALID_FLAGS": \
	- ((x) == 0x6 ? "INVALID_ENABLES": \
	- ((x) == 0x0 ? "SUCCESS": \
	- ((x) == 0x1 ? "FAIL": \
	- ((x) == 0x2 ? "INVALID_PARAMS": \
	- ((x) == 0x3 ? "RESOURCE_ACCESS_DENIED": \
	- "Unknown error_code"))))))))))
	+/* Send Raw Ethernet and QP1 SQ WQE (40 bytes) */

	-/* Return Codes (8 bytes) */
	+struct sq_send_raweth_qp1 {
	+ uint8_t wqe_type;
	+ /* This field defines the type of SQ WQE. */
	+ /* Send */
	+ #define SQ_SEND_RAWETH_QP1_WQE_TYPE_SEND UINT32_C(0x0)
	+ uint8_t flags;
	+ /*
	+ * Set if completion signaling is requested. If this bit is 0, and the
	+ * SQ is configured to support Unsignaled completion the controller
	+ * should not generate a CQE unless there was an error. This refers to
	+ * the CQE on the sender side. (The se flag refers to the receiver
	+ * side).
	+ */
	+ #define SQ_SEND_RAWETH_QP1_FLAGS_SIGNAL_COMP UINT32_C(0x1)
	+ /* This flag must be zero for a Raweth or QP1 send. */
	+ #define SQ_SEND_RAWETH_QP1_FLAGS_RD_OR_ATOMIC_FENCE UINT32_C(0x2)
	+ /* This flag must be zero for a Raweth or QP1 send. */
	+ #define SQ_SEND_RAWETH_QP1_FLAGS_UC_FENCE UINT32_C(0x4)
	+ /* This flag must be zero for a Raweth or QP1 send. */
	+ #define SQ_SEND_RAWETH_QP1_FLAGS_SE UINT32_C(0x8)
	+ /*
	+ * Indicate that inline data is posted to the SQ in the data area of
	+ * this WQE.
	+ */
	+ #define SQ_SEND_RAWETH_QP1_FLAGS_INLINE UINT32_C(0x10)
	+ uint8_t wqe_size;
	+ /*
	+ * The number of 16 bytes chunks of data including this first word of
	+ * the request that are a valid part of the request. The valid 16 bytes
	+ * units other than the WQE structure can be SGEs (Scatter Gather
	+ * Elements) OR inline data. While this field defines the valid WQE
	+ * size. The actual total WQE size is always 128B.
	+ */
	+ uint8_t reserved8;
	+ uint16_t lflags;
	+ /*
	+ * All bits in this field must be valid on the first BD of a packet.
	+ * Their value on other BDs of the packet will be ignored.
	+ */
	+ /*
	+ * If set to 1, the controller replaces the TCP/UPD checksum fields of
	+ * normal TCP/UPD checksum, or the inner TCP/UDP checksum field of the
	+ * encapsulated TCP/UDP packets with the hardware calculated TCP/UDP
	+ * checksum for the packet associated with this descriptor. This bit
	+ * must be valid on the first BD of a packet.
	+ */
	+ #define SQ_SEND_RAWETH_QP1_LFLAGS_TCP_UDP_CHKSUM UINT32_C(0x1)
	+ /*
	+ * If set to 1, the controller replaces the IP checksum of the normal
	+ * packets, or the inner IP checksum of the encapsulated packets with
	+ * the hardware calculated IP checksum for the packet associated with
	+ * this descriptor. This bit must be valid on the first BD of a packet.
	+ */
	+ #define SQ_SEND_RAWETH_QP1_LFLAGS_IP_CHKSUM UINT32_C(0x2)
	+ /*
	+ * If set to 1, the controller will not append an Ethernet CRC to the
	+ * end of the frame. This bit must be valid on the first BD of a packet.
	+ * Packet must be 64B or longer when this flag is set. It is not useful
	+ * to use this bit with any form of TX offload such as CSO or LSO. The
	+ * intent is that the packet from the host already has a valid Ethernet
	+ * CRC on the packet.
	+ */
	+ #define SQ_SEND_RAWETH_QP1_LFLAGS_NOCRC UINT32_C(0x4)
	+ /*
	+ * If set to 1, the device will record the time at which the packet was
	+ * actually transmitted at the TX MAC. This bit must be valid on the
	+ * first BD of a packet.
	+ */
	+ #define SQ_SEND_RAWETH_QP1_LFLAGS_STAMP UINT32_C(0x8)
	+ /*
	+ * If set to 1, The controller replaces the tunnel IP checksum field
	+ * with hardware calculated IP checksum for the IP header of the packet
	+ * associated with this descriptor. In case of VXLAN, the controller
	+ * also replaces the outer header UDP checksum with hardware calculated
	+ * UDP checksum for the packet associated with this descriptor.
	+ */
	+ #define SQ_SEND_RAWETH_QP1_LFLAGS_T_IP_CHKSUM UINT32_C(0x10)
	+ /* This bit is reserved and should be zero. */
	+ #define SQ_SEND_RAWETH_QP1_LFLAGS_RESERVED1_1 UINT32_C(0x20)
	+ /* This bit is reserved and should be zero. */
	+ #define SQ_SEND_RAWETH_QP1_LFLAGS_RESERVED1_2 UINT32_C(0x40)
	+ /* This bit is reserved and should be zero. */
	+ #define SQ_SEND_RAWETH_QP1_LFLAGS_RESERVED1_3 UINT32_C(0x80)
	+ /*
	+ * If set to '1', then the RoCE ICRC will be appended to the packet.
	+ * Packet must be a valid RoCE format packet.
	+ */
	+ #define SQ_SEND_RAWETH_QP1_LFLAGS_ROCE_CRC UINT32_C(0x100)
	+ /*
	+ * If set to '1', then the FCoE CRC will be appended to the packet.
	+ * Packet must be a valid FCoE format packet.
	+ */
	+ #define SQ_SEND_RAWETH_QP1_LFLAGS_FCOE_CRC UINT32_C(0x200)
	+ uint16_t cfa_action;
	+ /*
	+ * This value selects a CFA action to perform on the packet. Set this
	+ * value to zero if no CFA action is desired. This value must be valid
	+ * on the first BD of a packet.
	+ */
	+ uint32_t length;
	+ /*
	+ * This field represents a 32-bit total data length, in bytes. Note,
	+ * however, that the length cannot exceed the MTU.
	+ */
	+ uint32_t reserved32_1;
	+ uint32_t cfa_meta;
	+ /*
	+ * This value is action meta-data that defines CFA edit operations that
	+ * are done in addition to any action editing.
	+ */
	+ /* When key=1, This is the VLAN tag VID value. */
	+ #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_VID_MASK UINT32_C(0xfff)
	+ #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_VID_SFT 0
	+ /* When key=1, This is the VLAN tag DE value. */
	+ #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_DE UINT32_C(0x1000)
	+ /* When key=1, This is the VLAN tag PRI value. */
	+ #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_PRI_MASK UINT32_C(0xe000)
	+ #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_PRI_SFT 13
	+ /* When key=1, This is the VLAN tag TPID select value. */
	+ #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_MASK UINT32_C(0x70000)
	+ #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_SFT 16
	+ /* 0x88a8 */
	+ #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_TPID88A8 (UINT32_C(0x0) << 16)
	+ /* 0x8100 */
	+ #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_TPID8100 (UINT32_C(0x1) << 16)
	+ /* 0x9100 */
	+ #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_TPID9100 (UINT32_C(0x2) << 16)
	+ /* 0x9200 */
	+ #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_TPID9200 (UINT32_C(0x3) << 16)
	+ /* 0x9300 */
	+ #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_TPID9300 (UINT32_C(0x4) << 16)
	+ /* Value programmed in CFA VLANTPID register. */
	+ #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_TPIDCFG (UINT32_C(0x5) << 16)
	+ #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_LAST SQ_SEND_RAWETH_QP1_CFA_META_VLAN_TPID_TPIDCFG
	+ /* When key=1, This is the VLAN tag TPID select value. */
	+ #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_RESERVED_MASK UINT32_C(0xff80000)
	+ #define SQ_SEND_RAWETH_QP1_CFA_META_VLAN_RESERVED_SFT 19
	+ /*
	+ * This field identifies the type of edit to be performed on the packet.
	+ * This value must be valid on the first BD of a packet.
	+ */
	+ #define SQ_SEND_RAWETH_QP1_CFA_META_KEY_MASK UINT32_C(0xf0000000)
	+ #define SQ_SEND_RAWETH_QP1_CFA_META_KEY_SFT 28
	+ /* No editing */
	+ #define SQ_SEND_RAWETH_QP1_CFA_META_KEY_NONE (UINT32_C(0x0) << 28)
	+ /*
	+ * - meta[17:16] - TPID select value (0 = 0x8100). - meta[15:12]
	+ * - PRI/DE value. - meta[11:0] - VID value.
	+ */
	+ #define SQ_SEND_RAWETH_QP1_CFA_META_KEY_VLAN_TAG (UINT32_C(0x1) << 28)
	+ #define SQ_SEND_RAWETH_QP1_CFA_META_KEY_LAST SQ_SEND_RAWETH_QP1_CFA_META_KEY_VLAN_TAG
	+ uint32_t reserved32_2;
	+ uint64_t reserved64;
	+ uint32_t data[24];
	+ /*
	+ * When inline=0, then this area is filled with from 1 to 6 SGEs based
	+ * on the wqe_size field. When inline=1, this area is filled with
	+ * payload data for the send based on the length_or_AVID field. Bits
	+ * [7:0] of word 0 hold the first byte to go out on the wire.
	+ */
	+} __attribute__((packed));

	-struct ret_codes {
	- uint16_t error_code;
	- /* These are numbers assigned to return/error codes. */
	- /* Request was successfully executed by the HWRM. */
	- #define HWRM_ERR_CODE_SUCCESS (UINT32_C(0x0))
	- /* THe HWRM failed to execute the request. */
	- #define HWRM_ERR_CODE_FAIL (UINT32_C(0x1))
	- /* The request contains invalid argument(s) or input parameters. */
	- #define HWRM_ERR_CODE_INVALID_PARAMS (UINT32_C(0x2))
	+/* RDMA SQ WQE (40 bytes) */
	+
	+struct sq_rdma {
	+ uint8_t wqe_type;
	+ /* This field defines the type of SQ WQE. */
	+ /* RDMA Write. Allowed only on reliable connection (RC) SQ's. */
	+ #define SQ_RDMA_WQE_TYPE_WRITE_WQE UINT32_C(0x4)
	/*
	- * The requester is not allowed to access the requested
	- * resource. This error code shall be provided in a response to
	- * a request to query or modify an existing resource that is not
	- * accessible by the requester.
	+ * RDMA Write with Immediate. Allowed only on reliable
	+ * connection (RC) SQ's.
	*/
	- #define HWRM_ERR_CODE_RESOURCE_ACCESS_DENIED (UINT32_C(0x3))
	+ #define SQ_RDMA_WQE_TYPE_WRITE_W_IMMEAD UINT32_C(0x5)
	+ /* RDMA Read. Allowed only on reliable connection (RC) SQ's. */
	+ #define SQ_RDMA_WQE_TYPE_READ_WQE UINT32_C(0x6)
	+ uint8_t flags;
	/*
	- * The HWRM is unable to allocate the requested resource. This
	- * code only applies to requests for HWRM resource allocations.
	+ * Set if completion signaling is requested. If this bit is 0, and the
	+ * SQ is configured to support Unsignaled completion the controller
	+ * should not generate a CQE unless there was an error. This refer to
	+ * CQE on the sender side (se_flag refer to the receiver side)
	*/
	- #define HWRM_ERR_CODE_RESOURCE_ALLOC_ERROR (UINT32_C(0x4))
	- /* Invalid combination of flags is specified in the request. */
	- #define HWRM_ERR_CODE_INVALID_FLAGS (UINT32_C(0x5))
	+ #define SQ_RDMA_FLAGS_SIGNAL_COMP UINT32_C(0x1)
	/*
	- * Invalid combination of enables fields is specified in the
	- * request.
	+ * Indication to complete all previous RDMA Read or Atomic WQEs on the
	+ * SQ before executing this WQE
	*/
	- #define HWRM_ERR_CODE_INVALID_ENABLES (UINT32_C(0x6))
	- /* Generic HWRM execution error that represents an internal error. */
	- #define HWRM_ERR_CODE_HWRM_ERROR (UINT32_C(0xf))
	- /* Unknown error */
	- #define HWRM_ERR_CODE_UNKNOWN_ERR (UINT32_C(0xfffe))
	- /* Unsupported or invalid command */
	- #define HWRM_ERR_CODE_CMD_NOT_SUPPORTED (UINT32_C(0xffff))
	- uint16_t unused_0[3];
	+ #define SQ_RDMA_FLAGS_RD_OR_ATOMIC_FENCE UINT32_C(0x2)
	+ /*
	+ * Unconditional fence. Indicate to complete all previous SQ's WQEs
	+ * before executing this WQE.
	+ */
	+ #define SQ_RDMA_FLAGS_UC_FENCE UINT32_C(0x4)
	+ /*
	+ * Solicit event. Indication sent in BTH header to the receiver to
	+ * generate a Completion Event Notification, i.e. CNQE. This bit should
	+ * be set only in the last (or only) packet of the message.
	+ */
	+ #define SQ_RDMA_FLAGS_SE UINT32_C(0x8)
	+ /*
	+ * Indicate that inline data is posted to the SQ following this WQE.
	+ * This bit may be 1 only for write operations.
	+ */
	+ #define SQ_RDMA_FLAGS_INLINE UINT32_C(0x10)
	+ uint8_t wqe_size;
	+ /*
	+ * The number of 16 bytes chunks of data including this first wqe of the
	+ * request that are a valid part of the request. The valid 16 bytes
	+ * units other than the WQE structure can be SGEs (Scatter Gather
	+ * Elements) OR inline data. While this field defines the valid WQE
	+ * size. The actual total WQE size is always 128B.
	+ */
	+ uint8_t reserved8;
	+ uint32_t imm_data;
	+ /*
	+ * Immediate data - valid for RDMA Write with immediate and causes the
	+ * controller to add immDt header with this value
	+ */
	+ uint32_t length;
	+ /* Total data length in bytes */
	+ uint32_t reserved32_1;
	+ uint64_t remote_va;
	+ /* Remote VA sent to the destination QP */
	+ uint32_t remote_key;
	+ /*
	+ * R_Key provided by remote node when the connection was established and
	+ * placed in the RETH header. It identify the MRW on the remote host
	+ */
	+ uint32_t reserved32_2;
	+ uint32_t data[24];
	+ /*
	+ * When inline=0, then this area is filled with from 1 to 6 SGEs based
	+ * on the wqe_size field. When inline=1, this area is filled with
	+ * payload data for the write based on the length field. Bits [7:0] of
	+ * word 0 hold the first byte to go out on the wire.
	+ */
	} __attribute__((packed));

	-/* Output (16 bytes) */
	+/* Atomic SQ WQE (40 bytes) */

	-struct hwrm_err_output {
	- uint16_t error_code;
	+struct sq_atomic {
	+ uint8_t wqe_type;
	+ /* This field defines the type of SQ WQE. */
	/*
	- * Pass/Fail or error type Note: receiver to verify the in parameters,
	- * and fail the call with an error when appropriate
	+ * Atomic Compare/Swap. Allowed only on reliable connection (RC)
	+ * SQ's.
	*/
	- uint16_t req_type;
	- /* This field returns the type of original request. */
	- uint16_t seq_id;
	- /* This field provides original sequence number of the command. */
	- uint16_t resp_len;
	+ #define SQ_ATOMIC_WQE_TYPE_ATOMIC_CS UINT32_C(0x8)
	+ /* Atomic Fetch/Add. Allowed only on reliable connection (RC) SQ's. */
	+ #define SQ_ATOMIC_WQE_TYPE_ATOMIC_FA UINT32_C(0xb)
	+ uint8_t flags;
	/*
	- * This field is the length of the response in bytes. The last byte of
	- * the response is a valid flag that will read as '1' when the command
	- * has been completely written to memory.
	+ * Set if completion signaling is requested. If this bit is 0, and the
	+ * SQ is configured to support Unsignaled completion the controller
	+ * should not generate a CQE unless there was an error. This refer to
	+ * CQE on the sender side (se_flag refer to the receiver side)
	*/
	- uint32_t opaque_0;
	- /* debug info for this error response. */
	- uint16_t opaque_1;
	- /* debug info for this error response. */
	- uint8_t cmd_err;
	+ #define SQ_ATOMIC_FLAGS_SIGNAL_COMP UINT32_C(0x1)
	/*
	- * In the case of an error response, command specific error code is
	- * returned in this field.
	+ * Indication to complete all previous RDMA Read or Atomic WQEs on the
	+ * SQ before executing this WQE
	*/
	- uint8_t valid;
	+ #define SQ_ATOMIC_FLAGS_RD_OR_ATOMIC_FENCE UINT32_C(0x2)
	/*
	- * This field is used in Output records to indicate that the output is
	- * completely written to RAM. This field should be read as '1' to
	- * indicate that the output has been completely written. When writing a
	- * command completion or response to an internal processor, the order of
	- * writes has to be such that this field is written last.
	+ * Unconditional fence. Indicate to complete all previous SQ's WQEs
	+ * before executing this WQE.
	*/
	+ #define SQ_ATOMIC_FLAGS_UC_FENCE UINT32_C(0x4)
	+ /*
	+ * Solicit event. Indication sent in BTH header to the receiver to
	+ * generate a Completion Event Notification, i.e. CNQE. This bit should
	+ * be set only in the last (or only) packet of the message.
	+ */
	+ #define SQ_ATOMIC_FLAGS_SE UINT32_C(0x8)
	+ /* NA for this WQE. */
	+ #define SQ_ATOMIC_FLAGS_INLINE UINT32_C(0x10)
	+ uint16_t reserved16;
	+ uint32_t remote_key;
	+ /*
	+ * R_Key provided by remote node when the connection was established and
	+ * placed in the AETH header. It identify the MRW on the remote host
	+ */
	+ uint64_t remote_va;
	+ /* Remote VA sent to the destination QP */
	+ uint64_t swap_data;
	+ /* Data value to be placed in remote host specified address */
	+ uint64_t cmp_data;
	+ /*
	+ * Data value to be compared with the value in the remote host specified
	+ * address
	+ */
	+ uint32_t data[24];
	+ /*
	+ * The first 16B of the data field must be filled with a single SGE.
	+ * This will be used to store the return value from the Atomic Ack
	+ * response. The size of the single SGE must be 8B.
	+ */
	} __attribute__((packed));

	-/* Port Tx Statistics Formats (408 bytes) */
	+/* Local Invalidate SQ WQE (40 bytes) */

	-struct tx_port_stats {
	- uint64_t tx_64b_frames;
	- /* Total Number of 64 Bytes frames transmitted */
	- uint64_t tx_65b_127b_frames;
	- /* Total Number of 65-127 Bytes frames transmitted */
	- uint64_t tx_128b_255b_frames;
	- /* Total Number of 128-255 Bytes frames transmitted */
	- uint64_t tx_256b_511b_frames;
	- /* Total Number of 256-511 Bytes frames transmitted */
	- uint64_t tx_512b_1023b_frames;
	- /* Total Number of 512-1023 Bytes frames transmitted */
	- uint64_t tx_1024b_1518_frames;
	- /* Total Number of 1024-1518 Bytes frames transmitted */
	- uint64_t tx_good_vlan_frames;
	+struct sq_localinvalidate {
	+ uint8_t wqe_type;
	+ /* This field defines the type of SQ WQE. */
	+ /* Local Invalidate. Allowed only on reliable connection (RC) SQ's. */
	+ #define SQ_LOCALINVALIDATE_WQE_TYPE_LOCAL_INVALID UINT32_C(0xc)
	+ uint8_t flags;
	/*
	- * Total Number of each good VLAN (exludes FCS errors) frame transmitted
	- * which is 1519 to 1522 bytes in length inclusive (excluding framing
	- * bits but including FCS bytes).
	+ * Set if completion signaling is requested. If this bit is 0, and the
	+ * SQ is configured to support Unsignaled completion the controller
	+ * should not generate a CQE unless there was an error. This refer to
	+ * CQE on the sender side (se_flag refer to the receiver side)
	*/
	- uint64_t tx_1519b_2047_frames;
	- /* Total Number of 1519-2047 Bytes frames transmitted */
	- uint64_t tx_2048b_4095b_frames;
	- /* Total Number of 2048-4095 Bytes frames transmitted */
	- uint64_t tx_4096b_9216b_frames;
	- /* Total Number of 4096-9216 Bytes frames transmitted */
	- uint64_t tx_9217b_16383b_frames;
	- /* Total Number of 9217-16383 Bytes frames transmitted */
	- uint64_t tx_good_frames;
	- /* Total Number of good frames transmitted */
	- uint64_t tx_total_frames;
	- /* Total Number of frames transmitted */
	- uint64_t tx_ucast_frames;
	- /* Total number of unicast frames transmitted */
	- uint64_t tx_mcast_frames;
	- /* Total number of multicast frames transmitted */
	- uint64_t tx_bcast_frames;
	- /* Total number of broadcast frames transmitted */
	- uint64_t tx_pause_frames;
	- /* Total number of PAUSE control frames transmitted */
	- uint64_t tx_pfc_frames;
	- /* Total number of PFC/per-priority PAUSE control frames transmitted */
	- uint64_t tx_jabber_frames;
	- /* Total number of jabber frames transmitted */
	- uint64_t tx_fcs_err_frames;
	- /* Total number of frames transmitted with FCS error */
	- uint64_t tx_control_frames;
	- /* Total number of control frames transmitted */
	- uint64_t tx_oversz_frames;
	- /* Total number of over-sized frames transmitted */
	- uint64_t tx_single_dfrl_frames;
	- /* Total number of frames with single deferral */
	- uint64_t tx_multi_dfrl_frames;
	- /* Total number of frames with multiple deferrals */
	- uint64_t tx_single_coll_frames;
	- /* Total number of frames with single collision */
	- uint64_t tx_multi_coll_frames;
	- /* Total number of frames with multiple collisions */
	- uint64_t tx_late_coll_frames;
	- /* Total number of frames with late collisions */
	- uint64_t tx_excessive_coll_frames;
	- /* Total number of frames with excessive collisions */
	- uint64_t tx_frag_frames;
	- /* Total number of fragmented frames transmitted */
	- uint64_t tx_err;
	- /* Total number of transmit errors */
	- uint64_t tx_tagged_frames;
	- /* Total number of single VLAN tagged frames transmitted */
	- uint64_t tx_dbl_tagged_frames;
	- /* Total number of double VLAN tagged frames transmitted */
	- uint64_t tx_runt_frames;
	- /* Total number of runt frames transmitted */
	- uint64_t tx_fifo_underruns;
	- /* Total number of TX FIFO under runs */
	- uint64_t tx_pfc_ena_frames_pri0;
	- /* Total number of PFC frames with PFC enabled bit for Pri 0 transmitted */
	- uint64_t tx_pfc_ena_frames_pri1;
	- /* Total number of PFC frames with PFC enabled bit for Pri 1 transmitted */
	- uint64_t tx_pfc_ena_frames_pri2;
	- /* Total number of PFC frames with PFC enabled bit for Pri 2 transmitted */
	- uint64_t tx_pfc_ena_frames_pri3;
	- /* Total number of PFC frames with PFC enabled bit for Pri 3 transmitted */
	- uint64_t tx_pfc_ena_frames_pri4;
	- /* Total number of PFC frames with PFC enabled bit for Pri 4 transmitted */
	- uint64_t tx_pfc_ena_frames_pri5;
	- /* Total number of PFC frames with PFC enabled bit for Pri 5 transmitted */
	- uint64_t tx_pfc_ena_frames_pri6;
	- /* Total number of PFC frames with PFC enabled bit for Pri 6 transmitted */
	- uint64_t tx_pfc_ena_frames_pri7;
	- /* Total number of PFC frames with PFC enabled bit for Pri 7 transmitted */
	- uint64_t tx_eee_lpi_events;
	- /* Total number of EEE LPI Events on TX */
	- uint64_t tx_eee_lpi_duration;
	- /* EEE LPI Duration Counter on TX */
	- uint64_t tx_llfc_logical_msgs;
	- /* Total number of Link Level Flow Control (LLFC) messages transmitted */
	- uint64_t tx_hcfc_msgs;
	- /* Total number of HCFC messages transmitted */
	- uint64_t tx_total_collisions;
	- /* Total number of TX collisions */
	- uint64_t tx_bytes;
	- /* Total number of transmitted bytes */
	- uint64_t tx_xthol_frames;
	- /* Total number of end-to-end HOL frames */
	- uint64_t tx_stat_discard;
	- /* Total Tx Drops per Port reported by STATS block */
	- uint64_t tx_stat_error;
	- /* Total Tx Error Drops per Port reported by STATS block */
	+ #define SQ_LOCALINVALIDATE_FLAGS_SIGNAL_COMP UINT32_C(0x1)
	+ /*
	+ * Indication to complete all previous RDMA Read or Atomic WQEs on the
	+ * SQ before executing this WQE
	+ */
	+ #define SQ_LOCALINVALIDATE_FLAGS_RD_OR_ATOMIC_FENCE UINT32_C(0x2)
	+ /*
	+ * Unconditional fence. Indicate to complete all previous SQ's WQEs
	+ * before executing this WQE.
	+ */
	+ #define SQ_LOCALINVALIDATE_FLAGS_UC_FENCE UINT32_C(0x4)
	+ /*
	+ * Solicit event. Indication sent in BTH header to the receiver to
	+ * generate a Completion Event Notification, i.e. CNQE. This bit should
	+ * be set only in the last (or only) packet of the message.
	+ */
	+ #define SQ_LOCALINVALIDATE_FLAGS_SE UINT32_C(0x8)
	+ /* NA for this WQE */
	+ #define SQ_LOCALINVALIDATE_FLAGS_INLINE UINT32_C(0x10)
	+ uint16_t reserved16;
	+ uint32_t inv_l_key;
	+ /*
	+ * The local key for the MR/W to invalidate; 24 msb of the key are used
	+ * to index the MRW table, 8 lsb are compared with the 8 bit key in the
	+ * MRWC
	+ */
	+ uint64_t reserved64;
	+ uint32_t reserved128[4];
	+ uint32_t data[24];
	+ /* The data field for local invalidate is not used. */
	} __attribute__((packed));

	-/* Port Rx Statistics Formats (528 bytes) */
	+/* FR-PMR SQ WQE (40 bytes) */

	-struct rx_port_stats {
	- uint64_t rx_64b_frames;
	- /* Total Number of 64 Bytes frames received */
	- uint64_t rx_65b_127b_frames;
	- /* Total Number of 65-127 Bytes frames received */
	- uint64_t rx_128b_255b_frames;
	- /* Total Number of 128-255 Bytes frames received */
	- uint64_t rx_256b_511b_frames;
	- /* Total Number of 256-511 Bytes frames received */
	- uint64_t rx_512b_1023b_frames;
	- /* Total Number of 512-1023 Bytes frames received */
	- uint64_t rx_1024b_1518_frames;
	- /* Total Number of 1024-1518 Bytes frames received */
	- uint64_t rx_good_vlan_frames;
	+struct sq_fr_pmr {
	+ uint8_t wqe_type;
	+ /* This field defines the type of SQ WQE. */
	/*
	- * Total Number of each good VLAN (exludes FCS errors) frame received
	- * which is 1519 to 1522 bytes in length inclusive (excluding framing
	- * bits but including FCS bytes).
	+ * FR-PMR (Fast Register Physical Memory Region) Allowed only on
	+ * reliable connection (RC) SQ's.
	*/
	- uint64_t rx_1519b_2047b_frames;
	- /* Total Number of 1519-2047 Bytes frames received */
	- uint64_t rx_2048b_4095b_frames;
	- /* Total Number of 2048-4095 Bytes frames received */
	- uint64_t rx_4096b_9216b_frames;
	- /* Total Number of 4096-9216 Bytes frames received */
	- uint64_t rx_9217b_16383b_frames;
	- /* Total Number of 9217-16383 Bytes frames received */
	- uint64_t rx_total_frames;
	- /* Total number of frames received */
	- uint64_t rx_ucast_frames;
	- /* Total number of unicast frames received */
	- uint64_t rx_mcast_frames;
	- /* Total number of multicast frames received */
	- uint64_t rx_bcast_frames;
	- /* Total number of broadcast frames received */
	- uint64_t rx_fcs_err_frames;
	- /* Total number of received frames with FCS error */
	- uint64_t rx_ctrl_frames;
	- /* Total number of control frames received */
	- uint64_t rx_pause_frames;
	- /* Total number of PAUSE frames received */
	- uint64_t rx_pfc_frames;
	- /* Total number of PFC frames received */
	- uint64_t rx_unsupported_opcode_frames;
	- /* Total number of frames received with an unsupported opcode */
	- uint64_t rx_unsupported_da_pausepfc_frames;
	+ #define SQ_FR_PMR_WQE_TYPE_FR_PMR UINT32_C(0xd)
	+ uint8_t flags;
	/*
	- * Total number of frames received with an unsupported DA for pause and
	- * PFC
	+ * Set if completion signaling is requested. If this bit is 0, and the
	+ * SQ is configured to support Unsignaled completion the controller
	+ * should not generate a CQE unless there was an error. This refer to
	+ * CQE on the sender side (se_flag refer to the receiver side)
	*/
	- uint64_t rx_wrong_sa_frames;
	- /* Total number of frames received with an unsupported SA */
	- uint64_t rx_align_err_frames;
	- /* Total number of received packets with alignment error */
	- uint64_t rx_oor_len_frames;
	- /* Total number of received frames with out-of-range length */
	- uint64_t rx_code_err_frames;
	- /* Total number of received frames with error termination */
	- uint64_t rx_false_carrier_frames;
	+ #define SQ_FR_PMR_FLAGS_SIGNAL_COMP UINT32_C(0x1)
	/*
	- * Total number of received frames with a false carrier is detected
	- * during idle, as defined by RX_ER samples active and RXD is 0xE. The
	- * event is reported along with the statistics generated on the next
	- * received frame. Only one false carrier condition can be detected and
	- * logged between frames. Carrier event, valid for 10M/100M speed modes
	- * only.
	+ * Indication to complete all previous RDMA Read or Atomic WQEs on the
	+ * SQ before executing this WQE
	*/
	- uint64_t rx_ovrsz_frames;
	- /* Total number of over-sized frames received */
	- uint64_t rx_jbr_frames;
	- /* Total number of jabber packets received */
	- uint64_t rx_mtu_err_frames;
	- /* Total number of received frames with MTU error */
	- uint64_t rx_match_crc_frames;
	- /* Total number of received frames with CRC match */
	- uint64_t rx_promiscuous_frames;
	- /* Total number of frames received promiscuously */
	- uint64_t rx_tagged_frames;
	- /* Total number of received frames with one or two VLAN tags */
	- uint64_t rx_double_tagged_frames;
	- /* Total number of received frames with two VLAN tags */
	- uint64_t rx_trunc_frames;
	- /* Total number of truncated frames received */
	- uint64_t rx_good_frames;
	- /* Total number of good frames (without errors) received */
	- uint64_t rx_pfc_xon2xoff_frames_pri0;
	+ #define SQ_FR_PMR_FLAGS_RD_OR_ATOMIC_FENCE UINT32_C(0x2)
	/*
	- * Total number of received PFC frames with transition from XON to XOFF
	- * on Pri 0
	+ * Unconditional fence. Indicate to complete all previous SQ's WQEs
	+ * before executing this WQE.
	*/
	- uint64_t rx_pfc_xon2xoff_frames_pri1;
	+ #define SQ_FR_PMR_FLAGS_UC_FENCE UINT32_C(0x4)
	+ /* Not Applicable for FR_PMR. Nothing is sent */
	+ #define SQ_FR_PMR_FLAGS_SE UINT32_C(0x8)
	+ /* NA. */
	+ #define SQ_FR_PMR_FLAGS_INLINE UINT32_C(0x10)
	+ uint8_t access_cntl;
	/*
	- * Total number of received PFC frames with transition from XON to XOFF
	- * on Pri 1
	+ * This is the new access control for the MR. '1' means the operation is
	+ * allowed. '0' means operation is not allowed.
	*/
	- uint64_t rx_pfc_xon2xoff_frames_pri2;
	+ /* Local Write Access */
	+ #define SQ_FR_PMR_ACCESS_CNTL_LOCAL_WRITE UINT32_C(0x1)
	+ /* Remote Read Access */
	+ #define SQ_FR_PMR_ACCESS_CNTL_REMOTE_READ UINT32_C(0x2)
	+ /* Remote Write Access */
	+ #define SQ_FR_PMR_ACCESS_CNTL_REMOTE_WRITE UINT32_C(0x4)
	+ /* Remote Atomic Access */
	+ #define SQ_FR_PMR_ACCESS_CNTL_REMOTE_ATOMIC UINT32_C(0x8)
	+ /* Window Binding Allowed */
	+ #define SQ_FR_PMR_ACCESS_CNTL_WINDOW_BIND UINT32_C(0x10)
	+ uint8_t zero_based_page_size_log;
	+ /* 0 for 4KB page size, ... to 8GB */
	+ #define SQ_FR_PMR_PAGE_SIZE_LOG_MASK UINT32_C(0x1f)
	+ #define SQ_FR_PMR_PAGE_SIZE_LOG_SFT 0
	+ /* Page size is 4KB. */
	+ #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_4K UINT32_C(0x0)
	+ /* Page size is 8KB. */
	+ #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_8K UINT32_C(0x1)
	+ /* Page size is 64KB. */
	+ #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_64K UINT32_C(0x4)
	+ /* Page size is 256KB. */
	+ #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_256K UINT32_C(0x6)
	+ /* Page size is 1MB. */
	+ #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_1M UINT32_C(0x8)
	+ /* Page size is 2MB. */
	+ #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_2M UINT32_C(0x9)
	+ /* Page size is 4MB. */
	+ #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_4M UINT32_C(0xa)
	+ /* Page size is 1GB. */
	+ #define SQ_FR_PMR_PAGE_SIZE_LOG_PGSZ_1G UINT32_C(0x12)
	+ /* Indicate the MR is ZBVA (Zero Base VA) */
	+ #define SQ_FR_PMR_ZERO_BASED UINT32_C(0x20)
	+ #define SQ_FR_PMR_RESERVED2_MASK UINT32_C(0xc0)
	+ #define SQ_FR_PMR_RESERVED2_SFT 6
	+ uint32_t l_key;
	/*
	- * Total number of received PFC frames with transition from XON to XOFF
	- * on Pri 2
	+ * Local Key; 24 msb of the key are used to index the MRW table, 8 lsb
	+ * are assigned to the 8 bit key_lsb field in the MRWC.
	*/
	- uint64_t rx_pfc_xon2xoff_frames_pri3;
	+ uint8_t length[5];
	+ /* Length in bytes of registered MR */
	+ uint8_t reserved8_1;
	+ uint8_t reserved8_2;
	+ uint8_t numlevels_pbl_page_size_log;
	+ /* Number of levels of PBL for translation */
	+ /* PBL page size. 0 for 4KB page size. */
	+ #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_MASK UINT32_C(0x1f)
	+ #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_SFT 0
	+ /* Page size is 4KB. */
	+ #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_4K UINT32_C(0x0)
	+ /* Page size is 8KB. */
	+ #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_8K UINT32_C(0x1)
	+ /* Page size is 64KB. */
	+ #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_64K UINT32_C(0x4)
	+ /* Page size is 256KB. */
	+ #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_256K UINT32_C(0x6)
	+ /* Page size is 1MB. */
	+ #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_1M UINT32_C(0x8)
	+ /* Page size is 2MB. */
	+ #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_2M UINT32_C(0x9)
	+ /* Page size is 4MB. */
	+ #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_4M UINT32_C(0xa)
	+ /* Page size is 1GB. */
	+ #define SQ_FR_PMR_PBL_PAGE_SIZE_LOG_PGSZ_1G UINT32_C(0x12)
	+ #define SQ_FR_PMR_RESERVED1 UINT32_C(0x20)
	+ /* Number of levels of PBL for translation */
	+ #define SQ_FR_PMR_NUMLEVELS_MASK UINT32_C(0xc0)
	+ #define SQ_FR_PMR_NUMLEVELS_SFT 6
	/*
	- * Total number of received PFC frames with transition from XON to XOFF
	- * on Pri 3
	+ * A zero level PBL means that the VA is the physical address
	+ * used for the operation. No translation is done by the PTU.
	*/
	- uint64_t rx_pfc_xon2xoff_frames_pri4;
	+ #define SQ_FR_PMR_NUMLEVELS_PHYSICAL (UINT32_C(0x0) << 6)
	/*
	- * Total number of received PFC frames with transition from XON to XOFF
	- * on Pri 4
	+ * A one layer translation is provided between the logical and
	+ * physical address. The PBL points to a physical page that
	+ * contains PBE values that point to actual pg_size physical
	+ * pages.
	*/
	- uint64_t rx_pfc_xon2xoff_frames_pri5;
	+ #define SQ_FR_PMR_NUMLEVELS_LAYER1 (UINT32_C(0x1) << 6)
	/*
	- * Total number of received PFC frames with transition from XON to XOFF
	- * on Pri 5
	+ * A two layer translation is provided between the logical and
	+ * physical address. The PBL points to a physical page that
	+ * contains PDE values that in turn point to pbl_pg_size
	+ * physical pages that contain PBE values that point to actual
	+ * physical pages.
	*/
	- uint64_t rx_pfc_xon2xoff_frames_pri6;
	+ #define SQ_FR_PMR_NUMLEVELS_LAYER2 (UINT32_C(0x2) << 6)
	+ uint64_t pblptr;
	+ /* Pointer to the PBL, or PDL depending on number of levels */
	+ uint64_t va;
	+ /* Local Virtual Address */
	+ uint32_t data[24];
	+ /* The data field for FR-PRM is not used. */
	+} __attribute__((packed));
	+
	+/* Bind SQ WQE (40 bytes) */
	+
	+struct sq_bind {
	+ uint8_t wqe_type;
	+ /* This field defines the type of SQ WQE. */
	+ /* Memory Bind Allowed only on reliable connection (RC) SQ's. */
	+ #define SQ_BIND_WQE_TYPE_BIND UINT32_C(0xe)
	+ uint8_t flags;
	/*
	- * Total number of received PFC frames with transition from XON to XOFF
	- * on Pri 6
	+ * Set if completion signaling is requested. If this bit is 0, and the
	+ * SQ is configured to support Unsignaled completion the controller
	+ * should not generate a CQE unless there was an error. This refer to
	+ * CQE on the sender side (se_flag refer to the receiver side)
	*/
	- uint64_t rx_pfc_xon2xoff_frames_pri7;
	+ #define SQ_BIND_FLAGS_SIGNAL_COMP UINT32_C(0x1)
	/*
	- * Total number of received PFC frames with transition from XON to XOFF
	- * on Pri 7
	+ * Indication to complete all previous RDMA Read or Atomic WQEs on the
	+ * SQ before executing this WQE
	*/
	- uint64_t rx_pfc_ena_frames_pri0;
	- /* Total number of received PFC frames with PFC enabled bit for Pri 0 */
	- uint64_t rx_pfc_ena_frames_pri1;
	- /* Total number of received PFC frames with PFC enabled bit for Pri 1 */
	- uint64_t rx_pfc_ena_frames_pri2;
	- /* Total number of received PFC frames with PFC enabled bit for Pri 2 */
	- uint64_t rx_pfc_ena_frames_pri3;
	- /* Total number of received PFC frames with PFC enabled bit for Pri 3 */
	- uint64_t rx_pfc_ena_frames_pri4;
	- /* Total number of received PFC frames with PFC enabled bit for Pri 4 */
	- uint64_t rx_pfc_ena_frames_pri5;
	- /* Total number of received PFC frames with PFC enabled bit for Pri 5 */
	- uint64_t rx_pfc_ena_frames_pri6;
	- /* Total number of received PFC frames with PFC enabled bit for Pri 6 */
	- uint64_t rx_pfc_ena_frames_pri7;
	- /* Total number of received PFC frames with PFC enabled bit for Pri 7 */
	- uint64_t rx_sch_crc_err_frames;
	- /* Total Number of frames received with SCH CRC error */
	- uint64_t rx_undrsz_frames;
	- /* Total Number of under-sized frames received */
	- uint64_t rx_frag_frames;
	- /* Total Number of fragmented frames received */
	- uint64_t rx_eee_lpi_events;
	- /* Total number of RX EEE LPI Events */
	- uint64_t rx_eee_lpi_duration;
	- /* EEE LPI Duration Counter on RX */
	- uint64_t rx_llfc_physical_msgs;
	+ #define SQ_BIND_FLAGS_RD_OR_ATOMIC_FENCE UINT32_C(0x2)
	/*
	- * Total number of physical type Link Level Flow Control (LLFC) messages
	- * received
	+ * Unconditional fence. Indicate to complete all previous SQ's WQEs
	+ * before executing this WQE.
	*/
	- uint64_t rx_llfc_logical_msgs;
	+ #define SQ_BIND_FLAGS_UC_FENCE UINT32_C(0x4)
	+ /* NA, nothing is sent. */
	+ #define SQ_BIND_FLAGS_SE UINT32_C(0x8)
	+ /* NA */
	+ #define SQ_BIND_FLAGS_INLINE UINT32_C(0x10)
	+ uint8_t access_cntl;
	/*
	- * Total number of logical type Link Level Flow Control (LLFC) messages
	- * received
	+ * This is the new access control for the MR. '1' means the operation is
	+ * allowed. '0' means operation is not allowed.
	*/
	- uint64_t rx_llfc_msgs_with_crc_err;
	/*
	- * Total number of logical type Link Level Flow Control (LLFC) messages
	- * received with CRC error
	+ * Local Write Access. Local accesses are never allowed for memory
	+ * windows, so this bit must always be zero in a bind WQE. If this bit
	+ * is ever set, the bind will fail with an errored completion.
	*/
	- uint64_t rx_hcfc_msgs;
	- /* Total number of HCFC messages received */
	- uint64_t rx_hcfc_msgs_with_crc_err;
	- /* Total number of HCFC messages received with CRC error */
	- uint64_t rx_bytes;
	- /* Total number of received bytes */
	- uint64_t rx_runt_bytes;
	- /* Total number of bytes received in runt frames */
	- uint64_t rx_runt_frames;
	- /* Total number of runt frames received */
	- uint64_t rx_stat_discard;
	- /* Total Rx Discards per Port reported by STATS block */
	- uint64_t rx_stat_err;
	- /* Total Rx Error Drops per Port reported by STATS block */
	+ #define SQ_BIND_ACCESS_CNTL_LOCAL_WRITE UINT32_C(0x1)
	+ /* Remote Read Access */
	+ #define SQ_BIND_ACCESS_CNTL_REMOTE_READ UINT32_C(0x2)
	+ /*
	+ * Remote Write Access. Note that, if this bit is set, then the parent
	+ * region to which the window is being bound must allow local writes. If
	+ * this is not the case, then the bind will fail with an errored
	+ * completion.
	+ */
	+ #define SQ_BIND_ACCESS_CNTL_REMOTE_WRITE UINT32_C(0x4)
	+ /*
	+ * Remote Atomic Access. Note that, if this bit is set, then the parent
	+ * region to which the window is being bound must allow local writes. If
	+ * this is not the case, then the bind will fail with an errored
	+ * completion.
	+ */
	+ #define SQ_BIND_ACCESS_CNTL_REMOTE_ATOMIC UINT32_C(0x8)
	+ /*
	+ * Window Binding Allowed. It is never allowed to bind windows to
	+ * windows, so this bit must always be zero in a bind WQE. If this bit
	+ * is ever set, the bind will fail with an errored completion.
	+ */
	+ #define SQ_BIND_ACCESS_CNTL_WINDOW_BIND UINT32_C(0x10)
	+ uint8_t reserved8_1;
	+ /* reserved8_1 is 8 b */
	+ uint8_t mw_type_zero_based;
	+ /*
	+ * If this bit is set, then the newly-bound memory window will be zero-
	+ * based. If clear, then the newly-bound memory window will be non-zero-
	+ * based.
	+ */
	+ #define SQ_BIND_ZERO_BASED UINT32_C(0x1)
	+ /*
	+ * If type1 is specified, then this WQE performs a "bind memory window"
	+ * operation on a type1 window. If type2 is specified, then this WQE
	+ * performs a "post send bind memory window" operation on a type2
	+ * window. Note that the bind WQE cannot change the type of the memory
	+ * window. If a "bind memory window" operation is attempted on a memory
	+ * window that was allocated as type2, then the bind will fail with an
	+ * errored completion, as "bind memory window" is allowed only on type1
	+ * memory windows. Similarly, if a "post send bind memory window"
	+ * operation is attempted on a memory window that was allocated as
	+ * type1, then the bind will fail with an errored completions, as "post
	+ * send bind memory window" is allowed only on type2 memory windows.
	+ */
	+ #define SQ_BIND_MW_TYPE UINT32_C(0x2)
	+ /* Type 1 Bind Memory Window */
	+ #define SQ_BIND_MW_TYPE_TYPE1 (UINT32_C(0x0) << 1)
	+ /* Type 2 Post Send Bind Memory Window */
	+ #define SQ_BIND_MW_TYPE_TYPE2 (UINT32_C(0x1) << 1)
	+ #define SQ_BIND_RESERVED6_MASK UINT32_C(0xfc)
	+ #define SQ_BIND_RESERVED6_SFT 2
	+ uint8_t reserved8_2;
	+ uint16_t reserved16;
	+ uint32_t parent_l_key;
	+ /*
	+ * The L_Key of the parent MR; 24 msb of the key are used to index the
	+ * MRW table, 8 lsb are compared with the 8 bit key in the MRWC.
	+ */
	+ uint32_t l_key;
	+ /*
	+ * Local Key; 24 msb of the key are used to index the memory window
	+ * being bound in the MRW table, 8 lsb are assign to the 8 bit key_lsb
	+ * field in the MRWC.
	+ */
	+ uint64_t va;
	+ /* Local Virtual Address */
	+ uint8_t length[5];
	+ /*
	+ * Length in bytes of registered MW; 40 bits as this is the max size of
	+ * an MR/W
	+ */
	+ uint8_t data_reserved24[99];
	+ /* The data field for Bind is not used. */
	+ #define SQ_BIND_RESERVED24_MASK UINT32_C(0xffffff00)
	+ #define SQ_BIND_RESERVED24_SFT 8
	+ /* The data field for Bind is not used. */
	+ #define SQ_BIND_DATA_MASK UINT32_C(0xffffffff)
	+ #define SQ_BIND_DATA_SFT 0
	} __attribute__((packed));

	+/* RQ/SRQ WQE Structures */
	+/* Description: This is the RQ/SRQ WQE structure. */
	+/* RQ/SRQ WQE (40 bytes) */
	+
	+struct rq_wqe {
	+ uint8_t wqe_type;
	+ /* wqe_type is 8 b */
	+ /* RQ/SRQ WQE. This WQE is used for posting buffers on an RQ or SRQ. */
	+ #define RQ_WQE_WQE_TYPE_RCV UINT32_C(0x80)
	+ uint8_t flags;
	+ /* No flags supported for this WQE type. */
	+ uint8_t wqe_size;
	+ /*
	+ * Specify the total number 16B chunks that make up the valid portion of
	+ * the WQE. This includes the first chunk that is the WQE structure and
	+ * up to 6 SGE structures. While the valid area is defined by the
	+ * wqe_size field, the actual WQE size is fixed at 128B.
	+ */
	+ uint8_t reserved8;
	+ uint32_t reserved32;
	+ uint64_t wr_id;
	+ /*
	+ * Opaque value used by upper layer SW to identify the id of the WR
	+ * which generated the WQE. Used in CQE. Valid in the first SGE of an
	+ * SRQ WQE.
	+ */
	+ #define RQ_WQE_WR_ID_MASK UINT32_C(0xfffff)
	+ #define RQ_WQE_WR_ID_SFT 0
	+ #define RQ_WQE_RESERVED44_MASK UINT32_C(0xfff00000)
	+ #define RQ_WQE_RESERVED44_SFT 20
	+ uint32_t reserved128[4];
	+ uint32_t data[24];
	+ /*
	+ * The data field for RQ WQE is filled with from 1 to 6 SGE structures
	+ * as defined by the wqe_size field.
	+ */
	+} __attribute__((packed));
	+
	+/* CQ CQE Structures */
	+/* Description: This is the Cutoff CQE structure. */
	+/* Base CQE (32 bytes) */
	+
	+struct cq_base {
	+ uint64_t reserved64_1;
	+ uint64_t reserved64_2;
	+ uint64_t reserved64_3;
	+ uint8_t cqe_type_toggle;
	+ /*
	+ * Indicate valid completion - written by the chip. Cumulus toggle this
	+ * bit each time it finished consuming all PBL entries
	+ */
	+ #define CQ_BASE_TOGGLE UINT32_C(0x1)
	+ /* This field defines the type of SQ WQE. */
	+ #define CQ_BASE_CQE_TYPE_MASK UINT32_C(0x1e)
	+ #define CQ_BASE_CQE_TYPE_SFT 1
	+ /*
	+ * Requester completion - This is used for both RC and UD SQ
	+ * completions.
	+ */
	+ #define CQ_BASE_CQE_TYPE_REQ (UINT32_C(0x0) << 1)
	+ /*
	+ * Responder RC Completion - This is used for both RQ and SRQ
	+ * completions for RC service QPs.
	+ */
	+ #define CQ_BASE_CQE_TYPE_RES_RC (UINT32_C(0x1) << 1)
	+ /*
	+ * Responder UD Completion - This is used for both RQ and SRQ
	+ * completion for UD service QPs.
	+ */
	+ #define CQ_BASE_CQE_TYPE_RES_UD (UINT32_C(0x2) << 1)
	+ /*
	+ * Responder RawEth and QP1 Completion - This is used for RQ
	+ * completion for RawEth service and QP1 service QPs.
	+ */
	+ #define CQ_BASE_CQE_TYPE_RES_RAWETH_QP1 (UINT32_C(0x3) << 1)
	+ /*
	+ * Terminal completion - This is used to indicate that no
	+ * further completions will be made for this QP on this CQ.
	+ */
	+ #define CQ_BASE_CQE_TYPE_TERMINAL (UINT32_C(0xe) << 1)
	+ /* Cut off CQE; for CQ resize see CQ and SRQ Resize */
	+ #define CQ_BASE_CQE_TYPE_CUT_OFF (UINT32_C(0xf) << 1)
	+ #define CQ_BASE_RESERVED3_MASK UINT32_C(0xe0)
	+ #define CQ_BASE_RESERVED3_SFT 5
	+ uint8_t status;
	+ /* This field indicates the status for the CQE. */
	+ uint16_t reserved16;
	+ uint32_t reserved32;
	+} __attribute__((packed));
	+
	+/* Requester CQ CQE (32 bytes) */
	+
	+struct cq_req {
	+ uint64_t qp_handle;
	+ /*
	+ * This is an application level ID used to identify the QP and its SQ
	+ * and RQ.
	+ */
	+ uint16_t sq_cons_idx;
	+ /*
	+ * SQ Consumer Index - points to the entry just past the last WQE that
	+ * has been completed by the chip. Wraps around at QPC.sq_size (i.e. the
	+ * valid range of the SQ Consumer Index is 0 to (QPC.sq_size - 1)).
	+ */
	+ uint16_t reserved16_1;
	+ uint32_t reserved32_2;
	+ uint64_t reserved64;
	+ uint8_t cqe_type_toggle;
	+ /*
	+ * Indicate valid completion - written by the chip. Cumulus toggle this
	+ * bit each time it finished consuming all PBL entries
	+ */
	+ #define CQ_REQ_TOGGLE UINT32_C(0x1)
	+ /* This field defines the type of SQ WQE. */
	+ #define CQ_REQ_CQE_TYPE_MASK UINT32_C(0x1e)
	+ #define CQ_REQ_CQE_TYPE_SFT 1
	+ /*
	+ * Requester completion - This is used for both RC and UD SQ
	+ * completions.
	+ */
	+ #define CQ_REQ_CQE_TYPE_REQ (UINT32_C(0x0) << 1)
	+ #define CQ_REQ_RESERVED3_MASK UINT32_C(0xe0)
	+ #define CQ_REQ_RESERVED3_SFT 5
	+ uint8_t status;
	+ /* This field indicates the status for the CQE. */
	+ /* OK is 0 */
	+ #define CQ_REQ_STATUS_OK UINT32_C(0x0)
	+ /* BAD_RESPONSE_ERR is 1 */
	+ #define CQ_REQ_STATUS_BAD_RESPONSE_ERR UINT32_C(0x1)
	+ /* LOCAL_LENGTH_ERR is 2 */
	+ #define CQ_REQ_STATUS_LOCAL_LENGTH_ERR UINT32_C(0x2)
	+ /* LOCAL_QP_OPERATION_ERR is 3 */
	+ #define CQ_REQ_STATUS_LOCAL_QP_OPERATION_ERR UINT32_C(0x3)
	+ /* LOCAL_PROTECTION_ERR is 4 */
	+ #define CQ_REQ_STATUS_LOCAL_PROTECTION_ERR UINT32_C(0x4)
	+ /* MEMORY_MGT_OPERATION_ERR is 5 */
	+ #define CQ_REQ_STATUS_MEMORY_MGT_OPERATION_ERR UINT32_C(0x5)
	+ /* REMOTE_INVALID_REQUEST_ERR is 6 */
	+ #define CQ_REQ_STATUS_REMOTE_INVALID_REQUEST_ERR UINT32_C(0x6)
	+ /* REMOTE_ACCESS_ERR is 7 */
	+ #define CQ_REQ_STATUS_REMOTE_ACCESS_ERR UINT32_C(0x7)
	+ /* REMOTE_OPERATION_ERR is 8 */
	+ #define CQ_REQ_STATUS_REMOTE_OPERATION_ERR UINT32_C(0x8)
	+ /* RNR_NAK_RETRY_CNT_ERR is 9 */
	+ #define CQ_REQ_STATUS_RNR_NAK_RETRY_CNT_ERR UINT32_C(0x9)
	+ /* TRANSPORT_RETRY_CNT_ERR is 10 */
	+ #define CQ_REQ_STATUS_TRANSPORT_RETRY_CNT_ERR UINT32_C(0xa)
	+ /* WORK_REQUEST_FLUSHED_ERR is 11 */
	+ #define CQ_REQ_STATUS_WORK_REQUEST_FLUSHED_ERR UINT32_C(0xb)
	+ uint16_t reserved16_2;
	+ uint32_t reserved32_1;
	+} __attribute__((packed));
	+
	+/* Responder RC CQE (32 bytes) */
	+
	+struct cq_res_rc {
	+ uint32_t length;
	+ /* The length of the message's payload in bytes, stored in the SGEs */
	+ uint32_t imm_data_or_inv_r_key;
	+ /*
	+ * Immediate data in case the imm_flag set, R_Key to be invalidated in
	+ * case inv_flag is set.
	+ */
	+ uint64_t qp_handle;
	+ /*
	+ * This is an application level ID used to identify the QP and its SQ
	+ * and RQ.
	+ */
	+ uint64_t mr_handle;
	+ /*
	+ * Opaque value - valid when inv_flag is set. Used by driver to
	+ * reference the buffer used to store PBL when the MR was fast
	+ * registered. The driver can reclaim this buffer after an MR was
	+ * remotely invalidated. The controller take that value from the MR
	+ * referenced by R_Key
	+ */
	+ uint8_t cqe_type_toggle;
	+ /*
	+ * Indicate valid completion - written by the chip. Cumulus toggle this
	+ * bit each time it finished consuming all PBL entries
	+ */
	+ #define CQ_RES_RC_TOGGLE UINT32_C(0x1)
	+ /* This field defines the type of SQ WQE. */
	+ #define CQ_RES_RC_CQE_TYPE_MASK UINT32_C(0x1e)
	+ #define CQ_RES_RC_CQE_TYPE_SFT 1
	+ /*
	+ * Responder RC Completion - This is used for both RQ and SRQ
	+ * completions for RC service QPs.
	+ */
	+ #define CQ_RES_RC_CQE_TYPE_RES_RC (UINT32_C(0x1) << 1)
	+ #define CQ_RES_RC_RESERVED3_MASK UINT32_C(0xe0)
	+ #define CQ_RES_RC_RESERVED3_SFT 5
	+ uint8_t status;
	+ /* This field indicates the status for the CQE. */
	+ /* OK is 0 */
	+ #define CQ_RES_RC_STATUS_OK UINT32_C(0x0)
	+ /* LOCAL_ACCESS_ERROR is 1 */
	+ #define CQ_RES_RC_STATUS_LOCAL_ACCESS_ERROR UINT32_C(0x1)
	+ /* LOCAL_LENGTH_ERR is 2 */
	+ #define CQ_RES_RC_STATUS_LOCAL_LENGTH_ERR UINT32_C(0x2)
	+ /* LOCAL_PROTECTION_ERR is 3 */
	+ #define CQ_RES_RC_STATUS_LOCAL_PROTECTION_ERR UINT32_C(0x3)
	+ /* LOCAL_QP_OPERATION_ERR is 4 */
	+ #define CQ_RES_RC_STATUS_LOCAL_QP_OPERATION_ERR UINT32_C(0x4)
	+ /* MEMORY_MGT_OPERATION_ERR is 5 */
	+ #define CQ_RES_RC_STATUS_MEMORY_MGT_OPERATION_ERR UINT32_C(0x5)
	+ /* REMOTE_INVALID_REQUEST_ERR is 6 */
	+ #define CQ_RES_RC_STATUS_REMOTE_INVALID_REQUEST_ERR UINT32_C(0x6)
	+ /* WORK_REQUEST_FLUSHED_ERR is 7 */
	+ #define CQ_RES_RC_STATUS_WORK_REQUEST_FLUSHED_ERR UINT32_C(0x7)
	+ /* HW_FLUSH_ERR is 8 */
	+ #define CQ_RES_RC_STATUS_HW_FLUSH_ERR UINT32_C(0x8)
	+ uint16_t flags;
	+ /*
	+ * This flag indicates that the completion is for a SRQ entry rather
	+ * than for an RQ entry.
	+ */
	+ #define CQ_RES_RC_FLAGS_SRQ UINT32_C(0x1)
	+ /* CQE relates to RQ WQE. */
	+ #define CQ_RES_RC_FLAGS_SRQ_RQ (UINT32_C(0x0) << 0)
	+ /* CQE relates to SRQ WQE. */
	+ #define CQ_RES_RC_FLAGS_SRQ_SRQ (UINT32_C(0x1) << 0)
	+ #define CQ_RES_RC_FLAGS_SRQ_LAST CQ_RES_RC_FLAGS_SRQ_SRQ
	+ /* Immediate data indicator */
	+ #define CQ_RES_RC_FLAGS_IMM UINT32_C(0x2)
	+ /* R_Key invalidate indicator */
	+ #define CQ_RES_RC_FLAGS_INV UINT32_C(0x4)
	+ #define CQ_RES_RC_FLAGS_RDMA UINT32_C(0x8)
	+ /* CQE relates to an incoming Send request */
	+ #define CQ_RES_RC_FLAGS_RDMA_SEND (UINT32_C(0x0) << 3)
	+ /* CQE relates to incoming RDMA Write request */
	+ #define CQ_RES_RC_FLAGS_RDMA_RDMA_WRITE (UINT32_C(0x1) << 3)
	+ #define CQ_RES_RC_FLAGS_RDMA_LAST CQ_RES_RC_FLAGS_RDMA_RDMA_WRITE
	+ uint32_t srq_or_rq_wr_id;
	+ /*
	+ * Opaque value from RQ or SRQ WQE. Used by driver/lib to reference the
	+ * WQE in order to claim the received data and reuse the WQE space
	+ */
	+ #define CQ_RES_RC_SRQ_OR_RQ_WR_ID_MASK UINT32_C(0xfffff)
	+ #define CQ_RES_RC_SRQ_OR_RQ_WR_ID_SFT 0
	+ #define CQ_RES_RC_RESERVED12_MASK UINT32_C(0xfff00000)
	+ #define CQ_RES_RC_RESERVED12_SFT 20
	+} __attribute__((packed));
	+
	+/* Responder UD CQE (32 bytes) */
	+
	+struct cq_res_ud {
	+ uint32_t length;
	+ /* The length of the message's payload in bytes, stored in the SGEs */
	+ #define CQ_RES_UD_LENGTH_MASK UINT32_C(0x3fff)
	+ #define CQ_RES_UD_LENGTH_SFT 0
	+ #define CQ_RES_UD_RESERVED18_MASK UINT32_C(0xffffc000)
	+ #define CQ_RES_UD_RESERVED18_SFT 14
	+ uint32_t imm_data;
	+ /* Immediate data in case the imm_flag set. */
	+ uint64_t qp_handle;
	+ /*
	+ * This is an application level ID used to identify the QP and its SQ
	+ * and RQ.
	+ */
	+ uint16_t src_mac[3];
	+ /*
	+ * Source MAC address for the UD message placed in the WQE that is
	+ * completed by this CQE.
	+ */
	+ uint16_t src_qp_low;
	+ /* Lower 16b of the Source QP value from the DETH header. */
	+ uint8_t cqe_type_toggle;
	+ /*
	+ * Indicate valid completion - written by the chip. Cumulus toggle this
	+ * bit each time it finished consuming all PBL entries
	+ */
	+ #define CQ_RES_UD_TOGGLE UINT32_C(0x1)
	+ /* This field defines the type of SQ WQE. */
	+ #define CQ_RES_UD_CQE_TYPE_MASK UINT32_C(0x1e)
	+ #define CQ_RES_UD_CQE_TYPE_SFT 1
	+ /*
	+ * Responder UD Completion - This is used for both RQ and SRQ
	+ * completion for UD service QPs.
	+ */
	+ #define CQ_RES_UD_CQE_TYPE_RES_UD (UINT32_C(0x2) << 1)
	+ #define CQ_RES_UD_RESERVED3_MASK UINT32_C(0xe0)
	+ #define CQ_RES_UD_RESERVED3_SFT 5
	+ uint8_t status;
	+ /* This field indicates the status for the CQE. */
	+ /*
	+ * This indicates that the completion is without error. All
	+ * fields are valid.
	+ */
	+ #define CQ_RES_UD_STATUS_OK UINT32_C(0x0)
	+ /*
	+ * This indicates that write access was not allowed for at least
	+ * one of the SGEs in the WQE. This is a fatal error. Only the
	+ * srq_or_rq_wr_id is field is valid.
	+ */
	+ #define CQ_RES_UD_STATUS_LOCAL_ACCESS_ERROR UINT32_C(0x1)
	+ /*
	+ * This indicates that the packet was too long for the WQE
	+ * provided on the SRQ/RQ. This is not a fatal error. All the
	+ * fields are valid.
	+ */
	+ #define CQ_RES_UD_STATUS_HW_LOCAL_LENGTH_ERR UINT32_C(0x2)
	+ /* LOCAL_PROTECTION_ERR is 3 */
	+ #define CQ_RES_UD_STATUS_LOCAL_PROTECTION_ERR UINT32_C(0x3)
	+ /* LOCAL_QP_OPERATION_ERR is 4 */
	+ #define CQ_RES_UD_STATUS_LOCAL_QP_OPERATION_ERR UINT32_C(0x4)
	+ /* MEMORY_MGT_OPERATION_ERR is 5 */
	+ #define CQ_RES_UD_STATUS_MEMORY_MGT_OPERATION_ERR UINT32_C(0x5)
	+ /* WORK_REQUEST_FLUSHED_ERR is 7 */
	+ #define CQ_RES_UD_STATUS_WORK_REQUEST_FLUSHED_ERR UINT32_C(0x7)
	+ /* HW_FLUSH_ERR is 8 */
	+ #define CQ_RES_UD_STATUS_HW_FLUSH_ERR UINT32_C(0x8)
	+ uint16_t flags;
	+ /*
	+ * This flag indicates that the completion is for a SRQ entry rather
	+ * than for an RQ entry.
	+ */
	+ #define CQ_RES_UD_FLAGS_SRQ UINT32_C(0x1)
	+ /* CQE relates to RQ WQE. */
	+ #define CQ_RES_UD_FLAGS_SRQ_RQ (UINT32_C(0x0) << 0)
	+ /* CQE relates to SRQ WQE. */
	+ #define CQ_RES_UD_FLAGS_SRQ_SRQ (UINT32_C(0x1) << 0)
	+ #define CQ_RES_UD_FLAGS_SRQ_LAST CQ_RES_UD_FLAGS_SRQ_SRQ
	+ /* Immediate data indicator */
	+ #define CQ_RES_UD_FLAGS_IMM UINT32_C(0x2)
	+ #define CQ_RES_UD_FLAGS_ROCE_IP_VER_MASK UINT32_C(0xc)
	+ #define CQ_RES_UD_FLAGS_ROCE_IP_VER_SFT 2
	+ /* RoCEv1 Message */
	+ #define CQ_RES_UD_FLAGS_ROCE_IP_VER_V1 (UINT32_C(0x0) << 2)
	+ /* RoCEv2 IPv4 Message */
	+ #define CQ_RES_UD_FLAGS_ROCE_IP_VER_V2IPV4 (UINT32_C(0x2) << 2)
	+ /* RoCEv2 IPv6 Message */
	+ #define CQ_RES_UD_FLAGS_ROCE_IP_VER_V2IPV6 (UINT32_C(0x3) << 2)
	+ #define CQ_RES_UD_FLAGS_ROCE_IP_VER_LAST CQ_RES_UD_FLAGS_ROCE_IP_VER_V2IPV6
	+ uint32_t src_qp_high_srq_or_rq_wr_id;
	+ /* Upper 8b of the Source QP value from the DETH header. */
	+ /*
	+ * Opaque value from RQ or SRQ WQE. Used by driver/lib to reference the
	+ * WQE in order to claim the received data and reuse the WQE space
	+ */
	+ #define CQ_RES_UD_SRQ_OR_RQ_WR_ID_MASK UINT32_C(0xfffff)
	+ #define CQ_RES_UD_SRQ_OR_RQ_WR_ID_SFT 0
	+ #define CQ_RES_UD_RESERVED4_MASK UINT32_C(0xf00000)
	+ #define CQ_RES_UD_RESERVED4_SFT 20
	+ /* Upper 8b of the Source QP value from the DETH header. */
	+ #define CQ_RES_UD_SRC_QP_HIGH_MASK UINT32_C(0xff000000)
	+ #define CQ_RES_UD_SRC_QP_HIGH_SFT 24
	+} __attribute__((packed));
	+
	+/* Responder RawEth and QP1 CQE (32 bytes) */
	+
	+struct cq_res_raweth_qp1 {
	+ uint16_t length;
	+ /* The length of the message's payload in bytes, stored in the SGEs */
	+ #define CQ_RES_RAWETH_QP1_LENGTH_MASK UINT32_C(0x3fff)
	+ #define CQ_RES_RAWETH_QP1_LENGTH_SFT 0
	+ #define CQ_RES_RAWETH_QP1_RESERVED2_MASK UINT32_C(0xc000)
	+ #define CQ_RES_RAWETH_QP1_RESERVED2_SFT 14
	+ uint16_t raweth_qp1_flags;
	+ /*
	+ * When this bit is '1', it indicates a packet that has an error of some
	+ * type. Type of error is indicated in raweth_qp1_errors.
	+ */
	+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ERROR UINT32_C(0x1)
	+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_RESERVED5_1_MASK UINT32_C(0x3e)
	+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_RESERVED5_1_SFT 1
	+ /*
	+ * This value indicates what the inner packet determined for the packet
	+ * was.
	+ */
	+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_MASK UINT32_C(0x3c0)
	+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_SFT 6
	+ /* Not Known: Indicates that the packet type was not known. */
	+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_NOT_KNOWN (UINT32_C(0x0) << 6)
	+ /*
	+ * IP Packet: Indicates that the packet was an IP packet, but
	+ * further classification was not possible.
	+ */
	+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_IP (UINT32_C(0x1) << 6)
	+ /*
	+ * TCP Packet: Indicates that the packet was IP and TCP. This
	+ * indicates that the raweth_qp1_payload_offset field is valid.
	+ */
	+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_TCP (UINT32_C(0x2) << 6)
	+ /*
	+ * UDP Packet: Indicates that the packet was IP and UDP. This
	+ * indicates that the raweth_qp1_payload_offset field is valid.
	+ */
	+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_UDP (UINT32_C(0x3) << 6)
	+ /*
	+ * FCoE Packet: Indicates that the packet was recognized as a
	+ * FCoE. This also indicates that the raweth_qp1_payload_offset
	+ * field is valid.
	+ */
	+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_FCOE (UINT32_C(0x4) << 6)
	+ /*
	+ * RoCE Packet: Indicates that the packet was recognized as a
	+ * RoCE. This also indicates that the raweth_qp1_payload_offset
	+ * field is valid.
	+ */
	+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_ROCE (UINT32_C(0x5) << 6)
	+ /*
	+ * ICMP Packet: Indicates that the packet was recognized as
	+ * ICMP. This indicates that the raweth_qp1_payload_offset field
	+ * is valid.
	+ */
	+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_ICMP (UINT32_C(0x7) << 6)
	+ /*
	+ * PtP packet wo/timestamp: Indicates that the packet was
	+ * recognized as a PtP packet.
	+ */
	+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_PTP_WO_TIMESTAMP (UINT32_C(0x8) << 6)
	+ /*
	+ * PtP packet w/timestamp: Indicates that the packet was
	+ * recognized as a PtP packet and that a timestamp was taken for
	+ * the packet.
	+ */
	+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_PTP_W_TIMESTAMP (UINT32_C(0x9) << 6)
	+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_LAST CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_PTP_W_TIMESTAMP
	+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_MASK UINT32_C(0x3ff)
	+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_SFT 0
	+ #define CQ_RES_RAWETH_QP1_RESERVED6_MASK UINT32_C(0xfc00)
	+ #define CQ_RES_RAWETH_QP1_RESERVED6_SFT 10
	+ uint16_t raweth_qp1_errors;
	+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_RESERVED4_MASK UINT32_C(0xf)
	+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_RESERVED4_SFT 0
	+ /* This indicates that there was an error in the IP header checksum. */
	+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_IP_CS_ERROR UINT32_C(0x10)
	+ /*
	+ * This indicates that there was an error in the TCP, UDP or ICMP
	+ * checksum.
	+ */
	+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_L4_CS_ERROR UINT32_C(0x20)
	+ /*
	+ * This indicates that there was an error in the tunnel IP header
	+ * checksum.
	+ */
	+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_IP_CS_ERROR UINT32_C(0x40)
	+ /* This indicates that there was an error in the tunnel UDP checksum. */
	+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_L4_CS_ERROR UINT32_C(0x80)
	+ /*
	+ * This indicates that there was a CRC error on either an FCoE or RoCE
	+ * packet. The itype indicates the packet type.
	+ */
	+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_CRC_ERROR UINT32_C(0x100)
	+ /*
	+ * This indicates that there was an error in the tunnel portion of the
	+ * packet when this field is non-zero.
	+ */
	+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_MASK UINT32_C(0xe00)
	+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_SFT 9
	+ /*
	+ * No additional error occurred on the tunnel portion of the
	+ * packet of the packet does not have a tunnel.
	+ */
	+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_NO_ERROR (UINT32_C(0x0) << 9)
	+ /*
	+ * Indicates that IP header version does not match expectation
	+ * from L2 Ethertype for IPv4 and IPv6 in the tunnel header.
	+ */
	+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_T_L3_BAD_VERSION (UINT32_C(0x1) << 9)
	+ /*
	+ * Indicates that header length is out of range in the tunnel
	+ * header. Valid for IPv4.
	+ */
	+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_T_L3_BAD_HDR_LEN (UINT32_C(0x2) << 9)
	+ /*
	+ * Indicates that the physical packet is shorter than that
	+ * claimed by the PPPoE header length for a tunnel PPPoE packet.
	+ */
	+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_TUNNEL_TOTAL_ERROR (UINT32_C(0x3) << 9)
	+ /*
	+ * Indicates that physical packet is shorter than that claimed
	+ * by the tunnel l3 header length. Valid for IPv4, or IPv6
	+ * tunnel packet packets.
	+ */
	+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_T_IP_TOTAL_ERROR (UINT32_C(0x4) << 9)
	+ /*
	+ * Indicates that the physical packet is shorter than that
	+ * claimed by the tunnel UDP header length for a tunnel UDP
	+ * packet that is not fragmented.
	+ */
	+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_T_UDP_TOTAL_ERROR (UINT32_C(0x5) << 9)
	+ /*
	+ * indicates that the IPv4 TTL or IPv6 hop limit check have
	+ * failed (e.g. TTL = 0) in the tunnel header. Valid for IPv4,
	+ * and IPv6.
	+ */
	+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_T_L3_BAD_TTL (UINT32_C(0x6) << 9)
	+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_LAST CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_T_PKT_ERROR_T_L3_BAD_TTL
	+ /*
	+ * This indicates that there was an error in the inner portion of the
	+ * packet when this field is non-zero.
	+ */
	+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_MASK UINT32_C(0xf000)
	+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_SFT 12
	+ /*
	+ * No additional error occurred on the tunnel portion of the
	+ * packet of the packet does not have a tunnel.
	+ */
	+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_NO_ERROR (UINT32_C(0x0) << 12)
	+ /*
	+ * Indicates that IP header version does not match expectation
	+ * from L2 Ethertype for IPv4 and IPv6 or that option other than
	+ * VFT was parsed on FCoE packet.
	+ */
	+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_L3_BAD_VERSION (UINT32_C(0x1) << 12)
	+ /*
	+ * indicates that header length is out of range. Valid for IPv4
	+ * and RoCE
	+ */
	+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_L3_BAD_HDR_LEN (UINT32_C(0x2) << 12)
	+ /*
	+ * indicates that the IPv4 TTL or IPv6 hop limit check have
	+ * failed (e.g. TTL = 0). Valid for IPv4, and IPv6
	+ */
	+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_L3_BAD_TTL (UINT32_C(0x3) << 12)
	+ /*
	+ * Indicates that physical packet is shorter than that claimed
	+ * by the l3 header length. Valid for IPv4, IPv6 packet or RoCE
	+ * packets.
	+ */
	+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_IP_TOTAL_ERROR (UINT32_C(0x4) << 12)
	+ /*
	+ * Indicates that the physical packet is shorter than that
	+ * claimed by the UDP header length for a UDP packet that is not
	+ * fragmented.
	+ */
	+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_UDP_TOTAL_ERROR (UINT32_C(0x5) << 12)
	+ /*
	+ * Indicates that TCP header length > IP payload. Valid for TCP
	+ * packets only.
	+ */
	+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_L4_BAD_HDR_LEN (UINT32_C(0x6) << 12)
	+ /* Indicates that TCP header length < 5. Valid for TCP. */
	+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_L4_BAD_HDR_LEN_TOO_SMALL (UINT32_C(0x7) << 12)
	+ /*
	+ * Indicates that TCP option headers result in a TCP header size
	+ * that does not match data offset in TCP header. Valid for TCP.
	+ */
	+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_L4_BAD_OPT_LEN (UINT32_C(0x8) << 12)
	+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_LAST CQ_RES_RAWETH_QP1_RAWETH_QP1_ERRORS_PKT_ERROR_L4_BAD_OPT_LEN
	+ uint16_t raweth_qp1_cfa_code;
	+ /*
	+ * This field identifies the CFA action rule that was used for this
	+ * packet.
	+ */
	+ uint64_t qp_handle;
	+ /*
	+ * This is an application level ID used to identify the QP and its SQ
	+ * and RQ.
	+ */
	+ uint32_t raweth_qp1_flags2;
	+ /*
	+ * This indicates that the ip checksum was calculated for the inner
	+ * packet and that the ip_cs_error field indicates if there was an
	+ * error.
	+ */
	+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_IP_CS_CALC UINT32_C(0x1)
	+ /*
	+ * This indicates that the TCP, UDP or ICMP checksum was calculated for
	+ * the inner packet and that the l4_cs_error field indicates if there
	+ * was an error.
	+ */
	+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_L4_CS_CALC UINT32_C(0x2)
	+ /*
	+ * This indicates that the ip checksum was calculated for the tunnel
	+ * header and that the t_ip_cs_error field indicates if there was an
	+ * error.
	+ */
	+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_T_IP_CS_CALC UINT32_C(0x4)
	+ /*
	+ * This indicates that the UDP checksum was calculated for the tunnel
	+ * packet and that the t_l4_cs_error field indicates if there was an
	+ * error.
	+ */
	+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_T_L4_CS_CALC UINT32_C(0x8)
	+ /* This value indicates what format the raweth_qp1_metadata field is. */
	+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_MASK UINT32_C(0xf0)
	+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_SFT 4
	+ /* No metadata information. Value is zero. */
	+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_NONE (UINT32_C(0x0) << 4)
	+ /*
	+ * The raweth_qp1_metadata field contains the VLAN tag and TPID
	+ * value. - raweth_qp1_metadata[11:0] contains the vlan VID
	+ * value. - raweth_qp1_metadata[12] contains the vlan DE value.
	+ * - raweth_qp1_metadata[15:13] contains the vlan PRI value. -
	+ * raweth_qp1_metadata[31:16] contains the vlan TPID value.
	+ */
	+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_VLAN (UINT32_C(0x1) << 4)
	+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_LAST CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_VLAN
	+ /*
	+ * This field indicates the IP type for the inner-most IP header. A
	+ * value of '0' indicates IPv4. A value of '1' indicates IPv6. This
	+ * value is only valid if itype indicates a packet with an IP header.
	+ */
	+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_IP_TYPE UINT32_C(0x100)
	+ uint32_t raweth_qp1_metadata;
	+ /*
	+ * This is data from the CFA block as indicated by the meta_format
	+ * field.
	+ */
	+ /* When meta_format=1, this value is the VLAN VID. */
	+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_VID_MASK UINT32_C(0xfff)
	+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_VID_SFT 0
	+ /* When meta_format=1, this value is the VLAN DE. */
	+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_DE UINT32_C(0x1000)
	+ /* When meta_format=1, this value is the VLAN PRI. */
	+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_PRI_MASK UINT32_C(0xe000)
	+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_PRI_SFT 13
	+ /* When meta_format=1, this value is the VLAN TPID. */
	+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_TPID_MASK UINT32_C(0xffff0000)
	+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_TPID_SFT 16
	+ uint8_t cqe_type_toggle;
	+ /*
	+ * Indicate valid completion - written by the chip. Cumulus toggle this
	+ * bit each time it finished consuming all PBL entries
	+ */
	+ #define CQ_RES_RAWETH_QP1_TOGGLE UINT32_C(0x1)
	+ /* This field defines the type of SQ WQE. */
	+ #define CQ_RES_RAWETH_QP1_CQE_TYPE_MASK UINT32_C(0x1e)
	+ #define CQ_RES_RAWETH_QP1_CQE_TYPE_SFT 1
	+ /*
	+ * Responder RawEth and QP1 Completion - This is used for RQ
	+ * completion for RawEth service and QP1 service QPs.
	+ */
	+ #define CQ_RES_RAWETH_QP1_CQE_TYPE_RES_RAWETH_QP1 (UINT32_C(0x3) << 1)
	+ #define CQ_RES_RAWETH_QP1_RESERVED3_MASK UINT32_C(0xe0)
	+ #define CQ_RES_RAWETH_QP1_RESERVED3_SFT 5
	+ uint8_t status;
	+ /* This field indicates the status for the CQE. */
	+ /*
	+ * This indicates that the completion is without error. All
	+ * fields are valid.
	+ */
	+ #define CQ_RES_RAWETH_QP1_STATUS_OK UINT32_C(0x0)
	+ /*
	+ * This indicates that write access was not allowed for at least
	+ * one of the SGEs in the WQE. This is a fatal error. Only the
	+ * srq_or_rq_wr_id is field is valid.
	+ */
	+ #define CQ_RES_RAWETH_QP1_STATUS_LOCAL_ACCESS_ERROR UINT32_C(0x1)
	+ /*
	+ * This indicates that the packet was too long for the WQE
	+ * provided on the RQ. This is not a fatal error. All the fields
	+ * are valid.
	+ */
	+ #define CQ_RES_RAWETH_QP1_STATUS_HW_LOCAL_LENGTH_ERR UINT32_C(0x2)
	+ /* LOCAL_PROTECTION_ERR is 3 */
	+ #define CQ_RES_RAWETH_QP1_STATUS_LOCAL_PROTECTION_ERR UINT32_C(0x3)
	+ /* LOCAL_QP_OPERATION_ERR is 4 */
	+ #define CQ_RES_RAWETH_QP1_STATUS_LOCAL_QP_OPERATION_ERR UINT32_C(0x4)
	+ /* MEMORY_MGT_OPERATION_ERR is 5 */
	+ #define CQ_RES_RAWETH_QP1_STATUS_MEMORY_MGT_OPERATION_ERR UINT32_C(0x5)
	+ /* WORK_REQUEST_FLUSHED_ERR is 7 */
	+ #define CQ_RES_RAWETH_QP1_STATUS_WORK_REQUEST_FLUSHED_ERR UINT32_C(0x7)
	+ /* HW_FLUSH_ERR is 8 */
	+ #define CQ_RES_RAWETH_QP1_STATUS_HW_FLUSH_ERR UINT32_C(0x8)
	+ uint16_t flags;
	+ /*
	+ * This flag indicates that the completion is for a SRQ entry rather
	+ * than for an RQ entry.
	+ */
	+ #define CQ_RES_RAWETH_QP1_FLAGS_SRQ UINT32_C(0x1)
	+ /* CQE relates to RQ WQE. */
	+ #define CQ_RES_RAWETH_QP1_FLAGS_SRQ_RQ UINT32_C(0x0)
	+ /* CQE relates to SRQ WQE. */
	+ #define CQ_RES_RAWETH_QP1_FLAGS_SRQ_SRQ UINT32_C(0x1)
	+ #define CQ_RES_RAWETH_QP1_FLAGS_SRQ_LAST CQ_RES_RAWETH_QP1_FLAGS_SRQ_SRQ
	+ uint32_t raweth_qp1_payload_offset_srq_or_rq_wr_id;
	+ /*
	+ * This value indicates the offset in bytes from the beginning of the
	+ * packet where the inner payload starts. This value is valid for TCP,
	+ * UDP, FCoE, and RoCE packets. A value of zero indicates an offset of
	+ * 256 bytes.
	+ */
	+ /*
	+ * Opaque value from RQ or SRQ WQE. Used by driver/lib to reference the
	+ * WQE in order to claim the received data and reuse the WQE space
	+ */
	+ #define CQ_RES_RAWETH_QP1_SRQ_OR_RQ_WR_ID_MASK UINT32_C(0xfffff)
	+ #define CQ_RES_RAWETH_QP1_SRQ_OR_RQ_WR_ID_SFT 0
	+ #define CQ_RES_RAWETH_QP1_RESERVED4_MASK UINT32_C(0xf00000)
	+ #define CQ_RES_RAWETH_QP1_RESERVED4_SFT 20
	+ /*
	+ * This value indicates the offset in bytes from the beginning of the
	+ * packet where the inner payload starts. This value is valid for TCP,
	+ * UDP, FCoE, and RoCE packets. A value of zero indicates an offset of
	+ * 256 bytes.
	+ */
	+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_PAYLOAD_OFFSET_MASK UINT32_C(0xff000000)
	+ #define CQ_RES_RAWETH_QP1_RAWETH_QP1_PAYLOAD_OFFSET_SFT 24
	+} __attribute__((packed));
	+
	+/* Terminal CQE (32 bytes) */
	+
	+struct cq_terminal {
	+ uint64_t qp_handle;
	+ /*
	+ * This is an application level ID used to identify the QP and its SQ
	+ * and RQ.
	+ */
	+ uint16_t sq_cons_idx;
	+ /*
	+ * Final SQ Consumer Index value. Any additional SQ WQEs will have to be
	+ * completed by the user provider.
	+ */
	+ uint16_t rq_cons_idx;
	+ /*
	+ * Final RQ Consumer Index value. Any additional RQ WQEs will have to be
	+ * completed by the user provider.
	+ */
	+ uint32_t reserved32_1;
	+ uint64_t reserved64_3;
	+ uint8_t cqe_type_toggle;
	+ /*
	+ * Indicate valid completion - written by the chip. Cumulus toggle this
	+ * bit each time it finished consuming all PBL entries
	+ */
	+ #define CQ_TERMINAL_TOGGLE UINT32_C(0x1)
	+ /* This field defines the type of SQ WQE. */
	+ #define CQ_TERMINAL_CQE_TYPE_MASK UINT32_C(0x1e)
	+ #define CQ_TERMINAL_CQE_TYPE_SFT 1
	+ /*
	+ * Terminal completion - This is used to indicate that no
	+ * further completions will be made for this QP on this CQ.
	+ */
	+ #define CQ_TERMINAL_CQE_TYPE_TERMINAL (UINT32_C(0xe) << 1)
	+ #define CQ_TERMINAL_RESERVED3_MASK UINT32_C(0xe0)
	+ #define CQ_TERMINAL_RESERVED3_SFT 5
	+ uint8_t status;
	+ /* This field indicates the status for the CQE. */
	+ /* OK is 0 */
	+ #define CQ_TERMINAL_STATUS_OK UINT32_C(0x0)
	+ uint16_t reserved16;
	+ uint32_t reserved32_2;
	+} __attribute__((packed));
	+
	+/* Cutoff CQE (32 bytes) */
	+
	+struct cq_cutoff {
	+ uint64_t reserved64_1;
	+ uint64_t reserved64_2;
	+ uint64_t reserved64_3;
	+ uint8_t cqe_type_toggle;
	+ /*
	+ * Indicate valid completion - written by the chip. Cumulus toggle this
	+ * bit each time it finished consuming all PBL entries
	+ */
	+ #define CQ_CUTOFF_TOGGLE UINT32_C(0x1)
	+ /* This field defines the type of SQ WQE. */
	+ #define CQ_CUTOFF_CQE_TYPE_MASK UINT32_C(0x1e)
	+ #define CQ_CUTOFF_CQE_TYPE_SFT 1
	+ /* Cut off CQE; for CQ resize see CQ and SRQ Resize */
	+ #define CQ_CUTOFF_CQE_TYPE_CUT_OFF (UINT32_C(0xf) << 1)
	+ #define CQ_CUTOFF_RESERVED3_MASK UINT32_C(0xe0)
	+ #define CQ_CUTOFF_RESERVED3_SFT 5
	+ uint8_t status;
	+ /* This field indicates the status for the CQE. */
	+ /* OK is 0 */
	+ #define CQ_CUTOFF_STATUS_OK UINT32_C(0x0)
	+ uint16_t reserved16;
	+ uint32_t reserved32;
	+} __attribute__((packed));
	+
	+/* Notification Queue (NQ) Structures */
	+/*
	+ * Description: This completion indicates that the DBQ has reached the
	+ * programmed threshold value.
	+ */
	+/* Base NQ Record (16 bytes) */
	+
	+struct nq_base {
	+ uint16_t info10_type;
	+ /* info10 is 10 b */
	+ /*
	+ * This field indicates the exact type of the completion. By convention,
	+ * the LSB identifies the length of the record in 16B units. Even values
	+ * indicate 16B records. Odd values indicate 32B records.
	+ */
	+ #define NQ_BASE_TYPE_MASK UINT32_C(0x3f)
	+ #define NQ_BASE_TYPE_SFT 0
	+ /* CQ Notification */
	+ #define NQ_BASE_TYPE_CQ_NOTIFICATION UINT32_C(0x30)
	+ /* SRQ Threshold Event */
	+ #define NQ_BASE_TYPE_SRQ_EVENT UINT32_C(0x32)
	+ /* DBQ Threshold Event */
	+ #define NQ_BASE_TYPE_DBQ_EVENT UINT32_C(0x34)
	+ /* QP Async Notification */
	+ #define NQ_BASE_TYPE_QP_EVENT UINT32_C(0x38)
	+ /* Function Async Notification */
	+ #define NQ_BASE_TYPE_FUNC_EVENT UINT32_C(0x3a)
	+ /* info10 is 10 b */
	+ #define NQ_BASE_INFO10_MASK UINT32_C(0xffc0)
	+ #define NQ_BASE_INFO10_SFT 6
	+ uint16_t info16;
	+ /* info16 is 16 b */
	+ uint32_t info32;
	+ /* info32 is 32 b */
	+ uint64_t info63_v;
	+ /* info63 is 63 b */
	+ /*
	+ * This value is written by the NIC such that it will be different for
	+ * each pass through the completion queue. The even passes will write 1.
	+ * The odd passes will write 0.
	+ */
	+ #define NQ_BASE_V UINT32_C(0x1)
	+ /* info63 is 63 b */
	+ #define NQ_BASE_INFO63_MASK UINT32_C(0xfffffffe)
	+ #define NQ_BASE_INFO63_SFT 1
	+} __attribute__((packed));
	+
	+/* Completion Queue Notification (16 bytes) */
	+
	+struct nq_cn {
	+ uint16_t type;
	+ /*
	+ * This field indicates the exact type of the completion. By convention,
	+ * the LSB identifies the length of the record in 16B units. Even values
	+ * indicate 16B records. Odd values indicate 32B records.
	+ */
	+ #define NQ_CN_TYPE_MASK UINT32_C(0x3f)
	+ #define NQ_CN_TYPE_SFT 0
	+ /* CQ Notification */
	+ #define NQ_CN_TYPE_CQ_NOTIFICATION UINT32_C(0x30)
	+ #define NQ_CN_RESERVED9_MASK UINT32_C(0xffc0)
	+ #define NQ_CN_RESERVED9_SFT 6
	+ uint16_t reserved16;
	+ uint32_t cq_handle_low;
	+ /*
	+ * This is an application level ID used to identify the CQ. This field
	+ * carries the lower 32b of the value.
	+ */
	+ uint32_t v;
	+ /*
	+ * This value is written by the NIC such that it will be different for
	+ * each pass through the completion queue. The even passes will write 1.
	+ * The odd passes will write 0.
	+ */
	+ #define NQ_CN_V UINT32_C(0x1)
	+ #define NQ_CN_RESERVED31_MASK UINT32_C(0xfffffffe)
	+ #define NQ_CN_RESERVED31_SFT 1
	+ uint32_t cq_handle_high;
	+ /*
	+ * This is an application level ID used to identify the CQ. This field
	+ * carries the upper 32b of the value.
	+ */
	+} __attribute__((packed));
	+
	+/* SRQ Event Notification (16 bytes) */
	+
	+struct nq_srq_event {
	+ uint8_t type;
	+ /*
	+ * This field indicates the exact type of the completion. By convention,
	+ * the LSB identifies the length of the record in 16B units. Even values
	+ * indicate 16B records. Odd values indicate 32B records.
	+ */
	+ #define NQ_SRQ_EVENT_TYPE_MASK UINT32_C(0x3f)
	+ #define NQ_SRQ_EVENT_TYPE_SFT 0
	+ /* SRQ Threshold Event */
	+ #define NQ_SRQ_EVENT_TYPE_SRQ_EVENT UINT32_C(0x32)
	+ #define NQ_SRQ_EVENT_RESERVED1_MASK UINT32_C(0xc0)
	+ #define NQ_SRQ_EVENT_RESERVED1_SFT 6
	+ uint8_t event;
	+ /* This value define what type of async event has occurred on the SRQ. */
	+ /* The threshold event has occurred on the specified SRQ. */
	+ #define NQ_SRQ_EVENT_EVENT_SRQ_THRESHOLD_EVENT UINT32_C(0x1)
	+ uint16_t reserved16;
	+ uint32_t srq_handle_low;
	+ /*
	+ * This is the SRQ handle value for the queue that has reached it's
	+ * event threshold. This field carries the lower 32b of the value.
	+ */
	+ uint32_t v;
	+ /*
	+ * This value is written by the NIC such that it will be different for
	+ * each pass through the completion queue. The even passes will write 1.
	+ * The odd passes will write 0.
	+ */
	+ #define NQ_SRQ_EVENT_V UINT32_C(0x1)
	+ #define NQ_SRQ_EVENT_RESERVED31_MASK UINT32_C(0xfffffffe)
	+ #define NQ_SRQ_EVENT_RESERVED31_SFT 1
	+ uint32_t srq_handle_high;
	+ /*
	+ * This is the SRQ handle value for the queue that has reached it's
	+ * event threshold. This field carries the upper 32b of the value.
	+ */
	+} __attribute__((packed));
	+
	+/* DBQ Async Event Notification (16 bytes) */
	+
	+struct nq_dbq_event {
	+ uint8_t type;
	+ /*
	+ * This field indicates the exact type of the completion. By convention,
	+ * the LSB identifies the length of the record in 16B units. Even values
	+ * indicate 16B records. Odd values indicate 32B records.
	+ */
	+ #define NQ_DBQ_EVENT_TYPE_MASK UINT32_C(0x3f)
	+ #define NQ_DBQ_EVENT_TYPE_SFT 0
	+ /* DBQ Threshold Event */
	+ #define NQ_DBQ_EVENT_TYPE_DBQ_EVENT UINT32_C(0x34)
	+ #define NQ_DBQ_EVENT_RESERVED1_MASK UINT32_C(0xc0)
	+ #define NQ_DBQ_EVENT_RESERVED1_SFT 6
	+ uint8_t event;
	+ /* This value define what type of action the driver should take. */
	+ /*
	+ * The driver should start writing dummy values to the the
	+ * doorbell in an attempt to consume all the PCIE posted write
	+ * resources and prevent doorbell overflow.
	+ */
	+ #define NQ_DBQ_EVENT_EVENT_DBQ_THRESHOLD_EVENT UINT32_C(0x1)
	+ uint16_t db_pfid;
	+ /*
	+ * This is the PFID of function that wrote the doorbell that crossed the
	+ * async event threshold.
	+ */
	+ #define NQ_DBQ_EVENT_DB_PFID_MASK UINT32_C(0xf)
	+ #define NQ_DBQ_EVENT_DB_PFID_SFT 0
	+ #define NQ_DBQ_EVENT_RESERVED12_MASK UINT32_C(0xfff0)
	+ #define NQ_DBQ_EVENT_RESERVED12_SFT 4
	+ uint32_t db_dpi;
	+ /*
	+ * This is the DPI of the doorbell write that crossed the async event
	+ * threshold.
	+ */
	+ #define NQ_DBQ_EVENT_DB_DPI_MASK UINT32_C(0xfffff)
	+ #define NQ_DBQ_EVENT_DB_DPI_SFT 0
	+ #define NQ_DBQ_EVENT_RESERVED12_2_MASK UINT32_C(0xfff00000)
	+ #define NQ_DBQ_EVENT_RESERVED12_2_SFT 20
	+ uint32_t v;
	+ /*
	+ * This value is written by the NIC such that it will be different for
	+ * each pass through the completion queue. The even passes will write 1.
	+ * The odd passes will write 0.
	+ */
	+ #define NQ_DBQ_EVENT_V UINT32_C(0x1)
	+ #define NQ_DBQ_EVENT_RESERVED32_MASK UINT32_C(0xfffffffe)
	+ #define NQ_DBQ_EVENT_RESERVED32_SFT 1
	+ uint32_t db_type_db_xid;
	+ /* DB 'type' field from doorbell that crossed the async event threshold. */
	+ /*
	+ * DB 'XID' field from doorbell that crossed the async event threshold.
	+ * This is a QPID, SID, or CID, depending on the db_type field.
	+ */
	+ #define NQ_DBQ_EVENT_DB_XID_MASK UINT32_C(0xfffff)
	+ #define NQ_DBQ_EVENT_DB_XID_SFT 0
	+ #define NQ_DBQ_EVENT_RESERVED8_MASK UINT32_C(0xff00000)
	+ #define NQ_DBQ_EVENT_RESERVED8_SFT 20
	+ /* DB 'type' field from doorbell that crossed the async event threshold. */
	+ #define NQ_DBQ_EVENT_DB_TYPE_MASK UINT32_C(0xf0000000)
	+ #define NQ_DBQ_EVENT_DB_TYPE_SFT 28
	+} __attribute__((packed));
	+
	+/* Read Request/Response Queue Structures */
	+/*
	+ * Description: This queue messages is used on the ORRQ to indicate output read
	+ * requests to the RX side of the chip.
	+ */
	+/* Input Read Request Queue (IRRQ) Message (32 bytes) */
	+
	+struct xrrq_irrq {
	+ uint16_t credits_type;
	+ /*
	+ * The credit code calculated by Rx path when receiving the request. It
	+ * will be placed in the syndrome credit code with the acks on first and
	+ * last response.
	+ */
	+ /* Type indication */
	+ #define XRRQ_IRRQ_TYPE UINT32_C(0x1)
	+ /* RDMA Read */
	+ #define XRRQ_IRRQ_TYPE_READ_REQ UINT32_C(0x0)
	+ /* Atomic */
	+ #define XRRQ_IRRQ_TYPE_ATOMIC_REQ UINT32_C(0x1)
	+ #define XRRQ_IRRQ_RESERVED10_MASK UINT32_C(0x7fe)
	+ #define XRRQ_IRRQ_RESERVED10_SFT 1
	+ /*
	+ * The credit code calculated by Rx path when receiving the request. It
	+ * will be placed in the syndrome credit code with the acks on first and
	+ * last response.
	+ */
	+ #define XRRQ_IRRQ_CREDITS_MASK UINT32_C(0xf800)
	+ #define XRRQ_IRRQ_CREDITS_SFT 11
	+ uint16_t reserved16;
	+ uint32_t reserved32;
	+ uint32_t psn;
	+ /* The PSN of the outstanding incoming request */
	+ #define XRRQ_IRRQ_PSN_MASK UINT32_C(0xffffff)
	+ #define XRRQ_IRRQ_PSN_SFT 0
	+ #define XRRQ_IRRQ_RESERVED8_1_MASK UINT32_C(0xff000000)
	+ #define XRRQ_IRRQ_RESERVED8_1_SFT 24
	+ uint32_t msn;
	+ /*
	+ * The value of QPC.pending_ack_msn after it is incremented as a result
	+ * of receiving the read/atomic request. IRRQ.msn-1 will be placed in
	+ * the MSN field of the first response and IRRQ.msn will placed in the
	+ * MSN field of the last or only response.
	+ */
	+ #define XRRQ_IRRQ_MSN_MASK UINT32_C(0xffffff)
	+ #define XRRQ_IRRQ_MSN_SFT 0
	+ #define XRRQ_IRRQ_RESERVED8_2_MASK UINT32_C(0xff000000)
	+ #define XRRQ_IRRQ_RESERVED8_2_SFT 24
	+ uint64_t va_or_atomic_result;
	+ /*
	+ * Virtual address on local host for RDMA READ In case of duplicate
	+ * Atomic, the VA is not required to be validated, only the PSN is, thus
	+ * this field is used to store the value returned in the Ack to the
	+ * atomic request, and if duplicate arrives, this value is used again
	+ * for resending the ack.
	+ */
	+ uint32_t rdma_r_key;
	+ /* The key to the MR/W in the request */
	+ uint32_t length;
	+ /*
	+ * Length in bytes of the data requested. Length must be 8 if type is
	+ * atomic.
	+ */
	+} __attribute__((packed));
	+
	+/* Output Read Request Queue (ORRQ) Message (32 bytes) */
	+
	+struct xrrq_orrq {
	+ uint16_t num_sges_type;
	+ /*
	+ * Up to 6 SGEs. This value is 1 if type is atomic as one SGE is
	+ * required to store Atomic response result field. 2 more bits allocated
	+ * for future growth. Note that, if num_sges is 1 for an RDMA Read
	+ * request, then the first_sge_phy_or_sing_sge_va, single_sge_l_key, and
	+ * single_sge_size fields will be populated from the single SGE. If
	+ * num_sges is 2 or more for an RDMA Read request, then the
	+ * first_sge_phy_or_sing_sge_va field carries the physical address in
	+ * host memory where the first sge is stored. The single_sge_l_key and
	+ * single_sge_size fields are unused in this case. A special case is a
	+ * zero-length, zero-sge RDMA read request WQE. In this situation,
	+ * num_sges will be 1. However, first_sge_phy_or_sing_sge_va,
	+ * single_sge_l_key, and single_sge_size will all be populated with
	+ * zeros.
	+ */
	+ /* Type indication */
	+ #define XRRQ_ORRQ_TYPE UINT32_C(0x1)
	+ /* RDMA Read */
	+ #define XRRQ_ORRQ_TYPE_READ_REQ UINT32_C(0x0)
	+ /* Atomic */
	+ #define XRRQ_ORRQ_TYPE_ATOMIC_REQ UINT32_C(0x1)
	+ #define XRRQ_ORRQ_RESERVED10_MASK UINT32_C(0x7fe)
	+ #define XRRQ_ORRQ_RESERVED10_SFT 1
	+ /*
	+ * Up to 6 SGEs. This value is 1 if type is atomic as one SGE is
	+ * required to store Atomic response result field. 2 more bits allocated
	+ * for future growth. Note that, if num_sges is 1 for an RDMA Read
	+ * request, then the first_sge_phy_or_sing_sge_va, single_sge_l_key, and
	+ * single_sge_size fields will be populated from the single SGE. If
	+ * num_sges is 2 or more for an RDMA Read request, then the
	+ * first_sge_phy_or_sing_sge_va field carries the physical address in
	+ * host memory where the first sge is stored. The single_sge_l_key and
	+ * single_sge_size fields are unused in this case. A special case is a
	+ * zero-length, zero-sge RDMA read request WQE. In this situation,
	+ * num_sges will be 1. However, first_sge_phy_or_sing_sge_va,
	+ * single_sge_l_key, and single_sge_size will all be populated with
	+ * zeros.
	+ */
	+ #define XRRQ_ORRQ_NUM_SGES_MASK UINT32_C(0xf800)
	+ #define XRRQ_ORRQ_NUM_SGES_SFT 11
	+ uint16_t reserved16;
	+ uint32_t length;
	+ /*
	+ * Length in bytes of the data requested. Length must be 8 if type is
	+ * atomic.
	+ */
	+ uint32_t psn;
	+ /* The PSN of the outstanding outgoing request */
	+ #define XRRQ_ORRQ_PSN_MASK UINT32_C(0xffffff)
	+ #define XRRQ_ORRQ_PSN_SFT 0
	+ #define XRRQ_ORRQ_RESERVED8_1_MASK UINT32_C(0xff000000)
	+ #define XRRQ_ORRQ_RESERVED8_1_SFT 24
	+ uint32_t end_psn;
	+ /*
	+ * The expected last PSN on a response to this request where an ack with
	+ * response, rather than just response, should arrive. If ack arrive
	+ * with smaller PSN than end_psn then it is considered a NAK.
	+ */
	+ #define XRRQ_ORRQ_END_PSN_MASK UINT32_C(0xffffff)
	+ #define XRRQ_ORRQ_END_PSN_SFT 0
	+ #define XRRQ_ORRQ_RESERVED8_2_MASK UINT32_C(0xff000000)
	+ #define XRRQ_ORRQ_RESERVED8_2_SFT 24
	+ uint64_t first_sge_phy_or_sing_sge_va;
	+ /*
	+ * If num_sges == 1 this is the va of that SGE. Otherwise, physical
	+ * address to the first SGE specified by the WQE. Points to the first
	+ * SGE in the Request's WQE in the SQ. It is assumed that WQE does not
	+ * cross page boundaries! Driver is responsible to enforce that. SGEs
	+ * are 16B aligned 0b0000 lsb added to get 64 bit address.
	+ */
	+ uint32_t single_sge_l_key;
	+ /* The L_Key of a single SGE if used */
	+ uint32_t single_sge_size;
	+ /* The size in bytes of the single SGE if used */
	+} __attribute__((packed));
	+
	+/* Page Buffer List Memory Structures (PBL) */
	+/*
	+ * Description: Page directory entries point to a page directories made up of
	+ * PTE values.
	+ */
	+/* Page Table Entry (PTE) (8 bytes) */
	+
	+struct ptu_pte {
	+ uint64_t page_next_to_last_last_valid;
	+ /*
	+ * This is the upper bits of the physical page controlled by this PTE.
	+ * If the page is larger than 4KB, then the unused lower bits of the
	+ * page address should be zero.
	+ */
	+ /*
	+ * This field indicates if the PTE is valid. A value of '0' indicates
	+ * that the page is not valid. A value of '1' indicates that the page is
	+ * valid. A reference to an invalid page will return a PTU error.
	+ */
	+ #define PTU_PTE_VALID UINT32_C(0x1)
	+ /*
	+ * This field is used only for "ring" PBLs that are used for SQ, RQ,
	+ * SRQ, or CQ structures. For all other PBL structures, this bit should
	+ * be zero. When this bit is '1', it indicates that the page pointed to
	+ * by this PTE is the last page in the ring. A prefetch for the ring
	+ * should use the first PTE in the PBL.
	+ */
	+ #define PTU_PTE_LAST UINT32_C(0x2)
	+ /*
	+ * This field is used only for "ring" PBLs that are used for SQ, RQ,
	+ * SRQ, or CQ structures. For all other PBL structures, this bit should
	+ * be zero. When this bit is '1', it indicates that this is the next-to-
	+ * last page of the PBL.
	+ */
	+ #define PTU_PTE_NEXT_TO_LAST UINT32_C(0x4)
	+ /* These bits should be programmed to zero. */
	+ /*
	+ * This is the upper bits of the physical page controlled by this PTE.
	+ * If the page is larger than 4KB, then the unused lower bits of the
	+ * page address should be zero.
	+ */
	+ #define PTU_PTE_PAGE_MASK UINT32_C(0xfffff000)
	+ #define PTU_PTE_PAGE_SFT 12
	+} __attribute__((packed));
	+
	+/* Page Directory Entry (PDE) (8 bytes) */
	+
	+struct ptu_pde {
	+ uint64_t page_valid;
	+ /*
	+ * This is the upper bits of the physical page controlled by this PTE.
	+ * If the page is larger than 4KB, then the unused lower bits of the
	+ * page address should be zero.
	+ */
	+ /*
	+ * This field indicates if the PTE is valid. A value of '0' indicates
	+ * that the page is not valid. A value of '1' indicates that the page is
	+ * valid. A reference to an invalid page will return a PTU error.
	+ */
	+ #define PTU_PDE_VALID UINT32_C(0x1)
	+ /* These bits should be programmed to zero. */
	+ /*
	+ * This is the upper bits of the physical page controlled by this PTE.
	+ * If the page is larger than 4KB, then the unused lower bits of the
	+ * page address should be zero.
	+ */
	+ #define PTU_PDE_PAGE_MASK UINT32_C(0xfffff000)
	+ #define PTU_PDE_PAGE_SFT 12
	+} __attribute__((packed));
	+
	+/* RoCE Fastpath Host Structures */
	+/*
	+ * Note: This section documents the host structures used between RoCE state
	+ * machines and RoCE drivers/libraries.
	+ */
	/* hwrm_ver_get */
	/*
	* Description: This function is called by a driver to determine the HWRM
	@@ -4134,11 +5631,32 @@
	* firmware branches or customer specific releases tied to a specific
	* (major,minor,update) version
	*/
	- uint32_t reserved1;
	+ uint32_t dev_caps_cfg;
	/*
	- * This field is reserved for future use. The responder should set it to
	- * 0. The requester should ignore this field.
	+ * This field is used to indicate device's capabilities and
	+ * configurations.
	*/
	+ /*
	+ * If set to 1, then secure firmware update behavior is supported. If
	+ * set to 0, then secure firmware update behavior is not supported.
	+ */
	+ #define HWRM_VER_GET_OUTPUT_DEV_CAPS_CFG_SECURE_FW_UPD_SUPPORTED UINT32_C(0x1)
	+ /*
	+ * If set to 1, then firmware based DCBX agent is supported. If set to
	+ * 0, then firmware based DCBX agent capability is not supported on this
	+ * device.
	+ */
	+ #define HWRM_VER_GET_OUTPUT_DEV_CAPS_CFG_FW_DCBX_AGENT_SUPPORTED UINT32_C(0x2)
	+ /*
	+ * If set to 1, then HWRM short command format is supported. If set to
	+ * 0, then HWRM short command format is not supported.
	+ */
	+ #define HWRM_VER_GET_OUTPUT_DEV_CAPS_CFG_SHORT_CMD_SUPPORTED UINT32_C(0x4)
	+ /*
	+ * If set to 1, then HWRM short command format is required. If set to 0,
	+ * then HWRM short command format is not required.
	+ */
	+ #define HWRM_VER_GET_OUTPUT_DEV_CAPS_CFG_SHORT_CMD_REQUIRED UINT32_C(0x8)
	uint8_t roce_fw_maj;
	/*
	* This field represents the major version of RoCE firmware. A change in
	@@ -4214,9 +5732,15 @@
	/* This field returns the maximum value of response buffer in bytes. */
	uint16_t def_req_timeout;
	/* This field returns the default request timeout value in milliseconds. */
	+ uint8_t init_pending;
	+ /* This field will indicate if any subsystems is not fully initialized. */
	+ /*
	+ * If set to 1, device is not ready. If set to 0, device is ready to
	+ * accept all HWRM commands.
	+ */
	+ #define HWRM_VER_GET_OUTPUT_INIT_PENDING_DEV_NOT_RDY UINT32_C(0x1)
	uint8_t unused_0;
	uint8_t unused_1;
	- uint8_t unused_2;
	uint8_t valid;
	/*
	* This field is used in Output records to indicate that the output is
	@@ -4383,7 +5907,7 @@
	/*
	* This value is the PCI ID of the queried function. If ARI is enabled,
	* then it is Bus Number (8b):Function Number(8b). Otherwise, it is Bus
	- * Number (8b):Device Number (4b):Function Number(4b).
	+ * Number (8b):Device Number (5b):Function Number(3b).
	*/
	uint16_t unused_0;
	} __attribute__((packed));
	@@ -4807,15 +6331,51 @@
	*/
	#define HWRM_FUNC_QCAPS_OUTPUT_FLAGS_ROCE_V2_SUPPORTED UINT32_C(0x10)
	/*
	- * If 1, then control and configuration of WoL magic packet is supported
	- * on this function.
	+ * If 1, then control and configuration of WoL magic packet are
	+ * supported on this function.
	*/
	#define HWRM_FUNC_QCAPS_OUTPUT_FLAGS_WOL_MAGICPKT_SUPPORTED UINT32_C(0x20)
	/*
	- * If 1, then control and configuration of bitmap pattern packet is
	+ * If 1, then control and configuration of bitmap pattern packet are
	* supported on this function.
	*/
	#define HWRM_FUNC_QCAPS_OUTPUT_FLAGS_WOL_BMP_SUPPORTED UINT32_C(0x40)
	+ /*
	+ * If set to 1, then the control and configuration of rate limit of an
	+ * allocated TX ring on the queried function is supported.
	+ */
	+ #define HWRM_FUNC_QCAPS_OUTPUT_FLAGS_TX_RING_RL_SUPPORTED UINT32_C(0x80)
	+ /*
	+ * If 1, then control and configuration of minimum and maximum
	+ * bandwidths are supported on the queried function.
	+ */
	+ #define HWRM_FUNC_QCAPS_OUTPUT_FLAGS_TX_BW_CFG_SUPPORTED UINT32_C(0x100)
	+ /*
	+ * If the query is for a VF, then this flag shall be ignored. If this
	+ * query is for a PF and this flag is set to 1, then the PF has the
	+ * capability to set the rate limits on the TX rings of its children
	+ * VFs. If this query is for a PF and this flag is set to 0, then the PF
	+ * does not have the capability to set the rate limits on the TX rings
	+ * of its children VFs.
	+ */
	+ #define HWRM_FUNC_QCAPS_OUTPUT_FLAGS_VF_TX_RING_RL_SUPPORTED UINT32_C(0x200)
	+ /*
	+ * If the query is for a VF, then this flag shall be ignored. If this
	+ * query is for a PF and this flag is set to 1, then the PF has the
	+ * capability to set the minimum and/or maximum bandwidths for its
	+ * children VFs. If this query is for a PF and this flag is set to 0,
	+ * then the PF does not have the capability to set the minimum or
	+ * maximum bandwidths for its children VFs.
	+ */
	+ #define HWRM_FUNC_QCAPS_OUTPUT_FLAGS_VF_BW_CFG_SUPPORTED UINT32_C(0x400)
	+ /*
	+ * Standard TX Ring mode is used for the allocation of TX ring and
	+ * underlying scheduling resources that allow bandwidth reservation and
	+ * limit settings on the queried function. If set to 1, then standard TX
	+ * ring mode is supported on the queried function. If set to 0, then
	+ * standard TX ring mode is not available on the queried function.
	+ */
	+ #define HWRM_FUNC_QCAPS_OUTPUT_FLAGS_STD_TX_RING_MODE_SUPPORTED UINT32_C(0x800)
	uint8_t mac_address[6];
	/*
	* This value is current MAC address configured for this function. A
	@@ -4911,9 +6471,15 @@
	* The maximum number of HW ring groups that can be supported on this
	* function.
	*/
	+ uint16_t max_sp_tx_rings;
	+ /*
	+ * The maximum number of strict priority transmit rings that can be
	+ * allocated to the function. This number indicates the maximum number
	+ * of TX rings that can be assigned strict priorities out of the maximum
	+ * number of TX rings that can be allocated (max_tx_rings) to the
	+ * function.
	+ */
	uint8_t unused_0;
	- uint8_t unused_1;
	- uint8_t unused_2;
	uint8_t valid;
	/*
	* This field is used in Output records to indicate that the output is
	@@ -4931,7 +6497,11 @@
	* allows a physical function driver to query virtual functions that are
	* children of the physical function. The output FID value is needed to
	* configure Rings and MSI-X vectors so their DMA operations appear correctly on
	- * the PCI bus.
	+ * the PCI bus. This command should be called by every driver after
	+ * 'hwrm_func_cfg' to get the actual number of resources allocated by the HWRM.
	+ * The values returned by hwrm_func_qcfg are the values the driver shall use.
	+ * These values may be different than what was originally requested in the
	+ * 'hwrm_func_cfg' command.
	*/
	/* Input (24 bytes) */

	@@ -5016,6 +6586,35 @@
	* the port associated with this function.
	*/
	#define HWRM_FUNC_QCFG_OUTPUT_FLAGS_OOB_WOL_BMP_ENABLED UINT32_C(0x2)
	+ /*
	+ * If set to 1, then FW based DCBX agent is enabled and running on the
	+ * port associated with this function. If set to 0, then DCBX agent is
	+ * not running in the firmware.
	+ */
	+ #define HWRM_FUNC_QCFG_OUTPUT_FLAGS_FW_DCBX_AGENT_ENABLED UINT32_C(0x4)
	+ /*
	+ * Standard TX Ring mode is used for the allocation of TX ring and
	+ * underlying scheduling resources that allow bandwidth reservation and
	+ * limit settings on the queried function. If set to 1, then standard TX
	+ * ring mode is enabled on the queried function. If set to 0, then the
	+ * standard TX ring mode is disabled on the queried function. In this
	+ * extended TX ring resource mode, the minimum and maximum bandwidth
	+ * settings are not supported to allow the allocation of TX rings to
	+ * span multiple scheduler nodes.
	+ */
	+ #define HWRM_FUNC_QCFG_OUTPUT_FLAGS_STD_TX_RING_MODE_ENABLED UINT32_C(0x8)
	+ /*
	+ * If set to 1 then FW based LLDP agent is enabled and running on the
	+ * port associated with this function. If set to 0 then the LLDP agent
	+ * is not running in the firmware.
	+ */
	+ #define HWRM_FUNC_QCFG_OUTPUT_FLAGS_FW_LLDP_AGENT_ENABLED UINT32_C(0x10)
	+ /*
	+ * If set to 1, then multi-host mode is active for this function. If set
	+ * to 0, then multi-host mode is inactive for this function or not
	+ * applicable for this device.
	+ */
	+ #define HWRM_FUNC_QCFG_OUTPUT_FLAGS_MULTI_HOST UINT32_C(0x20)
	uint8_t mac_address[6];
	/*
	* This value is current MAC address configured for this function. A
	@@ -5026,7 +6625,9 @@
	/*
	* This value is current PCI ID of this function. If ARI is enabled,
	* then it is Bus Number (8b):Function Number(8b). Otherwise, it is Bus
	- * Number (8b):Device Number (4b):Function Number(4b).
	+ * Number (8b):Device Number (4b):Function Number(4b). If multi-host
	+ * mode is active, the 4 lsb will indicate the PF index for this
	+ * function.
	*/
	uint16_t alloc_rsscos_ctx;
	/* The number of RSS/COS contexts currently allocated to the function. */
	@@ -5082,25 +6683,85 @@
	#define HWRM_FUNC_QCFG_OUTPUT_PORT_PARTITION_TYPE_NPAR2_0 UINT32_C(0x4)
	/* Unknown */
	#define HWRM_FUNC_QCFG_OUTPUT_PORT_PARTITION_TYPE_UNKNOWN UINT32_C(0xff)
	- uint8_t unused_0;
	+ uint8_t port_pf_cnt;
	+ /*
	+ * This field will indicate number of physical functions on this
	+ * port_partition. HWRM shall return unavail (i.e. value of 0) for this
	+ * field when this command is used to query VF's configuration or from
	+ * older firmware that doesn't support this field.
	+ */
	+ /* number of PFs is not available */
	+ #define HWRM_FUNC_QCFG_OUTPUT_PORT_PF_CNT_UNAVAIL UINT32_C(0x0)
	uint16_t dflt_vnic_id;
	/* The default VNIC ID assigned to a function that is being queried. */
	+ uint8_t unused_0;
	uint8_t unused_1;
	- uint8_t unused_2;
	uint32_t min_bw;
	/*
	- * Minimum BW allocated for this function in Mbps. The HWRM will
	- * translate this value into byte counter and time interval used for the
	- * scheduler inside the device. A value of 0 indicates the minimum
	- * bandwidth is not configured.
	+ * Minimum BW allocated for this function. The HWRM will translate this
	+ * value into byte counter and time interval used for the scheduler
	+ * inside the device. A value of 0 indicates the minimum bandwidth is
	+ * not configured.
	*/
	+ /* The bandwidth value. */
	+ #define HWRM_FUNC_QCFG_OUTPUT_MIN_BW_BW_VALUE_MASK UINT32_C(0xfffffff)
	+ #define HWRM_FUNC_QCFG_OUTPUT_MIN_BW_BW_VALUE_SFT 0
	+ /* The granularity of the value (bits or bytes). */
	+ #define HWRM_FUNC_QCFG_OUTPUT_MIN_BW_SCALE UINT32_C(0x10000000)
	+ /* Value is in bits. */
	+ #define HWRM_FUNC_QCFG_OUTPUT_MIN_BW_SCALE_BITS (UINT32_C(0x0) << 28)
	+ /* Value is in bytes. */
	+ #define HWRM_FUNC_QCFG_OUTPUT_MIN_BW_SCALE_BYTES (UINT32_C(0x1) << 28)
	+ #define HWRM_FUNC_QCFG_OUTPUT_MIN_BW_SCALE_LAST HWRM_FUNC_QCFG_OUTPUT_MIN_BW_SCALE_BYTES
	+ /* bw_value_unit is 3 b */
	+ #define HWRM_FUNC_QCFG_OUTPUT_MIN_BW_BW_VALUE_UNIT_MASK UINT32_C(0xe0000000)
	+ #define HWRM_FUNC_QCFG_OUTPUT_MIN_BW_BW_VALUE_UNIT_SFT 29
	+ /* Value is in Mb or MB (base 10). */
	+ #define HWRM_FUNC_QCFG_OUTPUT_MIN_BW_BW_VALUE_UNIT_MEGA (UINT32_C(0x0) << 29)
	+ /* Value is in Kb or KB (base 10). */
	+ #define HWRM_FUNC_QCFG_OUTPUT_MIN_BW_BW_VALUE_UNIT_KILO (UINT32_C(0x2) << 29)
	+ /* Value is in bits or bytes. */
	+ #define HWRM_FUNC_QCFG_OUTPUT_MIN_BW_BW_VALUE_UNIT_BASE (UINT32_C(0x4) << 29)
	+ /* Value is in Gb or GB (base 10). */
	+ #define HWRM_FUNC_QCFG_OUTPUT_MIN_BW_BW_VALUE_UNIT_GIGA (UINT32_C(0x6) << 29)
	+ /* Value is in 1/100th of a percentage of total bandwidth. */
	+ #define HWRM_FUNC_QCFG_OUTPUT_MIN_BW_BW_VALUE_UNIT_PERCENT1_100 (UINT32_C(0x1) << 29)
	+ /* Invalid unit */
	+ #define HWRM_FUNC_QCFG_OUTPUT_MIN_BW_BW_VALUE_UNIT_INVALID (UINT32_C(0x7) << 29)
	+ #define HWRM_FUNC_QCFG_OUTPUT_MIN_BW_BW_VALUE_UNIT_LAST HWRM_FUNC_QCFG_OUTPUT_MIN_BW_BW_VALUE_UNIT_INVALID
	uint32_t max_bw;
	/*
	- * Maximum BW allocated for this function in Mbps. The HWRM will
	- * translate this value into byte counter and time interval used for the
	- * scheduler inside the device. A value of 0 indicates that the maximum
	- * bandwidth is not configured.
	+ * Maximum BW allocated for this function. The HWRM will translate this
	+ * value into byte counter and time interval used for the scheduler
	+ * inside the device. A value of 0 indicates that the maximum bandwidth
	+ * is not configured.
	*/
	+ /* The bandwidth value. */
	+ #define HWRM_FUNC_QCFG_OUTPUT_MAX_BW_BW_VALUE_MASK UINT32_C(0xfffffff)
	+ #define HWRM_FUNC_QCFG_OUTPUT_MAX_BW_BW_VALUE_SFT 0
	+ /* The granularity of the value (bits or bytes). */
	+ #define HWRM_FUNC_QCFG_OUTPUT_MAX_BW_SCALE UINT32_C(0x10000000)
	+ /* Value is in bits. */
	+ #define HWRM_FUNC_QCFG_OUTPUT_MAX_BW_SCALE_BITS (UINT32_C(0x0) << 28)
	+ /* Value is in bytes. */
	+ #define HWRM_FUNC_QCFG_OUTPUT_MAX_BW_SCALE_BYTES (UINT32_C(0x1) << 28)
	+ #define HWRM_FUNC_QCFG_OUTPUT_MAX_BW_SCALE_LAST HWRM_FUNC_QCFG_OUTPUT_MAX_BW_SCALE_BYTES
	+ /* bw_value_unit is 3 b */
	+ #define HWRM_FUNC_QCFG_OUTPUT_MAX_BW_BW_VALUE_UNIT_MASK UINT32_C(0xe0000000)
	+ #define HWRM_FUNC_QCFG_OUTPUT_MAX_BW_BW_VALUE_UNIT_SFT 29
	+ /* Value is in Mb or MB (base 10). */
	+ #define HWRM_FUNC_QCFG_OUTPUT_MAX_BW_BW_VALUE_UNIT_MEGA (UINT32_C(0x0) << 29)
	+ /* Value is in Kb or KB (base 10). */
	+ #define HWRM_FUNC_QCFG_OUTPUT_MAX_BW_BW_VALUE_UNIT_KILO (UINT32_C(0x2) << 29)
	+ /* Value is in bits or bytes. */
	+ #define HWRM_FUNC_QCFG_OUTPUT_MAX_BW_BW_VALUE_UNIT_BASE (UINT32_C(0x4) << 29)
	+ /* Value is in Gb or GB (base 10). */
	+ #define HWRM_FUNC_QCFG_OUTPUT_MAX_BW_BW_VALUE_UNIT_GIGA (UINT32_C(0x6) << 29)
	+ /* Value is in 1/100th of a percentage of total bandwidth. */
	+ #define HWRM_FUNC_QCFG_OUTPUT_MAX_BW_BW_VALUE_UNIT_PERCENT1_100 (UINT32_C(0x1) << 29)
	+ /* Invalid unit */
	+ #define HWRM_FUNC_QCFG_OUTPUT_MAX_BW_BW_VALUE_UNIT_INVALID (UINT32_C(0x7) << 29)
	+ #define HWRM_FUNC_QCFG_OUTPUT_MAX_BW_BW_VALUE_UNIT_LAST HWRM_FUNC_QCFG_OUTPUT_MAX_BW_BW_VALUE_UNIT_INVALID
	uint8_t evb_mode;
	/*
	* This value indicates the Edge virtual bridge mode for the domain that
	@@ -5112,8 +6773,13 @@
	#define HWRM_FUNC_QCFG_OUTPUT_EVB_MODE_VEB UINT32_C(0x1)
	/* Virtual Ethernet Port Aggregator (VEPA) */
	#define HWRM_FUNC_QCFG_OUTPUT_EVB_MODE_VEPA UINT32_C(0x2)
	- uint8_t unused_3;
	- uint16_t unused_4;
	+ uint8_t unused_2;
	+ uint16_t alloc_vfs;
	+ /*
	+ * The number of VFs that are allocated to the function. This is valid
	+ * only on the PF with SR-IOV enabled. 0xFF... (All Fs) if this command
	+ * is called on a PF with SR-IOV disabled or on a VF.
	+ */
	uint32_t alloc_mcast_filters;
	/*
	* The number of allocated multicast filters for this function on the RX
	@@ -5121,9 +6787,225 @@
	*/
	uint32_t alloc_hw_ring_grps;
	/* The number of allocated HW ring groups for this function. */
	+ uint16_t alloc_sp_tx_rings;
	+ /*
	+ * The number of strict priority transmit rings out of currently
	+ * allocated TX rings to the function (alloc_tx_rings).
	+ */
	+ uint8_t unused_3;
	+ uint8_t valid;
	+ /*
	+ * This field is used in Output records to indicate that the output is
	+ * completely written to RAM. This field should be read as '1' to
	+ * indicate that the output has been completely written. When writing a
	+ * command completion or response to an internal processor, the order of
	+ * writes has to be such that this field is written last.
	+ */
	+} __attribute__((packed));
	+
	+/* hwrm_func_vlan_qcfg */
	+/*
	+ * Description: This command should be called by PF driver to get the current
	+ * C-TAG, S-TAG and correcponsing PCP and TPID values configured for the
	+ * function.
	+ */
	+/* Input (24 bytes) */
	+
	+struct hwrm_func_vlan_qcfg_input {
	+ uint16_t req_type;
	+ /*
	+ * This value indicates what type of request this is. The format for the
	+ * rest of the command is determined by this field.
	+ */
	+ uint16_t cmpl_ring;
	+ /*
	+ * This value indicates the what completion ring the request will be
	+ * optionally completed on. If the value is -1, then no CR completion
	+ * will be generated. Any other value must be a valid CR ring_id value
	+ * for this function.
	+ */
	+ uint16_t seq_id;
	+ /* This value indicates the command sequence number. */
	+ uint16_t target_id;
	+ /*
	+ * Target ID of this command. 0x0 - 0xFFF8 - Used for function ids
	+ * 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF - HWRM
	+ */
	+ uint64_t resp_addr;
	+ /*
	+ * This is the host address where the response will be written when the
	+ * request is complete. This area must be 16B aligned and must be
	+ * cleared to zero before the request is made.
	+ */
	+ uint16_t fid;
	+ /*
	+ * Function ID of the function that is being configured. If set to
	+ * 0xFF... (All Fs), then the configuration is for the requesting
	+ * function.
	+ */
	+ uint16_t unused_0[3];
	+} __attribute__((packed));
	+
	+/* Output (40 bytes) */
	+
	+struct hwrm_func_vlan_qcfg_output {
	+ uint16_t error_code;
	+ /*
	+ * Pass/Fail or error type Note: receiver to verify the in parameters,
	+ * and fail the call with an error when appropriate
	+ */
	+ uint16_t req_type;
	+ /* This field returns the type of original request. */
	+ uint16_t seq_id;
	+ /* This field provides original sequence number of the command. */
	+ uint16_t resp_len;
	+ /*
	+ * This field is the length of the response in bytes. The last byte of
	+ * the response is a valid flag that will read as '1' when the command
	+ * has been completely written to memory.
	+ */
	+ uint32_t unused_0;
	+ uint8_t unused_1;
	+ uint8_t unused_2;
	+ uint8_t unused_3;
	+ uint8_t valid;
	+ /*
	+ * This field is used in Output records to indicate that the output is
	+ * completely written to RAM. This field should be read as '1' to
	+ * indicate that the output has been completely written. When writing a
	+ * command completion or response to an internal processor, the order of
	+ * writes has to be such that this field is written last.
	+ */
	+ uint16_t stag_vid;
	+ /* S-TAG VLAN identifier configured for the function. */
	+ uint8_t stag_pcp;
	+ /* S-TAG PCP value configured for the function. */
	+ uint8_t unused_4;
	+ uint16_t stag_tpid; /* big endian */
	+ /*
	+ * S-TAG TPID value configured for the function. This field is specified
	+ * in network byte order.
	+ */
	+ uint16_t ctag_vid;
	+ /* C-TAG VLAN identifier configured for the function. */
	+ uint8_t ctag_pcp;
	+ /* C-TAG PCP value configured for the function. */
	uint8_t unused_5;
	- uint8_t unused_6;
	- uint8_t unused_7;
	+ uint16_t ctag_tpid; /* big endian */
	+ /*
	+ * C-TAG TPID value configured for the function. This field is specified
	+ * in network byte order.
	+ */
	+ uint32_t rsvd2;
	+ /* Future use. */
	+ uint32_t rsvd3;
	+ /* Future use. */
	+ uint32_t unused_6;
	+} __attribute__((packed));
	+
	+/* hwrm_func_vlan_cfg */
	+/*
	+ * Description: This command allows PF driver to configure C-TAG, S-TAG and
	+ * corresponding PCP and TPID values for a function.
	+ */
	+/* Input (48 bytes) */
	+
	+struct hwrm_func_vlan_cfg_input {
	+ uint16_t req_type;
	+ /*
	+ * This value indicates what type of request this is. The format for the
	+ * rest of the command is determined by this field.
	+ */
	+ uint16_t cmpl_ring;
	+ /*
	+ * This value indicates the what completion ring the request will be
	+ * optionally completed on. If the value is -1, then no CR completion
	+ * will be generated. Any other value must be a valid CR ring_id value
	+ * for this function.
	+ */
	+ uint16_t seq_id;
	+ /* This value indicates the command sequence number. */
	+ uint16_t target_id;
	+ /*
	+ * Target ID of this command. 0x0 - 0xFFF8 - Used for function ids
	+ * 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF - HWRM
	+ */
	+ uint64_t resp_addr;
	+ /*
	+ * This is the host address where the response will be written when the
	+ * request is complete. This area must be 16B aligned and must be
	+ * cleared to zero before the request is made.
	+ */
	+ uint16_t fid;
	+ /*
	+ * Function ID of the function that is being configured. If set to
	+ * 0xFF... (All Fs), then the configuration is for the requesting
	+ * function.
	+ */
	+ uint8_t unused_0;
	+ uint8_t unused_1;
	+ uint32_t enables;
	+ /* This bit must be '1' for the stag_vid field to be configured. */
	+ #define HWRM_FUNC_VLAN_CFG_INPUT_ENABLES_STAG_VID UINT32_C(0x1)
	+ /* This bit must be '1' for the ctag_vid field to be configured. */
	+ #define HWRM_FUNC_VLAN_CFG_INPUT_ENABLES_CTAG_VID UINT32_C(0x2)
	+ /* This bit must be '1' for the stag_pcp field to be configured. */
	+ #define HWRM_FUNC_VLAN_CFG_INPUT_ENABLES_STAG_PCP UINT32_C(0x4)
	+ /* This bit must be '1' for the ctag_pcp field to be configured. */
	+ #define HWRM_FUNC_VLAN_CFG_INPUT_ENABLES_CTAG_PCP UINT32_C(0x8)
	+ /* This bit must be '1' for the stag_tpid field to be configured. */
	+ #define HWRM_FUNC_VLAN_CFG_INPUT_ENABLES_STAG_TPID UINT32_C(0x10)
	+ /* This bit must be '1' for the ctag_tpid field to be configured. */
	+ #define HWRM_FUNC_VLAN_CFG_INPUT_ENABLES_CTAG_TPID UINT32_C(0x20)
	+ uint16_t stag_vid;
	+ /* S-TAG VLAN identifier configured for the function. */
	+ uint8_t stag_pcp;
	+ /* S-TAG PCP value configured for the function. */
	+ uint8_t unused_2;
	+ uint16_t stag_tpid; /* big endian */
	+ /*
	+ * S-TAG TPID value configured for the function. This field is specified
	+ * in network byte order.
	+ */
	+ uint16_t ctag_vid;
	+ /* C-TAG VLAN identifier configured for the function. */
	+ uint8_t ctag_pcp;
	+ /* C-TAG PCP value configured for the function. */
	+ uint8_t unused_3;
	+ uint16_t ctag_tpid; /* big endian */
	+ /*
	+ * C-TAG TPID value configured for the function. This field is specified
	+ * in network byte order.
	+ */
	+ uint32_t rsvd1;
	+ /* Future use. */
	+ uint32_t rsvd2;
	+ /* Future use. */
	+ uint32_t unused_4;
	+} __attribute__((packed));
	+
	+/* Output (16 bytes) */
	+
	+struct hwrm_func_vlan_cfg_output {
	+ uint16_t error_code;
	+ /*
	+ * Pass/Fail or error type Note: receiver to verify the in parameters,
	+ * and fail the call with an error when appropriate
	+ */
	+ uint16_t req_type;
	+ /* This field returns the type of original request. */
	+ uint16_t seq_id;
	+ /* This field provides original sequence number of the command. */
	+ uint16_t resp_len;
	+ /*
	+ * This field is the length of the response in bytes. The last byte of
	+ * the response is a valid flag that will read as '1' when the command
	+ * has been completely written to memory.
	+ */
	+ uint32_t unused_0;
	+ uint8_t unused_1;
	+ uint8_t unused_2;
	+ uint8_t unused_3;
	uint8_t valid;
	/*
	* This field is used in Output records to indicate that the output is
	@@ -5154,7 +7036,20 @@
	* set MTU/MRU for a VF using this command. This is to allow MTU/MRU setting by
	* the VF driver. If the MTU or MRU for a VF is set by the PF driver, then the
	* HWRM should ignore it. A function's MTU/MRU should be set prior to allocating
	- * RX VNICs or TX rings.
	+ * RX VNICs or TX rings. A PF driver calls hwrm_func_cfg to allocate resources
	+ * for itself or its children VFs. All function drivers shall call hwrm_func_cfg
	+ * to reserve resources. A request to hwrm_func_cfg may not be fully granted;
	+ * that is, a request for resources may be larger than what can be supported by
	+ * the device and the HWRM will allocate the best set of resources available,
	+ * but that may be less than requested. If all the amounts requested could not
	+ * be fulfilled, the HWRM shall allocate what it could and return a status code
	+ * of success. A function driver should call hwrm_func_qcfg immediately after
	+ * hwrm_func_cfg to determine what resources were assigned to the configured
	+ * function. A call by a PF driver to hwrm_func_cfg to allocate resources for
	+ * itself shall only allocate resources for the PF driver to use, not for its
	+ * children VFs. Likewise, a call to hwrm_func_qcfg shall return the resources
	+ * available for the PF driver to use, not what is available to its children
	+ * VFs.
	*/
	/* Input (88 bytes) */

	@@ -5194,66 +7089,61 @@
	uint8_t unused_1;
	uint32_t flags;
	/*
	- * When this bit is '1', the function is requested to be put in the
	- * promiscuous mode.
	+ * When this bit is '1', the function is disabled with source MAC
	+ * address check. This is an anti-spoofing check. If this flag is set,
	+ * then the function shall be configured to disallow transmission of
	+ * frames with the source MAC address that is configured for this
	+ * function.
	*/
	- #define HWRM_FUNC_CFG_INPUT_FLAGS_PROM_MODE UINT32_C(0x1)
	+ #define HWRM_FUNC_CFG_INPUT_FLAGS_SRC_MAC_ADDR_CHECK_DISABLE UINT32_C(0x1)
	/*
	* When this bit is '1', the function is enabled with source MAC address
	* check. This is an anti-spoofing check. If this flag is set, then the
	* function shall be configured to allow transmission of frames with the
	* source MAC address that is configured for this function.
	*/
	- #define HWRM_FUNC_CFG_INPUT_FLAGS_SRC_MAC_ADDR_CHECK UINT32_C(0x2)
	+ #define HWRM_FUNC_CFG_INPUT_FLAGS_SRC_MAC_ADDR_CHECK_ENABLE UINT32_C(0x2)
	+ /* reserved */
	+ #define HWRM_FUNC_CFG_INPUT_FLAGS_RSVD_MASK UINT32_C(0x1fc)
	+ #define HWRM_FUNC_CFG_INPUT_FLAGS_RSVD_SFT 2
	/*
	- * When this bit is '1', the function is enabled with source IP address
	- * check. This is an anti-spoofing check. If this flag is set, then the
	- * function shall be configured to allow transmission of frames with the
	- * source IP address that is configured for this function.
	+ * Standard TX Ring mode is used for the allocation of TX ring and
	+ * underlying scheduling resources that allow bandwidth reservation and
	+ * limit settings on the queried function. If set to 1, then standard TX
	+ * ring mode is requested to be enabled on the function being
	+ * configured.
	*/
	- #define HWRM_FUNC_CFG_INPUT_FLAGS_SRC_IP_ADDR_CHECK UINT32_C(0x4)
	+ #define HWRM_FUNC_CFG_INPUT_FLAGS_STD_TX_RING_MODE_ENABLE UINT32_C(0x200)
	/*
	- * When this bit is set to '1', the function shall be configured with
	- * VLAN priority match. If the VLAN PRI of a packet originated from this
	- * function does not match, then the packet shall be discarded.
	+ * Standard TX Ring mode is used for the allocation of TX ring and
	+ * underlying scheduling resources that allow bandwidth reservation and
	+ * limit settings on the queried function. If set to 1, then the
	+ * standard TX ring mode is requested to be disabled on the function
	+ * being configured. In this extended TX ring resource mode, the minimum
	+ * and maximum bandwidth settings are not supported to allow the
	+ * allocation of TX rings to span multiple scheduler nodes.
	*/
	- #define HWRM_FUNC_CFG_INPUT_FLAGS_VLAN_PRI_MATCH UINT32_C(0x8)
	+ #define HWRM_FUNC_CFG_INPUT_FLAGS_STD_TX_RING_MODE_DISABLE UINT32_C(0x400)
	/*
	- * When this bit is set to '1', the function shall be configured to
	- * check for VLAN priority match. If the VLAN PRI of a packet originated
	- * from this function does not match, then the default VLAN PRI shall be
	- * used.
	+ * If this bit is set, virtual mac address configured in this command
	+ * will be persistent over warm boot.
	*/
	- #define HWRM_FUNC_CFG_INPUT_FLAGS_DFLT_PRI_NOMATCH UINT32_C(0x10)
	+ #define HWRM_FUNC_CFG_INPUT_FLAGS_VIRT_MAC_PERSIST UINT32_C(0x800)
	/*
	- * When this bit is set to '1', the function shall be configured to not
	- * allow the transmission of pause frames. PAUSE frames use 48-bit
	- * destination multicast MAC address 01-80-C2-00-00-01.
	+ * This bit only applies to the VF. If this bit is set, the statistic
	+ * context counters will not be cleared when the statistic context is
	+ * freed or a function reset is called on VF. This bit will be cleared
	+ * when the PF is unloaded or a function reset is called on the PF.
	*/
	- #define HWRM_FUNC_CFG_INPUT_FLAGS_DISABLE_PAUSE UINT32_C(0x20)
	+ #define HWRM_FUNC_CFG_INPUT_FLAGS_NO_AUTOCLEAR_STATISTIC UINT32_C(0x1000)
	/*
	- * When this bit is set to '1', the function shall be configured to not
	- * allow the transmission of Spanning Tree Protocol (STP) frames. STP
	- * frames use Ethertype 0x0802 and 48-bit destination multicast MAC
	- * address 01-80-C2-00-00-00 and 01-80-C2-00-00-08 for 802.1D and
	- * 802.1ad respectively.
	+ * This bit requests that the firmware test to see if all the assets
	+ * requested in this command (i.e. number of TX rings) are available.
	+ * The firmware will return an error if the requested assets are not
	+ * available. The firwmare will NOT reserve the assets if they are
	+ * available.
	*/
	- #define HWRM_FUNC_CFG_INPUT_FLAGS_DISABLE_STP UINT32_C(0x40)
	- /*
	- * When this bit is set to '1', the function shall be configured to not
	- * allow the transmission of Link Layer Discovery Protocol (LLDP)
	- * frames. LLDP frames use Ethertype 0x88CC and 48-bit destination
	- * multicast MAC address 01-80-C2-00-00-00 or 01-80-C2-00-00-03 or
	- * 01-80-C2-00-00-0E.
	- */
	- #define HWRM_FUNC_CFG_INPUT_FLAGS_DISABLE_LLDP UINT32_C(0x80)
	- /*
	- * When this bit is set to '1', the function shall be configured to not
	- * allow the transmission of Precision Time Protocol (PTP) v2 frames.
	- * PTP frames use Ethertype 0x88F7 and 48-bit destination multicast MAC
	- * address 01-80-C2-00-00-0E or 01-1B-19-00-00-00.
	- */
	- #define HWRM_FUNC_CFG_INPUT_FLAGS_DISABLE_PTPV2 UINT32_C(0x100)
	+ #define HWRM_FUNC_CFG_INPUT_FLAGS_TX_ASSETS_TEST UINT32_C(0x2000)
	uint32_t enables;
	/* This bit must be '1' for the mtu field to be configured. */
	#define HWRM_FUNC_CFG_INPUT_ENABLES_MTU UINT32_C(0x1)
	@@ -5369,16 +7259,68 @@
	*/
	uint32_t min_bw;
	/*
	- * Minimum BW allocated for this function in Mbps. The HWRM will
	- * translate this value into byte counter and time interval used for the
	- * scheduler inside the device.
	+ * Minimum BW allocated for this function. The HWRM will translate this
	+ * value into byte counter and time interval used for the scheduler
	+ * inside the device.
	*/
	+ /* The bandwidth value. */
	+ #define HWRM_FUNC_CFG_INPUT_MIN_BW_BW_VALUE_MASK UINT32_C(0xfffffff)
	+ #define HWRM_FUNC_CFG_INPUT_MIN_BW_BW_VALUE_SFT 0
	+ /* The granularity of the value (bits or bytes). */
	+ #define HWRM_FUNC_CFG_INPUT_MIN_BW_SCALE UINT32_C(0x10000000)
	+ /* Value is in bits. */
	+ #define HWRM_FUNC_CFG_INPUT_MIN_BW_SCALE_BITS (UINT32_C(0x0) << 28)
	+ /* Value is in bytes. */
	+ #define HWRM_FUNC_CFG_INPUT_MIN_BW_SCALE_BYTES (UINT32_C(0x1) << 28)
	+ #define HWRM_FUNC_CFG_INPUT_MIN_BW_SCALE_LAST HWRM_FUNC_CFG_INPUT_MIN_BW_SCALE_BYTES
	+ /* bw_value_unit is 3 b */
	+ #define HWRM_FUNC_CFG_INPUT_MIN_BW_BW_VALUE_UNIT_MASK UINT32_C(0xe0000000)
	+ #define HWRM_FUNC_CFG_INPUT_MIN_BW_BW_VALUE_UNIT_SFT 29
	+ /* Value is in Mb or MB (base 10). */
	+ #define HWRM_FUNC_CFG_INPUT_MIN_BW_BW_VALUE_UNIT_MEGA (UINT32_C(0x0) << 29)
	+ /* Value is in Kb or KB (base 10). */
	+ #define HWRM_FUNC_CFG_INPUT_MIN_BW_BW_VALUE_UNIT_KILO (UINT32_C(0x2) << 29)
	+ /* Value is in bits or bytes. */
	+ #define HWRM_FUNC_CFG_INPUT_MIN_BW_BW_VALUE_UNIT_BASE (UINT32_C(0x4) << 29)
	+ /* Value is in Gb or GB (base 10). */
	+ #define HWRM_FUNC_CFG_INPUT_MIN_BW_BW_VALUE_UNIT_GIGA (UINT32_C(0x6) << 29)
	+ /* Value is in 1/100th of a percentage of total bandwidth. */
	+ #define HWRM_FUNC_CFG_INPUT_MIN_BW_BW_VALUE_UNIT_PERCENT1_100 (UINT32_C(0x1) << 29)
	+ /* Invalid unit */
	+ #define HWRM_FUNC_CFG_INPUT_MIN_BW_BW_VALUE_UNIT_INVALID (UINT32_C(0x7) << 29)
	+ #define HWRM_FUNC_CFG_INPUT_MIN_BW_BW_VALUE_UNIT_LAST HWRM_FUNC_CFG_INPUT_MIN_BW_BW_VALUE_UNIT_INVALID
	uint32_t max_bw;
	/*
	- * Maximum BW allocated for this function in Mbps. The HWRM will
	- * translate this value into byte counter and time interval used for the
	- * scheduler inside the device.
	+ * Maximum BW allocated for this function. The HWRM will translate this
	+ * value into byte counter and time interval used for the scheduler
	+ * inside the device.
	*/
	+ /* The bandwidth value. */
	+ #define HWRM_FUNC_CFG_INPUT_MAX_BW_BW_VALUE_MASK UINT32_C(0xfffffff)
	+ #define HWRM_FUNC_CFG_INPUT_MAX_BW_BW_VALUE_SFT 0
	+ /* The granularity of the value (bits or bytes). */
	+ #define HWRM_FUNC_CFG_INPUT_MAX_BW_SCALE UINT32_C(0x10000000)
	+ /* Value is in bits. */
	+ #define HWRM_FUNC_CFG_INPUT_MAX_BW_SCALE_BITS (UINT32_C(0x0) << 28)
	+ /* Value is in bytes. */
	+ #define HWRM_FUNC_CFG_INPUT_MAX_BW_SCALE_BYTES (UINT32_C(0x1) << 28)
	+ #define HWRM_FUNC_CFG_INPUT_MAX_BW_SCALE_LAST HWRM_FUNC_CFG_INPUT_MAX_BW_SCALE_BYTES
	+ /* bw_value_unit is 3 b */
	+ #define HWRM_FUNC_CFG_INPUT_MAX_BW_BW_VALUE_UNIT_MASK UINT32_C(0xe0000000)
	+ #define HWRM_FUNC_CFG_INPUT_MAX_BW_BW_VALUE_UNIT_SFT 29
	+ /* Value is in Mb or MB (base 10). */
	+ #define HWRM_FUNC_CFG_INPUT_MAX_BW_BW_VALUE_UNIT_MEGA (UINT32_C(0x0) << 29)
	+ /* Value is in Kb or KB (base 10). */
	+ #define HWRM_FUNC_CFG_INPUT_MAX_BW_BW_VALUE_UNIT_KILO (UINT32_C(0x2) << 29)
	+ /* Value is in bits or bytes. */
	+ #define HWRM_FUNC_CFG_INPUT_MAX_BW_BW_VALUE_UNIT_BASE (UINT32_C(0x4) << 29)
	+ /* Value is in Gb or GB (base 10). */
	+ #define HWRM_FUNC_CFG_INPUT_MAX_BW_BW_VALUE_UNIT_GIGA (UINT32_C(0x6) << 29)
	+ /* Value is in 1/100th of a percentage of total bandwidth. */
	+ #define HWRM_FUNC_CFG_INPUT_MAX_BW_BW_VALUE_UNIT_PERCENT1_100 (UINT32_C(0x1) << 29)
	+ /* Invalid unit */
	+ #define HWRM_FUNC_CFG_INPUT_MAX_BW_BW_VALUE_UNIT_INVALID (UINT32_C(0x7) << 29)
	+ #define HWRM_FUNC_CFG_INPUT_MAX_BW_BW_VALUE_UNIT_LAST HWRM_FUNC_CFG_INPUT_MAX_BW_BW_VALUE_UNIT_INVALID
	uint16_t async_event_cr;
	/*
	* ID of the target completion ring for receiving asynchronous event
	@@ -5529,9 +7471,9 @@
	/* Number of transmitted multicast packets on the function. */
	uint64_t tx_bcast_pkts;
	/* Number of transmitted broadcast packets on the function. */
	- uint64_t tx_err_pkts;
	+ uint64_t tx_discard_pkts;
	/*
	- * Number of transmitted packets that were dropped due to internal NIC
	+ * Number of transmitted packets that were discarded due to internal NIC
	* resource problems. For transmit, this can only happen if TMP is
	* configured to allow dropping in HOL blocking conditions, which is not
	* a normal configuration.
	@@ -5554,9 +7496,9 @@
	/* Number of received multicast packets on the function. */
	uint64_t rx_bcast_pkts;
	/* Number of received broadcast packets on the function. */
	- uint64_t rx_err_pkts;
	+ uint64_t rx_discard_pkts;
	/*
	- * Number of received packets that were dropped on the function due to
	+ * Number of received packets that were discarded on the function due to
	* resource limitations. This can happen for 3 reasons. # The BD used
	* for the packet has a bad format. # There were no BDs available in the
	* ring for the packet. # There were no BDs available on-chip for the
	@@ -5878,7 +7820,10 @@
	/* This bit must be '1' for the async_event_fwd field to be configured. */
	#define HWRM_FUNC_DRV_RGTR_INPUT_ENABLES_ASYNC_EVENT_FWD UINT32_C(0x10)
	uint16_t os_type;
	- /* This value indicates the type of OS. */
	+ /*
	+ * This value indicates the type of OS. The values are based on
	+ * CIM_OperatingSystem.mof file as published by the DMTF.
	+ */
	/* Unknown */
	#define HWRM_FUNC_DRV_RGTR_INPUT_OS_TYPE_UNKNOWN UINT32_C(0x0)
	/* Other OS not listed below. */
	@@ -6315,7 +8260,10 @@
	* has been completely written to memory.
	*/
	uint16_t os_type;
	- /* This value indicates the type of OS. */
	+ /*
	+ * This value indicates the type of OS. The values are based on
	+ * CIM_OperatingSystem.mof file as published by the DMTF.
	+ */
	/* Unknown */
	#define HWRM_FUNC_DRV_QVER_OUTPUT_OS_TYPE_UNKNOWN UINT32_C(0x0)
	/* Other OS not listed below. */
	@@ -6403,19 +8351,9 @@
	* command.
	*/
	#define HWRM_PORT_PHY_CFG_INPUT_FLAGS_RESET_PHY UINT32_C(0x1)
	+ /* deprecated bit. Do not use!!! */
	+ #define HWRM_PORT_PHY_CFG_INPUT_FLAGS_DEPRECATED UINT32_C(0x2)
	/*
	- * When this bit is set to '1', the link shall be forced to be taken
	- * down. # When this bit is set to '1", all other command input settings
	- * related to the link speed shall be ignored. Once the link state is
	- * forced down, it can be explicitly cleared from that state by setting
	- * this flag to '0'. # If this flag is set to '0', then the link shall
	- * be cleared from forced down state if the link is in forced down
	- * state. There may be conditions (e.g. out-of-band or sideband
	- * configuration changes for the link) outside the scope of the HWRM
	- * implementation that may clear forced down link state.
	- */
	- #define HWRM_PORT_PHY_CFG_INPUT_FLAGS_FORCE_LINK_DOWN UINT32_C(0x2)
	- /*
	* When this bit is set to '1', the link shall be forced to the
	* force_link_speed value. When this bit is set to '1', the HWRM client
	* should not enable any of the auto negotiation related fields
	@@ -6498,6 +8436,18 @@
	* shall ignore this flag.
	*/
	#define HWRM_PORT_PHY_CFG_INPUT_FLAGS_FEC_CLAUSE91_DISABLE UINT32_C(0x2000)
	+ /*
	+ * When this bit is set to '1', the link shall be forced to be taken
	+ * down. # When this bit is set to '1", all other command input settings
	+ * related to the link speed shall be ignored. Once the link state is
	+ * forced down, it can be explicitly cleared from that state by setting
	+ * this flag to '0'. # If this flag is set to '0', then the link shall
	+ * be cleared from forced down state if the link is in forced down
	+ * state. There may be conditions (e.g. out-of-band or sideband
	+ * configuration changes for the link) outside the scope of the HWRM
	+ * implementation that may clear forced down link state.
	+ */
	+ #define HWRM_PORT_PHY_CFG_INPUT_FLAGS_FORCE_LINK_DWN UINT32_C(0x4000)
	uint32_t enables;
	/* This bit must be '1' for the auto_mode field to be configured. */
	#define HWRM_PORT_PHY_CFG_INPUT_ENABLES_AUTO_MODE UINT32_C(0x1)
	@@ -6878,12 +8828,12 @@
	#define HWRM_PORT_PHY_QCFG_OUTPUT_LINK_SPEED_100GB UINT32_C(0x3e8)
	/* 10Mb link speed */
	#define HWRM_PORT_PHY_QCFG_OUTPUT_LINK_SPEED_10MB UINT32_C(0xffff)
	- uint8_t duplex;
	- /* This value is indicates the duplex of the current connection. */
	+ uint8_t duplex_cfg;
	+ /* This value is indicates the duplex of the current configuration. */
	/* Half Duplex connection. */
	- #define HWRM_PORT_PHY_QCFG_OUTPUT_DUPLEX_HALF UINT32_C(0x0)
	+ #define HWRM_PORT_PHY_QCFG_OUTPUT_DUPLEX_CFG_HALF UINT32_C(0x0)
	/* Full duplex connection. */
	- #define HWRM_PORT_PHY_QCFG_OUTPUT_DUPLEX_FULL UINT32_C(0x1)
	+ #define HWRM_PORT_PHY_QCFG_OUTPUT_DUPLEX_CFG_FULL UINT32_C(0x1)
	uint8_t pause;
	/*
	* This value is used to indicate the current pause configuration. When
	@@ -7157,6 +9107,40 @@
	#define HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASETE UINT32_C(0x9)
	/* SGMII connected external PHY */
	#define HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_SGMIIEXTPHY UINT32_C(0xa)
	+ /* 25G_BASECR_CA_L */
	+ #define HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_25G_BASECR_CA_L UINT32_C(0xb)
	+ /* 25G_BASECR_CA_S */
	+ #define HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_25G_BASECR_CA_S UINT32_C(0xc)
	+ /* 25G_BASECR_CA_N */
	+ #define HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_25G_BASECR_CA_N UINT32_C(0xd)
	+ /* 25G_BASESR */
	+ #define HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_25G_BASESR UINT32_C(0xe)
	+ /* 100G_BASECR4 */
	+ #define HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_100G_BASECR4 UINT32_C(0xf)
	+ /* 100G_BASESR4 */
	+ #define HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_100G_BASESR4 UINT32_C(0x10)
	+ /* 100G_BASELR4 */
	+ #define HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_100G_BASELR4 UINT32_C(0x11)
	+ /* 100G_BASEER4 */
	+ #define HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_100G_BASEER4 UINT32_C(0x12)
	+ /* 100G_BASESR10 */
	+ #define HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_100G_BASESR10 UINT32_C(0x13)
	+ /* 40G_BASECR4 */
	+ #define HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_40G_BASECR4 UINT32_C(0x14)
	+ /* 40G_BASESR4 */
	+ #define HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_40G_BASESR4 UINT32_C(0x15)
	+ /* 40G_BASELR4 */
	+ #define HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_40G_BASELR4 UINT32_C(0x16)
	+ /* 40G_BASEER4 */
	+ #define HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_40G_BASEER4 UINT32_C(0x17)
	+ /* 40G_ACTIVE_CABLE */
	+ #define HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_40G_ACTIVE_CABLE UINT32_C(0x18)
	+ /* 1G_baseT */
	+ #define HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_1G_BASET UINT32_C(0x19)
	+ /* 1G_baseSX */
	+ #define HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_1G_BASESX UINT32_C(0x1a)
	+ /* 1G_baseCX */
	+ #define HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_1G_BASECX UINT32_C(0x1b)
	uint8_t media_type;
	/* This value represents a media type. */
	/* Unknown */
	@@ -7414,8 +9398,13 @@
	* supported on this port.
	*/
	#define HWRM_PORT_PHY_QCFG_OUTPUT_FEC_CFG_FEC_CLAUSE91_ENABLED UINT32_C(0x40)
	+ uint8_t duplex_state;
	+ /* This value is indicates the duplex of the current connection state. */
	+ /* Half Duplex connection. */
	+ #define HWRM_PORT_PHY_QCFG_OUTPUT_DUPLEX_STATE_HALF UINT32_C(0x0)
	+ /* Full duplex connection. */
	+ #define HWRM_PORT_PHY_QCFG_OUTPUT_DUPLEX_STATE_FULL UINT32_C(0x1)
	uint8_t unused_1;
	- uint8_t unused_2;
	char phy_vendor_name[16];
	/*
	* Up to 16 bytes of null padded ASCII string representing PHY vendor.
	@@ -7427,10 +9416,10 @@
	* specific part number of the PHY. If the string is set to null, then
	* the vendor specific part number is not available.
	*/
	- uint32_t unused_3;
	+ uint32_t unused_2;
	+ uint8_t unused_3;
	uint8_t unused_4;
	uint8_t unused_5;
	- uint8_t unused_6;
	uint8_t valid;
	/*
	* This field is used in Output records to indicate that the output is
	@@ -7442,7 +9431,17 @@
	} __attribute__((packed));

	/* hwrm_port_mac_cfg */
	-/* Description: This command configures the MAC block for the port. */
	+/*
	+ * Description: This command configures the MAC block for the port. # Only PF
	+ * drivers shall be allowed to configure MAC. # A VF driver should not be
	+ * allowed to configure MAC using this command. # In a network partition mode, a
	+ * PF driver should not be allowed to configure MAC using this command. The QoS
	+ * settings in port_mac_cfg() are global for all ports/functions. If multiple PF
	+ * drivers on different ports are configuring QoS settings, then the HWRM is not
	+ * responsible for maintaining consistency between them. A PF driver changing
	+ * global QoS settings using this command may impact other PF drivers on
	+ * different ports.
	+ */
	/* Input (40 bytes) */

	struct hwrm_port_mac_cfg_input {
	@@ -7473,26 +9472,38 @@
	*/
	uint32_t flags;
	/*
	+ * In this field, there are a number of CoS mappings related flags that
	+ * are used to configure CoS mappings and their corresponding priorities
	+ * in the hardware. For the priorities of CoS mappings, the HWRM uses
	+ * the following priority order (high to low) by default: # vlan pri #
	+ * ip_dscp # tunnel_vlan_pri # default cos A subset of CoS mappings can
	+ * be enabled. If a priority is not specified for an enabled CoS
	+ * mapping, the priority will be assigned in the above order for the
	+ * enabled CoS mappings. For example, if vlan_pri and ip_dscp CoS
	+ * mappings are enabled and their priorities are not specified, the
	+ * following priority order (high to low) will be used by the HWRM: #
	+ * vlan_pri # ip_dscp # default cos vlan_pri CoS mapping together with
	+ * default CoS with lower priority are enabled by default by the HWRM.
	+ */
	+ /*
	* When this bit is '1', this command will configure the MAC to match
	* the current link state of the PHY. If the link is not established on
	* the PHY, then this bit has no effect.
	*/
	#define HWRM_PORT_MAC_CFG_INPUT_FLAGS_MATCH_LINK UINT32_C(0x1)
	/*
	- * When this bit is '1', the CoS assignment logic is enabled. When this
	- * logic is enabled, then inner VLAN PRI to CoS mapping is enabled. If
	- * this bit is '0', then the default CoS is used.
	+ * When this bit is set to '1', the inner VLAN PRI to CoS mapping is
	+ * requested to be enabled.
	*/
	- #define HWRM_PORT_MAC_CFG_INPUT_FLAGS_COS_ASSIGNMENT_ENABLE UINT32_C(0x2)
	+ #define HWRM_PORT_MAC_CFG_INPUT_FLAGS_VLAN_PRI2COS_ENABLE UINT32_C(0x2)
	/*
	- * When this bit is '1', tunnel or outer VLAN PRI field to CoS mapping
	- * is enabled. If this bit is '0', then outer VLAN PRI bits are not used
	- * in determining CoS.
	+ * When this bit is set to '1', tunnel VLAN PRI field to CoS mapping is
	+ * requested to be enabled.
	*/
	#define HWRM_PORT_MAC_CFG_INPUT_FLAGS_TUNNEL_PRI2COS_ENABLE UINT32_C(0x4)
	/*
	- * When this bit is '1', the IP DSCP to CoS mapping is enabled. If this
	- * bit is '0', then IP DSCP bits are not used in determining CoS.
	+ * When this bit is set to '1', the IP DSCP to CoS mapping is requested
	+ * to be enabled.
	*/
	#define HWRM_PORT_MAC_CFG_INPUT_FLAGS_IP_DSCP2COS_ENABLE UINT32_C(0x8)
	/*
	@@ -7525,18 +9536,33 @@
	* disabled on this port.
	*/
	#define HWRM_PORT_MAC_CFG_INPUT_FLAGS_OOB_WOL_DISABLE UINT32_C(0x200)
	+ /*
	+ * When this bit is set to '1', the inner VLAN PRI to CoS mapping is
	+ * requested to be disabled.
	+ */
	+ #define HWRM_PORT_MAC_CFG_INPUT_FLAGS_VLAN_PRI2COS_DISABLE UINT32_C(0x400)
	+ /*
	+ * When this bit is set to '1', tunnel VLAN PRI field to CoS mapping is
	+ * requested to be disabled.
	+ */
	+ #define HWRM_PORT_MAC_CFG_INPUT_FLAGS_TUNNEL_PRI2COS_DISABLE UINT32_C(0x800)
	+ /*
	+ * When this bit is set to '1', the IP DSCP to CoS mapping is requested
	+ * to be disabled.
	+ */
	+ #define HWRM_PORT_MAC_CFG_INPUT_FLAGS_IP_DSCP2COS_DISABLE UINT32_C(0x1000)
	uint32_t enables;
	/* This bit must be '1' for the ipg field to be configured. */
	#define HWRM_PORT_MAC_CFG_INPUT_ENABLES_IPG UINT32_C(0x1)
	/* This bit must be '1' for the lpbk field to be configured. */
	#define HWRM_PORT_MAC_CFG_INPUT_ENABLES_LPBK UINT32_C(0x2)
	/*
	- * This bit must be '1' for the ivlan_pri2cos_map_pri field to be
	+ * This bit must be '1' for the vlan_pri2cos_map_pri field to be
	* configured.
	*/
	- #define HWRM_PORT_MAC_CFG_INPUT_ENABLES_IVLAN_PRI2COS_MAP_PRI UINT32_C(0x4)
	- /* This bit must be '1' for the lcos_map_pri field to be configured. */
	- #define HWRM_PORT_MAC_CFG_INPUT_ENABLES_LCOS_MAP_PRI UINT32_C(0x8)
	+ #define HWRM_PORT_MAC_CFG_INPUT_ENABLES_VLAN_PRI2COS_MAP_PRI UINT32_C(0x4)
	+ /* This bit must be '1' for the Reserved1 field to be configured. */
	+ #define HWRM_PORT_MAC_CFG_INPUT_ENABLES_RESERVED1 UINT32_C(0x8)
	/*
	* This bit must be '1' for the tunnel_pri2cos_map_pri field to be
	* configured.
	@@ -7554,6 +9580,8 @@
	* configured.
	*/
	#define HWRM_PORT_MAC_CFG_INPUT_ENABLES_TX_TS_CAPTURE_PTP_MSG_TYPE UINT32_C(0x80)
	+ /* This bit must be '1' for the cos_field_cfg field to be configured. */
	+ #define HWRM_PORT_MAC_CFG_INPUT_ENABLES_COS_FIELD_CFG UINT32_C(0x100)
	uint16_t port_id;
	/* Port ID of port that is to be configured. */
	uint8_t ipg;
	@@ -7576,25 +9604,37 @@
	* received.
	*/
	#define HWRM_PORT_MAC_CFG_INPUT_LPBK_REMOTE UINT32_C(0x2)
	- uint8_t ivlan_pri2cos_map_pri;
	+ uint8_t vlan_pri2cos_map_pri;
	/*
	- * This value controls the priority of mapping. Valid values: 1-4 Higher
	- * the number, higher the priority
	+ * This value controls the priority setting of VLAN PRI to CoS mapping
	+ * based on VLAN Tags of inner packet headers of tunneled packets or
	+ * packet headers of non-tunneled packets. # Each XXX_pri variable shall
	+ * have a unique priority value when it is being specified. # When
	+ * comparing priorities of mappings, higher value indicates higher
	+ * priority. For example, a value of 0-3 is returned where 0 is being
	+ * the lowest priority and 3 is being the highest priority.
	*/
	- uint8_t lcos_map_pri;
	- /*
	- * This value controls the priority of mapping. Valid values: 1-4 Higher
	- * the number, higher the priority
	- */
	+ uint8_t reserved1;
	+ /* Reserved field */
	uint8_t tunnel_pri2cos_map_pri;
	/*
	- * This value controls the priority of mapping. Valid values: 1-4 Higher
	- * the number, higher the priority
	+ * This value controls the priority setting of VLAN PRI to CoS mapping
	+ * based on VLAN Tags of tunneled header. This mapping only applies when
	+ * tunneled headers are present. # Each XXX_pri variable shall have a
	+ * unique priority value when it is being specified. # When comparing
	+ * priorities of mappings, higher value indicates higher priority. For
	+ * example, a value of 0-3 is returned where 0 is being the lowest
	+ * priority and 3 is being the highest priority.
	*/
	uint8_t dscp2pri_map_pri;
	/*
	- * This value controls the priority of mapping. Valid values: 1-4 Higher
	- * the number, higher the priority
	+ * This value controls the priority setting of IP DSCP to CoS mapping
	+ * based on inner IP header of tunneled packets or IP header of non-
	+ * tunneled packets. # Each XXX_pri variable shall have a unique
	+ * priority value when it is being specified. # When comparing
	+ * priorities of mappings, higher value indicates higher priority. For
	+ * example, a value of 0-3 is returned where 0 is being the lowest
	+ * priority and 3 is being the highest priority.
	*/
	uint16_t rx_ts_capture_ptp_msg_type;
	/*
	@@ -7616,7 +9656,77 @@
	* transmit sied of the port to capture the time stamp of every
	* transmitted PTP message with messageType field value set to i.
	*/
	- uint32_t unused_0;
	+ uint8_t cos_field_cfg;
	+ /* Configuration of CoS fields. */
	+ /* Reserved. */
	+ #define HWRM_PORT_MAC_CFG_INPUT_COS_FIELD_CFG_RSVD1 UINT32_C(0x1)
	+ /*
	+ * This field is used to specify selection of VLAN PRI value based on
	+ * whether one or two VLAN Tags are present in the inner packet headers
	+ * of tunneled packets or non-tunneled packets. This field is valid only
	+ * if inner VLAN PRI to CoS mapping is enabled. If VLAN PRI to CoS
	+ * mapping is not enabled, then this field shall be ignored.
	+ */
	+ #define HWRM_PORT_MAC_CFG_INPUT_COS_FIELD_CFG_VLAN_PRI_SEL_MASK UINT32_C(0x6)
	+ #define HWRM_PORT_MAC_CFG_INPUT_COS_FIELD_CFG_VLAN_PRI_SEL_SFT 1
	+ /*
	+ * Select inner VLAN PRI when 1 or 2 VLAN Tags are present in
	+ * the inner packet headers
	+ */
	+ #define HWRM_PORT_MAC_CFG_INPUT_COS_FIELD_CFG_VLAN_PRI_SEL_INNERMOST (UINT32_C(0x0) << 1)
	+ /*
	+ * Select outer VLAN Tag PRI when 2 VLAN Tags are present in the
	+ * inner packet headers. No VLAN PRI shall be selected for this
	+ * configuration if only one VLAN Tag is present in the inner
	+ * packet headers.
	+ */
	+ #define HWRM_PORT_MAC_CFG_INPUT_COS_FIELD_CFG_VLAN_PRI_SEL_OUTER (UINT32_C(0x1) << 1)
	+ /*
	+ * Select outermost VLAN PRI when 1 or 2 VLAN Tags are present
	+ * in the inner packet headers
	+ */
	+ #define HWRM_PORT_MAC_CFG_INPUT_COS_FIELD_CFG_VLAN_PRI_SEL_OUTERMOST (UINT32_C(0x2) << 1)
	+ /* Unspecified */
	+ #define HWRM_PORT_MAC_CFG_INPUT_COS_FIELD_CFG_VLAN_PRI_SEL_UNSPECIFIED (UINT32_C(0x3) << 1)
	+ #define HWRM_PORT_MAC_CFG_INPUT_COS_FIELD_CFG_VLAN_PRI_SEL_LAST HWRM_PORT_MAC_CFG_INPUT_COS_FIELD_CFG_VLAN_PRI_SEL_UNSPECIFIED
	+ /*
	+ * This field is used to specify selection of tunnel VLAN PRI value
	+ * based on whether one or two VLAN Tags are present in tunnel headers.
	+ * This field is valid only if tunnel VLAN PRI to CoS mapping is
	+ * enabled. If tunnel VLAN PRI to CoS mapping is not enabled, then this
	+ * field shall be ignored.
	+ */
	+ #define HWRM_PORT_MAC_CFG_INPUT_COS_FIELD_CFG_T_VLAN_PRI_SEL_MASK UINT32_C(0x18)
	+ #define HWRM_PORT_MAC_CFG_INPUT_COS_FIELD_CFG_T_VLAN_PRI_SEL_SFT 3
	+ /*
	+ * Select inner VLAN PRI when 1 or 2 VLAN Tags are present in
	+ * the tunnel packet headers
	+ */
	+ #define HWRM_PORT_MAC_CFG_INPUT_COS_FIELD_CFG_T_VLAN_PRI_SEL_INNERMOST (UINT32_C(0x0) << 3)
	+ /*
	+ * Select outer VLAN Tag PRI when 2 VLAN Tags are present in the
	+ * tunnel packet headers. No tunnel VLAN PRI shall be selected
	+ * for this configuration if only one VLAN Tag is present in the
	+ * tunnel packet headers.
	+ */
	+ #define HWRM_PORT_MAC_CFG_INPUT_COS_FIELD_CFG_T_VLAN_PRI_SEL_OUTER (UINT32_C(0x1) << 3)
	+ /*
	+ * Select outermost VLAN PRI when 1 or 2 VLAN Tags are present
	+ * in the tunnel packet headers
	+ */
	+ #define HWRM_PORT_MAC_CFG_INPUT_COS_FIELD_CFG_T_VLAN_PRI_SEL_OUTERMOST (UINT32_C(0x2) << 3)
	+ /* Unspecified */
	+ #define HWRM_PORT_MAC_CFG_INPUT_COS_FIELD_CFG_T_VLAN_PRI_SEL_UNSPECIFIED (UINT32_C(0x3) << 3)
	+ #define HWRM_PORT_MAC_CFG_INPUT_COS_FIELD_CFG_T_VLAN_PRI_SEL_LAST HWRM_PORT_MAC_CFG_INPUT_COS_FIELD_CFG_T_VLAN_PRI_SEL_UNSPECIFIED
	+ /*
	+ * This field shall be used to provide default CoS value that has been
	+ * configured on this port. This field is valid only if default CoS
	+ * mapping is enabled. If default CoS mapping is not enabled, then this
	+ * field shall be ignored.
	+ */
	+ #define HWRM_PORT_MAC_CFG_INPUT_COS_FIELD_CFG_DEFAULT_COS_MASK UINT32_C(0xe0)
	+ #define HWRM_PORT_MAC_CFG_INPUT_COS_FIELD_CFG_DEFAULT_COS_SFT 5
	+ uint8_t unused_0[3];
	} __attribute__((packed));

	/* Output (16 bytes) */
	@@ -7759,27 +9869,61 @@
	* received.
	*/
	#define HWRM_PORT_MAC_QCFG_OUTPUT_LPBK_REMOTE UINT32_C(0x2)
	- uint8_t ivlan_pri2cos_map_pri;
	+ uint8_t vlan_pri2cos_map_pri;
	/*
	- * Priority of pri to CoS mapping. Valid values: 1-4 Higher the number,
	- * higher the priority Value 0 indicates that this mapping is not used.
	+ * Priority setting for VLAN PRI to CoS mapping. # Each XXX_pri variable
	+ * shall have a unique priority value when it is being used. # When
	+ * comparing priorities of mappings, higher value indicates higher
	+ * priority. For example, a value of 0-3 is returned where 0 is being
	+ * the lowest priority and 3 is being the highest priority. # If the
	+ * corresponding CoS mapping is not enabled, then this field should be
	+ * ignored. # This value indicates the normalized priority value
	+ * retained in the HWRM.
	*/
	- uint8_t lcos_map_pri;
	+ uint8_t flags;
	/*
	- * Priority of local CoS to PRI mapping. Valid values: 1-4 Higher the
	- * number, higher the priority Value 0 indicates that this mapping is
	- * not used.
	+ * In this field, a number of CoS mappings related flags are used to
	+ * indicate configured CoS mappings.
	*/
	+ /*
	+ * When this bit is set to '1', the inner VLAN PRI to CoS mapping is
	+ * enabled.
	+ */
	+ #define HWRM_PORT_MAC_QCFG_OUTPUT_FLAGS_VLAN_PRI2COS_ENABLE UINT32_C(0x1)
	+ /*
	+ * When this bit is set to '1', tunnel VLAN PRI field to CoS mapping is
	+ * enabled.
	+ */
	+ #define HWRM_PORT_MAC_QCFG_OUTPUT_FLAGS_TUNNEL_PRI2COS_ENABLE UINT32_C(0x2)
	+ /* When this bit is set to '1', the IP DSCP to CoS mapping is enabled. */
	+ #define HWRM_PORT_MAC_QCFG_OUTPUT_FLAGS_IP_DSCP2COS_ENABLE UINT32_C(0x4)
	+ /* When this bit is '1', the Out-Of-Box WoL is enabled on this port. */
	+ #define HWRM_PORT_MAC_QCFG_OUTPUT_FLAGS_OOB_WOL_ENABLE UINT32_C(0x8)
	+ /* When this bit is '1', PTP is enabled for RX on this port. */
	+ #define HWRM_PORT_MAC_QCFG_OUTPUT_FLAGS_PTP_RX_TS_CAPTURE_ENABLE UINT32_C(0x10)
	+ /* When this bit is '1', PTP is enabled for TX on this port. */
	+ #define HWRM_PORT_MAC_QCFG_OUTPUT_FLAGS_PTP_TX_TS_CAPTURE_ENABLE UINT32_C(0x20)
	uint8_t tunnel_pri2cos_map_pri;
	/*
	- * Priority of tunnel PRI to CoS mapping. Valid values: 1-4 Higher the
	- * number, higher the priority Value 0 indicates that this mapping is
	- * not used.
	+ * Priority setting for tunnel VLAN PRI to CoS mapping. # Each XXX_pri
	+ * variable shall have a unique priority value when it is being used. #
	+ * When comparing priorities of mappings, higher value indicates higher
	+ * priority. For example, a value of 0-3 is returned where 0 is being
	+ * the lowest priority and 3 is being the highest priority. # If the
	+ * corresponding CoS mapping is not enabled, then this field should be
	+ * ignored. # This value indicates the normalized priority value
	+ * retained in the HWRM.
	*/
	uint8_t dscp2pri_map_pri;
	/*
	- * Priority of DSCP to PRI mapping. Valid values: 1-4 Higher the number,
	- * higher the priority Value 0 indicates that this mapping is not used.
	+ * Priority setting for DSCP to PRI mapping. # Each XXX_pri variable
	+ * shall have a unique priority value when it is being used. # When
	+ * comparing priorities of mappings, higher value indicates higher
	+ * priority. For example, a value of 0-3 is returned where 0 is being
	+ * the lowest priority and 3 is being the highest priority. # If the
	+ * corresponding CoS mapping is not enabled, then this field should be
	+ * ignored. # This value indicates the normalized priority value
	+ * retained in the HWRM.
	*/
	uint16_t rx_ts_capture_ptp_msg_type;
	/*
	@@ -7805,7 +9949,185 @@
	* to 1, then the transmit side of the port is configured to capture
	* timestamp for all PTP messages.
	*/
	+ uint8_t cos_field_cfg;
	+ /* Configuration of CoS fields. */
	+ /* Reserved */
	+ #define HWRM_PORT_MAC_QCFG_OUTPUT_COS_FIELD_CFG_RSVD UINT32_C(0x1)
	+ /*
	+ * This field is used for selecting VLAN PRI value based on whether one
	+ * or two VLAN Tags are present in the inner packet headers of tunneled
	+ * packets or non-tunneled packets.
	+ */
	+ #define HWRM_PORT_MAC_QCFG_OUTPUT_COS_FIELD_CFG_VLAN_PRI_SEL_MASK UINT32_C(0x6)
	+ #define HWRM_PORT_MAC_QCFG_OUTPUT_COS_FIELD_CFG_VLAN_PRI_SEL_SFT 1
	+ /*
	+ * Select inner VLAN PRI when 1 or 2 VLAN Tags are present in
	+ * the inner packet headers
	+ */
	+ #define HWRM_PORT_MAC_QCFG_OUTPUT_COS_FIELD_CFG_VLAN_PRI_SEL_INNERMOST (UINT32_C(0x0) << 1)
	+ /*
	+ * Select outer VLAN Tag PRI when 2 VLAN Tags are present in the
	+ * inner packet headers. No VLAN PRI is selected for this
	+ * configuration if only one VLAN Tag is present in the inner
	+ * packet headers.
	+ */
	+ #define HWRM_PORT_MAC_QCFG_OUTPUT_COS_FIELD_CFG_VLAN_PRI_SEL_OUTER (UINT32_C(0x1) << 1)
	+ /*
	+ * Select outermost VLAN PRI when 1 or 2 VLAN Tags are present
	+ * in the inner packet headers
	+ */
	+ #define HWRM_PORT_MAC_QCFG_OUTPUT_COS_FIELD_CFG_VLAN_PRI_SEL_OUTERMOST (UINT32_C(0x2) << 1)
	+ /* Unspecified */
	+ #define HWRM_PORT_MAC_QCFG_OUTPUT_COS_FIELD_CFG_VLAN_PRI_SEL_UNSPECIFIED (UINT32_C(0x3) << 1)
	+ #define HWRM_PORT_MAC_QCFG_OUTPUT_COS_FIELD_CFG_VLAN_PRI_SEL_LAST HWRM_PORT_MAC_QCFG_OUTPUT_COS_FIELD_CFG_VLAN_PRI_SEL_UNSPECIFIED
	+ /*
	+ * This field is used for selecting tunnel VLAN PRI value based on
	+ * whether one or two VLAN Tags are present in the tunnel headers of
	+ * tunneled packets. This selection does not apply to non-tunneled
	+ * packets.
	+ */
	+ #define HWRM_PORT_MAC_QCFG_OUTPUT_COS_FIELD_CFG_T_VLAN_PRI_SEL_MASK UINT32_C(0x18)
	+ #define HWRM_PORT_MAC_QCFG_OUTPUT_COS_FIELD_CFG_T_VLAN_PRI_SEL_SFT 3
	+ /*
	+ * Select inner VLAN PRI when 1 or 2 VLAN Tags are present in
	+ * the tunnel packet headers
	+ */
	+ #define HWRM_PORT_MAC_QCFG_OUTPUT_COS_FIELD_CFG_T_VLAN_PRI_SEL_INNERMOST (UINT32_C(0x0) << 3)
	+ /*
	+ * Select outer VLAN Tag PRI when 2 VLAN Tags are present in the
	+ * tunnel packet headers. No VLAN PRI is selected for this
	+ * configuration if only one VLAN Tag is present in the tunnel
	+ * packet headers.
	+ */
	+ #define HWRM_PORT_MAC_QCFG_OUTPUT_COS_FIELD_CFG_T_VLAN_PRI_SEL_OUTER (UINT32_C(0x1) << 3)
	+ /*
	+ * Select outermost VLAN PRI when 1 or 2 VLAN Tags are present
	+ * in the tunnel packet headers
	+ */
	+ #define HWRM_PORT_MAC_QCFG_OUTPUT_COS_FIELD_CFG_T_VLAN_PRI_SEL_OUTERMOST (UINT32_C(0x2) << 3)
	+ /* Unspecified */
	+ #define HWRM_PORT_MAC_QCFG_OUTPUT_COS_FIELD_CFG_T_VLAN_PRI_SEL_UNSPECIFIED (UINT32_C(0x3) << 3)
	+ #define HWRM_PORT_MAC_QCFG_OUTPUT_COS_FIELD_CFG_T_VLAN_PRI_SEL_LAST HWRM_PORT_MAC_QCFG_OUTPUT_COS_FIELD_CFG_T_VLAN_PRI_SEL_UNSPECIFIED
	+ /*
	+ * This field is used to provide default CoS value that has been
	+ * configured on this port.
	+ */
	+ #define HWRM_PORT_MAC_QCFG_OUTPUT_COS_FIELD_CFG_DEFAULT_COS_MASK UINT32_C(0xe0)
	+ #define HWRM_PORT_MAC_QCFG_OUTPUT_COS_FIELD_CFG_DEFAULT_COS_SFT 5
	+ uint8_t valid;
	+ /*
	+ * This field is used in Output records to indicate that the output is
	+ * completely written to RAM. This field should be read as '1' to
	+ * indicate that the output has been completely written. When writing a
	+ * command completion or response to an internal processor, the order of
	+ * writes has to be such that this field is written last.
	+ */
	+} __attribute__((packed));
	+
	+/* hwrm_port_mac_ptp_qcfg */
	+/* Description: This command queries the PTP information for the port. */
	+/* Input (24 bytes) */
	+
	+struct hwrm_port_mac_ptp_qcfg_input {
	+ uint16_t req_type;
	+ /*
	+ * This value indicates what type of request this is. The format for the
	+ * rest of the command is determined by this field.
	+ */
	+ uint16_t cmpl_ring;
	+ /*
	+ * This value indicates the what completion ring the request will be
	+ * optionally completed on. If the value is -1, then no CR completion
	+ * will be generated. Any other value must be a valid CR ring_id value
	+ * for this function.
	+ */
	+ uint16_t seq_id;
	+ /* This value indicates the command sequence number. */
	+ uint16_t target_id;
	+ /*
	+ * Target ID of this command. 0x0 - 0xFFF8 - Used for function ids
	+ * 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF - HWRM
	+ */
	+ uint64_t resp_addr;
	+ /*
	+ * This is the host address where the response will be written when the
	+ * request is complete. This area must be 16B aligned and must be
	+ * cleared to zero before the request is made.
	+ */
	+ uint16_t port_id;
	+ /* Port ID of port that is being queried. */
	+ uint16_t unused_0[3];
	+} __attribute__((packed));
	+
	+/* Output (80 bytes) */
	+
	+struct hwrm_port_mac_ptp_qcfg_output {
	+ uint16_t error_code;
	+ /*
	+ * Pass/Fail or error type Note: receiver to verify the in parameters,
	+ * and fail the call with an error when appropriate
	+ */
	+ uint16_t req_type;
	+ /* This field returns the type of original request. */
	+ uint16_t seq_id;
	+ /* This field provides original sequence number of the command. */
	+ uint16_t resp_len;
	+ /*
	+ * This field is the length of the response in bytes. The last byte of
	+ * the response is a valid flag that will read as '1' when the command
	+ * has been completely written to memory.
	+ */
	+ uint8_t flags;
	+ /*
	+ * In this field, a number of PTP related flags are used to indicate
	+ * configured PTP capabilities.
	+ */
	+ /*
	+ * When this bit is set to '1', the PTP related registers are directly
	+ * accessible by the host.
	+ */
	+ #define HWRM_PORT_MAC_PTP_QCFG_OUTPUT_FLAGS_DIRECT_ACCESS UINT32_C(0x1)
	+ /*
	+ * When this bit is set to '1', the PTP information is accessible via
	+ * HWRM commands.
	+ */
	+ #define HWRM_PORT_MAC_PTP_QCFG_OUTPUT_FLAGS_HWRM_ACCESS UINT32_C(0x2)
	uint8_t unused_0;
	+ uint16_t unused_1;
	+ uint32_t rx_ts_reg_off_lower;
	+ /* Offset of the PTP register for the lower 32 bits of timestamp for RX. */
	+ uint32_t rx_ts_reg_off_upper;
	+ /* Offset of the PTP register for the upper 32 bits of timestamp for RX. */
	+ uint32_t rx_ts_reg_off_seq_id;
	+ /* Offset of the PTP register for the sequence ID for RX. */
	+ uint32_t rx_ts_reg_off_src_id_0;
	+ /* Offset of the first PTP source ID for RX. */
	+ uint32_t rx_ts_reg_off_src_id_1;
	+ /* Offset of the second PTP source ID for RX. */
	+ uint32_t rx_ts_reg_off_src_id_2;
	+ /* Offset of the third PTP source ID for RX. */
	+ uint32_t rx_ts_reg_off_domain_id;
	+ /* Offset of the domain ID for RX. */
	+ uint32_t rx_ts_reg_off_fifo;
	+ /* Offset of the PTP FIFO register for RX. */
	+ uint32_t rx_ts_reg_off_fifo_adv;
	+ /* Offset of the PTP advance FIFO register for RX. */
	+ uint32_t rx_ts_reg_off_granularity;
	+ /* PTP timestamp granularity for RX. */
	+ uint32_t tx_ts_reg_off_lower;
	+ /* Offset of the PTP register for the lower 32 bits of timestamp for TX. */
	+ uint32_t tx_ts_reg_off_upper;
	+ /* Offset of the PTP register for the upper 32 bits of timestamp for TX. */
	+ uint32_t tx_ts_reg_off_seq_id;
	+ /* Offset of the PTP register for the sequence ID for TX. */
	+ uint32_t tx_ts_reg_off_fifo;
	+ /* Offset of the PTP FIFO register for TX. */
	+ uint32_t tx_ts_reg_off_granularity;
	+ /* PTP timestamp granularity for TX. */
	+ uint32_t unused_2;
	+ uint8_t unused_3;
	+ uint8_t unused_4;
	+ uint8_t unused_5;
	uint8_t valid;
	/*
	* This field is used in Output records to indicate that the output is
	@@ -8118,76 +10440,6 @@
	*/
	} __attribute__((packed));

	-/* hwrm_port_blink_led */
	-/*
	- * Description: This function blinks the port LED for the specified number of
	- * times.
	- */
	-/* Input (24 bytes) */
	-
	-struct hwrm_port_blink_led_input {
	- uint16_t req_type;
	- /*
	- * This value indicates what type of request this is. The format for the
	- * rest of the command is determined by this field.
	- */
	- uint16_t cmpl_ring;
	- /*
	- * This value indicates the what completion ring the request will be
	- * optionally completed on. If the value is -1, then no CR completion
	- * will be generated. Any other value must be a valid CR ring_id value
	- * for this function.
	- */
	- uint16_t seq_id;
	- /* This value indicates the command sequence number. */
	- uint16_t target_id;
	- /*
	- * Target ID of this command. 0x0 - 0xFFF8 - Used for function ids
	- * 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF - HWRM
	- */
	- uint64_t resp_addr;
	- /*
	- * This is the host address where the response will be written when the
	- * request is complete. This area must be 16B aligned and must be
	- * cleared to zero before the request is made.
	- */
	- uint32_t num_blinks;
	- /* Number of blinks. */
	- uint32_t unused_0;
	-} __attribute__((packed));
	-
	-/* Output (16 bytes) */
	-
	-struct hwrm_port_blink_led_output {
	- uint16_t error_code;
	- /*
	- * Pass/Fail or error type Note: receiver to verify the in parameters,
	- * and fail the call with an error when appropriate
	- */
	- uint16_t req_type;
	- /* This field returns the type of original request. */
	- uint16_t seq_id;
	- /* This field provides original sequence number of the command. */
	- uint16_t resp_len;
	- /*
	- * This field is the length of the response in bytes. The last byte of
	- * the response is a valid flag that will read as '1' when the command
	- * has been completely written to memory.
	- */
	- uint32_t unused_0;
	- uint8_t unused_1;
	- uint8_t unused_2;
	- uint8_t unused_3;
	- uint8_t valid;
	- /*
	- * This field is used in Output records to indicate that the output is
	- * completely written to RAM. This field should be read as '1' to
	- * indicate that the output has been completely written. When writing a
	- * command completion or response to an internal processor, the order of
	- * writes has to be such that this field is written last.
	- */
	-} __attribute__((packed));
	-
	/* hwrm_port_ts_query */
	/*
	* Description: This function is used to read timestamp information captured for
	@@ -8331,23 +10583,31 @@
	* the response is a valid flag that will read as '1' when the command
	* has been completely written to memory.
	*/
	- uint8_t eee_supported;
	+ uint8_t flags;
	+ /* PHY capability flags */
	/*
	- * Reserved field. The HWRM shall set this field to 0. An HWRM client
	- * shall ignore this field.
	- */
	- /*
	* If set to 1, then this field indicates that the link is capable of
	* supporting EEE.
	*/
	- #define HWRM_PORT_PHY_QCAPS_OUTPUT_EEE_SUPPORTED UINT32_C(0x1)
	+ #define HWRM_PORT_PHY_QCAPS_OUTPUT_FLAGS_EEE_SUPPORTED UINT32_C(0x1)
	/*
	* Reserved field. The HWRM shall set this field to 0. An HWRM client
	* shall ignore this field.
	*/
	- #define HWRM_PORT_PHY_QCAPS_OUTPUT_RSVD1_MASK UINT32_C(0xfe)
	- #define HWRM_PORT_PHY_QCAPS_OUTPUT_RSVD1_SFT 1
	- uint8_t unused_0;
	+ #define HWRM_PORT_PHY_QCAPS_OUTPUT_FLAGS_RSVD1_MASK UINT32_C(0xfe)
	+ #define HWRM_PORT_PHY_QCAPS_OUTPUT_FLAGS_RSVD1_SFT 1
	+ uint8_t port_cnt;
	+ /* Number of front panel ports for this device. */
	+ /* Not supported or unknown */
	+ #define HWRM_PORT_PHY_QCAPS_OUTPUT_PORT_CNT_UNKNOWN UINT32_C(0x0)
	+ /* single port device */
	+ #define HWRM_PORT_PHY_QCAPS_OUTPUT_PORT_CNT_1 UINT32_C(0x1)
	+ /* 2-port device */
	+ #define HWRM_PORT_PHY_QCAPS_OUTPUT_PORT_CNT_2 UINT32_C(0x2)
	+ /* 3-port device */
	+ #define HWRM_PORT_PHY_QCAPS_OUTPUT_PORT_CNT_3 UINT32_C(0x3)
	+ /* 4-port device */
	+ #define HWRM_PORT_PHY_QCAPS_OUTPUT_PORT_CNT_4 UINT32_C(0x4)
	uint16_t supported_speeds_force_mode;
	/*
	* This is a bit mask to indicate what speeds are supported as forced
	@@ -8482,6 +10742,910 @@
	#define HWRM_PORT_PHY_QCAPS_OUTPUT_VALID_SFT 24
	} __attribute__((packed));

	+/* hwrm_port_led_cfg */
	+/*
	+ * Description: This function is used to configure LEDs on a given port. Each
	+ * port has individual set of LEDs associated with it. These LEDs are used for
	+ * speed/link configuration as well as activity indicator configuration. Up to
	+ * three LEDs can be configured, one for activity and two for speeds.
	+ */
	+/* Input (64 bytes) */
	+
	+struct hwrm_port_led_cfg_input {
	+ uint16_t req_type;
	+ /*
	+ * This value indicates what type of request this is. The format for the
	+ * rest of the command is determined by this field.
	+ */
	+ uint16_t cmpl_ring;
	+ /*
	+ * This value indicates the what completion ring the request will be
	+ * optionally completed on. If the value is -1, then no CR completion
	+ * will be generated. Any other value must be a valid CR ring_id value
	+ * for this function.
	+ */
	+ uint16_t seq_id;
	+ /* This value indicates the command sequence number. */
	+ uint16_t target_id;
	+ /*
	+ * Target ID of this command. 0x0 - 0xFFF8 - Used for function ids
	+ * 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF - HWRM
	+ */
	+ uint64_t resp_addr;
	+ /*
	+ * This is the host address where the response will be written when the
	+ * request is complete. This area must be 16B aligned and must be
	+ * cleared to zero before the request is made.
	+ */
	+ uint32_t enables;
	+ /* This bit must be '1' for the led0_id field to be configured. */
	+ #define HWRM_PORT_LED_CFG_INPUT_ENABLES_LED0_ID UINT32_C(0x1)
	+ /* This bit must be '1' for the led0_state field to be configured. */
	+ #define HWRM_PORT_LED_CFG_INPUT_ENABLES_LED0_STATE UINT32_C(0x2)
	+ /* This bit must be '1' for the led0_color field to be configured. */
	+ #define HWRM_PORT_LED_CFG_INPUT_ENABLES_LED0_COLOR UINT32_C(0x4)
	+ /* This bit must be '1' for the led0_blink_on field to be configured. */
	+ #define HWRM_PORT_LED_CFG_INPUT_ENABLES_LED0_BLINK_ON UINT32_C(0x8)
	+ /* This bit must be '1' for the led0_blink_off field to be configured. */
	+ #define HWRM_PORT_LED_CFG_INPUT_ENABLES_LED0_BLINK_OFF UINT32_C(0x10)
	+ /* This bit must be '1' for the led0_group_id field to be configured. */
	+ #define HWRM_PORT_LED_CFG_INPUT_ENABLES_LED0_GROUP_ID UINT32_C(0x20)
	+ /* This bit must be '1' for the led1_id field to be configured. */
	+ #define HWRM_PORT_LED_CFG_INPUT_ENABLES_LED1_ID UINT32_C(0x40)
	+ /* This bit must be '1' for the led1_state field to be configured. */
	+ #define HWRM_PORT_LED_CFG_INPUT_ENABLES_LED1_STATE UINT32_C(0x80)
	+ /* This bit must be '1' for the led1_color field to be configured. */
	+ #define HWRM_PORT_LED_CFG_INPUT_ENABLES_LED1_COLOR UINT32_C(0x100)
	+ /* This bit must be '1' for the led1_blink_on field to be configured. */
	+ #define HWRM_PORT_LED_CFG_INPUT_ENABLES_LED1_BLINK_ON UINT32_C(0x200)
	+ /* This bit must be '1' for the led1_blink_off field to be configured. */
	+ #define HWRM_PORT_LED_CFG_INPUT_ENABLES_LED1_BLINK_OFF UINT32_C(0x400)
	+ /* This bit must be '1' for the led1_group_id field to be configured. */
	+ #define HWRM_PORT_LED_CFG_INPUT_ENABLES_LED1_GROUP_ID UINT32_C(0x800)
	+ /* This bit must be '1' for the led2_id field to be configured. */
	+ #define HWRM_PORT_LED_CFG_INPUT_ENABLES_LED2_ID UINT32_C(0x1000)
	+ /* This bit must be '1' for the led2_state field to be configured. */
	+ #define HWRM_PORT_LED_CFG_INPUT_ENABLES_LED2_STATE UINT32_C(0x2000)
	+ /* This bit must be '1' for the led2_color field to be configured. */
	+ #define HWRM_PORT_LED_CFG_INPUT_ENABLES_LED2_COLOR UINT32_C(0x4000)
	+ /* This bit must be '1' for the led2_blink_on field to be configured. */
	+ #define HWRM_PORT_LED_CFG_INPUT_ENABLES_LED2_BLINK_ON UINT32_C(0x8000)
	+ /* This bit must be '1' for the led2_blink_off field to be configured. */
	+ #define HWRM_PORT_LED_CFG_INPUT_ENABLES_LED2_BLINK_OFF UINT32_C(0x10000)
	+ /* This bit must be '1' for the led2_group_id field to be configured. */
	+ #define HWRM_PORT_LED_CFG_INPUT_ENABLES_LED2_GROUP_ID UINT32_C(0x20000)
	+ /* This bit must be '1' for the led3_id field to be configured. */
	+ #define HWRM_PORT_LED_CFG_INPUT_ENABLES_LED3_ID UINT32_C(0x40000)
	+ /* This bit must be '1' for the led3_state field to be configured. */
	+ #define HWRM_PORT_LED_CFG_INPUT_ENABLES_LED3_STATE UINT32_C(0x80000)
	+ /* This bit must be '1' for the led3_color field to be configured. */
	+ #define HWRM_PORT_LED_CFG_INPUT_ENABLES_LED3_COLOR UINT32_C(0x100000)
	+ /* This bit must be '1' for the led3_blink_on field to be configured. */
	+ #define HWRM_PORT_LED_CFG_INPUT_ENABLES_LED3_BLINK_ON UINT32_C(0x200000)
	+ /* This bit must be '1' for the led3_blink_off field to be configured. */
	+ #define HWRM_PORT_LED_CFG_INPUT_ENABLES_LED3_BLINK_OFF UINT32_C(0x400000)
	+ /* This bit must be '1' for the led3_group_id field to be configured. */
	+ #define HWRM_PORT_LED_CFG_INPUT_ENABLES_LED3_GROUP_ID UINT32_C(0x800000)
	+ uint16_t port_id;
	+ /* Port ID of port whose LEDs are configured. */
	+ uint8_t num_leds;
	+ /*
	+ * The number of LEDs that are being configured. Up to 4 LEDs can be
	+ * configured with this command.
	+ */
	+ uint8_t rsvd;
	+ /* Reserved field. */
	+ uint8_t led0_id;
	+ /* An identifier for the LED #0. */
	+ uint8_t led0_state;
	+ /* The requested state of the LED #0. */
	+ /* Default state of the LED */
	+ #define HWRM_PORT_LED_CFG_INPUT_LED0_STATE_DEFAULT UINT32_C(0x0)
	+ /* Off */
	+ #define HWRM_PORT_LED_CFG_INPUT_LED0_STATE_OFF UINT32_C(0x1)
	+ /* On */
	+ #define HWRM_PORT_LED_CFG_INPUT_LED0_STATE_ON UINT32_C(0x2)
	+ /* Blink */
	+ #define HWRM_PORT_LED_CFG_INPUT_LED0_STATE_BLINK UINT32_C(0x3)
	+ /* Blink Alternately */
	+ #define HWRM_PORT_LED_CFG_INPUT_LED0_STATE_BLINKALT UINT32_C(0x4)
	+ uint8_t led0_color;
	+ /* The requested color of LED #0. */
	+ /* Default */
	+ #define HWRM_PORT_LED_CFG_INPUT_LED0_COLOR_DEFAULT UINT32_C(0x0)
	+ /* Amber */
	+ #define HWRM_PORT_LED_CFG_INPUT_LED0_COLOR_AMBER UINT32_C(0x1)
	+ /* Green */
	+ #define HWRM_PORT_LED_CFG_INPUT_LED0_COLOR_GREEN UINT32_C(0x2)
	+ /* Green or Amber */
	+ #define HWRM_PORT_LED_CFG_INPUT_LED0_COLOR_GREENAMBER UINT32_C(0x3)
	+ uint8_t unused_0;
	+ uint16_t led0_blink_on;
	+ /*
	+ * If the LED #0 state is "blink" or "blinkalt", then this field
	+ * represents the requested time in milliseconds to keep LED on between
	+ * cycles.
	+ */
	+ uint16_t led0_blink_off;
	+ /*
	+ * If the LED #0 state is "blink" or "blinkalt", then this field
	+ * represents the requested time in milliseconds to keep LED off between
	+ * cycles.
	+ */
	+ uint8_t led0_group_id;
	+ /*
	+ * An identifier for the group of LEDs that LED #0 belongs to. If set to
	+ * 0, then the LED #0 shall not be grouped and shall be treated as an
	+ * individual resource. For all other non-zero values of this field, LED
	+ * #0 shall be grouped together with the LEDs with the same group ID
	+ * value.
	+ */
	+ uint8_t rsvd0;
	+ /* Reserved field. */
	+ uint8_t led1_id;
	+ /* An identifier for the LED #1. */
	+ uint8_t led1_state;
	+ /* The requested state of the LED #1. */
	+ /* Default state of the LED */
	+ #define HWRM_PORT_LED_CFG_INPUT_LED1_STATE_DEFAULT UINT32_C(0x0)
	+ /* Off */
	+ #define HWRM_PORT_LED_CFG_INPUT_LED1_STATE_OFF UINT32_C(0x1)
	+ /* On */
	+ #define HWRM_PORT_LED_CFG_INPUT_LED1_STATE_ON UINT32_C(0x2)
	+ /* Blink */
	+ #define HWRM_PORT_LED_CFG_INPUT_LED1_STATE_BLINK UINT32_C(0x3)
	+ /* Blink Alternately */
	+ #define HWRM_PORT_LED_CFG_INPUT_LED1_STATE_BLINKALT UINT32_C(0x4)
	+ uint8_t led1_color;
	+ /* The requested color of LED #1. */
	+ /* Default */
	+ #define HWRM_PORT_LED_CFG_INPUT_LED1_COLOR_DEFAULT UINT32_C(0x0)
	+ /* Amber */
	+ #define HWRM_PORT_LED_CFG_INPUT_LED1_COLOR_AMBER UINT32_C(0x1)
	+ /* Green */
	+ #define HWRM_PORT_LED_CFG_INPUT_LED1_COLOR_GREEN UINT32_C(0x2)
	+ /* Green or Amber */
	+ #define HWRM_PORT_LED_CFG_INPUT_LED1_COLOR_GREENAMBER UINT32_C(0x3)
	+ uint8_t unused_1;
	+ uint16_t led1_blink_on;
	+ /*
	+ * If the LED #1 state is "blink" or "blinkalt", then this field
	+ * represents the requested time in milliseconds to keep LED on between
	+ * cycles.
	+ */
	+ uint16_t led1_blink_off;
	+ /*
	+ * If the LED #1 state is "blink" or "blinkalt", then this field
	+ * represents the requested time in milliseconds to keep LED off between
	+ * cycles.
	+ */
	+ uint8_t led1_group_id;
	+ /*
	+ * An identifier for the group of LEDs that LED #1 belongs to. If set to
	+ * 0, then the LED #1 shall not be grouped and shall be treated as an
	+ * individual resource. For all other non-zero values of this field, LED
	+ * #1 shall be grouped together with the LEDs with the same group ID
	+ * value.
	+ */
	+ uint8_t rsvd1;
	+ /* Reserved field. */
	+ uint8_t led2_id;
	+ /* An identifier for the LED #2. */
	+ uint8_t led2_state;
	+ /* The requested state of the LED #2. */
	+ /* Default state of the LED */
	+ #define HWRM_PORT_LED_CFG_INPUT_LED2_STATE_DEFAULT UINT32_C(0x0)
	+ /* Off */
	+ #define HWRM_PORT_LED_CFG_INPUT_LED2_STATE_OFF UINT32_C(0x1)
	+ /* On */
	+ #define HWRM_PORT_LED_CFG_INPUT_LED2_STATE_ON UINT32_C(0x2)
	+ /* Blink */
	+ #define HWRM_PORT_LED_CFG_INPUT_LED2_STATE_BLINK UINT32_C(0x3)
	+ /* Blink Alternately */
	+ #define HWRM_PORT_LED_CFG_INPUT_LED2_STATE_BLINKALT UINT32_C(0x4)
	+ uint8_t led2_color;
	+ /* The requested color of LED #2. */
	+ /* Default */
	+ #define HWRM_PORT_LED_CFG_INPUT_LED2_COLOR_DEFAULT UINT32_C(0x0)
	+ /* Amber */
	+ #define HWRM_PORT_LED_CFG_INPUT_LED2_COLOR_AMBER UINT32_C(0x1)
	+ /* Green */
	+ #define HWRM_PORT_LED_CFG_INPUT_LED2_COLOR_GREEN UINT32_C(0x2)
	+ /* Green or Amber */
	+ #define HWRM_PORT_LED_CFG_INPUT_LED2_COLOR_GREENAMBER UINT32_C(0x3)
	+ uint8_t unused_2;
	+ uint16_t led2_blink_on;
	+ /*
	+ * If the LED #2 state is "blink" or "blinkalt", then this field
	+ * represents the requested time in milliseconds to keep LED on between
	+ * cycles.
	+ */
	+ uint16_t led2_blink_off;
	+ /*
	+ * If the LED #2 state is "blink" or "blinkalt", then this field
	+ * represents the requested time in milliseconds to keep LED off between
	+ * cycles.
	+ */
	+ uint8_t led2_group_id;
	+ /*
	+ * An identifier for the group of LEDs that LED #2 belongs to. If set to
	+ * 0, then the LED #2 shall not be grouped and shall be treated as an
	+ * individual resource. For all other non-zero values of this field, LED
	+ * #2 shall be grouped together with the LEDs with the same group ID
	+ * value.
	+ */
	+ uint8_t rsvd2;
	+ /* Reserved field. */
	+ uint8_t led3_id;
	+ /* An identifier for the LED #3. */
	+ uint8_t led3_state;
	+ /* The requested state of the LED #3. */
	+ /* Default state of the LED */
	+ #define HWRM_PORT_LED_CFG_INPUT_LED3_STATE_DEFAULT UINT32_C(0x0)
	+ /* Off */
	+ #define HWRM_PORT_LED_CFG_INPUT_LED3_STATE_OFF UINT32_C(0x1)
	+ /* On */
	+ #define HWRM_PORT_LED_CFG_INPUT_LED3_STATE_ON UINT32_C(0x2)
	+ /* Blink */
	+ #define HWRM_PORT_LED_CFG_INPUT_LED3_STATE_BLINK UINT32_C(0x3)
	+ /* Blink Alternately */
	+ #define HWRM_PORT_LED_CFG_INPUT_LED3_STATE_BLINKALT UINT32_C(0x4)
	+ uint8_t led3_color;
	+ /* The requested color of LED #3. */
	+ /* Default */
	+ #define HWRM_PORT_LED_CFG_INPUT_LED3_COLOR_DEFAULT UINT32_C(0x0)
	+ /* Amber */
	+ #define HWRM_PORT_LED_CFG_INPUT_LED3_COLOR_AMBER UINT32_C(0x1)
	+ /* Green */
	+ #define HWRM_PORT_LED_CFG_INPUT_LED3_COLOR_GREEN UINT32_C(0x2)
	+ /* Green or Amber */
	+ #define HWRM_PORT_LED_CFG_INPUT_LED3_COLOR_GREENAMBER UINT32_C(0x3)
	+ uint8_t unused_3;
	+ uint16_t led3_blink_on;
	+ /*
	+ * If the LED #3 state is "blink" or "blinkalt", then this field
	+ * represents the requested time in milliseconds to keep LED on between
	+ * cycles.
	+ */
	+ uint16_t led3_blink_off;
	+ /*
	+ * If the LED #3 state is "blink" or "blinkalt", then this field
	+ * represents the requested time in milliseconds to keep LED off between
	+ * cycles.
	+ */
	+ uint8_t led3_group_id;
	+ /*
	+ * An identifier for the group of LEDs that LED #3 belongs to. If set to
	+ * 0, then the LED #3 shall not be grouped and shall be treated as an
	+ * individual resource. For all other non-zero values of this field, LED
	+ * #3 shall be grouped together with the LEDs with the same group ID
	+ * value.
	+ */
	+ uint8_t rsvd3;
	+ /* Reserved field. */
	+} __attribute__((packed));
	+
	+/* Output (16 bytes) */
	+
	+struct hwrm_port_led_cfg_output {
	+ uint16_t error_code;
	+ /*
	+ * Pass/Fail or error type Note: receiver to verify the in parameters,
	+ * and fail the call with an error when appropriate
	+ */
	+ uint16_t req_type;
	+ /* This field returns the type of original request. */
	+ uint16_t seq_id;
	+ /* This field provides original sequence number of the command. */
	+ uint16_t resp_len;
	+ /*
	+ * This field is the length of the response in bytes. The last byte of
	+ * the response is a valid flag that will read as '1' when the command
	+ * has been completely written to memory.
	+ */
	+ uint32_t unused_0;
	+ uint8_t unused_1;
	+ uint8_t unused_2;
	+ uint8_t unused_3;
	+ uint8_t valid;
	+ /*
	+ * This field is used in Output records to indicate that the output is
	+ * completely written to RAM. This field should be read as '1' to
	+ * indicate that the output has been completely written. When writing a
	+ * command completion or response to an internal processor, the order of
	+ * writes has to be such that this field is written last.
	+ */
	+} __attribute__((packed));
	+
	+/* hwrm_port_led_qcfg */
	+/*
	+ * Description: This function is used to query configuration of LEDs on a given
	+ * port. Each port has individual set of LEDs associated with it. These LEDs are
	+ * used for speed/link configuration as well as activity indicator
	+ * configuration. Up to three LEDs can be configured, one for activity and two
	+ * for speeds.
	+ */
	+/* Input (24 bytes) */
	+
	+struct hwrm_port_led_qcfg_input {
	+ uint16_t req_type;
	+ /*
	+ * This value indicates what type of request this is. The format for the
	+ * rest of the command is determined by this field.
	+ */
	+ uint16_t cmpl_ring;
	+ /*
	+ * This value indicates the what completion ring the request will be
	+ * optionally completed on. If the value is -1, then no CR completion
	+ * will be generated. Any other value must be a valid CR ring_id value
	+ * for this function.
	+ */
	+ uint16_t seq_id;
	+ /* This value indicates the command sequence number. */
	+ uint16_t target_id;
	+ /*
	+ * Target ID of this command. 0x0 - 0xFFF8 - Used for function ids
	+ * 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF - HWRM
	+ */
	+ uint64_t resp_addr;
	+ /*
	+ * This is the host address where the response will be written when the
	+ * request is complete. This area must be 16B aligned and must be
	+ * cleared to zero before the request is made.
	+ */
	+ uint16_t port_id;
	+ /* Port ID of port whose LED configuration is being queried. */
	+ uint16_t unused_0[3];
	+} __attribute__((packed));
	+
	+/* Output (56 bytes) */
	+
	+struct hwrm_port_led_qcfg_output {
	+ uint16_t error_code;
	+ /*
	+ * Pass/Fail or error type Note: receiver to verify the in parameters,
	+ * and fail the call with an error when appropriate
	+ */
	+ uint16_t req_type;
	+ /* This field returns the type of original request. */
	+ uint16_t seq_id;
	+ /* This field provides original sequence number of the command. */
	+ uint16_t resp_len;
	+ /*
	+ * This field is the length of the response in bytes. The last byte of
	+ * the response is a valid flag that will read as '1' when the command
	+ * has been completely written to memory.
	+ */
	+ uint8_t num_leds;
	+ /*
	+ * The number of LEDs that are configured on this port. Up to 4 LEDs can
	+ * be returned in the response.
	+ */
	+ uint8_t led0_id;
	+ /* An identifier for the LED #0. */
	+ uint8_t led0_type;
	+ /* The type of LED #0. */
	+ /* Speed LED */
	+ #define HWRM_PORT_LED_QCFG_OUTPUT_LED0_TYPE_SPEED UINT32_C(0x0)
	+ /* Activity LED */
	+ #define HWRM_PORT_LED_QCFG_OUTPUT_LED0_TYPE_ACTIVITY UINT32_C(0x1)
	+ /* Invalid */
	+ #define HWRM_PORT_LED_QCFG_OUTPUT_LED0_TYPE_INVALID UINT32_C(0xff)
	+ uint8_t led0_state;
	+ /* The current state of the LED #0. */
	+ /* Default state of the LED */
	+ #define HWRM_PORT_LED_QCFG_OUTPUT_LED0_STATE_DEFAULT UINT32_C(0x0)
	+ /* Off */
	+ #define HWRM_PORT_LED_QCFG_OUTPUT_LED0_STATE_OFF UINT32_C(0x1)
	+ /* On */
	+ #define HWRM_PORT_LED_QCFG_OUTPUT_LED0_STATE_ON UINT32_C(0x2)
	+ /* Blink */
	+ #define HWRM_PORT_LED_QCFG_OUTPUT_LED0_STATE_BLINK UINT32_C(0x3)
	+ /* Blink Alternately */
	+ #define HWRM_PORT_LED_QCFG_OUTPUT_LED0_STATE_BLINKALT UINT32_C(0x4)
	+ uint8_t led0_color;
	+ /* The color of LED #0. */
	+ /* Default */
	+ #define HWRM_PORT_LED_QCFG_OUTPUT_LED0_COLOR_DEFAULT UINT32_C(0x0)
	+ /* Amber */
	+ #define HWRM_PORT_LED_QCFG_OUTPUT_LED0_COLOR_AMBER UINT32_C(0x1)
	+ /* Green */
	+ #define HWRM_PORT_LED_QCFG_OUTPUT_LED0_COLOR_GREEN UINT32_C(0x2)
	+ /* Green or Amber */
	+ #define HWRM_PORT_LED_QCFG_OUTPUT_LED0_COLOR_GREENAMBER UINT32_C(0x3)
	+ uint8_t unused_0;
	+ uint16_t led0_blink_on;
	+ /*
	+ * If the LED #0 state is "blink" or "blinkalt", then this field
	+ * represents the requested time in milliseconds to keep LED on between
	+ * cycles.
	+ */
	+ uint16_t led0_blink_off;
	+ /*
	+ * If the LED #0 state is "blink" or "blinkalt", then this field
	+ * represents the requested time in milliseconds to keep LED off between
	+ * cycles.
	+ */
	+ uint8_t led0_group_id;
	+ /*
	+ * An identifier for the group of LEDs that LED #0 belongs to. If set to
	+ * 0, then the LED #0 is not grouped. For all other non-zero values of
	+ * this field, LED #0 is grouped together with the LEDs with the same
	+ * group ID value.
	+ */
	+ uint8_t led1_id;
	+ /* An identifier for the LED #1. */
	+ uint8_t led1_type;
	+ /* The type of LED #1. */
	+ /* Speed LED */
	+ #define HWRM_PORT_LED_QCFG_OUTPUT_LED1_TYPE_SPEED UINT32_C(0x0)
	+ /* Activity LED */
	+ #define HWRM_PORT_LED_QCFG_OUTPUT_LED1_TYPE_ACTIVITY UINT32_C(0x1)
	+ /* Invalid */
	+ #define HWRM_PORT_LED_QCFG_OUTPUT_LED1_TYPE_INVALID UINT32_C(0xff)
	+ uint8_t led1_state;
	+ /* The current state of the LED #1. */
	+ /* Default state of the LED */
	+ #define HWRM_PORT_LED_QCFG_OUTPUT_LED1_STATE_DEFAULT UINT32_C(0x0)
	+ /* Off */
	+ #define HWRM_PORT_LED_QCFG_OUTPUT_LED1_STATE_OFF UINT32_C(0x1)
	+ /* On */
	+ #define HWRM_PORT_LED_QCFG_OUTPUT_LED1_STATE_ON UINT32_C(0x2)
	+ /* Blink */
	+ #define HWRM_PORT_LED_QCFG_OUTPUT_LED1_STATE_BLINK UINT32_C(0x3)
	+ /* Blink Alternately */
	+ #define HWRM_PORT_LED_QCFG_OUTPUT_LED1_STATE_BLINKALT UINT32_C(0x4)
	+ uint8_t led1_color;
	+ /* The color of LED #1. */
	+ /* Default */
	+ #define HWRM_PORT_LED_QCFG_OUTPUT_LED1_COLOR_DEFAULT UINT32_C(0x0)
	+ /* Amber */
	+ #define HWRM_PORT_LED_QCFG_OUTPUT_LED1_COLOR_AMBER UINT32_C(0x1)
	+ /* Green */
	+ #define HWRM_PORT_LED_QCFG_OUTPUT_LED1_COLOR_GREEN UINT32_C(0x2)
	+ /* Green or Amber */
	+ #define HWRM_PORT_LED_QCFG_OUTPUT_LED1_COLOR_GREENAMBER UINT32_C(0x3)
	+ uint8_t unused_1;
	+ uint16_t led1_blink_on;
	+ /*
	+ * If the LED #1 state is "blink" or "blinkalt", then this field
	+ * represents the requested time in milliseconds to keep LED on between
	+ * cycles.
	+ */
	+ uint16_t led1_blink_off;
	+ /*
	+ * If the LED #1 state is "blink" or "blinkalt", then this field
	+ * represents the requested time in milliseconds to keep LED off between
	+ * cycles.
	+ */
	+ uint8_t led1_group_id;
	+ /*
	+ * An identifier for the group of LEDs that LED #1 belongs to. If set to
	+ * 0, then the LED #1 is not grouped. For all other non-zero values of
	+ * this field, LED #1 is grouped together with the LEDs with the same
	+ * group ID value.
	+ */
	+ uint8_t led2_id;
	+ /* An identifier for the LED #2. */
	+ uint8_t led2_type;
	+ /* The type of LED #2. */
	+ /* Speed LED */
	+ #define HWRM_PORT_LED_QCFG_OUTPUT_LED2_TYPE_SPEED UINT32_C(0x0)
	+ /* Activity LED */
	+ #define HWRM_PORT_LED_QCFG_OUTPUT_LED2_TYPE_ACTIVITY UINT32_C(0x1)
	+ /* Invalid */
	+ #define HWRM_PORT_LED_QCFG_OUTPUT_LED2_TYPE_INVALID UINT32_C(0xff)
	+ uint8_t led2_state;
	+ /* The current state of the LED #2. */
	+ /* Default state of the LED */
	+ #define HWRM_PORT_LED_QCFG_OUTPUT_LED2_STATE_DEFAULT UINT32_C(0x0)
	+ /* Off */
	+ #define HWRM_PORT_LED_QCFG_OUTPUT_LED2_STATE_OFF UINT32_C(0x1)
	+ /* On */
	+ #define HWRM_PORT_LED_QCFG_OUTPUT_LED2_STATE_ON UINT32_C(0x2)
	+ /* Blink */
	+ #define HWRM_PORT_LED_QCFG_OUTPUT_LED2_STATE_BLINK UINT32_C(0x3)
	+ /* Blink Alternately */
	+ #define HWRM_PORT_LED_QCFG_OUTPUT_LED2_STATE_BLINKALT UINT32_C(0x4)
	+ uint8_t led2_color;
	+ /* The color of LED #2. */
	+ /* Default */
	+ #define HWRM_PORT_LED_QCFG_OUTPUT_LED2_COLOR_DEFAULT UINT32_C(0x0)
	+ /* Amber */
	+ #define HWRM_PORT_LED_QCFG_OUTPUT_LED2_COLOR_AMBER UINT32_C(0x1)
	+ /* Green */
	+ #define HWRM_PORT_LED_QCFG_OUTPUT_LED2_COLOR_GREEN UINT32_C(0x2)
	+ /* Green or Amber */
	+ #define HWRM_PORT_LED_QCFG_OUTPUT_LED2_COLOR_GREENAMBER UINT32_C(0x3)
	+ uint8_t unused_2;
	+ uint16_t led2_blink_on;
	+ /*
	+ * If the LED #2 state is "blink" or "blinkalt", then this field
	+ * represents the requested time in milliseconds to keep LED on between
	+ * cycles.
	+ */
	+ uint16_t led2_blink_off;
	+ /*
	+ * If the LED #2 state is "blink" or "blinkalt", then this field
	+ * represents the requested time in milliseconds to keep LED off between
	+ * cycles.
	+ */
	+ uint8_t led2_group_id;
	+ /*
	+ * An identifier for the group of LEDs that LED #2 belongs to. If set to
	+ * 0, then the LED #2 is not grouped. For all other non-zero values of
	+ * this field, LED #2 is grouped together with the LEDs with the same
	+ * group ID value.
	+ */
	+ uint8_t led3_id;
	+ /* An identifier for the LED #3. */
	+ uint8_t led3_type;
	+ /* The type of LED #3. */
	+ /* Speed LED */
	+ #define HWRM_PORT_LED_QCFG_OUTPUT_LED3_TYPE_SPEED UINT32_C(0x0)
	+ /* Activity LED */
	+ #define HWRM_PORT_LED_QCFG_OUTPUT_LED3_TYPE_ACTIVITY UINT32_C(0x1)
	+ /* Invalid */
	+ #define HWRM_PORT_LED_QCFG_OUTPUT_LED3_TYPE_INVALID UINT32_C(0xff)
	+ uint8_t led3_state;
	+ /* The current state of the LED #3. */
	+ /* Default state of the LED */
	+ #define HWRM_PORT_LED_QCFG_OUTPUT_LED3_STATE_DEFAULT UINT32_C(0x0)
	+ /* Off */
	+ #define HWRM_PORT_LED_QCFG_OUTPUT_LED3_STATE_OFF UINT32_C(0x1)
	+ /* On */
	+ #define HWRM_PORT_LED_QCFG_OUTPUT_LED3_STATE_ON UINT32_C(0x2)
	+ /* Blink */
	+ #define HWRM_PORT_LED_QCFG_OUTPUT_LED3_STATE_BLINK UINT32_C(0x3)
	+ /* Blink Alternately */
	+ #define HWRM_PORT_LED_QCFG_OUTPUT_LED3_STATE_BLINKALT UINT32_C(0x4)
	+ uint8_t led3_color;
	+ /* The color of LED #3. */
	+ /* Default */
	+ #define HWRM_PORT_LED_QCFG_OUTPUT_LED3_COLOR_DEFAULT UINT32_C(0x0)
	+ /* Amber */
	+ #define HWRM_PORT_LED_QCFG_OUTPUT_LED3_COLOR_AMBER UINT32_C(0x1)
	+ /* Green */
	+ #define HWRM_PORT_LED_QCFG_OUTPUT_LED3_COLOR_GREEN UINT32_C(0x2)
	+ /* Green or Amber */
	+ #define HWRM_PORT_LED_QCFG_OUTPUT_LED3_COLOR_GREENAMBER UINT32_C(0x3)
	+ uint8_t unused_3;
	+ uint16_t led3_blink_on;
	+ /*
	+ * If the LED #3 state is "blink" or "blinkalt", then this field
	+ * represents the requested time in milliseconds to keep LED on between
	+ * cycles.
	+ */
	+ uint16_t led3_blink_off;
	+ /*
	+ * If the LED #3 state is "blink" or "blinkalt", then this field
	+ * represents the requested time in milliseconds to keep LED off between
	+ * cycles.
	+ */
	+ uint8_t led3_group_id;
	+ /*
	+ * An identifier for the group of LEDs that LED #3 belongs to. If set to
	+ * 0, then the LED #3 is not grouped. For all other non-zero values of
	+ * this field, LED #3 is grouped together with the LEDs with the same
	+ * group ID value.
	+ */
	+ uint8_t unused_4;
	+ uint16_t unused_5;
	+ uint8_t unused_6;
	+ uint8_t unused_7;
	+ uint8_t unused_8;
	+ uint8_t valid;
	+ /*
	+ * This field is used in Output records to indicate that the output is
	+ * completely written to RAM. This field should be read as '1' to
	+ * indicate that the output has been completely written. When writing a
	+ * command completion or response to an internal processor, the order of
	+ * writes has to be such that this field is written last.
	+ */
	+} __attribute__((packed));
	+
	+/* hwrm_port_led_qcaps */
	+/*
	+ * Description: This function is used to query capabilities of LEDs on a given
	+ * port. Each port has individual set of LEDs associated with it. These LEDs are
	+ * used for speed/link configuration as well as activity indicator
	+ * configuration.
	+ */
	+/* Input (24 bytes) */
	+
	+struct hwrm_port_led_qcaps_input {
	+ uint16_t req_type;
	+ /*
	+ * This value indicates what type of request this is. The format for the
	+ * rest of the command is determined by this field.
	+ */
	+ uint16_t cmpl_ring;
	+ /*
	+ * This value indicates the what completion ring the request will be
	+ * optionally completed on. If the value is -1, then no CR completion
	+ * will be generated. Any other value must be a valid CR ring_id value
	+ * for this function.
	+ */
	+ uint16_t seq_id;
	+ /* This value indicates the command sequence number. */
	+ uint16_t target_id;
	+ /*
	+ * Target ID of this command. 0x0 - 0xFFF8 - Used for function ids
	+ * 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF - HWRM
	+ */
	+ uint64_t resp_addr;
	+ /*
	+ * This is the host address where the response will be written when the
	+ * request is complete. This area must be 16B aligned and must be
	+ * cleared to zero before the request is made.
	+ */
	+ uint16_t port_id;
	+ /* Port ID of port whose LED configuration is being queried. */
	+ uint16_t unused_0[3];
	+} __attribute__((packed));
	+
	+/* Output (48 bytes) */
	+
	+struct hwrm_port_led_qcaps_output {
	+ uint16_t error_code;
	+ /*
	+ * Pass/Fail or error type Note: receiver to verify the in parameters,
	+ * and fail the call with an error when appropriate
	+ */
	+ uint16_t req_type;
	+ /* This field returns the type of original request. */
	+ uint16_t seq_id;
	+ /* This field provides original sequence number of the command. */
	+ uint16_t resp_len;
	+ /*
	+ * This field is the length of the response in bytes. The last byte of
	+ * the response is a valid flag that will read as '1' when the command
	+ * has been completely written to memory.
	+ */
	+ uint8_t num_leds;
	+ /*
	+ * The number of LEDs that are configured on this port. Up to 4 LEDs can
	+ * be returned in the response.
	+ */
	+ uint8_t unused_0[3];
	+ /* Reserved for future use. */
	+ uint8_t led0_id;
	+ /* An identifier for the LED #0. */
	+ uint8_t led0_type;
	+ /* The type of LED #0. */
	+ /* Speed LED */
	+ #define HWRM_PORT_LED_QCAPS_OUTPUT_LED0_TYPE_SPEED UINT32_C(0x0)
	+ /* Activity LED */
	+ #define HWRM_PORT_LED_QCAPS_OUTPUT_LED0_TYPE_ACTIVITY UINT32_C(0x1)
	+ /* Invalid */
	+ #define HWRM_PORT_LED_QCAPS_OUTPUT_LED0_TYPE_INVALID UINT32_C(0xff)
	+ uint8_t led0_group_id;
	+ /*
	+ * An identifier for the group of LEDs that LED #0 belongs to. If set to
	+ * 0, then the LED #0 cannot be grouped. For all other non-zero values
	+ * of this field, LED #0 is grouped together with the LEDs with the same
	+ * group ID value.
	+ */
	+ uint8_t unused_1;
	+ uint16_t led0_state_caps;
	+ /* The states supported by LED #0. */
	+ /* If set to 1, this LED is enabled. If set to 0, this LED is disabled. */
	+ #define HWRM_PORT_LED_QCAPS_OUTPUT_LED0_STATE_CAPS_ENABLED UINT32_C(0x1)
	+ /*
	+ * If set to 1, off state is supported on this LED. If set to 0, off
	+ * state is not supported on this LED.
	+ */
	+ #define HWRM_PORT_LED_QCAPS_OUTPUT_LED0_STATE_CAPS_OFF_SUPPORTED UINT32_C(0x2)
	+ /*
	+ * If set to 1, on state is supported on this LED. If set to 0, on state
	+ * is not supported on this LED.
	+ */
	+ #define HWRM_PORT_LED_QCAPS_OUTPUT_LED0_STATE_CAPS_ON_SUPPORTED UINT32_C(0x4)
	+ /*
	+ * If set to 1, blink state is supported on this LED. If set to 0, blink
	+ * state is not supported on this LED.
	+ */
	+ #define HWRM_PORT_LED_QCAPS_OUTPUT_LED0_STATE_CAPS_BLINK_SUPPORTED UINT32_C(0x8)
	+ /*
	+ * If set to 1, blink_alt state is supported on this LED. If set to 0,
	+ * blink_alt state is not supported on this LED.
	+ */
	+ #define HWRM_PORT_LED_QCAPS_OUTPUT_LED0_STATE_CAPS_BLINK_ALT_SUPPORTED UINT32_C(0x10)
	+ uint16_t led0_color_caps;
	+ /* The colors supported by LED #0. */
	+ /* reserved */
	+ #define HWRM_PORT_LED_QCAPS_OUTPUT_LED0_COLOR_CAPS_RSVD UINT32_C(0x1)
	+ /*
	+ * If set to 1, Amber color is supported on this LED. If set to 0, Amber
	+ * color is not supported on this LED.
	+ */
	+ #define HWRM_PORT_LED_QCAPS_OUTPUT_LED0_COLOR_CAPS_AMBER_SUPPORTED UINT32_C(0x2)
	+ /*
	+ * If set to 1, Green color is supported on this LED. If set to 0, Green
	+ * color is not supported on this LED.
	+ */
	+ #define HWRM_PORT_LED_QCAPS_OUTPUT_LED0_COLOR_CAPS_GREEN_SUPPORTED UINT32_C(0x4)
	+ uint8_t led1_id;
	+ /* An identifier for the LED #1. */
	+ uint8_t led1_type;
	+ /* The type of LED #1. */
	+ /* Speed LED */
	+ #define HWRM_PORT_LED_QCAPS_OUTPUT_LED1_TYPE_SPEED UINT32_C(0x0)
	+ /* Activity LED */
	+ #define HWRM_PORT_LED_QCAPS_OUTPUT_LED1_TYPE_ACTIVITY UINT32_C(0x1)
	+ /* Invalid */
	+ #define HWRM_PORT_LED_QCAPS_OUTPUT_LED1_TYPE_INVALID UINT32_C(0xff)
	+ uint8_t led1_group_id;
	+ /*
	+ * An identifier for the group of LEDs that LED #1 belongs to. If set to
	+ * 0, then the LED #0 cannot be grouped. For all other non-zero values
	+ * of this field, LED #0 is grouped together with the LEDs with the same
	+ * group ID value.
	+ */
	+ uint8_t unused_2;
	+ uint16_t led1_state_caps;
	+ /* The states supported by LED #1. */
	+ /* If set to 1, this LED is enabled. If set to 0, this LED is disabled. */
	+ #define HWRM_PORT_LED_QCAPS_OUTPUT_LED1_STATE_CAPS_ENABLED UINT32_C(0x1)
	+ /*
	+ * If set to 1, off state is supported on this LED. If set to 0, off
	+ * state is not supported on this LED.
	+ */
	+ #define HWRM_PORT_LED_QCAPS_OUTPUT_LED1_STATE_CAPS_OFF_SUPPORTED UINT32_C(0x2)
	+ /*
	+ * If set to 1, on state is supported on this LED. If set to 0, on state
	+ * is not supported on this LED.
	+ */
	+ #define HWRM_PORT_LED_QCAPS_OUTPUT_LED1_STATE_CAPS_ON_SUPPORTED UINT32_C(0x4)
	+ /*
	+ * If set to 1, blink state is supported on this LED. If set to 0, blink
	+ * state is not supported on this LED.
	+ */
	+ #define HWRM_PORT_LED_QCAPS_OUTPUT_LED1_STATE_CAPS_BLINK_SUPPORTED UINT32_C(0x8)
	+ /*
	+ * If set to 1, blink_alt state is supported on this LED. If set to 0,
	+ * blink_alt state is not supported on this LED.
	+ */
	+ #define HWRM_PORT_LED_QCAPS_OUTPUT_LED1_STATE_CAPS_BLINK_ALT_SUPPORTED UINT32_C(0x10)
	+ uint16_t led1_color_caps;
	+ /* The colors supported by LED #1. */
	+ /* reserved */
	+ #define HWRM_PORT_LED_QCAPS_OUTPUT_LED1_COLOR_CAPS_RSVD UINT32_C(0x1)
	+ /*
	+ * If set to 1, Amber color is supported on this LED. If set to 0, Amber
	+ * color is not supported on this LED.
	+ */
	+ #define HWRM_PORT_LED_QCAPS_OUTPUT_LED1_COLOR_CAPS_AMBER_SUPPORTED UINT32_C(0x2)
	+ /*
	+ * If set to 1, Green color is supported on this LED. If set to 0, Green
	+ * color is not supported on this LED.
	+ */
	+ #define HWRM_PORT_LED_QCAPS_OUTPUT_LED1_COLOR_CAPS_GREEN_SUPPORTED UINT32_C(0x4)
	+ uint8_t led2_id;
	+ /* An identifier for the LED #2. */
	+ uint8_t led2_type;
	+ /* The type of LED #2. */
	+ /* Speed LED */
	+ #define HWRM_PORT_LED_QCAPS_OUTPUT_LED2_TYPE_SPEED UINT32_C(0x0)
	+ /* Activity LED */
	+ #define HWRM_PORT_LED_QCAPS_OUTPUT_LED2_TYPE_ACTIVITY UINT32_C(0x1)
	+ /* Invalid */
	+ #define HWRM_PORT_LED_QCAPS_OUTPUT_LED2_TYPE_INVALID UINT32_C(0xff)
	+ uint8_t led2_group_id;
	+ /*
	+ * An identifier for the group of LEDs that LED #0 belongs to. If set to
	+ * 0, then the LED #0 cannot be grouped. For all other non-zero values
	+ * of this field, LED #0 is grouped together with the LEDs with the same
	+ * group ID value.
	+ */
	+ uint8_t unused_3;
	+ uint16_t led2_state_caps;
	+ /* The states supported by LED #2. */
	+ /* If set to 1, this LED is enabled. If set to 0, this LED is disabled. */
	+ #define HWRM_PORT_LED_QCAPS_OUTPUT_LED2_STATE_CAPS_ENABLED UINT32_C(0x1)
	+ /*
	+ * If set to 1, off state is supported on this LED. If set to 0, off
	+ * state is not supported on this LED.
	+ */
	+ #define HWRM_PORT_LED_QCAPS_OUTPUT_LED2_STATE_CAPS_OFF_SUPPORTED UINT32_C(0x2)
	+ /*
	+ * If set to 1, on state is supported on this LED. If set to 0, on state
	+ * is not supported on this LED.
	+ */
	+ #define HWRM_PORT_LED_QCAPS_OUTPUT_LED2_STATE_CAPS_ON_SUPPORTED UINT32_C(0x4)
	+ /*
	+ * If set to 1, blink state is supported on this LED. If set to 0, blink
	+ * state is not supported on this LED.
	+ */
	+ #define HWRM_PORT_LED_QCAPS_OUTPUT_LED2_STATE_CAPS_BLINK_SUPPORTED UINT32_C(0x8)
	+ /*
	+ * If set to 1, blink_alt state is supported on this LED. If set to 0,
	+ * blink_alt state is not supported on this LED.
	+ */
	+ #define HWRM_PORT_LED_QCAPS_OUTPUT_LED2_STATE_CAPS_BLINK_ALT_SUPPORTED UINT32_C(0x10)
	+ uint16_t led2_color_caps;
	+ /* The colors supported by LED #2. */
	+ /* reserved */
	+ #define HWRM_PORT_LED_QCAPS_OUTPUT_LED2_COLOR_CAPS_RSVD UINT32_C(0x1)
	+ /*
	+ * If set to 1, Amber color is supported on this LED. If set to 0, Amber
	+ * color is not supported on this LED.
	+ */
	+ #define HWRM_PORT_LED_QCAPS_OUTPUT_LED2_COLOR_CAPS_AMBER_SUPPORTED UINT32_C(0x2)
	+ /*
	+ * If set to 1, Green color is supported on this LED. If set to 0, Green
	+ * color is not supported on this LED.
	+ */
	+ #define HWRM_PORT_LED_QCAPS_OUTPUT_LED2_COLOR_CAPS_GREEN_SUPPORTED UINT32_C(0x4)
	+ uint8_t led3_id;
	+ /* An identifier for the LED #3. */
	+ uint8_t led3_type;
	+ /* The type of LED #3. */
	+ /* Speed LED */
	+ #define HWRM_PORT_LED_QCAPS_OUTPUT_LED3_TYPE_SPEED UINT32_C(0x0)
	+ /* Activity LED */
	+ #define HWRM_PORT_LED_QCAPS_OUTPUT_LED3_TYPE_ACTIVITY UINT32_C(0x1)
	+ /* Invalid */
	+ #define HWRM_PORT_LED_QCAPS_OUTPUT_LED3_TYPE_INVALID UINT32_C(0xff)
	+ uint8_t led3_group_id;
	+ /*
	+ * An identifier for the group of LEDs that LED #3 belongs to. If set to
	+ * 0, then the LED #0 cannot be grouped. For all other non-zero values
	+ * of this field, LED #0 is grouped together with the LEDs with the same
	+ * group ID value.
	+ */
	+ uint8_t unused_4;
	+ uint16_t led3_state_caps;
	+ /* The states supported by LED #3. */
	+ /* If set to 1, this LED is enabled. If set to 0, this LED is disabled. */
	+ #define HWRM_PORT_LED_QCAPS_OUTPUT_LED3_STATE_CAPS_ENABLED UINT32_C(0x1)
	+ /*
	+ * If set to 1, off state is supported on this LED. If set to 0, off
	+ * state is not supported on this LED.
	+ */
	+ #define HWRM_PORT_LED_QCAPS_OUTPUT_LED3_STATE_CAPS_OFF_SUPPORTED UINT32_C(0x2)
	+ /*
	+ * If set to 1, on state is supported on this LED. If set to 0, on state
	+ * is not supported on this LED.
	+ */
	+ #define HWRM_PORT_LED_QCAPS_OUTPUT_LED3_STATE_CAPS_ON_SUPPORTED UINT32_C(0x4)
	+ /*
	+ * If set to 1, blink state is supported on this LED. If set to 0, blink
	+ * state is not supported on this LED.
	+ */
	+ #define HWRM_PORT_LED_QCAPS_OUTPUT_LED3_STATE_CAPS_BLINK_SUPPORTED UINT32_C(0x8)
	+ /*
	+ * If set to 1, blink_alt state is supported on this LED. If set to 0,
	+ * blink_alt state is not supported on this LED.
	+ */
	+ #define HWRM_PORT_LED_QCAPS_OUTPUT_LED3_STATE_CAPS_BLINK_ALT_SUPPORTED UINT32_C(0x10)
	+ uint16_t led3_color_caps;
	+ /* The colors supported by LED #3. */
	+ /* reserved */
	+ #define HWRM_PORT_LED_QCAPS_OUTPUT_LED3_COLOR_CAPS_RSVD UINT32_C(0x1)
	+ /*
	+ * If set to 1, Amber color is supported on this LED. If set to 0, Amber
	+ * color is not supported on this LED.
	+ */
	+ #define HWRM_PORT_LED_QCAPS_OUTPUT_LED3_COLOR_CAPS_AMBER_SUPPORTED UINT32_C(0x2)
	+ /*
	+ * If set to 1, Green color is supported on this LED. If set to 0, Green
	+ * color is not supported on this LED.
	+ */
	+ #define HWRM_PORT_LED_QCAPS_OUTPUT_LED3_COLOR_CAPS_GREEN_SUPPORTED UINT32_C(0x4)
	+ uint8_t unused_5;
	+ uint8_t unused_6;
	+ uint8_t unused_7;
	+ uint8_t valid;
	+ /*
	+ * This field is used in Output records to indicate that the output is
	+ * completely written to RAM. This field should be read as '1' to
	+ * indicate that the output has been completely written. When writing a
	+ * command completion or response to an internal processor, the order of
	+ * writes has to be such that this field is written last.
	+ */
	+} __attribute__((packed));
	+
	/* hwrm_queue_qportcfg */
	/*
	* Description: This function is called by a driver to query queue configuration
	@@ -8560,41 +11724,77 @@
	* has been completely written to memory.
	*/
	uint8_t max_configurable_queues;
	- /* The maximum number of queues that can be configured. */
	+ /*
	+ * The maximum number of queues that can be configured on this port.
	+ * Valid values range from 1 through 8.
	+ */
	uint8_t max_configurable_lossless_queues;
	- /* The maximum number of lossless queues that can be configured. */
	+ /*
	+ * The maximum number of lossless queues that can be configured on this
	+ * port. Valid values range from 0 through 8.
	+ */
	uint8_t queue_cfg_allowed;
	/*
	- * 0 - Not allowed. Non-zero - Allowed. If this value is non-zero, then
	- * the HWRM shall allow the host SW driver to configure queues using
	- * hwrm_queue_cfg.
	+ * Bitmask indicating which queues can be configured by the
	+ * hwrm_queue_cfg command. Each bit represents a specific queue where
	+ * bit 0 represents queue 0 and bit 7 represents queue 7. # A value of 0
	+ * indicates that the queue is not configurable by the hwrm_queue_cfg
	+ * command. # A value of 1 indicates that the queue is configurable. # A
	+ * hwrm_queue_cfg command shall return error when trying to configure a
	+ * queue not configurable.
	*/
	- uint8_t queue_buffers_cfg_allowed;
	+ uint8_t queue_cfg_info;
	+ /* Information about queue configuration. */
	/*
	- * 0 - Not allowed. Non-zero - Allowed If this value is non-zero, then
	- * the HWRM shall allow the host SW driver to configure queue buffers
	- * using hwrm_queue_buffers_cfg.
	+ * If this flag is set to '1', then the queues are configured
	+ * asymmetrically on TX and RX sides. If this flag is set to '0', then
	+ * the queues are configured symmetrically on TX and RX sides. For
	+ * symmetric configuration, the queue configuration including queue ids
	+ * and service profiles on the TX side is the same as the corresponding
	+ * queue configuration on the RX side.
	*/
	+ #define HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_CFG_INFO_ASYM_CFG UINT32_C(0x1)
	uint8_t queue_pfcenable_cfg_allowed;
	/*
	- * 0 - Not allowed. Non-zero - Allowed If this value is non-zero, then
	- * the HWRM shall allow the host SW driver to configure PFC using
	- * hwrm_queue_pfcenable_cfg.
	+ * Bitmask indicating which queues can be configured by the
	+ * hwrm_queue_pfcenable_cfg command. Each bit represents a specific
	+ * priority where bit 0 represents priority 0 and bit 7 represents
	+ * priority 7. # A value of 0 indicates that the priority is not
	+ * configurable by the hwrm_queue_pfcenable_cfg command. # A value of 1
	+ * indicates that the priority is configurable. # A
	+ * hwrm_queue_pfcenable_cfg command shall return error when trying to
	+ * configure a priority that is not configurable.
	*/
	uint8_t queue_pri2cos_cfg_allowed;
	/*
	- * 0 - Not allowed. Non-zero - Allowed If this value is non-zero, then
	- * the HWRM shall allow the host SW driver to configure Priority to CoS
	- * mapping using hwrm_queue_pri2cos_cfg.
	+ * Bitmask indicating which queues can be configured by the
	+ * hwrm_queue_pri2cos_cfg command. Each bit represents a specific queue
	+ * where bit 0 represents queue 0 and bit 7 represents queue 7. # A
	+ * value of 0 indicates that the queue is not configurable by the
	+ * hwrm_queue_pri2cos_cfg command. # A value of 1 indicates that the
	+ * queue is configurable. # A hwrm_queue_pri2cos_cfg command shall
	+ * return error when trying to configure a queue that is not
	+ * configurable.
	*/
	uint8_t queue_cos2bw_cfg_allowed;
	/*
	- * 0 - Not allowed. Non-zero - Allowed If this value is non-zero, then
	- * the HWRM shall allow the host SW driver to configure CoS Bandwidth
	- * configuration using hwrm_queue_cos2bw_cfg.
	+ * Bitmask indicating which queues can be configured by the
	+ * hwrm_queue_pri2cos_cfg command. Each bit represents a specific queue
	+ * where bit 0 represents queue 0 and bit 7 represents queue 7. # A
	+ * value of 0 indicates that the queue is not configurable by the
	+ * hwrm_queue_pri2cos_cfg command. # A value of 1 indicates that the
	+ * queue is configurable. # A hwrm_queue_pri2cos_cfg command shall
	+ * return error when trying to configure a queue not configurable.
	*/
	uint8_t queue_id0;
	- /* ID of CoS Queue 0. FF - Invalid id */
	+ /*
	+ * ID of CoS Queue 0. FF - Invalid id # This ID can be used on any
	+ * subsequent call to an hwrm command that takes a queue id. # IDs must
	+ * always be queried by this command before any use by the driver or
	+ * software. # Any driver or software should not make any assumptions
	+ * about queue IDs. # A value of 0xff indicates that the queue is not
	+ * available. # Available queues may not be in sequential order.
	+ */
	uint8_t queue_id0_service_profile;
	/* This value is applicable to CoS queues only. */
	/* Lossy (best-effort) */
	@@ -8604,7 +11804,14 @@
	/* Set to 0xFF... (All Fs) if there is no service profile specified */
	#define HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID0_SERVICE_PROFILE_UNKNOWN UINT32_C(0xff)
	uint8_t queue_id1;
	- /* ID of CoS Queue 1. FF - Invalid id */
	+ /*
	+ * ID of CoS Queue 1. FF - Invalid id # This ID can be used on any
	+ * subsequent call to an hwrm command that takes a queue id. # IDs must
	+ * always be queried by this command before any use by the driver or
	+ * software. # Any driver or software should not make any assumptions
	+ * about queue IDs. # A value of 0xff indicates that the queue is not
	+ * available. # Available queues may not be in sequential order.
	+ */
	uint8_t queue_id1_service_profile;
	/* This value is applicable to CoS queues only. */
	/* Lossy (best-effort) */
	@@ -8614,7 +11821,14 @@
	/* Set to 0xFF... (All Fs) if there is no service profile specified */
	#define HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID1_SERVICE_PROFILE_UNKNOWN UINT32_C(0xff)
	uint8_t queue_id2;
	- /* ID of CoS Queue 2. FF - Invalid id */
	+ /*
	+ * ID of CoS Queue 2. FF - Invalid id # This ID can be used on any
	+ * subsequent call to an hwrm command that takes a queue id. # IDs must
	+ * always be queried by this command before any use by the driver or
	+ * software. # Any driver or software should not make any assumptions
	+ * about queue IDs. # A value of 0xff indicates that the queue is not
	+ * available. # Available queues may not be in sequential order.
	+ */
	uint8_t queue_id2_service_profile;
	/* This value is applicable to CoS queues only. */
	/* Lossy (best-effort) */
	@@ -8624,7 +11838,14 @@
	/* Set to 0xFF... (All Fs) if there is no service profile specified */
	#define HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID2_SERVICE_PROFILE_UNKNOWN UINT32_C(0xff)
	uint8_t queue_id3;
	- /* ID of CoS Queue 3. FF - Invalid id */
	+ /*
	+ * ID of CoS Queue 3. FF - Invalid id # This ID can be used on any
	+ * subsequent call to an hwrm command that takes a queue id. # IDs must
	+ * always be queried by this command before any use by the driver or
	+ * software. # Any driver or software should not make any assumptions
	+ * about queue IDs. # A value of 0xff indicates that the queue is not
	+ * available. # Available queues may not be in sequential order.
	+ */
	uint8_t queue_id3_service_profile;
	/* This value is applicable to CoS queues only. */
	/* Lossy (best-effort) */
	@@ -8634,7 +11855,14 @@
	/* Set to 0xFF... (All Fs) if there is no service profile specified */
	#define HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID3_SERVICE_PROFILE_UNKNOWN UINT32_C(0xff)
	uint8_t queue_id4;
	- /* ID of CoS Queue 4. FF - Invalid id */
	+ /*
	+ * ID of CoS Queue 4. FF - Invalid id # This ID can be used on any
	+ * subsequent call to an hwrm command that takes a queue id. # IDs must
	+ * always be queried by this command before any use by the driver or
	+ * software. # Any driver or software should not make any assumptions
	+ * about queue IDs. # A value of 0xff indicates that the queue is not
	+ * available. # Available queues may not be in sequential order.
	+ */
	uint8_t queue_id4_service_profile;
	/* This value is applicable to CoS queues only. */
	/* Lossy (best-effort) */
	@@ -8644,7 +11872,14 @@
	/* Set to 0xFF... (All Fs) if there is no service profile specified */
	#define HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID4_SERVICE_PROFILE_UNKNOWN UINT32_C(0xff)
	uint8_t queue_id5;
	- /* ID of CoS Queue 5. FF - Invalid id */
	+ /*
	+ * ID of CoS Queue 5. FF - Invalid id # This ID can be used on any
	+ * subsequent call to an hwrm command that takes a queue id. # IDs must
	+ * always be queried by this command before any use by the driver or
	+ * software. # Any driver or software should not make any assumptions
	+ * about queue IDs. # A value of 0xff indicates that the queue is not
	+ * available. # Available queues may not be in sequential order.
	+ */
	uint8_t queue_id5_service_profile;
	/* This value is applicable to CoS queues only. */
	/* Lossy (best-effort) */
	@@ -8654,7 +11889,14 @@
	/* Set to 0xFF... (All Fs) if there is no service profile specified */
	#define HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID5_SERVICE_PROFILE_UNKNOWN UINT32_C(0xff)
	uint8_t queue_id6;
	- /* ID of CoS Queue 6. FF - Invalid id */
	+ /*
	+ * ID of CoS Queue 6. FF - Invalid id # This ID can be used on any
	+ * subsequent call to an hwrm command that takes a queue id. # IDs must
	+ * always be queried by this command before any use by the driver or
	+ * software. # Any driver or software should not make any assumptions
	+ * about queue IDs. # A value of 0xff indicates that the queue is not
	+ * available. # Available queues may not be in sequential order.
	+ */
	uint8_t queue_id6_service_profile;
	/* This value is applicable to CoS queues only. */
	/* Lossy (best-effort) */
	@@ -8664,7 +11906,14 @@
	/* Set to 0xFF... (All Fs) if there is no service profile specified */
	#define HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID6_SERVICE_PROFILE_UNKNOWN UINT32_C(0xff)
	uint8_t queue_id7;
	- /* ID of CoS Queue 7. FF - Invalid id */
	+ /*
	+ * ID of CoS Queue 7. FF - Invalid id # This ID can be used on any
	+ * subsequent call to an hwrm command that takes a queue id. # IDs must
	+ * always be queried by this command before any use by the driver or
	+ * software. # Any driver or software should not make any assumptions
	+ * about queue IDs. # A value of 0xff indicates that the queue is not
	+ * available. # Available queues may not be in sequential order.
	+ */
	uint8_t queue_id7_service_profile;
	/* This value is applicable to CoS queues only. */
	/* Lossy (best-effort) */
	@@ -8760,8 +12009,15 @@
	#define HWRM_QUEUE_QCFG_OUTPUT_SERVICE_PROFILE_LOSSLESS UINT32_C(0x1)
	/* Set to 0xFF... (All Fs) if there is no service profile specified */
	#define HWRM_QUEUE_QCFG_OUTPUT_SERVICE_PROFILE_UNKNOWN UINT32_C(0xff)
	+ uint8_t queue_cfg_info;
	+ /* Information about queue configuration. */
	+ /*
	+ * If this flag is set to '1', then the queue is configured
	+ * asymmetrically on TX and RX sides. If this flag is set to '0', then
	+ * this queue is configured symmetrically on TX and RX sides.
	+ */
	+ #define HWRM_QUEUE_QCFG_OUTPUT_QUEUE_CFG_INFO_ASYM_CFG UINT32_C(0x1)
	uint8_t unused_0;
	- uint8_t unused_1;
	uint8_t valid;
	/*
	* This field is used in Output records to indicate that the output is
	@@ -8804,16 +12060,19 @@
	*/
	uint32_t flags;
	/*
	- * Enumeration denoting the RX, TX type of the resource. This
	- * enumeration is used for resources that are similar for both TX and RX
	- * paths of the chip.
	+ * Enumeration denoting the RX, TX, or both directions applicable to the
	+ * resource. This enumeration is used for resources that are similar for
	+ * both TX and RX paths of the chip.
	*/
	- #define HWRM_QUEUE_CFG_INPUT_FLAGS_PATH UINT32_C(0x1)
	+ #define HWRM_QUEUE_CFG_INPUT_FLAGS_PATH_MASK UINT32_C(0x3)
	+ #define HWRM_QUEUE_CFG_INPUT_FLAGS_PATH_SFT 0
	/* tx path */
	#define HWRM_QUEUE_CFG_INPUT_FLAGS_PATH_TX UINT32_C(0x0)
	/* rx path */
	#define HWRM_QUEUE_CFG_INPUT_FLAGS_PATH_RX UINT32_C(0x1)
	- #define HWRM_QUEUE_CFG_INPUT_FLAGS_PATH_LAST HWRM_QUEUE_CFG_INPUT_FLAGS_PATH_RX
	+ /* Bi-directional (Symmetrically applicable to TX and RX paths) */
	+ #define HWRM_QUEUE_CFG_INPUT_FLAGS_PATH_BIDIR UINT32_C(0x2)
	+ #define HWRM_QUEUE_CFG_INPUT_FLAGS_PATH_LAST HWRM_QUEUE_CFG_INPUT_FLAGS_PATH_BIDIR
	uint32_t enables;
	/* This bit must be '1' for the dflt_len field to be configured. */
	#define HWRM_QUEUE_CFG_INPUT_ENABLES_DFLT_LEN UINT32_C(0x1)
	@@ -8869,258 +12128,6 @@
	*/
	} __attribute__((packed));

	-/* hwrm_queue_buffers_qcfg */
	-/*
	- * Description: This function is called by a driver to query configuration of
	- * the buffers assigned to a queue.
	- */
	-/* Input (24 bytes) */
	-
	-struct hwrm_queue_buffers_qcfg_input {
	- uint16_t req_type;
	- /*
	- * This value indicates what type of request this is. The format for the
	- * rest of the command is determined by this field.
	- */
	- uint16_t cmpl_ring;
	- /*
	- * This value indicates the what completion ring the request will be
	- * optionally completed on. If the value is -1, then no CR completion
	- * will be generated. Any other value must be a valid CR ring_id value
	- * for this function.
	- */
	- uint16_t seq_id;
	- /* This value indicates the command sequence number. */
	- uint16_t target_id;
	- /*
	- * Target ID of this command. 0x0 - 0xFFF8 - Used for function ids
	- * 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF - HWRM
	- */
	- uint64_t resp_addr;
	- /*
	- * This is the host address where the response will be written when the
	- * request is complete. This area must be 16B aligned and must be
	- * cleared to zero before the request is made.
	- */
	- uint32_t flags;
	- /*
	- * Enumeration denoting the RX, TX type of the resource. This
	- * enumeration is used for resources that are similar for both TX and RX
	- * paths of the chip.
	- */
	- #define HWRM_QUEUE_BUFFERS_QCFG_INPUT_FLAGS_PATH UINT32_C(0x1)
	- /* tx path */
	- #define HWRM_QUEUE_BUFFERS_QCFG_INPUT_FLAGS_PATH_TX UINT32_C(0x0)
	- /* rx path */
	- #define HWRM_QUEUE_BUFFERS_QCFG_INPUT_FLAGS_PATH_RX UINT32_C(0x1)
	- #define HWRM_QUEUE_BUFFERS_QCFG_INPUT_FLAGS_PATH_LAST HWRM_QUEUE_BUFFERS_QCFG_INPUT_FLAGS_PATH_RX
	- uint32_t queue_id;
	- /* Queue ID of queue that is to be configured by this function. */
	-} __attribute__((packed));
	-
	-/* Output (40 bytes) */
	-
	-struct hwrm_queue_buffers_qcfg_output {
	- uint16_t error_code;
	- /*
	- * Pass/Fail or error type Note: receiver to verify the in parameters,
	- * and fail the call with an error when appropriate
	- */
	- uint16_t req_type;
	- /* This field returns the type of original request. */
	- uint16_t seq_id;
	- /* This field provides original sequence number of the command. */
	- uint16_t resp_len;
	- /*
	- * This field is the length of the response in bytes. The last byte of
	- * the response is a valid flag that will read as '1' when the command
	- * has been completely written to memory.
	- */
	- uint32_t reserved;
	- /* Number of bytes allocated as reserved space for this queue. */
	- uint32_t shared;
	- /*
	- * Number of bytes of shared buffer space for this queue. The changing
	- * of shared buffer size for one CoS may create an adverse effect on
	- * other CoSs sharing the same buffer. It is recommended that the driver
	- * does not modify the shared mbuf size without understanding the
	- * consequence of it.
	- */
	- uint32_t xoff;
	- /*
	- * XOFF threshold of the queue. This is a high threshold value used to
	- * trigger XOFF.
	- */
	- uint32_t xon;
	- /*
	- * XON threshold of the queue. This is the low threshold value used to
	- * trigger XON.
	- */
	- uint32_t full;
	- /*
	- * FULL threshold of the queue. At this threshold, buffers allocated for
	- * this queue are full. Once this condition is asserted, packets on that
	- * queue are dropped.
	- */
	- uint32_t notfull;
	- /*
	- * NOTFULL threshold of the queue. This threshold is used for the de-
	- * assertion of buffers full condition.
	- */
	- uint32_t max;
	- /*
	- * The maximum number of bytes that will be allowed to be consumed by
	- * the queue. This value is the sum of both the number of bytes reserved
	- * for this queue and the maximum number of bytes of shared buffers
	- * allowed to be consumed by this queue.
	- */
	- uint8_t unused_0;
	- uint8_t unused_1;
	- uint8_t unused_2;
	- uint8_t valid;
	- /*
	- * This field is used in Output records to indicate that the output is
	- * completely written to RAM. This field should be read as '1' to
	- * indicate that the output has been completely written. When writing a
	- * command completion or response to an internal processor, the order of
	- * writes has to be such that this field is written last.
	- */
	-} __attribute__((packed));
	-
	-/* hwrm_queue_buffers_cfg */
	-/*
	- * Description: This function is called by a driver to configure the buffering
	- * for a queue.
	- */
	-/* Input (56 bytes) */
	-
	-struct hwrm_queue_buffers_cfg_input {
	- uint16_t req_type;
	- /*
	- * This value indicates what type of request this is. The format for the
	- * rest of the command is determined by this field.
	- */
	- uint16_t cmpl_ring;
	- /*
	- * This value indicates the what completion ring the request will be
	- * optionally completed on. If the value is -1, then no CR completion
	- * will be generated. Any other value must be a valid CR ring_id value
	- * for this function.
	- */
	- uint16_t seq_id;
	- /* This value indicates the command sequence number. */
	- uint16_t target_id;
	- /*
	- * Target ID of this command. 0x0 - 0xFFF8 - Used for function ids
	- * 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF - HWRM
	- */
	- uint64_t resp_addr;
	- /*
	- * This is the host address where the response will be written when the
	- * request is complete. This area must be 16B aligned and must be
	- * cleared to zero before the request is made.
	- */
	- uint32_t flags;
	- /*
	- * Enumeration denoting the RX, TX type of the resource. This
	- * enumeration is used for resources that are similar for both TX and RX
	- * paths of the chip.
	- */
	- #define HWRM_QUEUE_BUFFERS_CFG_INPUT_FLAGS_PATH UINT32_C(0x1)
	- /* tx path */
	- #define HWRM_QUEUE_BUFFERS_CFG_INPUT_FLAGS_PATH_TX UINT32_C(0x0)
	- /* rx path */
	- #define HWRM_QUEUE_BUFFERS_CFG_INPUT_FLAGS_PATH_RX UINT32_C(0x1)
	- #define HWRM_QUEUE_BUFFERS_CFG_INPUT_FLAGS_PATH_LAST HWRM_QUEUE_BUFFERS_CFG_INPUT_FLAGS_PATH_RX
	- uint32_t enables;
	- /* This bit must be '1' for the reserved field to be configured. */
	- #define HWRM_QUEUE_BUFFERS_CFG_INPUT_ENABLES_RESERVED UINT32_C(0x1)
	- /* This bit must be '1' for the shared field to be configured. */
	- #define HWRM_QUEUE_BUFFERS_CFG_INPUT_ENABLES_SHARED UINT32_C(0x2)
	- /* This bit must be '1' for the xoff field to be configured. */
	- #define HWRM_QUEUE_BUFFERS_CFG_INPUT_ENABLES_XOFF UINT32_C(0x4)
	- /* This bit must be '1' for the xon field to be configured. */
	- #define HWRM_QUEUE_BUFFERS_CFG_INPUT_ENABLES_XON UINT32_C(0x8)
	- /* This bit must be '1' for the full field to be configured. */
	- #define HWRM_QUEUE_BUFFERS_CFG_INPUT_ENABLES_FULL UINT32_C(0x10)
	- /* This bit must be '1' for the notfull field to be configured. */
	- #define HWRM_QUEUE_BUFFERS_CFG_INPUT_ENABLES_NOTFULL UINT32_C(0x20)
	- /* This bit must be '1' for the max field to be configured. */
	- #define HWRM_QUEUE_BUFFERS_CFG_INPUT_ENABLES_MAX UINT32_C(0x40)
	- uint32_t queue_id;
	- /* Queue ID of queue that is to be configured by this function. */
	- uint32_t reserved;
	- /* Number of bytes to be allocated as reserved space for this queue. */
	- uint32_t shared;
	- /*
	- * Number of bytes of shared buffer space for this queue. The changing
	- * of shared buffer size for one CoS may create an adverse effect on
	- * other CoSs sharing the same buffer. It is recommended that the driver
	- * does not modify the shared mbuf size without understanding the
	- * consequence of it.
	- */
	- uint32_t xoff;
	- /*
	- * XOFF threshold of the queue. This is a high threshold value used to
	- * trigger XOFF.
	- */
	- uint32_t xon;
	- /*
	- * XON threshold of the queue. This is the low threshold value used to
	- * trigger XON.
	- */
	- uint32_t full;
	- /*
	- * FULL threshold of the queue. At this threshold, buffers allocated for
	- * this queue are full. Once this condition is asserted, packets on that
	- * queue are dropped.
	- */
	- uint32_t notfull;
	- /*
	- * NOTFULL threshold of the queue. This threshold is used for the de-
	- * assertion of buffers full condition.
	- */
	- uint32_t max;
	- /*
	- * The maximum number of bytes that will be allowed to be consumed by
	- * the queue. This value is the sum of both the number of bytes reserved
	- * for this queue and the maximum number of bytes of shared buffers
	- * allowed to be consumed by this queue.
	- */
	-} __attribute__((packed));
	-
	-/* Output (16 bytes) */
	-
	-struct hwrm_queue_buffers_cfg_output {
	- uint16_t error_code;
	- /*
	- * Pass/Fail or error type Note: receiver to verify the in parameters,
	- * and fail the call with an error when appropriate
	- */
	- uint16_t req_type;
	- /* This field returns the type of original request. */
	- uint16_t seq_id;
	- /* This field provides original sequence number of the command. */
	- uint16_t resp_len;
	- /*
	- * This field is the length of the response in bytes. The last byte of
	- * the response is a valid flag that will read as '1' when the command
	- * has been completely written to memory.
	- */
	- uint32_t unused_0;
	- uint8_t unused_1;
	- uint8_t unused_2;
	- uint8_t unused_3;
	- uint8_t valid;
	- /*
	- * This field is used in Output records to indicate that the output is
	- * completely written to RAM. This field should be read as '1' to
	- * indicate that the output has been completely written. When writing a
	- * command completion or response to an internal processor, the order of
	- * writes has to be such that this field is written last.
	- */
	-} __attribute__((packed));
	-
	/* hwrm_queue_pfcenable_qcfg */
	/*
	* Description: This function is called by a driver to query PFC configuration
	@@ -9353,8 +12360,9 @@
	#define HWRM_QUEUE_PRI2COS_QCFG_INPUT_FLAGS_PATH_RX (UINT32_C(0x1) << 0)
	#define HWRM_QUEUE_PRI2COS_QCFG_INPUT_FLAGS_PATH_LAST HWRM_QUEUE_PRI2COS_QCFG_INPUT_FLAGS_PATH_RX
	/*
	- * When this bit is set to '1', the mapping is requested for inner VLAN
	- * PRI.
	+ * When this bit is set to '0', the query is for VLAN PRI field in
	+ * tunnel headers. When this bit is set to '1', the query is for VLAN
	+ * PRI field in inner packet headers.
	*/
	#define HWRM_QUEUE_PRI2COS_QCFG_INPUT_FLAGS_IVLAN UINT32_C(0x2)
	uint8_t port_id;
	@@ -9387,47 +12395,64 @@
	uint8_t pri0_cos_queue_id;
	/*
	* CoS Queue assigned to priority 0. This value can only be changed
	- * before traffic has started.
	+ * before traffic has started. A value of 0xff indicates that no CoS
	+ * queue is assigned to the specified priority.
	*/
	uint8_t pri1_cos_queue_id;
	/*
	* CoS Queue assigned to priority 1. This value can only be changed
	- * before traffic has started.
	+ * before traffic has started. A value of 0xff indicates that no CoS
	+ * queue is assigned to the specified priority.
	*/
	uint8_t pri2_cos_queue_id;
	/*
	* CoS Queue assigned to priority 2 This value can only be changed
	- * before traffic has started.
	+ * before traffic has started. A value of 0xff indicates that no CoS
	+ * queue is assigned to the specified priority.
	*/
	uint8_t pri3_cos_queue_id;
	/*
	* CoS Queue assigned to priority 3. This value can only be changed
	- * before traffic has started.
	+ * before traffic has started. A value of 0xff indicates that no CoS
	+ * queue is assigned to the specified priority.
	*/
	uint8_t pri4_cos_queue_id;
	/*
	* CoS Queue assigned to priority 4. This value can only be changed
	- * before traffic has started.
	+ * before traffic has started. A value of 0xff indicates that no CoS
	+ * queue is assigned to the specified priority.
	*/
	uint8_t pri5_cos_queue_id;
	/*
	* CoS Queue assigned to priority 5. This value can only be changed
	- * before traffic has started.
	+ * before traffic has started. A value of 0xff indicates that no CoS
	+ * queue is assigned to the specified priority.
	*/
	uint8_t pri6_cos_queue_id;
	/*
	* CoS Queue assigned to priority 6. This value can only be changed
	- * before traffic has started.
	+ * before traffic has started. A value of 0xff indicates that no CoS
	+ * queue is assigned to the specified priority.
	*/
	uint8_t pri7_cos_queue_id;
	/*
	* CoS Queue assigned to priority 7. This value can only be changed
	- * before traffic has started.
	+ * before traffic has started. A value of 0xff indicates that no CoS
	+ * queue is assigned to the specified priority.
	*/
	- uint32_t unused_0;
	- uint8_t unused_1;
	+ uint8_t queue_cfg_info;
	+ /* Information about queue configuration. */
	+ /*
	+ * If this flag is set to '1', then the PRI to CoS configuration is
	+ * asymmetric on TX and RX sides. If this flag is set to '0', then PRI
	+ * to CoS configuration is symmetric on TX and RX sides.
	+ */
	+ #define HWRM_QUEUE_PRI2COS_QCFG_OUTPUT_QUEUE_CFG_INFO_ASYM_CFG UINT32_C(0x1)
	+ uint8_t unused_0;
	+ uint16_t unused_1;
	uint8_t unused_2;
	uint8_t unused_3;
	+ uint8_t unused_4;
	uint8_t valid;
	/*
	* This field is used in Output records to indicate that the output is
	@@ -9477,19 +12502,66 @@
	*/
	uint32_t flags;
	/*
	- * Enumeration denoting the RX, TX type of the resource. This
	- * enumeration is used for resources that are similar for both TX and RX
	- * paths of the chip.
	+ * Enumeration denoting the RX, TX, or both directions applicable to the
	+ * resource. This enumeration is used for resources that are similar for
	+ * both TX and RX paths of the chip.
	*/
	- #define HWRM_QUEUE_PRI2COS_CFG_INPUT_FLAGS_PATH UINT32_C(0x1)
	+ #define HWRM_QUEUE_PRI2COS_CFG_INPUT_FLAGS_PATH_MASK UINT32_C(0x3)
	+ #define HWRM_QUEUE_PRI2COS_CFG_INPUT_FLAGS_PATH_SFT 0
	/* tx path */
	#define HWRM_QUEUE_PRI2COS_CFG_INPUT_FLAGS_PATH_TX (UINT32_C(0x0) << 0)
	/* rx path */
	#define HWRM_QUEUE_PRI2COS_CFG_INPUT_FLAGS_PATH_RX (UINT32_C(0x1) << 0)
	- #define HWRM_QUEUE_PRI2COS_CFG_INPUT_FLAGS_PATH_LAST HWRM_QUEUE_PRI2COS_CFG_INPUT_FLAGS_PATH_RX
	- /* When this bit is '1', the mapping is for inner VLAN PRI. */
	- #define HWRM_QUEUE_PRI2COS_CFG_INPUT_FLAGS_IVLAN UINT32_C(0x2)
	+ /* Bi-directional (Symmetrically applicable to TX and RX paths) */
	+ #define HWRM_QUEUE_PRI2COS_CFG_INPUT_FLAGS_PATH_BIDIR (UINT32_C(0x2) << 0)
	+ #define HWRM_QUEUE_PRI2COS_CFG_INPUT_FLAGS_PATH_LAST HWRM_QUEUE_PRI2COS_CFG_INPUT_FLAGS_PATH_BIDIR
	+ /*
	+ * When this bit is set to '0', the mapping is requested for VLAN PRI
	+ * field in tunnel headers. When this bit is set to '1', the mapping is
	+ * requested for VLAN PRI field in inner packet headers.
	+ */
	+ #define HWRM_QUEUE_PRI2COS_CFG_INPUT_FLAGS_IVLAN UINT32_C(0x4)
	uint32_t enables;
	+ /*
	+ * This bit must be '1' for the pri0_cos_queue_id field to be
	+ * configured.
	+ */
	+ #define HWRM_QUEUE_PRI2COS_CFG_INPUT_ENABLES_PRI0_COS_QUEUE_ID UINT32_C(0x1)
	+ /*
	+ * This bit must be '1' for the pri1_cos_queue_id field to be
	+ * configured.
	+ */
	+ #define HWRM_QUEUE_PRI2COS_CFG_INPUT_ENABLES_PRI1_COS_QUEUE_ID UINT32_C(0x2)
	+ /*
	+ * This bit must be '1' for the pri2_cos_queue_id field to be
	+ * configured.
	+ */
	+ #define HWRM_QUEUE_PRI2COS_CFG_INPUT_ENABLES_PRI2_COS_QUEUE_ID UINT32_C(0x4)
	+ /*
	+ * This bit must be '1' for the pri3_cos_queue_id field to be
	+ * configured.
	+ */
	+ #define HWRM_QUEUE_PRI2COS_CFG_INPUT_ENABLES_PRI3_COS_QUEUE_ID UINT32_C(0x8)
	+ /*
	+ * This bit must be '1' for the pri4_cos_queue_id field to be
	+ * configured.
	+ */
	+ #define HWRM_QUEUE_PRI2COS_CFG_INPUT_ENABLES_PRI4_COS_QUEUE_ID UINT32_C(0x10)
	+ /*
	+ * This bit must be '1' for the pri5_cos_queue_id field to be
	+ * configured.
	+ */
	+ #define HWRM_QUEUE_PRI2COS_CFG_INPUT_ENABLES_PRI5_COS_QUEUE_ID UINT32_C(0x20)
	+ /*
	+ * This bit must be '1' for the pri6_cos_queue_id field to be
	+ * configured.
	+ */
	+ #define HWRM_QUEUE_PRI2COS_CFG_INPUT_ENABLES_PRI6_COS_QUEUE_ID UINT32_C(0x40)
	+ /*
	+ * This bit must be '1' for the pri7_cos_queue_id field to be
	+ * configured.
	+ */
	+ #define HWRM_QUEUE_PRI2COS_CFG_INPUT_ENABLES_PRI7_COS_QUEUE_ID UINT32_C(0x80)
	uint8_t port_id;
	/*
	* Port ID of port for which the table is being configured. The HWRM
	@@ -9680,16 +12752,68 @@
	uint16_t unused_1;
	uint32_t queue_id0_min_bw;
	/*
	- * Minimum BW allocated to CoS Queue in Mbps. The HWRM will translate
	- * this value into byte counter and time interval used for this COS
	- * inside the device.
	+ * Minimum BW allocated to CoS Queue. The HWRM will translate this value
	+ * into byte counter and time interval used for this COS inside the
	+ * device.
	*/
	+ /* The bandwidth value. */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID0_MIN_BW_BW_VALUE_MASK UINT32_C(0xfffffff)
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID0_MIN_BW_BW_VALUE_SFT 0
	+ /* The granularity of the value (bits or bytes). */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID0_MIN_BW_SCALE UINT32_C(0x10000000)
	+ /* Value is in bits. */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID0_MIN_BW_SCALE_BITS (UINT32_C(0x0) << 28)
	+ /* Value is in bytes. */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID0_MIN_BW_SCALE_BYTES (UINT32_C(0x1) << 28)
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID0_MIN_BW_SCALE_LAST HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID0_MIN_BW_SCALE_BYTES
	+ /* bw_value_unit is 3 b */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID0_MIN_BW_BW_VALUE_UNIT_MASK UINT32_C(0xe0000000)
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID0_MIN_BW_BW_VALUE_UNIT_SFT 29
	+ /* Value is in Mb or MB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID0_MIN_BW_BW_VALUE_UNIT_MEGA (UINT32_C(0x0) << 29)
	+ /* Value is in Kb or KB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID0_MIN_BW_BW_VALUE_UNIT_KILO (UINT32_C(0x2) << 29)
	+ /* Value is in bits or bytes. */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID0_MIN_BW_BW_VALUE_UNIT_BASE (UINT32_C(0x4) << 29)
	+ /* Value is in Gb or GB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID0_MIN_BW_BW_VALUE_UNIT_GIGA (UINT32_C(0x6) << 29)
	+ /* Value is in 1/100th of a percentage of total bandwidth. */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID0_MIN_BW_BW_VALUE_UNIT_PERCENT1_100 (UINT32_C(0x1) << 29)
	+ /* Invalid unit */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID0_MIN_BW_BW_VALUE_UNIT_INVALID (UINT32_C(0x7) << 29)
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID0_MIN_BW_BW_VALUE_UNIT_LAST HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID0_MIN_BW_BW_VALUE_UNIT_INVALID
	uint32_t queue_id0_max_bw;
	/*
	- * Maximum BW allocated to CoS Queue in Mbps. The HWRM will translate
	- * this value into byte counter and time interval used for this COS
	- * inside the device.
	+ * Maximum BW allocated to CoS Queue. The HWRM will translate this value
	+ * into byte counter and time interval used for this COS inside the
	+ * device.
	*/
	+ /* The bandwidth value. */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID0_MAX_BW_BW_VALUE_MASK UINT32_C(0xfffffff)
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID0_MAX_BW_BW_VALUE_SFT 0
	+ /* The granularity of the value (bits or bytes). */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID0_MAX_BW_SCALE UINT32_C(0x10000000)
	+ /* Value is in bits. */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID0_MAX_BW_SCALE_BITS (UINT32_C(0x0) << 28)
	+ /* Value is in bytes. */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID0_MAX_BW_SCALE_BYTES (UINT32_C(0x1) << 28)
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID0_MAX_BW_SCALE_LAST HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID0_MAX_BW_SCALE_BYTES
	+ /* bw_value_unit is 3 b */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID0_MAX_BW_BW_VALUE_UNIT_MASK UINT32_C(0xe0000000)
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID0_MAX_BW_BW_VALUE_UNIT_SFT 29
	+ /* Value is in Mb or MB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID0_MAX_BW_BW_VALUE_UNIT_MEGA (UINT32_C(0x0) << 29)
	+ /* Value is in Kb or KB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID0_MAX_BW_BW_VALUE_UNIT_KILO (UINT32_C(0x2) << 29)
	+ /* Value is in bits or bytes. */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID0_MAX_BW_BW_VALUE_UNIT_BASE (UINT32_C(0x4) << 29)
	+ /* Value is in Gb or GB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID0_MAX_BW_BW_VALUE_UNIT_GIGA (UINT32_C(0x6) << 29)
	+ /* Value is in 1/100th of a percentage of total bandwidth. */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID0_MAX_BW_BW_VALUE_UNIT_PERCENT1_100 (UINT32_C(0x1) << 29)
	+ /* Invalid unit */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID0_MAX_BW_BW_VALUE_UNIT_INVALID (UINT32_C(0x7) << 29)
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID0_MAX_BW_BW_VALUE_UNIT_LAST HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID0_MAX_BW_BW_VALUE_UNIT_INVALID
	uint8_t queue_id0_tsa_assign;
	/* Transmission Selection Algorithm (TSA) for CoS Queue. */
	/* Strict Priority */
	@@ -9714,16 +12838,68 @@
	/* ID of CoS Queue 1. */
	uint32_t queue_id1_min_bw;
	/*
	- * Minimum BW allocated to CoS Queue in Mbps. The HWRM will translate
	- * this value into byte counter and time interval used for this COS
	- * inside the device.
	+ * Minimum BW allocated to CoS Queue. The HWRM will translate this value
	+ * into byte counter and time interval used for this COS inside the
	+ * device.
	*/
	+ /* The bandwidth value. */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID1_MIN_BW_BW_VALUE_MASK UINT32_C(0xfffffff)
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID1_MIN_BW_BW_VALUE_SFT 0
	+ /* The granularity of the value (bits or bytes). */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID1_MIN_BW_SCALE UINT32_C(0x10000000)
	+ /* Value is in bits. */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID1_MIN_BW_SCALE_BITS (UINT32_C(0x0) << 28)
	+ /* Value is in bytes. */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID1_MIN_BW_SCALE_BYTES (UINT32_C(0x1) << 28)
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID1_MIN_BW_SCALE_LAST HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID1_MIN_BW_SCALE_BYTES
	+ /* bw_value_unit is 3 b */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID1_MIN_BW_BW_VALUE_UNIT_MASK UINT32_C(0xe0000000)
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID1_MIN_BW_BW_VALUE_UNIT_SFT 29
	+ /* Value is in Mb or MB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID1_MIN_BW_BW_VALUE_UNIT_MEGA (UINT32_C(0x0) << 29)
	+ /* Value is in Kb or KB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID1_MIN_BW_BW_VALUE_UNIT_KILO (UINT32_C(0x2) << 29)
	+ /* Value is in bits or bytes. */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID1_MIN_BW_BW_VALUE_UNIT_BASE (UINT32_C(0x4) << 29)
	+ /* Value is in Gb or GB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID1_MIN_BW_BW_VALUE_UNIT_GIGA (UINT32_C(0x6) << 29)
	+ /* Value is in 1/100th of a percentage of total bandwidth. */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID1_MIN_BW_BW_VALUE_UNIT_PERCENT1_100 (UINT32_C(0x1) << 29)
	+ /* Invalid unit */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID1_MIN_BW_BW_VALUE_UNIT_INVALID (UINT32_C(0x7) << 29)
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID1_MIN_BW_BW_VALUE_UNIT_LAST HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID1_MIN_BW_BW_VALUE_UNIT_INVALID
	uint32_t queue_id1_max_bw;
	/*
	- * Maximum BW allocated to CoS queue in Mbps. The HWRM will translate
	- * this value into byte counter and time interval used for this COS
	- * inside the device.
	+ * Maximum BW allocated to CoS queue. The HWRM will translate this value
	+ * into byte counter and time interval used for this COS inside the
	+ * device.
	*/
	+ /* The bandwidth value. */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID1_MAX_BW_BW_VALUE_MASK UINT32_C(0xfffffff)
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID1_MAX_BW_BW_VALUE_SFT 0
	+ /* The granularity of the value (bits or bytes). */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID1_MAX_BW_SCALE UINT32_C(0x10000000)
	+ /* Value is in bits. */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID1_MAX_BW_SCALE_BITS (UINT32_C(0x0) << 28)
	+ /* Value is in bytes. */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID1_MAX_BW_SCALE_BYTES (UINT32_C(0x1) << 28)
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID1_MAX_BW_SCALE_LAST HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID1_MAX_BW_SCALE_BYTES
	+ /* bw_value_unit is 3 b */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID1_MAX_BW_BW_VALUE_UNIT_MASK UINT32_C(0xe0000000)
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID1_MAX_BW_BW_VALUE_UNIT_SFT 29
	+ /* Value is in Mb or MB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID1_MAX_BW_BW_VALUE_UNIT_MEGA (UINT32_C(0x0) << 29)
	+ /* Value is in Kb or KB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID1_MAX_BW_BW_VALUE_UNIT_KILO (UINT32_C(0x2) << 29)
	+ /* Value is in bits or bytes. */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID1_MAX_BW_BW_VALUE_UNIT_BASE (UINT32_C(0x4) << 29)
	+ /* Value is in Gb or GB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID1_MAX_BW_BW_VALUE_UNIT_GIGA (UINT32_C(0x6) << 29)
	+ /* Value is in 1/100th of a percentage of total bandwidth. */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID1_MAX_BW_BW_VALUE_UNIT_PERCENT1_100 (UINT32_C(0x1) << 29)
	+ /* Invalid unit */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID1_MAX_BW_BW_VALUE_UNIT_INVALID (UINT32_C(0x7) << 29)
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID1_MAX_BW_BW_VALUE_UNIT_LAST HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID1_MAX_BW_BW_VALUE_UNIT_INVALID
	uint8_t queue_id1_tsa_assign;
	/* Transmission Selection Algorithm (TSA) for CoS Queue. */
	/* Strict Priority */
	@@ -9748,16 +12924,68 @@
	/* ID of CoS Queue 2. */
	uint32_t queue_id2_min_bw;
	/*
	- * Minimum BW allocated to CoS Queue in Mbps. The HWRM will translate
	- * this value into byte counter and time interval used for this COS
	- * inside the device.
	+ * Minimum BW allocated to CoS Queue. The HWRM will translate this value
	+ * into byte counter and time interval used for this COS inside the
	+ * device.
	*/
	+ /* The bandwidth value. */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID2_MIN_BW_BW_VALUE_MASK UINT32_C(0xfffffff)
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID2_MIN_BW_BW_VALUE_SFT 0
	+ /* The granularity of the value (bits or bytes). */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID2_MIN_BW_SCALE UINT32_C(0x10000000)
	+ /* Value is in bits. */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID2_MIN_BW_SCALE_BITS (UINT32_C(0x0) << 28)
	+ /* Value is in bytes. */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID2_MIN_BW_SCALE_BYTES (UINT32_C(0x1) << 28)
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID2_MIN_BW_SCALE_LAST HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID2_MIN_BW_SCALE_BYTES
	+ /* bw_value_unit is 3 b */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID2_MIN_BW_BW_VALUE_UNIT_MASK UINT32_C(0xe0000000)
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID2_MIN_BW_BW_VALUE_UNIT_SFT 29
	+ /* Value is in Mb or MB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID2_MIN_BW_BW_VALUE_UNIT_MEGA (UINT32_C(0x0) << 29)
	+ /* Value is in Kb or KB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID2_MIN_BW_BW_VALUE_UNIT_KILO (UINT32_C(0x2) << 29)
	+ /* Value is in bits or bytes. */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID2_MIN_BW_BW_VALUE_UNIT_BASE (UINT32_C(0x4) << 29)
	+ /* Value is in Gb or GB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID2_MIN_BW_BW_VALUE_UNIT_GIGA (UINT32_C(0x6) << 29)
	+ /* Value is in 1/100th of a percentage of total bandwidth. */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID2_MIN_BW_BW_VALUE_UNIT_PERCENT1_100 (UINT32_C(0x1) << 29)
	+ /* Invalid unit */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID2_MIN_BW_BW_VALUE_UNIT_INVALID (UINT32_C(0x7) << 29)
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID2_MIN_BW_BW_VALUE_UNIT_LAST HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID2_MIN_BW_BW_VALUE_UNIT_INVALID
	uint32_t queue_id2_max_bw;
	/*
	- * Maximum BW allocated to CoS queue in Mbps. The HWRM will translate
	- * this value into byte counter and time interval used for this COS
	- * inside the device.
	+ * Maximum BW allocated to CoS queue. The HWRM will translate this value
	+ * into byte counter and time interval used for this COS inside the
	+ * device.
	*/
	+ /* The bandwidth value. */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID2_MAX_BW_BW_VALUE_MASK UINT32_C(0xfffffff)
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID2_MAX_BW_BW_VALUE_SFT 0
	+ /* The granularity of the value (bits or bytes). */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID2_MAX_BW_SCALE UINT32_C(0x10000000)
	+ /* Value is in bits. */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID2_MAX_BW_SCALE_BITS (UINT32_C(0x0) << 28)
	+ /* Value is in bytes. */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID2_MAX_BW_SCALE_BYTES (UINT32_C(0x1) << 28)
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID2_MAX_BW_SCALE_LAST HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID2_MAX_BW_SCALE_BYTES
	+ /* bw_value_unit is 3 b */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID2_MAX_BW_BW_VALUE_UNIT_MASK UINT32_C(0xe0000000)
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID2_MAX_BW_BW_VALUE_UNIT_SFT 29
	+ /* Value is in Mb or MB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID2_MAX_BW_BW_VALUE_UNIT_MEGA (UINT32_C(0x0) << 29)
	+ /* Value is in Kb or KB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID2_MAX_BW_BW_VALUE_UNIT_KILO (UINT32_C(0x2) << 29)
	+ /* Value is in bits or bytes. */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID2_MAX_BW_BW_VALUE_UNIT_BASE (UINT32_C(0x4) << 29)
	+ /* Value is in Gb or GB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID2_MAX_BW_BW_VALUE_UNIT_GIGA (UINT32_C(0x6) << 29)
	+ /* Value is in 1/100th of a percentage of total bandwidth. */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID2_MAX_BW_BW_VALUE_UNIT_PERCENT1_100 (UINT32_C(0x1) << 29)
	+ /* Invalid unit */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID2_MAX_BW_BW_VALUE_UNIT_INVALID (UINT32_C(0x7) << 29)
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID2_MAX_BW_BW_VALUE_UNIT_LAST HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID2_MAX_BW_BW_VALUE_UNIT_INVALID
	uint8_t queue_id2_tsa_assign;
	/* Transmission Selection Algorithm (TSA) for CoS Queue. */
	/* Strict Priority */
	@@ -9782,16 +13010,68 @@
	/* ID of CoS Queue 3. */
	uint32_t queue_id3_min_bw;
	/*
	- * Minimum BW allocated to CoS Queue in Mbps. The HWRM will translate
	- * this value into byte counter and time interval used for this COS
	- * inside the device.
	+ * Minimum BW allocated to CoS Queue. The HWRM will translate this value
	+ * into byte counter and time interval used for this COS inside the
	+ * device.
	*/
	+ /* The bandwidth value. */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID3_MIN_BW_BW_VALUE_MASK UINT32_C(0xfffffff)
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID3_MIN_BW_BW_VALUE_SFT 0
	+ /* The granularity of the value (bits or bytes). */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID3_MIN_BW_SCALE UINT32_C(0x10000000)
	+ /* Value is in bits. */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID3_MIN_BW_SCALE_BITS (UINT32_C(0x0) << 28)
	+ /* Value is in bytes. */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID3_MIN_BW_SCALE_BYTES (UINT32_C(0x1) << 28)
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID3_MIN_BW_SCALE_LAST HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID3_MIN_BW_SCALE_BYTES
	+ /* bw_value_unit is 3 b */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID3_MIN_BW_BW_VALUE_UNIT_MASK UINT32_C(0xe0000000)
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID3_MIN_BW_BW_VALUE_UNIT_SFT 29
	+ /* Value is in Mb or MB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID3_MIN_BW_BW_VALUE_UNIT_MEGA (UINT32_C(0x0) << 29)
	+ /* Value is in Kb or KB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID3_MIN_BW_BW_VALUE_UNIT_KILO (UINT32_C(0x2) << 29)
	+ /* Value is in bits or bytes. */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID3_MIN_BW_BW_VALUE_UNIT_BASE (UINT32_C(0x4) << 29)
	+ /* Value is in Gb or GB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID3_MIN_BW_BW_VALUE_UNIT_GIGA (UINT32_C(0x6) << 29)
	+ /* Value is in 1/100th of a percentage of total bandwidth. */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID3_MIN_BW_BW_VALUE_UNIT_PERCENT1_100 (UINT32_C(0x1) << 29)
	+ /* Invalid unit */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID3_MIN_BW_BW_VALUE_UNIT_INVALID (UINT32_C(0x7) << 29)
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID3_MIN_BW_BW_VALUE_UNIT_LAST HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID3_MIN_BW_BW_VALUE_UNIT_INVALID
	uint32_t queue_id3_max_bw;
	/*
	- * Maximum BW allocated to CoS queue in Mbps. The HWRM will translate
	- * this value into byte counter and time interval used for this COS
	- * inside the device.
	+ * Maximum BW allocated to CoS queue. The HWRM will translate this value
	+ * into byte counter and time interval used for this COS inside the
	+ * device.
	*/
	+ /* The bandwidth value. */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID3_MAX_BW_BW_VALUE_MASK UINT32_C(0xfffffff)
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID3_MAX_BW_BW_VALUE_SFT 0
	+ /* The granularity of the value (bits or bytes). */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID3_MAX_BW_SCALE UINT32_C(0x10000000)
	+ /* Value is in bits. */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID3_MAX_BW_SCALE_BITS (UINT32_C(0x0) << 28)
	+ /* Value is in bytes. */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID3_MAX_BW_SCALE_BYTES (UINT32_C(0x1) << 28)
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID3_MAX_BW_SCALE_LAST HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID3_MAX_BW_SCALE_BYTES
	+ /* bw_value_unit is 3 b */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID3_MAX_BW_BW_VALUE_UNIT_MASK UINT32_C(0xe0000000)
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID3_MAX_BW_BW_VALUE_UNIT_SFT 29
	+ /* Value is in Mb or MB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID3_MAX_BW_BW_VALUE_UNIT_MEGA (UINT32_C(0x0) << 29)
	+ /* Value is in Kb or KB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID3_MAX_BW_BW_VALUE_UNIT_KILO (UINT32_C(0x2) << 29)
	+ /* Value is in bits or bytes. */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID3_MAX_BW_BW_VALUE_UNIT_BASE (UINT32_C(0x4) << 29)
	+ /* Value is in Gb or GB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID3_MAX_BW_BW_VALUE_UNIT_GIGA (UINT32_C(0x6) << 29)
	+ /* Value is in 1/100th of a percentage of total bandwidth. */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID3_MAX_BW_BW_VALUE_UNIT_PERCENT1_100 (UINT32_C(0x1) << 29)
	+ /* Invalid unit */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID3_MAX_BW_BW_VALUE_UNIT_INVALID (UINT32_C(0x7) << 29)
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID3_MAX_BW_BW_VALUE_UNIT_LAST HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID3_MAX_BW_BW_VALUE_UNIT_INVALID
	uint8_t queue_id3_tsa_assign;
	/* Transmission Selection Algorithm (TSA) for CoS Queue. */
	/* Strict Priority */
	@@ -9816,16 +13096,68 @@
	/* ID of CoS Queue 4. */
	uint32_t queue_id4_min_bw;
	/*
	- * Minimum BW allocated to CoS Queue in Mbps. The HWRM will translate
	- * this value into byte counter and time interval used for this COS
	- * inside the device.
	+ * Minimum BW allocated to CoS Queue. The HWRM will translate this value
	+ * into byte counter and time interval used for this COS inside the
	+ * device.
	*/
	+ /* The bandwidth value. */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID4_MIN_BW_BW_VALUE_MASK UINT32_C(0xfffffff)
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID4_MIN_BW_BW_VALUE_SFT 0
	+ /* The granularity of the value (bits or bytes). */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID4_MIN_BW_SCALE UINT32_C(0x10000000)
	+ /* Value is in bits. */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID4_MIN_BW_SCALE_BITS (UINT32_C(0x0) << 28)
	+ /* Value is in bytes. */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID4_MIN_BW_SCALE_BYTES (UINT32_C(0x1) << 28)
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID4_MIN_BW_SCALE_LAST HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID4_MIN_BW_SCALE_BYTES
	+ /* bw_value_unit is 3 b */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID4_MIN_BW_BW_VALUE_UNIT_MASK UINT32_C(0xe0000000)
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID4_MIN_BW_BW_VALUE_UNIT_SFT 29
	+ /* Value is in Mb or MB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID4_MIN_BW_BW_VALUE_UNIT_MEGA (UINT32_C(0x0) << 29)
	+ /* Value is in Kb or KB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID4_MIN_BW_BW_VALUE_UNIT_KILO (UINT32_C(0x2) << 29)
	+ /* Value is in bits or bytes. */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID4_MIN_BW_BW_VALUE_UNIT_BASE (UINT32_C(0x4) << 29)
	+ /* Value is in Gb or GB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID4_MIN_BW_BW_VALUE_UNIT_GIGA (UINT32_C(0x6) << 29)
	+ /* Value is in 1/100th of a percentage of total bandwidth. */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID4_MIN_BW_BW_VALUE_UNIT_PERCENT1_100 (UINT32_C(0x1) << 29)
	+ /* Invalid unit */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID4_MIN_BW_BW_VALUE_UNIT_INVALID (UINT32_C(0x7) << 29)
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID4_MIN_BW_BW_VALUE_UNIT_LAST HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID4_MIN_BW_BW_VALUE_UNIT_INVALID
	uint32_t queue_id4_max_bw;
	/*
	- * Maximum BW allocated to CoS queue in Mbps. The HWRM will translate
	- * this value into byte counter and time interval used for this COS
	- * inside the device.
	+ * Maximum BW allocated to CoS queue. The HWRM will translate this value
	+ * into byte counter and time interval used for this COS inside the
	+ * device.
	*/
	+ /* The bandwidth value. */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID4_MAX_BW_BW_VALUE_MASK UINT32_C(0xfffffff)
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID4_MAX_BW_BW_VALUE_SFT 0
	+ /* The granularity of the value (bits or bytes). */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID4_MAX_BW_SCALE UINT32_C(0x10000000)
	+ /* Value is in bits. */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID4_MAX_BW_SCALE_BITS (UINT32_C(0x0) << 28)
	+ /* Value is in bytes. */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID4_MAX_BW_SCALE_BYTES (UINT32_C(0x1) << 28)
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID4_MAX_BW_SCALE_LAST HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID4_MAX_BW_SCALE_BYTES
	+ /* bw_value_unit is 3 b */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID4_MAX_BW_BW_VALUE_UNIT_MASK UINT32_C(0xe0000000)
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID4_MAX_BW_BW_VALUE_UNIT_SFT 29
	+ /* Value is in Mb or MB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID4_MAX_BW_BW_VALUE_UNIT_MEGA (UINT32_C(0x0) << 29)
	+ /* Value is in Kb or KB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID4_MAX_BW_BW_VALUE_UNIT_KILO (UINT32_C(0x2) << 29)
	+ /* Value is in bits or bytes. */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID4_MAX_BW_BW_VALUE_UNIT_BASE (UINT32_C(0x4) << 29)
	+ /* Value is in Gb or GB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID4_MAX_BW_BW_VALUE_UNIT_GIGA (UINT32_C(0x6) << 29)
	+ /* Value is in 1/100th of a percentage of total bandwidth. */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID4_MAX_BW_BW_VALUE_UNIT_PERCENT1_100 (UINT32_C(0x1) << 29)
	+ /* Invalid unit */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID4_MAX_BW_BW_VALUE_UNIT_INVALID (UINT32_C(0x7) << 29)
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID4_MAX_BW_BW_VALUE_UNIT_LAST HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID4_MAX_BW_BW_VALUE_UNIT_INVALID
	uint8_t queue_id4_tsa_assign;
	/* Transmission Selection Algorithm (TSA) for CoS Queue. */
	/* Strict Priority */
	@@ -9850,16 +13182,68 @@
	/* ID of CoS Queue 5. */
	uint32_t queue_id5_min_bw;
	/*
	- * Minimum BW allocated to CoS Queue in Mbps. The HWRM will translate
	- * this value into byte counter and time interval used for this COS
	- * inside the device.
	+ * Minimum BW allocated to CoS Queue. The HWRM will translate this value
	+ * into byte counter and time interval used for this COS inside the
	+ * device.
	*/
	+ /* The bandwidth value. */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID5_MIN_BW_BW_VALUE_MASK UINT32_C(0xfffffff)
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID5_MIN_BW_BW_VALUE_SFT 0
	+ /* The granularity of the value (bits or bytes). */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID5_MIN_BW_SCALE UINT32_C(0x10000000)
	+ /* Value is in bits. */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID5_MIN_BW_SCALE_BITS (UINT32_C(0x0) << 28)
	+ /* Value is in bytes. */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID5_MIN_BW_SCALE_BYTES (UINT32_C(0x1) << 28)
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID5_MIN_BW_SCALE_LAST HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID5_MIN_BW_SCALE_BYTES
	+ /* bw_value_unit is 3 b */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID5_MIN_BW_BW_VALUE_UNIT_MASK UINT32_C(0xe0000000)
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID5_MIN_BW_BW_VALUE_UNIT_SFT 29
	+ /* Value is in Mb or MB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID5_MIN_BW_BW_VALUE_UNIT_MEGA (UINT32_C(0x0) << 29)
	+ /* Value is in Kb or KB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID5_MIN_BW_BW_VALUE_UNIT_KILO (UINT32_C(0x2) << 29)
	+ /* Value is in bits or bytes. */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID5_MIN_BW_BW_VALUE_UNIT_BASE (UINT32_C(0x4) << 29)
	+ /* Value is in Gb or GB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID5_MIN_BW_BW_VALUE_UNIT_GIGA (UINT32_C(0x6) << 29)
	+ /* Value is in 1/100th of a percentage of total bandwidth. */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID5_MIN_BW_BW_VALUE_UNIT_PERCENT1_100 (UINT32_C(0x1) << 29)
	+ /* Invalid unit */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID5_MIN_BW_BW_VALUE_UNIT_INVALID (UINT32_C(0x7) << 29)
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID5_MIN_BW_BW_VALUE_UNIT_LAST HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID5_MIN_BW_BW_VALUE_UNIT_INVALID
	uint32_t queue_id5_max_bw;
	/*
	- * Maximum BW allocated to CoS queue in Mbps. The HWRM will translate
	- * this value into byte counter and time interval used for this COS
	- * inside the device.
	+ * Maximum BW allocated to CoS queue. The HWRM will translate this value
	+ * into byte counter and time interval used for this COS inside the
	+ * device.
	*/
	+ /* The bandwidth value. */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID5_MAX_BW_BW_VALUE_MASK UINT32_C(0xfffffff)
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID5_MAX_BW_BW_VALUE_SFT 0
	+ /* The granularity of the value (bits or bytes). */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID5_MAX_BW_SCALE UINT32_C(0x10000000)
	+ /* Value is in bits. */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID5_MAX_BW_SCALE_BITS (UINT32_C(0x0) << 28)
	+ /* Value is in bytes. */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID5_MAX_BW_SCALE_BYTES (UINT32_C(0x1) << 28)
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID5_MAX_BW_SCALE_LAST HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID5_MAX_BW_SCALE_BYTES
	+ /* bw_value_unit is 3 b */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID5_MAX_BW_BW_VALUE_UNIT_MASK UINT32_C(0xe0000000)
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID5_MAX_BW_BW_VALUE_UNIT_SFT 29
	+ /* Value is in Mb or MB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID5_MAX_BW_BW_VALUE_UNIT_MEGA (UINT32_C(0x0) << 29)
	+ /* Value is in Kb or KB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID5_MAX_BW_BW_VALUE_UNIT_KILO (UINT32_C(0x2) << 29)
	+ /* Value is in bits or bytes. */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID5_MAX_BW_BW_VALUE_UNIT_BASE (UINT32_C(0x4) << 29)
	+ /* Value is in Gb or GB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID5_MAX_BW_BW_VALUE_UNIT_GIGA (UINT32_C(0x6) << 29)
	+ /* Value is in 1/100th of a percentage of total bandwidth. */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID5_MAX_BW_BW_VALUE_UNIT_PERCENT1_100 (UINT32_C(0x1) << 29)
	+ /* Invalid unit */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID5_MAX_BW_BW_VALUE_UNIT_INVALID (UINT32_C(0x7) << 29)
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID5_MAX_BW_BW_VALUE_UNIT_LAST HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID5_MAX_BW_BW_VALUE_UNIT_INVALID
	uint8_t queue_id5_tsa_assign;
	/* Transmission Selection Algorithm (TSA) for CoS Queue. */
	/* Strict Priority */
	@@ -9884,16 +13268,68 @@
	/* ID of CoS Queue 6. */
	uint32_t queue_id6_min_bw;
	/*
	- * Minimum BW allocated to CoS Queue in Mbps. The HWRM will translate
	- * this value into byte counter and time interval used for this COS
	- * inside the device.
	+ * Minimum BW allocated to CoS Queue. The HWRM will translate this value
	+ * into byte counter and time interval used for this COS inside the
	+ * device.
	*/
	+ /* The bandwidth value. */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID6_MIN_BW_BW_VALUE_MASK UINT32_C(0xfffffff)
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID6_MIN_BW_BW_VALUE_SFT 0
	+ /* The granularity of the value (bits or bytes). */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID6_MIN_BW_SCALE UINT32_C(0x10000000)
	+ /* Value is in bits. */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID6_MIN_BW_SCALE_BITS (UINT32_C(0x0) << 28)
	+ /* Value is in bytes. */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID6_MIN_BW_SCALE_BYTES (UINT32_C(0x1) << 28)
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID6_MIN_BW_SCALE_LAST HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID6_MIN_BW_SCALE_BYTES
	+ /* bw_value_unit is 3 b */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID6_MIN_BW_BW_VALUE_UNIT_MASK UINT32_C(0xe0000000)
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID6_MIN_BW_BW_VALUE_UNIT_SFT 29
	+ /* Value is in Mb or MB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID6_MIN_BW_BW_VALUE_UNIT_MEGA (UINT32_C(0x0) << 29)
	+ /* Value is in Kb or KB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID6_MIN_BW_BW_VALUE_UNIT_KILO (UINT32_C(0x2) << 29)
	+ /* Value is in bits or bytes. */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID6_MIN_BW_BW_VALUE_UNIT_BASE (UINT32_C(0x4) << 29)
	+ /* Value is in Gb or GB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID6_MIN_BW_BW_VALUE_UNIT_GIGA (UINT32_C(0x6) << 29)
	+ /* Value is in 1/100th of a percentage of total bandwidth. */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID6_MIN_BW_BW_VALUE_UNIT_PERCENT1_100 (UINT32_C(0x1) << 29)
	+ /* Invalid unit */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID6_MIN_BW_BW_VALUE_UNIT_INVALID (UINT32_C(0x7) << 29)
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID6_MIN_BW_BW_VALUE_UNIT_LAST HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID6_MIN_BW_BW_VALUE_UNIT_INVALID
	uint32_t queue_id6_max_bw;
	/*
	- * Maximum BW allocated to CoS queue in Mbps. The HWRM will translate
	- * this value into byte counter and time interval used for this COS
	- * inside the device.
	+ * Maximum BW allocated to CoS queue. The HWRM will translate this value
	+ * into byte counter and time interval used for this COS inside the
	+ * device.
	*/
	+ /* The bandwidth value. */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID6_MAX_BW_BW_VALUE_MASK UINT32_C(0xfffffff)
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID6_MAX_BW_BW_VALUE_SFT 0
	+ /* The granularity of the value (bits or bytes). */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID6_MAX_BW_SCALE UINT32_C(0x10000000)
	+ /* Value is in bits. */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID6_MAX_BW_SCALE_BITS (UINT32_C(0x0) << 28)
	+ /* Value is in bytes. */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID6_MAX_BW_SCALE_BYTES (UINT32_C(0x1) << 28)
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID6_MAX_BW_SCALE_LAST HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID6_MAX_BW_SCALE_BYTES
	+ /* bw_value_unit is 3 b */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID6_MAX_BW_BW_VALUE_UNIT_MASK UINT32_C(0xe0000000)
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID6_MAX_BW_BW_VALUE_UNIT_SFT 29
	+ /* Value is in Mb or MB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID6_MAX_BW_BW_VALUE_UNIT_MEGA (UINT32_C(0x0) << 29)
	+ /* Value is in Kb or KB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID6_MAX_BW_BW_VALUE_UNIT_KILO (UINT32_C(0x2) << 29)
	+ /* Value is in bits or bytes. */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID6_MAX_BW_BW_VALUE_UNIT_BASE (UINT32_C(0x4) << 29)
	+ /* Value is in Gb or GB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID6_MAX_BW_BW_VALUE_UNIT_GIGA (UINT32_C(0x6) << 29)
	+ /* Value is in 1/100th of a percentage of total bandwidth. */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID6_MAX_BW_BW_VALUE_UNIT_PERCENT1_100 (UINT32_C(0x1) << 29)
	+ /* Invalid unit */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID6_MAX_BW_BW_VALUE_UNIT_INVALID (UINT32_C(0x7) << 29)
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID6_MAX_BW_BW_VALUE_UNIT_LAST HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID6_MAX_BW_BW_VALUE_UNIT_INVALID
	uint8_t queue_id6_tsa_assign;
	/* Transmission Selection Algorithm (TSA) for CoS Queue. */
	/* Strict Priority */
	@@ -9918,16 +13354,68 @@
	/* ID of CoS Queue 7. */
	uint32_t queue_id7_min_bw;
	/*
	- * Minimum BW allocated to CoS Queue in Mbps. The HWRM will translate
	- * this value into byte counter and time interval used for this COS
	- * inside the device.
	+ * Minimum BW allocated to CoS Queue. The HWRM will translate this value
	+ * into byte counter and time interval used for this COS inside the
	+ * device.
	*/
	+ /* The bandwidth value. */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID7_MIN_BW_BW_VALUE_MASK UINT32_C(0xfffffff)
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID7_MIN_BW_BW_VALUE_SFT 0
	+ /* The granularity of the value (bits or bytes). */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID7_MIN_BW_SCALE UINT32_C(0x10000000)
	+ /* Value is in bits. */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID7_MIN_BW_SCALE_BITS (UINT32_C(0x0) << 28)
	+ /* Value is in bytes. */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID7_MIN_BW_SCALE_BYTES (UINT32_C(0x1) << 28)
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID7_MIN_BW_SCALE_LAST HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID7_MIN_BW_SCALE_BYTES
	+ /* bw_value_unit is 3 b */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID7_MIN_BW_BW_VALUE_UNIT_MASK UINT32_C(0xe0000000)
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID7_MIN_BW_BW_VALUE_UNIT_SFT 29
	+ /* Value is in Mb or MB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID7_MIN_BW_BW_VALUE_UNIT_MEGA (UINT32_C(0x0) << 29)
	+ /* Value is in Kb or KB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID7_MIN_BW_BW_VALUE_UNIT_KILO (UINT32_C(0x2) << 29)
	+ /* Value is in bits or bytes. */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID7_MIN_BW_BW_VALUE_UNIT_BASE (UINT32_C(0x4) << 29)
	+ /* Value is in Gb or GB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID7_MIN_BW_BW_VALUE_UNIT_GIGA (UINT32_C(0x6) << 29)
	+ /* Value is in 1/100th of a percentage of total bandwidth. */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID7_MIN_BW_BW_VALUE_UNIT_PERCENT1_100 (UINT32_C(0x1) << 29)
	+ /* Invalid unit */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID7_MIN_BW_BW_VALUE_UNIT_INVALID (UINT32_C(0x7) << 29)
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID7_MIN_BW_BW_VALUE_UNIT_LAST HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID7_MIN_BW_BW_VALUE_UNIT_INVALID
	uint32_t queue_id7_max_bw;
	/*
	- * Maximum BW allocated to CoS queue in Mbps. The HWRM will translate
	- * this value into byte counter and time interval used for this COS
	- * inside the device.
	+ * Maximum BW allocated to CoS queue. The HWRM will translate this value
	+ * into byte counter and time interval used for this COS inside the
	+ * device.
	*/
	+ /* The bandwidth value. */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID7_MAX_BW_BW_VALUE_MASK UINT32_C(0xfffffff)
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID7_MAX_BW_BW_VALUE_SFT 0
	+ /* The granularity of the value (bits or bytes). */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID7_MAX_BW_SCALE UINT32_C(0x10000000)
	+ /* Value is in bits. */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID7_MAX_BW_SCALE_BITS (UINT32_C(0x0) << 28)
	+ /* Value is in bytes. */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID7_MAX_BW_SCALE_BYTES (UINT32_C(0x1) << 28)
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID7_MAX_BW_SCALE_LAST HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID7_MAX_BW_SCALE_BYTES
	+ /* bw_value_unit is 3 b */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID7_MAX_BW_BW_VALUE_UNIT_MASK UINT32_C(0xe0000000)
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID7_MAX_BW_BW_VALUE_UNIT_SFT 29
	+ /* Value is in Mb or MB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID7_MAX_BW_BW_VALUE_UNIT_MEGA (UINT32_C(0x0) << 29)
	+ /* Value is in Kb or KB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID7_MAX_BW_BW_VALUE_UNIT_KILO (UINT32_C(0x2) << 29)
	+ /* Value is in bits or bytes. */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID7_MAX_BW_BW_VALUE_UNIT_BASE (UINT32_C(0x4) << 29)
	+ /* Value is in Gb or GB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID7_MAX_BW_BW_VALUE_UNIT_GIGA (UINT32_C(0x6) << 29)
	+ /* Value is in 1/100th of a percentage of total bandwidth. */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID7_MAX_BW_BW_VALUE_UNIT_PERCENT1_100 (UINT32_C(0x1) << 29)
	+ /* Invalid unit */
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID7_MAX_BW_BW_VALUE_UNIT_INVALID (UINT32_C(0x7) << 29)
	+ #define HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID7_MAX_BW_BW_VALUE_UNIT_LAST HWRM_QUEUE_COS2BW_QCFG_OUTPUT_QUEUE_ID7_MAX_BW_BW_VALUE_UNIT_INVALID
	uint8_t queue_id7_tsa_assign;
	/* Transmission Selection Algorithm (TSA) for CoS Queue. */
	/* Strict Priority */
	@@ -9971,20 +13459,22 @@
	* following: - Minimum bandwidth - Maximum bandwidth - Transmission selection
	* algorithm (TSA) - Priority Level (only applies to strict priority COS) -
	* Bandwidth weight # A CoS can be SP or non-SP: A SP CoS always gets the strict
	- * priority. Is an COS min BW is set to 0x0 then it is considered to be non-SP;
	- * this is a valid configuration. Note: SP provides lower latency in addition to
	- * reserved bandwidth # For both SP CoS and non-SP CoS, min BW can be specified
	- * to reserve specific amount of the port BW. # The min BW specified for a CoS
	- * shall not exceed max port bandwidth. # The total of min BWs specified for all
	- * CoS shall not exceed max port bandwidth. # For any non-SP CoS, the minimum
	- * bandwidth guarantees are subject to round-robin scheduling. This allows BW
	- * reservation with anti-starvation; one CoS will not block another CoS using
	- * RR. Note: The bandwidth guarantees for any non-SP CoS are met after servicing
	- * all SP CoS. # An SP CoS can potentially starve other lower priority SP CoS
	- * and non-SP CoS queues. This can occur to the extent the SP min exceeds the
	- * available port BW. # For any CoS, max BW can be specified to limit the BW
	- * consumed by the CoS. # The max BW specified for a CoS shall not exceed the
	- * max port bandwidth. # The WFQ provides a mechanism for sharing available
	+ * priority. Note: SP provides lower latency in addition to reserved bandwidth #
	+ * For non-SP CoS, min BW can be specified to reserve specific amount of the
	+ * port BW. # The min BW specified for a CoS shall not exceed max port
	+ * bandwidth. # The total of min BWs specified for all CoS shall not exceed max
	+ * port bandwidth. # For any non-SP CoS, the minimum bandwidth guarantees are
	+ * subject to round-robin scheduling. This allows BW reservation with anti-
	+ * starvation; one CoS will not block another CoS using RR. Note: The bandwidth
	+ * guarantees for any non-SP CoS are met after servicing all SP CoS. # An SP CoS
	+ * can potentially starve other lower priority SP CoS and non-SP CoS queues.
	+ * This can occur to the extent the SP min exceeds the available port BW. # For
	+ * any CoS, max BW can be specified to limit the BW consumed by the CoS. # A max
	+ * BW can be used for a SP CoS to limit the starvation of other CoS, but using
	+ * this will cause some characteristics of any ETS CoS to be violated. # The max
	+ * BW specified for a CoS shall not exceed the max port bandwidth. # For SP CoS,
	+ * it is recommended to set min and max BW to 0. This instructs the adapter to
	+ * use default values. # The WFQ provides a mechanism for sharing available
	* bandwidth beyond the reserved minimums configured for each CoS. The WFQ
	* scheduler is used to provide the percentages of remaining bandwidth after: -
	* first servicing the reserved bandwidth for all SP CoS, - followed by the
	@@ -9995,20 +13485,21 @@
	* this COS is not guaranteed any bandwidth. A value of 0xFF.. (all Fs) means
	* min BW is not specified. When the min BW is not specified, the HWRM can set
	* it to any value it considers appropriate. Note: For a non-SP COS, the HWRM
	- * should set min BW to 0 when the min BW is not specified. For an SP COS, the
	- * HWRM should set min BW to some small value when the min BW is not specified.
	- * Maximum Bandwidth: # This is the bandwidth limit of the COS. # Values 0x0 and
	- * 0xFF.. (all Fs) are considered unspecified and the HWRM will set the maximum
	- * bandwidth to maximum port bandwidth. Priority Level: # It applies only to SP.
	- * # This parameter is ignored for non-SP. # 0-7 are valid values (higher value
	- * means higher priority) # A priority level can be assigned to at most one SP.
	- * # Invalid priority levels assignment for SPs shall result in failure.
	- * Additional notes: # The HWRM may have to use min and (max - min) to set
	- * appropriate counters of hardware rate limiters. # The bandwidth percentage as
	- * specified in the DCB TC BW assignment should be used by the driver to specify
	- * maximum bandwidth and bandwidth weight for a COS. For example, the driver
	- * should set max BW to 20 Gbps and weight to 50 for two COSs when these two
	- * COSs are assigned 50% share of 40 Gbps max port bandwidth.
	+ * should set min BW to 0 when the min BW is not specified. For an SP COS, min
	+ * BW value is ignored. Maximum Bandwidth: # This is the bandwidth limit of the
	+ * COS. # Values 0x0 and 0xFF.. (all Fs) are considered unspecified and the HWRM
	+ * will set the maximum bandwidth to maximum port bandwidth. Priority Level: #
	+ * It applies only to SP. # This parameter is ignored for non-SP. # 0-7 are
	+ * valid values (higher value means higher priority) # A priority level can be
	+ * assigned to at most one SP. # Invalid priority levels assignment for SPs
	+ * shall result in failure. Additional notes: # The HWRM may have to use min and
	+ * (max - min) to set appropriate counters of hardware rate limiters. # The
	+ * bandwidth percentage as specified in the DCB TC BW assignment should be used
	+ * by the driver to specify minimum bandwidth and bandwidth weight for a COS.
	+ * For example, the driver should set max BW to 20 Gbps and weight to 50 for two
	+ * COSs when these two COSs are assigned 50% share of 40 Gbps max port
	+ * bandwidth. DCBX use cases should always use max BW of 100% for all ETS CoS
	+ * queues.
	*/
	/* Input (128 bytes) */

	@@ -10041,43 +13532,43 @@
	uint32_t flags;
	uint32_t enables;
	/*
	- * This bit must be '1' for the cos_queue_id0_valid field to be
	- * configured.
	+ * If this bit is set to 1, then all queue_id0 related parameters in
	+ * this command are valid.
	*/
	#define HWRM_QUEUE_COS2BW_CFG_INPUT_ENABLES_COS_QUEUE_ID0_VALID UINT32_C(0x1)
	/*
	- * This bit must be '1' for the cos_queue_id1_valid field to be
	- * configured.
	+ * If this bit is set to 1, then all queue_id1 related parameters in
	+ * this command are valid.
	*/
	#define HWRM_QUEUE_COS2BW_CFG_INPUT_ENABLES_COS_QUEUE_ID1_VALID UINT32_C(0x2)
	/*
	- * This bit must be '1' for the cos_queue_id2_valid field to be
	- * configured.
	+ * If this bit is set to 1, then all queue_id2 related parameters in
	+ * this command are valid.
	*/
	#define HWRM_QUEUE_COS2BW_CFG_INPUT_ENABLES_COS_QUEUE_ID2_VALID UINT32_C(0x4)
	/*
	- * This bit must be '1' for the cos_queue_id3_valid field to be
	- * configured.
	+ * If this bit is set to 1, then all queue_id3 related parameters in
	+ * this command are valid.
	*/
	#define HWRM_QUEUE_COS2BW_CFG_INPUT_ENABLES_COS_QUEUE_ID3_VALID UINT32_C(0x8)
	/*
	- * This bit must be '1' for the cos_queue_id4_valid field to be
	- * configured.
	+ * If this bit is set to 1, then all queue_id4 related parameters in
	+ * this command are valid.
	*/
	#define HWRM_QUEUE_COS2BW_CFG_INPUT_ENABLES_COS_QUEUE_ID4_VALID UINT32_C(0x10)
	/*
	- * This bit must be '1' for the cos_queue_id5_valid field to be
	- * configured.
	+ * If this bit is set to 1, then all queue_id5 related parameters in
	+ * this command are valid.
	*/
	#define HWRM_QUEUE_COS2BW_CFG_INPUT_ENABLES_COS_QUEUE_ID5_VALID UINT32_C(0x20)
	/*
	- * This bit must be '1' for the cos_queue_id6_valid field to be
	- * configured.
	+ * If this bit is set to 1, then all queue_id6 related parameters in
	+ * this command are valid.
	*/
	#define HWRM_QUEUE_COS2BW_CFG_INPUT_ENABLES_COS_QUEUE_ID6_VALID UINT32_C(0x40)
	/*
	- * This bit must be '1' for the cos_queue_id7_valid field to be
	- * configured.
	+ * If this bit is set to 1, then all queue_id7 related parameters in
	+ * this command are valid.
	*/
	#define HWRM_QUEUE_COS2BW_CFG_INPUT_ENABLES_COS_QUEUE_ID7_VALID UINT32_C(0x80)
	uint16_t port_id;
	@@ -10091,16 +13582,68 @@
	uint8_t unused_0;
	uint32_t queue_id0_min_bw;
	/*
	- * Minimum BW allocated to CoS Queue in Mbps. The HWRM will translate
	- * this value into byte counter and time interval used for this COS
	- * inside the device.
	+ * Minimum BW allocated to CoS Queue. The HWRM will translate this value
	+ * into byte counter and time interval used for this COS inside the
	+ * device.
	*/
	+ /* The bandwidth value. */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID0_MIN_BW_BW_VALUE_MASK UINT32_C(0xfffffff)
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID0_MIN_BW_BW_VALUE_SFT 0
	+ /* The granularity of the value (bits or bytes). */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID0_MIN_BW_SCALE UINT32_C(0x10000000)
	+ /* Value is in bits. */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID0_MIN_BW_SCALE_BITS (UINT32_C(0x0) << 28)
	+ /* Value is in bytes. */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID0_MIN_BW_SCALE_BYTES (UINT32_C(0x1) << 28)
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID0_MIN_BW_SCALE_LAST HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID0_MIN_BW_SCALE_BYTES
	+ /* bw_value_unit is 3 b */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID0_MIN_BW_BW_VALUE_UNIT_MASK UINT32_C(0xe0000000)
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID0_MIN_BW_BW_VALUE_UNIT_SFT 29
	+ /* Value is in Mb or MB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID0_MIN_BW_BW_VALUE_UNIT_MEGA (UINT32_C(0x0) << 29)
	+ /* Value is in Kb or KB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID0_MIN_BW_BW_VALUE_UNIT_KILO (UINT32_C(0x2) << 29)
	+ /* Value is in bits or bytes. */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID0_MIN_BW_BW_VALUE_UNIT_BASE (UINT32_C(0x4) << 29)
	+ /* Value is in Gb or GB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID0_MIN_BW_BW_VALUE_UNIT_GIGA (UINT32_C(0x6) << 29)
	+ /* Value is in 1/100th of a percentage of total bandwidth. */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID0_MIN_BW_BW_VALUE_UNIT_PERCENT1_100 (UINT32_C(0x1) << 29)
	+ /* Invalid unit */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID0_MIN_BW_BW_VALUE_UNIT_INVALID (UINT32_C(0x7) << 29)
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID0_MIN_BW_BW_VALUE_UNIT_LAST HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID0_MIN_BW_BW_VALUE_UNIT_INVALID
	uint32_t queue_id0_max_bw;
	/*
	- * Maximum BW allocated to CoS Queue in Mbps. The HWRM will translate
	- * this value into byte counter and time interval used for this COS
	- * inside the device.
	+ * Maximum BW allocated to CoS Queue. The HWRM will translate this value
	+ * into byte counter and time interval used for this COS inside the
	+ * device.
	*/
	+ /* The bandwidth value. */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID0_MAX_BW_BW_VALUE_MASK UINT32_C(0xfffffff)
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID0_MAX_BW_BW_VALUE_SFT 0
	+ /* The granularity of the value (bits or bytes). */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID0_MAX_BW_SCALE UINT32_C(0x10000000)
	+ /* Value is in bits. */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID0_MAX_BW_SCALE_BITS (UINT32_C(0x0) << 28)
	+ /* Value is in bytes. */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID0_MAX_BW_SCALE_BYTES (UINT32_C(0x1) << 28)
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID0_MAX_BW_SCALE_LAST HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID0_MAX_BW_SCALE_BYTES
	+ /* bw_value_unit is 3 b */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID0_MAX_BW_BW_VALUE_UNIT_MASK UINT32_C(0xe0000000)
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID0_MAX_BW_BW_VALUE_UNIT_SFT 29
	+ /* Value is in Mb or MB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID0_MAX_BW_BW_VALUE_UNIT_MEGA (UINT32_C(0x0) << 29)
	+ /* Value is in Kb or KB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID0_MAX_BW_BW_VALUE_UNIT_KILO (UINT32_C(0x2) << 29)
	+ /* Value is in bits or bytes. */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID0_MAX_BW_BW_VALUE_UNIT_BASE (UINT32_C(0x4) << 29)
	+ /* Value is in Gb or GB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID0_MAX_BW_BW_VALUE_UNIT_GIGA (UINT32_C(0x6) << 29)
	+ /* Value is in 1/100th of a percentage of total bandwidth. */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID0_MAX_BW_BW_VALUE_UNIT_PERCENT1_100 (UINT32_C(0x1) << 29)
	+ /* Invalid unit */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID0_MAX_BW_BW_VALUE_UNIT_INVALID (UINT32_C(0x7) << 29)
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID0_MAX_BW_BW_VALUE_UNIT_LAST HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID0_MAX_BW_BW_VALUE_UNIT_INVALID
	uint8_t queue_id0_tsa_assign;
	/* Transmission Selection Algorithm (TSA) for CoS Queue. */
	/* Strict Priority */
	@@ -10125,16 +13668,68 @@
	/* ID of CoS Queue 1. */
	uint32_t queue_id1_min_bw;
	/*
	- * Minimum BW allocated to CoS Queue in Mbps. The HWRM will translate
	- * this value into byte counter and time interval used for this COS
	- * inside the device.
	+ * Minimum BW allocated to CoS Queue. The HWRM will translate this value
	+ * into byte counter and time interval used for this COS inside the
	+ * device.
	*/
	+ /* The bandwidth value. */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID1_MIN_BW_BW_VALUE_MASK UINT32_C(0xfffffff)
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID1_MIN_BW_BW_VALUE_SFT 0
	+ /* The granularity of the value (bits or bytes). */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID1_MIN_BW_SCALE UINT32_C(0x10000000)
	+ /* Value is in bits. */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID1_MIN_BW_SCALE_BITS (UINT32_C(0x0) << 28)
	+ /* Value is in bytes. */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID1_MIN_BW_SCALE_BYTES (UINT32_C(0x1) << 28)
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID1_MIN_BW_SCALE_LAST HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID1_MIN_BW_SCALE_BYTES
	+ /* bw_value_unit is 3 b */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID1_MIN_BW_BW_VALUE_UNIT_MASK UINT32_C(0xe0000000)
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID1_MIN_BW_BW_VALUE_UNIT_SFT 29
	+ /* Value is in Mb or MB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID1_MIN_BW_BW_VALUE_UNIT_MEGA (UINT32_C(0x0) << 29)
	+ /* Value is in Kb or KB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID1_MIN_BW_BW_VALUE_UNIT_KILO (UINT32_C(0x2) << 29)
	+ /* Value is in bits or bytes. */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID1_MIN_BW_BW_VALUE_UNIT_BASE (UINT32_C(0x4) << 29)
	+ /* Value is in Gb or GB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID1_MIN_BW_BW_VALUE_UNIT_GIGA (UINT32_C(0x6) << 29)
	+ /* Value is in 1/100th of a percentage of total bandwidth. */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID1_MIN_BW_BW_VALUE_UNIT_PERCENT1_100 (UINT32_C(0x1) << 29)
	+ /* Invalid unit */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID1_MIN_BW_BW_VALUE_UNIT_INVALID (UINT32_C(0x7) << 29)
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID1_MIN_BW_BW_VALUE_UNIT_LAST HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID1_MIN_BW_BW_VALUE_UNIT_INVALID
	uint32_t queue_id1_max_bw;
	/*
	- * Maximum BW allocated to CoS queue in Mbps. The HWRM will translate
	- * this value into byte counter and time interval used for this COS
	- * inside the device.
	+ * Maximum BW allocated to CoS queue. The HWRM will translate this value
	+ * into byte counter and time interval used for this COS inside the
	+ * device.
	*/
	+ /* The bandwidth value. */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID1_MAX_BW_BW_VALUE_MASK UINT32_C(0xfffffff)
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID1_MAX_BW_BW_VALUE_SFT 0
	+ /* The granularity of the value (bits or bytes). */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID1_MAX_BW_SCALE UINT32_C(0x10000000)
	+ /* Value is in bits. */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID1_MAX_BW_SCALE_BITS (UINT32_C(0x0) << 28)
	+ /* Value is in bytes. */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID1_MAX_BW_SCALE_BYTES (UINT32_C(0x1) << 28)
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID1_MAX_BW_SCALE_LAST HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID1_MAX_BW_SCALE_BYTES
	+ /* bw_value_unit is 3 b */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID1_MAX_BW_BW_VALUE_UNIT_MASK UINT32_C(0xe0000000)
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID1_MAX_BW_BW_VALUE_UNIT_SFT 29
	+ /* Value is in Mb or MB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID1_MAX_BW_BW_VALUE_UNIT_MEGA (UINT32_C(0x0) << 29)
	+ /* Value is in Kb or KB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID1_MAX_BW_BW_VALUE_UNIT_KILO (UINT32_C(0x2) << 29)
	+ /* Value is in bits or bytes. */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID1_MAX_BW_BW_VALUE_UNIT_BASE (UINT32_C(0x4) << 29)
	+ /* Value is in Gb or GB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID1_MAX_BW_BW_VALUE_UNIT_GIGA (UINT32_C(0x6) << 29)
	+ /* Value is in 1/100th of a percentage of total bandwidth. */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID1_MAX_BW_BW_VALUE_UNIT_PERCENT1_100 (UINT32_C(0x1) << 29)
	+ /* Invalid unit */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID1_MAX_BW_BW_VALUE_UNIT_INVALID (UINT32_C(0x7) << 29)
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID1_MAX_BW_BW_VALUE_UNIT_LAST HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID1_MAX_BW_BW_VALUE_UNIT_INVALID
	uint8_t queue_id1_tsa_assign;
	/* Transmission Selection Algorithm (TSA) for CoS Queue. */
	/* Strict Priority */
	@@ -10159,16 +13754,68 @@
	/* ID of CoS Queue 2. */
	uint32_t queue_id2_min_bw;
	/*
	- * Minimum BW allocated to CoS Queue in Mbps. The HWRM will translate
	- * this value into byte counter and time interval used for this COS
	- * inside the device.
	+ * Minimum BW allocated to CoS Queue. The HWRM will translate this value
	+ * into byte counter and time interval used for this COS inside the
	+ * device.
	*/
	+ /* The bandwidth value. */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID2_MIN_BW_BW_VALUE_MASK UINT32_C(0xfffffff)
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID2_MIN_BW_BW_VALUE_SFT 0
	+ /* The granularity of the value (bits or bytes). */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID2_MIN_BW_SCALE UINT32_C(0x10000000)
	+ /* Value is in bits. */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID2_MIN_BW_SCALE_BITS (UINT32_C(0x0) << 28)
	+ /* Value is in bytes. */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID2_MIN_BW_SCALE_BYTES (UINT32_C(0x1) << 28)
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID2_MIN_BW_SCALE_LAST HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID2_MIN_BW_SCALE_BYTES
	+ /* bw_value_unit is 3 b */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID2_MIN_BW_BW_VALUE_UNIT_MASK UINT32_C(0xe0000000)
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID2_MIN_BW_BW_VALUE_UNIT_SFT 29
	+ /* Value is in Mb or MB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID2_MIN_BW_BW_VALUE_UNIT_MEGA (UINT32_C(0x0) << 29)
	+ /* Value is in Kb or KB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID2_MIN_BW_BW_VALUE_UNIT_KILO (UINT32_C(0x2) << 29)
	+ /* Value is in bits or bytes. */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID2_MIN_BW_BW_VALUE_UNIT_BASE (UINT32_C(0x4) << 29)
	+ /* Value is in Gb or GB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID2_MIN_BW_BW_VALUE_UNIT_GIGA (UINT32_C(0x6) << 29)
	+ /* Value is in 1/100th of a percentage of total bandwidth. */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID2_MIN_BW_BW_VALUE_UNIT_PERCENT1_100 (UINT32_C(0x1) << 29)
	+ /* Invalid unit */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID2_MIN_BW_BW_VALUE_UNIT_INVALID (UINT32_C(0x7) << 29)
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID2_MIN_BW_BW_VALUE_UNIT_LAST HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID2_MIN_BW_BW_VALUE_UNIT_INVALID
	uint32_t queue_id2_max_bw;
	/*
	- * Maximum BW allocated to CoS queue in Mbps. The HWRM will translate
	- * this value into byte counter and time interval used for this COS
	- * inside the device.
	+ * Maximum BW allocated to CoS queue. The HWRM will translate this value
	+ * into byte counter and time interval used for this COS inside the
	+ * device.
	*/
	+ /* The bandwidth value. */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID2_MAX_BW_BW_VALUE_MASK UINT32_C(0xfffffff)
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID2_MAX_BW_BW_VALUE_SFT 0
	+ /* The granularity of the value (bits or bytes). */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID2_MAX_BW_SCALE UINT32_C(0x10000000)
	+ /* Value is in bits. */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID2_MAX_BW_SCALE_BITS (UINT32_C(0x0) << 28)
	+ /* Value is in bytes. */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID2_MAX_BW_SCALE_BYTES (UINT32_C(0x1) << 28)
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID2_MAX_BW_SCALE_LAST HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID2_MAX_BW_SCALE_BYTES
	+ /* bw_value_unit is 3 b */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID2_MAX_BW_BW_VALUE_UNIT_MASK UINT32_C(0xe0000000)
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID2_MAX_BW_BW_VALUE_UNIT_SFT 29
	+ /* Value is in Mb or MB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID2_MAX_BW_BW_VALUE_UNIT_MEGA (UINT32_C(0x0) << 29)
	+ /* Value is in Kb or KB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID2_MAX_BW_BW_VALUE_UNIT_KILO (UINT32_C(0x2) << 29)
	+ /* Value is in bits or bytes. */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID2_MAX_BW_BW_VALUE_UNIT_BASE (UINT32_C(0x4) << 29)
	+ /* Value is in Gb or GB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID2_MAX_BW_BW_VALUE_UNIT_GIGA (UINT32_C(0x6) << 29)
	+ /* Value is in 1/100th of a percentage of total bandwidth. */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID2_MAX_BW_BW_VALUE_UNIT_PERCENT1_100 (UINT32_C(0x1) << 29)
	+ /* Invalid unit */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID2_MAX_BW_BW_VALUE_UNIT_INVALID (UINT32_C(0x7) << 29)
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID2_MAX_BW_BW_VALUE_UNIT_LAST HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID2_MAX_BW_BW_VALUE_UNIT_INVALID
	uint8_t queue_id2_tsa_assign;
	/* Transmission Selection Algorithm (TSA) for CoS Queue. */
	/* Strict Priority */
	@@ -10193,16 +13840,68 @@
	/* ID of CoS Queue 3. */
	uint32_t queue_id3_min_bw;
	/*
	- * Minimum BW allocated to CoS Queue in Mbps. The HWRM will translate
	- * this value into byte counter and time interval used for this COS
	- * inside the device.
	+ * Minimum BW allocated to CoS Queue. The HWRM will translate this value
	+ * into byte counter and time interval used for this COS inside the
	+ * device.
	*/
	+ /* The bandwidth value. */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID3_MIN_BW_BW_VALUE_MASK UINT32_C(0xfffffff)
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID3_MIN_BW_BW_VALUE_SFT 0
	+ /* The granularity of the value (bits or bytes). */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID3_MIN_BW_SCALE UINT32_C(0x10000000)
	+ /* Value is in bits. */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID3_MIN_BW_SCALE_BITS (UINT32_C(0x0) << 28)
	+ /* Value is in bytes. */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID3_MIN_BW_SCALE_BYTES (UINT32_C(0x1) << 28)
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID3_MIN_BW_SCALE_LAST HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID3_MIN_BW_SCALE_BYTES
	+ /* bw_value_unit is 3 b */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID3_MIN_BW_BW_VALUE_UNIT_MASK UINT32_C(0xe0000000)
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID3_MIN_BW_BW_VALUE_UNIT_SFT 29
	+ /* Value is in Mb or MB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID3_MIN_BW_BW_VALUE_UNIT_MEGA (UINT32_C(0x0) << 29)
	+ /* Value is in Kb or KB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID3_MIN_BW_BW_VALUE_UNIT_KILO (UINT32_C(0x2) << 29)
	+ /* Value is in bits or bytes. */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID3_MIN_BW_BW_VALUE_UNIT_BASE (UINT32_C(0x4) << 29)
	+ /* Value is in Gb or GB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID3_MIN_BW_BW_VALUE_UNIT_GIGA (UINT32_C(0x6) << 29)
	+ /* Value is in 1/100th of a percentage of total bandwidth. */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID3_MIN_BW_BW_VALUE_UNIT_PERCENT1_100 (UINT32_C(0x1) << 29)
	+ /* Invalid unit */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID3_MIN_BW_BW_VALUE_UNIT_INVALID (UINT32_C(0x7) << 29)
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID3_MIN_BW_BW_VALUE_UNIT_LAST HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID3_MIN_BW_BW_VALUE_UNIT_INVALID
	uint32_t queue_id3_max_bw;
	/*
	- * Maximum BW allocated to CoS queue in Mbps. The HWRM will translate
	- * this value into byte counter and time interval used for this COS
	- * inside the device.
	+ * Maximum BW allocated to CoS queue. The HWRM will translate this value
	+ * into byte counter and time interval used for this COS inside the
	+ * device.
	*/
	+ /* The bandwidth value. */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID3_MAX_BW_BW_VALUE_MASK UINT32_C(0xfffffff)
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID3_MAX_BW_BW_VALUE_SFT 0
	+ /* The granularity of the value (bits or bytes). */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID3_MAX_BW_SCALE UINT32_C(0x10000000)
	+ /* Value is in bits. */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID3_MAX_BW_SCALE_BITS (UINT32_C(0x0) << 28)
	+ /* Value is in bytes. */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID3_MAX_BW_SCALE_BYTES (UINT32_C(0x1) << 28)
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID3_MAX_BW_SCALE_LAST HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID3_MAX_BW_SCALE_BYTES
	+ /* bw_value_unit is 3 b */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID3_MAX_BW_BW_VALUE_UNIT_MASK UINT32_C(0xe0000000)
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID3_MAX_BW_BW_VALUE_UNIT_SFT 29
	+ /* Value is in Mb or MB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID3_MAX_BW_BW_VALUE_UNIT_MEGA (UINT32_C(0x0) << 29)
	+ /* Value is in Kb or KB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID3_MAX_BW_BW_VALUE_UNIT_KILO (UINT32_C(0x2) << 29)
	+ /* Value is in bits or bytes. */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID3_MAX_BW_BW_VALUE_UNIT_BASE (UINT32_C(0x4) << 29)
	+ /* Value is in Gb or GB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID3_MAX_BW_BW_VALUE_UNIT_GIGA (UINT32_C(0x6) << 29)
	+ /* Value is in 1/100th of a percentage of total bandwidth. */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID3_MAX_BW_BW_VALUE_UNIT_PERCENT1_100 (UINT32_C(0x1) << 29)
	+ /* Invalid unit */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID3_MAX_BW_BW_VALUE_UNIT_INVALID (UINT32_C(0x7) << 29)
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID3_MAX_BW_BW_VALUE_UNIT_LAST HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID3_MAX_BW_BW_VALUE_UNIT_INVALID
	uint8_t queue_id3_tsa_assign;
	/* Transmission Selection Algorithm (TSA) for CoS Queue. */
	/* Strict Priority */
	@@ -10227,16 +13926,68 @@
	/* ID of CoS Queue 4. */
	uint32_t queue_id4_min_bw;
	/*
	- * Minimum BW allocated to CoS Queue in Mbps. The HWRM will translate
	- * this value into byte counter and time interval used for this COS
	- * inside the device.
	+ * Minimum BW allocated to CoS Queue. The HWRM will translate this value
	+ * into byte counter and time interval used for this COS inside the
	+ * device.
	*/
	+ /* The bandwidth value. */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID4_MIN_BW_BW_VALUE_MASK UINT32_C(0xfffffff)
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID4_MIN_BW_BW_VALUE_SFT 0
	+ /* The granularity of the value (bits or bytes). */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID4_MIN_BW_SCALE UINT32_C(0x10000000)
	+ /* Value is in bits. */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID4_MIN_BW_SCALE_BITS (UINT32_C(0x0) << 28)
	+ /* Value is in bytes. */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID4_MIN_BW_SCALE_BYTES (UINT32_C(0x1) << 28)
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID4_MIN_BW_SCALE_LAST HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID4_MIN_BW_SCALE_BYTES
	+ /* bw_value_unit is 3 b */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID4_MIN_BW_BW_VALUE_UNIT_MASK UINT32_C(0xe0000000)
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID4_MIN_BW_BW_VALUE_UNIT_SFT 29
	+ /* Value is in Mb or MB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID4_MIN_BW_BW_VALUE_UNIT_MEGA (UINT32_C(0x0) << 29)
	+ /* Value is in Kb or KB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID4_MIN_BW_BW_VALUE_UNIT_KILO (UINT32_C(0x2) << 29)
	+ /* Value is in bits or bytes. */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID4_MIN_BW_BW_VALUE_UNIT_BASE (UINT32_C(0x4) << 29)
	+ /* Value is in Gb or GB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID4_MIN_BW_BW_VALUE_UNIT_GIGA (UINT32_C(0x6) << 29)
	+ /* Value is in 1/100th of a percentage of total bandwidth. */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID4_MIN_BW_BW_VALUE_UNIT_PERCENT1_100 (UINT32_C(0x1) << 29)
	+ /* Invalid unit */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID4_MIN_BW_BW_VALUE_UNIT_INVALID (UINT32_C(0x7) << 29)
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID4_MIN_BW_BW_VALUE_UNIT_LAST HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID4_MIN_BW_BW_VALUE_UNIT_INVALID
	uint32_t queue_id4_max_bw;
	/*
	- * Maximum BW allocated to CoS queue in Mbps. The HWRM will translate
	- * this value into byte counter and time interval used for this COS
	- * inside the device.
	+ * Maximum BW allocated to CoS queue. The HWRM will translate this value
	+ * into byte counter and time interval used for this COS inside the
	+ * device.
	*/
	+ /* The bandwidth value. */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID4_MAX_BW_BW_VALUE_MASK UINT32_C(0xfffffff)
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID4_MAX_BW_BW_VALUE_SFT 0
	+ /* The granularity of the value (bits or bytes). */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID4_MAX_BW_SCALE UINT32_C(0x10000000)
	+ /* Value is in bits. */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID4_MAX_BW_SCALE_BITS (UINT32_C(0x0) << 28)
	+ /* Value is in bytes. */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID4_MAX_BW_SCALE_BYTES (UINT32_C(0x1) << 28)
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID4_MAX_BW_SCALE_LAST HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID4_MAX_BW_SCALE_BYTES
	+ /* bw_value_unit is 3 b */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID4_MAX_BW_BW_VALUE_UNIT_MASK UINT32_C(0xe0000000)
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID4_MAX_BW_BW_VALUE_UNIT_SFT 29
	+ /* Value is in Mb or MB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID4_MAX_BW_BW_VALUE_UNIT_MEGA (UINT32_C(0x0) << 29)
	+ /* Value is in Kb or KB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID4_MAX_BW_BW_VALUE_UNIT_KILO (UINT32_C(0x2) << 29)
	+ /* Value is in bits or bytes. */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID4_MAX_BW_BW_VALUE_UNIT_BASE (UINT32_C(0x4) << 29)
	+ /* Value is in Gb or GB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID4_MAX_BW_BW_VALUE_UNIT_GIGA (UINT32_C(0x6) << 29)
	+ /* Value is in 1/100th of a percentage of total bandwidth. */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID4_MAX_BW_BW_VALUE_UNIT_PERCENT1_100 (UINT32_C(0x1) << 29)
	+ /* Invalid unit */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID4_MAX_BW_BW_VALUE_UNIT_INVALID (UINT32_C(0x7) << 29)
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID4_MAX_BW_BW_VALUE_UNIT_LAST HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID4_MAX_BW_BW_VALUE_UNIT_INVALID
	uint8_t queue_id4_tsa_assign;
	/* Transmission Selection Algorithm (TSA) for CoS Queue. */
	/* Strict Priority */
	@@ -10261,16 +14012,68 @@
	/* ID of CoS Queue 5. */
	uint32_t queue_id5_min_bw;
	/*
	- * Minimum BW allocated to CoS Queue in Mbps. The HWRM will translate
	- * this value into byte counter and time interval used for this COS
	- * inside the device.
	+ * Minimum BW allocated to CoS Queue. The HWRM will translate this value
	+ * into byte counter and time interval used for this COS inside the
	+ * device.
	*/
	+ /* The bandwidth value. */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID5_MIN_BW_BW_VALUE_MASK UINT32_C(0xfffffff)
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID5_MIN_BW_BW_VALUE_SFT 0
	+ /* The granularity of the value (bits or bytes). */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID5_MIN_BW_SCALE UINT32_C(0x10000000)
	+ /* Value is in bits. */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID5_MIN_BW_SCALE_BITS (UINT32_C(0x0) << 28)
	+ /* Value is in bytes. */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID5_MIN_BW_SCALE_BYTES (UINT32_C(0x1) << 28)
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID5_MIN_BW_SCALE_LAST HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID5_MIN_BW_SCALE_BYTES
	+ /* bw_value_unit is 3 b */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID5_MIN_BW_BW_VALUE_UNIT_MASK UINT32_C(0xe0000000)
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID5_MIN_BW_BW_VALUE_UNIT_SFT 29
	+ /* Value is in Mb or MB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID5_MIN_BW_BW_VALUE_UNIT_MEGA (UINT32_C(0x0) << 29)
	+ /* Value is in Kb or KB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID5_MIN_BW_BW_VALUE_UNIT_KILO (UINT32_C(0x2) << 29)
	+ /* Value is in bits or bytes. */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID5_MIN_BW_BW_VALUE_UNIT_BASE (UINT32_C(0x4) << 29)
	+ /* Value is in Gb or GB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID5_MIN_BW_BW_VALUE_UNIT_GIGA (UINT32_C(0x6) << 29)
	+ /* Value is in 1/100th of a percentage of total bandwidth. */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID5_MIN_BW_BW_VALUE_UNIT_PERCENT1_100 (UINT32_C(0x1) << 29)
	+ /* Invalid unit */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID5_MIN_BW_BW_VALUE_UNIT_INVALID (UINT32_C(0x7) << 29)
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID5_MIN_BW_BW_VALUE_UNIT_LAST HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID5_MIN_BW_BW_VALUE_UNIT_INVALID
	uint32_t queue_id5_max_bw;
	/*
	- * Maximum BW allocated to CoS queue in Mbps. The HWRM will translate
	- * this value into byte counter and time interval used for this COS
	- * inside the device.
	+ * Maximum BW allocated to CoS queue. The HWRM will translate this value
	+ * into byte counter and time interval used for this COS inside the
	+ * device.
	*/
	+ /* The bandwidth value. */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID5_MAX_BW_BW_VALUE_MASK UINT32_C(0xfffffff)
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID5_MAX_BW_BW_VALUE_SFT 0
	+ /* The granularity of the value (bits or bytes). */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID5_MAX_BW_SCALE UINT32_C(0x10000000)
	+ /* Value is in bits. */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID5_MAX_BW_SCALE_BITS (UINT32_C(0x0) << 28)
	+ /* Value is in bytes. */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID5_MAX_BW_SCALE_BYTES (UINT32_C(0x1) << 28)
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID5_MAX_BW_SCALE_LAST HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID5_MAX_BW_SCALE_BYTES
	+ /* bw_value_unit is 3 b */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID5_MAX_BW_BW_VALUE_UNIT_MASK UINT32_C(0xe0000000)
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID5_MAX_BW_BW_VALUE_UNIT_SFT 29
	+ /* Value is in Mb or MB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID5_MAX_BW_BW_VALUE_UNIT_MEGA (UINT32_C(0x0) << 29)
	+ /* Value is in Kb or KB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID5_MAX_BW_BW_VALUE_UNIT_KILO (UINT32_C(0x2) << 29)
	+ /* Value is in bits or bytes. */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID5_MAX_BW_BW_VALUE_UNIT_BASE (UINT32_C(0x4) << 29)
	+ /* Value is in Gb or GB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID5_MAX_BW_BW_VALUE_UNIT_GIGA (UINT32_C(0x6) << 29)
	+ /* Value is in 1/100th of a percentage of total bandwidth. */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID5_MAX_BW_BW_VALUE_UNIT_PERCENT1_100 (UINT32_C(0x1) << 29)
	+ /* Invalid unit */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID5_MAX_BW_BW_VALUE_UNIT_INVALID (UINT32_C(0x7) << 29)
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID5_MAX_BW_BW_VALUE_UNIT_LAST HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID5_MAX_BW_BW_VALUE_UNIT_INVALID
	uint8_t queue_id5_tsa_assign;
	/* Transmission Selection Algorithm (TSA) for CoS Queue. */
	/* Strict Priority */
	@@ -10295,16 +14098,68 @@
	/* ID of CoS Queue 6. */
	uint32_t queue_id6_min_bw;
	/*
	- * Minimum BW allocated to CoS Queue in Mbps. The HWRM will translate
	- * this value into byte counter and time interval used for this COS
	- * inside the device.
	+ * Minimum BW allocated to CoS Queue. The HWRM will translate this value
	+ * into byte counter and time interval used for this COS inside the
	+ * device.
	*/
	+ /* The bandwidth value. */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID6_MIN_BW_BW_VALUE_MASK UINT32_C(0xfffffff)
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID6_MIN_BW_BW_VALUE_SFT 0
	+ /* The granularity of the value (bits or bytes). */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID6_MIN_BW_SCALE UINT32_C(0x10000000)
	+ /* Value is in bits. */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID6_MIN_BW_SCALE_BITS (UINT32_C(0x0) << 28)
	+ /* Value is in bytes. */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID6_MIN_BW_SCALE_BYTES (UINT32_C(0x1) << 28)
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID6_MIN_BW_SCALE_LAST HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID6_MIN_BW_SCALE_BYTES
	+ /* bw_value_unit is 3 b */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID6_MIN_BW_BW_VALUE_UNIT_MASK UINT32_C(0xe0000000)
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID6_MIN_BW_BW_VALUE_UNIT_SFT 29
	+ /* Value is in Mb or MB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID6_MIN_BW_BW_VALUE_UNIT_MEGA (UINT32_C(0x0) << 29)
	+ /* Value is in Kb or KB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID6_MIN_BW_BW_VALUE_UNIT_KILO (UINT32_C(0x2) << 29)
	+ /* Value is in bits or bytes. */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID6_MIN_BW_BW_VALUE_UNIT_BASE (UINT32_C(0x4) << 29)
	+ /* Value is in Gb or GB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID6_MIN_BW_BW_VALUE_UNIT_GIGA (UINT32_C(0x6) << 29)
	+ /* Value is in 1/100th of a percentage of total bandwidth. */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID6_MIN_BW_BW_VALUE_UNIT_PERCENT1_100 (UINT32_C(0x1) << 29)
	+ /* Invalid unit */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID6_MIN_BW_BW_VALUE_UNIT_INVALID (UINT32_C(0x7) << 29)
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID6_MIN_BW_BW_VALUE_UNIT_LAST HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID6_MIN_BW_BW_VALUE_UNIT_INVALID
	uint32_t queue_id6_max_bw;
	/*
	- * Maximum BW allocated to CoS queue in Mbps. The HWRM will translate
	- * this value into byte counter and time interval used for this COS
	- * inside the device.
	+ * Maximum BW allocated to CoS queue. The HWRM will translate this value
	+ * into byte counter and time interval used for this COS inside the
	+ * device.
	*/
	+ /* The bandwidth value. */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID6_MAX_BW_BW_VALUE_MASK UINT32_C(0xfffffff)
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID6_MAX_BW_BW_VALUE_SFT 0
	+ /* The granularity of the value (bits or bytes). */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID6_MAX_BW_SCALE UINT32_C(0x10000000)
	+ /* Value is in bits. */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID6_MAX_BW_SCALE_BITS (UINT32_C(0x0) << 28)
	+ /* Value is in bytes. */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID6_MAX_BW_SCALE_BYTES (UINT32_C(0x1) << 28)
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID6_MAX_BW_SCALE_LAST HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID6_MAX_BW_SCALE_BYTES
	+ /* bw_value_unit is 3 b */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID6_MAX_BW_BW_VALUE_UNIT_MASK UINT32_C(0xe0000000)
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID6_MAX_BW_BW_VALUE_UNIT_SFT 29
	+ /* Value is in Mb or MB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID6_MAX_BW_BW_VALUE_UNIT_MEGA (UINT32_C(0x0) << 29)
	+ /* Value is in Kb or KB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID6_MAX_BW_BW_VALUE_UNIT_KILO (UINT32_C(0x2) << 29)
	+ /* Value is in bits or bytes. */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID6_MAX_BW_BW_VALUE_UNIT_BASE (UINT32_C(0x4) << 29)
	+ /* Value is in Gb or GB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID6_MAX_BW_BW_VALUE_UNIT_GIGA (UINT32_C(0x6) << 29)
	+ /* Value is in 1/100th of a percentage of total bandwidth. */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID6_MAX_BW_BW_VALUE_UNIT_PERCENT1_100 (UINT32_C(0x1) << 29)
	+ /* Invalid unit */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID6_MAX_BW_BW_VALUE_UNIT_INVALID (UINT32_C(0x7) << 29)
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID6_MAX_BW_BW_VALUE_UNIT_LAST HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID6_MAX_BW_BW_VALUE_UNIT_INVALID
	uint8_t queue_id6_tsa_assign;
	/* Transmission Selection Algorithm (TSA) for CoS Queue. */
	/* Strict Priority */
	@@ -10329,16 +14184,68 @@
	/* ID of CoS Queue 7. */
	uint32_t queue_id7_min_bw;
	/*
	- * Minimum BW allocated to CoS Queue in Mbps. The HWRM will translate
	- * this value into byte counter and time interval used for this COS
	- * inside the device.
	+ * Minimum BW allocated to CoS Queue. The HWRM will translate this value
	+ * into byte counter and time interval used for this COS inside the
	+ * device.
	*/
	+ /* The bandwidth value. */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID7_MIN_BW_BW_VALUE_MASK UINT32_C(0xfffffff)
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID7_MIN_BW_BW_VALUE_SFT 0
	+ /* The granularity of the value (bits or bytes). */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID7_MIN_BW_SCALE UINT32_C(0x10000000)
	+ /* Value is in bits. */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID7_MIN_BW_SCALE_BITS (UINT32_C(0x0) << 28)
	+ /* Value is in bytes. */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID7_MIN_BW_SCALE_BYTES (UINT32_C(0x1) << 28)
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID7_MIN_BW_SCALE_LAST HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID7_MIN_BW_SCALE_BYTES
	+ /* bw_value_unit is 3 b */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID7_MIN_BW_BW_VALUE_UNIT_MASK UINT32_C(0xe0000000)
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID7_MIN_BW_BW_VALUE_UNIT_SFT 29
	+ /* Value is in Mb or MB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID7_MIN_BW_BW_VALUE_UNIT_MEGA (UINT32_C(0x0) << 29)
	+ /* Value is in Kb or KB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID7_MIN_BW_BW_VALUE_UNIT_KILO (UINT32_C(0x2) << 29)
	+ /* Value is in bits or bytes. */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID7_MIN_BW_BW_VALUE_UNIT_BASE (UINT32_C(0x4) << 29)
	+ /* Value is in Gb or GB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID7_MIN_BW_BW_VALUE_UNIT_GIGA (UINT32_C(0x6) << 29)
	+ /* Value is in 1/100th of a percentage of total bandwidth. */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID7_MIN_BW_BW_VALUE_UNIT_PERCENT1_100 (UINT32_C(0x1) << 29)
	+ /* Invalid unit */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID7_MIN_BW_BW_VALUE_UNIT_INVALID (UINT32_C(0x7) << 29)
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID7_MIN_BW_BW_VALUE_UNIT_LAST HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID7_MIN_BW_BW_VALUE_UNIT_INVALID
	uint32_t queue_id7_max_bw;
	/*
	- * Maximum BW allocated to CoS queue in Mbps. The HWRM will translate
	- * this value into byte counter and time interval used for this COS
	- * inside the device.
	+ * Maximum BW allocated to CoS queue. The HWRM will translate this value
	+ * into byte counter and time interval used for this COS inside the
	+ * device.
	*/
	+ /* The bandwidth value. */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID7_MAX_BW_BW_VALUE_MASK UINT32_C(0xfffffff)
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID7_MAX_BW_BW_VALUE_SFT 0
	+ /* The granularity of the value (bits or bytes). */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID7_MAX_BW_SCALE UINT32_C(0x10000000)
	+ /* Value is in bits. */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID7_MAX_BW_SCALE_BITS (UINT32_C(0x0) << 28)
	+ /* Value is in bytes. */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID7_MAX_BW_SCALE_BYTES (UINT32_C(0x1) << 28)
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID7_MAX_BW_SCALE_LAST HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID7_MAX_BW_SCALE_BYTES
	+ /* bw_value_unit is 3 b */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID7_MAX_BW_BW_VALUE_UNIT_MASK UINT32_C(0xe0000000)
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID7_MAX_BW_BW_VALUE_UNIT_SFT 29
	+ /* Value is in Mb or MB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID7_MAX_BW_BW_VALUE_UNIT_MEGA (UINT32_C(0x0) << 29)
	+ /* Value is in Kb or KB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID7_MAX_BW_BW_VALUE_UNIT_KILO (UINT32_C(0x2) << 29)
	+ /* Value is in bits or bytes. */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID7_MAX_BW_BW_VALUE_UNIT_BASE (UINT32_C(0x4) << 29)
	+ /* Value is in Gb or GB (base 10). */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID7_MAX_BW_BW_VALUE_UNIT_GIGA (UINT32_C(0x6) << 29)
	+ /* Value is in 1/100th of a percentage of total bandwidth. */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID7_MAX_BW_BW_VALUE_UNIT_PERCENT1_100 (UINT32_C(0x1) << 29)
	+ /* Invalid unit */
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID7_MAX_BW_BW_VALUE_UNIT_INVALID (UINT32_C(0x7) << 29)
	+ #define HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID7_MAX_BW_BW_VALUE_UNIT_LAST HWRM_QUEUE_COS2BW_CFG_INPUT_QUEUE_ID7_MAX_BW_BW_VALUE_UNIT_INVALID
	uint8_t queue_id7_tsa_assign;
	/* Transmission Selection Algorithm (TSA) for CoS Queue. */
	/* Strict Priority */
	@@ -10394,6 +14301,273 @@
	*/
	} __attribute__((packed));

	+/* hwrm_queue_dscp_qcaps */
	+/*
	+ * Description: This command is called by a driver to query the DSCP
	+ * capabilities for a port.
	+ */
	+/* Input (24 bytes) */
	+
	+struct hwrm_queue_dscp_qcaps_input {
	+ uint16_t req_type;
	+ /*
	+ * This value indicates what type of request this is. The format for the
	+ * rest of the command is determined by this field.
	+ */
	+ uint16_t cmpl_ring;
	+ /*
	+ * This value indicates the what completion ring the request will be
	+ * optionally completed on. If the value is -1, then no CR completion
	+ * will be generated. Any other value must be a valid CR ring_id value
	+ * for this function.
	+ */
	+ uint16_t seq_id;
	+ /* This value indicates the command sequence number. */
	+ uint16_t target_id;
	+ /*
	+ * Target ID of this command. 0x0 - 0xFFF8 - Used for function ids
	+ * 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF - HWRM
	+ */
	+ uint64_t resp_addr;
	+ /*
	+ * This is the host address where the response will be written when the
	+ * request is complete. This area must be 16B aligned and must be
	+ * cleared to zero before the request is made.
	+ */
	+ uint8_t port_id;
	+ /*
	+ * Port ID of port for which the table is being configured. The HWRM
	+ * needs to check whether this function is allowed to configure pri2cos
	+ * mapping on this port.
	+ */
	+ uint8_t unused_0[7];
	+} __attribute__((packed));
	+
	+/* Output (16 bytes) */
	+
	+struct hwrm_queue_dscp_qcaps_output {
	+ uint16_t error_code;
	+ /*
	+ * Pass/Fail or error type Note: receiver to verify the in parameters,
	+ * and fail the call with an error when appropriate
	+ */
	+ uint16_t req_type;
	+ /* This field returns the type of original request. */
	+ uint16_t seq_id;
	+ /* This field provides original sequence number of the command. */
	+ uint16_t resp_len;
	+ /*
	+ * This field is the length of the response in bytes. The last byte of
	+ * the response is a valid flag that will read as '1' when the command
	+ * has been completely written to memory.
	+ */
	+ uint8_t num_dscp_bits;
	+ /* The number of bits provided by the hardware for the DSCP value. */
	+ uint8_t unused_0;
	+ uint16_t max_entries;
	+ /* Max number of DSCP-MASK-PRI entries supported. */
	+ uint8_t unused_1;
	+ uint8_t unused_2;
	+ uint8_t unused_3;
	+ uint8_t valid;
	+ /*
	+ * This field is used in Output records to indicate that the output is
	+ * completely written to RAM. This field should be read as '1' to
	+ * indicate that the output has been completely written. When writing a
	+ * command completion or response to an internal processor, the order of
	+ * writes has to be such that this field is written last.
	+ */
	+} __attribute__((packed));
	+
	+/* hwrm_queue_dscp2pri_qcfg */
	+/*
	+ * Description: This command is called by a driver to query configuration of the
	+ * DSCP to PRI mapping on the receive side. This mapping can be different on
	+ * different ports.
	+ */
	+/* Input (32 bytes) */
	+
	+struct hwrm_queue_dscp2pri_qcfg_input {
	+ uint16_t req_type;
	+ /*
	+ * This value indicates what type of request this is. The format for the
	+ * rest of the command is determined by this field.
	+ */
	+ uint16_t cmpl_ring;
	+ /*
	+ * This value indicates the what completion ring the request will be
	+ * optionally completed on. If the value is -1, then no CR completion
	+ * will be generated. Any other value must be a valid CR ring_id value
	+ * for this function.
	+ */
	+ uint16_t seq_id;
	+ /* This value indicates the command sequence number. */
	+ uint16_t target_id;
	+ /*
	+ * Target ID of this command. 0x0 - 0xFFF8 - Used for function ids
	+ * 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF - HWRM
	+ */
	+ uint64_t resp_addr;
	+ /*
	+ * This is the host address where the response will be written when the
	+ * request is complete. This area must be 16B aligned and must be
	+ * cleared to zero before the request is made.
	+ */
	+ uint64_t dest_data_addr;
	+ /*
	+ * This is the host address where the 24-bits DSCP-MASK-PRI tuple(s)
	+ * will be copied to.
	+ */
	+ uint8_t port_id;
	+ /*
	+ * Port ID of port for which the table is being configured. The HWRM
	+ * needs to check whether this function is allowed to configure pri2cos
	+ * mapping on this port.
	+ */
	+ uint8_t unused_0;
	+ uint16_t dest_data_buffer_size;
	+ /* Size of the buffer pointed to by dest_data_addr. */
	+ uint32_t unused_1;
	+} __attribute__((packed));
	+
	+/* Output (16 bytes) */
	+
	+struct hwrm_queue_dscp2pri_qcfg_output {
	+ uint16_t error_code;
	+ /*
	+ * Pass/Fail or error type Note: receiver to verify the in parameters,
	+ * and fail the call with an error when appropriate
	+ */
	+ uint16_t req_type;
	+ /* This field returns the type of original request. */
	+ uint16_t seq_id;
	+ /* This field provides original sequence number of the command. */
	+ uint16_t resp_len;
	+ /*
	+ * This field is the length of the response in bytes. The last byte of
	+ * the response is a valid flag that will read as '1' when the command
	+ * has been completely written to memory.
	+ */
	+ uint16_t entry_cnt;
	+ /*
	+ * A count of the number of DSCP-MASK-PRI tuple(s) pointed to by the
	+ * dest_data_addr.
	+ */
	+ uint8_t default_pri;
	+ /*
	+ * This is the default PRI which un-initialized DSCP values are mapped
	+ * to.
	+ */
	+ uint8_t unused_0;
	+ uint8_t unused_1;
	+ uint8_t unused_2;
	+ uint8_t unused_3;
	+ uint8_t valid;
	+ /*
	+ * This field is used in Output records to indicate that the output is
	+ * completely written to RAM. This field should be read as '1' to
	+ * indicate that the output has been completely written. When writing a
	+ * command completion or response to an internal processor, the order of
	+ * writes has to be such that this field is written last.
	+ */
	+} __attribute__((packed));
	+
	+/* hwrm_queue_dscp2pri_cfg */
	+/*
	+ * Description: This command is called by a driver to configure the DSCP to PRI
	+ * mapping on the receive side. This mapping can be different on different
	+ * ports.
	+ */
	+/* Input (40 bytes) */
	+
	+struct hwrm_queue_dscp2pri_cfg_input {
	+ uint16_t req_type;
	+ /*
	+ * This value indicates what type of request this is. The format for the
	+ * rest of the command is determined by this field.
	+ */
	+ uint16_t cmpl_ring;
	+ /*
	+ * This value indicates the what completion ring the request will be
	+ * optionally completed on. If the value is -1, then no CR completion
	+ * will be generated. Any other value must be a valid CR ring_id value
	+ * for this function.
	+ */
	+ uint16_t seq_id;
	+ /* This value indicates the command sequence number. */
	+ uint16_t target_id;
	+ /*
	+ * Target ID of this command. 0x0 - 0xFFF8 - Used for function ids
	+ * 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF - HWRM
	+ */
	+ uint64_t resp_addr;
	+ /*
	+ * This is the host address where the response will be written when the
	+ * request is complete. This area must be 16B aligned and must be
	+ * cleared to zero before the request is made.
	+ */
	+ uint64_t src_data_addr;
	+ /*
	+ * This is the host address where the 24-bits DSCP-MASK-PRI tuple will
	+ * be copied from.
	+ */
	+ uint32_t flags;
	+ /* use_hw_default_pri is 1 b */
	+ #define HWRM_QUEUE_DSCP2PRI_CFG_INPUT_FLAGS_USE_HW_DEFAULT_PRI UINT32_C(0x1)
	+ uint32_t enables;
	+ /* This bit must be '1' for the default_pri field to be configured. */
	+ #define HWRM_QUEUE_DSCP2PRI_CFG_INPUT_ENABLES_DEFAULT_PRI UINT32_C(0x1)
	+ uint8_t port_id;
	+ /*
	+ * Port ID of port for which the table is being configured. The HWRM
	+ * needs to check whether this function is allowed to configure pri2cos
	+ * mapping on this port.
	+ */
	+ uint8_t default_pri;
	+ /*
	+ * This is the default PRI which un-initialized DSCP values will be
	+ * mapped to.
	+ */
	+ uint16_t entry_cnt;
	+ /*
	+ * A count of the number of DSCP-MASK-PRI tuple(s) in the data pointed
	+ * to by src_data_addr.
	+ */
	+ uint32_t unused_0;
	+} __attribute__((packed));
	+
	+/* Output (16 bytes) */
	+
	+struct hwrm_queue_dscp2pri_cfg_output {
	+ uint16_t error_code;
	+ /*
	+ * Pass/Fail or error type Note: receiver to verify the in parameters,
	+ * and fail the call with an error when appropriate
	+ */
	+ uint16_t req_type;
	+ /* This field returns the type of original request. */
	+ uint16_t seq_id;
	+ /* This field provides original sequence number of the command. */
	+ uint16_t resp_len;
	+ /*
	+ * This field is the length of the response in bytes. The last byte of
	+ * the response is a valid flag that will read as '1' when the command
	+ * has been completely written to memory.
	+ */
	+ uint32_t unused_0;
	+ uint8_t unused_1;
	+ uint8_t unused_2;
	+ uint8_t unused_3;
	+ uint8_t valid;
	+ /*
	+ * This field is used in Output records to indicate that the output is
	+ * completely written to RAM. This field should be read as '1' to
	+ * indicate that the output has been completely written. When writing a
	+ * command completion or response to an internal processor, the order of
	+ * writes has to be such that this field is written last.
	+ */
	+} __attribute__((packed));
	+
	/* hwrm_vnic_alloc */
	/*
	* Description: This VNIC is a resource in the RX side of the chip that is used
	@@ -10900,6 +15074,7 @@
	uint8_t unused_1;
	uint32_t flags;
	/* Unused. */
	+ #define HWRM_VNIC_QCAPS_OUTPUT_FLAGS_UNUSED UINT32_C(0x1)
	/*
	* When this bit is '1', the capability of stripping VLAN in the RX path
	* is supported on VNIC(s). If set to '0', then VLAN stripping
	@@ -11937,8 +16112,8 @@
	uint32_t enables;
	/* This bit must be '1' for the Reserved1 field to be configured. */
	#define HWRM_RING_ALLOC_INPUT_ENABLES_RESERVED1 UINT32_C(0x1)
	- /* This bit must be '1' for the Reserved2 field to be configured. */
	- #define HWRM_RING_ALLOC_INPUT_ENABLES_RESERVED2 UINT32_C(0x2)
	+ /* This bit must be '1' for the ring_arb_cfg field to be configured. */
	+ #define HWRM_RING_ALLOC_INPUT_ENABLES_RING_ARB_CFG UINT32_C(0x2)
	/* This bit must be '1' for the Reserved3 field to be configured. */
	#define HWRM_RING_ALLOC_INPUT_ENABLES_RESERVED3 UINT32_C(0x4)
	/*
	@@ -11952,12 +16127,14 @@
	#define HWRM_RING_ALLOC_INPUT_ENABLES_MAX_BW_VALID UINT32_C(0x20)
	uint8_t ring_type;
	/* Ring Type. */
	- /* Completion Ring (CR) */
	- #define HWRM_RING_ALLOC_INPUT_RING_TYPE_CMPL UINT32_C(0x0)
	+ /* L2 Completion Ring (CR) */
	+ #define HWRM_RING_ALLOC_INPUT_RING_TYPE_L2_CMPL UINT32_C(0x0)
	/* TX Ring (TR) */
	#define HWRM_RING_ALLOC_INPUT_RING_TYPE_TX UINT32_C(0x1)
	/* RX Ring (RR) */
	#define HWRM_RING_ALLOC_INPUT_RING_TYPE_RX UINT32_C(0x2)
	+ /* RoCE Notification Completion Ring (ROCE_CR) */
	+ #define HWRM_RING_ALLOC_INPUT_RING_TYPE_ROCE_CMPL UINT32_C(0x3)
	uint8_t unused_0;
	uint16_t unused_1;
	uint64_t page_tbl_addr;
	@@ -12010,8 +16187,39 @@
	uint8_t unused_5;
	uint32_t reserved1;
	/* This field is reserved for the future use. It shall be set to 0. */
	- uint16_t reserved2;
	- /* This field is reserved for the future use. It shall be set to 0. */
	+ uint16_t ring_arb_cfg;
	+ /*
	+ * This field is used only when ring_type is a TX ring. This field is
	+ * used to configure arbitration related parameters for a TX ring.
	+ */
	+ /* Arbitration policy used for the ring. */
	+ #define HWRM_RING_ALLOC_INPUT_RING_ARB_CFG_ARB_POLICY_MASK UINT32_C(0xf)
	+ #define HWRM_RING_ALLOC_INPUT_RING_ARB_CFG_ARB_POLICY_SFT 0
	+ /*
	+ * Use strict priority for the TX ring. Priority value is
	+ * specified in arb_policy_param
	+ */
	+ #define HWRM_RING_ALLOC_INPUT_RING_ARB_CFG_ARB_POLICY_SP (UINT32_C(0x1) << 0)
	+ /*
	+ * Use weighted fair queue arbitration for the TX ring. Weight
	+ * is specified in arb_policy_param
	+ */
	+ #define HWRM_RING_ALLOC_INPUT_RING_ARB_CFG_ARB_POLICY_WFQ (UINT32_C(0x2) << 0)
	+ #define HWRM_RING_ALLOC_INPUT_RING_ARB_CFG_ARB_POLICY_LAST HWRM_RING_ALLOC_INPUT_RING_ARB_CFG_ARB_POLICY_WFQ
	+ /* Reserved field. */
	+ #define HWRM_RING_ALLOC_INPUT_RING_ARB_CFG_RSVD_MASK UINT32_C(0xf0)
	+ #define HWRM_RING_ALLOC_INPUT_RING_ARB_CFG_RSVD_SFT 4
	+ /*
	+ * Arbitration policy specific parameter. # For strict priority
	+ * arbitration policy, this field represents a priority value. If set to
	+ * 0, then the priority is not specified and the HWRM is allowed to
	+ * select any priority for this TX ring. # For weighted fair queue
	+ * arbitration policy, this field represents a weight value. If set to
	+ * 0, then the weight is not specified and the HWRM is allowed to select
	+ * any weight for this TX ring.
	+ */
	+ #define HWRM_RING_ALLOC_INPUT_RING_ARB_CFG_ARB_POLICY_PARAM_MASK UINT32_C(0xff00)
	+ #define HWRM_RING_ALLOC_INPUT_RING_ARB_CFG_ARB_POLICY_PARAM_SFT 8
	uint8_t unused_6;
	uint8_t unused_7;
	uint32_t reserved3;
	@@ -12026,11 +16234,37 @@
	/* This field is reserved for the future use. It shall be set to 0. */
	uint32_t max_bw;
	/*
	- * This field is used only when ring_type is a TX ring. Maximum BW
	- * allocated to this TX ring in Mbps. The HWRM will translate this value
	- * into byte counter and time interval used for this ring inside the
	- * device.
	+ * This field is used only when ring_type is a TX ring to specify
	+ * maximum BW allocated to the TX ring. The HWRM will translate this
	+ * value into byte counter and time interval used for this ring inside
	+ * the device.
	*/
	+ /* The bandwidth value. */
	+ #define HWRM_RING_ALLOC_INPUT_MAX_BW_BW_VALUE_MASK UINT32_C(0xfffffff)
	+ #define HWRM_RING_ALLOC_INPUT_MAX_BW_BW_VALUE_SFT 0
	+ /* The granularity of the value (bits or bytes). */
	+ #define HWRM_RING_ALLOC_INPUT_MAX_BW_SCALE UINT32_C(0x10000000)
	+ /* Value is in bits. */
	+ #define HWRM_RING_ALLOC_INPUT_MAX_BW_SCALE_BITS (UINT32_C(0x0) << 28)
	+ /* Value is in bytes. */
	+ #define HWRM_RING_ALLOC_INPUT_MAX_BW_SCALE_BYTES (UINT32_C(0x1) << 28)
	+ #define HWRM_RING_ALLOC_INPUT_MAX_BW_SCALE_LAST HWRM_RING_ALLOC_INPUT_MAX_BW_SCALE_BYTES
	+ /* bw_value_unit is 3 b */
	+ #define HWRM_RING_ALLOC_INPUT_MAX_BW_BW_VALUE_UNIT_MASK UINT32_C(0xe0000000)
	+ #define HWRM_RING_ALLOC_INPUT_MAX_BW_BW_VALUE_UNIT_SFT 29
	+ /* Value is in Mb or MB (base 10). */
	+ #define HWRM_RING_ALLOC_INPUT_MAX_BW_BW_VALUE_UNIT_MEGA (UINT32_C(0x0) << 29)
	+ /* Value is in Kb or KB (base 10). */
	+ #define HWRM_RING_ALLOC_INPUT_MAX_BW_BW_VALUE_UNIT_KILO (UINT32_C(0x2) << 29)
	+ /* Value is in bits or bytes. */
	+ #define HWRM_RING_ALLOC_INPUT_MAX_BW_BW_VALUE_UNIT_BASE (UINT32_C(0x4) << 29)
	+ /* Value is in Gb or GB (base 10). */
	+ #define HWRM_RING_ALLOC_INPUT_MAX_BW_BW_VALUE_UNIT_GIGA (UINT32_C(0x6) << 29)
	+ /* Value is in 1/100th of a percentage of total bandwidth. */
	+ #define HWRM_RING_ALLOC_INPUT_MAX_BW_BW_VALUE_UNIT_PERCENT1_100 (UINT32_C(0x1) << 29)
	+ /* Invalid unit */
	+ #define HWRM_RING_ALLOC_INPUT_MAX_BW_BW_VALUE_UNIT_INVALID (UINT32_C(0x7) << 29)
	+ #define HWRM_RING_ALLOC_INPUT_MAX_BW_BW_VALUE_UNIT_LAST HWRM_RING_ALLOC_INPUT_MAX_BW_BW_VALUE_UNIT_INVALID
	uint8_t int_mode;
	/*
	* This field is used only when ring_type is a Completion ring. This
	@@ -12068,7 +16302,10 @@
	* has been completely written to memory.
	*/
	uint16_t ring_id;
	- /* Physical number of ring allocated. */
	+ /*
	+ * Physical number of ring allocated. This value shall be unique for a
	+ * ring type.
	+ */
	uint16_t logical_ring_id;
	/* Logical number of ring allocated. */
	uint8_t unused_0;
	@@ -12087,6 +16324,14 @@
	/* hwrm_ring_free */
	/*
	* Description: This command is used to free a ring and associated resources.
	+ * With QoS and DCBx agents, it is possible the traffic classes will be moved
	+ * from one CoS queue to another. When this occurs, the driver shall call
	+ * 'hwrm_ring_free' to free the allocated rings and then call 'hwrm_ring_alloc'
	+ * to re-allocate each ring and assign it to a new CoS queue. hwrm_ring_free
	+ * shall be called on a ring only after it has been idle for 500ms or more and
	+ * no frames have been posted to the ring during this time. All frames queued
	+ * for transmission shall be completed and at least 500ms time elapsed from the
	+ * last completion before calling this command.
	*/
	/* Input (24 bytes) */

	@@ -12118,12 +16363,14 @@
	*/
	uint8_t ring_type;
	/* Ring Type. */
	- /* Completion Ring (CR) */
	- #define HWRM_RING_FREE_INPUT_RING_TYPE_CMPL UINT32_C(0x0)
	+ /* L2 Completion Ring (CR) */
	+ #define HWRM_RING_FREE_INPUT_RING_TYPE_L2_CMPL UINT32_C(0x0)
	/* TX Ring (TR) */
	#define HWRM_RING_FREE_INPUT_RING_TYPE_TX UINT32_C(0x1)
	/* RX Ring (RR) */
	#define HWRM_RING_FREE_INPUT_RING_TYPE_RX UINT32_C(0x2)
	+ /* RoCE Notification Completion Ring (ROCE_CR) */
	+ #define HWRM_RING_FREE_INPUT_RING_TYPE_ROCE_CMPL UINT32_C(0x3)
	uint8_t unused_0;
	uint16_t ring_id;
	/* Physical number of ring allocated. */
	@@ -12418,12 +16665,14 @@
	*/
	uint8_t ring_type;
	/* Ring Type. */
	- /* Completion Ring (CR) */
	- #define HWRM_RING_RESET_INPUT_RING_TYPE_CMPL UINT32_C(0x0)
	+ /* L2 Completion Ring (CR) */
	+ #define HWRM_RING_RESET_INPUT_RING_TYPE_L2_CMPL UINT32_C(0x0)
	/* TX Ring (TR) */
	#define HWRM_RING_RESET_INPUT_RING_TYPE_TX UINT32_C(0x1)
	/* RX Ring (RR) */
	#define HWRM_RING_RESET_INPUT_RING_TYPE_RX UINT32_C(0x2)
	+ /* RoCE Notification Completion Ring (ROCE_CR) */
	+ #define HWRM_RING_RESET_INPUT_RING_TYPE_ROCE_CMPL UINT32_C(0x3)
	uint8_t unused_0;
	uint16_t ring_id;
	/* Physical number of the ring. */
	@@ -13281,6 +17530,191 @@
	*/
	} __attribute__((packed));

	+/* Command specific Error Codes (8 bytes) */
	+
	+struct hwrm_cfa_l2_set_rx_mask_cmd_err {
	+ uint8_t code;
	+ /*
	+ * command specific error codes that goes to the cmd_err field in Common
	+ * HWRM Error Response.
	+ */
	+ /* Unknown error */
	+ #define HWRM_CFA_L2_SET_RX_MASK_CMD_ERR_CODE_UNKNOWN UINT32_C(0x0)
	+ /* Unable to complete operation due to conflict with Ntuple Filter */
	+ #define HWRM_CFA_L2_SET_RX_MASK_CMD_ERR_CODE_NTUPLE_FILTER_CONFLICT_ERR UINT32_C(0x1)
	+ uint8_t unused_0[7];
	+} __attribute__((packed));
	+
	+/* hwrm_cfa_vlan_antispoof_cfg */
	+/* Description: Configures vlan anti-spoof filters for VF. */
	+/* Input (32 bytes) */
	+
	+struct hwrm_cfa_vlan_antispoof_cfg_input {
	+ uint16_t req_type;
	+ /*
	+ * This value indicates what type of request this is. The format for the
	+ * rest of the command is determined by this field.
	+ */
	+ uint16_t cmpl_ring;
	+ /*
	+ * This value indicates the what completion ring the request will be
	+ * optionally completed on. If the value is -1, then no CR completion
	+ * will be generated. Any other value must be a valid CR ring_id value
	+ * for this function.
	+ */
	+ uint16_t seq_id;
	+ /* This value indicates the command sequence number. */
	+ uint16_t target_id;
	+ /*
	+ * Target ID of this command. 0x0 - 0xFFF8 - Used for function ids
	+ * 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF - HWRM
	+ */
	+ uint64_t resp_addr;
	+ /*
	+ * This is the host address where the response will be written when the
	+ * request is complete. This area must be 16B aligned and must be
	+ * cleared to zero before the request is made.
	+ */
	+ uint16_t fid;
	+ /*
	+ * Function ID of the function that is being configured. Only valid for
	+ * a VF FID configured by the PF.
	+ */
	+ uint8_t unused_0;
	+ uint8_t unused_1;
	+ uint32_t num_vlan_entries;
	+ /* Number of VLAN entries in the vlan_tag_mask_tbl. */
	+ uint64_t vlan_tag_mask_tbl_addr;
	+ /*
	+ * The vlan_tag_mask_tbl_addr is the DMA address of the VLAN antispoof
	+ * table. Each table entry contains the 16-bit TPID (0x8100 or 0x88a8
	+ * only), 16-bit VLAN ID, and a 16-bit mask, all in network order to
	+ * match hwrm_cfa_l2_set_rx_mask. For an individual VLAN entry, the mask
	+ * value should be 0xfff for the 12-bit VLAN ID.
	+ */
	+} __attribute__((packed));
	+
	+/* Output (16 bytes) */
	+
	+struct hwrm_cfa_vlan_antispoof_cfg_output {
	+ uint16_t error_code;
	+ /*
	+ * Pass/Fail or error type Note: receiver to verify the in parameters,
	+ * and fail the call with an error when appropriate
	+ */
	+ uint16_t req_type;
	+ /* This field returns the type of original request. */
	+ uint16_t seq_id;
	+ /* This field provides original sequence number of the command. */
	+ uint16_t resp_len;
	+ /*
	+ * This field is the length of the response in bytes. The last byte of
	+ * the response is a valid flag that will read as '1' when the command
	+ * has been completely written to memory.
	+ */
	+ uint32_t unused_0;
	+ uint8_t unused_1;
	+ uint8_t unused_2;
	+ uint8_t unused_3;
	+ uint8_t valid;
	+ /*
	+ * This field is used in Output records to indicate that the output is
	+ * completely written to RAM. This field should be read as '1' to
	+ * indicate that the output has been completely written. When writing a
	+ * command completion or response to an internal processor, the order of
	+ * writes has to be such that this field is written last.
	+ */
	+} __attribute__((packed));
	+
	+/* hwrm_cfa_vlan_antispoof_qcfg */
	+/*
	+ * Description: Returns the current configuration of the vlan anti-spoof filters
	+ * for VF.
	+ */
	+/* Input (32 bytes) */
	+
	+struct hwrm_cfa_vlan_antispoof_qcfg_input {
	+ uint16_t req_type;
	+ /*
	+ * This value indicates what type of request this is. The format for the
	+ * rest of the command is determined by this field.
	+ */
	+ uint16_t cmpl_ring;
	+ /*
	+ * This value indicates the what completion ring the request will be
	+ * optionally completed on. If the value is -1, then no CR completion
	+ * will be generated. Any other value must be a valid CR ring_id value
	+ * for this function.
	+ */
	+ uint16_t seq_id;
	+ /* This value indicates the command sequence number. */
	+ uint16_t target_id;
	+ /*
	+ * Target ID of this command. 0x0 - 0xFFF8 - Used for function ids
	+ * 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF - HWRM
	+ */
	+ uint64_t resp_addr;
	+ /*
	+ * This is the host address where the response will be written when the
	+ * request is complete. This area must be 16B aligned and must be
	+ * cleared to zero before the request is made.
	+ */
	+ uint16_t fid;
	+ /*
	+ * Function ID of the function that is being queried. Only valid for a
	+ * VF FID queried by the PF.
	+ */
	+ uint8_t unused_0;
	+ uint8_t unused_1;
	+ uint32_t max_vlan_entries;
	+ /*
	+ * Maximum number of VLAN entries the firmware is allowed to DMA to
	+ * vlan_tag_mask_tbl.
	+ */
	+ uint64_t vlan_tag_mask_tbl_addr;
	+ /*
	+ * The vlan_tag_mask_tbl_addr is the DMA address of the VLAN antispoof
	+ * table to which firmware will DMA to. Each table entry will contain
	+ * the 16-bit TPID (0x8100 or 0x88a8 only), 16-bit VLAN ID, and a 16-bit
	+ * mask, all in network order to match hwrm_cfa_l2_set_rx_mask. For an
	+ * individual VLAN entry, the mask value should be 0xfff for the 12-bit
	+ * VLAN ID.
	+ */
	+} __attribute__((packed));
	+
	+/* Output (16 bytes) */
	+
	+struct hwrm_cfa_vlan_antispoof_qcfg_output {
	+ uint16_t error_code;
	+ /*
	+ * Pass/Fail or error type Note: receiver to verify the in parameters,
	+ * and fail the call with an error when appropriate
	+ */
	+ uint16_t req_type;
	+ /* This field returns the type of original request. */
	+ uint16_t seq_id;
	+ /* This field provides original sequence number of the command. */
	+ uint16_t resp_len;
	+ /*
	+ * This field is the length of the response in bytes. The last byte of
	+ * the response is a valid flag that will read as '1' when the command
	+ * has been completely written to memory.
	+ */
	+ uint32_t num_vlan_entries;
	+ /* Number of valid entries DMAd by firmware to vlan_tag_mask_tbl. */
	+ uint8_t unused_0;
	+ uint8_t unused_1;
	+ uint8_t unused_2;
	+ uint8_t valid;
	+ /*
	+ * This field is used in Output records to indicate that the output is
	+ * completely written to RAM. This field should be read as '1' to
	+ * indicate that the output has been completely written. When writing a
	+ * command completion or response to an internal processor, the order of
	+ * writes has to be such that this field is written last.
	+ */
	+} __attribute__((packed));
	+
	/* hwrm_cfa_tunnel_filter_alloc */
	/*
	* Description: This is a tunnel filter that uses fields from tunnel header in
	@@ -13582,7 +18016,7 @@
	#define HWRM_CFA_ENCAP_RECORD_ALLOC_INPUT_ENCAP_TYPE_IPGRE UINT32_C(0x8)
	uint8_t unused_0;
	uint16_t unused_1;
	- uint32_t encap_data[16];
	+ uint32_t encap_data[20];
	/* This value is encap data used for the given encap type. */
	} __attribute__((packed));

	@@ -13735,6 +18169,12 @@
	* then it should be considered accept action.
	*/
	#define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_FLAGS_DROP UINT32_C(0x2)
	+ /*
	+ * Setting of this flag indicates that a meter is expected to be
	+ * attached to this flow. This hint can be used when choosing the action
	+ * record format required for the flow.
	+ */
	+ #define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_FLAGS_METER UINT32_C(0x4)
	uint32_t enables;
	/* This bit must be '1' for the l2_filter_id field to be configured. */
	#define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_L2_FILTER_ID UINT32_C(0x1)
	@@ -13797,14 +18237,14 @@
	uint8_t ip_protocol;
	/*
	* The value of protocol filed in IP header. Applies to UDP and TCP
	- * traffic. 6 - UDP 17 - TCP
	+ * traffic. 6 - TCP 17 - UDP
	*/
	/* invalid */
	#define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_IP_PROTOCOL_UNKNOWN UINT32_C(0x0)
	- /* UDP */
	- #define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_IP_PROTOCOL_UDP UINT32_C(0x6)
	/* TCP */
	- #define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_IP_PROTOCOL_TCP UINT32_C(0x11)
	+ #define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_IP_PROTOCOL_TCP UINT32_C(0x6)
	+ /* UDP */
	+ #define HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_IP_PROTOCOL_UDP UINT32_C(0x11)
	uint16_t dst_id;
	/*
	* If set, this value shall represent the Logical VNIC ID of the
	@@ -13939,6 +18379,21 @@
	*/
	} __attribute__((packed));

	+/* Command specific Error Codes (8 bytes) */
	+
	+struct hwrm_cfa_ntuple_filter_alloc_cmd_err {
	+ uint8_t code;
	+ /*
	+ * command specific error codes that goes to the cmd_err field in Common
	+ * HWRM Error Response.
	+ */
	+ /* Unknown error */
	+ #define HWRM_CFA_NTUPLE_FILTER_ALLOC_CMD_ERR_CODE_UNKNOWN UINT32_C(0x0)
	+ /* Unable to complete operation due to conflict with Rx Mask VLAN */
	+ #define HWRM_CFA_NTUPLE_FILTER_ALLOC_CMD_ERR_CODE_RX_MASK_VLAN_CONFLICT_ERR UINT32_C(0x1)
	+ uint8_t unused_0[7];
	+} __attribute__((packed));
	+
	/* hwrm_cfa_ntuple_filter_free */
	/* Description: Free an ntuple filter */
	/* Input (24 bytes) */
	@@ -14006,8 +18461,11 @@
	} __attribute__((packed));

	/* hwrm_cfa_ntuple_filter_cfg */
	-/* Description: Configure an ntuple filter with new destination VNIC */
	-/* Input (40 bytes) */
	+/*
	+ * Description: Configure an ntuple filter with a new destination VNIC and/or
	+ * meter.
	+ */
	+/* Input (48 bytes) */

	struct hwrm_cfa_ntuple_filter_cfg_input {
	uint16_t req_type;
	@@ -14043,6 +18501,11 @@
	* configured.
	*/
	#define HWRM_CFA_NTUPLE_FILTER_CFG_INPUT_ENABLES_NEW_MIRROR_VNIC_ID UINT32_C(0x2)
	+ /*
	+ * This bit must be '1' for the new_meter_instance_id field to be
	+ * configured.
	+ */
	+ #define HWRM_CFA_NTUPLE_FILTER_CFG_INPUT_ENABLES_NEW_METER_INSTANCE_ID UINT32_C(0x4)
	uint32_t unused_0;
	uint64_t ntuple_filter_id;
	/* This value is an opaque id into CFA data structures. */
	@@ -14054,6 +18517,17 @@
	*/
	uint32_t new_mirror_vnic_id;
	/* New Logical VNIC ID of the VNIC where traffic is mirrored. */
	+ uint16_t new_meter_instance_id;
	+ /*
	+ * New meter to attach to the flow. Specifying the invalid instance ID
	+ * is used to remove any existing meter from the flow.
	+ */
	+ /*
	+ * A value of 0xfff is considered invalid and implies the
	+ * instance is not configured.
	+ */
	+ #define HWRM_CFA_NTUPLE_FILTER_CFG_INPUT_NEW_METER_INSTANCE_ID_INVALID UINT32_C(0xffff)
	+ uint16_t unused_1[3];
	} __attribute__((packed));

	/* Output (16 bytes) */
	@@ -14164,6 +18638,12 @@
	* then it should be considered accept action.
	*/
	#define HWRM_CFA_EM_FLOW_ALLOC_INPUT_FLAGS_DROP UINT32_C(0x20)
	+ /*
	+ * Setting of this flag indicates that a meter is expected to be
	+ * attached to this flow. This hint can be used when choosing the action
	+ * record format required for the flow.
	+ */
	+ #define HWRM_CFA_EM_FLOW_ALLOC_INPUT_FLAGS_METER UINT32_C(0x40)
	uint32_t enables;
	/* This bit must be '1' for the l2_filter_id field to be configured. */
	#define HWRM_CFA_EM_FLOW_ALLOC_INPUT_ENABLES_L2_FILTER_ID UINT32_C(0x1)
	@@ -14199,6 +18679,11 @@
	#define HWRM_CFA_EM_FLOW_ALLOC_INPUT_ENABLES_MIRROR_VNIC_ID UINT32_C(0x8000)
	/* This bit must be '1' for the encap_record_id field to be configured. */
	#define HWRM_CFA_EM_FLOW_ALLOC_INPUT_ENABLES_ENCAP_RECORD_ID UINT32_C(0x10000)
	+ /*
	+ * This bit must be '1' for the meter_instance_id field to be
	+ * configured.
	+ */
	+ #define HWRM_CFA_EM_FLOW_ALLOC_INPUT_ENABLES_METER_INSTANCE_ID UINT32_C(0x20000)
	uint64_t l2_filter_id;
	/*
	* This value identifies a set of CFA data structures used for an L2
	@@ -14236,8 +18721,13 @@
	*/
	uint8_t src_macaddr[6];
	/* This value indicates the source MAC address in the Ethernet header. */
	- uint8_t unused_2;
	- uint8_t unused_3;
	+ uint16_t meter_instance_id;
	+ /* The meter instance to attach to the flow. */
	+ /*
	+ * A value of 0xfff is considered invalid and implies the
	+ * instance is not configured.
	+ */
	+ #define HWRM_CFA_EM_FLOW_ALLOC_INPUT_METER_INSTANCE_ID_INVALID UINT32_C(0xffff)
	uint8_t dst_macaddr[6];
	/*
	* This value indicates the destination MAC address in the Ethernet
	@@ -14269,16 +18759,16 @@
	uint8_t ip_protocol;
	/*
	* The value of protocol filed in IP header. Applies to UDP and TCP
	- * traffic. 6 - UDP 17 - TCP
	+ * traffic. 6 - TCP 17 - UDP
	*/
	/* invalid */
	#define HWRM_CFA_EM_FLOW_ALLOC_INPUT_IP_PROTOCOL_UNKNOWN UINT32_C(0x0)
	- /* UDP */
	- #define HWRM_CFA_EM_FLOW_ALLOC_INPUT_IP_PROTOCOL_UDP UINT32_C(0x6)
	/* TCP */
	- #define HWRM_CFA_EM_FLOW_ALLOC_INPUT_IP_PROTOCOL_TCP UINT32_C(0x11)
	- uint8_t unused_4;
	- uint8_t unused_5;
	+ #define HWRM_CFA_EM_FLOW_ALLOC_INPUT_IP_PROTOCOL_TCP UINT32_C(0x6)
	+ /* UDP */
	+ #define HWRM_CFA_EM_FLOW_ALLOC_INPUT_IP_PROTOCOL_UDP UINT32_C(0x11)
	+ uint8_t unused_2;
	+ uint8_t unused_3;
	uint32_t src_ipaddr[4]; /* big endian */
	/*
	* The value of source IP address to be used in filtering. For IPv4,
	@@ -14286,8 +18776,8 @@
	*/
	uint32_t dst_ipaddr[4]; /* big endian */
	/*
	- * The value of destination IP address to be used in filtering. For
	- * IPv4, first four bytes represent the IP address.
	+ * big_endian = True The value of destination IP address to be used in
	+ * filtering. For IPv4, first four bytes represent the IP address.
	*/
	uint16_t src_port; /* big endian */
	/*
	@@ -14309,7 +18799,7 @@
	/* Logical VNIC ID of the VNIC where traffic is mirrored. */
	uint32_t encap_record_id;
	/* Logical ID of the encapsulation record. */
	- uint32_t unused_6;
	+ uint32_t unused_4;
	} __attribute__((packed));

	/* Output (24 bytes) */
	@@ -14418,8 +18908,10 @@
	} __attribute__((packed));

	/* hwrm_cfa_em_flow_cfg */
	-/* Description: Configure an EM flow with new destination VNIC */
	-/* Input (40 bytes) */
	+/*
	+ * Description: Configure an EM flow with a new destination VNIC and/or meter.
	+ */
	+/* Input (48 bytes) */

	struct hwrm_cfa_em_flow_cfg_input {
	uint16_t req_type;
	@@ -14455,6 +18947,11 @@
	* configured.
	*/
	#define HWRM_CFA_EM_FLOW_CFG_INPUT_ENABLES_NEW_MIRROR_VNIC_ID UINT32_C(0x2)
	+ /*
	+ * This bit must be '1' for the new_meter_instance_id field to be
	+ * configured.
	+ */
	+ #define HWRM_CFA_EM_FLOW_CFG_INPUT_ENABLES_NEW_METER_INSTANCE_ID UINT32_C(0x4)
	uint32_t unused_0;
	uint64_t em_filter_id;
	/* This value is an opaque id into CFA data structures. */
	@@ -14466,6 +18963,17 @@
	*/
	uint32_t new_mirror_vnic_id;
	/* New Logical VNIC ID of the VNIC where traffic is mirrored. */
	+ uint16_t new_meter_instance_id;
	+ /*
	+ * New meter to attach to the flow. Specifying the invalid instance ID
	+ * is used to remove any existing meter from the flow.
	+ */
	+ /*
	+ * A value of 0xfff is considered invalid and implies the
	+ * instance is not configured.
	+ */
	+ #define HWRM_CFA_EM_FLOW_CFG_INPUT_NEW_METER_INSTANCE_ID_INVALID UINT32_C(0xffff)
	+ uint16_t unused_1[3];
	} __attribute__((packed));

	/* Output (16 bytes) */
	@@ -14500,6 +19008,1885 @@
	*/
	} __attribute__((packed));

	+/* hwrm_cfa_meter_profile_alloc */
	+/*
	+ * Description: This is a meter profile that defines the characteristics of the
	+ * meter. This includes the algorithm, information rates, and burst sizes. No
	+ * running state is kept in a profile and instead meter instances are allocated
	+ * that reference a profile.
	+ */
	+/* Input (40 bytes) */
	+
	+struct hwrm_cfa_meter_profile_alloc_input {
	+ uint16_t req_type;
	+ /*
	+ * This value indicates what type of request this is. The format for the
	+ * rest of the command is determined by this field.
	+ */
	+ uint16_t cmpl_ring;
	+ /*
	+ * This value indicates the what completion ring the request will be
	+ * optionally completed on. If the value is -1, then no CR completion
	+ * will be generated. Any other value must be a valid CR ring_id value
	+ * for this function.
	+ */
	+ uint16_t seq_id;
	+ /* This value indicates the command sequence number. */
	+ uint16_t target_id;
	+ /*
	+ * Target ID of this command. 0x0 - 0xFFF8 - Used for function ids
	+ * 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF - HWRM
	+ */
	+ uint64_t resp_addr;
	+ /*
	+ * This is the host address where the response will be written when the
	+ * request is complete. This area must be 16B aligned and must be
	+ * cleared to zero before the request is made.
	+ */
	+ uint8_t flags;
	+ /*
	+ * Enumeration denoting the RX, TX type of the resource. This
	+ * enumeration is used for resources that are similar for both TX and RX
	+ * paths of the chip.
	+ */
	+ #define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_FLAGS_PATH UINT32_C(0x1)
	+ /* tx path */
	+ #define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_FLAGS_PATH_TX UINT32_C(0x0)
	+ /* rx path */
	+ #define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_FLAGS_PATH_RX UINT32_C(0x1)
	+ #define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_FLAGS_PATH_LAST HWRM_CFA_METER_PROFILE_ALLOC_INPUT_FLAGS_PATH_RX
	+ uint8_t meter_type;
	+ /* The meter algorithm type. */
	+ /* RFC 2697 (srTCM) */
	+ #define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_METER_TYPE_RFC2697 UINT32_C(0x0)
	+ /* RFC 2698 (trTCM) */
	+ #define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_METER_TYPE_RFC2698 UINT32_C(0x1)
	+ /* RFC 4115 (trTCM) */
	+ #define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_METER_TYPE_RFC4115 UINT32_C(0x2)
	+ uint16_t reserved1;
	+ /* This field is reserved for the future use. It shall be set to 0. */
	+ uint32_t reserved2;
	+ /* This field is reserved for the future use. It shall be set to 0. */
	+ uint32_t commit_rate;
	+ /* A meter rate specified in bytes-per-second. */
	+ /* The bandwidth value. */
	+ #define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_COMMIT_RATE_BW_VALUE_MASK UINT32_C(0xfffffff)
	+ #define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_COMMIT_RATE_BW_VALUE_SFT 0
	+ /* The granularity of the value (bits or bytes). */
	+ #define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_COMMIT_RATE_SCALE UINT32_C(0x10000000)
	+ /* Value is in bits. */
	+ #define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_COMMIT_RATE_SCALE_BITS (UINT32_C(0x0) << 28)
	+ /* Value is in bytes. */
	+ #define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_COMMIT_RATE_SCALE_BYTES (UINT32_C(0x1) << 28)
	+ #define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_COMMIT_RATE_SCALE_LAST HWRM_CFA_METER_PROFILE_ALLOC_INPUT_COMMIT_RATE_SCALE_BYTES
	+ /* bw_value_unit is 3 b */
	+ #define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_COMMIT_RATE_BW_VALUE_UNIT_MASK UINT32_C(0xe0000000)
	+ #define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_COMMIT_RATE_BW_VALUE_UNIT_SFT 29
	+ /* Value is in Mb or MB (base 10). */
	+ #define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_COMMIT_RATE_BW_VALUE_UNIT_MEGA (UINT32_C(0x0) << 29)
	+ /* Value is in Kb or KB (base 10). */
	+ #define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_COMMIT_RATE_BW_VALUE_UNIT_KILO (UINT32_C(0x2) << 29)
	+ /* Value is in bits or bytes. */
	+ #define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_COMMIT_RATE_BW_VALUE_UNIT_BASE (UINT32_C(0x4) << 29)
	+ /* Value is in Gb or GB (base 10). */
	+ #define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_COMMIT_RATE_BW_VALUE_UNIT_GIGA (UINT32_C(0x6) << 29)
	+ /* Value is in 1/100th of a percentage of total bandwidth. */
	+ #define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_COMMIT_RATE_BW_VALUE_UNIT_PERCENT1_100 (UINT32_C(0x1) << 29)
	+ /* Invalid unit */
	+ #define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_COMMIT_RATE_BW_VALUE_UNIT_INVALID (UINT32_C(0x7) << 29)
	+ #define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_COMMIT_RATE_BW_VALUE_UNIT_LAST HWRM_CFA_METER_PROFILE_ALLOC_INPUT_COMMIT_RATE_BW_VALUE_UNIT_INVALID
	+ uint32_t commit_burst;
	+ /* A meter burst size specified in bytes. */
	+ /* The bandwidth value. */
	+ #define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_COMMIT_BURST_BW_VALUE_MASK UINT32_C(0xfffffff)
	+ #define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_COMMIT_BURST_BW_VALUE_SFT 0
	+ /* The granularity of the value (bits or bytes). */
	+ #define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_COMMIT_BURST_SCALE UINT32_C(0x10000000)
	+ /* Value is in bits. */
	+ #define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_COMMIT_BURST_SCALE_BITS (UINT32_C(0x0) << 28)
	+ /* Value is in bytes. */
	+ #define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_COMMIT_BURST_SCALE_BYTES (UINT32_C(0x1) << 28)
	+ #define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_COMMIT_BURST_SCALE_LAST HWRM_CFA_METER_PROFILE_ALLOC_INPUT_COMMIT_BURST_SCALE_BYTES
	+ /* bw_value_unit is 3 b */
	+ #define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_COMMIT_BURST_BW_VALUE_UNIT_MASK UINT32_C(0xe0000000)
	+ #define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_COMMIT_BURST_BW_VALUE_UNIT_SFT 29
	+ /* Value is in Mb or MB (base 10). */
	+ #define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_COMMIT_BURST_BW_VALUE_UNIT_MEGA (UINT32_C(0x0) << 29)
	+ /* Value is in Kb or KB (base 10). */
	+ #define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_COMMIT_BURST_BW_VALUE_UNIT_KILO (UINT32_C(0x2) << 29)
	+ /* Value is in bits or bytes. */
	+ #define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_COMMIT_BURST_BW_VALUE_UNIT_BASE (UINT32_C(0x4) << 29)
	+ /* Value is in Gb or GB (base 10). */
	+ #define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_COMMIT_BURST_BW_VALUE_UNIT_GIGA (UINT32_C(0x6) << 29)
	+ /* Value is in 1/100th of a percentage of total bandwidth. */
	+ #define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_COMMIT_BURST_BW_VALUE_UNIT_PERCENT1_100 (UINT32_C(0x1) << 29)
	+ /* Invalid unit */
	+ #define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_COMMIT_BURST_BW_VALUE_UNIT_INVALID (UINT32_C(0x7) << 29)
	+ #define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_COMMIT_BURST_BW_VALUE_UNIT_LAST HWRM_CFA_METER_PROFILE_ALLOC_INPUT_COMMIT_BURST_BW_VALUE_UNIT_INVALID
	+ uint32_t excess_peak_rate;
	+ /* A meter rate specified in bytes-per-second. */
	+ /* The bandwidth value. */
	+ #define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_EXCESS_PEAK_RATE_BW_VALUE_MASK UINT32_C(0xfffffff)
	+ #define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_EXCESS_PEAK_RATE_BW_VALUE_SFT 0
	+ /* The granularity of the value (bits or bytes). */
	+ #define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_EXCESS_PEAK_RATE_SCALE UINT32_C(0x10000000)
	+ /* Value is in bits. */
	+ #define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_EXCESS_PEAK_RATE_SCALE_BITS (UINT32_C(0x0) << 28)
	+ /* Value is in bytes. */
	+ #define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_EXCESS_PEAK_RATE_SCALE_BYTES (UINT32_C(0x1) << 28)
	+ #define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_EXCESS_PEAK_RATE_SCALE_LAST HWRM_CFA_METER_PROFILE_ALLOC_INPUT_EXCESS_PEAK_RATE_SCALE_BYTES
	+ /* bw_value_unit is 3 b */
	+ #define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_EXCESS_PEAK_RATE_BW_VALUE_UNIT_MASK UINT32_C(0xe0000000)
	+ #define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_EXCESS_PEAK_RATE_BW_VALUE_UNIT_SFT 29
	+ /* Value is in Mb or MB (base 10). */
	+ #define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_EXCESS_PEAK_RATE_BW_VALUE_UNIT_MEGA (UINT32_C(0x0) << 29)
	+ /* Value is in Kb or KB (base 10). */
	+ #define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_EXCESS_PEAK_RATE_BW_VALUE_UNIT_KILO (UINT32_C(0x2) << 29)
	+ /* Value is in bits or bytes. */
	+ #define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_EXCESS_PEAK_RATE_BW_VALUE_UNIT_BASE (UINT32_C(0x4) << 29)
	+ /* Value is in Gb or GB (base 10). */
	+ #define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_EXCESS_PEAK_RATE_BW_VALUE_UNIT_GIGA (UINT32_C(0x6) << 29)
	+ /* Value is in 1/100th of a percentage of total bandwidth. */
	+ #define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_EXCESS_PEAK_RATE_BW_VALUE_UNIT_PERCENT1_100 (UINT32_C(0x1) << 29)
	+ /* Invalid unit */
	+ #define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_EXCESS_PEAK_RATE_BW_VALUE_UNIT_INVALID (UINT32_C(0x7) << 29)
	+ #define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_EXCESS_PEAK_RATE_BW_VALUE_UNIT_LAST HWRM_CFA_METER_PROFILE_ALLOC_INPUT_EXCESS_PEAK_RATE_BW_VALUE_UNIT_INVALID
	+ uint32_t excess_peak_burst;
	+ /* A meter burst size specified in bytes. */
	+ /* The bandwidth value. */
	+ #define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_EXCESS_PEAK_BURST_BW_VALUE_MASK UINT32_C(0xfffffff)
	+ #define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_EXCESS_PEAK_BURST_BW_VALUE_SFT 0
	+ /* The granularity of the value (bits or bytes). */
	+ #define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_EXCESS_PEAK_BURST_SCALE UINT32_C(0x10000000)
	+ /* Value is in bits. */
	+ #define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_EXCESS_PEAK_BURST_SCALE_BITS (UINT32_C(0x0) << 28)
	+ /* Value is in bytes. */
	+ #define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_EXCESS_PEAK_BURST_SCALE_BYTES (UINT32_C(0x1) << 28)
	+ #define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_EXCESS_PEAK_BURST_SCALE_LAST HWRM_CFA_METER_PROFILE_ALLOC_INPUT_EXCESS_PEAK_BURST_SCALE_BYTES
	+ /* bw_value_unit is 3 b */
	+ #define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_EXCESS_PEAK_BURST_BW_VALUE_UNIT_MASK UINT32_C(0xe0000000)
	+ #define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_EXCESS_PEAK_BURST_BW_VALUE_UNIT_SFT 29
	+ /* Value is in Mb or MB (base 10). */
	+ #define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_EXCESS_PEAK_BURST_BW_VALUE_UNIT_MEGA (UINT32_C(0x0) << 29)
	+ /* Value is in Kb or KB (base 10). */
	+ #define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_EXCESS_PEAK_BURST_BW_VALUE_UNIT_KILO (UINT32_C(0x2) << 29)
	+ /* Value is in bits or bytes. */
	+ #define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_EXCESS_PEAK_BURST_BW_VALUE_UNIT_BASE (UINT32_C(0x4) << 29)
	+ /* Value is in Gb or GB (base 10). */
	+ #define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_EXCESS_PEAK_BURST_BW_VALUE_UNIT_GIGA (UINT32_C(0x6) << 29)
	+ /* Value is in 1/100th of a percentage of total bandwidth. */
	+ #define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_EXCESS_PEAK_BURST_BW_VALUE_UNIT_PERCENT1_100 (UINT32_C(0x1) << 29)
	+ /* Invalid unit */
	+ #define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_EXCESS_PEAK_BURST_BW_VALUE_UNIT_INVALID (UINT32_C(0x7) << 29)
	+ #define HWRM_CFA_METER_PROFILE_ALLOC_INPUT_EXCESS_PEAK_BURST_BW_VALUE_UNIT_LAST HWRM_CFA_METER_PROFILE_ALLOC_INPUT_EXCESS_PEAK_BURST_BW_VALUE_UNIT_INVALID
	+} __attribute__((packed));
	+
	+/* Output (16 bytes) */
	+
	+struct hwrm_cfa_meter_profile_alloc_output {
	+ uint16_t error_code;
	+ /*
	+ * Pass/Fail or error type Note: receiver to verify the in parameters,
	+ * and fail the call with an error when appropriate
	+ */
	+ uint16_t req_type;
	+ /* This field returns the type of original request. */
	+ uint16_t seq_id;
	+ /* This field provides original sequence number of the command. */
	+ uint16_t resp_len;
	+ /*
	+ * This field is the length of the response in bytes. The last byte of
	+ * the response is a valid flag that will read as '1' when the command
	+ * has been completely written to memory.
	+ */
	+ uint16_t meter_profile_id;
	+ /* This value identifies a meter profile in CFA. */
	+ /*
	+ * A value of 0xfff is considered invalid and implies the
	+ * profile is not configured.
	+ */
	+ #define HWRM_CFA_METER_PROFILE_ALLOC_OUTPUT_METER_PROFILE_ID_INVALID UINT32_C(0xffff)
	+ uint8_t unused_0;
	+ uint8_t unused_1;
	+ uint8_t unused_2;
	+ uint8_t unused_3;
	+ uint8_t unused_4;
	+ uint8_t valid;
	+ /*
	+ * This field is used in Output records to indicate that the output is
	+ * completely written to RAM. This field should be read as '1' to
	+ * indicate that the output has been completely written. When writing a
	+ * command completion or response to an internal processor, the order of
	+ * writes has to be such that this field is written last.
	+ */
	+} __attribute__((packed));
	+
	+/* hwrm_cfa_meter_profile_free */
	+/* Description: Free a meter profile. */
	+/* Input (24 bytes) */
	+
	+struct hwrm_cfa_meter_profile_free_input {
	+ uint16_t req_type;
	+ /*
	+ * This value indicates what type of request this is. The format for the
	+ * rest of the command is determined by this field.
	+ */
	+ uint16_t cmpl_ring;
	+ /*
	+ * This value indicates the what completion ring the request will be
	+ * optionally completed on. If the value is -1, then no CR completion
	+ * will be generated. Any other value must be a valid CR ring_id value
	+ * for this function.
	+ */
	+ uint16_t seq_id;
	+ /* This value indicates the command sequence number. */
	+ uint16_t target_id;
	+ /*
	+ * Target ID of this command. 0x0 - 0xFFF8 - Used for function ids
	+ * 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF - HWRM
	+ */
	+ uint64_t resp_addr;
	+ /*
	+ * This is the host address where the response will be written when the
	+ * request is complete. This area must be 16B aligned and must be
	+ * cleared to zero before the request is made.
	+ */
	+ uint8_t flags;
	+ /*
	+ * Enumeration denoting the RX, TX type of the resource. This
	+ * enumeration is used for resources that are similar for both TX and RX
	+ * paths of the chip.
	+ */
	+ #define HWRM_CFA_METER_PROFILE_FREE_INPUT_FLAGS_PATH UINT32_C(0x1)
	+ /* tx path */
	+ #define HWRM_CFA_METER_PROFILE_FREE_INPUT_FLAGS_PATH_TX UINT32_C(0x0)
	+ /* rx path */
	+ #define HWRM_CFA_METER_PROFILE_FREE_INPUT_FLAGS_PATH_RX UINT32_C(0x1)
	+ #define HWRM_CFA_METER_PROFILE_FREE_INPUT_FLAGS_PATH_LAST HWRM_CFA_METER_PROFILE_FREE_INPUT_FLAGS_PATH_RX
	+ uint8_t unused_0;
	+ uint16_t meter_profile_id;
	+ /* This value identifies a meter profile in CFA. */
	+ /*
	+ * A value of 0xfff is considered invalid and implies the
	+ * profile is not configured.
	+ */
	+ #define HWRM_CFA_METER_PROFILE_FREE_INPUT_METER_PROFILE_ID_INVALID UINT32_C(0xffff)
	+ uint32_t unused_1;
	+} __attribute__((packed));
	+
	+/* Output (16 bytes) */
	+
	+struct hwrm_cfa_meter_profile_free_output {
	+ uint16_t error_code;
	+ /*
	+ * Pass/Fail or error type Note: receiver to verify the in parameters,
	+ * and fail the call with an error when appropriate
	+ */
	+ uint16_t req_type;
	+ /* This field returns the type of original request. */
	+ uint16_t seq_id;
	+ /* This field provides original sequence number of the command. */
	+ uint16_t resp_len;
	+ /*
	+ * This field is the length of the response in bytes. The last byte of
	+ * the response is a valid flag that will read as '1' when the command
	+ * has been completely written to memory.
	+ */
	+ uint32_t unused_0;
	+ uint8_t unused_1;
	+ uint8_t unused_2;
	+ uint8_t unused_3;
	+ uint8_t valid;
	+ /*
	+ * This field is used in Output records to indicate that the output is
	+ * completely written to RAM. This field should be read as '1' to
	+ * indicate that the output has been completely written. When writing a
	+ * command completion or response to an internal processor, the order of
	+ * writes has to be such that this field is written last.
	+ */
	+} __attribute__((packed));
	+
	+/* hwrm_cfa_meter_profile_cfg */
	+/* Description: Reconfigure a meter profile. */
	+/* Input (40 bytes) */
	+
	+struct hwrm_cfa_meter_profile_cfg_input {
	+ uint16_t req_type;
	+ /*
	+ * This value indicates what type of request this is. The format for the
	+ * rest of the command is determined by this field.
	+ */
	+ uint16_t cmpl_ring;
	+ /*
	+ * This value indicates the what completion ring the request will be
	+ * optionally completed on. If the value is -1, then no CR completion
	+ * will be generated. Any other value must be a valid CR ring_id value
	+ * for this function.
	+ */
	+ uint16_t seq_id;
	+ /* This value indicates the command sequence number. */
	+ uint16_t target_id;
	+ /*
	+ * Target ID of this command. 0x0 - 0xFFF8 - Used for function ids
	+ * 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF - HWRM
	+ */
	+ uint64_t resp_addr;
	+ /*
	+ * This is the host address where the response will be written when the
	+ * request is complete. This area must be 16B aligned and must be
	+ * cleared to zero before the request is made.
	+ */
	+ uint8_t flags;
	+ /*
	+ * Enumeration denoting the RX, TX type of the resource. This
	+ * enumeration is used for resources that are similar for both TX and RX
	+ * paths of the chip.
	+ */
	+ #define HWRM_CFA_METER_PROFILE_CFG_INPUT_FLAGS_PATH UINT32_C(0x1)
	+ /* tx path */
	+ #define HWRM_CFA_METER_PROFILE_CFG_INPUT_FLAGS_PATH_TX UINT32_C(0x0)
	+ /* rx path */
	+ #define HWRM_CFA_METER_PROFILE_CFG_INPUT_FLAGS_PATH_RX UINT32_C(0x1)
	+ #define HWRM_CFA_METER_PROFILE_CFG_INPUT_FLAGS_PATH_LAST HWRM_CFA_METER_PROFILE_CFG_INPUT_FLAGS_PATH_RX
	+ uint8_t meter_type;
	+ /* The meter algorithm type. */
	+ /* RFC 2697 (srTCM) */
	+ #define HWRM_CFA_METER_PROFILE_CFG_INPUT_METER_TYPE_RFC2697 UINT32_C(0x0)
	+ /* RFC 2698 (trTCM) */
	+ #define HWRM_CFA_METER_PROFILE_CFG_INPUT_METER_TYPE_RFC2698 UINT32_C(0x1)
	+ /* RFC 4115 (trTCM) */
	+ #define HWRM_CFA_METER_PROFILE_CFG_INPUT_METER_TYPE_RFC4115 UINT32_C(0x2)
	+ uint16_t meter_profile_id;
	+ /* This value identifies a meter profile in CFA. */
	+ /*
	+ * A value of 0xfff is considered invalid and implies the
	+ * profile is not configured.
	+ */
	+ #define HWRM_CFA_METER_PROFILE_CFG_INPUT_METER_PROFILE_ID_INVALID UINT32_C(0xffff)
	+ uint32_t reserved;
	+ /* This field is reserved for the future use. It shall be set to 0. */
	+ uint32_t commit_rate;
	+ /* A meter rate specified in bytes-per-second. */
	+ /* The bandwidth value. */
	+ #define HWRM_CFA_METER_PROFILE_CFG_INPUT_COMMIT_RATE_BW_VALUE_MASK UINT32_C(0xfffffff)
	+ #define HWRM_CFA_METER_PROFILE_CFG_INPUT_COMMIT_RATE_BW_VALUE_SFT 0
	+ /* The granularity of the value (bits or bytes). */
	+ #define HWRM_CFA_METER_PROFILE_CFG_INPUT_COMMIT_RATE_SCALE UINT32_C(0x10000000)
	+ /* Value is in bits. */
	+ #define HWRM_CFA_METER_PROFILE_CFG_INPUT_COMMIT_RATE_SCALE_BITS (UINT32_C(0x0) << 28)
	+ /* Value is in bytes. */
	+ #define HWRM_CFA_METER_PROFILE_CFG_INPUT_COMMIT_RATE_SCALE_BYTES (UINT32_C(0x1) << 28)
	+ #define HWRM_CFA_METER_PROFILE_CFG_INPUT_COMMIT_RATE_SCALE_LAST HWRM_CFA_METER_PROFILE_CFG_INPUT_COMMIT_RATE_SCALE_BYTES
	+ /* bw_value_unit is 3 b */
	+ #define HWRM_CFA_METER_PROFILE_CFG_INPUT_COMMIT_RATE_BW_VALUE_UNIT_MASK UINT32_C(0xe0000000)
	+ #define HWRM_CFA_METER_PROFILE_CFG_INPUT_COMMIT_RATE_BW_VALUE_UNIT_SFT 29
	+ /* Value is in Mb or MB (base 10). */
	+ #define HWRM_CFA_METER_PROFILE_CFG_INPUT_COMMIT_RATE_BW_VALUE_UNIT_MEGA (UINT32_C(0x0) << 29)
	+ /* Value is in Kb or KB (base 10). */
	+ #define HWRM_CFA_METER_PROFILE_CFG_INPUT_COMMIT_RATE_BW_VALUE_UNIT_KILO (UINT32_C(0x2) << 29)
	+ /* Value is in bits or bytes. */
	+ #define HWRM_CFA_METER_PROFILE_CFG_INPUT_COMMIT_RATE_BW_VALUE_UNIT_BASE (UINT32_C(0x4) << 29)
	+ /* Value is in Gb or GB (base 10). */
	+ #define HWRM_CFA_METER_PROFILE_CFG_INPUT_COMMIT_RATE_BW_VALUE_UNIT_GIGA (UINT32_C(0x6) << 29)
	+ /* Value is in 1/100th of a percentage of total bandwidth. */
	+ #define HWRM_CFA_METER_PROFILE_CFG_INPUT_COMMIT_RATE_BW_VALUE_UNIT_PERCENT1_100 (UINT32_C(0x1) << 29)
	+ /* Invalid unit */
	+ #define HWRM_CFA_METER_PROFILE_CFG_INPUT_COMMIT_RATE_BW_VALUE_UNIT_INVALID (UINT32_C(0x7) << 29)
	+ #define HWRM_CFA_METER_PROFILE_CFG_INPUT_COMMIT_RATE_BW_VALUE_UNIT_LAST HWRM_CFA_METER_PROFILE_CFG_INPUT_COMMIT_RATE_BW_VALUE_UNIT_INVALID
	+ uint32_t commit_burst;
	+ /* A meter burst size specified in bytes. */
	+ /* The bandwidth value. */
	+ #define HWRM_CFA_METER_PROFILE_CFG_INPUT_COMMIT_BURST_BW_VALUE_MASK UINT32_C(0xfffffff)
	+ #define HWRM_CFA_METER_PROFILE_CFG_INPUT_COMMIT_BURST_BW_VALUE_SFT 0
	+ /* The granularity of the value (bits or bytes). */
	+ #define HWRM_CFA_METER_PROFILE_CFG_INPUT_COMMIT_BURST_SCALE UINT32_C(0x10000000)
	+ /* Value is in bits. */
	+ #define HWRM_CFA_METER_PROFILE_CFG_INPUT_COMMIT_BURST_SCALE_BITS (UINT32_C(0x0) << 28)
	+ /* Value is in bytes. */
	+ #define HWRM_CFA_METER_PROFILE_CFG_INPUT_COMMIT_BURST_SCALE_BYTES (UINT32_C(0x1) << 28)
	+ #define HWRM_CFA_METER_PROFILE_CFG_INPUT_COMMIT_BURST_SCALE_LAST HWRM_CFA_METER_PROFILE_CFG_INPUT_COMMIT_BURST_SCALE_BYTES
	+ /* bw_value_unit is 3 b */
	+ #define HWRM_CFA_METER_PROFILE_CFG_INPUT_COMMIT_BURST_BW_VALUE_UNIT_MASK UINT32_C(0xe0000000)
	+ #define HWRM_CFA_METER_PROFILE_CFG_INPUT_COMMIT_BURST_BW_VALUE_UNIT_SFT 29
	+ /* Value is in Mb or MB (base 10). */
	+ #define HWRM_CFA_METER_PROFILE_CFG_INPUT_COMMIT_BURST_BW_VALUE_UNIT_MEGA (UINT32_C(0x0) << 29)
	+ /* Value is in Kb or KB (base 10). */
	+ #define HWRM_CFA_METER_PROFILE_CFG_INPUT_COMMIT_BURST_BW_VALUE_UNIT_KILO (UINT32_C(0x2) << 29)
	+ /* Value is in bits or bytes. */
	+ #define HWRM_CFA_METER_PROFILE_CFG_INPUT_COMMIT_BURST_BW_VALUE_UNIT_BASE (UINT32_C(0x4) << 29)
	+ /* Value is in Gb or GB (base 10). */
	+ #define HWRM_CFA_METER_PROFILE_CFG_INPUT_COMMIT_BURST_BW_VALUE_UNIT_GIGA (UINT32_C(0x6) << 29)
	+ /* Value is in 1/100th of a percentage of total bandwidth. */
	+ #define HWRM_CFA_METER_PROFILE_CFG_INPUT_COMMIT_BURST_BW_VALUE_UNIT_PERCENT1_100 (UINT32_C(0x1) << 29)
	+ /* Invalid unit */
	+ #define HWRM_CFA_METER_PROFILE_CFG_INPUT_COMMIT_BURST_BW_VALUE_UNIT_INVALID (UINT32_C(0x7) << 29)
	+ #define HWRM_CFA_METER_PROFILE_CFG_INPUT_COMMIT_BURST_BW_VALUE_UNIT_LAST HWRM_CFA_METER_PROFILE_CFG_INPUT_COMMIT_BURST_BW_VALUE_UNIT_INVALID
	+ uint32_t excess_peak_rate;
	+ /* A meter rate specified in bytes-per-second. */
	+ /* The bandwidth value. */
	+ #define HWRM_CFA_METER_PROFILE_CFG_INPUT_EXCESS_PEAK_RATE_BW_VALUE_MASK UINT32_C(0xfffffff)
	+ #define HWRM_CFA_METER_PROFILE_CFG_INPUT_EXCESS_PEAK_RATE_BW_VALUE_SFT 0
	+ /* The granularity of the value (bits or bytes). */
	+ #define HWRM_CFA_METER_PROFILE_CFG_INPUT_EXCESS_PEAK_RATE_SCALE UINT32_C(0x10000000)
	+ /* Value is in bits. */
	+ #define HWRM_CFA_METER_PROFILE_CFG_INPUT_EXCESS_PEAK_RATE_SCALE_BITS (UINT32_C(0x0) << 28)
	+ /* Value is in bytes. */
	+ #define HWRM_CFA_METER_PROFILE_CFG_INPUT_EXCESS_PEAK_RATE_SCALE_BYTES (UINT32_C(0x1) << 28)
	+ #define HWRM_CFA_METER_PROFILE_CFG_INPUT_EXCESS_PEAK_RATE_SCALE_LAST HWRM_CFA_METER_PROFILE_CFG_INPUT_EXCESS_PEAK_RATE_SCALE_BYTES
	+ /* bw_value_unit is 3 b */
	+ #define HWRM_CFA_METER_PROFILE_CFG_INPUT_EXCESS_PEAK_RATE_BW_VALUE_UNIT_MASK UINT32_C(0xe0000000)
	+ #define HWRM_CFA_METER_PROFILE_CFG_INPUT_EXCESS_PEAK_RATE_BW_VALUE_UNIT_SFT 29
	+ /* Value is in Mb or MB (base 10). */
	+ #define HWRM_CFA_METER_PROFILE_CFG_INPUT_EXCESS_PEAK_RATE_BW_VALUE_UNIT_MEGA (UINT32_C(0x0) << 29)
	+ /* Value is in Kb or KB (base 10). */
	+ #define HWRM_CFA_METER_PROFILE_CFG_INPUT_EXCESS_PEAK_RATE_BW_VALUE_UNIT_KILO (UINT32_C(0x2) << 29)
	+ /* Value is in bits or bytes. */
	+ #define HWRM_CFA_METER_PROFILE_CFG_INPUT_EXCESS_PEAK_RATE_BW_VALUE_UNIT_BASE (UINT32_C(0x4) << 29)
	+ /* Value is in Gb or GB (base 10). */
	+ #define HWRM_CFA_METER_PROFILE_CFG_INPUT_EXCESS_PEAK_RATE_BW_VALUE_UNIT_GIGA (UINT32_C(0x6) << 29)
	+ /* Value is in 1/100th of a percentage of total bandwidth. */
	+ #define HWRM_CFA_METER_PROFILE_CFG_INPUT_EXCESS_PEAK_RATE_BW_VALUE_UNIT_PERCENT1_100 (UINT32_C(0x1) << 29)
	+ /* Invalid unit */
	+ #define HWRM_CFA_METER_PROFILE_CFG_INPUT_EXCESS_PEAK_RATE_BW_VALUE_UNIT_INVALID (UINT32_C(0x7) << 29)
	+ #define HWRM_CFA_METER_PROFILE_CFG_INPUT_EXCESS_PEAK_RATE_BW_VALUE_UNIT_LAST HWRM_CFA_METER_PROFILE_CFG_INPUT_EXCESS_PEAK_RATE_BW_VALUE_UNIT_INVALID
	+ uint32_t excess_peak_burst;
	+ /* A meter burst size specified in bytes. */
	+ /* The bandwidth value. */
	+ #define HWRM_CFA_METER_PROFILE_CFG_INPUT_EXCESS_PEAK_BURST_BW_VALUE_MASK UINT32_C(0xfffffff)
	+ #define HWRM_CFA_METER_PROFILE_CFG_INPUT_EXCESS_PEAK_BURST_BW_VALUE_SFT 0
	+ /* The granularity of the value (bits or bytes). */
	+ #define HWRM_CFA_METER_PROFILE_CFG_INPUT_EXCESS_PEAK_BURST_SCALE UINT32_C(0x10000000)
	+ /* Value is in bits. */
	+ #define HWRM_CFA_METER_PROFILE_CFG_INPUT_EXCESS_PEAK_BURST_SCALE_BITS (UINT32_C(0x0) << 28)
	+ /* Value is in bytes. */
	+ #define HWRM_CFA_METER_PROFILE_CFG_INPUT_EXCESS_PEAK_BURST_SCALE_BYTES (UINT32_C(0x1) << 28)
	+ #define HWRM_CFA_METER_PROFILE_CFG_INPUT_EXCESS_PEAK_BURST_SCALE_LAST HWRM_CFA_METER_PROFILE_CFG_INPUT_EXCESS_PEAK_BURST_SCALE_BYTES
	+ /* bw_value_unit is 3 b */
	+ #define HWRM_CFA_METER_PROFILE_CFG_INPUT_EXCESS_PEAK_BURST_BW_VALUE_UNIT_MASK UINT32_C(0xe0000000)
	+ #define HWRM_CFA_METER_PROFILE_CFG_INPUT_EXCESS_PEAK_BURST_BW_VALUE_UNIT_SFT 29
	+ /* Value is in Mb or MB (base 10). */
	+ #define HWRM_CFA_METER_PROFILE_CFG_INPUT_EXCESS_PEAK_BURST_BW_VALUE_UNIT_MEGA (UINT32_C(0x0) << 29)
	+ /* Value is in Kb or KB (base 10). */
	+ #define HWRM_CFA_METER_PROFILE_CFG_INPUT_EXCESS_PEAK_BURST_BW_VALUE_UNIT_KILO (UINT32_C(0x2) << 29)
	+ /* Value is in bits or bytes. */
	+ #define HWRM_CFA_METER_PROFILE_CFG_INPUT_EXCESS_PEAK_BURST_BW_VALUE_UNIT_BASE (UINT32_C(0x4) << 29)
	+ /* Value is in Gb or GB (base 10). */
	+ #define HWRM_CFA_METER_PROFILE_CFG_INPUT_EXCESS_PEAK_BURST_BW_VALUE_UNIT_GIGA (UINT32_C(0x6) << 29)
	+ /* Value is in 1/100th of a percentage of total bandwidth. */
	+ #define HWRM_CFA_METER_PROFILE_CFG_INPUT_EXCESS_PEAK_BURST_BW_VALUE_UNIT_PERCENT1_100 (UINT32_C(0x1) << 29)
	+ /* Invalid unit */
	+ #define HWRM_CFA_METER_PROFILE_CFG_INPUT_EXCESS_PEAK_BURST_BW_VALUE_UNIT_INVALID (UINT32_C(0x7) << 29)
	+ #define HWRM_CFA_METER_PROFILE_CFG_INPUT_EXCESS_PEAK_BURST_BW_VALUE_UNIT_LAST HWRM_CFA_METER_PROFILE_CFG_INPUT_EXCESS_PEAK_BURST_BW_VALUE_UNIT_INVALID
	+} __attribute__((packed));
	+
	+/* Output (16 bytes) */
	+
	+struct hwrm_cfa_meter_profile_cfg_output {
	+ uint16_t error_code;
	+ /*
	+ * Pass/Fail or error type Note: receiver to verify the in parameters,
	+ * and fail the call with an error when appropriate
	+ */
	+ uint16_t req_type;
	+ /* This field returns the type of original request. */
	+ uint16_t seq_id;
	+ /* This field provides original sequence number of the command. */
	+ uint16_t resp_len;
	+ /*
	+ * This field is the length of the response in bytes. The last byte of
	+ * the response is a valid flag that will read as '1' when the command
	+ * has been completely written to memory.
	+ */
	+ uint32_t unused_0;
	+ uint8_t unused_1;
	+ uint8_t unused_2;
	+ uint8_t unused_3;
	+ uint8_t valid;
	+ /*
	+ * This field is used in Output records to indicate that the output is
	+ * completely written to RAM. This field should be read as '1' to
	+ * indicate that the output has been completely written. When writing a
	+ * command completion or response to an internal processor, the order of
	+ * writes has to be such that this field is written last.
	+ */
	+} __attribute__((packed));
	+
	+/* hwrm_cfa_meter_instance_alloc */
	+/*
	+ * Description: This is a meter instance which is used to track a meter's bucket
	+ * fill values for a flow. Each meter instance references a meter profile that
	+ * defines the meter algorithm in use.
	+ */
	+/* Input (24 bytes) */
	+
	+struct hwrm_cfa_meter_instance_alloc_input {
	+ uint16_t req_type;
	+ /*
	+ * This value indicates what type of request this is. The format for the
	+ * rest of the command is determined by this field.
	+ */
	+ uint16_t cmpl_ring;
	+ /*
	+ * This value indicates the what completion ring the request will be
	+ * optionally completed on. If the value is -1, then no CR completion
	+ * will be generated. Any other value must be a valid CR ring_id value
	+ * for this function.
	+ */
	+ uint16_t seq_id;
	+ /* This value indicates the command sequence number. */
	+ uint16_t target_id;
	+ /*
	+ * Target ID of this command. 0x0 - 0xFFF8 - Used for function ids
	+ * 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF - HWRM
	+ */
	+ uint64_t resp_addr;
	+ /*
	+ * This is the host address where the response will be written when the
	+ * request is complete. This area must be 16B aligned and must be
	+ * cleared to zero before the request is made.
	+ */
	+ uint8_t flags;
	+ /*
	+ * Enumeration denoting the RX, TX type of the resource. This
	+ * enumeration is used for resources that are similar for both TX and RX
	+ * paths of the chip.
	+ */
	+ #define HWRM_CFA_METER_INSTANCE_ALLOC_INPUT_FLAGS_PATH UINT32_C(0x1)
	+ /* tx path */
	+ #define HWRM_CFA_METER_INSTANCE_ALLOC_INPUT_FLAGS_PATH_TX UINT32_C(0x0)
	+ /* rx path */
	+ #define HWRM_CFA_METER_INSTANCE_ALLOC_INPUT_FLAGS_PATH_RX UINT32_C(0x1)
	+ #define HWRM_CFA_METER_INSTANCE_ALLOC_INPUT_FLAGS_PATH_LAST HWRM_CFA_METER_INSTANCE_ALLOC_INPUT_FLAGS_PATH_RX
	+ uint8_t unused_0;
	+ uint16_t meter_profile_id;
	+ /* This value identifies a meter profile in CFA. */
	+ /*
	+ * A value of 0xfff is considered invalid and implies the
	+ * profile is not configured.
	+ */
	+ #define HWRM_CFA_METER_INSTANCE_ALLOC_INPUT_METER_PROFILE_ID_INVALID UINT32_C(0xffff)
	+ uint32_t unused_1;
	+} __attribute__((packed));
	+
	+/* Output (16 bytes) */
	+
	+struct hwrm_cfa_meter_instance_alloc_output {
	+ uint16_t error_code;
	+ /*
	+ * Pass/Fail or error type Note: receiver to verify the in parameters,
	+ * and fail the call with an error when appropriate
	+ */
	+ uint16_t req_type;
	+ /* This field returns the type of original request. */
	+ uint16_t seq_id;
	+ /* This field provides original sequence number of the command. */
	+ uint16_t resp_len;
	+ /*
	+ * This field is the length of the response in bytes. The last byte of
	+ * the response is a valid flag that will read as '1' when the command
	+ * has been completely written to memory.
	+ */
	+ uint16_t meter_instance_id;
	+ /* This value identifies a meter instance in CFA. */
	+ /*
	+ * A value of 0xfff is considered invalid and implies the
	+ * instance is not configured.
	+ */
	+ #define HWRM_CFA_METER_INSTANCE_ALLOC_OUTPUT_METER_INSTANCE_ID_INVALID UINT32_C(0xffff)
	+ uint8_t unused_0;
	+ uint8_t unused_1;
	+ uint8_t unused_2;
	+ uint8_t unused_3;
	+ uint8_t unused_4;
	+ uint8_t valid;
	+ /*
	+ * This field is used in Output records to indicate that the output is
	+ * completely written to RAM. This field should be read as '1' to
	+ * indicate that the output has been completely written. When writing a
	+ * command completion or response to an internal processor, the order of
	+ * writes has to be such that this field is written last.
	+ */
	+} __attribute__((packed));
	+
	+/* hwrm_cfa_meter_instance_free */
	+/* Description: Free a meter instance. */
	+/* Input (24 bytes) */
	+
	+struct hwrm_cfa_meter_instance_free_input {
	+ uint16_t req_type;
	+ /*
	+ * This value indicates what type of request this is. The format for the
	+ * rest of the command is determined by this field.
	+ */
	+ uint16_t cmpl_ring;
	+ /*
	+ * This value indicates the what completion ring the request will be
	+ * optionally completed on. If the value is -1, then no CR completion
	+ * will be generated. Any other value must be a valid CR ring_id value
	+ * for this function.
	+ */
	+ uint16_t seq_id;
	+ /* This value indicates the command sequence number. */
	+ uint16_t target_id;
	+ /*
	+ * Target ID of this command. 0x0 - 0xFFF8 - Used for function ids
	+ * 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF - HWRM
	+ */
	+ uint64_t resp_addr;
	+ /*
	+ * This is the host address where the response will be written when the
	+ * request is complete. This area must be 16B aligned and must be
	+ * cleared to zero before the request is made.
	+ */
	+ uint8_t flags;
	+ /*
	+ * Enumeration denoting the RX, TX type of the resource. This
	+ * enumeration is used for resources that are similar for both TX and RX
	+ * paths of the chip.
	+ */
	+ #define HWRM_CFA_METER_INSTANCE_FREE_INPUT_FLAGS_PATH UINT32_C(0x1)
	+ /* tx path */
	+ #define HWRM_CFA_METER_INSTANCE_FREE_INPUT_FLAGS_PATH_TX UINT32_C(0x0)
	+ /* rx path */
	+ #define HWRM_CFA_METER_INSTANCE_FREE_INPUT_FLAGS_PATH_RX UINT32_C(0x1)
	+ #define HWRM_CFA_METER_INSTANCE_FREE_INPUT_FLAGS_PATH_LAST HWRM_CFA_METER_INSTANCE_FREE_INPUT_FLAGS_PATH_RX
	+ uint8_t unused_0;
	+ uint16_t meter_instance_id;
	+ /* This value identifies a meter instance in CFA. */
	+ /*
	+ * A value of 0xfff is considered invalid and implies the
	+ * instance is not configured.
	+ */
	+ #define HWRM_CFA_METER_INSTANCE_FREE_INPUT_METER_INSTANCE_ID_INVALID UINT32_C(0xffff)
	+ uint32_t unused_1;
	+} __attribute__((packed));
	+
	+/* Output (16 bytes) */
	+
	+struct hwrm_cfa_meter_instance_free_output {
	+ uint16_t error_code;
	+ /*
	+ * Pass/Fail or error type Note: receiver to verify the in parameters,
	+ * and fail the call with an error when appropriate
	+ */
	+ uint16_t req_type;
	+ /* This field returns the type of original request. */
	+ uint16_t seq_id;
	+ /* This field provides original sequence number of the command. */
	+ uint16_t resp_len;
	+ /*
	+ * This field is the length of the response in bytes. The last byte of
	+ * the response is a valid flag that will read as '1' when the command
	+ * has been completely written to memory.
	+ */
	+ uint32_t unused_0;
	+ uint8_t unused_1;
	+ uint8_t unused_2;
	+ uint8_t unused_3;
	+ uint8_t valid;
	+ /*
	+ * This field is used in Output records to indicate that the output is
	+ * completely written to RAM. This field should be read as '1' to
	+ * indicate that the output has been completely written. When writing a
	+ * command completion or response to an internal processor, the order of
	+ * writes has to be such that this field is written last.
	+ */
	+} __attribute__((packed));
	+
	+/* hwrm_cfa_decap_filter_alloc */
	+/*
	+ * Description: This command uses fields from L4/L3/L2 headers. All L2/L3/L4
	+ * header fields are specified in network byte order.
	+ */
	+/* Input (104 bytes) */
	+
	+struct hwrm_cfa_decap_filter_alloc_input {
	+ uint16_t req_type;
	+ /*
	+ * This value indicates what type of request this is. The format for the
	+ * rest of the command is determined by this field.
	+ */
	+ uint16_t cmpl_ring;
	+ /*
	+ * This value indicates the what completion ring the request will be
	+ * optionally completed on. If the value is -1, then no CR completion
	+ * will be generated. Any other value must be a valid CR ring_id value
	+ * for this function.
	+ */
	+ uint16_t seq_id;
	+ /* This value indicates the command sequence number. */
	+ uint16_t target_id;
	+ /*
	+ * Target ID of this command. 0x0 - 0xFFF8 - Used for function ids
	+ * 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF - HWRM
	+ */
	+ uint64_t resp_addr;
	+ /*
	+ * This is the host address where the response will be written when the
	+ * request is complete. This area must be 16B aligned and must be
	+ * cleared to zero before the request is made.
	+ */
	+ uint32_t flags;
	+ /* ovs_tunnel is 1 b */
	+ #define HWRM_CFA_DECAP_FILTER_ALLOC_INPUT_FLAGS_OVS_TUNNEL UINT32_C(0x1)
	+ uint32_t enables;
	+ /* This bit must be '1' for the tunnel_type field to be configured. */
	+ #define HWRM_CFA_DECAP_FILTER_ALLOC_INPUT_ENABLES_TUNNEL_TYPE UINT32_C(0x1)
	+ /* This bit must be '1' for the tunnel_id field to be configured. */
	+ #define HWRM_CFA_DECAP_FILTER_ALLOC_INPUT_ENABLES_TUNNEL_ID UINT32_C(0x2)
	+ /* This bit must be '1' for the src_macaddr field to be configured. */
	+ #define HWRM_CFA_DECAP_FILTER_ALLOC_INPUT_ENABLES_SRC_MACADDR UINT32_C(0x4)
	+ /* This bit must be '1' for the dst_macaddr field to be configured. */
	+ #define HWRM_CFA_DECAP_FILTER_ALLOC_INPUT_ENABLES_DST_MACADDR UINT32_C(0x8)
	+ /* This bit must be '1' for the ovlan_vid field to be configured. */
	+ #define HWRM_CFA_DECAP_FILTER_ALLOC_INPUT_ENABLES_OVLAN_VID UINT32_C(0x10)
	+ /* This bit must be '1' for the ivlan_vid field to be configured. */
	+ #define HWRM_CFA_DECAP_FILTER_ALLOC_INPUT_ENABLES_IVLAN_VID UINT32_C(0x20)
	+ /* This bit must be '1' for the t_ovlan_vid field to be configured. */
	+ #define HWRM_CFA_DECAP_FILTER_ALLOC_INPUT_ENABLES_T_OVLAN_VID UINT32_C(0x40)
	+ /* This bit must be '1' for the t_ivlan_vid field to be configured. */
	+ #define HWRM_CFA_DECAP_FILTER_ALLOC_INPUT_ENABLES_T_IVLAN_VID UINT32_C(0x80)
	+ /* This bit must be '1' for the ethertype field to be configured. */
	+ #define HWRM_CFA_DECAP_FILTER_ALLOC_INPUT_ENABLES_ETHERTYPE UINT32_C(0x100)
	+ /* This bit must be '1' for the src_ipaddr field to be configured. */
	+ #define HWRM_CFA_DECAP_FILTER_ALLOC_INPUT_ENABLES_SRC_IPADDR UINT32_C(0x200)
	+ /* This bit must be '1' for the dst_ipaddr field to be configured. */
	+ #define HWRM_CFA_DECAP_FILTER_ALLOC_INPUT_ENABLES_DST_IPADDR UINT32_C(0x400)
	+ /* This bit must be '1' for the ipaddr_type field to be configured. */
	+ #define HWRM_CFA_DECAP_FILTER_ALLOC_INPUT_ENABLES_IPADDR_TYPE UINT32_C(0x800)
	+ /* This bit must be '1' for the ip_protocol field to be configured. */
	+ #define HWRM_CFA_DECAP_FILTER_ALLOC_INPUT_ENABLES_IP_PROTOCOL UINT32_C(0x1000)
	+ /* This bit must be '1' for the src_port field to be configured. */
	+ #define HWRM_CFA_DECAP_FILTER_ALLOC_INPUT_ENABLES_SRC_PORT UINT32_C(0x2000)
	+ /* This bit must be '1' for the dst_port field to be configured. */
	+ #define HWRM_CFA_DECAP_FILTER_ALLOC_INPUT_ENABLES_DST_PORT UINT32_C(0x4000)
	+ /* This bit must be '1' for the dst_id field to be configured. */
	+ #define HWRM_CFA_DECAP_FILTER_ALLOC_INPUT_ENABLES_DST_ID UINT32_C(0x8000)
	+ /* This bit must be '1' for the mirror_vnic_id field to be configured. */
	+ #define HWRM_CFA_DECAP_FILTER_ALLOC_INPUT_ENABLES_MIRROR_VNIC_ID UINT32_C(0x10000)
	+ uint32_t tunnel_id;
	+ /*
	+ * Tunnel identifier. Virtual Network Identifier (VNI). Only valid with
	+ * tunnel_types VXLAN, NVGRE, and Geneve. Only lower 24-bits of VNI
	+ * field are used in setting up the filter.
	+ */
	+ uint8_t tunnel_type;
	+ /* Tunnel Type. */
	+ /* Non-tunnel */
	+ #define HWRM_CFA_DECAP_FILTER_ALLOC_INPUT_TUNNEL_TYPE_NONTUNNEL UINT32_C(0x0)
	+ /* Virtual eXtensible Local Area Network (VXLAN) */
	+ #define HWRM_CFA_DECAP_FILTER_ALLOC_INPUT_TUNNEL_TYPE_VXLAN UINT32_C(0x1)
	+ /* Network Virtualization Generic Routing Encapsulation (NVGRE) */
	+ #define HWRM_CFA_DECAP_FILTER_ALLOC_INPUT_TUNNEL_TYPE_NVGRE UINT32_C(0x2)
	+ /* Generic Routing Encapsulation (GRE) inside Ethernet payload */
	+ #define HWRM_CFA_DECAP_FILTER_ALLOC_INPUT_TUNNEL_TYPE_L2GRE UINT32_C(0x3)
	+ /* IP in IP */
	+ #define HWRM_CFA_DECAP_FILTER_ALLOC_INPUT_TUNNEL_TYPE_IPIP UINT32_C(0x4)
	+ /* Generic Network Virtualization Encapsulation (Geneve) */
	+ #define HWRM_CFA_DECAP_FILTER_ALLOC_INPUT_TUNNEL_TYPE_GENEVE UINT32_C(0x5)
	+ /* Multi-Protocol Lable Switching (MPLS) */
	+ #define HWRM_CFA_DECAP_FILTER_ALLOC_INPUT_TUNNEL_TYPE_MPLS UINT32_C(0x6)
	+ /* Stateless Transport Tunnel (STT) */
	+ #define HWRM_CFA_DECAP_FILTER_ALLOC_INPUT_TUNNEL_TYPE_STT UINT32_C(0x7)
	+ /* Generic Routing Encapsulation (GRE) inside IP datagram payload */
	+ #define HWRM_CFA_DECAP_FILTER_ALLOC_INPUT_TUNNEL_TYPE_IPGRE UINT32_C(0x8)
	+ /* Any tunneled traffic */
	+ #define HWRM_CFA_DECAP_FILTER_ALLOC_INPUT_TUNNEL_TYPE_ANYTUNNEL UINT32_C(0xff)
	+ uint8_t unused_0;
	+ uint16_t unused_1;
	+ uint8_t src_macaddr[6];
	+ /* This value indicates the source MAC address in the Ethernet header. */
	+ uint8_t unused_2;
	+ uint8_t unused_3;
	+ uint8_t dst_macaddr[6];
	+ /*
	+ * This value indicates the destination MAC address in the Ethernet
	+ * header.
	+ */
	+ uint16_t ovlan_vid;
	+ /*
	+ * This value indicates the VLAN ID of the outer VLAN tag in the
	+ * Ethernet header.
	+ */
	+ uint16_t ivlan_vid;
	+ /*
	+ * This value indicates the VLAN ID of the inner VLAN tag in the
	+ * Ethernet header.
	+ */
	+ uint16_t t_ovlan_vid;
	+ /*
	+ * This value indicates the VLAN ID of the outer VLAN tag in the tunnel
	+ * Ethernet header.
	+ */
	+ uint16_t t_ivlan_vid;
	+ /*
	+ * This value indicates the VLAN ID of the inner VLAN tag in the tunnel
	+ * Ethernet header.
	+ */
	+ uint16_t ethertype; /* big endian */
	+ /* This value indicates the ethertype in the Ethernet header. */
	+ uint8_t ip_addr_type;
	+ /*
	+ * This value indicates the type of IP address. 4 - IPv4 6 - IPv6 All
	+ * others are invalid.
	+ */
	+ /* invalid */
	+ #define HWRM_CFA_DECAP_FILTER_ALLOC_INPUT_IP_ADDR_TYPE_UNKNOWN UINT32_C(0x0)
	+ /* IPv4 */
	+ #define HWRM_CFA_DECAP_FILTER_ALLOC_INPUT_IP_ADDR_TYPE_IPV4 UINT32_C(0x4)
	+ /* IPv6 */
	+ #define HWRM_CFA_DECAP_FILTER_ALLOC_INPUT_IP_ADDR_TYPE_IPV6 UINT32_C(0x6)
	+ uint8_t ip_protocol;
	+ /*
	+ * The value of protocol filed in IP header. Applies to UDP and TCP
	+ * traffic. 6 - TCP 17 - UDP
	+ */
	+ /* invalid */
	+ #define HWRM_CFA_DECAP_FILTER_ALLOC_INPUT_IP_PROTOCOL_UNKNOWN UINT32_C(0x0)
	+ /* TCP */
	+ #define HWRM_CFA_DECAP_FILTER_ALLOC_INPUT_IP_PROTOCOL_TCP UINT32_C(0x6)
	+ /* UDP */
	+ #define HWRM_CFA_DECAP_FILTER_ALLOC_INPUT_IP_PROTOCOL_UDP UINT32_C(0x11)
	+ uint8_t unused_4;
	+ uint8_t unused_5;
	+ uint8_t unused_6[3];
	+ uint8_t unused_7;
	+ uint32_t src_ipaddr[4]; /* big endian */
	+ /*
	+ * The value of source IP address to be used in filtering. For IPv4,
	+ * first four bytes represent the IP address.
	+ */
	+ uint32_t dst_ipaddr[4]; /* big endian */
	+ /*
	+ * The value of destination IP address to be used in filtering. For
	+ * IPv4, first four bytes represent the IP address.
	+ */
	+ uint16_t src_port; /* big endian */
	+ /*
	+ * The value of source port to be used in filtering. Applies to UDP and
	+ * TCP traffic.
	+ */
	+ uint16_t dst_port; /* big endian */
	+ /*
	+ * The value of destination port to be used in filtering. Applies to UDP
	+ * and TCP traffic.
	+ */
	+ uint16_t dst_id;
	+ /*
	+ * If set, this value shall represent the Logical VNIC ID of the
	+ * destination VNIC for the RX path.
	+ */
	+ uint16_t l2_ctxt_ref_id;
	+ /*
	+ * If set, this value shall represent the L2 context that matches the L2
	+ * information of the decap filter.
	+ */
	+} __attribute__((packed));
	+
	+/* Output (16 bytes) */
	+
	+struct hwrm_cfa_decap_filter_alloc_output {
	+ uint16_t error_code;
	+ /*
	+ * Pass/Fail or error type Note: receiver to verify the in parameters,
	+ * and fail the call with an error when appropriate
	+ */
	+ uint16_t req_type;
	+ /* This field returns the type of original request. */
	+ uint16_t seq_id;
	+ /* This field provides original sequence number of the command. */
	+ uint16_t resp_len;
	+ /*
	+ * This field is the length of the response in bytes. The last byte of
	+ * the response is a valid flag that will read as '1' when the command
	+ * has been completely written to memory.
	+ */
	+ uint32_t decap_filter_id;
	+ /* This value is an opaque id into CFA data structures. */
	+ uint8_t unused_0;
	+ uint8_t unused_1;
	+ uint8_t unused_2;
	+ uint8_t valid;
	+ /*
	+ * This field is used in Output records to indicate that the output is
	+ * completely written to RAM. This field should be read as '1' to
	+ * indicate that the output has been completely written. When writing a
	+ * command completion or response to an internal processor, the order of
	+ * writes has to be such that this field is written last.
	+ */
	+} __attribute__((packed));
	+
	+/* hwrm_cfa_decap_filter_free */
	+/* Description: Free an decap filter table entry */
	+/* Input (24 bytes) */
	+
	+struct hwrm_cfa_decap_filter_free_input {
	+ uint16_t req_type;
	+ /*
	+ * This value indicates what type of request this is. The format for the
	+ * rest of the command is determined by this field.
	+ */
	+ uint16_t cmpl_ring;
	+ /*
	+ * This value indicates the what completion ring the request will be
	+ * optionally completed on. If the value is -1, then no CR completion
	+ * will be generated. Any other value must be a valid CR ring_id value
	+ * for this function.
	+ */
	+ uint16_t seq_id;
	+ /* This value indicates the command sequence number. */
	+ uint16_t target_id;
	+ /*
	+ * Target ID of this command. 0x0 - 0xFFF8 - Used for function ids
	+ * 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF - HWRM
	+ */
	+ uint64_t resp_addr;
	+ /*
	+ * This is the host address where the response will be written when the
	+ * request is complete. This area must be 16B aligned and must be
	+ * cleared to zero before the request is made.
	+ */
	+ uint32_t decap_filter_id;
	+ /* This value is an opaque id into CFA data structures. */
	+ uint32_t unused_0;
	+} __attribute__((packed));
	+
	+/* Output (16 bytes) */
	+
	+struct hwrm_cfa_decap_filter_free_output {
	+ uint16_t error_code;
	+ /*
	+ * Pass/Fail or error type Note: receiver to verify the in parameters,
	+ * and fail the call with an error when appropriate
	+ */
	+ uint16_t req_type;
	+ /* This field returns the type of original request. */
	+ uint16_t seq_id;
	+ /* This field provides original sequence number of the command. */
	+ uint16_t resp_len;
	+ /*
	+ * This field is the length of the response in bytes. The last byte of
	+ * the response is a valid flag that will read as '1' when the command
	+ * has been completely written to memory.
	+ */
	+ uint32_t unused_0;
	+ uint8_t unused_1;
	+ uint8_t unused_2;
	+ uint8_t unused_3;
	+ uint8_t valid;
	+ /*
	+ * This field is used in Output records to indicate that the output is
	+ * completely written to RAM. This field should be read as '1' to
	+ * indicate that the output has been completely written. When writing a
	+ * command completion or response to an internal processor, the order of
	+ * writes has to be such that this field is written last.
	+ */
	+} __attribute__((packed));
	+
	+/* hwrm_cfa_flow_alloc */
	+/* Description: Flow is added to table and resources are allocated. */
	+/* Input (128 bytes) */
	+
	+struct hwrm_cfa_flow_alloc_input {
	+ uint16_t req_type;
	+ /*
	+ * This value indicates what type of request this is. The format for the
	+ * rest of the command is determined by this field.
	+ */
	+ uint16_t cmpl_ring;
	+ /*
	+ * This value indicates the what completion ring the request will be
	+ * optionally completed on. If the value is -1, then no CR completion
	+ * will be generated. Any other value must be a valid CR ring_id value
	+ * for this function.
	+ */
	+ uint16_t seq_id;
	+ /* This value indicates the command sequence number. */
	+ uint16_t target_id;
	+ /*
	+ * Target ID of this command. 0x0 - 0xFFF8 - Used for function ids
	+ * 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF - HWRM
	+ */
	+ uint64_t resp_addr;
	+ /*
	+ * This is the host address where the response will be written when the
	+ * request is complete. This area must be 16B aligned and must be
	+ * cleared to zero before the request is made.
	+ */
	+ uint16_t flags;
	+ /* tunnel is 1 b */
	+ #define HWRM_CFA_FLOW_ALLOC_INPUT_FLAGS_TUNNEL UINT32_C(0x1)
	+ /* num_vlan is 2 b */
	+ #define HWRM_CFA_FLOW_ALLOC_INPUT_FLAGS_NUM_VLAN_MASK UINT32_C(0x6)
	+ #define HWRM_CFA_FLOW_ALLOC_INPUT_FLAGS_NUM_VLAN_SFT 1
	+ /* no tags */
	+ #define HWRM_CFA_FLOW_ALLOC_INPUT_FLAGS_NUM_VLAN_NONE (UINT32_C(0x0) << 1)
	+ /* 1 tag */
	+ #define HWRM_CFA_FLOW_ALLOC_INPUT_FLAGS_NUM_VLAN_ONE (UINT32_C(0x1) << 1)
	+ /* 2 tags */
	+ #define HWRM_CFA_FLOW_ALLOC_INPUT_FLAGS_NUM_VLAN_TWO (UINT32_C(0x2) << 1)
	+ #define HWRM_CFA_FLOW_ALLOC_INPUT_FLAGS_NUM_VLAN_LAST HWRM_CFA_FLOW_ALLOC_INPUT_FLAGS_NUM_VLAN_TWO
	+ /* Enumeration denoting the Flow Type. */
	+ #define HWRM_CFA_FLOW_ALLOC_INPUT_FLAGS_FLOWTYPE_MASK UINT32_C(0x38)
	+ #define HWRM_CFA_FLOW_ALLOC_INPUT_FLAGS_FLOWTYPE_SFT 3
	+ /* L2 flow */
	+ #define HWRM_CFA_FLOW_ALLOC_INPUT_FLAGS_FLOWTYPE_L2 (UINT32_C(0x0) << 3)
	+ /* IPV4 flow */
	+ #define HWRM_CFA_FLOW_ALLOC_INPUT_FLAGS_FLOWTYPE_IPV4 (UINT32_C(0x1) << 3)
	+ /* IPV6 flow */
	+ #define HWRM_CFA_FLOW_ALLOC_INPUT_FLAGS_FLOWTYPE_IPV6 (UINT32_C(0x2) << 3)
	+ #define HWRM_CFA_FLOW_ALLOC_INPUT_FLAGS_FLOWTYPE_LAST HWRM_CFA_FLOW_ALLOC_INPUT_FLAGS_FLOWTYPE_IPV6
	+ uint16_t src_fid;
	+ /* Tx Flow: vf fid. Rx Flow: pf fid. */
	+ uint32_t tunnel_handle;
	+ /* Tunnel handle valid when tunnel flag is set. */
	+ uint16_t action_flags;
	+ /*
	+ * Setting of this flag indicates drop action. If this flag is not set,
	+ * then it should be considered accept action.
	+ */
	+ #define HWRM_CFA_FLOW_ALLOC_INPUT_ACTION_FLAGS_FWD UINT32_C(0x1)
	+ /* recycle is 1 b */
	+ #define HWRM_CFA_FLOW_ALLOC_INPUT_ACTION_FLAGS_RECYCLE UINT32_C(0x2)
	+ /*
	+ * Setting of this flag indicates drop action. If this flag is not set,
	+ * then it should be considered accept action.
	+ */
	+ #define HWRM_CFA_FLOW_ALLOC_INPUT_ACTION_FLAGS_DROP UINT32_C(0x4)
	+ /* meter is 1 b */
	+ #define HWRM_CFA_FLOW_ALLOC_INPUT_ACTION_FLAGS_METER UINT32_C(0x8)
	+ /* tunnel is 1 b */
	+ #define HWRM_CFA_FLOW_ALLOC_INPUT_ACTION_FLAGS_TUNNEL UINT32_C(0x10)
	+ /* nat_src is 1 b */
	+ #define HWRM_CFA_FLOW_ALLOC_INPUT_ACTION_FLAGS_NAT_SRC UINT32_C(0x20)
	+ /* nat_dest is 1 b */
	+ #define HWRM_CFA_FLOW_ALLOC_INPUT_ACTION_FLAGS_NAT_DEST UINT32_C(0x40)
	+ /* nat_ipv4_address is 1 b */
	+ #define HWRM_CFA_FLOW_ALLOC_INPUT_ACTION_FLAGS_NAT_IPV4_ADDRESS UINT32_C(0x80)
	+ /* l2_header_rewrite is 1 b */
	+ #define HWRM_CFA_FLOW_ALLOC_INPUT_ACTION_FLAGS_L2_HEADER_REWRITE UINT32_C(0x100)
	+ /* ttl_decrement is 1 b */
	+ #define HWRM_CFA_FLOW_ALLOC_INPUT_ACTION_FLAGS_TTL_DECREMENT UINT32_C(0x200)
	+ uint16_t dst_fid;
	+ /* Tx Flow: pf or vf fid. Rx Flow: vf fid. */
	+ uint16_t l2_rewrite_vlan_tpid; /* big endian */
	+ /* VLAN tpid, valid when push_vlan flag is set. */
	+ uint16_t l2_rewrite_vlan_tci; /* big endian */
	+ /* VLAN tci, valid when push_vlan flag is set. */
	+ uint16_t act_meter_id;
	+ /* Meter id, valid when meter flag is set. */
	+ uint16_t ref_flow_handle;
	+ /* Flow with the same l2 context tcam key. */
	+ uint16_t ethertype; /* big endian */
	+ /* This value sets the match value for the ethertype. */
	+ uint16_t outer_vlan_tci; /* big endian */
	+ /* valid when num tags is 1 or 2. */
	+ uint16_t dmac[3]; /* big endian */
	+ /* This value sets the match value for the Destination MAC address. */
	+ uint16_t inner_vlan_tci; /* big endian */
	+ /* valid when num tags is 2. */
	+ uint16_t smac[3]; /* big endian */
	+ /* This value sets the match value for the Source MAC address. */
	+ uint8_t ip_dst_mask_len;
	+ /* The bit length of destination IP address mask. */
	+ uint8_t ip_src_mask_len;
	+ /* The bit length of source IP address mask. */
	+ uint32_t ip_dst[4]; /* big endian */
	+ /* The value of destination IPv4/IPv6 address. */
	+ uint32_t ip_src[4]; /* big endian */
	+ /* The source IPv4/IPv6 address. */
	+ uint16_t l4_src_port; /* big endian */
	+ /* The value of source port. Applies to UDP and TCP traffic. */
	+ uint16_t l4_src_port_mask; /* big endian */
	+ /* The value of source port mask. Applies to UDP and TCP traffic. */
	+ uint16_t l4_dst_port; /* big endian */
	+ /* The value of destination port. Applies to UDP and TCP traffic. */
	+ uint16_t l4_dst_port_mask; /* big endian */
	+ /* The value of destination port mask. Applies to UDP and TCP traffic. */
	+ uint32_t nat_ip_address[4]; /* big endian */
	+ /* NAT IPv4/6 address based on address type flag. 0 values are ignored. */
	+ uint16_t l2_rewrite_dmac[3]; /* big endian */
	+ /* L2 header re-write Destination MAC address. */
	+ uint16_t nat_port; /* big endian */
	+ /*
	+ * The NAT source/destination port based on direction flag. Applies to
	+ * UDP and TCP traffic. 0 values are ignored.
	+ */
	+ uint16_t l2_rewrite_smac[3]; /* big endian */
	+ /* L2 header re-write Source MAC address. */
	+ uint8_t ip_proto;
	+ /* The value of ip protocol. */
	+ uint8_t unused_0;
	+} __attribute__((packed));
	+
	+/* Output (16 bytes) */
	+
	+struct hwrm_cfa_flow_alloc_output {
	+ uint16_t error_code;
	+ /*
	+ * Pass/Fail or error type Note: receiver to verify the in parameters,
	+ * and fail the call with an error when appropriate
	+ */
	+ uint16_t req_type;
	+ /* This field returns the type of original request. */
	+ uint16_t seq_id;
	+ /* This field provides original sequence number of the command. */
	+ uint16_t resp_len;
	+ /*
	+ * This field is the length of the response in bytes. The last byte of
	+ * the response is a valid flag that will read as '1' when the command
	+ * has been completely written to memory.
	+ */
	+ uint16_t flow_handle;
	+ /* Flow record index. */
	+ uint8_t unused_0;
	+ uint8_t unused_1;
	+ uint8_t unused_2;
	+ uint8_t unused_3;
	+ uint8_t unused_4;
	+ uint8_t valid;
	+ /*
	+ * This field is used in Output records to indicate that the output is
	+ * completely written to RAM. This field should be read as '1' to
	+ * indicate that the output has been completely written. When writing a
	+ * command completion or response to an internal processor, the order of
	+ * writes has to be such that this field is written last.
	+ */
	+} __attribute__((packed));
	+
	+/* hwrm_cfa_flow_free */
	+/* Description: Flow is removed from table and resources are released. */
	+/* Input (24 bytes) */
	+
	+struct hwrm_cfa_flow_free_input {
	+ uint16_t req_type;
	+ /*
	+ * This value indicates what type of request this is. The format for the
	+ * rest of the command is determined by this field.
	+ */
	+ uint16_t cmpl_ring;
	+ /*
	+ * This value indicates the what completion ring the request will be
	+ * optionally completed on. If the value is -1, then no CR completion
	+ * will be generated. Any other value must be a valid CR ring_id value
	+ * for this function.
	+ */
	+ uint16_t seq_id;
	+ /* This value indicates the command sequence number. */
	+ uint16_t target_id;
	+ /*
	+ * Target ID of this command. 0x0 - 0xFFF8 - Used for function ids
	+ * 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF - HWRM
	+ */
	+ uint64_t resp_addr;
	+ /*
	+ * This is the host address where the response will be written when the
	+ * request is complete. This area must be 16B aligned and must be
	+ * cleared to zero before the request is made.
	+ */
	+ uint16_t flow_handle;
	+ /* Flow record index. */
	+ uint16_t unused_0[3];
	+} __attribute__((packed));
	+
	+/* Output (32 bytes) */
	+
	+struct hwrm_cfa_flow_free_output {
	+ uint16_t error_code;
	+ /*
	+ * Pass/Fail or error type Note: receiver to verify the in parameters,
	+ * and fail the call with an error when appropriate
	+ */
	+ uint16_t req_type;
	+ /* This field returns the type of original request. */
	+ uint16_t seq_id;
	+ /* This field provides original sequence number of the command. */
	+ uint16_t resp_len;
	+ /*
	+ * This field is the length of the response in bytes. The last byte of
	+ * the response is a valid flag that will read as '1' when the command
	+ * has been completely written to memory.
	+ */
	+ uint64_t packet;
	+ /* packet is 64 b */
	+ uint64_t byte;
	+ /* byte is 64 b */
	+ uint32_t unused_0;
	+ uint8_t unused_1;
	+ uint8_t unused_2;
	+ uint8_t unused_3;
	+ uint8_t valid;
	+ /*
	+ * This field is used in Output records to indicate that the output is
	+ * completely written to RAM. This field should be read as '1' to
	+ * indicate that the output has been completely written. When writing a
	+ * command completion or response to an internal processor, the order of
	+ * writes has to be such that this field is written last.
	+ */
	+} __attribute__((packed));
	+
	+/* hwrm_cfa_flow_info */
	+/* Description: Flow record content for specified flow is returned. */
	+/* Input (24 bytes) */
	+
	+struct hwrm_cfa_flow_info_input {
	+ uint16_t req_type;
	+ /*
	+ * This value indicates what type of request this is. The format for the
	+ * rest of the command is determined by this field.
	+ */
	+ uint16_t cmpl_ring;
	+ /*
	+ * This value indicates the what completion ring the request will be
	+ * optionally completed on. If the value is -1, then no CR completion
	+ * will be generated. Any other value must be a valid CR ring_id value
	+ * for this function.
	+ */
	+ uint16_t seq_id;
	+ /* This value indicates the command sequence number. */
	+ uint16_t target_id;
	+ /*
	+ * Target ID of this command. 0x0 - 0xFFF8 - Used for function ids
	+ * 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF - HWRM
	+ */
	+ uint64_t resp_addr;
	+ /*
	+ * This is the host address where the response will be written when the
	+ * request is complete. This area must be 16B aligned and must be
	+ * cleared to zero before the request is made.
	+ */
	+ uint16_t flow_handle;
	+ /* Flow record index. */
	+ /* Max flow handle */
	+ #define HWRM_CFA_FLOW_INFO_INPUT_FLOW_HANDLE_MAX_MASK UINT32_C(0xfff)
	+ #define HWRM_CFA_FLOW_INFO_INPUT_FLOW_HANDLE_MAX_SFT 0
	+ /* CNP flow handle */
	+ #define HWRM_CFA_FLOW_INFO_INPUT_FLOW_HANDLE_CNP_CNT UINT32_C(0x1000)
	+ /* Reserved */
	+ #define HWRM_CFA_FLOW_INFO_INPUT_FLOW_HANDLE_RESERVED_MASK UINT32_C(0x6000)
	+ #define HWRM_CFA_FLOW_INFO_INPUT_FLOW_HANDLE_RESERVED_SFT 13
	+ /* Direction rx = 1 */
	+ #define HWRM_CFA_FLOW_INFO_INPUT_FLOW_HANDLE_DIR_RX UINT32_C(0x8000)
	+ uint16_t unused_0[3];
	+} __attribute__((packed));
	+
	+/* Output (56 bytes) */
	+
	+struct hwrm_cfa_flow_info_output {
	+ uint16_t error_code;
	+ /*
	+ * Pass/Fail or error type Note: receiver to verify the in parameters,
	+ * and fail the call with an error when appropriate
	+ */
	+ uint16_t req_type;
	+ /* This field returns the type of original request. */
	+ uint16_t seq_id;
	+ /* This field provides original sequence number of the command. */
	+ uint16_t resp_len;
	+ /*
	+ * This field is the length of the response in bytes. The last byte of
	+ * the response is a valid flag that will read as '1' when the command
	+ * has been completely written to memory.
	+ */
	+ uint8_t flags;
	+ /* flags is 8 b */
	+ uint8_t profile;
	+ /* profile is 8 b */
	+ uint16_t src_fid;
	+ /* src_fid is 16 b */
	+ uint16_t dst_fid;
	+ /* dst_fid is 16 b */
	+ uint16_t l2_ctxt_id;
	+ /* l2_ctxt_id is 16 b */
	+ uint64_t em_info;
	+ /* em_info is 64 b */
	+ uint64_t tcam_info;
	+ /* tcam_info is 64 b */
	+ uint64_t vfp_tcam_info;
	+ /* vfp_tcam_info is 64 b */
	+ uint16_t ar_id;
	+ /* ar_id is 16 b */
	+ uint16_t flow_handle;
	+ /* flow_handle is 16 b */
	+ uint32_t tunnel_handle;
	+ /* tunnel_handle is 32 b */
	+ uint32_t unused_0;
	+ uint8_t unused_1;
	+ uint8_t unused_2;
	+ uint8_t unused_3;
	+ uint8_t valid;
	+ /*
	+ * This field is used in Output records to indicate that the output is
	+ * completely written to RAM. This field should be read as '1' to
	+ * indicate that the output has been completely written. When writing a
	+ * command completion or response to an internal processor, the order of
	+ * writes has to be such that this field is written last.
	+ */
	+} __attribute__((packed));
	+
	+/* hwrm_cfa_flow_flush */
	+/* Description: All flows are removed from table and resources are released. */
	+/* Input (24 bytes) */
	+
	+struct hwrm_cfa_flow_flush_input {
	+ uint16_t req_type;
	+ /*
	+ * This value indicates what type of request this is. The format for the
	+ * rest of the command is determined by this field.
	+ */
	+ uint16_t cmpl_ring;
	+ /*
	+ * This value indicates the what completion ring the request will be
	+ * optionally completed on. If the value is -1, then no CR completion
	+ * will be generated. Any other value must be a valid CR ring_id value
	+ * for this function.
	+ */
	+ uint16_t seq_id;
	+ /* This value indicates the command sequence number. */
	+ uint16_t target_id;
	+ /*
	+ * Target ID of this command. 0x0 - 0xFFF8 - Used for function ids
	+ * 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF - HWRM
	+ */
	+ uint64_t resp_addr;
	+ /*
	+ * This is the host address where the response will be written when the
	+ * request is complete. This area must be 16B aligned and must be
	+ * cleared to zero before the request is made.
	+ */
	+ uint32_t flags;
	+ uint32_t unused_0;
	+} __attribute__((packed));
	+
	+/* Output (16 bytes) */
	+
	+struct hwrm_cfa_flow_flush_output {
	+ uint16_t error_code;
	+ /*
	+ * Pass/Fail or error type Note: receiver to verify the in parameters,
	+ * and fail the call with an error when appropriate
	+ */
	+ uint16_t req_type;
	+ /* This field returns the type of original request. */
	+ uint16_t seq_id;
	+ /* This field provides original sequence number of the command. */
	+ uint16_t resp_len;
	+ /*
	+ * This field is the length of the response in bytes. The last byte of
	+ * the response is a valid flag that will read as '1' when the command
	+ * has been completely written to memory.
	+ */
	+ uint32_t unused_0;
	+ uint8_t unused_1;
	+ uint8_t unused_2;
	+ uint8_t unused_3;
	+ uint8_t valid;
	+ /*
	+ * This field is used in Output records to indicate that the output is
	+ * completely written to RAM. This field should be read as '1' to
	+ * indicate that the output has been completely written. When writing a
	+ * command completion or response to an internal processor, the order of
	+ * writes has to be such that this field is written last.
	+ */
	+} __attribute__((packed));
	+
	+/* hwrm_cfa_flow_stats */
	+/* Description: Flow is removed from table and resources are released. */
	+/* Input (40 bytes) */
	+
	+struct hwrm_cfa_flow_stats_input {
	+ uint16_t req_type;
	+ /*
	+ * This value indicates what type of request this is. The format for the
	+ * rest of the command is determined by this field.
	+ */
	+ uint16_t cmpl_ring;
	+ /*
	+ * This value indicates the what completion ring the request will be
	+ * optionally completed on. If the value is -1, then no CR completion
	+ * will be generated. Any other value must be a valid CR ring_id value
	+ * for this function.
	+ */
	+ uint16_t seq_id;
	+ /* This value indicates the command sequence number. */
	+ uint16_t target_id;
	+ /*
	+ * Target ID of this command. 0x0 - 0xFFF8 - Used for function ids
	+ * 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF - HWRM
	+ */
	+ uint64_t resp_addr;
	+ /*
	+ * This is the host address where the response will be written when the
	+ * request is complete. This area must be 16B aligned and must be
	+ * cleared to zero before the request is made.
	+ */
	+ uint16_t num_flows;
	+ /* Flow handle. */
	+ uint16_t flow_handle_0;
	+ /* Flow handle. */
	+ uint16_t flow_handle_1;
	+ /* Flow handle. */
	+ uint16_t flow_handle_2;
	+ /* Flow handle. */
	+ uint16_t flow_handle_3;
	+ /* Flow handle. */
	+ uint16_t flow_handle_4;
	+ /* Flow handle. */
	+ uint16_t flow_handle_5;
	+ /* Flow handle. */
	+ uint16_t flow_handle_6;
	+ /* Flow handle. */
	+ uint16_t flow_handle_7;
	+ /* Flow handle. */
	+ uint16_t flow_handle_8;
	+ /* Flow handle. */
	+ uint16_t flow_handle_9;
	+ /* Flow handle. */
	+ uint16_t unused_0;
	+} __attribute__((packed));
	+
	+/* Output (176 bytes) */
	+
	+struct hwrm_cfa_flow_stats_output {
	+ uint16_t error_code;
	+ /*
	+ * Pass/Fail or error type Note: receiver to verify the in parameters,
	+ * and fail the call with an error when appropriate
	+ */
	+ uint16_t req_type;
	+ /* This field returns the type of original request. */
	+ uint16_t seq_id;
	+ /* This field provides original sequence number of the command. */
	+ uint16_t resp_len;
	+ /*
	+ * This field is the length of the response in bytes. The last byte of
	+ * the response is a valid flag that will read as '1' when the command
	+ * has been completely written to memory.
	+ */
	+ uint64_t packet_0;
	+ /* packet_0 is 64 b */
	+ uint64_t packet_1;
	+ /* packet_1 is 64 b */
	+ uint64_t packet_2;
	+ /* packet_2 is 64 b */
	+ uint64_t packet_3;
	+ /* packet_3 is 64 b */
	+ uint64_t packet_4;
	+ /* packet_4 is 64 b */
	+ uint64_t packet_5;
	+ /* packet_5 is 64 b */
	+ uint64_t packet_6;
	+ /* packet_6 is 64 b */
	+ uint64_t packet_7;
	+ /* packet_7 is 64 b */
	+ uint64_t packet_8;
	+ /* packet_8 is 64 b */
	+ uint64_t packet_9;
	+ /* packet_9 is 64 b */
	+ uint64_t byte_0;
	+ /* byte_0 is 64 b */
	+ uint64_t byte_1;
	+ /* byte_1 is 64 b */
	+ uint64_t byte_2;
	+ /* byte_2 is 64 b */
	+ uint64_t byte_3;
	+ /* byte_3 is 64 b */
	+ uint64_t byte_4;
	+ /* byte_4 is 64 b */
	+ uint64_t byte_5;
	+ /* byte_5 is 64 b */
	+ uint64_t byte_6;
	+ /* byte_6 is 64 b */
	+ uint64_t byte_7;
	+ /* byte_7 is 64 b */
	+ uint64_t byte_8;
	+ /* byte_8 is 64 b */
	+ uint64_t byte_9;
	+ /* byte_9 is 64 b */
	+ uint32_t unused_0;
	+ uint8_t unused_1;
	+ uint8_t unused_2;
	+ uint8_t unused_3;
	+ uint8_t valid;
	+ /*
	+ * This field is used in Output records to indicate that the output is
	+ * completely written to RAM. This field should be read as '1' to
	+ * indicate that the output has been completely written. When writing a
	+ * command completion or response to an internal processor, the order of
	+ * writes has to be such that this field is written last.
	+ */
	+} __attribute__((packed));
	+
	+/* hwrm_cfa_vf_pair_alloc */
	+/* Description: VF pair is added to table and resources are allocated. */
	+/* Input (32 bytes) */
	+
	+struct hwrm_cfa_vf_pair_alloc_input {
	+ uint16_t req_type;
	+ /*
	+ * This value indicates what type of request this is. The format for the
	+ * rest of the command is determined by this field.
	+ */
	+ uint16_t cmpl_ring;
	+ /*
	+ * This value indicates the what completion ring the request will be
	+ * optionally completed on. If the value is -1, then no CR completion
	+ * will be generated. Any other value must be a valid CR ring_id value
	+ * for this function.
	+ */
	+ uint16_t seq_id;
	+ /* This value indicates the command sequence number. */
	+ uint16_t target_id;
	+ /*
	+ * Target ID of this command. 0x0 - 0xFFF8 - Used for function ids
	+ * 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF - HWRM
	+ */
	+ uint64_t resp_addr;
	+ /*
	+ * This is the host address where the response will be written when the
	+ * request is complete. This area must be 16B aligned and must be
	+ * cleared to zero before the request is made.
	+ */
	+ uint16_t vf_a_id;
	+ /* Logical VF number (range: 0 -> MAX_VFS -1). */
	+ uint16_t vf_b_id;
	+ /* Logical VF number (range: 0 -> MAX_VFS -1). */
	+ uint32_t unused_0;
	+ char pair_name[32];
	+ /* VF Pair name (32 byte string). */
	+} __attribute__((packed));
	+
	+/* Output (16 bytes) */
	+
	+struct hwrm_cfa_vf_pair_alloc_output {
	+ uint16_t error_code;
	+ /*
	+ * Pass/Fail or error type Note: receiver to verify the in parameters,
	+ * and fail the call with an error when appropriate
	+ */
	+ uint16_t req_type;
	+ /* This field returns the type of original request. */
	+ uint16_t seq_id;
	+ /* This field provides original sequence number of the command. */
	+ uint16_t resp_len;
	+ /*
	+ * This field is the length of the response in bytes. The last byte of
	+ * the response is a valid flag that will read as '1' when the command
	+ * has been completely written to memory.
	+ */
	+ uint32_t unused_0;
	+ uint8_t unused_1;
	+ uint8_t unused_2;
	+ uint8_t unused_3;
	+ uint8_t valid;
	+ /*
	+ * This field is used in Output records to indicate that the output is
	+ * completely written to RAM. This field should be read as '1' to
	+ * indicate that the output has been completely written. When writing a
	+ * command completion or response to an internal processor, the order of
	+ * writes has to be such that this field is written last.
	+ */
	+} __attribute__((packed));
	+
	+/* hwrm_cfa_vf_pair_free */
	+/* Description: VF Pair is removed from table and resources are released. */
	+/* Input (24 bytes) */
	+
	+struct hwrm_cfa_vf_pair_free_input {
	+ uint16_t req_type;
	+ /*
	+ * This value indicates what type of request this is. The format for the
	+ * rest of the command is determined by this field.
	+ */
	+ uint16_t cmpl_ring;
	+ /*
	+ * This value indicates the what completion ring the request will be
	+ * optionally completed on. If the value is -1, then no CR completion
	+ * will be generated. Any other value must be a valid CR ring_id value
	+ * for this function.
	+ */
	+ uint16_t seq_id;
	+ /* This value indicates the command sequence number. */
	+ uint16_t target_id;
	+ /*
	+ * Target ID of this command. 0x0 - 0xFFF8 - Used for function ids
	+ * 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF - HWRM
	+ */
	+ uint64_t resp_addr;
	+ /*
	+ * This is the host address where the response will be written when the
	+ * request is complete. This area must be 16B aligned and must be
	+ * cleared to zero before the request is made.
	+ */
	+ char pair_name[32];
	+ /* VF Pair name (32 byte string). */
	+} __attribute__((packed));
	+
	+/* Output (16 bytes) */
	+
	+struct hwrm_cfa_vf_pair_free_output {
	+ uint16_t error_code;
	+ /*
	+ * Pass/Fail or error type Note: receiver to verify the in parameters,
	+ * and fail the call with an error when appropriate
	+ */
	+ uint16_t req_type;
	+ /* This field returns the type of original request. */
	+ uint16_t seq_id;
	+ /* This field provides original sequence number of the command. */
	+ uint16_t resp_len;
	+ /*
	+ * This field is the length of the response in bytes. The last byte of
	+ * the response is a valid flag that will read as '1' when the command
	+ * has been completely written to memory.
	+ */
	+ uint32_t unused_0;
	+ uint8_t unused_1;
	+ uint8_t unused_2;
	+ uint8_t unused_3;
	+ uint8_t valid;
	+ /*
	+ * This field is used in Output records to indicate that the output is
	+ * completely written to RAM. This field should be read as '1' to
	+ * indicate that the output has been completely written. When writing a
	+ * command completion or response to an internal processor, the order of
	+ * writes has to be such that this field is written last.
	+ */
	+} __attribute__((packed));
	+
	+/* hwrm_cfa_vf_pair_info */
	+/* Description: VF pair information is returned. */
	+/* Input (32 bytes) */
	+
	+struct hwrm_cfa_vf_pair_info_input {
	+ uint16_t req_type;
	+ /*
	+ * This value indicates what type of request this is. The format for the
	+ * rest of the command is determined by this field.
	+ */
	+ uint16_t cmpl_ring;
	+ /*
	+ * This value indicates the what completion ring the request will be
	+ * optionally completed on. If the value is -1, then no CR completion
	+ * will be generated. Any other value must be a valid CR ring_id value
	+ * for this function.
	+ */
	+ uint16_t seq_id;
	+ /* This value indicates the command sequence number. */
	+ uint16_t target_id;
	+ /*
	+ * Target ID of this command. 0x0 - 0xFFF8 - Used for function ids
	+ * 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF - HWRM
	+ */
	+ uint64_t resp_addr;
	+ /*
	+ * This is the host address where the response will be written when the
	+ * request is complete. This area must be 16B aligned and must be
	+ * cleared to zero before the request is made.
	+ */
	+ uint32_t flags;
	+ /* If this flag is set, lookup by name else lookup by index. */
	+ #define HWRM_CFA_VF_PAIR_INFO_INPUT_FLAGS_LOOKUP_TYPE UINT32_C(0x1)
	+ uint16_t vf_pair_index;
	+ /* vf pair table index. */
	+ uint8_t unused_0;
	+ uint8_t unused_1;
	+ char vf_pair_name[32];
	+ /* VF Pair name (32 byte string). */
	+} __attribute__((packed));
	+
	+/* Output (64 bytes) */
	+
	+struct hwrm_cfa_vf_pair_info_output {
	+ uint16_t error_code;
	+ /*
	+ * Pass/Fail or error type Note: receiver to verify the in parameters,
	+ * and fail the call with an error when appropriate
	+ */
	+ uint16_t req_type;
	+ /* This field returns the type of original request. */
	+ uint16_t seq_id;
	+ /* This field provides original sequence number of the command. */
	+ uint16_t resp_len;
	+ /*
	+ * This field is the length of the response in bytes. The last byte of
	+ * the response is a valid flag that will read as '1' when the command
	+ * has been completely written to memory.
	+ */
	+ uint16_t next_vf_pair_index;
	+ /* vf pair table index. */
	+ uint16_t vf_a_fid;
	+ /* vf pair member a's vf_fid. */
	+ uint16_t vf_a_index;
	+ /* vf pair member a's Linux logical VF number. */
	+ uint16_t vf_b_fid;
	+ /* vf pair member b's vf_fid. */
	+ uint16_t vf_b_index;
	+ /* vf pair member a's Linux logical VF number. */
	+ uint8_t pair_state;
	+ /* vf pair state. */
	+ /* Pair has been allocated */
	+ #define HWRM_CFA_VF_PAIR_INFO_OUTPUT_PAIR_STATE_ALLOCATED UINT32_C(0x1)
	+ /* Both pair members are active */
	+ #define HWRM_CFA_VF_PAIR_INFO_OUTPUT_PAIR_STATE_ACTIVE UINT32_C(0x2)
	+ uint8_t unused_0;
	+ uint32_t unused_1;
	+ char pair_name[32];
	+ /* VF Pair name (32 byte string). */
	+ uint32_t unused_2;
	+ uint8_t unused_3;
	+ uint8_t unused_4;
	+ uint8_t unused_5;
	+ uint8_t valid;
	+ /*
	+ * This field is used in Output records to indicate that the output is
	+ * completely written to RAM. This field should be read as '1' to
	+ * indicate that the output has been completely written. When writing a
	+ * command completion or response to an internal processor, the order of
	+ * writes has to be such that this field is written last.
	+ */
	+} __attribute__((packed));
	+
	+/* hwrm_cfa_vfr_alloc */
	+/* Description: VF-R is added to table and resources are allocated. */
	+/* Input (32 bytes) */
	+
	+struct hwrm_cfa_vfr_alloc_input {
	+ uint16_t req_type;
	+ /*
	+ * This value indicates what type of request this is. The format for the
	+ * rest of the command is determined by this field.
	+ */
	+ uint16_t cmpl_ring;
	+ /*
	+ * This value indicates the what completion ring the request will be
	+ * optionally completed on. If the value is -1, then no CR completion
	+ * will be generated. Any other value must be a valid CR ring_id value
	+ * for this function.
	+ */
	+ uint16_t seq_id;
	+ /* This value indicates the command sequence number. */
	+ uint16_t target_id;
	+ /*
	+ * Target ID of this command. 0x0 - 0xFFF8 - Used for function ids
	+ * 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF - HWRM
	+ */
	+ uint64_t resp_addr;
	+ /*
	+ * This is the host address where the response will be written when the
	+ * request is complete. This area must be 16B aligned and must be
	+ * cleared to zero before the request is made.
	+ */
	+ uint16_t vf_id;
	+ /* Logical VF number (range: 0 -> MAX_VFS -1). */
	+ uint16_t reserved;
	+ /* This field is reserved for the future use. It shall be set to 0. */
	+ uint32_t unused_0;
	+ char vfr_name[32];
	+ /* VF Representor name (32 byte string). */
	+} __attribute__((packed));
	+
	+/* Output (16 bytes) */
	+
	+struct hwrm_cfa_vfr_alloc_output {
	+ uint16_t error_code;
	+ /*
	+ * Pass/Fail or error type Note: receiver to verify the in parameters,
	+ * and fail the call with an error when appropriate
	+ */
	+ uint16_t req_type;
	+ /* This field returns the type of original request. */
	+ uint16_t seq_id;
	+ /* This field provides original sequence number of the command. */
	+ uint16_t resp_len;
	+ /*
	+ * This field is the length of the response in bytes. The last byte of
	+ * the response is a valid flag that will read as '1' when the command
	+ * has been completely written to memory.
	+ */
	+ uint16_t rx_cfa_code;
	+ /* Rx CFA code. */
	+ uint16_t tx_cfa_action;
	+ /* Tx CFA action. */
	+ uint8_t unused_0;
	+ uint8_t unused_1;
	+ uint8_t unused_2;
	+ uint8_t valid;
	+ /*
	+ * This field is used in Output records to indicate that the output is
	+ * completely written to RAM. This field should be read as '1' to
	+ * indicate that the output has been completely written. When writing a
	+ * command completion or response to an internal processor, the order of
	+ * writes has to be such that this field is written last.
	+ */
	+} __attribute__((packed));
	+
	+/* hwrm_cfa_vfr_free */
	+/* Description: VF-R is removed from table and resources are released. */
	+/* Input (24 bytes) */
	+
	+struct hwrm_cfa_vfr_free_input {
	+ uint16_t req_type;
	+ /*
	+ * This value indicates what type of request this is. The format for the
	+ * rest of the command is determined by this field.
	+ */
	+ uint16_t cmpl_ring;
	+ /*
	+ * This value indicates the what completion ring the request will be
	+ * optionally completed on. If the value is -1, then no CR completion
	+ * will be generated. Any other value must be a valid CR ring_id value
	+ * for this function.
	+ */
	+ uint16_t seq_id;
	+ /* This value indicates the command sequence number. */
	+ uint16_t target_id;
	+ /*
	+ * Target ID of this command. 0x0 - 0xFFF8 - Used for function ids
	+ * 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF - HWRM
	+ */
	+ uint64_t resp_addr;
	+ /*
	+ * This is the host address where the response will be written when the
	+ * request is complete. This area must be 16B aligned and must be
	+ * cleared to zero before the request is made.
	+ */
	+ char vfr_name[32];
	+ /* VF Representor name (32 byte string). */
	+} __attribute__((packed));
	+
	+/* Output (16 bytes) */
	+
	+struct hwrm_cfa_vfr_free_output {
	+ uint16_t error_code;
	+ /*
	+ * Pass/Fail or error type Note: receiver to verify the in parameters,
	+ * and fail the call with an error when appropriate
	+ */
	+ uint16_t req_type;
	+ /* This field returns the type of original request. */
	+ uint16_t seq_id;
	+ /* This field provides original sequence number of the command. */
	+ uint16_t resp_len;
	+ /*
	+ * This field is the length of the response in bytes. The last byte of
	+ * the response is a valid flag that will read as '1' when the command
	+ * has been completely written to memory.
	+ */
	+ uint32_t unused_0;
	+ uint8_t unused_1;
	+ uint8_t unused_2;
	+ uint8_t unused_3;
	+ uint8_t valid;
	+ /*
	+ * This field is used in Output records to indicate that the output is
	+ * completely written to RAM. This field should be read as '1' to
	+ * indicate that the output has been completely written. When writing a
	+ * command completion or response to an internal processor, the order of
	+ * writes has to be such that this field is written last.
	+ */
	+} __attribute__((packed));
	+
	/* hwrm_tunnel_dst_port_query */
	/*
	* Description: This function is called by a driver to query tunnel type
	@@ -14804,9 +21191,24 @@
	uint32_t update_period_ms;
	/*
	* The statistic block update period in ms. e.g. 250ms, 500ms, 750ms,
	- * 1000ms.
	+ * 1000ms. If update_period_ms is 0, then the stats update shall be
	+ * never done and the DMA address shall not be used. In this case, the
	+ * stat block can only be read by hwrm_stat_ctx_query command.
	*/
	- uint32_t unused_0;
	+ uint8_t stat_ctx_flags;
	+ /*
	+ * This field is used to specify statistics context specific
	+ * configuration flags.
	+ */
	+ /*
	+ * When this bit is set to '1', the statistics context shall be
	+ * allocated for RoCE traffic only. In this case, traffic other than
	+ * offloaded RoCE traffic shall not be included in this statistic
	+ * context. When this bit is set to '0', the statistics context shall be
	+ * used for the network traffic other than offloaded RoCE traffic.
	+ */
	+ #define HWRM_STAT_CTX_ALLOC_INPUT_STAT_CTX_FLAGS_ROCE UINT32_C(0x1)
	+ uint8_t unused_0[3];
	} __attribute__((packed));

	/* Output (16 bytes) */
	@@ -15127,8 +21529,11 @@
	#define HWRM_FW_RESET_INPUT_EMBEDDED_PROC_TYPE_NETCTRL UINT32_C(0x2)
	/* RoCE control processor */
	#define HWRM_FW_RESET_INPUT_EMBEDDED_PROC_TYPE_ROCE UINT32_C(0x3)
	- /* Reserved */
	- #define HWRM_FW_RESET_INPUT_EMBEDDED_PROC_TYPE_RSVD UINT32_C(0x4)
	+ /*
	+ * Host (in multi-host environment): This is only valid if
	+ * requester is IPC
	+ */
	+ #define HWRM_FW_RESET_INPUT_EMBEDDED_PROC_TYPE_HOST UINT32_C(0x4)
	uint8_t selfrst_status;
	/* Type of self reset. */
	/* No Self Reset */
	@@ -15137,7 +21542,12 @@
	#define HWRM_FW_RESET_INPUT_SELFRST_STATUS_SELFRSTASAP UINT32_C(0x1)
	/* Self Reset on PCIe Reset */
	#define HWRM_FW_RESET_INPUT_SELFRST_STATUS_SELFRSTPCIERST UINT32_C(0x2)
	- uint16_t unused_0[3];
	+ uint8_t host_idx;
	+ /*
	+ * Indicate which host is being reset. 0 means first host. Only valid
	+ * when embedded_proc_type is host in multihost environment
	+ */
	+ uint8_t unused_0[5];
	} __attribute__((packed));

	/* Output (16 bytes) */
	@@ -15224,8 +21634,11 @@
	#define HWRM_FW_QSTATUS_INPUT_EMBEDDED_PROC_TYPE_NETCTRL UINT32_C(0x2)
	/* RoCE control processor */
	#define HWRM_FW_QSTATUS_INPUT_EMBEDDED_PROC_TYPE_ROCE UINT32_C(0x3)
	- /* Reserved */
	- #define HWRM_FW_QSTATUS_INPUT_EMBEDDED_PROC_TYPE_RSVD UINT32_C(0x4)
	+ /*
	+ * Host (in multi-host environment): This is only valid if
	+ * requester is IPC
	+ */
	+ #define HWRM_FW_QSTATUS_INPUT_EMBEDDED_PROC_TYPE_HOST UINT32_C(0x4)
	uint8_t unused_0[7];
	} __attribute__((packed));

	@@ -15452,6 +21865,316 @@
	*/
	} __attribute__((packed));

	+/* hwrm_fw_set_structured_data */
	+/*
	+ * Description: There can be a variable number of Structure Data Headers (SDH)
	+ * between offset 0x0 and the 'valid' field to handle customizable return
	+ * values. Each Structure Data Header will include one defined structure. The
	+ * number of returned structures can be 0, in which case the 'valid' field
	+ * starts at offset 0x8. The 'valid' field offset is adjusted based on the
	+ * Structure Data Header length and the length of the structured data it
	+ * contains.
	+ */
	+/* Input (32 bytes) */
	+
	+struct hwrm_fw_set_structured_data_input {
	+ uint16_t req_type;
	+ /*
	+ * This value indicates what type of request this is. The format for the
	+ * rest of the command is determined by this field.
	+ */
	+ uint16_t cmpl_ring;
	+ /*
	+ * This value indicates the what completion ring the request will be
	+ * optionally completed on. If the value is -1, then no CR completion
	+ * will be generated. Any other value must be a valid CR ring_id value
	+ * for this function.
	+ */
	+ uint16_t seq_id;
	+ /* This value indicates the command sequence number. */
	+ uint16_t target_id;
	+ /*
	+ * Target ID of this command. 0x0 - 0xFFF8 - Used for function ids
	+ * 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF - HWRM
	+ */
	+ uint64_t resp_addr;
	+ /*
	+ * This is the host address where the response will be written when the
	+ * request is complete. This area must be 16B aligned and must be
	+ * cleared to zero before the request is made.
	+ */
	+ uint64_t src_data_addr;
	+ /* This is the host address where structured data will be copied from */
	+ uint16_t data_len;
	+ /* size of data in bytes */
	+ uint8_t hdr_cnt;
	+ /*
	+ * a count of the number of Structured Data Headers in the data pointed
	+ * by src_data_addr.
	+ */
	+ uint8_t unused_0[5];
	+} __attribute__((packed));
	+
	+/* Output (16 bytes) */
	+
	+struct hwrm_fw_set_structured_data_output {
	+ uint16_t error_code;
	+ /*
	+ * Pass/Fail or error type Note: receiver to verify the in parameters,
	+ * and fail the call with an error when appropriate
	+ */
	+ uint16_t req_type;
	+ /* This field returns the type of original request. */
	+ uint16_t seq_id;
	+ /* This field provides original sequence number of the command. */
	+ uint16_t resp_len;
	+ /*
	+ * This field is the length of the response in bytes. The last byte of
	+ * the response is a valid flag that will read as '1' when the command
	+ * has been completely written to memory.
	+ */
	+ uint32_t unused_0;
	+ uint8_t unused_1;
	+ uint8_t unused_2;
	+ uint8_t unused_3;
	+ uint8_t valid;
	+ /*
	+ * This field is used in Output records to indicate that the output is
	+ * completely written to RAM. This field should be read as '1' to
	+ * indicate that the output has been completely written. When writing a
	+ * command completion or response to an internal processor, the order of
	+ * writes has to be such that this field is written last.
	+ */
	+} __attribute__((packed));
	+
	+/* Command specific Error Codes (8 bytes) */
	+
	+struct hwrm_fw_set_structured_data_cmd_err {
	+ uint8_t code;
	+ /*
	+ * command specific error codes that goes to the cmd_err field in Common
	+ * HWRM Error Response.
	+ */
	+ /* Unknown error */
	+ #define HWRM_FW_SET_STRUCTURED_DATA_CMD_ERR_CODE_UNKNOWN UINT32_C(0x0)
	+ /* count_of_headers is incorrect */
	+ #define HWRM_FW_SET_STRUCTURED_DATA_CMD_ERR_CODE_BAD_HDR_CNT UINT32_C(0x1)
	+ /* data improperly formatted */
	+ #define HWRM_FW_SET_STRUCTURED_DATA_CMD_ERR_CODE_BAD_FMT UINT32_C(0x2)
	+ /* unknown structure ID(s) */
	+ #define HWRM_FW_SET_STRUCTURED_DATA_CMD_ERR_CODE_BAD_ID UINT32_C(0x3)
	+ uint8_t unused_0[7];
	+} __attribute__((packed));
	+
	+/* hwrm_fw_get_structured_data */
	+/* Input (32 bytes) */
	+
	+struct hwrm_fw_get_structured_data_input {
	+ uint16_t req_type;
	+ /*
	+ * This value indicates what type of request this is. The format for the
	+ * rest of the command is determined by this field.
	+ */
	+ uint16_t cmpl_ring;
	+ /*
	+ * This value indicates the what completion ring the request will be
	+ * optionally completed on. If the value is -1, then no CR completion
	+ * will be generated. Any other value must be a valid CR ring_id value
	+ * for this function.
	+ */
	+ uint16_t seq_id;
	+ /* This value indicates the command sequence number. */
	+ uint16_t target_id;
	+ /*
	+ * Target ID of this command. 0x0 - 0xFFF8 - Used for function ids
	+ * 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF - HWRM
	+ */
	+ uint64_t resp_addr;
	+ /*
	+ * This is the host address where the response will be written when the
	+ * request is complete. This area must be 16B aligned and must be
	+ * cleared to zero before the request is made.
	+ */
	+ uint64_t dest_data_addr;
	+ /* This is the host address where structured data will be copied to */
	+ uint16_t data_len;
	+ /* size of data in bytes */
	+ uint16_t structure_id;
	+ /*
	+ * Structure_id is the id of the structure data requesting and count is
	+ * a requested number of instances of this data requested. The actual
	+ * number will be returned in count_of_headers
	+ */
	+ uint16_t subtype;
	+ /*
	+ * Subtype is an optional field used to specify additional information
	+ * of the data being retrieved. For example, if data can be categorized
	+ * as "live" vs "saved" then this field can be used to provide an
	+ * indication of "saved" vs "live" data. Not all structured data
	+ * supports subtypes and if they are supported then the structured data
	+ * will specify the valid values. If structured data is requested that
	+ * supports subtypes but no subtype is given then it is implementation
	+ * specific what will be returned. Some structure data can support a
	+ * subtype of "All" which would cause a list of structures to be
	+ * returned for all supported subtypes. "All" is only used on the
	+ * hwrm_get_structured_data command.
	+ */
	+ #define HWRM_FW_GET_STRUCTURED_DATA_INPUT_SUBTYPE_ALL UINT32_C(0xffff)
	+ #define HWRM_FW_GET_STRUCTURED_DATA_INPUT_SUBTYPE_NEAR_BRIDGE_ADMIN UINT32_C(0x100)
	+ #define HWRM_FW_GET_STRUCTURED_DATA_INPUT_SUBTYPE_NEAR_BRIDGE_PEER UINT32_C(0x101)
	+ #define HWRM_FW_GET_STRUCTURED_DATA_INPUT_SUBTYPE_NEAR_BRIDGE_OPERATIONAL UINT32_C(0x102)
	+ #define HWRM_FW_GET_STRUCTURED_DATA_INPUT_SUBTYPE_NON_TPMR_ADMIN UINT32_C(0x200)
	+ #define HWRM_FW_GET_STRUCTURED_DATA_INPUT_SUBTYPE_NON_TPMR_PEER UINT32_C(0x201)
	+ #define HWRM_FW_GET_STRUCTURED_DATA_INPUT_SUBTYPE_NON_TPMR_OPERATIONAL UINT32_C(0x202)
	+ #define HWRM_FW_GET_STRUCTURED_DATA_INPUT_SUBTYPE_HOST_OPERATIONAL UINT32_C(0x300)
	+ uint8_t count;
	+ /* Number of elements. This allows support of arrayed data */
	+ uint8_t unused_0;
	+} __attribute__((packed));
	+
	+/* Output (16 bytes) */
	+
	+struct hwrm_fw_get_structured_data_output {
	+ uint16_t error_code;
	+ /*
	+ * Pass/Fail or error type Note: receiver to verify the in parameters,
	+ * and fail the call with an error when appropriate
	+ */
	+ uint16_t req_type;
	+ /* This field returns the type of original request. */
	+ uint16_t seq_id;
	+ /* This field provides original sequence number of the command. */
	+ uint16_t resp_len;
	+ /*
	+ * This field is the length of the response in bytes. The last byte of
	+ * the response is a valid flag that will read as '1' when the command
	+ * has been completely written to memory.
	+ */
	+ uint8_t hdr_cnt;
	+ /*
	+ * a count of the number of Structured Data Headers in the data pointed
	+ * by dest_data_addr.
	+ */
	+ uint8_t unused_0;
	+ uint16_t unused_1;
	+ uint8_t unused_2;
	+ uint8_t unused_3;
	+ uint8_t unused_4;
	+ uint8_t valid;
	+ /*
	+ * This field is used in Output records to indicate that the output is
	+ * completely written to RAM. This field should be read as '1' to
	+ * indicate that the output has been completely written. When writing a
	+ * command completion or response to an internal processor, the order of
	+ * writes has to be such that this field is written last.
	+ */
	+} __attribute__((packed));
	+
	+/* Command specific Error Codes (8 bytes) */
	+
	+struct hwrm_fw_get_structured_data_cmd_err {
	+ uint8_t code;
	+ /*
	+ * command specific error codes that goes to the cmd_err field in Common
	+ * HWRM Error Response.
	+ */
	+ /* Unknown error */
	+ #define HWRM_FW_GET_STRUCTURED_DATA_CMD_ERR_CODE_UNKNOWN UINT32_C(0x0)
	+ /* unknown structure ID(s) */
	+ #define HWRM_FW_GET_STRUCTURED_DATA_CMD_ERR_CODE_BAD_ID UINT32_C(0x3)
	+ uint8_t unused_0[7];
	+} __attribute__((packed));
	+
	+/* hwrm_fw_ipc_mailbox */
	+/* Input (32 bytes) */
	+
	+struct hwrm_fw_ipc_mailbox_input {
	+ uint16_t req_type;
	+ /*
	+ * This value indicates what type of request this is. The format for the
	+ * rest of the command is determined by this field.
	+ */
	+ uint16_t cmpl_ring;
	+ /*
	+ * This value indicates the what completion ring the request will be
	+ * optionally completed on. If the value is -1, then no CR completion
	+ * will be generated. Any other value must be a valid CR ring_id value
	+ * for this function.
	+ */
	+ uint16_t seq_id;
	+ /* This value indicates the command sequence number. */
	+ uint16_t target_id;
	+ /*
	+ * Target ID of this command. 0x0 - 0xFFF8 - Used for function ids
	+ * 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF - HWRM
	+ */
	+ uint64_t resp_addr;
	+ /*
	+ * This is the host address where the response will be written when the
	+ * request is complete. This area must be 16B aligned and must be
	+ * cleared to zero before the request is made.
	+ */
	+ uint8_t flags;
	+ uint8_t unused_0;
	+ /* unused is 8 b */
	+ uint8_t event_id;
	+ /* asynchronous event to hosts. */
	+ uint8_t port_id;
	+ /* PORT ID */
	+ uint32_t event_data1;
	+ /* event data1 of asynchronous event */
	+ uint32_t event_data2;
	+ /* event data2 of asynchronous event */
	+ uint32_t unused_1;
	+} __attribute__((packed));
	+
	+/* Output (16 bytes) */
	+
	+struct hwrm_fw_ipc_mailbox_output {
	+ uint16_t error_code;
	+ /*
	+ * Pass/Fail or error type Note: receiver to verify the in parameters,
	+ * and fail the call with an error when appropriate
	+ */
	+ uint16_t req_type;
	+ /* This field returns the type of original request. */
	+ uint16_t seq_id;
	+ /* This field provides original sequence number of the command. */
	+ uint16_t resp_len;
	+ /*
	+ * This field is the length of the response in bytes. The last byte of
	+ * the response is a valid flag that will read as '1' when the command
	+ * has been completely written to memory.
	+ */
	+ uint32_t unused_0;
	+ uint8_t unused_1;
	+ uint8_t unused_2;
	+ uint8_t unused_3;
	+ uint8_t valid;
	+ /*
	+ * This field is used in Output records to indicate that the output is
	+ * completely written to RAM. This field should be read as '1' to
	+ * indicate that the output has been completely written. When writing a
	+ * command completion or response to an internal processor, the order of
	+ * writes has to be such that this field is written last.
	+ */
	+} __attribute__((packed));
	+
	+/* Command specific Error Codes (8 bytes) */
	+
	+struct hwrm_fw_ipc_mailbox_cmd_err {
	+ uint8_t code;
	+ /*
	+ * command specific error codes that goes to the cmd_err field in Common
	+ * HWRM Error Response.
	+ */
	+ /* Unknown error */
	+ #define HWRM_FW_IPC_MAILBOX_CMD_ERR_CODE_UNKNOWN UINT32_C(0x0)
	+ /* invalid event */
	+ #define HWRM_FW_IPC_MAILBOX_CMD_ERR_CODE_BAD_ID UINT32_C(0x3)
	+ uint8_t unused_0[7];
	+} __attribute__((packed));
	+
	/* hwrm_exec_fwd_resp */
	/*
	* Description: This command is used to send an encapsulated request to the
	@@ -16158,8 +22881,12 @@
	* HWRM is not required to provide pattern when the response contains a
	* bitmap WoL filter.
	*/
	- uint64_t pattern_buf_size;
	- /* The sixe of the pattern buffer. Applies to bitmap WoL filter only. */
	+ uint16_t pattern_buf_size;
	+ /* The size of the pattern buffer. Applies to bitmap WoL filter only. */
	+ uint8_t unused_1;
	+ uint8_t unused_2;
	+ uint8_t unused_3[3];
	+ uint8_t unused_4;
	uint64_t pattern_mask_addr;
	/*
	* Physical address of the pattern mask. Applies to bitmap WoL filter
	@@ -16169,11 +22896,12 @@
	* the HWRM is not required to provide mask when the response contains a
	* bitmap WoL filter.
	*/
	- uint64_t pattern_mask_size;
	+ uint16_t pattern_mask_size;
	/*
	* The size of the buffer for pattern mask. Applies to bitmap WoL filter
	* only.
	*/
	+ uint16_t unused_5[3];
	} __attribute__((packed));

	/* Output (32 bytes) */
	@@ -16291,8 +23019,9 @@
	uint8_t unused_3;
	uint64_t wol_pkt_buf_addr;
	/* Physical address of the packet buffer for querying WoL packet. */
	- uint64_t wol_pkt_buf_size;
	+ uint16_t wol_pkt_buf_size;
	/* The size of the buffer for the WoL packet. */
	+ uint16_t unused_4[3];
	} __attribute__((packed));

	/* Output (16 bytes) */
	@@ -16330,11 +23059,12 @@
	#define HWRM_WOL_REASON_QCFG_OUTPUT_WOL_REASON_BMP UINT32_C(0x1)
	/* Invalid */
	#define HWRM_WOL_REASON_QCFG_OUTPUT_WOL_REASON_INVALID UINT32_C(0xff)
	+ uint8_t wol_pkt_len;
	+ /* The value identifies the length of the WoL packet in bytes. */
	uint8_t unused_0;
	uint8_t unused_1;
	uint8_t unused_2;
	uint8_t unused_3;
	- uint8_t unused_4;
	uint8_t valid;
	/*
	* This field is used in Output records to indicate that the output is
	@@ -16940,6 +23670,23 @@
	*/
	} __attribute__((packed));

	+/* Command specific Error Codes (8 bytes) */
	+
	+struct hwrm_nvm_write_cmd_err {
	+ uint8_t code;
	+ /*
	+ * command specific error codes that goes to the cmd_err field in Common
	+ * HWRM Error Response.
	+ */
	+ /* Unknown error */
	+ #define HWRM_NVM_WRITE_CMD_ERR_CODE_UNKNOWN UINT32_C(0x0)
	+ /* Unable to complete operation due to fragmentation */
	+ #define HWRM_NVM_WRITE_CMD_ERR_CODE_FRAG_ERR UINT32_C(0x1)
	+ /* nvm is completely full. */
	+ #define HWRM_NVM_WRITE_CMD_ERR_CODE_NO_SPACE UINT32_C(0x2)
	+ uint8_t unused_0[7];
	+} __attribute__((packed));
	+
	/* hwrm_nvm_modify */
	/*
	* Note: Modify the contents of an NVRAM item as referenced (indexed) by an
	@@ -17503,7 +24250,25 @@
	* an installation directive of 'install').
	*/
	#define HWRM_NVM_INSTALL_UPDATE_INPUT_INSTALL_TYPE_ALL UINT32_C(0xffffffff)
	- uint32_t unused_0;
	+ uint16_t flags;
	+ /*
	+ * If set to 1, then securely erase all unused locations in persistent
	+ * storage.
	+ */
	+ #define HWRM_NVM_INSTALL_UPDATE_INPUT_FLAGS_ERASE_UNUSED_SPACE UINT32_C(0x1)
	+ /*
	+ * If set to 1, then unspecifed images, images not in the package file,
	+ * will be safely deleted. When combined with erase_unused_space then
	+ * unspecified images will be securely erased.
	+ */
	+ #define HWRM_NVM_INSTALL_UPDATE_INPUT_FLAGS_REMOVE_UNUSED_PKG UINT32_C(0x2)
	+ /*
	+ * If set to 1, FW will defragment the NVM if defragmentation is
	+ * required for the update. Allow additional time for this command to
	+ * complete if this bit is set to 1.
	+ */
	+ #define HWRM_NVM_INSTALL_UPDATE_INPUT_FLAGS_ALLOWED_TO_DEFRAG UINT32_C(0x4)
	+ uint16_t unused_0;
	} __attribute__((packed));

	/* Output (24 bytes) */
	@@ -17571,6 +24336,6050 @@
	* command completion or response to an internal processor, the order of
	* writes has to be such that this field is written last.
	*/
	+} __attribute__((packed));
	+
	+/* Command specific Error Codes (8 bytes) */
	+
	+struct hwrm_nvm_install_update_cmd_err {
	+ uint8_t code;
	+ /*
	+ * command specific error codes that goes to the cmd_err field in Common
	+ * HWRM Error Response.
	+ */
	+ /* Unknown error */
	+ #define HWRM_NVM_INSTALL_UPDATE_CMD_ERR_CODE_UNKNOWN UINT32_C(0x0)
	+ /* Unable to complete operation due to fragmentation */
	+ #define HWRM_NVM_INSTALL_UPDATE_CMD_ERR_CODE_FRAG_ERR UINT32_C(0x1)
	+ /* nvm is completely full. */
	+ #define HWRM_NVM_INSTALL_UPDATE_CMD_ERR_CODE_NO_SPACE UINT32_C(0x2)
	+ uint8_t unused_0[7];
	+} __attribute__((packed));
	+
	+/* hwrm_nvm_flush */
	+/* Input (16 bytes) */
	+
	+struct hwrm_nvm_flush_input {
	+ uint16_t req_type;
	+ /*
	+ * This value indicates what type of request this is. The format for the
	+ * rest of the command is determined by this field.
	+ */
	+ uint16_t cmpl_ring;
	+ /*
	+ * This value indicates the what completion ring the request will be
	+ * optionally completed on. If the value is -1, then no CR completion
	+ * will be generated. Any other value must be a valid CR ring_id value
	+ * for this function.
	+ */
	+ uint16_t seq_id;
	+ /* This value indicates the command sequence number. */
	+ uint16_t target_id;
	+ /*
	+ * Target ID of this command. 0x0 - 0xFFF8 - Used for function ids
	+ * 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF - HWRM
	+ */
	+ uint64_t resp_addr;
	+ /*
	+ * This is the host address where the response will be written when the
	+ * request is complete. This area must be 16B aligned and must be
	+ * cleared to zero before the request is made.
	+ */
	+} __attribute__((packed));
	+
	+/* Output (16 bytes) */
	+
	+struct hwrm_nvm_flush_output {
	+ uint16_t error_code;
	+ /*
	+ * Pass/Fail or error type Note: receiver to verify the in parameters,
	+ * and fail the call with an error when appropriate
	+ */
	+ uint16_t req_type;
	+ /* This field returns the type of original request. */
	+ uint16_t seq_id;
	+ /* This field provides original sequence number of the command. */
	+ uint16_t resp_len;
	+ /*
	+ * This field is the length of the response in bytes. The last byte of
	+ * the response is a valid flag that will read as '1' when the command
	+ * has been completely written to memory.
	+ */
	+ uint32_t unused_0;
	+ uint8_t unused_1;
	+ uint8_t unused_2;
	+ uint8_t unused_3;
	+ uint8_t valid;
	+ /*
	+ * This field is used in Output records to indicate that the output is
	+ * completely written to RAM. This field should be read as '1' to
	+ * indicate that the output has been completely written. When writing a
	+ * command completion or response to an internal processor, the order of
	+ * writes has to be such that this field is written last.
	+ */
	+} __attribute__((packed));
	+
	+/* Command specific Error Codes (8 bytes) */
	+
	+struct hwrm_nvm_flush_cmd_err {
	+ uint8_t code;
	+ /*
	+ * command specific error codes that goes to the cmd_err field in Common
	+ * HWRM Error Response.
	+ */
	+ /* Unknown error */
	+ #define HWRM_NVM_FLUSH_CMD_ERR_CODE_UNKNOWN UINT32_C(0x0)
	+ /* flush could not be performed */
	+ #define HWRM_NVM_FLUSH_CMD_ERR_CODE_FAIL UINT32_C(0x1)
	+ uint8_t unused_0[7];
	+} __attribute__((packed));
	+
	+/* hwrm_nvm_get_variable */
	+/* Input (40 bytes) */
	+
	+struct hwrm_nvm_get_variable_input {
	+ uint16_t req_type;
	+ /*
	+ * This value indicates what type of request this is. The format for the
	+ * rest of the command is determined by this field.
	+ */
	+ uint16_t cmpl_ring;
	+ /*
	+ * This value indicates the what completion ring the request will be
	+ * optionally completed on. If the value is -1, then no CR completion
	+ * will be generated. Any other value must be a valid CR ring_id value
	+ * for this function.
	+ */
	+ uint16_t seq_id;
	+ /* This value indicates the command sequence number. */
	+ uint16_t target_id;
	+ /*
	+ * Target ID of this command. 0x0 - 0xFFF8 - Used for function ids
	+ * 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF - HWRM
	+ */
	+ uint64_t resp_addr;
	+ /*
	+ * This is the host address where the response will be written when the
	+ * request is complete. This area must be 16B aligned and must be
	+ * cleared to zero before the request is made.
	+ */
	+ uint64_t dest_data_addr;
	+ /* This is the host address where nvm variable will be stored */
	+ uint16_t data_len;
	+ /* size of data in bits */
	+ uint16_t option_num;
	+ /* nvm cfg option number */
	+ /* reserved. */
	+ #define HWRM_NVM_GET_VARIABLE_INPUT_OPTION_NUM_RSVD_0 UINT32_C(0x0)
	+ /* reserved. */
	+ #define HWRM_NVM_GET_VARIABLE_INPUT_OPTION_NUM_RSVD_FFFF UINT32_C(0xffff)
	+ uint16_t dimensions;
	+ /*
	+ * Number of dimensions for this nvm configuration variable. This value
	+ * indicates how many of the indexN values to use. A value of 0 means
	+ * that none of the indexN values are valid. A value of 1 requires at
	+ * index0 is valued, a value of 2 requires that index0 and index1 are
	+ * valid, and so forth
	+ */
	+ uint16_t index_0;
	+ /* index for the 1st dimensions */
	+ uint16_t index_1;
	+ /* index for the 2nd dimensions */
	+ uint16_t index_2;
	+ /* index for the 3rd dimensions */
	+ uint16_t index_3;
	+ /* index for the 4th dimensions */
	+ uint8_t flags;
	+ /*
	+ * When this bit is set to 1, the factory default value will be
	+ * returned, 0 returns the operational value.
	+ */
	+ #define HWRM_NVM_GET_VARIABLE_INPUT_FLAGS_FACTORY_DFLT UINT32_C(0x1)
	+ uint8_t unused_0;
	+} __attribute__((packed));
	+
	+/* Output (16 bytes) */
	+
	+struct hwrm_nvm_get_variable_output {
	+ uint16_t error_code;
	+ /*
	+ * Pass/Fail or error type Note: receiver to verify the in parameters,
	+ * and fail the call with an error when appropriate
	+ */
	+ uint16_t req_type;
	+ /* This field returns the type of original request. */
	+ uint16_t seq_id;
	+ /* This field provides original sequence number of the command. */
	+ uint16_t resp_len;
	+ /*
	+ * This field is the length of the response in bytes. The last byte of
	+ * the response is a valid flag that will read as '1' when the command
	+ * has been completely written to memory.
	+ */
	+ uint16_t data_len;
	+ /* size of data of the actual variable retrieved in bits */
	+ uint16_t option_num;
	+ /*
	+ * option_num is the option number for the data retrieved. It is
	+ * possible in the future that the option number returned would be
	+ * different than requested. This condition could occur if an option is
	+ * deprecated and a new option id is defined with similar
	+ * characteristics, but has a slightly different definition. This also
	+ * makes it convenient for the caller to identify the variable result
	+ * with the option id from the response.
	+ */
	+ /* reserved. */
	+ #define HWRM_NVM_GET_VARIABLE_OUTPUT_OPTION_NUM_RSVD_0 UINT32_C(0x0)
	+ /* reserved. */
	+ #define HWRM_NVM_GET_VARIABLE_OUTPUT_OPTION_NUM_RSVD_FFFF UINT32_C(0xffff)
	+ uint8_t unused_0;
	+ uint8_t unused_1;
	+ uint8_t unused_2;
	+ uint8_t valid;
	+ /*
	+ * This field is used in Output records to indicate that the output is
	+ * completely written to RAM. This field should be read as '1' to
	+ * indicate that the output has been completely written. When writing a
	+ * command completion or response to an internal processor, the order of
	+ * writes has to be such that this field is written last.
	+ */
	+} __attribute__((packed));
	+
	+/* Command specific Error Codes (8 bytes) */
	+
	+struct hwrm_nvm_get_variable_cmd_err {
	+ uint8_t code;
	+ /*
	+ * command specific error codes that goes to the cmd_err field in Common
	+ * HWRM Error Response.
	+ */
	+ /* Unknown error */
	+ #define HWRM_NVM_GET_VARIABLE_CMD_ERR_CODE_UNKNOWN UINT32_C(0x0)
	+ /* variable does not exist */
	+ #define HWRM_NVM_GET_VARIABLE_CMD_ERR_CODE_VAR_NOT_EXIST UINT32_C(0x1)
	+ /* configuration is corrupted and the variable cannot be saved */
	+ #define HWRM_NVM_GET_VARIABLE_CMD_ERR_CODE_CORRUPT_VAR UINT32_C(0x2)
	+ /* length specified is too small */
	+ #define HWRM_NVM_GET_VARIABLE_CMD_ERR_CODE_LEN_TOO_SHORT UINT32_C(0x3)
	+ uint8_t unused_0[7];
	+} __attribute__((packed));
	+
	+/* hwrm_nvm_set_variable */
	+/* Input (40 bytes) */
	+
	+struct hwrm_nvm_set_variable_input {
	+ uint16_t req_type;
	+ /*
	+ * This value indicates what type of request this is. The format for the
	+ * rest of the command is determined by this field.
	+ */
	+ uint16_t cmpl_ring;
	+ /*
	+ * This value indicates the what completion ring the request will be
	+ * optionally completed on. If the value is -1, then no CR completion
	+ * will be generated. Any other value must be a valid CR ring_id value
	+ * for this function.
	+ */
	+ uint16_t seq_id;
	+ /* This value indicates the command sequence number. */
	+ uint16_t target_id;
	+ /*
	+ * Target ID of this command. 0x0 - 0xFFF8 - Used for function ids
	+ * 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF - HWRM
	+ */
	+ uint64_t resp_addr;
	+ /*
	+ * This is the host address where the response will be written when the
	+ * request is complete. This area must be 16B aligned and must be
	+ * cleared to zero before the request is made.
	+ */
	+ uint64_t src_data_addr;
	+ /* This is the host address where nvm variable will be copied from */
	+ uint16_t data_len;
	+ /* size of data in bits */
	+ uint16_t option_num;
	+ /* nvm cfg option number */
	+ /* reserved. */
	+ #define HWRM_NVM_SET_VARIABLE_INPUT_OPTION_NUM_RSVD_0 UINT32_C(0x0)
	+ /* reserved. */
	+ #define HWRM_NVM_SET_VARIABLE_INPUT_OPTION_NUM_RSVD_FFFF UINT32_C(0xffff)
	+ uint16_t dimensions;
	+ /*
	+ * Number of dimensions for this nvm configuration variable. This value
	+ * indicates how many of the indexN values to use. A value of 0 means
	+ * that none of the indexN values are valid. A value of 1 requires at
	+ * index0 is valued, a value of 2 requires that index0 and index1 are
	+ * valid, and so forth
	+ */
	+ uint16_t index_0;
	+ /* index for the 1st dimensions */
	+ uint16_t index_1;
	+ /* index for the 2nd dimensions */
	+ uint16_t index_2;
	+ /* index for the 3rd dimensions */
	+ uint16_t index_3;
	+ /* index for the 4th dimensions */
	+ uint8_t flags;
	+ /*
	+ * When this bit is 1, flush internal cache after this write operation
	+ * (see hwrm_nvm_flush command.)
	+ */
	+ #define HWRM_NVM_SET_VARIABLE_INPUT_FLAGS_FORCE_FLUSH UINT32_C(0x1)
	+ /* encryption method */
	+ #define HWRM_NVM_SET_VARIABLE_INPUT_FLAGS_ENCRYPT_MODE_MASK UINT32_C(0xe)
	+ #define HWRM_NVM_SET_VARIABLE_INPUT_FLAGS_ENCRYPT_MODE_SFT 1
	+ /* No encryption. */
	+ #define HWRM_NVM_SET_VARIABLE_INPUT_FLAGS_ENCRYPT_MODE_NONE (UINT32_C(0x0) << 1)
	+ /* one-way encryption. */
	+ #define HWRM_NVM_SET_VARIABLE_INPUT_FLAGS_ENCRYPT_MODE_HMAC_SHA1 (UINT32_C(0x1) << 1)
	+ #define HWRM_NVM_SET_VARIABLE_INPUT_FLAGS_ENCRYPT_MODE_LAST HWRM_NVM_SET_VARIABLE_INPUT_FLAGS_ENCRYPT_MODE_HMAC_SHA1
	+ uint8_t unused_0;
	+} __attribute__((packed));
	+
	+/* Output (16 bytes) */
	+
	+struct hwrm_nvm_set_variable_output {
	+ uint16_t error_code;
	+ /*
	+ * Pass/Fail or error type Note: receiver to verify the in parameters,
	+ * and fail the call with an error when appropriate
	+ */
	+ uint16_t req_type;
	+ /* This field returns the type of original request. */
	+ uint16_t seq_id;
	+ /* This field provides original sequence number of the command. */
	+ uint16_t resp_len;
	+ /*
	+ * This field is the length of the response in bytes. The last byte of
	+ * the response is a valid flag that will read as '1' when the command
	+ * has been completely written to memory.
	+ */
	+ uint32_t unused_0;
	+ uint8_t unused_1;
	+ uint8_t unused_2;
	+ uint8_t unused_3;
	+ uint8_t valid;
	+ /*
	+ * This field is used in Output records to indicate that the output is
	+ * completely written to RAM. This field should be read as '1' to
	+ * indicate that the output has been completely written. When writing a
	+ * command completion or response to an internal processor, the order of
	+ * writes has to be such that this field is written last.
	+ */
	+} __attribute__((packed));
	+
	+/* Command specific Error Codes (8 bytes) */
	+
	+struct hwrm_nvm_set_variable_cmd_err {
	+ uint8_t code;
	+ /*
	+ * command specific error codes that goes to the cmd_err field in Common
	+ * HWRM Error Response.
	+ */
	+ /* Unknown error */
	+ #define HWRM_NVM_SET_VARIABLE_CMD_ERR_CODE_UNKNOWN UINT32_C(0x0)
	+ /* variable does not exist */
	+ #define HWRM_NVM_SET_VARIABLE_CMD_ERR_CODE_VAR_NOT_EXIST UINT32_C(0x1)
	+ /* configuration is corrupted and the variable cannot be saved */
	+ #define HWRM_NVM_SET_VARIABLE_CMD_ERR_CODE_CORRUPT_VAR UINT32_C(0x2)
	+ uint8_t unused_0[7];
	+} __attribute__((packed));
	+
	+/* hwrm_nvm_validate_option */
	+/* Input (40 bytes) */
	+
	+struct hwrm_nvm_validate_option_input {
	+ uint16_t req_type;
	+ /*
	+ * This value indicates what type of request this is. The format for the
	+ * rest of the command is determined by this field.
	+ */
	+ uint16_t cmpl_ring;
	+ /*
	+ * This value indicates the what completion ring the request will be
	+ * optionally completed on. If the value is -1, then no CR completion
	+ * will be generated. Any other value must be a valid CR ring_id value
	+ * for this function.
	+ */
	+ uint16_t seq_id;
	+ /* This value indicates the command sequence number. */
	+ uint16_t target_id;
	+ /*
	+ * Target ID of this command. 0x0 - 0xFFF8 - Used for function ids
	+ * 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF - HWRM
	+ */
	+ uint64_t resp_addr;
	+ /*
	+ * This is the host address where the response will be written when the
	+ * request is complete. This area must be 16B aligned and must be
	+ * cleared to zero before the request is made.
	+ */
	+ uint64_t src_data_addr;
	+ /* This is the host address where nvm variable will be copied from */
	+ uint16_t data_len;
	+ /* size of data in bits */
	+ uint16_t option_num;
	+ /* nvm cfg option number */
	+ /* reserved. */
	+ #define HWRM_NVM_VALIDATE_OPTION_INPUT_OPTION_NUM_RSVD_0 UINT32_C(0x0)
	+ /* reserved. */
	+ #define HWRM_NVM_VALIDATE_OPTION_INPUT_OPTION_NUM_RSVD_FFFF UINT32_C(0xffff)
	+ uint16_t dimensions;
	+ /*
	+ * Number of dimensions for this nvm configuration variable. This value
	+ * indicates how many of the indexN values to use. A value of 0 means
	+ * that none of the indexN values are valid. A value of 1 requires at
	+ * index0 is valued, a value of 2 requires that index0 and index1 are
	+ * valid, and so forth
	+ */
	+ uint16_t index_0;
	+ /* index for the 1st dimensions */
	+ uint16_t index_1;
	+ /* index for the 2nd dimensions */
	+ uint16_t index_2;
	+ /* index for the 3rd dimensions */
	+ uint16_t index_3;
	+ /* index for the 4th dimensions */
	+ uint16_t unused_0;
	+} __attribute__((packed));
	+
	+/* Output (16 bytes) */
	+
	+struct hwrm_nvm_validate_option_output {
	+ uint16_t error_code;
	+ /*
	+ * Pass/Fail or error type Note: receiver to verify the in parameters,
	+ * and fail the call with an error when appropriate
	+ */
	+ uint16_t req_type;
	+ /* This field returns the type of original request. */
	+ uint16_t seq_id;
	+ /* This field provides original sequence number of the command. */
	+ uint16_t resp_len;
	+ /*
	+ * This field is the length of the response in bytes. The last byte of
	+ * the response is a valid flag that will read as '1' when the command
	+ * has been completely written to memory.
	+ */
	+ uint8_t result;
	+ /*
	+ * indicates that the value provided for the option is not
	+ * matching with the saved data.
	+ */
	+ #define HWRM_NVM_VALIDATE_OPTION_OUTPUT_RESULT_NOT_MATCH UINT32_C(0x0)
	+ /*
	+ * indicates that the value provided for the option is matching
	+ * the saved data.
	+ */
	+ #define HWRM_NVM_VALIDATE_OPTION_OUTPUT_RESULT_MATCH UINT32_C(0x1)
	+ uint8_t unused_0;
	+ uint16_t unused_1;
	+ uint8_t unused_2;
	+ uint8_t unused_3;
	+ uint8_t unused_4;
	+ uint8_t valid;
	+ /*
	+ * This field is used in Output records to indicate that the output is
	+ * completely written to RAM. This field should be read as '1' to
	+ * indicate that the output has been completely written. When writing a
	+ * command completion or response to an internal processor, the order of
	+ * writes has to be such that this field is written last.
	+ */
	+} __attribute__((packed));
	+
	+/* Command specific Error Codes (8 bytes) */
	+
	+struct hwrm_nvm_validate_option_cmd_err {
	+ uint8_t code;
	+ /*
	+ * command specific error codes that goes to the cmd_err field in Common
	+ * HWRM Error Response.
	+ */
	+ /* Unknown error */
	+ #define HWRM_NVM_VALIDATE_OPTION_CMD_ERR_CODE_UNKNOWN UINT32_C(0x0)
	+ uint8_t unused_0[7];
	+} __attribute__((packed));
	+
	+/* Command Queue (CMDQ) Interface */
	+/* Description: This command queries congestion control settings. */
	+/* Init CMDQ (16 bytes) */
	+
	+struct cmdq_init {
	+ uint64_t cmdq_pbl;
	+ /* CMDQ PBL physical address. */
	+ uint16_t cmdq_size_cmdq_lvl;
	+ /* CMDQ size. */
	+ /* CMDQ PBL indirection levels. */
	+ #define CMDQ_INIT_CMDQ_LVL_MASK UINT32_C(0x3)
	+ #define CMDQ_INIT_CMDQ_LVL_SFT 0
	+ /* CMDQ size. */
	+ #define CMDQ_INIT_CMDQ_SIZE_MASK UINT32_C(0xfffc)
	+ #define CMDQ_INIT_CMDQ_SIZE_SFT 2
	+ uint16_t creq_ring_id;
	+ /* CREQ completion ring id. */
	+ uint32_t prod_idx;
	+ /* Mailbox producer index. MSB must also be set. */
	+} __attribute__((packed));
	+
	+/* Update CMDQ producer index (16 bytes) */
	+
	+struct cmdq_update {
	+ uint64_t reserved64;
	+ /* reserved64 is 64 b */
	+ uint32_t reserved32;
	+ /* reserved32 is 32 b */
	+ uint32_t prod_idx;
	+ /* Mailbox producer index. */
	+} __attribute__((packed));
	+
	+/* CMDQ common header structure (16 bytes) */
	+
	+struct cmdq_base {
	+ uint8_t opcode;
	+ /* Command opcode. */
	+ /*
	+ * Create QP command allocates QP context with the specified SQ,
	+ * RQ/SRQ, CQ and other parameters.
	+ */
	+ #define CMDQ_BASE_OPCODE_CREATE_QP UINT32_C(0x1)
	+ /*
	+ * Destroy QP command deletes the QP context and ceases any
	+ * further reference.
	+ */
	+ #define CMDQ_BASE_OPCODE_DESTROY_QP UINT32_C(0x2)
	+ /*
	+ * Modify QP command changes QP states and other QP specific
	+ * parameters.
	+ */
	+ #define CMDQ_BASE_OPCODE_MODIFY_QP UINT32_C(0x3)
	+ /* Query QP command retrieves info about the specified QP. */
	+ #define CMDQ_BASE_OPCODE_QUERY_QP UINT32_C(0x4)
	+ /* Create SRQ command allocates a SRQ with the specified parameters. */
	+ #define CMDQ_BASE_OPCODE_CREATE_SRQ UINT32_C(0x5)
	+ /* Destroy SRQ command deletes and flushes the specified SRQ. */
	+ #define CMDQ_BASE_OPCODE_DESTROY_SRQ UINT32_C(0x6)
	+ /* Query SRP command retrieves info about the specified SRQ. */
	+ #define CMDQ_BASE_OPCODE_QUERY_SRQ UINT32_C(0x8)
	+ /* Create CQ command allocates a CQ with the specified parameters. */
	+ #define CMDQ_BASE_OPCODE_CREATE_CQ UINT32_C(0x9)
	+ /* Destroy CQ command deletes and flushes the specified CQ. */
	+ #define CMDQ_BASE_OPCODE_DESTROY_CQ UINT32_C(0xa)
	+ /* Resize CQ command resizes the specified CQ. */
	+ #define CMDQ_BASE_OPCODE_RESIZE_CQ UINT32_C(0xc)
	+ /*
	+ * Allocate MRW command allocates a MR/MW with the specified
	+ * parameters and returns the region's L_KEY/R_KEY
	+ */
	+ #define CMDQ_BASE_OPCODE_ALLOCATE_MRW UINT32_C(0xd)
	+ /*
	+ * De-allocate key command frees a MR/MW entry associated with
	+ * the specified key.
	+ */
	+ #define CMDQ_BASE_OPCODE_DEALLOCATE_KEY UINT32_C(0xe)
	+ /* Register MR command registers memory to the specified MR. */
	+ #define CMDQ_BASE_OPCODE_REGISTER_MR UINT32_C(0xf)
	+ /* Deregister MR command de-registers memory from the specified MR. */
	+ #define CMDQ_BASE_OPCODE_DEREGISTER_MR UINT32_C(0x10)
	+ /* Add GID command adds a GID to the local address table. */
	+ #define CMDQ_BASE_OPCODE_ADD_GID UINT32_C(0x11)
	+ /* Delete GID command deletes a GID from the local address table. */
	+ #define CMDQ_BASE_OPCODE_DELETE_GID UINT32_C(0x12)
	+ /* Modify GID command modifies a GID in the local address table. */
	+ #define CMDQ_BASE_OPCODE_MODIFY_GID UINT32_C(0x17)
	+ /* Query GID command queries a GID in the local address table. */
	+ #define CMDQ_BASE_OPCODE_QUERY_GID UINT32_C(0x18)
	+ /* Create QP1 command allocates a QP1 only. */
	+ #define CMDQ_BASE_OPCODE_CREATE_QP1 UINT32_C(0x13)
	+ /* Destroy QP1 command deletes and flushes the specified QP1. */
	+ #define CMDQ_BASE_OPCODE_DESTROY_QP1 UINT32_C(0x14)
	+ /* Create AH command allocates an AH with the specified parameters. */
	+ #define CMDQ_BASE_OPCODE_CREATE_AH UINT32_C(0x15)
	+ /* Destroy AH command deletes the specified AH. */
	+ #define CMDQ_BASE_OPCODE_DESTROY_AH UINT32_C(0x16)
	+ /*
	+ * Initialize firmware command initializes the firmware with the
	+ * specified parameters.
	+ */
	+ #define CMDQ_BASE_OPCODE_INITIALIZE_FW UINT32_C(0x80)
	+ /* De-initialize firmware command deinitializes the firmware. */
	+ #define CMDQ_BASE_OPCODE_DEINITIALIZE_FW UINT32_C(0x81)
	+ /* Stop the function */
	+ #define CMDQ_BASE_OPCODE_STOP_FUNC UINT32_C(0x82)
	+ /* Query the HW capabilities for the function. */
	+ #define CMDQ_BASE_OPCODE_QUERY_FUNC UINT32_C(0x83)
	+ /*
	+ * Set the following resources for the function: - Max QP, CQ,
	+ * MR+MW, SRQ per PF - Max QP, CQ, MR+MW, SRQ per VF
	+ */
	+ #define CMDQ_BASE_OPCODE_SET_FUNC_RESOURCES UINT32_C(0x84)
	+ /*
	+ * Read the current state of any internal resource context. Can
	+ * only be issued from a PF.
	+ */
	+ #define CMDQ_BASE_OPCODE_READ_CONTEXT UINT32_C(0x85)
	+ /*
	+ * Send a request from VF to pass a command to the PF. VF HSI is
	+ * suspended until the PF returns the response
	+ */
	+ #define CMDQ_BASE_OPCODE_VF_BACKCHANNEL_REQUEST UINT32_C(0x86)
	+ /*
	+ * Read VF memory (primarily to get the backchannel request
	+ * blob). Can only be issued from a PF.
	+ */
	+ #define CMDQ_BASE_OPCODE_READ_VF_MEMORY UINT32_C(0x87)
	+ /*
	+ * Write VF memory (primarily to put the backchannel response
	+ * blob), and re-enable VF HSI (post a CAG completion to it). Can
	+ * only be issued from a PF.
	+ */
	+ #define CMDQ_BASE_OPCODE_COMPLETE_VF_REQUEST UINT32_C(0x88)
	+ /*
	+ * Extend resource (QPC, MRW, CQ, SRQ) array, after the host
	+ * allocates more. Can only be issued from a PF.
	+ */
	+ #define CMDQ_BASE_OPCODE_EXTEND_CONTEXT_ARRRAY UINT32_C(0x89)
	+ /* Map TC to COS. Can only be issued from a PF. */
	+ #define CMDQ_BASE_OPCODE_MAP_TC_TO_COS UINT32_C(0x8a)
	+ /* Query version. */
	+ #define CMDQ_BASE_OPCODE_QUERY_VERSION UINT32_C(0x8b)
	+ /* Modify congestion control. Can only be issued from a PF. */
	+ #define CMDQ_BASE_OPCODE_MODIFY_ROCE_CC UINT32_C(0x8c)
	+ /* Query congestion control. */
	+ #define CMDQ_BASE_OPCODE_QUERY_ROCE_CC UINT32_C(0x8d)
	+ uint8_t cmd_size;
	+ /* Size of the command in 16-byte units. */
	+ uint16_t flags;
	+ /* Flags and attribs of the command. */
	+ uint16_t cookie;
	+ /* Driver supplied handle to associate the command and the response. */
	+ uint8_t resp_size;
	+ /* Size of the response buffer in 16-byte units. */
	+ uint8_t reserved8;
	+ uint64_t resp_addr;
	+ /* Host address of the response. */
	+} __attribute__((packed));
	+
	+/* Create QP command (96 bytes) */
	+
	+struct cmdq_create_qp {
	+ uint8_t opcode;
	+ /* Command opcode. */
	+ /*
	+ * Create QP command allocates QP context with the specified SQ,
	+ * RQ/SRQ, CQ and other parameters.
	+ */
	+ #define CMDQ_CREATE_QP_OPCODE_CREATE_QP UINT32_C(0x1)
	+ uint8_t cmd_size;
	+ /* Size of the command in 16-byte units. */
	+ uint16_t flags;
	+ /* Flags and attribs of the command. */
	+ uint16_t cookie;
	+ /* Driver supplied handle to associate the command and the response. */
	+ uint8_t resp_size;
	+ /* Size of the response buffer in 16-byte units. */
	+ uint8_t reserved8;
	+ uint64_t resp_addr;
	+ /* Host address of the response. */
	+ uint64_t qp_handle;
	+ /* QP handle. */
	+ uint32_t qp_flags;
	+ /* Create QP flags. */
	+ /* SRQ is used. */
	+ #define CMDQ_CREATE_QP_QP_FLAGS_SRQ_USED UINT32_C(0x1)
	+ /* post CQE for all SQ WQEs. */
	+ #define CMDQ_CREATE_QP_QP_FLAGS_FORCE_COMPLETION UINT32_C(0x2)
	+ /* This QP can use reserved L_Key */
	+ #define CMDQ_CREATE_QP_QP_FLAGS_RESERVED_LKEY_ENABLE UINT32_C(0x4)
	+ /* This QP can fast register physical memory */
	+ #define CMDQ_CREATE_QP_QP_FLAGS_FR_PMR_ENABLED UINT32_C(0x8)
	+ uint8_t type;
	+ /* Supported QP types. */
	+ /* Reliable Connection. */
	+ #define CMDQ_CREATE_QP_TYPE_RC UINT32_C(0x2)
	+ /* Unreliable Datagram. */
	+ #define CMDQ_CREATE_QP_TYPE_UD UINT32_C(0x4)
	+ /* Raw Ethertype. */
	+ #define CMDQ_CREATE_QP_TYPE_RAW_ETHERTYPE UINT32_C(0x6)
	+ uint8_t sq_pg_size_sq_lvl;
	+ /* SQ page size. */
	+ /* SQ PBL indirect levels. */
	+ #define CMDQ_CREATE_QP_SQ_LVL_MASK UINT32_C(0xf)
	+ #define CMDQ_CREATE_QP_SQ_LVL_SFT 0
	+ /* PBL pointer is physical start address. */
	+ #define CMDQ_CREATE_QP_SQ_LVL_LVL_0 UINT32_C(0x0)
	+ /* PBL pointer points to PTE table. */
	+ #define CMDQ_CREATE_QP_SQ_LVL_LVL_1 UINT32_C(0x1)
	+ /*
	+ * PBL pointer points to PDE table with each entry pointing to
	+ * PTE tables.
	+ */
	+ #define CMDQ_CREATE_QP_SQ_LVL_LVL_2 UINT32_C(0x2)
	+ /* SQ page size. */
	+ #define CMDQ_CREATE_QP_SQ_PG_SIZE_MASK UINT32_C(0xf0)
	+ #define CMDQ_CREATE_QP_SQ_PG_SIZE_SFT 4
	+ /* 4KB. */
	+ #define CMDQ_CREATE_QP_SQ_PG_SIZE_PG_4K (UINT32_C(0x0) << 4)
	+ /* 8KB. */
	+ #define CMDQ_CREATE_QP_SQ_PG_SIZE_PG_8K (UINT32_C(0x1) << 4)
	+ /* 64KB. */
	+ #define CMDQ_CREATE_QP_SQ_PG_SIZE_PG_64K (UINT32_C(0x2) << 4)
	+ /* 2MB. */
	+ #define CMDQ_CREATE_QP_SQ_PG_SIZE_PG_2M (UINT32_C(0x3) << 4)
	+ /* 8MB. */
	+ #define CMDQ_CREATE_QP_SQ_PG_SIZE_PG_8M (UINT32_C(0x4) << 4)
	+ /* 1GB. */
	+ #define CMDQ_CREATE_QP_SQ_PG_SIZE_PG_1G (UINT32_C(0x5) << 4)
	+ uint8_t rq_pg_size_rq_lvl;
	+ /* RQ page size. */
	+ /* RQ PBL indirect levels. */
	+ #define CMDQ_CREATE_QP_RQ_LVL_MASK UINT32_C(0xf)
	+ #define CMDQ_CREATE_QP_RQ_LVL_SFT 0
	+ /* PBL pointer is physical start address. */
	+ #define CMDQ_CREATE_QP_RQ_LVL_LVL_0 UINT32_C(0x0)
	+ /* PBL pointer points to PTE table. */
	+ #define CMDQ_CREATE_QP_RQ_LVL_LVL_1 UINT32_C(0x1)
	+ /*
	+ * PBL pointer points to PDE table with each entry pointing to
	+ * PTE tables.
	+ */
	+ #define CMDQ_CREATE_QP_RQ_LVL_LVL_2 UINT32_C(0x2)
	+ /* RQ page size. */
	+ #define CMDQ_CREATE_QP_RQ_PG_SIZE_MASK UINT32_C(0xf0)
	+ #define CMDQ_CREATE_QP_RQ_PG_SIZE_SFT 4
	+ /* 4KB. */
	+ #define CMDQ_CREATE_QP_RQ_PG_SIZE_PG_4K (UINT32_C(0x0) << 4)
	+ /* 8KB. */
	+ #define CMDQ_CREATE_QP_RQ_PG_SIZE_PG_8K (UINT32_C(0x1) << 4)
	+ /* 64KB. */
	+ #define CMDQ_CREATE_QP_RQ_PG_SIZE_PG_64K (UINT32_C(0x2) << 4)
	+ /* 2MB. */
	+ #define CMDQ_CREATE_QP_RQ_PG_SIZE_PG_2M (UINT32_C(0x3) << 4)
	+ /* 8MB. */
	+ #define CMDQ_CREATE_QP_RQ_PG_SIZE_PG_8M (UINT32_C(0x4) << 4)
	+ /* 1GB. */
	+ #define CMDQ_CREATE_QP_RQ_PG_SIZE_PG_1G (UINT32_C(0x5) << 4)
	+ uint8_t unused_0;
	+ uint32_t dpi;
	+ /* Doorbell page index. */
	+ uint32_t sq_size;
	+ /* Max number of SQ wqes. */
	+ uint32_t rq_size;
	+ /* Max number of RQ wqes. */
	+ uint16_t sq_fwo_sq_sge;
	+ /* Offset of First WQE in the first SQ page, in 128 byte units */
	+ /* Max send SGEs per SWQE. */
	+ #define CMDQ_CREATE_QP_SQ_SGE_MASK UINT32_C(0xf)
	+ #define CMDQ_CREATE_QP_SQ_SGE_SFT 0
	+ /* Offset of First WQE in the first SQ page, in 128 byte units */
	+ #define CMDQ_CREATE_QP_SQ_FWO_MASK UINT32_C(0xfff0)
	+ #define CMDQ_CREATE_QP_SQ_FWO_SFT 4
	+ uint16_t rq_fwo_rq_sge;
	+ /* Offset of First WQE in the first RQ page, in 128 byte units */
	+ /* Max recv SGEs per RWQE (NOT SUPPORTED BY HARDWARE). */
	+ #define CMDQ_CREATE_QP_RQ_SGE_MASK UINT32_C(0xf)
	+ #define CMDQ_CREATE_QP_RQ_SGE_SFT 0
	+ /* Offset of First WQE in the first RQ page, in 128 byte units */
	+ #define CMDQ_CREATE_QP_RQ_FWO_MASK UINT32_C(0xfff0)
	+ #define CMDQ_CREATE_QP_RQ_FWO_SFT 4
	+ uint32_t scq_cid;
	+ /* Send CQ context id. */
	+ uint32_t rcq_cid;
	+ /* Receive CQ context id. */
	+ uint32_t srq_cid;
	+ /* SRQ CQ context id. */
	+ uint32_t pd_id;
	+ /* Protection domain id. */
	+ uint64_t sq_pbl;
	+ /* SQ PBL physical address. */
	+ uint64_t rq_pbl;
	+ /* RQ PBL physical address. */
	+ uint64_t irrq_addr;
	+ /* IRRQ address. */
	+ uint64_t orrq_addr;
	+ /* ORRQ address. */
	+} __attribute__((packed));
	+
	+/* Destroy QP command (24 bytes) */
	+
	+struct cmdq_destroy_qp {
	+ uint8_t opcode;
	+ /* Command opcode. */
	+ /*
	+ * Destroy QP command deletes the QP context and ceases any
	+ * further reference.
	+ */
	+ #define CMDQ_DESTROY_QP_OPCODE_DESTROY_QP UINT32_C(0x2)
	+ uint8_t cmd_size;
	+ /* Size of the command in 16-byte units. */
	+ uint16_t flags;
	+ /* Flags and attribs of the command. */
	+ uint16_t cookie;
	+ /* Driver supplied handle to associate the command and the response. */
	+ uint8_t resp_size;
	+ /* Size of the response buffer in 16-byte units. */
	+ uint8_t reserved8;
	+ uint64_t resp_addr;
	+ /* Host address of the response. */
	+ uint32_t qp_cid;
	+ /* QP context id */
	+ uint32_t unused_0;
	+} __attribute__((packed));
	+
	+/* Modify QP command (112 bytes) */
	+
	+struct cmdq_modify_qp {
	+ uint8_t opcode;
	+ /* Command opcode. */
	+ /*
	+ * Modify QP command changes QP states and other QP specific
	+ * parameters.
	+ */
	+ #define CMDQ_MODIFY_QP_OPCODE_MODIFY_QP UINT32_C(0x3)
	+ uint8_t cmd_size;
	+ /* Size of the command in 16-byte units. */
	+ uint16_t flags;
	+ /* Flags and attribs of the command. */
	+ uint16_t cookie;
	+ /* Driver supplied handle to associate the command and the response. */
	+ uint8_t resp_size;
	+ /* Size of the response buffer in 16-byte units. */
	+ uint8_t reserved8;
	+ uint64_t resp_addr;
	+ /* Host address of the response. */
	+ uint32_t modify_mask;
	+ /* Modify mask signifies the field that is requesting the change. */
	+ /* QP state change. */
	+ #define CMDQ_MODIFY_QP_MODIFY_MASK_STATE UINT32_C(0x1)
	+ /* Enable SQ drain asynchronous notification change. */
	+ #define CMDQ_MODIFY_QP_MODIFY_MASK_EN_SQD_ASYNC_NOTIFY UINT32_C(0x2)
	+ /* Access change. */
	+ #define CMDQ_MODIFY_QP_MODIFY_MASK_ACCESS UINT32_C(0x4)
	+ /* P_KEY change. */
	+ #define CMDQ_MODIFY_QP_MODIFY_MASK_PKEY UINT32_C(0x8)
	+ /* Q_KEY index change. */
	+ #define CMDQ_MODIFY_QP_MODIFY_MASK_QKEY UINT32_C(0x10)
	+ /* Destination GID change. */
	+ #define CMDQ_MODIFY_QP_MODIFY_MASK_DGID UINT32_C(0x20)
	+ /* Flow label change. */
	+ #define CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL UINT32_C(0x40)
	+ /* SGID change. */
	+ #define CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX UINT32_C(0x80)
	+ /* Hop limit change. */
	+ #define CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT UINT32_C(0x100)
	+ /* Traffic class change. */
	+ #define CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS UINT32_C(0x200)
	+ /* destination MAC change. */
	+ #define CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC UINT32_C(0x400)
	+ /* unused is 1 b */
	+ #define CMDQ_MODIFY_QP_MODIFY_MASK_UNUSED UINT32_C(0x800)
	+ /* Path MTU change. */
	+ #define CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU UINT32_C(0x1000)
	+ /* Timeout change. */
	+ #define CMDQ_MODIFY_QP_MODIFY_MASK_TIMEOUT UINT32_C(0x2000)
	+ /* Retry count change. */
	+ #define CMDQ_MODIFY_QP_MODIFY_MASK_RETRY_CNT UINT32_C(0x4000)
	+ /* RNR Retry change. */
	+ #define CMDQ_MODIFY_QP_MODIFY_MASK_RNR_RETRY UINT32_C(0x8000)
	+ /* RQ start packet sequence number change. */
	+ #define CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN UINT32_C(0x10000)
	+ /* Max outstanding RDMA read atomic change. */
	+ #define CMDQ_MODIFY_QP_MODIFY_MASK_MAX_RD_ATOMIC UINT32_C(0x20000)
	+ /* RNR minimum timer change. */
	+ #define CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER UINT32_C(0x40000)
	+ /* SQ start packet sequence number change. */
	+ #define CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN UINT32_C(0x80000)
	+ /* Max destination outstanding RDMA read atomic change. */
	+ #define CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC UINT32_C(0x100000)
	+ /* Max send WQE change. */
	+ #define CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SIZE UINT32_C(0x200000)
	+ /* Max recv WQE change. */
	+ #define CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SIZE UINT32_C(0x400000)
	+ /* Max recv SGEs per SWQE change. */
	+ #define CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SGE UINT32_C(0x800000)
	+ /* Max send SGEs per RWQE change. */
	+ #define CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SGE UINT32_C(0x1000000)
	+ /* Max inline data length change. */
	+ #define CMDQ_MODIFY_QP_MODIFY_MASK_MAX_INLINE_DATA UINT32_C(0x2000000)
	+ /* Destination QP id change. */
	+ #define CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID UINT32_C(0x4000000)
	+ /* Source MAC change. */
	+ #define CMDQ_MODIFY_QP_MODIFY_MASK_SRC_MAC UINT32_C(0x8000000)
	+ /* Source VLAN id change. */
	+ #define CMDQ_MODIFY_QP_MODIFY_MASK_VLAN_ID UINT32_C(0x10000000)
	+ /* Congestion control RoCE v2 change. */
	+ #define CMDQ_MODIFY_QP_MODIFY_MASK_ENABLE_CC UINT32_C(0x20000000)
	+ /* IP TOS ECN change */
	+ #define CMDQ_MODIFY_QP_MODIFY_MASK_TOS_ECN UINT32_C(0x40000000)
	+ /* IP TOS DSCP change */
	+ #define CMDQ_MODIFY_QP_MODIFY_MASK_TOS_DSCP UINT32_C(0x80000000)
	+ uint32_t qp_cid;
	+ /* QP context id. */
	+ uint8_t network_type_en_sqd_async_notify_new_state;
	+ /* network type. */
	+ /* New QP state. */
	+ #define CMDQ_MODIFY_QP_NEW_STATE_MASK UINT32_C(0xf)
	+ #define CMDQ_MODIFY_QP_NEW_STATE_SFT 0
	+ /* Reset. */
	+ #define CMDQ_MODIFY_QP_NEW_STATE_RESET UINT32_C(0x0)
	+ /* Init. */
	+ #define CMDQ_MODIFY_QP_NEW_STATE_INIT UINT32_C(0x1)
	+ /* Ready To Receive. */
	+ #define CMDQ_MODIFY_QP_NEW_STATE_RTR UINT32_C(0x2)
	+ /* Ready To Send. */
	+ #define CMDQ_MODIFY_QP_NEW_STATE_RTS UINT32_C(0x3)
	+ /* SQ Drain. */
	+ #define CMDQ_MODIFY_QP_NEW_STATE_SQD UINT32_C(0x4)
	+ /* SQ Error. */
	+ #define CMDQ_MODIFY_QP_NEW_STATE_SQE UINT32_C(0x5)
	+ /* Error. */
	+ #define CMDQ_MODIFY_QP_NEW_STATE_ERR UINT32_C(0x6)
	+ /* Enable SQ drain asynchronous notification. */
	+ #define CMDQ_MODIFY_QP_EN_SQD_ASYNC_NOTIFY UINT32_C(0x10)
	+ /* unused1 is 1 b */
	+ /* network type. */
	+ #define CMDQ_MODIFY_QP_NETWORK_TYPE_MASK UINT32_C(0xc0)
	+ #define CMDQ_MODIFY_QP_NETWORK_TYPE_SFT 6
	+ /* RoCEv1. */
	+ #define CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV1 (UINT32_C(0x0) << 6)
	+ /* RoCEv2 IPv4. */
	+ #define CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV4 (UINT32_C(0x2) << 6)
	+ /* RoCEv2 IPv6. */
	+ #define CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV6 (UINT32_C(0x3) << 6)
	+ uint8_t access;
	+ /* Access flags. */
	+ /* Local write access. */
	+ #define CMDQ_MODIFY_QP_ACCESS_LOCAL_WRITE UINT32_C(0x1)
	+ /* Remote write access. */
	+ #define CMDQ_MODIFY_QP_ACCESS_REMOTE_WRITE UINT32_C(0x2)
	+ /* Remote read access. */
	+ #define CMDQ_MODIFY_QP_ACCESS_REMOTE_READ UINT32_C(0x4)
	+ /* Remote atomic access. */
	+ #define CMDQ_MODIFY_QP_ACCESS_REMOTE_ATOMIC UINT32_C(0x8)
	+ uint16_t pkey;
	+ /* P_KEY. */
	+ uint32_t qkey;
	+ /* Q_KEY. */
	+ uint32_t dgid[4];
	+ /* Destination GID. */
	+ uint32_t flow_label;
	+ /* Flow label. */
	+ uint16_t sgid_index;
	+ /* Source GID index. */
	+ uint8_t hop_limit;
	+ /* Hop limit. */
	+ uint8_t traffic_class;
	+ /* Traffic class. */
	+ uint16_t dest_mac[3];
	+ /* Destination MAC address. */
	+ uint8_t tos_dscp_tos_ecn;
	+ /* IP TOS DSCP. */
	+ /* IP TOS ECN. Valid values are 1 or 2 when ECN is enabled. */
	+ #define CMDQ_MODIFY_QP_TOS_ECN_MASK UINT32_C(0x3)
	+ #define CMDQ_MODIFY_QP_TOS_ECN_SFT 0
	+ /* IP TOS DSCP. */
	+ #define CMDQ_MODIFY_QP_TOS_DSCP_MASK UINT32_C(0xfc)
	+ #define CMDQ_MODIFY_QP_TOS_DSCP_SFT 2
	+ uint8_t path_mtu;
	+ /* Path MTU. */
	+ /* unused4 is 4 b */
	+ /* Path MTU. */
	+ #define CMDQ_MODIFY_QP_PATH_MTU_MASK UINT32_C(0xf0)
	+ #define CMDQ_MODIFY_QP_PATH_MTU_SFT 4
	+ /* 256. */
	+ #define CMDQ_MODIFY_QP_PATH_MTU_MTU_256 (UINT32_C(0x0) << 4)
	+ /* 512. */
	+ #define CMDQ_MODIFY_QP_PATH_MTU_MTU_512 (UINT32_C(0x1) << 4)
	+ /* 1024. */
	+ #define CMDQ_MODIFY_QP_PATH_MTU_MTU_1024 (UINT32_C(0x2) << 4)
	+ /* 2048. */
	+ #define CMDQ_MODIFY_QP_PATH_MTU_MTU_2048 (UINT32_C(0x3) << 4)
	+ /* 4096. */
	+ #define CMDQ_MODIFY_QP_PATH_MTU_MTU_4096 (UINT32_C(0x4) << 4)
	+ /* 8192. */
	+ #define CMDQ_MODIFY_QP_PATH_MTU_MTU_8192 (UINT32_C(0x5) << 4)
	+ uint8_t timeout;
	+ /* Timeout value for SWQEs. */
	+ uint8_t retry_cnt;
	+ /* Max retry count for WQEs. */
	+ uint8_t rnr_retry;
	+ /* Max RNR retry count for WQEs. */
	+ uint8_t min_rnr_timer;
	+ /* Min RNR timer that the QP will report to the remote. */
	+ uint32_t rq_psn;
	+ /* RQ start packet sequence number. */
	+ uint32_t sq_psn;
	+ /* SQ start packet sequence number. */
	+ uint8_t max_rd_atomic;
	+ /* Max outstanding RDMA read atomic. */
	+ uint8_t max_dest_rd_atomic;
	+ /* Max destination outstanding RDMA read atomic. */
	+ uint16_t enable_cc;
	+ /* unused15 is 15 b */
	+ /* Enable congestion control. */
	+ #define CMDQ_MODIFY_QP_ENABLE_CC UINT32_C(0x1)
	+ /* unused15 is 15 b */
	+ uint32_t sq_size;
	+ /* Max send WQE. */
	+ uint32_t rq_size;
	+ /* Max recv WQE. */
	+ uint16_t sq_sge;
	+ /* Max send SGEs per SWQE. */
	+ uint16_t rq_sge;
	+ /* Max recv SGEs per RWQE. */
	+ uint32_t max_inline_data;
	+ /* Max inline data length (upto 120 bytes). */
	+ uint32_t dest_qp_id;
	+ /* Destination QP id. */
	+ uint32_t unused_3;
	+ uint16_t src_mac[3];
	+ /* Source MAC. (Unused. Comes from Source GID index) */
	+ uint16_t vlan_pcp_vlan_dei_vlan_id;
	+ /* VLAN PCP field - Priority Code Point. */
	+ /* VLAN id. (Unused. Comes from Source GID index) */
	+ #define CMDQ_MODIFY_QP_VLAN_ID_MASK UINT32_C(0xfff)
	+ #define CMDQ_MODIFY_QP_VLAN_ID_SFT 0
	+ /* VLAN DEI field - Drop Eligibility Indicator. */
	+ #define CMDQ_MODIFY_QP_VLAN_DEI UINT32_C(0x1000)
	+ /* VLAN PCP field - Priority Code Point. */
	+ #define CMDQ_MODIFY_QP_VLAN_PCP_MASK UINT32_C(0xe000)
	+ #define CMDQ_MODIFY_QP_VLAN_PCP_SFT 13
	+} __attribute__((packed));
	+
	+/* Query QP command (24 bytes) */
	+
	+struct cmdq_query_qp {
	+ uint8_t opcode;
	+ /* Command opcode. */
	+ /* Query QP command retrieves info about the specified QP. */
	+ #define CMDQ_QUERY_QP_OPCODE_QUERY_QP UINT32_C(0x4)
	+ uint8_t cmd_size;
	+ /* Size of the command in 16-byte units. */
	+ uint16_t flags;
	+ /* Flags and attribs of the command. */
	+ uint16_t cookie;
	+ /* Driver supplied handle to associate the command and the response. */
	+ uint8_t resp_size;
	+ /* Size of the response buffer in 16-byte units. */
	+ uint8_t reserved8;
	+ uint64_t resp_addr;
	+ /* Host address of the response. */
	+ uint32_t qp_cid;
	+ /* QP context id */
	+ uint32_t unused_0;
	+} __attribute__((packed));
	+
	+/* Create SRQ command (48 bytes) */
	+
	+struct cmdq_create_srq {
	+ uint8_t opcode;
	+ /* Command opcode. */
	+ /* Create SRQ command allocates a SRQ with the specified parameters. */
	+ #define CMDQ_CREATE_SRQ_OPCODE_CREATE_SRQ UINT32_C(0x5)
	+ uint8_t cmd_size;
	+ /* Size of the command in 16-byte units. */
	+ uint16_t flags;
	+ /* Flags and attribs of the command. */
	+ uint16_t cookie;
	+ /* Driver supplied handle to associate the command and the response. */
	+ uint8_t resp_size;
	+ /* Size of the response buffer in 16-byte units. */
	+ uint8_t reserved8;
	+ uint64_t resp_addr;
	+ /* Host address of the response. */
	+ uint64_t srq_handle;
	+ /* SRQ handle. */
	+ uint16_t pg_size_lvl;
	+ /* unused11 is 11 b */
	+ /* SRQ PBL indirect levels. */
	+ #define CMDQ_CREATE_SRQ_LVL_MASK UINT32_C(0x3)
	+ #define CMDQ_CREATE_SRQ_LVL_SFT 0
	+ /* PBL pointer is physical start address. */
	+ #define CMDQ_CREATE_SRQ_LVL_LVL_0 UINT32_C(0x0)
	+ /* PBL pointer points to PTE table. */
	+ #define CMDQ_CREATE_SRQ_LVL_LVL_1 UINT32_C(0x1)
	+ /*
	+ * PBL pointer points to PDE table with each entry pointing to
	+ * PTE tables.
	+ */
	+ #define CMDQ_CREATE_SRQ_LVL_LVL_2 UINT32_C(0x2)
	+ /* page size. */
	+ #define CMDQ_CREATE_SRQ_PG_SIZE_MASK UINT32_C(0x1c)
	+ #define CMDQ_CREATE_SRQ_PG_SIZE_SFT 2
	+ /* 4KB. */
	+ #define CMDQ_CREATE_SRQ_PG_SIZE_PG_4K (UINT32_C(0x0) << 2)
	+ /* 8KB. */
	+ #define CMDQ_CREATE_SRQ_PG_SIZE_PG_8K (UINT32_C(0x1) << 2)
	+ /* 64KB. */
	+ #define CMDQ_CREATE_SRQ_PG_SIZE_PG_64K (UINT32_C(0x2) << 2)
	+ /* 2MB. */
	+ #define CMDQ_CREATE_SRQ_PG_SIZE_PG_2M (UINT32_C(0x3) << 2)
	+ /* 8MB. */
	+ #define CMDQ_CREATE_SRQ_PG_SIZE_PG_8M (UINT32_C(0x4) << 2)
	+ /* 1GB. */
	+ #define CMDQ_CREATE_SRQ_PG_SIZE_PG_1G (UINT32_C(0x5) << 2)
	+ /* unused11 is 11 b */
	+ uint16_t eventq_id;
	+ /* unused4 is 4 b */
	+ /* eventq_id is 12 b */
	+ #define CMDQ_CREATE_SRQ_EVENTQ_ID_MASK UINT32_C(0xfff)
	+ #define CMDQ_CREATE_SRQ_EVENTQ_ID_SFT 0
	+ /* unused4 is 4 b */
	+ uint16_t srq_size;
	+ /* Max number of SRQ wqes. */
	+ uint16_t srq_fwo;
	+ /* Offsetof first WQE in the first page of SRQ, in 128 byte units */
	+ uint32_t dpi;
	+ /* Doorbell page index. */
	+ uint32_t pd_id;
	+ /* Protection domain id. */
	+ uint64_t pbl;
	+ /* RQ PBL physical address. */
	+} __attribute__((packed));
	+
	+/* Destroy SRQ command (24 bytes) */
	+
	+struct cmdq_destroy_srq {
	+ uint8_t opcode;
	+ /* Command opcode. */
	+ /* Destroy SRQ command deletes and flushes the specified SRQ. */
	+ #define CMDQ_DESTROY_SRQ_OPCODE_DESTROY_SRQ UINT32_C(0x6)
	+ uint8_t cmd_size;
	+ /* Size of the command in 16-byte units. */
	+ uint16_t flags;
	+ /* Flags and attribs of the command. */
	+ uint16_t cookie;
	+ /* Driver supplied handle to associate the command and the response. */
	+ uint8_t resp_size;
	+ /* Size of the response buffer in 16-byte units. */
	+ uint8_t reserved8;
	+ uint64_t resp_addr;
	+ /* Host address of the response. */
	+ uint32_t srq_cid;
	+ /* SRQ context id */
	+ uint32_t unused_0;
	+} __attribute__((packed));
	+
	+/* Query SRQ command (24 bytes) */
	+
	+struct cmdq_query_srq {
	+ uint8_t opcode;
	+ /* Command opcode. */
	+ /* Query SRP command retrieves info about the specified SRQ. */
	+ #define CMDQ_QUERY_SRQ_OPCODE_QUERY_SRQ UINT32_C(0x8)
	+ uint8_t cmd_size;
	+ /* Size of the command in 16-byte units. */
	+ uint16_t flags;
	+ /* Flags and attribs of the command. */
	+ uint16_t cookie;
	+ /* Driver supplied handle to associate the command and the response. */
	+ uint8_t resp_size;
	+ /* Size of the response buffer in 16-byte units. */
	+ uint8_t reserved8;
	+ uint64_t resp_addr;
	+ /* Host address of the response. */
	+ uint32_t srq_cid;
	+ /* SRQ context id */
	+ uint32_t unused_0;
	+} __attribute__((packed));
	+
	+/* Create CQ command (48 bytes) */
	+
	+struct cmdq_create_cq {
	+ uint8_t opcode;
	+ /* Command opcode. */
	+ /* Create CQ command allocates a CQ with the specified parameters. */
	+ #define CMDQ_CREATE_CQ_OPCODE_CREATE_CQ UINT32_C(0x9)
	+ uint8_t cmd_size;
	+ /* Size of the command in 16-byte units. */
	+ uint16_t flags;
	+ /* Flags and attribs of the command. */
	+ uint16_t cookie;
	+ /* Driver supplied handle to associate the command and the response. */
	+ uint8_t resp_size;
	+ /* Size of the response buffer in 16-byte units. */
	+ uint8_t reserved8;
	+ uint64_t resp_addr;
	+ /* Host address of the response. */
	+ uint64_t cq_handle;
	+ /* CQ handle. */
	+ uint32_t pg_size_lvl;
	+ /* unused27 is 27 b */
	+ /* PBL indirect levels. */
	+ #define CMDQ_CREATE_CQ_LVL_MASK UINT32_C(0x3)
	+ #define CMDQ_CREATE_CQ_LVL_SFT 0
	+ /* PBL pointer is physical start address. */
	+ #define CMDQ_CREATE_CQ_LVL_LVL_0 UINT32_C(0x0)
	+ /* PBL pointer points to PTE table. */
	+ #define CMDQ_CREATE_CQ_LVL_LVL_1 UINT32_C(0x1)
	+ /*
	+ * PBL pointer points to PDE table with each entry pointing to
	+ * PTE tables.
	+ */
	+ #define CMDQ_CREATE_CQ_LVL_LVL_2 UINT32_C(0x2)
	+ /* page size. */
	+ #define CMDQ_CREATE_CQ_PG_SIZE_MASK UINT32_C(0x1c)
	+ #define CMDQ_CREATE_CQ_PG_SIZE_SFT 2
	+ /* 4KB. */
	+ #define CMDQ_CREATE_CQ_PG_SIZE_PG_4K (UINT32_C(0x0) << 2)
	+ /* 8KB. */
	+ #define CMDQ_CREATE_CQ_PG_SIZE_PG_8K (UINT32_C(0x1) << 2)
	+ /* 64KB. */
	+ #define CMDQ_CREATE_CQ_PG_SIZE_PG_64K (UINT32_C(0x2) << 2)
	+ /* 2MB. */
	+ #define CMDQ_CREATE_CQ_PG_SIZE_PG_2M (UINT32_C(0x3) << 2)
	+ /* 8MB. */
	+ #define CMDQ_CREATE_CQ_PG_SIZE_PG_8M (UINT32_C(0x4) << 2)
	+ /* 1GB. */
	+ #define CMDQ_CREATE_CQ_PG_SIZE_PG_1G (UINT32_C(0x5) << 2)
	+ /* unused27 is 27 b */
	+ uint32_t cq_fco_cnq_id;
	+ /* Offset of first CQE in the first Page, in 32 byte units */
	+ /* cnq_id is 12 b */
	+ #define CMDQ_CREATE_CQ_CNQ_ID_MASK UINT32_C(0xfff)
	+ #define CMDQ_CREATE_CQ_CNQ_ID_SFT 0
	+ /* Offset of first CQE in the first Page, in 32 byte units */
	+ #define CMDQ_CREATE_CQ_CQ_FCO_MASK UINT32_C(0xfffff000)
	+ #define CMDQ_CREATE_CQ_CQ_FCO_SFT 12
	+ uint32_t dpi;
	+ /* Doorbell page index. */
	+ uint32_t cq_size;
	+ /* Max number of CQ wqes. */
	+ uint64_t pbl;
	+ /* CQ PBL physical address. */
	+} __attribute__((packed));
	+
	+/* Destroy CQ command (24 bytes) */
	+
	+struct cmdq_destroy_cq {
	+ uint8_t opcode;
	+ /* Command opcode. */
	+ /* Destroy CQ command deletes and flushes the specified CQ. */
	+ #define CMDQ_DESTROY_CQ_OPCODE_DESTROY_CQ UINT32_C(0xa)
	+ uint8_t cmd_size;
	+ /* Size of the command in 16-byte units. */
	+ uint16_t flags;
	+ /* Flags and attribs of the command. */
	+ uint16_t cookie;
	+ /* Driver supplied handle to associate the command and the response. */
	+ uint8_t resp_size;
	+ /* Size of the response buffer in 16-byte units. */
	+ uint8_t reserved8;
	+ uint64_t resp_addr;
	+ /* Host address of the response. */
	+ uint32_t cq_cid;
	+ /* CQ context id */
	+ uint32_t unused_0;
	+} __attribute__((packed));
	+
	+/* Resize CQ command (40 bytes) */
	+
	+struct cmdq_resize_cq {
	+ uint8_t opcode;
	+ /* Command opcode. */
	+ /* Resize CQ command resizes the specified CQ. */
	+ #define CMDQ_RESIZE_CQ_OPCODE_RESIZE_CQ UINT32_C(0xc)
	+ uint8_t cmd_size;
	+ /* Size of the command in 16-byte units. */
	+ uint16_t flags;
	+ /* Flags and attribs of the command. */
	+ uint16_t cookie;
	+ /* Driver supplied handle to associate the command and the response. */
	+ uint8_t resp_size;
	+ /* Size of the response buffer in 16-byte units. */
	+ uint8_t reserved8;
	+ uint64_t resp_addr;
	+ /* Host address of the response. */
	+ uint32_t cq_cid;
	+ /* CQ context id */
	+ uint32_t new_cq_size_pg_size_lvl;
	+ /* PBL indirect levels. */
	+ #define CMDQ_RESIZE_CQ_LVL_MASK UINT32_C(0x3)
	+ #define CMDQ_RESIZE_CQ_LVL_SFT 0
	+ /* PBL pointer is physical start address. */
	+ #define CMDQ_RESIZE_CQ_LVL_LVL_0 UINT32_C(0x0)
	+ /* PBL pointer points to PTE table. */
	+ #define CMDQ_RESIZE_CQ_LVL_LVL_1 UINT32_C(0x1)
	+ /*
	+ * PBL pointer points to PDE table with each entry pointing to
	+ * PTE tables.
	+ */
	+ #define CMDQ_RESIZE_CQ_LVL_LVL_2 UINT32_C(0x2)
	+ /* page size. */
	+ #define CMDQ_RESIZE_CQ_PG_SIZE_MASK UINT32_C(0x1c)
	+ #define CMDQ_RESIZE_CQ_PG_SIZE_SFT 2
	+ /* 4KB. */
	+ #define CMDQ_RESIZE_CQ_PG_SIZE_PG_4K (UINT32_C(0x0) << 2)
	+ /* 8KB. */
	+ #define CMDQ_RESIZE_CQ_PG_SIZE_PG_8K (UINT32_C(0x1) << 2)
	+ /* 64KB. */
	+ #define CMDQ_RESIZE_CQ_PG_SIZE_PG_64K (UINT32_C(0x2) << 2)
	+ /* 2MB. */
	+ #define CMDQ_RESIZE_CQ_PG_SIZE_PG_2M (UINT32_C(0x3) << 2)
	+ /* 8MB. */
	+ #define CMDQ_RESIZE_CQ_PG_SIZE_PG_8M (UINT32_C(0x4) << 2)
	+ /* 1GB. */
	+ #define CMDQ_RESIZE_CQ_PG_SIZE_PG_1G (UINT32_C(0x5) << 2)
	+ /* New max number of CQ wqes. */
	+ #define CMDQ_RESIZE_CQ_NEW_CQ_SIZE_MASK UINT32_C(0x1fffe0)
	+ #define CMDQ_RESIZE_CQ_NEW_CQ_SIZE_SFT 5
	+ uint64_t new_pbl;
	+ /* CQ PBL physical address. */
	+ uint32_t new_cq_fco;
	+ /* Offset of first CQE in the first Page, in 32 byte units */
	+ uint32_t unused_2;
	+} __attribute__((packed));
	+
	+/* Allocate MRW command (32 bytes) */
	+
	+struct cmdq_allocate_mrw {
	+ uint8_t opcode;
	+ /* Command opcode. */
	+ /*
	+ * Allocate MRW command allocates a MR/MW with the specified
	+ * parameters and returns the region's L_KEY/R_KEY
	+ */
	+ #define CMDQ_ALLOCATE_MRW_OPCODE_ALLOCATE_MRW UINT32_C(0xd)
	+ uint8_t cmd_size;
	+ /* Size of the command in 16-byte units. */
	+ uint16_t flags;
	+ /* Flags and attribs of the command. */
	+ uint16_t cookie;
	+ /* Driver supplied handle to associate the command and the response. */
	+ uint8_t resp_size;
	+ /* Size of the response buffer in 16-byte units. */
	+ uint8_t reserved8;
	+ uint64_t resp_addr;
	+ /* Host address of the response. */
	+ uint64_t mrw_handle;
	+ /* MRW handle. */
	+ uint8_t mrw_flags;
	+ /* unused4 is 4 b */
	+ /* Allocate MRW flags. */
	+ #define CMDQ_ALLOCATE_MRW_MRW_FLAGS_MASK UINT32_C(0xf)
	+ #define CMDQ_ALLOCATE_MRW_MRW_FLAGS_SFT 0
	+ /* Allocate Memory Region */
	+ #define CMDQ_ALLOCATE_MRW_MRW_FLAGS_MR UINT32_C(0x0)
	+ /* Allocate Physical Memory Region */
	+ #define CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR UINT32_C(0x1)
	+ /* Allocate Memory Window (type 1) */
	+ #define CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE1 UINT32_C(0x2)
	+ /* Allocate Memory Window (type 2A) */
	+ #define CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2A UINT32_C(0x3)
	+ /* Allocate Memory Window (type 2B) */
	+ #define CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2B UINT32_C(0x4)
	+ /* unused4 is 4 b */
	+ uint8_t access;
	+ /* Access flags. */
	+ #define CMDQ_ALLOCATE_MRW_ACCESS_RESERVED_MASK UINT32_C(0x1f)
	+ #define CMDQ_ALLOCATE_MRW_ACCESS_RESERVED_SFT 0
	+ /* Consumer owns the key */
	+ #define CMDQ_ALLOCATE_MRW_ACCESS_CONSUMER_OWNED_KEY UINT32_C(0x20)
	+ uint16_t unused_1;
	+ /* unused16 is 16 b */
	+ uint32_t pd_id;
	+ /* Protection domain id. */
	+} __attribute__((packed));
	+
	+/* De-allocate key command (24 bytes) */
	+
	+struct cmdq_deallocate_key {
	+ uint8_t opcode;
	+ /* Command opcode. */
	+ /*
	+ * De-allocate key command frees a MR/MW entry associated with
	+ * the specified key.
	+ */
	+ #define CMDQ_DEALLOCATE_KEY_OPCODE_DEALLOCATE_KEY UINT32_C(0xe)
	+ uint8_t cmd_size;
	+ /* Size of the command in 16-byte units. */
	+ uint16_t flags;
	+ /* Flags and attribs of the command. */
	+ uint16_t cookie;
	+ /* Driver supplied handle to associate the command and the response. */
	+ uint8_t resp_size;
	+ /* Size of the response buffer in 16-byte units. */
	+ uint8_t reserved8;
	+ uint64_t resp_addr;
	+ /* Host address of the response. */
	+ uint8_t mrw_flags;
	+ /* unused4 is 4 b */
	+ /* Deallocate MRW flags. */
	+ #define CMDQ_DEALLOCATE_KEY_MRW_FLAGS_MASK UINT32_C(0xf)
	+ #define CMDQ_DEALLOCATE_KEY_MRW_FLAGS_SFT 0
	+ /* Deallocate Memory Region */
	+ #define CMDQ_DEALLOCATE_KEY_MRW_FLAGS_MR UINT32_C(0x0)
	+ /* Deallocate Physical Memory Region */
	+ #define CMDQ_DEALLOCATE_KEY_MRW_FLAGS_PMR UINT32_C(0x1)
	+ /* Deallocate Memory Window (type 1) */
	+ #define CMDQ_DEALLOCATE_KEY_MRW_FLAGS_MW_TYPE1 UINT32_C(0x2)
	+ /* Deallocate Memory Window (type 2A) */
	+ #define CMDQ_DEALLOCATE_KEY_MRW_FLAGS_MW_TYPE2A UINT32_C(0x3)
	+ /* Deallocate Memory Window (type 2B) */
	+ #define CMDQ_DEALLOCATE_KEY_MRW_FLAGS_MW_TYPE2B UINT32_C(0x4)
	+ /* unused4 is 4 b */
	+ uint8_t unused_1[3];
	+ /* unused24 is 24 b */
	+ uint32_t key;
	+ /* key is 32 b */
	+} __attribute__((packed));
	+
	+/* Register MR command (48 bytes) */
	+
	+struct cmdq_register_mr {
	+ uint8_t opcode;
	+ /* Command opcode. */
	+ /* Register MR command registers memory to the specified MR. */
	+ #define CMDQ_REGISTER_MR_OPCODE_REGISTER_MR UINT32_C(0xf)
	+ uint8_t cmd_size;
	+ /* Size of the command in 16-byte units. */
	+ uint16_t flags;
	+ /* Flags and attribs of the command. */
	+ uint16_t cookie;
	+ /* Driver supplied handle to associate the command and the response. */
	+ uint8_t resp_size;
	+ /* Size of the response buffer in 16-byte units. */
	+ uint8_t reserved8;
	+ uint64_t resp_addr;
	+ /* Host address of the response. */
	+ uint8_t log2_pg_size_lvl;
	+ /* unused1 is 1 b */
	+ /* PBL indirect levels. */
	+ #define CMDQ_REGISTER_MR_LVL_MASK UINT32_C(0x3)
	+ #define CMDQ_REGISTER_MR_LVL_SFT 0
	+ /* PBL pointer is physical start address. */
	+ #define CMDQ_REGISTER_MR_LVL_LVL_0 UINT32_C(0x0)
	+ /* PBL pointer points to PTE table. */
	+ #define CMDQ_REGISTER_MR_LVL_LVL_1 UINT32_C(0x1)
	+ /*
	+ * PBL pointer points to PDE table with each entry pointing to
	+ * PTE tables.
	+ */
	+ #define CMDQ_REGISTER_MR_LVL_LVL_2 UINT32_C(0x2)
	+ /*
	+ * Log base 2 of page size; 12 is the minimum for 4KB. HW supported
	+ * values are enumerated below.
	+ */
	+ #define CMDQ_REGISTER_MR_LOG2_PG_SIZE_MASK UINT32_C(0x7c)
	+ #define CMDQ_REGISTER_MR_LOG2_PG_SIZE_SFT 2
	+ /* 4KB. */
	+ #define CMDQ_REGISTER_MR_LOG2_PG_SIZE_PG_4K (UINT32_C(0xc) << 2)
	+ /* 8KB. */
	+ #define CMDQ_REGISTER_MR_LOG2_PG_SIZE_PG_8K (UINT32_C(0xd) << 2)
	+ /* 64KB. */
	+ #define CMDQ_REGISTER_MR_LOG2_PG_SIZE_PG_64K (UINT32_C(0x10) << 2)
	+ /* 256KB. */
	+ #define CMDQ_REGISTER_MR_LOG2_PG_SIZE_PG_256K (UINT32_C(0x12) << 2)
	+ /* 1MB. */
	+ #define CMDQ_REGISTER_MR_LOG2_PG_SIZE_PG_1M (UINT32_C(0x14) << 2)
	+ /* 2MB. */
	+ #define CMDQ_REGISTER_MR_LOG2_PG_SIZE_PG_2M (UINT32_C(0x15) << 2)
	+ /* 4MB. */
	+ #define CMDQ_REGISTER_MR_LOG2_PG_SIZE_PG_4M (UINT32_C(0x16) << 2)
	+ /* 1GB. */
	+ #define CMDQ_REGISTER_MR_LOG2_PG_SIZE_PG_1G (UINT32_C(0x1e) << 2)
	+ /* unused1 is 1 b */
	+ uint8_t access;
	+ /* Access flags. */
	+ /* Local write access. */
	+ #define CMDQ_REGISTER_MR_ACCESS_LOCAL_WRITE UINT32_C(0x1)
	+ /* Remote read access. */
	+ #define CMDQ_REGISTER_MR_ACCESS_REMOTE_READ UINT32_C(0x2)
	+ /* Remote write access. */
	+ #define CMDQ_REGISTER_MR_ACCESS_REMOTE_WRITE UINT32_C(0x4)
	+ /* Remote atomic access. */
	+ #define CMDQ_REGISTER_MR_ACCESS_REMOTE_ATOMIC UINT32_C(0x8)
	+ /* Bind access allowed. */
	+ #define CMDQ_REGISTER_MR_ACCESS_MW_BIND UINT32_C(0x10)
	+ /* Indicate Zero Based Virtual Address (ZBVA). */
	+ #define CMDQ_REGISTER_MR_ACCESS_ZERO_BASED UINT32_C(0x20)
	+ uint16_t log2_pbl_pg_size;
	+ /* unused11 is 11 b */
	+ /*
	+ * Log base 2 of PBL page size; 12 is the minimum for 4KB. HW supported
	+ * values are enumerated below
	+ */
	+ #define CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_MASK UINT32_C(0x1f)
	+ #define CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_SFT 0
	+ /* 4KB. */
	+ #define CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_4K UINT32_C(0xc)
	+ /* 8KB. */
	+ #define CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_8K UINT32_C(0xd)
	+ /* 64KB. */
	+ #define CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_64K UINT32_C(0x10)
	+ /* 256KB. */
	+ #define CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_256K UINT32_C(0x12)
	+ /* 1MB. */
	+ #define CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_1M UINT32_C(0x14)
	+ /* 2MB. */
	+ #define CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_2M UINT32_C(0x15)
	+ /* 4MB. */
	+ #define CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_4M UINT32_C(0x16)
	+ /* 1GB. */
	+ #define CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_1G UINT32_C(0x1e)
	+ /* unused11 is 11 b */
	+ uint32_t key;
	+ /* KEY of the MR. */
	+ uint64_t pbl;
	+ /* Page table of the MR memory. */
	+ uint64_t va;
	+ /* Virtual address of the MR. */
	+ uint64_t mr_size;
	+ /* Size of the MR. */
	+} __attribute__((packed));
	+
	+/* Deregister MR command (24 bytes) */
	+
	+struct cmdq_deregister_mr {
	+ uint8_t opcode;
	+ /* Command opcode. */
	+ /* Deregister MR command de-registers memory from the specified MR. */
	+ #define CMDQ_DEREGISTER_MR_OPCODE_DEREGISTER_MR UINT32_C(0x10)
	+ uint8_t cmd_size;
	+ /* Size of the command in 16-byte units. */
	+ uint16_t flags;
	+ /* Flags and attribs of the command. */
	+ uint16_t cookie;
	+ /* Driver supplied handle to associate the command and the response. */
	+ uint8_t resp_size;
	+ /* Size of the response buffer in 16-byte units. */
	+ uint8_t reserved8;
	+ uint64_t resp_addr;
	+ /* Host address of the response. */
	+ uint32_t lkey;
	+ /* L_KEY of the MR. */
	+ uint32_t unused_0;
	+} __attribute__((packed));
	+
	+/* Add GID command (48 bytes) */
	+
	+struct cmdq_add_gid {
	+ uint8_t opcode;
	+ /* Command opcode. */
	+ /* Add GID command adds a GID to the local address table. */
	+ #define CMDQ_ADD_GID_OPCODE_ADD_GID UINT32_C(0x11)
	+ uint8_t cmd_size;
	+ /* Size of the command in 16-byte units. */
	+ uint16_t flags;
	+ /* Flags and attribs of the command. */
	+ uint16_t cookie;
	+ /* Driver supplied handle to associate the command and the response. */
	+ uint8_t resp_size;
	+ /* Size of the response buffer in 16-byte units. */
	+ uint8_t reserved8;
	+ uint64_t resp_addr;
	+ /* Host address of the response. */
	+ uint32_t gid[4];
	+ /* GID */
	+ uint16_t src_mac[3];
	+ /* Source MAC. */
	+ uint16_t vlan;
	+ /* flags. */
	+ /* Source VLAN id. */
	+ #define CMDQ_ADD_GID_VLAN_VLAN_ID_MASK UINT32_C(0xfff)
	+ #define CMDQ_ADD_GID_VLAN_VLAN_ID_SFT 0
	+ /* This set of bits select the TPID of the VLAN Tag. */
	+ #define CMDQ_ADD_GID_VLAN_TPID_MASK UINT32_C(0x7000)
	+ #define CMDQ_ADD_GID_VLAN_TPID_SFT 12
	+ /* TPID = 0x88A8. */
	+ #define CMDQ_ADD_GID_VLAN_TPID_TPID_88A8 (UINT32_C(0x0) << 12)
	+ /* TPID = 0x8100. */
	+ #define CMDQ_ADD_GID_VLAN_TPID_TPID_8100 (UINT32_C(0x1) << 12)
	+ /* TPID = 0x9100. */
	+ #define CMDQ_ADD_GID_VLAN_TPID_TPID_9100 (UINT32_C(0x2) << 12)
	+ /* TPID = 0x9200. */
	+ #define CMDQ_ADD_GID_VLAN_TPID_TPID_9200 (UINT32_C(0x3) << 12)
	+ /* TPID = 0x9300. */
	+ #define CMDQ_ADD_GID_VLAN_TPID_TPID_9300 (UINT32_C(0x4) << 12)
	+ /* TPID = Configurable 1. */
	+ #define CMDQ_ADD_GID_VLAN_TPID_TPID_CFG1 (UINT32_C(0x5) << 12)
	+ /* TPID = Configurable 2. */
	+ #define CMDQ_ADD_GID_VLAN_TPID_TPID_CFG2 (UINT32_C(0x6) << 12)
	+ /* TPID = Configurable 3. */
	+ #define CMDQ_ADD_GID_VLAN_TPID_TPID_CFG3 (UINT32_C(0x7) << 12)
	+ #define CMDQ_ADD_GID_VLAN_TPID_LAST CMDQ_ADD_GID_VLAN_TPID_TPID_CFG3
	+ /*
	+ * Setting this bit to 1 enables insertion of a VLAN Tag to a RoCE
	+ * header.
	+ */
	+ #define CMDQ_ADD_GID_VLAN_VLAN_EN UINT32_C(0x8000)
	+ uint16_t ipid;
	+ /* Identifier field in the IP header. */
	+ uint16_t stats_ctx;
	+ /* Stats context ID to use with this SGID */
	+ /* stats_ctx_id is 15 b */
	+ #define CMDQ_ADD_GID_STATS_CTX_STATS_CTX_ID_MASK UINT32_C(0x7fff)
	+ #define CMDQ_ADD_GID_STATS_CTX_STATS_CTX_ID_SFT 0
	+ /*
	+ * Setting this bit to 1 enables use of own stats context ID instead of
	+ * per-function
	+ */
	+ #define CMDQ_ADD_GID_STATS_CTX_STATS_CTX_VALID UINT32_C(0x8000)
	+ uint32_t unused_0;
	+} __attribute__((packed));
	+
	+/* Delete GID command (24 bytes) */
	+
	+struct cmdq_delete_gid {
	+ uint8_t opcode;
	+ /* Command opcode. */
	+ /* Delete GID command deletes a GID from the local address table. */
	+ #define CMDQ_DELETE_GID_OPCODE_DELETE_GID UINT32_C(0x12)
	+ uint8_t cmd_size;
	+ /* Size of the command in 16-byte units. */
	+ uint16_t flags;
	+ /* Flags and attribs of the command. */
	+ uint16_t cookie;
	+ /* Driver supplied handle to associate the command and the response. */
	+ uint8_t resp_size;
	+ /* Size of the response buffer in 16-byte units. */
	+ uint8_t reserved8;
	+ uint64_t resp_addr;
	+ /* Host address of the response. */
	+ uint16_t gid_index;
	+ /* GID index */
	+ uint16_t unused_0;
	+ /* unused16 is 16 b */
	+ uint32_t unused_1;
	+} __attribute__((packed));
	+
	+/* Modify GID command (48 bytes) */
	+
	+struct cmdq_modify_gid {
	+ uint8_t opcode;
	+ /* Command opcode. */
	+ /* Modify GID command modifies a GID in the local address table. */
	+ #define CMDQ_MODIFY_GID_OPCODE_MODIFY_GID UINT32_C(0x17)
	+ uint8_t cmd_size;
	+ /* Size of the command in 16-byte units. */
	+ uint16_t flags;
	+ /* Flags and attribs of the command. */
	+ uint16_t cookie;
	+ /* Driver supplied handle to associate the command and the response. */
	+ uint8_t resp_size;
	+ /* Size of the response buffer in 16-byte units. */
	+ uint8_t reserved8;
	+ uint64_t resp_addr;
	+ /* Host address of the response. */
	+ uint32_t gid[4];
	+ /* GID */
	+ uint16_t src_mac[3];
	+ /* Source MAC. */
	+ uint16_t vlan;
	+ /* flags. */
	+ /* Source VLAN id. */
	+ #define CMDQ_MODIFY_GID_VLAN_VLAN_ID_MASK UINT32_C(0xfff)
	+ #define CMDQ_MODIFY_GID_VLAN_VLAN_ID_SFT 0
	+ /* This set of bits select the TPID of the VLAN Tag. */
	+ #define CMDQ_MODIFY_GID_VLAN_TPID_MASK UINT32_C(0x7000)
	+ #define CMDQ_MODIFY_GID_VLAN_TPID_SFT 12
	+ /* TPID = 0x88A8. */
	+ #define CMDQ_MODIFY_GID_VLAN_TPID_TPID_88A8 (UINT32_C(0x0) << 12)
	+ /* TPID = 0x8100. */
	+ #define CMDQ_MODIFY_GID_VLAN_TPID_TPID_8100 (UINT32_C(0x1) << 12)
	+ /* TPID = 0x9100. */
	+ #define CMDQ_MODIFY_GID_VLAN_TPID_TPID_9100 (UINT32_C(0x2) << 12)
	+ /* TPID = 0x9200. */
	+ #define CMDQ_MODIFY_GID_VLAN_TPID_TPID_9200 (UINT32_C(0x3) << 12)
	+ /* TPID = 0x9300. */
	+ #define CMDQ_MODIFY_GID_VLAN_TPID_TPID_9300 (UINT32_C(0x4) << 12)
	+ /* TPID = Configurable 1. */
	+ #define CMDQ_MODIFY_GID_VLAN_TPID_TPID_CFG1 (UINT32_C(0x5) << 12)
	+ /* TPID = Configurable 2. */
	+ #define CMDQ_MODIFY_GID_VLAN_TPID_TPID_CFG2 (UINT32_C(0x6) << 12)
	+ /* TPID = Configurable 3. */
	+ #define CMDQ_MODIFY_GID_VLAN_TPID_TPID_CFG3 (UINT32_C(0x7) << 12)
	+ #define CMDQ_MODIFY_GID_VLAN_TPID_LAST CMDQ_MODIFY_GID_VLAN_TPID_TPID_CFG3
	+ /*
	+ * Setting this bit to 1 enables insertion of a VLAN Tag to a RoCE
	+ * header.
	+ */
	+ #define CMDQ_MODIFY_GID_VLAN_VLAN_EN UINT32_C(0x8000)
	+ uint16_t ipid;
	+ /* Identifier field in the IP header. */
	+ uint16_t gid_index;
	+ /* GID index */
	+ uint16_t stats_ctx;
	+ /* Stats context ID to use with this SGID */
	+ /* stats_ctx_id is 15 b */
	+ #define CMDQ_MODIFY_GID_STATS_CTX_STATS_CTX_ID_MASK UINT32_C(0x7fff)
	+ #define CMDQ_MODIFY_GID_STATS_CTX_STATS_CTX_ID_SFT 0
	+ /*
	+ * Setting this bit to 1 enables use of own stats context ID instead of
	+ * per-function
	+ */
	+ #define CMDQ_MODIFY_GID_STATS_CTX_STATS_CTX_VALID UINT32_C(0x8000)
	+ uint16_t unused_0;
	+} __attribute__((packed));
	+
	+/* Query GID command (24 bytes) */
	+
	+struct cmdq_query_gid {
	+ uint8_t opcode;
	+ /* Command opcode. */
	+ /* Query GID command queries a GID in the local address table. */
	+ #define CMDQ_QUERY_GID_OPCODE_QUERY_GID UINT32_C(0x18)
	+ uint8_t cmd_size;
	+ /* Size of the command in 16-byte units. */
	+ uint16_t flags;
	+ /* Flags and attribs of the command. */
	+ uint16_t cookie;
	+ /* Driver supplied handle to associate the command and the response. */
	+ uint8_t resp_size;
	+ /* Size of the response buffer in 16-byte units. */
	+ uint8_t reserved8;
	+ uint64_t resp_addr;
	+ /* Host address of the response. */
	+ uint16_t gid_index;
	+ /* GID index */
	+ uint16_t unused_0;
	+ /* unused16 is 16 b */
	+ uint32_t unused_1;
	+} __attribute__((packed));
	+
	+/* Create QP1 command (80 bytes) */
	+
	+struct cmdq_create_qp1 {
	+ uint8_t opcode;
	+ /* Command opcode. */
	+ /* Create QP1 command allocates a QP1 only. */
	+ #define CMDQ_CREATE_QP1_OPCODE_CREATE_QP1 UINT32_C(0x13)
	+ uint8_t cmd_size;
	+ /* Size of the command in 16-byte units. */
	+ uint16_t flags;
	+ /* Flags and attribs of the command. */
	+ uint16_t cookie;
	+ /* Driver supplied handle to associate the command and the response. */
	+ uint8_t resp_size;
	+ /* Size of the response buffer in 16-byte units. */
	+ uint8_t reserved8;
	+ uint64_t resp_addr;
	+ /* Host address of the response. */
	+ uint64_t qp_handle;
	+ /* QP1 handle. */
	+ uint32_t qp_flags;
	+ /* Create QP1 flags. */
	+ /* SRQ is used. */
	+ #define CMDQ_CREATE_QP1_QP_FLAGS_SRQ_USED UINT32_C(0x1)
	+ /* post CQE for all SQ WQEs. */
	+ #define CMDQ_CREATE_QP1_QP_FLAGS_FORCE_COMPLETION UINT32_C(0x2)
	+ /* This QP can use reserved L_Key */
	+ #define CMDQ_CREATE_QP1_QP_FLAGS_RESERVED_LKEY_ENABLE UINT32_C(0x4)
	+ uint8_t type;
	+ /* Supported QP1 types. */
	+ /* General Services Interface on QP 1. */
	+ #define CMDQ_CREATE_QP1_TYPE_GSI UINT32_C(0x1)
	+ uint8_t sq_pg_size_sq_lvl;
	+ /* SQ page size. */
	+ /* SQ PBL indirect levels. */
	+ #define CMDQ_CREATE_QP1_SQ_LVL_MASK UINT32_C(0xf)
	+ #define CMDQ_CREATE_QP1_SQ_LVL_SFT 0
	+ /* PBL pointer is physical start address. */
	+ #define CMDQ_CREATE_QP1_SQ_LVL_LVL_0 UINT32_C(0x0)
	+ /* PBL pointer points to PTE table. */
	+ #define CMDQ_CREATE_QP1_SQ_LVL_LVL_1 UINT32_C(0x1)
	+ /*
	+ * PBL pointer points to PDE table with each entry pointing to
	+ * PTE tables.
	+ */
	+ #define CMDQ_CREATE_QP1_SQ_LVL_LVL_2 UINT32_C(0x2)
	+ /* SQ page size. */
	+ #define CMDQ_CREATE_QP1_SQ_PG_SIZE_MASK UINT32_C(0xf0)
	+ #define CMDQ_CREATE_QP1_SQ_PG_SIZE_SFT 4
	+ /* 4KB. */
	+ #define CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_4K (UINT32_C(0x0) << 4)
	+ /* 8KB. */
	+ #define CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_8K (UINT32_C(0x1) << 4)
	+ /* 64KB. */
	+ #define CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_64K (UINT32_C(0x2) << 4)
	+ /* 2MB. */
	+ #define CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_2M (UINT32_C(0x3) << 4)
	+ /* 8MB. */
	+ #define CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_8M (UINT32_C(0x4) << 4)
	+ /* 1GB. */
	+ #define CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_1G (UINT32_C(0x5) << 4)
	+ uint8_t rq_pg_size_rq_lvl;
	+ /* RQ page size. */
	+ /* RQ PBL indirect levels. */
	+ #define CMDQ_CREATE_QP1_RQ_LVL_MASK UINT32_C(0xf)
	+ #define CMDQ_CREATE_QP1_RQ_LVL_SFT 0
	+ /* PBL pointer is physical start address. */
	+ #define CMDQ_CREATE_QP1_RQ_LVL_LVL_0 UINT32_C(0x0)
	+ /* PBL pointer points to PTE table. */
	+ #define CMDQ_CREATE_QP1_RQ_LVL_LVL_1 UINT32_C(0x1)
	+ /*
	+ * PBL pointer points to PDE table with each entry pointing to
	+ * PTE tables.
	+ */
	+ #define CMDQ_CREATE_QP1_RQ_LVL_LVL_2 UINT32_C(0x2)
	+ /* RQ page size. */
	+ #define CMDQ_CREATE_QP1_RQ_PG_SIZE_MASK UINT32_C(0xf0)
	+ #define CMDQ_CREATE_QP1_RQ_PG_SIZE_SFT 4
	+ /* 4KB. */
	+ #define CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_4K (UINT32_C(0x0) << 4)
	+ /* 8KB. */
	+ #define CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_8K (UINT32_C(0x1) << 4)
	+ /* 64KB. */
	+ #define CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_64K (UINT32_C(0x2) << 4)
	+ /* 2MB. */
	+ #define CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_2M (UINT32_C(0x3) << 4)
	+ /* 8MB. */
	+ #define CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_8M (UINT32_C(0x4) << 4)
	+ /* 1GB. */
	+ #define CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_1G (UINT32_C(0x5) << 4)
	+ uint8_t unused_0;
	+ uint32_t dpi;
	+ /* Doorbell page index. */
	+ uint32_t sq_size;
	+ /* Max number of SQ wqes. */
	+ uint32_t rq_size;
	+ /* Max number of RQ wqes. */
	+ uint16_t sq_fwo_sq_sge;
	+ /* Offset of First WQE in the first SQ page, in 128 byte units */
	+ /* Max send SGEs per SWQE. */
	+ #define CMDQ_CREATE_QP1_SQ_SGE_MASK UINT32_C(0xf)
	+ #define CMDQ_CREATE_QP1_SQ_SGE_SFT 0
	+ /* Offset of First WQE in the first SQ page, in 128 byte units */
	+ #define CMDQ_CREATE_QP1_SQ_FWO_MASK UINT32_C(0xfff0)
	+ #define CMDQ_CREATE_QP1_SQ_FWO_SFT 4
	+ uint16_t rq_fwo_rq_sge;
	+ /* Offset of First WQE in the first RQ page, in 128 byte units */
	+ /* Max recv SGEs per RWQE (NOT SUPPORTED BY HARDWARE). */
	+ #define CMDQ_CREATE_QP1_RQ_SGE_MASK UINT32_C(0xf)
	+ #define CMDQ_CREATE_QP1_RQ_SGE_SFT 0
	+ /* Offset of First WQE in the first RQ page, in 128 byte units */
	+ #define CMDQ_CREATE_QP1_RQ_FWO_MASK UINT32_C(0xfff0)
	+ #define CMDQ_CREATE_QP1_RQ_FWO_SFT 4
	+ uint32_t scq_cid;
	+ /* Send CQ context id. */
	+ uint32_t rcq_cid;
	+ /* Receive CQ context id. */
	+ uint32_t srq_cid;
	+ /* SRQ CQ context id. */
	+ uint32_t pd_id;
	+ /* Protection domain id. */
	+ uint64_t sq_pbl;
	+ /* SQ PBL physical address. */
	+ uint64_t rq_pbl;
	+ /* RQ PBL physical address. */
	+} __attribute__((packed));
	+
	+/* Destroy QP1 command (24 bytes) */
	+
	+struct cmdq_destroy_qp1 {
	+ uint8_t opcode;
	+ /* Command opcode. */
	+ /* Destroy QP1 command deletes and flushes the specified QP1. */
	+ #define CMDQ_DESTROY_QP1_OPCODE_DESTROY_QP1 UINT32_C(0x14)
	+ uint8_t cmd_size;
	+ /* Size of the command in 16-byte units. */
	+ uint16_t flags;
	+ /* Flags and attribs of the command. */
	+ uint16_t cookie;
	+ /* Driver supplied handle to associate the command and the response. */
	+ uint8_t resp_size;
	+ /* Size of the response buffer in 16-byte units. */
	+ uint8_t reserved8;
	+ uint64_t resp_addr;
	+ /* Host address of the response. */
	+ uint32_t qp1_cid;
	+ /* QP1 context id */
	+ uint32_t unused_0;
	+} __attribute__((packed));
	+
	+/* Create AH command (64 bytes) */
	+
	+struct cmdq_create_ah {
	+ uint8_t opcode;
	+ /* Command opcode. */
	+ /* Create AH command allocates an AH with the specified parameters. */
	+ #define CMDQ_CREATE_AH_OPCODE_CREATE_AH UINT32_C(0x15)
	+ uint8_t cmd_size;
	+ /* Size of the command in 16-byte units. */
	+ uint16_t flags;
	+ /* Flags and attribs of the command. */
	+ uint16_t cookie;
	+ /* Driver supplied handle to associate the command and the response. */
	+ uint8_t resp_size;
	+ /* Size of the response buffer in 16-byte units. */
	+ uint8_t reserved8;
	+ uint64_t resp_addr;
	+ /* Host address of the response. */
	+ uint64_t ah_handle;
	+ /* AH handle. */
	+ uint32_t dgid[4];
	+ /* Destination GID. */
	+ uint8_t type;
	+ /* V1, V2IPv4 or V2IPv6. */
	+ /* V2IPv4. */
	+ #define CMDQ_CREATE_AH_TYPE_V1 UINT32_C(0x0)
	+ /* V2IPv4. */
	+ #define CMDQ_CREATE_AH_TYPE_V2IPV4 UINT32_C(0x2)
	+ /* V2IPv6. */
	+ #define CMDQ_CREATE_AH_TYPE_V2IPV6 UINT32_C(0x3)
	+ uint8_t hop_limit;
	+ /* IPv6 Hop limit. */
	+ uint16_t sgid_index;
	+ /* SGID index. */
	+ uint32_t dest_vlan_id_flow_label;
	+ /* Destination VLAN ID. */
	+ /* Flow label. */
	+ #define CMDQ_CREATE_AH_FLOW_LABEL_MASK UINT32_C(0xfffff)
	+ #define CMDQ_CREATE_AH_FLOW_LABEL_SFT 0
	+ /* Destination VLAN ID. */
	+ #define CMDQ_CREATE_AH_DEST_VLAN_ID_MASK UINT32_C(0xfff00000)
	+ #define CMDQ_CREATE_AH_DEST_VLAN_ID_SFT 20
	+ uint32_t pd_id;
	+ /* Protection domain id. */
	+ uint32_t unused_0;
	+ uint16_t dest_mac[3];
	+ /* Destination MAC address. */
	+ uint8_t traffic_class;
	+ /* Traffic class. */
	+ uint8_t enable_cc;
	+ /* Enable congestion control. */
	+ #define CMDQ_CREATE_AH_ENABLE_CC UINT32_C(0x1)
	+} __attribute__((packed));
	+
	+/* Destroy AH command (24 bytes) */
	+
	+struct cmdq_destroy_ah {
	+ uint8_t opcode;
	+ /* Command opcode. */
	+ /* Destroy AH command deletes the specified AH. */
	+ #define CMDQ_DESTROY_AH_OPCODE_DESTROY_AH UINT32_C(0x16)
	+ uint8_t cmd_size;
	+ /* Size of the command in 16-byte units. */
	+ uint16_t flags;
	+ /* Flags and attribs of the command. */
	+ uint16_t cookie;
	+ /* Driver supplied handle to associate the command and the response. */
	+ uint8_t resp_size;
	+ /* Size of the response buffer in 16-byte units. */
	+ uint8_t reserved8;
	+ uint64_t resp_addr;
	+ /* Host address of the response. */
	+ uint32_t ah_cid;
	+ /* AH context id */
	+ uint32_t unused_0;
	+} __attribute__((packed));
	+
	+/* Initialize Firmware command (112 bytes) */
	+
	+struct cmdq_initialize_fw {
	+ uint8_t opcode;
	+ /* Command opcode. */
	+ /*
	+ * Initialize firmware command initializes the firmware with the
	+ * specified parameters.
	+ */
	+ #define CMDQ_INITIALIZE_FW_OPCODE_INITIALIZE_FW UINT32_C(0x80)
	+ uint8_t cmd_size;
	+ /* Size of the command in 16-byte units. */
	+ uint16_t flags;
	+ /* Flags and attribs of the command. */
	+ uint16_t cookie;
	+ /* Driver supplied handle to associate the command and the response. */
	+ uint8_t resp_size;
	+ /* Size of the response buffer in 16-byte units. */
	+ uint8_t reserved8;
	+ uint64_t resp_addr;
	+ /* Host address of the response. */
	+ uint8_t qpc_pg_size_qpc_lvl;
	+ /* QPC page size. */
	+ /* QPC PBL indirect levels. */
	+ #define CMDQ_INITIALIZE_FW_QPC_LVL_MASK UINT32_C(0xf)
	+ #define CMDQ_INITIALIZE_FW_QPC_LVL_SFT 0
	+ /* PBL pointer is physical start address. */
	+ #define CMDQ_INITIALIZE_FW_QPC_LVL_LVL_0 UINT32_C(0x0)
	+ /* PBL pointer points to PTE table. */
	+ #define CMDQ_INITIALIZE_FW_QPC_LVL_LVL_1 UINT32_C(0x1)
	+ /*
	+ * PBL pointer points to PDE table with each entry pointing to
	+ * PTE tables.
	+ */
	+ #define CMDQ_INITIALIZE_FW_QPC_LVL_LVL_2 UINT32_C(0x2)
	+ /* QPC page size. */
	+ #define CMDQ_INITIALIZE_FW_QPC_PG_SIZE_MASK UINT32_C(0xf0)
	+ #define CMDQ_INITIALIZE_FW_QPC_PG_SIZE_SFT 4
	+ /* 4KB. */
	+ #define CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_4K (UINT32_C(0x0) << 4)
	+ /* 8KB. */
	+ #define CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_8K (UINT32_C(0x1) << 4)
	+ /* 64KB. */
	+ #define CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_64K (UINT32_C(0x2) << 4)
	+ /* 2MB. */
	+ #define CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_2M (UINT32_C(0x3) << 4)
	+ /* 8MB. */
	+ #define CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_8M (UINT32_C(0x4) << 4)
	+ /* 1GB. */
	+ #define CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_1G (UINT32_C(0x5) << 4)
	+ uint8_t mrw_pg_size_mrw_lvl;
	+ /* MRW page size. */
	+ /* MRW PBL indirect levels. */
	+ #define CMDQ_INITIALIZE_FW_MRW_LVL_MASK UINT32_C(0xf)
	+ #define CMDQ_INITIALIZE_FW_MRW_LVL_SFT 0
	+ /* PBL pointer is physical start address. */
	+ #define CMDQ_INITIALIZE_FW_MRW_LVL_LVL_0 UINT32_C(0x0)
	+ /* PBL pointer points to PTE table. */
	+ #define CMDQ_INITIALIZE_FW_MRW_LVL_LVL_1 UINT32_C(0x1)
	+ /*
	+ * PBL pointer points to PDE table with each entry pointing to
	+ * PTE tables.
	+ */
	+ #define CMDQ_INITIALIZE_FW_MRW_LVL_LVL_2 UINT32_C(0x2)
	+ /* MRW page size. */
	+ #define CMDQ_INITIALIZE_FW_MRW_PG_SIZE_MASK UINT32_C(0xf0)
	+ #define CMDQ_INITIALIZE_FW_MRW_PG_SIZE_SFT 4
	+ /* 4KB. */
	+ #define CMDQ_INITIALIZE_FW_MRW_PG_SIZE_PG_4K (UINT32_C(0x0) << 4)
	+ /* 8KB. */
	+ #define CMDQ_INITIALIZE_FW_MRW_PG_SIZE_PG_8K (UINT32_C(0x1) << 4)
	+ /* 64KB. */
	+ #define CMDQ_INITIALIZE_FW_MRW_PG_SIZE_PG_64K (UINT32_C(0x2) << 4)
	+ /* 2MB. */
	+ #define CMDQ_INITIALIZE_FW_MRW_PG_SIZE_PG_2M (UINT32_C(0x3) << 4)
	+ /* 8MB. */
	+ #define CMDQ_INITIALIZE_FW_MRW_PG_SIZE_PG_8M (UINT32_C(0x4) << 4)
	+ /* 1GB. */
	+ #define CMDQ_INITIALIZE_FW_MRW_PG_SIZE_PG_1G (UINT32_C(0x5) << 4)
	+ uint8_t srq_pg_size_srq_lvl;
	+ /* SRQ page size. */
	+ /* SRQ PBL indirect levels. */
	+ #define CMDQ_INITIALIZE_FW_SRQ_LVL_MASK UINT32_C(0xf)
	+ #define CMDQ_INITIALIZE_FW_SRQ_LVL_SFT 0
	+ /* PBL pointer is physical start address. */
	+ #define CMDQ_INITIALIZE_FW_SRQ_LVL_LVL_0 UINT32_C(0x0)
	+ /* PBL pointer points to PTE table. */
	+ #define CMDQ_INITIALIZE_FW_SRQ_LVL_LVL_1 UINT32_C(0x1)
	+ /*
	+ * PBL pointer points to PDE table with each entry pointing to
	+ * PTE tables.
	+ */
	+ #define CMDQ_INITIALIZE_FW_SRQ_LVL_LVL_2 UINT32_C(0x2)
	+ /* SRQ page size. */
	+ #define CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_MASK UINT32_C(0xf0)
	+ #define CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_SFT 4
	+ /* 4KB. */
	+ #define CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_PG_4K (UINT32_C(0x0) << 4)
	+ /* 8KB. */
	+ #define CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_PG_8K (UINT32_C(0x1) << 4)
	+ /* 64KB. */
	+ #define CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_PG_64K (UINT32_C(0x2) << 4)
	+ /* 2MB. */
	+ #define CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_PG_2M (UINT32_C(0x3) << 4)
	+ /* 8MB. */
	+ #define CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_PG_8M (UINT32_C(0x4) << 4)
	+ /* 1GB. */
	+ #define CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_PG_1G (UINT32_C(0x5) << 4)
	+ uint8_t cq_pg_size_cq_lvl;
	+ /* CQ page size. */
	+ /* CQ PBL indirect levels. */
	+ #define CMDQ_INITIALIZE_FW_CQ_LVL_MASK UINT32_C(0xf)
	+ #define CMDQ_INITIALIZE_FW_CQ_LVL_SFT 0
	+ /* PBL pointer is physical start address. */
	+ #define CMDQ_INITIALIZE_FW_CQ_LVL_LVL_0 UINT32_C(0x0)
	+ /* PBL pointer points to PTE table. */
	+ #define CMDQ_INITIALIZE_FW_CQ_LVL_LVL_1 UINT32_C(0x1)
	+ /*
	+ * PBL pointer points to PDE table with each entry pointing to
	+ * PTE tables.
	+ */
	+ #define CMDQ_INITIALIZE_FW_CQ_LVL_LVL_2 UINT32_C(0x2)
	+ /* CQ page size. */
	+ #define CMDQ_INITIALIZE_FW_CQ_PG_SIZE_MASK UINT32_C(0xf0)
	+ #define CMDQ_INITIALIZE_FW_CQ_PG_SIZE_SFT 4
	+ /* 4KB. */
	+ #define CMDQ_INITIALIZE_FW_CQ_PG_SIZE_PG_4K (UINT32_C(0x0) << 4)
	+ /* 8KB. */
	+ #define CMDQ_INITIALIZE_FW_CQ_PG_SIZE_PG_8K (UINT32_C(0x1) << 4)
	+ /* 64KB. */
	+ #define CMDQ_INITIALIZE_FW_CQ_PG_SIZE_PG_64K (UINT32_C(0x2) << 4)
	+ /* 2MB. */
	+ #define CMDQ_INITIALIZE_FW_CQ_PG_SIZE_PG_2M (UINT32_C(0x3) << 4)
	+ /* 8MB. */
	+ #define CMDQ_INITIALIZE_FW_CQ_PG_SIZE_PG_8M (UINT32_C(0x4) << 4)
	+ /* 1GB. */
	+ #define CMDQ_INITIALIZE_FW_CQ_PG_SIZE_PG_1G (UINT32_C(0x5) << 4)
	+ uint8_t tqm_pg_size_tqm_lvl;
	+ /* TQM page size. */
	+ /* TQM PBL indirect levels. */
	+ #define CMDQ_INITIALIZE_FW_TQM_LVL_MASK UINT32_C(0xf)
	+ #define CMDQ_INITIALIZE_FW_TQM_LVL_SFT 0
	+ /* PBL pointer is physical start address. */
	+ #define CMDQ_INITIALIZE_FW_TQM_LVL_LVL_0 UINT32_C(0x0)
	+ /* PBL pointer points to PTE table. */
	+ #define CMDQ_INITIALIZE_FW_TQM_LVL_LVL_1 UINT32_C(0x1)
	+ /*
	+ * PBL pointer points to PDE table with each entry pointing to
	+ * PTE tables.
	+ */
	+ #define CMDQ_INITIALIZE_FW_TQM_LVL_LVL_2 UINT32_C(0x2)
	+ /* TQM page size. */
	+ #define CMDQ_INITIALIZE_FW_TQM_PG_SIZE_MASK UINT32_C(0xf0)
	+ #define CMDQ_INITIALIZE_FW_TQM_PG_SIZE_SFT 4
	+ /* 4KB. */
	+ #define CMDQ_INITIALIZE_FW_TQM_PG_SIZE_PG_4K (UINT32_C(0x0) << 4)
	+ /* 8KB. */
	+ #define CMDQ_INITIALIZE_FW_TQM_PG_SIZE_PG_8K (UINT32_C(0x1) << 4)
	+ /* 64KB. */
	+ #define CMDQ_INITIALIZE_FW_TQM_PG_SIZE_PG_64K (UINT32_C(0x2) << 4)
	+ /* 2MB. */
	+ #define CMDQ_INITIALIZE_FW_TQM_PG_SIZE_PG_2M (UINT32_C(0x3) << 4)
	+ /* 8MB. */
	+ #define CMDQ_INITIALIZE_FW_TQM_PG_SIZE_PG_8M (UINT32_C(0x4) << 4)
	+ /* 1GB. */
	+ #define CMDQ_INITIALIZE_FW_TQM_PG_SIZE_PG_1G (UINT32_C(0x5) << 4)
	+ uint8_t tim_pg_size_tim_lvl;
	+ /* TIM page size. */
	+ /* TIM PBL indirect levels. */
	+ #define CMDQ_INITIALIZE_FW_TIM_LVL_MASK UINT32_C(0xf)
	+ #define CMDQ_INITIALIZE_FW_TIM_LVL_SFT 0
	+ /* PBL pointer is physical start address. */
	+ #define CMDQ_INITIALIZE_FW_TIM_LVL_LVL_0 UINT32_C(0x0)
	+ /* PBL pointer points to PTE table. */
	+ #define CMDQ_INITIALIZE_FW_TIM_LVL_LVL_1 UINT32_C(0x1)
	+ /*
	+ * PBL pointer points to PDE table with each entry pointing to
	+ * PTE tables.
	+ */
	+ #define CMDQ_INITIALIZE_FW_TIM_LVL_LVL_2 UINT32_C(0x2)
	+ /* TIM page size. */
	+ #define CMDQ_INITIALIZE_FW_TIM_PG_SIZE_MASK UINT32_C(0xf0)
	+ #define CMDQ_INITIALIZE_FW_TIM_PG_SIZE_SFT 4
	+ /* 4KB. */
	+ #define CMDQ_INITIALIZE_FW_TIM_PG_SIZE_PG_4K (UINT32_C(0x0) << 4)
	+ /* 8KB. */
	+ #define CMDQ_INITIALIZE_FW_TIM_PG_SIZE_PG_8K (UINT32_C(0x1) << 4)
	+ /* 64KB. */
	+ #define CMDQ_INITIALIZE_FW_TIM_PG_SIZE_PG_64K (UINT32_C(0x2) << 4)
	+ /* 2MB. */
	+ #define CMDQ_INITIALIZE_FW_TIM_PG_SIZE_PG_2M (UINT32_C(0x3) << 4)
	+ /* 8MB. */
	+ #define CMDQ_INITIALIZE_FW_TIM_PG_SIZE_PG_8M (UINT32_C(0x4) << 4)
	+ /* 1GB. */
	+ #define CMDQ_INITIALIZE_FW_TIM_PG_SIZE_PG_1G (UINT32_C(0x5) << 4)
	+ uint16_t reserved16;
	+ uint64_t qpc_page_dir;
	+ /* Kernel notification queue page directory. */
	+ uint64_t mrw_page_dir;
	+ /* MRW page directory. */
	+ uint64_t srq_page_dir;
	+ /* SRQ page directory. */
	+ uint64_t cq_page_dir;
	+ /* CQ page directory. */
	+ uint64_t tqm_page_dir;
	+ /* TQM page directory. */
	+ uint64_t tim_page_dir;
	+ /* TIM page directory. */
	+ uint32_t number_of_qp;
	+ /* Number of QPs. */
	+ uint32_t number_of_mrw;
	+ /* Number of MRWs. */
	+ uint32_t number_of_srq;
	+ /* Number of SRQs. */
	+ uint32_t number_of_cq;
	+ /* Number of CQs. */
	+ uint32_t max_qp_per_vf;
	+ /* Number of QPs per VF. */
	+ uint32_t max_mrw_per_vf;
	+ /* Number of MRWs per VF. */
	+ uint32_t max_srq_per_vf;
	+ /* Number of SRQs per VF. */
	+ uint32_t max_cq_per_vf;
	+ /* Number of CQs per VF. */
	+ uint32_t max_gid_per_vf;
	+ /* Number of GIDs per VF. */
	+ uint32_t stat_ctx_id;
	+ /* Statistics context index for this function. */
	+} __attribute__((packed));
	+
	+/* De-initialize Firmware command (16 bytes) */
	+
	+struct cmdq_deinitialize_fw {
	+ uint8_t opcode;
	+ /* Command opcode. */
	+ /* De-initialize firmware command deinitializes the firmware. */
	+ #define CMDQ_DEINITIALIZE_FW_OPCODE_DEINITIALIZE_FW UINT32_C(0x81)
	+ uint8_t cmd_size;
	+ /* Size of the command in 16-byte units. */
	+ uint16_t flags;
	+ /* Flags and attribs of the command. */
	+ uint16_t cookie;
	+ /* Driver supplied handle to associate the command and the response. */
	+ uint8_t resp_size;
	+ /* Size of the response buffer in 16-byte units. */
	+ uint8_t reserved8;
	+ uint64_t resp_addr;
	+ /* Host address of the response. */
	+} __attribute__((packed));
	+
	+/* Stop function command (16 bytes) */
	+
	+struct cmdq_stop_func {
	+ uint8_t opcode;
	+ /* Command opcode. */
	+ /* Stop the function */
	+ #define CMDQ_STOP_FUNC_OPCODE_STOP_FUNC UINT32_C(0x82)
	+ uint8_t cmd_size;
	+ /* Size of the command in 16-byte units. */
	+ uint16_t flags;
	+ /* Flags and attribs of the command. */
	+ uint16_t cookie;
	+ /* Driver supplied handle to associate the command and the response. */
	+ uint8_t resp_size;
	+ /* Size of the response buffer in 16-byte units. */
	+ uint8_t reserved8;
	+ uint64_t resp_addr;
	+ /* Host address of the response. */
	+} __attribute__((packed));
	+
	+/* Query function command (16 bytes) */
	+
	+struct cmdq_query_func {
	+ uint8_t opcode;
	+ /* Command opcode. */
	+ /* Query the HW capabilities for the function. */
	+ #define CMDQ_QUERY_FUNC_OPCODE_QUERY_FUNC UINT32_C(0x83)
	+ uint8_t cmd_size;
	+ /* Size of the command in 16-byte units. */
	+ uint16_t flags;
	+ /* Flags and attribs of the command. */
	+ uint16_t cookie;
	+ /* Driver supplied handle to associate the command and the response. */
	+ uint8_t resp_size;
	+ /* Size of the response buffer in 16-byte units. */
	+ uint8_t reserved8;
	+ uint64_t resp_addr;
	+ /* Host address of the response. */
	+} __attribute__((packed));
	+
	+/* Set function resources command (56 bytes) */
	+
	+struct cmdq_set_func_resources {
	+ uint8_t opcode;
	+ /* Command opcode. */
	+ /*
	+ * Set the following resources for the function: - Max QP, CQ,
	+ * MR+MW, SRQ per PF - Max QP, CQ, MR+MW, SRQ per VF
	+ */
	+ #define CMDQ_SET_FUNC_RESOURCES_OPCODE_SET_FUNC_RESOURCES UINT32_C(0x84)
	+ uint8_t cmd_size;
	+ /* Size of the command in 16-byte units. */
	+ uint16_t flags;
	+ /* Flags and attribs of the command. */
	+ uint16_t cookie;
	+ /* Driver supplied handle to associate the command and the response. */
	+ uint8_t resp_size;
	+ /* Size of the response buffer in 16-byte units. */
	+ uint8_t reserved8;
	+ uint64_t resp_addr;
	+ /* Host address of the response. */
	+ uint32_t number_of_qp;
	+ /*
	+ * Number of QPs. It is the responsibility of the host to first extend
	+ * the existing PBL with new addresses to pages to handle the
	+ * adjustment. Must be greater or equal to current.
	+ */
	+ uint32_t number_of_mrw;
	+ /*
	+ * Number of MRWs. It is the responsibility of the host to first extend
	+ * the existing PBL with new addresses to pages to handle the
	+ * adjustment. Must be greater or equal to current.
	+ */
	+ uint32_t number_of_srq;
	+ /*
	+ * Number of SRQs. It is the responsibility of the host to first extend
	+ * the existing PBL with new addresses to pages to handle the
	+ * adjustment. Must be greater or equal to current.
	+ */
	+ uint32_t number_of_cq;
	+ /*
	+ * Number of CQs. It is the responsibility of the host to first extend
	+ * the existing PBL with new addresses to pages to handle the
	+ * adjustment. Must be greater or equal to current.
	+ */
	+ uint32_t max_qp_per_vf;
	+ /* Number of QPs per VF. */
	+ uint32_t max_mrw_per_vf;
	+ /* Number of MRWs per VF. */
	+ uint32_t max_srq_per_vf;
	+ /* Number of SRQs per VF. */
	+ uint32_t max_cq_per_vf;
	+ /* Number of CQs per VF. */
	+ uint32_t max_gid_per_vf;
	+ /* Number of GIDs per VF. */
	+ uint32_t stat_ctx_id;
	+ /* Statistics context index for this function. */
	+} __attribute__((packed));
	+
	+/* Read hardware resource context command (24 bytes) */
	+
	+struct cmdq_read_context {
	+ uint8_t opcode;
	+ /* Command opcode. */
	+ /*
	+ * Read the current state of any internal resource context. Can
	+ * only be issued from a PF.
	+ */
	+ #define CMDQ_READ_CONTEXT_OPCODE_READ_CONTEXT UINT32_C(0x85)
	+ uint8_t cmd_size;
	+ /* Size of the command in 16-byte units. */
	+ uint16_t flags;
	+ /* Flags and attribs of the command. */
	+ uint16_t cookie;
	+ /* Driver supplied handle to associate the command and the response. */
	+ uint8_t resp_size;
	+ /* Size of the response buffer in 16-byte units. */
	+ uint8_t reserved8;
	+ uint64_t resp_addr;
	+ /* Host address of the response. */
	+ uint32_t type_xid;
	+ /* Context type */
	+ /* Context ID */
	+ #define CMDQ_READ_CONTEXT_XID_MASK UINT32_C(0xffffff)
	+ #define CMDQ_READ_CONTEXT_XID_SFT 0
	+ /* Context type */
	+ #define CMDQ_READ_CONTEXT_TYPE_MASK UINT32_C(0xff000000)
	+ #define CMDQ_READ_CONTEXT_TYPE_SFT 24
	+ /*
	+ * Read QPC. The context (448 bytes) goes to resp_addr (as is,
	+ * without a header), and resp_size should be set to 28 (448/16)
	+ */
	+ #define CMDQ_READ_CONTEXT_TYPE_QPC (UINT32_C(0x0) << 24)
	+ /*
	+ * Read CQ. The context (64 bytes) goes to resp_addr (as is,
	+ * without a header), and resp_size should be set to 4 (64/16)
	+ */
	+ #define CMDQ_READ_CONTEXT_TYPE_CQ (UINT32_C(0x1) << 24)
	+ /*
	+ * Read MRW. The context (128 bytes) goes to resp_addr (as is,
	+ * without a header), and resp_size should be set to 8 (128/16)
	+ */
	+ #define CMDQ_READ_CONTEXT_TYPE_MRW (UINT32_C(0x2) << 24)
	+ /*
	+ * Read SRQ. The context (64 bytes) goes to resp_addr (as is,
	+ * without a header), and resp_size should be set to 4 (64/16)
	+ */
	+ #define CMDQ_READ_CONTEXT_TYPE_SRQ (UINT32_C(0x3) << 24)
	+ uint32_t unused_0;
	+} __attribute__((packed));
	+
	+/* Send a request from VF to pass a command to the PF. VF HSI is suspended until the PF returns the response (32 bytes) */
	+
	+struct cmdq_vf_backchannel_request {
	+ uint8_t opcode;
	+ /* Command opcode. */
	+ /*
	+ * Send a request from VF to pass a command to the PF. VF HSI is
	+ * suspended until the PF returns the response
	+ */
	+ #define CMDQ_VF_BACKCHANNEL_REQUEST_OPCODE_VF_BACKCHANNEL_REQUEST UINT32_C(0x86)
	+ uint8_t cmd_size;
	+ /* Size of the command in 16-byte units. */
	+ uint16_t flags;
	+ /* Flags and attribs of the command. */
	+ uint16_t cookie;
	+ /* Driver supplied handle to associate the command and the response. */
	+ uint8_t resp_size;
	+ /* Size of the response buffer in 16-byte units. */
	+ uint8_t reserved8;
	+ uint64_t resp_addr;
	+ /* Host address of the response. */
	+ uint64_t command_addr;
	+ /* Address of command request structure in VF space */
	+ uint16_t command_length;
	+ /*
	+ * Command request length (up to 4K). An optional address of the
	+ * extended response buffer should be provided in the request
	+ */
	+ uint16_t unused_0[3];
	+} __attribute__((packed));
	+
	+/* Read VF memory (primarily to get the backchannel request blob). Can only be issued from a PF. (32 bytes) */
	+
	+struct cmdq_read_vf_memory {
	+ uint8_t opcode;
	+ /* Command opcode. */
	+ /*
	+ * Read VF memory (primarily to get the backchannel request
	+ * blob). Can only be issued from a PF.
	+ */
	+ #define CMDQ_READ_VF_MEMORY_OPCODE_READ_VF_MEMORY UINT32_C(0x87)
	+ uint8_t cmd_size;
	+ /* Size of the command in 16-byte units. */
	+ uint16_t flags;
	+ /* Flags and attribs of the command. */
	+ uint16_t cookie;
	+ /* Driver supplied handle to associate the command and the response. */
	+ uint8_t resp_size;
	+ /* Size of the response buffer in 16-byte units. */
	+ uint8_t reserved8;
	+ uint64_t resp_addr;
	+ /* Host address of the response. */
	+ uint64_t addr;
	+ /* Address of memory in VF space to read */
	+ uint16_t vf_id;
	+ /* VF id, as provided in 0xC0 VF request notification */
	+ uint16_t length;
	+ /* Length to read, up to 4K */
	+ uint32_t unused_0;
	+} __attribute__((packed));
	+
	+/* Write VF memory (primarily to put the backchannel response blob), and re-enable VF HSI (post a CAG completion to it). Can only be issued from a PF. (40 bytes) */
	+
	+struct cmdq_complete_vf_request {
	+ uint8_t opcode;
	+ /* Command opcode. */
	+ /*
	+ * Write VF memory (primarily to put the backchannel response
	+ * blob), and re-enable VF HSI (post a CAG completion to it). Can
	+ * only be issued from a PF.
	+ */
	+ #define CMDQ_COMPLETE_VF_REQUEST_OPCODE_COMPLETE_VF_REQUEST UINT32_C(0x88)
	+ uint8_t cmd_size;
	+ /* Size of the command in 16-byte units. */
	+ uint16_t flags;
	+ /* Flags and attribs of the command. */
	+ uint16_t cookie;
	+ /* Driver supplied handle to associate the command and the response. */
	+ uint8_t resp_size;
	+ /* Size of the response buffer in 16-byte units. */
	+ uint8_t reserved8;
	+ uint64_t resp_addr;
	+ /* Host address of the response. */
	+ uint64_t addr;
	+ /*
	+ * Optional address of extended response in VF space to write. Length is
	+ * in resp_size in 16 byte units.
	+ */
	+ uint32_t vf_misc;
	+ /* Completion misc field to VF CREQ */
	+ uint16_t vf_id;
	+ /* VF id, as provided in 0xC0 VF request notification */
	+ uint16_t vf_cookie;
	+ /* Completion cookie for the VF command, goes to VF CREQ */
	+ uint8_t vf_status;
	+ /* Completion status for the VF command, goes to VF CREQ */
	+ uint8_t unused_0[7];
	+} __attribute__((packed));
	+
	+/* Map TC to COS. Can only be issued from a PF (24 bytes) */
	+
	+struct cmdq_map_tc_to_cos {
	+ uint8_t opcode;
	+ /* Command opcode. */
	+ /* Map TC to COS. Can only be issued from a PF. */
	+ #define CMDQ_MAP_TC_TO_COS_OPCODE_MAP_TC_TO_COS UINT32_C(0x8a)
	+ uint8_t cmd_size;
	+ /* Size of the command in 16-byte units. */
	+ uint16_t flags;
	+ /* Flags and attribs of the command. */
	+ uint16_t cookie;
	+ /* Driver supplied handle to associate the command and the response. */
	+ uint8_t resp_size;
	+ /* Size of the response buffer in 16-byte units. */
	+ uint8_t reserved8;
	+ uint64_t resp_addr;
	+ /* Host address of the response. */
	+ uint16_t cos0;
	+ /* 1st COS index mapped to RoCE */
	+ /* Don't change this COS. */
	+ #define CMDQ_MAP_TC_TO_COS_COS0_NO_CHANGE UINT32_C(0xffff)
	+ uint16_t cos1;
	+ /* 2nd COS index mapped to RoCE */
	+ /* Disable this COS. */
	+ #define CMDQ_MAP_TC_TO_COS_COS1_DISABLE UINT32_C(0x8000)
	+ /* Don't change this COS. */
	+ #define CMDQ_MAP_TC_TO_COS_COS1_NO_CHANGE UINT32_C(0xffff)
	+ uint32_t unused_0;
	+} __attribute__((packed));
	+
	+/* Query version command (16 bytes) */
	+
	+struct cmdq_query_version {
	+ uint8_t opcode;
	+ /* Command opcode. */
	+ /* Query version. */
	+ #define CMDQ_QUERY_VERSION_OPCODE_QUERY_VERSION UINT32_C(0x8b)
	+ uint8_t cmd_size;
	+ /* Size of the command in 16-byte units. */
	+ uint16_t flags;
	+ /* Flags and attribs of the command. */
	+ uint16_t cookie;
	+ /* Driver supplied handle to associate the command and the response. */
	+ uint8_t resp_size;
	+ /* Size of the response buffer in 16-byte units. */
	+ uint8_t reserved8;
	+ uint64_t resp_addr;
	+ /* Host address of the response. */
	+} __attribute__((packed));
	+
	+/* Modify congestion control command (56 bytes) */
	+
	+struct cmdq_modify_roce_cc {
	+ uint8_t opcode;
	+ /* Command opcode. */
	+ /* Modify congestion control. Can only be issued from a PF. */
	+ #define CMDQ_MODIFY_ROCE_CC_OPCODE_MODIFY_ROCE_CC UINT32_C(0x8c)
	+ uint8_t cmd_size;
	+ /* Size of the command in 16-byte units. */
	+ uint16_t flags;
	+ /* Flags and attribs of the command. */
	+ uint16_t cookie;
	+ /* Driver supplied handle to associate the command and the response. */
	+ uint8_t resp_size;
	+ /* Size of the response buffer in 16-byte units. */
	+ uint8_t reserved8;
	+ uint64_t resp_addr;
	+ /* Host address of the response. */
	+ uint32_t modify_mask;
	+ /* Modify mask signifies the field that is requesting the change. */
	+ /* Enable change. */
	+ #define CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_ENABLE_CC UINT32_C(0x1)
	+ /* Running average weight change. */
	+ #define CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_G UINT32_C(0x2)
	+ /* Number of phases in Fast Recovery. */
	+ #define CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_NUMPHASEPERSTATE UINT32_C(0x4)
	+ /* The starting value of rate change. */
	+ #define CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_INIT_CR UINT32_C(0x8)
	+ /* The starting value of target rate change. */
	+ #define CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_INIT_TR UINT32_C(0x10)
	+ /* IP TOS ECN change */
	+ #define CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TOS_ECN UINT32_C(0x20)
	+ /* IP TOS DSCP change */
	+ #define CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TOS_DSCP UINT32_C(0x40)
	+ /* Alternate IP TOS ECN change */
	+ #define CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_ALT_VLAN_PCP UINT32_C(0x80)
	+ /* Alternate IP TOS DSCP change */
	+ #define CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_ALT_TOS_DSCP UINT32_C(0x100)
	+ /* Round trip time in units of usecs */
	+ #define CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_RTT UINT32_C(0x200)
	+ /* 0 for DCTCP , 1 for TCP */
	+ #define CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_CC_MODE UINT32_C(0x400)
	+ /* The value used as CP when cc_mode is 1(TCP) */
	+ #define CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TCP_CP UINT32_C(0x800)
	+ /* Specifies the RoCE Tx Queue ( o to 3) to use for sending CNP packets */
	+ #define CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TX_QUEUE UINT32_C(0x1000)
	+ /* Inactivity time after which QP CC parameters are initialized */
	+ #define CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_INACTIVITY_CP UINT32_C(0x2000)
	+ uint8_t enable_cc;
	+ /* rsvd1 is 7 b */
	+ /* Enable. */
	+ #define CMDQ_MODIFY_ROCE_CC_ENABLE_CC UINT32_C(0x1)
	+ /* rsvd1 is 7 b */
	+ #define CMDQ_MODIFY_ROCE_CC_RSVD1_MASK UINT32_C(0xfe)
	+ #define CMDQ_MODIFY_ROCE_CC_RSVD1_SFT 1
	+ uint8_t g;
	+ /* rsvd2 is 5 b */
	+ /* Congestion Probability averaging factor. */
	+ #define CMDQ_MODIFY_ROCE_CC_G_MASK UINT32_C(0x7)
	+ #define CMDQ_MODIFY_ROCE_CC_G_SFT 0
	+ /* rsvd2 is 5 b */
	+ #define CMDQ_MODIFY_ROCE_CC_RSVD2_MASK UINT32_C(0xf8)
	+ #define CMDQ_MODIFY_ROCE_CC_RSVD2_SFT 3
	+ uint8_t num_phases_per_state;
	+ /* Number of phases in Fast Recovery. */
	+ uint8_t rsvd9;
	+ /* rsvd9 is 8 b */
	+ uint16_t init_cr;
	+ /* The starting value of rate. */
	+ uint16_t init_tr;
	+ /* The starting value of target rate. */
	+ uint8_t tos_dscp_tos_ecn;
	+ /* IP TOS DSCP. */
	+ /* IP TOS ECN. Valid values are 1 or 2 when ECN is enabled. */
	+ #define CMDQ_MODIFY_ROCE_CC_TOS_ECN_MASK UINT32_C(0x3)
	+ #define CMDQ_MODIFY_ROCE_CC_TOS_ECN_SFT 0
	+ /* IP TOS DSCP. */
	+ #define CMDQ_MODIFY_ROCE_CC_TOS_DSCP_MASK UINT32_C(0xfc)
	+ #define CMDQ_MODIFY_ROCE_CC_TOS_DSCP_SFT 2
	+ uint8_t alt_vlan_pcp;
	+ /* rsvd3 is 5 b */
	+ /* Alternate vlan pcp value for CNP packets. */
	+ #define CMDQ_MODIFY_ROCE_CC_ALT_VLAN_PCP_MASK UINT32_C(0x7)
	+ #define CMDQ_MODIFY_ROCE_CC_ALT_VLAN_PCP_SFT 0
	+ /* rsvd3 is 5 b */
	+ #define CMDQ_MODIFY_ROCE_CC_RSVD3_MASK UINT32_C(0xf8)
	+ #define CMDQ_MODIFY_ROCE_CC_RSVD3_SFT 3
	+ uint16_t alt_tos_dscp;
	+ /* rsvd4 is 10 b */
	+ /* Alternate IP TOS DSCP. */
	+ #define CMDQ_MODIFY_ROCE_CC_ALT_TOS_DSCP_MASK UINT32_C(0x3f)
	+ #define CMDQ_MODIFY_ROCE_CC_ALT_TOS_DSCP_SFT 0
	+ /* rsvd4 is 10 b */
	+ #define CMDQ_MODIFY_ROCE_CC_RSVD4_MASK UINT32_C(0xffc0)
	+ #define CMDQ_MODIFY_ROCE_CC_RSVD4_SFT 6
	+ uint16_t rtt;
	+ /* rsvd5 is 2 b */
	+ /* Round trip time in units of usecs */
	+ #define CMDQ_MODIFY_ROCE_CC_RTT_MASK UINT32_C(0x3fff)
	+ #define CMDQ_MODIFY_ROCE_CC_RTT_SFT 0
	+ /* rsvd5 is 2 b */
	+ #define CMDQ_MODIFY_ROCE_CC_RSVD5_MASK UINT32_C(0xc000)
	+ #define CMDQ_MODIFY_ROCE_CC_RSVD5_SFT 14
	+ uint16_t tcp_cp;
	+ /* rsvd6 is 6 b */
	+ /* The value used as CP when cc_mode is 1(TCP) */
	+ #define CMDQ_MODIFY_ROCE_CC_TCP_CP_MASK UINT32_C(0x3ff)
	+ #define CMDQ_MODIFY_ROCE_CC_TCP_CP_SFT 0
	+ /* rsvd6 is 6 b */
	+ #define CMDQ_MODIFY_ROCE_CC_RSVD6_MASK UINT32_C(0xfc00)
	+ #define CMDQ_MODIFY_ROCE_CC_RSVD6_SFT 10
	+ uint8_t cc_mode;
	+ /* rsvd7 is 7 b */
	+ /* 0 for DCTCP , 1 for TCP */
	+ #define CMDQ_MODIFY_ROCE_CC_CC_MODE UINT32_C(0x1)
	+ /* rsvd7 is 7 b */
	+ #define CMDQ_MODIFY_ROCE_CC_RSVD7_MASK UINT32_C(0xfe)
	+ #define CMDQ_MODIFY_ROCE_CC_RSVD7_SFT 1
	+ uint8_t tx_queue;
	+ /* rsvd8 is 6 b */
	+ /* Specifies the RoCE Tx Queue ( o to 3) to use for sending CNP packets */
	+ #define CMDQ_MODIFY_ROCE_CC_TX_QUEUE_MASK UINT32_C(0x3)
	+ #define CMDQ_MODIFY_ROCE_CC_TX_QUEUE_SFT 0
	+ /* rsvd8 is 6 b */
	+ #define CMDQ_MODIFY_ROCE_CC_RSVD8_MASK UINT32_C(0xfc)
	+ #define CMDQ_MODIFY_ROCE_CC_RSVD8_SFT 2
	+ uint16_t inactivity_th;
	+ /* Inactivity time after which QP CC parameters are initialized */
	+ uint64_t reserved64;
	+ uint64_t reserved64_1;
	+} __attribute__((packed));
	+
	+/* Query congestion control command (16 bytes) */
	+
	+struct cmdq_query_roce_cc {
	+ uint8_t opcode;
	+ /* Command opcode. */
	+ /* Query congestion control. */
	+ #define CMDQ_QUERY_ROCE_CC_OPCODE_QUERY_ROCE_CC UINT32_C(0x8d)
	+ uint8_t cmd_size;
	+ /* Size of the command in 16-byte units. */
	+ uint16_t flags;
	+ /* Flags and attribs of the command. */
	+ uint16_t cookie;
	+ /* Driver supplied handle to associate the command and the response. */
	+ uint8_t resp_size;
	+ /* Size of the response buffer in 16-byte units. */
	+ uint8_t reserved8;
	+ uint64_t resp_addr;
	+ /* Host address of the response. */
	+} __attribute__((packed));
	+
	+/* Command-Response Event Queue (CREQ) Structures */
	+/* Description: This is an async event indicating error happened on a QP. */
	+/* Base CREQ Record (16 bytes) */
	+
	+struct creq_base {
	+ uint8_t type;
	+ /*
	+ * This field indicates the exact type of the completion. By convention,
	+ * the LSB identifies the length of the record in 16B units. Even values
	+ * indicate 16B records. Odd values indicate 32B records.
	+ */
	+ #define CREQ_BASE_TYPE_MASK UINT32_C(0x3f)
	+ #define CREQ_BASE_TYPE_SFT 0
	+ /* QP Async Notification */
	+ #define CREQ_BASE_TYPE_QP_EVENT UINT32_C(0x38)
	+ /* Function Async Notification */
	+ #define CREQ_BASE_TYPE_FUNC_EVENT UINT32_C(0x3a)
	+ #define CREQ_BASE_RESERVED2_MASK UINT32_C(0xc0)
	+ #define CREQ_BASE_RESERVED2_SFT 6
	+ uint8_t reserved56[7];
	+ uint8_t v;
	+ /*
	+ * This value is written by the NIC such that it will be different for
	+ * each pass through the completion queue. The even passes will write 1.
	+ * The odd passes will write 0.
	+ */
	+ #define CREQ_BASE_V UINT32_C(0x1)
	+ #define CREQ_BASE_RESERVED7_MASK UINT32_C(0xfe)
	+ #define CREQ_BASE_RESERVED7_SFT 1
	+ uint8_t event;
	+ /* This is the modifier on to the type field. */
	+ uint16_t reserved48[3];
	+} __attribute__((packed));
	+
	+/* RoCE Function Async Event Notification (16 bytes) */
	+
	+struct creq_func_event {
	+ uint8_t type;
	+ /*
	+ * This field indicates the exact type of the completion. By convention,
	+ * the LSB identifies the length of the record in 16B units. Even values
	+ * indicate 16B records. Odd values indicate 32B records.
	+ */
	+ #define CREQ_FUNC_EVENT_TYPE_MASK UINT32_C(0x3f)
	+ #define CREQ_FUNC_EVENT_TYPE_SFT 0
	+ /* Function Async Notification */
	+ #define CREQ_FUNC_EVENT_TYPE_FUNC_EVENT UINT32_C(0x3a)
	+ #define CREQ_FUNC_EVENT_RESERVED2_MASK UINT32_C(0xc0)
	+ #define CREQ_FUNC_EVENT_RESERVED2_SFT 6
	+ uint8_t reserved56[7];
	+ uint8_t v;
	+ /*
	+ * This value is written by the NIC such that it will be different for
	+ * each pass through the completion queue. The even passes will write 1.
	+ * The odd passes will write 0.
	+ */
	+ #define CREQ_FUNC_EVENT_V UINT32_C(0x1)
	+ #define CREQ_FUNC_EVENT_RESERVED7_MASK UINT32_C(0xfe)
	+ #define CREQ_FUNC_EVENT_RESERVED7_SFT 1
	+ uint8_t event;
	+ /*
	+ * This value defines what type of async event has occurred on the
	+ * function.
	+ */
	+ /*
	+ * Invalid PBL or PCIE UR response occurred in SQ WQE or IRRQ
	+ * read access.
	+ */
	+ #define CREQ_FUNC_EVENT_EVENT_TX_WQE_ERROR UINT32_C(0x1)
	+ /* Invalid PBL or PCIE UR response occurred during data read access. */
	+ #define CREQ_FUNC_EVENT_EVENT_TX_DATA_ERROR UINT32_C(0x2)
	+ /*
	+ * Invalid PBL or PCIE UR response occurred in RQ/SRQ WQE or
	+ * ORRQ read access.
	+ */
	+ #define CREQ_FUNC_EVENT_EVENT_RX_WQE_ERROR UINT32_C(0x3)
	+ /* Invalid PBL occurred during data write access. */
	+ #define CREQ_FUNC_EVENT_EVENT_RX_DATA_ERROR UINT32_C(0x4)
	+ /* Invalid PBL occurred during CQ write access. */
	+ #define CREQ_FUNC_EVENT_EVENT_CQ_ERROR UINT32_C(0x5)
	+ /* Invalid PBL or PCIE UR response occurred in TQM read access. */
	+ #define CREQ_FUNC_EVENT_EVENT_TQM_ERROR UINT32_C(0x6)
	+ /* PCIE UR response occurred in CFC read access. */
	+ #define CREQ_FUNC_EVENT_EVENT_CFCQ_ERROR UINT32_C(0x7)
	+ /* PCIE UR response occurred in CFC read access. */
	+ #define CREQ_FUNC_EVENT_EVENT_CFCS_ERROR UINT32_C(0x8)
	+ /* PCIE UR response occurred in CFC read access. */
	+ #define CREQ_FUNC_EVENT_EVENT_CFCC_ERROR UINT32_C(0x9)
	+ /* PCIE UR response occurred in CFC read access. */
	+ #define CREQ_FUNC_EVENT_EVENT_CFCM_ERROR UINT32_C(0xa)
	+ /* Invalid PBL or PCIE UR response occurred on timer read access. */
	+ #define CREQ_FUNC_EVENT_EVENT_TIM_ERROR UINT32_C(0xb)
	+ /* A VF sent a backchannel command request */
	+ #define CREQ_FUNC_EVENT_EVENT_VF_COMM_REQUEST UINT32_C(0x80)
	+ /*
	+ * Communication resource (QPC, CQ, SRQ, MRW) exhausted, and
	+ * resource array extension is enabled
	+ */
	+ #define CREQ_FUNC_EVENT_EVENT_RESOURCE_EXHAUSTED UINT32_C(0x81)
	+ uint16_t reserved48[3];
	+} __attribute__((packed));
	+
	+/* RoCE Slowpath Command Completion (16 bytes) */
	+
	+struct creq_qp_event {
	+ uint8_t type;
	+ /*
	+ * This field indicates the exact type of the completion. By convention,
	+ * the LSB identifies the length of the record in 16B units. Even values
	+ * indicate 16B records. Odd values indicate 32B records.
	+ */
	+ #define CREQ_QP_EVENT_TYPE_MASK UINT32_C(0x3f)
	+ #define CREQ_QP_EVENT_TYPE_SFT 0
	+ /* QP Async Notification */
	+ #define CREQ_QP_EVENT_TYPE_QP_EVENT UINT32_C(0x38)
	+ #define CREQ_QP_EVENT_RESERVED2_MASK UINT32_C(0xc0)
	+ #define CREQ_QP_EVENT_RESERVED2_SFT 6
	+ uint8_t status;
	+ /* Status of the response. */
	+ /* Success. */
	+ #define CREQ_QP_EVENT_STATUS_SUCCESS UINT32_C(0x0)
	+ /* Fail. */
	+ #define CREQ_QP_EVENT_STATUS_FAIL UINT32_C(0x1)
	+ /* Resources. */
	+ #define CREQ_QP_EVENT_STATUS_RESOURCES UINT32_C(0x2)
	+ /* Invalid command. */
	+ #define CREQ_QP_EVENT_STATUS_INVALID_CMD UINT32_C(0x3)
	+ /* Not implemented. */
	+ #define CREQ_QP_EVENT_STATUS_NOT_IMPLEMENTED UINT32_C(0x4)
	+ /* Invalid parameter. */
	+ #define CREQ_QP_EVENT_STATUS_INVALID_PARAMETER UINT32_C(0x5)
	+ /* Hardware operation failed. */
	+ #define CREQ_QP_EVENT_STATUS_HARDWARE_ERROR UINT32_C(0x6)
	+ /* Firmware operation failed due to internal error. */
	+ #define CREQ_QP_EVENT_STATUS_INTERNAL_ERROR UINT32_C(0x7)
	+ uint16_t cookie;
	+ /* Driver supplied handle to associate the command and the response. */
	+ uint32_t reserved32;
	+ uint8_t v;
	+ /*
	+ * This value is written by the NIC such that it will be different for
	+ * each pass through the completion queue. The even passes will write 1.
	+ * The odd passes will write 0.
	+ */
	+ #define CREQ_QP_EVENT_V UINT32_C(0x1)
	+ #define CREQ_QP_EVENT_RESERVED7_MASK UINT32_C(0xfe)
	+ #define CREQ_QP_EVENT_RESERVED7_SFT 1
	+ uint8_t event;
	+ /* Event or command opcode. */
	+ /* Create QP command response. */
	+ #define CREQ_QP_EVENT_EVENT_CREATE_QP UINT32_C(0x1)
	+ /* Destroy QP command response. */
	+ #define CREQ_QP_EVENT_EVENT_DESTROY_QP UINT32_C(0x2)
	+ /* Modify QP command response. */
	+ #define CREQ_QP_EVENT_EVENT_MODIFY_QP UINT32_C(0x3)
	+ /* Query QP command response. */
	+ #define CREQ_QP_EVENT_EVENT_QUERY_QP UINT32_C(0x4)
	+ /* Create SRQ command response. */
	+ #define CREQ_QP_EVENT_EVENT_CREATE_SRQ UINT32_C(0x5)
	+ /* Destroy SRQ command response. */
	+ #define CREQ_QP_EVENT_EVENT_DESTROY_SRQ UINT32_C(0x6)
	+ /* Query SRQ command response. */
	+ #define CREQ_QP_EVENT_EVENT_QUERY_SRQ UINT32_C(0x8)
	+ /* Create CQ command response. */
	+ #define CREQ_QP_EVENT_EVENT_CREATE_CQ UINT32_C(0x9)
	+ /* Destroy CQ command response. */
	+ #define CREQ_QP_EVENT_EVENT_DESTROY_CQ UINT32_C(0xa)
	+ /* Resize CQ command response. */
	+ #define CREQ_QP_EVENT_EVENT_RESIZE_CQ UINT32_C(0xc)
	+ /* Allocate MRW command response. */
	+ #define CREQ_QP_EVENT_EVENT_ALLOCATE_MRW UINT32_C(0xd)
	+ /* De-allocate key command response. */
	+ #define CREQ_QP_EVENT_EVENT_DEALLOCATE_KEY UINT32_C(0xe)
	+ /* Register MR command response. */
	+ #define CREQ_QP_EVENT_EVENT_REGISTER_MR UINT32_C(0xf)
	+ /* Deregister MR command response. */
	+ #define CREQ_QP_EVENT_EVENT_DEREGISTER_MR UINT32_C(0x10)
	+ /* Add GID command response. */
	+ #define CREQ_QP_EVENT_EVENT_ADD_GID UINT32_C(0x11)
	+ /* Delete GID command response. */
	+ #define CREQ_QP_EVENT_EVENT_DELETE_GID UINT32_C(0x12)
	+ /* Modify GID command response. */
	+ #define CREQ_QP_EVENT_EVENT_MODIFY_GID UINT32_C(0x17)
	+ /* Query GID command response. */
	+ #define CREQ_QP_EVENT_EVENT_QUERY_GID UINT32_C(0x18)
	+ /* Create QP1 command response. */
	+ #define CREQ_QP_EVENT_EVENT_CREATE_QP1 UINT32_C(0x13)
	+ /* Destroy QP1 command response. */
	+ #define CREQ_QP_EVENT_EVENT_DESTROY_QP1 UINT32_C(0x14)
	+ /* Create AH command response. */
	+ #define CREQ_QP_EVENT_EVENT_CREATE_AH UINT32_C(0x15)
	+ /* Destroy AH command response. */
	+ #define CREQ_QP_EVENT_EVENT_DESTROY_AH UINT32_C(0x16)
	+ /* Initialize firmware command response. */
	+ #define CREQ_QP_EVENT_EVENT_INITIALIZE_FW UINT32_C(0x80)
	+ /* De-initialize firmware command response. */
	+ #define CREQ_QP_EVENT_EVENT_DEINITIALIZE_FW UINT32_C(0x81)
	+ /* Stop PF command response. */
	+ #define CREQ_QP_EVENT_EVENT_STOP_FUNC UINT32_C(0x82)
	+ /* Query info PF command response. */
	+ #define CREQ_QP_EVENT_EVENT_QUERY_FUNC UINT32_C(0x83)
	+ /* Set function resources command response. */
	+ #define CREQ_QP_EVENT_EVENT_SET_FUNC_RESOURCES UINT32_C(0x84)
	+ /* Map TC to COS response. */
	+ #define CREQ_QP_EVENT_EVENT_MAP_TC_TO_COS UINT32_C(0x8a)
	+ /* Query firmware and interface version response. */
	+ #define CREQ_QP_EVENT_EVENT_QUERY_VERSION UINT32_C(0x8b)
	+ /* Modify congestion control response. */
	+ #define CREQ_QP_EVENT_EVENT_MODIFY_ROCE_CC UINT32_C(0x8c)
	+ /* Query congestion control response. */
	+ #define CREQ_QP_EVENT_EVENT_QUERY_ROCE_CC UINT32_C(0x8d)
	+ /* QP error notification event. */
	+ #define CREQ_QP_EVENT_EVENT_QP_ERROR_NOTIFICATION UINT32_C(0xc0)
	+ uint16_t reserved48[3];
	+} __attribute__((packed));
	+
	+/* Create QP command response (16 bytes) */
	+
	+struct creq_create_qp_resp {
	+ uint8_t type;
	+ /*
	+ * This field indicates the exact type of the completion. By convention,
	+ * the LSB identifies the length of the record in 16B units. Even values
	+ * indicate 16B records. Odd values indicate 32B records.
	+ */
	+ #define CREQ_CREATE_QP_RESP_TYPE_MASK UINT32_C(0x3f)
	+ #define CREQ_CREATE_QP_RESP_TYPE_SFT 0
	+ /* QP Async Notification */
	+ #define CREQ_CREATE_QP_RESP_TYPE_QP_EVENT UINT32_C(0x38)
	+ #define CREQ_CREATE_QP_RESP_RESERVED2_MASK UINT32_C(0xc0)
	+ #define CREQ_CREATE_QP_RESP_RESERVED2_SFT 6
	+ uint8_t status;
	+ /* Status of the response. */
	+ uint16_t cookie;
	+ /* Driver supplied handle to associate the command and the response. */
	+ uint32_t xid;
	+ /* QP context id */
	+ uint8_t v;
	+ /*
	+ * This value is written by the NIC such that it will be different for
	+ * each pass through the completion queue. The even passes will write 1.
	+ * The odd passes will write 0.
	+ */
	+ #define CREQ_CREATE_QP_RESP_V UINT32_C(0x1)
	+ #define CREQ_CREATE_QP_RESP_RESERVED7_MASK UINT32_C(0xfe)
	+ #define CREQ_CREATE_QP_RESP_RESERVED7_SFT 1
	+ uint8_t event;
	+ /* Event or command opcode. */
	+ /* Create QP command response. */
	+ #define CREQ_CREATE_QP_RESP_EVENT_CREATE_QP UINT32_C(0x1)
	+ uint16_t reserved48[3];
	+} __attribute__((packed));
	+
	+/* Destroy QP command response (16 bytes) */
	+
	+struct creq_destroy_qp_resp {
	+ uint8_t type;
	+ /*
	+ * This field indicates the exact type of the completion. By convention,
	+ * the LSB identifies the length of the record in 16B units. Even values
	+ * indicate 16B records. Odd values indicate 32B records.
	+ */
	+ #define CREQ_DESTROY_QP_RESP_TYPE_MASK UINT32_C(0x3f)
	+ #define CREQ_DESTROY_QP_RESP_TYPE_SFT 0
	+ /* QP Async Notification */
	+ #define CREQ_DESTROY_QP_RESP_TYPE_QP_EVENT UINT32_C(0x38)
	+ #define CREQ_DESTROY_QP_RESP_RESERVED2_MASK UINT32_C(0xc0)
	+ #define CREQ_DESTROY_QP_RESP_RESERVED2_SFT 6
	+ uint8_t status;
	+ /* Status of the response. */
	+ uint16_t cookie;
	+ /* Driver supplied handle to associate the command and the response. */
	+ uint32_t xid;
	+ /* QP context id */
	+ uint8_t v;
	+ /*
	+ * This value is written by the NIC such that it will be different for
	+ * each pass through the completion queue. The even passes will write 1.
	+ * The odd passes will write 0.
	+ */
	+ #define CREQ_DESTROY_QP_RESP_V UINT32_C(0x1)
	+ #define CREQ_DESTROY_QP_RESP_RESERVED7_MASK UINT32_C(0xfe)
	+ #define CREQ_DESTROY_QP_RESP_RESERVED7_SFT 1
	+ uint8_t event;
	+ /* Event or command opcode. */
	+ /* Destroy QP command response. */
	+ #define CREQ_DESTROY_QP_RESP_EVENT_DESTROY_QP UINT32_C(0x2)
	+ uint16_t reserved48[3];
	+} __attribute__((packed));
	+
	+/* Modify QP command response (16 bytes) */
	+
	+struct creq_modify_qp_resp {
	+ uint8_t type;
	+ /*
	+ * This field indicates the exact type of the completion. By convention,
	+ * the LSB identifies the length of the record in 16B units. Even values
	+ * indicate 16B records. Odd values indicate 32B records.
	+ */
	+ #define CREQ_MODIFY_QP_RESP_TYPE_MASK UINT32_C(0x3f)
	+ #define CREQ_MODIFY_QP_RESP_TYPE_SFT 0
	+ /* QP Async Notification */
	+ #define CREQ_MODIFY_QP_RESP_TYPE_QP_EVENT UINT32_C(0x38)
	+ #define CREQ_MODIFY_QP_RESP_RESERVED2_MASK UINT32_C(0xc0)
	+ #define CREQ_MODIFY_QP_RESP_RESERVED2_SFT 6
	+ uint8_t status;
	+ /* Status of the response. */
	+ uint16_t cookie;
	+ /* Driver supplied handle to associate the command and the response. */
	+ uint32_t xid;
	+ /* QP context id */
	+ uint8_t v;
	+ /*
	+ * This value is written by the NIC such that it will be different for
	+ * each pass through the completion queue. The even passes will write 1.
	+ * The odd passes will write 0.
	+ */
	+ #define CREQ_MODIFY_QP_RESP_V UINT32_C(0x1)
	+ #define CREQ_MODIFY_QP_RESP_RESERVED7_MASK UINT32_C(0xfe)
	+ #define CREQ_MODIFY_QP_RESP_RESERVED7_SFT 1
	+ uint8_t event;
	+ /* Event or command opcode. */
	+ /* Modify QP command response. */
	+ #define CREQ_MODIFY_QP_RESP_EVENT_MODIFY_QP UINT32_C(0x3)
	+ uint16_t reserved48[3];
	+} __attribute__((packed));
	+
	+/* Query QP command response (16 bytes) */
	+
	+struct creq_query_qp_resp {
	+ uint8_t type;
	+ /*
	+ * This field indicates the exact type of the completion. By convention,
	+ * the LSB identifies the length of the record in 16B units. Even values
	+ * indicate 16B records. Odd values indicate 32B records.
	+ */
	+ #define CREQ_QUERY_QP_RESP_TYPE_MASK UINT32_C(0x3f)
	+ #define CREQ_QUERY_QP_RESP_TYPE_SFT 0
	+ /* QP Async Notification */
	+ #define CREQ_QUERY_QP_RESP_TYPE_QP_EVENT UINT32_C(0x38)
	+ #define CREQ_QUERY_QP_RESP_RESERVED2_MASK UINT32_C(0xc0)
	+ #define CREQ_QUERY_QP_RESP_RESERVED2_SFT 6
	+ uint8_t status;
	+ /* Status of the response. */
	+ uint16_t cookie;
	+ /* Driver supplied handle to associate the command and the response. */
	+ uint32_t size;
	+ /* Side buffer size in 16-byte units */
	+ uint8_t v;
	+ /*
	+ * This value is written by the NIC such that it will be different for
	+ * each pass through the completion queue. The even passes will write 1.
	+ * The odd passes will write 0.
	+ */
	+ #define CREQ_QUERY_QP_RESP_V UINT32_C(0x1)
	+ #define CREQ_QUERY_QP_RESP_RESERVED7_MASK UINT32_C(0xfe)
	+ #define CREQ_QUERY_QP_RESP_RESERVED7_SFT 1
	+ uint8_t event;
	+ /* Event or command opcode. */
	+ /* Query QP command response. */
	+ #define CREQ_QUERY_QP_RESP_EVENT_QUERY_QP UINT32_C(0x4)
	+ uint16_t reserved48[3];
	+} __attribute__((packed));
	+
	+/* Query QP command response side buffer structure (104 bytes) */
	+
	+struct creq_query_qp_resp_sb {
	+ uint8_t opcode;
	+ /* Command opcode. */
	+ /* Query QP command response. */
	+ #define CREQ_QUERY_QP_RESP_SB_OPCODE_QUERY_QP UINT32_C(0x4)
	+ uint8_t status;
	+ /* Status of the response. */
	+ uint16_t cookie;
	+ /* Driver supplied handle to associate the command and the response. */
	+ uint16_t flags;
	+ /* Flags and attribs of the command. */
	+ uint8_t resp_size;
	+ /* Size of the response buffer in 16-byte units. */
	+ uint8_t reserved8;
	+ uint32_t xid;
	+ /* QP context id */
	+ uint8_t en_sqd_async_notify_state;
	+ /* Enable congestion control. */
	+ /* QP state */
	+ #define CREQ_QUERY_QP_RESP_SB_STATE_MASK UINT32_C(0xf)
	+ #define CREQ_QUERY_QP_RESP_SB_STATE_SFT 0
	+ /* Reset. */
	+ #define CREQ_QUERY_QP_RESP_SB_STATE_RESET UINT32_C(0x0)
	+ /* Init. */
	+ #define CREQ_QUERY_QP_RESP_SB_STATE_INIT UINT32_C(0x1)
	+ /* Ready To Receive. */
	+ #define CREQ_QUERY_QP_RESP_SB_STATE_RTR UINT32_C(0x2)
	+ /* Ready To Send. */
	+ #define CREQ_QUERY_QP_RESP_SB_STATE_RTS UINT32_C(0x3)
	+ /* SQ Drain. */
	+ #define CREQ_QUERY_QP_RESP_SB_STATE_SQD UINT32_C(0x4)
	+ /* SQ Error. */
	+ #define CREQ_QUERY_QP_RESP_SB_STATE_SQE UINT32_C(0x5)
	+ /* Error. */
	+ #define CREQ_QUERY_QP_RESP_SB_STATE_ERR UINT32_C(0x6)
	+ /* SQ drain asynchronous notification. */
	+ #define CREQ_QUERY_QP_RESP_SB_EN_SQD_ASYNC_NOTIFY UINT32_C(0x10)
	+ /* Enable congestion control. */
	+ uint8_t access;
	+ /* Access flags. */
	+ /* Local write access. */
	+ #define CREQ_QUERY_QP_RESP_SB_ACCESS_LOCAL_WRITE UINT32_C(0x1)
	+ /* Remote write access. */
	+ #define CREQ_QUERY_QP_RESP_SB_ACCESS_REMOTE_WRITE UINT32_C(0x2)
	+ /* Remote read access. */
	+ #define CREQ_QUERY_QP_RESP_SB_ACCESS_REMOTE_READ UINT32_C(0x4)
	+ /* Remote atomic access. */
	+ #define CREQ_QUERY_QP_RESP_SB_ACCESS_REMOTE_ATOMIC UINT32_C(0x8)
	+ uint16_t pkey;
	+ /* P_KEY index. */
	+ uint32_t qkey;
	+ /* Q_KEY. */
	+ uint32_t reserved32;
	+ uint32_t dgid[4];
	+ /* Destination GID. */
	+ uint32_t flow_label;
	+ /* Flow label. */
	+ uint16_t sgid_index;
	+ /* Source GID index. */
	+ uint8_t hop_limit;
	+ /* Hop limit. */
	+ uint8_t traffic_class;
	+ /* Traffic class. */
	+ uint16_t dest_mac[3];
	+ /* Destination MAC address. */
	+ uint16_t path_mtu_dest_vlan_id;
	+ /* Path MTU. */
	+ /* Destination VLAN ID. */
	+ #define CREQ_QUERY_QP_RESP_SB_DEST_VLAN_ID_MASK UINT32_C(0xfff)
	+ #define CREQ_QUERY_QP_RESP_SB_DEST_VLAN_ID_SFT 0
	+ /* Path MTU. */
	+ #define CREQ_QUERY_QP_RESP_SB_PATH_MTU_MASK UINT32_C(0xf000)
	+ #define CREQ_QUERY_QP_RESP_SB_PATH_MTU_SFT 12
	+ /* 256. */
	+ #define CREQ_QUERY_QP_RESP_SB_PATH_MTU_MTU_256 (UINT32_C(0x0) << 12)
	+ /* 512. */
	+ #define CREQ_QUERY_QP_RESP_SB_PATH_MTU_MTU_512 (UINT32_C(0x1) << 12)
	+ /* 1024. */
	+ #define CREQ_QUERY_QP_RESP_SB_PATH_MTU_MTU_1024 (UINT32_C(0x2) << 12)
	+ /* 2048. */
	+ #define CREQ_QUERY_QP_RESP_SB_PATH_MTU_MTU_2048 (UINT32_C(0x3) << 12)
	+ /* 4096. */
	+ #define CREQ_QUERY_QP_RESP_SB_PATH_MTU_MTU_4096 (UINT32_C(0x4) << 12)
	+ /* 8192. */
	+ #define CREQ_QUERY_QP_RESP_SB_PATH_MTU_MTU_8192 (UINT32_C(0x5) << 12)
	+ uint8_t timeout;
	+ /* Timeout value for SWQEs. */
	+ uint8_t retry_cnt;
	+ /* Max retry count for WQEs. */
	+ uint8_t rnr_retry;
	+ /* Max RNR retry count for WQEs. */
	+ uint8_t min_rnr_timer;
	+ /* Min RNR timer that the QP will report to the remote. */
	+ uint32_t rq_psn;
	+ /* RQ start packet sequence number. */
	+ uint32_t sq_psn;
	+ /* SQ start packet sequence number. */
	+ uint8_t max_rd_atomic;
	+ /* Max outstanding RDMA read atomic. */
	+ uint8_t max_dest_rd_atomic;
	+ /* Max destination outstanding RDMA read atomic. */
	+ uint8_t tos_dscp_tos_ecn;
	+ /* IP TOS DSCP. */
	+ /* IP TOS ECN. */
	+ #define CREQ_QUERY_QP_RESP_SB_TOS_ECN_MASK UINT32_C(0x3)
	+ #define CREQ_QUERY_QP_RESP_SB_TOS_ECN_SFT 0
	+ /* IP TOS DSCP. */
	+ #define CREQ_QUERY_QP_RESP_SB_TOS_DSCP_MASK UINT32_C(0xfc)
	+ #define CREQ_QUERY_QP_RESP_SB_TOS_DSCP_SFT 2
	+ uint8_t enable_cc;
	+ /* enable_cc is 1 b */
	+ #define CREQ_QUERY_QP_RESP_SB_ENABLE_CC UINT32_C(0x1)
	+ #define CREQ_QUERY_QP_RESP_SB_RESERVED7_MASK UINT32_C(0xfe)
	+ #define CREQ_QUERY_QP_RESP_SB_RESERVED7_SFT 1
	+ uint32_t sq_size;
	+ /* Max send WQE. */
	+ uint32_t rq_size;
	+ /* Max recv WQE. */
	+ uint16_t sq_sge;
	+ /* Max send SGEs per SWQE. */
	+ uint16_t rq_sge;
	+ /* Max recv SGEs per RWQE (NOT SUPPORTED BY HARDWARE). */
	+ uint32_t max_inline_data;
	+ /* Max inline data length (upto 120 bytes). */
	+ uint32_t dest_qp_id;
	+ /* Destination QP id. */
	+ uint32_t unused_1;
	+ uint16_t src_mac[3];
	+ /* Source MAC. */
	+ uint16_t vlan_pcp_vlan_dei_vlan_id;
	+ /* VLAN PCP field - Priority Code Point. */
	+ /* Source VLAN id. */
	+ #define CREQ_QUERY_QP_RESP_SB_VLAN_ID_MASK UINT32_C(0xfff)
	+ #define CREQ_QUERY_QP_RESP_SB_VLAN_ID_SFT 0
	+ /* VLAN DEI field - Drop Eligibility Indicator. */
	+ #define CREQ_QUERY_QP_RESP_SB_VLAN_DEI UINT32_C(0x1000)
	+ /* VLAN PCP field - Priority Code Point. */
	+ #define CREQ_QUERY_QP_RESP_SB_VLAN_PCP_MASK UINT32_C(0xe000)
	+ #define CREQ_QUERY_QP_RESP_SB_VLAN_PCP_SFT 13
	+} __attribute__((packed));
	+
	+/* Create SRQ command response (16 bytes) */
	+
	+struct creq_create_srq_resp {
	+ uint8_t type;
	+ /*
	+ * This field indicates the exact type of the completion. By convention,
	+ * the LSB identifies the length of the record in 16B units. Even values
	+ * indicate 16B records. Odd values indicate 32B records.
	+ */
	+ #define CREQ_CREATE_SRQ_RESP_TYPE_MASK UINT32_C(0x3f)
	+ #define CREQ_CREATE_SRQ_RESP_TYPE_SFT 0
	+ /* QP Async Notification */
	+ #define CREQ_CREATE_SRQ_RESP_TYPE_QP_EVENT UINT32_C(0x38)
	+ #define CREQ_CREATE_SRQ_RESP_RESERVED2_MASK UINT32_C(0xc0)
	+ #define CREQ_CREATE_SRQ_RESP_RESERVED2_SFT 6
	+ uint8_t status;
	+ /* Status of the response. */
	+ uint16_t cookie;
	+ /* Driver supplied handle to associate the command and the response. */
	+ uint32_t xid;
	+ /* SRQ context id */
	+ uint8_t v;
	+ /*
	+ * This value is written by the NIC such that it will be different for
	+ * each pass through the completion queue. The even passes will write 1.
	+ * The odd passes will write 0.
	+ */
	+ #define CREQ_CREATE_SRQ_RESP_V UINT32_C(0x1)
	+ #define CREQ_CREATE_SRQ_RESP_RESERVED7_MASK UINT32_C(0xfe)
	+ #define CREQ_CREATE_SRQ_RESP_RESERVED7_SFT 1
	+ uint8_t event;
	+ /* Event or command opcode. */
	+ /* Create SRQ command response. */
	+ #define CREQ_CREATE_SRQ_RESP_EVENT_CREATE_SRQ UINT32_C(0x5)
	+ uint16_t reserved48[3];
	+} __attribute__((packed));
	+
	+/* Destroy SRQ command response (16 bytes) */
	+
	+struct creq_destroy_srq_resp {
	+ uint8_t type;
	+ /*
	+ * This field indicates the exact type of the completion. By convention,
	+ * the LSB identifies the length of the record in 16B units. Even values
	+ * indicate 16B records. Odd values indicate 32B records.
	+ */
	+ #define CREQ_DESTROY_SRQ_RESP_TYPE_MASK UINT32_C(0x3f)
	+ #define CREQ_DESTROY_SRQ_RESP_TYPE_SFT 0
	+ /* QP Async Notification */
	+ #define CREQ_DESTROY_SRQ_RESP_TYPE_QP_EVENT UINT32_C(0x38)
	+ #define CREQ_DESTROY_SRQ_RESP_RESERVED2_MASK UINT32_C(0xc0)
	+ #define CREQ_DESTROY_SRQ_RESP_RESERVED2_SFT 6
	+ uint8_t status;
	+ /* Status of the response. */
	+ uint16_t cookie;
	+ /* Driver supplied handle to associate the command and the response. */
	+ uint32_t xid;
	+ /* SRQ context id */
	+ uint8_t v;
	+ /*
	+ * This value is written by the NIC such that it will be different for
	+ * each pass through the completion queue. The even passes will write 1.
	+ * The odd passes will write 0.
	+ */
	+ #define CREQ_DESTROY_SRQ_RESP_V UINT32_C(0x1)
	+ #define CREQ_DESTROY_SRQ_RESP_RESERVED7_MASK UINT32_C(0xfe)
	+ #define CREQ_DESTROY_SRQ_RESP_RESERVED7_SFT 1
	+ uint8_t event;
	+ /* Event or command opcode. */
	+ /* Destroy SRQ command response. */
	+ #define CREQ_DESTROY_SRQ_RESP_EVENT_DESTROY_SRQ UINT32_C(0x6)
	+ uint16_t enable_for_arm[3];
	+ /* Set to 1 if this SRQ is allowed to be armed for threshold async event */
	+ #define CREQ_DESTROY_SRQ_RESP_ENABLE_FOR_ARM_MASK UINT32_C(0x30000)
	+ #define CREQ_DESTROY_SRQ_RESP_ENABLE_FOR_ARM_SFT 16
	+ #define CREQ_DESTROY_SRQ_RESP_RESERVED46_MASK UINT32_C(0xfffc0000)
	+ #define CREQ_DESTROY_SRQ_RESP_RESERVED46_SFT 18
	+} __attribute__((packed));
	+
	+/* Query SRQ command response (16 bytes) */
	+
	+struct creq_query_srq_resp {
	+ uint8_t type;
	+ /*
	+ * This field indicates the exact type of the completion. By convention,
	+ * the LSB identifies the length of the record in 16B units. Even values
	+ * indicate 16B records. Odd values indicate 32B records.
	+ */
	+ #define CREQ_QUERY_SRQ_RESP_TYPE_MASK UINT32_C(0x3f)
	+ #define CREQ_QUERY_SRQ_RESP_TYPE_SFT 0
	+ /* QP Async Notification */
	+ #define CREQ_QUERY_SRQ_RESP_TYPE_QP_EVENT UINT32_C(0x38)
	+ #define CREQ_QUERY_SRQ_RESP_RESERVED2_MASK UINT32_C(0xc0)
	+ #define CREQ_QUERY_SRQ_RESP_RESERVED2_SFT 6
	+ uint8_t status;
	+ /* Status of the response. */
	+ uint16_t cookie;
	+ /* Driver supplied handle to associate the command and the response. */
	+ uint32_t size;
	+ /* Side buffer size in 16-byte units */
	+ uint8_t v;
	+ /*
	+ * This value is written by the NIC such that it will be different for
	+ * each pass through the completion queue. The even passes will write 1.
	+ * The odd passes will write 0.
	+ */
	+ #define CREQ_QUERY_SRQ_RESP_V UINT32_C(0x1)
	+ #define CREQ_QUERY_SRQ_RESP_RESERVED7_MASK UINT32_C(0xfe)
	+ #define CREQ_QUERY_SRQ_RESP_RESERVED7_SFT 1
	+ uint8_t event;
	+ /* Event or command opcode. */
	+ /* Query SRQ command response. */
	+ #define CREQ_QUERY_SRQ_RESP_EVENT_QUERY_SRQ UINT32_C(0x8)
	+ uint16_t reserved48[3];
	+} __attribute__((packed));
	+
	+/* Query SRQ command response side buffer structure (24 bytes) */
	+
	+struct creq_query_srq_resp_sb {
	+ uint8_t opcode;
	+ /* Command opcode. */
	+ /* Query SRQ command response. */
	+ #define CREQ_QUERY_SRQ_RESP_SB_OPCODE_QUERY_SRQ UINT32_C(0x8)
	+ uint8_t status;
	+ /* Status of the response. */
	+ uint16_t cookie;
	+ /* Driver supplied handle to associate the command and the response. */
	+ uint16_t flags;
	+ /* Flags and attribs of the command. */
	+ uint8_t resp_size;
	+ /* Size of the response buffer in 16-byte units. */
	+ uint8_t reserved8;
	+ uint32_t xid;
	+ /* SRQ context id */
	+ uint16_t srq_limit;
	+ /* Watermark value to generate a SRQ limit event. */
	+ uint16_t reserved16;
	+ uint32_t data[4];
	+ /* data is 128 b */
	+} __attribute__((packed));
	+
	+/* Create CQ command Response (16 bytes) */
	+
	+struct creq_create_cq_resp {
	+ uint8_t type;
	+ /*
	+ * This field indicates the exact type of the completion. By convention,
	+ * the LSB identifies the length of the record in 16B units. Even values
	+ * indicate 16B records. Odd values indicate 32B records.
	+ */
	+ #define CREQ_CREATE_CQ_RESP_TYPE_MASK UINT32_C(0x3f)
	+ #define CREQ_CREATE_CQ_RESP_TYPE_SFT 0
	+ /* QP Async Notification */
	+ #define CREQ_CREATE_CQ_RESP_TYPE_QP_EVENT UINT32_C(0x38)
	+ #define CREQ_CREATE_CQ_RESP_RESERVED2_MASK UINT32_C(0xc0)
	+ #define CREQ_CREATE_CQ_RESP_RESERVED2_SFT 6
	+ uint8_t status;
	+ /* Status of the response. */
	+ uint16_t cookie;
	+ /* Driver supplied handle to associate the command and the response. */
	+ uint32_t xid;
	+ /* CQ context id */
	+ uint8_t v;
	+ /*
	+ * This value is written by the NIC such that it will be different for
	+ * each pass through the completion queue. The even passes will write 1.
	+ * The odd passes will write 0.
	+ */
	+ #define CREQ_CREATE_CQ_RESP_V UINT32_C(0x1)
	+ #define CREQ_CREATE_CQ_RESP_RESERVED7_MASK UINT32_C(0xfe)
	+ #define CREQ_CREATE_CQ_RESP_RESERVED7_SFT 1
	+ uint8_t event;
	+ /* Event or command opcode. */
	+ /* Create CQ command response. */
	+ #define CREQ_CREATE_CQ_RESP_EVENT_CREATE_CQ UINT32_C(0x9)
	+ uint16_t reserved48[3];
	+} __attribute__((packed));
	+
	+/* Destroy CQ command response (16 bytes) */
	+
	+struct creq_destroy_cq_resp {
	+ uint8_t type;
	+ /*
	+ * This field indicates the exact type of the completion. By convention,
	+ * the LSB identifies the length of the record in 16B units. Even values
	+ * indicate 16B records. Odd values indicate 32B records.
	+ */
	+ #define CREQ_DESTROY_CQ_RESP_TYPE_MASK UINT32_C(0x3f)
	+ #define CREQ_DESTROY_CQ_RESP_TYPE_SFT 0
	+ /* QP Async Notification */
	+ #define CREQ_DESTROY_CQ_RESP_TYPE_QP_EVENT UINT32_C(0x38)
	+ #define CREQ_DESTROY_CQ_RESP_RESERVED2_MASK UINT32_C(0xc0)
	+ #define CREQ_DESTROY_CQ_RESP_RESERVED2_SFT 6
	+ uint8_t status;
	+ /* Status of the response. */
	+ uint16_t cookie;
	+ /* Driver supplied handle to associate the command and the response. */
	+ uint32_t xid;
	+ /* CQ context id */
	+ uint8_t v;
	+ /*
	+ * This value is written by the NIC such that it will be different for
	+ * each pass through the completion queue. The even passes will write 1.
	+ * The odd passes will write 0.
	+ */
	+ #define CREQ_DESTROY_CQ_RESP_V UINT32_C(0x1)
	+ #define CREQ_DESTROY_CQ_RESP_RESERVED7_MASK UINT32_C(0xfe)
	+ #define CREQ_DESTROY_CQ_RESP_RESERVED7_SFT 1
	+ uint8_t event;
	+ /* Event or command opcode. */
	+ /* Destroy CQ command response. */
	+ #define CREQ_DESTROY_CQ_RESP_EVENT_DESTROY_CQ UINT32_C(0xa)
	+ uint16_t cq_arm_lvl;
	+ /*
	+ * CQ ARM Level: 0 ? Not Armed 1 ? Arm SE Only, Generate CNQE only for
	+ * incoming Solicted Events 2 ? Arm all, Generate CNQE for Rx and Tx
	+ */
	+ #define CREQ_DESTROY_CQ_RESP_CQ_ARM_LVL_MASK UINT32_C(0x3)
	+ #define CREQ_DESTROY_CQ_RESP_CQ_ARM_LVL_SFT 0
	+ #define CREQ_DESTROY_CQ_RESP_RESERVED14_MASK UINT32_C(0xfffc)
	+ #define CREQ_DESTROY_CQ_RESP_RESERVED14_SFT 2
	+ uint16_t total_cnq_events;
	+ /*
	+ * The total number of CNQ events for the CQ, incremented on each CNQ
	+ * event for the CQ (including firmware-generated CQ error
	+ * notification).
	+ */
	+ uint16_t reserved16;
	+} __attribute__((packed));
	+
	+/* Resize CQ command response (16 bytes) */
	+
	+struct creq_resize_cq_resp {
	+ uint8_t type;
	+ /*
	+ * This field indicates the exact type of the completion. By convention,
	+ * the LSB identifies the length of the record in 16B units. Even values
	+ * indicate 16B records. Odd values indicate 32B records.
	+ */
	+ #define CREQ_RESIZE_CQ_RESP_TYPE_MASK UINT32_C(0x3f)
	+ #define CREQ_RESIZE_CQ_RESP_TYPE_SFT 0
	+ /* QP Async Notification */
	+ #define CREQ_RESIZE_CQ_RESP_TYPE_QP_EVENT UINT32_C(0x38)
	+ #define CREQ_RESIZE_CQ_RESP_RESERVED2_MASK UINT32_C(0xc0)
	+ #define CREQ_RESIZE_CQ_RESP_RESERVED2_SFT 6
	+ uint8_t status;
	+ /* Status of the response. */
	+ uint16_t cookie;
	+ /* Driver supplied handle to associate the command and the response. */
	+ uint32_t xid;
	+ /* CQ context id */
	+ uint8_t v;
	+ /*
	+ * This value is written by the NIC such that it will be different for
	+ * each pass through the completion queue. The even passes will write 1.
	+ * The odd passes will write 0.
	+ */
	+ #define CREQ_RESIZE_CQ_RESP_V UINT32_C(0x1)
	+ #define CREQ_RESIZE_CQ_RESP_RESERVED7_MASK UINT32_C(0xfe)
	+ #define CREQ_RESIZE_CQ_RESP_RESERVED7_SFT 1
	+ uint8_t event;
	+ /* Event or command opcode. */
	+ /* Resize CQ command response. */
	+ #define CREQ_RESIZE_CQ_RESP_EVENT_RESIZE_CQ UINT32_C(0xc)
	+ uint16_t reserved48[3];
	+} __attribute__((packed));
	+
	+/* Allocate MRW command response (16 bytes) */
	+
	+struct creq_allocate_mrw_resp {
	+ uint8_t type;
	+ /*
	+ * This field indicates the exact type of the completion. By convention,
	+ * the LSB identifies the length of the record in 16B units. Even values
	+ * indicate 16B records. Odd values indicate 32B records.
	+ */
	+ #define CREQ_ALLOCATE_MRW_RESP_TYPE_MASK UINT32_C(0x3f)
	+ #define CREQ_ALLOCATE_MRW_RESP_TYPE_SFT 0
	+ /* QP Async Notification */
	+ #define CREQ_ALLOCATE_MRW_RESP_TYPE_QP_EVENT UINT32_C(0x38)
	+ #define CREQ_ALLOCATE_MRW_RESP_RESERVED2_MASK UINT32_C(0xc0)
	+ #define CREQ_ALLOCATE_MRW_RESP_RESERVED2_SFT 6
	+ uint8_t status;
	+ /* Status of the response. */
	+ uint16_t cookie;
	+ /* Driver supplied handle to associate the command and the response. */
	+ uint32_t xid;
	+ /* L_KEY for MR, R_KEY for MW */
	+ uint8_t v;
	+ /*
	+ * This value is written by the NIC such that it will be different for
	+ * each pass through the completion queue. The even passes will write 1.
	+ * The odd passes will write 0.
	+ */
	+ #define CREQ_ALLOCATE_MRW_RESP_V UINT32_C(0x1)
	+ #define CREQ_ALLOCATE_MRW_RESP_RESERVED7_MASK UINT32_C(0xfe)
	+ #define CREQ_ALLOCATE_MRW_RESP_RESERVED7_SFT 1
	+ uint8_t event;
	+ /* Event or command opcode. */
	+ /* Allocate MRW command response. */
	+ #define CREQ_ALLOCATE_MRW_RESP_EVENT_ALLOCATE_MRW UINT32_C(0xd)
	+ uint16_t reserved48[3];
	+} __attribute__((packed));
	+
	+/* De-allocate key command response (16 bytes) */
	+
	+struct creq_deallocate_key_resp {
	+ uint8_t type;
	+ /*
	+ * This field indicates the exact type of the completion. By convention,
	+ * the LSB identifies the length of the record in 16B units. Even values
	+ * indicate 16B records. Odd values indicate 32B records.
	+ */
	+ #define CREQ_DEALLOCATE_KEY_RESP_TYPE_MASK UINT32_C(0x3f)
	+ #define CREQ_DEALLOCATE_KEY_RESP_TYPE_SFT 0
	+ /* QP Async Notification */
	+ #define CREQ_DEALLOCATE_KEY_RESP_TYPE_QP_EVENT UINT32_C(0x38)
	+ #define CREQ_DEALLOCATE_KEY_RESP_RESERVED2_MASK UINT32_C(0xc0)
	+ #define CREQ_DEALLOCATE_KEY_RESP_RESERVED2_SFT 6
	+ uint8_t status;
	+ /* Status of the response. */
	+ uint16_t cookie;
	+ /* Driver supplied handle to associate the command and the response. */
	+ uint32_t xid;
	+ /* L_KEY for MR, R_KEY for MW */
	+ uint8_t v;
	+ /*
	+ * This value is written by the NIC such that it will be different for
	+ * each pass through the completion queue. The even passes will write 1.
	+ * The odd passes will write 0.
	+ */
	+ #define CREQ_DEALLOCATE_KEY_RESP_V UINT32_C(0x1)
	+ #define CREQ_DEALLOCATE_KEY_RESP_RESERVED7_MASK UINT32_C(0xfe)
	+ #define CREQ_DEALLOCATE_KEY_RESP_RESERVED7_SFT 1
	+ uint8_t event;
	+ /* Event or command opcode. */
	+ /* De-allocate key command response. */
	+ #define CREQ_DEALLOCATE_KEY_RESP_EVENT_DEALLOCATE_KEY UINT32_C(0xe)
	+ uint16_t reserved16;
	+ uint32_t bound_window_info;
	+ /*
	+ * This is advisory data to facilitate eventual descruction of lingering
	+ * memory regions in Windows. For memory window, it contains non-zero
	+ * HWID of a region this window was bound to (without the 8-bit key
	+ * portion). The host may check if the region is lingering in destroyed
	+ * state and try to destroy it now. For memory region, if deallocation
	+ * fails because there are windows bound to this region, this field will
	+ * contain approximate number of those windows. This number is read from
	+ * the context right before the deregistration is attempted and can
	+ * potentially be slightly different from the current number.
	+ */
	+} __attribute__((packed));
	+
	+/* Register MR command response (16 bytes) */
	+
	+struct creq_register_mr_resp {
	+ uint8_t type;
	+ /*
	+ * This field indicates the exact type of the completion. By convention,
	+ * the LSB identifies the length of the record in 16B units. Even values
	+ * indicate 16B records. Odd values indicate 32B records.
	+ */
	+ #define CREQ_REGISTER_MR_RESP_TYPE_MASK UINT32_C(0x3f)
	+ #define CREQ_REGISTER_MR_RESP_TYPE_SFT 0
	+ /* QP Async Notification */
	+ #define CREQ_REGISTER_MR_RESP_TYPE_QP_EVENT UINT32_C(0x38)
	+ #define CREQ_REGISTER_MR_RESP_RESERVED2_MASK UINT32_C(0xc0)
	+ #define CREQ_REGISTER_MR_RESP_RESERVED2_SFT 6
	+ uint8_t status;
	+ /* Status of the response. */
	+ uint16_t cookie;
	+ /* Driver supplied handle to associate the command and the response. */
	+ uint32_t xid;
	+ /* L_KEY */
	+ uint8_t v;
	+ /*
	+ * This value is written by the NIC such that it will be different for
	+ * each pass through the completion queue. The even passes will write 1.
	+ * The odd passes will write 0.
	+ */
	+ #define CREQ_REGISTER_MR_RESP_V UINT32_C(0x1)
	+ #define CREQ_REGISTER_MR_RESP_RESERVED7_MASK UINT32_C(0xfe)
	+ #define CREQ_REGISTER_MR_RESP_RESERVED7_SFT 1
	+ uint8_t event;
	+ /* Event or command opcode. */
	+ /* Register MR command response. */
	+ #define CREQ_REGISTER_MR_RESP_EVENT_REGISTER_MR UINT32_C(0xf)
	+ uint16_t reserved48[3];
	+} __attribute__((packed));
	+
	+/* Deregister MR command response (16 bytes) */
	+
	+struct creq_deregister_mr_resp {
	+ uint8_t type;
	+ /*
	+ * This field indicates the exact type of the completion. By convention,
	+ * the LSB identifies the length of the record in 16B units. Even values
	+ * indicate 16B records. Odd values indicate 32B records.
	+ */
	+ #define CREQ_DEREGISTER_MR_RESP_TYPE_MASK UINT32_C(0x3f)
	+ #define CREQ_DEREGISTER_MR_RESP_TYPE_SFT 0
	+ /* QP Async Notification */
	+ #define CREQ_DEREGISTER_MR_RESP_TYPE_QP_EVENT UINT32_C(0x38)
	+ #define CREQ_DEREGISTER_MR_RESP_RESERVED2_MASK UINT32_C(0xc0)
	+ #define CREQ_DEREGISTER_MR_RESP_RESERVED2_SFT 6
	+ uint8_t status;
	+ /* Status of the response. */
	+ uint16_t cookie;
	+ /* Driver supplied handle to associate the command and the response. */
	+ uint32_t xid;
	+ /* L_KEY */
	+ uint8_t v;
	+ /*
	+ * This value is written by the NIC such that it will be different for
	+ * each pass through the completion queue. The even passes will write 1.
	+ * The odd passes will write 0.
	+ */
	+ #define CREQ_DEREGISTER_MR_RESP_V UINT32_C(0x1)
	+ #define CREQ_DEREGISTER_MR_RESP_RESERVED7_MASK UINT32_C(0xfe)
	+ #define CREQ_DEREGISTER_MR_RESP_RESERVED7_SFT 1
	+ uint8_t event;
	+ /* Event or command opcode. */
	+ /* Deregister MR command response. */
	+ #define CREQ_DEREGISTER_MR_RESP_EVENT_DEREGISTER_MR UINT32_C(0x10)
	+ uint16_t reserved16;
	+ uint32_t bound_windows;
	+ /*
	+ * If deregister fails because there are windows bound to this region,
	+ * this field will contain approximate number of those windows. This
	+ * number is read from the context right before the deregistration is
	+ * attempted and can potentially be slightly different from the current
	+ * number.
	+ */
	+} __attribute__((packed));
	+
	+/* Add GID command response (16 bytes) */
	+
	+struct creq_add_gid_resp {
	+ uint8_t type;
	+ /*
	+ * This field indicates the exact type of the completion. By convention,
	+ * the LSB identifies the length of the record in 16B units. Even values
	+ * indicate 16B records. Odd values indicate 32B records.
	+ */
	+ #define CREQ_ADD_GID_RESP_TYPE_MASK UINT32_C(0x3f)
	+ #define CREQ_ADD_GID_RESP_TYPE_SFT 0
	+ /* QP Async Notification */
	+ #define CREQ_ADD_GID_RESP_TYPE_QP_EVENT UINT32_C(0x38)
	+ #define CREQ_ADD_GID_RESP_RESERVED2_MASK UINT32_C(0xc0)
	+ #define CREQ_ADD_GID_RESP_RESERVED2_SFT 6
	+ uint8_t status;
	+ /* Status of the response. */
	+ uint16_t cookie;
	+ /* Driver supplied handle to associate the command and the response. */
	+ uint32_t xid;
	+ /* GID index */
	+ uint8_t v;
	+ /*
	+ * This value is written by the NIC such that it will be different for
	+ * each pass through the completion queue. The even passes will write 1.
	+ * The odd passes will write 0.
	+ */
	+ #define CREQ_ADD_GID_RESP_V UINT32_C(0x1)
	+ #define CREQ_ADD_GID_RESP_RESERVED7_MASK UINT32_C(0xfe)
	+ #define CREQ_ADD_GID_RESP_RESERVED7_SFT 1
	+ uint8_t event;
	+ /* Event or command opcode. */
	+ /* Add GID command response. */
	+ #define CREQ_ADD_GID_RESP_EVENT_ADD_GID UINT32_C(0x11)
	+ uint16_t reserved48[3];
	+} __attribute__((packed));
	+
	+/* Delete GID command response (16 bytes) */
	+
	+struct creq_delete_gid_resp {
	+ uint8_t type;
	+ /*
	+ * This field indicates the exact type of the completion. By convention,
	+ * the LSB identifies the length of the record in 16B units. Even values
	+ * indicate 16B records. Odd values indicate 32B records.
	+ */
	+ #define CREQ_DELETE_GID_RESP_TYPE_MASK UINT32_C(0x3f)
	+ #define CREQ_DELETE_GID_RESP_TYPE_SFT 0
	+ /* QP Async Notification */
	+ #define CREQ_DELETE_GID_RESP_TYPE_QP_EVENT UINT32_C(0x38)
	+ #define CREQ_DELETE_GID_RESP_RESERVED2_MASK UINT32_C(0xc0)
	+ #define CREQ_DELETE_GID_RESP_RESERVED2_SFT 6
	+ uint8_t status;
	+ /* Status of the response. */
	+ uint16_t cookie;
	+ /* Driver supplied handle to associate the command and the response. */
	+ uint32_t xid;
	+ /* GID index */
	+ uint8_t v;
	+ /*
	+ * This value is written by the NIC such that it will be different for
	+ * each pass through the completion queue. The even passes will write 1.
	+ * The odd passes will write 0.
	+ */
	+ #define CREQ_DELETE_GID_RESP_V UINT32_C(0x1)
	+ #define CREQ_DELETE_GID_RESP_RESERVED7_MASK UINT32_C(0xfe)
	+ #define CREQ_DELETE_GID_RESP_RESERVED7_SFT 1
	+ uint8_t event;
	+ /* Event or command opcode. */
	+ /* Delete GID command response. */
	+ #define CREQ_DELETE_GID_RESP_EVENT_DELETE_GID UINT32_C(0x12)
	+ uint16_t reserved48[3];
	+} __attribute__((packed));
	+
	+/* Modify GID command response (16 bytes) */
	+
	+struct creq_modify_gid_resp {
	+ uint8_t type;
	+ /*
	+ * This field indicates the exact type of the completion. By convention,
	+ * the LSB identifies the length of the record in 16B units. Even values
	+ * indicate 16B records. Odd values indicate 32B records.
	+ */
	+ #define CREQ_MODIFY_GID_RESP_TYPE_MASK UINT32_C(0x3f)
	+ #define CREQ_MODIFY_GID_RESP_TYPE_SFT 0
	+ /* QP Async Notification */
	+ #define CREQ_MODIFY_GID_RESP_TYPE_QP_EVENT UINT32_C(0x38)
	+ #define CREQ_MODIFY_GID_RESP_RESERVED2_MASK UINT32_C(0xc0)
	+ #define CREQ_MODIFY_GID_RESP_RESERVED2_SFT 6
	+ uint8_t status;
	+ /* Status of the response. */
	+ uint16_t cookie;
	+ /* Driver supplied handle to associate the command and the response. */
	+ uint32_t xid;
	+ /* GID index */
	+ uint8_t v;
	+ /*
	+ * This value is written by the NIC such that it will be different for
	+ * each pass through the completion queue. The even passes will write 1.
	+ * The odd passes will write 0.
	+ */
	+ #define CREQ_MODIFY_GID_RESP_V UINT32_C(0x1)
	+ #define CREQ_MODIFY_GID_RESP_RESERVED7_MASK UINT32_C(0xfe)
	+ #define CREQ_MODIFY_GID_RESP_RESERVED7_SFT 1
	+ uint8_t event;
	+ /* Event or command opcode. */
	+ /* Add GID command response. */
	+ #define CREQ_MODIFY_GID_RESP_EVENT_ADD_GID UINT32_C(0x11)
	+ uint16_t reserved48[3];
	+} __attribute__((packed));
	+
	+/* Query GID command response (16 bytes) */
	+
	+struct creq_query_gid_resp {
	+ uint8_t type;
	+ /*
	+ * This field indicates the exact type of the completion. By convention,
	+ * the LSB identifies the length of the record in 16B units. Even values
	+ * indicate 16B records. Odd values indicate 32B records.
	+ */
	+ #define CREQ_QUERY_GID_RESP_TYPE_MASK UINT32_C(0x3f)
	+ #define CREQ_QUERY_GID_RESP_TYPE_SFT 0
	+ /* QP Async Notification */
	+ #define CREQ_QUERY_GID_RESP_TYPE_QP_EVENT UINT32_C(0x38)
	+ #define CREQ_QUERY_GID_RESP_RESERVED2_MASK UINT32_C(0xc0)
	+ #define CREQ_QUERY_GID_RESP_RESERVED2_SFT 6
	+ uint8_t status;
	+ /* Status of the response. */
	+ uint16_t cookie;
	+ /* Driver supplied handle to associate the command and the response. */
	+ uint32_t size;
	+ /* Side buffer size in 16-byte units */
	+ uint8_t v;
	+ /*
	+ * This value is written by the NIC such that it will be different for
	+ * each pass through the completion queue. The even passes will write 1.
	+ * The odd passes will write 0.
	+ */
	+ #define CREQ_QUERY_GID_RESP_V UINT32_C(0x1)
	+ #define CREQ_QUERY_GID_RESP_RESERVED7_MASK UINT32_C(0xfe)
	+ #define CREQ_QUERY_GID_RESP_RESERVED7_SFT 1
	+ uint8_t event;
	+ /* Event or command opcode. */
	+ /* Query GID command response. */
	+ #define CREQ_QUERY_GID_RESP_EVENT_QUERY_GID UINT32_C(0x18)
	+ uint16_t reserved48[3];
	+} __attribute__((packed));
	+
	+/* Query GID command response side buffer structure (40 bytes) */
	+
	+struct creq_query_gid_resp_sb {
	+ uint8_t opcode;
	+ /* Command opcode. */
	+ /* Query GID command response. */
	+ #define CREQ_QUERY_GID_RESP_SB_OPCODE_QUERY_GID UINT32_C(0x18)
	+ uint8_t status;
	+ /* Status of the response. */
	+ uint16_t cookie;
	+ /* Driver supplied handle to associate the command and the response. */
	+ uint16_t flags;
	+ /* Flags and attribs of the command. */
	+ uint8_t resp_size;
	+ /* Size of the response buffer in 16-byte units. */
	+ uint8_t reserved8;
	+ uint32_t gid[4];
	+ /* GID */
	+ uint16_t src_mac[3];
	+ /* Source MAC. */
	+ uint16_t vlan;
	+ /* flags. */
	+ /* Source VLAN id. */
	+ #define CREQ_QUERY_GID_RESP_SB_VLAN_VLAN_ID_MASK UINT32_C(0xfff)
	+ #define CREQ_QUERY_GID_RESP_SB_VLAN_VLAN_ID_SFT 0
	+ /* This set of bits select the TPID of the VLAN Tag. */
	+ #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_MASK UINT32_C(0x7000)
	+ #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_SFT 12
	+ /* TPID = 0x88A8. */
	+ #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_88A8 (UINT32_C(0x0) << 12)
	+ /* TPID = 0x8100. */
	+ #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_8100 (UINT32_C(0x1) << 12)
	+ /* TPID = 0x9100. */
	+ #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_9100 (UINT32_C(0x2) << 12)
	+ /* TPID = 0x9200. */
	+ #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_9200 (UINT32_C(0x3) << 12)
	+ /* TPID = 0x9300. */
	+ #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_9300 (UINT32_C(0x4) << 12)
	+ /* TPID = Configurable 1. */
	+ #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_CFG1 (UINT32_C(0x5) << 12)
	+ /* TPID = Configurable 2. */
	+ #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_CFG2 (UINT32_C(0x6) << 12)
	+ /* TPID = Configurable 3. */
	+ #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_CFG3 (UINT32_C(0x7) << 12)
	+ #define CREQ_QUERY_GID_RESP_SB_VLAN_TPID_LAST CREQ_QUERY_GID_RESP_SB_VLAN_TPID_TPID_CFG3
	+ /*
	+ * Setting this bit to 1 enables insertion of a VLAN Tag to a RoCE
	+ * header.
	+ */
	+ #define CREQ_QUERY_GID_RESP_SB_VLAN_VLAN_EN UINT32_C(0x8000)
	+ uint16_t ipid;
	+ /* Identifier field in the IP header. */
	+ uint16_t gid_index;
	+ /* GID index */
	+ uint32_t unused_0;
	+} __attribute__((packed));
	+
	+/* Create QP1 command response (16 bytes) */
	+
	+struct creq_create_qp1_resp {
	+ uint8_t type;
	+ /*
	+ * This field indicates the exact type of the completion. By convention,
	+ * the LSB identifies the length of the record in 16B units. Even values
	+ * indicate 16B records. Odd values indicate 32B records.
	+ */
	+ #define CREQ_CREATE_QP1_RESP_TYPE_MASK UINT32_C(0x3f)
	+ #define CREQ_CREATE_QP1_RESP_TYPE_SFT 0
	+ /* QP Async Notification */
	+ #define CREQ_CREATE_QP1_RESP_TYPE_QP_EVENT UINT32_C(0x38)
	+ #define CREQ_CREATE_QP1_RESP_RESERVED2_MASK UINT32_C(0xc0)
	+ #define CREQ_CREATE_QP1_RESP_RESERVED2_SFT 6
	+ uint8_t status;
	+ /* Status of the response. */
	+ uint16_t cookie;
	+ /* Driver supplied handle to associate the command and the response. */
	+ uint32_t xid;
	+ /* QP1 context id */
	+ uint8_t v;
	+ /*
	+ * This value is written by the NIC such that it will be different for
	+ * each pass through the completion queue. The even passes will write 1.
	+ * The odd passes will write 0.
	+ */
	+ #define CREQ_CREATE_QP1_RESP_V UINT32_C(0x1)
	+ #define CREQ_CREATE_QP1_RESP_RESERVED7_MASK UINT32_C(0xfe)
	+ #define CREQ_CREATE_QP1_RESP_RESERVED7_SFT 1
	+ uint8_t event;
	+ /* Event or command opcode. */
	+ /* Create QP1 command response. */
	+ #define CREQ_CREATE_QP1_RESP_EVENT_CREATE_QP1 UINT32_C(0x13)
	+ uint16_t reserved48[3];
	+} __attribute__((packed));
	+
	+/* Destroy QP1 command response (16 bytes) */
	+
	+struct creq_destroy_qp1_resp {
	+ uint8_t type;
	+ /*
	+ * This field indicates the exact type of the completion. By convention,
	+ * the LSB identifies the length of the record in 16B units. Even values
	+ * indicate 16B records. Odd values indicate 32B records.
	+ */
	+ #define CREQ_DESTROY_QP1_RESP_TYPE_MASK UINT32_C(0x3f)
	+ #define CREQ_DESTROY_QP1_RESP_TYPE_SFT 0
	+ /* QP Async Notification */
	+ #define CREQ_DESTROY_QP1_RESP_TYPE_QP_EVENT UINT32_C(0x38)
	+ #define CREQ_DESTROY_QP1_RESP_RESERVED2_MASK UINT32_C(0xc0)
	+ #define CREQ_DESTROY_QP1_RESP_RESERVED2_SFT 6
	+ uint8_t status;
	+ /* Status of the response. */
	+ uint16_t cookie;
	+ /* Driver supplied handle to associate the command and the response. */
	+ uint32_t xid;
	+ /* QP1 context id */
	+ uint8_t v;
	+ /*
	+ * This value is written by the NIC such that it will be different for
	+ * each pass through the completion queue. The even passes will write 1.
	+ * The odd passes will write 0.
	+ */
	+ #define CREQ_DESTROY_QP1_RESP_V UINT32_C(0x1)
	+ #define CREQ_DESTROY_QP1_RESP_RESERVED7_MASK UINT32_C(0xfe)
	+ #define CREQ_DESTROY_QP1_RESP_RESERVED7_SFT 1
	+ uint8_t event;
	+ /* Event or command opcode. */
	+ /* Destroy QP1 command response. */
	+ #define CREQ_DESTROY_QP1_RESP_EVENT_DESTROY_QP1 UINT32_C(0x14)
	+ uint16_t reserved48[3];
	+} __attribute__((packed));
	+
	+/* Create AH command response (16 bytes) */
	+
	+struct creq_create_ah_resp {
	+ uint8_t type;
	+ /*
	+ * This field indicates the exact type of the completion. By convention,
	+ * the LSB identifies the length of the record in 16B units. Even values
	+ * indicate 16B records. Odd values indicate 32B records.
	+ */
	+ #define CREQ_CREATE_AH_RESP_TYPE_MASK UINT32_C(0x3f)
	+ #define CREQ_CREATE_AH_RESP_TYPE_SFT 0
	+ /* QP Async Notification */
	+ #define CREQ_CREATE_AH_RESP_TYPE_QP_EVENT UINT32_C(0x38)
	+ #define CREQ_CREATE_AH_RESP_RESERVED2_MASK UINT32_C(0xc0)
	+ #define CREQ_CREATE_AH_RESP_RESERVED2_SFT 6
	+ uint8_t status;
	+ /* Status of the response. */
	+ uint16_t cookie;
	+ /* Driver supplied handle to associate the command and the response. */
	+ uint32_t xid;
	+ /* AH context id */
	+ uint8_t v;
	+ /*
	+ * This value is written by the NIC such that it will be different for
	+ * each pass through the completion queue. The even passes will write 1.
	+ * The odd passes will write 0.
	+ */
	+ #define CREQ_CREATE_AH_RESP_V UINT32_C(0x1)
	+ #define CREQ_CREATE_AH_RESP_RESERVED7_MASK UINT32_C(0xfe)
	+ #define CREQ_CREATE_AH_RESP_RESERVED7_SFT 1
	+ uint8_t event;
	+ /* Event or command opcode. */
	+ /* Create AH command response. */
	+ #define CREQ_CREATE_AH_RESP_EVENT_CREATE_AH UINT32_C(0x15)
	+ uint16_t reserved48[3];
	+} __attribute__((packed));
	+
	+/* Destroy AH command response (16 bytes) */
	+
	+struct creq_destroy_ah_resp {
	+ uint8_t type;
	+ /*
	+ * This field indicates the exact type of the completion. By convention,
	+ * the LSB identifies the length of the record in 16B units. Even values
	+ * indicate 16B records. Odd values indicate 32B records.
	+ */
	+ #define CREQ_DESTROY_AH_RESP_TYPE_MASK UINT32_C(0x3f)
	+ #define CREQ_DESTROY_AH_RESP_TYPE_SFT 0
	+ /* QP Async Notification */
	+ #define CREQ_DESTROY_AH_RESP_TYPE_QP_EVENT UINT32_C(0x38)
	+ #define CREQ_DESTROY_AH_RESP_RESERVED2_MASK UINT32_C(0xc0)
	+ #define CREQ_DESTROY_AH_RESP_RESERVED2_SFT 6
	+ uint8_t status;
	+ /* Status of the response. */
	+ uint16_t cookie;
	+ /* Driver supplied handle to associate the command and the response. */
	+ uint32_t xid;
	+ /* AH context id */
	+ uint8_t v;
	+ /*
	+ * This value is written by the NIC such that it will be different for
	+ * each pass through the completion queue. The even passes will write 1.
	+ * The odd passes will write 0.
	+ */
	+ #define CREQ_DESTROY_AH_RESP_V UINT32_C(0x1)
	+ #define CREQ_DESTROY_AH_RESP_RESERVED7_MASK UINT32_C(0xfe)
	+ #define CREQ_DESTROY_AH_RESP_RESERVED7_SFT 1
	+ uint8_t event;
	+ /* Event or command opcode. */
	+ /* Destroy AH command response. */
	+ #define CREQ_DESTROY_AH_RESP_EVENT_DESTROY_AH UINT32_C(0x16)
	+ uint16_t reserved48[3];
	+} __attribute__((packed));
	+
	+/* Initialize Firmware command response (16 bytes) */
	+
	+struct creq_initialize_fw_resp {
	+ uint8_t type;
	+ /*
	+ * This field indicates the exact type of the completion. By convention,
	+ * the LSB identifies the length of the record in 16B units. Even values
	+ * indicate 16B records. Odd values indicate 32B records.
	+ */
	+ #define CREQ_INITIALIZE_FW_RESP_TYPE_MASK UINT32_C(0x3f)
	+ #define CREQ_INITIALIZE_FW_RESP_TYPE_SFT 0
	+ /* QP Async Notification */
	+ #define CREQ_INITIALIZE_FW_RESP_TYPE_QP_EVENT UINT32_C(0x38)
	+ #define CREQ_INITIALIZE_FW_RESP_RESERVED2_MASK UINT32_C(0xc0)
	+ #define CREQ_INITIALIZE_FW_RESP_RESERVED2_SFT 6
	+ uint8_t status;
	+ /* Status of the response. */
	+ uint16_t cookie;
	+ /* Driver supplied handle to associate the command and the response. */
	+ uint32_t reserved32;
	+ uint8_t v;
	+ /*
	+ * This value is written by the NIC such that it will be different for
	+ * each pass through the completion queue. The even passes will write 1.
	+ * The odd passes will write 0.
	+ */
	+ #define CREQ_INITIALIZE_FW_RESP_V UINT32_C(0x1)
	+ #define CREQ_INITIALIZE_FW_RESP_RESERVED7_MASK UINT32_C(0xfe)
	+ #define CREQ_INITIALIZE_FW_RESP_RESERVED7_SFT 1
	+ uint8_t event;
	+ /* Event or command opcode. */
	+ /* Initialize firmware command response. */
	+ #define CREQ_INITIALIZE_FW_RESP_EVENT_INITIALIZE_FW UINT32_C(0x80)
	+ uint16_t reserved48[3];
	+} __attribute__((packed));
	+
	+/* De-initialize Firmware command response (16 bytes) */
	+
	+struct creq_deinitialize_fw_resp {
	+ uint8_t type;
	+ /*
	+ * This field indicates the exact type of the completion. By convention,
	+ * the LSB identifies the length of the record in 16B units. Even values
	+ * indicate 16B records. Odd values indicate 32B records.
	+ */
	+ #define CREQ_DEINITIALIZE_FW_RESP_TYPE_MASK UINT32_C(0x3f)
	+ #define CREQ_DEINITIALIZE_FW_RESP_TYPE_SFT 0
	+ /* QP Async Notification */
	+ #define CREQ_DEINITIALIZE_FW_RESP_TYPE_QP_EVENT UINT32_C(0x38)
	+ #define CREQ_DEINITIALIZE_FW_RESP_RESERVED2_MASK UINT32_C(0xc0)
	+ #define CREQ_DEINITIALIZE_FW_RESP_RESERVED2_SFT 6
	+ uint8_t status;
	+ /* Status of the response. */
	+ uint16_t cookie;
	+ /* Driver supplied handle to associate the command and the response. */
	+ uint32_t reserved32;
	+ uint8_t v;
	+ /*
	+ * This value is written by the NIC such that it will be different for
	+ * each pass through the completion queue. The even passes will write 1.
	+ * The odd passes will write 0.
	+ */
	+ #define CREQ_DEINITIALIZE_FW_RESP_V UINT32_C(0x1)
	+ #define CREQ_DEINITIALIZE_FW_RESP_RESERVED7_MASK UINT32_C(0xfe)
	+ #define CREQ_DEINITIALIZE_FW_RESP_RESERVED7_SFT 1
	+ uint8_t event;
	+ /* Event or command opcode. */
	+ /* De-initialize firmware command response. */
	+ #define CREQ_DEINITIALIZE_FW_RESP_EVENT_DEINITIALIZE_FW UINT32_C(0x81)
	+ uint16_t reserved48[3];
	+} __attribute__((packed));
	+
	+/* Stop function command response (16 bytes) */
	+
	+struct creq_stop_func_resp {
	+ uint8_t type;
	+ /*
	+ * This field indicates the exact type of the completion. By convention,
	+ * the LSB identifies the length of the record in 16B units. Even values
	+ * indicate 16B records. Odd values indicate 32B records.
	+ */
	+ #define CREQ_STOP_FUNC_RESP_TYPE_MASK UINT32_C(0x3f)
	+ #define CREQ_STOP_FUNC_RESP_TYPE_SFT 0
	+ /* QP Async Notification */
	+ #define CREQ_STOP_FUNC_RESP_TYPE_QP_EVENT UINT32_C(0x38)
	+ #define CREQ_STOP_FUNC_RESP_RESERVED2_MASK UINT32_C(0xc0)
	+ #define CREQ_STOP_FUNC_RESP_RESERVED2_SFT 6
	+ uint8_t status;
	+ /* Status of the response. */
	+ uint16_t cookie;
	+ /* Driver supplied handle to associate the command and the response. */
	+ uint32_t reserved32;
	+ uint8_t v;
	+ /*
	+ * This value is written by the NIC such that it will be different for
	+ * each pass through the completion queue. The even passes will write 1.
	+ * The odd passes will write 0.
	+ */
	+ #define CREQ_STOP_FUNC_RESP_V UINT32_C(0x1)
	+ #define CREQ_STOP_FUNC_RESP_RESERVED7_MASK UINT32_C(0xfe)
	+ #define CREQ_STOP_FUNC_RESP_RESERVED7_SFT 1
	+ uint8_t event;
	+ /* Event or command opcode. */
	+ /* Stop PF command response. */
	+ #define CREQ_STOP_FUNC_RESP_EVENT_STOP_FUNC UINT32_C(0x82)
	+ uint16_t reserved48[3];
	+} __attribute__((packed));
	+
	+/* Query function command response (16 bytes) */
	+
	+struct creq_query_func_resp {
	+ uint8_t type;
	+ /*
	+ * This field indicates the exact type of the completion. By convention,
	+ * the LSB identifies the length of the record in 16B units. Even values
	+ * indicate 16B records. Odd values indicate 32B records.
	+ */
	+ #define CREQ_QUERY_FUNC_RESP_TYPE_MASK UINT32_C(0x3f)
	+ #define CREQ_QUERY_FUNC_RESP_TYPE_SFT 0
	+ /* QP Async Notification */
	+ #define CREQ_QUERY_FUNC_RESP_TYPE_QP_EVENT UINT32_C(0x38)
	+ #define CREQ_QUERY_FUNC_RESP_RESERVED2_MASK UINT32_C(0xc0)
	+ #define CREQ_QUERY_FUNC_RESP_RESERVED2_SFT 6
	+ uint8_t status;
	+ /* Status of the response. */
	+ uint16_t cookie;
	+ /* Driver supplied handle to associate the command and the response. */
	+ uint32_t size;
	+ /* Side buffer size in 16-byte units */
	+ uint8_t v;
	+ /*
	+ * This value is written by the NIC such that it will be different for
	+ * each pass through the completion queue. The even passes will write 1.
	+ * The odd passes will write 0.
	+ */
	+ #define CREQ_QUERY_FUNC_RESP_V UINT32_C(0x1)
	+ #define CREQ_QUERY_FUNC_RESP_RESERVED7_MASK UINT32_C(0xfe)
	+ #define CREQ_QUERY_FUNC_RESP_RESERVED7_SFT 1
	+ uint8_t event;
	+ /* Event or command opcode. */
	+ /* Query info PF command response. */
	+ #define CREQ_QUERY_FUNC_RESP_EVENT_QUERY_FUNC UINT32_C(0x83)
	+ uint16_t reserved48[3];
	+} __attribute__((packed));
	+
	+/* Query function command response side buffer structure (88 bytes) */
	+
	+struct creq_query_func_resp_sb {
	+ uint8_t opcode;
	+ /* Command opcode. */
	+ /* Query info PF command response. */
	+ #define CREQ_QUERY_FUNC_RESP_SB_OPCODE_QUERY_FUNC UINT32_C(0x83)
	+ uint8_t status;
	+ /* Status of the response. */
	+ uint16_t cookie;
	+ /* Driver supplied handle to associate the command and the response. */
	+ uint16_t flags;
	+ /* Flags and attribs of the command. */
	+ uint8_t resp_size;
	+ /* Size of the response buffer in 16-byte units. */
	+ uint8_t reserved8;
	+ uint64_t max_mr_size;
	+ /* Max MR size supported. */
	+ uint32_t max_qp;
	+ /* Max QP supported. */
	+ uint16_t max_qp_wr;
	+ /* Max WQEs per QP. */
	+ uint16_t dev_cap_flags;
	+ /* Device capability flags. */
	+ /* Allow QP resizing. */
	+ #define CREQ_QUERY_FUNC_RESP_SB_DEV_CAP_FLAGS_RESIZE_QP UINT32_C(0x1)
	+ uint32_t max_cq;
	+ /* Max CQs supported. */
	+ uint32_t max_cqe;
	+ /* Max CQEs per CQ supported. */
	+ uint32_t max_pd;
	+ /* Max PDs supported. */
	+ uint8_t max_sge;
	+ /* Max SGEs per QP WQE supported. */
	+ uint8_t max_srq_sge;
	+ /* Max SGEs per SRQ WQE supported. */
	+ uint8_t max_qp_rd_atom;
	+ /* Max outstanding RDMA read & atomic supported. */
	+ uint8_t max_qp_init_rd_atom;
	+ /*
	+ * Max outstanding RDMA read & atomic that can be sent from an
	+ * initiator.
	+ */
	+ uint32_t max_mr;
	+ /* Max MRs supported. */
	+ uint32_t max_mw;
	+ /* Max MWs supported. */
	+ uint32_t max_raw_eth_qp;
	+ /* Max Raw Ethertype QPs supported. */
	+ uint32_t max_ah;
	+ /* Max AHs supported. */
	+ uint32_t max_fmr;
	+ /* Max FMRs supported. */
	+ uint32_t max_srq_wr;
	+ /* Max WQEs per SRQ supported. */
	+ uint32_t max_pkeys;
	+ /* Max PKEYs supported. */
	+ uint32_t max_inline_data;
	+ /* Max inline data supported. */
	+ uint8_t max_map_per_fmr;
	+ /* Max mappings per FMR supported. */
	+ uint8_t l2_db_space_size;
	+ /* L2 DB space size in pages. */
	+ uint16_t max_srq;
	+ /* Max SRQs supported. */
	+ uint32_t max_gid;
	+ /* Max GIDs supported. */
	+ uint32_t tqm_alloc_reqs[12];
	+ /*
	+ * An array of 48 8-bit values to specify allocation multiplier for TQM
	+ * host buffer regions. Each region occupies 16 MB of TQM PBL address
	+ * space: 0x00000000, 0x01000000, 0x02000000, etc. The host needs to
	+ * allocate (<Number of QPs>*multiplier, rounded up to page size) of
	+ * physical memory for non-zero slots and map the pages to the
	+ * corresponding 16MB regions. Typically there are total 3 non-zero
	+ * values in this array, their values are 16, 16, 12. Cu+ will only
	+ * populate up to index 11. SR may populate up to index 47.
	+ */
	+} __attribute__((packed));
	+
	+/* Set resources command response (16 bytes) */
	+
	+struct creq_set_func_resources_resp {
	+ uint8_t type;
	+ /*
	+ * This field indicates the exact type of the completion. By convention,
	+ * the LSB identifies the length of the record in 16B units. Even values
	+ * indicate 16B records. Odd values indicate 32B records.
	+ */
	+ #define CREQ_SET_FUNC_RESOURCES_RESP_TYPE_MASK UINT32_C(0x3f)
	+ #define CREQ_SET_FUNC_RESOURCES_RESP_TYPE_SFT 0
	+ /* QP Async Notification */
	+ #define CREQ_SET_FUNC_RESOURCES_RESP_TYPE_QP_EVENT UINT32_C(0x38)
	+ #define CREQ_SET_FUNC_RESOURCES_RESP_RESERVED2_MASK UINT32_C(0xc0)
	+ #define CREQ_SET_FUNC_RESOURCES_RESP_RESERVED2_SFT 6
	+ uint8_t status;
	+ /* Status of the response. */
	+ uint16_t cookie;
	+ /* Driver supplied handle to associate the command and the response. */
	+ uint32_t reserved32;
	+ uint8_t v;
	+ /*
	+ * This value is written by the NIC such that it will be different for
	+ * each pass through the completion queue. The even passes will write 1.
	+ * The odd passes will write 0.
	+ */
	+ #define CREQ_SET_FUNC_RESOURCES_RESP_V UINT32_C(0x1)
	+ #define CREQ_SET_FUNC_RESOURCES_RESP_RESERVED7_MASK UINT32_C(0xfe)
	+ #define CREQ_SET_FUNC_RESOURCES_RESP_RESERVED7_SFT 1
	+ uint8_t event;
	+ /* Event or command opcode. */
	+ /* Set function resources command response. */
	+ #define CREQ_SET_FUNC_RESOURCES_RESP_EVENT_SET_FUNC_RESOURCES UINT32_C(0x84)
	+ uint16_t reserved48[3];
	+} __attribute__((packed));
	+
	+/* Map TC to COS response (16 bytes) */
	+
	+struct creq_map_tc_to_cos_resp {
	+ uint8_t type;
	+ /*
	+ * This field indicates the exact type of the completion. By convention,
	+ * the LSB identifies the length of the record in 16B units. Even values
	+ * indicate 16B records. Odd values indicate 32B records.
	+ */
	+ #define CREQ_MAP_TC_TO_COS_RESP_TYPE_MASK UINT32_C(0x3f)
	+ #define CREQ_MAP_TC_TO_COS_RESP_TYPE_SFT 0
	+ /* QP Async Notification */
	+ #define CREQ_MAP_TC_TO_COS_RESP_TYPE_QP_EVENT UINT32_C(0x38)
	+ #define CREQ_MAP_TC_TO_COS_RESP_RESERVED2_MASK UINT32_C(0xc0)
	+ #define CREQ_MAP_TC_TO_COS_RESP_RESERVED2_SFT 6
	+ uint8_t status;
	+ /* Status of the response. */
	+ uint16_t cookie;
	+ /* Driver supplied handle to associate the command and the response. */
	+ uint32_t reserved32;
	+ uint8_t v;
	+ /*
	+ * This value is written by the NIC such that it will be different for
	+ * each pass through the completion queue. The even passes will write 1.
	+ * The odd passes will write 0.
	+ */
	+ #define CREQ_MAP_TC_TO_COS_RESP_V UINT32_C(0x1)
	+ #define CREQ_MAP_TC_TO_COS_RESP_RESERVED7_MASK UINT32_C(0xfe)
	+ #define CREQ_MAP_TC_TO_COS_RESP_RESERVED7_SFT 1
	+ uint8_t event;
	+ /* Event or command opcode. */
	+ /* Map TC to COS response. */
	+ #define CREQ_MAP_TC_TO_COS_RESP_EVENT_MAP_TC_TO_COS UINT32_C(0x8a)
	+ uint16_t reserved48[3];
	+} __attribute__((packed));
	+
	+/* Query version response (16 bytes) */
	+
	+struct creq_query_version_resp {
	+ uint8_t type;
	+ /*
	+ * This field indicates the exact type of the completion. By convention,
	+ * the LSB identifies the length of the record in 16B units. Even values
	+ * indicate 16B records. Odd values indicate 32B records.
	+ */
	+ #define CREQ_QUERY_VERSION_RESP_TYPE_MASK UINT32_C(0x3f)
	+ #define CREQ_QUERY_VERSION_RESP_TYPE_SFT 0
	+ /* QP Async Notification */
	+ #define CREQ_QUERY_VERSION_RESP_TYPE_QP_EVENT UINT32_C(0x38)
	+ #define CREQ_QUERY_VERSION_RESP_RESERVED2_MASK UINT32_C(0xc0)
	+ #define CREQ_QUERY_VERSION_RESP_RESERVED2_SFT 6
	+ uint8_t status;
	+ /* Status of the response. */
	+ uint16_t cookie;
	+ /* Driver supplied handle to associate the command and the response. */
	+ uint8_t fw_maj;
	+ /* firmware major version */
	+ uint8_t fw_minor;
	+ /* firmware minor version */
	+ uint8_t fw_bld;
	+ /* firmware build version */
	+ uint8_t fw_rsvd;
	+ /* firmware reserved version */
	+ uint8_t v;
	+ /*
	+ * This value is written by the NIC such that it will be different for
	+ * each pass through the completion queue. The even passes will write 1.
	+ * The odd passes will write 0.
	+ */
	+ #define CREQ_QUERY_VERSION_RESP_V UINT32_C(0x1)
	+ #define CREQ_QUERY_VERSION_RESP_RESERVED7_MASK UINT32_C(0xfe)
	+ #define CREQ_QUERY_VERSION_RESP_RESERVED7_SFT 1
	+ uint8_t event;
	+ /* Event or command opcode. */
	+ /* Query firmware and interface version response. */
	+ #define CREQ_QUERY_VERSION_RESP_EVENT_QUERY_VERSION UINT32_C(0x8b)
	+ uint16_t reserved16;
	+ uint8_t intf_maj;
	+ /* interface major version */
	+ uint8_t intf_minor;
	+ /* interface minor version */
	+ uint8_t intf_bld;
	+ /* interface build version */
	+ uint8_t intf_rsvd;
	+ /* interface reserved version */
	+} __attribute__((packed));
	+
	+/* Modify congestion control command response (16 bytes) */
	+
	+struct creq_modify_roce_cc_resp {
	+ uint8_t type;
	+ /*
	+ * This field indicates the exact type of the completion. By convention,
	+ * the LSB identifies the length of the record in 16B units. Even values
	+ * indicate 16B records. Odd values indicate 32B records.
	+ */
	+ #define CREQ_MODIFY_ROCE_CC_RESP_TYPE_MASK UINT32_C(0x3f)
	+ #define CREQ_MODIFY_ROCE_CC_RESP_TYPE_SFT 0
	+ /* QP Async Notification */
	+ #define CREQ_MODIFY_ROCE_CC_RESP_TYPE_QP_EVENT UINT32_C(0x38)
	+ #define CREQ_MODIFY_ROCE_CC_RESP_RESERVED2_MASK UINT32_C(0xc0)
	+ #define CREQ_MODIFY_ROCE_CC_RESP_RESERVED2_SFT 6
	+ uint8_t status;
	+ /* Status of the response. */
	+ uint16_t cookie;
	+ /* Driver supplied handle to associate the command and the response. */
	+ uint32_t reserved32;
	+ uint8_t v;
	+ /*
	+ * This value is written by the NIC such that it will be different for
	+ * each pass through the completion queue. The even passes will write 1.
	+ * The odd passes will write 0.
	+ */
	+ #define CREQ_MODIFY_ROCE_CC_RESP_V UINT32_C(0x1)
	+ #define CREQ_MODIFY_ROCE_CC_RESP_RESERVED7_MASK UINT32_C(0xfe)
	+ #define CREQ_MODIFY_ROCE_CC_RESP_RESERVED7_SFT 1
	+ uint8_t event;
	+ /* Event or command opcode. */
	+ /* Modify congestion control response. */
	+ #define CREQ_MODIFY_ROCE_CC_RESP_EVENT_MODIFY_ROCE_CC UINT32_C(0x8c)
	+ uint16_t reserved48[3];
	+} __attribute__((packed));
	+
	+/* Query congestion control command response (16 bytes) */
	+
	+struct creq_query_roce_cc_resp {
	+ uint8_t type;
	+ /*
	+ * This field indicates the exact type of the completion. By convention,
	+ * the LSB identifies the length of the record in 16B units. Even values
	+ * indicate 16B records. Odd values indicate 32B records.
	+ */
	+ #define CREQ_QUERY_ROCE_CC_RESP_TYPE_MASK UINT32_C(0x3f)
	+ #define CREQ_QUERY_ROCE_CC_RESP_TYPE_SFT 0
	+ /* QP Async Notification */
	+ #define CREQ_QUERY_ROCE_CC_RESP_TYPE_QP_EVENT UINT32_C(0x38)
	+ #define CREQ_QUERY_ROCE_CC_RESP_RESERVED2_MASK UINT32_C(0xc0)
	+ #define CREQ_QUERY_ROCE_CC_RESP_RESERVED2_SFT 6
	+ uint8_t status;
	+ /* Status of the response. */
	+ uint16_t cookie;
	+ /* Driver supplied handle to associate the command and the response. */
	+ uint32_t size;
	+ /* Side buffer size in 16-byte units */
	+ uint8_t v;
	+ /*
	+ * This value is written by the NIC such that it will be different for
	+ * each pass through the completion queue. The even passes will write 1.
	+ * The odd passes will write 0.
	+ */
	+ #define CREQ_QUERY_ROCE_CC_RESP_V UINT32_C(0x1)
	+ #define CREQ_QUERY_ROCE_CC_RESP_RESERVED7_MASK UINT32_C(0xfe)
	+ #define CREQ_QUERY_ROCE_CC_RESP_RESERVED7_SFT 1
	+ uint8_t event;
	+ /* Event or command opcode. */
	+ /* Query congestion control response. */
	+ #define CREQ_QUERY_ROCE_CC_RESP_EVENT_QUERY_ROCE_CC UINT32_C(0x8d)
	+ uint16_t reserved48[3];
	+} __attribute__((packed));
	+
	+/* Query congestion control command response side buffer structure (32 bytes) */
	+
	+struct creq_query_roce_cc_resp_sb {
	+ uint8_t opcode;
	+ /* Command opcode. */
	+ /* Query congestion control response. */
	+ #define CREQ_QUERY_ROCE_CC_RESP_SB_OPCODE_QUERY_ROCE_CC UINT32_C(0x8d)
	+ uint8_t status;
	+ /* Status of the response. */
	+ uint16_t cookie;
	+ /* Driver supplied handle to associate the command and the response. */
	+ uint16_t flags;
	+ /* Flags and attribs of the command. */
	+ uint8_t resp_size;
	+ /* Size of the response buffer in 16-byte units. */
	+ uint8_t reserved8;
	+ uint8_t enable_cc;
	+ /* unused7 is 7 b */
	+ /* Enable. */
	+ #define CREQ_QUERY_ROCE_CC_RESP_SB_ENABLE_CC UINT32_C(0x1)
	+ /* unused7 is 7 b */
	+ uint8_t tos_dscp_tos_ecn;
	+ /* IP TOS DSCP. */
	+ /* IP TOS ECN. */
	+ #define CREQ_QUERY_ROCE_CC_RESP_SB_TOS_ECN_MASK UINT32_C(0x3)
	+ #define CREQ_QUERY_ROCE_CC_RESP_SB_TOS_ECN_SFT 0
	+ /* IP TOS DSCP. */
	+ #define CREQ_QUERY_ROCE_CC_RESP_SB_TOS_DSCP_MASK UINT32_C(0xfc)
	+ #define CREQ_QUERY_ROCE_CC_RESP_SB_TOS_DSCP_SFT 2
	+ uint8_t g;
	+ /* unused5 is 5 b */
	+ /* Congestion Probability averaging factor. */
	+ #define CREQ_QUERY_ROCE_CC_RESP_SB_G_MASK UINT32_C(0x7)
	+ #define CREQ_QUERY_ROCE_CC_RESP_SB_G_SFT 0
	+ /* unused5 is 5 b */
	+ uint8_t num_phases_per_state;
	+ /* Number of phases in Fast Recovery and Active Increase. */
	+ uint16_t init_cr;
	+ /* The starting value of rate. */
	+ uint16_t init_tr;
	+ /* The starting value of target rate. */
	+ uint8_t alt_vlan_pcp;
	+ /* rsvd1 is 5 b */
	+ /* Alternate vlan pcp value for CNP packets. */
	+ #define CREQ_QUERY_ROCE_CC_RESP_SB_ALT_VLAN_PCP_MASK UINT32_C(0x7)
	+ #define CREQ_QUERY_ROCE_CC_RESP_SB_ALT_VLAN_PCP_SFT 0
	+ /* rsvd1 is 5 b */
	+ #define CREQ_QUERY_ROCE_CC_RESP_SB_RSVD1_MASK UINT32_C(0xf8)
	+ #define CREQ_QUERY_ROCE_CC_RESP_SB_RSVD1_SFT 3
	+ uint8_t alt_tos_dscp;
	+ /* rsvd4 is 2 b */
	+ /* Alternate IP TOS DSCP. */
	+ #define CREQ_QUERY_ROCE_CC_RESP_SB_ALT_TOS_DSCP_MASK UINT32_C(0x3f)
	+ #define CREQ_QUERY_ROCE_CC_RESP_SB_ALT_TOS_DSCP_SFT 0
	+ /* rsvd4 is 2 b */
	+ #define CREQ_QUERY_ROCE_CC_RESP_SB_RSVD4_MASK UINT32_C(0xc0)
	+ #define CREQ_QUERY_ROCE_CC_RESP_SB_RSVD4_SFT 6
	+ uint8_t cc_mode;
	+ /* rsvd2 is 7 b */
	+ /* 0 for DCTCP , 1 for TCP */
	+ #define CREQ_QUERY_ROCE_CC_RESP_SB_CC_MODE UINT32_C(0x1)
	+ /* rsvd2 is 7 b */
	+ #define CREQ_QUERY_ROCE_CC_RESP_SB_RSVD2_MASK UINT32_C(0xfe)
	+ #define CREQ_QUERY_ROCE_CC_RESP_SB_RSVD2_SFT 1
	+ uint8_t tx_queue;
	+ /* rsvd3 is 6 b */
	+ /* Specifies the RoCE Tx Queue ( o to 3) to use for sending CNP packets */
	+ #define CREQ_QUERY_ROCE_CC_RESP_SB_TX_QUEUE_MASK UINT32_C(0x3)
	+ #define CREQ_QUERY_ROCE_CC_RESP_SB_TX_QUEUE_SFT 0
	+ /* rsvd3 is 6 b */
	+ #define CREQ_QUERY_ROCE_CC_RESP_SB_RSVD3_MASK UINT32_C(0xfc)
	+ #define CREQ_QUERY_ROCE_CC_RESP_SB_RSVD3_SFT 2
	+ uint16_t rtt;
	+ /* rsvd5 is 2 b */
	+ /* Round trip time in units of usecs */
	+ #define CREQ_QUERY_ROCE_CC_RESP_SB_RTT_MASK UINT32_C(0x3fff)
	+ #define CREQ_QUERY_ROCE_CC_RESP_SB_RTT_SFT 0
	+ /* rsvd5 is 2 b */
	+ #define CREQ_QUERY_ROCE_CC_RESP_SB_RSVD5_MASK UINT32_C(0xc000)
	+ #define CREQ_QUERY_ROCE_CC_RESP_SB_RSVD5_SFT 14
	+ uint16_t tcp_cp;
	+ /* rsvd6 is 6 b */
	+ /* The value used as CP when cc_mode is 1(TCP) */
	+ #define CREQ_QUERY_ROCE_CC_RESP_SB_TCP_CP_MASK UINT32_C(0x3ff)
	+ #define CREQ_QUERY_ROCE_CC_RESP_SB_TCP_CP_SFT 0
	+ /* rsvd6 is 6 b */
	+ #define CREQ_QUERY_ROCE_CC_RESP_SB_RSVD6_MASK UINT32_C(0xfc00)
	+ #define CREQ_QUERY_ROCE_CC_RESP_SB_RSVD6_SFT 10
	+ uint16_t inactivity_th;
	+ /* Inactivity time after which QP CC parameters are initialized */
	+ uint16_t reserved16;
	+ uint32_t reserved32;
	+} __attribute__((packed));
	+
	+/* QP error notification event (16 bytes) */
	+
	+struct creq_qp_error_notification {
	+ uint8_t type;
	+ /*
	+ * This field indicates the exact type of the completion. By convention,
	+ * the LSB identifies the length of the record in 16B units. Even values
	+ * indicate 16B records. Odd values indicate 32B records.
	+ */
	+ #define CREQ_QP_ERROR_NOTIFICATION_TYPE_MASK UINT32_C(0x3f)
	+ #define CREQ_QP_ERROR_NOTIFICATION_TYPE_SFT 0
	+ /* QP Async Notification */
	+ #define CREQ_QP_ERROR_NOTIFICATION_TYPE_QP_EVENT UINT32_C(0x38)
	+ #define CREQ_QP_ERROR_NOTIFICATION_RESERVED2_MASK UINT32_C(0xc0)
	+ #define CREQ_QP_ERROR_NOTIFICATION_RESERVED2_SFT 6
	+ uint8_t status;
	+ /* Status of the response. */
	+ uint8_t req_slow_path_state;
	+ /* requestor slow path state */
	+ uint8_t req_err_state_reason;
	+ /* requestor error reason */
	+ uint32_t xid;
	+ /* QP context id */
	+ uint8_t v;
	+ /*
	+ * This value is written by the NIC such that it will be different for
	+ * each pass through the completion queue. The even passes will write 1.
	+ * The odd passes will write 0.
	+ */
	+ #define CREQ_QP_ERROR_NOTIFICATION_V UINT32_C(0x1)
	+ #define CREQ_QP_ERROR_NOTIFICATION_RESERVED7_MASK UINT32_C(0xfe)
	+ #define CREQ_QP_ERROR_NOTIFICATION_RESERVED7_SFT 1
	+ uint8_t event;
	+ /* Event or command opcode. */
	+ /* QP error notification event. */
	+ #define CREQ_QP_ERROR_NOTIFICATION_EVENT_QP_ERROR_NOTIFICATION UINT32_C(0xc0)
	+ uint8_t res_slow_path_state;
	+ /* responder slow path state */
	+ uint8_t res_err_state_reason;
	+ uint16_t sq_cons_idx;
	+ /*
	+ * Final SQ Consumer Index value. Any additional SQ WQEs will have to be
	+ * completed by the user provider.
	+ */
	+ uint16_t rq_cons_idx;
	+ /*
	+ * Final RQ Consumer Index value. Any additional RQ WQEs will have to be
	+ * completed by the user provider.
	+ */
	+} __attribute__((packed));
	+
	+/* RoCE Slowpath Data Structures */
	+/*
	+ * Note: This section documents the Host Structures used between software and
	+ * RoCE control firmware.
	+ */
	+/* hwrm_selftest_qlist */
	+/*
	+ * Description: This function is called by a driver to determine which selftests
	+ * are available to be run against the requested function.
	+ */
	+/* Input (16 bytes) */
	+
	+struct hwrm_selftest_qlist_input {
	+ uint16_t req_type;
	+ /*
	+ * This value indicates what type of request this is. The format for the
	+ * rest of the command is determined by this field.
	+ */
	+ uint16_t cmpl_ring;
	+ /*
	+ * This value indicates the what completion ring the request will be
	+ * optionally completed on. If the value is -1, then no CR completion
	+ * will be generated. Any other value must be a valid CR ring_id value
	+ * for this function.
	+ */
	+ uint16_t seq_id;
	+ /* This value indicates the command sequence number. */
	+ uint16_t target_id;
	+ /*
	+ * Target ID of this command. 0x0 - 0xFFF8 - Used for function ids
	+ * 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF - HWRM
	+ */
	+ uint64_t resp_addr;
	+ /*
	+ * This is the host address where the response will be written when the
	+ * request is complete. This area must be 16B aligned and must be
	+ * cleared to zero before the request is made.
	+ */
	+} __attribute__((packed));
	+
	+/* Output (272 bytes) */
	+
	+struct hwrm_selftest_qlist_output {
	+ uint16_t error_code;
	+ /*
	+ * Pass/Fail or error type Note: receiver to verify the in parameters,
	+ * and fail the call with an error when appropriate
	+ */
	+ uint16_t req_type;
	+ /* This field returns the type of original request. */
	+ uint16_t seq_id;
	+ /* This field provides original sequence number of the command. */
	+ uint16_t resp_len;
	+ /*
	+ * This field is the length of the response in bytes. The last byte of
	+ * the response is a valid flag that will read as '1' when the command
	+ * has been completely written to memory.
	+ */
	+ uint8_t num_tests;
	+ /*
	+ * This field represents the number of tests available to be requested
	+ * by a driver.
	+ */
	+ uint8_t available_tests;
	+ /* This field indicates which self-test is available to be run. */
	+ /* Can run the NVM test. */
	+ #define HWRM_SELFTEST_QLIST_OUTPUT_AVAILABLE_TESTS_NVM_TEST UINT32_C(0x1)
	+ /* Can run the link test. */
	+ #define HWRM_SELFTEST_QLIST_OUTPUT_AVAILABLE_TESTS_LINK_TEST UINT32_C(0x2)
	+ /* Can run the register test. */
	+ #define HWRM_SELFTEST_QLIST_OUTPUT_AVAILABLE_TESTS_REGISTER_TEST UINT32_C(0x4)
	+ /* Can run the memory test. */
	+ #define HWRM_SELFTEST_QLIST_OUTPUT_AVAILABLE_TESTS_MEMORY_TEST UINT32_C(0x8)
	+ /* Can run the PCIe serdes test. */
	+ #define HWRM_SELFTEST_QLIST_OUTPUT_AVAILABLE_TESTS_PCIE_SERDES_TEST UINT32_C(0x10)
	+ /* Can run the Ethernet serdes test. */
	+ #define HWRM_SELFTEST_QLIST_OUTPUT_AVAILABLE_TESTS_ETHERNET_SERDES_TEST UINT32_C(0x20)
	+ uint8_t offline_tests;
	+ /* The NVM test is an offline test. */
	+ #define HWRM_SELFTEST_QLIST_OUTPUT_OFFLINE_TESTS_NVM_TEST UINT32_C(0x1)
	+ /* The link test is an offline test. */
	+ #define HWRM_SELFTEST_QLIST_OUTPUT_OFFLINE_TESTS_LINK_TEST UINT32_C(0x2)
	+ /* The register test is an offline test. */
	+ #define HWRM_SELFTEST_QLIST_OUTPUT_OFFLINE_TESTS_REGISTER_TEST UINT32_C(0x4)
	+ /* The memory test is an offline test. */
	+ #define HWRM_SELFTEST_QLIST_OUTPUT_OFFLINE_TESTS_MEMORY_TEST UINT32_C(0x8)
	+ /* The PCIe serdes test is an offline test. */
	+ #define HWRM_SELFTEST_QLIST_OUTPUT_OFFLINE_TESTS_PCIE_SERDES_TEST UINT32_C(0x10)
	+ /* The Ethernet serdes test is an offline test. */
	+ #define HWRM_SELFTEST_QLIST_OUTPUT_OFFLINE_TESTS_ETHERNET_SERDES_TEST UINT32_C(0x20)
	+ uint8_t unused_0;
	+ uint16_t test_timeout;
	+ /*
	+ * This field represents the the maximum timeout for all the tests to
	+ * complete in milliseconds.
	+ */
	+ uint8_t unused_1;
	+ uint8_t unused_2;
	+ char test0_name[32];
	+ /*
	+ * This field represents the name of the NVM test (ASCII chars with NULL
	+ * at the end).
	+ */
	+ char test1_name[32];
	+ /*
	+ * This field represents the name of the link test (ASCII chars with
	+ * NULL at the end).
	+ */
	+ char test2_name[32];
	+ /*
	+ * This field represents the name of the register test (ASCII chars with
	+ * NULL at the end).
	+ */
	+ char test3_name[32];
	+ /*
	+ * This field represents the name of the memory test (ASCII chars with
	+ * NULL at the end).
	+ */
	+ char test4_name[32];
	+ /*
	+ * This field represents the name of the PCIe serdes test (ASCII chars
	+ * with NULL at the end).
	+ */
	+ char test5_name[32];
	+ /*
	+ * This field represents the name of the Ethernet serdes test (ASCII
	+ * chars with NULL at the end).
	+ */
	+ char test6_name[32];
	+ /*
	+ * This field represents the name of some future test (ASCII chars with
	+ * NULL at the end).
	+ */
	+ char test7_name[32];
	+ /*
	+ * This field represents the name of some future test (ASCII chars with
	+ * NULL at the end).
	+ */
	+} __attribute__((packed));
	+
	+/* hwrm_selftest_exec */
	+/*
	+ * Description: This function is called by a driver to request which self tests
	+ * are to be run.
	+ */
	+/* Input (24 bytes) */
	+
	+struct hwrm_selftest_exec_input {
	+ uint16_t req_type;
	+ /*
	+ * This value indicates what type of request this is. The format for the
	+ * rest of the command is determined by this field.
	+ */
	+ uint16_t cmpl_ring;
	+ /*
	+ * This value indicates the what completion ring the request will be
	+ * optionally completed on. If the value is -1, then no CR completion
	+ * will be generated. Any other value must be a valid CR ring_id value
	+ * for this function.
	+ */
	+ uint16_t seq_id;
	+ /* This value indicates the command sequence number. */
	+ uint16_t target_id;
	+ /*
	+ * Target ID of this command. 0x0 - 0xFFF8 - Used for function ids
	+ * 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF - HWRM
	+ */
	+ uint64_t resp_addr;
	+ /*
	+ * This is the host address where the response will be written when the
	+ * request is complete. This area must be 16B aligned and must be
	+ * cleared to zero before the request is made.
	+ */
	+ uint8_t flags;
	+ /* This field indicates which self-test is being requested to run. */
	+ /* Run the NVM test. */
	+ #define HWRM_SELFTEST_EXEC_INPUT_FLAGS_NVM_TEST UINT32_C(0x1)
	+ /* Run the link test. */
	+ #define HWRM_SELFTEST_EXEC_INPUT_FLAGS_LINK_TEST UINT32_C(0x2)
	+ /* Run the register test. */
	+ #define HWRM_SELFTEST_EXEC_INPUT_FLAGS_REGISTER_TEST UINT32_C(0x4)
	+ /* Run the memory test. */
	+ #define HWRM_SELFTEST_EXEC_INPUT_FLAGS_MEMORY_TEST UINT32_C(0x8)
	+ /* Run the PCIe serdes test. */
	+ #define HWRM_SELFTEST_EXEC_INPUT_FLAGS_PCIE_SERDES_TEST UINT32_C(0x10)
	+ /* Run the Ethernet serdes test. */
	+ #define HWRM_SELFTEST_EXEC_INPUT_FLAGS_ETHERNET_SERDES_TEST UINT32_C(0x20)
	+ uint8_t unused_0[7];
	+} __attribute__((packed));
	+
	+/* Output (16 bytes) */
	+
	+struct hwrm_selftest_exec_output {
	+ uint16_t error_code;
	+ /*
	+ * Pass/Fail or error type Note: receiver to verify the in parameters,
	+ * and fail the call with an error when appropriate
	+ */
	+ uint16_t req_type;
	+ /* This field returns the type of original request. */
	+ uint16_t seq_id;
	+ /* This field provides original sequence number of the command. */
	+ uint16_t resp_len;
	+ /*
	+ * This field is the length of the response in bytes. The last byte of
	+ * the response is a valid flag that will read as '1' when the command
	+ * has been completely written to memory.
	+ */
	+ uint8_t requested_tests;
	+ /* The following tests were requested to be run. */
	+ /* A reqeust was made to run the NVM test. */
	+ #define HWRM_SELFTEST_EXEC_OUTPUT_REQUESTED_TESTS_NVM_TEST UINT32_C(0x1)
	+ /* A request was made to run the link test. */
	+ #define HWRM_SELFTEST_EXEC_OUTPUT_REQUESTED_TESTS_LINK_TEST UINT32_C(0x2)
	+ /* A request was made to run the register test. */
	+ #define HWRM_SELFTEST_EXEC_OUTPUT_REQUESTED_TESTS_REGISTER_TEST UINT32_C(0x4)
	+ /* A request was made to run the memory test. */
	+ #define HWRM_SELFTEST_EXEC_OUTPUT_REQUESTED_TESTS_MEMORY_TEST UINT32_C(0x8)
	+ /* A request was made to run the PCIe serdes test. */
	+ #define HWRM_SELFTEST_EXEC_OUTPUT_REQUESTED_TESTS_PCIE_SERDES_TEST UINT32_C(0x10)
	+ /* A request was made to run the Ethernet serdes test. */
	+ #define HWRM_SELFTEST_EXEC_OUTPUT_REQUESTED_TESTS_ETHERNET_SERDES_TEST UINT32_C(0x20)
	+ uint8_t test_success;
	+ /*
	+ * If a test was requested to be run as seen in the requested_tests
	+ * field, this bit indicates whether the test was successful(1) or
	+ * failed(0).
	+ */
	+ /*
	+ * If requested, a value of 1 indicates the NVM test completed
	+ * successfully.
	+ */
	+ #define HWRM_SELFTEST_EXEC_OUTPUT_TEST_SUCCESS_NVM_TEST UINT32_C(0x1)
	+ /*
	+ * If requested, a value of 1 indicates the link test completed
	+ * successfully.
	+ */
	+ #define HWRM_SELFTEST_EXEC_OUTPUT_TEST_SUCCESS_LINK_TEST UINT32_C(0x2)
	+ /*
	+ * If requested, a value of 1 indicates the register test completed
	+ * successfully.
	+ */
	+ #define HWRM_SELFTEST_EXEC_OUTPUT_TEST_SUCCESS_REGISTER_TEST UINT32_C(0x4)
	+ /*
	+ * If requested, a value of 1 indicates the memory test completed
	+ * successfully.
	+ */
	+ #define HWRM_SELFTEST_EXEC_OUTPUT_TEST_SUCCESS_MEMORY_TEST UINT32_C(0x8)
	+ /*
	+ * If requested, a value of 1 indicates the PCIe serdes test completed
	+ * successfully.
	+ */
	+ #define HWRM_SELFTEST_EXEC_OUTPUT_TEST_SUCCESS_PCIE_SERDES_TEST UINT32_C(0x10)
	+ /*
	+ * If requested, a value of 1 indicates the Ethernet serdes test
	+ * completed successfully.
	+ */
	+ #define HWRM_SELFTEST_EXEC_OUTPUT_TEST_SUCCESS_ETHERNET_SERDES_TEST UINT32_C(0x20)
	+ uint16_t unused_0[3];
	+} __attribute__((packed));
	+
	+/* hwrm_selftest_irq */
	+/*
	+ * Description: This function is called by a driver to request the interrupt
	+ * test be run. In response to this request the interrupt associated with the
	+ * completion ring specified in the cmpl_ring field will be asserted to the
	+ * host.
	+ */
	+/* Input (16 bytes) */
	+
	+struct hwrm_selftest_irq_input {
	+ uint16_t req_type;
	+ /*
	+ * This value indicates what type of request this is. The format for the
	+ * rest of the command is determined by this field.
	+ */
	+ uint16_t cmpl_ring;
	+ /*
	+ * This value indicates the what completion ring the request will be
	+ * optionally completed on. If the value is -1, then no CR completion
	+ * will be generated. Any other value must be a valid CR ring_id value
	+ * for this function.
	+ */
	+ uint16_t seq_id;
	+ /* This value indicates the command sequence number. */
	+ uint16_t target_id;
	+ /*
	+ * Target ID of this command. 0x0 - 0xFFF8 - Used for function ids
	+ * 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF - HWRM
	+ */
	+ uint64_t resp_addr;
	+ /*
	+ * This is the host address where the response will be written when the
	+ * request is complete. This area must be 16B aligned and must be
	+ * cleared to zero before the request is made.
	+ */
	+} __attribute__((packed));
	+
	+/* Output (8 bytes) */
	+
	+struct hwrm_selftest_irq_output {
	+ uint16_t error_code;
	+ /*
	+ * Pass/Fail or error type Note: receiver to verify the in parameters,
	+ * and fail the call with an error when appropriate
	+ */
	+ uint16_t req_type;
	+ /* This field returns the type of original request. */
	+ uint16_t seq_id;
	+ /* This field provides original sequence number of the command. */
	+ uint16_t resp_len;
	+ /*
	+ * This field is the length of the response in bytes. The last byte of
	+ * the response is a valid flag that will read as '1' when the command
	+ * has been completely written to memory.
	+ */
	+} __attribute__((packed));
	+
	+/* hwrm_selftest_retreive_serdes_data */
	+/*
	+ * Description: This function is called by a driver to retreieve the data
	+ * collected when running the previous PCIe or Ethernet serdes test. The driver
	+ * can use multiple calls to this command to retrieve the entire stored buffer
	+ * in the event it cannot do so with a single call.
	+ */
	+/* Input (32 bytes) */
	+
	+struct hwrm_selftest_retreive_serdes_data_input {
	+ uint16_t req_type;
	+ /*
	+ * This value indicates what type of request this is. The format for the
	+ * rest of the command is determined by this field.
	+ */
	+ uint16_t cmpl_ring;
	+ /*
	+ * This value indicates the what completion ring the request will be
	+ * optionally completed on. If the value is -1, then no CR completion
	+ * will be generated. Any other value must be a valid CR ring_id value
	+ * for this function.
	+ */
	+ uint16_t seq_id;
	+ /* This value indicates the command sequence number. */
	+ uint16_t target_id;
	+ /*
	+ * Target ID of this command. 0x0 - 0xFFF8 - Used for function ids
	+ * 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF - HWRM
	+ */
	+ uint64_t resp_addr;
	+ /*
	+ * This is the host address where the response will be written when the
	+ * request is complete. This area must be 16B aligned and must be
	+ * cleared to zero before the request is made.
	+ */
	+ uint64_t resp_data_addr;
	+ /* Host address data is to DMA'd to. */
	+ uint32_t resp_data_offset;
	+ /*
	+ * This field contains the offset into the captured data to begin
	+ * copying the data to the host from. This should be set to 0 on the
	+ * initial call to this command.
	+ */
	+ uint16_t data_len;
	+ /*
	+ * Size of the buffer pointed to by resp_data_addr. The firmware may use
	+ * this entire buffer or less than the entire buffer, but never more.
	+ */
	+ uint8_t flags;
	+ /*
	+ * This field allows this command to request the individual serdes tests
	+ * to be run using this command.
	+ */
	+ /* Unused. */
	+ #define HWRM_SELFTEST_RETREIVE_SERDES_DATA_INPUT_FLAGS_UNUSED_TEST_MASK UINT32_C(0xf)
	+ #define HWRM_SELFTEST_RETREIVE_SERDES_DATA_INPUT_FLAGS_UNUSED_TEST_SFT 0
	+ /* Run the PCIe serdes test. */
	+ #define HWRM_SELFTEST_RETREIVE_SERDES_DATA_INPUT_FLAGS_PCIE_SERDES_TEST UINT32_C(0x10)
	+ /* Run the Ethernet serdes test. */
	+ #define HWRM_SELFTEST_RETREIVE_SERDES_DATA_INPUT_FLAGS_ETHERNET_SERDES_TEST UINT32_C(0x20)
	+ uint8_t unused_0;
	+} __attribute__((packed));
	+
	+/* Output (16 bytes) */
	+
	+struct hwrm_selftest_retreive_serdes_data_output {
	+ uint16_t error_code;
	+ /*
	+ * Pass/Fail or error type Note: receiver to verify the in parameters,
	+ * and fail the call with an error when appropriate
	+ */
	+ uint16_t req_type;
	+ /* This field returns the type of original request. */
	+ uint16_t seq_id;
	+ /* This field provides original sequence number of the command. */
	+ uint16_t resp_len;
	+ /*
	+ * This field is the length of the response in bytes. The last byte of
	+ * the response is a valid flag that will read as '1' when the command
	+ * has been completely written to memory.
	+ */
	+ uint16_t total_data_len;
	+ /* Total length of stored data. */
	+ uint16_t copied_data_len;
	+ /*
	+ * Amount of data DMA'd to host by this call. The driver can use this
	+ * field along with the total_data_len field above to determine the
	+ * value to write to the resp_data_offset field in the next call if more
	+ * than one call to these commands is required to retrieve all the
	+ * stored data.
	+ */
	+ uint32_t unused_0;
	+} __attribute__((packed));
	+
	+/* Hardware Resource Manager Specification */
	+/* Description: This structure is used to configure a RSS Context. */
	+/*
	+ * Note: The Hardware Resource Manager (HWRM) manages various hardware resources
	+ * inside the chip. The HWRM is implemented in firmware, and runs on embedded
	+ * processors inside the chip. This firmware service is vital part of the chip.
	+ * The chip can not be used by a driver or HWRM client without the HWRM.
	+ */
	+/* Input (16 bytes) */
	+
	+struct input {
	+ uint16_t req_type;
	+ /*
	+ * This value indicates what type of request this is. The format for the
	+ * rest of the command is determined by this field.
	+ */
	+ uint16_t cmpl_ring;
	+ /*
	+ * This value indicates the what completion ring the request will be
	+ * optionally completed on. If the value is -1, then no CR completion
	+ * will be generated. Any other value must be a valid CR ring_id value
	+ * for this function.
	+ */
	+ uint16_t seq_id;
	+ /* This value indicates the command sequence number. */
	+ uint16_t target_id;
	+ /*
	+ * Target ID of this command. 0x0 - 0xFFF8 - Used for function ids
	+ * 0xFFF8 - 0xFFFE - Reserved for internal processors 0xFFFF - HWRM
	+ */
	+ uint64_t resp_addr;
	+ /*
	+ * This is the host address where the response will be written when the
	+ * request is complete. This area must be 16B aligned and must be
	+ * cleared to zero before the request is made.
	+ */
	+} __attribute__((packed));
	+
	+/* Output (8 bytes) */
	+
	+struct output {
	+ uint16_t error_code;
	+ /*
	+ * Pass/Fail or error type Note: receiver to verify the in parameters,
	+ * and fail the call with an error when appropriate
	+ */
	+ uint16_t req_type;
	+ /* This field returns the type of original request. */
	+ uint16_t seq_id;
	+ /* This field provides original sequence number of the command. */
	+ uint16_t resp_len;
	+ /*
	+ * This field is the length of the response in bytes. The last byte of
	+ * the response is a valid flag that will read as '1' when the command
	+ * has been completely written to memory.
	+ */
	+} __attribute__((packed));
	+
	+/* Short Command Structure (16 bytes) */
	+
	+struct hwrm_short_input {
	+ uint16_t req_type;
	+ /*
	+ * This field indicates the type of request in the request buffer. The
	+ * format for the rest of the command (request) is determined by this
	+ * field.
	+ */
	+ uint16_t signature;
	+ /*
	+ * This field indicates a signature that is used to identify short form
	+ * of the command listed here. This field shall be set to 17185
	+ * (0x4321).
	+ */
	+ /* Signature indicating this is a short form of HWRM command */
	+ #define HWRM_SHORT_INPUT_SIGNATURE_SHORT_CMD UINT32_C(0x4321)
	+ uint16_t unused_0;
	+ /* Reserved for future use. */
	+ uint16_t size;
	+ /* This value indicates the length of the request. */
	+ uint64_t req_addr;
	+ /*
	+ * This is the host address where the request was written. This area
	+ * must be 16B aligned.
	+ */
	+} __attribute__((packed));
	+
	+#define GET_HWRM_REQ_TYPE(x) \
	+ ((x) == 0x99 ? "HWRM_CFA_NTUPLE_FILTER_ALLOC": \
	+ ((x) == 0x90 ? "HWRM_CFA_L2_FILTER_ALLOC": \
	+ ((x) == 0x91 ? "HWRM_CFA_L2_FILTER_FREE": \
	+ ((x) == 0x92 ? "HWRM_CFA_L2_FILTER_CFG": \
	+ ((x) == 0x93 ? "HWRM_CFA_L2_SET_RX_MASK": \
	+ ((x) == 0x94 ? "HWRM_CFA_VLAN_ANTISPOOF_CFG": \
	+ ((x) == 0x95 ? "HWRM_CFA_TUNNEL_FILTER_ALLOC": \
	+ ((x) == 0x96 ? "HWRM_CFA_TUNNEL_FILTER_FREE": \
	+ ((x) == 0x10 ? "RESERVED1": \
	+ ((x) == 0x11 ? "HWRM_FUNC_RESET": \
	+ ((x) == 0x12 ? "HWRM_FUNC_GETFID": \
	+ ((x) == 0x13 ? "HWRM_FUNC_VF_ALLOC": \
	+ ((x) == 0x14 ? "HWRM_FUNC_VF_FREE": \
	+ ((x) == 0x15 ? "HWRM_FUNC_QCAPS": \
	+ ((x) == 0x16 ? "HWRM_FUNC_QCFG": \
	+ ((x) == 0x17 ? "HWRM_FUNC_CFG": \
	+ ((x) == 0x18 ? "HWRM_FUNC_QSTATS": \
	+ ((x) == 0x19 ? "HWRM_FUNC_CLR_STATS": \
	+ ((x) == 0xe0 ? "HWRM_TEMP_MONITOR_QUERY": \
	+ ((x) == 0x1a ? "HWRM_FUNC_DRV_UNRGTR": \
	+ ((x) == 0x1b ? "HWRM_FUNC_VF_RESC_FREE": \
	+ ((x) == 0x1c ? "HWRM_FUNC_VF_VNIC_IDS_QUERY": \
	+ ((x) == 0x1d ? "HWRM_FUNC_DRV_RGTR": \
	+ ((x) == 0x1e ? "HWRM_FUNC_DRV_QVER": \
	+ ((x) == 0x1f ? "HWRM_FUNC_BUF_RGTR": \
	+ ((x) == 0x9a ? "HWRM_CFA_NTUPLE_FILTER_FREE": \
	+ ((x) == 0x9b ? "HWRM_CFA_NTUPLE_FILTER_CFG": \
	+ ((x) == 0xd3 ? "HWRM_FWD_ASYNC_EVENT_CMPL": \
	+ ((x) == 0xd2 ? "HWRM_FWD_RESP": \
	+ ((x) == 0xd1 ? "HWRM_REJECT_FWD_RESP": \
	+ ((x) == 0xd0 ? "HWRM_EXEC_FWD_RESP": \
	+ ((x) == 0xc0 ? "HWRM_FW_RESET": \
	+ ((x) == 0xc1 ? "HWRM_FW_QSTATUS": \
	+ ((x) == 0x70 ? "HWRM_VNIC_RSS_COS_LB_CTX_ALLOC": \
	+ ((x) == 0x71 ? "HWRM_VNIC_RSS_COS_LB_CTX_FREE": \
	+ ((x) == 0xb1 ? "HWRM_STAT_CTX_FREE": \
	+ ((x) == 0xb0 ? "HWRM_STAT_CTX_ALLOC": \
	+ ((x) == 0xb3 ? "HWRM_STAT_CTX_CLR_STATS": \
	+ ((x) == 0xb2 ? "HWRM_STAT_CTX_QUERY": \
	+ ((x) == 0xfff6 ? "HWRM_NVM_GET_DEV_INFO": \
	+ ((x) == 0x61 ? "HWRM_RING_GRP_FREE": \
	+ ((x) == 0x60 ? "HWRM_RING_GRP_ALLOC": \
	+ ((x) == 0x24 ? "HWRM_PORT_LPBK_QSTATS": \
	+ ((x) == 0xf3 ? "HWRM_WOL_REASON_QCFG": \
	+ ((x) == 0xa0 ? "HWRM_TUNNEL_DST_PORT_QUERY": \
	+ ((x) == 0xa1 ? "HWRM_TUNNEL_DST_PORT_ALLOC": \
	+ ((x) == 0xa2 ? "HWRM_TUNNEL_DST_PORT_FREE": \
	+ ((x) == 0xfffc ? "HWRM_NVM_RAW_DUMP": \
	+ ((x) == 0xfffb ? "HWRM_NVM_GET_DIR_INFO": \
	+ ((x) == 0xfffa ? "HWRM_NVM_GET_DIR_ENTRIES": \
	+ ((x) == 0x10a ? "HWRM_CFA_VLAN_ANTISPOOF_QCFG": \
	+ ((x) == 0xe ? "HWRM_FUNC_BUF_UNRGTR": \
	+ ((x) == 0xf ? "HWRM_FUNC_VF_CFG": \
	+ ((x) == 0xffff ? "HWRM_NVM_RAW_WRITE_BLK": \
	+ ((x) == 0xfffe ? "HWRM_NVM_WRITE": \
	+ ((x) == 0xfffd ? "HWRM_NVM_READ": \
	+ ((x) == 0x50 ? "HWRM_RING_ALLOC": \
	+ ((x) == 0x51 ? "HWRM_RING_FREE": \
	+ ((x) == 0x52 ? "HWRM_RING_CMPL_RING_QAGGINT_PARAMS": \
	+ ((x) == 0x53 ? "HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS": \
	+ ((x) == 0x4a ? "HWRM_VNIC_QCAPS": \
	+ ((x) == 0x49 ? "HWRM_VNIC_PLCMODES_QCFG": \
	+ ((x) == 0x48 ? "HWRM_VNIC_PLCMODES_CFG": \
	+ ((x) == 0x47 ? "HWRM_VNIC_RSS_QCFG": \
	+ ((x) == 0x46 ? "HWRM_VNIC_RSS_CFG": \
	+ ((x) == 0x44 ? "HWRM_VNIC_TPA_CFG": \
	+ ((x) == 0x43 ? "HWRM_VNIC_QCFG": \
	+ ((x) == 0x42 ? "HWRM_VNIC_CFG": \
	+ ((x) == 0x41 ? "HWRM_VNIC_FREE": \
	+ ((x) == 0x40 ? "HWRM_VNIC_ALLOC": \
	+ ((x) == 0x0 ? "HWRM_VER_GET": \
	+ ((x) == 0xfff9 ? "HWRM_NVM_FIND_DIR_ENTRY": \
	+ ((x) == 0xfff8 ? "HWRM_NVM_MOD_DIR_ENTRY": \
	+ ((x) == 0xfff7 ? "HWRM_NVM_ERASE_DIR_ENTRY": \
	+ ((x) == 0x5e ? "HWRM_RING_RESET": \
	+ ((x) == 0xfff5 ? "HWRM_NVM_VERIFY_UPDATE": \
	+ ((x) == 0xfff4 ? "HWRM_NVM_MODIFY": \
	+ ((x) == 0xfff3 ? "HWRM_NVM_INSTALL_UPDATE": \
	+ ((x) == 0xfff2 ? "HWRM_NVM_SET_VARIABLE": \
	+ ((x) == 0xfff1 ? "HWRM_NVM_GET_VARIABLE": \
	+ ((x) == 0xfff0 ? "HWRM_NVM_FLUSH": \
	+ ((x) == 0x2e ? "HWRM_PORT_LED_QCFG": \
	+ ((x) == 0x2d ? "HWRM_PORT_LED_CFG": \
	+ ((x) == 0x2f ? "HWRM_PORT_LED_QCAPS": \
	+ ((x) == 0x2a ? "HWRM_PORT_PHY_QCAPS": \
	+ ((x) == 0x38 ? "HWRM_QUEUE_PRI2COS_CFG": \
	+ ((x) == 0x39 ? "HWRM_QUEUE_COS2BW_QCFG": \
	+ ((x) == 0x32 ? "HWRM_QUEUE_CFG": \
	+ ((x) == 0x33 ? "HWRM_FUNC_VLAN_CFG": \
	+ ((x) == 0x30 ? "HWRM_QUEUE_QPORTCFG": \
	+ ((x) == 0x31 ? "HWRM_QUEUE_QCFG": \
	+ ((x) == 0x36 ? "HWRM_QUEUE_PFCENABLE_CFG": \
	+ ((x) == 0x37 ? "HWRM_QUEUE_PRI2COS_QCFG": \
	+ ((x) == 0x34 ? "HWRM_FUNC_VLAN_QCFG": \
	+ ((x) == 0x35 ? "HWRM_QUEUE_PFCENABLE_QCFG": \
	+ ((x) == 0xff14 ? "HWRM_DBG_DUMP": \
	+ ((x) == 0xc8 ? "HWRM_FW_SET_TIME": \
	+ ((x) == 0xc9 ? "HWRM_FW_GET_TIME": \
	+ ((x) == 0xf1 ? "HWRM_WOL_FILTER_FREE": \
	+ ((x) == 0xf0 ? "HWRM_WOL_FILTER_ALLOC": \
	+ ((x) == 0x27 ? "HWRM_PORT_PHY_QCFG": \
	+ ((x) == 0xf2 ? "HWRM_WOL_FILTER_QCFG": \
	+ ((x) == 0x21 ? "HWRM_PORT_MAC_CFG": \
	+ ((x) == 0x20 ? "HWRM_PORT_PHY_CFG": \
	+ ((x) == 0x23 ? "HWRM_PORT_QSTATS": \
	+ ((x) == 0x28 ? "HWRM_PORT_MAC_QCFG": \
	+ ((x) == 0xffef ? "HWRM_NVM_VALIDATE_OPTION": \
	+ ((x) == 0x3a ? "HWRM_QUEUE_COS2BW_CFG": \
	+ "Unknown req_type"))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))
	+
	+/* Command numbering (8 bytes) */
	+
	+struct cmd_nums {
	+ uint16_t req_type;
	+ /*
	+ * This version of the specification defines the commands listed in the
	+ * table below. The following are general implementation requirements
	+ * for these commands: # All commands listed below that are marked
	+ * neither reserved nor experimental shall be implemented by the HWRM. #
	+ * A HWRM client compliant to this specification should not use commands
	+ * outside of the list below. # A HWRM client compliant to this
	+ * specification should not use command numbers marked reserved below. #
	+ * A command marked experimental below may not be implemented by the
	+ * HWRM. # A command marked experimental may change in the future
	+ * version of the HWRM specification. # A command not listed below may
	+ * be implemented by the HWRM. The behavior of commands that are not
	+ * listed below is outside the scope of this specification.
	+ */
	+ #define HWRM_VER_GET (UINT32_C(0x0))
	+ #define HWRM_FUNC_BUF_UNRGTR (UINT32_C(0xe))
	+ #define HWRM_FUNC_VF_CFG (UINT32_C(0xf))
	+ /* Reserved for future use */
	+ #define RESERVED1 (UINT32_C(0x10))
	+ #define HWRM_FUNC_RESET (UINT32_C(0x11))
	+ #define HWRM_FUNC_GETFID (UINT32_C(0x12))
	+ #define HWRM_FUNC_VF_ALLOC (UINT32_C(0x13))
	+ #define HWRM_FUNC_VF_FREE (UINT32_C(0x14))
	+ #define HWRM_FUNC_QCAPS (UINT32_C(0x15))
	+ #define HWRM_FUNC_QCFG (UINT32_C(0x16))
	+ #define HWRM_FUNC_CFG (UINT32_C(0x17))
	+ #define HWRM_FUNC_QSTATS (UINT32_C(0x18))
	+ #define HWRM_FUNC_CLR_STATS (UINT32_C(0x19))
	+ #define HWRM_FUNC_DRV_UNRGTR (UINT32_C(0x1a))
	+ #define HWRM_FUNC_VF_RESC_FREE (UINT32_C(0x1b))
	+ #define HWRM_FUNC_VF_VNIC_IDS_QUERY (UINT32_C(0x1c))
	+ #define HWRM_FUNC_DRV_RGTR (UINT32_C(0x1d))
	+ #define HWRM_FUNC_DRV_QVER (UINT32_C(0x1e))
	+ #define HWRM_FUNC_BUF_RGTR (UINT32_C(0x1f))
	+ #define HWRM_PORT_PHY_CFG (UINT32_C(0x20))
	+ #define HWRM_PORT_MAC_CFG (UINT32_C(0x21))
	+ #define HWRM_PORT_QSTATS (UINT32_C(0x23))
	+ #define HWRM_PORT_LPBK_QSTATS (UINT32_C(0x24))
	+ #define HWRM_PORT_PHY_QCFG (UINT32_C(0x27))
	+ #define HWRM_PORT_MAC_QCFG (UINT32_C(0x28))
	+ #define HWRM_PORT_PHY_QCAPS (UINT32_C(0x2a))
	+ #define HWRM_PORT_LED_CFG (UINT32_C(0x2d))
	+ #define HWRM_PORT_LED_QCFG (UINT32_C(0x2e))
	+ #define HWRM_PORT_LED_QCAPS (UINT32_C(0x2f))
	+ #define HWRM_QUEUE_QPORTCFG (UINT32_C(0x30))
	+ #define HWRM_QUEUE_QCFG (UINT32_C(0x31))
	+ #define HWRM_QUEUE_CFG (UINT32_C(0x32))
	+ #define HWRM_FUNC_VLAN_CFG (UINT32_C(0x33))
	+ #define HWRM_FUNC_VLAN_QCFG (UINT32_C(0x34))
	+ #define HWRM_QUEUE_PFCENABLE_QCFG (UINT32_C(0x35))
	+ #define HWRM_QUEUE_PFCENABLE_CFG (UINT32_C(0x36))
	+ #define HWRM_QUEUE_PRI2COS_QCFG (UINT32_C(0x37))
	+ #define HWRM_QUEUE_PRI2COS_CFG (UINT32_C(0x38))
	+ #define HWRM_QUEUE_COS2BW_QCFG (UINT32_C(0x39))
	+ #define HWRM_QUEUE_COS2BW_CFG (UINT32_C(0x3a))
	+ #define HWRM_VNIC_ALLOC (UINT32_C(0x40))
	+ #define HWRM_VNIC_FREE (UINT32_C(0x41))
	+ #define HWRM_VNIC_CFG (UINT32_C(0x42))
	+ #define HWRM_VNIC_QCFG (UINT32_C(0x43))
	+ #define HWRM_VNIC_TPA_CFG (UINT32_C(0x44))
	+ #define HWRM_VNIC_RSS_CFG (UINT32_C(0x46))
	+ #define HWRM_VNIC_RSS_QCFG (UINT32_C(0x47))
	+ #define HWRM_VNIC_PLCMODES_CFG (UINT32_C(0x48))
	+ #define HWRM_VNIC_PLCMODES_QCFG (UINT32_C(0x49))
	+ #define HWRM_VNIC_QCAPS (UINT32_C(0x4a))
	+ #define HWRM_RING_ALLOC (UINT32_C(0x50))
	+ #define HWRM_RING_FREE (UINT32_C(0x51))
	+ #define HWRM_RING_CMPL_RING_QAGGINT_PARAMS (UINT32_C(0x52))
	+ #define HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS (UINT32_C(0x53))
	+ #define HWRM_RING_RESET (UINT32_C(0x5e))
	+ #define HWRM_RING_GRP_ALLOC (UINT32_C(0x60))
	+ #define HWRM_RING_GRP_FREE (UINT32_C(0x61))
	+ #define HWRM_VNIC_RSS_COS_LB_CTX_ALLOC (UINT32_C(0x70))
	+ #define HWRM_VNIC_RSS_COS_LB_CTX_FREE (UINT32_C(0x71))
	+ #define HWRM_CFA_L2_FILTER_ALLOC (UINT32_C(0x90))
	+ #define HWRM_CFA_L2_FILTER_FREE (UINT32_C(0x91))
	+ #define HWRM_CFA_L2_FILTER_CFG (UINT32_C(0x92))
	+ #define HWRM_CFA_L2_SET_RX_MASK (UINT32_C(0x93))
	+ #define HWRM_CFA_VLAN_ANTISPOOF_CFG (UINT32_C(0x94))
	+ #define HWRM_CFA_TUNNEL_FILTER_ALLOC (UINT32_C(0x95))
	+ #define HWRM_CFA_TUNNEL_FILTER_FREE (UINT32_C(0x96))
	+ #define HWRM_CFA_NTUPLE_FILTER_ALLOC (UINT32_C(0x99))
	+ #define HWRM_CFA_NTUPLE_FILTER_FREE (UINT32_C(0x9a))
	+ #define HWRM_CFA_NTUPLE_FILTER_CFG (UINT32_C(0x9b))
	+ #define HWRM_TUNNEL_DST_PORT_QUERY (UINT32_C(0xa0))
	+ #define HWRM_TUNNEL_DST_PORT_ALLOC (UINT32_C(0xa1))
	+ #define HWRM_TUNNEL_DST_PORT_FREE (UINT32_C(0xa2))
	+ #define HWRM_STAT_CTX_ALLOC (UINT32_C(0xb0))
	+ #define HWRM_STAT_CTX_FREE (UINT32_C(0xb1))
	+ #define HWRM_STAT_CTX_QUERY (UINT32_C(0xb2))
	+ #define HWRM_STAT_CTX_CLR_STATS (UINT32_C(0xb3))
	+ #define HWRM_FW_RESET (UINT32_C(0xc0))
	+ #define HWRM_FW_QSTATUS (UINT32_C(0xc1))
	+ #define HWRM_FW_SET_TIME (UINT32_C(0xc8))
	+ #define HWRM_FW_GET_TIME (UINT32_C(0xc9))
	+ #define HWRM_EXEC_FWD_RESP (UINT32_C(0xd0))
	+ #define HWRM_REJECT_FWD_RESP (UINT32_C(0xd1))
	+ #define HWRM_FWD_RESP (UINT32_C(0xd2))
	+ #define HWRM_FWD_ASYNC_EVENT_CMPL (UINT32_C(0xd3))
	+ #define HWRM_TEMP_MONITOR_QUERY (UINT32_C(0xe0))
	+ #define HWRM_WOL_FILTER_ALLOC (UINT32_C(0xf0))
	+ #define HWRM_WOL_FILTER_FREE (UINT32_C(0xf1))
	+ #define HWRM_WOL_FILTER_QCFG (UINT32_C(0xf2))
	+ #define HWRM_WOL_REASON_QCFG (UINT32_C(0xf3))
	+ #define HWRM_CFA_VLAN_ANTISPOOF_QCFG (UINT32_C(0x10a))
	+ #define HWRM_DBG_DUMP (UINT32_C(0xff14))
	+ #define HWRM_NVM_VALIDATE_OPTION (UINT32_C(0xffef))
	+ #define HWRM_NVM_FLUSH (UINT32_C(0xfff0))
	+ #define HWRM_NVM_GET_VARIABLE (UINT32_C(0xfff1))
	+ #define HWRM_NVM_SET_VARIABLE (UINT32_C(0xfff2))
	+ #define HWRM_NVM_INSTALL_UPDATE (UINT32_C(0xfff3))
	+ #define HWRM_NVM_MODIFY (UINT32_C(0xfff4))
	+ #define HWRM_NVM_VERIFY_UPDATE (UINT32_C(0xfff5))
	+ #define HWRM_NVM_GET_DEV_INFO (UINT32_C(0xfff6))
	+ #define HWRM_NVM_ERASE_DIR_ENTRY (UINT32_C(0xfff7))
	+ #define HWRM_NVM_MOD_DIR_ENTRY (UINT32_C(0xfff8))
	+ #define HWRM_NVM_FIND_DIR_ENTRY (UINT32_C(0xfff9))
	+ #define HWRM_NVM_GET_DIR_ENTRIES (UINT32_C(0xfffa))
	+ #define HWRM_NVM_GET_DIR_INFO (UINT32_C(0xfffb))
	+ #define HWRM_NVM_RAW_DUMP (UINT32_C(0xfffc))
	+ #define HWRM_NVM_READ (UINT32_C(0xfffd))
	+ #define HWRM_NVM_WRITE (UINT32_C(0xfffe))
	+ #define HWRM_NVM_RAW_WRITE_BLK (UINT32_C(0xffff))
	+ uint16_t unused_0[3];
	+} __attribute__((packed));
	+
	+#define GET_HWRM_ERROR_CODE(x) \
	+ ((x) == 0xf ? "HWRM_ERROR": \
	+ ((x) == 0xffff ? "CMD_NOT_SUPPORTED": \
	+ ((x) == 0xfffe ? "UNKNOWN_ERR": \
	+ ((x) == 0x4 ? "RESOURCE_ALLOC_ERROR": \
	+ ((x) == 0x5 ? "INVALID_FLAGS": \
	+ ((x) == 0x6 ? "INVALID_ENABLES": \
	+ ((x) == 0x0 ? "SUCCESS": \
	+ ((x) == 0x1 ? "FAIL": \
	+ ((x) == 0x2 ? "INVALID_PARAMS": \
	+ ((x) == 0x3 ? "RESOURCE_ACCESS_DENIED": \
	+ "Unknown error_code"))))))))))
	+
	+/* Return Codes (8 bytes) */
	+
	+struct ret_codes {
	+ uint16_t error_code;
	+ /* These are numbers assigned to return/error codes. */
	+ /* Request was successfully executed by the HWRM. */
	+ #define HWRM_ERR_CODE_SUCCESS (UINT32_C(0x0))
	+ /* THe HWRM failed to execute the request. */
	+ #define HWRM_ERR_CODE_FAIL (UINT32_C(0x1))
	+ /* The request contains invalid argument(s) or input parameters. */
	+ #define HWRM_ERR_CODE_INVALID_PARAMS (UINT32_C(0x2))
	+ /*
	+ * The requester is not allowed to access the requested
	+ * resource. This error code shall be provided in a response to
	+ * a request to query or modify an existing resource that is not
	+ * accessible by the requester.
	+ */
	+ #define HWRM_ERR_CODE_RESOURCE_ACCESS_DENIED (UINT32_C(0x3))
	+ /*
	+ * The HWRM is unable to allocate the requested resource. This
	+ * code only applies to requests for HWRM resource allocations.
	+ */
	+ #define HWRM_ERR_CODE_RESOURCE_ALLOC_ERROR (UINT32_C(0x4))
	+ /* Invalid combination of flags is specified in the request. */
	+ #define HWRM_ERR_CODE_INVALID_FLAGS (UINT32_C(0x5))
	+ /*
	+ * Invalid combination of enables fields is specified in the
	+ * request.
	+ */
	+ #define HWRM_ERR_CODE_INVALID_ENABLES (UINT32_C(0x6))
	+ /* Generic HWRM execution error that represents an internal error. */
	+ #define HWRM_ERR_CODE_HWRM_ERROR (UINT32_C(0xf))
	+ /* Unknown error */
	+ #define HWRM_ERR_CODE_UNKNOWN_ERR (UINT32_C(0xfffe))
	+ /* Unsupported or invalid command */
	+ #define HWRM_ERR_CODE_CMD_NOT_SUPPORTED (UINT32_C(0xffff))
	+ uint16_t unused_0[3];
	+} __attribute__((packed));
	+
	+/* Output (16 bytes) */
	+
	+struct hwrm_err_output {
	+ uint16_t error_code;
	+ /*
	+ * Pass/Fail or error type Note: receiver to verify the in parameters,
	+ * and fail the call with an error when appropriate
	+ */
	+ uint16_t req_type;
	+ /* This field returns the type of original request. */
	+ uint16_t seq_id;
	+ /* This field provides original sequence number of the command. */
	+ uint16_t resp_len;
	+ /*
	+ * This field is the length of the response in bytes. The last byte of
	+ * the response is a valid flag that will read as '1' when the command
	+ * has been completely written to memory.
	+ */
	+ uint32_t opaque_0;
	+ /* debug info for this error response. */
	+ uint16_t opaque_1;
	+ /* debug info for this error response. */
	+ uint8_t cmd_err;
	+ /*
	+ * In the case of an error response, command specific error code is
	+ * returned in this field.
	+ */
	+ uint8_t valid;
	+ /*
	+ * This field is used in Output records to indicate that the output is
	+ * completely written to RAM. This field should be read as '1' to
	+ * indicate that the output has been completely written. When writing a
	+ * command completion or response to an internal processor, the order of
	+ * writes has to be such that this field is written last.
	+ */
	+} __attribute__((packed));
	+
	+/* Port Tx Statistics Formats (408 bytes) */
	+
	+struct tx_port_stats {
	+ uint64_t tx_64b_frames;
	+ /* Total Number of 64 Bytes frames transmitted */
	+ uint64_t tx_65b_127b_frames;
	+ /* Total Number of 65-127 Bytes frames transmitted */
	+ uint64_t tx_128b_255b_frames;
	+ /* Total Number of 128-255 Bytes frames transmitted */
	+ uint64_t tx_256b_511b_frames;
	+ /* Total Number of 256-511 Bytes frames transmitted */
	+ uint64_t tx_512b_1023b_frames;
	+ /* Total Number of 512-1023 Bytes frames transmitted */
	+ uint64_t tx_1024b_1518_frames;
	+ /* Total Number of 1024-1518 Bytes frames transmitted */
	+ uint64_t tx_good_vlan_frames;
	+ /*
	+ * Total Number of each good VLAN (exludes FCS errors) frame transmitted
	+ * which is 1519 to 1522 bytes in length inclusive (excluding framing
	+ * bits but including FCS bytes).
	+ */
	+ uint64_t tx_1519b_2047_frames;
	+ /* Total Number of 1519-2047 Bytes frames transmitted */
	+ uint64_t tx_2048b_4095b_frames;
	+ /* Total Number of 2048-4095 Bytes frames transmitted */
	+ uint64_t tx_4096b_9216b_frames;
	+ /* Total Number of 4096-9216 Bytes frames transmitted */
	+ uint64_t tx_9217b_16383b_frames;
	+ /* Total Number of 9217-16383 Bytes frames transmitted */
	+ uint64_t tx_good_frames;
	+ /* Total Number of good frames transmitted */
	+ uint64_t tx_total_frames;
	+ /* Total Number of frames transmitted */
	+ uint64_t tx_ucast_frames;
	+ /* Total number of unicast frames transmitted */
	+ uint64_t tx_mcast_frames;
	+ /* Total number of multicast frames transmitted */
	+ uint64_t tx_bcast_frames;
	+ /* Total number of broadcast frames transmitted */
	+ uint64_t tx_pause_frames;
	+ /* Total number of PAUSE control frames transmitted */
	+ uint64_t tx_pfc_frames;
	+ /* Total number of PFC/per-priority PAUSE control frames transmitted */
	+ uint64_t tx_jabber_frames;
	+ /* Total number of jabber frames transmitted */
	+ uint64_t tx_fcs_err_frames;
	+ /* Total number of frames transmitted with FCS error */
	+ uint64_t tx_control_frames;
	+ /* Total number of control frames transmitted */
	+ uint64_t tx_oversz_frames;
	+ /* Total number of over-sized frames transmitted */
	+ uint64_t tx_single_dfrl_frames;
	+ /* Total number of frames with single deferral */
	+ uint64_t tx_multi_dfrl_frames;
	+ /* Total number of frames with multiple deferrals */
	+ uint64_t tx_single_coll_frames;
	+ /* Total number of frames with single collision */
	+ uint64_t tx_multi_coll_frames;
	+ /* Total number of frames with multiple collisions */
	+ uint64_t tx_late_coll_frames;
	+ /* Total number of frames with late collisions */
	+ uint64_t tx_excessive_coll_frames;
	+ /* Total number of frames with excessive collisions */
	+ uint64_t tx_frag_frames;
	+ /* Total number of fragmented frames transmitted */
	+ uint64_t tx_err;
	+ /* Total number of transmit errors */
	+ uint64_t tx_tagged_frames;
	+ /* Total number of single VLAN tagged frames transmitted */
	+ uint64_t tx_dbl_tagged_frames;
	+ /* Total number of double VLAN tagged frames transmitted */
	+ uint64_t tx_runt_frames;
	+ /* Total number of runt frames transmitted */
	+ uint64_t tx_fifo_underruns;
	+ /* Total number of TX FIFO under runs */
	+ uint64_t tx_pfc_ena_frames_pri0;
	+ /* Total number of PFC frames with PFC enabled bit for Pri 0 transmitted */
	+ uint64_t tx_pfc_ena_frames_pri1;
	+ /* Total number of PFC frames with PFC enabled bit for Pri 1 transmitted */
	+ uint64_t tx_pfc_ena_frames_pri2;
	+ /* Total number of PFC frames with PFC enabled bit for Pri 2 transmitted */
	+ uint64_t tx_pfc_ena_frames_pri3;
	+ /* Total number of PFC frames with PFC enabled bit for Pri 3 transmitted */
	+ uint64_t tx_pfc_ena_frames_pri4;
	+ /* Total number of PFC frames with PFC enabled bit for Pri 4 transmitted */
	+ uint64_t tx_pfc_ena_frames_pri5;
	+ /* Total number of PFC frames with PFC enabled bit for Pri 5 transmitted */
	+ uint64_t tx_pfc_ena_frames_pri6;
	+ /* Total number of PFC frames with PFC enabled bit for Pri 6 transmitted */
	+ uint64_t tx_pfc_ena_frames_pri7;
	+ /* Total number of PFC frames with PFC enabled bit for Pri 7 transmitted */
	+ uint64_t tx_eee_lpi_events;
	+ /* Total number of EEE LPI Events on TX */
	+ uint64_t tx_eee_lpi_duration;
	+ /* EEE LPI Duration Counter on TX */
	+ uint64_t tx_llfc_logical_msgs;
	+ /* Total number of Link Level Flow Control (LLFC) messages transmitted */
	+ uint64_t tx_hcfc_msgs;
	+ /* Total number of HCFC messages transmitted */
	+ uint64_t tx_total_collisions;
	+ /* Total number of TX collisions */
	+ uint64_t tx_bytes;
	+ /* Total number of transmitted bytes */
	+ uint64_t tx_xthol_frames;
	+ /* Total number of end-to-end HOL frames */
	+ uint64_t tx_stat_discard;
	+ /* Total Tx Drops per Port reported by STATS block */
	+ uint64_t tx_stat_error;
	+ /* Total Tx Error Drops per Port reported by STATS block */
	+} __attribute__((packed));
	+
	+/* Port Rx Statistics Formats (528 bytes) */
	+
	+struct rx_port_stats {
	+ uint64_t rx_64b_frames;
	+ /* Total Number of 64 Bytes frames received */
	+ uint64_t rx_65b_127b_frames;
	+ /* Total Number of 65-127 Bytes frames received */
	+ uint64_t rx_128b_255b_frames;
	+ /* Total Number of 128-255 Bytes frames received */
	+ uint64_t rx_256b_511b_frames;
	+ /* Total Number of 256-511 Bytes frames received */
	+ uint64_t rx_512b_1023b_frames;
	+ /* Total Number of 512-1023 Bytes frames received */
	+ uint64_t rx_1024b_1518_frames;
	+ /* Total Number of 1024-1518 Bytes frames received */
	+ uint64_t rx_good_vlan_frames;
	+ /*
	+ * Total Number of each good VLAN (exludes FCS errors) frame received
	+ * which is 1519 to 1522 bytes in length inclusive (excluding framing
	+ * bits but including FCS bytes).
	+ */
	+ uint64_t rx_1519b_2047b_frames;
	+ /* Total Number of 1519-2047 Bytes frames received */
	+ uint64_t rx_2048b_4095b_frames;
	+ /* Total Number of 2048-4095 Bytes frames received */
	+ uint64_t rx_4096b_9216b_frames;
	+ /* Total Number of 4096-9216 Bytes frames received */
	+ uint64_t rx_9217b_16383b_frames;
	+ /* Total Number of 9217-16383 Bytes frames received */
	+ uint64_t rx_total_frames;
	+ /* Total number of frames received */
	+ uint64_t rx_ucast_frames;
	+ /* Total number of unicast frames received */
	+ uint64_t rx_mcast_frames;
	+ /* Total number of multicast frames received */
	+ uint64_t rx_bcast_frames;
	+ /* Total number of broadcast frames received */
	+ uint64_t rx_fcs_err_frames;
	+ /* Total number of received frames with FCS error */
	+ uint64_t rx_ctrl_frames;
	+ /* Total number of control frames received */
	+ uint64_t rx_pause_frames;
	+ /* Total number of PAUSE frames received */
	+ uint64_t rx_pfc_frames;
	+ /* Total number of PFC frames received */
	+ uint64_t rx_unsupported_opcode_frames;
	+ /* Total number of frames received with an unsupported opcode */
	+ uint64_t rx_unsupported_da_pausepfc_frames;
	+ /*
	+ * Total number of frames received with an unsupported DA for pause and
	+ * PFC
	+ */
	+ uint64_t rx_wrong_sa_frames;
	+ /* Total number of frames received with an unsupported SA */
	+ uint64_t rx_align_err_frames;
	+ /* Total number of received packets with alignment error */
	+ uint64_t rx_oor_len_frames;
	+ /* Total number of received frames with out-of-range length */
	+ uint64_t rx_code_err_frames;
	+ /* Total number of received frames with error termination */
	+ uint64_t rx_false_carrier_frames;
	+ /*
	+ * Total number of received frames with a false carrier is detected
	+ * during idle, as defined by RX_ER samples active and RXD is 0xE. The
	+ * event is reported along with the statistics generated on the next
	+ * received frame. Only one false carrier condition can be detected and
	+ * logged between frames. Carrier event, valid for 10M/100M speed modes
	+ * only.
	+ */
	+ uint64_t rx_ovrsz_frames;
	+ /* Total number of over-sized frames received */
	+ uint64_t rx_jbr_frames;
	+ /* Total number of jabber packets received */
	+ uint64_t rx_mtu_err_frames;
	+ /* Total number of received frames with MTU error */
	+ uint64_t rx_match_crc_frames;
	+ /* Total number of received frames with CRC match */
	+ uint64_t rx_promiscuous_frames;
	+ /* Total number of frames received promiscuously */
	+ uint64_t rx_tagged_frames;
	+ /* Total number of received frames with one or two VLAN tags */
	+ uint64_t rx_double_tagged_frames;
	+ /* Total number of received frames with two VLAN tags */
	+ uint64_t rx_trunc_frames;
	+ /* Total number of truncated frames received */
	+ uint64_t rx_good_frames;
	+ /* Total number of good frames (without errors) received */
	+ uint64_t rx_pfc_xon2xoff_frames_pri0;
	+ /*
	+ * Total number of received PFC frames with transition from XON to XOFF
	+ * on Pri 0
	+ */
	+ uint64_t rx_pfc_xon2xoff_frames_pri1;
	+ /*
	+ * Total number of received PFC frames with transition from XON to XOFF
	+ * on Pri 1
	+ */
	+ uint64_t rx_pfc_xon2xoff_frames_pri2;
	+ /*
	+ * Total number of received PFC frames with transition from XON to XOFF
	+ * on Pri 2
	+ */
	+ uint64_t rx_pfc_xon2xoff_frames_pri3;
	+ /*
	+ * Total number of received PFC frames with transition from XON to XOFF
	+ * on Pri 3
	+ */
	+ uint64_t rx_pfc_xon2xoff_frames_pri4;
	+ /*
	+ * Total number of received PFC frames with transition from XON to XOFF
	+ * on Pri 4
	+ */
	+ uint64_t rx_pfc_xon2xoff_frames_pri5;
	+ /*
	+ * Total number of received PFC frames with transition from XON to XOFF
	+ * on Pri 5
	+ */
	+ uint64_t rx_pfc_xon2xoff_frames_pri6;
	+ /*
	+ * Total number of received PFC frames with transition from XON to XOFF
	+ * on Pri 6
	+ */
	+ uint64_t rx_pfc_xon2xoff_frames_pri7;
	+ /*
	+ * Total number of received PFC frames with transition from XON to XOFF
	+ * on Pri 7
	+ */
	+ uint64_t rx_pfc_ena_frames_pri0;
	+ /* Total number of received PFC frames with PFC enabled bit for Pri 0 */
	+ uint64_t rx_pfc_ena_frames_pri1;
	+ /* Total number of received PFC frames with PFC enabled bit for Pri 1 */
	+ uint64_t rx_pfc_ena_frames_pri2;
	+ /* Total number of received PFC frames with PFC enabled bit for Pri 2 */
	+ uint64_t rx_pfc_ena_frames_pri3;
	+ /* Total number of received PFC frames with PFC enabled bit for Pri 3 */
	+ uint64_t rx_pfc_ena_frames_pri4;
	+ /* Total number of received PFC frames with PFC enabled bit for Pri 4 */
	+ uint64_t rx_pfc_ena_frames_pri5;
	+ /* Total number of received PFC frames with PFC enabled bit for Pri 5 */
	+ uint64_t rx_pfc_ena_frames_pri6;
	+ /* Total number of received PFC frames with PFC enabled bit for Pri 6 */
	+ uint64_t rx_pfc_ena_frames_pri7;
	+ /* Total number of received PFC frames with PFC enabled bit for Pri 7 */
	+ uint64_t rx_sch_crc_err_frames;
	+ /* Total Number of frames received with SCH CRC error */
	+ uint64_t rx_undrsz_frames;
	+ /* Total Number of under-sized frames received */
	+ uint64_t rx_frag_frames;
	+ /* Total Number of fragmented frames received */
	+ uint64_t rx_eee_lpi_events;
	+ /* Total number of RX EEE LPI Events */
	+ uint64_t rx_eee_lpi_duration;
	+ /* EEE LPI Duration Counter on RX */
	+ uint64_t rx_llfc_physical_msgs;
	+ /*
	+ * Total number of physical type Link Level Flow Control (LLFC) messages
	+ * received
	+ */
	+ uint64_t rx_llfc_logical_msgs;
	+ /*
	+ * Total number of logical type Link Level Flow Control (LLFC) messages
	+ * received
	+ */
	+ uint64_t rx_llfc_msgs_with_crc_err;
	+ /*
	+ * Total number of logical type Link Level Flow Control (LLFC) messages
	+ * received with CRC error
	+ */
	+ uint64_t rx_hcfc_msgs;
	+ /* Total number of HCFC messages received */
	+ uint64_t rx_hcfc_msgs_with_crc_err;
	+ /* Total number of HCFC messages received with CRC error */
	+ uint64_t rx_bytes;
	+ /* Total number of received bytes */
	+ uint64_t rx_runt_bytes;
	+ /* Total number of bytes received in runt frames */
	+ uint64_t rx_runt_frames;
	+ /* Total number of runt frames received */
	+ uint64_t rx_stat_discard;
	+ /* Total Rx Discards per Port reported by STATS block */
	+ uint64_t rx_stat_err;
	+ /* Total Rx Error Drops per Port reported by STATS block */
	+} __attribute__((packed));
	+
	+/* Periodic Statistics Context DMA to host (160 bytes) */
	+
	+struct ctx_hw_stats {
	+ uint64_t rx_ucast_pkts;
	+ /* Number of received unicast packets */
	+ uint64_t rx_mcast_pkts;
	+ /* Number of received multicast packets */
	+ uint64_t rx_bcast_pkts;
	+ /* Number of received broadcast packets */
	+ uint64_t rx_discard_pkts;
	+ /* Number of discarded packets on received path */
	+ uint64_t rx_drop_pkts;
	+ /* Number of dropped packets on received path */
	+ uint64_t rx_ucast_bytes;
	+ /* Number of received bytes for unicast traffic */
	+ uint64_t rx_mcast_bytes;
	+ /* Number of received bytes for multicast traffic */
	+ uint64_t rx_bcast_bytes;
	+ /* Number of received bytes for broadcast traffic */
	+ uint64_t tx_ucast_pkts;
	+ /* Number of transmitted unicast packets */
	+ uint64_t tx_mcast_pkts;
	+ /* Number of transmitted multicast packets */
	+ uint64_t tx_bcast_pkts;
	+ /* Number of transmitted broadcast packets */
	+ uint64_t tx_discard_pkts;
	+ /* Number of discarded packets on transmit path */
	+ uint64_t tx_drop_pkts;
	+ /* Number of dropped packets on transmit path */
	+ uint64_t tx_ucast_bytes;
	+ /* Number of transmitted bytes for unicast traffic */
	+ uint64_t tx_mcast_bytes;
	+ /* Number of transmitted bytes for multicast traffic */
	+ uint64_t tx_bcast_bytes;
	+ /* Number of transmitted bytes for broadcast traffic */
	+ uint64_t tpa_pkts;
	+ /* Number of TPA packets */
	+ uint64_t tpa_bytes;
	+ /* Number of TPA bytes */
	+ uint64_t tpa_events;
	+ /* Number of TPA events */
	+ uint64_t tpa_aborts;
	+ /* Number of TPA aborts */
	+} __attribute__((packed));
	+
	+/* Structure data header (16 bytes) */
	+
	+struct hwrm_struct_hdr {
	+ uint16_t struct_id;
	+ /* This value indicates the structured data ID. */
	+ /* LLDP configuration structured data ID. */
	+ #define HWRM_STRUCT_HDR_STRUCT_ID_LLDP_CFG UINT32_C(0x41b)
	+ /* DCBX ETS configuration structured data ID. */
	+ #define HWRM_STRUCT_HDR_STRUCT_ID_DCBX_ETS UINT32_C(0x41d)
	+ /* DCBX PFC configuration structured data ID. */
	+ #define HWRM_STRUCT_HDR_STRUCT_ID_DCBX_PFC UINT32_C(0x41f)
	+ /* DCBX APP configuration structured data ID. */
	+ #define HWRM_STRUCT_HDR_STRUCT_ID_DCBX_APP UINT32_C(0x421)
	+ /*
	+ * DCBX state configuration structured data ID for all DCBX
	+ * features.
	+ */
	+ #define HWRM_STRUCT_HDR_STRUCT_ID_DCBX_FEATURE_STATE UINT32_C(0x422)
	+ /*
	+ * LLDP generic structured data ID. This is used with
	+ * GET_STRUCTURED_DATA only.
	+ */
	+ #define HWRM_STRUCT_HDR_STRUCT_ID_LLDP_GENERIC UINT32_C(0x424)
	+ /*
	+ * LLDP device structured data ID. This is used with
	+ * GET_STRUCTURED_DATA only.
	+ */
	+ #define HWRM_STRUCT_HDR_STRUCT_ID_LLDP_DEVICE UINT32_C(0x426)
	+ /* reserved for AFM usage. */
	+ #define HWRM_STRUCT_HDR_STRUCT_ID_AFM_OPAQUE UINT32_C(0x1)
	+ /* Port description. */
	+ #define HWRM_STRUCT_HDR_STRUCT_ID_PORT_DESCRIPTION UINT32_C(0xa)
	+ /* RSSv2 Configuration. */
	+ #define HWRM_STRUCT_HDR_STRUCT_ID_RSS_V2 UINT32_C(0x64)
	+ uint16_t len;
	+ /* This value indicates the length of structured data. */
	+ uint8_t version;
	+ /* This value indicates the version of structured data. */
	+ uint8_t count;
	+ /* This value indicates the number of structured data elements. */
	+ uint16_t subtype;
	+ /* This value indicates the subtype. */
	+ uint16_t next_offset;
	+ /*
	+ * This value indicates the count of 64-bit values that point to the
	+ * next header. A value of 0 means that this is the last element. The
	+ * value is a count of 64-bit words from the beginning of the current
	+ * header.
	+ */
	+ /* This value indicates this is the last element */
	+ #define HWRM_STRUCT_HDR_NEXT_OFFSET_LAST UINT32_C(0x0)
	+ uint16_t unused_0[3];
	+} __attribute__((packed));
	+
	+/* DCBX ETS configuration structure (1053) (32 bytes) */
	+
	+struct hwrm_struct_data_dcbx_ets {
	+ uint8_t destination;
	+ /*
	+ * This field indicates if this configuration is ETS recommendation or
	+ * ETS configuration. A value 1 means it is ETS configuration, A value
	+ * of 2 means it is a ETS recommendation.
	+ */
	+ /* ETS configuration */
	+ #define HWRM_STRUCT_DATA_DCBX_ETS_DESTINATION_CONFIGURATION UINT32_C(0x1)
	+ /* ETS recommendation */
	+ #define HWRM_STRUCT_DATA_DCBX_ETS_DESTINATION_RECOMMMENDATION UINT32_C(0x2)
	+ uint8_t max_tcs;
	+ /* This value indicates maximum ETS TCs supported. */
	+ uint16_t unused_0;
	+ /* unused. */
	+ uint8_t pri0_to_tc_map;
	+ /* ETS priority 0 to TC map. */
	+ uint8_t pri1_to_tc_map;
	+ /* ETS priority 1 to TC map. */
	+ uint8_t pri2_to_tc_map;
	+ /* ETS priority 2 to TC map. */
	+ uint8_t pri3_to_tc_map;
	+ /* ETS priority 3 to TC map. */
	+ uint8_t pri4_to_tc_map;
	+ /* ETS priority 4 to TC map. */
	+ uint8_t pri5_to_tc_map;
	+ /* ETS priority 5 to TC map. */
	+ uint8_t pri6_to_tc_map;
	+ /* ETS priority 6 to TC map. */
	+ uint8_t pri7_to_tc_map;
	+ /* ETS priority 7 to TC map. */
	+ uint8_t tc0_to_bw_map;
	+ /* ETS TC 0 to bandwidth map. */
	+ uint8_t tc1_to_bw_map;
	+ /* ETS TC 1 to bandwidth map. */
	+ uint8_t tc2_to_bw_map;
	+ /* ETS TC 2 to bandwidth map. */
	+ uint8_t tc3_to_bw_map;
	+ /* ETS TC 3 to bandwidth map. */
	+ uint8_t tc4_to_bw_map;
	+ /* ETS TC 4 to bandwidth map. */
	+ uint8_t tc5_to_bw_map;
	+ /* ETS TC 5 to bandwidth map. */
	+ uint8_t tc6_to_bw_map;
	+ /* ETS TC 6 to bandwidth map. */
	+ uint8_t tc7_to_bw_map;
	+ /* ETS TC 7 to bandwidth map. */
	+ uint8_t tc0_to_tsa_map;
	+ /* ETS TC 0 to TSA map. */
	+ /* strict priority */
	+ #define HWRM_STRUCT_DATA_DCBX_ETS_TC0_TO_TSA_MAP_TSA_TYPE_SP UINT32_C(0x0)
	+ /* credit based shaper */
	+ #define HWRM_STRUCT_DATA_DCBX_ETS_TC0_TO_TSA_MAP_TSA_TYPE_CBS UINT32_C(0x1)
	+ /* ETS */
	+ #define HWRM_STRUCT_DATA_DCBX_ETS_TC0_TO_TSA_MAP_TSA_TYPE_ETS UINT32_C(0x2)
	+ /* vendor specific */
	+ #define HWRM_STRUCT_DATA_DCBX_ETS_TC0_TO_TSA_MAP_TSA_TYPE_VENDOR_SPECIFIC UINT32_C(0xff)
	+ uint8_t tc1_to_tsa_map;
	+ /* ETS TC 1 to TSA map. */
	+ uint8_t tc2_to_tsa_map;
	+ /* ETS TC 2 to TSA map. */
	+ uint8_t tc3_to_tsa_map;
	+ /* ETS TC 3 to TSA map. */
	+ uint8_t tc4_to_tsa_map;
	+ /* ETS TC 4 to TSA map. */
	+ uint8_t tc5_to_tsa_map;
	+ /* ETS TC 5 to TSA map. */
	+ uint8_t tc6_to_tsa_map;
	+ /* ETS TC 6 to TSA map. */
	+ uint8_t tc7_to_tsa_map;
	+ /* ETS TC 7 to TSA map. */
	+ uint32_t unused_1;
	+} __attribute__((packed));
	+
	+/* DCBX PFC configuration structure (1055) (8 bytes) */
	+
	+struct hwrm_struct_data_dcbx_pfc {
	+ uint8_t pfc_priority_bitmap;
	+ /*
	+ * This field indicates PFC priority bit map. A value of '0' indicates
	+ * PFC is disabled. A value of '1' indicates PFC is enabled on that
	+ * priority.
	+ */
	+ uint8_t max_pfc_tcs;
	+ /*
	+ * This field indicates max PFC TCs supported. Each PFC TC will map to a
	+ * lossless CoS queue.
	+ */
	+ uint8_t mbc;
	+ /*
	+ * This field indicates if MACSec bypass capability is enabled. A value
	+ * of '1' indicates MBC is enabled. A value of '0' indicates MBC is
	+ * disabled.
	+ */
	+ uint8_t unused_0[5];
	+} __attribute__((packed));
	+
	+/* DCBX Application configuration structure (1057) (8 bytes) */
	+
	+struct hwrm_struct_data_dcbx_app {
	+ uint16_t protocol_id; /* big endian */
	+ /*
	+ * This field indicates the protocol identifier. This should be
	+ * specified in big endian format.
	+ */
	+ uint8_t protocol_selector;
	+ /*
	+ * This field indicates the protocol selector. The valid values are
	+ * mentioned below.
	+ */
	+ /* ether type */
	+ #define HWRM_STRUCT_DATA_DCBX_APP_PROTOCOL_SELECTOR_ETHER_TYPE UINT32_C(0x1)
	+ /* TCP port */
	+ #define HWRM_STRUCT_DATA_DCBX_APP_PROTOCOL_SELECTOR_TCP_PORT UINT32_C(0x2)
	+ /* UDP port */
	+ #define HWRM_STRUCT_DATA_DCBX_APP_PROTOCOL_SELECTOR_UDP_PORT UINT32_C(0x3)
	+ /* TCP & UDP port */
	+ #define HWRM_STRUCT_DATA_DCBX_APP_PROTOCOL_SELECTOR_TCP_UDP_PORT UINT32_C(0x4)
	+ uint8_t priority;
	+ /* This field indicates application priority. */
	+ uint8_t valid;
	+ /* This field indicates this entry is valid. */
	+ uint8_t unused_0[3];
	+} __attribute__((packed));
	+
	+/* DCBX feature states configuration structure (1058) (8 bytes) */
	+
	+struct hwrm_struct_data_dcbx_feature_state {
	+ uint8_t dcbx_mode;
	+ /* DCBX mode - IEEE or CEE. This is read only field. */
	+ /* DCBX disabled mode. */
	+ #define HWRM_STRUCT_DATA_DCBX_FEATURE_STATE_DCBX_MODE_DCBX_DISABLED UINT32_C(0x0)
	+ /* DCBX IEEE mode. */
	+ #define HWRM_STRUCT_DATA_DCBX_FEATURE_STATE_DCBX_MODE_DCBX_IEEE UINT32_C(0x1)
	+ /* DCBX CEE mode. */
	+ #define HWRM_STRUCT_DATA_DCBX_FEATURE_STATE_DCBX_MODE_DCBX_CEE UINT32_C(0x2)
	+ uint8_t ets_state;
	+ /* ETS TLV state. */
	+ uint8_t pfc_state;
	+ /* PFC TLV state. */
	+ uint8_t app_state;
	+ /* App TLV state. */
	+ /* Feature enable bit position. */
	+ #define HWRM_STRUCT_DATA_DCBX_FEATURE_STATE_APP_STATE_ENABLE_BIT_POS UINT32_C(0x7)
	+ /* Feature willing bit position. */
	+ #define HWRM_STRUCT_DATA_DCBX_FEATURE_STATE_APP_STATE_WILLING_BIT_POS UINT32_C(0x6)
	+ /* Feature advertise bit position. */
	+ #define HWRM_STRUCT_DATA_DCBX_FEATURE_STATE_APP_STATE_ADVERTISE_BIT_POS UINT32_C(0x5)
	+ uint8_t unused_0[3];
	+ /* unused. */
	+ uint8_t resets;
	+ /*
	+ * This field is used to reset the DCBX configuration to factory
	+ * defaults.
	+ */
	+ /* reset ETS configuration. */
	+ #define HWRM_STRUCT_DATA_DCBX_FEATURE_STATE_RESETS_RESET_ETS UINT32_C(0x1)
	+ /* reset PFC configuration. */
	+ #define HWRM_STRUCT_DATA_DCBX_FEATURE_STATE_RESETS_RESET_PFC UINT32_C(0x2)
	+ /* reset application configuration. */
	+ #define HWRM_STRUCT_DATA_DCBX_FEATURE_STATE_RESETS_RESET_APP UINT32_C(0x4)
	+ /* reset DCBX state configuration. */
	+ #define HWRM_STRUCT_DATA_DCBX_FEATURE_STATE_RESETS_RESET_STATE UINT32_C(0x8)
	+} __attribute__((packed));
	+
	+/* LLDP TLVs transmit configuration structure (1051) (8 bytes) */
	+
	+struct hwrm_struct_data_lldp {
	+ uint8_t admin_state;
	+ /* Port admin state */
	+ /* Disable both Tx and Rx */
	+ #define HWRM_STRUCT_DATA_LLDP_ADMIN_STATE_DISABLE UINT32_C(0x0)
	+ /* Enable Tx only */
	+ #define HWRM_STRUCT_DATA_LLDP_ADMIN_STATE_TX UINT32_C(0x1)
	+ /* Enable Rx only */
	+ #define HWRM_STRUCT_DATA_LLDP_ADMIN_STATE_RX UINT32_C(0x2)
	+ /* Enable both Tx and Rx */
	+ #define HWRM_STRUCT_DATA_LLDP_ADMIN_STATE_ENABLE UINT32_C(0x3)
	+ uint8_t port_description_state;
	+ /* Port desciption TLV transmit state (enable(1)/disable(0)). */
	+ /* Disable */
	+ #define HWRM_STRUCT_DATA_LLDP_PORT_DESCRIPTION_STATE_DISABLE UINT32_C(0x0)
	+ /* Enable */
	+ #define HWRM_STRUCT_DATA_LLDP_PORT_DESCRIPTION_STATE_ENABLE UINT32_C(0x1)
	+ uint8_t system_name_state;
	+ /* System name TLV transmit state (enable(1)/disable(0)). */
	+ /* Disable */
	+ #define HWRM_STRUCT_DATA_LLDP_SYSTEM_NAME_STATE_DISABLE UINT32_C(0x0)
	+ /* Enable */
	+ #define HWRM_STRUCT_DATA_LLDP_SYSTEM_NAME_STATE_ENABLE UINT32_C(0x1)
	+ uint8_t system_desc_state;
	+ /* System desciption TLV transmit state (enable(1)/disable(0)). */
	+ /* Disable */
	+ #define HWRM_STRUCT_DATA_LLDP_SYSTEM_DESC_STATE_DISABLE UINT32_C(0x0)
	+ /* Enable */
	+ #define HWRM_STRUCT_DATA_LLDP_SYSTEM_DESC_STATE_ENABLE UINT32_C(0x1)
	+ uint8_t system_cap_state;
	+ /* System capabilities TLV transmit state (enable(1)/disable(0)). */
	+ /* Disable */
	+ #define HWRM_STRUCT_DATA_LLDP_SYSTEM_CAP_STATE_DISABLE UINT32_C(0x0)
	+ /* Enable */
	+ #define HWRM_STRUCT_DATA_LLDP_SYSTEM_CAP_STATE_ENABLE UINT32_C(0x1)
	+ uint8_t mgmt_addr_state;
	+ /* Management address TLV transmit state (enable(1)/disable(0)). */
	+ /* Disable */
	+ #define HWRM_STRUCT_DATA_LLDP_MGMT_ADDR_STATE_DISABLE UINT32_C(0x0)
	+ /* Enable */
	+ #define HWRM_STRUCT_DATA_LLDP_MGMT_ADDR_STATE_ENABLE UINT32_C(0x1)
	+ uint8_t async_event_notification_state;
	+ /* Async event notification state (enable(1)/disable(0)). */
	+ /* Disable */
	+ #define HWRM_STRUCT_DATA_LLDP_ASYNC_EVENT_NOTIFICATION_STATE_DISABLE UINT32_C(0x0)
	+ /* Enable */
	+ #define HWRM_STRUCT_DATA_LLDP_ASYNC_EVENT_NOTIFICATION_STATE_ENABLE UINT32_C(0x1)
	+ uint8_t unused_0;
	+} __attribute__((packed));
	+
	+/* LLDP generic TLV configuration (1060) (16 bytes) */
	+
	+struct hwrm_struct_data_lldp_generic {
	+ uint8_t tlv_type;
	+ /* TLV type. */
	+ /* Chassis ID TLV */
	+ #define HWRM_STRUCT_DATA_LLDP_GENERIC_TLV_TYPE_CHASSIS UINT32_C(0x1)
	+ /* Port ID TLV */
	+ #define HWRM_STRUCT_DATA_LLDP_GENERIC_TLV_TYPE_PORT UINT32_C(0x2)
	+ /* System name TLV */
	+ #define HWRM_STRUCT_DATA_LLDP_GENERIC_TLV_TYPE_SYSTEM_NAME UINT32_C(0x3)
	+ /* System description TLV */
	+ #define HWRM_STRUCT_DATA_LLDP_GENERIC_TLV_TYPE_SYSTEM_DESCRIPTION UINT32_C(0x4)
	+ /* Port name TLV */
	+ #define HWRM_STRUCT_DATA_LLDP_GENERIC_TLV_TYPE_PORT_NAME UINT32_C(0x5)
	+ /* Port description TLV */
	+ #define HWRM_STRUCT_DATA_LLDP_GENERIC_TLV_TYPE_PORT_DESCRIPTION UINT32_C(0x6)
	+ uint8_t subtype;
	+ /* TLV sub-type. */
	+ uint8_t length;
	+ /* Length. */
	+ uint8_t unused_0;
	+ /* unused. */
	+ uint32_t unused_1;
	+ uint32_t tlv_value[64];
	+ /* TLV value. */
	+} __attribute__((packed));
	+
	+/* LLDP device TLV configuration (1062) (64 bytes) */
	+
	+struct hwrm_struct_data_lldp_device {
	+ uint16_t ttl;
	+ /* Time to Live. */
	+ uint8_t mgmt_addr_len;
	+ /* Management address length. */
	+ uint8_t mgmt_addr_type;
	+ /* Management address type. */
	+ uint32_t unused_0;
	+ uint32_t mgmt_addr[8];
	+ /* Management address. */
	+ uint32_t system_caps;
	+ /* System capabilities. */
	+ uint8_t intf_num_type;
	+ /* Interface number type. */
	+ uint8_t mgmt_addr_oid_length;
	+ /* Management address OID length. */
	+ uint8_t unused_1;
	+ uint8_t unused_2;
	+ uint32_t intf_num;
	+ /* Interface number. */
	+ uint32_t unused_3;
	+ uint32_t mgmt_addr_oid[32];
	+ /* Management address OID. */
	+} __attribute__((packed));
	+
	+/* port description (10) (8 bytes) */
	+
	+struct hwrm_struct_data_port_description {
	+ uint8_t port_id;
	+ /*
	+ * Port #. Port number starts at 0 and anything greater than number of
	+ * ports minus 1 is an error.
	+ */
	+ uint8_t unused_0[7];
	+} __attribute__((packed));
	+
	+/* RSSv2 Configuration (100) (16 bytes) */
	+
	+struct hwrm_struct_data_rss_v2 {
	+ uint16_t flags;
	+ /* When this bit is '1', the hash type and hash key are included. */
	+ #define HWRM_STRUCT_DATA_RSS_V2_FLAGS_HASH_VALID UINT32_C(0x1)
	+ uint16_t rss_ctx_id;
	+ /* RSS Context index. */
	+ uint16_t num_ring_groups;
	+ /* Number ring group IDs. */
	+ uint16_t hash_type;
	+ /*
	+ * When this bit is '1', the RSS hash shall be computed over source and
	+ * destination IPv4 addresses of IPv4 packets.
	+ */
	+ #define HWRM_STRUCT_DATA_RSS_V2_HASH_TYPE_IPV4 UINT32_C(0x1)
	+ /*
	+ * When this bit is '1', the RSS hash shall be computed over
	+ * source/destination IPv4 addresses and source/destination ports of
	+ * TCP/IPv4 packets.
	+ */
	+ #define HWRM_STRUCT_DATA_RSS_V2_HASH_TYPE_TCP_IPV4 UINT32_C(0x2)
	+ /*
	+ * When this bit is '1', the RSS hash shall be computed over
	+ * source/destination IPv4 addresses and source/destination ports of
	+ * UDP/IPv4 packets.
	+ */
	+ #define HWRM_STRUCT_DATA_RSS_V2_HASH_TYPE_UDP_IPV4 UINT32_C(0x4)
	+ /*
	+ * When this bit is '1', the RSS hash shall be computed over source and
	+ * destination IPv4 addresses of IPv6 packets.
	+ */
	+ #define HWRM_STRUCT_DATA_RSS_V2_HASH_TYPE_IPV6 UINT32_C(0x8)
	+ /*
	+ * When this bit is '1', the RSS hash shall be computed over
	+ * source/destination IPv6 addresses and source/destination ports of
	+ * TCP/IPv6 packets.
	+ */
	+ #define HWRM_STRUCT_DATA_RSS_V2_HASH_TYPE_TCP_IPV6 UINT32_C(0x10)
	+ /*
	+ * When this bit is '1', the RSS hash shall be computed over
	+ * source/destination IPv6 addresses and source/destination ports of
	+ * UDP/IPv6 packets.
	+ */
	+ #define HWRM_STRUCT_DATA_RSS_V2_HASH_TYPE_UDP_IPV6 UINT32_C(0x20)
	+ uint64_t hash_key_ring_group_ids;
	+ /* Variable size data. Hash key (optional) followed by ring_group_ids. */
	} __attribute__((packed));

	#endif /* _HSI_STRUCT_DEF_EXTERNAL_H_ */
	Index: sys/dev/bnxt/if_bnxt.c
	===================================================================
	--- sys/dev/bnxt/if_bnxt.c
	+++ sys/dev/bnxt/if_bnxt.c
	@@ -176,10 +176,12 @@
	static int bnxt_promisc_set(if_ctx_t ctx, int flags);
	static uint64_t bnxt_get_counter(if_ctx_t, ift_counter);
	static void bnxt_update_admin_status(if_ctx_t ctx);
	+static void bnxt_if_timer(if_ctx_t ctx, uint16_t qid);

	/* Interrupt enable / disable */
	static void bnxt_intr_enable(if_ctx_t ctx);
	-static int bnxt_queue_intr_enable(if_ctx_t ctx, uint16_t qid);
	+static int bnxt_rx_queue_intr_enable(if_ctx_t ctx, uint16_t qid);
	+static int bnxt_tx_queue_intr_enable(if_ctx_t ctx, uint16_t qid);
	static void bnxt_disable_intr(if_ctx_t ctx);
	static int bnxt_msix_intr_assign(if_ctx_t ctx, int msix);

	@@ -190,6 +192,10 @@
	/* ioctl */
	static int bnxt_priv_ioctl(if_ctx_t ctx, u_long command, caddr_t data);

	+static int bnxt_shutdown(if_ctx_t ctx);
	+static int bnxt_suspend(if_ctx_t ctx);
	+static int bnxt_resume(if_ctx_t ctx);
	+
	/* Internal support functions */
	static int bnxt_probe_phy(struct bnxt_softc *softc);
	static void bnxt_add_media_types(struct bnxt_softc *softc);
	@@ -207,6 +213,8 @@
	struct cmpl_base *cmpl);
	static uint8_t get_phy_type(struct bnxt_softc *softc);
	static uint64_t bnxt_get_baudrate(struct bnxt_link_info *link);
	+static void bnxt_get_wol_settings(struct bnxt_softc *softc);
	+static int bnxt_wol_config(if_ctx_t ctx);

	/*
	* Device Interface Declaration
	@@ -235,6 +243,8 @@
	MODULE_DEPEND(bnxt, ether, 1, 1, 1);
	MODULE_DEPEND(bnxt, iflib, 1, 1, 1);

	+IFLIB_PNP_INFO(pci, bnxt, bnxt_vendor_info_array);
	+
	static device_method_t bnxt_iflib_methods[] = {
	DEVMETHOD(ifdi_tx_queues_alloc, bnxt_tx_queues_alloc),
	DEVMETHOD(ifdi_rx_queues_alloc, bnxt_rx_queues_alloc),
	@@ -253,9 +263,11 @@
	DEVMETHOD(ifdi_promisc_set, bnxt_promisc_set),
	DEVMETHOD(ifdi_get_counter, bnxt_get_counter),
	DEVMETHOD(ifdi_update_admin_status, bnxt_update_admin_status),
	+ DEVMETHOD(ifdi_timer, bnxt_if_timer),

	DEVMETHOD(ifdi_intr_enable, bnxt_intr_enable),
	- DEVMETHOD(ifdi_queue_intr_enable, bnxt_queue_intr_enable),
	+ DEVMETHOD(ifdi_tx_queue_intr_enable, bnxt_tx_queue_intr_enable),
	+ DEVMETHOD(ifdi_rx_queue_intr_enable, bnxt_rx_queue_intr_enable),
	DEVMETHOD(ifdi_intr_disable, bnxt_disable_intr),
	DEVMETHOD(ifdi_msix_intr_assign, bnxt_msix_intr_assign),

	@@ -264,6 +276,10 @@

	DEVMETHOD(ifdi_priv_ioctl, bnxt_priv_ioctl),

	+ DEVMETHOD(ifdi_suspend, bnxt_suspend),
	+ DEVMETHOD(ifdi_shutdown, bnxt_shutdown),
	+ DEVMETHOD(ifdi_resume, bnxt_resume),
	+
	DEVMETHOD_END
	};

	@@ -275,14 +291,14 @@
	* iflib shared context
	*/

	-char bnxt_driver_version[] = "FreeBSD base";
	+#define BNXT_DRIVER_VERSION "1.0.0.2"
	+char bnxt_driver_version[] = BNXT_DRIVER_VERSION;
	extern struct if_txrx bnxt_txrx;
	static struct if_shared_ctx bnxt_sctx_init = {
	.isc_magic = IFLIB_MAGIC,
	- .isc_txrx = &bnxt_txrx,
	.isc_driver = &bnxt_iflib_driver,
	.isc_nfl = 2, // Number of Free Lists
	- .isc_flags = IFLIB_HAS_RXCQ \| IFLIB_HAS_TXCQ,
	+ .isc_flags = IFLIB_HAS_RXCQ \| IFLIB_HAS_TXCQ \| IFLIB_NEED_ETHER_PAD,
	.isc_q_align = PAGE_SIZE,
	.isc_tx_maxsize = BNXT_TSO_SIZE,
	.isc_tx_maxsegsize = BNXT_TSO_SIZE,
	@@ -412,6 +428,8 @@

	// Free RX queues
	iflib_dma_free(&softc->rx_stats);
	+ iflib_dma_free(&softc->hw_tx_port_stats);
	+ iflib_dma_free(&softc->hw_rx_port_stats);
	free(softc->grp_info, M_DEVBUF);
	free(softc->ag_rings, M_DEVBUF);
	free(softc->rx_rings, M_DEVBUF);
	@@ -468,6 +486,33 @@
	bus_dmamap_sync(softc->rx_stats.idi_tag, softc->rx_stats.idi_map,
	BUS_DMASYNC_PREREAD);

	+/*
	+ * Additional 512 bytes for future expansion.
	+ * To prevent corruption when loaded with newer firmwares with added counters.
	+ * This can be deleted when there will be no further additions of counters.
	+ */
	+#define BNXT_PORT_STAT_PADDING 512
	+
	+ rc = iflib_dma_alloc(ctx, sizeof(struct rx_port_stats) + BNXT_PORT_STAT_PADDING,
	+ &softc->hw_rx_port_stats, 0);
	+ if (rc)
	+ goto hw_port_rx_stats_alloc_fail;
	+
	+ bus_dmamap_sync(softc->hw_rx_port_stats.idi_tag,
	+ softc->hw_rx_port_stats.idi_map, BUS_DMASYNC_PREREAD);
	+
	+ rc = iflib_dma_alloc(ctx, sizeof(struct tx_port_stats) + BNXT_PORT_STAT_PADDING,
	+ &softc->hw_tx_port_stats, 0);
	+
	+ if (rc)
	+ goto hw_port_tx_stats_alloc_fail;
	+
	+ bus_dmamap_sync(softc->hw_tx_port_stats.idi_tag,
	+ softc->hw_tx_port_stats.idi_map, BUS_DMASYNC_PREREAD);
	+
	+ softc->rx_port_stats = (void *) softc->hw_rx_port_stats.idi_vaddr;
	+ softc->tx_port_stats = (void *) softc->hw_tx_port_stats.idi_vaddr;
	+
	for (i = 0; i < nrxqsets; i++) {
	/* Allocation the completion ring */
	softc->rx_cp_rings[i].stats_ctx_id = HWRM_NA_SIGNATURE;
	@@ -494,6 +539,17 @@
	softc->rx_rings[i].vaddr = vaddrs[i * nrxqs + 1];
	softc->rx_rings[i].paddr = paddrs[i * nrxqs + 1];

	+ /* Allocate the TPA start buffer */
	+ softc->rx_rings[i].tpa_start = malloc(sizeof(struct bnxt_full_tpa_start) *
	+ (RX_TPA_START_CMPL_AGG_ID_MASK >> RX_TPA_START_CMPL_AGG_ID_SFT),
	+ M_DEVBUF, M_NOWAIT \| M_ZERO);
	+ if (softc->rx_rings[i].tpa_start == NULL) {
	+ rc = -ENOMEM;
	+ device_printf(softc->dev,
	+ "Unable to allocate space for TPA\n");
	+ goto tpa_alloc_fail;
	+ }
	+
	/* Allocate the AG ring */
	softc->ag_rings[i].phys_id = (uint16_t)HWRM_NA_SIGNATURE;
	softc->ag_rings[i].softc = softc;
	@@ -515,6 +571,13 @@
	bnxt_create_rx_sysctls(softc, i);
	}

	+ /*
	+ * When SR-IOV is enabled, avoid each VF sending PORT_QSTATS
	+ * HWRM every sec with which firmware timeouts can happen
	+ */
	+ if (BNXT_PF(softc))
	+ bnxt_create_port_stats_sysctls(softc);
	+
	/* And finally, the VNIC */
	softc->vnic_info.id = (uint16_t)HWRM_NA_SIGNATURE;
	softc->vnic_info.flow_id = (uint16_t)HWRM_NA_SIGNATURE;
	@@ -559,7 +622,14 @@
	iflib_dma_free(&softc->vnic_info.rss_hash_key_tbl);
	rss_hash_alloc_fail:
	iflib_dma_free(&softc->vnic_info.mc_list);
	+tpa_alloc_fail:
	mc_list_alloc_fail:
	+ for (i = i - 1; i >= 0; i--)
	+ free(softc->rx_rings[i].tpa_start, M_DEVBUF);
	+ iflib_dma_free(&softc->hw_tx_port_stats);
	+hw_port_tx_stats_alloc_fail:
	+ iflib_dma_free(&softc->hw_rx_port_stats);
	+hw_port_rx_stats_alloc_fail:
	iflib_dma_free(&softc->rx_stats);
	hw_stats_alloc_fail:
	free(softc->grp_info, M_DEVBUF);
	@@ -573,6 +643,23 @@
	return rc;
	}

	+static void bnxt_free_hwrm_short_cmd_req(struct bnxt_softc *softc)
	+{
	+ if (softc->hwrm_short_cmd_req_addr.idi_vaddr)
	+ iflib_dma_free(&softc->hwrm_short_cmd_req_addr);
	+ softc->hwrm_short_cmd_req_addr.idi_vaddr = NULL;
	+}
	+
	+static int bnxt_alloc_hwrm_short_cmd_req(struct bnxt_softc *softc)
	+{
	+ int rc;
	+
	+ rc = iflib_dma_alloc(softc->ctx, softc->hwrm_max_req_len,
	+ &softc->hwrm_short_cmd_req_addr, BUS_DMA_NOWAIT);
	+
	+ return rc;
	+}
	+
	/* Device setup and teardown */
	static int
	bnxt_attach_pre(if_ctx_t ctx)
	@@ -589,7 +676,7 @@
	scctx = softc->scctx;

	/* TODO: Better way of detecting NPAR/VF is needed */
	- switch (softc->sctx->isc_vendor_info->pvi_device_id) {
	+ switch (pci_get_device(softc->dev)) {
	case BCM57402_NPAR:
	case BCM57404_NPAR:
	case BCM57406_NPAR:
	@@ -623,16 +710,6 @@
	if (rc)
	goto dma_fail;

	- /* Allocate the TPA start buffer */
	- softc->tpa_start = malloc(sizeof(struct bnxt_full_tpa_start) *
	- (RX_TPA_START_CMPL_AGG_ID_MASK >> RX_TPA_START_CMPL_AGG_ID_SFT),
	- M_DEVBUF, M_NOWAIT \| M_ZERO);
	- if (softc->tpa_start == NULL) {
	- rc = ENOMEM;
	- device_printf(softc->dev,
	- "Unable to allocate space for TPA\n");
	- goto tpa_failed;
	- }

	/* Get firmware version and compare with driver */
	softc->ver_info = malloc(sizeof(struct bnxt_ver_info),
	@@ -654,19 +731,28 @@
	goto ver_fail;
	}

	+ if (softc->flags & BNXT_FLAG_SHORT_CMD) {
	+ rc = bnxt_alloc_hwrm_short_cmd_req(softc);
	+ if (rc)
	+ goto hwrm_short_cmd_alloc_fail;
	+ }
	+
	/* Get NVRAM info */
	- softc->nvm_info = malloc(sizeof(struct bnxt_nvram_info),
	- M_DEVBUF, M_NOWAIT \| M_ZERO);
	- if (softc->nvm_info == NULL) {
	- rc = ENOMEM;
	- device_printf(softc->dev,
	- "Unable to allocate space for NVRAM info\n");
	- goto nvm_alloc_fail;
	+ if (BNXT_PF(softc)) {
	+ softc->nvm_info = malloc(sizeof(struct bnxt_nvram_info),
	+ M_DEVBUF, M_NOWAIT \| M_ZERO);
	+ if (softc->nvm_info == NULL) {
	+ rc = ENOMEM;
	+ device_printf(softc->dev,
	+ "Unable to allocate space for NVRAM info\n");
	+ goto nvm_alloc_fail;
	+ }
	+
	+ rc = bnxt_hwrm_nvm_get_dev_info(softc, &softc->nvm_info->mfg_id,
	+ &softc->nvm_info->device_id, &softc->nvm_info->sector_size,
	+ &softc->nvm_info->size, &softc->nvm_info->reserved_size,
	+ &softc->nvm_info->available_size);
	}
	- rc = bnxt_hwrm_nvm_get_dev_info(softc, &softc->nvm_info->mfg_id,
	- &softc->nvm_info->device_id, &softc->nvm_info->sector_size,
	- &softc->nvm_info->size, &softc->nvm_info->reserved_size,
	- &softc->nvm_info->available_size);

	/* Register the driver with the FW */
	rc = bnxt_hwrm_func_drv_rgtr(softc);
	@@ -675,12 +761,43 @@
	goto drv_rgtr_fail;
	}

	+ rc = bnxt_hwrm_func_rgtr_async_events(softc, NULL, 0);
	+ if (rc) {
	+ device_printf(softc->dev, "attach: hwrm rgtr async evts failed\n");
	+ goto drv_rgtr_fail;
	+ }
	+
	/* Get the HW capabilities */
	rc = bnxt_hwrm_func_qcaps(softc);
	if (rc)
	goto failed;
	+
	+ /* Get the current configuration of this function */
	+ rc = bnxt_hwrm_func_qcfg(softc);
	+ if (rc) {
	+ device_printf(softc->dev, "attach: hwrm func qcfg failed\n");
	+ goto failed;
	+ }
	+
	iflib_set_mac(ctx, softc->func.mac_addr);

	+ scctx->isc_txrx = &bnxt_txrx;
	+ scctx->isc_tx_csum_flags = (CSUM_IP \| CSUM_TCP \| CSUM_UDP \|
	+ CSUM_TCP_IPV6 \| CSUM_UDP_IPV6 \| CSUM_TSO);
	+ scctx->isc_capenable =
	+ /* These are translated to hwassit bits */
	+ IFCAP_TXCSUM \| IFCAP_TXCSUM_IPV6 \| IFCAP_TSO4 \| IFCAP_TSO6 \|
	+ /* These are checked by iflib */
	+ IFCAP_LRO \| IFCAP_VLAN_HWFILTER \|
	+ /* These are part of the iflib mask */
	+ IFCAP_RXCSUM \| IFCAP_RXCSUM_IPV6 \| IFCAP_VLAN_MTU \|
	+ IFCAP_VLAN_HWTAGGING \| IFCAP_VLAN_HWTSO \|
	+ /* These likely get lost... */
	+ IFCAP_VLAN_HWCSUM \| IFCAP_JUMBO_MTU;
	+
	+ if (bnxt_wol_supported(softc))
	+ scctx->isc_capenable \|= IFCAP_WOL_MAGIC;
	+
	/* Get the queue config */
	rc = bnxt_hwrm_queue_qportcfg(softc);
	if (rc) {
	@@ -688,6 +805,8 @@
	goto failed;
	}

	+ bnxt_get_wol_settings(softc);
	+
	/* Now perform a function reset */
	rc = bnxt_hwrm_func_reset(softc);
	bnxt_clear_ids(softc);
	@@ -700,6 +819,9 @@
	scctx->isc_tx_tso_size_max = BNXT_TSO_SIZE;
	scctx->isc_tx_tso_segsize_max = BNXT_TSO_SIZE;
	scctx->isc_vectors = softc->func.max_cp_rings;
	+ scctx->isc_min_frame_size = BNXT_MIN_FRAME_SIZE;
	+ scctx->isc_txrx = &bnxt_txrx;
	+
	if (scctx->isc_nrxd[0] <
	((scctx->isc_nrxd[1] * 4) + scctx->isc_nrxd[2]))
	device_printf(softc->dev,
	@@ -717,18 +839,33 @@
	scctx->isc_nrxd[1];
	scctx->isc_rxqsizes[2] = sizeof(struct rx_prod_pkt_bd) *
	scctx->isc_nrxd[2];
	- scctx->isc_max_rxqsets = min(pci_msix_count(softc->dev)-1,
	- softc->func.max_cp_rings - 1);
	- scctx->isc_max_rxqsets = min(scctx->isc_max_rxqsets,
	- softc->func.max_rx_rings);
	- scctx->isc_max_txqsets = min(softc->func.max_rx_rings,
	- softc->func.max_cp_rings - scctx->isc_max_rxqsets - 1);
	+
	+ scctx->isc_nrxqsets_max = min(pci_msix_count(softc->dev)-1,
	+ softc->fn_qcfg.alloc_completion_rings - 1);
	+ scctx->isc_nrxqsets_max = min(scctx->isc_nrxqsets_max,
	+ softc->fn_qcfg.alloc_rx_rings);
	+ scctx->isc_nrxqsets_max = min(scctx->isc_nrxqsets_max,
	+ softc->fn_qcfg.alloc_vnics);
	+ scctx->isc_ntxqsets_max = min(softc->fn_qcfg.alloc_tx_rings,
	+ softc->fn_qcfg.alloc_completion_rings - scctx->isc_nrxqsets_max - 1);
	+
	scctx->isc_rss_table_size = HW_HASH_INDEX_SIZE;
	scctx->isc_rss_table_mask = scctx->isc_rss_table_size - 1;

	/* iflib will map and release this bar */
	scctx->isc_msix_bar = pci_msix_table_bar(softc->dev);

	+ /*
	+ * Default settings for HW LRO (TPA):
	+ * Disable HW LRO by default
	+ * Can be enabled after taking care of 'packet forwarding'
	+ */
	+ softc->hw_lro.enable = 0;
	+ softc->hw_lro.is_mode_gro = 0;
	+ softc->hw_lro.max_agg_segs = 5; /* 2^5 = 32 segs */
	+ softc->hw_lro.max_aggs = HWRM_VNIC_TPA_CFG_INPUT_MAX_AGGS_MAX;
	+ softc->hw_lro.min_agg_len = 512;
	+
	/* Allocate the default completion ring */
	softc->def_cp_ring.stats_ctx_id = HWRM_NA_SIGNATURE;
	softc->def_cp_ring.ring.phys_id = (uint16_t)HWRM_NA_SIGNATURE;
	@@ -748,9 +885,11 @@
	rc = bnxt_init_sysctl_ctx(softc);
	if (rc)
	goto init_sysctl_failed;
	- rc = bnxt_create_nvram_sysctls(softc->nvm_info);
	- if (rc)
	- goto failed;
	+ if (BNXT_PF(softc)) {
	+ rc = bnxt_create_nvram_sysctls(softc->nvm_info);
	+ if (rc)
	+ goto failed;
	+ }

	arc4rand(softc->vnic_info.rss_hash_key, HW_HASH_KEY_SIZE, 0);
	softc->vnic_info.rss_hash_type =
	@@ -764,6 +903,14 @@
	if (rc)
	goto failed;

	+ rc = bnxt_create_hw_lro_sysctls(softc);
	+ if (rc)
	+ goto failed;
	+
	+ rc = bnxt_create_pause_fc_sysctls(softc);
	+ if (rc)
	+ goto failed;
	+
	/* Initialize the vlan list */
	SLIST_INIT(&softc->vnic_info.vlan_tags);
	softc->vnic_info.vlan_tag_list.idi_vaddr = NULL;
	@@ -775,13 +922,14 @@
	init_sysctl_failed:
	bnxt_hwrm_func_drv_unrgtr(softc, false);
	drv_rgtr_fail:
	- free(softc->nvm_info, M_DEVBUF);
	+ if (BNXT_PF(softc))
	+ free(softc->nvm_info, M_DEVBUF);
	nvm_alloc_fail:
	+ bnxt_free_hwrm_short_cmd_req(softc);
	+hwrm_short_cmd_alloc_fail:
	ver_fail:
	free(softc->ver_info, M_DEVBUF);
	ver_alloc_fail:
	- free(softc->tpa_start, M_DEVBUF);
	-tpa_failed:
	bnxt_free_hwrm_dma_mem(softc);
	dma_fail:
	BNXT_HWRM_LOCK_DESTROY(softc);
	@@ -795,7 +943,6 @@
	{
	struct bnxt_softc *softc = iflib_get_softc(ctx);
	if_t ifp = iflib_get_ifp(ctx);
	- int capabilities, enabling;
	int rc;

	bnxt_create_config_sysctls_post(softc);
	@@ -810,26 +957,6 @@
	bnxt_add_media_types(softc);
	ifmedia_set(softc->media, IFM_ETHER \| IFM_AUTO);

	- if_sethwassist(ifp, (CSUM_TCP \| CSUM_UDP \| CSUM_TCP_IPV6 \|
	- CSUM_UDP_IPV6 \| CSUM_TSO));
	-
	- capabilities =
	- /* These are translated to hwassit bits */
	- IFCAP_TXCSUM \| IFCAP_TXCSUM_IPV6 \| IFCAP_TSO4 \| IFCAP_TSO6 \|
	- /* These are checked by iflib */
	- IFCAP_LRO \| IFCAP_VLAN_HWFILTER \|
	- /* These are part of the iflib mask */
	- IFCAP_RXCSUM \| IFCAP_RXCSUM_IPV6 \| IFCAP_VLAN_MTU \|
	- IFCAP_VLAN_HWTAGGING \| IFCAP_VLAN_HWTSO \|
	- /* These likely get lost... */
	- IFCAP_VLAN_HWCSUM \| IFCAP_JUMBO_MTU;
	-
	- if_setcapabilities(ifp, capabilities);
	-
	- enabling = capabilities;
	-
	- if_setcapenable(ifp, enabling);
	-
	softc->scctx->isc_max_frame_size = ifp->if_mtu + ETHER_HDR_LEN +
	ETHER_CRC_LEN;

	@@ -845,6 +972,7 @@
	struct bnxt_vlan_tag *tmp;
	int i;

	+ bnxt_wol_config(ctx);
	bnxt_do_disable_intr(&softc->def_cp_ring);
	bnxt_free_sysctl_ctx(softc);
	bnxt_hwrm_func_reset(softc);
	@@ -863,12 +991,15 @@
	SLIST_FOREACH_SAFE(tag, &softc->vnic_info.vlan_tags, next, tmp)
	free(tag, M_DEVBUF);
	iflib_dma_free(&softc->def_cp_ring_mem);
	- free(softc->tpa_start, M_DEVBUF);
	+ for (i = 0; i < softc->nrxqsets; i++)
	+ free(softc->rx_rings[i].tpa_start, M_DEVBUF);
	free(softc->ver_info, M_DEVBUF);
	- free(softc->nvm_info, M_DEVBUF);
	+ if (BNXT_PF(softc))
	+ free(softc->nvm_info, M_DEVBUF);

	bnxt_hwrm_func_drv_unrgtr(softc, false);
	bnxt_free_hwrm_dma_mem(softc);
	+ bnxt_free_hwrm_short_cmd_req(softc);
	BNXT_HWRM_LOCK_DESTROY(softc);

	pci_disable_busmaster(softc->dev);
	@@ -896,7 +1027,7 @@
	softc->def_cp_ring.v_bit = 1;
	bnxt_mark_cpr_invalid(&softc->def_cp_ring);
	rc = bnxt_hwrm_ring_alloc(softc,
	- HWRM_RING_ALLOC_INPUT_RING_TYPE_CMPL,
	+ HWRM_RING_ALLOC_INPUT_RING_TYPE_L2_CMPL,
	&softc->def_cp_ring.ring,
	(uint16_t)HWRM_NA_SIGNATURE,
	HWRM_NA_SIGNATURE, true);
	@@ -904,7 +1035,7 @@
	goto fail;

	/* And now set the default CP ring as the async CP ring */
	- rc = bnxt_hwrm_func_cfg(softc);
	+ rc = bnxt_cfg_async_cr(softc);
	if (rc)
	goto fail;

	@@ -922,7 +1053,7 @@
	softc->rx_cp_rings[i].last_idx = UINT32_MAX;
	bnxt_mark_cpr_invalid(&softc->rx_cp_rings[i]);
	rc = bnxt_hwrm_ring_alloc(softc,
	- HWRM_RING_ALLOC_INPUT_RING_TYPE_CMPL,
	+ HWRM_RING_ALLOC_INPUT_RING_TYPE_L2_CMPL,
	&softc->rx_cp_rings[i].ring, (uint16_t)HWRM_NA_SIGNATURE,
	HWRM_NA_SIGNATURE, true);
	if (rc)
	@@ -995,14 +1126,9 @@
	if (rc)
	goto fail;

	-#ifdef notyet
	- /* Enable LRO/TPA/GRO */
	- rc = bnxt_hwrm_vnic_tpa_cfg(softc, &softc->vnic_info,
	- (if_getcapenable(iflib_get_ifp(ctx)) & IFCAP_LRO) ?
	- HWRM_VNIC_TPA_CFG_INPUT_FLAGS_TPA : 0);
	+ rc = bnxt_hwrm_vnic_tpa_cfg(softc);
	if (rc)
	goto fail;
	-#endif

	for (i = 0; i < softc->ntxqsets; i++) {
	/* Allocate the statistics context */
	@@ -1017,7 +1143,7 @@
	softc->tx_cp_rings[i].v_bit = 1;
	bnxt_mark_cpr_invalid(&softc->tx_cp_rings[i]);
	rc = bnxt_hwrm_ring_alloc(softc,
	- HWRM_RING_ALLOC_INPUT_RING_TYPE_CMPL,
	+ HWRM_RING_ALLOC_INPUT_RING_TYPE_L2_CMPL,
	&softc->tx_cp_rings[i].ring, (uint16_t)HWRM_NA_SIGNATURE,
	HWRM_NA_SIGNATURE, false);
	if (rc)
	@@ -1037,6 +1163,7 @@

	bnxt_do_enable_intr(&softc->def_cp_ring);
	bnxt_media_status(softc->ctx, &ifmr);
	+ bnxt_hwrm_cfa_l2_set_rx_mask(softc, &softc->vnic_info);
	return;

	fail:
	@@ -1105,8 +1232,11 @@
	{
	struct bnxt_softc *softc = iflib_get_softc(ctx);
	struct bnxt_link_info *link_info = &softc->link_info;
	- uint8_t phy_type = get_phy_type(softc);
	+ struct ifmedia_entry *next;
	+ uint64_t target_baudrate = bnxt_get_baudrate(link_info);
	+ int active_media = IFM_UNKNOWN;

	+
	bnxt_update_link(softc, true);

	ifmr->ifm_status = IFM_AVALID;
	@@ -1117,159 +1247,27 @@
	else
	ifmr->ifm_status &= ~IFM_ACTIVE;

	- if (link_info->duplex == HWRM_PORT_PHY_QCFG_OUTPUT_DUPLEX_FULL)
	+ if (link_info->duplex == HWRM_PORT_PHY_QCFG_OUTPUT_DUPLEX_CFG_FULL)
	ifmr->ifm_active \|= IFM_FDX;
	else
	ifmr->ifm_active \|= IFM_HDX;

	- switch (link_info->link_speed) {
	- case HWRM_PORT_PHY_QCFG_OUTPUT_LINK_SPEED_100MB:
	- ifmr->ifm_active \|= IFM_100_T;
	- break;
	- case HWRM_PORT_PHY_QCFG_OUTPUT_LINK_SPEED_1GB:
	- switch (phy_type) {
	- case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASEKX:
	- ifmr->ifm_active \|= IFM_1000_KX;
	+ /*
	+ * Go through the list of supported media which got prepared
	+ * as part of bnxt_add_media_types() using api ifmedia_add().
	+ */
	+ LIST_FOREACH(next, &(iflib_get_media(ctx)->ifm_list), ifm_list) {
	+ if (ifmedia_baudrate(next->ifm_media) == target_baudrate) {
	+ active_media = next->ifm_media;
	break;
	- case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASET:
	- ifmr->ifm_active \|= IFM_1000_T;
	- break;
	- case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_SGMIIEXTPHY:
	- ifmr->ifm_active \|= IFM_1000_SGMII;
	- break;
	- default:
	- ifmr->ifm_active \|= IFM_UNKNOWN;
	- break;
	}
	- break;
	- case HWRM_PORT_PHY_QCFG_OUTPUT_LINK_SPEED_2_5GB:
	- switch (phy_type) {
	- case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASEKX:
	- ifmr->ifm_active \|= IFM_2500_KX;
	- break;
	- case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASET:
	- ifmr->ifm_active \|= IFM_2500_T;
	- break;
	- default:
	- ifmr->ifm_active \|= IFM_UNKNOWN;
	- break;
	- }
	- break;
	- case HWRM_PORT_PHY_QCFG_OUTPUT_LINK_SPEED_10GB:
	- switch (phy_type) {
	- case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASECR:
	- ifmr->ifm_active \|= IFM_10G_CR1;
	- break;
	- case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASEKR4:
	- case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASEKR2:
	- case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASEKR:
	- ifmr->ifm_active \|= IFM_10G_KR;
	- break;
	- case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASELR:
	- ifmr->ifm_active \|= IFM_10G_LR;
	- break;
	- case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASESR:
	- ifmr->ifm_active \|= IFM_10G_SR;
	- break;
	- case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASEKX:
	- ifmr->ifm_active \|= IFM_10G_KX4;
	- break;
	- case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASET:
	- ifmr->ifm_active \|= IFM_10G_T;
	- break;
	- default:
	- ifmr->ifm_active \|= IFM_UNKNOWN;
	- break;
	- }
	- break;
	- case HWRM_PORT_PHY_QCFG_OUTPUT_LINK_SPEED_20GB:
	- ifmr->ifm_active \|= IFM_20G_KR2;
	- break;
	- case HWRM_PORT_PHY_QCFG_OUTPUT_LINK_SPEED_25GB:
	- switch (phy_type) {
	- case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASECR:
	- ifmr->ifm_active \|= IFM_25G_CR;
	- break;
	- case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASEKR4:
	- case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASEKR2:
	- case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASEKR:
	- ifmr->ifm_active \|= IFM_25G_KR;
	- break;
	- case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASESR:
	- ifmr->ifm_active \|= IFM_25G_SR;
	- break;
	- default:
	- ifmr->ifm_active \|= IFM_UNKNOWN;
	- break;
	- }
	- break;
	- case HWRM_PORT_PHY_QCFG_OUTPUT_LINK_SPEED_40GB:
	- switch (phy_type) {
	- case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASECR:
	- ifmr->ifm_active \|= IFM_40G_CR4;
	- break;
	- case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASEKR4:
	- case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASEKR2:
	- case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASEKR:
	- ifmr->ifm_active \|= IFM_40G_KR4;
	- break;
	- case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASELR:
	- ifmr->ifm_active \|= IFM_40G_LR4;
	- break;
	- case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASESR:
	- ifmr->ifm_active \|= IFM_40G_SR4;
	- break;
	- default:
	- ifmr->ifm_active \|= IFM_UNKNOWN;
	- break;
	- }
	- break;
	- case HWRM_PORT_PHY_QCFG_OUTPUT_LINK_SPEED_50GB:
	- switch (phy_type) {
	- case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASECR:
	- ifmr->ifm_active \|= IFM_50G_CR2;
	- break;
	- case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASEKR4:
	- case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASEKR2:
	- case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASEKR:
	- ifmr->ifm_active \|= IFM_50G_KR2;
	- break;
	- default:
	- ifmr->ifm_active \|= IFM_UNKNOWN;
	- break;
	- }
	- break;
	- case HWRM_PORT_PHY_QCFG_OUTPUT_LINK_SPEED_100GB:
	- switch (phy_type) {
	- case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASECR:
	- ifmr->ifm_active \|= IFM_100G_CR4;
	- break;
	- case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASEKR4:
	- case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASEKR2:
	- case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASEKR:
	- ifmr->ifm_active \|= IFM_100G_KR4;
	- break;
	- case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASELR:
	- ifmr->ifm_active \|= IFM_100G_LR4;
	- break;
	- case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASESR:
	- ifmr->ifm_active \|= IFM_100G_SR4;
	- break;
	- default:
	- ifmr->ifm_active \|= IFM_UNKNOWN;
	- break;
	- }
	- default:
	- return;
	}
	+ ifmr->ifm_active \|= active_media;

	- if (link_info->pause == (HWRM_PORT_PHY_QCFG_OUTPUT_PAUSE_TX \|
	- HWRM_PORT_PHY_QCFG_OUTPUT_PAUSE_RX))
	- ifmr->ifm_active \|= (IFM_ETH_RXPAUSE \| IFM_ETH_TXPAUSE);
	- else if (link_info->pause == HWRM_PORT_PHY_QCFG_OUTPUT_PAUSE_TX)
	- ifmr->ifm_active \|= IFM_ETH_TXPAUSE;
	- else if (link_info->pause == HWRM_PORT_PHY_QCFG_OUTPUT_PAUSE_RX)
	+ if (link_info->flow_ctrl.rx)
	ifmr->ifm_active \|= IFM_ETH_RXPAUSE;
	+ if (link_info->flow_ctrl.tx)
	+ ifmr->ifm_active \|= IFM_ETH_TXPAUSE;

	bnxt_report_link(softc);
	return;
	@@ -1357,7 +1355,7 @@
	softc->link_info.autoneg \|= BNXT_AUTONEG_SPEED;
	break;
	}
	- rc = bnxt_hwrm_set_link_setting(softc, true, true);
	+ rc = bnxt_hwrm_set_link_setting(softc, true, true, true);
	bnxt_media_status(softc->ctx, &ifmr);
	return rc;
	}
	@@ -1405,10 +1403,35 @@
	static void
	bnxt_update_admin_status(if_ctx_t ctx)
	{
	- /* TODO: do we need to do anything here? */
	+ struct bnxt_softc *softc = iflib_get_softc(ctx);
	+
	+ /*
	+ * When SR-IOV is enabled, avoid each VF sending this HWRM
	+ * request every sec with which firmware timeouts can happen
	+ */
	+ if (BNXT_PF(softc)) {
	+ bnxt_hwrm_port_qstats(softc);
	+ }
	+
	return;
	}

	+static void
	+bnxt_if_timer(if_ctx_t ctx, uint16_t qid)
	+{
	+
	+ struct bnxt_softc *softc = iflib_get_softc(ctx);
	+ uint64_t ticks_now = ticks;
	+
	+ /* Schedule bnxt_update_admin_status() once per sec */
	+ if (ticks_now - softc->admin_ticks >= hz) {
	+ softc->admin_ticks = ticks_now;
	+ iflib_admin_intr_deferred(ctx);
	+ }
	+
	+ return;
	+}
	+
	static void inline
	bnxt_do_enable_intr(struct bnxt_cp_ring *cpr)
	{
	@@ -1444,10 +1467,19 @@

	/* Enable interrupt for a single queue */
	static int
	-bnxt_queue_intr_enable(if_ctx_t ctx, uint16_t qid)
	+bnxt_tx_queue_intr_enable(if_ctx_t ctx, uint16_t qid)
	{
	struct bnxt_softc *softc = iflib_get_softc(ctx);

	+ bnxt_do_enable_intr(&softc->tx_cp_rings[qid]);
	+ return 0;
	+}
	+
	+static int
	+bnxt_rx_queue_intr_enable(if_ctx_t ctx, uint16_t qid)
	+{
	+ struct bnxt_softc *softc = iflib_get_softc(ctx);
	+
	bnxt_do_enable_intr(&softc->rx_cp_rings[qid]);
	return 0;
	}
	@@ -1477,6 +1509,7 @@
	struct bnxt_softc *softc = iflib_get_softc(ctx);
	int rc;
	int i;
	+ char irq_name[16];

	rc = iflib_irq_alloc_generic(ctx, &softc->def_cp_ring.irq,
	softc->def_cp_ring.ring.id + 1, IFLIB_INTR_ADMIN,
	@@ -1488,9 +1521,10 @@
	}

	for (i=0; i<softc->scctx->isc_nrxqsets; i++) {
	+ snprintf(irq_name, sizeof(irq_name), "rxq%d", i);
	rc = iflib_irq_alloc_generic(ctx, &softc->rx_cp_rings[i].irq,
	softc->rx_cp_rings[i].ring.id + 1, IFLIB_INTR_RX,
	- bnxt_handle_rx_cp, &softc->rx_cp_rings[i], i, "rx_cp");
	+ bnxt_handle_rx_cp, &softc->rx_cp_rings[i], i, irq_name);
	if (rc) {
	device_printf(iflib_get_dev(ctx),
	"Failed to register RX completion ring handler\n");
	@@ -1500,8 +1534,7 @@
	}

	for (i=0; i<softc->scctx->isc_ntxqsets; i++)
	- iflib_softirq_alloc_generic(ctx, i + 1, IFLIB_INTR_TX, NULL, i,
	- "tx_cp");
	+ iflib_softirq_alloc_generic(ctx, NULL, IFLIB_INTR_TX, NULL, i, "tx_cp");

	return rc;

	@@ -1547,6 +1580,58 @@
	}

	static int
	+bnxt_wol_config(if_ctx_t ctx)
	+{
	+ struct bnxt_softc *softc = iflib_get_softc(ctx);
	+ if_t ifp = iflib_get_ifp(ctx);
	+
	+ if (!softc)
	+ return -EBUSY;
	+
	+ if (!bnxt_wol_supported(softc))
	+ return -ENOTSUP;
	+
	+ if (if_getcapabilities(ifp) & IFCAP_WOL_MAGIC) {
	+ if (!softc->wol) {
	+ if (bnxt_hwrm_alloc_wol_fltr(softc))
	+ return -EBUSY;
	+ softc->wol = 1;
	+ }
	+ } else {
	+ if (softc->wol) {
	+ if (bnxt_hwrm_free_wol_fltr(softc))
	+ return -EBUSY;
	+ softc->wol = 0;
	+ }
	+ }
	+
	+ return 0;
	+}
	+
	+static int
	+bnxt_shutdown(if_ctx_t ctx)
	+{
	+ bnxt_wol_config(ctx);
	+ return 0;
	+}
	+
	+static int
	+bnxt_suspend(if_ctx_t ctx)
	+{
	+ bnxt_wol_config(ctx);
	+ return 0;
	+}
	+
	+static int
	+bnxt_resume(if_ctx_t ctx)
	+{
	+ struct bnxt_softc *softc = iflib_get_softc(ctx);
	+
	+ bnxt_get_wol_settings(softc);
	+ return 0;
	+}
	+
	+static int
	bnxt_priv_ioctl(if_ctx_t ctx, u_long command, caddr_t data)
	{
	struct bnxt_softc *softc = iflib_get_softc(ctx);
	@@ -1895,18 +1980,6 @@
	if (link_info->auto_mode != HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_MODE_NONE)
	link_info->autoneg \|= BNXT_AUTONEG_SPEED;

	- if (link_info->auto_pause & (HWRM_PORT_PHY_QCFG_OUTPUT_PAUSE_TX \|
	- HWRM_PORT_PHY_QCFG_OUTPUT_PAUSE_RX)) {
	- if (link_info->auto_pause == (
	- HWRM_PORT_PHY_QCFG_OUTPUT_PAUSE_TX \|
	- HWRM_PORT_PHY_QCFG_OUTPUT_PAUSE_RX))
	- link_info->autoneg \|= BNXT_AUTONEG_FLOW_CTRL;
	- link_info->req_flow_ctrl = link_info->auto_pause;
	- } else if (link_info->force_pause & (
	- HWRM_PORT_PHY_QCFG_OUTPUT_PAUSE_TX \|
	- HWRM_PORT_PHY_QCFG_OUTPUT_PAUSE_RX)) {
	- link_info->req_flow_ctrl = link_info->force_pause;
	- }
	link_info->req_duplex = link_info->duplex_setting;
	if (link_info->autoneg & BNXT_AUTONEG_SPEED)
	link_info->req_link_speed = link_info->auto_link_speed;
	@@ -1931,107 +2004,95 @@
	return;

	switch (phy_type) {
	+ case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_100G_BASECR4:
	+ case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_40G_BASECR4:
	+ case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_25G_BASECR_CA_L:
	+ case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_25G_BASECR_CA_S:
	+ case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_25G_BASECR_CA_N:
	case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASECR:
	- if (supported & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_100GB)
	- ifmedia_add(softc->media, IFM_ETHER \| IFM_100G_CR4, 0,
	- NULL);
	- if (supported & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_50GB)
	- ifmedia_add(softc->media, IFM_ETHER \| IFM_50G_CR2, 0,
	- NULL);
	- if (supported & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_40GB)
	- ifmedia_add(softc->media, IFM_ETHER \| IFM_40G_CR4, 0,
	- NULL);
	- if (supported & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_25GB)
	- ifmedia_add(softc->media, IFM_ETHER \| IFM_25G_CR, 0,
	- NULL);
	- if (supported & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_10GB)
	- ifmedia_add(softc->media, IFM_ETHER \| IFM_10G_CR1, 0,
	- NULL);
	+ BNXT_IFMEDIA_ADD(supported, SPEEDS_100GB, IFM_100G_CR4);
	+ BNXT_IFMEDIA_ADD(supported, SPEEDS_50GB, IFM_50G_CR2);
	+ BNXT_IFMEDIA_ADD(supported, SPEEDS_40GB, IFM_40G_CR4);
	+ BNXT_IFMEDIA_ADD(supported, SPEEDS_25GB, IFM_25G_CR);
	+ BNXT_IFMEDIA_ADD(supported, SPEEDS_10GB, IFM_10G_CR1);
	+ BNXT_IFMEDIA_ADD(supported, SPEEDS_1GB, IFM_1000_T);
	break;
	- case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_UNKNOWN:
	- /* Auto only */
	+
	+ case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_100G_BASELR4:
	+ case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_40G_BASELR4:
	+ case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASELR:
	+ BNXT_IFMEDIA_ADD(supported, SPEEDS_100GB, IFM_100G_LR4);
	+ BNXT_IFMEDIA_ADD(supported, SPEEDS_40GB, IFM_40G_LR4);
	+ BNXT_IFMEDIA_ADD(supported, SPEEDS_25GB, IFM_25G_LR);
	+ BNXT_IFMEDIA_ADD(supported, SPEEDS_10GB, IFM_10G_LR);
	+ BNXT_IFMEDIA_ADD(supported, SPEEDS_1GB, IFM_1000_LX);
	break;
	+
	+ case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_100G_BASESR10:
	+ case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_100G_BASESR4:
	+ case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_40G_BASESR4:
	+ case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASESR:
	+ case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_40G_BASEER4:
	+ case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_100G_BASEER4:
	+ case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_25G_BASESR:
	+ case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_1G_BASESX:
	+ BNXT_IFMEDIA_ADD(supported, SPEEDS_100GB, IFM_100G_SR4);
	+ BNXT_IFMEDIA_ADD(supported, SPEEDS_40GB, IFM_40G_SR4);
	+ BNXT_IFMEDIA_ADD(supported, SPEEDS_25GB, IFM_25G_SR);
	+ BNXT_IFMEDIA_ADD(supported, SPEEDS_10GB, IFM_10G_SR);
	+ BNXT_IFMEDIA_ADD(supported, SPEEDS_1GB, IFM_1000_SX);
	+ break;
	+
	case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASEKR4:
	case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASEKR2:
	case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASEKR:
	- if (supported & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_100GB)
	- ifmedia_add(softc->media, IFM_ETHER \| IFM_100G_KR4, 0,
	- NULL);
	- if (supported & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_50GB)
	- ifmedia_add(softc->media, IFM_ETHER \| IFM_50G_KR2, 0,
	- NULL);
	- if (supported & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_40GB)
	- ifmedia_add(softc->media, IFM_ETHER \| IFM_40G_KR4, 0,
	- NULL);
	- if (supported & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_25GB)
	- ifmedia_add(softc->media, IFM_ETHER \| IFM_25G_KR, 0,
	- NULL);
	- if (supported & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_20GB)
	- ifmedia_add(softc->media, IFM_ETHER \| IFM_20G_KR2, 0,
	- NULL);
	- if (supported & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_10GB)
	- ifmedia_add(softc->media, IFM_ETHER \| IFM_10G_KR, 0,
	- NULL);
	+ BNXT_IFMEDIA_ADD(supported, SPEEDS_100GB, IFM_100G_KR4);
	+ BNXT_IFMEDIA_ADD(supported, SPEEDS_50GB, IFM_50G_KR2);
	+ BNXT_IFMEDIA_ADD(supported, SPEEDS_40GB, IFM_40G_KR4);
	+ BNXT_IFMEDIA_ADD(supported, SPEEDS_25GB, IFM_25G_KR);
	+ BNXT_IFMEDIA_ADD(supported, SPEEDS_20GB, IFM_20G_KR2);
	+ BNXT_IFMEDIA_ADD(supported, SPEEDS_10GB, IFM_10G_KR);
	+ BNXT_IFMEDIA_ADD(supported, SPEEDS_1GB, IFM_1000_KX);
	break;
	- case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASELR:
	- if (supported & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_100GB)
	- ifmedia_add(softc->media, IFM_ETHER \| IFM_100G_LR4, 0,
	- NULL);
	- if (supported & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_40GB)
	- ifmedia_add(softc->media, IFM_ETHER \| IFM_40G_LR4, 0,
	- NULL);
	- if (supported & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_10GB)
	- ifmedia_add(softc->media, IFM_ETHER \| IFM_10G_LR, 0,
	- NULL);
	+
	+ case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_40G_ACTIVE_CABLE:
	+ BNXT_IFMEDIA_ADD(supported, SPEEDS_25GB, IFM_25G_ACC);
	+ BNXT_IFMEDIA_ADD(supported, SPEEDS_10GB, IFM_10G_AOC);
	break;
	- case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASESR:
	- if (supported & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_100GB)
	- ifmedia_add(softc->media, IFM_ETHER \| IFM_100G_SR4, 0,
	- NULL);
	- if (supported & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_40GB)
	- ifmedia_add(softc->media, IFM_ETHER \| IFM_40G_SR4, 0,
	- NULL);
	- if (supported & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_25GB)
	- ifmedia_add(softc->media, IFM_ETHER \| IFM_25G_SR, 0,
	- NULL);
	- if (supported & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_10GB)
	- ifmedia_add(softc->media, IFM_ETHER \| IFM_10G_SR, 0,
	- NULL);
	+
	+ case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_1G_BASECX:
	+ BNXT_IFMEDIA_ADD(supported, SPEEDS_1GBHD, IFM_1000_CX);
	break;
	- case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASEKX:
	- if (supported & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_10GB)
	- ifmedia_add(softc->media, IFM_ETHER \| IFM_10G_KX4, 0,
	- NULL);
	- if (supported & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_2_5GB)
	- ifmedia_add(softc->media, IFM_ETHER \| IFM_2500_KX, 0,
	- NULL);
	- if (supported & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_1GB)
	- ifmedia_add(softc->media, IFM_ETHER \| IFM_1000_KX, 0,
	- NULL);
	- break;
	+
	+ case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_1G_BASET:
	case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASET:
	case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASETE:
	- if (supported & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_10MB)
	- ifmedia_add(softc->media, IFM_ETHER \| IFM_10_T, 0,
	- NULL);
	- if (supported & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_100MB)
	- ifmedia_add(softc->media, IFM_ETHER \| IFM_100_T, 0,
	- NULL);
	- if (supported & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_1GB)
	- ifmedia_add(softc->media, IFM_ETHER \| IFM_1000_T, 0,
	- NULL);
	- if (supported & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_2_5GB)
	- ifmedia_add(softc->media, IFM_ETHER \| IFM_2500_T, 0,
	- NULL);
	- if (supported & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_10GB)
	- ifmedia_add(softc->media, IFM_ETHER \| IFM_10G_T, 0,
	- NULL);
	+ BNXT_IFMEDIA_ADD(supported, SPEEDS_10GB, IFM_10G_T);
	+ BNXT_IFMEDIA_ADD(supported, SPEEDS_2_5GB, IFM_2500_T);
	+ BNXT_IFMEDIA_ADD(supported, SPEEDS_1GB, IFM_1000_T);
	+ BNXT_IFMEDIA_ADD(supported, SPEEDS_100MB, IFM_100_T);
	+ BNXT_IFMEDIA_ADD(supported, SPEEDS_10MB, IFM_10_T);
	break;
	+
	+ case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASEKX:
	+ BNXT_IFMEDIA_ADD(supported, SPEEDS_10GB, IFM_10G_KR);
	+ BNXT_IFMEDIA_ADD(supported, SPEEDS_2_5GB, IFM_2500_KX);
	+ BNXT_IFMEDIA_ADD(supported, SPEEDS_1GB, IFM_1000_KX);
	+ break;
	+
	case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_SGMIIEXTPHY:
	- if (supported & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_1GB)
	- ifmedia_add(softc->media, IFM_ETHER \| IFM_1000_SGMII, 0,
	- NULL);
	+ BNXT_IFMEDIA_ADD(supported, SPEEDS_1GB, IFM_1000_SGMII);
	break;
	+
	+ case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_UNKNOWN:
	+ /* Only Autoneg is supported for TYPE_UNKNOWN */
	+ device_printf(softc->dev, "Unknown phy type\n");
	+ break;
	+
	+ default:
	+ /* Only Autoneg is supported for new phy type values */
	+ device_printf(softc->dev, "phy type %d not supported by driver\n", phy_type);
	+ break;
	}

	return;
	@@ -2128,47 +2189,50 @@
	void
	bnxt_report_link(struct bnxt_softc *softc)
	{
	+ struct bnxt_link_info *link_info = &softc->link_info;
	const char duplex = NULL, flow_ctrl = NULL;

	- if (softc->link_info.link_up == softc->link_info.last_link_up) {
	- if (!softc->link_info.link_up)
	+ if (link_info->link_up == link_info->last_link_up) {
	+ if (!link_info->link_up)
	return;
	- if (softc->link_info.pause == softc->link_info.last_pause &&
	- softc->link_info.duplex == softc->link_info.last_duplex)
	+ if ((link_info->duplex == link_info->last_duplex) &&
	+ (!(BNXT_IS_FLOW_CTRL_CHANGED(link_info))))
	return;
	}

	- if (softc->link_info.link_up) {
	- if (softc->link_info.duplex ==
	- HWRM_PORT_PHY_QCFG_OUTPUT_DUPLEX_FULL)
	+ if (link_info->link_up) {
	+ if (link_info->duplex ==
	+ HWRM_PORT_PHY_QCFG_OUTPUT_DUPLEX_CFG_FULL)
	duplex = "full duplex";
	else
	duplex = "half duplex";
	- if (softc->link_info.pause == (
	- HWRM_PORT_PHY_QCFG_OUTPUT_PAUSE_TX \|
	- HWRM_PORT_PHY_QCFG_OUTPUT_PAUSE_RX))
	+ if (link_info->flow_ctrl.tx & link_info->flow_ctrl.rx)
	flow_ctrl = "FC - receive & transmit";
	- else if (softc->link_info.pause ==
	- HWRM_PORT_PHY_QCFG_OUTPUT_PAUSE_TX)
	+ else if (link_info->flow_ctrl.tx)
	flow_ctrl = "FC - transmit";
	- else if (softc->link_info.pause ==
	- HWRM_PORT_PHY_QCFG_OUTPUT_PAUSE_RX)
	+ else if (link_info->flow_ctrl.rx)
	flow_ctrl = "FC - receive";
	else
	- flow_ctrl = "none";
	+ flow_ctrl = "FC - none";
	iflib_link_state_change(softc->ctx, LINK_STATE_UP,
	IF_Gbps(100));
	- device_printf(softc->dev, "Link is UP %s, %s\n", duplex,
	- flow_ctrl);
	+ device_printf(softc->dev, "Link is UP %s, %s - %d Mbps \n", duplex,
	+ flow_ctrl, (link_info->link_speed * 100));
	} else {
	iflib_link_state_change(softc->ctx, LINK_STATE_DOWN,
	bnxt_get_baudrate(&softc->link_info));
	device_printf(softc->dev, "Link is Down\n");
	}

	- softc->link_info.last_link_up = softc->link_info.link_up;
	- softc->link_info.last_pause = softc->link_info.pause;
	- softc->link_info.last_duplex = softc->link_info.duplex;
	+ link_info->last_link_up = link_info->link_up;
	+ link_info->last_duplex = link_info->duplex;
	+ link_info->last_flow_ctrl.tx = link_info->flow_ctrl.tx;
	+ link_info->last_flow_ctrl.rx = link_info->flow_ctrl.rx;
	+ link_info->last_flow_ctrl.autoneg = link_info->flow_ctrl.autoneg;
	+ /* update media types */
	+ ifmedia_removeall(softc->media);
	+ bnxt_add_media_types(softc);
	+ ifmedia_set(softc->media, IFM_ETHER \| IFM_AUTO);
	}

	static int
	@@ -2418,4 +2482,17 @@
	return IF_Mbps(10);
	}
	return IF_Gbps(100);
	+}
	+
	+static void
	+bnxt_get_wol_settings(struct bnxt_softc *softc)
	+{
	+ uint16_t wol_handle = 0;
	+
	+ if (!bnxt_wol_supported(softc))
	+ return;
	+
	+ do {
	+ wol_handle = bnxt_hwrm_get_wol_fltrs(softc, wol_handle);
	+ } while (wol_handle && wol_handle != BNXT_NO_MORE_WOL_FILTERS);
	}
	Index: sys/kern/kern_cpuset.c
	===================================================================
	--- sys/kern/kern_cpuset.c
	+++ sys/kern/kern_cpuset.c
	@@ -1127,6 +1127,8 @@
	case CPU_WHICH_JAIL:
	break;
	case CPU_WHICH_IRQ:
	+ case CPU_WHICH_INTRHANDLER:
	+ case CPU_WHICH_ITHREAD:
	case CPU_WHICH_DOMAIN:
	error = EINVAL;
	goto out;
	@@ -1157,7 +1159,9 @@
	CPU_COPY(&set->cs_mask, mask);
	break;
	case CPU_WHICH_IRQ:
	- error = intr_getaffinity(id, mask);
	+ case CPU_WHICH_INTRHANDLER:
	+ case CPU_WHICH_ITHREAD:
	+ error = intr_getaffinity(id, which, mask);
	break;
	case CPU_WHICH_DOMAIN:
	if (id < 0 \|\| id >= MAXMEMDOM)
	@@ -1260,6 +1264,8 @@
	case CPU_WHICH_JAIL:
	break;
	case CPU_WHICH_IRQ:
	+ case CPU_WHICH_INTRHANDLER:
	+ case CPU_WHICH_ITHREAD:
	case CPU_WHICH_DOMAIN:
	error = EINVAL;
	goto out;
	@@ -1289,7 +1295,9 @@
	}
	break;
	case CPU_WHICH_IRQ:
	- error = intr_setaffinity(id, mask);
	+ case CPU_WHICH_INTRHANDLER:
	+ case CPU_WHICH_ITHREAD:
	+ error = intr_setaffinity(id, which, mask);
	break;
	default:
	error = EINVAL;
	Index: sys/kern/kern_intr.c
	===================================================================
	--- sys/kern/kern_intr.c
	+++ sys/kern/kern_intr.c
	@@ -287,13 +287,11 @@
	/*
	* Bind an interrupt event to the specified CPU. Note that not all
	* platforms support binding an interrupt to a CPU. For those
	- * platforms this request will fail. For supported platforms, any
	- * associated ithreads as well as the primary interrupt context will
	- * be bound to the specificed CPU. Using a cpu id of NOCPU unbinds
	+ * platforms this request will fail. Using a cpu id of NOCPU unbinds
	* the interrupt event.
	*/
	-int
	-intr_event_bind(struct intr_event *ie, int cpu)
	+static int
	+_intr_event_bind(struct intr_event *ie, int cpu, bool bindirq, bool bindithread)
	{
	lwpid_t id;
	int error;
	@@ -313,35 +311,75 @@
	* If we have any ithreads try to set their mask first to verify
	* permissions, etc.
	*/
	- mtx_lock(&ie->ie_lock);
	- if (ie->ie_thread != NULL) {
	- id = ie->ie_thread->it_thread->td_tid;
	- mtx_unlock(&ie->ie_lock);
	- error = cpuset_setithread(id, cpu);
	- if (error)
	- return (error);
	- } else
	- mtx_unlock(&ie->ie_lock);
	- error = ie->ie_assign_cpu(ie->ie_source, cpu);
	- if (error) {
	+ if (bindithread) {
	mtx_lock(&ie->ie_lock);
	if (ie->ie_thread != NULL) {
	- cpu = ie->ie_cpu;
	id = ie->ie_thread->it_thread->td_tid;
	mtx_unlock(&ie->ie_lock);
	- (void)cpuset_setithread(id, cpu);
	+ error = cpuset_setithread(id, cpu);
	+ if (error)
	+ return (error);
	} else
	mtx_unlock(&ie->ie_lock);
	+ }
	+ if (bindirq)
	+ error = ie->ie_assign_cpu(ie->ie_source, cpu);
	+ if (error) {
	+ if (bindithread) {
	+ mtx_lock(&ie->ie_lock);
	+ if (ie->ie_thread != NULL) {
	+ cpu = ie->ie_cpu;
	+ id = ie->ie_thread->it_thread->td_tid;
	+ mtx_unlock(&ie->ie_lock);
	+ (void)cpuset_setithread(id, cpu);
	+ } else
	+ mtx_unlock(&ie->ie_lock);
	+ }
	return (error);
	}

	- mtx_lock(&ie->ie_lock);
	- ie->ie_cpu = cpu;
	- mtx_unlock(&ie->ie_lock);
	+ if (bindirq) {
	+ mtx_lock(&ie->ie_lock);
	+ ie->ie_cpu = cpu;
	+ mtx_unlock(&ie->ie_lock);
	+ }

	return (error);
	}

	+/*
	+ * Bind an interrupt event to the specified CPU. For supported platforms, any
	+ * associated ithreads as well as the primary interrupt context will be bound
	+ * to the specificed CPU.
	+ */
	+int
	+intr_event_bind(struct intr_event *ie, int cpu)
	+{
	+
	+ return (_intr_event_bind(ie, cpu, true, true));
	+}
	+
	+/*
	+ * Bind an interrupt event to the specified CPU, but do not bind associated
	+ * ithreads.
	+ */
	+int
	+intr_event_bind_irqonly(struct intr_event *ie, int cpu)
	+{
	+
	+ return (_intr_event_bind(ie, cpu, true, false));
	+}
	+
	+/*
	+ * Bind an interrupt event's ithread to the specified CPU.
	+ */
	+int
	+intr_event_bind_ithread(struct intr_event *ie, int cpu)
	+{
	+
	+ return (_intr_event_bind(ie, cpu, false, true));
	+}
	+
	static struct intr_event *
	intr_lookup(int irq)
	{
	@@ -358,7 +396,7 @@
	}

	int
	-intr_setaffinity(int irq, void *m)
	+intr_setaffinity(int irq, int mode, void *m)
	{
	struct intr_event *ie;
	cpuset_t *mask;
	@@ -382,26 +420,62 @@
	ie = intr_lookup(irq);
	if (ie == NULL)
	return (ESRCH);
	- return (intr_event_bind(ie, cpu));
	+ switch (mode) {
	+ case CPU_WHICH_IRQ:
	+ return (intr_event_bind(ie, cpu));
	+ case CPU_WHICH_INTRHANDLER:
	+ return (intr_event_bind_irqonly(ie, cpu));
	+ case CPU_WHICH_ITHREAD:
	+ return (intr_event_bind_ithread(ie, cpu));
	+ default:
	+ return (EINVAL);
	+ }
	}

	int
	-intr_getaffinity(int irq, void *m)
	+intr_getaffinity(int irq, int mode, void *m)
	{
	struct intr_event *ie;
	+ struct thread *td;
	+ struct proc *p;
	cpuset_t *mask;
	+ lwpid_t id;
	+ int error;

	mask = m;
	ie = intr_lookup(irq);
	if (ie == NULL)
	return (ESRCH);
	+
	+ error = 0;
	CPU_ZERO(mask);
	- mtx_lock(&ie->ie_lock);
	- if (ie->ie_cpu == NOCPU)
	- CPU_COPY(cpuset_root, mask);
	- else
	- CPU_SET(ie->ie_cpu, mask);
	- mtx_unlock(&ie->ie_lock);
	+ switch (mode) {
	+ case CPU_WHICH_IRQ:
	+ case CPU_WHICH_INTRHANDLER:
	+ mtx_lock(&ie->ie_lock);
	+ if (ie->ie_cpu == NOCPU)
	+ CPU_COPY(cpuset_root, mask);
	+ else
	+ CPU_SET(ie->ie_cpu, mask);
	+ mtx_unlock(&ie->ie_lock);
	+ break;
	+ case CPU_WHICH_ITHREAD:
	+ mtx_lock(&ie->ie_lock);
	+ if (ie->ie_thread == NULL) {
	+ mtx_unlock(&ie->ie_lock);
	+ CPU_COPY(cpuset_root, mask);
	+ } else {
	+ id = ie->ie_thread->it_thread->td_tid;
	+ mtx_unlock(&ie->ie_lock);
	+ error = cpuset_which(CPU_WHICH_TID, id, &p, &td, NULL);
	+ if (error != 0)
	+ return (error);
	+ CPU_COPY(&td->td_cpuset->cs_mask, mask);
	+ PROC_UNLOCK(p);
	+ }
	+ default:
	+ return (EINVAL);
	+ }
	return (0);
	}

	Index: sys/kern/subr_gtaskqueue.c
	===================================================================
	--- sys/kern/subr_gtaskqueue.c
	+++ sys/kern/subr_gtaskqueue.c
	@@ -48,7 +48,7 @@
	#include <sys/unistd.h>
	#include <machine/stdarg.h>

	-static MALLOC_DEFINE(M_GTASKQUEUE, "taskqueue", "Task Queues");
	+static MALLOC_DEFINE(M_GTASKQUEUE, "gtaskqueue", "Group Task Queues");
	static void gtaskqueue_thread_enqueue(void *);
	static void gtaskqueue_thread_loop(void *arg);

	@@ -134,8 +134,10 @@
	snprintf(tq_name, TASKQUEUE_NAMELEN, "%s", (name) ? name : "taskqueue");

	queue = malloc(sizeof(struct gtaskqueue), M_GTASKQUEUE, mflags \| M_ZERO);
	- if (!queue)
	+ if (!queue) {
	+ free(tq_name, M_GTASKQUEUE);
	return (NULL);
	+ }

	STAILQ_INIT(&queue->tq_queue);
	TAILQ_INIT(&queue->tq_active);
	@@ -663,10 +665,10 @@
	void uniq, int irq, char name)
	{
	cpuset_t mask;
	- int qid;
	+ int qid, error;

	gtask->gt_uniq = uniq;
	- gtask->gt_name = name;
	+ snprintf(gtask->gt_name, GROUPTASK_NAMELEN, "%s", name ? name : "grouptask");
	gtask->gt_irq = irq;
	gtask->gt_cpu = -1;
	mtx_lock(&qgroup->tqg_lock);
	@@ -679,7 +681,9 @@
	CPU_ZERO(&mask);
	CPU_SET(qgroup->tqg_queue[qid].tgc_cpu, &mask);
	mtx_unlock(&qgroup->tqg_lock);
	- intr_setaffinity(irq, &mask);
	+ error = intr_setaffinity(irq, CPU_WHICH_IRQ, &mask);
	+ if (error)
	+ printf("%s: setaffinity failed for %s: %d\n", __func__, gtask->gt_name, error);
	} else
	mtx_unlock(&qgroup->tqg_lock);
	}
	@@ -688,7 +692,7 @@
	taskqgroup_attach_deferred(struct taskqgroup qgroup, struct grouptask gtask)
	{
	cpuset_t mask;
	- int qid, cpu;
	+ int qid, cpu, error;

	mtx_lock(&qgroup->tqg_lock);
	qid = taskqgroup_find(qgroup, gtask->gt_uniq);
	@@ -698,9 +702,11 @@

	CPU_ZERO(&mask);
	CPU_SET(cpu, &mask);
	- intr_setaffinity(gtask->gt_irq, &mask);
	-
	+ error = intr_setaffinity(gtask->gt_irq, CPU_WHICH_IRQ, &mask);
	mtx_lock(&qgroup->tqg_lock);
	+ if (error)
	+ printf("%s: %s setaffinity failed: %d\n", __func__, gtask->gt_name, error);
	+
	}
	qgroup->tqg_queue[qid].tgc_cnt++;

	@@ -716,11 +722,11 @@
	void uniq, int cpu, int irq, char name)
	{
	cpuset_t mask;
	- int i, qid;
	+ int i, qid, error;

	qid = -1;
	gtask->gt_uniq = uniq;
	- gtask->gt_name = name;
	+ snprintf(gtask->gt_name, GROUPTASK_NAMELEN, "%s", name ? name : "grouptask");
	gtask->gt_irq = irq;
	gtask->gt_cpu = cpu;
	mtx_lock(&qgroup->tqg_lock);
	@@ -732,6 +738,7 @@
	}
	if (qid == -1) {
	mtx_unlock(&qgroup->tqg_lock);
	+ printf("%s: qid not found for %s cpu=%d\n", __func__, gtask->gt_name, cpu);
	return (EINVAL);
	}
	} else
	@@ -744,8 +751,11 @@

	CPU_ZERO(&mask);
	CPU_SET(cpu, &mask);
	- if (irq != -1 && tqg_smp_started)
	- intr_setaffinity(irq, &mask);
	+ if (irq != -1 && tqg_smp_started) {
	+ error = intr_setaffinity(irq, CPU_WHICH_IRQ, &mask);
	+ if (error)
	+ printf("%s: setaffinity failed: %d\n", __func__, error);
	+ }
	return (0);
	}

	@@ -753,7 +763,7 @@
	taskqgroup_attach_cpu_deferred(struct taskqgroup qgroup, struct grouptask gtask)
	{
	cpuset_t mask;
	- int i, qid, irq, cpu;
	+ int i, qid, irq, cpu, error;

	qid = -1;
	irq = gtask->gt_irq;
	@@ -767,6 +777,7 @@
	}
	if (qid == -1) {
	mtx_unlock(&qgroup->tqg_lock);
	+ printf("%s: qid not found for %s cpu=%d\n", __func__, gtask->gt_name, cpu);
	return (EINVAL);
	}
	qgroup->tqg_queue[qid].tgc_cnt++;
	@@ -778,8 +789,11 @@
	CPU_ZERO(&mask);
	CPU_SET(cpu, &mask);

	- if (irq != -1)
	- intr_setaffinity(irq, &mask);
	+ if (irq != -1) {
	+ error = intr_setaffinity(irq, CPU_WHICH_IRQ, &mask);
	+ if (error)
	+ printf("%s: setaffinity failed: %d\n", __func__, error);
	+ }
	return (0);
	}

	@@ -793,7 +807,7 @@
	if (qgroup->tqg_queue[i].tgc_taskq == gtask->gt_taskqueue)
	break;
	if (i == qgroup->tqg_cnt)
	- panic("taskqgroup_detach: task not in group\n");
	+ panic("taskqgroup_detach: task %s not in group\n", gtask->gt_name);
	qgroup->tqg_queue[i].tgc_cnt--;
	LIST_REMOVE(gtask, gt_list);
	mtx_unlock(&qgroup->tqg_lock);
	@@ -815,7 +829,7 @@
	thread_unlock(curthread);

	if (error)
	- printf("taskqgroup_binder: setaffinity failed: %d\n",
	+ printf("%s: setaffinity failed: %d\n", __func__,
	error);
	free(gtask, M_DEVBUF);
	}
	@@ -858,7 +872,7 @@
	return (EINVAL);
	}
	if (qgroup->tqg_adjusting) {
	- printf("taskqgroup_adjust failed: adjusting\n");
	+ printf("%s failed: adjusting\n", __func__);
	return (EBUSY);
	}
	qgroup->tqg_adjusting = 1;
	Index: sys/net/ifdi_if.m
	===================================================================
	--- sys/net/ifdi_if.m
	+++ sys/net/ifdi_if.m
	@@ -195,11 +195,16 @@
	if_ctx_t _ctx;
	};

	-METHOD int queue_intr_enable {
	+METHOD int rx_queue_intr_enable {
	if_ctx_t _ctx;
	uint16_t _qid;
	} DEFAULT null_queue_intr_enable;

	+METHOD int tx_queue_intr_enable {
	+ if_ctx_t _ctx;
	+ uint16_t _qid;
	+} DEFAULT null_queue_intr_enable;
	+
	METHOD void link_intr_enable {
	if_ctx_t _ctx;
	} DEFAULT null_void_op;
	@@ -229,6 +234,7 @@
	METHOD void crcstrip_set {
	if_ctx_t _ctx;
	int _onoff;
	+ int _strip;
	};

	#
	@@ -332,4 +338,6 @@
	if_int_delay_info_t _iidi;
	} DEFAULT null_sysctl_int_delay;

	-
	+METHOD void debug {
	+ if_ctx_t _ctx;
	+} DEFAULT null_void_op;
	Index: sys/net/iflib.h
	===================================================================
	--- sys/net/iflib.h
	+++ sys/net/iflib.h
	@@ -1,5 +1,5 @@
	/*-
	- * Copyright (c) 2014-2015, Matthew Macy (mmacy@nextbsd.org)
	+ * Copyright (c) 2014-2017, Matthew Macy (mmacy@nextbsd.org)
	* All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	@@ -37,8 +37,14 @@
	#include <sys/nv.h>
	#include <sys/gtaskqueue.h>

	-
	/*
	+ * The value type for indexing, limits max descriptors
	+ * to 65535 can be conditionally redefined to uint32_t
	+ * in the future if the need arises.
	+ */
	+typedef uint16_t qidx_t;
	+#define QIDX_INVALID 0xFFFF
	+/*
	* Most cards can handle much larger TSO requests
	* but the FreeBSD TCP stack will break on larger
	* values
	@@ -63,7 +69,7 @@

	typedef struct if_rxd_frag {
	uint8_t irf_flid;
	- uint16_t irf_idx;
	+ qidx_t irf_idx;
	uint16_t irf_len;
	} *if_rxd_frag_t;

	@@ -73,47 +79,61 @@
	uint16_t iri_vtag; /* vlan tag - if flag set */
	/* XXX redundant with the new irf_len field */
	uint16_t iri_len; /* packet length */
	- uint16_t iri_cidx; /* consumer index of cq */
	+ qidx_t iri_cidx; /* consumer index of cq */
	struct ifnet iri_ifp; / some drivers >1 interface per softc */

	/* updated by driver */
	- uint16_t iri_flags; /* mbuf flags for packet */
	+ if_rxd_frag_t iri_frags;
	uint32_t iri_flowid; /* RSS hash for packet */
	uint32_t iri_csum_flags; /* m_pkthdr csum flags */
	+
	uint32_t iri_csum_data; /* m_pkthdr csum data */
	+ uint8_t iri_flags; /* mbuf flags for packet */
	uint8_t iri_nfrags; /* number of fragments in packet */
	uint8_t iri_rsstype; /* RSS hash type */
	uint8_t iri_pad; /* any padding in the received data */
	- if_rxd_frag_t iri_frags;
	} *if_rxd_info_t;

	+typedef struct if_rxd_update {
	+ uint64_t *iru_paddrs;
	+ caddr_t *iru_vaddrs;
	+ qidx_t *iru_idxs;
	+ qidx_t iru_pidx;
	+ uint16_t iru_qsidx;
	+ uint16_t iru_count;
	+ uint16_t iru_buf_size;
	+ uint8_t iru_flidx;
	+} *if_rxd_update_t;
	+
	#define IPI_TX_INTR 0x1 /* send an interrupt when this packet is sent */
	#define IPI_TX_IPV4 0x2 /* ethertype IPv4 */
	#define IPI_TX_IPV6 0x4 /* ethertype IPv6 */

	typedef struct if_pkt_info {
	- uint32_t ipi_len; /* packet length */
	- bus_dma_segment_t ipi_segs; / physical addresses */
	- uint16_t ipi_qsidx; /* queue set index */
	- uint16_t ipi_nsegs; /* number of segments */
	- uint16_t ipi_ndescs; /* number of descriptors used by encap */
	- uint16_t ipi_flags; /* iflib per-packet flags */
	- uint32_t ipi_pidx; /* start pidx for encap */
	- uint32_t ipi_new_pidx; /* next available pidx post-encap */
	+ bus_dma_segment_t ipi_segs; / physical addresses */
	+ uint32_t ipi_len; /* packet length */
	+ uint16_t ipi_qsidx; /* queue set index */
	+ qidx_t ipi_nsegs; /* number of segments */
	+
	+ qidx_t ipi_ndescs; /* number of descriptors used by encap */
	+ uint16_t ipi_flags; /* iflib per-packet flags */
	+ qidx_t ipi_pidx; /* start pidx for encap */
	+ qidx_t ipi_new_pidx; /* next available pidx post-encap */
	/* offload handling */
	- uint64_t ipi_csum_flags; /* packet checksum flags */
	- uint16_t ipi_tso_segsz; /* tso segment size */
	- uint16_t ipi_mflags; /* packet mbuf flags */
	- uint16_t ipi_vtag; /* VLAN tag */
	- uint16_t ipi_etype; /* ether header type */
	- uint8_t ipi_ehdrlen; /* ether header length */
	- uint8_t ipi_ip_hlen; /* ip header length */
	- uint8_t ipi_tcp_hlen; /* tcp header length */
	- uint8_t ipi_tcp_hflags; /* tcp header flags */
	- uint8_t ipi_ipproto; /* ip protocol */
	- /* implied padding */
	- uint32_t ipi_tcp_seq; /* tcp seqno */
	- uint32_t ipi_tcp_sum; /* tcp csum */
	+ uint8_t ipi_ehdrlen; /* ether header length */
	+ uint8_t ipi_ip_hlen; /* ip header length */
	+ uint8_t ipi_tcp_hlen; /* tcp header length */
	+ uint8_t ipi_ipproto; /* ip protocol */
	+
	+ uint32_t ipi_csum_flags; /* packet checksum flags */
	+ uint16_t ipi_tso_segsz; /* tso segment size */
	+ uint16_t ipi_vtag; /* VLAN tag */
	+ uint16_t ipi_etype; /* ether header type */
	+ uint8_t ipi_tcp_hflags; /* tcp header flags */
	+ uint8_t ipi_mflags; /* packet mbuf flags */
	+
	+ uint32_t ipi_tcp_seq; /* tcp seqno */
	+ uint32_t ipi_tcp_sum; /* tcp csum */
	} *if_pkt_info_t;

	typedef struct if_irq {
	@@ -154,17 +174,20 @@
	#define PVID_OEM(vendor, devid, svid, sdevid, revid, name) {vendor, devid, svid, sdevid, revid, 0, name}
	#define PVID_END {0, 0, 0, 0, 0, 0, NULL}

	+#define IFLIB_PNP_DESCR "U32:vendor;U32:device;U32:subvendor;U32:subdevice;" \
	+ "U32:revision;U32:class;D:human"
	+#define IFLIB_PNP_INFO(b, u, t) \
	+ MODULE_PNP_INFO(IFLIB_PNP_DESCR, b, u, t, sizeof(t[0]), nitems(t) - 1)
	+
	typedef struct if_txrx {
	int (ift_txd_encap) (void , if_pkt_info_t);
	- void (ift_txd_flush) (void , uint16_t, uint32_t);
	- int (ift_txd_credits_update) (void , uint16_t, uint32_t, bool);
	+ void (ift_txd_flush) (void , uint16_t, qidx_t pidx);
	+ int (ift_txd_credits_update) (void , uint16_t qsidx, bool clear);

	- int (ift_rxd_available) (void , uint16_t qsidx, uint32_t pidx,
	- int budget);
	+ int (ift_rxd_available) (void , uint16_t qsidx, qidx_t pidx, qidx_t budget);
	int (ift_rxd_pkt_get) (void , if_rxd_info_t ri);
	- void (ift_rxd_refill) (void , uint16_t qsidx, uint8_t flidx, uint32_t pidx,
	- uint64_t paddrs, caddr_t vaddrs, uint16_t count, uint16_t buf_size);
	- void (ift_rxd_flush) (void , uint16_t qsidx, uint8_t flidx, uint32_t pidx);
	+ void (ift_rxd_refill) (void , if_rxd_update_t iru);
	+ void (ift_rxd_flush) (void , uint16_t qsidx, uint8_t flidx, qidx_t pidx);
	int (ift_legacy_intr) (void );
	} *if_txrx_t;

	@@ -179,11 +202,15 @@

	uint32_t isc_txqsizes[8];
	uint32_t isc_rxqsizes[8];
	- int isc_max_txqsets;
	- int isc_max_rxqsets;
	+ /* is there such thing as a descriptor that is more than 248 bytes ? */
	+ uint8_t isc_txd_size[8];
	+ uint8_t isc_rxd_size[8];
	+
	int isc_tx_tso_segments_max;
	int isc_tx_tso_size_max;
	int isc_tx_tso_segsize_max;
	+ int isc_tx_csum_flags;
	+ int isc_capenable;
	int isc_rss_table_size;
	int isc_rss_table_mask;
	int isc_nrxqsets_max;
	@@ -191,32 +218,28 @@

	iflib_intr_mode_t isc_intr;
	uint16_t isc_max_frame_size; /* set at init time by driver */
	+ uint16_t isc_min_frame_size; /* set at init time by driver, only used if
	+ IFLIB_NEED_ETHER_PAD is set. */
	+ uint32_t isc_pause_frames; /* set by driver for iflib_timer to detect */
	pci_vendor_info_t isc_vendor_info; /* set by iflib prior to attach_pre */
	+ int isc_disable_msix;
	+ if_txrx_t isc_txrx;
	} *if_softc_ctx_t;

	/*
	* Initialization values for device
	*/
	struct if_shared_ctx {
	- int isc_magic;
	- if_txrx_t isc_txrx;
	+ unsigned isc_magic;
	driver_t *isc_driver;
	- int isc_nfl;
	- int isc_flags;
	bus_size_t isc_q_align;
	bus_size_t isc_tx_maxsize;
	bus_size_t isc_tx_maxsegsize;
	bus_size_t isc_rx_maxsize;
	bus_size_t isc_rx_maxsegsize;
	int isc_rx_nsegments;
	- int isc_rx_process_limit;
	- int isc_ntxqs; /* # of tx queues per tx qset - usually 1 */
	- int isc_nrxqs; /* # of rx queues per rx qset - intel 1, chelsio 2, broadcom 3 */
	int isc_admin_intrcnt; /* # of admin/link interrupts */

	-
	- int isc_tx_reclaim_thresh;
	-
	/* fields necessary for probe */
	pci_vendor_info_t *isc_vendor_info;
	char *isc_driver_version;
	@@ -229,6 +252,14 @@
	int isc_ntxd_min[8];
	int isc_ntxd_default[8];
	int isc_ntxd_max[8];
	+
	+ /* actively used during operation */
	+ int isc_nfl __aligned(CACHE_LINE_SIZE);
	+ int isc_ntxqs; /* # of tx queues per tx qset - usually 1 */
	+ int isc_nrxqs; /* # of rx queues per rx qset - intel 1, chelsio 2, broadcom 3 */
	+ int isc_rx_process_limit;
	+ int isc_tx_reclaim_thresh;
	+ int isc_flags;
	};

	typedef struct iflib_dma_info {
	@@ -242,8 +273,9 @@
	#define IFLIB_MAGIC 0xCAFEF00D

	typedef enum {
	- IFLIB_INTR_TX,
	IFLIB_INTR_RX,
	+ IFLIB_INTR_TX,
	+ IFLIB_INTR_RXTX,
	IFLIB_INTR_ADMIN,
	IFLIB_INTR_IOV,
	} iflib_intr_type_t;
	@@ -256,22 +288,42 @@
	/*
	* Interface has a separate command queue for RX
	*/
	-#define IFLIB_HAS_RXCQ 0x1
	+#define IFLIB_HAS_RXCQ 0x01
	/*
	* Driver has already allocated vectors
	*/
	-#define IFLIB_SKIP_MSIX 0x2
	-
	+#define IFLIB_SKIP_MSIX 0x02
	/*
	* Interface is a virtual function
	*/
	-#define IFLIB_IS_VF 0x4
	+#define IFLIB_IS_VF 0x04
	/*
	* Interface has a separate command queue for TX
	*/
	-#define IFLIB_HAS_TXCQ 0x8
	+#define IFLIB_HAS_TXCQ 0x08
	+/*
	+ * Interface does checksum in place
	+ */
	+#define IFLIB_NEED_SCRATCH 0x10
	+/*
	+ * Interface doesn't expect in_pseudo for th_sum
	+ */
	+#define IFLIB_TSO_INIT_IP 0x20
	+/*
	+ * Interface doesn't align IP header
	+ */
	+#define IFLIB_DO_RX_FIXUP 0x40
	+/*
	+ * Driver needs csum zeroed for offloading
	+ */
	+#define IFLIB_NEED_ZERO_CSUM 0x80
	+/*
	+ * Driver needs frames padded to some minimum length
	+ */
	+#define IFLIB_NEED_ETHER_PAD 0x100


	+
	/*
	* field accessors
	*/
	@@ -288,9 +340,6 @@

	void iflib_set_mac(if_ctx_t ctx, uint8_t mac[ETHER_ADDR_LEN]);

	-
	-
	-
	/*
	* If the driver can plug cleanly in to newbus use these
	*/
	@@ -319,7 +368,7 @@
	int iflib_irq_alloc_generic(if_ctx_t ctx, if_irq_t irq, int rid,
	iflib_intr_type_t type, driver_filter_t *filter,
	void filter_arg, int qid, char name);
	-void iflib_softirq_alloc_generic(if_ctx_t ctx, int rid, iflib_intr_type_t type, void arg, int qid, char name);
	+void iflib_softirq_alloc_generic(if_ctx_t ctx, if_irq_t irq, iflib_intr_type_t type, void arg, int qid, char name);

	void iflib_irq_free(if_ctx_t ctx, if_irq_t irq);

	Index: sys/net/iflib.c
	===================================================================
	--- sys/net/iflib.c
	+++ sys/net/iflib.c
	@@ -1,5 +1,5 @@
	/*-
	- * Copyright (c) 2014-2016, Matthew Macy <mmacy@nextbsd.org>
	+ * Copyright (c) 2014-2017, Matthew Macy <mmacy@nextbsd.org>
	* All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	@@ -31,6 +31,7 @@
	#include "opt_inet.h"
	#include "opt_inet6.h"
	#include "opt_acpi.h"
	+#include "opt_sched.h"

	#include <sys/param.h>
	#include <sys/types.h>
	@@ -59,6 +60,7 @@
	#include <net/bpf.h>
	#include <net/ethernet.h>
	#include <net/mp_ring.h>
	+#include <net/vnet.h>

	#include <netinet/in.h>
	#include <netinet/in_pcb.h>
	@@ -68,6 +70,8 @@
	#include <netinet/ip.h>
	#include <netinet/ip6.h>
	#include <netinet/tcp.h>
	+#include <netinet/ip_var.h>
	+#include <netinet6/ip6_var.h>

	#include <machine/bus.h>
	#include <machine/in_cksum.h>
	@@ -93,9 +97,10 @@
	#include <x86/iommu/busdma_dmar.h>
	#endif

	-
	+#include <sys/bitstring.h>
	/*
	- * enable accounting of every mbuf as it comes in to and goes out of iflib's software descriptor references
	+ * enable accounting of every mbuf as it comes in to and goes out of
	+ * iflib's software descriptor references
	*/
	#define MEMORY_LOGGING 0
	/*
	@@ -134,10 +139,13 @@
	struct iflib_fl;
	typedef struct iflib_fl *iflib_fl_t;

	+static void iru_init(if_rxd_update_t iru, iflib_rxq_t rxq, uint8_t flid);
	+
	typedef struct iflib_filter_info {
	driver_filter_t *ifi_filter;
	void *ifi_filter_arg;
	struct grouptask *ifi_task;
	+ void *ifi_ctx;
	} *iflib_filter_info_t;

	struct iflib_ctx {
	@@ -156,7 +164,6 @@
	struct mtx ifc_mtx;

	uint16_t ifc_nhwtxqs;
	- uint16_t ifc_nhwrxqs;

	iflib_txq_t ifc_txqs;
	iflib_rxq_t ifc_rxqs;
	@@ -167,7 +174,6 @@

	int ifc_link_state;
	int ifc_link_irq;
	- int ifc_pause_frames;
	int ifc_watchdog_events;
	struct cdev *ifc_led_dev;
	struct resource *ifc_msix_mem;
	@@ -182,9 +188,10 @@
	uint16_t ifc_sysctl_ntxqs;
	uint16_t ifc_sysctl_nrxqs;
	uint16_t ifc_sysctl_qs_eq_override;
	+ uint16_t ifc_sysctl_rx_budget;

	- uint16_t ifc_sysctl_ntxds[8];
	- uint16_t ifc_sysctl_nrxds[8];
	+ qidx_t ifc_sysctl_ntxds[8];
	+ qidx_t ifc_sysctl_nrxds[8];
	struct if_txrx ifc_txrx;
	#define isc_txd_encap ifc_txrx.ift_txd_encap
	#define isc_txd_flush ifc_txrx.ift_txd_flush
	@@ -252,7 +259,9 @@
	return (ctx->ifc_sctx);
	}

	+#define IP_ALIGNED(m) ((((uintptr_t)(m)->m_data) & 0x3) == 0x2)
	#define CACHE_PTR_INCREMENT (CACHE_LINE_SIZE/sizeof(void*))
	+#define CACHE_PTR_NEXT(ptr) ((void *)(((uintptr_t)(ptr)+CACHE_LINE_SIZE-1) & (CACHE_LINE_SIZE-1)))

	#define LINK_ACTIVE(ctx) ((ctx)->ifc_link_state == LINK_STATE_UP)
	#define CTX_IS_VF(ctx) ((ctx)->ifc_sctx->isc_flags & IFLIB_IS_VF)
	@@ -262,62 +271,70 @@
	#define RX_SW_DESC_INUSE (1 << 3)
	#define TX_SW_DESC_MAPPED (1 << 4)

	-typedef struct iflib_sw_rx_desc {
	- bus_dmamap_t ifsd_map; /* bus_dma map for packet */
	- struct mbuf ifsd_m; / rx: uninitialized mbuf */
	- caddr_t ifsd_cl; /* direct cluster pointer for rx */
	- uint16_t ifsd_flags;
	-} *iflib_rxsd_t;
	+#define M_TOOBIG M_PROTO1

	-typedef struct iflib_sw_tx_desc_val {
	- bus_dmamap_t ifsd_map; /* bus_dma map for packet */
	- struct mbuf ifsd_m; / pkthdr mbuf */
	- uint8_t ifsd_flags;
	-} *iflib_txsd_val_t;
	+typedef struct iflib_sw_rx_desc_array {
	+ bus_dmamap_t ifsd_map; / bus_dma maps for packet */
	+ struct mbuf *ifsd_m; / pkthdr mbufs */
	+ caddr_t ifsd_cl; / direct cluster pointer for rx */
	+ uint8_t *ifsd_flags;
	+} iflib_rxsd_array_t;

	typedef struct iflib_sw_tx_desc_array {
	bus_dmamap_t ifsd_map; / bus_dma maps for packet */
	struct mbuf *ifsd_m; / pkthdr mbufs */
	uint8_t *ifsd_flags;
	-} iflib_txsd_array_t;
	+} if_txsd_vec_t;


	/* magic number that should be high enough for any hardware */
	#define IFLIB_MAX_TX_SEGS 128
	-#define IFLIB_MAX_RX_SEGS 32
	+/* bnxt supports 64 with hardware LRO enabled */
	+#define IFLIB_MAX_RX_SEGS 64
	#define IFLIB_RX_COPY_THRESH 128
	#define IFLIB_MAX_RX_REFRESH 32
	+/* The minimum descriptors per second before we start coalescing */
	+#define IFLIB_MIN_DESC_SEC 16384
	+#define IFLIB_DEFAULT_TX_UPDATE_FREQ 16
	#define IFLIB_QUEUE_IDLE 0
	#define IFLIB_QUEUE_HUNG 1
	#define IFLIB_QUEUE_WORKING 2
	+/* maximum number of txqs that can share an rx interrupt */
	+#define IFLIB_MAX_TX_SHARED_INTR 4

	-/* this should really scale with ring size - 32 is a fairly arbitrary value for this */
	-#define TX_BATCH_SIZE 16
	+/* this should really scale with ring size - this is a fairly arbitrary value */
	+#define TX_BATCH_SIZE 32

	#define IFLIB_RESTART_BUDGET 8

	-#define IFC_LEGACY 0x01
	-#define IFC_QFLUSH 0x02
	-#define IFC_MULTISEG 0x04
	-#define IFC_DMAR 0x08
	-#define IFC_SC_ALLOCATED 0x10
	+#define IFC_LEGACY 0x001
	+#define IFC_QFLUSH 0x002
	+#define IFC_MULTISEG 0x004
	+#define IFC_DMAR 0x008
	+#define IFC_SC_ALLOCATED 0x010
	+#define IFC_INIT_DONE 0x020
	+#define IFC_PREFETCH 0x040
	+#define IFC_DO_RESET 0x080
	+#define IFC_CHECK_HUNG 0x100

	#define CSUM_OFFLOAD (CSUM_IP_TSO\|CSUM_IP6_TSO\|CSUM_IP\| \
	CSUM_IP_UDP\|CSUM_IP_TCP\|CSUM_IP_SCTP\| \
	CSUM_IP6_UDP\|CSUM_IP6_TCP\|CSUM_IP6_SCTP)
	struct iflib_txq {
	- uint16_t ift_in_use;
	- uint16_t ift_cidx;
	- uint16_t ift_cidx_processed;
	- uint16_t ift_pidx;
	+ qidx_t ift_in_use;
	+ qidx_t ift_cidx;
	+ qidx_t ift_cidx_processed;
	+ qidx_t ift_pidx;
	uint8_t ift_gen;
	- uint8_t ift_db_pending;
	- uint8_t ift_db_pending_queued;
	- uint8_t ift_npending;
	uint8_t ift_br_offset;
	+ uint16_t ift_npending;
	+ uint16_t ift_db_pending;
	+ uint16_t ift_rs_pending;
	/* implicit pad */
	+ uint8_t ift_txd_size[8];
	uint64_t ift_processed;
	uint64_t ift_cleaned;
	+ uint64_t ift_cleaned_prev;
	#if MEMORY_LOGGING
	uint64_t ift_enqueued;
	uint64_t ift_dequeued;
	@@ -335,19 +352,16 @@

	/* constant values */
	if_ctx_t ift_ctx;
	- struct ifmp_ring **ift_br;
	+ struct ifmp_ring *ift_br;
	struct grouptask ift_task;
	- uint16_t ift_size;
	+ qidx_t ift_size;
	uint16_t ift_id;
	struct callout ift_timer;
	- struct callout ift_db_check;

	- iflib_txsd_array_t ift_sds;
	- uint8_t ift_nbr;
	- uint8_t ift_qstatus;
	- uint8_t ift_active;
	- uint8_t ift_closed;
	- int ift_watchdog_time;
	+ if_txsd_vec_t ift_sds;
	+ uint8_t ift_qstatus;
	+ uint8_t ift_closed;
	+ uint8_t ift_update_freq;
	struct iflib_filter_info ift_filter_info;
	bus_dma_tag_t ift_desc_tag;
	bus_dma_tag_t ift_tso_desc_tag;
	@@ -356,13 +370,17 @@
	char ift_mtx_name[MTX_NAME_LEN];
	char ift_db_mtx_name[MTX_NAME_LEN];
	bus_dma_segment_t ift_segs[IFLIB_MAX_TX_SEGS] __aligned(CACHE_LINE_SIZE);
	+#ifdef IFLIB_DIAGNOSTICS
	+ uint64_t ift_cpu_exec_count[256];
	+#endif
	} __aligned(CACHE_LINE_SIZE);

	struct iflib_fl {
	- uint16_t ifl_cidx;
	- uint16_t ifl_pidx;
	- uint16_t ifl_credits;
	+ qidx_t ifl_cidx;
	+ qidx_t ifl_pidx;
	+ qidx_t ifl_credits;
	uint8_t ifl_gen;
	+ uint8_t ifl_rxd_size;
	#if MEMORY_LOGGING
	uint64_t ifl_m_enqueued;
	uint64_t ifl_m_dequeued;
	@@ -371,24 +389,27 @@
	#endif
	/* implicit pad */

	+ bitstr_t *ifl_rx_bitmap;
	+ qidx_t ifl_fragidx;
	/* constant */
	- uint16_t ifl_size;
	+ qidx_t ifl_size;
	uint16_t ifl_buf_size;
	uint16_t ifl_cltype;
	uma_zone_t ifl_zone;
	- iflib_rxsd_t ifl_sds;
	+ iflib_rxsd_array_t ifl_sds;
	iflib_rxq_t ifl_rxq;
	uint8_t ifl_id;
	bus_dma_tag_t ifl_desc_tag;
	iflib_dma_info_t ifl_ifdi;
	uint64_t ifl_bus_addrs[IFLIB_MAX_RX_REFRESH] __aligned(CACHE_LINE_SIZE);
	caddr_t ifl_vm_addrs[IFLIB_MAX_RX_REFRESH];
	+ qidx_t ifl_rxd_idxs[IFLIB_MAX_RX_REFRESH];
	} __aligned(CACHE_LINE_SIZE);

	-static inline int
	-get_inuse(int size, int cidx, int pidx, int gen)
	+static inline qidx_t
	+get_inuse(int size, qidx_t cidx, qidx_t pidx, uint8_t gen)
	{
	- int used;
	+ qidx_t used;

	if (pidx > cidx)
	used = pidx - cidx;
	@@ -414,9 +435,9 @@
	* these are the cq cidx and pidx. Otherwise
	* these are unused.
	*/
	- uint16_t ifr_size;
	- uint16_t ifr_cq_cidx;
	- uint16_t ifr_cq_pidx;
	+ qidx_t ifr_size;
	+ qidx_t ifr_cq_cidx;
	+ qidx_t ifr_cq_pidx;
	uint8_t ifr_cq_gen;
	uint8_t ifr_fl_offset;

	@@ -426,26 +447,89 @@
	uint16_t ifr_id;
	uint8_t ifr_lro_enabled;
	uint8_t ifr_nfl;
	+ uint8_t ifr_ntxqirq;
	+ uint8_t ifr_txqid[IFLIB_MAX_TX_SHARED_INTR];
	struct lro_ctrl ifr_lc;
	struct grouptask ifr_task;
	struct iflib_filter_info ifr_filter_info;
	iflib_dma_info_t ifr_ifdi;
	+
	/* dynamically allocate if any drivers need a value substantially larger than this */
	struct if_rxd_frag ifr_frags[IFLIB_MAX_RX_SEGS] __aligned(CACHE_LINE_SIZE);
	+#ifdef IFLIB_DIAGNOSTICS
	+ uint64_t ifr_cpu_exec_count[256];
	+#endif
	} __aligned(CACHE_LINE_SIZE);

	+typedef struct if_rxsd {
	+ caddr_t *ifsd_cl;
	+ struct mbuf **ifsd_m;
	+ iflib_fl_t ifsd_fl;
	+ qidx_t ifsd_cidx;
	+} *if_rxsd_t;
	+
	+/* multiple of word size */
	+#ifdef __LP64__
	+#define PKT_INFO_SIZE 6
	+#define RXD_INFO_SIZE 5
	+#define PKT_TYPE uint64_t
	+#else
	+#define PKT_INFO_SIZE 11
	+#define RXD_INFO_SIZE 8
	+#define PKT_TYPE uint32_t
	+#endif
	+#define PKT_LOOP_BOUND ((PKT_INFO_SIZE/3)*3)
	+#define RXD_LOOP_BOUND ((RXD_INFO_SIZE/4)*4)
	+
	+typedef struct if_pkt_info_pad {
	+ PKT_TYPE pkt_val[PKT_INFO_SIZE];
	+} *if_pkt_info_pad_t;
	+typedef struct if_rxd_info_pad {
	+ PKT_TYPE rxd_val[RXD_INFO_SIZE];
	+} *if_rxd_info_pad_t;
	+
	+CTASSERT(sizeof(struct if_pkt_info_pad) == sizeof(struct if_pkt_info));
	+CTASSERT(sizeof(struct if_rxd_info_pad) == sizeof(struct if_rxd_info));
	+
	+
	+static inline void
	+pkt_info_zero(if_pkt_info_t pi)
	+{
	+ if_pkt_info_pad_t pi_pad;
	+
	+ pi_pad = (if_pkt_info_pad_t)pi;
	+ pi_pad->pkt_val[0] = 0; pi_pad->pkt_val[1] = 0; pi_pad->pkt_val[2] = 0;
	+ pi_pad->pkt_val[3] = 0; pi_pad->pkt_val[4] = 0; pi_pad->pkt_val[5] = 0;
	+#ifndef __LP64__
	+ pi_pad->pkt_val[6] = 0; pi_pad->pkt_val[7] = 0; pi_pad->pkt_val[8] = 0;
	+ pi_pad->pkt_val[9] = 0; pi_pad->pkt_val[10] = 0;
	+#endif
	+}
	+
	+static inline void
	+rxd_info_zero(if_rxd_info_t ri)
	+{
	+ if_rxd_info_pad_t ri_pad;
	+ int i;
	+
	+ ri_pad = (if_rxd_info_pad_t)ri;
	+ for (i = 0; i < RXD_LOOP_BOUND; i += 4) {
	+ ri_pad->rxd_val[i] = 0;
	+ ri_pad->rxd_val[i+1] = 0;
	+ ri_pad->rxd_val[i+2] = 0;
	+ ri_pad->rxd_val[i+3] = 0;
	+ }
	+#ifdef __LP64__
	+ ri_pad->rxd_val[RXD_INFO_SIZE-1] = 0;
	+#endif
	+}
	+
	/*
	* Only allow a single packet to take up most 1/nth of the tx ring
	*/
	#define MAX_SINGLE_PACKET_FRACTION 12
	#define IF_BAD_DMA (bus_addr_t)-1

	-static int enable_msix = 1;
	-
	-#define mtx_held(m) (((m)->mtx_lock & ~MTX_FLAGMASK) != (uintptr_t)0)
	-
	-
	-
	#define CTX_ACTIVE(ctx) ((if_getdrvflags((ctx)->ifc_ifp) & IFF_DRV_RUNNING))

	#define CTX_LOCK_INIT(_sc, _name) mtx_init(&(_sc)->ifc_mtx, _name, "iflib ctx lock", MTX_DEF)
	@@ -455,12 +539,6 @@
	#define CTX_LOCK_DESTROY(ctx) mtx_destroy(&(ctx)->ifc_mtx)


	-#define TXDB_LOCK_INIT(txq) mtx_init(&(txq)->ift_db_mtx, (txq)->ift_db_mtx_name, NULL, MTX_DEF)
	-#define TXDB_TRYLOCK(txq) mtx_trylock(&(txq)->ift_db_mtx)
	-#define TXDB_LOCK(txq) mtx_lock(&(txq)->ift_db_mtx)
	-#define TXDB_UNLOCK(txq) mtx_unlock(&(txq)->ift_db_mtx)
	-#define TXDB_LOCK_DESTROY(txq) mtx_destroy(&(txq)->ift_db_mtx)
	-
	#define CALLOUT_LOCK(txq) mtx_lock(&txq->ift_mtx)
	#define CALLOUT_UNLOCK(txq) mtx_unlock(&txq->ift_mtx)

	@@ -480,6 +558,7 @@
	MODULE_DEPEND(iflib, pci, 1, 1, 1);
	MODULE_DEPEND(iflib, ether, 1, 1, 1);

	+TASKQGROUP_DEFINE(if_io_tqg, mp_ncpus, 1);
	TASKQGROUP_DEFINE(if_config_tqg, 1, 1);

	#ifndef IFLIB_DEBUG_COUNTERS
	@@ -497,9 +576,11 @@
	* XXX need to ensure that this can't accidentally cause the head to be moved backwards
	*/
	static int iflib_min_tx_latency = 0;
	-
	SYSCTL_INT(_net_iflib, OID_AUTO, min_tx_latency, CTLFLAG_RW,
	- &iflib_min_tx_latency, 0, "minimize transmit latency at the possibel expense of throughput");
	+ &iflib_min_tx_latency, 0, "minimize transmit latency at the possible expense of throughput");
	+static int iflib_no_tx_batch = 0;
	+SYSCTL_INT(_net_iflib, OID_AUTO, no_tx_batch, CTLFLAG_RW,
	+ &iflib_no_tx_batch, 0, "minimize transmit latency at the possible expense of throughput");


	#if IFLIB_DEBUG_COUNTERS
	@@ -544,11 +625,14 @@


	static int iflib_encap_load_mbuf_fail;
	+static int iflib_encap_pad_mbuf_fail;
	static int iflib_encap_txq_avail_fail;
	static int iflib_encap_txd_encap_fail;

	SYSCTL_INT(_net_iflib, OID_AUTO, encap_load_mbuf_fail, CTLFLAG_RD,
	&iflib_encap_load_mbuf_fail, 0, "# busdma load failures");
	+SYSCTL_INT(_net_iflib, OID_AUTO, encap_pad_mbuf_fail, CTLFLAG_RD,
	+ &iflib_encap_pad_mbuf_fail, 0, "# runt frame pad failures");
	SYSCTL_INT(_net_iflib, OID_AUTO, encap_txq_avail_fail, CTLFLAG_RD,
	&iflib_encap_txq_avail_fail, 0, "# txq avail failures");
	SYSCTL_INT(_net_iflib, OID_AUTO, encap_txd_encap_fail, CTLFLAG_RD,
	@@ -594,10 +678,24 @@
	&iflib_verbose_debug, 0, "enable verbose debugging");

	#define DBG_COUNTER_INC(name) atomic_add_int(&(iflib_ ## name), 1)
	+static void
	+iflib_debug_reset(void)
	+{
	+ iflib_tx_seen = iflib_tx_sent = iflib_tx_encap = iflib_rx_allocs =
	+ iflib_fl_refills = iflib_fl_refills_large = iflib_tx_frees =
	+ iflib_txq_drain_flushing = iflib_txq_drain_oactive =
	+ iflib_txq_drain_notready = iflib_txq_drain_encapfail =
	+ iflib_encap_load_mbuf_fail = iflib_encap_pad_mbuf_fail =
	+ iflib_encap_txq_avail_fail = iflib_encap_txd_encap_fail =
	+ iflib_task_fn_rxs = iflib_rx_intr_enables = iflib_fast_intrs =
	+ iflib_intr_link = iflib_intr_msix = iflib_rx_unavail =
	+ iflib_rx_ctx_inactive = iflib_rx_zero_len = iflib_rx_if_input =
	+ iflib_rx_mbuf_null = iflib_rxd_flush = 0;
	+}

	#else
	#define DBG_COUNTER_INC(name)
	-
	+static void iflib_debug_reset(void) {}
	#endif


	@@ -608,7 +706,7 @@
	static void iflib_rx_structures_free(if_ctx_t ctx);
	static int iflib_queues_alloc(if_ctx_t ctx);
	static int iflib_tx_credits_update(if_ctx_t ctx, iflib_txq_t txq);
	-static int iflib_rxd_avail(if_ctx_t ctx, iflib_rxq_t rxq, int cidx, int budget);
	+static int iflib_rxd_avail(if_ctx_t ctx, iflib_rxq_t rxq, qidx_t cidx, qidx_t budget);
	static int iflib_qset_structures_setup(if_ctx_t ctx);
	static int iflib_msix_init(if_ctx_t ctx);
	static int iflib_legacy_setup(if_ctx_t ctx, driver_filter_t filter, void filterarg, int rid, char *str);
	@@ -618,8 +716,14 @@
	static void iflib_init_locked(if_ctx_t ctx);
	static void iflib_add_device_sysctl_pre(if_ctx_t ctx);
	static void iflib_add_device_sysctl_post(if_ctx_t ctx);
	+static void iflib_ifmp_purge(iflib_txq_t txq);
	+static void _iflib_pre_assert(if_softc_ctx_t scctx);
	+static void iflib_stop(if_ctx_t ctx);
	+static void iflib_if_init_locked(if_ctx_t ctx);
	+#ifndef __NO_STRICT_ALIGNMENT
	+static struct mbuf * iflib_fixup_rx(struct mbuf *m);
	+#endif

	-
	#ifdef DEV_NETMAP
	#include <sys/selinfo.h>
	#include <net/netmap.h>
	@@ -627,6 +731,8 @@

	MODULE_DEPEND(iflib, netmap, 1, 1, 1);

	+static int netmap_fl_refill(iflib_rxq_t rxq, struct netmap_kring *kring, uint32_t nm_i, bool init);
	+
	/*
	* device-specific sysctl variables:
	*
	@@ -662,6 +768,7 @@
	{
	struct ifnet *ifp = na->ifp;
	if_ctx_t ctx = ifp->if_softc;
	+ int status;

	CTX_LOCK(ctx);
	IFDI_INTR_DISABLE(ctx);
	@@ -670,7 +777,7 @@
	ifp->if_drv_flags &= ~(IFF_DRV_RUNNING \| IFF_DRV_OACTIVE);

	if (!CTX_IS_VF(ctx))
	- IFDI_CRCSTRIP_SET(ctx, onoff);
	+ IFDI_CRCSTRIP_SET(ctx, onoff, iflib_crcstrip);

	/* enable or disable flags and callbacks in na and ifp */
	if (onoff) {
	@@ -678,12 +785,90 @@
	} else {
	nm_clear_native_flags(na);
	}
	- IFDI_INIT(ctx);
	- IFDI_CRCSTRIP_SET(ctx, onoff); // XXX why twice ?
	+ iflib_stop(ctx);
	+ iflib_init_locked(ctx);
	+ IFDI_CRCSTRIP_SET(ctx, onoff, iflib_crcstrip); // XXX why twice ?
	+ status = ifp->if_drv_flags & IFF_DRV_RUNNING ? 0 : 1;
	+ if (status)
	+ nm_clear_native_flags(na);
	CTX_UNLOCK(ctx);
	- return (ifp->if_drv_flags & IFF_DRV_RUNNING ? 0 : 1);
	+ return (status);
	}

	+static int
	+netmap_fl_refill(iflib_rxq_t rxq, struct netmap_kring *kring, uint32_t nm_i, bool init)
	+{
	+ struct netmap_adapter *na = kring->na;
	+ u_int const lim = kring->nkr_num_slots - 1;
	+ u_int head = kring->rhead;
	+ struct netmap_ring *ring = kring->ring;
	+ bus_dmamap_t *map;
	+ struct if_rxd_update iru;
	+ if_ctx_t ctx = rxq->ifr_ctx;
	+ iflib_fl_t fl = &rxq->ifr_fl[0];
	+ uint32_t refill_pidx, nic_i;
	+
	+ if (nm_i == head && __predict_true(!init))
	+ return 0;
	+ iru_init(&iru, rxq, 0 /* flid */);
	+ map = fl->ifl_sds.ifsd_map;
	+ refill_pidx = netmap_idx_k2n(kring, nm_i);
	+ /*
	+ * IMPORTANT: we must leave one free slot in the ring,
	+ * so move head back by one unit
	+ */
	+ head = nm_prev(head, lim);
	+ while (nm_i != head) {
	+ for (int tmp_pidx = 0; tmp_pidx < IFLIB_MAX_RX_REFRESH && nm_i != head; tmp_pidx++) {
	+ struct netmap_slot *slot = &ring->slot[nm_i];
	+ void *addr = PNMB(na, slot, &fl->ifl_bus_addrs[tmp_pidx]);
	+ uint32_t nic_i_dma = refill_pidx;
	+ nic_i = netmap_idx_k2n(kring, nm_i);
	+
	+ MPASS(tmp_pidx < IFLIB_MAX_RX_REFRESH);
	+
	+ if (addr == NETMAP_BUF_BASE(na)) /* bad buf */
	+ return netmap_ring_reinit(kring);
	+
	+ fl->ifl_vm_addrs[tmp_pidx] = addr;
	+ if (__predict_false(init) && map) {
	+ netmap_load_map(na, fl->ifl_ifdi->idi_tag, map[nic_i], addr);
	+ } else if (map && (slot->flags & NS_BUF_CHANGED)) {
	+ /* buffer has changed, reload map */
	+ netmap_reload_map(na, fl->ifl_ifdi->idi_tag, map[nic_i], addr);
	+ }
	+ slot->flags &= ~NS_BUF_CHANGED;
	+
	+ nm_i = nm_next(nm_i, lim);
	+ fl->ifl_rxd_idxs[tmp_pidx] = nic_i = nm_next(nic_i, lim);
	+ if (nm_i != head && tmp_pidx < IFLIB_MAX_RX_REFRESH-1)
	+ continue;
	+
	+ iru.iru_pidx = refill_pidx;
	+ iru.iru_count = tmp_pidx+1;
	+ ctx->isc_rxd_refill(ctx->ifc_softc, &iru);
	+
	+ refill_pidx = nic_i;
	+ if (map == NULL)
	+ continue;
	+
	+ for (int n = 0; n < iru.iru_count; n++) {
	+ bus_dmamap_sync(fl->ifl_ifdi->idi_tag, map[nic_i_dma],
	+ BUS_DMASYNC_PREREAD);
	+ /* XXX - change this to not use the netmap func*/
	+ nic_i_dma = nm_next(nic_i_dma, lim);
	+ }
	+ }
	+ }
	+ kring->nr_hwcur = head;
	+
	+ if (map)
	+ bus_dmamap_sync(fl->ifl_ifdi->idi_tag, fl->ifl_ifdi->idi_map,
	+ BUS_DMASYNC_PREREAD \| BUS_DMASYNC_PREWRITE);
	+ ctx->isc_rxd_flush(ctx->ifc_softc, rxq->ifr_id, fl->ifl_id, nic_i);
	+ return (0);
	+}
	+
	/*
	* Reconcile kernel and user view of the transmit ring.
	*
	@@ -720,14 +905,11 @@
	if_ctx_t ctx = ifp->if_softc;
	iflib_txq_t txq = &ctx->ifc_txqs[kring->ring_id];

	- pi.ipi_segs = txq->ift_segs;
	- pi.ipi_qsidx = kring->ring_id;
	- pi.ipi_ndescs = 0;
	+ if (txq->ift_sds.ifsd_map)
	+ bus_dmamap_sync(txq->ift_desc_tag, txq->ift_ifdi->idi_map,
	+ BUS_DMASYNC_POSTREAD \| BUS_DMASYNC_POSTWRITE);

	- bus_dmamap_sync(txq->ift_desc_tag, txq->ift_ifdi->idi_map,
	- BUS_DMASYNC_POSTREAD \| BUS_DMASYNC_POSTWRITE);

	-
	/*
	* First part: process new packets to send.
	* nm_i is the current index in the netmap ring,
	@@ -750,13 +932,17 @@
	* to prefetch the next slot and txr entry.
	*/

	- nm_i = kring->nr_hwcur;
	+ nm_i = netmap_idx_n2k(kring, kring->nr_hwcur);
	+ pkt_info_zero(&pi);
	+ pi.ipi_segs = txq->ift_segs;
	+ pi.ipi_qsidx = kring->ring_id;
	if (nm_i != head) { /* we have new packets to send */
	nic_i = netmap_idx_k2n(kring, nm_i);

	__builtin_prefetch(&ring->slot[nm_i]);
	__builtin_prefetch(&txq->ift_sds.ifsd_m[nic_i]);
	- __builtin_prefetch(&txq->ift_sds.ifsd_map[nic_i]);
	+ if (txq->ift_sds.ifsd_map)
	+ __builtin_prefetch(&txq->ift_sds.ifsd_map[nic_i]);

	for (n = 0; nm_i != head; n++) {
	struct netmap_slot *slot = &ring->slot[nm_i];
	@@ -768,6 +954,11 @@
	IPI_TX_INTR : 0;

	/* device-specific */
	+ pi.ipi_len = len;
	+ pi.ipi_segs[0].ds_addr = paddr;
	+ pi.ipi_segs[0].ds_len = len;
	+ pi.ipi_nsegs = 1;
	+ pi.ipi_ndescs = 0;
	pi.ipi_pidx = nic_i;
	pi.ipi_flags = flags;

	@@ -777,27 +968,28 @@
	/* prefetch for next round */
	__builtin_prefetch(&ring->slot[nm_i + 1]);
	__builtin_prefetch(&txq->ift_sds.ifsd_m[nic_i + 1]);
	- __builtin_prefetch(&txq->ift_sds.ifsd_map[nic_i + 1]);
	+ if (txq->ift_sds.ifsd_map) {
	+ __builtin_prefetch(&txq->ift_sds.ifsd_map[nic_i + 1]);

	- NM_CHECK_ADDR_LEN(na, addr, len);
	+ NM_CHECK_ADDR_LEN(na, addr, len);

	- if (slot->flags & NS_BUF_CHANGED) {
	- /* buffer has changed, reload map */
	- netmap_reload_map(na, txq->ift_desc_tag, txq->ift_sds.ifsd_map[nic_i], addr);
	+ if (slot->flags & NS_BUF_CHANGED) {
	+ /* buffer has changed, reload map */
	+ netmap_reload_map(na, txq->ift_desc_tag, txq->ift_sds.ifsd_map[nic_i], addr);
	+ }
	+ /* make sure changes to the buffer are synced */
	+ bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_sds.ifsd_map[nic_i],
	+ BUS_DMASYNC_PREWRITE);
	}
	slot->flags &= ~(NS_REPORT \| NS_BUF_CHANGED);
	-
	- /* make sure changes to the buffer are synced */
	- bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_sds.ifsd_map[nic_i],
	- BUS_DMASYNC_PREWRITE);
	-
	nm_i = nm_next(nm_i, lim);
	nic_i = nm_next(nic_i, lim);
	}
	kring->nr_hwcur = head;

	/* synchronize the NIC ring */
	- bus_dmamap_sync(txq->ift_desc_tag, txq->ift_ifdi->idi_map,
	+ if (txq->ift_sds.ifsd_map)
	+ bus_dmamap_sync(txq->ift_desc_tag, txq->ift_ifdi->idi_map,
	BUS_DMASYNC_PREREAD \| BUS_DMASYNC_PREWRITE);

	/* (re)start the tx unit up to slot nic_i (excluded) */
	@@ -832,30 +1024,29 @@
	iflib_netmap_rxsync(struct netmap_kring *kring, int flags)
	{
	struct netmap_adapter *na = kring->na;
	- struct ifnet *ifp = na->ifp;
	struct netmap_ring *ring = kring->ring;
	- u_int nm_i; /* index into the netmap ring */
	- u_int nic_i; /* index into the NIC ring */
	+ uint32_t nm_i; /* index into the netmap ring */
	+ uint32_t nic_i; /* index into the NIC ring */
	u_int i, n;
	u_int const lim = kring->nkr_num_slots - 1;
	- u_int const head = kring->rhead;
	+ u_int const head = netmap_idx_n2k(kring, kring->rhead);
	int force_update = (flags & NAF_FORCE_READ) \|\| kring->nr_kflags & NKR_PENDINTR;
	struct if_rxd_info ri;
	- /* device-specific */
	+
	+ struct ifnet *ifp = na->ifp;
	if_ctx_t ctx = ifp->if_softc;
	iflib_rxq_t rxq = &ctx->ifc_rxqs[kring->ring_id];
	iflib_fl_t fl = rxq->ifr_fl;
	if (head > lim)
	return netmap_ring_reinit(kring);

	- bzero(&ri, sizeof(ri));
	- ri.iri_qsidx = kring->ring_id;
	- ri.iri_ifp = ctx->ifc_ifp;
	/* XXX check sync modes */
	- for (i = 0, fl = rxq->ifr_fl; i < rxq->ifr_nfl; i++, fl++)
	+ for (i = 0, fl = rxq->ifr_fl; i < rxq->ifr_nfl; i++, fl++) {
	+ if (fl->ifl_sds.ifsd_map == NULL)
	+ continue;
	bus_dmamap_sync(rxq->ifr_fl[i].ifl_desc_tag, fl->ifl_ifdi->idi_map,
	BUS_DMASYNC_POSTREAD \| BUS_DMASYNC_POSTWRITE);
	-
	+ }
	/*
	* First part: import newly received packets.
	*
	@@ -876,19 +1067,24 @@
	int error, avail;
	uint16_t slot_flags = kring->nkr_slot_flags;

	- for (fl = rxq->ifr_fl, i = 0; i < rxq->ifr_nfl; i++, fl++) {
	+ for (i = 0; i < rxq->ifr_nfl; i++) {
	+ fl = &rxq->ifr_fl[i];
	nic_i = fl->ifl_cidx;
	nm_i = netmap_idx_n2k(kring, nic_i);
	- avail = ctx->isc_rxd_available(ctx->ifc_softc, kring->ring_id, nic_i, INT_MAX);
	+ avail = iflib_rxd_avail(ctx, rxq, nic_i, USHRT_MAX);
	for (n = 0; avail > 0; n++, avail--) {
	+ rxd_info_zero(&ri);
	+ ri.iri_frags = rxq->ifr_frags;
	+ ri.iri_qsidx = kring->ring_id;
	+ ri.iri_ifp = ctx->ifc_ifp;
	+ ri.iri_cidx = nic_i;
	+
	error = ctx->isc_rxd_pkt_get(ctx->ifc_softc, &ri);
	- if (error)
	- ring->slot[nm_i].len = 0;
	- else
	- ring->slot[nm_i].len = ri.iri_len - crclen;
	+ ring->slot[nm_i].len = error ? 0 : ri.iri_len - crclen;
	ring->slot[nm_i].flags = slot_flags;
	- bus_dmamap_sync(fl->ifl_ifdi->idi_tag,
	- fl->ifl_sds[nic_i].ifsd_map, BUS_DMASYNC_POSTREAD);
	+ if (fl->ifl_sds.ifsd_map)
	+ bus_dmamap_sync(fl->ifl_ifdi->idi_tag,
	+ fl->ifl_sds.ifsd_map[nic_i], BUS_DMASYNC_POSTREAD);
	nm_i = nm_next(nm_i, lim);
	nic_i = nm_next(nic_i, lim);
	}
	@@ -899,7 +1095,7 @@
	iflib_rx_miss_bufs += n;
	}
	fl->ifl_cidx = nic_i;
	- kring->nr_hwtail = nm_i;
	+ kring->nr_hwtail = netmap_idx_k2n(kring, nm_i);
	}
	kring->nr_kflags &= ~NKR_PENDINTR;
	}
	@@ -913,49 +1109,9 @@
	* nm_i == (nic_i + kring->nkr_hwofs) % ring_size
	*/
	/* XXX not sure how this will work with multiple free lists */
	- nm_i = kring->nr_hwcur;
	- if (nm_i != head) {
	- nic_i = netmap_idx_k2n(kring, nm_i);
	- for (n = 0; nm_i != head; n++) {
	- struct netmap_slot *slot = &ring->slot[nm_i];
	- uint64_t paddr;
	- caddr_t vaddr;
	- void *addr = PNMB(na, slot, &paddr);
	+ nm_i = netmap_idx_n2k(kring, kring->nr_hwcur);

	- if (addr == NETMAP_BUF_BASE(na)) /* bad buf */
	- goto ring_reset;
	-
	- vaddr = addr;
	- if (slot->flags & NS_BUF_CHANGED) {
	- /* buffer has changed, reload map */
	- netmap_reload_map(na, fl->ifl_ifdi->idi_tag, fl->ifl_sds[nic_i].ifsd_map, addr);
	- slot->flags &= ~NS_BUF_CHANGED;
	- }
	- /*
	- * XXX we should be batching this operation - TODO
	- */
	- ctx->isc_rxd_refill(ctx->ifc_softc, rxq->ifr_id, fl->ifl_id, nic_i, &paddr, &vaddr, 1, fl->ifl_buf_size);
	- bus_dmamap_sync(fl->ifl_ifdi->idi_tag, fl->ifl_sds[nic_i].ifsd_map,
	- BUS_DMASYNC_PREREAD);
	- nm_i = nm_next(nm_i, lim);
	- nic_i = nm_next(nic_i, lim);
	- }
	- kring->nr_hwcur = head;
	-
	- bus_dmamap_sync(fl->ifl_ifdi->idi_tag, fl->ifl_ifdi->idi_map,
	- BUS_DMASYNC_PREREAD \| BUS_DMASYNC_PREWRITE);
	- /*
	- * IMPORTANT: we must leave one free slot in the ring,
	- * so move nic_i back by one unit
	- */
	- nic_i = nm_prev(nic_i, lim);
	- ctx->isc_rxd_flush(ctx->ifc_softc, rxq->ifr_id, fl->ifl_id, nic_i);
	- }
	-
	- return 0;
	-
	-ring_reset:
	- return netmap_ring_reinit(kring);
	+ return (netmap_fl_refill(rxq, kring, nm_i, false));
	}

	static int
	@@ -990,6 +1146,8 @@
	slot = netmap_reset(na, NR_TX, txq->ift_id, 0);
	if (slot == NULL)
	return;
	+ if (txq->ift_sds.ifsd_map == NULL)
	+ return;

	for (int i = 0; i < ctx->ifc_softc_ctx.isc_ntxd[0]; i++) {

	@@ -1004,37 +1162,20 @@
	netmap_load_map(na, txq->ift_desc_tag, txq->ift_sds.ifsd_map[i], NMB(na, slot + si));
	}
	}
	+
	static void
	iflib_netmap_rxq_init(if_ctx_t ctx, iflib_rxq_t rxq)
	{
	struct netmap_adapter *na = NA(ctx->ifc_ifp);
	+ struct netmap_kring *kring = &na->rx_rings[rxq->ifr_id];
	struct netmap_slot *slot;
	- iflib_rxsd_t sd;
	- int nrxd;
	+ uint32_t nm_i;

	slot = netmap_reset(na, NR_RX, rxq->ifr_id, 0);
	if (slot == NULL)
	return;
	- sd = rxq->ifr_fl[0].ifl_sds;
	- nrxd = ctx->ifc_softc_ctx.isc_nrxd[0];
	- for (int i = 0; i < nrxd; i++, sd++) {
	- int sj = netmap_idx_n2k(&na->rx_rings[rxq->ifr_id], i);
	- uint64_t paddr;
	- void *addr;
	- caddr_t vaddr;
	-
	- vaddr = addr = PNMB(na, slot + sj, &paddr);
	- netmap_load_map(na, rxq->ifr_fl[0].ifl_ifdi->idi_tag, sd->ifsd_map, addr);
	- /* Update descriptor and the cached value */
	- ctx->isc_rxd_refill(ctx->ifc_softc, rxq->ifr_id, 0 /* fl_id */, i, &paddr, &vaddr, 1, rxq->ifr_fl[0].ifl_buf_size);
	- }
	- /* preserve queue */
	- if (ctx->ifc_ifp->if_capenable & IFCAP_NETMAP) {
	- struct netmap_kring *kring = &na->rx_rings[rxq->ifr_id];
	- int t = na->num_rx_desc - 1 - nm_kr_rxspace(kring);
	- ctx->isc_rxd_flush(ctx->ifc_softc, rxq->ifr_id, 0 /* fl_id */, t);
	- } else
	- ctx->isc_rxd_flush(ctx->ifc_softc, rxq->ifr_id, 0 /* fl_id */, nrxd-1);
	+ nm_i = netmap_idx_n2k(kring, 0);
	+ netmap_fl_refill(rxq, kring, nm_i, true);
	}

	#define iflib_netmap_detach(ifp) netmap_detach(ifp)
	@@ -1046,6 +1187,7 @@

	#define iflib_netmap_attach(ctx) (0)
	#define netmap_rx_irq(ifp, qid, budget) (0)
	+#define netmap_tx_irq(ifp, qid) do {} while (0)

	#endif

	@@ -1055,11 +1197,34 @@
	{
	__asm volatile("prefetcht0 %0" :: "m" ((unsigned long )x));
	}
	+static __inline void
	+prefetch2cachelines(void *x)
	+{
	+ __asm volatile("prefetcht0 %0" :: "m" ((unsigned long )x));
	+#if (CACHE_LINE_SIZE < 128)
	+ __asm volatile("prefetcht0 %0" :: "m" ((((unsigned long )x)+CACHE_LINE_SIZE/(sizeof(unsigned long)))));
	+#endif
	+}
	#else
	#define prefetch(x)
	+#define prefetch2cachelines(x)
	#endif

	static void
	+iru_init(if_rxd_update_t iru, iflib_rxq_t rxq, uint8_t flid)
	+{
	+ iflib_fl_t fl;
	+
	+ fl = &rxq->ifr_fl[flid];
	+ iru->iru_paddrs = fl->ifl_bus_addrs;
	+ iru->iru_vaddrs = &fl->ifl_vm_addrs[0];
	+ iru->iru_idxs = fl->ifl_rxd_idxs;
	+ iru->iru_qsidx = rxq->ifr_id;
	+ iru->iru_buf_size = fl->ifl_buf_size;
	+ iru->iru_flidx = fl->ifl_id;
	+}
	+
	+static void
	_iflib_dmamap_cb(void arg, bus_dma_segment_t segs, int nseg, int err)
	{
	if (err)
	@@ -1172,11 +1337,36 @@
	iflib_dma_free(*dmaiter);
	}

	+#ifdef EARLY_AP_STARTUP
	+static const int iflib_started = 1;
	+#else
	+/*
	+ * We used to abuse the smp_started flag to decide if the queues have been
	+ * fully initialized (by late taskqgroup_adjust() calls in a SYSINIT()).
	+ * That gave bad races, since the SYSINIT() runs strictly after smp_started
	+ * is set. Run a SYSINIT() strictly after that to just set a usable
	+ * completion flag.
	+ */
	+
	+static int iflib_started;
	+
	+static void
	+iflib_record_started(void *arg)
	+{
	+ iflib_started = 1;
	+}
	+
	+SYSINIT(iflib_record_started, SI_SUB_SMP + 1, SI_ORDER_FIRST,
	+ iflib_record_started, NULL);
	+#endif
	+
	static int
	iflib_fast_intr(void *arg)
	{
	iflib_filter_info_t info = arg;
	struct grouptask *gtask = info->ifi_task;
	+ if (!iflib_started)
	+ return (FILTER_HANDLED);

	DBG_COUNTER_INC(fast_intrs);
	if (info->ifi_filter != NULL && info->ifi_filter(info->ifi_filter_arg) == FILTER_HANDLED)
	@@ -1187,19 +1377,77 @@
	}

	static int
	+iflib_fast_intr_rxtx(void *arg)
	+{
	+ iflib_filter_info_t info = arg;
	+ struct grouptask *gtask = info->ifi_task;
	+ iflib_rxq_t rxq = (iflib_rxq_t)info->ifi_ctx;
	+ if_ctx_t ctx;
	+ int i, cidx;
	+
	+ if (!iflib_started)
	+ return (FILTER_HANDLED);
	+
	+ DBG_COUNTER_INC(fast_intrs);
	+ if (info->ifi_filter != NULL && info->ifi_filter(info->ifi_filter_arg) == FILTER_HANDLED)
	+ return (FILTER_HANDLED);
	+
	+ for (i = 0; i < rxq->ifr_ntxqirq; i++) {
	+ qidx_t txqid = rxq->ifr_txqid[i];
	+
	+ ctx = rxq->ifr_ctx;
	+
	+ if (!ctx->isc_txd_credits_update(ctx->ifc_softc, txqid, false)) {
	+ IFDI_TX_QUEUE_INTR_ENABLE(ctx, txqid);
	+ continue;
	+ }
	+ GROUPTASK_ENQUEUE(&ctx->ifc_txqs[txqid].ift_task);
	+ }
	+ if (ctx->ifc_sctx->isc_flags & IFLIB_HAS_RXCQ)
	+ cidx = rxq->ifr_cq_cidx;
	+ else
	+ cidx = rxq->ifr_fl[0].ifl_cidx;
	+ if (iflib_rxd_avail(ctx, rxq, cidx, 1))
	+ GROUPTASK_ENQUEUE(gtask);
	+ else
	+ IFDI_RX_QUEUE_INTR_ENABLE(ctx, rxq->ifr_id);
	+ return (FILTER_HANDLED);
	+}
	+
	+
	+static int
	+iflib_fast_intr_ctx(void *arg)
	+{
	+ iflib_filter_info_t info = arg;
	+ struct grouptask *gtask = info->ifi_task;
	+
	+ if (!iflib_started)
	+ return (FILTER_HANDLED);
	+
	+ DBG_COUNTER_INC(fast_intrs);
	+ if (info->ifi_filter != NULL && info->ifi_filter(info->ifi_filter_arg) == FILTER_HANDLED)
	+ return (FILTER_HANDLED);
	+
	+ GROUPTASK_ENQUEUE(gtask);
	+ return (FILTER_HANDLED);
	+}
	+
	+static int
	_iflib_irq_alloc(if_ctx_t ctx, if_irq_t irq, int rid,
	driver_filter_t filter, driver_intr_t handler, void *arg,
	char *name)
	{
	- int rc;
	+ int rc, flags;
	struct resource *res;
	- void *tag;
	+ void *tag = NULL;
	device_t dev = ctx->ifc_dev;

	+ flags = RF_ACTIVE;
	+ if (ctx->ifc_flags & IFC_LEGACY)
	+ flags \|= RF_SHAREABLE;
	MPASS(rid < 512);
	irq->ii_rid = rid;
	- res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &irq->ii_rid,
	- RF_SHAREABLE \| RF_ACTIVE);
	+ res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &irq->ii_rid, flags);
	if (res == NULL) {
	device_printf(dev,
	"failed to allocate IRQ for rid %d, name %s.\n", rid, name);
	@@ -1265,11 +1513,6 @@
	(uintmax_t)sctx->isc_tx_maxsize, nsegments, (uintmax_t)sctx->isc_tx_maxsegsize);
	goto fail;
	}
	-#ifdef IFLIB_DIAGNOSTICS
	- device_printf(dev,"maxsize: %zd nsegments: %d maxsegsize: %zd\n",
	- sctx->isc_tx_maxsize, nsegments, sctx->isc_tx_maxsegsize);
	-
	-#endif
	if ((err = bus_dma_tag_create(bus_get_dma_tag(dev),
	1, 0, /* alignment, bounds */
	BUS_SPACE_MAXADDR, /* lowaddr */
	@@ -1286,11 +1529,6 @@

	goto fail;
	}
	-#ifdef IFLIB_DIAGNOSTICS
	- device_printf(dev,"TSO maxsize: %d ntsosegments: %d maxsegsize: %d\n",
	- scctx->isc_tx_tso_size_max, ntsosegments,
	- scctx->isc_tx_tso_segsize_max);
	-#endif
	if (!(txq->ift_sds.ifsd_flags =
	(uint8_t ) malloc(sizeof(uint8_t)
	scctx->isc_ntxd[txq->ift_br_offset], M_IFLIB, M_NOWAIT \| M_ZERO))) {
	@@ -1307,7 +1545,7 @@
	}

	/* Create the descriptor buffer dma maps */
	-#if defined(ACPI_DMAR) \|\| (!(defined(__i386__) && !defined(__amd64__)))
	+#if defined(ACPI_DMAR) \|\| (! (defined(__i386__) \|\| defined(__amd64__)))
	if ((ctx->ifc_flags & IFC_DMAR) == 0)
	return (0);

	@@ -1406,11 +1644,14 @@
	iflib_dma_info_t di;
	int i;

	- /* Set number of descriptors available */
	+ /* Set number of descriptors available */
	txq->ift_qstatus = IFLIB_QUEUE_IDLE;
	+ /* XXX make configurable */
	+ txq->ift_update_freq = IFLIB_DEFAULT_TX_UPDATE_FREQ;

	/* Reset indices */
	- txq->ift_cidx_processed = txq->ift_pidx = txq->ift_cidx = txq->ift_npending = 0;
	+ txq->ift_cidx_processed = 0;
	+ txq->ift_pidx = txq->ift_cidx = txq->ift_npending = 0;
	txq->ift_size = scctx->isc_ntxd[txq->ift_br_offset];

	for (i = 0, di = txq->ift_ifdi; i < ctx->ifc_nhwtxqs; i++, di++)
	@@ -1439,7 +1680,6 @@
	if_softc_ctx_t scctx = &ctx->ifc_softc_ctx;
	device_t dev = ctx->ifc_dev;
	iflib_fl_t fl;
	- iflib_rxsd_t rxsd;
	int err;

	MPASS(scctx->isc_nrxd[0] > 0);
	@@ -1447,13 +1687,6 @@

	fl = rxq->ifr_fl;
	for (int i = 0; i < rxq->ifr_nfl; i++, fl++) {
	- fl->ifl_sds = malloc(sizeof(struct iflib_sw_rx_desc) *
	- scctx->isc_nrxd[rxq->ifr_fl_offset], M_IFLIB,
	- M_WAITOK \| M_ZERO);
	- if (fl->ifl_sds == NULL) {
	- device_printf(dev, "Unable to allocate rx sw desc memory\n");
	- return (ENOMEM);
	- }
	fl->ifl_size = scctx->isc_nrxd[rxq->ifr_fl_offset]; /* this isn't necessarily the same */
	err = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */
	1, 0, /* alignment, bounds */
	@@ -1472,16 +1705,48 @@
	__func__, err);
	goto fail;
	}
	+ if (!(fl->ifl_sds.ifsd_flags =
	+ (uint8_t ) malloc(sizeof(uint8_t)
	+ scctx->isc_nrxd[rxq->ifr_fl_offset], M_IFLIB, M_NOWAIT \| M_ZERO))) {
	+ device_printf(dev, "Unable to allocate tx_buffer memory\n");
	+ err = ENOMEM;
	+ goto fail;
	+ }
	+ if (!(fl->ifl_sds.ifsd_m =
	+ (struct mbuf *) malloc(sizeof(struct mbuf ) *
	+ scctx->isc_nrxd[rxq->ifr_fl_offset], M_IFLIB, M_NOWAIT \| M_ZERO))) {
	+ device_printf(dev, "Unable to allocate tx_buffer memory\n");
	+ err = ENOMEM;
	+ goto fail;
	+ }
	+ if (!(fl->ifl_sds.ifsd_cl =
	+ (caddr_t ) malloc(sizeof(caddr_t)
	+ scctx->isc_nrxd[rxq->ifr_fl_offset], M_IFLIB, M_NOWAIT \| M_ZERO))) {
	+ device_printf(dev, "Unable to allocate tx_buffer memory\n");
	+ err = ENOMEM;
	+ goto fail;
	+ }

	- rxsd = fl->ifl_sds;
	- for (int i = 0; i < scctx->isc_nrxd[rxq->ifr_fl_offset]; i++, rxsd++) {
	- err = bus_dmamap_create(fl->ifl_desc_tag, 0, &rxsd->ifsd_map);
	- if (err) {
	- device_printf(dev, "%s: bus_dmamap_create failed: %d\n",
	- __func__, err);
	+ /* Create the descriptor buffer dma maps */
	+#if defined(ACPI_DMAR) \|\| (! (defined(__i386__) \|\| defined(__amd64__)))
	+ if ((ctx->ifc_flags & IFC_DMAR) == 0)
	+ continue;
	+
	+ if (!(fl->ifl_sds.ifsd_map =
	+ (bus_dmamap_t ) malloc(sizeof(bus_dmamap_t) scctx->isc_nrxd[rxq->ifr_fl_offset], M_IFLIB, M_NOWAIT \| M_ZERO))) {
	+ device_printf(dev, "Unable to allocate tx_buffer map memory\n");
	+ err = ENOMEM;
	+ goto fail;
	+ }
	+
	+ for (int i = 0; i < scctx->isc_nrxd[rxq->ifr_fl_offset]; i++) {
	+ err = bus_dmamap_create(fl->ifl_desc_tag, 0, &fl->ifl_sds.ifsd_map[i]);
	+ if (err != 0) {
	+ device_printf(dev, "Unable to create RX buffer DMA map\n");
	goto fail;
	}
	}
	+#endif
	}
	return (0);

	@@ -1531,20 +1796,32 @@
	_iflib_fl_refill(if_ctx_t ctx, iflib_fl_t fl, int count)
	{
	struct mbuf *m;
	- int pidx = fl->ifl_pidx;
	- iflib_rxsd_t rxsd = &fl->ifl_sds[pidx];
	- caddr_t cl;
	+ int idx, frag_idx = fl->ifl_fragidx;
	+ int pidx = fl->ifl_pidx;
	+ caddr_t cl, *sd_cl;
	+ struct mbuf **sd_m;
	+ uint8_t *sd_flags;
	+ struct if_rxd_update iru;
	+ bus_dmamap_t *sd_map;
	int n, i = 0;
	uint64_t bus_addr;
	int err;
	+ qidx_t credits;

	+ sd_m = fl->ifl_sds.ifsd_m;
	+ sd_map = fl->ifl_sds.ifsd_map;
	+ sd_cl = fl->ifl_sds.ifsd_cl;
	+ sd_flags = fl->ifl_sds.ifsd_flags;
	+ idx = pidx;
	+ credits = fl->ifl_credits;
	+
	n = count;
	MPASS(n > 0);
	- MPASS(fl->ifl_credits + n <= fl->ifl_size);
	+ MPASS(credits + n <= fl->ifl_size);

	if (pidx < fl->ifl_cidx)
	MPASS(pidx + n <= fl->ifl_cidx);
	- if (pidx == fl->ifl_cidx && (fl->ifl_credits < fl->ifl_size))
	+ if (pidx == fl->ifl_cidx && (credits < fl->ifl_size))
	MPASS(fl->ifl_gen == 0);
	if (pidx > fl->ifl_cidx)
	MPASS(n <= fl->ifl_size - pidx + fl->ifl_cidx);
	@@ -1552,7 +1829,7 @@
	DBG_COUNTER_INC(fl_refills);
	if (n > 8)
	DBG_COUNTER_INC(fl_refills_large);
	-
	+ iru_init(&iru, fl->ifl_rxq, fl->ifl_id);
	while (n--) {
	/*
	* We allocate an uninitialized mbuf + cluster, mbuf is
	@@ -1560,8 +1837,11 @@
	*
	* If the cluster is still set then we know a minimum sized packet was received
	*/
	- if ((cl = rxsd->ifsd_cl) == NULL) {
	- if ((cl = rxsd->ifsd_cl = m_cljget(NULL, M_NOWAIT, fl->ifl_buf_size)) == NULL)
	+ bit_ffc_at(fl->ifl_rx_bitmap, frag_idx, fl->ifl_size, &frag_idx);
	+ if ((frag_idx < 0) \|\| (frag_idx >= fl->ifl_size))
	+ bit_ffc(fl->ifl_rx_bitmap, fl->ifl_size, &frag_idx);
	+ if ((cl = sd_cl[frag_idx]) == NULL) {
	+ if ((cl = sd_cl[frag_idx] = m_cljget(NULL, M_NOWAIT, fl->ifl_buf_size)) == NULL)
	break;
	#if MEMORY_LOGGING
	fl->ifl_cl_enqueued++;
	@@ -1575,19 +1855,6 @@
	#endif

	DBG_COUNTER_INC(rx_allocs);
	-#ifdef notyet
	- if ((rxsd->ifsd_flags & RX_SW_DESC_MAP_CREATED) == 0) {
	- int err;
	-
	- if ((err = bus_dmamap_create(fl->ifl_ifdi->idi_tag, 0, &rxsd->ifsd_map))) {
	- log(LOG_WARNING, "bus_dmamap_create failed %d\n", err);
	- uma_zfree(fl->ifl_zone, cl);
	- n = 0;
	- goto done;
	- }
	- rxsd->ifsd_flags \|= RX_SW_DESC_MAP_CREATED;
	- }
	-#endif
	#if defined(__i386__) \|\| defined(__amd64__)
	if (!IS_DMAR(ctx)) {
	bus_addr = pmap_kextract((vm_offset_t)cl);
	@@ -1599,8 +1866,12 @@

	cb_arg.error = 0;
	q = fl->ifl_rxq;
	- err = bus_dmamap_load(fl->ifl_desc_tag, rxsd->ifsd_map,
	+ MPASS(sd_map != NULL);
	+ MPASS(sd_map[frag_idx] != NULL);
	+ err = bus_dmamap_load(fl->ifl_desc_tag, sd_map[frag_idx],
	cl, fl->ifl_buf_size, _rxq_refill_cb, &cb_arg, 0);
	+ bus_dmamap_sync(fl->ifl_desc_tag, sd_map[frag_idx],
	+ BUS_DMASYNC_PREREAD);

	if (err != 0 \|\| cb_arg.error) {
	/*
	@@ -1614,36 +1885,52 @@
	}
	bus_addr = cb_arg.seg.ds_addr;
	}
	- rxsd->ifsd_flags \|= RX_SW_DESC_INUSE;
	+ bit_set(fl->ifl_rx_bitmap, frag_idx);
	+ sd_flags[frag_idx] \|= RX_SW_DESC_INUSE;

	- MPASS(rxsd->ifsd_m == NULL);
	- rxsd->ifsd_cl = cl;
	- rxsd->ifsd_m = m;
	+ MPASS(sd_m[frag_idx] == NULL);
	+ sd_cl[frag_idx] = cl;
	+ sd_m[frag_idx] = m;
	+ fl->ifl_rxd_idxs[i] = frag_idx;
	fl->ifl_bus_addrs[i] = bus_addr;
	fl->ifl_vm_addrs[i] = cl;
	- rxsd++;
	- fl->ifl_credits++;
	+ credits++;
	i++;
	- MPASS(fl->ifl_credits <= fl->ifl_size);
	- if (++fl->ifl_pidx == fl->ifl_size) {
	- fl->ifl_pidx = 0;
	+ MPASS(credits <= fl->ifl_size);
	+ if (++idx == fl->ifl_size) {
	fl->ifl_gen = 1;
	- rxsd = fl->ifl_sds;
	+ idx = 0;
	}
	if (n == 0 \|\| i == IFLIB_MAX_RX_REFRESH) {
	- ctx->isc_rxd_refill(ctx->ifc_softc, fl->ifl_rxq->ifr_id, fl->ifl_id, pidx,
	- fl->ifl_bus_addrs, fl->ifl_vm_addrs, i, fl->ifl_buf_size);
	+ iru.iru_pidx = pidx;
	+ iru.iru_count = i;
	+ ctx->isc_rxd_refill(ctx->ifc_softc, &iru);
	i = 0;
	- pidx = fl->ifl_pidx;
	+ pidx = idx;
	+ fl->ifl_pidx = idx;
	+ fl->ifl_credits = credits;
	}
	+
	}
	done:
	+ if (i) {
	+ iru.iru_pidx = pidx;
	+ iru.iru_count = i;
	+ ctx->isc_rxd_refill(ctx->ifc_softc, &iru);
	+ fl->ifl_pidx = idx;
	+ fl->ifl_credits = credits;
	+ }
	DBG_COUNTER_INC(rxd_flush);
	if (fl->ifl_pidx == 0)
	pidx = fl->ifl_size - 1;
	else
	pidx = fl->ifl_pidx - 1;
	+
	+ if (sd_map)
	+ bus_dmamap_sync(fl->ifl_ifdi->idi_tag, fl->ifl_ifdi->idi_map,
	+ BUS_DMASYNC_PREREAD \| BUS_DMASYNC_PREWRITE);
	ctx->isc_rxd_flush(ctx->ifc_softc, fl->ifl_rxq->ifr_id, fl->ifl_id, pidx);
	+ fl->ifl_fragidx = frag_idx;
	}

	static __inline void
	@@ -1669,33 +1956,46 @@
	uint32_t i;

	for (i = 0; i < fl->ifl_size; i++) {
	- iflib_rxsd_t d = &fl->ifl_sds[i];
	+ struct mbuf **sd_m = &fl->ifl_sds.ifsd_m[i];
	+ uint8_t *sd_flags = &fl->ifl_sds.ifsd_flags[i];
	+ caddr_t *sd_cl = &fl->ifl_sds.ifsd_cl[i];

	- if (d->ifsd_flags & RX_SW_DESC_INUSE) {
	- bus_dmamap_unload(fl->ifl_desc_tag, d->ifsd_map);
	- bus_dmamap_destroy(fl->ifl_desc_tag, d->ifsd_map);
	- if (d->ifsd_m != NULL) {
	- m_init(d->ifsd_m, M_NOWAIT, MT_DATA, 0);
	- uma_zfree(zone_mbuf, d->ifsd_m);
	+ if (*sd_flags & RX_SW_DESC_INUSE) {
	+ if (fl->ifl_sds.ifsd_map != NULL) {
	+ bus_dmamap_t sd_map = fl->ifl_sds.ifsd_map[i];
	+ bus_dmamap_unload(fl->ifl_desc_tag, sd_map);
	+ if (fl->ifl_rxq->ifr_ctx->ifc_in_detach)
	+ bus_dmamap_destroy(fl->ifl_desc_tag, sd_map);
	}
	- if (d->ifsd_cl != NULL)
	- uma_zfree(fl->ifl_zone, d->ifsd_cl);
	- d->ifsd_flags = 0;
	+ if (*sd_m != NULL) {
	+ m_init(*sd_m, M_NOWAIT, MT_DATA, 0);
	+ uma_zfree(zone_mbuf, *sd_m);
	+ }
	+ if (*sd_cl != NULL)
	+ uma_zfree(fl->ifl_zone, *sd_cl);
	+ *sd_flags = 0;
	} else {
	- MPASS(d->ifsd_cl == NULL);
	- MPASS(d->ifsd_m == NULL);
	+ MPASS(*sd_cl == NULL);
	+ MPASS(*sd_m == NULL);
	}
	#if MEMORY_LOGGING
	fl->ifl_m_dequeued++;
	fl->ifl_cl_dequeued++;
	#endif
	- d->ifsd_cl = NULL;
	- d->ifsd_m = NULL;
	+ *sd_cl = NULL;
	+ *sd_m = NULL;
	}
	+#ifdef INVARIANTS
	+ for (i = 0; i < fl->ifl_size; i++) {
	+ MPASS(fl->ifl_sds.ifsd_flags[i] == 0);
	+ MPASS(fl->ifl_sds.ifsd_cl[i] == NULL);
	+ MPASS(fl->ifl_sds.ifsd_m[i] == NULL);
	+ }
	+#endif
	/*
	* Reset free list values
	*/
	- fl->ifl_credits = fl->ifl_cidx = fl->ifl_pidx = fl->ifl_gen = 0;;
	+ fl->ifl_credits = fl->ifl_cidx = fl->ifl_pidx = fl->ifl_gen = fl->ifl_fragidx = 0;
	bzero(idi->idi_vaddr, idi->idi_size);
	}

	@@ -1711,6 +2011,7 @@
	if_ctx_t ctx = rxq->ifr_ctx;
	if_softc_ctx_t sctx = &ctx->ifc_softc_ctx;

	+ bit_nclear(fl->ifl_rx_bitmap, 0, fl->ifl_size - 1);
	/*
	** Free current RX buffer structs and their mbufs
	*/
	@@ -1723,12 +2024,17 @@
	*/
	if (sctx->isc_max_frame_size <= 2048)
	fl->ifl_buf_size = MCLBYTES;
	+#ifndef CONTIGMALLOC_WORKS
	+ else
	+ fl->ifl_buf_size = MJUMPAGESIZE;
	+#else
	else if (sctx->isc_max_frame_size <= 4096)
	fl->ifl_buf_size = MJUMPAGESIZE;
	else if (sctx->isc_max_frame_size <= 9216)
	fl->ifl_buf_size = MJUM9BYTES;
	else
	fl->ifl_buf_size = MJUM16BYTES;
	+#endif
	if (fl->ifl_buf_size > ctx->ifc_max_fl_buf_size)
	ctx->ifc_max_fl_buf_size = fl->ifl_buf_size;
	fl->ifl_cltype = m_gettype(fl->ifl_buf_size);
	@@ -1770,10 +2076,14 @@
	bus_dma_tag_destroy(fl->ifl_desc_tag);
	fl->ifl_desc_tag = NULL;
	}
	+ free(fl->ifl_sds.ifsd_m, M_IFLIB);
	+ free(fl->ifl_sds.ifsd_cl, M_IFLIB);
	+ /* XXX destroy maps first */
	+ free(fl->ifl_sds.ifsd_map, M_IFLIB);
	+ fl->ifl_sds.ifsd_m = NULL;
	+ fl->ifl_sds.ifsd_cl = NULL;
	+ fl->ifl_sds.ifsd_map = NULL;
	}
	- if (rxq->ifr_fl->ifl_sds != NULL)
	- free(rxq->ifr_fl->ifl_sds, M_IFLIB);
	-
	free(rxq->ifr_fl, M_IFLIB);
	rxq->ifr_fl = NULL;
	rxq->ifr_cq_gen = rxq->ifr_cq_cidx = rxq->ifr_cq_pidx = 0;
	@@ -1789,7 +2099,7 @@
	{
	iflib_txq_t txq = arg;
	if_ctx_t ctx = txq->ift_ctx;
	- if_softc_ctx_t scctx = &ctx->ifc_softc_ctx;
	+ if_softc_ctx_t sctx = &ctx->ifc_softc_ctx;

	if (!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING))
	return;
	@@ -1800,28 +2110,32 @@
	*/
	IFDI_TIMER(ctx, txq->ift_id);
	if ((txq->ift_qstatus == IFLIB_QUEUE_HUNG) &&
	- (ctx->ifc_pause_frames == 0))
	+ ((txq->ift_cleaned_prev == txq->ift_cleaned) \|\|
	+ (sctx->isc_pause_frames == 0)))
	goto hung;

	- if (TXQ_AVAIL(txq) <= 2*scctx->isc_tx_nsegments \|\|
	- ifmp_ring_is_stalled(txq->ift_br[0]))
	+ if (ifmp_ring_is_stalled(txq->ift_br))
	+ txq->ift_qstatus = IFLIB_QUEUE_HUNG;
	+ txq->ift_cleaned_prev = txq->ift_cleaned;
	+ /* handle any laggards */
	+ if (txq->ift_db_pending)
	GROUPTASK_ENQUEUE(&txq->ift_task);

	- ctx->ifc_pause_frames = 0;
	+ sctx->isc_pause_frames = 0;
	if (if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING)
	callout_reset_on(&txq->ift_timer, hz/2, iflib_timer, txq, txq->ift_timer.c_cpu);
	return;
	hung:
	CTX_LOCK(ctx);
	- if_setdrvflagbits(ctx->ifc_ifp, 0, IFF_DRV_RUNNING);
	+ if_setdrvflagbits(ctx->ifc_ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING);
	device_printf(ctx->ifc_dev, "TX(%d) desc avail = %d, pidx = %d\n",
	txq->ift_id, TXQ_AVAIL(txq), txq->ift_pidx);

	IFDI_WATCHDOG_RESET(ctx);
	ctx->ifc_watchdog_events++;
	- ctx->ifc_pause_frames = 0;

	- iflib_init_locked(ctx);
	+ ctx->ifc_flags \|= IFC_DO_RESET;
	+ iflib_admin_intr_deferred(ctx);
	CTX_UNLOCK(ctx);
	}

	@@ -1829,22 +2143,25 @@
	iflib_init_locked(if_ctx_t ctx)
	{
	if_softc_ctx_t sctx = &ctx->ifc_softc_ctx;
	+ if_softc_ctx_t scctx = &ctx->ifc_softc_ctx;
	if_t ifp = ctx->ifc_ifp;
	iflib_fl_t fl;
	iflib_txq_t txq;
	iflib_rxq_t rxq;
	- int i, j;
	+ int i, j, tx_ip_csum_flags, tx_ip6_csum_flags;


	if_setdrvflagbits(ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING);
	IFDI_INTR_DISABLE(ctx);

	+ tx_ip_csum_flags = scctx->isc_tx_csum_flags & (CSUM_IP \| CSUM_TCP \| CSUM_UDP \| CSUM_SCTP);
	+ tx_ip6_csum_flags = scctx->isc_tx_csum_flags & (CSUM_IP6_TCP \| CSUM_IP6_UDP \| CSUM_IP6_SCTP);
	/* Set hardware offload abilities */
	if_clearhwassist(ifp);
	if (if_getcapenable(ifp) & IFCAP_TXCSUM)
	- if_sethwassistbits(ifp, CSUM_IP \| CSUM_TCP \| CSUM_UDP, 0);
	+ if_sethwassistbits(ifp, tx_ip_csum_flags, 0);
	if (if_getcapenable(ifp) & IFCAP_TXCSUM_IPV6)
	- if_sethwassistbits(ifp, (CSUM_TCP_IPV6 \| CSUM_UDP_IPV6), 0);
	+ if_sethwassistbits(ifp, tx_ip6_csum_flags, 0);
	if (if_getcapenable(ifp) & IFCAP_TSO4)
	if_sethwassistbits(ifp, CSUM_IP_TSO, 0);
	if (if_getcapenable(ifp) & IFCAP_TSO6)
	@@ -1853,19 +2170,21 @@
	for (i = 0, txq = ctx->ifc_txqs; i < sctx->isc_ntxqsets; i++, txq++) {
	CALLOUT_LOCK(txq);
	callout_stop(&txq->ift_timer);
	- callout_stop(&txq->ift_db_check);
	CALLOUT_UNLOCK(txq);
	iflib_netmap_txq_init(ctx, txq);
	}
	- for (i = 0, rxq = ctx->ifc_rxqs; i < sctx->isc_nrxqsets; i++, rxq++) {
	- iflib_netmap_rxq_init(ctx, rxq);
	- }
	#ifdef INVARIANTS
	i = if_getdrvflags(ifp);
	#endif
	IFDI_INIT(ctx);
	MPASS(if_getdrvflags(ifp) == i);
	for (i = 0, rxq = ctx->ifc_rxqs; i < sctx->isc_nrxqsets; i++, rxq++) {
	+ /* XXX this should really be done on a per-queue basis */
	+ if (if_getcapenable(ifp) & IFCAP_NETMAP) {
	+ MPASS(rxq->ifr_id == i);
	+ iflib_netmap_rxq_init(ctx, rxq);
	+ continue;
	+ }
	for (j = 0, fl = rxq->ifr_fl; j < rxq->ifr_nfl; j++, fl++) {
	if (iflib_fl_setup(fl)) {
	device_printf(ctx->ifc_dev, "freelist setup failed - check cluster settings\n");
	@@ -1920,102 +2239,159 @@
	if_setdrvflagbits(ctx->ifc_ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING);

	IFDI_INTR_DISABLE(ctx);
	- msleep(ctx, &ctx->ifc_mtx, PUSER, "iflib_init", hz);
	+ DELAY(1000);
	+ IFDI_STOP(ctx);
	+ DELAY(1000);

	+ iflib_debug_reset();
	/* Wait for current tx queue users to exit to disarm watchdog timer. */
	for (i = 0; i < scctx->isc_ntxqsets; i++, txq++) {
	/* make sure all transmitters have completed before proceeding XXX */

	+ CALLOUT_LOCK(txq);
	+ callout_stop(&txq->ift_timer);
	+ CALLOUT_UNLOCK(txq);
	+
	/* clean any enqueued buffers */
	- iflib_txq_check_drain(txq, 0);
	+ iflib_ifmp_purge(txq);
	/* Free any existing tx buffers. */
	for (j = 0; j < txq->ift_size; j++) {
	iflib_txsd_free(ctx, txq, j);
	}
	txq->ift_processed = txq->ift_cleaned = txq->ift_cidx_processed = 0;
	- txq->ift_in_use = txq->ift_cidx = txq->ift_pidx = txq->ift_no_desc_avail = 0;
	+ txq->ift_in_use = txq->ift_gen = txq->ift_cidx = txq->ift_pidx = txq->ift_no_desc_avail = 0;
	txq->ift_closed = txq->ift_mbuf_defrag = txq->ift_mbuf_defrag_failed = 0;
	txq->ift_no_tx_dma_setup = txq->ift_txd_encap_efbig = txq->ift_map_failed = 0;
	txq->ift_pullups = 0;
	- ifmp_ring_reset_stats(txq->ift_br[0]);
	+ ifmp_ring_reset_stats(txq->ift_br);
	for (j = 0, di = txq->ift_ifdi; j < ctx->ifc_nhwtxqs; j++, di++)
	bzero((void *)di->idi_vaddr, di->idi_size);
	}
	for (i = 0; i < scctx->isc_nrxqsets; i++, rxq++) {
	/* make sure all transmitters have completed before proceeding XXX */

	- for (j = 0, di = txq->ift_ifdi; j < ctx->ifc_nhwrxqs; j++, di++)
	+ for (j = 0, di = rxq->ifr_ifdi; j < rxq->ifr_nfl; j++, di++)
	bzero((void *)di->idi_vaddr, di->idi_size);
	/* also resets the free lists pidx/cidx */
	for (j = 0, fl = rxq->ifr_fl; j < rxq->ifr_nfl; j++, fl++)
	iflib_fl_bufs_free(fl);
	}
	- IFDI_STOP(ctx);
	}

	-static iflib_rxsd_t
	-rxd_frag_to_sd(iflib_rxq_t rxq, if_rxd_frag_t irf, int *cltype, int unload)
	+static inline caddr_t
	+calc_next_rxd(iflib_fl_t fl, int cidx)
	{
	+ qidx_t size;
	+ int nrxd;
	+ caddr_t start, end, cur, next;
	+
	+ nrxd = fl->ifl_size;
	+ size = fl->ifl_rxd_size;
	+ start = fl->ifl_ifdi->idi_vaddr;
	+
	+ if (__predict_false(size == 0))
	+ return (start);
	+ cur = start + size*cidx;
	+ end = start + size*nrxd;
	+ next = CACHE_PTR_NEXT(cur);
	+ return (next < end ? next : start);
	+}
	+
	+static inline void
	+prefetch_pkts(iflib_fl_t fl, int cidx)
	+{
	+ int nextptr;
	+ int nrxd = fl->ifl_size;
	+ caddr_t next_rxd;
	+
	+
	+ nextptr = (cidx + CACHE_PTR_INCREMENT) & (nrxd-1);
	+ prefetch(&fl->ifl_sds.ifsd_m[nextptr]);
	+ prefetch(&fl->ifl_sds.ifsd_cl[nextptr]);
	+ next_rxd = calc_next_rxd(fl, cidx);
	+ prefetch(next_rxd);
	+ prefetch(fl->ifl_sds.ifsd_m[(cidx + 1) & (nrxd-1)]);
	+ prefetch(fl->ifl_sds.ifsd_m[(cidx + 2) & (nrxd-1)]);
	+ prefetch(fl->ifl_sds.ifsd_m[(cidx + 3) & (nrxd-1)]);
	+ prefetch(fl->ifl_sds.ifsd_m[(cidx + 4) & (nrxd-1)]);
	+ prefetch(fl->ifl_sds.ifsd_cl[(cidx + 1) & (nrxd-1)]);
	+ prefetch(fl->ifl_sds.ifsd_cl[(cidx + 2) & (nrxd-1)]);
	+ prefetch(fl->ifl_sds.ifsd_cl[(cidx + 3) & (nrxd-1)]);
	+ prefetch(fl->ifl_sds.ifsd_cl[(cidx + 4) & (nrxd-1)]);
	+}
	+
	+static void
	+rxd_frag_to_sd(iflib_rxq_t rxq, if_rxd_frag_t irf, int unload, if_rxsd_t sd)
	+{
	int flid, cidx;
	- iflib_rxsd_t sd;
	+ bus_dmamap_t map;
	iflib_fl_t fl;
	iflib_dma_info_t di;
	+ int next;

	+ map = NULL;
	flid = irf->irf_flid;
	cidx = irf->irf_idx;
	fl = &rxq->ifr_fl[flid];
	+ sd->ifsd_fl = fl;
	+ sd->ifsd_cidx = cidx;
	+ sd->ifsd_m = &fl->ifl_sds.ifsd_m[cidx];
	+ sd->ifsd_cl = &fl->ifl_sds.ifsd_cl[cidx];
	fl->ifl_credits--;
	#if MEMORY_LOGGING
	fl->ifl_m_dequeued++;
	- if (cltype)
	- fl->ifl_cl_dequeued++;
	#endif
	- sd = &fl->ifl_sds[cidx];
	- di = fl->ifl_ifdi;
	- bus_dmamap_sync(di->idi_tag, di->idi_map,
	- BUS_DMASYNC_POSTREAD \| BUS_DMASYNC_POSTWRITE);
	+ if (rxq->ifr_ctx->ifc_flags & IFC_PREFETCH)
	+ prefetch_pkts(fl, cidx);
	+ if (fl->ifl_sds.ifsd_map != NULL) {
	+ next = (cidx + CACHE_PTR_INCREMENT) & (fl->ifl_size-1);
	+ prefetch(&fl->ifl_sds.ifsd_map[next]);
	+ map = fl->ifl_sds.ifsd_map[cidx];
	+ di = fl->ifl_ifdi;
	+ next = (cidx + CACHE_LINE_SIZE) & (fl->ifl_size-1);
	+ prefetch(&fl->ifl_sds.ifsd_flags[next]);
	+ bus_dmamap_sync(di->idi_tag, di->idi_map,
	+ BUS_DMASYNC_POSTREAD \| BUS_DMASYNC_POSTWRITE);

	/* not valid assert if bxe really does SGE from non-contiguous elements */
	- MPASS(fl->ifl_cidx == cidx);
	- if (unload)
	- bus_dmamap_unload(fl->ifl_desc_tag, sd->ifsd_map);
	-
	- if (__predict_false(++fl->ifl_cidx == fl->ifl_size)) {
	- fl->ifl_cidx = 0;
	- fl->ifl_gen = 0;
	+ MPASS(fl->ifl_cidx == cidx);
	+ if (unload)
	+ bus_dmamap_unload(fl->ifl_desc_tag, map);
	}
	- /* YES ick */
	- if (cltype)
	- *cltype = fl->ifl_cltype;
	- return (sd);
	+ fl->ifl_cidx = (fl->ifl_cidx + 1) & (fl->ifl_size-1);
	+ if (__predict_false(fl->ifl_cidx == 0))
	+ fl->ifl_gen = 0;
	+ if (map != NULL)
	+ bus_dmamap_sync(fl->ifl_ifdi->idi_tag, fl->ifl_ifdi->idi_map,
	+ BUS_DMASYNC_PREREAD \| BUS_DMASYNC_PREWRITE);
	+ bit_clear(fl->ifl_rx_bitmap, cidx);
	}

	static struct mbuf *
	-assemble_segments(iflib_rxq_t rxq, if_rxd_info_t ri)
	+assemble_segments(iflib_rxq_t rxq, if_rxd_info_t ri, if_rxsd_t sd)
	{
	- int i, padlen , flags, cltype;
	+ int i, padlen , flags;
	struct mbuf m, mh, *mt;
	- iflib_rxsd_t sd;
	caddr_t cl;

	i = 0;
	mh = NULL;
	do {
	- sd = rxd_frag_to_sd(rxq, &ri->iri_frags[i], &cltype, TRUE);
	+ rxd_frag_to_sd(rxq, &ri->iri_frags[i], TRUE, sd);

	- MPASS(sd->ifsd_cl != NULL);
	- MPASS(sd->ifsd_m != NULL);
	+ MPASS(*sd->ifsd_cl != NULL);
	+ MPASS(*sd->ifsd_m != NULL);

	/* Don't include zero-length frags */
	if (ri->iri_frags[i].irf_len == 0) {
	/* XXX we can save the cluster here, but not the mbuf */
	- m_init(sd->ifsd_m, M_NOWAIT, MT_DATA, 0);
	- m_free(sd->ifsd_m);
	- sd->ifsd_m = NULL;
	+ m_init(*sd->ifsd_m, M_NOWAIT, MT_DATA, 0);
	+ m_free(*sd->ifsd_m);
	+ *sd->ifsd_m = NULL;
	continue;
	}
	-
	- m = sd->ifsd_m;
	+ m = *sd->ifsd_m;
	+ *sd->ifsd_m = NULL;
	if (mh == NULL) {
	flags = M_PKTHDR\|M_EXT;
	mh = mt = m;
	@@ -2027,13 +2403,12 @@
	/* assuming padding is only on the first fragment */
	padlen = 0;
	}
	- sd->ifsd_m = NULL;
	- cl = sd->ifsd_cl;
	- sd->ifsd_cl = NULL;
	+ cl = *sd->ifsd_cl;
	+ *sd->ifsd_cl = NULL;

	/* Can these two be made one ? */
	m_init(m, M_NOWAIT, MT_DATA, flags);
	- m_cljset(m, cl, cltype);
	+ m_cljset(m, cl, sd->ifsd_fl->ifl_cltype);
	/*
	* These must follow m_init and m_cljset
	*/
	@@ -2051,20 +2426,24 @@
	static struct mbuf *
	iflib_rxd_pkt_get(iflib_rxq_t rxq, if_rxd_info_t ri)
	{
	+ struct if_rxsd sd;
	struct mbuf *m;
	- iflib_rxsd_t sd;

	/* should I merge this back in now that the two paths are basically duplicated? */
	if (ri->iri_nfrags == 1 &&
	ri->iri_frags[0].irf_len <= MIN(IFLIB_RX_COPY_THRESH, MHLEN)) {
	- sd = rxd_frag_to_sd(rxq, &ri->iri_frags[0], NULL, FALSE);
	- m = sd->ifsd_m;
	- sd->ifsd_m = NULL;
	+ rxd_frag_to_sd(rxq, &ri->iri_frags[0], FALSE, &sd);
	+ m = *sd.ifsd_m;
	+ *sd.ifsd_m = NULL;
	m_init(m, M_NOWAIT, MT_DATA, M_PKTHDR);
	- memcpy(m->m_data, sd->ifsd_cl, ri->iri_len);
	+#ifndef __NO_STRICT_ALIGNMENT
	+ if (!IP_ALIGNED(m))
	+ m->m_data += 2;
	+#endif
	+ memcpy(m->m_data, *sd.ifsd_cl, ri->iri_len);
	m->m_len = ri->iri_frags[0].irf_len;
	} else {
	- m = assemble_segments(rxq, ri);
	+ m = assemble_segments(rxq, ri, &sd);
	}
	m->m_pkthdr.len = ri->iri_len;
	m->m_pkthdr.rcvif = ri->iri_ifp;
	@@ -2077,29 +2456,76 @@
	return (m);
	}

	+#if defined(INET6) \|\| defined(INET)
	+static void
	+iflib_get_ip_forwarding(struct lro_ctrl lc, bool v4, bool *v6)
	+{
	+ CURVNET_SET(lc->ifp->if_vnet);
	+#if defined(INET6)
	+ *v6 = VNET(ip6_forwarding);
	+#endif
	+#if defined(INET)
	+ *v4 = VNET(ipforwarding);
	+#endif
	+ CURVNET_RESTORE();
	+}
	+
	+/*
	+ * Returns true if it's possible this packet could be LROed.
	+ * if it returns false, it is guaranteed that tcp_lro_rx()
	+ * would not return zero.
	+ */
	static bool
	-iflib_rxeof(iflib_rxq_t rxq, int budget)
	+iflib_check_lro_possible(struct mbuf *m, bool v4_forwarding, bool v6_forwarding)
	{
	+ struct ether_header *eh;
	+ uint16_t eh_type;
	+
	+ eh = mtod(m, struct ether_header *);
	+ eh_type = ntohs(eh->ether_type);
	+ switch (eh_type) {
	+#if defined(INET6)
	+ case ETHERTYPE_IPV6:
	+ return !v6_forwarding;
	+#endif
	+#if defined (INET)
	+ case ETHERTYPE_IP:
	+ return !v4_forwarding;
	+#endif
	+ }
	+
	+ return false;
	+}
	+#else
	+static void
	+iflib_get_ip_forwarding(struct lro_ctrl lc __unused, bool v4 __unused, bool *v6 __unused)
	+{
	+}
	+#endif
	+
	+static bool
	+iflib_rxeof(iflib_rxq_t rxq, qidx_t budget)
	+{
	if_ctx_t ctx = rxq->ifr_ctx;
	if_shared_ctx_t sctx = ctx->ifc_sctx;
	if_softc_ctx_t scctx = &ctx->ifc_softc_ctx;
	int avail, i;
	- uint16_t *cidxp;
	+ qidx_t *cidxp;
	struct if_rxd_info ri;
	int err, budget_left, rx_bytes, rx_pkts;
	iflib_fl_t fl;
	struct ifnet *ifp;
	int lro_enabled;
	+ bool lro_possible = false;
	+ bool v4_forwarding, v6_forwarding;
	+
	/*
	* XXX early demux data packets so that if_input processing only handles
	* acks in interrupt context
	*/
	- struct mbuf m, mh, *mt;
	+ struct mbuf m, mh, mt, mf;

	- if (netmap_rx_irq(ctx->ifc_ifp, rxq->ifr_id, &budget)) {
	- return (FALSE);
	- }
	-
	+ ifp = ctx->ifc_ifp;
	mh = mt = NULL;
	MPASS(budget > 0);
	rx_pkts = rx_bytes = 0;
	@@ -2122,18 +2548,19 @@
	/*
	* Reset client set fields to their default values
	*/
	- bzero(&ri, sizeof(ri));
	+ rxd_info_zero(&ri);
	ri.iri_qsidx = rxq->ifr_id;
	ri.iri_cidx = *cidxp;
	- ri.iri_ifp = ctx->ifc_ifp;
	+ ri.iri_ifp = ifp;
	ri.iri_frags = rxq->ifr_frags;
	err = ctx->isc_rxd_pkt_get(ctx->ifc_softc, &ri);

	- /* in lieu of handling correctly - make sure it isn't being unhandled */
	- MPASS(err == 0);
	+ if (err)
	+ goto err;
	if (sctx->isc_flags & IFLIB_HAS_RXCQ) {
	*cidxp = ri.iri_cidx;
	/* Update our consumer index */
	+ /* XXX NB: shurd - check if this is still safe */
	while (rxq->ifr_cq_cidx >= scctx->isc_nrxd[0]) {
	rxq->ifr_cq_cidx -= scctx->isc_nrxd[0];
	rxq->ifr_cq_gen = 0;
	@@ -2166,20 +2593,52 @@
	for (i = 0, fl = &rxq->ifr_fl[0]; i < sctx->isc_nfl; i++, fl++)
	__iflib_fl_refill_lt(ctx, fl, budget + 8);

	- ifp = ctx->ifc_ifp;
	lro_enabled = (if_getcapenable(ifp) & IFCAP_LRO);
	+ if (lro_enabled)
	+ iflib_get_ip_forwarding(&rxq->ifr_lc, &v4_forwarding, &v6_forwarding);
	+ mt = mf = NULL;
	while (mh != NULL) {
	m = mh;
	mh = mh->m_nextpkt;
	m->m_nextpkt = NULL;
	+#ifndef __NO_STRICT_ALIGNMENT
	+ if (!IP_ALIGNED(m) && (m = iflib_fixup_rx(m)) == NULL)
	+ continue;
	+#endif
	rx_bytes += m->m_pkthdr.len;
	rx_pkts++;
	#if defined(INET6) \|\| defined(INET)
	- if (lro_enabled && tcp_lro_rx(&rxq->ifr_lc, m, 0) == 0)
	- continue;
	+ if (lro_enabled) {
	+ if (!lro_possible) {
	+ lro_possible = iflib_check_lro_possible(m, v4_forwarding, v6_forwarding);
	+ if (lro_possible && mf != NULL) {
	+ ifp->if_input(ifp, mf);
	+ DBG_COUNTER_INC(rx_if_input);
	+ mt = mf = NULL;
	+ }
	+ }
	+ if ((m->m_pkthdr.csum_flags & (CSUM_L4_CALC\|CSUM_L4_VALID)) ==
	+ (CSUM_L4_CALC\|CSUM_L4_VALID)) {
	+ if (lro_possible && tcp_lro_rx(&rxq->ifr_lc, m, 0) == 0)
	+ continue;
	+ }
	+ }
	#endif
	+ if (lro_possible) {
	+ ifp->if_input(ifp, m);
	+ DBG_COUNTER_INC(rx_if_input);
	+ continue;
	+ }
	+
	+ if (mf == NULL)
	+ mf = m;
	+ if (mt != NULL)
	+ mt->m_nextpkt = m;
	+ mt = m;
	+ }
	+ if (mf != NULL) {
	+ ifp->if_input(ifp, mf);
	DBG_COUNTER_INC(rx_if_input);
	- ifp->if_input(ifp, m);
	}

	if_inc_counter(ifp, IFCOUNTER_IBYTES, rx_bytes);
	@@ -2194,45 +2653,78 @@
	if (avail)
	return true;
	return (iflib_rxd_avail(ctx, rxq, *cidxp, 1));
	+err:
	+ CTX_LOCK(ctx);
	+ ctx->ifc_flags \|= IFC_DO_RESET;
	+ iflib_admin_intr_deferred(ctx);
	+ CTX_UNLOCK(ctx);
	+ return (false);
	}

	+#define TXD_NOTIFY_COUNT(txq) (((txq)->ift_size / (txq)->ift_update_freq)-1)
	+static inline qidx_t
	+txq_max_db_deferred(iflib_txq_t txq, qidx_t in_use)
	+{
	+ qidx_t notify_count = TXD_NOTIFY_COUNT(txq);
	+ qidx_t minthresh = txq->ift_size / 8;
	+ if (in_use > 4*minthresh)
	+ return (notify_count);
	+ if (in_use > 2*minthresh)
	+ return (notify_count >> 1);
	+ if (in_use > minthresh)
	+ return (notify_count >> 3);
	+ return (0);
	+}
	+
	+static inline qidx_t
	+txq_max_rs_deferred(iflib_txq_t txq)
	+{
	+ qidx_t notify_count = TXD_NOTIFY_COUNT(txq);
	+ qidx_t minthresh = txq->ift_size / 8;
	+ if (txq->ift_in_use > 4*minthresh)
	+ return (notify_count);
	+ if (txq->ift_in_use > 2*minthresh)
	+ return (notify_count >> 1);
	+ if (txq->ift_in_use > minthresh)
	+ return (notify_count >> 2);
	+ return (2);
	+}
	+
	#define M_CSUM_FLAGS(m) ((m)->m_pkthdr.csum_flags)
	#define M_HAS_VLANTAG(m) (m->m_flags & M_VLANTAG)
	-#define TXQ_MAX_DB_DEFERRED(size) (size >> 5)
	+
	+#define TXQ_MAX_DB_DEFERRED(txq, in_use) txq_max_db_deferred((txq), (in_use))
	+#define TXQ_MAX_RS_DEFERRED(txq) txq_max_rs_deferred(txq)
	#define TXQ_MAX_DB_CONSUMED(size) (size >> 4)

	-static __inline void
	-iflib_txd_db_check(if_ctx_t ctx, iflib_txq_t txq, int ring)
	-{
	- uint32_t dbval;
	+/* forward compatibility for cxgb */
	+#define FIRST_QSET(ctx) 0
	+#define NTXQSETS(ctx) ((ctx)->ifc_softc_ctx.isc_ntxqsets)
	+#define NRXQSETS(ctx) ((ctx)->ifc_softc_ctx.isc_nrxqsets)
	+#define QIDX(ctx, m) ((((m)->m_pkthdr.flowid & ctx->ifc_softc_ctx.isc_rss_table_mask) % NTXQSETS(ctx)) + FIRST_QSET(ctx))
	+#define DESC_RECLAIMABLE(q) ((int)((q)->ift_processed - (q)->ift_cleaned - (q)->ift_ctx->ifc_softc_ctx.isc_tx_nsegments))

	- if (ring \|\| txq->ift_db_pending >=
	- TXQ_MAX_DB_DEFERRED(txq->ift_size)) {
	+/* XXX we should be setting this to something other than zero */
	+#define RECLAIM_THRESH(ctx) ((ctx)->ifc_sctx->isc_tx_reclaim_thresh)
	+#define MAX_TX_DESC(ctx) ((ctx)->ifc_softc_ctx.isc_tx_tso_segments_max)

	- /* the lock will only ever be contended in the !min_latency case */
	- if (!TXDB_TRYLOCK(txq))
	- return;
	+static inline bool
	+iflib_txd_db_check(if_ctx_t ctx, iflib_txq_t txq, int ring, qidx_t in_use)
	+{
	+ qidx_t dbval, max;
	+ bool rang;
	+
	+ rang = false;
	+ max = TXQ_MAX_DB_DEFERRED(txq, in_use);
	+ if (ring \|\| txq->ift_db_pending >= max) {
	dbval = txq->ift_npending ? txq->ift_npending : txq->ift_pidx;
	ctx->isc_txd_flush(ctx->ifc_softc, txq->ift_id, dbval);
	txq->ift_db_pending = txq->ift_npending = 0;
	- TXDB_UNLOCK(txq);
	+ rang = true;
	}
	+ return (rang);
	}

	-static void
	-iflib_txd_deferred_db_check(void * arg)
	-{
	- iflib_txq_t txq = arg;
	-
	- /* simple non-zero boolean so use bitwise OR */
	- if ((txq->ift_db_pending \| txq->ift_npending) &&
	- txq->ift_db_pending >= txq->ift_db_pending_queued)
	- iflib_txd_db_check(txq->ift_ctx, txq, TRUE);
	- txq->ift_db_pending_queued = 0;
	- if (ifmp_ring_is_stalled(txq->ift_br[0]))
	- iflib_txq_check_drain(txq, 4);
	-}
	-
	#ifdef PKT_DEBUG
	static void
	print_pkt(if_pkt_info_t pi)
	@@ -2252,10 +2744,21 @@
	static int
	iflib_parse_header(iflib_txq_t txq, if_pkt_info_t pi, struct mbuf **mp)
	{
	+ if_shared_ctx_t sctx = txq->ift_ctx->ifc_sctx;
	struct ether_vlan_header *eh;
	struct mbuf m, n;

	n = m = *mp;
	+ if ((sctx->isc_flags & IFLIB_NEED_SCRATCH) &&
	+ M_WRITABLE(m) == 0) {
	+ if ((m = m_dup(m, M_NOWAIT)) == NULL) {
	+ return (ENOMEM);
	+ } else {
	+ m_freem(*mp);
	+ n = *mp = m;
	+ }
	+ }
	+
	/*
	* Determine where frame payload starts.
	* Jump over vlan headers if already present,
	@@ -2319,26 +2822,30 @@
	pi->ipi_ipproto = ip->ip_p;
	pi->ipi_flags \|= IPI_TX_IPV4;

	- if (pi->ipi_csum_flags & CSUM_IP)
	+ if ((sctx->isc_flags & IFLIB_NEED_ZERO_CSUM) && (pi->ipi_csum_flags & CSUM_IP))
	ip->ip_sum = 0;

	- if (pi->ipi_ipproto == IPPROTO_TCP) {
	- if (__predict_false(th == NULL)) {
	- txq->ift_pullups++;
	- if (__predict_false((m = m_pullup(m, (ip->ip_hl << 2) + sizeof(*th))) == NULL))
	- return (ENOMEM);
	- th = (struct tcphdr *)((caddr_t)ip + pi->ipi_ip_hlen);
	- }
	- pi->ipi_tcp_hflags = th->th_flags;
	- pi->ipi_tcp_hlen = th->th_off << 2;
	- pi->ipi_tcp_seq = th->th_seq;
	- }
	if (IS_TSO4(pi)) {
	+ if (pi->ipi_ipproto == IPPROTO_TCP) {
	+ if (__predict_false(th == NULL)) {
	+ txq->ift_pullups++;
	+ if (__predict_false((m = m_pullup(m, (ip->ip_hl << 2) + sizeof(*th))) == NULL))
	+ return (ENOMEM);
	+ th = (struct tcphdr *)((caddr_t)ip + pi->ipi_ip_hlen);
	+ }
	+ pi->ipi_tcp_hflags = th->th_flags;
	+ pi->ipi_tcp_hlen = th->th_off << 2;
	+ pi->ipi_tcp_seq = th->th_seq;
	+ }
	if (__predict_false(ip->ip_p != IPPROTO_TCP))
	return (ENXIO);
	th->th_sum = in_pseudo(ip->ip_src.s_addr,
	ip->ip_dst.s_addr, htons(IPPROTO_TCP));
	pi->ipi_tso_segsz = m->m_pkthdr.tso_segsz;
	+ if (sctx->isc_flags & IFLIB_TSO_INIT_IP) {
	+ ip->ip_sum = 0;
	+ ip->ip_len = htons(pi->ipi_ip_hlen + pi->ipi_tcp_hlen + pi->ipi_tso_segsz);
	+ }
	}
	break;
	}
	@@ -2360,15 +2867,15 @@
	pi->ipi_ipproto = ip6->ip6_nxt;
	pi->ipi_flags \|= IPI_TX_IPV6;

	- if (pi->ipi_ipproto == IPPROTO_TCP) {
	- if (__predict_false(m->m_len < pi->ipi_ehdrlen + sizeof(struct ip6_hdr) + sizeof(struct tcphdr))) {
	- if (__predict_false((m = m_pullup(m, pi->ipi_ehdrlen + sizeof(struct ip6_hdr) + sizeof(struct tcphdr))) == NULL))
	- return (ENOMEM);
	- }
	- pi->ipi_tcp_hflags = th->th_flags;
	- pi->ipi_tcp_hlen = th->th_off << 2;
	- }
	if (IS_TSO6(pi)) {
	+ if (pi->ipi_ipproto == IPPROTO_TCP) {
	+ if (__predict_false(m->m_len < pi->ipi_ehdrlen + sizeof(struct ip6_hdr) + sizeof(struct tcphdr))) {
	+ if (__predict_false((m = m_pullup(m, pi->ipi_ehdrlen + sizeof(struct ip6_hdr) + sizeof(struct tcphdr))) == NULL))
	+ return (ENOMEM);
	+ }
	+ pi->ipi_tcp_hflags = th->th_flags;
	+ pi->ipi_tcp_hlen = th->th_off << 2;
	+ }

	if (__predict_false(ip6->ip6_nxt != IPPROTO_TCP))
	return (ENXIO);
	@@ -2390,10 +2897,10 @@
	break;
	}
	*mp = m;
	+
	return (0);
	}

	-
	static __noinline struct mbuf *
	collapse_pkthdr(struct mbuf *m0)
	{
	@@ -2460,8 +2967,8 @@
	if_ctx_t ctx;
	if_shared_ctx_t sctx;
	if_softc_ctx_t scctx;
	- int i, next, pidx, mask, err, maxsegsz, ntxd, count;
	- struct mbuf m, tmp, ifsd_m, mp;
	+ int i, next, pidx, err, ntxd, count;
	+ struct mbuf m, tmp, **ifsd_m;

	m = *m0;

	@@ -2485,28 +2992,46 @@
	if (err)
	return (err);
	ifsd_flags[pidx] \|= TX_SW_DESC_MAPPED;
	- i = 0;
	- next = pidx;
	- mask = (txq->ift_size-1);
	+ count = 0;
	m = *m0;
	do {
	- mp = &ifsd_m[next];
	- *mp = m;
	+ if (__predict_false(m->m_len <= 0)) {
	+ tmp = m;
	+ m = m->m_next;
	+ tmp->m_next = NULL;
	+ m_free(tmp);
	+ continue;
	+ }
	m = m->m_next;
	- if (__predict_false((*mp)->m_len == 0)) {
	- m_free(*mp);
	- *mp = NULL;
	- } else
	- next = (pidx + i) & (ntxd-1);
	+ count++;
	} while (m != NULL);
	+ if (count > *nsegs) {
	+ ifsd_m[pidx] = *m0;
	+ ifsd_m[pidx]->m_flags \|= M_TOOBIG;
	+ return (0);
	+ }
	+ m = *m0;
	+ count = 0;
	+ do {
	+ next = (pidx + count) & (ntxd-1);
	+ MPASS(ifsd_m[next] == NULL);
	+ ifsd_m[next] = m;
	+ count++;
	+ tmp = m;
	+ m = m->m_next;
	+ } while (m != NULL);
	} else {
	- int buflen, sgsize, max_sgsize;
	+ int buflen, sgsize, maxsegsz, max_sgsize;
	vm_offset_t vaddr;
	vm_paddr_t curaddr;

	count = i = 0;
	- maxsegsz = sctx->isc_tx_maxsize;
	m = *m0;
	+ if (m->m_pkthdr.csum_flags & CSUM_TSO)
	+ maxsegsz = scctx->isc_tx_tso_segsize_max;
	+ else
	+ maxsegsz = sctx->isc_tx_maxsegsize;
	+
	do {
	if (__predict_false(m->m_len <= 0)) {
	tmp = m;
	@@ -2528,6 +3053,8 @@
	#endif
	ifsd_m[next] = m;
	while (buflen > 0) {
	+ if (i >= max_segs)
	+ goto err;
	max_sgsize = MIN(buflen, maxsegsz);
	curaddr = pmap_kextract(vaddr);
	sgsize = PAGE_SIZE - (curaddr & PAGE_MASK);
	@@ -2537,8 +3064,6 @@
	vaddr += sgsize;
	buflen -= sgsize;
	i++;
	- if (i >= max_segs)
	- goto err;
	}
	count++;
	tmp = m;
	@@ -2552,6 +3077,67 @@
	return (EFBIG);
	}

	+static inline caddr_t
	+calc_next_txd(iflib_txq_t txq, int cidx, uint8_t qid)
	+{
	+ qidx_t size;
	+ int ntxd;
	+ caddr_t start, end, cur, next;
	+
	+ ntxd = txq->ift_size;
	+ size = txq->ift_txd_size[qid];
	+ start = txq->ift_ifdi[qid].idi_vaddr;
	+
	+ if (__predict_false(size == 0))
	+ return (start);
	+ cur = start + size*cidx;
	+ end = start + size*ntxd;
	+ next = CACHE_PTR_NEXT(cur);
	+ return (next < end ? next : start);
	+}
	+
	+/*
	+ * Pad an mbuf to ensure a minimum ethernet frame size.
	+ * min_frame_size is the frame size (less CRC) to pad the mbuf to
	+ */
	+static __noinline int
	+iflib_ether_pad(device_t dev, struct mbuf **m_head, uint16_t min_frame_size)
	+{
	+ /*
	+ * 18 is enough bytes to pad an ARP packet to 46 bytes, and
	+ * and ARP message is the smallest common payload I can think of
	+ */
	+ static char pad[18]; /* just zeros */
	+ int n;
	+ struct mbuf *new_head;
	+
	+ if (!M_WRITABLE(*m_head)) {
	+ new_head = m_dup(*m_head, M_NOWAIT);
	+ if (new_head == NULL) {
	+ m_freem(*m_head);
	+ device_printf(dev, "cannot pad short frame, m_dup() failed");
	+ DBG_COUNTER_INC(encap_pad_mbuf_fail);
	+ return ENOMEM;
	+ }
	+ m_freem(*m_head);
	+ *m_head = new_head;
	+ }
	+
	+ for (n = min_frame_size - (*m_head)->m_pkthdr.len;
	+ n > 0; n -= sizeof(pad))
	+ if (!m_append(*m_head, min(n, sizeof(pad)), pad))
	+ break;
	+
	+ if (n > 0) {
	+ m_freem(*m_head);
	+ device_printf(dev, "cannot pad short frame\n");
	+ DBG_COUNTER_INC(encap_pad_mbuf_fail);
	+ return (ENOBUFS);
	+ }
	+
	+ return 0;
	+}
	+
	static int
	iflib_encap(iflib_txq_t txq, struct mbuf **m_headp)
	{
	@@ -2560,6 +3146,7 @@
	if_softc_ctx_t scctx;
	bus_dma_segment_t *segs;
	struct mbuf *m_head;
	+ void *next_txd;
	bus_dmamap_t map;
	struct if_pkt_info pi;
	int remap = 0;
	@@ -2580,18 +3167,23 @@
	*/
	cidx = txq->ift_cidx;
	pidx = txq->ift_pidx;
	- next = (cidx + CACHE_PTR_INCREMENT) & (ntxd-1);
	+ if (ctx->ifc_flags & IFC_PREFETCH) {
	+ next = (cidx + CACHE_PTR_INCREMENT) & (ntxd-1);
	+ if (!(ctx->ifc_flags & IFLIB_HAS_TXCQ)) {
	+ next_txd = calc_next_txd(txq, cidx, 0);
	+ prefetch(next_txd);
	+ }

	- /* prefetch the next cache line of mbuf pointers and flags */
	- prefetch(&txq->ift_sds.ifsd_m[next]);
	- if (txq->ift_sds.ifsd_map != NULL) {
	- prefetch(&txq->ift_sds.ifsd_map[next]);
	+ /* prefetch the next cache line of mbuf pointers and flags */
	+ prefetch(&txq->ift_sds.ifsd_m[next]);
	+ if (txq->ift_sds.ifsd_map != NULL) {
	+ prefetch(&txq->ift_sds.ifsd_map[next]);
	+ next = (cidx + CACHE_LINE_SIZE) & (ntxd-1);
	+ prefetch(&txq->ift_sds.ifsd_flags[next]);
	+ }
	+ } else if (txq->ift_sds.ifsd_map != NULL)
	map = txq->ift_sds.ifsd_map[pidx];
	- next = (cidx + CACHE_LINE_SIZE) & (ntxd-1);
	- prefetch(&txq->ift_sds.ifsd_flags[next]);
	- }

	-
	if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
	desc_tag = txq->ift_tso_desc_tag;
	max_segs = scctx->isc_tx_tso_segments_max;
	@@ -2599,14 +3191,21 @@
	desc_tag = txq->ift_desc_tag;
	max_segs = scctx->isc_tx_nsegments;
	}
	+ if ((sctx->isc_flags & IFLIB_NEED_ETHER_PAD) &&
	+ __predict_false(m_head->m_pkthdr.len < scctx->isc_min_frame_size)) {
	+ err = iflib_ether_pad(ctx->ifc_dev, m_headp, scctx->isc_min_frame_size);
	+ if (err)
	+ return err;
	+ }
	m_head = *m_headp;
	- bzero(&pi, sizeof(pi));
	- pi.ipi_len = m_head->m_pkthdr.len;
	+
	+ pkt_info_zero(&pi);
	pi.ipi_mflags = (m_head->m_flags & (M_VLANTAG\|M_BCAST\|M_MCAST));
	- pi.ipi_csum_flags = m_head->m_pkthdr.csum_flags;
	- pi.ipi_vtag = (m_head->m_flags & M_VLANTAG) ? m_head->m_pkthdr.ether_vtag : 0;
	pi.ipi_pidx = pidx;
	pi.ipi_qsidx = txq->ift_id;
	+ pi.ipi_len = m_head->m_pkthdr.len;
	+ pi.ipi_csum_flags = m_head->m_pkthdr.csum_flags;
	+ pi.ipi_vtag = (m_head->m_flags & M_VLANTAG) ? m_head->m_pkthdr.ether_vtag : 0;

	/* deliberate bitwise OR to make one condition */
	if (__predict_true((pi.ipi_csum_flags \| pi.ipi_vtag))) {
	@@ -2662,6 +3261,19 @@
	GROUPTASK_ENQUEUE(&txq->ift_task);
	return (ENOBUFS);
	}
	+ /*
	+ * On Intel cards we can greatly reduce the number of TX interrupts
	+ * we see by only setting report status on every Nth descriptor.
	+ * However, this also means that the driver will need to keep track
	+ * of the descriptors that RS was set on to check them for the DD bit.
	+ */
	+ txq->ift_rs_pending += nsegs + 1;
	+ if (txq->ift_rs_pending > TXQ_MAX_RS_DEFERRED(txq) \|\|
	+ iflib_no_tx_batch \|\| (TXQ_AVAIL(txq) - nsegs - 1) <= MAX_TX_DESC(ctx)) {
	+ pi.ipi_flags \|= IPI_TX_INTR;
	+ txq->ift_rs_pending = 0;
	+ }
	+
	pi.ipi_segs = segs;
	pi.ipi_nsegs = nsegs;

	@@ -2669,22 +3281,29 @@
	#ifdef PKT_DEBUG
	print_pkt(&pi);
	#endif
	+ if (map != NULL)
	+ bus_dmamap_sync(desc_tag, map, BUS_DMASYNC_PREWRITE);
	if ((err = ctx->isc_txd_encap(ctx->ifc_softc, &pi)) == 0) {
	- bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_ifdi->idi_map,
	- BUS_DMASYNC_PREREAD \| BUS_DMASYNC_PREWRITE);
	-
	+ if (map != NULL)
	+ bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_ifdi->idi_map,
	+ BUS_DMASYNC_PREREAD \| BUS_DMASYNC_PREWRITE);
	DBG_COUNTER_INC(tx_encap);
	- MPASS(pi.ipi_new_pidx >= 0 &&
	- pi.ipi_new_pidx < txq->ift_size);
	+ MPASS(pi.ipi_new_pidx < txq->ift_size);

	ndesc = pi.ipi_new_pidx - pi.ipi_pidx;
	if (pi.ipi_new_pidx < pi.ipi_pidx) {
	ndesc += txq->ift_size;
	txq->ift_gen = 1;
	}
	+ /*
	+ * drivers can need as many as
	+ * two sentinels
	+ */
	+ MPASS(ndesc <= pi.ipi_nsegs + 2);
	MPASS(pi.ipi_new_pidx != pidx);
	MPASS(ndesc > 0);
	txq->ift_in_use += ndesc;
	+
	/*
	* We update the last software descriptor again here because there may
	* be a sentinel and/or there may be more mbufs than segments
	@@ -2709,36 +3328,6 @@
	return (ENOMEM);
	}

	-/* forward compatibility for cxgb */
	-#define FIRST_QSET(ctx) 0
	-
	-#define NTXQSETS(ctx) ((ctx)->ifc_softc_ctx.isc_ntxqsets)
	-#define NRXQSETS(ctx) ((ctx)->ifc_softc_ctx.isc_nrxqsets)
	-#define QIDX(ctx, m) ((((m)->m_pkthdr.flowid & ctx->ifc_softc_ctx.isc_rss_table_mask) % NTXQSETS(ctx)) + FIRST_QSET(ctx))
	-#define DESC_RECLAIMABLE(q) ((int)((q)->ift_processed - (q)->ift_cleaned - (q)->ift_ctx->ifc_softc_ctx.isc_tx_nsegments))
	-#define RECLAIM_THRESH(ctx) ((ctx)->ifc_sctx->isc_tx_reclaim_thresh)
	-#define MAX_TX_DESC(ctx) ((ctx)->ifc_softc_ctx.isc_tx_tso_segments_max)
	-
	-
	-
	-/* if there are more than TXQ_MIN_OCCUPANCY packets pending we consider deferring
	- * doorbell writes
	- *
	- * ORing with 2 assures that min occupancy is never less than 2 without any conditional logic
	- */
	-#define TXQ_MIN_OCCUPANCY(size) ((size >> 6)\| 0x2)
	-
	-static inline int
	-iflib_txq_min_occupancy(iflib_txq_t txq)
	-{
	- if_ctx_t ctx;
	-
	- ctx = txq->ift_ctx;
	- return (get_inuse(txq->ift_size, txq->ift_cidx, txq->ift_pidx,
	- txq->ift_gen) < TXQ_MIN_OCCUPANCY(txq->ift_size) +
	- MAX_TX_DESC(ctx));
	-}
	-
	static void
	iflib_tx_desc_free(iflib_txq_t txq, int n)
	{
	@@ -2747,6 +3336,7 @@
	struct mbuf m, *ifsd_m;
	uint8_t *ifsd_flags;
	bus_dmamap_t *ifsd_map;
	+ bool do_prefetch;

	cidx = txq->ift_cidx;
	gen = txq->ift_gen;
	@@ -2756,11 +3346,13 @@
	ifsd_flags = txq->ift_sds.ifsd_flags;
	ifsd_m = txq->ift_sds.ifsd_m;
	ifsd_map = txq->ift_sds.ifsd_map;
	+ do_prefetch = (txq->ift_ctx->ifc_flags & IFC_PREFETCH);

	while (n--) {
	- prefetch(ifsd_m[(cidx + 3) & mask]);
	- prefetch(ifsd_m[(cidx + 4) & mask]);
	-
	+ if (do_prefetch) {
	+ prefetch(ifsd_m[(cidx + 3) & mask]);
	+ prefetch(ifsd_m[(cidx + 4) & mask]);
	+ }
	if (ifsd_m[cidx] != NULL) {
	prefetch(&ifsd_m[(cidx + CACHE_PTR_INCREMENT) & mask]);
	prefetch(&ifsd_flags[(cidx + CACHE_PTR_INCREMENT) & mask]);
	@@ -2775,8 +3367,15 @@
	if ((m = ifsd_m[cidx]) != NULL) {
	/* XXX we don't support any drivers that batch packets yet */
	MPASS(m->m_nextpkt == NULL);
	-
	- m_free(m);
	+ /* if the number of clusters exceeds the number of segments
	+ * there won't be space on the ring to save a pointer to each
	+ * cluster so we simply free the list here
	+ */
	+ if (m->m_flags & M_TOOBIG) {
	+ m_freem(m);
	+ } else {
	+ m_free(m);
	+ }
	ifsd_m[cidx] = NULL;
	#if MEMORY_LOGGING
	txq->ift_dequeued++;
	@@ -2823,24 +3422,34 @@
	txq->ift_cleaned += reclaim;
	txq->ift_in_use -= reclaim;

	- if (txq->ift_active == FALSE)
	- txq->ift_active = TRUE;
	-
	return (reclaim);
	}

	static struct mbuf **
	-_ring_peek_one(struct ifmp_ring *r, int cidx, int offset)
	+_ring_peek_one(struct ifmp_ring *r, int cidx, int offset, int remaining)
	{
	+ int next, size;
	+ struct mbuf **items;

	- return (__DEVOLATILE(struct mbuf **, &r->items[(cidx + offset) & (r->size-1)]));
	+ size = r->size;
	+ next = (cidx + CACHE_PTR_INCREMENT) & (size-1);
	+ items = __DEVOLATILE(struct mbuf **, &r->items[0]);
	+
	+ prefetch(items[(cidx + offset) & (size-1)]);
	+ if (remaining > 1) {
	+ prefetch2cachelines(&items[next]);
	+ prefetch2cachelines(items[(cidx + offset + 1) & (size-1)]);
	+ prefetch2cachelines(items[(cidx + offset + 2) & (size-1)]);
	+ prefetch2cachelines(items[(cidx + offset + 3) & (size-1)]);
	+ }
	+ return (__DEVOLATILE(struct mbuf **, &r->items[(cidx + offset) & (size-1)]));
	}

	static void
	iflib_txq_check_drain(iflib_txq_t txq, int budget)
	{

	- ifmp_ring_check_drainage(txq->ift_br[0], budget);
	+ ifmp_ring_check_drainage(txq->ift_br, budget);
	}

	static uint32_t
	@@ -2849,8 +3458,8 @@
	iflib_txq_t txq = r->cookie;
	if_ctx_t ctx = txq->ift_ctx;

	- return ((TXQ_AVAIL(txq) >= MAX_TX_DESC(ctx)) \|\|
	- ctx->isc_txd_credits_update(ctx->ifc_softc, txq->ift_id, txq->ift_cidx_processed, false));
	+ return ((TXQ_AVAIL(txq) > MAX_TX_DESC(ctx) + 2) \|\|
	+ ctx->isc_txd_credits_update(ctx->ifc_softc, txq->ift_id, false));
	}

	static uint32_t
	@@ -2858,16 +3467,19 @@
	{
	iflib_txq_t txq = r->cookie;
	if_ctx_t ctx = txq->ift_ctx;
	- if_t ifp = ctx->ifc_ifp;
	+ struct ifnet *ifp = ctx->ifc_ifp;
	struct mbuf *mp, m;
	- int i, count, consumed, pkt_sent, bytes_sent, mcast_sent, avail, err, in_use_prev, desc_used;
	+ int i, count, consumed, pkt_sent, bytes_sent, mcast_sent, avail;
	+ int reclaimed, err, in_use_prev, desc_used;
	+ bool do_prefetch, ring, rang;

	if (__predict_false(!(if_getdrvflags(ifp) & IFF_DRV_RUNNING) \|\|
	!LINK_ACTIVE(ctx))) {
	DBG_COUNTER_INC(txq_drain_notready);
	return (0);
	}
	-
	+ reclaimed = iflib_completed_tx_reclaim(txq, RECLAIM_THRESH(ctx));
	+ rang = iflib_txd_db_check(ctx, txq, reclaimed, txq->ift_in_use);
	avail = IDXDIFF(pidx, cidx, r->size);
	if (__predict_false(ctx->ifc_flags & IFC_QFLUSH)) {
	DBG_COUNTER_INC(txq_drain_flushing);
	@@ -2877,77 +3489,153 @@
	}
	return (avail);
	}
	- iflib_completed_tx_reclaim(txq, RECLAIM_THRESH(ctx));
	+
	if (__predict_false(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_OACTIVE)) {
	txq->ift_qstatus = IFLIB_QUEUE_IDLE;
	CALLOUT_LOCK(txq);
	callout_stop(&txq->ift_timer);
	- callout_stop(&txq->ift_db_check);
	CALLOUT_UNLOCK(txq);
	DBG_COUNTER_INC(txq_drain_oactive);
	return (0);
	}
	+ if (reclaimed)
	+ txq->ift_qstatus = IFLIB_QUEUE_IDLE;
	consumed = mcast_sent = bytes_sent = pkt_sent = 0;
	count = MIN(avail, TX_BATCH_SIZE);
	+#ifdef INVARIANTS
	+ if (iflib_verbose_debug)
	+ printf("%s avail=%d ifc_flags=%x txq_avail=%d ", __FUNCTION__,
	+ avail, ctx->ifc_flags, TXQ_AVAIL(txq));
	+#endif
	+ do_prefetch = (ctx->ifc_flags & IFC_PREFETCH);
	+ avail = TXQ_AVAIL(txq);
	+ for (desc_used = i = 0; i < count && avail > MAX_TX_DESC(ctx) + 2; i++) {
	+ int pidx_prev, rem = do_prefetch ? count - i : 0;

	- for (desc_used = i = 0; i < count && TXQ_AVAIL(txq) > MAX_TX_DESC(ctx) + 2; i++) {
	- mp = _ring_peek_one(r, cidx, i);
	+ mp = _ring_peek_one(r, cidx, i, rem);
	+ MPASS(mp != NULL && *mp != NULL);
	+ if (__predict_false(mp == (struct mbuf )txq)) {
	+ consumed++;
	+ reclaimed++;
	+ continue;
	+ }
	in_use_prev = txq->ift_in_use;
	+ pidx_prev = txq->ift_pidx;
	err = iflib_encap(txq, mp);
	- /*
	- * What other errors should we bail out for?
	- */
	- if (err == ENOBUFS) {
	+ if (__predict_false(err)) {
	DBG_COUNTER_INC(txq_drain_encapfail);
	- break;
	+ /* no room - bail out */
	+ if (err == ENOBUFS)
	+ break;
	+ consumed++;
	+ DBG_COUNTER_INC(txq_drain_encapfail);
	+ /* we can't send this packet - skip it */
	+ continue;
	}
	consumed++;
	- if (err)
	- continue;
	-
	pkt_sent++;
	m = *mp;
	DBG_COUNTER_INC(tx_sent);
	bytes_sent += m->m_pkthdr.len;
	- if (m->m_flags & M_MCAST)
	- mcast_sent++;
	+ mcast_sent += !!(m->m_flags & M_MCAST);
	+ avail = TXQ_AVAIL(txq);

	txq->ift_db_pending += (txq->ift_in_use - in_use_prev);
	desc_used += (txq->ift_in_use - in_use_prev);
	- iflib_txd_db_check(ctx, txq, FALSE);
	ETHER_BPF_MTAP(ifp, m);
	- if (__predict_false(!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING)))
	+ if (__predict_false(!(ifp->if_drv_flags & IFF_DRV_RUNNING)))
	break;
	-
	- if (desc_used > TXQ_MAX_DB_CONSUMED(txq->ift_size))
	- break;
	+ rang = iflib_txd_db_check(ctx, txq, false, in_use_prev);
	}

	- if ((iflib_min_tx_latency \|\| iflib_txq_min_occupancy(txq)) && txq->ift_db_pending)
	- iflib_txd_db_check(ctx, txq, TRUE);
	- else if ((txq->ift_db_pending \|\| TXQ_AVAIL(txq) < MAX_TX_DESC(ctx)) &&
	- (callout_pending(&txq->ift_db_check) == 0)) {
	- txq->ift_db_pending_queued = txq->ift_db_pending;
	- callout_reset_on(&txq->ift_db_check, 1, iflib_txd_deferred_db_check,
	- txq, txq->ift_db_check.c_cpu);
	- }
	+ /* deliberate use of bitwise or to avoid gratuitous short-circuit */
	+ ring = rang ? false : (iflib_min_tx_latency \| err) \|\| (TXQ_AVAIL(txq) < MAX_TX_DESC(ctx));
	+ iflib_txd_db_check(ctx, txq, ring, txq->ift_in_use);
	if_inc_counter(ifp, IFCOUNTER_OBYTES, bytes_sent);
	if_inc_counter(ifp, IFCOUNTER_OPACKETS, pkt_sent);
	if (mcast_sent)
	if_inc_counter(ifp, IFCOUNTER_OMCASTS, mcast_sent);
	-
	+#ifdef INVARIANTS
	+ if (iflib_verbose_debug)
	+ printf("consumed=%d\n", consumed);
	+#endif
	return (consumed);
	}

	+static uint32_t
	+iflib_txq_drain_always(struct ifmp_ring *r)
	+{
	+ return (1);
	+}
	+
	+static uint32_t
	+iflib_txq_drain_free(struct ifmp_ring *r, uint32_t cidx, uint32_t pidx)
	+{
	+ int i, avail;
	+ struct mbuf **mp;
	+ iflib_txq_t txq;
	+
	+ txq = r->cookie;
	+
	+ txq->ift_qstatus = IFLIB_QUEUE_IDLE;
	+ CALLOUT_LOCK(txq);
	+ callout_stop(&txq->ift_timer);
	+ CALLOUT_UNLOCK(txq);
	+
	+ avail = IDXDIFF(pidx, cidx, r->size);
	+ for (i = 0; i < avail; i++) {
	+ mp = _ring_peek_one(r, cidx, i, avail - i);
	+ if (__predict_false(mp == (struct mbuf )txq))
	+ continue;
	+ m_freem(*mp);
	+ }
	+ MPASS(ifmp_ring_is_stalled(r) == 0);
	+ return (avail);
	+}
	+
	static void
	+iflib_ifmp_purge(iflib_txq_t txq)
	+{
	+ struct ifmp_ring *r;
	+
	+ r = txq->ift_br;
	+ r->drain = iflib_txq_drain_free;
	+ r->can_drain = iflib_txq_drain_always;
	+
	+ ifmp_ring_check_drainage(r, r->size);
	+
	+ r->drain = iflib_txq_drain;
	+ r->can_drain = iflib_txq_can_drain;
	+}
	+
	+static void
	_task_fn_tx(void *context)
	{
	iflib_txq_t txq = context;
	if_ctx_t ctx = txq->ift_ctx;
	+ struct ifnet *ifp = ctx->ifc_ifp;
	+ int rc;

	+#ifdef IFLIB_DIAGNOSTICS
	+ txq->ift_cpu_exec_count[curcpu]++;
	+#endif
	if (!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING))
	return;
	- ifmp_ring_check_drainage(txq->ift_br[0], TX_BATCH_SIZE);
	+ if (if_getcapenable(ifp) & IFCAP_NETMAP) {
	+ if (ctx->isc_txd_credits_update(ctx->ifc_softc, txq->ift_id, false))
	+ netmap_tx_irq(ifp, txq->ift_id);
	+ IFDI_TX_QUEUE_INTR_ENABLE(ctx, txq->ift_id);
	+ return;
	+ }
	+ if (txq->ift_db_pending)
	+ ifmp_ring_enqueue(txq->ift_br, (void **)&txq, 1, TX_BATCH_SIZE);
	+ ifmp_ring_check_drainage(txq->ift_br, TX_BATCH_SIZE);
	+ if (ctx->ifc_flags & IFC_LEGACY)
	+ IFDI_INTR_ENABLE(ctx);
	+ else {
	+ rc = IFDI_TX_QUEUE_INTR_ENABLE(ctx, txq->ift_id);
	+ KASSERT(rc != ENOTSUP, ("MSI-X support requires queue_intr_enable, but not implemented in driver"));
	+ }
	}

	static void
	@@ -2957,17 +3645,32 @@
	if_ctx_t ctx = rxq->ifr_ctx;
	bool more;
	int rc;
	+ uint16_t budget;

	+#ifdef IFLIB_DIAGNOSTICS
	+ rxq->ifr_cpu_exec_count[curcpu]++;
	+#endif
	DBG_COUNTER_INC(task_fn_rxs);
	if (__predict_false(!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING)))
	return;
	-
	- if ((more = iflib_rxeof(rxq, 16 /* XXX */)) == false) {
	+ more = true;
	+#ifdef DEV_NETMAP
	+ if (if_getcapenable(ctx->ifc_ifp) & IFCAP_NETMAP) {
	+ u_int work = 0;
	+ if (netmap_rx_irq(ctx->ifc_ifp, rxq->ifr_id, &work)) {
	+ more = false;
	+ }
	+ }
	+#endif
	+ budget = ctx->ifc_sysctl_rx_budget;
	+ if (budget == 0)
	+ budget = 16; /* XXX */
	+ if (more == false \|\| (more = iflib_rxeof(rxq, budget)) == false) {
	if (ctx->ifc_flags & IFC_LEGACY)
	IFDI_INTR_ENABLE(ctx);
	else {
	DBG_COUNTER_INC(rx_intr_enables);
	- rc = IFDI_QUEUE_INTR_ENABLE(ctx, rxq->ifr_id);
	+ rc = IFDI_RX_QUEUE_INTR_ENABLE(ctx, rxq->ifr_id);
	KASSERT(rc != ENOTSUP, ("MSI-X support requires queue_intr_enable, but not implemented in driver"));
	}
	}
	@@ -2985,8 +3688,11 @@
	iflib_txq_t txq;
	int i;

	- if (!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING))
	- return;
	+ if (!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING)) {
	+ if (!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_OACTIVE)) {
	+ return;
	+ }
	+ }

	CTX_LOCK(ctx);
	for (txq = ctx->ifc_txqs, i = 0; i < sctx->isc_ntxqsets; i++, txq++) {
	@@ -2998,6 +3704,10 @@
	for (txq = ctx->ifc_txqs, i = 0; i < sctx->isc_ntxqsets; i++, txq++)
	callout_reset_on(&txq->ift_timer, hz/2, iflib_timer, txq, txq->ift_timer.c_cpu);
	IFDI_LINK_INTR_ENABLE(ctx);
	+ if (ctx->ifc_flags & IFC_DO_RESET) {
	+ ctx->ifc_flags &= ~IFC_DO_RESET;
	+ iflib_if_init_locked(ctx);
	+ }
	CTX_UNLOCK(ctx);

	if (LINK_ACTIVE(ctx) == 0)
	@@ -3072,7 +3782,7 @@
	if (__predict_false((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0 \|\| !LINK_ACTIVE(ctx))) {
	DBG_COUNTER_INC(tx_frees);
	m_freem(m);
	- return (0);
	+ return (ENOBUFS);
	}

	MPASS(m->m_nextpkt == NULL);
	@@ -3119,18 +3829,16 @@
	}
	#endif
	DBG_COUNTER_INC(tx_seen);
	- err = ifmp_ring_enqueue(txq->ift_br[0], (void **)&m, 1, TX_BATCH_SIZE);
	+ err = ifmp_ring_enqueue(txq->ift_br, (void **)&m, 1, TX_BATCH_SIZE);

	+ GROUPTASK_ENQUEUE(&txq->ift_task);
	if (err) {
	- GROUPTASK_ENQUEUE(&txq->ift_task);
	/* support forthcoming later */
	#ifdef DRIVER_BACKPRESSURE
	txq->ift_closed = TRUE;
	#endif
	- ifmp_ring_check_drainage(txq->ift_br[0], TX_BATCH_SIZE);
	+ ifmp_ring_check_drainage(txq->ift_br, TX_BATCH_SIZE);
	m_freem(m);
	- } else if (TXQ_AVAIL(txq) < (txq->ift_size >> 1)) {
	- GROUPTASK_ENQUEUE(&txq->ift_task);
	}

	return (err);
	@@ -3147,7 +3855,7 @@
	ctx->ifc_flags \|= IFC_QFLUSH;
	CTX_UNLOCK(ctx);
	for (i = 0; i < NTXQSETS(ctx); i++, txq++)
	- while (!(ifmp_ring_is_idle(txq->ift_br[0]) \|\| ifmp_ring_is_stalled(txq->ift_br[0])))
	+ while (!(ifmp_ring_is_idle(txq->ift_br) \|\| ifmp_ring_is_stalled(txq->ift_br)))
	iflib_txq_check_drain(txq, 0);
	CTX_LOCK(ctx);
	ctx->ifc_flags &= ~IFC_QFLUSH;
	@@ -3158,11 +3866,9 @@


	#define IFCAP_FLAGS (IFCAP_TXCSUM_IPV6 \| IFCAP_RXCSUM_IPV6 \| IFCAP_HWCSUM \| IFCAP_LRO \| \
	- IFCAP_TSO4 \| IFCAP_TSO6 \| IFCAP_VLAN_HWTAGGING \| \
	+ IFCAP_TSO4 \| IFCAP_TSO6 \| IFCAP_VLAN_HWTAGGING \| IFCAP_HWSTATS \| \
	IFCAP_VLAN_MTU \| IFCAP_VLAN_HWFILTER \| IFCAP_VLAN_HWTSO)

	-#define IFCAP_REINIT IFCAP_FLAGS
	-
	static int
	iflib_if_ioctl(if_t ifp, u_long command, caddr_t data)
	{
	@@ -3236,8 +3942,6 @@
	ctx->ifc_if_flags = if_getflags(ifp);
	CTX_UNLOCK(ctx);
	break;
	-
	- break;
	case SIOCADDMULTI:
	case SIOCDELMULTI:
	if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) {
	@@ -3254,6 +3958,7 @@
	CTX_UNLOCK(ctx);
	/* falls thru */
	case SIOCGIFMEDIA:
	+ case SIOCGIFXMEDIA:
	err = ifmedia_ioctl(ifp, ifr, &ctx->ifc_media, command);
	break;
	case SIOCGI2C:
	@@ -3288,6 +3993,8 @@
	#endif
	setmask \|= (mask & IFCAP_FLAGS);

	+ if (setmask & (IFCAP_RXCSUM \| IFCAP_RXCSUM_IPV6))
	+ setmask \|= (IFCAP_RXCSUM \| IFCAP_RXCSUM_IPV6);
	if ((mask & IFCAP_WOL) &&
	(if_getcapabilities(ifp) & IFCAP_WOL) != 0)
	setmask \|= (mask & (IFCAP_WOL_MCAST\|IFCAP_WOL_MAGIC));
	@@ -3298,10 +4005,10 @@
	if (setmask) {
	CTX_LOCK(ctx);
	bits = if_getdrvflags(ifp);
	- if (setmask & IFCAP_REINIT)
	+ if (bits & IFF_DRV_RUNNING)
	iflib_stop(ctx);
	if_togglecapenable(ifp, setmask);
	- if (setmask & IFCAP_REINIT)
	+ if (bits & IFF_DRV_RUNNING)
	iflib_init_locked(ctx);
	if_setdrvflags(ifp, bits);
	CTX_UNLOCK(ctx);
	@@ -3353,7 +4060,7 @@
	IFDI_VLAN_REGISTER(ctx, vtag);
	/* Re-init to load the changes */
	if (if_getcapenable(ifp) & IFCAP_VLAN_HWFILTER)
	- iflib_init_locked(ctx);
	+ iflib_if_init_locked(ctx);
	CTX_UNLOCK(ctx);
	}

	@@ -3372,7 +4079,7 @@
	IFDI_VLAN_UNREGISTER(ctx, vtag);
	/* Re-init to load the changes */
	if (if_getcapenable(ifp) & IFCAP_VLAN_HWFILTER)
	- iflib_init_locked(ctx);
	+ iflib_if_init_locked(ctx);
	CTX_UNLOCK(ctx);
	}

	@@ -3462,7 +4169,6 @@

	ctx->ifc_sctx = sctx;
	ctx->ifc_dev = dev;
	- ctx->ifc_txrx = *sctx->isc_txrx;
	ctx->ifc_softc = sc;

	if ((err = iflib_register(ctx)) != 0) {
	@@ -3472,6 +4178,9 @@
	iflib_add_device_sysctl_pre(ctx);

	scctx = &ctx->ifc_softc_ctx;
	+ ifp = ctx->ifc_ifp;
	+ ctx->ifc_nhwtxqs = sctx->isc_ntxqs;
	+
	/*
	* XXX sanity check that ntxd & nrxd are a power of 2
	*/
	@@ -3524,30 +4233,35 @@
	device_printf(dev, "IFDI_ATTACH_PRE failed %d\n", err);
	return (err);
	}
	- if (scctx->isc_ntxqsets_max)
	- scctx->isc_ntxqsets = min(scctx->isc_ntxqsets, scctx->isc_ntxqsets_max);
	- if (scctx->isc_nrxqsets_max)
	- scctx->isc_nrxqsets = min(scctx->isc_nrxqsets, scctx->isc_nrxqsets_max);
	+ _iflib_pre_assert(scctx);
	+ ctx->ifc_txrx = *scctx->isc_txrx;

	+#ifdef INVARIANTS
	+ MPASS(scctx->isc_capenable);
	+ if (scctx->isc_capenable & IFCAP_TXCSUM)
	+ MPASS(scctx->isc_tx_csum_flags);
	+#endif
	+
	+ if_setcapabilities(ifp, scctx->isc_capenable \| IFCAP_HWSTATS);
	+ if_setcapenable(ifp, scctx->isc_capenable \| IFCAP_HWSTATS);
	+
	+ if (scctx->isc_ntxqsets == 0 \|\| (scctx->isc_ntxqsets_max && scctx->isc_ntxqsets_max < scctx->isc_ntxqsets))
	+ scctx->isc_ntxqsets = scctx->isc_ntxqsets_max;
	+ if (scctx->isc_nrxqsets == 0 \|\| (scctx->isc_nrxqsets_max && scctx->isc_nrxqsets_max < scctx->isc_nrxqsets))
	+ scctx->isc_nrxqsets = scctx->isc_nrxqsets_max;
	+
	#ifdef ACPI_DMAR
	if (dmar_get_dma_tag(device_get_parent(dev), dev) != NULL)
	ctx->ifc_flags \|= IFC_DMAR;
	+#elif !(defined(__i386__) \|\| defined(__amd64__))
	+ /* set unconditionally for !x86 */
	+ ctx->ifc_flags \|= IFC_DMAR;
	#endif

	msix_bar = scctx->isc_msix_bar;
	+ main_txq = (sctx->isc_flags & IFLIB_HAS_TXCQ) ? 1 : 0;
	+ main_rxq = (sctx->isc_flags & IFLIB_HAS_RXCQ) ? 1 : 0;

	- ifp = ctx->ifc_ifp;
	-
	- if(sctx->isc_flags & IFLIB_HAS_TXCQ)
	- main_txq = 1;
	- else
	- main_txq = 0;
	-
	- if(sctx->isc_flags & IFLIB_HAS_RXCQ)
	- main_rxq = 1;
	- else
	- main_rxq = 0;
	-
	/* XXX change for per-queue sizes */
	device_printf(dev, "using %d tx descriptors and %d rx descriptors\n",
	scctx->isc_ntxd[main_txq], scctx->isc_nrxd[main_rxq]);
	@@ -3590,6 +4304,18 @@
	if (scctx->isc_rss_table_size == 0)
	scctx->isc_rss_table_size = 64;
	scctx->isc_rss_table_mask = scctx->isc_rss_table_size-1;
	+
	+ GROUPTASK_INIT(&ctx->ifc_admin_task, 0, _task_fn_admin, ctx);
	+ /* XXX format name */
	+ taskqgroup_attach(qgroup_if_config_tqg, &ctx->ifc_admin_task, ctx, -1, "admin");
	+
	+ /* Set up cpu set. If it fails, use the set of all CPUs. */
	+ if (bus_get_cpus(dev, INTR_CPUS, sizeof(ctx->ifc_cpus), &ctx->ifc_cpus) != 0) {
	+ device_printf(dev, "Unable to fetch CPU list\n");
	+ CPU_COPY(&all_cpus, &ctx->ifc_cpus);
	+ }
	+ MPASS(CPU_COUNT(&ctx->ifc_cpus) > 0);
	+
	/*
	** Now setup MSI or MSI/X, should
	** return us the number of supported
	@@ -3598,6 +4324,10 @@
	if (sctx->isc_flags & IFLIB_SKIP_MSIX) {
	msix = scctx->isc_vectors;
	} else if (scctx->isc_msix_bar != 0)
	+ /*
	+ * The simple fact that isc_msix_bar is not 0 does not mean we
	+ * we have a good value there that is known to work.
	+ */
	msix = iflib_msix_init(ctx);
	else {
	scctx->isc_vectors = 1;
	@@ -3616,7 +4346,17 @@
	device_printf(dev, "qset structure setup failed %d\n", err);
	goto fail_queues;
	}
	-
	+ /*
	+ * Group taskqueues aren't properly set up until SMP is started,
	+ * so we disable interrupts until we can handle them post
	+ * SI_SUB_SMP.
	+ *
	+ * XXX: disabling interrupts doesn't actually work, at least for
	+ * the non-MSI case. When they occur before SI_SUB_SMP completes,
	+ * we do null handling and depend on this not causing too large an
	+ * interrupt storm.
	+ */
	+ IFDI_INTR_DISABLE(ctx);
	if (msix > 1 && (err = IFDI_MSIX_INTR_ASSIGN(ctx, msix)) != 0) {
	device_printf(dev, "IFDI_MSIX_INTR_ASSIGN failed %d\n", err);
	goto fail_intr_free;
	@@ -3679,8 +4419,9 @@
	iflib_txq_t txq;
	iflib_rxq_t rxq;
	device_t dev = ctx->ifc_dev;
	- int i;
	+ int i, j;
	struct taskqgroup *tqg;
	+ iflib_fl_t fl;

	/* Make sure VLANS are not using driver */
	if (if_vlantrunkinuse(ifp)) {
	@@ -3706,16 +4447,19 @@
	if (ctx->ifc_led_dev != NULL)
	led_destroy(ctx->ifc_led_dev);
	/* XXX drain any dependent tasks */
	- tqg = qgroup_softirq;
	+ tqg = qgroup_if_io_tqg;
	for (txq = ctx->ifc_txqs, i = 0; i < NTXQSETS(ctx); i++, txq++) {
	callout_drain(&txq->ift_timer);
	- callout_drain(&txq->ift_db_check);
	if (txq->ift_task.gt_uniq != NULL)
	taskqgroup_detach(tqg, &txq->ift_task);
	}
	for (i = 0, rxq = ctx->ifc_rxqs; i < NRXQSETS(ctx); i++, rxq++) {
	if (rxq->ifr_task.gt_uniq != NULL)
	taskqgroup_detach(tqg, &rxq->ifr_task);
	+
	+ for (j = 0, fl = rxq->ifr_fl; j < rxq->ifr_nfl; j++, fl++)
	+ free(fl->ifl_rx_bitmap, M_IFLIB);
	+
	}
	tqg = qgroup_if_config_tqg;
	if (ctx->ifc_admin_task.gt_uniq != NULL)
	@@ -3886,15 +4630,6 @@
	MPASS(sctx->isc_rx_nsegments);
	MPASS(sctx->isc_rx_maxsegsize);

	-
	- MPASS(sctx->isc_txrx->ift_txd_encap);
	- MPASS(sctx->isc_txrx->ift_txd_flush);
	- MPASS(sctx->isc_txrx->ift_txd_credits_update);
	- MPASS(sctx->isc_txrx->ift_rxd_available);
	- MPASS(sctx->isc_txrx->ift_rxd_pkt_get);
	- MPASS(sctx->isc_txrx->ift_rxd_refill);
	- MPASS(sctx->isc_txrx->ift_rxd_flush);
	-
	MPASS(sctx->isc_nrxd_min[0]);
	MPASS(sctx->isc_nrxd_max[0]);
	MPASS(sctx->isc_nrxd_default[0]);
	@@ -3903,6 +4638,19 @@
	MPASS(sctx->isc_ntxd_default[0]);
	}

	+static void
	+_iflib_pre_assert(if_softc_ctx_t scctx)
	+{
	+
	+ MPASS(scctx->isc_txrx->ift_txd_encap);
	+ MPASS(scctx->isc_txrx->ift_txd_flush);
	+ MPASS(scctx->isc_txrx->ift_txd_credits_update);
	+ MPASS(scctx->isc_txrx->ift_rxd_available);
	+ MPASS(scctx->isc_txrx->ift_rxd_pkt_get);
	+ MPASS(scctx->isc_txrx->ift_rxd_refill);
	+ MPASS(scctx->isc_txrx->ift_rxd_flush);
	+}
	+
	static int
	iflib_register(if_ctx_t ctx)
	{
	@@ -3937,9 +4685,6 @@
	if_setqflushfn(ifp, iflib_if_qflush);
	if_setflags(ifp, IFF_BROADCAST \| IFF_SIMPLEX \| IFF_MULTICAST);

	- if_setcapabilities(ifp, 0);
	- if_setcapenable(ifp, 0);
	-
	ctx->ifc_vlan_attach_event =
	EVENTHANDLER_REGISTER(vlan_config, iflib_vlan_register, ctx,
	EVENTHANDLER_PRI_FIRST);
	@@ -3975,7 +4720,6 @@
	caddr_t *vaddrs;
	uint64_t *paddrs;
	struct ifmp_ring **brscp;
	- int nbuf_rings = 1; /* XXX determine dynamically */

	KASSERT(ntxqs > 0, ("number of queues per qset must be at least 1"));
	KASSERT(nrxqs > 0, ("number of queues per qset must be at least 1"));
	@@ -4001,11 +4745,6 @@
	err = ENOMEM;
	goto rx_fail;
	}
	- if (!(brscp = malloc(sizeof(void ) nbuf_rings * nrxqsets, M_IFLIB, M_NOWAIT \| M_ZERO))) {
	- device_printf(dev, "Unable to buf_ring_sc * memory\n");
	- err = ENOMEM;
	- goto rx_fail;
	- }

	ctx->ifc_txqs = txq;
	ctx->ifc_rxqs = rxq;
	@@ -4028,6 +4767,7 @@
	err = ENOMEM;
	goto err_tx_desc;
	}
	+ txq->ift_txd_size[j] = scctx->isc_txd_size[j];
	bzero((void *)ifdip->idi_vaddr, txqsizes[j]);
	}
	txq->ift_ctx = ctx;
	@@ -4039,8 +4779,6 @@
	}
	/* XXX fix this */
	txq->ift_timer.c_cpu = cpu;
	- txq->ift_db_check.c_cpu = cpu;
	- txq->ift_nbr = nbuf_rings;

	if (iflib_txsd_alloc(txq)) {
	device_printf(dev, "Critical Failure setting up TX buffers\n");
	@@ -4053,21 +4791,16 @@
	device_get_nameunit(dev), txq->ift_id);
	mtx_init(&txq->ift_mtx, txq->ift_mtx_name, NULL, MTX_DEF);
	callout_init_mtx(&txq->ift_timer, &txq->ift_mtx, 0);
	- callout_init_mtx(&txq->ift_db_check, &txq->ift_mtx, 0);

	snprintf(txq->ift_db_mtx_name, MTX_NAME_LEN, "%s:tx(%d):db",
	device_get_nameunit(dev), txq->ift_id);
	- TXDB_LOCK_INIT(txq);

	- txq->ift_br = brscp + i*nbuf_rings;
	- for (j = 0; j < nbuf_rings; j++) {
	- err = ifmp_ring_alloc(&txq->ift_br[j], 2048, txq, iflib_txq_drain,
	- iflib_txq_can_drain, M_IFLIB, M_WAITOK);
	- if (err) {
	- /* XXX free any allocated rings */
	- device_printf(dev, "Unable to allocate buf_ring\n");
	- goto err_tx_desc;
	- }
	+ err = ifmp_ring_alloc(&txq->ift_br, 2048, txq, iflib_txq_drain,
	+ iflib_txq_can_drain, M_IFLIB, M_WAITOK);
	+ if (err) {
	+ /* XXX free any allocated rings */
	+ device_printf(dev, "Unable to allocate buf_ring\n");
	+ goto err_tx_desc;
	}
	}

	@@ -4081,6 +4814,9 @@
	}

	rxq->ifr_ifdi = ifdip;
	+ /* XXX this needs to be changed if #rx queues != #tx queues */
	+ rxq->ifr_ntxqirq = 1;
	+ rxq->ifr_txqid[0] = i;
	for (j = 0; j < nrxqs; j++, ifdip++) {
	if (iflib_dma_alloc(ctx, rxqsizes[j], ifdip, BUS_DMA_NOWAIT)) {
	device_printf(dev, "Unable to allocate Descriptor memory\n");
	@@ -4105,10 +4841,10 @@
	}
	rxq->ifr_fl = fl;
	for (j = 0; j < nfree_lists; j++) {
	- rxq->ifr_fl[j].ifl_rxq = rxq;
	- rxq->ifr_fl[j].ifl_id = j;
	- rxq->ifr_fl[j].ifl_ifdi =
	- &rxq->ifr_ifdi[j + rxq->ifr_fl_offset];
	+ fl[j].ifl_rxq = rxq;
	+ fl[j].ifl_id = j;
	+ fl[j].ifl_ifdi = &rxq->ifr_ifdi[j + rxq->ifr_fl_offset];
	+ fl[j].ifl_rxd_size = scctx->isc_rxd_size[j];
	}
	/* Allocate receive buffers for the ring*/
	if (iflib_rxsd_alloc(rxq)) {
	@@ -4117,6 +4853,9 @@
	err = ENOMEM;
	goto err_rx_desc;
	}
	+
	+ for (j = 0, fl = rxq->ifr_fl; j < rxq->ifr_nfl; j++, fl++)
	+ fl->ifl_rx_bitmap = bit_alloc(fl->ifl_size, M_IFLIB, M_WAITOK\|M_ZERO);
	}

	/* TXQs */
	@@ -4294,19 +5033,152 @@
	return (_iflib_irq_alloc(ctx, irq, rid, filter, handler, arg, name));
	}

	-static void
	-find_nth(if_ctx_t ctx, cpuset_t *cpus, int qid)
	+#ifdef SMP
	+static int
	+find_nth(if_ctx_t ctx, int qid)
	{
	- int i, cpuid;
	+ cpuset_t cpus;
	+ int i, cpuid, eqid, count;

	- CPU_COPY(&ctx->ifc_cpus, cpus);
	+ CPU_COPY(&ctx->ifc_cpus, &cpus);
	+ count = CPU_COUNT(&cpus);
	+ eqid = qid % count;
	/* clear up to the qid'th bit */
	- for (i = 0; i < qid; i++) {
	- cpuid = CPU_FFS(cpus);
	- CPU_CLR(cpuid, cpus);
	+ for (i = 0; i < eqid; i++) {
	+ cpuid = CPU_FFS(&cpus);
	+ MPASS(cpuid != 0);
	+ CPU_CLR(cpuid-1, &cpus);
	}
	+ cpuid = CPU_FFS(&cpus);
	+ MPASS(cpuid != 0);
	+ return (cpuid-1);
	}

	+#ifdef SCHED_ULE
	+extern struct cpu_group cpu_top; / CPU topology */
	+
	+static int
	+find_child_with_core(int cpu, struct cpu_group *grp)
	+{
	+ int i;
	+
	+ if (grp->cg_children == 0)
	+ return -1;
	+
	+ MPASS(grp->cg_child);
	+ for (i = 0; i < grp->cg_children; i++) {
	+ if (CPU_ISSET(cpu, &grp->cg_child[i].cg_mask))
	+ return i;
	+ }
	+
	+ return -1;
	+}
	+
	+/*
	+ * Find the nth "close" core to the specified core
	+ * "close" is defined as the deepest level that shares
	+ * at least an L2 cache. With threads, this will be
	+ * threads on the same core. If the sahred cache is L3
	+ * or higher, simply returns the same core.
	+ */
	+static int
	+find_close_core(int cpu, int core_offset)
	+{
	+ struct cpu_group *grp;
	+ int i;
	+ int fcpu;
	+ cpuset_t cs;
	+
	+ grp = cpu_top;
	+ if (grp == NULL)
	+ return cpu;
	+ i = 0;
	+ while ((i = find_child_with_core(cpu, grp)) != -1) {
	+ /* If the child only has one cpu, don't descend */
	+ if (grp->cg_child[i].cg_count <= 1)
	+ break;
	+ grp = &grp->cg_child[i];
	+ }
	+
	+ /* If they don't share at least an L2 cache, use the same CPU */
	+ if (grp->cg_level > CG_SHARE_L2 \|\| grp->cg_level == CG_SHARE_NONE)
	+ return cpu;
	+
	+ /* Now pick one */
	+ CPU_COPY(&grp->cg_mask, &cs);
	+
	+ /* Add the selected CPU offset to core offset. */
	+ for (i = 0; (fcpu = CPU_FFS(&cs)) != 0; i++) {
	+ if (fcpu - 1 == cpu)
	+ break;
	+ CPU_CLR(fcpu - 1, &cs);
	+ }
	+ MPASS(fcpu);
	+
	+ core_offset += i;
	+
	+ CPU_COPY(&grp->cg_mask, &cs);
	+ for (i = core_offset % grp->cg_count; i > 0; i--) {
	+ MPASS(CPU_FFS(&cs));
	+ CPU_CLR(CPU_FFS(&cs) - 1, &cs);
	+ }
	+ MPASS(CPU_FFS(&cs));
	+ return CPU_FFS(&cs) - 1;
	+}
	+#else
	+static int
	+find_close_core(int cpu, int core_offset __unused)
	+{
	+ return cpu;
	+}
	+#endif
	+
	+static int
	+get_core_offset(if_ctx_t ctx, iflib_intr_type_t type, int qid)
	+{
	+ switch (type) {
	+ case IFLIB_INTR_TX:
	+ /* TX queues get cores which share at least an L2 cache with the corresponding RX queue */
	+ /* XXX handle multiple RX threads per core and more than two core per L2 group */
	+ return qid / CPU_COUNT(&ctx->ifc_cpus) + 1;
	+ case IFLIB_INTR_RX:
	+ case IFLIB_INTR_RXTX:
	+ /* RX queues get the specified core */
	+ return qid / CPU_COUNT(&ctx->ifc_cpus);
	+ default:
	+ return -1;
	+ }
	+}
	+#else
	+#define get_core_offset(ctx, type, qid) CPU_FIRST()
	+#define find_close_core(cpuid, tid) CPU_FIRST()
	+#define find_nth(ctx, gid) CPU_FIRST()
	+#endif
	+
	+/* Just to avoid copy/paste */
	+static inline int
	+iflib_irq_set_affinity(if_ctx_t ctx, int irq, iflib_intr_type_t type, int qid,
	+ struct grouptask gtask, struct taskqgroup tqg, void uniq, char name)
	+{
	+ int cpuid;
	+ int err, tid;
	+
	+ cpuid = find_nth(ctx, qid);
	+ tid = get_core_offset(ctx, type, qid);
	+ MPASS(tid >= 0);
	+ cpuid = find_close_core(cpuid, tid);
	+ err = taskqgroup_attach_cpu(tqg, gtask, uniq, cpuid, irq, name);
	+ if (err) {
	+ device_printf(ctx->ifc_dev, "taskqgroup_attach_cpu failed %d\n", err);
	+ return (err);
	+ }
	+#ifdef notyet
	+ if (cpuid > ctx->ifc_cpuid_highest)
	+ ctx->ifc_cpuid_highest = cpuid;
	+#endif
	+ return 0;
	+}
	+
	int
	iflib_irq_alloc_generic(if_ctx_t ctx, if_irq_t irq, int rid,
	iflib_intr_type_t type, driver_filter_t *filter,
	@@ -4315,12 +5187,13 @@
	struct grouptask *gtask;
	struct taskqgroup *tqg;
	iflib_filter_info_t info;
	- cpuset_t cpus;
	gtask_fn_t *fn;
	int tqrid, err;
	+ driver_filter_t *intr_fast;
	void *q;

	info = &ctx->ifc_filter_info;
	+ tqrid = rid;

	switch (type) {
	/* XXX merge tx/rx for netmap? */
	@@ -4328,90 +5201,111 @@
	q = &ctx->ifc_txqs[qid];
	info = &ctx->ifc_txqs[qid].ift_filter_info;
	gtask = &ctx->ifc_txqs[qid].ift_task;
	- tqg = qgroup_softirq;
	- tqrid = irq->ii_rid;
	+ tqg = qgroup_if_io_tqg;
	fn = _task_fn_tx;
	+ intr_fast = iflib_fast_intr;
	+ GROUPTASK_INIT(gtask, 0, fn, q);
	break;
	case IFLIB_INTR_RX:
	q = &ctx->ifc_rxqs[qid];
	info = &ctx->ifc_rxqs[qid].ifr_filter_info;
	gtask = &ctx->ifc_rxqs[qid].ifr_task;
	- tqg = qgroup_softirq;
	- tqrid = irq->ii_rid;
	+ tqg = qgroup_if_io_tqg;
	fn = _task_fn_rx;
	+ intr_fast = iflib_fast_intr;
	+ GROUPTASK_INIT(gtask, 0, fn, q);
	break;
	+ case IFLIB_INTR_RXTX:
	+ q = &ctx->ifc_rxqs[qid];
	+ info = &ctx->ifc_rxqs[qid].ifr_filter_info;
	+ gtask = &ctx->ifc_rxqs[qid].ifr_task;
	+ tqg = qgroup_if_io_tqg;
	+ fn = _task_fn_rx;
	+ intr_fast = iflib_fast_intr_rxtx;
	+ GROUPTASK_INIT(gtask, 0, fn, q);
	+ break;
	case IFLIB_INTR_ADMIN:
	q = ctx;
	+ tqrid = -1;
	info = &ctx->ifc_filter_info;
	gtask = &ctx->ifc_admin_task;
	tqg = qgroup_if_config_tqg;
	- tqrid = -1;
	fn = _task_fn_admin;
	+ intr_fast = iflib_fast_intr_ctx;
	break;
	default:
	panic("unknown net intr type");
	}
	- GROUPTASK_INIT(gtask, 0, fn, q);

	info->ifi_filter = filter;
	info->ifi_filter_arg = filter_arg;
	info->ifi_task = gtask;
	+ info->ifi_ctx = q;

	- /* XXX query cpu that rid belongs to */
	-
	- err = _iflib_irq_alloc(ctx, irq, rid, iflib_fast_intr, NULL, info, name);
	- if (err != 0)
	+ err = _iflib_irq_alloc(ctx, irq, rid, intr_fast, NULL, info, name);
	+ if (err != 0) {
	+ device_printf(ctx->ifc_dev, "_iflib_irq_alloc failed %d\n", err);
	return (err);
	+ }
	+ if (type == IFLIB_INTR_ADMIN)
	+ return (0);
	+
	if (tqrid != -1) {
	- find_nth(ctx, &cpus, qid);
	- taskqgroup_attach_cpu(tqg, gtask, q, CPU_FFS(&cpus), irq->ii_rid, name);
	- } else
	- taskqgroup_attach(tqg, gtask, q, tqrid, name);
	+ err = iflib_irq_set_affinity(ctx, rman_get_start(irq->ii_res), type, qid, gtask, tqg, q, name);
	+ if (err)
	+ return (err);
	+ } else {
	+ taskqgroup_attach(tqg, gtask, q, rman_get_start(irq->ii_res), name);
	+ }

	-
	return (0);
	}

	void
	-iflib_softirq_alloc_generic(if_ctx_t ctx, int rid, iflib_intr_type_t type, void arg, int qid, char name)
	+iflib_softirq_alloc_generic(if_ctx_t ctx, if_irq_t irq, iflib_intr_type_t type, void arg, int qid, char name)
	{
	struct grouptask *gtask;
	struct taskqgroup *tqg;
	gtask_fn_t *fn;
	void *q;
	+ int irq_num = -1;
	+ int err;

	switch (type) {
	case IFLIB_INTR_TX:
	q = &ctx->ifc_txqs[qid];
	gtask = &ctx->ifc_txqs[qid].ift_task;
	- tqg = qgroup_softirq;
	+ tqg = qgroup_if_io_tqg;
	fn = _task_fn_tx;
	+ if (irq != NULL)
	+ irq_num = rman_get_start(irq->ii_res);
	break;
	case IFLIB_INTR_RX:
	q = &ctx->ifc_rxqs[qid];
	gtask = &ctx->ifc_rxqs[qid].ifr_task;
	- tqg = qgroup_softirq;
	+ tqg = qgroup_if_io_tqg;
	fn = _task_fn_rx;
	+ if (irq != NULL)
	+ irq_num = rman_get_start(irq->ii_res);
	break;
	- case IFLIB_INTR_ADMIN:
	- q = ctx;
	- gtask = &ctx->ifc_admin_task;
	- tqg = qgroup_if_config_tqg;
	- rid = -1;
	- fn = _task_fn_admin;
	- break;
	case IFLIB_INTR_IOV:
	q = ctx;
	gtask = &ctx->ifc_vflr_task;
	tqg = qgroup_if_config_tqg;
	- rid = -1;
	fn = _task_fn_iov;
	break;
	default:
	panic("unknown net intr type");
	}
	GROUPTASK_INIT(gtask, 0, fn, q);
	- taskqgroup_attach(tqg, gtask, q, rid, name);
	+ if (irq_num != -1) {
	+ err = iflib_irq_set_affinity(ctx, irq_num, type, qid, gtask, tqg, q, name);
	+ if (err)
	+ taskqgroup_attach(tqg, gtask, q, irq_num, name);
	+ }
	+ else {
	+ taskqgroup_attach(tqg, gtask, q, irq_num, name);
	+ }
	}

	void
	@@ -4441,7 +5335,7 @@
	q = &ctx->ifc_rxqs[0];
	info = &rxq[0].ifr_filter_info;
	gtask = &rxq[0].ifr_task;
	- tqg = qgroup_softirq;
	+ tqg = qgroup_if_io_tqg;
	tqrid = irq->ii_rid = *rid;
	fn = _task_fn_rx;

	@@ -4451,16 +5345,13 @@
	info->ifi_task = gtask;

	/* We allocate a single interrupt resource */
	- if ((err = _iflib_irq_alloc(ctx, irq, tqrid, iflib_fast_intr, NULL, info, name)) != 0)
	+ if ((err = _iflib_irq_alloc(ctx, irq, tqrid, iflib_fast_intr_ctx, NULL, info, name)) != 0)
	return (err);
	GROUPTASK_INIT(gtask, 0, fn, q);
	- taskqgroup_attach(tqg, gtask, q, tqrid, name);
	+ taskqgroup_attach(tqg, gtask, q, rman_get_start(irq->ii_res), name);

	GROUPTASK_INIT(&txq->ift_task, 0, _task_fn_tx, txq);
	- taskqgroup_attach(qgroup_softirq, &txq->ift_task, txq, tqrid, "tx");
	- GROUPTASK_INIT(&ctx->ifc_admin_task, 0, _task_fn_admin, ctx);
	- taskqgroup_attach(qgroup_if_config_tqg, &ctx->ifc_admin_task, ctx, -1, "admin/link");
	-
	+ taskqgroup_attach(qgroup_if_io_tqg, &txq->ift_task, txq, rman_get_start(irq->ii_res), "tx");
	return (0);
	}

	@@ -4469,7 +5360,7 @@
	{

	ctx->ifc_led_dev = led_create(iflib_led_func, ctx,
	- device_get_nameunit(ctx->ifc_dev));
	+ device_get_nameunit(ctx->ifc_dev));
	}

	void
	@@ -4489,7 +5380,13 @@
	void
	iflib_admin_intr_deferred(if_ctx_t ctx)
	{
	+#ifdef INVARIANTS
	+ struct grouptask *gtask;

	+ gtask = &ctx->ifc_admin_task;
	+ MPASS(gtask != NULL && gtask->gt_taskqueue != NULL);
	+#endif
	+
	GROUPTASK_ENQUEUE(&ctx->ifc_admin_task);
	}

	@@ -4504,7 +5401,7 @@
	iflib_io_tqg_attach(struct grouptask gt, void uniq, int cpu, char *name)
	{

	- taskqgroup_attach_cpu(qgroup_softirq, gt, uniq, cpu, -1, name);
	+ taskqgroup_attach_cpu(qgroup_if_io_tqg, gt, uniq, cpu, -1, name);
	}

	void
	@@ -4529,8 +5426,9 @@
	if_t ifp = ctx->ifc_ifp;
	iflib_txq_t txq = ctx->ifc_txqs;

	-
	if_setbaudrate(ifp, baudrate);
	+ if (baudrate >= IF_Gbps(10))
	+ ctx->ifc_flags \|= IFC_PREFETCH;

	/* If link down, disable watchdog */
	if ((ctx->ifc_link_state == LINK_STATE_UP) && (link_state == LINK_STATE_DOWN)) {
	@@ -4545,23 +5443,27 @@
	iflib_tx_credits_update(if_ctx_t ctx, iflib_txq_t txq)
	{
	int credits;
	+#ifdef INVARIANTS
	+ int credits_pre = txq->ift_cidx_processed;
	+#endif

	if (ctx->isc_txd_credits_update == NULL)
	return (0);

	- if ((credits = ctx->isc_txd_credits_update(ctx->ifc_softc, txq->ift_id, txq->ift_cidx_processed, true)) == 0)
	+ if ((credits = ctx->isc_txd_credits_update(ctx->ifc_softc, txq->ift_id, true)) == 0)
	return (0);

	txq->ift_processed += credits;
	txq->ift_cidx_processed += credits;

	+ MPASS(credits_pre + credits == txq->ift_cidx_processed);
	if (txq->ift_cidx_processed >= txq->ift_size)
	txq->ift_cidx_processed -= txq->ift_size;
	return (credits);
	}

	static int
	-iflib_rxd_avail(if_ctx_t ctx, iflib_rxq_t rxq, int cidx, int budget)
	+iflib_rxd_avail(if_ctx_t ctx, iflib_rxq_t rxq, qidx_t cidx, qidx_t budget)
	{

	return (ctx->isc_rxd_available(ctx->ifc_softc, rxq->ifr_id, cidx,
	@@ -4599,13 +5501,28 @@
	int iflib_num_tx_queues, iflib_num_rx_queues;
	int err, admincnt, bar;

	- iflib_num_tx_queues = scctx->isc_ntxqsets;
	- iflib_num_rx_queues = scctx->isc_nrxqsets;
	+ iflib_num_tx_queues = ctx->ifc_sysctl_ntxqs;
	+ iflib_num_rx_queues = ctx->ifc_sysctl_nrxqs;

	+ device_printf(dev, "msix_init qsets capped at %d\n", imax(scctx->isc_ntxqsets, scctx->isc_nrxqsets));
	+
	bar = ctx->ifc_softc_ctx.isc_msix_bar;
	admincnt = sctx->isc_admin_intrcnt;
	+ /* Override by global tuneable */
	+ {
	+ int i;
	+ size_t len = sizeof(i);
	+ err = kernel_sysctlbyname(curthread, "hw.pci.enable_msix", &i, &len, NULL, 0, NULL, 0);
	+ if (err == 0) {
	+ if (i == 0)
	+ goto msi;
	+ }
	+ else {
	+ device_printf(dev, "unable to read hw.pci.enable_msix.");
	+ }
	+ }
	/* Override by tuneable */
	- if (enable_msix == 0)
	+ if (scctx->isc_disable_msix)
	goto msi;

	/*
	@@ -4617,18 +5534,20 @@
	** successfully initialize us.
	*/
	{
	- uint16_t pci_cmd_word;
	int msix_ctrl, rid;

	+ pci_enable_busmaster(dev);
	rid = 0;
	- pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2);
	- pci_cmd_word \|= PCIM_CMD_BUSMASTEREN;
	- pci_write_config(dev, PCIR_COMMAND, pci_cmd_word, 2);
	- pci_find_cap(dev, PCIY_MSIX, &rid);
	- rid += PCIR_MSIX_CTRL;
	- msix_ctrl = pci_read_config(dev, rid, 2);
	- msix_ctrl \|= PCIM_MSIXCTRL_MSIX_ENABLE;
	- pci_write_config(dev, rid, msix_ctrl, 2);
	+ if (pci_find_cap(dev, PCIY_MSIX, &rid) == 0 && rid != 0) {
	+ rid += PCIR_MSIX_CTRL;
	+ msix_ctrl = pci_read_config(dev, rid, 2);
	+ msix_ctrl \|= PCIM_MSIXCTRL_MSIX_ENABLE;
	+ pci_write_config(dev, rid, msix_ctrl, 2);
	+ } else {
	+ device_printf(dev, "PCIY_MSIX capability not found; "
	+ "or rid %d == 0.\n", rid);
	+ goto msi;
	+ }
	}

	/*
	@@ -4661,20 +5580,14 @@
	#else
	queuemsgs = msgs - admincnt;
	#endif
	- if (bus_get_cpus(dev, INTR_CPUS, sizeof(ctx->ifc_cpus), &ctx->ifc_cpus) == 0) {
	#ifdef RSS
	- queues = imin(queuemsgs, rss_getnumbuckets());
	+ queues = imin(queuemsgs, rss_getnumbuckets());
	#else
	- queues = queuemsgs;
	+ queues = queuemsgs;
	#endif
	- queues = imin(CPU_COUNT(&ctx->ifc_cpus), queues);
	- device_printf(dev, "pxm cpus: %d queue msgs: %d admincnt: %d\n",
	- CPU_COUNT(&ctx->ifc_cpus), queuemsgs, admincnt);
	- } else {
	- device_printf(dev, "Unable to fetch CPU list\n");
	- /* Figure out a reasonable auto config value */
	- queues = min(queuemsgs, mp_ncpus);
	- }
	+ queues = imin(CPU_COUNT(&ctx->ifc_cpus), queues);
	+ device_printf(dev, "pxm cpus: %d queue msgs: %d admincnt: %d\n",
	+ CPU_COUNT(&ctx->ifc_cpus), queuemsgs, admincnt);
	#ifdef RSS
	/* If we're doing RSS, clamp at the number of RSS buckets */
	if (queues > rss_getnumbuckets())
	@@ -4684,6 +5597,10 @@
	rx_queues = iflib_num_rx_queues;
	else
	rx_queues = queues;
	+
	+ if (rx_queues > scctx->isc_nrxqsets)
	+ rx_queues = scctx->isc_nrxqsets;
	+
	/*
	* We want this to be all logical CPUs by default
	*/
	@@ -4692,6 +5609,9 @@
	else
	tx_queues = mp_ncpus;

	+ if (tx_queues > scctx->isc_ntxqsets)
	+ tx_queues = scctx->isc_ntxqsets;
	+
	if (ctx->ifc_sysctl_qs_eq_override == 0) {
	#ifdef INVARIANTS
	if (tx_queues != rx_queues)
	@@ -4773,7 +5693,7 @@
	if_ctx_t ctx = (void *)arg1;
	enum iflib_ndesc_handler type = arg2;
	char buf[256] = {0};
	- uint16_t *ndesc;
	+ qidx_t *ndesc;
	char p, next;
	int nqs, rc, i;

	@@ -4843,6 +5763,12 @@
	SYSCTL_ADD_U16(ctx_list, oid_list, OID_AUTO, "override_qs_enable",
	CTLFLAG_RWTUN, &ctx->ifc_sysctl_qs_eq_override, 0,
	"permit #txq != #rxq");
	+ SYSCTL_ADD_INT(ctx_list, oid_list, OID_AUTO, "disable_msix",
	+ CTLFLAG_RWTUN, &ctx->ifc_softc_ctx.isc_disable_msix, 0,
	+ "disable MSIX (default 0)");
	+ SYSCTL_ADD_U16(ctx_list, oid_list, OID_AUTO, "rx_budget",
	+ CTLFLAG_RWTUN, &ctx->ifc_sysctl_rx_budget, 0,
	+ "set the rx budget");

	/* XXX change for per-queue sizes */
	SYSCTL_ADD_PROC(ctx_list, oid_list, OID_AUTO, "override_ntxds",
	@@ -4935,27 +5861,26 @@
	CTLFLAG_RD,
	&txq->ift_cleaned, "total cleaned");
	SYSCTL_ADD_PROC(ctx_list, queue_list, OID_AUTO, "ring_state",
	- CTLTYPE_STRING \| CTLFLAG_RD, __DEVOLATILE(uint64_t *, &txq->ift_br[0]->state),
	+ CTLTYPE_STRING \| CTLFLAG_RD, __DEVOLATILE(uint64_t *, &txq->ift_br->state),
	0, mp_ring_state_handler, "A", "soft ring state");
	SYSCTL_ADD_COUNTER_U64(ctx_list, queue_list, OID_AUTO, "r_enqueues",
	- CTLFLAG_RD, &txq->ift_br[0]->enqueues,
	+ CTLFLAG_RD, &txq->ift_br->enqueues,
	"# of enqueues to the mp_ring for this queue");
	SYSCTL_ADD_COUNTER_U64(ctx_list, queue_list, OID_AUTO, "r_drops",
	- CTLFLAG_RD, &txq->ift_br[0]->drops,
	+ CTLFLAG_RD, &txq->ift_br->drops,
	"# of drops in the mp_ring for this queue");
	SYSCTL_ADD_COUNTER_U64(ctx_list, queue_list, OID_AUTO, "r_starts",
	- CTLFLAG_RD, &txq->ift_br[0]->starts,
	+ CTLFLAG_RD, &txq->ift_br->starts,
	"# of normal consumer starts in the mp_ring for this queue");
	SYSCTL_ADD_COUNTER_U64(ctx_list, queue_list, OID_AUTO, "r_stalls",
	- CTLFLAG_RD, &txq->ift_br[0]->stalls,
	+ CTLFLAG_RD, &txq->ift_br->stalls,
	"# of consumer stalls in the mp_ring for this queue");
	SYSCTL_ADD_COUNTER_U64(ctx_list, queue_list, OID_AUTO, "r_restarts",
	- CTLFLAG_RD, &txq->ift_br[0]->restarts,
	+ CTLFLAG_RD, &txq->ift_br->restarts,
	"# of consumer restarts in the mp_ring for this queue");
	SYSCTL_ADD_COUNTER_U64(ctx_list, queue_list, OID_AUTO, "r_abdications",
	- CTLFLAG_RD, &txq->ift_br[0]->abdications,
	+ CTLFLAG_RD, &txq->ift_br->abdications,
	"# of consumer abdications in the mp_ring for this queue");
	-
	}

	if (scctx->isc_nrxqsets > 100)
	@@ -4977,6 +5902,7 @@
	CTLFLAG_RD,
	&rxq->ifr_cq_cidx, 1, "Consumer Index");
	}
	+
	for (j = 0, fl = rxq->ifr_fl; j < rxq->ifr_nfl; j++, fl++) {
	snprintf(namebuf, NAME_BUFLEN, "rxq_fl%d", j);
	fl_node = SYSCTL_ADD_NODE(ctx_list, queue_list, OID_AUTO, namebuf,
	@@ -5010,3 +5936,30 @@
	}

	}
	+
	+#ifndef __NO_STRICT_ALIGNMENT
	+static struct mbuf *
	+iflib_fixup_rx(struct mbuf *m)
	+{
	+ struct mbuf *n;
	+
	+ if (m->m_len <= (MCLBYTES - ETHER_HDR_LEN)) {
	+ bcopy(m->m_data, m->m_data + ETHER_HDR_LEN, m->m_len);
	+ m->m_data += ETHER_HDR_LEN;
	+ n = m;
	+ } else {
	+ MGETHDR(n, M_NOWAIT, MT_DATA);
	+ if (n == NULL) {
	+ m_freem(m);
	+ return (NULL);
	+ }
	+ bcopy(m->m_data, n->m_data, ETHER_HDR_LEN);
	+ m->m_data += ETHER_HDR_LEN;
	+ m->m_len -= ETHER_HDR_LEN;
	+ n->m_len = ETHER_HDR_LEN;
	+ M_MOVE_PKTHDR(n, m);
	+ n->m_next = m;
	+ }
	+ return (n);
	+}
	+#endif
	Index: sys/net/mp_ring.c
	===================================================================
	--- sys/net/mp_ring.c
	+++ sys/net/mp_ring.c
	@@ -454,18 +454,12 @@
	do {
	os.state = ns.state = r->state;
	ns.pidx_tail = pidx_stop;
	- ns.flags = BUSY;
	+ if (os.flags == IDLE)
	+ ns.flags = ABDICATED;
	} while (atomic_cmpset_rel_64(&r->state, os.state, ns.state) == 0);
	critical_exit();
	counter_u64_add(r->enqueues, n);

	- /*
	- * Turn into a consumer if some other thread isn't active as a consumer
	- * already.
	- */
	- if (os.flags != BUSY)
	- drain_ring_lockless(r, ns, os.flags, budget);
	-
	return (0);
	}
	#endif
	@@ -476,7 +470,9 @@
	union ring_state os, ns;

	os.state = r->state;
	- if (os.flags != STALLED \|\| os.pidx_head != os.pidx_tail \|\| r->can_drain(r) == 0)
	+ if ((os.flags != STALLED && os.flags != ABDICATED) \|\| // Only continue in STALLED and ABDICATED
	+ os.pidx_head != os.pidx_tail \|\| // Require work to be available
	+ (os.flags != ABDICATED && r->can_drain(r) == 0)) // Can either drain, or everyone left
	return;

	MPASS(os.cidx != os.pidx_tail); /* implied by STALLED */
	Index: sys/sys/_task.h
	===================================================================
	--- sys/sys/_task.h
	+++ sys/sys/_task.h
	@@ -65,7 +65,8 @@
	void *gt_taskqueue;
	LIST_ENTRY(grouptask) gt_list;
	void *gt_uniq;
	- char *gt_name;
	+#define GROUPTASK_NAMELEN 32
	+ char gt_name[GROUPTASK_NAMELEN];
	int16_t gt_irq;
	int16_t gt_cpu;
	};
	Index: sys/sys/cpuset.h
	===================================================================
	--- sys/sys/cpuset.h
	+++ sys/sys/cpuset.h
	@@ -83,6 +83,8 @@
	#define CPU_WHICH_IRQ 4 /* Specifies an irq #. */
	#define CPU_WHICH_JAIL 5 /* Specifies a jail id. */
	#define CPU_WHICH_DOMAIN 6 /* Specifies a NUMA domain id. */
	+#define CPU_WHICH_INTRHANDLER 7 /* Specifies an irq # (not ithread). */
	+#define CPU_WHICH_ITHREAD 8 /* Specifies an irq's ithread. */

	/*
	* Reserved cpuset identifiers.
	Index: sys/sys/interrupt.h
	===================================================================
	--- sys/sys/interrupt.h
	+++ sys/sys/interrupt.h
	@@ -162,6 +162,8 @@
	driver_filter_t filter, driver_intr_t handler, void *arg,
	u_char pri, enum intr_type flags, void **cookiep);
	int intr_event_bind(struct intr_event *ie, int cpu);
	+int intr_event_bind_irqonly(struct intr_event *ie, int cpu);
	+int intr_event_bind_ithread(struct intr_event *ie, int cpu);
	int intr_event_create(struct intr_event *event, void source,
	int flags, int irq, void (pre_ithread)(void ),
	void (post_ithread)(void ), void (post_filter)(void ),
	@@ -173,9 +175,9 @@
	void intr_event_execute_handlers(struct proc p, struct intr_event ie);
	int intr_event_handle(struct intr_event ie, struct trapframe frame);
	int intr_event_remove_handler(void *cookie);
	-int intr_getaffinity(int irq, void *mask);
	+int intr_getaffinity(int irq, int mode, void *mask);
	void intr_handler_source(void cookie);
	-int intr_setaffinity(int irq, void *mask);
	+int intr_setaffinity(int irq, int mode, void *mask);
	void _intr_drain(int irq); /* Linux compat only. */
	int swi_add(struct intr_event *eventp, const char name,
	driver_intr_t handler, void *arg, int pri, enum intr_type flags,

File Metadata

Mime Type: text/plain
Expires: Tue, Jul 28, 10:48 AM (1 h, 10 m)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 29289943
Default Alt Text: D15143.1785235709.diff (916 KB)

D15143.1785235709.diffNo OneTemporaryActions

D15143.1785235709.diffView Options

File Metadata

Event Timeline

D15143.1785235709.diff
No OneTemporary
Actions

D15143.1785235709.diff
View Options