D11378.1791083393.diff
No OneTemporary
Actions

Size

872 KB

Referenced Files

None

Subscribers

None

D11378.1791083393.diff
View Options

This file is larger than 256 KB, so syntax highlighting was skipped.

	Index: share/man/man4/iw_ixl.4
	===================================================================
	--- /dev/null
	+++ share/man/man4/iw_ixl.4
	@@ -0,0 +1,133 @@
	+.\" Copyright (c) 2017 Intel Corporation. All rights reserved.
	+.\"
	+.\" This software is available to you under a choice of one of two
	+.\" licenses. You may choose to be licensed under the terms of the GNU
	+.\" General Public License (GPL) Version 2, available from the file
	+.\" COPYING in the main directory of this source tree, or the
	+.\" OpenFabrics.org BSD license below:
	+.\"
	+.\" Redistribution and use in source and binary forms, with or
	+.\" without modification, are permitted provided that the following
	+.\" conditions are met:
	+.\"
	+.\" - Redistributions of source code must retain the above
	+.\" copyright notice, this list of conditions and the following
	+.\" disclaimer.
	+.\"
	+.\" - Redistributions in binary form must reproduce the above
	+.\" copyright notice, this list of conditions and the following
	+.\" disclaimer in the documentation and/or other materials
	+.\" provided with the distribution.
	+.\"
	+.\" THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
	+.\" EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
	+.\" MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
	+.\" NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
	+.\" BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
	+.\" ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
	+.\" CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
	+.\" SOFTWARE.
	+.Dd July 31, 2017
	+.Dt IW_IXL 4
	+.Os
	+.Sh NAME
	+.Nm iw_ixl
	+.Nd "Beta iWARP Driver for the Intel(R) Ethernet Connection X722 for the FreeBSD operating system"
	+.Sh SYNOPSIS
	+This module relies on
	+.Xr if_ixl 4
	+and requires a number of adjustments before it is ready to use.
	+.Bl -tag -nested-width indent
	+.It The following kernel options should be included in the configuration:
	+.Bl -tag
	+.It IXL_IW
	+.It OFED
	+.It OFED_DEBUG_INIT
	+.It IPOIB_CM
	+.It COMPAT_LINUXKPI
	+.El
	+.It The following tunable should be set:
	+.Bl -tag
	+.It hw.ixl.enable_iwarp=1
	+.El
	+.El
	+.Pp
	+.Sh DESCRIPTION
	+The
	+.Nm
	+driver provides support for iWARP capability of the Intel(R) Ethernet Controller X722 family of Ethernet devices.
	+The driver requires
	+.Xr if_ixl 4
	+ version 1.7.12 or higher.
	+Currently, only kernel verbs are supported.
	+For further hardware information and questions related to hardware requirements, see
	+.Lk http://support.intel.com/ .
	+.Sh HARDWARE
	+The driver supports the Intel(R) Ethernet Controller X722 family of Ethernet devices with iWARP support.
	+.Sh LOADER TUNABLES
	+Tunables can be set at the
	+.Xr loader 8
	+prompt before booting the kernel or stored in
	+.Xr loader.conf 5 .
	+.Bl -tag -width indent
	+.It Va hw.iw_ixl.max_ceq
	+The maximum number of MSI-X vectors available to the driver for CEQ usage.
	+The value is usually evaluated with respect to maximal number of MSI-X vectors on the pci, hardware limitations and number of CPUs.
	+The tunable controls how many MSI-X vectors are actually used by the driver.
	+If equal to 0, the value will be ignored.
	+.It Va hw.iw_ixl.debug
	+The amount of debug messages to print.
	+.It Va hw.iw_ixl.mpa_version
	+Select which MPA version to use.
	+The default is 2.
	+.El
	+.Sh SYSCTL PROCEDURES
	+.Bl -tag -width indent
	+.It Va hw.iw_ixl.debug
	+The amount of debug messages to print.
	+.El
	+.Sh KNOWN ISSUES
	+.Bl -dash -nested
	+.It
	+Note that 40G operation can generate high numbers of interrupts, often incorrectly being interpreted as a storm condition in the kernel.
	+Resolve this by setting:
	+.Bl -tag -width indent
	+.It Va hw.intr_storm_threshold: 0
	+.El
	+Consult
	+.Xr ixl 4 ,
	+or
	+.Lk http://support.intel.com/
	+for details.
	+.It
	+Loopback connection is not supported.
	+.It
	+Connection using IPv6 is not supported.
	+.It
	+Changing the MTU size is not supported.
	+.It
	+MW memory mode is not supported.
	+.It
	+MR memory mode supports only single buffer.
	+.It
	+ib_cq_resize function is not supported.
	+.El
	+.Sh SUPPORT
	+For general information and support,
	+go to the Intel support website at:
	+.Lk http://support.intel.com/ .
	+.Pp
	+If an issue with this driver is identified on a supported adapter,
	+email all the specific information related to the issue to
	+.Aq freebsd@intel.com .
	+.Sh SEE ALSO
	+.Xr ixl 4 ,
	+.Xr ixlv 4 ,
	+.Sh AUTHORS
	+.An -nosplit
	+The
	+.Nm
	+driver has been ported, prepared and maintained by
	+.An Maciej Sosnowski Aq Mt maciej.m.sosnowski@gmail.com
	+and
	+.An Bartosz Sobczak Aq Mt bartosz.m.sobczak@gmail.com .
	Index: sys/amd64/conf/NOTES
	===================================================================
	--- sys/amd64/conf/NOTES
	+++ sys/amd64/conf/NOTES
	@@ -315,6 +315,7 @@
	# Requires the iwn firmware module
	# ixl: Intel XL710 40Gbe PCIE Ethernet
	# ixlv: Intel XL710 40Gbe VF PCIE Ethernet
	+# iw_ixl: Intel X722 40Gbe PCIE iWARP driver
	# mlx4ib: Mellanox ConnectX HCA InfiniBand
	# mlx4en: Mellanox ConnectX HCA Ethernet
	# nfe: nVidia nForce MCP on-board Ethernet Networking (BSD open source)
	@@ -334,6 +335,8 @@
	device ixl # Intel XL710 40Gbe PCIE Ethernet
	options IXL_IW # Enable iWARP Client Interface in ixl(4)
	device ixlv # Intel XL710 40Gbe VF PCIE Ethernet
	+nodevice iw_ixl # Intel X722 40Gbe PCIE iWARP driver
	+options IW_IXL_RDMA_4_9 # Enable compatibility with newest OFED
	device mlx4 # Shared code module between IB and Ethernet
	device mlx4ib # Mellanox ConnectX HCA InfiniBand
	device mlx4en # Mellanox ConnectX HCA Ethernet
	Index: sys/conf/files.amd64
	===================================================================
	--- sys/conf/files.amd64
	+++ sys/conf/files.amd64
	@@ -288,6 +288,28 @@
	compile-with "${NORMAL_C} -I$S/dev/ixl"
	dev/ixl/i40e_adminq.c optional ixl pci \| ixlv pci \
	compile-with "${NORMAL_C} -I$S/dev/ixl"
	+dev/ixl/iwarp/iw_ixl.c optional iw_ixl ixl pci \
	+ compile-with "${OFED_C} -I$S/dev/ixl/iwarp -I$S/dev/ixl/"
	+dev/ixl/iwarp/iw_ixl_cm.c optional iw_ixl ixl pci \
	+ compile-with "${OFED_C} -I$S/dev/ixl/iwarp -I$S/dev/ixl/"
	+dev/ixl/iwarp/iw_ixl_hw.c optional iw_ixl ixl pci \
	+ compile-with "${OFED_C} -I$S/dev/ixl/iwarp -I$S/dev/ixl/"
	+dev/ixl/iwarp/iw_ixl_linux_genalloc.c optional iw_ixl ixl pci \
	+ compile-with "${OFED_C} -I$S/dev/ixl/iwarp -I$S/dev/ixl/"
	+dev/ixl/iwarp/iw_ixl_pble.c optional iw_ixl ixl pci \
	+ compile-with "${OFED_C} -I$S/dev/ixl/iwarp -I$S/dev/ixl/"
	+dev/ixl/iwarp/iw_ixl_utils.c optional iw_ixl ixl pci \
	+ compile-with "${OFED_C} -I$S/dev/ixl/iwarp -I$S/dev/ixl/"
	+dev/ixl/iwarp/iw_ixl_verbs.c optional iw_ixl ixl pci \
	+ compile-with "${OFED_C} -I$S/dev/ixl/iwarp -I$S/dev/ixl/"
	+dev/ixl/iwarp/i40iw_ctrl.c optional iw_ixl ixl pci \
	+ compile-with "${OFED_C} -I$S/dev/ixl/iwarp -I$S/dev/ixl/"
	+dev/ixl/iwarp/i40iw_hmc.c optional iw_ixl ixl pci \
	+ compile-with "${OFED_C} -I$S/dev/ixl/iwarp -I$S/dev/ixl/"
	+dev/ixl/iwarp/i40iw_puda.c optional iw_ixl ixl pci \
	+ compile-with "${OFED_C} -I$S/dev/ixl/iwarp -I$S/dev/ixl/"
	+dev/ixl/iwarp/i40iw_uk.c optional iw_ixl ixl pci \
	+ compile-with "${OFED_C} -I$S/dev/ixl/iwarp -I$S/dev/ixl/"
	dev/fdc/fdc.c optional fdc
	dev/fdc/fdc_acpi.c optional fdc
	dev/fdc/fdc_isa.c optional fdc isa
	Index: sys/conf/options.amd64
	===================================================================
	--- sys/conf/options.amd64
	+++ sys/conf/options.amd64
	@@ -48,6 +48,7 @@

	# iWARP client interface support in ixl
	IXL_IW opt_ixl.h
	+IW_IXL_RDMA_4_9 opt_ixl.h

	# -------------------------------
	# EOF
	Index: sys/dev/ixl/iwarp/COPYING
	===================================================================
	--- /dev/null
	+++ sys/dev/ixl/iwarp/COPYING
	@@ -0,0 +1,339 @@
	+ GNU GENERAL PUBLIC LICENSE
	+ Version 2, June 1991
	+
	+ Copyright (C) 1989, 1991 Free Software Foundation, Inc.,
	+ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
	+ Everyone is permitted to copy and distribute verbatim copies
	+ of this license document, but changing it is not allowed.
	+
	+ Preamble
	+
	+ The licenses for most software are designed to take away your
	+freedom to share and change it. By contrast, the GNU General Public
	+License is intended to guarantee your freedom to share and change free
	+software--to make sure the software is free for all its users. This
	+General Public License applies to most of the Free Software
	+Foundation's software and to any other program whose authors commit to
	+using it. (Some other Free Software Foundation software is covered by
	+the GNU Lesser General Public License instead.) You can apply it to
	+your programs, too.
	+
	+ When we speak of free software, we are referring to freedom, not
	+price. Our General Public Licenses are designed to make sure that you
	+have the freedom to distribute copies of free software (and charge for
	+this service if you wish), that you receive source code or can get it
	+if you want it, that you can change the software or use pieces of it
	+in new free programs; and that you know you can do these things.
	+
	+ To protect your rights, we need to make restrictions that forbid
	+anyone to deny you these rights or to ask you to surrender the rights.
	+These restrictions translate to certain responsibilities for you if you
	+distribute copies of the software, or if you modify it.
	+
	+ For example, if you distribute copies of such a program, whether
	+gratis or for a fee, you must give the recipients all the rights that
	+you have. You must make sure that they, too, receive or can get the
	+source code. And you must show them these terms so they know their
	+rights.
	+
	+ We protect your rights with two steps: (1) copyright the software, and
	+(2) offer you this license which gives you legal permission to copy,
	+distribute and/or modify the software.
	+
	+ Also, for each author's protection and ours, we want to make certain
	+that everyone understands that there is no warranty for this free
	+software. If the software is modified by someone else and passed on, we
	+want its recipients to know that what they have is not the original, so
	+that any problems introduced by others will not reflect on the original
	+authors' reputations.
	+
	+ Finally, any free program is threatened constantly by software
	+patents. We wish to avoid the danger that redistributors of a free
	+program will individually obtain patent licenses, in effect making the
	+program proprietary. To prevent this, we have made it clear that any
	+patent must be licensed for everyone's free use or not licensed at all.
	+
	+ The precise terms and conditions for copying, distribution and
	+modification follow.
	+
	+ GNU GENERAL PUBLIC LICENSE
	+ TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
	+
	+ 0. This License applies to any program or other work which contains
	+a notice placed by the copyright holder saying it may be distributed
	+under the terms of this General Public License. The "Program", below,
	+refers to any such program or work, and a "work based on the Program"
	+means either the Program or any derivative work under copyright law:
	+that is to say, a work containing the Program or a portion of it,
	+either verbatim or with modifications and/or translated into another
	+language. (Hereinafter, translation is included without limitation in
	+the term "modification".) Each licensee is addressed as "you".
	+
	+Activities other than copying, distribution and modification are not
	+covered by this License; they are outside its scope. The act of
	+running the Program is not restricted, and the output from the Program
	+is covered only if its contents constitute a work based on the
	+Program (independent of having been made by running the Program).
	+Whether that is true depends on what the Program does.
	+
	+ 1. You may copy and distribute verbatim copies of the Program's
	+source code as you receive it, in any medium, provided that you
	+conspicuously and appropriately publish on each copy an appropriate
	+copyright notice and disclaimer of warranty; keep intact all the
	+notices that refer to this License and to the absence of any warranty;
	+and give any other recipients of the Program a copy of this License
	+along with the Program.
	+
	+You may charge a fee for the physical act of transferring a copy, and
	+you may at your option offer warranty protection in exchange for a fee.
	+
	+ 2. You may modify your copy or copies of the Program or any portion
	+of it, thus forming a work based on the Program, and copy and
	+distribute such modifications or work under the terms of Section 1
	+above, provided that you also meet all of these conditions:
	+
	+ a) You must cause the modified files to carry prominent notices
	+ stating that you changed the files and the date of any change.
	+
	+ b) You must cause any work that you distribute or publish, that in
	+ whole or in part contains or is derived from the Program or any
	+ part thereof, to be licensed as a whole at no charge to all third
	+ parties under the terms of this License.
	+
	+ c) If the modified program normally reads commands interactively
	+ when run, you must cause it, when started running for such
	+ interactive use in the most ordinary way, to print or display an
	+ announcement including an appropriate copyright notice and a
	+ notice that there is no warranty (or else, saying that you provide
	+ a warranty) and that users may redistribute the program under
	+ these conditions, and telling the user how to view a copy of this
	+ License. (Exception: if the Program itself is interactive but
	+ does not normally print such an announcement, your work based on
	+ the Program is not required to print an announcement.)
	+
	+These requirements apply to the modified work as a whole. If
	+identifiable sections of that work are not derived from the Program,
	+and can be reasonably considered independent and separate works in
	+themselves, then this License, and its terms, do not apply to those
	+sections when you distribute them as separate works. But when you
	+distribute the same sections as part of a whole which is a work based
	+on the Program, the distribution of the whole must be on the terms of
	+this License, whose permissions for other licensees extend to the
	+entire whole, and thus to each and every part regardless of who wrote it.
	+
	+Thus, it is not the intent of this section to claim rights or contest
	+your rights to work written entirely by you; rather, the intent is to
	+exercise the right to control the distribution of derivative or
	+collective works based on the Program.
	+
	+In addition, mere aggregation of another work not based on the Program
	+with the Program (or with a work based on the Program) on a volume of
	+a storage or distribution medium does not bring the other work under
	+the scope of this License.
	+
	+ 3. You may copy and distribute the Program (or a work based on it,
	+under Section 2) in object code or executable form under the terms of
	+Sections 1 and 2 above provided that you also do one of the following:
	+
	+ a) Accompany it with the complete corresponding machine-readable
	+ source code, which must be distributed under the terms of Sections
	+ 1 and 2 above on a medium customarily used for software interchange; or,
	+
	+ b) Accompany it with a written offer, valid for at least three
	+ years, to give any third party, for a charge no more than your
	+ cost of physically performing source distribution, a complete
	+ machine-readable copy of the corresponding source code, to be
	+ distributed under the terms of Sections 1 and 2 above on a medium
	+ customarily used for software interchange; or,
	+
	+ c) Accompany it with the information you received as to the offer
	+ to distribute corresponding source code. (This alternative is
	+ allowed only for noncommercial distribution and only if you
	+ received the program in object code or executable form with such
	+ an offer, in accord with Subsection b above.)
	+
	+The source code for a work means the preferred form of the work for
	+making modifications to it. For an executable work, complete source
	+code means all the source code for all modules it contains, plus any
	+associated interface definition files, plus the scripts used to
	+control compilation and installation of the executable. However, as a
	+special exception, the source code distributed need not include
	+anything that is normally distributed (in either source or binary
	+form) with the major components (compiler, kernel, and so on) of the
	+operating system on which the executable runs, unless that component
	+itself accompanies the executable.
	+
	+If distribution of executable or object code is made by offering
	+access to copy from a designated place, then offering equivalent
	+access to copy the source code from the same place counts as
	+distribution of the source code, even though third parties are not
	+compelled to copy the source along with the object code.
	+
	+ 4. You may not copy, modify, sublicense, or distribute the Program
	+except as expressly provided under this License. Any attempt
	+otherwise to copy, modify, sublicense or distribute the Program is
	+void, and will automatically terminate your rights under this License.
	+However, parties who have received copies, or rights, from you under
	+this License will not have their licenses terminated so long as such
	+parties remain in full compliance.
	+
	+ 5. You are not required to accept this License, since you have not
	+signed it. However, nothing else grants you permission to modify or
	+distribute the Program or its derivative works. These actions are
	+prohibited by law if you do not accept this License. Therefore, by
	+modifying or distributing the Program (or any work based on the
	+Program), you indicate your acceptance of this License to do so, and
	+all its terms and conditions for copying, distributing or modifying
	+the Program or works based on it.
	+
	+ 6. Each time you redistribute the Program (or any work based on the
	+Program), the recipient automatically receives a license from the
	+original licensor to copy, distribute or modify the Program subject to
	+these terms and conditions. You may not impose any further
	+restrictions on the recipients' exercise of the rights granted herein.
	+You are not responsible for enforcing compliance by third parties to
	+this License.
	+
	+ 7. If, as a consequence of a court judgment or allegation of patent
	+infringement or for any other reason (not limited to patent issues),
	+conditions are imposed on you (whether by court order, agreement or
	+otherwise) that contradict the conditions of this License, they do not
	+excuse you from the conditions of this License. If you cannot
	+distribute so as to satisfy simultaneously your obligations under this
	+License and any other pertinent obligations, then as a consequence you
	+may not distribute the Program at all. For example, if a patent
	+license would not permit royalty-free redistribution of the Program by
	+all those who receive copies directly or indirectly through you, then
	+the only way you could satisfy both it and this License would be to
	+refrain entirely from distribution of the Program.
	+
	+If any portion of this section is held invalid or unenforceable under
	+any particular circumstance, the balance of the section is intended to
	+apply and the section as a whole is intended to apply in other
	+circumstances.
	+
	+It is not the purpose of this section to induce you to infringe any
	+patents or other property right claims or to contest validity of any
	+such claims; this section has the sole purpose of protecting the
	+integrity of the free software distribution system, which is
	+implemented by public license practices. Many people have made
	+generous contributions to the wide range of software distributed
	+through that system in reliance on consistent application of that
	+system; it is up to the author/donor to decide if he or she is willing
	+to distribute software through any other system and a licensee cannot
	+impose that choice.
	+
	+This section is intended to make thoroughly clear what is believed to
	+be a consequence of the rest of this License.
	+
	+ 8. If the distribution and/or use of the Program is restricted in
	+certain countries either by patents or by copyrighted interfaces, the
	+original copyright holder who places the Program under this License
	+may add an explicit geographical distribution limitation excluding
	+those countries, so that distribution is permitted only in or among
	+countries not thus excluded. In such case, this License incorporates
	+the limitation as if written in the body of this License.
	+
	+ 9. The Free Software Foundation may publish revised and/or new versions
	+of the General Public License from time to time. Such new versions will
	+be similar in spirit to the present version, but may differ in detail to
	+address new problems or concerns.
	+
	+Each version is given a distinguishing version number. If the Program
	+specifies a version number of this License which applies to it and "any
	+later version", you have the option of following the terms and conditions
	+either of that version or of any later version published by the Free
	+Software Foundation. If the Program does not specify a version number of
	+this License, you may choose any version ever published by the Free Software
	+Foundation.
	+
	+ 10. If you wish to incorporate parts of the Program into other free
	+programs whose distribution conditions are different, write to the author
	+to ask for permission. For software which is copyrighted by the Free
	+Software Foundation, write to the Free Software Foundation; we sometimes
	+make exceptions for this. Our decision will be guided by the two goals
	+of preserving the free status of all derivatives of our free software and
	+of promoting the sharing and reuse of software generally.
	+
	+ NO WARRANTY
	+
	+ 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
	+FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN
	+OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
	+PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
	+OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
	+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS
	+TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE
	+PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
	+REPAIR OR CORRECTION.
	+
	+ 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
	+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
	+REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
	+INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
	+OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
	+TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
	+YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
	+PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
	+POSSIBILITY OF SUCH DAMAGES.
	+
	+ END OF TERMS AND CONDITIONS
	+
	+ How to Apply These Terms to Your New Programs
	+
	+ If you develop a new program, and you want it to be of the greatest
	+possible use to the public, the best way to achieve this is to make it
	+free software which everyone can redistribute and change under these terms.
	+
	+ To do so, attach the following notices to the program. It is safest
	+to attach them to the start of each source file to most effectively
	+convey the exclusion of warranty; and each file should have at least
	+the "copyright" line and a pointer to where the full notice is found.
	+
	+ <one line to give the program's name and a brief idea of what it does.>
	+ Copyright (C) <year> <name of author>
	+
	+ This program is free software; you can redistribute it and/or modify
	+ it under the terms of the GNU General Public License as published by
	+ the Free Software Foundation; either version 2 of the License, or
	+ (at your option) any later version.
	+
	+ This program is distributed in the hope that it will be useful,
	+ but WITHOUT ANY WARRANTY; without even the implied warranty of
	+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	+ GNU General Public License for more details.
	+
	+ You should have received a copy of the GNU General Public License along
	+ with this program; if not, write to the Free Software Foundation, Inc.,
	+ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
	+
	+Also add information on how to contact you by electronic and paper mail.
	+
	+If the program is interactive, make it output a short notice like this
	+when it starts in an interactive mode:
	+
	+ Gnomovision version 69, Copyright (C) year name of author
	+ Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
	+ This is free software, and you are welcome to redistribute it
	+ under certain conditions; type `show c' for details.
	+
	+The hypothetical commands `show w' and `show c' should show the appropriate
	+parts of the General Public License. Of course, the commands you use may
	+be called something other than `show w' and `show c'; they could even be
	+mouse-clicks or menu items--whatever suits your program.
	+
	+You should also get your employer (if you work as a programmer) or your
	+school, if any, to sign a "copyright disclaimer" for the program, if
	+necessary. Here is a sample; alter the names:
	+
	+ Yoyodyne, Inc., hereby disclaims all copyright interest in the program
	+ `Gnomovision' (which makes passes at compilers) written by James Hacker.
	+
	+ <signature of Ty Coon>, 1 April 1989
	+ Ty Coon, President of Vice
	+
	+This General Public License does not permit incorporating your program into
	+proprietary programs. If your program is a subroutine library, you may
	+consider it more useful to permit linking proprietary applications with the
	+library. If this is what you want to do, use the GNU Lesser General
	+Public License instead of this License.
	Index: sys/dev/ixl/iwarp/README
	===================================================================
	--- /dev/null
	+++ sys/dev/ixl/iwarp/README
	@@ -0,0 +1,312 @@
	+ixl iWARP FreeBSD* driver for Intel(R) Ethernet Connection X722
	+================================================================
	+December 15, 2017
	+
	+Contents
	+========
	+
	+- Prerequisites
	+- Building and Installation
	+- Testing
	+- Configuration
	+- Interoperability
	+- Known Issues
	+
	+
	+================================================================================
	+
	+
	+Prerequisites
	+-------------
	+
	+- FreeBSD version 10.3 or 11.0, or later.
	+- Kernel configuration:
	+ Please add the following kernel configuration options:
	+ include GENERIC
	+ options OFED
	+ options OFED_DEBUG_INIT
	+ options COMPAT_LINUXKPI
	+ options SDP
	+ options IPOIB_CM
	+ options IXL_IW
	+
	+ nodevice ixl
	+ nodevice ixlv
	+ Note: IXL_IW is required for FreeBSD-CURRENT branch.
	+- For the iw_ixl driver to work, an if_ixl driver with iwarp interface
	+ is required. The interface is available in if_ixl version 1.7.12 or later.
	+ It should be enabled prior to usage, as the setting is switched off by
	+ default. To enable iwarp compatibility, add
	+ hw.ixl.enable_iwarp=1
	+to
	+ /boot/loader.conf
	+
	+The lan driver can be downloaded from
	+https://downloadcenter.intel.com/download/25160/Ethernet-Intel-Network-Adapter-D
	+river-for-PCIe-40-Gigabit-Ethernet-Network-Connection-under-FreeBSD
	+Or search on downloadcenter.intel.com using '40 Gigabit Ethernet Network
	+Connection under FreeBSD'. Newer OS releases contain the if_ixl driver in
	+the ixl driver version 1.7.12-k or later.
	+
	+There are some known issues with the interface on if_ixl-1.7.12. Please
	+use version 1.7.13 or later.
	+
	+- fastreg memory mode in krping might need a patch applied to krping.
	+ Refer to the 'Testing' and 'Known Issues' sections for details.
	+- fr test in krping in FreeBSD 10.3 needs a patch applied to krping.
	+ Refer to 'Testing' and 'Known Issues' section for details.
	+
	+
	+Building and Installation
	+-------------------------
	+
	+1. Untar ixl-<version>.tar.gz and iw_ixl-<version>.tar.gz
	+ tar -xf ixl-<version>.tar.gz
	+ tar -xf iw_ixl-<version>.tar.gz
	+2. Install the if_ixl driver:
	+ cd ixl-<version>/src directory
	+ make
	+ make install
	+3. Install the iw_ixl driver:
	+ cd iw_ixl-<version>/src
	+ make clean
	+ make IXL_DIR=$PATH_TO_IXL/ixl-<version>/src
	+ make install
	+4. Install the man page for the iw_ixl driver by copying the iw_ixl.4.gz file
	+ to the directory where manual pages are held on your system, for instance:
	+ cp iw_ixl-<version>/doc/iw_ixl.4.gz /usr/share/man/man4/
	+
	+For in-tree driver if_ixl-1.7.12-k or later, it is sufficient to follow
	+the instruction from point 3 but ensure the correct path to if_ixl source
	+folder is supplied, for instance:
	+ IXL_DIR=/usr/src/sys/dev/ixl/
	+
	+
	+Testing
	+-------
	+1. To load iw_ixl driver call:
	+ kldload iw_ixl
	+ If if_ixl is not already loaded, the system will load it on its own.
	+ Please remember to add
	+ hw.ixl.enable_iwarp=1
	+ to /boot/loader.conf file prior to if_ixl loading, to ensure the ixl
	+ driver has the iwarp interface enabled.
	+2. To validate the load of the driver check:
	+ sysctl -a \| grep infiniband
	+ a number of sys.class.infiniband should appear provided at least one
	+ port of the X722 is up.
	+3. The source code for krping software is provided with the kernel in
	+/usr/src/sys/contrib/rdma/krping/. To compile the software, change directory
	+to /usr/src/sys/modules/rdma/krping/ and invoke:
	+ make clean
	+ make
	+ make install
	+ Note: In FreeBSD 10.3, some changes need to be applied to the krping.c file
	+ in order to enable fr test.
	+ For details please refer to Known Issues section.
	+4. Start krping server on one machine:
	+ echo size=64,count=1,port=6601,addr=100.0.0.189,server > /dev/krping
	+5. Connect client from another machine:
	+ echo size=64,count=1,port=6601,addr=100.0.0.189,client > /dev/krping
	+
	+
	+================================================================================
	+
	+
	+Configuration
	+-------------
	+
	+The following sysctl options are visible:
	+ - hw.iw_ixl.max_ceq
	+ determines the maximum number of msix vectors available to the driver
	+ for CEQ usage.
	+ - hw.iw_ixl.debug
	+ defines level of debug messages.
	+ - hw.iw_ixl.mpa_version
	+ shows the current MPA version used.
	+
	+The max_ceq setting may be changed by adding:
	+ hw.iw_ixl.max_ceq=$value
	+to /boot/loader.conf file. The final number of CEQ is evaluated depending
	+on the available msix vectors, number of cpu cores, and hardware limits.
	+
	+If max_ceq=0, the value is ignored.
	+
	+The debug setting may be changed either by adding:
	+ hw.iw_ixl.debug=$value
	+to the /boot/loader.conf file or by calling
	+ sysctl hw.iw_ixl.debug=$value
	+
	+The mpa_version may be changed by adding:
	+ hw.iw_ixl.mpa_version=$value
	+to the /boot/loader.conf file.
	+
	+
	+Interoperability
	+----------------
	+
	+To interoperate with Chelsio iWARP devices:
	+
	+1. Load the ixl driver with parameter mpa_version set to 1.
	+Add the line:
	+ hw.iw_ixl.mpa_version=1
	+to /boot/loader.conf
	+
	+2. Load Chelsio T4/T5 RDMA driver (iw_cxgb4) with parameters
	+ c4iw_max_read_depth set to 63 and dack_mode set to 0.
	+
	+
	+================================================================================
	+
	+
	+Known Issues
	+------------
	+
	+- Loopback is not supported.
	+- MTU changes are not supported.
	+- IPv6 is not supported in FreeBSD 10.3, 10.4, 11.0 or 11.1, the support is
	+ added in FreeBSD-CURRENT with IW_IXL_RDMA_4_9 flag enabled.
	+- MW memory mode is not supported.
	+- MR memory mode supports only single buffer.
	+- The function ib_cq_resize is not supported.
	+- Krping with read_inv option ends with error on FreeBSD-CURRENT system during
	+ first iteration.
	+- The max number of registered cq, qp, pd or mr reported by the device may
	+ differ from the actual number of registrations achievable.
	+- A kernel crash may occur when trying to run krping without ensuring that the
	+ two machines are able to ping each other.
	+- A kernel crash may occur when trying to load the iw_ixl driver when
	+ hw.ixl.enable_iwarp=0 (fixed with if_ixl 1.7.13).
	+- A kernel crash may occur when loading the iw_ixl driver on a card that is
	+ supported by if_ixl driver, but does not have iWARP capability (fixed with
	+ if_ixl 1.7.13).
	+- On FreeBSD 10.3 you can run krping traffic with a Chelsio T5 card if
	+ mpa_version on the X722 card is set to '1'. To set the MPA version, add:
	+ hw.iw_ixl.mpa_version=1
	+ to /boot/loader.conf.
	+ Setting MPA version on both the X722 and the Chelsio T5 card to '2' may
	+ result in a kernel crash.
	+ On FreeBSD 11.0, krping traffic is possible with the default settings.
	+- Krping with fastreg memory mode will not work on driver 0.1.17 or older,
	+ unless some changes are made to krping. To workaround the issue, modify
	+ the krping_rdma_rkey function such that, in the case of FASTREG memory mode,
	+ the ib_post_send function with &cd->invalidate_wr parameter is not called
	+ during the first run of the function
	+- In FreeBSD 10.3, krping fr test is incorrectly implemented. Please consult
	+ the krping.c file provided with the FreeBSD 11.0 kernel for the required
	+ fixes. Needed changes include, but are not limited to:
	+ - The fr test requires the server side (unlike what is written in
	+ the comments).
	+ function: krping_doit
	+ function: krping_fr_test_server
	+ function: krping_run_server
	+ - The fr test require MR memory mode.
	+ function: krping_doit
	+ - Wrong device is queried on the client side when checking fastreg support.
	+ function: fastreg_supported
	+ - Note: the fr itself is designed to work endlessly, however lack of
	+ wait_event_interruptible_timeout function support causes the test to do
	+ nothing after the first iteration.
	+ function: krping_fr_test
	+
	+--------------------
	+Changes since 0.1.12
	+--------------------
	+- Adjustments for changes in the ixl driver.
	+- Adjustments for changes in the kernel's OFED of FreeBSD-CURRENT.
	+- Fix for running rdma traffic using krping via tagged vlan.
	+- Rewritten address resolve algorithm for systems with VIMAGE option enabled.
	+- Fix for fast memory registration problem seen in krping fastreg tests,
	+ the krping patch for these tests will not be needed from version 0.1.18
	+ of the driver
	+
	+-------------------
	+Changes since 0.1.2
	+-------------------
	+- Fix for kernel crash on FreeBSD 11.0 when setting ixl3 down.
	+- Fix for krping test with fastreg and server_inv options enabled.
	+
	+--------------------
	+Changes since 0.0.24
	+--------------------
	+- Added patch for 1.7.12 to fix a number of bugs (These changes are available
	+ in the ixl driver, version 1.7.13 and later):
	+ o kernel crash on loading iw_ixl on XL710 cards.
	+ o kernel crash on loading iw_ixl with enable_iwarp tunable set to 0.
	+- MSIX vector utilization has been reworked.
	+- CPU affinity of the CEQ has been implemented.
	+- Added sysctl controls.
	+
	+--------------------
	+Changes since 0.0.18
	+--------------------
	+- No patch for OFED is necessary for FreeBSD 11.0.
	+
	+--------------------
	+Changes since 0.0.11
	+--------------------
	+- Support for FreeBSD 11.0 connection is added.
	+- No patch for OFED is necessary for FreeBSD 10.3.
	+
	+
	+================================================================================
	+
	+
	+Support
	+-------
	+For general information, go to the Intel support website at:
	+www.intel.com/support/
	+
	+or the Intel Wired Networking project hosted by Sourceforge at:
	+http://sourceforge.net/projects/e1000
	+If an issue is identified with the released source code on a supported kernel
	+with a supported adapter, email the specific information related to the issue
	+to e1000-rdma@lists.sourceforge.net
	+
	+
	+================================================================================
	+
	+
	+License
	+-------
	+
	+This software is available to you under a choice of one of two
	+licenses. You may choose to be licensed under the terms of the GNU
	+General Public License (GPL) Version 2, available from the file
	+COPYING in the main directory of this source tree, or the
	+OpenFabrics.org BSD license below:
	+
	+ Redistribution and use in source and binary forms, with or
	+ without modification, are permitted provided that the following
	+ conditions are met:
	+
	+ - Redistributions of source code must retain the above
	+ copyright notice, this list of conditions and the following
	+ disclaimer.
	+
	+ - Redistributions in binary form must reproduce the above
	+ copyright notice, this list of conditions and the following
	+ disclaimer in the documentation and/or other materials
	+ provided with the distribution.
	+
	+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
	+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
	+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
	+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
	+BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
	+ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
	+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
	+SOFTWARE.
	+
	+
	+================================================================================
	+
	+
	+Trademarks
	+----------
	+Intel and Itanium are trademarks or registered trademarks of Intel Corporation
	+or its subsidiaries in the United States and/or other countries.
	+
	+* Other names and brands may be claimed as the property of others.
	+
	+
	Index: sys/dev/ixl/iwarp/i40iw_ctrl.c
	===================================================================
	--- /dev/null
	+++ sys/dev/ixl/iwarp/i40iw_ctrl.c
	@@ -0,0 +1,4990 @@
	+/*******************************************************************************
	+*
	+* Copyright (c) 2015-2017 Intel Corporation. All rights reserved.
	+*
	+* This software is available to you under a choice of one of two
	+* licenses. You may choose to be licensed under the terms of the GNU
	+* General Public License (GPL) Version 2, available from the file
	+* COPYING in the main directory of this source tree, or the
	+* OpenFabrics.org BSD license below:
	+*
	+* Redistribution and use in source and binary forms, with or
	+* without modification, are permitted provided that the following
	+* conditions are met:
	+*
	+* - Redistributions of source code must retain the above
	+* copyright notice, this list of conditions and the following
	+* disclaimer.
	+*
	+* - Redistributions in binary form must reproduce the above
	+* copyright notice, this list of conditions and the following
	+* disclaimer in the documentation and/or other materials
	+* provided with the distribution.
	+*
	+* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
	+* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
	+* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
	+* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
	+* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
	+* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
	+* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
	+* SOFTWARE.
	+*
	+*******************************************************************************/
	+/$FreeBSD$/
	+
	+#include "i40iw_osdep.h"
	+#include "i40iw_register.h"
	+#include "i40iw_status.h"
	+#include "i40iw_hmc.h"
	+
	+#include "i40iw_d.h"
	+#include "i40iw_type.h"
	+#include "i40iw_p.h"
	+
	+inline void
	+update_high_watermark(u64 create, u64 destroy, u64 *high_watermark)
	+{
	+ u64 _tmp = create - destroy;
	+ if (_tmp > *high_watermark)
	+ *high_watermark = _tmp;
	+}
	+
	+/**
	+ * i40iw_insert_wqe_hdr - write wqe header
	+ * @wqe: cqp wqe for header
	+ * @header: header for the cqp wqe
	+ */
	+void
	+i40iw_insert_wqe_hdr(u64 *wqe, u64 header)
	+{
	+ i40iw_wmb(); /* make sure WQE is populated before polarity
	+ * is set */
	+ set_64bit_val(wqe, I40IW_BYTE_24, header);
	+}
	+
	+void
	+i40iw_check_cqp_progress(struct i40iw_cqp_timeout cqp_timeout, struct i40iw_sc_dev dev)
	+{
	+ if (cqp_timeout->compl_cqp_cmds != dev->cqp_cmd_stats[OP_COMPLETED_COMMANDS]) {
	+ cqp_timeout->compl_cqp_cmds = dev->cqp_cmd_stats[OP_COMPLETED_COMMANDS];
	+ cqp_timeout->count = 0;
	+ } else {
	+ if (dev->cqp_cmd_stats[OP_REQUESTED_COMMANDS] != cqp_timeout->compl_cqp_cmds)
	+ cqp_timeout->count++;
	+ }
	+}
	+
	+/**
	+ * i40iw_get_cqp_reg_info - get head and tail for cqp using registers
	+ * @cqp: struct for cqp hw
	+ * @val: cqp tail register value
	+ * @tail:wqtail register value
	+ * @error: cqp processing err
	+ */
	+static inline void
	+i40iw_get_cqp_reg_info(struct i40iw_sc_cqp *cqp,
	+ u32 *val,
	+ u32 *tail,
	+ u32 *error)
	+{
	+ if (cqp->dev->is_pf) {
	+ *val = rd32(cqp->dev->hw, I40E_PFPE_CQPTAIL);
	+ tail = RS_32(val, I40E_PFPE_CQPTAIL_WQTAIL);
	+ error = RS_32(val, I40E_PFPE_CQPTAIL_CQP_OP_ERR);
	+ } else {
	+ *val = rd32(cqp->dev->hw, I40E_VFPE_CQPTAIL1);
	+ tail = RS_32(val, I40E_VFPE_CQPTAIL_WQTAIL);
	+ error = RS_32(val, I40E_VFPE_CQPTAIL_CQP_OP_ERR);
	+ }
	+}
	+
	+/**
	+ * i40iw_cqp_poll_registers - poll cqp registers
	+ * @cqp: struct for cqp hw
	+ * @tail:wqtail register value
	+ * @count: how many times to try for completion
	+ */
	+static enum i40iw_status_code
	+i40iw_cqp_poll_registers(
	+ struct i40iw_sc_cqp *cqp,
	+ u32 tail,
	+ u32 count)
	+{
	+ u32 i = 0;
	+ u32 newtail, error, val;
	+
	+ while (i < count) {
	+ i++;
	+ i40iw_get_cqp_reg_info(cqp, &val, &newtail, &error);
	+ if (error) {
	+ error = (cqp->dev->is_pf) ?
	+ rd32(cqp->dev->hw, I40E_PFPE_CQPERRCODES) :
	+ rd32(cqp->dev->hw, I40E_VFPE_CQPERRCODES1);
	+ i40iw_debug(cqp->dev, I40IW_DEBUG_CQP, "[%s] I40E_PFPE_CQPERRCODES error_code[x%08X]\n",
	+ __func__,
	+ error);
	+ return I40IW_ERR_CQP_COMPL_ERROR;
	+ }
	+ if (newtail != tail) {
	+ /* SUCCESS */
	+ I40IW_RING_MOVE_TAIL(cqp->sq_ring);
	+ cqp->dev->cqp_cmd_stats[OP_COMPLETED_COMMANDS]++;
	+ return 0;
	+ }
	+ i40iw_usec_delay(I40IW_SLEEP_COUNT);
	+ }
	+ return I40IW_ERR_TIMEOUT;
	+}
	+
	+/**
	+ * i40iw_sc_parse_fpm_commit_buf - parse fpm commit buffer
	+ * @buf: ptr to fpm commit buffer
	+ * @info: ptr to i40iw_hmc_obj_info struct
	+ * @sd: number of SDs for HMC objects
	+ *
	+ * parses fpm commit info and copy base value
	+ * of hmc objects in hmc_info
	+ */
	+static enum i40iw_status_code
	+i40iw_sc_parse_fpm_commit_buf(
	+ u64 *buf,
	+ struct i40iw_hmc_obj_info *info,
	+ u32 *sd)
	+{
	+ u64 temp;
	+ u64 size;
	+ u64 base = 0;
	+ u32 i, j;
	+ u32 k = 0;
	+
	+ /* copy base values in obj_info */
	+ for (i = I40IW_HMC_IW_QP, j = 0; i <= I40IW_HMC_IW_PBLE; i++, j += 8) {
	+ if ((i == I40IW_HMC_IW_SRQ) \|\|
	+ (i == I40IW_HMC_IW_FSIMC) \|\|
	+ (i == I40IW_HMC_IW_FSIAV)) {
	+ info[i].base = 0;
	+ info[i].cnt = 0;
	+ continue;
	+ }
	+ get_64bit_val(buf, j, &temp);
	+ info[i].base = RS_64_1(temp, 32) * 512;
	+ if (info[i].base > base) {
	+ base = info[i].base;
	+ k = i;
	+ }
	+ if (i == I40IW_HMC_IW_APBVT_ENTRY) {
	+ info[i].cnt = 1;
	+ continue;
	+ }
	+ if (i == I40IW_HMC_IW_QP)
	+ info[i].cnt = (u32)RS_64(temp, I40IW_QUERY_FPM_MAX_QPS);
	+ else if (i == I40IW_HMC_IW_CQ)
	+ info[i].cnt = (u32)RS_64(temp, I40IW_QUERY_FPM_MAX_CQS);
	+ else
	+ info[i].cnt = (u32)(temp);
	+ }
	+ size = info[k].cnt * info[k].size + info[k].base;
	+ if (size & 0x1FFFFF)
	+ sd = (u32)((size >> 21) + 1); / add 1 for remainder */
	+ else
	+ *sd = (u32)(size >> 21);
	+
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_sc_decode_fpm_query() - Decode a 64 bit value into max count and size
	+ * @buf: ptr to fpm query buffer
	+ * @buf_idx: index into buf
	+ * @info: ptr to i40iw_hmc_obj_info struct
	+ * @rsrc_idx: resource index into info
	+ *
	+ * Decode a 64 bit value from fpm query buffer into max count and size
	+ */
	+static u64
	+i40iw_sc_decode_fpm_query(u64 *buf,
	+ u32 buf_idx,
	+ struct i40iw_hmc_obj_info *obj_info,
	+ u32 rsrc_idx)
	+{
	+ u64 temp;
	+ u32 size;
	+
	+ get_64bit_val(buf, buf_idx, &temp);
	+ obj_info[rsrc_idx].max_cnt = (u32)temp;
	+ size = (u32)RS_64_1(temp, 32);
	+ obj_info[rsrc_idx].size = LS_64_1(1, size);
	+
	+ return temp;
	+}
	+
	+/**
	+ * i40iw_sc_parse_fpm_query_buf() - parses fpm query buffer
	+ * @buf: ptr to fpm query buffer
	+ * @info: ptr to i40iw_hmc_obj_info struct
	+ * @hmc_fpm_misc: ptr to fpm data
	+ *
	+ * parses fpm query buffer and copy max_cnt and
	+ * size value of hmc objects in hmc_info
	+ */
	+static enum i40iw_status_code
	+i40iw_sc_parse_fpm_query_buf(
	+ u64 *buf,
	+ struct i40iw_hmc_info *hmc_info,
	+ struct i40iw_hmc_fpm_misc *hmc_fpm_misc)
	+{
	+ struct i40iw_hmc_obj_info *obj_info;
	+ u64 temp;
	+ u32 size;
	+ u16 max_pe_sds;
	+
	+ obj_info = hmc_info->hmc_obj;
	+
	+ get_64bit_val(buf, 0, &temp);
	+ hmc_info->first_sd_index = (u16)RS_64(temp, I40IW_QUERY_FPM_FIRST_PE_SD_INDEX);
	+ max_pe_sds = (u16)RS_64(temp, I40IW_QUERY_FPM_MAX_PE_SDS);
	+
	+ /*
	+ * Reduce SD count for VFs by 1 to account for PBLE backing page
	+ * rounding
	+ */
	+ if (hmc_info->hmc_fn_id >= I40IW_FIRST_VF_FPM_ID)
	+ max_pe_sds--;
	+ hmc_fpm_misc->max_sds = max_pe_sds;
	+ hmc_info->sd_table.sd_cnt = max_pe_sds + hmc_info->first_sd_index;
	+
	+ get_64bit_val(buf, 8, &temp);
	+ obj_info[I40IW_HMC_IW_QP].max_cnt = (u32)RS_64(temp, I40IW_QUERY_FPM_MAX_QPS);
	+ size = (u32)RS_64_1(temp, 32);
	+ obj_info[I40IW_HMC_IW_QP].size = LS_64_1(1, size);
	+
	+ get_64bit_val(buf, 16, &temp);
	+ obj_info[I40IW_HMC_IW_CQ].max_cnt = (u32)RS_64(temp, I40IW_QUERY_FPM_MAX_CQS);
	+ size = (u32)RS_64_1(temp, 32);
	+ obj_info[I40IW_HMC_IW_CQ].size = LS_64_1(1, size);
	+
	+ i40iw_sc_decode_fpm_query(buf, 32, obj_info, I40IW_HMC_IW_HTE);
	+ i40iw_sc_decode_fpm_query(buf, 40, obj_info, I40IW_HMC_IW_ARP);
	+
	+ obj_info[I40IW_HMC_IW_APBVT_ENTRY].size = 8192;
	+ obj_info[I40IW_HMC_IW_APBVT_ENTRY].max_cnt = 1;
	+
	+ i40iw_sc_decode_fpm_query(buf, 48, obj_info, I40IW_HMC_IW_MR);
	+ i40iw_sc_decode_fpm_query(buf, 56, obj_info, I40IW_HMC_IW_XF);
	+
	+ get_64bit_val(buf, 64, &temp);
	+ obj_info[I40IW_HMC_IW_XFFL].max_cnt = (u32)temp;
	+ obj_info[I40IW_HMC_IW_XFFL].size = 4;
	+ hmc_fpm_misc->xf_block_size = RS_64(temp, I40IW_QUERY_FPM_XFBLOCKSIZE);
	+ if (!hmc_fpm_misc->xf_block_size)
	+ return I40IW_ERR_INVALID_SIZE;
	+
	+ i40iw_sc_decode_fpm_query(buf, 72, obj_info, I40IW_HMC_IW_Q1);
	+
	+ get_64bit_val(buf, 80, &temp);
	+ obj_info[I40IW_HMC_IW_Q1FL].max_cnt = (u32)temp;
	+ obj_info[I40IW_HMC_IW_Q1FL].size = 4;
	+ hmc_fpm_misc->q1_block_size = RS_64(temp, I40IW_QUERY_FPM_Q1BLOCKSIZE);
	+ if (!hmc_fpm_misc->q1_block_size)
	+ return I40IW_ERR_INVALID_SIZE;
	+
	+ i40iw_sc_decode_fpm_query(buf, 88, obj_info, I40IW_HMC_IW_TIMER);
	+
	+ get_64bit_val(buf, 112, &temp);
	+ obj_info[I40IW_HMC_IW_PBLE].max_cnt = (u32)temp;
	+ obj_info[I40IW_HMC_IW_PBLE].size = 8;
	+
	+ get_64bit_val(buf, 120, &temp);
	+ hmc_fpm_misc->max_ceqs = (u8)RS_64(temp, I40IW_QUERY_FPM_MAX_CEQS);
	+ hmc_fpm_misc->ht_multiplier = RS_64(temp, I40IW_QUERY_FPM_HTMULTIPLIER);
	+ hmc_fpm_misc->timer_bucket = RS_64(temp, I40IW_QUERY_FPM_TIMERBUCKET);
	+
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_sc_pd_init - initialize sc pd struct
	+ * @dev: sc device struct
	+ * @pd: sc pd ptr
	+ * @pd_id: pd_id for allocated pd
	+ * @abi_ver: ABI version from user context, -1 if not valid
	+ */
	+static void
	+i40iw_sc_pd_init(struct i40iw_sc_dev *dev,
	+ struct i40iw_sc_pd *pd,
	+ u16 pd_id)
	+{
	+ pd->size = sizeof(*pd);
	+ pd->pd_id = pd_id;
	+ pd->dev = dev;
	+}
	+
	+/**
	+ * i40iw_get_encoded_wqe_size - given wq size, returns hardware encoded size
	+ * @wqsize: size of the wq (sq, rq, srq) to encoded_size
	+ * @cqpsq: encoded size for sq for cqp as its encoded size is 1+ other wq's
	+ */
	+u8
	+i40iw_get_encoded_wqe_size(u32 wqsize, bool cqpsq)
	+{
	+ u8 encoded_size = 0;
	+
	+ /*
	+ * cqp sq's hw coded value starts from 1 for size of 4 while it
	+ * starts from 0 for qp' wq's.
	+ */
	+ if (cqpsq)
	+ encoded_size = 1;
	+ wqsize >>= 2;
	+ while (wqsize >>= 1)
	+ encoded_size++;
	+ return encoded_size;
	+}
	+
	+/**
	+ * i40iw_sc_cqp_init - Initialize buffers for a control Queue Pair
	+ * @cqp: IWARP control queue pair pointer
	+ * @info: IWARP control queue pair init info pointer
	+ *
	+ * Initializes the object and context buffers for a control Queue Pair.
	+ */
	+static enum i40iw_status_code
	+i40iw_sc_cqp_init(struct i40iw_sc_cqp *cqp,
	+ struct i40iw_cqp_init_info *info)
	+{
	+ u8 hw_sq_size;
	+
	+ if ((info->sq_size > I40IW_CQP_SW_SQSIZE_2048) \|\|
	+ (info->sq_size < I40IW_CQP_SW_SQSIZE_4) \|\|
	+ ((info->sq_size & (info->sq_size - 1))))
	+ return I40IW_ERR_INVALID_SIZE;
	+
	+ hw_sq_size = i40iw_get_encoded_wqe_size(info->sq_size, TRUE);
	+ cqp->size = sizeof(*cqp);
	+ cqp->sq_size = info->sq_size;
	+ cqp->hw_sq_size = hw_sq_size;
	+ cqp->sq_base = info->sq;
	+ cqp->host_ctx = info->host_ctx;
	+ cqp->sq_pa = info->sq_pa;
	+ cqp->host_ctx_pa = info->host_ctx_pa;
	+ cqp->dev = info->dev;
	+ cqp->struct_ver = info->struct_ver;
	+ cqp->scratch_array = info->scratch_array;
	+ cqp->polarity = 0;
	+ cqp->en_datacenter_tcp = info->en_datacenter_tcp;
	+ cqp->enabled_vf_count = info->enabled_vf_count;
	+ cqp->hmc_profile = info->hmc_profile;
	+ info->dev->cqp = cqp;
	+
	+ I40IW_RING_INIT(cqp->sq_ring, cqp->sq_size);
	+ cqp->dev->cqp_cmd_stats[OP_REQUESTED_COMMANDS] = 0;
	+ cqp->dev->cqp_cmd_stats[OP_COMPLETED_COMMANDS] = 0;
	+ INIT_LIST_HEAD(&cqp->dev->cqp_cmd_head); /* for the cqp commands
	+ * backlog. */
	+
	+ if (cqp->dev->is_pf) {
	+ wr32(cqp->dev->hw, I40E_PFPE_CQPTAIL, 0);
	+ wr32(cqp->dev->hw, I40E_PFPE_CQPDB, 0);
	+ } else {
	+ wr32(cqp->dev->hw, I40E_VFPE_CQPTAIL1, 0);
	+ wr32(cqp->dev->hw, I40E_VFPE_CQPDB1, 0);
	+ }
	+ i40iw_debug(cqp->dev, I40IW_DEBUG_WQE,
	+ "%s: sq_size[%04d] hw_sq_size[%04d] sq_base[%p] sq_pa[%lxh] cqp[%p] polarity[x%04X]\n",
	+ __func__, cqp->sq_size, cqp->hw_sq_size,
	+ cqp->sq_base, cqp->sq_pa, cqp, cqp->polarity);
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_sc_cqp_create - create cqp during bringup
	+ * @cqp: struct for cqp hw
	+ * @disable_pfpdus: if pfpdu to be disabled
	+ * @maj_err: If error, major err number
	+ * @min_err: If error, minor err number
	+ */
	+static enum i40iw_status_code
	+i40iw_sc_cqp_create(struct i40iw_sc_cqp *cqp,
	+ u16 *maj_err,
	+ u16 *min_err)
	+{
	+ u64 temp;
	+ u32 cnt = 0, p1, p2, val = 0, err_code;
	+ enum i40iw_status_code ret_code;
	+
	+ ret_code = i40iw_allocate_dma_mem(cqp->dev->hw,
	+ &cqp->sdbuf,
	+ 128 * cqp->sq_size,
	+ I40IW_SD_BUF_ALIGNMENT);
	+
	+ if (ret_code)
	+ goto exit;
	+
	+ temp = LS_64(cqp->hw_sq_size, I40IW_CQPHC_SQSIZE) \|
	+ LS_64(cqp->struct_ver, I40IW_CQPHC_SVER);
	+
	+ set_64bit_val(cqp->host_ctx, I40IW_BYTE_0, temp);
	+ set_64bit_val(cqp->host_ctx, I40IW_BYTE_8, cqp->sq_pa);
	+ temp = LS_64(cqp->enabled_vf_count, I40IW_CQPHC_ENABLED_VFS) \|
	+ LS_64(cqp->hmc_profile, I40IW_CQPHC_HMC_PROFILE);
	+ set_64bit_val(cqp->host_ctx, I40IW_BYTE_16, temp);
	+ set_64bit_val(cqp->host_ctx, I40IW_BYTE_24, (uintptr_t)cqp);
	+ set_64bit_val(cqp->host_ctx, I40IW_BYTE_32, 0);
	+ set_64bit_val(cqp->host_ctx, I40IW_BYTE_40, 0);
	+ set_64bit_val(cqp->host_ctx, I40IW_BYTE_48, 0);
	+ set_64bit_val(cqp->host_ctx, I40IW_BYTE_56, 0);
	+
	+ i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "CQP_HOST_CTX",
	+ cqp->host_ctx, I40IW_CQP_CTX_SIZE * 8);
	+
	+ p1 = RS_32_1(cqp->host_ctx_pa, 32);
	+ p2 = (u32)cqp->host_ctx_pa;
	+
	+ if (cqp->dev->is_pf) {
	+ wr32(cqp->dev->hw, I40E_PFPE_CCQPHIGH, p1);
	+ wr32(cqp->dev->hw, I40E_PFPE_CCQPLOW, p2);
	+ } else {
	+ wr32(cqp->dev->hw, I40E_VFPE_CCQPHIGH1, p1);
	+ wr32(cqp->dev->hw, I40E_VFPE_CCQPLOW1, p2);
	+ }
	+ do {
	+ if (cnt++ > I40IW_DONE_COUNT) {
	+ i40iw_free_dma_mem(cqp->dev->hw, &cqp->sdbuf);
	+ ret_code = I40IW_ERR_TIMEOUT;
	+ /*
	+ * read PFPE_CQPERRORCODES register to get the minor
	+ * and major error code
	+ */
	+ if (cqp->dev->is_pf)
	+ err_code = rd32(cqp->dev->hw, I40E_PFPE_CQPERRCODES);
	+ else
	+ err_code = rd32(cqp->dev->hw, I40E_VFPE_CQPERRCODES1);
	+ *min_err = RS_32(err_code, I40E_PFPE_CQPERRCODES_CQP_MINOR_CODE);
	+ *maj_err = RS_32(err_code, I40E_PFPE_CQPERRCODES_CQP_MAJOR_CODE);
	+ goto exit;
	+ }
	+ i40iw_usec_delay(I40IW_SLEEP_COUNT);
	+ if (cqp->dev->is_pf)
	+ val = rd32(cqp->dev->hw, I40E_PFPE_CCQPSTATUS);
	+ else
	+ val = rd32(cqp->dev->hw, I40E_VFPE_CCQPSTATUS1);
	+ } while (!val);
	+
	+exit:
	+ if (!ret_code)
	+ cqp->process_cqp_sds = i40iw_update_sds_noccq;
	+ return ret_code;
	+}
	+
	+/**
	+ * i40iw_sc_cqp_post_sq - post of cqp's sq
	+ * @cqp: struct for cqp hw
	+ */
	+void
	+i40iw_sc_cqp_post_sq(struct i40iw_sc_cqp *cqp)
	+{
	+ if (cqp->dev->is_pf)
	+ wr32(cqp->dev->hw, I40E_PFPE_CQPDB, I40IW_RING_GETCURRENT_HEAD(cqp->sq_ring));
	+ else
	+ wr32(cqp->dev->hw, I40E_VFPE_CQPDB1, I40IW_RING_GETCURRENT_HEAD(cqp->sq_ring));
	+
	+ i40iw_debug(cqp->dev,
	+ I40IW_DEBUG_WQE,
	+ "%s: HEAD_TAIL[%04d,%04d,%04d]\n",
	+ __func__,
	+ cqp->sq_ring.head,
	+ cqp->sq_ring.tail,
	+ cqp->sq_ring.size);
	+}
	+
	+/**
	+ * i40iw_sc_cqp_get_next_send_wqe_idx - get next wqe on cqp sq and pass back index
	+ * @cqp: CQP HW structure
	+ * @scratch: private data for CQP WQE
	+ * @wqe_idx: WQE index of CQP SQ
	+ */
	+static u64 *
	+i40iw_sc_cqp_get_next_send_wqe_idx(struct i40iw_sc_cqp cqp, u64 scratch, u32 wqe_idx)
	+{
	+ u64 *wqe = NULL;
	+ enum i40iw_status_code ret_code;
	+
	+ if (I40IW_RING_FULL_ERR(cqp->sq_ring)) {
	+ i40iw_debug(cqp->dev,
	+ I40IW_DEBUG_WQE,
	+ "%s: ring is full head %x tail %x size %x\n",
	+ __func__,
	+ cqp->sq_ring.head,
	+ cqp->sq_ring.tail,
	+ cqp->sq_ring.size);
	+ return NULL;
	+ }
	+ I40IW_ATOMIC_RING_MOVE_HEAD(cqp->sq_ring, *wqe_idx, ret_code);
	+ cqp->dev->cqp_cmd_stats[OP_REQUESTED_COMMANDS]++;
	+ if (ret_code)
	+ return NULL;
	+ if (!*wqe_idx)
	+ cqp->polarity = !cqp->polarity;
	+
	+ wqe = cqp->sq_base[*wqe_idx].elem;
	+ cqp->scratch_array[*wqe_idx] = scratch;
	+ I40IW_CQP_INIT_WQE(wqe);
	+
	+ return wqe;
	+}
	+/**
	+ * i40iw_sc_cqp_get_next_send_wqe - get next wqe on cqp sq
	+ * @cqp: struct for cqp hw
	+ * @scratch: private data for CQP WQE
	+ */
	+u64 *
	+i40iw_sc_cqp_get_next_send_wqe(struct i40iw_sc_cqp *cqp, u64 scratch)
	+{
	+ u32 wqe_idx;
	+
	+ return i40iw_sc_cqp_get_next_send_wqe_idx(cqp, scratch, &wqe_idx);
	+}
	+
	+/**
	+ * i40iw_sc_cqp_destroy - destroy cqp during close
	+ * @cqp: struct for cqp hw
	+ */
	+static enum i40iw_status_code
	+i40iw_sc_cqp_destroy(struct i40iw_sc_cqp *cqp)
	+{
	+ u32 cnt = 0, val = 1;
	+ enum i40iw_status_code ret_code = 0;
	+ u32 cqpstat_addr;
	+
	+ if (cqp->dev->is_pf) {
	+ wr32(cqp->dev->hw, I40E_PFPE_CCQPHIGH, 0);
	+ wr32(cqp->dev->hw, I40E_PFPE_CCQPLOW, 0);
	+ cqpstat_addr = I40E_PFPE_CCQPSTATUS;
	+ } else {
	+ wr32(cqp->dev->hw, I40E_VFPE_CCQPHIGH1, 0);
	+ wr32(cqp->dev->hw, I40E_VFPE_CCQPLOW1, 0);
	+ cqpstat_addr = I40E_VFPE_CCQPSTATUS1;
	+ }
	+ do {
	+ if (cnt++ > I40IW_DONE_COUNT) {
	+ ret_code = I40IW_ERR_TIMEOUT;
	+ break;
	+ }
	+ i40iw_usec_delay(I40IW_SLEEP_COUNT);
	+ val = rd32(cqp->dev->hw, cqpstat_addr);
	+ } while (val);
	+
	+ i40iw_free_dma_mem(cqp->dev->hw, &cqp->sdbuf);
	+ return ret_code;
	+}
	+
	+/**
	+ * i40iw_sc_ccq_arm - enable intr for control cq
	+ * @ccq: ccq sc struct
	+ */
	+static void
	+i40iw_sc_ccq_arm(struct i40iw_sc_cq *ccq)
	+{
	+ u64 temp_val;
	+ u16 sw_cq_sel;
	+ u8 arm_next_se;
	+ u8 arm_seq_num;
	+
	+ /* write to cq doorbell shadow area */
	+ /* arm next se should always be zero */
	+ get_64bit_val(ccq->cq_uk.shadow_area, I40IW_BYTE_32, &temp_val);
	+
	+ sw_cq_sel = (u16)RS_64(temp_val, I40IW_CQ_DBSA_SW_CQ_SELECT);
	+ arm_next_se = (u8)RS_64(temp_val, I40IW_CQ_DBSA_ARM_NEXT_SE);
	+
	+ arm_seq_num = (u8)RS_64(temp_val, I40IW_CQ_DBSA_ARM_SEQ_NUM);
	+ arm_seq_num++;
	+
	+ temp_val = LS_64(arm_seq_num, I40IW_CQ_DBSA_ARM_SEQ_NUM) \|
	+ LS_64(sw_cq_sel, I40IW_CQ_DBSA_SW_CQ_SELECT) \|
	+ LS_64(arm_next_se, I40IW_CQ_DBSA_ARM_NEXT_SE) \|
	+ LS_64(1, I40IW_CQ_DBSA_ARM_NEXT);
	+
	+ set_64bit_val(ccq->cq_uk.shadow_area, I40IW_BYTE_32, temp_val);
	+
	+ i40iw_wmb(); /* make sure shadow area is updated before
	+ * arming */
	+
	+ if (ccq->dev->is_pf)
	+ wr32(ccq->dev->hw, I40E_PFPE_CQARM, ccq->cq_uk.cq_id);
	+ else
	+ wr32(ccq->dev->hw, I40E_VFPE_CQARM1, ccq->cq_uk.cq_id);
	+}
	+
	+/**
	+ * i40iw_sc_ccq_get_cqe_info - get ccq's cq entry
	+ * @ccq: ccq sc struct
	+ * @info: completion q entry to return
	+ */
	+static enum i40iw_status_code
	+i40iw_sc_ccq_get_cqe_info(
	+ struct i40iw_sc_cq *ccq,
	+ struct i40iw_ccq_cqe_info *info)
	+{
	+ u64 qp_ctx, temp, temp1;
	+ u64 *cqe;
	+ struct i40iw_sc_cqp *cqp;
	+ u32 wqe_idx;
	+ u32 error;
	+ u8 polarity;
	+ enum i40iw_status_code ret_code = 0;
	+
	+ if (ccq->cq_uk.avoid_mem_cflct)
	+ cqe = (u64 *)I40IW_GET_CURRENT_EXTENDED_CQ_ELEMENT(&ccq->cq_uk);
	+ else
	+ cqe = (u64 *)I40IW_GET_CURRENT_CQ_ELEMENT(&ccq->cq_uk);
	+
	+ get_64bit_val(cqe, I40IW_BYTE_24, &temp);
	+ polarity = (u8)RS_64(temp, I40IW_CQ_VALID);
	+ if (polarity != ccq->cq_uk.polarity)
	+ return I40IW_ERR_QUEUE_EMPTY;
	+
	+ get_64bit_val(cqe, I40IW_BYTE_8, &qp_ctx);
	+ cqp = (struct i40iw_sc_cqp *)(i40iw_uintptr)qp_ctx;
	+ info->error = (bool)RS_64(temp, I40IW_CQ_ERROR);
	+ info->min_err_code = (u16)RS_64(temp, I40IW_CQ_MINERR);
	+ if (info->error) {
	+ info->maj_err_code = (u16)RS_64(temp, I40IW_CQ_MAJERR);
	+ info->min_err_code = (u16)RS_64(temp, I40IW_CQ_MINERR);
	+ error = (cqp->dev->is_pf) ?
	+ rd32(cqp->dev->hw, I40E_PFPE_CQPERRCODES) :
	+ rd32(cqp->dev->hw, I40E_VFPE_CQPERRCODES1);
	+ i40iw_debug(cqp->dev, I40IW_DEBUG_CQP, "[%s] I40E_PFPE_CQPERRCODES error_code[x%08X]\n",
	+ __func__,
	+ error);
	+ }
	+ wqe_idx = (u32)RS_64(temp, I40IW_CQ_WQEIDX);
	+ info->scratch = cqp->scratch_array[wqe_idx];
	+
	+ get_64bit_val(cqe, I40IW_BYTE_16, &temp1);
	+ info->op_ret_val = (u32)RS_64(temp1, I40IW_CCQ_OPRETVAL);
	+ get_64bit_val(cqp->sq_base[wqe_idx].elem, I40IW_BYTE_24, &temp1);
	+ info->op_code = (u8)RS_64(temp1, I40IW_CQPSQ_OPCODE);
	+ info->cqp = cqp;
	+
	+ /* move the head for cq */
	+ I40IW_RING_MOVE_HEAD(ccq->cq_uk.cq_ring, ret_code);
	+ if (I40IW_RING_GETCURRENT_HEAD(ccq->cq_uk.cq_ring) == 0)
	+ ccq->cq_uk.polarity ^= 1;
	+
	+ /* update cq tail in cq shadow memory also */
	+ I40IW_RING_MOVE_TAIL(ccq->cq_uk.cq_ring);
	+ set_64bit_val(ccq->cq_uk.shadow_area,
	+ I40IW_BYTE_0,
	+ I40IW_RING_GETCURRENT_HEAD(ccq->cq_uk.cq_ring));
	+ i40iw_wmb(); /* write shadow area before tail */
	+ I40IW_RING_MOVE_TAIL(cqp->sq_ring);
	+ ccq->dev->cqp_cmd_stats[OP_COMPLETED_COMMANDS]++;
	+
	+ return ret_code;
	+}
	+
	+/**
	+ * i40iw_sc_poll_for_cqp_op_done - Waits for last write to complete in CQP SQ
	+ * @cqp: struct for cqp hw
	+ * @op_code: cqp opcode for completion
	+ * @info: completion q entry to return
	+ */
	+static enum i40iw_status_code
	+i40iw_sc_poll_for_cqp_op_done(
	+ struct i40iw_sc_cqp *cqp,
	+ u8 op_code,
	+ struct i40iw_ccq_cqe_info *compl_info)
	+{
	+ struct i40iw_ccq_cqe_info info;
	+ struct i40iw_sc_cq *ccq;
	+ enum i40iw_status_code ret_code = 0;
	+ u32 cnt = 0;
	+
	+ memset(&info, 0, sizeof(info));
	+ ccq = cqp->dev->ccq;
	+ while (1) {
	+ if (cnt++ > I40IW_DONE_COUNT)
	+ return I40IW_ERR_TIMEOUT;
	+
	+ if (i40iw_sc_ccq_get_cqe_info(ccq, &info)) {
	+ i40iw_usec_delay(I40IW_SLEEP_COUNT);
	+ continue;
	+ }
	+ if (info.error) {
	+ ret_code = I40IW_ERR_CQP_COMPL_ERROR;
	+ break;
	+ }
	+ /* check if opcode is cq create */
	+ if (op_code != info.op_code) {
	+ i40iw_debug(cqp->dev, I40IW_DEBUG_WQE,
	+ "%s: opcode mismatch for my op code 0x%x, returned opcode %x\n",
	+ __func__, op_code, info.op_code);
	+ }
	+ /* success, exit out of the loop */
	+ if (op_code == info.op_code)
	+ break;
	+ }
	+
	+ if (compl_info)
	+ i40iw_memcpy(compl_info, &info, sizeof(*compl_info));
	+
	+ return ret_code;
	+}
	+
	+/**
	+ * i40iw_sc_manage_push_page - Handle push page
	+ * @cqp: struct for cqp hw
	+ * @info: push page info
	+ * @scratch: u64 saved to be used during cqp completion
	+ * @post_sq: flag for cqp db to ring
	+ */
	+static enum i40iw_status_code
	+i40iw_sc_manage_push_page(
	+ struct i40iw_sc_cqp *cqp,
	+ struct i40iw_cqp_manage_push_page_info *info,
	+ u64 scratch,
	+ bool post_sq)
	+{
	+ u64 *wqe;
	+ u64 header;
	+
	+ if (info->push_idx >= I40IW_MAX_PUSH_PAGE_COUNT)
	+ return I40IW_ERR_INVALID_PUSH_PAGE_INDEX;
	+
	+ wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
	+ if (!wqe)
	+ return I40IW_ERR_RING_FULL;
	+
	+ set_64bit_val(wqe, I40IW_BYTE_16, info->qs_handle);
	+
	+ header = LS_64(info->push_idx, I40IW_CQPSQ_MPP_PPIDX) \|
	+ LS_64(I40IW_CQP_OP_MANAGE_PUSH_PAGES, I40IW_CQPSQ_OPCODE) \|
	+ LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID) \|
	+ LS_64(info->free_page, I40IW_CQPSQ_MPP_FREE_PAGE);
	+
	+ i40iw_insert_wqe_hdr(wqe, header);
	+
	+ i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "MANAGE_PUSH_PAGES WQE",
	+ wqe, I40IW_CQP_WQE_SIZE * 8);
	+
	+ if (post_sq)
	+ i40iw_sc_cqp_post_sq(cqp);
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_sc_manage_hmc_pm_func_table - manage of function table
	+ * @cqp: struct for cqp hw
	+ * @scratch: u64 saved to be used during cqp completion
	+ * @vf_index: vf index for cqp
	+ * @free_pm_fcn: function number
	+ * @post_sq: flag for cqp db to ring
	+ */
	+static enum i40iw_status_code
	+i40iw_sc_manage_hmc_pm_func_table(
	+ struct i40iw_sc_cqp *cqp,
	+ u64 scratch,
	+ u8 vf_index,
	+ bool free_pm_fcn,
	+ bool post_sq)
	+{
	+ u64 *wqe;
	+ u64 header;
	+
	+ wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
	+ if (!wqe)
	+ return I40IW_ERR_RING_FULL;
	+
	+ header = LS_64(vf_index, I40IW_CQPSQ_MHMC_VFIDX) \|
	+ LS_64(I40IW_CQP_OP_MANAGE_HMC_PM_FUNC_TABLE, I40IW_CQPSQ_OPCODE) \|
	+ LS_64(free_pm_fcn, I40IW_CQPSQ_MHMC_FREEPMFN) \|
	+ LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
	+
	+ i40iw_insert_wqe_hdr(wqe, header);
	+ i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "MANAGE_HMC_PM_FUNC_TABLE WQE",
	+ wqe, I40IW_CQP_WQE_SIZE * 8);
	+ if (post_sq)
	+ i40iw_sc_cqp_post_sq(cqp);
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_sc_manage_hmc_pm_func_table_done - wait for cqp wqe completion for function table
	+ * @cqp: struct for cqp hw
	+ */
	+static enum i40iw_status_code
	+i40iw_sc_manage_hmc_pm_func_table_done(struct i40iw_sc_cqp *cqp)
	+{
	+ return i40iw_sc_poll_for_cqp_op_done(cqp, I40IW_CQP_OP_MANAGE_HMC_PM_FUNC_TABLE, NULL);
	+}
	+
	+/**
	+ * i40iw_sc_commit_fpm_values_done - wait for cqp eqe completion for fpm commit
	+ * @cqp: struct for cqp hw
	+ */
	+static enum i40iw_status_code
	+i40iw_sc_commit_fpm_values_done(struct i40iw_sc_cqp *cqp)
	+{
	+ return i40iw_sc_poll_for_cqp_op_done(cqp, I40IW_CQP_OP_COMMIT_FPM_VALUES, NULL);
	+}
	+
	+/**
	+ * i40iw_sc_commit_fpm_values - cqp wqe for commit fpm values
	+ * @cqp: struct for cqp hw
	+ * @scratch: u64 saved to be used during cqp completion
	+ * @hmc_fn_id: hmc function id
	+ * @commit_fpm_mem; Memory for fpm values
	+ * @post_sq: flag for cqp db to ring
	+ * @wait_type: poll ccq or cqp registers for cqp completion
	+ */
	+static enum i40iw_status_code
	+i40iw_sc_commit_fpm_values(
	+ struct i40iw_sc_cqp *cqp,
	+ u64 scratch,
	+ u8 hmc_fn_id,
	+ struct i40iw_dma_mem *commit_fpm_mem,
	+ bool post_sq,
	+ u8 wait_type)
	+{
	+ u64 *wqe;
	+ u64 header;
	+ u32 tail, val, error;
	+ enum i40iw_status_code ret_code = 0;
	+
	+ wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
	+ if (!wqe)
	+ return I40IW_ERR_RING_FULL;
	+
	+ set_64bit_val(wqe, I40IW_BYTE_16, hmc_fn_id);
	+ set_64bit_val(wqe, I40IW_BYTE_32, commit_fpm_mem->pa);
	+
	+ header = LS_64(I40IW_CQP_OP_COMMIT_FPM_VALUES, I40IW_CQPSQ_OPCODE) \|
	+ LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
	+
	+ i40iw_insert_wqe_hdr(wqe, header);
	+
	+ i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "COMMIT_FPM_VALUES WQE",
	+ wqe, I40IW_CQP_WQE_SIZE * 8);
	+
	+ i40iw_get_cqp_reg_info(cqp, &val, &tail, &error);
	+ if (error)
	+ return I40IW_ERR_CQP_COMPL_ERROR;
	+
	+ if (post_sq) {
	+ i40iw_sc_cqp_post_sq(cqp);
	+
	+ if (wait_type == I40IW_CQP_WAIT_POLL_REGS)
	+ ret_code = i40iw_cqp_poll_registers(cqp, tail, I40IW_DONE_COUNT);
	+ else if (wait_type == I40IW_CQP_WAIT_POLL_CQ)
	+ ret_code = i40iw_sc_commit_fpm_values_done(cqp);
	+ }
	+ return ret_code;
	+}
	+
	+/**
	+ * i40iw_sc_query_fpm_values_done - poll for cqp wqe completion for query fpm
	+ * @cqp: struct for cqp hw
	+ */
	+static enum i40iw_status_code
	+i40iw_sc_query_fpm_values_done(struct i40iw_sc_cqp *cqp)
	+{
	+ return i40iw_sc_poll_for_cqp_op_done(cqp, I40IW_CQP_OP_QUERY_FPM_VALUES, NULL);
	+}
	+
	+/**
	+ * i40iw_sc_query_fpm_values - cqp wqe query fpm values
	+ * @cqp: struct for cqp hw
	+ * @scratch: u64 saved to be used during cqp completion
	+ * @hmc_fn_id: hmc function id
	+ * @query_fpm_mem: memory for return fpm values
	+ * @post_sq: flag for cqp db to ring
	+ * @wait_type: poll ccq or cqp registers for cqp completion
	+ */
	+static enum i40iw_status_code
	+i40iw_sc_query_fpm_values(
	+ struct i40iw_sc_cqp *cqp,
	+ u64 scratch,
	+ u8 hmc_fn_id,
	+ struct i40iw_dma_mem *query_fpm_mem,
	+ bool post_sq,
	+ u8 wait_type)
	+{
	+ u64 *wqe;
	+ u64 header;
	+ u32 tail, val, error;
	+ enum i40iw_status_code ret_code = 0;
	+
	+ wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
	+ if (!wqe)
	+ return I40IW_ERR_RING_FULL;
	+
	+ set_64bit_val(wqe, I40IW_BYTE_16, hmc_fn_id);
	+ set_64bit_val(wqe, I40IW_BYTE_32, query_fpm_mem->pa);
	+
	+ header = LS_64(I40IW_CQP_OP_QUERY_FPM_VALUES, I40IW_CQPSQ_OPCODE) \|
	+ LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
	+
	+ i40iw_insert_wqe_hdr(wqe, header);
	+
	+ i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "QUERY_FPM WQE",
	+ wqe, I40IW_CQP_WQE_SIZE * 8);
	+
	+ /* read the tail from CQP_TAIL register */
	+ i40iw_get_cqp_reg_info(cqp, &val, &tail, &error);
	+
	+ if (error)
	+ return I40IW_ERR_CQP_COMPL_ERROR;
	+
	+ if (post_sq) {
	+ i40iw_sc_cqp_post_sq(cqp);
	+ if (wait_type == I40IW_CQP_WAIT_POLL_REGS)
	+ ret_code = i40iw_cqp_poll_registers(cqp, tail, I40IW_DONE_COUNT);
	+ else if (wait_type == I40IW_CQP_WAIT_POLL_CQ)
	+ ret_code = i40iw_sc_query_fpm_values_done(cqp);
	+ }
	+ return ret_code;
	+}
	+
	+/**
	+ * i40iw_sc_add_arp_cache_entry - cqp wqe add arp cache entry
	+ * @cqp: struct for cqp hw
	+ * @info: arp entry information
	+ * @scratch: u64 saved to be used during cqp completion
	+ * @post_sq: flag for cqp db to ring
	+ */
	+static enum i40iw_status_code
	+i40iw_sc_add_arp_cache_entry(
	+ struct i40iw_sc_cqp *cqp,
	+ struct i40iw_add_arp_cache_entry_info *info,
	+ u64 scratch,
	+ bool post_sq)
	+{
	+ u64 *wqe;
	+ u64 temp, header;
	+
	+ wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
	+ if (!wqe)
	+ return I40IW_ERR_RING_FULL;
	+ set_64bit_val(wqe, I40IW_BYTE_8, info->reach_max);
	+
	+ temp = info->mac_addr[5] \|
	+ LS_64_1(info->mac_addr[4], 8) \|
	+ LS_64_1(info->mac_addr[3], 16) \|
	+ LS_64_1(info->mac_addr[2], 24) \|
	+ LS_64_1(info->mac_addr[1], 32) \|
	+ LS_64_1(info->mac_addr[0], 40);
	+
	+ set_64bit_val(wqe, I40IW_BYTE_16, temp);
	+
	+ header = info->arp_index \|
	+ LS_64(I40IW_CQP_OP_MANAGE_ARP, I40IW_CQPSQ_OPCODE) \|
	+ LS_64((info->permanent ? 1 : 0), I40IW_CQPSQ_MAT_PERMANENT) \|
	+ LS_64(1, I40IW_CQPSQ_MAT_ENTRYVALID) \|
	+ LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
	+
	+ i40iw_insert_wqe_hdr(wqe, header);
	+
	+ i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "ARP_CACHE_ENTRY WQE",
	+ wqe, I40IW_CQP_WQE_SIZE * 8);
	+
	+ if (post_sq)
	+ i40iw_sc_cqp_post_sq(cqp);
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_sc_del_arp_cache_entry - dele arp cache entry
	+ * @cqp: struct for cqp hw
	+ * @scratch: u64 saved to be used during cqp completion
	+ * @arp_index: arp index to delete arp entry
	+ * @post_sq: flag for cqp db to ring
	+ */
	+static enum i40iw_status_code
	+i40iw_sc_del_arp_cache_entry(
	+ struct i40iw_sc_cqp *cqp,
	+ u64 scratch,
	+ u16 arp_index,
	+ bool post_sq)
	+{
	+ u64 *wqe;
	+ u64 header;
	+
	+ wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
	+ if (!wqe)
	+ return I40IW_ERR_RING_FULL;
	+
	+ header = arp_index \|
	+ LS_64(I40IW_CQP_OP_MANAGE_ARP, I40IW_CQPSQ_OPCODE) \|
	+ LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
	+ i40iw_insert_wqe_hdr(wqe, header);
	+
	+ i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "ARP_CACHE_DEL_ENTRY WQE",
	+ wqe, I40IW_CQP_WQE_SIZE * 8);
	+
	+ if (post_sq)
	+ i40iw_sc_cqp_post_sq(cqp);
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_sc_query_arp_cache_entry - cqp wqe to query arp and arp index
	+ * @cqp: struct for cqp hw
	+ * @scratch: u64 saved to be used during cqp completion
	+ * @arp_index: arp index to delete arp entry
	+ * @post_sq: flag for cqp db to ring
	+ */
	+static enum i40iw_status_code
	+i40iw_sc_query_arp_cache_entry(
	+ struct i40iw_sc_cqp *cqp,
	+ u64 scratch,
	+ u16 arp_index,
	+ bool post_sq)
	+{
	+ u64 *wqe;
	+ u64 header;
	+
	+ wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
	+ if (!wqe)
	+ return I40IW_ERR_RING_FULL;
	+
	+ header = arp_index \|
	+ LS_64(I40IW_CQP_OP_MANAGE_ARP, I40IW_CQPSQ_OPCODE) \|
	+ LS_64(1, I40IW_CQPSQ_MAT_QUERY) \|
	+ LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
	+
	+ i40iw_insert_wqe_hdr(wqe, header);
	+
	+ i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "QUERY_ARP_CACHE_ENTRY WQE",
	+ wqe, I40IW_CQP_WQE_SIZE * 8);
	+
	+ if (post_sq)
	+ i40iw_sc_cqp_post_sq(cqp);
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_sc_manage_apbvt_entry - for adding and deleting apbvt entries
	+ * @cqp: struct for cqp hw
	+ * @info: info for apbvt entry to add or delete
	+ * @scratch: u64 saved to be used during cqp completion
	+ * @post_sq: flag for cqp db to ring
	+ */
	+static enum i40iw_status_code
	+i40iw_sc_manage_apbvt_entry(
	+ struct i40iw_sc_cqp *cqp,
	+ struct i40iw_apbvt_info *info,
	+ u64 scratch,
	+ bool post_sq)
	+{
	+ u64 *wqe;
	+ u64 header;
	+
	+ wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
	+ if (!wqe)
	+ return I40IW_ERR_RING_FULL;
	+
	+ set_64bit_val(wqe, I40IW_BYTE_16, info->port);
	+
	+ header = LS_64(I40IW_CQP_OP_MANAGE_APBVT, I40IW_CQPSQ_OPCODE) \|
	+ LS_64(info->add, I40IW_CQPSQ_MAPT_ADDPORT) \|
	+ LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
	+
	+ i40iw_insert_wqe_hdr(wqe, header);
	+
	+ i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "MANAGE_APBVT WQE",
	+ wqe, I40IW_CQP_WQE_SIZE * 8);
	+
	+ if (post_sq)
	+ i40iw_sc_cqp_post_sq(cqp);
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_sc_manage_qhash_table_entry - manage quad hash entries
	+ * @cqp: struct for cqp hw
	+ * @info: info for quad hash to manage
	+ * @scratch: u64 saved to be used during cqp completion
	+ * @post_sq: flag for cqp db to ring
	+ *
	+ * This is called before connection establishment is started. For passive connections, when
	+ * listener is created, it will call with entry type of I40IW_QHASH_TYPE_TCP_SYN with local
	+ * ip address and tcp port. When SYN is received (passive connections) or
	+ * sent (active connections), this routine is called with entry type of
	+ * I40IW_QHASH_TYPE_TCP_ESTABLISHED and quad is passed in info.
	+ *
	+ * When iwarp connection is done and its state moves to RTS, the quad hash entry in
	+ * the hardware will point to iwarp's qp number and requires no calls from the driver.
	+ */
	+static enum i40iw_status_code
	+i40iw_sc_manage_qhash_table_entry(
	+ struct i40iw_sc_cqp *cqp,
	+ struct i40iw_qhash_table_info *info,
	+ u64 scratch,
	+ bool post_sq)
	+{
	+ u64 *wqe;
	+ u64 qw1 = 0;
	+ u64 qw2 = 0;
	+ u64 temp;
	+ struct i40iw_sc_vsi *vsi = info->vsi;
	+
	+ wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
	+ if (!wqe)
	+ return I40IW_ERR_RING_FULL;
	+
	+ temp = info->mac_addr[5] \|
	+ LS_64_1(info->mac_addr[4], 8) \|
	+ LS_64_1(info->mac_addr[3], 16) \|
	+ LS_64_1(info->mac_addr[2], 24) \|
	+ LS_64_1(info->mac_addr[1], 32) \|
	+ LS_64_1(info->mac_addr[0], 40);
	+
	+ set_64bit_val(wqe, I40IW_BYTE_0, temp);
	+
	+ qw1 = LS_64(info->qp_num, I40IW_CQPSQ_QHASH_QPN) \|
	+ LS_64(info->dest_port, I40IW_CQPSQ_QHASH_DEST_PORT);
	+ if (info->ipv4_valid) {
	+ set_64bit_val(wqe,
	+ I40IW_BYTE_48,
	+ LS_64(info->dest_ip[0], I40IW_CQPSQ_QHASH_ADDR3));
	+ } else {
	+ set_64bit_val(wqe,
	+ I40IW_BYTE_56,
	+ LS_64(info->dest_ip[0], I40IW_CQPSQ_QHASH_ADDR0) \|
	+ LS_64(info->dest_ip[1], I40IW_CQPSQ_QHASH_ADDR1));
	+
	+ set_64bit_val(wqe,
	+ I40IW_BYTE_48,
	+ LS_64(info->dest_ip[2], I40IW_CQPSQ_QHASH_ADDR2) \|
	+ LS_64(info->dest_ip[3], I40IW_CQPSQ_QHASH_ADDR3));
	+ }
	+ qw2 = LS_64(vsi->qos[0].qs_handle, I40IW_CQPSQ_QHASH_QS_HANDLE);
	+ if (info->vlan_valid)
	+ qw2 \|= LS_64(info->vlan_id, I40IW_CQPSQ_QHASH_VLANID);
	+ set_64bit_val(wqe, I40IW_BYTE_16, qw2);
	+ if (info->entry_type == I40IW_QHASH_TYPE_TCP_ESTABLISHED) {
	+ qw1 \|= LS_64(info->src_port, I40IW_CQPSQ_QHASH_SRC_PORT);
	+ if (!info->ipv4_valid) {
	+ set_64bit_val(wqe,
	+ I40IW_BYTE_40,
	+ LS_64(info->src_ip[0], I40IW_CQPSQ_QHASH_ADDR0) \|
	+ LS_64(info->src_ip[1], I40IW_CQPSQ_QHASH_ADDR1));
	+ set_64bit_val(wqe,
	+ I40IW_BYTE_32,
	+ LS_64(info->src_ip[2], I40IW_CQPSQ_QHASH_ADDR2) \|
	+ LS_64(info->src_ip[3], I40IW_CQPSQ_QHASH_ADDR3));
	+ } else {
	+ set_64bit_val(wqe,
	+ I40IW_BYTE_32,
	+ LS_64(info->src_ip[0], I40IW_CQPSQ_QHASH_ADDR3));
	+ }
	+ }
	+ set_64bit_val(wqe, I40IW_BYTE_8, qw1);
	+ temp = LS_64(cqp->polarity, I40IW_CQPSQ_QHASH_WQEVALID) \|
	+ LS_64(I40IW_CQP_OP_MANAGE_QUAD_HASH_TABLE_ENTRY, I40IW_CQPSQ_QHASH_OPCODE) \|
	+ LS_64(info->manage, I40IW_CQPSQ_QHASH_MANAGE) \|
	+ LS_64(info->ipv4_valid, I40IW_CQPSQ_QHASH_IPV4VALID) \|
	+ LS_64(info->vlan_valid, I40IW_CQPSQ_QHASH_VLANVALID) \|
	+ LS_64(info->entry_type, I40IW_CQPSQ_QHASH_ENTRYTYPE);
	+
	+ i40iw_insert_wqe_hdr(wqe, temp);
	+
	+ i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "MANAGE_QHASH WQE",
	+ wqe, I40IW_CQP_WQE_SIZE * 8);
	+
	+ if (post_sq)
	+ i40iw_sc_cqp_post_sq(cqp);
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_sc_alloc_local_mac_ipaddr_entry - cqp wqe for loc mac entry
	+ * @cqp: struct for cqp hw
	+ * @scratch: u64 saved to be used during cqp completion
	+ * @post_sq: flag for cqp db to ring
	+ */
	+static enum i40iw_status_code
	+i40iw_sc_alloc_local_mac_ipaddr_entry(
	+ struct i40iw_sc_cqp *cqp,
	+ u64 scratch,
	+ bool post_sq)
	+{
	+ u64 *wqe;
	+ u64 header;
	+
	+ wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
	+ if (!wqe)
	+ return I40IW_ERR_RING_FULL;
	+ header = LS_64(I40IW_CQP_OP_ALLOCATE_LOC_MAC_IP_TABLE_ENTRY, I40IW_CQPSQ_OPCODE) \|
	+ LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
	+
	+ i40iw_insert_wqe_hdr(wqe, header);
	+ i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "ALLOCATE_LOCAL_MAC_IPADDR WQE",
	+ wqe, I40IW_CQP_WQE_SIZE * 8);
	+ if (post_sq)
	+ i40iw_sc_cqp_post_sq(cqp);
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_sc_add_local_mac_ipaddr_entry - add mac enry
	+ * @cqp: struct for cqp hw
	+ * @info:mac addr info
	+ * @scratch: u64 saved to be used during cqp completion
	+ * @post_sq: flag for cqp db to ring
	+ */
	+static enum i40iw_status_code
	+i40iw_sc_add_local_mac_ipaddr_entry(
	+ struct i40iw_sc_cqp *cqp,
	+ struct i40iw_local_mac_ipaddr_entry_info *info,
	+ u64 scratch,
	+ bool post_sq)
	+{
	+ u64 *wqe;
	+ u64 temp, header;
	+
	+ wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
	+ if (!wqe)
	+ return I40IW_ERR_RING_FULL;
	+ temp = info->mac_addr[5] \|
	+ LS_64_1(info->mac_addr[4], 8) \|
	+ LS_64_1(info->mac_addr[3], 16) \|
	+ LS_64_1(info->mac_addr[2], 24) \|
	+ LS_64_1(info->mac_addr[1], 32) \|
	+ LS_64_1(info->mac_addr[0], 40);
	+
	+ set_64bit_val(wqe, I40IW_BYTE_32, temp);
	+
	+ header = LS_64(info->entry_idx, I40IW_CQPSQ_MLIPA_IPTABLEIDX) \|
	+ LS_64(I40IW_CQP_OP_MANAGE_LOC_MAC_IP_TABLE, I40IW_CQPSQ_OPCODE) \|
	+ LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
	+
	+ i40iw_insert_wqe_hdr(wqe, header);
	+
	+ i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "ADD_LOCAL_MAC_IPADDR WQE",
	+ wqe, I40IW_CQP_WQE_SIZE * 8);
	+
	+ if (post_sq)
	+ i40iw_sc_cqp_post_sq(cqp);
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_sc_del_local_mac_ipaddr_entry - cqp wqe to dele local mac
	+ * @cqp: struct for cqp hw
	+ * @scratch: u64 saved to be used during cqp completion
	+ * @entry_idx: index of mac entry
	+ * @ ignore_ref_count: to force mac adde delete
	+ * @post_sq: flag for cqp db to ring
	+ */
	+static enum i40iw_status_code
	+i40iw_sc_del_local_mac_ipaddr_entry(
	+ struct i40iw_sc_cqp *cqp,
	+ u64 scratch,
	+ u8 entry_idx,
	+ u8 ignore_ref_count,
	+ bool post_sq)
	+{
	+ u64 *wqe;
	+ u64 header;
	+
	+ wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
	+ if (!wqe)
	+ return I40IW_ERR_RING_FULL;
	+ header = LS_64(entry_idx, I40IW_CQPSQ_MLIPA_IPTABLEIDX) \|
	+ LS_64(I40IW_CQP_OP_MANAGE_LOC_MAC_IP_TABLE, I40IW_CQPSQ_OPCODE) \|
	+ LS_64(1, I40IW_CQPSQ_MLIPA_FREEENTRY) \|
	+ LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID) \|
	+ LS_64(ignore_ref_count, I40IW_CQPSQ_MLIPA_IGNORE_REF_CNT);
	+
	+ i40iw_insert_wqe_hdr(wqe, header);
	+
	+ i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "DEL_LOCAL_MAC_IPADDR WQE",
	+ wqe, I40IW_CQP_WQE_SIZE * 8);
	+
	+ if (post_sq)
	+ i40iw_sc_cqp_post_sq(cqp);
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_sc_cqp_nop - send a nop wqe
	+ * @cqp: struct for cqp hw
	+ * @scratch: u64 saved to be used during cqp completion
	+ * @post_sq: flag for cqp db to ring
	+ */
	+static enum i40iw_status_code
	+i40iw_sc_cqp_nop(struct i40iw_sc_cqp *cqp,
	+ u64 scratch,
	+ bool post_sq)
	+{
	+ u64 *wqe;
	+ u64 header;
	+
	+ wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
	+ if (!wqe)
	+ return I40IW_ERR_RING_FULL;
	+ header = LS_64(I40IW_CQP_OP_NOP, I40IW_CQPSQ_OPCODE) \|
	+ LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
	+ i40iw_insert_wqe_hdr(wqe, header);
	+ i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "NOP WQE",
	+ wqe, I40IW_CQP_WQE_SIZE * 8);
	+
	+ if (post_sq)
	+ i40iw_sc_cqp_post_sq(cqp);
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_sc_ceq_init - initialize ceq
	+ * @ceq: ceq sc structure
	+ * @info: ceq initialization info
	+ */
	+static enum i40iw_status_code
	+i40iw_sc_ceq_init(struct i40iw_sc_ceq *ceq,
	+ struct i40iw_ceq_init_info *info)
	+{
	+ u32 pble_obj_cnt;
	+
	+ if ((info->elem_cnt < I40IW_MIN_CEQ_ENTRIES) \|\|
	+ (info->elem_cnt > I40IW_MAX_CEQ_ENTRIES))
	+ return I40IW_ERR_INVALID_SIZE;
	+
	+ if (info->ceq_id >= I40IW_MAX_CEQID)
	+ return I40IW_ERR_INVALID_CEQ_ID;
	+
	+ pble_obj_cnt = info->dev->hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].cnt;
	+
	+ if (info->virtual_map && (info->first_pm_pbl_idx >= pble_obj_cnt))
	+ return I40IW_ERR_INVALID_PBLE_INDEX;
	+
	+ ceq->size = sizeof(*ceq);
	+ ceq->ceqe_base = (struct i40iw_ceqe *)info->ceqe_base;
	+ ceq->ceq_id = info->ceq_id;
	+ ceq->dev = info->dev;
	+ ceq->elem_cnt = info->elem_cnt;
	+ ceq->ceq_elem_pa = info->ceqe_pa;
	+ ceq->virtual_map = info->virtual_map;
	+
	+ ceq->pbl_chunk_size = (ceq->virtual_map ? info->pbl_chunk_size : 0);
	+ ceq->first_pm_pbl_idx = (ceq->virtual_map ? info->first_pm_pbl_idx : 0);
	+ ceq->pbl_list = (ceq->virtual_map ? info->pbl_list : NULL);
	+
	+ ceq->tph_en = info->tph_en;
	+ ceq->tph_val = info->tph_val;
	+ ceq->polarity = 1;
	+ I40IW_RING_INIT(ceq->ceq_ring, ceq->elem_cnt);
	+ ceq->dev->ceq[info->ceq_id] = ceq;
	+
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_sc_ceq_create - create ceq wqe
	+ * @ceq: ceq sc structure
	+ * @scratch: u64 saved to be used during cqp completion
	+ * @post_sq: flag for cqp db to ring
	+ */
	+static enum i40iw_status_code
	+i40iw_sc_ceq_create(struct i40iw_sc_ceq *ceq,
	+ u64 scratch,
	+ bool post_sq)
	+{
	+ struct i40iw_sc_cqp *cqp;
	+ u64 *wqe;
	+ u64 header;
	+
	+ cqp = ceq->dev->cqp;
	+ wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
	+ if (!wqe)
	+ return I40IW_ERR_RING_FULL;
	+ set_64bit_val(wqe, I40IW_BYTE_16, ceq->elem_cnt);
	+ set_64bit_val(wqe, I40IW_BYTE_32, (ceq->virtual_map ? 0 : ceq->ceq_elem_pa));
	+ set_64bit_val(wqe, I40IW_BYTE_48, (ceq->virtual_map ? ceq->first_pm_pbl_idx : 0));
	+ set_64bit_val(wqe, I40IW_BYTE_56, LS_64(ceq->tph_val, I40IW_CQPSQ_TPHVAL));
	+
	+ header = ceq->ceq_id \|
	+ LS_64(I40IW_CQP_OP_CREATE_CEQ, I40IW_CQPSQ_OPCODE) \|
	+ LS_64(ceq->pbl_chunk_size, I40IW_CQPSQ_CEQ_LPBLSIZE) \|
	+ LS_64(ceq->virtual_map, I40IW_CQPSQ_CEQ_VMAP) \|
	+ LS_64(ceq->tph_en, I40IW_CQPSQ_TPHEN) \|
	+ LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
	+
	+ i40iw_insert_wqe_hdr(wqe, header);
	+
	+ i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "CEQ_CREATE WQE",
	+ wqe, I40IW_CQP_WQE_SIZE * 8);
	+
	+ if (post_sq)
	+ i40iw_sc_cqp_post_sq(cqp);
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_sc_cceq_create_done - poll for control ceq wqe to complete
	+ * @ceq: ceq sc structure
	+ */
	+static enum i40iw_status_code
	+i40iw_sc_cceq_create_done(struct i40iw_sc_ceq *ceq)
	+{
	+ struct i40iw_sc_cqp *cqp;
	+
	+ cqp = ceq->dev->cqp;
	+ return i40iw_sc_poll_for_cqp_op_done(cqp, I40IW_CQP_OP_CREATE_CEQ, NULL);
	+}
	+
	+/**
	+ * i40iw_sc_cceq_destroy_done - poll for destroy cceq to complete
	+ * @ceq: ceq sc structure
	+ */
	+static enum i40iw_status_code
	+i40iw_sc_cceq_destroy_done(struct i40iw_sc_ceq *ceq)
	+{
	+ struct i40iw_sc_cqp *cqp;
	+
	+ cqp = ceq->dev->cqp;
	+ cqp->process_cqp_sds = i40iw_update_sds_noccq;
	+ return i40iw_sc_poll_for_cqp_op_done(cqp, I40IW_CQP_OP_DESTROY_CEQ, NULL);
	+}
	+
	+/**
	+ * i40iw_sc_cceq_create - create cceq
	+ * @ceq: ceq sc structure
	+ * @scratch: u64 saved to be used during cqp completion
	+ */
	+static enum i40iw_status_code
	+i40iw_sc_cceq_create(struct i40iw_sc_ceq *ceq, u64 scratch)
	+{
	+ enum i40iw_status_code ret_code;
	+
	+ ret_code = i40iw_sc_ceq_create(ceq, scratch, TRUE);
	+ if (!ret_code)
	+ ret_code = i40iw_sc_cceq_create_done(ceq);
	+ return ret_code;
	+}
	+
	+/**
	+ * i40iw_sc_ceq_destroy - destroy ceq
	+ * @ceq: ceq sc structure
	+ * @scratch: u64 saved to be used during cqp completion
	+ * @post_sq: flag for cqp db to ring
	+ */
	+static enum i40iw_status_code
	+i40iw_sc_ceq_destroy(struct i40iw_sc_ceq *ceq,
	+ u64 scratch,
	+ bool post_sq)
	+{
	+ struct i40iw_sc_cqp *cqp;
	+ u64 *wqe;
	+ u64 header;
	+
	+ cqp = ceq->dev->cqp;
	+ wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
	+ if (!wqe)
	+ return I40IW_ERR_RING_FULL;
	+
	+ if (ceq->ceq_id >= I40IW_MAX_CEQID)
	+ return I40IW_ERR_INVALID_CEQ_ID;
	+
	+ set_64bit_val(wqe, I40IW_BYTE_16, ceq->elem_cnt);
	+ set_64bit_val(wqe, I40IW_BYTE_48, ceq->first_pm_pbl_idx);
	+ header = ceq->ceq_id \|
	+ LS_64(I40IW_CQP_OP_DESTROY_CEQ, I40IW_CQPSQ_OPCODE) \|
	+ LS_64(ceq->pbl_chunk_size, I40IW_CQPSQ_CEQ_LPBLSIZE) \|
	+ LS_64(ceq->virtual_map, I40IW_CQPSQ_CEQ_VMAP) \|
	+ LS_64(ceq->tph_en, I40IW_CQPSQ_TPHEN) \|
	+ LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
	+ i40iw_insert_wqe_hdr(wqe, header);
	+ i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "CEQ_DESTROY WQE",
	+ wqe, I40IW_CQP_WQE_SIZE * 8);
	+
	+ if (post_sq)
	+ i40iw_sc_cqp_post_sq(cqp);
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_sc_process_ceq - process ceq
	+ * @dev: sc device struct
	+ * @ceq: ceq sc structure
	+ */
	+static void *
	+i40iw_sc_process_ceq(struct i40iw_sc_dev dev, struct i40iw_sc_ceq ceq)
	+{
	+ u64 temp;
	+ u64 *ceqe;
	+ struct i40iw_sc_cq *cq = NULL;
	+ u8 polarity;
	+
	+ ceqe = (u64 *)I40IW_GET_CURRENT_CEQ_ELEMENT(ceq);
	+ get_64bit_val(ceqe, I40IW_BYTE_0, &temp);
	+ polarity = (u8)RS_64(temp, I40IW_CEQE_VALID);
	+ if (polarity != ceq->polarity)
	+ return cq;
	+
	+ cq = (struct i40iw_sc_cq *)(i40iw_uintptr)LS_64_1(temp, 1);
	+ if (!cq)
	+ return NULL;
	+
	+ I40IW_RING_MOVE_TAIL(ceq->ceq_ring);
	+ if (I40IW_RING_GETCURRENT_TAIL(ceq->ceq_ring) == 0)
	+ ceq->polarity ^= 1;
	+
	+ if (dev->is_pf)
	+ wr32(dev->hw, I40E_PFPE_CQACK, cq->cq_uk.cq_id);
	+ else
	+ wr32(dev->hw, I40E_VFPE_CQACK1, cq->cq_uk.cq_id);
	+
	+ return cq;
	+}
	+
	+/**
	+ * i40iw_sc_aeq_init - initialize aeq
	+ * @aeq: aeq structure ptr
	+ * @info: aeq initialization info
	+ */
	+static enum i40iw_status_code
	+i40iw_sc_aeq_init(struct i40iw_sc_aeq *aeq,
	+ struct i40iw_aeq_init_info *info)
	+{
	+ u32 pble_obj_cnt;
	+
	+ if ((info->elem_cnt < I40IW_MIN_AEQ_ENTRIES) \|\|
	+ (info->elem_cnt > I40IW_MAX_AEQ_ENTRIES))
	+ return I40IW_ERR_INVALID_SIZE;
	+ pble_obj_cnt = info->dev->hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].cnt;
	+
	+ if (info->virtual_map && (info->first_pm_pbl_idx >= pble_obj_cnt))
	+ return I40IW_ERR_INVALID_PBLE_INDEX;
	+
	+ aeq->size = sizeof(*aeq);
	+ aeq->polarity = 1;
	+ aeq->aeqe_base = (struct i40iw_sc_aeqe *)info->aeqe_base;
	+ aeq->dev = info->dev;
	+ aeq->elem_cnt = info->elem_cnt;
	+
	+ aeq->aeq_elem_pa = info->aeq_elem_pa;
	+ I40IW_RING_INIT(aeq->aeq_ring, aeq->elem_cnt);
	+ info->dev->aeq = aeq;
	+
	+ aeq->virtual_map = info->virtual_map;
	+ aeq->pbl_list = (aeq->virtual_map ? info->pbl_list : NULL);
	+ aeq->pbl_chunk_size = (aeq->virtual_map ? info->pbl_chunk_size : 0);
	+ aeq->first_pm_pbl_idx = (aeq->virtual_map ? info->first_pm_pbl_idx : 0);
	+ info->dev->aeq = aeq;
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_sc_aeq_create - create aeq
	+ * @aeq: aeq structure ptr
	+ * @scratch: u64 saved to be used during cqp completion
	+ * @post_sq: flag for cqp db to ring
	+ */
	+static enum i40iw_status_code
	+i40iw_sc_aeq_create(struct i40iw_sc_aeq *aeq,
	+ u64 scratch,
	+ bool post_sq)
	+{
	+ u64 *wqe;
	+ struct i40iw_sc_cqp *cqp;
	+ struct i40iw_sc_dev *dev;
	+ u64 header;
	+
	+ dev = aeq->dev;
	+
	+ if (dev->is_pf)
	+ wr32(dev->hw, I40E_PFINT_AEQCTL, 0);
	+
	+ cqp = dev->cqp;
	+ wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
	+ if (!wqe)
	+ return I40IW_ERR_RING_FULL;
	+ set_64bit_val(wqe, I40IW_BYTE_16, aeq->elem_cnt);
	+ set_64bit_val(wqe, I40IW_BYTE_32,
	+ (aeq->virtual_map ? 0 : aeq->aeq_elem_pa));
	+ set_64bit_val(wqe, I40IW_BYTE_48,
	+ (aeq->virtual_map ? aeq->first_pm_pbl_idx : 0));
	+
	+ header = LS_64(I40IW_CQP_OP_CREATE_AEQ, I40IW_CQPSQ_OPCODE) \|
	+ LS_64(aeq->pbl_chunk_size, I40IW_CQPSQ_AEQ_LPBLSIZE) \|
	+ LS_64(aeq->virtual_map, I40IW_CQPSQ_AEQ_VMAP) \|
	+ LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
	+
	+ i40iw_insert_wqe_hdr(wqe, header);
	+ i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "AEQ_CREATE WQE",
	+ wqe, I40IW_CQP_WQE_SIZE * 8);
	+ if (post_sq)
	+ i40iw_sc_cqp_post_sq(cqp);
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_sc_aeq_destroy - destroy aeq during close
	+ * @aeq: aeq structure ptr
	+ * @scratch: u64 saved to be used during cqp completion
	+ * @post_sq: flag for cqp db to ring
	+ */
	+static enum i40iw_status_code
	+i40iw_sc_aeq_destroy(struct i40iw_sc_aeq *aeq,
	+ u64 scratch,
	+ bool post_sq)
	+{
	+ u64 *wqe;
	+ struct i40iw_sc_cqp *cqp;
	+ u64 header;
	+
	+ cqp = aeq->dev->cqp;
	+ wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
	+ if (!wqe)
	+ return I40IW_ERR_RING_FULL;
	+ set_64bit_val(wqe, I40IW_BYTE_16, aeq->elem_cnt);
	+ set_64bit_val(wqe, I40IW_BYTE_48, aeq->first_pm_pbl_idx);
	+ header = LS_64(I40IW_CQP_OP_DESTROY_AEQ, I40IW_CQPSQ_OPCODE) \|
	+ LS_64(aeq->pbl_chunk_size, I40IW_CQPSQ_AEQ_LPBLSIZE) \|
	+ LS_64(aeq->virtual_map, I40IW_CQPSQ_AEQ_VMAP) \|
	+ LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
	+ i40iw_insert_wqe_hdr(wqe, header);
	+
	+ i40iw_debug_buf(aeq->dev, I40IW_DEBUG_WQE, "AEQ_DESTROY WQE",
	+ wqe, I40IW_CQP_WQE_SIZE * 8);
	+ if (post_sq)
	+ i40iw_sc_cqp_post_sq(cqp);
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_sc_get_next_aeqe - get next aeq entry
	+ * @aeq: aeq structure ptr
	+ * @info: aeqe info to be returned
	+ */
	+static enum i40iw_status_code
	+i40iw_sc_get_next_aeqe(struct i40iw_sc_aeq *aeq,
	+ struct i40iw_aeqe_info *info)
	+{
	+ u64 temp, compl_ctx;
	+ u64 *aeqe;
	+ u16 wqe_idx;
	+ u8 ae_src;
	+ u8 polarity;
	+
	+ aeqe = (u64 *)I40IW_GET_CURRENT_AEQ_ELEMENT(aeq);
	+ get_64bit_val(aeqe, I40IW_BYTE_0, &compl_ctx);
	+ get_64bit_val(aeqe, I40IW_BYTE_8, &temp);
	+ polarity = (u8)RS_64(temp, I40IW_AEQE_VALID);
	+
	+ if (aeq->polarity != polarity)
	+ return I40IW_ERR_QUEUE_EMPTY;
	+
	+ i40iw_debug_buf(aeq->dev, I40IW_DEBUG_WQE, "AEQ_ENTRY", aeqe, 16);
	+
	+ ae_src = (u8)RS_64(temp, I40IW_AEQE_AESRC);
	+ wqe_idx = (u16)RS_64(temp, I40IW_AEQE_WQDESCIDX);
	+ info->qp_cq_id = (u32)RS_64(temp, I40IW_AEQE_QPCQID);
	+ info->ae_id = (u16)RS_64(temp, I40IW_AEQE_AECODE);
	+ info->tcp_state = (u8)RS_64(temp, I40IW_AEQE_TCPSTATE);
	+ info->iwarp_state = (u8)RS_64(temp, I40IW_AEQE_IWSTATE);
	+ info->q2_data_written = (u8)RS_64(temp, I40IW_AEQE_Q2DATA);
	+ info->aeqe_overflow = (bool)RS_64(temp, I40IW_AEQE_OVERFLOW);
	+
	+ switch (info->ae_id) {
	+ case I40IW_AE_PRIV_OPERATION_DENIED:
	+ case I40IW_AE_UDA_XMIT_DGRAM_TOO_LONG:
	+ case I40IW_AE_UDA_XMIT_DGRAM_TOO_SHORT:
	+ case I40IW_AE_BAD_CLOSE:
	+ case I40IW_AE_RDMAP_ROE_BAD_LLP_CLOSE:
	+ case I40IW_AE_RDMA_READ_WHILE_ORD_ZERO:
	+ case I40IW_AE_STAG_ZERO_INVALID:
	+ case I40IW_AE_IB_RREQ_AND_Q1_FULL:
	+ case I40IW_AE_WQE_UNEXPECTED_OPCODE:
	+ case I40IW_AE_DDP_UBE_INVALID_DDP_VERSION:
	+ case I40IW_AE_DDP_UBE_INVALID_MO:
	+ case I40IW_AE_DDP_UBE_INVALID_QN:
	+ case I40IW_AE_DDP_NO_L_BIT:
	+ case I40IW_AE_RDMAP_ROE_INVALID_RDMAP_VERSION:
	+ case I40IW_AE_RDMAP_ROE_UNEXPECTED_OPCODE:
	+ case I40IW_AE_ROE_INVALID_RDMA_READ_REQUEST:
	+ case I40IW_AE_ROE_INVALID_RDMA_WRITE_OR_READ_RESP:
	+ case I40IW_AE_INVALID_ARP_ENTRY:
	+ case I40IW_AE_INVALID_TCP_OPTION_RCVD:
	+ case I40IW_AE_STALE_ARP_ENTRY:
	+ case I40IW_AE_LLP_CLOSE_COMPLETE:
	+ case I40IW_AE_LLP_CONNECTION_RESET:
	+ case I40IW_AE_LLP_FIN_RECEIVED:
	+ case I40IW_AE_LLP_RECEIVED_MPA_CRC_ERROR:
	+ case I40IW_AE_LLP_SEGMENT_TOO_SMALL:
	+ case I40IW_AE_LLP_SYN_RECEIVED:
	+ case I40IW_AE_LLP_TERMINATE_RECEIVED:
	+ case I40IW_AE_LLP_TOO_MANY_RETRIES:
	+ case I40IW_AE_LLP_DOUBT_REACHABILITY:
	+ case I40IW_AE_RESET_SENT:
	+ case I40IW_AE_TERMINATE_SENT:
	+ case I40IW_AE_RESET_NOT_SENT:
	+ case I40IW_AE_LCE_QP_CATASTROPHIC:
	+ case I40IW_AE_QP_SUSPEND_COMPLETE:
	+ info->qp = TRUE;
	+ info->compl_ctx = compl_ctx;
	+ ae_src = I40IW_AE_SOURCE_RSVD;
	+ break;
	+ case I40IW_AE_LCE_CQ_CATASTROPHIC:
	+ info->cq = TRUE;
	+ info->compl_ctx = LS_64_1(compl_ctx, 1);
	+ ae_src = I40IW_AE_SOURCE_RSVD;
	+ break;
	+ }
	+
	+ switch (ae_src) {
	+ case I40IW_AE_SOURCE_RQ:
	+ case I40IW_AE_SOURCE_RQ_0011:
	+ info->qp = TRUE;
	+ info->wqe_idx = wqe_idx;
	+ info->compl_ctx = compl_ctx;
	+ break;
	+ case I40IW_AE_SOURCE_CQ:
	+ case I40IW_AE_SOURCE_CQ_0110:
	+ case I40IW_AE_SOURCE_CQ_1010:
	+ case I40IW_AE_SOURCE_CQ_1110:
	+ info->cq = TRUE;
	+ info->compl_ctx = LS_64_1(compl_ctx, 1);
	+ break;
	+ case I40IW_AE_SOURCE_SQ:
	+ case I40IW_AE_SOURCE_SQ_0111:
	+ info->qp = TRUE;
	+ info->sq = TRUE;
	+ info->wqe_idx = wqe_idx;
	+ info->compl_ctx = compl_ctx;
	+ break;
	+ case I40IW_AE_SOURCE_IN_RR_WR:
	+ case I40IW_AE_SOURCE_IN_RR_WR_1011:
	+ info->qp = TRUE;
	+ info->compl_ctx = compl_ctx;
	+ info->in_rdrsp_wr = TRUE;
	+ break;
	+ case I40IW_AE_SOURCE_OUT_RR:
	+ case I40IW_AE_SOURCE_OUT_RR_1111:
	+ info->qp = TRUE;
	+ info->compl_ctx = compl_ctx;
	+ info->out_rdrsp = TRUE;
	+ break;
	+ case I40IW_AE_SOURCE_RSVD:
	+ /* fallthrough */
	+ default:
	+ break;
	+ }
	+ I40IW_RING_MOVE_TAIL(aeq->aeq_ring);
	+ if (I40IW_RING_GETCURRENT_TAIL(aeq->aeq_ring) == 0)
	+ aeq->polarity ^= 1;
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_sc_repost_aeq_entries - repost completed aeq entries
	+ * @dev: sc device struct
	+ * @count: allocate count
	+ */
	+static enum i40iw_status_code
	+i40iw_sc_repost_aeq_entries(struct i40iw_sc_dev *dev,
	+ u32 count)
	+{
	+ if (dev->is_pf)
	+ wr32(dev->hw, I40E_PFPE_AEQALLOC, count);
	+ else
	+ wr32(dev->hw, I40E_VFPE_AEQALLOC1, count);
	+
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_sc_aeq_create_done - create aeq
	+ * @aeq: aeq structure ptr
	+ */
	+static enum i40iw_status_code
	+i40iw_sc_aeq_create_done(struct i40iw_sc_aeq *aeq)
	+{
	+ struct i40iw_sc_cqp *cqp;
	+
	+ cqp = aeq->dev->cqp;
	+ return i40iw_sc_poll_for_cqp_op_done(cqp, I40IW_CQP_OP_CREATE_AEQ, NULL);
	+}
	+
	+/**
	+ * i40iw_sc_aeq_destroy_done - destroy of aeq during close
	+ * @aeq: aeq structure ptr
	+ */
	+static enum i40iw_status_code
	+i40iw_sc_aeq_destroy_done(struct i40iw_sc_aeq *aeq)
	+{
	+ struct i40iw_sc_cqp *cqp;
	+
	+ cqp = aeq->dev->cqp;
	+ return i40iw_sc_poll_for_cqp_op_done(cqp, I40IW_CQP_OP_DESTROY_AEQ, NULL);
	+}
	+
	+/**
	+ * i40iw_sc_ccq_init - initialize control cq
	+ * @cq: sc's cq ctruct
	+ * @info: info for control cq initialization
	+ */
	+static enum i40iw_status_code
	+i40iw_sc_ccq_init(struct i40iw_sc_cq *cq,
	+ struct i40iw_ccq_init_info *info)
	+{
	+ u32 pble_obj_cnt;
	+
	+ if (info->num_elem < I40IW_MIN_CQ_SIZE \|\| info->num_elem > I40IW_MAX_CQ_SIZE)
	+ return I40IW_ERR_INVALID_SIZE;
	+
	+ if (info->ceq_id >= I40IW_MAX_CEQID)
	+ return I40IW_ERR_INVALID_CEQ_ID;
	+
	+ pble_obj_cnt = info->dev->hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].cnt;
	+
	+ if (info->virtual_map && (info->first_pm_pbl_idx >= pble_obj_cnt))
	+ return I40IW_ERR_INVALID_PBLE_INDEX;
	+
	+ cq->cq_pa = info->cq_pa;
	+ cq->cq_uk.cq_base = info->cq_base;
	+ cq->shadow_area_pa = info->shadow_area_pa;
	+ cq->cq_uk.shadow_area = info->shadow_area;
	+ cq->shadow_read_threshold = info->shadow_read_threshold;
	+ cq->dev = info->dev;
	+ cq->ceq_id = info->ceq_id;
	+ cq->cq_uk.cq_size = info->num_elem;
	+ cq->cq_type = I40IW_CQ_TYPE_CQP;
	+ cq->ceqe_mask = info->ceqe_mask;
	+ I40IW_RING_INIT(cq->cq_uk.cq_ring, info->num_elem);
	+
	+ cq->cq_uk.cq_id = 0; /* control cq is id 0 always */
	+ cq->ceq_id_valid = info->ceq_id_valid;
	+ cq->tph_en = info->tph_en;
	+ cq->tph_val = info->tph_val;
	+ cq->cq_uk.avoid_mem_cflct = info->avoid_mem_cflct;
	+
	+ cq->pbl_list = info->pbl_list;
	+ cq->virtual_map = info->virtual_map;
	+ cq->pbl_chunk_size = info->pbl_chunk_size;
	+ cq->first_pm_pbl_idx = info->first_pm_pbl_idx;
	+ cq->cq_uk.polarity = TRUE;
	+
	+ /* following are only for iw cqs so initialize them to zero */
	+ cq->cq_uk.cqe_alloc_reg = NULL;
	+ info->dev->ccq = cq;
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_sc_ccq_create_done - poll cqp for ccq create
	+ * @ccq: ccq sc struct
	+ */
	+static enum i40iw_status_code
	+i40iw_sc_ccq_create_done(struct i40iw_sc_cq *ccq)
	+{
	+ struct i40iw_sc_cqp *cqp;
	+
	+ cqp = ccq->dev->cqp;
	+ return i40iw_sc_poll_for_cqp_op_done(cqp, I40IW_CQP_OP_CREATE_CQ, NULL);
	+}
	+
	+/**
	+ * i40iw_sc_ccq_create - create control cq
	+ * @ccq: ccq sc struct
	+ * @scratch: u64 saved to be used during cqp completion
	+ * @check_overflow: overlow flag for ccq
	+ * @post_sq: flag for cqp db to ring
	+ */
	+static enum i40iw_status_code
	+i40iw_sc_ccq_create(struct i40iw_sc_cq *ccq,
	+ u64 scratch,
	+ bool check_overflow,
	+ bool post_sq)
	+{
	+ u64 *wqe;
	+ struct i40iw_sc_cqp *cqp;
	+ u64 header;
	+ enum i40iw_status_code ret_code;
	+
	+ cqp = ccq->dev->cqp;
	+ wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
	+ if (!wqe)
	+ return I40IW_ERR_RING_FULL;
	+ set_64bit_val(wqe, I40IW_BYTE_0, ccq->cq_uk.cq_size);
	+ set_64bit_val(wqe, I40IW_BYTE_8, RS_64_1(ccq, 1));
	+ set_64bit_val(wqe, I40IW_BYTE_16,
	+ LS_64(ccq->shadow_read_threshold, I40IW_CQPSQ_CQ_SHADOW_READ_THRESHOLD));
	+ set_64bit_val(wqe, I40IW_BYTE_32, (ccq->virtual_map ? 0 : ccq->cq_pa));
	+ set_64bit_val(wqe, I40IW_BYTE_40, ccq->shadow_area_pa);
	+ set_64bit_val(wqe, I40IW_BYTE_48,
	+ (ccq->virtual_map ? ccq->first_pm_pbl_idx : 0));
	+ set_64bit_val(wqe, I40IW_BYTE_56,
	+ LS_64(ccq->tph_val, I40IW_CQPSQ_TPHVAL));
	+
	+ header = ccq->cq_uk.cq_id \|
	+ LS_64((ccq->ceq_id_valid ? ccq->ceq_id : 0), I40IW_CQPSQ_CQ_CEQID) \|
	+ LS_64(I40IW_CQP_OP_CREATE_CQ, I40IW_CQPSQ_OPCODE) \|
	+ LS_64(ccq->pbl_chunk_size, I40IW_CQPSQ_CQ_LPBLSIZE) \|
	+ LS_64(check_overflow, I40IW_CQPSQ_CQ_CHKOVERFLOW) \|
	+ LS_64(ccq->virtual_map, I40IW_CQPSQ_CQ_VIRTMAP) \|
	+ LS_64(ccq->ceqe_mask, I40IW_CQPSQ_CQ_ENCEQEMASK) \|
	+ LS_64(ccq->ceq_id_valid, I40IW_CQPSQ_CQ_CEQIDVALID) \|
	+ LS_64(ccq->tph_en, I40IW_CQPSQ_TPHEN) \|
	+ LS_64(ccq->cq_uk.avoid_mem_cflct, I40IW_CQPSQ_CQ_AVOIDMEMCNFLCT) \|
	+ LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
	+
	+ i40iw_insert_wqe_hdr(wqe, header);
	+
	+ i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "CCQ_CREATE WQE",
	+ wqe, I40IW_CQP_WQE_SIZE * 8);
	+
	+ if (post_sq) {
	+ i40iw_sc_cqp_post_sq(cqp);
	+ ret_code = i40iw_sc_ccq_create_done(ccq);
	+ if (ret_code)
	+ return ret_code;
	+ }
	+ cqp->process_cqp_sds = i40iw_cqp_sds_cmd;
	+
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_sc_ccq_destroy - destroy ccq during close
	+ * @ccq: ccq sc struct
	+ * @scratch: u64 saved to be used during cqp completion
	+ * @post_sq: flag for cqp db to ring
	+ */
	+static enum i40iw_status_code
	+i40iw_sc_ccq_destroy(struct i40iw_sc_cq *ccq,
	+ u64 scratch,
	+ bool post_sq)
	+{
	+ struct i40iw_sc_cqp *cqp;
	+ u64 *wqe;
	+ u64 header;
	+ enum i40iw_status_code ret_code = 0;
	+ u32 tail, val, error;
	+
	+ cqp = ccq->dev->cqp;
	+ wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
	+ if (!wqe)
	+ return I40IW_ERR_RING_FULL;
	+ set_64bit_val(wqe, I40IW_BYTE_0, ccq->cq_uk.cq_size);
	+ set_64bit_val(wqe, I40IW_BYTE_8, RS_64_1(ccq, 1));
	+ set_64bit_val(wqe, I40IW_BYTE_40, ccq->shadow_area_pa);
	+
	+ header = ccq->cq_uk.cq_id \|
	+ LS_64((ccq->ceq_id_valid ? ccq->ceq_id : 0), I40IW_CQPSQ_CQ_CEQID) \|
	+ LS_64(I40IW_CQP_OP_DESTROY_CQ, I40IW_CQPSQ_OPCODE) \|
	+ LS_64(ccq->ceqe_mask, I40IW_CQPSQ_CQ_ENCEQEMASK) \|
	+ LS_64(ccq->ceq_id_valid, I40IW_CQPSQ_CQ_CEQIDVALID) \|
	+ LS_64(ccq->tph_en, I40IW_CQPSQ_TPHEN) \|
	+ LS_64(ccq->cq_uk.avoid_mem_cflct, I40IW_CQPSQ_CQ_AVOIDMEMCNFLCT) \|
	+ LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
	+
	+ i40iw_insert_wqe_hdr(wqe, header);
	+
	+ i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "CCQ_DESTROY WQE",
	+ wqe, I40IW_CQP_WQE_SIZE * 8);
	+
	+ i40iw_get_cqp_reg_info(cqp, &val, &tail, &error);
	+ if (error)
	+ return I40IW_ERR_CQP_COMPL_ERROR;
	+
	+ if (post_sq) {
	+ i40iw_sc_cqp_post_sq(cqp);
	+ ret_code = i40iw_cqp_poll_registers(cqp, tail, 1000);
	+ }
	+ cqp->process_cqp_sds = i40iw_update_sds_noccq;
	+
	+ return ret_code;
	+}
	+
	+/**
	+ * i40iw_sc_cq_init - initialize completion q
	+ * @cq: cq struct
	+ * @info: cq initialization info
	+ */
	+static enum i40iw_status_code
	+i40iw_sc_cq_init(struct i40iw_sc_cq *cq,
	+ struct i40iw_cq_init_info *info)
	+{
	+ u32 IOMEM *cqe_alloc_reg = NULL;
	+ enum i40iw_status_code ret_code;
	+ u32 pble_obj_cnt;
	+ u32 arm_offset;
	+
	+ pble_obj_cnt = info->dev->hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].cnt;
	+
	+ if (info->virtual_map && (info->first_pm_pbl_idx >= pble_obj_cnt))
	+ return I40IW_ERR_INVALID_PBLE_INDEX;
	+
	+ cq->cq_pa = info->cq_base_pa;
	+ cq->dev = info->dev;
	+ cq->ceq_id = info->ceq_id;
	+ arm_offset = (info->dev->is_pf) ? I40E_PFPE_CQARM : I40E_VFPE_CQARM1;
	+ if (i40iw_get_hw_addr(cq->dev))
	+ cqe_alloc_reg = (u32 IOMEM *) (i40iw_get_hw_addr(cq->dev) +
	+ arm_offset);
	+ info->cq_uk_init_info.cqe_alloc_reg = cqe_alloc_reg;
	+ ret_code = i40iw_cq_uk_init(&cq->cq_uk, &info->cq_uk_init_info);
	+ if (ret_code)
	+ return ret_code;
	+ cq->virtual_map = info->virtual_map;
	+ cq->pbl_chunk_size = info->pbl_chunk_size;
	+ cq->ceqe_mask = info->ceqe_mask;
	+ cq->cq_type = (info->type) ? info->type : I40IW_CQ_TYPE_IWARP;
	+
	+ cq->shadow_area_pa = info->shadow_area_pa;
	+ cq->shadow_read_threshold = info->shadow_read_threshold;
	+
	+ cq->ceq_id_valid = info->ceq_id_valid;
	+ cq->tph_en = info->tph_en;
	+ cq->tph_val = info->tph_val;
	+
	+ cq->first_pm_pbl_idx = info->first_pm_pbl_idx;
	+
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_sc_cq_create - create completion q
	+ * @cq: cq struct
	+ * @scratch: u64 saved to be used during cqp completion
	+ * @check_overflow: flag for overflow check
	+ * @post_sq: flag for cqp db to ring
	+ */
	+static enum i40iw_status_code
	+i40iw_sc_cq_create(struct i40iw_sc_cq *cq,
	+ u64 scratch,
	+ bool check_overflow,
	+ bool post_sq)
	+{
	+ u64 *wqe;
	+ struct i40iw_sc_cqp *cqp;
	+ u64 header;
	+
	+ if (cq->cq_uk.cq_id > I40IW_MAX_CQID)
	+ return I40IW_ERR_INVALID_CQ_ID;
	+
	+ if (cq->ceq_id >= I40IW_MAX_CEQID)
	+ return I40IW_ERR_INVALID_CEQ_ID;
	+
	+ cqp = cq->dev->cqp;
	+ wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
	+ if (!wqe)
	+ return I40IW_ERR_RING_FULL;
	+
	+ set_64bit_val(wqe, I40IW_BYTE_0, cq->cq_uk.cq_size);
	+ set_64bit_val(wqe, I40IW_BYTE_8, RS_64_1(cq, 1));
	+ set_64bit_val(wqe,
	+ I40IW_BYTE_16,
	+ LS_64(cq->shadow_read_threshold, I40IW_CQPSQ_CQ_SHADOW_READ_THRESHOLD));
	+
	+ set_64bit_val(wqe, I40IW_BYTE_32, (cq->virtual_map ? 0 : cq->cq_pa));
	+
	+ set_64bit_val(wqe, I40IW_BYTE_40, cq->shadow_area_pa);
	+ set_64bit_val(wqe, I40IW_BYTE_48, (cq->virtual_map ? cq->first_pm_pbl_idx : 0));
	+ set_64bit_val(wqe, I40IW_BYTE_56, LS_64(cq->tph_val, I40IW_CQPSQ_TPHVAL));
	+
	+ header = cq->cq_uk.cq_id \|
	+ LS_64((cq->ceq_id_valid ? cq->ceq_id : 0), I40IW_CQPSQ_CQ_CEQID) \|
	+ LS_64(I40IW_CQP_OP_CREATE_CQ, I40IW_CQPSQ_OPCODE) \|
	+ LS_64(cq->pbl_chunk_size, I40IW_CQPSQ_CQ_LPBLSIZE) \|
	+ LS_64(check_overflow, I40IW_CQPSQ_CQ_CHKOVERFLOW) \|
	+ LS_64(cq->virtual_map, I40IW_CQPSQ_CQ_VIRTMAP) \|
	+ LS_64(cq->ceqe_mask, I40IW_CQPSQ_CQ_ENCEQEMASK) \|
	+ LS_64(cq->ceq_id_valid, I40IW_CQPSQ_CQ_CEQIDVALID) \|
	+ LS_64(cq->tph_en, I40IW_CQPSQ_TPHEN) \|
	+ LS_64(cq->cq_uk.avoid_mem_cflct, I40IW_CQPSQ_CQ_AVOIDMEMCNFLCT) \|
	+ LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
	+
	+ i40iw_insert_wqe_hdr(wqe, header);
	+
	+ i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "CQ_CREATE WQE",
	+ wqe, I40IW_CQP_WQE_SIZE * 8);
	+
	+ if (post_sq)
	+ i40iw_sc_cqp_post_sq(cqp);
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_sc_cq_destroy - destroy completion q
	+ * @cq: cq struct
	+ * @scratch: u64 saved to be used during cqp completion
	+ * @post_sq: flag for cqp db to ring
	+ */
	+static enum i40iw_status_code
	+i40iw_sc_cq_destroy(struct i40iw_sc_cq *cq,
	+ u64 scratch,
	+ bool post_sq)
	+{
	+ struct i40iw_sc_cqp *cqp;
	+ u64 *wqe;
	+ u64 header;
	+
	+ cqp = cq->dev->cqp;
	+ wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
	+ if (!wqe)
	+ return I40IW_ERR_RING_FULL;
	+
	+ set_64bit_val(wqe, I40IW_BYTE_0, cq->cq_uk.cq_size);
	+ set_64bit_val(wqe, I40IW_BYTE_8, RS_64_1(cq, 1));
	+ set_64bit_val(wqe, I40IW_BYTE_40, cq->shadow_area_pa);
	+ set_64bit_val(wqe, I40IW_BYTE_48, (cq->virtual_map ? cq->first_pm_pbl_idx : 0));
	+
	+ header = cq->cq_uk.cq_id \|
	+ LS_64((cq->ceq_id_valid ? cq->ceq_id : 0), I40IW_CQPSQ_CQ_CEQID) \|
	+ LS_64(I40IW_CQP_OP_DESTROY_CQ, I40IW_CQPSQ_OPCODE) \|
	+ LS_64(cq->pbl_chunk_size, I40IW_CQPSQ_CQ_LPBLSIZE) \|
	+ LS_64(cq->virtual_map, I40IW_CQPSQ_CQ_VIRTMAP) \|
	+ LS_64(cq->ceqe_mask, I40IW_CQPSQ_CQ_ENCEQEMASK) \|
	+ LS_64(cq->ceq_id_valid, I40IW_CQPSQ_CQ_CEQIDVALID) \|
	+ LS_64(cq->tph_en, I40IW_CQPSQ_TPHEN) \|
	+ LS_64(cq->cq_uk.avoid_mem_cflct, I40IW_CQPSQ_CQ_AVOIDMEMCNFLCT) \|
	+ LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
	+
	+ i40iw_insert_wqe_hdr(wqe, header);
	+
	+ i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "CQ_DESTROY WQE",
	+ wqe, I40IW_CQP_WQE_SIZE * 8);
	+
	+ if (post_sq)
	+ i40iw_sc_cqp_post_sq(cqp);
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_sc_cq_modify - modify a Completion Queue
	+ * @cq: cq struct
	+ * @info: modification info struct
	+ * @scratch:
	+ * @post_sq: flag to post to sq
	+ */
	+static enum i40iw_status_code
	+i40iw_sc_cq_modify(struct i40iw_sc_cq *cq,
	+ struct i40iw_modify_cq_info *info,
	+ u64 scratch,
	+ bool post_sq)
	+{
	+ struct i40iw_sc_cqp *cqp;
	+ u64 *wqe;
	+ u64 header;
	+ u32 cq_size, ceq_id, first_pm_pbl_idx;
	+ u8 pbl_chunk_size;
	+ bool virtual_map, ceq_id_valid, check_overflow;
	+ u32 pble_obj_cnt;
	+
	+ if (info->ceq_valid && (info->ceq_id >= I40IW_MAX_CEQID))
	+ return I40IW_ERR_INVALID_CEQ_ID;
	+
	+ pble_obj_cnt = cq->dev->hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].cnt;
	+
	+ if (info->cq_resize && info->virtual_map &&
	+ (info->first_pm_pbl_idx >= pble_obj_cnt))
	+ return I40IW_ERR_INVALID_PBLE_INDEX;
	+
	+ cqp = cq->dev->cqp;
	+ wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
	+ if (!wqe)
	+ return I40IW_ERR_RING_FULL;
	+
	+ cq->pbl_list = info->pbl_list;
	+ cq->cq_pa = info->cq_pa;
	+ cq->first_pm_pbl_idx = info->first_pm_pbl_idx;
	+
	+ cq_size = info->cq_resize ? info->cq_size : cq->cq_uk.cq_size;
	+ if (info->ceq_change) {
	+ ceq_id_valid = TRUE;
	+ ceq_id = info->ceq_id;
	+ } else {
	+ ceq_id_valid = cq->ceq_id_valid;
	+ ceq_id = ceq_id_valid ? cq->ceq_id : 0;
	+ }
	+ virtual_map = info->cq_resize ? info->virtual_map : cq->virtual_map;
	+ first_pm_pbl_idx = (info->cq_resize ?
	+ (info->virtual_map ? info->first_pm_pbl_idx : 0) :
	+ (cq->virtual_map ? cq->first_pm_pbl_idx : 0));
	+ pbl_chunk_size = (info->cq_resize ?
	+ (info->virtual_map ? info->pbl_chunk_size : 0) :
	+ (cq->virtual_map ? cq->pbl_chunk_size : 0));
	+ check_overflow = info->check_overflow_change ? info->check_overflow :
	+ cq->check_overflow;
	+ cq->cq_uk.cq_size = cq_size;
	+ cq->ceq_id_valid = ceq_id_valid;
	+ cq->ceq_id = ceq_id;
	+ cq->virtual_map = virtual_map;
	+ cq->first_pm_pbl_idx = first_pm_pbl_idx;
	+ cq->pbl_chunk_size = pbl_chunk_size;
	+ cq->check_overflow = check_overflow;
	+
	+ set_64bit_val(wqe, I40IW_BYTE_0, cq_size);
	+ set_64bit_val(wqe, I40IW_BYTE_8, RS_64_1(cq, 1));
	+ set_64bit_val(wqe, I40IW_BYTE_16,
	+ LS_64(info->shadow_read_threshold, I40IW_CQPSQ_CQ_SHADOW_READ_THRESHOLD));
	+ set_64bit_val(wqe, I40IW_BYTE_32, (cq->virtual_map ? 0 : cq->cq_pa));
	+ set_64bit_val(wqe, I40IW_BYTE_40, cq->shadow_area_pa);
	+ set_64bit_val(wqe, I40IW_BYTE_48, (cq->virtual_map ? first_pm_pbl_idx : 0));
	+ set_64bit_val(wqe, I40IW_BYTE_56, LS_64(cq->tph_val, I40IW_CQPSQ_TPHVAL));
	+
	+ header = cq->cq_uk.cq_id \|
	+ LS_64(ceq_id, I40IW_CQPSQ_CQ_CEQID) \|
	+ LS_64(I40IW_CQP_OP_MODIFY_CQ, I40IW_CQPSQ_OPCODE) \|
	+ LS_64(info->cq_resize, I40IW_CQPSQ_CQ_CQRESIZE) \|
	+ LS_64(pbl_chunk_size, I40IW_CQPSQ_CQ_LPBLSIZE) \|
	+ LS_64(check_overflow, I40IW_CQPSQ_CQ_CHKOVERFLOW) \|
	+ LS_64(virtual_map, I40IW_CQPSQ_CQ_VIRTMAP) \|
	+ LS_64(cq->ceqe_mask, I40IW_CQPSQ_CQ_ENCEQEMASK) \|
	+ LS_64(ceq_id_valid, I40IW_CQPSQ_CQ_CEQIDVALID) \|
	+ LS_64(cq->tph_en, I40IW_CQPSQ_TPHEN) \|
	+ LS_64(cq->cq_uk.avoid_mem_cflct, I40IW_CQPSQ_CQ_AVOIDMEMCNFLCT) \|
	+ LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
	+
	+ i40iw_insert_wqe_hdr(wqe, header);
	+
	+ i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "CQ_MODIFY WQE",
	+ wqe, I40IW_CQP_WQE_SIZE * 8);
	+
	+ if (post_sq)
	+ i40iw_sc_cqp_post_sq(cqp);
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_sc_qp_init - initialize qp
	+ * @qp: sc qp
	+ * @info: initialization qp info
	+ */
	+static enum i40iw_status_code
	+i40iw_sc_qp_init(struct i40iw_sc_qp *qp,
	+ struct i40iw_qp_init_info *info)
	+{
	+ u32 IOMEM *wqe_alloc_reg = NULL;
	+ enum i40iw_status_code ret_code;
	+ u32 pble_obj_cnt;
	+ u8 wqe_size;
	+ u32 offset;
	+
	+ qp->dev = info->pd->dev;
	+ qp->vsi = info->vsi;
	+ qp->sq_pa = info->sq_pa;
	+ qp->rq_pa = info->rq_pa;
	+ qp->hw_host_ctx_pa = info->host_ctx_pa;
	+ qp->q2_pa = info->q2_pa;
	+ qp->shadow_area_pa = info->shadow_area_pa;
	+
	+ qp->q2_buf = info->q2;
	+ qp->pd = info->pd;
	+ qp->hw_host_ctx = info->host_ctx;
	+ offset = (qp->pd->dev->is_pf) ? I40E_PFPE_WQEALLOC : I40E_VFPE_WQEALLOC1;
	+ if (i40iw_get_hw_addr(qp->pd->dev))
	+ wqe_alloc_reg = (u32 IOMEM *) (i40iw_get_hw_addr(qp->pd->dev) +
	+ offset);
	+
	+ info->qp_uk_init_info.wqe_alloc_reg = wqe_alloc_reg;
	+ ret_code = i40iw_qp_uk_init(&qp->qp_uk, &info->qp_uk_init_info);
	+ if (ret_code)
	+ return ret_code;
	+ qp->virtual_map = info->virtual_map;
	+
	+ pble_obj_cnt = info->pd->dev->hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].cnt;
	+
	+ if ((info->virtual_map && (info->sq_pa >= pble_obj_cnt)) \|\|
	+ (info->virtual_map && (info->rq_pa >= pble_obj_cnt)))
	+ return I40IW_ERR_INVALID_PBLE_INDEX;
	+
	+ qp->llp_stream_handle = (void *)(-1);
	+ qp->qp_type = (info->type) ? info->type : I40IW_QP_TYPE_IWARP;
	+
	+ qp->hw_sq_size = i40iw_get_encoded_wqe_size(qp->qp_uk.sq_ring.size,
	+ FALSE);
	+ i40iw_debug(qp->dev, I40IW_DEBUG_WQE, "%s: hw_sq_size[%04d] sq_ring.size[%04d]\n",
	+ __func__, qp->hw_sq_size, qp->qp_uk.sq_ring.size);
	+ ret_code = i40iw_fragcnt_to_wqesize_rq(qp->qp_uk.max_rq_frag_cnt,
	+ &wqe_size);
	+ if (ret_code)
	+ return ret_code;
	+ qp->hw_rq_size = i40iw_get_encoded_wqe_size(qp->qp_uk.rq_size *
	+ (wqe_size / I40IW_QP_WQE_MIN_SIZE), FALSE);
	+ i40iw_debug(qp->dev, I40IW_DEBUG_WQE,
	+ "%s: hw_rq_size[%04d] qp_uk.rq_size[%04d] wqe_size[%04d]\n",
	+ __func__, qp->hw_rq_size, qp->qp_uk.rq_size, wqe_size);
	+ qp->sq_tph_val = info->sq_tph_val;
	+ qp->rq_tph_val = info->rq_tph_val;
	+ qp->sq_tph_en = info->sq_tph_en;
	+ qp->rq_tph_en = info->rq_tph_en;
	+ qp->rcv_tph_en = info->rcv_tph_en;
	+ qp->xmit_tph_en = info->xmit_tph_en;
	+ qp->qs_handle = qp->vsi->qos[0].qs_handle;
	+
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_sc_qp_create - create qp
	+ * @qp: sc qp
	+ * @info: qp create info
	+ * @scratch: u64 saved to be used during cqp completion
	+ * @post_sq: flag for cqp db to ring
	+ */
	+static enum i40iw_status_code
	+i40iw_sc_qp_create(
	+ struct i40iw_sc_qp *qp,
	+ struct i40iw_create_qp_info *info,
	+ u64 scratch,
	+ bool post_sq)
	+{
	+ struct i40iw_sc_cqp *cqp;
	+ u64 *wqe;
	+ u64 header;
	+
	+ if ((qp->qp_uk.qp_id < I40IW_MIN_IW_QP_ID) \|\|
	+ (qp->qp_uk.qp_id > I40IW_MAX_IW_QP_ID))
	+ return I40IW_ERR_INVALID_QP_ID;
	+
	+ cqp = qp->pd->dev->cqp;
	+ wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
	+ if (!wqe)
	+ return I40IW_ERR_RING_FULL;
	+
	+ set_64bit_val(wqe, I40IW_BYTE_16, qp->hw_host_ctx_pa);
	+
	+ set_64bit_val(wqe, I40IW_BYTE_40, qp->shadow_area_pa);
	+
	+ header = qp->qp_uk.qp_id \|
	+ LS_64(I40IW_CQP_OP_CREATE_QP, I40IW_CQPSQ_OPCODE) \|
	+ LS_64((info->ord_valid ? 1 : 0), I40IW_CQPSQ_QP_ORDVALID) \|
	+ LS_64(info->tcp_ctx_valid, I40IW_CQPSQ_QP_TOECTXVALID) \|
	+ LS_64(qp->qp_type, I40IW_CQPSQ_QP_QPTYPE) \|
	+ LS_64(qp->virtual_map, I40IW_CQPSQ_QP_VQ) \|
	+ LS_64(info->cq_num_valid, I40IW_CQPSQ_QP_CQNUMVALID) \|
	+ LS_64(info->arp_cache_idx_valid, I40IW_CQPSQ_QP_ARPTABIDXVALID) \|
	+ LS_64(info->next_iwarp_state, I40IW_CQPSQ_QP_NEXTIWSTATE) \|
	+ LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
	+
	+ i40iw_insert_wqe_hdr(wqe, header);
	+ i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "QP_CREATE WQE",
	+ wqe, I40IW_CQP_WQE_SIZE * 8);
	+
	+ if (post_sq)
	+ i40iw_sc_cqp_post_sq(cqp);
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_sc_qp_modify - modify qp cqp wqe
	+ * @qp: sc qp
	+ * @info: modify qp info
	+ * @scratch: u64 saved to be used during cqp completion
	+ * @post_sq: flag for cqp db to ring
	+ */
	+static enum i40iw_status_code
	+i40iw_sc_qp_modify(
	+ struct i40iw_sc_qp *qp,
	+ struct i40iw_modify_qp_info *info,
	+ u64 scratch,
	+ bool post_sq)
	+{
	+ u64 *wqe;
	+ struct i40iw_sc_cqp *cqp;
	+ u64 header;
	+ u8 term_actions = 0;
	+ u8 term_len = 0;
	+
	+ cqp = qp->pd->dev->cqp;
	+ wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
	+ if (!wqe)
	+ return I40IW_ERR_RING_FULL;
	+ if (info->next_iwarp_state == I40IW_QP_STATE_TERMINATE) {
	+ if (info->dont_send_fin)
	+ term_actions += I40IWQP_TERM_SEND_TERM_ONLY;
	+ if (info->dont_send_term)
	+ term_actions += I40IWQP_TERM_SEND_FIN_ONLY;
	+ if ((term_actions == I40IWQP_TERM_SEND_TERM_AND_FIN) \|\|
	+ (term_actions == I40IWQP_TERM_SEND_TERM_ONLY))
	+ term_len = info->termlen;
	+ }
	+ set_64bit_val(wqe,
	+ I40IW_BYTE_8,
	+ LS_64(term_len, I40IW_CQPSQ_QP_TERMLEN));
	+
	+ set_64bit_val(wqe, I40IW_BYTE_16, qp->hw_host_ctx_pa);
	+ set_64bit_val(wqe, I40IW_BYTE_40, qp->shadow_area_pa);
	+
	+ header = qp->qp_uk.qp_id \|
	+ LS_64(I40IW_CQP_OP_MODIFY_QP, I40IW_CQPSQ_OPCODE) \|
	+ LS_64(info->ord_valid, I40IW_CQPSQ_QP_ORDVALID) \|
	+ LS_64(info->tcp_ctx_valid, I40IW_CQPSQ_QP_TOECTXVALID) \|
	+ LS_64(info->cached_var_valid, I40IW_CQPSQ_QP_CACHEDVARVALID) \|
	+ LS_64(qp->virtual_map, I40IW_CQPSQ_QP_VQ) \|
	+ LS_64(info->cq_num_valid, I40IW_CQPSQ_QP_CQNUMVALID) \|
	+ LS_64(info->force_loopback, I40IW_CQPSQ_QP_FORCELOOPBACK) \|
	+ LS_64(qp->qp_type, I40IW_CQPSQ_QP_QPTYPE) \|
	+ LS_64(info->remove_hash_idx, I40IW_CQPSQ_QP_REMOVEHASHENTRY) \|
	+ LS_64(term_actions, I40IW_CQPSQ_QP_TERMACT) \|
	+ LS_64(info->reset_tcp_conn, I40IW_CQPSQ_QP_RESETCON) \|
	+ LS_64(info->arp_cache_idx_valid, I40IW_CQPSQ_QP_ARPTABIDXVALID) \|
	+ LS_64(info->next_iwarp_state, I40IW_CQPSQ_QP_NEXTIWSTATE) \|
	+ LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
	+
	+ i40iw_insert_wqe_hdr(wqe, header);
	+
	+ i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "QP_MODIFY WQE",
	+ wqe, I40IW_CQP_WQE_SIZE * 8);
	+
	+ if (post_sq)
	+ i40iw_sc_cqp_post_sq(cqp);
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_sc_qp_destroy - cqp destroy qp
	+ * @qp: sc qp
	+ * @scratch: u64 saved to be used during cqp completion
	+ * @remove_hash_idx: flag if to remove hash idx
	+ * @ignore_mw_bnd: memory window bind flag
	+ * @post_sq: flag for cqp db to ring
	+ */
	+static enum i40iw_status_code
	+i40iw_sc_qp_destroy(
	+ struct i40iw_sc_qp *qp,
	+ u64 scratch,
	+ bool remove_hash_idx,
	+ bool ignore_mw_bnd,
	+ bool post_sq)
	+{
	+ u64 *wqe;
	+ struct i40iw_sc_cqp *cqp;
	+ u64 header;
	+
	+ cqp = qp->pd->dev->cqp;
	+ wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
	+ if (!wqe)
	+ return I40IW_ERR_RING_FULL;
	+ set_64bit_val(wqe, I40IW_BYTE_16, qp->hw_host_ctx_pa);
	+ set_64bit_val(wqe, I40IW_BYTE_40, qp->shadow_area_pa);
	+
	+ header = qp->qp_uk.qp_id \|
	+ LS_64(I40IW_CQP_OP_DESTROY_QP, I40IW_CQPSQ_OPCODE) \|
	+ LS_64(qp->qp_type, I40IW_CQPSQ_QP_QPTYPE) \|
	+ LS_64(ignore_mw_bnd, I40IW_CQPSQ_QP_IGNOREMWBOUND) \|
	+ LS_64(remove_hash_idx, I40IW_CQPSQ_QP_REMOVEHASHENTRY) \|
	+ LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
	+
	+ i40iw_insert_wqe_hdr(wqe, header);
	+ i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "QP_DESTROY WQE",
	+ wqe, I40IW_CQP_WQE_SIZE * 8);
	+
	+ if (post_sq)
	+ i40iw_sc_cqp_post_sq(cqp);
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_sc_qp_flush_wqes - flush qp's wqe
	+ * @qp: sc qp
	+ * @info: dlush information
	+ * @scratch: u64 saved to be used during cqp completion
	+ * @post_sq: flag for cqp db to ring
	+ */
	+static enum i40iw_status_code
	+i40iw_sc_qp_flush_wqes(
	+ struct i40iw_sc_qp *qp,
	+ struct i40iw_qp_flush_info *info,
	+ u64 scratch,
	+ bool post_sq)
	+{
	+ u64 temp = 0;
	+ u64 *wqe;
	+ struct i40iw_sc_cqp *cqp;
	+ u64 header;
	+ bool flush_sq = FALSE, flush_rq = false;
	+
	+ if (info->rq && !qp->flush_rq)
	+ flush_rq = TRUE;
	+
	+ if (info->sq && !qp->flush_sq)
	+ flush_sq = TRUE;
	+
	+ qp->flush_sq \|= flush_sq;
	+ qp->flush_rq \|= flush_rq;
	+ if (!flush_sq && !flush_rq)
	+ return 0;
	+
	+ cqp = qp->pd->dev->cqp;
	+ wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
	+ if (!wqe)
	+ return I40IW_ERR_RING_FULL;
	+ if (info->userflushcode) {
	+ if (flush_rq) {
	+ temp \|= LS_64(info->rq_minor_code, I40IW_CQPSQ_FWQE_RQMNERR) \|
	+ LS_64(info->rq_major_code, I40IW_CQPSQ_FWQE_RQMJERR);
	+ }
	+ if (flush_sq) {
	+ temp \|= LS_64(info->sq_minor_code, I40IW_CQPSQ_FWQE_SQMNERR) \|
	+ LS_64(info->sq_major_code, I40IW_CQPSQ_FWQE_SQMJERR);
	+ }
	+ }
	+ set_64bit_val(wqe, I40IW_BYTE_16, temp);
	+
	+ temp = (info->generate_ae) ?
	+ info->ae_code \| LS_64(info->ae_source, I40IW_CQPSQ_FWQE_AESOURCE) : 0;
	+
	+ set_64bit_val(wqe, I40IW_BYTE_8, temp);
	+
	+ header = qp->qp_uk.qp_id \|
	+ LS_64(I40IW_CQP_OP_FLUSH_WQES, I40IW_CQPSQ_OPCODE) \|
	+ LS_64(info->generate_ae, I40IW_CQPSQ_FWQE_GENERATE_AE) \|
	+ LS_64(info->userflushcode, I40IW_CQPSQ_FWQE_USERFLCODE) \|
	+ LS_64(flush_sq, I40IW_CQPSQ_FWQE_FLUSHSQ) \|
	+ LS_64(flush_rq, I40IW_CQPSQ_FWQE_FLUSHRQ) \|
	+ LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
	+
	+ i40iw_insert_wqe_hdr(wqe, header);
	+
	+ i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "QP_FLUSH WQE",
	+ wqe, I40IW_CQP_WQE_SIZE * 8);
	+
	+ if (post_sq)
	+ i40iw_sc_cqp_post_sq(cqp);
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_sc_gen_ae - generate AE, currently uses flush WQE CQP OP
	+ * @qp: sc qp
	+ * @info: gen ae information
	+ * @scratch: u64 saved to be used during cqp completion
	+ * @post_sq: flag for cqp db to ring
	+ */
	+static enum i40iw_status_code
	+i40iw_sc_gen_ae(
	+ struct i40iw_sc_qp *qp,
	+ struct i40iw_gen_ae_info *info,
	+ u64 scratch,
	+ bool post_sq)
	+{
	+ u64 temp;
	+ u64 *wqe;
	+ struct i40iw_sc_cqp *cqp;
	+ u64 header;
	+
	+ cqp = qp->pd->dev->cqp;
	+ wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
	+ if (!wqe)
	+ return I40IW_ERR_RING_FULL;
	+
	+ temp = info->ae_code \| LS_64(info->ae_source, I40IW_CQPSQ_FWQE_AESOURCE);
	+
	+ set_64bit_val(wqe, I40IW_BYTE_8, temp);
	+
	+ header = qp->qp_uk.qp_id \|
	+ LS_64(I40IW_CQP_OP_GEN_AE, I40IW_CQPSQ_OPCODE) \|
	+ LS_64(1, I40IW_CQPSQ_FWQE_GENERATE_AE) \|
	+ LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
	+
	+ i40iw_insert_wqe_hdr(wqe, header);
	+
	+ i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "GEN_AE WQE",
	+ wqe, I40IW_CQP_WQE_SIZE * 8);
	+
	+ if (post_sq)
	+ i40iw_sc_cqp_post_sq(cqp);
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_sc_qp_upload_context - upload qp's context
	+ * @dev: sc device struct
	+ * @info: upload context info ptr for return
	+ * @scratch: u64 saved to be used during cqp completion
	+ * @post_sq: flag for cqp db to ring
	+ */
	+static enum i40iw_status_code
	+i40iw_sc_qp_upload_context(
	+ struct i40iw_sc_dev *dev,
	+ struct i40iw_upload_context_info *info,
	+ u64 scratch,
	+ bool post_sq)
	+{
	+ u64 *wqe;
	+ struct i40iw_sc_cqp *cqp;
	+ u64 header;
	+
	+ cqp = dev->cqp;
	+ wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
	+ if (!wqe)
	+ return I40IW_ERR_RING_FULL;
	+ set_64bit_val(wqe, I40IW_BYTE_16, info->buf_pa);
	+
	+ header = LS_64(info->qp_id, I40IW_CQPSQ_UCTX_QPID) \|
	+ LS_64(I40IW_CQP_OP_UPLOAD_CONTEXT, I40IW_CQPSQ_OPCODE) \|
	+ LS_64(info->qp_type, I40IW_CQPSQ_UCTX_QPTYPE) \|
	+ LS_64(info->raw_format, I40IW_CQPSQ_UCTX_RAWFORMAT) \|
	+ LS_64(info->freeze_qp, I40IW_CQPSQ_UCTX_FREEZEQP) \|
	+ LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
	+
	+ i40iw_insert_wqe_hdr(wqe, header);
	+
	+ i40iw_debug_buf(dev, I40IW_DEBUG_WQE, "QP_UPLOAD_CTX WQE",
	+ wqe, I40IW_CQP_WQE_SIZE * 8);
	+
	+ if (post_sq)
	+ i40iw_sc_cqp_post_sq(cqp);
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_sc_qp_setctx - set qp's context
	+ * @qp: sc qp
	+ * @qp_ctx: context ptr
	+ * @info: ctx info
	+ */
	+static enum i40iw_status_code
	+i40iw_sc_qp_setctx(
	+ struct i40iw_sc_qp *qp,
	+ u64 *qp_ctx,
	+ struct i40iw_qp_host_ctx_info *info)
	+{
	+ struct i40iwarp_offload_info *iw;
	+ struct i40iw_tcp_offload_info *tcp;
	+ struct i40iw_sc_vsi *vsi;
	+ struct i40iw_sc_dev *dev;
	+ u64 qw0, qw3, qw7 = 0;
	+
	+ iw = info->iwarp_info;
	+ tcp = info->tcp_info;
	+ vsi = qp->vsi;
	+ dev = qp->dev;
	+ qw0 = LS_64(qp->qp_uk.rq_wqe_size, I40IWQPC_RQWQESIZE) \|
	+ LS_64(info->iwarp_info->err_rq_idx_valid, I40IWQPC_ERR_RQ_IDX_VALID) \|
	+ LS_64(qp->rcv_tph_en, I40IWQPC_RCVTPHEN) \|
	+ LS_64(qp->xmit_tph_en, I40IWQPC_XMITTPHEN) \|
	+ LS_64(qp->rq_tph_en, I40IWQPC_RQTPHEN) \|
	+ LS_64(qp->sq_tph_en, I40IWQPC_SQTPHEN) \|
	+ LS_64(info->push_idx, I40IWQPC_PPIDX) \|
	+ LS_64(info->push_mode_en, I40IWQPC_PMENA);
	+
	+ set_64bit_val(qp_ctx, I40IW_BYTE_8, qp->sq_pa);
	+ set_64bit_val(qp_ctx, I40IW_BYTE_16, qp->rq_pa);
	+
	+ qw3 = LS_64(qp->src_mac_addr_idx, I40IWQPC_SRCMACADDRIDX) \|
	+ LS_64(qp->hw_rq_size, I40IWQPC_RQSIZE) \|
	+ LS_64(qp->hw_sq_size, I40IWQPC_SQSIZE);
	+
	+ set_64bit_val(qp_ctx,
	+ I40IW_BYTE_128,
	+ LS_64(info->iwarp_info->err_rq_idx, I40IWQPC_ERR_RQ_IDX));
	+
	+ set_64bit_val(qp_ctx,
	+ I40IW_BYTE_136,
	+ LS_64(info->send_cq_num, I40IWQPC_TXCQNUM) \|
	+ LS_64(info->rcv_cq_num, I40IWQPC_RXCQNUM));
	+
	+ set_64bit_val(qp_ctx,
	+ I40IW_BYTE_168,
	+ LS_64(info->qp_compl_ctx, I40IWQPC_QPCOMPCTX));
	+ set_64bit_val(qp_ctx,
	+ I40IW_BYTE_176,
	+ LS_64(qp->sq_tph_val, I40IWQPC_SQTPHVAL) \|
	+ LS_64(qp->rq_tph_val, I40IWQPC_RQTPHVAL) \|
	+ LS_64(qp->qs_handle, I40IWQPC_QSHANDLE) \|
	+ LS_64(vsi->exception_lan_queue, I40IWQPC_EXCEPTION_LAN_QUEUE));
	+
	+ if (info->iwarp_info_valid) {
	+ qw0 \|= LS_64(iw->ddp_ver, I40IWQPC_DDP_VER) \|
	+ LS_64(iw->rdmap_ver, I40IWQPC_RDMAP_VER);
	+
	+ qw7 \|= LS_64(iw->pd_id, I40IWQPC_PDIDX);
	+ set_64bit_val(qp_ctx,
	+ I40IW_BYTE_144,
	+ LS_64(qp->q2_pa, I40IWQPC_Q2ADDR) \|
	+ LS_64((vsi->fcn_id - I40IW_FIRST_NON_PF_STAT), I40IWQPC_STAT_INDEX));
	+ set_64bit_val(qp_ctx,
	+ I40IW_BYTE_152,
	+ LS_64(iw->last_byte_sent, I40IWQPC_LASTBYTESENT));
	+
	+ set_64bit_val(qp_ctx,
	+ I40IW_BYTE_160,
	+ LS_64(iw->ord_size, I40IWQPC_ORDSIZE) \|
	+ LS_64(iw->ird_size, I40IWQPC_IRDSIZE) \|
	+ LS_64(iw->wr_rdresp_en, I40IWQPC_WRRDRSPOK) \|
	+ LS_64(iw->rd_enable, I40IWQPC_RDOK) \|
	+ LS_64(iw->snd_mark_en, I40IWQPC_SNDMARKERS) \|
	+ LS_64(iw->bind_en, I40IWQPC_BINDEN) \|
	+ LS_64(iw->fast_reg_en, I40IWQPC_FASTREGEN) \|
	+ LS_64(iw->priv_mode_en, I40IWQPC_PRIVEN) \|
	+ LS_64((((vsi->stats_fcn_id_alloc) &&
	+ (dev->is_pf) && (vsi->fcn_id >= I40IW_FIRST_NON_PF_STAT)) ? 1 : 0),
	+ I40IWQPC_USESTATSINSTANCE) \|
	+ LS_64(1, I40IWQPC_IWARPMODE) \|
	+ LS_64(iw->rcv_mark_en, I40IWQPC_RCVMARKERS) \|
	+ LS_64(iw->align_hdrs, I40IWQPC_ALIGNHDRS) \|
	+ LS_64(iw->rcv_no_mpa_crc, I40IWQPC_RCVNOMPACRC) \|
	+ LS_64(iw->rcv_mark_offset, I40IWQPC_RCVMARKOFFSET) \|
	+ LS_64(iw->snd_mark_offset, I40IWQPC_SNDMARKOFFSET));
	+ }
	+ if (info->tcp_info_valid) {
	+ qw0 \|= LS_64(tcp->ipv4, I40IWQPC_IPV4) \|
	+ LS_64(tcp->no_nagle, I40IWQPC_NONAGLE) \|
	+ LS_64(tcp->insert_vlan_tag, I40IWQPC_INSERTVLANTAG) \|
	+ LS_64(tcp->time_stamp, I40IWQPC_TIMESTAMP) \|
	+ LS_64(tcp->cwnd_inc_limit, I40IWQPC_LIMIT) \|
	+ LS_64(tcp->drop_ooo_seg, I40IWQPC_DROPOOOSEG) \|
	+ LS_64(tcp->dup_ack_thresh, I40IWQPC_DUPACK_THRESH);
	+
	+ qw3 \|= LS_64(tcp->ttl, I40IWQPC_TTL) \|
	+ LS_64(tcp->src_mac_addr_idx, I40IWQPC_SRCMACADDRIDX) \|
	+ LS_64(tcp->avoid_stretch_ack, I40IWQPC_AVOIDSTRETCHACK) \|
	+ LS_64(tcp->tos, I40IWQPC_TOS) \|
	+ LS_64(tcp->src_port, I40IWQPC_SRCPORTNUM) \|
	+ LS_64(tcp->dst_port, I40IWQPC_DESTPORTNUM);
	+
	+ qp->src_mac_addr_idx = tcp->src_mac_addr_idx;
	+ set_64bit_val(qp_ctx,
	+ I40IW_BYTE_32,
	+ LS_64(tcp->dest_ip_addr2, I40IWQPC_DESTIPADDR2) \|
	+ LS_64(tcp->dest_ip_addr3, I40IWQPC_DESTIPADDR3));
	+
	+ set_64bit_val(qp_ctx,
	+ I40IW_BYTE_40,
	+ LS_64(tcp->dest_ip_addr0, I40IWQPC_DESTIPADDR0) \|
	+ LS_64(tcp->dest_ip_addr1, I40IWQPC_DESTIPADDR1));
	+
	+ set_64bit_val(qp_ctx,
	+ I40IW_BYTE_48,
	+ LS_64(tcp->snd_mss, I40IWQPC_SNDMSS) \|
	+ LS_64(tcp->vlan_tag, I40IWQPC_VLANTAG) \|
	+ LS_64(tcp->arp_idx, I40IWQPC_ARPIDX));
	+
	+ qw7 \|= LS_64(tcp->flow_label, I40IWQPC_FLOWLABEL) \|
	+ LS_64(tcp->wscale, I40IWQPC_WSCALE) \|
	+ LS_64(tcp->ignore_tcp_opt, I40IWQPC_IGNORE_TCP_OPT) \|
	+ LS_64(tcp->ignore_tcp_uns_opt, I40IWQPC_IGNORE_TCP_UNS_OPT) \|
	+ LS_64(tcp->tcp_state, I40IWQPC_TCPSTATE) \|
	+ LS_64(tcp->rcv_wscale, I40IWQPC_RCVSCALE) \|
	+ LS_64(tcp->snd_wscale, I40IWQPC_SNDSCALE);
	+
	+ set_64bit_val(qp_ctx,
	+ I40IW_BYTE_72,
	+ LS_64(tcp->time_stamp_recent, I40IWQPC_TIMESTAMP_RECENT) \|
	+ LS_64(tcp->time_stamp_age, I40IWQPC_TIMESTAMP_AGE));
	+ set_64bit_val(qp_ctx,
	+ I40IW_BYTE_80,
	+ LS_64(tcp->snd_nxt, I40IWQPC_SNDNXT) \|
	+ LS_64(tcp->snd_wnd, I40IWQPC_SNDWND));
	+
	+ set_64bit_val(qp_ctx,
	+ I40IW_BYTE_88,
	+ LS_64(tcp->rcv_nxt, I40IWQPC_RCVNXT) \|
	+ LS_64(tcp->rcv_wnd, I40IWQPC_RCVWND));
	+ set_64bit_val(qp_ctx,
	+ I40IW_BYTE_96,
	+ LS_64(tcp->snd_max, I40IWQPC_SNDMAX) \|
	+ LS_64(tcp->snd_una, I40IWQPC_SNDUNA));
	+ set_64bit_val(qp_ctx,
	+ I40IW_BYTE_104,
	+ LS_64(tcp->srtt, I40IWQPC_SRTT) \|
	+ LS_64(tcp->rtt_var, I40IWQPC_RTTVAR));
	+ set_64bit_val(qp_ctx,
	+ I40IW_BYTE_112,
	+ LS_64(tcp->ss_thresh, I40IWQPC_SSTHRESH) \|
	+ LS_64(tcp->cwnd, I40IWQPC_CWND));
	+ set_64bit_val(qp_ctx,
	+ I40IW_BYTE_120,
	+ LS_64(tcp->snd_wl1, I40IWQPC_SNDWL1) \|
	+ LS_64(tcp->snd_wl2, I40IWQPC_SNDWL2));
	+ set_64bit_val(qp_ctx,
	+ I40IW_BYTE_128,
	+ LS_64(tcp->max_snd_window, I40IWQPC_MAXSNDWND) \|
	+ LS_64(tcp->rexmit_thresh, I40IWQPC_REXMIT_THRESH));
	+ set_64bit_val(qp_ctx,
	+ I40IW_BYTE_184,
	+ LS_64(tcp->local_ipaddr3, I40IWQPC_LOCAL_IPADDR3) \|
	+ LS_64(tcp->local_ipaddr2, I40IWQPC_LOCAL_IPADDR2));
	+ set_64bit_val(qp_ctx,
	+ I40IW_BYTE_192,
	+ LS_64(tcp->local_ipaddr1, I40IWQPC_LOCAL_IPADDR1) \|
	+ LS_64(tcp->local_ipaddr0, I40IWQPC_LOCAL_IPADDR0));
	+ }
	+ set_64bit_val(qp_ctx, I40IW_BYTE_0, qw0);
	+ set_64bit_val(qp_ctx, I40IW_BYTE_24, qw3);
	+ set_64bit_val(qp_ctx, I40IW_BYTE_56, qw7);
	+
	+ i40iw_debug_buf(qp->dev, I40IW_DEBUG_WQE, "QP_HOST CTX WQE",
	+ qp_ctx, I40IW_QP_CTX_SIZE);
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_sc_alloc_stag - mr stag alloc
	+ * @dev: sc device struct
	+ * @info: stag info
	+ * @scratch: u64 saved to be used during cqp completion
	+ * @post_sq: flag for cqp db to ring
	+ */
	+static enum i40iw_status_code
	+i40iw_sc_alloc_stag(
	+ struct i40iw_sc_dev *dev,
	+ struct i40iw_allocate_stag_info *info,
	+ u64 scratch,
	+ bool post_sq)
	+{
	+ u64 *wqe;
	+ struct i40iw_sc_cqp *cqp;
	+ u64 header;
	+ enum i40iw_page_size page_size;
	+
	+ page_size = (info->page_size == 0x200000) ? I40IW_PAGE_SIZE_2M : I40IW_PAGE_SIZE_4K;
	+ cqp = dev->cqp;
	+ wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
	+ if (!wqe)
	+ return I40IW_ERR_RING_FULL;
	+ set_64bit_val(wqe,
	+ I40IW_BYTE_8,
	+ LS_64(info->pd_id, I40IW_CQPSQ_STAG_PDID) \|
	+ LS_64(info->total_len, I40IW_CQPSQ_STAG_STAGLEN));
	+ set_64bit_val(wqe,
	+ I40IW_BYTE_16,
	+ LS_64(info->stag_idx, I40IW_CQPSQ_STAG_IDX));
	+ set_64bit_val(wqe,
	+ I40IW_BYTE_40,
	+ LS_64(info->hmc_fcn_index, I40IW_CQPSQ_STAG_HMCFNIDX));
	+
	+ header = LS_64(I40IW_CQP_OP_ALLOC_STAG, I40IW_CQPSQ_OPCODE) \|
	+ LS_64(1, I40IW_CQPSQ_STAG_MR) \|
	+ LS_64(info->access_rights, I40IW_CQPSQ_STAG_ARIGHTS) \|
	+ LS_64(info->chunk_size, I40IW_CQPSQ_STAG_LPBLSIZE) \|
	+ LS_64(page_size, I40IW_CQPSQ_STAG_HPAGESIZE) \|
	+ LS_64(info->remote_access, I40IW_CQPSQ_STAG_REMACCENABLED) \|
	+ LS_64(info->use_hmc_fcn_index, I40IW_CQPSQ_STAG_USEHMCFNIDX) \|
	+ LS_64(info->use_pf_rid, I40IW_CQPSQ_STAG_USEPFRID) \|
	+ LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
	+
	+ i40iw_insert_wqe_hdr(wqe, header);
	+
	+ i40iw_debug_buf(dev, I40IW_DEBUG_WQE, "ALLOC_STAG WQE",
	+ wqe, I40IW_CQP_WQE_SIZE * 8);
	+
	+ if (post_sq)
	+ i40iw_sc_cqp_post_sq(cqp);
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_sc_mr_reg_non_shared - non-shared mr registration
	+ * @dev: sc device struct
	+ * @info: mr info
	+ * @scratch: u64 saved to be used during cqp completion
	+ * @post_sq: flag for cqp db to ring
	+ */
	+static enum i40iw_status_code
	+i40iw_sc_mr_reg_non_shared(
	+ struct i40iw_sc_dev *dev,
	+ struct i40iw_reg_ns_stag_info *info,
	+ u64 scratch,
	+ bool post_sq)
	+{
	+ u64 *wqe;
	+ u64 temp;
	+ struct i40iw_sc_cqp *cqp;
	+ u64 header;
	+ u32 pble_obj_cnt;
	+ bool remote_access;
	+ u8 addr_type;
	+ enum i40iw_page_size page_size;
	+
	+ page_size = (info->page_size == 0x200000) ? I40IW_PAGE_SIZE_2M : I40IW_PAGE_SIZE_4K;
	+ if (info->access_rights & (I40IW_ACCESS_FLAGS_REMOTEREAD_ONLY \|
	+ I40IW_ACCESS_FLAGS_REMOTEWRITE_ONLY))
	+ remote_access = TRUE;
	+ else
	+ remote_access = FALSE;
	+
	+ pble_obj_cnt = dev->hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].cnt;
	+
	+ if (info->chunk_size && (info->first_pm_pbl_index >= pble_obj_cnt))
	+ return I40IW_ERR_INVALID_PBLE_INDEX;
	+
	+ cqp = dev->cqp;
	+ wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
	+ if (!wqe)
	+ return I40IW_ERR_RING_FULL;
	+
	+ temp = (info->addr_type == I40IW_ADDR_TYPE_VA_BASED) ? (uintptr_t)info->va : info->fbo;
	+ set_64bit_val(wqe, I40IW_BYTE_0, temp);
	+
	+ set_64bit_val(wqe,
	+ I40IW_BYTE_8,
	+ LS_64(info->total_len, I40IW_CQPSQ_STAG_STAGLEN) \|
	+ LS_64(info->pd_id, I40IW_CQPSQ_STAG_PDID));
	+
	+ set_64bit_val(wqe,
	+ I40IW_BYTE_16,
	+ LS_64(info->stag_key, I40IW_CQPSQ_STAG_KEY) \|
	+ LS_64(info->stag_idx, I40IW_CQPSQ_STAG_IDX));
	+ if (!info->chunk_size) {
	+ set_64bit_val(wqe, I40IW_BYTE_32, info->reg_addr_pa);
	+ set_64bit_val(wqe, I40IW_BYTE_48, 0);
	+ } else {
	+ set_64bit_val(wqe, I40IW_BYTE_32, 0);
	+ set_64bit_val(wqe, I40IW_BYTE_48, info->first_pm_pbl_index);
	+ }
	+ set_64bit_val(wqe, I40IW_BYTE_40, info->hmc_fcn_index);
	+ set_64bit_val(wqe, I40IW_BYTE_56, 0);
	+
	+ addr_type = (info->addr_type == I40IW_ADDR_TYPE_VA_BASED) ? 1 : 0;
	+ header = LS_64(I40IW_CQP_OP_REG_MR, I40IW_CQPSQ_OPCODE) \|
	+ LS_64(1, I40IW_CQPSQ_STAG_MR) \|
	+ LS_64(info->chunk_size, I40IW_CQPSQ_STAG_LPBLSIZE) \|
	+ LS_64(page_size, I40IW_CQPSQ_STAG_HPAGESIZE) \|
	+ LS_64(info->access_rights, I40IW_CQPSQ_STAG_ARIGHTS) \|
	+ LS_64(remote_access, I40IW_CQPSQ_STAG_REMACCENABLED) \|
	+ LS_64(addr_type, I40IW_CQPSQ_STAG_VABASEDTO) \|
	+ LS_64(info->use_hmc_fcn_index, I40IW_CQPSQ_STAG_USEHMCFNIDX) \|
	+ LS_64(info->use_pf_rid, I40IW_CQPSQ_STAG_USEPFRID) \|
	+ LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
	+
	+ i40iw_insert_wqe_hdr(wqe, header);
	+
	+ i40iw_debug_buf(dev, I40IW_DEBUG_WQE, "MR_REG_NS WQE",
	+ wqe, I40IW_CQP_WQE_SIZE * 8);
	+
	+ if (post_sq)
	+ i40iw_sc_cqp_post_sq(cqp);
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_sc_mr_reg_shared - registered shared memory region
	+ * @dev: sc device struct
	+ * @info: info for shared memory registeration
	+ * @scratch: u64 saved to be used during cqp completion
	+ * @post_sq: flag for cqp db to ring
	+ */
	+static enum i40iw_status_code
	+i40iw_sc_mr_reg_shared(
	+ struct i40iw_sc_dev *dev,
	+ struct i40iw_register_shared_stag *info,
	+ u64 scratch,
	+ bool post_sq)
	+{
	+ u64 *wqe;
	+ struct i40iw_sc_cqp *cqp;
	+ u64 temp, va64, fbo, header;
	+ u32 va32;
	+ bool remote_access;
	+ u8 addr_type;
	+
	+ if (info->access_rights & (I40IW_ACCESS_FLAGS_REMOTEREAD_ONLY \|
	+ I40IW_ACCESS_FLAGS_REMOTEWRITE_ONLY))
	+ remote_access = TRUE;
	+ else
	+ remote_access = FALSE;
	+ cqp = dev->cqp;
	+ wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
	+ if (!wqe)
	+ return I40IW_ERR_RING_FULL;
	+ va64 = (uintptr_t)(info->va);
	+ va32 = (u32)(va64 & 0x00000000FFFFFFFF);
	+ fbo = (u64)(va32 & (4096 - 1));
	+
	+ set_64bit_val(wqe,
	+ I40IW_BYTE_0,
	+ (info->addr_type == I40IW_ADDR_TYPE_VA_BASED ? (uintptr_t)info->va : fbo));
	+
	+ set_64bit_val(wqe,
	+ I40IW_BYTE_8,
	+ LS_64(info->pd_id, I40IW_CQPSQ_STAG_PDID));
	+ temp = LS_64(info->new_stag_key, I40IW_CQPSQ_STAG_KEY) \|
	+ LS_64(info->new_stag_idx, I40IW_CQPSQ_STAG_IDX) \|
	+ LS_64(info->parent_stag_idx, I40IW_CQPSQ_STAG_PARENTSTAGIDX);
	+ set_64bit_val(wqe, I40IW_BYTE_16, temp);
	+
	+ addr_type = (info->addr_type == I40IW_ADDR_TYPE_VA_BASED) ? 1 : 0;
	+ header = LS_64(I40IW_CQP_OP_REG_SMR, I40IW_CQPSQ_OPCODE) \|
	+ LS_64(1, I40IW_CQPSQ_STAG_MR) \|
	+ LS_64(info->access_rights, I40IW_CQPSQ_STAG_ARIGHTS) \|
	+ LS_64(remote_access, I40IW_CQPSQ_STAG_REMACCENABLED) \|
	+ LS_64(addr_type, I40IW_CQPSQ_STAG_VABASEDTO) \|
	+ LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
	+
	+ i40iw_insert_wqe_hdr(wqe, header);
	+
	+ i40iw_debug_buf(dev, I40IW_DEBUG_WQE, "MR_REG_SHARED WQE",
	+ wqe, I40IW_CQP_WQE_SIZE * 8);
	+
	+ if (post_sq)
	+ i40iw_sc_cqp_post_sq(cqp);
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_sc_dealloc_stag - deallocate stag
	+ * @dev: sc device struct
	+ * @info: dealloc stag info
	+ * @scratch: u64 saved to be used during cqp completion
	+ * @post_sq: flag for cqp db to ring
	+ */
	+static enum i40iw_status_code
	+i40iw_sc_dealloc_stag(
	+ struct i40iw_sc_dev *dev,
	+ struct i40iw_dealloc_stag_info *info,
	+ u64 scratch,
	+ bool post_sq)
	+{
	+ u64 header;
	+ u64 *wqe;
	+ struct i40iw_sc_cqp *cqp;
	+
	+ cqp = dev->cqp;
	+ wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
	+ if (!wqe)
	+ return I40IW_ERR_RING_FULL;
	+ set_64bit_val(wqe,
	+ I40IW_BYTE_8,
	+ LS_64(info->pd_id, I40IW_CQPSQ_STAG_PDID));
	+ set_64bit_val(wqe,
	+ I40IW_BYTE_16,
	+ LS_64(info->stag_idx, I40IW_CQPSQ_STAG_IDX));
	+
	+ header = LS_64(I40IW_CQP_OP_DEALLOC_STAG, I40IW_CQPSQ_OPCODE) \|
	+ LS_64(info->mr, I40IW_CQPSQ_STAG_MR) \|
	+ LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
	+
	+ i40iw_insert_wqe_hdr(wqe, header);
	+
	+ i40iw_debug_buf(dev, I40IW_DEBUG_WQE, "DEALLOC_STAG WQE",
	+ wqe, I40IW_CQP_WQE_SIZE * 8);
	+
	+ if (post_sq)
	+ i40iw_sc_cqp_post_sq(cqp);
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_sc_query_stag - query hardware for stag
	+ * @dev: sc device struct
	+ * @scratch: u64 saved to be used during cqp completion
	+ * @stag_index: stag index for query
	+ * @post_sq: flag for cqp db to ring
	+ */
	+static enum i40iw_status_code
	+i40iw_sc_query_stag(struct i40iw_sc_dev *dev,
	+ u64 scratch,
	+ u32 stag_index,
	+ bool post_sq)
	+{
	+ u64 header;
	+ u64 *wqe;
	+ struct i40iw_sc_cqp *cqp;
	+
	+ cqp = dev->cqp;
	+ wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
	+ if (!wqe)
	+ return I40IW_ERR_RING_FULL;
	+ set_64bit_val(wqe,
	+ I40IW_BYTE_16,
	+ LS_64(stag_index, I40IW_CQPSQ_QUERYSTAG_IDX));
	+
	+ header = LS_64(I40IW_CQP_OP_QUERY_STAG, I40IW_CQPSQ_OPCODE) \|
	+ LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
	+
	+ i40iw_insert_wqe_hdr(wqe, header);
	+
	+ i40iw_debug_buf(dev, I40IW_DEBUG_WQE, "QUERY_STAG WQE",
	+ wqe, I40IW_CQP_WQE_SIZE * 8);
	+
	+ if (post_sq)
	+ i40iw_sc_cqp_post_sq(cqp);
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_sc_mw_alloc - mw allocate
	+ * @dev: sc device struct
	+ * @scratch: u64 saved to be used during cqp completion
	+ * @mw_stag_index:stag index
	+ * @pd_id: pd is for this mw
	+ * @post_sq: flag for cqp db to ring
	+ */
	+static enum i40iw_status_code
	+i40iw_sc_mw_alloc(
	+ struct i40iw_sc_dev *dev,
	+ u64 scratch,
	+ u32 mw_stag_index,
	+ u16 pd_id,
	+ bool post_sq)
	+{
	+ u64 header;
	+ struct i40iw_sc_cqp *cqp;
	+ u64 *wqe;
	+
	+ cqp = dev->cqp;
	+ wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
	+ if (!wqe)
	+ return I40IW_ERR_RING_FULL;
	+ set_64bit_val(wqe, I40IW_BYTE_8, LS_64(pd_id, I40IW_CQPSQ_STAG_PDID));
	+ set_64bit_val(wqe,
	+ I40IW_BYTE_16,
	+ LS_64(mw_stag_index, I40IW_CQPSQ_STAG_IDX));
	+
	+ header = LS_64(I40IW_CQP_OP_ALLOC_STAG, I40IW_CQPSQ_OPCODE) \|
	+ LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
	+
	+ i40iw_insert_wqe_hdr(wqe, header);
	+
	+ i40iw_debug_buf(dev, I40IW_DEBUG_WQE, "MW_ALLOC WQE",
	+ wqe, I40IW_CQP_WQE_SIZE * 8);
	+
	+ if (post_sq)
	+ i40iw_sc_cqp_post_sq(cqp);
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_sc_mr_fast_register - Posts RDMA fast register mr WR to iwarp qp
	+ * @qp: sc qp struct
	+ * @info: fast mr info
	+ * @post_sq: flag for cqp db to ring
	+ */
	+enum i40iw_status_code
	+i40iw_sc_mr_fast_register(
	+ struct i40iw_sc_qp *qp,
	+ struct i40iw_fast_reg_stag_info *info,
	+ bool post_sq)
	+{
	+ u64 temp, header;
	+ u64 *wqe;
	+ u32 wqe_idx;
	+ enum i40iw_page_size page_size;
	+
	+ page_size = (info->page_size == 0x200000) ? I40IW_PAGE_SIZE_2M : I40IW_PAGE_SIZE_4K;
	+
	+ wqe = i40iw_qp_get_next_send_wqe(&qp->qp_uk, &wqe_idx, I40IW_QP_WQE_MIN_SIZE,
	+ 0, info->wr_id);
	+ if (!wqe)
	+ return I40IW_ERR_QP_TOOMANY_WRS_POSTED;
	+
	+ i40iw_debug(qp->dev, I40IW_DEBUG_MR, "%s: wr_id[%lxh] wqe_idx[%04d] location[%p]\n",
	+ __func__, info->wr_id, wqe_idx,
	+ &qp->qp_uk.sq_wrtrk_array[wqe_idx].wrid);
	+ temp = (info->addr_type == I40IW_ADDR_TYPE_VA_BASED) ? (uintptr_t)info->va : info->fbo;
	+ set_64bit_val(wqe, I40IW_BYTE_0, temp);
	+
	+ temp = RS_64(info->first_pm_pbl_index >> 16, I40IWQPSQ_FIRSTPMPBLIDXHI);
	+ set_64bit_val(wqe,
	+ I40IW_BYTE_8,
	+ LS_64(temp, I40IWQPSQ_FIRSTPMPBLIDXHI) \|
	+ LS_64(info->reg_addr_pa >> I40IWQPSQ_PBLADDR_SHIFT, I40IWQPSQ_PBLADDR));
	+
	+ set_64bit_val(wqe,
	+ I40IW_BYTE_16,
	+ info->total_len \|
	+ LS_64(info->first_pm_pbl_index, I40IWQPSQ_FIRSTPMPBLIDXLO));
	+
	+ header = LS_64(info->stag_key, I40IWQPSQ_STAGKEY) \|
	+ LS_64(info->stag_idx, I40IWQPSQ_STAGINDEX) \|
	+ LS_64(I40IWQP_OP_FAST_REGISTER, I40IWQPSQ_OPCODE) \|
	+ LS_64(info->chunk_size, I40IWQPSQ_LPBLSIZE) \|
	+ LS_64(page_size, I40IWQPSQ_HPAGESIZE) \|
	+ LS_64(info->access_rights, I40IWQPSQ_STAGRIGHTS) \|
	+ LS_64(info->addr_type, I40IWQPSQ_VABASEDTO) \|
	+ LS_64(info->read_fence, I40IWQPSQ_READFENCE) \|
	+ LS_64(info->local_fence, I40IWQPSQ_LOCALFENCE) \|
	+ LS_64(info->signaled, I40IWQPSQ_SIGCOMPL) \|
	+ LS_64(qp->qp_uk.swqe_polarity, I40IWQPSQ_VALID);
	+
	+ i40iw_insert_wqe_hdr(wqe, header);
	+
	+ i40iw_debug_buf(qp->dev, I40IW_DEBUG_WQE, "FAST_REG WQE",
	+ wqe, I40IW_QP_WQE_MIN_SIZE);
	+
	+ if (post_sq)
	+ i40iw_qp_post_wr(&qp->qp_uk);
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_sc_send_lsmm - send last streaming mode message
	+ * @qp: sc qp struct
	+ * @lsmm_buf: buffer with lsmm message
	+ * @size: size of lsmm buffer
	+ * @stag: stag of lsmm buffer
	+ */
	+static void
	+i40iw_sc_send_lsmm(struct i40iw_sc_qp *qp,
	+ void *lsmm_buf,
	+ u32 size,
	+ i40iw_stag stag)
	+{
	+ u64 *wqe;
	+ u64 header;
	+ struct i40iw_qp_uk *qp_uk;
	+
	+ qp_uk = &qp->qp_uk;
	+ wqe = qp_uk->sq_base->elem;
	+
	+ set_64bit_val(wqe, I40IW_BYTE_0, (uintptr_t)lsmm_buf);
	+
	+ set_64bit_val(wqe, I40IW_BYTE_8, (size \| LS_64(stag, I40IWQPSQ_FRAG_STAG)));
	+
	+ set_64bit_val(wqe, I40IW_BYTE_16, 0);
	+
	+ header = LS_64(I40IWQP_OP_RDMA_SEND, I40IWQPSQ_OPCODE) \|
	+ LS_64(1, I40IWQPSQ_STREAMMODE) \|
	+ LS_64(1, I40IWQPSQ_WAITFORRCVPDU) \|
	+ LS_64(qp->qp_uk.swqe_polarity, I40IWQPSQ_VALID);
	+
	+ i40iw_insert_wqe_hdr(wqe, header);
	+
	+ i40iw_debug_buf(qp->dev, I40IW_DEBUG_QP, "SEND_LSMM WQE",
	+ wqe, I40IW_QP_WQE_MIN_SIZE);
	+}
	+
	+/**
	+ * i40iw_sc_send_lsmm_nostag - for privilege qp
	+ * @qp: sc qp struct
	+ * @lsmm_buf: buffer with lsmm message
	+ * @size: size of lsmm buffer
	+ */
	+static void
	+i40iw_sc_send_lsmm_nostag(struct i40iw_sc_qp *qp,
	+ void *lsmm_buf,
	+ u32 size)
	+{
	+ u64 *wqe;
	+ u64 header;
	+ struct i40iw_qp_uk *qp_uk;
	+
	+ qp_uk = &qp->qp_uk;
	+ wqe = qp_uk->sq_base->elem;
	+
	+ set_64bit_val(wqe, I40IW_BYTE_0, (uintptr_t)lsmm_buf);
	+
	+ set_64bit_val(wqe, I40IW_BYTE_8, size);
	+
	+ set_64bit_val(wqe, I40IW_BYTE_16, 0);
	+
	+ header = LS_64(I40IWQP_OP_RDMA_SEND, I40IWQPSQ_OPCODE) \|
	+ LS_64(1, I40IWQPSQ_STREAMMODE) \|
	+ LS_64(1, I40IWQPSQ_WAITFORRCVPDU) \|
	+ LS_64(qp->qp_uk.swqe_polarity, I40IWQPSQ_VALID);
	+
	+ i40iw_insert_wqe_hdr(wqe, header);
	+
	+ i40iw_debug_buf(qp->dev, I40IW_DEBUG_WQE, "SEND_LSMM_NOSTAG WQE",
	+ wqe, I40IW_QP_WQE_MIN_SIZE);
	+}
	+
	+/**
	+ * i40iw_sc_send_rtt - send last read0 or write0
	+ * @qp: sc qp struct
	+ * @read: Do read0 or write0
	+ */
	+static void
	+i40iw_sc_send_rtt(struct i40iw_sc_qp *qp, bool read)
	+{
	+ u64 *wqe;
	+ u64 header;
	+ struct i40iw_qp_uk *qp_uk;
	+
	+ qp_uk = &qp->qp_uk;
	+ wqe = qp_uk->sq_base->elem;
	+
	+ set_64bit_val(wqe, I40IW_BYTE_0, 0);
	+ set_64bit_val(wqe, I40IW_BYTE_8, 0);
	+ set_64bit_val(wqe, I40IW_BYTE_16, 0);
	+ if (read) {
	+ header = LS_64(0x1234, I40IWQPSQ_REMSTAG) \|
	+ LS_64(I40IWQP_OP_RDMA_READ, I40IWQPSQ_OPCODE) \|
	+ LS_64(qp->qp_uk.swqe_polarity, I40IWQPSQ_VALID);
	+ set_64bit_val(wqe, I40IW_BYTE_8, ((u64)0xabcd << 32));
	+ } else {
	+ header = LS_64(I40IWQP_OP_RDMA_WRITE, I40IWQPSQ_OPCODE) \|
	+ LS_64(qp->qp_uk.swqe_polarity, I40IWQPSQ_VALID);
	+ }
	+
	+ i40iw_insert_wqe_hdr(wqe, header);
	+
	+ i40iw_debug_buf(qp->dev, I40IW_DEBUG_WQE, "RTR WQE",
	+ wqe, I40IW_QP_WQE_MIN_SIZE);
	+}
	+
	+/**
	+ * i40iw_sc_init_iw_hmc() - queries fpm values using cqp and populates hmc_info
	+ * @dev : ptr to i40iw_dev struct
	+ * @hmc_fn_id: hmc function id
	+ */
	+enum i40iw_status_code
	+i40iw_sc_init_iw_hmc(struct i40iw_sc_dev *dev, u8 hmc_fn_id)
	+{
	+ struct i40iw_hmc_info *hmc_info;
	+ struct i40iw_dma_mem query_fpm_mem;
	+ enum i40iw_status_code ret_code = 0;
	+ bool poll_registers = TRUE;
	+ u8 wait_type;
	+
	+ if (hmc_fn_id > I40IW_MAX_VF_FPM_ID \|\|
	+ (dev->hmc_fn_id != hmc_fn_id && hmc_fn_id < I40IW_FIRST_VF_FPM_ID))
	+ return I40IW_ERR_INVALID_HMCFN_ID;
	+
	+ i40iw_debug(dev, I40IW_DEBUG_HMC, "hmc_fn_id %u, dev->hmc_fn_id %u\n", hmc_fn_id,
	+ dev->hmc_fn_id);
	+ hmc_info = dev->hmc_info;
	+ query_fpm_mem.pa = dev->fpm_query_buf_pa;
	+ query_fpm_mem.va = dev->fpm_query_buf;
	+
	+ hmc_info->hmc_fn_id = hmc_fn_id;
	+
	+ wait_type = poll_registers ? (u8)I40IW_CQP_WAIT_POLL_REGS :
	+ (u8)I40IW_CQP_WAIT_POLL_CQ;
	+
	+ ret_code = i40iw_sc_query_fpm_values(
	+ dev->cqp,
	+ 0,
	+ hmc_info->hmc_fn_id,
	+ &query_fpm_mem,
	+ TRUE,
	+ wait_type);
	+ if (ret_code)
	+ return ret_code;
	+
	+ /* parse the fpm_query_buf and fill hmc obj info */
	+ ret_code =
	+ i40iw_sc_parse_fpm_query_buf((u64 *)query_fpm_mem.va,
	+ hmc_info,
	+ &dev->hmc_fpm_misc);
	+ if (ret_code)
	+ return ret_code;
	+ i40iw_debug_buf(dev, I40IW_DEBUG_HMC, "QUERY FPM BUFFER",
	+ query_fpm_mem.va, I40IW_QUERY_FPM_BUF_SIZE);
	+
	+ return ret_code;
	+}
	+
	+/**
	+ * i40iw_sc_configure_iw_fpm() - commits hmc obj cnt values using cqp command and
	+ * populates fpm base address in hmc_info
	+ * @dev : ptr to i40iw_dev struct
	+ * @hmc_fn_id: hmc function id
	+ */
	+static enum i40iw_status_code
	+i40iw_sc_configure_iw_fpm(struct i40iw_sc_dev *dev,
	+ u8 hmc_fn_id)
	+{
	+ struct i40iw_hmc_info *hmc_info;
	+ struct i40iw_hmc_obj_info *obj_info;
	+ u64 *buf;
	+ struct i40iw_dma_mem commit_fpm_mem;
	+ u32 i, j;
	+ enum i40iw_status_code ret_code = 0;
	+ bool poll_registers = TRUE;
	+ u8 wait_type;
	+
	+ if (hmc_fn_id > I40IW_MAX_VF_FPM_ID \|\|
	+ (dev->hmc_fn_id != hmc_fn_id && hmc_fn_id < I40IW_FIRST_VF_FPM_ID))
	+ return I40IW_ERR_INVALID_HMCFN_ID;
	+
	+ if (hmc_fn_id == dev->hmc_fn_id) {
	+ hmc_info = dev->hmc_info;
	+ } else {
	+ hmc_info = NULL;
	+ }
	+ if (!hmc_info)
	+ return I40IW_ERR_BAD_PTR;
	+
	+ obj_info = hmc_info->hmc_obj;
	+ buf = dev->fpm_commit_buf;
	+
	+ /* copy cnt values in commit buf */
	+ for (i = I40IW_HMC_IW_QP, j = I40IW_BYTE_0; i <= I40IW_HMC_IW_PBLE;
	+ i++, j += I40IW_BYTE_8)
	+ set_64bit_val(buf, j, (u64)obj_info[i].cnt);
	+
	+ set_64bit_val(buf, I40IW_BYTE_40, 0); /* APBVT rsvd */
	+
	+ commit_fpm_mem.pa = dev->fpm_commit_buf_pa;
	+ commit_fpm_mem.va = dev->fpm_commit_buf;
	+ wait_type = poll_registers ? (u8)I40IW_CQP_WAIT_POLL_REGS :
	+ (u8)I40IW_CQP_WAIT_POLL_CQ;
	+ ret_code = i40iw_sc_commit_fpm_values(
	+ dev->cqp,
	+ 0,
	+ hmc_info->hmc_fn_id,
	+ &commit_fpm_mem,
	+ TRUE,
	+ wait_type);
	+
	+ /* parse the fpm_commit_buf and fill hmc obj info */
	+ if (!ret_code)
	+ ret_code = i40iw_sc_parse_fpm_commit_buf(dev->fpm_commit_buf,
	+ hmc_info->hmc_obj,
	+ &hmc_info->sd_table.sd_cnt);
	+
	+ i40iw_debug_buf(dev, I40IW_DEBUG_HMC, "COMMIT FPM BUFFER",
	+ commit_fpm_mem.va, I40IW_COMMIT_FPM_BUF_SIZE);
	+
	+ return ret_code;
	+}
	+
	+/**
	+ * cqp_sds_wqe_fill - fill cqp wqe doe sd
	+ * @cqp: struct for cqp hw
	+ * @info; sd info for wqe
	+ * @scratch: u64 saved to be used during cqp completion
	+ */
	+static enum i40iw_status_code
	+cqp_sds_wqe_fill(struct i40iw_sc_cqp *cqp,
	+ struct i40iw_update_sds_info *info,
	+ u64 scratch)
	+{
	+ u64 data;
	+ u64 header;
	+ u64 *wqe;
	+ int mem_entries, wqe_entries;
	+ struct i40iw_dma_mem *sdbuf = &cqp->sdbuf;
	+ u64 offset;
	+ u32 wqe_idx;
	+
	+ wqe = i40iw_sc_cqp_get_next_send_wqe_idx(cqp, scratch, &wqe_idx);
	+ if (!wqe)
	+ return I40IW_ERR_RING_FULL;
	+
	+ I40IW_CQP_INIT_WQE(wqe);
	+ wqe_entries = (info->cnt > 3) ? 3 : info->cnt;
	+ mem_entries = info->cnt - wqe_entries;
	+
	+ header = LS_64(I40IW_CQP_OP_UPDATE_PE_SDS, I40IW_CQPSQ_OPCODE) \|
	+ LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID) \|
	+ LS_64(mem_entries, I40IW_CQPSQ_UPESD_ENTRY_COUNT);
	+
	+ if (mem_entries) {
	+ offset = wqe_idx * 128;
	+ i40iw_memcpy(((char *)sdbuf->va + offset), &info->entry[3], mem_entries << 4);
	+ data = (u64)sdbuf->pa + offset;
	+ } else {
	+ data = 0;
	+ }
	+ data \|= LS_64(info->hmc_fn_id, I40IW_CQPSQ_UPESD_HMCFNID);
	+
	+ set_64bit_val(wqe, I40IW_BYTE_16, data);
	+
	+ switch (wqe_entries) {
	+ case 3:
	+ set_64bit_val(wqe, I40IW_BYTE_48,
	+ (LS_64(info->entry[2].cmd, I40IW_CQPSQ_UPESD_SDCMD) \|
	+ LS_64(1, I40IW_CQPSQ_UPESD_ENTRY_VALID)));
	+
	+ set_64bit_val(wqe, I40IW_BYTE_56, info->entry[2].data);
	+ /* fallthrough */
	+ case 2:
	+ set_64bit_val(wqe, I40IW_BYTE_32,
	+ (LS_64(info->entry[1].cmd, I40IW_CQPSQ_UPESD_SDCMD) \|
	+ LS_64(1, I40IW_CQPSQ_UPESD_ENTRY_VALID)));
	+
	+ set_64bit_val(wqe, I40IW_BYTE_40, info->entry[1].data);
	+ /* fallthrough */
	+ case 1:
	+ set_64bit_val(wqe, I40IW_BYTE_0,
	+ LS_64(info->entry[0].cmd, I40IW_CQPSQ_UPESD_SDCMD));
	+
	+ set_64bit_val(wqe, I40IW_BYTE_8, info->entry[0].data);
	+ break;
	+ default:
	+ break;
	+ }
	+
	+ i40iw_insert_wqe_hdr(wqe, header);
	+
	+ i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "UPDATE_PE_SDS WQE",
	+ wqe, I40IW_CQP_WQE_SIZE * 8);
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_update_pe_sds - cqp wqe for sd
	+ * @dev: ptr to i40iw_dev struct
	+ * @info: sd info for sd's
	+ * @scratch: u64 saved to be used during cqp completion
	+ */
	+static enum i40iw_status_code
	+i40iw_update_pe_sds(struct i40iw_sc_dev *dev,
	+ struct i40iw_update_sds_info *info,
	+ u64 scratch)
	+{
	+ struct i40iw_sc_cqp *cqp = dev->cqp;
	+ enum i40iw_status_code ret_code;
	+
	+ ret_code = cqp_sds_wqe_fill(cqp, info, scratch);
	+ if (!ret_code)
	+ i40iw_sc_cqp_post_sq(cqp);
	+
	+ return ret_code;
	+}
	+
	+/**
	+ * i40iw_update_sds_noccq - update sd before ccq created
	+ * @dev: sc device struct
	+ * @info: sd info for sd's
	+ */
	+enum i40iw_status_code
	+i40iw_update_sds_noccq(struct i40iw_sc_dev *dev,
	+ struct i40iw_update_sds_info *info)
	+{
	+ u32 error, val, tail;
	+ struct i40iw_sc_cqp *cqp = dev->cqp;
	+ enum i40iw_status_code ret_code;
	+
	+ ret_code = cqp_sds_wqe_fill(cqp, info, 0);
	+ if (ret_code)
	+ return ret_code;
	+ i40iw_get_cqp_reg_info(cqp, &val, &tail, &error);
	+ if (error)
	+ return I40IW_ERR_CQP_COMPL_ERROR;
	+
	+ i40iw_sc_cqp_post_sq(cqp);
	+ ret_code = i40iw_cqp_poll_registers(cqp, tail, I40IW_DONE_COUNT);
	+
	+ return ret_code;
	+}
	+
	+/**
	+ * i40iw_sc_suspend_qp - suspend qp for param change
	+ * @cqp: struct for cqp hw
	+ * @qp: sc qp struct
	+ * @scratch: u64 saved to be used during cqp completion
	+ */
	+enum i40iw_status_code
	+i40iw_sc_suspend_qp(struct i40iw_sc_cqp *cqp,
	+ struct i40iw_sc_qp *qp,
	+ u64 scratch)
	+{
	+ u64 header;
	+ u64 *wqe;
	+
	+ wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
	+ if (!wqe)
	+ return I40IW_ERR_RING_FULL;
	+ header = LS_64(qp->qp_uk.qp_id, I40IW_CQPSQ_SUSPENDQP_QPID) \|
	+ LS_64(I40IW_CQP_OP_SUSPEND_QP, I40IW_CQPSQ_OPCODE) \|
	+ LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
	+
	+ i40iw_insert_wqe_hdr(wqe, header);
	+
	+ i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "SUSPEND_QP WQE",
	+ wqe, I40IW_CQP_WQE_SIZE * 8);
	+
	+ i40iw_sc_cqp_post_sq(cqp);
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_sc_resume_qp - resume qp after suspend
	+ * @cqp: struct for cqp hw
	+ * @qp: sc qp struct
	+ * @scratch: u64 saved to be used during cqp completion
	+ */
	+enum i40iw_status_code
	+i40iw_sc_resume_qp(struct i40iw_sc_cqp *cqp,
	+ struct i40iw_sc_qp *qp,
	+ u64 scratch)
	+{
	+ u64 header;
	+ u64 *wqe;
	+
	+ wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
	+ if (!wqe)
	+ return I40IW_ERR_RING_FULL;
	+ set_64bit_val(wqe,
	+ I40IW_BYTE_16,
	+ LS_64(qp->qs_handle, I40IW_CQPSQ_RESUMEQP_QSHANDLE));
	+
	+ header = LS_64(qp->qp_uk.qp_id, I40IW_CQPSQ_RESUMEQP_QPID) \|
	+ LS_64(I40IW_CQP_OP_RESUME_QP, I40IW_CQPSQ_OPCODE) \|
	+ LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
	+
	+ i40iw_insert_wqe_hdr(wqe, header);
	+
	+ i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "RESUME_QP WQE",
	+ wqe, I40IW_CQP_WQE_SIZE * 8);
	+
	+ i40iw_sc_cqp_post_sq(cqp);
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_sc_static_hmc_pages_allocated - cqp wqe to allocate hmc pages
	+ * @cqp: struct for cqp hw
	+ * @scratch: u64 saved to be used during cqp completion
	+ * @hmc_fn_id: hmc function id
	+ * @post_sq: flag for cqp db to ring
	+ * @poll_registers: flag to poll register for cqp completion
	+ */
	+enum i40iw_status_code
	+i40iw_sc_static_hmc_pages_allocated(
	+ struct i40iw_sc_cqp *cqp,
	+ u64 scratch,
	+ u8 hmc_fn_id,
	+ bool post_sq,
	+ bool poll_registers)
	+{
	+ u64 header;
	+ u64 *wqe;
	+ u32 tail, val, error;
	+ enum i40iw_status_code ret_code = 0;
	+
	+ wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
	+ if (!wqe)
	+ return I40IW_ERR_RING_FULL;
	+ set_64bit_val(wqe,
	+ I40IW_BYTE_16,
	+ LS_64(hmc_fn_id, I40IW_SHMC_PAGE_ALLOCATED_HMC_FN_ID));
	+
	+ header = LS_64(I40IW_CQP_OP_SHMC_PAGES_ALLOCATED, I40IW_CQPSQ_OPCODE) \|
	+ LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
	+
	+ i40iw_insert_wqe_hdr(wqe, header);
	+
	+ i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "SHMC_PAGES_ALLOCATED WQE",
	+ wqe, I40IW_CQP_WQE_SIZE * 8);
	+ i40iw_get_cqp_reg_info(cqp, &val, &tail, &error);
	+ if (error) {
	+ ret_code = I40IW_ERR_CQP_COMPL_ERROR;
	+ return ret_code;
	+ }
	+ if (post_sq) {
	+ i40iw_sc_cqp_post_sq(cqp);
	+ if (poll_registers)
	+ /* check for cqp sq tail update */
	+ ret_code = i40iw_cqp_poll_registers(cqp, tail, 1000);
	+ else
	+ ret_code = i40iw_sc_poll_for_cqp_op_done(cqp,
	+ I40IW_CQP_OP_SHMC_PAGES_ALLOCATED,
	+ NULL);
	+ }
	+ return ret_code;
	+}
	+
	+/**
	+ * i40iw_ring_full - check if cqp ring is full
	+ * @cqp: struct for cqp hw
	+ */
	+static bool
	+i40iw_ring_full(struct i40iw_sc_cqp *cqp)
	+{
	+ return I40IW_RING_FULL_ERR(cqp->sq_ring);
	+}
	+
	+/**
	+ * i40iw_est_sd - returns approximate number of SDs for HMC
	+ * @dev: sc device struct
	+ * @hmc_info: hmc structure, size and count for HMC objects
	+ */
	+static u64
	+i40iw_est_sd(struct i40iw_sc_dev dev, struct i40iw_hmc_info hmc_info)
	+{
	+ int i;
	+ u64 size = 0;
	+ u64 sd;
	+
	+ for (i = I40IW_HMC_IW_QP; i < I40IW_HMC_IW_PBLE; i++)
	+ size += hmc_info->hmc_obj[i].cnt * hmc_info->hmc_obj[i].size;
	+
	+ if (dev->is_pf)
	+ size += hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].cnt * hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].size;
	+
	+ if (size & 0x1FFFFF)
	+ sd = (size >> 21) + 1; /* add 1 for remainder */
	+ else
	+ sd = size >> 21;
	+
	+ if (!dev->is_pf) {
	+ /* 2MB alignment for VF PBLE HMC */
	+ size = hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].cnt * hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].size;
	+ if (size & 0x1FFFFF)
	+ sd += (size >> 21) + 1; /* add 1 for remainder */
	+ else
	+ sd += size >> 21;
	+ }
	+ return sd;
	+}
	+
	+/**
	+ * i40iw_config_fpm_values - configure HMC objects
	+ * @dev: sc device struct
	+ * @qp_count: desired qp count
	+ */
	+enum i40iw_status_code
	+i40iw_config_fpm_values(struct i40iw_sc_dev *dev, u32 qp_count)
	+{
	+ struct i40iw_virt_mem virt_mem;
	+ u32 i, mem_size;
	+ u32 qpwantedoriginal, qpwanted, mrwanted, pblewanted;
	+ u64 sd_needed;
	+ u32 loop_count = 0;
	+
	+ struct i40iw_hmc_info *hmc_info;
	+ struct i40iw_hmc_fpm_misc *hmc_fpm_misc;
	+ enum i40iw_status_code ret_code = 0;
	+
	+ hmc_info = dev->hmc_info;
	+ hmc_fpm_misc = &dev->hmc_fpm_misc;
	+
	+ ret_code = i40iw_sc_init_iw_hmc(dev, dev->hmc_fn_id);
	+ if (ret_code) {
	+ i40iw_debug(dev, I40IW_DEBUG_HMC,
	+ "i40iw_sc_init_iw_hmc returned error_code = %d\n",
	+ ret_code);
	+ return ret_code;
	+ }
	+ for (i = I40IW_HMC_IW_QP; i < I40IW_HMC_IW_MAX; i++)
	+ hmc_info->hmc_obj[i].cnt = hmc_info->hmc_obj[i].max_cnt;
	+ sd_needed = i40iw_est_sd(dev, hmc_info);
	+ i40iw_debug(dev, I40IW_DEBUG_HMC,
	+ "%s: FW initial max sd_count[%08ld] first_sd_index[%04d]\n",
	+ __func__, sd_needed, hmc_info->first_sd_index);
	+
	+ i40iw_debug(dev, I40IW_DEBUG_HMC,
	+ "%s: sd count %d where max sd is %d\n",
	+ __func__, hmc_info->sd_table.sd_cnt,
	+ hmc_fpm_misc->max_sds);
	+
	+ qpwanted = min(qp_count, hmc_info->hmc_obj[I40IW_HMC_IW_QP].max_cnt);
	+ qpwantedoriginal = qpwanted;
	+ mrwanted = hmc_info->hmc_obj[I40IW_HMC_IW_MR].max_cnt;
	+ pblewanted = hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].max_cnt;
	+
	+ i40iw_debug(dev, I40IW_DEBUG_HMC,
	+ "req_qp=%d max_sd=%d, max_qp = %d, max_cq=%d, max_mr=%d, max_pble=%d\n",
	+ qp_count, hmc_fpm_misc->max_sds,
	+ hmc_info->hmc_obj[I40IW_HMC_IW_QP].max_cnt,
	+ hmc_info->hmc_obj[I40IW_HMC_IW_CQ].max_cnt,
	+ hmc_info->hmc_obj[I40IW_HMC_IW_MR].max_cnt,
	+ hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].max_cnt);
	+
	+ do {
	+ ++loop_count;
	+ hmc_info->hmc_obj[I40IW_HMC_IW_QP].cnt = qpwanted;
	+ hmc_info->hmc_obj[I40IW_HMC_IW_CQ].cnt =
	+ min(2 * qpwanted, hmc_info->hmc_obj[I40IW_HMC_IW_CQ].cnt);
	+ hmc_info->hmc_obj[I40IW_HMC_IW_SRQ].cnt = 0x00; /* Reserved */
	+ hmc_info->hmc_obj[I40IW_HMC_IW_HTE].cnt =
	+ qpwanted * hmc_fpm_misc->ht_multiplier;
	+ hmc_info->hmc_obj[I40IW_HMC_IW_ARP].cnt =
	+ hmc_info->hmc_obj[I40IW_HMC_IW_ARP].max_cnt;
	+ hmc_info->hmc_obj[I40IW_HMC_IW_APBVT_ENTRY].cnt = 1;
	+ hmc_info->hmc_obj[I40IW_HMC_IW_MR].cnt = mrwanted;
	+
	+ hmc_info->hmc_obj[I40IW_HMC_IW_XF].cnt =
	+ i40iw_powerof2_roundup(I40IW_MAX_WQ_ENTRIES * qpwanted);
	+ hmc_info->hmc_obj[I40IW_HMC_IW_Q1].cnt =
	+ i40iw_powerof2_roundup(2 * I40IW_MAX_IRD_SIZE * qpwanted);
	+ hmc_info->hmc_obj[I40IW_HMC_IW_XFFL].cnt =
	+ hmc_info->hmc_obj[I40IW_HMC_IW_XF].cnt / hmc_fpm_misc->xf_block_size;
	+ hmc_info->hmc_obj[I40IW_HMC_IW_Q1FL].cnt =
	+ hmc_info->hmc_obj[I40IW_HMC_IW_Q1].cnt / hmc_fpm_misc->q1_block_size;
	+ hmc_info->hmc_obj[I40IW_HMC_IW_TIMER].cnt =
	+ ((qpwanted) / 512 + 1) * hmc_fpm_misc->timer_bucket;
	+ hmc_info->hmc_obj[I40IW_HMC_IW_FSIMC].cnt = 0x00;
	+ hmc_info->hmc_obj[I40IW_HMC_IW_FSIAV].cnt = 0x00;
	+ hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].cnt = pblewanted;
	+
	+ /* How much memory is needed for all the objects. */
	+ sd_needed = i40iw_est_sd(dev, hmc_info);
	+ if ((loop_count > 1000) \|\|
	+ ((!(loop_count % 10)) &&
	+ (qpwanted > qpwantedoriginal * 2 / 3))) {
	+ if (qpwanted > FPM_MULTIPLIER)
	+ qpwanted = i40iw_powerof2_roundup(qpwanted - FPM_MULTIPLIER) >> 1;
	+ else
	+ qpwanted /= 2;
	+ }
	+ if (mrwanted > FPM_MULTIPLIER * 10)
	+ mrwanted -= FPM_MULTIPLIER * 10;
	+ if (pblewanted > FPM_MULTIPLIER * 1000)
	+ pblewanted -= FPM_MULTIPLIER * 1000;
	+ } while (sd_needed > hmc_fpm_misc->max_sds && loop_count < 2000);
	+
	+ sd_needed = i40iw_est_sd(dev, hmc_info);
	+
	+ i40iw_debug(dev, I40IW_DEBUG_HMC,
	+ "loop_cnt=%d, sd_needed=%ld, qpcnt = %d, cqcnt=%d, mrcnt=%d, pblecnt=%d\n",
	+ loop_count, sd_needed,
	+ hmc_info->hmc_obj[I40IW_HMC_IW_QP].cnt,
	+ hmc_info->hmc_obj[I40IW_HMC_IW_CQ].cnt,
	+ hmc_info->hmc_obj[I40IW_HMC_IW_MR].cnt,
	+ hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].cnt);
	+
	+ ret_code = i40iw_sc_configure_iw_fpm(dev, dev->hmc_fn_id);
	+ if (ret_code) {
	+ i40iw_debug(dev, I40IW_DEBUG_HMC,
	+ "configure_iw_fpm returned error_code[x%08X]\n",
	+ rd32(dev->hw, dev->is_pf ? I40E_PFPE_CQPERRCODES : I40E_VFPE_CQPERRCODES1));
	+ return ret_code;
	+ }
	+ mem_size = sizeof(struct i40iw_hmc_sd_entry) *
	+ (hmc_info->sd_table.sd_cnt + hmc_info->first_sd_index + 1);
	+ ret_code = i40iw_allocate_virt_mem(dev->hw, &virt_mem, mem_size);
	+ if (ret_code) {
	+ i40iw_debug(dev, I40IW_DEBUG_HMC,
	+ "%s: failed to allocate memory for sd_entry buffer\n",
	+ __func__);
	+ return ret_code;
	+ }
	+ hmc_info->sd_table.sd_entry = virt_mem.va;
	+
	+ return ret_code;
	+}
	+
	+/**
	+ * i40iw_exec_cqp_cmd - execute cqp cmd when wqe are available
	+ * @dev: rdma device
	+ * @pcmdinfo: cqp command info
	+ */
	+static enum i40iw_status_code
	+i40iw_exec_cqp_cmd(struct i40iw_sc_dev *dev,
	+ struct cqp_commands_info *pcmdinfo)
	+{
	+ enum i40iw_status_code status;
	+ struct i40iw_dma_mem values_mem;
	+
	+ dev->cqp_cmd_stats[pcmdinfo->cqp_cmd]++;
	+
	+ switch (pcmdinfo->cqp_cmd) {
	+ case OP_DELETE_LOCAL_MAC_IPADDR_ENTRY:
	+ status = i40iw_sc_del_local_mac_ipaddr_entry(
	+ pcmdinfo->in.u.del_local_mac_ipaddr_entry.cqp,
	+ pcmdinfo->in.u.del_local_mac_ipaddr_entry.scratch,
	+ pcmdinfo->in.u.del_local_mac_ipaddr_entry.entry_idx,
	+ pcmdinfo->in.u.del_local_mac_ipaddr_entry.ignore_ref_count,
	+ pcmdinfo->post_sq);
	+ break;
	+ case OP_CEQ_DESTROY:
	+ status = i40iw_sc_ceq_destroy(pcmdinfo->in.u.ceq_destroy.ceq,
	+ pcmdinfo->in.u.ceq_destroy.scratch,
	+ pcmdinfo->post_sq);
	+ break;
	+ case OP_AEQ_DESTROY:
	+ status = i40iw_sc_aeq_destroy(pcmdinfo->in.u.aeq_destroy.aeq,
	+ pcmdinfo->in.u.aeq_destroy.scratch,
	+ pcmdinfo->post_sq);
	+
	+ break;
	+ case OP_DELETE_ARP_CACHE_ENTRY:
	+ status = i40iw_sc_del_arp_cache_entry(
	+ pcmdinfo->in.u.del_arp_cache_entry.cqp,
	+ pcmdinfo->in.u.del_arp_cache_entry.scratch,
	+ pcmdinfo->in.u.del_arp_cache_entry.arp_index,
	+ pcmdinfo->post_sq);
	+ break;
	+ case OP_MANAGE_APBVT_ENTRY:
	+ status = i40iw_sc_manage_apbvt_entry(
	+ pcmdinfo->in.u.manage_apbvt_entry.cqp,
	+ &pcmdinfo->in.u.manage_apbvt_entry.info,
	+ pcmdinfo->in.u.manage_apbvt_entry.scratch,
	+ pcmdinfo->post_sq);
	+ break;
	+ case OP_CEQ_CREATE:
	+ status = i40iw_sc_ceq_create(pcmdinfo->in.u.ceq_create.ceq,
	+ pcmdinfo->in.u.ceq_create.scratch,
	+ pcmdinfo->post_sq);
	+ break;
	+ case OP_AEQ_CREATE:
	+ status = i40iw_sc_aeq_create(pcmdinfo->in.u.aeq_create.aeq,
	+ pcmdinfo->in.u.aeq_create.scratch,
	+ pcmdinfo->post_sq);
	+ break;
	+ case OP_ALLOC_LOCAL_MAC_IPADDR_ENTRY:
	+ status = i40iw_sc_alloc_local_mac_ipaddr_entry(
	+ pcmdinfo->in.u.alloc_local_mac_ipaddr_entry.cqp,
	+ pcmdinfo->in.u.alloc_local_mac_ipaddr_entry.scratch,
	+ pcmdinfo->post_sq);
	+ break;
	+ case OP_ADD_LOCAL_MAC_IPADDR_ENTRY:
	+ status = i40iw_sc_add_local_mac_ipaddr_entry(
	+ pcmdinfo->in.u.add_local_mac_ipaddr_entry.cqp,
	+ &pcmdinfo->in.u.add_local_mac_ipaddr_entry.info,
	+ pcmdinfo->in.u.add_local_mac_ipaddr_entry.scratch,
	+ pcmdinfo->post_sq);
	+ break;
	+ case OP_MANAGE_QHASH_TABLE_ENTRY:
	+ status = i40iw_sc_manage_qhash_table_entry(
	+ pcmdinfo->in.u.manage_qhash_table_entry.cqp,
	+ &pcmdinfo->in.u.manage_qhash_table_entry.info,
	+ pcmdinfo->in.u.manage_qhash_table_entry.scratch,
	+ pcmdinfo->post_sq);
	+
	+ break;
	+ case OP_QP_MODIFY:
	+ status = i40iw_sc_qp_modify(
	+ pcmdinfo->in.u.qp_modify.qp,
	+ &pcmdinfo->in.u.qp_modify.info,
	+ pcmdinfo->in.u.qp_modify.scratch,
	+ pcmdinfo->post_sq);
	+
	+ break;
	+ case OP_QP_UPLOAD_CONTEXT:
	+ status = i40iw_sc_qp_upload_context(
	+ pcmdinfo->in.u.qp_upload_context.dev,
	+ &pcmdinfo->in.u.qp_upload_context.info,
	+ pcmdinfo->in.u.qp_upload_context.scratch,
	+ pcmdinfo->post_sq);
	+
	+ break;
	+ case OP_CQ_CREATE:
	+ status = i40iw_sc_cq_create(
	+ pcmdinfo->in.u.cq_create.cq,
	+ pcmdinfo->in.u.cq_create.scratch,
	+ pcmdinfo->in.u.cq_create.check_overflow,
	+ pcmdinfo->post_sq);
	+ break;
	+ case OP_CQ_DESTROY:
	+ status = i40iw_sc_cq_destroy(
	+ pcmdinfo->in.u.cq_destroy.cq,
	+ pcmdinfo->in.u.cq_destroy.scratch,
	+ pcmdinfo->post_sq);
	+
	+ break;
	+ case OP_QP_CREATE:
	+ status = i40iw_sc_qp_create(
	+ pcmdinfo->in.u.qp_create.qp,
	+ &pcmdinfo->in.u.qp_create.info,
	+ pcmdinfo->in.u.qp_create.scratch,
	+ pcmdinfo->post_sq);
	+ break;
	+ case OP_QP_DESTROY:
	+ status = i40iw_sc_qp_destroy(
	+ pcmdinfo->in.u.qp_destroy.qp,
	+ pcmdinfo->in.u.qp_destroy.scratch,
	+ pcmdinfo->in.u.qp_destroy.remove_hash_idx,
	+ pcmdinfo->in.u.qp_destroy.
	+ ignore_mw_bnd,
	+ pcmdinfo->post_sq);
	+
	+ break;
	+ case OP_ALLOC_STAG:
	+ status = i40iw_sc_alloc_stag(
	+ pcmdinfo->in.u.alloc_stag.dev,
	+ &pcmdinfo->in.u.alloc_stag.info,
	+ pcmdinfo->in.u.alloc_stag.scratch,
	+ pcmdinfo->post_sq);
	+ break;
	+ case OP_MR_REG_NON_SHARED:
	+ status = i40iw_sc_mr_reg_non_shared(
	+ pcmdinfo->in.u.mr_reg_non_shared.dev,
	+ &pcmdinfo->in.u.mr_reg_non_shared.info,
	+ pcmdinfo->in.u.mr_reg_non_shared.scratch,
	+ pcmdinfo->post_sq);
	+ break;
	+ case OP_DEALLOC_STAG:
	+ status = i40iw_sc_dealloc_stag(
	+ pcmdinfo->in.u.dealloc_stag.dev,
	+ &pcmdinfo->in.u.dealloc_stag.info,
	+ pcmdinfo->in.u.dealloc_stag.scratch,
	+ pcmdinfo->post_sq);
	+
	+ break;
	+ case OP_MW_ALLOC:
	+ status = i40iw_sc_mw_alloc(
	+ pcmdinfo->in.u.mw_alloc.dev,
	+ pcmdinfo->in.u.mw_alloc.scratch,
	+ pcmdinfo->in.u.mw_alloc.mw_stag_index,
	+ pcmdinfo->in.u.mw_alloc.pd_id,
	+ pcmdinfo->post_sq);
	+
	+ break;
	+ case OP_QP_FLUSH_WQES:
	+ status = i40iw_sc_qp_flush_wqes(
	+ pcmdinfo->in.u.qp_flush_wqes.qp,
	+ &pcmdinfo->in.u.qp_flush_wqes.info,
	+ pcmdinfo->in.u.qp_flush_wqes.
	+ scratch, pcmdinfo->post_sq);
	+ break;
	+ case OP_GEN_AE:
	+ status = i40iw_sc_gen_ae(
	+ pcmdinfo->in.u.gen_ae.qp,
	+ &pcmdinfo->in.u.gen_ae.info,
	+ pcmdinfo->in.u.gen_ae.scratch,
	+ pcmdinfo->post_sq);
	+ break;
	+ case OP_ADD_ARP_CACHE_ENTRY:
	+ status = i40iw_sc_add_arp_cache_entry(
	+ pcmdinfo->in.u.add_arp_cache_entry.cqp,
	+ &pcmdinfo->in.u.add_arp_cache_entry.info,
	+ pcmdinfo->in.u.add_arp_cache_entry.scratch,
	+ pcmdinfo->post_sq);
	+ break;
	+ case OP_MANAGE_PUSH_PAGE:
	+ status = i40iw_sc_manage_push_page(
	+ pcmdinfo->in.u.manage_push_page.cqp,
	+ &pcmdinfo->in.u.manage_push_page.info,
	+ pcmdinfo->in.u.manage_push_page.scratch,
	+ pcmdinfo->post_sq);
	+ break;
	+ case OP_UPDATE_PE_SDS:
	+ /* case I40IW_CQP_OP_UPDATE_PE_SDS */
	+ status = i40iw_update_pe_sds(
	+ pcmdinfo->in.u.update_pe_sds.dev,
	+ &pcmdinfo->in.u.update_pe_sds.info,
	+ pcmdinfo->in.u.update_pe_sds.
	+ scratch);
	+
	+ break;
	+ case OP_MANAGE_HMC_PM_FUNC_TABLE:
	+ status = i40iw_sc_manage_hmc_pm_func_table(
	+ pcmdinfo->in.u.manage_hmc_pm.dev->cqp,
	+ pcmdinfo->in.u.manage_hmc_pm.scratch,
	+ (u8)pcmdinfo->in.u.manage_hmc_pm.info.vf_id,
	+ pcmdinfo->in.u.manage_hmc_pm.info.free_fcn,
	+ TRUE);
	+ break;
	+ case OP_SUSPEND:
	+ status = i40iw_sc_suspend_qp(
	+ pcmdinfo->in.u.suspend_resume.cqp,
	+ pcmdinfo->in.u.suspend_resume.qp,
	+ pcmdinfo->in.u.suspend_resume.scratch);
	+ break;
	+ case OP_RESUME:
	+ status = i40iw_sc_resume_qp(
	+ pcmdinfo->in.u.suspend_resume.cqp,
	+ pcmdinfo->in.u.suspend_resume.qp,
	+ pcmdinfo->in.u.suspend_resume.scratch);
	+ break;
	+ case OP_QUERY_FPM_VALUES:
	+ values_mem.pa = pcmdinfo->in.u.query_fpm_values.fpm_values_pa;
	+ values_mem.va = pcmdinfo->in.u.query_fpm_values.fpm_values_va;
	+ status = i40iw_sc_query_fpm_values(
	+ pcmdinfo->in.u.query_fpm_values.cqp,
	+ pcmdinfo->in.u.query_fpm_values.scratch,
	+ pcmdinfo->in.u.query_fpm_values.hmc_fn_id,
	+ &values_mem, TRUE, I40IW_CQP_WAIT_EVENT);
	+ break;
	+ case OP_COMMIT_FPM_VALUES:
	+ values_mem.pa = pcmdinfo->in.u.commit_fpm_values.fpm_values_pa;
	+ values_mem.va = pcmdinfo->in.u.commit_fpm_values.fpm_values_va;
	+ status = i40iw_sc_commit_fpm_values(
	+ pcmdinfo->in.u.commit_fpm_values.cqp,
	+ pcmdinfo->in.u.commit_fpm_values.scratch,
	+ pcmdinfo->in.u.commit_fpm_values.hmc_fn_id,
	+ &values_mem,
	+ TRUE,
	+ I40IW_CQP_WAIT_EVENT);
	+ break;
	+ default:
	+ status = I40IW_NOT_SUPPORTED;
	+ break;
	+ }
	+
	+ if (status)
	+ dev->cqp_cmd_stats[pcmdinfo->cqp_cmd]--;
	+
	+ return status;
	+}
	+
	+/**
	+ * i40iw_process_cqp_cmd - process all cqp commands
	+ * @dev: sc device struct
	+ * @pcmdinfo: cqp command info
	+ */
	+enum i40iw_status_code
	+i40iw_process_cqp_cmd(struct i40iw_sc_dev *dev,
	+ struct cqp_commands_info *pcmdinfo)
	+{
	+ enum i40iw_status_code status = 0;
	+ unsigned long flags;
	+
	+ spin_lock_irqsave(&dev->cqp_lock, flags);
	+ if (list_empty(&dev->cqp_cmd_head) && !i40iw_ring_full(dev->cqp))
	+ status = i40iw_exec_cqp_cmd(dev, pcmdinfo);
	+ else
	+ list_add_tail(&pcmdinfo->cqp_cmd_entry, &dev->cqp_cmd_head);
	+ spin_unlock_irqrestore(&dev->cqp_lock, flags);
	+ return status;
	+}
	+
	+/**
	+ * i40iw_process_bh - called from tasklet for cqp list
	+ * @dev: sc device struct
	+ */
	+enum i40iw_status_code
	+i40iw_process_bh(struct i40iw_sc_dev *dev)
	+{
	+ enum i40iw_status_code status = 0;
	+ struct cqp_commands_info *pcmdinfo;
	+ unsigned long flags;
	+
	+ spin_lock_irqsave(&dev->cqp_lock, flags);
	+ while (!list_empty(&dev->cqp_cmd_head) && !i40iw_ring_full(dev->cqp)) {
	+ pcmdinfo = (struct cqp_commands_info *)i40iw_remove_head(&dev->cqp_cmd_head);
	+
	+ status = i40iw_exec_cqp_cmd(dev, pcmdinfo);
	+ if (status)
	+ break;
	+ }
	+ spin_unlock_irqrestore(&dev->cqp_lock, flags);
	+ return status;
	+}
	+
	+/**
	+ * i40iw_iwarp_opcode - determine if incoming is rdma layer
	+ * @info: aeq info for the packet
	+ * @pkt: packet for error
	+ */
	+static u32
	+i40iw_iwarp_opcode(struct i40iw_aeqe_info info, u8 pkt)
	+{
	+ BE16 *mpa;
	+ u32 opcode = 0xffffffff;
	+
	+ if (info->q2_data_written) {
	+ mpa = (BE16 *) pkt;
	+ opcode = I40IW_NTOHS(mpa[1]) & 0xf;
	+ }
	+ return opcode;
	+}
	+
	+/**
	+ * i40iw_locate_mpa - return pointer to mpa in the pkt
	+ * @pkt: packet with data
	+ */
	+static u8 *
	+i40iw_locate_mpa(u8 *pkt)
	+{
	+ /* skip over ethernet header */
	+ pkt += I40IW_MAC_HLEN;
	+
	+ /* Skip over IP and TCP headers */
	+ pkt += 4 * (pkt[0] & 0x0f);
	+ pkt += 4 * ((pkt[12] >> 4) & 0x0f);
	+ return pkt;
	+}
	+
	+/**
	+ * i40iw_setup_termhdr - termhdr for terminate pkt
	+ * @qp: sc qp ptr for pkt
	+ * @hdr: term hdr
	+ * @opcode: flush opcode for termhdr
	+ * @layer_etype: error layer + error type
	+ * @err: error cod ein the header
	+ */
	+static void
	+i40iw_setup_termhdr(struct i40iw_sc_qp *qp,
	+ struct i40iw_terminate_hdr *hdr,
	+ enum i40iw_flush_opcode opcode,
	+ u8 layer_etype,
	+ u8 err)
	+{
	+ qp->flush_code = opcode;
	+ hdr->layer_etype = layer_etype;
	+ hdr->error_code = err;
	+}
	+
	+/**
	+ * i40iw_bld_terminate_hdr - build terminate message header
	+ * @qp: qp associated with received terminate AE
	+ * @info: the struct contiaing AE information
	+ */
	+static int
	+i40iw_bld_terminate_hdr(struct i40iw_sc_qp *qp,
	+ struct i40iw_aeqe_info *info)
	+{
	+ u8 *pkt = qp->q2_buf + Q2_BAD_FRAME_OFFSET;
	+ u16 ddp_seg_len;
	+ int copy_len = 0;
	+ u8 is_tagged = 0;
	+ u32 opcode;
	+ struct i40iw_terminate_hdr *termhdr;
	+
	+ termhdr = (struct i40iw_terminate_hdr *)qp->q2_buf;
	+ memset(termhdr, 0, Q2_BAD_FRAME_OFFSET);
	+
	+ if (info->q2_data_written) {
	+ /* Use data from offending packet to fill in ddp & rdma hdrs */
	+ pkt = i40iw_locate_mpa(pkt);
	+ ddp_seg_len = I40IW_NTOHS((BE16 ) pkt);
	+ if (ddp_seg_len) {
	+ copy_len = 2;
	+ termhdr->hdrct = DDP_LEN_FLAG;
	+ if (pkt[2] & 0x80) {
	+ is_tagged = 1;
	+ if (ddp_seg_len >= TERM_DDP_LEN_TAGGED) {
	+ copy_len += TERM_DDP_LEN_TAGGED;
	+ termhdr->hdrct \|= DDP_HDR_FLAG;
	+ }
	+ } else {
	+ if (ddp_seg_len >= TERM_DDP_LEN_UNTAGGED) {
	+ copy_len += TERM_DDP_LEN_UNTAGGED;
	+ termhdr->hdrct \|= DDP_HDR_FLAG;
	+ }
	+ if (ddp_seg_len >= (TERM_DDP_LEN_UNTAGGED + TERM_RDMA_LEN)) {
	+ if ((pkt[3] & RDMA_OPCODE_MASK) == RDMA_READ_REQ_OPCODE) {
	+ copy_len += TERM_RDMA_LEN;
	+ termhdr->hdrct \|= RDMA_HDR_FLAG;
	+ }
	+ }
	+ }
	+ }
	+ }
	+ opcode = i40iw_iwarp_opcode(info, pkt);
	+
	+ switch (info->ae_id) {
	+ case I40IW_AE_AMP_UNALLOCATED_STAG:
	+ qp->eventtype = TERM_EVENT_QP_ACCESS_ERR;
	+ if (opcode == I40IW_OP_TYPE_RDMA_WRITE)
	+ i40iw_setup_termhdr(qp, termhdr, FLUSH_PROT_ERR,
	+ (LAYER_DDP << 4) \| DDP_TAGGED_BUFFER, DDP_TAGGED_INV_STAG);
	+ else
	+ i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_ACCESS_ERR,
	+ (LAYER_RDMA << 4) \| RDMAP_REMOTE_PROT, RDMAP_INV_STAG);
	+ break;
	+ case I40IW_AE_AMP_BOUNDS_VIOLATION:
	+ qp->eventtype = TERM_EVENT_QP_ACCESS_ERR;
	+ if (info->q2_data_written)
	+ i40iw_setup_termhdr(qp, termhdr, FLUSH_PROT_ERR,
	+ (LAYER_DDP << 4) \| DDP_TAGGED_BUFFER, DDP_TAGGED_BOUNDS);
	+ else
	+ i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_ACCESS_ERR,
	+ (LAYER_RDMA << 4) \| RDMAP_REMOTE_PROT, RDMAP_INV_BOUNDS);
	+ break;
	+ case I40IW_AE_AMP_BAD_PD:
	+ switch (opcode) {
	+ case I40IW_OP_TYPE_RDMA_WRITE:
	+ i40iw_setup_termhdr(qp, termhdr, FLUSH_PROT_ERR,
	+ (LAYER_DDP << 4) \| DDP_TAGGED_BUFFER, DDP_TAGGED_UNASSOC_STAG);
	+ break;
	+ case I40IW_OP_TYPE_SEND_INV:
	+ case I40IW_OP_TYPE_SEND_SOL_INV:
	+ i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_ACCESS_ERR,
	+ (LAYER_RDMA << 4) \| RDMAP_REMOTE_PROT, RDMAP_CANT_INV_STAG);
	+ break;
	+ default:
	+ i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_ACCESS_ERR,
	+ (LAYER_RDMA << 4) \| RDMAP_REMOTE_PROT, RDMAP_UNASSOC_STAG);
	+ }
	+ break;
	+ case I40IW_AE_AMP_INVALID_STAG:
	+ qp->eventtype = TERM_EVENT_QP_ACCESS_ERR;
	+ i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_ACCESS_ERR,
	+ (LAYER_RDMA << 4) \| RDMAP_REMOTE_PROT, RDMAP_INV_STAG);
	+ break;
	+ case I40IW_AE_AMP_BAD_QP:
	+ i40iw_setup_termhdr(qp, termhdr, FLUSH_LOC_QP_OP_ERR,
	+ (LAYER_DDP << 4) \| DDP_UNTAGGED_BUFFER, DDP_UNTAGGED_INV_QN);
	+ break;
	+ case I40IW_AE_AMP_BAD_STAG_KEY:
	+ case I40IW_AE_AMP_BAD_STAG_INDEX:
	+ qp->eventtype = TERM_EVENT_QP_ACCESS_ERR;
	+ switch (opcode) {
	+ case I40IW_OP_TYPE_SEND_INV:
	+ case I40IW_OP_TYPE_SEND_SOL_INV:
	+ i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_OP_ERR,
	+ (LAYER_RDMA << 4) \| RDMAP_REMOTE_OP, RDMAP_CANT_INV_STAG);
	+ break;
	+ default:
	+ i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_ACCESS_ERR,
	+ (LAYER_RDMA << 4) \| RDMAP_REMOTE_OP, RDMAP_INV_STAG);
	+ }
	+ break;
	+ case I40IW_AE_AMP_RIGHTS_VIOLATION:
	+ case I40IW_AE_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS:
	+ case I40IW_AE_PRIV_OPERATION_DENIED:
	+ qp->eventtype = TERM_EVENT_QP_ACCESS_ERR;
	+ i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_ACCESS_ERR,
	+ (LAYER_RDMA << 4) \| RDMAP_REMOTE_PROT, RDMAP_ACCESS);
	+ break;
	+ case I40IW_AE_AMP_TO_WRAP:
	+ qp->eventtype = TERM_EVENT_QP_ACCESS_ERR;
	+ i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_ACCESS_ERR,
	+ (LAYER_RDMA << 4) \| RDMAP_REMOTE_PROT, RDMAP_TO_WRAP);
	+ break;
	+ case I40IW_AE_LLP_RECEIVED_MPA_CRC_ERROR:
	+ i40iw_setup_termhdr(qp, termhdr, FLUSH_GENERAL_ERR,
	+ (LAYER_MPA << 4) \| DDP_LLP, MPA_CRC);
	+ break;
	+ case I40IW_AE_LLP_SEGMENT_TOO_LARGE:
	+ case I40IW_AE_LLP_SEGMENT_TOO_SMALL:
	+ i40iw_setup_termhdr(qp, termhdr, FLUSH_LOC_LEN_ERR,
	+ (LAYER_DDP << 4) \| DDP_CATASTROPHIC, DDP_CATASTROPHIC_LOCAL);
	+ break;
	+ case I40IW_AE_LCE_QP_CATASTROPHIC:
	+ case I40IW_AE_DDP_NO_L_BIT:
	+ i40iw_setup_termhdr(qp, termhdr, FLUSH_FATAL_ERR,
	+ (LAYER_DDP << 4) \| DDP_CATASTROPHIC, DDP_CATASTROPHIC_LOCAL);
	+ break;
	+ case I40IW_AE_DDP_INVALID_MSN_GAP_IN_MSN:
	+ i40iw_setup_termhdr(qp, termhdr, FLUSH_GENERAL_ERR,
	+ (LAYER_DDP << 4) \| DDP_UNTAGGED_BUFFER, DDP_UNTAGGED_INV_MSN_RANGE);
	+ break;
	+ case I40IW_AE_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER:
	+ qp->eventtype = TERM_EVENT_QP_ACCESS_ERR;
	+ i40iw_setup_termhdr(qp, termhdr, FLUSH_LOC_LEN_ERR,
	+ (LAYER_DDP << 4) \| DDP_UNTAGGED_BUFFER, DDP_UNTAGGED_INV_TOO_LONG);
	+ break;
	+ case I40IW_AE_DDP_UBE_INVALID_DDP_VERSION:
	+ if (is_tagged)
	+ i40iw_setup_termhdr(qp, termhdr, FLUSH_GENERAL_ERR,
	+ (LAYER_DDP << 4) \| DDP_TAGGED_BUFFER, DDP_TAGGED_INV_DDP_VER);
	+ else
	+ i40iw_setup_termhdr(qp, termhdr, FLUSH_GENERAL_ERR,
	+ (LAYER_DDP << 4) \| DDP_UNTAGGED_BUFFER, DDP_UNTAGGED_INV_DDP_VER);
	+ break;
	+ case I40IW_AE_DDP_UBE_INVALID_MO:
	+ i40iw_setup_termhdr(qp, termhdr, FLUSH_GENERAL_ERR,
	+ (LAYER_DDP << 4) \| DDP_UNTAGGED_BUFFER, DDP_UNTAGGED_INV_MO);
	+ break;
	+ case I40IW_AE_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE:
	+ i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_OP_ERR,
	+ (LAYER_DDP << 4) \| DDP_UNTAGGED_BUFFER, DDP_UNTAGGED_INV_MSN_NO_BUF);
	+ break;
	+ case I40IW_AE_DDP_UBE_INVALID_QN:
	+ i40iw_setup_termhdr(qp, termhdr, FLUSH_GENERAL_ERR,
	+ (LAYER_DDP << 4) \| DDP_UNTAGGED_BUFFER, DDP_UNTAGGED_INV_QN);
	+ break;
	+ case I40IW_AE_RDMAP_ROE_INVALID_RDMAP_VERSION:
	+ i40iw_setup_termhdr(qp, termhdr, FLUSH_GENERAL_ERR,
	+ (LAYER_RDMA << 4) \| RDMAP_REMOTE_OP, RDMAP_INV_RDMAP_VER);
	+ break;
	+ case I40IW_AE_RDMAP_ROE_UNEXPECTED_OPCODE:
	+ i40iw_setup_termhdr(qp, termhdr, FLUSH_LOC_QP_OP_ERR,
	+ (LAYER_RDMA << 4) \| RDMAP_REMOTE_OP, RDMAP_UNEXPECTED_OP);
	+ break;
	+ default:
	+ i40iw_setup_termhdr(qp, termhdr, FLUSH_FATAL_ERR,
	+ (LAYER_RDMA << 4) \| RDMAP_REMOTE_OP, RDMAP_UNSPECIFIED);
	+ break;
	+ }
	+
	+ if (copy_len)
	+ i40iw_memcpy(termhdr + 1, pkt, copy_len);
	+
	+ return sizeof(struct i40iw_terminate_hdr) + copy_len;
	+}
	+
	+/**
	+ * i40iw_terminate_send_fin() - Send fin for terminate message
	+ * @qp: qp associated with received terminate AE
	+ */
	+void
	+i40iw_terminate_send_fin(struct i40iw_sc_qp *qp)
	+{
	+ /* Send the fin only */
	+ i40iw_term_modify_qp(qp,
	+ I40IW_QP_STATE_TERMINATE,
	+ I40IWQP_TERM_SEND_FIN_ONLY,
	+ 0);
	+}
	+
	+/**
	+ * i40iw_terminate_connection() - Bad AE and send terminate to remote QP
	+ * @qp: qp associated with received terminate AE
	+ * @info: the struct contiaing AE information
	+ */
	+void
	+i40iw_terminate_connection(struct i40iw_sc_qp qp, struct i40iw_aeqe_info info)
	+{
	+ u8 termlen = 0;
	+
	+ if (qp->term_flags & I40IW_TERM_SENT)
	+ return; /* Sanity check */
	+
	+ /* Eventtype can change from bld_terminate_hdr */
	+ qp->eventtype = TERM_EVENT_QP_FATAL;
	+ termlen = i40iw_bld_terminate_hdr(qp, info);
	+ i40iw_terminate_start_timer(qp);
	+ qp->term_flags \|= I40IW_TERM_SENT;
	+ i40iw_term_modify_qp(qp, I40IW_QP_STATE_TERMINATE,
	+ I40IWQP_TERM_SEND_TERM_ONLY, termlen);
	+}
	+
	+/**
	+ * i40iw_terminate_received - handle terminate received AE
	+ * @qp: qp associated with received terminate AE
	+ * @info: the struct contiaing AE information
	+ */
	+void
	+i40iw_terminate_received(struct i40iw_sc_qp qp, struct i40iw_aeqe_info info)
	+{
	+ u8 *pkt = qp->q2_buf + Q2_BAD_FRAME_OFFSET;
	+ BE32 *mpa;
	+ u8 ddp_ctl;
	+ u8 rdma_ctl;
	+ u16 aeq_id = 0;
	+ struct i40iw_terminate_hdr *termhdr;
	+
	+ mpa = (BE32 *) i40iw_locate_mpa(pkt);
	+ if (info->q2_data_written) {
	+ /* did not validate the frame - do it now */
	+ ddp_ctl = (I40IW_NTOHL(mpa[0]) >> 8) & 0xff;
	+ rdma_ctl = I40IW_NTOHL(mpa[0]) & 0xff;
	+ if ((ddp_ctl & 0xc0) != 0x40)
	+ aeq_id = I40IW_AE_LCE_QP_CATASTROPHIC;
	+ else if ((ddp_ctl & 0x03) != 1)
	+ aeq_id = I40IW_AE_DDP_UBE_INVALID_DDP_VERSION;
	+ else if (I40IW_NTOHL(mpa[2]) != 2)
	+ aeq_id = I40IW_AE_DDP_UBE_INVALID_QN;
	+ else if (I40IW_NTOHL(mpa[3]) != 1)
	+ aeq_id = I40IW_AE_DDP_INVALID_MSN_GAP_IN_MSN;
	+ else if (I40IW_NTOHL(mpa[4]) != 0)
	+ aeq_id = I40IW_AE_DDP_UBE_INVALID_MO;
	+ else if ((rdma_ctl & 0xc0) != 0x40)
	+ aeq_id = I40IW_AE_RDMAP_ROE_INVALID_RDMAP_VERSION;
	+
	+ info->ae_id = aeq_id;
	+ if (info->ae_id) {
	+ /* Bad terminate recvd - send back a terminate */
	+ i40iw_terminate_connection(qp, info);
	+ return;
	+ }
	+ }
	+ qp->term_flags \|= I40IW_TERM_RCVD;
	+ qp->eventtype = TERM_EVENT_QP_FATAL;
	+ termhdr = (struct i40iw_terminate_hdr *)&mpa[5];
	+ if (termhdr->layer_etype == RDMAP_REMOTE_PROT \|\|
	+ termhdr->layer_etype == RDMAP_REMOTE_OP) {
	+ i40iw_terminate_done(qp, 0);
	+ } else {
	+ i40iw_terminate_start_timer(qp);
	+ i40iw_terminate_send_fin(qp);
	+ }
	+}
	+
	+/**
	+ * i40iw_sc_vsi_init - Init the vsi structure
	+ * @vsi: pointer to vsi structure to initialize
	+ * @info: the info used to initialize the vsi struct
	+ */
	+void
	+i40iw_sc_vsi_init(struct i40iw_sc_vsi vsi, struct i40iw_vsi_init_info info)
	+{
	+ vsi->qos[0].qs_handle = info->params->qs_handle_list[0];
	+ vsi->dev = info->dev;
	+ vsi->back_vsi = info->back_vsi;
	+ vsi->mtu = info->params->mtu;
	+ vsi->exception_lan_queue = info->exception_lan_queue;
	+}
	+
	+/**
	+ * i40iw_hw_stats_init - Initialize HW stats table
	+ * @stats: pestat struct
	+ * @fcn_idx: PCI fn id
	+ * @hw: PF i40iw_hw structure.
	+ * @is_pf: Is it a PF?
	+ *
	+ * Populate the HW stats table with register offset addr for each
	+ * stats. And start the perioidic stats timer.
	+ */
	+void
	+i40iw_hw_stats_init(struct i40iw_vsi_pestat *stats, u8 fcn_idx, bool is_pf)
	+{
	+ u32 stats_reg_offset;
	+ u32 stats_index;
	+ struct i40iw_dev_hw_stats_offsets *stats_table =
	+ &stats->hw_stats_offsets;
	+ struct i40iw_dev_hw_stats *last_rd_stats = &stats->last_read_hw_stats;
	+
	+ if (is_pf) {
	+ stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP4RXDISCARD] =
	+ I40E_GLPES_PFIP4RXDISCARD(fcn_idx);
	+ stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP4RXTRUNC] =
	+ I40E_GLPES_PFIP4RXTRUNC(fcn_idx);
	+ stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP4TXNOROUTE] =
	+ I40E_GLPES_PFIP4TXNOROUTE(fcn_idx);
	+ stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP6RXDISCARD] =
	+ I40E_GLPES_PFIP6RXDISCARD(fcn_idx);
	+ stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP6RXTRUNC] =
	+ I40E_GLPES_PFIP6RXTRUNC(fcn_idx);
	+ stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP6TXNOROUTE] =
	+ I40E_GLPES_PFIP6TXNOROUTE(fcn_idx);
	+ stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_TCPRTXSEG] =
	+ I40E_GLPES_PFTCPRTXSEG(fcn_idx);
	+ stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_TCPRXOPTERR] =
	+ I40E_GLPES_PFTCPRXOPTERR(fcn_idx);
	+ stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_TCPRXPROTOERR] =
	+ I40E_GLPES_PFTCPRXPROTOERR(fcn_idx);
	+
	+ stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4RXOCTS] =
	+ I40E_GLPES_PFIP4RXOCTSLO(fcn_idx);
	+ stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4RXPKTS] =
	+ I40E_GLPES_PFIP4RXPKTSLO(fcn_idx);
	+ stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4RXFRAGS] =
	+ I40E_GLPES_PFIP4RXFRAGSLO(fcn_idx);
	+ stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4RXMCPKTS] =
	+ I40E_GLPES_PFIP4RXMCPKTSLO(fcn_idx);
	+ stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4TXOCTS] =
	+ I40E_GLPES_PFIP4TXOCTSLO(fcn_idx);
	+ stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4TXPKTS] =
	+ I40E_GLPES_PFIP4TXPKTSLO(fcn_idx);
	+ stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4TXFRAGS] =
	+ I40E_GLPES_PFIP4TXFRAGSLO(fcn_idx);
	+ stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4TXMCPKTS] =
	+ I40E_GLPES_PFIP4TXMCPKTSLO(fcn_idx);
	+ stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6RXOCTS] =
	+ I40E_GLPES_PFIP6RXOCTSLO(fcn_idx);
	+ stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6RXPKTS] =
	+ I40E_GLPES_PFIP6RXPKTSLO(fcn_idx);
	+ stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6RXFRAGS] =
	+ I40E_GLPES_PFIP6RXFRAGSLO(fcn_idx);
	+ stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6RXMCPKTS] =
	+ I40E_GLPES_PFIP6RXMCPKTSLO(fcn_idx);
	+ stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6TXOCTS] =
	+ I40E_GLPES_PFIP6TXOCTSLO(fcn_idx);
	+ stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6TXPKTS] =
	+ I40E_GLPES_PFIP6TXPKTSLO(fcn_idx);
	+ stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6TXPKTS] =
	+ I40E_GLPES_PFIP6TXPKTSLO(fcn_idx);
	+ stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6TXFRAGS] =
	+ I40E_GLPES_PFIP6TXFRAGSLO(fcn_idx);
	+ stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_TCPRXSEGS] =
	+ I40E_GLPES_PFTCPRXSEGSLO(fcn_idx);
	+ stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_TCPTXSEG] =
	+ I40E_GLPES_PFTCPTXSEGLO(fcn_idx);
	+ stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMARXRDS] =
	+ I40E_GLPES_PFRDMARXRDSLO(fcn_idx);
	+ stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMARXSNDS] =
	+ I40E_GLPES_PFRDMARXSNDSLO(fcn_idx);
	+ stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMARXWRS] =
	+ I40E_GLPES_PFRDMARXWRSLO(fcn_idx);
	+ stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMATXRDS] =
	+ I40E_GLPES_PFRDMATXRDSLO(fcn_idx);
	+ stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMATXSNDS] =
	+ I40E_GLPES_PFRDMATXSNDSLO(fcn_idx);
	+ stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMATXWRS] =
	+ I40E_GLPES_PFRDMATXWRSLO(fcn_idx);
	+ stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMAVBND] =
	+ I40E_GLPES_PFRDMAVBNDLO(fcn_idx);
	+ stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMAVINV] =
	+ I40E_GLPES_PFRDMAVINVLO(fcn_idx);
	+ } else {
	+ fcn_idx -= I40IW_FIRST_VF_FPM_ID;
	+ stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP4RXDISCARD] =
	+ I40E_GLPES_VFIP4RXDISCARD(fcn_idx);
	+ stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP4RXTRUNC] =
	+ I40E_GLPES_VFIP4RXTRUNC(fcn_idx);
	+ stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP4TXNOROUTE] =
	+ I40E_GLPES_VFIP4TXNOROUTE(fcn_idx);
	+ stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP6RXDISCARD] =
	+ I40E_GLPES_VFIP6RXDISCARD(fcn_idx);
	+ stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP6RXTRUNC] =
	+ I40E_GLPES_VFIP6RXTRUNC(fcn_idx);
	+ stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP6TXNOROUTE] =
	+ I40E_GLPES_VFIP6TXNOROUTE(fcn_idx);
	+ stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_TCPRTXSEG] =
	+ I40E_GLPES_VFTCPRTXSEG(fcn_idx);
	+ stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_TCPRXOPTERR] =
	+ I40E_GLPES_VFTCPRXOPTERR(fcn_idx);
	+ stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_TCPRXPROTOERR] =
	+ I40E_GLPES_VFTCPRXPROTOERR(fcn_idx);
	+
	+ stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4RXOCTS] =
	+ I40E_GLPES_VFIP4RXOCTSLO(fcn_idx);
	+ stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4RXPKTS] =
	+ I40E_GLPES_VFIP4RXPKTSLO(fcn_idx);
	+ stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4RXFRAGS] =
	+ I40E_GLPES_VFIP4RXFRAGSLO(fcn_idx);
	+ stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4RXMCPKTS] =
	+ I40E_GLPES_VFIP4RXMCPKTSLO(fcn_idx);
	+ stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4TXOCTS] =
	+ I40E_GLPES_VFIP4TXOCTSLO(fcn_idx);
	+ stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4TXPKTS] =
	+ I40E_GLPES_VFIP4TXPKTSLO(fcn_idx);
	+ stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4TXFRAGS] =
	+ I40E_GLPES_VFIP4TXFRAGSLO(fcn_idx);
	+ stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4TXMCPKTS] =
	+ I40E_GLPES_VFIP4TXMCPKTSLO(fcn_idx);
	+ stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6RXOCTS] =
	+ I40E_GLPES_VFIP6RXOCTSLO(fcn_idx);
	+ stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6RXPKTS] =
	+ I40E_GLPES_VFIP6RXPKTSLO(fcn_idx);
	+ stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6RXFRAGS] =
	+ I40E_GLPES_VFIP6RXFRAGSLO(fcn_idx);
	+ stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6RXMCPKTS] =
	+ I40E_GLPES_VFIP6RXMCPKTSLO(fcn_idx);
	+ stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6TXOCTS] =
	+ I40E_GLPES_VFIP6TXOCTSLO(fcn_idx);
	+ stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6TXPKTS] =
	+ I40E_GLPES_VFIP6TXPKTSLO(fcn_idx);
	+ stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6TXPKTS] =
	+ I40E_GLPES_VFIP6TXPKTSLO(fcn_idx);
	+ stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6TXFRAGS] =
	+ I40E_GLPES_VFIP6TXFRAGSLO(fcn_idx);
	+ stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_TCPRXSEGS] =
	+ I40E_GLPES_VFTCPRXSEGSLO(fcn_idx);
	+ stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_TCPTXSEG] =
	+ I40E_GLPES_VFTCPTXSEGLO(fcn_idx);
	+ stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMARXRDS] =
	+ I40E_GLPES_VFRDMARXRDSLO(fcn_idx);
	+ stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMARXSNDS] =
	+ I40E_GLPES_VFRDMARXSNDSLO(fcn_idx);
	+ stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMARXWRS] =
	+ I40E_GLPES_VFRDMARXWRSLO(fcn_idx);
	+ stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMATXRDS] =
	+ I40E_GLPES_VFRDMATXRDSLO(fcn_idx);
	+ stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMATXSNDS] =
	+ I40E_GLPES_VFRDMATXSNDSLO(fcn_idx);
	+ stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMATXWRS] =
	+ I40E_GLPES_VFRDMATXWRSLO(fcn_idx);
	+ stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMAVBND] =
	+ I40E_GLPES_VFRDMAVBNDLO(fcn_idx);
	+ stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMAVINV] =
	+ I40E_GLPES_VFRDMAVINVLO(fcn_idx);
	+ }
	+
	+ for (stats_index = 0; stats_index < I40IW_HW_STAT_INDEX_MAX_64;
	+ stats_index++) {
	+ stats_reg_offset = stats_table->stats_offset_64[stats_index];
	+ last_rd_stats->stats_value_64[stats_index] =
	+ rd64(stats->hw, stats_reg_offset);
	+ }
	+
	+ for (stats_index = 0; stats_index < I40IW_HW_STAT_INDEX_MAX_32;
	+ stats_index++) {
	+ stats_reg_offset = stats_table->stats_offset_32[stats_index];
	+ last_rd_stats->stats_value_32[stats_index] =
	+ rd32(stats->hw, stats_reg_offset);
	+ }
	+}
	+
	+/**
	+ * i40iw_hw_stats_read_32 - Read 32-bit HW stats counters and accommodates for roll-overs.
	+ * @stat: pestat struct
	+ * @index: index in HW stats table which contains offset reg-addr
	+ * @value: hw stats value
	+ */
	+void
	+i40iw_hw_stats_read_32(struct i40iw_vsi_pestat *stats,
	+ enum i40iw_hw_stats_index_32b index,
	+ u64 *value)
	+{
	+ struct i40iw_dev_hw_stats_offsets *stats_table =
	+ &stats->hw_stats_offsets;
	+ struct i40iw_dev_hw_stats *last_rd_stats = &stats->last_read_hw_stats;
	+ struct i40iw_dev_hw_stats *hw_stats = &stats->hw_stats;
	+ u64 new_stats_value = 0;
	+ u32 stats_reg_offset = stats_table->stats_offset_32[index];
	+
	+ new_stats_value = rd32(stats->hw, stats_reg_offset);
	+ /* roll-over case */
	+ if (new_stats_value < last_rd_stats->stats_value_32[index])
	+ hw_stats->stats_value_32[index] += new_stats_value;
	+ else
	+ hw_stats->stats_value_32[index] +=
	+ new_stats_value - last_rd_stats->stats_value_32[index];
	+ last_rd_stats->stats_value_32[index] = new_stats_value;
	+ *value = hw_stats->stats_value_32[index];
	+}
	+
	+/**
	+ * i40iw_hw_stats_read_64 - Read HW stats counters (greater than 32-bit) and accommodates for roll-overs.
	+ * @stats: pestat struct
	+ * @index: index in HW stats table which contains offset reg-addr
	+ * @value: hw stats value
	+ */
	+void
	+i40iw_hw_stats_read_64(struct i40iw_vsi_pestat *stats,
	+ enum i40iw_hw_stats_index_64b index,
	+ u64 *value)
	+{
	+ struct i40iw_dev_hw_stats_offsets *stats_table =
	+ &stats->hw_stats_offsets;
	+ struct i40iw_dev_hw_stats *last_rd_stats = &stats->last_read_hw_stats;
	+ struct i40iw_dev_hw_stats *hw_stats = &stats->hw_stats;
	+ u64 new_stats_value = 0;
	+ u32 stats_reg_offset = stats_table->stats_offset_64[index];
	+
	+ new_stats_value = rd64(stats->hw, stats_reg_offset);
	+ /* roll-over case */
	+ if (new_stats_value < last_rd_stats->stats_value_64[index])
	+ hw_stats->stats_value_64[index] += new_stats_value;
	+ else
	+ hw_stats->stats_value_64[index] +=
	+ new_stats_value - last_rd_stats->stats_value_64[index];
	+ last_rd_stats->stats_value_64[index] = new_stats_value;
	+ *value = hw_stats->stats_value_64[index];
	+}
	+
	+/**
	+ * i40iw_hw_stats_read_all - read all HW stat counters
	+ * @stats: pestat struct
	+ * @stats_values: hw stats structure
	+ *
	+ * Read all the HW stat counters and populates hw_stats structure
	+ * of passed-in dev's pestat as well as copy created in stat_values.
	+ */
	+void
	+i40iw_hw_stats_read_all(struct i40iw_vsi_pestat *stats,
	+ struct i40iw_dev_hw_stats *stats_values)
	+{
	+ u32 stats_index;
	+ unsigned long flags;
	+
	+ spin_lock_irqsave(&stats->lock, flags);
	+
	+ for (stats_index = 0; stats_index < I40IW_HW_STAT_INDEX_MAX_32;
	+ stats_index++)
	+ i40iw_hw_stats_read_32(stats, stats_index,
	+ &stats_values->stats_value_32[stats_index]);
	+ for (stats_index = 0; stats_index < I40IW_HW_STAT_INDEX_MAX_64;
	+ stats_index++)
	+ i40iw_hw_stats_read_64(stats, stats_index,
	+ &stats_values->stats_value_64[stats_index]);
	+ spin_unlock_irqrestore(&stats->lock, flags);
	+}
	+
	+/**
	+ * i40iw_hw_stats_refresh_all - Update all HW stats structs
	+ * @stats: pestat struct
	+ *
	+ * Read all the HW stats counters to refresh values in hw_stats structure
	+ * of passed-in dev's pestat
	+ */
	+void
	+i40iw_hw_stats_refresh_all(struct i40iw_vsi_pestat *stats)
	+{
	+ u64 stats_value;
	+ u32 stats_index;
	+ unsigned long flags;
	+
	+ spin_lock_irqsave(&stats->lock, flags);
	+
	+ for (stats_index = 0; stats_index < I40IW_HW_STAT_INDEX_MAX_32;
	+ stats_index++)
	+ i40iw_hw_stats_read_32(stats, stats_index, &stats_value);
	+ for (stats_index = 0; stats_index < I40IW_HW_STAT_INDEX_MAX_64;
	+ stats_index++)
	+ i40iw_hw_stats_read_64(stats, stats_index, &stats_value);
	+ spin_unlock_irqrestore(&stats->lock, flags);
	+}
	+
	+/**
	+ * i40iw_get_fcn_id - Return the funcion id
	+ * @dev: pointer to the device
	+ */
	+static u8
	+i40iw_get_fcn_id(struct i40iw_sc_dev *dev)
	+{
	+ u8 fcn_id = I40IW_INVALID_FCN_ID;
	+ u8 i;
	+
	+ for (i = I40IW_FIRST_NON_PF_STAT; i < I40IW_MAX_STATS_COUNT; i++)
	+ if (!dev->fcn_id_array[i]) {
	+ fcn_id = i;
	+ dev->fcn_id_array[i] = TRUE;
	+ break;
	+ }
	+ return fcn_id;
	+}
	+
	+/**
	+ * i40iw_vsi_stats_init - Initialize the vsi statistics
	+ * @vsi: pointer to the vsi structure
	+ * @info: The info structure used for initialization
	+ */
	+enum i40iw_status_code
	+i40iw_vsi_stats_init(struct i40iw_sc_vsi vsi, struct i40iw_vsi_stats_info info)
	+{
	+ u8 fcn_id = info->fcn_id;
	+
	+ if (info->alloc_fcn_id)
	+ fcn_id = i40iw_get_fcn_id(vsi->dev);
	+
	+ if (fcn_id == I40IW_INVALID_FCN_ID)
	+ return I40IW_ERR_NOT_READY;
	+
	+ vsi->pestat = info->pestat;
	+ vsi->pestat->hw = vsi->dev->hw;
	+
	+ if (vsi->dev->is_pf) {
	+ i40iw_hw_stats_init(vsi->pestat, fcn_id, TRUE);
	+ spin_lock_init(&vsi->pestat->lock);
	+ i40iw_hw_stats_start_timer(vsi);
	+ }
	+ vsi->stats_fcn_id_alloc = info->alloc_fcn_id;
	+ vsi->fcn_id = fcn_id;
	+ return I40IW_SUCCESS;
	+}
	+
	+/**
	+ * i40iw_vsi_stats_free - Free the vsi stats
	+ * @vsi: pointer to the vsi structure
	+ */
	+void
	+i40iw_vsi_stats_free(struct i40iw_sc_vsi *vsi)
	+{
	+ u8 fcn_id = vsi->fcn_id;
	+
	+ if (vsi->stats_fcn_id_alloc && fcn_id < I40IW_MAX_STATS_COUNT)
	+ vsi->dev->fcn_id_array[fcn_id] = FALSE;
	+}
	+
	+static struct i40iw_cqp_ops iw_cqp_ops = {
	+ i40iw_sc_cqp_init,
	+ i40iw_sc_cqp_create,
	+ i40iw_sc_cqp_post_sq,
	+ i40iw_sc_cqp_get_next_send_wqe,
	+ i40iw_sc_cqp_destroy,
	+ i40iw_sc_poll_for_cqp_op_done
	+};
	+
	+static struct i40iw_ccq_ops iw_ccq_ops = {
	+ i40iw_sc_ccq_init,
	+ i40iw_sc_ccq_create,
	+ i40iw_sc_ccq_destroy,
	+ i40iw_sc_ccq_create_done,
	+ i40iw_sc_ccq_get_cqe_info,
	+ i40iw_sc_ccq_arm
	+};
	+
	+static struct i40iw_ceq_ops iw_ceq_ops = {
	+ i40iw_sc_ceq_init,
	+ i40iw_sc_ceq_create,
	+ i40iw_sc_cceq_create_done,
	+ i40iw_sc_cceq_destroy_done,
	+ i40iw_sc_cceq_create,
	+ i40iw_sc_ceq_destroy,
	+ i40iw_sc_process_ceq
	+};
	+
	+static struct i40iw_aeq_ops iw_aeq_ops = {
	+ i40iw_sc_aeq_init,
	+ i40iw_sc_aeq_create,
	+ i40iw_sc_aeq_destroy,
	+ i40iw_sc_get_next_aeqe,
	+ i40iw_sc_repost_aeq_entries,
	+ i40iw_sc_aeq_create_done,
	+ i40iw_sc_aeq_destroy_done
	+};
	+
	+/* iwarp pd ops */
	+static struct i40iw_pd_ops iw_pd_ops = {
	+ i40iw_sc_pd_init,
	+};
	+
	+static struct i40iw_priv_qp_ops iw_priv_qp_ops = {
	+ .qp_init = i40iw_sc_qp_init,
	+ .qp_create = i40iw_sc_qp_create,
	+ .qp_modify = i40iw_sc_qp_modify,
	+ .qp_destroy = i40iw_sc_qp_destroy,
	+ .qp_flush_wqes = i40iw_sc_qp_flush_wqes,
	+ .qp_upload_context = i40iw_sc_qp_upload_context,
	+ .qp_setctx = i40iw_sc_qp_setctx,
	+ .qp_send_lsmm = i40iw_sc_send_lsmm,
	+ .qp_send_lsmm_nostag = i40iw_sc_send_lsmm_nostag,
	+ .qp_send_rtt = i40iw_sc_send_rtt,
	+ .iw_mr_fast_register = i40iw_sc_mr_fast_register
	+};
	+
	+static struct i40iw_priv_cq_ops iw_priv_cq_ops = {
	+ i40iw_sc_cq_init,
	+ i40iw_sc_cq_create,
	+ i40iw_sc_cq_destroy,
	+ i40iw_sc_cq_modify,
	+};
	+
	+static struct i40iw_mr_ops iw_mr_ops = {
	+ i40iw_sc_alloc_stag,
	+ i40iw_sc_mr_reg_non_shared,
	+ i40iw_sc_mr_reg_shared,
	+ i40iw_sc_dealloc_stag,
	+ i40iw_sc_query_stag,
	+ i40iw_sc_mw_alloc
	+};
	+
	+static struct i40iw_cqp_misc_ops iw_cqp_misc_ops = {
	+ i40iw_sc_manage_push_page,
	+ i40iw_sc_manage_hmc_pm_func_table,
	+ i40iw_sc_commit_fpm_values,
	+ i40iw_sc_query_fpm_values,
	+ i40iw_sc_static_hmc_pages_allocated,
	+ i40iw_sc_add_arp_cache_entry,
	+ i40iw_sc_del_arp_cache_entry,
	+ i40iw_sc_query_arp_cache_entry,
	+ i40iw_sc_manage_apbvt_entry,
	+ i40iw_sc_manage_qhash_table_entry,
	+ i40iw_sc_alloc_local_mac_ipaddr_entry,
	+ i40iw_sc_add_local_mac_ipaddr_entry,
	+ i40iw_sc_del_local_mac_ipaddr_entry,
	+ i40iw_sc_cqp_nop,
	+ i40iw_sc_commit_fpm_values_done,
	+ i40iw_sc_query_fpm_values_done,
	+ i40iw_sc_manage_hmc_pm_func_table_done,
	+ i40iw_sc_suspend_qp,
	+ i40iw_sc_resume_qp
	+};
	+
	+static struct i40iw_hmc_ops iw_hmc_ops = {
	+ i40iw_sc_init_iw_hmc,
	+ i40iw_sc_parse_fpm_query_buf,
	+ i40iw_sc_configure_iw_fpm,
	+ i40iw_sc_parse_fpm_commit_buf,
	+ i40iw_sc_create_hmc_obj,
	+ i40iw_sc_del_hmc_obj,
	+ NULL,
	+ NULL
	+};
	+
	+/**
	+ * i40iw_device_init - Initialize IWARP device
	+ * @dev: IWARP device pointer
	+ * @info: IWARP init info
	+ */
	+enum i40iw_status_code
	+i40iw_device_init(struct i40iw_sc_dev *dev,
	+ struct i40iw_device_init_info *info)
	+{
	+ u32 val;
	+ enum i40iw_status_code ret_code = 0;
	+ u8 db_size;
	+
	+ spin_lock_init(&dev->cqp_lock);
	+
	+ i40iw_device_init_uk(&dev->dev_uk);
	+
	+ dev->debug_mask = info->debug_mask;
	+
	+ dev->hmc_fn_id = info->hmc_fn_id;
	+ dev->is_pf = info->is_pf;
	+
	+ dev->fpm_query_buf_pa = info->fpm_query_buf_pa;
	+ dev->fpm_query_buf = info->fpm_query_buf;
	+
	+ dev->fpm_commit_buf_pa = info->fpm_commit_buf_pa;
	+ dev->fpm_commit_buf = info->fpm_commit_buf;
	+
	+ dev->hw = info->hw;
	+ dev->hw->hw_addr = info->bar0;
	+
	+ if (dev->is_pf) {
	+ val = rd32(dev->hw, I40E_GLPCI_DREVID);
	+ dev->hw_rev = (u8)RS_32(val, I40E_GLPCI_DREVID_DEFAULT_REVID);
	+
	+ val = rd32(dev->hw, I40E_GLPCI_LBARCTRL);
	+ db_size = (u8)RS_32(val, I40E_GLPCI_LBARCTRL_PE_DB_SIZE);
	+ if ((db_size != I40IW_PE_DB_SIZE_4M) &&
	+ (db_size != I40IW_PE_DB_SIZE_8M)) {
	+ i40iw_debug(dev, I40IW_DEBUG_DEV,
	+ "%s: PE doorbell is not enabled in CSR val 0x%x\n",
	+ __func__, val);
	+ pr_err("RDMA feature not enabled!\n");
	+ ret_code = I40IW_ERR_PE_DOORBELL_NOT_ENABLED;
	+ return ret_code;
	+ }
	+ dev->db_addr = dev->hw->hw_addr + I40IW_DB_ADDR_OFFSET;
	+ } else {
	+ dev->db_addr = dev->hw->hw_addr + I40IW_VF_DB_ADDR_OFFSET;
	+ }
	+
	+ dev->cqp_ops = &iw_cqp_ops;
	+ dev->ccq_ops = &iw_ccq_ops;
	+ dev->ceq_ops = &iw_ceq_ops;
	+ dev->aeq_ops = &iw_aeq_ops;
	+ dev->cqp_misc_ops = &iw_cqp_misc_ops;
	+ dev->iw_pd_ops = &iw_pd_ops;
	+ dev->iw_priv_qp_ops = &iw_priv_qp_ops;
	+ dev->iw_priv_cq_ops = &iw_priv_cq_ops;
	+ dev->mr_ops = &iw_mr_ops;
	+ dev->hmc_ops = &iw_hmc_ops;
	+
	+ return ret_code;
	+}
	Index: sys/dev/ixl/iwarp/i40iw_d.h
	===================================================================
	--- /dev/null
	+++ sys/dev/ixl/iwarp/i40iw_d.h
	@@ -0,0 +1,1751 @@
	+/*******************************************************************************
	+*
	+* Copyright (c) 2015-2017 Intel Corporation. All rights reserved.
	+*
	+* This software is available to you under a choice of one of two
	+* licenses. You may choose to be licensed under the terms of the GNU
	+* General Public License (GPL) Version 2, available from the file
	+* COPYING in the main directory of this source tree, or the
	+* OpenFabrics.org BSD license below:
	+*
	+* Redistribution and use in source and binary forms, with or
	+* without modification, are permitted provided that the following
	+* conditions are met:
	+*
	+* - Redistributions of source code must retain the above
	+* copyright notice, this list of conditions and the following
	+* disclaimer.
	+*
	+* - Redistributions in binary form must reproduce the above
	+* copyright notice, this list of conditions and the following
	+* disclaimer in the documentation and/or other materials
	+* provided with the distribution.
	+*
	+* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
	+* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
	+* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
	+* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
	+* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
	+* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
	+* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
	+* SOFTWARE.
	+*
	+*******************************************************************************/
	+/$FreeBSD$/
	+
	+#ifndef I40IW_D_H
	+#define I40IW_D_H
	+
	+#define I40IW_FIRST_USER_QP_ID 2
	+
	+#define I40IW_DB_ADDR_OFFSET (4 * 1024 * 1024 - 64 * 1024)
	+#define I40IW_VF_DB_ADDR_OFFSET (64 * 1024)
	+
	+#define I40IW_PUSH_OFFSET (4 * 1024 * 1024)
	+#define I40IW_PF_FIRST_PUSH_PAGE_INDEX 16
	+#define I40IW_VF_PUSH_OFFSET ((8 + 64) * 1024)
	+#define I40IW_VF_FIRST_PUSH_PAGE_INDEX 2
	+
	+#define I40IW_PE_DB_SIZE_4M 1
	+#define I40IW_PE_DB_SIZE_8M 2
	+
	+#define I40IW_DDP_VER 1
	+#define I40IW_RDMAP_VER 1
	+
	+#define I40IW_RDMA_MODE_RDMAC 0
	+#define I40IW_RDMA_MODE_IETF 1
	+
	+#define I40IW_QP_STATE_INVALID 0
	+#define I40IW_QP_STATE_IDLE 1
	+#define I40IW_QP_STATE_RTS 2
	+#define I40IW_QP_STATE_CLOSING 3
	+#define I40IW_QP_STATE_RESERVED 4
	+#define I40IW_QP_STATE_TERMINATE 5
	+#define I40IW_QP_STATE_ERROR 6
	+
	+#define I40IW_STAG_STATE_INVALID 0
	+#define I40IW_STAG_STATE_VALID 1
	+
	+#define I40IW_STAG_TYPE_SHARED 0
	+#define I40IW_STAG_TYPE_NONSHARED 1
	+
	+#define I40IW_MAX_USER_PRIORITY 8
	+#define I40IW_MAX_STATS_COUNT 16
	+#define I40IW_FIRST_NON_PF_STAT 4
	+
	+#define I40IW_MIN_MTU_IPV4 576
	+#define I40IW_MIN_MTU_IPV6 1280
	+#define I40IW_MTU_TO_MSS_IPV4 40
	+#define I40IW_MTU_TO_MSS_IPV6 60
	+#define I40IW_DEFAULT_MTU 1500
	+
	+#define LS_64_1(val, bits) ((u64)(uintptr_t)val << bits)
	+#define RS_64_1(val, bits) ((u64)(uintptr_t)val >> bits)
	+#define LS_32_1(val, bits) (u32)(val << bits)
	+#define RS_32_1(val, bits) (u32)(val >> bits)
	+#define I40E_HI_DWORD(x) ((u32)((((x) >> 16) >> 16) & 0xFFFFFFFF))
	+
	+#define LS_64(val, field) (((u64)val << field ## _SHIFT) & (field ## _MASK))
	+
	+#define RS_64(val, field) ((u64)(val & field ## _MASK) >> field ## _SHIFT)
	+#define LS_32(val, field) ((val << field ## _SHIFT) & (field ## _MASK))
	+#define RS_32(val, field) ((val & field ## _MASK) >> field ## _SHIFT)
	+
	+#define TERM_DDP_LEN_TAGGED 14
	+#define TERM_DDP_LEN_UNTAGGED 18
	+#define TERM_RDMA_LEN 28
	+#define RDMA_OPCODE_MASK 0x0f
	+#define RDMA_READ_REQ_OPCODE 1
	+#define Q2_BAD_FRAME_OFFSET 72
	+#define Q2_FPSN_OFFSET 64
	+#define CQE_MAJOR_DRV 0x8000
	+
	+#define I40IW_TERM_SENT 0x01
	+#define I40IW_TERM_RCVD 0x02
	+#define I40IW_TERM_DONE 0x04
	+#define I40IW_MAC_HLEN 14
	+#define I40IW_BYTE_0 0
	+#define I40IW_BYTE_8 8
	+#define I40IW_BYTE_16 16
	+#define I40IW_BYTE_24 24
	+#define I40IW_BYTE_32 32
	+#define I40IW_BYTE_40 40
	+#define I40IW_BYTE_48 48
	+#define I40IW_BYTE_56 56
	+#define I40IW_BYTE_64 64
	+#define I40IW_BYTE_72 72
	+#define I40IW_BYTE_80 80
	+#define I40IW_BYTE_88 88
	+#define I40IW_BYTE_96 96
	+#define I40IW_BYTE_104 104
	+#define I40IW_BYTE_112 112
	+#define I40IW_BYTE_120 120
	+#define I40IW_BYTE_128 128
	+#define I40IW_BYTE_136 136
	+#define I40IW_BYTE_144 144
	+#define I40IW_BYTE_152 152
	+#define I40IW_BYTE_160 160
	+#define I40IW_BYTE_168 168
	+#define I40IW_BYTE_176 176
	+#define I40IW_BYTE_184 184
	+#define I40IW_BYTE_192 192
	+#define I40IW_BYTE_200 200
	+#define I40IW_BYTE_208 208
	+
	+#define I40IW_INVALID_WQE_INDEX 0xffffffff
	+
	+#define I40IW_CQP_WAIT_POLL_REGS 1
	+#define I40IW_CQP_WAIT_POLL_CQ 2
	+#define I40IW_CQP_WAIT_EVENT 3
	+
	+#define I40IW_CQP_INIT_WQE(wqe) memset(wqe, 0, 64)
	+
	+#define I40IW_GET_CURRENT_CQ_ELEMENT(_cq) \
	+ ( \
	+ &((_cq)->cq_base[I40IW_RING_GETCURRENT_HEAD((_cq)->cq_ring)]) \
	+ )
	+#define I40IW_GET_CURRENT_EXTENDED_CQ_ELEMENT(_cq) \
	+ ( \
	+ &(((struct i40iw_extended_cqe *) \
	+ ((_cq)->cq_base))[I40IW_RING_GETCURRENT_HEAD((_cq)->cq_ring)]) \
	+ )
	+
	+#define I40IW_GET_CURRENT_AEQ_ELEMENT(_aeq) \
	+ ( \
	+ &_aeq->aeqe_base[I40IW_RING_GETCURRENT_TAIL(_aeq->aeq_ring)] \
	+ )
	+
	+#define I40IW_GET_CURRENT_CEQ_ELEMENT(_ceq) \
	+ ( \
	+ &_ceq->ceqe_base[I40IW_RING_GETCURRENT_TAIL(_ceq->ceq_ring)] \
	+ )
	+
	+#define I40IW_AE_SOURCE_RSVD 0x0
	+#define I40IW_AE_SOURCE_RQ 0x1
	+#define I40IW_AE_SOURCE_RQ_0011 0x3
	+
	+#define I40IW_AE_SOURCE_CQ 0x2
	+#define I40IW_AE_SOURCE_CQ_0110 0x6
	+#define I40IW_AE_SOURCE_CQ_1010 0xA
	+#define I40IW_AE_SOURCE_CQ_1110 0xE
	+
	+#define I40IW_AE_SOURCE_SQ 0x5
	+#define I40IW_AE_SOURCE_SQ_0111 0x7
	+
	+#define I40IW_AE_SOURCE_IN_RR_WR 0x9
	+#define I40IW_AE_SOURCE_IN_RR_WR_1011 0xB
	+#define I40IW_AE_SOURCE_OUT_RR 0xD
	+#define I40IW_AE_SOURCE_OUT_RR_1111 0xF
	+
	+#define I40IW_TCP_STATE_NON_EXISTENT 0
	+#define I40IW_TCP_STATE_CLOSED 1
	+#define I40IW_TCP_STATE_LISTEN 2
	+#define I40IW_STATE_SYN_SEND 3
	+#define I40IW_TCP_STATE_SYN_RECEIVED 4
	+#define I40IW_TCP_STATE_ESTABLISHED 5
	+#define I40IW_TCP_STATE_CLOSE_WAIT 6
	+#define I40IW_TCP_STATE_FIN_WAIT_1 7
	+#define I40IW_TCP_STATE_CLOSING 8
	+#define I40IW_TCP_STATE_LAST_ACK 9
	+#define I40IW_TCP_STATE_FIN_WAIT_2 10
	+#define I40IW_TCP_STATE_TIME_WAIT 11
	+#define I40IW_TCP_STATE_RESERVED_1 12
	+#define I40IW_TCP_STATE_RESERVED_2 13
	+#define I40IW_TCP_STATE_RESERVED_3 14
	+#define I40IW_TCP_STATE_RESERVED_4 15
	+
	+/* ILQ CQP hash table fields */
	+#define I40IW_CQPSQ_QHASH_VLANID_SHIFT 32
	+#define I40IW_CQPSQ_QHASH_VLANID_MASK \
	+ ((u64)0xfff << I40IW_CQPSQ_QHASH_VLANID_SHIFT)
	+
	+#define I40IW_CQPSQ_QHASH_QPN_SHIFT 32
	+#define I40IW_CQPSQ_QHASH_QPN_MASK \
	+ ((u64)0x3ffff << I40IW_CQPSQ_QHASH_QPN_SHIFT)
	+
	+#define I40IW_CQPSQ_QHASH_QS_HANDLE_SHIFT 0
	+#define I40IW_CQPSQ_QHASH_QS_HANDLE_MASK ((u64)0x3ff << I40IW_CQPSQ_QHASH_QS_HANDLE_SHIFT)
	+
	+#define I40IW_CQPSQ_QHASH_SRC_PORT_SHIFT 16
	+#define I40IW_CQPSQ_QHASH_SRC_PORT_MASK \
	+ ((u64)0xffff << I40IW_CQPSQ_QHASH_SRC_PORT_SHIFT)
	+
	+#define I40IW_CQPSQ_QHASH_DEST_PORT_SHIFT 0
	+#define I40IW_CQPSQ_QHASH_DEST_PORT_MASK \
	+ ((u64)0xffff << I40IW_CQPSQ_QHASH_DEST_PORT_SHIFT)
	+
	+#define I40IW_CQPSQ_QHASH_ADDR0_SHIFT 32
	+#define I40IW_CQPSQ_QHASH_ADDR0_MASK \
	+ ((u64)0xffffffff << I40IW_CQPSQ_QHASH_ADDR0_SHIFT)
	+
	+#define I40IW_CQPSQ_QHASH_ADDR1_SHIFT 0
	+#define I40IW_CQPSQ_QHASH_ADDR1_MASK \
	+ ((u64)0xffffffff << I40IW_CQPSQ_QHASH_ADDR1_SHIFT)
	+
	+#define I40IW_CQPSQ_QHASH_ADDR2_SHIFT 32
	+#define I40IW_CQPSQ_QHASH_ADDR2_MASK \
	+ ((u64)0xffffffff << I40IW_CQPSQ_QHASH_ADDR2_SHIFT)
	+
	+#define I40IW_CQPSQ_QHASH_ADDR3_SHIFT 0
	+#define I40IW_CQPSQ_QHASH_ADDR3_MASK \
	+ ((u64)0xffffffff << I40IW_CQPSQ_QHASH_ADDR3_SHIFT)
	+
	+#define I40IW_CQPSQ_QHASH_WQEVALID_SHIFT 63
	+#define I40IW_CQPSQ_QHASH_WQEVALID_MASK \
	+ ((u64)0x1 << I40IW_CQPSQ_QHASH_WQEVALID_SHIFT)
	+#define I40IW_CQPSQ_QHASH_OPCODE_SHIFT 32
	+#define I40IW_CQPSQ_QHASH_OPCODE_MASK \
	+ ((u64)0x3f << I40IW_CQPSQ_QHASH_OPCODE_SHIFT)
	+
	+#define I40IW_CQPSQ_QHASH_MANAGE_SHIFT 61
	+#define I40IW_CQPSQ_QHASH_MANAGE_MASK \
	+ ((u64)0x3 << I40IW_CQPSQ_QHASH_MANAGE_SHIFT)
	+
	+#define I40IW_CQPSQ_QHASH_IPV4VALID_SHIFT 60
	+#define I40IW_CQPSQ_QHASH_IPV4VALID_MASK \
	+ ((u64)0x1 << I40IW_CQPSQ_QHASH_IPV4VALID_SHIFT)
	+
	+#define I40IW_CQPSQ_QHASH_VLANVALID_SHIFT 59
	+#define I40IW_CQPSQ_QHASH_VLANVALID_MASK \
	+ ((u64)0x1 << I40IW_CQPSQ_QHASH_VLANVALID_SHIFT)
	+
	+#define I40IW_CQPSQ_QHASH_ENTRYTYPE_SHIFT 42
	+#define I40IW_CQPSQ_QHASH_ENTRYTYPE_MASK \
	+ ((u64)0x7 << I40IW_CQPSQ_QHASH_ENTRYTYPE_SHIFT)
	+/* CQP Host Context */
	+#define I40IW_CQPHC_EN_DC_TCP_SHIFT 0
	+#define I40IW_CQPHC_EN_DC_TCP_MASK (1UL << I40IW_CQPHC_EN_DC_TCP_SHIFT)
	+
	+#define I40IW_CQPHC_SQSIZE_SHIFT 8
	+#define I40IW_CQPHC_SQSIZE_MASK (0xfUL << I40IW_CQPHC_SQSIZE_SHIFT)
	+
	+#define I40IW_CQPHC_DISABLE_PFPDUS_SHIFT 1
	+#define I40IW_CQPHC_DISABLE_PFPDUS_MASK (0x1UL << I40IW_CQPHC_DISABLE_PFPDUS_SHIFT)
	+
	+#define I40IW_CQPHC_ENABLED_VFS_SHIFT 32
	+#define I40IW_CQPHC_ENABLED_VFS_MASK (0x3fULL << I40IW_CQPHC_ENABLED_VFS_SHIFT)
	+
	+#define I40IW_CQPHC_HMC_PROFILE_SHIFT 0
	+#define I40IW_CQPHC_HMC_PROFILE_MASK (0x7ULL << I40IW_CQPHC_HMC_PROFILE_SHIFT)
	+
	+#define I40IW_CQPHC_SVER_SHIFT 24
	+#define I40IW_CQPHC_SVER_MASK (0xffUL << I40IW_CQPHC_SVER_SHIFT)
	+
	+#define I40IW_CQPHC_SQBASE_SHIFT 9
	+#define I40IW_CQPHC_SQBASE_MASK \
	+ (0xfffffffffffffeULL << I40IW_CQPHC_SQBASE_SHIFT)
	+
	+#define I40IW_CQPHC_QPCTX_SHIFT 0
	+#define I40IW_CQPHC_QPCTX_MASK \
	+ (0xffffffffffffffffULL << I40IW_CQPHC_QPCTX_SHIFT)
	+#define I40IW_CQPHC_SVER 1
	+
	+#define I40IW_CQP_SW_SQSIZE_4 4
	+#define I40IW_CQP_SW_SQSIZE_2048 2048
	+
	+/* iWARP QP Doorbell shadow area */
	+#define I40IW_QP_DBSA_HW_SQ_TAIL_SHIFT 0
	+#define I40IW_QP_DBSA_HW_SQ_TAIL_MASK \
	+ (0x3fffUL << I40IW_QP_DBSA_HW_SQ_TAIL_SHIFT)
	+
	+/* Completion Queue Doorbell shadow area */
	+#define I40IW_CQ_DBSA_CQEIDX_SHIFT 0
	+#define I40IW_CQ_DBSA_CQEIDX_MASK (0xfffffUL << I40IW_CQ_DBSA_CQEIDX_SHIFT)
	+
	+#define I40IW_CQ_DBSA_SW_CQ_SELECT_SHIFT 0
	+#define I40IW_CQ_DBSA_SW_CQ_SELECT_MASK \
	+ (0x3fffUL << I40IW_CQ_DBSA_SW_CQ_SELECT_SHIFT)
	+
	+#define I40IW_CQ_DBSA_ARM_NEXT_SHIFT 14
	+#define I40IW_CQ_DBSA_ARM_NEXT_MASK (1UL << I40IW_CQ_DBSA_ARM_NEXT_SHIFT)
	+
	+#define I40IW_CQ_DBSA_ARM_NEXT_SE_SHIFT 15
	+#define I40IW_CQ_DBSA_ARM_NEXT_SE_MASK (1UL << I40IW_CQ_DBSA_ARM_NEXT_SE_SHIFT)
	+
	+#define I40IW_CQ_DBSA_ARM_SEQ_NUM_SHIFT 16
	+#define I40IW_CQ_DBSA_ARM_SEQ_NUM_MASK \
	+ (0x3UL << I40IW_CQ_DBSA_ARM_SEQ_NUM_SHIFT)
	+
	+/* CQP and iWARP Completion Queue */
	+#define I40IW_CQ_QPCTX_SHIFT I40IW_CQPHC_QPCTX_SHIFT
	+#define I40IW_CQ_QPCTX_MASK I40IW_CQPHC_QPCTX_MASK
	+
	+#define I40IW_CCQ_OPRETVAL_SHIFT 0
	+#define I40IW_CCQ_OPRETVAL_MASK (0xffffffffUL << I40IW_CCQ_OPRETVAL_SHIFT)
	+
	+#define I40IW_CQ_MINERR_SHIFT 0
	+#define I40IW_CQ_MINERR_MASK (0xffffUL << I40IW_CQ_MINERR_SHIFT)
	+
	+#define I40IW_CQ_MAJERR_SHIFT 16
	+#define I40IW_CQ_MAJERR_MASK (0xffffUL << I40IW_CQ_MAJERR_SHIFT)
	+
	+#define I40IW_CQ_WQEIDX_SHIFT 32
	+#define I40IW_CQ_WQEIDX_MASK (0x3fffULL << I40IW_CQ_WQEIDX_SHIFT)
	+
	+#define I40IW_CQ_ERROR_SHIFT 55
	+#define I40IW_CQ_ERROR_MASK (1ULL << I40IW_CQ_ERROR_SHIFT)
	+
	+#define I40IW_CQ_SQ_SHIFT 62
	+#define I40IW_CQ_SQ_MASK (1ULL << I40IW_CQ_SQ_SHIFT)
	+
	+#define I40IW_CQ_VALID_SHIFT 63
	+#define I40IW_CQ_VALID_MASK (1ULL << I40IW_CQ_VALID_SHIFT)
	+
	+#define I40IWCQ_PAYLDLEN_SHIFT 0
	+#define I40IWCQ_PAYLDLEN_MASK (0xffffffffUL << I40IWCQ_PAYLDLEN_SHIFT)
	+
	+#define I40IWCQ_TCPSEQNUM_SHIFT 32
	+#define I40IWCQ_TCPSEQNUM_MASK (0xffffffffULL << I40IWCQ_TCPSEQNUM_SHIFT)
	+
	+#define I40IWCQ_INVSTAG_SHIFT 0
	+#define I40IWCQ_INVSTAG_MASK (0xffffffffUL << I40IWCQ_INVSTAG_SHIFT)
	+
	+#define I40IWCQ_QPID_SHIFT 32
	+#define I40IWCQ_QPID_MASK (0x3ffffULL << I40IWCQ_QPID_SHIFT)
	+
	+#define I40IWCQ_PSHDROP_SHIFT 51
	+#define I40IWCQ_PSHDROP_MASK (1ULL << I40IWCQ_PSHDROP_SHIFT)
	+
	+#define I40IWCQ_SRQ_SHIFT 52
	+#define I40IWCQ_SRQ_MASK (1ULL << I40IWCQ_SRQ_SHIFT)
	+
	+#define I40IWCQ_STAG_SHIFT 53
	+#define I40IWCQ_STAG_MASK (1ULL << I40IWCQ_STAG_SHIFT)
	+
	+#define I40IWCQ_SOEVENT_SHIFT 54
	+#define I40IWCQ_SOEVENT_MASK (1ULL << I40IWCQ_SOEVENT_SHIFT)
	+
	+#define I40IWCQ_OP_SHIFT 56
	+#define I40IWCQ_OP_MASK (0x3fULL << I40IWCQ_OP_SHIFT)
	+
	+/* CEQE format */
	+#define I40IW_CEQE_CQCTX_SHIFT 0
	+#define I40IW_CEQE_CQCTX_MASK \
	+ (0x7fffffffffffffffULL << I40IW_CEQE_CQCTX_SHIFT)
	+
	+#define I40IW_CEQE_VALID_SHIFT 63
	+#define I40IW_CEQE_VALID_MASK (1ULL << I40IW_CEQE_VALID_SHIFT)
	+
	+/* AEQE format */
	+#define I40IW_AEQE_COMPCTX_SHIFT I40IW_CQPHC_QPCTX_SHIFT
	+#define I40IW_AEQE_COMPCTX_MASK I40IW_CQPHC_QPCTX_MASK
	+
	+#define I40IW_AEQE_QPCQID_SHIFT 0
	+#define I40IW_AEQE_QPCQID_MASK (0x3ffffUL << I40IW_AEQE_QPCQID_SHIFT)
	+
	+#define I40IW_AEQE_WQDESCIDX_SHIFT 18
	+#define I40IW_AEQE_WQDESCIDX_MASK (0x3fffULL << I40IW_AEQE_WQDESCIDX_SHIFT)
	+
	+#define I40IW_AEQE_OVERFLOW_SHIFT 33
	+#define I40IW_AEQE_OVERFLOW_MASK (1ULL << I40IW_AEQE_OVERFLOW_SHIFT)
	+
	+#define I40IW_AEQE_AECODE_SHIFT 34
	+#define I40IW_AEQE_AECODE_MASK (0xffffULL << I40IW_AEQE_AECODE_SHIFT)
	+
	+#define I40IW_AEQE_AESRC_SHIFT 50
	+#define I40IW_AEQE_AESRC_MASK (0xfULL << I40IW_AEQE_AESRC_SHIFT)
	+
	+#define I40IW_AEQE_IWSTATE_SHIFT 54
	+#define I40IW_AEQE_IWSTATE_MASK (0x7ULL << I40IW_AEQE_IWSTATE_SHIFT)
	+
	+#define I40IW_AEQE_TCPSTATE_SHIFT 57
	+#define I40IW_AEQE_TCPSTATE_MASK (0xfULL << I40IW_AEQE_TCPSTATE_SHIFT)
	+
	+#define I40IW_AEQE_Q2DATA_SHIFT 61
	+#define I40IW_AEQE_Q2DATA_MASK (0x3ULL << I40IW_AEQE_Q2DATA_SHIFT)
	+
	+#define I40IW_AEQE_VALID_SHIFT 63
	+#define I40IW_AEQE_VALID_MASK (1ULL << I40IW_AEQE_VALID_SHIFT)
	+
	+/* CQP SQ WQES */
	+#define I40IW_QP_TYPE_IWARP 1
	+#define I40IW_QP_TYPE_UDA 2
	+#define I40IW_QP_TYPE_CQP 4
	+
	+#define I40IW_CQ_TYPE_IWARP 1
	+#define I40IW_CQ_TYPE_ILQ 2
	+#define I40IW_CQ_TYPE_IEQ 3
	+#define I40IW_CQ_TYPE_CQP 4
	+
	+#define I40IWQP_TERM_SEND_TERM_AND_FIN 0
	+#define I40IWQP_TERM_SEND_TERM_ONLY 1
	+#define I40IWQP_TERM_SEND_FIN_ONLY 2
	+#define I40IWQP_TERM_DONOT_SEND_TERM_OR_FIN 3
	+
	+#define I40IW_CQP_OP_CREATE_QP 0
	+#define I40IW_CQP_OP_MODIFY_QP 0x1
	+#define I40IW_CQP_OP_DESTROY_QP 0x02
	+#define I40IW_CQP_OP_CREATE_CQ 0x03
	+#define I40IW_CQP_OP_MODIFY_CQ 0x04
	+#define I40IW_CQP_OP_DESTROY_CQ 0x05
	+#define I40IW_CQP_OP_CREATE_SRQ 0x06
	+#define I40IW_CQP_OP_MODIFY_SRQ 0x07
	+#define I40IW_CQP_OP_DESTROY_SRQ 0x08
	+#define I40IW_CQP_OP_ALLOC_STAG 0x09
	+#define I40IW_CQP_OP_REG_MR 0x0a
	+#define I40IW_CQP_OP_QUERY_STAG 0x0b
	+#define I40IW_CQP_OP_REG_SMR 0x0c
	+#define I40IW_CQP_OP_DEALLOC_STAG 0x0d
	+#define I40IW_CQP_OP_MANAGE_LOC_MAC_IP_TABLE 0x0e
	+#define I40IW_CQP_OP_MANAGE_ARP 0x0f
	+#define I40IW_CQP_OP_MANAGE_VF_PBLE_BP 0x10
	+#define I40IW_CQP_OP_MANAGE_PUSH_PAGES 0x11
	+#define I40IW_CQP_OP_MANAGE_PE_TEAM 0x12
	+#define I40IW_CQP_OP_UPLOAD_CONTEXT 0x13
	+#define I40IW_CQP_OP_ALLOCATE_LOC_MAC_IP_TABLE_ENTRY 0x14
	+#define I40IW_CQP_OP_MANAGE_HMC_PM_FUNC_TABLE 0x15
	+#define I40IW_CQP_OP_CREATE_CEQ 0x16
	+#define I40IW_CQP_OP_DESTROY_CEQ 0x18
	+#define I40IW_CQP_OP_CREATE_AEQ 0x19
	+#define I40IW_CQP_OP_DESTROY_AEQ 0x1b
	+#define I40IW_CQP_OP_CREATE_ADDR_VECT 0x1c
	+#define I40IW_CQP_OP_MODIFY_ADDR_VECT 0x1d
	+#define I40IW_CQP_OP_DESTROY_ADDR_VECT 0x1e
	+#define I40IW_CQP_OP_UPDATE_PE_SDS 0x1f
	+#define I40IW_CQP_OP_QUERY_FPM_VALUES 0x20
	+#define I40IW_CQP_OP_COMMIT_FPM_VALUES 0x21
	+#define I40IW_CQP_OP_FLUSH_WQES 0x22
	+/* I40IW_CQP_OP_GEN_AE is the same value as I40IW_CQP_OP_FLUSH_WQES */
	+#define I40IW_CQP_OP_GEN_AE 0x22
	+#define I40IW_CQP_OP_MANAGE_APBVT 0x23
	+#define I40IW_CQP_OP_NOP 0x24
	+#define I40IW_CQP_OP_MANAGE_QUAD_HASH_TABLE_ENTRY 0x25
	+#define I40IW_CQP_OP_CREATE_UDA_MCAST_GROUP 0x26
	+#define I40IW_CQP_OP_MODIFY_UDA_MCAST_GROUP 0x27
	+#define I40IW_CQP_OP_DESTROY_UDA_MCAST_GROUP 0x28
	+#define I40IW_CQP_OP_SUSPEND_QP 0x29
	+#define I40IW_CQP_OP_RESUME_QP 0x2a
	+#define I40IW_CQP_OP_SHMC_PAGES_ALLOCATED 0x2b
	+
	+#define I40IW_UDA_QPSQ_NEXT_HEADER_SHIFT 16
	+#define I40IW_UDA_QPSQ_NEXT_HEADER_MASK ((u64)0xff << I40IW_UDA_QPSQ_NEXT_HEADER_SHIFT)
	+
	+#define I40IW_UDA_QPSQ_OPCODE_SHIFT 32
	+#define I40IW_UDA_QPSQ_OPCODE_MASK ((u64)0x3f << I40IW_UDA_QPSQ_OPCODE_SHIFT)
	+
	+#define I40IW_UDA_QPSQ_MACLEN_SHIFT 56
	+#define I40IW_UDA_QPSQ_MACLEN_MASK \
	+ ((u64)0x7f << I40IW_UDA_QPSQ_MACLEN_SHIFT)
	+
	+#define I40IW_UDA_QPSQ_IPLEN_SHIFT 48
	+#define I40IW_UDA_QPSQ_IPLEN_MASK \
	+ ((u64)0x7f << I40IW_UDA_QPSQ_IPLEN_SHIFT)
	+
	+#define I40IW_UDA_QPSQ_L4T_SHIFT 30
	+#define I40IW_UDA_QPSQ_L4T_MASK \
	+ ((u64)0x3 << I40IW_UDA_QPSQ_L4T_SHIFT)
	+
	+#define I40IW_UDA_QPSQ_IIPT_SHIFT 28
	+#define I40IW_UDA_QPSQ_IIPT_MASK \
	+ ((u64)0x3 << I40IW_UDA_QPSQ_IIPT_SHIFT)
	+
	+#define I40IW_UDA_QPSQ_L4LEN_SHIFT 24
	+#define I40IW_UDA_QPSQ_L4LEN_MASK ((u64)0xf << I40IW_UDA_QPSQ_L4LEN_SHIFT)
	+
	+#define I40IW_UDA_QPSQ_AVIDX_SHIFT 0
	+#define I40IW_UDA_QPSQ_AVIDX_MASK ((u64)0xffff << I40IW_UDA_QPSQ_AVIDX_SHIFT)
	+
	+#define I40IW_UDA_QPSQ_VALID_SHIFT 63
	+#define I40IW_UDA_QPSQ_VALID_MASK \
	+ ((u64)0x1 << I40IW_UDA_QPSQ_VALID_SHIFT)
	+
	+#define I40IW_UDA_QPSQ_SIGCOMPL_SHIFT 62
	+#define I40IW_UDA_QPSQ_SIGCOMPL_MASK ((u64)0x1 << I40IW_UDA_QPSQ_SIGCOMPL_SHIFT)
	+
	+#define I40IW_UDA_PAYLOADLEN_SHIFT 0
	+#define I40IW_UDA_PAYLOADLEN_MASK ((u64)0x3fff << I40IW_UDA_PAYLOADLEN_SHIFT)
	+
	+#define I40IW_UDA_HDRLEN_SHIFT 16
	+#define I40IW_UDA_HDRLEN_MASK ((u64)0x1ff << I40IW_UDA_HDRLEN_SHIFT)
	+
	+#define I40IW_VLAN_TAG_VALID_SHIFT 50
	+#define I40IW_VLAN_TAG_VALID_MASK ((u64)0x1 << I40IW_VLAN_TAG_VALID_SHIFT)
	+
	+#define I40IW_UDA_L3PROTO_SHIFT 0
	+#define I40IW_UDA_L3PROTO_MASK ((u64)0x3 << I40IW_UDA_L3PROTO_SHIFT)
	+
	+#define I40IW_UDA_L4PROTO_SHIFT 16
	+#define I40IW_UDA_L4PROTO_MASK ((u64)0x3 << I40IW_UDA_L4PROTO_SHIFT)
	+
	+#define I40IW_UDA_QPSQ_DOLOOPBACK_SHIFT 44
	+#define I40IW_UDA_QPSQ_DOLOOPBACK_MASK \
	+ ((u64)0x1 << I40IW_UDA_QPSQ_DOLOOPBACK_SHIFT)
	+
	+/* CQP SQ WQE common fields */
	+#define I40IW_CQPSQ_OPCODE_SHIFT 32
	+#define I40IW_CQPSQ_OPCODE_MASK (0x3fULL << I40IW_CQPSQ_OPCODE_SHIFT)
	+
	+#define I40IW_CQPSQ_WQEVALID_SHIFT 63
	+#define I40IW_CQPSQ_WQEVALID_MASK (1ULL << I40IW_CQPSQ_WQEVALID_SHIFT)
	+
	+#define I40IW_CQPSQ_TPHVAL_SHIFT 0
	+#define I40IW_CQPSQ_TPHVAL_MASK (0xffUL << I40IW_CQPSQ_TPHVAL_SHIFT)
	+
	+#define I40IW_CQPSQ_TPHEN_SHIFT 60
	+#define I40IW_CQPSQ_TPHEN_MASK (1ULL << I40IW_CQPSQ_TPHEN_SHIFT)
	+
	+#define I40IW_CQPSQ_PBUFADDR_SHIFT I40IW_CQPHC_QPCTX_SHIFT
	+#define I40IW_CQPSQ_PBUFADDR_MASK I40IW_CQPHC_QPCTX_MASK
	+
	+/* Create/Modify/Destroy QP */
	+
	+#define I40IW_CQPSQ_QP_NEWMSS_SHIFT 32
	+#define I40IW_CQPSQ_QP_NEWMSS_MASK (0x3fffULL << I40IW_CQPSQ_QP_NEWMSS_SHIFT)
	+
	+#define I40IW_CQPSQ_QP_TERMLEN_SHIFT 48
	+#define I40IW_CQPSQ_QP_TERMLEN_MASK (0xfULL << I40IW_CQPSQ_QP_TERMLEN_SHIFT)
	+
	+#define I40IW_CQPSQ_QP_QPCTX_SHIFT I40IW_CQPHC_QPCTX_SHIFT
	+#define I40IW_CQPSQ_QP_QPCTX_MASK I40IW_CQPHC_QPCTX_MASK
	+
	+#define I40IW_CQPSQ_QP_QPID_SHIFT 0
	+#define I40IW_CQPSQ_QP_QPID_MASK (0x3FFFFUL)
	+/* I40IWCQ_QPID_MASK */
	+
	+#define I40IW_CQPSQ_QP_OP_SHIFT 32
	+#define I40IW_CQPSQ_QP_OP_MASK I40IWCQ_OP_MASK
	+
	+#define I40IW_CQPSQ_QP_ORDVALID_SHIFT 42
	+#define I40IW_CQPSQ_QP_ORDVALID_MASK (1ULL << I40IW_CQPSQ_QP_ORDVALID_SHIFT)
	+
	+#define I40IW_CQPSQ_QP_TOECTXVALID_SHIFT 43
	+#define I40IW_CQPSQ_QP_TOECTXVALID_MASK \
	+ (1ULL << I40IW_CQPSQ_QP_TOECTXVALID_SHIFT)
	+
	+#define I40IW_CQPSQ_QP_CACHEDVARVALID_SHIFT 44
	+#define I40IW_CQPSQ_QP_CACHEDVARVALID_MASK \
	+ (1ULL << I40IW_CQPSQ_QP_CACHEDVARVALID_SHIFT)
	+
	+#define I40IW_CQPSQ_QP_VQ_SHIFT 45
	+#define I40IW_CQPSQ_QP_VQ_MASK (1ULL << I40IW_CQPSQ_QP_VQ_SHIFT)
	+
	+#define I40IW_CQPSQ_QP_FORCELOOPBACK_SHIFT 46
	+#define I40IW_CQPSQ_QP_FORCELOOPBACK_MASK \
	+ (1ULL << I40IW_CQPSQ_QP_FORCELOOPBACK_SHIFT)
	+
	+#define I40IW_CQPSQ_QP_CQNUMVALID_SHIFT 47
	+#define I40IW_CQPSQ_QP_CQNUMVALID_MASK \
	+ (1ULL << I40IW_CQPSQ_QP_CQNUMVALID_SHIFT)
	+
	+#define I40IW_CQPSQ_QP_QPTYPE_SHIFT 48
	+#define I40IW_CQPSQ_QP_QPTYPE_MASK (0x3ULL << I40IW_CQPSQ_QP_QPTYPE_SHIFT)
	+
	+#define I40IW_CQPSQ_QP_MSSCHANGE_SHIFT 52
	+#define I40IW_CQPSQ_QP_MSSCHANGE_MASK (1ULL << I40IW_CQPSQ_QP_MSSCHANGE_SHIFT)
	+
	+#define I40IW_CQPSQ_QP_IGNOREMWBOUND_SHIFT 54
	+#define I40IW_CQPSQ_QP_IGNOREMWBOUND_MASK \
	+ (1ULL << I40IW_CQPSQ_QP_IGNOREMWBOUND_SHIFT)
	+
	+#define I40IW_CQPSQ_QP_REMOVEHASHENTRY_SHIFT 55
	+#define I40IW_CQPSQ_QP_REMOVEHASHENTRY_MASK \
	+ (1ULL << I40IW_CQPSQ_QP_REMOVEHASHENTRY_SHIFT)
	+
	+#define I40IW_CQPSQ_QP_TERMACT_SHIFT 56
	+#define I40IW_CQPSQ_QP_TERMACT_MASK (0x3ULL << I40IW_CQPSQ_QP_TERMACT_SHIFT)
	+
	+#define I40IW_CQPSQ_QP_RESETCON_SHIFT 58
	+#define I40IW_CQPSQ_QP_RESETCON_MASK (1ULL << I40IW_CQPSQ_QP_RESETCON_SHIFT)
	+
	+#define I40IW_CQPSQ_QP_ARPTABIDXVALID_SHIFT 59
	+#define I40IW_CQPSQ_QP_ARPTABIDXVALID_MASK \
	+ (1ULL << I40IW_CQPSQ_QP_ARPTABIDXVALID_SHIFT)
	+
	+#define I40IW_CQPSQ_QP_NEXTIWSTATE_SHIFT 60
	+#define I40IW_CQPSQ_QP_NEXTIWSTATE_MASK \
	+ (0x7ULL << I40IW_CQPSQ_QP_NEXTIWSTATE_SHIFT)
	+
	+#define I40IW_CQPSQ_QP_DBSHADOWADDR_SHIFT I40IW_CQPHC_QPCTX_SHIFT
	+#define I40IW_CQPSQ_QP_DBSHADOWADDR_MASK I40IW_CQPHC_QPCTX_MASK
	+
	+/* Create/Modify/Destroy CQ */
	+#define I40IW_CQPSQ_CQ_CQSIZE_SHIFT 0
	+#define I40IW_CQPSQ_CQ_CQSIZE_MASK (0x3ffffUL << I40IW_CQPSQ_CQ_CQSIZE_SHIFT)
	+
	+#define I40IW_CQPSQ_CQ_CQCTX_SHIFT 0
	+#define I40IW_CQPSQ_CQ_CQCTX_MASK \
	+ (0x7fffffffffffffffULL << I40IW_CQPSQ_CQ_CQCTX_SHIFT)
	+
	+#define I40IW_CQPSQ_CQ_CQCTX_SHIFT 0
	+#define I40IW_CQPSQ_CQ_CQCTX_MASK \
	+ (0x7fffffffffffffffULL << I40IW_CQPSQ_CQ_CQCTX_SHIFT)
	+
	+#define I40IW_CQPSQ_CQ_SHADOW_READ_THRESHOLD_SHIFT 0
	+#define I40IW_CQPSQ_CQ_SHADOW_READ_THRESHOLD_MASK \
	+ (0x3ffff << I40IW_CQPSQ_CQ_SHADOW_READ_THRESHOLD_SHIFT)
	+
	+#define I40IW_CQPSQ_CQ_CEQID_SHIFT 24
	+#define I40IW_CQPSQ_CQ_CEQID_MASK (0x7fUL << I40IW_CQPSQ_CQ_CEQID_SHIFT)
	+
	+#define I40IW_CQPSQ_CQ_OP_SHIFT 32
	+#define I40IW_CQPSQ_CQ_OP_MASK (0x3fULL << I40IW_CQPSQ_CQ_OP_SHIFT)
	+
	+#define I40IW_CQPSQ_CQ_CQRESIZE_SHIFT 43
	+#define I40IW_CQPSQ_CQ_CQRESIZE_MASK (1ULL << I40IW_CQPSQ_CQ_CQRESIZE_SHIFT)
	+
	+#define I40IW_CQPSQ_CQ_LPBLSIZE_SHIFT 44
	+#define I40IW_CQPSQ_CQ_LPBLSIZE_MASK (3ULL << I40IW_CQPSQ_CQ_LPBLSIZE_SHIFT)
	+
	+#define I40IW_CQPSQ_CQ_CHKOVERFLOW_SHIFT 46
	+#define I40IW_CQPSQ_CQ_CHKOVERFLOW_MASK \
	+ (1ULL << I40IW_CQPSQ_CQ_CHKOVERFLOW_SHIFT)
	+
	+#define I40IW_CQPSQ_CQ_VIRTMAP_SHIFT 47
	+#define I40IW_CQPSQ_CQ_VIRTMAP_MASK (1ULL << I40IW_CQPSQ_CQ_VIRTMAP_SHIFT)
	+
	+#define I40IW_CQPSQ_CQ_ENCEQEMASK_SHIFT 48
	+#define I40IW_CQPSQ_CQ_ENCEQEMASK_MASK \
	+ (1ULL << I40IW_CQPSQ_CQ_ENCEQEMASK_SHIFT)
	+
	+#define I40IW_CQPSQ_CQ_CEQIDVALID_SHIFT 49
	+#define I40IW_CQPSQ_CQ_CEQIDVALID_MASK \
	+ (1ULL << I40IW_CQPSQ_CQ_CEQIDVALID_SHIFT)
	+
	+#define I40IW_CQPSQ_CQ_AVOIDMEMCNFLCT_SHIFT 61
	+#define I40IW_CQPSQ_CQ_AVOIDMEMCNFLCT_MASK \
	+ (1ULL << I40IW_CQPSQ_CQ_AVOIDMEMCNFLCT_SHIFT)
	+
	+/* Create/Modify/Destroy Shared Receive Queue */
	+
	+#define I40IW_CQPSQ_SRQ_RQSIZE_SHIFT 0
	+#define I40IW_CQPSQ_SRQ_RQSIZE_MASK (0xfUL << I40IW_CQPSQ_SRQ_RQSIZE_SHIFT)
	+
	+#define I40IW_CQPSQ_SRQ_RQWQESIZE_SHIFT 4
	+#define I40IW_CQPSQ_SRQ_RQWQESIZE_MASK \
	+ (0x7UL << I40IW_CQPSQ_SRQ_RQWQESIZE_SHIFT)
	+
	+#define I40IW_CQPSQ_SRQ_SRQLIMIT_SHIFT 32
	+#define I40IW_CQPSQ_SRQ_SRQLIMIT_MASK \
	+ (0xfffULL << I40IW_CQPSQ_SRQ_SRQLIMIT_SHIFT)
	+
	+#define I40IW_CQPSQ_SRQ_SRQCTX_SHIFT I40IW_CQPHC_QPCTX_SHIFT
	+#define I40IW_CQPSQ_SRQ_SRQCTX_MASK I40IW_CQPHC_QPCTX_MASK
	+
	+#define I40IW_CQPSQ_SRQ_PDID_SHIFT 16
	+#define I40IW_CQPSQ_SRQ_PDID_MASK \
	+ (0x7fffULL << I40IW_CQPSQ_SRQ_PDID_SHIFT)
	+
	+#define I40IW_CQPSQ_SRQ_SRQID_SHIFT 0
	+#define I40IW_CQPSQ_SRQ_SRQID_MASK (0x7fffUL << I40IW_CQPSQ_SRQ_SRQID_SHIFT)
	+
	+#define I40IW_CQPSQ_SRQ_LPBLSIZE_SHIFT I40IW_CQPSQ_CQ_LPBLSIZE_SHIFT
	+#define I40IW_CQPSQ_SRQ_LPBLSIZE_MASK I40IW_CQPSQ_CQ_LPBLSIZE_MASK
	+
	+#define I40IW_CQPSQ_SRQ_VIRTMAP_SHIFT I40IW_CQPSQ_CQ_VIRTMAP_SHIFT
	+#define I40IW_CQPSQ_SRQ_VIRTMAP_MASK I40IW_CQPSQ_CQ_VIRTMAP_MASK
	+
	+#define I40IW_CQPSQ_SRQ_TPHEN_SHIFT I40IW_CQPSQ_TPHEN_SHIFT
	+#define I40IW_CQPSQ_SRQ_TPHEN_MASK I40IW_CQPSQ_TPHEN_MASK
	+
	+#define I40IW_CQPSQ_SRQ_ARMLIMITEVENT_SHIFT 61
	+#define I40IW_CQPSQ_SRQ_ARMLIMITEVENT_MASK \
	+ (1ULL << I40IW_CQPSQ_SRQ_ARMLIMITEVENT_SHIFT)
	+
	+#define I40IW_CQPSQ_SRQ_DBSHADOWAREA_SHIFT 6
	+#define I40IW_CQPSQ_SRQ_DBSHADOWAREA_MASK \
	+ (0x3ffffffffffffffULL << I40IW_CQPSQ_SRQ_DBSHADOWAREA_SHIFT)
	+
	+#define I40IW_CQPSQ_SRQ_FIRSTPMPBLIDX_SHIFT 0
	+#define I40IW_CQPSQ_SRQ_FIRSTPMPBLIDX_MASK \
	+ (0xfffffffUL << I40IW_CQPSQ_SRQ_FIRSTPMPBLIDX_SHIFT)
	+
	+/* Allocate/Register/Register Shared/Deallocate Stag */
	+#define I40IW_CQPSQ_STAG_VA_FBO_SHIFT I40IW_CQPHC_QPCTX_SHIFT
	+#define I40IW_CQPSQ_STAG_VA_FBO_MASK I40IW_CQPHC_QPCTX_MASK
	+
	+#define I40IW_CQPSQ_STAG_STAGLEN_SHIFT 0
	+#define I40IW_CQPSQ_STAG_STAGLEN_MASK \
	+ (0x3fffffffffffULL << I40IW_CQPSQ_STAG_STAGLEN_SHIFT)
	+
	+#define I40IW_CQPSQ_STAG_PDID_SHIFT 48
	+#define I40IW_CQPSQ_STAG_PDID_MASK (0x7fffULL << I40IW_CQPSQ_STAG_PDID_SHIFT)
	+
	+#define I40IW_CQPSQ_STAG_KEY_SHIFT 0
	+#define I40IW_CQPSQ_STAG_KEY_MASK (0xffUL << I40IW_CQPSQ_STAG_KEY_SHIFT)
	+
	+#define I40IW_CQPSQ_STAG_IDX_SHIFT 8
	+#define I40IW_CQPSQ_STAG_IDX_MASK (0xffffffUL << I40IW_CQPSQ_STAG_IDX_SHIFT)
	+
	+#define I40IW_CQPSQ_STAG_PARENTSTAGIDX_SHIFT 32
	+#define I40IW_CQPSQ_STAG_PARENTSTAGIDX_MASK \
	+ (0xffffffULL << I40IW_CQPSQ_STAG_PARENTSTAGIDX_SHIFT)
	+
	+#define I40IW_CQPSQ_STAG_MR_SHIFT 43
	+#define I40IW_CQPSQ_STAG_MR_MASK (1ULL << I40IW_CQPSQ_STAG_MR_SHIFT)
	+
	+#define I40IW_CQPSQ_STAG_LPBLSIZE_SHIFT I40IW_CQPSQ_CQ_LPBLSIZE_SHIFT
	+#define I40IW_CQPSQ_STAG_LPBLSIZE_MASK I40IW_CQPSQ_CQ_LPBLSIZE_MASK
	+
	+#define I40IW_CQPSQ_STAG_HPAGESIZE_SHIFT 46
	+#define I40IW_CQPSQ_STAG_HPAGESIZE_MASK \
	+ (1ULL << I40IW_CQPSQ_STAG_HPAGESIZE_SHIFT)
	+
	+#define I40IW_CQPSQ_STAG_ARIGHTS_SHIFT 48
	+#define I40IW_CQPSQ_STAG_ARIGHTS_MASK \
	+ (0x1fULL << I40IW_CQPSQ_STAG_ARIGHTS_SHIFT)
	+
	+#define I40IW_CQPSQ_STAG_REMACCENABLED_SHIFT 53
	+#define I40IW_CQPSQ_STAG_REMACCENABLED_MASK \
	+ (1ULL << I40IW_CQPSQ_STAG_REMACCENABLED_SHIFT)
	+
	+#define I40IW_CQPSQ_STAG_VABASEDTO_SHIFT 59
	+#define I40IW_CQPSQ_STAG_VABASEDTO_MASK \
	+ (1ULL << I40IW_CQPSQ_STAG_VABASEDTO_SHIFT)
	+
	+#define I40IW_CQPSQ_STAG_USEHMCFNIDX_SHIFT 60
	+#define I40IW_CQPSQ_STAG_USEHMCFNIDX_MASK \
	+ (1ULL << I40IW_CQPSQ_STAG_USEHMCFNIDX_SHIFT)
	+
	+#define I40IW_CQPSQ_STAG_USEPFRID_SHIFT 61
	+#define I40IW_CQPSQ_STAG_USEPFRID_MASK \
	+ (1ULL << I40IW_CQPSQ_STAG_USEPFRID_SHIFT)
	+
	+#define I40IW_CQPSQ_STAG_PBA_SHIFT I40IW_CQPHC_QPCTX_SHIFT
	+#define I40IW_CQPSQ_STAG_PBA_MASK I40IW_CQPHC_QPCTX_MASK
	+
	+#define I40IW_CQPSQ_STAG_HMCFNIDX_SHIFT 0
	+#define I40IW_CQPSQ_STAG_HMCFNIDX_MASK \
	+ (0x3fUL << I40IW_CQPSQ_STAG_HMCFNIDX_SHIFT)
	+
	+#define I40IW_CQPSQ_STAG_FIRSTPMPBLIDX_SHIFT 0
	+#define I40IW_CQPSQ_STAG_FIRSTPMPBLIDX_MASK \
	+ (0xfffffffUL << I40IW_CQPSQ_STAG_FIRSTPMPBLIDX_SHIFT)
	+
	+/* Query stag */
	+#define I40IW_CQPSQ_QUERYSTAG_IDX_SHIFT I40IW_CQPSQ_STAG_IDX_SHIFT
	+#define I40IW_CQPSQ_QUERYSTAG_IDX_MASK I40IW_CQPSQ_STAG_IDX_MASK
	+
	+/* Allocate Local IP Address Entry */
	+
	+/* Manage Local IP Address Table - MLIPA */
	+#define I40IW_CQPSQ_MLIPA_IPV6LO_SHIFT I40IW_CQPHC_QPCTX_SHIFT
	+#define I40IW_CQPSQ_MLIPA_IPV6LO_MASK I40IW_CQPHC_QPCTX_MASK
	+
	+#define I40IW_CQPSQ_MLIPA_IPV6HI_SHIFT I40IW_CQPHC_QPCTX_SHIFT
	+#define I40IW_CQPSQ_MLIPA_IPV6HI_MASK I40IW_CQPHC_QPCTX_MASK
	+
	+#define I40IW_CQPSQ_MLIPA_IPV4_SHIFT 0
	+#define I40IW_CQPSQ_MLIPA_IPV4_MASK \
	+ (0xffffffffUL << I40IW_CQPSQ_MLIPA_IPV4_SHIFT)
	+
	+#define I40IW_CQPSQ_MLIPA_IPTABLEIDX_SHIFT 0
	+#define I40IW_CQPSQ_MLIPA_IPTABLEIDX_MASK \
	+ (0x3fUL << I40IW_CQPSQ_MLIPA_IPTABLEIDX_SHIFT)
	+
	+#define I40IW_CQPSQ_MLIPA_IPV4VALID_SHIFT 42
	+#define I40IW_CQPSQ_MLIPA_IPV4VALID_MASK \
	+ (1ULL << I40IW_CQPSQ_MLIPA_IPV4VALID_SHIFT)
	+
	+#define I40IW_CQPSQ_MLIPA_IPV6VALID_SHIFT 43
	+#define I40IW_CQPSQ_MLIPA_IPV6VALID_MASK \
	+ (1ULL << I40IW_CQPSQ_MLIPA_IPV6VALID_SHIFT)
	+
	+#define I40IW_CQPSQ_MLIPA_FREEENTRY_SHIFT 62
	+#define I40IW_CQPSQ_MLIPA_FREEENTRY_MASK \
	+ (1ULL << I40IW_CQPSQ_MLIPA_FREEENTRY_SHIFT)
	+
	+#define I40IW_CQPSQ_MLIPA_IGNORE_REF_CNT_SHIFT 61
	+#define I40IW_CQPSQ_MLIPA_IGNORE_REF_CNT_MASK \
	+ (1ULL << I40IW_CQPSQ_MLIPA_IGNORE_REF_CNT_SHIFT)
	+
	+#define I40IW_CQPSQ_MLIPA_MAC0_SHIFT 0
	+#define I40IW_CQPSQ_MLIPA_MAC0_MASK (0xffUL << I40IW_CQPSQ_MLIPA_MAC0_SHIFT)
	+
	+#define I40IW_CQPSQ_MLIPA_MAC1_SHIFT 8
	+#define I40IW_CQPSQ_MLIPA_MAC1_MASK (0xffUL << I40IW_CQPSQ_MLIPA_MAC1_SHIFT)
	+
	+#define I40IW_CQPSQ_MLIPA_MAC2_SHIFT 16
	+#define I40IW_CQPSQ_MLIPA_MAC2_MASK (0xffUL << I40IW_CQPSQ_MLIPA_MAC2_SHIFT)
	+
	+#define I40IW_CQPSQ_MLIPA_MAC3_SHIFT 24
	+#define I40IW_CQPSQ_MLIPA_MAC3_MASK (0xffUL << I40IW_CQPSQ_MLIPA_MAC3_SHIFT)
	+
	+#define I40IW_CQPSQ_MLIPA_MAC4_SHIFT 32
	+#define I40IW_CQPSQ_MLIPA_MAC4_MASK (0xffULL << I40IW_CQPSQ_MLIPA_MAC4_SHIFT)
	+
	+#define I40IW_CQPSQ_MLIPA_MAC5_SHIFT 40
	+#define I40IW_CQPSQ_MLIPA_MAC5_MASK (0xffULL << I40IW_CQPSQ_MLIPA_MAC5_SHIFT)
	+
	+/* Manage ARP Table - MAT */
	+#define I40IW_CQPSQ_MAT_REACHMAX_SHIFT 0
	+#define I40IW_CQPSQ_MAT_REACHMAX_MASK \
	+ (0xffffffffUL << I40IW_CQPSQ_MAT_REACHMAX_SHIFT)
	+
	+#define I40IW_CQPSQ_MAT_MACADDR_SHIFT 0
	+#define I40IW_CQPSQ_MAT_MACADDR_MASK \
	+ (0xffffffffffffULL << I40IW_CQPSQ_MAT_MACADDR_SHIFT)
	+
	+#define I40IW_CQPSQ_MAT_ARPENTRYIDX_SHIFT 0
	+#define I40IW_CQPSQ_MAT_ARPENTRYIDX_MASK \
	+ (0xfffUL << I40IW_CQPSQ_MAT_ARPENTRYIDX_SHIFT)
	+
	+#define I40IW_CQPSQ_MAT_ENTRYVALID_SHIFT 42
	+#define I40IW_CQPSQ_MAT_ENTRYVALID_MASK \
	+ (1ULL << I40IW_CQPSQ_MAT_ENTRYVALID_SHIFT)
	+
	+#define I40IW_CQPSQ_MAT_PERMANENT_SHIFT 43
	+#define I40IW_CQPSQ_MAT_PERMANENT_MASK \
	+ (1ULL << I40IW_CQPSQ_MAT_PERMANENT_SHIFT)
	+
	+#define I40IW_CQPSQ_MAT_QUERY_SHIFT 44
	+#define I40IW_CQPSQ_MAT_QUERY_MASK (1ULL << I40IW_CQPSQ_MAT_QUERY_SHIFT)
	+
	+/* Manage VF PBLE Backing Pages - MVPBP*/
	+#define I40IW_CQPSQ_MVPBP_PD_ENTRY_CNT_SHIFT 0
	+#define I40IW_CQPSQ_MVPBP_PD_ENTRY_CNT_MASK \
	+ (0x3ffULL << I40IW_CQPSQ_MVPBP_PD_ENTRY_CNT_SHIFT)
	+
	+#define I40IW_CQPSQ_MVPBP_FIRST_PD_INX_SHIFT 16
	+#define I40IW_CQPSQ_MVPBP_FIRST_PD_INX_MASK \
	+ (0x1ffULL << I40IW_CQPSQ_MVPBP_FIRST_PD_INX_SHIFT)
	+
	+#define I40IW_CQPSQ_MVPBP_SD_INX_SHIFT 32
	+#define I40IW_CQPSQ_MVPBP_SD_INX_MASK \
	+ (0xfffULL << I40IW_CQPSQ_MVPBP_SD_INX_SHIFT)
	+
	+#define I40IW_CQPSQ_MVPBP_INV_PD_ENT_SHIFT 62
	+#define I40IW_CQPSQ_MVPBP_INV_PD_ENT_MASK \
	+ (0x1ULL << I40IW_CQPSQ_MVPBP_INV_PD_ENT_SHIFT)
	+
	+#define I40IW_CQPSQ_MVPBP_PD_PLPBA_SHIFT 3
	+#define I40IW_CQPSQ_MVPBP_PD_PLPBA_MASK \
	+ (0x1fffffffffffffffULL << I40IW_CQPSQ_MVPBP_PD_PLPBA_SHIFT)
	+
	+/* Manage Push Page - MPP */
	+#define I40IW_INVALID_PUSH_PAGE_INDEX 0xffff
	+
	+#define I40IW_CQPSQ_MPP_QS_HANDLE_SHIFT 0
	+#define I40IW_CQPSQ_MPP_QS_HANDLE_MASK (0xffffUL << \
	+ I40IW_CQPSQ_MPP_QS_HANDLE_SHIFT)
	+
	+#define I40IW_CQPSQ_MPP_PPIDX_SHIFT 0
	+#define I40IW_CQPSQ_MPP_PPIDX_MASK (0x3ffUL << I40IW_CQPSQ_MPP_PPIDX_SHIFT)
	+
	+#define I40IW_CQPSQ_MPP_FREE_PAGE_SHIFT 62
	+#define I40IW_CQPSQ_MPP_FREE_PAGE_MASK (1ULL << I40IW_CQPSQ_MPP_FREE_PAGE_SHIFT)
	+
	+/* Upload Context - UCTX */
	+#define I40IW_CQPSQ_UCTX_QPCTXADDR_SHIFT I40IW_CQPHC_QPCTX_SHIFT
	+#define I40IW_CQPSQ_UCTX_QPCTXADDR_MASK I40IW_CQPHC_QPCTX_MASK
	+
	+#define I40IW_CQPSQ_UCTX_QPID_SHIFT 0
	+#define I40IW_CQPSQ_UCTX_QPID_MASK (0x3ffffUL << I40IW_CQPSQ_UCTX_QPID_SHIFT)
	+
	+#define I40IW_CQPSQ_UCTX_QPTYPE_SHIFT 48
	+#define I40IW_CQPSQ_UCTX_QPTYPE_MASK (0xfULL << I40IW_CQPSQ_UCTX_QPTYPE_SHIFT)
	+
	+#define I40IW_CQPSQ_UCTX_RAWFORMAT_SHIFT 61
	+#define I40IW_CQPSQ_UCTX_RAWFORMAT_MASK \
	+ (1ULL << I40IW_CQPSQ_UCTX_RAWFORMAT_SHIFT)
	+
	+#define I40IW_CQPSQ_UCTX_FREEZEQP_SHIFT 62
	+#define I40IW_CQPSQ_UCTX_FREEZEQP_MASK \
	+ (1ULL << I40IW_CQPSQ_UCTX_FREEZEQP_SHIFT)
	+
	+/* Manage HMC PM Function Table - MHMC */
	+#define I40IW_CQPSQ_MHMC_VFIDX_SHIFT 0
	+#define I40IW_CQPSQ_MHMC_VFIDX_MASK (0x7fUL << I40IW_CQPSQ_MHMC_VFIDX_SHIFT)
	+
	+#define I40IW_CQPSQ_MHMC_FREEPMFN_SHIFT 62
	+#define I40IW_CQPSQ_MHMC_FREEPMFN_MASK \
	+ (1ULL << I40IW_CQPSQ_MHMC_FREEPMFN_SHIFT)
	+
	+/* Set HMC Resource Profile - SHMCRP */
	+#define I40IW_CQPSQ_SHMCRP_HMC_PROFILE_SHIFT 0
	+#define I40IW_CQPSQ_SHMCRP_HMC_PROFILE_MASK \
	+ (0x7ULL << I40IW_CQPSQ_SHMCRP_HMC_PROFILE_SHIFT)
	+#define I40IW_CQPSQ_SHMCRP_VFNUM_SHIFT 32
	+#define I40IW_CQPSQ_SHMCRP_VFNUM_MASK (0x3fULL << I40IW_CQPSQ_SHMCRP_VFNUM_SHIFT)
	+
	+/* Create/Destroy CEQ */
	+#define I40IW_CQPSQ_CEQ_CEQSIZE_SHIFT 0
	+#define I40IW_CQPSQ_CEQ_CEQSIZE_MASK \
	+ (0x1ffffUL << I40IW_CQPSQ_CEQ_CEQSIZE_SHIFT)
	+
	+#define I40IW_CQPSQ_CEQ_CEQID_SHIFT 0
	+#define I40IW_CQPSQ_CEQ_CEQID_MASK (0x7fUL << I40IW_CQPSQ_CEQ_CEQID_SHIFT)
	+
	+#define I40IW_CQPSQ_CEQ_LPBLSIZE_SHIFT I40IW_CQPSQ_CQ_LPBLSIZE_SHIFT
	+#define I40IW_CQPSQ_CEQ_LPBLSIZE_MASK I40IW_CQPSQ_CQ_LPBLSIZE_MASK
	+
	+#define I40IW_CQPSQ_CEQ_VMAP_SHIFT 47
	+#define I40IW_CQPSQ_CEQ_VMAP_MASK (1ULL << I40IW_CQPSQ_CEQ_VMAP_SHIFT)
	+
	+#define I40IW_CQPSQ_CEQ_FIRSTPMPBLIDX_SHIFT 0
	+#define I40IW_CQPSQ_CEQ_FIRSTPMPBLIDX_MASK \
	+ (0xfffffffUL << I40IW_CQPSQ_CEQ_FIRSTPMPBLIDX_SHIFT)
	+
	+/* Create/Destroy AEQ */
	+#define I40IW_CQPSQ_AEQ_AEQECNT_SHIFT 0
	+#define I40IW_CQPSQ_AEQ_AEQECNT_MASK \
	+ (0x7ffffUL << I40IW_CQPSQ_AEQ_AEQECNT_SHIFT)
	+
	+#define I40IW_CQPSQ_AEQ_LPBLSIZE_SHIFT I40IW_CQPSQ_CQ_LPBLSIZE_SHIFT
	+#define I40IW_CQPSQ_AEQ_LPBLSIZE_MASK I40IW_CQPSQ_CQ_LPBLSIZE_MASK
	+
	+#define I40IW_CQPSQ_AEQ_VMAP_SHIFT 47
	+#define I40IW_CQPSQ_AEQ_VMAP_MASK (1ULL << I40IW_CQPSQ_AEQ_VMAP_SHIFT)
	+
	+#define I40IW_CQPSQ_AEQ_FIRSTPMPBLIDX_SHIFT 0
	+#define I40IW_CQPSQ_AEQ_FIRSTPMPBLIDX_MASK \
	+ (0xfffffffUL << I40IW_CQPSQ_AEQ_FIRSTPMPBLIDX_SHIFT)
	+
	+/* Commit FPM Values - CFPM */
	+#define I40IW_CQPSQ_CFPM_HMCFNID_SHIFT 0
	+#define I40IW_CQPSQ_CFPM_HMCFNID_MASK (0x3fUL << I40IW_CQPSQ_CFPM_HMCFNID_SHIFT)
	+
	+/* Flush WQEs - FWQE */
	+#define I40IW_CQPSQ_FWQE_AECODE_SHIFT 0
	+#define I40IW_CQPSQ_FWQE_AECODE_MASK (0xffffUL << I40IW_CQPSQ_FWQE_AECODE_SHIFT)
	+
	+#define I40IW_CQPSQ_FWQE_AESOURCE_SHIFT 16
	+#define I40IW_CQPSQ_FWQE_AESOURCE_MASK \
	+ (0xfUL << I40IW_CQPSQ_FWQE_AESOURCE_SHIFT)
	+
	+#define I40IW_CQPSQ_FWQE_RQMNERR_SHIFT 0
	+#define I40IW_CQPSQ_FWQE_RQMNERR_MASK \
	+ (0xffffUL << I40IW_CQPSQ_FWQE_RQMNERR_SHIFT)
	+
	+#define I40IW_CQPSQ_FWQE_RQMJERR_SHIFT 16
	+#define I40IW_CQPSQ_FWQE_RQMJERR_MASK \
	+ (0xffffUL << I40IW_CQPSQ_FWQE_RQMJERR_SHIFT)
	+
	+#define I40IW_CQPSQ_FWQE_SQMNERR_SHIFT 32
	+#define I40IW_CQPSQ_FWQE_SQMNERR_MASK \
	+ (0xffffULL << I40IW_CQPSQ_FWQE_SQMNERR_SHIFT)
	+
	+#define I40IW_CQPSQ_FWQE_SQMJERR_SHIFT 48
	+#define I40IW_CQPSQ_FWQE_SQMJERR_MASK \
	+ (0xffffULL << I40IW_CQPSQ_FWQE_SQMJERR_SHIFT)
	+
	+#define I40IW_CQPSQ_FWQE_QPID_SHIFT 0
	+#define I40IW_CQPSQ_FWQE_QPID_MASK (0x3ffffULL << I40IW_CQPSQ_FWQE_QPID_SHIFT)
	+
	+#define I40IW_CQPSQ_FWQE_GENERATE_AE_SHIFT 59
	+#define I40IW_CQPSQ_FWQE_GENERATE_AE_MASK (1ULL << \
	+ I40IW_CQPSQ_FWQE_GENERATE_AE_SHIFT)
	+
	+#define I40IW_CQPSQ_FWQE_USERFLCODE_SHIFT 60
	+#define I40IW_CQPSQ_FWQE_USERFLCODE_MASK \
	+ (1ULL << I40IW_CQPSQ_FWQE_USERFLCODE_SHIFT)
	+
	+#define I40IW_CQPSQ_FWQE_FLUSHSQ_SHIFT 61
	+#define I40IW_CQPSQ_FWQE_FLUSHSQ_MASK (1ULL << I40IW_CQPSQ_FWQE_FLUSHSQ_SHIFT)
	+
	+#define I40IW_CQPSQ_FWQE_FLUSHRQ_SHIFT 62
	+#define I40IW_CQPSQ_FWQE_FLUSHRQ_MASK (1ULL << I40IW_CQPSQ_FWQE_FLUSHRQ_SHIFT)
	+
	+/* Manage Accelerated Port Table - MAPT */
	+#define I40IW_CQPSQ_MAPT_PORT_SHIFT 0
	+#define I40IW_CQPSQ_MAPT_PORT_MASK (0xffffUL << I40IW_CQPSQ_MAPT_PORT_SHIFT)
	+
	+#define I40IW_CQPSQ_MAPT_ADDPORT_SHIFT 62
	+#define I40IW_CQPSQ_MAPT_ADDPORT_MASK (1ULL << I40IW_CQPSQ_MAPT_ADDPORT_SHIFT)
	+
	+/* Update Protocol Engine SDs */
	+#define I40IW_CQPSQ_UPESD_SDCMD_SHIFT 0
	+#define I40IW_CQPSQ_UPESD_SDCMD_MASK (0xffffffffUL << I40IW_CQPSQ_UPESD_SDCMD_SHIFT)
	+
	+#define I40IW_CQPSQ_UPESD_SDDATALOW_SHIFT 0
	+#define I40IW_CQPSQ_UPESD_SDDATALOW_MASK \
	+ (0xffffffffUL << I40IW_CQPSQ_UPESD_SDDATALOW_SHIFT)
	+
	+#define I40IW_CQPSQ_UPESD_SDDATAHI_SHIFT 32
	+#define I40IW_CQPSQ_UPESD_SDDATAHI_MASK \
	+ (0xffffffffULL << I40IW_CQPSQ_UPESD_SDDATAHI_SHIFT)
	+#define I40IW_CQPSQ_UPESD_HMCFNID_SHIFT 0
	+#define I40IW_CQPSQ_UPESD_HMCFNID_MASK \
	+ (0x3fUL << I40IW_CQPSQ_UPESD_HMCFNID_SHIFT)
	+
	+#define I40IW_CQPSQ_UPESD_ENTRY_VALID_SHIFT 63
	+#define I40IW_CQPSQ_UPESD_ENTRY_VALID_MASK \
	+ ((u64)1 << I40IW_CQPSQ_UPESD_ENTRY_VALID_SHIFT)
	+
	+#define I40IW_CQPSQ_UPESD_ENTRY_COUNT_SHIFT 0
	+#define I40IW_CQPSQ_UPESD_ENTRY_COUNT_MASK \
	+ (0xfUL << I40IW_CQPSQ_UPESD_ENTRY_COUNT_SHIFT)
	+
	+#define I40IW_CQPSQ_UPESD_SKIP_ENTRY_SHIFT 7
	+#define I40IW_CQPSQ_UPESD_SKIP_ENTRY_MASK \
	+ (0x1UL << I40IW_CQPSQ_UPESD_SKIP_ENTRY_SHIFT)
	+
	+/* Suspend QP */
	+#define I40IW_CQPSQ_SUSPENDQP_QPID_SHIFT 0
	+#define I40IW_CQPSQ_SUSPENDQP_QPID_MASK (0x3FFFFUL)
	+/* I40IWCQ_QPID_MASK */
	+
	+/* Resume QP */
	+#define I40IW_CQPSQ_RESUMEQP_QSHANDLE_SHIFT 0
	+#define I40IW_CQPSQ_RESUMEQP_QSHANDLE_MASK \
	+ (0xffffffffUL << I40IW_CQPSQ_RESUMEQP_QSHANDLE_SHIFT)
	+
	+#define I40IW_CQPSQ_RESUMEQP_QPID_SHIFT 0
	+#define I40IW_CQPSQ_RESUMEQP_QPID_MASK (0x3FFFFUL)
	+/* I40IWCQ_QPID_MASK */
	+
	+/* IW QP Context */
	+#define I40IWQPC_DDP_VER_SHIFT 0
	+#define I40IWQPC_DDP_VER_MASK (3UL << I40IWQPC_DDP_VER_SHIFT)
	+
	+#define I40IWQPC_SNAP_SHIFT 2
	+#define I40IWQPC_SNAP_MASK (1UL << I40IWQPC_SNAP_SHIFT)
	+
	+#define I40IWQPC_IPV4_SHIFT 3
	+#define I40IWQPC_IPV4_MASK (1UL << I40IWQPC_IPV4_SHIFT)
	+
	+#define I40IWQPC_NONAGLE_SHIFT 4
	+#define I40IWQPC_NONAGLE_MASK (1UL << I40IWQPC_NONAGLE_SHIFT)
	+
	+#define I40IWQPC_INSERTVLANTAG_SHIFT 5
	+#define I40IWQPC_INSERTVLANTAG_MASK (1 << I40IWQPC_INSERTVLANTAG_SHIFT)
	+
	+#define I40IWQPC_USESRQ_SHIFT 6
	+#define I40IWQPC_USESRQ_MASK (1UL << I40IWQPC_USESRQ_SHIFT)
	+
	+#define I40IWQPC_TIMESTAMP_SHIFT 7
	+#define I40IWQPC_TIMESTAMP_MASK (1UL << I40IWQPC_TIMESTAMP_SHIFT)
	+
	+#define I40IWQPC_RQWQESIZE_SHIFT 8
	+#define I40IWQPC_RQWQESIZE_MASK (3UL << I40IWQPC_RQWQESIZE_SHIFT)
	+
	+#define I40IWQPC_INSERTL2TAG2_SHIFT 11
	+#define I40IWQPC_INSERTL2TAG2_MASK (1UL << I40IWQPC_INSERTL2TAG2_SHIFT)
	+
	+#define I40IWQPC_LIMIT_SHIFT 12
	+#define I40IWQPC_LIMIT_MASK (3UL << I40IWQPC_LIMIT_SHIFT)
	+
	+#define I40IWQPC_DROPOOOSEG_SHIFT 15
	+#define I40IWQPC_DROPOOOSEG_MASK (1UL << I40IWQPC_DROPOOOSEG_SHIFT)
	+
	+#define I40IWQPC_DUPACK_THRESH_SHIFT 16
	+#define I40IWQPC_DUPACK_THRESH_MASK (7UL << I40IWQPC_DUPACK_THRESH_SHIFT)
	+
	+#define I40IWQPC_ERR_RQ_IDX_VALID_SHIFT 19
	+#define I40IWQPC_ERR_RQ_IDX_VALID_MASK (1UL << I40IWQPC_ERR_RQ_IDX_VALID_SHIFT)
	+
	+#define I40IWQPC_DIS_VLAN_CHECKS_SHIFT 19
	+#define I40IWQPC_DIS_VLAN_CHECKS_MASK (7UL << I40IWQPC_DIS_VLAN_CHECKS_SHIFT)
	+
	+#define I40IWQPC_RCVTPHEN_SHIFT 28
	+#define I40IWQPC_RCVTPHEN_MASK (1UL << I40IWQPC_RCVTPHEN_SHIFT)
	+
	+#define I40IWQPC_XMITTPHEN_SHIFT 29
	+#define I40IWQPC_XMITTPHEN_MASK (1ULL << I40IWQPC_XMITTPHEN_SHIFT)
	+
	+#define I40IWQPC_RQTPHEN_SHIFT 30
	+#define I40IWQPC_RQTPHEN_MASK (1UL << I40IWQPC_RQTPHEN_SHIFT)
	+
	+#define I40IWQPC_SQTPHEN_SHIFT 31
	+#define I40IWQPC_SQTPHEN_MASK (1ULL << I40IWQPC_SQTPHEN_SHIFT)
	+
	+#define I40IWQPC_PPIDX_SHIFT 32
	+#define I40IWQPC_PPIDX_MASK (0x3ffULL << I40IWQPC_PPIDX_SHIFT)
	+
	+#define I40IWQPC_PMENA_SHIFT 47
	+#define I40IWQPC_PMENA_MASK (1ULL << I40IWQPC_PMENA_SHIFT)
	+
	+#define I40IWQPC_RDMAP_VER_SHIFT 62
	+#define I40IWQPC_RDMAP_VER_MASK (3ULL << I40IWQPC_RDMAP_VER_SHIFT)
	+
	+#define I40IWQPC_SQADDR_SHIFT I40IW_CQPHC_QPCTX_SHIFT
	+#define I40IWQPC_SQADDR_MASK I40IW_CQPHC_QPCTX_MASK
	+
	+#define I40IWQPC_RQADDR_SHIFT I40IW_CQPHC_QPCTX_SHIFT
	+#define I40IWQPC_RQADDR_MASK I40IW_CQPHC_QPCTX_MASK
	+
	+#define I40IWQPC_TTL_SHIFT 0
	+#define I40IWQPC_TTL_MASK (0xffUL << I40IWQPC_TTL_SHIFT)
	+
	+#define I40IWQPC_RQSIZE_SHIFT 8
	+#define I40IWQPC_RQSIZE_MASK (0xfUL << I40IWQPC_RQSIZE_SHIFT)
	+
	+#define I40IWQPC_SQSIZE_SHIFT 12
	+#define I40IWQPC_SQSIZE_MASK (0xfUL << I40IWQPC_SQSIZE_SHIFT)
	+
	+#define I40IWQPC_SRCMACADDRIDX_SHIFT 16
	+#define I40IWQPC_SRCMACADDRIDX_MASK (0x3fUL << I40IWQPC_SRCMACADDRIDX_SHIFT)
	+
	+#define I40IWQPC_AVOIDSTRETCHACK_SHIFT 23
	+#define I40IWQPC_AVOIDSTRETCHACK_MASK (1UL << I40IWQPC_AVOIDSTRETCHACK_SHIFT)
	+
	+#define I40IWQPC_TOS_SHIFT 24
	+#define I40IWQPC_TOS_MASK (0xffUL << I40IWQPC_TOS_SHIFT)
	+
	+#define I40IWQPC_SRCPORTNUM_SHIFT 32
	+#define I40IWQPC_SRCPORTNUM_MASK (0xffffULL << I40IWQPC_SRCPORTNUM_SHIFT)
	+
	+#define I40IWQPC_DESTPORTNUM_SHIFT 48
	+#define I40IWQPC_DESTPORTNUM_MASK (0xffffULL << I40IWQPC_DESTPORTNUM_SHIFT)
	+
	+#define I40IWQPC_DESTIPADDR0_SHIFT 32
	+#define I40IWQPC_DESTIPADDR0_MASK \
	+ (0xffffffffULL << I40IWQPC_DESTIPADDR0_SHIFT)
	+
	+#define I40IWQPC_DESTIPADDR1_SHIFT 0
	+#define I40IWQPC_DESTIPADDR1_MASK \
	+ (0xffffffffULL << I40IWQPC_DESTIPADDR1_SHIFT)
	+
	+#define I40IWQPC_DESTIPADDR2_SHIFT 32
	+#define I40IWQPC_DESTIPADDR2_MASK \
	+ (0xffffffffULL << I40IWQPC_DESTIPADDR2_SHIFT)
	+
	+#define I40IWQPC_DESTIPADDR3_SHIFT 0
	+#define I40IWQPC_DESTIPADDR3_MASK \
	+ (0xffffffffULL << I40IWQPC_DESTIPADDR3_SHIFT)
	+
	+#define I40IWQPC_SNDMSS_SHIFT 16
	+#define I40IWQPC_SNDMSS_MASK (0x3fffUL << I40IWQPC_SNDMSS_SHIFT)
	+
	+#define I40IWQPC_MAXFRAMESIZE_SHIFT 16
	+#define I40IWQPC_MAXFRAMESIZE_MASK (0x3fffUL << I40IWQPC_MAXFRAMESIZE_SHIFT)
	+
	+#define I40IWQPC_VLANTAG_SHIFT 32
	+#define I40IWQPC_VLANTAG_MASK (0xffffULL << I40IWQPC_VLANTAG_SHIFT)
	+
	+#define I40IWQPC_ARPIDX_SHIFT 48
	+#define I40IWQPC_ARPIDX_MASK (0xffffULL << I40IWQPC_ARPIDX_SHIFT)
	+
	+#define I40IWQPC_FLOWLABEL_SHIFT 0
	+#define I40IWQPC_FLOWLABEL_MASK (0xfffffUL << I40IWQPC_FLOWLABEL_SHIFT)
	+
	+#define I40IWQPC_WSCALE_SHIFT 20
	+#define I40IWQPC_WSCALE_MASK (1UL << I40IWQPC_WSCALE_SHIFT)
	+
	+#define I40IWQPC_KEEPALIVE_SHIFT 21
	+#define I40IWQPC_KEEPALIVE_MASK (1UL << I40IWQPC_KEEPALIVE_SHIFT)
	+
	+#define I40IWQPC_IGNORE_TCP_OPT_SHIFT 22
	+#define I40IWQPC_IGNORE_TCP_OPT_MASK (1UL << I40IWQPC_IGNORE_TCP_OPT_SHIFT)
	+
	+#define I40IWQPC_IGNORE_TCP_UNS_OPT_SHIFT 23
	+#define I40IWQPC_IGNORE_TCP_UNS_OPT_MASK \
	+ (1UL << I40IWQPC_IGNORE_TCP_UNS_OPT_SHIFT)
	+
	+#define I40IWQPC_TCPSTATE_SHIFT 28
	+#define I40IWQPC_TCPSTATE_MASK (0xfUL << I40IWQPC_TCPSTATE_SHIFT)
	+
	+#define I40IWQPC_RCVSCALE_SHIFT 32
	+#define I40IWQPC_RCVSCALE_MASK (0xfULL << I40IWQPC_RCVSCALE_SHIFT)
	+
	+#define I40IWQPC_SNDSCALE_SHIFT 40
	+#define I40IWQPC_SNDSCALE_MASK (0xfULL << I40IWQPC_SNDSCALE_SHIFT)
	+
	+#define I40IWQPC_PDIDX_SHIFT 48
	+#define I40IWQPC_PDIDX_MASK (0x7fffULL << I40IWQPC_PDIDX_SHIFT)
	+
	+#define I40IWQPC_KALIVE_TIMER_MAX_PROBES_SHIFT 16
	+#define I40IWQPC_KALIVE_TIMER_MAX_PROBES_MASK \
	+ (0xffUL << I40IWQPC_KALIVE_TIMER_MAX_PROBES_SHIFT)
	+
	+#define I40IWQPC_KEEPALIVE_INTERVAL_SHIFT 24
	+#define I40IWQPC_KEEPALIVE_INTERVAL_MASK \
	+ (0xffUL << I40IWQPC_KEEPALIVE_INTERVAL_SHIFT)
	+
	+#define I40IWQPC_TIMESTAMP_RECENT_SHIFT 0
	+#define I40IWQPC_TIMESTAMP_RECENT_MASK \
	+ (0xffffffffUL << I40IWQPC_TIMESTAMP_RECENT_SHIFT)
	+
	+#define I40IWQPC_TIMESTAMP_AGE_SHIFT 32
	+#define I40IWQPC_TIMESTAMP_AGE_MASK \
	+ (0xffffffffULL << I40IWQPC_TIMESTAMP_AGE_SHIFT)
	+
	+#define I40IWQPC_SNDNXT_SHIFT 0
	+#define I40IWQPC_SNDNXT_MASK (0xffffffffUL << I40IWQPC_SNDNXT_SHIFT)
	+
	+#define I40IWQPC_SNDWND_SHIFT 32
	+#define I40IWQPC_SNDWND_MASK (0xffffffffULL << I40IWQPC_SNDWND_SHIFT)
	+
	+#define I40IWQPC_RCVNXT_SHIFT 0
	+#define I40IWQPC_RCVNXT_MASK (0xffffffffUL << I40IWQPC_RCVNXT_SHIFT)
	+
	+#define I40IWQPC_RCVWND_SHIFT 32
	+#define I40IWQPC_RCVWND_MASK (0xffffffffULL << I40IWQPC_RCVWND_SHIFT)
	+
	+#define I40IWQPC_SNDMAX_SHIFT 0
	+#define I40IWQPC_SNDMAX_MASK (0xffffffffUL << I40IWQPC_SNDMAX_SHIFT)
	+
	+#define I40IWQPC_SNDUNA_SHIFT 32
	+#define I40IWQPC_SNDUNA_MASK (0xffffffffULL << I40IWQPC_SNDUNA_SHIFT)
	+
	+#define I40IWQPC_SRTT_SHIFT 0
	+#define I40IWQPC_SRTT_MASK (0xffffffffUL << I40IWQPC_SRTT_SHIFT)
	+
	+#define I40IWQPC_RTTVAR_SHIFT 32
	+#define I40IWQPC_RTTVAR_MASK (0xffffffffULL << I40IWQPC_RTTVAR_SHIFT)
	+
	+#define I40IWQPC_SSTHRESH_SHIFT 0
	+#define I40IWQPC_SSTHRESH_MASK (0xffffffffUL << I40IWQPC_SSTHRESH_SHIFT)
	+
	+#define I40IWQPC_CWND_SHIFT 32
	+#define I40IWQPC_CWND_MASK (0xffffffffULL << I40IWQPC_CWND_SHIFT)
	+
	+#define I40IWQPC_SNDWL1_SHIFT 0
	+#define I40IWQPC_SNDWL1_MASK (0xffffffffUL << I40IWQPC_SNDWL1_SHIFT)
	+
	+#define I40IWQPC_SNDWL2_SHIFT 32
	+#define I40IWQPC_SNDWL2_MASK (0xffffffffULL << I40IWQPC_SNDWL2_SHIFT)
	+
	+#define I40IWQPC_ERR_RQ_IDX_SHIFT 32
	+#define I40IWQPC_ERR_RQ_IDX_MASK (0x3fffULL << I40IWQPC_ERR_RQ_IDX_SHIFT)
	+
	+#define I40IWQPC_MAXSNDWND_SHIFT 0
	+#define I40IWQPC_MAXSNDWND_MASK (0xffffffffUL << I40IWQPC_MAXSNDWND_SHIFT)
	+
	+#define I40IWQPC_REXMIT_THRESH_SHIFT 48
	+#define I40IWQPC_REXMIT_THRESH_MASK (0x3fULL << I40IWQPC_REXMIT_THRESH_SHIFT)
	+
	+#define I40IWQPC_TXCQNUM_SHIFT 0
	+#define I40IWQPC_TXCQNUM_MASK (0x1ffffUL << I40IWQPC_TXCQNUM_SHIFT)
	+
	+#define I40IWQPC_RXCQNUM_SHIFT 32
	+#define I40IWQPC_RXCQNUM_MASK (0x1ffffULL << I40IWQPC_RXCQNUM_SHIFT)
	+
	+#define I40IWQPC_STAT_INDEX_SHIFT 0
	+#define I40IWQPC_STAT_INDEX_MASK (0x1fULL << I40IWQPC_STAT_INDEX_SHIFT)
	+
	+#define I40IWQPC_Q2ADDR_SHIFT 0
	+#define I40IWQPC_Q2ADDR_MASK (0xffffffffffffff00ULL << I40IWQPC_Q2ADDR_SHIFT)
	+
	+#define I40IWQPC_LASTBYTESENT_SHIFT 0
	+#define I40IWQPC_LASTBYTESENT_MASK (0xffUL << I40IWQPC_LASTBYTESENT_SHIFT)
	+
	+#define I40IWQPC_SRQID_SHIFT 32
	+#define I40IWQPC_SRQID_MASK (0xffULL << I40IWQPC_SRQID_SHIFT)
	+
	+#define I40IWQPC_ORDSIZE_SHIFT 0
	+#define I40IWQPC_ORDSIZE_MASK (0x7fUL << I40IWQPC_ORDSIZE_SHIFT)
	+
	+#define I40IWQPC_IRDSIZE_SHIFT 16
	+#define I40IWQPC_IRDSIZE_MASK (0x3UL << I40IWQPC_IRDSIZE_SHIFT)
	+
	+#define I40IWQPC_WRRDRSPOK_SHIFT 20
	+#define I40IWQPC_WRRDRSPOK_MASK (1UL << I40IWQPC_WRRDRSPOK_SHIFT)
	+
	+#define I40IWQPC_RDOK_SHIFT 21
	+#define I40IWQPC_RDOK_MASK (1UL << I40IWQPC_RDOK_SHIFT)
	+
	+#define I40IWQPC_SNDMARKERS_SHIFT 22
	+#define I40IWQPC_SNDMARKERS_MASK (1UL << I40IWQPC_SNDMARKERS_SHIFT)
	+
	+#define I40IWQPC_BINDEN_SHIFT 23
	+#define I40IWQPC_BINDEN_MASK (1UL << I40IWQPC_BINDEN_SHIFT)
	+
	+#define I40IWQPC_FASTREGEN_SHIFT 24
	+#define I40IWQPC_FASTREGEN_MASK (1UL << I40IWQPC_FASTREGEN_SHIFT)
	+
	+#define I40IWQPC_PRIVEN_SHIFT 25
	+#define I40IWQPC_PRIVEN_MASK (1UL << I40IWQPC_PRIVEN_SHIFT)
	+
	+#define I40IWQPC_USESTATSINSTANCE_SHIFT 26
	+#define I40IWQPC_USESTATSINSTANCE_MASK (1UL << I40IWQPC_USESTATSINSTANCE_SHIFT)
	+
	+#define I40IWQPC_IWARPMODE_SHIFT 28
	+#define I40IWQPC_IWARPMODE_MASK (1UL << I40IWQPC_IWARPMODE_SHIFT)
	+
	+#define I40IWQPC_RCVMARKERS_SHIFT 29
	+#define I40IWQPC_RCVMARKERS_MASK (1UL << I40IWQPC_RCVMARKERS_SHIFT)
	+
	+#define I40IWQPC_ALIGNHDRS_SHIFT 30
	+#define I40IWQPC_ALIGNHDRS_MASK (1UL << I40IWQPC_ALIGNHDRS_SHIFT)
	+
	+#define I40IWQPC_RCVNOMPACRC_SHIFT 31
	+#define I40IWQPC_RCVNOMPACRC_MASK (1UL << I40IWQPC_RCVNOMPACRC_SHIFT)
	+
	+#define I40IWQPC_RCVMARKOFFSET_SHIFT 33
	+#define I40IWQPC_RCVMARKOFFSET_MASK (0x1ffULL << I40IWQPC_RCVMARKOFFSET_SHIFT)
	+
	+#define I40IWQPC_SNDMARKOFFSET_SHIFT 48
	+#define I40IWQPC_SNDMARKOFFSET_MASK (0x1ffULL << I40IWQPC_SNDMARKOFFSET_SHIFT)
	+
	+#define I40IWQPC_QPCOMPCTX_SHIFT I40IW_CQPHC_QPCTX_SHIFT
	+#define I40IWQPC_QPCOMPCTX_MASK I40IW_CQPHC_QPCTX_MASK
	+
	+#define I40IWQPC_SQTPHVAL_SHIFT 0
	+#define I40IWQPC_SQTPHVAL_MASK (0xffUL << I40IWQPC_SQTPHVAL_SHIFT)
	+
	+#define I40IWQPC_RQTPHVAL_SHIFT 8
	+#define I40IWQPC_RQTPHVAL_MASK (0xffUL << I40IWQPC_RQTPHVAL_SHIFT)
	+
	+#define I40IWQPC_QSHANDLE_SHIFT 16
	+#define I40IWQPC_QSHANDLE_MASK (0x3ffUL << I40IWQPC_QSHANDLE_SHIFT)
	+
	+#define I40IWQPC_EXCEPTION_LAN_QUEUE_SHIFT 32
	+#define I40IWQPC_EXCEPTION_LAN_QUEUE_MASK (0xfffULL << \
	+ I40IWQPC_EXCEPTION_LAN_QUEUE_SHIFT)
	+
	+#define I40IWQPC_LOCAL_IPADDR3_SHIFT 0
	+#define I40IWQPC_LOCAL_IPADDR3_MASK \
	+ (0xffffffffUL << I40IWQPC_LOCAL_IPADDR3_SHIFT)
	+
	+#define I40IWQPC_LOCAL_IPADDR2_SHIFT 32
	+#define I40IWQPC_LOCAL_IPADDR2_MASK \
	+ (0xffffffffULL << I40IWQPC_LOCAL_IPADDR2_SHIFT)
	+
	+#define I40IWQPC_LOCAL_IPADDR1_SHIFT 0
	+#define I40IWQPC_LOCAL_IPADDR1_MASK \
	+ (0xffffffffUL << I40IWQPC_LOCAL_IPADDR1_SHIFT)
	+
	+#define I40IWQPC_LOCAL_IPADDR0_SHIFT 32
	+#define I40IWQPC_LOCAL_IPADDR0_MASK \
	+ (0xffffffffULL << I40IWQPC_LOCAL_IPADDR0_SHIFT)
	+
	+/* wqe size considering 32 bytes per wqe*/
	+#define I40IW_QP_SW_MIN_WQSIZE 4 /in WRs/
	+#define I40IW_QP_SW_MAX_WQ_QUANTAS 2048
	+#define I40IWQP_OP_RDMA_WRITE 0
	+#define I40IWQP_OP_RDMA_READ 1
	+#define I40IWQP_OP_RDMA_SEND 3
	+#define I40IWQP_OP_RDMA_SEND_INV 4
	+#define I40IWQP_OP_RDMA_SEND_SOL_EVENT 5
	+#define I40IWQP_OP_RDMA_SEND_SOL_EVENT_INV 6
	+#define I40IWQP_OP_BIND_MW 8
	+#define I40IWQP_OP_FAST_REGISTER 9
	+#define I40IWQP_OP_LOCAL_INVALIDATE 10
	+#define I40IWQP_OP_RDMA_READ_LOC_INV 11
	+#define I40IWQP_OP_NOP 12
	+
	+#define I40IW_RSVD_SHIFT 41
	+#define I40IW_RSVD_MASK (0x7fffULL << I40IW_RSVD_SHIFT)
	+
	+/* iwarp QP SQ WQE common fields */
	+#define I40IWQPSQ_OPCODE_SHIFT 32
	+#define I40IWQPSQ_OPCODE_MASK (0x3fULL << I40IWQPSQ_OPCODE_SHIFT)
	+
	+#define I40IWQPSQ_ADDFRAGCNT_SHIFT 38
	+#define I40IWQPSQ_ADDFRAGCNT_MASK (0x7ULL << I40IWQPSQ_ADDFRAGCNT_SHIFT)
	+
	+#define I40IWQPSQ_PUSHWQE_SHIFT 56
	+#define I40IWQPSQ_PUSHWQE_MASK (1ULL << I40IWQPSQ_PUSHWQE_SHIFT)
	+
	+#define I40IWQPSQ_STREAMMODE_SHIFT 58
	+#define I40IWQPSQ_STREAMMODE_MASK (1ULL << I40IWQPSQ_STREAMMODE_SHIFT)
	+
	+#define I40IWQPSQ_WAITFORRCVPDU_SHIFT 59
	+#define I40IWQPSQ_WAITFORRCVPDU_MASK (1ULL << I40IWQPSQ_WAITFORRCVPDU_SHIFT)
	+
	+#define I40IWQPSQ_READFENCE_SHIFT 60
	+#define I40IWQPSQ_READFENCE_MASK (1ULL << I40IWQPSQ_READFENCE_SHIFT)
	+
	+#define I40IWQPSQ_LOCALFENCE_SHIFT 61
	+#define I40IWQPSQ_LOCALFENCE_MASK (1ULL << I40IWQPSQ_LOCALFENCE_SHIFT)
	+
	+#define I40IWQPSQ_SIGCOMPL_SHIFT 62
	+#define I40IWQPSQ_SIGCOMPL_MASK (1ULL << I40IWQPSQ_SIGCOMPL_SHIFT)
	+
	+#define I40IWQPSQ_VALID_SHIFT 63
	+#define I40IWQPSQ_VALID_MASK (1ULL << I40IWQPSQ_VALID_SHIFT)
	+
	+#define I40IWQPSQ_FRAG_TO_SHIFT I40IW_CQPHC_QPCTX_SHIFT
	+#define I40IWQPSQ_FRAG_TO_MASK I40IW_CQPHC_QPCTX_MASK
	+
	+#define I40IWQPSQ_FRAG_LEN_SHIFT 0
	+#define I40IWQPSQ_FRAG_LEN_MASK (0xffffffffUL << I40IWQPSQ_FRAG_LEN_SHIFT)
	+
	+#define I40IWQPSQ_FRAG_STAG_SHIFT 32
	+#define I40IWQPSQ_FRAG_STAG_MASK (0xffffffffULL << I40IWQPSQ_FRAG_STAG_SHIFT)
	+
	+#define I40IWQPSQ_REMSTAGINV_SHIFT 0
	+#define I40IWQPSQ_REMSTAGINV_MASK (0xffffffffUL << I40IWQPSQ_REMSTAGINV_SHIFT)
	+
	+#define I40IWQPSQ_INLINEDATAFLAG_SHIFT 57
	+#define I40IWQPSQ_INLINEDATAFLAG_MASK (1ULL << I40IWQPSQ_INLINEDATAFLAG_SHIFT)
	+
	+#define I40IWQPSQ_INLINEDATALEN_SHIFT 48
	+#define I40IWQPSQ_INLINEDATALEN_MASK \
	+ (0x7fULL << I40IWQPSQ_INLINEDATALEN_SHIFT)
	+
	+/* iwarp send with push mode */
	+#define I40IWQPSQ_WQDESCIDX_SHIFT 0
	+#define I40IWQPSQ_WQDESCIDX_MASK (0x3fffUL << I40IWQPSQ_WQDESCIDX_SHIFT)
	+
	+/* rdma write */
	+#define I40IWQPSQ_REMSTAG_SHIFT 0
	+#define I40IWQPSQ_REMSTAG_MASK (0xffffffffUL << I40IWQPSQ_REMSTAG_SHIFT)
	+
	+#define I40IWQPSQ_REMTO_SHIFT I40IW_CQPHC_QPCTX_SHIFT
	+#define I40IWQPSQ_REMTO_MASK I40IW_CQPHC_QPCTX_MASK
	+
	+/* memory window */
	+#define I40IWQPSQ_STAGRIGHTS_SHIFT 48
	+#define I40IWQPSQ_STAGRIGHTS_MASK (0x1fULL << I40IWQPSQ_STAGRIGHTS_SHIFT)
	+
	+#define I40IWQPSQ_VABASEDTO_SHIFT 53
	+#define I40IWQPSQ_VABASEDTO_MASK (1ULL << I40IWQPSQ_VABASEDTO_SHIFT)
	+
	+#define I40IWQPSQ_MWLEN_SHIFT I40IW_CQPHC_QPCTX_SHIFT
	+#define I40IWQPSQ_MWLEN_MASK I40IW_CQPHC_QPCTX_MASK
	+
	+#define I40IWQPSQ_PARENTMRSTAG_SHIFT 0
	+#define I40IWQPSQ_PARENTMRSTAG_MASK \
	+ (0xffffffffUL << I40IWQPSQ_PARENTMRSTAG_SHIFT)
	+
	+#define I40IWQPSQ_MWSTAG_SHIFT 32
	+#define I40IWQPSQ_MWSTAG_MASK (0xffffffffULL << I40IWQPSQ_MWSTAG_SHIFT)
	+
	+#define I40IWQPSQ_BASEVA_TO_FBO_SHIFT I40IW_CQPHC_QPCTX_SHIFT
	+#define I40IWQPSQ_BASEVA_TO_FBO_MASK I40IW_CQPHC_QPCTX_MASK
	+
	+/* Local Invalidate */
	+#define I40IWQPSQ_LOCSTAG_SHIFT 32
	+#define I40IWQPSQ_LOCSTAG_MASK (0xffffffffULL << I40IWQPSQ_LOCSTAG_SHIFT)
	+
	+/* Fast Register */
	+#define I40IWQPSQ_STAGKEY_SHIFT 0
	+#define I40IWQPSQ_STAGKEY_MASK (0xffUL << I40IWQPSQ_STAGKEY_SHIFT)
	+
	+#define I40IWQPSQ_STAGINDEX_SHIFT 8
	+#define I40IWQPSQ_STAGINDEX_MASK (0xffffffUL << I40IWQPSQ_STAGINDEX_SHIFT)
	+
	+#define I40IWQPSQ_COPYHOSTPBLS_SHIFT 43
	+#define I40IWQPSQ_COPYHOSTPBLS_MASK (1ULL << I40IWQPSQ_COPYHOSTPBLS_SHIFT)
	+
	+#define I40IWQPSQ_LPBLSIZE_SHIFT 44
	+#define I40IWQPSQ_LPBLSIZE_MASK (3ULL << I40IWQPSQ_LPBLSIZE_SHIFT)
	+
	+#define I40IWQPSQ_HPAGESIZE_SHIFT 46
	+#define I40IWQPSQ_HPAGESIZE_MASK (3ULL << I40IWQPSQ_HPAGESIZE_SHIFT)
	+
	+#define I40IWQPSQ_STAGLEN_SHIFT 0
	+#define I40IWQPSQ_STAGLEN_MASK (0x1ffffffffffULL << I40IWQPSQ_STAGLEN_SHIFT)
	+
	+#define I40IWQPSQ_FIRSTPMPBLIDXLO_SHIFT 48
	+#define I40IWQPSQ_FIRSTPMPBLIDXLO_MASK \
	+ (0xffffULL << I40IWQPSQ_FIRSTPMPBLIDXLO_SHIFT)
	+
	+#define I40IWQPSQ_FIRSTPMPBLIDXHI_SHIFT 0
	+#define I40IWQPSQ_FIRSTPMPBLIDXHI_MASK \
	+ (0xfffUL << I40IWQPSQ_FIRSTPMPBLIDXHI_SHIFT)
	+
	+#define I40IWQPSQ_PBLADDR_SHIFT 12
	+#define I40IWQPSQ_PBLADDR_MASK (0xfffffffffffffULL << I40IWQPSQ_PBLADDR_SHIFT)
	+
	+/* iwarp QP RQ WQE common fields */
	+#define I40IWQPRQ_ADDFRAGCNT_SHIFT I40IWQPSQ_ADDFRAGCNT_SHIFT
	+#define I40IWQPRQ_ADDFRAGCNT_MASK I40IWQPSQ_ADDFRAGCNT_MASK
	+
	+#define I40IWQPRQ_VALID_SHIFT I40IWQPSQ_VALID_SHIFT
	+#define I40IWQPRQ_VALID_MASK I40IWQPSQ_VALID_MASK
	+
	+#define I40IWQPRQ_COMPLCTX_SHIFT I40IW_CQPHC_QPCTX_SHIFT
	+#define I40IWQPRQ_COMPLCTX_MASK I40IW_CQPHC_QPCTX_MASK
	+
	+#define I40IWQPRQ_FRAG_LEN_SHIFT I40IWQPSQ_FRAG_LEN_SHIFT
	+#define I40IWQPRQ_FRAG_LEN_MASK I40IWQPSQ_FRAG_LEN_MASK
	+
	+#define I40IWQPRQ_STAG_SHIFT I40IWQPSQ_FRAG_STAG_SHIFT
	+#define I40IWQPRQ_STAG_MASK I40IWQPSQ_FRAG_STAG_MASK
	+
	+#define I40IWQPRQ_TO_SHIFT I40IWQPSQ_FRAG_TO_SHIFT
	+#define I40IWQPRQ_TO_MASK I40IWQPSQ_FRAG_TO_MASK
	+
	+/* Query FPM CQP buf */
	+#define I40IW_QUERY_FPM_MAX_CQS_SHIFT 0
	+#define I40IW_QUERY_FPM_MAX_CQS_MASK \
	+ (0x3ffffUL << I40IW_QUERY_FPM_MAX_CQS_SHIFT)
	+
	+#define I40IW_QUERY_FPM_FIRST_PE_SD_INDEX_SHIFT 0
	+#define I40IW_QUERY_FPM_FIRST_PE_SD_INDEX_MASK \
	+ (0x3fffUL << I40IW_QUERY_FPM_FIRST_PE_SD_INDEX_SHIFT)
	+
	+#define I40IW_QUERY_FPM_MAX_PE_SDS_SHIFT 32
	+#define I40IW_QUERY_FPM_MAX_PE_SDS_MASK \
	+ (0x3fffULL << I40IW_QUERY_FPM_MAX_PE_SDS_SHIFT)
	+
	+#define I40IW_QUERY_FPM_MAX_QPS_SHIFT 0
	+#define I40IW_QUERY_FPM_MAX_QPS_MASK \
	+ (0x7ffffUL << I40IW_QUERY_FPM_MAX_QPS_SHIFT)
	+
	+#define I40IW_QUERY_FPM_MAX_CQS_SHIFT 0
	+#define I40IW_QUERY_FPM_MAX_CQS_MASK \
	+ (0x3ffffUL << I40IW_QUERY_FPM_MAX_CQS_SHIFT)
	+
	+#define I40IW_QUERY_FPM_MAX_CEQS_SHIFT 0
	+#define I40IW_QUERY_FPM_MAX_CEQS_MASK \
	+ (0xffUL << I40IW_QUERY_FPM_MAX_CEQS_SHIFT)
	+
	+#define I40IW_QUERY_FPM_XFBLOCKSIZE_SHIFT 32
	+#define I40IW_QUERY_FPM_XFBLOCKSIZE_MASK \
	+ (0xffffffffULL << I40IW_QUERY_FPM_XFBLOCKSIZE_SHIFT)
	+
	+#define I40IW_QUERY_FPM_Q1BLOCKSIZE_SHIFT 32
	+#define I40IW_QUERY_FPM_Q1BLOCKSIZE_MASK \
	+ (0xffffffffULL << I40IW_QUERY_FPM_Q1BLOCKSIZE_SHIFT)
	+
	+#define I40IW_QUERY_FPM_HTMULTIPLIER_SHIFT 16
	+#define I40IW_QUERY_FPM_HTMULTIPLIER_MASK \
	+ (0xfUL << I40IW_QUERY_FPM_HTMULTIPLIER_SHIFT)
	+
	+#define I40IW_QUERY_FPM_TIMERBUCKET_SHIFT 32
	+#define I40IW_QUERY_FPM_TIMERBUCKET_MASK \
	+ (0xffFFULL << I40IW_QUERY_FPM_TIMERBUCKET_SHIFT)
	+
	+/* Static HMC pages allocated buf */
	+#define I40IW_SHMC_PAGE_ALLOCATED_HMC_FN_ID_SHIFT 0
	+#define I40IW_SHMC_PAGE_ALLOCATED_HMC_FN_ID_MASK \
	+ (0x3fUL << I40IW_SHMC_PAGE_ALLOCATED_HMC_FN_ID_SHIFT)
	+
	+#define I40IW_HW_PAGE_SIZE 4096
	+#define I40IW_DONE_COUNT 1000
	+#define I40IW_SLEEP_COUNT 10
	+
	+enum {
	+ I40IW_QUEUES_ALIGNMENT_MASK = (128 - 1),
	+ I40IW_AEQ_ALIGNMENT_MASK = (256 - 1),
	+ I40IW_Q2_ALIGNMENT_MASK = (256 - 1),
	+ I40IW_CEQ_ALIGNMENT_MASK = (256 - 1),
	+ I40IW_CQ0_ALIGNMENT_MASK = (256 - 1),
	+ I40IW_HOST_CTX_ALIGNMENT_MASK = (4 - 1),
	+ I40IW_SHADOWAREA_MASK = (128 - 1),
	+ I40IW_FPM_QUERY_BUF_ALIGNMENT_MASK = (4 - 1),
	+ I40IW_FPM_COMMIT_BUF_ALIGNMENT_MASK = (4 - 1)
	+};
	+
	+enum i40iw_alignment {
	+ I40IW_CQP_ALIGNMENT = 0x200,
	+ I40IW_AEQ_ALIGNMENT = 0x100,
	+ I40IW_CEQ_ALIGNMENT = 0x100,
	+ I40IW_CQ0_ALIGNMENT = 0x100,
	+ I40IW_SD_BUF_ALIGNMENT = 0x80
	+};
	+
	+#define I40IW_WQE_SIZE_64 64
	+
	+#define I40IW_QP_WQE_MIN_SIZE 32
	+#define I40IW_QP_WQE_MAX_SIZE 128
	+
	+#define I40IW_CQE_QTYPE_RQ 0
	+#define I40IW_CQE_QTYPE_SQ 1
	+
	+#define I40IW_RING_INIT(_ring, _size) \
	+ { \
	+ (_ring).head = 0; \
	+ (_ring).tail = 0; \
	+ (_ring).size = (_size); \
	+ }
	+#define I40IW_RING_GETSIZE(_ring) ((_ring).size)
	+#define I40IW_RING_GETCURRENT_HEAD(_ring) ((_ring).head)
	+#define I40IW_RING_GETCURRENT_TAIL(_ring) ((_ring).tail)
	+
	+#define I40IW_RING_MOVE_HEAD(_ring, _retcode) \
	+ { \
	+ register u32 size; \
	+ size = (_ring).size; \
	+ if (!I40IW_RING_FULL_ERR(_ring)) { \
	+ (_ring).head = ((_ring).head + 1) % size; \
	+ (_retcode) = 0; \
	+ } else { \
	+ (_retcode) = I40IW_ERR_RING_FULL; \
	+ } \
	+ }
	+
	+#define I40IW_RING_MOVE_HEAD_BY_COUNT(_ring, _count, _retcode) \
	+ { \
	+ register u32 size; \
	+ size = (_ring).size; \
	+ if ((I40IW_RING_WORK_AVAILABLE(_ring) + (_count)) < size) { \
	+ (_ring).head = ((_ring).head + (_count)) % size; \
	+ (_retcode) = 0; \
	+ } else { \
	+ (_retcode) = I40IW_ERR_RING_FULL; \
	+ } \
	+ }
	+
	+#define I40IW_RING_MOVE_TAIL(_ring) \
	+ (_ring).tail = ((_ring).tail + 1) % (_ring).size
	+
	+#define I40IW_RING_MOVE_HEAD_NOCHECK(_ring) \
	+ (_ring).head = ((_ring).head + 1) % (_ring).size
	+
	+#define I40IW_RING_MOVE_TAIL_BY_COUNT(_ring, _count) \
	+ (_ring).tail = ((_ring).tail + (_count)) % (_ring).size
	+
	+#define I40IW_RING_SET_TAIL(_ring, _pos) \
	+ (_ring).tail = (_pos) % (_ring).size
	+
	+#define I40IW_RING_FULL_ERR(_ring) \
	+ ( \
	+ (I40IW_RING_WORK_AVAILABLE(_ring) == ((_ring).size - 1)) \
	+ )
	+
	+#define I40IW_ERR_RING_FULL2(_ring) \
	+ ( \
	+ (I40IW_RING_WORK_AVAILABLE(_ring) == ((_ring).size - 2)) \
	+ )
	+
	+#define I40IW_ERR_RING_FULL3(_ring) \
	+ ( \
	+ (I40IW_RING_WORK_AVAILABLE(_ring) == ((_ring).size - 3)) \
	+ )
	+
	+#define I40IW_RING_MORE_WORK(_ring) \
	+ ( \
	+ (I40IW_RING_WORK_AVAILABLE(_ring) != 0) \
	+ )
	+
	+#define I40IW_RING_WORK_AVAILABLE(_ring) \
	+ ( \
	+ (((_ring).head + (_ring).size - (_ring).tail) % (_ring).size) \
	+ )
	+
	+#define I40IW_RING_GET_WQES_AVAILABLE(_ring) \
	+ ( \
	+ ((_ring).size - I40IW_RING_WORK_AVAILABLE(_ring) - 1) \
	+ )
	+
	+#define I40IW_ATOMIC_RING_MOVE_HEAD(_ring, index, _retcode) \
	+ { \
	+ index = I40IW_RING_GETCURRENT_HEAD(_ring); \
	+ I40IW_RING_MOVE_HEAD(_ring, _retcode); \
	+ }
	+
	+/* Async Events codes */
	+#define I40IW_AE_AMP_UNALLOCATED_STAG 0x0102
	+#define I40IW_AE_AMP_INVALID_STAG 0x0103
	+#define I40IW_AE_AMP_BAD_QP 0x0104
	+#define I40IW_AE_AMP_BAD_PD 0x0105
	+#define I40IW_AE_AMP_BAD_STAG_KEY 0x0106
	+#define I40IW_AE_AMP_BAD_STAG_INDEX 0x0107
	+#define I40IW_AE_AMP_BOUNDS_VIOLATION 0x0108
	+#define I40IW_AE_AMP_RIGHTS_VIOLATION 0x0109
	+#define I40IW_AE_AMP_TO_WRAP 0x010a
	+#define I40IW_AE_AMP_FASTREG_SHARED 0x010b
	+#define I40IW_AE_AMP_FASTREG_VALID_STAG 0x010c
	+#define I40IW_AE_AMP_FASTREG_MW_STAG 0x010d
	+#define I40IW_AE_AMP_FASTREG_INVALID_RIGHTS 0x010e
	+#define I40IW_AE_AMP_FASTREG_PBL_TABLE_OVERFLOW 0x010f
	+#define I40IW_AE_AMP_FASTREG_INVALID_LENGTH 0x0110
	+#define I40IW_AE_AMP_INVALIDATE_SHARED 0x0111
	+#define I40IW_AE_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS 0x0112
	+#define I40IW_AE_AMP_INVALIDATE_MR_WITH_BOUND_WINDOWS 0x0113
	+#define I40IW_AE_AMP_MWBIND_VALID_STAG 0x0114
	+#define I40IW_AE_AMP_MWBIND_OF_MR_STAG 0x0115
	+#define I40IW_AE_AMP_MWBIND_TO_ZERO_BASED_STAG 0x0116
	+#define I40IW_AE_AMP_MWBIND_TO_MW_STAG 0x0117
	+#define I40IW_AE_AMP_MWBIND_INVALID_RIGHTS 0x0118
	+#define I40IW_AE_AMP_MWBIND_INVALID_BOUNDS 0x0119
	+#define I40IW_AE_AMP_MWBIND_TO_INVALID_PARENT 0x011a
	+#define I40IW_AE_AMP_MWBIND_BIND_DISABLED 0x011b
	+#define I40IW_AE_UDA_XMIT_DGRAM_TOO_LONG 0x0132
	+#define I40IW_AE_UDA_XMIT_DGRAM_TOO_SHORT 0x0134
	+#define I40IW_AE_BAD_CLOSE 0x0201
	+#define I40IW_AE_RDMAP_ROE_BAD_LLP_CLOSE 0x0202
	+#define I40IW_AE_CQ_OPERATION_ERROR 0x0203
	+#define I40IW_AE_PRIV_OPERATION_DENIED 0x011c
	+#define I40IW_AE_RDMA_READ_WHILE_ORD_ZERO 0x0205
	+#define I40IW_AE_STAG_ZERO_INVALID 0x0206
	+#define I40IW_AE_IB_RREQ_AND_Q1_FULL 0x0207
	+#define I40IW_AE_WQE_UNEXPECTED_OPCODE 0x020a
	+#define I40IW_AE_WQE_INVALID_PARAMETER 0x020b
	+#define I40IW_AE_WQE_LSMM_TOO_LONG 0x0220
	+#define I40IW_AE_DDP_INVALID_MSN_GAP_IN_MSN 0x0301
	+#define I40IW_AE_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER 0x0303
	+#define I40IW_AE_DDP_UBE_INVALID_DDP_VERSION 0x0304
	+#define I40IW_AE_DDP_UBE_INVALID_MO 0x0305
	+#define I40IW_AE_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE 0x0306
	+#define I40IW_AE_DDP_UBE_INVALID_QN 0x0307
	+#define I40IW_AE_DDP_NO_L_BIT 0x0308
	+#define I40IW_AE_RDMAP_ROE_INVALID_RDMAP_VERSION 0x0311
	+#define I40IW_AE_RDMAP_ROE_UNEXPECTED_OPCODE 0x0312
	+#define I40IW_AE_ROE_INVALID_RDMA_READ_REQUEST 0x0313
	+#define I40IW_AE_ROE_INVALID_RDMA_WRITE_OR_READ_RESP 0x0314
	+#define I40IW_AE_INVALID_ARP_ENTRY 0x0401
	+#define I40IW_AE_INVALID_TCP_OPTION_RCVD 0x0402
	+#define I40IW_AE_STALE_ARP_ENTRY 0x0403
	+#define I40IW_AE_INVALID_MAC_ENTRY 0x0405
	+#define I40IW_AE_LLP_CLOSE_COMPLETE 0x0501
	+#define I40IW_AE_LLP_CONNECTION_RESET 0x0502
	+#define I40IW_AE_LLP_FIN_RECEIVED 0x0503
	+#define I40IW_AE_LLP_RECEIVED_MPA_CRC_ERROR 0x0505
	+#define I40IW_AE_LLP_SEGMENT_TOO_LARGE 0x0506
	+#define I40IW_AE_LLP_SEGMENT_TOO_SMALL 0x0507
	+#define I40IW_AE_LLP_SYN_RECEIVED 0x0508
	+#define I40IW_AE_LLP_TERMINATE_RECEIVED 0x0509
	+#define I40IW_AE_LLP_TOO_MANY_RETRIES 0x050a
	+#define I40IW_AE_LLP_TOO_MANY_KEEPALIVE_RETRIES 0x050b
	+#define I40IW_AE_LLP_DOUBT_REACHABILITY 0x050c
	+#define I40IW_AE_LLP_RX_VLAN_MISMATCH 0x050d
	+#define I40IW_AE_RESOURCE_EXHAUSTION 0x0520
	+#define I40IW_AE_RESET_SENT 0x0601
	+#define I40IW_AE_TERMINATE_SENT 0x0602
	+#define I40IW_AE_RESET_NOT_SENT 0x0603
	+#define I40IW_AE_LCE_QP_CATASTROPHIC 0x0700
	+#define I40IW_AE_LCE_FUNCTION_CATASTROPHIC 0x0701
	+#define I40IW_AE_LCE_CQ_CATASTROPHIC 0x0702
	+#define I40IW_AE_QP_SUSPEND_COMPLETE 0x0900
	+
	+#define OP_DELETE_LOCAL_MAC_IPADDR_ENTRY 1
	+#define OP_CEQ_DESTROY 2
	+#define OP_AEQ_DESTROY 3
	+#define OP_DELETE_ARP_CACHE_ENTRY 4
	+#define OP_MANAGE_APBVT_ENTRY 5
	+#define OP_CEQ_CREATE 6
	+#define OP_AEQ_CREATE 7
	+#define OP_ALLOC_LOCAL_MAC_IPADDR_ENTRY 8
	+#define OP_ADD_LOCAL_MAC_IPADDR_ENTRY 9
	+#define OP_MANAGE_QHASH_TABLE_ENTRY 10
	+#define OP_QP_MODIFY 11
	+#define OP_QP_UPLOAD_CONTEXT 12
	+#define OP_CQ_CREATE 13
	+#define OP_CQ_DESTROY 14
	+#define OP_QP_CREATE 15
	+#define OP_QP_DESTROY 16
	+#define OP_ALLOC_STAG 17
	+#define OP_MR_REG_NON_SHARED 18
	+#define OP_DEALLOC_STAG 19
	+#define OP_MW_ALLOC 20
	+#define OP_QP_FLUSH_WQES 21
	+#define OP_ADD_ARP_CACHE_ENTRY 22
	+#define OP_MANAGE_PUSH_PAGE 23
	+#define OP_UPDATE_PE_SDS 24
	+#define OP_MANAGE_HMC_PM_FUNC_TABLE 25
	+#define OP_SUSPEND 26
	+#define OP_RESUME 27
	+#define OP_MANAGE_VF_PBLE_BP 28
	+#define OP_QUERY_FPM_VALUES 29
	+#define OP_COMMIT_FPM_VALUES 30
	+#define OP_REQUESTED_COMMANDS 31
	+#define OP_COMPLETED_COMMANDS 32
	+#define OP_GEN_AE 33
	+#define OP_SIZE_CQP_STAT_ARRAY 34
	+
	+#endif
	Index: sys/dev/ixl/iwarp/i40iw_hmc.h
	===================================================================
	--- /dev/null
	+++ sys/dev/ixl/iwarp/i40iw_hmc.h
	@@ -0,0 +1,239 @@
	+/*******************************************************************************
	+*
	+* Copyright (c) 2015-2017 Intel Corporation. All rights reserved.
	+*
	+* This software is available to you under a choice of one of two
	+* licenses. You may choose to be licensed under the terms of the GNU
	+* General Public License (GPL) Version 2, available from the file
	+* COPYING in the main directory of this source tree, or the
	+* OpenFabrics.org BSD license below:
	+*
	+* Redistribution and use in source and binary forms, with or
	+* without modification, are permitted provided that the following
	+* conditions are met:
	+*
	+* - Redistributions of source code must retain the above
	+* copyright notice, this list of conditions and the following
	+* disclaimer.
	+*
	+* - Redistributions in binary form must reproduce the above
	+* copyright notice, this list of conditions and the following
	+* disclaimer in the documentation and/or other materials
	+* provided with the distribution.
	+*
	+* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
	+* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
	+* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
	+* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
	+* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
	+* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
	+* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
	+* SOFTWARE.
	+*
	+*******************************************************************************/
	+/$FreeBSD$/
	+
	+#ifndef I40IW_HMC_H
	+#define I40IW_HMC_H
	+
	+#include "i40iw_d.h"
	+
	+struct i40iw_hw;
	+enum i40iw_status_code;
	+
	+#define I40IW_HMC_MAX_BP_COUNT 512
	+#define I40IW_MAX_SD_ENTRIES 11
	+#define I40IW_HW_DBG_HMC_INVALID_BP_MARK 0xCA
	+
	+#define I40IW_HMC_INFO_SIGNATURE 0x484D5347
	+#define I40IW_HMC_PD_CNT_IN_SD 512
	+#define I40IW_HMC_DIRECT_BP_SIZE 0x200000
	+#define I40IW_HMC_MAX_SD_COUNT 4096
	+#define I40IW_HMC_PAGED_BP_SIZE 4096
	+#define I40IW_HMC_PD_BP_BUF_ALIGNMENT 4096
	+#define I40IW_FIRST_VF_FPM_ID 16
	+#define FPM_MULTIPLIER 1024
	+
	+#define I40IW_INC_SD_REFCNT(sd_table) ((sd_table)->ref_cnt++)
	+#define I40IW_INC_PD_REFCNT(pd_table) ((pd_table)->ref_cnt++)
	+#define I40IW_INC_BP_REFCNT(bp) ((bp)->ref_cnt++)
	+
	+#define I40IW_DEC_SD_REFCNT(sd_table) ((sd_table)->ref_cnt--)
	+#define I40IW_DEC_PD_REFCNT(pd_table) ((pd_table)->ref_cnt--)
	+#define I40IW_DEC_BP_REFCNT(bp) ((bp)->ref_cnt--)
	+
	+/**
	+ * I40IW_INVALIDATE_PF_HMC_PD - Invalidates the pd cache in the hardware
	+ * @hw: pointer to our hw struct
	+ * @sd_idx: segment descriptor index
	+ * @pd_idx: page descriptor index
	+ */
	+#define I40IW_INVALIDATE_PF_HMC_PD(hw, sd_idx, pd_idx) \
	+ wr32((hw), I40E_PFHMC_PDINV, \
	+ (((sd_idx) << I40E_PFHMC_PDINV_PMSDIDX_SHIFT) \| \
	+ (0x1 << I40E_PFHMC_PDINV_PMSDPARTSEL_SHIFT) \| \
	+ ((pd_idx) << I40E_PFHMC_PDINV_PMPDIDX_SHIFT)))
	+
	+/**
	+ * I40IW_INVALIDATE_VF_HMC_PD - Invalidates the pd cache in the hardware
	+ * @hw: pointer to our hw struct
	+ * @sd_idx: segment descriptor index
	+ * @pd_idx: page descriptor index
	+ * @hmc_fn_id: VF's function id
	+ */
	+#define I40IW_INVALIDATE_VF_HMC_PD(hw, sd_idx, pd_idx, hmc_fn_id) \
	+ wr32(hw, I40E_GLHMC_VFPDINV(hmc_fn_id - I40IW_FIRST_VF_FPM_ID), \
	+ ((sd_idx << I40E_PFHMC_PDINV_PMSDIDX_SHIFT) \| \
	+ (pd_idx << I40E_PFHMC_PDINV_PMPDIDX_SHIFT)))
	+
	+struct i40iw_hmc_obj_info {
	+ u64 base;
	+ u32 max_cnt;
	+ u32 cnt;
	+ u64 size;
	+};
	+
	+enum i40iw_sd_entry_type {
	+ I40IW_SD_TYPE_INVALID = 0,
	+ I40IW_SD_TYPE_PAGED = 1,
	+ I40IW_SD_TYPE_DIRECT = 2
	+};
	+
	+struct i40iw_hmc_bp {
	+ enum i40iw_sd_entry_type entry_type;
	+ struct i40iw_dma_mem addr;
	+ u32 sd_pd_index;
	+ u32 ref_cnt;
	+};
	+
	+struct i40iw_hmc_pd_entry {
	+ struct i40iw_hmc_bp bp;
	+ u32 sd_index;
	+ bool rsrc_pg;
	+ bool valid;
	+};
	+
	+struct i40iw_hmc_pd_table {
	+ struct i40iw_dma_mem pd_page_addr;
	+ struct i40iw_hmc_pd_entry *pd_entry;
	+ struct i40iw_virt_mem pd_entry_virt_mem;
	+ u32 ref_cnt;
	+ u32 sd_index;
	+};
	+
	+struct i40iw_hmc_sd_entry {
	+ enum i40iw_sd_entry_type entry_type;
	+ bool valid;
	+
	+ union {
	+ struct i40iw_hmc_pd_table pd_table;
	+ struct i40iw_hmc_bp bp;
	+ } u;
	+};
	+
	+struct i40iw_hmc_sd_table {
	+ struct i40iw_virt_mem addr;
	+ u32 sd_cnt;
	+ u32 ref_cnt;
	+ struct i40iw_hmc_sd_entry *sd_entry;
	+};
	+
	+struct i40iw_hmc_info {
	+ u32 signature;
	+ u8 hmc_fn_id;
	+ u16 first_sd_index;
	+
	+ struct i40iw_hmc_obj_info *hmc_obj;
	+ struct i40iw_virt_mem hmc_obj_virt_mem;
	+ struct i40iw_hmc_sd_table sd_table;
	+ u16 sd_indexes[I40IW_HMC_MAX_SD_COUNT];
	+};
	+
	+struct update_sd_entry {
	+ u64 cmd;
	+ u64 data;
	+};
	+
	+struct i40iw_update_sds_info {
	+ u32 cnt;
	+ u8 hmc_fn_id;
	+ struct update_sd_entry entry[I40IW_MAX_SD_ENTRIES];
	+};
	+
	+struct i40iw_ccq_cqe_info;
	+struct i40iw_hmc_fcn_info {
	+ void (callback_fcn)(struct i40iw_sc_dev , void *,
	+ struct i40iw_ccq_cqe_info *);
	+ void *cqp_callback_param;
	+ u32 vf_id;
	+ u16 iw_vf_idx;
	+ bool free_fcn;
	+};
	+
	+enum i40iw_hmc_rsrc_type {
	+ I40IW_HMC_IW_QP = 0,
	+ I40IW_HMC_IW_CQ = 1,
	+ I40IW_HMC_IW_SRQ = 2,
	+ I40IW_HMC_IW_HTE = 3,
	+ I40IW_HMC_IW_ARP = 4,
	+ I40IW_HMC_IW_APBVT_ENTRY = 5,
	+ I40IW_HMC_IW_MR = 6,
	+ I40IW_HMC_IW_XF = 7,
	+ I40IW_HMC_IW_XFFL = 8,
	+ I40IW_HMC_IW_Q1 = 9,
	+ I40IW_HMC_IW_Q1FL = 10,
	+ I40IW_HMC_IW_TIMER = 11,
	+ I40IW_HMC_IW_FSIMC = 12,
	+ I40IW_HMC_IW_FSIAV = 13,
	+ I40IW_HMC_IW_PBLE = 14,
	+ I40IW_HMC_IW_MAX = 15,
	+};
	+
	+struct i40iw_hmc_create_obj_info {
	+ struct i40iw_hmc_info *hmc_info;
	+ struct i40iw_virt_mem add_sd_virt_mem;
	+ u32 rsrc_type;
	+ u32 start_idx;
	+ u32 count;
	+ u32 add_sd_cnt;
	+ enum i40iw_sd_entry_type entry_type;
	+ bool is_pf;
	+};
	+
	+struct i40iw_hmc_del_obj_info {
	+ struct i40iw_hmc_info *hmc_info;
	+ struct i40iw_virt_mem del_sd_virt_mem;
	+ u32 rsrc_type;
	+ u32 start_idx;
	+ u32 count;
	+ u32 del_sd_cnt;
	+ bool is_pf;
	+};
	+
	+enum i40iw_status_code i40iw_copy_dma_mem(struct i40iw_hw hw, void dest_buf,
	+ struct i40iw_dma_mem *src_mem, u64 src_offset, u64 size);
	+enum i40iw_status_code i40iw_sc_create_hmc_obj(struct i40iw_sc_dev *dev,
	+ struct i40iw_hmc_create_obj_info *info);
	+enum i40iw_status_code i40iw_sc_del_hmc_obj(struct i40iw_sc_dev *dev,
	+ struct i40iw_hmc_del_obj_info *info,
	+ bool reset);
	+enum i40iw_status_code i40iw_hmc_sd_one(struct i40iw_sc_dev *dev, u8 hmc_fn_id,
	+ u64 pa, u32 sd_idx, enum i40iw_sd_entry_type type,
	+ bool setsd);
	+enum i40iw_status_code i40iw_update_sds_noccq(struct i40iw_sc_dev *dev,
	+ struct i40iw_update_sds_info *info);
	+enum i40iw_status_code i40iw_add_sd_table_entry(struct i40iw_hw *hw,
	+ struct i40iw_hmc_info *hmc_info, u32 sd_index,
	+ enum i40iw_sd_entry_type type, u64 direct_mode_sz);
	+enum i40iw_status_code i40iw_add_pd_table_entry(struct i40iw_hw *hw,
	+ struct i40iw_hmc_info *hmc_info, u32 pd_index,
	+ struct i40iw_dma_mem *rsrc_pg);
	+enum i40iw_status_code i40iw_remove_pd_bp(struct i40iw_sc_dev *dev,
	+ struct i40iw_hmc_info *hmc_info, u32 idx);
	+enum i40iw_status_code i40iw_prep_remove_sd_bp(struct i40iw_hmc_info *hmc_info, u32 idx);
	+enum i40iw_status_code i40iw_prep_remove_pd_page(struct i40iw_hmc_info *hmc_info, u32 idx);
	+
	+#define ENTER_SHARED_FUNCTION()
	+#define EXIT_SHARED_FUNCTION()
	+
	+#endif /* I40IW_HMC_H */
	Index: sys/dev/ixl/iwarp/i40iw_hmc.c
	===================================================================
	--- /dev/null
	+++ sys/dev/ixl/iwarp/i40iw_hmc.c
	@@ -0,0 +1,739 @@
	+/*******************************************************************************
	+*
	+* Copyright (c) 2015-2017 Intel Corporation. All rights reserved.
	+*
	+* This software is available to you under a choice of one of two
	+* licenses. You may choose to be licensed under the terms of the GNU
	+* General Public License (GPL) Version 2, available from the file
	+* COPYING in the main directory of this source tree, or the
	+* OpenFabrics.org BSD license below:
	+*
	+* Redistribution and use in source and binary forms, with or
	+* without modification, are permitted provided that the following
	+* conditions are met:
	+*
	+* - Redistributions of source code must retain the above
	+* copyright notice, this list of conditions and the following
	+* disclaimer.
	+*
	+* - Redistributions in binary form must reproduce the above
	+* copyright notice, this list of conditions and the following
	+* disclaimer in the documentation and/or other materials
	+* provided with the distribution.
	+*
	+* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
	+* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
	+* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
	+* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
	+* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
	+* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
	+* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
	+* SOFTWARE.
	+*
	+*******************************************************************************/
	+/$FreeBSD$/
	+
	+#include "i40iw_osdep.h"
	+#include "i40iw_register.h"
	+#include "i40iw_status.h"
	+#include "i40iw_hmc.h"
	+#include "i40iw_d.h"
	+#include "i40iw_type.h"
	+#include "i40iw_p.h"
	+
	+/**
	+ * i40iw_find_sd_index_limit - finds segment descriptor index limit
	+ * @hmc_info: pointer to the HMC configuration information structure
	+ * @type: type of HMC resources we're searching
	+ * @index: starting index for the object
	+ * @cnt: number of objects we're trying to create
	+ * @sd_idx: pointer to return index of the segment descriptor in question
	+ * @sd_limit: pointer to return the maximum number of segment descriptors
	+ *
	+ * This function calculates the segment descriptor index and index limit
	+ * for the resource defined by i40iw_hmc_rsrc_type.
	+ */
	+
	+static void
	+i40iw_find_sd_index_limit(struct i40iw_hmc_info *hmc_info,
	+ u32 type,
	+ u32 idx,
	+ u32 cnt,
	+ u32 *sd_idx,
	+ u32 *sd_limit)
	+{
	+ u64 fpm_addr, fpm_limit;
	+
	+ fpm_addr = hmc_info->hmc_obj[(type)].base +
	+ hmc_info->hmc_obj[type].size * idx;
	+ fpm_limit = fpm_addr + hmc_info->hmc_obj[type].size * cnt;
	+ *sd_idx = (u32)(fpm_addr / I40IW_HMC_DIRECT_BP_SIZE);
	+ *sd_limit = (u32)((fpm_limit - 1) / I40IW_HMC_DIRECT_BP_SIZE);
	+ *sd_limit += 1;
	+}
	+
	+/**
	+ * i40iw_find_pd_index_limit - finds page descriptor index limit
	+ * @hmc_info: pointer to the HMC configuration information struct
	+ * @type: HMC resource type we're examining
	+ * @idx: starting index for the object
	+ * @cnt: number of objects we're trying to create
	+ * @pd_index: pointer to return page descriptor index
	+ * @pd_limit: pointer to return page descriptor index limit
	+ *
	+ * Calculates the page descriptor index and index limit for the resource
	+ * defined by i40iw_hmc_rsrc_type.
	+ */
	+
	+static void
	+i40iw_find_pd_index_limit(struct i40iw_hmc_info *hmc_info,
	+ u32 type,
	+ u32 idx,
	+ u32 cnt,
	+ u32 *pd_idx,
	+ u32 *pd_limit)
	+{
	+ u64 fpm_adr, fpm_limit;
	+
	+ fpm_adr = hmc_info->hmc_obj[type].base +
	+ hmc_info->hmc_obj[type].size * idx;
	+ fpm_limit = fpm_adr + (hmc_info)->hmc_obj[(type)].size * (cnt);
	+ *(pd_idx) = (u32)(fpm_adr / I40IW_HMC_PAGED_BP_SIZE);
	+ *(pd_limit) = (u32)((fpm_limit - 1) / I40IW_HMC_PAGED_BP_SIZE);
	+ *(pd_limit) += 1;
	+}
	+
	+/**
	+ * i40iw_set_sd_entry - setup entry for sd programming
	+ * @pa: physical addr
	+ * @idx: sd index
	+ * @type: paged or direct sd
	+ * @entry: sd entry ptr
	+ */
	+static inline void
	+i40iw_set_sd_entry(u64 pa,
	+ u32 idx,
	+ enum i40iw_sd_entry_type type,
	+ struct update_sd_entry *entry)
	+{
	+ entry->data = pa \| (I40IW_HMC_MAX_BP_COUNT << I40E_PFHMC_SDDATALOW_PMSDBPCOUNT_SHIFT) \|
	+ (((type == I40IW_SD_TYPE_PAGED) ? 0 : 1) <<
	+ I40E_PFHMC_SDDATALOW_PMSDTYPE_SHIFT) \|
	+ (1 << I40E_PFHMC_SDDATALOW_PMSDVALID_SHIFT);
	+ entry->cmd = (idx \| (1 << I40E_PFHMC_SDCMD_PMSDWR_SHIFT) \| (1 << 15));
	+}
	+
	+/**
	+ * i40iw_clr_sd_entry - setup entry for sd clear
	+ * @idx: sd index
	+ * @type: paged or direct sd
	+ * @entry: sd entry ptr
	+ */
	+static inline void
	+i40iw_clr_sd_entry(u32 idx, enum i40iw_sd_entry_type type,
	+ struct update_sd_entry *entry)
	+{
	+ entry->data = (I40IW_HMC_MAX_BP_COUNT <<
	+ I40E_PFHMC_SDDATALOW_PMSDBPCOUNT_SHIFT) \|
	+ (((type == I40IW_SD_TYPE_PAGED) ? 0 : 1) <<
	+ I40E_PFHMC_SDDATALOW_PMSDTYPE_SHIFT);
	+ entry->cmd = (idx \| (1 << I40E_PFHMC_SDCMD_PMSDWR_SHIFT) \| (1 << 15));
	+}
	+
	+/**
	+ * i40iw_hmc_sd_one - setup 1 sd entry for cqp
	+ * @dev: pointer to the device structure
	+ * @hmc_fn_id: hmc's function id
	+ * @pa: physical addr
	+ * @sd_idx: sd index
	+ * @type: paged or direct sd
	+ * @setsd: flag to set or clear sd
	+ */
	+enum i40iw_status_code
	+i40iw_hmc_sd_one(struct i40iw_sc_dev *dev,
	+ u8 hmc_fn_id,
	+ u64 pa, u32 sd_idx,
	+ enum i40iw_sd_entry_type type,
	+ bool setsd)
	+{
	+ struct i40iw_update_sds_info sdinfo;
	+
	+ sdinfo.cnt = 1;
	+ sdinfo.hmc_fn_id = hmc_fn_id;
	+ if (setsd)
	+ i40iw_set_sd_entry(pa, sd_idx, type, sdinfo.entry);
	+ else
	+ i40iw_clr_sd_entry(sd_idx, type, sdinfo.entry);
	+
	+ return dev->cqp->process_cqp_sds(dev, &sdinfo);
	+}
	+
	+/**
	+ * i40iw_hmc_sd_grp - setup group od sd entries for cqp
	+ * @dev: pointer to the device structure
	+ * @hmc_info: pointer to the HMC configuration information struct
	+ * @sd_index: sd index
	+ * @sd_cnt: number of sd entries
	+ * @setsd: flag to set or clear sd
	+ */
	+static enum i40iw_status_code
	+i40iw_hmc_sd_grp(struct i40iw_sc_dev *dev,
	+ struct i40iw_hmc_info *hmc_info,
	+ u32 sd_index,
	+ u32 sd_cnt,
	+ bool setsd)
	+{
	+ struct i40iw_hmc_sd_entry *sd_entry;
	+ struct i40iw_update_sds_info sdinfo;
	+ u64 pa;
	+ u32 i;
	+ enum i40iw_status_code ret_code = 0;
	+
	+ memset(&sdinfo, 0, sizeof(sdinfo));
	+ sdinfo.hmc_fn_id = hmc_info->hmc_fn_id;
	+ for (i = sd_index; i < sd_index + sd_cnt; i++) {
	+ sd_entry = &hmc_info->sd_table.sd_entry[i];
	+ if (!sd_entry \|\|
	+ (!sd_entry->valid && setsd) \|\|
	+ (sd_entry->valid && !setsd))
	+ continue;
	+ if (setsd) {
	+ pa = (sd_entry->entry_type == I40IW_SD_TYPE_PAGED) ?
	+ sd_entry->u.pd_table.pd_page_addr.pa :
	+ sd_entry->u.bp.addr.pa;
	+ i40iw_set_sd_entry(pa, i, sd_entry->entry_type,
	+ &sdinfo.entry[sdinfo.cnt]);
	+ } else {
	+ i40iw_clr_sd_entry(i, sd_entry->entry_type,
	+ &sdinfo.entry[sdinfo.cnt]);
	+ }
	+ sdinfo.cnt++;
	+ if (sdinfo.cnt == I40IW_MAX_SD_ENTRIES) {
	+ ret_code = dev->cqp->process_cqp_sds(dev, &sdinfo);
	+ if (ret_code) {
	+ i40iw_debug(dev, I40IW_DEBUG_HMC,
	+ "i40iw_hmc_sd_grp: sd_programming failed err=%d\n",
	+ ret_code);
	+ return ret_code;
	+ }
	+ sdinfo.cnt = 0;
	+ }
	+ }
	+ if (sdinfo.cnt)
	+ ret_code = dev->cqp->process_cqp_sds(dev, &sdinfo);
	+
	+ return ret_code;
	+}
	+
	+/**
	+ * i40iw_hmc_finish_add_sd_reg - program sd entries for objects
	+ * @dev: pointer to the device structure
	+ * @info: create obj info
	+ */
	+static enum i40iw_status_code
	+i40iw_hmc_finish_add_sd_reg(struct i40iw_sc_dev *dev,
	+ struct i40iw_hmc_create_obj_info *info)
	+{
	+ if (info->start_idx >= info->hmc_info->hmc_obj[info->rsrc_type].cnt)
	+ return I40IW_ERR_INVALID_HMC_OBJ_INDEX;
	+
	+ if ((info->start_idx + info->count) >
	+ info->hmc_info->hmc_obj[info->rsrc_type].cnt)
	+ return I40IW_ERR_INVALID_HMC_OBJ_COUNT;
	+
	+ if (!info->add_sd_cnt)
	+ return 0;
	+
	+ return i40iw_hmc_sd_grp(dev, info->hmc_info,
	+ info->hmc_info->sd_indexes[0],
	+ info->add_sd_cnt, TRUE);
	+}
	+
	+/**
	+ * i40iw_create_iw_hmc_obj - allocate backing store for hmc objects
	+ * @dev: pointer to the device structure
	+ * @info: pointer to i40iw_hmc_iw_create_obj_info struct
	+ *
	+ * This will allocate memory for PDs and backing pages and populate
	+ * the sd and pd entries.
	+ */
	+enum i40iw_status_code
	+i40iw_sc_create_hmc_obj(struct i40iw_sc_dev *dev,
	+ struct i40iw_hmc_create_obj_info *info)
	+{
	+ struct i40iw_hmc_sd_entry *sd_entry;
	+ u32 sd_idx, sd_lmt;
	+ u32 pd_idx = 0, pd_lmt = 0;
	+ u32 pd_idx1 = 0, pd_lmt1 = 0;
	+ u32 i, j;
	+ bool pd_error = FALSE;
	+ enum i40iw_status_code ret_code = 0;
	+
	+ if (info->start_idx >= info->hmc_info->hmc_obj[info->rsrc_type].cnt)
	+ return I40IW_ERR_INVALID_HMC_OBJ_INDEX;
	+
	+ if ((info->start_idx + info->count) >
	+ info->hmc_info->hmc_obj[info->rsrc_type].cnt) {
	+ i40iw_debug(dev, I40IW_DEBUG_HMC,
	+ "%s: error type %u, start = %u, req cnt %u, cnt = %u\n",
	+ __func__, info->rsrc_type, info->start_idx, info->count,
	+ info->hmc_info->hmc_obj[info->rsrc_type].cnt);
	+ return I40IW_ERR_INVALID_HMC_OBJ_COUNT;
	+ }
	+ i40iw_find_sd_index_limit(info->hmc_info, info->rsrc_type,
	+ info->start_idx, info->count,
	+ &sd_idx, &sd_lmt);
	+ if (sd_idx >= info->hmc_info->sd_table.sd_cnt \|\|
	+ sd_lmt > info->hmc_info->sd_table.sd_cnt) {
	+ return I40IW_ERR_INVALID_SD_INDEX;
	+ }
	+ i40iw_find_pd_index_limit(info->hmc_info, info->rsrc_type,
	+ info->start_idx, info->count, &pd_idx, &pd_lmt);
	+
	+ for (j = sd_idx; j < sd_lmt; j++) {
	+ ret_code = i40iw_add_sd_table_entry(dev->hw, info->hmc_info,
	+ j,
	+ info->entry_type,
	+ I40IW_HMC_DIRECT_BP_SIZE);
	+ if (ret_code)
	+ goto exit_sd_error;
	+ sd_entry = &info->hmc_info->sd_table.sd_entry[j];
	+
	+ if ((sd_entry->entry_type == I40IW_SD_TYPE_PAGED) &&
	+ ((dev->hmc_info == info->hmc_info) &&
	+ (info->rsrc_type != I40IW_HMC_IW_PBLE))) {
	+ pd_idx1 = max(pd_idx, (j * I40IW_HMC_MAX_BP_COUNT));
	+ pd_lmt1 = min(pd_lmt,
	+ (j + 1) * I40IW_HMC_MAX_BP_COUNT);
	+ for (i = pd_idx1; i < pd_lmt1; i++) {
	+ /* update the pd table entry */
	+ ret_code = i40iw_add_pd_table_entry(dev->hw, info->hmc_info,
	+ i, NULL);
	+ if (ret_code) {
	+ pd_error = TRUE;
	+ break;
	+ }
	+ }
	+ if (pd_error) {
	+ while (i && (i > pd_idx1)) {
	+ i40iw_remove_pd_bp(dev, info->hmc_info, (i - 1));
	+ i--;
	+ }
	+ }
	+ }
	+ if (sd_entry->valid)
	+ continue;
	+
	+ info->hmc_info->sd_indexes[info->add_sd_cnt] = (u16)j;
	+ info->add_sd_cnt++;
	+ sd_entry->valid = TRUE;
	+ }
	+ return i40iw_hmc_finish_add_sd_reg(dev, info);
	+
	+exit_sd_error:
	+ while (j && (j > sd_idx)) {
	+ sd_entry = &info->hmc_info->sd_table.sd_entry[j - 1];
	+ switch (sd_entry->entry_type) {
	+ case I40IW_SD_TYPE_PAGED:
	+ pd_idx1 = max(pd_idx,
	+ (j - 1) * I40IW_HMC_MAX_BP_COUNT);
	+ pd_lmt1 = min(pd_lmt, (j * I40IW_HMC_MAX_BP_COUNT));
	+ for (i = pd_idx1; i < pd_lmt1; i++)
	+ i40iw_prep_remove_pd_page(info->hmc_info, i);
	+ break;
	+ case I40IW_SD_TYPE_DIRECT:
	+ i40iw_prep_remove_pd_page(info->hmc_info, (j - 1));
	+ break;
	+ default:
	+ ret_code = I40IW_ERR_INVALID_SD_TYPE;
	+ break;
	+ }
	+ j--;
	+ }
	+
	+ return ret_code;
	+}
	+
	+/**
	+ * i40iw_finish_del_sd_reg - delete sd entries for objects
	+ * @dev: pointer to the device structure
	+ * @info: dele obj info
	+ * @reset: TRUE if called before reset
	+ */
	+static enum i40iw_status_code
	+i40iw_finish_del_sd_reg(struct i40iw_sc_dev *dev,
	+ struct i40iw_hmc_del_obj_info *info,
	+ bool reset)
	+{
	+ struct i40iw_hmc_sd_entry *sd_entry;
	+ enum i40iw_status_code ret_code = 0;
	+ u32 i, sd_idx;
	+ struct i40iw_dma_mem *mem;
	+
	+ if (dev->is_pf && !reset)
	+ ret_code = i40iw_hmc_sd_grp(dev, info->hmc_info,
	+ info->hmc_info->sd_indexes[0],
	+ info->del_sd_cnt, FALSE);
	+
	+ if (ret_code)
	+ i40iw_debug(dev, I40IW_DEBUG_HMC, "%s: error cqp sd sd_grp\n", __func__);
	+
	+ for (i = 0; i < info->del_sd_cnt; i++) {
	+ sd_idx = info->hmc_info->sd_indexes[i];
	+ sd_entry = &info->hmc_info->sd_table.sd_entry[sd_idx];
	+ if (!sd_entry)
	+ continue;
	+ mem = (sd_entry->entry_type == I40IW_SD_TYPE_PAGED) ?
	+ &sd_entry->u.pd_table.pd_page_addr :
	+ &sd_entry->u.bp.addr;
	+
	+ if (!mem \|\| !mem->va)
	+ i40iw_debug(dev, I40IW_DEBUG_HMC, "%s: error cqp sd mem\n", __func__);
	+ else
	+ i40iw_free_dma_mem(dev->hw, mem);
	+ }
	+ return ret_code;
	+}
	+
	+/**
	+ * i40iw_del_iw_hmc_obj - remove pe hmc objects
	+ * @dev: pointer to the device structure
	+ * @info: pointer to i40iw_hmc_del_obj_info struct
	+ * @reset: TRUE if called before reset
	+ *
	+ * This will de-populate the SDs and PDs. It frees
	+ * the memory for PDS and backing storage. After this function is returned,
	+ * caller should deallocate memory allocated previously for
	+ * book-keeping information about PDs and backing storage.
	+ */
	+enum i40iw_status_code
	+i40iw_sc_del_hmc_obj(struct i40iw_sc_dev *dev,
	+ struct i40iw_hmc_del_obj_info *info,
	+ bool reset)
	+{
	+ struct i40iw_hmc_pd_table *pd_table;
	+ u32 sd_idx, sd_lmt;
	+ u32 pd_idx, pd_lmt, rel_pd_idx;
	+ u32 i, j;
	+ enum i40iw_status_code ret_code = 0;
	+
	+ if (info->start_idx >= info->hmc_info->hmc_obj[info->rsrc_type].cnt) {
	+ i40iw_debug(dev, I40IW_DEBUG_HMC,
	+ "%s: error start_idx[%04d] >= [type %04d].cnt[%04d]\n",
	+ __func__, info->start_idx, info->rsrc_type,
	+ info->hmc_info->hmc_obj[info->rsrc_type].cnt);
	+ return I40IW_ERR_INVALID_HMC_OBJ_INDEX;
	+ }
	+ if ((info->start_idx + info->count) >
	+ info->hmc_info->hmc_obj[info->rsrc_type].cnt) {
	+ i40iw_debug(dev, I40IW_DEBUG_HMC,
	+ "%s: error start_idx[%04d] + count %04d >= [type %04d].cnt[%04d]\n",
	+ __func__, info->start_idx, info->count,
	+ info->rsrc_type,
	+ info->hmc_info->hmc_obj[info->rsrc_type].cnt);
	+ return I40IW_ERR_INVALID_HMC_OBJ_COUNT;
	+ }
	+ i40iw_find_pd_index_limit(info->hmc_info, info->rsrc_type,
	+ info->start_idx, info->count, &pd_idx, &pd_lmt);
	+
	+ for (j = pd_idx; j < pd_lmt; j++) {
	+ sd_idx = j / I40IW_HMC_PD_CNT_IN_SD;
	+
	+ if (info->hmc_info->sd_table.sd_entry[sd_idx].entry_type !=
	+ I40IW_SD_TYPE_PAGED)
	+ continue;
	+
	+ rel_pd_idx = j % I40IW_HMC_PD_CNT_IN_SD;
	+ pd_table = &info->hmc_info->sd_table.sd_entry[sd_idx].u.pd_table;
	+ if (pd_table->pd_entry[rel_pd_idx].valid) {
	+ ret_code = i40iw_remove_pd_bp(dev, info->hmc_info, j);
	+ if (ret_code) {
	+ i40iw_debug(dev, I40IW_DEBUG_HMC, "%s: error\n", __func__);
	+ return ret_code;
	+ }
	+ }
	+ }
	+
	+ i40iw_find_sd_index_limit(info->hmc_info, info->rsrc_type,
	+ info->start_idx, info->count, &sd_idx, &sd_lmt);
	+ if (sd_idx >= info->hmc_info->sd_table.sd_cnt \|\|
	+ sd_lmt > info->hmc_info->sd_table.sd_cnt) {
	+ i40iw_debug(dev, I40IW_DEBUG_HMC, "%s: error invalid sd_idx\n", __func__);
	+ return I40IW_ERR_INVALID_SD_INDEX;
	+ }
	+ for (i = sd_idx; i < sd_lmt; i++) {
	+ if (!info->hmc_info->sd_table.sd_entry[i].valid)
	+ continue;
	+ switch (info->hmc_info->sd_table.sd_entry[i].entry_type) {
	+ case I40IW_SD_TYPE_DIRECT:
	+ ret_code = i40iw_prep_remove_sd_bp(info->hmc_info, i);
	+ if (!ret_code) {
	+ info->hmc_info->sd_indexes[info->del_sd_cnt] = (u16)i;
	+ info->del_sd_cnt++;
	+ }
	+ break;
	+ case I40IW_SD_TYPE_PAGED:
	+ ret_code = i40iw_prep_remove_pd_page(info->hmc_info, i);
	+ if (!ret_code) {
	+ info->hmc_info->sd_indexes[info->del_sd_cnt] = (u16)i;
	+ info->del_sd_cnt++;
	+ }
	+ break;
	+ default:
	+ break;
	+ }
	+ }
	+ return i40iw_finish_del_sd_reg(dev, info, reset);
	+}
	+
	+/**
	+ * i40iw_add_sd_table_entry - Adds a segment descriptor to the table
	+ * @hw: pointer to our hw struct
	+ * @hmc_info: pointer to the HMC configuration information struct
	+ * @sd_index: segment descriptor index to manipulate
	+ * @type: what type of segment descriptor we're manipulating
	+ * @direct_mode_sz: size to alloc in direct mode
	+ */
	+enum i40iw_status_code
	+i40iw_add_sd_table_entry(struct i40iw_hw *hw,
	+ struct i40iw_hmc_info *hmc_info,
	+ u32 sd_index,
	+ enum i40iw_sd_entry_type type,
	+ u64 direct_mode_sz)
	+{
	+ enum i40iw_status_code ret_code = 0;
	+ struct i40iw_hmc_sd_entry *sd_entry;
	+ bool dma_mem_alloc_done = FALSE;
	+ struct i40iw_dma_mem mem;
	+ u64 alloc_len;
	+
	+ sd_entry = &hmc_info->sd_table.sd_entry[sd_index];
	+ if (!sd_entry->valid) {
	+ if (type == I40IW_SD_TYPE_PAGED)
	+ alloc_len = I40IW_HMC_PAGED_BP_SIZE;
	+ else
	+ alloc_len = direct_mode_sz;
	+
	+ /* allocate a 4K pd page or 2M backing page */
	+ ret_code = i40iw_allocate_dma_mem(hw, &mem, alloc_len,
	+ I40IW_HMC_PD_BP_BUF_ALIGNMENT);
	+ if (ret_code)
	+ goto exit;
	+ dma_mem_alloc_done = TRUE;
	+ if (type == I40IW_SD_TYPE_PAGED) {
	+ ret_code = i40iw_allocate_virt_mem(hw,
	+ &sd_entry->u.pd_table.pd_entry_virt_mem,
	+ sizeof(struct i40iw_hmc_pd_entry) * 512);
	+ if (ret_code)
	+ goto exit;
	+ sd_entry->u.pd_table.pd_entry = (struct i40iw_hmc_pd_entry *)
	+ sd_entry->u.pd_table.pd_entry_virt_mem.va;
	+
	+ i40iw_memcpy(&sd_entry->u.pd_table.pd_page_addr, &mem, sizeof(sd_entry->u.pd_table.pd_page_addr));
	+ } else {
	+ i40iw_memcpy(&sd_entry->u.bp.addr, &mem, sizeof(sd_entry->u.bp.addr));
	+ sd_entry->u.bp.sd_pd_index = sd_index;
	+ }
	+
	+ hmc_info->sd_table.sd_entry[sd_index].entry_type = type;
	+
	+ I40IW_INC_SD_REFCNT(&hmc_info->sd_table);
	+ }
	+ if (sd_entry->entry_type == I40IW_SD_TYPE_DIRECT)
	+ I40IW_INC_BP_REFCNT(&sd_entry->u.bp);
	+exit:
	+ if (ret_code)
	+ if (dma_mem_alloc_done)
	+ i40iw_free_dma_mem(hw, &mem);
	+
	+ return ret_code;
	+}
	+
	+/**
	+ * i40iw_add_pd_table_entry - Adds page descriptor to the specified table
	+ * @hw: pointer to our HW structure
	+ * @hmc_info: pointer to the HMC configuration information structure
	+ * @pd_index: which page descriptor index to manipulate
	+ * @rsrc_pg: if not NULL, use preallocated page instead of allocating new one.
	+ *
	+ * This function:
	+ * 1. Initializes the pd entry
	+ * 2. Adds pd_entry in the pd_table
	+ * 3. Mark the entry valid in i40iw_hmc_pd_entry structure
	+ * 4. Initializes the pd_entry's ref count to 1
	+ * assumptions:
	+ * 1. The memory for pd should be pinned down, physically contiguous and
	+ * aligned on 4K boundary and zeroed memory.
	+ * 2. It should be 4K in size.
	+ */
	+enum i40iw_status_code
	+i40iw_add_pd_table_entry(struct i40iw_hw *hw,
	+ struct i40iw_hmc_info *hmc_info,
	+ u32 pd_index,
	+ struct i40iw_dma_mem *rsrc_pg)
	+{
	+ enum i40iw_status_code ret_code = 0;
	+ struct i40iw_hmc_pd_table *pd_table;
	+ struct i40iw_hmc_pd_entry *pd_entry;
	+ struct i40iw_dma_mem mem;
	+ struct i40iw_dma_mem *page = &mem;
	+ u32 sd_idx, rel_pd_idx;
	+ u64 *pd_addr;
	+ u64 page_desc;
	+
	+ if (pd_index / I40IW_HMC_PD_CNT_IN_SD >= hmc_info->sd_table.sd_cnt)
	+ return I40IW_ERR_INVALID_PAGE_DESC_INDEX;
	+
	+ sd_idx = (pd_index / I40IW_HMC_PD_CNT_IN_SD);
	+ if (hmc_info->sd_table.sd_entry[sd_idx].entry_type != I40IW_SD_TYPE_PAGED)
	+ return 0;
	+
	+ rel_pd_idx = (pd_index % I40IW_HMC_PD_CNT_IN_SD);
	+ pd_table = &hmc_info->sd_table.sd_entry[sd_idx].u.pd_table;
	+ pd_entry = &pd_table->pd_entry[rel_pd_idx];
	+ if (!pd_entry->valid) {
	+ if (rsrc_pg) {
	+ pd_entry->rsrc_pg = TRUE;
	+ page = rsrc_pg;
	+ } else {
	+ ret_code = i40iw_allocate_dma_mem(hw, page,
	+ I40IW_HMC_PAGED_BP_SIZE,
	+ I40IW_HMC_PD_BP_BUF_ALIGNMENT);
	+ if (ret_code)
	+ return ret_code;
	+ pd_entry->rsrc_pg = FALSE;
	+ }
	+
	+ i40iw_memcpy(&pd_entry->bp.addr, page, sizeof(pd_entry->bp.addr));
	+ pd_entry->bp.sd_pd_index = pd_index;
	+ pd_entry->bp.entry_type = I40IW_SD_TYPE_PAGED;
	+ page_desc = page->pa \| 0x1;
	+
	+ pd_addr = (u64 *)pd_table->pd_page_addr.va;
	+ pd_addr += rel_pd_idx;
	+
	+ i40iw_memcpy(pd_addr, &page_desc, sizeof(*pd_addr));
	+
	+ pd_entry->sd_index = sd_idx;
	+ pd_entry->valid = TRUE;
	+ I40IW_INC_PD_REFCNT(pd_table);
	+ if (hmc_info->hmc_fn_id < I40IW_FIRST_VF_FPM_ID)
	+ I40IW_INVALIDATE_PF_HMC_PD(hw, sd_idx, rel_pd_idx);
	+ else if (hw->hmc.hmc_fn_id != hmc_info->hmc_fn_id)
	+ I40IW_INVALIDATE_VF_HMC_PD(hw, sd_idx, rel_pd_idx,
	+ hmc_info->hmc_fn_id);
	+ }
	+ I40IW_INC_BP_REFCNT(&pd_entry->bp);
	+
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_remove_pd_bp - remove a backing page from a page descriptor
	+ * @hw: pointer to our HW structure
	+ * @hmc_info: pointer to the HMC configuration information structure
	+ * @idx: the page index
	+ * @is_pf: distinguishes a VF from a PF
	+ *
	+ * This function:
	+ * 1. Marks the entry in pd table (for paged address mode) or in sd table
	+ * (for direct address mode) invalid.
	+ * 2. Write to register PMPDINV to invalidate the backing page in FV cache
	+ * 3. Decrement the ref count for the pd _entry
	+ * assumptions:
	+ * 1. Caller can deallocate the memory used by backing storage after this
	+ * function returns.
	+ */
	+enum i40iw_status_code
	+i40iw_remove_pd_bp(struct i40iw_sc_dev *dev,
	+ struct i40iw_hmc_info *hmc_info,
	+ u32 idx)
	+{
	+ struct i40iw_hmc_pd_entry *pd_entry;
	+ struct i40iw_hmc_pd_table *pd_table;
	+ struct i40iw_hmc_sd_entry *sd_entry;
	+ u32 sd_idx, rel_pd_idx;
	+ struct i40iw_dma_mem *mem;
	+ u64 *pd_addr;
	+
	+ sd_idx = idx / I40IW_HMC_PD_CNT_IN_SD;
	+ rel_pd_idx = idx % I40IW_HMC_PD_CNT_IN_SD;
	+ if (sd_idx >= hmc_info->sd_table.sd_cnt)
	+ return I40IW_ERR_INVALID_PAGE_DESC_INDEX;
	+
	+ sd_entry = &hmc_info->sd_table.sd_entry[sd_idx];
	+ if (sd_entry->entry_type != I40IW_SD_TYPE_PAGED)
	+ return I40IW_ERR_INVALID_SD_TYPE;
	+
	+ pd_table = &hmc_info->sd_table.sd_entry[sd_idx].u.pd_table;
	+ pd_entry = &pd_table->pd_entry[rel_pd_idx];
	+ I40IW_DEC_BP_REFCNT(&pd_entry->bp);
	+ if (pd_entry->bp.ref_cnt)
	+ return 0;
	+
	+ pd_entry->valid = FALSE;
	+ I40IW_DEC_PD_REFCNT(pd_table);
	+ pd_addr = (u64 *)pd_table->pd_page_addr.va;
	+ pd_addr += rel_pd_idx;
	+ i40iw_memset(pd_addr, 0, sizeof(u64));
	+ if (dev->is_pf) {
	+ if (dev->hmc_fn_id == hmc_info->hmc_fn_id)
	+ I40IW_INVALIDATE_PF_HMC_PD(dev->hw, sd_idx, idx);
	+ else
	+ I40IW_INVALIDATE_VF_HMC_PD(dev->hw, sd_idx, idx,
	+ hmc_info->hmc_fn_id);
	+ }
	+ if (!pd_entry->rsrc_pg) {
	+ mem = &pd_entry->bp.addr;
	+ if (!mem \|\| !mem->va)
	+ return I40IW_ERR_PARAM;
	+ i40iw_free_dma_mem(dev->hw, mem);
	+ }
	+ if (!pd_table->ref_cnt)
	+ i40iw_free_virt_mem(dev->hw, &pd_table->pd_entry_virt_mem);
	+
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_prep_remove_sd_bp - Prepares to remove a backing page from a sd entry
	+ * @hmc_info: pointer to the HMC configuration information structure
	+ * @idx: the page index
	+ */
	+enum i40iw_status_code
	+i40iw_prep_remove_sd_bp(struct i40iw_hmc_info *hmc_info, u32 idx)
	+{
	+ struct i40iw_hmc_sd_entry *sd_entry;
	+
	+ sd_entry = &hmc_info->sd_table.sd_entry[idx];
	+ I40IW_DEC_BP_REFCNT(&sd_entry->u.bp);
	+ if (sd_entry->u.bp.ref_cnt)
	+ return I40IW_ERR_NOT_READY;
	+
	+ I40IW_DEC_SD_REFCNT(&hmc_info->sd_table);
	+ sd_entry->valid = FALSE;
	+
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_prep_remove_pd_page - Prepares to remove a PD page from sd entry.
	+ * @hmc_info: pointer to the HMC configuration information structure
	+ * @idx: segment descriptor index to find the relevant page descriptor
	+ */
	+enum i40iw_status_code
	+i40iw_prep_remove_pd_page(struct i40iw_hmc_info *hmc_info,
	+ u32 idx)
	+{
	+ struct i40iw_hmc_sd_entry *sd_entry;
	+
	+ sd_entry = &hmc_info->sd_table.sd_entry[idx];
	+
	+ if (sd_entry->u.pd_table.ref_cnt)
	+ return I40IW_ERR_NOT_READY;
	+
	+ sd_entry->valid = FALSE;
	+ I40IW_DEC_SD_REFCNT(&hmc_info->sd_table);
	+
	+ return 0;
	+}
	Index: sys/dev/ixl/iwarp/i40iw_osdep.h
	===================================================================
	--- /dev/null
	+++ sys/dev/ixl/iwarp/i40iw_osdep.h
	@@ -0,0 +1,171 @@
	+/*******************************************************************************
	+*
	+* Copyright (c) 2015-2017 Intel Corporation. All rights reserved.
	+*
	+* This software is available to you under a choice of one of two
	+* licenses. You may choose to be licensed under the terms of the GNU
	+* General Public License (GPL) Version 2, available from the file
	+* COPYING in the main directory of this source tree, or the
	+* OpenFabrics.org BSD license below:
	+*
	+* Redistribution and use in source and binary forms, with or
	+* without modification, are permitted provided that the following
	+* conditions are met:
	+*
	+* - Redistributions of source code must retain the above
	+* copyright notice, this list of conditions and the following
	+* disclaimer.
	+*
	+* - Redistributions in binary form must reproduce the above
	+* copyright notice, this list of conditions and the following
	+* disclaimer in the documentation and/or other materials
	+* provided with the distribution.
	+*
	+* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
	+* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
	+* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
	+* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
	+* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
	+* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
	+* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
	+* SOFTWARE.
	+*
	+*******************************************************************************/
	+/$FreeBSD$/
	+
	+#ifndef _I40IW_OSDEP_H_
	+#define _I40IW_OSDEP_H_
	+
	+#include <linux/list.h>
	+#include <linux/timer.h>
	+#include <linux/spinlock.h>
	+#include <linux/mutex.h>
	+#include <linux/workqueue.h>
	+#include <linux/if_ether.h>
	+#include <linux/etherdevice.h>
	+#include <linux/fs.h>
	+#include <linux/io.h>
	+#include <linux/random.h>
	+
	+#include <sys/bus.h>
	+#include <machine/bus.h>
	+
	+#define IOMEM
	+#define OS_TIMER timer_list
	+#define ATOMIC atomic_t
	+#define i40iw_get_virt_to_phy vtophys
	+
	+#define I40IW_NTOHS(a) ntohs(a)
	+
	+#define CPU_TO_LE16(o) htole16(o)
	+#define CPU_TO_LE32(s) htole32(s)
	+#define CPU_TO_LE64(h) htole64(h)
	+#define LE16_TO_CPU(a) le16toh(a)
	+#define LE32_TO_CPU(c) le32toh(c)
	+#define LE64_TO_CPU(k) le64toh(k)
	+
	+#define i40iw_memcpy(a, b, c) memcpy((a), (b), (c))
	+#define i40iw_memset(a, b, c) memset((a), (b), (c))
	+
	+#define i40iw_mb() mb()
	+#define i40iw_wmb() wmb()
	+
	+#define i40iw_usec_delay(x) DELAY(x)
	+#define mdelay(x) DELAY((x) * 1000)
	+
	+typedef unsigned long *i40iw_bits_t;
	+typedef uintptr_t i40iw_uintptr;
	+
	+typedef __be16 BE16;
	+typedef __be32 BE32;
	+
	+#define I40IW_NTOHL(a) ntohl(a)
	+
	+#define i40iw_free_dma_mem(hw, dma_mem) ixliw_free_dma_mem(dma_mem)
	+
	+#define rd32(a, reg) i40iw_rd32((a)->dev_context, (reg))
	+#define wr32(a, reg, value) i40iw_wr32((a)->dev_context, (reg), (value))
	+
	+#define rd64(a, reg) i40iw_rd64((a)->dev_context, (reg))
	+#define wr64(a, reg, value) i40iw_wr64((a)->dev_context, (reg), (value))
	+
	+#define db_wr32(value, a) writel((value), (a))
	+
	+#define i40iw_debug(h, m, s, ...) \
	+do { \
	+ if (((m) & (h)->debug_mask)) \
	+ printf("i40iw " s, ##__VA_ARGS__); \
	+} while (0)
	+
	+struct i40iw_dev_ctx;
	+struct i40iw_sc_dev;
	+struct i40iw_sc_qp;
	+struct i40iw_sc_vsi;
	+struct i40iw_update_sds_info;
	+
	+struct i40iw_dma_mem {
	+ void *va;
	+ u64 pa;
	+ bus_dma_tag_t tag;
	+ bus_dmamap_t map;
	+ bus_dma_segment_t seg;
	+ bus_size_t size;
	+ int nseg;
	+ int flags;
	+};
	+
	+struct i40iw_virt_mem {
	+ void *va;
	+ u32 size;
	+};
	+
	+u8 i40iw_get_hw_addr(void par);
	+enum i40iw_status_code i40iw_cqp_sds_cmd(struct i40iw_sc_dev , struct i40iw_update_sds_info );
	+void i40iw_remove_head(struct list_head list);
	+
	+void i40iw_hw_stats_start_timer(struct i40iw_sc_vsi *vsi);
	+void i40iw_hw_stats_stop_timer(struct i40iw_sc_vsi *vsi);
	+
	+void ixliw_free_dma_mem(struct i40iw_dma_mem *mem);
	+
	+u32 i40iw_rd32(struct i40iw_dev_ctx *dev_ctx, u32 reg);
	+void i40iw_wr32(struct i40iw_dev_ctx *dev_ctx, u32 reg, u32 value);
	+u64 i40iw_rd64(struct i40iw_dev_ctx *dev_ctx, u32 reg);
	+void i40iw_wr64(struct i40iw_dev_ctx *dev_ctx, u32 reg, u64 value);
	+
	+void i40iw_term_modify_qp(struct i40iw_sc_qp *qp, u8 next_state, u8 term, u8 term_len);
	+void i40iw_terminate_done(struct i40iw_sc_qp *qp, int timeout_occurred);
	+void i40iw_terminate_start_timer(struct i40iw_sc_qp *qp);
	+void i40iw_terminate_del_timer(struct i40iw_sc_qp *qp);
	+
	+static inline void
	+set_64bit_val(u64 *wqe_words, u32 byte_index, u64 value)
	+{
	+ wqe_words[byte_index >> 3] = value;
	+}
	+
	+/**
	+ * set_32bit_val - set 32 value to hw wqe
	+ * @wqe_words: wqe addr to write
	+ * @byte_index: index in wqe
	+ * @value: value to write
	+ **/
	+static inline void
	+set_32bit_val(u32 *wqe_words, u32 byte_index, u32 value)
	+{
	+ wqe_words[byte_index >> 2] = value;
	+}
	+
	+/**
	+ * get_64bit_val - read 64 bit value from wqe
	+ * @wqe_words: wqe addr
	+ * @byte_index: index to read from
	+ * @value: read value
	+ **/
	+static inline void
	+get_64bit_val(u64 wqe_words, u32 byte_index, u64 value)
	+{
	+ *value = wqe_words[byte_index >> 3];
	+}
	+
	+#endif /* _I40IW_OSDEP_H_ */
	Index: sys/dev/ixl/iwarp/i40iw_p.h
	===================================================================
	--- /dev/null
	+++ sys/dev/ixl/iwarp/i40iw_p.h
	@@ -0,0 +1,124 @@
	+/*******************************************************************************
	+*
	+* Copyright (c) 2015-2017 Intel Corporation. All rights reserved.
	+*
	+* This software is available to you under a choice of one of two
	+* licenses. You may choose to be licensed under the terms of the GNU
	+* General Public License (GPL) Version 2, available from the file
	+* COPYING in the main directory of this source tree, or the
	+* OpenFabrics.org BSD license below:
	+*
	+* Redistribution and use in source and binary forms, with or
	+* without modification, are permitted provided that the following
	+* conditions are met:
	+*
	+* - Redistributions of source code must retain the above
	+* copyright notice, this list of conditions and the following
	+* disclaimer.
	+*
	+* - Redistributions in binary form must reproduce the above
	+* copyright notice, this list of conditions and the following
	+* disclaimer in the documentation and/or other materials
	+* provided with the distribution.
	+*
	+* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
	+* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
	+* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
	+* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
	+* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
	+* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
	+* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
	+* SOFTWARE.
	+*
	+*******************************************************************************/
	+/$FreeBSD$/
	+
	+#ifndef I40IW_P_H
	+#define I40IW_P_H
	+
	+#define PAUSE_TIMER_VALUE 0xFFFF
	+#define REFRESH_THRESHOLD 0x7FFF
	+#define HIGH_THRESHOLD 0x800
	+#define LOW_THRESHOLD 0x200
	+#define ALL_TC2PFC 0xFF
	+#define CQP_COMPL_WAIT_TIME 0x3E8
	+#define CQP_TIMEOUT_THRESHOLD 5
	+
	+void i40iw_debug_buf(struct i40iw_sc_dev *dev, enum i40iw_debug_flag mask,
	+ char desc, u64 buf, u32 size);
	+/* init operations */
	+enum i40iw_status_code i40iw_device_init(struct i40iw_sc_dev *dev,
	+ struct i40iw_device_init_info *info);
	+
	+void i40iw_device_init_pestat(struct i40iw_vsi_pestat *);
	+
	+void i40iw_sc_cqp_post_sq(struct i40iw_sc_cqp *cqp);
	+
	+u64 i40iw_sc_cqp_get_next_send_wqe(struct i40iw_sc_cqp cqp, u64 scratch);
	+
	+void i40iw_check_cqp_progress(struct i40iw_cqp_timeout cqp_timeout, struct i40iw_sc_dev dev);
	+
	+enum i40iw_status_code i40iw_sc_mr_fast_register(struct i40iw_sc_qp *qp,
	+ struct i40iw_fast_reg_stag_info *info,
	+ bool post_sq);
	+
	+/* HMC/FPM functions */
	+enum i40iw_status_code i40iw_sc_init_iw_hmc(struct i40iw_sc_dev *dev,
	+ u8 hmc_fn_id);
	+
	+void i40iw_insert_wqe_hdr(u64 *wqe, u64 header);
	+
	+/* stats functions */
	+void i40iw_hw_stats_refresh_all(struct i40iw_vsi_pestat *stats);
	+void i40iw_hw_stats_read_all(struct i40iw_vsi_pestat stats, struct i40iw_dev_hw_stats stats_values);
	+void i40iw_hw_stats_read_32(struct i40iw_vsi_pestat *stats,
	+ enum i40iw_hw_stats_index_32b index,
	+ u64 *value);
	+void i40iw_hw_stats_read_64(struct i40iw_vsi_pestat *stats,
	+ enum i40iw_hw_stats_index_64b index,
	+ u64 *value);
	+void i40iw_hw_stats_init(struct i40iw_vsi_pestat *stats, u8 index, bool is_pf);
	+
	+/* vsi misc functions */
	+enum i40iw_status_code i40iw_vsi_stats_init(struct i40iw_sc_vsi vsi, struct i40iw_vsi_stats_info info);
	+void i40iw_vsi_stats_free(struct i40iw_sc_vsi *vsi);
	+void i40iw_sc_vsi_init(struct i40iw_sc_vsi vsi, struct i40iw_vsi_init_info info);
	+
	+void i40iw_terminate_send_fin(struct i40iw_sc_qp *qp);
	+
	+void i40iw_terminate_connection(struct i40iw_sc_qp qp, struct i40iw_aeqe_info info);
	+
	+void i40iw_terminate_received(struct i40iw_sc_qp qp, struct i40iw_aeqe_info info);
	+
	+enum i40iw_status_code i40iw_sc_suspend_qp(struct i40iw_sc_cqp *cqp,
	+ struct i40iw_sc_qp *qp, u64 scratch);
	+
	+enum i40iw_status_code i40iw_sc_resume_qp(struct i40iw_sc_cqp *cqp,
	+ struct i40iw_sc_qp *qp, u64 scratch);
	+
	+enum i40iw_status_code i40iw_sc_static_hmc_pages_allocated(struct i40iw_sc_cqp *cqp,
	+ u64 scratch, u8 hmc_fn_id,
	+ bool post_sq,
	+ bool poll_registers);
	+
	+enum i40iw_status_code i40iw_config_fpm_values(struct i40iw_sc_dev *dev, u32 qp_count);
	+
	+void free_sd_mem(struct i40iw_sc_dev *dev);
	+
	+enum i40iw_status_code i40iw_process_cqp_cmd(struct i40iw_sc_dev *dev,
	+ struct cqp_commands_info *pcmdinfo);
	+
	+enum i40iw_status_code i40iw_process_bh(struct i40iw_sc_dev *dev);
	+
	+/* prototype for functions used for dynamic memory allocation */
	+enum i40iw_status_code i40iw_allocate_dma_mem(struct i40iw_hw *hw,
	+ struct i40iw_dma_mem *mem, u64 size,
	+ u32 alignment);
	+enum i40iw_status_code i40iw_allocate_virt_mem(struct i40iw_hw *hw,
	+ struct i40iw_virt_mem *mem, u32 size);
	+enum i40iw_status_code i40iw_free_virt_mem(struct i40iw_hw *hw,
	+ struct i40iw_virt_mem *mem);
	+u8 i40iw_get_encoded_wqe_size(u32 wqsize, bool cqpsq);
	+void i40iw_reinitialize_ieq(struct i40iw_sc_dev *dev);
	+
	+#endif
	Index: sys/dev/ixl/iwarp/i40iw_puda.h
	===================================================================
	--- /dev/null
	+++ sys/dev/ixl/iwarp/i40iw_puda.h
	@@ -0,0 +1,188 @@
	+/*******************************************************************************
	+*
	+* Copyright (c) 2015-2017 Intel Corporation. All rights reserved.
	+*
	+* This software is available to you under a choice of one of two
	+* licenses. You may choose to be licensed under the terms of the GNU
	+* General Public License (GPL) Version 2, available from the file
	+* COPYING in the main directory of this source tree, or the
	+* OpenFabrics.org BSD license below:
	+*
	+* Redistribution and use in source and binary forms, with or
	+* without modification, are permitted provided that the following
	+* conditions are met:
	+*
	+* - Redistributions of source code must retain the above
	+* copyright notice, this list of conditions and the following
	+* disclaimer.
	+*
	+* - Redistributions in binary form must reproduce the above
	+* copyright notice, this list of conditions and the following
	+* disclaimer in the documentation and/or other materials
	+* provided with the distribution.
	+*
	+* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
	+* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
	+* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
	+* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
	+* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
	+* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
	+* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
	+* SOFTWARE.
	+*
	+*******************************************************************************/
	+/$FreeBSD$/
	+
	+#ifndef I40IW_PUDA_H
	+#define I40IW_PUDA_H
	+
	+#define I40IW_IEQ_MPA_FRAMING 6
	+
	+struct i40iw_sc_dev;
	+struct i40iw_sc_qp;
	+struct i40iw_sc_cq;
	+
	+enum puda_resource_type {
	+ I40IW_PUDA_RSRC_TYPE_ILQ = 1,
	+ I40IW_PUDA_RSRC_TYPE_IEQ
	+};
	+
	+enum puda_rsrc_complete {
	+ PUDA_CQ_CREATED = 1,
	+ PUDA_QP_CREATED,
	+ PUDA_TX_COMPLETE,
	+ PUDA_RX_COMPLETE,
	+ PUDA_HASH_CRC_COMPLETE
	+};
	+
	+struct i40iw_puda_completion_info {
	+ struct i40iw_qp_uk *qp;
	+ u8 q_type;
	+ u8 vlan_valid;
	+ u8 l3proto;
	+ u8 l4proto;
	+ u16 payload_len;
	+ u32 compl_error; /* No_err=0, else major and minor err code */
	+ u32 qp_id;
	+ u32 wqe_idx;
	+};
	+
	+struct i40iw_puda_send_info {
	+ u64 paddr; /* Physical address */
	+ u32 len;
	+ u8 tcplen;
	+ u8 maclen;
	+ bool ipv4;
	+ bool doloopback;
	+ void *scratch;
	+};
	+
	+struct i40iw_puda_buf {
	+ struct list_head list; /* MUST be first entry */
	+ struct i40iw_dma_mem mem; /* DMA memory for the buffer */
	+ struct i40iw_puda_buf next; / for alloclist in rsrc struct */
	+ struct i40iw_virt_mem buf_mem; /* Buffer memory for this buffer */
	+ void *scratch;
	+ u8 *iph;
	+ u8 *tcph;
	+ u8 *data;
	+ u16 datalen;
	+ u16 vlan_id;
	+ u8 tcphlen; /* tcp length in bytes */
	+ u8 maclen; /* mac length in bytes */
	+ u32 totallen; /* machlen+iphlen+tcphlen+datalen */
	+ ATOMIC refcount;
	+ u8 hdrlen;
	+ bool ipv4;
	+ u32 seqnum;
	+};
	+
	+struct i40iw_puda_rsrc_info {
	+ enum puda_resource_type type; /* ILQ or IEQ */
	+ u32 count;
	+ u16 pd_id;
	+ u32 cq_id;
	+ u32 qp_id;
	+ u32 sq_size;
	+ u32 rq_size;
	+ u16 buf_size;
	+ u16 mss;
	+ u32 tx_buf_cnt; /* total bufs allocated will be rq_size + tx_buf_cnt */
	+ void (receive)(struct i40iw_sc_vsi , struct i40iw_puda_buf *);
	+ void (xmit_complete)(struct i40iw_sc_vsi , void *);
	+};
	+
	+struct i40iw_puda_rsrc {
	+ struct i40iw_sc_cq cq;
	+ struct i40iw_sc_qp qp;
	+ struct i40iw_sc_pd sc_pd;
	+ struct i40iw_sc_dev *dev;
	+ struct i40iw_sc_vsi *vsi;
	+ struct i40iw_dma_mem cqmem;
	+ struct i40iw_dma_mem qpmem;
	+ struct i40iw_virt_mem ilq_mem;
	+ enum puda_rsrc_complete completion;
	+ enum puda_resource_type type;
	+ u16 buf_size; /buffer must be max datalen + tcpip hdr + mac /
	+ u32 cq_id;
	+ u32 qp_id;
	+ u32 sq_size;
	+ u32 rq_size;
	+ u32 cq_size;
	+ struct i40iw_sq_uk_wr_trk_info *sq_wrtrk_array;
	+ u64 *rq_wrid_array;
	+ u32 compl_rxwqe_idx;
	+ u32 rx_wqe_idx;
	+ u32 rxq_invalid_cnt;
	+ u32 tx_wqe_avail_cnt;
	+ bool check_crc;
	+ void *hash_desc;
	+ struct list_head txpend;
	+ struct list_head bufpool; /* free buffers pool list for recv and xmit */
	+ u32 alloc_buf_count;
	+ u32 avail_buf_count; /* snapshot of currently available buffers */
	+ spinlock_t bufpool_lock;
	+ struct i40iw_puda_buf *alloclist;
	+ void (receive)(struct i40iw_sc_vsi , struct i40iw_puda_buf *);
	+ void (xmit_complete)(struct i40iw_sc_vsi , void *);
	+ /* puda stats */
	+ u64 stats_buf_alloc_fail;
	+ u64 stats_pkt_rcvd;
	+ u64 stats_pkt_sent;
	+ u64 stats_rcvd_pkt_err;
	+ u64 stats_sent_pkt_q;
	+ u64 stats_bad_qp_id;
	+};
	+
	+struct i40iw_puda_buf i40iw_puda_get_bufpool(struct i40iw_puda_rsrc rsrc);
	+void i40iw_puda_ret_bufpool(struct i40iw_puda_rsrc *rsrc,
	+ struct i40iw_puda_buf *buf);
	+void i40iw_puda_send_buf(struct i40iw_puda_rsrc *rsrc,
	+ struct i40iw_puda_buf *buf);
	+enum i40iw_status_code i40iw_puda_send(struct i40iw_sc_qp *qp,
	+ struct i40iw_puda_send_info *info);
	+enum i40iw_status_code i40iw_puda_create_rsrc(struct i40iw_sc_vsi *vsi,
	+ struct i40iw_puda_rsrc_info *info);
	+void i40iw_puda_dele_resources(struct i40iw_sc_vsi *vsi,
	+ enum puda_resource_type type,
	+ bool reset);
	+enum i40iw_status_code i40iw_puda_poll_completion(struct i40iw_sc_dev *dev,
	+ struct i40iw_sc_cq cq, u32 compl_err);
	+
	+struct i40iw_sc_qp i40iw_ieq_get_qp(struct i40iw_sc_dev dev,
	+ struct i40iw_puda_buf *buf);
	+enum i40iw_status_code i40iw_puda_get_tcpip_info(struct i40iw_puda_completion_info *info,
	+ struct i40iw_puda_buf *buf);
	+enum i40iw_status_code i40iw_ieq_check_mpacrc(void *desc,
	+ void *addr, u32 length, u32 value);
	+enum i40iw_status_code i40iw_init_hash_desc(void **desc);
	+void i40iw_ieq_mpa_crc_ae(struct i40iw_sc_dev dev, struct i40iw_sc_qp qp);
	+void i40iw_free_hash_desc(void *desc);
	+void i40iw_ieq_update_tcpip_info(struct i40iw_puda_buf *buf, u16 length,
	+ u32 seqnum);
	+enum i40iw_status_code i40iw_cqp_qp_create_cmd(struct i40iw_sc_dev dev, struct i40iw_sc_qp qp);
	+enum i40iw_status_code i40iw_cqp_cq_create_cmd(struct i40iw_sc_dev dev, struct i40iw_sc_cq cq);
	+void i40iw_cqp_qp_destroy_cmd(struct i40iw_sc_dev dev, struct i40iw_sc_qp qp);
	+void i40iw_cqp_cq_destroy_cmd(struct i40iw_sc_dev dev, struct i40iw_sc_cq cq);
	+void i40iw_ieq_cleanup_qp(struct i40iw_puda_rsrc ieq, struct i40iw_sc_qp qp);
	+#endif
	Index: sys/dev/ixl/iwarp/i40iw_puda.c
	===================================================================
	--- /dev/null
	+++ sys/dev/ixl/iwarp/i40iw_puda.c
	@@ -0,0 +1,1513 @@
	+/*******************************************************************************
	+*
	+* Copyright (c) 2015-2017 Intel Corporation. All rights reserved.
	+*
	+* This software is available to you under a choice of one of two
	+* licenses. You may choose to be licensed under the terms of the GNU
	+* General Public License (GPL) Version 2, available from the file
	+* COPYING in the main directory of this source tree, or the
	+* OpenFabrics.org BSD license below:
	+*
	+* Redistribution and use in source and binary forms, with or
	+* without modification, are permitted provided that the following
	+* conditions are met:
	+*
	+* - Redistributions of source code must retain the above
	+* copyright notice, this list of conditions and the following
	+* disclaimer.
	+*
	+* - Redistributions in binary form must reproduce the above
	+* copyright notice, this list of conditions and the following
	+* disclaimer in the documentation and/or other materials
	+* provided with the distribution.
	+*
	+* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
	+* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
	+* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
	+* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
	+* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
	+* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
	+* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
	+* SOFTWARE.
	+*
	+*******************************************************************************/
	+/$FreeBSD$/
	+
	+#include "i40iw_osdep.h"
	+#include "i40iw_register.h"
	+#include "i40iw_status.h"
	+#include "i40iw_hmc.h"
	+
	+#include "i40iw_d.h"
	+#include "i40iw_type.h"
	+#include "i40iw_p.h"
	+#include "i40iw_puda.h"
	+
	+static void
	+i40iw_ieq_receive(struct i40iw_sc_vsi *vsi,
	+ struct i40iw_puda_buf *buf);
	+static void i40iw_ieq_tx_compl(struct i40iw_sc_vsi vsi, void sqwrid);
	+static void i40iw_ilq_putback_rcvbuf(struct i40iw_sc_qp *qp, u32 wqe_idx);
	+/**
	+ * i40iw_puda_get_listbuf - get buffer from puda list
	+ * @list: list to use for buffers (ILQ or IEQ)
	+ */
	+static struct i40iw_puda_buf *
	+i40iw_puda_get_listbuf(struct list_head *list)
	+{
	+ struct i40iw_puda_buf *buf = NULL;
	+
	+ if (!list_empty(list)) {
	+ buf = (struct i40iw_puda_buf *)list->next;
	+ list_del((struct list_head *)&buf->list);
	+ }
	+ return buf;
	+}
	+
	+/**
	+ * i40iw_puda_get_bufpool - return buffer from resource
	+ * @rsrc: resource to use for buffer
	+ */
	+struct i40iw_puda_buf *
	+i40iw_puda_get_bufpool(struct i40iw_puda_rsrc *rsrc)
	+{
	+ struct i40iw_puda_buf *buf = NULL;
	+ struct list_head *list = &rsrc->bufpool;
	+ unsigned long flags;
	+
	+ spin_lock_irqsave(&rsrc->bufpool_lock, flags);
	+ buf = i40iw_puda_get_listbuf(list);
	+ if (buf)
	+ rsrc->avail_buf_count--;
	+ else
	+ rsrc->stats_buf_alloc_fail++;
	+ spin_unlock_irqrestore(&rsrc->bufpool_lock, flags);
	+ return buf;
	+}
	+
	+/**
	+ * i40iw_puda_ret_bufpool - return buffer to rsrc list
	+ * @rsrc: resource to use for buffer
	+ * @buf: buffe to return to resouce
	+ */
	+void
	+i40iw_puda_ret_bufpool(struct i40iw_puda_rsrc *rsrc,
	+ struct i40iw_puda_buf *buf)
	+{
	+ unsigned long flags;
	+
	+ spin_lock_irqsave(&rsrc->bufpool_lock, flags);
	+ list_add(&buf->list, &rsrc->bufpool);
	+ spin_unlock_irqrestore(&rsrc->bufpool_lock, flags);
	+ rsrc->avail_buf_count++;
	+}
	+
	+/**
	+ * i40iw_puda_post_recvbuf - set wqe for rcv buffer
	+ * @rsrc: resource ptr
	+ * @wqe_idx: wqe index to use
	+ * @buf: puda buffer for rcv q
	+ * @initial: flag if during init time
	+ */
	+static void
	+i40iw_puda_post_recvbuf(struct i40iw_puda_rsrc *rsrc, u32 wqe_idx,
	+ struct i40iw_puda_buf *buf, bool initial)
	+{
	+ u64 *wqe;
	+ struct i40iw_sc_qp *qp = &rsrc->qp;
	+ u64 offset24 = 0;
	+
	+ qp->qp_uk.rq_wrid_array[wqe_idx] = (uintptr_t)buf;
	+ wqe = qp->qp_uk.rq_base[wqe_idx].elem;
	+ i40iw_debug(rsrc->dev, I40IW_DEBUG_PUDA,
	+ "%s: wqe_idx= %d buf = %p wqe = %p\n", __func__,
	+ wqe_idx, buf, wqe);
	+ if (!initial)
	+ get_64bit_val(wqe, I40IW_BYTE_24, &offset24);
	+
	+ offset24 = (offset24) ? 0 : LS_64(1, I40IWQPSQ_VALID);
	+
	+ set_64bit_val(wqe, 0, buf->mem.pa);
	+ set_64bit_val(wqe, I40IW_BYTE_8,
	+ LS_64(buf->mem.size, I40IWQPSQ_FRAG_LEN));
	+ i40iw_insert_wqe_hdr(wqe, offset24);
	+}
	+
	+/**
	+ * i40iw_puda_replenish_rq - post rcv buffers
	+ * @rsrc: resource to use for buffer
	+ * @initial: flag if during init time
	+ */
	+static enum i40iw_status_code
	+i40iw_puda_replenish_rq(struct i40iw_puda_rsrc *rsrc,
	+ bool initial)
	+{
	+ u32 i;
	+ u32 invalid_cnt = rsrc->rxq_invalid_cnt;
	+ struct i40iw_puda_buf *buf = NULL;
	+
	+ for (i = 0; i < invalid_cnt; i++) {
	+ buf = i40iw_puda_get_bufpool(rsrc);
	+ if (!buf)
	+ return I40IW_ERR_LIST_EMPTY;
	+ i40iw_puda_post_recvbuf(rsrc, rsrc->rx_wqe_idx, buf,
	+ initial);
	+ rsrc->rx_wqe_idx =
	+ ((rsrc->rx_wqe_idx + 1) % rsrc->rq_size);
	+ rsrc->rxq_invalid_cnt--;
	+ }
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_puda_alloc_buf - allocate mem for buffer
	+ * @dev: iwarp device
	+ * @length: length of buffer
	+ */
	+static struct i40iw_puda_buf *
	+i40iw_puda_alloc_buf(struct i40iw_sc_dev *dev,
	+ u32 length)
	+{
	+ struct i40iw_puda_buf *buf = NULL;
	+ struct i40iw_virt_mem buf_mem;
	+ enum i40iw_status_code ret;
	+
	+ ret = i40iw_allocate_virt_mem(dev->hw, &buf_mem,
	+ sizeof(struct i40iw_puda_buf));
	+ if (ret) {
	+ i40iw_debug(dev, I40IW_DEBUG_PUDA,
	+ "%s: error mem for buf\n", __func__);
	+ return NULL;
	+ }
	+ buf = (struct i40iw_puda_buf *)buf_mem.va;
	+ ret = i40iw_allocate_dma_mem(dev->hw, &buf->mem, length, 1);
	+ if (ret) {
	+ i40iw_debug(dev, I40IW_DEBUG_PUDA,
	+ "%s: error dma mem for buf\n", __func__);
	+ i40iw_free_virt_mem(dev->hw, &buf_mem);
	+ return NULL;
	+ }
	+ buf->buf_mem.va = buf_mem.va;
	+ buf->buf_mem.size = buf_mem.size;
	+ return buf;
	+}
	+
	+/**
	+ * i40iw_puda_dele_buf - delete buffer back to system
	+ * @dev: iwarp device
	+ * @buf: buffer to free
	+ */
	+static void
	+i40iw_puda_dele_buf(struct i40iw_sc_dev *dev,
	+ struct i40iw_puda_buf *buf)
	+{
	+ i40iw_free_dma_mem(dev->hw, &buf->mem);
	+ i40iw_free_virt_mem(dev->hw, &buf->buf_mem);
	+
	+}
	+
	+/**
	+ * i40iw_puda_get_next_send_wqe - return next wqe for processing
	+ * @qp: puda qp for wqe
	+ * @wqe_idx: wqe index for caller
	+ */
	+static u64 *
	+i40iw_puda_get_next_send_wqe(struct i40iw_qp_uk qp, u32 wqe_idx)
	+{
	+ u64 *wqe = NULL;
	+ enum i40iw_status_code ret_code = 0;
	+
	+ *wqe_idx = I40IW_RING_GETCURRENT_HEAD(qp->sq_ring);
	+ if (!*wqe_idx)
	+ qp->swqe_polarity = !qp->swqe_polarity;
	+ I40IW_RING_MOVE_HEAD(qp->sq_ring, ret_code);
	+ if (ret_code)
	+ return wqe;
	+ wqe = qp->sq_base[*wqe_idx].elem;
	+
	+ return wqe;
	+}
	+
	+/**
	+ * i40iw_puda_poll_info - poll cq for completion
	+ * @cq: cq for poll
	+ * @info: info return for successful completion
	+ */
	+static enum i40iw_status_code
	+i40iw_puda_poll_info(struct i40iw_sc_cq *cq,
	+ struct i40iw_puda_completion_info *info)
	+{
	+ u64 qword0, qword2, qword3;
	+ u64 *cqe;
	+ u64 comp_ctx;
	+ bool valid_bit;
	+ u32 major_err, minor_err;
	+ bool error;
	+
	+ cqe = (u64 *)I40IW_GET_CURRENT_CQ_ELEMENT(&cq->cq_uk);
	+ get_64bit_val(cqe, I40IW_BYTE_24, &qword3);
	+ valid_bit = (bool)RS_64(qword3, I40IW_CQ_VALID);
	+
	+ if (valid_bit != cq->cq_uk.polarity)
	+ return I40IW_ERR_QUEUE_EMPTY;
	+
	+ i40iw_debug_buf(cq->dev, I40IW_DEBUG_PUDA, "PUDA CQE", cqe, 32);
	+ error = (bool)RS_64(qword3, I40IW_CQ_ERROR);
	+ if (error) {
	+ i40iw_debug(cq->dev, I40IW_DEBUG_PUDA, "%s receive error\n", __func__);
	+ major_err = (u32)(RS_64(qword3, I40IW_CQ_MAJERR));
	+ minor_err = (u32)(RS_64(qword3, I40IW_CQ_MINERR));
	+ info->compl_error = major_err << 16 \| minor_err;
	+ return I40IW_ERR_CQ_COMPL_ERROR;
	+ }
	+ get_64bit_val(cqe, I40IW_BYTE_0, &qword0);
	+ get_64bit_val(cqe, I40IW_BYTE_16, &qword2);
	+
	+ info->q_type = (u8)RS_64(qword3, I40IW_CQ_SQ);
	+ info->qp_id = (u32)RS_64(qword2, I40IWCQ_QPID);
	+
	+ get_64bit_val(cqe, I40IW_BYTE_8, &comp_ctx);
	+ info->qp = (struct i40iw_qp_uk *)(i40iw_uintptr)comp_ctx;
	+ info->wqe_idx = (u32)RS_64(qword3, I40IW_CQ_WQEIDX);
	+
	+ if (info->q_type == I40IW_CQE_QTYPE_RQ) {
	+ info->vlan_valid = (bool)RS_64(qword3, I40IW_VLAN_TAG_VALID);
	+ info->l4proto = (u8)RS_64(qword2, I40IW_UDA_L4PROTO);
	+ info->l3proto = (u8)RS_64(qword2, I40IW_UDA_L3PROTO);
	+ info->payload_len = (u16)RS_64(qword0, I40IW_UDA_PAYLOADLEN);
	+ }
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_puda_poll_completion - processes completion for cq
	+ * @dev: iwarp device
	+ * @cq: cq getting interrupt
	+ * @compl_err: return any completion err
	+ */
	+enum i40iw_status_code
	+i40iw_puda_poll_completion(struct i40iw_sc_dev *dev,
	+ struct i40iw_sc_cq cq, u32 compl_err)
	+{
	+ struct i40iw_qp_uk *qp;
	+ struct i40iw_cq_uk *cq_uk = &cq->cq_uk;
	+ struct i40iw_puda_completion_info info;
	+ enum i40iw_status_code ret = 0;
	+ struct i40iw_puda_buf *buf;
	+ struct i40iw_puda_rsrc *rsrc;
	+ void *sqwrid;
	+ u8 cq_type = cq->cq_type;
	+ unsigned long flags;
	+
	+ if ((cq_type == I40IW_CQ_TYPE_ILQ) \|\| (cq_type == I40IW_CQ_TYPE_IEQ)) {
	+ rsrc = (cq_type == I40IW_CQ_TYPE_ILQ) ? cq->vsi->ilq : cq->vsi->ieq;
	+ } else {
	+ i40iw_debug(dev, I40IW_DEBUG_PUDA, "%s qp_type error\n", __func__);
	+ return I40IW_ERR_BAD_PTR;
	+ }
	+ memset(&info, 0, sizeof(info));
	+ ret = i40iw_puda_poll_info(cq, &info);
	+ *compl_err = info.compl_error;
	+ if (ret == I40IW_ERR_QUEUE_EMPTY)
	+ return ret;
	+ if (ret)
	+ goto done;
	+
	+ qp = info.qp;
	+ if (!qp \|\| !rsrc) {
	+ ret = I40IW_ERR_BAD_PTR;
	+ goto done;
	+ }
	+ if (qp->qp_id != rsrc->qp_id) {
	+ ret = I40IW_ERR_BAD_PTR;
	+ goto done;
	+ }
	+ if (info.q_type == I40IW_CQE_QTYPE_RQ) {
	+ buf = (struct i40iw_puda_buf *)(uintptr_t)qp->rq_wrid_array[info.wqe_idx];
	+ /* Get all the tcpip information in the buf header */
	+ ret = i40iw_puda_get_tcpip_info(&info, buf);
	+ if (ret) {
	+ rsrc->stats_rcvd_pkt_err++;
	+ if (cq_type == I40IW_CQ_TYPE_ILQ) {
	+ i40iw_ilq_putback_rcvbuf(&rsrc->qp,
	+ info.wqe_idx);
	+ } else {
	+ i40iw_puda_ret_bufpool(rsrc, buf);
	+ i40iw_puda_replenish_rq(rsrc, FALSE);
	+ }
	+ goto done;
	+ }
	+ rsrc->stats_pkt_rcvd++;
	+ rsrc->compl_rxwqe_idx = info.wqe_idx;
	+ i40iw_debug(dev, I40IW_DEBUG_PUDA, "%s RQ completion\n", __func__);
	+ rsrc->receive(rsrc->vsi, buf);
	+ if (cq_type == I40IW_CQ_TYPE_ILQ)
	+ i40iw_ilq_putback_rcvbuf(&rsrc->qp, info.wqe_idx);
	+ else
	+ i40iw_puda_replenish_rq(rsrc, FALSE);
	+
	+ } else {
	+ i40iw_debug(dev, I40IW_DEBUG_PUDA, "%s SQ completion\n", __func__);
	+ sqwrid = (void *)(uintptr_t)qp->sq_wrtrk_array[info.wqe_idx].wrid;
	+ I40IW_RING_SET_TAIL(qp->sq_ring, info.wqe_idx);
	+ rsrc->xmit_complete(rsrc->vsi, sqwrid);
	+ spin_lock_irqsave(&rsrc->bufpool_lock, flags);
	+ rsrc->tx_wqe_avail_cnt++;
	+ spin_unlock_irqrestore(&rsrc->bufpool_lock, flags);
	+ if (!list_empty(&rsrc->vsi->ilq->txpend))
	+ i40iw_puda_send_buf(rsrc->vsi->ilq, NULL);
	+ }
	+
	+done:
	+ I40IW_RING_MOVE_HEAD(cq_uk->cq_ring, ret);
	+ if (I40IW_RING_GETCURRENT_HEAD(cq_uk->cq_ring) == 0)
	+ cq_uk->polarity = !cq_uk->polarity;
	+ /* update cq tail in cq shadow memory also */
	+ I40IW_RING_MOVE_TAIL(cq_uk->cq_ring);
	+ set_64bit_val(cq_uk->shadow_area, I40IW_BYTE_0,
	+ I40IW_RING_GETCURRENT_HEAD(cq_uk->cq_ring));
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_puda_send - complete send wqe for transmit
	+ * @qp: puda qp for send
	+ * @info: buffer information for transmit
	+ */
	+enum i40iw_status_code
	+i40iw_puda_send(struct i40iw_sc_qp *qp,
	+ struct i40iw_puda_send_info *info)
	+{
	+ u64 *wqe;
	+ u32 iplen, l4len;
	+ u64 header[2];
	+ u32 wqe_idx;
	+ u8 iipt;
	+
	+ /* number of 32 bits DWORDS in header */
	+ l4len = info->tcplen >> 2;
	+ if (info->ipv4) {
	+ iipt = 3;
	+ iplen = 5;
	+ } else {
	+ iipt = 1;
	+ iplen = 10;
	+ }
	+
	+ wqe = i40iw_puda_get_next_send_wqe(&qp->qp_uk, &wqe_idx);
	+ if (!wqe)
	+ return I40IW_ERR_QP_TOOMANY_WRS_POSTED;
	+ qp->qp_uk.sq_wrtrk_array[wqe_idx].wrid = (uintptr_t)info->scratch;
	+ /* Third line of WQE descriptor */
	+ /* maclen is in words */
	+ header[0] = LS_64((info->maclen >> 1), I40IW_UDA_QPSQ_MACLEN) \|
	+ LS_64(iplen, I40IW_UDA_QPSQ_IPLEN) \| LS_64(1, I40IW_UDA_QPSQ_L4T) \|
	+ LS_64(iipt, I40IW_UDA_QPSQ_IIPT) \|
	+ LS_64(l4len, I40IW_UDA_QPSQ_L4LEN);
	+ /* Forth line of WQE descriptor */
	+ header[1] = LS_64(I40IW_OP_TYPE_SEND, I40IW_UDA_QPSQ_OPCODE) \|
	+ LS_64(1, I40IW_UDA_QPSQ_SIGCOMPL) \|
	+ LS_64(info->doloopback, I40IW_UDA_QPSQ_DOLOOPBACK) \|
	+ LS_64(qp->qp_uk.swqe_polarity, I40IW_UDA_QPSQ_VALID);
	+
	+ set_64bit_val(wqe, I40IW_BYTE_0, info->paddr);
	+ set_64bit_val(wqe, I40IW_BYTE_8, LS_64(info->len, I40IWQPSQ_FRAG_LEN));
	+ set_64bit_val(wqe, I40IW_BYTE_16, header[0]);
	+
	+ i40iw_insert_wqe_hdr(wqe, header[1]);
	+
	+ i40iw_debug_buf(qp->dev, I40IW_DEBUG_PUDA, "PUDA SEND WQE", wqe, 32);
	+ i40iw_qp_post_wr(&qp->qp_uk);
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_puda_send_buf - transmit puda buffer
	+ * @rsrc: resource to use for buffer
	+ * @buf: puda buffer to transmit
	+ */
	+void
	+i40iw_puda_send_buf(struct i40iw_puda_rsrc rsrc, struct i40iw_puda_buf buf)
	+{
	+ struct i40iw_puda_send_info info;
	+ enum i40iw_status_code ret = 0;
	+ unsigned long flags;
	+
	+ spin_lock_irqsave(&rsrc->bufpool_lock, flags);
	+ /*
	+ * if no wqe available or not from a completion and we have pending
	+ * buffers, we must queue new buffer
	+ */
	+ if (!rsrc->tx_wqe_avail_cnt \|\| (buf && !list_empty(&rsrc->txpend))) {
	+ list_add_tail(&buf->list, &rsrc->txpend);
	+ spin_unlock_irqrestore(&rsrc->bufpool_lock, flags);
	+ rsrc->stats_sent_pkt_q++;
	+ if (rsrc->type == I40IW_PUDA_RSRC_TYPE_ILQ)
	+ i40iw_debug(rsrc->dev, I40IW_DEBUG_PUDA,
	+ "%s: adding to txpend\n", __func__);
	+ return;
	+ }
	+ rsrc->tx_wqe_avail_cnt--;
	+ /*
	+ * if we are coming from a completion and have pending buffers then
	+ * Get one from pending list
	+ */
	+ if (!buf) {
	+ buf = i40iw_puda_get_listbuf(&rsrc->txpend);
	+ if (!buf)
	+ goto done;
	+ }
	+ info.scratch = (void *)buf;
	+ info.paddr = buf->mem.pa;
	+ info.len = buf->totallen;
	+ info.tcplen = buf->tcphlen;
	+ info.maclen = buf->maclen;
	+ info.ipv4 = buf->ipv4;
	+ info.doloopback = (rsrc->type == I40IW_PUDA_RSRC_TYPE_IEQ);
	+
	+ ret = i40iw_puda_send(&rsrc->qp, &info);
	+ if (ret) {
	+ rsrc->tx_wqe_avail_cnt++;
	+ rsrc->stats_sent_pkt_q++;
	+ list_add(&buf->list, &rsrc->txpend);
	+ if (rsrc->type == I40IW_PUDA_RSRC_TYPE_ILQ)
	+ i40iw_debug(rsrc->dev, I40IW_DEBUG_PUDA,
	+ "%s: adding to puda_send\n", __func__);
	+ } else {
	+ rsrc->stats_pkt_sent++;
	+ }
	+done:
	+ spin_unlock_irqrestore(&rsrc->bufpool_lock, flags);
	+}
	+
	+/**
	+ * i40iw_puda_qp_setctx - during init, set qp's context
	+ * @rsrc: qp's resource
	+ */
	+static void
	+i40iw_puda_qp_setctx(struct i40iw_puda_rsrc *rsrc)
	+{
	+ struct i40iw_sc_qp *qp = &rsrc->qp;
	+ u64 *qp_ctx = qp->hw_host_ctx;
	+
	+ set_64bit_val(qp_ctx, I40IW_BYTE_8, qp->sq_pa);
	+ set_64bit_val(qp_ctx, I40IW_BYTE_16, qp->rq_pa);
	+
	+ set_64bit_val(qp_ctx, I40IW_BYTE_24,
	+ LS_64(qp->hw_rq_size, I40IWQPC_RQSIZE) \|
	+ LS_64(qp->hw_sq_size, I40IWQPC_SQSIZE));
	+
	+ set_64bit_val(qp_ctx, I40IW_BYTE_48, LS_64(rsrc->buf_size, I40IWQPC_MAXFRAMESIZE));
	+ set_64bit_val(qp_ctx, I40IW_BYTE_56, 0);
	+ set_64bit_val(qp_ctx, I40IW_BYTE_64, 1);
	+
	+ set_64bit_val(qp_ctx, I40IW_BYTE_136,
	+ LS_64(rsrc->cq_id, I40IWQPC_TXCQNUM) \|
	+ LS_64(rsrc->cq_id, I40IWQPC_RXCQNUM));
	+
	+ set_64bit_val(qp_ctx, I40IW_BYTE_160, LS_64(1, I40IWQPC_PRIVEN));
	+
	+ set_64bit_val(qp_ctx, I40IW_BYTE_168,
	+ LS_64((uintptr_t)qp, I40IWQPC_QPCOMPCTX));
	+
	+ set_64bit_val(qp_ctx, I40IW_BYTE_176,
	+ LS_64(qp->sq_tph_val, I40IWQPC_SQTPHVAL) \|
	+ LS_64(qp->rq_tph_val, I40IWQPC_RQTPHVAL) \|
	+ LS_64(qp->qs_handle, I40IWQPC_QSHANDLE));
	+
	+ i40iw_debug_buf(rsrc->dev, I40IW_DEBUG_PUDA, "PUDA QP CONTEXT",
	+ qp_ctx, I40IW_QP_CTX_SIZE);
	+}
	+
	+/**
	+ * i40iw_puda_qp_wqe - setup wqe for qp create
	+ * @rsrc: resource for qp
	+ */
	+static enum i40iw_status_code
	+i40iw_puda_qp_wqe(struct i40iw_sc_dev dev, struct i40iw_sc_qp qp)
	+{
	+ struct i40iw_sc_cqp *cqp;
	+ u64 *wqe;
	+ u64 header;
	+ struct i40iw_ccq_cqe_info compl_info;
	+ enum i40iw_status_code status = 0;
	+
	+ cqp = dev->cqp;
	+ wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, 0);
	+ if (!wqe)
	+ return I40IW_ERR_RING_FULL;
	+
	+ set_64bit_val(wqe, I40IW_BYTE_16, qp->hw_host_ctx_pa);
	+ set_64bit_val(wqe, I40IW_BYTE_40, qp->shadow_area_pa);
	+ header = qp->qp_uk.qp_id \|
	+ LS_64(I40IW_CQP_OP_CREATE_QP, I40IW_CQPSQ_OPCODE) \|
	+ LS_64(I40IW_QP_TYPE_UDA, I40IW_CQPSQ_QP_QPTYPE) \|
	+ LS_64(1, I40IW_CQPSQ_QP_CQNUMVALID) \|
	+ LS_64(2, I40IW_CQPSQ_QP_NEXTIWSTATE) \|
	+ LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
	+
	+ i40iw_insert_wqe_hdr(wqe, header);
	+
	+ i40iw_debug_buf(cqp->dev, I40IW_DEBUG_PUDA, "PUDA CQE", wqe, 32);
	+ i40iw_sc_cqp_post_sq(cqp);
	+ status = dev->cqp_ops->poll_for_cqp_op_done(dev->cqp,
	+ I40IW_CQP_OP_CREATE_QP,
	+ &compl_info);
	+ return status;
	+}
	+
	+/**
	+ * i40iw_puda_qp_create - create qp for resource
	+ * @rsrc: resource to use for buffer
	+ */
	+static enum i40iw_status_code
	+i40iw_puda_qp_create(struct i40iw_puda_rsrc *rsrc)
	+{
	+ struct i40iw_sc_qp *qp = &rsrc->qp;
	+ struct i40iw_qp_uk *ukqp = &qp->qp_uk;
	+ enum i40iw_status_code ret = 0;
	+ u32 sq_size, rq_size, t_size;
	+ struct i40iw_dma_mem *mem;
	+
	+ sq_size = rsrc->sq_size * I40IW_QP_WQE_MIN_SIZE;
	+ rq_size = rsrc->rq_size * I40IW_QP_WQE_MIN_SIZE;
	+ t_size = (sq_size + rq_size + (I40IW_SHADOW_AREA_SIZE << 3) +
	+ I40IW_QP_CTX_SIZE);
	+ /* Get page aligned memory */
	+ ret =
	+ i40iw_allocate_dma_mem(rsrc->dev->hw, &rsrc->qpmem, t_size,
	+ I40IW_HW_PAGE_SIZE);
	+ if (ret) {
	+ i40iw_debug(rsrc->dev, I40IW_DEBUG_PUDA, "%s: error dma mem\n", __func__);
	+ return ret;
	+ }
	+ mem = &rsrc->qpmem;
	+ memset(mem->va, 0, t_size);
	+ qp->hw_sq_size = i40iw_get_encoded_wqe_size(rsrc->sq_size, FALSE);
	+ qp->hw_rq_size = i40iw_get_encoded_wqe_size(rsrc->rq_size, FALSE);
	+ qp->pd = &rsrc->sc_pd;
	+ qp->qp_type = I40IW_QP_TYPE_UDA;
	+ qp->dev = rsrc->dev;
	+ qp->back_qp = (void *)rsrc;
	+ qp->sq_pa = mem->pa;
	+ qp->rq_pa = qp->sq_pa + sq_size;
	+ qp->vsi = rsrc->vsi;
	+ ukqp->sq_base = mem->va;
	+ ukqp->rq_base = &ukqp->sq_base[rsrc->sq_size];
	+ ukqp->shadow_area = ukqp->rq_base[rsrc->rq_size].elem;
	+ qp->shadow_area_pa = qp->rq_pa + rq_size;
	+ qp->hw_host_ctx = ukqp->shadow_area + I40IW_SHADOW_AREA_SIZE;
	+ qp->hw_host_ctx_pa =
	+ qp->shadow_area_pa + (I40IW_SHADOW_AREA_SIZE << 3);
	+ ukqp->qp_id = rsrc->qp_id;
	+ ukqp->sq_wrtrk_array = rsrc->sq_wrtrk_array;
	+ ukqp->rq_wrid_array = rsrc->rq_wrid_array;
	+
	+ ukqp->qp_id = rsrc->qp_id;
	+ ukqp->sq_size = rsrc->sq_size;
	+ ukqp->rq_size = rsrc->rq_size;
	+
	+ I40IW_RING_INIT(ukqp->sq_ring, ukqp->sq_size);
	+ I40IW_RING_INIT(ukqp->initial_ring, ukqp->sq_size);
	+ I40IW_RING_INIT(ukqp->rq_ring, ukqp->rq_size);
	+
	+ if (qp->pd->dev->is_pf)
	+ ukqp->wqe_alloc_reg = (u32 IOMEM *) (i40iw_get_hw_addr(qp->pd->dev) +
	+ I40E_PFPE_WQEALLOC);
	+ else
	+ ukqp->wqe_alloc_reg = (u32 IOMEM *) (i40iw_get_hw_addr(qp->pd->dev) +
	+ I40E_VFPE_WQEALLOC1);
	+
	+ qp->qs_handle = qp->vsi->qos[0].qs_handle;
	+ i40iw_puda_qp_setctx(rsrc);
	+ if (rsrc->dev->ceq_valid)
	+ ret = i40iw_cqp_qp_create_cmd(rsrc->dev, qp);
	+ else
	+ ret = i40iw_puda_qp_wqe(rsrc->dev, qp);
	+ if (ret)
	+ i40iw_free_dma_mem(rsrc->dev->hw, &rsrc->qpmem);
	+ return ret;
	+}
	+
	+/**
	+ * i40iw_puda_cq_wqe - setup wqe for cq create
	+ * @rsrc: resource for cq
	+ */
	+static enum i40iw_status_code
	+i40iw_puda_cq_wqe(struct i40iw_sc_dev dev, struct i40iw_sc_cq cq)
	+{
	+ u64 *wqe;
	+ struct i40iw_sc_cqp *cqp;
	+ u64 header;
	+ struct i40iw_ccq_cqe_info compl_info;
	+ enum i40iw_status_code status = 0;
	+
	+ cqp = dev->cqp;
	+ wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, 0);
	+ if (!wqe)
	+ return I40IW_ERR_RING_FULL;
	+
	+ set_64bit_val(wqe, I40IW_BYTE_0, cq->cq_uk.cq_size);
	+ set_64bit_val(wqe, I40IW_BYTE_8, RS_64_1(cq, 1));
	+ set_64bit_val(wqe, I40IW_BYTE_16,
	+ LS_64(cq->shadow_read_threshold,
	+ I40IW_CQPSQ_CQ_SHADOW_READ_THRESHOLD));
	+ set_64bit_val(wqe, I40IW_BYTE_32, cq->cq_pa);
	+
	+ set_64bit_val(wqe, I40IW_BYTE_40, cq->shadow_area_pa);
	+
	+ header = cq->cq_uk.cq_id \|
	+ LS_64(I40IW_CQP_OP_CREATE_CQ, I40IW_CQPSQ_OPCODE) \|
	+ LS_64(1, I40IW_CQPSQ_CQ_CHKOVERFLOW) \|
	+ LS_64(1, I40IW_CQPSQ_CQ_ENCEQEMASK) \|
	+ LS_64(1, I40IW_CQPSQ_CQ_CEQIDVALID) \|
	+ LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
	+ i40iw_insert_wqe_hdr(wqe, header);
	+
	+ i40iw_debug_buf(dev, I40IW_DEBUG_PUDA, "PUDA CQE",
	+ wqe, I40IW_CQP_WQE_SIZE * 8);
	+
	+ i40iw_sc_cqp_post_sq(dev->cqp);
	+ status = dev->cqp_ops->poll_for_cqp_op_done(dev->cqp,
	+ I40IW_CQP_OP_CREATE_CQ,
	+ &compl_info);
	+ return status;
	+}
	+
	+/**
	+ * i40iw_puda_cq_create - create cq for resource
	+ * @rsrc: resource for which cq to create
	+ */
	+static enum i40iw_status_code
	+i40iw_puda_cq_create(struct i40iw_puda_rsrc *rsrc)
	+{
	+ struct i40iw_sc_dev *dev = rsrc->dev;
	+ struct i40iw_sc_cq *cq = &rsrc->cq;
	+ enum i40iw_status_code ret = 0;
	+ u32 tsize, cqsize;
	+ struct i40iw_dma_mem *mem;
	+ struct i40iw_cq_init_info info;
	+ struct i40iw_cq_uk_init_info *init_info = &info.cq_uk_init_info;
	+
	+ cq->vsi = rsrc->vsi;
	+ cqsize = rsrc->cq_size * (sizeof(struct i40iw_cqe));
	+ tsize = cqsize + sizeof(struct i40iw_cq_shadow_area);
	+ ret = i40iw_allocate_dma_mem(dev->hw, &rsrc->cqmem, tsize,
	+ I40IW_CQ0_ALIGNMENT);
	+ if (ret)
	+ return ret;
	+
	+ mem = &rsrc->cqmem;
	+ memset(&info, 0, sizeof(info));
	+ info.dev = dev;
	+ info.type = (rsrc->type == I40IW_PUDA_RSRC_TYPE_ILQ) ?
	+ I40IW_CQ_TYPE_ILQ : I40IW_CQ_TYPE_IEQ;
	+ info.shadow_read_threshold = rsrc->cq_size >> 2;
	+ info.cq_base_pa = mem->pa;
	+ info.shadow_area_pa = mem->pa + cqsize;
	+ init_info->cq_base = mem->va;
	+ init_info->shadow_area = (u64 )((u8 )mem->va + cqsize);
	+ init_info->cq_size = rsrc->cq_size;
	+ init_info->cq_id = rsrc->cq_id;
	+ info.ceqe_mask = TRUE;
	+ info.ceq_id_valid = TRUE;
	+
	+ ret = dev->iw_priv_cq_ops->cq_init(cq, &info);
	+ if (ret)
	+ goto error;
	+ if (rsrc->dev->ceq_valid)
	+ ret = i40iw_cqp_cq_create_cmd(dev, cq);
	+ else
	+ ret = i40iw_puda_cq_wqe(dev, cq);
	+error:
	+ if (ret)
	+ i40iw_free_dma_mem(dev->hw, &rsrc->cqmem);
	+ return ret;
	+}
	+
	+/**
	+ * i40iw_puda_free_qp - free qp for resource
	+ * @rsrc: resource for which qp to free
	+ */
	+static void
	+i40iw_puda_free_qp(struct i40iw_puda_rsrc *rsrc)
	+{
	+ enum i40iw_status_code ret;
	+ struct i40iw_ccq_cqe_info compl_info;
	+ struct i40iw_sc_dev *dev = rsrc->dev;
	+
	+ if (rsrc->dev->ceq_valid) {
	+ i40iw_cqp_qp_destroy_cmd(dev, &rsrc->qp);
	+ return;
	+ }
	+ ret = dev->iw_priv_qp_ops->qp_destroy(&rsrc->qp,
	+ 0, FALSE, TRUE, true);
	+ if (ret)
	+ i40iw_debug(dev, I40IW_DEBUG_PUDA,
	+ "%s error puda qp destroy wqe\n",
	+ __func__);
	+
	+ if (!ret) {
	+ ret = dev->cqp_ops->poll_for_cqp_op_done(dev->cqp,
	+ I40IW_CQP_OP_DESTROY_QP,
	+ &compl_info);
	+ if (ret)
	+ i40iw_debug(dev, I40IW_DEBUG_PUDA,
	+ "%s error puda qp destroy failed\n",
	+ __func__);
	+ }
	+}
	+
	+/**
	+ * i40iw_puda_free_cq - free cq for resource
	+ * @rsrc: resource for which cq to free
	+ */
	+static void
	+i40iw_puda_free_cq(struct i40iw_puda_rsrc *rsrc)
	+{
	+ enum i40iw_status_code ret;
	+ struct i40iw_ccq_cqe_info compl_info;
	+ struct i40iw_sc_dev *dev = rsrc->dev;
	+
	+ if (rsrc->dev->ceq_valid) {
	+ i40iw_cqp_cq_destroy_cmd(dev, &rsrc->cq);
	+ return;
	+ }
	+ ret = dev->iw_priv_cq_ops->cq_destroy(&rsrc->cq, 0, TRUE);
	+ if (ret)
	+ i40iw_debug(dev, I40IW_DEBUG_PUDA,
	+ "%s error ieq cq destroy\n",
	+ __func__);
	+
	+ if (!ret) {
	+ ret = dev->cqp_ops->poll_for_cqp_op_done(dev->cqp,
	+ I40IW_CQP_OP_DESTROY_CQ,
	+ &compl_info);
	+ if (ret)
	+ i40iw_debug(dev, I40IW_DEBUG_PUDA,
	+ "%s error ieq qp destroy done\n",
	+ __func__);
	+ }
	+}
	+
	+/**
	+ * i40iw_puda_dele_resources - delete all resources during close
	+ * @dev: iwarp device
	+ * @type: type of resource to dele
	+ * @reset: TRUE if reset chip
	+ */
	+void
	+i40iw_puda_dele_resources(struct i40iw_sc_vsi *vsi,
	+ enum puda_resource_type type,
	+ bool reset)
	+{
	+ struct i40iw_sc_dev *dev = vsi->dev;
	+ struct i40iw_puda_rsrc *rsrc;
	+ struct i40iw_puda_buf *buf = NULL;
	+ struct i40iw_puda_buf *nextbuf = NULL;
	+ struct i40iw_virt_mem *vmem;
	+ switch (type) {
	+ case I40IW_PUDA_RSRC_TYPE_ILQ:
	+ rsrc = vsi->ilq;
	+ vmem = &vsi->ilq_mem;
	+ break;
	+ case I40IW_PUDA_RSRC_TYPE_IEQ:
	+ rsrc = vsi->ieq;
	+ vmem = &vsi->ieq_mem;
	+ break;
	+ default:
	+ i40iw_debug(dev, I40IW_DEBUG_PUDA, "%s: error resource type = 0x%x\n",
	+ __func__, type);
	+ return;
	+ }
	+
	+ switch (rsrc->completion) {
	+ case PUDA_HASH_CRC_COMPLETE:
	+ i40iw_free_hash_desc(rsrc->hash_desc);
	+ case PUDA_QP_CREATED:
	+ if (!reset)
	+ i40iw_puda_free_qp(rsrc);
	+
	+ i40iw_free_dma_mem(dev->hw, &rsrc->qpmem);
	+ /* fallthrough */
	+ case PUDA_CQ_CREATED:
	+ if (!reset)
	+ i40iw_puda_free_cq(rsrc);
	+
	+ i40iw_free_dma_mem(dev->hw, &rsrc->cqmem);
	+ break;
	+ default:
	+ i40iw_debug(rsrc->dev, I40IW_DEBUG_PUDA, "%s error no resources\n", __func__);
	+ break;
	+ }
	+ /* Free all allocated puda buffers for both tx and rx */
	+ buf = rsrc->alloclist;
	+ while (buf) {
	+ nextbuf = buf->next;
	+ i40iw_puda_dele_buf(dev, buf);
	+ buf = nextbuf;
	+ rsrc->alloc_buf_count--;
	+ }
	+ i40iw_free_virt_mem(dev->hw, vmem);
	+}
	+
	+/**
	+ * i40iw_puda_allocbufs - allocate buffers for resource
	+ * @rsrc: resource for buffer allocation
	+ * @count: number of buffers to create
	+ */
	+static enum i40iw_status_code
	+i40iw_puda_allocbufs(struct i40iw_puda_rsrc *rsrc,
	+ u32 count)
	+{
	+ u32 i;
	+ struct i40iw_puda_buf *buf;
	+ struct i40iw_puda_buf *nextbuf;
	+
	+ for (i = 0; i < count; i++) {
	+ buf = i40iw_puda_alloc_buf(rsrc->dev, rsrc->buf_size);
	+ if (!buf) {
	+ rsrc->stats_buf_alloc_fail++;
	+ return I40IW_ERR_NO_MEMORY;
	+ }
	+ i40iw_puda_ret_bufpool(rsrc, buf);
	+ rsrc->alloc_buf_count++;
	+ if (!rsrc->alloclist) {
	+ rsrc->alloclist = buf;
	+ } else {
	+ nextbuf = rsrc->alloclist;
	+ rsrc->alloclist = buf;
	+ buf->next = nextbuf;
	+ }
	+ }
	+ rsrc->avail_buf_count = rsrc->alloc_buf_count;
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_puda_create_rsrc - create resouce (ilq or ieq)
	+ * @dev: iwarp device
	+ * @info: resource information
	+ */
	+enum i40iw_status_code
	+i40iw_puda_create_rsrc(struct i40iw_sc_vsi *vsi,
	+ struct i40iw_puda_rsrc_info *info)
	+{
	+ struct i40iw_sc_dev *dev = vsi->dev;
	+ enum i40iw_status_code ret = 0;
	+ struct i40iw_puda_rsrc *rsrc;
	+ u32 pudasize;
	+ u32 sqwridsize, rqwridsize;
	+ struct i40iw_virt_mem *vmem;
	+
	+ info->count = 1;
	+ pudasize = sizeof(struct i40iw_puda_rsrc);
	+ sqwridsize = info->sq_size * sizeof(struct i40iw_sq_uk_wr_trk_info);
	+ rqwridsize = info->rq_size * 8;
	+ switch (info->type) {
	+ case I40IW_PUDA_RSRC_TYPE_ILQ:
	+ vmem = &vsi->ilq_mem;
	+ break;
	+ case I40IW_PUDA_RSRC_TYPE_IEQ:
	+ vmem = &vsi->ieq_mem;
	+ break;
	+ default:
	+ return I40IW_NOT_SUPPORTED;
	+ }
	+ ret =
	+ i40iw_allocate_virt_mem(dev->hw, vmem,
	+ pudasize + sqwridsize + rqwridsize);
	+ if (ret)
	+ return ret;
	+ rsrc = (struct i40iw_puda_rsrc *)vmem->va;
	+ spin_lock_init(&rsrc->bufpool_lock);
	+ if (info->type == I40IW_PUDA_RSRC_TYPE_ILQ) {
	+ vsi->ilq = (struct i40iw_puda_rsrc *)vmem->va;
	+ vsi->ilq_count = info->count;
	+ rsrc->receive = info->receive;
	+ rsrc->xmit_complete = info->xmit_complete;
	+ } else {
	+ vmem = &vsi->ieq_mem;
	+ vsi->ieq_count = info->count;
	+ vsi->ieq = (struct i40iw_puda_rsrc *)vmem->va;
	+ rsrc->receive = i40iw_ieq_receive;
	+ rsrc->xmit_complete = i40iw_ieq_tx_compl;
	+ }
	+
	+ rsrc->type = info->type;
	+ rsrc->sq_wrtrk_array = (struct i40iw_sq_uk_wr_trk_info )((u8 )vmem->va + pudasize);
	+ rsrc->rq_wrid_array = (u64 )((u8 )vmem->va + pudasize + sqwridsize);
	+ /* Initialize all ieq lists */
	+ INIT_LIST_HEAD(&rsrc->bufpool);
	+ INIT_LIST_HEAD(&rsrc->txpend);
	+
	+ rsrc->tx_wqe_avail_cnt = info->sq_size - 1;
	+ dev->iw_pd_ops->pd_init(dev, &rsrc->sc_pd, info->pd_id);
	+ rsrc->qp_id = info->qp_id;
	+ rsrc->cq_id = info->cq_id;
	+ rsrc->sq_size = info->sq_size;
	+ rsrc->rq_size = info->rq_size;
	+ rsrc->cq_size = info->rq_size + info->sq_size;
	+ rsrc->buf_size = info->buf_size;
	+ rsrc->dev = dev;
	+ rsrc->vsi = vsi;
	+
	+ ret = i40iw_puda_cq_create(rsrc);
	+ if (!ret) {
	+ rsrc->completion = PUDA_CQ_CREATED;
	+ ret = i40iw_puda_qp_create(rsrc);
	+ }
	+ if (ret) {
	+ i40iw_debug(dev, I40IW_DEBUG_PUDA, "[%s] error qp_create\n", __func__);
	+ goto error;
	+ }
	+ rsrc->completion = PUDA_QP_CREATED;
	+
	+ ret = i40iw_puda_allocbufs(rsrc, info->tx_buf_cnt + info->rq_size);
	+ if (ret) {
	+ i40iw_debug(dev, I40IW_DEBUG_PUDA, "[%s] error allloc_buf\n", __func__);
	+ goto error;
	+ }
	+ rsrc->rxq_invalid_cnt = info->rq_size;
	+ ret = i40iw_puda_replenish_rq(rsrc, TRUE);
	+ if (ret)
	+ goto error;
	+
	+ if (info->type == I40IW_PUDA_RSRC_TYPE_IEQ) {
	+ if (!i40iw_init_hash_desc(&rsrc->hash_desc)) {
	+ rsrc->check_crc = TRUE;
	+ rsrc->completion = PUDA_HASH_CRC_COMPLETE;
	+ ret = 0;
	+ }
	+ }
	+ dev->ccq_ops->ccq_arm(&rsrc->cq);
	+ return ret;
	+error:
	+
	+ i40iw_puda_dele_resources(vsi, info->type, FALSE);
	+
	+ return ret;
	+}
	+
	+/**
	+ * i40iw_ilq_putback_rcvbuf - ilq buffer to put back on rq
	+ * @qp: ilq's qp resource
	+ * @wqe_idx: wqe index of completed rcvbuf
	+ */
	+static void
	+i40iw_ilq_putback_rcvbuf(struct i40iw_sc_qp *qp, u32 wqe_idx)
	+{
	+ u64 *wqe;
	+ u64 offset24;
	+
	+ wqe = qp->qp_uk.rq_base[wqe_idx].elem;
	+ get_64bit_val(wqe, I40IW_BYTE_24, &offset24);
	+ offset24 = (offset24) ? 0 : LS_64(1, I40IWQPSQ_VALID);
	+ set_64bit_val(wqe, I40IW_BYTE_24, offset24);
	+}
	+
	+/**
	+ * i40iw_ieq_get_fpdu - given length return fpdu length
	+ * @length: length if fpdu
	+ */
	+static u16
	+i40iw_ieq_get_fpdu_length(u16 length)
	+{
	+ u16 fpdu_len;
	+
	+ fpdu_len = length + I40IW_IEQ_MPA_FRAMING;
	+ fpdu_len = (fpdu_len + 3) & 0xfffffffc;
	+ return fpdu_len;
	+}
	+
	+/**
	+ * i40iw_ieq_copy_to_txbuf - copydata from rcv buf to tx buf
	+ * @buf: rcv buffer with partial
	+ * @txbuf: tx buffer for sendign back
	+ * @buf_offset: rcv buffer offset to copy from
	+ * @txbuf_offset: at offset in tx buf to copy
	+ * @length: length of data to copy
	+ */
	+static void
	+i40iw_ieq_copy_to_txbuf(struct i40iw_puda_buf *buf,
	+ struct i40iw_puda_buf *txbuf,
	+ u16 buf_offset, u32 txbuf_offset,
	+ u32 length)
	+{
	+ void mem1 = (u8 )buf->mem.va + buf_offset;
	+ void mem2 = (u8 )txbuf->mem.va + txbuf_offset;
	+
	+ i40iw_memcpy(mem2, mem1, length);
	+}
	+
	+/**
	+ * i40iw_ieq_setup_tx_buf - setup tx buffer for partial handling
	+ * @buf: reeive buffer with partial
	+ * @txbuf: buffer to prepare
	+ */
	+static void
	+i40iw_ieq_setup_tx_buf(struct i40iw_puda_buf *buf,
	+ struct i40iw_puda_buf *txbuf)
	+{
	+ txbuf->maclen = buf->maclen;
	+ txbuf->tcphlen = buf->tcphlen;
	+ txbuf->ipv4 = buf->ipv4;
	+ txbuf->hdrlen = buf->hdrlen;
	+ i40iw_ieq_copy_to_txbuf(buf, txbuf, 0, 0, buf->hdrlen);
	+}
	+
	+/**
	+ * i40iw_ieq_check_first_buf - check if rcv buffer's seq is in range
	+ * @buf: receive exception buffer
	+ * @fps: first partial sequence number
	+ */
	+static void
	+i40iw_ieq_check_first_buf(struct i40iw_puda_buf *buf, u32 fps)
	+{
	+ u32 offset;
	+
	+ if (buf->seqnum < fps) {
	+ offset = fps - buf->seqnum;
	+ if (offset > buf->datalen)
	+ return;
	+ buf->data += offset;
	+ buf->datalen -= (u16)offset;
	+ buf->seqnum = fps;
	+ }
	+}
	+
	+/**
	+ * i40iw_ieq_compl_pfpdu - write txbuf with full fpdu
	+ * @ieq: ieq resource
	+ * @rxlist: ieq's received buffer list
	+ * @pbufl: temporary list for buffers for fpddu
	+ * @txbuf: tx buffer for fpdu
	+ * @fpdu_len: total length of fpdu
	+ */
	+static void
	+i40iw_ieq_compl_pfpdu(struct i40iw_puda_rsrc *ieq,
	+ struct list_head *rxlist,
	+ struct list_head *pbufl,
	+ struct i40iw_puda_buf *txbuf,
	+ u16 fpdu_len)
	+{
	+ struct i40iw_puda_buf *buf;
	+ u32 nextseqnum;
	+ u16 txoffset, bufoffset;
	+
	+ buf = i40iw_puda_get_listbuf(pbufl);
	+ if (!buf)
	+ return;
	+ nextseqnum = buf->seqnum + fpdu_len;
	+ txbuf->totallen = buf->hdrlen + fpdu_len;
	+ txbuf->data = (u8 *)txbuf->mem.va + buf->hdrlen;
	+ i40iw_ieq_setup_tx_buf(buf, txbuf);
	+
	+ txoffset = buf->hdrlen;
	+ bufoffset = (u16)(buf->data - (u8 *)buf->mem.va);
	+
	+ do {
	+ if (buf->datalen >= fpdu_len) {
	+ /* copied full fpdu */
	+ i40iw_ieq_copy_to_txbuf(buf, txbuf, bufoffset, txoffset, fpdu_len);
	+ buf->datalen -= fpdu_len;
	+ buf->data += fpdu_len;
	+ buf->seqnum = nextseqnum;
	+ break;
	+ }
	+ /* copy partial fpdu */
	+ i40iw_ieq_copy_to_txbuf(buf, txbuf, bufoffset, txoffset, buf->datalen);
	+ txoffset += buf->datalen;
	+ fpdu_len -= buf->datalen;
	+ i40iw_puda_ret_bufpool(ieq, buf);
	+ buf = i40iw_puda_get_listbuf(pbufl);
	+ if (!buf)
	+ return;
	+ bufoffset = (u16)(buf->data - (u8 *)buf->mem.va);
	+ } while (1);
	+
	+ /* last buffer on the list */
	+ if (buf->datalen)
	+ list_add(&buf->list, rxlist);
	+ else
	+ i40iw_puda_ret_bufpool(ieq, buf);
	+}
	+
	+/**
	+ * i40iw_ieq_create_pbufl - create buffer list for single fpdu
	+ * @rxlist: resource list for receive ieq buffes
	+ * @pbufl: temp. list for buffers for fpddu
	+ * @buf: first receive buffer
	+ * @fpdu_len: total length of fpdu
	+ */
	+static enum i40iw_status_code
	+i40iw_ieq_create_pbufl(
	+ struct i40iw_pfpdu *pfpdu,
	+ struct list_head *rxlist,
	+ struct list_head *pbufl,
	+ struct i40iw_puda_buf *buf,
	+ u16 fpdu_len)
	+{
	+ enum i40iw_status_code status = 0;
	+ struct i40iw_puda_buf *nextbuf;
	+ u32 nextseqnum;
	+ u16 plen = fpdu_len - buf->datalen;
	+ bool done = FALSE;
	+
	+ nextseqnum = buf->seqnum + buf->datalen;
	+ do {
	+ nextbuf = i40iw_puda_get_listbuf(rxlist);
	+ if (!nextbuf) {
	+ status = I40IW_ERR_LIST_EMPTY;
	+ break;
	+ }
	+ list_add_tail(&nextbuf->list, pbufl);
	+ if (nextbuf->seqnum != nextseqnum) {
	+ pfpdu->bad_seq_num++;
	+ status = I40IW_ERR_SEQ_NUM;
	+ break;
	+ }
	+ if (nextbuf->datalen >= plen) {
	+ done = TRUE;
	+ } else {
	+ plen -= nextbuf->datalen;
	+ nextseqnum = nextbuf->seqnum + nextbuf->datalen;
	+ }
	+
	+ } while (!done);
	+
	+ return status;
	+}
	+
	+/**
	+ * i40iw_ieq_handle_partial - process partial fpdu buffer
	+ * @ieq: ieq resource
	+ * @pfpdu: partial management per user qp
	+ * @buf: receive buffer
	+ * @fpdu_len: fpdu len in the buffer
	+ */
	+static enum i40iw_status_code
	+i40iw_ieq_handle_partial(struct i40iw_puda_rsrc *ieq,
	+ struct i40iw_pfpdu *pfpdu,
	+ struct i40iw_puda_buf *buf,
	+ u16 fpdu_len)
	+{
	+ enum i40iw_status_code status = 0;
	+ u8 *crcptr;
	+ u32 mpacrc;
	+ u32 seqnum = buf->seqnum;
	+ struct list_head pbufl; /* partial buffer list */
	+ struct i40iw_puda_buf *txbuf = NULL;
	+ struct list_head *rxlist = &pfpdu->rxlist;
	+
	+ INIT_LIST_HEAD(&pbufl);
	+ list_add(&buf->list, &pbufl);
	+
	+ status = i40iw_ieq_create_pbufl(pfpdu, rxlist, &pbufl, buf, fpdu_len);
	+ if (status)
	+ goto error;
	+
	+ txbuf = i40iw_puda_get_bufpool(ieq);
	+ if (!txbuf) {
	+ pfpdu->no_tx_bufs++;
	+ status = I40IW_ERR_NO_TXBUFS;
	+ goto error;
	+ }
	+ i40iw_ieq_compl_pfpdu(ieq, rxlist, &pbufl, txbuf, fpdu_len);
	+ i40iw_ieq_update_tcpip_info(txbuf, fpdu_len, seqnum);
	+ crcptr = txbuf->data + fpdu_len - 4;
	+ mpacrc = (u32 )crcptr;
	+ if (ieq->check_crc) {
	+ status = i40iw_ieq_check_mpacrc(ieq->hash_desc, txbuf->data,
	+ (fpdu_len - 4), mpacrc);
	+ if (status) {
	+ i40iw_debug(ieq->dev, I40IW_DEBUG_IEQ,
	+ "%s: error bad crc\n", __func__);
	+ goto error;
	+ }
	+ }
	+ i40iw_debug_buf(ieq->dev, I40IW_DEBUG_IEQ, "IEQ TX BUFFER",
	+ txbuf->mem.va, txbuf->totallen);
	+ i40iw_puda_send_buf(ieq, txbuf);
	+ pfpdu->rcv_nxt = seqnum + fpdu_len;
	+ return status;
	+error:
	+ while (!list_empty(&pbufl)) {
	+ buf = (struct i40iw_puda_buf *)(pbufl.prev);
	+ list_del(&buf->list);
	+ list_add(&buf->list, rxlist);
	+ }
	+ if (txbuf)
	+ i40iw_puda_ret_bufpool(ieq, txbuf);
	+ return status;
	+}
	+
	+/**
	+ * i40iw_ieq_process_buf - process buffer rcvd for ieq
	+ * @ieq: ieq resource
	+ * @pfpdu: partial management per user qp
	+ * @buf: receive buffer
	+ */
	+static enum i40iw_status_code
	+i40iw_ieq_process_buf(struct i40iw_puda_rsrc *ieq,
	+ struct i40iw_pfpdu *pfpdu,
	+ struct i40iw_puda_buf *buf)
	+{
	+ u16 fpdu_len = 0;
	+ u16 datalen = buf->datalen;
	+ u8 *datap = buf->data;
	+ u8 *crcptr;
	+ u16 ioffset = 0;
	+ u32 mpacrc;
	+ u32 seqnum = buf->seqnum;
	+ u16 length = 0;
	+ u16 full = 0;
	+ bool partial = FALSE;
	+ struct i40iw_puda_buf *txbuf;
	+ struct list_head *rxlist = &pfpdu->rxlist;
	+ enum i40iw_status_code ret = 0;
	+ enum i40iw_status_code status = 0;
	+
	+ ioffset = (u16)(buf->data - (u8 *)buf->mem.va);
	+ while (datalen) {
	+ fpdu_len = i40iw_ieq_get_fpdu_length(I40IW_NTOHS((__be16 ) datap));
	+ if (fpdu_len > pfpdu->max_fpdu_data) {
	+ i40iw_debug(ieq->dev, I40IW_DEBUG_IEQ,
	+ "%s: error bad fpdu_len\n", __func__);
	+ status = I40IW_ERR_MPA_CRC;
	+ list_add(&buf->list, rxlist);
	+ return status;
	+ }
	+ if (datalen < fpdu_len) {
	+ partial = TRUE;
	+ break;
	+ }
	+ crcptr = datap + fpdu_len - 4;
	+ mpacrc = (u32 )crcptr;
	+ if (ieq->check_crc)
	+ ret = i40iw_ieq_check_mpacrc(ieq->hash_desc,
	+ datap, fpdu_len - 4, mpacrc);
	+ if (ret) {
	+ status = I40IW_ERR_MPA_CRC;
	+ list_add(&buf->list, rxlist);
	+ return status;
	+ }
	+ full++;
	+ pfpdu->fpdu_processed++;
	+ datap += fpdu_len;
	+ length += fpdu_len;
	+ datalen -= fpdu_len;
	+ }
	+ if (full) {
	+ /* copy full pdu's in the txbuf and send them out */
	+ txbuf = i40iw_puda_get_bufpool(ieq);
	+ if (!txbuf) {
	+ pfpdu->no_tx_bufs++;
	+ status = I40IW_ERR_NO_TXBUFS;
	+ list_add(&buf->list, rxlist);
	+ return status;
	+ }
	+ /* modify txbuf's buffer header */
	+ i40iw_ieq_setup_tx_buf(buf, txbuf);
	+ /* copy full fpdu's to new buffer */
	+ i40iw_ieq_copy_to_txbuf(buf, txbuf, ioffset, buf->hdrlen,
	+ length);
	+ txbuf->totallen = buf->hdrlen + length;
	+
	+ i40iw_ieq_update_tcpip_info(txbuf, length, buf->seqnum);
	+ i40iw_puda_send_buf(ieq, txbuf);
	+
	+ if (!datalen) {
	+ pfpdu->rcv_nxt = buf->seqnum + length;
	+ i40iw_puda_ret_bufpool(ieq, buf);
	+ return status;
	+ }
	+ buf->data = datap;
	+ buf->seqnum = seqnum + length;
	+ buf->datalen = datalen;
	+ pfpdu->rcv_nxt = buf->seqnum;
	+ }
	+ if (partial)
	+ status = i40iw_ieq_handle_partial(ieq, pfpdu, buf, fpdu_len);
	+
	+ return status;
	+}
	+
	+/**
	+ * i40iw_ieq_process_fpdus - process fpdu's buffers on its list
	+ * @qp: qp for which partial fpdus
	+ * @ieq: ieq resource
	+ */
	+static void
	+i40iw_ieq_process_fpdus(struct i40iw_sc_qp *qp,
	+ struct i40iw_puda_rsrc *ieq)
	+{
	+ struct i40iw_pfpdu *pfpdu = &qp->pfpdu;
	+ struct list_head *rxlist = &pfpdu->rxlist;
	+ struct i40iw_puda_buf *buf;
	+ enum i40iw_status_code status;
	+
	+ do {
	+ if (list_empty(rxlist))
	+ break;
	+ buf = i40iw_puda_get_listbuf(rxlist);
	+ if (!buf) {
	+ i40iw_debug(ieq->dev, I40IW_DEBUG_IEQ,
	+ "%s: error no buf\n", __func__);
	+ break;
	+ }
	+ if (buf->seqnum != pfpdu->rcv_nxt) {
	+ /* This could be out of order or missing packet */
	+ pfpdu->out_of_order++;
	+ list_add(&buf->list, rxlist);
	+ break;
	+ }
	+ /* keep processing buffers from the head of the list */
	+ status = i40iw_ieq_process_buf(ieq, pfpdu, buf);
	+ if (status == I40IW_ERR_MPA_CRC) {
	+ pfpdu->mpa_crc_err = TRUE;
	+ while (!list_empty(rxlist)) {
	+ buf = i40iw_puda_get_listbuf(rxlist);
	+ i40iw_puda_ret_bufpool(ieq, buf);
	+ pfpdu->crc_err++;
	+ }
	+ /* create CQP for AE */
	+ i40iw_ieq_mpa_crc_ae(ieq->dev, qp);
	+ }
	+ } while (!status);
	+}
	+
	+/**
	+ * i40iw_ieq_handle_exception - handle qp's exception
	+ * @ieq: ieq resource
	+ * @qp: qp receiving excpetion
	+ * @buf: receive buffer
	+ */
	+static void
	+i40iw_ieq_handle_exception(struct i40iw_puda_rsrc *ieq,
	+ struct i40iw_sc_qp *qp,
	+ struct i40iw_puda_buf *buf)
	+{
	+ struct i40iw_puda_buf *tmpbuf = NULL;
	+ struct i40iw_pfpdu *pfpdu = &qp->pfpdu;
	+ u32 hw_host_ctx = (u32 )qp->hw_host_ctx;
	+ u32 rcv_wnd = hw_host_ctx[23];
	+ /* first partial seq # in q2 */
	+ u32 fps = (u32 )(qp->q2_buf + Q2_FPSN_OFFSET);
	+ struct list_head *rxlist = &pfpdu->rxlist;
	+ struct list_head *plist;
	+
	+ pfpdu->total_ieq_bufs++;
	+
	+ if (pfpdu->mpa_crc_err) {
	+ pfpdu->crc_err++;
	+ goto error;
	+ }
	+ if (pfpdu->mode && (fps != pfpdu->fps)) {
	+ /* clean up qp as it is new partial sequence */
	+ i40iw_ieq_cleanup_qp(ieq, qp);
	+ i40iw_debug(ieq->dev, I40IW_DEBUG_IEQ,
	+ "%s: restarting new partial\n", __func__);
	+ pfpdu->mode = FALSE;
	+ }
	+ if (!pfpdu->mode) {
	+ i40iw_debug_buf(ieq->dev, I40IW_DEBUG_IEQ, "Q2 BUFFER", (u64 *)qp->q2_buf, 128);
	+ /* First_Partial_Sequence_Number check */
	+ pfpdu->rcv_nxt = fps;
	+ pfpdu->fps = fps;
	+ pfpdu->mode = TRUE;
	+ pfpdu->max_fpdu_data = (buf->ipv4) ? (ieq->vsi->mtu - I40IW_MTU_TO_MSS_IPV4) :
	+ (ieq->vsi->mtu - I40IW_MTU_TO_MSS_IPV6);
	+ pfpdu->pmode_count++;
	+ INIT_LIST_HEAD(rxlist);
	+ i40iw_ieq_check_first_buf(buf, fps);
	+ }
	+ if (!(rcv_wnd >= (buf->seqnum - pfpdu->rcv_nxt))) {
	+ pfpdu->bad_seq_num++;
	+ goto error;
	+ }
	+ if (!list_empty(rxlist)) {
	+ tmpbuf = (struct i40iw_puda_buf *)rxlist->next;
	+ while ((struct list_head *)tmpbuf != rxlist) {
	+ if ((int)(buf->seqnum - tmpbuf->seqnum) < 0)
	+ break;
	+ plist = &tmpbuf->list;
	+ tmpbuf = (struct i40iw_puda_buf *)plist->next;
	+ }
	+ /* Insert buf before tmpbuf */
	+ list_add_tail(&buf->list, &tmpbuf->list);
	+ } else {
	+ list_add_tail(&buf->list, rxlist);
	+ }
	+ i40iw_ieq_process_fpdus(qp, ieq);
	+ return;
	+error:
	+ i40iw_puda_ret_bufpool(ieq, buf);
	+}
	+
	+/**
	+ * i40iw_ieq_receive - received exception buffer
	+ * @dev: iwarp device
	+ * @buf: exception buffer received
	+ */
	+static void
	+i40iw_ieq_receive(struct i40iw_sc_vsi *vsi,
	+ struct i40iw_puda_buf *buf)
	+{
	+ struct i40iw_puda_rsrc *ieq = vsi->ieq;
	+ struct i40iw_sc_qp *qp = NULL;
	+ u32 wqe_idx = ieq->compl_rxwqe_idx;
	+
	+ qp = i40iw_ieq_get_qp(vsi->dev, buf);
	+ if (!qp) {
	+ ieq->stats_bad_qp_id++;
	+ i40iw_puda_ret_bufpool(ieq, buf);
	+ } else {
	+ i40iw_ieq_handle_exception(ieq, qp, buf);
	+ }
	+ /*
	+ * ieq->rx_wqe_idx is used by i40iw_puda_replenish_rq() on which
	+ * wqe_idx to start replenish rq
	+ */
	+ if (!ieq->rxq_invalid_cnt)
	+ ieq->rx_wqe_idx = wqe_idx;
	+ ieq->rxq_invalid_cnt++;
	+}
	+
	+/**
	+ * i40iw_ieq_tx_compl - put back after sending completed exception buffer
	+ * @dev: iwarp device
	+ * @sqwrid: pointer to puda buffer
	+ */
	+static void
	+i40iw_ieq_tx_compl(struct i40iw_sc_vsi vsi, void sqwrid)
	+{
	+ struct i40iw_puda_rsrc *ieq = vsi->ieq;
	+ struct i40iw_puda_buf buf = (struct i40iw_puda_buf )sqwrid;
	+
	+ i40iw_puda_ret_bufpool(ieq, buf);
	+ if (!list_empty(&ieq->txpend)) {
	+ buf = i40iw_puda_get_listbuf(&ieq->txpend);
	+ i40iw_puda_send_buf(ieq, buf);
	+ }
	+}
	+
	+/**
	+ * i40iw_ieq_cleanup_qp - qp is being destroyed
	+ * @dev: iwarp device
	+ * @qp: all pending fpdu buffers
	+ */
	+void
	+i40iw_ieq_cleanup_qp(struct i40iw_puda_rsrc ieq, struct i40iw_sc_qp qp)
	+{
	+ struct i40iw_puda_buf *buf;
	+ struct i40iw_pfpdu *pfpdu = &qp->pfpdu;
	+ struct list_head *rxlist = &pfpdu->rxlist;
	+
	+ if (!pfpdu->mode)
	+ return;
	+ while (!list_empty(rxlist)) {
	+ buf = i40iw_puda_get_listbuf(rxlist);
	+ i40iw_puda_ret_bufpool(ieq, buf);
	+ }
	+}
	Index: sys/dev/ixl/iwarp/i40iw_register.h
	===================================================================
	--- /dev/null
	+++ sys/dev/ixl/iwarp/i40iw_register.h
	@@ -0,0 +1,1032 @@
	+/*******************************************************************************
	+*
	+* Copyright (c) 2015-2017 Intel Corporation. All rights reserved.
	+*
	+* This software is available to you under a choice of one of two
	+* licenses. You may choose to be licensed under the terms of the GNU
	+* General Public License (GPL) Version 2, available from the file
	+* COPYING in the main directory of this source tree, or the
	+* OpenFabrics.org BSD license below:
	+*
	+* Redistribution and use in source and binary forms, with or
	+* without modification, are permitted provided that the following
	+* conditions are met:
	+*
	+* - Redistributions of source code must retain the above
	+* copyright notice, this list of conditions and the following
	+* disclaimer.
	+*
	+* - Redistributions in binary form must reproduce the above
	+* copyright notice, this list of conditions and the following
	+* disclaimer in the documentation and/or other materials
	+* provided with the distribution.
	+*
	+* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
	+* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
	+* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
	+* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
	+* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
	+* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
	+* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
	+* SOFTWARE.
	+*
	+*******************************************************************************/
	+/$FreeBSD$/
	+
	+#ifndef I40IW_REGISTER_H
	+#define I40IW_REGISTER_H
	+
	+#define I40E_GLGEN_STAT 0x000B612C /* Reset: POR */
	+
	+#define I40E_PFHMC_PDINV 0x000C0300 /* Reset: PFR */
	+#define I40E_PFHMC_PDINV_PMSDIDX_SHIFT 0
	+#define I40E_PFHMC_PDINV_PMSDIDX_MASK (0xFFF << I40E_PFHMC_PDINV_PMSDIDX_SHIFT)
	+#define I40E_PFHMC_PDINV_PMPDIDX_SHIFT 16
	+#define I40E_PFHMC_PDINV_PMPDIDX_MASK (0x1FF << I40E_PFHMC_PDINV_PMPDIDX_SHIFT)
	+#define I40E_PFHMC_SDCMD_PMSDWR_SHIFT 31
	+#define I40E_PFHMC_SDCMD_PMSDWR_MASK (0x1 << I40E_PFHMC_SDCMD_PMSDWR_SHIFT)
	+#define I40E_PFHMC_SDDATALOW_PMSDVALID_SHIFT 0
	+#define I40E_PFHMC_SDDATALOW_PMSDVALID_MASK (0x1 << I40E_PFHMC_SDDATALOW_PMSDVALID_SHIFT)
	+#define I40E_PFHMC_SDDATALOW_PMSDTYPE_SHIFT 1
	+#define I40E_PFHMC_SDDATALOW_PMSDTYPE_MASK (0x1 << I40E_PFHMC_SDDATALOW_PMSDTYPE_SHIFT)
	+#define I40E_PFHMC_SDDATALOW_PMSDBPCOUNT_SHIFT 2
	+#define I40E_PFHMC_SDDATALOW_PMSDBPCOUNT_MASK (0x3FF << I40E_PFHMC_SDDATALOW_PMSDBPCOUNT_SHIFT)
	+
	+#define I40E_PFINT_DYN_CTLN(_INTPF) (0x00034800 + ((_INTPF) * 4)) /* _i=0...511 / / Reset: PFR */
	+#define I40E_PFINT_DYN_CTLN_INTENA_SHIFT 0
	+#define I40E_PFINT_DYN_CTLN_INTENA_MASK (0x1 << I40E_PFINT_DYN_CTLN_INTENA_SHIFT)
	+#define I40E_PFINT_DYN_CTLN_CLEARPBA_SHIFT 1
	+#define I40E_PFINT_DYN_CTLN_CLEARPBA_MASK (0x1 << I40E_PFINT_DYN_CTLN_CLEARPBA_SHIFT)
	+#define I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT 3
	+#define I40E_PFINT_DYN_CTLN_ITR_INDX_MASK (0x3 << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT)
	+
	+#define I40E_VFINT_DYN_CTLN1(_INTVF) (0x00003800 + ((_INTVF) * 4)) /* _i=0...15 / / Reset: VFR */
	+#define I40E_GLHMC_VFPDINV(_i) (0x000C8300 + ((_i) * 4)) /* _i=0...31 / / Reset: CORER */
	+
	+#define I40E_PFHMC_PDINV_PMSDPARTSEL_SHIFT 15
	+#define I40E_PFHMC_PDINV_PMSDPARTSEL_MASK (0x1 << I40E_PFHMC_PDINV_PMSDPARTSEL_SHIFT)
	+#define I40E_GLPCI_LBARCTRL 0x000BE484 /* Reset: POR */
	+#define I40E_GLPCI_LBARCTRL_PE_DB_SIZE_SHIFT 4
	+#define I40E_GLPCI_LBARCTRL_PE_DB_SIZE_MASK (0x3 << I40E_GLPCI_LBARCTRL_PE_DB_SIZE_SHIFT)
	+#define I40E_GLPCI_DREVID 0x0009C480 /* Reset: PCIR */
	+#define I40E_GLPCI_DREVID_DEFAULT_REVID_SHIFT 0
	+#define I40E_GLPCI_DREVID_DEFAULT_REVID_MASK 0xFF
	+
	+#define I40E_PFPE_AEQALLOC 0x00131180 /* Reset: PFR */
	+#define I40E_PFPE_AEQALLOC_AECOUNT_SHIFT 0
	+#define I40E_PFPE_AEQALLOC_AECOUNT_MASK (0xFFFFFFFF << I40E_PFPE_AEQALLOC_AECOUNT_SHIFT)
	+#define I40E_PFPE_CCQPHIGH 0x00008200 /* Reset: PFR */
	+#define I40E_PFPE_CCQPHIGH_PECCQPHIGH_SHIFT 0
	+#define I40E_PFPE_CCQPHIGH_PECCQPHIGH_MASK (0xFFFFFFFF << I40E_PFPE_CCQPHIGH_PECCQPHIGH_SHIFT)
	+#define I40E_PFPE_CCQPLOW 0x00008180 /* Reset: PFR */
	+#define I40E_PFPE_CCQPLOW_PECCQPLOW_SHIFT 0
	+#define I40E_PFPE_CCQPLOW_PECCQPLOW_MASK (0xFFFFFFFF << I40E_PFPE_CCQPLOW_PECCQPLOW_SHIFT)
	+#define I40E_PFPE_CCQPSTATUS 0x00008100 /* Reset: PFR */
	+#define I40E_PFPE_CCQPSTATUS_CCQP_DONE_SHIFT 0
	+#define I40E_PFPE_CCQPSTATUS_CCQP_DONE_MASK (0x1 << I40E_PFPE_CCQPSTATUS_CCQP_DONE_SHIFT)
	+#define I40E_PFPE_CCQPSTATUS_HMC_PROFILE_SHIFT 4
	+#define I40E_PFPE_CCQPSTATUS_HMC_PROFILE_MASK (0x7 << I40E_PFPE_CCQPSTATUS_HMC_PROFILE_SHIFT)
	+#define I40E_PFPE_CCQPSTATUS_RDMA_EN_VFS_SHIFT 16
	+#define I40E_PFPE_CCQPSTATUS_RDMA_EN_VFS_MASK (0x3F << I40E_PFPE_CCQPSTATUS_RDMA_EN_VFS_SHIFT)
	+#define I40E_PFPE_CCQPSTATUS_CCQP_ERR_SHIFT 31
	+#define I40E_PFPE_CCQPSTATUS_CCQP_ERR_MASK (0x1 << I40E_PFPE_CCQPSTATUS_CCQP_ERR_SHIFT)
	+#define I40E_PFPE_CQACK 0x00131100 /* Reset: PFR */
	+#define I40E_PFPE_CQACK_PECQID_SHIFT 0
	+#define I40E_PFPE_CQACK_PECQID_MASK (0x1FFFF << I40E_PFPE_CQACK_PECQID_SHIFT)
	+#define I40E_PFPE_CQARM 0x00131080 /* Reset: PFR */
	+#define I40E_PFPE_CQARM_PECQID_SHIFT 0
	+#define I40E_PFPE_CQARM_PECQID_MASK (0x1FFFF << I40E_PFPE_CQARM_PECQID_SHIFT)
	+#define I40E_PFPE_CQPDB 0x00008000 /* Reset: PFR */
	+#define I40E_PFPE_CQPDB_WQHEAD_SHIFT 0
	+#define I40E_PFPE_CQPDB_WQHEAD_MASK (0x7FF << I40E_PFPE_CQPDB_WQHEAD_SHIFT)
	+#define I40E_PFPE_CQPERRCODES 0x00008880 /* Reset: PFR */
	+#define I40E_PFPE_CQPERRCODES_CQP_MINOR_CODE_SHIFT 0
	+#define I40E_PFPE_CQPERRCODES_CQP_MINOR_CODE_MASK (0xFFFF << I40E_PFPE_CQPERRCODES_CQP_MINOR_CODE_SHIFT)
	+#define I40E_PFPE_CQPERRCODES_CQP_MAJOR_CODE_SHIFT 16
	+#define I40E_PFPE_CQPERRCODES_CQP_MAJOR_CODE_MASK (0xFFFF << I40E_PFPE_CQPERRCODES_CQP_MAJOR_CODE_SHIFT)
	+#define I40E_PFPE_CQPTAIL 0x00008080 /* Reset: PFR */
	+#define I40E_PFPE_CQPTAIL_WQTAIL_SHIFT 0
	+#define I40E_PFPE_CQPTAIL_WQTAIL_MASK (0x7FF << I40E_PFPE_CQPTAIL_WQTAIL_SHIFT)
	+#define I40E_PFPE_CQPTAIL_CQP_OP_ERR_SHIFT 31
	+#define I40E_PFPE_CQPTAIL_CQP_OP_ERR_MASK (0x1 << I40E_PFPE_CQPTAIL_CQP_OP_ERR_SHIFT)
	+#define I40E_PFPE_FLMQ1ALLOCERR 0x00008980 /* Reset: PFR */
	+#define I40E_PFPE_FLMQ1ALLOCERR_ERROR_COUNT_SHIFT 0
	+#define I40E_PFPE_FLMQ1ALLOCERR_ERROR_COUNT_MASK (0xFFFF << I40E_PFPE_FLMQ1ALLOCERR_ERROR_COUNT_SHIFT)
	+#define I40E_PFPE_FLMXMITALLOCERR 0x00008900 /* Reset: PFR */
	+#define I40E_PFPE_FLMXMITALLOCERR_ERROR_COUNT_SHIFT 0
	+#define I40E_PFPE_FLMXMITALLOCERR_ERROR_COUNT_MASK (0xFFFF << I40E_PFPE_FLMXMITALLOCERR_ERROR_COUNT_SHIFT)
	+#define I40E_PFPE_IPCONFIG0 0x00008280 /* Reset: PFR */
	+#define I40E_PFPE_IPCONFIG0_PEIPID_SHIFT 0
	+#define I40E_PFPE_IPCONFIG0_PEIPID_MASK (0xFFFF << I40E_PFPE_IPCONFIG0_PEIPID_SHIFT)
	+#define I40E_PFPE_IPCONFIG0_USEENTIREIDRANGE_SHIFT 16
	+#define I40E_PFPE_IPCONFIG0_USEENTIREIDRANGE_MASK (0x1 << I40E_PFPE_IPCONFIG0_USEENTIREIDRANGE_SHIFT)
	+#define I40E_PFPE_MRTEIDXMASK 0x00008600 /* Reset: PFR */
	+#define I40E_PFPE_MRTEIDXMASK_MRTEIDXMASKBITS_SHIFT 0
	+#define I40E_PFPE_MRTEIDXMASK_MRTEIDXMASKBITS_MASK (0x1F << I40E_PFPE_MRTEIDXMASK_MRTEIDXMASKBITS_SHIFT)
	+#define I40E_PFPE_RCVUNEXPECTEDERROR 0x00008680 /* Reset: PFR */
	+#define I40E_PFPE_RCVUNEXPECTEDERROR_TCP_RX_UNEXP_ERR_SHIFT 0
	+#define I40E_PFPE_RCVUNEXPECTEDERROR_TCP_RX_UNEXP_ERR_MASK (0xFFFFFF << I40E_PFPE_RCVUNEXPECTEDERROR_TCP_RX_UNEXP_ERR_SHIFT)
	+#define I40E_PFPE_TCPNOWTIMER 0x00008580 /* Reset: PFR */
	+#define I40E_PFPE_TCPNOWTIMER_TCP_NOW_SHIFT 0
	+#define I40E_PFPE_TCPNOWTIMER_TCP_NOW_MASK (0xFFFFFFFF << I40E_PFPE_TCPNOWTIMER_TCP_NOW_SHIFT)
	+
	+#define I40E_PFPE_WQEALLOC 0x00138C00 /* Reset: PFR */
	+#define I40E_PFPE_WQEALLOC_PEQPID_SHIFT 0
	+#define I40E_PFPE_WQEALLOC_PEQPID_MASK (0x3FFFF << I40E_PFPE_WQEALLOC_PEQPID_SHIFT)
	+#define I40E_PFPE_WQEALLOC_WQE_DESC_INDEX_SHIFT 20
	+#define I40E_PFPE_WQEALLOC_WQE_DESC_INDEX_MASK (0xFFF << I40E_PFPE_WQEALLOC_WQE_DESC_INDEX_SHIFT)
	+
	+#define I40E_VFPE_AEQALLOC(_VF) (0x00130C00 + ((_VF) * 4)) /* _i=0...127 / / Reset: VFR */
	+#define I40E_VFPE_AEQALLOC_MAX_INDEX 127
	+#define I40E_VFPE_AEQALLOC_AECOUNT_SHIFT 0
	+#define I40E_VFPE_AEQALLOC_AECOUNT_MASK (0xFFFFFFFF << I40E_VFPE_AEQALLOC_AECOUNT_SHIFT)
	+#define I40E_VFPE_CCQPHIGH(_VF) (0x00001000 + ((_VF) * 4)) /* _i=0...127 / / Reset: VFR */
	+#define I40E_VFPE_CCQPHIGH_MAX_INDEX 127
	+#define I40E_VFPE_CCQPHIGH_PECCQPHIGH_SHIFT 0
	+#define I40E_VFPE_CCQPHIGH_PECCQPHIGH_MASK (0xFFFFFFFF << I40E_VFPE_CCQPHIGH_PECCQPHIGH_SHIFT)
	+#define I40E_VFPE_CCQPLOW(_VF) (0x00000C00 + ((_VF) * 4)) /* _i=0...127 / / Reset: VFR */
	+#define I40E_VFPE_CCQPLOW_MAX_INDEX 127
	+#define I40E_VFPE_CCQPLOW_PECCQPLOW_SHIFT 0
	+#define I40E_VFPE_CCQPLOW_PECCQPLOW_MASK (0xFFFFFFFF << I40E_VFPE_CCQPLOW_PECCQPLOW_SHIFT)
	+#define I40E_VFPE_CCQPSTATUS(_VF) (0x00000800 + ((_VF) * 4)) /* _i=0...127 / / Reset: VFR */
	+#define I40E_VFPE_CCQPSTATUS_MAX_INDEX 127
	+#define I40E_VFPE_CCQPSTATUS_CCQP_DONE_SHIFT 0
	+#define I40E_VFPE_CCQPSTATUS_CCQP_DONE_MASK (0x1 << I40E_VFPE_CCQPSTATUS_CCQP_DONE_SHIFT)
	+#define I40E_VFPE_CCQPSTATUS_HMC_PROFILE_SHIFT 4
	+#define I40E_VFPE_CCQPSTATUS_HMC_PROFILE_MASK (0x7 << I40E_VFPE_CCQPSTATUS_HMC_PROFILE_SHIFT)
	+#define I40E_VFPE_CCQPSTATUS_RDMA_EN_VFS_SHIFT 16
	+#define I40E_VFPE_CCQPSTATUS_RDMA_EN_VFS_MASK (0x3F << I40E_VFPE_CCQPSTATUS_RDMA_EN_VFS_SHIFT)
	+#define I40E_VFPE_CCQPSTATUS_CCQP_ERR_SHIFT 31
	+#define I40E_VFPE_CCQPSTATUS_CCQP_ERR_MASK (0x1 << I40E_VFPE_CCQPSTATUS_CCQP_ERR_SHIFT)
	+#define I40E_VFPE_CQACK(_VF) (0x00130800 + ((_VF) * 4)) /* _i=0...127 / / Reset: VFR */
	+#define I40E_VFPE_CQACK_MAX_INDEX 127
	+#define I40E_VFPE_CQACK_PECQID_SHIFT 0
	+#define I40E_VFPE_CQACK_PECQID_MASK (0x1FFFF << I40E_VFPE_CQACK_PECQID_SHIFT)
	+#define I40E_VFPE_CQARM(_VF) (0x00130400 + ((_VF) * 4)) /* _i=0...127 / / Reset: VFR */
	+#define I40E_VFPE_CQARM_MAX_INDEX 127
	+#define I40E_VFPE_CQARM_PECQID_SHIFT 0
	+#define I40E_VFPE_CQARM_PECQID_MASK (0x1FFFF << I40E_VFPE_CQARM_PECQID_SHIFT)
	+#define I40E_VFPE_CQPDB(_VF) (0x00000000 + ((_VF) * 4)) /* _i=0...127 / / Reset: VFR */
	+#define I40E_VFPE_CQPDB_MAX_INDEX 127
	+#define I40E_VFPE_CQPDB_WQHEAD_SHIFT 0
	+#define I40E_VFPE_CQPDB_WQHEAD_MASK (0x7FF << I40E_VFPE_CQPDB_WQHEAD_SHIFT)
	+#define I40E_VFPE_CQPERRCODES(_VF) (0x00001800 + ((_VF) * 4)) /* _i=0...127 / / Reset: VFR */
	+#define I40E_VFPE_CQPERRCODES_MAX_INDEX 127
	+#define I40E_VFPE_CQPERRCODES_CQP_MINOR_CODE_SHIFT 0
	+#define I40E_VFPE_CQPERRCODES_CQP_MINOR_CODE_MASK (0xFFFF << I40E_VFPE_CQPERRCODES_CQP_MINOR_CODE_SHIFT)
	+#define I40E_VFPE_CQPERRCODES_CQP_MAJOR_CODE_SHIFT 16
	+#define I40E_VFPE_CQPERRCODES_CQP_MAJOR_CODE_MASK (0xFFFF << I40E_VFPE_CQPERRCODES_CQP_MAJOR_CODE_SHIFT)
	+#define I40E_VFPE_CQPTAIL(_VF) (0x00000400 + ((_VF) * 4)) /* _i=0...127 / / Reset: VFR */
	+#define I40E_VFPE_CQPTAIL_MAX_INDEX 127
	+#define I40E_VFPE_CQPTAIL_WQTAIL_SHIFT 0
	+#define I40E_VFPE_CQPTAIL_WQTAIL_MASK (0x7FF << I40E_VFPE_CQPTAIL_WQTAIL_SHIFT)
	+#define I40E_VFPE_CQPTAIL_CQP_OP_ERR_SHIFT 31
	+#define I40E_VFPE_CQPTAIL_CQP_OP_ERR_MASK (0x1 << I40E_VFPE_CQPTAIL_CQP_OP_ERR_SHIFT)
	+#define I40E_VFPE_IPCONFIG0(_VF) (0x00001400 + ((_VF) * 4)) /* _i=0...127 / / Reset: VFR */
	+#define I40E_VFPE_IPCONFIG0_MAX_INDEX 127
	+#define I40E_VFPE_IPCONFIG0_PEIPID_SHIFT 0
	+#define I40E_VFPE_IPCONFIG0_PEIPID_MASK (0xFFFF << I40E_VFPE_IPCONFIG0_PEIPID_SHIFT)
	+#define I40E_VFPE_IPCONFIG0_USEENTIREIDRANGE_SHIFT 16
	+#define I40E_VFPE_IPCONFIG0_USEENTIREIDRANGE_MASK (0x1 << I40E_VFPE_IPCONFIG0_USEENTIREIDRANGE_SHIFT)
	+#define I40E_VFPE_MRTEIDXMASK(_VF) (0x00003000 + ((_VF) * 4)) /* _i=0...127 / / Reset: VFR */
	+#define I40E_VFPE_MRTEIDXMASK_MAX_INDEX 127
	+#define I40E_VFPE_MRTEIDXMASK_MRTEIDXMASKBITS_SHIFT 0
	+#define I40E_VFPE_MRTEIDXMASK_MRTEIDXMASKBITS_MASK (0x1F << I40E_VFPE_MRTEIDXMASK_MRTEIDXMASKBITS_SHIFT)
	+#define I40E_VFPE_RCVUNEXPECTEDERROR(_VF) (0x00003400 + ((_VF) * 4)) /* _i=0...127 / / Reset: VFR */
	+#define I40E_VFPE_RCVUNEXPECTEDERROR_MAX_INDEX 127
	+#define I40E_VFPE_RCVUNEXPECTEDERROR_TCP_RX_UNEXP_ERR_SHIFT 0
	+#define I40E_VFPE_RCVUNEXPECTEDERROR_TCP_RX_UNEXP_ERR_MASK (0xFFFFFF << I40E_VFPE_RCVUNEXPECTEDERROR_TCP_RX_UNEXP_ERR_SHIFT)
	+#define I40E_VFPE_TCPNOWTIMER(_VF) (0x00002C00 + ((_VF) * 4)) /* _i=0...127 / / Reset: VFR */
	+#define I40E_VFPE_TCPNOWTIMER_MAX_INDEX 127
	+#define I40E_VFPE_TCPNOWTIMER_TCP_NOW_SHIFT 0
	+#define I40E_VFPE_TCPNOWTIMER_TCP_NOW_MASK (0xFFFFFFFF << I40E_VFPE_TCPNOWTIMER_TCP_NOW_SHIFT)
	+#define I40E_VFPE_WQEALLOC(_VF) (0x00138000 + ((_VF) * 4)) /* _i=0...127 / / Reset: VFR */
	+#define I40E_VFPE_WQEALLOC_MAX_INDEX 127
	+#define I40E_VFPE_WQEALLOC_PEQPID_SHIFT 0
	+#define I40E_VFPE_WQEALLOC_PEQPID_MASK (0x3FFFF << I40E_VFPE_WQEALLOC_PEQPID_SHIFT)
	+#define I40E_VFPE_WQEALLOC_WQE_DESC_INDEX_SHIFT 20
	+#define I40E_VFPE_WQEALLOC_WQE_DESC_INDEX_MASK (0xFFF << I40E_VFPE_WQEALLOC_WQE_DESC_INDEX_SHIFT)
	+
	+#define I40E_GLPE_CPUSTATUS0 0x0000D040 /* Reset: PE_CORER */
	+#define I40E_GLPE_CPUSTATUS0_PECPUSTATUS0_SHIFT 0
	+#define I40E_GLPE_CPUSTATUS0_PECPUSTATUS0_MASK (0xFFFFFFFF << I40E_GLPE_CPUSTATUS0_PECPUSTATUS0_SHIFT)
	+#define I40E_GLPE_CPUSTATUS1 0x0000D044 /* Reset: PE_CORER */
	+#define I40E_GLPE_CPUSTATUS1_PECPUSTATUS1_SHIFT 0
	+#define I40E_GLPE_CPUSTATUS1_PECPUSTATUS1_MASK (0xFFFFFFFF << I40E_GLPE_CPUSTATUS1_PECPUSTATUS1_SHIFT)
	+#define I40E_GLPE_CPUSTATUS2 0x0000D048 /* Reset: PE_CORER */
	+#define I40E_GLPE_CPUSTATUS2_PECPUSTATUS2_SHIFT 0
	+#define I40E_GLPE_CPUSTATUS2_PECPUSTATUS2_MASK (0xFFFFFFFF << I40E_GLPE_CPUSTATUS2_PECPUSTATUS2_SHIFT)
	+#define I40E_GLPE_CPUTRIG0 0x0000D060 /* Reset: PE_CORER */
	+#define I40E_GLPE_CPUTRIG0_PECPUTRIG0_SHIFT 0
	+#define I40E_GLPE_CPUTRIG0_PECPUTRIG0_MASK (0xFFFF << I40E_GLPE_CPUTRIG0_PECPUTRIG0_SHIFT)
	+#define I40E_GLPE_CPUTRIG0_TEPREQUEST0_SHIFT 17
	+#define I40E_GLPE_CPUTRIG0_TEPREQUEST0_MASK (0x1 << I40E_GLPE_CPUTRIG0_TEPREQUEST0_SHIFT)
	+#define I40E_GLPE_CPUTRIG0_OOPREQUEST0_SHIFT 18
	+#define I40E_GLPE_CPUTRIG0_OOPREQUEST0_MASK (0x1 << I40E_GLPE_CPUTRIG0_OOPREQUEST0_SHIFT)
	+#define I40E_GLPE_DUAL40_RUPM 0x0000DA04 /* Reset: PE_CORER */
	+#define I40E_GLPE_DUAL40_RUPM_DUAL_40G_MODE_SHIFT 0
	+#define I40E_GLPE_DUAL40_RUPM_DUAL_40G_MODE_MASK (0x1 << I40E_GLPE_DUAL40_RUPM_DUAL_40G_MODE_SHIFT)
	+#define I40E_GLPE_PFAEQEDROPCNT(_i) (0x00131440 + ((_i) * 4)) /* _i=0...15 / / Reset: CORER */
	+#define I40E_GLPE_PFAEQEDROPCNT_MAX_INDEX 15
	+#define I40E_GLPE_PFAEQEDROPCNT_AEQEDROPCNT_SHIFT 0
	+#define I40E_GLPE_PFAEQEDROPCNT_AEQEDROPCNT_MASK (0xFFFF << I40E_GLPE_PFAEQEDROPCNT_AEQEDROPCNT_SHIFT)
	+#define I40E_GLPE_PFCEQEDROPCNT(_i) (0x001313C0 + ((_i) * 4)) /* _i=0...15 / / Reset: CORER */
	+#define I40E_GLPE_PFCEQEDROPCNT_MAX_INDEX 15
	+#define I40E_GLPE_PFCEQEDROPCNT_CEQEDROPCNT_SHIFT 0
	+#define I40E_GLPE_PFCEQEDROPCNT_CEQEDROPCNT_MASK (0xFFFF << I40E_GLPE_PFCEQEDROPCNT_CEQEDROPCNT_SHIFT)
	+#define I40E_GLPE_PFCQEDROPCNT(_i) (0x00131340 + ((_i) * 4)) /* _i=0...15 / / Reset: CORER */
	+#define I40E_GLPE_PFCQEDROPCNT_MAX_INDEX 15
	+#define I40E_GLPE_PFCQEDROPCNT_CQEDROPCNT_SHIFT 0
	+#define I40E_GLPE_PFCQEDROPCNT_CQEDROPCNT_MASK (0xFFFF << I40E_GLPE_PFCQEDROPCNT_CQEDROPCNT_SHIFT)
	+#define I40E_GLPE_RUPM_CQPPOOL 0x0000DACC /* Reset: PE_CORER */
	+#define I40E_GLPE_RUPM_CQPPOOL_CQPSPADS_SHIFT 0
	+#define I40E_GLPE_RUPM_CQPPOOL_CQPSPADS_MASK (0xFF << I40E_GLPE_RUPM_CQPPOOL_CQPSPADS_SHIFT)
	+#define I40E_GLPE_RUPM_FLRPOOL 0x0000DAC4 /* Reset: PE_CORER */
	+#define I40E_GLPE_RUPM_FLRPOOL_FLRSPADS_SHIFT 0
	+#define I40E_GLPE_RUPM_FLRPOOL_FLRSPADS_MASK (0xFF << I40E_GLPE_RUPM_FLRPOOL_FLRSPADS_SHIFT)
	+#define I40E_GLPE_RUPM_GCTL 0x0000DA00 /* Reset: PE_CORER */
	+#define I40E_GLPE_RUPM_GCTL_ALLOFFTH_SHIFT 0
	+#define I40E_GLPE_RUPM_GCTL_ALLOFFTH_MASK (0xFF << I40E_GLPE_RUPM_GCTL_ALLOFFTH_SHIFT)
	+#define I40E_GLPE_RUPM_GCTL_RUPM_P0_DIS_SHIFT 26
	+#define I40E_GLPE_RUPM_GCTL_RUPM_P0_DIS_MASK (0x1 << I40E_GLPE_RUPM_GCTL_RUPM_P0_DIS_SHIFT)
	+#define I40E_GLPE_RUPM_GCTL_RUPM_P1_DIS_SHIFT 27
	+#define I40E_GLPE_RUPM_GCTL_RUPM_P1_DIS_MASK (0x1 << I40E_GLPE_RUPM_GCTL_RUPM_P1_DIS_SHIFT)
	+#define I40E_GLPE_RUPM_GCTL_RUPM_P2_DIS_SHIFT 28
	+#define I40E_GLPE_RUPM_GCTL_RUPM_P2_DIS_MASK (0x1 << I40E_GLPE_RUPM_GCTL_RUPM_P2_DIS_SHIFT)
	+#define I40E_GLPE_RUPM_GCTL_RUPM_P3_DIS_SHIFT 29
	+#define I40E_GLPE_RUPM_GCTL_RUPM_P3_DIS_MASK (0x1 << I40E_GLPE_RUPM_GCTL_RUPM_P3_DIS_SHIFT)
	+#define I40E_GLPE_RUPM_GCTL_RUPM_DIS_SHIFT 30
	+#define I40E_GLPE_RUPM_GCTL_RUPM_DIS_MASK (0x1 << I40E_GLPE_RUPM_GCTL_RUPM_DIS_SHIFT)
	+#define I40E_GLPE_RUPM_GCTL_SWLB_MODE_SHIFT 31
	+#define I40E_GLPE_RUPM_GCTL_SWLB_MODE_MASK (0x1 << I40E_GLPE_RUPM_GCTL_SWLB_MODE_SHIFT)
	+#define I40E_GLPE_RUPM_PTXPOOL 0x0000DAC8 /* Reset: PE_CORER */
	+#define I40E_GLPE_RUPM_PTXPOOL_PTXSPADS_SHIFT 0
	+#define I40E_GLPE_RUPM_PTXPOOL_PTXSPADS_MASK (0xFF << I40E_GLPE_RUPM_PTXPOOL_PTXSPADS_SHIFT)
	+#define I40E_GLPE_RUPM_PUSHPOOL 0x0000DAC0 /* Reset: PE_CORER */
	+#define I40E_GLPE_RUPM_PUSHPOOL_PUSHSPADS_SHIFT 0
	+#define I40E_GLPE_RUPM_PUSHPOOL_PUSHSPADS_MASK (0xFF << I40E_GLPE_RUPM_PUSHPOOL_PUSHSPADS_SHIFT)
	+#define I40E_GLPE_RUPM_TXHOST_EN 0x0000DA08 /* Reset: PE_CORER */
	+#define I40E_GLPE_RUPM_TXHOST_EN_TXHOST_EN_SHIFT 0
	+#define I40E_GLPE_RUPM_TXHOST_EN_TXHOST_EN_MASK (0x1 << I40E_GLPE_RUPM_TXHOST_EN_TXHOST_EN_SHIFT)
	+#define I40E_GLPE_VFAEQEDROPCNT(_i) (0x00132540 + ((_i) * 4)) /* _i=0...31 / / Reset: CORER */
	+#define I40E_GLPE_VFAEQEDROPCNT_MAX_INDEX 31
	+#define I40E_GLPE_VFAEQEDROPCNT_AEQEDROPCNT_SHIFT 0
	+#define I40E_GLPE_VFAEQEDROPCNT_AEQEDROPCNT_MASK (0xFFFF << I40E_GLPE_VFAEQEDROPCNT_AEQEDROPCNT_SHIFT)
	+#define I40E_GLPE_VFCEQEDROPCNT(_i) (0x00132440 + ((_i) * 4)) /* _i=0...31 / / Reset: CORER */
	+#define I40E_GLPE_VFCEQEDROPCNT_MAX_INDEX 31
	+#define I40E_GLPE_VFCEQEDROPCNT_CEQEDROPCNT_SHIFT 0
	+#define I40E_GLPE_VFCEQEDROPCNT_CEQEDROPCNT_MASK (0xFFFF << I40E_GLPE_VFCEQEDROPCNT_CEQEDROPCNT_SHIFT)
	+#define I40E_GLPE_VFCQEDROPCNT(_i) (0x00132340 + ((_i) * 4)) /* _i=0...31 / / Reset: CORER */
	+#define I40E_GLPE_VFCQEDROPCNT_MAX_INDEX 31
	+#define I40E_GLPE_VFCQEDROPCNT_CQEDROPCNT_SHIFT 0
	+#define I40E_GLPE_VFCQEDROPCNT_CQEDROPCNT_MASK (0xFFFF << I40E_GLPE_VFCQEDROPCNT_CQEDROPCNT_SHIFT)
	+#define I40E_GLPE_VFFLMOBJCTRL(_i) (0x0000D400 + ((_i) * 4)) /* _i=0...31 / / Reset: PE_CORER */
	+#define I40E_GLPE_VFFLMOBJCTRL_MAX_INDEX 31
	+#define I40E_GLPE_VFFLMOBJCTRL_XMIT_BLOCKSIZE_SHIFT 0
	+#define I40E_GLPE_VFFLMOBJCTRL_XMIT_BLOCKSIZE_MASK (0x7 << I40E_GLPE_VFFLMOBJCTRL_XMIT_BLOCKSIZE_SHIFT)
	+#define I40E_GLPE_VFFLMOBJCTRL_Q1_BLOCKSIZE_SHIFT 8
	+#define I40E_GLPE_VFFLMOBJCTRL_Q1_BLOCKSIZE_MASK (0x7 << I40E_GLPE_VFFLMOBJCTRL_Q1_BLOCKSIZE_SHIFT)
	+#define I40E_GLPE_VFFLMQ1ALLOCERR(_i) (0x0000C700 + ((_i) * 4)) /* _i=0...31 / / Reset: PE_CORER */
	+#define I40E_GLPE_VFFLMQ1ALLOCERR_MAX_INDEX 31
	+#define I40E_GLPE_VFFLMQ1ALLOCERR_ERROR_COUNT_SHIFT 0
	+#define I40E_GLPE_VFFLMQ1ALLOCERR_ERROR_COUNT_MASK (0xFFFF << I40E_GLPE_VFFLMQ1ALLOCERR_ERROR_COUNT_SHIFT)
	+#define I40E_GLPE_VFFLMXMITALLOCERR(_i) (0x0000C600 + ((_i) * 4)) /* _i=0...31 / / Reset: PE_CORER */
	+#define I40E_GLPE_VFFLMXMITALLOCERR_MAX_INDEX 31
	+#define I40E_GLPE_VFFLMXMITALLOCERR_ERROR_COUNT_SHIFT 0
	+#define I40E_GLPE_VFFLMXMITALLOCERR_ERROR_COUNT_MASK (0xFFFF << I40E_GLPE_VFFLMXMITALLOCERR_ERROR_COUNT_SHIFT)
	+#define I40E_GLPE_VFUDACTRL(_i) (0x0000C000 + ((_i) * 4)) /* _i=0...31 / / Reset: PE_CORER */
	+#define I40E_GLPE_VFUDACTRL_MAX_INDEX 31
	+#define I40E_GLPE_VFUDACTRL_IPV4MCFRAGRESBP_SHIFT 0
	+#define I40E_GLPE_VFUDACTRL_IPV4MCFRAGRESBP_MASK (0x1 << I40E_GLPE_VFUDACTRL_IPV4MCFRAGRESBP_SHIFT)
	+#define I40E_GLPE_VFUDACTRL_IPV4UCFRAGRESBP_SHIFT 1
	+#define I40E_GLPE_VFUDACTRL_IPV4UCFRAGRESBP_MASK (0x1 << I40E_GLPE_VFUDACTRL_IPV4UCFRAGRESBP_SHIFT)
	+#define I40E_GLPE_VFUDACTRL_IPV6MCFRAGRESBP_SHIFT 2
	+#define I40E_GLPE_VFUDACTRL_IPV6MCFRAGRESBP_MASK (0x1 << I40E_GLPE_VFUDACTRL_IPV6MCFRAGRESBP_SHIFT)
	+#define I40E_GLPE_VFUDACTRL_IPV6UCFRAGRESBP_SHIFT 3
	+#define I40E_GLPE_VFUDACTRL_IPV6UCFRAGRESBP_MASK (0x1 << I40E_GLPE_VFUDACTRL_IPV6UCFRAGRESBP_SHIFT)
	+#define I40E_GLPE_VFUDACTRL_UDPMCFRAGRESFAIL_SHIFT 4
	+#define I40E_GLPE_VFUDACTRL_UDPMCFRAGRESFAIL_MASK (0x1 << I40E_GLPE_VFUDACTRL_UDPMCFRAGRESFAIL_SHIFT)
	+#define I40E_GLPE_VFUDAUCFBQPN(_i) (0x0000C100 + ((_i) * 4)) /* _i=0...31 / / Reset: PE_CORER */
	+#define I40E_GLPE_VFUDAUCFBQPN_MAX_INDEX 31
	+#define I40E_GLPE_VFUDAUCFBQPN_QPN_SHIFT 0
	+#define I40E_GLPE_VFUDAUCFBQPN_QPN_MASK (0x3FFFF << I40E_GLPE_VFUDAUCFBQPN_QPN_SHIFT)
	+#define I40E_GLPE_VFUDAUCFBQPN_VALID_SHIFT 31
	+#define I40E_GLPE_VFUDAUCFBQPN_VALID_MASK (0x1 << I40E_GLPE_VFUDAUCFBQPN_VALID_SHIFT)
	+
	+#define I40E_GLPES_PFIP4RXDISCARD(_i) (0x00010600 + ((_i) * 4)) /* _i=0...15 / / Reset: PE_CORER */
	+#define I40E_GLPES_PFIP4RXDISCARD_MAX_INDEX 15
	+#define I40E_GLPES_PFIP4RXDISCARD_IP4RXDISCARD_SHIFT 0
	+#define I40E_GLPES_PFIP4RXDISCARD_IP4RXDISCARD_MASK (0xFFFFFFFF << I40E_GLPES_PFIP4RXDISCARD_IP4RXDISCARD_SHIFT)
	+#define I40E_GLPES_PFIP4RXFRAGSHI(_i) (0x00010804 + ((_i) * 8)) /* _i=0...15 / / Reset: PE_CORER */
	+#define I40E_GLPES_PFIP4RXFRAGSHI_MAX_INDEX 15
	+#define I40E_GLPES_PFIP4RXFRAGSHI_IP4RXFRAGSHI_SHIFT 0
	+#define I40E_GLPES_PFIP4RXFRAGSHI_IP4RXFRAGSHI_MASK (0xFFFF << I40E_GLPES_PFIP4RXFRAGSHI_IP4RXFRAGSHI_SHIFT)
	+#define I40E_GLPES_PFIP4RXFRAGSLO(_i) (0x00010800 + ((_i) * 8)) /* _i=0...15 / / Reset: PE_CORER */
	+#define I40E_GLPES_PFIP4RXFRAGSLO_MAX_INDEX 15
	+#define I40E_GLPES_PFIP4RXFRAGSLO_IP4RXFRAGSLO_SHIFT 0
	+#define I40E_GLPES_PFIP4RXFRAGSLO_IP4RXFRAGSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFIP4RXFRAGSLO_IP4RXFRAGSLO_SHIFT)
	+#define I40E_GLPES_PFIP4RXMCOCTSHI(_i) (0x00010A04 + ((_i) * 8)) /* _i=0...15 / / Reset: PE_CORER */
	+#define I40E_GLPES_PFIP4RXMCOCTSHI_MAX_INDEX 15
	+#define I40E_GLPES_PFIP4RXMCOCTSHI_IP4RXMCOCTSHI_SHIFT 0
	+#define I40E_GLPES_PFIP4RXMCOCTSHI_IP4RXMCOCTSHI_MASK (0xFFFF << I40E_GLPES_PFIP4RXMCOCTSHI_IP4RXMCOCTSHI_SHIFT)
	+#define I40E_GLPES_PFIP4RXMCOCTSLO(_i) (0x00010A00 + ((_i) * 8)) /* _i=0...15 / / Reset: PE_CORER */
	+#define I40E_GLPES_PFIP4RXMCOCTSLO_MAX_INDEX 15
	+#define I40E_GLPES_PFIP4RXMCOCTSLO_IP4RXMCOCTSLO_SHIFT 0
	+#define I40E_GLPES_PFIP4RXMCOCTSLO_IP4RXMCOCTSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFIP4RXMCOCTSLO_IP4RXMCOCTSLO_SHIFT)
	+#define I40E_GLPES_PFIP4RXMCPKTSHI(_i) (0x00010C04 + ((_i) * 8)) /* _i=0...15 / / Reset: PE_CORER */
	+#define I40E_GLPES_PFIP4RXMCPKTSHI_MAX_INDEX 15
	+#define I40E_GLPES_PFIP4RXMCPKTSHI_IP4RXMCPKTSHI_SHIFT 0
	+#define I40E_GLPES_PFIP4RXMCPKTSHI_IP4RXMCPKTSHI_MASK (0xFFFF << I40E_GLPES_PFIP4RXMCPKTSHI_IP4RXMCPKTSHI_SHIFT)
	+#define I40E_GLPES_PFIP4RXMCPKTSLO(_i) (0x00010C00 + ((_i) * 8)) /* _i=0...15 / / Reset: PE_CORER */
	+#define I40E_GLPES_PFIP4RXMCPKTSLO_MAX_INDEX 15
	+#define I40E_GLPES_PFIP4RXMCPKTSLO_IP4RXMCPKTSLO_SHIFT 0
	+#define I40E_GLPES_PFIP4RXMCPKTSLO_IP4RXMCPKTSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFIP4RXMCPKTSLO_IP4RXMCPKTSLO_SHIFT)
	+#define I40E_GLPES_PFIP4RXOCTSHI(_i) (0x00010204 + ((_i) * 8)) /* _i=0...15 / / Reset: PE_CORER */
	+#define I40E_GLPES_PFIP4RXOCTSHI_MAX_INDEX 15
	+#define I40E_GLPES_PFIP4RXOCTSHI_IP4RXOCTSHI_SHIFT 0
	+#define I40E_GLPES_PFIP4RXOCTSHI_IP4RXOCTSHI_MASK (0xFFFF << I40E_GLPES_PFIP4RXOCTSHI_IP4RXOCTSHI_SHIFT)
	+#define I40E_GLPES_PFIP4RXOCTSLO(_i) (0x00010200 + ((_i) * 8)) /* _i=0...15 / / Reset: PE_CORER */
	+#define I40E_GLPES_PFIP4RXOCTSLO_MAX_INDEX 15
	+#define I40E_GLPES_PFIP4RXOCTSLO_IP4RXOCTSLO_SHIFT 0
	+#define I40E_GLPES_PFIP4RXOCTSLO_IP4RXOCTSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFIP4RXOCTSLO_IP4RXOCTSLO_SHIFT)
	+#define I40E_GLPES_PFIP4RXPKTSHI(_i) (0x00010404 + ((_i) * 8)) /* _i=0...15 / / Reset: PE_CORER */
	+#define I40E_GLPES_PFIP4RXPKTSHI_MAX_INDEX 15
	+#define I40E_GLPES_PFIP4RXPKTSHI_IP4RXPKTSHI_SHIFT 0
	+#define I40E_GLPES_PFIP4RXPKTSHI_IP4RXPKTSHI_MASK (0xFFFF << I40E_GLPES_PFIP4RXPKTSHI_IP4RXPKTSHI_SHIFT)
	+#define I40E_GLPES_PFIP4RXPKTSLO(_i) (0x00010400 + ((_i) * 8)) /* _i=0...15 / / Reset: PE_CORER */
	+#define I40E_GLPES_PFIP4RXPKTSLO_MAX_INDEX 15
	+#define I40E_GLPES_PFIP4RXPKTSLO_IP4RXPKTSLO_SHIFT 0
	+#define I40E_GLPES_PFIP4RXPKTSLO_IP4RXPKTSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFIP4RXPKTSLO_IP4RXPKTSLO_SHIFT)
	+#define I40E_GLPES_PFIP4RXTRUNC(_i) (0x00010700 + ((_i) * 4)) /* _i=0...15 / / Reset: PE_CORER */
	+#define I40E_GLPES_PFIP4RXTRUNC_MAX_INDEX 15
	+#define I40E_GLPES_PFIP4RXTRUNC_IP4RXTRUNC_SHIFT 0
	+#define I40E_GLPES_PFIP4RXTRUNC_IP4RXTRUNC_MASK (0xFFFFFFFF << I40E_GLPES_PFIP4RXTRUNC_IP4RXTRUNC_SHIFT)
	+#define I40E_GLPES_PFIP4TXFRAGSHI(_i) (0x00011E04 + ((_i) * 8)) /* _i=0...15 / / Reset: PE_CORER */
	+#define I40E_GLPES_PFIP4TXFRAGSHI_MAX_INDEX 15
	+#define I40E_GLPES_PFIP4TXFRAGSHI_IP4TXFRAGSHI_SHIFT 0
	+#define I40E_GLPES_PFIP4TXFRAGSHI_IP4TXFRAGSHI_MASK (0xFFFF << I40E_GLPES_PFIP4TXFRAGSHI_IP4TXFRAGSHI_SHIFT)
	+#define I40E_GLPES_PFIP4TXFRAGSLO(_i) (0x00011E00 + ((_i) * 8)) /* _i=0...15 / / Reset: PE_CORER */
	+#define I40E_GLPES_PFIP4TXFRAGSLO_MAX_INDEX 15
	+#define I40E_GLPES_PFIP4TXFRAGSLO_IP4TXFRAGSLO_SHIFT 0
	+#define I40E_GLPES_PFIP4TXFRAGSLO_IP4TXFRAGSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFIP4TXFRAGSLO_IP4TXFRAGSLO_SHIFT)
	+#define I40E_GLPES_PFIP4TXMCOCTSHI(_i) (0x00012004 + ((_i) * 8)) /* _i=0...15 / / Reset: PE_CORER */
	+#define I40E_GLPES_PFIP4TXMCOCTSHI_MAX_INDEX 15
	+#define I40E_GLPES_PFIP4TXMCOCTSHI_IP4TXMCOCTSHI_SHIFT 0
	+#define I40E_GLPES_PFIP4TXMCOCTSHI_IP4TXMCOCTSHI_MASK (0xFFFF << I40E_GLPES_PFIP4TXMCOCTSHI_IP4TXMCOCTSHI_SHIFT)
	+#define I40E_GLPES_PFIP4TXMCOCTSLO(_i) (0x00012000 + ((_i) * 8)) /* _i=0...15 / / Reset: PE_CORER */
	+#define I40E_GLPES_PFIP4TXMCOCTSLO_MAX_INDEX 15
	+#define I40E_GLPES_PFIP4TXMCOCTSLO_IP4TXMCOCTSLO_SHIFT 0
	+#define I40E_GLPES_PFIP4TXMCOCTSLO_IP4TXMCOCTSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFIP4TXMCOCTSLO_IP4TXMCOCTSLO_SHIFT)
	+#define I40E_GLPES_PFIP4TXMCPKTSHI(_i) (0x00012204 + ((_i) * 8)) /* _i=0...15 / / Reset: PE_CORER */
	+#define I40E_GLPES_PFIP4TXMCPKTSHI_MAX_INDEX 15
	+#define I40E_GLPES_PFIP4TXMCPKTSHI_IP4TXMCPKTSHI_SHIFT 0
	+#define I40E_GLPES_PFIP4TXMCPKTSHI_IP4TXMCPKTSHI_MASK (0xFFFF << I40E_GLPES_PFIP4TXMCPKTSHI_IP4TXMCPKTSHI_SHIFT)
	+#define I40E_GLPES_PFIP4TXMCPKTSLO(_i) (0x00012200 + ((_i) * 8)) /* _i=0...15 / / Reset: PE_CORER */
	+#define I40E_GLPES_PFIP4TXMCPKTSLO_MAX_INDEX 15
	+#define I40E_GLPES_PFIP4TXMCPKTSLO_IP4TXMCPKTSLO_SHIFT 0
	+#define I40E_GLPES_PFIP4TXMCPKTSLO_IP4TXMCPKTSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFIP4TXMCPKTSLO_IP4TXMCPKTSLO_SHIFT)
	+#define I40E_GLPES_PFIP4TXNOROUTE(_i) (0x00012E00 + ((_i) * 4)) /* _i=0...15 / / Reset: PE_CORER */
	+#define I40E_GLPES_PFIP4TXNOROUTE_MAX_INDEX 15
	+#define I40E_GLPES_PFIP4TXNOROUTE_IP4TXNOROUTE_SHIFT 0
	+#define I40E_GLPES_PFIP4TXNOROUTE_IP4TXNOROUTE_MASK (0xFFFFFF << I40E_GLPES_PFIP4TXNOROUTE_IP4TXNOROUTE_SHIFT)
	+#define I40E_GLPES_PFIP4TXOCTSHI(_i) (0x00011A04 + ((_i) * 8)) /* _i=0...15 / / Reset: PE_CORER */
	+#define I40E_GLPES_PFIP4TXOCTSHI_MAX_INDEX 15
	+#define I40E_GLPES_PFIP4TXOCTSHI_IP4TXOCTSHI_SHIFT 0
	+#define I40E_GLPES_PFIP4TXOCTSHI_IP4TXOCTSHI_MASK (0xFFFF << I40E_GLPES_PFIP4TXOCTSHI_IP4TXOCTSHI_SHIFT)
	+#define I40E_GLPES_PFIP4TXOCTSLO(_i) (0x00011A00 + ((_i) * 8)) /* _i=0...15 / / Reset: PE_CORER */
	+#define I40E_GLPES_PFIP4TXOCTSLO_MAX_INDEX 15
	+#define I40E_GLPES_PFIP4TXOCTSLO_IP4TXOCTSLO_SHIFT 0
	+#define I40E_GLPES_PFIP4TXOCTSLO_IP4TXOCTSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFIP4TXOCTSLO_IP4TXOCTSLO_SHIFT)
	+#define I40E_GLPES_PFIP4TXPKTSHI(_i) (0x00011C04 + ((_i) * 8)) /* _i=0...15 / / Reset: PE_CORER */
	+#define I40E_GLPES_PFIP4TXPKTSHI_MAX_INDEX 15
	+#define I40E_GLPES_PFIP4TXPKTSHI_IP4TXPKTSHI_SHIFT 0
	+#define I40E_GLPES_PFIP4TXPKTSHI_IP4TXPKTSHI_MASK (0xFFFF << I40E_GLPES_PFIP4TXPKTSHI_IP4TXPKTSHI_SHIFT)
	+#define I40E_GLPES_PFIP4TXPKTSLO(_i) (0x00011C00 + ((_i) * 8)) /* _i=0...15 / / Reset: PE_CORER */
	+#define I40E_GLPES_PFIP4TXPKTSLO_MAX_INDEX 15
	+#define I40E_GLPES_PFIP4TXPKTSLO_IP4TXPKTSLO_SHIFT 0
	+#define I40E_GLPES_PFIP4TXPKTSLO_IP4TXPKTSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFIP4TXPKTSLO_IP4TXPKTSLO_SHIFT)
	+#define I40E_GLPES_PFIP6RXDISCARD(_i) (0x00011200 + ((_i) * 4)) /* _i=0...15 / / Reset: PE_CORER */
	+#define I40E_GLPES_PFIP6RXDISCARD_MAX_INDEX 15
	+#define I40E_GLPES_PFIP6RXDISCARD_IP6RXDISCARD_SHIFT 0
	+#define I40E_GLPES_PFIP6RXDISCARD_IP6RXDISCARD_MASK (0xFFFFFFFF << I40E_GLPES_PFIP6RXDISCARD_IP6RXDISCARD_SHIFT)
	+#define I40E_GLPES_PFIP6RXFRAGSHI(_i) (0x00011404 + ((_i) * 8)) /* _i=0...15 / / Reset: PE_CORER */
	+#define I40E_GLPES_PFIP6RXFRAGSHI_MAX_INDEX 15
	+#define I40E_GLPES_PFIP6RXFRAGSHI_IP6RXFRAGSHI_SHIFT 0
	+#define I40E_GLPES_PFIP6RXFRAGSHI_IP6RXFRAGSHI_MASK (0xFFFF << I40E_GLPES_PFIP6RXFRAGSHI_IP6RXFRAGSHI_SHIFT)
	+#define I40E_GLPES_PFIP6RXFRAGSLO(_i) (0x00011400 + ((_i) * 8)) /* _i=0...15 / / Reset: PE_CORER */
	+#define I40E_GLPES_PFIP6RXFRAGSLO_MAX_INDEX 15
	+#define I40E_GLPES_PFIP6RXFRAGSLO_IP6RXFRAGSLO_SHIFT 0
	+#define I40E_GLPES_PFIP6RXFRAGSLO_IP6RXFRAGSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFIP6RXFRAGSLO_IP6RXFRAGSLO_SHIFT)
	+#define I40E_GLPES_PFIP6RXMCOCTSHI(_i) (0x00011604 + ((_i) * 8)) /* _i=0...15 / / Reset: PE_CORER */
	+#define I40E_GLPES_PFIP6RXMCOCTSHI_MAX_INDEX 15
	+#define I40E_GLPES_PFIP6RXMCOCTSHI_IP6RXMCOCTSHI_SHIFT 0
	+#define I40E_GLPES_PFIP6RXMCOCTSHI_IP6RXMCOCTSHI_MASK (0xFFFF << I40E_GLPES_PFIP6RXMCOCTSHI_IP6RXMCOCTSHI_SHIFT)
	+#define I40E_GLPES_PFIP6RXMCOCTSLO(_i) (0x00011600 + ((_i) * 8)) /* _i=0...15 / / Reset: PE_CORER */
	+#define I40E_GLPES_PFIP6RXMCOCTSLO_MAX_INDEX 15
	+#define I40E_GLPES_PFIP6RXMCOCTSLO_IP6RXMCOCTSLO_SHIFT 0
	+#define I40E_GLPES_PFIP6RXMCOCTSLO_IP6RXMCOCTSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFIP6RXMCOCTSLO_IP6RXMCOCTSLO_SHIFT)
	+#define I40E_GLPES_PFIP6RXMCPKTSHI(_i) (0x00011804 + ((_i) * 8)) /* _i=0...15 / / Reset: PE_CORER */
	+#define I40E_GLPES_PFIP6RXMCPKTSHI_MAX_INDEX 15
	+#define I40E_GLPES_PFIP6RXMCPKTSHI_IP6RXMCPKTSHI_SHIFT 0
	+#define I40E_GLPES_PFIP6RXMCPKTSHI_IP6RXMCPKTSHI_MASK (0xFFFF << I40E_GLPES_PFIP6RXMCPKTSHI_IP6RXMCPKTSHI_SHIFT)
	+#define I40E_GLPES_PFIP6RXMCPKTSLO(_i) (0x00011800 + ((_i) * 8)) /* _i=0...15 / / Reset: PE_CORER */
	+#define I40E_GLPES_PFIP6RXMCPKTSLO_MAX_INDEX 15
	+#define I40E_GLPES_PFIP6RXMCPKTSLO_IP6RXMCPKTSLO_SHIFT 0
	+#define I40E_GLPES_PFIP6RXMCPKTSLO_IP6RXMCPKTSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFIP6RXMCPKTSLO_IP6RXMCPKTSLO_SHIFT)
	+#define I40E_GLPES_PFIP6RXOCTSHI(_i) (0x00010E04 + ((_i) * 8)) /* _i=0...15 / / Reset: PE_CORER */
	+#define I40E_GLPES_PFIP6RXOCTSHI_MAX_INDEX 15
	+#define I40E_GLPES_PFIP6RXOCTSHI_IP6RXOCTSHI_SHIFT 0
	+#define I40E_GLPES_PFIP6RXOCTSHI_IP6RXOCTSHI_MASK (0xFFFF << I40E_GLPES_PFIP6RXOCTSHI_IP6RXOCTSHI_SHIFT)
	+#define I40E_GLPES_PFIP6RXOCTSLO(_i) (0x00010E00 + ((_i) * 8)) /* _i=0...15 / / Reset: PE_CORER */
	+#define I40E_GLPES_PFIP6RXOCTSLO_MAX_INDEX 15
	+#define I40E_GLPES_PFIP6RXOCTSLO_IP6RXOCTSLO_SHIFT 0
	+#define I40E_GLPES_PFIP6RXOCTSLO_IP6RXOCTSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFIP6RXOCTSLO_IP6RXOCTSLO_SHIFT)
	+#define I40E_GLPES_PFIP6RXPKTSHI(_i) (0x00011004 + ((_i) * 8)) /* _i=0...15 / / Reset: PE_CORER */
	+#define I40E_GLPES_PFIP6RXPKTSHI_MAX_INDEX 15
	+#define I40E_GLPES_PFIP6RXPKTSHI_IP6RXPKTSHI_SHIFT 0
	+#define I40E_GLPES_PFIP6RXPKTSHI_IP6RXPKTSHI_MASK (0xFFFF << I40E_GLPES_PFIP6RXPKTSHI_IP6RXPKTSHI_SHIFT)
	+#define I40E_GLPES_PFIP6RXPKTSLO(_i) (0x00011000 + ((_i) * 8)) /* _i=0...15 / / Reset: PE_CORER */
	+#define I40E_GLPES_PFIP6RXPKTSLO_MAX_INDEX 15
	+#define I40E_GLPES_PFIP6RXPKTSLO_IP6RXPKTSLO_SHIFT 0
	+#define I40E_GLPES_PFIP6RXPKTSLO_IP6RXPKTSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFIP6RXPKTSLO_IP6RXPKTSLO_SHIFT)
	+#define I40E_GLPES_PFIP6RXTRUNC(_i) (0x00011300 + ((_i) * 4)) /* _i=0...15 / / Reset: PE_CORER */
	+#define I40E_GLPES_PFIP6RXTRUNC_MAX_INDEX 15
	+#define I40E_GLPES_PFIP6RXTRUNC_IP6RXTRUNC_SHIFT 0
	+#define I40E_GLPES_PFIP6RXTRUNC_IP6RXTRUNC_MASK (0xFFFFFFFF << I40E_GLPES_PFIP6RXTRUNC_IP6RXTRUNC_SHIFT)
	+#define I40E_GLPES_PFIP6TXFRAGSHI(_i) (0x00012804 + ((_i) * 8)) /* _i=0...15 / / Reset: PE_CORER */
	+#define I40E_GLPES_PFIP6TXFRAGSHI_MAX_INDEX 15
	+#define I40E_GLPES_PFIP6TXFRAGSHI_IP6TXFRAGSHI_SHIFT 0
	+#define I40E_GLPES_PFIP6TXFRAGSHI_IP6TXFRAGSHI_MASK (0xFFFF << I40E_GLPES_PFIP6TXFRAGSHI_IP6TXFRAGSHI_SHIFT)
	+#define I40E_GLPES_PFIP6TXFRAGSLO(_i) (0x00012800 + ((_i) * 8)) /* _i=0...15 / / Reset: PE_CORER */
	+#define I40E_GLPES_PFIP6TXFRAGSLO_MAX_INDEX 15
	+#define I40E_GLPES_PFIP6TXFRAGSLO_IP6TXFRAGSLO_SHIFT 0
	+#define I40E_GLPES_PFIP6TXFRAGSLO_IP6TXFRAGSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFIP6TXFRAGSLO_IP6TXFRAGSLO_SHIFT)
	+#define I40E_GLPES_PFIP6TXMCOCTSHI(_i) (0x00012A04 + ((_i) * 8)) /* _i=0...15 / / Reset: PE_CORER */
	+#define I40E_GLPES_PFIP6TXMCOCTSHI_MAX_INDEX 15
	+#define I40E_GLPES_PFIP6TXMCOCTSHI_IP6TXMCOCTSHI_SHIFT 0
	+#define I40E_GLPES_PFIP6TXMCOCTSHI_IP6TXMCOCTSHI_MASK (0xFFFF << I40E_GLPES_PFIP6TXMCOCTSHI_IP6TXMCOCTSHI_SHIFT)
	+#define I40E_GLPES_PFIP6TXMCOCTSLO(_i) (0x00012A00 + ((_i) * 8)) /* _i=0...15 / / Reset: PE_CORER */
	+#define I40E_GLPES_PFIP6TXMCOCTSLO_MAX_INDEX 15
	+#define I40E_GLPES_PFIP6TXMCOCTSLO_IP6TXMCOCTSLO_SHIFT 0
	+#define I40E_GLPES_PFIP6TXMCOCTSLO_IP6TXMCOCTSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFIP6TXMCOCTSLO_IP6TXMCOCTSLO_SHIFT)
	+#define I40E_GLPES_PFIP6TXMCPKTSHI(_i) (0x00012C04 + ((_i) * 8)) /* _i=0...15 / / Reset: PE_CORER */
	+#define I40E_GLPES_PFIP6TXMCPKTSHI_MAX_INDEX 15
	+#define I40E_GLPES_PFIP6TXMCPKTSHI_IP6TXMCPKTSHI_SHIFT 0
	+#define I40E_GLPES_PFIP6TXMCPKTSHI_IP6TXMCPKTSHI_MASK (0xFFFF << I40E_GLPES_PFIP6TXMCPKTSHI_IP6TXMCPKTSHI_SHIFT)
	+#define I40E_GLPES_PFIP6TXMCPKTSLO(_i) (0x00012C00 + ((_i) * 8)) /* _i=0...15 / / Reset: PE_CORER */
	+#define I40E_GLPES_PFIP6TXMCPKTSLO_MAX_INDEX 15
	+#define I40E_GLPES_PFIP6TXMCPKTSLO_IP6TXMCPKTSLO_SHIFT 0
	+#define I40E_GLPES_PFIP6TXMCPKTSLO_IP6TXMCPKTSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFIP6TXMCPKTSLO_IP6TXMCPKTSLO_SHIFT)
	+#define I40E_GLPES_PFIP6TXNOROUTE(_i) (0x00012F00 + ((_i) * 4)) /* _i=0...15 / / Reset: PE_CORER */
	+#define I40E_GLPES_PFIP6TXNOROUTE_MAX_INDEX 15
	+#define I40E_GLPES_PFIP6TXNOROUTE_IP6TXNOROUTE_SHIFT 0
	+#define I40E_GLPES_PFIP6TXNOROUTE_IP6TXNOROUTE_MASK (0xFFFFFF << I40E_GLPES_PFIP6TXNOROUTE_IP6TXNOROUTE_SHIFT)
	+#define I40E_GLPES_PFIP6TXOCTSHI(_i) (0x00012404 + ((_i) * 8)) /* _i=0...15 / / Reset: PE_CORER */
	+#define I40E_GLPES_PFIP6TXOCTSHI_MAX_INDEX 15
	+#define I40E_GLPES_PFIP6TXOCTSHI_IP6TXOCTSHI_SHIFT 0
	+#define I40E_GLPES_PFIP6TXOCTSHI_IP6TXOCTSHI_MASK (0xFFFF << I40E_GLPES_PFIP6TXOCTSHI_IP6TXOCTSHI_SHIFT)
	+#define I40E_GLPES_PFIP6TXOCTSLO(_i) (0x00012400 + ((_i) * 8)) /* _i=0...15 / / Reset: PE_CORER */
	+#define I40E_GLPES_PFIP6TXOCTSLO_MAX_INDEX 15
	+#define I40E_GLPES_PFIP6TXOCTSLO_IP6TXOCTSLO_SHIFT 0
	+#define I40E_GLPES_PFIP6TXOCTSLO_IP6TXOCTSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFIP6TXOCTSLO_IP6TXOCTSLO_SHIFT)
	+#define I40E_GLPES_PFIP6TXPKTSHI(_i) (0x00012604 + ((_i) * 8)) /* _i=0...15 / / Reset: PE_CORER */
	+#define I40E_GLPES_PFIP6TXPKTSHI_MAX_INDEX 15
	+#define I40E_GLPES_PFIP6TXPKTSHI_IP6TXPKTSHI_SHIFT 0
	+#define I40E_GLPES_PFIP6TXPKTSHI_IP6TXPKTSHI_MASK (0xFFFF << I40E_GLPES_PFIP6TXPKTSHI_IP6TXPKTSHI_SHIFT)
	+#define I40E_GLPES_PFIP6TXPKTSLO(_i) (0x00012600 + ((_i) * 8)) /* _i=0...15 / / Reset: PE_CORER */
	+#define I40E_GLPES_PFIP6TXPKTSLO_MAX_INDEX 15
	+#define I40E_GLPES_PFIP6TXPKTSLO_IP6TXPKTSLO_SHIFT 0
	+#define I40E_GLPES_PFIP6TXPKTSLO_IP6TXPKTSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFIP6TXPKTSLO_IP6TXPKTSLO_SHIFT)
	+#define I40E_GLPES_PFRDMARXRDSHI(_i) (0x00013E04 + ((_i) * 8)) /* _i=0...15 / / Reset: PE_CORER */
	+#define I40E_GLPES_PFRDMARXRDSHI_MAX_INDEX 15
	+#define I40E_GLPES_PFRDMARXRDSHI_RDMARXRDSHI_SHIFT 0
	+#define I40E_GLPES_PFRDMARXRDSHI_RDMARXRDSHI_MASK (0xFFFF << I40E_GLPES_PFRDMARXRDSHI_RDMARXRDSHI_SHIFT)
	+#define I40E_GLPES_PFRDMARXRDSLO(_i) (0x00013E00 + ((_i) * 8)) /* _i=0...15 / / Reset: PE_CORER */
	+#define I40E_GLPES_PFRDMARXRDSLO_MAX_INDEX 15
	+#define I40E_GLPES_PFRDMARXRDSLO_RDMARXRDSLO_SHIFT 0
	+#define I40E_GLPES_PFRDMARXRDSLO_RDMARXRDSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFRDMARXRDSLO_RDMARXRDSLO_SHIFT)
	+#define I40E_GLPES_PFRDMARXSNDSHI(_i) (0x00014004 + ((_i) * 8)) /* _i=0...15 / / Reset: PE_CORER */
	+#define I40E_GLPES_PFRDMARXSNDSHI_MAX_INDEX 15
	+#define I40E_GLPES_PFRDMARXSNDSHI_RDMARXSNDSHI_SHIFT 0
	+#define I40E_GLPES_PFRDMARXSNDSHI_RDMARXSNDSHI_MASK (0xFFFF << I40E_GLPES_PFRDMARXSNDSHI_RDMARXSNDSHI_SHIFT)
	+#define I40E_GLPES_PFRDMARXSNDSLO(_i) (0x00014000 + ((_i) * 8)) /* _i=0...15 / / Reset: PE_CORER */
	+#define I40E_GLPES_PFRDMARXSNDSLO_MAX_INDEX 15
	+#define I40E_GLPES_PFRDMARXSNDSLO_RDMARXSNDSLO_SHIFT 0
	+#define I40E_GLPES_PFRDMARXSNDSLO_RDMARXSNDSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFRDMARXSNDSLO_RDMARXSNDSLO_SHIFT)
	+#define I40E_GLPES_PFRDMARXWRSHI(_i) (0x00013C04 + ((_i) * 8)) /* _i=0...15 / / Reset: PE_CORER */
	+#define I40E_GLPES_PFRDMARXWRSHI_MAX_INDEX 15
	+#define I40E_GLPES_PFRDMARXWRSHI_RDMARXWRSHI_SHIFT 0
	+#define I40E_GLPES_PFRDMARXWRSHI_RDMARXWRSHI_MASK (0xFFFF << I40E_GLPES_PFRDMARXWRSHI_RDMARXWRSHI_SHIFT)
	+#define I40E_GLPES_PFRDMARXWRSLO(_i) (0x00013C00 + ((_i) * 8)) /* _i=0...15 / / Reset: PE_CORER */
	+#define I40E_GLPES_PFRDMARXWRSLO_MAX_INDEX 15
	+#define I40E_GLPES_PFRDMARXWRSLO_RDMARXWRSLO_SHIFT 0
	+#define I40E_GLPES_PFRDMARXWRSLO_RDMARXWRSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFRDMARXWRSLO_RDMARXWRSLO_SHIFT)
	+#define I40E_GLPES_PFRDMATXRDSHI(_i) (0x00014404 + ((_i) * 8)) /* _i=0...15 / / Reset: PE_CORER */
	+#define I40E_GLPES_PFRDMATXRDSHI_MAX_INDEX 15
	+#define I40E_GLPES_PFRDMATXRDSHI_RDMARXRDSHI_SHIFT 0
	+#define I40E_GLPES_PFRDMATXRDSHI_RDMARXRDSHI_MASK (0xFFFF << I40E_GLPES_PFRDMATXRDSHI_RDMARXRDSHI_SHIFT)
	+#define I40E_GLPES_PFRDMATXRDSLO(_i) (0x00014400 + ((_i) * 8)) /* _i=0...15 / / Reset: PE_CORER */
	+#define I40E_GLPES_PFRDMATXRDSLO_MAX_INDEX 15
	+#define I40E_GLPES_PFRDMATXRDSLO_RDMARXRDSLO_SHIFT 0
	+#define I40E_GLPES_PFRDMATXRDSLO_RDMARXRDSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFRDMATXRDSLO_RDMARXRDSLO_SHIFT)
	+#define I40E_GLPES_PFRDMATXSNDSHI(_i) (0x00014604 + ((_i) * 8)) /* _i=0...15 / / Reset: PE_CORER */
	+#define I40E_GLPES_PFRDMATXSNDSHI_MAX_INDEX 15
	+#define I40E_GLPES_PFRDMATXSNDSHI_RDMARXSNDSHI_SHIFT 0
	+#define I40E_GLPES_PFRDMATXSNDSHI_RDMARXSNDSHI_MASK (0xFFFF << I40E_GLPES_PFRDMATXSNDSHI_RDMARXSNDSHI_SHIFT)
	+#define I40E_GLPES_PFRDMATXSNDSLO(_i) (0x00014600 + ((_i) * 8)) /* _i=0...15 / / Reset: PE_CORER */
	+#define I40E_GLPES_PFRDMATXSNDSLO_MAX_INDEX 15
	+#define I40E_GLPES_PFRDMATXSNDSLO_RDMARXSNDSLO_SHIFT 0
	+#define I40E_GLPES_PFRDMATXSNDSLO_RDMARXSNDSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFRDMATXSNDSLO_RDMARXSNDSLO_SHIFT)
	+#define I40E_GLPES_PFRDMATXWRSHI(_i) (0x00014204 + ((_i) * 8)) /* _i=0...15 / / Reset: PE_CORER */
	+#define I40E_GLPES_PFRDMATXWRSHI_MAX_INDEX 15
	+#define I40E_GLPES_PFRDMATXWRSHI_RDMARXWRSHI_SHIFT 0
	+#define I40E_GLPES_PFRDMATXWRSHI_RDMARXWRSHI_MASK (0xFFFF << I40E_GLPES_PFRDMATXWRSHI_RDMARXWRSHI_SHIFT)
	+#define I40E_GLPES_PFRDMATXWRSLO(_i) (0x00014200 + ((_i) * 8)) /* _i=0...15 / / Reset: PE_CORER */
	+#define I40E_GLPES_PFRDMATXWRSLO_MAX_INDEX 15
	+#define I40E_GLPES_PFRDMATXWRSLO_RDMARXWRSLO_SHIFT 0
	+#define I40E_GLPES_PFRDMATXWRSLO_RDMARXWRSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFRDMATXWRSLO_RDMARXWRSLO_SHIFT)
	+#define I40E_GLPES_PFRDMAVBNDHI(_i) (0x00014804 + ((_i) * 8)) /* _i=0...15 / / Reset: PE_CORER */
	+#define I40E_GLPES_PFRDMAVBNDHI_MAX_INDEX 15
	+#define I40E_GLPES_PFRDMAVBNDHI_RDMAVBNDHI_SHIFT 0
	+#define I40E_GLPES_PFRDMAVBNDHI_RDMAVBNDHI_MASK (0xFFFFFFFF << I40E_GLPES_PFRDMAVBNDHI_RDMAVBNDHI_SHIFT)
	+#define I40E_GLPES_PFRDMAVBNDLO(_i) (0x00014800 + ((_i) * 8)) /* _i=0...15 / / Reset: PE_CORER */
	+#define I40E_GLPES_PFRDMAVBNDLO_MAX_INDEX 15
	+#define I40E_GLPES_PFRDMAVBNDLO_RDMAVBNDLO_SHIFT 0
	+#define I40E_GLPES_PFRDMAVBNDLO_RDMAVBNDLO_MASK (0xFFFFFFFF << I40E_GLPES_PFRDMAVBNDLO_RDMAVBNDLO_SHIFT)
	+#define I40E_GLPES_PFRDMAVINVHI(_i) (0x00014A04 + ((_i) * 8)) /* _i=0...15 / / Reset: PE_CORER */
	+#define I40E_GLPES_PFRDMAVINVHI_MAX_INDEX 15
	+#define I40E_GLPES_PFRDMAVINVHI_RDMAVINVHI_SHIFT 0
	+#define I40E_GLPES_PFRDMAVINVHI_RDMAVINVHI_MASK (0xFFFFFFFF << I40E_GLPES_PFRDMAVINVHI_RDMAVINVHI_SHIFT)
	+#define I40E_GLPES_PFRDMAVINVLO(_i) (0x00014A00 + ((_i) * 8)) /* _i=0...15 / / Reset: PE_CORER */
	+#define I40E_GLPES_PFRDMAVINVLO_MAX_INDEX 15
	+#define I40E_GLPES_PFRDMAVINVLO_RDMAVINVLO_SHIFT 0
	+#define I40E_GLPES_PFRDMAVINVLO_RDMAVINVLO_MASK (0xFFFFFFFF << I40E_GLPES_PFRDMAVINVLO_RDMAVINVLO_SHIFT)
	+#define I40E_GLPES_PFRXVLANERR(_i) (0x00010000 + ((_i) * 4)) /* _i=0...15 / / Reset: PE_CORER */
	+#define I40E_GLPES_PFRXVLANERR_MAX_INDEX 15
	+#define I40E_GLPES_PFRXVLANERR_RXVLANERR_SHIFT 0
	+#define I40E_GLPES_PFRXVLANERR_RXVLANERR_MASK (0xFFFFFF << I40E_GLPES_PFRXVLANERR_RXVLANERR_SHIFT)
	+#define I40E_GLPES_PFTCPRTXSEG(_i) (0x00013600 + ((_i) * 4)) /* _i=0...15 / / Reset: PE_CORER */
	+#define I40E_GLPES_PFTCPRTXSEG_MAX_INDEX 15
	+#define I40E_GLPES_PFTCPRTXSEG_TCPRTXSEG_SHIFT 0
	+#define I40E_GLPES_PFTCPRTXSEG_TCPRTXSEG_MASK (0xFFFFFFFF << I40E_GLPES_PFTCPRTXSEG_TCPRTXSEG_SHIFT)
	+#define I40E_GLPES_PFTCPRXOPTERR(_i) (0x00013200 + ((_i) * 4)) /* _i=0...15 / / Reset: PE_CORER */
	+#define I40E_GLPES_PFTCPRXOPTERR_MAX_INDEX 15
	+#define I40E_GLPES_PFTCPRXOPTERR_TCPRXOPTERR_SHIFT 0
	+#define I40E_GLPES_PFTCPRXOPTERR_TCPRXOPTERR_MASK (0xFFFFFF << I40E_GLPES_PFTCPRXOPTERR_TCPRXOPTERR_SHIFT)
	+#define I40E_GLPES_PFTCPRXPROTOERR(_i) (0x00013300 + ((_i) * 4)) /* _i=0...15 / / Reset: PE_CORER */
	+#define I40E_GLPES_PFTCPRXPROTOERR_MAX_INDEX 15
	+#define I40E_GLPES_PFTCPRXPROTOERR_TCPRXPROTOERR_SHIFT 0
	+#define I40E_GLPES_PFTCPRXPROTOERR_TCPRXPROTOERR_MASK (0xFFFFFF << I40E_GLPES_PFTCPRXPROTOERR_TCPRXPROTOERR_SHIFT)
	+#define I40E_GLPES_PFTCPRXSEGSHI(_i) (0x00013004 + ((_i) * 8)) /* _i=0...15 / / Reset: PE_CORER */
	+#define I40E_GLPES_PFTCPRXSEGSHI_MAX_INDEX 15
	+#define I40E_GLPES_PFTCPRXSEGSHI_TCPRXSEGSHI_SHIFT 0
	+#define I40E_GLPES_PFTCPRXSEGSHI_TCPRXSEGSHI_MASK (0xFFFF << I40E_GLPES_PFTCPRXSEGSHI_TCPRXSEGSHI_SHIFT)
	+#define I40E_GLPES_PFTCPRXSEGSLO(_i) (0x00013000 + ((_i) * 8)) /* _i=0...15 / / Reset: PE_CORER */
	+#define I40E_GLPES_PFTCPRXSEGSLO_MAX_INDEX 15
	+#define I40E_GLPES_PFTCPRXSEGSLO_TCPRXSEGSLO_SHIFT 0
	+#define I40E_GLPES_PFTCPRXSEGSLO_TCPRXSEGSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFTCPRXSEGSLO_TCPRXSEGSLO_SHIFT)
	+#define I40E_GLPES_PFTCPTXSEGHI(_i) (0x00013404 + ((_i) * 8)) /* _i=0...15 / / Reset: PE_CORER */
	+#define I40E_GLPES_PFTCPTXSEGHI_MAX_INDEX 15
	+#define I40E_GLPES_PFTCPTXSEGHI_TCPTXSEGHI_SHIFT 0
	+#define I40E_GLPES_PFTCPTXSEGHI_TCPTXSEGHI_MASK (0xFFFF << I40E_GLPES_PFTCPTXSEGHI_TCPTXSEGHI_SHIFT)
	+#define I40E_GLPES_PFTCPTXSEGLO(_i) (0x00013400 + ((_i) * 8)) /* _i=0...15 / / Reset: PE_CORER */
	+#define I40E_GLPES_PFTCPTXSEGLO_MAX_INDEX 15
	+#define I40E_GLPES_PFTCPTXSEGLO_TCPTXSEGLO_SHIFT 0
	+#define I40E_GLPES_PFTCPTXSEGLO_TCPTXSEGLO_MASK (0xFFFFFFFF << I40E_GLPES_PFTCPTXSEGLO_TCPTXSEGLO_SHIFT)
	+#define I40E_GLPES_PFUDPRXPKTSHI(_i) (0x00013804 + ((_i) * 8)) /* _i=0...15 / / Reset: PE_CORER */
	+#define I40E_GLPES_PFUDPRXPKTSHI_MAX_INDEX 15
	+#define I40E_GLPES_PFUDPRXPKTSHI_UDPRXPKTSHI_SHIFT 0
	+#define I40E_GLPES_PFUDPRXPKTSHI_UDPRXPKTSHI_MASK (0xFFFF << I40E_GLPES_PFUDPRXPKTSHI_UDPRXPKTSHI_SHIFT)
	+#define I40E_GLPES_PFUDPRXPKTSLO(_i) (0x00013800 + ((_i) * 8)) /* _i=0...15 / / Reset: PE_CORER */
	+#define I40E_GLPES_PFUDPRXPKTSLO_MAX_INDEX 15
	+#define I40E_GLPES_PFUDPRXPKTSLO_UDPRXPKTSLO_SHIFT 0
	+#define I40E_GLPES_PFUDPRXPKTSLO_UDPRXPKTSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFUDPRXPKTSLO_UDPRXPKTSLO_SHIFT)
	+#define I40E_GLPES_PFUDPTXPKTSHI(_i) (0x00013A04 + ((_i) * 8)) /* _i=0...15 / / Reset: PE_CORER */
	+#define I40E_GLPES_PFUDPTXPKTSHI_MAX_INDEX 15
	+#define I40E_GLPES_PFUDPTXPKTSHI_UDPTXPKTSHI_SHIFT 0
	+#define I40E_GLPES_PFUDPTXPKTSHI_UDPTXPKTSHI_MASK (0xFFFF << I40E_GLPES_PFUDPTXPKTSHI_UDPTXPKTSHI_SHIFT)
	+#define I40E_GLPES_PFUDPTXPKTSLO(_i) (0x00013A00 + ((_i) * 8)) /* _i=0...15 / / Reset: PE_CORER */
	+#define I40E_GLPES_PFUDPTXPKTSLO_MAX_INDEX 15
	+#define I40E_GLPES_PFUDPTXPKTSLO_UDPTXPKTSLO_SHIFT 0
	+#define I40E_GLPES_PFUDPTXPKTSLO_UDPTXPKTSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFUDPTXPKTSLO_UDPTXPKTSLO_SHIFT)
	+#define I40E_GLPES_RDMARXMULTFPDUSHI 0x0001E014 /* Reset: PE_CORER */
	+#define I40E_GLPES_RDMARXMULTFPDUSHI_RDMARXMULTFPDUSHI_SHIFT 0
	+#define I40E_GLPES_RDMARXMULTFPDUSHI_RDMARXMULTFPDUSHI_MASK (0xFFFFFF << I40E_GLPES_RDMARXMULTFPDUSHI_RDMARXMULTFPDUSHI_SHIFT)
	+#define I40E_GLPES_RDMARXMULTFPDUSLO 0x0001E010 /* Reset: PE_CORER */
	+#define I40E_GLPES_RDMARXMULTFPDUSLO_RDMARXMULTFPDUSLO_SHIFT 0
	+#define I40E_GLPES_RDMARXMULTFPDUSLO_RDMARXMULTFPDUSLO_MASK (0xFFFFFFFF << I40E_GLPES_RDMARXMULTFPDUSLO_RDMARXMULTFPDUSLO_SHIFT)
	+#define I40E_GLPES_RDMARXOOODDPHI 0x0001E01C /* Reset: PE_CORER */
	+#define I40E_GLPES_RDMARXOOODDPHI_RDMARXOOODDPHI_SHIFT 0
	+#define I40E_GLPES_RDMARXOOODDPHI_RDMARXOOODDPHI_MASK (0xFFFFFF << I40E_GLPES_RDMARXOOODDPHI_RDMARXOOODDPHI_SHIFT)
	+#define I40E_GLPES_RDMARXOOODDPLO 0x0001E018 /* Reset: PE_CORER */
	+#define I40E_GLPES_RDMARXOOODDPLO_RDMARXOOODDPLO_SHIFT 0
	+#define I40E_GLPES_RDMARXOOODDPLO_RDMARXOOODDPLO_MASK (0xFFFFFFFF << I40E_GLPES_RDMARXOOODDPLO_RDMARXOOODDPLO_SHIFT)
	+#define I40E_GLPES_RDMARXOOONOMARK 0x0001E004 /* Reset: PE_CORER */
	+#define I40E_GLPES_RDMARXOOONOMARK_RDMAOOONOMARK_SHIFT 0
	+#define I40E_GLPES_RDMARXOOONOMARK_RDMAOOONOMARK_MASK (0xFFFFFFFF << I40E_GLPES_RDMARXOOONOMARK_RDMAOOONOMARK_SHIFT)
	+#define I40E_GLPES_RDMARXUNALIGN 0x0001E000 /* Reset: PE_CORER */
	+#define I40E_GLPES_RDMARXUNALIGN_RDMRXAUNALIGN_SHIFT 0
	+#define I40E_GLPES_RDMARXUNALIGN_RDMRXAUNALIGN_MASK (0xFFFFFFFF << I40E_GLPES_RDMARXUNALIGN_RDMRXAUNALIGN_SHIFT)
	+#define I40E_GLPES_TCPRXFOURHOLEHI 0x0001E044 /* Reset: PE_CORER */
	+#define I40E_GLPES_TCPRXFOURHOLEHI_TCPRXFOURHOLEHI_SHIFT 0
	+#define I40E_GLPES_TCPRXFOURHOLEHI_TCPRXFOURHOLEHI_MASK (0xFFFFFF << I40E_GLPES_TCPRXFOURHOLEHI_TCPRXFOURHOLEHI_SHIFT)
	+#define I40E_GLPES_TCPRXFOURHOLELO 0x0001E040 /* Reset: PE_CORER */
	+#define I40E_GLPES_TCPRXFOURHOLELO_TCPRXFOURHOLELO_SHIFT 0
	+#define I40E_GLPES_TCPRXFOURHOLELO_TCPRXFOURHOLELO_MASK (0xFFFFFFFF << I40E_GLPES_TCPRXFOURHOLELO_TCPRXFOURHOLELO_SHIFT)
	+#define I40E_GLPES_TCPRXONEHOLEHI 0x0001E02C /* Reset: PE_CORER */
	+#define I40E_GLPES_TCPRXONEHOLEHI_TCPRXONEHOLEHI_SHIFT 0
	+#define I40E_GLPES_TCPRXONEHOLEHI_TCPRXONEHOLEHI_MASK (0xFFFFFF << I40E_GLPES_TCPRXONEHOLEHI_TCPRXONEHOLEHI_SHIFT)
	+#define I40E_GLPES_TCPRXONEHOLELO 0x0001E028 /* Reset: PE_CORER */
	+#define I40E_GLPES_TCPRXONEHOLELO_TCPRXONEHOLELO_SHIFT 0
	+#define I40E_GLPES_TCPRXONEHOLELO_TCPRXONEHOLELO_MASK (0xFFFFFFFF << I40E_GLPES_TCPRXONEHOLELO_TCPRXONEHOLELO_SHIFT)
	+#define I40E_GLPES_TCPRXPUREACKHI 0x0001E024 /* Reset: PE_CORER */
	+#define I40E_GLPES_TCPRXPUREACKHI_TCPRXPUREACKSHI_SHIFT 0
	+#define I40E_GLPES_TCPRXPUREACKHI_TCPRXPUREACKSHI_MASK (0xFFFFFF << I40E_GLPES_TCPRXPUREACKHI_TCPRXPUREACKSHI_SHIFT)
	+#define I40E_GLPES_TCPRXPUREACKSLO 0x0001E020 /* Reset: PE_CORER */
	+#define I40E_GLPES_TCPRXPUREACKSLO_TCPRXPUREACKLO_SHIFT 0
	+#define I40E_GLPES_TCPRXPUREACKSLO_TCPRXPUREACKLO_MASK (0xFFFFFFFF << I40E_GLPES_TCPRXPUREACKSLO_TCPRXPUREACKLO_SHIFT)
	+#define I40E_GLPES_TCPRXTHREEHOLEHI 0x0001E03C /* Reset: PE_CORER */
	+#define I40E_GLPES_TCPRXTHREEHOLEHI_TCPRXTHREEHOLEHI_SHIFT 0
	+#define I40E_GLPES_TCPRXTHREEHOLEHI_TCPRXTHREEHOLEHI_MASK (0xFFFFFF << I40E_GLPES_TCPRXTHREEHOLEHI_TCPRXTHREEHOLEHI_SHIFT)
	+#define I40E_GLPES_TCPRXTHREEHOLELO 0x0001E038 /* Reset: PE_CORER */
	+#define I40E_GLPES_TCPRXTHREEHOLELO_TCPRXTHREEHOLELO_SHIFT 0
	+#define I40E_GLPES_TCPRXTHREEHOLELO_TCPRXTHREEHOLELO_MASK (0xFFFFFFFF << I40E_GLPES_TCPRXTHREEHOLELO_TCPRXTHREEHOLELO_SHIFT)
	+#define I40E_GLPES_TCPRXTWOHOLEHI 0x0001E034 /* Reset: PE_CORER */
	+#define I40E_GLPES_TCPRXTWOHOLEHI_TCPRXTWOHOLEHI_SHIFT 0
	+#define I40E_GLPES_TCPRXTWOHOLEHI_TCPRXTWOHOLEHI_MASK (0xFFFFFF << I40E_GLPES_TCPRXTWOHOLEHI_TCPRXTWOHOLEHI_SHIFT)
	+#define I40E_GLPES_TCPRXTWOHOLELO 0x0001E030 /* Reset: PE_CORER */
	+#define I40E_GLPES_TCPRXTWOHOLELO_TCPRXTWOHOLELO_SHIFT 0
	+#define I40E_GLPES_TCPRXTWOHOLELO_TCPRXTWOHOLELO_MASK (0xFFFFFFFF << I40E_GLPES_TCPRXTWOHOLELO_TCPRXTWOHOLELO_SHIFT)
	+#define I40E_GLPES_TCPTXRETRANSFASTHI 0x0001E04C /* Reset: PE_CORER */
	+#define I40E_GLPES_TCPTXRETRANSFASTHI_TCPTXRETRANSFASTHI_SHIFT 0
	+#define I40E_GLPES_TCPTXRETRANSFASTHI_TCPTXRETRANSFASTHI_MASK (0xFFFFFF << I40E_GLPES_TCPTXRETRANSFASTHI_TCPTXRETRANSFASTHI_SHIFT)
	+#define I40E_GLPES_TCPTXRETRANSFASTLO 0x0001E048 /* Reset: PE_CORER */
	+#define I40E_GLPES_TCPTXRETRANSFASTLO_TCPTXRETRANSFASTLO_SHIFT 0
	+#define I40E_GLPES_TCPTXRETRANSFASTLO_TCPTXRETRANSFASTLO_MASK (0xFFFFFFFF << I40E_GLPES_TCPTXRETRANSFASTLO_TCPTXRETRANSFASTLO_SHIFT)
	+#define I40E_GLPES_TCPTXTOUTSFASTHI 0x0001E054 /* Reset: PE_CORER */
	+#define I40E_GLPES_TCPTXTOUTSFASTHI_TCPTXTOUTSFASTHI_SHIFT 0
	+#define I40E_GLPES_TCPTXTOUTSFASTHI_TCPTXTOUTSFASTHI_MASK (0xFFFFFF << I40E_GLPES_TCPTXTOUTSFASTHI_TCPTXTOUTSFASTHI_SHIFT)
	+#define I40E_GLPES_TCPTXTOUTSFASTLO 0x0001E050 /* Reset: PE_CORER */
	+#define I40E_GLPES_TCPTXTOUTSFASTLO_TCPTXTOUTSFASTLO_SHIFT 0
	+#define I40E_GLPES_TCPTXTOUTSFASTLO_TCPTXTOUTSFASTLO_MASK (0xFFFFFFFF << I40E_GLPES_TCPTXTOUTSFASTLO_TCPTXTOUTSFASTLO_SHIFT)
	+#define I40E_GLPES_TCPTXTOUTSHI 0x0001E05C /* Reset: PE_CORER */
	+#define I40E_GLPES_TCPTXTOUTSHI_TCPTXTOUTSHI_SHIFT 0
	+#define I40E_GLPES_TCPTXTOUTSHI_TCPTXTOUTSHI_MASK (0xFFFFFF << I40E_GLPES_TCPTXTOUTSHI_TCPTXTOUTSHI_SHIFT)
	+#define I40E_GLPES_TCPTXTOUTSLO 0x0001E058 /* Reset: PE_CORER */
	+#define I40E_GLPES_TCPTXTOUTSLO_TCPTXTOUTSLO_SHIFT 0
	+#define I40E_GLPES_TCPTXTOUTSLO_TCPTXTOUTSLO_MASK (0xFFFFFFFF << I40E_GLPES_TCPTXTOUTSLO_TCPTXTOUTSLO_SHIFT)
	+#define I40E_GLPES_VFIP4RXDISCARD(_i) (0x00018600 + ((_i) * 4)) /* _i=0...31 / / Reset: PE_CORER */
	+#define I40E_GLPES_VFIP4RXDISCARD_MAX_INDEX 31
	+#define I40E_GLPES_VFIP4RXDISCARD_IP4RXDISCARD_SHIFT 0
	+#define I40E_GLPES_VFIP4RXDISCARD_IP4RXDISCARD_MASK (0xFFFFFFFF << I40E_GLPES_VFIP4RXDISCARD_IP4RXDISCARD_SHIFT)
	+#define I40E_GLPES_VFIP4RXFRAGSHI(_i) (0x00018804 + ((_i) * 8)) /* _i=0...31 / / Reset: PE_CORER */
	+#define I40E_GLPES_VFIP4RXFRAGSHI_MAX_INDEX 31
	+#define I40E_GLPES_VFIP4RXFRAGSHI_IP4RXFRAGSHI_SHIFT 0
	+#define I40E_GLPES_VFIP4RXFRAGSHI_IP4RXFRAGSHI_MASK (0xFFFF << I40E_GLPES_VFIP4RXFRAGSHI_IP4RXFRAGSHI_SHIFT)
	+#define I40E_GLPES_VFIP4RXFRAGSLO(_i) (0x00018800 + ((_i) * 8)) /* _i=0...31 / / Reset: PE_CORER */
	+#define I40E_GLPES_VFIP4RXFRAGSLO_MAX_INDEX 31
	+#define I40E_GLPES_VFIP4RXFRAGSLO_IP4RXFRAGSLO_SHIFT 0
	+#define I40E_GLPES_VFIP4RXFRAGSLO_IP4RXFRAGSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFIP4RXFRAGSLO_IP4RXFRAGSLO_SHIFT)
	+#define I40E_GLPES_VFIP4RXMCOCTSHI(_i) (0x00018A04 + ((_i) * 8)) /* _i=0...31 / / Reset: PE_CORER */
	+#define I40E_GLPES_VFIP4RXMCOCTSHI_MAX_INDEX 31
	+#define I40E_GLPES_VFIP4RXMCOCTSHI_IP4RXMCOCTSHI_SHIFT 0
	+#define I40E_GLPES_VFIP4RXMCOCTSHI_IP4RXMCOCTSHI_MASK (0xFFFF << I40E_GLPES_VFIP4RXMCOCTSHI_IP4RXMCOCTSHI_SHIFT)
	+#define I40E_GLPES_VFIP4RXMCOCTSLO(_i) (0x00018A00 + ((_i) * 8)) /* _i=0...31 / / Reset: PE_CORER */
	+#define I40E_GLPES_VFIP4RXMCOCTSLO_MAX_INDEX 31
	+#define I40E_GLPES_VFIP4RXMCOCTSLO_IP4RXMCOCTSLO_SHIFT 0
	+#define I40E_GLPES_VFIP4RXMCOCTSLO_IP4RXMCOCTSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFIP4RXMCOCTSLO_IP4RXMCOCTSLO_SHIFT)
	+#define I40E_GLPES_VFIP4RXMCPKTSHI(_i) (0x00018C04 + ((_i) * 8)) /* _i=0...31 / / Reset: PE_CORER */
	+#define I40E_GLPES_VFIP4RXMCPKTSHI_MAX_INDEX 31
	+#define I40E_GLPES_VFIP4RXMCPKTSHI_IP4RXMCPKTSHI_SHIFT 0
	+#define I40E_GLPES_VFIP4RXMCPKTSHI_IP4RXMCPKTSHI_MASK (0xFFFF << I40E_GLPES_VFIP4RXMCPKTSHI_IP4RXMCPKTSHI_SHIFT)
	+#define I40E_GLPES_VFIP4RXMCPKTSLO(_i) (0x00018C00 + ((_i) * 8)) /* _i=0...31 / / Reset: PE_CORER */
	+#define I40E_GLPES_VFIP4RXMCPKTSLO_MAX_INDEX 31
	+#define I40E_GLPES_VFIP4RXMCPKTSLO_IP4RXMCPKTSLO_SHIFT 0
	+#define I40E_GLPES_VFIP4RXMCPKTSLO_IP4RXMCPKTSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFIP4RXMCPKTSLO_IP4RXMCPKTSLO_SHIFT)
	+#define I40E_GLPES_VFIP4RXOCTSHI(_i) (0x00018204 + ((_i) * 8)) /* _i=0...31 / / Reset: PE_CORER */
	+#define I40E_GLPES_VFIP4RXOCTSHI_MAX_INDEX 31
	+#define I40E_GLPES_VFIP4RXOCTSHI_IP4RXOCTSHI_SHIFT 0
	+#define I40E_GLPES_VFIP4RXOCTSHI_IP4RXOCTSHI_MASK (0xFFFF << I40E_GLPES_VFIP4RXOCTSHI_IP4RXOCTSHI_SHIFT)
	+#define I40E_GLPES_VFIP4RXOCTSLO(_i) (0x00018200 + ((_i) * 8)) /* _i=0...31 / / Reset: PE_CORER */
	+#define I40E_GLPES_VFIP4RXOCTSLO_MAX_INDEX 31
	+#define I40E_GLPES_VFIP4RXOCTSLO_IP4RXOCTSLO_SHIFT 0
	+#define I40E_GLPES_VFIP4RXOCTSLO_IP4RXOCTSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFIP4RXOCTSLO_IP4RXOCTSLO_SHIFT)
	+#define I40E_GLPES_VFIP4RXPKTSHI(_i) (0x00018404 + ((_i) * 8)) /* _i=0...31 / / Reset: PE_CORER */
	+#define I40E_GLPES_VFIP4RXPKTSHI_MAX_INDEX 31
	+#define I40E_GLPES_VFIP4RXPKTSHI_IP4RXPKTSHI_SHIFT 0
	+#define I40E_GLPES_VFIP4RXPKTSHI_IP4RXPKTSHI_MASK (0xFFFF << I40E_GLPES_VFIP4RXPKTSHI_IP4RXPKTSHI_SHIFT)
	+#define I40E_GLPES_VFIP4RXPKTSLO(_i) (0x00018400 + ((_i) * 8)) /* _i=0...31 / / Reset: PE_CORER */
	+#define I40E_GLPES_VFIP4RXPKTSLO_MAX_INDEX 31
	+#define I40E_GLPES_VFIP4RXPKTSLO_IP4RXPKTSLO_SHIFT 0
	+#define I40E_GLPES_VFIP4RXPKTSLO_IP4RXPKTSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFIP4RXPKTSLO_IP4RXPKTSLO_SHIFT)
	+#define I40E_GLPES_VFIP4RXTRUNC(_i) (0x00018700 + ((_i) * 4)) /* _i=0...31 / / Reset: PE_CORER */
	+#define I40E_GLPES_VFIP4RXTRUNC_MAX_INDEX 31
	+#define I40E_GLPES_VFIP4RXTRUNC_IP4RXTRUNC_SHIFT 0
	+#define I40E_GLPES_VFIP4RXTRUNC_IP4RXTRUNC_MASK (0xFFFFFFFF << I40E_GLPES_VFIP4RXTRUNC_IP4RXTRUNC_SHIFT)
	+#define I40E_GLPES_VFIP4TXFRAGSHI(_i) (0x00019E04 + ((_i) * 8)) /* _i=0...31 / / Reset: PE_CORER */
	+#define I40E_GLPES_VFIP4TXFRAGSHI_MAX_INDEX 31
	+#define I40E_GLPES_VFIP4TXFRAGSHI_IP4TXFRAGSHI_SHIFT 0
	+#define I40E_GLPES_VFIP4TXFRAGSHI_IP4TXFRAGSHI_MASK (0xFFFF << I40E_GLPES_VFIP4TXFRAGSHI_IP4TXFRAGSHI_SHIFT)
	+#define I40E_GLPES_VFIP4TXFRAGSLO(_i) (0x00019E00 + ((_i) * 8)) /* _i=0...31 / / Reset: PE_CORER */
	+#define I40E_GLPES_VFIP4TXFRAGSLO_MAX_INDEX 31
	+#define I40E_GLPES_VFIP4TXFRAGSLO_IP4TXFRAGSLO_SHIFT 0
	+#define I40E_GLPES_VFIP4TXFRAGSLO_IP4TXFRAGSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFIP4TXFRAGSLO_IP4TXFRAGSLO_SHIFT)
	+#define I40E_GLPES_VFIP4TXMCOCTSHI(_i) (0x0001A004 + ((_i) * 8)) /* _i=0...31 / / Reset: PE_CORER */
	+#define I40E_GLPES_VFIP4TXMCOCTSHI_MAX_INDEX 31
	+#define I40E_GLPES_VFIP4TXMCOCTSHI_IP4TXMCOCTSHI_SHIFT 0
	+#define I40E_GLPES_VFIP4TXMCOCTSHI_IP4TXMCOCTSHI_MASK (0xFFFF << I40E_GLPES_VFIP4TXMCOCTSHI_IP4TXMCOCTSHI_SHIFT)
	+#define I40E_GLPES_VFIP4TXMCOCTSLO(_i) (0x0001A000 + ((_i) * 8)) /* _i=0...31 / / Reset: PE_CORER */
	+#define I40E_GLPES_VFIP4TXMCOCTSLO_MAX_INDEX 31
	+#define I40E_GLPES_VFIP4TXMCOCTSLO_IP4TXMCOCTSLO_SHIFT 0
	+#define I40E_GLPES_VFIP4TXMCOCTSLO_IP4TXMCOCTSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFIP4TXMCOCTSLO_IP4TXMCOCTSLO_SHIFT)
	+#define I40E_GLPES_VFIP4TXMCPKTSHI(_i) (0x0001A204 + ((_i) * 8)) /* _i=0...31 / / Reset: PE_CORER */
	+#define I40E_GLPES_VFIP4TXMCPKTSHI_MAX_INDEX 31
	+#define I40E_GLPES_VFIP4TXMCPKTSHI_IP4TXMCPKTSHI_SHIFT 0
	+#define I40E_GLPES_VFIP4TXMCPKTSHI_IP4TXMCPKTSHI_MASK (0xFFFF << I40E_GLPES_VFIP4TXMCPKTSHI_IP4TXMCPKTSHI_SHIFT)
	+#define I40E_GLPES_VFIP4TXMCPKTSLO(_i) (0x0001A200 + ((_i) * 8)) /* _i=0...31 / / Reset: PE_CORER */
	+#define I40E_GLPES_VFIP4TXMCPKTSLO_MAX_INDEX 31
	+#define I40E_GLPES_VFIP4TXMCPKTSLO_IP4TXMCPKTSLO_SHIFT 0
	+#define I40E_GLPES_VFIP4TXMCPKTSLO_IP4TXMCPKTSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFIP4TXMCPKTSLO_IP4TXMCPKTSLO_SHIFT)
	+#define I40E_GLPES_VFIP4TXNOROUTE(_i) (0x0001AE00 + ((_i) * 4)) /* _i=0...31 / / Reset: PE_CORER */
	+#define I40E_GLPES_VFIP4TXNOROUTE_MAX_INDEX 31
	+#define I40E_GLPES_VFIP4TXNOROUTE_IP4TXNOROUTE_SHIFT 0
	+#define I40E_GLPES_VFIP4TXNOROUTE_IP4TXNOROUTE_MASK (0xFFFFFF << I40E_GLPES_VFIP4TXNOROUTE_IP4TXNOROUTE_SHIFT)
	+#define I40E_GLPES_VFIP4TXOCTSHI(_i) (0x00019A04 + ((_i) * 8)) /* _i=0...31 / / Reset: PE_CORER */
	+#define I40E_GLPES_VFIP4TXOCTSHI_MAX_INDEX 31
	+#define I40E_GLPES_VFIP4TXOCTSHI_IP4TXOCTSHI_SHIFT 0
	+#define I40E_GLPES_VFIP4TXOCTSHI_IP4TXOCTSHI_MASK (0xFFFF << I40E_GLPES_VFIP4TXOCTSHI_IP4TXOCTSHI_SHIFT)
	+#define I40E_GLPES_VFIP4TXOCTSLO(_i) (0x00019A00 + ((_i) * 8)) /* _i=0...31 / / Reset: PE_CORER */
	+#define I40E_GLPES_VFIP4TXOCTSLO_MAX_INDEX 31
	+#define I40E_GLPES_VFIP4TXOCTSLO_IP4TXOCTSLO_SHIFT 0
	+#define I40E_GLPES_VFIP4TXOCTSLO_IP4TXOCTSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFIP4TXOCTSLO_IP4TXOCTSLO_SHIFT)
	+#define I40E_GLPES_VFIP4TXPKTSHI(_i) (0x00019C04 + ((_i) * 8)) /* _i=0...31 / / Reset: PE_CORER */
	+#define I40E_GLPES_VFIP4TXPKTSHI_MAX_INDEX 31
	+#define I40E_GLPES_VFIP4TXPKTSHI_IP4TXPKTSHI_SHIFT 0
	+#define I40E_GLPES_VFIP4TXPKTSHI_IP4TXPKTSHI_MASK (0xFFFF << I40E_GLPES_VFIP4TXPKTSHI_IP4TXPKTSHI_SHIFT)
	+#define I40E_GLPES_VFIP4TXPKTSLO(_i) (0x00019C00 + ((_i) * 8)) /* _i=0...31 / / Reset: PE_CORER */
	+#define I40E_GLPES_VFIP4TXPKTSLO_MAX_INDEX 31
	+#define I40E_GLPES_VFIP4TXPKTSLO_IP4TXPKTSLO_SHIFT 0
	+#define I40E_GLPES_VFIP4TXPKTSLO_IP4TXPKTSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFIP4TXPKTSLO_IP4TXPKTSLO_SHIFT)
	+#define I40E_GLPES_VFIP6RXDISCARD(_i) (0x00019200 + ((_i) * 4)) /* _i=0...31 / / Reset: PE_CORER */
	+#define I40E_GLPES_VFIP6RXDISCARD_MAX_INDEX 31
	+#define I40E_GLPES_VFIP6RXDISCARD_IP6RXDISCARD_SHIFT 0
	+#define I40E_GLPES_VFIP6RXDISCARD_IP6RXDISCARD_MASK (0xFFFFFFFF << I40E_GLPES_VFIP6RXDISCARD_IP6RXDISCARD_SHIFT)
	+#define I40E_GLPES_VFIP6RXFRAGSHI(_i) (0x00019404 + ((_i) * 8)) /* _i=0...31 / / Reset: PE_CORER */
	+#define I40E_GLPES_VFIP6RXFRAGSHI_MAX_INDEX 31
	+#define I40E_GLPES_VFIP6RXFRAGSHI_IP6RXFRAGSHI_SHIFT 0
	+#define I40E_GLPES_VFIP6RXFRAGSHI_IP6RXFRAGSHI_MASK (0xFFFF << I40E_GLPES_VFIP6RXFRAGSHI_IP6RXFRAGSHI_SHIFT)
	+#define I40E_GLPES_VFIP6RXFRAGSLO(_i) (0x00019400 + ((_i) * 8)) /* _i=0...31 / / Reset: PE_CORER */
	+#define I40E_GLPES_VFIP6RXFRAGSLO_MAX_INDEX 31
	+#define I40E_GLPES_VFIP6RXFRAGSLO_IP6RXFRAGSLO_SHIFT 0
	+#define I40E_GLPES_VFIP6RXFRAGSLO_IP6RXFRAGSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFIP6RXFRAGSLO_IP6RXFRAGSLO_SHIFT)
	+#define I40E_GLPES_VFIP6RXMCOCTSHI(_i) (0x00019604 + ((_i) * 8)) /* _i=0...31 / / Reset: PE_CORER */
	+#define I40E_GLPES_VFIP6RXMCOCTSHI_MAX_INDEX 31
	+#define I40E_GLPES_VFIP6RXMCOCTSHI_IP6RXMCOCTSHI_SHIFT 0
	+#define I40E_GLPES_VFIP6RXMCOCTSHI_IP6RXMCOCTSHI_MASK (0xFFFF << I40E_GLPES_VFIP6RXMCOCTSHI_IP6RXMCOCTSHI_SHIFT)
	+#define I40E_GLPES_VFIP6RXMCOCTSLO(_i) (0x00019600 + ((_i) * 8)) /* _i=0...31 / / Reset: PE_CORER */
	+#define I40E_GLPES_VFIP6RXMCOCTSLO_MAX_INDEX 31
	+#define I40E_GLPES_VFIP6RXMCOCTSLO_IP6RXMCOCTSLO_SHIFT 0
	+#define I40E_GLPES_VFIP6RXMCOCTSLO_IP6RXMCOCTSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFIP6RXMCOCTSLO_IP6RXMCOCTSLO_SHIFT)
	+#define I40E_GLPES_VFIP6RXMCPKTSHI(_i) (0x00019804 + ((_i) * 8)) /* _i=0...31 / / Reset: PE_CORER */
	+#define I40E_GLPES_VFIP6RXMCPKTSHI_MAX_INDEX 31
	+#define I40E_GLPES_VFIP6RXMCPKTSHI_IP6RXMCPKTSHI_SHIFT 0
	+#define I40E_GLPES_VFIP6RXMCPKTSHI_IP6RXMCPKTSHI_MASK (0xFFFF << I40E_GLPES_VFIP6RXMCPKTSHI_IP6RXMCPKTSHI_SHIFT)
	+#define I40E_GLPES_VFIP6RXMCPKTSLO(_i) (0x00019800 + ((_i) * 8)) /* _i=0...31 / / Reset: PE_CORER */
	+#define I40E_GLPES_VFIP6RXMCPKTSLO_MAX_INDEX 31
	+#define I40E_GLPES_VFIP6RXMCPKTSLO_IP6RXMCPKTSLO_SHIFT 0
	+#define I40E_GLPES_VFIP6RXMCPKTSLO_IP6RXMCPKTSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFIP6RXMCPKTSLO_IP6RXMCPKTSLO_SHIFT)
	+#define I40E_GLPES_VFIP6RXOCTSHI(_i) (0x00018E04 + ((_i) * 8)) /* _i=0...31 / / Reset: PE_CORER */
	+#define I40E_GLPES_VFIP6RXOCTSHI_MAX_INDEX 31
	+#define I40E_GLPES_VFIP6RXOCTSHI_IP6RXOCTSHI_SHIFT 0
	+#define I40E_GLPES_VFIP6RXOCTSHI_IP6RXOCTSHI_MASK (0xFFFF << I40E_GLPES_VFIP6RXOCTSHI_IP6RXOCTSHI_SHIFT)
	+#define I40E_GLPES_VFIP6RXOCTSLO(_i) (0x00018E00 + ((_i) * 8)) /* _i=0...31 / / Reset: PE_CORER */
	+#define I40E_GLPES_VFIP6RXOCTSLO_MAX_INDEX 31
	+#define I40E_GLPES_VFIP6RXOCTSLO_IP6RXOCTSLO_SHIFT 0
	+#define I40E_GLPES_VFIP6RXOCTSLO_IP6RXOCTSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFIP6RXOCTSLO_IP6RXOCTSLO_SHIFT)
	+#define I40E_GLPES_VFIP6RXPKTSHI(_i) (0x00019004 + ((_i) * 8)) /* _i=0...31 / / Reset: PE_CORER */
	+#define I40E_GLPES_VFIP6RXPKTSHI_MAX_INDEX 31
	+#define I40E_GLPES_VFIP6RXPKTSHI_IP6RXPKTSHI_SHIFT 0
	+#define I40E_GLPES_VFIP6RXPKTSHI_IP6RXPKTSHI_MASK (0xFFFF << I40E_GLPES_VFIP6RXPKTSHI_IP6RXPKTSHI_SHIFT)
	+#define I40E_GLPES_VFIP6RXPKTSLO(_i) (0x00019000 + ((_i) * 8)) /* _i=0...31 / / Reset: PE_CORER */
	+#define I40E_GLPES_VFIP6RXPKTSLO_MAX_INDEX 31
	+#define I40E_GLPES_VFIP6RXPKTSLO_IP6RXPKTSLO_SHIFT 0
	+#define I40E_GLPES_VFIP6RXPKTSLO_IP6RXPKTSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFIP6RXPKTSLO_IP6RXPKTSLO_SHIFT)
	+#define I40E_GLPES_VFIP6RXTRUNC(_i) (0x00019300 + ((_i) * 4)) /* _i=0...31 / / Reset: PE_CORER */
	+#define I40E_GLPES_VFIP6RXTRUNC_MAX_INDEX 31
	+#define I40E_GLPES_VFIP6RXTRUNC_IP6RXTRUNC_SHIFT 0
	+#define I40E_GLPES_VFIP6RXTRUNC_IP6RXTRUNC_MASK (0xFFFFFFFF << I40E_GLPES_VFIP6RXTRUNC_IP6RXTRUNC_SHIFT)
	+#define I40E_GLPES_VFIP6TXFRAGSHI(_i) (0x0001A804 + ((_i) * 8)) /* _i=0...31 / / Reset: PE_CORER */
	+#define I40E_GLPES_VFIP6TXFRAGSHI_MAX_INDEX 31
	+#define I40E_GLPES_VFIP6TXFRAGSHI_IP6TXFRAGSHI_SHIFT 0
	+#define I40E_GLPES_VFIP6TXFRAGSHI_IP6TXFRAGSHI_MASK (0xFFFF << I40E_GLPES_VFIP6TXFRAGSHI_IP6TXFRAGSHI_SHIFT)
	+#define I40E_GLPES_VFIP6TXFRAGSLO(_i) (0x0001A800 + ((_i) * 8)) /* _i=0...31 / / Reset: PE_CORER */
	+#define I40E_GLPES_VFIP6TXFRAGSLO_MAX_INDEX 31
	+#define I40E_GLPES_VFIP6TXFRAGSLO_IP6TXFRAGSLO_SHIFT 0
	+#define I40E_GLPES_VFIP6TXFRAGSLO_IP6TXFRAGSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFIP6TXFRAGSLO_IP6TXFRAGSLO_SHIFT)
	+#define I40E_GLPES_VFIP6TXMCOCTSHI(_i) (0x0001AA04 + ((_i) * 8)) /* _i=0...31 / / Reset: PE_CORER */
	+#define I40E_GLPES_VFIP6TXMCOCTSHI_MAX_INDEX 31
	+#define I40E_GLPES_VFIP6TXMCOCTSHI_IP6TXMCOCTSHI_SHIFT 0
	+#define I40E_GLPES_VFIP6TXMCOCTSHI_IP6TXMCOCTSHI_MASK (0xFFFF << I40E_GLPES_VFIP6TXMCOCTSHI_IP6TXMCOCTSHI_SHIFT)
	+#define I40E_GLPES_VFIP6TXMCOCTSLO(_i) (0x0001AA00 + ((_i) * 8)) /* _i=0...31 / / Reset: PE_CORER */
	+#define I40E_GLPES_VFIP6TXMCOCTSLO_MAX_INDEX 31
	+#define I40E_GLPES_VFIP6TXMCOCTSLO_IP6TXMCOCTSLO_SHIFT 0
	+#define I40E_GLPES_VFIP6TXMCOCTSLO_IP6TXMCOCTSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFIP6TXMCOCTSLO_IP6TXMCOCTSLO_SHIFT)
	+#define I40E_GLPES_VFIP6TXMCPKTSHI(_i) (0x0001AC04 + ((_i) * 8)) /* _i=0...31 / / Reset: PE_CORER */
	+#define I40E_GLPES_VFIP6TXMCPKTSHI_MAX_INDEX 31
	+#define I40E_GLPES_VFIP6TXMCPKTSHI_IP6TXMCPKTSHI_SHIFT 0
	+#define I40E_GLPES_VFIP6TXMCPKTSHI_IP6TXMCPKTSHI_MASK (0xFFFF << I40E_GLPES_VFIP6TXMCPKTSHI_IP6TXMCPKTSHI_SHIFT)
	+#define I40E_GLPES_VFIP6TXMCPKTSLO(_i) (0x0001AC00 + ((_i) * 8)) /* _i=0...31 / / Reset: PE_CORER */
	+#define I40E_GLPES_VFIP6TXMCPKTSLO_MAX_INDEX 31
	+#define I40E_GLPES_VFIP6TXMCPKTSLO_IP6TXMCPKTSLO_SHIFT 0
	+#define I40E_GLPES_VFIP6TXMCPKTSLO_IP6TXMCPKTSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFIP6TXMCPKTSLO_IP6TXMCPKTSLO_SHIFT)
	+#define I40E_GLPES_VFIP6TXNOROUTE(_i) (0x0001AF00 + ((_i) * 4)) /* _i=0...31 / / Reset: PE_CORER */
	+#define I40E_GLPES_VFIP6TXNOROUTE_MAX_INDEX 31
	+#define I40E_GLPES_VFIP6TXNOROUTE_IP6TXNOROUTE_SHIFT 0
	+#define I40E_GLPES_VFIP6TXNOROUTE_IP6TXNOROUTE_MASK (0xFFFFFF << I40E_GLPES_VFIP6TXNOROUTE_IP6TXNOROUTE_SHIFT)
	+#define I40E_GLPES_VFIP6TXOCTSHI(_i) (0x0001A404 + ((_i) * 8)) /* _i=0...31 / / Reset: PE_CORER */
	+#define I40E_GLPES_VFIP6TXOCTSHI_MAX_INDEX 31
	+#define I40E_GLPES_VFIP6TXOCTSHI_IP6TXOCTSHI_SHIFT 0
	+#define I40E_GLPES_VFIP6TXOCTSHI_IP6TXOCTSHI_MASK (0xFFFF << I40E_GLPES_VFIP6TXOCTSHI_IP6TXOCTSHI_SHIFT)
	+#define I40E_GLPES_VFIP6TXOCTSLO(_i) (0x0001A400 + ((_i) * 8)) /* _i=0...31 / / Reset: PE_CORER */
	+#define I40E_GLPES_VFIP6TXOCTSLO_MAX_INDEX 31
	+#define I40E_GLPES_VFIP6TXOCTSLO_IP6TXOCTSLO_SHIFT 0
	+#define I40E_GLPES_VFIP6TXOCTSLO_IP6TXOCTSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFIP6TXOCTSLO_IP6TXOCTSLO_SHIFT)
	+#define I40E_GLPES_VFIP6TXPKTSHI(_i) (0x0001A604 + ((_i) * 8)) /* _i=0...31 / / Reset: PE_CORER */
	+#define I40E_GLPES_VFIP6TXPKTSHI_MAX_INDEX 31
	+#define I40E_GLPES_VFIP6TXPKTSHI_IP6TXPKTSHI_SHIFT 0
	+#define I40E_GLPES_VFIP6TXPKTSHI_IP6TXPKTSHI_MASK (0xFFFF << I40E_GLPES_VFIP6TXPKTSHI_IP6TXPKTSHI_SHIFT)
	+#define I40E_GLPES_VFIP6TXPKTSLO(_i) (0x0001A600 + ((_i) * 8)) /* _i=0...31 / / Reset: PE_CORER */
	+#define I40E_GLPES_VFIP6TXPKTSLO_MAX_INDEX 31
	+#define I40E_GLPES_VFIP6TXPKTSLO_IP6TXPKTSLO_SHIFT 0
	+#define I40E_GLPES_VFIP6TXPKTSLO_IP6TXPKTSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFIP6TXPKTSLO_IP6TXPKTSLO_SHIFT)
	+#define I40E_GLPES_VFRDMARXRDSHI(_i) (0x0001BE04 + ((_i) * 8)) /* _i=0...31 / / Reset: PE_CORER */
	+#define I40E_GLPES_VFRDMARXRDSHI_MAX_INDEX 31
	+#define I40E_GLPES_VFRDMARXRDSHI_RDMARXRDSHI_SHIFT 0
	+#define I40E_GLPES_VFRDMARXRDSHI_RDMARXRDSHI_MASK (0xFFFF << I40E_GLPES_VFRDMARXRDSHI_RDMARXRDSHI_SHIFT)
	+#define I40E_GLPES_VFRDMARXRDSLO(_i) (0x0001BE00 + ((_i) * 8)) /* _i=0...31 / / Reset: PE_CORER */
	+#define I40E_GLPES_VFRDMARXRDSLO_MAX_INDEX 31
	+#define I40E_GLPES_VFRDMARXRDSLO_RDMARXRDSLO_SHIFT 0
	+#define I40E_GLPES_VFRDMARXRDSLO_RDMARXRDSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFRDMARXRDSLO_RDMARXRDSLO_SHIFT)
	+#define I40E_GLPES_VFRDMARXSNDSHI(_i) (0x0001C004 + ((_i) * 8)) /* _i=0...31 / / Reset: PE_CORER */
	+#define I40E_GLPES_VFRDMARXSNDSHI_MAX_INDEX 31
	+#define I40E_GLPES_VFRDMARXSNDSHI_RDMARXSNDSHI_SHIFT 0
	+#define I40E_GLPES_VFRDMARXSNDSHI_RDMARXSNDSHI_MASK (0xFFFF << I40E_GLPES_VFRDMARXSNDSHI_RDMARXSNDSHI_SHIFT)
	+#define I40E_GLPES_VFRDMARXSNDSLO(_i) (0x0001C000 + ((_i) * 8)) /* _i=0...31 / / Reset: PE_CORER */
	+#define I40E_GLPES_VFRDMARXSNDSLO_MAX_INDEX 31
	+#define I40E_GLPES_VFRDMARXSNDSLO_RDMARXSNDSLO_SHIFT 0
	+#define I40E_GLPES_VFRDMARXSNDSLO_RDMARXSNDSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFRDMARXSNDSLO_RDMARXSNDSLO_SHIFT)
	+#define I40E_GLPES_VFRDMARXWRSHI(_i) (0x0001BC04 + ((_i) * 8)) /* _i=0...31 / / Reset: PE_CORER */
	+#define I40E_GLPES_VFRDMARXWRSHI_MAX_INDEX 31
	+#define I40E_GLPES_VFRDMARXWRSHI_RDMARXWRSHI_SHIFT 0
	+#define I40E_GLPES_VFRDMARXWRSHI_RDMARXWRSHI_MASK (0xFFFF << I40E_GLPES_VFRDMARXWRSHI_RDMARXWRSHI_SHIFT)
	+#define I40E_GLPES_VFRDMARXWRSLO(_i) (0x0001BC00 + ((_i) * 8)) /* _i=0...31 / / Reset: PE_CORER */
	+#define I40E_GLPES_VFRDMARXWRSLO_MAX_INDEX 31
	+#define I40E_GLPES_VFRDMARXWRSLO_RDMARXWRSLO_SHIFT 0
	+#define I40E_GLPES_VFRDMARXWRSLO_RDMARXWRSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFRDMARXWRSLO_RDMARXWRSLO_SHIFT)
	+#define I40E_GLPES_VFRDMATXRDSHI(_i) (0x0001C404 + ((_i) * 8)) /* _i=0...31 / / Reset: PE_CORER */
	+#define I40E_GLPES_VFRDMATXRDSHI_MAX_INDEX 31
	+#define I40E_GLPES_VFRDMATXRDSHI_RDMARXRDSHI_SHIFT 0
	+#define I40E_GLPES_VFRDMATXRDSHI_RDMARXRDSHI_MASK (0xFFFF << I40E_GLPES_VFRDMATXRDSHI_RDMARXRDSHI_SHIFT)
	+#define I40E_GLPES_VFRDMATXRDSLO(_i) (0x0001C400 + ((_i) * 8)) /* _i=0...31 / / Reset: PE_CORER */
	+#define I40E_GLPES_VFRDMATXRDSLO_MAX_INDEX 31
	+#define I40E_GLPES_VFRDMATXRDSLO_RDMARXRDSLO_SHIFT 0
	+#define I40E_GLPES_VFRDMATXRDSLO_RDMARXRDSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFRDMATXRDSLO_RDMARXRDSLO_SHIFT)
	+#define I40E_GLPES_VFRDMATXSNDSHI(_i) (0x0001C604 + ((_i) * 8)) /* _i=0...31 / / Reset: PE_CORER */
	+#define I40E_GLPES_VFRDMATXSNDSHI_MAX_INDEX 31
	+#define I40E_GLPES_VFRDMATXSNDSHI_RDMARXSNDSHI_SHIFT 0
	+#define I40E_GLPES_VFRDMATXSNDSHI_RDMARXSNDSHI_MASK (0xFFFF << I40E_GLPES_VFRDMATXSNDSHI_RDMARXSNDSHI_SHIFT)
	+#define I40E_GLPES_VFRDMATXSNDSLO(_i) (0x0001C600 + ((_i) * 8)) /* _i=0...31 / / Reset: PE_CORER */
	+#define I40E_GLPES_VFRDMATXSNDSLO_MAX_INDEX 31
	+#define I40E_GLPES_VFRDMATXSNDSLO_RDMARXSNDSLO_SHIFT 0
	+#define I40E_GLPES_VFRDMATXSNDSLO_RDMARXSNDSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFRDMATXSNDSLO_RDMARXSNDSLO_SHIFT)
	+#define I40E_GLPES_VFRDMATXWRSHI(_i) (0x0001C204 + ((_i) * 8)) /* _i=0...31 / / Reset: PE_CORER */
	+#define I40E_GLPES_VFRDMATXWRSHI_MAX_INDEX 31
	+#define I40E_GLPES_VFRDMATXWRSHI_RDMARXWRSHI_SHIFT 0
	+#define I40E_GLPES_VFRDMATXWRSHI_RDMARXWRSHI_MASK (0xFFFF << I40E_GLPES_VFRDMATXWRSHI_RDMARXWRSHI_SHIFT)
	+#define I40E_GLPES_VFRDMATXWRSLO(_i) (0x0001C200 + ((_i) * 8)) /* _i=0...31 / / Reset: PE_CORER */
	+#define I40E_GLPES_VFRDMATXWRSLO_MAX_INDEX 31
	+#define I40E_GLPES_VFRDMATXWRSLO_RDMARXWRSLO_SHIFT 0
	+#define I40E_GLPES_VFRDMATXWRSLO_RDMARXWRSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFRDMATXWRSLO_RDMARXWRSLO_SHIFT)
	+#define I40E_GLPES_VFRDMAVBNDHI(_i) (0x0001C804 + ((_i) * 8)) /* _i=0...31 / / Reset: PE_CORER */
	+#define I40E_GLPES_VFRDMAVBNDHI_MAX_INDEX 31
	+#define I40E_GLPES_VFRDMAVBNDHI_RDMAVBNDHI_SHIFT 0
	+#define I40E_GLPES_VFRDMAVBNDHI_RDMAVBNDHI_MASK (0xFFFFFFFF << I40E_GLPES_VFRDMAVBNDHI_RDMAVBNDHI_SHIFT)
	+#define I40E_GLPES_VFRDMAVBNDLO(_i) (0x0001C800 + ((_i) * 8)) /* _i=0...31 / / Reset: PE_CORER */
	+#define I40E_GLPES_VFRDMAVBNDLO_MAX_INDEX 31
	+#define I40E_GLPES_VFRDMAVBNDLO_RDMAVBNDLO_SHIFT 0
	+#define I40E_GLPES_VFRDMAVBNDLO_RDMAVBNDLO_MASK (0xFFFFFFFF << I40E_GLPES_VFRDMAVBNDLO_RDMAVBNDLO_SHIFT)
	+#define I40E_GLPES_VFRDMAVINVHI(_i) (0x0001CA04 + ((_i) * 8)) /* _i=0...31 / / Reset: PE_CORER */
	+#define I40E_GLPES_VFRDMAVINVHI_MAX_INDEX 31
	+#define I40E_GLPES_VFRDMAVINVHI_RDMAVINVHI_SHIFT 0
	+#define I40E_GLPES_VFRDMAVINVHI_RDMAVINVHI_MASK (0xFFFFFFFF << I40E_GLPES_VFRDMAVINVHI_RDMAVINVHI_SHIFT)
	+#define I40E_GLPES_VFRDMAVINVLO(_i) (0x0001CA00 + ((_i) * 8)) /* _i=0...31 / / Reset: PE_CORER */
	+#define I40E_GLPES_VFRDMAVINVLO_MAX_INDEX 31
	+#define I40E_GLPES_VFRDMAVINVLO_RDMAVINVLO_SHIFT 0
	+#define I40E_GLPES_VFRDMAVINVLO_RDMAVINVLO_MASK (0xFFFFFFFF << I40E_GLPES_VFRDMAVINVLO_RDMAVINVLO_SHIFT)
	+#define I40E_GLPES_VFRXVLANERR(_i) (0x00018000 + ((_i) * 4)) /* _i=0...31 / / Reset: PE_CORER */
	+#define I40E_GLPES_VFRXVLANERR_MAX_INDEX 31
	+#define I40E_GLPES_VFRXVLANERR_RXVLANERR_SHIFT 0
	+#define I40E_GLPES_VFRXVLANERR_RXVLANERR_MASK (0xFFFFFF << I40E_GLPES_VFRXVLANERR_RXVLANERR_SHIFT)
	+#define I40E_GLPES_VFTCPRTXSEG(_i) (0x0001B600 + ((_i) * 4)) /* _i=0...31 / / Reset: PE_CORER */
	+#define I40E_GLPES_VFTCPRTXSEG_MAX_INDEX 31
	+#define I40E_GLPES_VFTCPRTXSEG_TCPRTXSEG_SHIFT 0
	+#define I40E_GLPES_VFTCPRTXSEG_TCPRTXSEG_MASK (0xFFFFFFFF << I40E_GLPES_VFTCPRTXSEG_TCPRTXSEG_SHIFT)
	+#define I40E_GLPES_VFTCPRXOPTERR(_i) (0x0001B200 + ((_i) * 4)) /* _i=0...31 / / Reset: PE_CORER */
	+#define I40E_GLPES_VFTCPRXOPTERR_MAX_INDEX 31
	+#define I40E_GLPES_VFTCPRXOPTERR_TCPRXOPTERR_SHIFT 0
	+#define I40E_GLPES_VFTCPRXOPTERR_TCPRXOPTERR_MASK (0xFFFFFF << I40E_GLPES_VFTCPRXOPTERR_TCPRXOPTERR_SHIFT)
	+#define I40E_GLPES_VFTCPRXPROTOERR(_i) (0x0001B300 + ((_i) * 4)) /* _i=0...31 / / Reset: PE_CORER */
	+#define I40E_GLPES_VFTCPRXPROTOERR_MAX_INDEX 31
	+#define I40E_GLPES_VFTCPRXPROTOERR_TCPRXPROTOERR_SHIFT 0
	+#define I40E_GLPES_VFTCPRXPROTOERR_TCPRXPROTOERR_MASK (0xFFFFFF << I40E_GLPES_VFTCPRXPROTOERR_TCPRXPROTOERR_SHIFT)
	+#define I40E_GLPES_VFTCPRXSEGSHI(_i) (0x0001B004 + ((_i) * 8)) /* _i=0...31 / / Reset: PE_CORER */
	+#define I40E_GLPES_VFTCPRXSEGSHI_MAX_INDEX 31
	+#define I40E_GLPES_VFTCPRXSEGSHI_TCPRXSEGSHI_SHIFT 0
	+#define I40E_GLPES_VFTCPRXSEGSHI_TCPRXSEGSHI_MASK (0xFFFF << I40E_GLPES_VFTCPRXSEGSHI_TCPRXSEGSHI_SHIFT)
	+#define I40E_GLPES_VFTCPRXSEGSLO(_i) (0x0001B000 + ((_i) * 8)) /* _i=0...31 / / Reset: PE_CORER */
	+#define I40E_GLPES_VFTCPRXSEGSLO_MAX_INDEX 31
	+#define I40E_GLPES_VFTCPRXSEGSLO_TCPRXSEGSLO_SHIFT 0
	+#define I40E_GLPES_VFTCPRXSEGSLO_TCPRXSEGSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFTCPRXSEGSLO_TCPRXSEGSLO_SHIFT)
	+#define I40E_GLPES_VFTCPTXSEGHI(_i) (0x0001B404 + ((_i) * 8)) /* _i=0...31 / / Reset: PE_CORER */
	+#define I40E_GLPES_VFTCPTXSEGHI_MAX_INDEX 31
	+#define I40E_GLPES_VFTCPTXSEGHI_TCPTXSEGHI_SHIFT 0
	+#define I40E_GLPES_VFTCPTXSEGHI_TCPTXSEGHI_MASK (0xFFFF << I40E_GLPES_VFTCPTXSEGHI_TCPTXSEGHI_SHIFT)
	+#define I40E_GLPES_VFTCPTXSEGLO(_i) (0x0001B400 + ((_i) * 8)) /* _i=0...31 / / Reset: PE_CORER */
	+#define I40E_GLPES_VFTCPTXSEGLO_MAX_INDEX 31
	+#define I40E_GLPES_VFTCPTXSEGLO_TCPTXSEGLO_SHIFT 0
	+#define I40E_GLPES_VFTCPTXSEGLO_TCPTXSEGLO_MASK (0xFFFFFFFF << I40E_GLPES_VFTCPTXSEGLO_TCPTXSEGLO_SHIFT)
	+#define I40E_GLPES_VFUDPRXPKTSHI(_i) (0x0001B804 + ((_i) * 8)) /* _i=0...31 / / Reset: PE_CORER */
	+#define I40E_GLPES_VFUDPRXPKTSHI_MAX_INDEX 31
	+#define I40E_GLPES_VFUDPRXPKTSHI_UDPRXPKTSHI_SHIFT 0
	+#define I40E_GLPES_VFUDPRXPKTSHI_UDPRXPKTSHI_MASK (0xFFFF << I40E_GLPES_VFUDPRXPKTSHI_UDPRXPKTSHI_SHIFT)
	+#define I40E_GLPES_VFUDPRXPKTSLO(_i) (0x0001B800 + ((_i) * 8)) /* _i=0...31 / / Reset: PE_CORER */
	+#define I40E_GLPES_VFUDPRXPKTSLO_MAX_INDEX 31
	+#define I40E_GLPES_VFUDPRXPKTSLO_UDPRXPKTSLO_SHIFT 0
	+#define I40E_GLPES_VFUDPRXPKTSLO_UDPRXPKTSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFUDPRXPKTSLO_UDPRXPKTSLO_SHIFT)
	+#define I40E_GLPES_VFUDPTXPKTSHI(_i) (0x0001BA04 + ((_i) * 8)) /* _i=0...31 / / Reset: PE_CORER */
	+#define I40E_GLPES_VFUDPTXPKTSHI_MAX_INDEX 31
	+#define I40E_GLPES_VFUDPTXPKTSHI_UDPTXPKTSHI_SHIFT 0
	+#define I40E_GLPES_VFUDPTXPKTSHI_UDPTXPKTSHI_MASK (0xFFFF << I40E_GLPES_VFUDPTXPKTSHI_UDPTXPKTSHI_SHIFT)
	+#define I40E_GLPES_VFUDPTXPKTSLO(_i) (0x0001BA00 + ((_i) * 8)) /* _i=0...31 / / Reset: PE_CORER */
	+#define I40E_GLPES_VFUDPTXPKTSLO_MAX_INDEX 31
	+#define I40E_GLPES_VFUDPTXPKTSLO_UDPTXPKTSLO_SHIFT 0
	+#define I40E_GLPES_VFUDPTXPKTSLO_UDPTXPKTSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFUDPTXPKTSLO_UDPTXPKTSLO_SHIFT)
	+
	+#define I40E_PFINT_AEQCTL 0x00038700 /* Reset: CORER */
	+#define I40E_VFPE_AEQALLOC1 0x0000A400 /* Reset: VFR */
	+#define I40E_VFPE_AEQALLOC1_AECOUNT_SHIFT 0
	+#define I40E_VFPE_AEQALLOC1_AECOUNT_MASK (0xFFFFFFFF << I40E_VFPE_AEQALLOC1_AECOUNT_SHIFT)
	+#define I40E_VFPE_CCQPHIGH1 0x00009800 /* Reset: VFR */
	+#define I40E_VFPE_CCQPHIGH1_PECCQPHIGH_SHIFT 0
	+#define I40E_VFPE_CCQPHIGH1_PECCQPHIGH_MASK (0xFFFFFFFF << I40E_VFPE_CCQPHIGH1_PECCQPHIGH_SHIFT)
	+#define I40E_VFPE_CCQPLOW1 0x0000AC00 /* Reset: VFR */
	+#define I40E_VFPE_CCQPLOW1_PECCQPLOW_SHIFT 0
	+#define I40E_VFPE_CCQPLOW1_PECCQPLOW_MASK (0xFFFFFFFF << I40E_VFPE_CCQPLOW1_PECCQPLOW_SHIFT)
	+#define I40E_VFPE_CCQPSTATUS1 0x0000B800 /* Reset: VFR */
	+#define I40E_VFPE_CCQPSTATUS1_CCQP_DONE_SHIFT 0
	+#define I40E_VFPE_CCQPSTATUS1_CCQP_DONE_MASK (0x1 << I40E_VFPE_CCQPSTATUS1_CCQP_DONE_SHIFT)
	+#define I40E_VFPE_CCQPSTATUS1_HMC_PROFILE_SHIFT 4
	+#define I40E_VFPE_CCQPSTATUS1_HMC_PROFILE_MASK (0x7 << I40E_VFPE_CCQPSTATUS1_HMC_PROFILE_SHIFT)
	+#define I40E_VFPE_CCQPSTATUS1_RDMA_EN_VFS_SHIFT 16
	+#define I40E_VFPE_CCQPSTATUS1_RDMA_EN_VFS_MASK (0x3F << I40E_VFPE_CCQPSTATUS1_RDMA_EN_VFS_SHIFT)
	+#define I40E_VFPE_CCQPSTATUS1_CCQP_ERR_SHIFT 31
	+#define I40E_VFPE_CCQPSTATUS1_CCQP_ERR_MASK (0x1 << I40E_VFPE_CCQPSTATUS1_CCQP_ERR_SHIFT)
	+#define I40E_VFPE_CQACK1 0x0000B000 /* Reset: VFR */
	+#define I40E_VFPE_CQACK1_PECQID_SHIFT 0
	+#define I40E_VFPE_CQACK1_PECQID_MASK (0x1FFFF << I40E_VFPE_CQACK1_PECQID_SHIFT)
	+#define I40E_VFPE_CQARM1 0x0000B400 /* Reset: VFR */
	+#define I40E_VFPE_CQARM1_PECQID_SHIFT 0
	+#define I40E_VFPE_CQARM1_PECQID_MASK (0x1FFFF << I40E_VFPE_CQARM1_PECQID_SHIFT)
	+#define I40E_VFPE_CQPDB1 0x0000BC00 /* Reset: VFR */
	+#define I40E_VFPE_CQPDB1_WQHEAD_SHIFT 0
	+#define I40E_VFPE_CQPDB1_WQHEAD_MASK (0x7FF << I40E_VFPE_CQPDB1_WQHEAD_SHIFT)
	+#define I40E_VFPE_CQPERRCODES1 0x00009C00 /* Reset: VFR */
	+#define I40E_VFPE_CQPERRCODES1_CQP_MINOR_CODE_SHIFT 0
	+#define I40E_VFPE_CQPERRCODES1_CQP_MINOR_CODE_MASK (0xFFFF << I40E_VFPE_CQPERRCODES1_CQP_MINOR_CODE_SHIFT)
	+#define I40E_VFPE_CQPERRCODES1_CQP_MAJOR_CODE_SHIFT 16
	+#define I40E_VFPE_CQPERRCODES1_CQP_MAJOR_CODE_MASK (0xFFFF << I40E_VFPE_CQPERRCODES1_CQP_MAJOR_CODE_SHIFT)
	+#define I40E_VFPE_CQPTAIL1 0x0000A000 /* Reset: VFR */
	+#define I40E_VFPE_CQPTAIL1_WQTAIL_SHIFT 0
	+#define I40E_VFPE_CQPTAIL1_WQTAIL_MASK (0x7FF << I40E_VFPE_CQPTAIL1_WQTAIL_SHIFT)
	+#define I40E_VFPE_CQPTAIL1_CQP_OP_ERR_SHIFT 31
	+#define I40E_VFPE_CQPTAIL1_CQP_OP_ERR_MASK (0x1 << I40E_VFPE_CQPTAIL1_CQP_OP_ERR_SHIFT)
	+#define I40E_VFPE_IPCONFIG01 0x00008C00 /* Reset: VFR */
	+#define I40E_VFPE_IPCONFIG01_PEIPID_SHIFT 0
	+#define I40E_VFPE_IPCONFIG01_PEIPID_MASK (0xFFFF << I40E_VFPE_IPCONFIG01_PEIPID_SHIFT)
	+#define I40E_VFPE_IPCONFIG01_USEENTIREIDRANGE_SHIFT 16
	+#define I40E_VFPE_IPCONFIG01_USEENTIREIDRANGE_MASK (0x1 << I40E_VFPE_IPCONFIG01_USEENTIREIDRANGE_SHIFT)
	+#define I40E_VFPE_MRTEIDXMASK1 0x00009000 /* Reset: VFR */
	+#define I40E_VFPE_MRTEIDXMASK1_MRTEIDXMASKBITS_SHIFT 0
	+#define I40E_VFPE_MRTEIDXMASK1_MRTEIDXMASKBITS_MASK (0x1F << I40E_VFPE_MRTEIDXMASK1_MRTEIDXMASKBITS_SHIFT)
	+#define I40E_VFPE_RCVUNEXPECTEDERROR1 0x00009400 /* Reset: VFR */
	+#define I40E_VFPE_RCVUNEXPECTEDERROR1_TCP_RX_UNEXP_ERR_SHIFT 0
	+#define I40E_VFPE_RCVUNEXPECTEDERROR1_TCP_RX_UNEXP_ERR_MASK (0xFFFFFF << I40E_VFPE_RCVUNEXPECTEDERROR1_TCP_RX_UNEXP_ERR_SHIFT)
	+#define I40E_VFPE_TCPNOWTIMER1 0x0000A800 /* Reset: VFR */
	+#define I40E_VFPE_TCPNOWTIMER1_TCP_NOW_SHIFT 0
	+#define I40E_VFPE_TCPNOWTIMER1_TCP_NOW_MASK (0xFFFFFFFF << I40E_VFPE_TCPNOWTIMER1_TCP_NOW_SHIFT)
	+#define I40E_VFPE_WQEALLOC1 0x0000C000 /* Reset: VFR */
	+#define I40E_VFPE_WQEALLOC1_PEQPID_SHIFT 0
	+#define I40E_VFPE_WQEALLOC1_PEQPID_MASK (0x3FFFF << I40E_VFPE_WQEALLOC1_PEQPID_SHIFT)
	+#define I40E_VFPE_WQEALLOC1_WQE_DESC_INDEX_SHIFT 20
	+#define I40E_VFPE_WQEALLOC1_WQE_DESC_INDEX_MASK (0xFFF << I40E_VFPE_WQEALLOC1_WQE_DESC_INDEX_SHIFT)
	+#endif /* I40IW_REGISTER_H */
	Index: sys/dev/ixl/iwarp/i40iw_status.h
	===================================================================
	--- /dev/null
	+++ sys/dev/ixl/iwarp/i40iw_status.h
	@@ -0,0 +1,101 @@
	+/*******************************************************************************
	+*
	+* Copyright (c) 2015-2017 Intel Corporation. All rights reserved.
	+*
	+* This software is available to you under a choice of one of two
	+* licenses. You may choose to be licensed under the terms of the GNU
	+* General Public License (GPL) Version 2, available from the file
	+* COPYING in the main directory of this source tree, or the
	+* OpenFabrics.org BSD license below:
	+*
	+* Redistribution and use in source and binary forms, with or
	+* without modification, are permitted provided that the following
	+* conditions are met:
	+*
	+* - Redistributions of source code must retain the above
	+* copyright notice, this list of conditions and the following
	+* disclaimer.
	+*
	+* - Redistributions in binary form must reproduce the above
	+* copyright notice, this list of conditions and the following
	+* disclaimer in the documentation and/or other materials
	+* provided with the distribution.
	+*
	+* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
	+* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
	+* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
	+* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
	+* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
	+* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
	+* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
	+* SOFTWARE.
	+*
	+*******************************************************************************/
	+/$FreeBSD$/
	+
	+#ifndef I40IW_STATUS_H
	+#define I40IW_STATUS_H
	+
	+/* Error Codes */
	+enum i40iw_status_code {
	+ I40IW_SUCCESS = 0,
	+ I40IW_ERR_NVM = -1,
	+ I40IW_ERR_NVM_CHECKSUM = -2,
	+ I40IW_ERR_CONFIG = -4,
	+ I40IW_ERR_PARAM = -5,
	+ I40IW_ERR_DEVICE_NOT_SUPPORTED = -6,
	+ I40IW_ERR_RESET_FAILED = -7,
	+ I40IW_ERR_SWFW_SYNC = -8,
	+ I40IW_ERR_NO_MEMORY = -9,
	+ I40IW_ERR_BAD_PTR = -10,
	+ I40IW_ERR_INVALID_PD_ID = -11,
	+ I40IW_ERR_INVALID_QP_ID = -12,
	+ I40IW_ERR_INVALID_CQ_ID = -13,
	+ I40IW_ERR_INVALID_CEQ_ID = -14,
	+ I40IW_ERR_INVALID_AEQ_ID = -15,
	+ I40IW_ERR_INVALID_SIZE = -16,
	+ I40IW_ERR_INVALID_ARP_INDEX = -17,
	+ I40IW_ERR_INVALID_FPM_FUNC_ID = -18,
	+ I40IW_ERR_QP_INVALID_MSG_SIZE = -19,
	+ I40IW_ERR_QP_TOOMANY_WRS_POSTED = -20,
	+ I40IW_ERR_INVALID_FRAG_COUNT = -21,
	+ I40IW_ERR_QUEUE_EMPTY = -22,
	+ I40IW_ERR_INVALID_ALIGNMENT = -23,
	+ I40IW_ERR_FLUSHED_QUEUE = -24,
	+ I40IW_ERR_INVALID_PUSH_PAGE_INDEX = -25,
	+ I40IW_ERR_INVALID_INLINE_DATA_SIZE = -26,
	+ I40IW_ERR_TIMEOUT = -27,
	+ I40IW_ERR_OPCODE_MISMATCH = -28,
	+ I40IW_ERR_CQP_COMPL_ERROR = -29,
	+ I40IW_ERR_INVALID_VF_ID = -30,
	+ I40IW_ERR_INVALID_HMCFN_ID = -31,
	+ I40IW_ERR_BACKING_PAGE_ERROR = -32,
	+ I40IW_ERR_NO_PBLCHUNKS_AVAILABLE = -33,
	+ I40IW_ERR_INVALID_PBLE_INDEX = -34,
	+ I40IW_ERR_INVALID_SD_INDEX = -35,
	+ I40IW_ERR_INVALID_PAGE_DESC_INDEX = -36,
	+ I40IW_ERR_INVALID_SD_TYPE = -37,
	+ I40IW_ERR_MEMCPY_FAILED = -38,
	+ I40IW_ERR_INVALID_HMC_OBJ_INDEX = -39,
	+ I40IW_ERR_INVALID_HMC_OBJ_COUNT = -40,
	+ I40IW_ERR_INVALID_SRQ_ARM_LIMIT = -41,
	+ I40IW_ERR_SRQ_ENABLED = -42,
	+ I40IW_ERR_BUF_TOO_SHORT = -43,
	+ I40IW_ERR_BAD_IWARP_CQE = -44,
	+ I40IW_ERR_NVM_BLANK_MODE = -45,
	+ I40IW_ERR_NOT_IMPLEMENTED = -46,
	+ I40IW_ERR_PE_DOORBELL_NOT_ENABLED = -47,
	+ I40IW_ERR_NOT_READY = -48,
	+ I40IW_NOT_SUPPORTED = -49,
	+ I40IW_ERR_FIRMWARE_API_VERSION = -50,
	+ I40IW_ERR_RING_FULL = -51,
	+ I40IW_ERR_MPA_CRC = -61,
	+ I40IW_ERR_NO_TXBUFS = -62,
	+ I40IW_ERR_SEQ_NUM = -63,
	+ I40IW_ERR_LIST_EMPTY = -64,
	+ I40IW_ERR_INVALID_MAC_ADDR = -65,
	+ I40IW_ERR_BAD_STAG = -66,
	+ I40IW_ERR_CQ_COMPL_ERROR = -67,
	+ I40IW_ERR_QUEUE_DESTROYED = -68
	+};
	+#endif
	Index: sys/dev/ixl/iwarp/i40iw_type.h
	===================================================================
	--- /dev/null
	+++ sys/dev/ixl/iwarp/i40iw_type.h
	@@ -0,0 +1,1238 @@
	+/*******************************************************************************
	+*
	+* Copyright (c) 2015-2017 Intel Corporation. All rights reserved.
	+*
	+* This software is available to you under a choice of one of two
	+* licenses. You may choose to be licensed under the terms of the GNU
	+* General Public License (GPL) Version 2, available from the file
	+* COPYING in the main directory of this source tree, or the
	+* OpenFabrics.org BSD license below:
	+*
	+* Redistribution and use in source and binary forms, with or
	+* without modification, are permitted provided that the following
	+* conditions are met:
	+*
	+* - Redistributions of source code must retain the above
	+* copyright notice, this list of conditions and the following
	+* disclaimer.
	+*
	+* - Redistributions in binary form must reproduce the above
	+* copyright notice, this list of conditions and the following
	+* disclaimer in the documentation and/or other materials
	+* provided with the distribution.
	+*
	+* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
	+* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
	+* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
	+* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
	+* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
	+* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
	+* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
	+* SOFTWARE.
	+*
	+*******************************************************************************/
	+/$FreeBSD$/
	+
	+#ifndef I40IW_TYPE_H
	+#define I40IW_TYPE_H
	+#include "i40iw_user.h"
	+#include "i40iw_hmc.h"
	+
	+struct i40iw_cqp_sq_wqe {
	+ u64 buf[I40IW_CQP_WQE_SIZE];
	+};
	+
	+struct i40iw_sc_aeqe {
	+ u64 buf[I40IW_AEQE_SIZE];
	+};
	+
	+struct i40iw_ceqe {
	+ u64 buf[I40IW_CEQE_SIZE];
	+};
	+
	+struct i40iw_cqp_ctx {
	+ u64 buf[I40IW_CQP_CTX_SIZE];
	+};
	+
	+struct i40iw_cq_shadow_area {
	+ u64 buf[I40IW_SHADOW_AREA_SIZE];
	+};
	+
	+struct i40iw_vsi_pestat;
	+
	+struct i40iw_cqp_ops;
	+struct i40iw_ccq_ops;
	+struct i40iw_ceq_ops;
	+struct i40iw_aeq_ops;
	+struct i40iw_mr_ops;
	+struct i40iw_cqp_misc_ops;
	+struct i40iw_pd_ops;
	+struct i40iw_priv_qp_ops;
	+struct i40iw_priv_cq_ops;
	+struct i40iw_hmc_ops;
	+
	+enum i40iw_page_size {
	+ I40IW_PAGE_SIZE_4K = 0,
	+ I40IW_PAGE_SIZE_2M
	+};
	+
	+enum i40iw_hdrct_flags {
	+ DDP_LEN_FLAG = 0x80,
	+ DDP_HDR_FLAG = 0x40,
	+ RDMA_HDR_FLAG = 0x20
	+};
	+
	+enum i40iw_term_layers {
	+ LAYER_RDMA = 0,
	+ LAYER_DDP = 1,
	+ LAYER_MPA = 2
	+};
	+
	+enum i40iw_term_error_types {
	+ RDMAP_REMOTE_PROT = 1,
	+ RDMAP_REMOTE_OP = 2,
	+ DDP_CATASTROPHIC = 0,
	+ DDP_TAGGED_BUFFER = 1,
	+ DDP_UNTAGGED_BUFFER = 2,
	+ DDP_LLP = 3
	+};
	+
	+enum i40iw_term_rdma_errors {
	+ RDMAP_INV_STAG = 0x00,
	+ RDMAP_INV_BOUNDS = 0x01,
	+ RDMAP_ACCESS = 0x02,
	+ RDMAP_UNASSOC_STAG = 0x03,
	+ RDMAP_TO_WRAP = 0x04,
	+ RDMAP_INV_RDMAP_VER = 0x05,
	+ RDMAP_UNEXPECTED_OP = 0x06,
	+ RDMAP_CATASTROPHIC_LOCAL = 0x07,
	+ RDMAP_CATASTROPHIC_GLOBAL = 0x08,
	+ RDMAP_CANT_INV_STAG = 0x09,
	+ RDMAP_UNSPECIFIED = 0xff
	+};
	+
	+enum i40iw_term_ddp_errors {
	+ DDP_CATASTROPHIC_LOCAL = 0x00,
	+ DDP_TAGGED_INV_STAG = 0x00,
	+ DDP_TAGGED_BOUNDS = 0x01,
	+ DDP_TAGGED_UNASSOC_STAG = 0x02,
	+ DDP_TAGGED_TO_WRAP = 0x03,
	+ DDP_TAGGED_INV_DDP_VER = 0x04,
	+ DDP_UNTAGGED_INV_QN = 0x01,
	+ DDP_UNTAGGED_INV_MSN_NO_BUF = 0x02,
	+ DDP_UNTAGGED_INV_MSN_RANGE = 0x03,
	+ DDP_UNTAGGED_INV_MO = 0x04,
	+ DDP_UNTAGGED_INV_TOO_LONG = 0x05,
	+ DDP_UNTAGGED_INV_DDP_VER = 0x06
	+};
	+
	+enum i40iw_term_mpa_errors {
	+ MPA_CLOSED = 0x01,
	+ MPA_CRC = 0x02,
	+ MPA_MARKER = 0x03,
	+ MPA_REQ_RSP = 0x04,
	+};
	+
	+enum i40iw_flush_opcode {
	+ FLUSH_INVALID = 0,
	+ FLUSH_PROT_ERR,
	+ FLUSH_REM_ACCESS_ERR,
	+ FLUSH_LOC_QP_OP_ERR,
	+ FLUSH_REM_OP_ERR,
	+ FLUSH_LOC_LEN_ERR,
	+ FLUSH_GENERAL_ERR,
	+ FLUSH_FATAL_ERR
	+};
	+
	+enum i40iw_term_eventtypes {
	+ TERM_EVENT_QP_FATAL,
	+ TERM_EVENT_QP_ACCESS_ERR
	+};
	+
	+struct i40iw_terminate_hdr {
	+ u8 layer_etype;
	+ u8 error_code;
	+ u8 hdrct;
	+ u8 rsvd;
	+};
	+
	+enum i40iw_debug_flag {
	+ I40IW_DEBUG_NONE = 0x00000000,
	+ I40IW_DEBUG_ERR = 0x00000001,
	+ I40IW_DEBUG_INIT = 0x00000002,
	+ I40IW_DEBUG_DEV = 0x00000004,
	+ I40IW_DEBUG_CM = 0x00000008,
	+ I40IW_DEBUG_VERBS = 0x00000010,
	+ I40IW_DEBUG_PUDA = 0x00000020,
	+ I40IW_DEBUG_ILQ = 0x00000040,
	+ I40IW_DEBUG_IEQ = 0x00000080,
	+ I40IW_DEBUG_QP = 0x00000100,
	+ I40IW_DEBUG_CQ = 0x00000200,
	+ I40IW_DEBUG_MR = 0x00000400,
	+ I40IW_DEBUG_PBLE = 0x00000800,
	+ I40IW_DEBUG_WQE = 0x00001000,
	+ I40IW_DEBUG_AEQ = 0x00002000,
	+ I40IW_DEBUG_CQP = 0x00004000,
	+ I40IW_DEBUG_HMC = 0x00008000,
	+ I40IW_DEBUG_USER = 0x00010000,
	+ I40IW_DEBUG_VIRT = 0x00020000,
	+ I40IW_DEBUG_DCB = 0x00040000,
	+ I40IW_DEBUG_CQE = 0x00800000,
	+ I40IW_DEBUG_CLNT = 0x01000000,
	+ I40IW_DEBUG_ALL = 0xFFFFFFFF
	+};
	+
	+enum i40iw_hw_stats_index_32b {
	+ I40IW_HW_STAT_INDEX_IP4RXDISCARD = 0,
	+ I40IW_HW_STAT_INDEX_IP4RXTRUNC,
	+ I40IW_HW_STAT_INDEX_IP4TXNOROUTE,
	+ I40IW_HW_STAT_INDEX_IP6RXDISCARD,
	+ I40IW_HW_STAT_INDEX_IP6RXTRUNC,
	+ I40IW_HW_STAT_INDEX_IP6TXNOROUTE,
	+ I40IW_HW_STAT_INDEX_TCPRTXSEG,
	+ I40IW_HW_STAT_INDEX_TCPRXOPTERR,
	+ I40IW_HW_STAT_INDEX_TCPRXPROTOERR,
	+ I40IW_HW_STAT_INDEX_MAX_32
	+};
	+
	+enum i40iw_hw_stats_index_64b {
	+ I40IW_HW_STAT_INDEX_IP4RXOCTS = 0,
	+ I40IW_HW_STAT_INDEX_IP4RXPKTS,
	+ I40IW_HW_STAT_INDEX_IP4RXFRAGS,
	+ I40IW_HW_STAT_INDEX_IP4RXMCPKTS,
	+ I40IW_HW_STAT_INDEX_IP4TXOCTS,
	+ I40IW_HW_STAT_INDEX_IP4TXPKTS,
	+ I40IW_HW_STAT_INDEX_IP4TXFRAGS,
	+ I40IW_HW_STAT_INDEX_IP4TXMCPKTS,
	+ I40IW_HW_STAT_INDEX_IP6RXOCTS,
	+ I40IW_HW_STAT_INDEX_IP6RXPKTS,
	+ I40IW_HW_STAT_INDEX_IP6RXFRAGS,
	+ I40IW_HW_STAT_INDEX_IP6RXMCPKTS,
	+ I40IW_HW_STAT_INDEX_IP6TXOCTS,
	+ I40IW_HW_STAT_INDEX_IP6TXPKTS,
	+ I40IW_HW_STAT_INDEX_IP6TXFRAGS,
	+ I40IW_HW_STAT_INDEX_IP6TXMCPKTS,
	+ I40IW_HW_STAT_INDEX_TCPRXSEGS,
	+ I40IW_HW_STAT_INDEX_TCPTXSEG,
	+ I40IW_HW_STAT_INDEX_RDMARXRDS,
	+ I40IW_HW_STAT_INDEX_RDMARXSNDS,
	+ I40IW_HW_STAT_INDEX_RDMARXWRS,
	+ I40IW_HW_STAT_INDEX_RDMATXRDS,
	+ I40IW_HW_STAT_INDEX_RDMATXSNDS,
	+ I40IW_HW_STAT_INDEX_RDMATXWRS,
	+ I40IW_HW_STAT_INDEX_RDMAVBND,
	+ I40IW_HW_STAT_INDEX_RDMAVINV,
	+ I40IW_HW_STAT_INDEX_MAX_64
	+};
	+
	+struct i40iw_dev_hw_stats_offsets {
	+ u32 stats_offset_32[I40IW_HW_STAT_INDEX_MAX_32];
	+ u32 stats_offset_64[I40IW_HW_STAT_INDEX_MAX_64];
	+};
	+
	+struct i40iw_dev_hw_stats {
	+ u64 stats_value_32[I40IW_HW_STAT_INDEX_MAX_32];
	+ u64 stats_value_64[I40IW_HW_STAT_INDEX_MAX_64];
	+};
	+
	+struct i40iw_vsi_pestat {
	+ struct i40iw_hw *hw;
	+ struct i40iw_dev_hw_stats hw_stats;
	+ struct i40iw_dev_hw_stats last_read_hw_stats;
	+ struct i40iw_dev_hw_stats_offsets hw_stats_offsets;
	+ struct OS_TIMER stats_timer;
	+ spinlock_t lock; /* rdma stats lock */
	+};
	+
	+struct i40iw_hw {
	+ u8 IOMEM *hw_addr;
	+ void *dev_context;
	+ struct i40iw_hmc_info hmc;
	+};
	+
	+struct i40iw_pfpdu {
	+ struct list_head rxlist;
	+ u32 rcv_nxt;
	+ u32 fps;
	+ u32 max_fpdu_data;
	+ bool mode;
	+ bool mpa_crc_err;
	+ u64 total_ieq_bufs;
	+ u64 fpdu_processed;
	+ u64 bad_seq_num;
	+ u64 crc_err;
	+ u64 no_tx_bufs;
	+ u64 tx_err;
	+ u64 out_of_order;
	+ u64 pmode_count;
	+};
	+
	+struct i40iw_sc_pd {
	+ u32 size;
	+ struct i40iw_sc_dev *dev;
	+ u16 pd_id;
	+};
	+
	+struct i40iw_cqp_quanta {
	+ u64 elem[I40IW_CQP_WQE_SIZE];
	+};
	+
	+struct i40iw_sc_cqp {
	+ u32 size;
	+ u64 sq_pa;
	+ u64 host_ctx_pa;
	+ void *back_cqp;
	+ struct i40iw_sc_dev *dev;
	+ enum i40iw_status_code (process_cqp_sds)(struct i40iw_sc_dev ,
	+ struct i40iw_update_sds_info *);
	+ struct i40iw_dma_mem sdbuf;
	+ struct i40iw_ring sq_ring;
	+ struct i40iw_cqp_quanta *sq_base;
	+ u64 *host_ctx;
	+ u64 *scratch_array;
	+ u32 cqp_id;
	+ u32 sq_size;
	+ u32 hw_sq_size;
	+ u8 struct_ver;
	+ u8 polarity;
	+ bool en_datacenter_tcp;
	+ u8 hmc_profile;
	+ u8 enabled_vf_count;
	+ u8 timeout_count;
	+};
	+
	+struct i40iw_sc_aeq {
	+ u32 size;
	+ u64 aeq_elem_pa;
	+ struct i40iw_sc_dev *dev;
	+ struct i40iw_sc_aeqe *aeqe_base;
	+ void *pbl_list;
	+ u32 elem_cnt;
	+ struct i40iw_ring aeq_ring;
	+ bool virtual_map;
	+ u8 pbl_chunk_size;
	+ u32 first_pm_pbl_idx;
	+ u8 polarity;
	+};
	+
	+struct i40iw_sc_ceq {
	+ u32 size;
	+ u64 ceq_elem_pa;
	+ struct i40iw_sc_dev *dev;
	+ struct i40iw_ceqe *ceqe_base;
	+ void *pbl_list;
	+ u32 ceq_id;
	+ u32 elem_cnt;
	+ struct i40iw_ring ceq_ring;
	+ bool virtual_map;
	+ u8 pbl_chunk_size;
	+ bool tph_en;
	+ u8 tph_val;
	+ u32 first_pm_pbl_idx;
	+ u8 polarity;
	+};
	+
	+struct i40iw_sc_cq {
	+ struct i40iw_cq_uk cq_uk;
	+ u64 cq_pa;
	+ u64 shadow_area_pa;
	+ struct i40iw_sc_dev *dev;
	+ struct i40iw_sc_vsi *vsi;
	+ void *pbl_list;
	+ void *back_cq;
	+ u32 ceq_id;
	+ u32 shadow_read_threshold;
	+ bool ceqe_mask;
	+ bool virtual_map;
	+ u8 pbl_chunk_size;
	+ u8 cq_type;
	+ bool ceq_id_valid;
	+ bool tph_en;
	+ u8 tph_val;
	+ u32 first_pm_pbl_idx;
	+ bool check_overflow;
	+};
	+
	+struct i40iw_sc_qp {
	+ struct i40iw_qp_uk qp_uk;
	+ u64 sq_pa;
	+ u64 rq_pa;
	+ u64 hw_host_ctx_pa;
	+ u64 shadow_area_pa;
	+ u64 q2_pa;
	+ struct i40iw_sc_dev *dev;
	+ struct i40iw_sc_vsi *vsi;
	+ struct i40iw_sc_pd *pd;
	+ u64 *hw_host_ctx;
	+ void *llp_stream_handle;
	+ void *back_qp;
	+ struct i40iw_pfpdu pfpdu;
	+ u8 *q2_buf;
	+ u64 qp_compl_ctx;
	+ u16 qs_handle;
	+ u16 push_idx;
	+ u8 sq_tph_val;
	+ u8 rq_tph_val;
	+ u8 qp_state;
	+ u8 qp_type;
	+ u8 hw_sq_size;
	+ u8 hw_rq_size;
	+ u8 src_mac_addr_idx;
	+ bool sq_tph_en;
	+ bool rq_tph_en;
	+ bool rcv_tph_en;
	+ bool xmit_tph_en;
	+ bool virtual_map;
	+ bool flush_sq;
	+ bool flush_rq;
	+ bool sq_flush;
	+ enum i40iw_flush_opcode flush_code;
	+ enum i40iw_term_eventtypes eventtype;
	+ u8 term_flags;
	+};
	+
	+struct i40iw_hmc_fpm_misc {
	+ u32 max_ceqs;
	+ u32 max_sds;
	+ u32 xf_block_size;
	+ u32 q1_block_size;
	+ u32 ht_multiplier;
	+ u32 timer_bucket;
	+};
	+
	+struct i40iw_qos {
	+ struct list_head qplist;
	+ spinlock_t lock; /* qos list */
	+ u16 qs_handle;
	+};
	+
	+#define I40IW_INVALID_FCN_ID 0xff
	+struct i40iw_sc_vsi {
	+ struct i40iw_sc_dev *dev;
	+ void back_vsi; / Owned by OS */
	+ u32 ilq_count;
	+ struct i40iw_virt_mem ilq_mem;
	+ struct i40iw_puda_rsrc *ilq;
	+ u32 ieq_count;
	+ struct i40iw_virt_mem ieq_mem;
	+ struct i40iw_puda_rsrc *ieq;
	+ u16 exception_lan_queue;
	+ u16 mtu;
	+ u8 fcn_id;
	+ bool stats_fcn_id_alloc;
	+ struct i40iw_qos qos[I40IW_MAX_USER_PRIORITY];
	+ struct i40iw_vsi_pestat *pestat;
	+};
	+
	+struct i40iw_sc_dev {
	+ struct list_head cqp_cmd_head; /* head of the CQP command list */
	+ spinlock_t cqp_lock; /* cqp list sync */
	+ struct i40iw_dev_uk dev_uk;
	+ bool fcn_id_array[I40IW_MAX_STATS_COUNT];
	+ u64 fpm_query_buf_pa;
	+ u64 fpm_commit_buf_pa;
	+ u64 *fpm_query_buf;
	+ u64 *fpm_commit_buf;
	+ void *back_dev;
	+ struct i40iw_hw *hw;
	+ u8 IOMEM *db_addr;
	+ struct i40iw_hmc_info *hmc_info;
	+ struct i40iw_sc_cqp *cqp;
	+ struct i40iw_sc_aeq *aeq;
	+ struct i40iw_sc_ceq *ceq[I40IW_CEQ_MAX_COUNT];
	+ struct i40iw_sc_cq *ccq;
	+ struct i40iw_cqp_ops *cqp_ops;
	+ struct i40iw_ccq_ops *ccq_ops;
	+ struct i40iw_ceq_ops *ceq_ops;
	+ struct i40iw_aeq_ops *aeq_ops;
	+ struct i40iw_pd_ops *iw_pd_ops;
	+ struct i40iw_priv_qp_ops *iw_priv_qp_ops;
	+ struct i40iw_priv_cq_ops *iw_priv_cq_ops;
	+ struct i40iw_mr_ops *mr_ops;
	+ struct i40iw_cqp_misc_ops *cqp_misc_ops;
	+ struct i40iw_hmc_ops *hmc_ops;
	+
	+ struct i40iw_hmc_fpm_misc hmc_fpm_misc;
	+ u32 debug_mask;
	+ u8 hmc_fn_id;
	+ bool is_pf;
	+ bool vchnl_up;
	+ bool ceq_valid;
	+ u64 cqp_cmd_stats[OP_SIZE_CQP_STAT_ARRAY];
	+ u8 hw_rev;
	+};
	+
	+struct i40iw_modify_cq_info {
	+ u64 cq_pa;
	+ struct i40iw_cqe *cq_base;
	+ void *pbl_list;
	+ u32 ceq_id;
	+ u32 cq_size;
	+ u32 shadow_read_threshold;
	+ bool virtual_map;
	+ u8 pbl_chunk_size;
	+ bool check_overflow;
	+ bool cq_resize;
	+ bool ceq_change;
	+ bool check_overflow_change;
	+ u32 first_pm_pbl_idx;
	+ bool ceq_valid;
	+};
	+
	+struct i40iw_create_qp_info {
	+ u8 next_iwarp_state;
	+ bool ord_valid;
	+ bool tcp_ctx_valid;
	+ bool cq_num_valid;
	+ bool arp_cache_idx_valid;
	+};
	+
	+struct i40iw_modify_qp_info {
	+ u64 rx_win0;
	+ u64 rx_win1;
	+ u8 next_iwarp_state;
	+ u8 termlen;
	+ bool ord_valid;
	+ bool tcp_ctx_valid;
	+ bool cq_num_valid;
	+ bool arp_cache_idx_valid;
	+ bool reset_tcp_conn;
	+ bool remove_hash_idx;
	+ bool dont_send_term;
	+ bool dont_send_fin;
	+ bool cached_var_valid;
	+ bool force_loopback;
	+};
	+
	+struct i40iw_ccq_cqe_info {
	+ struct i40iw_sc_cqp *cqp;
	+ u64 scratch;
	+ u32 op_ret_val;
	+ u16 maj_err_code;
	+ u16 min_err_code;
	+ u8 op_code;
	+ bool error;
	+};
	+
	+struct i40iw_l2params {
	+ u16 qs_handle_list[I40IW_MAX_USER_PRIORITY];
	+ u16 mtu;
	+};
	+
	+struct i40iw_vsi_init_info {
	+ struct i40iw_sc_dev *dev;
	+ void *back_vsi;
	+ struct i40iw_l2params *params;
	+ u16 exception_lan_queue;
	+};
	+
	+struct i40iw_vsi_stats_info {
	+ struct i40iw_vsi_pestat *pestat;
	+ u8 fcn_id;
	+ bool alloc_fcn_id;
	+};
	+
	+struct i40iw_device_init_info {
	+ u64 fpm_query_buf_pa;
	+ u64 fpm_commit_buf_pa;
	+ u64 *fpm_query_buf;
	+ u64 *fpm_commit_buf;
	+ struct i40iw_hw *hw;
	+ void IOMEM *bar0;
	+ enum i40iw_status_code (vchnl_send)(struct i40iw_sc_dev , u32, u8 *, u16);
	+ u8 hmc_fn_id;
	+ bool is_pf;
	+ u32 debug_mask;
	+};
	+
	+enum i40iw_cqp_hmc_profile {
	+ I40IW_HMC_PROFILE_DEFAULT = 1,
	+ I40IW_HMC_PROFILE_FAVOR_VF = 2,
	+ I40IW_HMC_PROFILE_EQUAL = 3,
	+};
	+
	+struct i40iw_cqp_init_info {
	+ u64 cqp_compl_ctx;
	+ u64 host_ctx_pa;
	+ u64 sq_pa;
	+ struct i40iw_sc_dev *dev;
	+ struct i40iw_cqp_quanta *sq;
	+ u64 *host_ctx;
	+ u64 *scratch_array;
	+ u32 sq_size;
	+ u8 struct_ver;
	+ bool en_datacenter_tcp;
	+ u8 hmc_profile;
	+ u8 enabled_vf_count;
	+};
	+
	+struct i40iw_ceq_init_info {
	+ u64 ceqe_pa;
	+ struct i40iw_sc_dev *dev;
	+ u64 *ceqe_base;
	+ void *pbl_list;
	+ u32 elem_cnt;
	+ u32 ceq_id;
	+ bool virtual_map;
	+ u8 pbl_chunk_size;
	+ bool tph_en;
	+ u8 tph_val;
	+ u32 first_pm_pbl_idx;
	+};
	+
	+struct i40iw_aeq_init_info {
	+ u64 aeq_elem_pa;
	+ struct i40iw_sc_dev *dev;
	+ u32 *aeqe_base;
	+ void *pbl_list;
	+ u32 elem_cnt;
	+ bool virtual_map;
	+ u8 pbl_chunk_size;
	+ u32 first_pm_pbl_idx;
	+};
	+
	+struct i40iw_ccq_init_info {
	+ u64 cq_pa;
	+ u64 shadow_area_pa;
	+ struct i40iw_sc_dev *dev;
	+ struct i40iw_cqe *cq_base;
	+ u64 *shadow_area;
	+ void *pbl_list;
	+ u32 num_elem;
	+ u32 ceq_id;
	+ u32 shadow_read_threshold;
	+ bool ceqe_mask;
	+ bool ceq_id_valid;
	+ bool tph_en;
	+ u8 tph_val;
	+ bool avoid_mem_cflct;
	+ bool virtual_map;
	+ u8 pbl_chunk_size;
	+ u32 first_pm_pbl_idx;
	+};
	+
	+struct i40iwarp_offload_info {
	+ u16 rcv_mark_offset;
	+ u16 snd_mark_offset;
	+ u16 pd_id;
	+ u8 ddp_ver;
	+ u8 rdmap_ver;
	+ u8 ord_size;
	+ u8 ird_size;
	+ bool wr_rdresp_en;
	+ bool rd_enable;
	+ bool snd_mark_en;
	+ bool rcv_mark_en;
	+ bool bind_en;
	+ bool fast_reg_en;
	+ bool priv_mode_en;
	+ bool lsmm_present;
	+ u8 iwarp_mode;
	+ bool align_hdrs;
	+ bool rcv_no_mpa_crc;
	+
	+ u8 last_byte_sent;
	+ bool err_rq_idx_valid;
	+ u16 err_rq_idx;
	+};
	+
	+struct i40iw_tcp_offload_info {
	+ bool ipv4;
	+ bool no_nagle;
	+ bool insert_vlan_tag;
	+ bool time_stamp;
	+ u8 cwnd_inc_limit;
	+ bool drop_ooo_seg;
	+ u8 dup_ack_thresh;
	+ u8 ttl;
	+ u8 src_mac_addr_idx;
	+ bool avoid_stretch_ack;
	+ u8 tos;
	+ u16 src_port;
	+ u16 dst_port;
	+ u32 dest_ip_addr0;
	+ u32 dest_ip_addr1;
	+ u32 dest_ip_addr2;
	+ u32 dest_ip_addr3;
	+ u32 snd_mss;
	+ u16 vlan_tag;
	+ u16 arp_idx;
	+ u32 flow_label;
	+ bool wscale;
	+ u8 tcp_state;
	+ u8 snd_wscale;
	+ u8 rcv_wscale;
	+ u32 time_stamp_recent;
	+ u32 time_stamp_age;
	+ u32 snd_nxt;
	+ u32 snd_wnd;
	+ u32 rcv_nxt;
	+ u32 rcv_wnd;
	+ u32 snd_max;
	+ u32 snd_una;
	+ u32 srtt;
	+ u32 rtt_var;
	+ u32 ss_thresh;
	+ u32 cwnd;
	+ u32 snd_wl1;
	+ u32 snd_wl2;
	+ u32 max_snd_window;
	+ u8 rexmit_thresh;
	+ u32 local_ipaddr0;
	+ u32 local_ipaddr1;
	+ u32 local_ipaddr2;
	+ u32 local_ipaddr3;
	+ bool ignore_tcp_opt;
	+ bool ignore_tcp_uns_opt;
	+};
	+
	+struct i40iw_qp_host_ctx_info {
	+ u64 qp_compl_ctx;
	+ struct i40iw_tcp_offload_info *tcp_info;
	+ struct i40iwarp_offload_info *iwarp_info;
	+ u32 send_cq_num;
	+ u32 rcv_cq_num;
	+ u16 push_idx;
	+ bool push_mode_en;
	+ bool tcp_info_valid;
	+ bool iwarp_info_valid;
	+};
	+
	+struct i40iw_aeqe_info {
	+ u64 compl_ctx;
	+ u32 qp_cq_id;
	+ u16 ae_id;
	+ u16 wqe_idx;
	+ u8 tcp_state;
	+ u8 iwarp_state;
	+ bool qp;
	+ bool cq;
	+ bool sq;
	+ bool in_rdrsp_wr;
	+ bool out_rdrsp;
	+ u8 q2_data_written;
	+ bool aeqe_overflow;
	+};
	+
	+struct i40iw_allocate_stag_info {
	+ u64 total_len;
	+ u32 chunk_size;
	+ u32 stag_idx;
	+ u32 page_size;
	+ u16 pd_id;
	+ u16 access_rights;
	+ bool remote_access;
	+ bool use_hmc_fcn_index;
	+ u8 hmc_fcn_index;
	+ bool use_pf_rid;
	+};
	+
	+struct i40iw_reg_ns_stag_info {
	+ u64 reg_addr_pa;
	+ u64 fbo;
	+ void *va;
	+ u64 total_len;
	+ u32 page_size;
	+ u32 chunk_size;
	+ u32 first_pm_pbl_index;
	+ enum i40iw_addressing_type addr_type;
	+ i40iw_stag_index stag_idx;
	+ u16 access_rights;
	+ u16 pd_id;
	+ i40iw_stag_key stag_key;
	+ bool use_hmc_fcn_index;
	+ u8 hmc_fcn_index;
	+ bool use_pf_rid;
	+};
	+
	+struct i40iw_fast_reg_stag_info {
	+ u64 wr_id;
	+ u64 reg_addr_pa;
	+ u64 fbo;
	+ void *va;
	+ u64 total_len;
	+ u32 page_size;
	+ u32 chunk_size;
	+ u32 first_pm_pbl_index;
	+ enum i40iw_addressing_type addr_type;
	+ i40iw_stag_index stag_idx;
	+ u16 access_rights;
	+ u16 pd_id;
	+ i40iw_stag_key stag_key;
	+ bool local_fence;
	+ bool read_fence;
	+ bool signaled;
	+ bool use_hmc_fcn_index;
	+ u8 hmc_fcn_index;
	+ bool use_pf_rid;
	+ bool defer_flag;
	+};
	+
	+struct i40iw_dealloc_stag_info {
	+ u32 stag_idx;
	+ u16 pd_id;
	+ bool mr;
	+ bool dealloc_pbl;
	+};
	+
	+struct i40iw_register_shared_stag {
	+ void *va;
	+ enum i40iw_addressing_type addr_type;
	+ i40iw_stag_index new_stag_idx;
	+ i40iw_stag_index parent_stag_idx;
	+ u32 access_rights;
	+ u16 pd_id;
	+ i40iw_stag_key new_stag_key;
	+};
	+
	+struct i40iw_qp_init_info {
	+ struct i40iw_qp_uk_init_info qp_uk_init_info;
	+ struct i40iw_sc_pd *pd;
	+ struct i40iw_sc_vsi *vsi;
	+ u64 *host_ctx;
	+ u8 *q2;
	+ u64 sq_pa;
	+ u64 rq_pa;
	+ u64 host_ctx_pa;
	+ u64 q2_pa;
	+ u64 shadow_area_pa;
	+ u8 sq_tph_val;
	+ u8 rq_tph_val;
	+ u8 type;
	+ bool sq_tph_en;
	+ bool rq_tph_en;
	+ bool rcv_tph_en;
	+ bool xmit_tph_en;
	+ bool virtual_map;
	+};
	+
	+struct i40iw_cq_init_info {
	+ struct i40iw_sc_dev *dev;
	+ u64 cq_base_pa;
	+ u64 shadow_area_pa;
	+ u32 ceq_id;
	+ u32 shadow_read_threshold;
	+ bool virtual_map;
	+ bool ceqe_mask;
	+ u8 pbl_chunk_size;
	+ u32 first_pm_pbl_idx;
	+ bool ceq_id_valid;
	+ bool tph_en;
	+ u8 tph_val;
	+ u8 type;
	+ struct i40iw_cq_uk_init_info cq_uk_init_info;
	+};
	+
	+struct i40iw_upload_context_info {
	+ u64 buf_pa;
	+ bool freeze_qp;
	+ bool raw_format;
	+ u32 qp_id;
	+ u8 qp_type;
	+};
	+
	+struct i40iw_add_arp_cache_entry_info {
	+ u8 mac_addr[6];
	+ u32 reach_max;
	+ u16 arp_index;
	+ bool permanent;
	+};
	+
	+struct i40iw_apbvt_info {
	+ u16 port;
	+ bool add;
	+};
	+
	+enum i40iw_quad_entry_type {
	+ I40IW_QHASH_TYPE_TCP_ESTABLISHED = 1,
	+ I40IW_QHASH_TYPE_TCP_SYN,
	+};
	+
	+enum i40iw_quad_hash_manage_type {
	+ I40IW_QHASH_MANAGE_TYPE_DELETE = 0,
	+ I40IW_QHASH_MANAGE_TYPE_ADD,
	+ I40IW_QHASH_MANAGE_TYPE_MODIFY
	+};
	+
	+struct i40iw_qhash_table_info {
	+ struct i40iw_sc_vsi *vsi;
	+ enum i40iw_quad_hash_manage_type manage;
	+ enum i40iw_quad_entry_type entry_type;
	+ bool vlan_valid;
	+ bool ipv4_valid;
	+ u8 mac_addr[6];
	+ u16 vlan_id;
	+ u32 qp_num;
	+ u32 dest_ip[4];
	+ u32 src_ip[4];
	+ u16 dest_port;
	+ u16 src_port;
	+};
	+
	+struct i40iw_local_mac_ipaddr_entry_info {
	+ u8 mac_addr[6];
	+ u8 entry_idx;
	+};
	+
	+struct i40iw_cqp_manage_push_page_info {
	+ u32 push_idx;
	+ u16 qs_handle;
	+ u8 free_page;
	+};
	+
	+struct i40iw_qp_flush_info {
	+ u16 sq_minor_code;
	+ u16 sq_major_code;
	+ u16 rq_minor_code;
	+ u16 rq_major_code;
	+ u16 ae_code;
	+ u8 ae_source;
	+ bool sq;
	+ bool rq;
	+ bool userflushcode;
	+ bool generate_ae;
	+};
	+
	+struct i40iw_gen_ae_info {
	+ u16 ae_code;
	+ u8 ae_source;
	+};
	+
	+struct i40iw_cqp_ops {
	+ enum i40iw_status_code (cqp_init)(struct i40iw_sc_cqp ,
	+ struct i40iw_cqp_init_info *);
	+ enum i40iw_status_code (cqp_create)(struct i40iw_sc_cqp , u16 , u16 );
	+ void (cqp_post_sq)(struct i40iw_sc_cqp );
	+ u64 (cqp_get_next_send_wqe)(struct i40iw_sc_cqp *, u64 scratch);
	+ enum i40iw_status_code (cqp_destroy)(struct i40iw_sc_cqp );
	+ enum i40iw_status_code (poll_for_cqp_op_done)(struct i40iw_sc_cqp , u8,
	+ struct i40iw_ccq_cqe_info *);
	+};
	+
	+struct i40iw_ccq_ops {
	+ enum i40iw_status_code (ccq_init)(struct i40iw_sc_cq ,
	+ struct i40iw_ccq_init_info *);
	+ enum i40iw_status_code (ccq_create)(struct i40iw_sc_cq , u64, bool, bool);
	+ enum i40iw_status_code (ccq_destroy)(struct i40iw_sc_cq , u64, bool);
	+ enum i40iw_status_code (ccq_create_done)(struct i40iw_sc_cq );
	+ enum i40iw_status_code (ccq_get_cqe_info)(struct i40iw_sc_cq ,
	+ struct i40iw_ccq_cqe_info *);
	+ void (ccq_arm)(struct i40iw_sc_cq );
	+};
	+
	+struct i40iw_ceq_ops {
	+ enum i40iw_status_code (ceq_init)(struct i40iw_sc_ceq ,
	+ struct i40iw_ceq_init_info *);
	+ enum i40iw_status_code (ceq_create)(struct i40iw_sc_ceq , u64, bool);
	+ enum i40iw_status_code (cceq_create_done)(struct i40iw_sc_ceq );
	+ enum i40iw_status_code (cceq_destroy_done)(struct i40iw_sc_ceq );
	+ enum i40iw_status_code (cceq_create)(struct i40iw_sc_ceq , u64);
	+ enum i40iw_status_code (ceq_destroy)(struct i40iw_sc_ceq , u64, bool);
	+ void (process_ceq)(struct i40iw_sc_dev , struct i40iw_sc_ceq );
	+};
	+
	+struct i40iw_aeq_ops {
	+ enum i40iw_status_code (aeq_init)(struct i40iw_sc_aeq ,
	+ struct i40iw_aeq_init_info *);
	+ enum i40iw_status_code (aeq_create)(struct i40iw_sc_aeq , u64, bool);
	+ enum i40iw_status_code (aeq_destroy)(struct i40iw_sc_aeq , u64, bool);
	+ enum i40iw_status_code (get_next_aeqe)(struct i40iw_sc_aeq ,
	+ struct i40iw_aeqe_info *);
	+ enum i40iw_status_code (repost_aeq_entries)(struct i40iw_sc_dev , u32);
	+ enum i40iw_status_code (aeq_create_done)(struct i40iw_sc_aeq );
	+ enum i40iw_status_code (aeq_destroy_done)(struct i40iw_sc_aeq );
	+};
	+
	+struct i40iw_pd_ops {
	+ void (pd_init)(struct i40iw_sc_dev , struct i40iw_sc_pd *, u16);
	+};
	+
	+struct i40iw_priv_qp_ops {
	+ enum i40iw_status_code (qp_init)(struct i40iw_sc_qp , struct i40iw_qp_init_info *);
	+ enum i40iw_status_code (qp_create)(struct i40iw_sc_qp ,
	+ struct i40iw_create_qp_info *, u64, bool);
	+ enum i40iw_status_code (qp_modify)(struct i40iw_sc_qp ,
	+ struct i40iw_modify_qp_info *, u64, bool);
	+ enum i40iw_status_code (qp_destroy)(struct i40iw_sc_qp , u64, bool, bool, bool);
	+ enum i40iw_status_code (qp_flush_wqes)(struct i40iw_sc_qp ,
	+ struct i40iw_qp_flush_info *, u64, bool);
	+ enum i40iw_status_code (qp_upload_context)(struct i40iw_sc_dev ,
	+ struct i40iw_upload_context_info *,
	+ u64, bool);
	+ enum i40iw_status_code (qp_setctx)(struct i40iw_sc_qp , u64 *,
	+ struct i40iw_qp_host_ctx_info *);
	+
	+ void (qp_send_lsmm)(struct i40iw_sc_qp , void *, u32, i40iw_stag);
	+ void (qp_send_lsmm_nostag)(struct i40iw_sc_qp , void *, u32);
	+ void (qp_send_rtt)(struct i40iw_sc_qp , bool);
	+ enum i40iw_status_code (iw_mr_fast_register)(struct i40iw_sc_qp ,
	+ struct i40iw_fast_reg_stag_info *,
	+ bool);
	+};
	+
	+struct i40iw_priv_cq_ops {
	+ enum i40iw_status_code (cq_init)(struct i40iw_sc_cq , struct i40iw_cq_init_info *);
	+ enum i40iw_status_code (cq_create)(struct i40iw_sc_cq , u64, bool, bool);
	+ enum i40iw_status_code (cq_destroy)(struct i40iw_sc_cq , u64, bool);
	+ enum i40iw_status_code (cq_modify)(struct i40iw_sc_cq ,
	+ struct i40iw_modify_cq_info *, u64, bool);
	+};
	+
	+struct i40iw_mr_ops {
	+ enum i40iw_status_code (alloc_stag)(struct i40iw_sc_dev ,
	+ struct i40iw_allocate_stag_info *, u64, bool);
	+ enum i40iw_status_code (mr_reg_non_shared)(struct i40iw_sc_dev ,
	+ struct i40iw_reg_ns_stag_info *,
	+ u64, bool);
	+ enum i40iw_status_code (mr_reg_shared)(struct i40iw_sc_dev ,
	+ struct i40iw_register_shared_stag *,
	+ u64, bool);
	+ enum i40iw_status_code (dealloc_stag)(struct i40iw_sc_dev ,
	+ struct i40iw_dealloc_stag_info *,
	+ u64, bool);
	+ enum i40iw_status_code (query_stag)(struct i40iw_sc_dev , u64, u32, bool);
	+ enum i40iw_status_code (mw_alloc)(struct i40iw_sc_dev , u64, u32, u16, bool);
	+};
	+
	+struct i40iw_cqp_misc_ops {
	+ enum i40iw_status_code (manage_push_page)(struct i40iw_sc_cqp ,
	+ struct i40iw_cqp_manage_push_page_info *,
	+ u64, bool);
	+ enum i40iw_status_code (manage_hmc_pm_func_table)(struct i40iw_sc_cqp ,
	+ u64, u8, bool, bool);
	+ enum i40iw_status_code (commit_fpm_values)(struct i40iw_sc_cqp , u64, u8,
	+ struct i40iw_dma_mem *, bool, u8);
	+ enum i40iw_status_code (query_fpm_values)(struct i40iw_sc_cqp , u64, u8,
	+ struct i40iw_dma_mem *, bool, u8);
	+ enum i40iw_status_code (static_hmc_pages_allocated)(struct i40iw_sc_cqp ,
	+ u64, u8, bool, bool);
	+ enum i40iw_status_code (add_arp_cache_entry)(struct i40iw_sc_cqp ,
	+ struct i40iw_add_arp_cache_entry_info *,
	+ u64, bool);
	+ enum i40iw_status_code (del_arp_cache_entry)(struct i40iw_sc_cqp , u64, u16, bool);
	+ enum i40iw_status_code (query_arp_cache_entry)(struct i40iw_sc_cqp , u64, u16, bool);
	+ enum i40iw_status_code (manage_apbvt_entry)(struct i40iw_sc_cqp ,
	+ struct i40iw_apbvt_info *, u64, bool);
	+ enum i40iw_status_code (manage_qhash_table_entry)(struct i40iw_sc_cqp ,
	+ struct i40iw_qhash_table_info *, u64, bool);
	+ enum i40iw_status_code (alloc_local_mac_ipaddr_table_entry)(struct i40iw_sc_cqp , u64, bool);
	+ enum i40iw_status_code (add_local_mac_ipaddr_entry)(struct i40iw_sc_cqp ,
	+ struct i40iw_local_mac_ipaddr_entry_info *,
	+ u64, bool);
	+ enum i40iw_status_code (del_local_mac_ipaddr_entry)(struct i40iw_sc_cqp , u64, u8, u8, bool);
	+ enum i40iw_status_code (cqp_nop)(struct i40iw_sc_cqp , u64, bool);
	+ enum i40iw_status_code (*commit_fpm_values_done)(struct i40iw_sc_cqp
	+ *);
	+ enum i40iw_status_code (query_fpm_values_done)(struct i40iw_sc_cqp );
	+ enum i40iw_status_code (manage_hmc_pm_func_table_done)(struct i40iw_sc_cqp );
	+ enum i40iw_status_code (update_suspend_qp)(struct i40iw_sc_cqp , struct i40iw_sc_qp *, u64);
	+ enum i40iw_status_code (update_resume_qp)(struct i40iw_sc_cqp , struct i40iw_sc_qp *, u64);
	+};
	+
	+struct i40iw_hmc_ops {
	+ enum i40iw_status_code (init_iw_hmc)(struct i40iw_sc_dev , u8);
	+ enum i40iw_status_code (parse_fpm_query_buf)(u64 , struct i40iw_hmc_info *,
	+ struct i40iw_hmc_fpm_misc *);
	+ enum i40iw_status_code (configure_iw_fpm)(struct i40iw_sc_dev , u8);
	+ enum i40iw_status_code (parse_fpm_commit_buf)(u64 , struct i40iw_hmc_obj_info , u32 sd);
	+ enum i40iw_status_code (create_hmc_object)(struct i40iw_sc_dev dev,
	+ struct i40iw_hmc_create_obj_info *);
	+ enum i40iw_status_code (del_hmc_object)(struct i40iw_sc_dev dev,
	+ struct i40iw_hmc_del_obj_info *,
	+ bool reset);
	+ enum i40iw_status_code (pf_init_vfhmc)(struct i40iw_sc_dev , u8, u32 *);
	+ enum i40iw_status_code (vf_configure_vffpm)(struct i40iw_sc_dev , u32 *);
	+};
	+
	+struct cqp_info {
	+ union {
	+ struct {
	+ struct i40iw_sc_qp *qp;
	+ struct i40iw_create_qp_info info;
	+ u64 scratch;
	+ } qp_create;
	+
	+ struct {
	+ struct i40iw_sc_qp *qp;
	+ struct i40iw_modify_qp_info info;
	+ u64 scratch;
	+ } qp_modify;
	+
	+ struct {
	+ struct i40iw_sc_qp *qp;
	+ u64 scratch;
	+ bool remove_hash_idx;
	+ bool ignore_mw_bnd;
	+ } qp_destroy;
	+
	+ struct {
	+ struct i40iw_sc_cq *cq;
	+ u64 scratch;
	+ bool check_overflow;
	+ } cq_create;
	+
	+ struct {
	+ struct i40iw_sc_cq *cq;
	+ u64 scratch;
	+ } cq_destroy;
	+
	+ struct {
	+ struct i40iw_sc_dev *dev;
	+ struct i40iw_allocate_stag_info info;
	+ u64 scratch;
	+ } alloc_stag;
	+
	+ struct {
	+ struct i40iw_sc_dev *dev;
	+ u64 scratch;
	+ u32 mw_stag_index;
	+ u16 pd_id;
	+ } mw_alloc;
	+
	+ struct {
	+ struct i40iw_sc_dev *dev;
	+ struct i40iw_reg_ns_stag_info info;
	+ u64 scratch;
	+ } mr_reg_non_shared;
	+
	+ struct {
	+ struct i40iw_sc_dev *dev;
	+ struct i40iw_dealloc_stag_info info;
	+ u64 scratch;
	+ } dealloc_stag;
	+
	+ struct {
	+ struct i40iw_sc_cqp *cqp;
	+ struct i40iw_local_mac_ipaddr_entry_info info;
	+ u64 scratch;
	+ } add_local_mac_ipaddr_entry;
	+
	+ struct {
	+ struct i40iw_sc_cqp *cqp;
	+ struct i40iw_add_arp_cache_entry_info info;
	+ u64 scratch;
	+ } add_arp_cache_entry;
	+
	+ struct {
	+ struct i40iw_sc_cqp *cqp;
	+ u64 scratch;
	+ u8 entry_idx;
	+ u8 ignore_ref_count;
	+ } del_local_mac_ipaddr_entry;
	+
	+ struct {
	+ struct i40iw_sc_cqp *cqp;
	+ u64 scratch;
	+ u16 arp_index;
	+ } del_arp_cache_entry;
	+
	+ struct {
	+ struct i40iw_sc_cqp *cqp;
	+ struct i40iw_cqp_manage_push_page_info info;
	+ u64 scratch;
	+ } manage_push_page;
	+
	+ struct {
	+ struct i40iw_sc_dev *dev;
	+ struct i40iw_upload_context_info info;
	+ u64 scratch;
	+ } qp_upload_context;
	+
	+ struct {
	+ struct i40iw_sc_cqp *cqp;
	+ u64 scratch;
	+ } alloc_local_mac_ipaddr_entry;
	+
	+ struct {
	+ struct i40iw_sc_dev *dev;
	+ struct i40iw_hmc_fcn_info info;
	+ u64 scratch;
	+ } manage_hmc_pm;
	+
	+ struct {
	+ struct i40iw_sc_ceq *ceq;
	+ u64 scratch;
	+ } ceq_create;
	+
	+ struct {
	+ struct i40iw_sc_ceq *ceq;
	+ u64 scratch;
	+ } ceq_destroy;
	+
	+ struct {
	+ struct i40iw_sc_aeq *aeq;
	+ u64 scratch;
	+ } aeq_create;
	+
	+ struct {
	+ struct i40iw_sc_aeq *aeq;
	+ u64 scratch;
	+ } aeq_destroy;
	+
	+ struct {
	+ struct i40iw_sc_qp *qp;
	+ struct i40iw_qp_flush_info info;
	+ u64 scratch;
	+ } qp_flush_wqes;
	+
	+ struct {
	+ struct i40iw_sc_qp *qp;
	+ struct i40iw_gen_ae_info info;
	+ u64 scratch;
	+ } gen_ae;
	+
	+ struct {
	+ struct i40iw_sc_cqp *cqp;
	+ void *fpm_values_va;
	+ u64 fpm_values_pa;
	+ u8 hmc_fn_id;
	+ u64 scratch;
	+ } query_fpm_values;
	+
	+ struct {
	+ struct i40iw_sc_cqp *cqp;
	+ void *fpm_values_va;
	+ u64 fpm_values_pa;
	+ u8 hmc_fn_id;
	+ u64 scratch;
	+ } commit_fpm_values;
	+
	+ struct {
	+ struct i40iw_sc_cqp *cqp;
	+ struct i40iw_apbvt_info info;
	+ u64 scratch;
	+ } manage_apbvt_entry;
	+
	+ struct {
	+ struct i40iw_sc_cqp *cqp;
	+ struct i40iw_qhash_table_info info;
	+ u64 scratch;
	+ } manage_qhash_table_entry;
	+
	+ struct {
	+ struct i40iw_sc_dev *dev;
	+ struct i40iw_update_sds_info info;
	+ u64 scratch;
	+ } update_pe_sds;
	+
	+ struct {
	+ struct i40iw_sc_cqp *cqp;
	+ struct i40iw_sc_qp *qp;
	+ u64 scratch;
	+ } suspend_resume;
	+ } u;
	+};
	+
	+struct cqp_commands_info {
	+ struct list_head cqp_cmd_entry;
	+ u8 cqp_cmd;
	+ u8 post_sq;
	+ struct cqp_info in;
	+};
	+
	+struct i40iw_cqp_timeout {
	+ u64 compl_cqp_cmds;
	+ u8 count;
	+};
	+
	+#endif
	Index: sys/dev/ixl/iwarp/i40iw_uk.c
	===================================================================
	--- /dev/null
	+++ sys/dev/ixl/iwarp/i40iw_uk.c
	@@ -0,0 +1,1309 @@
	+/*******************************************************************************
	+*
	+* Copyright (c) 2015-2017 Intel Corporation. All rights reserved.
	+*
	+* This software is available to you under a choice of one of two
	+* licenses. You may choose to be licensed under the terms of the GNU
	+* General Public License (GPL) Version 2, available from the file
	+* COPYING in the main directory of this source tree, or the
	+* OpenFabrics.org BSD license below:
	+*
	+* Redistribution and use in source and binary forms, with or
	+* without modification, are permitted provided that the following
	+* conditions are met:
	+*
	+* - Redistributions of source code must retain the above
	+* copyright notice, this list of conditions and the following
	+* disclaimer.
	+*
	+* - Redistributions in binary form must reproduce the above
	+* copyright notice, this list of conditions and the following
	+* disclaimer in the documentation and/or other materials
	+* provided with the distribution.
	+*
	+* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
	+* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
	+* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
	+* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
	+* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
	+* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
	+* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
	+* SOFTWARE.
	+*
	+*******************************************************************************/
	+/$FreeBSD$/
	+
	+#include "i40iw_osdep.h"
	+#include "i40iw_status.h"
	+#include "i40iw_d.h"
	+#include "i40iw_user.h"
	+#include "i40iw_register.h"
	+
	+static u32 nop_signature = 0x55550000;
	+
	+/**
	+ * i40iw_nop_1 - insert a nop wqe and move head. no post work
	+ * @qp: hw qp ptr
	+ */
	+static enum i40iw_status_code
	+i40iw_nop_1(struct i40iw_qp_uk *qp)
	+{
	+ u64 header, *wqe;
	+ u64 *wqe_0 = NULL;
	+ u32 wqe_idx, peek_head;
	+ bool signaled = FALSE;
	+
	+ if (!qp->sq_ring.head)
	+ return I40IW_ERR_PARAM;
	+
	+ wqe_idx = I40IW_RING_GETCURRENT_HEAD(qp->sq_ring);
	+ wqe = qp->sq_base[wqe_idx].elem;
	+
	+ qp->sq_wrtrk_array[wqe_idx].wqe_size = I40IW_QP_WQE_MIN_SIZE;
	+
	+ peek_head = (qp->sq_ring.head + 1) % qp->sq_ring.size;
	+ wqe_0 = qp->sq_base[peek_head].elem;
	+ if (peek_head)
	+ wqe_0[3] = LS_64(!qp->swqe_polarity, I40IWQPSQ_VALID);
	+ else
	+ wqe_0[3] = LS_64(qp->swqe_polarity, I40IWQPSQ_VALID);
	+ set_64bit_val(wqe, I40IW_BYTE_0, 0);
	+ set_64bit_val(wqe, I40IW_BYTE_8, 0);
	+ set_64bit_val(wqe, I40IW_BYTE_16, 0);
	+
	+ header = LS_64(I40IWQP_OP_NOP, I40IWQPSQ_OPCODE) \|
	+ LS_64(signaled, I40IWQPSQ_SIGCOMPL) \|
	+ LS_64(qp->swqe_polarity, I40IWQPSQ_VALID) \| nop_signature++;
	+
	+ i40iw_wmb(); /* Memory barrier to ensure data is written
	+ * before valid bit is set */
	+
	+ set_64bit_val(wqe, I40IW_BYTE_24, header);
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_qp_post_wr - post wr to hrdware
	+ * @qp: hw qp ptr
	+ */
	+void
	+i40iw_qp_post_wr(struct i40iw_qp_uk *qp)
	+{
	+ u64 temp;
	+ u32 hw_sq_tail;
	+ u32 sw_sq_head;
	+
	+ i40iw_mb(); /* valid bit is written and loads completed
	+ * before reading shadow */
	+
	+ /* read the doorbell shadow area */
	+ get_64bit_val(qp->shadow_area, I40IW_BYTE_0, &temp);
	+
	+ hw_sq_tail = (u32)RS_64(temp, I40IW_QP_DBSA_HW_SQ_TAIL);
	+ sw_sq_head = I40IW_RING_GETCURRENT_HEAD(qp->sq_ring);
	+ if (sw_sq_head != hw_sq_tail) {
	+ if (sw_sq_head > qp->initial_ring.head) {
	+ if ((hw_sq_tail >= qp->initial_ring.head) &&
	+ (hw_sq_tail < sw_sq_head)) {
	+ db_wr32(qp->qp_id, qp->wqe_alloc_reg);
	+ }
	+ } else if (sw_sq_head != qp->initial_ring.head) {
	+ if ((hw_sq_tail >= qp->initial_ring.head) \|\|
	+ (hw_sq_tail < sw_sq_head)) {
	+ db_wr32(qp->qp_id, qp->wqe_alloc_reg);
	+ }
	+ }
	+ }
	+ qp->initial_ring.head = qp->sq_ring.head;
	+}
	+
	+/**
	+ * i40iw_qp_ring_push_db - ring qp doorbell
	+ * @qp: hw qp ptr
	+ * @wqe_idx: wqe index
	+ */
	+static void
	+i40iw_qp_ring_push_db(struct i40iw_qp_uk *qp, u32 wqe_idx)
	+{
	+ set_32bit_val(qp->push_db, 0, LS_32((wqe_idx >> 2), I40E_PFPE_WQEALLOC_WQE_DESC_INDEX) \| qp->qp_id);
	+ qp->initial_ring.head = I40IW_RING_GETCURRENT_HEAD(qp->sq_ring);
	+}
	+
	+/**
	+ * i40iw_qp_get_next_send_wqe - return next wqe ptr
	+ * @qp: hw qp ptr
	+ * @wqe_idx: return wqe index
	+ * @wqe_size: size of sq wqe
	+ */
	+u64 *
	+i40iw_qp_get_next_send_wqe(struct i40iw_qp_uk *qp,
	+ u32 *wqe_idx,
	+ u8 wqe_size,
	+ u32 total_size,
	+ u64 wr_id)
	+{
	+ u64 *wqe = NULL;
	+ u64 wqe_ptr;
	+ u32 peek_head = 0;
	+ u16 offset;
	+ enum i40iw_status_code ret_code = 0;
	+ u8 nop_wqe_cnt = 0, i;
	+ u64 *wqe_0 = NULL;
	+
	+ *wqe_idx = I40IW_RING_GETCURRENT_HEAD(qp->sq_ring);
	+
	+ if (!*wqe_idx)
	+ qp->swqe_polarity = !qp->swqe_polarity;
	+ wqe_ptr = (uintptr_t)qp->sq_base[*wqe_idx].elem;
	+ offset = (u16)(wqe_ptr) & 0x7F;
	+ if ((offset + wqe_size) > I40IW_QP_WQE_MAX_SIZE) {
	+ nop_wqe_cnt = (u8)(I40IW_QP_WQE_MAX_SIZE - offset) / I40IW_QP_WQE_MIN_SIZE;
	+ for (i = 0; i < nop_wqe_cnt; i++) {
	+ i40iw_nop_1(qp);
	+ I40IW_RING_MOVE_HEAD(qp->sq_ring, ret_code);
	+ if (ret_code)
	+ return NULL;
	+ }
	+
	+ *wqe_idx = I40IW_RING_GETCURRENT_HEAD(qp->sq_ring);
	+ if (!*wqe_idx)
	+ qp->swqe_polarity = !qp->swqe_polarity;
	+ }
	+ if (((*wqe_idx & 3) == 1) && (wqe_size == I40IW_WQE_SIZE_64)) {
	+ i40iw_nop_1(qp);
	+ I40IW_RING_MOVE_HEAD(qp->sq_ring, ret_code);
	+ if (ret_code)
	+ return NULL;
	+ *wqe_idx = I40IW_RING_GETCURRENT_HEAD(qp->sq_ring);
	+ if (!*wqe_idx)
	+ qp->swqe_polarity = !qp->swqe_polarity;
	+ }
	+ I40IW_RING_MOVE_HEAD_BY_COUNT(qp->sq_ring,
	+ (wqe_size / I40IW_QP_WQE_MIN_SIZE), ret_code);
	+ if (ret_code)
	+ return NULL;
	+
	+ wqe = qp->sq_base[*wqe_idx].elem;
	+
	+ peek_head = I40IW_RING_GETCURRENT_HEAD(qp->sq_ring);
	+ wqe_0 = qp->sq_base[peek_head].elem;
	+
	+ if (((peek_head & 3) == 1) \|\| ((peek_head & 3) == 3)) {
	+ if (RS_64(wqe_0[3], I40IWQPSQ_VALID) != !qp->swqe_polarity)
	+ wqe_0[3] = LS_64(!qp->swqe_polarity, I40IWQPSQ_VALID);
	+ }
	+#if defined(NDK_DEBUG)
	+ insert_send_idx(qp, *wqe_idx);
	+#endif
	+
	+ qp->sq_wrtrk_array[*wqe_idx].wrid = wr_id;
	+ qp->sq_wrtrk_array[*wqe_idx].wr_len = total_size;
	+ qp->sq_wrtrk_array[*wqe_idx].wqe_size = wqe_size;
	+ return wqe;
	+}
	+
	+/**
	+ * i40iw_set_fragment - set fragment in wqe
	+ * @wqe: wqe for setting fragment
	+ * @offset: offset value
	+ * @sge: sge length and stag
	+ */
	+static void
	+i40iw_set_fragment(u64 wqe, u32 offset, struct i40iw_sge sge)
	+{
	+ if (sge) {
	+ set_64bit_val(wqe, offset, LS_64(sge->tag_off, I40IWQPSQ_FRAG_TO));
	+ set_64bit_val(wqe, (offset + I40IW_BYTE_8),
	+ (LS_64(sge->len, I40IWQPSQ_FRAG_LEN) \|
	+ LS_64(sge->stag, I40IWQPSQ_FRAG_STAG)));
	+ }
	+}
	+
	+/**
	+ * i40iw_qp_get_next_recv_wqe - get next qp's rcv wqe
	+ * @qp: hw qp ptr
	+ * @wqe_idx: return wqe index
	+ */
	+u64 *
	+i40iw_qp_get_next_recv_wqe(struct i40iw_qp_uk qp, u32 wqe_idx)
	+{
	+ u64 *wqe = NULL;
	+ enum i40iw_status_code ret_code;
	+
	+ if (I40IW_RING_FULL_ERR(qp->rq_ring))
	+ return NULL;
	+
	+ I40IW_ATOMIC_RING_MOVE_HEAD(qp->rq_ring, *wqe_idx, ret_code);
	+ if (ret_code)
	+ return NULL;
	+ if (!*wqe_idx)
	+ qp->rwqe_polarity = !qp->rwqe_polarity;
	+ /* rq_wqe_size_multiplier is no of qwords in one rq wqe */
	+ wqe = qp->rq_base[wqe_idx (qp->rq_wqe_size_multiplier >> 2)].elem;
	+#if defined(NDK_DEBUG)
	+ insert_recv_idx(qp, *wqe_idx);
	+#endif
	+
	+ return wqe;
	+}
	+
	+/**
	+ * i40iw_rdma_write - rdma write operation
	+ * @qp: hw qp ptr
	+ * @info: post sq information
	+ * @post_sq: flag to post sq
	+ */
	+static enum i40iw_status_code
	+i40iw_rdma_write(struct i40iw_qp_uk *qp,
	+ struct i40iw_post_sq_info *info,
	+ bool post_sq)
	+{
	+ u64 header;
	+ u64 *wqe;
	+ struct i40iw_rdma_write *op_info;
	+ u32 i, wqe_idx;
	+ u32 total_size = 0, byte_off;
	+ enum i40iw_status_code ret_code;
	+ bool read_fence = FALSE;
	+ u8 wqe_size;
	+
	+ op_info = &info->op.rdma_write;
	+ if (op_info->num_lo_sges > qp->max_sq_frag_cnt)
	+ return I40IW_ERR_INVALID_FRAG_COUNT;
	+
	+ for (i = 0; i < op_info->num_lo_sges; i++)
	+ total_size += op_info->lo_sg_list[i].len;
	+
	+ if (total_size > I40IW_MAX_OUTBOUND_MESSAGE_SIZE)
	+ return I40IW_ERR_QP_INVALID_MSG_SIZE;
	+
	+ read_fence \|= info->read_fence;
	+#if defined(READ_FENCE_ERRATA)
	+ read_fence = 1;
	+#endif
	+
	+ ret_code = i40iw_fragcnt_to_wqesize_sq(op_info->num_lo_sges, &wqe_size);
	+ if (ret_code)
	+ return ret_code;
	+
	+ wqe = i40iw_qp_get_next_send_wqe(qp, &wqe_idx, wqe_size,
	+ total_size, info->wr_id);
	+ if (!wqe)
	+ return I40IW_ERR_QP_TOOMANY_WRS_POSTED;
	+ set_64bit_val(wqe, I40IW_BYTE_16,
	+ LS_64(op_info->rem_addr.tag_off, I40IWQPSQ_FRAG_TO));
	+ if (!op_info->rem_addr.stag)
	+ return I40IW_ERR_BAD_STAG;
	+
	+ header = LS_64(op_info->rem_addr.stag, I40IWQPSQ_REMSTAG) \|
	+ LS_64(I40IWQP_OP_RDMA_WRITE, I40IWQPSQ_OPCODE) \|
	+ LS_64((op_info->num_lo_sges > 1 ? (op_info->num_lo_sges - 1) : 0), I40IWQPSQ_ADDFRAGCNT) \|
	+ LS_64(read_fence, I40IWQPSQ_READFENCE) \|
	+ LS_64(info->local_fence, I40IWQPSQ_LOCALFENCE) \|
	+ LS_64(info->signaled, I40IWQPSQ_SIGCOMPL) \|
	+ LS_64(qp->swqe_polarity, I40IWQPSQ_VALID);
	+
	+ i40iw_set_fragment(wqe, I40IW_BYTE_0, op_info->lo_sg_list);
	+
	+ for (i = 1, byte_off = I40IW_BYTE_32; i < op_info->num_lo_sges; i++) {
	+ i40iw_set_fragment(wqe, byte_off, &op_info->lo_sg_list[i]);
	+ byte_off += 16;
	+ }
	+
	+ i40iw_wmb(); /* make sure WQE is populated before valid
	+ * bit is set */
	+
	+ set_64bit_val(wqe, I40IW_BYTE_24, header);
	+
	+ if (post_sq)
	+ i40iw_qp_post_wr(qp);
	+
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_rdma_read - rdma read command
	+ * @qp: hw qp ptr
	+ * @info: post sq information
	+ * @inv_stag: flag for inv_stag
	+ * @post_sq: flag to post sq
	+ */
	+static enum i40iw_status_code
	+i40iw_rdma_read(struct i40iw_qp_uk *qp,
	+ struct i40iw_post_sq_info *info,
	+ bool inv_stag,
	+ bool post_sq)
	+{
	+ u64 *wqe;
	+ struct i40iw_rdma_read *op_info;
	+ u64 header;
	+ u32 wqe_idx;
	+ enum i40iw_status_code ret_code;
	+ u8 wqe_size;
	+ bool local_fence = FALSE;
	+
	+ op_info = &info->op.rdma_read;
	+ ret_code = i40iw_fragcnt_to_wqesize_sq(1, &wqe_size);
	+ if (ret_code)
	+ return ret_code;
	+ wqe = i40iw_qp_get_next_send_wqe(qp, &wqe_idx, wqe_size,
	+ op_info->lo_addr.len, info->wr_id);
	+ if (!wqe)
	+ return I40IW_ERR_QP_TOOMANY_WRS_POSTED;
	+ local_fence \|= info->local_fence;
	+
	+#if defined(READ_FENCE_ERRATA)
	+ local_fence = 1;
	+ info->read_fence = 1;
	+#endif
	+ set_64bit_val(wqe, I40IW_BYTE_16, LS_64(op_info->rem_addr.tag_off, I40IWQPSQ_FRAG_TO));
	+ header = LS_64(op_info->rem_addr.stag, I40IWQPSQ_REMSTAG) \|
	+ LS_64((inv_stag ? I40IWQP_OP_RDMA_READ_LOC_INV : I40IWQP_OP_RDMA_READ), I40IWQPSQ_OPCODE) \|
	+ LS_64(info->read_fence, I40IWQPSQ_READFENCE) \|
	+ LS_64(local_fence, I40IWQPSQ_LOCALFENCE) \|
	+ LS_64(info->signaled, I40IWQPSQ_SIGCOMPL) \|
	+ LS_64(qp->swqe_polarity, I40IWQPSQ_VALID);
	+
	+ i40iw_set_fragment(wqe, I40IW_BYTE_0, &op_info->lo_addr);
	+
	+ i40iw_wmb(); /* make sure WQE is populated before valid
	+ * bit is set */
	+
	+ set_64bit_val(wqe, I40IW_BYTE_24, header);
	+ if (post_sq)
	+ i40iw_qp_post_wr(qp);
	+
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_send - rdma send command
	+ * @qp: hw qp ptr
	+ * @info: post sq information
	+ * @post_sq: flag to post sq
	+ */
	+static enum i40iw_status_code
	+i40iw_send(struct i40iw_qp_uk *qp,
	+ struct i40iw_post_sq_info *info,
	+ bool post_sq)
	+{
	+ u64 *wqe;
	+ struct i40iw_post_send *op_info;
	+ u64 header;
	+ u32 i, wqe_idx, total_size = 0, byte_off;
	+ enum i40iw_status_code ret_code;
	+ bool read_fence = FALSE;
	+ u8 wqe_size;
	+
	+ op_info = &info->op.send;
	+ if (qp->max_sq_frag_cnt < op_info->num_sges)
	+ return I40IW_ERR_INVALID_FRAG_COUNT;
	+
	+ for (i = 0; i < op_info->num_sges; i++)
	+ total_size += op_info->sg_list[i].len;
	+ ret_code = i40iw_fragcnt_to_wqesize_sq(op_info->num_sges, &wqe_size);
	+ if (ret_code)
	+ return ret_code;
	+
	+ wqe = i40iw_qp_get_next_send_wqe(qp, &wqe_idx, wqe_size,
	+ total_size, info->wr_id);
	+ if (!wqe)
	+ return I40IW_ERR_QP_TOOMANY_WRS_POSTED;
	+
	+ read_fence \|= info->read_fence;
	+#if defined(READ_FENCE_ERRATA)
	+ read_fence = 1;
	+#endif
	+
	+ set_64bit_val(wqe, I40IW_BYTE_16, 0);
	+ header = LS_64(info->stag_to_inv, I40IWQPSQ_REMSTAG) \|
	+ LS_64(info->op_type, I40IWQPSQ_OPCODE) \|
	+ LS_64((op_info->num_sges > 1 ? (op_info->num_sges - 1) : 0),
	+ I40IWQPSQ_ADDFRAGCNT) \|
	+ LS_64(read_fence, I40IWQPSQ_READFENCE) \|
	+ LS_64(info->local_fence, I40IWQPSQ_LOCALFENCE) \|
	+ LS_64(info->signaled, I40IWQPSQ_SIGCOMPL) \|
	+ LS_64(qp->swqe_polarity, I40IWQPSQ_VALID);
	+
	+ i40iw_set_fragment(wqe, I40IW_BYTE_0, op_info->sg_list);
	+
	+ for (i = 1, byte_off = I40IW_BYTE_32; i < op_info->num_sges; i++) {
	+ i40iw_set_fragment(wqe, byte_off, &op_info->sg_list[i]);
	+ byte_off += 16;
	+ }
	+
	+ i40iw_wmb(); /* make sure WQE is populated before valid
	+ * bit is set */
	+
	+ set_64bit_val(wqe, I40IW_BYTE_24, header);
	+ if (post_sq)
	+ i40iw_qp_post_wr(qp);
	+
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_inline_rdma_write - inline rdma write operation
	+ * @qp: hw qp ptr
	+ * @info: post sq information
	+ * @post_sq: flag to post sq
	+ */
	+static enum i40iw_status_code
	+i40iw_inline_rdma_write(struct i40iw_qp_uk *qp,
	+ struct i40iw_post_sq_info *info,
	+ bool post_sq)
	+{
	+ u64 *wqe;
	+ u8 dest, src;
	+ struct i40iw_inline_rdma_write *op_info;
	+ u64 *push;
	+ u64 header = 0;
	+ u32 wqe_idx;
	+ enum i40iw_status_code ret_code;
	+ bool read_fence = FALSE;
	+ u8 wqe_size;
	+
	+ op_info = &info->op.inline_rdma_write;
	+ if (op_info->len > I40IW_MAX_INLINE_DATA_SIZE)
	+ return I40IW_ERR_INVALID_INLINE_DATA_SIZE;
	+
	+ ret_code = i40iw_inline_data_size_to_wqesize(op_info->len, &wqe_size);
	+ if (ret_code)
	+ return ret_code;
	+
	+ wqe = i40iw_qp_get_next_send_wqe(qp, &wqe_idx, wqe_size,
	+ op_info->len, info->wr_id);
	+ if (!wqe)
	+ return I40IW_ERR_QP_TOOMANY_WRS_POSTED;
	+
	+ read_fence \|= info->read_fence;
	+ set_64bit_val(wqe, I40IW_BYTE_16,
	+ LS_64(op_info->rem_addr.tag_off, I40IWQPSQ_FRAG_TO));
	+
	+ header = LS_64(op_info->rem_addr.stag, I40IWQPSQ_REMSTAG) \|
	+ LS_64(I40IWQP_OP_RDMA_WRITE, I40IWQPSQ_OPCODE) \|
	+ LS_64(op_info->len, I40IWQPSQ_INLINEDATALEN) \|
	+ LS_64(1, I40IWQPSQ_INLINEDATAFLAG) \|
	+ LS_64((qp->push_db ? 1 : 0), I40IWQPSQ_PUSHWQE) \|
	+ LS_64(read_fence, I40IWQPSQ_READFENCE) \|
	+ LS_64(info->local_fence, I40IWQPSQ_LOCALFENCE) \|
	+ LS_64(info->signaled, I40IWQPSQ_SIGCOMPL) \|
	+ LS_64(qp->swqe_polarity, I40IWQPSQ_VALID);
	+
	+ dest = (u8 *)wqe;
	+ src = (u8 *)(op_info->data);
	+
	+ if (op_info->len <= I40IW_BYTE_16) {
	+ i40iw_memcpy(dest, src, op_info->len);
	+ } else {
	+ i40iw_memcpy(dest, src, I40IW_BYTE_16);
	+ src += I40IW_BYTE_16;
	+ dest = (u8 *)wqe + I40IW_BYTE_32;
	+ i40iw_memcpy(dest, src, op_info->len - I40IW_BYTE_16);
	+ }
	+
	+ i40iw_wmb(); /* make sure WQE is populated before valid
	+ * bit is set */
	+
	+ set_64bit_val(wqe, I40IW_BYTE_24, header);
	+
	+ if (qp->push_db) {
	+ push = (u64 )((uintptr_t)qp->push_wqe + (wqe_idx & 0x3) 0x20);
	+ i40iw_memcpy(push, wqe, (op_info->len > 16) ? op_info->len + 16 : 32);
	+ i40iw_qp_ring_push_db(qp, wqe_idx);
	+ } else {
	+ if (post_sq)
	+ i40iw_qp_post_wr(qp);
	+ }
	+
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_inline_send - inline send operation
	+ * @qp: hw qp ptr
	+ * @info: post sq information
	+ * @post_sq: flag to post sq
	+ */
	+static enum i40iw_status_code
	+i40iw_inline_send(struct i40iw_qp_uk *qp,
	+ struct i40iw_post_sq_info *info,
	+ bool post_sq)
	+{
	+ u64 *wqe;
	+ u8 dest, src;
	+ struct i40iw_post_inline_send *op_info;
	+ u64 header;
	+ u32 wqe_idx;
	+ enum i40iw_status_code ret_code;
	+ bool read_fence = FALSE;
	+ u8 wqe_size;
	+ u64 *push;
	+
	+ op_info = &info->op.inline_send;
	+ if (op_info->len > I40IW_MAX_INLINE_DATA_SIZE)
	+ return I40IW_ERR_INVALID_INLINE_DATA_SIZE;
	+
	+ ret_code = i40iw_inline_data_size_to_wqesize(op_info->len, &wqe_size);
	+ if (ret_code)
	+ return ret_code;
	+
	+ wqe = i40iw_qp_get_next_send_wqe(qp, &wqe_idx, wqe_size,
	+ op_info->len, info->wr_id);
	+ if (!wqe)
	+ return I40IW_ERR_QP_TOOMANY_WRS_POSTED;
	+
	+ read_fence \|= info->read_fence;
	+ header = LS_64(info->stag_to_inv, I40IWQPSQ_REMSTAG) \|
	+ LS_64(info->op_type, I40IWQPSQ_OPCODE) \|
	+ LS_64(op_info->len, I40IWQPSQ_INLINEDATALEN) \|
	+ LS_64(1, I40IWQPSQ_INLINEDATAFLAG) \|
	+ LS_64((qp->push_db ? 1 : 0), I40IWQPSQ_PUSHWQE) \|
	+ LS_64(read_fence, I40IWQPSQ_READFENCE) \|
	+ LS_64(info->local_fence, I40IWQPSQ_LOCALFENCE) \|
	+ LS_64(info->signaled, I40IWQPSQ_SIGCOMPL) \|
	+ LS_64(qp->swqe_polarity, I40IWQPSQ_VALID);
	+
	+ dest = (u8 *)wqe;
	+ src = (u8 *)(op_info->data);
	+
	+ if (op_info->len <= I40IW_BYTE_16) {
	+ i40iw_memcpy(dest, src, op_info->len);
	+ } else {
	+ i40iw_memcpy(dest, src, I40IW_BYTE_16);
	+ src += I40IW_BYTE_16;
	+ dest = (u8 *)wqe + I40IW_BYTE_32;
	+ i40iw_memcpy(dest, src, op_info->len - I40IW_BYTE_16);
	+ }
	+
	+ i40iw_wmb(); /* make sure WQE is populated before valid
	+ * bit is set */
	+
	+ set_64bit_val(wqe, I40IW_BYTE_24, header);
	+
	+ if (qp->push_db) {
	+ push = (u64 )((uintptr_t)qp->push_wqe + (wqe_idx & 0x3) 0x20);
	+ i40iw_memcpy(push, wqe, (op_info->len > 16) ? op_info->len + 16 : 32);
	+ i40iw_qp_ring_push_db(qp, wqe_idx);
	+ } else {
	+ if (post_sq)
	+ i40iw_qp_post_wr(qp);
	+ }
	+
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_stag_local_invalidate - stag invalidate operation
	+ * @qp: hw qp ptr
	+ * @info: post sq information
	+ * @post_sq: flag to post sq
	+ */
	+static enum i40iw_status_code
	+i40iw_stag_local_invalidate(struct i40iw_qp_uk *qp,
	+ struct i40iw_post_sq_info *info,
	+ bool post_sq)
	+{
	+ u64 *wqe;
	+ struct i40iw_inv_local_stag *op_info;
	+ u64 header;
	+ u32 wqe_idx;
	+ bool local_fence = FALSE;
	+
	+ op_info = &info->op.inv_local_stag;
	+ local_fence = info->local_fence;
	+#if defined(READ_FENCE_ERRATA)
	+ info->read_fence = 1;
	+#endif
	+
	+ wqe = i40iw_qp_get_next_send_wqe(qp, &wqe_idx, I40IW_QP_WQE_MIN_SIZE,
	+ 0, info->wr_id);
	+ if (!wqe)
	+ return I40IW_ERR_QP_TOOMANY_WRS_POSTED;
	+ set_64bit_val(wqe, I40IW_BYTE_0, 0);
	+ set_64bit_val(wqe, I40IW_BYTE_8,
	+ LS_64(op_info->target_stag, I40IWQPSQ_LOCSTAG));
	+ set_64bit_val(wqe, I40IW_BYTE_16, 0);
	+ header = LS_64(I40IW_OP_TYPE_INV_STAG, I40IWQPSQ_OPCODE) \|
	+ LS_64(info->read_fence, I40IWQPSQ_READFENCE) \|
	+ LS_64(local_fence, I40IWQPSQ_LOCALFENCE) \|
	+ LS_64(info->signaled, I40IWQPSQ_SIGCOMPL) \|
	+ LS_64(qp->swqe_polarity, I40IWQPSQ_VALID);
	+
	+ i40iw_wmb(); /* make sure WQE is populated before valid
	+ * bit is set */
	+
	+ set_64bit_val(wqe, I40IW_BYTE_24, header);
	+
	+ if (post_sq)
	+ i40iw_qp_post_wr(qp);
	+
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_mw_bind - Memory Window bind operation
	+ * @qp: hw qp ptr
	+ * @info: post sq information
	+ * @post_sq: flag to post sq
	+ */
	+static enum i40iw_status_code
	+i40iw_mw_bind(struct i40iw_qp_uk *qp,
	+ struct i40iw_post_sq_info *info,
	+ bool post_sq)
	+{
	+ u64 *wqe;
	+ struct i40iw_bind_window *op_info;
	+ u64 header;
	+ u32 wqe_idx;
	+ bool local_fence = FALSE;
	+
	+ op_info = &info->op.bind_window;
	+
	+ local_fence \|= info->local_fence;
	+ wqe = i40iw_qp_get_next_send_wqe(qp, &wqe_idx, I40IW_QP_WQE_MIN_SIZE,
	+ 0, info->wr_id);
	+ if (!wqe)
	+ return I40IW_ERR_QP_TOOMANY_WRS_POSTED;
	+ set_64bit_val(wqe, I40IW_BYTE_0, (uintptr_t)op_info->va);
	+ set_64bit_val(wqe, I40IW_BYTE_8,
	+ LS_64(op_info->mr_stag, I40IWQPSQ_PARENTMRSTAG) \|
	+ LS_64(op_info->mw_stag, I40IWQPSQ_MWSTAG));
	+ set_64bit_val(wqe, I40IW_BYTE_16, op_info->bind_length);
	+ header = LS_64(I40IW_OP_TYPE_BIND_MW, I40IWQPSQ_OPCODE) \|
	+ LS_64(((op_info->enable_reads << 2) \|
	+ (op_info->enable_writes << 3)),
	+ I40IWQPSQ_STAGRIGHTS) \|
	+ LS_64((op_info->addressing_type == I40IW_ADDR_TYPE_VA_BASED ? 1 : 0),
	+ I40IWQPSQ_VABASEDTO) \|
	+ LS_64(info->read_fence, I40IWQPSQ_READFENCE) \|
	+ LS_64(local_fence, I40IWQPSQ_LOCALFENCE) \|
	+ LS_64(info->signaled, I40IWQPSQ_SIGCOMPL) \|
	+ LS_64(qp->swqe_polarity, I40IWQPSQ_VALID);
	+
	+ i40iw_wmb(); /* make sure WQE is populated before valid
	+ * bit is set */
	+
	+ set_64bit_val(wqe, I40IW_BYTE_24, header);
	+
	+ if (post_sq)
	+ i40iw_qp_post_wr(qp);
	+
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_post_receive - post receive wqe
	+ * @qp: hw qp ptr
	+ * @info: post rq information
	+ */
	+static enum i40iw_status_code
	+i40iw_post_receive(struct i40iw_qp_uk *qp,
	+ struct i40iw_post_rq_info *info)
	+{
	+ u64 *wqe;
	+ u64 header;
	+ u32 total_size = 0, wqe_idx, i, byte_off;
	+
	+ if (qp->max_rq_frag_cnt < info->num_sges)
	+ return I40IW_ERR_INVALID_FRAG_COUNT;
	+ for (i = 0; i < info->num_sges; i++)
	+ total_size += info->sg_list[i].len;
	+ wqe = i40iw_qp_get_next_recv_wqe(qp, &wqe_idx);
	+ if (!wqe)
	+ return I40IW_ERR_QP_TOOMANY_WRS_POSTED;
	+
	+ qp->rq_wrid_array[wqe_idx] = info->wr_id;
	+ set_64bit_val(wqe, I40IW_BYTE_16, 0);
	+
	+ header = LS_64((info->num_sges > 1 ? (info->num_sges - 1) : 0),
	+ I40IWQPSQ_ADDFRAGCNT) \|
	+ LS_64(qp->rwqe_polarity, I40IWQPSQ_VALID);
	+
	+ i40iw_set_fragment(wqe, I40IW_BYTE_0, info->sg_list);
	+
	+ for (i = 1, byte_off = I40IW_BYTE_32; i < info->num_sges; i++) {
	+ i40iw_set_fragment(wqe, byte_off, &info->sg_list[i]);
	+ byte_off += 16;
	+ }
	+
	+ i40iw_wmb(); /* make sure WQE is populated before valid
	+ * bit is set */
	+
	+ set_64bit_val(wqe, I40IW_BYTE_24, header);
	+
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_cq_request_notification - cq notification request (door bell)
	+ * @cq: hw cq
	+ * @cq_notify: notification type
	+ */
	+static void
	+i40iw_cq_request_notification(struct i40iw_cq_uk *cq,
	+ enum i40iw_completion_notify cq_notify)
	+{
	+ u64 temp_val;
	+ u16 sw_cq_sel;
	+ u8 arm_next_se = 0;
	+ u8 arm_next = 0;
	+ u8 arm_seq_num;
	+
	+ get_64bit_val(cq->shadow_area, I40IW_BYTE_32, &temp_val);
	+ arm_seq_num = (u8)RS_64(temp_val, I40IW_CQ_DBSA_ARM_SEQ_NUM);
	+ arm_seq_num++;
	+
	+ sw_cq_sel = (u16)RS_64(temp_val, I40IW_CQ_DBSA_SW_CQ_SELECT);
	+ arm_next_se = (u8)RS_64(temp_val, I40IW_CQ_DBSA_ARM_NEXT_SE);
	+ arm_next_se \|= 1;
	+ if (cq_notify == IW_CQ_COMPL_EVENT)
	+ arm_next = 1;
	+ temp_val = LS_64(arm_seq_num, I40IW_CQ_DBSA_ARM_SEQ_NUM) \|
	+ LS_64(sw_cq_sel, I40IW_CQ_DBSA_SW_CQ_SELECT) \|
	+ LS_64(arm_next_se, I40IW_CQ_DBSA_ARM_NEXT_SE) \|
	+ LS_64(arm_next, I40IW_CQ_DBSA_ARM_NEXT);
	+
	+ set_64bit_val(cq->shadow_area, I40IW_BYTE_32, temp_val);
	+
	+ i40iw_wmb(); /* make sure WQE is populated before valid
	+ * bit is set */
	+
	+ db_wr32(cq->cq_id, cq->cqe_alloc_reg);
	+}
	+
	+/**
	+ * i40iw_cq_post_entries - update tail in shadow memory
	+ * @cq: hw cq
	+ * @count: # of entries processed
	+ */
	+static enum i40iw_status_code
	+i40iw_cq_post_entries(struct i40iw_cq_uk *cq,
	+ u8 count)
	+{
	+ I40IW_RING_MOVE_TAIL_BY_COUNT(cq->cq_ring, count);
	+ set_64bit_val(cq->shadow_area, I40IW_BYTE_0,
	+ I40IW_RING_GETCURRENT_HEAD(cq->cq_ring));
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_find_next_expected_sq_wqe - find the next expected sq wqe
	+ * @qp
	+ */
	+static u32
	+i40iw_find_next_expected_sq_wqe(struct i40iw_qp_uk *qp)
	+{
	+ u8 op_type, addl_wqes;
	+ u32 tail = qp->sq_ring.tail;
	+ u32 attempt = 0;
	+ u64 *sw_wqe;
	+ u64 wqe_qword;
	+ bool sig_compl;
	+
	+ while (qp->sq_ring.tail != qp->sq_ring.head) {
	+ sw_wqe = qp->sq_base[tail].elem;
	+ get_64bit_val(sw_wqe, I40IW_BYTE_24, &wqe_qword);
	+ op_type = (u8)RS_64(wqe_qword, I40IWQPSQ_OPCODE);
	+ sig_compl = (bool)RS_64(wqe_qword, I40IWQPSQ_SIGCOMPL);
	+ if (op_type != I40IWQP_OP_NOP && sig_compl)
	+ return tail;
	+ if (!qp->sq_wrtrk_array[tail].wqe_size \|\| attempt++ >= qp->sq_ring.size)
	+ break;
	+ addl_wqes = (u8)(qp->sq_wrtrk_array[tail].wqe_size / I40IW_QP_WQE_MIN_SIZE);
	+ tail = (tail + addl_wqes) % qp->sq_ring.size;
	+ }
	+ return 0xffffffff;
	+}
	+
	+/**
	+ * i40iw_cq_poll_completion - get cq completion info
	+ * @cq: hw cq
	+ * @info: cq poll information returned
	+ * @post_cq: update cq tail
	+ */
	+static enum i40iw_status_code
	+i40iw_cq_poll_completion(struct i40iw_cq_uk *cq,
	+ struct i40iw_cq_poll_info *info)
	+{
	+ u64 comp_ctx, qword0, qword2, qword3, wqe_qword;
	+ u64 cqe, sw_wqe;
	+ struct i40iw_qp_uk *qp;
	+ struct i40iw_ring *pring = NULL;
	+ u32 wqe_idx, q_type, array_idx = 0;
	+ enum i40iw_status_code ret_code = 0;
	+ bool move_cq_head = TRUE;
	+ u8 polarity;
	+ u8 addl_wqes = 0;
	+
	+ if (cq->avoid_mem_cflct)
	+ cqe = (u64 *)I40IW_GET_CURRENT_EXTENDED_CQ_ELEMENT(cq);
	+ else
	+ cqe = (u64 *)I40IW_GET_CURRENT_CQ_ELEMENT(cq);
	+
	+ get_64bit_val(cqe, I40IW_BYTE_24, &qword3);
	+ polarity = (u8)RS_64(qword3, I40IW_CQ_VALID);
	+
	+ if (polarity != cq->polarity)
	+ return I40IW_ERR_QUEUE_EMPTY;
	+
	+ q_type = (u8)RS_64(qword3, I40IW_CQ_SQ);
	+ info->error = (bool)RS_64(qword3, I40IW_CQ_ERROR);
	+ info->push_dropped = (bool)RS_64(qword3, I40IWCQ_PSHDROP);
	+ if (info->error) {
	+ info->major_err = (bool)RS_64(qword3, I40IW_CQ_MAJERR);
	+ info->minor_err = (bool)RS_64(qword3, I40IW_CQ_MINERR);
	+ if (info->major_err != 1 \|\| info->minor_err != 1) {
	+ info->comp_status = I40IW_COMPL_STATUS_INVALID_WQE;
	+ return I40IW_ERR_CQ_COMPL_ERROR;
	+ }
	+ info->comp_status = I40IW_COMPL_STATUS_FLUSHED;
	+ } else {
	+ info->comp_status = I40IW_COMPL_STATUS_SUCCESS;
	+ }
	+
	+ get_64bit_val(cqe, I40IW_BYTE_0, &qword0);
	+ get_64bit_val(cqe, I40IW_BYTE_16, &qword2);
	+
	+ info->tcp_seq_num = (u32)RS_64(qword0, I40IWCQ_TCPSEQNUM);
	+
	+ info->qp_id = (u32)RS_64(qword2, I40IWCQ_QPID);
	+
	+ get_64bit_val(cqe, I40IW_BYTE_8, &comp_ctx);
	+
	+ info->solicited_event = (bool)RS_64(qword3, I40IWCQ_SOEVENT);
	+ info->is_srq = (bool)RS_64(qword3, I40IWCQ_SRQ);
	+
	+ qp = (struct i40iw_qp_uk *)(i40iw_uintptr)comp_ctx;
	+ if (!qp) {
	+ ret_code = I40IW_ERR_QUEUE_DESTROYED;
	+ goto exit;
	+ }
	+ wqe_idx = (u32)RS_64(qword3, I40IW_CQ_WQEIDX);
	+ info->qp_handle = (i40iw_qp_handle)(i40iw_uintptr)qp;
	+
	+ if (q_type == I40IW_CQE_QTYPE_RQ) {
	+ if (info->comp_status == I40IW_COMPL_STATUS_FLUSHED) {
	+ info->wr_id = qp->rq_wrid_array[qp->rq_ring.tail];
	+ array_idx = qp->rq_ring.tail;
	+ } else {
	+ array_idx = (wqe_idx * 4) / qp->rq_wqe_size_multiplier;
	+ if (array_idx != qp->rq_ring.tail) {
	+ info->comp_status = I40IW_COMPL_STATUS_INVALID_WQE;
	+ return I40IW_ERR_CQ_COMPL_ERROR;
	+ }
	+ info->wr_id = qp->rq_wrid_array[array_idx];
	+#if defined(NDK_DEBUG)
	+ if (info->comp_status != I40IW_COMPL_STATUS_FLUSHED)
	+ check_recv_idx(qp, wqe_idx);
	+
	+#endif
	+ }
	+
	+ info->op_type = I40IW_OP_TYPE_REC;
	+ if (qword3 & I40IWCQ_STAG_MASK) {
	+ info->stag_invalid_set = TRUE;
	+ info->inv_stag = (u32)RS_64(qword2, I40IWCQ_INVSTAG);
	+ } else {
	+ info->stag_invalid_set = FALSE;
	+ }
	+ info->bytes_xfered = (u32)RS_64(qword0, I40IWCQ_PAYLDLEN);
	+ I40IW_RING_SET_TAIL(qp->rq_ring, array_idx + 1);
	+ pring = &qp->rq_ring;
	+ } else {
	+
	+ if (qp->first_sq_wq) {
	+ qp->first_sq_wq = FALSE;
	+ if ((!wqe_idx) && (qp->sq_ring.head == qp->sq_ring.tail)) {
	+ I40IW_RING_MOVE_HEAD_NOCHECK(cq->cq_ring);
	+ I40IW_RING_MOVE_TAIL(cq->cq_ring);
	+ set_64bit_val(cq->shadow_area, I40IW_BYTE_0,
	+ I40IW_RING_GETCURRENT_HEAD(cq->cq_ring));
	+ memset(info, 0, sizeof(struct i40iw_cq_poll_info));
	+ return i40iw_cq_poll_completion(cq, info);
	+ }
	+ }
	+ if (info->comp_status != I40IW_COMPL_STATUS_FLUSHED) {
	+ if (wqe_idx != i40iw_find_next_expected_sq_wqe(qp)) {
	+ info->comp_status = I40IW_COMPL_STATUS_INVALID_WQE;
	+ return I40IW_ERR_CQ_COMPL_ERROR;
	+ }
	+ info->wr_id = qp->sq_wrtrk_array[wqe_idx].wrid;
	+ info->bytes_xfered = qp->sq_wrtrk_array[wqe_idx].wr_len;
	+
	+#if defined(NDK_DEBUG)
	+ check_send_idx(qp, wqe_idx);
	+#endif
	+ info->op_type = (u8)RS_64(qword3, I40IWCQ_OP);
	+ sw_wqe = qp->sq_base[wqe_idx].elem;
	+ get_64bit_val(sw_wqe, I40IW_BYTE_24, &wqe_qword);
	+ addl_wqes = qp->sq_wrtrk_array[wqe_idx].wqe_size / I40IW_QP_WQE_MIN_SIZE;
	+ I40IW_RING_SET_TAIL(qp->sq_ring, (wqe_idx + addl_wqes));
	+ } else {
	+ do {
	+ u8 op_type;
	+ u32 tail;
	+
	+ tail = qp->sq_ring.tail;
	+ sw_wqe = qp->sq_base[tail].elem;
	+ get_64bit_val(sw_wqe, I40IW_BYTE_24, &wqe_qword);
	+ op_type = (u8)RS_64(wqe_qword, I40IWQPSQ_OPCODE);
	+ info->op_type = op_type;
	+ addl_wqes = qp->sq_wrtrk_array[tail].wqe_size / I40IW_QP_WQE_MIN_SIZE;
	+ I40IW_RING_SET_TAIL(qp->sq_ring, (tail + addl_wqes));
	+ if (op_type != I40IWQP_OP_NOP) {
	+ info->wr_id = qp->sq_wrtrk_array[tail].wrid;
	+ info->bytes_xfered = qp->sq_wrtrk_array[tail].wr_len;
	+ break;
	+ }
	+ } while (1);
	+ }
	+ pring = &qp->sq_ring;
	+ }
	+
	+ ret_code = 0;
	+
	+exit:
	+ if (!ret_code &&
	+ (info->comp_status == I40IW_COMPL_STATUS_FLUSHED))
	+ if (pring && (I40IW_RING_MORE_WORK(*pring)))
	+ move_cq_head = FALSE;
	+ if (move_cq_head) {
	+ I40IW_RING_MOVE_HEAD_NOCHECK(cq->cq_ring);
	+
	+ if (I40IW_RING_GETCURRENT_HEAD(cq->cq_ring) == 0)
	+ cq->polarity ^= 1;
	+
	+ I40IW_RING_MOVE_TAIL(cq->cq_ring);
	+ set_64bit_val(cq->shadow_area, I40IW_BYTE_0,
	+ I40IW_RING_GETCURRENT_HEAD(cq->cq_ring));
	+ } else {
	+ if (info->is_srq)
	+ return ret_code;
	+ qword3 &= ~I40IW_CQ_WQEIDX_MASK;
	+ qword3 \|= LS_64(pring->tail, I40IW_CQ_WQEIDX);
	+ set_64bit_val(cqe, I40IW_BYTE_24, qword3);
	+ }
	+
	+ return ret_code;
	+}
	+
	+/**
	+ * i40iw_powerof2_roundup - return round up to power of 2
	+ * @value: value to round up
	+ */
	+u32
	+i40iw_powerof2_roundup(u32 value)
	+{
	+ int scount;
	+
	+ value--;
	+
	+ for (scount = 1; scount <= (sizeof(value) << 2); scount *= 2)
	+ value \|= value >> scount;
	+
	+ return ++value;
	+}
	+
	+/**
	+ * i40iw_get_wqe_shift - get shift count for maximum wqe size
	+ * @wqsize: wq size in WRs
	+ * @sge: Maximum Scatter Gather Elements wqe
	+ * @inline_data: Maximum inline data size
	+ * @shift: Returns the shift needed based on sge
	+ *
	+ * Shift can be used to left shift the wqe size based on number of SGEs and inlind data size.
	+ * For 1 SGE or inline data <= 16, shift = 0 (wqe size of 32 bytes).
	+ * For 2 or 3 SGEs or inline data <= 48, shift = 1 (wqe size of 64 bytes).
	+ * Shift of 2 otherwise (wqe size of 128 bytes).
	+ */
	+enum i40iw_status_code
	+i40iw_get_wqe_shift(u32 wqsize, u32 sge, u32 inline_data, u8 *shift)
	+{
	+ u32 wqdepth;
	+ *shift = 0;
	+ if (sge > 1 \|\| inline_data > 16)
	+ *shift = (sge < 4 && inline_data <= 48) ? 1 : 2;
	+
	+ /* check if wqsize is multiple of 2 or not */
	+
	+ if ((wqsize < I40IW_QP_SW_MIN_WQSIZE) \|\| (wqsize & (wqsize - 1)))
	+ return I40IW_ERR_INVALID_SIZE;
	+
	+ wqdepth = wqsize << shift; / in quantas */
	+ if (wqdepth > I40IW_QP_SW_MAX_WQ_QUANTAS)
	+ return I40IW_ERR_INVALID_SIZE;
	+ return 0;
	+}
	+
	+static struct i40iw_qp_uk_ops iw_qp_uk_ops = {
	+ i40iw_qp_post_wr,
	+ i40iw_qp_ring_push_db,
	+ i40iw_rdma_write,
	+ i40iw_rdma_read,
	+ i40iw_send,
	+ i40iw_inline_rdma_write,
	+ i40iw_inline_send,
	+ i40iw_stag_local_invalidate,
	+ i40iw_mw_bind,
	+ i40iw_post_receive,
	+ i40iw_nop
	+};
	+
	+static struct i40iw_cq_ops iw_cq_ops = {
	+ i40iw_cq_request_notification,
	+ i40iw_cq_poll_completion,
	+ i40iw_cq_post_entries,
	+ i40iw_clean_cq
	+};
	+
	+static struct i40iw_device_uk_ops iw_device_uk_ops = {
	+ i40iw_cq_uk_init,
	+ i40iw_qp_uk_init,
	+};
	+
	+/**
	+ * i40iw_qp_uk_init - initialize shared qp
	+ * @qp: hw qp (user and kernel)
	+ * @info: qp initialization info
	+ *
	+ * initializes the vars used in both user and kernel mode.
	+ * size of the wqe depends on numbers of max. fragements
	+ * allowed. Then size of wqe * the number of wqes should be the
	+ * amount of memory allocated for sq and rq. If srq is used,
	+ * then rq_base will point to one rq wqe only (not the whole
	+ * array of wqes)
	+ */
	+enum i40iw_status_code
	+i40iw_qp_uk_init(struct i40iw_qp_uk *qp,
	+ struct i40iw_qp_uk_init_info *info)
	+{
	+ enum i40iw_status_code ret_code = 0;
	+ u32 sq_ring_size;
	+ u8 sqshift, rqshift;
	+
	+ if (info->max_sq_frag_cnt > I40IW_MAX_WQ_FRAGMENT_COUNT)
	+ return I40IW_ERR_INVALID_FRAG_COUNT;
	+
	+ if (info->max_rq_frag_cnt > I40IW_MAX_WQ_FRAGMENT_COUNT)
	+ return I40IW_ERR_INVALID_FRAG_COUNT;
	+ ret_code = i40iw_get_wqe_shift(info->sq_size, info->max_sq_frag_cnt, info->max_inline_data, &sqshift);
	+ if (ret_code)
	+ return ret_code;
	+ rqshift = I40IW_MAX_RQ_WQE_SHIFT;
	+
	+ qp->sq_base = info->sq;
	+ qp->rq_base = info->rq;
	+ qp->shadow_area = info->shadow_area;
	+ qp->sq_wrtrk_array = info->sq_wrtrk_array;
	+ qp->rq_wrid_array = info->rq_wrid_array;
	+
	+ qp->wqe_alloc_reg = info->wqe_alloc_reg;
	+ qp->qp_id = info->qp_id;
	+
	+ qp->sq_size = info->sq_size;
	+ qp->push_db = info->push_db;
	+ qp->push_wqe = info->push_wqe;
	+
	+ qp->max_sq_frag_cnt = info->max_sq_frag_cnt;
	+ sq_ring_size = qp->sq_size << sqshift;
	+
	+ I40IW_RING_INIT(qp->sq_ring, sq_ring_size);
	+ I40IW_RING_INIT(qp->initial_ring, sq_ring_size);
	+ I40IW_RING_MOVE_HEAD(qp->sq_ring, ret_code);
	+ I40IW_RING_MOVE_TAIL(qp->sq_ring);
	+ I40IW_RING_MOVE_HEAD(qp->initial_ring, ret_code);
	+ qp->swqe_polarity = 1;
	+ qp->first_sq_wq = 1;
	+ qp->swqe_polarity_deferred = 1;
	+ qp->rwqe_polarity = 0;
	+
	+ if (!qp->use_srq) {
	+ qp->rq_size = info->rq_size;
	+ qp->max_rq_frag_cnt = info->max_rq_frag_cnt;
	+ I40IW_RING_INIT(qp->rq_ring, qp->rq_size);
	+ qp->rq_wqe_size = rqshift;
	+ qp->rq_wqe_size_multiplier = 4 << rqshift;
	+ }
	+ qp->ops = iw_qp_uk_ops;
	+
	+ return ret_code;
	+}
	+
	+/**
	+ * i40iw_cq_uk_init - initialize shared cq (user and kernel)
	+ * @cq: hw cq
	+ * @info: hw cq initialization info
	+ */
	+enum i40iw_status_code
	+i40iw_cq_uk_init(struct i40iw_cq_uk *cq,
	+ struct i40iw_cq_uk_init_info *info)
	+{
	+ if ((info->cq_size < I40IW_MIN_CQ_SIZE) \|\|
	+ (info->cq_size > I40IW_MAX_CQ_SIZE))
	+ return I40IW_ERR_INVALID_SIZE;
	+ cq->cq_base = (struct i40iw_cqe *)info->cq_base;
	+ cq->cq_id = info->cq_id;
	+ cq->cq_size = info->cq_size;
	+ cq->cqe_alloc_reg = info->cqe_alloc_reg;
	+ cq->shadow_area = info->shadow_area;
	+ cq->avoid_mem_cflct = info->avoid_mem_cflct;
	+
	+ I40IW_RING_INIT(cq->cq_ring, cq->cq_size);
	+ cq->polarity = 1;
	+ cq->ops = iw_cq_ops;
	+
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_device_init_uk - setup routines for iwarp shared device
	+ * @dev: iwarp shared (user and kernel)
	+ */
	+void
	+i40iw_device_init_uk(struct i40iw_dev_uk *dev)
	+{
	+ dev->ops_uk = iw_device_uk_ops;
	+}
	+
	+/**
	+ * i40iw_clean_cq - clean cq entries
	+ * @ queue completion context
	+ * @cq: cq to clean
	+ */
	+void
	+i40iw_clean_cq(void queue, struct i40iw_cq_uk cq)
	+{
	+ u64 *cqe;
	+ u64 qword3, comp_ctx;
	+ u32 cq_head;
	+ u8 polarity, temp;
	+
	+ cq_head = cq->cq_ring.head;
	+ temp = cq->polarity;
	+ do {
	+ if (cq->avoid_mem_cflct)
	+ cqe = (u64 )&(((struct i40iw_extended_cqe )cq->cq_base)[cq_head]);
	+ else
	+ cqe = (u64 *)&cq->cq_base[cq_head];
	+ get_64bit_val(cqe, I40IW_BYTE_24, &qword3);
	+ polarity = (u8)RS_64(qword3, I40IW_CQ_VALID);
	+
	+ if (polarity != temp)
	+ break;
	+
	+ get_64bit_val(cqe, I40IW_BYTE_8, &comp_ctx);
	+ if ((void *)(i40iw_uintptr)comp_ctx == queue)
	+ set_64bit_val(cqe, I40IW_BYTE_8, 0);
	+
	+ cq_head = (cq_head + 1) % cq->cq_ring.size;
	+ if (!cq_head)
	+ temp ^= 1;
	+ } while (TRUE);
	+}
	+
	+/**
	+ * i40iw_nop - send a nop
	+ * @qp: hw qp ptr
	+ * @wr_id: work request id
	+ * @signaled: flag if signaled for completion
	+ * @post_sq: flag to post sq
	+ */
	+enum i40iw_status_code
	+i40iw_nop(struct i40iw_qp_uk *qp,
	+ u64 wr_id,
	+ bool signaled,
	+ bool post_sq)
	+{
	+ u64 header, *wqe;
	+ u32 wqe_idx;
	+
	+ wqe = i40iw_qp_get_next_send_wqe(qp, &wqe_idx, I40IW_QP_WQE_MIN_SIZE,
	+ 0, wr_id);
	+ if (!wqe)
	+ return I40IW_ERR_QP_TOOMANY_WRS_POSTED;
	+ set_64bit_val(wqe, I40IW_BYTE_0, 0);
	+ set_64bit_val(wqe, I40IW_BYTE_8, 0);
	+ set_64bit_val(wqe, I40IW_BYTE_16, 0);
	+
	+ header = LS_64(I40IWQP_OP_NOP, I40IWQPSQ_OPCODE) \|
	+ LS_64(signaled, I40IWQPSQ_SIGCOMPL) \|
	+ LS_64(qp->swqe_polarity, I40IWQPSQ_VALID);
	+
	+ i40iw_wmb(); /* make sure WQE is populated before valid
	+ * bit is set */
	+
	+ set_64bit_val(wqe, I40IW_BYTE_24, header);
	+ if (post_sq)
	+ i40iw_qp_post_wr(qp);
	+
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_fragcnt_to_wqesize_sq - calculate wqe size based on fragment count for SQ
	+ * @frag_cnt: number of fragments
	+ * @wqe_size: size of sq wqe returned
	+ */
	+enum i40iw_status_code
	+i40iw_fragcnt_to_wqesize_sq(u32 frag_cnt, u8 *wqe_size)
	+{
	+ switch (frag_cnt) {
	+ case 0:
	+ case 1:
	+ *wqe_size = I40IW_QP_WQE_MIN_SIZE;
	+ break;
	+ case 2:
	+ case 3:
	+ *wqe_size = 64;
	+ break;
	+ case 4:
	+ case 5:
	+ *wqe_size = 96;
	+ break;
	+ case 6:
	+ case 7:
	+ *wqe_size = 128;
	+ break;
	+ default:
	+ return I40IW_ERR_INVALID_FRAG_COUNT;
	+ }
	+
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_fragcnt_to_wqesize_rq - calculate wqe size based on fragment count for RQ
	+ * @frag_cnt: number of fragments
	+ * @wqe_size: size of rq wqe returned
	+ */
	+enum i40iw_status_code
	+i40iw_fragcnt_to_wqesize_rq(u32 frag_cnt, u8 *wqe_size)
	+{
	+ if (frag_cnt > I40IW_MAX_WQ_FRAGMENT_COUNT) {
	+ return I40IW_ERR_INVALID_FRAG_COUNT;
	+ } else {
	+ *wqe_size = 128;
	+ return 0;
	+ }
	+}
	+
	+/**
	+ * i40iw_inline_data_size_to_wqesize - based on inline data, wqe size
	+ * @data_size: data size for inline
	+ * @wqe_size: size of sq wqe returned
	+ */
	+enum i40iw_status_code
	+i40iw_inline_data_size_to_wqesize(u32 data_size,
	+ u8 *wqe_size)
	+{
	+ if (data_size > I40IW_MAX_INLINE_DATA_SIZE)
	+ return I40IW_ERR_INVALID_INLINE_DATA_SIZE;
	+
	+ if (data_size <= 16)
	+ *wqe_size = I40IW_QP_WQE_MIN_SIZE;
	+ else
	+ *wqe_size = 64;
	+
	+ return 0;
	+}
	Index: sys/dev/ixl/iwarp/i40iw_user.h
	===================================================================
	--- /dev/null
	+++ sys/dev/ixl/iwarp/i40iw_user.h
	@@ -0,0 +1,437 @@
	+/*******************************************************************************
	+*
	+* Copyright (c) 2015-2017 Intel Corporation. All rights reserved.
	+*
	+* This software is available to you under a choice of one of two
	+* licenses. You may choose to be licensed under the terms of the GNU
	+* General Public License (GPL) Version 2, available from the file
	+* COPYING in the main directory of this source tree, or the
	+* OpenFabrics.org BSD license below:
	+*
	+* Redistribution and use in source and binary forms, with or
	+* without modification, are permitted provided that the following
	+* conditions are met:
	+*
	+* - Redistributions of source code must retain the above
	+* copyright notice, this list of conditions and the following
	+* disclaimer.
	+*
	+* - Redistributions in binary form must reproduce the above
	+* copyright notice, this list of conditions and the following
	+* disclaimer in the documentation and/or other materials
	+* provided with the distribution.
	+*
	+* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
	+* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
	+* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
	+* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
	+* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
	+* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
	+* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
	+* SOFTWARE.
	+*
	+*******************************************************************************/
	+/$FreeBSD$/
	+
	+#ifndef I40IW_USER_H
	+#define I40IW_USER_H
	+
	+enum i40iw_device_capabilities_const {
	+ I40IW_WQE_SIZE = 4,
	+ I40IW_CQP_WQE_SIZE = 8,
	+ I40IW_CQE_SIZE = 4,
	+ I40IW_EXTENDED_CQE_SIZE = 8,
	+ I40IW_AEQE_SIZE = 2,
	+ I40IW_CEQE_SIZE = 1,
	+ I40IW_CQP_CTX_SIZE = 8,
	+ I40IW_SHADOW_AREA_SIZE = 8,
	+ I40IW_CEQ_MAX_COUNT = 256,
	+ I40IW_QUERY_FPM_BUF_SIZE = 128,
	+ I40IW_COMMIT_FPM_BUF_SIZE = 128,
	+ I40IW_MIN_IW_QP_ID = 1,
	+ I40IW_MAX_IW_QP_ID = 262143,
	+ I40IW_MIN_CEQID = 0,
	+ I40IW_MAX_CEQID = 256,
	+ I40IW_MIN_CQID = 0,
	+ I40IW_MAX_CQID = 131071,
	+ I40IW_MIN_AEQ_ENTRIES = 1,
	+ I40IW_MAX_AEQ_ENTRIES = 524287,
	+ I40IW_MIN_CEQ_ENTRIES = 1,
	+ I40IW_MAX_CEQ_ENTRIES = 131071,
	+ I40IW_MIN_CQ_SIZE = 1,
	+ I40IW_MAX_CQ_SIZE = 1048575,
	+ I40IW_DB_ID_ZERO = 0,
	+ I40IW_MAX_WQ_FRAGMENT_COUNT = 3,
	+ I40IW_MAX_SGE_RD = 1,
	+ I40IW_MAX_OUTBOUND_MESSAGE_SIZE = 2147483647,
	+ I40IW_MAX_INBOUND_MESSAGE_SIZE = 2147483647,
	+ I40IW_MAX_PUSH_PAGE_COUNT = 4096,
	+ I40IW_MAX_PE_ENABLED_VF_COUNT = 32,
	+ I40IW_MAX_VF_FPM_ID = 47,
	+ I40IW_MAX_SQ_PAYLOAD_SIZE = 2145386496,
	+ I40IW_MAX_INLINE_DATA_SIZE = 48,
	+ I40IW_MAX_PUSHMODE_INLINE_DATA_SIZE = 48,
	+ I40IW_MAX_IRD_SIZE = 64,
	+ I40IW_MAX_ORD_SIZE = 127,
	+ I40IW_MAX_WQ_ENTRIES = 2048,
	+ I40IW_Q2_BUFFER_SIZE = (248 + 100),
	+ I40IW_MAX_WQE_SIZE_RQ = 128,
	+ I40IW_QP_CTX_SIZE = 248,
	+ I40IW_MAX_PDS = 32768
	+};
	+
	+#define i40iw_handle void *
	+#define i40iw_adapter_handle i40iw_handle
	+#define i40iw_qp_handle i40iw_handle
	+#define i40iw_cq_handle i40iw_handle
	+#define i40iw_srq_handle i40iw_handle
	+#define i40iw_pd_id i40iw_handle
	+#define i40iw_stag_handle i40iw_handle
	+#define i40iw_stag_index u32
	+#define i40iw_stag u32
	+#define i40iw_stag_key u8
	+
	+#define i40iw_tagged_offset u64
	+#define i40iw_access_privileges u32
	+#define i40iw_physical_fragment u64
	+#define i40iw_address_list u64 *
	+
	+#define I40IW_CREATE_STAG(index, key) (((index) << 8) + (key))
	+
	+#define I40IW_STAG_KEY_FROM_STAG(stag) ((stag) && 0x000000FF)
	+
	+#define I40IW_STAG_INDEX_FROM_STAG(stag) (((stag) && 0xFFFFFF00) >> 8)
	+
	+#define I40IW_MAX_MR_SIZE 0x10000000000L
	+#define I40IW_MAX_RQ_WQE_SHIFT 2
	+
	+struct i40iw_qp_uk;
	+struct i40iw_cq_uk;
	+struct i40iw_srq_uk;
	+struct i40iw_qp_uk_init_info;
	+struct i40iw_cq_uk_init_info;
	+struct i40iw_srq_uk_init_info;
	+
	+struct i40iw_sge {
	+ i40iw_tagged_offset tag_off;
	+ u32 len;
	+ i40iw_stag stag;
	+};
	+
	+#define i40iw_sgl struct i40iw_sge *
	+
	+struct i40iw_ring {
	+ volatile u32 head;
	+ volatile u32 tail;
	+ u32 size;
	+};
	+
	+struct i40iw_cqe {
	+ u64 buf[I40IW_CQE_SIZE];
	+};
	+
	+struct i40iw_extended_cqe {
	+ u64 buf[I40IW_EXTENDED_CQE_SIZE];
	+};
	+
	+enum i40iw_addressing_type {
	+ I40IW_ADDR_TYPE_ZERO_BASED = 0,
	+ I40IW_ADDR_TYPE_VA_BASED = 1,
	+};
	+
	+#define I40IW_ACCESS_FLAGS_LOCALREAD 0x01
	+#define I40IW_ACCESS_FLAGS_LOCALWRITE 0x02
	+#define I40IW_ACCESS_FLAGS_REMOTEREAD_ONLY 0x04
	+#define I40IW_ACCESS_FLAGS_REMOTEREAD 0x05
	+#define I40IW_ACCESS_FLAGS_REMOTEWRITE_ONLY 0x08
	+#define I40IW_ACCESS_FLAGS_REMOTEWRITE 0x0a
	+#define I40IW_ACCESS_FLAGS_BIND_WINDOW 0x10
	+#define I40IW_ACCESS_FLAGS_ALL 0x1F
	+
	+#define I40IW_OP_TYPE_RDMA_WRITE 0
	+#define I40IW_OP_TYPE_RDMA_READ 1
	+#define I40IW_OP_TYPE_SEND 3
	+#define I40IW_OP_TYPE_SEND_INV 4
	+#define I40IW_OP_TYPE_SEND_SOL 5
	+#define I40IW_OP_TYPE_SEND_SOL_INV 6
	+#define I40IW_OP_TYPE_REC 7
	+#define I40IW_OP_TYPE_BIND_MW 8
	+#define I40IW_OP_TYPE_FAST_REG_NSMR 9
	+#define I40IW_OP_TYPE_INV_STAG 10
	+#define I40IW_OP_TYPE_RDMA_READ_INV_STAG 11
	+#define I40IW_OP_TYPE_NOP 12
	+
	+enum i40iw_completion_status {
	+ I40IW_COMPL_STATUS_SUCCESS = 0,
	+ I40IW_COMPL_STATUS_FLUSHED,
	+ I40IW_COMPL_STATUS_INVALID_WQE,
	+ I40IW_COMPL_STATUS_QP_CATASTROPHIC,
	+ I40IW_COMPL_STATUS_REMOTE_TERMINATION,
	+ I40IW_COMPL_STATUS_INVALID_STAG,
	+ I40IW_COMPL_STATUS_BASE_BOUND_VIOLATION,
	+ I40IW_COMPL_STATUS_ACCESS_VIOLATION,
	+ I40IW_COMPL_STATUS_INVALID_PD_ID,
	+ I40IW_COMPL_STATUS_WRAP_ERROR,
	+ I40IW_COMPL_STATUS_STAG_INVALID_PDID,
	+ I40IW_COMPL_STATUS_RDMA_READ_ZERO_ORD,
	+ I40IW_COMPL_STATUS_QP_NOT_PRIVLEDGED,
	+ I40IW_COMPL_STATUS_STAG_NOT_INVALID,
	+ I40IW_COMPL_STATUS_INVALID_PHYS_BUFFER_SIZE,
	+ I40IW_COMPL_STATUS_INVALID_PHYS_BUFFER_ENTRY,
	+ I40IW_COMPL_STATUS_INVALID_FBO,
	+ I40IW_COMPL_STATUS_INVALID_LENGTH,
	+ I40IW_COMPL_STATUS_INVALID_ACCESS,
	+ I40IW_COMPL_STATUS_PHYS_BUFFER_LIST_TOO_LONG,
	+ I40IW_COMPL_STATUS_INVALID_VIRT_ADDRESS,
	+ I40IW_COMPL_STATUS_INVALID_REGION,
	+ I40IW_COMPL_STATUS_INVALID_WINDOW,
	+ I40IW_COMPL_STATUS_INVALID_TOTAL_LENGTH
	+};
	+
	+enum i40iw_completion_notify {
	+ IW_CQ_COMPL_EVENT = 0,
	+ IW_CQ_COMPL_SOLICITED = 1
	+};
	+
	+struct i40iw_post_send {
	+ i40iw_sgl sg_list;
	+ u32 num_sges;
	+};
	+
	+struct i40iw_post_inline_send {
	+ void *data;
	+ u32 len;
	+};
	+
	+struct i40iw_rdma_write {
	+ i40iw_sgl lo_sg_list;
	+ u32 num_lo_sges;
	+ struct i40iw_sge rem_addr;
	+};
	+
	+struct i40iw_inline_rdma_write {
	+ void *data;
	+ u32 len;
	+ struct i40iw_sge rem_addr;
	+};
	+
	+struct i40iw_rdma_read {
	+ struct i40iw_sge lo_addr;
	+ struct i40iw_sge rem_addr;
	+};
	+
	+struct i40iw_bind_window {
	+ i40iw_stag mr_stag;
	+ u64 bind_length;
	+ void *va;
	+ enum i40iw_addressing_type addressing_type;
	+ bool enable_reads;
	+ bool enable_writes;
	+ i40iw_stag mw_stag;
	+};
	+
	+struct i40iw_inv_local_stag {
	+ i40iw_stag target_stag;
	+};
	+
	+struct i40iw_post_sq_info {
	+ u64 wr_id;
	+ u8 op_type;
	+ bool signaled;
	+ bool read_fence;
	+ bool local_fence;
	+ bool inline_data;
	+ bool defer_flag;
	+ u32 stag_to_inv;
	+ union {
	+ struct i40iw_post_send send;
	+ struct i40iw_rdma_write rdma_write;
	+ struct i40iw_rdma_read rdma_read;
	+ struct i40iw_rdma_read rdma_read_inv;
	+ struct i40iw_bind_window bind_window;
	+ struct i40iw_inv_local_stag inv_local_stag;
	+ struct i40iw_inline_rdma_write inline_rdma_write;
	+ struct i40iw_post_inline_send inline_send;
	+ } op;
	+};
	+
	+struct i40iw_post_rq_info {
	+ u64 wr_id;
	+ i40iw_sgl sg_list;
	+ u32 num_sges;
	+};
	+
	+struct i40iw_cq_poll_info {
	+ u64 wr_id;
	+ i40iw_qp_handle qp_handle;
	+ u32 bytes_xfered;
	+ u32 tcp_seq_num;
	+ u32 qp_id;
	+ i40iw_stag inv_stag;
	+ enum i40iw_completion_status comp_status;
	+ u16 major_err;
	+ u16 minor_err;
	+ u8 op_type;
	+ bool stag_invalid_set;
	+ bool push_dropped;
	+ bool error;
	+ bool is_srq;
	+ bool solicited_event;
	+};
	+
	+struct i40iw_qp_uk_ops {
	+ void (iw_qp_post_wr)(struct i40iw_qp_uk );
	+ void (iw_qp_ring_push_db)(struct i40iw_qp_uk , u32);
	+ enum i40iw_status_code (iw_rdma_write)(struct i40iw_qp_uk ,
	+ struct i40iw_post_sq_info *, bool);
	+ enum i40iw_status_code (iw_rdma_read)(struct i40iw_qp_uk ,
	+ struct i40iw_post_sq_info *, bool, bool);
	+ enum i40iw_status_code (iw_send)(struct i40iw_qp_uk ,
	+ struct i40iw_post_sq_info *, bool);
	+ enum i40iw_status_code (iw_inline_rdma_write)(struct i40iw_qp_uk ,
	+ struct i40iw_post_sq_info *, bool);
	+ enum i40iw_status_code (iw_inline_send)(struct i40iw_qp_uk ,
	+ struct i40iw_post_sq_info *, bool);
	+ enum i40iw_status_code (iw_stag_local_invalidate)(struct i40iw_qp_uk ,
	+ struct i40iw_post_sq_info *, bool);
	+ enum i40iw_status_code (iw_mw_bind)(struct i40iw_qp_uk ,
	+ struct i40iw_post_sq_info *, bool);
	+ enum i40iw_status_code (iw_post_receive)(struct i40iw_qp_uk ,
	+ struct i40iw_post_rq_info *);
	+ enum i40iw_status_code (iw_post_nop)(struct i40iw_qp_uk , u64, bool, bool);
	+};
	+
	+struct i40iw_cq_ops {
	+ void (iw_cq_request_notification)(struct i40iw_cq_uk ,
	+ enum i40iw_completion_notify);
	+ enum i40iw_status_code (iw_cq_poll_completion)(struct i40iw_cq_uk ,
	+ struct i40iw_cq_poll_info *);
	+ enum i40iw_status_code (iw_cq_post_entries)(struct i40iw_cq_uk , u8 count);
	+ void (iw_cq_clean)(void , struct i40iw_cq_uk *);
	+};
	+
	+struct i40iw_dev_uk;
	+
	+struct i40iw_device_uk_ops {
	+ enum i40iw_status_code (iwarp_cq_uk_init)(struct i40iw_cq_uk ,
	+ struct i40iw_cq_uk_init_info *);
	+ enum i40iw_status_code (iwarp_qp_uk_init)(struct i40iw_qp_uk ,
	+ struct i40iw_qp_uk_init_info *);
	+};
	+
	+struct i40iw_dev_uk {
	+ struct i40iw_device_uk_ops ops_uk;
	+};
	+
	+struct i40iw_sq_uk_wr_trk_info {
	+ u64 wrid;
	+ u32 wr_len;
	+ u8 wqe_size;
	+ u8 reserved[3];
	+};
	+
	+struct i40iw_qp_quanta {
	+ u64 elem[I40IW_WQE_SIZE];
	+};
	+
	+struct i40iw_qp_uk {
	+ struct i40iw_qp_quanta *sq_base;
	+ struct i40iw_qp_quanta *rq_base;
	+ u32 IOMEM *wqe_alloc_reg;
	+ struct i40iw_sq_uk_wr_trk_info *sq_wrtrk_array;
	+ u64 *rq_wrid_array;
	+ u64 *shadow_area;
	+ u32 *push_db;
	+ u64 *push_wqe;
	+ struct i40iw_ring sq_ring;
	+ struct i40iw_ring rq_ring;
	+ struct i40iw_ring initial_ring;
	+ u32 qp_id;
	+ u32 sq_size;
	+ u32 rq_size;
	+ u32 max_sq_frag_cnt;
	+ u32 max_rq_frag_cnt;
	+ struct i40iw_qp_uk_ops ops;
	+ bool use_srq;
	+ u8 swqe_polarity;
	+ u8 swqe_polarity_deferred;
	+ u8 rwqe_polarity;
	+ u8 rq_wqe_size;
	+ u8 rq_wqe_size_multiplier;
	+ bool deferred_flag;
	+#if defined(NDK_DEBUG)
	+ struct i40iw_ring ndk_sq_ring;
	+ struct i40iw_ring ndk_rq_ring;
	+ u32 ndk_debug_sq_array; / array of send wqe_index to be stored */
	+ u32 ndk_debug_rq_array; / array of rq wqe_index to be stored */
	+#endif
	+ bool first_sq_wq;
	+};
	+
	+struct i40iw_cq_uk {
	+ struct i40iw_cqe *cq_base;
	+ u32 IOMEM *cqe_alloc_reg;
	+ u64 *shadow_area;
	+ u32 cq_id;
	+ u32 cq_size;
	+ struct i40iw_ring cq_ring;
	+ u8 polarity;
	+ bool avoid_mem_cflct;
	+
	+ struct i40iw_cq_ops ops;
	+};
	+
	+struct i40iw_qp_uk_init_info {
	+ struct i40iw_qp_quanta *sq;
	+ struct i40iw_qp_quanta *rq;
	+ u32 IOMEM *wqe_alloc_reg;
	+ u64 *shadow_area;
	+ struct i40iw_sq_uk_wr_trk_info *sq_wrtrk_array;
	+ u64 *rq_wrid_array;
	+ u32 *push_db;
	+ u64 *push_wqe;
	+ u32 qp_id;
	+ u32 sq_size;
	+ u32 rq_size;
	+ u32 max_sq_frag_cnt;
	+ u32 max_rq_frag_cnt;
	+ u32 max_inline_data;
	+ u8 first_sq_wq;
	+};
	+
	+struct i40iw_cq_uk_init_info {
	+ u32 IOMEM *cqe_alloc_reg;
	+ struct i40iw_cqe *cq_base;
	+ u64 *shadow_area;
	+ u32 cq_size;
	+ u32 cq_id;
	+ bool avoid_mem_cflct;
	+};
	+
	+void i40iw_device_init_uk(struct i40iw_dev_uk *dev);
	+
	+void i40iw_qp_post_wr(struct i40iw_qp_uk *qp);
	+u64 i40iw_qp_get_next_send_wqe(struct i40iw_qp_uk qp, u32 *wqe_idx,
	+ u8 wqe_size,
	+ u32 total_size,
	+ u64 wr_id
	+ );
	+
	+u64 i40iw_qp_get_next_recv_wqe(struct i40iw_qp_uk qp, u32 *wqe_idx);
	+u64 i40iw_qp_get_next_srq_wqe(struct i40iw_srq_uk srq, u32 *wqe_idx);
	+
	+enum i40iw_status_code i40iw_cq_uk_init(struct i40iw_cq_uk *cq,
	+ struct i40iw_cq_uk_init_info *info);
	+enum i40iw_status_code i40iw_qp_uk_init(struct i40iw_qp_uk *qp,
	+ struct i40iw_qp_uk_init_info *info);
	+
	+void i40iw_clean_cq(void queue, struct i40iw_cq_uk cq);
	+enum i40iw_status_code i40iw_nop(struct i40iw_qp_uk *qp, u64 wr_id,
	+ bool signaled, bool post_sq);
	+enum i40iw_status_code i40iw_fragcnt_to_wqesize_sq(u32 frag_cnt, u8 *wqe_size);
	+enum i40iw_status_code i40iw_fragcnt_to_wqesize_rq(u32 frag_cnt, u8 *wqe_size);
	+enum i40iw_status_code i40iw_inline_data_size_to_wqesize(u32 data_size,
	+ u8 *wqe_size);
	+u32 i40iw_powerof2_roundup(u32 value);
	+enum i40iw_status_code i40iw_get_wqe_shift(u32 wqdepth, u32 sge, u32 inline_data, u8 *shift);
	+#endif
	Index: sys/dev/ixl/iwarp/iw_ixl.h
	===================================================================
	--- /dev/null
	+++ sys/dev/ixl/iwarp/iw_ixl.h
	@@ -0,0 +1,443 @@
	+/*******************************************************************************
	+*
	+* Copyright (c) 2015-2017 Intel Corporation. All rights reserved.
	+*
	+* This software is available to you under a choice of one of two
	+* licenses. You may choose to be licensed under the terms of the GNU
	+* General Public License (GPL) Version 2, available from the file
	+* COPYING in the main directory of this source tree, or the
	+* OpenFabrics.org BSD license below:
	+*
	+* Redistribution and use in source and binary forms, with or
	+* without modification, are permitted provided that the following
	+* conditions are met:
	+*
	+* - Redistributions of source code must retain the above
	+* copyright notice, this list of conditions and the following
	+* disclaimer.
	+*
	+* - Redistributions in binary form must reproduce the above
	+* copyright notice, this list of conditions and the following
	+* disclaimer in the documentation and/or other materials
	+* provided with the distribution.
	+*
	+* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
	+* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
	+* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
	+* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
	+* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
	+* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
	+* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
	+* SOFTWARE.
	+*
	+*******************************************************************************/
	+/$FreeBSD$/
	+
	+#ifndef _IW_IXL_H_
	+#define _IW_IXL_H_
	+
	+#include <sys/param.h>
	+#include <rdma/ib_verbs.h>
	+#include <rdma/iw_cm.h>
	+
	+#include "i40iw_status.h"
	+#include "i40iw_osdep.h"
	+#include "i40iw_d.h"
	+#include "i40iw_hmc.h"
	+#include "i40iw_type.h"
	+#include "i40iw_p.h"
	+#include "i40iw_user.h"
	+#include "i40iw_puda.h"
	+#include "iw_ixl_ucontext.h"
	+#include "iw_ixl_pble.h"
	+#include "iw_ixl_verbs.h"
	+#include "iw_ixl_cm.h"
	+#include "iw_ixl_dbg.h"
	+
	+#include "ixl_iw.h"
	+
	+#if defined(INET6) && defined(IW_IXL_RDMA_4_9)
	+#define IPV6_SUPPORT
	+#else
	+#undef IPV6_SUPPORT
	+#endif
	+
	+#define I40IW_FW_VERSION 2
	+
	+#define I40IW_ARP_ADD 1
	+#define I40IW_ARP_DELETE 2
	+#define I40IW_ARP_RESOLVE 3
	+
	+#define IW_CCQ_SIZE (I40IW_CQP_SW_SQSIZE_2048 + 1)
	+
	+#define IW_FIRST_QPN 1
	+
	+#define I40IW_EVENT_TIMEOUT 100000
	+
	+#define I40IW_NO_VLAN 0xffff
	+#define I40IW_NO_QSET 0xffff
	+
	+#define IW_HMC_OBJ_TYPE_NUM ARRAY_SIZE(iw_hmc_obj_types)
	+
	+#define I40IW_MAX_PAGES_PER_FMR 512
	+#define I40IW_MIN_PAGES_PER_FMR 1
	+
	+#define IW_CFG_FPM_QP_COUNT 32768
	+
	+#ifndef VLAN_ETH_HLEN
	+#define VLAN_ETH_HLEN 18
	+#endif
	+
	+#ifdef IW_IXL_RDMA_4_9
	+#define I40IW_CQP_COMPL_RQ_WQE_FLUSHED 2
	+#define I40IW_CQP_COMPL_SQ_WQE_FLUSHED 3
	+#endif
	+
	+struct i40iw_handler;
	+
	+struct i40iw_cqp_compl_info {
	+ u32 op_ret_val;
	+ u16 maj_err_code;
	+ u16 min_err_code;
	+ bool error;
	+ u8 op_code;
	+};
	+
	+#define i40iw_pr_err(fmt, args ...) printf("%s: "fmt, __func__, ## args)
	+
	+struct i40iw_cqp_request {
	+ struct cqp_commands_info info;
	+ wait_queue_head_t waitq;
	+ struct list_head list;
	+ atomic_t refcount;
	+ void (callback_fcn) (struct i40iw_cqp_request , u32);
	+ void *param;
	+ struct i40iw_cqp_compl_info compl_info;
	+ bool waiting;
	+ bool request_done;
	+ bool dynamic;
	+};
	+
	+struct i40iw_cqp {
	+ struct i40iw_sc_cqp sc_cqp;
	+ spinlock_t req_lock; /* cqp request list */
	+ wait_queue_head_t waitq;
	+ struct i40iw_dma_mem sq;
	+ struct i40iw_dma_mem host_ctx;
	+ u64 *scratch_array;
	+ struct i40iw_cqp_request *cqp_requests;
	+ struct list_head cqp_avail_reqs;
	+ struct list_head cqp_pending_reqs;
	+};
	+
	+struct i40iw_ccq {
	+ struct i40iw_sc_cq sc_cq;
	+ spinlock_t lock; /* ccq control */
	+ wait_queue_head_t waitq;
	+ struct i40iw_dma_mem mem_cq;
	+ struct i40iw_dma_mem shadow_area;
	+};
	+
	+struct i40iw_ceq {
	+ struct i40iw_sc_ceq sc_ceq;
	+ struct i40iw_dma_mem mem;
	+ u32 msix_idx;
	+ struct i40iw_device *iwdev;
	+ struct task irq_task;
	+ struct taskqueue *irq_tq;
	+};
	+
	+struct i40iw_aeq {
	+ struct i40iw_sc_aeq sc_aeq;
	+ struct i40iw_dma_mem mem;
	+};
	+
	+struct i40iw_arp_entry {
	+ u32 ip_addr[4];
	+ u8 mac_addr[ETH_ALEN];
	+};
	+
	+enum init_completion_state {
	+ INVALID_STATE = 0,
	+ INITIAL_STATE,
	+ CQP_CREATED,
	+ HMC_OBJS_CREATED,
	+ PBLE_CHUNK_MEM,
	+ CCQ_CREATED,
	+ AEQ_CREATED,
	+ CEQ_CREATED,
	+ ILQ_CREATED,
	+ IEQ_CREATED,
	+ IP_ADDR_REGISTERED,
	+ RDMA_DEV_REGISTERED
	+};
	+
	+struct i40iw_msix_vector {
	+ u32 idx;
	+ struct resource *res;
	+ void *tag;
	+ u32 cpu_affinity;
	+};
	+
	+struct i40iw_dev_ctx {
	+ bus_space_tag_t mem_bus_space_tag;
	+ bus_space_handle_t mem_bus_space_handle;
	+ bus_size_t mem_bus_space_size;
	+ struct device *dev;
	+};
	+
	+struct i40iw_device {
	+ struct i40iw_ib_device *iwibdev;
	+ struct ifnet *ifp;
	+ wait_queue_head_t vchnl_waitq;
	+ struct i40iw_sc_dev sc_dev;
	+ struct i40iw_sc_vsi vsi;/* Added for FSL VSI SUPPORT */
	+ struct i40iw_handler *hdl;
	+ struct ixl_iw_pf *ldev;
	+ struct i40iw_hw hw;
	+ struct i40iw_dev_ctx dev_ctx;
	+ struct i40iw_cm_core cm_core;
	+ u8 *mem_resources;
	+ i40iw_bits_t allocated_qps;
	+ i40iw_bits_t allocated_cqs;
	+ i40iw_bits_t allocated_mrs;
	+ i40iw_bits_t allocated_pds;
	+ i40iw_bits_t allocated_arps;
	+ struct i40iw_qp **qp_table;
	+ struct ixl_iw_msix msix_info;
	+ bool msix_shared;
	+ u32 msix_count;
	+ struct i40iw_msix_vector *iw_msixtbl;
	+ struct ixl_iw_msix_mapping msix_mapping;
	+
	+ struct i40iw_hmc_pble_rsrc *pble_rsrc;
	+ struct i40iw_arp_entry *arp_table;
	+ struct i40iw_cqp cqp;
	+ struct i40iw_ccq ccq;
	+ u32 ceqs_count;
	+ struct i40iw_ceq *ceqlist;
	+ struct i40iw_aeq aeq;
	+ u32 arp_table_size;
	+ u32 next_arp_index;
	+ spinlock_t resource_lock; /* hw resource access */
	+ spinlock_t qptable_lock;
	+ u32 device_cap_flags;
	+ u8 resource_profile;
	+ u8 max_rdma_vfs;
	+ u8 iw_status;
	+ bool push_mode; /* Initialized with tunable */
	+ struct mutex pbl_mutex;
	+ struct task irq_task;
	+ struct taskqueue *irq_tq;
	+ struct i40iw_dma_mem obj_mem;
	+ struct i40iw_dma_mem obj_next;
	+ u8 *hmc_info_mem;
	+ u32 sd_type;
	+ enum init_completion_state init_state;
	+ u16 mac_ip_table_idx;
	+ u32 max_mr;
	+ u32 max_qp;
	+ u32 max_cq;
	+ u32 max_pd;
	+ u32 next_qp;
	+ u32 next_cq;
	+ u32 next_pd;
	+ u32 max_cqe;
	+ u32 mr_stagmask;
	+ u32 mpa_version;
	+ bool closing;
	+ bool reset;
	+ u32 used_pds;
	+ u32 used_cqs;
	+ u32 used_mrs;
	+ u32 used_qps;
	+ wait_queue_head_t close_wq;
	+ atomic_t use_count;
	+};
	+
	+struct i40iw_ib_device {
	+ struct ib_device ibdev;
	+ struct i40iw_device *iwdev;
	+};
	+
	+struct i40iw_handler {
	+ struct list_head list;
	+ struct i40iw_device device;
	+ struct ixl_iw_pf ldev;
	+};
	+
	+/**
	+ * i40iw_alloc_resource - allocate a resource
	+ * @iwdev: device pointer
	+ * @resource_array: resource bit array:
	+ * @max_resources: maximum resource number
	+ * @req_resources_num: Allocated resource number
	+ * @next: next free id
	+ **/
	+static inline int
	+i40iw_alloc_resource(struct i40iw_device *iwdev,
	+ i40iw_bits_t resource_array,
	+ u32 max_resources,
	+ u32 *req_resource_num,
	+ u32 *next)
	+{
	+ u32 resource_num;
	+ unsigned long flags;
	+
	+ spin_lock_irqsave(&iwdev->resource_lock, flags);
	+ resource_num = find_next_zero_bit(resource_array, max_resources, *next);
	+ if (resource_num >= max_resources) {
	+ resource_num = find_first_zero_bit(resource_array, max_resources);
	+ if (resource_num >= max_resources) {
	+ spin_unlock_irqrestore(&iwdev->resource_lock, flags);
	+ return -EOVERFLOW;
	+ }
	+ }
	+ set_bit(resource_num, resource_array);
	+ *next = resource_num + 1;
	+ if (*next == max_resources)
	+ *next = 0;
	+ *req_resource_num = resource_num;
	+ spin_unlock_irqrestore(&iwdev->resource_lock, flags);
	+
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_free_resource - free a resource
	+ * @iwdev: device pointer
	+ * @resource_array: resource array for the resource_num
	+ * @resource_num: resource number to free
	+ **/
	+static inline void
	+i40iw_free_resource(struct i40iw_device *iwdev,
	+ i40iw_bits_t resource_array,
	+ u32 resource_num)
	+{
	+ unsigned long flags;
	+
	+ spin_lock_irqsave(&iwdev->resource_lock, flags);
	+ clear_bit(resource_num, resource_array);
	+ spin_unlock_irqrestore(&iwdev->resource_lock, flags);
	+}
	+
	+/**
	+ * to_iwdev - get device
	+ * @ibdev: ib device
	+ **/
	+static inline struct i40iw_device *
	+to_iwdev(struct ib_device *ibdev)
	+{
	+ return container_of(ibdev, struct i40iw_ib_device, ibdev)->iwdev;
	+}
	+
	+/**
	+ * to_ucontext - get user context
	+ * @ibucontext: ib user context
	+ **/
	+static inline struct i40iw_ucontext *
	+to_ucontext(struct ib_ucontext *ibucontext)
	+{
	+ return container_of(ibucontext, struct i40iw_ucontext, ibucontext);
	+}
	+
	+/**
	+ * to_iwpd - get protection domain
	+ * @ibpd: ib pd
	+ **/
	+static inline struct i40iw_pd *
	+to_iwpd(struct ib_pd *ibpd)
	+{
	+ return container_of(ibpd, struct i40iw_pd, ibpd);
	+}
	+
	+/**
	+ * to_iwmr - get device memory region
	+ * @ibdev: ib memory region
	+ **/
	+static inline struct i40iw_mr *
	+to_iwmr(struct ib_mr *ibmr)
	+{
	+ return container_of(ibmr, struct i40iw_mr, ibmr);
	+}
	+
	+/**
	+ * to_iwcq - get completion queue
	+ * @ibcq: ib cqdevice
	+ **/
	+static inline struct i40iw_cq *
	+to_iwcq(struct ib_cq *ibcq)
	+{
	+ return container_of(ibcq, struct i40iw_cq, ibcq);
	+}
	+
	+/**
	+ * to_iwqp - get device qp
	+ * @ibqp: ib qp
	+ **/
	+static inline struct i40iw_qp *
	+to_iwqp(struct ib_qp *ibqp)
	+{
	+ return container_of(ibqp, struct i40iw_qp, ibqp);
	+}
	+
	+#ifndef IW_IXL_RDMA_4_9
	+/**
	+ * to_i40iw_fr_page_list - get fast memreg page list
	+ * @ibfpl: ib fast memreg page list
	+ **/
	+static inline struct i40iw_fast_reg_page_list *
	+to_i40iw_fr_page_list(struct ib_fast_reg_page_list *ibfpl)
	+{
	+ return container_of(ibfpl, struct i40iw_fast_reg_page_list, ibfpl);
	+}
	+
	+#endif
	+void i40iw_flush_wqes(struct i40iw_device , struct i40iw_qp );
	+void i40iw_manage_arp_cache(struct i40iw_device , unsigned char , u32 *, bool, u32);
	+int i40iw_manage_apbvt(struct i40iw_device *, u16, bool);
	+void i40iw_put_cqp_request(struct i40iw_cqp , struct i40iw_cqp_request );
	+struct i40iw_cqp_request i40iw_get_cqp_request(struct i40iw_cqp , bool);
	+void i40iw_free_cqp_request(struct i40iw_cqp , struct i40iw_cqp_request );
	+u32 i40iw_initialize_hw_resources(struct i40iw_device *);
	+int i40iw_register_rdma_device(struct i40iw_device *);
	+void i40iw_port_ibevent(struct i40iw_device *);
	+enum i40iw_status_code i40iw_handle_cqp_op(struct i40iw_device , struct i40iw_cqp_request );
	+enum i40iw_status_code i40iw_manage_qhash(struct i40iw_device , struct i40iw_cm_info , enum i40iw_quad_entry_type, enum i40iw_quad_hash_manage_type, void *, bool);
	+void i40iw_receive_ilq(struct i40iw_sc_vsi , struct i40iw_puda_buf );
	+void i40iw_free_sqbuf(struct i40iw_sc_vsi , void );
	+enum i40iw_status_code i40iw_obj_aligned_mem(struct i40iw_device , struct i40iw_dma_mem , u32, u32);
	+void i40iw_request_reset(struct i40iw_device *);
	+void i40iw_destroy_rdma_device(struct i40iw_ib_device *);
	+void i40iw_setup_cm_core(struct i40iw_device *);
	+int i40iw_modify_qp(struct ib_qp , struct ib_qp_attr , int, struct ib_udata *);
	+void i40iw_hw_modify_qp(struct i40iw_device , struct i40iw_qp , struct i40iw_modify_qp_info *, bool);
	+void i40iw_cq_wq_destroy(struct i40iw_device , struct i40iw_sc_cq );
	+void i40iw_next_iw_state(struct i40iw_qp *, u8, u8, u8, u8);
	+void i40iw_cm_disconn(struct i40iw_qp *);
	+void i40iw_free_qp_resources(struct i40iw_device , struct i40iw_qp , u32);
	+void i40iw_cleanup_cm_core(struct i40iw_cm_core *);
	+void i40iw_process_ceq(struct i40iw_device , struct i40iw_ceq );
	+void i40iw_process_aeq(struct i40iw_device *);
	+int i40iw_send_syn(struct i40iw_cm_node *, u32);
	+int i40iw_send_reset(struct i40iw_cm_node *cm_node);
	+struct i40iw_cm_node i40iw_find_node(struct i40iw_cm_core , u16, u32 , u16, u32 , bool);
	+enum i40iw_status_code i40iw_hw_flush_wqes(struct i40iw_device , struct i40iw_sc_qp , struct i40iw_qp_flush_info *, bool);
	+void i40iw_gen_ae(struct i40iw_device , struct i40iw_sc_qp , struct i40iw_gen_ae_info *, bool);
	+#ifdef IW_IXL_RDMA_4_9
	+struct ib_mr i40iw_reg_phys_mr(struct ib_pd , u64, u64, int, u64 *);
	+#else
	+struct ib_mr i40iw_reg_phys_mr(struct ib_pd , struct ib_phys_buf , int, int, u64 );
	+#endif
	+void i40iw_cleanup_pending_cqp_op(struct i40iw_device *);
	+void i40iw_rem_pdusecount(struct i40iw_pd , struct i40iw_device );
	+void i40iw_add_pdusecount(struct i40iw_pd *);
	+void i40iw_rem_devusecount(struct i40iw_device *);
	+void i40iw_add_devusecount(struct i40iw_device *);
	+void i40iw_copy_ip_ntohl(u32 , __be32 );
	+void i40iw_add_ref(struct ib_qp *);
	+void i40iw_rem_ref(struct ib_qp *);
	+struct ib_qp i40iw_get_qp(struct ib_device , int);
	+void i40iw_cq_ceq_mapping_watch(struct i40iw_device *, unsigned int, bool);
	+
	+#endif /* _IW_IXL_H_ */
	Index: sys/dev/ixl/iwarp/iw_ixl.c
	===================================================================
	--- /dev/null
	+++ sys/dev/ixl/iwarp/iw_ixl.c
	@@ -0,0 +1,1935 @@
	+/*******************************************************************************
	+*
	+* Copyright (c) 2015-2017 Intel Corporation. All rights reserved.
	+*
	+* This software is available to you under a choice of one of two
	+* licenses. You may choose to be licensed under the terms of the GNU
	+* General Public License (GPL) Version 2, available from the file
	+* COPYING in the main directory of this source tree, or the
	+* OpenFabrics.org BSD license below:
	+*
	+* Redistribution and use in source and binary forms, with or
	+* without modification, are permitted provided that the following
	+* conditions are met:
	+*
	+* - Redistributions of source code must retain the above
	+* copyright notice, this list of conditions and the following
	+* disclaimer.
	+*
	+* - Redistributions in binary form must reproduce the above
	+* copyright notice, this list of conditions and the following
	+* disclaimer in the documentation and/or other materials
	+* provided with the distribution.
	+*
	+* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
	+* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
	+* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
	+* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
	+* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
	+* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
	+* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
	+* SOFTWARE.
	+*
	+*******************************************************************************/
	+/$FreeBSD$/
	+
	+#include <sys/param.h>
	+#include <sys/module.h>
	+#include <sys/kernel.h>
	+#include <sys/systm.h>
	+#include <sys/bus.h>
	+#include <sys/rman.h>
	+#include <sys/socket.h>
	+
	+#include <net/if.h>
	+#include <net/if_var.h>
	+#include <net/if_dl.h>
	+
	+#include <machine/resource.h>
	+
	+#include "iw_ixl.h"
	+#include "i40iw_register.h"
	+
	+/**
	+ * Driver version
	+ */
	+char ixliw_driver_version[] = "0.2.0";
	+
	+/*
	+ * TUNEABLE PARAMETERS:
	+ */
	+static
	+SYSCTL_NODE(_hw, OID_AUTO, iw_ixl, CTLFLAG_RD, 0,
	+ "IW_IXL driver parameters");
	+
	+static int ixliw_max_ceq = 0;
	+TUNABLE_INT("hw.iw_ixl.max_ceq", &ixliw_max_ceq);
	+SYSCTL_INT(_hw_iw_ixl, OID_AUTO, max_ceq, CTLFLAG_RDTUN,
	+ &ixliw_max_ceq, 0, "iWARP max CEQ");
	+
	+static int ixliw_debug = 0;
	+TUNABLE_INT("hw.iw_ixl.debug", &ixliw_debug);
	+SYSCTL_INT(_hw_iw_ixl, OID_AUTO, debug, CTLFLAG_RWTUN,
	+ &ixliw_debug, 0, "iWARP debug");
	+
	+static int ixliw_mpa_version = 2;
	+TUNABLE_INT("hw.iw_ixl.mpa_version", &ixliw_mpa_version);
	+SYSCTL_INT(_hw_iw_ixl, OID_AUTO, mpa_version, CTLFLAG_RDTUN,
	+ &ixliw_mpa_version, 0, "iWARP mpa version");
	+
	+static int ixliw_push_mode = 0;
	+
	+static LIST_HEAD(i40iw_handlers);
	+static spinlock_t i40iw_handler_lock;
	+
	+/**
	+ * i40iw_find_i40e_handler - find a handler given a client info
	+ * @ldev: pointer to a client info
	+ */
	+static struct i40iw_handler *
	+i40iw_find_handler(struct ixl_iw_pf *ldev)
	+{
	+ struct i40iw_handler *hdl;
	+ unsigned long flags;
	+
	+ spin_lock_irqsave(&i40iw_handler_lock, flags);
	+ list_for_each_entry(hdl, &i40iw_handlers, list) {
	+ if (hdl->ldev.dev == ldev->dev) {
	+ spin_unlock_irqrestore(&i40iw_handler_lock, flags);
	+ return hdl;
	+ }
	+ }
	+ spin_unlock_irqrestore(&i40iw_handler_lock, flags);
	+ return NULL;
	+}
	+
	+/**
	+ * i40iw_add_handler - add a handler to the list
	+ * @hdl: handler to be added to the handler list
	+ */
	+static void
	+i40iw_add_handler(struct i40iw_handler *hdl)
	+{
	+ unsigned long flags;
	+
	+ spin_lock_irqsave(&i40iw_handler_lock, flags);
	+ list_add(&hdl->list, &i40iw_handlers);
	+ spin_unlock_irqrestore(&i40iw_handler_lock, flags);
	+}
	+
	+/**
	+ * i40iw_del_handler - delete a handler from the list
	+ * @hdl: handler to be deleted from the handler list
	+ */
	+static int
	+i40iw_del_handler(struct i40iw_handler *hdl)
	+{
	+ unsigned long flags;
	+
	+ spin_lock_irqsave(&i40iw_handler_lock, flags);
	+ list_del(&hdl->list);
	+ spin_unlock_irqrestore(&i40iw_handler_lock, flags);
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_enable_intr - set up device interrupts
	+ * @dev: hardware control device structure
	+ * @msix_id: id of the interrupt to be enabled
	+ */
	+static void
	+i40iw_enable_intr(struct i40iw_sc_dev *dev, u32 msix_id)
	+{
	+ u32 val;
	+
	+ val = I40E_PFINT_DYN_CTLN_INTENA_MASK \|
	+ I40E_PFINT_DYN_CTLN_CLEARPBA_MASK \|
	+ (3 << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT);
	+ if (dev->is_pf)
	+ wr32(dev->hw, I40E_PFINT_DYN_CTLN(msix_id - 1), val);
	+ else
	+ wr32(dev->hw, I40E_VFINT_DYN_CTLN1(msix_id - 1), val);
	+}
	+
	+/**
	+ * i40iw_dpc - tasklet for aeq and ceq 0
	+ * @data: iwarp device
	+ */
	+static void
	+i40iw_dpc(void *context, int pending)
	+{
	+ struct i40iw_device iwdev = (struct i40iw_device )context;
	+
	+ if (iwdev->msix_shared)
	+ i40iw_process_ceq(iwdev, iwdev->ceqlist);
	+ i40iw_process_aeq(iwdev);
	+ i40iw_enable_intr(&iwdev->sc_dev, iwdev->iw_msixtbl[0].idx);
	+}
	+
	+/**
	+ * i40iw_ceq_dpc - dpc handler for CEQ
	+ * @data: data points to CEQ
	+ */
	+static void
	+i40iw_ceq_dpc(void *context, int pending)
	+{
	+ struct i40iw_ceq iwceq = (struct i40iw_ceq )context;
	+ struct i40iw_device *iwdev = iwceq->iwdev;
	+
	+ i40iw_process_ceq(iwdev, iwceq);
	+ i40iw_enable_intr(&iwdev->sc_dev, iwceq->msix_idx);
	+}
	+
	+/**
	+ * i40iw_irq_handler - interrupt handler for aeq and ceq0
	+ * @irq: Interrupt request number
	+ * @data: iwarp device
	+ */
	+static void
	+i40iw_irq_handler(void *data)
	+{
	+ struct i40iw_device iwdev = (struct i40iw_device )data;
	+
	+ taskqueue_enqueue(iwdev->irq_tq, &iwdev->irq_task);
	+ return;
	+}
	+
	+/**
	+ * i40iw_destroy_cqp - destroy control qp
	+ * @iwdev: iwarp device
	+ * @create_done: 1 if cqp create poll was success
	+ *
	+ * Issue destroy cqp request and
	+ * free the resources associated with the cqp
	+ */
	+static void
	+i40iw_destroy_cqp(struct i40iw_device *iwdev, bool free_hwcqp)
	+{
	+ struct i40iw_sc_dev *dev = &iwdev->sc_dev;
	+ struct i40iw_cqp *cqp = &iwdev->cqp;
	+
	+ INIT_DBG_DEV(iwdev->ldev->dev, "begin");
	+
	+ if (free_hwcqp)
	+ dev->cqp_ops->cqp_destroy(dev->cqp);
	+
	+ i40iw_cleanup_pending_cqp_op(iwdev);
	+
	+ ixliw_free_dma_mem(&cqp->sq);
	+ kfree(cqp->scratch_array);
	+ iwdev->cqp.scratch_array = NULL;
	+
	+ kfree(cqp->cqp_requests);
	+ cqp->cqp_requests = NULL;
	+}
	+
	+/**
	+ * i40iw_disable_irqs - disable device interrupts
	+ * @dev: hardware control device structure
	+ * @msic_vec: msix vector to disable irq
	+ * @dev_id: parameter to pass to free_irq (used during irq setup)
	+ *
	+ * The function is called when destroying aeq/ceq
	+ */
	+static void
	+i40iw_disable_irq(struct i40iw_device *iwdev,
	+ struct i40iw_msix_vector *msix_vec)
	+{
	+ struct i40iw_sc_dev *dev = &iwdev->sc_dev;
	+ int rid = msix_vec->idx + 1;
	+
	+ wr32(dev->hw, I40E_PFINT_DYN_CTLN(msix_vec->idx - 1), 0);
	+
	+ if (msix_vec->tag != NULL) {
	+ bus_teardown_intr(iwdev->ldev->dev, msix_vec->res,
	+ msix_vec->tag);
	+ msix_vec->tag = NULL;
	+ }
	+ if (msix_vec->res != NULL) {
	+ bus_release_resource(iwdev->ldev->dev, SYS_RES_IRQ, rid,
	+ msix_vec->res);
	+ msix_vec->res = NULL;
	+ }
	+}
	+
	+/**
	+ * i40iw_destroy_aeq - destroy aeq
	+ * @iwdev: iwarp device
	+ * @reset: TRUE if called before reset
	+ *
	+ * Issue a destroy aeq request and
	+ * free the resources associated with the aeq
	+ * The function is called during driver unload
	+ */
	+static void
	+i40iw_destroy_aeq(struct i40iw_device *iwdev)
	+{
	+ struct i40iw_aeq *aeq = &iwdev->aeq;
	+ struct i40iw_sc_dev *dev = &iwdev->sc_dev;
	+ enum i40iw_status_code status = I40IW_ERR_NOT_READY;
	+
	+ i40iw_debug(dev, I40IW_DEBUG_INIT, "%s: begin\n", __func__);
	+
	+ if (!iwdev->msix_shared) {
	+ i40iw_disable_irq(iwdev, iwdev->iw_msixtbl);
	+ taskqueue_drain(iwdev->irq_tq, &iwdev->irq_task);
	+ taskqueue_free(iwdev->irq_tq);
	+ }
	+ if (!dev->aeq_ops->aeq_destroy(&aeq->sc_aeq, 0, 1))
	+ status = dev->aeq_ops->aeq_destroy_done(&aeq->sc_aeq);
	+ if (status)
	+ i40iw_debug(dev, I40IW_DEBUG_AEQ, "destroy aeq failed %d\n", status);
	+
	+ ixliw_free_dma_mem(&aeq->mem);
	+}
	+
	+/**
	+ * i40iw_destroy_ceq - destroy ceq
	+ * @iwdev: iwarp device
	+ * @iwceq: ceq to be destroyed
	+ * @reset: TRUE if called before reset
	+ *
	+ * Issue a destroy ceq request and
	+ * free the resources associated with the ceq
	+ */
	+static void
	+i40iw_destroy_ceq(struct i40iw_device iwdev, struct i40iw_ceq iwceq)
	+{
	+ struct i40iw_sc_dev *dev = &iwdev->sc_dev;
	+ enum i40iw_status_code status;
	+
	+ INIT_DBG_DEV(iwdev->ldev->dev, "begin");
	+
	+ status = dev->ceq_ops->ceq_destroy(&iwceq->sc_ceq, 0, 1);
	+ if (status) {
	+ i40iw_debug(dev, I40IW_DEBUG_DEV,
	+ "ceq destroy command failed %d\n", status);
	+ goto exit;
	+ }
	+ status = dev->ceq_ops->cceq_destroy_done(&iwceq->sc_ceq);
	+ if (status)
	+ i40iw_debug(dev, I40IW_DEBUG_DEV,
	+ "ceq destroy completion failed %d\n", status);
	+
	+exit:
	+ ixliw_free_dma_mem(&iwceq->mem);
	+}
	+
	+/**
	+ * i40iw_del_ceqs - destroy all ceq's
	+ * @iwdev: iwarp device
	+ *
	+ * Go through all of the device ceq's and for each ceq
	+ * disable the ceq interrupt and destroy the ceq
	+ */
	+static void
	+i40iw_del_ceqs(struct i40iw_device *iwdev)
	+{
	+ struct i40iw_ceq *iwceq = iwdev->ceqlist;
	+ struct i40iw_msix_vector *msix_vec = iwdev->iw_msixtbl;
	+ u32 i = 0;
	+
	+ if (iwdev->msix_shared) {
	+ i40iw_disable_irq(iwdev, msix_vec);
	+ taskqueue_drain(iwdev->irq_tq, &iwdev->irq_task);
	+ taskqueue_free(iwdev->irq_tq);
	+ i40iw_destroy_ceq(iwdev, iwceq);
	+ iwceq++;
	+ i++;
	+ }
	+ for (msix_vec++; i < iwdev->ceqs_count; i++, msix_vec++, iwceq++) {
	+ i40iw_disable_irq(iwdev, msix_vec);
	+ taskqueue_drain(iwceq->irq_tq, &iwceq->irq_task);
	+ taskqueue_free(iwceq->irq_tq);
	+ i40iw_destroy_ceq(iwdev, iwceq);
	+ }
	+
	+ iwdev->sc_dev.ceq_valid = FALSE;
	+}
	+
	+/**
	+ * i40iw_destroy_ccq - destroy control cq
	+ * @iwdev: iwarp device
	+ *
	+ * Issue destroy ccq request and
	+ * free the resources associated with the ccq
	+ */
	+static void
	+i40iw_destroy_ccq(struct i40iw_device *iwdev)
	+{
	+ struct i40iw_ccq *ccq = &iwdev->ccq;
	+ struct i40iw_sc_dev *dev = &iwdev->sc_dev;
	+ enum i40iw_status_code status = 0;
	+
	+ status = dev->ccq_ops->ccq_destroy(dev->ccq, 0, TRUE);
	+ if (status)
	+ i40iw_debug(dev, I40IW_DEBUG_DEV,
	+ "ccq destroy failed %d\n", status);
	+ ixliw_free_dma_mem(&ccq->mem_cq);
	+}
	+
	+/* types of hmc objects */
	+static enum i40iw_hmc_rsrc_type iw_hmc_obj_types[] = {
	+ I40IW_HMC_IW_QP,
	+ I40IW_HMC_IW_CQ,
	+ I40IW_HMC_IW_HTE,
	+ I40IW_HMC_IW_ARP,
	+ I40IW_HMC_IW_APBVT_ENTRY,
	+ I40IW_HMC_IW_MR,
	+ I40IW_HMC_IW_XF,
	+ I40IW_HMC_IW_XFFL,
	+ I40IW_HMC_IW_Q1,
	+ I40IW_HMC_IW_Q1FL,
	+ I40IW_HMC_IW_TIMER,
	+};
	+
	+/**
	+ * i40iw_close_hmc_objects_type - delete hmc objects of a given type
	+ * @iwdev: iwarp device
	+ * @obj_type: the hmc object type to be deleted
	+ * @is_pf: TRUE if the function is PF otherwise FALSE
	+ * @reset: TRUE if called before reset
	+ */
	+static void
	+i40iw_close_hmc_objects_type(struct i40iw_sc_dev *dev,
	+ enum i40iw_hmc_rsrc_type obj_type, struct i40iw_hmc_info *hmc_info)
	+{
	+ struct i40iw_hmc_del_obj_info info;
	+
	+ memset(&info, 0, sizeof(info));
	+ info.hmc_info = hmc_info;
	+ info.rsrc_type = obj_type;
	+ info.count = hmc_info->hmc_obj[obj_type].cnt;
	+ info.is_pf = TRUE;
	+ if (dev->hmc_ops->del_hmc_object(dev, &info, FALSE))
	+ DPRINTF("iw_ixl: del obj of type %d failed\n", obj_type);
	+}
	+
	+/**
	+ * i40iw_del_hmc_objects - remove all device hmc objects
	+ * @dev: iwarp device
	+ * @hmc_info: hmc_info to free
	+ */
	+static void
	+i40iw_del_hmc_objects(struct i40iw_sc_dev *dev,
	+ struct i40iw_hmc_info *hmc_info)
	+{
	+ unsigned int i;
	+
	+ for (i = 0; i < IW_HMC_OBJ_TYPE_NUM; i++)
	+ i40iw_close_hmc_objects_type(dev, iw_hmc_obj_types[i], hmc_info);
	+}
	+
	+/**
	+ * i40iw_ceq_handler - interrupt handler for ceq
	+ * @data: ceq pointer
	+ */
	+static void
	+i40iw_ceq_handler(void *data)
	+{
	+ struct i40iw_ceq iwceq = (struct i40iw_ceq )data;
	+
	+ taskqueue_enqueue(iwceq->irq_tq, &iwceq->irq_task);
	+ return;
	+}
	+
	+/**
	+ * i40iw_create_hmc_obj_type - create hmc object of a given type
	+ * @dev: hardware control device structure
	+ * @info: information for the hmc object to create
	+ */
	+static enum i40iw_status_code
	+i40iw_create_hmc_obj_type(struct i40iw_sc_dev *dev,
	+ struct i40iw_hmc_create_obj_info *info)
	+{
	+ return dev->hmc_ops->create_hmc_object(dev, info);
	+}
	+
	+/**
	+ * i40iw_create_hmc_objs - create all hmc objects for the device
	+ * @iwdev: iwarp device
	+ * @is_pf: TRUE if the function is PF otherwise FALSE
	+ *
	+ * Create the device hmc objects and allocate hmc pages
	+ * Return 0 if successful, otherwise clean up and return error
	+ */
	+static enum i40iw_status_code
	+i40iw_create_hmc_objs(struct i40iw_device *iwdev, bool is_pf)
	+{
	+ struct i40iw_sc_dev *dev = &iwdev->sc_dev;
	+ struct i40iw_hmc_create_obj_info info;
	+ enum i40iw_status_code status;
	+ int i;
	+
	+ i40iw_debug(dev, I40IW_DEBUG_INIT, "%s: begin\n", __func__);
	+
	+ memset(&info, 0, sizeof(info));
	+ info.hmc_info = dev->hmc_info;
	+ info.is_pf = is_pf;
	+ info.entry_type = iwdev->sd_type;
	+ for (i = 0; i < IW_HMC_OBJ_TYPE_NUM; i++) {
	+ info.rsrc_type = iw_hmc_obj_types[i];
	+ info.count = dev->hmc_info->hmc_obj[info.rsrc_type].cnt;
	+ info.add_sd_cnt = 0;
	+ status = i40iw_create_hmc_obj_type(dev, &info);
	+ if (status) {
	+ i40iw_debug(dev, I40IW_DEBUG_HMC, "create obj type %d status = %d\n",
	+ iw_hmc_obj_types[i], status);
	+ break;
	+ }
	+ }
	+ if (!status)
	+ return (dev->cqp_misc_ops->static_hmc_pages_allocated(dev->cqp, 0,
	+ dev->hmc_fn_id,
	+ TRUE, true));
	+
	+ while (i) {
	+ i--;
	+ /* destroy the hmc objects of a given type */
	+ i40iw_close_hmc_objects_type(dev,
	+ iw_hmc_obj_types[i],
	+ dev->hmc_info);
	+ }
	+ return status;
	+}
	+
	+/**
	+ * i40iw_obj_aligned_mem - get aligned memory from device allocated memory
	+ * @iwdev: iwarp device
	+ * @memptr: points to the memory addresses
	+ * @size: size of memory needed
	+ * @mask: mask for the aligned memory
	+ *
	+ * Get aligned memory of the requested size and
	+ * update the memptr to point to the new aligned memory
	+ * Return 0 if successful, otherwise return no memory error
	+ */
	+enum i40iw_status_code
	+i40iw_obj_aligned_mem(struct i40iw_device iwdev, struct i40iw_dma_mem memptr,
	+ u32 size, u32 mask)
	+{
	+ uintptr_t va, newva;
	+ uintptr_t extra;
	+
	+ va = (uintptr_t)iwdev->obj_next.va;
	+ newva = va;
	+ if (mask)
	+ newva = ALIGN(va, (uintptr_t)(mask + 1));
	+ extra = newva - va;
	+ memptr->va = (u8 *)va + extra;
	+ memptr->pa = iwdev->obj_next.pa + extra;
	+ memptr->size = size;
	+ if (((u8 )memptr->va + size) > ((u8 )iwdev->obj_mem.va + iwdev->obj_mem.size))
	+ return I40IW_ERR_NO_MEMORY;
	+
	+ iwdev->obj_next.va = (u8 *)memptr->va + size;
	+ iwdev->obj_next.pa = memptr->pa + size;
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_create_ccq - create control cq
	+ * @iwdev: iwarp device
	+ *
	+ * Return 0, if the ccq and the resources associated with it
	+ * are successfully created, otherwise return error
	+ */
	+static enum i40iw_status_code
	+i40iw_create_ccq(struct i40iw_device *iwdev)
	+{
	+ struct i40iw_ccq *ccq = &iwdev->ccq;
	+ struct i40iw_sc_dev *dev = &iwdev->sc_dev;
	+ struct i40iw_ccq_init_info info;
	+ struct i40iw_dma_mem mem;
	+ enum i40iw_status_code status;
	+
	+ i40iw_debug(dev, I40IW_DEBUG_INIT, "%s: begin\n", __func__);
	+
	+ dev->ccq = &ccq->sc_cq;
	+ dev->ccq->dev = dev;
	+ ccq->shadow_area.size = sizeof(struct i40iw_cq_shadow_area);
	+ ccq->mem_cq.size = sizeof(struct i40iw_cqe) * IW_CCQ_SIZE;
	+ status = i40iw_allocate_dma_mem(dev->hw, &ccq->mem_cq,
	+ ccq->mem_cq.size, I40IW_CQ0_ALIGNMENT);
	+ if (status)
	+ goto exit;
	+ status = i40iw_obj_aligned_mem(iwdev, &mem, ccq->shadow_area.size,
	+ I40IW_SHADOWAREA_MASK);
	+ if (status)
	+ goto exit;
	+ ccq->sc_cq.back_cq = (void *)ccq;
	+ /* populate the ccq init info */
	+ memset(&info, 0, sizeof(info));
	+ info.dev = dev;
	+ info.cq_base = ccq->mem_cq.va;
	+ info.cq_pa = ccq->mem_cq.pa;
	+ info.num_elem = IW_CCQ_SIZE;
	+ info.shadow_area = mem.va;
	+ info.shadow_area_pa = mem.pa;
	+ info.ceqe_mask = FALSE;
	+ info.ceq_id_valid = TRUE;
	+ info.shadow_read_threshold = 16;
	+ status = dev->ccq_ops->ccq_init(dev->ccq, &info);
	+ if (!status)
	+ status = dev->ccq_ops->ccq_create(dev->ccq, 0, TRUE, true);
	+exit:
	+ if (status)
	+ ixliw_free_dma_mem(&ccq->mem_cq);
	+ return status;
	+}
	+
	+/**
	+ * i40iw_create_cqp - create control qp
	+ * @iwdev: iwarp device
	+ *
	+ * Return 0, if the cqp and all the resources associated with it
	+ * are successfully created, otherwise return error
	+ */
	+static enum i40iw_status_code
	+i40iw_create_cqp(struct i40iw_device *iwdev)
	+{
	+ struct i40iw_dma_mem mem;
	+ struct i40iw_sc_dev *dev = &iwdev->sc_dev;
	+ struct i40iw_cqp_init_info cqp_init_info;
	+ struct i40iw_cqp *cqp = &iwdev->cqp;
	+ int i;
	+ u32 sqsize = I40IW_CQP_SW_SQSIZE_2048;
	+ u16 maj_err, min_err;
	+ enum i40iw_status_code status;
	+
	+ i40iw_debug(dev, I40IW_DEBUG_INIT, "%s: begin\n", __func__);
	+
	+ cqp->cqp_requests = kcalloc(sqsize, sizeof(*cqp->cqp_requests), GFP_KERNEL);
	+ if (!cqp->cqp_requests)
	+ return I40IW_ERR_NO_MEMORY;
	+ cqp->scratch_array = kcalloc(sqsize, sizeof(*cqp->scratch_array), GFP_KERNEL);
	+ if (!cqp->scratch_array) {
	+ kfree(cqp->cqp_requests);
	+ return I40IW_ERR_NO_MEMORY;
	+ }
	+ dev->cqp = &cqp->sc_cqp;
	+ dev->cqp->dev = dev;
	+ status = i40iw_allocate_dma_mem(dev->hw, &cqp->sq,
	+ (sizeof(struct i40iw_cqp_sq_wqe) * sqsize), I40IW_CQP_ALIGNMENT);
	+ if (status)
	+ goto exit;
	+ status = i40iw_obj_aligned_mem(iwdev, &mem, sizeof(struct i40iw_cqp_ctx),
	+ I40IW_HOST_CTX_ALIGNMENT_MASK);
	+ if (status)
	+ goto exit;
	+ dev->cqp->host_ctx_pa = mem.pa;
	+ dev->cqp->host_ctx = mem.va;
	+ /* populate the cqp init info */
	+ memset(&cqp_init_info, 0, sizeof(cqp_init_info));
	+ cqp_init_info.dev = dev;
	+ cqp_init_info.sq_size = sqsize;
	+ cqp_init_info.sq = cqp->sq.va;
	+ cqp_init_info.sq_pa = cqp->sq.pa;
	+ cqp_init_info.host_ctx_pa = mem.pa;
	+ cqp_init_info.host_ctx = mem.va;
	+ cqp_init_info.hmc_profile = iwdev->resource_profile;
	+ cqp_init_info.enabled_vf_count = iwdev->max_rdma_vfs;
	+ cqp_init_info.scratch_array = cqp->scratch_array;
	+ status = dev->cqp_ops->cqp_init(dev->cqp, &cqp_init_info);
	+ if (status) {
	+ i40iw_debug(dev, I40IW_DEBUG_CQP, "cqp init status %d\n", status);
	+ goto exit;
	+ }
	+ status = dev->cqp_ops->cqp_create(dev->cqp, &maj_err, &min_err);
	+ if (status) {
	+ i40iw_debug(dev, I40IW_DEBUG_ERR \| I40IW_DEBUG_CQP,
	+ "cqp create status %d maj_err %d min_err %d\n", status,
	+ maj_err, min_err);
	+ goto exit;
	+ }
	+ spin_lock_init(&cqp->req_lock);
	+ INIT_LIST_HEAD(&cqp->cqp_avail_reqs);
	+ INIT_LIST_HEAD(&cqp->cqp_pending_reqs);
	+ /* init the waitq of the cqp_requests and add them to the list */
	+ for (i = 0; i < sqsize; i++) {
	+ init_waitqueue_head(&cqp->cqp_requests[i].waitq);
	+ list_add_tail(&cqp->cqp_requests[i].list, &cqp->cqp_avail_reqs);
	+ }
	+ return 0;
	+exit:
	+ /* clean up the created resources */
	+ i40iw_destroy_cqp(iwdev, FALSE);
	+ return status;
	+}
	+
	+/**
	+ * i40iw_configure_ceq_vector - set up the msix interrupt vector for ceq
	+ * @iwdev: iwarp device
	+ * @msix_vec: interrupt vector information
	+ * @iwceq: ceq associated with the vector
	+ * @ceq_id: the id number of the iwceq
	+ *
	+ * Allocate interrupt resources and enable irq handling
	+ * Return 0 if successful, otherwise return error
	+ */
	+static enum i40iw_status_code
	+i40iw_configure_ceq_vector(struct i40iw_device iwdev, struct i40iw_ceq iwceq,
	+ u32 ceq_id, struct i40iw_msix_vector *msix_vec)
	+{
	+ device_t dev = iwdev->ldev->dev;
	+ int err;
	+ int rid = msix_vec->idx + 1;
	+ enum i40iw_status_code status;
	+
	+ i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_INIT, "%s: begin\n", __func__);
	+
	+ if (ceq_id == 0) {
	+ msix_vec->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
	+ RF_SHAREABLE \| RF_ACTIVE);
	+ if (!msix_vec->res) {
	+ i40iw_debug(&iwdev->sc_dev,
	+ I40IW_DEBUG_ERR \| I40IW_DEBUG_DEV \| I40IW_DEBUG_AEQ,
	+ "Unable to allocate bus resource: aeq-ceq interrupt [%d]\n",
	+ rid);
	+ return I40IW_ERR_CONFIG;
	+ }
	+ err = bus_setup_intr(dev, msix_vec->res, INTR_TYPE_NET \| INTR_MPSAFE,
	+ NULL, i40iw_irq_handler, iwdev, &msix_vec->tag);
	+ if (err) {
	+ i40iw_debug(&iwdev->sc_dev,
	+ I40IW_DEBUG_ERR \| I40IW_DEBUG_DEV \| I40IW_DEBUG_AEQ,
	+ "Failed to register aeq-ceq handler\n");
	+ status = I40IW_ERR_CONFIG;
	+ goto fail_intr;
	+ }
	+ bus_describe_intr(dev, msix_vec->res, msix_vec->tag,
	+ "aeq-ceq");
	+
	+ TASK_INIT(&iwdev->irq_task, 0, i40iw_dpc, iwdev);
	+
	+ iwdev->irq_tq = taskqueue_create_fast("iw_ixl_aeq-ceq",
	+ M_NOWAIT, taskqueue_thread_enqueue, &iwdev->irq_tq);
	+ if (iwdev->irq_tq == NULL) {
	+ i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_ERR \| I40IW_DEBUG_DEV \| I40IW_DEBUG_AEQ,
	+ "Failed to create aeq-ceq taskqueue\n");
	+ status = I40IW_ERR_NO_MEMORY;
	+ goto fail_tq;
	+ }
	+ taskqueue_start_threads(&iwdev->irq_tq, 1, PI_NET,
	+ "%s iwarp aeq-ceq", device_get_nameunit(dev));
	+ } else {
	+ msix_vec->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
	+ RF_SHAREABLE \| RF_ACTIVE);
	+ if (!msix_vec->res) {
	+ i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_ERR \| I40IW_DEBUG_DEV,
	+ "Unable to allocate bus resource: ceq interrupt [%d]\n",
	+ rid);
	+ return I40IW_ERR_CONFIG;
	+ }
	+ err = bus_setup_intr(dev, msix_vec->res, INTR_TYPE_NET \| INTR_MPSAFE,
	+ NULL, i40iw_ceq_handler, iwceq, &msix_vec->tag);
	+ if (err) {
	+ i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_ERR \| I40IW_DEBUG_DEV,
	+ "Failed to register ceq handler\n");
	+ status = I40IW_ERR_CONFIG;
	+ goto fail_intr;
	+ }
	+ bus_describe_intr(dev, msix_vec->res, msix_vec->tag, "ceq");
	+
	+ TASK_INIT(&iwceq->irq_task, 0, i40iw_ceq_dpc, iwceq);
	+
	+ iwceq->irq_tq = taskqueue_create_fast("iw_ixl_ceq", M_NOWAIT,
	+ taskqueue_thread_enqueue, &iwceq->irq_tq);
	+ if (iwceq->irq_tq == NULL) {
	+ i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_ERR \| I40IW_DEBUG_DEV,
	+ "Failed to create ceq taskqueue\n");
	+ status = I40IW_ERR_NO_MEMORY;
	+ goto fail_tq;
	+ }
	+ taskqueue_start_threads(&iwceq->irq_tq, 1, PI_NET,
	+ "%s iwarp ceq-%d", device_get_nameunit(dev), ceq_id);
	+ }
	+ i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_INIT,
	+ "%s ceq_id=%d, msix_vec->idx=%d, cpu_id=%d\n", __func__, ceq_id,
	+ msix_vec->idx, msix_vec->cpu_affinity);
	+ bus_bind_intr(dev, msix_vec->res, msix_vec->cpu_affinity);
	+
	+ return 0;
	+
	+fail_tq:
	+ bus_teardown_intr(dev, msix_vec->res, msix_vec->tag);
	+ msix_vec->tag = NULL;
	+fail_intr:
	+ bus_release_resource(dev, SYS_RES_IRQ, rid, msix_vec->res);
	+ msix_vec->res = NULL;
	+ return (status);
	+}
	+
	+/**
	+ * i40iw_create_ceq - create completion event queue
	+ * @iwdev: iwarp device
	+ * @iwceq: pointer to the ceq resources to be created
	+ * @ceq_id: the id number of the iwceq
	+ *
	+ * Return 0, if the ceq and the resources associated with it
	+ * are successfully created, otherwise return error
	+ */
	+static enum i40iw_status_code
	+i40iw_create_ceq(struct i40iw_device iwdev, struct i40iw_ceq iwceq,
	+ u32 ceq_id)
	+{
	+ struct i40iw_sc_dev *dev = &iwdev->sc_dev;
	+ struct i40iw_ceq_init_info info;
	+ u64 scratch;
	+ enum i40iw_status_code status;
	+
	+ i40iw_debug(dev, I40IW_DEBUG_INIT, "%s: begin\n", __func__);
	+
	+ memset(&info, 0, sizeof(info));
	+ info.ceq_id = ceq_id;
	+ iwceq->iwdev = iwdev;
	+ iwceq->mem.size = sizeof(struct i40iw_ceqe) *
	+ iwdev->sc_dev.hmc_info->hmc_obj[I40IW_HMC_IW_CQ].cnt;
	+ status = i40iw_allocate_dma_mem(dev->hw, &iwceq->mem, iwceq->mem.size,
	+ I40IW_CEQ_ALIGNMENT);
	+ if (status)
	+ goto exit;
	+ info.ceq_id = ceq_id;
	+ info.ceqe_base = iwceq->mem.va;
	+ info.ceqe_pa = iwceq->mem.pa;
	+
	+ info.elem_cnt = iwdev->sc_dev.hmc_info->hmc_obj[I40IW_HMC_IW_CQ].cnt;
	+ iwceq->sc_ceq.ceq_id = ceq_id;
	+ info.dev = dev;
	+ scratch = (uintptr_t)&iwdev->cqp.sc_cqp;
	+ status = dev->ceq_ops->ceq_init(&iwceq->sc_ceq, &info);
	+ if (!status)
	+ status = dev->ceq_ops->cceq_create(&iwceq->sc_ceq, scratch);
	+
	+exit:
	+ if (status)
	+ ixliw_free_dma_mem(&iwceq->mem);
	+ return status;
	+}
	+
	+void
	+i40iw_request_reset(struct i40iw_device *iwdev)
	+{
	+ i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_ERR \| I40IW_DEBUG_INIT \|
	+ I40IW_DEBUG_CQP, "%s: begin\n", __func__);
	+
	+ ixl_iw_pf_reset(iwdev->hdl->ldev.handle);
	+}
	+
	+/**
	+ * i40iw_setup_ceqs - manage the device ceq's and their interrupt resources
	+ * @iwdev: iwarp device
	+ * @ldev: i40e lan device
	+ *
	+ * Allocate a list for all device completion event queues
	+ * Create the ceq's and configure their msix interrupt vectors
	+ * Return 0, if at least one ceq is successfully set up, otherwise return error
	+ */
	+static enum i40iw_status_code
	+i40iw_setup_ceqs(struct i40iw_device *iwdev)
	+{
	+ struct i40iw_ceq *iwceq;
	+ struct i40iw_msix_vector *msix_vec;
	+ enum i40iw_status_code status = 0;
	+ int err;
	+ u32 ceq_id;
	+
	+ i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_INIT, "%s: begin\n", __func__);
	+
	+ err = ixl_iw_pf_msix_init(iwdev->hdl->ldev.handle,
	+ &iwdev->msix_mapping);
	+ if (err) {
	+ status = I40IW_ERR_CONFIG;
	+ goto exit;
	+ }
	+ iwdev->ceqlist = kcalloc(iwdev->msix_count, sizeof(*iwdev->ceqlist), GFP_KERNEL);
	+ if (!iwdev->ceqlist) {
	+ status = I40IW_ERR_NO_MEMORY;
	+ goto exit;
	+ }
	+ for (ceq_id = 0; ceq_id < iwdev->msix_count; ceq_id++) {
	+ iwceq = &iwdev->ceqlist[ceq_id];
	+
	+ status = i40iw_create_ceq(iwdev, iwceq, ceq_id);
	+ if (status) {
	+ i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_DEV, "create ceq status = %d\n", status);
	+ break;
	+ }
	+ msix_vec = &iwdev->iw_msixtbl[ceq_id];
	+ iwceq->msix_idx = msix_vec->idx;
	+ status = i40iw_configure_ceq_vector(iwdev, iwceq, ceq_id, msix_vec);
	+ if (status) {
	+ i40iw_destroy_ceq(iwdev, iwceq);
	+ break;
	+ }
	+ i40iw_enable_intr(&iwdev->sc_dev, msix_vec->idx);
	+ iwdev->ceqs_count++;
	+ }
	+ i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_INIT \| I40IW_DEBUG_DEV,
	+ "%d MSIX interrupts used for this device\n", iwdev->ceqs_count);
	+exit:
	+ if (status && !iwdev->ceqs_count) {
	+ kfree(iwdev->ceqlist);
	+ iwdev->ceqlist = NULL;
	+ return status;
	+ } else {
	+ iwdev->sc_dev.ceq_valid = TRUE;
	+ return 0;
	+ }
	+}
	+
	+/**
	+ * i40iw_create_aeq - create async event queue
	+ * @iwdev: iwarp device
	+ *
	+ * Return 0, if the aeq and the resources associated with it
	+ * are successfully created, otherwise return error
	+ */
	+static enum i40iw_status_code
	+i40iw_create_aeq(struct i40iw_device *iwdev)
	+{
	+ struct i40iw_aeq *aeq = &iwdev->aeq;
	+ struct i40iw_sc_dev *dev = &iwdev->sc_dev;
	+ struct i40iw_aeq_init_info info;
	+ u64 scratch = 0;
	+ u32 aeq_size;
	+ enum i40iw_status_code status;
	+
	+ i40iw_debug(dev, I40IW_DEBUG_INIT, "%s: begin\n", __func__);
	+
	+ aeq_size = 2 * iwdev->sc_dev.hmc_info->hmc_obj[I40IW_HMC_IW_QP].cnt +
	+ iwdev->sc_dev.hmc_info->hmc_obj[I40IW_HMC_IW_CQ].cnt;
	+ aeq->mem.size = sizeof(struct i40iw_sc_aeqe) * aeq_size;
	+ status = i40iw_allocate_dma_mem(dev->hw, &aeq->mem, aeq->mem.size,
	+ I40IW_AEQ_ALIGNMENT);
	+ if (status)
	+ goto exit;
	+
	+ memset(&info, 0, sizeof(info));
	+ info.aeqe_base = aeq->mem.va;
	+ info.aeq_elem_pa = aeq->mem.pa;
	+ info.elem_cnt = aeq_size;
	+ info.dev = dev;
	+ status = dev->aeq_ops->aeq_init(&aeq->sc_aeq, &info);
	+ if (status)
	+ goto exit;
	+ status = dev->aeq_ops->aeq_create(&aeq->sc_aeq, scratch, 1);
	+ if (!status)
	+ status = dev->aeq_ops->aeq_create_done(&aeq->sc_aeq);
	+exit:
	+ if (status)
	+ ixliw_free_dma_mem(&aeq->mem);
	+ return status;
	+}
	+
	+/**
	+ * i40iw_setup_aeq - set up the device aeq
	+ * @iwdev: iwarp device
	+ *
	+ * Create the aeq and configure its msix interrupt vector
	+ * Return 0 if successful, otherwise return error
	+ */
	+static enum i40iw_status_code
	+i40iw_setup_aeq(struct i40iw_device *iwdev)
	+{
	+ enum i40iw_status_code status;
	+
	+ i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_INIT, "%s: begin\n", __func__);
	+
	+ status = i40iw_create_aeq(iwdev);
	+ if (status)
	+ return status;
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_initialize_ilq - create iwarp local queue for cm
	+ * @iwdev: iwarp device
	+ *
	+ * Return 0 if successful, otherwise return error
	+ */
	+static enum i40iw_status_code
	+i40iw_initialize_ilq(struct i40iw_device *iwdev)
	+{
	+ struct i40iw_puda_rsrc_info info;
	+ enum i40iw_status_code status;
	+
	+ i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_INIT, "%s: begin\n", __func__);
	+
	+ memset(&info, 0, sizeof(info));
	+ info.type = I40IW_PUDA_RSRC_TYPE_ILQ;
	+ info.cq_id = 1;
	+ info.qp_id = 0;
	+ info.count = 1;
	+ info.pd_id = 1;
	+ info.sq_size = 8192;
	+ info.rq_size = 8192;
	+ info.buf_size = 1024;
	+ info.tx_buf_cnt = 16384;
	+
	+ info.receive = i40iw_receive_ilq;
	+ info.xmit_complete = i40iw_free_sqbuf;
	+ info.mss = iwdev->vsi.mtu;
	+
	+ status = i40iw_puda_create_rsrc(&iwdev->vsi, &info);
	+
	+ if (status)
	+ i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_ILQ, "ilq create fail\n");
	+ return status;
	+}
	+
	+/**
	+ * i40iw_initialize_ieq - create iwarp exception queue
	+ * @iwdev: iwarp device
	+ *
	+ * Return 0 if successful, otherwise return error
	+ */
	+static enum i40iw_status_code
	+i40iw_initialize_ieq(struct i40iw_device *iwdev)
	+{
	+ struct i40iw_puda_rsrc_info info;
	+ enum i40iw_status_code status;
	+
	+ i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_INIT, "%s: begin\n", __func__);
	+
	+ memset(&info, 0, sizeof(info));
	+ info.type = I40IW_PUDA_RSRC_TYPE_IEQ;
	+ info.cq_id = 2;
	+ info.qp_id = iwdev->vsi.exception_lan_queue;
	+ info.count = 1;
	+ info.pd_id = 2;
	+ info.sq_size = 8192;
	+ info.rq_size = 8192;
	+ info.buf_size = iwdev->vsi.mtu + VLAN_ETH_HLEN;
	+
	+ info.tx_buf_cnt = 4096;
	+ status = i40iw_puda_create_rsrc(&iwdev->vsi, &info);
	+ if (status)
	+ i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_ERR \| I40IW_DEBUG_IEQ,
	+ "ieq create fail\n");
	+ return status;
	+}
	+
	+/**
	+ * i40iw_reinitialize_ieq - destroy and re-create ieq
	+ * @dev: iwarp device
	+ */
	+void
	+i40iw_reinitialize_ieq(struct i40iw_sc_dev *dev)
	+{
	+ struct i40iw_device iwdev = (struct i40iw_device )dev->back_dev;
	+
	+ i40iw_puda_dele_resources(&iwdev->vsi, I40IW_PUDA_RSRC_TYPE_IEQ, FALSE);
	+ if (i40iw_initialize_ieq(iwdev)) {
	+ iwdev->reset = TRUE;
	+ i40iw_request_reset(iwdev);
	+ }
	+}
	+
	+/**
	+ * i40iw_hmc_setup - create hmc objects for the device
	+ * @iwdev: iwarp device
	+ *
	+ * Set up the device private memory space for the number and size of
	+ * the hmc objects and create the objects
	+ * Return 0 if successful, otherwise return error
	+ */
	+static enum i40iw_status_code
	+i40iw_hmc_setup(struct i40iw_device *iwdev)
	+{
	+ enum i40iw_status_code status;
	+
	+ i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_INIT, "%s: begin\n", __func__);
	+
	+ iwdev->sd_type = I40IW_SD_TYPE_DIRECT;
	+ status = i40iw_config_fpm_values(&iwdev->sc_dev, IW_CFG_FPM_QP_COUNT);
	+ if (status)
	+ goto exit;
	+ status = i40iw_create_hmc_objs(iwdev, TRUE);
	+ if (status)
	+ goto exit;
	+ iwdev->init_state = HMC_OBJS_CREATED;
	+exit:
	+ return status;
	+}
	+
	+/**
	+ * i40iw_del_init_mem - deallocate memory resources
	+ * @iwdev: iwarp device
	+ */
	+static void
	+i40iw_del_init_mem(struct i40iw_device *iwdev)
	+{
	+ struct i40iw_sc_dev *dev = &iwdev->sc_dev;
	+
	+ ixliw_free_dma_mem(&iwdev->obj_mem);
	+ kfree(dev->hmc_info->sd_table.sd_entry);
	+ dev->hmc_info->sd_table.sd_entry = NULL;
	+ kfree(iwdev->mem_resources);
	+ iwdev->mem_resources = NULL;
	+ kfree(iwdev->ceqlist);
	+ iwdev->ceqlist = NULL;
	+ kfree(iwdev->iw_msixtbl);
	+ iwdev->iw_msixtbl = NULL;
	+ kfree(iwdev->hmc_info_mem);
	+ iwdev->hmc_info_mem = NULL;
	+}
	+
	+/**
	+ * i40iw_del_macip_entry - remove a mac ip address entry from the hw table
	+ * @iwdev: iwarp device
	+ * @idx: the index of the mac ip address to delete
	+ */
	+static void
	+i40iw_del_macip_entry(struct i40iw_device *iwdev, u8 idx)
	+{
	+ struct i40iw_cqp *iwcqp = &iwdev->cqp;
	+ struct i40iw_cqp_request *cqp_request;
	+ struct cqp_commands_info *cqp_info;
	+ enum i40iw_status_code status = 0;
	+
	+ i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_INIT, "%s: begin\n", __func__);
	+
	+ cqp_request = i40iw_get_cqp_request(iwcqp, TRUE);
	+ if (!cqp_request) {
	+ i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_ERR \| I40IW_DEBUG_CQP, "cqp_request memory failed\n");
	+ return;
	+ }
	+ cqp_info = &cqp_request->info;
	+ cqp_info->cqp_cmd = OP_DELETE_LOCAL_MAC_IPADDR_ENTRY;
	+ cqp_info->post_sq = 1;
	+ cqp_info->in.u.del_local_mac_ipaddr_entry.cqp = &iwcqp->sc_cqp;
	+ cqp_info->in.u.del_local_mac_ipaddr_entry.scratch = (uintptr_t)cqp_request;
	+ cqp_info->in.u.del_local_mac_ipaddr_entry.entry_idx = idx;
	+ cqp_info->in.u.del_local_mac_ipaddr_entry.ignore_ref_count = 0;
	+ status = i40iw_handle_cqp_op(iwdev, cqp_request);
	+ if (status)
	+ i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_CQP, "CQP-OP Del MAC Ip entry fail");
	+}
	+
	+/**
	+ * i40iw_add_mac_ipaddr_entry - add a mac ip address entry to the hw table
	+ * @iwdev: iwarp device
	+ * @mac_addr: pointer to mac address
	+ * @idx: the index of the mac ip address to add
	+ */
	+static enum i40iw_status_code
	+i40iw_add_mac_ipaddr_entry(struct i40iw_device iwdev, u8 mac_addr, u8 idx)
	+{
	+ struct i40iw_local_mac_ipaddr_entry_info *info;
	+ struct i40iw_cqp *iwcqp = &iwdev->cqp;
	+ struct i40iw_cqp_request *cqp_request;
	+ struct cqp_commands_info *cqp_info;
	+ enum i40iw_status_code status = 0;
	+
	+ i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_INIT, "%s: begin\n", __func__);
	+
	+ cqp_request = i40iw_get_cqp_request(iwcqp, TRUE);
	+ if (!cqp_request) {
	+ i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_ERR \| I40IW_DEBUG_CQP, "cqp_request memory failed\n");
	+ return I40IW_ERR_NO_MEMORY;
	+ }
	+ cqp_info = &cqp_request->info;
	+
	+ cqp_info->post_sq = 1;
	+ info = &cqp_info->in.u.add_local_mac_ipaddr_entry.info;
	+ ether_addr_copy(info->mac_addr, mac_addr);
	+ info->entry_idx = idx;
	+ cqp_info->in.u.add_local_mac_ipaddr_entry.scratch = (uintptr_t)cqp_request;
	+ cqp_info->cqp_cmd = OP_ADD_LOCAL_MAC_IPADDR_ENTRY;
	+ cqp_info->in.u.add_local_mac_ipaddr_entry.cqp = &iwcqp->sc_cqp;
	+ cqp_info->in.u.add_local_mac_ipaddr_entry.scratch = (uintptr_t)cqp_request;
	+ status = i40iw_handle_cqp_op(iwdev, cqp_request);
	+ if (status)
	+ i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_CQP, "CQP-OP Add MAC Ip entry fail");
	+ return status;
	+}
	+
	+/**
	+ * i40iw_alloc_local_mac_ipaddr_entry - allocate a mac ip address entry
	+ * @iwdev: iwarp device
	+ * @mac_ip_tbl_idx: the index of the new mac ip address
	+ *
	+ * Allocate a mac ip address entry and update the mac_ip_tbl_idx
	+ * to hold the index of the newly created mac ip address
	+ * Return 0 if successful, otherwise return error
	+ */
	+static enum i40iw_status_code
	+i40iw_alloc_local_mac_ipaddr_entry(struct i40iw_device *iwdev,
	+ u16 *mac_ip_tbl_idx)
	+{
	+ struct i40iw_cqp *iwcqp = &iwdev->cqp;
	+ struct i40iw_cqp_request *cqp_request;
	+ struct cqp_commands_info *cqp_info;
	+ enum i40iw_status_code status = 0;
	+
	+ i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_INIT, "%s: begin\n", __func__);
	+
	+ cqp_request = i40iw_get_cqp_request(iwcqp, TRUE);
	+ if (!cqp_request) {
	+ i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_CQP, "cqp_request memory failed\n");
	+ return I40IW_ERR_NO_MEMORY;
	+ }
	+ /* increment refcount, because we need the cqp request ret value */
	+ atomic_inc(&cqp_request->refcount);
	+
	+ cqp_info = &cqp_request->info;
	+ cqp_info->cqp_cmd = OP_ALLOC_LOCAL_MAC_IPADDR_ENTRY;
	+ cqp_info->post_sq = 1;
	+ cqp_info->in.u.alloc_local_mac_ipaddr_entry.cqp = &iwcqp->sc_cqp;
	+ cqp_info->in.u.alloc_local_mac_ipaddr_entry.scratch = (uintptr_t)cqp_request;
	+
	+ status = i40iw_handle_cqp_op(iwdev, cqp_request);
	+ if (!status)
	+ *mac_ip_tbl_idx = cqp_request->compl_info.op_ret_val;
	+ else
	+ i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_CQP, "CQP-OP Alloc MAC Ip entry fail");
	+
	+ /* decrement refcount and free the cqp request, if no longer used */
	+ i40iw_put_cqp_request(iwcqp, cqp_request);
	+ return status;
	+}
	+
	+/**
	+ * i40iw_alloc_set_mac_ipaddr - set up a mac ip address table entry
	+ * @iwdev: iwarp device
	+ * @macaddr: pointer to mac address
	+ *
	+ * Allocate a mac ip address entry and add it to the hw table
	+ * Return 0 if successful, otherwise return error
	+ */
	+static enum i40iw_status_code
	+i40iw_alloc_set_mac_ipaddr(struct i40iw_device iwdev, u8 macaddr)
	+{
	+ enum i40iw_status_code status;
	+
	+ i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_INIT, "%s: begin\n", __func__);
	+
	+ status = i40iw_alloc_local_mac_ipaddr_entry(iwdev, &iwdev->mac_ip_table_idx);
	+ if (!status) {
	+ status = i40iw_add_mac_ipaddr_entry(iwdev, macaddr,
	+ (u8)iwdev->mac_ip_table_idx);
	+ if (status)
	+ i40iw_del_macip_entry(iwdev, (u8)iwdev->mac_ip_table_idx);
	+ }
	+ return status;
	+}
	+
	+/**
	+ * i40iw_add_ipv6_addr - add ipv6 address to the hw arp table
	+ * @iwdev: iwarp device
	+ */
	+static void
	+i40iw_add_ipv6_addr(struct i40iw_device *iwdev)
	+{
	+ struct ifnet *ifp = iwdev->ifp;
	+ struct ifaddr ifa, tmp;
	+ struct sockaddr_in6 *sin6;
	+ u32 local_ipaddr6[4];
	+ u8 *mac_addr;
	+ char ip6buf[INET6_ADDRSTRLEN];
	+
	+ i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_INIT, "%s: begin\n", __func__);
	+
	+ if_addr_rlock(ifp);
	+ TAILQ_FOREACH_SAFE(ifa, &ifp->if_addrhead, ifa_link, tmp) {
	+ sin6 = (struct sockaddr_in6 *)ifa->ifa_addr;
	+ if (sin6->sin6_family != AF_INET6)
	+ continue;
	+
	+ i40iw_copy_ip_ntohl(local_ipaddr6, (u32 *)&sin6->sin6_addr);
	+ mac_addr = IF_LLADDR(ifp);
	+
	+ i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_CM,
	+ "IP=%s, MAC=%02x:%02x:%02x:%02x:%02x:%02x\n",
	+ ip6_sprintf(ip6buf, &sin6->sin6_addr),
	+ mac_addr[0], mac_addr[1], mac_addr[2],
	+ mac_addr[3], mac_addr[4], mac_addr[5]);
	+
	+ i40iw_manage_arp_cache(iwdev, mac_addr, local_ipaddr6, FALSE,
	+ I40IW_ARP_ADD);
	+
	+ }
	+ if_addr_runlock(ifp);
	+}
	+
	+/**
	+ * i40iw_add_ipv4_addr - add ipv4 address to the hw arp table
	+ * @iwdev: iwarp device
	+ */
	+static void
	+i40iw_add_ipv4_addr(struct i40iw_device *iwdev)
	+{
	+ struct ifnet *ifp = iwdev->ifp;
	+ struct ifaddr *ifa;
	+ struct sockaddr_in *sin;
	+ in_addr_t ip_addr;
	+ u8 *mac_addr;
	+
	+ i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_INIT, "%s: begin\n", __func__);
	+
	+ if_addr_rlock(ifp);
	+ TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
	+ sin = (struct sockaddr_in *)ifa->ifa_addr;
	+ if (sin->sin_family != AF_INET)
	+ continue;
	+
	+ ip_addr = ntohl(sin->sin_addr.s_addr);
	+ mac_addr = IF_LLADDR(ifp);
	+
	+ i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_CM,
	+ "IP=%d.%d.%d.%d, MAC=%02x:%02x:%02x:%02x:%02x:%02x\n",
	+ ip_addr >> 24,
	+ (ip_addr >> 16) & 0xFF,
	+ (ip_addr >> 8) & 0xFF,
	+ ip_addr & 0xFF,
	+ mac_addr[0], mac_addr[1], mac_addr[2],
	+ mac_addr[3], mac_addr[4], mac_addr[5]);
	+
	+ i40iw_manage_arp_cache(iwdev, mac_addr, &ip_addr, TRUE,
	+ I40IW_ARP_ADD);
	+ }
	+ if_addr_runlock(ifp);
	+}
	+
	+/**
	+ * i40iw_add_mac_ip - add mac and ip addresses
	+ * @iwdev: iwarp device
	+ *
	+ * Create and add a mac ip address entry to the hw table and
	+ * ipv4/ipv6 addresses to the arp cache
	+ * Return 0 if successful, otherwise return error
	+ */
	+static enum i40iw_status_code
	+i40iw_add_mac_ip(struct i40iw_device *iwdev)
	+{
	+ struct ifnet *ifp = iwdev->ifp;
	+ enum i40iw_status_code status;
	+
	+ i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_INIT, "%s: begin\n", __func__);
	+
	+ status = i40iw_alloc_set_mac_ipaddr(iwdev, IF_LLADDR(ifp));
	+ if (status)
	+ return status;
	+
	+ i40iw_add_ipv4_addr(iwdev);
	+ i40iw_add_ipv6_addr(iwdev);
	+
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_wait_pe_ready - Check if firmware is ready
	+ * @hw: provides access to registers
	+ */
	+static void
	+i40iw_wait_pe_ready(struct i40iw_hw *hw)
	+{
	+ u32 statuscpu0;
	+ u32 statuscpu1;
	+ u32 statuscpu2;
	+ u32 retrycount = 0;
	+
	+ INIT_DEBUGOUT("begin");
	+
	+ do {
	+ statuscpu0 = rd32(hw, I40E_GLPE_CPUSTATUS0);
	+ HW_DEBUGOUT("CSR_CQP status[x%04X]", statuscpu0);
	+ statuscpu1 = rd32(hw, I40E_GLPE_CPUSTATUS1);
	+ HW_DEBUGOUT("I40E_GLPE_CPUSTATUS1 status[x%04X]",
	+ statuscpu1);
	+ statuscpu2 = rd32(hw, I40E_GLPE_CPUSTATUS2);
	+ HW_DEBUGOUT("I40E_GLPE_CPUSTATUS2 status[x%04X]",
	+ statuscpu2);
	+ if ((statuscpu0 == 0x80) && (statuscpu1 == 0x80) &&
	+ (statuscpu2 == 0x80))
	+ break; /* SUCCESS */
	+ mdelay(1000);
	+ retrycount++;
	+ } while (retrycount < 14);
	+}
	+
	+/**
	+ * i40iw_initialize_dev - initialize device
	+ * @iwdev: iwarp device
	+ * @pf_info: lan device information
	+ *
	+ * Allocate memory for the hmc objects and initialize iwdev
	+ * Return 0 if successful, otherwise clean up the resources
	+ * and return error
	+ */
	+static enum i40iw_status_code
	+i40iw_initialize_dev(struct i40iw_device iwdev, struct ixl_iw_pf pf_info)
	+{
	+ struct i40iw_sc_dev *dev = &iwdev->sc_dev;
	+ struct i40iw_device_init_info info;
	+ struct i40iw_l2params l2params;
	+ struct i40iw_dma_mem mem;
	+ struct i40iw_vsi_stats_info stats_info;
	+ struct i40iw_vsi_init_info vsi_info;
	+ u32 i;
	+ u32 size;
	+ u16 qset;
	+ enum i40iw_status_code status;
	+
	+ i40iw_debug(dev, I40IW_DEBUG_INIT, "%s: begin\n", __func__);
	+
	+ memset(&l2params, 0, sizeof(l2params));
	+ memset(&info, 0, sizeof(info));
	+ size = sizeof(struct i40iw_hmc_pble_rsrc) + sizeof(struct i40iw_hmc_info) +
	+ (sizeof(struct i40iw_hmc_obj_info) * I40IW_HMC_IW_MAX);
	+ iwdev->hmc_info_mem = kzalloc(size, GFP_KERNEL);
	+ if (!iwdev->hmc_info_mem) {
	+ i40iw_debug(dev, I40IW_DEBUG_ERR \| I40IW_DEBUG_INIT,
	+ "%s: memory alloc fail\n", __func__);
	+ return I40IW_ERR_NO_MEMORY;
	+ }
	+ iwdev->pble_rsrc = (struct i40iw_hmc_pble_rsrc *)iwdev->hmc_info_mem;
	+ dev->hmc_info = &iwdev->hw.hmc;
	+ dev->hmc_info->hmc_obj = (struct i40iw_hmc_obj_info *)(iwdev->pble_rsrc + 1);
	+ status = i40iw_obj_aligned_mem(iwdev, &mem, I40IW_QUERY_FPM_BUF_SIZE,
	+ I40IW_FPM_QUERY_BUF_ALIGNMENT_MASK);
	+ if (status)
	+ goto exit;
	+ info.fpm_query_buf_pa = mem.pa;
	+ info.fpm_query_buf = mem.va;
	+ status = i40iw_obj_aligned_mem(iwdev, &mem, I40IW_COMMIT_FPM_BUF_SIZE,
	+ I40IW_FPM_COMMIT_BUF_ALIGNMENT_MASK);
	+
	+ if (status)
	+ goto exit;
	+ info.fpm_commit_buf_pa = mem.pa;
	+ info.fpm_commit_buf = mem.va;
	+ info.hmc_fn_id = pf_info->pf_id;
	+ info.is_pf = TRUE;
	+ info.bar0 = iwdev->hw.hw_addr;
	+ info.hw = &iwdev->hw;
	+ info.debug_mask = ixliw_debug;
	+ l2params.mtu = (pf_info->mtu) ? pf_info->mtu : iwdev->vsi.mtu;
	+ for (i = 0; i < IXL_IW_MAX_USER_PRIORITY; i++) {
	+ qset = pf_info->qs_handle[i];
	+ l2params.qs_handle_list[i] = qset;
	+ }
	+
	+ info.vchnl_send = NULL; /* virtual channel not supported */
	+
	+ memset(&vsi_info, 0, sizeof(vsi_info));
	+ vsi_info.dev = &iwdev->sc_dev;
	+ vsi_info.back_vsi = (void *)iwdev;
	+ vsi_info.params = &l2params;
	+ vsi_info.exception_lan_queue = 1;
	+ i40iw_sc_vsi_init(&iwdev->vsi, &vsi_info);
	+
	+ memset(&stats_info, 0, sizeof(stats_info));
	+ stats_info.fcn_id = iwdev->ldev->pf_id;
	+ stats_info.pestat = kzalloc(sizeof(*stats_info.pestat), GFP_KERNEL);
	+ if (!stats_info.pestat) {
	+ i40iw_pr_err("memory alloc fail\n");
	+ status = I40IW_ERR_NO_MEMORY;
	+ goto exit;
	+ }
	+ status = i40iw_device_init(&iwdev->sc_dev, &info);
	+exit:
	+ if (status) {
	+ kfree(iwdev->hmc_info_mem);
	+ iwdev->hmc_info_mem = NULL;
	+ }
	+ return status;
	+}
	+
	+/**
	+ * i40iw_save_msix_info - copy msix vector information to iwarp device
	+ * @iwdev: iwarp device
	+ * @pf_info: lan device information
	+ *
	+ * Allocate iwdev msix table and copy the pf_info msix info to the table
	+ * Return 0 if successful, otherwise return error
	+ */
	+static enum i40iw_status_code
	+i40iw_save_msix_info(struct i40iw_device iwdev, struct ixl_iw_pf pf_info)
	+{
	+ i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_INIT, "%s: begin\n", __func__);
	+
	+ iwdev->msix_info.base = pf_info->iw_msix.base;
	+ iwdev->msix_info.count = pf_info->iw_msix.count;
	+ iwdev->msix_shared = TRUE;
	+
	+ iwdev->msix_mapping.itr_indx = 3;
	+ iwdev->msix_mapping.aeq_vector = pf_info->iw_msix.base;
	+
	+ return 0;
	+}
	+
	+static int
	+ixliw_get_cpu_cnt(struct i40iw_device *iwdev)
	+{
	+#if __FreeBSD_version >= 1100000
	+ device_t dev = iwdev->ldev->dev;
	+ cpuset_t cpu_set;
	+ int cpu_cnt;
	+
	+ if (bus_get_cpus(dev, INTR_CPUS, sizeof(cpu_set), &cpu_set)) {
	+ cpu_cnt = mp_ncpus;
	+ } else {
	+ cpu_cnt = CPU_COUNT(&cpu_set);
	+ }
	+#else
	+ int cpu_cnt = mp_ncpus;
	+#endif
	+
	+ return cpu_cnt;
	+}
	+/**
	+ * ixliw_get_int_cpu - determine which cpu core to use for msix vector,
	+ * uses bus_get_cpus on FreeBSD11 or newer
	+ * @iwdev: iwarp device
	+ * @msix_taken: which intr cpu to use
	+ * returns id of intr cpu that will be used, or msix_taken for FBSD10.3
	+ */
	+static int
	+ixliw_get_int_cpu(struct i40iw_device *iwdev, int msix_taken)
	+{
	+ int result = -1;
	+#if __FreeBSD_version >= 1100000
	+ device_t dev = iwdev->ldev->dev;
	+ cpuset_t cpu_set;
	+ u64 i = 0;
	+ int intr_cpu_cnt = -1;
	+
	+ if (!bus_get_cpus(dev, INTR_CPUS, sizeof(cpu_set), &cpu_set)) {
	+ while (intr_cpu_cnt < msix_taken) {
	+ if ((cpu_set.__bits[i / 64] & (((u64)1) << (i % 64)))) {
	+ ++intr_cpu_cnt;
	+ if (intr_cpu_cnt == msix_taken)
	+ break;
	+ }
	+ ++i;
	+ if (i >= mp_ncpus)
	+ i = i % mp_ncpus;
	+ }
	+ result = i;
	+ }
	+#endif
	+ /*
	+ * if FBSD10 or bus_get_cpus returns error
	+ */
	+ if (result < 0) {
	+ /*
	+ * ensure non-negative value will be returned
	+ */
	+ if (iwdev->msix_count <= mp_ncpus / 2)
	+ /* get next even id */
	+ result = max(((msix_taken + 1) & 0xFFFFFFFE) % mp_ncpus, 0);
	+ else
	+ result = max((msix_taken) % mp_ncpus, 0);
	+ }
	+ return result;
	+}
	+
	+/**
	+ * ixliw_setup_msix - prepare msix vector infrastructure
	+ * @iwdev: iwarp device
	+ *
	+ * Determine the number of interrupt vectors, alloc structures - iw_msixtbl
	+ * and ceq_vectors
	+ *
	+ */
	+static int
	+ixliw_setup_msix(struct i40iw_device *iwdev)
	+{
	+ int cpu_id;
	+ int inc_cpuid = 1;
	+ int vector;
	+ u32 i;
	+ u32 size;
	+
	+ iwdev->msix_count = min(iwdev->msix_info.count, iwdev->sc_dev.hmc_fpm_misc.max_ceqs);
	+ iwdev->msix_count = min(iwdev->msix_count, ixliw_get_cpu_cnt(iwdev));
	+ /*
	+ * if ixliw_max_ceq is different than 0, add upper constraint for the
	+ * number of ceq.
	+ */
	+ if (ixliw_max_ceq)
	+ iwdev->msix_count = min(iwdev->msix_count, ixliw_max_ceq);
	+
	+ size = sizeof(struct i40iw_msix_vector) * iwdev->msix_count;
	+ iwdev->iw_msixtbl = kzalloc(size, GFP_KERNEL);
	+ if (!iwdev->iw_msixtbl)
	+ return I40IW_ERR_NO_MEMORY;
	+
	+ iwdev->msix_mapping.ceq_cnt = iwdev->msix_count;
	+ size = sizeof(int) * iwdev->msix_mapping.ceq_cnt;
	+ iwdev->msix_mapping.ceq_vector = kzalloc(size, GFP_KERNEL);
	+ if (!iwdev->msix_mapping.ceq_vector) {
	+ kfree(iwdev->iw_msixtbl);
	+ return I40IW_ERR_NO_MEMORY;
	+ }
	+ /*
	+ * if there is less vectors than half of cpus the chances are we will
	+ * be able to assign them onto physical cores only
	+ */
	+ if (iwdev->msix_count < ixliw_get_cpu_cnt(iwdev) / 2)
	+ inc_cpuid = 2;
	+ else
	+ inc_cpuid = 1;
	+
	+ vector = cpu_id = iwdev->msix_info.base;
	+ /* avoid ceq_vector[0] being asigned to the same core on every port */
	+ cpu_id += (iwdev->ldev->pf_id) * inc_cpuid;
	+ for (i = 0; i < iwdev->msix_mapping.ceq_cnt; cpu_id += inc_cpuid, i++) {
	+ iwdev->msix_mapping.ceq_vector[i] = vector;
	+ iwdev->iw_msixtbl[i].idx = vector++;
	+ iwdev->iw_msixtbl[i].cpu_affinity = ixliw_get_int_cpu(iwdev, cpu_id);
	+ }
	+
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_deinit_device - clean up the device resources
	+ * @iwdev: iwarp device
	+ *
	+ * Destroy the ib device interface, remove the mac ip entry and ipv4/ipv6
	+ * addresses, destroy the device queues and free the pble and the hmc objects
	+ */
	+static void
	+i40iw_deinit_device(struct i40iw_device *iwdev)
	+{
	+ struct i40iw_sc_dev *dev = &iwdev->sc_dev;
	+
	+ i40iw_debug(dev, I40IW_DEBUG_INIT, "state = %d", iwdev->init_state);
	+
	+ switch (iwdev->init_state) {
	+ case RDMA_DEV_REGISTERED:
	+ iwdev->iw_status = 0;
	+ i40iw_port_ibevent(iwdev);
	+ i40iw_destroy_rdma_device(iwdev->iwibdev);
	+ /* fallthrough */
	+ case IP_ADDR_REGISTERED:
	+ i40iw_del_macip_entry(iwdev, (u8)iwdev->mac_ip_table_idx);
	+ /* fallthrough */
	+ case PBLE_CHUNK_MEM:
	+ i40iw_destroy_pble_pool(dev, iwdev->pble_rsrc);
	+ /* fallthrough */
	+ case CEQ_CREATED:
	+ i40iw_del_ceqs(iwdev);
	+ /* fallthrough */
	+ case AEQ_CREATED:
	+ i40iw_destroy_aeq(iwdev);
	+ /* fallthrough */
	+ case IEQ_CREATED:
	+ i40iw_puda_dele_resources(&iwdev->vsi, I40IW_PUDA_RSRC_TYPE_IEQ, FALSE);
	+ /* fallthrough */
	+ case ILQ_CREATED:
	+ i40iw_puda_dele_resources(&iwdev->vsi, I40IW_PUDA_RSRC_TYPE_ILQ, FALSE);
	+ /* fallthrough */
	+ case CCQ_CREATED:
	+ i40iw_destroy_ccq(iwdev);
	+ /* fallthrough */
	+ case HMC_OBJS_CREATED:
	+ i40iw_del_hmc_objects(dev, dev->hmc_info);
	+ /* fallthrough */
	+ case CQP_CREATED:
	+ i40iw_destroy_cqp(iwdev, TRUE);
	+ /* fallthrough */
	+ case INITIAL_STATE:
	+ i40iw_del_init_mem(iwdev);
	+ break;
	+ case INVALID_STATE:
	+ /* fallthrough */
	+ default:
	+ i40iw_debug(dev, I40IW_DEBUG_ERR \| I40IW_DEBUG_INIT,
	+ "bad init_state = %d\n", iwdev->init_state);
	+ break;
	+ }
	+
	+ i40iw_cleanup_cm_core(&iwdev->cm_core);
	+ i40iw_del_handler(i40iw_find_handler(iwdev->ldev));
	+ kfree(iwdev->hdl);
	+}
	+
	+/**
	+ * i40iw_setup_init_state - set up the initial device struct
	+ * @hdl: handler for iwarp device - one per instance
	+ * @pf_info: lan device information
	+ *
	+ * Initialize the iwarp device and its hdl information
	+ * using the ldev and client information
	+ * Return 0 if successful, otherwise return error
	+ */
	+static enum i40iw_status_code
	+i40iw_setup_init_state(struct i40iw_handler hdl, struct ixl_iw_pf pf_info)
	+{
	+ struct i40iw_device *iwdev = &hdl->device;
	+ enum i40iw_status_code status;
	+
	+ i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_INIT, "begin");
	+
	+ memcpy(&hdl->ldev, pf_info, sizeof(*pf_info));
	+
	+ iwdev->mpa_version = ixliw_mpa_version;
	+ iwdev->resource_profile = I40IW_HMC_PROFILE_DEFAULT;
	+ iwdev->max_rdma_vfs = 0;
	+ iwdev->ifp = pf_info->ifp;
	+ iwdev->vsi.mtu = pf_info->ifp->if_mtu;
	+
	+ status = i40iw_save_msix_info(iwdev, pf_info);
	+ if (status)
	+ goto exit;
	+
	+ iwdev->dev_ctx.dev = pf_info->dev;
	+ iwdev->dev_ctx.mem_bus_space_tag = rman_get_bustag(pf_info->pci_mem);
	+ iwdev->dev_ctx.mem_bus_space_handle = rman_get_bushandle(pf_info->pci_mem);
	+ iwdev->dev_ctx.mem_bus_space_size = rman_get_size(pf_info->pci_mem);
	+
	+ iwdev->hw.dev_context = &iwdev->dev_ctx;
	+ iwdev->hw.hw_addr = (u8 *)rman_get_virtual(pf_info->pci_mem);
	+
	+ status = i40iw_allocate_dma_mem(&iwdev->hw,
	+ &iwdev->obj_mem, 8192, 4096);
	+ if (status)
	+ goto exit;
	+
	+ iwdev->obj_next = iwdev->obj_mem;
	+ iwdev->push_mode = ixliw_push_mode;
	+
	+ init_waitqueue_head(&iwdev->vchnl_waitq);
	+ init_waitqueue_head(&iwdev->close_wq);
	+
	+ status = i40iw_initialize_dev(iwdev, pf_info);
	+
	+exit:
	+ if (status) {
	+ kfree(iwdev->iw_msixtbl);
	+ ixliw_free_dma_mem(&iwdev->obj_mem);
	+ iwdev->iw_msixtbl = NULL;
	+ }
	+ return status;
	+}
	+
	+/*
	+ * i40iw_get_used_rsrc - determine resources used internally
	+ */
	+static void
	+i40iw_get_used_rsrc(struct i40iw_device *iwdev)
	+{
	+ iwdev->used_pds = find_next_zero_bit(iwdev->allocated_pds, iwdev->max_pd, 0);
	+ iwdev->used_qps = find_next_zero_bit(iwdev->allocated_qps, iwdev->max_qp, 0);
	+ iwdev->used_cqs = find_next_zero_bit(iwdev->allocated_cqs, iwdev->max_cq, 0);
	+ iwdev->used_mrs = find_next_zero_bit(iwdev->allocated_mrs, iwdev->max_mr, 0);
	+}
	+
	+static int
	+ixliw_pf_init(struct ixl_iw_pf *pf_info)
	+{
	+ struct i40iw_sc_dev *dev;
	+ struct i40iw_handler *hdl;
	+ struct i40iw_device *iwdev;
	+ int err;
	+ enum i40iw_status_code status;
	+
	+ INIT_DBG_DEV(pf_info->dev, "begin");
	+
	+#ifdef IW_IXL_DEBUG
	+ printf("%s:\tdevice %s\n", __func__,
	+ device_get_nameunit(pf_info->dev));
	+ printf("%s:\tpf_id=%u\n", __func__, pf_info->pf_id);
	+ printf("%s:\tdev=%p, ifp=%p\n", __func__,
	+ pf_info->dev, pf_info->ifp);
	+ printf("%s:\tmsix: base=%d, count=%d\n", __func__,
	+ pf_info->iw_msix.base, pf_info->iw_msix.count);
	+ printf("%s:\tmtu=%u\n", __func__, pf_info->mtu);
	+ for (int i = 0; i < IXL_IW_MAX_USER_PRIORITY; i++)
	+ printf("%s:\tqs_handle[%i]=%u\n", __func__,
	+ i, pf_info->qs_handle[i]);
	+#endif /* IW_IXL_DEBUG */
	+
	+ hdl = kzalloc(sizeof(*hdl), GFP_KERNEL);
	+ if (!hdl)
	+ return ENOMEM;
	+ iwdev = &hdl->device;
	+ iwdev->hdl = hdl;
	+ dev = &iwdev->sc_dev;
	+ i40iw_debug(dev, I40IW_DEBUG_CLNT, "Entering %s, iwdev is %p\n", __func__, iwdev);
	+
	+ i40iw_setup_cm_core(iwdev);
	+
	+ dev->back_dev = (void *)iwdev;
	+ iwdev->ldev = &hdl->ldev;
	+
	+ mutex_init(&iwdev->pbl_mutex);
	+
	+ i40iw_add_handler(hdl);
	+
	+ do {
	+ status = i40iw_setup_init_state(hdl, pf_info);
	+ if (status)
	+ break;
	+ iwdev->init_state = INITIAL_STATE;
	+ if (dev->is_pf)
	+ i40iw_wait_pe_ready(dev->hw);
	+ status = i40iw_create_cqp(iwdev);
	+ if (status)
	+ break;
	+ iwdev->init_state = CQP_CREATED;
	+ status = i40iw_hmc_setup(iwdev);
	+ if (status)
	+ break;
	+ status = i40iw_create_ccq(iwdev);
	+ if (status)
	+ break;
	+ iwdev->init_state = CCQ_CREATED;
	+ status = i40iw_initialize_ilq(iwdev);
	+ if (status)
	+ break;
	+ iwdev->init_state = ILQ_CREATED;
	+ status = i40iw_initialize_ieq(iwdev);
	+ if (status)
	+ break;
	+ iwdev->init_state = IEQ_CREATED;
	+ status = ixliw_setup_msix(iwdev);
	+ if (status)
	+ break;
	+ status = i40iw_setup_aeq(iwdev);
	+ if (status)
	+ break;
	+ iwdev->init_state = AEQ_CREATED;
	+ status = i40iw_setup_ceqs(iwdev);
	+ if (status)
	+ break;
	+ iwdev->init_state = CEQ_CREATED;
	+
	+ status = i40iw_initialize_hw_resources(iwdev);
	+ if (status)
	+ break;
	+ dev->ccq_ops->ccq_arm(dev->ccq);
	+ status = i40iw_hmc_init_pble(&iwdev->sc_dev, iwdev->pble_rsrc);
	+ if (status)
	+ break;
	+ iwdev->init_state = PBLE_CHUNK_MEM;
	+ status = i40iw_add_mac_ip(iwdev);
	+ if (status)
	+ break;
	+ iwdev->init_state = IP_ADDR_REGISTERED;
	+
	+ err = i40iw_register_rdma_device(iwdev);
	+ if (err) {
	+ i40iw_debug(dev, I40IW_DEBUG_ERR \| I40IW_DEBUG_INIT,
	+ "register rdma device fail (err = %d)\n", err);
	+ break;
	+ };
	+
	+ iwdev->init_state = RDMA_DEV_REGISTERED;
	+ iwdev->iw_status = 1;
	+
	+ i40iw_port_ibevent(iwdev);
	+ i40iw_get_used_rsrc(iwdev);
	+ i40iw_debug(dev, I40IW_DEBUG_CLNT,
	+ "Exiting %s, iwdev is %p. Returning 0.\n", __func__, iwdev);
	+ INIT_DBG_DEV(iwdev->ldev->dev, "completed");
	+
	+ return (0);
	+ } while (0);
	+
	+ device_printf(iwdev->ldev->dev,
	+ "%s: failed (status = %d, last completion = %d)\n",
	+ __func__, status, iwdev->init_state);
	+ i40iw_deinit_device(iwdev);
	+
	+ return (ERESTART);
	+}
	+
	+static int
	+ixliw_pf_stop(struct ixl_iw_pf *pf_info)
	+{
	+ struct i40iw_handler *hdl;
	+ struct i40iw_device *iwdev;
	+
	+ INIT_DBG_DEV(pf_info->dev, "begin");
	+
	+ hdl = i40iw_find_handler(pf_info);
	+ if (!hdl) {
	+ printf("%s: PF handler not found\n", __func__);
	+ return (ENOENT);
	+ }
	+ iwdev = &hdl->device;
	+ i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_CLNT, "Entering %s, iwdev is %p\n",
	+ __func__, iwdev);
	+
	+ iwdev->closing = TRUE;
	+
	+ i40iw_deinit_device(iwdev);
	+ i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_CLNT, "Exiting %s, iwdev is %p\n",
	+ __func__, iwdev);
	+ INIT_DBG_DEV(pf_info->dev, "completed");
	+
	+ return (0);
	+}
	+
	+static int
	+ixliw_mod_load(void)
	+{
	+ struct ixl_iw_ops iw_ops;
	+ int err;
	+
	+ INIT_DEBUGOUT("begin");
	+ printf("iw_ixl module version: %s\n", ixliw_driver_version);
	+
	+ spin_lock_init(&i40iw_handler_lock);
	+ INIT_LIST_HEAD(&i40iw_handlers);
	+
	+ iw_ops.init = ixliw_pf_init;
	+ iw_ops.stop = ixliw_pf_stop;
	+
	+ err = ixl_iw_register(&iw_ops);
	+ if (err)
	+ printf("%s: failed to register to if_ixl (err=%d)\n", __func__, err);
	+
	+ return (err);
	+}
	+
	+static int
	+ixliw_mod_unload(void)
	+{
	+ int err;
	+
	+ INIT_DEBUGOUT("begin");
	+
	+ err = ixl_iw_unregister();
	+ if (err)
	+ printf("%s: failed to unregister from if_ixl (err=%d)\n", __func__,
	+ err);
	+
	+ return (err);
	+}
	+
	+static int
	+ixliw_modevent(module_t mod __unused, int event, void *arg __unused)
	+{
	+ int err = 0;
	+
	+ switch (event) {
	+ case MOD_LOAD:
	+ err = ixliw_mod_load();
	+ break;
	+ case MOD_UNLOAD:
	+ err = ixliw_mod_unload();
	+ break;
	+ default:
	+ err = EOPNOTSUPP;
	+ break;
	+ }
	+
	+ return (err);
	+}
	+
	+static moduledata_t ixliw_mod = {
	+ "iw_ixl",
	+ ixliw_modevent,
	+ NULL
	+};
	+
	+DECLARE_MODULE(iw_ixl, ixliw_mod, SI_SUB_DRIVERS, SI_ORDER_ANY);
	+MODULE_VERSION(iw_ixl, 1);
	+MODULE_DEPEND(iw_ixl, ixl, 1, 2, 2);
	+MODULE_DEPEND(iw_ixl, ibcore, 1, 1, 1);
	Index: sys/dev/ixl/iwarp/iw_ixl_cm.h
	===================================================================
	--- /dev/null
	+++ sys/dev/ixl/iwarp/iw_ixl_cm.h
	@@ -0,0 +1,443 @@
	+/*******************************************************************************
	+*
	+* Copyright (c) 2015-2017 Intel Corporation. All rights reserved.
	+*
	+* This software is available to you under a choice of one of two
	+* licenses. You may choose to be licensed under the terms of the GNU
	+* General Public License (GPL) Version 2, available from the file
	+* COPYING in the main directory of this source tree, or the
	+* OpenFabrics.org BSD license below:
	+*
	+* Redistribution and use in source and binary forms, with or
	+* without modification, are permitted provided that the following
	+* conditions are met:
	+*
	+* - Redistributions of source code must retain the above
	+* copyright notice, this list of conditions and the following
	+* disclaimer.
	+*
	+* - Redistributions in binary form must reproduce the above
	+* copyright notice, this list of conditions and the following
	+* disclaimer in the documentation and/or other materials
	+* provided with the distribution.
	+*
	+* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
	+* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
	+* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
	+* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
	+* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
	+* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
	+* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
	+* SOFTWARE.
	+*
	+*******************************************************************************/
	+/$FreeBSD$/
	+
	+#ifndef _IW_IXL_CM_H_
	+#define _IW_IXL_CM_H_
	+
	+#include <net/ethernet.h>
	+
	+#if defined(VIMAGE) \|\| defined(VNET_DEBUG)
	+#define VIMAGE_READY
	+#else
	+#undef VIMAGE_READY
	+#endif
	+
	+#define VLAN_TAG_PRESENT 0x1000
	+
	+#define I40IW_MANAGE_APBVT_DEL 0
	+#define I40IW_MANAGE_APBVT_ADD 1
	+
	+#define I40IW_MPA_REQUEST_ACCEPT 1
	+#define I40IW_MPA_REQUEST_REJECT 2
	+
	+/* IETF MPA -- defines, enums, structs */
	+#define IEFT_MPA_KEY_REQ "MPA ID Req Frame"
	+#define IEFT_MPA_KEY_REP "MPA ID Rep Frame"
	+#define IETF_MPA_KEY_SIZE 16
	+#define IETF_MAX_PRIV_DATA_LEN 512
	+#define IETF_RTR_MSG_SIZE 4
	+#define IETF_MPA_V2_FLAG 0x10
	+#define SNDMARKER_SEQNMASK 0x000001FF
	+
	+#define I40IW_MAX_IETF_SIZE 32
	+
	+/* IETF RTR MSG Fields */
	+#define IETF_PEER_TO_PEER 0x8000
	+#define IETF_FLPDU_ZERO_LEN 0x4000
	+#define IETF_RDMA0_WRITE 0x8000
	+#define IETF_RDMA0_READ 0x4000
	+#define IETF_NO_IRD_ORD 0x3FFF
	+
	+/* HW-supported IRD sizes */
	+#define I40IW_HW_IRD_SETTING_2 2
	+#define I40IW_HW_IRD_SETTING_4 4
	+#define I40IW_HW_IRD_SETTING_8 8
	+#define I40IW_HW_IRD_SETTING_16 16
	+#define I40IW_HW_IRD_SETTING_32 32
	+#define I40IW_HW_IRD_SETTING_64 64
	+
	+#define MAX_PORTS 65536
	+#define I40IW_VLAN_PRIO_SHIFT 13
	+
	+enum ietf_mpa_flags {
	+ IETF_MPA_FLAGS_MARKERS = 0x80, /* receive Markers */
	+ IETF_MPA_FLAGS_CRC = 0x40, /* receive Markers */
	+ IETF_MPA_FLAGS_REJECT = 0x20, /* Reject */
	+};
	+
	+struct ietf_mpa_v1 {
	+ u8 key[IETF_MPA_KEY_SIZE];
	+ u8 flags;
	+ u8 rev;
	+ __be16 priv_data_len;
	+ u8 priv_data[0];
	+};
	+
	+struct ietf_rtr_msg {
	+ __be16 ctrl_ird;
	+ __be16 ctrl_ord;
	+};
	+
	+struct ietf_mpa_v2 {
	+ u8 key[IETF_MPA_KEY_SIZE];
	+ u8 flags;
	+ u8 rev;
	+ __be16 priv_data_len;
	+ struct ietf_rtr_msg rtr_msg;
	+ u8 priv_data[0];
	+};
	+
	+enum i40iw_timer_type {
	+ I40IW_TIMER_TYPE_SEND,
	+ I40IW_TIMER_TYPE_RECV,
	+ I40IW_TIMER_NODE_CLEANUP,
	+ I40IW_TIMER_TYPE_CLOSE,
	+};
	+
	+#define I40IW_PASSIVE_STATE_INDICATED 0
	+#define I40IW_DO_NOT_SEND_RESET_EVENT 1
	+#define I40IW_SEND_RESET_EVENT 2
	+
	+#define SET_ACK 0x1
	+#define SET_SYN 0x2
	+#define SET_FIN 0x4
	+#define SET_RST 0x8
	+
	+#define TCP_OPTIONS_PADDING 3
	+
	+struct option_base {
	+ u8 optionnum;
	+ u8 length;
	+};
	+
	+enum option_numbers {
	+ OPTION_NUMBER_END,
	+ OPTION_NUMBER_NONE,
	+ OPTION_NUMBER_MSS,
	+ OPTION_NUMBER_WINDOW_SCALE,
	+ OPTION_NUMBER_SACK_PERM,
	+ OPTION_NUMBER_SACK,
	+ OPTION_NUMBER_WRITE0 = 0xbc
	+};
	+
	+struct option_mss {
	+ u8 optionnum;
	+ u8 length;
	+ __be16 mss;
	+};
	+
	+struct option_windowscale {
	+ u8 optionnum;
	+ u8 length;
	+ u8 shiftcount;
	+};
	+
	+union all_known_options {
	+ char as_end;
	+ struct option_base as_base;
	+ struct option_mss as_mss;
	+ struct option_windowscale as_windowscale;
	+};
	+
	+struct i40iw_timer_entry {
	+ struct list_head list;
	+ unsigned long timetosend; /* jiffies */
	+ struct i40iw_puda_buf *sqbuf;
	+ u32 type;
	+ u32 retrycount;
	+ u32 retranscount;
	+ u32 context;
	+ u32 send_retrans;
	+ int close_when_complete;
	+};
	+
	+#define I40IW_DEFAULT_RETRYS 64
	+#define I40IW_DEFAULT_RETRANS 8
	+#define I40IW_DEFAULT_TTL 0x40
	+#define I40IW_DEFAULT_RTT_VAR 0x6
	+#define I40IW_DEFAULT_SS_THRESH 0x3FFFFFFF
	+#define I40IW_DEFAULT_REXMIT_THRESH 8
	+
	+#define I40IW_RETRY_TIMEOUT HZ
	+
	+#define I40IW_LONG_TIME (2 * HZ)
	+#define I40IW_MAX_TIMEOUT ((unsigned long)(12 * HZ))
	+
	+#define I40IW_CM_DEFAULT_RCV_WND_SCALED 0x3fffc
	+#define I40IW_CM_DEFAULT_RCV_WND_SCALE 2
	+
	+#define I40IW_CM_DEFAULT_MSS 536
	+
	+#define I40IW_CM_DEF_LOCAL_ID 0x3b47
	+
	+#define MAX_CM_BUFFER (I40IW_MAX_IETF_SIZE + IETF_MAX_PRIV_DATA_LEN)
	+
	+#define i40iw_cm_tsa_context i40iw_qp_context
	+
	+/* cm node transition states */
	+enum i40iw_cm_node_state {
	+ I40IW_CM_STATE_UNKNOWN,
	+ I40IW_CM_STATE_INITED,
	+ I40IW_CM_STATE_LISTENING,
	+ I40IW_CM_STATE_SYN_RCVD,
	+ I40IW_CM_STATE_SYN_SENT,
	+ I40IW_CM_STATE_ONE_SIDE_ESTABLISHED,
	+ I40IW_CM_STATE_ESTABLISHED,
	+ I40IW_CM_STATE_ACCEPTING,
	+ I40IW_CM_STATE_MPAREQ_SENT,
	+ I40IW_CM_STATE_MPAREQ_RCVD,
	+ I40IW_CM_STATE_MPAREJ_RCVD,
	+ I40IW_CM_STATE_OFFLOADED,
	+ I40IW_CM_STATE_FIN_WAIT1,
	+ I40IW_CM_STATE_FIN_WAIT2,
	+ I40IW_CM_STATE_CLOSE_WAIT,
	+ I40IW_CM_STATE_TIME_WAIT,
	+ I40IW_CM_STATE_LAST_ACK,
	+ I40IW_CM_STATE_CLOSING,
	+ I40IW_CM_STATE_LISTENER_DESTROYED,
	+ I40IW_CM_STATE_CLOSED
	+};
	+
	+enum mpa_frame_version {
	+ IETF_MPA_V1 = 1,
	+ IETF_MPA_V2 = 2
	+};
	+
	+enum mpa_frame_key {
	+ MPA_KEY_REQUEST,
	+ MPA_KEY_REPLY
	+};
	+
	+enum send_rdma0 {
	+ SEND_RDMA_READ_ZERO = 1,
	+ SEND_RDMA_WRITE_ZERO = 2
	+};
	+
	+enum i40iw_tcpip_pkt_type {
	+ I40IW_PKT_TYPE_UNKNOWN,
	+ I40IW_PKT_TYPE_SYN,
	+ I40IW_PKT_TYPE_SYNACK,
	+ I40IW_PKT_TYPE_ACK,
	+ I40IW_PKT_TYPE_FIN,
	+ I40IW_PKT_TYPE_RST
	+};
	+
	+/* CM context params */
	+struct i40iw_cm_tcp_context {
	+ u8 client;
	+
	+ u32 loc_seq_num;
	+ u32 loc_ack_num;
	+ u32 rem_ack_num;
	+ u32 rcv_nxt;
	+
	+ u32 loc_id;
	+ u32 rem_id;
	+
	+ u32 snd_wnd;
	+ u32 max_snd_wnd;
	+
	+ u32 rcv_wnd;
	+ u32 mss;
	+ u8 snd_wscale;
	+ u8 rcv_wscale;
	+
	+ struct timeval sent_ts;
	+};
	+
	+enum i40iw_cm_listener_state {
	+ I40IW_CM_LISTENER_PASSIVE_STATE = 1,
	+ I40IW_CM_LISTENER_ACTIVE_STATE = 2,
	+ I40IW_CM_LISTENER_EITHER_STATE = 3
	+};
	+
	+struct i40iw_cm_listener {
	+ struct list_head list;
	+ struct i40iw_cm_core *cm_core;
	+ u8 loc_mac[ETHER_ADDR_LEN];
	+ u32 loc_addr[4];
	+ u16 loc_port;
	+ struct iw_cm_id *cm_id;
	+ atomic_t ref_count;
	+ struct i40iw_device *iwdev;
	+ atomic_t pend_accepts_cnt;
	+ int backlog;
	+ enum i40iw_cm_listener_state listener_state;
	+ u32 reused_node;
	+ u8 user_pri;
	+ u8 tos;
	+ u16 vlan_id;
	+ bool qhash_set;
	+ bool ipv4;
	+ struct list_head child_listen_list;
	+};
	+
	+struct i40iw_kmem_info {
	+ void *addr;
	+ u32 size;
	+};
	+
	+struct i40iw_kmem_const_info {
	+ const void *addr;
	+ u32 size;
	+};
	+
	+/* per connection node and node state information */
	+struct i40iw_cm_node {
	+ u32 loc_addr[4], rem_addr[4];
	+ u16 loc_port, rem_port;
	+ u16 vlan_id;
	+ enum i40iw_cm_node_state state;
	+ u8 loc_mac[ETHER_ADDR_LEN];
	+ u8 rem_mac[ETHER_ADDR_LEN];
	+ atomic_t ref_count;
	+ struct i40iw_qp *iwqp;
	+ struct i40iw_device *iwdev;
	+ struct i40iw_sc_dev *dev;
	+ struct i40iw_cm_tcp_context tcp_cntxt;
	+ struct i40iw_cm_core *cm_core;
	+ struct i40iw_cm_node *loopbackpartner;
	+ struct i40iw_timer_entry *send_entry;
	+ struct i40iw_timer_entry *close_entry;
	+ spinlock_t retrans_list_lock; /* cm transmit packet */
	+ enum send_rdma0 send_rdma0_op;
	+ u16 ird_size;
	+ u16 ord_size;
	+ u16 mpav2_ird_ord;
	+ struct iw_cm_id *cm_id;
	+ struct list_head list;
	+ int accelerated;
	+ struct i40iw_cm_listener *listener;
	+ int apbvt_set;
	+ int accept_pend;
	+ struct list_head timer_entry;
	+ struct list_head reset_entry;
	+ struct list_head connected_entry;
	+ atomic_t passive_state;
	+ bool qhash_set;
	+ u8 user_pri;
	+ u8 tos;
	+ bool ipv4;
	+ bool snd_mark_en;
	+ u16 lsmm_size;
	+ enum mpa_frame_version mpa_frame_rev;
	+ struct i40iw_kmem_info pdata;
	+ union {
	+ struct ietf_mpa_v1 mpa_frame;
	+ struct ietf_mpa_v2 mpa_v2_frame;
	+ };
	+
	+ u8 pdata_buf[IETF_MAX_PRIV_DATA_LEN];
	+ struct i40iw_kmem_info mpa_hdr;
	+ bool ack_rcvd;
	+};
	+
	+/* structure for client or CM to fill when making CM api calls. */
	+/* - only need to set relevant data, based on op. */
	+struct i40iw_cm_info {
	+ struct iw_cm_id *cm_id;
	+ u16 loc_port;
	+ u16 rem_port;
	+ u32 loc_addr[4];
	+ u32 rem_addr[4];
	+ u16 vlan_id;
	+ int backlog;
	+ u8 user_pri;
	+ u8 tos;
	+ bool ipv4;
	+#ifdef VIMAGE_READY
	+ u8 loc_dev_addr[MAX_ADDR_LEN];
	+ u8 rem_dev_addr[MAX_ADDR_LEN];
	+#endif
	+};
	+
	+/* CM event codes */
	+enum i40iw_cm_event_type {
	+ I40IW_CM_EVENT_UNKNOWN,
	+ I40IW_CM_EVENT_ESTABLISHED,
	+ I40IW_CM_EVENT_MPA_REQ,
	+ I40IW_CM_EVENT_MPA_CONNECT,
	+ I40IW_CM_EVENT_MPA_ACCEPT,
	+ I40IW_CM_EVENT_MPA_REJECT,
	+ I40IW_CM_EVENT_MPA_ESTABLISHED,
	+ I40IW_CM_EVENT_CONNECTED,
	+ I40IW_CM_EVENT_RESET,
	+ I40IW_CM_EVENT_ABORTED
	+};
	+
	+/* event to post to CM event handler */
	+struct i40iw_cm_event {
	+ enum i40iw_cm_event_type type;
	+ struct i40iw_cm_info cm_info;
	+ struct work_struct event_work;
	+ struct i40iw_cm_node *cm_node;
	+};
	+
	+struct i40iw_cm_core {
	+ struct i40iw_device *iwdev;
	+ struct i40iw_sc_dev *dev;
	+
	+ struct list_head listen_nodes;
	+ struct list_head connected_nodes;
	+
	+ struct timer_list tcp_timer;
	+
	+ struct workqueue_struct *event_wq;
	+ struct workqueue_struct *disconn_wq;
	+
	+ spinlock_t ht_lock; /* manage hash table */
	+ spinlock_t listen_list_lock; /* listen list */
	+
	+ unsigned long active_side_ports[BITS_TO_LONGS(MAX_PORTS)];
	+
	+ u64 stats_nodes_created;
	+ u64 stats_nodes_destroyed;
	+ u64 stats_listen_created;
	+ u64 stats_listen_destroyed;
	+ u64 stats_listen_nodes_created;
	+ u64 stats_listen_nodes_destroyed;
	+ u64 stats_loopbacks;
	+ u64 stats_accepts;
	+ u64 stats_rejects;
	+ u64 stats_connect_errs;
	+ u64 stats_passive_errs;
	+ u64 stats_pkt_retrans;
	+ u64 stats_backlog_drops;
	+};
	+
	+int i40iw_schedule_cm_timer(struct i40iw_cm_node , struct i40iw_puda_buf , enum i40iw_timer_type, int, int);
	+int i40iw_accept(struct iw_cm_id , struct iw_cm_conn_param );
	+int i40iw_reject(struct iw_cm_id , const void , u8);
	+int i40iw_connect(struct iw_cm_id , struct iw_cm_conn_param );
	+int i40iw_create_listen(struct iw_cm_id *, int);
	+#if __FreeBSD_version >= 1100000 && !defined(IW_IXL_RDMA_4_9)
	+void i40iw_destroy_listen(struct iw_cm_id *);
	+#else
	+int i40iw_destroy_listen(struct iw_cm_id *);
	+#endif /* IW_IXL_FREEBSD11 */
	+
	+int i40iw_arp_table(struct i40iw_device , u32 , bool, u8 *, u32);
	+
	+#endif /* _IW_IXL_CM_H_ */
	Index: sys/dev/ixl/iwarp/iw_ixl_cm.c
	===================================================================
	--- /dev/null
	+++ sys/dev/ixl/iwarp/iw_ixl_cm.c
	@@ -0,0 +1,4153 @@
	+/*******************************************************************************
	+*
	+* Copyright (c) 2015-2017 Intel Corporation. All rights reserved.
	+*
	+* This software is available to you under a choice of one of two
	+* licenses. You may choose to be licensed under the terms of the GNU
	+* General Public License (GPL) Version 2, available from the file
	+* COPYING in the main directory of this source tree, or the
	+* OpenFabrics.org BSD license below:
	+*
	+* Redistribution and use in source and binary forms, with or
	+* without modification, are permitted provided that the following
	+* conditions are met:
	+*
	+* - Redistributions of source code must retain the above
	+* copyright notice, this list of conditions and the following
	+* disclaimer.
	+*
	+* - Redistributions in binary form must reproduce the above
	+* copyright notice, this list of conditions and the following
	+* disclaimer in the documentation and/or other materials
	+* provided with the distribution.
	+*
	+* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
	+* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
	+* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
	+* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
	+* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
	+* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
	+* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
	+* SOFTWARE.
	+*
	+*******************************************************************************/
	+/$FreeBSD$/
	+
	+#include "iw_ixl.h"
	+
	+#include <net/if_var.h>
	+#include <net/if_dl.h>
	+#include <net/ethernet.h>
	+#include <net/route.h>
	+#include <net/if_vlan_var.h>
	+#include <netinet/if_ether.h>
	+#include <netinet/ip.h>
	+#include <netinet/ip6.h>
	+#include <netinet/tcp.h>
	+
	+#include <linux/inetdevice.h>
	+#include <net/ipv6.h>
	+#include <rdma/ib_addr.h>
	+#include <rdma/rdma_cm.h>
	+
	+static inline bool
	+between(u32 seq1, u32 seq2, u32 seq3)
	+{
	+ return (seq3 - seq2 >= seq1 - seq2);
	+}
	+
	+static void i40iw_rem_ref_cm_node(struct i40iw_cm_node *);
	+static void i40iw_cm_post_event(struct i40iw_cm_event *event);
	+static void i40iw_disconnect_worker(struct work_struct *work);
	+
	+/**
	+ * i40iw_derive_hw_ird_setting - Calculate IRD
	+ *
	+ * @cm_ird: IRD of connection's node
	+ *
	+ * The ird from the connection is rounded to a supported HW
	+ * setting (2,8,32,64) and then encoded for ird_size field of
	+ * qp_ctx
	+ */
	+static u8
	+i40iw_derive_hw_ird_setting(u16 cm_ird)
	+{
	+ u8 encoded_ird_size;
	+
	+ /* ird_size field is encoded in qp_ctx */
	+ switch (i40iw_powerof2_roundup(cm_ird)) {
	+ case I40IW_HW_IRD_SETTING_64:
	+ encoded_ird_size = 3;
	+ break;
	+ case I40IW_HW_IRD_SETTING_32:
	+ case I40IW_HW_IRD_SETTING_16:
	+ encoded_ird_size = 2;
	+ break;
	+ case I40IW_HW_IRD_SETTING_8:
	+ case I40IW_HW_IRD_SETTING_4:
	+ encoded_ird_size = 1;
	+ break;
	+ case I40IW_HW_IRD_SETTING_2:
	+ default:
	+ encoded_ird_size = 0;
	+ break;
	+ }
	+ return encoded_ird_size;
	+}
	+
	+/**
	+ * i40iw_record_ird_ord - Record IRD/ORD passed in
	+ * @cm_node: connection's node
	+ * @conn_ird: connection IRD
	+ * @conn_ord: connection ORD
	+ */
	+static void
	+i40iw_record_ird_ord(struct i40iw_cm_node *cm_node, u32 conn_ird, u32 conn_ord)
	+{
	+ if (conn_ird > I40IW_MAX_IRD_SIZE)
	+ conn_ird = I40IW_MAX_IRD_SIZE;
	+
	+ if (conn_ord > I40IW_MAX_ORD_SIZE)
	+ conn_ord = I40IW_MAX_ORD_SIZE;
	+ else if (!conn_ord && cm_node->send_rdma0_op == SEND_RDMA_READ_ZERO)
	+ conn_ord = 1;
	+
	+ cm_node->ird_size = conn_ird;
	+ cm_node->ord_size = conn_ord;
	+}
	+
	+/**
	+ * i40iw_free_sqbuf - put back puda buffer if refcount = 0
	+ * @dev: FPK device
	+ * @buf: puda buffer to free
	+ */
	+void
	+i40iw_free_sqbuf(struct i40iw_sc_vsi vsi, void bufp)
	+{
	+ struct i40iw_puda_buf buf = (struct i40iw_puda_buf )bufp;
	+ struct i40iw_puda_rsrc *ilq = vsi->ilq;
	+
	+ if (!atomic_dec_return(&buf->refcount))
	+ i40iw_puda_ret_bufpool(ilq, buf);
	+}
	+
	+/**
	+ * i40iw_copy_ip_ntohl - change network to host ip
	+ * @dst: host ip
	+ * @src: big endian
	+ */
	+void
	+i40iw_copy_ip_ntohl(u32 dst, __be32 src)
	+{
	+ dst++ = ntohl(src++);
	+ dst++ = ntohl(src++);
	+ dst++ = ntohl(src++);
	+ dst = ntohl(src);
	+}
	+
	+#ifdef IPV6_SUPPORT
	+/**
	+ * i40iw_copy_ip_htonl - change host addr to network ip
	+ * @dst: host ip
	+ * @src: little endian
	+ */
	+static inline void
	+i40iw_copy_ip_htonl(__be32 dst, u32 src)
	+{
	+ dst++ = htonl(src++);
	+ dst++ = htonl(src++);
	+ dst++ = htonl(src++);
	+ dst = htonl(src);
	+}
	+
	+#endif
	+/**
	+ * i40iw_fill_sockaddr4 - get addr info for passive connection
	+ * @cm_node: connection's node
	+ * @event: upper layer's cm event
	+ */
	+static inline void
	+i40iw_fill_sockaddr4(struct i40iw_cm_node cm_node, struct iw_cm_event event)
	+{
	+ struct sockaddr_in laddr = (struct sockaddr_in )&event->local_addr;
	+ struct sockaddr_in raddr = (struct sockaddr_in )&event->remote_addr;
	+
	+ laddr->sin_family = AF_INET;
	+ raddr->sin_family = AF_INET;
	+
	+ laddr->sin_port = htons(cm_node->loc_port);
	+ raddr->sin_port = htons(cm_node->rem_port);
	+
	+ laddr->sin_addr.s_addr = htonl(cm_node->loc_addr[0]);
	+ raddr->sin_addr.s_addr = htonl(cm_node->rem_addr[0]);
	+}
	+
	+#ifdef IPV6_SUPPORT
	+/**
	+ * i40iw_fill_sockaddr6 - get ipv6 addr info for passive side
	+ * @cm_node: connection's node
	+ * @event: upper layer's cm event
	+ */
	+static inline void
	+i40iw_fill_sockaddr6(struct i40iw_cm_node cm_node, struct iw_cm_event event)
	+{
	+ struct sockaddr_in6 laddr6 = (struct sockaddr_in6 )&event->local_addr;
	+ struct sockaddr_in6 raddr6 = (struct sockaddr_in6 )&event->remote_addr;
	+
	+ laddr6->sin6_family = AF_INET6;
	+ raddr6->sin6_family = AF_INET6;
	+
	+ laddr6->sin6_port = htons(cm_node->loc_port);
	+ raddr6->sin6_port = htons(cm_node->rem_port);
	+
	+ i40iw_copy_ip_htonl((__be32 *)&laddr6->sin6_addr, cm_node->loc_addr);
	+ i40iw_copy_ip_htonl((__be32 *)&raddr6->sin6_addr, cm_node->rem_addr);
	+}
	+
	+#endif
	+/**
	+ * i40iw_get_addr_info
	+ * @cm_node: contains ip/tcp info
	+ * @cm_info: to get a copy of the cm_node ip/tcp info
	+ */
	+static void
	+i40iw_get_addr_info(struct i40iw_cm_node *cm_node,
	+ struct i40iw_cm_info *cm_info)
	+{
	+ cm_info->ipv4 = cm_node->ipv4;
	+ cm_info->vlan_id = cm_node->vlan_id;
	+ memcpy(cm_info->loc_addr, cm_node->loc_addr, sizeof(cm_info->loc_addr));
	+ memcpy(cm_info->rem_addr, cm_node->rem_addr, sizeof(cm_info->rem_addr));
	+ cm_info->loc_port = cm_node->loc_port;
	+ cm_info->rem_port = cm_node->rem_port;
	+}
	+
	+/**
	+ * i40iw_get_cmevent_info - for cm event upcall
	+ * @cm_node: connection's node
	+ * @cm_id: upper layers cm struct for the event
	+ * @event: upper layer's cm event
	+ */
	+static inline void
	+i40iw_get_cmevent_info(struct i40iw_cm_node cm_node, struct iw_cm_id cm_id,
	+ struct iw_cm_event *event)
	+{
	+ memcpy(&event->local_addr, &cm_id->local_addr,
	+ sizeof(event->local_addr));
	+ memcpy(&event->remote_addr, &cm_id->remote_addr,
	+ sizeof(event->remote_addr));
	+ if (cm_node) {
	+ event->private_data = (void *)cm_node->pdata_buf;
	+ event->private_data_len = (u8)cm_node->pdata.size;
	+#if __FreeBSD_version >= 1100000
	+ event->ird = cm_node->ird_size;
	+ event->ord = cm_node->ord_size;
	+#endif /* IW_IXL_FREEBSD11 */
	+ }
	+}
	+
	+/**
	+ * i40iw_send_cm_event - upcall cm's event handler
	+ * @cm_node: connection's node
	+ * @cm_id: upper layer's cm info struct
	+ * @type: Event type to indicate
	+ * @status: status for the event type
	+ */
	+static int
	+i40iw_send_cm_event(struct i40iw_cm_node cm_node, struct iw_cm_id cm_id,
	+ enum iw_cm_event_type type, int status)
	+{
	+ struct iw_cm_event event;
	+
	+ memset(&event, 0, sizeof(event));
	+ event.event = type;
	+ event.status = status;
	+ switch (type) {
	+ case IW_CM_EVENT_CONNECT_REQUEST:
	+ if (cm_node->ipv4)
	+ i40iw_fill_sockaddr4(cm_node, &event);
	+#ifdef IPV6_SUPPORT
	+ else
	+ i40iw_fill_sockaddr6(cm_node, &event);
	+#endif
	+ event.provider_data = (void *)cm_node;
	+ event.private_data = (void *)cm_node->pdata_buf;
	+ event.private_data_len = (u8)cm_node->pdata.size;
	+#if __FreeBSD_version >= 1100000
	+ event.ird = cm_node->ird_size;
	+#endif /* IW_IXL_FREEBSD11 */
	+ break;
	+ case IW_CM_EVENT_CONNECT_REPLY:
	+ i40iw_get_cmevent_info(cm_node, cm_id, &event);
	+ break;
	+ case IW_CM_EVENT_ESTABLISHED:
	+#if __FreeBSD_version >= 1100000
	+ event.ird = cm_node->ird_size;
	+ event.ord = cm_node->ord_size;
	+#endif /* IW_IXL_FREEBSD11 */
	+ break;
	+ case IW_CM_EVENT_DISCONNECT:
	+ break;
	+ case IW_CM_EVENT_CLOSE:
	+ break;
	+ default:
	+ i40iw_pr_err("event type received type = %d\n", type);
	+ return -1;
	+ }
	+ return cm_id->event_handler(cm_id, &event);
	+}
	+
	+/**
	+ * i40iw_create_event - create cm event
	+ * @cm_node: connection's node
	+ * @type: Event type to generate
	+ */
	+static struct i40iw_cm_event *
	+i40iw_create_event(struct i40iw_cm_node *cm_node,
	+ enum i40iw_cm_event_type type)
	+{
	+ struct i40iw_cm_event *event;
	+
	+ if (!cm_node->cm_id)
	+ return NULL;
	+
	+ event = kzalloc(sizeof(*event), GFP_ATOMIC);
	+
	+ if (!event)
	+ return NULL;
	+
	+ event->type = type;
	+ event->cm_node = cm_node;
	+ memcpy(event->cm_info.rem_addr, cm_node->rem_addr,
	+ sizeof(event->cm_info.rem_addr));
	+ memcpy(event->cm_info.loc_addr, cm_node->loc_addr,
	+ sizeof(event->cm_info.loc_addr));
	+ event->cm_info.rem_port = cm_node->rem_port;
	+ event->cm_info.loc_port = cm_node->loc_port;
	+ event->cm_info.cm_id = cm_node->cm_id;
	+
	+ i40iw_debug(cm_node->dev, I40IW_DEBUG_CM,
	+ "node=%p event=%p type=%u dst=%d.%d.%d.%d src=%d.%d.%d.%d\n",
	+ cm_node, event, type,
	+ event->cm_info.loc_addr[0] >> 24,
	+ (event->cm_info.loc_addr[0] >> 16) & 0xFF,
	+ (event->cm_info.loc_addr[0] >> 8) & 0xFF,
	+ event->cm_info.loc_addr[0] & 0xFF,
	+ event->cm_info.rem_addr[0] >> 24,
	+ (event->cm_info.rem_addr[0] >> 16) & 0xFF,
	+ (event->cm_info.rem_addr[0] >> 8) & 0xFF,
	+ event->cm_info.rem_addr[0] & 0xFF);
	+
	+ i40iw_cm_post_event(event);
	+ return event;
	+}
	+
	+/**
	+ * i40iw_free_retrans_entry - free send entry
	+ * @cm_node: connection's node
	+ */
	+static void
	+i40iw_free_retrans_entry(struct i40iw_cm_node *cm_node)
	+{
	+ struct i40iw_device *iwdev = cm_node->iwdev;
	+ struct i40iw_timer_entry *send_entry;
	+
	+ send_entry = cm_node->send_entry;
	+ if (send_entry) {
	+ cm_node->send_entry = NULL;
	+ i40iw_free_sqbuf(&iwdev->vsi, (void *)send_entry->sqbuf);
	+ kfree(send_entry);
	+ atomic_dec(&cm_node->ref_count);
	+ }
	+}
	+
	+/**
	+ * i40iw_cleanup_retrans_entry - free send entry with lock
	+ * @cm_node: connection's node
	+ */
	+static void
	+i40iw_cleanup_retrans_entry(struct i40iw_cm_node *cm_node)
	+{
	+ unsigned long flags;
	+
	+ spin_lock_irqsave(&cm_node->retrans_list_lock, flags);
	+ i40iw_free_retrans_entry(cm_node);
	+ spin_unlock_irqrestore(&cm_node->retrans_list_lock, flags);
	+}
	+
	+static void
	+i40iw_fill_ip4_struct(struct i40iw_cm_node cm_node, struct ip iph, u16 packetsize)
	+{
	+ iph->ip_v = IPVERSION;
	+ iph->ip_hl = 5; /* 5 * 4Byte words, IP headr len */
	+ iph->ip_tos = cm_node->tos;
	+ iph->ip_len = htons(packetsize);
	+ iph->ip_id = htons(++cm_node->tcp_cntxt.loc_id);
	+
	+ iph->ip_off = htons(0x4000);
	+ iph->ip_ttl = 0x40;
	+ iph->ip_p = IPPROTO_TCP;
	+ iph->ip_src.s_addr = htonl(cm_node->loc_addr[0]);
	+ iph->ip_dst.s_addr = htonl(cm_node->rem_addr[0]);
	+}
	+
	+#ifdef IPV6_SUPPORT
	+static void
	+i40iw_fill_ip6_struct(struct i40iw_cm_node cm_node, struct ip6_hdr ip6h, u16 packetsize)
	+{
	+ ip6h->ip6_flow = 0;
	+ ip6h->ip6_vfc = IPV6_VERSION;
	+ ip6h->ip6_plen = htons(packetsize - sizeof(*ip6h));
	+ ip6h->ip6_nxt = 6;
	+ ip6h->ip6_hlim = 128;
	+ i40iw_copy_ip_htonl((__be32 *)&ip6h->ip6_src,
	+ cm_node->loc_addr);
	+ i40iw_copy_ip_htonl((__be32 *)&ip6h->ip6_dst,
	+ cm_node->rem_addr);
	+}
	+
	+#endif
	+/**
	+ * i40iw_form_cm_frame - get a free packet and build frame
	+ * @cm_node: connection's node ionfo to use in frame
	+ * @options: pointer to options info
	+ * @hdr: pointer mpa header
	+ * @pdata: pointer to private data
	+ * @flags: indicates FIN or ACK
	+ */
	+static struct i40iw_puda_buf *
	+i40iw_form_cm_frame(struct i40iw_cm_node *cm_node,
	+ struct i40iw_kmem_info options, struct i40iw_kmem_info hdr,
	+ struct i40iw_kmem_const_info *pdata, u8 flags)
	+{
	+ struct ether_header *ethh;
	+ struct ip *iph;
	+#ifdef IPV6_SUPPORT
	+ struct ip6_hdr *ip6h;
	+#endif
	+ struct i40iw_puda_buf *sqbuf;
	+ struct tcphdr *tcph;
	+ struct i40iw_sc_vsi *vsi = &cm_node->iwdev->vsi;
	+ u32 hdr_len = 0;
	+ u32 opts_len = 0;
	+ u32 pd_len = 0;
	+ u16 eth_hlen = ETH_HLEN;
	+ u16 packetsize;
	+ u8 *buf;
	+
	+#ifndef IPV6_SUPPORT
	+ /*
	+ * When the ipv6 support is not built into the driver (either because
	+ * the INET6 is not added to kernel configuration, or IW_IXL_RDMA_4_9
	+ * is not used in this driver), the cm_node->ipv4 variable must not
	+ * take FALSE value. It should return with 'Protocol not supported'
	+ * error code, but it does return just NULL.
	+ */
	+ if (!cm_node->ipv4)
	+ return NULL;
	+#endif
	+ sqbuf = i40iw_puda_get_bufpool(vsi->ilq);
	+ if (!sqbuf)
	+ return NULL;
	+ buf = sqbuf->mem.va;
	+
	+ if (options)
	+ opts_len = (u32)options->size;
	+
	+ if (hdr)
	+ hdr_len = hdr->size;
	+
	+ if (pdata) {
	+ pd_len = pdata->size;
	+ }
	+ if (cm_node->vlan_id < VLAN_TAG_PRESENT)
	+ eth_hlen += 4;
	+
	+ packetsize = opts_len + hdr_len + pd_len + sizeof(*tcph);
	+ if (cm_node->ipv4)
	+ packetsize += sizeof(*iph);
	+#ifdef IPV6_SUPPORT
	+ else
	+ packetsize += sizeof(*ip6h);
	+ /*
	+ * No ifndef IPV6_SUPPORT needed, as it should not reach this stage
	+ * with such cm_node->ipv4 value.
	+ */
	+#endif
	+
	+ memset(buf, 0x00, eth_hlen + packetsize);
	+
	+ sqbuf->totallen = packetsize + eth_hlen;
	+ sqbuf->maclen = eth_hlen;
	+ sqbuf->tcphlen = sizeof(*tcph) + opts_len;
	+ sqbuf->scratch = (void *)cm_node;
	+
	+ ethh = (struct ether_header *)buf;
	+ buf += eth_hlen;
	+
	+ if (cm_node->ipv4) {
	+ sqbuf->ipv4 = TRUE;
	+ iph = (struct ip *)buf;
	+ buf += sizeof(*iph);
	+ }
	+#ifdef IPV6_SUPPORT
	+ else {
	+ sqbuf->ipv4 = FALSE;
	+ ip6h = (struct ip6_hdr *)buf;
	+ buf += sizeof(*ip6h);
	+ }
	+ /*
	+ * No ifndef IPV6_SUPPORT needed, as it should not reach this stage
	+ * with such cm_node->ipv4 value.
	+ */
	+#endif
	+ tcph = (struct tcphdr *)buf;
	+ buf += sizeof(*tcph);
	+
	+ ether_addr_copy(ethh->ether_dhost, cm_node->rem_mac);
	+ ether_addr_copy(ethh->ether_shost, cm_node->loc_mac);
	+ if (cm_node->vlan_id < VLAN_TAG_PRESENT) {
	+ ((struct ether_vlan_header *)ethh)->evl_encap_proto = htons(ETH_P_8021Q);
	+ ((struct ether_vlan_header *)ethh)->evl_tag = htons(cm_node->vlan_id);
	+ if (cm_node->ipv4)
	+ ((struct ether_vlan_header *)ethh)->evl_proto = htons(ETH_P_IP);
	+#ifdef IPV6_SUPPORT
	+ else
	+ ((struct ether_vlan_header *)ethh)->evl_proto = htons(ETHERTYPE_IPV6);
	+#endif
	+ } else {
	+ ethh->ether_type = htons(ETH_P_IP);
	+#ifdef IPV6_SUPPORT
	+ if (!cm_node->ipv4)
	+ ethh->ether_type = htons(ETHERTYPE_IPV6);
	+#endif
	+ }
	+ if (cm_node->ipv4)
	+ i40iw_fill_ip4_struct(cm_node, iph, packetsize);
	+#ifdef IPV6_SUPPORT
	+ else
	+ i40iw_fill_ip6_struct(cm_node, ip6h, packetsize);
	+#endif
	+
	+ tcph->th_sport = htons(cm_node->loc_port);
	+ tcph->th_dport = htons(cm_node->rem_port);
	+
	+ tcph->th_seq = htonl(cm_node->tcp_cntxt.loc_seq_num);
	+
	+ if (flags & SET_ACK) {
	+ cm_node->tcp_cntxt.loc_ack_num = cm_node->tcp_cntxt.rcv_nxt;
	+ tcph->th_ack = htonl(cm_node->tcp_cntxt.loc_ack_num);
	+ tcph->th_flags \|= TH_ACK;
	+ } else {
	+ tcph->th_ack = 0;
	+ }
	+
	+ if (flags & SET_SYN) {
	+ cm_node->tcp_cntxt.loc_seq_num++;
	+ tcph->th_flags \|= TH_SYN;
	+ } else {
	+ cm_node->tcp_cntxt.loc_seq_num += hdr_len + pd_len;
	+ }
	+
	+ if (flags & SET_FIN) {
	+ cm_node->tcp_cntxt.loc_seq_num++;
	+ tcph->th_flags \|= TH_FIN;
	+ }
	+ if (flags & SET_RST)
	+ tcph->th_flags \|= TH_RST;
	+
	+ tcph->th_off = (u16)((sizeof(*tcph) + opts_len + 3) >> 2);
	+ sqbuf->tcphlen = tcph->th_off << 2;
	+ tcph->th_win = htons(cm_node->tcp_cntxt.rcv_wnd);
	+ tcph->th_urp = 0;
	+
	+ if (opts_len) {
	+ memcpy(buf, options->addr, opts_len);
	+ buf += opts_len;
	+ }
	+ if (hdr_len) {
	+ memcpy(buf, hdr->addr, hdr_len);
	+ buf += hdr_len;
	+ }
	+ if (pdata && pdata->addr)
	+ memcpy(buf, pdata->addr, pdata->size);
	+
	+ atomic_set(&sqbuf->refcount, 1);
	+
	+ return sqbuf;
	+}
	+
	+/**
	+ * i40iw_send_reset - Send RST packet
	+ * @cm_node: connection's node
	+ */
	+int
	+i40iw_send_reset(struct i40iw_cm_node *cm_node)
	+{
	+ struct i40iw_puda_buf *sqbuf;
	+ int flags = SET_RST \| SET_ACK;
	+
	+ sqbuf = i40iw_form_cm_frame(cm_node, NULL, NULL, NULL, flags);
	+ if (!sqbuf) {
	+ i40iw_pr_err("no sqbuf\n");
	+ return -1;
	+ }
	+ return i40iw_schedule_cm_timer(cm_node, sqbuf, I40IW_TIMER_TYPE_SEND,
	+ 0, 1);
	+}
	+
	+/**
	+ * i40iw_active_open_err - send event for active side cm error
	+ * @cm_node: connection's node
	+ * @reset: Flag to send reset or not
	+ */
	+static void
	+i40iw_active_open_err(struct i40iw_cm_node *cm_node, bool reset)
	+{
	+ i40iw_cleanup_retrans_entry(cm_node);
	+ cm_node->cm_core->stats_connect_errs++;
	+ if (reset) {
	+ i40iw_debug(cm_node->dev, I40IW_DEBUG_CM, "%s cm_node=%p state=%d\n",
	+ __func__, cm_node, cm_node->state);
	+ atomic_inc(&cm_node->ref_count);
	+ i40iw_send_reset(cm_node);
	+ }
	+ cm_node->state = I40IW_CM_STATE_CLOSED;
	+ i40iw_create_event(cm_node, I40IW_CM_EVENT_ABORTED);
	+}
	+
	+/**
	+ * i40iw_passive_open_err - handle passive side cm error
	+ * @cm_node: connection's node
	+ * @reset: send reset or just free cm_node
	+ */
	+static void
	+i40iw_passive_open_err(struct i40iw_cm_node *cm_node, bool reset)
	+{
	+ i40iw_cleanup_retrans_entry(cm_node);
	+ cm_node->cm_core->stats_passive_errs++;
	+ cm_node->state = I40IW_CM_STATE_CLOSED;
	+ i40iw_debug(cm_node->dev, I40IW_DEBUG_CM, "%s cm_node=%p state =%d\n",
	+ __func__, cm_node, cm_node->state);
	+ if (reset)
	+ i40iw_send_reset(cm_node);
	+ else
	+ i40iw_rem_ref_cm_node(cm_node);
	+}
	+
	+/**
	+ * i40iw_event_connect_error - to create connect error event
	+ * @event: cm information for connect event
	+ */
	+static void
	+i40iw_event_connect_error(struct i40iw_cm_event *event)
	+{
	+ struct i40iw_qp *iwqp;
	+ struct iw_cm_id *cm_id;
	+
	+ cm_id = event->cm_node->cm_id;
	+ if (!cm_id)
	+ return;
	+
	+ iwqp = cm_id->provider_data;
	+
	+ if (!iwqp \|\| !iwqp->iwdev)
	+ return;
	+
	+ iwqp->cm_id = NULL;
	+ cm_id->provider_data = NULL;
	+ i40iw_send_cm_event(event->cm_node, cm_id,
	+ IW_CM_EVENT_CONNECT_REPLY,
	+ -ECONNRESET);
	+ cm_id->rem_ref(cm_id);
	+ i40iw_rem_ref_cm_node(event->cm_node);
	+}
	+
	+/**
	+ * i40iw_process_options
	+ * @cm_node: connection's node
	+ * @optionsloc: point to start of options
	+ * @optionsize: size of all options
	+ * @syn_packet: flag if syn packet
	+ */
	+static int
	+i40iw_process_options(struct i40iw_cm_node cm_node, u8 optionsloc,
	+ u32 optionsize, u32 syn_packet)
	+{
	+ union all_known_options *all_options;
	+ u32 offset = 0;
	+ u32 tmp;
	+ char got_mss_option = 0;
	+
	+ while (offset < optionsize) {
	+ all_options = (union all_known_options *)(optionsloc + offset);
	+ switch (all_options->as_base.optionnum) {
	+ case OPTION_NUMBER_END:
	+ offset = optionsize;
	+ break;
	+ case OPTION_NUMBER_NONE:
	+ offset += 1;
	+ continue;
	+ case OPTION_NUMBER_MSS:
	+ i40iw_debug(cm_node->dev, I40IW_DEBUG_CM,
	+ "%s: MSS Length: %d Offset: %d Size: %d\n", __func__,
	+ all_options->as_mss.length, offset, optionsize);
	+ got_mss_option = 1;
	+ if (all_options->as_mss.length != 4)
	+ return -1;
	+ tmp = ntohs(all_options->as_mss.mss);
	+ if (tmp > 0 && tmp < cm_node->tcp_cntxt.mss)
	+ cm_node->tcp_cntxt.mss = tmp;
	+ break;
	+ case OPTION_NUMBER_WINDOW_SCALE:
	+ cm_node->tcp_cntxt.snd_wscale =
	+ all_options->as_windowscale.shiftcount;
	+ break;
	+ default:
	+ i40iw_debug(cm_node->dev,
	+ I40IW_DEBUG_CM,
	+ "TCP Option not understood: %x\n",
	+ all_options->as_base.optionnum);
	+ break;
	+ }
	+ offset += all_options->as_base.length;
	+ }
	+ if (!got_mss_option && syn_packet)
	+ cm_node->tcp_cntxt.mss = I40IW_CM_DEFAULT_MSS;
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_handle_tcp_options -
	+ * @cm_node: connection's node
	+ * @tcph: pointer tcp header
	+ * @optionsize: size of options rcvd
	+ * @passive: active or passive flag
	+ */
	+static int
	+i40iw_handle_tcp_options(struct i40iw_cm_node cm_node, struct tcphdr tcph,
	+ int optionsize, int passive)
	+{
	+ u8 optionsloc = (u8 )&tcph[1];
	+
	+ if (optionsize) {
	+ if (i40iw_process_options(cm_node, optionsloc, optionsize,
	+ (u32)(tcph->th_flags & TH_SYN))) {
	+ i40iw_debug(cm_node->dev, I40IW_DEBUG_CM,
	+ "%s: Node %p, Sending RESET\n", __func__, cm_node);
	+ if (passive)
	+ i40iw_passive_open_err(cm_node, TRUE);
	+ else
	+ i40iw_active_open_err(cm_node, TRUE);
	+ return -1;
	+ }
	+ }
	+ cm_node->tcp_cntxt.snd_wnd = ntohs(tcph->th_win) <<
	+ cm_node->tcp_cntxt.snd_wscale;
	+
	+ if (cm_node->tcp_cntxt.snd_wnd > cm_node->tcp_cntxt.max_snd_wnd)
	+ cm_node->tcp_cntxt.max_snd_wnd = cm_node->tcp_cntxt.snd_wnd;
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_build_mpa_v1 - build a MPA V1 frame
	+ * @cm_node: connection's node
	+ * @mpa_key: to do read0 or write0
	+ */
	+static void
	+i40iw_build_mpa_v1(struct i40iw_cm_node cm_node, void start_addr, u8 mpa_key)
	+{
	+ struct ietf_mpa_v1 mpa_frame = (struct ietf_mpa_v1 )start_addr;
	+
	+ switch (mpa_key) {
	+ case MPA_KEY_REQUEST:
	+ memcpy(mpa_frame->key, IEFT_MPA_KEY_REQ, IETF_MPA_KEY_SIZE);
	+ break;
	+ case MPA_KEY_REPLY:
	+ memcpy(mpa_frame->key, IEFT_MPA_KEY_REP, IETF_MPA_KEY_SIZE);
	+ break;
	+ default:
	+ break;
	+ }
	+ mpa_frame->flags = IETF_MPA_FLAGS_CRC;
	+ mpa_frame->rev = cm_node->mpa_frame_rev;
	+ mpa_frame->priv_data_len = htons(cm_node->pdata.size);
	+}
	+
	+/**
	+ * i40iw_build_mpa_v2 - build a MPA V2 frame
	+ * @cm_node: connection's node
	+ * @start_addr: buffer start address
	+ * @mpa_key: to do read0 or write0
	+ */
	+static void
	+i40iw_build_mpa_v2(struct i40iw_cm_node cm_node, void start_addr, u8 mpa_key)
	+{
	+ struct ietf_mpa_v2 mpa_frame = (struct ietf_mpa_v2 )start_addr;
	+ struct ietf_rtr_msg *rtr_msg = &mpa_frame->rtr_msg;
	+ u16 ctrl_ird, ctrl_ord;
	+
	+ /* initialize the upper 5 bytes of the frame */
	+ i40iw_build_mpa_v1(cm_node, start_addr, mpa_key);
	+ mpa_frame->flags \|= IETF_MPA_V2_FLAG;
	+ mpa_frame->priv_data_len += htons(IETF_RTR_MSG_SIZE);
	+
	+ /* initialize RTR msg */
	+ if (cm_node->mpav2_ird_ord == IETF_NO_IRD_ORD) {
	+ ctrl_ird = IETF_NO_IRD_ORD;
	+ ctrl_ord = IETF_NO_IRD_ORD;
	+ } else {
	+ ctrl_ird = (cm_node->ird_size > IETF_NO_IRD_ORD) ?
	+ IETF_NO_IRD_ORD : cm_node->ird_size;
	+ ctrl_ord = (cm_node->ord_size > IETF_NO_IRD_ORD) ?
	+ IETF_NO_IRD_ORD : cm_node->ord_size;
	+ }
	+
	+ ctrl_ird \|= IETF_PEER_TO_PEER;
	+
	+ switch (mpa_key) {
	+ case MPA_KEY_REQUEST:
	+ ctrl_ord \|= IETF_RDMA0_WRITE;
	+ ctrl_ord \|= IETF_RDMA0_READ;
	+ break;
	+ case MPA_KEY_REPLY:
	+ switch (cm_node->send_rdma0_op) {
	+ case SEND_RDMA_WRITE_ZERO:
	+ ctrl_ord \|= IETF_RDMA0_WRITE;
	+ break;
	+ case SEND_RDMA_READ_ZERO:
	+ ctrl_ord \|= IETF_RDMA0_READ;
	+ break;
	+ }
	+ break;
	+ default:
	+ break;
	+ }
	+ rtr_msg->ctrl_ird = htons(ctrl_ird);
	+ rtr_msg->ctrl_ord = htons(ctrl_ord);
	+}
	+
	+/**
	+ * i40iw_cm_build_mpa_frame - build mpa frame for mpa version 1 or version 2
	+ * @cm_node: connection's node
	+ * @mpa: mpa: data buffer
	+ * @mpa_key: to do read0 or write0
	+ */
	+static int
	+i40iw_cm_build_mpa_frame(struct i40iw_cm_node *cm_node,
	+ struct i40iw_kmem_info *mpa, u8 mpa_key)
	+{
	+ int hdr_len = 0;
	+
	+ switch (cm_node->mpa_frame_rev) {
	+ case IETF_MPA_V1:
	+ hdr_len = sizeof(struct ietf_mpa_v1);
	+ i40iw_build_mpa_v1(cm_node, mpa->addr, mpa_key);
	+ break;
	+ case IETF_MPA_V2:
	+ hdr_len = sizeof(struct ietf_mpa_v2);
	+ i40iw_build_mpa_v2(cm_node, mpa->addr, mpa_key);
	+ break;
	+ default:
	+ break;
	+ }
	+
	+ return hdr_len;
	+}
	+
	+/**
	+ * i40iw_send_mpa_request - active node send mpa request to passive node
	+ * @cm_node: connection's node
	+ */
	+static int
	+i40iw_send_mpa_request(struct i40iw_cm_node *cm_node)
	+{
	+ struct i40iw_puda_buf *sqbuf;
	+
	+ if (!cm_node) {
	+ i40iw_pr_err("cm_node == NULL\n");
	+ return -1;
	+ }
	+ cm_node->mpa_hdr.addr = &cm_node->mpa_frame;
	+ cm_node->mpa_hdr.size = i40iw_cm_build_mpa_frame(cm_node,
	+ &cm_node->mpa_hdr, MPA_KEY_REQUEST);
	+ if (!cm_node->mpa_hdr.size) {
	+ i40iw_pr_err("mpa size = %d\n", cm_node->mpa_hdr.size);
	+ return -1;
	+ }
	+ sqbuf = i40iw_form_cm_frame(cm_node,
	+ NULL,
	+ &cm_node->mpa_hdr,
	+ (struct i40iw_kmem_const_info *)&cm_node->pdata,
	+ SET_ACK);
	+ if (!sqbuf) {
	+ i40iw_pr_err("sq_buf == NULL\n");
	+ return -1;
	+ }
	+ return i40iw_schedule_cm_timer(cm_node, sqbuf, I40IW_TIMER_TYPE_SEND, 1, 0);
	+}
	+
	+/**
	+ * i40iw_send_mpa_reject -
	+ * @cm_node: connection's node
	+ * @pdata: reject data for connection
	+ * @plen: length of reject data
	+ */
	+static int
	+i40iw_send_mpa_reject(struct i40iw_cm_node cm_node, const void pdata,
	+ u8 plen)
	+{
	+ struct i40iw_puda_buf *sqbuf;
	+ struct i40iw_kmem_const_info priv_info;
	+
	+ cm_node->mpa_hdr.addr = &cm_node->mpa_frame;
	+ cm_node->mpa_hdr.size = i40iw_cm_build_mpa_frame(cm_node,
	+ &cm_node->mpa_hdr,
	+ MPA_KEY_REPLY);
	+
	+ cm_node->mpa_frame.flags \|= IETF_MPA_FLAGS_REJECT;
	+ priv_info.addr = pdata;
	+ priv_info.size = plen;
	+
	+ sqbuf = i40iw_form_cm_frame(cm_node,
	+ NULL,
	+ &cm_node->mpa_hdr,
	+ &priv_info,
	+ SET_ACK \| SET_FIN);
	+ if (!sqbuf) {
	+ i40iw_pr_err("no sqbuf\n");
	+ return -ENOMEM;
	+ }
	+ cm_node->state = I40IW_CM_STATE_FIN_WAIT1;
	+ return i40iw_schedule_cm_timer(cm_node, sqbuf, I40IW_TIMER_TYPE_SEND, 1, 0);
	+}
	+
	+/**
	+ * recv_mpa - process an IETF MPA frame
	+ * @cm_node: connection's node
	+ * @buffer: Data pointer
	+ * @type: to return accept or reject
	+ * @len: Len of mpa buffer
	+ */
	+static int
	+i40iw_parse_mpa(struct i40iw_cm_node cm_node, u8 buffer, u32 *type, u32 len)
	+{
	+ struct ietf_mpa_v1 *mpa_frame;
	+ struct ietf_mpa_v2 *mpa_v2_frame;
	+ struct ietf_rtr_msg *rtr_msg;
	+ int mpa_hdr_len;
	+ int priv_data_len;
	+
	+ *type = I40IW_MPA_REQUEST_ACCEPT;
	+
	+ if (len < sizeof(struct ietf_mpa_v1)) {
	+ i40iw_pr_err("ietf buffer small (%x)\n", len);
	+ return -1;
	+ }
	+ mpa_frame = (struct ietf_mpa_v1 *)buffer;
	+ mpa_hdr_len = sizeof(struct ietf_mpa_v1);
	+ priv_data_len = ntohs(mpa_frame->priv_data_len);
	+
	+ if (priv_data_len > IETF_MAX_PRIV_DATA_LEN) {
	+ i40iw_pr_err("large pri_data %d\n", priv_data_len);
	+ return -1;
	+ }
	+ if (mpa_frame->rev != IETF_MPA_V1 && mpa_frame->rev != IETF_MPA_V2) {
	+ i40iw_pr_err("unsupported mpa rev = %d\n", mpa_frame->rev);
	+ return -1;
	+ }
	+ if (mpa_frame->rev > cm_node->mpa_frame_rev) {
	+ i40iw_pr_err("rev %d\n", mpa_frame->rev);
	+ return -1;
	+ }
	+ cm_node->mpa_frame_rev = mpa_frame->rev;
	+
	+ if (cm_node->state != I40IW_CM_STATE_MPAREQ_SENT) {
	+ if (memcmp(mpa_frame->key, IEFT_MPA_KEY_REQ, IETF_MPA_KEY_SIZE)) {
	+ i40iw_pr_err("Unexpected MPA Key received\n");
	+ return -1;
	+ }
	+ } else {
	+ if (memcmp(mpa_frame->key, IEFT_MPA_KEY_REP, IETF_MPA_KEY_SIZE)) {
	+ i40iw_pr_err("Unexpected MPA Key received\n");
	+ return -1;
	+ }
	+ }
	+
	+ if (priv_data_len + mpa_hdr_len > len) {
	+ i40iw_pr_err("ietf buffer len(%x + %x != %x)\n",
	+ priv_data_len, mpa_hdr_len, len);
	+ return -1;
	+ }
	+ if (len > MAX_CM_BUFFER) {
	+ i40iw_pr_err("ietf buffer large len = %d\n", len);
	+ return -1;
	+ }
	+ switch (mpa_frame->rev) {
	+ case IETF_MPA_V2:{
	+ u16 ird_size;
	+ u16 ord_size;
	+ u16 ctrl_ord;
	+ u16 ctrl_ird;
	+
	+ mpa_v2_frame = (struct ietf_mpa_v2 *)buffer;
	+ mpa_hdr_len += IETF_RTR_MSG_SIZE;
	+ rtr_msg = &mpa_v2_frame->rtr_msg;
	+
	+ /* parse rtr message */
	+ ctrl_ord = ntohs(rtr_msg->ctrl_ord);
	+ ctrl_ird = ntohs(rtr_msg->ctrl_ird);
	+ ird_size = ctrl_ird & IETF_NO_IRD_ORD;
	+ ord_size = ctrl_ord & IETF_NO_IRD_ORD;
	+
	+ if (!(ctrl_ird & IETF_PEER_TO_PEER))
	+ return -1;
	+
	+ if (ird_size == IETF_NO_IRD_ORD \|\| ord_size == IETF_NO_IRD_ORD) {
	+ cm_node->mpav2_ird_ord = IETF_NO_IRD_ORD;
	+ goto negotiate_done;
	+ }
	+ if (cm_node->state != I40IW_CM_STATE_MPAREQ_SENT) {
	+ /* responder */
	+ if (!ord_size && (ctrl_ord & IETF_RDMA0_READ))
	+ cm_node->ird_size = 1;
	+ if (cm_node->ord_size > ird_size)
	+ cm_node->ord_size = ird_size;
	+ } else {
	+ /* initiator */
	+ if (!ird_size && (ctrl_ord & IETF_RDMA0_READ))
	+ return -1;
	+ if (cm_node->ord_size > ird_size)
	+ cm_node->ord_size = ird_size;
	+
	+ if (cm_node->ird_size < ord_size)
	+ /* no resources available */
	+ return -1;
	+ }
	+
	+ negotiate_done:
	+ if (ctrl_ord & IETF_RDMA0_READ)
	+ cm_node->send_rdma0_op = SEND_RDMA_READ_ZERO;
	+ else if (ctrl_ord & IETF_RDMA0_WRITE)
	+ cm_node->send_rdma0_op = SEND_RDMA_WRITE_ZERO;
	+ else /* Not supported RDMA0 operation */
	+ return -1;
	+ i40iw_debug(cm_node->dev, I40IW_DEBUG_CM,
	+ "MPAV2: Negotiated ORD: %d, IRD: %d\n",
	+ cm_node->ord_size, cm_node->ird_size);
	+ break;
	+ }
	+ break;
	+ case IETF_MPA_V1:
	+ default:
	+ break;
	+ }
	+
	+ memcpy(cm_node->pdata_buf, buffer + mpa_hdr_len, priv_data_len);
	+ cm_node->pdata.size = priv_data_len;
	+
	+ if (mpa_frame->flags & IETF_MPA_FLAGS_REJECT)
	+ *type = I40IW_MPA_REQUEST_REJECT;
	+
	+ if (mpa_frame->flags & IETF_MPA_FLAGS_MARKERS)
	+ cm_node->snd_mark_en = TRUE;
	+
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_schedule_cm_timer
	+ * @@cm_node: connection's node
	+ * @sqbuf: buffer to send
	+ * @type: if it es send or close
	+ * @send_retrans: if rexmits to be done
	+ * @close_when_complete: is cm_node to be removed
	+ *
	+ * note - cm_node needs to be protected before calling this. Encase in:
	+ * i40iw_rem_ref_cm_node(cm_core, cm_node);
	+ * i40iw_schedule_cm_timer(...)
	+ * atomic_inc(&cm_node->ref_count);
	+ */
	+int
	+i40iw_schedule_cm_timer(struct i40iw_cm_node *cm_node,
	+ struct i40iw_puda_buf *sqbuf, enum i40iw_timer_type type, int send_retrans,
	+ int close_when_complete)
	+{
	+ struct i40iw_sc_vsi *vsi = &cm_node->iwdev->vsi;
	+ struct i40iw_cm_core *cm_core = cm_node->cm_core;
	+ struct i40iw_timer_entry *new_send;
	+ int ret = 0;
	+ u32 was_timer_set;
	+ unsigned long flags;
	+
	+ new_send = kzalloc(sizeof(*new_send), GFP_ATOMIC);
	+ if (!new_send) {
	+ if (type != I40IW_TIMER_TYPE_CLOSE)
	+ i40iw_free_sqbuf(vsi, (void *)sqbuf);
	+ return -ENOMEM;
	+ }
	+ new_send->retrycount = I40IW_DEFAULT_RETRYS;
	+ new_send->retranscount = I40IW_DEFAULT_RETRANS;
	+ new_send->sqbuf = sqbuf;
	+ new_send->timetosend = jiffies;
	+ new_send->type = type;
	+ new_send->send_retrans = send_retrans;
	+ new_send->close_when_complete = close_when_complete;
	+
	+ if (type == I40IW_TIMER_TYPE_CLOSE) {
	+ new_send->timetosend += (HZ / 10);
	+ if (cm_node->close_entry) {
	+ kfree(new_send);
	+ i40iw_pr_err("already close entry\n");
	+ return -EINVAL;
	+ }
	+ cm_node->close_entry = new_send;
	+ }
	+ if (type == I40IW_TIMER_TYPE_SEND) {
	+ spin_lock_irqsave(&cm_node->retrans_list_lock, flags);
	+ cm_node->send_entry = new_send;
	+ atomic_inc(&cm_node->ref_count);
	+ spin_unlock_irqrestore(&cm_node->retrans_list_lock, flags);
	+ new_send->timetosend = jiffies + I40IW_RETRY_TIMEOUT;
	+
	+ atomic_inc(&sqbuf->refcount);
	+ i40iw_puda_send_buf(vsi->ilq, sqbuf);
	+ if (!send_retrans) {
	+ i40iw_cleanup_retrans_entry(cm_node);
	+ if (close_when_complete)
	+ i40iw_rem_ref_cm_node(cm_node);
	+ return ret;
	+ }
	+ }
	+ spin_lock_irqsave(&cm_core->ht_lock, flags);
	+ was_timer_set = timer_pending(&cm_core->tcp_timer);
	+
	+ if (!was_timer_set) {
	+ cm_core->tcp_timer.expires = new_send->timetosend;
	+ add_timer(&cm_core->tcp_timer);
	+ }
	+ spin_unlock_irqrestore(&cm_core->ht_lock, flags);
	+
	+ return ret;
	+}
	+
	+/**
	+ * i40iw_retrans_expired - Could not rexmit the packet
	+ * @cm_node: connection's node
	+ */
	+static void
	+i40iw_retrans_expired(struct i40iw_cm_node *cm_node)
	+{
	+ struct iw_cm_id *cm_id = cm_node->cm_id;
	+ enum i40iw_cm_node_state state = cm_node->state;
	+
	+ cm_node->state = I40IW_CM_STATE_CLOSED;
	+ switch (state) {
	+ case I40IW_CM_STATE_SYN_RCVD:
	+ case I40IW_CM_STATE_CLOSING:
	+ i40iw_rem_ref_cm_node(cm_node);
	+ break;
	+ case I40IW_CM_STATE_FIN_WAIT1:
	+ case I40IW_CM_STATE_LAST_ACK:
	+ if (cm_node->cm_id)
	+ cm_id->rem_ref(cm_id);
	+ i40iw_send_reset(cm_node);
	+ break;
	+ default:
	+ atomic_inc(&cm_node->ref_count);
	+ i40iw_send_reset(cm_node);
	+ i40iw_create_event(cm_node, I40IW_CM_EVENT_ABORTED);
	+ break;
	+ }
	+}
	+
	+/**
	+ * i40iw_handle_close_entry - for handling retry/timeouts
	+ * @cm_node: connection's node
	+ * @rem_node: flag for remove cm_node
	+ */
	+static void
	+i40iw_handle_close_entry(struct i40iw_cm_node *cm_node, u32 rem_node)
	+{
	+ struct i40iw_timer_entry *close_entry = cm_node->close_entry;
	+ struct iw_cm_id *cm_id = cm_node->cm_id;
	+ struct i40iw_qp *iwqp;
	+ unsigned long flags;
	+
	+ if (!close_entry)
	+ return;
	+ iwqp = (struct i40iw_qp *)close_entry->sqbuf;
	+ if (iwqp) {
	+ spin_lock_irqsave(&iwqp->lock, flags);
	+ if (iwqp->cm_id) {
	+ iwqp->hw_tcp_state = I40IW_TCP_STATE_CLOSED;
	+ iwqp->hw_iwarp_state = I40IW_QP_STATE_ERROR;
	+ iwqp->last_aeq = I40IW_AE_RESET_SENT;
	+ iwqp->ibqp_state = IB_QPS_ERR;
	+ spin_unlock_irqrestore(&iwqp->lock, flags);
	+ i40iw_cm_disconn(iwqp);
	+ } else {
	+ spin_unlock_irqrestore(&iwqp->lock, flags);
	+ }
	+ } else if (rem_node) {
	+ /* TIME_WAIT state */
	+ i40iw_rem_ref_cm_node(cm_node);
	+ }
	+ if (cm_id)
	+ cm_id->rem_ref(cm_id);
	+ kfree(close_entry);
	+ cm_node->close_entry = NULL;
	+}
	+
	+/**
	+ * i40iw_cm_timer_tick - system's timer expired callback
	+ * @pass: Pointing to cm_core
	+ */
	+static void
	+i40iw_cm_timer_tick(unsigned long pass)
	+{
	+ struct i40iw_cm_core cm_core = (struct i40iw_cm_core )pass;
	+ struct i40iw_cm_node *cm_node;
	+ struct i40iw_timer_entry send_entry, close_entry;
	+ struct i40iw_sc_dev *dev;
	+ struct list_head *list_core_temp;
	+ struct list_head *list_node;
	+ struct list_head timer_list;
	+ struct i40iw_sc_vsi *vsi;
	+ unsigned long nexttimeout = jiffies + I40IW_LONG_TIME;
	+ unsigned long timetosend;
	+ unsigned long flags;
	+ u32 settimer = 0;
	+
	+ INIT_LIST_HEAD(&timer_list);
	+ spin_lock_irqsave(&cm_core->ht_lock, flags);
	+
	+ list_for_each_safe(list_node, list_core_temp, &cm_core->connected_nodes) {
	+ cm_node = container_of(list_node, struct i40iw_cm_node, list);
	+ if (cm_node->close_entry \|\| cm_node->send_entry) {
	+ atomic_inc(&cm_node->ref_count);
	+ list_add(&cm_node->timer_entry, &timer_list);
	+ }
	+ }
	+ spin_unlock_irqrestore(&cm_core->ht_lock, flags);
	+
	+ list_for_each_safe(list_node, list_core_temp, &timer_list) {
	+ cm_node = container_of(list_node, struct i40iw_cm_node, timer_entry);
	+ close_entry = cm_node->close_entry;
	+
	+ if (close_entry) {
	+ if (time_after(close_entry->timetosend, jiffies)) {
	+ if (nexttimeout > close_entry->timetosend \|\|
	+ !settimer) {
	+ nexttimeout = close_entry->timetosend;
	+ settimer = 1;
	+ }
	+ } else {
	+ i40iw_handle_close_entry(cm_node, 1);
	+ }
	+ }
	+ spin_lock_irqsave(&cm_node->retrans_list_lock, flags);
	+
	+ send_entry = cm_node->send_entry;
	+ if (!send_entry)
	+ goto done;
	+ if (time_after(send_entry->timetosend, jiffies)) {
	+ if (cm_node->state != I40IW_CM_STATE_OFFLOADED) {
	+ if ((nexttimeout > send_entry->timetosend) \|\|
	+ !settimer) {
	+ nexttimeout = send_entry->timetosend;
	+ settimer = 1;
	+ }
	+ } else {
	+ i40iw_free_retrans_entry(cm_node);
	+ }
	+ goto done;
	+ }
	+ if ((cm_node->state == I40IW_CM_STATE_OFFLOADED) \|\|
	+ (cm_node->state == I40IW_CM_STATE_CLOSED)) {
	+ i40iw_free_retrans_entry(cm_node);
	+ goto done;
	+ }
	+ if (!send_entry->retranscount \|\| !send_entry->retrycount) {
	+ i40iw_free_retrans_entry(cm_node);
	+
	+ spin_unlock_irqrestore(&cm_node->retrans_list_lock, flags);
	+
	+ i40iw_retrans_expired(cm_node);
	+ cm_node->state = I40IW_CM_STATE_CLOSED;
	+ spin_lock_irqsave(&cm_node->retrans_list_lock, flags);
	+ goto done;
	+ }
	+ spin_unlock_irqrestore(&cm_node->retrans_list_lock, flags);
	+
	+ vsi = &cm_node->iwdev->vsi;
	+ dev = cm_node->dev;
	+
	+ if (!cm_node->ack_rcvd) {
	+ atomic_inc(&send_entry->sqbuf->refcount);
	+ i40iw_puda_send_buf(vsi->ilq, send_entry->sqbuf);
	+ cm_node->cm_core->stats_pkt_retrans++;
	+ }
	+ spin_lock_irqsave(&cm_node->retrans_list_lock, flags);
	+ if (send_entry->send_retrans) {
	+ send_entry->retranscount--;
	+ timetosend = (I40IW_RETRY_TIMEOUT <<
	+ (I40IW_DEFAULT_RETRANS -
	+ send_entry->retranscount));
	+
	+ send_entry->timetosend = jiffies +
	+ min(timetosend, I40IW_MAX_TIMEOUT);
	+ if (nexttimeout > send_entry->timetosend \|\| !settimer) {
	+ nexttimeout = send_entry->timetosend;
	+ settimer = 1;
	+ }
	+ } else {
	+ int close_when_complete;
	+
	+ close_when_complete = send_entry->close_when_complete;
	+ i40iw_debug(cm_node->dev, I40IW_DEBUG_CM, "cm_node=%p state=%d\n",
	+ cm_node, cm_node->state);
	+ i40iw_free_retrans_entry(cm_node);
	+ if (close_when_complete)
	+ i40iw_rem_ref_cm_node(cm_node);
	+ }
	+done:
	+ spin_unlock_irqrestore(&cm_node->retrans_list_lock, flags);
	+ i40iw_rem_ref_cm_node(cm_node);
	+ }
	+
	+ if (settimer) {
	+ spin_lock_irqsave(&cm_core->ht_lock, flags);
	+ if (!timer_pending(&cm_core->tcp_timer)) {
	+ cm_core->tcp_timer.expires = nexttimeout;
	+ add_timer(&cm_core->tcp_timer);
	+ }
	+ spin_unlock_irqrestore(&cm_core->ht_lock, flags);
	+ }
	+}
	+
	+/**
	+ * i40iw_send_syn - send SYN packet
	+ * @cm_node: connection's node
	+ * @sendack: flag to set ACK bit or not
	+ */
	+int
	+i40iw_send_syn(struct i40iw_cm_node *cm_node, u32 sendack)
	+{
	+ struct i40iw_kmem_info opts;
	+ struct i40iw_puda_buf *sqbuf;
	+ union all_known_options options; / Sending MSS option */
	+ int flags = SET_SYN;
	+ int optionssize = 0;
	+ char optionsbuffer[sizeof(struct option_mss) +
	+ sizeof(struct option_windowscale) +
	+ sizeof(struct option_base) + TCP_OPTIONS_PADDING];
	+
	+ opts.addr = optionsbuffer;
	+ if (!cm_node) {
	+ i40iw_pr_err("no cm_node\n");
	+ return -EINVAL;
	+ }
	+ options = (union all_known_options *)&optionsbuffer[optionssize];
	+ options->as_mss.optionnum = OPTION_NUMBER_MSS;
	+ options->as_mss.length = sizeof(struct option_mss);
	+ options->as_mss.mss = htons(cm_node->tcp_cntxt.mss);
	+ optionssize += sizeof(struct option_mss);
	+
	+ options = (union all_known_options *)&optionsbuffer[optionssize];
	+ options->as_windowscale.optionnum = OPTION_NUMBER_WINDOW_SCALE;
	+ options->as_windowscale.length = sizeof(struct option_windowscale);
	+ options->as_windowscale.shiftcount = cm_node->tcp_cntxt.rcv_wscale;
	+ optionssize += sizeof(struct option_windowscale);
	+ options = (union all_known_options *)&optionsbuffer[optionssize];
	+ options->as_end = OPTION_NUMBER_END;
	+ optionssize += 1;
	+
	+ if (sendack)
	+ flags \|= SET_ACK;
	+
	+ opts.size = optionssize;
	+
	+ sqbuf = i40iw_form_cm_frame(cm_node, &opts, NULL, NULL, flags);
	+ if (!sqbuf) {
	+ i40iw_pr_err("no sqbuf\n");
	+ return -1;
	+ }
	+ return i40iw_schedule_cm_timer(cm_node, sqbuf, I40IW_TIMER_TYPE_SEND, 1,
	+ 0);
	+}
	+
	+/**
	+ * i40iw_send_ack - Send ACK packet
	+ * @cm_node: connection's node
	+ */
	+static void
	+i40iw_send_ack(struct i40iw_cm_node *cm_node)
	+{
	+ struct i40iw_puda_buf *sqbuf;
	+ struct i40iw_sc_vsi *vsi = &cm_node->iwdev->vsi;
	+
	+ sqbuf = i40iw_form_cm_frame(cm_node, NULL, NULL, NULL, SET_ACK);
	+ if (sqbuf)
	+ i40iw_puda_send_buf(vsi->ilq, sqbuf);
	+ else
	+ i40iw_pr_err("no sqbuf\n");
	+}
	+
	+/**
	+ * i40iw_send_fin - Send FIN pkt
	+ * @cm_node: connection's node
	+ */
	+static int
	+i40iw_send_fin(struct i40iw_cm_node *cm_node)
	+{
	+ struct i40iw_puda_buf *sqbuf;
	+
	+ sqbuf = i40iw_form_cm_frame(cm_node, NULL, NULL, NULL, SET_ACK \| SET_FIN);
	+ if (!sqbuf) {
	+ i40iw_pr_err("no sqbuf\n");
	+ return -1;
	+ }
	+ return i40iw_schedule_cm_timer(cm_node, sqbuf, I40IW_TIMER_TYPE_SEND, 1,
	+ 0);
	+}
	+
	+/**
	+ * i40iw_find_node - find a cm node that matches the reference cm node
	+ * @cm_core: cm's core
	+ * @rem_port: remote tcp port num
	+ * @rem_addr: remote ip addr
	+ * @loc_port: local tcp port num
	+ * @loc_addr: loc ip addr
	+ * @add_refcnt: flag to increment refcount of cm_node
	+ */
	+struct i40iw_cm_node *
	+i40iw_find_node(struct i40iw_cm_core cm_core, u16 rem_port, u32 rem_addr,
	+ u16 loc_port, u32 *loc_addr, bool add_refcnt)
	+{
	+ struct list_head *hte;
	+ struct i40iw_cm_node *cm_node;
	+ unsigned long flags;
	+
	+ hte = &cm_core->connected_nodes;
	+
	+ /* walk list and find cm_node associated with this session ID */
	+ spin_lock_irqsave(&cm_core->ht_lock, flags);
	+ list_for_each_entry(cm_node, hte, list) {
	+ if (!memcmp(cm_node->loc_addr, loc_addr, sizeof(cm_node->loc_addr)) &&
	+ (cm_node->loc_port == loc_port) &&
	+ !memcmp(cm_node->rem_addr, rem_addr, sizeof(cm_node->rem_addr)) &&
	+ (cm_node->rem_port == rem_port)) {
	+ if (add_refcnt)
	+ atomic_inc(&cm_node->ref_count);
	+ spin_unlock_irqrestore(&cm_core->ht_lock, flags);
	+ return cm_node;
	+ }
	+ }
	+ spin_unlock_irqrestore(&cm_core->ht_lock, flags);
	+
	+ /* no owner node */
	+ return NULL;
	+}
	+
	+static struct i40iw_cm_listener *
	+i40iw_find_listener(struct i40iw_cm_core cm_core, u32 dst_addr, u16 dst_port,
	+ u16 vlan_id, enum i40iw_cm_listener_state listener_state)
	+{
	+ struct i40iw_cm_listener *listen_node;
	+ static const u32 ip_zero[4] = {0, 0, 0, 0};
	+ unsigned long flags;
	+ u32 listen_addr[4];
	+ u16 listen_port;
	+
	+ /* walk list and find cm_node associated with this session ID */
	+ spin_lock_irqsave(&cm_core->listen_list_lock, flags);
	+ list_for_each_entry(listen_node, &cm_core->listen_nodes, list) {
	+ memcpy(listen_addr, listen_node->loc_addr, sizeof(listen_addr));
	+ listen_port = listen_node->loc_port;
	+ /* compare node pair, return node handle if a match */
	+ if ((!memcmp(listen_addr, dst_addr, sizeof(listen_addr)) \|\|
	+ !memcmp(listen_addr, ip_zero, sizeof(listen_addr))) &&
	+ (listen_port == dst_port) &&
	+ (listener_state & listen_node->listener_state)) {
	+ atomic_inc(&listen_node->ref_count);
	+ spin_unlock_irqrestore(&cm_core->listen_list_lock, flags);
	+ return listen_node;
	+ }
	+ }
	+ spin_unlock_irqrestore(&cm_core->listen_list_lock, flags);
	+ return NULL;
	+}
	+
	+/**
	+ * i40iw_add_hte_node - add a cm node to the hash table
	+ * @cm_core: cm's core
	+ * @cm_node: connection's node
	+ */
	+static void
	+i40iw_add_hte_node(struct i40iw_cm_core *cm_core,
	+ struct i40iw_cm_node *cm_node)
	+{
	+ struct list_head *hte;
	+ unsigned long flags;
	+
	+ if (!cm_node \|\| !cm_core) {
	+ i40iw_pr_err("cm_node or cm_core == NULL\n");
	+ return;
	+ }
	+ spin_lock_irqsave(&cm_core->ht_lock, flags);
	+
	+ /* get a handle on the hash table element (list head for this slot) */
	+ hte = &cm_core->connected_nodes;
	+ list_add_tail(&cm_node->list, hte);
	+ spin_unlock_irqrestore(&cm_core->ht_lock, flags);
	+}
	+
	+/**
	+ * i40iw_port_in_use - determine if port is in use
	+ * @port: port number
	+ * @active_side: flag for listener side vs active side
	+ */
	+static bool
	+i40iw_port_in_use(struct i40iw_cm_core *cm_core, u16 port, bool active_side)
	+{
	+ struct i40iw_cm_listener *listen_node;
	+ struct i40iw_cm_node *cm_node;
	+ unsigned long flags;
	+ bool ret = FALSE;
	+
	+ if (active_side) {
	+ /* search connected node list */
	+ spin_lock_irqsave(&cm_core->ht_lock, flags);
	+ list_for_each_entry(cm_node, &cm_core->connected_nodes, list) {
	+ if (cm_node->loc_port == port) {
	+ ret = TRUE;
	+ break;
	+ }
	+ }
	+ if (!ret)
	+ clear_bit(port, cm_core->active_side_ports);
	+ spin_unlock_irqrestore(&cm_core->ht_lock, flags);
	+ } else {
	+ spin_lock_irqsave(&cm_core->listen_list_lock, flags);
	+ list_for_each_entry(listen_node, &cm_core->listen_nodes, list) {
	+ if (listen_node->loc_port == port) {
	+ ret = TRUE;
	+ break;
	+ }
	+ }
	+ spin_unlock_irqrestore(&cm_core->listen_list_lock, flags);
	+ }
	+
	+ return ret;
	+}
	+
	+/**
	+ * i40iw_del_multiple_qhash - Remove qhash and child listens
	+ * @iwdev: iWarp device
	+ * @cm_info: CM info for parent listen node
	+ * @cm_parent_listen_node: The parent listen node
	+ */
	+static enum i40iw_status_code
	+i40iw_del_multiple_qhash(struct i40iw_device *iwdev,
	+ struct i40iw_cm_info *cm_info,
	+ struct i40iw_cm_listener *cm_parent_listen_node)
	+{
	+ struct i40iw_cm_listener *child_listen_node;
	+ struct list_head pos, tpos;
	+ unsigned long flags;
	+#ifdef IPV6_SUPPORT
	+ char ip6buf[INET6_ADDRSTRLEN];
	+#endif
	+ enum i40iw_status_code ret = I40IW_ERR_CONFIG;
	+
	+ spin_lock_irqsave(&iwdev->cm_core.listen_list_lock, flags);
	+ list_for_each_safe(pos, tpos, &cm_parent_listen_node->child_listen_list) {
	+ child_listen_node = list_entry(pos, struct i40iw_cm_listener,
	+ child_listen_list);
	+ if (child_listen_node->ipv4)
	+ i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_CM,
	+ "removing child listen for IP=%d.%d.%d.%d, port=%d, vlan=%d\n",
	+ child_listen_node->loc_addr[0] >> 24,
	+ (child_listen_node->loc_addr[0] >> 16) & 0xFF,
	+ (child_listen_node->loc_addr[0] >> 8) & 0xFF,
	+ child_listen_node->loc_addr[0] & 0xFF,
	+ child_listen_node->loc_port,
	+ child_listen_node->vlan_id);
	+#ifdef IPV6_SUPPORT
	+ else
	+ i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_CM,
	+ "removing child listen for IP=%s, port=%d, vlan=%d\n",
	+ ip6_sprintf(ip6buf,
	+ (const struct in6_addr *)child_listen_node->loc_addr),
	+ child_listen_node->loc_port,
	+ child_listen_node->vlan_id);
	+#endif
	+ list_del(pos);
	+ memcpy(cm_info->loc_addr, child_listen_node->loc_addr,
	+ sizeof(cm_info->loc_addr));
	+ cm_info->vlan_id = child_listen_node->vlan_id;
	+ if (child_listen_node->qhash_set) {
	+ ret = i40iw_manage_qhash(iwdev, cm_info,
	+ I40IW_QHASH_TYPE_TCP_SYN,
	+ I40IW_QHASH_MANAGE_TYPE_DELETE,
	+ NULL, FALSE);
	+ child_listen_node->qhash_set = FALSE;
	+ } else {
	+ ret = I40IW_SUCCESS;
	+ }
	+ i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_CM, "freed pointer = %p\n",
	+ child_listen_node);
	+ kfree(child_listen_node);
	+ cm_parent_listen_node->cm_core->stats_listen_nodes_destroyed++;
	+ }
	+ spin_unlock_irqrestore(&iwdev->cm_core.listen_list_lock, flags);
	+
	+ return ret;
	+}
	+
	+#ifdef IPV6_SUPPORT
	+/**
	+ * i40iw_get_vlan_ipv6 - Returns the vlan_id for ipv6
	+ * @addr: local IPv6 address
	+ *
	+ */
	+static u16
	+i40iw_get_vlan_ipv6(u32 *addr)
	+{
	+ struct ifnet *ifp;
	+ struct sockaddr_in6 sin6;
	+ u16 vlan_id;
	+
	+ memset(&sin6, 0, sizeof(sin6));
	+ ipv6_addr_copy(&sin6.sin6_addr, addr);
	+ sin6.sin6_port = 0;
	+ sin6.sin6_len = sizeof(sin6);
	+ sin6.sin6_family = AF_INET6;
	+
	+ ifp = ip6_dev_find(&init_net, sin6.sin6_addr);
	+ if (ifp) {
	+ vlan_id = rdma_vlan_dev_vlan_id(ifp);
	+ dev_put(ifp);
	+ } else {
	+ vlan_id = I40IW_NO_VLAN;
	+ }
	+
	+ return vlan_id;
	+}
	+
	+#endif
	+/**
	+ * i40iw_get_vlan_ipv4 - Returns the vlan_id for IPv4 address
	+ * @addr: local IPv4 address
	+ */
	+static u16
	+i40iw_get_vlan_ipv4(u32 *addr)
	+{
	+ struct net_device *netdev;
	+ u16 vlan_id = I40IW_NO_VLAN;
	+
	+ netdev = ip_dev_find(&init_net, htonl(addr[0]));
	+ if (netdev) {
	+ vlan_id = rdma_vlan_dev_vlan_id(netdev);
	+ dev_put(netdev);
	+ }
	+ return vlan_id;
	+}
	+
	+#ifdef IPV6_SUPPORT
	+/**
	+ * i40iw_add_mqh_6 - Adds multiple qhashes for IPv6
	+ * @iwdev: iWarp device
	+ * @cm_info: CM info for parent listen node
	+ * @cm_parent_listen_node: The parent listen node
	+ *
	+ * Adds a qhash and a child listen node for every IPv6 address
	+ * on the adapter and adds the associated qhash filter
	+ */
	+static enum i40iw_status_code
	+i40iw_add_mqh_6(struct i40iw_device iwdev, struct i40iw_cm_info cm_info,
	+ struct i40iw_cm_listener *cm_parent_listen_node)
	+{
	+ struct i40iw_cm_listener *child_listen_node;
	+ struct ifnet *ifp;
	+ struct ifaddr *ifa;
	+ struct sockaddr_in6 *sin6;
	+ unsigned long flags;
	+ char ip6buf[INET6_ADDRSTRLEN];
	+ u8 *mac_addr;
	+ enum i40iw_status_code ret = 0;
	+
	+ IFNET_RLOCK();
	+ TAILQ_FOREACH(ifp, &V_ifnet, if_link) {
	+ if ((((rdma_vlan_dev_vlan_id(ifp) < I40IW_NO_VLAN) &&
	+ (rdma_vlan_dev_real_dev(ifp) == iwdev->ifp)) \|\|
	+ (ifp == iwdev->ifp)) && (ifp->if_flags & IFF_UP)) {
	+ if_addr_rlock(ifp);
	+ TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
	+ sin6 = (struct sockaddr_in6 *)ifa->ifa_addr;
	+ if (sin6->sin6_family != AF_INET6)
	+ continue;
	+ mac_addr = IF_LLADDR(ifp);
	+
	+ i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_CM,
	+ "Allocating child CM Listener for "
	+ "IP=%s, vlan_id=%d, "
	+ "MAC=%02x:%02x:%02x:%02x:%02x:%02x\n",
	+ ip6_sprintf(ip6buf, &sin6->sin6_addr),
	+ rdma_vlan_dev_vlan_id(ifp),
	+ mac_addr[0], mac_addr[1], mac_addr[2],
	+ mac_addr[3], mac_addr[4], mac_addr[5]);
	+
	+ child_listen_node =
	+ kzalloc(sizeof(*child_listen_node), GFP_ATOMIC);
	+ i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_CM,
	+ "Allocating child listener %p\n", child_listen_node);
	+ if (!child_listen_node) {
	+ i40iw_pr_err("listener memory allocation\n");
	+ if_addr_runlock(ifp);
	+ ret = I40IW_ERR_NO_MEMORY;
	+ goto exit;
	+ }
	+ cm_info->vlan_id = rdma_vlan_dev_vlan_id(ifp);
	+ cm_parent_listen_node->vlan_id = cm_info->vlan_id;
	+
	+ memcpy(child_listen_node, cm_parent_listen_node,
	+ sizeof(*child_listen_node));
	+
	+ i40iw_copy_ip_ntohl(child_listen_node->loc_addr,
	+ (__be32 *)&sin6->sin6_addr);
	+ memcpy(cm_info->loc_addr, child_listen_node->loc_addr,
	+ sizeof(cm_info->loc_addr));
	+
	+ ret = i40iw_manage_qhash(iwdev, cm_info,
	+ I40IW_QHASH_TYPE_TCP_SYN,
	+ I40IW_QHASH_MANAGE_TYPE_ADD,
	+ NULL, TRUE);
	+ if (!ret) {
	+ child_listen_node->qhash_set = TRUE;
	+ spin_lock_irqsave(&iwdev->cm_core.listen_list_lock, flags);
	+ list_add(&child_listen_node->child_listen_list,
	+ &cm_parent_listen_node->child_listen_list);
	+ spin_unlock_irqrestore(&iwdev->cm_core.listen_list_lock,
	+ flags);
	+ cm_parent_listen_node->cm_core->stats_listen_nodes_created++;
	+ } else {
	+ kfree(child_listen_node);
	+ }
	+ }
	+ if_addr_runlock(ifp);
	+ }
	+ }
	+exit:
	+ IFNET_RUNLOCK();
	+ return ret;
	+}
	+
	+#endif
	+/**
	+ * i40iw_add_mqh_4 - Adds multiple qhashes for IPv4
	+ * @iwdev: iWarp device
	+ * @cm_info: CM info for parent listen node
	+ * @cm_parent_listen_node: The parent listen node
	+ *
	+ * Adds a qhash and a child listen node for every IPv4 address
	+ * on the adapter and adds the associated qhash filter
	+ */
	+static enum i40iw_status_code
	+i40iw_add_mqh_4(struct i40iw_device iwdev, struct i40iw_cm_info cm_info,
	+ struct i40iw_cm_listener *cm_parent_listen_node)
	+{
	+ struct ifnet *ifp;
	+ struct ifaddr *ifa;
	+ struct sockaddr_in *sin;
	+ struct i40iw_cm_listener *child_listen_node;
	+ in_addr_t ip_addr;
	+ unsigned long flags;
	+ u8 *mac_addr;
	+ enum i40iw_status_code ret = 0;
	+
	+ IFNET_RLOCK();
	+ TAILQ_FOREACH(ifp, &V_ifnet, if_link) {
	+ if ((((rdma_vlan_dev_vlan_id(ifp) < I40IW_NO_VLAN) &&
	+ (rdma_vlan_dev_real_dev(ifp) == iwdev->ifp)) \|\|
	+ (ifp == iwdev->ifp)) && (ifp->if_flags & IFF_UP)) {
	+ if_addr_rlock(ifp);
	+ TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
	+ sin = (struct sockaddr_in *)ifa->ifa_addr;
	+ if (sin->sin_family != AF_INET)
	+ continue;
	+ ip_addr = ntohl(sin->sin_addr.s_addr);
	+ mac_addr = IF_LLADDR(ifp);
	+
	+ i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_CM,
	+ "Allocating child CM Listener for "
	+ "IP=%d.%d.%d.%d, vlan_id=%d, "
	+ "MAC=%02x:%02x:%02x:%02x:%02x:%02x\n",
	+ ip_addr >> 24, (ip_addr >> 16) & 0xFF,
	+ (ip_addr >> 8) & 0xFF, ip_addr & 0xFF,
	+ rdma_vlan_dev_vlan_id(ifp),
	+ mac_addr[0], mac_addr[1], mac_addr[2],
	+ mac_addr[3], mac_addr[4], mac_addr[5]);
	+
	+ child_listen_node = kzalloc(sizeof(*child_listen_node),
	+ GFP_ATOMIC);
	+ cm_parent_listen_node->cm_core->stats_listen_nodes_created++;
	+ i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_CM,
	+ "Allocating child listener %p\n", child_listen_node);
	+ if (!child_listen_node) {
	+ i40iw_pr_err("listener memory allocation\n");
	+ if_addr_runlock(ifp);
	+ ret = I40IW_ERR_NO_MEMORY;
	+ goto exit;
	+ }
	+ cm_info->vlan_id = rdma_vlan_dev_vlan_id(ifp);
	+ cm_parent_listen_node->vlan_id = cm_info->vlan_id;
	+ memcpy(child_listen_node,
	+ cm_parent_listen_node,
	+ sizeof(*child_listen_node));
	+
	+ child_listen_node->loc_addr[0] = ip_addr;
	+ memcpy(cm_info->loc_addr, child_listen_node->loc_addr,
	+ sizeof(cm_info->loc_addr));
	+
	+ ret = i40iw_manage_qhash(iwdev,
	+ cm_info,
	+ I40IW_QHASH_TYPE_TCP_SYN,
	+ I40IW_QHASH_MANAGE_TYPE_ADD,
	+ NULL,
	+ TRUE);
	+ if (!ret) {
	+ child_listen_node->qhash_set = TRUE;
	+ spin_lock_irqsave(&iwdev->cm_core.listen_list_lock, flags);
	+ list_add(&child_listen_node->child_listen_list,
	+ &cm_parent_listen_node->child_listen_list);
	+ spin_unlock_irqrestore(&iwdev->cm_core.listen_list_lock,
	+ flags);
	+ } else {
	+ kfree(child_listen_node);
	+ cm_parent_listen_node->cm_core->stats_listen_nodes_created--;
	+ }
	+ }
	+ if_addr_runlock(ifp);
	+ }
	+ }
	+exit:
	+ IFNET_RUNLOCK();
	+ return ret;
	+}
	+
	+/**
	+ * i40iw_dec_refcnt_listen - delete listener and associated cm nodes
	+ * @cm_core: cm's core
	+ * @free_hanging_nodes: to free associated cm_nodes
	+ * @apbvt_del: flag to delete the apbvt
	+ */
	+static int
	+i40iw_dec_refcnt_listen(struct i40iw_cm_core *cm_core,
	+ struct i40iw_cm_listener *listener, int free_hanging_nodes, bool apbvt_del)
	+{
	+ struct list_head *list_pos;
	+ struct list_head *list_temp;
	+ struct i40iw_cm_node *cm_node;
	+ struct list_head reset_list;
	+ struct i40iw_cm_info nfo;
	+ struct i40iw_cm_node *loopback;
	+ int err = 0;
	+ int ret = -EINVAL;
	+ unsigned long flags;
	+ enum i40iw_cm_node_state old_state;
	+
	+ /* free non-accelerated child nodes for this listener */
	+ INIT_LIST_HEAD(&reset_list);
	+ if (free_hanging_nodes) {
	+ spin_lock_irqsave(&cm_core->ht_lock, flags);
	+ list_for_each_safe(list_pos, list_temp, &cm_core->connected_nodes) {
	+ cm_node = container_of(list_pos, struct i40iw_cm_node, list);
	+ if ((cm_node->listener == listener) && !cm_node->accelerated) {
	+ atomic_inc(&cm_node->ref_count);
	+ list_add(&cm_node->reset_entry, &reset_list);
	+ }
	+ }
	+ spin_unlock_irqrestore(&cm_core->ht_lock, flags);
	+ }
	+ list_for_each_safe(list_pos, list_temp, &reset_list) {
	+ cm_node = container_of(list_pos, struct i40iw_cm_node, reset_entry);
	+ loopback = cm_node->loopbackpartner;
	+ if (cm_node->state >= I40IW_CM_STATE_FIN_WAIT1) {
	+ i40iw_rem_ref_cm_node(cm_node);
	+ } else {
	+ if (!loopback) {
	+ i40iw_cleanup_retrans_entry(cm_node);
	+ err = i40iw_send_reset(cm_node);
	+ if (err) {
	+ cm_node->state = I40IW_CM_STATE_CLOSED;
	+ i40iw_pr_err("send reset\n");
	+ } else {
	+ old_state = cm_node->state;
	+ cm_node->state = I40IW_CM_STATE_LISTENER_DESTROYED;
	+ if (old_state != I40IW_CM_STATE_MPAREQ_RCVD)
	+ i40iw_rem_ref_cm_node(cm_node);
	+ }
	+ } else {
	+ struct i40iw_cm_event event;
	+
	+ event.cm_node = loopback;
	+ memcpy(event.cm_info.rem_addr,
	+ loopback->rem_addr, sizeof(event.cm_info.rem_addr));
	+ memcpy(event.cm_info.loc_addr,
	+ loopback->loc_addr, sizeof(event.cm_info.loc_addr));
	+ event.cm_info.rem_port = loopback->rem_port;
	+ event.cm_info.loc_port = loopback->loc_port;
	+ event.cm_info.cm_id = loopback->cm_id;
	+ event.cm_info.ipv4 = loopback->ipv4;
	+ atomic_inc(&loopback->ref_count);
	+ loopback->state = I40IW_CM_STATE_CLOSED;
	+ i40iw_event_connect_error(&event);
	+ cm_node->state = I40IW_CM_STATE_LISTENER_DESTROYED;
	+ i40iw_rem_ref_cm_node(cm_node);
	+ }
	+ }
	+ }
	+
	+ if (!atomic_dec_return(&listener->ref_count)) {
	+ spin_lock_irqsave(&cm_core->listen_list_lock, flags);
	+ list_del(&listener->list);
	+ spin_unlock_irqrestore(&cm_core->listen_list_lock, flags);
	+
	+ if (listener->iwdev) {
	+ if (apbvt_del && !i40iw_port_in_use(cm_core, listener->loc_port, FALSE))
	+ i40iw_manage_apbvt(listener->iwdev,
	+ listener->loc_port,
	+ I40IW_MANAGE_APBVT_DEL);
	+ memcpy(nfo.loc_addr, listener->loc_addr, sizeof(nfo.loc_addr));
	+ nfo.loc_port = listener->loc_port;
	+ nfo.ipv4 = listener->ipv4;
	+ nfo.vlan_id = listener->vlan_id;
	+
	+ if (!list_empty(&listener->child_listen_list)) {
	+ i40iw_del_multiple_qhash(listener->iwdev, &nfo, listener);
	+ } else {
	+ if (listener->qhash_set)
	+ i40iw_manage_qhash(listener->iwdev,
	+ &nfo,
	+ I40IW_QHASH_TYPE_TCP_SYN,
	+ I40IW_QHASH_MANAGE_TYPE_DELETE,
	+ NULL,
	+ FALSE);
	+ }
	+ }
	+ cm_core->stats_listen_destroyed++;
	+ kfree(listener);
	+ cm_core->stats_listen_nodes_destroyed++;
	+ listener = NULL;
	+ ret = 0;
	+ }
	+ if (listener) {
	+ if (atomic_read(&listener->pend_accepts_cnt) > 0)
	+ i40iw_debug(cm_core->dev, I40IW_DEBUG_CM,
	+ "%s: listener (%p) pending accepts=%u\n", __func__,
	+ listener, atomic_read(&listener->pend_accepts_cnt));
	+ }
	+ return ret;
	+}
	+
	+/**
	+ * i40iw_cm_del_listen - delete a linstener
	+ * @cm_core: cm's core
	+ * @listener: passive connection's listener
	+ * @apbvt_del: flag to delete apbvt
	+ */
	+static int
	+i40iw_cm_del_listen(struct i40iw_cm_core *cm_core,
	+ struct i40iw_cm_listener *listener, bool apbvt_del)
	+{
	+ listener->listener_state = I40IW_CM_LISTENER_PASSIVE_STATE;
	+ listener->cm_id = NULL; /* going to be destroyed pretty soon */
	+ return i40iw_dec_refcnt_listen(cm_core, listener, 1, apbvt_del);
	+}
	+
	+#ifndef VIMAGE_READY
	+/**
	+ * i40iw_addr_resolve_neigh - resolve neighbor address
	+ * @iwdev: iwarp device structure
	+ * @src_ip: local ip address
	+ * @dst_ip: remote ip address
	+ * @arpindex: if there is an arp entry
	+ */
	+static int
	+i40iw_addr_resolve_neigh(struct i40iw_device *iwdev, u32 src_ip, u32 dst_ip,
	+ int arpindex)
	+{
	+ struct sockaddr_in dst_sin;
	+ struct llentry *lle;
	+ struct sockaddr *nexthop;
	+ struct rtentry *rte;
	+ int err;
	+ u8 dst_mac[MAX_ADDR_LEN];
	+
	+ memset(&dst_sin, 0, sizeof(dst_sin));
	+ dst_sin.sin_len = sizeof(dst_sin);
	+ dst_sin.sin_family = AF_INET;
	+ dst_sin.sin_port = 0;
	+ dst_sin.sin_addr.s_addr = htonl(dst_ip);
	+
	+ rte = rtalloc1((struct sockaddr *)&dst_sin, 0, 0);
	+ if ((rte == NULL) \|\| ((rte->rt_ifp != iwdev->ifp) &&
	+ (rdma_vlan_dev_real_dev(rte->rt_ifp) != iwdev->ifp))) {
	+ i40iw_pr_err("route not found\n");
	+ RTFREE_LOCKED(rte);
	+ return (arpindex);
	+ }
	+ nexthop = (rte->rt_flags & RTF_GATEWAY) ? rte->rt_gateway :
	+ (struct sockaddr *)&dst_sin;
	+
	+#if __FreeBSD_version >= 1100000
	+ err = arpresolve(rte->rt_ifp, (rte->rt_flags & RTF_GATEWAY), NULL, nexthop,
	+ dst_mac, NULL, &lle);
	+#else
	+ err = arpresolve(rte->rt_ifp, rte, NULL, nexthop, dst_mac, &lle);
	+#endif /* IW_IXL_FREEBSD11 */
	+ rtfree(rte);
	+ if (err) {
	+ i40iw_pr_err("failed to resolve neighbor address (err=%d)\n",
	+ err);
	+ return (arpindex);
	+ }
	+ if (arpindex >= 0) {
	+ if (memcmp(iwdev->arp_table[arpindex].mac_addr, dst_mac,
	+ MAX_ADDR_LEN) == 0)
	+ /* Mac address same as arp table */
	+ return (arpindex);
	+ i40iw_manage_arp_cache(iwdev, iwdev->arp_table[arpindex].mac_addr,
	+ &dst_ip, TRUE, I40IW_ARP_DELETE);
	+ }
	+ i40iw_manage_arp_cache(iwdev, dst_mac, &dst_ip, TRUE, I40IW_ARP_ADD);
	+
	+ return (i40iw_arp_table(iwdev, &dst_ip, TRUE, NULL, I40IW_ARP_RESOLVE));
	+}
	+
	+#endif
	+
	+/**
	+ * i40iw_is_loopback - check if loopback
	+ * @loc_addr: local addr to compare
	+ * @rem_addr: remote address
	+ */
	+static bool
	+i40iw_is_loopback(bool ipv4, u32 loc_addr, u32 rem_addr)
	+{
	+#ifdef IPV6_SUPPORT
	+ struct in6_addr raddr6;
	+
	+#endif
	+ if (ipv4)
	+ return ipv4_is_loopback(htonl(rem_addr)) \|\| (loc_addr == *rem_addr);
	+#ifdef IPV6_SUPPORT
	+ else {
	+ i40iw_copy_ip_htonl((__be32 *)&raddr6, rem_addr);
	+ return !memcmp(loc_addr, rem_addr, 16) \|\| ipv6_addr_loopback(&raddr6);
	+ }
	+#endif
	+ return FALSE;
	+}
	+
	+/**
	+ * i40iw_make_cm_node - create a new instance of a cm node
	+ * @cm_core: cm's core
	+ * @iwdev: iwarp device structure
	+ * @cm_info: quad info for connection
	+ * @listener: passive connection's listener
	+ */
	+static struct i40iw_cm_node *
	+i40iw_make_cm_node(struct i40iw_cm_core cm_core, struct i40iw_device iwdev,
	+ struct i40iw_cm_info cm_info, struct i40iw_cm_listener listener)
	+{
	+ struct i40iw_cm_node *cm_node;
	+ struct timespec ts;
	+ int oldarpindex;
	+ int arpindex;
	+
	+ /* create an hte and cm_node for this instance */
	+ cm_node = kzalloc(sizeof(*cm_node), GFP_ATOMIC);
	+ if (!cm_node)
	+ return NULL;
	+
	+ /* set our node specific transport info */
	+ cm_node->ipv4 = cm_info->ipv4;
	+ cm_node->vlan_id = cm_info->vlan_id;
	+ memcpy(cm_node->loc_addr, cm_info->loc_addr, sizeof(cm_node->loc_addr));
	+ memcpy(cm_node->rem_addr, cm_info->rem_addr, sizeof(cm_node->rem_addr));
	+ cm_node->loc_port = cm_info->loc_port;
	+ cm_node->rem_port = cm_info->rem_port;
	+ cm_node->mpa_frame_rev = iwdev->mpa_version;
	+ cm_node->send_rdma0_op = SEND_RDMA_READ_ZERO;
	+ cm_node->ird_size = I40IW_MAX_IRD_SIZE;
	+ cm_node->ord_size = I40IW_MAX_ORD_SIZE;
	+
	+ cm_node->listener = listener;
	+ cm_node->cm_id = cm_info->cm_id;
	+ ether_addr_copy(cm_node->loc_mac, IF_LLADDR(iwdev->ifp));
	+ spin_lock_init(&cm_node->retrans_list_lock);
	+ cm_node->ack_rcvd = FALSE;
	+
	+ atomic_set(&cm_node->ref_count, 1);
	+ /* associate our parent CM core */
	+ cm_node->cm_core = cm_core;
	+ cm_node->tcp_cntxt.loc_id = I40IW_CM_DEF_LOCAL_ID;
	+ cm_node->tcp_cntxt.rcv_wscale = I40IW_CM_DEFAULT_RCV_WND_SCALE;
	+ cm_node->tcp_cntxt.rcv_wnd =
	+ I40IW_CM_DEFAULT_RCV_WND_SCALED >> I40IW_CM_DEFAULT_RCV_WND_SCALE;
	+ getnanotime(&ts);
	+ cm_node->tcp_cntxt.loc_seq_num = ts.tv_nsec;
	+ cm_node->tcp_cntxt.mss = (cm_node->ipv4) ? (iwdev->vsi.mtu - I40IW_MTU_TO_MSS_IPV4) :
	+ (iwdev->vsi.mtu - I40IW_MTU_TO_MSS_IPV6);
	+
	+ cm_node->iwdev = iwdev;
	+ cm_node->dev = &iwdev->sc_dev;
	+
	+ if (i40iw_is_loopback(cm_node->ipv4, cm_node->loc_addr, cm_node->rem_addr)) {
	+ arpindex = i40iw_arp_table(iwdev, cm_node->rem_addr, FALSE, NULL, I40IW_ARP_RESOLVE);
	+ } else {
	+ oldarpindex = i40iw_arp_table(iwdev, cm_node->rem_addr, FALSE, NULL, I40IW_ARP_RESOLVE);
	+#ifdef VIMAGE_READY
	+ do {
	+ if (oldarpindex >= 0) {
	+ if (memcmp(iwdev->arp_table[oldarpindex].mac_addr, cm_info->rem_dev_addr, MAX_ADDR_LEN) == 0) {
	+ arpindex = oldarpindex;
	+ break;
	+ }
	+ i40iw_manage_arp_cache(iwdev, iwdev->arp_table[oldarpindex].mac_addr, &cm_info->rem_addr[0], FALSE, I40IW_ARP_DELETE);
	+ }
	+ i40iw_manage_arp_cache(iwdev, cm_info->rem_dev_addr, &cm_info->rem_addr[0], FALSE, I40IW_ARP_ADD);
	+ arpindex = (i40iw_arp_table(iwdev, &cm_info->rem_addr[0], FALSE, NULL, I40IW_ARP_RESOLVE));
	+ } while (FALSE);
	+#else
	+ if (cm_node->ipv4)
	+ arpindex = i40iw_addr_resolve_neigh(iwdev, cm_info->loc_addr[0], cm_info->rem_addr[0], oldarpindex);
	+ else
	+ arpindex = -EINVAL;
	+#endif /* VIMAGE_READY */
	+ }
	+ if (arpindex < 0) {
	+ i40iw_pr_err("cm_node arpindex\n");
	+ kfree(cm_node);
	+ return NULL;
	+ }
	+ ether_addr_copy(cm_node->rem_mac, iwdev->arp_table[arpindex].mac_addr);
	+ i40iw_add_hte_node(cm_core, cm_node);
	+ cm_core->stats_nodes_created++;
	+ return cm_node;
	+}
	+
	+/**
	+ * i40iw_rem_ref_cm_node - destroy an instance of a cm node
	+ * @cm_node: connection's node
	+ */
	+static void
	+i40iw_rem_ref_cm_node(struct i40iw_cm_node *cm_node)
	+{
	+ struct i40iw_cm_core *cm_core = cm_node->cm_core;
	+ struct i40iw_qp *iwqp;
	+ struct i40iw_cm_info nfo;
	+ unsigned long flags;
	+
	+ spin_lock_irqsave(&cm_node->cm_core->ht_lock, flags);
	+ if (atomic_dec_return(&cm_node->ref_count)) {
	+ spin_unlock_irqrestore(&cm_node->cm_core->ht_lock, flags);
	+ return;
	+ }
	+ list_del(&cm_node->list);
	+ spin_unlock_irqrestore(&cm_node->cm_core->ht_lock, flags);
	+
	+ /* if the node is destroyed before connection was accelerated */
	+ if (!cm_node->accelerated && cm_node->accept_pend) {
	+ pr_err("node destroyed before established\n");
	+ atomic_dec(&cm_node->listener->pend_accepts_cnt);
	+ }
	+ if (cm_node->close_entry)
	+ i40iw_handle_close_entry(cm_node, 0);
	+ if (cm_node->listener) {
	+ i40iw_dec_refcnt_listen(cm_core, cm_node->listener, 0, TRUE);
	+ } else {
	+ if (!i40iw_port_in_use(cm_core, cm_node->loc_port, TRUE) &&
	+ cm_node->apbvt_set) {
	+ i40iw_manage_apbvt(cm_node->iwdev,
	+ cm_node->loc_port,
	+ I40IW_MANAGE_APBVT_DEL);
	+ cm_node->apbvt_set = 0;
	+ }
	+ i40iw_get_addr_info(cm_node, &nfo);
	+ if (cm_node->qhash_set) {
	+ i40iw_manage_qhash(cm_node->iwdev,
	+ &nfo,
	+ I40IW_QHASH_TYPE_TCP_ESTABLISHED,
	+ I40IW_QHASH_MANAGE_TYPE_DELETE,
	+ NULL,
	+ FALSE);
	+ cm_node->qhash_set = 0;
	+ }
	+ }
	+
	+ iwqp = cm_node->iwqp;
	+ if (iwqp) {
	+ iwqp->cm_node = NULL;
	+ i40iw_rem_ref(&iwqp->ibqp);
	+ cm_node->iwqp = NULL;
	+ } else if (cm_node->qhash_set) {
	+ i40iw_get_addr_info(cm_node, &nfo);
	+ i40iw_manage_qhash(cm_node->iwdev,
	+ &nfo,
	+ I40IW_QHASH_TYPE_TCP_ESTABLISHED,
	+ I40IW_QHASH_MANAGE_TYPE_DELETE,
	+ NULL,
	+ FALSE);
	+ cm_node->qhash_set = 0;
	+ }
	+ cm_node->cm_core->stats_nodes_destroyed++;
	+ kfree(cm_node);
	+}
	+
	+/**
	+ * i40iw_handle_fin_pkt - FIN packet received
	+ * @cm_node: connection's node
	+ */
	+static void
	+i40iw_handle_fin_pkt(struct i40iw_cm_node *cm_node)
	+{
	+ u32 ret;
	+
	+ switch (cm_node->state) {
	+ case I40IW_CM_STATE_SYN_RCVD:
	+ case I40IW_CM_STATE_SYN_SENT:
	+ case I40IW_CM_STATE_ESTABLISHED:
	+ case I40IW_CM_STATE_MPAREJ_RCVD:
	+ cm_node->tcp_cntxt.rcv_nxt++;
	+ i40iw_cleanup_retrans_entry(cm_node);
	+ cm_node->state = I40IW_CM_STATE_LAST_ACK;
	+ i40iw_send_fin(cm_node);
	+ break;
	+ case I40IW_CM_STATE_MPAREQ_SENT:
	+ i40iw_create_event(cm_node, I40IW_CM_EVENT_ABORTED);
	+ cm_node->tcp_cntxt.rcv_nxt++;
	+ i40iw_cleanup_retrans_entry(cm_node);
	+ cm_node->state = I40IW_CM_STATE_CLOSED;
	+ atomic_inc(&cm_node->ref_count);
	+ i40iw_send_reset(cm_node);
	+ break;
	+ case I40IW_CM_STATE_FIN_WAIT1:
	+ cm_node->tcp_cntxt.rcv_nxt++;
	+ i40iw_cleanup_retrans_entry(cm_node);
	+ cm_node->state = I40IW_CM_STATE_CLOSING;
	+ i40iw_send_ack(cm_node);
	+ /*
	+ * Wait for ACK as this is simultaneous close. After we
	+ * receive ACK, do not send anything. Just rm the node.
	+ */
	+ break;
	+ case I40IW_CM_STATE_FIN_WAIT2:
	+ cm_node->tcp_cntxt.rcv_nxt++;
	+ i40iw_cleanup_retrans_entry(cm_node);
	+ cm_node->state = I40IW_CM_STATE_TIME_WAIT;
	+ i40iw_send_ack(cm_node);
	+ ret =
	+ i40iw_schedule_cm_timer(cm_node, NULL, I40IW_TIMER_TYPE_CLOSE, 1,
	+ 0);
	+ if (ret)
	+ i40iw_pr_err("node %p state = %d\n", cm_node, cm_node->state);
	+ break;
	+ case I40IW_CM_STATE_TIME_WAIT:
	+ cm_node->tcp_cntxt.rcv_nxt++;
	+ i40iw_cleanup_retrans_entry(cm_node);
	+ cm_node->state = I40IW_CM_STATE_CLOSED;
	+ i40iw_rem_ref_cm_node(cm_node);
	+ break;
	+ case I40IW_CM_STATE_OFFLOADED:
	+ default:
	+ i40iw_pr_err("bad state node %p state = %d\n", cm_node,
	+ cm_node->state);
	+ break;
	+ }
	+}
	+
	+/**
	+ * i40iw_handle_rst_pkt - process received RST packet
	+ * @cm_node: connection's node
	+ * @rbuf: receive buffer
	+ */
	+static void
	+i40iw_handle_rst_pkt(struct i40iw_cm_node *cm_node,
	+ struct i40iw_puda_buf *rbuf)
	+{
	+ i40iw_cleanup_retrans_entry(cm_node);
	+ switch (cm_node->state) {
	+ case I40IW_CM_STATE_SYN_SENT:
	+ case I40IW_CM_STATE_MPAREQ_SENT:
	+ switch (cm_node->mpa_frame_rev) {
	+ case IETF_MPA_V2:
	+ cm_node->mpa_frame_rev = IETF_MPA_V1;
	+ /* send a syn and goto syn sent state */
	+ cm_node->state = I40IW_CM_STATE_SYN_SENT;
	+ if (i40iw_send_syn(cm_node, 0))
	+ i40iw_active_open_err(cm_node, FALSE);
	+ break;
	+ case IETF_MPA_V1:
	+ default:
	+ i40iw_active_open_err(cm_node, FALSE);
	+ break;
	+ }
	+ break;
	+ case I40IW_CM_STATE_MPAREQ_RCVD:
	+ atomic_add_return(1, &cm_node->passive_state);
	+ break;
	+ case I40IW_CM_STATE_ESTABLISHED:
	+ case I40IW_CM_STATE_SYN_RCVD:
	+ case I40IW_CM_STATE_LISTENING:
	+ i40iw_pr_err("Bad state state = %d\n", cm_node->state);
	+ i40iw_passive_open_err(cm_node, FALSE);
	+ break;
	+ case I40IW_CM_STATE_OFFLOADED:
	+ i40iw_active_open_err(cm_node, FALSE);
	+ break;
	+ case I40IW_CM_STATE_CLOSED:
	+ break;
	+ case I40IW_CM_STATE_FIN_WAIT2:
	+ case I40IW_CM_STATE_FIN_WAIT1:
	+ case I40IW_CM_STATE_LAST_ACK:
	+ cm_node->cm_id->rem_ref(cm_node->cm_id);
	+ case I40IW_CM_STATE_TIME_WAIT:
	+ cm_node->state = I40IW_CM_STATE_CLOSED;
	+ i40iw_rem_ref_cm_node(cm_node);
	+ break;
	+ default:
	+ break;
	+ }
	+}
	+
	+/**
	+ * i40iw_handle_rcv_mpa - Process a recv'd mpa buffer
	+ * @cm_node: connection's node
	+ * @rbuf: receive buffer
	+ */
	+static void
	+i40iw_handle_rcv_mpa(struct i40iw_cm_node *cm_node,
	+ struct i40iw_puda_buf *rbuf)
	+{
	+ int datasize = rbuf->datalen;
	+ int ret;
	+ u32 res_type;
	+ u8 *dataloc = rbuf->data;
	+ enum i40iw_cm_event_type type = I40IW_CM_EVENT_UNKNOWN;
	+
	+ ret = i40iw_parse_mpa(cm_node, dataloc, &res_type, datasize);
	+ if (ret) {
	+ if (cm_node->state == I40IW_CM_STATE_MPAREQ_SENT)
	+ i40iw_active_open_err(cm_node, TRUE);
	+ else
	+ i40iw_passive_open_err(cm_node, TRUE);
	+ return;
	+ }
	+ switch (cm_node->state) {
	+ case I40IW_CM_STATE_ESTABLISHED:
	+ if (res_type == I40IW_MPA_REQUEST_REJECT)
	+ i40iw_pr_err("state for reject\n");
	+ cm_node->state = I40IW_CM_STATE_MPAREQ_RCVD;
	+ type = I40IW_CM_EVENT_MPA_REQ;
	+ i40iw_send_ack(cm_node); /* ACK received MPA request */
	+ atomic_set(&cm_node->passive_state,
	+ I40IW_PASSIVE_STATE_INDICATED);
	+ break;
	+ case I40IW_CM_STATE_MPAREQ_SENT:
	+ i40iw_cleanup_retrans_entry(cm_node);
	+ if (res_type == I40IW_MPA_REQUEST_REJECT) {
	+ type = I40IW_CM_EVENT_MPA_REJECT;
	+ cm_node->state = I40IW_CM_STATE_MPAREJ_RCVD;
	+ } else {
	+ type = I40IW_CM_EVENT_CONNECTED;
	+ cm_node->state = I40IW_CM_STATE_OFFLOADED;
	+ }
	+ i40iw_send_ack(cm_node);
	+ break;
	+ default:
	+ pr_err("%s wrong cm_node state =%d\n", __func__, cm_node->state);
	+ break;
	+ }
	+ i40iw_create_event(cm_node, type);
	+}
	+
	+/**
	+ * i40iw_indicate_pkt_err - Send up err event to cm
	+ * @cm_node: connection's node
	+ */
	+static void
	+i40iw_indicate_pkt_err(struct i40iw_cm_node *cm_node)
	+{
	+ switch (cm_node->state) {
	+ case I40IW_CM_STATE_SYN_SENT:
	+ case I40IW_CM_STATE_MPAREQ_SENT:
	+ i40iw_active_open_err(cm_node, TRUE);
	+ break;
	+ case I40IW_CM_STATE_ESTABLISHED:
	+ case I40IW_CM_STATE_SYN_RCVD:
	+ i40iw_passive_open_err(cm_node, TRUE);
	+ break;
	+ case I40IW_CM_STATE_OFFLOADED:
	+ default:
	+ break;
	+ }
	+}
	+
	+/**
	+ * i40iw_check_syn - Check for error on received syn ack
	+ * @cm_node: connection's node
	+ * @tcph: pointer tcp header
	+ */
	+static int
	+i40iw_check_syn(struct i40iw_cm_node cm_node, struct tcphdr tcph)
	+{
	+ int err = 0;
	+
	+ if (ntohl(tcph->th_ack) != cm_node->tcp_cntxt.loc_seq_num) {
	+ err = 1;
	+ i40iw_active_open_err(cm_node, TRUE);
	+ }
	+ return err;
	+}
	+
	+/**
	+ * i40iw_check_seq - check seq numbers if OK
	+ * @cm_node: connection's node
	+ * @tcph: pointer tcp header
	+ */
	+static int
	+i40iw_check_seq(struct i40iw_cm_node cm_node, struct tcphdr tcph)
	+{
	+ int err = 0;
	+ u32 seq;
	+ u32 ack_seq;
	+ u32 loc_seq_num = cm_node->tcp_cntxt.loc_seq_num;
	+ u32 rcv_nxt = cm_node->tcp_cntxt.rcv_nxt;
	+ u32 rcv_wnd;
	+
	+ seq = ntohl(tcph->th_seq);
	+ ack_seq = ntohl(tcph->th_ack);
	+ rcv_wnd = cm_node->tcp_cntxt.rcv_wnd;
	+ if (ack_seq != loc_seq_num)
	+ err = -1;
	+ else if (!between(seq, rcv_nxt, (rcv_nxt + rcv_wnd)))
	+ err = -1;
	+ if (err) {
	+ i40iw_pr_err("seq number\n");
	+ i40iw_indicate_pkt_err(cm_node);
	+ }
	+ return err;
	+}
	+
	+/**
	+ * i40iw_handle_syn_pkt - is for Passive node
	+ * @cm_node: connection's node
	+ * @rbuf: receive buffer
	+ */
	+static void
	+i40iw_handle_syn_pkt(struct i40iw_cm_node *cm_node,
	+ struct i40iw_puda_buf *rbuf)
	+{
	+ struct i40iw_cm_info nfo;
	+ struct tcphdr tcph = (struct tcphdr )rbuf->tcph;
	+ int optionsize;
	+ int ret;
	+ u32 inc_sequence;
	+
	+ optionsize = (tcph->th_off << 2) - sizeof(struct tcphdr);
	+ inc_sequence = ntohl(tcph->th_seq);
	+
	+ switch (cm_node->state) {
	+ case I40IW_CM_STATE_SYN_SENT:
	+ case I40IW_CM_STATE_MPAREQ_SENT:
	+ /* Rcvd syn on active open connection */
	+ i40iw_active_open_err(cm_node, 1);
	+ break;
	+ case I40IW_CM_STATE_LISTENING:
	+ /* Passive OPEN */
	+ if (atomic_read(&cm_node->listener->pend_accepts_cnt) >
	+ cm_node->listener->backlog) {
	+ cm_node->cm_core->stats_backlog_drops++;
	+ i40iw_passive_open_err(cm_node, FALSE);
	+ break;
	+ }
	+ ret = i40iw_handle_tcp_options(cm_node, tcph, optionsize, 1);
	+ if (ret) {
	+ i40iw_passive_open_err(cm_node, FALSE);
	+ /* drop pkt */
	+ break;
	+ }
	+ cm_node->tcp_cntxt.rcv_nxt = inc_sequence + 1;
	+ cm_node->accept_pend = 1;
	+ atomic_inc(&cm_node->listener->pend_accepts_cnt);
	+
	+ cm_node->state = I40IW_CM_STATE_SYN_RCVD;
	+ i40iw_get_addr_info(cm_node, &nfo);
	+ ret = i40iw_manage_qhash(cm_node->iwdev, &nfo,
	+ I40IW_QHASH_TYPE_TCP_ESTABLISHED,
	+ I40IW_QHASH_MANAGE_TYPE_ADD, (void *)cm_node, FALSE);
	+ cm_node->qhash_set = TRUE;
	+ break;
	+ case I40IW_CM_STATE_CLOSED:
	+ i40iw_cleanup_retrans_entry(cm_node);
	+ atomic_inc(&cm_node->ref_count);
	+ i40iw_send_reset(cm_node);
	+ break;
	+ case I40IW_CM_STATE_OFFLOADED:
	+ case I40IW_CM_STATE_ESTABLISHED:
	+ case I40IW_CM_STATE_FIN_WAIT1:
	+ case I40IW_CM_STATE_FIN_WAIT2:
	+ case I40IW_CM_STATE_MPAREQ_RCVD:
	+ case I40IW_CM_STATE_LAST_ACK:
	+ case I40IW_CM_STATE_CLOSING:
	+ case I40IW_CM_STATE_UNKNOWN:
	+ default:
	+ break;
	+ }
	+}
	+
	+/**
	+ * i40iw_handle_synack_pkt - Process SYN+ACK packet (active side)
	+ * @cm_node: connection's node
	+ * @rbuf: receive buffer
	+ */
	+static void
	+i40iw_handle_synack_pkt(struct i40iw_cm_node *cm_node,
	+ struct i40iw_puda_buf *rbuf)
	+{
	+ struct tcphdr tcph = (struct tcphdr )rbuf->tcph;
	+ int optionsize;
	+ int ret;
	+ u32 inc_sequence;
	+
	+ optionsize = (tcph->th_off << 2) - sizeof(struct tcphdr);
	+ inc_sequence = ntohl(tcph->th_seq);
	+ switch (cm_node->state) {
	+ case I40IW_CM_STATE_SYN_SENT:
	+ i40iw_cleanup_retrans_entry(cm_node);
	+ /* active open */
	+ if (i40iw_check_syn(cm_node, tcph)) {
	+ i40iw_pr_err("check syn fail\n");
	+ return;
	+ }
	+ cm_node->tcp_cntxt.rem_ack_num = ntohl(tcph->th_ack);
	+ /* setup options */
	+ ret = i40iw_handle_tcp_options(cm_node, tcph, optionsize, 0);
	+ if (ret) {
	+ i40iw_debug(cm_node->dev, I40IW_DEBUG_CM,
	+ "cm_node=%p tcp_options failed\n", cm_node);
	+ break;
	+ }
	+ i40iw_cleanup_retrans_entry(cm_node);
	+ cm_node->tcp_cntxt.rcv_nxt = inc_sequence + 1;
	+
	+ i40iw_send_ack(cm_node); /* ACK for the syn_ack */
	+ ret = i40iw_send_mpa_request(cm_node);
	+ if (ret) {
	+ i40iw_debug(cm_node->dev, I40IW_DEBUG_CM,
	+ "cm_node=%p i40iw_send_mpa_request failed\n", cm_node);
	+ break;
	+ }
	+ cm_node->state = I40IW_CM_STATE_MPAREQ_SENT;
	+ break;
	+ case I40IW_CM_STATE_MPAREQ_RCVD:
	+ i40iw_passive_open_err(cm_node, TRUE);
	+ break;
	+ case I40IW_CM_STATE_LISTENING:
	+ cm_node->tcp_cntxt.loc_seq_num = ntohl(tcph->th_ack);
	+ i40iw_cleanup_retrans_entry(cm_node);
	+ cm_node->state = I40IW_CM_STATE_CLOSED;
	+ i40iw_send_reset(cm_node);
	+ break;
	+ case I40IW_CM_STATE_CLOSED:
	+ cm_node->tcp_cntxt.loc_seq_num = ntohl(tcph->th_ack);
	+ i40iw_cleanup_retrans_entry(cm_node);
	+ atomic_inc(&cm_node->ref_count);
	+ i40iw_send_reset(cm_node);
	+ break;
	+ case I40IW_CM_STATE_ESTABLISHED:
	+ case I40IW_CM_STATE_FIN_WAIT1:
	+ case I40IW_CM_STATE_FIN_WAIT2:
	+ case I40IW_CM_STATE_LAST_ACK:
	+ case I40IW_CM_STATE_OFFLOADED:
	+ case I40IW_CM_STATE_CLOSING:
	+ case I40IW_CM_STATE_UNKNOWN:
	+ case I40IW_CM_STATE_MPAREQ_SENT:
	+ default:
	+ break;
	+ }
	+}
	+
	+/**
	+ * i40iw_handle_ack_pkt - process packet with ACK
	+ * @cm_node: connection's node
	+ * @rbuf: receive buffer
	+ */
	+static int
	+i40iw_handle_ack_pkt(struct i40iw_cm_node *cm_node,
	+ struct i40iw_puda_buf *rbuf)
	+{
	+ struct tcphdr tcph = (struct tcphdr )rbuf->tcph;
	+ int optionsize;
	+ int ret = 0;
	+ u32 datasize = rbuf->datalen;
	+ u32 inc_sequence;
	+
	+ optionsize = (tcph->th_off << 2) - sizeof(struct tcphdr);
	+
	+ if (i40iw_check_seq(cm_node, tcph))
	+ return -EINVAL;
	+
	+ inc_sequence = ntohl(tcph->th_seq);
	+ switch (cm_node->state) {
	+ case I40IW_CM_STATE_SYN_RCVD:
	+ i40iw_cleanup_retrans_entry(cm_node);
	+ ret = i40iw_handle_tcp_options(cm_node, tcph, optionsize, 1);
	+ if (ret)
	+ break;
	+ cm_node->tcp_cntxt.rem_ack_num = ntohl(tcph->th_ack);
	+ cm_node->state = I40IW_CM_STATE_ESTABLISHED;
	+ if (datasize) {
	+ cm_node->tcp_cntxt.rcv_nxt = inc_sequence + datasize;
	+ cm_node->ack_rcvd = FALSE;
	+ i40iw_handle_rcv_mpa(cm_node, rbuf);
	+ } else {
	+ cm_node->ack_rcvd = TRUE;
	+ }
	+ break;
	+ case I40IW_CM_STATE_ESTABLISHED:
	+ i40iw_cleanup_retrans_entry(cm_node);
	+ if (datasize) {
	+ cm_node->tcp_cntxt.rcv_nxt = inc_sequence + datasize;
	+ i40iw_handle_rcv_mpa(cm_node, rbuf);
	+ }
	+ break;
	+ case I40IW_CM_STATE_MPAREQ_SENT:
	+ cm_node->tcp_cntxt.rem_ack_num = ntohl(tcph->th_ack);
	+ if (datasize) {
	+ cm_node->tcp_cntxt.rcv_nxt = inc_sequence + datasize;
	+ i40iw_handle_rcv_mpa(cm_node, rbuf);
	+ }
	+ break;
	+ case I40IW_CM_STATE_LISTENING:
	+ i40iw_cleanup_retrans_entry(cm_node);
	+ cm_node->state = I40IW_CM_STATE_CLOSED;
	+ i40iw_send_reset(cm_node);
	+ break;
	+ case I40IW_CM_STATE_CLOSED:
	+ i40iw_cleanup_retrans_entry(cm_node);
	+ atomic_inc(&cm_node->ref_count);
	+ i40iw_send_reset(cm_node);
	+ break;
	+ case I40IW_CM_STATE_LAST_ACK:
	+ case I40IW_CM_STATE_CLOSING:
	+ i40iw_cleanup_retrans_entry(cm_node);
	+ cm_node->state = I40IW_CM_STATE_CLOSED;
	+ if (!cm_node->accept_pend)
	+ cm_node->cm_id->rem_ref(cm_node->cm_id);
	+ i40iw_rem_ref_cm_node(cm_node);
	+ break;
	+ case I40IW_CM_STATE_FIN_WAIT1:
	+ i40iw_cleanup_retrans_entry(cm_node);
	+ cm_node->state = I40IW_CM_STATE_FIN_WAIT2;
	+ break;
	+ case I40IW_CM_STATE_SYN_SENT:
	+ case I40IW_CM_STATE_FIN_WAIT2:
	+ case I40IW_CM_STATE_OFFLOADED:
	+ case I40IW_CM_STATE_MPAREQ_RCVD:
	+ case I40IW_CM_STATE_UNKNOWN:
	+ default:
	+ i40iw_cleanup_retrans_entry(cm_node);
	+ break;
	+ }
	+ return ret;
	+}
	+
	+/**
	+ * i40iw_process_packet - process cm packet
	+ * @cm_node: connection's node
	+ * @rbuf: receive buffer
	+ */
	+static void
	+i40iw_process_packet(struct i40iw_cm_node *cm_node,
	+ struct i40iw_puda_buf *rbuf)
	+{
	+ struct tcphdr tcph = (struct tcphdr )rbuf->tcph;
	+ int ret;
	+ u32 fin_set = 0;
	+ enum i40iw_tcpip_pkt_type pkt_type = I40IW_PKT_TYPE_UNKNOWN;
	+
	+ if (tcph->th_flags & TH_RST) {
	+ pkt_type = I40IW_PKT_TYPE_RST;
	+ } else if (tcph->th_flags & TH_SYN) {
	+ pkt_type = I40IW_PKT_TYPE_SYN;
	+ if (tcph->th_flags & TH_ACK)
	+ pkt_type = I40IW_PKT_TYPE_SYNACK;
	+ } else if (tcph->th_flags & TH_ACK) {
	+ pkt_type = I40IW_PKT_TYPE_ACK;
	+ }
	+ if (tcph->th_flags & TH_FIN)
	+ fin_set = 1;
	+
	+ switch (pkt_type) {
	+ case I40IW_PKT_TYPE_SYN:
	+ i40iw_handle_syn_pkt(cm_node, rbuf);
	+ break;
	+ case I40IW_PKT_TYPE_SYNACK:
	+ i40iw_handle_synack_pkt(cm_node, rbuf);
	+ break;
	+ case I40IW_PKT_TYPE_ACK:
	+ ret = i40iw_handle_ack_pkt(cm_node, rbuf);
	+ if (fin_set && !ret)
	+ i40iw_handle_fin_pkt(cm_node);
	+ break;
	+ case I40IW_PKT_TYPE_RST:
	+ i40iw_handle_rst_pkt(cm_node, rbuf);
	+ break;
	+ default:
	+ if (fin_set &&
	+ (!i40iw_check_seq(cm_node, (struct tcphdr *)rbuf->tcph)))
	+ i40iw_handle_fin_pkt(cm_node);
	+ break;
	+ }
	+}
	+
	+/**
	+ * i40iw_make_listen_node - create a listen node with params
	+ * @cm_core: cm's core
	+ * @iwdev: iwarp device structure
	+ * @cm_info: quad info for connection
	+ */
	+static struct i40iw_cm_listener *
	+i40iw_make_listen_node(struct i40iw_cm_core *cm_core,
	+ struct i40iw_device iwdev, struct i40iw_cm_info cm_info)
	+{
	+ struct i40iw_cm_listener *listener;
	+ unsigned long flags;
	+
	+ /* cannot have multiple matching listeners */
	+ listener = i40iw_find_listener(cm_core, cm_info->loc_addr,
	+ cm_info->loc_port,
	+ cm_info->vlan_id,
	+ I40IW_CM_LISTENER_EITHER_STATE);
	+ if (listener &&
	+ (listener->listener_state == I40IW_CM_LISTENER_ACTIVE_STATE)) {
	+ atomic_dec(&listener->ref_count);
	+ i40iw_debug(cm_core->dev, I40IW_DEBUG_CM,
	+ "Not creating listener since it already exists\n");
	+ return NULL;
	+ }
	+ if (!listener) {
	+ /*
	+ * create a CM listen node (1/2 node to compare incoming
	+ * traffic to)
	+ */
	+ listener = kzalloc(sizeof(*listener), GFP_ATOMIC);
	+ if (!listener)
	+ return NULL;
	+ cm_core->stats_listen_nodes_created++;
	+ memcpy(listener->loc_addr, cm_info->loc_addr,
	+ sizeof(listener->loc_addr));
	+ listener->loc_port = cm_info->loc_port;
	+
	+ INIT_LIST_HEAD(&listener->child_listen_list);
	+
	+ atomic_set(&listener->ref_count, 1);
	+ } else {
	+ listener->reused_node = 1;
	+ }
	+
	+ listener->cm_id = cm_info->cm_id;
	+ listener->ipv4 = cm_info->ipv4;
	+ listener->vlan_id = cm_info->vlan_id;
	+ atomic_set(&listener->pend_accepts_cnt, 0);
	+ listener->cm_core = cm_core;
	+ listener->iwdev = iwdev;
	+
	+ listener->backlog = cm_info->backlog;
	+ listener->listener_state = I40IW_CM_LISTENER_ACTIVE_STATE;
	+
	+ if (!listener->reused_node) {
	+ spin_lock_irqsave(&cm_core->listen_list_lock, flags);
	+ list_add(&listener->list, &cm_core->listen_nodes);
	+ spin_unlock_irqrestore(&cm_core->listen_list_lock, flags);
	+ }
	+ return listener;
	+}
	+
	+/**
	+ * i40iw_create_cm_node - make a connection node with params
	+ * @cm_core: cm's core
	+ * @iwdev: iwarp device structure
	+ * @conn_param: upper layer connection parameters
	+ * @cm_info: quad info for connection
	+ */
	+static struct i40iw_cm_node *
	+i40iw_create_cm_node(struct i40iw_cm_core cm_core, struct i40iw_device iwdev,
	+ struct iw_cm_conn_param conn_param, struct i40iw_cm_info cm_info)
	+{
	+ struct i40iw_cm_node *cm_node;
	+ struct i40iw_cm_listener *loopback_remotelistener;
	+ struct i40iw_cm_node *loopback_remotenode;
	+ struct i40iw_cm_info loopback_cm_info;
	+ u16 private_data_len = conn_param->private_data_len;
	+ const void *private_data = conn_param->private_data;
	+
	+ /* create a CM connection node */
	+ cm_node = i40iw_make_cm_node(cm_core, iwdev, cm_info, NULL);
	+ if (!cm_node)
	+ return ERR_PTR(-ENOMEM);
	+ /* set our node side to client (active) side */
	+ cm_node->tcp_cntxt.client = 1;
	+ cm_node->tcp_cntxt.rcv_wscale = I40IW_CM_DEFAULT_RCV_WND_SCALE;
	+ i40iw_record_ird_ord(cm_node, conn_param->ird, conn_param->ord);
	+
	+ if (!memcmp(cm_info->loc_addr, cm_info->rem_addr, sizeof(cm_info->loc_addr))) {
	+ loopback_remotelistener = i40iw_find_listener(cm_core,
	+ cm_info->rem_addr, cm_node->rem_port, cm_node->vlan_id,
	+ I40IW_CM_LISTENER_ACTIVE_STATE);
	+ if (!loopback_remotelistener) {
	+ i40iw_rem_ref_cm_node(cm_node);
	+ return ERR_PTR(-ECONNREFUSED);
	+ } else {
	+ loopback_cm_info = *cm_info;
	+ loopback_cm_info.loc_port = cm_info->rem_port;
	+ loopback_cm_info.rem_port = cm_info->loc_port;
	+ loopback_cm_info.cm_id = loopback_remotelistener->cm_id;
	+ loopback_cm_info.ipv4 = cm_info->ipv4;
	+ loopback_remotenode = i40iw_make_cm_node(cm_core, iwdev,
	+ &loopback_cm_info, loopback_remotelistener);
	+ if (!loopback_remotenode) {
	+ i40iw_rem_ref_cm_node(cm_node);
	+ return ERR_PTR(-ENOMEM);
	+ }
	+ cm_core->stats_loopbacks++;
	+ loopback_remotenode->loopbackpartner = cm_node;
	+ loopback_remotenode->tcp_cntxt.rcv_wscale =
	+ I40IW_CM_DEFAULT_RCV_WND_SCALE;
	+ cm_node->loopbackpartner = loopback_remotenode;
	+ memcpy(loopback_remotenode->pdata_buf, private_data,
	+ private_data_len);
	+ loopback_remotenode->pdata.size = private_data_len;
	+ if (loopback_remotenode->ord_size > cm_node->ird_size)
	+ loopback_remotenode->ord_size = cm_node->ird_size;
	+
	+ cm_node->state = I40IW_CM_STATE_OFFLOADED;
	+ cm_node->tcp_cntxt.rcv_nxt =
	+ loopback_remotenode->tcp_cntxt.loc_seq_num;
	+ loopback_remotenode->tcp_cntxt.rcv_nxt =
	+ cm_node->tcp_cntxt.loc_seq_num;
	+ cm_node->tcp_cntxt.max_snd_wnd =
	+ loopback_remotenode->tcp_cntxt.rcv_wnd;
	+ loopback_remotenode->tcp_cntxt.max_snd_wnd =
	+ cm_node->tcp_cntxt.rcv_wnd;
	+ cm_node->tcp_cntxt.snd_wnd =
	+ loopback_remotenode->tcp_cntxt.rcv_wnd;
	+ loopback_remotenode->tcp_cntxt.snd_wnd =
	+ cm_node->tcp_cntxt.rcv_wnd;
	+ cm_node->tcp_cntxt.snd_wscale =
	+ loopback_remotenode->tcp_cntxt.rcv_wscale;
	+ loopback_remotenode->tcp_cntxt.snd_wscale =
	+ cm_node->tcp_cntxt.rcv_wscale;
	+ }
	+ return cm_node;
	+ }
	+ cm_node->pdata.size = private_data_len;
	+ cm_node->pdata.addr = cm_node->pdata_buf;
	+
	+ memcpy(cm_node->pdata_buf, private_data, private_data_len);
	+ return cm_node;
	+}
	+
	+/**
	+ * i40iw_cm_reject - reject and teardown a connection
	+ * @cm_node: connection's node
	+ * @pdate: ptr to private data for reject
	+ * @plen: size of private data
	+ */
	+static int
	+i40iw_cm_reject(struct i40iw_cm_node cm_node, const void pdata, u8 plen)
	+{
	+ struct iw_cm_id *cm_id = cm_node->cm_id;
	+ struct i40iw_cm_node *loopback = cm_node->loopbackpartner;
	+ int err;
	+ int passive_state;
	+ int ret = 0;
	+
	+ if (cm_node->tcp_cntxt.client)
	+ return ret;
	+ i40iw_cleanup_retrans_entry(cm_node);
	+
	+ if (!loopback) {
	+ passive_state = atomic_add_return(1, &cm_node->passive_state);
	+ if (passive_state == I40IW_SEND_RESET_EVENT) {
	+ cm_node->state = I40IW_CM_STATE_CLOSED;
	+ i40iw_rem_ref_cm_node(cm_node);
	+ } else {
	+ if (cm_node->state == I40IW_CM_STATE_LISTENER_DESTROYED) {
	+ i40iw_rem_ref_cm_node(cm_node);
	+ } else {
	+ ret = i40iw_send_mpa_reject(cm_node, pdata, plen);
	+ if (ret) {
	+ cm_node->state = I40IW_CM_STATE_CLOSED;
	+ err = i40iw_send_reset(cm_node);
	+ if (err)
	+ i40iw_pr_err("send reset failed\n");
	+ } else {
	+ cm_id->add_ref(cm_id);
	+ }
	+ }
	+ }
	+ } else {
	+ cm_node->cm_id = NULL;
	+ if (cm_node->state == I40IW_CM_STATE_LISTENER_DESTROYED) {
	+ i40iw_rem_ref_cm_node(cm_node);
	+ i40iw_rem_ref_cm_node(loopback);
	+ } else {
	+ ret = i40iw_send_cm_event(loopback,
	+ loopback->cm_id,
	+ IW_CM_EVENT_CONNECT_REPLY,
	+ -ECONNREFUSED);
	+ i40iw_rem_ref_cm_node(cm_node);
	+ loopback->state = I40IW_CM_STATE_CLOSING;
	+
	+ cm_id = loopback->cm_id;
	+ i40iw_rem_ref_cm_node(loopback);
	+ cm_id->rem_ref(cm_id);
	+ }
	+ }
	+
	+ return ret;
	+}
	+
	+/**
	+ * i40iw_cm_close - close of cm connection
	+ * @cm_node: connection's node
	+ */
	+static int
	+i40iw_cm_close(struct i40iw_cm_node *cm_node)
	+{
	+ int ret = 0;
	+
	+ if (!cm_node)
	+ return -EINVAL;
	+
	+ switch (cm_node->state) {
	+ case I40IW_CM_STATE_SYN_RCVD:
	+ case I40IW_CM_STATE_SYN_SENT:
	+ case I40IW_CM_STATE_ONE_SIDE_ESTABLISHED:
	+ case I40IW_CM_STATE_ESTABLISHED:
	+ case I40IW_CM_STATE_ACCEPTING:
	+ case I40IW_CM_STATE_MPAREQ_SENT:
	+ case I40IW_CM_STATE_MPAREQ_RCVD:
	+ i40iw_cleanup_retrans_entry(cm_node);
	+ i40iw_send_reset(cm_node);
	+ break;
	+ case I40IW_CM_STATE_CLOSE_WAIT:
	+ cm_node->state = I40IW_CM_STATE_LAST_ACK;
	+ i40iw_send_fin(cm_node);
	+ break;
	+ case I40IW_CM_STATE_FIN_WAIT1:
	+ case I40IW_CM_STATE_FIN_WAIT2:
	+ case I40IW_CM_STATE_LAST_ACK:
	+ case I40IW_CM_STATE_TIME_WAIT:
	+ case I40IW_CM_STATE_CLOSING:
	+ ret = -1;
	+ break;
	+ case I40IW_CM_STATE_LISTENING:
	+ i40iw_cleanup_retrans_entry(cm_node);
	+ i40iw_send_reset(cm_node);
	+ break;
	+ case I40IW_CM_STATE_MPAREJ_RCVD:
	+ case I40IW_CM_STATE_UNKNOWN:
	+ case I40IW_CM_STATE_INITED:
	+ case I40IW_CM_STATE_CLOSED:
	+ case I40IW_CM_STATE_LISTENER_DESTROYED:
	+ i40iw_rem_ref_cm_node(cm_node);
	+ break;
	+ case I40IW_CM_STATE_OFFLOADED:
	+ if (cm_node->send_entry)
	+ i40iw_pr_err("send_entry\n");
	+ i40iw_rem_ref_cm_node(cm_node);
	+ break;
	+ }
	+ return ret;
	+}
	+
	+/**
	+ * i40iw_receive_ilq - recv an ETHERNET packet, and process it
	+ * through CM
	+ * @dev: FPK dev struct
	+ * @rbuf: receive buffer
	+ */
	+void
	+i40iw_receive_ilq(struct i40iw_sc_vsi vsi, struct i40iw_puda_buf rbuf)
	+{
	+ struct i40iw_cm_node *cm_node;
	+ struct i40iw_cm_listener *listener;
	+ struct ip *iph;
	+#ifdef IPV6_SUPPORT
	+ struct ip6_hdr *ip6h;
	+#endif
	+ struct tcphdr *tcph;
	+ struct i40iw_cm_info cm_info;
	+ struct i40iw_sc_dev *dev = vsi->dev;
	+ struct i40iw_device iwdev = (struct i40iw_device )dev->back_dev;
	+ struct i40iw_cm_core *cm_core = &iwdev->cm_core;
	+ struct ether_vlan_header *ethh;
	+
	+ /* if vlan, then maclen = 18 else 14 */
	+ memset(&cm_info, 0, sizeof(cm_info));
	+
	+ i40iw_debug_buf(dev, I40IW_DEBUG_ILQ, "RECEIVE ILQ BUFFER", rbuf->mem.va,
	+ rbuf->totallen);
	+ ethh = (struct ether_vlan_header *)rbuf->mem.va;
	+
	+ if (ethh->evl_encap_proto == htons(ETH_P_8021Q)) {
	+ cm_info.vlan_id = ntohs(ethh->evl_tag) & EVL_VLID_MASK;
	+ i40iw_debug(cm_core->dev, I40IW_DEBUG_CM, "%s vlan_id=%d\n", __func__,
	+ cm_info.vlan_id);
	+ } else {
	+ cm_info.vlan_id = I40IW_NO_VLAN;
	+ }
	+#ifdef VIMAGE_READY
	+ memcpy(cm_info.loc_dev_addr, ethh->evl_dhost, ETHER_ADDR_LEN);
	+ memcpy(cm_info.rem_dev_addr, ethh->evl_shost, ETHER_ADDR_LEN);
	+#endif
	+ tcph = (struct tcphdr *)rbuf->tcph;
	+
	+ cm_info.ipv4 = rbuf->ipv4;
	+ iph = (struct ip *)rbuf->iph;
	+#ifdef IPV6_SUPPORT
	+ ip6h = (struct ip6_hdr *)rbuf->iph;
	+
	+ if (cm_info.ipv4) {
	+ cm_info.loc_addr[0] = ntohl(iph->ip_dst.s_addr);
	+ cm_info.rem_addr[0] = ntohl(iph->ip_src.s_addr);
	+ } else {
	+ i40iw_copy_ip_ntohl(cm_info.loc_addr, (__be32 *)&ip6h->ip6_dst);
	+ i40iw_copy_ip_ntohl(cm_info.rem_addr, (__be32 *)&ip6h->ip6_src);
	+ }
	+#else
	+ cm_info.loc_addr[0] = ntohl(iph->ip_dst.s_addr);
	+ cm_info.rem_addr[0] = ntohl(iph->ip_src.s_addr);
	+#endif
	+ cm_info.loc_port = ntohs(tcph->th_dport);
	+ cm_info.rem_port = ntohs(tcph->th_sport);
	+ cm_node = i40iw_find_node(cm_core, cm_info.rem_port, cm_info.rem_addr,
	+ cm_info.loc_port, cm_info.loc_addr, TRUE);
	+
	+ if (!cm_node) {
	+ /* Only type of packet accepted are for */
	+ /* the PASSIVE open (syn only) */
	+ if (!(tcph->th_flags & TH_SYN) \|\| (tcph->th_flags & TH_ACK))
	+ return;
	+ listener = i40iw_find_listener(cm_core,
	+ cm_info.loc_addr, cm_info.loc_port, cm_info.vlan_id,
	+ I40IW_CM_LISTENER_ACTIVE_STATE);
	+ if (!listener) {
	+ cm_info.cm_id = NULL;
	+ i40iw_debug(cm_core->dev, I40IW_DEBUG_CM, "%s no listener found\n",
	+ __func__);
	+ return;
	+ }
	+ cm_info.cm_id = listener->cm_id;
	+ cm_node = i40iw_make_cm_node(cm_core, iwdev, &cm_info, listener);
	+ if (!cm_node) {
	+ i40iw_debug(cm_core->dev, I40IW_DEBUG_CM,
	+ "%s allocate node failed\n", __func__);
	+ atomic_dec(&listener->ref_count);
	+ return;
	+ }
	+ if (!(tcph->th_flags & (TH_RST \| TH_FIN))) {
	+ cm_node->state = I40IW_CM_STATE_LISTENING;
	+ } else {
	+ i40iw_rem_ref_cm_node(cm_node);
	+ return;
	+ }
	+ atomic_inc(&cm_node->ref_count);
	+ } else if (cm_node->state == I40IW_CM_STATE_OFFLOADED) {
	+ i40iw_rem_ref_cm_node(cm_node);
	+ return;
	+ }
	+ i40iw_process_packet(cm_node, rbuf);
	+ i40iw_rem_ref_cm_node(cm_node);
	+}
	+
	+/**
	+ * i40iw_setup_cm_core - allocate a top level instance of a cm
	+ * core
	+ * @iwdev: iwarp device structure
	+ */
	+void
	+i40iw_setup_cm_core(struct i40iw_device *iwdev)
	+{
	+ struct i40iw_cm_core *cm_core = &iwdev->cm_core;
	+
	+ cm_core->iwdev = iwdev;
	+ cm_core->dev = &iwdev->sc_dev;
	+
	+ INIT_LIST_HEAD(&cm_core->connected_nodes);
	+ INIT_LIST_HEAD(&cm_core->listen_nodes);
	+
	+ init_timer(&cm_core->tcp_timer);
	+ cm_core->tcp_timer.function = i40iw_cm_timer_tick;
	+ cm_core->tcp_timer.data = (unsigned long)cm_core;
	+
	+ spin_lock_init(&cm_core->ht_lock);
	+ spin_lock_init(&cm_core->listen_list_lock);
	+
	+ cm_core->event_wq = create_singlethread_workqueue("iwewq");
	+ cm_core->disconn_wq = create_singlethread_workqueue("iwdwq");
	+}
	+
	+/**
	+ * i40iw_cleanup_cm_core - deallocate a top level instance of a
	+ * cm core
	+ * @cm_core: cm's core
	+ */
	+void
	+i40iw_cleanup_cm_core(struct i40iw_cm_core *cm_core)
	+{
	+ unsigned long flags;
	+
	+ if (!cm_core)
	+ return;
	+
	+ spin_lock_irqsave(&cm_core->ht_lock, flags);
	+ if (timer_pending(&cm_core->tcp_timer))
	+ del_timer_sync(&cm_core->tcp_timer);
	+ spin_unlock_irqrestore(&cm_core->ht_lock, flags);
	+
	+ destroy_workqueue(cm_core->event_wq);
	+ destroy_workqueue(cm_core->disconn_wq);
	+}
	+
	+/**
	+ * i40iw_init_tcp_ctx - setup qp context
	+ * @cm_node: connection's node
	+ * @tcp_info: offload info for tcp
	+ * @iwqp: associate qp for the connection
	+ */
	+static void
	+i40iw_init_tcp_ctx(struct i40iw_cm_node *cm_node,
	+ struct i40iw_tcp_offload_info tcp_info, struct i40iw_qp iwqp)
	+{
	+ tcp_info->ipv4 = cm_node->ipv4;
	+ tcp_info->drop_ooo_seg = TRUE;
	+ tcp_info->wscale = TRUE;
	+ tcp_info->ignore_tcp_opt = TRUE;
	+ tcp_info->ignore_tcp_uns_opt = TRUE;
	+ tcp_info->no_nagle = FALSE;
	+
	+ tcp_info->ttl = I40IW_DEFAULT_TTL;
	+ tcp_info->rtt_var = cpu_to_le32(I40IW_DEFAULT_RTT_VAR);
	+ tcp_info->ss_thresh = cpu_to_le32(I40IW_DEFAULT_SS_THRESH);
	+ tcp_info->rexmit_thresh = I40IW_DEFAULT_REXMIT_THRESH;
	+
	+ tcp_info->tcp_state = I40IW_TCP_STATE_ESTABLISHED;
	+ tcp_info->snd_wscale = cm_node->tcp_cntxt.snd_wscale;
	+ tcp_info->rcv_wscale = cm_node->tcp_cntxt.rcv_wscale;
	+
	+ tcp_info->snd_nxt = cpu_to_le32(cm_node->tcp_cntxt.loc_seq_num);
	+ tcp_info->snd_wnd = cpu_to_le32(cm_node->tcp_cntxt.snd_wnd);
	+ tcp_info->rcv_nxt = cpu_to_le32(cm_node->tcp_cntxt.rcv_nxt);
	+ tcp_info->snd_max = cpu_to_le32(cm_node->tcp_cntxt.loc_seq_num);
	+
	+ tcp_info->snd_una = cpu_to_le32(cm_node->tcp_cntxt.loc_seq_num);
	+ tcp_info->cwnd = cpu_to_le32(2 * cm_node->tcp_cntxt.mss);
	+ tcp_info->snd_wl1 = cpu_to_le32(cm_node->tcp_cntxt.rcv_nxt);
	+ tcp_info->snd_wl2 = cpu_to_le32(cm_node->tcp_cntxt.loc_seq_num);
	+ tcp_info->max_snd_window = cpu_to_le32(cm_node->tcp_cntxt.max_snd_wnd);
	+ tcp_info->rcv_wnd = cpu_to_le32(cm_node->tcp_cntxt.rcv_wnd <<
	+ cm_node->tcp_cntxt.rcv_wscale);
	+
	+ tcp_info->flow_label = 0;
	+ tcp_info->snd_mss = cpu_to_le32(((u32)cm_node->tcp_cntxt.mss));
	+ if (cm_node->vlan_id < VLAN_TAG_PRESENT) {
	+ tcp_info->insert_vlan_tag = TRUE;
	+ tcp_info->vlan_tag = cpu_to_le16(
	+ ((u16)cm_node->user_pri << I40IW_VLAN_PRIO_SHIFT) \|
	+ cm_node->vlan_id);
	+ }
	+ if (cm_node->ipv4) {
	+ tcp_info->src_port = cpu_to_le16(cm_node->loc_port);
	+ tcp_info->dst_port = cpu_to_le16(cm_node->rem_port);
	+
	+ tcp_info->dest_ip_addr3 = cpu_to_le32(cm_node->rem_addr[0]);
	+ tcp_info->local_ipaddr3 = cpu_to_le32(cm_node->loc_addr[0]);
	+ tcp_info->arp_idx = cpu_to_le16((u16)i40iw_arp_table(iwqp->iwdev,
	+ &tcp_info->dest_ip_addr3, TRUE, NULL, I40IW_ARP_RESOLVE));
	+ }
	+#ifdef IPV6_SUPPORT
	+ else {
	+ tcp_info->src_port = cpu_to_le16(cm_node->loc_port);
	+ tcp_info->dst_port = cpu_to_le16(cm_node->rem_port);
	+ tcp_info->dest_ip_addr0 = cpu_to_le32(cm_node->rem_addr[0]);
	+ tcp_info->dest_ip_addr1 = cpu_to_le32(cm_node->rem_addr[1]);
	+ tcp_info->dest_ip_addr2 = cpu_to_le32(cm_node->rem_addr[2]);
	+ tcp_info->dest_ip_addr3 = cpu_to_le32(cm_node->rem_addr[3]);
	+ tcp_info->local_ipaddr0 = cpu_to_le32(cm_node->loc_addr[0]);
	+ tcp_info->local_ipaddr1 = cpu_to_le32(cm_node->loc_addr[1]);
	+ tcp_info->local_ipaddr2 = cpu_to_le32(cm_node->loc_addr[2]);
	+ tcp_info->local_ipaddr3 = cpu_to_le32(cm_node->loc_addr[3]);
	+ tcp_info->arp_idx = cpu_to_le16((u16)i40iw_arp_table(iwqp->iwdev,
	+ &tcp_info->dest_ip_addr0, FALSE, NULL, I40IW_ARP_RESOLVE));
	+ }
	+#endif
	+}
	+
	+/**
	+ * i40iw_cm_init_tsa_conn - setup qp for RTS
	+ * @iwqp: associate qp for the connection
	+ * @cm_node: connection's node
	+ */
	+static void
	+i40iw_cm_init_tsa_conn(struct i40iw_qp iwqp, struct i40iw_cm_node cm_node)
	+{
	+ struct i40iw_tcp_offload_info tcp_info;
	+ struct i40iwarp_offload_info *iwarp_info;
	+ struct i40iw_qp_host_ctx_info *ctx_info;
	+ struct i40iw_device *iwdev = iwqp->iwdev;
	+ struct i40iw_sc_dev *dev = &iwqp->iwdev->sc_dev;
	+
	+ memset(&tcp_info, 0x00, sizeof(struct i40iw_tcp_offload_info));
	+ iwarp_info = &iwqp->iwarp_info;
	+ ctx_info = &iwqp->ctx_info;
	+
	+ ctx_info->tcp_info = &tcp_info;
	+ ctx_info->send_cq_num = iwqp->iwscq->sc_cq.cq_uk.cq_id;
	+ ctx_info->rcv_cq_num = iwqp->iwrcq->sc_cq.cq_uk.cq_id;
	+
	+ iwarp_info->ord_size = cm_node->ord_size;
	+ iwarp_info->ird_size = i40iw_derive_hw_ird_setting(cm_node->ird_size);
	+
	+ if (iwarp_info->ord_size == 1)
	+ iwarp_info->ord_size = 2;
	+
	+ iwarp_info->rd_enable = TRUE;
	+ iwarp_info->rdmap_ver = 1;
	+ iwarp_info->ddp_ver = 1;
	+
	+ iwarp_info->pd_id = iwqp->iwpd->sc_pd.pd_id;
	+
	+ ctx_info->tcp_info_valid = TRUE;
	+ ctx_info->iwarp_info_valid = TRUE;
	+
	+ i40iw_init_tcp_ctx(cm_node, &tcp_info, iwqp);
	+ if (cm_node->snd_mark_en) {
	+ iwarp_info->snd_mark_en = TRUE;
	+ iwarp_info->snd_mark_offset = (tcp_info.snd_nxt &
	+ SNDMARKER_SEQNMASK) + cm_node->lsmm_size;
	+ }
	+ cm_node->state = I40IW_CM_STATE_OFFLOADED;
	+ tcp_info.tcp_state = I40IW_TCP_STATE_ESTABLISHED;
	+ tcp_info.src_mac_addr_idx = iwdev->mac_ip_table_idx;
	+
	+ dev->iw_priv_qp_ops->qp_setctx(&iwqp->sc_qp, (u64 *)(iwqp->host_ctx.va),
	+ ctx_info);
	+
	+ /* once tcp_info is set, no need to do it again */
	+ ctx_info->tcp_info_valid = FALSE;
	+ ctx_info->iwarp_info_valid = FALSE;
	+}
	+
	+/**
	+ * i40iw_cm_disconn - when a connection is being closed
	+ * @iwqp: associate qp for the connection
	+ */
	+void
	+i40iw_cm_disconn(struct i40iw_qp *iwqp)
	+{
	+ struct i40iw_disconn_work *work;
	+ struct i40iw_device *iwdev = iwqp->iwdev;
	+ struct i40iw_cm_core *cm_core = &iwdev->cm_core;
	+ unsigned long flags;
	+
	+ work = kzalloc(sizeof(*work), GFP_ATOMIC);
	+ if (!work)
	+ return; /* Timer will clean up */
	+
	+ spin_lock_irqsave(&iwdev->qptable_lock, flags);
	+ if (!iwdev->qp_table[iwqp->ibqp.qp_num]) {
	+ spin_unlock_irqrestore(&iwdev->qptable_lock, flags);
	+ i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_CM,
	+ "%s qp_id %d is already freed\n",
	+ __func__, iwqp->ibqp.qp_num);
	+ kfree(work);
	+ return;
	+ }
	+ i40iw_add_ref(&iwqp->ibqp);
	+ spin_unlock_irqrestore(&iwdev->qptable_lock, flags);
	+
	+ work->iwqp = iwqp;
	+ INIT_WORK(&work->work, i40iw_disconnect_worker);
	+ queue_work(cm_core->disconn_wq, &work->work);
	+ return;
	+}
	+
	+/**
	+ * i40iw_qp_disconnect - free qp and close cm
	+ * @iwqp: associate qp for the connection
	+ */
	+static void
	+i40iw_qp_disconnect(struct i40iw_qp *iwqp)
	+{
	+ struct i40iw_device *iwdev;
	+ struct i40iw_ib_device *iwibdev;
	+
	+ iwdev = to_iwdev(iwqp->ibqp.device);
	+ if (!iwdev) {
	+ i40iw_pr_err("iwdev == NULL\n");
	+ return;
	+ }
	+ iwibdev = iwdev->iwibdev;
	+
	+ if (iwqp->active_conn) {
	+ /* indicate this connection is NOT active */
	+ iwqp->active_conn = 0;
	+ } else {
	+ /* Need to free the Last Streaming Mode Message */
	+ if (iwqp->ietf_mem.va) {
	+ if (iwqp->lsmm_mr)
	+ iwibdev->ibdev.dereg_mr(iwqp->lsmm_mr);
	+ ixliw_free_dma_mem(&iwqp->ietf_mem);
	+ }
	+ }
	+
	+ /* close the CM node down if it is still active */
	+ if (iwqp->cm_node) {
	+ i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_CM, "%s Call close API\n",
	+ __func__);
	+ i40iw_cm_close(iwqp->cm_node);
	+ }
	+}
	+
	+/**
	+ * i40iw_cm_disconn_true - called by worker thread to disconnect qp
	+ * @iwqp: associate qp for the connection
	+ */
	+static void
	+i40iw_cm_disconn_true(struct i40iw_qp *iwqp)
	+{
	+ struct iw_cm_id *cm_id;
	+ struct i40iw_device *iwdev;
	+ struct ib_event ibevent;
	+ struct i40iw_sc_qp *qp = &iwqp->sc_qp;
	+ int disconn_status = 0;
	+ int issue_disconn = 0;
	+ int issue_close = 0;
	+ int issue_flush = 0;
	+ int ret;
	+ unsigned long flags;
	+ u16 last_ae;
	+ u8 original_hw_tcp_state;
	+ u8 original_ibqp_state;
	+
	+ if (!iwqp) {
	+ i40iw_pr_err("iwqp == NULL\n");
	+ return;
	+ }
	+ spin_lock_irqsave(&iwqp->lock, flags);
	+ cm_id = iwqp->cm_id;
	+ /* make sure we havent already closed this connection */
	+ if (!cm_id) {
	+ spin_unlock_irqrestore(&iwqp->lock, flags);
	+ return;
	+ }
	+ iwdev = to_iwdev(iwqp->ibqp.device);
	+
	+ original_hw_tcp_state = iwqp->hw_tcp_state;
	+ original_ibqp_state = iwqp->ibqp_state;
	+ last_ae = iwqp->last_aeq;
	+ if (qp->term_flags) {
	+ issue_disconn = 1;
	+ issue_close = 1;
	+ iwqp->cm_id = NULL;
	+ del_timer(&iwqp->terminate_timer);
	+ if (!iwqp->flush_issued) {
	+ iwqp->flush_issued = 1;
	+ issue_flush = 1;
	+ }
	+ } else if ((original_hw_tcp_state == I40IW_TCP_STATE_CLOSE_WAIT) \|\|
	+ ((original_ibqp_state == IB_QPS_RTS) &&
	+ (last_ae == I40IW_AE_LLP_CONNECTION_RESET))) {
	+ issue_disconn = 1;
	+ if (last_ae == I40IW_AE_LLP_CONNECTION_RESET)
	+ disconn_status = -ECONNRESET;
	+ }
	+ if (((original_hw_tcp_state == I40IW_TCP_STATE_CLOSED) \|\|
	+ (original_hw_tcp_state == I40IW_TCP_STATE_TIME_WAIT) \|\|
	+ (last_ae == I40IW_AE_RDMAP_ROE_BAD_LLP_CLOSE) \|\|
	+ (last_ae == I40IW_AE_LLP_CONNECTION_RESET))) {
	+ issue_close = 1;
	+ iwqp->cm_id = NULL;
	+ if (!iwqp->flush_issued) {
	+ iwqp->flush_issued = 1;
	+ issue_flush = 1;
	+ }
	+ }
	+ spin_unlock_irqrestore(&iwqp->lock, flags);
	+ if (issue_flush && !iwqp->destroyed) {
	+ /* Flush the queues */
	+ i40iw_flush_wqes(iwdev, iwqp);
	+ if (qp->term_flags && iwqp->ibqp.event_handler) {
	+ ibevent.device = iwqp->ibqp.device;
	+ ibevent.event = (qp->eventtype == TERM_EVENT_QP_FATAL) ?
	+ IB_EVENT_QP_FATAL : IB_EVENT_QP_ACCESS_ERR;
	+ ibevent.element.qp = &iwqp->ibqp;
	+ iwqp->ibqp.event_handler(&ibevent, iwqp->ibqp.qp_context);
	+ }
	+ }
	+ if (cm_id && cm_id->event_handler) {
	+ if (issue_disconn) {
	+ ret = i40iw_send_cm_event(NULL,
	+ cm_id,
	+ IW_CM_EVENT_DISCONNECT,
	+ disconn_status);
	+
	+ if (ret)
	+ i40iw_debug(&iwdev->sc_dev,
	+ I40IW_DEBUG_CM,
	+ "disconnect event failed %s: - cm_id = %p\n",
	+ __func__, cm_id);
	+ }
	+ if (issue_close) {
	+ i40iw_qp_disconnect(iwqp);
	+ cm_id->provider_data = iwqp;
	+ ret = i40iw_send_cm_event(NULL, cm_id, IW_CM_EVENT_CLOSE, 0);
	+ if (ret)
	+ i40iw_debug(&iwdev->sc_dev,
	+ I40IW_DEBUG_CM,
	+ "close event failed %s: - cm_id = %p\n",
	+ __func__, cm_id);
	+ cm_id->rem_ref(cm_id);
	+ }
	+ }
	+}
	+
	+/**
	+ * i40iw_disconnect_worker - worker for connection close
	+ * @work: points or disconn structure
	+ */
	+static void
	+i40iw_disconnect_worker(struct work_struct *work)
	+{
	+ struct i40iw_disconn_work *dwork =
	+ container_of(work, struct i40iw_disconn_work, work);
	+ struct i40iw_qp *iwqp = dwork->iwqp;
	+
	+ kfree(dwork);
	+
	+ i40iw_cm_disconn_true(iwqp);
	+ i40iw_rem_ref(&iwqp->ibqp);
	+}
	+
	+/**
	+ * i40iw_accept - registered call for connection to be accepted
	+ * @cm_id: cm information for passive connection
	+ * @conn_param: accpet parameters
	+ */
	+int
	+i40iw_accept(struct iw_cm_id cm_id, struct iw_cm_conn_param conn_param)
	+{
	+ struct ib_qp_attr attr;
	+ struct i40iw_cm_node *cm_node;
	+ struct i40iw_sc_dev *dev;
	+ struct ib_mr *ibmr;
	+ struct ib_qp *ibqp;
	+ struct i40iw_device *iwdev;
	+ struct i40iw_qp *iwqp;
	+ struct i40iw_pd *iwpd;
	+ struct i40iw_kmem_info accept;
	+#ifndef IW_IXL_RDMA_4_9
	+ struct ib_phys_buf phys_buf;
	+#endif
	+ u64 tagged_offset;
	+ int passive_state;
	+ u16 buf_len = 0;
	+ enum i40iw_status_code status;
	+
	+ memset(&attr, 0, sizeof(attr));
	+ ibqp = i40iw_get_qp(cm_id->device, conn_param->qpn);
	+ if (!ibqp)
	+ return -EINVAL;
	+
	+ iwqp = to_iwqp(ibqp);
	+ iwdev = iwqp->iwdev;
	+ dev = &iwdev->sc_dev;
	+ cm_node = (struct i40iw_cm_node *)cm_id->provider_data;
	+
	+ if (((struct sockaddr_in *)&cm_id->local_addr)->sin_family == AF_INET) {
	+ cm_node->ipv4 = TRUE;
	+ cm_node->vlan_id = i40iw_get_vlan_ipv4(cm_node->loc_addr);
	+ } else {
	+ cm_node->ipv4 = FALSE;
	+#ifdef IPV6_SUPPORT
	+ cm_node->vlan_id = i40iw_get_vlan_ipv6(cm_node->loc_addr);
	+#else
	+ return -EAFNOSUPPORT;
	+#endif
	+ }
	+ i40iw_debug(cm_node->dev, I40IW_DEBUG_CM, "Accept vlan_id=%d\n",
	+ cm_node->vlan_id);
	+ if (cm_node->state == I40IW_CM_STATE_LISTENER_DESTROYED) {
	+ if (cm_node->loopbackpartner)
	+ i40iw_rem_ref_cm_node(cm_node->loopbackpartner);
	+ i40iw_rem_ref_cm_node(cm_node);
	+ return -EINVAL;
	+ }
	+ passive_state = atomic_add_return(1, &cm_node->passive_state);
	+ if (passive_state == I40IW_SEND_RESET_EVENT) {
	+ i40iw_rem_ref_cm_node(cm_node);
	+ return -ECONNRESET;
	+ }
	+ cm_node->cm_core->stats_accepts++;
	+ iwqp->cm_node = (void *)cm_node;
	+ cm_node->iwqp = iwqp;
	+
	+ buf_len = conn_param->private_data_len + I40IW_MAX_IETF_SIZE;
	+
	+ status = i40iw_allocate_dma_mem(dev->hw, &iwqp->ietf_mem, buf_len, 1);
	+
	+ if (status)
	+ return -ENOMEM;
	+ cm_node->pdata.size = conn_param->private_data_len;
	+ accept.addr = iwqp->ietf_mem.va;
	+ accept.size = i40iw_cm_build_mpa_frame(cm_node, &accept, MPA_KEY_REPLY);
	+ memcpy((void *)((uintptr_t)accept.addr + accept.size),
	+ conn_param->private_data, conn_param->private_data_len);
	+
	+ /* setup our first outgoing iWarp send WQE (the IETF frame response) */
	+ if (!i40iw_is_loopback(cm_node->ipv4, cm_node->loc_addr, cm_node->rem_addr)) {
	+ iwpd = iwqp->iwpd;
	+ tagged_offset = (uintptr_t)iwqp->ietf_mem.va;
	+#ifndef IW_IXL_RDMA_4_9
	+ phys_buf.addr = iwqp->ietf_mem.pa;
	+ phys_buf.size = buf_len;
	+
	+ ibmr = i40iw_reg_phys_mr(&iwpd->ibpd, &phys_buf, 1,
	+ IB_ACCESS_LOCAL_WRITE, &tagged_offset);
	+#else
	+ ibmr = i40iw_reg_phys_mr(&iwpd->ibpd, iwqp->ietf_mem.pa, buf_len,
	+ IB_ACCESS_LOCAL_WRITE, &tagged_offset);
	+#endif
	+ if (IS_ERR(ibmr)) {
	+ ixliw_free_dma_mem(&iwqp->ietf_mem);
	+ return -ENOMEM;
	+ }
	+ ibmr->pd = &iwpd->ibpd;
	+ ibmr->device = iwpd->ibpd.device;
	+ iwqp->lsmm_mr = ibmr;
	+ dev->iw_priv_qp_ops->qp_send_lsmm(&iwqp->sc_qp, iwqp->ietf_mem.va,
	+ (accept.size + conn_param->private_data_len), ibmr->lkey);
	+
	+ } else {
	+ dev->iw_priv_qp_ops->qp_send_lsmm(&iwqp->sc_qp, NULL, 0, 0);
	+ }
	+
	+ iwqp->cm_id = cm_id;
	+ cm_node->cm_id = cm_id;
	+
	+ cm_id->provider_data = (void *)iwqp;
	+ iwqp->active_conn = 0;
	+
	+ cm_node->lsmm_size = accept.size + conn_param->private_data_len;
	+ i40iw_cm_init_tsa_conn(iwqp, cm_node);
	+ cm_id->add_ref(cm_id);
	+ i40iw_add_ref(&iwqp->ibqp);
	+
	+ attr.qp_state = IB_QPS_RTS;
	+ cm_node->qhash_set = FALSE;
	+ i40iw_modify_qp(&iwqp->ibqp, &attr, IB_QP_STATE, NULL);
	+
	+ cm_node->accelerated = 1;
	+ i40iw_send_cm_event(cm_node, cm_id, IW_CM_EVENT_ESTABLISHED, 0);
	+
	+ if (cm_node->loopbackpartner) {
	+ cm_node->loopbackpartner->pdata.size = conn_param->private_data_len;
	+
	+ /* copy entire MPA frame to our cm_node's frame */
	+ memcpy(cm_node->loopbackpartner->pdata_buf,
	+ conn_param->private_data,
	+ conn_param->private_data_len);
	+ i40iw_create_event(cm_node->loopbackpartner, I40IW_CM_EVENT_CONNECTED);
	+ }
	+ if (cm_node->accept_pend) {
	+ atomic_dec(&cm_node->listener->pend_accepts_cnt);
	+ cm_node->accept_pend = 0;
	+ }
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_reject - registered call for connection to be rejected
	+ * @cm_id: cm information for passive connection
	+ * @pdata: private data to be sent
	+ * @pdata_len: private data length
	+ */
	+int
	+i40iw_reject(struct iw_cm_id cm_id, const void pdata, u8 pdata_len)
	+{
	+ struct i40iw_device *iwdev;
	+ struct i40iw_cm_node *cm_node;
	+ struct i40iw_cm_node *loopback;
	+
	+ cm_node = (struct i40iw_cm_node *)cm_id->provider_data;
	+ loopback = cm_node->loopbackpartner;
	+ cm_node->cm_id = cm_id;
	+ cm_node->pdata.size = pdata_len;
	+
	+ iwdev = to_iwdev(cm_id->device);
	+ if (!iwdev)
	+ return -EINVAL;
	+ cm_node->cm_core->stats_rejects++;
	+
	+ if (pdata_len + sizeof(struct ietf_mpa_v2) > MAX_CM_BUFFER)
	+ return -EINVAL;
	+
	+ if (loopback) {
	+ memcpy(&loopback->pdata_buf, pdata, pdata_len);
	+ loopback->pdata.size = pdata_len;
	+ }
	+ return i40iw_cm_reject(cm_node, pdata, pdata_len);
	+}
	+
	+/**
	+ * i40iw_connect - registered call for connection to be established
	+ * @cm_id: cm information for passive connection
	+ * @conn_param: Information about the connection
	+ */
	+int
	+i40iw_connect(struct iw_cm_id cm_id, struct iw_cm_conn_param conn_param)
	+{
	+ struct i40iw_cm_info cm_info;
	+ struct i40iw_cm_node *cm_node;
	+ struct ib_qp *ibqp;
	+ struct i40iw_device *iwdev;
	+ struct i40iw_qp *iwqp;
	+#ifdef VIMAGE_READY
	+ struct rdma_cm_id rdma_id = (struct rdma_cm_id )cm_id->context;
	+ struct rdma_route *rdma_id_route;
	+#endif
	+#ifndef IW_IXL_RDMA_4_9
	+ struct rtentry *rte = NULL;
	+#endif
	+ struct sockaddr_in *laddr;
	+ struct sockaddr_in *raddr;
	+#if defined(IPV6_SUPPORT) \|\| defined(IW_IXL_RDMA_4_9)
	+ struct sockaddr_in6 *laddr6;
	+ struct sockaddr_in6 *raddr6;
	+#endif
	+ int ret = 0;
	+ u32 *local_address = NULL;
	+ u32 *remote_address = NULL;
	+ unsigned long flags;
	+#ifndef IW_IXL_RDMA_4_9
	+ u16 lport = 0;
	+#endif
	+#ifdef VIMAGE_READY
	+ unsigned char *dst_dev_addr;
	+ unsigned char *src_dev_addr;
	+#endif
	+
	+ ibqp = i40iw_get_qp(cm_id->device, conn_param->qpn);
	+ if (!ibqp)
	+ return -EINVAL;
	+ iwqp = to_iwqp(ibqp);
	+ if (!iwqp)
	+ return -EINVAL;
	+ iwdev = to_iwdev(iwqp->ibqp.device);
	+ if (!iwdev)
	+ return -EINVAL;
	+
	+#ifdef VIMAGE_READY
	+ rdma_id_route = &rdma_id->route;
	+ dst_dev_addr = rdma_id_route->addr.dev_addr.dst_dev_addr;
	+ src_dev_addr = rdma_id_route->addr.dev_addr.src_dev_addr;
	+#endif
	+#ifdef IW_IXL_RDMA_4_9
	+ if (cm_id->local_addr.ss_family == AF_INET && cm_id->remote_addr.ss_family == AF_INET) {
	+ laddr = (struct sockaddr_in *)&cm_id->local_addr;
	+ raddr = (struct sockaddr_in *)&cm_id->remote_addr;
	+ laddr6 = NULL;
	+ raddr6 = NULL;
	+ i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_CM, "Connect: local address: %u.%u.%u.%u:%u\n",
	+ ((u8 )&(laddr->sin_addr.s_addr))[0], ((u8 )&(laddr->sin_addr.s_addr))[1],
	+ ((u8 )&(laddr->sin_addr.s_addr))[2], ((u8 )&(laddr->sin_addr.s_addr))[3],
	+ laddr->sin_port);
	+ i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_CM, "Connect: remote address %d.%d.%d.%d:%u\n",
	+ ((u8 )&(raddr->sin_addr.s_addr))[0], ((u8 )&(raddr->sin_addr.s_addr))[1],
	+ ((u8 )&(raddr->sin_addr.s_addr))[2], ((u8 )&(raddr->sin_addr.s_addr))[3],
	+ raddr->sin_port);
	+ if (!(laddr->sin_port) \|\| !(raddr->sin_port))
	+ return -EINVAL;
	+
	+ } else if (cm_id->local_addr.ss_family == AF_INET6 && cm_id->remote_addr.ss_family == AF_INET6) {
	+ laddr = NULL;
	+ raddr = NULL;
	+ laddr6 = (struct sockaddr_in6 *)&cm_id->local_addr;
	+ raddr6 = (struct sockaddr_in6 *)&cm_id->remote_addr;
	+ i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_CM, "Connect: local port %u, remote port %u\n",
	+ laddr6->sin6_port, raddr6->sin6_port);
	+ if (!(laddr6->sin6_port) \|\| !(raddr6->sin6_port))
	+ return -EINVAL;
	+ } else {
	+ return -EFAULT;
	+ }
	+#else
	+ laddr = (struct sockaddr_in *)&cm_id->local_addr;
	+ raddr = (struct sockaddr_in *)&cm_id->remote_addr;
	+#ifdef IPV6_SUPPORT
	+ laddr6 = (struct sockaddr_in6 *)&cm_id->local_addr;
	+ raddr6 = (struct sockaddr_in6 *)&cm_id->remote_addr;
	+#endif
	+ if (cm_id->remote_addr.sin_family == AF_INET) {
	+ lport = laddr->sin_port;
	+ if (laddr->sin_addr.s_addr == 0) {
	+ rte = rtalloc1((struct sockaddr *)raddr, 1, 0);
	+ if (rte == NULL \|\| rte->rt_ifp == NULL \|\| !RT_LINK_IS_UP(rte->rt_ifp)) {
	+ if (rte)
	+ RTFREE_LOCKED(rte);
	+ return -EHOSTUNREACH;
	+ }
	+ memcpy(laddr, rte->rt_ifa->ifa_addr, ip_addr_size(rte->rt_ifa->ifa_addr));
	+ RTFREE_LOCKED(rte);
	+ }
	+ if (!lport) {
	+ lport = sotoinpcb(cm_id->so)->inp_lport;
	+ }
	+ laddr->sin_port = lport;
	+ i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_CM,
	+ "Connect: local address: %u.%u.%u.%u:%u\n",
	+ ((u8 *)&(laddr->sin_addr.s_addr))[0],
	+ ((u8 *)&(laddr->sin_addr.s_addr))[1],
	+ ((u8 *)&(laddr->sin_addr.s_addr))[2],
	+ ((u8 *)&(laddr->sin_addr.s_addr))[3],
	+ laddr->sin_port);
	+ i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_CM,
	+ "Connect: remote address %d.%d.%d.%d:%u\n",
	+ ((u8 *)&(raddr->sin_addr.s_addr))[0],
	+ ((u8 *)&(raddr->sin_addr.s_addr))[1],
	+ ((u8 *)&(raddr->sin_addr.s_addr))[2],
	+ ((u8 *)&(raddr->sin_addr.s_addr))[3],
	+ raddr->sin_port);
	+ } else {
	+#ifdef IPV6_SUPPORT
	+ if (IN6_IS_ADDR_UNSPECIFIED(&(laddr6->sin6_addr))) {
	+ printf("i40iw_connect inside if\n");
	+ rte = rtalloc1((struct sockaddr *)raddr6, 1, 0);
	+ if (rte == NULL \|\| rte->rt_ifp == NULL \|\| !RT_LINK_IS_UP(rte->rt_ifp)) {
	+ if (rte)
	+ RTFREE_LOCKED(rte);
	+ return -EHOSTUNREACH;
	+ }
	+ lport = laddr6->sin6_port;
	+ memcpy(laddr6, rte->rt_ifa->ifa_addr, ip_addr_size(rte->rt_ifa->ifa_addr));
	+ laddr6->sin6_port = lport;
	+ i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_CM,
	+ "Connect: local port %u, remote port %u\n",
	+ laddr6->sin6_port,
	+ raddr6->sin6_port);
	+ RTFREE_LOCKED(rte);
	+ }
	+ }
	+#else
	+ return -EAFNOSUPPORT;
	+ }
	+#endif
	+
	+ if (!(laddr->sin_port) \|\| !(raddr->sin_port))
	+ return -EINVAL;
	+#endif /* IW_IXL_RDMA_4_9 */
	+
	+ iwqp->active_conn = 1;
	+ iwqp->cm_id = NULL;
	+ cm_id->provider_data = iwqp;
	+
	+ /* set up the connection params for the node */
	+#ifndef IW_IXL_RDMA_4_9
	+ if (cm_id->remote_addr.sin_family == AF_INET) {
	+#else
	+ if (cm_id->remote_addr.ss_family == AF_INET) {
	+#endif
	+ cm_info.ipv4 = TRUE;
	+ memset(cm_info.loc_addr, 0, sizeof(cm_info.loc_addr));
	+ memset(cm_info.rem_addr, 0, sizeof(cm_info.rem_addr));
	+ cm_info.loc_addr[0] = ntohl(laddr->sin_addr.s_addr);
	+ cm_info.rem_addr[0] = ntohl(raddr->sin_addr.s_addr);
	+#ifdef VIMAGE_READY
	+ memcpy(cm_info.loc_dev_addr, src_dev_addr, MAX_ADDR_LEN);
	+ memcpy(cm_info.rem_dev_addr, dst_dev_addr, MAX_ADDR_LEN);
	+#endif
	+ cm_info.loc_port = ntohs(laddr->sin_port);
	+ cm_info.rem_port = ntohs(raddr->sin_port);
	+ cm_info.vlan_id = i40iw_get_vlan_ipv4(cm_info.loc_addr);
	+ }
	+#ifdef IPV6_SUPPORT
	+ else {
	+ cm_info.ipv4 = FALSE;
	+ i40iw_copy_ip_ntohl(cm_info.loc_addr,
	+ (__be32 *)&laddr6->sin6_addr);
	+ i40iw_copy_ip_ntohl(cm_info.rem_addr,
	+ (__be32 *)&raddr6->sin6_addr);
	+#ifdef VIMAGE_READY
	+ memcpy(cm_info.loc_dev_addr, src_dev_addr, MAX_ADDR_LEN);
	+ memcpy(cm_info.rem_dev_addr, dst_dev_addr, MAX_ADDR_LEN);
	+#endif
	+ cm_info.loc_port = ntohs(laddr6->sin6_port);
	+ cm_info.rem_port = ntohs(raddr6->sin6_port);
	+ cm_info.vlan_id = i40iw_get_vlan_ipv6(cm_info.loc_addr);
	+ }
	+#endif
	+
	+ cm_info.cm_id = cm_id;
	+ cm_node = i40iw_create_cm_node(&iwdev->cm_core, iwdev, conn_param, &cm_info);
	+
	+ if (IS_ERR(cm_node)) {
	+ ret = PTR_ERR(cm_node);
	+ return ret;
	+ }
	+#ifdef IPV6_SUPPORT
	+ if (cm_info.ipv4) {
	+ local_address = (u32 *)&laddr->sin_addr.s_addr;
	+ remote_address = (u32 *)&raddr->sin_addr.s_addr;
	+ } else {
	+ local_address = (u32 *)&laddr6->sin6_addr;
	+ remote_address = (u32 *)&raddr6->sin6_addr;
	+ }
	+#else
	+ local_address = (u32 *)&laddr->sin_addr.s_addr;
	+ remote_address = (u32 *)&raddr->sin_addr.s_addr;
	+#endif
	+ if (!i40iw_is_loopback(cm_info.ipv4, local_address, remote_address)) {
	+ if (i40iw_manage_qhash(iwdev, &cm_info, I40IW_QHASH_TYPE_TCP_ESTABLISHED,
	+ I40IW_QHASH_MANAGE_TYPE_ADD, NULL, TRUE)) {
	+ ret = -EINVAL;
	+ goto err;
	+ }
	+ cm_node->qhash_set = TRUE;
	+ }
	+ spin_lock_irqsave(&iwdev->cm_core.ht_lock, flags);
	+ if (!test_and_set_bit(cm_info.loc_port, iwdev->cm_core.active_side_ports)) {
	+ spin_unlock_irqrestore(&iwdev->cm_core.ht_lock, flags);
	+ if (i40iw_manage_apbvt(iwdev, cm_info.loc_port, I40IW_MANAGE_APBVT_ADD)) {
	+ ret = -EINVAL;
	+ goto err;
	+ }
	+ } else {
	+ spin_unlock_irqrestore(&iwdev->cm_core.ht_lock, flags);
	+ }
	+
	+ cm_node->apbvt_set = TRUE;
	+ iwqp->cm_node = cm_node;
	+ cm_node->iwqp = iwqp;
	+ iwqp->cm_id = cm_id;
	+
	+ if (cm_node->state != I40IW_CM_STATE_OFFLOADED) {
	+ cm_node->state = I40IW_CM_STATE_SYN_SENT;
	+ ret = i40iw_send_syn(cm_node, 0);
	+ if (ret)
	+ goto err;
	+ }
	+ if (cm_node->loopbackpartner) {
	+ cm_node->loopbackpartner->state = I40IW_CM_STATE_MPAREQ_RCVD;
	+ i40iw_create_event(cm_node->loopbackpartner, I40IW_CM_EVENT_MPA_REQ);
	+ }
	+ i40iw_debug(cm_node->dev, I40IW_DEBUG_CM,
	+ "Api - connect(): port=0x%04x, cm_node=%p, cm_id = %p.\n",
	+ cm_node->rem_port, cm_node, cm_node->cm_id);
	+ cm_id->add_ref(cm_id);
	+ i40iw_add_ref(&iwqp->ibqp);
	+ return 0;
	+
	+err:
	+ if (cm_info.ipv4) {
	+ i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_CM,
	+ "Api - connect() FAILED: dest addr=%d.%d.%d.%d",
	+ cm_node->rem_addr[0] >> 24,
	+ (cm_node->rem_addr[0] >> 16) & 0xFF,
	+ (cm_node->rem_addr[0] >> 8) & 0xFF,
	+ cm_node->rem_addr[0] & 0xFF);
	+ }
	+#ifdef IPV6_SUPPORT
	+ else {
	+ char ip6buf[INET6_ADDRSTRLEN];
	+ i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_CM,
	+ "Api - connect() FAILED: dest addr=%s",
	+ ip6_sprintf(ip6buf,
	+ (const struct in6_addr *)cm_node->rem_addr));
	+ }
	+#endif
	+ i40iw_rem_ref_cm_node(cm_node);
	+ iwdev->cm_core.stats_connect_errs++;
	+ return ret;
	+}
	+
	+/**
	+ * i40iw_create_listen - registered call creating listener
	+ * @cm_id: cm information for passive connection
	+ * @backlog: to max accept pending count
	+ */
	+int
	+i40iw_create_listen(struct iw_cm_id *cm_id, int backlog)
	+{
	+ struct i40iw_device *iwdev;
	+ struct i40iw_cm_info cm_info;
	+ struct i40iw_cm_listener *cm_listen_node;
	+ struct sockaddr_in *laddr;
	+#ifdef IPV6_SUPPORT
	+ struct sockaddr_in6 *laddr6;
	+#endif
	+ bool wildcard = FALSE;
	+ enum i40iw_status_code ret = 0;
	+
	+ iwdev = to_iwdev(cm_id->device);
	+ if (!iwdev)
	+ return -EINVAL;
	+
	+ laddr = (struct sockaddr_in *)&cm_id->local_addr;
	+#ifdef IPV6_SUPPORT
	+ laddr6 = (struct sockaddr_in6 *)&cm_id->local_addr;
	+#endif
	+
	+ memset(&cm_info, 0, sizeof(cm_info));
	+ if (laddr->sin_family == AF_INET) {
	+ cm_info.ipv4 = TRUE;
	+ cm_info.loc_addr[0] = ntohl(laddr->sin_addr.s_addr);
	+ cm_info.loc_port = ntohs(laddr->sin_port);
	+
	+ if (laddr->sin_addr.s_addr != INADDR_ANY)
	+ cm_info.vlan_id = i40iw_get_vlan_ipv4(cm_info.loc_addr);
	+ else
	+ wildcard = TRUE;
	+
	+ } else {
	+#ifdef IPV6_SUPPORT
	+ cm_info.ipv4 = FALSE;
	+ i40iw_copy_ip_ntohl(cm_info.loc_addr, (__be32 *)&laddr6->sin6_addr);
	+ cm_info.loc_port = ntohs(laddr6->sin6_port);
	+ if (!IN6_IS_ADDR_UNSPECIFIED(&laddr6->sin6_addr))
	+ cm_info.vlan_id = i40iw_get_vlan_ipv6(cm_info.loc_addr);
	+ else
	+ wildcard = TRUE;
	+#else
	+ return -EAFNOSUPPORT;
	+#endif
	+ }
	+ cm_info.backlog = backlog;
	+ cm_info.cm_id = cm_id;
	+
	+ cm_listen_node = i40iw_make_listen_node(&iwdev->cm_core, iwdev, &cm_info);
	+ if (!cm_listen_node) {
	+ i40iw_pr_err("cm_listen_node == NULL\n");
	+ return -ENOMEM;
	+ }
	+ cm_id->provider_data = cm_listen_node;
	+
	+ if (!cm_listen_node->reused_node) {
	+ if (wildcard) {
	+ if (cm_info.ipv4)
	+ ret = i40iw_add_mqh_4(iwdev, &cm_info, cm_listen_node);
	+#ifdef IPV6_SUPPORT
	+ else
	+ ret = i40iw_add_mqh_6(iwdev, &cm_info, cm_listen_node);
	+#endif
	+ if (ret)
	+ goto error;
	+
	+ ret = i40iw_manage_apbvt(iwdev, cm_info.loc_port, I40IW_MANAGE_APBVT_ADD);
	+ if (ret)
	+ goto error;
	+ } else {
	+ ret = i40iw_manage_qhash(iwdev, &cm_info, I40IW_QHASH_TYPE_TCP_SYN,
	+ I40IW_QHASH_MANAGE_TYPE_ADD, NULL, TRUE);
	+ if (ret)
	+ goto error;
	+ cm_listen_node->qhash_set = TRUE;
	+ ret = i40iw_manage_apbvt(iwdev, cm_info.loc_port, I40IW_MANAGE_APBVT_ADD);
	+ if (ret)
	+ goto error;
	+ }
	+ }
	+ cm_id->add_ref(cm_id);
	+ cm_listen_node->cm_core->stats_listen_created++;
	+ return 0;
	+error:
	+ i40iw_cm_del_listen(&iwdev->cm_core, (void *)cm_listen_node, FALSE);
	+ return -EINVAL;
	+}
	+
	+/**
	+ * i40iw_destroy_listen - registered call to destroy listener
	+ * @cm_id: cm information for passive connection
	+ */
	+#if __FreeBSD_version >= 1100000 && !defined(IW_IXL_RDMA_4_9)
	+void
	+#else
	+int
	+#endif /* IW_IXL_FREEBSD11 */
	+i40iw_destroy_listen(struct iw_cm_id *cm_id)
	+{
	+ struct i40iw_device *iwdev;
	+
	+ iwdev = to_iwdev(cm_id->device);
	+ if (cm_id->provider_data)
	+ i40iw_cm_del_listen(&iwdev->cm_core, cm_id->provider_data, TRUE);
	+ else
	+ i40iw_pr_err("cm_id->provider_data was NULL\n");
	+
	+ cm_id->rem_ref(cm_id);
	+#if __FreeBSD_version < 1100000 \|\| defined(IW_IXL_RDMA_4_9)
	+ return 0;
	+#endif /* IW_IXL_FREEBSD10 */
	+}
	+
	+/**
	+ * i40iw_cm_event_connected - handle connected active node
	+ * @event: the info for cm_node of connection
	+ */
	+static void
	+i40iw_cm_event_connected(struct i40iw_cm_event *event)
	+{
	+ struct ib_qp_attr attr;
	+ struct iw_cm_id *cm_id;
	+ struct i40iw_cm_node *cm_node;
	+ struct i40iw_device *iwdev;
	+ struct i40iw_qp *iwqp;
	+ struct i40iw_sc_dev *dev;
	+ int status;
	+ bool read0;
	+
	+ cm_node = event->cm_node;
	+ cm_id = cm_node->cm_id;
	+ iwqp = (struct i40iw_qp *)cm_id->provider_data;
	+ iwdev = to_iwdev(iwqp->ibqp.device);
	+ dev = &iwdev->sc_dev;
	+
	+ if (iwqp->destroyed) {
	+ status = -ETIMEDOUT;
	+ goto error;
	+ }
	+ i40iw_cm_init_tsa_conn(iwqp, cm_node);
	+ read0 = (cm_node->send_rdma0_op == SEND_RDMA_READ_ZERO);
	+ dev->iw_priv_qp_ops->qp_send_rtt(&iwqp->sc_qp, read0);
	+
	+ memset(&attr, 0, sizeof(attr));
	+ attr.qp_state = IB_QPS_RTS;
	+ cm_node->qhash_set = FALSE;
	+ i40iw_modify_qp(&iwqp->ibqp, &attr, IB_QP_STATE, NULL);
	+
	+ cm_node->accelerated = 1;
	+ i40iw_send_cm_event(cm_node, cm_id, IW_CM_EVENT_CONNECT_REPLY, 0);
	+ return;
	+
	+error:
	+ iwqp->cm_id = NULL;
	+ cm_id->provider_data = NULL;
	+ i40iw_send_cm_event(event->cm_node, cm_id, IW_CM_EVENT_CONNECT_REPLY,
	+ status);
	+ cm_id->rem_ref(cm_id);
	+ i40iw_rem_ref_cm_node(event->cm_node);
	+}
	+
	+/**
	+ * i40iw_cm_event_reset - handle reset
	+ * @event: the info for cm_node of connection
	+ */
	+static void
	+i40iw_cm_event_reset(struct i40iw_cm_event *event)
	+{
	+ struct i40iw_cm_node *cm_node = event->cm_node;
	+ struct iw_cm_id *cm_id = cm_node->cm_id;
	+ struct i40iw_qp *iwqp;
	+
	+ if (!cm_id)
	+ return;
	+
	+ iwqp = cm_id->provider_data;
	+ if (!iwqp)
	+ return;
	+
	+ i40iw_debug(cm_node->dev, I40IW_DEBUG_CM, "reset event %p - cm_id = %p\n",
	+ event->cm_node, cm_id);
	+ iwqp->cm_id = NULL;
	+
	+ i40iw_send_cm_event(cm_node, cm_node->cm_id, IW_CM_EVENT_DISCONNECT,
	+ -ECONNRESET);
	+ i40iw_send_cm_event(cm_node, cm_node->cm_id, IW_CM_EVENT_CLOSE, 0);
	+}
	+
	+/**
	+ * i40iw_cm_event_handler - worker thread callback to send event
	+ * to cm upper layer
	+ * @work: pointer of cm event info.
	+ */
	+static void
	+i40iw_cm_event_handler(struct work_struct *work)
	+{
	+ struct i40iw_cm_event *event = container_of(work, struct i40iw_cm_event,
	+ event_work);
	+ struct i40iw_cm_node *cm_node;
	+
	+ if (!event \|\| !event->cm_node \|\| !event->cm_node->cm_core)
	+ return;
	+
	+ cm_node = event->cm_node;
	+
	+ switch (event->type) {
	+ case I40IW_CM_EVENT_MPA_REQ:
	+ i40iw_send_cm_event(cm_node, cm_node->cm_id,
	+ IW_CM_EVENT_CONNECT_REQUEST, 0);
	+ break;
	+ case I40IW_CM_EVENT_RESET:
	+ i40iw_cm_event_reset(event);
	+ break;
	+ case I40IW_CM_EVENT_CONNECTED:
	+ if (!event->cm_node->cm_id \|\|
	+ (event->cm_node->state != I40IW_CM_STATE_OFFLOADED))
	+ break;
	+ i40iw_cm_event_connected(event);
	+ break;
	+ case I40IW_CM_EVENT_MPA_REJECT:
	+ if (!event->cm_node->cm_id \|\|
	+ (cm_node->state == I40IW_CM_STATE_OFFLOADED))
	+ break;
	+ i40iw_send_cm_event(cm_node, cm_node->cm_id, IW_CM_EVENT_CONNECT_REPLY,
	+ -ECONNREFUSED);
	+ break;
	+ case I40IW_CM_EVENT_ABORTED:
	+ if (!event->cm_node->cm_id \|\|
	+ (event->cm_node->state == I40IW_CM_STATE_OFFLOADED))
	+ break;
	+ i40iw_event_connect_error(event);
	+ break;
	+ default:
	+ i40iw_pr_err("event type = %d\n", event->type);
	+ break;
	+ }
	+
	+ event->cm_info.cm_id->rem_ref(event->cm_info.cm_id);
	+ i40iw_rem_ref_cm_node(event->cm_node);
	+ kfree(event);
	+}
	+
	+/**
	+ * i40iw_cm_post_event - queue event request for worker thread
	+ * @event: cm node's info for up event call
	+ */
	+static void
	+i40iw_cm_post_event(struct i40iw_cm_event *event)
	+{
	+ atomic_inc(&event->cm_node->ref_count);
	+ event->cm_info.cm_id->add_ref(event->cm_info.cm_id);
	+
	+ INIT_WORK(&event->event_work, i40iw_cm_event_handler);
	+
	+ queue_work(event->cm_node->cm_core->event_wq, &event->event_work);
	+}
	Index: sys/dev/ixl/iwarp/iw_ixl_dbg.h
	===================================================================
	--- /dev/null
	+++ sys/dev/ixl/iwarp/iw_ixl_dbg.h
	@@ -0,0 +1,81 @@
	+/*******************************************************************************
	+*
	+* Copyright (c) 2015-2017 Intel Corporation. All rights reserved.
	+*
	+* This software is available to you under a choice of one of two
	+* licenses. You may choose to be licensed under the terms of the GNU
	+* General Public License (GPL) Version 2, available from the file
	+* COPYING in the main directory of this source tree, or the
	+* OpenFabrics.org BSD license below:
	+*
	+* Redistribution and use in source and binary forms, with or
	+* without modification, are permitted provided that the following
	+* conditions are met:
	+*
	+* - Redistributions of source code must retain the above
	+* copyright notice, this list of conditions and the following
	+* disclaimer.
	+*
	+* - Redistributions in binary form must reproduce the above
	+* copyright notice, this list of conditions and the following
	+* disclaimer in the documentation and/or other materials
	+* provided with the distribution.
	+*
	+* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
	+* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
	+* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
	+* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
	+* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
	+* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
	+* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
	+* SOFTWARE.
	+*
	+*******************************************************************************/
	+/$FreeBSD$/
	+
	+#ifndef _IW_IXL_DBG_H_
	+#define _IW_IXL_DBG_H_
	+
	+#ifdef IW_IXL_DEBUG
	+
	+#define _DBG_PRINTF(S, ...) printf("%s: " S "\n", __func__, ##__VA_ARGS__)
	+#define _DEV_DBG_PRINTF(dev, S, ...) device_printf(dev, "%s: " S "\n", __func__, ##__VA_ARGS__)
	+#define _IF_DBG_PRINTF(ifp, S, ...) if_printf(ifp, "%s: " S "\n", __func__, ##__VA_ARGS__)
	+
	+/* Defines for printing generic debug information */
	+#define DPRINTF(...) _DBG_PRINTF(__VA_ARGS__)
	+#define DDPRINTF(...) _DEV_DBG_PRINTF(__VA_ARGS__)
	+#define IDPRINTF(...) _IF_DBG_PRINTF(__VA_ARGS__)
	+
	+/* Defines for printing specific debug information */
	+#define DEBUG_INIT 1
	+#define DEBUG_HW 1
	+
	+#define INIT_DEBUGOUT(...) if (DEBUG_INIT) _DBG_PRINTF(__VA_ARGS__)
	+#define INIT_DBG_DEV(...) if (DEBUG_INIT) _DEV_DBG_PRINTF(__VA_ARGS__)
	+#define INIT_DBG_IF(...) if (DEBUG_INIT) _IF_DBG_PRINTF(__VA_ARGS__)
	+
	+#define HW_DEBUGOUT(...) if (DEBUG_HW) _DBG_PRINTF(__VA_ARGS__)
	+
	+#define IW_IXL_ASSERT(x) if(!(x)) panic("iw_ixl: %s: x", __func__)
	+
	+#else /* no IW_IXL_DEBUG */
	+
	+#define DEBUG_INIT 0
	+#define DEBUG_HW 0
	+
	+#define DPRINTF(...)
	+#define DDPRINTF(...)
	+#define IDPRINTF(...)
	+
	+#define INIT_DEBUGOUT(...)
	+#define INIT_DBG_DEV(...)
	+#define INIT_DBG_IF(...)
	+
	+#define HW_DEBUGOUT(...)
	+
	+#define IW_IXL_ASSERT(x)
	+
	+#endif /* IW_IXL_DEBUG */
	+
	+#endif /* _IW_IXL_DBG_H_ */
	Index: sys/dev/ixl/iwarp/iw_ixl_hw.c
	===================================================================
	--- /dev/null
	+++ sys/dev/ixl/iwarp/iw_ixl_hw.c
	@@ -0,0 +1,799 @@
	+/*******************************************************************************
	+*
	+* Copyright (c) 2015-2017 Intel Corporation. All rights reserved.
	+*
	+* This software is available to you under a choice of one of two
	+* licenses. You may choose to be licensed under the terms of the GNU
	+* General Public License (GPL) Version 2, available from the file
	+* COPYING in the main directory of this source tree, or the
	+* OpenFabrics.org BSD license below:
	+*
	+* Redistribution and use in source and binary forms, with or
	+* without modification, are permitted provided that the following
	+* conditions are met:
	+*
	+* - Redistributions of source code must retain the above
	+* copyright notice, this list of conditions and the following
	+* disclaimer.
	+*
	+* - Redistributions in binary form must reproduce the above
	+* copyright notice, this list of conditions and the following
	+* disclaimer in the documentation and/or other materials
	+* provided with the distribution.
	+*
	+* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
	+* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
	+* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
	+* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
	+* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
	+* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
	+* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
	+* SOFTWARE.
	+*
	+*******************************************************************************/
	+/$FreeBSD$/
	+
	+#include "iw_ixl.h"
	+
	+#include <net/if_var.h>
	+#include <net/if_dl.h>
	+
	+/**
	+ * i40iw_initialize_hw_resources - initialize hw resource during open
	+ * @iwdev: iwarp device
	+ */
	+u32
	+i40iw_initialize_hw_resources(struct i40iw_device *iwdev)
	+{
	+ unsigned long num_pds;
	+ u32 resources_size;
	+ u32 max_mr;
	+ u32 max_qp;
	+ u32 max_cq;
	+ u32 arp_table_size;
	+ u32 mrdrvbits;
	+ void *resource_ptr;
	+
	+ max_qp = iwdev->sc_dev.hmc_info->hmc_obj[I40IW_HMC_IW_QP].cnt;
	+ max_cq = iwdev->sc_dev.hmc_info->hmc_obj[I40IW_HMC_IW_CQ].cnt;
	+ max_mr = iwdev->sc_dev.hmc_info->hmc_obj[I40IW_HMC_IW_MR].cnt;
	+ arp_table_size = iwdev->sc_dev.hmc_info->hmc_obj[I40IW_HMC_IW_ARP].cnt;
	+ iwdev->max_cqe = 0xFFFFF;
	+ num_pds = I40IW_MAX_PDS;
	+ resources_size = sizeof(struct i40iw_arp_entry) * arp_table_size;
	+ resources_size += sizeof(unsigned long) * BITS_TO_LONGS(max_qp);
	+ resources_size += sizeof(unsigned long) * BITS_TO_LONGS(max_mr);
	+ resources_size += sizeof(unsigned long) * BITS_TO_LONGS(max_cq);
	+ resources_size += sizeof(unsigned long) * BITS_TO_LONGS(num_pds);
	+ resources_size += sizeof(unsigned long) * BITS_TO_LONGS(arp_table_size);
	+ resources_size += sizeof(struct i40iw_qp *) max_qp;
	+ iwdev->mem_resources = kzalloc(resources_size, GFP_KERNEL);
	+
	+ if (!iwdev->mem_resources)
	+ return -ENOMEM;
	+
	+ iwdev->max_qp = max_qp;
	+ iwdev->max_mr = max_mr;
	+ iwdev->max_cq = max_cq;
	+ iwdev->max_pd = num_pds;
	+ iwdev->arp_table_size = arp_table_size;
	+ iwdev->arp_table = (struct i40iw_arp_entry *)iwdev->mem_resources;
	+ resource_ptr = iwdev->mem_resources + (sizeof(struct i40iw_arp_entry) * arp_table_size);
	+
	+ iwdev->device_cap_flags = IB_DEVICE_LOCAL_DMA_LKEY \|
	+ IB_DEVICE_MEM_WINDOW \| IB_DEVICE_MEM_MGT_EXTENSIONS;
	+
	+ iwdev->allocated_qps = resource_ptr;
	+ iwdev->allocated_cqs = &iwdev->allocated_qps[BITS_TO_LONGS(max_qp)];
	+ iwdev->allocated_mrs = &iwdev->allocated_cqs[BITS_TO_LONGS(max_cq)];
	+ iwdev->allocated_pds = &iwdev->allocated_mrs[BITS_TO_LONGS(max_mr)];
	+ iwdev->allocated_arps = &iwdev->allocated_pds[BITS_TO_LONGS(num_pds)];
	+ iwdev->qp_table = (struct i40iw_qp **)(&iwdev->allocated_arps[BITS_TO_LONGS(arp_table_size)]);
	+
	+ set_bit(0, iwdev->allocated_mrs);
	+ set_bit(0, iwdev->allocated_qps);
	+ set_bit(0, iwdev->allocated_cqs);
	+ set_bit(0, iwdev->allocated_pds);
	+ set_bit(0, iwdev->allocated_arps);
	+
	+ /* Following for ILQ/IEQ */
	+ set_bit(1, iwdev->allocated_qps);
	+ set_bit(1, iwdev->allocated_cqs);
	+ set_bit(1, iwdev->allocated_pds);
	+ set_bit(2, iwdev->allocated_cqs);
	+ set_bit(2, iwdev->allocated_pds);
	+
	+ spin_lock_init(&iwdev->resource_lock);
	+ spin_lock_init(&iwdev->qptable_lock);
	+ /* stag index mask has a minimum of 14 bits */
	+ mrdrvbits = 24 - max(get_count_order(iwdev->max_mr), 14);
	+ iwdev->mr_stagmask = ~(((1 << mrdrvbits) - 1) << (32 - mrdrvbits));
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_cqp_ce_handler - handle cqp completions
	+ * @iwdev: iwarp device
	+ * @arm: flag to arm after completions
	+ * @cq: cq for cqp completions
	+ */
	+static void
	+i40iw_cqp_ce_handler(struct i40iw_device iwdev, struct i40iw_sc_cq cq,
	+ bool arm)
	+{
	+ struct i40iw_cqp_request *cqp_request;
	+ struct i40iw_ccq_cqe_info info;
	+ struct i40iw_sc_dev *dev = &iwdev->sc_dev;
	+ int ret;
	+ u32 cqe_count = 0;
	+
	+ do {
	+ memset(&info, 0, sizeof(info));
	+ ret = dev->ccq_ops->ccq_get_cqe_info(cq, &info);
	+ if (ret)
	+ break;
	+ cqp_request = (struct i40iw_cqp_request *)(unsigned long)info.scratch;
	+ if (info.error)
	+ device_printf(iwdev->ldev->dev,
	+ "opcode = 0x%x maj_err_code = 0x%x min_err_code = 0x%x\n",
	+ info.op_code, info.maj_err_code, info.min_err_code);
	+ if (cqp_request) {
	+ cqp_request->compl_info.maj_err_code = info.maj_err_code;
	+ cqp_request->compl_info.min_err_code = info.min_err_code;
	+ cqp_request->compl_info.op_ret_val = info.op_ret_val;
	+ cqp_request->compl_info.error = info.error;
	+
	+ if (cqp_request->waiting) {
	+ cqp_request->request_done = TRUE;
	+ wake_up(&cqp_request->waitq);
	+ i40iw_put_cqp_request(&iwdev->cqp, cqp_request);
	+ } else {
	+ if (cqp_request->callback_fcn)
	+ cqp_request->callback_fcn(cqp_request, 1);
	+ i40iw_put_cqp_request(&iwdev->cqp, cqp_request);
	+ }
	+ }
	+ cqe_count++;
	+ } while (1);
	+
	+ if (arm && cqe_count) {
	+ i40iw_process_bh(dev);
	+ dev->ccq_ops->ccq_arm(cq);
	+ }
	+}
	+
	+/**
	+ * i40iw_iwarp_ce_handler - handle iwarp completions
	+ * @iwdev: iwarp device
	+ * @iwcp: iwarp cq receiving event
	+ */
	+static void
	+i40iw_iwarp_ce_handler(struct i40iw_device iwdev, struct i40iw_sc_cq iwcq)
	+{
	+ struct i40iw_cq *i40iwcq = iwcq->back_cq;
	+
	+ if (i40iwcq->ibcq.comp_handler)
	+ i40iwcq->ibcq.comp_handler(&i40iwcq->ibcq,
	+ i40iwcq->ibcq.cq_context);
	+}
	+
	+/**
	+ * i40iw_puda_ce_handler - handle puda completion events
	+ * @iwdev: iwarp device
	+ * @cq: puda completion q for event
	+ */
	+static void
	+i40iw_puda_ce_handler(struct i40iw_device iwdev, struct i40iw_sc_cq cq)
	+{
	+ struct i40iw_sc_dev dev = (struct i40iw_sc_dev )&iwdev->sc_dev;
	+ u32 compl_error;
	+ enum i40iw_status_code status;
	+
	+ do {
	+ status = i40iw_puda_poll_completion(dev, cq, &compl_error);
	+ if (status == I40IW_ERR_QUEUE_EMPTY)
	+ break;
	+ if (status) {
	+ device_printf(iwdev->ldev->dev,
	+ "puda status = %d\n", status);
	+ break;
	+ }
	+ if (compl_error) {
	+ device_printf(iwdev->ldev->dev,
	+ "puda compl_err =0x%x\n", compl_error);
	+ break;
	+ }
	+ } while (1);
	+
	+ dev->ccq_ops->ccq_arm(cq);
	+}
	+
	+/**
	+ * i40iw_process_ceq - handle ceq for completions
	+ * @iwdev: iwarp device
	+ * @ceq: ceq having cq for completion
	+ */
	+void
	+i40iw_process_ceq(struct i40iw_device iwdev, struct i40iw_ceq ceq)
	+{
	+ struct i40iw_sc_dev *dev = &iwdev->sc_dev;
	+ struct i40iw_sc_ceq *sc_ceq;
	+ struct i40iw_sc_cq *cq;
	+ bool arm = TRUE;
	+
	+ sc_ceq = &ceq->sc_ceq;
	+ do {
	+ cq = dev->ceq_ops->process_ceq(dev, sc_ceq);
	+ if (!cq)
	+ break;
	+
	+ if (cq->cq_type == I40IW_CQ_TYPE_CQP)
	+ i40iw_cqp_ce_handler(iwdev, cq, arm);
	+ else if (cq->cq_type == I40IW_CQ_TYPE_IWARP)
	+ i40iw_iwarp_ce_handler(iwdev, cq);
	+ else if ((cq->cq_type == I40IW_CQ_TYPE_ILQ) \|\|
	+ (cq->cq_type == I40IW_CQ_TYPE_IEQ))
	+ i40iw_puda_ce_handler(iwdev, cq);
	+ } while (1);
	+}
	+
	+/**
	+ * i40iw_next_iw_state - modify qp state
	+ * @iwqp: iwarp qp to modify
	+ * @state: next state for qp
	+ * @del_hash: del hash
	+ * @term: term message
	+ * @termlen: length of term message
	+ */
	+void
	+i40iw_next_iw_state(struct i40iw_qp *iwqp, u8 state, u8 del_hash, u8 term,
	+ u8 termlen)
	+{
	+ struct i40iw_modify_qp_info info;
	+
	+ memset(&info, 0, sizeof(info));
	+ info.next_iwarp_state = state;
	+ info.remove_hash_idx = del_hash;
	+ info.cq_num_valid = TRUE;
	+ info.arp_cache_idx_valid = TRUE;
	+ info.dont_send_term = TRUE;
	+ info.dont_send_fin = TRUE;
	+ info.termlen = termlen;
	+
	+ if (term & I40IWQP_TERM_SEND_TERM_ONLY)
	+ info.dont_send_term = FALSE;
	+ if (term & I40IWQP_TERM_SEND_FIN_ONLY)
	+ info.dont_send_fin = FALSE;
	+ if (iwqp->sc_qp.term_flags && (state == I40IW_QP_STATE_ERROR))
	+ info.reset_tcp_conn = TRUE;
	+ iwqp->hw_iwarp_state = state;
	+ i40iw_hw_modify_qp(iwqp->iwdev, iwqp, &info, 0);
	+}
	+
	+/**
	+ * i40iw_process_aeq - handle aeq events
	+ * @iwdev: iwarp device
	+ */
	+void
	+i40iw_process_aeq(struct i40iw_device *iwdev)
	+{
	+ struct i40iw_sc_dev *dev = &iwdev->sc_dev;
	+ struct i40iw_aeq *aeq = &iwdev->aeq;
	+ struct i40iw_sc_aeq *sc_aeq = &aeq->sc_aeq;
	+ struct i40iw_aeqe_info aeinfo;
	+ struct i40iw_aeqe_info *info = &aeinfo;
	+ struct i40iw_qp *iwqp = NULL;
	+ struct i40iw_sc_cq *cq = NULL;
	+ struct i40iw_cq *iwcq = NULL;
	+ struct i40iw_sc_qp *qp = NULL;
	+ struct i40iw_qp_host_ctx_info *ctx_info = NULL;
	+ int ret;
	+ unsigned long flags;
	+
	+ u32 aeqcnt = 0;
	+
	+ if (!sc_aeq->size)
	+ return;
	+
	+ do {
	+ memset(info, 0, sizeof(*info));
	+ ret = dev->aeq_ops->get_next_aeqe(sc_aeq, info);
	+ if (ret)
	+ break;
	+
	+ aeqcnt++;
	+ i40iw_debug(dev, I40IW_DEBUG_AEQ,
	+ "%s ae_id = 0x%x bool qp=%d qp_id = %d\n",
	+ __func__, info->ae_id, info->qp, info->qp_cq_id);
	+ if (info->qp) {
	+ spin_lock_irqsave(&iwdev->qptable_lock, flags);
	+ iwqp = iwdev->qp_table[info->qp_cq_id];
	+ if (!iwqp) {
	+ spin_unlock_irqrestore(&iwdev->qptable_lock, flags);
	+ i40iw_debug(dev, I40IW_DEBUG_AEQ, "qp_id %d is already freed\n",
	+ info->qp_cq_id);
	+ continue;
	+ }
	+ i40iw_add_ref(&iwqp->ibqp);
	+ spin_unlock_irqrestore(&iwdev->qptable_lock, flags);
	+ qp = &iwqp->sc_qp;
	+ spin_lock_irqsave(&iwqp->lock, flags);
	+ iwqp->hw_tcp_state = info->tcp_state;
	+ iwqp->hw_iwarp_state = info->iwarp_state;
	+ iwqp->last_aeq = info->ae_id;
	+ spin_unlock_irqrestore(&iwqp->lock, flags);
	+ ctx_info = &iwqp->ctx_info;
	+ ctx_info->iwarp_info->err_rq_idx_valid = TRUE;
	+ } else {
	+ if (info->ae_id != I40IW_AE_CQ_OPERATION_ERROR)
	+ continue;
	+ }
	+
	+ switch (info->ae_id) {
	+ case I40IW_AE_LLP_FIN_RECEIVED:
	+ case I40IW_AE_RDMAP_ROE_BAD_LLP_CLOSE:
	+ if (qp->term_flags)
	+ continue;
	+ if (atomic_inc_return(&iwqp->close_timer_started) == 1) {
	+ iwqp->hw_tcp_state = I40IW_TCP_STATE_CLOSE_WAIT;
	+ if ((iwqp->hw_tcp_state == I40IW_TCP_STATE_CLOSE_WAIT) &&
	+ (iwqp->ibqp_state == IB_QPS_RTS)) {
	+ i40iw_next_iw_state(iwqp,
	+ I40IW_QP_STATE_CLOSING, 0, 0, 0);
	+ i40iw_cm_disconn(iwqp);
	+ }
	+ iwqp->cm_id->add_ref(iwqp->cm_id);
	+ i40iw_schedule_cm_timer(iwqp->cm_node,
	+ (struct i40iw_puda_buf *)iwqp,
	+ I40IW_TIMER_TYPE_CLOSE, 1, 0);
	+ }
	+ break;
	+ case I40IW_AE_LLP_CLOSE_COMPLETE:
	+ if (qp->term_flags)
	+ i40iw_terminate_done(qp, 0);
	+ else
	+ i40iw_cm_disconn(iwqp);
	+ break;
	+ case I40IW_AE_BAD_CLOSE:
	+ /* fall through */
	+ case I40IW_AE_RESET_SENT:
	+ i40iw_next_iw_state(iwqp, I40IW_QP_STATE_ERROR, 1, 0, 0);
	+ i40iw_cm_disconn(iwqp);
	+ break;
	+ case I40IW_AE_LLP_CONNECTION_RESET:
	+ if (atomic_read(&iwqp->close_timer_started))
	+ continue;
	+ i40iw_cm_disconn(iwqp);
	+ break;
	+ case I40IW_AE_TERMINATE_SENT:
	+ i40iw_terminate_send_fin(qp);
	+ break;
	+ case I40IW_AE_LLP_TERMINATE_RECEIVED:
	+ i40iw_terminate_received(qp, info);
	+ break;
	+ case I40IW_AE_CQ_OPERATION_ERROR:
	+ device_printf(iwdev->ldev->dev,
	+ "Processing an iWARP related AE for CQ misc = 0x%04X\n",
	+ info->ae_id);
	+ cq = (struct i40iw_sc_cq *)(unsigned long)info->compl_ctx;
	+ iwcq = (struct i40iw_cq *)cq->back_cq;
	+
	+ if (iwcq->ibcq.event_handler) {
	+ struct ib_event ibevent;
	+
	+ ibevent.device = iwcq->ibcq.device;
	+ ibevent.event = IB_EVENT_CQ_ERR;
	+ ibevent.element.cq = &iwcq->ibcq;
	+ iwcq->ibcq.event_handler(&ibevent, iwcq->ibcq.cq_context);
	+ }
	+ break;
	+ case I40IW_AE_LLP_DOUBT_REACHABILITY:
	+ break;
	+ case I40IW_AE_PRIV_OPERATION_DENIED:
	+ case I40IW_AE_STAG_ZERO_INVALID:
	+ case I40IW_AE_IB_RREQ_AND_Q1_FULL:
	+ case I40IW_AE_DDP_UBE_INVALID_DDP_VERSION:
	+ case I40IW_AE_DDP_UBE_INVALID_MO:
	+ case I40IW_AE_DDP_UBE_INVALID_QN:
	+ case I40IW_AE_DDP_NO_L_BIT:
	+ case I40IW_AE_RDMAP_ROE_INVALID_RDMAP_VERSION:
	+ case I40IW_AE_RDMAP_ROE_UNEXPECTED_OPCODE:
	+ case I40IW_AE_ROE_INVALID_RDMA_READ_REQUEST:
	+ case I40IW_AE_ROE_INVALID_RDMA_WRITE_OR_READ_RESP:
	+ case I40IW_AE_INVALID_ARP_ENTRY:
	+ case I40IW_AE_INVALID_TCP_OPTION_RCVD:
	+ case I40IW_AE_STALE_ARP_ENTRY:
	+ case I40IW_AE_LLP_RECEIVED_MPA_CRC_ERROR:
	+ case I40IW_AE_LLP_SEGMENT_TOO_SMALL:
	+ case I40IW_AE_LLP_SYN_RECEIVED:
	+ case I40IW_AE_LLP_TOO_MANY_RETRIES:
	+ case I40IW_AE_LCE_QP_CATASTROPHIC:
	+ case I40IW_AE_LCE_FUNCTION_CATASTROPHIC:
	+ case I40IW_AE_LCE_CQ_CATASTROPHIC:
	+ case I40IW_AE_UDA_XMIT_DGRAM_TOO_LONG:
	+ case I40IW_AE_UDA_XMIT_DGRAM_TOO_SHORT:
	+ case I40IW_AE_QP_SUSPEND_COMPLETE:
	+ ctx_info->iwarp_info->err_rq_idx_valid = FALSE;
	+ default:
	+ if (!info->sq && ctx_info->iwarp_info->err_rq_idx_valid) {
	+ ctx_info->iwarp_info->err_rq_idx = info->wqe_idx;
	+ ctx_info->tcp_info_valid = FALSE;
	+ ctx_info->iwarp_info_valid = FALSE;
	+ ret = dev->iw_priv_qp_ops->qp_setctx(&iwqp->sc_qp,
	+ iwqp->host_ctx.va,
	+ ctx_info);
	+ }
	+ i40iw_terminate_connection(qp, info);
	+ break;
	+ }
	+ if (info->qp)
	+ i40iw_rem_ref(&iwqp->ibqp);
	+ } while (1);
	+
	+ if (aeqcnt)
	+ dev->aeq_ops->repost_aeq_entries(dev, aeqcnt);
	+}
	+
	+/**
	+ * i40iw_manage_apbvt - add or delete tcp port
	+ * @iwdev: iwarp device
	+ * @accel_local_port: port for apbvt
	+ * @add_port: add or delete port
	+ */
	+int
	+i40iw_manage_apbvt(struct i40iw_device *iwdev, u16 accel_local_port,
	+ bool add_port)
	+{
	+ struct i40iw_cqp_request *cqp_request;
	+ struct cqp_commands_info *cqp_info;
	+ struct i40iw_apbvt_info *info;
	+ enum i40iw_status_code status;
	+
	+ cqp_request = i40iw_get_cqp_request(&iwdev->cqp, add_port);
	+ if (!cqp_request)
	+ return -ENOMEM;
	+
	+ cqp_info = &cqp_request->info;
	+ info = &cqp_info->in.u.manage_apbvt_entry.info;
	+
	+ memset(info, 0, sizeof(*info));
	+ info->add = add_port;
	+ info->port = cpu_to_le16(accel_local_port);
	+
	+ cqp_info->cqp_cmd = OP_MANAGE_APBVT_ENTRY;
	+ cqp_info->post_sq = 1;
	+ cqp_info->in.u.manage_apbvt_entry.cqp = &iwdev->cqp.sc_cqp;
	+ cqp_info->in.u.manage_apbvt_entry.scratch = (uintptr_t)cqp_request;
	+ status = i40iw_handle_cqp_op(iwdev, cqp_request);
	+ if (status)
	+ device_printf(iwdev->ldev->dev, "CQP-OP Manage APBVT entry fail");
	+ return status;
	+}
	+
	+/**
	+ * i40iw_manage_arp_cache - manage hw arp cache
	+ * @iwdev: iwarp device
	+ * @mac_addr: mac address ptr
	+ * @ip_addr: ip addr for arp cache
	+ * @action: add, delete or modify
	+ */
	+void
	+i40iw_manage_arp_cache(struct i40iw_device iwdev, unsigned char mac_addr,
	+ u32 *ip_addr, bool ipv4, u32 action)
	+{
	+ struct i40iw_add_arp_cache_entry_info *info;
	+ struct i40iw_cqp_request *cqp_request;
	+ struct cqp_commands_info *cqp_info;
	+ int arp_index;
	+
	+ arp_index = i40iw_arp_table(iwdev, ip_addr, ipv4, mac_addr, action);
	+ if (arp_index == -1)
	+ return;
	+ cqp_request = i40iw_get_cqp_request(&iwdev->cqp, FALSE);
	+ if (!cqp_request)
	+ return;
	+
	+ cqp_info = &cqp_request->info;
	+ if (action == I40IW_ARP_ADD) {
	+ cqp_info->cqp_cmd = OP_ADD_ARP_CACHE_ENTRY;
	+ info = &cqp_info->in.u.add_arp_cache_entry.info;
	+ memset(info, 0, sizeof(*info));
	+ info->arp_index = cpu_to_le16((u16)arp_index);
	+ info->permanent = TRUE;
	+ ether_addr_copy(info->mac_addr, mac_addr);
	+ cqp_info->in.u.add_arp_cache_entry.scratch = (uintptr_t)cqp_request;
	+ cqp_info->in.u.add_arp_cache_entry.cqp = &iwdev->cqp.sc_cqp;
	+ } else {
	+ cqp_info->cqp_cmd = OP_DELETE_ARP_CACHE_ENTRY;
	+ cqp_info->in.u.del_arp_cache_entry.scratch = (uintptr_t)cqp_request;
	+ cqp_info->in.u.del_arp_cache_entry.cqp = &iwdev->cqp.sc_cqp;
	+ cqp_info->in.u.del_arp_cache_entry.arp_index = arp_index;
	+ }
	+
	+ cqp_info->in.u.add_arp_cache_entry.cqp = &iwdev->cqp.sc_cqp;
	+ cqp_info->in.u.add_arp_cache_entry.scratch = (uintptr_t)cqp_request;
	+ cqp_info->post_sq = 1;
	+ if (i40iw_handle_cqp_op(iwdev, cqp_request))
	+ device_printf(iwdev->ldev->dev,
	+ "CQP-OP Add/Del Arp Cache entry fail");
	+}
	+
	+/**
	+ * i40iw_send_syn_cqp_callback - do syn/ack after qhash
	+ * @cqp_request: qhash cqp completion
	+ * @send_ack: flag send ack
	+ */
	+static void
	+i40iw_send_syn_cqp_callback(struct i40iw_cqp_request *cqp_request,
	+ u32 send_ack)
	+{
	+ i40iw_send_syn(cqp_request->param, send_ack);
	+}
	+
	+/**
	+ * i40iw_manage_qhash - add or modify qhash
	+ * @iwdev: iwarp device
	+ * @cminfo: cm info for qhash
	+ * @etype: type (syn or quad)
	+ * @mtype: type of qhash
	+ * @cmnode: cmnode associated with connection
	+ * @wait: wait for completion
	+ * @user_pri:user pri of the connection
	+ */
	+enum i40iw_status_code
	+i40iw_manage_qhash(struct i40iw_device iwdev, struct i40iw_cm_info cminfo,
	+ enum i40iw_quad_entry_type etype, enum i40iw_quad_hash_manage_type mtype,
	+ void *cmnode, bool wait)
	+{
	+ struct i40iw_qhash_table_info *info;
	+ struct i40iw_sc_dev *dev = &iwdev->sc_dev;
	+ struct i40iw_sc_vsi *vsi = &iwdev->vsi;
	+ struct i40iw_cqp *iwcqp = &iwdev->cqp;
	+ struct i40iw_cqp_request *cqp_request;
	+ struct cqp_commands_info *cqp_info;
	+#ifdef IPV6_SUPPORT
	+ char ip6buf[INET6_ADDRSTRLEN];
	+#endif
	+ enum i40iw_status_code status;
	+
	+ cqp_request = i40iw_get_cqp_request(iwcqp, wait);
	+ if (!cqp_request)
	+ return I40IW_ERR_NO_MEMORY;
	+ cqp_info = &cqp_request->info;
	+ info = &cqp_info->in.u.manage_qhash_table_entry.info;
	+ memset(info, 0, sizeof(*info));
	+
	+ info->vsi = &iwdev->vsi;
	+ info->manage = mtype;
	+ info->entry_type = etype;
	+ if (cminfo->vlan_id != 0xFFFF) {
	+ info->vlan_valid = TRUE;
	+ info->vlan_id = cpu_to_le16(cminfo->vlan_id);
	+ } else {
	+ info->vlan_valid = FALSE;
	+ }
	+
	+ info->ipv4_valid = cminfo->ipv4;
	+ ether_addr_copy(info->mac_addr, IF_LLADDR(iwdev->ifp));
	+ info->qp_num = cpu_to_le32(vsi->ilq->qp_id);
	+ info->dest_port = cpu_to_le16(cminfo->loc_port);
	+ info->dest_ip[0] = cpu_to_le32(cminfo->loc_addr[0]);
	+ info->dest_ip[1] = cpu_to_le32(cminfo->loc_addr[1]);
	+ info->dest_ip[2] = cpu_to_le32(cminfo->loc_addr[2]);
	+ info->dest_ip[3] = cpu_to_le32(cminfo->loc_addr[3]);
	+ if (etype == I40IW_QHASH_TYPE_TCP_ESTABLISHED) {
	+ info->src_port = cpu_to_le16(cminfo->rem_port);
	+ info->src_ip[0] = cpu_to_le32(cminfo->rem_addr[0]);
	+ info->src_ip[1] = cpu_to_le32(cminfo->rem_addr[1]);
	+ info->src_ip[2] = cpu_to_le32(cminfo->rem_addr[2]);
	+ info->src_ip[3] = cpu_to_le32(cminfo->rem_addr[3]);
	+ }
	+ if (cmnode) {
	+ cqp_request->callback_fcn = i40iw_send_syn_cqp_callback;
	+ cqp_request->param = (void *)cmnode;
	+ }
	+ if (info->ipv4_valid)
	+ i40iw_debug(dev, I40IW_DEBUG_CM,
	+ "%s:%s IP=%d.%d.%d.%d, port=%d, "
	+ "mac=%02x:%02x:%02x:%02x:%02x:%02x, vlan_id=%d\n",
	+ __func__, (!mtype) ? "DELETE" : "ADD",
	+ info->dest_ip[0] >> 24,
	+ (info->dest_ip[0] >> 16) & 0xFF,
	+ (info->dest_ip[0] >> 8) & 0xFF,
	+ info->dest_ip[0] & 0xFF,
	+ info->dest_port,
	+ info->mac_addr[0], info->mac_addr[1],
	+ info->mac_addr[2], info->mac_addr[3],
	+ info->mac_addr[4], info->mac_addr[5],
	+ cminfo->vlan_id);
	+#ifdef IPV6_SUPPORT
	+ else
	+ i40iw_debug(dev, I40IW_DEBUG_CM,
	+ "%s:%s IP=%s, port=%d, "
	+ "mac=%02x:%02x:%02x:%02x:%02x:%02x, vlan_id=%d\n",
	+ __func__, (!mtype) ? "DELETE" : "ADD",
	+ ip6_sprintf(ip6buf,
	+ (const struct in6_addr *)info->dest_ip),
	+ info->dest_port,
	+ info->mac_addr[0], info->mac_addr[1],
	+ info->mac_addr[2], info->mac_addr[3],
	+ info->mac_addr[4], info->mac_addr[5],
	+ cminfo->vlan_id);
	+#endif
	+ cqp_info->in.u.manage_qhash_table_entry.cqp = &iwdev->cqp.sc_cqp;
	+ cqp_info->in.u.manage_qhash_table_entry.scratch = (uintptr_t)cqp_request;
	+ cqp_info->cqp_cmd = OP_MANAGE_QHASH_TABLE_ENTRY;
	+ cqp_info->post_sq = 1;
	+ status = i40iw_handle_cqp_op(iwdev, cqp_request);
	+ if (status)
	+ device_printf(iwdev->ldev->dev, "CQP-OP Manage Qhash Entry fail");
	+ return status;
	+}
	+
	+/**
	+ * i40iw_hw_flush_wqes - flush qp's wqe
	+ * @iwdev: iwarp device
	+ * @qp: hardware control qp
	+ * @info: info for flush
	+ * @wait: flag wait for completion
	+ */
	+enum i40iw_status_code
	+i40iw_hw_flush_wqes(struct i40iw_device iwdev, struct i40iw_sc_qp qp,
	+ struct i40iw_qp_flush_info *info, bool wait)
	+{
	+ struct i40iw_cqp_request *cqp_request;
	+ struct cqp_commands_info *cqp_info;
	+ struct i40iw_qp_flush_info *hw_info;
	+#ifdef IW_IXL_RDMA_4_9
	+ struct i40iw_qp iwqp = (struct i40iw_qp )qp->back_qp;
	+#endif
	+ enum i40iw_status_code status;
	+
	+ cqp_request = i40iw_get_cqp_request(&iwdev->cqp, wait);
	+ if (!cqp_request)
	+ return I40IW_ERR_NO_MEMORY;
	+
	+ cqp_info = &cqp_request->info;
	+ hw_info = &cqp_request->info.in.u.qp_flush_wqes.info;
	+ memcpy(hw_info, info, sizeof(*hw_info));
	+
	+ cqp_info->cqp_cmd = OP_QP_FLUSH_WQES;
	+ cqp_info->post_sq = 1;
	+ cqp_info->in.u.qp_flush_wqes.qp = qp;
	+ cqp_info->in.u.qp_flush_wqes.scratch = (uintptr_t)cqp_request;
	+ status = i40iw_handle_cqp_op(iwdev, cqp_request);
	+ if (status) {
	+ i40iw_pr_err("CQP-OP Flush WQE's fail");
	+#ifdef IW_IXL_RDMA_4_9
	+ complete(&iwqp->sq_drained);
	+ complete(&iwqp->rq_drained);
	+#endif
	+ }
	+#ifdef IW_IXL_RDMA_4_9
	+ if (!cqp_request->compl_info.maj_err_code) {
	+ switch (cqp_request->compl_info.min_err_code) {
	+ case I40IW_CQP_COMPL_RQ_WQE_FLUSHED:
	+ complete(&iwqp->sq_drained);
	+ break;
	+ case I40IW_CQP_COMPL_SQ_WQE_FLUSHED:
	+ complete(&iwqp->rq_drained);
	+ break;
	+ default:
	+ break;
	+ }
	+ }
	+#endif
	+ return status;
	+}
	+
	+/**
	+ * i40iw_gen_ae - generate AE
	+ * @iwdev: iwarp device
	+ * @qp: qp associated with AE
	+ * @info: info for ae
	+ * @wait: wait for completion
	+ */
	+void
	+i40iw_gen_ae(struct i40iw_device *iwdev,
	+ struct i40iw_sc_qp *qp,
	+ struct i40iw_gen_ae_info *info,
	+ bool wait)
	+{
	+ struct i40iw_gen_ae_info *ae_info;
	+ struct i40iw_cqp_request *cqp_request;
	+ struct cqp_commands_info *cqp_info;
	+
	+ cqp_request = i40iw_get_cqp_request(&iwdev->cqp, wait);
	+ if (!cqp_request)
	+ return;
	+
	+ cqp_info = &cqp_request->info;
	+ ae_info = &cqp_request->info.in.u.gen_ae.info;
	+ memcpy(ae_info, info, sizeof(*ae_info));
	+
	+ cqp_info->cqp_cmd = OP_GEN_AE;
	+ cqp_info->post_sq = 1;
	+ cqp_info->in.u.gen_ae.qp = qp;
	+ cqp_info->in.u.gen_ae.scratch = (uintptr_t)cqp_request;
	+ if (i40iw_handle_cqp_op(iwdev, cqp_request))
	+ i40iw_pr_err("CQP OP failed attempting to generate ae_code=0x%x\n",
	+ info->ae_code);
	+}
	+
	+/**
	+ * i40iw_get_ib_wc - return change flush code to IB's
	+ * @opcode: iwarp flush code
	+ */
	+static enum ib_wc_status
	+i40iw_get_ib_wc(enum i40iw_flush_opcode opcode)
	+{
	+ switch (opcode) {
	+ case FLUSH_PROT_ERR:
	+ return IB_WC_LOC_PROT_ERR;
	+ case FLUSH_REM_ACCESS_ERR:
	+ return IB_WC_REM_ACCESS_ERR;
	+ case FLUSH_LOC_QP_OP_ERR:
	+ return IB_WC_LOC_QP_OP_ERR;
	+ case FLUSH_REM_OP_ERR:
	+ return IB_WC_REM_OP_ERR;
	+ case FLUSH_LOC_LEN_ERR:
	+ return IB_WC_LOC_LEN_ERR;
	+ case FLUSH_GENERAL_ERR:
	+ return IB_WC_GENERAL_ERR;
	+ case FLUSH_FATAL_ERR:
	+ default:
	+ return IB_WC_FATAL_ERR;
	+ }
	+}
	+
	+/**
	+ * i40iw_set_flush_info - set flush info
	+ * @pinfo: set flush info
	+ * @min: minor err
	+ * @maj: major err
	+ * @opcode: flush error code
	+ */
	+static void
	+i40iw_set_flush_info(struct i40iw_qp_flush_info pinfo, u16 min, u16 *maj,
	+ enum i40iw_flush_opcode opcode)
	+{
	+ *min = (u16)i40iw_get_ib_wc(opcode);
	+ *maj = CQE_MAJOR_DRV;
	+ pinfo->userflushcode = TRUE;
	+}
	+
	+/**
	+ * i40iw_flush_wqes - flush wqe for qp
	+ * @iwdev: iwarp device
	+ * @iwqp: qp to flush wqes
	+ */
	+void
	+i40iw_flush_wqes(struct i40iw_device iwdev, struct i40iw_qp iwqp)
	+{
	+ struct i40iw_qp_flush_info info;
	+ struct i40iw_qp_flush_info *pinfo = &info;
	+ struct i40iw_sc_qp *qp = &iwqp->sc_qp;
	+
	+ memset(pinfo, 0, sizeof(*pinfo));
	+ info.sq = TRUE;
	+ info.rq = TRUE;
	+ if (qp->term_flags) {
	+ i40iw_set_flush_info(pinfo, &pinfo->sq_minor_code,
	+ &pinfo->sq_major_code, qp->flush_code);
	+ i40iw_set_flush_info(pinfo, &pinfo->rq_minor_code,
	+ &pinfo->rq_major_code, qp->flush_code);
	+ }
	+ (void)i40iw_hw_flush_wqes(iwdev, &iwqp->sc_qp, &info, TRUE);
	+}
	+
	+void
	+i40iw_hw_stats_start_timer(struct i40iw_sc_vsi *vsi)
	+{
	+ return;
	+}
	+
	+void
	+i40iw_hw_stats_stop_timer(struct i40iw_sc_vsi *vsi)
	+{
	+ return;
	+}
	Index: sys/dev/ixl/iwarp/iw_ixl_linux_genalloc.h
	===================================================================
	--- /dev/null
	+++ sys/dev/ixl/iwarp/iw_ixl_linux_genalloc.h
	@@ -0,0 +1,62 @@
	+/*******************************************************************************
	+*
	+* Copyright (c) 2015-2017 Intel Corporation. All rights reserved.
	+*
	+* This software is available to you under a choice of one of two
	+* licenses. You may choose to be licensed under the terms of the GNU
	+* General Public License (GPL) Version 2, available from the file
	+* COPYING in the main directory of this source tree, or the
	+* OpenFabrics.org BSD license below:
	+*
	+* Redistribution and use in source and binary forms, with or
	+* without modification, are permitted provided that the following
	+* conditions are met:
	+*
	+* - Redistributions of source code must retain the above
	+* copyright notice, this list of conditions and the following
	+* disclaimer.
	+*
	+* - Redistributions in binary form must reproduce the above
	+* copyright notice, this list of conditions and the following
	+* disclaimer in the documentation and/or other materials
	+* provided with the distribution.
	+*
	+* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
	+* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
	+* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
	+* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
	+* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
	+* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
	+* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
	+* SOFTWARE.
	+*
	+*******************************************************************************/
	+/$FreeBSD$/
	+
	+#ifndef _IW_IXL_LINUX_GENALLOC_H_
	+#define _IW_IXL_LINUX_GENALLOC_H_
	+
	+#define gen_pool_create(s, n) \
	+ i40iw_gen_pool_create((s))
	+#define gen_pool_destroy(p) \
	+ i40iw_gen_pool_destroy((p))
	+#define gen_pool_add_virt(p, v, ph, s, n) \
	+ i40iw_gen_pool_add_virt((p), (v), (ph), (s))
	+#define gen_pool_alloc(p, s) \
	+ i40iw_gen_pool_alloc((p), (s))
	+#define gen_pool_free(p, a, s) \
	+ i40iw_gen_pool_free((p), (a), (s))
	+#define gen_pool_virt_to_phys(p, a) \
	+ i40iw_gen_pool_virt_to_phys((p), (a))
	+
	+#define I40IW_GEN_POOL_BLK_CNT(size, shift) \
	+ (((size) + (1 << (shift)) - 1) >> (shift))
	+
	+struct gen_pool *i40iw_gen_pool_create(int);
	+void i40iw_gen_pool_destroy(struct gen_pool *);
	+int i40iw_gen_pool_add_virt(struct gen_pool *, unsigned long, phys_addr_t, size_t);
	+unsigned long i40iw_gen_pool_alloc(struct gen_pool *, size_t);
	+void i40iw_gen_pool_free(struct gen_pool *, unsigned long, size_t);
	+phys_addr_t i40iw_gen_pool_virt_to_phys(struct gen_pool *, unsigned long);
	+
	+#endif /* _IW_IXL_LINUX_GENALLOC_H_ */
	Index: sys/dev/ixl/iwarp/iw_ixl_linux_genalloc.c
	===================================================================
	--- /dev/null
	+++ sys/dev/ixl/iwarp/iw_ixl_linux_genalloc.c
	@@ -0,0 +1,186 @@
	+/*******************************************************************************
	+*
	+* Copyright (c) 2015-2017 Intel Corporation. All rights reserved.
	+*
	+* This software is available to you under a choice of one of two
	+* licenses. You may choose to be licensed under the terms of the GNU
	+* General Public License (GPL) Version 2, available from the file
	+* COPYING in the main directory of this source tree, or the
	+* OpenFabrics.org BSD license below:
	+*
	+* Redistribution and use in source and binary forms, with or
	+* without modification, are permitted provided that the following
	+* conditions are met:
	+*
	+* - Redistributions of source code must retain the above
	+* copyright notice, this list of conditions and the following
	+* disclaimer.
	+*
	+* - Redistributions in binary form must reproduce the above
	+* copyright notice, this list of conditions and the following
	+* disclaimer in the documentation and/or other materials
	+* provided with the distribution.
	+*
	+* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
	+* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
	+* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
	+* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
	+* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
	+* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
	+* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
	+* SOFTWARE.
	+*
	+*******************************************************************************/
	+/$FreeBSD$/
	+
	+#include <linux/types.h>
	+#include <sys/blist.h>
	+#include <sys/malloc.h>
	+#include <sys/queue.h>
	+#include <sys/lock.h>
	+#include <sys/mutex.h>
	+#include "iw_ixl_linux_genalloc.h"
	+#include "iw_ixl_dbg.h"
	+
	+struct i40iw_pool_blist {
	+ LIST_ENTRY(i40iw_pool_blist) node;
	+ blist_t blist;
	+ phys_addr_t phys;
	+ unsigned long virt;
	+ size_t size;
	+};
	+
	+LIST_HEAD(i40iw_pool_blists_head, i40iw_pool_blist);
	+struct gen_pool {
	+ struct i40iw_pool_blists_head blists;
	+ int size_shift;
	+ struct mtx mtx;
	+};
	+
	+struct gen_pool *
	+i40iw_gen_pool_create(int size_shift)
	+{
	+ struct gen_pool *pool;
	+
	+ pool = malloc(sizeof(struct gen_pool), M_DEVBUF, M_NOWAIT \| M_ZERO);
	+ if (pool == NULL)
	+ return NULL;
	+
	+ mtx_init(&pool->mtx, "iw_ixl_gen_pool", NULL, MTX_DEF);
	+ LIST_INIT(&pool->blists);
	+ pool->size_shift = size_shift;
	+
	+ return (pool);
	+}
	+
	+void
	+i40iw_gen_pool_destroy(struct gen_pool *pool)
	+{
	+ struct i40iw_pool_blist bl, t_bl;
	+
	+ if (pool != NULL) {
	+ mtx_lock(&pool->mtx);
	+ LIST_FOREACH_SAFE(bl, &pool->blists, node, t_bl) {
	+ blist_destroy(bl->blist);
	+ LIST_REMOVE(bl, node);
	+ free(bl, M_DEVBUF);
	+ }
	+ mtx_unlock(&pool->mtx);
	+
	+ mtx_destroy(&pool->mtx);
	+ free(pool, M_DEVBUF);
	+ }
	+}
	+
	+int
	+i40iw_gen_pool_add_virt(struct gen_pool *pool, unsigned long virt,
	+ phys_addr_t phys, size_t size)
	+{
	+ struct i40iw_pool_blist *bl;
	+
	+ bl = malloc(sizeof(struct i40iw_pool_blist), M_DEVBUF,
	+ M_NOWAIT \| M_ZERO);
	+ if (bl == NULL)
	+ return (ENOMEM);
	+
	+ bl->blist = blist_create(size >> pool->size_shift, M_NOWAIT);
	+ if (bl->blist == NULL) {
	+ free(bl, M_DEVBUF);
	+ return (ENOMEM);
	+ }
	+ blist_free(bl->blist, 0, size >> pool->size_shift);
	+ bl->virt = virt;
	+ bl->phys = phys;
	+ bl->size = size;
	+
	+ mtx_lock(&pool->mtx);
	+ LIST_INSERT_HEAD(&pool->blists, bl, node);
	+ mtx_unlock(&pool->mtx);
	+
	+ return (0);
	+}
	+
	+unsigned long
	+i40iw_gen_pool_alloc(struct gen_pool *pool, size_t size)
	+{
	+ struct i40iw_pool_blist *bl;
	+ daddr_t blk_no = SWAPBLK_NONE;
	+ int blk_cnt;
	+
	+ blk_cnt = I40IW_GEN_POOL_BLK_CNT(size, pool->size_shift);
	+
	+ mtx_lock(&pool->mtx);
	+ LIST_FOREACH(bl, &pool->blists, node) {
	+ blk_no = blist_alloc(bl->blist, blk_cnt);
	+ if (blk_no != SWAPBLK_NONE)
	+ break;
	+ }
	+ mtx_unlock(&pool->mtx);
	+
	+ if (blk_no == SWAPBLK_NONE)
	+ return (0);
	+
	+ return (bl->virt + ((unsigned long)blk_no << pool->size_shift));
	+}
	+
	+void
	+i40iw_gen_pool_free(struct gen_pool *pool, unsigned long addr, size_t size)
	+{
	+ struct i40iw_pool_blist *bl;
	+ daddr_t blk_no;
	+ int blk_cnt;
	+
	+ mtx_lock(&pool->mtx);
	+ LIST_FOREACH(bl, &pool->blists, node) {
	+ if ((bl->virt <= addr) &&
	+ (bl->virt + bl->size >= addr + size)) {
	+ blk_no = (addr - bl->virt) >> pool->size_shift;
	+ blk_cnt = I40IW_GEN_POOL_BLK_CNT(size, pool->size_shift);
	+ blist_free(bl->blist, blk_no, blk_cnt);
	+ break;
	+ }
	+ }
	+ mtx_unlock(&pool->mtx);
	+
	+ IW_IXL_ASSERT(bl != NULL);
	+}
	+
	+phys_addr_t
	+i40iw_gen_pool_virt_to_phys(struct gen_pool *pool, unsigned long addr)
	+{
	+ struct i40iw_pool_blist *bl;
	+ phys_addr_t phys = -1;
	+
	+ mtx_lock(&pool->mtx);
	+ LIST_FOREACH(bl, &pool->blists, node) {
	+ if ((bl->virt <= addr) &&
	+ (bl->virt + bl->size > addr)) {
	+ phys = bl->phys + (addr - bl->virt);
	+ break;
	+ }
	+ }
	+ mtx_unlock(&pool->mtx);
	+
	+ IW_IXL_ASSERT(bl != NULL);
	+ return phys;
	+}
	Index: sys/dev/ixl/iwarp/iw_ixl_linux_jiffies.h
	===================================================================
	--- /dev/null
	+++ sys/dev/ixl/iwarp/iw_ixl_linux_jiffies.h
	@@ -0,0 +1,51 @@
	+/*******************************************************************************
	+*
	+* Copyright (c) 2015-2017 Intel Corporation. All rights reserved.
	+*
	+* This software is available to you under a choice of one of two
	+* licenses. You may choose to be licensed under the terms of the GNU
	+* General Public License (GPL) Version 2, available from the file
	+* COPYING in the main directory of this source tree, or the
	+* OpenFabrics.org BSD license below:
	+*
	+* Redistribution and use in source and binary forms, with or
	+* without modification, are permitted provided that the following
	+* conditions are met:
	+*
	+* - Redistributions of source code must retain the above
	+* copyright notice, this list of conditions and the following
	+* disclaimer.
	+*
	+* - Redistributions in binary form must reproduce the above
	+* copyright notice, this list of conditions and the following
	+* disclaimer in the documentation and/or other materials
	+* provided with the distribution.
	+*
	+* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
	+* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
	+* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
	+* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
	+* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
	+* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
	+* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
	+* SOFTWARE.
	+*
	+*******************************************************************************/
	+/$FreeBSD$/
	+
	+#ifndef _IW_IXL_LINUX_JIFFIES_H_
	+#define _IW_IXL_LINUX_JIFFIES_H_
	+
	+#include <linux/jiffies.h>
	+
	+static inline int
	+linux_timer_jiffies_until(unsigned long expires)
	+{
	+ int delta = expires - jiffies;
	+ /* guard against already expired values */
	+ if (delta < 1)
	+ delta = 1;
	+ return (delta);
	+}
	+
	+#endif /* _IW_IXL_LINUX_JIFIIES_H_ */
	Index: sys/dev/ixl/iwarp/iw_ixl_linux_wait.h
	===================================================================
	--- /dev/null
	+++ sys/dev/ixl/iwarp/iw_ixl_linux_wait.h
	@@ -0,0 +1,88 @@
	+/*******************************************************************************
	+*
	+* Copyright (c) 2015-2017 Intel Corporation. All rights reserved.
	+*
	+* This software is available to you under a choice of one of two
	+* licenses. You may choose to be licensed under the terms of the GNU
	+* General Public License (GPL) Version 2, available from the file
	+* COPYING in the main directory of this source tree, or the
	+* OpenFabrics.org BSD license below:
	+*
	+* Redistribution and use in source and binary forms, with or
	+* without modification, are permitted provided that the following
	+* conditions are met:
	+*
	+* - Redistributions of source code must retain the above
	+* copyright notice, this list of conditions and the following
	+* disclaimer.
	+*
	+* - Redistributions in binary form must reproduce the above
	+* copyright notice, this list of conditions and the following
	+* disclaimer in the documentation and/or other materials
	+* provided with the distribution.
	+*
	+* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
	+* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
	+* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
	+* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
	+* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
	+* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
	+* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
	+* SOFTWARE.
	+*
	+*******************************************************************************/
	+/$FreeBSD$/
	+
	+#ifndef _IW_IXL_LINUX_WAIT_H_
	+#define _IW_IXL_LINUX_WAIT_H_
	+
	+/*
	+ * TODO: Content of this file should ultimately land in linux-kpi
	+ * (linux/wait.h file)
	+ */
	+
	+#include <linux/wait.h>
	+
	+#ifndef wait_event_timeout
	+/*
	+ * This is left for backward compatibility with FreeBSD 10.3 and 11.0
	+ */
	+#define wait_event_timeout(q, cond, timeout) \
	+({ \
	+ void *c = &(q).wchan; \
	+ long end = jiffies + timeout; \
	+ int __ret = 0; \
	+ int __rc = 0; \
	+ \
	+ if (!(cond)) { \
	+ for (; __rc == 0;) { \
	+ if (SCHEDULER_STOPPED()) \
	+ break; \
	+ sleepq_lock(c); \
	+ if (cond) { \
	+ sleepq_release(c); \
	+ __ret = 1; \
	+ break; \
	+ } \
	+ sleepq_add(c, NULL, "completion", \
	+ SLEEPQ_SLEEP, 0); \
	+ sleepq_set_timeout(c, \
	+ linux_timer_jiffies_until(end));\
	+ __rc = sleepq_timedwait(c, 0); \
	+ if (__rc != 0) \
	+ __ret = (cond); \
	+ } \
	+ } else { \
	+ /* return remaining jiffies (at least 1) if the \
	+ * condition evaluated to TRUE before the \
	+ * timeout elapsed. \
	+ */ \
	+ __ret = (end - jiffies); \
	+ if( __ret < 1 ) \
	+ __ret = 1; \
	+ } \
	+ __ret; \
	+})
	+#endif
	+
	+#endif /* _IW_IXL_LINUX_WAIT_H_ */
	Index: sys/dev/ixl/iwarp/iw_ixl_pble.h
	===================================================================
	--- /dev/null
	+++ sys/dev/ixl/iwarp/iw_ixl_pble.h
	@@ -0,0 +1,128 @@
	+/*******************************************************************************
	+*
	+* Copyright (c) 2015-2017 Intel Corporation. All rights reserved.
	+*
	+* This software is available to you under a choice of one of two
	+* licenses. You may choose to be licensed under the terms of the GNU
	+* General Public License (GPL) Version 2, available from the file
	+* COPYING in the main directory of this source tree, or the
	+* OpenFabrics.org BSD license below:
	+*
	+* Redistribution and use in source and binary forms, with or
	+* without modification, are permitted provided that the following
	+* conditions are met:
	+*
	+* - Redistributions of source code must retain the above
	+* copyright notice, this list of conditions and the following
	+* disclaimer.
	+*
	+* - Redistributions in binary form must reproduce the above
	+* copyright notice, this list of conditions and the following
	+* disclaimer in the documentation and/or other materials
	+* provided with the distribution.
	+*
	+* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
	+* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
	+* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
	+* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
	+* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
	+* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
	+* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
	+* SOFTWARE.
	+*
	+*******************************************************************************/
	+/$FreeBSD$/
	+
	+#ifndef _IW_IXL_PBLE_H_
	+#define _IW_IXL_PBLE_H_
	+
	+#define POOL_SHIFT 6
	+#define PBLE_PER_PAGE 512
	+#define I40IW_HMC_PAGED_BP_SHIFT 12
	+#define PBLE_512_SHIFT 9
	+
	+enum i40iw_pble_level {
	+ I40IW_LEVEL_0 = 0,
	+ I40IW_LEVEL_1 = 1,
	+ I40IW_LEVEL_2 = 2
	+};
	+
	+enum i40iw_alloc_type {
	+ I40IW_NO_ALLOC = 0,
	+ I40IW_DMA_COHERENT = 1,
	+ I40IW_VMALLOC = 2
	+};
	+
	+struct i40iw_pble_info {
	+ unsigned long addr;
	+ u32 idx;
	+ u32 cnt;
	+};
	+
	+struct i40iw_pble_level2 {
	+ struct i40iw_pble_info root;
	+ struct i40iw_pble_info *leaf;
	+ u32 leaf_cnt;
	+};
	+
	+struct i40iw_pble_alloc {
	+ u32 total_cnt;
	+ enum i40iw_pble_level level;
	+ union {
	+ struct i40iw_pble_info level1;
	+ struct i40iw_pble_level2 level2;
	+ };
	+};
	+
	+struct i40iw_pble_pool {
	+ struct gen_pool *pool;
	+ struct list_head clist;
	+ u32 total_pble_alloc;
	+ u32 free_pble_cnt;
	+ u32 pool_shift;
	+};
	+
	+struct sd_pd_idx {
	+ u32 sd_idx;
	+ u32 pd_idx;
	+ u32 rel_pd_idx;
	+};
	+
	+struct i40iw_add_page_info {
	+ struct i40iw_chunk *chunk;
	+ struct i40iw_hmc_sd_entry *sd_entry;
	+ struct i40iw_hmc_info *hmc_info;
	+ struct sd_pd_idx idx;
	+ u32 pages;
	+};
	+
	+struct i40iw_chunk {
	+ struct list_head list;
	+ u32 size;
	+ void *vaddr;
	+ u64 fpm_addr;
	+ dma_addr_t *dmaaddrs;
	+ enum i40iw_alloc_type type;
	+};
	+
	+struct i40iw_hmc_pble_rsrc {
	+ u32 unallocated_pble;
	+ u64 fpm_base_addr;
	+ u64 next_fpm_addr;
	+ struct i40iw_pble_pool pinfo;
	+
	+ u32 stats_direct_sds;
	+ u32 stats_paged_sds;
	+ u64 stats_alloc_ok;
	+ u64 stats_alloc_fail;
	+ u64 stats_alloc_freed;
	+ u64 stats_lvl1;
	+ u64 stats_lvl2;
	+};
	+
	+void i40iw_destroy_pble_pool(struct i40iw_sc_dev , struct i40iw_hmc_pble_rsrc );
	+enum i40iw_status_code i40iw_hmc_init_pble(struct i40iw_sc_dev , struct i40iw_hmc_pble_rsrc );
	+void i40iw_free_pble(struct i40iw_hmc_pble_rsrc , struct i40iw_pble_alloc );
	+enum i40iw_status_code i40iw_get_pble(struct i40iw_sc_dev , struct i40iw_hmc_pble_rsrc , struct i40iw_pble_alloc *, u32);
	+
	+#endif /* _IW_IXL_PBLE_H_ */
	Index: sys/dev/ixl/iwarp/iw_ixl_pble.c
	===================================================================
	--- /dev/null
	+++ sys/dev/ixl/iwarp/iw_ixl_pble.c
	@@ -0,0 +1,587 @@
	+/*******************************************************************************
	+*
	+* Copyright (c) 2015-2017 Intel Corporation. All rights reserved.
	+*
	+* This software is available to you under a choice of one of two
	+* licenses. You may choose to be licensed under the terms of the GNU
	+* General Public License (GPL) Version 2, available from the file
	+* COPYING in the main directory of this source tree, or the
	+* OpenFabrics.org BSD license below:
	+*
	+* Redistribution and use in source and binary forms, with or
	+* without modification, are permitted provided that the following
	+* conditions are met:
	+*
	+* - Redistributions of source code must retain the above
	+* copyright notice, this list of conditions and the following
	+* disclaimer.
	+*
	+* - Redistributions in binary form must reproduce the above
	+* copyright notice, this list of conditions and the following
	+* disclaimer in the documentation and/or other materials
	+* provided with the distribution.
	+*
	+* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
	+* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
	+* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
	+* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
	+* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
	+* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
	+* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
	+* SOFTWARE.
	+*
	+*******************************************************************************/
	+/$FreeBSD$/
	+
	+#include "i40iw_status.h"
	+#include "i40iw_osdep.h"
	+#include "i40iw_register.h"
	+#include "i40iw_hmc.h"
	+
	+#include "i40iw_d.h"
	+#include "i40iw_type.h"
	+#include "i40iw_p.h"
	+
	+#include "iw_ixl_linux_genalloc.h"
	+
	+#include "iw_ixl.h"
	+
	+struct i40iw_device;
	+static enum i40iw_status_code
	+add_pble_pool(struct i40iw_sc_dev *dev,
	+ struct i40iw_hmc_pble_rsrc *pble_rsrc);
	+static void i40iw_free_vmalloc_mem(struct i40iw_chunk *chunk);
	+
	+/**
	+ * i40iw_destroy_pble_pool - destroy pool during module unload
	+ * @pble_rsrc: pble resources
	+ */
	+void
	+i40iw_destroy_pble_pool(struct i40iw_sc_dev *dev,
	+ struct i40iw_hmc_pble_rsrc *pble_rsrc)
	+{
	+ struct list_head *clist;
	+ struct list_head *tlist;
	+ struct i40iw_chunk *chunk;
	+ struct i40iw_pble_pool *pinfo = &pble_rsrc->pinfo;
	+
	+ if (pinfo->pool) {
	+ list_for_each_safe(clist, tlist, &pinfo->clist) {
	+ chunk = list_entry(clist, struct i40iw_chunk, list);
	+ if (chunk->type == I40IW_VMALLOC)
	+ i40iw_free_vmalloc_mem(chunk);
	+ kfree(chunk);
	+ }
	+ gen_pool_destroy(pinfo->pool);
	+ }
	+}
	+
	+/**
	+ * i40iw_hmc_init_pble - Initialize pble resources during module load
	+ * @dev: i40iw_sc_dev struct
	+ * @pble_rsrc: pble resources
	+ */
	+enum i40iw_status_code
	+i40iw_hmc_init_pble(struct i40iw_sc_dev *dev,
	+ struct i40iw_hmc_pble_rsrc *pble_rsrc)
	+{
	+ struct i40iw_hmc_info *hmc_info;
	+ u32 fpm_idx = 0;
	+
	+ hmc_info = dev->hmc_info;
	+ pble_rsrc->fpm_base_addr = hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].base;
	+ /* Now start the pble' on 4k boundary */
	+ if (pble_rsrc->fpm_base_addr & 0xfff)
	+ fpm_idx = (PAGE_SIZE - (pble_rsrc->fpm_base_addr & 0xfff)) >> 3;
	+
	+ pble_rsrc->unallocated_pble =
	+ hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].cnt - fpm_idx;
	+ pble_rsrc->next_fpm_addr = pble_rsrc->fpm_base_addr + (fpm_idx << 3);
	+
	+ pble_rsrc->pinfo.pool_shift = POOL_SHIFT;
	+ pble_rsrc->pinfo.pool = gen_pool_create(pble_rsrc->pinfo.pool_shift, -1);
	+ INIT_LIST_HEAD(&pble_rsrc->pinfo.clist);
	+ if (!pble_rsrc->pinfo.pool)
	+ goto error;
	+
	+ if (add_pble_pool(dev, pble_rsrc))
	+ goto error;
	+
	+ return 0;
	+
	+error: i40iw_destroy_pble_pool(dev, pble_rsrc);
	+ return I40IW_ERR_NO_MEMORY;
	+}
	+
	+/**
	+ * get_sd_pd_idx - Returns sd index, pd index and rel_pd_idx from fpm address
	+ * @ pble_rsrc: structure containing fpm address
	+ * @ idx: where to return indexes
	+ */
	+static inline void
	+get_sd_pd_idx(struct i40iw_hmc_pble_rsrc *pble_rsrc,
	+ struct sd_pd_idx *idx)
	+{
	+ idx->sd_idx = (u32)(pble_rsrc->next_fpm_addr) / I40IW_HMC_DIRECT_BP_SIZE;
	+ idx->pd_idx = (u32)(pble_rsrc->next_fpm_addr) / I40IW_HMC_PAGED_BP_SIZE;
	+ idx->rel_pd_idx = (idx->pd_idx % I40IW_HMC_PD_CNT_IN_SD);
	+}
	+
	+/**
	+ * add_sd_direct - add sd direct for pble
	+ * @dev: hardware control device structure
	+ * @pble_rsrc: pble resource ptr
	+ * @info: page info for sd
	+ */
	+static enum i40iw_status_code
	+add_sd_direct(struct i40iw_sc_dev *dev,
	+ struct i40iw_hmc_pble_rsrc *pble_rsrc,
	+ struct i40iw_add_page_info *info)
	+{
	+ struct sd_pd_idx *idx = &info->idx;
	+ struct i40iw_chunk *chunk = info->chunk;
	+ struct i40iw_hmc_info *hmc_info = info->hmc_info;
	+ struct i40iw_hmc_sd_entry *sd_entry = info->sd_entry;
	+ u32 offset = 0;
	+ enum i40iw_status_code ret_code = 0;
	+
	+ if (!sd_entry->valid) {
	+ if (dev->is_pf) {
	+ ret_code = i40iw_add_sd_table_entry(dev->hw, hmc_info,
	+ info->idx.sd_idx,
	+ I40IW_SD_TYPE_DIRECT,
	+ I40IW_HMC_DIRECT_BP_SIZE);
	+ if (ret_code)
	+ return ret_code;
	+ chunk->type = I40IW_DMA_COHERENT;
	+ }
	+ }
	+ offset = idx->rel_pd_idx << I40IW_HMC_PAGED_BP_SHIFT;
	+ chunk->size = info->pages << I40IW_HMC_PAGED_BP_SHIFT;
	+ chunk->vaddr = ((u8 *)sd_entry->u.bp.addr.va + offset);
	+ chunk->fpm_addr = pble_rsrc->next_fpm_addr;
	+ i40iw_debug(dev, I40IW_DEBUG_PBLE, "chunk_size[%d] = 0x%x vaddr=%p fpm_addr = %llx\n",
	+ chunk->size, chunk->size, chunk->vaddr, (unsigned long long)chunk->fpm_addr);
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_free_vmalloc_mem - free vmalloc during close
	+ * @hw: hw struct
	+ * @chunk: chunk information for vmalloc
	+ */
	+static void
	+i40iw_free_vmalloc_mem(struct i40iw_chunk *chunk)
	+{
	+ free(chunk->dmaaddrs, M_DEVBUF);
	+ chunk->dmaaddrs = NULL;
	+ kmem_free(kernel_arena, (vm_offset_t) chunk->vaddr, chunk->size);
	+ chunk->vaddr = NULL;
	+ chunk->type = 0;
	+}
	+
	+/**
	+ * i40iw_get_vmalloc_mem - get 2M page for sd
	+ * @hw: hardware address
	+ * @chunk: chunk to adf
	+ * @pg_cnt: #of 4 K pages
	+ */
	+static enum i40iw_status_code
	+i40iw_get_vmalloc_mem(struct i40iw_chunk *chunk, int pg_cnt)
	+{
	+ int i;
	+ u32 size;
	+ u8 *addr;
	+
	+ chunk->dmaaddrs = malloc(pg_cnt << 3, M_DEVBUF, M_NOWAIT \| M_ZERO);
	+ if (!chunk->dmaaddrs)
	+ return I40IW_ERR_NO_MEMORY;
	+ size = PAGE_SIZE * pg_cnt;
	+ chunk->vaddr = (void *)kmem_alloc_attr(kernel_arena, size,
	+ M_NOWAIT \| M_ZERO, 0, BUS_SPACE_MAXADDR, VM_MEMATTR_DEFAULT);
	+ if (!chunk->vaddr) {
	+ free(chunk->dmaaddrs, M_DEVBUF);
	+ chunk->dmaaddrs = NULL;
	+ return I40IW_ERR_NO_MEMORY;
	+ }
	+ chunk->size = size;
	+ addr = (u8 *)chunk->vaddr;
	+
	+ for (i = 0; i < pg_cnt; i++, addr += PAGE_SIZE)
	+ chunk->dmaaddrs[i] = vtophys(addr);
	+
	+ chunk->type = I40IW_VMALLOC;
	+ if (i == pg_cnt)
	+ return 0;
	+
	+ i40iw_free_vmalloc_mem(chunk);
	+ return I40IW_ERR_NO_MEMORY;
	+}
	+
	+/**
	+ * fpm_to_idx - given fpm address, get pble index
	+ * @pble_rsrc: pble resource management
	+ * @addr: fpm address for index
	+ */
	+static inline u32
	+fpm_to_idx(struct i40iw_hmc_pble_rsrc *pble_rsrc, u64 addr)
	+{
	+ return (addr - (pble_rsrc->fpm_base_addr)) >> 3;
	+}
	+
	+/**
	+ * add_bp_pages - add backing pages for sd
	+ * @dev: hardware control device structure
	+ * @pble_rsrc: pble resource management
	+ * @info: page info for sd
	+ */
	+static enum i40iw_status_code
	+add_bp_pages(struct i40iw_sc_dev dev, struct i40iw_hmc_pble_rsrc pble_rsrc,
	+ struct i40iw_add_page_info *info)
	+{
	+ struct i40iw_dma_mem mem;
	+ struct i40iw_hmc_pd_entry *pd_entry;
	+ struct i40iw_hmc_sd_entry *sd_entry = info->sd_entry;
	+ struct i40iw_hmc_info *hmc_info = info->hmc_info;
	+ struct i40iw_chunk *chunk = info->chunk;
	+ u32 i;
	+ u32 pd_idx = info->idx.pd_idx;
	+ u32 rel_pd_idx = info->idx.rel_pd_idx;
	+ u8 *addr;
	+ enum i40iw_status_code status = 0;
	+
	+ status = i40iw_get_vmalloc_mem(chunk, info->pages);
	+ if (status)
	+ return I40IW_ERR_NO_MEMORY;
	+ status = i40iw_add_sd_table_entry(dev->hw, hmc_info,
	+ info->idx.sd_idx, I40IW_SD_TYPE_PAGED,
	+ I40IW_HMC_DIRECT_BP_SIZE);
	+ if (status) {
	+ i40iw_free_vmalloc_mem(chunk);
	+ return status;
	+ }
	+ addr = chunk->vaddr;
	+ for (i = 0; i < info->pages; i++) {
	+ mem.pa = chunk->dmaaddrs[i];
	+ mem.size = PAGE_SIZE;
	+ mem.va = (void *)(addr);
	+ pd_entry = &sd_entry->u.pd_table.pd_entry[rel_pd_idx++];
	+ if (!pd_entry->valid) {
	+ status = i40iw_add_pd_table_entry(dev->hw, hmc_info, pd_idx++, &mem);
	+ if (status)
	+ goto error;
	+ addr += PAGE_SIZE;
	+ } else {
	+ DPRINTF("pd entry is valid expecting to be invalid\n");
	+ }
	+ }
	+ chunk->fpm_addr = pble_rsrc->next_fpm_addr;
	+ return 0;
	+error:
	+ i40iw_free_vmalloc_mem(chunk);
	+ return status;
	+}
	+
	+/**
	+ * add_pble_pool - add a sd entry for pble resoure
	+ * @dev: hardware control device structure
	+ * @pble_rsrc: pble resource management
	+ */
	+static enum i40iw_status_code
	+add_pble_pool(struct i40iw_sc_dev dev, struct i40iw_hmc_pble_rsrc pble_rsrc)
	+{
	+ struct i40iw_hmc_sd_entry *sd_entry;
	+ struct i40iw_hmc_info *hmc_info;
	+ struct i40iw_chunk *chunk;
	+ struct i40iw_add_page_info info;
	+ struct sd_pd_idx *idx = &info.idx;
	+ u64 sd_reg_val = 0;
	+ u32 pages;
	+ enum i40iw_status_code ret_code = 0;
	+ enum i40iw_sd_entry_type sd_entry_type;
	+
	+ if (pble_rsrc->unallocated_pble < PBLE_PER_PAGE)
	+ return I40IW_ERR_NO_MEMORY;
	+ if (pble_rsrc->next_fpm_addr & 0xfff) {
	+ DPRINTF("next fpm_addr %llx\n",
	+ (unsigned long long)pble_rsrc->next_fpm_addr);
	+ return I40IW_ERR_INVALID_PAGE_DESC_INDEX;
	+ }
	+ chunk = kzalloc(sizeof(*chunk), GFP_KERNEL);
	+ if (!chunk)
	+ return I40IW_ERR_NO_MEMORY;
	+ hmc_info = dev->hmc_info;
	+ chunk->fpm_addr = pble_rsrc->next_fpm_addr;
	+ get_sd_pd_idx(pble_rsrc, idx);
	+ sd_entry = &hmc_info->sd_table.sd_entry[idx->sd_idx];
	+ pages = (idx->rel_pd_idx) ? (I40IW_HMC_PD_CNT_IN_SD -
	+ idx->rel_pd_idx) : I40IW_HMC_PD_CNT_IN_SD;
	+ pages = min(pages, pble_rsrc->unallocated_pble >> PBLE_512_SHIFT);
	+ info.chunk = chunk;
	+ info.hmc_info = hmc_info;
	+ info.pages = pages;
	+ info.sd_entry = sd_entry;
	+ if (!sd_entry->valid) {
	+ sd_entry_type = (!idx->rel_pd_idx &&
	+ (pages == I40IW_HMC_PD_CNT_IN_SD) &&
	+ dev->is_pf) ? I40IW_SD_TYPE_DIRECT : I40IW_SD_TYPE_PAGED;
	+ } else {
	+ sd_entry_type = sd_entry->entry_type;
	+ }
	+ i40iw_debug(dev, I40IW_DEBUG_PBLE,
	+ "pages = %d, unallocated_pble[%u] current_fpm_addr = %llx\n",
	+ pages, pble_rsrc->unallocated_pble,
	+ (unsigned long long)pble_rsrc->next_fpm_addr);
	+ i40iw_debug(dev, I40IW_DEBUG_PBLE, "sd_entry_type = %d sd_entry valid = %d\n",
	+ sd_entry_type, sd_entry->valid);
	+
	+ if (sd_entry_type == I40IW_SD_TYPE_DIRECT)
	+ ret_code = add_sd_direct(dev, pble_rsrc, &info);
	+ if (ret_code)
	+ sd_entry_type = I40IW_SD_TYPE_PAGED;
	+ else
	+ pble_rsrc->stats_direct_sds++;
	+
	+ if (sd_entry_type == I40IW_SD_TYPE_PAGED) {
	+ ret_code = add_bp_pages(dev, pble_rsrc, &info);
	+ if (ret_code)
	+ goto error;
	+ else
	+ pble_rsrc->stats_paged_sds++;
	+ }
	+ if (gen_pool_add_virt(pble_rsrc->pinfo.pool, (unsigned long)chunk->vaddr,
	+ (phys_addr_t)chunk->fpm_addr, chunk->size, -1)) {
	+ DPRINTF("could not allocate memory by gen_pool_addr_virt()\n");
	+ ret_code = I40IW_ERR_NO_MEMORY;
	+ goto error;
	+ }
	+ pble_rsrc->next_fpm_addr += chunk->size;
	+ i40iw_debug(dev, I40IW_DEBUG_PBLE, "next_fpm_addr = %llx chunk_size[%u] = 0x%x\n",
	+ (unsigned long long)pble_rsrc->next_fpm_addr, chunk->size, chunk->size);
	+ pble_rsrc->unallocated_pble -= (chunk->size >> 3);
	+ list_add(&chunk->list, &pble_rsrc->pinfo.clist);
	+ sd_reg_val = (sd_entry_type == I40IW_SD_TYPE_PAGED) ?
	+ sd_entry->u.pd_table.pd_page_addr.pa : sd_entry->u.bp.addr.pa;
	+ if (sd_entry->valid)
	+ return 0;
	+ if (dev->is_pf) {
	+ ret_code = i40iw_hmc_sd_one(dev, hmc_info->hmc_fn_id,
	+ sd_reg_val, idx->sd_idx,
	+ sd_entry->entry_type, TRUE);
	+ if (ret_code) {
	+ DPRINTF("cqp cmd failed for sd (pbles)\n");
	+ goto error;
	+ }
	+ }
	+ sd_entry->valid = TRUE;
	+ return 0;
	+error:
	+ kfree(chunk);
	+ return ret_code;
	+}
	+
	+/**
	+ * free_lvl2 - fee level 2 pble
	+ * @pble_rsrc: pble resource management
	+ * @palloc: level 2 pble allocation
	+ */
	+static void
	+free_lvl2(struct i40iw_hmc_pble_rsrc *pble_rsrc,
	+ struct i40iw_pble_alloc *palloc)
	+{
	+ struct gen_pool *pool;
	+ struct i40iw_pble_level2 *lvl2 = &palloc->level2;
	+ struct i40iw_pble_info *root = &lvl2->root;
	+ struct i40iw_pble_info *leaf = lvl2->leaf;
	+ u32 i;
	+
	+ pool = pble_rsrc->pinfo.pool;
	+
	+ for (i = 0; i < lvl2->leaf_cnt; i++, leaf++) {
	+ if (leaf->addr)
	+ gen_pool_free(pool, leaf->addr, (leaf->cnt << 3));
	+ else
	+ break;
	+ }
	+
	+ if (root->addr)
	+ gen_pool_free(pool, root->addr, (root->cnt << 3));
	+
	+ kfree(lvl2->leaf);
	+ lvl2->leaf = NULL;
	+}
	+
	+/**
	+ * get_lvl2_pble - get level 2 pble resource
	+ * @pble_rsrc: pble resource management
	+ * @palloc: level 2 pble allocation
	+ * @pool: pool pointer
	+ */
	+static enum i40iw_status_code
	+get_lvl2_pble(struct i40iw_hmc_pble_rsrc *pble_rsrc,
	+ struct i40iw_pble_alloc *palloc,
	+ struct gen_pool *pool)
	+{
	+ struct i40iw_pble_level2 *lvl2 = &palloc->level2;
	+ struct i40iw_pble_info *root = &lvl2->root;
	+ struct i40iw_pble_info *leaf;
	+ u64 *addr;
	+ u32 lf4k, lflast, total, i;
	+ u32 pblcnt = PBLE_PER_PAGE;
	+
	+ /* number of full 512 (4K) leafs) */
	+ lf4k = palloc->total_cnt >> 9;
	+ lflast = palloc->total_cnt % PBLE_PER_PAGE;
	+ total = (lflast == 0) ? lf4k : lf4k + 1;
	+ lvl2->leaf_cnt = total;
	+
	+ leaf = kzalloc((sizeof(leaf) total), GFP_ATOMIC);
	+ if (!leaf)
	+ return I40IW_ERR_NO_MEMORY;
	+ lvl2->leaf = leaf;
	+ /* allocate pbles for the root */
	+ root->addr = gen_pool_alloc(pool, (total << 3));
	+ if (!root->addr) {
	+ kfree(lvl2->leaf);
	+ lvl2->leaf = NULL;
	+ return I40IW_ERR_NO_MEMORY;
	+ }
	+ root->idx = fpm_to_idx(pble_rsrc,
	+ (u64)gen_pool_virt_to_phys(pool, root->addr));
	+ root->cnt = total;
	+ addr = (u64 *)root->addr;
	+ for (i = 0; i < total; i++, leaf++) {
	+ pblcnt = (lflast && ((i + 1) == total)) ? lflast : PBLE_PER_PAGE;
	+ leaf->addr = gen_pool_alloc(pool, (pblcnt << 3));
	+ if (!leaf->addr)
	+ goto error;
	+ leaf->idx = fpm_to_idx(pble_rsrc, (u64)gen_pool_virt_to_phys(pool, leaf->addr));
	+
	+ leaf->cnt = pblcnt;
	+ *addr = (u64)leaf->idx;
	+ addr++;
	+ }
	+ palloc->level = I40IW_LEVEL_2;
	+ pble_rsrc->stats_lvl2++;
	+ return 0;
	+error:
	+ free_lvl2(pble_rsrc, palloc);
	+ return I40IW_ERR_NO_MEMORY;
	+}
	+
	+/**
	+ * get_lvl1_pble - get level 1 pble resource
	+ * @dev: hardware control device structure
	+ * @pble_rsrc: pble resource management
	+ * @palloc: level 1 pble allocation
	+ */
	+static enum i40iw_status_code
	+get_lvl1_pble(struct i40iw_sc_dev *dev,
	+ struct i40iw_hmc_pble_rsrc *pble_rsrc,
	+ struct i40iw_pble_alloc *palloc)
	+{
	+ struct gen_pool *pool;
	+ struct i40iw_pble_info *lvl1 = &palloc->level1;
	+ u64 *addr;
	+
	+ pool = pble_rsrc->pinfo.pool;
	+ addr = (u64 *)gen_pool_alloc(pool, (palloc->total_cnt << 3));
	+
	+ if (!addr)
	+ return I40IW_ERR_NO_MEMORY;
	+
	+ palloc->level = I40IW_LEVEL_1;
	+ lvl1->addr = (unsigned long)addr;
	+ lvl1->idx = fpm_to_idx(pble_rsrc, (u64)gen_pool_virt_to_phys(pool,
	+ (unsigned long)addr));
	+ lvl1->cnt = palloc->total_cnt;
	+ pble_rsrc->stats_lvl1++;
	+ return 0;
	+}
	+
	+/**
	+ * get_lvl1_lvl2_pble - calls get_lvl1 and get_lvl2 pble routine
	+ * @dev: i40iw_sc_dev struct
	+ * @pble_rsrc: pble resources
	+ * @palloc: contains all inforamtion regarding pble (idx + pble addr)
	+ * @pool: pointer to general purpose special memory pool descriptor
	+ */
	+static inline enum i40iw_status_code
	+get_lvl1_lvl2_pble(struct i40iw_sc_dev *dev,
	+ struct i40iw_hmc_pble_rsrc *pble_rsrc,
	+ struct i40iw_pble_alloc *palloc,
	+ struct gen_pool *pool)
	+{
	+ enum i40iw_status_code status = 0;
	+
	+ status = get_lvl1_pble(dev, pble_rsrc, palloc);
	+ if (status && (palloc->total_cnt > PBLE_PER_PAGE))
	+ status = get_lvl2_pble(pble_rsrc, palloc, pool);
	+ return status;
	+}
	+
	+/**
	+ * i40iw_get_pble - allocate pbles from the pool
	+ * @dev: i40iw_sc_dev struct
	+ * @pble_rsrc: pble resources
	+ * @palloc: contains all inforamtion regarding pble (idx + pble addr)
	+ * @pble_cnt: #of pbles requested
	+ */
	+enum i40iw_status_code
	+i40iw_get_pble(struct i40iw_sc_dev *dev,
	+ struct i40iw_hmc_pble_rsrc *pble_rsrc,
	+ struct i40iw_pble_alloc *palloc,
	+ u32 pble_cnt)
	+{
	+ struct gen_pool *pool;
	+ enum i40iw_status_code status = 0;
	+ u32 max_sds = 0;
	+ int i;
	+
	+ pool = pble_rsrc->pinfo.pool;
	+ palloc->total_cnt = pble_cnt;
	+ palloc->level = I40IW_LEVEL_0;
	+ /*
	+ * check first to see if we can get pble's without acquiring
	+ * additional sd's
	+ */
	+ status = get_lvl1_lvl2_pble(dev, pble_rsrc, palloc, pool);
	+ if (!status)
	+ goto exit;
	+ max_sds = (palloc->total_cnt >> 18) + 1;
	+ for (i = 0; i < max_sds; i++) {
	+ status = add_pble_pool(dev, pble_rsrc);
	+ if (status)
	+ break;
	+ status = get_lvl1_lvl2_pble(dev, pble_rsrc, palloc, pool);
	+ if (!status)
	+ break;
	+ }
	+exit:
	+ if (!status)
	+ pble_rsrc->stats_alloc_ok++;
	+ else
	+ pble_rsrc->stats_alloc_fail++;
	+
	+ return status;
	+}
	+
	+/**
	+ * i40iw_free_pble - put pbles back into pool
	+ * @pble_rsrc: pble resources
	+ * @palloc: contains all inforamtion regarding pble resource being freed
	+ */
	+void
	+i40iw_free_pble(struct i40iw_hmc_pble_rsrc *pble_rsrc,
	+ struct i40iw_pble_alloc *palloc)
	+{
	+ struct gen_pool *pool;
	+
	+ pool = pble_rsrc->pinfo.pool;
	+ if (palloc->level == I40IW_LEVEL_2)
	+ free_lvl2(pble_rsrc, palloc);
	+ else
	+ gen_pool_free(pool, palloc->level1.addr,
	+ (palloc->level1.cnt << 3));
	+ pble_rsrc->stats_alloc_freed++;
	+}
	Index: sys/dev/ixl/iwarp/iw_ixl_ucontext.h
	===================================================================
	--- /dev/null
	+++ sys/dev/ixl/iwarp/iw_ixl_ucontext.h
	@@ -0,0 +1,82 @@
	+/*******************************************************************************
	+*
	+* Copyright (c) 2015-2017 Intel Corporation. All rights reserved.
	+*
	+* This software is available to you under a choice of one of two
	+* licenses. You may choose to be licensed under the terms of the GNU
	+* General Public License (GPL) Version 2, available from the file
	+* COPYING in the main directory of this source tree, or the
	+* OpenFabrics.org BSD license below:
	+*
	+* Redistribution and use in source and binary forms, with or
	+* without modification, are permitted provided that the following
	+* conditions are met:
	+*
	+* - Redistributions of source code must retain the above
	+* copyright notice, this list of conditions and the following
	+* disclaimer.
	+*
	+* - Redistributions in binary form must reproduce the above
	+* copyright notice, this list of conditions and the following
	+* disclaimer in the documentation and/or other materials
	+* provided with the distribution.
	+*
	+* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
	+* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
	+* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
	+* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
	+* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
	+* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
	+* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
	+* SOFTWARE.
	+*
	+*******************************************************************************/
	+/$FreeBSD$/
	+
	+#ifndef _IW_IXL_UCONTEXT_H_
	+#define _IW_IXL_UCONTEXT_H_
	+
	+struct i40iw_alloc_pd_resp {
	+ __u32 pd_id;
	+ __u8 reserved[4];
	+};
	+
	+struct i40iw_create_cq_req {
	+ __u64 user_cq_buffer;
	+ __u64 user_shadow_area;
	+};
	+
	+struct i40iw_create_qp_req {
	+ __u64 user_wqe_buffers;
	+ __u64 user_compl_ctx;
	+
	+ /* UDA QP PHB */
	+ __u64 user_sq_phb; /* place for VA of the sq phb buff */
	+ __u64 user_rq_phb; /* place for VA of the rq phb buff */
	+};
	+
	+enum i40iw_memreg_type {
	+ IW_MEMREG_TYPE_MEM = 0x0000,
	+ IW_MEMREG_TYPE_QP = 0x0001,
	+ IW_MEMREG_TYPE_CQ = 0x0002,
	+ IW_MEMREG_TYPE_FMEM = 0x0003,
	+};
	+
	+struct i40iw_create_cq_resp {
	+ __u32 cq_id;
	+ __u32 cq_size;
	+ __u32 mmap_db_index;
	+ __u32 reserved;
	+};
	+
	+struct i40iw_create_qp_resp {
	+ __u32 qp_id;
	+ __u32 actual_sq_size;
	+ __u32 actual_rq_size;
	+ __u32 i40iw_drv_opt;
	+ __u16 push_idx;
	+ __u8 lsmm;
	+ __u8 rsvd2;
	+};
	+
	+#endif /* _IW_IXL_UCONTEXT_H_ */
	Index: sys/dev/ixl/iwarp/iw_ixl_utils.c
	===================================================================
	--- /dev/null
	+++ sys/dev/ixl/iwarp/iw_ixl_utils.c
	@@ -0,0 +1,1098 @@
	+/*******************************************************************************
	+*
	+* Copyright (c) 2015-2017 Intel Corporation. All rights reserved.
	+*
	+* This software is available to you under a choice of one of two
	+* licenses. You may choose to be licensed under the terms of the GNU
	+* General Public License (GPL) Version 2, available from the file
	+* COPYING in the main directory of this source tree, or the
	+* OpenFabrics.org BSD license below:
	+*
	+* Redistribution and use in source and binary forms, with or
	+* without modification, are permitted provided that the following
	+* conditions are met:
	+*
	+* - Redistributions of source code must retain the above
	+* copyright notice, this list of conditions and the following
	+* disclaimer.
	+*
	+* - Redistributions in binary form must reproduce the above
	+* copyright notice, this list of conditions and the following
	+* disclaimer in the documentation and/or other materials
	+* provided with the distribution.
	+*
	+* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
	+* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
	+* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
	+* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
	+* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
	+* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
	+* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
	+* SOFTWARE.
	+*
	+*******************************************************************************/
	+/$FreeBSD$/
	+
	+#include "iw_ixl_linux_wait.h"
	+#include "iw_ixl.h"
	+#if __FreeBSD_version < 1100000
	+#include "iw_ixl_linux_jiffies.h"
	+#endif /* IW_IXL_FREEBSD10 */
	+
	+#include <net/if_vlan_var.h>
	+#include <netinet/if_ether.h>
	+#include <netinet/ip.h>
	+#include <netinet/ip6.h>
	+#include <netinet/tcp.h>
	+
	+/**
	+ * i40iw_arp_table - manage arp table
	+ * @iwdev: iwarp device
	+ * @ip_addr: ip address for device
	+ * @mac_addr: mac address ptr
	+ * @action: modify, delete or add
	+ */
	+int
	+i40iw_arp_table(struct i40iw_device iwdev, u32 ip_addr, bool ipv4,
	+ u8 *mac_addr, u32 action)
	+{
	+ int arp_index;
	+ int err;
	+ u32 ip[4];
	+
	+ if (ipv4) {
	+ memset(ip, 0, sizeof(ip));
	+ ip[0] = *ip_addr;
	+ } else {
	+ memcpy(ip, ip_addr, sizeof(ip));
	+ }
	+
	+ for (arp_index = 0; (u32)arp_index < iwdev->arp_table_size; arp_index++)
	+ if (memcmp(iwdev->arp_table[arp_index].ip_addr, ip, sizeof(ip)) == 0)
	+ break;
	+ switch (action) {
	+ case I40IW_ARP_ADD:
	+ if (arp_index != iwdev->arp_table_size)
	+ return -1;
	+
	+ arp_index = 0;
	+ err = i40iw_alloc_resource(iwdev, iwdev->allocated_arps,
	+ iwdev->arp_table_size,
	+ (u32 *)&arp_index,
	+ &iwdev->next_arp_index);
	+
	+ if (err)
	+ return err;
	+
	+ memcpy(iwdev->arp_table[arp_index].ip_addr, ip, sizeof(ip));
	+ ether_addr_copy(iwdev->arp_table[arp_index].mac_addr, mac_addr);
	+ break;
	+ case I40IW_ARP_RESOLVE:
	+ if (arp_index == iwdev->arp_table_size)
	+ return -1;
	+ break;
	+ case I40IW_ARP_DELETE:
	+ if (arp_index == iwdev->arp_table_size)
	+ return -1;
	+ memset(iwdev->arp_table[arp_index].ip_addr, 0,
	+ sizeof(iwdev->arp_table[arp_index].ip_addr));
	+ memset(iwdev->arp_table[arp_index].mac_addr, 0x00,
	+ ETHER_ADDR_LEN);
	+ i40iw_free_resource(iwdev, iwdev->allocated_arps, arp_index);
	+ break;
	+ default:
	+ return -1;
	+ }
	+ return arp_index;
	+}
	+
	+/**
	+ * i40iw_get_cqp_request - get cqp struct
	+ * @cqp: device cqp ptr
	+ * @wait: cqp to be used in wait mode
	+ */
	+struct i40iw_cqp_request *
	+i40iw_get_cqp_request(struct i40iw_cqp *cqp, bool wait)
	+{
	+ struct i40iw_cqp_request *cqp_request = NULL;
	+ unsigned long flags;
	+
	+ spin_lock_irqsave(&cqp->req_lock, flags);
	+ if (!list_empty(&cqp->cqp_avail_reqs)) {
	+ cqp_request = list_entry(cqp->cqp_avail_reqs.next,
	+ struct i40iw_cqp_request, list);
	+ list_del_init(&cqp_request->list);
	+ }
	+ spin_unlock_irqrestore(&cqp->req_lock, flags);
	+ if (!cqp_request) {
	+ cqp_request = kzalloc(sizeof(*cqp_request), GFP_ATOMIC);
	+ if (cqp_request) {
	+ cqp_request->dynamic = TRUE;
	+ INIT_LIST_HEAD(&cqp_request->list);
	+ init_waitqueue_head(&cqp_request->waitq);
	+ }
	+ }
	+ if (!cqp_request) {
	+ DPRINTF("CQP Request Fail: No Memory");
	+ return NULL;
	+ }
	+ if (wait) {
	+ atomic_set(&cqp_request->refcount, 2);
	+ cqp_request->waiting = TRUE;
	+ } else {
	+ atomic_set(&cqp_request->refcount, 1);
	+ }
	+ return cqp_request;
	+}
	+
	+/**
	+ * i40iw_free_cqp_request - free cqp request
	+ * @cqp: cqp ptr
	+ * @cqp_request: to be put back in cqp list
	+ */
	+void
	+i40iw_free_cqp_request(struct i40iw_cqp *cqp,
	+ struct i40iw_cqp_request *cqp_request)
	+{
	+ unsigned long flags;
	+
	+ if (cqp_request->dynamic) {
	+ kfree(cqp_request);
	+ } else {
	+ cqp_request->request_done = FALSE;
	+ cqp_request->callback_fcn = NULL;
	+ cqp_request->waiting = FALSE;
	+
	+ spin_lock_irqsave(&cqp->req_lock, flags);
	+ list_add_tail(&cqp_request->list, &cqp->cqp_avail_reqs);
	+ spin_unlock_irqrestore(&cqp->req_lock, flags);
	+ }
	+}
	+
	+/**
	+ * i40iw_put_cqp_request - dec ref count and free if 0
	+ * @cqp: cqp ptr
	+ * @cqp_request: to be put back in cqp list
	+ */
	+void
	+i40iw_put_cqp_request(struct i40iw_cqp *cqp,
	+ struct i40iw_cqp_request *cqp_request)
	+{
	+ if (atomic_dec_and_test(&cqp_request->refcount))
	+ i40iw_free_cqp_request(cqp, cqp_request);
	+}
	+
	+/**
	+ * i40iw_free_pending_cqp_request -free pending cqp request objs
	+ * @cqp: cqp ptr
	+ * @cqp_request: to be put back in cqp list
	+ */
	+static void
	+i40iw_free_pending_cqp_request(struct i40iw_cqp *cqp,
	+ struct i40iw_cqp_request *cqp_request)
	+{
	+ struct i40iw_device *iwdev = container_of(cqp, struct i40iw_device, cqp);
	+
	+ if (cqp_request->waiting) {
	+ cqp_request->compl_info.error = TRUE;
	+ cqp_request->request_done = TRUE;
	+ wake_up(&cqp_request->waitq);
	+ }
	+ i40iw_put_cqp_request(cqp, cqp_request);
	+ wait_event_timeout(iwdev->close_wq,
	+ !atomic_read(&cqp_request->refcount),
	+ 1000);
	+}
	+
	+/**
	+ * i40iw_cleanup_pending_cqp_op - clean-up cqp with no completions
	+ * @iwdev: iwarp device
	+ */
	+void
	+i40iw_cleanup_pending_cqp_op(struct i40iw_device *iwdev)
	+{
	+ struct i40iw_sc_dev *dev = &iwdev->sc_dev;
	+ struct i40iw_cqp *cqp = &iwdev->cqp;
	+ struct i40iw_cqp_request *cqp_request = NULL;
	+ struct cqp_commands_info *pcmdinfo = NULL;
	+ u32 i, pending_work, wqe_idx;
	+
	+ pending_work = I40IW_RING_WORK_AVAILABLE(cqp->sc_cqp.sq_ring);
	+ wqe_idx = I40IW_RING_GETCURRENT_TAIL(cqp->sc_cqp.sq_ring);
	+ for (i = 0; i < pending_work; i++) {
	+ cqp_request = (struct i40iw_cqp_request *)(unsigned long)cqp->scratch_array[wqe_idx];
	+ if (cqp_request)
	+ i40iw_free_pending_cqp_request(cqp, cqp_request);
	+ wqe_idx = (wqe_idx + 1) % I40IW_RING_GETSIZE(cqp->sc_cqp.sq_ring);
	+ }
	+
	+ while (!list_empty(&dev->cqp_cmd_head)) {
	+ pcmdinfo = (struct cqp_commands_info *)i40iw_remove_head(&dev->cqp_cmd_head);
	+ cqp_request = container_of(pcmdinfo, struct i40iw_cqp_request, info);
	+ if (cqp_request)
	+ i40iw_free_pending_cqp_request(cqp, cqp_request);
	+ }
	+}
	+
	+/**
	+ * i40iw_free_qp - callback after destroy cqp completes
	+ * @cqp_request: cqp request for destroy qp
	+ * @num: not used
	+ */
	+static void
	+i40iw_free_qp(struct i40iw_cqp_request *cqp_request, u32 num)
	+{
	+ struct i40iw_sc_qp qp = (struct i40iw_sc_qp )cqp_request->param;
	+ struct i40iw_qp iwqp = (struct i40iw_qp )qp->back_qp;
	+ struct i40iw_device *iwdev;
	+ u32 qp_num = iwqp->ibqp.qp_num;
	+
	+ iwdev = iwqp->iwdev;
	+
	+ i40iw_rem_pdusecount(iwqp->iwpd, iwdev);
	+ i40iw_free_qp_resources(iwdev, iwqp, qp_num);
	+ i40iw_rem_devusecount(iwdev);
	+}
	+
	+/**
	+ * i40iw_wait_event - wait for completion
	+ * @iwdev: iwarp device
	+ * @cqp_request: cqp request to wait
	+ */
	+static int
	+i40iw_wait_event(struct i40iw_device *iwdev,
	+ struct i40iw_cqp_request *cqp_request)
	+{
	+ struct cqp_commands_info *info = &cqp_request->info;
	+ struct i40iw_cqp *iwcqp = &iwdev->cqp;
	+ struct i40iw_cqp_timeout cqp_timeout;
	+ int err_code = 0;
	+ int timeout_ret = 0;
	+ bool cqp_error = FALSE;
	+
	+ memset(&cqp_timeout, 0, sizeof(cqp_timeout));
	+ cqp_timeout.compl_cqp_cmds = iwdev->sc_dev.cqp_cmd_stats[OP_COMPLETED_COMMANDS];
	+ do {
	+ if (wait_event_timeout(cqp_request->waitq,
	+ cqp_request->request_done, CQP_COMPL_WAIT_TIME))
	+ break;
	+
	+ i40iw_check_cqp_progress(&cqp_timeout, &iwdev->sc_dev);
	+
	+ if (cqp_timeout.count < CQP_TIMEOUT_THRESHOLD)
	+ continue;
	+ device_printf(iwdev->ldev->dev,
	+ "error cqp command 0x%x timed out ret = %d\n",
	+ info->cqp_cmd, timeout_ret);
	+ err_code = -ETIMEDOUT;
	+ if (!iwdev->reset) {
	+ iwdev->reset = TRUE;
	+ i40iw_request_reset(iwdev);
	+ }
	+ goto done;
	+ } while (1);
	+
	+ cqp_error = cqp_request->compl_info.error;
	+ if (cqp_error) {
	+ device_printf(iwdev->ldev->dev,
	+ "error cqp command 0x%x completion maj = 0x%x min=0x%x\n",
	+ info->cqp_cmd, cqp_request->compl_info.maj_err_code,
	+ cqp_request->compl_info.min_err_code);
	+ err_code = -EPROTO;
	+ goto done;
	+ }
	+done:
	+ i40iw_put_cqp_request(iwcqp, cqp_request);
	+ return err_code;
	+}
	+
	+/**
	+ * i40iw_handle_cqp_op - process cqp command
	+ * @iwdev: iwarp device
	+ * @cqp_request: cqp request to process
	+ */
	+enum i40iw_status_code
	+i40iw_handle_cqp_op(struct i40iw_device *iwdev,
	+ struct i40iw_cqp_request *cqp_request)
	+{
	+ struct i40iw_sc_dev *dev = &iwdev->sc_dev;
	+ struct cqp_commands_info *info = &cqp_request->info;
	+ int err_code = 0;
	+ enum i40iw_status_code status;
	+
	+ if (iwdev->reset) {
	+ i40iw_free_cqp_request(&iwdev->cqp, cqp_request);
	+ return I40IW_ERR_CQP_COMPL_ERROR;
	+ }
	+ status = i40iw_process_cqp_cmd(dev, info);
	+ if (status) {
	+ device_printf(iwdev->ldev->dev,
	+ "error cqp command 0x%x failed\n", info->cqp_cmd);
	+ i40iw_free_cqp_request(&iwdev->cqp, cqp_request);
	+ return status;
	+ }
	+ if (cqp_request->waiting)
	+ err_code = i40iw_wait_event(iwdev, cqp_request);
	+ if (err_code)
	+ status = I40IW_ERR_CQP_COMPL_ERROR;
	+ return status;
	+}
	+
	+/**
	+ * i40iw_add_devusecount - add dev refcount
	+ * @iwdev: dev for refcount
	+ */
	+void
	+i40iw_add_devusecount(struct i40iw_device *iwdev)
	+{
	+ atomic_inc(&iwdev->use_count);
	+}
	+
	+/**
	+ * i40iw_rem_devusecount - decrement refcount for dev
	+ * @iwdev: device
	+ */
	+void
	+i40iw_rem_devusecount(struct i40iw_device *iwdev)
	+{
	+ if (!atomic_dec_and_test(&iwdev->use_count))
	+ return;
	+ wake_up(&iwdev->close_wq);
	+}
	+
	+/**
	+ * i40iw_add_pdusecount - add pd refcount
	+ * @iwpd: pd for refcount
	+ */
	+void
	+i40iw_add_pdusecount(struct i40iw_pd *iwpd)
	+{
	+ atomic_inc(&iwpd->usecount);
	+}
	+
	+/**
	+ * i40iw_rem_pdusecount - decrement refcount for pd and free if 0
	+ * @iwpd: pd for refcount
	+ * @iwdev: iwarp device
	+ */
	+void
	+i40iw_rem_pdusecount(struct i40iw_pd iwpd, struct i40iw_device iwdev)
	+{
	+ if (!atomic_dec_and_test(&iwpd->usecount))
	+ return;
	+ i40iw_free_resource(iwdev, iwdev->allocated_pds, iwpd->sc_pd.pd_id);
	+ kfree(iwpd);
	+}
	+
	+/**
	+ * i40iw_add_ref - add refcount for qp
	+ * @ibqp: iqarp qp
	+ */
	+void
	+i40iw_add_ref(struct ib_qp *ibqp)
	+{
	+ struct i40iw_qp iwqp = (struct i40iw_qp )ibqp;
	+
	+ atomic_inc(&iwqp->refcount);
	+}
	+
	+/**
	+ * i40iw_rem_ref - rem refcount for qp and free if 0
	+ * @ibqp: iqarp qp
	+ */
	+void
	+i40iw_rem_ref(struct ib_qp *ibqp)
	+{
	+ struct i40iw_qp *iwqp;
	+ struct i40iw_cqp_request *cqp_request;
	+ struct cqp_commands_info *cqp_info;
	+ struct i40iw_device *iwdev;
	+ unsigned long flags;
	+ u32 qp_num;
	+ enum i40iw_status_code status;
	+
	+ iwqp = to_iwqp(ibqp);
	+ iwdev = iwqp->iwdev;
	+ spin_lock_irqsave(&iwdev->qptable_lock, flags);
	+ if (!atomic_dec_and_test(&iwqp->refcount)) {
	+ spin_unlock_irqrestore(&iwdev->qptable_lock, flags);
	+ return;
	+ }
	+ qp_num = iwqp->ibqp.qp_num;
	+ iwdev->qp_table[qp_num] = NULL;
	+ spin_unlock_irqrestore(&iwdev->qptable_lock, flags);
	+ cqp_request = i40iw_get_cqp_request(&iwdev->cqp, FALSE);
	+ if (!cqp_request)
	+ return;
	+
	+ cqp_request->callback_fcn = i40iw_free_qp;
	+ cqp_request->param = (void *)&iwqp->sc_qp;
	+ cqp_info = &cqp_request->info;
	+ cqp_info->cqp_cmd = OP_QP_DESTROY;
	+ cqp_info->post_sq = 1;
	+ cqp_info->in.u.qp_destroy.qp = &iwqp->sc_qp;
	+ cqp_info->in.u.qp_destroy.scratch = (uintptr_t)cqp_request;
	+ cqp_info->in.u.qp_destroy.remove_hash_idx = TRUE;
	+ status = i40iw_handle_cqp_op(iwdev, cqp_request);
	+ if (status)
	+ device_printf(iwdev->ldev->dev, "CQP-OP Destroy QP fail");
	+}
	+
	+/**
	+ * i40iw_get_qp - get qp address
	+ * @device: iwarp device
	+ * @qpn: qp number
	+ */
	+struct ib_qp *
	+i40iw_get_qp(struct ib_device *device, int qpn)
	+{
	+ struct i40iw_device *iwdev = to_iwdev(device);
	+
	+ if ((qpn < IW_FIRST_QPN) \|\| (qpn >= iwdev->max_qp))
	+ return NULL;
	+
	+ return &iwdev->qp_table[qpn]->ibqp;
	+}
	+
	+/**
	+ * i40iw_debug_buf - print debug msg and buffer is mask set
	+ * @dev: hardware control device structure
	+ * @mask: mask to compare if to print debug buffer
	+ * @buf: points buffer addr
	+ * @size: saize of buffer to print
	+ */
	+void
	+i40iw_debug_buf(struct i40iw_sc_dev *dev, enum i40iw_debug_flag mask,
	+ char desc, u64 buf, u32 size)
	+{
	+ u32 i;
	+
	+ if (!(dev->debug_mask & mask))
	+ return;
	+ i40iw_debug(dev, mask, "%s\n", desc);
	+ i40iw_debug(dev, mask, "starting address virt=%p phy=%llxh\n", buf,
	+ (unsigned long long)i40iw_get_virt_to_phy(buf));
	+
	+ for (i = 0; i < size; i += 8)
	+ i40iw_debug(dev, mask, "index %03d val: %016lx\n", i, buf[i / 8]);
	+}
	+
	+/**
	+ * i40iw_get_hw_addr - return hw addr
	+ * @par: points to shared dev
	+ */
	+u8 *
	+i40iw_get_hw_addr(void *par)
	+{
	+ struct i40iw_sc_dev dev = (struct i40iw_sc_dev )par;
	+
	+ return dev->hw->hw_addr;
	+}
	+
	+/**
	+ * i40iw_remove_head - return head entry and remove from list
	+ * @list: list for entry
	+ */
	+void *
	+i40iw_remove_head(struct list_head *list)
	+{
	+ struct list_head *entry;
	+
	+ if (list_empty(list))
	+ return NULL;
	+
	+ entry = (void *)list->next;
	+ list_del(entry);
	+ return (void *)entry;
	+}
	+
	+inline u32
	+i40iw_rd32(struct i40iw_dev_ctx *dev_ctx, u32 reg)
	+{
	+
	+ KASSERT(reg < dev_ctx->mem_bus_space_size,
	+ ("iw_ixl: register offset %#jx too large (max is %#jx)",
	+ (uintmax_t)reg, (uintmax_t)dev_ctx->mem_bus_space_size));
	+
	+ return (bus_space_read_4(dev_ctx->mem_bus_space_tag,
	+ dev_ctx->mem_bus_space_handle, reg));
	+}
	+
	+inline void
	+i40iw_wr32(struct i40iw_dev_ctx *dev_ctx, u32 reg, u32 value)
	+{
	+
	+ KASSERT(reg < dev_ctx->mem_bus_space_size,
	+ ("iw_ixl: register offset %#jx too large (max is %#jx)",
	+ (uintmax_t)reg, (uintmax_t)dev_ctx->mem_bus_space_size));
	+
	+ bus_space_write_4(dev_ctx->mem_bus_space_tag,
	+ dev_ctx->mem_bus_space_handle, reg, value);
	+}
	+
	+inline u64
	+i40iw_rd64(struct i40iw_dev_ctx *dev_ctx, u32 reg)
	+{
	+
	+ KASSERT(reg < dev_ctx->mem_bus_space_size,
	+ ("iw_ixl: register offset %#jx too large (max is %#jx)",
	+ (uintmax_t)reg, (uintmax_t)dev_ctx->mem_bus_space_size));
	+
	+ return (bus_space_read_8(dev_ctx->mem_bus_space_tag,
	+ dev_ctx->mem_bus_space_handle, reg));
	+}
	+
	+inline void
	+i40iw_wr64(struct i40iw_dev_ctx *dev_ctx, u32 reg, u64 value)
	+{
	+
	+ KASSERT(reg < dev_ctx->mem_bus_space_size,
	+ ("iw_ixl: register offset %#jx too large (max is %#jx)",
	+ (uintmax_t)reg, (uintmax_t)dev_ctx->mem_bus_space_size));
	+
	+ bus_space_write_8(dev_ctx->mem_bus_space_tag,
	+ dev_ctx->mem_bus_space_handle, reg, value);
	+}
	+
	+static void
	+i40iw_dmamap_cb(void arg, bus_dma_segment_t segs, int nseg, int error)
	+{
	+ if (error)
	+ return;
	+ (bus_addr_t ) arg = segs->ds_addr;
	+ return;
	+}
	+
	+/**
	+ * i40iw_allocate_dma_mem - Memory alloc helper fn
	+ * @hw: pointer to the HW structure
	+ * @mem: ptr to mem struct to fill out
	+ * @size: size of memory requested
	+ * @alignment: what to align the allocation to
	+ */
	+enum i40iw_status_code
	+i40iw_allocate_dma_mem(struct i40iw_hw hw, struct i40iw_dma_mem mem,
	+ u64 size, u32 alignment)
	+{
	+ struct i40iw_dev_ctx *dev_ctx =
	+ (struct i40iw_dev_ctx *)hw->dev_context;
	+ device_t dev = dev_ctx->dev;
	+ int ret;
	+
	+ ret = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */
	+ alignment, 0, /* alignment, bounds */
	+ BUS_SPACE_MAXADDR, /* lowaddr */
	+ BUS_SPACE_MAXADDR, /* highaddr */
	+ NULL, NULL, /* filter, filterarg */
	+ size, /* maxsize */
	+ 1, /* nsegments */
	+ size, /* maxsegsize */
	+ BUS_DMA_ALLOCNOW, /* flags */
	+ NULL, /* lockfunc */
	+ NULL, /* lockfuncarg */
	+ &mem->tag);
	+ if (ret != 0) {
	+ device_printf(dev, "%s: bus_dma_tag_create failed, error %u\n",
	+ __func__, ret);
	+ goto fail_0;
	+ }
	+ ret = bus_dmamem_alloc(mem->tag, (void **)&mem->va,
	+ BUS_DMA_NOWAIT \| BUS_DMA_ZERO, &mem->map);
	+ if (ret != 0) {
	+ device_printf(dev, "%s: bus_dmamem_alloc failed, error %u\n",
	+ __func__, ret);
	+ goto fail_1;
	+ }
	+ ret = bus_dmamap_load(mem->tag, mem->map, mem->va, size,
	+ i40iw_dmamap_cb, &mem->pa, BUS_DMA_NOWAIT);
	+ if (ret != 0) {
	+ device_printf(dev, "%s: bus_dmamap_load failed, error %u\n",
	+ __func__, ret);
	+ goto fail_2;
	+ }
	+ mem->nseg = 1;
	+ mem->size = size;
	+ bus_dmamap_sync(mem->tag, mem->map,
	+ BUS_DMASYNC_PREREAD \| BUS_DMASYNC_PREWRITE);
	+ return (0);
	+fail_2:
	+ bus_dmamem_free(mem->tag, mem->va, mem->map);
	+fail_1:
	+ bus_dma_tag_destroy(mem->tag);
	+fail_0:
	+ mem->map = NULL;
	+ mem->tag = NULL;
	+
	+ return (I40IW_ERR_NO_MEMORY);
	+}
	+
	+/**
	+ * ixliw_free_dma_mem - Memory free helper fn
	+ * @mem: ptr to mem struct to free
	+ */
	+void
	+ixliw_free_dma_mem(struct i40iw_dma_mem *mem)
	+{
	+ if (!mem)
	+ return;
	+ bus_dmamap_sync(mem->tag, mem->map,
	+ BUS_DMASYNC_POSTREAD \| BUS_DMASYNC_POSTWRITE);
	+ bus_dmamap_unload(mem->tag, mem->map);
	+ if (!mem->va)
	+ return;
	+ bus_dmamem_free(mem->tag, mem->va, mem->map);
	+ bus_dma_tag_destroy(mem->tag);
	+}
	+
	+/**
	+ * i40iw_allocate_virt_mem - virtual memory alloc helper fn
	+ * @hw: pointer to the HW structure
	+ * @mem: ptr to mem struct to fill out
	+ * @size: size of memory requested
	+ */
	+enum i40iw_status_code
	+i40iw_allocate_virt_mem(struct i40iw_hw hw, struct i40iw_virt_mem mem,
	+ u32 size)
	+{
	+ if (!mem)
	+ return (I40IW_ERR_PARAM);
	+
	+ mem->size = size;
	+ mem->va = malloc(size, M_DEVBUF, M_NOWAIT \| M_ZERO);
	+
	+ if (mem->va)
	+ return (0);
	+ else
	+ return (I40IW_ERR_NO_MEMORY);
	+}
	+
	+/**
	+ * i40iw_free_virt_mem - virtual memory free helper fn
	+ * @hw: pointer to the HW structure
	+ * @mem: ptr to mem struct to free
	+ */
	+enum i40iw_status_code
	+i40iw_free_virt_mem(struct i40iw_hw hw, struct i40iw_virt_mem mem)
	+{
	+ if (!mem)
	+ return (I40IW_ERR_PARAM);
	+ free(mem->va, M_DEVBUF);
	+ return (0);
	+}
	+
	+/**
	+ * i40iw_cqp_sds_cmd - create cqp command for sd
	+ * @dev: hardware control device structure
	+ * @sd_info: information for sd cqp
	+ *
	+ */
	+enum i40iw_status_code
	+i40iw_cqp_sds_cmd(struct i40iw_sc_dev *dev,
	+ struct i40iw_update_sds_info *sdinfo)
	+{
	+ enum i40iw_status_code status;
	+ struct i40iw_cqp_request *cqp_request;
	+ struct cqp_commands_info *cqp_info;
	+ struct i40iw_device iwdev = (struct i40iw_device )dev->back_dev;
	+
	+ cqp_request = i40iw_get_cqp_request(&iwdev->cqp, TRUE);
	+ if (!cqp_request)
	+ return I40IW_ERR_NO_MEMORY;
	+ cqp_info = &cqp_request->info;
	+ memcpy(&cqp_info->in.u.update_pe_sds.info, sdinfo,
	+ sizeof(cqp_info->in.u.update_pe_sds.info));
	+ cqp_info->cqp_cmd = OP_UPDATE_PE_SDS;
	+ cqp_info->post_sq = 1;
	+ cqp_info->in.u.update_pe_sds.dev = dev;
	+ cqp_info->in.u.update_pe_sds.scratch = (uintptr_t)cqp_request;
	+ status = i40iw_handle_cqp_op(iwdev, cqp_request);
	+ if (status)
	+ DPRINTF("CQP-OP Update SD's fail");
	+ return status;
	+}
	+
	+/**
	+ * i40iw_cqp_cq_create_cmd - create a cq for the cqp
	+ * @dev: device pointer
	+ * @cq: pointer to created cq
	+ */
	+enum i40iw_status_code
	+i40iw_cqp_cq_create_cmd(struct i40iw_sc_dev *dev,
	+ struct i40iw_sc_cq *cq)
	+{
	+ struct i40iw_device iwdev = (struct i40iw_device )dev->back_dev;
	+ struct i40iw_cqp *iwcqp = &iwdev->cqp;
	+ struct i40iw_cqp_request *cqp_request;
	+ struct cqp_commands_info *cqp_info;
	+ enum i40iw_status_code status;
	+
	+ cqp_request = i40iw_get_cqp_request(iwcqp, TRUE);
	+ if (!cqp_request)
	+ return I40IW_ERR_NO_MEMORY;
	+
	+ cqp_info = &cqp_request->info;
	+ cqp_info->cqp_cmd = OP_CQ_CREATE;
	+ cqp_info->post_sq = 1;
	+ cqp_info->in.u.cq_create.cq = cq;
	+ cqp_info->in.u.cq_create.scratch = (uintptr_t)cqp_request;
	+ status = i40iw_handle_cqp_op(iwdev, cqp_request);
	+ if (status)
	+ i40iw_pr_err("CQP-OP Create QP fail");
	+
	+ return status;
	+}
	+
	+/**
	+ * i40iw_cqp_qp_create_cmd - create a qp for the cqp
	+ * @dev: device pointer
	+ * @qp: pointer to created qp
	+ */
	+enum i40iw_status_code
	+i40iw_cqp_qp_create_cmd(struct i40iw_sc_dev *dev,
	+ struct i40iw_sc_qp *qp)
	+{
	+ struct i40iw_device iwdev = (struct i40iw_device )dev->back_dev;
	+ struct i40iw_cqp *iwcqp = &iwdev->cqp;
	+ struct i40iw_cqp_request *cqp_request;
	+ struct cqp_commands_info *cqp_info;
	+ struct i40iw_create_qp_info *qp_info;
	+ enum i40iw_status_code status;
	+
	+ cqp_request = i40iw_get_cqp_request(iwcqp, TRUE);
	+ if (!cqp_request)
	+ return I40IW_ERR_NO_MEMORY;
	+
	+ cqp_info = &cqp_request->info;
	+ qp_info = &cqp_request->info.in.u.qp_create.info;
	+
	+ memset(qp_info, 0, sizeof(*qp_info));
	+
	+ qp_info->cq_num_valid = TRUE;
	+ qp_info->next_iwarp_state = I40IW_QP_STATE_RTS;
	+
	+ cqp_info->cqp_cmd = OP_QP_CREATE;
	+ cqp_info->post_sq = 1;
	+ cqp_info->in.u.qp_create.qp = qp;
	+ cqp_info->in.u.qp_create.scratch = (uintptr_t)cqp_request;
	+ status = i40iw_handle_cqp_op(iwdev, cqp_request);
	+ if (status)
	+ i40iw_pr_err("CQP-OP QP create fail");
	+ return status;
	+}
	+
	+/**
	+ * i40iw_cqp_cq_destroy_cmd - destroy the cqp cq
	+ * @dev: device pointer
	+ * @cq: pointer to cq
	+ */
	+void
	+i40iw_cqp_cq_destroy_cmd(struct i40iw_sc_dev dev, struct i40iw_sc_cq cq)
	+{
	+
	+ struct i40iw_device iwdev = (struct i40iw_device )dev->back_dev;
	+
	+ i40iw_cq_wq_destroy(iwdev, cq);
	+}
	+
	+/**
	+ * i40iw_cqp_qp_destroy_cmd - destroy the cqp
	+ * @dev: device pointer
	+ * @qp: pointer to qp
	+ */
	+void
	+i40iw_cqp_qp_destroy_cmd(struct i40iw_sc_dev dev, struct i40iw_sc_qp qp)
	+{
	+ struct i40iw_device iwdev = (struct i40iw_device )dev->back_dev;
	+ struct i40iw_cqp *iwcqp = &iwdev->cqp;
	+ struct i40iw_cqp_request *cqp_request;
	+ struct cqp_commands_info *cqp_info;
	+ enum i40iw_status_code status;
	+
	+ cqp_request = i40iw_get_cqp_request(iwcqp, TRUE);
	+ if (!cqp_request)
	+ return;
	+
	+ cqp_info = &cqp_request->info;
	+ memset(cqp_info, 0, sizeof(*cqp_info));
	+
	+ cqp_info->cqp_cmd = OP_QP_DESTROY;
	+ cqp_info->post_sq = 1;
	+ cqp_info->in.u.qp_destroy.qp = qp;
	+ cqp_info->in.u.qp_destroy.scratch = (uintptr_t)cqp_request;
	+ cqp_info->in.u.qp_destroy.remove_hash_idx = TRUE;
	+ status = i40iw_handle_cqp_op(iwdev, cqp_request);
	+ if (status)
	+ i40iw_pr_err("CQP QP_DESTROY fail");
	+}
	+
	+/**
	+ * i40iw_term_modify_qp - modify qp for term message
	+ * @qp: hardware control qp
	+ * @next_state: qp's next state
	+ * @term: terminate code
	+ * @term_len: length
	+ */
	+void
	+i40iw_term_modify_qp(struct i40iw_sc_qp *qp, u8 next_state, u8 term, u8 term_len)
	+{
	+ struct i40iw_qp *iwqp;
	+
	+ iwqp = (struct i40iw_qp *)qp->back_qp;
	+ i40iw_next_iw_state(iwqp, next_state, 0, term, term_len);
	+};
	+
	+/**
	+ * i40iw_terminate_done - after terminate is completed
	+ * @qp: hardware control qp
	+ * @timeout_occurred: indicates if terminate timer expired
	+ */
	+void
	+i40iw_terminate_done(struct i40iw_sc_qp *qp, int timeout_occurred)
	+{
	+ struct i40iw_qp *iwqp;
	+ unsigned long flags;
	+ u32 next_iwarp_state = I40IW_QP_STATE_ERROR;
	+ u8 hte = 0;
	+ bool first_time;
	+
	+ iwqp = (struct i40iw_qp *)qp->back_qp;
	+ spin_lock_irqsave(&iwqp->lock, flags);
	+ if (iwqp->hte_added) {
	+ iwqp->hte_added = 0;
	+ hte = 1;
	+ }
	+ first_time = !(qp->term_flags & I40IW_TERM_DONE);
	+ qp->term_flags \|= I40IW_TERM_DONE;
	+ spin_unlock_irqrestore(&iwqp->lock, flags);
	+ if (first_time) {
	+ if (!timeout_occurred)
	+ i40iw_terminate_del_timer(qp);
	+ else
	+ next_iwarp_state = I40IW_QP_STATE_CLOSING;
	+
	+ i40iw_next_iw_state(iwqp, next_iwarp_state, hte, 0, 0);
	+ i40iw_cm_disconn(iwqp);
	+ }
	+}
	+
	+/**
	+ * i40iw_terminate_imeout - timeout happened
	+ * @context: points to iwarp qp
	+ */
	+static void
	+i40iw_terminate_timeout(unsigned long context)
	+{
	+ struct i40iw_qp iwqp = (struct i40iw_qp )context;
	+ struct i40iw_sc_qp qp = (struct i40iw_sc_qp )&iwqp->sc_qp;
	+
	+ i40iw_terminate_done(qp, 1);
	+ i40iw_rem_ref(&iwqp->ibqp);
	+}
	+
	+/**
	+ * i40iw_terminate_start_timer - start terminate timeout
	+ * @qp: hardware control qp
	+ */
	+void
	+i40iw_terminate_start_timer(struct i40iw_sc_qp *qp)
	+{
	+ struct i40iw_qp *iwqp;
	+
	+ iwqp = (struct i40iw_qp *)qp->back_qp;
	+ i40iw_add_ref(&iwqp->ibqp);
	+ init_timer(&iwqp->terminate_timer);
	+ iwqp->terminate_timer.function = i40iw_terminate_timeout;
	+ iwqp->terminate_timer.expires = jiffies + HZ;
	+ iwqp->terminate_timer.data = (unsigned long)iwqp;
	+ add_timer(&iwqp->terminate_timer);
	+}
	+
	+/**
	+ * i40iw_terminate_del_timer - delete terminate timeout
	+ * @qp: hardware control qp
	+ */
	+void
	+i40iw_terminate_del_timer(struct i40iw_sc_qp *qp)
	+{
	+ struct i40iw_qp *iwqp;
	+#if __FreeBSD_version < 1200054
	+ int ret;
	+#endif
	+
	+ iwqp = (struct i40iw_qp *)qp->back_qp;
	+#if __FreeBSD_version >= 1200054
	+ del_timer(&iwqp->terminate_timer);
	+ if (timer_pending(&iwqp->terminate_timer))
	+ i40iw_rem_ref(&iwqp->ibqp);
	+#else
	+ ret = del_timer(&iwqp->terminate_timer);
	+ if (ret)
	+ i40iw_rem_ref(&iwqp->ibqp);
	+#endif
	+}
	+
	+/**
	+ * i40iw_ieq_mpa_crc_ae - generate AE for crc error
	+ * @dev: hardware control device structure
	+ * @qp: hardware control qp
	+ */
	+void
	+i40iw_ieq_mpa_crc_ae(struct i40iw_sc_dev dev, struct i40iw_sc_qp qp)
	+{
	+ struct i40iw_gen_ae_info info;
	+ struct i40iw_device iwdev = (struct i40iw_device )dev->back_dev;
	+
	+ i40iw_debug(dev, I40IW_DEBUG_AEQ, "%s entered\n", __func__);
	+ info.ae_code = I40IW_AE_LLP_RECEIVED_MPA_CRC_ERROR;
	+ info.ae_source = I40IW_AE_SOURCE_RQ;
	+ i40iw_gen_ae(iwdev, qp, &info, FALSE);
	+}
	+
	+enum i40iw_status_code
	+i40iw_init_hash_desc(void **desc)
	+{
	+ return 0;
	+}
	+
	+void
	+i40iw_free_hash_desc(void *desc)
	+{
	+ return;
	+}
	+
	+/**
	+ * i40iw_ieq_check_mpacrc - check if mpa crc is OK
	+ * @desc: desc for hash
	+ * @addr: address of buffer for crc
	+ * @length: length of buffer
	+ * @value: value to be compared
	+ */
	+enum i40iw_status_code
	+i40iw_ieq_check_mpacrc(void desc, void addr, u32 length,
	+ u32 value)
	+{
	+ u32 crc = calculate_crc32c(0xffffffff, addr, length) ^ 0xffffffff;
	+ enum i40iw_status_code ret_code = 0;
	+
	+ printf("%s: begin\n", __func__);
	+ if (crc != value) {
	+ i40iw_pr_err("mpa crc check fail\n");
	+ ret_code = I40IW_ERR_MPA_CRC;
	+ }
	+ printf("%s: result crc=%x value=%x\n", __func__, crc, value);
	+ return ret_code;
	+}
	+
	+/**
	+ * i40iw_ieq_get_qp - get qp based on quad in puda buffer
	+ * @dev: hardware control device structure
	+ * @buf: receive puda buffer on exception q
	+ */
	+struct i40iw_sc_qp *
	+i40iw_ieq_get_qp(struct i40iw_sc_dev dev, struct i40iw_puda_buf buf)
	+{
	+ struct i40iw_device iwdev = (struct i40iw_device )dev->back_dev;
	+ struct i40iw_qp *iwqp;
	+ struct i40iw_cm_node *cm_node;
	+ struct ip6_hdr *ip6h;
	+ struct ip iph = (struct ip )buf->iph;
	+ struct tcphdr tcph = (struct tcphdr )buf->tcph;
	+ u32 loc_addr[4], rem_addr[4];
	+ u16 loc_port, rem_port;
	+
	+ if (iph->ip_v == 4) {
	+ memset(loc_addr, 0, sizeof(loc_addr));
	+ loc_addr[0] = ntohl(iph->ip_dst.s_addr);
	+ memset(rem_addr, 0, sizeof(rem_addr));
	+ rem_addr[0] = ntohl(iph->ip_src.s_addr);
	+ } else {
	+ ip6h = (struct ip6_hdr *)buf->iph;
	+ i40iw_copy_ip_ntohl(loc_addr, (__be32 *)&ip6h->ip6_dst);
	+ i40iw_copy_ip_ntohl(rem_addr, (__be32 *)&ip6h->ip6_src);
	+ }
	+ loc_port = ntohs(tcph->th_sport);
	+ rem_port = ntohs(tcph->th_dport);
	+
	+ cm_node = i40iw_find_node(&iwdev->cm_core, rem_port, rem_addr, loc_port,
	+ loc_addr, FALSE);
	+ if (!cm_node)
	+ return NULL;
	+ iwqp = cm_node->iwqp;
	+ return &iwqp->sc_qp;
	+}
	+
	+/**
	+ * i40iw_ieq_update_tcpip_info - update tcpip in the buffer
	+ * @buf: puda to update
	+ * @length: length of buffer
	+ * @seqnum: seq number for tcp
	+ */
	+void
	+i40iw_ieq_update_tcpip_info(struct i40iw_puda_buf *buf, u16 length, u32 seqnum)
	+{
	+ struct tcphdr *tcph;
	+ struct ip *iph;
	+ u16 iphlen;
	+ u16 packetsize;
	+ u8 addr = (u8 )buf->mem.va;
	+
	+ iphlen = (buf->ipv4) ? 20 : 40;
	+ iph = (struct ip *)(addr + buf->maclen);
	+ tcph = (struct tcphdr *)(addr + buf->maclen + iphlen);
	+ packetsize = length + buf->tcphlen + iphlen;
	+
	+ iph->ip_len = htons(packetsize);
	+ tcph->th_seq = htonl(seqnum);
	+}
	+
	+/**
	+ * i40iw_puda_get_tcpip_info - get tcpip info from puda buffer
	+ * @info: to get information
	+ * @buf: puda buffer
	+ */
	+enum i40iw_status_code
	+i40iw_puda_get_tcpip_info(struct i40iw_puda_completion_info *info,
	+ struct i40iw_puda_buf *buf)
	+{
	+ struct ip *iph;
	+ struct ip6_hdr *ip6h;
	+ struct tcphdr *tcph;
	+ struct ether_header ethh = (struct ether_header )buf->mem.va;
	+ u16 iphlen;
	+ u16 pkt_len;
	+ u8 mem = (u8 )buf->mem.va;
	+
	+ if (ethh->ether_type == htons(ETH_P_8021Q)) {
	+ info->vlan_valid = TRUE;
	+ buf->vlan_id = ntohs(((struct ether_vlan_header *)ethh)->evl_tag) & EVL_VLID_MASK;
	+ }
	+ buf->maclen = (info->vlan_valid) ? 18 : 14;
	+ iphlen = (info->l3proto) ? 40 : 20;
	+ buf->ipv4 = (info->l3proto) ? FALSE : TRUE;
	+ buf->iph = mem + buf->maclen;
	+ iph = (struct ip *)buf->iph;
	+
	+ buf->tcph = buf->iph + iphlen;
	+ tcph = (struct tcphdr *)buf->tcph;
	+
	+ if (buf->ipv4) {
	+ pkt_len = ntohs(iph->ip_len);
	+ } else {
	+ ip6h = (struct ip6_hdr *)buf->iph;
	+ pkt_len = ntohs(ip6h->ip6_plen) + iphlen;
	+ }
	+
	+ buf->totallen = pkt_len + buf->maclen;
	+
	+ if (info->payload_len < buf->totallen) {
	+ i40iw_pr_err("payload_len = 0x%x totallen expected0x%x\n",
	+ info->payload_len, buf->totallen);
	+ return I40IW_ERR_INVALID_SIZE;
	+ }
	+ buf->tcphlen = (tcph->th_off) << 2;
	+ buf->datalen = pkt_len - iphlen - buf->tcphlen;
	+ buf->data = (buf->datalen) ? buf->tcph + buf->tcphlen : NULL;
	+ buf->hdrlen = buf->maclen + iphlen + buf->tcphlen;
	+ buf->seqnum = ntohl(tcph->th_seq);
	+ return 0;
	+}
	Index: sys/dev/ixl/iwarp/iw_ixl_verbs.h
	===================================================================
	--- /dev/null
	+++ sys/dev/ixl/iwarp/iw_ixl_verbs.h
	@@ -0,0 +1,186 @@
	+/*******************************************************************************
	+*
	+* Copyright (c) 2015-2017 Intel Corporation. All rights reserved.
	+*
	+* This software is available to you under a choice of one of two
	+* licenses. You may choose to be licensed under the terms of the GNU
	+* General Public License (GPL) Version 2, available from the file
	+* COPYING in the main directory of this source tree, or the
	+* OpenFabrics.org BSD license below:
	+*
	+* Redistribution and use in source and binary forms, with or
	+* without modification, are permitted provided that the following
	+* conditions are met:
	+*
	+* - Redistributions of source code must retain the above
	+* copyright notice, this list of conditions and the following
	+* disclaimer.
	+*
	+* - Redistributions in binary form must reproduce the above
	+* copyright notice, this list of conditions and the following
	+* disclaimer in the documentation and/or other materials
	+* provided with the distribution.
	+*
	+* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
	+* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
	+* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
	+* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
	+* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
	+* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
	+* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
	+* SOFTWARE.
	+*
	+*******************************************************************************/
	+/$FreeBSD$/
	+
	+#ifndef _IW_IXL_VERBS_H_
	+#define _IW_IXL_VERBS_H_
	+
	+struct i40iw_ucontext {
	+ struct ib_ucontext ibucontext;
	+ struct i40iw_device *iwdev;
	+ struct list_head cq_reg_mem_list;
	+ spinlock_t cq_reg_mem_list_lock; /* memory list for cq's */
	+ struct list_head qp_reg_mem_list;
	+ spinlock_t qp_reg_mem_list_lock; /* memory list for qp's */
	+};
	+
	+struct i40iw_pd {
	+ struct ib_pd ibpd;
	+ struct i40iw_sc_pd sc_pd;
	+ atomic_t usecount;
	+};
	+
	+struct i40iw_hmc_pble {
	+ union {
	+ u32 idx;
	+ dma_addr_t addr;
	+ };
	+};
	+
	+struct i40iw_cq_mr {
	+ struct i40iw_hmc_pble cq_pbl;
	+ dma_addr_t shadow;
	+};
	+
	+struct i40iw_qp_mr {
	+ struct i40iw_hmc_pble sq_pbl;
	+ struct i40iw_hmc_pble rq_pbl;
	+ dma_addr_t shadow;
	+ struct page *sq_page;
	+};
	+
	+struct i40iw_pbl {
	+ struct list_head list;
	+ union {
	+ struct i40iw_qp_mr qp_mr;
	+ struct i40iw_cq_mr cq_mr;
	+ };
	+
	+ bool pbl_allocated;
	+ u64 user_base;
	+ struct i40iw_pble_alloc pble_alloc;
	+ struct i40iw_mr *iwmr;
	+};
	+
	+#define MAX_SAVE_PAGE_ADDRS 4
	+struct i40iw_mr {
	+ union {
	+ struct ib_mr ibmr;
	+ struct ib_mw ibmw;
	+ struct ib_fmr ibfmr;
	+ };
	+ struct ib_umem *region;
	+ u16 type;
	+ u32 page_cnt;
	+#ifdef IW_IXL_RDMA_4_9
	+ u32 npages;
	+#endif
	+ u32 stag;
	+ u64 length;
	+ u64 pgaddrmem[MAX_SAVE_PAGE_ADDRS];
	+ struct i40iw_pbl iwpbl;
	+};
	+
	+#ifndef IW_IXL_RDMA_4_9
	+struct i40iw_fast_reg_page_list {
	+ struct ib_fast_reg_page_list ibfpl;
	+ struct i40iw_pble_alloc palloc;
	+ struct i40iw_device *iwdev;
	+};
	+
	+#endif
	+struct i40iw_cq {
	+ struct ib_cq ibcq;
	+ struct i40iw_sc_cq sc_cq;
	+ u16 cq_head;
	+ u16 cq_size;
	+ u16 cq_number;
	+ bool user_mode;
	+ u32 polled_completions;
	+ u32 cq_mem_size;
	+ struct i40iw_dma_mem kmem;
	+ spinlock_t lock; /* for poll cq */
	+ struct i40iw_pbl *iwpbl;
	+};
	+
	+struct i40iw_disconn_work {
	+ struct work_struct work;
	+ struct i40iw_qp *iwqp;
	+};
	+
	+struct i40iw_qp_kmode {
	+ struct i40iw_dma_mem dma_mem;
	+ u64 *wrid_mem;
	+};
	+
	+struct i40iw_qp {
	+ struct ib_qp ibqp;
	+ struct i40iw_sc_qp sc_qp;
	+ struct i40iw_device *iwdev;
	+ struct i40iw_cq *iwscq;
	+ struct i40iw_cq *iwrcq;
	+ struct i40iw_pd *iwpd;
	+ struct i40iw_qp_host_ctx_info ctx_info;
	+ struct i40iwarp_offload_info iwarp_info;
	+ void *allocated_buffer;
	+ atomic_t refcount;
	+ struct iw_cm_id *cm_id;
	+ void *cm_node;
	+ struct ib_mr *lsmm_mr;
	+ struct work_struct work;
	+ enum ib_qp_state ibqp_state;
	+ u32 iwarp_state;
	+ u32 qp_mem_size;
	+ u32 last_aeq;
	+ atomic_t close_timer_started;
	+ spinlock_t lock; /* for post work requests */
	+ struct i40iw_qp_context *iwqp_context;
	+ void *pbl_vbase;
	+ dma_addr_t pbl_pbase;
	+ struct page *page;
	+ u8 active_conn:1;
	+ u8 user_mode:1;
	+ u8 hte_added:1;
	+ u8 flush_issued:1;
	+ u8 destroyed:1;
	+ u8 sig_all:1;
	+ u8 pau_mode:1;
	+ u8 rsvd:1;
	+ u16 term_sq_flush_code;
	+ u16 term_rq_flush_code;
	+ u8 hw_iwarp_state;
	+ u8 hw_tcp_state;
	+ struct i40iw_qp_kmode kqp;
	+ struct i40iw_dma_mem host_ctx;
	+ struct timer_list terminate_timer;
	+ struct i40iw_pbl iwpbl;
	+ struct i40iw_dma_mem q2_ctx_mem;
	+ struct i40iw_dma_mem ietf_mem;
	+#ifdef IW_IXL_RDMA_4_9
	+ struct completion sq_drained;
	+ struct completion rq_drained;
	+#endif
	+};
	+
	+#endif /* _IW_IXL_VERBS_H_ */
	Index: sys/dev/ixl/iwarp/iw_ixl_verbs.c
	===================================================================
	--- /dev/null
	+++ sys/dev/ixl/iwarp/iw_ixl_verbs.c
	@@ -0,0 +1,2554 @@
	+/*******************************************************************************
	+*
	+* Copyright (c) 2015-2017 Intel Corporation. All rights reserved.
	+*
	+* This software is available to you under a choice of one of two
	+* licenses. You may choose to be licensed under the terms of the GNU
	+* General Public License (GPL) Version 2, available from the file
	+* COPYING in the main directory of this source tree, or the
	+* OpenFabrics.org BSD license below:
	+*
	+* Redistribution and use in source and binary forms, with or
	+* without modification, are permitted provided that the following
	+* conditions are met:
	+*
	+* - Redistributions of source code must retain the above
	+* copyright notice, this list of conditions and the following
	+* disclaimer.
	+*
	+* - Redistributions in binary form must reproduce the above
	+* copyright notice, this list of conditions and the following
	+* disclaimer in the documentation and/or other materials
	+* provided with the distribution.
	+*
	+* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
	+* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
	+* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
	+* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
	+* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
	+* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
	+* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
	+* SOFTWARE.
	+*
	+*******************************************************************************/
	+/$FreeBSD$/
	+
	+#include <rdma/ib_user_verbs.h>
	+#include <rdma/iw_cm.h>
	+#include <rdma/ib_umem.h>
	+
	+#include <net/if.h>
	+#include <net/if_var.h>
	+#include <net/if_dl.h>
	+
	+#include <dev/pci/pcivar.h>
	+
	+#include "iw_ixl.h"
	+#include "iw_ixl_linux_wait.h"
	+
	+#if __FreeBSD_version < 1100000
	+#include "iw_ixl_linux_jiffies.h"
	+#endif /* IW_IXL_FREEBSD10 */
	+
	+/**
	+ * i40iw_query_device - get device attributes
	+ * @ibdev: device pointer from stack
	+ * @props: returning device attributes
	+#ifdef IW_IXL_RDMA_4_9
	+ * @udata: user data
	+#endif
	+ */
	+static int
	+#ifdef IW_IXL_RDMA_4_9
	+i40iw_query_device(struct ib_device ibdev, struct ib_device_attr props, struct ib_udata *udata)
	+#else
	+i40iw_query_device(struct ib_device ibdev, struct ib_device_attr props)
	+#endif
	+{
	+ struct i40iw_device *iwdev = to_iwdev(ibdev);
	+
	+#ifdef IW_IXL_RDMA_4_9
	+ if (udata->inlen \|\| udata->outlen)
	+ return -EINVAL;
	+#endif
	+ memset(props, 0, sizeof(*props));
	+ ether_addr_copy((u8 *)&props->sys_image_guid, IF_LLADDR(iwdev->ifp));
	+ props->fw_ver = I40IW_FW_VERSION;
	+ props->device_cap_flags = iwdev->device_cap_flags;
	+ props->vendor_id = pci_get_vendor(iwdev->ldev->dev);
	+ props->vendor_part_id = pci_get_device(iwdev->ldev->dev);
	+ props->hw_ver = (u32)iwdev->sc_dev.hw_rev;
	+ props->max_mr_size = I40IW_MAX_OUTBOUND_MESSAGE_SIZE;
	+ props->max_qp = iwdev->max_qp - iwdev->used_qps;
	+ props->max_qp_wr = (I40IW_MAX_WQ_ENTRIES >> 2) - 1;
	+ props->max_sge = I40IW_MAX_WQ_FRAGMENT_COUNT;
	+ props->max_cq = iwdev->max_cq - iwdev->used_cqs;
	+ props->max_cqe = iwdev->max_cqe;
	+ props->max_mr = iwdev->max_mr - iwdev->used_mrs;
	+ props->max_pd = iwdev->max_pd - iwdev->used_pds;
	+ props->max_sge_rd = I40IW_MAX_SGE_RD;
	+ props->max_qp_rd_atom = I40IW_MAX_IRD_SIZE;
	+ props->max_qp_init_rd_atom = props->max_qp_rd_atom;
	+ props->atomic_cap = IB_ATOMIC_NONE;
	+ props->max_map_per_fmr = 1;
	+#ifdef IW_IXL_RDMA_4_9
	+ props->max_fast_reg_page_list_len = I40IW_MAX_PAGES_PER_FMR;
	+#endif
	+
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_query_port - get port attrubutes
	+ * @ibdev: device pointer from stack
	+ * @port: port number for query
	+ * @props: returning device attributes
	+ */
	+static int
	+i40iw_query_port(struct ib_device ibdev, u8 port, struct ib_port_attr props)
	+{
	+ struct i40iw_device *iwdev = to_iwdev(ibdev);
	+ struct ifnet *ifp = iwdev->ifp;
	+
	+ memset(props, 0, sizeof(*props));
	+
	+ props->max_mtu = IB_MTU_4096;
	+ if (ifp->if_mtu >= 4096)
	+ props->active_mtu = IB_MTU_4096;
	+ else if (ifp->if_mtu >= 2048)
	+ props->active_mtu = IB_MTU_2048;
	+ else if (ifp->if_mtu >= 1024)
	+ props->active_mtu = IB_MTU_1024;
	+ else if (ifp->if_mtu >= 512)
	+ props->active_mtu = IB_MTU_512;
	+ else
	+ props->active_mtu = IB_MTU_256;
	+
	+ props->lid = 1;
	+ if (ifp->if_drv_flags & IFF_DRV_RUNNING)
	+ props->state = IB_PORT_ACTIVE;
	+ else
	+ props->state = IB_PORT_DOWN;
	+ props->port_cap_flags = IB_PORT_CM_SUP \| IB_PORT_REINIT_SUP \|
	+ IB_PORT_VENDOR_CLASS_SUP \| IB_PORT_BOOT_MGMT_SUP;
	+ props->gid_tbl_len = 1;
	+ props->pkey_tbl_len = 1;
	+ props->active_width = IB_WIDTH_4X;
	+ props->active_speed = 1;
	+ props->max_msg_sz = I40IW_MAX_OUTBOUND_MESSAGE_SIZE;
	+
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_alloc_push_page - allocate a push page for qp
	+ * @iwdev: iwarp device
	+ * @qp: hardware control qp
	+ */
	+static void
	+i40iw_alloc_push_page(struct i40iw_device *iwdev,
	+ struct i40iw_sc_qp *qp)
	+{
	+ struct i40iw_cqp_request *cqp_request;
	+ struct cqp_commands_info *cqp_info;
	+ enum i40iw_status_code status;
	+
	+ if (qp->push_idx != I40IW_INVALID_PUSH_PAGE_INDEX)
	+ return;
	+
	+ cqp_request = i40iw_get_cqp_request(&iwdev->cqp, TRUE);
	+ if (!cqp_request)
	+ return;
	+
	+ atomic_inc(&cqp_request->refcount);
	+
	+ cqp_info = &cqp_request->info;
	+ cqp_info->cqp_cmd = OP_MANAGE_PUSH_PAGE;
	+ cqp_info->post_sq = 1;
	+
	+ cqp_info->in.u.manage_push_page.info.qs_handle =
	+ iwdev->vsi.qos[0].qs_handle;
	+ cqp_info->in.u.manage_push_page.info.free_page = 0;
	+ cqp_info->in.u.manage_push_page.cqp = &iwdev->cqp.sc_cqp;
	+ cqp_info->in.u.manage_push_page.scratch = (uintptr_t)cqp_request;
	+
	+ status = i40iw_handle_cqp_op(iwdev, cqp_request);
	+ if (!status)
	+ qp->push_idx = cqp_request->compl_info.op_ret_val;
	+ else
	+ device_printf(iwdev->ldev->dev, "CQP-OP Push page fail");
	+ i40iw_put_cqp_request(&iwdev->cqp, cqp_request);
	+}
	+
	+/**
	+ * i40iw_dealloc_push_page - free a push page for qp
	+ * @iwdev: iwarp device
	+ * @qp: hardware control qp
	+ */
	+static void
	+i40iw_dealloc_push_page(struct i40iw_device iwdev, struct i40iw_sc_qp qp)
	+{
	+ struct i40iw_cqp_request *cqp_request;
	+ struct cqp_commands_info *cqp_info;
	+ enum i40iw_status_code status;
	+
	+ if (qp->push_idx == I40IW_INVALID_PUSH_PAGE_INDEX)
	+ return;
	+
	+ cqp_request = i40iw_get_cqp_request(&iwdev->cqp, FALSE);
	+ if (!cqp_request)
	+ return;
	+
	+ cqp_info = &cqp_request->info;
	+ cqp_info->cqp_cmd = OP_MANAGE_PUSH_PAGE;
	+ cqp_info->post_sq = 1;
	+
	+ cqp_info->in.u.manage_push_page.info.push_idx = qp->push_idx;
	+ cqp_info->in.u.manage_push_page.info.qs_handle =
	+ iwdev->vsi.qos[0].qs_handle;
	+ cqp_info->in.u.manage_push_page.info.free_page = 1;
	+ cqp_info->in.u.manage_push_page.cqp = &iwdev->cqp.sc_cqp;
	+ cqp_info->in.u.manage_push_page.scratch = (uintptr_t)cqp_request;
	+
	+ status = i40iw_handle_cqp_op(iwdev, cqp_request);
	+ if (!status)
	+ qp->push_idx = I40IW_INVALID_PUSH_PAGE_INDEX;
	+ else
	+ device_printf(iwdev->ldev->dev, "CQP-OP Push page fail");
	+}
	+
	+/**
	+ * i40iw_alloc_pd - allocate protection domain
	+ * @ibdev: device pointer from stack
	+ * @context: user context created during alloc
	+ * @udata: user data
	+ */
	+static struct ib_pd *
	+i40iw_alloc_pd(struct ib_device ibdev, struct ib_ucontext context,
	+ struct ib_udata *udata)
	+{
	+ struct i40iw_pd *iwpd;
	+ struct i40iw_device *iwdev = to_iwdev(ibdev);
	+ struct i40iw_sc_dev *dev = &iwdev->sc_dev;
	+ struct i40iw_alloc_pd_resp uresp;
	+ struct i40iw_sc_pd *sc_pd;
	+ u32 pd_id = 0;
	+ int err;
	+
	+ if (iwdev->closing)
	+ return ERR_PTR(-ENODEV);
	+
	+ err = i40iw_alloc_resource(iwdev, iwdev->allocated_pds,
	+ iwdev->max_pd, &pd_id, &iwdev->next_pd);
	+ if (err) {
	+ device_printf(iwdev->ldev->dev, "alloc resource failed\n");
	+ return ERR_PTR(err);
	+ }
	+ iwpd = kzalloc(sizeof(*iwpd), GFP_KERNEL);
	+ if (!iwpd) {
	+ err = -ENOMEM;
	+ goto free_res;
	+ }
	+ sc_pd = &iwpd->sc_pd;
	+ dev->iw_pd_ops->pd_init(dev, sc_pd, pd_id);
	+
	+ if (context) {
	+ memset(&uresp, 0, sizeof(uresp));
	+ uresp.pd_id = pd_id;
	+ if (ib_copy_to_udata(udata, &uresp, sizeof(uresp))) {
	+ err = -EFAULT;
	+ goto error;
	+ }
	+ }
	+ i40iw_add_pdusecount(iwpd);
	+ return &iwpd->ibpd;
	+error:
	+ kfree(iwpd);
	+free_res:
	+ i40iw_free_resource(iwdev, iwdev->allocated_pds, pd_id);
	+ return ERR_PTR(err);
	+}
	+
	+/**
	+ * i40iw_dealloc_pd - deallocate pd
	+ * @ibpd: ptr of pd to be deallocated
	+ */
	+static int
	+i40iw_dealloc_pd(struct ib_pd *ibpd)
	+{
	+ struct i40iw_pd *iwpd = to_iwpd(ibpd);
	+ struct i40iw_device *iwdev = to_iwdev(ibpd->device);
	+
	+ i40iw_rem_pdusecount(iwpd, iwdev);
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_qp_roundup - return round up qp ring size
	+ * @wr_ring_size: ring size to round up
	+ */
	+static int
	+i40iw_qp_roundup(u32 wr_ring_size)
	+{
	+ int scount = 1;
	+
	+ if (wr_ring_size < I40IW_QP_SW_MIN_WQSIZE)
	+ wr_ring_size = I40IW_QP_SW_MIN_WQSIZE;
	+
	+ for (wr_ring_size--; scount <= 16; scount *= 2)
	+ wr_ring_size \|= wr_ring_size >> scount;
	+ return ++wr_ring_size;
	+}
	+
	+/**
	+ * i40iw_get_pbl - Retrieve pbl from a list given a virtual
	+ * address
	+ * @va: user virtual address
	+ * @pbl_list: pbl list to search in (QP's or CQ's)
	+ */
	+static struct i40iw_pbl *
	+i40iw_get_pbl(unsigned long va,
	+ struct list_head *pbl_list)
	+{
	+ struct i40iw_pbl *iwpbl;
	+
	+ list_for_each_entry(iwpbl, pbl_list, list) {
	+ if (iwpbl->user_base == va) {
	+ list_del(&iwpbl->list);
	+ return iwpbl;
	+ }
	+ }
	+ return NULL;
	+}
	+
	+/**
	+ * i40iw_free_qp_resources - free up memory resources for qp
	+ * @iwdev: iwarp device
	+ * @iwqp: qp ptr (user or kernel)
	+ * @qp_num: qp number assigned
	+ */
	+void
	+i40iw_free_qp_resources(struct i40iw_device iwdev, struct i40iw_qp iwqp,
	+ u32 qp_num)
	+{
	+ struct i40iw_pbl *iwpbl = &iwqp->iwpbl;
	+
	+ i40iw_ieq_cleanup_qp(iwdev->vsi.ieq, &iwqp->sc_qp);
	+ i40iw_dealloc_push_page(iwdev, &iwqp->sc_qp);
	+ if (qp_num)
	+ i40iw_free_resource(iwdev, iwdev->allocated_qps, qp_num);
	+ if (iwpbl->pbl_allocated)
	+ i40iw_free_pble(iwdev->pble_rsrc, &iwpbl->pble_alloc);
	+ ixliw_free_dma_mem(&iwqp->q2_ctx_mem);
	+ ixliw_free_dma_mem(&iwqp->kqp.dma_mem);
	+ kfree(iwqp->kqp.wrid_mem);
	+ iwqp->kqp.wrid_mem = NULL;
	+ kfree(iwqp->allocated_buffer);
	+}
	+
	+/**
	+ * i40iw_clean_cqes - clean cq entries for qp
	+ * @iwqp: qp ptr (user or kernel)
	+ * @iwcq: cq ptr
	+ */
	+static void
	+i40iw_clean_cqes(struct i40iw_qp iwqp, struct i40iw_cq iwcq)
	+{
	+ struct i40iw_cq_uk *ukcq = &iwcq->sc_cq.cq_uk;
	+
	+ ukcq->ops.iw_cq_clean(&iwqp->sc_qp.qp_uk, ukcq);
	+}
	+
	+/**
	+ * i40iw_destroy_qp - destroy qp
	+ * @ibqp: qp's ib pointer also to get to device's qp address
	+ */
	+static int
	+i40iw_destroy_qp(struct ib_qp *ibqp)
	+{
	+ struct i40iw_qp *iwqp = to_iwqp(ibqp);
	+
	+ iwqp->destroyed = 1;
	+
	+ if (iwqp->ibqp_state >= IB_QPS_INIT && iwqp->ibqp_state < IB_QPS_RTS)
	+ i40iw_next_iw_state(iwqp, I40IW_QP_STATE_ERROR, 0, 0, 0);
	+
	+ if (!iwqp->user_mode) {
	+ if (iwqp->iwscq) {
	+ i40iw_clean_cqes(iwqp, iwqp->iwscq);
	+ if (iwqp->iwrcq != iwqp->iwscq)
	+ i40iw_clean_cqes(iwqp, iwqp->iwrcq);
	+ }
	+ }
	+ i40iw_rem_ref(&iwqp->ibqp);
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_setup_virt_qp - setup for allocation of virtual qp
	+ * @dev: iwarp device
	+ * @qp: qp ptr
	+ * @init_info: initialize info to return
	+ */
	+static int
	+i40iw_setup_virt_qp(struct i40iw_device *iwdev,
	+ struct i40iw_qp *iwqp,
	+ struct i40iw_qp_init_info *init_info)
	+{
	+ struct i40iw_pbl *iwpbl = &iwqp->iwpbl;
	+ struct i40iw_qp_mr *qpmr = &iwpbl->qp_mr;
	+
	+ iwqp->page = qpmr->sq_page;
	+ init_info->shadow_area_pa = cpu_to_le64(qpmr->shadow);
	+ if (iwpbl->pbl_allocated) {
	+ init_info->virtual_map = TRUE;
	+ init_info->sq_pa = qpmr->sq_pbl.idx;
	+ init_info->rq_pa = qpmr->rq_pbl.idx;
	+ } else {
	+ init_info->sq_pa = qpmr->sq_pbl.addr;
	+ init_info->rq_pa = qpmr->rq_pbl.addr;
	+ }
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_setup_kmode_qp - setup initialization for kernel mode qp
	+ * @iwdev: iwarp device
	+ * @iwqp: qp ptr (user or kernel)
	+ * @info: initialize info to return
	+ */
	+static int
	+i40iw_setup_kmode_qp(struct i40iw_device *iwdev,
	+ struct i40iw_qp *iwqp,
	+ struct i40iw_qp_init_info *info)
	+{
	+ struct i40iw_dma_mem *mem = &iwqp->kqp.dma_mem;
	+ struct i40iw_qp_uk_init_info *ukinfo = &info->qp_uk_init_info;
	+ u32 size;
	+ u32 sqdepth, rqdepth;
	+ u32 sq_size, rq_size;
	+ u8 sqshift;
	+ enum i40iw_status_code status;
	+
	+ sq_size = i40iw_qp_roundup(ukinfo->sq_size + 2);
	+ rq_size = i40iw_qp_roundup(ukinfo->rq_size + 1);
	+
	+ status = i40iw_get_wqe_shift(sq_size, ukinfo->max_sq_frag_cnt,
	+ ukinfo->max_inline_data, &sqshift);
	+ if (status)
	+ return -ENOMEM;
	+
	+ sqdepth = sq_size << sqshift;
	+ rqdepth = rq_size << I40IW_MAX_RQ_WQE_SHIFT;
	+
	+ size = sqdepth * sizeof(struct i40iw_sq_uk_wr_trk_info) + (rqdepth << 3);
	+ iwqp->kqp.wrid_mem = kzalloc(size, GFP_KERNEL);
	+
	+ ukinfo->sq_wrtrk_array =
	+ (struct i40iw_sq_uk_wr_trk_info *)iwqp->kqp.wrid_mem;
	+ if (!ukinfo->sq_wrtrk_array)
	+ return -ENOMEM;
	+
	+ ukinfo->rq_wrid_array = (u64 *)&ukinfo->sq_wrtrk_array[sqdepth];
	+
	+ size = (sqdepth + rqdepth) * I40IW_QP_WQE_MIN_SIZE;
	+ size += (I40IW_SHADOW_AREA_SIZE << 3);
	+
	+ status = i40iw_allocate_dma_mem(iwdev->sc_dev.hw, mem, size, 256);
	+ if (status) {
	+ kfree(ukinfo->sq_wrtrk_array);
	+ ukinfo->sq_wrtrk_array = NULL;
	+ return -ENOMEM;
	+ }
	+ ukinfo->sq = mem->va;
	+ info->sq_pa = mem->pa;
	+
	+ ukinfo->rq = &ukinfo->sq[sqdepth];
	+ info->rq_pa = info->sq_pa + (sqdepth * I40IW_QP_WQE_MIN_SIZE);
	+
	+ ukinfo->shadow_area = ukinfo->rq[rqdepth].elem;
	+ info->shadow_area_pa = info->rq_pa + (rqdepth * I40IW_QP_WQE_MIN_SIZE);
	+
	+ ukinfo->sq_size = sq_size;
	+ ukinfo->rq_size = rq_size;
	+ ukinfo->qp_id = iwqp->ibqp.qp_num;
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_create_qp - create qp
	+ * @ibpd: ptr of pd
	+ * @init_attr: attributes for qp
	+ * @udata: user data for create qp
	+ */
	+static struct ib_qp *
	+i40iw_create_qp(struct ib_pd ibpd, struct ib_qp_init_attr init_attr,
	+ struct ib_udata *udata)
	+{
	+ struct i40iw_device *iwdev = to_iwdev(ibpd->device);
	+ struct i40iw_cqp_request *cqp_request;
	+ struct cqp_commands_info *cqp_info;
	+ struct i40iw_qp_host_ctx_info *ctx_info;
	+ struct i40iw_sc_dev *dev = &iwdev->sc_dev;
	+ struct i40iw_qp_init_info init_info;
	+ struct i40iwarp_offload_info *iwarp_info;
	+ struct i40iw_cqp *iwcqp = &iwdev->cqp;
	+ struct i40iw_pd *iwpd = to_iwpd(ibpd);
	+ struct i40iw_qp *iwqp;
	+ struct i40iw_sc_qp *qp;
	+ struct i40iw_create_qp_info *qp_info;
	+ struct i40iw_create_qp_req req;
	+ struct i40iw_ucontext *ucontext;
	+ struct i40iw_create_qp_resp uresp;
	+ void *mem;
	+ int err_code;
	+ int sq_size;
	+ int rq_size;
	+ unsigned long flags;
	+ u32 qp_num = 0;
	+ enum i40iw_status_code ret;
	+
	+ if (iwdev->closing)
	+ return ERR_PTR(-ENODEV);
	+
	+ if (init_attr->create_flags)
	+ return ERR_PTR(-EINVAL);
	+
	+ if (init_attr->cap.max_inline_data > I40IW_MAX_INLINE_DATA_SIZE)
	+ init_attr->cap.max_inline_data = I40IW_MAX_INLINE_DATA_SIZE;
	+
	+ if (init_attr->cap.max_send_sge > I40IW_MAX_WQ_FRAGMENT_COUNT)
	+ init_attr->cap.max_send_sge = I40IW_MAX_WQ_FRAGMENT_COUNT;
	+
	+ if (init_attr->cap.max_recv_sge > I40IW_MAX_WQ_FRAGMENT_COUNT)
	+ init_attr->cap.max_recv_sge = I40IW_MAX_WQ_FRAGMENT_COUNT;
	+
	+ memset(&init_info, 0, sizeof(init_info));
	+
	+ sq_size = init_attr->cap.max_send_wr;
	+ rq_size = init_attr->cap.max_recv_wr;
	+
	+ init_info.vsi = &iwdev->vsi;
	+ init_info.qp_uk_init_info.sq_size = sq_size;
	+ init_info.qp_uk_init_info.rq_size = rq_size;
	+ init_info.qp_uk_init_info.max_sq_frag_cnt = init_attr->cap.max_send_sge;
	+ init_info.qp_uk_init_info.max_rq_frag_cnt = init_attr->cap.max_recv_sge;
	+ init_info.qp_uk_init_info.max_inline_data =
	+ init_attr->cap.max_inline_data;
	+
	+ mem = kzalloc(sizeof(*iwqp), GFP_KERNEL);
	+ if (!mem)
	+ return ERR_PTR(-ENOMEM);
	+
	+ iwqp = (struct i40iw_qp *)mem;
	+ qp = &iwqp->sc_qp;
	+ qp->back_qp = (void *)iwqp;
	+ qp->push_idx = I40IW_INVALID_PUSH_PAGE_INDEX;
	+
	+ iwqp->ctx_info.iwarp_info = &iwqp->iwarp_info;
	+
	+ if (i40iw_allocate_dma_mem(dev->hw,
	+ &iwqp->q2_ctx_mem,
	+ I40IW_Q2_BUFFER_SIZE + I40IW_QP_CTX_SIZE,
	+ 256)) {
	+ device_printf(iwdev->ldev->dev, "dma_mem alloc failed\n");
	+ err_code = -ENOMEM;
	+ goto error;
	+ }
	+ init_info.q2 = iwqp->q2_ctx_mem.va;
	+ init_info.q2_pa = iwqp->q2_ctx_mem.pa;
	+
	+ init_info.host_ctx = (u64 *)((uintptr_t)init_info.q2 +
	+ I40IW_Q2_BUFFER_SIZE);
	+ init_info.host_ctx_pa = init_info.q2_pa + I40IW_Q2_BUFFER_SIZE;
	+
	+ err_code = i40iw_alloc_resource(iwdev, iwdev->allocated_qps, iwdev->max_qp,
	+ &qp_num, &iwdev->next_qp);
	+ if (err_code) {
	+ device_printf(iwdev->ldev->dev, "qp resource alloc failed\n");
	+ goto error;
	+ }
	+ iwqp->allocated_buffer = mem;
	+ iwqp->iwdev = iwdev;
	+ iwqp->iwpd = iwpd;
	+ iwqp->ibqp.qp_num = qp_num;
	+ qp = &iwqp->sc_qp;
	+ iwqp->iwscq = to_iwcq(init_attr->send_cq);
	+ iwqp->iwrcq = to_iwcq(init_attr->recv_cq);
	+
	+ iwqp->host_ctx.va = init_info.host_ctx;
	+ iwqp->host_ctx.pa = init_info.host_ctx_pa;
	+ iwqp->host_ctx.size = I40IW_QP_CTX_SIZE;
	+
	+ init_info.pd = &iwpd->sc_pd;
	+ init_info.qp_uk_init_info.qp_id = iwqp->ibqp.qp_num;
	+ iwqp->ctx_info.qp_compl_ctx = (uintptr_t)qp;
	+
	+ if (init_attr->qp_type != IB_QPT_RC) {
	+ err_code = -EINVAL;
	+ goto error;
	+ }
	+ if (iwdev->push_mode)
	+ i40iw_alloc_push_page(iwdev, qp);
	+ if (udata) {
	+ err_code = ib_copy_from_udata(&req, udata, sizeof(req));
	+ if (err_code) {
	+ i40iw_debug(dev, I40IW_DEBUG_ERR \| I40IW_DEBUG_VERBS \| I40IW_DEBUG_QP,
	+ "ib_copy_from_data error %d\n", err_code);
	+ goto error;
	+ }
	+ iwqp->ctx_info.qp_compl_ctx = req.user_compl_ctx;
	+ if (ibpd->uobject && ibpd->uobject->context) {
	+ iwqp->user_mode = 1;
	+ ucontext = to_ucontext(ibpd->uobject->context);
	+
	+ if (req.user_wqe_buffers) {
	+ struct i40iw_pbl *iwpbl;
	+
	+ spin_lock_irqsave(
	+ &ucontext->qp_reg_mem_list_lock, flags);
	+ iwpbl = i40iw_get_pbl(
	+ (unsigned long)req.user_wqe_buffers,
	+ &ucontext->qp_reg_mem_list);
	+ spin_unlock_irqrestore(
	+ &ucontext->qp_reg_mem_list_lock, flags);
	+
	+ if (iwpbl) {
	+ err_code = -ENODATA;
	+ i40iw_debug(dev, I40IW_DEBUG_ERR \| I40IW_DEBUG_PBLE, "no pbl info\n");
	+ goto error;
	+ }
	+ memcpy(&iwqp->iwpbl, iwpbl, sizeof(iwqp->iwpbl));
	+ }
	+ }
	+ err_code = i40iw_setup_virt_qp(iwdev, iwqp, &init_info);
	+ } else {
	+ err_code = i40iw_setup_kmode_qp(iwdev, iwqp, &init_info);
	+ }
	+
	+ if (err_code) {
	+ i40iw_debug(dev, I40IW_DEBUG_ERR \| I40IW_DEBUG_VERBS \|
	+ I40IW_DEBUG_QP, "setup qp failed\n");
	+ goto error;
	+ }
	+ init_info.type = I40IW_QP_TYPE_IWARP;
	+ ret = dev->iw_priv_qp_ops->qp_init(qp, &init_info);
	+ if (ret) {
	+ err_code = -EPROTO;
	+ i40iw_debug(dev, I40IW_DEBUG_ERR \| I40IW_DEBUG_VERBS \|
	+ I40IW_DEBUG_QP, "qp_init fail\n");
	+ goto error;
	+ }
	+ ctx_info = &iwqp->ctx_info;
	+
	+ iwarp_info = &iwqp->iwarp_info;
	+ iwarp_info->rd_enable = TRUE;
	+ iwarp_info->wr_rdresp_en = TRUE;
	+ if (!iwqp->user_mode) {
	+ iwarp_info->fast_reg_en = TRUE;
	+ iwarp_info->priv_mode_en = TRUE;
	+ }
	+ iwarp_info->ddp_ver = 1;
	+ iwarp_info->rdmap_ver = 1;
	+
	+ ctx_info->iwarp_info_valid = TRUE;
	+ ctx_info->send_cq_num = iwqp->iwscq->sc_cq.cq_uk.cq_id;
	+ ctx_info->rcv_cq_num = iwqp->iwrcq->sc_cq.cq_uk.cq_id;
	+ if (qp->push_idx == I40IW_INVALID_PUSH_PAGE_INDEX) {
	+ ctx_info->push_mode_en = FALSE;
	+ } else {
	+ ctx_info->push_mode_en = TRUE;
	+ ctx_info->push_idx = qp->push_idx;
	+ }
	+
	+ ret = dev->iw_priv_qp_ops->qp_setctx(&iwqp->sc_qp,
	+ (u64 *)iwqp->host_ctx.va,
	+ ctx_info);
	+ ctx_info->iwarp_info_valid = FALSE;
	+ cqp_request = i40iw_get_cqp_request(iwcqp, TRUE);
	+ if (!cqp_request) {
	+ err_code = -ENOMEM;
	+ goto error;
	+ }
	+ cqp_info = &cqp_request->info;
	+ qp_info = &cqp_request->info.in.u.qp_create.info;
	+
	+ memset(qp_info, 0, sizeof(*qp_info));
	+
	+ qp_info->cq_num_valid = TRUE;
	+ qp_info->next_iwarp_state = I40IW_QP_STATE_IDLE;
	+
	+ cqp_info->cqp_cmd = OP_QP_CREATE;
	+ cqp_info->post_sq = 1;
	+ cqp_info->in.u.qp_create.qp = qp;
	+ cqp_info->in.u.qp_create.scratch = (uintptr_t)cqp_request;
	+ ret = i40iw_handle_cqp_op(iwdev, cqp_request);
	+ if (ret) {
	+ i40iw_debug(dev, I40IW_DEBUG_ERR \| I40IW_DEBUG_VERBS \|
	+ I40IW_DEBUG_CQP, "CQP-OP QP create fail");
	+ err_code = -EACCES;
	+ goto error;
	+ }
	+ i40iw_add_ref(&iwqp->ibqp);
	+ spin_lock_init(&iwqp->lock);
	+ iwqp->sig_all = (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR) ? 1 : 0;
	+ iwdev->qp_table[qp_num] = iwqp;
	+ i40iw_add_pdusecount(iwqp->iwpd);
	+ i40iw_add_devusecount(iwdev);
	+ if (ibpd->uobject && udata) {
	+ memset(&uresp, 0, sizeof(uresp));
	+ uresp.actual_sq_size = sq_size;
	+ uresp.actual_rq_size = rq_size;
	+ uresp.qp_id = qp_num;
	+ uresp.push_idx = qp->push_idx;
	+ err_code = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
	+ if (err_code) {
	+ i40iw_debug(dev, I40IW_DEBUG_ERR \| I40IW_DEBUG_VERBS \|
	+ I40IW_DEBUG_QP, "copy_to_udata failed\n");
	+ i40iw_destroy_qp(&iwqp->ibqp);
	+ /* let the completion of the qp destroy free the qp */
	+ return ERR_PTR(err_code);
	+ }
	+ }
	+#ifdef IW_IXL_RDMA_4_9
	+ init_completion(&iwqp->sq_drained);
	+ init_completion(&iwqp->rq_drained);
	+#endif
	+
	+ return &iwqp->ibqp;
	+error:
	+ i40iw_free_qp_resources(iwdev, iwqp, qp_num);
	+ return ERR_PTR(err_code);
	+}
	+
	+/**
	+ * i40iw_query - query qp attributes
	+ * @ibqp: qp pointer
	+ * @attr: attributes pointer
	+ * @attr_mask: Not used
	+ * @init_attr: qp attributes to return
	+ */
	+static int
	+i40iw_query_qp(struct ib_qp ibqp, struct ib_qp_attr attr, int attr_mask,
	+ struct ib_qp_init_attr *init_attr)
	+{
	+ struct i40iw_qp *iwqp = to_iwqp(ibqp);
	+
	+ attr->qp_access_flags = 0;
	+ attr->cap.max_inline_data = I40IW_MAX_INLINE_DATA_SIZE;
	+ attr->cap.max_send_sge = I40IW_MAX_WQ_FRAGMENT_COUNT;
	+ attr->cap.max_recv_sge = I40IW_MAX_WQ_FRAGMENT_COUNT;
	+ attr->port_num = 1;
	+ init_attr->event_handler = iwqp->ibqp.event_handler;
	+ init_attr->qp_context = iwqp->ibqp.qp_context;
	+ init_attr->send_cq = iwqp->ibqp.send_cq;
	+ init_attr->recv_cq = iwqp->ibqp.recv_cq;
	+ init_attr->srq = iwqp->ibqp.srq;
	+ init_attr->cap = attr->cap;
	+ init_attr->port_num = 1;
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_hw_modify_qp - setup cqp for modify qp
	+ * @iwdev: iwarp device
	+ * @iwqp: qp ptr (user or kernel)
	+ * @info: info for modify qp
	+ * @wait: flag to wait or not for modify qp completion
	+ */
	+void
	+i40iw_hw_modify_qp(struct i40iw_device iwdev, struct i40iw_qp iwqp,
	+ struct i40iw_modify_qp_info *info, bool wait)
	+{
	+ struct i40iw_cqp_request *cqp_request;
	+ struct cqp_commands_info *cqp_info;
	+ struct i40iw_modify_qp_info *m_info;
	+ enum i40iw_status_code status;
	+
	+ cqp_request = i40iw_get_cqp_request(&iwdev->cqp, wait);
	+ if (!cqp_request)
	+ return;
	+
	+ cqp_info = &cqp_request->info;
	+ m_info = &cqp_info->in.u.qp_modify.info;
	+ memcpy(m_info, info, sizeof(*m_info));
	+ cqp_info->cqp_cmd = OP_QP_MODIFY;
	+ cqp_info->post_sq = 1;
	+ cqp_info->in.u.qp_modify.qp = &iwqp->sc_qp;
	+ cqp_info->in.u.qp_modify.scratch = (uintptr_t)cqp_request;
	+ status = i40iw_handle_cqp_op(iwdev, cqp_request);
	+ if (status) {
	+ switch (m_info->next_iwarp_state) {
	+ struct i40iw_gen_ae_info ae_info;
	+ case I40IW_QP_STATE_RTS:
	+ if (iwqp->iwarp_state == I40IW_QP_STATE_IDLE)
	+ i40iw_send_reset(iwqp->cm_node);
	+ /* fall through */
	+ case I40IW_QP_STATE_IDLE:
	+ case I40IW_QP_STATE_TERMINATE:
	+ case I40IW_QP_STATE_CLOSING:
	+ ae_info.ae_code = I40IW_AE_BAD_CLOSE;
	+ ae_info.ae_source = 0;
	+ i40iw_gen_ae(iwdev, &iwqp->sc_qp, &ae_info, FALSE);
	+ break;
	+ case I40IW_QP_STATE_ERROR:
	+ default:
	+ break;
	+ }
	+ }
	+}
	+
	+int
	+i40iw_modify_qp(struct ib_qp ibqp, struct ib_qp_attr attr, int attr_mask,
	+ struct ib_udata *udata)
	+{
	+ struct i40iw_qp *iwqp = to_iwqp(ibqp);
	+ struct i40iw_device *iwdev = iwqp->iwdev;
	+ struct i40iw_qp_host_ctx_info *ctx_info;
	+ struct i40iwarp_offload_info *iwarp_info;
	+ struct i40iw_modify_qp_info info;
	+ unsigned long flags;
	+ u32 err;
	+ u8 issue_modify_qp = 0;
	+ u8 dont_wait = 0;
	+
	+ memset(&info, 0, sizeof(info));
	+ ctx_info = &iwqp->ctx_info;
	+ iwarp_info = &iwqp->iwarp_info;
	+
	+ spin_lock_irqsave(&iwqp->lock, flags);
	+
	+ if (attr_mask & IB_QP_STATE) {
	+ if (iwdev->closing && attr->qp_state != IB_QPS_ERR) {
	+ err = -EINVAL;
	+ goto exit;
	+ }
	+ switch (attr->qp_state) {
	+ case IB_QPS_INIT:
	+ case IB_QPS_RTR:
	+ if (iwqp->iwarp_state > (u32)I40IW_QP_STATE_IDLE) {
	+ err = -EINVAL;
	+ goto exit;
	+ }
	+ if (iwqp->iwarp_state == I40IW_QP_STATE_INVALID) {
	+ info.next_iwarp_state = I40IW_QP_STATE_IDLE;
	+ issue_modify_qp = 1;
	+ }
	+ break;
	+ case IB_QPS_RTS:
	+ if ((iwqp->iwarp_state > (u32)I40IW_QP_STATE_RTS) \|\|
	+ (!iwqp->cm_id)) {
	+ err = -EINVAL;
	+ goto exit;
	+ }
	+ issue_modify_qp = 1;
	+ iwqp->hw_tcp_state = I40IW_TCP_STATE_ESTABLISHED;
	+ iwqp->hte_added = 1;
	+ info.next_iwarp_state = I40IW_QP_STATE_RTS;
	+ info.tcp_ctx_valid = TRUE;
	+ info.ord_valid = TRUE;
	+ info.arp_cache_idx_valid = TRUE;
	+ info.cq_num_valid = TRUE;
	+ break;
	+ case IB_QPS_SQD:
	+ if (iwqp->hw_iwarp_state > (u32)I40IW_QP_STATE_RTS) {
	+ err = 0;
	+ goto exit;
	+ }
	+ if ((iwqp->iwarp_state == (u32)I40IW_QP_STATE_CLOSING) \|\|
	+ (iwqp->iwarp_state < (u32)I40IW_QP_STATE_RTS)) {
	+ err = 0;
	+ goto exit;
	+ }
	+ if (iwqp->iwarp_state > (u32)I40IW_QP_STATE_CLOSING) {
	+ err = -EINVAL;
	+ goto exit;
	+ }
	+ info.next_iwarp_state = I40IW_QP_STATE_CLOSING;
	+ issue_modify_qp = 1;
	+ break;
	+ case IB_QPS_SQE:
	+ if (iwqp->iwarp_state >= (u32)I40IW_QP_STATE_TERMINATE) {
	+ err = -EINVAL;
	+ goto exit;
	+ }
	+ info.next_iwarp_state = I40IW_QP_STATE_TERMINATE;
	+ issue_modify_qp = 1;
	+ break;
	+ case IB_QPS_ERR:
	+ case IB_QPS_RESET:
	+ if (iwqp->iwarp_state == (u32)I40IW_QP_STATE_ERROR) {
	+ err = -EINVAL;
	+ goto exit;
	+ }
	+ if (iwqp->sc_qp.term_flags)
	+ del_timer(&iwqp->terminate_timer);
	+ info.next_iwarp_state = I40IW_QP_STATE_ERROR;
	+ if ((iwqp->hw_tcp_state > I40IW_TCP_STATE_CLOSED) &&
	+ iwdev->iw_status &&
	+ (iwqp->hw_tcp_state != I40IW_TCP_STATE_TIME_WAIT))
	+ info.reset_tcp_conn = TRUE;
	+ else
	+ dont_wait = 1;
	+ issue_modify_qp = 1;
	+ info.next_iwarp_state = I40IW_QP_STATE_ERROR;
	+ break;
	+ default:
	+ err = -EINVAL;
	+ goto exit;
	+ }
	+
	+ iwqp->ibqp_state = attr->qp_state;
	+
	+ }
	+ if (attr_mask & IB_QP_ACCESS_FLAGS) {
	+ ctx_info->iwarp_info_valid = TRUE;
	+ if (attr->qp_access_flags & IB_ACCESS_LOCAL_WRITE)
	+ iwarp_info->wr_rdresp_en = TRUE;
	+ if (attr->qp_access_flags & IB_ACCESS_REMOTE_WRITE)
	+ iwarp_info->wr_rdresp_en = TRUE;
	+ if (attr->qp_access_flags & IB_ACCESS_REMOTE_READ)
	+ iwarp_info->rd_enable = TRUE;
	+ if (attr->qp_access_flags & IB_ACCESS_MW_BIND)
	+ iwarp_info->bind_en = TRUE;
	+
	+ if (iwqp->user_mode) {
	+ iwarp_info->rd_enable = TRUE;
	+ iwarp_info->wr_rdresp_en = TRUE;
	+ iwarp_info->priv_mode_en = FALSE;
	+ }
	+ }
	+ if (ctx_info->iwarp_info_valid) {
	+ struct i40iw_sc_dev *dev = &iwdev->sc_dev;
	+ int ret;
	+
	+ ctx_info->send_cq_num = iwqp->iwscq->sc_cq.cq_uk.cq_id;
	+ ctx_info->rcv_cq_num = iwqp->iwrcq->sc_cq.cq_uk.cq_id;
	+ ret = dev->iw_priv_qp_ops->qp_setctx(&iwqp->sc_qp,
	+ (u64 *)iwqp->host_ctx.va,
	+ ctx_info);
	+ if (ret) {
	+ i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_VERBS \| I40IW_DEBUG_QP,
	+ "setting QP context\n");
	+ err = -EINVAL;
	+ goto exit;
	+ }
	+ }
	+ spin_unlock_irqrestore(&iwqp->lock, flags);
	+
	+ if (issue_modify_qp)
	+ i40iw_hw_modify_qp(iwdev, iwqp, &info, TRUE);
	+
	+ spin_lock_irqsave(&iwqp->lock, flags);
	+ if (attr_mask & IB_QP_STATE) {
	+ if (issue_modify_qp)
	+ iwqp->iwarp_state = info.next_iwarp_state;
	+ else
	+ info.next_iwarp_state = iwqp->iwarp_state;
	+ }
	+ spin_unlock_irqrestore(&iwqp->lock, flags);
	+
	+ if (issue_modify_qp && (iwqp->ibqp_state > IB_QPS_RTS)) {
	+ if (dont_wait) {
	+ if (iwqp->cm_id && iwqp->hw_tcp_state) {
	+ spin_lock_irqsave(&iwqp->lock, flags);
	+ iwqp->hw_tcp_state = I40IW_TCP_STATE_CLOSED;
	+ iwqp->last_aeq = I40IW_AE_RESET_SENT;
	+ spin_unlock_irqrestore(&iwqp->lock, flags);
	+ i40iw_cm_disconn(iwqp);
	+ }
	+ } else {
	+ spin_lock_irqsave(&iwqp->lock, flags);
	+ if (iwqp->cm_id) {
	+ if (atomic_inc_return(&iwqp->close_timer_started) == 1) {
	+ iwqp->cm_id->add_ref(iwqp->cm_id);
	+ i40iw_schedule_cm_timer(iwqp->cm_node,
	+ (struct i40iw_puda_buf *)iwqp,
	+ I40IW_TIMER_TYPE_CLOSE, 1, 0);
	+ }
	+ }
	+ spin_unlock_irqrestore(&iwqp->lock, flags);
	+ }
	+ }
	+ return 0;
	+exit:
	+ spin_unlock_irqrestore(&iwqp->lock, flags);
	+ return err;
	+}
	+
	+/**
	+ * cq_free_resources - free up recources for cq
	+ * @iwdev: iwarp device
	+ * @iwcq: cq ptr
	+ */
	+static void
	+cq_free_resources(struct i40iw_device *iwdev,
	+ struct i40iw_cq *iwcq)
	+{
	+ struct i40iw_sc_cq *cq = &iwcq->sc_cq;
	+
	+ if (!iwcq->user_mode)
	+ ixliw_free_dma_mem(&iwcq->kmem);
	+ i40iw_free_resource(iwdev, iwdev->allocated_cqs, cq->cq_uk.cq_id);
	+}
	+
	+/**
	+ * i40iw_cq_wq_destroy - send cq destroy cqp
	+ * @iwdev: iwarp device
	+ * @cq: hardware control cq
	+ */
	+void
	+i40iw_cq_wq_destroy(struct i40iw_device iwdev, struct i40iw_sc_cq cq)
	+{
	+ struct i40iw_cqp_request *cqp_request;
	+ struct cqp_commands_info *cqp_info;
	+ enum i40iw_status_code status;
	+
	+ cqp_request = i40iw_get_cqp_request(&iwdev->cqp, TRUE);
	+ if (!cqp_request)
	+ return;
	+
	+ cqp_info = &cqp_request->info;
	+
	+ cqp_info->cqp_cmd = OP_CQ_DESTROY;
	+ cqp_info->post_sq = 1;
	+ cqp_info->in.u.cq_destroy.cq = cq;
	+ cqp_info->in.u.cq_destroy.scratch = (uintptr_t)cqp_request;
	+ status = i40iw_handle_cqp_op(iwdev, cqp_request);
	+ if (status)
	+ i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_ERR \| I40IW_DEBUG_VERBS \|
	+ I40IW_DEBUG_CQ \| I40IW_DEBUG_CQP, "CQP-OP Destroy QP fail");
	+}
	+
	+/**
	+ * i40iw_destroy_cq - destroy cq
	+ * @ib_cq: cq pointer
	+ */
	+static int
	+i40iw_destroy_cq(struct ib_cq *ib_cq)
	+{
	+ struct i40iw_cq *iwcq;
	+ struct i40iw_device *iwdev;
	+ struct i40iw_sc_cq *cq;
	+
	+ if (!ib_cq) {
	+ DPRINTF("ib_cq == NULL\n");
	+ return 0;
	+ }
	+ iwcq = to_iwcq(ib_cq);
	+ iwdev = to_iwdev(ib_cq->device);
	+ cq = &iwcq->sc_cq;
	+ i40iw_cq_wq_destroy(iwdev, cq);
	+ cq_free_resources(iwdev, iwcq);
	+ kfree(iwcq);
	+ i40iw_rem_devusecount(iwdev);
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_create_cq - create cq
	+ * @ibdev: device pointer from stack
	+#if __FreeBSD_version >= 1100000
	+ * @attr: attributes for cq
	+#else
	+ * @entries: # of cq entries in cq
	+ * @comp_vector: not used
	+#endif
	+ * @context: user context created during alloc
	+ * @udata: user data
	+ */
	+static struct ib_cq *
	+i40iw_create_cq(struct ib_device *ibdev,
	+#if __FreeBSD_version >= 1100000
	+#ifdef IW_IXL_RDMA_4_9
	+ const struct ib_cq_init_attr *attr,
	+#else
	+ struct ib_cq_init_attr *attr,
	+#endif
	+#else
	+ int entries, int comp_vector,
	+#endif /* IW_IXL_FREEBSD11 */
	+ struct ib_ucontext context, struct ib_udata udata)
	+{
	+ struct i40iw_device *iwdev = to_iwdev(ibdev);
	+ struct i40iw_cq *iwcq;
	+ struct i40iw_pbl *iwpbl;
	+ struct i40iw_sc_cq *cq;
	+ struct i40iw_sc_dev *dev = &iwdev->sc_dev;
	+ struct i40iw_cq_init_info info;
	+ struct i40iw_cqp_request *cqp_request;
	+ struct cqp_commands_info *cqp_info;
	+ struct i40iw_cq_uk_init_info *ukinfo = &info.cq_uk_init_info;
	+ int err_code;
	+#if __FreeBSD_version >= 1100000
	+ int entries = attr->cqe;
	+ int comp_vector = attr->comp_vector;
	+#endif /* IW_IXL_FREEBSD11 */
	+ unsigned long flags;
	+ u32 cq_num = 0;
	+ enum i40iw_status_code status;
	+
	+ if (iwdev->closing)
	+ return ERR_PTR(-ENODEV);
	+
	+ if (entries > iwdev->max_cqe)
	+ return ERR_PTR(-EINVAL);
	+
	+ iwcq = kzalloc(sizeof(*iwcq), GFP_KERNEL);
	+ if (!iwcq)
	+ return ERR_PTR(-ENOMEM);
	+
	+ memset(&info, 0, sizeof(info));
	+
	+ err_code = i40iw_alloc_resource(iwdev, iwdev->allocated_cqs,
	+ iwdev->max_cq, &cq_num,
	+ &iwdev->next_cq);
	+ if (err_code)
	+ goto error;
	+
	+ cq = &iwcq->sc_cq;
	+ cq->back_cq = (void *)iwcq;
	+ spin_lock_init(&iwcq->lock);
	+
	+ info.dev = dev;
	+ ukinfo->cq_size = max(entries, 4);
	+ ukinfo->cq_id = cq_num;
	+ iwcq->ibcq.cqe = info.cq_uk_init_info.cq_size;
	+ info.ceqe_mask = 0;
	+ if (comp_vector < iwdev->ceqs_count)
	+ info.ceq_id = comp_vector;
	+ info.ceq_id_valid = TRUE;
	+ info.ceqe_mask = 1;
	+ info.type = I40IW_CQ_TYPE_IWARP;
	+ if (context) {
	+ struct i40iw_ucontext *ucontext;
	+ struct i40iw_create_cq_req req;
	+ struct i40iw_cq_mr *cqmr;
	+
	+ memset(&req, 0, sizeof(req));
	+ iwcq->user_mode = TRUE;
	+ ucontext = to_ucontext(context);
	+ if (ib_copy_from_udata(&req, udata,
	+ sizeof(struct i40iw_create_cq_req)))
	+ goto cq_free_resources;
	+
	+ spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags);
	+ iwpbl = i40iw_get_pbl((unsigned long)req.user_cq_buffer,
	+ &ucontext->cq_reg_mem_list);
	+ spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags);
	+ if (!iwpbl) {
	+ err_code = -EPROTO;
	+ goto cq_free_resources;
	+ }
	+ iwcq->iwpbl = iwpbl;
	+ iwcq->cq_mem_size = 0;
	+ cqmr = &iwpbl->cq_mr;
	+ info.shadow_area_pa = cpu_to_le64(cqmr->shadow);
	+ if (iwpbl->pbl_allocated) {
	+ info.virtual_map = TRUE;
	+ info.pbl_chunk_size = 1;
	+ info.first_pm_pbl_idx = cqmr->cq_pbl.idx;
	+ } else {
	+ info.cq_base_pa = cqmr->cq_pbl.addr;
	+ }
	+ } else {
	+ /* Kmode allocations */
	+ int rsize;
	+ int shadow;
	+
	+ rsize = info.cq_uk_init_info.cq_size * sizeof(struct i40iw_cqe);
	+ rsize = round_up(rsize, 256);
	+ shadow = I40IW_SHADOW_AREA_SIZE << 3;
	+ status = i40iw_allocate_dma_mem(dev->hw, &iwcq->kmem,
	+ rsize + shadow, 256);
	+ if (status) {
	+ err_code = -ENOMEM;
	+ goto cq_free_resources;
	+ }
	+ ukinfo->cq_base = iwcq->kmem.va;
	+ info.cq_base_pa = iwcq->kmem.pa;
	+ info.shadow_area_pa = info.cq_base_pa + rsize;
	+ ukinfo->shadow_area = (u64 *)((uintptr_t)iwcq->kmem.va + rsize);
	+ }
	+
	+ if (dev->iw_priv_cq_ops->cq_init(cq, &info)) {
	+ i40iw_debug(dev, I40IW_DEBUG_ERR \| I40IW_DEBUG_CQ, "init cq fail\n");
	+ err_code = -EPROTO;
	+ goto cq_free_resources;
	+ }
	+ cqp_request = i40iw_get_cqp_request(&iwdev->cqp, TRUE);
	+ if (!cqp_request) {
	+ err_code = -ENOMEM;
	+ goto cq_free_resources;
	+ }
	+ cqp_info = &cqp_request->info;
	+ cqp_info->cqp_cmd = OP_CQ_CREATE;
	+ cqp_info->post_sq = 1;
	+ cqp_info->in.u.cq_create.cq = cq;
	+ cqp_info->in.u.cq_create.scratch = (uintptr_t)cqp_request;
	+ status = i40iw_handle_cqp_op(iwdev, cqp_request);
	+ if (status) {
	+ i40iw_debug(dev, I40IW_DEBUG_ERR \| I40IW_DEBUG_QP \| I40IW_DEBUG_CQP,
	+ "CQP-OP Create QP fail");
	+ err_code = -EPROTO;
	+ goto cq_free_resources;
	+ }
	+ if (context) {
	+ struct i40iw_create_cq_resp resp;
	+
	+ memset(&resp, 0, sizeof(resp));
	+ resp.cq_id = info.cq_uk_init_info.cq_id;
	+ resp.cq_size = info.cq_uk_init_info.cq_size;
	+ if (ib_copy_to_udata(udata, &resp, sizeof(resp))) {
	+ i40iw_debug(dev, I40IW_DEBUG_ERR \| I40IW_DEBUG_VERBS \|
	+ I40IW_DEBUG_CQ, "copy_to_udata failed\n");
	+ err_code = -EPROTO;
	+ goto cq_destroy;
	+ }
	+ }
	+ i40iw_add_devusecount(iwdev);
	+ return (struct ib_cq *)iwcq;
	+
	+cq_destroy:
	+ i40iw_cq_wq_destroy(iwdev, cq);
	+cq_free_resources:
	+ cq_free_resources(iwdev, iwcq);
	+error:
	+ kfree(iwcq);
	+ return ERR_PTR(err_code);
	+}
	+
	+/**
	+ * i40iw_get_user_access - get hw access from IB access
	+ * @acc: IB access to return hw access
	+ */
	+static inline u16
	+i40iw_get_user_access(int acc)
	+{
	+ u16 access = 0;
	+
	+ access \|= (acc & IB_ACCESS_LOCAL_WRITE) ?
	+ I40IW_ACCESS_FLAGS_LOCALWRITE : 0;
	+ access \|= (acc & IB_ACCESS_REMOTE_WRITE) ?
	+ I40IW_ACCESS_FLAGS_REMOTEWRITE : 0;
	+ access \|= (acc & IB_ACCESS_REMOTE_READ) ?
	+ I40IW_ACCESS_FLAGS_REMOTEREAD : 0;
	+ access \|= (acc & IB_ACCESS_MW_BIND) ? I40IW_ACCESS_FLAGS_BIND_WINDOW : 0;
	+ return access;
	+}
	+
	+/**
	+ * i40iw_free_stag - free stag resource
	+ * @iwdev: iwarp device
	+ * @stag: stag to free
	+ */
	+static void
	+i40iw_free_stag(struct i40iw_device *iwdev, u32 stag)
	+{
	+ u32 stag_idx;
	+
	+ stag_idx = (stag & iwdev->mr_stagmask) >> I40IW_CQPSQ_STAG_IDX_SHIFT;
	+ i40iw_free_resource(iwdev, iwdev->allocated_mrs, stag_idx);
	+ i40iw_rem_devusecount(iwdev);
	+}
	+
	+/**
	+ * i40iw_create_stag - create random stag
	+ * @iwdev: iwarp device
	+ */
	+static u32
	+i40iw_create_stag(struct i40iw_device *iwdev)
	+{
	+ int ret;
	+ u32 stag = 0;
	+ u32 stag_index = 0;
	+ u32 next_stag_index;
	+ u32 driver_key;
	+ u32 random;
	+ u8 consumer_key;
	+
	+ get_random_bytes(&random, sizeof(random));
	+ consumer_key = (u8)random;
	+
	+ driver_key = random & ~iwdev->mr_stagmask;
	+ next_stag_index = (random & iwdev->mr_stagmask) >> 8;
	+ next_stag_index %= iwdev->max_mr;
	+
	+ ret = i40iw_alloc_resource(iwdev,
	+ iwdev->allocated_mrs, iwdev->max_mr,
	+ &stag_index, &next_stag_index);
	+ if (!ret) {
	+ stag = stag_index << I40IW_CQPSQ_STAG_IDX_SHIFT;
	+ stag \|= driver_key;
	+ stag += (u32)consumer_key;
	+ i40iw_add_devusecount(iwdev);
	+ }
	+ return stag;
	+}
	+
	+/**
	+ * i40iw_hw_alloc_stag - cqp command to allocate stag
	+ * @iwdev: iwarp device
	+ * @iwmr: iwarp mr pointer
	+ */
	+static int
	+i40iw_hw_alloc_stag(struct i40iw_device *iwdev,
	+ struct i40iw_mr *iwmr)
	+{
	+ struct i40iw_allocate_stag_info *info;
	+ struct i40iw_cqp_request *cqp_request;
	+ struct cqp_commands_info *cqp_info;
	+ struct i40iw_pd *iwpd = to_iwpd(iwmr->ibmr.pd);
	+ int err = 0;
	+ enum i40iw_status_code status;
	+
	+ cqp_request = i40iw_get_cqp_request(&iwdev->cqp, TRUE);
	+ if (!cqp_request)
	+ return -ENOMEM;
	+
	+ cqp_info = &cqp_request->info;
	+ info = &cqp_info->in.u.alloc_stag.info;
	+ memset(info, 0, sizeof(*info));
	+ info->page_size = PAGE_SIZE;
	+ info->stag_idx = iwmr->stag >> I40IW_CQPSQ_STAG_IDX_SHIFT;
	+ info->pd_id = iwpd->sc_pd.pd_id;
	+ info->total_len = iwmr->length;
	+ info->remote_access = TRUE;
	+ cqp_info->cqp_cmd = OP_ALLOC_STAG;
	+ cqp_info->post_sq = 1;
	+ cqp_info->in.u.alloc_stag.dev = &iwdev->sc_dev;
	+ cqp_info->in.u.alloc_stag.scratch = (uintptr_t)cqp_request;
	+
	+ status = i40iw_handle_cqp_op(iwdev, cqp_request);
	+ if (status) {
	+ err = -ENOMEM;
	+ i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_ERR \| I40IW_DEBUG_VERBS \|
	+ I40IW_DEBUG_CQP, "CQP-OP MR Reg fail");
	+ }
	+ return err;
	+}
	+
	+#ifdef IW_IXL_RDMA_4_9
	+/**
	+ * i40iw_alloc_mr - register stag for fast memory registration
	+ * @pd: ibpd pointer
	+ * @mr_type: memory for stag registrion
	+ * @max_num_sg: max number of pages
	+ */
	+#else
	+/**
	+ * i40iw_alloc_fast_reg_mr - register stag for fast memory registration
	+ * @pd: ibpd pointer
	+ * @max_page_list_len: max number of pages
	+ */
	+#endif
	+static struct ib_mr *
	+#ifdef IW_IXL_RDMA_4_9
	+i40iw_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type, u32 max_num_sg)
	+#else
	+i40iw_alloc_fast_reg_mr(struct ib_pd *pd, int max_page_list_len)
	+#endif
	+{
	+ struct i40iw_pd *iwpd = to_iwpd(pd);
	+ struct i40iw_device *iwdev = to_iwdev(pd->device);
	+ struct i40iw_pble_alloc *palloc;
	+ struct i40iw_pbl *iwpbl;
	+ struct i40iw_mr *iwmr;
	+#ifdef IW_IXL_RDMA_4_9
	+ enum i40iw_status_code status;
	+#endif
	+ u32 stag;
	+ int err_code = -ENOMEM;
	+
	+ iwmr = kzalloc(sizeof(*iwmr), GFP_KERNEL);
	+ if (!iwmr)
	+ return ERR_PTR(-ENOMEM);
	+
	+ stag = i40iw_create_stag(iwdev);
	+ if (!stag) {
	+ err_code = -EOVERFLOW;
	+ goto err;
	+ }
	+ stag &= ~I40IW_CQPSQ_STAG_KEY_MASK;
	+
	+ iwmr->stag = stag;
	+ iwmr->ibmr.rkey = stag;
	+ iwmr->ibmr.lkey = stag;
	+ iwmr->ibmr.pd = pd;
	+ iwmr->ibmr.device = pd->device;
	+ iwpbl = &iwmr->iwpbl;
	+ iwpbl->iwmr = iwmr;
	+#ifdef IW_IXL_RDMA_4_9
	+ iwmr->type = IW_MEMREG_TYPE_MEM;
	+ iwmr->page_cnt = max_num_sg;
	+#else
	+ iwmr->type = IW_MEMREG_TYPE_FMEM;
	+ iwmr->page_cnt = max_page_list_len;
	+#endif
	+
	+ palloc = &iwpbl->pble_alloc;
	+#ifdef IW_IXL_RDMA_4_9
	+ mutex_lock(&iwdev->pbl_mutex);
	+ status = i40iw_get_pble(&iwdev->sc_dev, iwdev->pble_rsrc, palloc, iwmr->page_cnt);
	+ mutex_unlock(&iwdev->pbl_mutex);
	+ if (status)
	+ goto err1;
	+
	+ if (palloc->level != I40IW_LEVEL_1)
	+ goto err2;
	+#endif
	+ err_code = i40iw_hw_alloc_stag(iwdev, iwmr);
	+ if (err_code)
	+#ifdef IW_IXL_RDMA_4_9
	+ goto err2;
	+ iwpbl->pbl_allocated = TRUE;
	+#else
	+ goto err1;
	+ iwpbl->pbl_allocated = FALSE;
	+#endif
	+ i40iw_add_pdusecount(iwpd);
	+ return &iwmr->ibmr;
	+#ifdef IW_IXL_RDMA_4_9
	+err2:
	+ i40iw_free_pble(iwdev->pble_rsrc, palloc);
	+#endif
	+err1:
	+ i40iw_free_stag(iwdev, stag);
	+err:
	+ kfree(iwmr);
	+ return ERR_PTR(err_code);
	+}
	+
	+#ifdef IW_IXL_RDMA_4_9
	+/**
	+ * i40iw_set_page - populate pbl list for fmr
	+ * @ibmr: ib mem to access iwarp mr pointer
	+ * @addr: page dma address fro pbl list
	+ */
	+static int
	+i40iw_set_page(struct ib_mr *ibmr, u64 addr)
	+{
	+ struct i40iw_mr *iwmr = to_iwmr(ibmr);
	+ struct i40iw_pbl *iwpbl = &iwmr->iwpbl;
	+ struct i40iw_pble_alloc *palloc = &iwpbl->pble_alloc;
	+ u64 *pbl;
	+
	+ if (unlikely(iwmr->npages == iwmr->page_cnt))
	+ return -ENOMEM;
	+
	+ pbl = (u64 *)palloc->level1.addr;
	+ pbl[iwmr->npages++] = cpu_to_le64(addr);
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_map_mr_sg - map of sg list for fmr
	+ * @ibmr: ib mem to access iwarp mr pointer
	+ * @sg: scatter gather list for fmr
	+ * @sg_nents: number of sg pages
	+ */
	+static int
	+i40iw_map_mr_sg(struct ib_mr ibmr, struct scatterlist sg,
	+ int sg_nents, unsigned int *sg_offset)
	+{
	+ struct i40iw_mr *iwmr = to_iwmr(ibmr);
	+
	+ iwmr->npages = 0;
	+ return ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, i40iw_set_page);
	+}
	+
	+/**
	+ * i40iw_drain_sq - drain the send queue
	+ * @ibqp: ib qp pointer
	+ */
	+static void
	+i40iw_drain_sq(struct ib_qp *ibqp)
	+{
	+ struct i40iw_qp *iwqp = to_iwqp(ibqp);
	+ struct i40iw_sc_qp *qp = &iwqp->sc_qp;
	+
	+ if (I40IW_RING_MORE_WORK(qp->qp_uk.sq_ring))
	+ wait_for_completion(&iwqp->sq_drained);
	+}
	+
	+/**
	+ * i40iw_drain_rq - drain the receive queue
	+ * @ibqp: ib qp pointer
	+ */
	+static void
	+i40iw_drain_rq(struct ib_qp *ibqp)
	+{
	+ struct i40iw_qp *iwqp = to_iwqp(ibqp);
	+ struct i40iw_sc_qp *qp = &iwqp->sc_qp;
	+
	+ if (I40IW_RING_MORE_WORK(qp->qp_uk.rq_ring))
	+ wait_for_completion(&iwqp->rq_drained);
	+}
	+
	+#else
	+static struct ib_fast_reg_page_list *
	+i40iw_alloc_fast_reg_page_list(struct ib_device *device, int page_list_len)
	+{
	+ struct i40iw_fast_reg_page_list *i40iw_fpl;
	+ struct ib_fast_reg_page_list *ibfpl;
	+ struct i40iw_device *iwdev = to_iwdev(device);
	+ struct i40iw_pble_alloc *palloc;
	+ enum i40iw_status_code status;
	+
	+ i40iw_fpl = kzalloc(sizeof(*i40iw_fpl), GFP_KERNEL);
	+ if (!i40iw_fpl)
	+ return ERR_PTR(-ENOMEM);
	+
	+ ibfpl = &i40iw_fpl->ibfpl;
	+ ibfpl->max_page_list_len = page_list_len;
	+ palloc = &i40iw_fpl->palloc;
	+
	+ mutex_lock(&iwdev->pbl_mutex);
	+ status = i40iw_get_pble(&iwdev->sc_dev, iwdev->pble_rsrc, palloc,
	+ page_list_len);
	+ mutex_unlock(&iwdev->pbl_mutex);
	+ if (status) {
	+ kfree(i40iw_fpl);
	+ return ERR_PTR(-ENOMEM);
	+ }
	+ if (palloc->level != I40IW_LEVEL_1) {
	+ i40iw_free_pble(iwdev->pble_rsrc, palloc);
	+ kfree(i40iw_fpl);
	+ return ERR_PTR(-ENOMEM);
	+ }
	+ ibfpl->page_list = (u64 *)palloc->level1.addr;
	+ i40iw_fpl->iwdev = iwdev;
	+
	+ return &i40iw_fpl->ibfpl;
	+}
	+
	+static void
	+i40iw_free_fast_reg_page_list(struct ib_fast_reg_page_list *ibfpl)
	+{
	+ struct i40iw_fast_reg_page_list *i40iw_fpl = to_i40iw_fr_page_list(ibfpl);
	+ struct i40iw_pble_alloc *palloc;
	+ struct i40iw_device *iwdev;
	+
	+ palloc = &i40iw_fpl->palloc;
	+ iwdev = i40iw_fpl->iwdev;
	+
	+ i40iw_free_pble(iwdev->pble_rsrc, palloc);
	+ kfree(i40iw_fpl);
	+}
	+
	+#endif
	+/**
	+ * i40iw_hwreg_mr - send cqp command for memory registration
	+ * @iwdev: iwarp device
	+ * @iwmr: iwarp mr pointer
	+ * @access: access for MR
	+ */
	+static int
	+i40iw_hwreg_mr(struct i40iw_device iwdev, struct i40iw_mr iwmr, u16 access)
	+{
	+ struct i40iw_pbl *iwpbl = &iwmr->iwpbl;
	+ struct i40iw_reg_ns_stag_info *stag_info;
	+ struct i40iw_pd *iwpd = to_iwpd(iwmr->ibmr.pd);
	+ struct i40iw_pble_alloc *palloc = &iwpbl->pble_alloc;
	+ struct i40iw_cqp_request *cqp_request;
	+ struct cqp_commands_info *cqp_info;
	+ int err = 0;
	+ enum i40iw_status_code status;
	+
	+ cqp_request = i40iw_get_cqp_request(&iwdev->cqp, TRUE);
	+ if (!cqp_request)
	+ return -ENOMEM;
	+
	+ cqp_info = &cqp_request->info;
	+ stag_info = &cqp_info->in.u.mr_reg_non_shared.info;
	+ memset(stag_info, 0, sizeof(*stag_info));
	+ stag_info->va = (void *)(unsigned long)iwpbl->user_base;
	+ stag_info->stag_idx = iwmr->stag >> I40IW_CQPSQ_STAG_IDX_SHIFT;
	+ stag_info->stag_key = (u8)iwmr->stag;
	+ stag_info->total_len = iwmr->length;
	+ stag_info->access_rights = access;
	+ stag_info->pd_id = iwpd->sc_pd.pd_id;
	+ stag_info->addr_type = I40IW_ADDR_TYPE_VA_BASED;
	+
	+ if (iwmr->page_cnt > 1) {
	+ if (palloc->level == I40IW_LEVEL_1) {
	+ stag_info->first_pm_pbl_index = palloc->level1.idx;
	+ stag_info->chunk_size = 1;
	+ } else {
	+ stag_info->first_pm_pbl_index = palloc->level2.root.idx;
	+ stag_info->chunk_size = 3;
	+ }
	+ } else {
	+ stag_info->reg_addr_pa = iwmr->pgaddrmem[0];
	+ }
	+
	+ cqp_info->cqp_cmd = OP_MR_REG_NON_SHARED;
	+ cqp_info->post_sq = 1;
	+ cqp_info->in.u.mr_reg_non_shared.dev = &iwdev->sc_dev;
	+ cqp_info->in.u.mr_reg_non_shared.scratch = (uintptr_t)cqp_request;
	+
	+ status = i40iw_handle_cqp_op(iwdev, cqp_request);
	+ if (status) {
	+ err = -ENOMEM;
	+ i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_ERR \| I40IW_DEBUG_VERBS \|
	+ I40IW_DEBUG_MR \| I40IW_DEBUG_CQP, "CQP-OP MR Reg fail");
	+ }
	+ return err;
	+}
	+
	+/**
	+ * i40iw_reg_phys_mr - register kernel physical memory
	+ * @pd: ibpd pointer
	+ * @phys_buf: array of buffer descriptors defining memory to register
	+ * @num_phys_buf: number of buffer descriptors in the array
	+ * @acc: Access rights
	+ * @iova_start: start of virtual address for physical buffers
	+ */
	+struct ib_mr *
	+#ifdef IW_IXL_RDMA_4_9
	+i40iw_reg_phys_mr(struct ib_pd *pd, u64 addr, u64 size, int acc,
	+ u64 *iova_start)
	+#else
	+i40iw_reg_phys_mr(struct ib_pd pd, struct ib_phys_buf phys_buf,
	+ int num_phys_buf, int acc, u64 *iova_start)
	+#endif
	+{
	+ struct i40iw_pd *iwpd = to_iwpd(pd);
	+ struct i40iw_device *iwdev = to_iwdev(pd->device);
	+ struct i40iw_pbl *iwpbl;
	+ struct i40iw_mr *iwmr;
	+ int ret;
	+ u32 stag;
	+ u16 access = I40IW_ACCESS_FLAGS_LOCALREAD;
	+ enum i40iw_status_code status;
	+
	+#ifndef IW_IXL_RDMA_4_9
	+ if (num_phys_buf > 1)
	+ return ERR_PTR(-ENOTSUP);
	+
	+#endif
	+ iwmr = kzalloc(sizeof(*iwmr), GFP_KERNEL);
	+ if (!iwmr)
	+ return ERR_PTR(-ENOMEM);
	+ iwmr->ibmr.pd = pd;
	+ iwmr->ibmr.device = pd->device;
	+ iwpbl = &iwmr->iwpbl;
	+ iwpbl->iwmr = iwmr;
	+ iwmr->type = IW_MEMREG_TYPE_MEM;
	+ iwpbl->user_base = *iova_start;
	+ stag = i40iw_create_stag(iwdev);
	+ if (!stag) {
	+ ret = -EOVERFLOW;
	+ goto err;
	+ }
	+ access \|= i40iw_get_user_access(acc);
	+ iwmr->stag = stag;
	+ iwmr->ibmr.rkey = stag;
	+ iwmr->ibmr.lkey = stag;
	+ iwmr->page_cnt = 1;
	+#ifdef IW_IXL_RDMA_4_9
	+ iwmr->pgaddrmem[0] = addr;
	+ iwmr->length = size;
	+#else
	+ iwmr->pgaddrmem[0] = phys_buf->addr;
	+ iwmr->length = phys_buf->size;
	+#endif
	+ status = i40iw_hwreg_mr(iwdev, iwmr, access);
	+ if (status) {
	+ i40iw_free_stag(iwdev, stag);
	+ ret = -ENOMEM;
	+ goto err;
	+ }
	+ i40iw_add_pdusecount(iwpd);
	+ return &iwmr->ibmr;
	+err:
	+ kfree(iwmr);
	+ return ERR_PTR(ret);
	+}
	+
	+/**
	+ * i40iw_get_dma_mr - register physical mem
	+ * @pd: ptr of pd
	+ * @acc: access for memory
	+ */
	+static struct ib_mr *
	+i40iw_get_dma_mr(struct ib_pd *pd, int acc)
	+{
	+#ifndef IW_IXL_RDMA_4_9
	+ struct ib_phys_buf phys_buf;
	+ u64 kva = 0;
	+
	+ phys_buf.addr = 0;
	+ phys_buf.size = 0;
	+ return i40iw_reg_phys_mr(pd, &phys_buf, 1, acc, &kva);
	+#else
	+ u64 kva = 0;
	+
	+ return i40iw_reg_phys_mr(pd, 0, 0, acc, &kva);
	+#endif
	+}
	+
	+/**
	+ * i40iw_del_mem_list - Deleting pbl list entries for CQ/QP
	+ * @iwmr: iwmr for IB's user page addresses
	+ * @ucontext: ptr to user context
	+ */
	+static void
	+i40iw_del_memlist(struct i40iw_mr iwmr, struct i40iw_ucontext ucontext)
	+{
	+ struct i40iw_pbl *iwpbl = &iwmr->iwpbl;
	+ unsigned long flags;
	+
	+ switch (iwmr->type) {
	+ case IW_MEMREG_TYPE_CQ:
	+ spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags);
	+ if (!list_empty(&ucontext->cq_reg_mem_list))
	+ list_del(&iwpbl->list);
	+ spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags);
	+ break;
	+ case IW_MEMREG_TYPE_QP:
	+ spin_lock_irqsave(&ucontext->qp_reg_mem_list_lock, flags);
	+ if (!list_empty(&ucontext->qp_reg_mem_list))
	+ list_del(&iwpbl->list);
	+ spin_unlock_irqrestore(&ucontext->qp_reg_mem_list_lock, flags);
	+ break;
	+ default:
	+ break;
	+ }
	+}
	+
	+/**
	+ * i40iw_dereg_mr - deregister mr
	+ * @ib_mr: mr ptr for dereg
	+ */
	+static int
	+i40iw_dereg_mr(struct ib_mr *ib_mr)
	+{
	+ struct ib_pd *ibpd = ib_mr->pd;
	+ struct i40iw_pd *iwpd = to_iwpd(ibpd);
	+ struct i40iw_mr *iwmr = to_iwmr(ib_mr);
	+ struct i40iw_device *iwdev = to_iwdev(ib_mr->device);
	+ struct i40iw_dealloc_stag_info *info;
	+ struct i40iw_pbl *iwpbl = &iwmr->iwpbl;
	+ struct i40iw_pble_alloc *palloc = &iwpbl->pble_alloc;
	+ struct i40iw_cqp_request *cqp_request;
	+ struct cqp_commands_info *cqp_info;
	+ u32 stag_idx;
	+ enum i40iw_status_code status;
	+
	+ if (iwmr->region)
	+ ib_umem_release(iwmr->region);
	+
	+ if (iwmr->type != IW_MEMREG_TYPE_MEM) {
	+ if (ibpd->uobject) {
	+ struct i40iw_ucontext *ucontext;
	+
	+ ucontext = to_ucontext(ibpd->uobject->context);
	+ i40iw_del_memlist(iwmr, ucontext);
	+ }
	+ if (iwpbl->pbl_allocated)
	+ i40iw_free_pble(iwdev->pble_rsrc, palloc);
	+ kfree(iwmr);
	+ return 0;
	+ }
	+ cqp_request = i40iw_get_cqp_request(&iwdev->cqp, TRUE);
	+ if (!cqp_request)
	+ return -ENOMEM;
	+
	+ cqp_info = &cqp_request->info;
	+ info = &cqp_info->in.u.dealloc_stag.info;
	+ memset(info, 0, sizeof(*info));
	+
	+ info->pd_id = CPU_TO_LE32(iwpd->sc_pd.pd_id & 0x00007fff);
	+ info->stag_idx = RS_64_1(ib_mr->rkey, I40IW_CQPSQ_STAG_IDX_SHIFT);
	+ stag_idx = info->stag_idx;
	+ info->mr = TRUE;
	+ if (iwpbl->pbl_allocated)
	+ info->dealloc_pbl = TRUE;
	+
	+ cqp_info->cqp_cmd = OP_DEALLOC_STAG;
	+ cqp_info->post_sq = 1;
	+ cqp_info->in.u.dealloc_stag.dev = &iwdev->sc_dev;
	+ cqp_info->in.u.dealloc_stag.scratch = (uintptr_t)cqp_request;
	+ status = i40iw_handle_cqp_op(iwdev, cqp_request);
	+ if (status)
	+ i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_ERR \| I40IW_DEBUG_VERBS \|
	+ I40IW_DEBUG_CQP \| I40IW_DEBUG_MR,
	+ "CQP-OP dealloc failed for stag_idx = 0x%x\n", stag_idx);
	+ i40iw_rem_pdusecount(iwpd, iwdev);
	+ i40iw_free_stag(iwdev, iwmr->stag);
	+ if (iwpbl->pbl_allocated && iwmr->type != IW_MEMREG_TYPE_QP)
	+ i40iw_free_pble(iwdev->pble_rsrc, palloc);
	+ kfree(iwmr);
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_show_rev
	+ */
	+static ssize_t
	+i40iw_show_rev(struct device dev, struct device_attribute attr, char *buf)
	+{
	+ struct i40iw_ib_device *iwibdev = container_of(dev,
	+ struct i40iw_ib_device,
	+ ibdev.dev);
	+ u32 hw_rev = iwibdev->iwdev->sc_dev.hw_rev;
	+
	+ return sprintf(buf, "%x\n", hw_rev);
	+}
	+
	+/**
	+ * i40iw_show_hca
	+ */
	+static ssize_t
	+i40iw_show_hca(struct device dev, struct device_attribute attr, char *buf)
	+{
	+ return sprintf(buf, "I40IW\n");
	+}
	+
	+/**
	+ * i40iw_show_board
	+ */
	+static ssize_t
	+i40iw_show_board(struct device dev, struct device_attribute attr, char *buf)
	+{
	+ return sprintf(buf, "%.*s\n", 32, "I40IW Board ID");
	+}
	+
	+static DEVICE_ATTR(hw_rev, S_IRUGO, i40iw_show_rev, NULL);
	+static DEVICE_ATTR(hca_type, S_IRUGO, i40iw_show_hca, NULL);
	+static DEVICE_ATTR(board_id, S_IRUGO, i40iw_show_board, NULL);
	+
	+static struct device_attribute *i40iw_dev_attributes[] = {
	+ &dev_attr_hw_rev,
	+ &dev_attr_hca_type,
	+ &dev_attr_board_id
	+};
	+
	+/**
	+ * i40iw_copy_sg_list - copy sg list for qp
	+ * @sg_list: copied into sg_list
	+ * @sgl: copy from sgl
	+ * @num_sges: count of sg entries
	+ */
	+static void
	+i40iw_copy_sg_list(struct i40iw_sge sg_list, struct ib_sge sgl, int num_sges)
	+{
	+ unsigned int i;
	+
	+ for (i = 0; (i < num_sges) && (i < I40IW_MAX_WQ_FRAGMENT_COUNT); i++) {
	+ sg_list[i].tag_off = sgl[i].addr;
	+ sg_list[i].len = sgl[i].length;
	+ sg_list[i].stag = sgl[i].lkey;
	+ }
	+}
	+
	+/**
	+ * i40iw_post_send - kernel application wr
	+ * @ibqp: qp ptr for wr
	+ * @ib_wr: work request ptr
	+ * @bad_wr: return of bad wr if err
	+ */
	+static int
	+i40iw_post_send(struct ib_qp ibqp, struct ib_send_wr ib_wr,
	+ struct ib_send_wr **bad_wr)
	+{
	+ struct i40iw_qp *iwqp;
	+ struct i40iw_qp_uk *ukqp;
	+ struct i40iw_post_sq_info info;
	+ int err = 0;
	+ unsigned long flags;
	+ bool inv_stag;
	+ enum i40iw_status_code ret;
	+
	+ iwqp = (struct i40iw_qp *)ibqp;
	+ ukqp = &iwqp->sc_qp.qp_uk;
	+
	+ spin_lock_irqsave(&iwqp->lock, flags);
	+
	+ if (iwqp->flush_issued) {
	+ err = -EINVAL;
	+ goto out;
	+ }
	+ while (ib_wr) {
	+ inv_stag = FALSE;
	+ memset(&info, 0, sizeof(info));
	+ info.wr_id = (u64)(ib_wr->wr_id);
	+ if ((ib_wr->send_flags & IB_SEND_SIGNALED) \|\| iwqp->sig_all)
	+ info.signaled = TRUE;
	+ if (ib_wr->send_flags & IB_SEND_FENCE)
	+ info.read_fence = TRUE;
	+
	+ switch (ib_wr->opcode) {
	+ case IB_WR_SEND:
	+ /* fall-through */
	+ case IB_WR_SEND_WITH_INV:
	+ if (ib_wr->opcode == IB_WR_SEND) {
	+ if (ib_wr->send_flags & IB_SEND_SOLICITED)
	+ info.op_type = I40IW_OP_TYPE_SEND_SOL;
	+ else
	+ info.op_type = I40IW_OP_TYPE_SEND;
	+ } else {
	+ if (ib_wr->send_flags & IB_SEND_SOLICITED)
	+ info.op_type = I40IW_OP_TYPE_SEND_SOL_INV;
	+ else
	+ info.op_type = I40IW_OP_TYPE_SEND_INV;
	+ info.stag_to_inv = ib_wr->ex.invalidate_rkey;
	+ }
	+
	+ if (ib_wr->send_flags & IB_SEND_INLINE) {
	+ info.op.inline_send.data =
	+ (void *)(unsigned long)ib_wr->sg_list[0].addr;
	+ info.op.inline_send.len = ib_wr->sg_list[0].length;
	+ ret = ukqp->ops.iw_inline_send(ukqp, &info, FALSE);
	+ } else {
	+ info.op.send.num_sges = ib_wr->num_sge;
	+ info.op.send.sg_list = (struct i40iw_sge *)ib_wr->sg_list;
	+ ret = ukqp->ops.iw_send(ukqp, &info, FALSE);
	+ }
	+
	+ if (ret) {
	+ if (ret == I40IW_ERR_QP_TOOMANY_WRS_POSTED)
	+ err = -ENOMEM;
	+ else
	+ err = -EINVAL;
	+ }
	+ break;
	+ case IB_WR_RDMA_WRITE:
	+ info.op_type = I40IW_OP_TYPE_RDMA_WRITE;
	+
	+ if (ib_wr->send_flags & IB_SEND_INLINE) {
	+ info.op.inline_rdma_write.data =
	+ (void *)(unsigned long)ib_wr->sg_list[0].addr;
	+ info.op.inline_rdma_write.len = ib_wr->sg_list[0].length;
	+#ifdef IW_IXL_RDMA_4_9
	+ info.op.inline_rdma_write.rem_addr.tag_off =
	+ rdma_wr(ib_wr)->remote_addr;
	+ info.op.inline_rdma_write.rem_addr.stag = rdma_wr(ib_wr)->rkey;
	+#else
	+ info.op.inline_rdma_write.rem_addr.tag_off =
	+ ib_wr->wr.rdma.remote_addr;
	+ info.op.inline_rdma_write.rem_addr.stag = ib_wr->wr.rdma.rkey;
	+#endif
	+ ret = ukqp->ops.iw_inline_rdma_write(ukqp, &info, FALSE);
	+ } else {
	+ info.op.rdma_write.lo_sg_list = (void *)ib_wr->sg_list;
	+ info.op.rdma_write.num_lo_sges = ib_wr->num_sge;
	+#ifdef IW_IXL_RDMA_4_9
	+ info.op.rdma_write.rem_addr.tag_off =
	+ rdma_wr(ib_wr)->remote_addr;
	+ info.op.rdma_write.rem_addr.stag = rdma_wr(ib_wr)->rkey;
	+#else
	+ info.op.rdma_write.rem_addr.tag_off =
	+ ib_wr->wr.rdma.remote_addr;
	+ info.op.rdma_write.rem_addr.stag = ib_wr->wr.rdma.rkey;
	+#endif
	+ ret = ukqp->ops.iw_rdma_write(ukqp, &info, FALSE);
	+ }
	+
	+ if (ret) {
	+ if (ret == I40IW_ERR_QP_TOOMANY_WRS_POSTED)
	+ err = -ENOMEM;
	+ else
	+ err = -EINVAL;
	+ }
	+ break;
	+ case IB_WR_RDMA_READ_WITH_INV:
	+ inv_stag = TRUE;
	+ /* fall-through */
	+ case IB_WR_RDMA_READ:
	+ if (ib_wr->num_sge > I40IW_MAX_SGE_RD) {
	+ err = -EINVAL;
	+ break;
	+ }
	+ info.op_type = I40IW_OP_TYPE_RDMA_READ;
	+#ifdef IW_IXL_RDMA_4_9
	+ info.op.rdma_read.rem_addr.tag_off = rdma_wr(ib_wr)->remote_addr;
	+ info.op.rdma_read.rem_addr.stag = rdma_wr(ib_wr)->rkey;
	+#else
	+ info.op.rdma_read.rem_addr.tag_off = ib_wr->wr.rdma.remote_addr;
	+ info.op.rdma_read.rem_addr.stag = ib_wr->wr.rdma.rkey;
	+#endif
	+ info.op.rdma_read.lo_addr.tag_off = ib_wr->sg_list->addr;
	+ info.op.rdma_read.lo_addr.stag = ib_wr->sg_list->lkey;
	+ info.op.rdma_read.lo_addr.len = ib_wr->sg_list->length;
	+ ret = ukqp->ops.iw_rdma_read(ukqp, &info, inv_stag, FALSE);
	+ if (ret) {
	+ if (ret == I40IW_ERR_QP_TOOMANY_WRS_POSTED)
	+ err = -ENOMEM;
	+ else
	+ err = -EINVAL;
	+ }
	+ break;
	+ case IB_WR_LOCAL_INV:
	+ info.op_type = I40IW_OP_TYPE_INV_STAG;
	+ info.op.inv_local_stag.target_stag = ib_wr->ex.invalidate_rkey;
	+ ret = ukqp->ops.iw_stag_local_invalidate(ukqp, &info, TRUE);
	+ if (ret)
	+ err = -ENOMEM;
	+ break;
	+#ifdef IW_IXL_RDMA_4_9
	+ case IB_WR_REG_MR:
	+ {
	+ struct i40iw_fast_reg_stag_info info;
	+ struct i40iw_mr *iwmr = to_iwmr(reg_wr(ib_wr)->mr);
	+ struct i40iw_pble_alloc *palloc = &iwmr->iwpbl.pble_alloc;
	+ struct i40iw_sc_dev *dev = &iwqp->iwdev->sc_dev;
	+ int flags = reg_wr(ib_wr)->access;
	+
	+ memset(&info, 0, sizeof(info));
	+ info.access_rights = I40IW_ACCESS_FLAGS_LOCALREAD;
	+ info.access_rights \|= i40iw_get_user_access(flags);
	+ info.stag_key = reg_wr(ib_wr)->key & 0xff;
	+ info.stag_idx = reg_wr(ib_wr)->key >> 8;
	+ info.page_size = reg_wr(ib_wr)->mr->page_size;
	+ info.wr_id = ib_wr->wr_id;
	+
	+ info.addr_type = I40IW_ADDR_TYPE_VA_BASED;
	+ info.va = (void *)(uintptr_t)iwmr->ibmr.iova;
	+ info.total_len = iwmr->ibmr.length;
	+ info.reg_addr_pa = (u64 )palloc->level1.addr;
	+ info.first_pm_pbl_index = palloc->level1.idx;
	+ info.local_fence = ib_wr->send_flags & IB_SEND_FENCE;
	+ info.signaled = ib_wr->send_flags & IB_SEND_SIGNALED;
	+ if (iwmr->npages > I40IW_MIN_PAGES_PER_FMR)
	+ info.chunk_size = 1;
	+
	+ ret = dev->iw_priv_qp_ops->iw_mr_fast_register(&iwqp->sc_qp, &info, TRUE);
	+ if (ret)
	+ err = -ENOMEM;
	+ break;
	+ }
	+#else
	+ case IB_WR_FAST_REG_MR:
	+ {
	+ struct i40iw_fast_reg_page_list *i40iw_fpl =
	+ to_i40iw_fr_page_list(ib_wr->wr.fast_reg.page_list);
	+ struct i40iw_pble_alloc *palloc = &i40iw_fpl->palloc;
	+ struct i40iw_sc_dev *dev = &iwqp->iwdev->sc_dev;
	+ struct i40iw_fast_reg_stag_info info;
	+ int i;
	+ int flags = ib_wr->wr.fast_reg.access_flags;
	+ int page_shift = ib_wr->wr.fast_reg.page_shift;
	+
	+ memset(&info, 0, sizeof(info));
	+ info.access_rights = I40IW_ACCESS_FLAGS_LOCALREAD;
	+ info.access_rights \|= i40iw_get_user_access(flags);
	+ info.stag_key = ib_wr->wr.fast_reg.rkey & 0xff;
	+ info.stag_idx = ib_wr->wr.fast_reg.rkey >> 8;
	+ if (page_shift == 21)
	+ info.page_size = 0x200000; /* 2M page */
	+ else
	+ info.page_size = 0x1000; /* 4K page */
	+ info.wr_id = ib_wr->wr_id;
	+
	+ info.addr_type = I40IW_ADDR_TYPE_VA_BASED;
	+ info.va = (void *)ib_wr->wr.fast_reg.iova_start;
	+ info.total_len = ib_wr->wr.fast_reg.length;
	+ info.reg_addr_pa = (u64 )palloc->level1.addr;
	+ info.first_pm_pbl_index = palloc->level1.idx;
	+ info.local_fence = ib_wr->send_flags & IB_SEND_FENCE;
	+ info.signaled = ib_wr->send_flags & IB_SEND_SIGNALED;
	+ info.chunk_size = 1;
	+
	+ ret = dev->iw_priv_qp_ops->iw_mr_fast_register(&iwqp->sc_qp,
	+ &info, TRUE);
	+ if (ret)
	+ err = -ENOMEM;
	+ for (i = 0; i < ib_wr->wr.fast_reg.page_list_len; i++)
	+ ib_wr->wr.fast_reg.page_list->page_list[i] = cpu_to_le64((u64)
	+ ib_wr->wr.fast_reg.page_list->page_list[i]);
	+ break;
	+ }
	+#endif /* IW_IXL_RDMA_4_9 */
	+ default:
	+ err = -EINVAL;
	+ i40iw_pr_err(" upost_send bad opcode = 0x%x\n",
	+ ib_wr->opcode);
	+ break;
	+ }
	+
	+ if (err)
	+ break;
	+ ib_wr = ib_wr->next;
	+ }
	+
	+out:
	+ if (err)
	+ *bad_wr = ib_wr;
	+ else
	+ ukqp->ops.iw_qp_post_wr(ukqp);
	+ spin_unlock_irqrestore(&iwqp->lock, flags);
	+
	+ return err;
	+}
	+
	+/**
	+ * i40iw_post_recv - post receive wr for kernel application
	+ * @ibqp: ib qp pointer
	+ * @ib_wr: work request for receive
	+ * @bad_wr: bad wr caused an error
	+ */
	+static int
	+i40iw_post_recv(struct ib_qp ibqp, struct ib_recv_wr ib_wr,
	+ struct ib_recv_wr **bad_wr)
	+{
	+ struct i40iw_qp *iwqp;
	+ struct i40iw_qp_uk *ukqp;
	+ struct i40iw_post_rq_info post_recv;
	+ struct i40iw_sge sg_list[I40IW_MAX_WQ_FRAGMENT_COUNT];
	+ int err = 0;
	+ unsigned long flags;
	+ enum i40iw_status_code ret = 0;
	+
	+ if (!ib_wr->sg_list && ib_wr->num_sge > 0) {
	+ i40iw_pr_err(" post_recv err, sg_list NULL, num_sge larger than 0\n");
	+ err = -EDOM;
	+ *bad_wr = ib_wr;
	+ return err;
	+ }
	+ iwqp = (struct i40iw_qp *)ibqp;
	+ ukqp = &iwqp->sc_qp.qp_uk;
	+
	+ memset(&post_recv, 0, sizeof(post_recv));
	+ spin_lock_irqsave(&iwqp->lock, flags);
	+
	+ if (iwqp->flush_issued) {
	+ err = -EINVAL;
	+ goto out;
	+ }
	+ while (ib_wr) {
	+ post_recv.num_sges = ib_wr->num_sge;
	+ post_recv.wr_id = ib_wr->wr_id;
	+ i40iw_copy_sg_list(sg_list, ib_wr->sg_list, ib_wr->num_sge);
	+ post_recv.sg_list = sg_list;
	+ ret = ukqp->ops.iw_post_receive(ukqp, &post_recv);
	+ if (ret) {
	+ i40iw_pr_err(" post_recv err %d\n", ret);
	+ if (ret == I40IW_ERR_QP_TOOMANY_WRS_POSTED)
	+ err = -ENOMEM;
	+ else
	+ err = -EINVAL;
	+ *bad_wr = ib_wr;
	+ goto out;
	+ }
	+ ib_wr = ib_wr->next;
	+ }
	+out:
	+ spin_unlock_irqrestore(&iwqp->lock, flags);
	+ return err;
	+}
	+
	+/**
	+ * i40iw_poll_cq - poll cq for completion (kernel apps)
	+ * @ibcq: cq to poll
	+ * @num_entries: number of entries to poll
	+ * @entry: wr of entry completed
	+ */
	+static int
	+i40iw_poll_cq(struct ib_cq ibcq, int num_entries, struct ib_wc entry)
	+{
	+ struct i40iw_cq *iwcq;
	+ struct i40iw_cq_poll_info cq_poll_info;
	+ struct i40iw_cq_uk *ukcq;
	+ struct i40iw_sc_qp *qp;
	+#ifdef IW_IXL_RDMA_4_9
	+ struct i40iw_qp *iwqp;
	+#endif
	+ int cqe_count = 0;
	+ unsigned long flags;
	+ enum i40iw_status_code ret;
	+
	+ iwcq = (struct i40iw_cq *)ibcq;
	+ ukcq = &iwcq->sc_cq.cq_uk;
	+
	+ spin_lock_irqsave(&iwcq->lock, flags);
	+ while (cqe_count < num_entries) {
	+ ret = ukcq->ops.iw_cq_poll_completion(ukcq, &cq_poll_info);
	+ if (ret == I40IW_ERR_QUEUE_EMPTY) {
	+ break;
	+ } else if (ret == I40IW_ERR_QUEUE_DESTROYED) {
	+ continue;
	+ } else if (ret) {
	+ if (!cqe_count)
	+ cqe_count = -1;
	+ break;
	+ }
	+ entry->wc_flags = 0;
	+ entry->wr_id = cq_poll_info.wr_id;
	+ if (cq_poll_info.error) {
	+ entry->status = IB_WC_WR_FLUSH_ERR;
	+ entry->vendor_err = cq_poll_info.major_err << 16 \| cq_poll_info.minor_err;
	+ } else {
	+ entry->status = IB_WC_SUCCESS;
	+ }
	+
	+ switch (cq_poll_info.op_type) {
	+ case I40IW_OP_TYPE_RDMA_WRITE:
	+ entry->opcode = IB_WC_RDMA_WRITE;
	+ break;
	+ case I40IW_OP_TYPE_RDMA_READ_INV_STAG:
	+ case I40IW_OP_TYPE_RDMA_READ:
	+ entry->opcode = IB_WC_RDMA_READ;
	+ break;
	+ case I40IW_OP_TYPE_SEND_SOL:
	+ case I40IW_OP_TYPE_SEND_SOL_INV:
	+ case I40IW_OP_TYPE_SEND_INV:
	+ case I40IW_OP_TYPE_SEND:
	+ entry->opcode = IB_WC_SEND;
	+ break;
	+ case I40IW_OP_TYPE_REC:
	+ entry->opcode = IB_WC_RECV;
	+ break;
	+ default:
	+ entry->opcode = IB_WC_RECV;
	+ break;
	+ }
	+
	+ entry->ex.imm_data = 0;
	+ qp = (struct i40iw_sc_qp *)cq_poll_info.qp_handle;
	+ entry->qp = (struct ib_qp *)qp->back_qp;
	+ entry->src_qp = cq_poll_info.qp_id;
	+#ifdef IW_IXL_RDMA_4_9
	+ iwqp = (struct i40iw_qp *)qp->back_qp;
	+ if (iwqp->iwarp_state > I40IW_QP_STATE_RTS) {
	+ if (!I40IW_RING_MORE_WORK(qp->qp_uk.sq_ring))
	+ complete(&iwqp->sq_drained);
	+ if (!I40IW_RING_MORE_WORK(qp->qp_uk.rq_ring))
	+ complete(&iwqp->rq_drained);
	+ }
	+#endif
	+ entry->byte_len = cq_poll_info.bytes_xfered;
	+ entry++;
	+ cqe_count++;
	+ }
	+ spin_unlock_irqrestore(&iwcq->lock, flags);
	+ return cqe_count;
	+}
	+
	+/**
	+ * i40iw_req_notify_cq - arm cq kernel application
	+ * @ibcq: cq to arm
	+ * @notify_flags: notofication flags
	+ */
	+static int
	+i40iw_req_notify_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags notify_flags)
	+{
	+ struct i40iw_cq *iwcq;
	+ struct i40iw_cq_uk *ukcq;
	+ unsigned long flags;
	+ enum i40iw_completion_notify cq_notify = IW_CQ_COMPL_EVENT;
	+
	+ iwcq = (struct i40iw_cq *)ibcq;
	+ ukcq = &iwcq->sc_cq.cq_uk;
	+ if (notify_flags == IB_CQ_SOLICITED)
	+ cq_notify = IW_CQ_COMPL_SOLICITED;
	+ spin_lock_irqsave(&iwcq->lock, flags);
	+ ukcq->ops.iw_cq_request_notification(ukcq, cq_notify);
	+ spin_unlock_irqrestore(&iwcq->lock, flags);
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_query_gid - Query port GID
	+ * @ibdev: device pointer from stack
	+ * @port: port number
	+ * @index: Entry index
	+ * @gid: Global ID
	+ */
	+static int
	+i40iw_query_gid(struct ib_device ibdev, u8 port, int index, union ib_gid gid)
	+{
	+ struct i40iw_device *iwdev = to_iwdev(ibdev);
	+
	+ memset(gid->raw, 0, sizeof(gid->raw));
	+ ether_addr_copy(gid->raw, IF_LLADDR(iwdev->ifp));
	+ return 0;
	+}
	+
	+/**
	+ * i40iw_modify_port Modify port properties
	+ * @ibdev: device pointer from stack
	+ * @port: port number
	+ * @port_modify_mask: mask for port modifications
	+ * @props: port properties
	+ */
	+static int
	+i40iw_modify_port(struct ib_device *ibdev, u8 port, int port_modify_mask,
	+ struct ib_port_modify *props)
	+{
	+ return -ENOSYS;
	+}
	+
	+/**
	+ * i40iw_query_pkey - Query partition key
	+ * @ibdev: device pointer from stack
	+ * @port: port number
	+ * @index: index of pkey
	+ * @pkey: pointer to store the pkey
	+ */
	+static int
	+i40iw_query_pkey(struct ib_device ibdev, u8 port, u16 index, u16 pkey)
	+{
	+ *pkey = 0;
	+
	+ return (0);
	+}
	+
	+/**
	+ * i40iw_create_ah - create address handle
	+ * @ibpd: ptr of pd
	+ * @ah_attr: address handle attributes
	+ */
	+static struct ib_ah *
	+i40iw_create_ah(struct ib_pd ibpd, struct ib_ah_attr attr)
	+{
	+ return ERR_PTR(-ENOSYS);
	+}
	+
	+/**
	+ * i40iw_destroy_ah - Destroy address handle
	+ * @ah: pointer to address handle
	+ */
	+static int
	+i40iw_destroy_ah(struct ib_ah *ah)
	+{
	+ return -ENOSYS;
	+}
	+
	+#if __FreeBSD_version >= 1200000
	+static int
	+i40iw_port_immutable(struct ib_device *ibdev, u8 port_num,
	+ struct ib_port_immutable *immutable)
	+{
	+ struct ib_port_attr attr;
	+ int err;
	+
	+ err = i40iw_query_port(ibdev, port_num, &attr);
	+
	+ if (err)
	+ return err;
	+
	+ immutable->pkey_tbl_len = attr.pkey_tbl_len;
	+ immutable->gid_tbl_len = attr.gid_tbl_len;
	+ immutable->core_cap_flags = RDMA_CORE_PORT_IWARP;
	+
	+ return 0;
	+}
	+#endif
	+#ifdef IW_IXL_RDMA_4_9
	+
	+static void
	+i40iw_get_dev_fw_str(struct ib_device dev, char str,
	+ size_t str_len)
	+{
	+ u32 firmware_version = I40IW_FW_VERSION;
	+
	+ snprintf(str, str_len, "%u.%u", firmware_version,
	+ (firmware_version & 0x000000ff));
	+}
	+#endif
	+
	+/**
	+ * i40iw_init_rdma_device - initialization of iwarp device
	+ * @iwdev: iwarp device
	+ */
	+static struct i40iw_ib_device *
	+i40iw_init_rdma_device(struct i40iw_device *iwdev)
	+{
	+ struct i40iw_ib_device *iwibdev;
	+
	+ iwibdev = (struct i40iw_ib_device )ib_alloc_device(sizeof(iwibdev));
	+ if (!iwibdev) {
	+ device_printf(iwdev->ldev->dev, "iwdev == NULL\n");
	+ return NULL;
	+ }
	+ strlcpy(iwibdev->ibdev.name, "ixliw%d", IB_DEVICE_NAME_MAX);
	+ iwibdev->ibdev.owner = THIS_MODULE;
	+ iwdev->iwibdev = iwibdev;
	+ iwibdev->iwdev = iwdev;
	+
	+ iwibdev->ibdev.node_type = RDMA_NODE_RNIC;
	+ ether_addr_copy((u8 *)&iwibdev->ibdev.node_guid, IF_LLADDR(iwdev->ifp));
	+
	+ iwibdev->ibdev.uverbs_cmd_mask =
	+ (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) \|
	+ (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) \|
	+ (1ull << IB_USER_VERBS_CMD_QUERY_PORT) \|
	+ (1ull << IB_USER_VERBS_CMD_ALLOC_PD) \|
	+ (1ull << IB_USER_VERBS_CMD_DEALLOC_PD) \|
	+ (1ull << IB_USER_VERBS_CMD_REG_MR) \|
	+ (1ull << IB_USER_VERBS_CMD_DEREG_MR) \|
	+ (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) \|
	+ (1ull << IB_USER_VERBS_CMD_CREATE_CQ) \|
	+ (1ull << IB_USER_VERBS_CMD_DESTROY_CQ) \|
	+ (1ull << IB_USER_VERBS_CMD_REQ_NOTIFY_CQ) \|
	+ (1ull << IB_USER_VERBS_CMD_CREATE_QP) \|
	+ (1ull << IB_USER_VERBS_CMD_MODIFY_QP) \|
	+ (1ull << IB_USER_VERBS_CMD_QUERY_QP) \|
	+ (1ull << IB_USER_VERBS_CMD_POLL_CQ) \|
	+ (1ull << IB_USER_VERBS_CMD_CREATE_AH) \|
	+ (1ull << IB_USER_VERBS_CMD_DESTROY_AH) \|
	+ (1ull << IB_USER_VERBS_CMD_DESTROY_QP) \|
	+ (1ull << IB_USER_VERBS_CMD_POST_RECV) \|
	+ (1ull << IB_USER_VERBS_CMD_POST_SEND);
	+ iwibdev->ibdev.phys_port_cnt = 1;
	+ iwibdev->ibdev.num_comp_vectors = 1;
	+ iwibdev->ibdev.dma_device = device_get_parent(iwdev->ldev->dev);
	+ iwibdev->ibdev.dev.parent = iwdev->ldev->dev;
	+ iwibdev->ibdev.query_port = i40iw_query_port;
	+ iwibdev->ibdev.modify_port = i40iw_modify_port;
	+ iwibdev->ibdev.query_pkey = i40iw_query_pkey;
	+ iwibdev->ibdev.query_gid = i40iw_query_gid;
	+ iwibdev->ibdev.alloc_pd = i40iw_alloc_pd;
	+ iwibdev->ibdev.dealloc_pd = i40iw_dealloc_pd;
	+ iwibdev->ibdev.create_qp = i40iw_create_qp;
	+ iwibdev->ibdev.modify_qp = i40iw_modify_qp;
	+ iwibdev->ibdev.query_qp = i40iw_query_qp;
	+ iwibdev->ibdev.destroy_qp = i40iw_destroy_qp;
	+ iwibdev->ibdev.create_cq = i40iw_create_cq;
	+ iwibdev->ibdev.destroy_cq = i40iw_destroy_cq;
	+ iwibdev->ibdev.get_dma_mr = i40iw_get_dma_mr;
	+#ifndef IW_IXL_RDMA_4_9
	+ iwibdev->ibdev.reg_phys_mr = i40iw_reg_phys_mr;
	+#endif
	+ iwibdev->ibdev.dereg_mr = i40iw_dereg_mr;
	+ iwibdev->ibdev.query_device = i40iw_query_device;
	+ iwibdev->ibdev.create_ah = i40iw_create_ah;
	+ iwibdev->ibdev.destroy_ah = i40iw_destroy_ah;
	+#ifdef IW_IXL_RDMA_4_9
	+ iwibdev->ibdev.drain_sq = i40iw_drain_sq;
	+ iwibdev->ibdev.drain_rq = i40iw_drain_rq;
	+ iwibdev->ibdev.alloc_mr = i40iw_alloc_mr;
	+ iwibdev->ibdev.map_mr_sg = i40iw_map_mr_sg;
	+ iwibdev->ibdev.get_dev_fw_str = i40iw_get_dev_fw_str;
	+#else
	+ iwibdev->ibdev.alloc_fast_reg_mr = i40iw_alloc_fast_reg_mr;
	+ iwibdev->ibdev.alloc_fast_reg_page_list = i40iw_alloc_fast_reg_page_list;
	+ iwibdev->ibdev.free_fast_reg_page_list = i40iw_free_fast_reg_page_list;
	+#endif
	+#if __FreeBSD_version >= 1200000
	+ iwibdev->ibdev.get_port_immutable = i40iw_port_immutable;
	+#endif
	+
	+ iwibdev->ibdev.iwcm = kzalloc(sizeof(*iwibdev->ibdev.iwcm), GFP_KERNEL);
	+ if (!iwibdev->ibdev.iwcm) {
	+ ib_dealloc_device(&iwibdev->ibdev);
	+ device_printf(iwdev->ldev->dev, "iwcm == NULL\n");
	+ return NULL;
	+ }
	+ iwibdev->ibdev.iwcm->add_ref = i40iw_add_ref;
	+ iwibdev->ibdev.iwcm->rem_ref = i40iw_rem_ref;
	+ iwibdev->ibdev.iwcm->get_qp = i40iw_get_qp;
	+ iwibdev->ibdev.iwcm->connect = i40iw_connect;
	+ iwibdev->ibdev.iwcm->accept = i40iw_accept;
	+ iwibdev->ibdev.iwcm->reject = i40iw_reject;
	+#if __FreeBSD_version >= 1100000 && !defined(IW_IXL_RDMA_4_9)
	+ iwibdev->ibdev.iwcm->create_listen_ep = i40iw_create_listen;
	+ iwibdev->ibdev.iwcm->destroy_listen_ep = i40iw_destroy_listen;
	+#else
	+ iwibdev->ibdev.iwcm->create_listen = i40iw_create_listen;
	+ iwibdev->ibdev.iwcm->destroy_listen = i40iw_destroy_listen;
	+#endif /* IW_IXL_FREEBSD11 */
	+#ifdef IW_IXL_RDMA_4_9
	+ memcpy(iwibdev->ibdev.iwcm->ifname, IF_LLADDR(iwdev->ifp),
	+ sizeof(iwibdev->ibdev.iwcm->ifname));
	+#endif
	+ iwibdev->ibdev.poll_cq = i40iw_poll_cq;
	+ iwibdev->ibdev.req_notify_cq = i40iw_req_notify_cq;
	+ iwibdev->ibdev.post_send = i40iw_post_send;
	+ iwibdev->ibdev.post_recv = i40iw_post_recv;
	+
	+ return iwibdev;
	+}
	+
	+/**
	+ * i40iw_port_ibevent - indicate port event
	+ * @iwdev: iwarp device
	+ */
	+void
	+i40iw_port_ibevent(struct i40iw_device *iwdev)
	+{
	+ struct i40iw_ib_device *iwibdev = iwdev->iwibdev;
	+ struct ib_event event;
	+
	+ event.device = &iwibdev->ibdev;
	+ event.element.port_num = 1;
	+ event.event = iwdev->iw_status ? IB_EVENT_PORT_ACTIVE : IB_EVENT_PORT_ERR;
	+ ib_dispatch_event(&event);
	+}
	+
	+/**
	+ * i40iw_unregister_rdma_device - unregister of iwarp from IB
	+ * @iwibdev: rdma device ptr
	+ */
	+static void
	+i40iw_unregister_rdma_device(struct i40iw_ib_device *iwibdev)
	+{
	+ int i;
	+
	+ for (i = 0; i < ARRAY_SIZE(i40iw_dev_attributes); ++i)
	+ device_remove_file(&iwibdev->ibdev.dev,
	+ i40iw_dev_attributes[i]);
	+ ib_unregister_device(&iwibdev->ibdev);
	+}
	+
	+/**
	+ * i40iw_destroy_rdma_device - destroy rdma device and free resources
	+ * @iwibdev: IB device ptr
	+ */
	+void
	+i40iw_destroy_rdma_device(struct i40iw_ib_device *iwibdev)
	+{
	+ if (!iwibdev)
	+ return;
	+
	+ i40iw_unregister_rdma_device(iwibdev);
	+ kfree(iwibdev->ibdev.iwcm);
	+ iwibdev->ibdev.iwcm = NULL;
	+ wait_event_timeout(iwibdev->iwdev->close_wq,
	+ !atomic_read(&iwibdev->iwdev->use_count),
	+ I40IW_EVENT_TIMEOUT);
	+ ib_dealloc_device(&iwibdev->ibdev);
	+}
	+
	+/**
	+ * i40iw_register_rdma_device - register iwarp device to IB
	+ * @iwdev: iwarp device
	+ */
	+int
	+i40iw_register_rdma_device(struct i40iw_device *iwdev)
	+{
	+ struct i40iw_ib_device *iwibdev;
	+ int i, ret;
	+
	+ iwdev->iwibdev = i40iw_init_rdma_device(iwdev);
	+ if (!iwdev->iwibdev)
	+ return -ENOMEM;
	+ iwibdev = iwdev->iwibdev;
	+
	+ ret = ib_register_device(&iwibdev->ibdev, NULL);
	+ if (ret)
	+ goto error;
	+
	+ for (i = 0; i < ARRAY_SIZE(i40iw_dev_attributes); ++i) {
	+ ret =
	+ device_create_file(&iwibdev->ibdev.dev,
	+ i40iw_dev_attributes[i]);
	+ if (ret) {
	+ while (i > 0) {
	+ i--;
	+ device_remove_file(&iwibdev->ibdev.dev,
	+ i40iw_dev_attributes[i]);
	+ }
	+ ib_unregister_device(&iwibdev->ibdev);
	+ goto error;
	+ }
	+ }
	+ return 0;
	+error:
	+ kfree(iwdev->iwibdev->ibdev.iwcm);
	+ iwdev->iwibdev->ibdev.iwcm = NULL;
	+ ib_dealloc_device(&iwdev->iwibdev->ibdev);
	+ return ret;
	+}
	Index: sys/modules/Makefile
	===================================================================
	--- sys/modules/Makefile
	+++ sys/modules/Makefile
	@@ -205,6 +205,7 @@
	${_ixgb} \
	${_ixl} \
	${_ixlv} \
	+ ${_iw_ixl} \
	jme \
	joy \
	kbdmux \
	@@ -707,6 +708,7 @@
	_ioat= ioat
	_ixl= ixl
	_ixlv= ixlv
	+_iw_ixl= ${_ixl}/iw_ixl
	_linux64= linux64
	_linux_common= linux_common
	_ntb= ntb
	Index: sys/modules/ixl/iw_ixl/Makefile
	===================================================================
	--- /dev/null
	+++ sys/modules/ixl/iw_ixl/Makefile
	@@ -0,0 +1,34 @@
	+.include <bsd.own.mk>
	+
	+IXL_DIR = ${.CURDIR}/../../../dev/ixl
	+OFED_INC_DIR = ${.CURDIR}/../../../ofed/include
	+.PATH: ${IXL_DIR}/iwarp
	+
	+KMOD= iw_ixl
	+SRCS= iw_ixl.c iw_ixl_cm.c iw_ixl_utils.c iw_ixl_hw.c iw_ixl_pble.c
	+SRCS+= iw_ixl_verbs.c iw_ixl_linux_genalloc.c
	+SRCS+= vnode_if.h device_if.h bus_if.h opt_inet.h pci_if.h opt_inet6.h
	+
	+# Shared source
	+SRCS += i40iw_ctrl.c i40iw_uk.c i40iw_hmc.c i40iw_puda.c
	+
	+CFLAGS+= -I${IXL_DIR} -I${OFED_INC_DIR}
	+CFLAGS+= -DINET -DINET6
	+
	+# check FreeBSD version
	+KERNVER != uname -K
	+VERS_CHECK != test $(KERNVER) -ge 1100000 ; echo $$?
	+RDMA_49 != test $(KERNVER) -ge 1200054 ; echo $$?
	+
	+.if !$(RDMA_49)
	+CFLAGS += -DIW_IXL_RDMA_4_9 -DVIMAGE
	+CFLAGS += -I${OFED_INC_DIR}/uapi
	+.endif
	+
	+.if !$(VERS_CHECK)
	+CFLAGS += -I/usr/src/sys/compat/linuxkpi/common/include
	+.endif
	+
	+# Debug messages
	+CFLAGS += -DIW_IXL_DEBUG
	+.include <bsd.kmod.mk>

File Metadata

Mime Type: text/plain
Expires: Sun, Oct 4, 3:09 AM (17 h, 41 m)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 30144978
Default Alt Text: D11378.1791083393.diff (872 KB)

D11378.1791083393.diffNo OneTemporaryActions

D11378.1791083393.diffView Options

File Metadata

Event Timeline

D11378.1791083393.diff
No OneTemporary
Actions

D11378.1791083393.diff
View Options