Page MenuHomeFreeBSD
Files F148053907

D36254.1785230803.diff
No OneTemporary
Actions

Size
564 KB
Referenced Files
None
Subscribers
None

D36254.1785230803.diff
View Options

This file is larger than 256 KB, so syntax highlighting was skipped.
Index: share/man/man4/qat.4
===================================================================
--- share/man/man4/qat.4
+++ share/man/man4/qat.4
@@ -1,7 +1,7 @@
.\" SPDX-License-Identifier: BSD-3-Clause
.\" Copyright(c) 2007-2022 Intel Corporation
.\" $FreeBSD$
-.Dd June 30, 2022
+.Dd September 1, 2022
.Dt QAT 4
.Os
.Sh NAME
@@ -30,6 +30,8 @@
.It
qat_dh895xcc_fw_load="YES"
.It
+qat_4xxx_fw_load="YES"
+.It
qat_load="YES"
.El
.Sh DESCRIPTION
@@ -53,6 +55,8 @@
Intel (R) Communications Chipset 8925 to 8955 Series
.It
Intel (R) Atom P5300 processor product family
+.It
+Intel (R) QAT 4xxx Series
.El
.Pp
The
Index: sys/contrib/dev/qat/qat_4xxx.bin
===================================================================
--- /dev/null
+++ sys/contrib/dev/qat/qat_4xxx.bin
@@ -0,0 +1 @@
+Placeholder for QAT 4xxx Series devices firmware.
Index: sys/contrib/dev/qat/qat_4xxx_mmp.bin
===================================================================
--- /dev/null
+++ sys/contrib/dev/qat/qat_4xxx_mmp.bin
@@ -0,0 +1 @@
+Placeholder for QAT 4xxx Series devices firmware.
Index: sys/dev/qat/include/common/adf_accel_devices.h
===================================================================
--- sys/dev/qat/include/common/adf_accel_devices.h
+++ sys/dev/qat/include/common/adf_accel_devices.h
@@ -19,6 +19,7 @@
#define ADF_200XXVF_DEVICE_NAME "200xxvf"
#define ADF_C4XXX_DEVICE_NAME "c4xxx"
#define ADF_C4XXXVF_DEVICE_NAME "c4xxxvf"
+#define ADF_4XXX_DEVICE_NAME "4xxx"
#define ADF_DH895XCC_PCI_DEVICE_ID 0x435
#define ADF_DH895XCCIOV_PCI_DEVICE_ID 0x443
#define ADF_C62X_PCI_DEVICE_ID 0x37c8
@@ -31,8 +32,11 @@
#define ADF_D15XXIOV_PCI_DEVICE_ID 0x6f55
#define ADF_C4XXX_PCI_DEVICE_ID 0x18a0
#define ADF_C4XXXIOV_PCI_DEVICE_ID 0x18a1
+#define ADF_4XXX_PCI_DEVICE_ID 0x4940
#define IS_QAT_GEN3(ID) ({ (ID == ADF_C4XXX_PCI_DEVICE_ID); })
+#define IS_QAT_GEN4(ID) ({ (ID == ADF_4XXX_PCI_DEVICE_ID); })
+#define IS_QAT_GEN3_OR_GEN4(ID) (IS_QAT_GEN3(ID) || IS_QAT_GEN4(ID))
#define ADF_VF2PF_SET_SIZE 32
#define ADF_MAX_VF2PF_SET 4
#define ADF_VF2PF_SET_OFFSET(set_nr) ((set_nr)*ADF_VF2PF_SET_SIZE)
@@ -50,7 +54,7 @@
#define ADF_PCI_MAX_BARS 3
#define ADF_DEVICE_NAME_LENGTH 32
#define ADF_ETR_MAX_RINGS_PER_BANK 16
-#define ADF_MAX_MSIX_VECTOR_NAME 16
+#define ADF_MAX_MSIX_VECTOR_NAME 32
#define ADF_DEVICE_NAME_PREFIX "qat_"
#define ADF_STOP_RETRY 50
#define ADF_NUM_THREADS_PER_AE (8)
@@ -58,7 +62,6 @@
#define ADF_NUM_PKE_STRAND (2)
#define ADF_AE_STRAND0_THREAD (8)
#define ADF_AE_STRAND1_THREAD (9)
-#define ADF_NUM_HB_CNT_PER_AE (ADF_NUM_THREADS_PER_AE + ADF_NUM_PKE_STRAND)
#define ADF_CFG_NUM_SERVICES 4
#define ADF_SRV_TYPE_BIT_LEN 3
#define ADF_SRV_TYPE_MASK 0x7
@@ -75,6 +78,8 @@
#define GET_SRV_TYPE(ena_srv_mask, srv) \
(((ena_srv_mask) >> (ADF_SRV_TYPE_BIT_LEN * (srv))) & ADF_SRV_TYPE_MASK)
+#define GET_CSR_OPS(accel_dev) (&(accel_dev)->hw_device->csr_info.csr_ops)
+
#define ADF_DEFAULT_RING_TO_SRV_MAP \
(CRYPTO | CRYPTO << ADF_CFG_SERV_RING_PAIR_1_SHIFT | \
NA << ADF_CFG_SERV_RING_PAIR_2_SHIFT | \
@@ -156,7 +161,9 @@
ADF_ACCEL_SERVICE_NULL = 0,
ADF_ACCEL_INLINE_CRYPTO = 1,
ADF_ACCEL_CRYPTO = 2,
- ADF_ACCEL_COMPRESSION = 4
+ ADF_ACCEL_COMPRESSION = 4,
+ ADF_ACCEL_ASYM = 8,
+ ADF_ACCEL_ADMIN = 16
};
struct adf_ae_info {
@@ -182,6 +189,7 @@
u32 dc_ae_msk;
u8 num_cy_au;
u8 num_dc_au;
+ u8 num_asym_au;
u8 num_inline_au;
struct adf_accel_unit *au;
const struct adf_ae_info *ae_info;
@@ -231,6 +239,60 @@
u32 mailbox_offset;
} __packed;
+struct adf_hw_csr_ops {
+ u64 (*build_csr_ring_base_addr)(bus_addr_t addr, u32 size);
+ u32 (*read_csr_ring_head)(struct resource *csr_base_addr,
+ u32 bank,
+ u32 ring);
+ void (*write_csr_ring_head)(struct resource *csr_base_addr,
+ u32 bank,
+ u32 ring,
+ u32 value);
+ u32 (*read_csr_ring_tail)(struct resource *csr_base_addr,
+ u32 bank,
+ u32 ring);
+ void (*write_csr_ring_tail)(struct resource *csr_base_addr,
+ u32 bank,
+ u32 ring,
+ u32 value);
+ u32 (*read_csr_e_stat)(struct resource *csr_base_addr, u32 bank);
+ void (*write_csr_ring_config)(struct resource *csr_base_addr,
+ u32 bank,
+ u32 ring,
+ u32 value);
+ void (*write_csr_ring_base)(struct resource *csr_base_addr,
+ u32 bank,
+ u32 ring,
+ bus_addr_t addr);
+ void (*write_csr_int_flag)(struct resource *csr_base_addr,
+ u32 bank,
+ u32 value);
+ void (*write_csr_int_srcsel)(struct resource *csr_base_addr, u32 bank);
+ void (*write_csr_int_col_en)(struct resource *csr_base_addr,
+ u32 bank,
+ u32 value);
+ void (*write_csr_int_col_ctl)(struct resource *csr_base_addr,
+ u32 bank,
+ u32 value);
+ void (*write_csr_int_flag_and_col)(struct resource *csr_base_addr,
+ u32 bank,
+ u32 value);
+ u32 (*read_csr_ring_srv_arb_en)(struct resource *csr_base_addr,
+ u32 bank);
+ void (*write_csr_ring_srv_arb_en)(struct resource *csr_base_addr,
+ u32 bank,
+ u32 value);
+};
+
+struct adf_hw_csr_info {
+ struct adf_hw_csr_ops csr_ops;
+ u32 csr_addr_offset;
+ u32 ring_bundle_size;
+ u32 bank_int_flag_clear_mask;
+ u32 num_rings_per_int_srcsel;
+ u32 arb_enable_mask;
+};
+
struct adf_cfg_device_data;
struct adf_accel_dev;
struct adf_etr_data;
@@ -282,8 +344,10 @@
void (*exit_arb)(struct adf_accel_dev *accel_dev);
void (*get_arb_mapping)(struct adf_accel_dev *accel_dev,
const uint32_t **cfg);
+ int (*init_device)(struct adf_accel_dev *accel_dev);
int (*get_heartbeat_status)(struct adf_accel_dev *accel_dev);
uint32_t (*get_ae_clock)(struct adf_hw_device_data *self);
+ uint32_t (*get_hb_clock)(struct adf_hw_device_data *self);
void (*disable_iov)(struct adf_accel_dev *accel_dev);
void (*configure_iov_threads)(struct adf_accel_dev *accel_dev,
bool enable);
@@ -298,6 +362,8 @@
void (*restore_device)(struct adf_accel_dev *accel_dev);
uint32_t (*get_obj_cfg_ae_mask)(struct adf_accel_dev *accel_dev,
enum adf_accel_unit_services services);
+ enum adf_accel_unit_services (
+ *get_service_type)(struct adf_accel_dev *accel_dev, s32 obj_num);
int (*add_pke_stats)(struct adf_accel_dev *accel_dev);
void (*remove_pke_stats)(struct adf_accel_dev *accel_dev);
int (*add_misc_error)(struct adf_accel_dev *accel_dev);
@@ -311,6 +377,14 @@
enum adf_accel_unit_services services);
void (*pre_reset)(struct adf_accel_dev *accel_dev);
void (*post_reset)(struct adf_accel_dev *accel_dev);
+ void (*set_msix_rttable)(struct adf_accel_dev *accel_dev);
+ void (*get_ring_svc_map_data)(int ring_pair_index,
+ u16 ring_to_svc_map,
+ u8 *serv_type,
+ int *ring_index,
+ int *num_rings_per_srv,
+ int bundle_num);
+ struct adf_hw_csr_info csr_info;
const char *fw_name;
const char *fw_mmp_name;
bool reset_ack;
@@ -320,7 +394,10 @@
uint16_t accel_mask;
u32 aerucm_mask;
u32 ae_mask;
+ u32 admin_ae_mask;
u32 service_mask;
+ u32 service_to_load_mask;
+ u32 heartbeat_ctr_num;
uint16_t tx_rings_mask;
uint8_t tx_rx_gap;
uint8_t num_banks;
Index: sys/dev/qat/include/common/adf_cfg.h
===================================================================
--- sys/dev/qat/include/common/adf_cfg.h
+++ sys/dev/qat/include/common/adf_cfg.h
@@ -76,4 +76,14 @@
int adf_cfg_restore_section(struct adf_accel_dev *accel_dev,
struct adf_cfg_section *section);
void adf_cfg_keyval_del_all(struct list_head *head);
+
+static inline int
+adf_cy_inst_cross_banks(struct adf_accel_dev *accel_dev)
+{
+ if (accel_dev->hw_device->num_rings_per_bank == 2)
+ return 1;
+ else
+ return 0;
+}
+
#endif
Index: sys/dev/qat/include/common/adf_cfg_common.h
===================================================================
--- sys/dev/qat/include/common/adf_cfg_common.h
+++ sys/dev/qat/include/common/adf_cfg_common.h
@@ -27,7 +27,7 @@
#define ADF_MAX_ACCELENGINES 12
#define ADF_CFG_STORAGE_ENABLED 1
#define ADF_DEVS_ARRAY_SIZE BITS_TO_LONGS(ADF_MAX_DEVICES)
-#define ADF_SSM_WDT_PKE_DEFAULT_VALUE 0x3000000
+#define ADF_GEN2_SSM_WDT_PKE_DEFAULT_VALUE 0x3000000
#define ADF_WDT_TIMER_SYM_COMP_MS 3
#define ADF_MIN_HB_TIMER_MS 100
#define ADF_CFG_MAX_NUM_OF_SECTIONS 16
@@ -87,7 +87,8 @@
DEV_200XX,
DEV_200XXVF,
DEV_C4XXX,
- DEV_C4XXXVF
+ DEV_C4XXXVF,
+ DEV_4XXX
};
enum adf_cfg_fw_image_type {
@@ -158,6 +159,7 @@
/* contains all the info about rings */
struct adf_cfg_ring **rings;
u16 in_use;
+ u16 max_cfg_svc_num;
};
struct adf_cfg_instance {
Index: sys/dev/qat/include/common/adf_cfg_strings.h
===================================================================
--- sys/dev/qat/include/common/adf_cfg_strings.h
+++ sys/dev/qat/include/common/adf_cfg_strings.h
@@ -22,6 +22,8 @@
#define ADF_RING_DC_RX "RingRx"
#define ADF_ETRMGR_BANK "Bank"
#define ADF_RING_BANK_NUM "BankNumber"
+#define ADF_RING_BANK_NUM_ASYM "BankNumberAsym"
+#define ADF_RING_BANK_NUM_SYM "BankNumberSym"
#define ADF_CY "Cy"
#define ADF_DC "Dc"
#define ADF_DC_EXTENDED_FEATURES "Device_DcExtendedFeatures"
@@ -112,6 +114,8 @@
#define ADF_CY_CORE_AFFINITY_FORMAT ADF_CY "%d" ADF_ETRMGR_CORE_AFFINITY
#define ADF_DC_CORE_AFFINITY_FORMAT ADF_DC "%d" ADF_ETRMGR_CORE_AFFINITY
#define ADF_CY_BANK_NUM_FORMAT ADF_CY "%d" ADF_RING_BANK_NUM
+#define ADF_CY_ASYM_BANK_NUM_FORMAT ADF_CY "%d" ADF_RING_BANK_NUM_ASYM
+#define ADF_CY_SYM_BANK_NUM_FORMAT ADF_CY "%d" ADF_RING_BANK_NUM_SYM
#define ADF_DC_BANK_NUM_FORMAT ADF_DC "%d" ADF_RING_BANK_NUM
#define ADF_CY_ASYM_TX_FORMAT ADF_CY "%d" ADF_RING_ASYM_TX
#define ADF_CY_SYM_TX_FORMAT ADF_CY "%d" ADF_RING_SYM_TX
Index: sys/dev/qat/include/common/adf_common_drv.h
===================================================================
--- sys/dev/qat/include/common/adf_common_drv.h
+++ sys/dev/qat/include/common/adf_common_drv.h
@@ -203,10 +203,14 @@
void adf_exit_arb(struct adf_accel_dev *accel_dev);
void adf_disable_arb(struct adf_accel_dev *accel_dev);
void adf_update_ring_arb(struct adf_etr_ring_data *ring);
-void
-adf_enable_ring_arb(void *csr_addr, unsigned int bank_nr, unsigned int mask);
-void
-adf_disable_ring_arb(void *csr_addr, unsigned int bank_nr, unsigned int mask);
+void adf_enable_ring_arb(struct adf_accel_dev *accel_dev,
+ void *csr_addr,
+ unsigned int bank_nr,
+ unsigned int mask);
+void adf_disable_ring_arb(struct adf_accel_dev *accel_dev,
+ void *csr_addr,
+ unsigned int bank_nr,
+ unsigned int mask);
int adf_set_ssm_wdtimer(struct adf_accel_dev *accel_dev);
struct adf_accel_dev *adf_devmgr_get_dev_by_bdf(struct adf_pci_address *addr);
struct adf_accel_dev *adf_devmgr_get_dev_by_pci_bus(u8 bus);
@@ -238,9 +242,7 @@
int qat_hal_init(struct adf_accel_dev *accel_dev);
void qat_hal_deinit(struct icp_qat_fw_loader_handle *handle);
-void qat_hal_start(struct icp_qat_fw_loader_handle *handle,
- unsigned char ae,
- unsigned int ctx_mask);
+int qat_hal_start(struct icp_qat_fw_loader_handle *handle);
void qat_hal_stop(struct icp_qat_fw_loader_handle *handle,
unsigned char ae,
unsigned int ctx_mask);
Index: sys/dev/qat/include/common/adf_gen2_hw_data.h
===================================================================
--- /dev/null
+++ sys/dev/qat/include/common/adf_gen2_hw_data.h
@@ -0,0 +1,172 @@
+/* SPDX-License-Identifier: BSD-3-Clause */
+/* Copyright(c) 2021 Intel Corporation */
+/* $FreeBSD$ */
+#ifndef ADF_GEN2_HW_DATA_H_
+#define ADF_GEN2_HW_DATA_H_
+
+#include "adf_accel_devices.h"
+#include "adf_cfg_common.h"
+
+/* Transport access */
+#define ADF_BANK_INT_SRC_SEL_MASK_0 0x4444444CUL
+#define ADF_BANK_INT_SRC_SEL_MASK_X 0x44444444UL
+#define ADF_RING_CSR_RING_CONFIG 0x000
+#define ADF_RING_CSR_RING_LBASE 0x040
+#define ADF_RING_CSR_RING_UBASE 0x080
+#define ADF_RING_CSR_RING_HEAD 0x0C0
+#define ADF_RING_CSR_RING_TAIL 0x100
+#define ADF_RING_CSR_E_STAT 0x14C
+#define ADF_RING_CSR_INT_FLAG 0x170
+#define ADF_RING_CSR_INT_SRCSEL 0x174
+#define ADF_RING_CSR_INT_SRCSEL_2 0x178
+#define ADF_RING_CSR_INT_COL_EN 0x17C
+#define ADF_RING_CSR_INT_COL_CTL 0x180
+#define ADF_RING_CSR_INT_FLAG_AND_COL 0x184
+#define ADF_RING_CSR_INT_COL_CTL_ENABLE 0x80000000
+#define ADF_RING_BUNDLE_SIZE 0x1000
+#define ADF_GEN2_RX_RINGS_OFFSET 8
+#define ADF_GEN2_TX_RINGS_MASK 0xFF
+
+#define BUILD_RING_BASE_ADDR(addr, size) \
+ (((addr) >> 6) & (GENMASK_ULL(63, 0) << (size)))
+#define READ_CSR_RING_HEAD(csr_base_addr, bank, ring) \
+ ADF_CSR_RD(csr_base_addr, \
+ (ADF_RING_BUNDLE_SIZE * (bank)) + ADF_RING_CSR_RING_HEAD + \
+ ((ring) << 2))
+#define READ_CSR_RING_TAIL(csr_base_addr, bank, ring) \
+ ADF_CSR_RD(csr_base_addr, \
+ (ADF_RING_BUNDLE_SIZE * (bank)) + ADF_RING_CSR_RING_TAIL + \
+ ((ring) << 2))
+#define READ_CSR_E_STAT(csr_base_addr, bank) \
+ ADF_CSR_RD(csr_base_addr, \
+ (ADF_RING_BUNDLE_SIZE * (bank)) + ADF_RING_CSR_E_STAT)
+#define WRITE_CSR_RING_CONFIG(csr_base_addr, bank, ring, value) \
+ ADF_CSR_WR(csr_base_addr, \
+ (ADF_RING_BUNDLE_SIZE * (bank)) + \
+ ADF_RING_CSR_RING_CONFIG + ((ring) << 2), \
+ value)
+#define WRITE_CSR_RING_BASE(csr_base_addr, bank, ring, value) \
+ do { \
+ u32 l_base = 0, u_base = 0; \
+ l_base = (u32)((value)&0xFFFFFFFF); \
+ u_base = (u32)(((value)&0xFFFFFFFF00000000ULL) >> 32); \
+ ADF_CSR_WR(csr_base_addr, \
+ (ADF_RING_BUNDLE_SIZE * (bank)) + \
+ ADF_RING_CSR_RING_LBASE + ((ring) << 2), \
+ l_base); \
+ ADF_CSR_WR(csr_base_addr, \
+ (ADF_RING_BUNDLE_SIZE * (bank)) + \
+ ADF_RING_CSR_RING_UBASE + ((ring) << 2), \
+ u_base); \
+ } while (0)
+
+#define WRITE_CSR_RING_HEAD(csr_base_addr, bank, ring, value) \
+ ADF_CSR_WR(csr_base_addr, \
+ (ADF_RING_BUNDLE_SIZE * (bank)) + ADF_RING_CSR_RING_HEAD + \
+ ((ring) << 2), \
+ value)
+#define WRITE_CSR_RING_TAIL(csr_base_addr, bank, ring, value) \
+ ADF_CSR_WR(csr_base_addr, \
+ (ADF_RING_BUNDLE_SIZE * (bank)) + ADF_RING_CSR_RING_TAIL + \
+ ((ring) << 2), \
+ value)
+#define WRITE_CSR_INT_FLAG(csr_base_addr, bank, value) \
+ ADF_CSR_WR(csr_base_addr, \
+ (ADF_RING_BUNDLE_SIZE * (bank)) + ADF_RING_CSR_INT_FLAG, \
+ value)
+#define WRITE_CSR_INT_SRCSEL(csr_base_addr, bank) \
+ do { \
+ ADF_CSR_WR(csr_base_addr, \
+ (ADF_RING_BUNDLE_SIZE * (bank)) + \
+ ADF_RING_CSR_INT_SRCSEL, \
+ ADF_BANK_INT_SRC_SEL_MASK_0); \
+ ADF_CSR_WR(csr_base_addr, \
+ (ADF_RING_BUNDLE_SIZE * (bank)) + \
+ ADF_RING_CSR_INT_SRCSEL_2, \
+ ADF_BANK_INT_SRC_SEL_MASK_X); \
+ } while (0)
+#define WRITE_CSR_INT_COL_EN(csr_base_addr, bank, value) \
+ ADF_CSR_WR(csr_base_addr, \
+ (ADF_RING_BUNDLE_SIZE * (bank)) + ADF_RING_CSR_INT_COL_EN, \
+ value)
+#define WRITE_CSR_INT_COL_CTL(csr_base_addr, bank, value) \
+ ADF_CSR_WR(csr_base_addr, \
+ (ADF_RING_BUNDLE_SIZE * (bank)) + ADF_RING_CSR_INT_COL_CTL, \
+ ADF_RING_CSR_INT_COL_CTL_ENABLE | (value))
+#define WRITE_CSR_INT_FLAG_AND_COL(csr_base_addr, bank, value) \
+ ADF_CSR_WR(csr_base_addr, \
+ (ADF_RING_BUNDLE_SIZE * (bank)) + \
+ ADF_RING_CSR_INT_FLAG_AND_COL, \
+ value)
+
+/* AE to function map */
+#define AE2FUNCTION_MAP_A_OFFSET (0x3A400 + 0x190)
+#define AE2FUNCTION_MAP_B_OFFSET (0x3A400 + 0x310)
+#define AE2FUNCTION_MAP_REG_SIZE 4
+#define AE2FUNCTION_MAP_VALID BIT(7)
+
+#define READ_CSR_AE2FUNCTION_MAP_A(pmisc_bar_addr, index) \
+ ADF_CSR_RD(pmisc_bar_addr, \
+ AE2FUNCTION_MAP_A_OFFSET + \
+ AE2FUNCTION_MAP_REG_SIZE * (index))
+#define WRITE_CSR_AE2FUNCTION_MAP_A(pmisc_bar_addr, index, value) \
+ ADF_CSR_WR(pmisc_bar_addr, \
+ AE2FUNCTION_MAP_A_OFFSET + \
+ AE2FUNCTION_MAP_REG_SIZE * (index), \
+ value)
+#define READ_CSR_AE2FUNCTION_MAP_B(pmisc_bar_addr, index) \
+ ADF_CSR_RD(pmisc_bar_addr, \
+ AE2FUNCTION_MAP_B_OFFSET + \
+ AE2FUNCTION_MAP_REG_SIZE * (index))
+#define WRITE_CSR_AE2FUNCTION_MAP_B(pmisc_bar_addr, index, value) \
+ ADF_CSR_WR(pmisc_bar_addr, \
+ AE2FUNCTION_MAP_B_OFFSET + \
+ AE2FUNCTION_MAP_REG_SIZE * (index), \
+ value)
+
+/* Admin Interface Offsets */
+#define ADF_ADMINMSGUR_OFFSET (0x3A000 + 0x574)
+#define ADF_ADMINMSGLR_OFFSET (0x3A000 + 0x578)
+#define ADF_MAILBOX_BASE_OFFSET 0x20970
+
+/* Arbiter configuration */
+#define ADF_ARB_OFFSET 0x30000
+#define ADF_ARB_WRK_2_SER_MAP_OFFSET 0x180
+#define ADF_ARB_CONFIG (BIT(31) | BIT(6) | BIT(0))
+#define ADF_ARB_REG_SLOT 0x1000
+#define ADF_ARB_RINGSRVARBEN_OFFSET 0x19C
+
+#define READ_CSR_RING_SRV_ARB_EN(csr_addr, index) \
+ ADF_CSR_RD(csr_addr, \
+ ADF_ARB_RINGSRVARBEN_OFFSET + (ADF_ARB_REG_SLOT * (index)))
+
+#define WRITE_CSR_RING_SRV_ARB_EN(csr_addr, index, value) \
+ ADF_CSR_WR(csr_addr, \
+ ADF_ARB_RINGSRVARBEN_OFFSET + (ADF_ARB_REG_SLOT * (index)), \
+ value)
+
+/* Power gating */
+#define ADF_POWERGATE_DC BIT(23)
+#define ADF_POWERGATE_PKE BIT(24)
+
+/* Default ring mapping */
+#define ADF_GEN2_DEFAULT_RING_TO_SRV_MAP \
+ (CRYPTO << ADF_CFG_SERV_RING_PAIR_0_SHIFT | \
+ CRYPTO << ADF_CFG_SERV_RING_PAIR_1_SHIFT | \
+ UNUSED << ADF_CFG_SERV_RING_PAIR_2_SHIFT | \
+ COMP << ADF_CFG_SERV_RING_PAIR_3_SHIFT)
+
+/* Error detection and correction */
+#define ADF_GEN2_AE_CTX_ENABLES(i) ((i)*0x1000 + 0x20818)
+#define ADF_GEN2_AE_MISC_CONTROL(i) ((i)*0x1000 + 0x20960)
+#define ADF_GEN2_ENABLE_AE_ECC_ERR BIT(28)
+#define ADF_GEN2_ENABLE_AE_ECC_PARITY_CORR (BIT(24) | BIT(12))
+#define ADF_GEN2_UERRSSMSH(i) ((i)*0x4000 + 0x18)
+#define ADF_GEN2_CERRSSMSH(i) ((i)*0x4000 + 0x10)
+#define ADF_GEN2_ERRSSMSH_EN BIT(3)
+
+#define ADF_NUM_HB_CNT_PER_AE (ADF_NUM_THREADS_PER_AE + ADF_NUM_PKE_STRAND)
+
+void adf_gen2_init_hw_csr_info(struct adf_hw_csr_info *csr_info);
+
+#endif
Index: sys/dev/qat/include/common/adf_gen4_hw_data.h
===================================================================
--- /dev/null
+++ sys/dev/qat/include/common/adf_gen4_hw_data.h
@@ -0,0 +1,132 @@
+/* SPDX-License-Identifier: BSD-3-Clause */
+/* Copyright(c) 2021 Intel Corporation */
+/* $FreeBSD$ */
+#ifndef ADF_GEN4_HW_CSR_DATA_H_
+#define ADF_GEN4_HW_CSR_DATA_H_
+
+#include "adf_accel_devices.h"
+
+/* Transport access */
+#define ADF_BANK_INT_SRC_SEL_MASK 0x44UL
+#define ADF_RING_CSR_RING_CONFIG 0x1000
+#define ADF_RING_CSR_RING_LBASE 0x1040
+#define ADF_RING_CSR_RING_UBASE 0x1080
+#define ADF_RING_CSR_RING_HEAD 0x0C0
+#define ADF_RING_CSR_RING_TAIL 0x100
+#define ADF_RING_CSR_E_STAT 0x14C
+#define ADF_RING_CSR_INT_FLAG 0x170
+#define ADF_RING_CSR_INT_SRCSEL 0x174
+#define ADF_RING_CSR_INT_COL_CTL 0x180
+#define ADF_RING_CSR_INT_FLAG_AND_COL 0x184
+#define ADF_RING_CSR_INT_COL_CTL_ENABLE 0x80000000
+#define ADF_RING_CSR_INT_COL_EN 0x17C
+#define ADF_RING_CSR_ADDR_OFFSET 0x100000
+#define ADF_RING_BUNDLE_SIZE 0x2000
+
+#define BUILD_RING_BASE_ADDR(addr, size) \
+ ((((addr) >> 6) & (GENMASK_ULL(63, 0) << (size))) << 6)
+#define READ_CSR_RING_HEAD(csr_base_addr, bank, ring) \
+ ADF_CSR_RD((csr_base_addr), \
+ ADF_RING_CSR_ADDR_OFFSET + ADF_RING_BUNDLE_SIZE * (bank) + \
+ ADF_RING_CSR_RING_HEAD + ((ring) << 2))
+#define READ_CSR_RING_TAIL(csr_base_addr, bank, ring) \
+ ADF_CSR_RD((csr_base_addr), \
+ ADF_RING_CSR_ADDR_OFFSET + ADF_RING_BUNDLE_SIZE * (bank) + \
+ ADF_RING_CSR_RING_TAIL + ((ring) << 2))
+#define READ_CSR_E_STAT(csr_base_addr, bank) \
+ ADF_CSR_RD((csr_base_addr), \
+ ADF_RING_CSR_ADDR_OFFSET + ADF_RING_BUNDLE_SIZE * (bank) + \
+ ADF_RING_CSR_E_STAT)
+#define WRITE_CSR_RING_CONFIG(csr_base_addr, bank, ring, value) \
+ ADF_CSR_WR((csr_base_addr), \
+ ADF_RING_CSR_ADDR_OFFSET + ADF_RING_BUNDLE_SIZE * (bank) + \
+ ADF_RING_CSR_RING_CONFIG + ((ring) << 2), \
+ value)
+#define WRITE_CSR_RING_BASE(csr_base_addr, bank, ring, value) \
+ do { \
+ struct resource *_csr_base_addr = csr_base_addr; \
+ u32 _bank = bank; \
+ u32 _ring = ring; \
+ dma_addr_t _value = value; \
+ u32 l_base = 0, u_base = 0; \
+ l_base = lower_32_bits(_value); \
+ u_base = upper_32_bits(_value); \
+ ADF_CSR_WR((_csr_base_addr), \
+ ADF_RING_CSR_ADDR_OFFSET + \
+ ADF_RING_BUNDLE_SIZE * (_bank) + \
+ ADF_RING_CSR_RING_LBASE + ((_ring) << 2), \
+ l_base); \
+ ADF_CSR_WR((_csr_base_addr), \
+ ADF_RING_CSR_ADDR_OFFSET + \
+ ADF_RING_BUNDLE_SIZE * (_bank) + \
+ ADF_RING_CSR_RING_UBASE + ((_ring) << 2), \
+ u_base); \
+ } while (0)
+
+#define WRITE_CSR_RING_HEAD(csr_base_addr, bank, ring, value) \
+ ADF_CSR_WR((csr_base_addr), \
+ ADF_RING_CSR_ADDR_OFFSET + ADF_RING_BUNDLE_SIZE * (bank) + \
+ ADF_RING_CSR_RING_HEAD + ((ring) << 2), \
+ value)
+#define WRITE_CSR_RING_TAIL(csr_base_addr, bank, ring, value) \
+ ADF_CSR_WR((csr_base_addr), \
+ ADF_RING_CSR_ADDR_OFFSET + ADF_RING_BUNDLE_SIZE * (bank) + \
+ ADF_RING_CSR_RING_TAIL + ((ring) << 2), \
+ value)
+#define WRITE_CSR_INT_FLAG(csr_base_addr, bank, value) \
+ ADF_CSR_WR((csr_base_addr), \
+ ADF_RING_CSR_ADDR_OFFSET + ADF_RING_BUNDLE_SIZE * (bank) + \
+ ADF_RING_CSR_INT_FLAG, \
+ (value))
+#define WRITE_CSR_INT_SRCSEL(csr_base_addr, bank) \
+ ADF_CSR_WR((csr_base_addr), \
+ ADF_RING_CSR_ADDR_OFFSET + ADF_RING_BUNDLE_SIZE * (bank) + \
+ ADF_RING_CSR_INT_SRCSEL, \
+ ADF_BANK_INT_SRC_SEL_MASK)
+#define WRITE_CSR_INT_COL_EN(csr_base_addr, bank, value) \
+ ADF_CSR_WR((csr_base_addr), \
+ ADF_RING_CSR_ADDR_OFFSET + ADF_RING_BUNDLE_SIZE * (bank) + \
+ ADF_RING_CSR_INT_COL_EN, \
+ (value))
+#define WRITE_CSR_INT_COL_CTL(csr_base_addr, bank, value) \
+ ADF_CSR_WR((csr_base_addr), \
+ ADF_RING_CSR_ADDR_OFFSET + ADF_RING_BUNDLE_SIZE * (bank) + \
+ ADF_RING_CSR_INT_COL_CTL, \
+ ADF_RING_CSR_INT_COL_CTL_ENABLE | (value))
+#define WRITE_CSR_INT_FLAG_AND_COL(csr_base_addr, bank, value) \
+ ADF_CSR_WR((csr_base_addr), \
+ ADF_RING_CSR_ADDR_OFFSET + ADF_RING_BUNDLE_SIZE * (bank) + \
+ ADF_RING_CSR_INT_FLAG_AND_COL, \
+ (value))
+
+/* Arbiter configuration */
+#define ADF_RING_CSR_RING_SRV_ARB_EN 0x19C
+
+#define READ_CSR_RING_SRV_ARB_EN(csr_base_addr, bank) \
+ ADF_CSR_RD((csr_base_addr), \
+ ADF_RING_CSR_ADDR_OFFSET + ADF_RING_BUNDLE_SIZE * (bank) + \
+ ADF_RING_CSR_RING_SRV_ARB_EN)
+
+#define WRITE_CSR_RING_SRV_ARB_EN(csr_base_addr, bank, value) \
+ ADF_CSR_WR((csr_base_addr), \
+ ADF_RING_CSR_ADDR_OFFSET + ADF_RING_BUNDLE_SIZE * (bank) + \
+ ADF_RING_CSR_RING_SRV_ARB_EN, \
+ (value))
+
+/* WDT timers
+ *
+ * Timeout is in cycles. Clock speed may vary across products but this
+ * value should be a few milli-seconds.
+ */
+#define ADF_SSM_WDT_DEFAULT_VALUE 0x200000
+#define ADF_SSM_WDT_PKE_DEFAULT_VALUE 0x8000000
+#define ADF_SSMWDTL_OFFSET 0x54
+#define ADF_SSMWDTH_OFFSET 0x5C
+#define ADF_SSMWDTPKEL_OFFSET 0x58
+#define ADF_SSMWDTPKEH_OFFSET 0x60
+
+#define ADF_NUM_HB_CNT_PER_AE (ADF_NUM_THREADS_PER_AE)
+
+int adf_gen4_set_ssm_wdtimer(struct adf_accel_dev *accel_dev);
+void adf_gen4_init_hw_csr_info(struct adf_hw_csr_info *csr_info);
+#endif
Index: sys/dev/qat/include/common/icp_qat_fw_loader_handle.h
===================================================================
--- sys/dev/qat/include/common/icp_qat_fw_loader_handle.h
+++ sys/dev/qat/include/common/icp_qat_fw_loader_handle.h
@@ -16,6 +16,7 @@
struct icp_qat_fw_loader_hal_handle {
struct icp_qat_fw_loader_ae_data aes[ICP_QAT_UCLO_MAX_AE];
unsigned int ae_mask;
+ unsigned int admin_ae_mask;
unsigned int slice_mask;
unsigned int revision_id;
unsigned int ae_max_num;
Index: sys/dev/qat/include/common/icp_qat_hal.h
===================================================================
--- sys/dev/qat/include/common/icp_qat_hal.h
+++ sys/dev/qat/include/common/icp_qat_hal.h
@@ -52,23 +52,32 @@
};
enum fcu_csr {
- FCU_CONTROL = 0x0,
- FCU_STATUS = 0x4,
- FCU_DRAM_ADDR_LO = 0xc,
+ FCU_CONTROL = 0x00,
+ FCU_STATUS = 0x04,
+ FCU_DRAM_ADDR_LO = 0x0c,
FCU_DRAM_ADDR_HI = 0x10,
FCU_RAMBASE_ADDR_HI = 0x14,
FCU_RAMBASE_ADDR_LO = 0x18
};
enum fcu_csr_c4xxx {
- FCU_CONTROL_C4XXX = 0x0,
- FCU_STATUS_C4XXX = 0x4,
- FCU_STATUS1_C4XXX = 0xc,
+ FCU_CONTROL_C4XXX = 0x00,
+ FCU_STATUS_C4XXX = 0x04,
+ FCU_STATUS1_C4XXX = 0x0c,
FCU_AE_LOADED_C4XXX = 0x10,
FCU_DRAM_ADDR_LO_C4XXX = 0x14,
FCU_DRAM_ADDR_HI_C4XXX = 0x18,
};
+enum fcu_csr_4xxx {
+ FCU_CONTROL_4XXX = 0x00,
+ FCU_STATUS_4XXX = 0x04,
+ FCU_ME_BROADCAST_MASK_TYPE = 0x08,
+ FCU_AE_LOADED_4XXX = 0x10,
+ FCU_DRAM_ADDR_LO_4XXX = 0x14,
+ FCU_DRAM_ADDR_HI_4XXX = 0x18,
+};
+
enum fcu_cmd {
FCU_CTRL_CMD_NOOP = 0,
FCU_CTRL_CMD_AUTH = 1,
@@ -104,6 +113,7 @@
#define LCS_STATUS (0x1)
#define MMC_SHARE_CS_BITPOS 2
#define GLOBAL_CSR 0xA00
+#define FCU_CTRL_BROADCAST_POS 0x4
#define FCU_CTRL_AE_POS 0x8
#define FCU_AUTH_STS_MASK 0x7
#define FCU_STS_DONE_POS 0x9
@@ -111,20 +121,26 @@
#define FCU_LOADED_AE_POS 0x16
#define FW_AUTH_WAIT_PERIOD 10
#define FW_AUTH_MAX_RETRY 300
+#define FW_BROADCAST_MAX_RETRY 300
#define FCU_OFFSET 0x8c0
#define FCU_OFFSET_C4XXX 0x1000
+#define FCU_OFFSET_4XXX 0x1000
#define MAX_CPP_NUM 2
#define AE_CPP_NUM 2
#define AES_PER_CPP 16
#define SLICES_PER_CPP 6
#define ICP_QAT_AE_OFFSET 0x20000
#define ICP_QAT_AE_OFFSET_C4XXX 0x40000
+#define ICP_QAT_AE_OFFSET_4XXX 0x600000
#define ICP_QAT_CAP_OFFSET (ICP_QAT_AE_OFFSET + 0x10000)
#define ICP_QAT_CAP_OFFSET_C4XXX 0x70000
+#define ICP_QAT_CAP_OFFSET_4XXX 0x640000
#define LOCAL_TO_XFER_REG_OFFSET 0x800
#define ICP_QAT_EP_OFFSET 0x3a000
#define ICP_QAT_EP_OFFSET_C4XXX 0x60000
+#define ICP_QAT_EP_OFFSET_4XXX 0x200000 /* HI MMIO CSRs */
#define MEM_CFG_ERR_BIT 0x20
+#define AE_TG_NUM_CPM2X 4
#define CAP_CSR_ADDR(csr) (csr + handle->hal_cap_g_ctl_csr_addr_v)
#define SET_CAP_CSR(handle, csr, val) \
@@ -133,20 +149,17 @@
ADF_CSR_RD(handle->hal_misc_addr_v, CAP_CSR_ADDR(csr))
#define SET_GLB_CSR(handle, csr, val) \
({ \
- typeof(handle) handle_ = (handle); \
- typeof(csr) csr_ = (csr); \
- typeof(val) val_ = (val); \
- (IS_QAT_GEN3(pci_get_device(GET_DEV(handle_->accel_dev)))) ? \
- SET_CAP_CSR(handle_, (csr_), (val_)) : \
- SET_CAP_CSR(handle_, csr_ + GLOBAL_CSR, val_); \
+ u32 dev_id = pci_get_device(GET_DEV((handle)->accel_dev)); \
+ (IS_QAT_GEN3_OR_GEN4(dev_id)) ? \
+ SET_CAP_CSR((handle), (csr), (val)) : \
+ SET_CAP_CSR((handle), (csr) + GLOBAL_CSR, val); \
})
#define GET_GLB_CSR(handle, csr) \
({ \
- typeof(handle) handle_ = (handle); \
- typeof(csr) csr_ = (csr); \
- (IS_QAT_GEN3(pci_get_device(GET_DEV(handle_->accel_dev)))) ? \
- (GET_CAP_CSR(handle_, (csr_))) : \
- (GET_CAP_CSR(handle_, (GLOBAL_CSR + (csr_)))); \
+ u32 dev_id = pci_get_device(GET_DEV((handle)->accel_dev)); \
+ (IS_QAT_GEN3_OR_GEN4(dev_id)) ? \
+ GET_CAP_CSR((handle), (csr)) : \
+ GET_CAP_CSR((handle), (csr) + GLOBAL_CSR); \
})
#define SET_FCU_CSR(handle, csr, val) \
({ \
@@ -157,7 +170,12 @@
SET_CAP_CSR(handle_, \
((csr_) + FCU_OFFSET_C4XXX), \
(val_)) : \
- SET_CAP_CSR(handle_, ((csr_) + FCU_OFFSET), (val_)); \
+ ((IS_QAT_GEN4( \
+ pci_get_device(GET_DEV(handle_->accel_dev)))) ? \
+ SET_CAP_CSR(handle_, \
+ ((csr_) + FCU_OFFSET_4XXX), \
+ (val_)) : \
+ SET_CAP_CSR(handle_, ((csr_) + FCU_OFFSET), (val_))); \
})
#define GET_FCU_CSR(handle, csr) \
({ \
@@ -165,7 +183,10 @@
typeof(csr) csr_ = (csr); \
(IS_QAT_GEN3(pci_get_device(GET_DEV(handle_->accel_dev)))) ? \
GET_CAP_CSR(handle_, (FCU_OFFSET_C4XXX + (csr_))) : \
- GET_CAP_CSR(handle_, (FCU_OFFSET + (csr_))); \
+ ((IS_QAT_GEN4( \
+ pci_get_device(GET_DEV(handle_->accel_dev)))) ? \
+ GET_CAP_CSR(handle_, (FCU_OFFSET_4XXX + (csr_))) : \
+ GET_CAP_CSR(handle_, (FCU_OFFSET + (csr_)))); \
})
#define AE_CSR(handle, ae) \
((handle)->hal_cap_ae_local_csr_addr_v + ((ae) << 12))
@@ -184,13 +205,19 @@
ADF_CSR_WR((handle)->hal_sram_addr_v, addr, val)
#define GET_CSR_OFFSET(device_id, cap_offset_, ae_offset_, ep_offset_) \
({ \
- int gen3 = IS_QAT_GEN3(device_id); \
- cap_offset_ = \
- (gen3 ? ICP_QAT_CAP_OFFSET_C4XXX : ICP_QAT_CAP_OFFSET); \
- ae_offset_ = \
- (gen3 ? ICP_QAT_AE_OFFSET_C4XXX : ICP_QAT_AE_OFFSET); \
- ep_offset_ = \
- (gen3 ? ICP_QAT_EP_OFFSET_C4XXX : ICP_QAT_EP_OFFSET); \
+ if (IS_QAT_GEN3(device_id)) { \
+ cap_offset_ = ICP_QAT_CAP_OFFSET_C4XXX; \
+ ae_offset_ = ICP_QAT_AE_OFFSET_C4XXX; \
+ ep_offset_ = ICP_QAT_EP_OFFSET_C4XXX; \
+ } else if (IS_QAT_GEN4(device_id)) { \
+ cap_offset_ = ICP_QAT_CAP_OFFSET_4XXX; \
+ ae_offset_ = ICP_QAT_AE_OFFSET_4XXX; \
+ ep_offset_ = ICP_QAT_EP_OFFSET_4XXX; \
+ } else { \
+ cap_offset_ = ICP_QAT_CAP_OFFSET; \
+ ae_offset_ = ICP_QAT_AE_OFFSET; \
+ ep_offset_ = ICP_QAT_EP_OFFSET; \
+ } \
})
#endif
Index: sys/dev/qat/include/common/icp_qat_uclo.h
===================================================================
--- sys/dev/qat/include/common/icp_qat_uclo.h
+++ sys/dev/qat/include/common/icp_qat_uclo.h
@@ -9,6 +9,7 @@
#define ICP_QAT_AC_C3XXX_DEV_TYPE 0x02000000
#define ICP_QAT_AC_200XX_DEV_TYPE 0x02000000
#define ICP_QAT_AC_C4XXX_DEV_TYPE 0x04000000
+#define ICP_QAT_AC_4XXX_A_DEV_TYPE 0x08000000
#define ICP_QAT_UCLO_MAX_AE 32
#define ICP_QAT_UCLO_MAX_CTX 8
#define ICP_QAT_UCLO_MAX_CPPNUM 2
@@ -17,6 +18,7 @@
#define ICP_QAT_UCLO_MAX_XFER_REG 128
#define ICP_QAT_UCLO_MAX_GPR_REG 128
#define ICP_QAT_UCLO_MAX_LMEM_REG 1024
+#define ICP_QAT_UCLO_MAX_LMEM_REG_2X 1280
#define ICP_QAT_UCLO_AE_ALL_CTX 0xff
#define ICP_QAT_UOF_OBJID_LEN 8
#define ICP_QAT_UOF_FID 0xc6c2
@@ -46,22 +48,42 @@
#define ICP_QAT_SUOF_IMAG "SUF_IMAG"
#define ICP_QAT_SIMG_AE_INIT_SEQ_LEN (50 * sizeof(unsigned long long))
#define ICP_QAT_SIMG_AE_INSTS_LEN (0x4000 * sizeof(unsigned long long))
-#define ICP_QAT_CSS_FWSK_MODULUS_LEN 256
-#define ICP_QAT_CSS_FWSK_EXPONENT_LEN 4
-#define ICP_QAT_CSS_FWSK_PAD_LEN 252
-#define ICP_QAT_CSS_FWSK_PUB_LEN \
- (ICP_QAT_CSS_FWSK_MODULUS_LEN + ICP_QAT_CSS_FWSK_EXPONENT_LEN + \
- ICP_QAT_CSS_FWSK_PAD_LEN)
-#define ICP_QAT_CSS_SIGNATURE_LEN 256
+
+#define DSS_FWSK_MODULUS_LEN 384 // RSA3K
+#define DSS_FWSK_EXPONENT_LEN 4
+#define DSS_FWSK_PADDING_LEN 380
+#define DSS_SIGNATURE_LEN 384 // RSA3K
+
+#define CSS_FWSK_MODULUS_LEN 256 // RSA2K
+#define CSS_FWSK_EXPONENT_LEN 4
+#define CSS_FWSK_PADDING_LEN 252
+#define CSS_SIGNATURE_LEN 256 // RSA2K
+
+#define ICP_QAT_CSS_FWSK_MODULUS_LEN(ID) \
+ (IS_QAT_GEN4(ID) ? DSS_FWSK_MODULUS_LEN : CSS_FWSK_MODULUS_LEN)
+
+#define ICP_QAT_CSS_FWSK_EXPONENT_LEN(ID) \
+ (IS_QAT_GEN4(ID) ? DSS_FWSK_EXPONENT_LEN : CSS_FWSK_EXPONENT_LEN)
+
+#define ICP_QAT_CSS_FWSK_PAD_LEN(ID) \
+ (IS_QAT_GEN4(ID) ? DSS_FWSK_PADDING_LEN : CSS_FWSK_PADDING_LEN)
+
+#define ICP_QAT_CSS_FWSK_PUB_LEN(ID) \
+ (ICP_QAT_CSS_FWSK_MODULUS_LEN(ID) + \
+ ICP_QAT_CSS_FWSK_EXPONENT_LEN(ID) + ICP_QAT_CSS_FWSK_PAD_LEN(ID))
+
+#define ICP_QAT_CSS_SIGNATURE_LEN(ID) \
+ (IS_QAT_GEN4(ID) ? DSS_SIGNATURE_LEN : CSS_SIGNATURE_LEN)
+
#define ICP_QAT_CSS_AE_IMG_LEN \
(sizeof(struct icp_qat_simg_ae_mode) + ICP_QAT_SIMG_AE_INIT_SEQ_LEN + \
ICP_QAT_SIMG_AE_INSTS_LEN)
-#define ICP_QAT_CSS_AE_SIMG_LEN \
- (sizeof(struct icp_qat_css_hdr) + ICP_QAT_CSS_FWSK_PUB_LEN + \
- ICP_QAT_CSS_SIGNATURE_LEN + ICP_QAT_CSS_AE_IMG_LEN)
-#define ICP_QAT_AE_IMG_OFFSET \
- (sizeof(struct icp_qat_css_hdr) + ICP_QAT_CSS_FWSK_MODULUS_LEN + \
- ICP_QAT_CSS_FWSK_EXPONENT_LEN + ICP_QAT_CSS_SIGNATURE_LEN)
+#define ICP_QAT_CSS_AE_SIMG_LEN(ID) \
+ (sizeof(struct icp_qat_css_hdr) + ICP_QAT_CSS_FWSK_PUB_LEN(ID) + \
+ ICP_QAT_CSS_SIGNATURE_LEN(ID) + ICP_QAT_CSS_AE_IMG_LEN)
+#define ICP_QAT_AE_IMG_OFFSET(ID) \
+ (sizeof(struct icp_qat_css_hdr) + ICP_QAT_CSS_FWSK_MODULUS_LEN(ID) + \
+ ICP_QAT_CSS_FWSK_EXPONENT_LEN(ID) + ICP_QAT_CSS_SIGNATURE_LEN(ID))
#define ICP_QAT_CSS_MAX_IMAGE_LEN 0x40000
#define ICP_QAT_CTX_MODE(ae_mode) ((ae_mode)&0xf)
Index: sys/dev/qat/include/icp_qat_fw_init_admin.h
===================================================================
--- sys/dev/qat/include/icp_qat_fw_init_admin.h
+++ sys/dev/qat/include/icp_qat_fw_init_admin.h
@@ -200,10 +200,6 @@
u16 req_heartbeat_cnt;
};
-struct icp_qat_fw_init_admin_hb_stats {
- struct icp_qat_fw_init_admin_hb_cnt stats[ADF_NUM_HB_CNT_PER_AE];
-};
-
#define ICP_QAT_FW_COMN_HEARTBEAT_OK 0
#define ICP_QAT_FW_COMN_HEARTBEAT_BLOCKED 1
#define ICP_QAT_FW_COMN_HEARTBEAT_FLAG_BITPOS 0
Index: sys/dev/qat/qat/qat_ocf.c
===================================================================
--- sys/dev/qat/qat/qat_ocf.c
+++ sys/dev/qat/qat/qat_ocf.c
@@ -1222,6 +1222,7 @@
MODULE_DEPEND(qat, qat_c3xxx, 1, 1, 1);
MODULE_DEPEND(qat, qat_c4xxx, 1, 1, 1);
MODULE_DEPEND(qat, qat_dh895xcc, 1, 1, 1);
+MODULE_DEPEND(qat, qat_4xxx, 1, 1, 1);
MODULE_DEPEND(qat, crypto, 1, 1, 1);
MODULE_DEPEND(qat, qat_common, 1, 1, 1);
MODULE_DEPEND(qat, qat_api, 1, 1, 1);
Index: sys/dev/qat/qat_api/common/compression/dc_buffers.c
===================================================================
--- sys/dev/qat/qat_api/common/compression/dc_buffers.c
+++ sys/dev/qat/qat_api/common/compression/dc_buffers.c
@@ -26,9 +26,14 @@
#include "sal_types_compression.h"
#include "icp_qat_fw_comp.h"
+#include "sal_hw_gen.h"
#define CPA_DC_CEIL_DIV(x, y) (((x) + (y)-1) / (y))
#define DC_DEST_BUFF_EXTRA_DEFLATE_GEN2 (55)
+#define DC_DEST_BUFF_EXTRA_DEFLATE_GEN4_STATIC (1029)
+#define DC_DEST_BUFF_EXTRA_DEFLATE_GEN4_DYN (512)
+#define DC_DEST_BUFF_MIN_EXTRA_BYTES(x) ((x < 8) ? (8 - x) : 0)
+#define DC_BUF_MAX_SIZE (0xFFFFFFFF)
CpaStatus
cpaDcBufferListGetMetaSize(const CpaInstanceHandle instanceHandle,
@@ -72,13 +77,60 @@
static inline CpaStatus
dcDeflateBoundGen2(CpaDcHuffType huffType, Cpa32U inputSize, Cpa32U *outputSize)
{
+ Cpa64U inBufferSize = inputSize;
+ Cpa64U outBufferSize = 0;
+
/* Formula for GEN2 deflate:
* ceil(9 * Total input bytes / 8) + 55 bytes.
* 55 bytes is the skid pad value for GEN2 devices.
+ * Adding extra bytes = `DC_DEST_BUFF_MIN_EXTRA_BYTES(inputSize)`
+ * when calculated value from `CPA_DC_CEIL_DIV(9 * inputSize, 8) +
+ * DC_DEST_BUFF_EXTRA_DEFLATE_GEN2` is less than 64 bytes to
+ * achieve a safer output buffer size of 64 bytes.
*/
- *outputSize =
- CPA_DC_CEIL_DIV(9 * inputSize, 8) + DC_DEST_BUFF_EXTRA_DEFLATE_GEN2;
+ outBufferSize = CPA_DC_CEIL_DIV(9 * inBufferSize, 8) +
+ DC_DEST_BUFF_EXTRA_DEFLATE_GEN2 +
+ DC_DEST_BUFF_MIN_EXTRA_BYTES(inputSize);
+
+ if (outBufferSize > DC_BUF_MAX_SIZE)
+ *outputSize = DC_BUF_MAX_SIZE;
+ else
+ *outputSize = (Cpa32U)outBufferSize;
+
+ return CPA_STATUS_SUCCESS;
+}
+
+static inline CpaStatus
+dcDeflateBoundGen4(CpaDcHuffType huffType, Cpa32U inputSize, Cpa32U *outputSize)
+{
+ Cpa64U outputSizeLong;
+ Cpa64U inputSizeLong = (Cpa64U)inputSize;
+
+ switch (huffType) {
+ case CPA_DC_HT_STATIC:
+ /* Formula for GEN4 static deflate:
+ * ceil((9*sourceLen)/8) + 5 + 1024. */
+ outputSizeLong = CPA_DC_CEIL_DIV(9 * inputSizeLong, 8) +
+ DC_DEST_BUFF_EXTRA_DEFLATE_GEN4_STATIC;
+ break;
+ case CPA_DC_HT_FULL_DYNAMIC:
+ /* Formula for GEN4 dynamic deflate:
+ * Ceil ((9*sourceLen)/8)▒| +
+ * ((((8/7) * sourceLen)/ 16KB) * (150+5)) + 512
+ */
+ outputSizeLong = DC_DEST_BUFF_EXTRA_DEFLATE_GEN4_DYN;
+ outputSizeLong += CPA_DC_CEIL_DIV(9 * inputSizeLong, 8);
+ outputSizeLong += ((8 * inputSizeLong * 155) / 7) / (16 * 1024);
+ break;
+ default:
+ return CPA_STATUS_INVALID_PARAM;
+ }
+ /* Avoid output size overflow */
+ if (outputSizeLong & 0xffffffff00000000UL)
+ return CPA_STATUS_INVALID_PARAM;
+
+ *outputSize = (Cpa32U)outputSizeLong;
return CPA_STATUS_SUCCESS;
}
@@ -88,6 +140,7 @@
Cpa32U inputSize,
Cpa32U *outputSize)
{
+ sal_compression_service_t *pService = NULL;
CpaInstanceHandle insHandle = NULL;
if (CPA_INSTANCE_HANDLE_SINGLE == dcInstance) {
@@ -112,5 +165,10 @@
return CPA_STATUS_INVALID_PARAM;
}
- return dcDeflateBoundGen2(huffType, inputSize, outputSize);
+ pService = (sal_compression_service_t *)insHandle;
+ if (isDcGen4x(pService)) {
+ return dcDeflateBoundGen4(huffType, inputSize, outputSize);
+ } else {
+ return dcDeflateBoundGen2(huffType, inputSize, outputSize);
+ }
}
Index: sys/dev/qat/qat_api/common/compression/dc_datapath.c
===================================================================
--- sys/dev/qat/qat_api/common/compression/dc_datapath.c
+++ sys/dev/qat/qat_api/common/compression/dc_datapath.c
@@ -42,6 +42,7 @@
#include "lac_sync.h"
#include "sal_service_state.h"
#include "sal_qat_cmn_msg.h"
+#include "sal_hw_gen.h"
#include "dc_error_counter.h"
#define DC_COMP_MAX_BUFF_SIZE (1024 * 64)
@@ -71,6 +72,28 @@
return 0;
}
+static inline void
+dcUpdateXltOverflowChecksumsGen4(const dc_compression_cookie_t *pCookie,
+ const icp_qat_fw_resp_comp_pars_t *pRespPars,
+ CpaDcRqResults *pDcResults)
+{
+ dc_session_desc_t *pSessionDesc =
+ DC_SESSION_DESC_FROM_CTX_GET(pCookie->pSessionHandle);
+
+ /* Recompute CRC checksum when either the checksum type
+ * is CPA_DC_CRC32 or when the integrity CRCs are enabled.
+ */
+ if (CPA_DC_CRC32 == pSessionDesc->checksumType) {
+ pDcResults->checksum = pRespPars->crc.legacy.curr_crc32;
+
+ /* No need to recalculate the swCrc64I here as this will get
+ * handled later in dcHandleIntegrityChecksumsGen4.
+ */
+ } else if (CPA_DC_ADLER32 == pSessionDesc->checksumType) {
+ pDcResults->checksum = pRespPars->crc.legacy.curr_adler_32;
+ }
+}
+
void
dcCompression_ProcessCallback(void *pRespMsg)
{
@@ -86,6 +109,8 @@
dc_compression_cookie_t *pCookie = NULL;
CpaDcOpData *pOpData = NULL;
CpaBoolean cmpPass = CPA_TRUE, xlatPass = CPA_TRUE;
+ CpaBoolean isDcDp = CPA_FALSE;
+ CpaBoolean integrityCrcCheck = CPA_FALSE;
CpaBoolean verifyHwIntegrityCrcs = CPA_FALSE;
Cpa8U cmpErr = ERR_CODE_NO_ERROR, xlatErr = ERR_CODE_NO_ERROR;
dc_request_dir_t compDecomp = DC_COMPRESSION_REQUEST;
@@ -102,17 +127,20 @@
pCookie = (dc_compression_cookie_t *)pReqData;
if (!pCookie)
return;
+
pSessionDesc = DC_SESSION_DESC_FROM_CTX_GET(pCookie->pSessionHandle);
+ pService = (sal_compression_service_t *)(pCookie->dcInstance);
- if (CPA_TRUE == pSessionDesc->isDcDp) {
+ isDcDp = pSessionDesc->isDcDp;
+ if (CPA_TRUE == isDcDp) {
pResponse = (CpaDcDpOpData *)pReqData;
pResults = &(pResponse->results);
if (CPA_DC_DIR_DECOMPRESS == pSessionDesc->sessDirection) {
compDecomp = DC_DECOMPRESSION_REQUEST;
}
+ pCookie = NULL;
} else {
- pSessionDesc = pCookie->pSessionDesc;
pResults = pCookie->pResults;
callbackTag = pCookie->callbackTag;
pCbFunc = pCookie->pSessionDesc->pCompressionCb;
@@ -120,8 +148,6 @@
pOpData = pCookie->pDcOpData;
}
- pService = (sal_compression_service_t *)(pCookie->dcInstance);
-
opStatus = pCompRespMsg->comn_resp.comn_status;
if (NULL != pOpData) {
@@ -142,15 +168,17 @@
pResults->status = (Cpa8S)cmpErr;
pResults->consumed = 0;
pResults->produced = 0;
- if (CPA_TRUE == pSessionDesc->isDcDp) {
+ if (CPA_TRUE == isDcDp) {
if (pResponse)
pResponse->responseStatus =
CPA_STATUS_UNSUPPORTED;
(pService->pDcDpCb)(pResponse);
} else {
/* Free the memory pool */
- Lac_MemPoolEntryFree(pCookie);
- pCookie = NULL;
+ if (NULL != pCookie) {
+ Lac_MemPoolEntryFree(pCookie);
+ pCookie = NULL;
+ }
if (NULL != pCbFunc) {
pCbFunc(callbackTag, status);
}
@@ -169,9 +197,23 @@
ICP_QAT_FW_COMN_RESP_CMP_STAT_GET(opStatus));
}
- if (CPA_DC_INCOMPLETE_FILE_ERR == (Cpa8S)cmpErr) {
- cmpPass = CPA_TRUE;
- cmpErr = ERR_CODE_NO_ERROR;
+ if (isDcGen2x(pService)) {
+ /* QAT1.7 and QAT 1.8 hardware */
+ if (CPA_DC_INCOMPLETE_FILE_ERR == (Cpa8S)cmpErr) {
+ cmpPass = CPA_TRUE;
+ cmpErr = ERR_CODE_NO_ERROR;
+ }
+ } else {
+ /* QAT2.0 hardware cancels the incomplete file errors
+ * only for DEFLATE algorithm.
+ * Decompression direction is not tested in the callback as
+ * the request does not allow it.
+ */
+ if ((pSessionDesc->compType == CPA_DC_DEFLATE) &&
+ (CPA_DC_INCOMPLETE_FILE_ERR == (Cpa8S)cmpErr)) {
+ cmpPass = CPA_TRUE;
+ cmpErr = ERR_CODE_NO_ERROR;
+ }
}
/* log the slice hang and endpoint push/pull error inside the response
*/
@@ -199,8 +241,7 @@
xlatErr = pCompRespMsg->comn_resp.comn_error.s1.xlat_err_code;
/* Return a fatal error or a potential error in the translator
- * slice
- * if the compression slice did not return any error */
+ * slice if the compression slice did not return any error */
if ((CPA_DC_OK == pResults->status) ||
(CPA_DC_FATALERR == (Cpa8S)xlatErr)) {
pResults->status = (Cpa8S)xlatErr;
@@ -209,7 +250,7 @@
/* Update dc error counter */
dcErrorLog(pResults->status);
- if (CPA_FALSE == pSessionDesc->isDcDp) {
+ if (CPA_FALSE == isDcDp) {
/* In case of any error for an end of packet request, we need to
* update
* the request type for the following request */
@@ -223,17 +264,27 @@
((CPA_DC_STATELESS == pSessionDesc->sessState) &&
(DC_COMPRESSION_REQUEST == compDecomp))) {
/* Overflow is a valid use case for Traditional API
- * only.
- * Stateful Overflow is supported in both compression
- * and
- * decompression direction.
- * Stateless Overflow is supported only in compression
- * direction.
+ * only. Stateful Overflow is supported in both
+ * compression and decompression direction. Stateless
+ * Overflow is supported only in compression direction.
*/
if (CPA_DC_OVERFLOW == (Cpa8S)cmpErr)
cmpPass = CPA_TRUE;
if (CPA_DC_OVERFLOW == (Cpa8S)xlatErr) {
+ if (isDcGen4x(pService) &&
+ (CPA_TRUE ==
+ pService->comp_device_data
+ .translatorOverflow)) {
+ pResults->consumed =
+ pCompRespMsg->comp_resp_pars
+ .input_byte_counter;
+
+ dcUpdateXltOverflowChecksumsGen4(
+ pCookie,
+ &pCompRespMsg->comp_resp_pars,
+ pResults);
+ }
xlatPass = CPA_TRUE;
}
}
@@ -242,6 +293,7 @@
cmpPass = CPA_FALSE;
}
if (CPA_DC_OVERFLOW == (Cpa8S)xlatErr) {
+ /* XLT overflow is not valid for Data Plane requests */
xlatPass = CPA_FALSE;
}
}
@@ -254,7 +306,13 @@
pCompRespMsg->comp_resp_pars.output_byte_counter;
pSessionDesc->cumulativeConsumedBytes += pResults->consumed;
- if (CPA_DC_OVERFLOW != (Cpa8S)xlatErr) {
+ /* Handle Checksum for end to end data integrity. */
+ if (CPA_TRUE ==
+ pService->generic_service_info.integrityCrcCheck &&
+ CPA_TRUE == integrityCrcCheck) {
+ pSessionDesc->previousChecksum =
+ pSessionDesc->seedSwCrc.swCrc32I;
+ } else if (CPA_DC_OVERFLOW != (Cpa8S)xlatErr) {
if (CPA_DC_CRC32 == pSessionDesc->checksumType) {
pResults->checksum =
pCompRespMsg->comp_resp_pars.crc.legacy
@@ -279,7 +337,7 @@
if ((CPA_DC_OVERFLOW != (Cpa8S)xlatErr) &&
(CPA_TRUE == verifyHwIntegrityCrcs)) {
pSessionDesc->previousChecksum =
- pSessionDesc->seedSwCrc.swCrcI;
+ pSessionDesc->seedSwCrc.swCrc32I;
}
/* Check if a CNV recovery happened and
@@ -292,7 +350,7 @@
pService);
}
- if (CPA_TRUE == pSessionDesc->isDcDp) {
+ if (CPA_TRUE == isDcDp) {
if (pResponse)
pResponse->responseStatus = CPA_STATUS_SUCCESS;
} else {
@@ -305,8 +363,26 @@
}
}
} else {
+#ifdef ICP_DC_RETURN_COUNTERS_ON_ERROR
+ /* Extract the response from the firmware */
+ pResults->consumed =
+ pCompRespMsg->comp_resp_pars.input_byte_counter;
+ pResults->produced =
+ pCompRespMsg->comp_resp_pars.output_byte_counter;
+
+ if (CPA_DC_STATEFUL == pSessionDesc->sessState) {
+ pSessionDesc->cumulativeConsumedBytes +=
+ pResults->consumed;
+ } else {
+ /* In the stateless case all requests have both SOP and
+ * EOP set */
+ pSessionDesc->cumulativeConsumedBytes =
+ pResults->consumed;
+ }
+#else
pResults->consumed = 0;
pResults->produced = 0;
+#endif
if (CPA_DC_OVERFLOW == pResults->status &&
CPA_DC_STATELESS == pSessionDesc->sessState) {
/* This error message will be returned by Data Plane API
@@ -319,7 +395,7 @@
"Unrecoverable error: stateless overflow. You may need to increase the size of your destination buffer.\n");
}
- if (CPA_TRUE == pSessionDesc->isDcDp) {
+ if (CPA_TRUE == isDcDp) {
if (pResponse)
pResponse->responseStatus = CPA_STATUS_FAIL;
} else {
@@ -338,7 +414,7 @@
}
}
- if (CPA_TRUE == pSessionDesc->isDcDp) {
+ if (CPA_TRUE == isDcDp) {
/* Decrement number of stateless pending callbacks for session
*/
pSessionDesc->pendingDpStatelessCbCount--;
@@ -383,7 +459,7 @@
* @retval CPA_STATUS_INVALID_PARAM Invalid parameter passed in
*
*****************************************************************************/
-static CpaStatus
+CpaStatus
dcCheckOpData(sal_compression_service_t *pService, CpaDcOpData *pOpData)
{
CpaDcSkipMode skipMode = 0;
@@ -440,6 +516,14 @@
"supported on this device");
return CPA_STATUS_INVALID_PARAM;
}
+
+ if (CPA_TRUE == pOpData->integrityCrcCheck &&
+ NULL == pOpData->pCrcData) {
+ LAC_INVALID_PARAM_LOG("Integrity CRC data structure "
+ "not intialized in CpaDcOpData");
+ return CPA_STATUS_INVALID_PARAM;
+ }
+
return CPA_STATUS_SUCCESS;
}
@@ -583,8 +667,9 @@
if (CPA_DC_HT_FULL_DYNAMIC == pSessionDesc->huffType) {
/* Check if intermediate buffers are supported */
- if ((0 == pService->pInterBuffPtrsArrayPhyAddr) ||
- (NULL == pService->pInterBuffPtrsArray)) {
+ if ((isDcGen2x(pService)) &&
+ ((0 == pService->pInterBuffPtrsArrayPhyAddr) ||
+ (NULL == pService->pInterBuffPtrsArray))) {
LAC_LOG_ERROR(
"No intermediate buffer defined for this instance "
"- see cpaDcStartInstance");
@@ -702,6 +787,7 @@
Cpa8U crcMode = ICP_QAT_FW_COMP_CRC_MODE_LEGACY;
Cpa8U cnvDecompReq = ICP_QAT_FW_COMP_NO_CNV;
Cpa8U cnvRecovery = ICP_QAT_FW_COMP_NO_CNV_RECOVERY;
+ CpaBoolean cnvErrorInjection = ICP_QAT_FW_COMP_NO_CNV_DFX;
CpaBoolean integrityCrcCheck = CPA_FALSE;
CpaStatus status = CPA_STATUS_SUCCESS;
CpaDcFlush flush = CPA_DC_FLUSH_NONE;
@@ -757,7 +843,7 @@
* bigger as allocated by the user. We ensure that this is not the case
* in dcCheckSourceData and cast the values to Cpa32U here */
pCookie->srcTotalDataLenInBytes = (Cpa32U)srcTotalDataLenInBytes;
- if ((DC_COMPRESSION_REQUEST == compDecomp) &&
+ if ((isDcGen2x(pService)) && (DC_COMPRESSION_REQUEST == compDecomp) &&
(CPA_DC_HT_FULL_DYNAMIC == pSessionDesc->huffType)) {
if (pService->minInterBuffSizeInBytes <
(Cpa32U)dstTotalDataLenInBytes) {
@@ -813,9 +899,9 @@
initial_crc32 = 0;
if (CPA_DC_ADLER32 == pSessionDesc->checksumType) {
- pSessionDesc->previousChecksum = 1;
+ pSessionDesc->previousChecksum = initial_adler;
} else {
- pSessionDesc->previousChecksum = 0;
+ pSessionDesc->previousChecksum = initial_crc32;
}
} else if (CPA_DC_STATELESS == pSessionDesc->sessState) {
pSessionDesc->previousChecksum = pResults->checksum;
@@ -906,7 +992,6 @@
eop = ICP_QAT_FW_COMP_NOT_EOP;
}
} else {
-
if (DC_REQUEST_FIRST == pSessionDesc->requestType) {
/* Reinitialise the cumulative amount of consumed bytes
*/
@@ -928,6 +1013,9 @@
* cnvDecompReq also needs to be set */
case DC_CNV:
cnvDecompReq = ICP_QAT_FW_COMP_CNV;
+ if (isDcGen4x(pService)) {
+ cnvErrorInjection = pSessionDesc->cnvErrorInjection;
+ }
break;
case DC_NO_CNV:
cnvDecompReq = ICP_QAT_FW_COMP_NO_CNV;
@@ -936,8 +1024,14 @@
}
/* LW 18 */
- rpCmdFlags = ICP_QAT_FW_COMP_REQ_PARAM_FLAGS_BUILD(
- sop, eop, bFinal, cnvDecompReq, cnvRecovery, crcMode);
+ rpCmdFlags = ICP_QAT_FW_COMP_REQ_PARAM_FLAGS_BUILD(sop,
+ eop,
+ bFinal,
+ cnvDecompReq,
+ cnvRecovery,
+ cnvErrorInjection,
+ crcMode);
+
pMsg->comp_pars.req_par_flags = rpCmdFlags;
/* Populates the QAT common request middle part of the message
@@ -1244,10 +1338,7 @@
COMPRESSION_STAT_INC(numDecompCompleted, pService);
}
- if (CPA_STATUS_SUCCESS !=
- LAC_SPINUNLOCK(&(pSessionDesc->sessionLock))) {
- LAC_LOG_ERROR("Cannot unlock session lock");
- }
+ LAC_SPINUNLOCK(&(pSessionDesc->sessionLock));
if ((NULL != pCbFunc) &&
(LacSync_GenWakeupSyncCaller != pCbFunc)) {
@@ -1371,7 +1462,7 @@
return dcCompDecompData(pService,
pSessionDesc,
- dcInstance,
+ insHandle,
pSessionHandle,
pSrcBuff,
pDestBuff,
@@ -1506,10 +1597,7 @@
if (CPA_DC_STATEFUL == pSessionDesc->sessState) {
/* Lock the session to check if there are in-flight stateful
* requests */
- if (CPA_STATUS_SUCCESS !=
- LAC_SPINLOCK(&(pSessionDesc->sessionLock))) {
- LAC_LOG_ERROR("Cannot unlock session lock");
- }
+ LAC_SPINLOCK(&(pSessionDesc->sessionLock));
/* Check if there is already one in-flight stateful request */
if (0 !=
@@ -1517,10 +1605,7 @@
&(pSessionDesc->pendingStatefulCbCount))) {
LAC_LOG_ERROR(
"Only one in-flight stateful request supported");
- if (CPA_STATUS_SUCCESS !=
- LAC_SPINUNLOCK(&(pSessionDesc->sessionLock))) {
- LAC_LOG_ERROR("Cannot unlock session lock");
- }
+ LAC_SPINUNLOCK(&(pSessionDesc->sessionLock));
return CPA_STATUS_RETRY;
}
@@ -1537,10 +1622,7 @@
}
qatUtilsAtomicInc(&(pSessionDesc->pendingStatefulCbCount));
- if (CPA_STATUS_SUCCESS !=
- LAC_SPINUNLOCK(&(pSessionDesc->sessionLock))) {
- LAC_LOG_ERROR("Cannot unlock session lock");
- }
+ LAC_SPINUNLOCK(&(pSessionDesc->sessionLock));
}
if (CPA_TRUE == pOpData->compressAndVerify) {
@@ -1549,7 +1631,7 @@
return dcCompDecompData(pService,
pSessionDesc,
- dcInstance,
+ insHandle,
pSessionHandle,
pSrcBuff,
pDestBuff,
@@ -1659,16 +1741,50 @@
pService = (sal_compression_service_t *)insHandle;
+ /* Check if SAL is initialised otherwise return an error */
+ SAL_RUNNING_CHECK(insHandle);
+
+ /* This check is outside the parameter checking as it is needed to
+ * manage zero length requests */
+ if (CPA_STATUS_SUCCESS !=
+ LacBuffDesc_BufferListVerifyNull(pSrcBuff,
+ &srcBuffSize,
+ LAC_NO_ALIGNMENT_SHIFT)) {
+ QAT_UTILS_LOG("Invalid source buffer list parameter");
+ return CPA_STATUS_INVALID_PARAM;
+ }
+
+ /* Ensure this is a compression instance */
+ SAL_CHECK_INSTANCE_TYPE(insHandle, SAL_SERVICE_TYPE_COMPRESSION);
+
+ if (dcCheckSourceData(pSessionHandle,
+ pSrcBuff,
+ pDestBuff,
+ pResults,
+ flushFlag,
+ srcBuffSize,
+ NULL) != CPA_STATUS_SUCCESS) {
+ return CPA_STATUS_INVALID_PARAM;
+ }
+ if (dcCheckDestinationData(pService,
+ pSessionHandle,
+ pDestBuff,
+ DC_DECOMPRESSION_REQUEST) !=
+ CPA_STATUS_SUCCESS) {
+ return CPA_STATUS_INVALID_PARAM;
+ }
pSessionDesc = DC_SESSION_DESC_FROM_CTX_GET(pSessionHandle);
+ if (CPA_DC_DIR_COMPRESS == pSessionDesc->sessDirection) {
+ QAT_UTILS_LOG("Invalid sessDirection value");
+ return CPA_STATUS_INVALID_PARAM;
+ }
+
+
if (CPA_DC_STATEFUL == pSessionDesc->sessState) {
/* Lock the session to check if there are in-flight stateful
* requests */
- if (CPA_STATUS_SUCCESS !=
- LAC_SPINLOCK(&(pSessionDesc->sessionLock))) {
- LAC_LOG_ERROR("Cannot lock session lock");
- return CPA_STATUS_RESOURCE;
- }
+ LAC_SPINLOCK(&(pSessionDesc->sessionLock));
/* Check if there is already one in-flight stateful request */
if (0 !=
@@ -1676,37 +1792,36 @@
&(pSessionDesc->pendingStatefulCbCount))) {
LAC_LOG_ERROR(
"Only one in-flight stateful request supported");
- if (CPA_STATUS_SUCCESS !=
- LAC_SPINUNLOCK(&(pSessionDesc->sessionLock))) {
- LAC_LOG_ERROR("Cannot unlock session lock");
- }
+ LAC_SPINUNLOCK(&(pSessionDesc->sessionLock));
return CPA_STATUS_RETRY;
}
- if ((0 == srcBuffSize) ||
- ((1 == srcBuffSize) && (CPA_DC_FLUSH_FINAL != flushFlag) &&
- (CPA_DC_FLUSH_FULL != flushFlag))) {
- if (CPA_TRUE ==
- dcZeroLengthRequests(pService,
- pSessionDesc,
- pResults,
- flushFlag,
- callbackTag,
- DC_DECOMPRESSION_REQUEST)) {
- return CPA_STATUS_SUCCESS;
+ /* Gen 4 handle 0 len requests in FW */
+ if (isDcGen2x(pService)) {
+ if ((0 == srcBuffSize) ||
+ ((1 == srcBuffSize) &&
+ (CPA_DC_FLUSH_FINAL != flushFlag) &&
+ (CPA_DC_FLUSH_FULL != flushFlag))) {
+ if (CPA_TRUE ==
+ dcZeroLengthRequests(
+ pService,
+ pSessionDesc,
+ pResults,
+ flushFlag,
+ callbackTag,
+ DC_DECOMPRESSION_REQUEST)) {
+ return CPA_STATUS_SUCCESS;
+ }
}
}
qatUtilsAtomicInc(&(pSessionDesc->pendingStatefulCbCount));
- if (CPA_STATUS_SUCCESS !=
- LAC_SPINUNLOCK(&(pSessionDesc->sessionLock))) {
- LAC_LOG_ERROR("Cannot unlock session lock");
- }
+ LAC_SPINUNLOCK(&(pSessionDesc->sessionLock));
}
return dcCompDecompData(pService,
pSessionDesc,
- dcInstance,
+ insHandle,
pSessionHandle,
pSrcBuff,
pDestBuff,
@@ -1768,12 +1883,89 @@
pSessionDesc = DC_SESSION_DESC_FROM_CTX_GET(pSessionHandle);
- if (CPA_DC_STATEFUL == pSessionDesc->sessState) {
- LAC_INVALID_PARAM_LOG("Invalid session: Stateful session is "
- "not supported");
+ LAC_CHECK_NULL_PARAM(insHandle);
+
+ /* Check if SAL is initialised otherwise return an error */
+ SAL_RUNNING_CHECK(insHandle);
+
+ /* This check is outside the parameter checking as it is needed to
+ * manage zero length requests */
+ if (CPA_STATUS_SUCCESS !=
+ LacBuffDesc_BufferListVerifyNull(pSrcBuff,
+ &srcBuffSize,
+ LAC_NO_ALIGNMENT_SHIFT)) {
+ QAT_UTILS_LOG("Invalid source buffer list parameter");
+ return CPA_STATUS_INVALID_PARAM;
+ }
+
+ /* Ensure this is a compression instance */
+ SAL_CHECK_INSTANCE_TYPE(insHandle, SAL_SERVICE_TYPE_COMPRESSION);
+
+ if (CPA_STATUS_SUCCESS !=
+ dcCheckSourceData(pSessionHandle,
+ pSrcBuff,
+ pDestBuff,
+ pResults,
+ CPA_DC_FLUSH_NONE,
+ srcBuffSize,
+ NULL)) {
+ return CPA_STATUS_INVALID_PARAM;
+ }
+ if (CPA_STATUS_SUCCESS !=
+ dcCheckDestinationData(pService,
+ pSessionHandle,
+ pDestBuff,
+ DC_DECOMPRESSION_REQUEST)) {
return CPA_STATUS_INVALID_PARAM;
}
+ if (CPA_STATUS_SUCCESS != dcCheckOpData(pService, pOpData)) {
+ return CPA_STATUS_INVALID_PARAM;
+ }
+
+ if (CPA_DC_DIR_COMPRESS == pSessionDesc->sessDirection) {
+ QAT_UTILS_LOG("Invalid sessDirection value");
+ return CPA_STATUS_INVALID_PARAM;
+ }
+
+
+ if (CPA_DC_STATEFUL == pSessionDesc->sessState) {
+ /* Lock the session to check if there are in-flight stateful
+ * requests */
+ LAC_SPINLOCK(&(pSessionDesc->sessionLock));
+
+ /* Check if there is already one in-flight stateful request */
+ if (0 !=
+ qatUtilsAtomicGet(
+ &(pSessionDesc->pendingStatefulCbCount))) {
+ LAC_LOG_ERROR(
+ "Only one in-flight stateful request supported");
+ LAC_SPINUNLOCK(&(pSessionDesc->sessionLock));
+ return CPA_STATUS_RETRY;
+ }
+
+ /* Gen 4 handle 0 len requests in FW */
+ if (isDcGen2x(pService)) {
+ if ((0 == srcBuffSize) ||
+ ((1 == srcBuffSize) &&
+ (CPA_DC_FLUSH_FINAL != pOpData->flushFlag) &&
+ (CPA_DC_FLUSH_FULL != pOpData->flushFlag))) {
+ if (CPA_TRUE ==
+ dcZeroLengthRequests(
+ pService,
+ pSessionDesc,
+ pResults,
+ pOpData->flushFlag,
+ callbackTag,
+ DC_DECOMPRESSION_REQUEST)) {
+ return CPA_STATUS_SUCCESS;
+ }
+ }
+ }
+ qatUtilsAtomicInc(&(pSessionDesc->pendingStatefulCbCount));
+ LAC_SPINUNLOCK(&(pSessionDesc->sessionLock));
+ }
+
return dcCompDecompData(pService,
pSessionDesc,
insHandle,
Index: sys/dev/qat/qat_api/common/compression/dc_dp.c
===================================================================
--- sys/dev/qat/qat_api/common/compression/dc_dp.c
+++ sys/dev/qat/qat_api/common/compression/dc_dp.c
@@ -41,6 +41,7 @@
#include "sal_service_state.h"
#include "sal_qat_cmn_msg.h"
#include "icp_sal_poll.h"
+#include "sal_hw_gen.h"
/**
*****************************************************************************
@@ -87,8 +88,8 @@
/* Compressing zero byte is not supported */
if ((CPA_DC_DIR_COMPRESS == pSessionDesc->sessDirection) &&
(0 == pOpData->bufferLenToCompress)) {
- QAT_UTILS_LOG(
- "The source buffer length to compress needs to be greater than zero byte.\n");
+ QAT_UTILS_LOG("The source buffer length to compress needs to "
+ "be greater than zero byte.\n");
return CPA_STATUS_INVALID_PARAM;
}
@@ -171,8 +172,7 @@
} else {
/* We are assuming that there is enough memory in the source and
* destination buffer lists. We only receive physical addresses
- * of the
- * buffers so we are unable to test it here */
+ * of the buffers so we are unable to test it here */
LAC_CHECK_8_BYTE_ALIGNMENT(pOpData->srcBuffer);
LAC_CHECK_8_BYTE_ALIGNMENT(pOpData->destBuffer);
}
@@ -183,8 +183,9 @@
(CPA_DC_DIR_COMBINED == pSessionDesc->sessDirection)) {
if (CPA_DC_HT_FULL_DYNAMIC == pSessionDesc->huffType) {
/* Check if Intermediate Buffer Array pointer is NULL */
- if ((0 == pService->pInterBuffPtrsArrayPhyAddr) ||
- (NULL == pService->pInterBuffPtrsArray)) {
+ if (isDcGen2x(pService) &&
+ ((0 == pService->pInterBuffPtrsArrayPhyAddr) ||
+ (NULL == pService->pInterBuffPtrsArray))) {
QAT_UTILS_LOG(
"No intermediate buffer defined for this instance - see cpaDcStartInstance.\n");
return CPA_STATUS_INVALID_PARAM;
@@ -312,7 +313,10 @@
Cpa8U cnvDecompReq = ICP_QAT_FW_COMP_NO_CNV;
Cpa8U cnvnrCompReq = ICP_QAT_FW_COMP_NO_CNV_RECOVERY;
+ CpaBoolean cnvErrorInjection = ICP_QAT_FW_COMP_NO_CNV_DFX;
+ sal_compression_service_t *pService = NULL;
+ pService = (sal_compression_service_t *)(pOpData->dcInstance);
pSessionDesc = DC_SESSION_DESC_FROM_CTX_GET(pOpData->pSessionHandle);
if (CPA_DC_DIR_COMPRESS == pOpData->sessDirection) {
@@ -320,6 +324,11 @@
/* CNV check */
if (CPA_TRUE == pOpData->compressAndVerify) {
cnvDecompReq = ICP_QAT_FW_COMP_CNV;
+ if (isDcGen4x(pService)) {
+ cnvErrorInjection =
+ pSessionDesc->cnvErrorInjection;
+ }
+
/* CNVNR check */
if (CPA_TRUE == pOpData->compressAndVerifyAndRecover) {
cnvnrCompReq = ICP_QAT_FW_COMP_CNV_RECOVERY;
@@ -343,7 +352,13 @@
pCurrentQatMsg->comp_pars.req_par_flags |=
ICP_QAT_FW_COMP_REQ_PARAM_FLAGS_BUILD(
- 0, 0, 0, cnvDecompReq, cnvnrCompReq, 0);
+ ICP_QAT_FW_COMP_NOT_SOP,
+ ICP_QAT_FW_COMP_NOT_EOP,
+ ICP_QAT_FW_COMP_NOT_BFINAL,
+ cnvDecompReq,
+ cnvnrCompReq,
+ cnvErrorInjection,
+ ICP_QAT_FW_COMP_CRC_MODE_LEGACY);
SalQatMsg_CmnMidWrite((icp_qat_fw_la_bulk_req_t *)pCurrentQatMsg,
pOpData,
Index: sys/dev/qat/qat_api/common/compression/dc_session.c
===================================================================
--- sys/dev/qat/qat_api/common/compression/dc_session.c
+++ sys/dev/qat/qat_api/common/compression/dc_session.c
@@ -23,6 +23,7 @@
#include "icp_qat_fw.h"
#include "icp_qat_fw_comp.h"
#include "icp_qat_hw.h"
+#include "icp_qat_hw_20_comp.h"
/*
*******************************************************************************
@@ -36,6 +37,7 @@
#include "lac_buffer_desc.h"
#include "sal_service_state.h"
#include "sal_qat_cmn_msg.h"
+#include "sal_hw_gen.h"
/**
*****************************************************************************
@@ -54,7 +56,7 @@
* @retval CPA_STATUS_UNSUPPORTED Unsupported algorithm/feature
*
*****************************************************************************/
-static CpaStatus
+CpaStatus
dcCheckSessionData(const CpaDcSessionSetupData *pSessionData,
CpaInstanceHandle dcInstance)
{
@@ -67,6 +69,7 @@
QAT_UTILS_LOG("Invalid compLevel value\n");
return CPA_STATUS_INVALID_PARAM;
}
+
if ((pSessionData->autoSelectBestHuffmanTree < CPA_DC_ASB_DISABLED) ||
(pSessionData->autoSelectBestHuffmanTree >
CPA_DC_ASB_UNCOMP_STATIC_DYNAMIC_WITH_NO_HDRS)) {
@@ -122,10 +125,10 @@
*
*****************************************************************************/
static void
-dcCompHwBlockPopulate(dc_session_desc_t *pSessionDesc,
+dcCompHwBlockPopulate(sal_compression_service_t *pService,
+ dc_session_desc_t *pSessionDesc,
icp_qat_hw_compression_config_t *pCompConfig,
- dc_request_dir_t compDecomp,
- icp_qat_hw_compression_delayed_match_t enableDmm)
+ dc_request_dir_t compDecomp)
{
icp_qat_hw_compression_direction_t dir =
ICP_QAT_HW_COMPRESSION_DIR_COMPRESS;
@@ -134,6 +137,7 @@
icp_qat_hw_compression_depth_t depth = ICP_QAT_HW_COMPRESSION_DEPTH_1;
icp_qat_hw_compression_file_type_t filetype =
ICP_QAT_HW_COMPRESSION_FILE_TYPE_0;
+ icp_qat_hw_compression_delayed_match_t dmm;
/* Set the direction */
if (DC_COMPRESSION_REQUEST == compDecomp) {
@@ -148,6 +152,13 @@
QAT_UTILS_LOG("Algorithm not supported for Compression\n");
}
+ /* Set delay match mode */
+ if (CPA_TRUE == pService->comp_device_data.enableDmm) {
+ dmm = ICP_QAT_HW_COMPRESSION_DELAYED_MATCH_ENABLED;
+ } else {
+ dmm = ICP_QAT_HW_COMPRESSION_DELAYED_MATCH_DISABLED;
+ }
+
/* Set the depth */
if (DC_DECOMPRESSION_REQUEST == compDecomp) {
depth = ICP_QAT_HW_COMPRESSION_DEPTH_1;
@@ -162,8 +173,13 @@
case CPA_DC_L3:
depth = ICP_QAT_HW_COMPRESSION_DEPTH_8;
break;
- default:
+ case CPA_DC_L4:
depth = ICP_QAT_HW_COMPRESSION_DEPTH_16;
+ break;
+ default:
+ depth = pService->comp_device_data
+ .highestHwCompressionDepth;
+ break;
}
}
@@ -171,10 +187,138 @@
* modes will be used in the future for precompiled huffman trees */
filetype = ICP_QAT_HW_COMPRESSION_FILE_TYPE_0;
- pCompConfig->val = ICP_QAT_HW_COMPRESSION_CONFIG_BUILD(
- dir, enableDmm, algo, depth, filetype);
+ pCompConfig->lower_val = ICP_QAT_HW_COMPRESSION_CONFIG_BUILD(
+ dir, dmm, algo, depth, filetype);
+
+ /* Upper 32-bits of the configuration word do not need to be
+ * configured with legacy devices.
+ */
+ pCompConfig->upper_val = 0;
+}
+
+static void
+dcCompHwBlockPopulateGen4(sal_compression_service_t *pService,
+ dc_session_desc_t *pSessionDesc,
+ icp_qat_hw_compression_config_t *pCompConfig,
+ dc_request_dir_t compDecomp)
+{
+ /* Compression related */
+ if (DC_COMPRESSION_REQUEST == compDecomp) {
+ icp_qat_hw_comp_20_config_csr_upper_t hw_comp_upper_csr;
+ icp_qat_hw_comp_20_config_csr_lower_t hw_comp_lower_csr;
+
+ memset(&hw_comp_upper_csr, 0, sizeof hw_comp_upper_csr);
+ memset(&hw_comp_lower_csr, 0, sizeof hw_comp_lower_csr);
+
+ /* Disable Literal + Length Limit Block Drop by default and
+ * enable it only for dynamic deflate compression.
+ */
+ hw_comp_lower_csr.lllbd =
+ ICP_QAT_HW_COMP_20_LLLBD_CTRL_LLLBD_DISABLED;
+
+ switch (pSessionDesc->compType) {
+ case CPA_DC_DEFLATE:
+ /* DEFLATE algorithm settings */
+ hw_comp_lower_csr.skip_ctrl =
+ ICP_QAT_HW_COMP_20_BYTE_SKIP_3BYTE_LITERAL;
+
+ if (CPA_DC_HT_FULL_DYNAMIC == pSessionDesc->huffType) {
+ hw_comp_lower_csr.algo =
+ ICP_QAT_HW_COMP_20_HW_COMP_FORMAT_ILZ77;
+ } else /* Static DEFLATE */
+ {
+ hw_comp_lower_csr.algo =
+ ICP_QAT_HW_COMP_20_HW_COMP_FORMAT_DEFLATE;
+ hw_comp_upper_csr.scb_ctrl =
+ ICP_QAT_HW_COMP_20_SCB_CONTROL_DISABLE;
+ }
+
+ if (CPA_DC_STATEFUL == pSessionDesc->sessState) {
+ hw_comp_upper_csr.som_ctrl =
+ ICP_QAT_HW_COMP_20_SOM_CONTROL_REPLAY_MODE;
+ }
+ break;
+ default:
+ QAT_UTILS_LOG("Compression algorithm not supported\n");
+ break;
+ }
+ /* Set the search depth */
+ switch (pSessionDesc->compLevel) {
+ case CPA_DC_L1:
+ case CPA_DC_L2:
+ case CPA_DC_L3:
+ case CPA_DC_L4:
+ case CPA_DC_L5:
+ hw_comp_lower_csr.sd =
+ ICP_QAT_HW_COMP_20_SEARCH_DEPTH_LEVEL_1;
+ hw_comp_lower_csr.hash_col =
+ ICP_QAT_HW_COMP_20_SKIP_HASH_COLLISION_DONT_ALLOW;
+ break;
+ case CPA_DC_L6:
+ case CPA_DC_L7:
+ case CPA_DC_L8:
+ hw_comp_lower_csr.sd =
+ ICP_QAT_HW_COMP_20_SEARCH_DEPTH_LEVEL_6;
+ break;
+ case CPA_DC_L9:
+ hw_comp_lower_csr.sd =
+ ICP_QAT_HW_COMP_20_SEARCH_DEPTH_LEVEL_9;
+ break;
+ default:
+ hw_comp_lower_csr.sd = pService->comp_device_data
+ .highestHwCompressionDepth;
+ if ((CPA_DC_HT_FULL_DYNAMIC ==
+ pSessionDesc->huffType) &&
+ (CPA_DC_DEFLATE == pSessionDesc->compType)) {
+ /* Enable Literal + Length Limit Block Drop
+ * with dynamic deflate compression when
+ * highest compression levels are selected.
+ */
+ hw_comp_lower_csr.lllbd =
+ ICP_QAT_HW_COMP_20_LLLBD_CTRL_LLLBD_ENABLED;
+ }
+ break;
+ }
+ /* Same for all algorithms */
+ hw_comp_lower_csr.abd = ICP_QAT_HW_COMP_20_ABD_ABD_DISABLED;
+ hw_comp_lower_csr.hash_update =
+ ICP_QAT_HW_COMP_20_SKIP_HASH_UPDATE_DONT_ALLOW;
+ hw_comp_lower_csr.edmm =
+ (CPA_TRUE == pService->comp_device_data.enableDmm) ?
+ ICP_QAT_HW_COMP_20_EXTENDED_DELAY_MATCH_MODE_EDMM_ENABLED :
+ ICP_QAT_HW_COMP_20_EXTENDED_DELAY_MATCH_MODE_EDMM_DISABLED;
+
+ /* Hard-coded HW-specific values */
+ hw_comp_upper_csr.nice =
+ ICP_QAT_HW_COMP_20_CONFIG_CSR_NICE_PARAM_DEFAULT_VAL;
+ hw_comp_upper_csr.lazy =
+ ICP_QAT_HW_COMP_20_CONFIG_CSR_LAZY_PARAM_DEFAULT_VAL;
+
+ pCompConfig->upper_val =
+ ICP_QAT_FW_COMP_20_BUILD_CONFIG_UPPER(hw_comp_upper_csr);
+
+ pCompConfig->lower_val =
+ ICP_QAT_FW_COMP_20_BUILD_CONFIG_LOWER(hw_comp_lower_csr);
+ } else /* Decompress */
+ {
+ icp_qat_hw_decomp_20_config_csr_lower_t hw_decomp_lower_csr;
+
+ memset(&hw_decomp_lower_csr, 0, sizeof hw_decomp_lower_csr);
+
+ /* Set the algorithm */
+ if (CPA_DC_DEFLATE == pSessionDesc->compType) {
+ hw_decomp_lower_csr.algo =
+ ICP_QAT_HW_DECOMP_20_HW_DECOMP_FORMAT_DEFLATE;
+ } else {
+ QAT_UTILS_LOG("Algorithm not supported for "
+ "Decompression\n");
+ }
- pCompConfig->reserved = 0;
+ pCompConfig->upper_val = 0;
+ pCompConfig->lower_val =
+ ICP_QAT_FW_DECOMP_20_BUILD_CONFIG_LOWER(
+ hw_decomp_lower_csr);
+ }
}
/**
@@ -268,10 +412,19 @@
pCompControlBlock->resrvd = 0;
/* Populate Compression Hardware Setup Block */
- dcCompHwBlockPopulate(pSessionDesc,
- pCompConfig,
- compDecomp,
- pService->comp_device_data.enableDmm);
+ if (isDcGen4x(pService)) {
+ dcCompHwBlockPopulateGen4(pService,
+ pSessionDesc,
+ pCompConfig,
+ compDecomp);
+ } else if (isDcGen2x(pService)) {
+ dcCompHwBlockPopulate(pService,
+ pSessionDesc,
+ pCompConfig,
+ compDecomp);
+ } else {
+ QAT_UTILS_LOG("Invalid QAT generation value\n");
+ }
}
/**
@@ -286,7 +439,7 @@
* @param[in] nextSlice Next slice
*
*****************************************************************************/
-static void
+void
dcTransContentDescPopulate(icp_qat_fw_comp_req_t *pMsg,
icp_qat_fw_slice_t nextSlice)
{
@@ -333,14 +486,70 @@
*pContextSize = 0;
if ((CPA_DC_STATEFUL == pSessionData->sessState) &&
- (CPA_DC_DEFLATE == pSessionData->compType) &&
(CPA_DC_DIR_COMPRESS != pSessionData->sessDirection)) {
- *pContextSize =
- pCompService->comp_device_data.inflateContextSize;
+ switch (pSessionData->compType) {
+ case CPA_DC_DEFLATE:
+ *pContextSize =
+ pCompService->comp_device_data.inflateContextSize;
+ break;
+ default:
+ QAT_UTILS_LOG("Invalid compression algorithm.");
+ return CPA_STATUS_FAIL;
+ }
}
return CPA_STATUS_SUCCESS;
}
+CpaStatus
+dcGetCompressCommandId(sal_compression_service_t *pService,
+ CpaDcSessionSetupData *pSessionData,
+ Cpa8U *pDcCmdId)
+{
+ CpaStatus status = CPA_STATUS_SUCCESS;
+ LAC_CHECK_NULL_PARAM(pService);
+ LAC_CHECK_NULL_PARAM(pSessionData);
+ LAC_CHECK_NULL_PARAM(pDcCmdId);
+
+ switch (pSessionData->compType) {
+ case CPA_DC_DEFLATE:
+ *pDcCmdId = (CPA_DC_HT_FULL_DYNAMIC == pSessionData->huffType) ?
+ ICP_QAT_FW_COMP_CMD_DYNAMIC :
+ ICP_QAT_FW_COMP_CMD_STATIC;
+ break;
+ default:
+ QAT_UTILS_LOG("Algorithm not supported for "
+ "compression\n");
+ status = CPA_STATUS_UNSUPPORTED;
+ break;
+ }
+
+ return status;
+}
+
+CpaStatus
+dcGetDecompressCommandId(sal_compression_service_t *pService,
+ CpaDcSessionSetupData *pSessionData,
+ Cpa8U *pDcCmdId)
+{
+ CpaStatus status = CPA_STATUS_SUCCESS;
+ LAC_CHECK_NULL_PARAM(pService);
+ LAC_CHECK_NULL_PARAM(pSessionData);
+ LAC_CHECK_NULL_PARAM(pDcCmdId);
+
+ switch (pSessionData->compType) {
+ case CPA_DC_DEFLATE:
+ *pDcCmdId = ICP_QAT_FW_COMP_CMD_DECOMPRESS;
+ break;
+ default:
+ QAT_UTILS_LOG("Algorithm not supported for "
+ "decompression\n");
+ status = CPA_STATUS_UNSUPPORTED;
+ break;
+ }
+
+ return status;
+}
+
CpaStatus
dcInitSession(CpaInstanceHandle dcInstance,
CpaDcSessionHandle pSessionHandle,
@@ -394,7 +603,17 @@
return CPA_STATUS_UNSUPPORTED;
}
- if (CPA_DC_HT_FULL_DYNAMIC == pSessionData->huffType) {
+ /* Check for Gen4 and stateful, return error if both exist */
+ if ((isDcGen4x(pService)) &&
+ (CPA_DC_STATEFUL == pSessionData->sessState &&
+ CPA_DC_DIR_DECOMPRESS != pSessionData->sessDirection)) {
+ QAT_UTILS_LOG("Stateful sessions are not supported for "
+ "compression direction");
+ return CPA_STATUS_UNSUPPORTED;
+ }
+
+ if ((isDcGen2x(pService)) &&
+ (CPA_DC_HT_FULL_DYNAMIC == pSessionData->huffType)) {
/* Test if DRAM is available for the intermediate buffers */
if ((NULL == pService->pInterBuffPtrsArray) &&
(0 == pService->pInterBuffPtrsArrayPhyAddr)) {
@@ -404,7 +623,8 @@
pSessionData->huffType = CPA_DC_HT_STATIC;
} else {
QAT_UTILS_LOG(
- "No buffer defined for this instance - see cpaDcStartInstance.\n");
+ "No buffer defined for this instance - "
+ "see cpaDcStartInstance.\n");
return CPA_STATUS_RESOURCE;
}
}
@@ -541,7 +761,8 @@
pSessionDesc->pendingDpStatelessCbCount = 0;
if (CPA_DC_DIR_DECOMPRESS != pSessionData->sessDirection) {
- if (CPA_DC_HT_FULL_DYNAMIC == pSessionData->huffType) {
+ if ((isDcGen2x(pService)) &&
+ CPA_DC_HT_FULL_DYNAMIC == pSessionData->huffType) {
/* Populate the compression section of the content
* descriptor */
dcCompContentDescPopulate(pService,
@@ -607,17 +828,29 @@
pDataIntegrityCrcs = &pSessionDesc->dataIntegrityCrcs;
pDataIntegrityCrcs->crc32 = 0;
pDataIntegrityCrcs->adler32 = 1;
- pDataIntegrityCrcs->oCrc32Cpr = DC_INVALID_CRC;
- pDataIntegrityCrcs->iCrc32Cpr = DC_INVALID_CRC;
- pDataIntegrityCrcs->oCrc32Xlt = DC_INVALID_CRC;
- pDataIntegrityCrcs->iCrc32Xlt = DC_INVALID_CRC;
- pDataIntegrityCrcs->xorFlags = DC_XOR_FLAGS_DEFAULT;
- pDataIntegrityCrcs->crcPoly = DC_CRC_POLY_DEFAULT;
- pDataIntegrityCrcs->xorOut = DC_XOR_OUT_DEFAULT;
-
- /* Initialise seed checksums */
- pSessionDesc->seedSwCrc.swCrcI = 0;
- pSessionDesc->seedSwCrc.swCrcO = 0;
+
+ if (isDcGen2x(pService)) {
+ pDataIntegrityCrcs->oCrc32Cpr = DC_INVALID_CRC;
+ pDataIntegrityCrcs->iCrc32Cpr = DC_INVALID_CRC;
+ pDataIntegrityCrcs->oCrc32Xlt = DC_INVALID_CRC;
+ pDataIntegrityCrcs->iCrc32Xlt = DC_INVALID_CRC;
+ pDataIntegrityCrcs->xorFlags = DC_XOR_FLAGS_DEFAULT;
+ pDataIntegrityCrcs->crcPoly = DC_CRC_POLY_DEFAULT;
+ pDataIntegrityCrcs->xorOut = DC_XOR_OUT_DEFAULT;
+ } else {
+ pDataIntegrityCrcs->iCrc64Cpr = DC_INVALID_CRC;
+ pDataIntegrityCrcs->oCrc64Cpr = DC_INVALID_CRC;
+ pDataIntegrityCrcs->iCrc64Xlt = DC_INVALID_CRC;
+ pDataIntegrityCrcs->oCrc64Xlt = DC_INVALID_CRC;
+ pDataIntegrityCrcs->crc64Poly = DC_CRC64_POLY_DEFAULT;
+ pDataIntegrityCrcs->xor64Out = DC_XOR64_OUT_DEFAULT;
+ }
+
+ /* Initialise seed checksums.
+ * It initializes swCrc32I, swCrc32O, too(union).
+ */
+ pSessionDesc->seedSwCrc.swCrc64I = 0;
+ pSessionDesc->seedSwCrc.swCrc64O = 0;
/* Populate the cmdFlags */
switch (pSessionDesc->autoSelectBestHuffmanTree) {
@@ -646,6 +879,7 @@
ICP_QAT_FW_COMP_BFINAL,
ICP_QAT_FW_COMP_NO_CNV,
ICP_QAT_FW_COMP_NO_CNV_RECOVERY,
+ ICP_QAT_FW_COMP_NO_CNV_DFX,
ICP_QAT_FW_COMP_CRC_MODE_LEGACY);
cmdFlags =
@@ -656,11 +890,16 @@
secureRam);
if (CPA_DC_DIR_DECOMPRESS != pSessionData->sessDirection) {
- if (CPA_DC_HT_FULL_DYNAMIC == pSessionDesc->huffType) {
- dcCmdId = (icp_qat_fw_la_cmd_id_t)(
- ICP_QAT_FW_COMP_CMD_DYNAMIC);
- }
+ status = dcGetCompressCommandId(pService,
+ pSessionData,
+ (Cpa8U *)&dcCmdId);
+ if (CPA_STATUS_SUCCESS != status) {
+ QAT_UTILS_LOG(
+ "Couldn't get compress command ID for current "
+ "session data.");
+ return status;
+ }
pReqCache = &(pSessionDesc->reqCacheComp);
pReqCache->comp_pars.req_par_flags = rpCmdFlags;
pReqCache->comp_pars.crc.legacy.initial_adler = 1;
@@ -675,8 +914,16 @@
}
if (CPA_DC_DIR_COMPRESS != pSessionData->sessDirection) {
- dcCmdId =
- (icp_qat_fw_la_cmd_id_t)(ICP_QAT_FW_COMP_CMD_DECOMPRESS);
+ status = dcGetDecompressCommandId(pService,
+ pSessionData,
+ (Cpa8U *)&dcCmdId);
+ if (CPA_STATUS_SUCCESS != status) {
+ QAT_UTILS_LOG(
+ "Couldn't get decompress command ID for current "
+ "session data.");
+
+ return status;
+ }
pReqCache = &(pSessionDesc->reqCacheDecomp);
pReqCache->comp_pars.req_par_flags = rpCmdFlags;
pReqCache->comp_pars.crc.legacy.initial_adler = 1;
@@ -730,10 +977,14 @@
{
CpaStatus status = CPA_STATUS_SUCCESS;
CpaInstanceHandle insHandle = NULL;
+ sal_compression_service_t *pService = NULL;
dc_session_desc_t *pSessionDesc = NULL;
Cpa64U numPendingStateless = 0;
Cpa64U numPendingStateful = 0;
icp_comms_trans_handle trans_handle = NULL;
+ dc_integrity_crc_fw_t *pDataIntegrityCrcs = NULL;
+ dc_sw_checksums_t *pSwCrcs = NULL;
+
LAC_CHECK_NULL_PARAM(pSessionHandle);
pSessionDesc = DC_SESSION_DESC_FROM_CTX_GET(pSessionHandle);
LAC_CHECK_NULL_PARAM(pSessionDesc);
@@ -752,7 +1003,7 @@
/* Check if SAL is running otherwise return an error */
SAL_RUNNING_CHECK(insHandle);
if (CPA_TRUE == pSessionDesc->isDcDp) {
- trans_handle = ((sal_compression_service_t *)dcInstance)
+ trans_handle = ((sal_compression_service_t *)insHandle)
->trans_handle_compression_tx;
if (CPA_TRUE == icp_adf_queueDataToSend(trans_handle)) {
/* Process the remaining messages on the ring */
@@ -798,7 +1049,32 @@
} else {
pSessionDesc->previousChecksum = 0;
}
+ pSessionDesc->cnvErrorInjection = ICP_QAT_FW_COMP_NO_CNV_DFX;
+
+ /* Reset integrity CRCs to default parameters. */
+ pDataIntegrityCrcs = &pSessionDesc->dataIntegrityCrcs;
+ memset(pDataIntegrityCrcs, 0, sizeof(dc_integrity_crc_fw_t));
+ pDataIntegrityCrcs->adler32 = 1;
+
+ pService = (sal_compression_service_t *)insHandle;
+ if (isDcGen2x(pService)) {
+ pDataIntegrityCrcs->xorFlags = DC_XOR_FLAGS_DEFAULT;
+ pDataIntegrityCrcs->crcPoly = DC_CRC_POLY_DEFAULT;
+ pDataIntegrityCrcs->xorOut = DC_XOR_OUT_DEFAULT;
+ } else {
+ pDataIntegrityCrcs->crc64Poly = DC_CRC64_POLY_DEFAULT;
+ pDataIntegrityCrcs->xor64Out = DC_XOR64_OUT_DEFAULT;
+ }
+
+ /* Reset seed SW checksums. */
+ pSwCrcs = &pSessionDesc->seedSwCrc;
+ memset(pSwCrcs, 0, sizeof(dc_sw_checksums_t));
+
+ /* Reset integrity SW checksums. */
+ pSwCrcs = &pSessionDesc->integritySwCrc;
+ memset(pSwCrcs, 0, sizeof(dc_sw_checksums_t));
}
+
/* Reset the pending callback counters */
qatUtilsAtomicSet(0, &pSessionDesc->pendingStatelessCbCount);
qatUtilsAtomicSet(0, &pSessionDesc->pendingStatefulCbCount);
@@ -886,12 +1162,7 @@
}
if ((CPA_DC_STATEFUL == pSessionDesc->sessState) &&
(CPA_STATUS_SUCCESS == status)) {
- if (CPA_STATUS_SUCCESS !=
- LAC_SPINLOCK_DESTROY(
- &(pSessionDesc->sessionLock))) {
- QAT_UTILS_LOG(
- "Failed to destory session lock.\n");
- }
+ LAC_SPINLOCK_DESTROY(&(pSessionDesc->sessionLock));
}
}
@@ -955,3 +1226,33 @@
pSessionSize,
pContextSize);
}
+
+CpaStatus
+dcSetCnvError(CpaInstanceHandle dcInstance, CpaDcSessionHandle pSessionHandle)
+{
+ LAC_CHECK_NULL_PARAM(pSessionHandle);
+
+ dc_session_desc_t *pSessionDesc = NULL;
+ CpaInstanceHandle insHandle = NULL;
+ sal_compression_service_t *pService = NULL;
+
+ if (CPA_INSTANCE_HANDLE_SINGLE == dcInstance) {
+ insHandle = dcGetFirstHandle();
+ } else {
+ insHandle = dcInstance;
+ }
+
+ pService = (sal_compression_service_t *)insHandle;
+
+ if (isDcGen2x(pService)) {
+ QAT_UTILS_LOG("Unsupported compression feature.\n");
+ return CPA_STATUS_UNSUPPORTED;
+ }
+ pSessionDesc = DC_SESSION_DESC_FROM_CTX_GET(pSessionHandle);
+
+ LAC_CHECK_NULL_PARAM(pSessionDesc);
+
+ pSessionDesc->cnvErrorInjection = ICP_QAT_FW_COMP_CNV_DFX;
+
+ return CPA_STATUS_SUCCESS;
+}
Index: sys/dev/qat/qat_api/common/compression/include/dc_datapath.h
===================================================================
--- sys/dev/qat/qat_api/common/compression/include/dc_datapath.h
+++ sys/dev/qat/qat_api/common/compression/include/dc_datapath.h
@@ -26,6 +26,8 @@
* the management of skid buffers in the firmware */
#define DC_DEST_BUFFER_DYN_MIN_SIZE (128)
#define DC_DEST_BUFFER_STA_MIN_SIZE (64)
+#define DC_DEST_BUFFER_DYN_MIN_SIZE_GEN4 (512)
+#define DC_DEST_BUFFER_STA_MIN_SIZE_GEN4 (1024)
/* C62x and C3xxx pcie rev0 devices require an additional 32bytes */
#define DC_DEST_BUFFER_STA_ADDITIONAL_SIZE (32)
@@ -93,8 +95,10 @@
* those are used to inform hardware of specifying CRC parameters to be used
* when calculating CRCs */
#define DC_CRC_POLY_DEFAULT 0x04c11db7
+#define DC_CRC64_POLY_DEFAULT 0x42f0e1eba9ea3693ULL
#define DC_XOR_FLAGS_DEFAULT 0xe0000
#define DC_XOR_OUT_DEFAULT 0xffffffff
+#define DC_XOR64_OUT_DEFAULT 0x0ULL
#define DC_INVALID_CRC 0x0
/**
@@ -147,6 +151,13 @@
/**< virtual userspace ptr to source SGL */
CpaBufferList *pUserDestBuff;
/**< virtual userspace ptr to destination SGL */
+ CpaDcCallbackFn pCbFunc;
+ /**< Callback function defined for the traditional sessionless API */
+ CpaDcChecksum checksumType;
+ /**< Type of checksum */
+ dc_integrity_crc_fw_t dataIntegrityCrcs;
+ /**< Data integrity table */
+
} dc_compression_cookie_t;
/**
@@ -165,6 +176,9 @@
*****************************************************************************/
void dcCompression_ProcessCallback(void *pRespMsg);
+CpaStatus dcCheckOpData(sal_compression_service_t *pService,
+ CpaDcOpData *pOpData);
+
/**
*****************************************************************************
* @ingroup Dc_DataCompression
Index: sys/dev/qat/qat_api/common/compression/include/dc_session.h
===================================================================
--- sys/dev/qat/qat_api/common/compression/include/dc_session.h
+++ sys/dev/qat/qat_api/common/compression/include/dc_session.h
@@ -17,6 +17,7 @@
#include "cpa_dc_dp.h"
#include "icp_qat_fw_comp.h"
#include "sal_qat_cmn_msg.h"
+#include "sal_types_compression.h"
/* Maximum number of intermediate buffers SGLs for devices
* with a maximum of 6 compression slices */
@@ -35,6 +36,7 @@
/* Size of the history window.
* Base 2 logarithm of maximum window size minus 8 */
+#define DC_4K_WINDOW_SIZE (4)
#define DC_8K_WINDOW_SIZE (5)
#define DC_16K_WINDOW_SIZE (6)
#define DC_32K_WINDOW_SIZE (7)
@@ -95,35 +97,81 @@
/* CRC32 checksum returned for compressed data */
Cpa32U adler32;
/* ADLER32 checksum returned for compressed data */
- Cpa32U oCrc32Cpr;
- /* CRC32 checksum returned for data output by compression accelerator */
- Cpa32U iCrc32Cpr;
- /* CRC32 checksum returned for input data to compression accelerator */
- Cpa32U oCrc32Xlt;
- /* CRC32 checksum returned for data output by translator accelerator */
- Cpa32U iCrc32Xlt;
- /* CRC32 checksum returned for input data to translator accelerator */
- Cpa32U xorFlags;
- /* Initialise transactor pCRC controls in state register */
- Cpa32U crcPoly;
- /* CRC32 polynomial used by hardware */
- Cpa32U xorOut;
- /* CRC32 from XOR stage (Input CRC is xor'ed with value in the state) */
- Cpa32U deflateBlockType;
- /* Bit 1 - Bit 0
- * 0 0 -> RAW DATA + Deflate header.
- * This will not produced any CRC check because
- * the output will not come from the slices.
- * It will be a simple copy from input to output
- * buffers list.
- * 0 1 -> Static deflate block type
- * 1 0 -> Dynamic deflate block type
- * 1 1 -> Invalid type */
+
+ union {
+ struct {
+ Cpa32U oCrc32Cpr;
+ /* CRC32 checksum returned for data output by
+ * compression accelerator */
+ Cpa32U iCrc32Cpr;
+ /* CRC32 checksum returned for input data to compression
+ * accelerator
+ */
+ Cpa32U oCrc32Xlt;
+ /* CRC32 checksum returned for data output by translator
+ * accelerator
+ */
+ Cpa32U iCrc32Xlt;
+ /* CRC32 checksum returned for input data to translator
+ * accelerator
+ */
+ Cpa32U xorFlags;
+ /* Initialise transactor pCRC controls in state register
+ */
+ Cpa32U crcPoly;
+ /* CRC32 polynomial used by hardware */
+ Cpa32U xorOut;
+ /* CRC32 from XOR stage (Input CRC is xor'ed with value
+ * in the state) */
+ Cpa32U deflateBlockType;
+ /* Bit 1 - Bit 0
+ * 0 0 -> RAW DATA + Deflate header.
+ * This will not produced any CRC check
+ * because the output will not come
+ * from the slices. It will be a simple
+ * copy from input to output buffer
+ * list. 0 1 -> Static deflate block type 1 0 ->
+ * Dynamic deflate block type 1 1 -> Invalid type
+ */
+ };
+
+ struct {
+ Cpa64U iCrc64Cpr;
+ /* CRC64 checksum returned for input data to compression
+ * accelerator
+ */
+ Cpa64U oCrc64Cpr;
+ /* CRC64 checksum returned for data output by
+ * compression accelerator */
+ Cpa64U iCrc64Xlt;
+ /* CRC64 checksum returned for input data to translator
+ * accelerator
+ */
+ Cpa64U oCrc64Xlt;
+ /* CRC64 checksum returned for data output by translator
+ * accelerator
+ */
+ Cpa64U crc64Poly;
+ /* CRC64 polynomial used by hardware */
+ Cpa64U xor64Out;
+ /* CRC64 from XOR stage (Input CRC is xor'ed with value
+ * in the state) */
+ };
+ };
} dc_integrity_crc_fw_t;
typedef struct dc_sw_checksums_s {
- Cpa32U swCrcI;
- Cpa32U swCrcO;
+ union {
+ struct {
+ Cpa32U swCrc32I;
+ Cpa32U swCrc32O;
+ };
+
+ struct {
+ Cpa64U swCrc64I;
+ Cpa64U swCrc64O;
+ };
+ };
} dc_sw_checksums_t;
/* Session descriptor structure for compression */
@@ -211,6 +259,8 @@
dc_sw_checksums_t seedSwCrc;
/* Driver calculated integrity software CRC */
dc_sw_checksums_t integritySwCrc;
+ /* Flag to disable or enable CnV Error Injection mechanism */
+ CpaBoolean cnvErrorInjection;
} dc_session_desc_t;
/**
@@ -275,4 +325,106 @@
Cpa32U *pSessionSize,
Cpa32U *pContextSize);
+/**
+ *****************************************************************************
+ * @ingroup Dc_DataCompression
+ * Set the cnvErrorInjection flag in session descriptor
+ *
+ * @description
+ * This function enables the CnVError injection for the session
+ * passed in. All Compression requests sent within the session
+ * are injected with CnV errors. This error injection is for the
+ * duration of the session. Resetting the session results in
+ * setting being cleared. CnV error injection does not apply to
+ * Data Plane API.
+ *
+ * @param[in] dcInstance Instance Handle
+ * @param[in] pSessionHandle Pointer to a session handle
+ *
+ * @retval CPA_STATUS_SUCCESS Function executed successfully
+ * @retval CPA_STATUS_INVALID_PARAM Invalid parameter passed in
+ * @retval CPA_STATUS_UNSUPPORTED Unsupported feature
+ *****************************************************************************/
+CpaStatus dcSetCnvError(CpaInstanceHandle dcInstance,
+ CpaDcSessionHandle pSessionHandle);
+
+/**
+ *****************************************************************************
+ * @ingroup Dc_DataCompression
+ * Check that pSessionData is valid
+ *
+ * @description
+ * Check that all the parameters defined in the pSessionData are valid
+ *
+ * @param[in] pSessionData Pointer to a user instantiated structure
+ * containing session data
+ *
+ * @retval CPA_STATUS_SUCCESS Function executed successfully
+ * @retval CPA_STATUS_FAIL Function failed to find device
+ * @retval CPA_STATUS_INVALID_PARAM Invalid parameter passed in
+ * @retval CPA_STATUS_UNSUPPORTED Unsupported algorithm/feature
+ *
+ *****************************************************************************/
+CpaStatus dcCheckSessionData(const CpaDcSessionSetupData *pSessionData,
+ CpaInstanceHandle dcInstance);
+
+/**
+ *****************************************************************************
+ * @ingroup Dc_DataCompression
+ * Get the compression command id for the given session setup data.
+ *
+ * @description
+ * This function will get the compression command id based on parameters
+ * passed in the given session setup data.
+ *
+ * @param[in] pService Pointer to the service
+ * @param[in] pSessionData Pointer to a user instantiated
+ * structure containing session data
+ * @param[out] pDcCmdId Pointer to the command id
+ *
+ * @retval CPA_STATUS_SUCCESS Function executed successfully
+ * @retval CPA_STATUS_UNSUPPORTED Unsupported algorithm/feature
+ *
+ *****************************************************************************/
+CpaStatus dcGetCompressCommandId(sal_compression_service_t *pService,
+ CpaDcSessionSetupData *pSessionData,
+ Cpa8U *pDcCmdId);
+
+/**
+ *****************************************************************************
+ * @ingroup Dc_DataCompression
+ * Get the decompression command id for the given session setup data.
+ *
+ * @description
+ * This function will get the decompression command id based on parameters
+ * passed in the given session setup data.
+ *
+ * @param[in] pService Pointer to the service
+ * @param[in] pSessionData Pointer to a user instantiated
+ * structure containing session data
+ * @param[out] pDcCmdId Pointer to the command id
+ *
+ * @retval CPA_STATUS_SUCCESS Function executed successfully
+ * @retval CPA_STATUS_UNSUPPORTED Unsupported algorithm/feature
+ *
+ *****************************************************************************/
+CpaStatus dcGetDecompressCommandId(sal_compression_service_t *pService,
+ CpaDcSessionSetupData *pSessionData,
+ Cpa8U *pDcCmdId);
+
+/**
+ *****************************************************************************
+ * @ingroup Dc_DataCompression
+ * Populate the translator content descriptor
+ *
+ * @description
+ * This function will populate the translator content descriptor
+ *
+ * @param[out] pMsg Pointer to the compression message
+ * @param[in] nextSlice Next slice
+ *
+ *****************************************************************************/
+void dcTransContentDescPopulate(icp_qat_fw_comp_req_t *pMsg,
+ icp_qat_fw_slice_t nextSlice);
+
#endif /* DC_SESSION_H */
Index: sys/dev/qat/qat_api/common/crypto/sym/include/lac_session.h
===================================================================
--- sys/dev/qat/qat_api/common/crypto/sym/include/lac_session.h
+++ sys/dev/qat/qat_api/common/crypto/sym/include/lac_session.h
@@ -107,6 +107,21 @@
* Include private header files
*******************************************************************************
*/
+/**
+*****************************************************************************
+* @ingroup LacSym
+* Spc state
+*
+* @description
+* This enum is used to indicate the Spc state.
+*
+*****************************************************************************/
+typedef enum lac_single_pass_state_e {
+ NON_SPC, /* Algorithms other than CHACHA-POLY and AES-GCM */
+ LIKELY_SPC, /* AES-GCM - Likely to handle it as single pass */
+ SPC /* CHACHA-POLY and AES-GCM */
+} lac_single_pass_state_t;
+
/**
*******************************************************************************
* @ingroup LacSym_Session
@@ -214,15 +229,17 @@
/**< Flag indicating whether the SymConstantsTable can be used or not */
CpaBoolean useOptimisedContentDesc : 1;
/**< Flag indicating whether to use the optimised CD or not */
+ CpaBoolean isPartialSupported : 1;
+ /**< Flag indicating whether symOperation support partial packet */
+ CpaBoolean useStatefulSha3ContentDesc : 1;
+ /**< Flag indicating whether to use the stateful SHA3 CD or not */
icp_qat_la_bulk_req_hdr_t shramReqCacheHdr;
icp_qat_fw_la_key_gen_common_t shramReqCacheMid;
icp_qat_la_bulk_req_ftr_t shramReqCacheFtr;
/**< Alternative pre-built request (header, mid & footer)
* for use with symConstantsTable. */
- CpaBoolean isPartialSupported : 1;
- /**< Flag indicating whether symOperation support partial packet */
- CpaBoolean isSinglePass : 1;
- /**< Flag indicating whether symOperation is single pass operation */
+ lac_single_pass_state_t singlePassState;
+ /**< Flag indicating whether symOperation support single pass */
icp_qat_fw_serv_specif_flags laCmdFlags;
/**< Common request - Service specific flags type */
icp_qat_fw_comn_flags cmnRequestFlags;
@@ -236,6 +253,23 @@
/**< Hash Mode for the qat slices. Not to be confused with QA-API
* hashMode
*/
+ Cpa32U cipherSliceType;
+ /**< Cipher slice type to be used, set at init session time */
+ Cpa8U cipherAesXtsKey1Forward[LAC_CIPHER_AES_XTS_KEY_MAX_LENGTH];
+ /**< Cached AES XTS Forward key
+ * For CPM2.0 AES XTS key convertion need to be done in SW.
+ * Because use can update session direction at any time,
+ * also forward key needs to be cached
+ */
+ Cpa8U cipherAesXtsKey1Reverse[LAC_CIPHER_AES_XTS_KEY_MAX_LENGTH];
+ /**< AES XTS Reverse key
+ * For CPM2.0 AES XTS key convertion need to be done in SW.
+ * Reverse key always will be calcilated at session setup time and
+ * cached to be used when needed */
+ Cpa8U cipherAesXtsKey2[LAC_CIPHER_AES_XTS_KEY_MAX_LENGTH];
+ /**< For AES XTS session need to store Key2 value in order to generate
+ * tweak
+ */
void *writeRingMsgFunc;
/**< function which will be called to write ring message */
Cpa32U aadLenInBytes;
@@ -382,15 +416,17 @@
/**< Flag indicating whether the SymConstantsTable can be used or not */
CpaBoolean useOptimisedContentDesc : 1;
/**< Flag indicating whether to use the optimised CD or not */
+ CpaBoolean isPartialSupported : 1;
+ /**< Flag indicating whether symOperation support partial packet */
+ CpaBoolean useStatefulSha3ContentDesc : 1;
+ /**< Flag indicating whether to use the stateful SHA3 CD or not */
icp_qat_la_bulk_req_hdr_t shramReqCacheHdr;
icp_qat_fw_la_key_gen_common_t shramReqCacheMid;
icp_qat_la_bulk_req_ftr_t shramReqCacheFtr;
/**< Alternative pre-built request (header, mid & footer)
* for use with symConstantsTable. */
- CpaBoolean isPartialSupported : 1;
- /**< Flag indicating whether symOperation support partial packet */
- CpaBoolean isSinglePass : 1;
- /**< Flag indicating whether symOperation is single pass operation */
+ lac_single_pass_state_t singlePassState;
+ /**< Flag indicating whether symOperation support single pass */
icp_qat_fw_serv_specif_flags laCmdFlags;
/**< Common request - Service specific flags type */
icp_qat_fw_comn_flags cmnRequestFlags;
@@ -404,6 +440,23 @@
/**< Hash Mode for the qat slices. Not to be confused with QA-API
* hashMode
*/
+ Cpa32U cipherSliceType;
+ /**< Cipher slice type to be used, set at init session time */
+ Cpa8U cipherAesXtsKey1Forward[LAC_CIPHER_AES_XTS_KEY_MAX_LENGTH];
+ /**< Cached AES XTS Forward key
+ * For CPM2.0 AES XTS key convertion need to be done in SW.
+ * Because use can update session direction at any time,
+ * also forward key needs to be cached
+ */
+ Cpa8U cipherAesXtsKey1Reverse[LAC_CIPHER_AES_XTS_KEY_MAX_LENGTH];
+ /**< AES XTS Reverse key
+ * For CPM2.0 AES XTS key convertion need to be done in SW.
+ * Reverse key always will be calcilated at session setup time and
+ * cached to be used when needed */
+ Cpa8U cipherAesXtsKey2[LAC_CIPHER_AES_XTS_KEY_MAX_LENGTH];
+ /**< For AES XTS session need to store Key2 value in order to generate
+ * tweak
+ */
void *writeRingMsgFunc;
/**< function which will be called to write ring message */
} lac_session_desc_d1_t;
@@ -444,8 +497,8 @@
/**< info on the hash state prefix buffer */
CpaCySymHashAlgorithm hashAlgorithm;
/**< hash algorithm */
- Cpa32U authKeyLenInBytes;
- /**< Authentication key length in bytes */
+ Cpa32U authKeyLenInBytes;
+ /**< Authentication key length in bytes */
CpaCySymHashMode hashMode;
/**< Mode of the hash operation. plain, auth or nested */
Cpa32U hashResultSize;
@@ -516,15 +569,17 @@
/**< Flag indicating whether the SymConstantsTable can be used or not */
CpaBoolean useOptimisedContentDesc : 1;
/**< Flag indicating whether to use the optimised CD or not */
+ CpaBoolean isPartialSupported : 1;
+ /**< Flag indicating whether symOperation support partial packet */
+ CpaBoolean useStatefulSha3ContentDesc : 1;
+ /**< Flag indicating whether to use the stateful SHA3 CD or not */
icp_qat_la_bulk_req_hdr_t shramReqCacheHdr;
icp_qat_fw_la_key_gen_common_t shramReqCacheMid;
icp_qat_la_bulk_req_ftr_t shramReqCacheFtr;
/**< Alternative pre-built request (header. mid & footer)
* for use with symConstantsTable. */
- CpaBoolean isPartialSupported : 1;
- /**< Flag indicating whether symOperation support partial packet */
- CpaBoolean isSinglePass : 1;
- /**< Flag indicating whether symOperation is single pass operation */
+ lac_single_pass_state_t singlePassState;
+ /**< Flag indicating whether symOperation support single pass */
icp_qat_fw_serv_specif_flags laCmdFlags;
/**< Common request - Service specific flags type */
icp_qat_fw_comn_flags cmnRequestFlags;
@@ -538,6 +593,23 @@
/**< Hash Mode for the qat slices. Not to be confused with QA-API
* hashMode
*/
+ Cpa32U cipherSliceType;
+ /**< Cipher slice type to be used, set at init session time */
+ Cpa8U cipherAesXtsKey1Forward[LAC_CIPHER_AES_XTS_KEY_MAX_LENGTH];
+ /**< Cached AES XTS Forward key
+ * For CPM2.0 AES XTS key convertion need to be done in SW.
+ * Because use can update session direction at any time,
+ * also forward key needs to be cached
+ */
+ Cpa8U cipherAesXtsKey1Reverse[LAC_CIPHER_AES_XTS_KEY_MAX_LENGTH];
+ /**< AES XTS Reverse key
+ * For CPM2.0 AES XTS key convertion need to be done in SW.
+ * Reverse key always will be calcilated at session setup time and
+ * cached to be used when needed */
+ Cpa8U cipherAesXtsKey2[LAC_CIPHER_AES_XTS_KEY_MAX_LENGTH];
+ /**< For AES XTS session need to store Key2 value in order to generate
+ * tweak
+ */
void *writeRingMsgFunc;
/**< function which will be called to write ring message */
Cpa32U aadLenInBytes;
Index: sys/dev/qat/qat_api/common/crypto/sym/include/lac_sym_cipher.h
===================================================================
--- sys/dev/qat/qat_api/common/crypto/sym/include/lac_sym_cipher.h
+++ sys/dev/qat/qat_api/common/crypto/sym/include/lac_sym_cipher.h
@@ -228,6 +228,7 @@
#include "lac_session.h"
#include "lac_sym.h"
+#include "lac_sal_types_crypto.h"
/*
* WARNING: There are no checks done on the parameters of the functions in
@@ -253,8 +254,9 @@
* @retval CPA_STATUS_INVALID_PARAM Invalid parameter.
*
*****************************************************************************/
-CpaStatus LacCipher_SessionSetupDataCheck(
- const CpaCySymCipherSetupData *pCipherSetupData);
+CpaStatus
+LacCipher_SessionSetupDataCheck(const CpaCySymCipherSetupData *pCipherSetupData,
+ Cpa32U capabilitiesMask);
/**
*******************************************************************************
@@ -309,4 +311,22 @@
Cpa32U qatPacketType,
Cpa8U **ppIvBuffer);
+/**
+ *****************************************************************************
+ * @ingroup LacCipher
+ * Return cipher slice type for given algorithm
+ *
+ * @description
+ * This function will check what cipher slice type should be used for given
+ * algorithms and CPM generation combination.
+ * Since CPM2.0 there is new UCS cipher slice available.
+ *
+ * @param[in] pService Pointer to service struct
+ * @param[in] cipherAlgorithm cipher algorithm
+ * @param[in] hashAlgorithm hash algorithm
+ *
+ *****************************************************************************/
+Cpa32U LacCipher_GetCipherSliceType(sal_crypto_service_t *pService,
+ CpaCySymCipherAlgorithm algorithm,
+ CpaCySymHashAlgorithm hash);
#endif /* LAC_SYM_CIPHER_H */
Index: sys/dev/qat/qat_api/common/crypto/sym/include/lac_sym_cipher_defs.h
===================================================================
--- sys/dev/qat/qat_api/common/crypto/sym/include/lac_sym_cipher_defs.h
+++ sys/dev/qat/qat_api/common/crypto/sym/include/lac_sym_cipher_defs.h
@@ -45,6 +45,14 @@
/* ARC4 256 bytes for key matrix, 2 for i and j and 6 bytes for padding */
#define LAC_CIPHER_ARC4_STATE_LEN_BYTES 264
+/*
+ * Constant values for CCM AAD buffer
+ */
+#define LAC_CIPHER_CCM_B0_SIZE 16
+#define LAC_CIPHER_CCM_ENCODED_AAD_LEN_SIZE 2
+#define LAC_CIPHER_CCM_AAD_OFFSET \
+ (LAC_CIPHER_CCM_B0_SIZE + LAC_CIPHER_CCM_ENCODED_AAD_LEN_SIZE)
+
#define LAC_SYM_SNOW3G_CIPHER_CONFIG_FOR_HASH_SZ 40
/* Snow3g cipher config required for performing a Snow3g hash operation.
* It contains 8 Bytes of config for hardware, 16 Bytes of Key and requires
@@ -64,6 +72,9 @@
/* The IV length for AES F8 is 16 bytes */
#define LAC_CIPHER_AES_F8_IV_LENGTH 16
+/* The max key length for AES XTS 32 is bytes*/
+#define LAC_CIPHER_AES_XTS_KEY_MAX_LENGTH 32
+
/* For Snow3G UEA2, need to make sure last 8 Bytes of IV buffer are
* zero. */
#define LAC_CIPHER_SNOW3G_UEA2_IV_BUFFER_ZERO_LENGTH 8
@@ -171,12 +182,17 @@
/* Macro to check if the Algorithm Mode is XTS */
#define LAC_CIPHER_IS_XTS_MODE(algo) (algo == CPA_CY_SYM_CIPHER_AES_XTS)
+/* Macro to check if the accelerator has AES V2 capability */
+#define LAC_CIPHER_AES_V2(mask) ((mask)&ICP_ACCEL_CAPABILITIES_AES_V2)
+
/* Macro to check if the Algorithm is single pass */
#define LAC_CIPHER_IS_SPC(cipher, hash, mask) \
- ((LAC_CIPHER_IS_CHACHA(cipher) && (CPA_CY_SYM_HASH_POLY == hash) && \
- ((mask)&ICP_ACCEL_CAPABILITIES_CHACHA_POLY)) || \
- (LAC_CIPHER_IS_GCM(cipher) && ((CPA_CY_SYM_HASH_AES_GCM == hash) || \
- (CPA_CY_SYM_HASH_AES_GMAC == hash)) && \
- ((mask)&ICP_ACCEL_CAPABILITIES_AESGCM_SPC)))
+ (((mask)&ICP_ACCEL_CAPABILITIES_CHACHA_POLY && \
+ LAC_CIPHER_IS_CHACHA(cipher) && (CPA_CY_SYM_HASH_POLY == hash)) || \
+ (((mask)&ICP_ACCEL_CAPABILITIES_AESGCM_SPC) && \
+ LAC_CIPHER_IS_GCM(cipher) && \
+ ((CPA_CY_SYM_HASH_AES_GCM == hash) || \
+ (CPA_CY_SYM_HASH_AES_GMAC == hash))) || \
+ (LAC_CIPHER_IS_CCM(cipher) && LAC_CIPHER_AES_V2(mask)))
#endif /* LAC_CIPHER_DEFS_H */
Index: sys/dev/qat/qat_api/common/crypto/sym/include/lac_sym_hash_defs.h
===================================================================
--- sys/dev/qat/qat_api/common/crypto/sym/include/lac_sym_hash_defs.h
+++ sys/dev/qat/qat_api/common/crypto/sym/include/lac_sym_hash_defs.h
@@ -108,40 +108,37 @@
/* Constants for SHA3_384 algorithm */
#define LAC_HASH_SHA3_384_BLOCK_SIZE 104
/**< @ingroup LacHashDefs
- * * SHA3_384 block size in bytes */
+ * SHA3_384 block size in bytes */
#define LAC_HASH_SHA3_384_DIGEST_SIZE 48
/**< @ingroup LacHashDefs
- * * SHA3_384 digest length in bytes */
+ * SHA3_384 digest length in bytes */
#define LAC_HASH_SHA3_384_STATE_SIZE 48
/**< @ingroup LacHashDefs
- * * SHA3_384 state size */
+ * SHA3_384 state size */
/* Constants for SHA3_512 algorithm */
#define LAC_HASH_SHA3_512_BLOCK_SIZE 72
/**< @ingroup LacHashDefs
- * * * SHA3_512 block size in bytes */
+ * SHA3_512 block size in bytes */
#define LAC_HASH_SHA3_512_DIGEST_SIZE 64
/**< @ingroup LacHashDefs
- * * * SHA3_512 digest length in bytes */
+ * SHA3_512 digest length in bytes */
#define LAC_HASH_SHA3_512_STATE_SIZE 64
/**< @ingroup LacHashDefs
- * * * SHA3_512 state size */
+ * SHA3_512 state size */
-/* Constants for SHAKE_128 algorithm */
-#define LAC_HASH_SHAKE_128_BLOCK_SIZE 168
-/**< @ingroup LacHashDefs
- * * * SHAKE_128 block size in bytes */
-#define LAC_HASH_SHAKE_128_DIGEST_SIZE 0xFFFFFFFF
-/**< @ingroup LacHashDefs
- * * * SHAKE_128 digest length in bytes ((2^32)-1)*/
+#define LAC_HASH_SHA3_STATEFUL_STATE_SIZE 200
-/* Constants for SHAKE_256 algorithm */
-#define LAC_HASH_SHAKE_256_BLOCK_SIZE 136
+/* Constants for SM3 algorithm */
+#define LAC_HASH_SM3_BLOCK_SIZE 64
+/**< @ingroup LacHashDefs
+ * SM3 block size in bytes */
+#define LAC_HASH_SM3_DIGEST_SIZE 32
/**< @ingroup LacHashDefs
- * * * SHAKE_256 block size in bytes */
-#define LAC_HASH_SHAKE_256_DIGEST_SIZE 0xFFFFFFFF
+ * SM3 digest length */
+#define LAC_HASH_SM3_STATE_SIZE 32
/**< @ingroup LacHashDefs
- * * * SHAKE_256 digest length in bytes ((2^ 32)-1)*/
+ * SM3 state size */
/* Constants for POLY algorithm */
#define LAC_HASH_POLY_BLOCK_SIZE 64
@@ -154,17 +151,6 @@
/**< @ingroup LacHashDefs
* POLY state size */
-/* Constants for SM3 algorithm */
-#define LAC_HASH_SM3_BLOCK_SIZE 64
-/**< @ingroup LacHashDefs
- * SM3 block size in bytes */
-#define LAC_HASH_SM3_DIGEST_SIZE 32
-/**< @ingroup LacHashDefs
- * SM3 digest length */
-#define LAC_HASH_SM3_STATE_SIZE 32
-/**< @ingroup LacHashDefs
- * SM3 state size */
-
/* Constants for XCBC precompute algorithm */
#define LAC_HASH_XCBC_PRECOMP_KEY_NUM 3
/**< @ingroup LacHashDefs
@@ -285,7 +271,7 @@
* the size in an 8bit field */
#define LAC_MAX_HASH_STATE_STORAGE_SIZE \
- (sizeof(icp_qat_hw_auth_counter_t) + LAC_HASH_SHA512_STATE_SIZE)
+ (sizeof(icp_qat_hw_auth_counter_t) + LAC_HASH_SHA3_STATEFUL_STATE_SIZE)
/**< Maximum size of the hash state storage section of the hash state prefix
* buffer */
@@ -307,11 +293,8 @@
(algorithm == CPA_CY_SYM_HASH_SHA256) || \
(algorithm == CPA_CY_SYM_HASH_SHA384) || \
(algorithm == CPA_CY_SYM_HASH_SHA512) || \
- (algorithm == CPA_CY_SYM_HASH_SHA3_224) || \
- (algorithm == CPA_CY_SYM_HASH_SHA3_256) || \
- (algorithm == CPA_CY_SYM_HASH_SHA3_384) || \
- (algorithm == CPA_CY_SYM_HASH_SHA3_512) || \
- (algorithm == CPA_CY_SYM_HASH_SM3))
+ (algorithm == CPA_CY_SYM_HASH_SM3)) || \
+ (LAC_HASH_IS_SHA3(algorithm))
/**< @ingroup LacSymQatHash
* Macro to detect if the hash algorithm is a HMAC algorithm */
@@ -341,4 +324,12 @@
* Nested. This applies to TLS. This is used to differentiate between
* TLS and HMAC */
+#define LAC_HASH_IS_SHA3(algo) \
+ ((algo == CPA_CY_SYM_HASH_SHA3_224) || \
+ (algo == CPA_CY_SYM_HASH_SHA3_256) || \
+ (algo == CPA_CY_SYM_HASH_SHA3_384) || \
+ (algo == CPA_CY_SYM_HASH_SHA3_512))
+/**< @ingroup LacSymQatHash
+ * Macro to check if the hash algorithm is SHA3 */
+
#endif /* LAC_SYM_HASH_DEFS_H */
Index: sys/dev/qat/qat_api/common/crypto/sym/include/lac_sym_qat.h
===================================================================
--- sys/dev/qat/qat_api/common/crypto/sym/include/lac_sym_qat.h
+++ sys/dev/qat/qat_api/common/crypto/sym/include/lac_sym_qat.h
@@ -206,4 +206,43 @@
void LacSymQat_LaSetDefaultFlags(icp_qat_fw_serv_specif_flags *laCmdFlags,
CpaCySymOp symOp);
+/**
+ ******************************************************************************
+ * @ingroup LacSymQat
+ *
+ *
+ * @description
+ * this function defines whether the shared constants table can be used
+ * for a particular cipher and hash algorithm
+ *
+ * @param[in] ptr to session
+
+ * @param[in] ptr to return offset into table for cipher config
+
+ * @param[in] ptr to return offset into table for hash config
+ *
+ * @return CPA_TRUE if Constants table is available for use, CPA_FALSE if it's
+ * not.
+ *
+ *****************************************************************************/
+CpaBoolean LacSymQat_UseSymConstantsTable(lac_session_desc_t *pSession,
+ Cpa8U *cipherOffset,
+ Cpa8U *hashOffset);
+
+/**
+ ******************************************************************************
+ * @ingroup LacSymQat
+ *
+ *
+ * @description
+ * this function calculates whether the optimized content descriptor can
+ * be used for a particular chained cipher and hash algorithm
+ *
+ * @param[in] ptr to session
+ *
+ * @return CPA_TRUE if optimized CD can be used, CPA_FALSE if it's not.
+ *
+ *****************************************************************************/
+CpaBoolean LacSymQat_UseOptimisedContentDesc(lac_session_desc_t *pSession);
+
#endif /* LAC_SYM_QAT_H */
Index: sys/dev/qat/qat_api/common/crypto/sym/include/lac_sym_qat_cipher.h
===================================================================
--- sys/dev/qat/qat_api/common/crypto/sym/include/lac_sym_qat_cipher.h
+++ sys/dev/qat/qat_api/common/crypto/sym/include/lac_sym_qat_cipher.h
@@ -42,9 +42,11 @@
#define LAC_SYM_QAT_CIPHER_NEXT_ID_BIT_OFFSET 24
#define LAC_SYM_QAT_CIPHER_CURR_ID_BIT_OFFSET 16
#define LAC_SYM_QAT_CIPHER_STATE_SIZE_BIT_OFFSET 8
-#define LAC_SYM_QAT_CIPHER_OFFSET_IN_DRAM_GCM_SPC 9
-#define LAC_SYM_QAT_CIPHER_OFFSET_IN_DRAM_CHACHA_SPC 2
-#define LAC_SYM_QAT_CIPHER_STATE_SIZE_SPC 48
+#define LAC_SYM_QAT_CIPHER_GCM_SPC_OFFSET_IN_DRAM 9
+#define LAC_SYM_QAT_CIPHER_CCM_SPC_OFFSET_IN_DRAM 8
+#define LAC_SYM_QAT_CIPHER_CHACHA_SPC_OFFSET_IN_DRAM 2
+#define LAC_SYM_QAT_CIPHER_SPC_STATE_SIZE 48
+
/**
******************************************************************************
* @ingroup LacSymQat_Cipher
@@ -111,11 +113,13 @@
* @retval void
*
*****************************************************************************/
-CpaStatus LacSymQat_CipherRequestParamsPopulate(icp_qat_fw_la_bulk_req_t *pReq,
- Cpa32U cipherOffsetInBytes,
- Cpa32U cipherLenInBytes,
- Cpa64U ivBufferPhysAddr,
- Cpa8U *pIvBufferVirt);
+CpaStatus
+LacSymQat_CipherRequestParamsPopulate(lac_session_desc_t *pSessionDesc,
+ icp_qat_fw_la_bulk_req_t *pReq,
+ Cpa32U cipherOffsetInBytes,
+ Cpa32U cipherLenInBytes,
+ Cpa64U ivBufferPhysAddr,
+ Cpa8U *pIvBufferVirt);
/**
******************************************************************************
@@ -194,6 +198,8 @@
* @param[in] targetKeyLenInBytes cipher key length in bytes of selected
* algorithm
*
+ * @param[in] sliceType Cipher slice type to be used
+ *
* @param[out] nextSlice SliceID for next control block
* entry. This value is known only by
* the calling component
@@ -206,6 +212,7 @@
void LacSymQat_CipherCtrlBlockWrite(icp_qat_la_bulk_req_ftr_t *pMsg,
Cpa32U cipherAlgorithm,
Cpa32U targetKeyLenInBytes,
+ Cpa32U sliceType,
icp_qat_fw_slice_t nextSlice,
Cpa8U cipherCfgOffsetInQuadWord);
@@ -274,6 +281,8 @@
* key length MUST match the key length
* in the cipher setup data.
*
+ * @param[in] sliceType Cipher slice type to be used
+ *
* @param[in] pCipherHwBlock Pointer to the cipher hardware block
*
* @param[out] pCipherHwBlockSizeBytes Size in bytes of cipher setup block
@@ -283,8 +292,10 @@
*
*****************************************************************************/
void LacSymQat_CipherHwBlockPopulateKeySetup(
+ lac_session_desc_t *pSessionDesc,
const CpaCySymCipherSetupData *pCipherSetupData,
Cpa32U targetKeyLenInBytes,
+ Cpa32U sliceType,
const void *pCipherHwBlock,
Cpa32U *pCipherHwBlockSizeBytes);
Index: sys/dev/qat/qat_api/common/crypto/sym/include/lac_sym_qat_constants_table.h
===================================================================
--- /dev/null
+++ sys/dev/qat/qat_api/common/crypto/sym/include/lac_sym_qat_constants_table.h
@@ -0,0 +1,111 @@
+/* SPDX-License-Identifier: BSD-3-Clause */
+/* Copyright(c) 2007-2022 Intel Corporation */
+/* $FreeBSD$ */
+
+/**
+ *****************************************************************************
+ * @file lac_sym_qat_constants_table.h
+ *
+ * @ingroup LacSymQat
+ *
+ * API to be used for the CySym constants table.
+ *
+ *****************************************************************************/
+
+#ifndef LAC_SYM_QAT_CONSTANTS_TABLE_H
+#define LAC_SYM_QAT_CONSTANTS_TABLE_H
+
+#include "cpa.h"
+#include "icp_qat_fw_la.h"
+
+typedef struct lac_sym_qat_constants_s {
+ /* Note these arrays must match the tables in lac_sym_qat_constants.c
+ * icp_qat_hw_cipher_lookup_tbl and icp_qat_hw_auth_lookup_tbl */
+ uint8_t cipher_offset[ICP_QAT_HW_CIPHER_DELIMITER]
+ [ICP_QAT_HW_CIPHER_MODE_DELIMITER][2][2];
+ uint8_t auth_offset[ICP_QAT_HW_AUTH_ALGO_DELIMITER]
+ [ICP_QAT_HW_AUTH_MODE_DELIMITER][2];
+} lac_sym_qat_constants_t;
+
+/**
+ *******************************************************************************
+ * @ingroup LacSymQat
+ * LacSymQat_ConstantsInitLookupTables
+ *
+ *
+ * @description
+ * The SymCy constants table is 1K of static data which is passed down
+ * to the FW to be stored in SHRAM for use by the FW.
+ * This function populates the associated lookup tables which the IA
+ * driver uses.
+ * Where there is config data available in the constants table the lookup
+ * table stores the offset into the constants table.
+ * Where there's no suitable config data available in the constants table
+ * zero is stored in the lookup table.
+ *
+ * @return none
+ *
+ *****************************************************************************/
+void LacSymQat_ConstantsInitLookupTables(CpaInstanceHandle instanceHandle);
+
+/**
+*******************************************************************************
+* @ingroup LacSymQat
+* LacSymQat_ConstantsGetCipherOffset
+*
+* @description
+* This function looks up the cipher constants lookup array for
+* a specific cipher algorithm, mode, direction and convert flag.
+* If the lookup table value is zero then there's no suitable config data
+* available in the constants table.
+* If the value > zero, then there is config data available in the constants
+* table which is stored in SHRAM for use by the FW. The value is the offset
+* into the constants table, it is returned to the caller in poffset.
+*
+*
+* @param[in] Cipher Algorithm
+* @param[in] Cipher Mode
+* @param[in] Direction - encrypt/decrypt
+* @param[in] convert / no convert
+* @param[out] offset into constants table
+*
+* @return none
+*
+*****************************************************************************/
+void LacSymQat_ConstantsGetCipherOffset(CpaInstanceHandle instanceHandle,
+ uint8_t algo,
+ uint8_t mode,
+ uint8_t direction,
+ uint8_t convert,
+ uint8_t *poffset);
+
+/**
+*******************************************************************************
+* @ingroup LacSymQat
+* LacSymQat_ConstantsGetAuthOffset
+*
+* @description
+* This function looks up the auth constants lookup array for
+* a specific auth algorithm, mode, direction and convert flag.
+* If the lookup table value is zero then there's no suitable config data
+* available in the constants table.
+* If the value > zero, then there is config data available in the constants
+* table which is stored in SHRAM for use by the FW. The value is the offset
+* into the constants table, it is returned to the caller in poffset.
+*
+*
+* @param[in] auth Algorithm
+* @param[in] auth Mode
+* @param[in] nested / no nested
+* @param[out] offset into constants table
+*
+* @return none
+*
+*****************************************************************************/
+void LacSymQat_ConstantsGetAuthOffset(CpaInstanceHandle instanceHandle,
+ uint8_t algo,
+ uint8_t mode,
+ uint8_t nested,
+ uint8_t *poffset);
+
+#endif /* LAC_SYM_QAT_SHRAM_CONSTANTS_TABLE_H */
Index: sys/dev/qat/qat_api/common/crypto/sym/include/lac_sym_qat_hash.h
===================================================================
--- sys/dev/qat/qat_api/common/crypto/sym/include/lac_sym_qat_hash.h
+++ sys/dev/qat/qat_api/common/crypto/sym/include/lac_sym_qat_hash.h
@@ -126,6 +126,10 @@
* @param[in] useOptimisedContentDesc Indicate if optimised content desc
* is used for this session.
*
+ * @param[in] useStatefulSha3ContentDesc
+ * Indicate if stateful SHA3 content desc
+ * is used for this session.
+ *
* @param[in] pPrecompute For auth mode, this is the pointer
* to the precompute data. Otherwise this
* should be set to NULL
@@ -145,6 +149,7 @@
icp_qat_hw_auth_mode_t qatHashMode,
CpaBoolean useSymConstantsTable,
CpaBoolean useOptimisedContentDesc,
+ CpaBoolean useStatefulSha3ContentDesc,
lac_sym_qat_hash_precompute_info_t *pPrecompute,
Cpa32U *pHashBlkSizeInBytes);
Index: sys/dev/qat/qat_api/common/crypto/sym/key/lac_sym_key.c
===================================================================
--- sys/dev/qat/qat_api/common/crypto/sym/key/lac_sym_key.c
+++ sys/dev/qat/qat_api/common/crypto/sym/key/lac_sym_key.c
@@ -798,8 +798,9 @@
ICP_QAT_FW_SLICE_DRAM_WR,
ICP_QAT_HW_AUTH_MODE0, /* just a plain hash */
CPA_FALSE, /* Not using sym Constants Table in Shared SRAM
- */
+ */
CPA_FALSE, /* not using the optimised Content Desc */
+ CPA_FALSE, /* Not using the stateful SHA3 Content Desc */
NULL,
&hashBlkSizeInBytes);
@@ -1550,9 +1551,11 @@
LAC_SYM_KEY_NO_HASH_BLK_OFFSET_QW,
ICP_QAT_FW_SLICE_DRAM_WR,
qatHashMode,
- CPA_FALSE, /* Not using sym Constants Table in SRAM */
- CPA_FALSE, /* Not using the optimised content Desc */
- NULL, /* Precompute data */
+ CPA_FALSE, /* Not using sym Constants Table in Shared SRAM
+ */
+ CPA_FALSE, /* not using the optimised content Desc */
+ CPA_FALSE, /* Not using the stateful SHA3 Content Desc */
+ NULL, /* precompute data */
&hashBlkSizeInBytes);
/* SSL3 */
Index: sys/dev/qat/qat_api/common/crypto/sym/lac_sym_alg_chain.c
===================================================================
--- sys/dev/qat/qat_api/common/crypto/sym/lac_sym_alg_chain.c
+++ sys/dev/qat/qat_api/common/crypto/sym/lac_sym_alg_chain.c
@@ -52,6 +52,72 @@
#include "sal_string_parse.h"
#include "lac_sym_auth_enc.h"
#include "lac_sym_qat.h"
+#include "sal_hw_gen.h"
+
+/**
+ * @ingroup LacAlgChain
+ * This callback function will be invoked whenever a hash precompute
+ * operation completes. It will dequeue and send any QAT requests
+ * which were queued up while the precompute was in progress.
+ *
+ * @param[in] callbackTag Opaque value provided by user. This will
+ * be a pointer to the session descriptor.
+ *
+ * @retval
+ * None
+ *
+ */
+static void
+LacSymAlgChain_HashPrecomputeDoneCb(void *callbackTag)
+{
+ LacSymCb_PendingReqsDequeue((lac_session_desc_t *)callbackTag);
+}
+
+/**
+ * @ingroup LacAlgChain
+ * Walk the buffer list and find the address for the given offset within
+ * a buffer.
+ *
+ * @param[in] pBufferList Buffer List
+ * @param[in] packetOffset Offset in the buffer list for which address
+ * is to be found.
+ * @param[out] ppDataPtr This is where the sought pointer will be put
+ * @param[out] pSpaceLeft Pointer to a variable in which information about
+ * available space from the given offset to the end
+ * of the flat buffer it is located in will be returned
+ *
+ * @retval CPA_STATUS_SUCCESS Address with a given offset is found in the list
+ * @retval CPA_STATUS_FAIL Address with a given offset not found in the list.
+ *
+ */
+static CpaStatus
+LacSymAlgChain_PtrFromOffsetGet(const CpaBufferList *pBufferList,
+ const Cpa32U packetOffset,
+ Cpa8U **ppDataPtr)
+{
+ Cpa32U currentOffset = 0;
+ Cpa32U i = 0;
+
+ for (i = 0; i < pBufferList->numBuffers; i++) {
+ Cpa8U *pCurrData = pBufferList->pBuffers[i].pData;
+ Cpa32U currDataSize = pBufferList->pBuffers[i].dataLenInBytes;
+
+ /* If the offset is within the address space of the current
+ * buffer */
+ if ((packetOffset >= currentOffset) &&
+ (packetOffset < (currentOffset + currDataSize))) {
+ /* increment by offset of the address in the current
+ * buffer */
+ *ppDataPtr = pCurrData + (packetOffset - currentOffset);
+ return CPA_STATUS_SUCCESS;
+ }
+
+ /* Increment by the size of the buffer */
+ currentOffset += currDataSize;
+ }
+
+ return CPA_STATUS_FAIL;
+}
/**
* @ingroup LacAlgChain
@@ -81,6 +147,7 @@
/* All others support partial */
default:
isCipherPartialSupported = CPA_TRUE;
+ break;
}
switch (pSessionDesc->hashAlgorithm) {
/* Following hash don't support partial */
@@ -88,8 +155,6 @@
case CPA_CY_SYM_HASH_SNOW3G_UIA2:
case CPA_CY_SYM_HASH_POLY:
case CPA_CY_SYM_HASH_ZUC_EIA3:
- case CPA_CY_SYM_HASH_SHAKE_128:
- case CPA_CY_SYM_HASH_SHAKE_256:
break;
/* Following hash may support partial based on device capabilities */
case CPA_CY_SYM_HASH_SHA3_256:
@@ -100,6 +165,7 @@
/* All others support partial */
default:
isHashPartialSupported = CPA_TRUE;
+ break;
}
switch (pSessionDesc->symOperation) {
case CPA_CY_SYM_OP_CIPHER:
@@ -115,73 +181,84 @@
break;
case CPA_CY_SYM_OP_NONE:
break;
+ default:
+ break;
+ }
+
+ if (ICP_QAT_FW_LA_USE_UCS_SLICE_TYPE == pSessionDesc->cipherSliceType) {
+ /* UCS slice has no support for state flush and
+ * because of that is not able to do partial processing */
+ isPartialSupported = CPA_FALSE;
}
+
pSessionDesc->isPartialSupported = isPartialSupported;
}
-/**
- * @ingroup LacAlgChain
- * This callback function will be invoked whenever a hash precompute
- * operation completes. It will dequeue and send any QAT requests
- * which were queued up while the precompute was in progress.
- *
- * @param[in] callbackTag Opaque value provided by user. This will
- * be a pointer to the session descriptor.
- *
- * @retval
- * None
- *
- */
static void
-LacSymAlgChain_HashPrecomputeDoneCb(void *callbackTag)
+LacAlgChain_CipherCDBuild_ForOptimisedCD(
+ const CpaCySymCipherSetupData *pCipherData,
+ lac_session_desc_t *pSessionDesc,
+ icp_qat_fw_slice_t nextSlice,
+ Cpa8U cipherOffsetInConstantsTable,
+ Cpa8U *pOptimisedHwBlockBaseInDRAM,
+ Cpa32U *pOptimisedHwBlockOffsetInDRAM)
{
- LacSymCb_PendingReqsDequeue((lac_session_desc_t *)callbackTag);
-}
+ Cpa8U *pCipherKeyField = NULL;
+ Cpa32U sizeInBytes = 0;
+ pCipherKeyField = pOptimisedHwBlockBaseInDRAM;
-/**
- * @ingroup LacAlgChain
- * Walk the buffer list and find the address for the given offset within
- * a buffer.
- *
- * @param[in] pBufferList Buffer List
- * @param[in] packetOffset Offset in the buffer list for which address
- * is to be found.
- * @param[out] ppDataPtr This is where the sought pointer will be put
- * @param[out] pSpaceLeft Pointer to a variable in which information about
- * available space from the given offset to the end
- * of the flat buffer it is located in will be returned
- *
- * @retval CPA_STATUS_SUCCESS Address with a given offset is found in the list
- * @retval CPA_STATUS_FAIL Address with a given offset not found in the list.
- *
- */
-static CpaStatus
-LacSymAlgChain_PtrFromOffsetGet(const CpaBufferList *pBufferList,
- const Cpa32U packetOffset,
- Cpa8U **ppDataPtr)
-{
- Cpa32U currentOffset = 0;
- Cpa32U i = 0;
+ /* Need to build up the alternative CD for SHRAM Constants Table use
+ * with an optimised content desc of 64 bytes for this case. Cipher key
+ * will be in the Content desc in DRAM, The cipher config data
+ * is now in the SHRAM constants table. */
- for (i = 0; i < pBufferList->numBuffers; i++) {
- Cpa8U *pCurrData = pBufferList->pBuffers[i].pData;
- Cpa32U currDataSize = pBufferList->pBuffers[i].dataLenInBytes;
+ LacSymQat_CipherHwBlockPopulateKeySetup(
+ pSessionDesc,
+ pCipherData,
+ pCipherData->cipherKeyLenInBytes,
+ pSessionDesc->cipherSliceType,
+ pCipherKeyField,
+ &sizeInBytes);
- /* If the offset is within the address space of the current
- * buffer */
- if ((packetOffset >= currentOffset) &&
- (packetOffset < (currentOffset + currDataSize))) {
- /* increment by offset of the address in the current
- * buffer */
- *ppDataPtr = pCurrData + (packetOffset - currentOffset);
- return CPA_STATUS_SUCCESS;
- }
+ LacSymQat_CipherCtrlBlockWrite(&(pSessionDesc->shramReqCacheFtr),
+ pSessionDesc->cipherAlgorithm,
+ pSessionDesc->cipherKeyLenInBytes,
+ pSessionDesc->cipherSliceType,
+ nextSlice,
+ cipherOffsetInConstantsTable);
- /* Increment by the size of the buffer */
- currentOffset += currDataSize;
- }
+ *pOptimisedHwBlockOffsetInDRAM += sizeInBytes;
+}
- return CPA_STATUS_FAIL;
+static void
+LacAlgChain_CipherCDBuild_ForSHRAM(const CpaCySymCipherSetupData *pCipherData,
+ lac_session_desc_t *pSessionDesc,
+ icp_qat_fw_slice_t nextSlice,
+ Cpa8U cipherOffsetInConstantsTable)
+{
+ Cpa32U sizeInBytes = 0;
+ Cpa8U *pCipherKeyField = NULL;
+ /* Need to build up the alternative CD for SHRAM Constants Table use
+ * Cipher key will be in the Request, The cipher config data is now in
+ * the SHRAM constants table. And nothing is now stored in the content
+ * desc */
+ pCipherKeyField = (Cpa8U *)&(
+ pSessionDesc->shramReqCacheHdr.cd_pars.s1.serv_specif_fields);
+
+ LacSymQat_CipherHwBlockPopulateKeySetup(
+ pSessionDesc,
+ pCipherData,
+ pCipherData->cipherKeyLenInBytes,
+ pSessionDesc->cipherSliceType,
+ pCipherKeyField,
+ &sizeInBytes);
+
+ LacSymQat_CipherCtrlBlockWrite(&(pSessionDesc->shramReqCacheFtr),
+ pSessionDesc->cipherAlgorithm,
+ pSessionDesc->cipherKeyLenInBytes,
+ pSessionDesc->cipherSliceType,
+ nextSlice,
+ cipherOffsetInConstantsTable);
}
static void
@@ -192,11 +269,14 @@
icp_qat_fw_comn_flags *pCmnRequestFlags,
icp_qat_fw_serv_specif_flags *pLaCmdFlags,
Cpa8U *pHwBlockBaseInDRAM,
- Cpa32U *pHwBlockOffsetInDRAM)
+ Cpa32U *pHwBlockOffsetInDRAM,
+ Cpa32U capabilitiesMask)
{
Cpa8U *pCipherKeyField = NULL;
Cpa8U cipherOffsetInReqQW = 0;
Cpa32U sizeInBytes = 0;
+ void *pCfgData = NULL;
+ Cpa32U cfgOffset = 0;
/* Construct the ContentDescriptor in DRAM */
cipherOffsetInReqQW = (*pHwBlockOffsetInDRAM / LAC_QUAD_WORD_IN_BYTES);
@@ -204,11 +284,15 @@
*pLaCmdFlags, ICP_QAT_FW_CIPH_AUTH_CFG_OFFSET_IN_CD_SETUP);
/* construct cipherConfig in CD in DRAM */
+ cfgOffset = *pHwBlockOffsetInDRAM;
+ pCfgData = pHwBlockBaseInDRAM + cfgOffset;
LacSymQat_CipherHwBlockPopulateCfgData(pSessionDesc,
- pHwBlockBaseInDRAM +
- *pHwBlockOffsetInDRAM,
+ pCfgData,
&sizeInBytes);
+ ICP_QAT_FW_LA_SLICE_TYPE_SET(*pLaCmdFlags,
+ pSessionDesc->cipherSliceType);
+
*pHwBlockOffsetInDRAM += sizeInBytes;
/* Cipher key will be in CD in DRAM.
@@ -220,8 +304,10 @@
QAT_COMN_CD_FLD_TYPE_64BIT_ADR);
LacSymQat_CipherHwBlockPopulateKeySetup(
+ pSessionDesc,
pCipherData,
pCipherData->cipherKeyLenInBytes,
+ pSessionDesc->cipherSliceType,
pCipherKeyField,
&sizeInBytes);
/* update offset */
@@ -230,14 +316,15 @@
LacSymQat_CipherCtrlBlockWrite(&(pSessionDesc->reqCacheFtr),
pSessionDesc->cipherAlgorithm,
pSessionDesc->cipherKeyLenInBytes,
+ pSessionDesc->cipherSliceType,
nextSlice,
cipherOffsetInReqQW);
- if (LAC_CIPHER_IS_GCM(pSessionDesc->cipherAlgorithm) ||
- LAC_CIPHER_IS_CHACHA(pSessionDesc->cipherAlgorithm)) {
+ if (NON_SPC != pSessionDesc->singlePassState) {
LacSymQat_CipherCtrlBlockWrite(
&(pSessionDesc->reqSpcCacheFtr),
pSessionDesc->cipherAlgorithm,
pSessionDesc->cipherKeyLenInBytes,
+ pSessionDesc->cipherSliceType,
ICP_QAT_FW_SLICE_DRAM_WR,
cipherOffsetInReqQW);
}
@@ -275,6 +362,7 @@
pSessionDesc->qatHashMode,
CPA_FALSE,
CPA_FALSE,
+ pSessionDesc->useStatefulSha3ContentDesc,
pPrecomputeData,
&sizeInBytes);
@@ -282,6 +370,69 @@
*pHwBlockOffsetInDRAM += sizeInBytes;
sizeInBytes = 0;
+
+ if (pSessionDesc->useOptimisedContentDesc) {
+ LacSymQat_HashContentDescInit(&(pSessionDesc->shramReqCacheFtr),
+ instanceHandle,
+ pHashData,
+ pOptimisedHwBlockBaseInDRAM,
+ hashOffsetInConstantsTable,
+ nextSlice,
+ pSessionDesc->qatHashMode,
+ CPA_TRUE,
+ CPA_TRUE,
+ CPA_FALSE,
+ pPrecomputeDataOptimisedCd,
+ &sizeInBytes);
+
+ *pOptimisedHwBlockOffsetInDRAM += sizeInBytes;
+ } else if (pSessionDesc->useSymConstantsTable) {
+ /* Need to build up the alternative CD for SHRAM Constants Table
+ * use */
+ LacSymQat_HashContentDescInit(&(pSessionDesc->shramReqCacheFtr),
+ instanceHandle,
+ pHashData,
+ pHwBlockBaseInDRAM,
+ hashOffsetInConstantsTable,
+ nextSlice,
+ pSessionDesc->qatHashMode,
+ CPA_TRUE,
+ CPA_FALSE,
+ CPA_FALSE,
+ pPrecomputeData,
+ &sizeInBytes);
+ }
+}
+
+static Cpa16U
+LacAlgChain_GetCipherConfigSize(lac_session_desc_t *pSessionDesc)
+{
+ if (ICP_QAT_FW_LA_USE_UCS_SLICE_TYPE == pSessionDesc->cipherSliceType) {
+ return sizeof(icp_qat_hw_ucs_cipher_config_t);
+ } else {
+ return sizeof(icp_qat_hw_cipher_config_t);
+ }
+}
+
+static Cpa16U
+LacAlgChain_GetCipherConfigOffset(lac_session_desc_t *pSessionDesc)
+{
+ Cpa16U offset = 0;
+
+ if (CPA_CY_SYM_OP_ALGORITHM_CHAINING == pSessionDesc->symOperation ||
+ SPC == pSessionDesc->singlePassState) {
+ icp_qat_fw_cipher_auth_cd_ctrl_hdr_t *cd_ctrl =
+ (icp_qat_fw_cipher_auth_cd_ctrl_hdr_t *)&pSessionDesc
+ ->reqCacheFtr.cd_ctrl;
+ offset = cd_ctrl->cipher_cfg_offset;
+ } else if (CPA_CY_SYM_OP_CIPHER == pSessionDesc->symOperation) {
+ icp_qat_fw_cipher_cd_ctrl_hdr_t *cd_ctrl =
+ (icp_qat_fw_cipher_cd_ctrl_hdr_t *)&pSessionDesc
+ ->reqCacheFtr.cd_ctrl;
+ offset = cd_ctrl->cipher_cfg_offset;
+ }
+
+ return offset * LAC_QUAD_WORD_IN_BYTES;
}
CpaStatus
@@ -298,7 +449,7 @@
req_params->u2.aad_sz =
LAC_ALIGN_POW2_ROUNDUP(newAADLength, LAC_HASH_AES_GCM_BLOCK_SIZE);
- if (CPA_TRUE == pSessionDesc->isSinglePass) {
+ if (SPC == pSessionDesc->singlePassState) {
Cpa8U *pHwBlockBaseInDRAM = NULL;
Cpa32U hwBlockOffsetInDRAM = 0;
Cpa32U pSizeInBytes = 0;
@@ -310,10 +461,10 @@
CPA_CY_SYM_CIPHER_DIRECTION_DECRYPT) {
if (LAC_CIPHER_IS_GCM(cipher)) {
hwBlockOffsetInDRAM = LAC_QUADWORDS_TO_BYTES(
- LAC_SYM_QAT_CIPHER_OFFSET_IN_DRAM_GCM_SPC);
+ LAC_SYM_QAT_CIPHER_GCM_SPC_OFFSET_IN_DRAM);
} else {
hwBlockOffsetInDRAM = LAC_QUADWORDS_TO_BYTES(
- LAC_SYM_QAT_CIPHER_OFFSET_IN_DRAM_CHACHA_SPC);
+ LAC_SYM_QAT_CIPHER_CHACHA_SPC_OFFSET_IN_DRAM);
}
}
LacSymQat_CipherHwBlockPopulateCfgData(pSessionDesc,
@@ -343,6 +494,8 @@
CpaCySymCipherSetupData cipherSetupData = { 0 };
Cpa32U sizeInBytes;
Cpa8U *pCipherKeyField;
+ Cpa16U cipherConfigSize;
+ Cpa16U cipherConfigOffset;
sal_qat_content_desc_info_t *pCdInfo =
&(pSessionDesc->contentDescInfo);
@@ -350,15 +503,23 @@
cipherSetupData.cipherKeyLenInBytes =
pSessionDesc->cipherKeyLenInBytes;
cipherSetupData.pCipherKey = pCipherKey;
+ cipherSetupData.cipherDirection = pSessionDesc->cipherDirection;
+
+ cipherConfigSize =
+ LacAlgChain_GetCipherConfigSize(pSessionDesc);
+ cipherConfigOffset =
+ LacAlgChain_GetCipherConfigOffset(pSessionDesc);
+
+ pCipherKeyField = (Cpa8U *)pCdInfo->pData + cipherConfigOffset +
+ cipherConfigSize;
switch (pSessionDesc->symOperation) {
case CPA_CY_SYM_OP_CIPHER: {
- pCipherKeyField = (Cpa8U *)pCdInfo->pData +
- sizeof(icp_qat_hw_cipher_config_t);
-
LacSymQat_CipherHwBlockPopulateKeySetup(
+ pSessionDesc,
&(cipherSetupData),
cipherSetupData.cipherKeyLenInBytes,
+ pSessionDesc->cipherSliceType,
pCipherKeyField,
&sizeInBytes);
@@ -368,26 +529,21 @@
.serv_specif_fields);
LacSymQat_CipherHwBlockPopulateKeySetup(
+ pSessionDesc,
&(cipherSetupData),
cipherSetupData.cipherKeyLenInBytes,
+ pSessionDesc->cipherSliceType,
pCipherKeyField,
&sizeInBytes);
}
} break;
case CPA_CY_SYM_OP_ALGORITHM_CHAINING: {
- icp_qat_fw_cipher_auth_cd_ctrl_hdr_t *cd_ctrl =
- (icp_qat_fw_cipher_auth_cd_ctrl_hdr_t
- *)&pSessionDesc->reqCacheFtr.cd_ctrl;
-
- pCipherKeyField = (Cpa8U *)pCdInfo->pData +
- cd_ctrl->cipher_cfg_offset *
- LAC_QUAD_WORD_IN_BYTES +
- sizeof(icp_qat_hw_cipher_config_t);
-
LacSymQat_CipherHwBlockPopulateKeySetup(
+ pSessionDesc,
&(cipherSetupData),
cipherSetupData.cipherKeyLenInBytes,
+ pSessionDesc->cipherSliceType,
pCipherKeyField,
&sizeInBytes);
} break;
@@ -411,10 +567,13 @@
Cpa8U *pInnerState1 = NULL;
Cpa8U *pInnerState2 = NULL;
CpaCySymSessionSetupData sessionSetup = { 0 };
+ Cpa16U cipherConfigSize;
if (pSessionDesc == NULL || pAuthKey == NULL)
return CPA_STATUS_FAIL;
+ cipherConfigSize = LacAlgChain_GetCipherConfigSize(pSessionDesc);
+
icp_qat_fw_cipher_auth_cd_ctrl_hdr_t *cd_ctrl =
(icp_qat_fw_cipher_auth_cd_ctrl_hdr_t *)&pSessionDesc->reqCacheFtr
.cd_ctrl;
@@ -448,30 +607,28 @@
/* Calculate offset of cipher key */
if (pSessionDesc->laCmdId == ICP_QAT_FW_LA_CMD_CIPHER_HASH) {
sessionSetup.cipherSetupData.pCipherKey =
- (Cpa8U *)pHwBlockBaseInDRAM +
- sizeof(icp_qat_hw_cipher_config_t);
+ (Cpa8U *)pHwBlockBaseInDRAM + cipherConfigSize;
} else if (pSessionDesc->laCmdId == ICP_QAT_FW_LA_CMD_HASH_CIPHER) {
sessionSetup.cipherSetupData.pCipherKey =
- pOutHashSetup + sizeof(icp_qat_hw_cipher_config_t);
- } else if (CPA_TRUE == pSessionDesc->isSinglePass) {
+ pOutHashSetup + cipherConfigSize;
+ } else if (SPC == pSessionDesc->singlePassState) {
CpaCySymCipherAlgorithm cipher = pSessionDesc->cipherAlgorithm;
Cpa32U hwBlockOffsetInDRAM = 0;
if (pSessionDesc->cipherDirection ==
CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT) {
sessionSetup.cipherSetupData.pCipherKey =
- (Cpa8U *)pHwBlockBaseInDRAM +
- sizeof(icp_qat_hw_cipher_config_t);
+ (Cpa8U *)pHwBlockBaseInDRAM + cipherConfigSize;
} else {
if (LAC_CIPHER_IS_GCM(cipher))
hwBlockOffsetInDRAM = LAC_QUADWORDS_TO_BYTES(
- LAC_SYM_QAT_CIPHER_OFFSET_IN_DRAM_GCM_SPC);
+ LAC_SYM_QAT_CIPHER_GCM_SPC_OFFSET_IN_DRAM);
else
hwBlockOffsetInDRAM = LAC_QUADWORDS_TO_BYTES(
- LAC_SYM_QAT_CIPHER_OFFSET_IN_DRAM_CHACHA_SPC);
+ LAC_SYM_QAT_CIPHER_CHACHA_SPC_OFFSET_IN_DRAM);
sessionSetup.cipherSetupData.pCipherKey =
(Cpa8U *)pHwBlockBaseInDRAM + hwBlockOffsetInDRAM +
- sizeof(icp_qat_hw_cipher_config_t);
+ cipherConfigSize;
}
}
@@ -497,8 +654,7 @@
/* SHRAM Constants Table not used for Auth-Enc */
}
} else if (CPA_CY_SYM_HASH_SNOW3G_UIA2 == pSessionDesc->hashAlgorithm) {
- Cpa8U *authKey =
- (Cpa8U *)pOutHashSetup + sizeof(icp_qat_hw_cipher_config_t);
+ Cpa8U *authKey = (Cpa8U *)pOutHashSetup + cipherConfigSize;
memcpy(authKey, pAuthKey, pSessionDesc->authKeyLenInBytes);
} else if (CPA_CY_SYM_HASH_ZUC_EIA3 == pSessionDesc->hashAlgorithm ||
CPA_CY_SYM_HASH_AES_CBC_MAC == pSessionDesc->hashAlgorithm) {
@@ -526,6 +682,242 @@
return status;
}
+static void
+buildCmdData(lac_session_desc_t *pSessionDesc,
+ CpaCySymAlgChainOrder *chainOrder,
+ Cpa16U *proto,
+ icp_qat_fw_serv_specif_flags *laCmdFlags,
+ icp_qat_fw_comn_flags *cmnRequestFlags)
+{
+ /* LW 28 is used to set hash flags for AlgChaining. */
+ icp_qat_fw_cipher_auth_cd_ctrl_hdr_t *cd_ctrl =
+ (icp_qat_fw_cipher_auth_cd_ctrl_hdr_t *)&pSessionDesc->reqCacheFtr
+ .cd_ctrl;
+
+ /* proto refers to Protocol Flags, which is legacy FW <=> IA interface
+ * for ZUC and Snow3G. Use extended protocol flags for AlgChaining.
+ */
+ *proto = ICP_QAT_FW_LA_NO_PROTO; /* no CCM/GCM/Snow3G */
+
+ switch (pSessionDesc->symOperation) {
+ case CPA_CY_SYM_OP_CIPHER:
+ pSessionDesc->laCmdId = ICP_QAT_FW_LA_CMD_CIPHER;
+
+ if (CPA_CY_SYM_CIPHER_SNOW3G_UEA2 ==
+ pSessionDesc->cipherAlgorithm) {
+ *proto = ICP_QAT_FW_LA_SNOW_3G_PROTO;
+ } else if (CPA_CY_SYM_CIPHER_ZUC_EEA3 ==
+ pSessionDesc->cipherAlgorithm) {
+ *proto = ICP_QAT_FW_LA_ZUC_3G_PROTO;
+ }
+ if (LAC_CIPHER_IS_CCM(pSessionDesc->cipherAlgorithm)) {
+ *proto = ICP_QAT_FW_LA_CCM_PROTO;
+ }
+ break;
+
+ case CPA_CY_SYM_OP_HASH:
+ pSessionDesc->laCmdId = ICP_QAT_FW_LA_CMD_AUTH;
+ if (CPA_CY_SYM_HASH_SNOW3G_UIA2 ==
+ pSessionDesc->hashAlgorithm) {
+ *proto = ICP_QAT_FW_LA_SNOW_3G_PROTO;
+ } else if (CPA_CY_SYM_HASH_ZUC_EIA3 ==
+ pSessionDesc->hashAlgorithm) {
+ *proto = ICP_QAT_FW_LA_ZUC_3G_PROTO;
+ }
+ break;
+
+ case CPA_CY_SYM_OP_ALGORITHM_CHAINING:
+ if (LAC_CIPHER_IS_CCM(pSessionDesc->cipherAlgorithm)) {
+ *proto = ICP_QAT_FW_LA_CCM_PROTO;
+
+ /* Derive chainOrder from direction for isAuthEncryptOp
+ * cases */
+ /* For CCM & GCM modes: force digest verify flag _TRUE
+ for decrypt and _FALSE for encrypt. For all other
+ cases use user defined value */
+
+ if (CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT ==
+ pSessionDesc->cipherDirection) {
+ *chainOrder =
+ CPA_CY_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER;
+ pSessionDesc->digestVerify = CPA_FALSE;
+ } else {
+ *chainOrder =
+ CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH;
+ pSessionDesc->digestVerify = CPA_TRUE;
+ }
+ } else if (LAC_CIPHER_IS_GCM(pSessionDesc->cipherAlgorithm)) {
+ *proto = ICP_QAT_FW_LA_GCM_PROTO;
+
+ /* Derive chainOrder from direction for isAuthEncryptOp
+ * cases */
+ /* For CCM & GCM modes: force digest verify flag _TRUE
+ for decrypt and _FALSE for encrypt. For all other
+ cases use user defined value */
+
+ if (CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT ==
+ pSessionDesc->cipherDirection) {
+ *chainOrder =
+ CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH;
+ pSessionDesc->digestVerify = CPA_FALSE;
+ } else {
+ *chainOrder =
+ CPA_CY_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER;
+ pSessionDesc->digestVerify = CPA_TRUE;
+ }
+ } else if (LAC_CIPHER_IS_CHACHA(
+ pSessionDesc->cipherAlgorithm)) {
+ if (CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT ==
+ pSessionDesc->cipherDirection) {
+ *chainOrder =
+ CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH;
+ } else {
+ *chainOrder =
+ CPA_CY_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER;
+ }
+ } else {
+ pSessionDesc->isAuthEncryptOp = CPA_FALSE;
+
+ if (CPA_CY_SYM_CIPHER_SNOW3G_UEA2 ==
+ pSessionDesc->cipherAlgorithm) {
+ *proto = ICP_QAT_FW_LA_SNOW_3G_PROTO;
+ } else if (CPA_CY_SYM_CIPHER_ZUC_EEA3 ==
+ pSessionDesc->cipherAlgorithm) {
+ *proto = ICP_QAT_FW_LA_ZUC_3G_PROTO;
+ }
+
+ if (CPA_CY_SYM_HASH_SNOW3G_UIA2 ==
+ pSessionDesc->hashAlgorithm) {
+ /* Need to set LW 28 hash flags as well. */
+ ICP_QAT_FW_HASH_FLAG_SNOW3G_UIA2_SET(
+ cd_ctrl->hash_flags, QAT_FW_LA_SNOW3G_UIA2);
+ } else if (CPA_CY_SYM_HASH_ZUC_EIA3 ==
+ pSessionDesc->hashAlgorithm) {
+ /* Need to set LW 28 hash flags as well. */
+ ICP_QAT_FW_HASH_FLAG_ZUC_EIA3_SET(
+ cd_ctrl->hash_flags, QAT_FW_LA_ZUC_EIA3);
+ }
+ }
+
+ if (CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH ==
+ *chainOrder) {
+ pSessionDesc->laCmdId = ICP_QAT_FW_LA_CMD_CIPHER_HASH;
+ } else if (CPA_CY_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER ==
+ *chainOrder) {
+ pSessionDesc->laCmdId = ICP_QAT_FW_LA_CMD_HASH_CIPHER;
+ }
+ break;
+
+ default:
+ break;
+ }
+
+ /*
+ * Build the header flags with the default settings for this session.
+ */
+ if (pSessionDesc->isDPSession == CPA_TRUE) {
+ *cmnRequestFlags =
+ ICP_QAT_FW_COMN_FLAGS_BUILD(QAT_COMN_CD_FLD_TYPE_64BIT_ADR,
+ LAC_SYM_DP_QAT_PTR_TYPE);
+ } else {
+ *cmnRequestFlags =
+ ICP_QAT_FW_COMN_FLAGS_BUILD(QAT_COMN_CD_FLD_TYPE_64BIT_ADR,
+ LAC_SYM_DEFAULT_QAT_PTR_TYPE);
+ }
+
+ LacSymQat_LaSetDefaultFlags(laCmdFlags, pSessionDesc->symOperation);
+
+ return;
+}
+
+static void
+updateLaCmdFlags(lac_session_desc_t *pSessionDesc,
+ Cpa16U proto,
+ icp_qat_fw_serv_specif_flags *laCmdFlags)
+{
+ if (pSessionDesc->isAuth) {
+ if (pSessionDesc->digestVerify) {
+ ICP_QAT_FW_LA_CMP_AUTH_SET(*laCmdFlags,
+ ICP_QAT_FW_LA_CMP_AUTH_RES);
+ ICP_QAT_FW_LA_RET_AUTH_SET(
+ *laCmdFlags, ICP_QAT_FW_LA_NO_RET_AUTH_RES);
+ } else {
+ ICP_QAT_FW_LA_RET_AUTH_SET(*laCmdFlags,
+ ICP_QAT_FW_LA_RET_AUTH_RES);
+ ICP_QAT_FW_LA_CMP_AUTH_SET(
+ *laCmdFlags, ICP_QAT_FW_LA_NO_CMP_AUTH_RES);
+ }
+ }
+
+ if ((CPA_CY_SYM_CIPHER_ZUC_EEA3 == pSessionDesc->cipherAlgorithm) ||
+ (CPA_CY_SYM_HASH_ZUC_EIA3 == pSessionDesc->hashAlgorithm)) {
+ /* New bit position (12) for ZUC. The FW provides a specific
+ * macro to use to set the ZUC proto flag. With the new FW I/F
+ * this needs to be set for both Cipher and Auth */
+ ICP_QAT_FW_LA_ZUC_3G_PROTO_FLAG_SET(*laCmdFlags, proto);
+ } else {
+ /* Configure the common header */
+ ICP_QAT_FW_LA_PROTO_SET(*laCmdFlags, proto);
+ }
+
+ /* set Append flag, if digest is appended */
+ if (pSessionDesc->digestIsAppended) {
+ ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(
+ *laCmdFlags, ICP_QAT_FW_LA_DIGEST_IN_BUFFER);
+ } else {
+ ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(
+ *laCmdFlags, ICP_QAT_FW_LA_NO_DIGEST_IN_BUFFER);
+ }
+}
+
+static lac_single_pass_state_t
+LacSymAlgChain_GetSpcState(CpaCySymCipherAlgorithm cipher,
+ CpaCySymHashAlgorithm hash,
+ Cpa32U capabilitiesMask)
+{
+ lac_single_pass_state_t state = NON_SPC;
+ if (capabilitiesMask & ICP_ACCEL_CAPABILITIES_CHACHA_POLY) {
+ switch (cipher) {
+ case CPA_CY_SYM_CIPHER_CHACHA: {
+ if (CPA_CY_SYM_HASH_POLY == hash)
+ state = SPC;
+ break;
+ }
+ case CPA_CY_SYM_CIPHER_AES_GCM: {
+ if ((CPA_CY_SYM_HASH_AES_GCM == hash) ||
+ (CPA_CY_SYM_HASH_AES_GMAC == hash))
+ state = LIKELY_SPC;
+ break;
+ }
+ case CPA_CY_SYM_CIPHER_AES_CCM: {
+ if (LAC_CIPHER_AES_V2(capabilitiesMask))
+ state = SPC;
+ }
+ default:
+ /* Do Nothing as it is NON_SPC */
+ break;
+ }
+ }
+ return state;
+}
+
+static CpaBoolean
+LacAlgChain_UseStatefulSha3ContentDesc(CpaBoolean partialsNotRequired,
+ Cpa32U capabilitiesMask,
+ lac_session_desc_t *pSessionDesc)
+{
+ CpaBoolean hasSha3Ext =
+ ICP_ACCEL_CAPABILITIES_SHA3_EXT & capabilitiesMask;
+ CpaBoolean useStatefulSha3DescFlag = CPA_FALSE;
+
+ if (hasSha3Ext && !partialsNotRequired &&
+ (pSessionDesc->symOperation == CPA_CY_SYM_OP_HASH) &&
+ LAC_HASH_IS_SHA3(pSessionDesc->hashAlgorithm)) {
+ useStatefulSha3DescFlag = CPA_TRUE;
+ }
+ return useStatefulSha3DescFlag;
+}
+
/** @ingroup LacAlgChain */
CpaStatus
LacAlgChain_SessionInit(const CpaInstanceHandle instanceHandle,
@@ -534,6 +926,7 @@
{
CpaStatus stat, status = CPA_STATUS_SUCCESS;
sal_qat_content_desc_info_t *pCdInfo = NULL;
+ sal_qat_content_desc_info_t *pCdInfoOptimised = NULL;
sal_crypto_service_t *pService = (sal_crypto_service_t *)instanceHandle;
Cpa32U capabilitiesMask =
pService->generic_service_info.capabilitiesMask;
@@ -543,7 +936,9 @@
Cpa32U optimisedHwBlockOffsetInDRAM = 0;
Cpa8U cipherOffsetInConstantsTable = 0;
Cpa8U hashOffsetInConstantsTable = 0;
+ icp_qat_fw_comn_flags cmnRequestFlags = 0;
icp_qat_fw_comn_req_t *pMsg = NULL;
+ icp_qat_fw_comn_req_t *pMsgS = NULL;
const CpaCySymCipherSetupData *pCipherData;
const CpaCySymHashSetupData *pHashData;
Cpa16U proto = ICP_QAT_FW_LA_NO_PROTO; /* no CCM/GCM/Snow3G */
@@ -584,147 +979,58 @@
pSessionDesc->symOperation = pSessionSetupData->symOperation;
switch (pSessionDesc->symOperation) {
case CPA_CY_SYM_OP_CIPHER:
- pSessionDesc->laCmdId = ICP_QAT_FW_LA_CMD_CIPHER;
- pSessionDesc->isCipher = TRUE;
- pSessionDesc->isAuth = FALSE;
+ pSessionDesc->isCipher = CPA_TRUE;
+ pSessionDesc->isAuth = CPA_FALSE;
pSessionDesc->isAuthEncryptOp = CPA_FALSE;
-
- if (CPA_CY_SYM_CIPHER_SNOW3G_UEA2 ==
- pSessionSetupData->cipherSetupData.cipherAlgorithm) {
- proto = ICP_QAT_FW_LA_SNOW_3G_PROTO;
- } else if (CPA_CY_SYM_CIPHER_ZUC_EEA3 ==
- pSessionSetupData->cipherSetupData.cipherAlgorithm) {
- proto = ICP_QAT_FW_LA_ZUC_3G_PROTO;
- }
+ pSessionDesc->singlePassState = NON_SPC;
break;
case CPA_CY_SYM_OP_HASH:
- pSessionDesc->laCmdId = ICP_QAT_FW_LA_CMD_AUTH;
- pSessionDesc->isCipher = FALSE;
- pSessionDesc->isAuth = TRUE;
+ pSessionDesc->isCipher = CPA_FALSE;
+ pSessionDesc->isAuth = CPA_TRUE;
pSessionDesc->isAuthEncryptOp = CPA_FALSE;
-
- if (CPA_CY_SYM_HASH_SNOW3G_UIA2 ==
- pSessionSetupData->hashSetupData.hashAlgorithm) {
- proto = ICP_QAT_FW_LA_SNOW_3G_PROTO;
- } else if (CPA_CY_SYM_HASH_ZUC_EIA3 ==
- pSessionSetupData->hashSetupData.hashAlgorithm) {
- proto = ICP_QAT_FW_LA_ZUC_3G_PROTO;
- }
-
+ pSessionDesc->singlePassState = NON_SPC;
break;
- case CPA_CY_SYM_OP_ALGORITHM_CHAINING:
- pSessionDesc->isCipher = TRUE;
- pSessionDesc->isAuth = TRUE;
-
- {
- /* set up some useful shortcuts */
- CpaCySymCipherAlgorithm cipherAlgorithm =
- pSessionSetupData->cipherSetupData.cipherAlgorithm;
- CpaCySymCipherDirection cipherDir =
- pSessionSetupData->cipherSetupData.cipherDirection;
-
- if (LAC_CIPHER_IS_CCM(cipherAlgorithm)) {
- pSessionDesc->isAuthEncryptOp = CPA_TRUE;
- pSessionDesc->digestIsAppended = CPA_TRUE;
- proto = ICP_QAT_FW_LA_CCM_PROTO;
-
- /* Derive chainOrder from direction for
- * isAuthEncryptOp
- * cases */
- /* For CCM & GCM modes: force digest verify flag
- _TRUE
- for decrypt and _FALSE for encrypt. For all
- other cases
- use user defined value */
-
- if (CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT ==
- cipherDir) {
- chainOrder =
- CPA_CY_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER;
- pSessionDesc->digestVerify = CPA_FALSE;
- } else {
- chainOrder =
- CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH;
- pSessionDesc->digestVerify = CPA_TRUE;
- }
- } else if (LAC_CIPHER_IS_GCM(cipherAlgorithm)) {
- pSessionDesc->isAuthEncryptOp = CPA_TRUE;
- proto = ICP_QAT_FW_LA_GCM_PROTO;
-
- /* Derive chainOrder from direction for
- * isAuthEncryptOp
- * cases */
- /* For CCM & GCM modes: force digest verify flag
- _TRUE
- for decrypt and _FALSE for encrypt. For all
- other cases
- use user defined value */
-
- if (CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT ==
- cipherDir) {
- chainOrder =
- CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH;
- pSessionDesc->digestVerify = CPA_FALSE;
- } else {
- chainOrder =
- CPA_CY_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER;
- pSessionDesc->digestVerify = CPA_TRUE;
- }
- } else if (LAC_CIPHER_IS_CHACHA(cipherAlgorithm)) {
- pSessionDesc->isAuthEncryptOp = CPA_TRUE;
- proto = ICP_QAT_FW_LA_SINGLE_PASS_PROTO;
-
- if (CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT ==
- cipherDir) {
- chainOrder =
- CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH;
- } else {
- chainOrder =
- CPA_CY_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER;
- }
- } else {
- pSessionDesc->isAuthEncryptOp = CPA_FALSE;
- /* Use the chainOrder passed in */
- chainOrder = pSessionSetupData->algChainOrder;
- if ((chainOrder !=
- CPA_CY_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER) &&
- (chainOrder !=
- CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH)) {
- LAC_INVALID_PARAM_LOG("algChainOrder");
- return CPA_STATUS_INVALID_PARAM;
- }
-
- if (CPA_CY_SYM_HASH_SNOW3G_UIA2 ==
- pSessionSetupData->hashSetupData
- .hashAlgorithm) {
- proto = ICP_QAT_FW_LA_SNOW_3G_PROTO;
- } else if (CPA_CY_SYM_HASH_ZUC_EIA3 ==
- pSessionSetupData->hashSetupData
- .hashAlgorithm) {
- proto = ICP_QAT_FW_LA_ZUC_3G_PROTO;
- }
- }
-
- if (CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH ==
- chainOrder) {
- pSessionDesc->laCmdId =
- ICP_QAT_FW_LA_CMD_CIPHER_HASH;
- } else if (
- CPA_CY_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER ==
- chainOrder) {
- pSessionDesc->laCmdId =
- ICP_QAT_FW_LA_CMD_HASH_CIPHER;
+ case CPA_CY_SYM_OP_ALGORITHM_CHAINING: {
+ pSessionDesc->isCipher = CPA_TRUE;
+ pSessionDesc->isAuth = CPA_TRUE;
+ pSessionDesc->singlePassState =
+ LacSymAlgChain_GetSpcState(pCipherData->cipherAlgorithm,
+ pHashData->hashAlgorithm,
+ capabilitiesMask);
+
+ switch (pSessionSetupData->cipherSetupData.cipherAlgorithm) {
+ case CPA_CY_SYM_CIPHER_AES_CCM: {
+ pSessionDesc->isAuthEncryptOp = CPA_TRUE;
+ pSessionDesc->digestIsAppended = CPA_TRUE;
+ } break;
+ case CPA_CY_SYM_CIPHER_AES_GCM:
+ case CPA_CY_SYM_CIPHER_CHACHA:
+ pSessionDesc->isAuthEncryptOp = CPA_TRUE;
+ break;
+ default: {
+ pSessionDesc->isAuthEncryptOp = CPA_FALSE;
+ /* Use the chainOrder passed in */
+ chainOrder = pSessionSetupData->algChainOrder;
+ if ((chainOrder !=
+ CPA_CY_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER) &&
+ (chainOrder !=
+ CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH)) {
+ LAC_INVALID_PARAM_LOG("algChainOrder");
+ return CPA_STATUS_INVALID_PARAM;
}
+ } break;
}
- break;
+ } break;
default:
+ pSessionDesc->singlePassState = NON_SPC;
break;
}
if (pSessionDesc->isCipher) {
-/* Populate cipher specific session data */
+ /* Populate cipher specific session data */
- status = LacCipher_SessionSetupDataCheck(pCipherData);
+ status = LacCipher_SessionSetupDataCheck(pCipherData,
+ capabilitiesMask);
if (CPA_STATUS_SUCCESS == status) {
pSessionDesc->cipherAlgorithm =
@@ -735,16 +1041,12 @@
pCipherData->cipherDirection;
/* ARC4 base key isn't added to the content descriptor,
- * because
- * we don't need to pass it directly to the QAT engine.
- * Instead
- * an initial cipher state & key matrix is derived from
- * the
- * base key and provided to the QAT through the state
- * pointer
- * in the request params. We'll store this initial state
- * in
- * the session descriptor. */
+ * because we don't need to pass it directly to the QAT
+ * engine. Instead an initial cipher state & key matrix
+ * is derived from the base key and provided to the QAT
+ * through the state pointer in the request params.
+ * We'll store this initial state in the session
+ * descriptor. */
if (LAC_CIPHER_IS_ARC4(pSessionDesc->cipherAlgorithm)) {
LacSymQat_CipherArc4StateInit(
@@ -809,27 +1111,20 @@
ICP_QAT_HW_AUTH_MODE0;
} else /* CPA_CY_SYM_HASH_MODE_AUTH
&& anything except CPA_CY_SYM_HASH_AES_CBC_MAC
- */
+ */
{
if (IS_HMAC_ALG(pHashData->hashAlgorithm)) {
- /* SHA3 and SM3 HMAC do not support
- * precompute, force MODE2
- * for AUTH */
- if ((CPA_CY_SYM_HASH_SHA3_224 ==
- pHashData->hashAlgorithm) ||
- (CPA_CY_SYM_HASH_SHA3_256 ==
- pHashData->hashAlgorithm) ||
- (CPA_CY_SYM_HASH_SHA3_384 ==
- pHashData->hashAlgorithm) ||
- (CPA_CY_SYM_HASH_SHA3_512 ==
- pHashData->hashAlgorithm) ||
+ /* SHA3 HMAC and SM3 do not support
+ * precompute, force MODE2 for AUTH */
+ if (LAC_HASH_IS_SHA3(
+ pHashData->hashAlgorithm) ||
(CPA_CY_SYM_HASH_SM3 ==
pHashData->hashAlgorithm)) {
pSessionDesc->qatHashMode =
ICP_QAT_HW_AUTH_MODE2;
} else {
pSessionDesc->qatHashMode =
- ICP_QAT_HW_AUTH_MODE1;
+ pService->qatHmacMode;
}
} else if (CPA_CY_SYM_HASH_ZUC_EIA3 ==
pHashData->hashAlgorithm) {
@@ -847,39 +1142,70 @@
* build the message templates
* create two content descriptors in the case we can support using SHRAM
* constants and an optimised content descriptor. we have to do this in
- *case
- * of partials.
- * 64 byte content desciptor is used in the SHRAM case for
- *AES-128-HMAC-SHA1
+ *case of partials. 64 byte content desciptor is used in the SHRAM case
+ *for AES-128-HMAC-SHA1
*-----------------------------------------------------------------------*/
if (CPA_STATUS_SUCCESS == status) {
+ pSessionDesc->cipherSliceType =
+ LacCipher_GetCipherSliceType(pService,
+ pSessionDesc->cipherAlgorithm,
+ pSessionDesc->hashAlgorithm);
LacSymCheck_IsPartialSupported(capabilitiesMask, pSessionDesc);
+ pSessionDesc->useOptimisedContentDesc = CPA_FALSE;
+ pSessionDesc->useStatefulSha3ContentDesc = CPA_FALSE;
+
+ /* Build configuration data */
+ buildCmdData(pSessionDesc,
+ &chainOrder,
+ &proto,
+ &pSessionDesc->laCmdFlags,
+ &cmnRequestFlags);
+
+ if (ICP_QAT_FW_LA_USE_UCS_SLICE_TYPE ==
+ pSessionDesc->cipherSliceType)
+ pSessionDesc->useSymConstantsTable = CPA_FALSE;
+ else
+ pSessionDesc->useSymConstantsTable =
+ LacSymQat_UseSymConstantsTable(
+ pSessionDesc,
+ &cipherOffsetInConstantsTable,
+ &hashOffsetInConstantsTable);
+
+ /* for a certain combination of Algorthm Chaining we want to
+ use an optimised cd block */
+
+ if (pSessionDesc->symOperation ==
+ CPA_CY_SYM_OP_ALGORITHM_CHAINING &&
+ pSessionDesc->useSymConstantsTable == CPA_TRUE) {
+ pSessionDesc->useOptimisedContentDesc =
+ LacSymQat_UseOptimisedContentDesc(pSessionDesc);
+ }
+
+ /* check whether we need to construct content desc for stateful
+ * SHA3 */
+ pSessionDesc->useStatefulSha3ContentDesc =
+ LacAlgChain_UseStatefulSha3ContentDesc(
+ pSessionSetupData->partialsNotRequired,
+ capabilitiesMask,
+ pSessionDesc);
/* setup some convenience pointers */
pCdInfo = &(pSessionDesc->contentDescInfo);
pHwBlockBaseInDRAM = (Cpa8U *)pCdInfo->pData;
hwBlockOffsetInDRAM = 0;
- /*
- * Build the header flags with the default settings for this
- * session.
- */
- if (pSessionDesc->isDPSession == CPA_TRUE) {
- pSessionDesc->cmnRequestFlags =
- ICP_QAT_FW_COMN_FLAGS_BUILD(
- QAT_COMN_CD_FLD_TYPE_64BIT_ADR,
- LAC_SYM_DP_QAT_PTR_TYPE);
- } else {
- pSessionDesc->cmnRequestFlags =
- ICP_QAT_FW_COMN_FLAGS_BUILD(
- QAT_COMN_CD_FLD_TYPE_64BIT_ADR,
- LAC_SYM_DEFAULT_QAT_PTR_TYPE);
+ /* set up the pointer for the optimised content desc if this is
+ * possible we still have to support both cd types in case of
+ * partials so we construct both */
+ if (pSessionDesc->useOptimisedContentDesc == CPA_TRUE) {
+ pCdInfoOptimised =
+ &(pSessionDesc->contentDescOptimisedInfo);
+ pOptimisedHwBlockBaseInDRAM =
+ (Cpa8U *)pCdInfoOptimised->pData;
+ optimisedHwBlockOffsetInDRAM = 0;
}
- LacSymQat_LaSetDefaultFlags(&pSessionDesc->laCmdFlags,
- pSessionDesc->symOperation);
-
switch (pSessionDesc->symOperation) {
case CPA_CY_SYM_OP_CIPHER: {
LacAlgChain_CipherCDBuild(
@@ -890,13 +1216,22 @@
&pSessionDesc->cmnRequestFlags,
&pSessionDesc->laCmdFlags,
pHwBlockBaseInDRAM,
- &hwBlockOffsetInDRAM);
+ &hwBlockOffsetInDRAM,
+ capabilitiesMask);
+
+ if (pSessionDesc->useSymConstantsTable) {
+ LacAlgChain_CipherCDBuild_ForSHRAM(
+ pCipherData,
+ pSessionDesc,
+ ICP_QAT_FW_SLICE_DRAM_WR,
+ cipherOffsetInConstantsTable);
+ }
} break;
case CPA_CY_SYM_OP_HASH:
LacAlgChain_HashCDBuild(pHashData,
instanceHandle,
pSessionDesc,
- ICP_QAT_FW_SLICE_DRAM_WR,
+ ICP_QAT_FW_SLICE_NULL,
hashOffsetInConstantsTable,
&pSessionDesc->cmnRequestFlags,
&pSessionDesc->laCmdFlags,
@@ -909,14 +1244,10 @@
break;
case CPA_CY_SYM_OP_ALGORITHM_CHAINING:
/* For CCM/GCM, CPM firmware currently expects the
- * cipher and
- * hash h/w setup blocks to be arranged according to the
- * chain
- * order (Except for GCM/CCM, order doesn't actually
- * matter as
- * long as the config offsets are set correctly in CD
- * control
- * blocks
+ * cipher and hash h/w setup blocks to be arranged
+ * according to the chain order (Except for GCM/CCM,
+ * order doesn't actually matter as long as the config
+ * offsets are set correctly in CD control blocks
*/
if (CPA_CY_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER ==
chainOrder) {
@@ -943,11 +1274,20 @@
&pSessionDesc->cmnRequestFlags,
&pSessionDesc->laCmdFlags,
pHwBlockBaseInDRAM,
- &hwBlockOffsetInDRAM);
- if (LAC_CIPHER_IS_SPC(
- pCipherData->cipherAlgorithm,
- pHashData->hashAlgorithm,
- capabilitiesMask)) {
+ &hwBlockOffsetInDRAM,
+ capabilitiesMask);
+
+ if (pSessionDesc->useOptimisedContentDesc) {
+ LacAlgChain_CipherCDBuild_ForOptimisedCD(
+ pCipherData,
+ pSessionDesc,
+ ICP_QAT_FW_SLICE_DRAM_WR,
+ cipherOffsetInConstantsTable,
+ pOptimisedHwBlockBaseInDRAM,
+ &optimisedHwBlockOffsetInDRAM);
+ }
+
+ if (NON_SPC != pSessionDesc->singlePassState) {
pCdInfo->hwBlkSzQuadWords =
(LAC_BYTES_TO_QUADWORDS(
hwBlockOffsetInDRAM));
@@ -957,7 +1297,8 @@
(icp_qat_fw_comn_req_t *)pMsg,
pCdInfo);
}
- } else {
+ } else /* CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH */
+ {
LacAlgChain_CipherCDBuild(
pCipherData,
pSessionDesc,
@@ -966,12 +1307,20 @@
&pSessionDesc->cmnRequestFlags,
&pSessionDesc->laCmdFlags,
pHwBlockBaseInDRAM,
- &hwBlockOffsetInDRAM);
+ &hwBlockOffsetInDRAM,
+ capabilitiesMask);
+
+ if (pSessionDesc->useOptimisedContentDesc) {
+ LacAlgChain_CipherCDBuild_ForOptimisedCD(
+ pCipherData,
+ pSessionDesc,
+ ICP_QAT_FW_SLICE_AUTH,
+ cipherOffsetInConstantsTable,
+ pOptimisedHwBlockBaseInDRAM,
+ &optimisedHwBlockOffsetInDRAM);
+ }
- if (LAC_CIPHER_IS_SPC(
- pCipherData->cipherAlgorithm,
- pHashData->hashAlgorithm,
- capabilitiesMask)) {
+ if (NON_SPC != pSessionDesc->singlePassState) {
pCdInfo->hwBlkSzQuadWords =
LAC_BYTES_TO_QUADWORDS(
hwBlockOffsetInDRAM);
@@ -1009,8 +1358,7 @@
CpaBoolean hashStateBuffer = CPA_TRUE;
/* set up fields in both the cd_ctrl and reqParams which
- * describe
- * the ReqParams block */
+ * describe the ReqParams block */
LacSymQat_HashSetupReqParamsMetaData(
&(pSessionDesc->reqCacheFtr),
instanceHandle,
@@ -1019,6 +1367,17 @@
pSessionDesc->qatHashMode,
pSessionDesc->digestVerify);
+ if (pSessionDesc->useSymConstantsTable) {
+ /* Need to set up for SHRAM Constants Table use also */
+ LacSymQat_HashSetupReqParamsMetaData(
+ &(pSessionDesc->shramReqCacheFtr),
+ instanceHandle,
+ pHashData,
+ hashStateBuffer,
+ pSessionDesc->qatHashMode,
+ pSessionDesc->digestVerify);
+ }
+
/* populate the hash state prefix buffer info structure
* (part of user allocated session memory & the
* buffer itself. For CCM/GCM the buffer is stored in the
@@ -1056,86 +1415,122 @@
pSessionDesc->hashStatePrefixBuffer,
precomputeData.pState1,
precomputeData.pState2);
+ if (pSessionDesc->useOptimisedContentDesc) {
+ status = LacHash_PrecomputeDataCreate(
+ instanceHandle,
+ (CpaCySymSessionSetupData *)
+ pSessionSetupData,
+ LacSymAlgChain_HashPrecomputeDoneCb,
+ pSessionDesc,
+ pSessionDesc->hashStatePrefixBuffer,
+ precomputeDataOptimisedCd.pState1,
+ precomputeDataOptimisedCd.pState2);
+ }
} else if (pHashData->hashAlgorithm ==
CPA_CY_SYM_HASH_AES_CBC_MAC) {
- LAC_OS_BZERO(precomputeData.pState2,
- precomputeData.state2Size);
- memcpy(precomputeData.pState2,
- pHashData->authModeSetupData.authKey,
- pHashData->authModeSetupData
- .authKeyLenInBytes);
- }
- }
- if (CPA_STATUS_SUCCESS == status) {
-
- if (pSessionDesc->digestVerify) {
-
- ICP_QAT_FW_LA_CMP_AUTH_SET(
- pSessionDesc->laCmdFlags,
- ICP_QAT_FW_LA_CMP_AUTH_RES);
- ICP_QAT_FW_LA_RET_AUTH_SET(
- pSessionDesc->laCmdFlags,
- ICP_QAT_FW_LA_NO_RET_AUTH_RES);
- } else {
-
- ICP_QAT_FW_LA_RET_AUTH_SET(
- pSessionDesc->laCmdFlags,
- ICP_QAT_FW_LA_RET_AUTH_RES);
- ICP_QAT_FW_LA_CMP_AUTH_SET(
- pSessionDesc->laCmdFlags,
- ICP_QAT_FW_LA_NO_CMP_AUTH_RES);
+ if (NULL != precomputeData.pState2) {
+ LAC_OS_BZERO(precomputeData.pState2,
+ precomputeData.state2Size);
+ memcpy(precomputeData.pState2,
+ pHashData->authModeSetupData
+ .authKey,
+ pHashData->authModeSetupData
+ .authKeyLenInBytes);
+ }
}
}
}
if (CPA_STATUS_SUCCESS == status) {
+ /* Configure the ContentDescriptor field
+ in the request if not done already */
pCdInfo->hwBlkSzQuadWords =
LAC_BYTES_TO_QUADWORDS(hwBlockOffsetInDRAM);
pMsg = (icp_qat_fw_comn_req_t *)&(pSessionDesc->reqCacheHdr);
-
- /* Configure the ContentDescriptor field
- * in the request if not done already */
SalQatMsg_ContentDescHdrWrite((icp_qat_fw_comn_req_t *)pMsg,
pCdInfo);
- if (CPA_CY_SYM_CIPHER_ZUC_EEA3 ==
- pSessionSetupData->cipherSetupData.cipherAlgorithm ||
- pHashData->hashAlgorithm == CPA_CY_SYM_HASH_ZUC_EIA3) {
- /* New bit position (12) for ZUC. The FW provides a
- * specific macro
- * to use to set the ZUC proto flag. With the new FW I/F
- * this needs
- * to be set for both Cipher and Auth */
- ICP_QAT_FW_LA_ZUC_3G_PROTO_FLAG_SET(
- pSessionDesc->laCmdFlags, proto);
- } else {
- /* Configure the common header */
- ICP_QAT_FW_LA_PROTO_SET(pSessionDesc->laCmdFlags,
- proto);
+ pMsgS =
+ (icp_qat_fw_comn_req_t *)&(pSessionDesc->shramReqCacheHdr);
+ /*If we are using the optimised CD then
+ we have to set this up correctly in the SHARM reqCache*/
+ if (pSessionDesc->useOptimisedContentDesc) {
+ pCdInfoOptimised->hwBlkSzQuadWords =
+ LAC_BYTES_TO_QUADWORDS(
+ optimisedHwBlockOffsetInDRAM);
+ SalQatMsg_ContentDescHdrWrite(
+ (icp_qat_fw_comn_req_t *)pMsgS, pCdInfoOptimised);
}
- /* set Append flag, if digest is appended */
- if (pSessionDesc->digestIsAppended) {
- ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(
- pSessionDesc->laCmdFlags,
- ICP_QAT_FW_LA_DIGEST_IN_BUFFER);
- } else {
- ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(
- pSessionDesc->laCmdFlags,
- ICP_QAT_FW_LA_NO_DIGEST_IN_BUFFER);
- }
+ /* Updates command flags basing on configured alg */
+ updateLaCmdFlags(pSessionDesc,
+ proto,
+ &pSessionDesc->laCmdFlags);
SalQatMsg_CmnHdrWrite((icp_qat_fw_comn_req_t *)pMsg,
ICP_QAT_FW_COMN_REQ_CPM_FW_LA,
pSessionDesc->laCmdId,
pSessionDesc->cmnRequestFlags,
pSessionDesc->laCmdFlags);
+
+ /* Need to duplicate if SHRAM Constants Table used */
+ if (pSessionDesc->useSymConstantsTable) {
+ ICP_QAT_FW_LA_CIPH_AUTH_CFG_OFFSET_FLAG_SET(
+ pSessionDesc->laCmdFlags,
+ ICP_QAT_FW_CIPH_AUTH_CFG_OFFSET_IN_SHRAM_CP);
+
+ if (pSessionDesc->isCipher &&
+ !pSessionDesc->useOptimisedContentDesc) {
+ ICP_QAT_FW_COMN_CD_FLD_TYPE_SET(
+ cmnRequestFlags,
+ QAT_COMN_CD_FLD_TYPE_16BYTE_DATA);
+ }
+
+ SalQatMsg_CmnHdrWrite((icp_qat_fw_comn_req_t *)pMsgS,
+ ICP_QAT_FW_COMN_REQ_CPM_FW_LA,
+ pSessionDesc->laCmdId,
+ cmnRequestFlags,
+ pSessionDesc->laCmdFlags);
+ }
}
return status;
}
+static void
+LacAlgChain_StatefulSha3_SkipStateLoadFlags(icp_qat_fw_la_bulk_req_t *pMsg,
+ Cpa32U packetType,
+ icp_qat_hw_auth_mode_t qatHashMode)
+{
+ icp_qat_fw_auth_cd_ctrl_hdr_t *pAuthCdCtrlHdr = NULL;
+
+ pAuthCdCtrlHdr = (icp_qat_fw_auth_cd_ctrl_hdr_t *)&(pMsg->cd_ctrl);
+
+ if (IS_HASH_MODE_2(qatHashMode)) {
+ if ((ICP_QAT_FW_LA_PARTIAL_START == packetType) ||
+ (ICP_QAT_FW_LA_PARTIAL_NONE == packetType)) {
+ ICP_QAT_FW_HASH_FLAG_SKIP_INNER_STATE1_LOAD_SET(
+ pAuthCdCtrlHdr->hash_flags,
+ QAT_FW_LA_SKIP_INNER_STATE1_LOAD);
+ ICP_QAT_FW_HASH_FLAG_SKIP_OUTER_STATE1_LOAD_SET(
+ pAuthCdCtrlHdr->hash_flags,
+ QAT_FW_LA_SKIP_OUTER_STATE1_LOAD);
+ } else if (ICP_QAT_FW_LA_PARTIAL_END == packetType) {
+ ICP_QAT_FW_HASH_FLAG_SKIP_OUTER_STATE1_LOAD_SET(
+ pAuthCdCtrlHdr->hash_flags,
+ QAT_FW_LA_SKIP_OUTER_STATE1_LOAD);
+ }
+ } else {
+ if ((ICP_QAT_FW_LA_PARTIAL_START == packetType) ||
+ (ICP_QAT_FW_LA_PARTIAL_NONE == packetType)) {
+ ICP_QAT_FW_HASH_FLAG_SKIP_INNER_STATE1_LOAD_SET(
+ pAuthCdCtrlHdr->hash_flags,
+ QAT_FW_LA_SKIP_INNER_STATE1_LOAD);
+ }
+ }
+}
+
/** @ingroup LacAlgChain */
CpaStatus
LacAlgChain_Perform(const CpaInstanceHandle instanceHandle,
@@ -1171,7 +1566,10 @@
CpaCySymHashAlgorithm hash;
Cpa8U paddingLen = 0;
Cpa8U blockLen = 0;
+ CpaBoolean digestIsAppended = CPA_FALSE;
+ Cpa32U aadLenInBytes = 0;
Cpa64U srcPktSize = 0;
+ Cpa64U dstPktSize = 0;
/* Set the command id */
laCmdId = pSessionDesc->laCmdId;
@@ -1179,35 +1577,51 @@
cipher = pSessionDesc->cipherAlgorithm;
hash = pSessionDesc->hashAlgorithm;
+ CpaBoolean isSpCcm =
+ (LAC_CIPHER_IS_CCM(cipher) && LAC_CIPHER_AES_V2(capabilitiesMask));
+
+ if (CPA_CY_SYM_HASH_AES_GMAC == hash) {
+ pSessionDesc->aadLenInBytes = pOpData->messageLenToHashInBytes;
+ if (pOpData->messageLenToHashInBytes == 0 ||
+ pOpData->pAdditionalAuthData != NULL) {
+ LAC_INVALID_PARAM_LOG(
+ "For AES_GMAC, AAD Length "
+ "(messageLenToHashInBytes) must "
+ "be non zero and pAdditionalAuthData "
+ "must be NULL");
+ return CPA_STATUS_INVALID_PARAM;
+ }
+ }
+
+ aadLenInBytes = pSessionDesc->aadLenInBytes;
+
/* Convert Alg Chain Request to Cipher Request for CCP and
* AES_GCM single pass */
- if (!pSessionDesc->isSinglePass &&
- LAC_CIPHER_IS_SPC(cipher, hash, capabilitiesMask) &&
- (LAC_CIPHER_SPC_IV_SIZE == pOpData->ivLenInBytes)) {
+ if ((NON_SPC != pSessionDesc->singlePassState) &&
+ (isSpCcm || (LAC_CIPHER_SPC_IV_SIZE == pOpData->ivLenInBytes))) {
pSessionDesc->laCmdId = ICP_QAT_FW_LA_CMD_CIPHER;
laCmdId = pSessionDesc->laCmdId;
pSessionDesc->symOperation = CPA_CY_SYM_OP_CIPHER;
- pSessionDesc->isSinglePass = CPA_TRUE;
+ pSessionDesc->singlePassState = SPC;
pSessionDesc->isCipher = CPA_TRUE;
pSessionDesc->isAuthEncryptOp = CPA_FALSE;
pSessionDesc->isAuth = CPA_FALSE;
- if (CPA_CY_SYM_HASH_AES_GMAC == pSessionDesc->hashAlgorithm) {
- pSessionDesc->aadLenInBytes =
- pOpData->messageLenToHashInBytes;
- if (ICP_QAT_FW_SPC_AAD_SZ_MAX <
- pSessionDesc->aadLenInBytes) {
+
+ if (CPA_CY_SYM_HASH_AES_GMAC == hash) {
+ if (ICP_QAT_FW_SPC_AAD_SZ_MAX < aadLenInBytes) {
LAC_INVALID_PARAM_LOG(
"aadLenInBytes for AES_GMAC");
return CPA_STATUS_INVALID_PARAM;
}
}
-
/* New bit position (13) for SINGLE PASS.
* The FW provides a specific macro to use to set the proto flag
*/
ICP_QAT_FW_LA_SINGLE_PASS_PROTO_FLAG_SET(
pSessionDesc->laCmdFlags, ICP_QAT_FW_LA_SINGLE_PASS_PROTO);
- ICP_QAT_FW_LA_PROTO_SET(pSessionDesc->laCmdFlags, 0);
+ if (isCyGen2x(pService)) {
+ ICP_QAT_FW_LA_PROTO_SET(pSessionDesc->laCmdFlags, 0);
+ }
pCdInfo = &(pSessionDesc->contentDescInfo);
pHwBlockBaseInDRAM = (Cpa8U *)pCdInfo->pData;
@@ -1215,11 +1629,24 @@
pSessionDesc->cipherDirection) {
if (LAC_CIPHER_IS_GCM(cipher))
hwBlockOffsetInDRAM = LAC_QUADWORDS_TO_BYTES(
- LAC_SYM_QAT_CIPHER_OFFSET_IN_DRAM_GCM_SPC);
- else
+ LAC_SYM_QAT_CIPHER_GCM_SPC_OFFSET_IN_DRAM);
+ else if (LAC_CIPHER_IS_CHACHA(cipher))
hwBlockOffsetInDRAM = LAC_QUADWORDS_TO_BYTES(
- LAC_SYM_QAT_CIPHER_OFFSET_IN_DRAM_CHACHA_SPC);
+ LAC_SYM_QAT_CIPHER_CHACHA_SPC_OFFSET_IN_DRAM);
+ } else if (isSpCcm) {
+ hwBlockOffsetInDRAM = LAC_QUADWORDS_TO_BYTES(
+ LAC_SYM_QAT_CIPHER_CCM_SPC_OFFSET_IN_DRAM);
}
+
+ /* Update cipher slice type */
+ pSessionDesc->cipherSliceType =
+ LacCipher_GetCipherSliceType(pService,
+ pSessionDesc->cipherAlgorithm,
+ pSessionDesc->hashAlgorithm);
+
+ ICP_QAT_FW_LA_SLICE_TYPE_SET(pSessionDesc->laCmdFlags,
+ pSessionDesc->cipherSliceType);
+
/* construct cipherConfig in CD in DRAM */
LacSymQat_CipherHwBlockPopulateCfgData(pSessionDesc,
pHwBlockBaseInDRAM +
@@ -1228,31 +1655,53 @@
SalQatMsg_CmnHdrWrite((icp_qat_fw_comn_req_t *)&(
pSessionDesc->reqSpcCacheHdr),
ICP_QAT_FW_COMN_REQ_CPM_FW_LA,
- pSessionDesc->laCmdId,
+ laCmdId,
+ pSessionDesc->cmnRequestFlags,
+ pSessionDesc->laCmdFlags);
+ } else if ((SPC == pSessionDesc->singlePassState) &&
+ (LAC_CIPHER_SPC_IV_SIZE != pOpData->ivLenInBytes)) {
+ pSessionDesc->symOperation = CPA_CY_SYM_OP_ALGORITHM_CHAINING;
+ pSessionDesc->singlePassState = LIKELY_SPC;
+ pSessionDesc->isCipher = CPA_TRUE;
+ pSessionDesc->isAuthEncryptOp = CPA_TRUE;
+ pSessionDesc->isAuth = CPA_TRUE;
+ pCdInfo = &(pSessionDesc->contentDescInfo);
+ pHwBlockBaseInDRAM = (Cpa8U *)pCdInfo->pData;
+
+ if (CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT ==
+ pSessionDesc->cipherDirection) {
+ pSessionDesc->laCmdId = ICP_QAT_FW_LA_CMD_CIPHER_HASH;
+ } else {
+ pSessionDesc->laCmdId = ICP_QAT_FW_LA_CMD_HASH_CIPHER;
+ }
+
+ laCmdId = pSessionDesc->laCmdId;
+ ICP_QAT_FW_LA_SINGLE_PASS_PROTO_FLAG_SET(
+ pSessionDesc->laCmdFlags, 0);
+ ICP_QAT_FW_LA_PROTO_SET(pSessionDesc->laCmdFlags,
+ ICP_QAT_FW_LA_GCM_PROTO);
+
+ LacSymQat_CipherHwBlockPopulateCfgData(pSessionDesc,
+ pHwBlockBaseInDRAM +
+ hwBlockOffsetInDRAM,
+ &sizeInBytes);
+
+ SalQatMsg_CmnHdrWrite((icp_qat_fw_comn_req_t *)&(
+ pSessionDesc->reqCacheHdr),
+ ICP_QAT_FW_COMN_REQ_CPM_FW_LA,
+ laCmdId,
pSessionDesc->cmnRequestFlags,
pSessionDesc->laCmdFlags);
- } else if (CPA_CY_SYM_HASH_AES_GMAC == pSessionDesc->hashAlgorithm) {
- pSessionDesc->aadLenInBytes = pOpData->messageLenToHashInBytes;
}
- if (LAC_CIPHER_IS_CHACHA(cipher) &&
- (LAC_CIPHER_SPC_IV_SIZE != pOpData->ivLenInBytes)) {
+ else if (LAC_CIPHER_IS_CHACHA(cipher) &&
+ (LAC_CIPHER_SPC_IV_SIZE != pOpData->ivLenInBytes)) {
LAC_INVALID_PARAM_LOG("IV for CHACHA");
return CPA_STATUS_INVALID_PARAM;
- } else if (CPA_CY_SYM_HASH_AES_GMAC == pSessionDesc->hashAlgorithm) {
- if (pOpData->messageLenToHashInBytes == 0 ||
- pOpData->pAdditionalAuthData != NULL) {
- LAC_INVALID_PARAM_LOG(
- "For AES_GMAC, AAD Length "
- "(messageLenToHashInBytes) must "
- "be non zero and pAdditionalAuthData "
- "must be NULL");
- status = CPA_STATUS_INVALID_PARAM;
- }
}
- if (CPA_TRUE == pSessionDesc->isAuthEncryptOp) {
- if (CPA_CY_SYM_HASH_AES_CCM == pSessionDesc->hashAlgorithm) {
+ if ((CPA_TRUE == pSessionDesc->isAuthEncryptOp) || isSpCcm) {
+ if (CPA_CY_SYM_HASH_AES_CCM == hash) {
status = LacSymAlgChain_CheckCCMData(
pOpData->pAdditionalAuthData,
pOpData->pIv,
@@ -1266,9 +1715,8 @@
pOpData->messageLenToCipherInBytes,
pOpData->ivLenInBytes);
}
- } else if (CPA_CY_SYM_HASH_AES_GCM ==
- pSessionDesc->hashAlgorithm) {
- if (pSessionDesc->aadLenInBytes != 0 &&
+ } else if (CPA_CY_SYM_HASH_AES_GCM == hash) {
+ if (aadLenInBytes != 0 &&
pOpData->pAdditionalAuthData == NULL) {
LAC_INVALID_PARAM_LOG("pAdditionalAuthData");
status = CPA_STATUS_INVALID_PARAM;
@@ -1304,37 +1752,48 @@
&srcAddrPhys,
CPA_FALSE,
&(pService->generic_service_info));
- } else {
- status = LacBuffDesc_BufferListDescWrite(
- (CpaBufferList *)pSrcBuffer,
- &srcAddrPhys,
- CPA_FALSE,
- &(pService->generic_service_info));
- }
- if (CPA_STATUS_SUCCESS != status) {
- LAC_LOG_ERROR("Unable to write src buffer descriptors");
- }
-
- /* For out of place operations */
- if ((pSrcBuffer != pDstBuffer) &&
- (CPA_STATUS_SUCCESS == status)) {
- if (IS_ZERO_LENGTH_BUFFER_SUPPORTED(cipher, hash)) {
+ if (CPA_STATUS_SUCCESS != status) {
+ LAC_LOG_ERROR(
+ "Unable to write src buffer descriptors");
+ }
+ /* For out of place operations */
+ if ((pSrcBuffer != pDstBuffer) &&
+ (CPA_STATUS_SUCCESS == status)) {
status =
LacBuffDesc_BufferListDescWriteAndAllowZeroBuffer(
pDstBuffer,
&dstAddrPhys,
CPA_FALSE,
&(pService->generic_service_info));
- } else {
+ if (CPA_STATUS_SUCCESS != status) {
+ LAC_LOG_ERROR(
+ "Unable to write dest buffer descriptors");
+ }
+ }
+ } else {
+ status = LacBuffDesc_BufferListDescWrite(
+ (CpaBufferList *)pSrcBuffer,
+ &srcAddrPhys,
+ CPA_FALSE,
+ &(pService->generic_service_info));
+ if (CPA_STATUS_SUCCESS != status) {
+ LAC_LOG_ERROR(
+ "Unable to write src buffer descriptors in "
+ "LacBuffDesc_BufferListDescWrite");
+ }
+ /* For out of place operations */
+ if ((pSrcBuffer != pDstBuffer) &&
+ (CPA_STATUS_SUCCESS == status)) {
status = LacBuffDesc_BufferListDescWrite(
pDstBuffer,
&dstAddrPhys,
CPA_FALSE,
&(pService->generic_service_info));
- }
- if (CPA_STATUS_SUCCESS != status) {
- LAC_LOG_ERROR(
- "Unable to write dest buffer descriptors");
+ if (CPA_STATUS_SUCCESS != status) {
+ LAC_LOG_ERROR(
+ "Unable to write dest buffer descriptors in "
+ "LacBuffDesc_BufferListDescWrite");
+ }
}
}
}
@@ -1359,9 +1818,9 @@
* the first partial has been sent. Set a flag here so the
* response knows to do this.
*/
- if ((laCmdId != ICP_QAT_FW_LA_CMD_AUTH) &&
+ if (LAC_CIPHER_IS_XTS_MODE(cipher) &&
+ (laCmdId != ICP_QAT_FW_LA_CMD_AUTH) &&
(CPA_CY_SYM_PACKET_TYPE_PARTIAL == pOpData->packetType) &&
- (LAC_CIPHER_IS_XTS_MODE(pSessionDesc->cipherAlgorithm)) &&
(qatPacketType == ICP_QAT_FW_LA_PARTIAL_START)) {
pCookie->updateKeySizeOnRecieve = CPA_TRUE;
}
@@ -1374,26 +1833,21 @@
pMsg = &(pCookie->qatMsg);
pMsgDummy = (Cpa8U *)pMsg;
- if (pSessionDesc->isSinglePass) {
+ if (SPC == pSessionDesc->singlePassState) {
pCacheDummyHdr =
(Cpa8U *)&(pSessionDesc->reqSpcCacheHdr);
pCacheDummyFtr =
(Cpa8U *)&(pSessionDesc->reqSpcCacheFtr);
} else {
/* Normally, we want to use the SHRAM Constants Table if
- * possible
- * for best performance (less DRAM accesses incurred by
- * CPM). But
- * we can't use it for partial-packet hash operations.
- * This is why
- * we build 2 versions of the message template at
- * sessionInit,
+ * possible for best performance (less DRAM accesses
+ * incurred by CPM). But we can't use it for
+ * partial-packet hash operations. This is why we build
+ * 2 versions of the message template at sessionInit,
* one for SHRAM Constants Table usage and the other
- * (default) for
- * Content Descriptor h/w setup data in DRAM. And we
- * chose between
- * them here on a per-request basis, when we know the
- * packetType
+ * (default) for Content Descriptor h/w setup data in
+ * DRAM. And we chose between them here on a
+ * per-request basis, when we know the packetType
*/
if ((!pSessionDesc->useSymConstantsTable) ||
(pSessionDesc->isAuth &&
@@ -1418,8 +1872,9 @@
0,
(LAC_LONG_WORD_IN_BYTES *
LAC_SIZE_OF_CACHE_TO_CLEAR_IN_LW));
- memcpy(pMsgDummy + (LAC_LONG_WORD_IN_BYTES *
- LAC_START_OF_CACHE_FTR_IN_LW),
+ memcpy(pMsgDummy +
+ (LAC_LONG_WORD_IN_BYTES *
+ LAC_START_OF_CACHE_FTR_IN_LW),
pCacheDummyFtr,
(LAC_LONG_WORD_IN_BYTES * LAC_SIZE_OF_CACHE_FTR_IN_LW));
/*
@@ -1442,11 +1897,11 @@
LacSymQat_LaPacketCommandFlagSet(
qatPacketType,
laCmdId,
- pSessionDesc->cipherAlgorithm,
+ cipher,
&pMsg->comn_hdr.serv_specif_flags,
pOpData->ivLenInBytes);
- if (pSessionDesc->isSinglePass) {
+ if (SPC == pSessionDesc->singlePassState) {
ICP_QAT_FW_LA_GCM_IV_LEN_FLAG_SET(
pMsg->comn_hdr.serv_specif_flags,
ICP_QAT_FW_LA_GCM_IV_LEN_NOT_12_OCTETS);
@@ -1463,7 +1918,11 @@
}
}
+ ICP_QAT_FW_LA_SLICE_TYPE_SET(pMsg->comn_hdr.serv_specif_flags,
+ pSessionDesc->cipherSliceType);
+
LacBuffDesc_BufferListTotalSizeGet(pSrcBuffer, &srcPktSize);
+ LacBuffDesc_BufferListTotalSizeGet(pDstBuffer, &dstPktSize);
/*
* Populate the CipherRequestParams section of the Request
@@ -1472,8 +1931,9 @@
Cpa8U *pIvBuffer = NULL;
- status = LacCipher_PerformParamCheck(
- pSessionDesc->cipherAlgorithm, pOpData, srcPktSize);
+ status = LacCipher_PerformParamCheck(cipher,
+ pOpData,
+ srcPktSize);
if (CPA_STATUS_SUCCESS != status) {
/* free the cookie */
Lac_MemPoolEntryFree(pCookie);
@@ -1488,19 +1948,18 @@
qatPacketType,
&pIvBuffer);
}
- if (pSessionDesc->isSinglePass &&
+ if ((SPC == pSessionDesc->singlePassState) &&
((ICP_QAT_FW_LA_PARTIAL_MID == qatPacketType) ||
(ICP_QAT_FW_LA_PARTIAL_END == qatPacketType))) {
/* For SPC stateful cipher state size for mid
- * and
- * end partial packet is 48 bytes
+ * and end partial packet is 48 bytes
*/
pSpcCdCtrlHdr =
(icp_qat_fw_cipher_cd_ctrl_hdr_t *)&(
pMsg->cd_ctrl);
pSpcCdCtrlHdr->cipher_state_sz =
LAC_BYTES_TO_QUADWORDS(
- LAC_SYM_QAT_CIPHER_STATE_SIZE_SPC);
+ LAC_SYM_QAT_CIPHER_SPC_STATE_SIZE);
}
/*populate the cipher request parameters */
if (CPA_STATUS_SUCCESS == status) {
@@ -1525,6 +1984,7 @@
if (status == CPA_STATUS_SUCCESS) {
status =
LacSymQat_CipherRequestParamsPopulate(
+ pSessionDesc,
pMsg,
pOpData
->cryptoStartSrcOffsetInBytes,
@@ -1535,39 +1995,45 @@
}
}
- if (CPA_STATUS_SUCCESS == status &&
- pSessionDesc->isSinglePass) {
+ if ((SPC == pSessionDesc->singlePassState) &&
+ CPA_STATUS_SUCCESS == status) {
Cpa64U aadBufferPhysAddr = 0;
/* For CHACHA and AES-GCM there is an AAD buffer
- * if
- * aadLenInBytes is nonzero In case of AES-GMAC,
- * AAD buffer
- * passed in the src buffer.
+ * if aadLenInBytes is nonzero In case of
+ * AES-GMAC, AAD buffer passed in the src
+ * buffer.
*/
- if (0 != pSessionDesc->aadLenInBytes &&
- CPA_CY_SYM_HASH_AES_GMAC !=
- pSessionDesc->hashAlgorithm) {
+ if ((0 != aadLenInBytes &&
+ CPA_CY_SYM_HASH_AES_GMAC != hash) ||
+ isSpCcm) {
LAC_CHECK_NULL_PARAM(
pOpData->pAdditionalAuthData);
+ Cpa32U aadDataLen =
+ pSessionDesc->aadLenInBytes;
+
+ /* In case of AES_CCM, B0 block size and
+ * 2 bytes of AAD len encoding need to
+ * be added to total AAD data len */
+ if (isSpCcm)
+ aadDataLen +=
+ LAC_CIPHER_CCM_AAD_OFFSET;
+
blockLen =
LacSymQat_CipherBlockSizeBytesGet(
- pSessionDesc->cipherAlgorithm);
- if ((pSessionDesc->aadLenInBytes %
- blockLen) != 0) {
+ cipher);
+ if ((aadDataLen % blockLen) != 0) {
paddingLen = blockLen -
- (pSessionDesc
- ->aadLenInBytes %
- blockLen);
+ (aadDataLen % blockLen);
memset(
- &pOpData->pAdditionalAuthData
- [pSessionDesc
- ->aadLenInBytes],
+ &pOpData
+ ->pAdditionalAuthData
+ [aadDataLen],
0,
paddingLen);
}
- /* User OpData memory being used for aad
+ /* User OpData memory being used for AAD
* buffer */
/* get the physical address */
aadBufferPhysAddr =
@@ -1588,10 +2054,33 @@
*)((Cpa8U *)&(
pMsg->serv_specif_rqpars) +
ICP_QAT_FW_CIPHER_REQUEST_PARAMETERS_OFFSET);
- pCipherReqParams->spc_aad_addr =
- aadBufferPhysAddr;
- pCipherReqParams->spc_aad_sz =
- pSessionDesc->aadLenInBytes;
+
+ icp_qat_fw_la_cipher_20_req_params_t
+ *pCipher20ReqParams =
+ (void
+ *)((Cpa8U *)&(
+ pMsg->serv_specif_rqpars) +
+ ICP_QAT_FW_CIPHER_REQUEST_PARAMETERS_OFFSET);
+
+ if (isCyGen4x(pService)) {
+ pCipher20ReqParams
+ ->spc_aad_addr =
+ aadBufferPhysAddr;
+ pCipher20ReqParams->spc_aad_sz =
+ pSessionDesc->aadLenInBytes;
+ pCipher20ReqParams
+ ->spc_aad_offset = 0;
+ if (isSpCcm)
+ pCipher20ReqParams
+ ->spc_aad_sz +=
+ LAC_CIPHER_CCM_AAD_OFFSET;
+ } else {
+ pCipherReqParams->spc_aad_addr =
+ aadBufferPhysAddr;
+ pCipherReqParams->spc_aad_sz =
+ (Cpa16U)pSessionDesc
+ ->aadLenInBytes;
+ }
if (CPA_TRUE !=
pSessionDesc->digestIsAppended) {
@@ -1605,13 +2094,24 @@
->generic_service_info,
pOpData->pDigestResult);
if (0 != digestBufferPhysAddr) {
- pCipherReqParams
- ->spc_auth_res_addr =
- digestBufferPhysAddr;
- pCipherReqParams
- ->spc_auth_res_sz =
- pSessionDesc
- ->hashResultSize;
+ if (isCyGen4x(
+ pService)) {
+ pCipher20ReqParams
+ ->spc_auth_res_addr =
+ digestBufferPhysAddr;
+ pCipher20ReqParams
+ ->spc_auth_res_sz =
+ (Cpa8U)pSessionDesc
+ ->hashResultSize;
+ } else {
+ pCipherReqParams
+ ->spc_auth_res_addr =
+ digestBufferPhysAddr;
+ pCipherReqParams
+ ->spc_auth_res_sz =
+ (Cpa8U)pSessionDesc
+ ->hashResultSize;
+ }
} else {
LAC_LOG_ERROR(
"Unable to get the physical address "
@@ -1619,6 +2119,35 @@
status =
CPA_STATUS_FAIL;
}
+ } else {
+ /* Check if the dest buffer can
+ * handle the digest, only for
+ * last packet */
+ if (((ICP_QAT_FW_LA_PARTIAL_NONE ==
+ qatPacketType) ||
+ (ICP_QAT_FW_LA_PARTIAL_END ==
+ qatPacketType))) {
+ if (dstPktSize <
+ (pOpData
+ ->cryptoStartSrcOffsetInBytes +
+ pOpData
+ ->messageLenToCipherInBytes +
+ pSessionDesc
+ ->hashResultSize))
+ status =
+ CPA_STATUS_INVALID_PARAM;
+ }
+ if (isCyGen4x(pService)) {
+ pCipher20ReqParams
+ ->spc_auth_res_sz =
+ (Cpa8U)pSessionDesc
+ ->hashResultSize;
+ } else {
+ pCipherReqParams
+ ->spc_auth_res_sz =
+ (Cpa8U)pSessionDesc
+ ->hashResultSize;
+ }
}
}
}
@@ -1683,8 +2212,8 @@
pHashReqParams->u2.aad_sz);
/* Overwrite hash state buffer info
- * structure pointer
- * with the one created for CCM/GCM */
+ * structure pointer with the one
+ * created for CCM/GCM */
pHashStateBufferInfo =
&hashStateBufferInfo;
@@ -1692,10 +2221,8 @@
* case */
if (0 ==
hashStateBufferInfo.pDataPhys &&
- CPA_CY_SYM_HASH_AES_GCM !=
- pSessionDesc->hashAlgorithm &&
- CPA_CY_SYM_HASH_AES_GMAC !=
- pSessionDesc->hashAlgorithm) {
+ CPA_CY_SYM_HASH_AES_GCM != hash &&
+ CPA_CY_SYM_HASH_AES_GMAC != hash) {
LAC_LOG_ERROR(
"Unable to get the physical address"
"of the AAD\n");
@@ -1703,31 +2230,26 @@
}
/* for CCM/GCM the hash and cipher data
- * regions
- * are equal */
+ * regions are equal */
authOffsetInBytes =
pOpData
->cryptoStartSrcOffsetInBytes;
/* For authenticated encryption,
- * authentication length is
- * determined by
- * messageLenToCipherInBytes for AES-GCM
- * and
- * AES-CCM, and by
+ * authentication length is determined
+ * by messageLenToCipherInBytes for
+ * AES-GCM and AES-CCM, and by
* messageLenToHashInBytes for AES-GMAC.
* You don't see the latter here, as
- * that is the initial
- * value of authLenInBytes. */
- if (pSessionDesc->hashAlgorithm !=
- CPA_CY_SYM_HASH_AES_GMAC)
+ * that is the initial value of
+ * authLenInBytes. */
+ if (hash != CPA_CY_SYM_HASH_AES_GMAC)
authLenInBytes =
pOpData
->messageLenToCipherInBytes;
} else if (CPA_CY_SYM_HASH_SNOW3G_UIA2 ==
- pSessionDesc->hashAlgorithm ||
- CPA_CY_SYM_HASH_ZUC_EIA3 ==
- pSessionDesc->hashAlgorithm) {
+ hash ||
+ CPA_CY_SYM_HASH_ZUC_EIA3 == hash) {
hashStateBufferInfo.pData =
pOpData->pAdditionalAuthData;
hashStateBufferInfo.pDataPhys =
@@ -1739,7 +2261,7 @@
hashStateBufferInfo
.prefixAadSzQuadWords =
LAC_BYTES_TO_QUADWORDS(
- pSessionDesc->aadLenInBytes);
+ aadLenInBytes);
pHashStateBufferInfo =
&hashStateBufferInfo;
@@ -1752,23 +2274,22 @@
status = CPA_STATUS_FAIL;
}
}
- if (CPA_CY_SYM_HASH_AES_CCM ==
- pSessionDesc->hashAlgorithm) {
+ if (CPA_CY_SYM_HASH_AES_CCM == hash) {
if (CPA_CY_SYM_CIPHER_DIRECTION_DECRYPT ==
pSessionDesc->cipherDirection) {
/* On a decrypt path pSrcBuffer
- * is used as this is
- * where encrypted digest is
- * located. Firmware
- * uses encrypted digest for
+ * is used as this is where
+ * encrypted digest is located.
+ * Firmware uses encrypted
+ * digest for
* compare/verification*/
pBufferList =
(CpaBufferList *)pSrcBuffer;
} else {
/* On an encrypt path pDstBuffer
- * is used as this is
- * where encrypted digest will
- * be written */
+ * is used as this is where
+ * encrypted digest will be
+ * written */
pBufferList =
(CpaBufferList *)pDstBuffer;
}
@@ -1787,21 +2308,29 @@
pDigestResult = pOpData->pDigestResult;
}
+ if (CPA_TRUE ==
+ pSessionDesc->useStatefulSha3ContentDesc) {
+ LacAlgChain_StatefulSha3_SkipStateLoadFlags(
+ pMsg,
+ qatPacketType,
+ pSessionDesc->qatHashMode);
+ }
+
if (CPA_CY_SYM_OP_ALGORITHM_CHAINING ==
pSessionDesc->symOperation) {
/* In alg chaining mode, packets are not
- * seen as partials
- * for hash operations. Override to
- * NONE.
+ * seen as partials for hash operations.
+ * Override to NONE.
*/
qatPacketType =
ICP_QAT_FW_LA_PARTIAL_NONE;
}
- if (CPA_TRUE ==
- pSessionDesc->digestIsAppended) {
+ digestIsAppended =
+ pSessionDesc->digestIsAppended;
+ if (CPA_TRUE == digestIsAppended) {
/*Check if the destination buffer can
- * handle the digest
- * if digestIsAppend is true*/
+ * handle the digest if digestIsAppend
+ * is true*/
if (srcPktSize <
(authOffsetInBytes +
authLenInBytes +
@@ -1824,10 +2353,10 @@
qatPacketType,
pSessionDesc->hashResultSize,
pSessionDesc->digestVerify,
- pSessionDesc->digestIsAppended ?
+ digestIsAppended ?
NULL :
pDigestResult,
- pSessionDesc->hashAlgorithm,
+ hash,
NULL);
}
}
Index: sys/dev/qat/qat_api/common/crypto/sym/lac_sym_api.c
===================================================================
--- sys/dev/qat/qat_api/common/crypto/sym/lac_sym_api.c
+++ sys/dev/qat/qat_api/common/crypto/sym/lac_sym_api.c
@@ -55,8 +55,9 @@
#include "lac_sal_types_crypto.h"
#include "sal_service_state.h"
-#define IS_EXT_ALG_CHAIN_UNSUPPORTED( \
- cipherAlgorithm, hashAlgorithm, extAlgchainSupported) \
+#define IS_EXT_ALG_CHAIN_UNSUPPORTED(cipherAlgorithm, \
+ hashAlgorithm, \
+ extAlgchainSupported) \
((((CPA_CY_SYM_CIPHER_ZUC_EEA3 == cipherAlgorithm || \
CPA_CY_SYM_CIPHER_SNOW3G_UEA2 == cipherAlgorithm) && \
CPA_CY_SYM_HASH_AES_CMAC == hashAlgorithm) || \
@@ -77,6 +78,10 @@
const lac_session_desc_t *const pSessionDesc,
const CpaCySymOpData *const pOpData);
+void LacDp_WriteRingMsgFull(CpaCySymDpOpData *pRequest,
+ icp_qat_fw_la_bulk_req_t *pCurrentQatMsg);
+void LacDp_WriteRingMsgOpt(CpaCySymDpOpData *pRequest,
+ icp_qat_fw_la_bulk_req_t *pCurrentQatMsg);
void getCtxSize(const CpaCySymSessionSetupData *pSessionSetupData,
Cpa32U *pSessionCtxSizeInBytes);
@@ -151,45 +156,40 @@
/* Protect against value of cipher outside the bitmap
* and check if cipher algorithm is correct
*/
- if ((pCipherSetupData->cipherAlgorithm >=
- CPA_CY_SYM_CIPHER_CAP_BITMAP_SIZE) ||
- (!CPA_BITMAP_BIT_TEST(capInfo.ciphers,
- pCipherSetupData->cipherAlgorithm))) {
+ if (pCipherSetupData->cipherAlgorithm >=
+ CPA_CY_SYM_CIPHER_CAP_BITMAP_SIZE) {
LAC_INVALID_PARAM_LOG("cipherAlgorithm");
return CPA_STATUS_INVALID_PARAM;
}
+ if (!CPA_BITMAP_BIT_TEST(capInfo.ciphers,
+ pCipherSetupData->cipherAlgorithm)) {
+ LAC_UNSUPPORTED_PARAM_LOG(
+ "UnSupported cipherAlgorithm");
+ return CPA_STATUS_UNSUPPORTED;
+ }
}
/* Ensure hash algorithm is correct and supported */
if ((CPA_CY_SYM_OP_ALGORITHM_CHAINING ==
pSessionSetupData->symOperation) ||
(CPA_CY_SYM_OP_HASH == pSessionSetupData->symOperation)) {
- /* Ensure SHAKE algorithms are not supported */
- if ((CPA_CY_SYM_HASH_SHAKE_128 ==
- pHashSetupData->hashAlgorithm) ||
- (CPA_CY_SYM_HASH_SHAKE_256 ==
- pHashSetupData->hashAlgorithm)) {
- LAC_INVALID_PARAM_LOG(
- "Hash algorithms SHAKE-128 and SHAKE-256 "
- "are not supported.");
- return CPA_STATUS_UNSUPPORTED;
- }
-
/* Protect against value of hash outside the bitmap
* and check if hash algorithm is correct
*/
- if ((pHashSetupData->hashAlgorithm >=
- CPA_CY_SYM_HASH_CAP_BITMAP_SIZE) ||
- (!CPA_BITMAP_BIT_TEST(capInfo.hashes,
- pHashSetupData->hashAlgorithm))) {
+ if (pHashSetupData->hashAlgorithm >=
+ CPA_CY_SYM_HASH_CAP_BITMAP_SIZE) {
LAC_INVALID_PARAM_LOG("hashAlgorithm");
return CPA_STATUS_INVALID_PARAM;
}
+ if (!CPA_BITMAP_BIT_TEST(capInfo.hashes,
+ pHashSetupData->hashAlgorithm)) {
+ LAC_UNSUPPORTED_PARAM_LOG("UnSupported hashAlgorithm");
+ return CPA_STATUS_UNSUPPORTED;
+ }
}
/* ensure CCM, GCM, Kasumi, Snow3G and ZUC cipher and hash algorithms
- * are
- * selected together for Algorithm Chaining */
+ * are selected together for Algorithm Chaining */
if (CPA_CY_SYM_OP_ALGORITHM_CHAINING ==
pSessionSetupData->symOperation) {
/* ensure both hash and cipher algorithms are POLY and CHACHA */
@@ -435,7 +435,11 @@
}
}
/* Check that src Buffer and dst Buffer Lengths are equal */
- if (srcBufferLen != dstBufferLen) {
+ /* CCM output needs to be longer than input buffer for appending
+ * tag*/
+ if (srcBufferLen != dstBufferLen &&
+ pSessionDesc->cipherAlgorithm !=
+ CPA_CY_SYM_CIPHER_AES_CCM) {
LAC_INVALID_PARAM_LOG(
"Source and Dest buffer lengths need to be equal ");
return CPA_STATUS_INVALID_PARAM;
@@ -451,8 +455,7 @@
return CPA_STATUS_INVALID_PARAM;
} else {
/* This function checks to see if the partial packet
- * sequence
- * is correct */
+ * sequence is correct */
if (CPA_STATUS_SUCCESS !=
LacSym_PartialPacketStateCheck(
pOpData->packetType,
@@ -551,7 +554,7 @@
const CpaCySymCipherSetupData *pCipherSetupData = NULL;
const CpaCySymHashSetupData *pHashSetupData = NULL;
-/* Instance param checking done by calling function */
+ /* Instance param checking done by calling function */
LAC_CHECK_NULL_PARAM(pSessionSetupData);
LAC_CHECK_NULL_PARAM(pSessionCtx);
@@ -580,7 +583,7 @@
if (0 == physAddress) {
LAC_LOG_ERROR(
- "Unable to get the physical address of the session");
+ "Unable to get the physical address of the session\n");
return CPA_STATUS_FAIL;
}
@@ -631,8 +634,8 @@
/* For asynchronous - use the user supplied callback
* for synchronous - use the internal synchronous callback */
pSessionDesc->pSymCb = ((void *)NULL != (void *)pSymCb) ?
- pSymCb :
- LacSync_GenBufListVerifyCb;
+ pSymCb :
+ LacSync_GenBufListVerifyCb;
}
pSessionDesc->isDPSession = isDPSession;
@@ -649,10 +652,8 @@
if (CPA_STATUS_SUCCESS == status) {
/* Session set up via API call (not internal one) */
/* Services such as DRBG call the crypto api as part of their
- * service
- * hence the need to for the flag, it is needed to distinguish
- * between
- * an internal and external session.
+ * service hence the need to for the flag, it is needed to
+ * distinguish between an internal and external session.
*/
pSessionDesc->internalSession = CPA_FALSE;
@@ -697,14 +698,11 @@
/*
* Based on one instance, we can initialize multiple sessions.
* For example, we can initialize the session "X" and session
- * "Y" with
- * the same instance "A". If there is no operation pending for
- * session
- * "X", we can remove the session "X".
+ * "Y" with the same instance "A". If there is no operation
+ * pending for session "X", we can remove the session "X".
*
* Now we only check the @pSessionDesc->pendingDpCbCount, if it
- * becomes
- * zero, we can remove the session.
+ * becomes zero, we can remove the session.
*
* Why?
* (1) We increase it in the cpaCySymDpEnqueueOp/
@@ -713,12 +711,10 @@
*
* If the @pSessionDesc->pendingDpCbCount becomes zero, it means
* there is no operation pending for the session "X" anymore, so
- * we can
- * remove this session. Maybe there is still some requests left
- * in the
- * instance's ring (icp_adf_queueDataToSend() returns true), but
- * the
- * request does not belong to "X", it belongs to session "Y".
+ * we can remove this session. Maybe there is still some
+ * requests left in the instance's ring
+ * (icp_adf_queueDataToSend() returns true), but the request
+ * does not belong to "X", it belongs to session "Y".
*/
numPendingRequests =
qatUtilsAtomicGet(&(pSessionDesc->u.pendingDpCbCount));
@@ -734,8 +730,7 @@
status = CPA_STATUS_RETRY;
if (CPA_TRUE == pSessionDesc->isDPSession) {
/* Need to update tail if messages queue on tx hi ring
- for
- data plane api */
+ for data plane api */
icp_comms_trans_handle trans_handle =
((sal_crypto_service_t *)instanceHandle)
->trans_handle_sym_tx;
@@ -752,10 +747,7 @@
}
}
if (CPA_STATUS_SUCCESS == status) {
- if (CPA_STATUS_SUCCESS !=
- LAC_SPINLOCK_DESTROY(&pSessionDesc->requestQueueLock)) {
- LAC_LOG_ERROR("Failed to destroy request queue lock");
- }
+ LAC_SPINLOCK_DESTROY(&pSessionDesc->requestQueueLock);
if (CPA_FALSE == pSessionDesc->isDPSession) {
LAC_SYM_STAT_INC(numSessionsRemoved, instanceHandle);
}
Index: sys/dev/qat/qat_api/common/crypto/sym/lac_sym_cb.c
===================================================================
--- sys/dev/qat/qat_api/common/crypto/sym/lac_sym_cb.c
+++ sys/dev/qat/qat_api/common/crypto/sym/lac_sym_cb.c
@@ -69,7 +69,7 @@
const CpaCySymOpData *pOpData,
CpaBoolean isCCM)
{
- Cpa8U authTagLen = 0;
+ Cpa32U authTagLen = 0;
/* Retrieve authTagLen */
authTagLen = pSessionDesc->hashResultSize;
@@ -138,10 +138,10 @@
/* For a digest verify operation - for full packet and final partial
* only, perform a comparison with the digest generated and with the one
- * supplied in the packet. */
+ * supplied in the packet. In case of AES_GCM in SPC mode, destination
+ * buffer needs to be cleared if digest verify operation fails */
- if (((pSessionDesc->isSinglePass &&
- (CPA_CY_SYM_CIPHER_AES_GCM == pSessionDesc->cipherAlgorithm)) ||
+ if (((SPC == pSessionDesc->singlePassState) ||
(CPA_CY_SYM_OP_CIPHER != operationType)) &&
(CPA_TRUE == pSessionDesc->digestVerify) &&
((CPA_CY_SYM_PACKET_TYPE_FULL == pOpData->packetType) ||
@@ -151,14 +151,10 @@
instanceHandle);
/* The comparison has failed at this point (status is
- * fail),
- * need to clean any sensitive calculated data up to
- * this point.
- * The data calculated is no longer useful to the end
- * result and
- * does not need to be returned to the user so setting
- * buffers to
- * zero.
+ * fail), need to clean any sensitive calculated data up
+ * to this point. The data calculated is no longer
+ * useful to the end result and does not need to be
+ * returned to the user so setting buffers to zero.
*/
if (pSessionDesc->cipherAlgorithm ==
CPA_CY_SYM_CIPHER_AES_CCM) {
@@ -200,8 +196,7 @@
/* Update the user's IV buffer
* Very important to do this BEFORE dequeuing
* subsequent partial requests, as the state
- * buffer
- * may get overwritten
+ * buffer may get overwritten
*/
memcpy(pCookie->pOpData->pIv,
pSessionDesc->cipherPartialOpState,
@@ -218,8 +213,9 @@
} else if (CPA_CY_SYM_PACKET_TYPE_LAST_PARTIAL == pOpData->packetType) {
if ((CPA_CY_SYM_OP_CIPHER == operationType) ||
(CPA_CY_SYM_OP_ALGORITHM_CHAINING == operationType)) {
- if (CPA_TRUE == LAC_CIPHER_IS_XTS_MODE(
- pSessionDesc->cipherAlgorithm)) {
+ if (CPA_TRUE ==
+ LAC_CIPHER_IS_XTS_MODE(
+ pSessionDesc->cipherAlgorithm)) {
/*
* For XTS mode, we replace the updated key with
* the original key - for subsequent partial
@@ -294,51 +290,46 @@
static void
LacSymCb_ProcessDpCallback(CpaCySymDpOpData *pResponse,
CpaBoolean qatRespStatusOkFlag,
+ CpaStatus status,
lac_session_desc_t *pSessionDesc)
{
- CpaStatus status = CPA_STATUS_SUCCESS;
+ CpaCySymDpCbFunc pSymDpCb = NULL;
/* For CCM and GCM, if qatRespStatusOkFlag is false, the data has to be
* cleaned as stated in RFC 3610; in DP mode, it is the user
- * responsability
- * to do so */
-
- if (CPA_FALSE == pSessionDesc->isSinglePass) {
- if ((CPA_CY_SYM_OP_CIPHER == pSessionDesc->symOperation) ||
- (CPA_FALSE == pSessionDesc->digestVerify)) {
- /* If not doing digest compare and qatRespStatusOkFlag
- !=
- CPA_TRUE
- then there is something very wrong */
- if (CPA_FALSE == qatRespStatusOkFlag) {
- LAC_LOG_ERROR(
- "Response status value not as expected");
- status = CPA_STATUS_FAIL;
- }
+ * responsability to do so */
+
+ if (((CPA_CY_SYM_OP_CIPHER == pSessionDesc->symOperation) &&
+ SPC != pSessionDesc->singlePassState) ||
+ (CPA_FALSE == pSessionDesc->digestVerify)) {
+ /* If not doing digest compare and qatRespStatusOkFlag !=
+ CPA_TRUE then there is something very wrong */
+ if ((CPA_FALSE == qatRespStatusOkFlag) &&
+ (status != CPA_STATUS_UNSUPPORTED)) {
+ LAC_LOG_ERROR("Response status value not as expected");
+ status = CPA_STATUS_FAIL;
}
}
- ((sal_crypto_service_t *)pResponse->instanceHandle)
- ->pSymDpCb(pResponse, status, qatRespStatusOkFlag);
+ pSymDpCb =
+ ((sal_crypto_service_t *)pResponse->instanceHandle)->pSymDpCb;
+
+ pSymDpCb(pResponse, status, qatRespStatusOkFlag);
+
/*
* Decrement the number of pending CB.
*
* If the @pendingDpCbCount becomes zero, we may remove the session,
- * please
- * read more information in the cpaCySymRemoveSession().
+ * please read more information in the cpaCySymRemoveSession().
*
* But there is a field in the @pResponse to store the session,
* the "sessionCtx". In another word, in the above @->pSymDpCb()
- * callback,
- * it may use the session again. If we decrease the @pendingDpCbCount
- * before
- * the @->pSymDpCb(), there is a _risk_ the @->pSymDpCb() may reference
- * to
- * a deleted session.
+ * callback, it may use the session again. If we decrease the
+ * @pendingDpCbCount before the @->pSymDpCb(), there is a _risk_ the
+ * @->pSymDpCb() may reference to a deleted session.
*
* So in order to avoid the risk, we decrease the @pendingDpCbCount
- * after
- * the @->pSymDpCb() callback.
+ * after the @->pSymDpCb() callback.
*/
qatUtilsAtomicDec(&pSessionDesc->u.pendingDpCbCount);
}
@@ -367,6 +358,7 @@
void *pOpaqueData,
icp_qat_fw_comn_flags cmnRespFlags)
{
+ CpaStatus status = CPA_STATUS_SUCCESS;
CpaCySymDpOpData *pDpOpData = (CpaCySymDpOpData *)pOpaqueData;
lac_session_desc_t *pSessionDesc =
LAC_SYM_SESSION_DESC_FROM_CTX_GET(pDpOpData->sessionCtx);
@@ -376,8 +368,13 @@
if (CPA_TRUE == pSessionDesc->isDPSession) {
/* DP session */
+ if (ICP_QAT_FW_COMN_RESP_UNSUPPORTED_REQUEST_STAT_GET(
+ cmnRespFlags)) {
+ status = CPA_STATUS_UNSUPPORTED;
+ }
LacSymCb_ProcessDpCallback(pDpOpData,
qatRespStatusOkFlag,
+ status,
pSessionDesc);
} else {
/* Trad session */
@@ -414,11 +411,7 @@
* be accessed by multiple contexts simultaneously for enqueue and
* dequeue operations
*/
- if (CPA_STATUS_SUCCESS !=
- LAC_SPINLOCK(&pSessionDesc->requestQueueLock)) {
- LAC_LOG_ERROR("Failed to lock request queue");
- return CPA_STATUS_RESOURCE;
- }
+ LAC_SPINLOCK(&pSessionDesc->requestQueueLock);
/* Clear the blocking flag in the session descriptor */
pSessionDesc->nonBlockingOpsInProgress = CPA_TRUE;
@@ -427,10 +420,9 @@
(CPA_TRUE == pSessionDesc->nonBlockingOpsInProgress)) {
/* If we send a partial packet request, set the
- * blockingOpsInProgress
- * flag for the session to indicate that subsequent requests
- * must be
- * queued up until this request completes
+ * blockingOpsInProgress flag for the session to indicate that
+ * subsequent requests must be queued up until this request
+ * completes
*/
if (CPA_CY_SYM_PACKET_TYPE_FULL !=
pSessionDesc->pRequestQueueHead->pOpData->packetType) {
@@ -438,14 +430,10 @@
}
/* At this point, we're clear to send the request. For cipher
- * requests,
- * we need to check if the session IV needs to be updated. This
- * can
- * only be done when no other partials are in flight for this
- * session,
- * to ensure the cipherPartialOpState buffer in the session
- * descriptor
- * is not currently in use
+ * requests, we need to check if the session IV needs to be
+ * updated. This can only be done when no other partials are in
+ * flight for this session, to ensure the cipherPartialOpState
+ * buffer in the session descriptor is not currently in use
*/
if (CPA_TRUE ==
pSessionDesc->pRequestQueueHead->updateSessionIvOnSend) {
@@ -464,13 +452,11 @@
/*
* Now we'll attempt to send the message directly to QAT. We'll
- * keep
- * looing until it succeeds (or at least a very high number of
- * retries),
- * as the failure only happens when the ring is full, and this
- * is only
- * a temporary situation. After a few retries, space will become
- * availble, allowing the putMsg to succeed.
+ * keep looing until it succeeds (or at least a very high number
+ * of retries), as the failure only happens when the ring is
+ * full, and this is only a temporary situation. After a few
+ * retries, space will become availble, allowing the putMsg to
+ * succeed.
*/
retries = 0;
do {
@@ -483,8 +469,7 @@
retries++;
/*
* Yield to allow other threads that may be on this
- * session to poll
- * and make some space on the ring
+ * session to poll and make some space on the ring
*/
if (CPA_STATUS_SUCCESS != status) {
qatUtilsYield();
@@ -509,10 +494,7 @@
}
cleanup:
- if (CPA_STATUS_SUCCESS !=
- LAC_SPINUNLOCK(&pSessionDesc->requestQueueLock)) {
- LAC_LOG_ERROR("Failed to unlock request queue");
- }
+ LAC_SPINUNLOCK(&pSessionDesc->requestQueueLock);
return status;
}
Index: sys/dev/qat/qat_api/common/crypto/sym/lac_sym_cipher.c
===================================================================
--- sys/dev/qat/qat_api/common/crypto/sym/lac_sym_cipher.c
+++ sys/dev/qat/qat_api/common/crypto/sym/lac_sym_cipher.c
@@ -49,6 +49,7 @@
#include "lac_sym_qat_cipher.h"
#include "lac_log.h"
#include "lac_buffer_desc.h"
+#include "sal_hw_gen.h"
/*
*******************************************************************************
@@ -66,14 +67,23 @@
lac_session_desc_t *pSessionDesc =
LAC_SYM_SESSION_DESC_FROM_CTX_GET(pOpData->sessionCtx);
CpaCySymCipherAlgorithm algorithm = pSessionDesc->cipherAlgorithm;
+ unsigned ivLenInBytes = 0;
- /* Perform IV check. */
- if (LAC_CIPHER_IS_CTR_MODE(algorithm) ||
- LAC_CIPHER_IS_CBC_MODE(algorithm) ||
- LAC_CIPHER_IS_AES_F8(algorithm) ||
- LAC_CIPHER_IS_XTS_MODE(algorithm)) {
- unsigned ivLenInBytes =
- LacSymQat_CipherIvSizeBytesGet(algorithm);
+ switch (algorithm) {
+ /* Perform IV check for CTR, CBC, XTS, F8 MODE. */
+ case CPA_CY_SYM_CIPHER_AES_CTR:
+ case CPA_CY_SYM_CIPHER_3DES_CTR:
+ case CPA_CY_SYM_CIPHER_SM4_CTR:
+ case CPA_CY_SYM_CIPHER_AES_CCM:
+ case CPA_CY_SYM_CIPHER_AES_GCM:
+ case CPA_CY_SYM_CIPHER_CHACHA:
+ case CPA_CY_SYM_CIPHER_AES_CBC:
+ case CPA_CY_SYM_CIPHER_DES_CBC:
+ case CPA_CY_SYM_CIPHER_3DES_CBC:
+ case CPA_CY_SYM_CIPHER_SM4_CBC:
+ case CPA_CY_SYM_CIPHER_AES_F8:
+ case CPA_CY_SYM_CIPHER_AES_XTS: {
+ ivLenInBytes = LacSymQat_CipherIvSizeBytesGet(algorithm);
LAC_CHECK_NULL_PARAM(pOpData->pIv);
if (pOpData->ivLenInBytes != ivLenInBytes) {
if (!(/* GCM with 12 byte IV is OK */
@@ -96,26 +106,21 @@
/* Set the value of the ppIvBuffer to that supplied
* by the user.
* NOTE: There is no guarantee that this address is
- * aligned on
- * an 8 or 64 Byte address. */
+ * aligned on an 8 or 64 Byte address. */
*ppIvBuffer = pOpData->pIv;
} else {
/* For partial packets, we use a per-session buffer to
- * maintain
- * the IV. This allows us to easily pass the updated IV
- * forward
- * to the next partial in the sequence. This makes
- * internal
- * buffering of partials easier to implement.
+ * maintain the IV. This allows us to easily pass the
+ * updated IV forward to the next partial in the
+ * sequence. This makes internal buffering of partials
+ * easier to implement.
*/
*ppIvBuffer = pSessionDesc->cipherPartialOpState;
/* Ensure that the user's IV buffer gets updated between
- * partial
- * requests so that they may also see the residue from
- * the
- * previous partial. Not needed for final partials
- * though.
+ * partial requests so that they may also see the
+ * residue from the previous partial. Not needed for
+ * final partials though.
*/
if ((ICP_QAT_FW_LA_PARTIAL_START == qatPacketType) ||
(ICP_QAT_FW_LA_PARTIAL_MID == qatPacketType)) {
@@ -124,97 +129,87 @@
if (ICP_QAT_FW_LA_PARTIAL_START ==
qatPacketType) {
/* if the previous partial state was
- * full, then this is
- * the first partial in the sequence so
- * we need to copy
- * in the user's IV. But, we have to be
- * very careful
- * here not to overwrite the
- * cipherPartialOpState just
- * yet in case there's a previous
- * partial sequence in
- * flight, so we defer the copy for now.
- * This will be
- * completed in the
- * LacSymQueue_RequestSend() function.
+ * full, then this is the first partial
+ * in the sequence so we need to copy in
+ * the user's IV. But, we have to be
+ * very careful here not to overwrite
+ * the cipherPartialOpState just yet in
+ * case there's a previous partial
+ * sequence in flight, so we defer the
+ * copy for now. This will be completed
+ * in the LacSymQueue_RequestSend()
+ * function.
*/
pCbCookie->updateSessionIvOnSend =
CPA_TRUE;
}
/* For subsequent partials in a sequence, we'll
- * re-use the
- * IV that was written back by the QAT, using
- * internal
- * request queueing if necessary to ensure that
- * the next
- * partial request isn't issued to the QAT until
- * the
+ * re-use the IV that was written back by the
+ * QAT, using internal request queueing if
+ * necessary to ensure that the next partial
+ * request isn't issued to the QAT until the
* previous one completes
*/
}
}
- } else if (LAC_CIPHER_IS_KASUMI(algorithm)) {
+ } break;
+ case CPA_CY_SYM_CIPHER_KASUMI_F8: {
LAC_CHECK_NULL_PARAM(pOpData->pIv);
- if (LAC_CIPHER_IS_KASUMI(algorithm) &&
- (pOpData->ivLenInBytes != LAC_CIPHER_KASUMI_F8_IV_LENGTH)) {
+ if (pOpData->ivLenInBytes != LAC_CIPHER_KASUMI_F8_IV_LENGTH) {
LAC_INVALID_PARAM_LOG("invalid cipher IV size");
return CPA_STATUS_INVALID_PARAM;
}
-
*ppIvBuffer = pOpData->pIv;
- } else if (LAC_CIPHER_IS_SNOW3G_UEA2(algorithm)) {
+ } break;
+ case CPA_CY_SYM_CIPHER_SNOW3G_UEA2: {
LAC_CHECK_NULL_PARAM(pOpData->pIv);
- if (LAC_CIPHER_IS_SNOW3G_UEA2(algorithm) &&
- (pOpData->ivLenInBytes != ICP_QAT_HW_SNOW_3G_UEA2_IV_SZ)) {
+ if (pOpData->ivLenInBytes != ICP_QAT_HW_SNOW_3G_UEA2_IV_SZ) {
LAC_INVALID_PARAM_LOG("invalid cipher IV size");
return CPA_STATUS_INVALID_PARAM;
}
*ppIvBuffer = pOpData->pIv;
- } else if (LAC_CIPHER_IS_ARC4(algorithm)) {
+ } break;
+ case CPA_CY_SYM_CIPHER_ARC4: {
if (ICP_QAT_FW_LA_PARTIAL_NONE == qatPacketType) {
/* For full packets, the initial ARC4 state is stored in
- * the
- * session descriptor. Use it directly.
+ * the session descriptor. Use it directly.
*/
*ppIvBuffer = pSessionDesc->cipherARC4InitialState;
} else {
/* For partial packets, we maintain the running ARC4
- * state in
- * dedicated buffer in the session descriptor
+ * state in dedicated buffer in the session descriptor
*/
*ppIvBuffer = pSessionDesc->cipherPartialOpState;
if (ICP_QAT_FW_LA_PARTIAL_START == qatPacketType) {
/* if the previous partial state was full, then
- * this is the
- * first partial in the sequence so we need to
- * (re-)initialise
- * the contents of the state buffer using the
- * initial state
- * that is stored in the session descriptor.
- * But, we have to be
- * very careful here not to overwrite the
- * cipherPartialOpState
- * just yet in case there's a previous partial
- * sequence in
+ * this is the first partial in the sequence so
+ * we need to (re-)initialise the contents of
+ * the state buffer using the initial state that
+ * is stored in the session descriptor. But, we
+ * have to be very careful here not to overwrite
+ * the cipherPartialOpState just yet in case
+ * there's a previous partial sequence in
* flight, so we defer the copy for now. This
- * will be completed
- * in the LacSymQueue_RequestSend() function
- * when clear to send.
+ * will be completed in the
+ * LacSymQueue_RequestSend() function when clear
+ * to send.
*/
pCbCookie->updateSessionIvOnSend = CPA_TRUE;
}
}
- } else if (LAC_CIPHER_IS_ZUC_EEA3(algorithm)) {
+ } break;
+ case CPA_CY_SYM_CIPHER_ZUC_EEA3: {
LAC_CHECK_NULL_PARAM(pOpData->pIv);
if (pOpData->ivLenInBytes != ICP_QAT_HW_ZUC_3G_EEA3_IV_SZ) {
LAC_INVALID_PARAM_LOG("invalid cipher IV size");
return CPA_STATUS_INVALID_PARAM;
}
*ppIvBuffer = pOpData->pIv;
- } else {
+ } break;
+ default:
*ppIvBuffer = NULL;
}
@@ -223,40 +218,50 @@
CpaStatus
-LacCipher_SessionSetupDataCheck(const CpaCySymCipherSetupData *pCipherSetupData)
+LacCipher_SessionSetupDataCheck(const CpaCySymCipherSetupData *pCipherSetupData,
+ Cpa32U capabilitiesMask)
{
/* No key required for NULL algorithm */
if (!LAC_CIPHER_IS_NULL(pCipherSetupData->cipherAlgorithm)) {
LAC_CHECK_NULL_PARAM(pCipherSetupData->pCipherKey);
/* Check that algorithm and keys passed in are correct size */
- if (LAC_CIPHER_IS_ARC4(pCipherSetupData->cipherAlgorithm)) {
+ switch (pCipherSetupData->cipherAlgorithm) {
+ case CPA_CY_SYM_CIPHER_ARC4:
if (pCipherSetupData->cipherKeyLenInBytes >
ICP_QAT_HW_ARC4_KEY_SZ) {
LAC_INVALID_PARAM_LOG(
"Invalid ARC4 cipher key length");
return CPA_STATUS_INVALID_PARAM;
}
- } else if (LAC_CIPHER_IS_CCM(
- pCipherSetupData->cipherAlgorithm)) {
- if (pCipherSetupData->cipherKeyLenInBytes !=
- ICP_QAT_HW_AES_128_KEY_SZ) {
+ break;
+ case CPA_CY_SYM_CIPHER_AES_CCM:
+ if (!LAC_CIPHER_AES_V2(capabilitiesMask) &&
+ pCipherSetupData->cipherKeyLenInBytes !=
+ ICP_QAT_HW_AES_128_KEY_SZ) {
LAC_INVALID_PARAM_LOG(
"Invalid AES CCM cipher key length");
return CPA_STATUS_INVALID_PARAM;
}
- } else if (LAC_CIPHER_IS_XTS_MODE(
- pCipherSetupData->cipherAlgorithm)) {
+ break;
+ case CPA_CY_SYM_CIPHER_AES_XTS:
if ((pCipherSetupData->cipherKeyLenInBytes !=
ICP_QAT_HW_AES_128_XTS_KEY_SZ) &&
(pCipherSetupData->cipherKeyLenInBytes !=
- ICP_QAT_HW_AES_256_XTS_KEY_SZ)) {
+ ICP_QAT_HW_AES_256_XTS_KEY_SZ) &&
+ (pCipherSetupData->cipherKeyLenInBytes !=
+ ICP_QAT_HW_UCS_AES_128_XTS_KEY_SZ) &&
+ (pCipherSetupData->cipherKeyLenInBytes !=
+ ICP_QAT_HW_UCS_AES_256_XTS_KEY_SZ)) {
LAC_INVALID_PARAM_LOG(
"Invalid AES XTS cipher key length");
return CPA_STATUS_INVALID_PARAM;
}
- } else if (LAC_CIPHER_IS_AES(
- pCipherSetupData->cipherAlgorithm)) {
+ break;
+ case CPA_CY_SYM_CIPHER_AES_ECB:
+ case CPA_CY_SYM_CIPHER_AES_CBC:
+ case CPA_CY_SYM_CIPHER_AES_CTR:
+ case CPA_CY_SYM_CIPHER_AES_GCM:
if ((pCipherSetupData->cipherKeyLenInBytes !=
ICP_QAT_HW_AES_128_KEY_SZ) &&
(pCipherSetupData->cipherKeyLenInBytes !=
@@ -267,8 +272,8 @@
"Invalid AES cipher key length");
return CPA_STATUS_INVALID_PARAM;
}
- } else if (LAC_CIPHER_IS_AES_F8(
- pCipherSetupData->cipherAlgorithm)) {
+ break;
+ case CPA_CY_SYM_CIPHER_AES_F8:
if ((pCipherSetupData->cipherKeyLenInBytes !=
ICP_QAT_HW_AES_128_F8_KEY_SZ) &&
(pCipherSetupData->cipherKeyLenInBytes !=
@@ -279,35 +284,37 @@
"Invalid AES cipher key length");
return CPA_STATUS_INVALID_PARAM;
}
- } else if (LAC_CIPHER_IS_DES(
- pCipherSetupData->cipherAlgorithm)) {
+ break;
+ case CPA_CY_SYM_CIPHER_DES_ECB:
+ case CPA_CY_SYM_CIPHER_DES_CBC:
if (pCipherSetupData->cipherKeyLenInBytes !=
ICP_QAT_HW_DES_KEY_SZ) {
LAC_INVALID_PARAM_LOG(
"Invalid DES cipher key length");
return CPA_STATUS_INVALID_PARAM;
}
- } else if (LAC_CIPHER_IS_TRIPLE_DES(
- pCipherSetupData->cipherAlgorithm)) {
+ break;
+ case CPA_CY_SYM_CIPHER_3DES_ECB:
+ case CPA_CY_SYM_CIPHER_3DES_CBC:
+ case CPA_CY_SYM_CIPHER_3DES_CTR:
if (pCipherSetupData->cipherKeyLenInBytes !=
ICP_QAT_HW_3DES_KEY_SZ) {
LAC_INVALID_PARAM_LOG(
"Invalid Triple-DES cipher key length");
return CPA_STATUS_INVALID_PARAM;
}
- } else if (LAC_CIPHER_IS_KASUMI(
- pCipherSetupData->cipherAlgorithm)) {
+ break;
+ case CPA_CY_SYM_CIPHER_KASUMI_F8:
/* QAT-FW only supports 128 bits Cipher Key size for
- * Kasumi F8
- * Ref: 3GPP TS 55.216 V6.2.0 */
+ * Kasumi F8 Ref: 3GPP TS 55.216 V6.2.0 */
if (pCipherSetupData->cipherKeyLenInBytes !=
ICP_QAT_HW_KASUMI_KEY_SZ) {
LAC_INVALID_PARAM_LOG(
"Invalid Kasumi cipher key length");
return CPA_STATUS_INVALID_PARAM;
}
- } else if (LAC_CIPHER_IS_SNOW3G_UEA2(
- pCipherSetupData->cipherAlgorithm)) {
+ break;
+ case CPA_CY_SYM_CIPHER_SNOW3G_UEA2:
/* QAT-FW only supports 256 bits Cipher Key size for
* Snow_3G */
if (pCipherSetupData->cipherKeyLenInBytes !=
@@ -316,8 +323,8 @@
"Invalid Snow_3G cipher key length");
return CPA_STATUS_INVALID_PARAM;
}
- } else if (LAC_CIPHER_IS_ZUC_EEA3(
- pCipherSetupData->cipherAlgorithm)) {
+ break;
+ case CPA_CY_SYM_CIPHER_ZUC_EEA3:
/* ZUC EEA3 */
if (pCipherSetupData->cipherKeyLenInBytes !=
ICP_QAT_HW_ZUC_3G_EEA3_KEY_SZ) {
@@ -325,23 +332,26 @@
"Invalid ZUC cipher key length");
return CPA_STATUS_INVALID_PARAM;
}
- } else if (LAC_CIPHER_IS_CHACHA(
- pCipherSetupData->cipherAlgorithm)) {
+ break;
+ case CPA_CY_SYM_CIPHER_CHACHA:
if (pCipherSetupData->cipherKeyLenInBytes !=
ICP_QAT_HW_CHACHAPOLY_KEY_SZ) {
LAC_INVALID_PARAM_LOG(
"Invalid CHACHAPOLY cipher key length");
return CPA_STATUS_INVALID_PARAM;
}
- } else if (LAC_CIPHER_IS_SM4(
- pCipherSetupData->cipherAlgorithm)) {
+ break;
+ case CPA_CY_SYM_CIPHER_SM4_ECB:
+ case CPA_CY_SYM_CIPHER_SM4_CBC:
+ case CPA_CY_SYM_CIPHER_SM4_CTR:
if (pCipherSetupData->cipherKeyLenInBytes !=
ICP_QAT_HW_SM4_KEY_SZ) {
LAC_INVALID_PARAM_LOG(
"Invalid SM4 cipher key length");
return CPA_STATUS_INVALID_PARAM;
}
- } else {
+ break;
+ default:
LAC_INVALID_PARAM_LOG("Invalid cipher algorithm");
return CPA_STATUS_INVALID_PARAM;
}
@@ -365,52 +375,86 @@
LAC_INVALID_PARAM_LOG("cipher len + offset greater than "
"srcBuffer packet len");
status = CPA_STATUS_INVALID_PARAM;
- }
-
- if (CPA_STATUS_SUCCESS == status) {
+ } else {
+ /* Perform algorithm-specific checks */
+ switch (algorithm) {
+ case CPA_CY_SYM_CIPHER_ARC4:
+ case CPA_CY_SYM_CIPHER_AES_CTR:
+ case CPA_CY_SYM_CIPHER_3DES_CTR:
+ case CPA_CY_SYM_CIPHER_SM4_CTR:
+ case CPA_CY_SYM_CIPHER_AES_CCM:
+ case CPA_CY_SYM_CIPHER_AES_GCM:
+ case CPA_CY_SYM_CIPHER_CHACHA:
+ case CPA_CY_SYM_CIPHER_KASUMI_F8:
+ case CPA_CY_SYM_CIPHER_AES_F8:
+ case CPA_CY_SYM_CIPHER_SNOW3G_UEA2:
+ case CPA_CY_SYM_CIPHER_ZUC_EEA3:
+ /* No action needed */
+ break;
/*
* XTS Mode allow for ciphers which are not multiples of
* the block size.
*/
- /* Perform algorithm-specific checks */
- if (LAC_CIPHER_IS_XTS_MODE(algorithm) &&
- ((pOpData->packetType == CPA_CY_SYM_PACKET_TYPE_FULL) ||
- (pOpData->packetType ==
- CPA_CY_SYM_PACKET_TYPE_LAST_PARTIAL))) {
- /*
- * If this is the last of a partial request
- */
- if (pOpData->messageLenToCipherInBytes <
- ICP_QAT_HW_AES_BLK_SZ) {
- LAC_INVALID_PARAM_LOG(
- "data size must be greater than block "
- "size for last XTS partial or XTS "
- "full packet");
- status = CPA_STATUS_INVALID_PARAM;
+ case CPA_CY_SYM_CIPHER_AES_XTS:
+ if ((pOpData->packetType ==
+ CPA_CY_SYM_PACKET_TYPE_FULL) ||
+ (pOpData->packetType ==
+ CPA_CY_SYM_PACKET_TYPE_LAST_PARTIAL)) {
+ /*
+ * If this is the last of a partial request
+ */
+ if (pOpData->messageLenToCipherInBytes <
+ ICP_QAT_HW_AES_BLK_SZ) {
+ LAC_INVALID_PARAM_LOG(
+ "data size must be greater than block"
+ " size for last XTS partial or XTS "
+ "full packet");
+ status = CPA_STATUS_INVALID_PARAM;
+ }
}
- } else if (!(LAC_CIPHER_IS_ARC4(algorithm) ||
- LAC_CIPHER_IS_CTR_MODE(algorithm) ||
- LAC_CIPHER_IS_F8_MODE(algorithm) ||
- LAC_CIPHER_IS_SNOW3G_UEA2(algorithm) ||
- LAC_CIPHER_IS_XTS_MODE(algorithm) ||
- LAC_CIPHER_IS_CHACHA(algorithm) ||
- LAC_CIPHER_IS_ZUC_EEA3(algorithm))) {
+ break;
+ default:
/* Mask & check below is based on assumption that block
- * size is
- * a power of 2. If data size is not a multiple of the
- * block size,
- * the "remainder" bits selected by the mask be non-zero
+ * size is a power of 2. If data size is not a multiple
+ * of the block size, the "remainder" bits selected by
+ * the mask be non-zero
*/
if (pOpData->messageLenToCipherInBytes &
(LacSymQat_CipherBlockSizeBytesGet(algorithm) -
1)) {
LAC_INVALID_PARAM_LOG(
- "data size must be block size multiple");
+ "data size must be block size"
+ " multiple");
status = CPA_STATUS_INVALID_PARAM;
}
}
}
-
return status;
}
+Cpa32U
+LacCipher_GetCipherSliceType(sal_crypto_service_t *pService,
+ CpaCySymCipherAlgorithm cipherAlgorithm,
+ CpaCySymHashAlgorithm hashAlgorithm)
+{
+ Cpa32U sliceType = ICP_QAT_FW_LA_USE_LEGACY_SLICE_TYPE;
+ Cpa32U capabilitiesMask =
+ pService->generic_service_info.capabilitiesMask;
+
+ /* UCS Slice is supproted only in Gen4 */
+ if (isCyGen4x(pService)) {
+ if (LAC_CIPHER_IS_XTS_MODE(cipherAlgorithm) ||
+ LAC_CIPHER_IS_CHACHA(cipherAlgorithm) ||
+ LAC_CIPHER_IS_GCM(cipherAlgorithm)) {
+ sliceType = ICP_QAT_FW_LA_USE_UCS_SLICE_TYPE;
+ } else if (LAC_CIPHER_IS_CCM(cipherAlgorithm) &&
+ LAC_CIPHER_AES_V2(capabilitiesMask)) {
+ sliceType = ICP_QAT_FW_LA_USE_LEGACY_SLICE_TYPE;
+ } else if (LAC_CIPHER_IS_AES(cipherAlgorithm) &&
+ LAC_CIPHER_IS_CTR_MODE(cipherAlgorithm)) {
+ sliceType = ICP_QAT_FW_LA_USE_UCS_SLICE_TYPE;
+ }
+ }
+
+ return sliceType;
+}
Index: sys/dev/qat/qat_api/common/crypto/sym/lac_sym_dp.c
===================================================================
--- sys/dev/qat/qat_api/common/crypto/sym/lac_sym_dp.c
+++ sys/dev/qat/qat_api/common/crypto/sym/lac_sym_dp.c
@@ -35,14 +35,16 @@
#include "qat_utils.h"
-#include "lac_mem.h"
+#include "lac_list.h"
#include "lac_log.h"
+#include "lac_mem.h"
+#include "lac_sal_types_crypto.h"
#include "lac_sym.h"
+#include "lac_sym_cipher.h"
+#include "lac_sym_auth_enc.h"
#include "lac_sym_qat_cipher.h"
-#include "lac_list.h"
-#include "lac_sal_types_crypto.h"
#include "sal_service_state.h"
-#include "lac_sym_auth_enc.h"
+#include "sal_hw_gen.h"
typedef void (*write_ringMsgFunc_t)(CpaCySymDpOpData *pRequest,
icp_qat_fw_la_bulk_req_t *pCurrentQatMsg);
@@ -133,7 +135,7 @@
/* digestVerify and digestIsAppended on Hash-Only operation not
* supported */
if (pSessionDesc->digestIsAppended && pSessionDesc->digestVerify &&
- (CPA_CY_SYM_OP_HASH == pSessionDesc->symOperation)) {
+ (pSessionDesc->symOperation == CPA_CY_SYM_OP_HASH)) {
LAC_INVALID_PARAM_LOG(
"digestVerify and digestIsAppended set "
"on Hash-Only operation is not supported");
@@ -143,10 +145,17 @@
/* Cipher specific tests */
if (CPA_CY_SYM_OP_HASH != pSessionDesc->symOperation) {
/* Perform IV check */
- if ((LAC_CIPHER_IS_CTR_MODE(pSessionDesc->cipherAlgorithm) ||
- LAC_CIPHER_IS_CBC_MODE(pSessionDesc->cipherAlgorithm) ||
- LAC_CIPHER_IS_AES_F8(pSessionDesc->cipherAlgorithm)) &&
- (!(LAC_CIPHER_IS_CCM(pSessionDesc->cipherAlgorithm)))) {
+ switch (pSessionDesc->cipherAlgorithm) {
+ case CPA_CY_SYM_CIPHER_AES_CTR:
+ case CPA_CY_SYM_CIPHER_3DES_CTR:
+ case CPA_CY_SYM_CIPHER_SM4_CTR:
+ case CPA_CY_SYM_CIPHER_AES_GCM:
+ case CPA_CY_SYM_CIPHER_CHACHA:
+ case CPA_CY_SYM_CIPHER_AES_CBC:
+ case CPA_CY_SYM_CIPHER_DES_CBC:
+ case CPA_CY_SYM_CIPHER_3DES_CBC:
+ case CPA_CY_SYM_CIPHER_SM4_CBC:
+ case CPA_CY_SYM_CIPHER_AES_F8: {
Cpa32U ivLenInBytes = LacSymQat_CipherIvSizeBytesGet(
pSessionDesc->cipherAlgorithm);
if (pRequest->ivLenInBytes != ivLenInBytes) {
@@ -164,11 +173,10 @@
LAC_INVALID_PARAM_LOG("invalid iv of 0");
return CPA_STATUS_INVALID_PARAM;
}
-
/* pRequest->pIv is only used for CCM so is not checked
* here */
- } else if (LAC_CIPHER_IS_KASUMI(
- pSessionDesc->cipherAlgorithm)) {
+ } break;
+ case CPA_CY_SYM_CIPHER_KASUMI_F8: {
if (LAC_CIPHER_KASUMI_F8_IV_LENGTH !=
pRequest->ivLenInBytes) {
LAC_INVALID_PARAM_LOG("invalid cipher IV size");
@@ -178,8 +186,8 @@
LAC_INVALID_PARAM_LOG("invalid iv of 0");
return CPA_STATUS_INVALID_PARAM;
}
- } else if (LAC_CIPHER_IS_SNOW3G_UEA2(
- pSessionDesc->cipherAlgorithm)) {
+ } break;
+ case CPA_CY_SYM_CIPHER_SNOW3G_UEA2: {
if (ICP_QAT_HW_SNOW_3G_UEA2_IV_SZ !=
pRequest->ivLenInBytes) {
LAC_INVALID_PARAM_LOG("invalid cipher IV size");
@@ -189,8 +197,8 @@
LAC_INVALID_PARAM_LOG("invalid iv of 0");
return CPA_STATUS_INVALID_PARAM;
}
- } else if (LAC_CIPHER_IS_ZUC_EEA3(
- pSessionDesc->cipherAlgorithm)) {
+ } break;
+ case CPA_CY_SYM_CIPHER_ZUC_EEA3: {
if (ICP_QAT_HW_ZUC_3G_EEA3_IV_SZ !=
pRequest->ivLenInBytes) {
LAC_INVALID_PARAM_LOG("invalid cipher IV size");
@@ -200,7 +208,8 @@
LAC_INVALID_PARAM_LOG("invalid iv of 0");
return CPA_STATUS_INVALID_PARAM;
}
- } else if (LAC_CIPHER_IS_CCM(pSessionDesc->cipherAlgorithm)) {
+ } break;
+ case CPA_CY_SYM_CIPHER_AES_CCM: {
if (CPA_STATUS_SUCCESS !=
LacSymAlgChain_CheckCCMData(
pRequest->pAdditionalAuthData,
@@ -209,30 +218,42 @@
pRequest->ivLenInBytes)) {
return CPA_STATUS_INVALID_PARAM;
}
+ } break;
+ default:
+ break;
}
-
/* Perform algorithm-specific checks */
- if (!(LAC_CIPHER_IS_ARC4(pSessionDesc->cipherAlgorithm) ||
- LAC_CIPHER_IS_CTR_MODE(pSessionDesc->cipherAlgorithm) ||
- LAC_CIPHER_IS_F8_MODE(pSessionDesc->cipherAlgorithm) ||
- LAC_CIPHER_IS_SNOW3G_UEA2(
- pSessionDesc->cipherAlgorithm) ||
- LAC_CIPHER_IS_ZUC_EEA3(pSessionDesc->cipherAlgorithm))) {
+ switch (pSessionDesc->cipherAlgorithm) {
+ case CPA_CY_SYM_CIPHER_ARC4:
+ case CPA_CY_SYM_CIPHER_AES_CTR:
+ case CPA_CY_SYM_CIPHER_3DES_CTR:
+ case CPA_CY_SYM_CIPHER_SM4_CTR:
+ case CPA_CY_SYM_CIPHER_AES_CCM:
+ case CPA_CY_SYM_CIPHER_AES_GCM:
+ case CPA_CY_SYM_CIPHER_CHACHA:
+ case CPA_CY_SYM_CIPHER_KASUMI_F8:
+ case CPA_CY_SYM_CIPHER_AES_F8:
+ case CPA_CY_SYM_CIPHER_SNOW3G_UEA2:
+ case CPA_CY_SYM_CIPHER_ZUC_EEA3:
+ /* No action needed */
+ break;
+ default: {
/* Mask & check below is based on assumption that block
- * size is
- * a power of 2. If data size is not a multiple of the
- * block size,
- * the "remainder" bits selected by the mask be non-zero
+ * size is a power of 2. If data size is not a multiple
+ * of the block size, the "remainder" bits selected by
+ * the mask be non-zero
*/
if (pRequest->messageLenToCipherInBytes &
(LacSymQat_CipherBlockSizeBytesGet(
pSessionDesc->cipherAlgorithm) -
1)) {
LAC_INVALID_PARAM_LOG(
- "Data size must be block size multiple");
+ "Data size must be block size"
+ " multiple");
return CPA_STATUS_INVALID_PARAM;
}
}
+ }
cipher = pSessionDesc->cipherAlgorithm;
hash = pSessionDesc->hashAlgorithm;
@@ -242,11 +263,9 @@
if (LAC_CIPHER_IS_SPC(cipher, hash, capabilitiesMask) &&
(LAC_CIPHER_SPC_IV_SIZE == pRequest->ivLenInBytes)) {
/* For CHACHA and AES_GCM single pass there is an AAD
- * buffer
- * if aadLenInBytes is nonzero. AES_GMAC AAD is stored
- * in
- * source buffer, therefore there is no separate AAD
- * buffer. */
+ * buffer if aadLenInBytes is nonzero. AES_GMAC AAD is
+ * stored in source buffer, therefore there is no
+ * separate AAD buffer. */
if ((0 != pSessionDesc->aadLenInBytes) &&
(CPA_CY_SYM_HASH_AES_GMAC !=
pSessionDesc->hashAlgorithm)) {
@@ -397,7 +416,7 @@
* @ingroup cpaCySymDp
* Write Message on the ring and write request params
* This is the optimized version, which should not be used for
- * algorithm of CCM, GCM and RC4
+ * algorithm of CCM, GCM, CHACHA and RC4
*
* @description
* Write Message on the ring and write request params
@@ -425,12 +444,15 @@
/* Write Request */
/*
* Fill in the header and footer bytes of the ET ring message - cached
- * from
- * the session descriptor.
+ * from the session descriptor.
*/
- pCacheDummyHdr = (Cpa8U *)&(pSessionDesc->reqCacheHdr);
- pCacheDummyFtr = (Cpa8U *)&(pSessionDesc->reqCacheFtr);
-
+ if (!pSessionDesc->useSymConstantsTable) {
+ pCacheDummyHdr = (Cpa8U *)&(pSessionDesc->reqCacheHdr);
+ pCacheDummyFtr = (Cpa8U *)&(pSessionDesc->reqCacheFtr);
+ } else {
+ pCacheDummyHdr = (Cpa8U *)&(pSessionDesc->shramReqCacheHdr);
+ pCacheDummyFtr = (Cpa8U *)&(pSessionDesc->shramReqCacheFtr);
+ }
memcpy(pMsgDummy,
pCacheDummyHdr,
(LAC_LONG_WORD_IN_BYTES * LAC_SIZE_OF_CACHE_HDR_IN_LW));
@@ -457,6 +479,7 @@
if (pSessionDesc->isCipher) {
LacSymQat_CipherRequestParamsPopulate(
+ pSessionDesc,
pCurrentQatMsg,
pRequest->cryptoStartSrcOffsetInBytes,
pRequest->messageLenToCipherInBytes,
@@ -493,14 +516,27 @@
* copied directly from the op request data because they share a
* corresponding layout. The remaining 4 bytes are taken
* from the session message template and use values
- * preconfigured at
- * sessionInit (updated per request for some specific cases
- * below)
+ * preconfigured at sessionInit (updated per request for some
+ * specific cases below)
*/
- memcpy(pAuthReqPars,
- (Cpa32U *)&(pRequest->hashStartSrcOffsetInBytes),
- ((unsigned long)&(pAuthReqPars->u2.inner_prefix_sz) -
- (unsigned long)pAuthReqPars));
+
+ /* We force a specific compiler optimisation here. The length
+ * to be copied turns out to be always 16, and by coding a
+ * memcpy with a literal value the compiler will compile inline
+ * code (in fact, only two vector instructions) to effect the
+ * copy. This gives us a huge performance increase.
+ */
+ unsigned long cplen =
+ (unsigned long)&(pAuthReqPars->u2.inner_prefix_sz) -
+ (unsigned long)pAuthReqPars;
+ if (cplen == 16)
+ memcpy(pAuthReqPars,
+ (Cpa32U *)&(pRequest->hashStartSrcOffsetInBytes),
+ 16);
+ else
+ memcpy(pAuthReqPars,
+ (Cpa32U *)&(pRequest->hashStartSrcOffsetInBytes),
+ cplen);
if (CPA_TRUE == pSessionDesc->isAuthEncryptOp) {
pAuthReqPars->hash_state_sz =
@@ -548,25 +584,29 @@
Cpa32U sizeInBytes = 0;
CpaCySymCipherAlgorithm cipher = pSessionDesc->cipherAlgorithm;
CpaCySymHashAlgorithm hash = pSessionDesc->hashAlgorithm;
+ sal_crypto_service_t *pService =
+ (sal_crypto_service_t *)pRequest->instanceHandle;
Cpa32U capabilitiesMask =
((sal_crypto_service_t *)pRequest->instanceHandle)
->generic_service_info.capabilitiesMask;
+ CpaBoolean isSpCcm = LAC_CIPHER_IS_CCM(cipher) &&
+ LAC_CIPHER_IS_SPC(cipher, hash, capabilitiesMask);
+
Cpa8U paddingLen = 0;
Cpa8U blockLen = 0;
+ Cpa32U aadDataLen = 0;
pMsgDummy = (Cpa8U *)pCurrentQatMsg;
/* Write Request */
/*
* Fill in the header and footer bytes of the ET ring message - cached
- * from
- * the session descriptor.
+ * from the session descriptor.
*/
- if (!pSessionDesc->isSinglePass &&
- LAC_CIPHER_IS_SPC(cipher, hash, capabilitiesMask) &&
- (LAC_CIPHER_SPC_IV_SIZE == pRequest->ivLenInBytes)) {
- pSessionDesc->isSinglePass = CPA_TRUE;
+ if ((NON_SPC != pSessionDesc->singlePassState) &&
+ (isSpCcm || (LAC_CIPHER_SPC_IV_SIZE == pRequest->ivLenInBytes))) {
+ pSessionDesc->singlePassState = SPC;
pSessionDesc->isCipher = CPA_TRUE;
pSessionDesc->isAuthEncryptOp = CPA_FALSE;
pSessionDesc->isAuth = CPA_FALSE;
@@ -581,7 +621,8 @@
*/
ICP_QAT_FW_LA_SINGLE_PASS_PROTO_FLAG_SET(
pSessionDesc->laCmdFlags, ICP_QAT_FW_LA_SINGLE_PASS_PROTO);
- ICP_QAT_FW_LA_PROTO_SET(pSessionDesc->laCmdFlags, 0);
+ if (isCyGen2x(pService))
+ ICP_QAT_FW_LA_PROTO_SET(pSessionDesc->laCmdFlags, 0);
pCdInfo = &(pSessionDesc->contentDescInfo);
pHwBlockBaseInDRAM = (Cpa8U *)pCdInfo->pData;
@@ -589,11 +630,22 @@
pSessionDesc->cipherDirection) {
if (LAC_CIPHER_IS_GCM(cipher))
hwBlockOffsetInDRAM = LAC_QUADWORDS_TO_BYTES(
- LAC_SYM_QAT_CIPHER_OFFSET_IN_DRAM_GCM_SPC);
- else
+ LAC_SYM_QAT_CIPHER_GCM_SPC_OFFSET_IN_DRAM);
+ else if (LAC_CIPHER_IS_CHACHA(cipher))
hwBlockOffsetInDRAM = LAC_QUADWORDS_TO_BYTES(
- LAC_SYM_QAT_CIPHER_OFFSET_IN_DRAM_CHACHA_SPC);
+ LAC_SYM_QAT_CIPHER_CHACHA_SPC_OFFSET_IN_DRAM);
+ } else if (isSpCcm) {
+ hwBlockOffsetInDRAM = LAC_QUADWORDS_TO_BYTES(
+ LAC_SYM_QAT_CIPHER_CCM_SPC_OFFSET_IN_DRAM);
}
+
+ /* Update slice type, as used algos changed */
+ pSessionDesc->cipherSliceType =
+ LacCipher_GetCipherSliceType(pService, cipher, hash);
+
+ ICP_QAT_FW_LA_SLICE_TYPE_SET(pSessionDesc->laCmdFlags,
+ pSessionDesc->cipherSliceType);
+
/* construct cipherConfig in CD in DRAM */
LacSymQat_CipherHwBlockPopulateCfgData(pSessionDesc,
pHwBlockBaseInDRAM +
@@ -605,10 +657,42 @@
pSessionDesc->laCmdId,
pSessionDesc->cmnRequestFlags,
pSessionDesc->laCmdFlags);
+ } else if ((SPC == pSessionDesc->singlePassState) &&
+ (LAC_CIPHER_SPC_IV_SIZE != pRequest->ivLenInBytes)) {
+ pSessionDesc->symOperation = CPA_CY_SYM_OP_ALGORITHM_CHAINING;
+ pSessionDesc->singlePassState = LIKELY_SPC;
+ pSessionDesc->isCipher = CPA_TRUE;
+ pSessionDesc->isAuthEncryptOp = CPA_TRUE;
+ pSessionDesc->isAuth = CPA_TRUE;
+ pCdInfo = &(pSessionDesc->contentDescInfo);
+ pHwBlockBaseInDRAM = (Cpa8U *)pCdInfo->pData;
+
+ if (CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT ==
+ pSessionDesc->cipherDirection) {
+ pSessionDesc->laCmdId = ICP_QAT_FW_LA_CMD_CIPHER_HASH;
+ } else {
+ pSessionDesc->laCmdId = ICP_QAT_FW_LA_CMD_HASH_CIPHER;
+ }
+
+ ICP_QAT_FW_LA_SINGLE_PASS_PROTO_FLAG_SET(
+ pSessionDesc->laCmdFlags, 0);
+ ICP_QAT_FW_LA_PROTO_SET(pSessionDesc->laCmdFlags,
+ ICP_QAT_FW_LA_GCM_PROTO);
+
+ LacSymQat_CipherHwBlockPopulateCfgData(pSessionDesc,
+ pHwBlockBaseInDRAM +
+ hwBlockOffsetInDRAM,
+ &sizeInBytes);
+ SalQatMsg_CmnHdrWrite((icp_qat_fw_comn_req_t *)&(
+ pSessionDesc->reqCacheHdr),
+ ICP_QAT_FW_COMN_REQ_CPM_FW_LA,
+ pSessionDesc->laCmdId,
+ pSessionDesc->cmnRequestFlags,
+ pSessionDesc->laCmdFlags);
} else if (CPA_CY_SYM_HASH_AES_GMAC == pSessionDesc->hashAlgorithm) {
pSessionDesc->aadLenInBytes = pRequest->messageLenToHashInBytes;
}
- if (pSessionDesc->isSinglePass) {
+ if (SPC == pSessionDesc->singlePassState) {
pCacheDummyHdr = (Cpa8U *)&(pSessionDesc->reqSpcCacheHdr);
pCacheDummyFtr = (Cpa8U *)&(pSessionDesc->reqSpcCacheFtr);
} else {
@@ -644,8 +728,9 @@
pRequest->srcBufferLen,
pRequest->dstBufferLen);
- if (CPA_CY_SYM_HASH_AES_CCM == pSessionDesc->hashAlgorithm &&
- pSessionDesc->isAuth == CPA_TRUE) {
+ if ((CPA_CY_SYM_HASH_AES_CCM == pSessionDesc->hashAlgorithm &&
+ pSessionDesc->isAuth == CPA_TRUE) ||
+ isSpCcm) {
/* prepare IV and AAD for CCM */
LacSymAlgChain_PrepareCCMData(
pSessionDesc,
@@ -655,15 +740,14 @@
pRequest->ivLenInBytes);
/* According to the API, for CCM and GCM,
- * messageLenToHashInBytes
- * and hashStartSrcOffsetInBytes are not initialized by the
- * user and must be set by the driver
+ * messageLenToHashInBytes and hashStartSrcOffsetInBytes are not
+ * initialized by the user and must be set by the driver
*/
pRequest->hashStartSrcOffsetInBytes =
pRequest->cryptoStartSrcOffsetInBytes;
pRequest->messageLenToHashInBytes =
pRequest->messageLenToCipherInBytes;
- } else if (!pSessionDesc->isSinglePass &&
+ } else if ((SPC != pSessionDesc->singlePassState) &&
(CPA_CY_SYM_HASH_AES_GCM == pSessionDesc->hashAlgorithm ||
CPA_CY_SYM_HASH_AES_GMAC == pSessionDesc->hashAlgorithm)) {
/* GCM case */
@@ -693,56 +777,87 @@
if (pSessionDesc->isCipher) {
if (CPA_CY_SYM_CIPHER_ARC4 == pSessionDesc->cipherAlgorithm) {
/* ARC4 does not have an IV but the field is used to
- * store the
- * initial state */
+ * store the initial state */
pRequest->iv =
pSessionDesc->cipherARC4InitialStatePhysAddr;
}
+ ICP_QAT_FW_LA_SLICE_TYPE_SET(
+ pCurrentQatMsg->comn_hdr.serv_specif_flags,
+ pSessionDesc->cipherSliceType);
+
LacSymQat_CipherRequestParamsPopulate(
+ pSessionDesc,
pCurrentQatMsg,
pRequest->cryptoStartSrcOffsetInBytes,
pRequest->messageLenToCipherInBytes,
pRequest->iv,
pRequest->pIv);
- if (pSessionDesc->isSinglePass) {
+ if (SPC == pSessionDesc->singlePassState) {
icp_qat_fw_la_cipher_req_params_t *pCipherReqParams =
(icp_qat_fw_la_cipher_req_params_t
*)((Cpa8U *)&(
pCurrentQatMsg->serv_specif_rqpars) +
ICP_QAT_FW_CIPHER_REQUEST_PARAMETERS_OFFSET);
- pCipherReqParams->spc_aad_addr =
- (uint64_t)pRequest->additionalAuthData;
- pCipherReqParams->spc_aad_sz =
- pSessionDesc->aadLenInBytes;
+ icp_qat_fw_la_cipher_20_req_params_t *pCipher20ReqParams =
+ (void
+ *)((Cpa8U *)&(
+ pCurrentQatMsg->serv_specif_rqpars) +
+ ICP_QAT_FW_CIPHER_REQUEST_PARAMETERS_OFFSET);
- pCipherReqParams->spc_auth_res_addr =
- (uint64_t)pRequest->digestResult;
- pCipherReqParams->spc_auth_res_sz =
- pSessionDesc->hashResultSize;
+ if (isCyGen4x(pService)) {
+ pCipher20ReqParams->spc_aad_addr =
+ (uint64_t)pRequest->additionalAuthData;
+ pCipher20ReqParams->spc_aad_sz =
+ pSessionDesc->aadLenInBytes;
+ pCipher20ReqParams->spc_aad_offset = 0;
+
+ if (isSpCcm)
+ pCipher20ReqParams->spc_aad_sz +=
+ LAC_CIPHER_CCM_AAD_OFFSET;
+
+ pCipher20ReqParams->spc_auth_res_addr =
+ (uint64_t)pRequest->digestResult;
+ pCipher20ReqParams->spc_auth_res_sz =
+ (Cpa8U)pSessionDesc->hashResultSize;
+ } else {
+ pCipherReqParams->spc_aad_addr =
+ (uint64_t)pRequest->additionalAuthData;
+ pCipherReqParams->spc_aad_sz =
+ (Cpa16U)pSessionDesc->aadLenInBytes;
+
+ pCipherReqParams->spc_auth_res_addr =
+ (uint64_t)pRequest->digestResult;
+ pCipherReqParams->spc_auth_res_sz =
+ (Cpa8U)pSessionDesc->hashResultSize;
+ }
/* For CHACHA and AES_GCM single pass AAD buffer needs
- * alignment
- * if aadLenInBytes is nonzero.
- * In case of AES-GMAC, AAD buffer passed in the src
- * buffer.
+ * alignment if aadLenInBytes is nonzero. In case of
+ * AES-GMAC, AAD buffer passed in the src buffer.
*/
if (0 != pSessionDesc->aadLenInBytes &&
CPA_CY_SYM_HASH_AES_GMAC !=
pSessionDesc->hashAlgorithm) {
blockLen = LacSymQat_CipherBlockSizeBytesGet(
pSessionDesc->cipherAlgorithm);
- if ((pSessionDesc->aadLenInBytes % blockLen) !=
- 0) {
- paddingLen = blockLen -
- (pSessionDesc->aadLenInBytes %
- blockLen);
- memset(
- &pRequest->pAdditionalAuthData
- [pSessionDesc->aadLenInBytes],
- 0,
- paddingLen);
+ aadDataLen = pSessionDesc->aadLenInBytes;
+
+ /* In case of AES_CCM, B0 block size and 2 bytes
+ * of AAD len
+ * encoding need to be added to total AAD data
+ * len */
+ if (isSpCcm)
+ aadDataLen += LAC_CIPHER_CCM_AAD_OFFSET;
+
+ if ((aadDataLen % blockLen) != 0) {
+ paddingLen =
+ blockLen - (aadDataLen % blockLen);
+ memset(&pRequest->pAdditionalAuthData
+ [aadDataLen],
+ 0,
+ paddingLen);
}
}
}
@@ -777,32 +892,14 @@
* copied directly from the op request data because they share a
* corresponding layout. The remaining 4 bytes are taken
* from the session message template and use values
- * preconfigured at
- * sessionInit (updated per request for some specific cases
- * below)
+ * preconfigured at sessionInit (updated per request for some
+ * specific cases below)
*/
-
- /* We force a specific compiler optimisation here. The length
- * to
- * be copied turns out to be always 16, and by coding a memcpy
- * with
- * a literal value the compiler will compile inline code (in
- * fact,
- * only two vector instructions) to effect the copy. This gives
- * us
- * a huge performance increase.
- */
- unsigned long cplen =
- (unsigned long)&(pAuthReqPars->u2.inner_prefix_sz) -
- (unsigned long)pAuthReqPars;
- if (cplen == 16)
- memcpy(pAuthReqPars,
- (Cpa32U *)&(pRequest->hashStartSrcOffsetInBytes),
- 16);
- else
- memcpy(pAuthReqPars,
- (Cpa32U *)&(pRequest->hashStartSrcOffsetInBytes),
- cplen);
+ memcpy(pAuthReqPars,
+ (Cpa32U *)&(pRequest->hashStartSrcOffsetInBytes),
+ ((uintptr_t) &
+ (pAuthReqPars->u2.inner_prefix_sz) -
+ (uintptr_t)pAuthReqPars));
if (CPA_TRUE == pSessionDesc->isAuthEncryptOp) {
pAuthReqPars->hash_state_sz =
@@ -892,7 +989,7 @@
/* CPA_INSTANCE_HANDLE_SINGLE is not supported on DP apis */
LAC_CHECK_INSTANCE_HANDLE(instanceHandle);
-/* All other param checks are common with trad api */
+ /* All other param checks are common with trad api */
return cpaCySymRemoveSession(instanceHandle, sessionCtx);
}
@@ -932,6 +1029,10 @@
return status;
}
+ /* Check if SAL is running in crypto data plane otherwise return an
+ * error */
+ SAL_RUNNING_CHECK(pRequest->instanceHandle);
+
trans_handle = ((sal_crypto_service_t *)pRequest->instanceHandle)
->trans_handle_sym_tx;
@@ -1029,6 +1130,10 @@
}
}
+ /* Check if SAL is running in crypto data plane otherwise return an
+ * error */
+ SAL_RUNNING_CHECK(pRequests[0]->instanceHandle);
+
trans_handle = ((sal_crypto_service_t *)pRequests[0]->instanceHandle)
->trans_handle_sym_tx;
pSessionDesc =
Index: sys/dev/qat/qat_api/common/crypto/sym/lac_sym_hash.c
===================================================================
--- sys/dev/qat/qat_api/common/crypto/sym/lac_sym_hash.c
+++ sys/dev/qat/qat_api/common/crypto/sym/lac_sym_hash.c
@@ -50,17 +50,16 @@
(CPA_CY_SYM_HASH_AES_CMAC == (alg)) || \
(CPA_CY_SYM_HASH_ZUC_EIA3 == (alg))) && \
(CPA_CY_SYM_HASH_MODE_AUTH != (mode))) || \
- (((CPA_CY_SYM_HASH_SHA3_224 == (alg)) || \
- (CPA_CY_SYM_HASH_SHA3_256 == (alg)) || \
- (CPA_CY_SYM_HASH_SHA3_384 == (alg)) || \
- (CPA_CY_SYM_HASH_SHA3_512 == (alg))) && \
- (CPA_CY_SYM_HASH_MODE_NESTED == (mode))) || \
- (((CPA_CY_SYM_HASH_SHAKE_128 == (alg)) || \
- (CPA_CY_SYM_HASH_SHAKE_256 == (alg))) && \
- (CPA_CY_SYM_HASH_MODE_AUTH == (mode))))
-
+ ((LAC_HASH_IS_SHA3(alg)) && (CPA_CY_SYM_HASH_MODE_NESTED == (mode))))
/**< Macro to check for valid algorithm-mode combination */
+void LacSync_GenBufListVerifyCb(void *pCallbackTag,
+ CpaStatus status,
+ CpaCySymOp operationType,
+ void *pOpData,
+ CpaBufferList *pDstBuffer,
+ CpaBoolean opResult);
+
/**
* @ingroup LacHash
* This callback function will be invoked whenever a synchronous
@@ -98,7 +97,7 @@
if (pHashStateBufferInfo->pDataPhys == 0) {
LAC_LOG_ERROR("Unable to get the physical address of "
- "the hash state buffer");
+ "the hash state buffer\n");
return CPA_STATUS_FAIL;
}
@@ -110,9 +109,11 @@
pHashStateBufferInfo,
pReq,
pHashSetupData->nestedModeSetupData.pInnerPrefixData,
- pHashSetupData->nestedModeSetupData.innerPrefixLenInBytes,
+ (Cpa8U)pHashSetupData->nestedModeSetupData
+ .innerPrefixLenInBytes,
pHashSetupData->nestedModeSetupData.pOuterPrefixData,
- pHashSetupData->nestedModeSetupData.outerPrefixLenInBytes);
+ (Cpa8U)pHashSetupData->nestedModeSetupData
+ .outerPrefixLenInBytes);
}
/* For mode2 HMAC the key gets copied into both the inner and
* outer prefix fields */
@@ -121,9 +122,9 @@
pHashStateBufferInfo,
pReq,
pHashSetupData->authModeSetupData.authKey,
- pHashSetupData->authModeSetupData.authKeyLenInBytes,
+ (Cpa8U)pHashSetupData->authModeSetupData.authKeyLenInBytes,
pHashSetupData->authModeSetupData.authKey,
- pHashSetupData->authModeSetupData.authKeyLenInBytes);
+ (Cpa8U)pHashSetupData->authModeSetupData.authKeyLenInBytes);
}
/* else do nothing for the other cases */
return CPA_STATUS_SUCCESS;
@@ -226,22 +227,32 @@
pCallbackTag);
} else if (CPA_CY_SYM_HASH_AES_CCM == hashAlgorithm) {
/*
- * The Inner Hash Initial State2 block must contain K
- * (the cipher key) and 16 zeroes which will be replaced with
- * EK(Ctr0) by the QAT-ME.
- */
-
- /* write the auth key which for CCM is equivalent to cipher key
+ * The Inner Hash Initial State2 block is 32 bytes long.
+ * Therefore, for keys bigger than 128 bits (16 bytes),
+ * there is no space for 16 zeroes.
*/
- memcpy(pState2,
- pSessionSetup->cipherSetupData.pCipherKey,
- pSessionSetup->cipherSetupData.cipherKeyLenInBytes);
+ if (pSessionSetup->cipherSetupData.cipherKeyLenInBytes ==
+ ICP_QAT_HW_AES_128_KEY_SZ) {
+ /*
+ * The Inner Hash Initial State2 block must contain K
+ * (the cipher key) and 16 zeroes which will be replaced
+ * with EK(Ctr0) by the QAT-ME.
+ */
- /* initialize remaining buffer space to all zeroes */
- LAC_OS_BZERO(
- pState2 +
- pSessionSetup->cipherSetupData.cipherKeyLenInBytes,
- ICP_QAT_HW_AES_CCM_CBC_E_CTR0_SZ);
+ /* write the auth key which for CCM is equivalent to
+ * cipher key
+ */
+ memcpy(
+ pState2,
+ pSessionSetup->cipherSetupData.pCipherKey,
+ pSessionSetup->cipherSetupData.cipherKeyLenInBytes);
+
+ /* initialize remaining buffer space to all zeroes */
+ LAC_OS_BZERO(pState2 +
+ pSessionSetup->cipherSetupData
+ .cipherKeyLenInBytes,
+ ICP_QAT_HW_AES_CCM_CBC_E_CTR0_SZ);
+ }
/* There is no request sent to the QAT for this operation,
* so just invoke the user's callback directly to signal
@@ -279,8 +290,8 @@
if (CPA_STATUS_SUCCESS == status) {
/* write len(A) (the length of A) into bytes 16-19 of
- * pState2
- * in big-endian format. This field is 8 bytes */
+ * pState2 in big-endian format. This field is 8 bytes
+ */
*(Cpa32U *)&pState2[ICP_QAT_HW_GALOIS_H_SZ] =
LAC_MEM_WR_32(pAuthModeSetupData->aadLenInBytes);
}
@@ -367,26 +378,15 @@
CpaCySymCapabilitiesInfo capInfo;
/*Protect against value of hash outside the bitmap*/
- if ((pHashSetupData->hashAlgorithm) >=
- CPA_CY_SYM_HASH_CAP_BITMAP_SIZE) {
+ if (pHashSetupData->hashAlgorithm >= CPA_CY_SYM_HASH_CAP_BITMAP_SIZE) {
LAC_INVALID_PARAM_LOG("hashAlgorithm");
return CPA_STATUS_INVALID_PARAM;
}
+
cpaCySymQueryCapabilities(instanceHandle, &capInfo);
if (!CPA_BITMAP_BIT_TEST(capInfo.hashes,
pHashSetupData->hashAlgorithm) &&
pHashSetupData->hashAlgorithm != CPA_CY_SYM_HASH_AES_CBC_MAC) {
- /* Ensure SHAKE algorithms are not supported */
- if ((CPA_CY_SYM_HASH_SHAKE_128 ==
- pHashSetupData->hashAlgorithm) ||
- (CPA_CY_SYM_HASH_SHAKE_256 ==
- pHashSetupData->hashAlgorithm)) {
- LAC_INVALID_PARAM_LOG(
- "Hash algorithms SHAKE-128 and SHAKE-256 "
- "are not supported.");
- return CPA_STATUS_UNSUPPORTED;
- }
-
LAC_INVALID_PARAM_LOG("hashAlgorithm");
return CPA_STATUS_INVALID_PARAM;
}
@@ -405,8 +405,9 @@
if (LAC_HASH_ALG_MODE_NOT_SUPPORTED(pHashSetupData->hashAlgorithm,
pHashSetupData->hashMode)) {
- LAC_INVALID_PARAM_LOG("hashAlgorithm and hashMode combination");
- return CPA_STATUS_INVALID_PARAM;
+ LAC_UNSUPPORTED_PARAM_LOG(
+ "hashAlgorithm and hashMode combination");
+ return CPA_STATUS_UNSUPPORTED;
}
LacSymQat_HashAlgLookupGet(instanceHandle,
@@ -432,10 +433,8 @@
Cpa32U aadDataSize = 0;
/* RFC 4106: Implementations MUST support a full-length
- * 16-octet
- * ICV, and MAY support 8 or 12 octet ICVs, and MUST NOT
- * support
- * other ICV lengths. */
+ * 16-octet ICV, and MAY support 8 or 12 octet ICVs, and
+ * MUST NOT support other ICV lengths. */
if ((pHashSetupData->digestResultLenInBytes !=
LAC_HASH_AES_GCM_ICV_SIZE_8) &&
(pHashSetupData->digestResultLenInBytes !=
@@ -490,15 +489,13 @@
aadDataSize = LAC_HASH_AES_CCM_BLOCK_SIZE;
/* then, if there is some 'a' data, the buffer will
- * store encoded
- * length of 'a' and 'a' itself */
+ * store encoded length of 'a' and 'a' itself */
if (pHashSetupData->authModeSetupData.aadLenInBytes >
0) {
/* as the QAT API puts the requirement on the
* pAdditionalAuthData not to be bigger than 240
- * bytes then we
- * just need 2 bytes to store encoded length of
- * 'a' */
+ * bytes then we just need 2 bytes to store
+ * encoded length of 'a' */
aadDataSize += sizeof(Cpa16U);
aadDataSize += pHashSetupData->authModeSetupData
.aadLenInBytes;
@@ -536,8 +533,7 @@
return CPA_STATUS_INVALID_PARAM;
}
/* For Snow3g hash aad field contains IV - it needs to
- * be 16
- * bytes long
+ * be 16 bytes long
*/
if (pHashSetupData->authModeSetupData.aadLenInBytes !=
ICP_QAT_HW_SNOW_3G_UEA2_IV_SZ) {
@@ -570,8 +566,7 @@
return CPA_STATUS_INVALID_PARAM;
}
/* For ZUC EIA3 hash aad field contains IV - it needs to
- * be 16
- * bytes long
+ * be 16 bytes long
*/
if (pHashSetupData->authModeSetupData.aadLenInBytes !=
ICP_QAT_HW_ZUC_3G_EEA3_IV_SZ) {
@@ -613,19 +608,6 @@
if (!CPA_BITMAP_BIT_TEST(capInfo.hashes,
pHashSetupData->nestedModeSetupData
.outerHashAlgorithm)) {
- /* Ensure SHAKE algorithms are not supported */
- if ((CPA_CY_SYM_HASH_SHAKE_128 ==
- pHashSetupData->nestedModeSetupData
- .outerHashAlgorithm) ||
- (CPA_CY_SYM_HASH_SHAKE_256 ==
- pHashSetupData->nestedModeSetupData
- .outerHashAlgorithm)) {
- LAC_INVALID_PARAM_LOG(
- "Hash algorithms SHAKE-128 and SHAKE-256 "
- "are not supported.");
- return CPA_STATUS_UNSUPPORTED;
- }
-
LAC_INVALID_PARAM_LOG("outerHashAlgorithm");
return CPA_STATUS_INVALID_PARAM;
}
@@ -689,11 +671,15 @@
{
CpaStatus status = CPA_STATUS_SUCCESS;
lac_sym_qat_hash_alg_info_t *pHashAlgInfo = NULL;
+ CpaBoolean digestIsAppended = pSessionDesc->digestIsAppended;
+ CpaBoolean digestVerify = pSessionDesc->digestVerify;
+ CpaCySymOp symOperation = pSessionDesc->symOperation;
+ CpaCySymHashAlgorithm hashAlgorithm = pSessionDesc->hashAlgorithm;
/* digestVerify and digestIsAppended on Hash-Only operation not
* supported */
- if (pSessionDesc->digestIsAppended && pSessionDesc->digestVerify &&
- (CPA_CY_SYM_OP_HASH == pSessionDesc->symOperation)) {
+ if (digestIsAppended && digestVerify &&
+ (CPA_CY_SYM_OP_HASH == symOperation)) {
LAC_INVALID_PARAM_LOG(
"digestVerify and digestIsAppended set "
"on Hash-Only operation is not supported");
@@ -702,21 +688,19 @@
/* check the digest result pointer */
if ((CPA_CY_SYM_PACKET_TYPE_PARTIAL != pOpData->packetType) &&
- !pSessionDesc->digestIsAppended &&
- (NULL == pOpData->pDigestResult)) {
+ !digestIsAppended && (NULL == pOpData->pDigestResult)) {
LAC_INVALID_PARAM_LOG("pDigestResult is NULL");
return CPA_STATUS_INVALID_PARAM;
}
/*
* Check if the pVerifyResult pointer is not null for hash operation
- * when
- * the packet is the last one and user has set verifyDigest flag
+ * when the packet is the last one and user has set verifyDigest flag
* Also, this is only needed for symchronous operation, so check if the
* callback pointer is the internal synchronous one rather than a user-
* supplied one.
*/
- if ((CPA_TRUE == pSessionDesc->digestVerify) &&
+ if ((CPA_TRUE == digestVerify) &&
(CPA_CY_SYM_PACKET_TYPE_PARTIAL != pOpData->packetType) &&
(LacSync_GenBufListVerifyCb == pSessionDesc->pSymCb)) {
if (NULL == pVerifyResult) {
@@ -732,8 +716,8 @@
* written anywhere so we cannot check for this been inside a buffer
* CCM/GCM specify the auth region using just the cipher params as this
* region is the same for auth and cipher. It is not checked here */
- if ((CPA_CY_SYM_HASH_AES_CCM == pSessionDesc->hashAlgorithm) ||
- (CPA_CY_SYM_HASH_AES_GCM == pSessionDesc->hashAlgorithm)) {
+ if ((CPA_CY_SYM_HASH_AES_CCM == hashAlgorithm) ||
+ (CPA_CY_SYM_HASH_AES_GCM == hashAlgorithm)) {
/* ensure AAD data pointer is non-NULL if AAD len > 0 */
if ((pSessionDesc->aadLenInBytes > 0) &&
(NULL == pOpData->pAdditionalAuthData)) {
@@ -752,8 +736,8 @@
/* For Snow3g & ZUC hash pAdditionalAuthData field
* of OpData should contain IV */
- if ((CPA_CY_SYM_HASH_SNOW3G_UIA2 == pSessionDesc->hashAlgorithm) ||
- (CPA_CY_SYM_HASH_ZUC_EIA3 == pSessionDesc->hashAlgorithm)) {
+ if ((CPA_CY_SYM_HASH_SNOW3G_UIA2 == hashAlgorithm) ||
+ (CPA_CY_SYM_HASH_ZUC_EIA3 == hashAlgorithm)) {
if (NULL == pOpData->pAdditionalAuthData) {
LAC_INVALID_PARAM_LOG("pAdditionalAuthData is NULL");
return CPA_STATUS_INVALID_PARAM;
@@ -761,12 +745,11 @@
}
/* partial packets need to be multiples of the algorithm block size in
- * hash
- * only mode (except for final partial packet) */
+ * hash only mode (except for final partial packet) */
if ((CPA_CY_SYM_PACKET_TYPE_PARTIAL == pOpData->packetType) &&
- (CPA_CY_SYM_OP_HASH == pSessionDesc->symOperation)) {
+ (CPA_CY_SYM_OP_HASH == symOperation)) {
LacSymQat_HashAlgLookupGet(instanceHandle,
- pSessionDesc->hashAlgorithm,
+ hashAlgorithm,
&pHashAlgInfo);
/* check if the message is a multiple of the block size. */
Index: sys/dev/qat/qat_api/common/crypto/sym/lac_sym_queue.c
===================================================================
--- sys/dev/qat/qat_api/common/crypto/sym/lac_sym_queue.c
+++ sys/dev/qat/qat_api/common/crypto/sym/lac_sym_queue.c
@@ -57,11 +57,7 @@
*/
if ((CPA_FALSE == pSessionDesc->nonBlockingOpsInProgress) ||
(NULL != pSessionDesc->pRequestQueueTail)) {
- if (CPA_STATUS_SUCCESS !=
- LAC_SPINLOCK(&pSessionDesc->requestQueueLock)) {
- LAC_LOG_ERROR("Failed to lock request queue");
- return CPA_STATUS_RESOURCE;
- }
+ LAC_SPINLOCK(&pSessionDesc->requestQueueLock);
/* Re-check blockingOpsInProgress and pRequestQueueTail in case
* either
@@ -95,10 +91,7 @@
/* request is queued, don't send to QAT here */
enqueued = CPA_TRUE;
}
- if (CPA_STATUS_SUCCESS !=
- LAC_SPINUNLOCK(&pSessionDesc->requestQueueLock)) {
- LAC_LOG_ERROR("Failed to unlock request queue");
- }
+ LAC_SPINUNLOCK(&pSessionDesc->requestQueueLock);
}
if (CPA_FALSE == enqueued) {
Index: sys/dev/qat/qat_api/common/crypto/sym/qat/lac_sym_qat.c
===================================================================
--- sys/dev/qat/qat_api/common/crypto/sym/qat/lac_sym_qat.c
+++ sys/dev/qat/qat_api/common/crypto/sym/qat/lac_sym_qat.c
@@ -31,6 +31,7 @@
#include "sal_string_parse.h"
#include "lac_sym_key.h"
#include "lac_sym_qat_hash_defs_lookup.h"
+#include "lac_sym_qat_constants_table.h"
#include "lac_sym_qat_cipher.h"
#include "lac_sym_qat_hash.h"
@@ -104,6 +105,9 @@
{
CpaStatus status = CPA_STATUS_SUCCESS;
+ /* Initialize the SHRAM constants table */
+ LacSymQat_ConstantsInitLookupTables(instanceHandle);
+
/* Initialise the Hash lookup table */
status = LacSymQat_HashLookupInit(instanceHandle);
@@ -130,10 +134,10 @@
Cpa16U *pLaCommandFlags,
Cpa32U ivLenInBytes)
{
- /* For Chacha ciphers set command flag as partial none to proceed
+ /* For SM4/Chacha ciphers set command flag as partial none to proceed
* with stateless processing */
- if (LAC_CIPHER_IS_CHACHA(cipherAlgorithm) ||
- LAC_CIPHER_IS_SM4(cipherAlgorithm)) {
+ if (LAC_CIPHER_IS_SM4(cipherAlgorithm) ||
+ LAC_CIPHER_IS_CHACHA(cipherAlgorithm)) {
ICP_QAT_FW_LA_PARTIAL_SET(*pLaCommandFlags,
ICP_QAT_FW_LA_PARTIAL_NONE);
return;
@@ -144,10 +148,10 @@
* must be disabled always.
* For all other ciphers and auth
* update state is disabled for full packets and final partials */
- if (((laCmdId != ICP_QAT_FW_LA_CMD_AUTH) &&
- LAC_CIPHER_IS_ECB_MODE(cipherAlgorithm)) ||
- (ICP_QAT_FW_LA_PARTIAL_NONE == qatPacketType) ||
- (ICP_QAT_FW_LA_PARTIAL_END == qatPacketType)) {
+ if ((ICP_QAT_FW_LA_PARTIAL_NONE == qatPacketType) ||
+ (ICP_QAT_FW_LA_PARTIAL_END == qatPacketType) ||
+ ((laCmdId != ICP_QAT_FW_LA_CMD_AUTH) &&
+ LAC_CIPHER_IS_ECB_MODE(cipherAlgorithm))) {
ICP_QAT_FW_LA_UPDATE_STATE_SET(*pLaCommandFlags,
ICP_QAT_FW_LA_NO_UPDATE_STATE);
}
@@ -182,8 +186,9 @@
CpaCySymPacketType packetState,
Cpa32U *pQatPacketType)
{
+ switch (packetType) {
/* partial */
- if (CPA_CY_SYM_PACKET_TYPE_PARTIAL == packetType) {
+ case CPA_CY_SYM_PACKET_TYPE_PARTIAL:
/* if the previous state was full, then this is the first packet
*/
if (CPA_CY_SYM_PACKET_TYPE_FULL == packetState) {
@@ -191,13 +196,15 @@
} else {
*pQatPacketType = ICP_QAT_FW_LA_PARTIAL_MID;
}
- }
+ break;
+
/* final partial */
- else if (CPA_CY_SYM_PACKET_TYPE_LAST_PARTIAL == packetType) {
+ case CPA_CY_SYM_PACKET_TYPE_LAST_PARTIAL:
*pQatPacketType = ICP_QAT_FW_LA_PARTIAL_END;
- }
+ break;
+
/* full packet - CPA_CY_SYM_PACKET_TYPE_FULL */
- else {
+ default:
*pQatPacketType = ICP_QAT_FW_LA_PARTIAL_NONE;
}
}
@@ -225,3 +232,101 @@
ICP_QAT_FW_LA_GCM_IV_LEN_FLAG_SET(
*laCmdFlags, ICP_QAT_FW_LA_GCM_IV_LEN_NOT_12_OCTETS);
}
+
+CpaBoolean
+LacSymQat_UseSymConstantsTable(lac_session_desc_t *pSession,
+ Cpa8U *pCipherOffset,
+ Cpa8U *pHashOffset)
+{
+
+ CpaBoolean useOptimisedContentDesc = CPA_FALSE;
+ CpaBoolean useSHRAMConstants = CPA_FALSE;
+
+ *pCipherOffset = 0;
+ *pHashOffset = 0;
+
+ /* for chaining can we use the optimised content descritor */
+ if (pSession->laCmdId == ICP_QAT_FW_LA_CMD_CIPHER_HASH ||
+ pSession->laCmdId == ICP_QAT_FW_LA_CMD_HASH_CIPHER) {
+ useOptimisedContentDesc =
+ LacSymQat_UseOptimisedContentDesc(pSession);
+ }
+
+ /* Cipher-only case or chaining */
+ if (pSession->laCmdId == ICP_QAT_FW_LA_CMD_CIPHER ||
+ useOptimisedContentDesc) {
+ icp_qat_hw_cipher_algo_t algorithm;
+ icp_qat_hw_cipher_mode_t mode;
+ icp_qat_hw_cipher_dir_t dir;
+ icp_qat_hw_cipher_convert_t key_convert;
+
+ if (pSession->cipherKeyLenInBytes >
+ sizeof(icp_qat_fw_comn_req_hdr_cd_pars_t)) {
+ return CPA_FALSE;
+ }
+
+ LacSymQat_CipherGetCfgData(
+ pSession, &algorithm, &mode, &dir, &key_convert);
+
+ /* Check if cipher config is available in table. */
+ LacSymQat_ConstantsGetCipherOffset(pSession->pInstance,
+ algorithm,
+ mode,
+ dir,
+ key_convert,
+ pCipherOffset);
+ if (*pCipherOffset > 0) {
+ useSHRAMConstants = CPA_TRUE;
+ } else {
+ useSHRAMConstants = CPA_FALSE;
+ }
+ }
+
+ /* hash only case or when chaining, cipher must be found in SHRAM table
+ * for
+ * optimised CD case */
+ if (pSession->laCmdId == ICP_QAT_FW_LA_CMD_AUTH ||
+ (useOptimisedContentDesc && useSHRAMConstants)) {
+ icp_qat_hw_auth_algo_t algorithm;
+ CpaBoolean nested;
+
+ if (pSession->digestVerify) {
+ return CPA_FALSE;
+ }
+
+ if ((!(useOptimisedContentDesc && useSHRAMConstants)) &&
+ (pSession->qatHashMode == ICP_QAT_HW_AUTH_MODE1)) {
+ /* we can only use the SHA1-mode1 in the SHRAM constants
+ * table when
+ * we are using the opimised content desc */
+ return CPA_FALSE;
+ }
+
+ LacSymQat_HashGetCfgData(pSession->pInstance,
+ pSession->qatHashMode,
+ pSession->hashMode,
+ pSession->hashAlgorithm,
+ &algorithm,
+ &nested);
+
+ /* Check if config data is available in table. */
+ LacSymQat_ConstantsGetAuthOffset(pSession->pInstance,
+ algorithm,
+ pSession->qatHashMode,
+ nested,
+ pHashOffset);
+ if (*pHashOffset > 0) {
+ useSHRAMConstants = CPA_TRUE;
+ } else {
+ useSHRAMConstants = CPA_FALSE;
+ }
+ }
+
+ return useSHRAMConstants;
+}
+
+CpaBoolean
+LacSymQat_UseOptimisedContentDesc(lac_session_desc_t *pSession)
+{
+ return CPA_FALSE;
+}
Index: sys/dev/qat/qat_api/common/crypto/sym/qat/lac_sym_qat_cipher.c
===================================================================
--- sys/dev/qat/qat_api/common/crypto/sym/qat/lac_sym_qat_cipher.c
+++ sys/dev/qat/qat_api/common/crypto/sym/qat/lac_sym_qat_cipher.c
@@ -29,6 +29,9 @@
#include "lac_sym_cipher_defs.h"
#include "icp_qat_hw.h"
#include "icp_qat_fw_la.h"
+#include "sal_hw_gen.h"
+
+#define LAC_UNUSED_POS_MASK 0x3
/*****************************************************************************
* Internal data
@@ -73,7 +76,7 @@
0,
0,
0,
- ICP_QAT_HW_CIPHER_ALGO_AES128, // ICP_QAT_HW_AES_128_XTS_KEY_SZ
+ ICP_QAT_HW_CIPHER_ALGO_AES128, /* ICP_QAT_HW_AES_128_XTS_KEY_SZ */
0,
0,
0,
@@ -105,7 +108,7 @@
0,
0,
0,
- ICP_QAT_HW_CIPHER_ALGO_AES256 // ICP_QAT_HW_AES_256_XTS_KEY_SZ
+ ICP_QAT_HW_CIPHER_ALGO_AES256 /* ICP_QAT_HW_AES_256_XTS_KEY_SZ */
};
/* LAC_CIPHER_IS_AES */
static const uint8_t key_size_aes[] = {
@@ -125,7 +128,7 @@
0,
0,
0,
- ICP_QAT_HW_CIPHER_ALGO_AES128, // ICP_QAT_HW_AES_128_KEY_SZ
+ ICP_QAT_HW_CIPHER_ALGO_AES128, /* ICP_QAT_HW_AES_128_KEY_SZ */
0,
0,
0,
@@ -133,7 +136,7 @@
0,
0,
0,
- ICP_QAT_HW_CIPHER_ALGO_AES192, // ICP_QAT_HW_AES_192_KEY_SZ
+ ICP_QAT_HW_CIPHER_ALGO_AES192, /* ICP_QAT_HW_AES_192_KEY_SZ */
0,
0,
0,
@@ -141,7 +144,7 @@
0,
0,
0,
- ICP_QAT_HW_CIPHER_ALGO_AES256 // ICP_QAT_HW_AES_256_KEY_SZ
+ ICP_QAT_HW_CIPHER_ALGO_AES256 /* ICP_QAT_HW_AES_256_KEY_SZ */
};
/* LAC_CIPHER_IS_AES_F8 */
static const uint8_t key_size_f8[] = {
@@ -177,23 +180,13 @@
0,
0,
0,
- ICP_QAT_HW_CIPHER_ALGO_AES128, // ICP_QAT_HW_AES_128_F8_KEY_SZ
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
+ ICP_QAT_HW_CIPHER_ALGO_AES128, /* ICP_QAT_HW_AES_128_F8_KEY_SZ */
0,
0,
0,
0,
0,
0,
- ICP_QAT_HW_CIPHER_ALGO_AES192, // ICP_QAT_HW_AES_192_F8_KEY_SZ
0,
0,
0,
@@ -203,16 +196,9 @@
0,
0,
0,
+ ICP_QAT_HW_CIPHER_ALGO_AES192, /* ICP_QAT_HW_AES_192_F8_KEY_SZ */
0,
0,
- 0,
- 0,
- 0,
- 0,
- ICP_QAT_HW_CIPHER_ALGO_AES256 // ICP_QAT_HW_AES_256_F8_KEY_SZ
-};
-/* LAC_CIPHER_IS_SM4 */
-static const uint8_t key_size_sm4[] = {
0,
0,
0,
@@ -226,10 +212,7 @@
0,
0,
0,
- 0,
- 0,
- 0,
- ICP_QAT_HW_CIPHER_ALGO_SM4 // ICP_QAT_HW_SM4_KEY_SZ
+ ICP_QAT_HW_CIPHER_ALGO_AES256 /* ICP_QAT_HW_AES_256_F8_KEY_SZ */
};
typedef struct _icp_qat_hw_cipher_info {
@@ -241,239 +224,237 @@
const uint8_t *pAlgByKeySize;
} icp_qat_hw_cipher_info;
-static const icp_qat_hw_cipher_info icp_qat_alg_info[] =
- {
- /* CPA_CY_SYM_CIPHER_NULL */
- {
- ICP_QAT_HW_CIPHER_ALGO_NULL,
- ICP_QAT_HW_CIPHER_ECB_MODE,
- { ICP_QAT_HW_CIPHER_NO_CONVERT, ICP_QAT_HW_CIPHER_NO_CONVERT },
- { ICP_QAT_HW_CIPHER_ENCRYPT, ICP_QAT_HW_CIPHER_DECRYPT },
- IS_KEY_DEP_NO,
- NULL,
- },
- /* CPA_CY_SYM_CIPHER_ARC4 */
- {
- ICP_QAT_HW_CIPHER_ALGO_ARC4,
- ICP_QAT_HW_CIPHER_ECB_MODE,
- { ICP_QAT_HW_CIPHER_NO_CONVERT, ICP_QAT_HW_CIPHER_NO_CONVERT },
- /* Streaming ciphers are a special case. Decrypt = encrypt */
- { ICP_QAT_HW_CIPHER_ENCRYPT, ICP_QAT_HW_CIPHER_ENCRYPT },
- IS_KEY_DEP_NO,
- NULL,
- },
- /* CPA_CY_SYM_CIPHER_AES_ECB */
- {
- ICP_QAT_HW_CIPHER_ALGO_AES128,
- ICP_QAT_HW_CIPHER_ECB_MODE,
- /* AES decrypt key needs to be reversed. Instead of reversing the key
- * at session registration, it is instead reversed on-the-fly by
- * setting the KEY_CONVERT bit here
- */
- { ICP_QAT_HW_CIPHER_NO_CONVERT, ICP_QAT_HW_CIPHER_KEY_CONVERT },
- { ICP_QAT_HW_CIPHER_ENCRYPT, ICP_QAT_HW_CIPHER_DECRYPT },
- IS_KEY_DEP_YES,
- key_size_aes,
- },
- /* CPA_CY_SYM_CIPHER_AES_CBC */
- {
- ICP_QAT_HW_CIPHER_ALGO_AES128,
- ICP_QAT_HW_CIPHER_CBC_MODE,
- /* AES decrypt key needs to be reversed. Instead of reversing the key
- * at session registration, it is instead reversed on-the-fly by
- * setting the KEY_CONVERT bit here
- */
- { ICP_QAT_HW_CIPHER_NO_CONVERT, ICP_QAT_HW_CIPHER_KEY_CONVERT },
- { ICP_QAT_HW_CIPHER_ENCRYPT, ICP_QAT_HW_CIPHER_DECRYPT },
- IS_KEY_DEP_YES,
- key_size_aes,
- },
- /* CPA_CY_SYM_CIPHER_AES_CTR */
- {
- ICP_QAT_HW_CIPHER_ALGO_AES128,
- ICP_QAT_HW_CIPHER_CTR_MODE,
- /* AES decrypt key needs to be reversed. Instead of reversing the key
- * at session registration, it is instead reversed on-the-fly by
- * setting the KEY_CONVERT bit here
- */
- { ICP_QAT_HW_CIPHER_NO_CONVERT, ICP_QAT_HW_CIPHER_NO_CONVERT },
- /* Streaming ciphers are a special case. Decrypt = encrypt
- * Overriding default values previously set for AES
- */
- { ICP_QAT_HW_CIPHER_ENCRYPT, ICP_QAT_HW_CIPHER_ENCRYPT },
- IS_KEY_DEP_YES,
- key_size_aes,
- },
- /* CPA_CY_SYM_CIPHER_AES_CCM */
- {
- ICP_QAT_HW_CIPHER_ALGO_AES128,
- ICP_QAT_HW_CIPHER_CTR_MODE,
- /* AES decrypt key needs to be reversed. Instead of reversing the key
- * at session registration, it is instead reversed on-the-fly by
- * setting the KEY_CONVERT bit here
- */
- { ICP_QAT_HW_CIPHER_NO_CONVERT, ICP_QAT_HW_CIPHER_NO_CONVERT },
- /* Streaming ciphers are a special case. Decrypt = encrypt
- * Overriding default values previously set for AES
- */
- { ICP_QAT_HW_CIPHER_ENCRYPT, ICP_QAT_HW_CIPHER_ENCRYPT },
- IS_KEY_DEP_YES,
- key_size_aes,
- },
- /* CPA_CY_SYM_CIPHER_AES_GCM */
- {
- ICP_QAT_HW_CIPHER_ALGO_AES128,
- ICP_QAT_HW_CIPHER_CTR_MODE,
- /* AES decrypt key needs to be reversed. Instead of reversing the key
- * at session registration, it is instead reversed on-the-fly by
- * setting the KEY_CONVERT bit here
- */
- { ICP_QAT_HW_CIPHER_NO_CONVERT, ICP_QAT_HW_CIPHER_NO_CONVERT },
- /* Streaming ciphers are a special case. Decrypt = encrypt
- * Overriding default values previously set for AES
- */
- { ICP_QAT_HW_CIPHER_ENCRYPT, ICP_QAT_HW_CIPHER_ENCRYPT },
- IS_KEY_DEP_YES,
- key_size_aes,
- },
- /* CPA_CY_SYM_CIPHER_DES_ECB */
- {
- ICP_QAT_HW_CIPHER_ALGO_DES,
- ICP_QAT_HW_CIPHER_ECB_MODE,
- { ICP_QAT_HW_CIPHER_NO_CONVERT, ICP_QAT_HW_CIPHER_NO_CONVERT },
- { ICP_QAT_HW_CIPHER_ENCRYPT, ICP_QAT_HW_CIPHER_DECRYPT },
- IS_KEY_DEP_NO,
- NULL,
- },
- /* CPA_CY_SYM_CIPHER_DES_CBC */
- {
- ICP_QAT_HW_CIPHER_ALGO_DES,
- ICP_QAT_HW_CIPHER_CBC_MODE,
- { ICP_QAT_HW_CIPHER_NO_CONVERT, ICP_QAT_HW_CIPHER_NO_CONVERT },
- { ICP_QAT_HW_CIPHER_ENCRYPT, ICP_QAT_HW_CIPHER_DECRYPT },
- IS_KEY_DEP_NO,
- NULL,
- },
- /* CPA_CY_SYM_CIPHER_3DES_ECB */
- {
- ICP_QAT_HW_CIPHER_ALGO_3DES,
- ICP_QAT_HW_CIPHER_ECB_MODE,
- { ICP_QAT_HW_CIPHER_NO_CONVERT, ICP_QAT_HW_CIPHER_NO_CONVERT },
- { ICP_QAT_HW_CIPHER_ENCRYPT, ICP_QAT_HW_CIPHER_DECRYPT },
- IS_KEY_DEP_NO,
- NULL,
- },
- /* CPA_CY_SYM_CIPHER_3DES_CBC */
- {
- ICP_QAT_HW_CIPHER_ALGO_3DES,
- ICP_QAT_HW_CIPHER_CBC_MODE,
- { ICP_QAT_HW_CIPHER_NO_CONVERT, ICP_QAT_HW_CIPHER_NO_CONVERT },
- { ICP_QAT_HW_CIPHER_ENCRYPT, ICP_QAT_HW_CIPHER_DECRYPT },
- IS_KEY_DEP_NO,
- NULL,
- },
- /* CPA_CY_SYM_CIPHER_3DES_CTR */
- {
- ICP_QAT_HW_CIPHER_ALGO_3DES,
- ICP_QAT_HW_CIPHER_CTR_MODE,
- { ICP_QAT_HW_CIPHER_NO_CONVERT, ICP_QAT_HW_CIPHER_NO_CONVERT },
- /* Streaming ciphers are a special case. Decrypt = encrypt
- * Overriding default values previously set for AES
- */
- { ICP_QAT_HW_CIPHER_ENCRYPT, ICP_QAT_HW_CIPHER_ENCRYPT },
- IS_KEY_DEP_NO,
- NULL,
- },
- /* CPA_CY_SYM_CIPHER_KASUMI_F8 */
- {
- ICP_QAT_HW_CIPHER_ALGO_KASUMI,
- ICP_QAT_HW_CIPHER_F8_MODE,
- { ICP_QAT_HW_CIPHER_NO_CONVERT, ICP_QAT_HW_CIPHER_NO_CONVERT },
- /* Streaming ciphers are a special case. Decrypt = encrypt */
- { ICP_QAT_HW_CIPHER_ENCRYPT, ICP_QAT_HW_CIPHER_ENCRYPT },
- IS_KEY_DEP_NO,
- NULL,
- },
- /* CPA_CY_SYM_CIPHER_SNOW3G_UEA2 */
- {
- /* The KEY_CONVERT bit has to be set for Snow_3G operation */
- ICP_QAT_HW_CIPHER_ALGO_SNOW_3G_UEA2,
- ICP_QAT_HW_CIPHER_ECB_MODE,
- { ICP_QAT_HW_CIPHER_KEY_CONVERT, ICP_QAT_HW_CIPHER_KEY_CONVERT },
- { ICP_QAT_HW_CIPHER_ENCRYPT, ICP_QAT_HW_CIPHER_DECRYPT },
- IS_KEY_DEP_NO,
- NULL,
- },
- /* CPA_CY_SYM_CIPHER_AES_F8 */
- {
- ICP_QAT_HW_CIPHER_ALGO_AES128,
- ICP_QAT_HW_CIPHER_F8_MODE,
- { ICP_QAT_HW_CIPHER_NO_CONVERT, ICP_QAT_HW_CIPHER_NO_CONVERT },
- /* Streaming ciphers are a special case. Decrypt = encrypt */
- { ICP_QAT_HW_CIPHER_ENCRYPT, ICP_QAT_HW_CIPHER_ENCRYPT },
- IS_KEY_DEP_YES,
- key_size_f8,
- },
- /* CPA_CY_SYM_CIPHER_AES_XTS */
- {
- ICP_QAT_HW_CIPHER_ALGO_AES128,
- ICP_QAT_HW_CIPHER_XTS_MODE,
- /* AES decrypt key needs to be reversed. Instead of reversing the key
- * at session registration, it is instead reversed on-the-fly by
- * setting the KEY_CONVERT bit here
- */
- { ICP_QAT_HW_CIPHER_NO_CONVERT, ICP_QAT_HW_CIPHER_KEY_CONVERT },
- { ICP_QAT_HW_CIPHER_ENCRYPT, ICP_QAT_HW_CIPHER_DECRYPT },
- IS_KEY_DEP_YES,
- key_size_xts,
- },
- /* CPA_CY_SYM_CIPHER_ZUC_EEA3 */
- {
- ICP_QAT_HW_CIPHER_ALGO_ZUC_3G_128_EEA3,
- ICP_QAT_HW_CIPHER_ECB_MODE,
- { ICP_QAT_HW_CIPHER_KEY_CONVERT, ICP_QAT_HW_CIPHER_KEY_CONVERT },
- { ICP_QAT_HW_CIPHER_ENCRYPT, ICP_QAT_HW_CIPHER_DECRYPT },
- IS_KEY_DEP_NO,
- NULL,
- },
- /* CPA_CY_SYM_CIPHER_CHACHA */
- {
- ICP_QAT_HW_CIPHER_ALGO_CHACHA20_POLY1305,
- ICP_QAT_HW_CIPHER_CTR_MODE,
- { ICP_QAT_HW_CIPHER_KEY_CONVERT, ICP_QAT_HW_CIPHER_KEY_CONVERT },
- { ICP_QAT_HW_CIPHER_ENCRYPT, ICP_QAT_HW_CIPHER_DECRYPT },
- IS_KEY_DEP_NO,
- NULL,
- },
- /* CPA_CY_SYM_CIPHER_SM4_ECB */
- {
- ICP_QAT_HW_CIPHER_ALGO_SM4,
- ICP_QAT_HW_CIPHER_ECB_MODE,
- { ICP_QAT_HW_CIPHER_NO_CONVERT, ICP_QAT_HW_CIPHER_KEY_CONVERT },
- { ICP_QAT_HW_CIPHER_ENCRYPT, ICP_QAT_HW_CIPHER_DECRYPT },
- IS_KEY_DEP_YES,
- key_size_sm4,
- },
- /* CPA_CY_SYM_CIPHER_SM4_CBC */
- {
- ICP_QAT_HW_CIPHER_ALGO_SM4,
- ICP_QAT_HW_CIPHER_CBC_MODE,
- { ICP_QAT_HW_CIPHER_NO_CONVERT, ICP_QAT_HW_CIPHER_KEY_CONVERT },
- { ICP_QAT_HW_CIPHER_ENCRYPT, ICP_QAT_HW_CIPHER_DECRYPT },
- IS_KEY_DEP_YES,
- key_size_sm4,
- },
- /* CPA_CY_SYM_CIPHER_SM4_CTR */
- {
- ICP_QAT_HW_CIPHER_ALGO_SM4,
- ICP_QAT_HW_CIPHER_CTR_MODE,
- { ICP_QAT_HW_CIPHER_NO_CONVERT, ICP_QAT_HW_CIPHER_NO_CONVERT },
- /* Streaming ciphers are a special case. Decrypt = encrypt */
- { ICP_QAT_HW_CIPHER_ENCRYPT, ICP_QAT_HW_CIPHER_ENCRYPT },
- IS_KEY_DEP_YES,
- key_size_sm4,
- },
- };
+static const icp_qat_hw_cipher_info icp_qat_alg_info[] = {
+ /* CPA_CY_SYM_CIPHER_NULL */
+ {
+ ICP_QAT_HW_CIPHER_ALGO_NULL,
+ ICP_QAT_HW_CIPHER_ECB_MODE,
+ { ICP_QAT_HW_CIPHER_NO_CONVERT, ICP_QAT_HW_CIPHER_NO_CONVERT },
+ { ICP_QAT_HW_CIPHER_ENCRYPT, ICP_QAT_HW_CIPHER_DECRYPT },
+ IS_KEY_DEP_NO,
+ NULL,
+ },
+ /* CPA_CY_SYM_CIPHER_ARC4 */
+ {
+ ICP_QAT_HW_CIPHER_ALGO_ARC4,
+ ICP_QAT_HW_CIPHER_ECB_MODE,
+ { ICP_QAT_HW_CIPHER_NO_CONVERT, ICP_QAT_HW_CIPHER_NO_CONVERT },
+ /* Streaming ciphers are a special case. Decrypt = encrypt */
+ { ICP_QAT_HW_CIPHER_ENCRYPT, ICP_QAT_HW_CIPHER_ENCRYPT },
+ IS_KEY_DEP_NO,
+ NULL,
+ },
+ /* CPA_CY_SYM_CIPHER_AES_ECB */
+ {
+ ICP_QAT_HW_CIPHER_ALGO_AES128,
+ ICP_QAT_HW_CIPHER_ECB_MODE,
+ /* AES decrypt key needs to be reversed. Instead of reversing the
+ * key at session registration, it is instead reversed on-the-fly by
+ * setting the KEY_CONVERT bit here
+ */
+ { ICP_QAT_HW_CIPHER_NO_CONVERT, ICP_QAT_HW_CIPHER_KEY_CONVERT },
+ { ICP_QAT_HW_CIPHER_ENCRYPT, ICP_QAT_HW_CIPHER_DECRYPT },
+ IS_KEY_DEP_YES,
+ key_size_aes,
+ },
+ /* CPA_CY_SYM_CIPHER_AES_CBC */
+ {
+ ICP_QAT_HW_CIPHER_ALGO_AES128,
+ ICP_QAT_HW_CIPHER_CBC_MODE,
+ /* AES decrypt key needs to be reversed. Instead of reversing the
+ * key at session registration, it is instead reversed on-the-fly by
+ * setting the KEY_CONVERT bit here
+ */
+ { ICP_QAT_HW_CIPHER_NO_CONVERT, ICP_QAT_HW_CIPHER_KEY_CONVERT },
+ { ICP_QAT_HW_CIPHER_ENCRYPT, ICP_QAT_HW_CIPHER_DECRYPT },
+ IS_KEY_DEP_YES,
+ key_size_aes,
+ },
+ /* CPA_CY_SYM_CIPHER_AES_CTR */
+ {
+ ICP_QAT_HW_CIPHER_ALGO_AES128,
+ ICP_QAT_HW_CIPHER_CTR_MODE,
+ /* AES decrypt key needs to be reversed. Instead of reversing the
+ * key at session registration, it is instead reversed on-the-fly by
+ * setting the KEY_CONVERT bit here
+ */
+ { ICP_QAT_HW_CIPHER_NO_CONVERT, ICP_QAT_HW_CIPHER_NO_CONVERT },
+ /* Streaming ciphers are a special case. Decrypt = encrypt
+ * Overriding default values previously set for AES
+ */
+ { ICP_QAT_HW_CIPHER_ENCRYPT, ICP_QAT_HW_CIPHER_ENCRYPT },
+ IS_KEY_DEP_YES,
+ key_size_aes,
+ },
+ /* CPA_CY_SYM_CIPHER_AES_CCM */
+ {
+ ICP_QAT_HW_CIPHER_ALGO_AES128,
+ ICP_QAT_HW_CIPHER_CTR_MODE,
+ /* AES decrypt key needs to be reversed. Instead of reversing the
+ * key at session registration, it is instead reversed on-the-fly by
+ * setting the KEY_CONVERT bit here
+ */
+ { ICP_QAT_HW_CIPHER_NO_CONVERT, ICP_QAT_HW_CIPHER_NO_CONVERT },
+ /* Streaming ciphers are a special case. Decrypt = encrypt
+ * Overriding default values previously set for AES
+ */
+ { ICP_QAT_HW_CIPHER_ENCRYPT, ICP_QAT_HW_CIPHER_ENCRYPT },
+ IS_KEY_DEP_YES,
+ key_size_aes,
+ },
+ /* CPA_CY_SYM_CIPHER_AES_GCM */
+ {
+ ICP_QAT_HW_CIPHER_ALGO_AES128,
+ ICP_QAT_HW_CIPHER_CTR_MODE,
+ /* AES decrypt key needs to be reversed. Instead of reversing the
+ * key at session registration, it is instead reversed on-the-fly by
+ * setting the KEY_CONVERT bit here
+ */
+ { ICP_QAT_HW_CIPHER_NO_CONVERT, ICP_QAT_HW_CIPHER_NO_CONVERT },
+ /* Streaming ciphers are a special case. Decrypt = encrypt
+ * Overriding default values previously set for AES
+ */
+ { ICP_QAT_HW_CIPHER_ENCRYPT, ICP_QAT_HW_CIPHER_ENCRYPT },
+ IS_KEY_DEP_YES,
+ key_size_aes,
+ },
+ /* CPA_CY_SYM_CIPHER_DES_ECB */
+ {
+ ICP_QAT_HW_CIPHER_ALGO_DES,
+ ICP_QAT_HW_CIPHER_ECB_MODE,
+ { ICP_QAT_HW_CIPHER_NO_CONVERT, ICP_QAT_HW_CIPHER_NO_CONVERT },
+ { ICP_QAT_HW_CIPHER_ENCRYPT, ICP_QAT_HW_CIPHER_DECRYPT },
+ IS_KEY_DEP_NO,
+ NULL,
+ },
+ /* CPA_CY_SYM_CIPHER_DES_CBC */
+ {
+ ICP_QAT_HW_CIPHER_ALGO_DES,
+ ICP_QAT_HW_CIPHER_CBC_MODE,
+ { ICP_QAT_HW_CIPHER_NO_CONVERT, ICP_QAT_HW_CIPHER_NO_CONVERT },
+ { ICP_QAT_HW_CIPHER_ENCRYPT, ICP_QAT_HW_CIPHER_DECRYPT },
+ IS_KEY_DEP_NO,
+ NULL,
+ },
+ /* CPA_CY_SYM_CIPHER_3DES_ECB */
+ {
+ ICP_QAT_HW_CIPHER_ALGO_3DES,
+ ICP_QAT_HW_CIPHER_ECB_MODE,
+ { ICP_QAT_HW_CIPHER_NO_CONVERT, ICP_QAT_HW_CIPHER_NO_CONVERT },
+ { ICP_QAT_HW_CIPHER_ENCRYPT, ICP_QAT_HW_CIPHER_DECRYPT },
+ IS_KEY_DEP_NO,
+ NULL,
+ },
+ /* CPA_CY_SYM_CIPHER_3DES_CBC */
+ {
+ ICP_QAT_HW_CIPHER_ALGO_3DES,
+ ICP_QAT_HW_CIPHER_CBC_MODE,
+ { ICP_QAT_HW_CIPHER_NO_CONVERT, ICP_QAT_HW_CIPHER_NO_CONVERT },
+ { ICP_QAT_HW_CIPHER_ENCRYPT, ICP_QAT_HW_CIPHER_DECRYPT },
+ IS_KEY_DEP_NO,
+ NULL,
+ },
+ /* CPA_CY_SYM_CIPHER_3DES_CTR */
+ {
+ ICP_QAT_HW_CIPHER_ALGO_3DES,
+ ICP_QAT_HW_CIPHER_CTR_MODE,
+ { ICP_QAT_HW_CIPHER_NO_CONVERT, ICP_QAT_HW_CIPHER_NO_CONVERT },
+ /* Streaming ciphers are a special case. Decrypt = encrypt
+ * Overriding default values previously set for AES
+ */
+ { ICP_QAT_HW_CIPHER_ENCRYPT, ICP_QAT_HW_CIPHER_ENCRYPT },
+ IS_KEY_DEP_NO,
+ NULL,
+ },
+ /* CPA_CY_SYM_CIPHER_KASUMI_F8 */
+ {
+ ICP_QAT_HW_CIPHER_ALGO_KASUMI,
+ ICP_QAT_HW_CIPHER_F8_MODE,
+ { ICP_QAT_HW_CIPHER_NO_CONVERT, ICP_QAT_HW_CIPHER_NO_CONVERT },
+ /* Streaming ciphers are a special case. Decrypt = encrypt */
+ { ICP_QAT_HW_CIPHER_ENCRYPT, ICP_QAT_HW_CIPHER_ENCRYPT },
+ IS_KEY_DEP_NO,
+ NULL,
+ },
+ /* CPA_CY_SYM_CIPHER_SNOW3G_UEA2 */
+ {
+ /* The KEY_CONVERT bit has to be set for Snow_3G operation */
+ ICP_QAT_HW_CIPHER_ALGO_SNOW_3G_UEA2,
+ ICP_QAT_HW_CIPHER_ECB_MODE,
+ { ICP_QAT_HW_CIPHER_KEY_CONVERT, ICP_QAT_HW_CIPHER_KEY_CONVERT },
+ { ICP_QAT_HW_CIPHER_ENCRYPT, ICP_QAT_HW_CIPHER_DECRYPT },
+ IS_KEY_DEP_NO,
+ NULL,
+ },
+ /* CPA_CY_SYM_CIPHER_AES_F8 */
+ {
+ ICP_QAT_HW_CIPHER_ALGO_AES128,
+ ICP_QAT_HW_CIPHER_F8_MODE,
+ { ICP_QAT_HW_CIPHER_NO_CONVERT, ICP_QAT_HW_CIPHER_NO_CONVERT },
+ /* Streaming ciphers are a special case. Decrypt = encrypt */
+ { ICP_QAT_HW_CIPHER_ENCRYPT, ICP_QAT_HW_CIPHER_ENCRYPT },
+ IS_KEY_DEP_YES,
+ key_size_f8,
+ },
+ /* CPA_CY_SYM_CIPHER_AES_XTS */
+ {
+ ICP_QAT_HW_CIPHER_ALGO_AES128,
+ ICP_QAT_HW_CIPHER_XTS_MODE,
+ /* AES decrypt key needs to be reversed. Instead of reversing the
+ * key at session registration, it is instead reversed on-the-fly by
+ * setting the KEY_CONVERT bit here
+ */
+ { ICP_QAT_HW_CIPHER_NO_CONVERT, ICP_QAT_HW_CIPHER_KEY_CONVERT },
+ { ICP_QAT_HW_CIPHER_ENCRYPT, ICP_QAT_HW_CIPHER_DECRYPT },
+ IS_KEY_DEP_YES,
+ key_size_xts,
+ },
+ /* CPA_CY_SYM_CIPHER_ZUC_EEA3 */
+ {
+ ICP_QAT_HW_CIPHER_ALGO_ZUC_3G_128_EEA3,
+ ICP_QAT_HW_CIPHER_ECB_MODE,
+ { ICP_QAT_HW_CIPHER_KEY_CONVERT, ICP_QAT_HW_CIPHER_KEY_CONVERT },
+ { ICP_QAT_HW_CIPHER_ENCRYPT, ICP_QAT_HW_CIPHER_DECRYPT },
+ IS_KEY_DEP_NO,
+ NULL,
+ },
+ /* CPA_CY_SYM_CIPHER_CHACHA */
+ {
+ ICP_QAT_HW_CIPHER_ALGO_CHACHA20_POLY1305,
+ ICP_QAT_HW_CIPHER_CTR_MODE,
+ { ICP_QAT_HW_CIPHER_NO_CONVERT, ICP_QAT_HW_CIPHER_NO_CONVERT },
+ { ICP_QAT_HW_CIPHER_ENCRYPT, ICP_QAT_HW_CIPHER_DECRYPT },
+ IS_KEY_DEP_NO,
+ NULL,
+ },
+ /* CPA_CY_SYM_CIPHER_SM4_ECB */
+ {
+ ICP_QAT_HW_CIPHER_ALGO_SM4,
+ ICP_QAT_HW_CIPHER_ECB_MODE,
+ { ICP_QAT_HW_CIPHER_NO_CONVERT, ICP_QAT_HW_CIPHER_KEY_CONVERT },
+ { ICP_QAT_HW_CIPHER_ENCRYPT, ICP_QAT_HW_CIPHER_DECRYPT },
+ IS_KEY_DEP_NO,
+ NULL,
+ },
+ /* CPA_CY_SYM_CIPHER_SM4_CBC */
+ {
+ ICP_QAT_HW_CIPHER_ALGO_SM4,
+ ICP_QAT_HW_CIPHER_CBC_MODE,
+ { ICP_QAT_HW_CIPHER_NO_CONVERT, ICP_QAT_HW_CIPHER_KEY_CONVERT },
+ { ICP_QAT_HW_CIPHER_ENCRYPT, ICP_QAT_HW_CIPHER_DECRYPT },
+ IS_KEY_DEP_NO,
+ NULL,
+ },
+ /* CPA_CY_SYM_CIPHER_SM4_CTR */
+ {
+ ICP_QAT_HW_CIPHER_ALGO_SM4,
+ ICP_QAT_HW_CIPHER_CTR_MODE,
+ { ICP_QAT_HW_CIPHER_NO_CONVERT, ICP_QAT_HW_CIPHER_NO_CONVERT },
+ { ICP_QAT_HW_CIPHER_ENCRYPT, ICP_QAT_HW_CIPHER_ENCRYPT },
+ IS_KEY_DEP_NO,
+ NULL,
+ },
+};
/*****************************************************************************
* Internal functions
@@ -483,6 +464,7 @@
LacSymQat_CipherCtrlBlockWrite(icp_qat_la_bulk_req_ftr_t *pMsg,
Cpa32U cipherAlgorithm,
Cpa32U targetKeyLenInBytes,
+ Cpa32U sliceType,
icp_qat_fw_slice_t nextSlice,
Cpa8U cipherCfgOffsetInQuadWord)
{
@@ -492,33 +474,52 @@
/* state_padding_sz is nonzero for f8 mode only */
cd_ctrl->cipher_padding_sz = 0;
+ /* Special handling of AES 192 key for UCS slice.
+ UCS requires it to have 32 bytes - set is as targetKeyLen
+ in this case, and add padding. It makes no sense
+ to force applications to provide such key length for couple reasons:
+ 1. It won't be possible to distinguish between AES 192 and 256 based
+ on key lenght only
+ 2. Only some modes of AES will use UCS slice, then application will
+ have to know which ones */
+ if (ICP_QAT_FW_LA_USE_UCS_SLICE_TYPE == sliceType &&
+ ICP_QAT_HW_AES_192_KEY_SZ == targetKeyLenInBytes) {
+ targetKeyLenInBytes = ICP_QAT_HW_UCS_AES_192_KEY_SZ;
+ }
+
+ switch (cipherAlgorithm) {
/* Base Key is not passed down to QAT in the case of ARC4 or NULL */
- if (LAC_CIPHER_IS_ARC4(cipherAlgorithm) ||
- LAC_CIPHER_IS_NULL(cipherAlgorithm)) {
+ case CPA_CY_SYM_CIPHER_ARC4:
+ case CPA_CY_SYM_CIPHER_NULL:
cd_ctrl->cipher_key_sz = 0;
- } else if (LAC_CIPHER_IS_KASUMI(cipherAlgorithm)) {
+ break;
+ case CPA_CY_SYM_CIPHER_KASUMI_F8:
cd_ctrl->cipher_key_sz =
LAC_BYTES_TO_QUADWORDS(ICP_QAT_HW_KASUMI_F8_KEY_SZ);
cd_ctrl->cipher_padding_sz =
ICP_QAT_HW_MODE_F8_NUM_REG_TO_CLEAR;
- } else if (LAC_CIPHER_IS_SNOW3G_UEA2(cipherAlgorithm)) {
- /* For Snow3G UEA2 content descriptor key size is
- key size plus iv size */
+ break;
+ /* For Snow3G UEA2 content descriptor key size is
+ key size plus iv size */
+ case CPA_CY_SYM_CIPHER_SNOW3G_UEA2:
cd_ctrl->cipher_key_sz =
LAC_BYTES_TO_QUADWORDS(ICP_QAT_HW_SNOW_3G_UEA2_KEY_SZ +
ICP_QAT_HW_SNOW_3G_UEA2_IV_SZ);
- } else if (LAC_CIPHER_IS_AES_F8(cipherAlgorithm)) {
+ break;
+ case CPA_CY_SYM_CIPHER_AES_F8:
cd_ctrl->cipher_key_sz =
LAC_BYTES_TO_QUADWORDS(targetKeyLenInBytes);
cd_ctrl->cipher_padding_sz =
- 2 * ICP_QAT_HW_MODE_F8_NUM_REG_TO_CLEAR;
- } else if (LAC_CIPHER_IS_ZUC_EEA3(cipherAlgorithm)) {
- /* For ZUC EEA3 content descriptor key size is
- key size plus iv size */
+ (2 * ICP_QAT_HW_MODE_F8_NUM_REG_TO_CLEAR);
+ break;
+ /* For ZUC EEA3 content descriptor key size is
+ key size plus iv size */
+ case CPA_CY_SYM_CIPHER_ZUC_EEA3:
cd_ctrl->cipher_key_sz =
LAC_BYTES_TO_QUADWORDS(ICP_QAT_HW_ZUC_3G_EEA3_KEY_SZ +
ICP_QAT_HW_ZUC_3G_EEA3_IV_SZ);
- } else {
+ break;
+ default:
cd_ctrl->cipher_key_sz =
LAC_BYTES_TO_QUADWORDS(targetKeyLenInBytes);
}
@@ -539,6 +540,8 @@
icp_qat_hw_cipher_dir_t *pDir,
icp_qat_hw_cipher_convert_t *pKey_convert)
{
+ sal_crypto_service_t *pService =
+ (sal_crypto_service_t *)pSession->pInstance;
CpaCySymCipherAlgorithm cipherAlgorithm = 0;
icp_qat_hw_cipher_dir_t cipherDirection = 0;
@@ -553,8 +556,8 @@
cipherAlgorithm = pSession->cipherAlgorithm - 1;
cipherDirection =
pSession->cipherDirection == CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT ?
- ICP_QAT_HW_CIPHER_ENCRYPT :
- ICP_QAT_HW_CIPHER_DECRYPT;
+ ICP_QAT_HW_CIPHER_ENCRYPT :
+ ICP_QAT_HW_CIPHER_DECRYPT;
*pAlgorithm = icp_qat_alg_info[cipherAlgorithm].algorithm;
*pMode = icp_qat_alg_info[cipherAlgorithm].mode;
@@ -566,23 +569,22 @@
*pAlgorithm = icp_qat_alg_info[cipherAlgorithm]
.pAlgByKeySize[pSession->cipherKeyLenInBytes];
}
- /* Set the mode */
- if (LAC_CIPHER_IS_CTR_MODE(pSession->cipherAlgorithm)) {
- *pMode = ICP_QAT_HW_CIPHER_CTR_MODE;
- *pKey_convert = ICP_QAT_HW_CIPHER_NO_CONVERT;
- /* CCP and AES_GCM single pass, despite being limited to
- * CTR/AEAD mode,
- * support both Encrypt/Decrypt modes - this is because of the
- * differences in the hash computation/verification paths in
- * encrypt/decrypt modes respectively.
- * By default CCP is set as CTR Mode.Set AEAD Mode for AES_GCM.
- */
- if (pSession->isSinglePass) {
- if (LAC_CIPHER_IS_GCM(pSession->cipherAlgorithm))
- *pMode = ICP_QAT_HW_CIPHER_AEAD_MODE;
- if (cipherDirection == ICP_QAT_HW_CIPHER_DECRYPT)
- *pDir = ICP_QAT_HW_CIPHER_DECRYPT;
- }
+
+ /* CCP and AES_GCM single pass, despite being limited to CTR/AEAD mode,
+ * support both Encrypt/Decrypt modes - this is because of the
+ * differences in the hash computation/verification paths in
+ * encrypt/decrypt modes respectively.
+ * By default CCP is set as CTR Mode.Set AEAD Mode for AES_GCM.
+ */
+ if (SPC == pSession->singlePassState) {
+ if (LAC_CIPHER_IS_GCM(pSession->cipherAlgorithm))
+ *pMode = ICP_QAT_HW_CIPHER_AEAD_MODE;
+ else if (isCyGen4x(pService) &&
+ LAC_CIPHER_IS_CCM(pSession->cipherAlgorithm))
+ *pMode = ICP_QAT_HW_CIPHER_CCM_MODE;
+
+ if (cipherDirection == ICP_QAT_HW_CIPHER_DECRYPT)
+ *pDir = ICP_QAT_HW_CIPHER_DECRYPT;
}
}
@@ -597,6 +599,10 @@
icp_qat_hw_cipher_convert_t key_convert;
icp_qat_hw_cipher_config_t *pCipherConfig =
(icp_qat_hw_cipher_config_t *)pCipherHwBlock;
+ icp_qat_hw_ucs_cipher_config_t *pUCSCipherConfig =
+ (icp_qat_hw_ucs_cipher_config_t *)pCipherHwBlock;
+
+ Cpa32U val, reserved;
Cpa32U aed_hash_cmp_length = 0;
*pSizeInBytes = 0;
@@ -605,24 +611,38 @@
pSession, &algorithm, &mode, &dir, &key_convert);
/* Build the cipher config into the hardware setup block */
- if (pSession->isSinglePass) {
+ if (SPC == pSession->singlePassState) {
aed_hash_cmp_length = pSession->hashResultSize;
- pCipherConfig->reserved = ICP_QAT_HW_CIPHER_CONFIG_BUILD_UPPER(
+ reserved = ICP_QAT_HW_CIPHER_CONFIG_BUILD_UPPER(
pSession->aadLenInBytes);
} else {
- pCipherConfig->reserved = 0;
+ reserved = 0;
}
- pCipherConfig->val = ICP_QAT_HW_CIPHER_CONFIG_BUILD(
+ val = ICP_QAT_HW_CIPHER_CONFIG_BUILD(
mode, algorithm, key_convert, dir, aed_hash_cmp_length);
- *pSizeInBytes = sizeof(icp_qat_hw_cipher_config_t);
+ /* UCS slice has 128-bit configuration register.
+ Leacy cipher slice has 64-bit config register */
+ if (ICP_QAT_FW_LA_USE_UCS_SLICE_TYPE == pSession->cipherSliceType) {
+ pUCSCipherConfig->val = val;
+ pUCSCipherConfig->reserved[0] = reserved;
+ pUCSCipherConfig->reserved[1] = 0;
+ pUCSCipherConfig->reserved[2] = 0;
+ *pSizeInBytes = sizeof(icp_qat_hw_ucs_cipher_config_t);
+ } else {
+ pCipherConfig->val = val;
+ pCipherConfig->reserved = reserved;
+ *pSizeInBytes = sizeof(icp_qat_hw_cipher_config_t);
+ }
}
void
LacSymQat_CipherHwBlockPopulateKeySetup(
+ lac_session_desc_t *pSessionDesc,
const CpaCySymCipherSetupData *pCipherSetupData,
Cpa32U targetKeyLenInBytes,
+ Cpa32U sliceType,
const void *pCipherHwBlock,
Cpa32U *pSizeInBytes)
{
@@ -635,6 +655,20 @@
* Arc4 and Null cipher */
if (!(LAC_CIPHER_IS_ARC4(pCipherSetupData->cipherAlgorithm) ||
LAC_CIPHER_IS_NULL(pCipherSetupData->cipherAlgorithm))) {
+ /* Special handling of AES 192 key for UCS slice.
+ UCS requires it to have 32 bytes - set is as targetKeyLen
+ in this case, and add padding. It makes no sense
+ to force applications to provide such key length for couple
+ reasons:
+ 1. It won't be possible to distinguish between AES 192 and
+ 256 based on key lenght only
+ 2. Only some modes of AES will use UCS slice, then
+ application will have to know which ones */
+ if (ICP_QAT_FW_LA_USE_UCS_SLICE_TYPE == sliceType &&
+ ICP_QAT_HW_AES_192_KEY_SZ == targetKeyLenInBytes) {
+ targetKeyLenInBytes = ICP_QAT_HW_UCS_AES_192_KEY_SZ;
+ }
+
/* Set the Cipher key field in the cipher block */
memcpy(pCipherKey,
pCipherSetupData->pCipherKey,
@@ -646,9 +680,10 @@
}
*pSizeInBytes += targetKeyLenInBytes;
- /* For Kasumi in F8 mode Cipher Key is concatenated with
- * Cipher Key XOR-ed with Key Modifier (CK||CK^KM) */
- if (LAC_CIPHER_IS_KASUMI(pCipherSetupData->cipherAlgorithm)) {
+ switch (pCipherSetupData->cipherAlgorithm) {
+ /* For Kasumi in F8 mode Cipher Key is concatenated with
+ * Cipher Key XOR-ed with Key Modifier (CK||CK^KM) */
+ case CPA_CY_SYM_CIPHER_KASUMI_F8: {
Cpa32U wordIndex = 0;
Cpa32U *pu32CipherKey =
(Cpa32U *)pCipherSetupData->pCipherKey;
@@ -672,11 +707,10 @@
LAC_OS_BZERO((Cpa8U *)pTempKey + targetKeyLenInBytes,
LAC_QUADWORDS_TO_BYTES(
ICP_QAT_HW_MODE_F8_NUM_REG_TO_CLEAR));
- }
- /* For AES in F8 mode Cipher Key is concatenated with
- * Cipher Key XOR-ed with Key Mask (CK||CK^KM) */
- else if (LAC_CIPHER_IS_AES_F8(
- pCipherSetupData->cipherAlgorithm)) {
+ } break;
+ /* For AES in F8 mode Cipher Key is concatenated with
+ * Cipher Key XOR-ed with Key Mask (CK||CK^KM) */
+ case CPA_CY_SYM_CIPHER_AES_F8: {
Cpa32U index = 0;
Cpa8U *pTempKey =
pCipherKey + (targetKeyLenInBytes / 2);
@@ -690,20 +724,59 @@
/* also add padding for AES F8 */
*pSizeInBytes += 2 * targetKeyLenInBytes;
LAC_OS_BZERO(pTempKey, 2 * targetKeyLenInBytes);
- } else if (LAC_CIPHER_IS_SNOW3G_UEA2(
- pCipherSetupData->cipherAlgorithm)) {
+ } break;
+ case CPA_CY_SYM_CIPHER_SNOW3G_UEA2: {
/* For Snow3G zero area after the key for FW */
LAC_OS_BZERO(pCipherKey + targetKeyLenInBytes,
ICP_QAT_HW_SNOW_3G_UEA2_IV_SZ);
*pSizeInBytes += ICP_QAT_HW_SNOW_3G_UEA2_IV_SZ;
- } else if (LAC_CIPHER_IS_ZUC_EEA3(
- pCipherSetupData->cipherAlgorithm)) {
+ } break;
+ case CPA_CY_SYM_CIPHER_ZUC_EEA3: {
/* For ZUC zero area after the key for FW */
LAC_OS_BZERO(pCipherKey + targetKeyLenInBytes,
ICP_QAT_HW_ZUC_3G_EEA3_IV_SZ);
*pSizeInBytes += ICP_QAT_HW_ZUC_3G_EEA3_IV_SZ;
+ } break;
+ case CPA_CY_SYM_CIPHER_AES_XTS: {
+ /* For AES in XTS mode Cipher Key is concatenated with
+ * second Cipher Key which is used for tweak calculation
+ * (CK1||CK2). For decryption Cipher Key needs to be
+ * converted to reverse key.*/
+ if (ICP_QAT_FW_LA_USE_UCS_SLICE_TYPE == sliceType) {
+ Cpa32U key_len =
+ pCipherSetupData->cipherKeyLenInBytes / 2;
+ memcpy(pSessionDesc->cipherAesXtsKey1Forward,
+ pCipherSetupData->pCipherKey,
+ key_len);
+
+ qatUtilsAESKeyExpansionForward(
+ pSessionDesc->cipherAesXtsKey1Forward,
+ key_len,
+ (uint32_t *)
+ pSessionDesc->cipherAesXtsKey1Reverse);
+
+ memcpy(pSessionDesc->cipherAesXtsKey2,
+ pCipherSetupData->pCipherKey + key_len,
+ key_len);
+
+ if (CPA_CY_SYM_CIPHER_DIRECTION_DECRYPT ==
+ pCipherSetupData->cipherDirection) {
+ memcpy(pCipherKey,
+ pSessionDesc
+ ->cipherAesXtsKey1Reverse,
+ key_len);
+ } else {
+ memcpy(pCipherKey,
+ pSessionDesc
+ ->cipherAesXtsKey1Forward,
+ key_len);
+ }
+ }
+ } break;
+ default:
+ break;
}
}
}
@@ -715,56 +788,94 @@
Cpa8U
LacSymQat_CipherBlockSizeBytesGet(CpaCySymCipherAlgorithm cipherAlgorithm)
{
- if (LAC_CIPHER_IS_ARC4(cipherAlgorithm)) {
- return LAC_CIPHER_ARC4_BLOCK_LEN_BYTES;
- } else if (LAC_CIPHER_IS_AES(cipherAlgorithm) ||
- LAC_CIPHER_IS_AES_F8(cipherAlgorithm)) {
- return ICP_QAT_HW_AES_BLK_SZ;
- } else if (LAC_CIPHER_IS_DES(cipherAlgorithm)) {
- return ICP_QAT_HW_DES_BLK_SZ;
- } else if (LAC_CIPHER_IS_TRIPLE_DES(cipherAlgorithm)) {
- return ICP_QAT_HW_3DES_BLK_SZ;
- } else if (LAC_CIPHER_IS_KASUMI(cipherAlgorithm)) {
- return ICP_QAT_HW_KASUMI_BLK_SZ;
- } else if (LAC_CIPHER_IS_SNOW3G_UEA2(cipherAlgorithm)) {
- return ICP_QAT_HW_SNOW_3G_BLK_SZ;
- } else if (LAC_CIPHER_IS_ZUC_EEA3(cipherAlgorithm)) {
- return ICP_QAT_HW_ZUC_3G_BLK_SZ;
- } else if (LAC_CIPHER_IS_NULL(cipherAlgorithm)) {
- return LAC_CIPHER_NULL_BLOCK_LEN_BYTES;
- } else if (LAC_CIPHER_IS_CHACHA(cipherAlgorithm)) {
- return ICP_QAT_HW_CHACHAPOLY_BLK_SZ;
- } else if (LAC_CIPHER_IS_SM4(cipherAlgorithm)) {
- return ICP_QAT_HW_SM4_BLK_SZ;
- } else {
- QAT_UTILS_LOG("Algorithm not supported in Cipher\n");
- return 0;
+ Cpa8U blockSize = 0;
+ switch (cipherAlgorithm) {
+ case CPA_CY_SYM_CIPHER_ARC4:
+ blockSize = LAC_CIPHER_ARC4_BLOCK_LEN_BYTES;
+ break;
+ /* Handle AES or AES_F8 */
+ case CPA_CY_SYM_CIPHER_AES_ECB:
+ case CPA_CY_SYM_CIPHER_AES_CBC:
+ case CPA_CY_SYM_CIPHER_AES_CTR:
+ case CPA_CY_SYM_CIPHER_AES_CCM:
+ case CPA_CY_SYM_CIPHER_AES_GCM:
+ case CPA_CY_SYM_CIPHER_AES_XTS:
+ case CPA_CY_SYM_CIPHER_AES_F8:
+ blockSize = ICP_QAT_HW_AES_BLK_SZ;
+ break;
+ /* Handle DES */
+ case CPA_CY_SYM_CIPHER_DES_ECB:
+ case CPA_CY_SYM_CIPHER_DES_CBC:
+ blockSize = ICP_QAT_HW_DES_BLK_SZ;
+ break;
+ /* Handle TRIPLE DES */
+ case CPA_CY_SYM_CIPHER_3DES_ECB:
+ case CPA_CY_SYM_CIPHER_3DES_CBC:
+ case CPA_CY_SYM_CIPHER_3DES_CTR:
+ blockSize = ICP_QAT_HW_3DES_BLK_SZ;
+ break;
+ case CPA_CY_SYM_CIPHER_KASUMI_F8:
+ blockSize = ICP_QAT_HW_KASUMI_BLK_SZ;
+ break;
+ case CPA_CY_SYM_CIPHER_SNOW3G_UEA2:
+ blockSize = ICP_QAT_HW_SNOW_3G_BLK_SZ;
+ break;
+ case CPA_CY_SYM_CIPHER_ZUC_EEA3:
+ blockSize = ICP_QAT_HW_ZUC_3G_BLK_SZ;
+ break;
+ case CPA_CY_SYM_CIPHER_NULL:
+ blockSize = LAC_CIPHER_NULL_BLOCK_LEN_BYTES;
+ break;
+ case CPA_CY_SYM_CIPHER_CHACHA:
+ blockSize = ICP_QAT_HW_CHACHAPOLY_BLK_SZ;
+ break;
+ case CPA_CY_SYM_CIPHER_SM4_ECB:
+ case CPA_CY_SYM_CIPHER_SM4_CBC:
+ case CPA_CY_SYM_CIPHER_SM4_CTR:
+ blockSize = ICP_QAT_HW_SM4_BLK_SZ;
+ break;
+ default:
+ QAT_UTILS_LOG("Algorithm not supported in Cipher");
}
+ return blockSize;
}
Cpa32U
LacSymQat_CipherIvSizeBytesGet(CpaCySymCipherAlgorithm cipherAlgorithm)
{
- if (CPA_CY_SYM_CIPHER_ARC4 == cipherAlgorithm) {
- return LAC_CIPHER_ARC4_STATE_LEN_BYTES;
- } else if (LAC_CIPHER_IS_KASUMI(cipherAlgorithm)) {
- return ICP_QAT_HW_KASUMI_BLK_SZ;
- } else if (LAC_CIPHER_IS_SNOW3G_UEA2(cipherAlgorithm)) {
- return ICP_QAT_HW_SNOW_3G_UEA2_IV_SZ;
- } else if (LAC_CIPHER_IS_ZUC_EEA3(cipherAlgorithm)) {
- return ICP_QAT_HW_ZUC_3G_EEA3_IV_SZ;
- } else if (LAC_CIPHER_IS_CHACHA(cipherAlgorithm)) {
- return ICP_QAT_HW_CHACHAPOLY_IV_SZ;
- } else if (LAC_CIPHER_IS_ECB_MODE(cipherAlgorithm)) {
- return 0;
- } else {
- return (Cpa32U)LacSymQat_CipherBlockSizeBytesGet(
- cipherAlgorithm);
+ Cpa32U ivSize = 0;
+ switch (cipherAlgorithm) {
+ case CPA_CY_SYM_CIPHER_ARC4:
+ ivSize = LAC_CIPHER_ARC4_STATE_LEN_BYTES;
+ break;
+ case CPA_CY_SYM_CIPHER_KASUMI_F8:
+ ivSize = ICP_QAT_HW_KASUMI_BLK_SZ;
+ break;
+ case CPA_CY_SYM_CIPHER_SNOW3G_UEA2:
+ ivSize = ICP_QAT_HW_SNOW_3G_UEA2_IV_SZ;
+ break;
+ case CPA_CY_SYM_CIPHER_ZUC_EEA3:
+ ivSize = ICP_QAT_HW_ZUC_3G_EEA3_IV_SZ;
+ break;
+ case CPA_CY_SYM_CIPHER_CHACHA:
+ ivSize = ICP_QAT_HW_CHACHAPOLY_IV_SZ;
+ break;
+ case CPA_CY_SYM_CIPHER_AES_ECB:
+ case CPA_CY_SYM_CIPHER_DES_ECB:
+ case CPA_CY_SYM_CIPHER_3DES_ECB:
+ case CPA_CY_SYM_CIPHER_SM4_ECB:
+ case CPA_CY_SYM_CIPHER_NULL:
+ /* for all ECB Mode IV size is 0 */
+ break;
+ default:
+ ivSize = LacSymQat_CipherBlockSizeBytesGet(cipherAlgorithm);
}
+ return ivSize;
}
inline CpaStatus
-LacSymQat_CipherRequestParamsPopulate(icp_qat_fw_la_bulk_req_t *pReq,
+LacSymQat_CipherRequestParamsPopulate(lac_session_desc_t *pSessionDesc,
+ icp_qat_fw_la_bulk_req_t *pReq,
Cpa32U cipherOffsetInBytes,
Cpa32U cipherLenInBytes,
Cpa64U ivBufferPhysAddr,
@@ -773,6 +884,8 @@
icp_qat_fw_la_cipher_req_params_t *pCipherReqParams;
icp_qat_fw_cipher_cd_ctrl_hdr_t *pCipherCdCtrlHdr;
icp_qat_fw_serv_specif_flags *pCipherSpecificFlags;
+ Cpa32U usedBufSize = 0;
+ Cpa32U totalBufSize = 0;
pCipherReqParams = (icp_qat_fw_la_cipher_req_params_t
*)((Cpa8U *)&(pReq->serv_specif_rqpars) +
@@ -805,31 +918,40 @@
} else {
/* Populate the field with the contents of the buffer,
* zero field first as data may be smaller than the field */
- memset(pCipherReqParams->u.cipher_IV_array,
- 0,
- LAC_LONGWORDS_TO_BYTES(ICP_QAT_FW_NUM_LONGWORDS_4));
-
- /* We force a specific compiler optimisation here. The length
- * to
- * be copied turns out to be always 16, and by coding a memcpy
- * with
- * a literal value the compiler will compile inline code (in
- * fact,
- * only two vector instructions) to effect the copy. This gives
- * us
- * a huge performance increase.
- */
- unsigned long cplen =
- LAC_QUADWORDS_TO_BYTES(pCipherCdCtrlHdr->cipher_state_sz);
-
- if (cplen == 16)
- memcpy(pCipherReqParams->u.cipher_IV_array,
- pIvBufferVirt,
- 16);
- else
+
+ /* In case of XTS mode using UCS slice always embedd IV.
+ * IV provided by user needs to be encrypted to calculate
+ * initial tweak, use pCipherReqParams->u.cipher_IV_array as
+ * destination buffer for tweak value */
+ if (ICP_QAT_FW_LA_USE_UCS_SLICE_TYPE ==
+ pSessionDesc->cipherSliceType &&
+ LAC_CIPHER_IS_XTS_MODE(pSessionDesc->cipherAlgorithm)) {
+ memset(pCipherReqParams->u.cipher_IV_array,
+ 0,
+ LAC_LONGWORDS_TO_BYTES(
+ ICP_QAT_FW_NUM_LONGWORDS_4));
+ qatUtilsAESEncrypt(
+ pSessionDesc->cipherAesXtsKey2,
+ pSessionDesc->cipherKeyLenInBytes / 2,
+ pIvBufferVirt,
+ (Cpa8U *)pCipherReqParams->u.cipher_IV_array);
+ } else {
+ totalBufSize =
+ LAC_LONGWORDS_TO_BYTES(ICP_QAT_FW_NUM_LONGWORDS_4);
+ usedBufSize = LAC_QUADWORDS_TO_BYTES(
+ pCipherCdCtrlHdr->cipher_state_sz);
+ /* Only initialise unused buffer if applicable*/
+ if (usedBufSize < totalBufSize) {
+ memset(
+ (&pCipherReqParams->u.cipher_IV_array
+ [usedBufSize & LAC_UNUSED_POS_MASK]),
+ 0,
+ totalBufSize - usedBufSize);
+ }
memcpy(pCipherReqParams->u.cipher_IV_array,
pIvBufferVirt,
- cplen);
+ usedBufSize);
+ }
/* Set the flag indicating the field format */
ICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_SET(
*pCipherSpecificFlags, ICP_QAT_FW_CIPH_IV_16BYTE_DATA);
@@ -851,7 +973,7 @@
pArc4CipherState[i] = (Cpa8U)i;
}
- for (i = 0; i < LAC_CIPHER_ARC4_KEY_MATRIX_LEN_BYTES; ++i) {
+ for (i = 0, k = 0; i < LAC_CIPHER_ARC4_KEY_MATRIX_LEN_BYTES; ++i, ++k) {
Cpa8U swap = 0;
if (k >= keyLenInBytes)
@@ -860,7 +982,6 @@
j = (j + pArc4CipherState[i] + pKey[k]);
if (j >= LAC_CIPHER_ARC4_KEY_MATRIX_LEN_BYTES)
j %= LAC_CIPHER_ARC4_KEY_MATRIX_LEN_BYTES;
- ++k;
/* Swap state[i] & state[j] */
swap = pArc4CipherState[i];
Index: sys/dev/qat/qat_api/common/crypto/sym/qat/lac_sym_qat_constants_table.c
===================================================================
--- /dev/null
+++ sys/dev/qat/qat_api/common/crypto/sym/qat/lac_sym_qat_constants_table.c
@@ -0,0 +1,257 @@
+/* SPDX-License-Identifier: BSD-3-Clause */
+/* Copyright(c) 2007-2022 Intel Corporation */
+/* $FreeBSD$ */
+
+/**
+ ***************************************************************************
+ * @file lac_sym_qat_constants_table.c
+ *
+ * @ingroup LacSymQat
+ ***************************************************************************/
+
+/*
+*******************************************************************************
+* Include public/global header files
+*******************************************************************************
+*/
+
+#include "cpa.h"
+
+/*
+*******************************************************************************
+* Include private header files
+*******************************************************************************
+*/
+
+#include "lac_common.h"
+#include "icp_qat_fw_la.h"
+#include "lac_log.h"
+#include "lac_mem.h"
+#include "sal_string_parse.h"
+#include "lac_sal_types_crypto.h"
+#include "sal_types_compression.h"
+
+static uint8_t icp_qat_hw_cipher_lookup_tbl[ICP_QAT_HW_CIPHER_DELIMITER]
+ [ICP_QAT_HW_CIPHER_MODE_DELIMITER][2]
+ [2]; /* IA version */
+static uint8_t icp_qat_hw_auth_lookup_tbl[ICP_QAT_HW_AUTH_ALGO_DELIMITER]
+ [ICP_QAT_HW_AUTH_MODE_DELIMITER]
+ [2]; /* IA version */
+
+#define ICP_QAT_HW_FILL_LOOKUP_TBLS \
+ { \
+ \
+ icp_qat_hw_cipher_lookup_tbl[ICP_QAT_HW_CIPHER_ALGO_DES] \
+ [ICP_QAT_HW_CIPHER_ECB_MODE] \
+ [ICP_QAT_HW_CIPHER_ENCRYPT] \
+ [ICP_QAT_HW_CIPHER_NO_CONVERT] = \
+ 9; \
+ icp_qat_hw_cipher_lookup_tbl[ICP_QAT_HW_CIPHER_ALGO_DES] \
+ [ICP_QAT_HW_CIPHER_ECB_MODE] \
+ [ICP_QAT_HW_CIPHER_DECRYPT] \
+ [ICP_QAT_HW_CIPHER_NO_CONVERT] = \
+ 10; \
+ icp_qat_hw_cipher_lookup_tbl[ICP_QAT_HW_CIPHER_ALGO_DES] \
+ [ICP_QAT_HW_CIPHER_CBC_MODE] \
+ [ICP_QAT_HW_CIPHER_ENCRYPT] \
+ [ICP_QAT_HW_CIPHER_NO_CONVERT] = \
+ 11; \
+ icp_qat_hw_cipher_lookup_tbl[ICP_QAT_HW_CIPHER_ALGO_DES] \
+ [ICP_QAT_HW_CIPHER_CBC_MODE] \
+ [ICP_QAT_HW_CIPHER_DECRYPT] \
+ [ICP_QAT_HW_CIPHER_NO_CONVERT] = \
+ 12; \
+ icp_qat_hw_cipher_lookup_tbl[ICP_QAT_HW_CIPHER_ALGO_DES] \
+ [ICP_QAT_HW_CIPHER_CTR_MODE] \
+ [ICP_QAT_HW_CIPHER_ENCRYPT] \
+ [ICP_QAT_HW_CIPHER_NO_CONVERT] = \
+ 13; \
+ icp_qat_hw_cipher_lookup_tbl[ICP_QAT_HW_CIPHER_ALGO_AES128] \
+ [ICP_QAT_HW_CIPHER_ECB_MODE] \
+ [ICP_QAT_HW_CIPHER_ENCRYPT] \
+ [ICP_QAT_HW_CIPHER_NO_CONVERT] = \
+ 14; \
+ icp_qat_hw_cipher_lookup_tbl[ICP_QAT_HW_CIPHER_ALGO_AES128] \
+ [ICP_QAT_HW_CIPHER_ECB_MODE] \
+ [ICP_QAT_HW_CIPHER_ENCRYPT] \
+ [ICP_QAT_HW_CIPHER_KEY_CONVERT] = \
+ 15; \
+ icp_qat_hw_cipher_lookup_tbl[ICP_QAT_HW_CIPHER_ALGO_AES128] \
+ [ICP_QAT_HW_CIPHER_ECB_MODE] \
+ [ICP_QAT_HW_CIPHER_DECRYPT] \
+ [ICP_QAT_HW_CIPHER_NO_CONVERT] = \
+ 16; \
+ icp_qat_hw_cipher_lookup_tbl[ICP_QAT_HW_CIPHER_ALGO_AES128] \
+ [ICP_QAT_HW_CIPHER_ECB_MODE] \
+ [ICP_QAT_HW_CIPHER_DECRYPT] \
+ [ICP_QAT_HW_CIPHER_KEY_CONVERT] = \
+ 17; \
+ icp_qat_hw_cipher_lookup_tbl[ICP_QAT_HW_CIPHER_ALGO_AES128] \
+ [ICP_QAT_HW_CIPHER_CBC_MODE] \
+ [ICP_QAT_HW_CIPHER_ENCRYPT] \
+ [ICP_QAT_HW_CIPHER_NO_CONVERT] = \
+ 18; \
+ icp_qat_hw_cipher_lookup_tbl[ICP_QAT_HW_CIPHER_ALGO_AES128] \
+ [ICP_QAT_HW_CIPHER_CBC_MODE] \
+ [ICP_QAT_HW_CIPHER_ENCRYPT] \
+ [ICP_QAT_HW_CIPHER_KEY_CONVERT] = \
+ 19; \
+ icp_qat_hw_cipher_lookup_tbl[ICP_QAT_HW_CIPHER_ALGO_AES128] \
+ [ICP_QAT_HW_CIPHER_CBC_MODE] \
+ [ICP_QAT_HW_CIPHER_DECRYPT] \
+ [ICP_QAT_HW_CIPHER_NO_CONVERT] = \
+ 20; \
+ icp_qat_hw_cipher_lookup_tbl[ICP_QAT_HW_CIPHER_ALGO_AES128] \
+ [ICP_QAT_HW_CIPHER_CBC_MODE] \
+ [ICP_QAT_HW_CIPHER_DECRYPT] \
+ [ICP_QAT_HW_CIPHER_KEY_CONVERT] = \
+ 21; \
+ icp_qat_hw_cipher_lookup_tbl[ICP_QAT_HW_CIPHER_ALGO_AES128] \
+ [ICP_QAT_HW_CIPHER_CTR_MODE] \
+ [ICP_QAT_HW_CIPHER_ENCRYPT] \
+ [ICP_QAT_HW_CIPHER_NO_CONVERT] = \
+ 22; \
+ icp_qat_hw_cipher_lookup_tbl[ICP_QAT_HW_CIPHER_ALGO_AES128] \
+ [ICP_QAT_HW_CIPHER_F8_MODE] \
+ [ICP_QAT_HW_CIPHER_ENCRYPT] \
+ [ICP_QAT_HW_CIPHER_NO_CONVERT] = \
+ 23; \
+ icp_qat_hw_cipher_lookup_tbl[ICP_QAT_HW_CIPHER_ALGO_ARC4] \
+ [ICP_QAT_HW_CIPHER_ECB_MODE] \
+ [ICP_QAT_HW_CIPHER_ENCRYPT] \
+ [ICP_QAT_HW_CIPHER_NO_CONVERT] = \
+ 24; \
+ icp_qat_hw_cipher_lookup_tbl[ICP_QAT_HW_CIPHER_ALGO_ARC4] \
+ [ICP_QAT_HW_CIPHER_ECB_MODE] \
+ [ICP_QAT_HW_CIPHER_ENCRYPT] \
+ [ICP_QAT_HW_CIPHER_KEY_CONVERT] = \
+ 25; \
+ \
+ icp_qat_hw_auth_lookup_tbl \
+ [ICP_QAT_HW_AUTH_ALGO_MD5][ICP_QAT_HW_AUTH_MODE0] \
+ [ICP_QAT_FW_AUTH_HDR_FLAG_NO_NESTED] = 37; \
+ icp_qat_hw_auth_lookup_tbl \
+ [ICP_QAT_HW_AUTH_ALGO_SHA1][ICP_QAT_HW_AUTH_MODE0] \
+ [ICP_QAT_FW_AUTH_HDR_FLAG_NO_NESTED] = 41; \
+ icp_qat_hw_auth_lookup_tbl \
+ [ICP_QAT_HW_AUTH_ALGO_SHA1][ICP_QAT_HW_AUTH_MODE1] \
+ [ICP_QAT_FW_AUTH_HDR_FLAG_NO_NESTED] = 46; \
+ icp_qat_hw_auth_lookup_tbl \
+ [ICP_QAT_HW_AUTH_ALGO_SHA224][ICP_QAT_HW_AUTH_MODE0] \
+ [ICP_QAT_FW_AUTH_HDR_FLAG_NO_NESTED] = 48; \
+ icp_qat_hw_auth_lookup_tbl \
+ [ICP_QAT_HW_AUTH_ALGO_SHA256][ICP_QAT_HW_AUTH_MODE0] \
+ [ICP_QAT_FW_AUTH_HDR_FLAG_NO_NESTED] = 54; \
+ icp_qat_hw_auth_lookup_tbl \
+ [ICP_QAT_HW_AUTH_ALGO_SHA384][ICP_QAT_HW_AUTH_MODE0] \
+ [ICP_QAT_FW_AUTH_HDR_FLAG_NO_NESTED] = 60; \
+ icp_qat_hw_auth_lookup_tbl \
+ [ICP_QAT_HW_AUTH_ALGO_SHA512][ICP_QAT_HW_AUTH_MODE0] \
+ [ICP_QAT_FW_AUTH_HDR_FLAG_NO_NESTED] = 70; \
+ }
+
+/**
+ *****************************************************************************
+ * @ingroup LacSymQat
+ * LacSymQat_ConstantsInitLookupTables
+ *
+ *
+ *****************************************************************************/
+void
+LacSymQat_ConstantsInitLookupTables(CpaInstanceHandle instanceHandle)
+{
+ sal_service_t *pService = (sal_service_t *)instanceHandle;
+ lac_sym_qat_constants_t *pConstantsLookupTables;
+
+ /* Note the global tables are initialised first, then copied
+ * to the service which probably seems like a waste of memory
+ * and processing cycles as the global tables are never needed again
+ * but this allows use of the ICP_QAT_HW_FILL_LOOKUP_TBLS macro
+ * supplied by FW without modification */
+
+ if (SAL_SERVICE_TYPE_COMPRESSION == pService->type) {
+ /* DC chaining not supported yet */
+ return;
+ } else {
+ pConstantsLookupTables = &(
+ ((sal_crypto_service_t *)pService)->constantsLookupTables);
+ }
+
+ /* First fill the global lookup tables with zeroes. */
+ memset(icp_qat_hw_cipher_lookup_tbl,
+ 0,
+ sizeof(icp_qat_hw_cipher_lookup_tbl));
+ memset(icp_qat_hw_auth_lookup_tbl,
+ 0,
+ sizeof(icp_qat_hw_auth_lookup_tbl));
+
+ /* Override lookup tables with the offsets into the SHRAM table
+ * for supported algorithms/modes */
+ ICP_QAT_HW_FILL_LOOKUP_TBLS;
+
+ /* Copy the global tables to the service instance */
+ memcpy(pConstantsLookupTables->cipher_offset,
+ icp_qat_hw_cipher_lookup_tbl,
+ sizeof(pConstantsLookupTables->cipher_offset));
+ memcpy(pConstantsLookupTables->auth_offset,
+ icp_qat_hw_auth_lookup_tbl,
+ sizeof(pConstantsLookupTables->auth_offset));
+}
+
+/**
+ *****************************************************************************
+ * @ingroup LacSymQat
+ * LacSymQat_ConstantsGetCipherOffset
+ *
+ *
+ *****************************************************************************/
+void
+LacSymQat_ConstantsGetCipherOffset(CpaInstanceHandle instanceHandle,
+ uint8_t algo,
+ uint8_t mode,
+ uint8_t direction,
+ uint8_t convert,
+ uint8_t *poffset)
+{
+ sal_service_t *pService = (sal_service_t *)instanceHandle;
+ lac_sym_qat_constants_t *pConstantsLookupTables;
+
+ if (SAL_SERVICE_TYPE_COMPRESSION == pService->type) {
+ /* DC chaining not supported yet */
+ return;
+ } else {
+ pConstantsLookupTables = &(
+ ((sal_crypto_service_t *)pService)->constantsLookupTables);
+ }
+
+ *poffset = pConstantsLookupTables
+ ->cipher_offset[algo][mode][direction][convert];
+}
+
+/**
+ *****************************************************************************
+ * @ingroup LacSymQat
+ * LacSymQat_ConstantsGetAuthOffset
+ *
+ *
+ *****************************************************************************/
+void
+LacSymQat_ConstantsGetAuthOffset(CpaInstanceHandle instanceHandle,
+ uint8_t algo,
+ uint8_t mode,
+ uint8_t nested,
+ uint8_t *poffset)
+{
+ sal_service_t *pService = (sal_service_t *)instanceHandle;
+ lac_sym_qat_constants_t *pConstantsLookupTables;
+
+ if (SAL_SERVICE_TYPE_COMPRESSION == pService->type) {
+ /* DC chaining not supported yet */
+ return;
+ } else {
+ pConstantsLookupTables = &(
+ ((sal_crypto_service_t *)pService)->constantsLookupTables);
+ }
+
+ *poffset = pConstantsLookupTables->auth_offset[algo][mode][nested];
+}
Index: sys/dev/qat/qat_api/common/crypto/sym/qat/lac_sym_qat_hash.c
===================================================================
--- sys/dev/qat/qat_api/common/crypto/sym/qat/lac_sym_qat_hash.c
+++ sys/dev/qat/qat_api/common/crypto/sym/qat/lac_sym_qat_hash.c
@@ -28,8 +28,10 @@
#include "lac_sym_qat.h"
#include "lac_list.h"
#include "lac_sal_types.h"
+#include "lac_sal_types_crypto.h"
#include "lac_sym_qat_hash.h"
#include "lac_sym_qat_hash_defs_lookup.h"
+#include "sal_hw_gen.h"
/**
* This structure contains pointers into the hash setup block of the
@@ -148,10 +150,10 @@
icp_qat_hw_auth_mode_t qatHashMode,
CpaBoolean useSymConstantsTable,
CpaBoolean useOptimisedContentDesc,
+ CpaBoolean useStatefulSha3ContentDesc,
lac_sym_qat_hash_precompute_info_t *pPrecompute,
Cpa32U *pHashBlkSizeInBytes)
{
-
icp_qat_fw_auth_cd_ctrl_hdr_t *cd_ctrl =
(icp_qat_fw_auth_cd_ctrl_hdr_t *)&(pMsg->cd_ctrl);
lac_sym_qat_hash_defs_t *pHashDefs = NULL;
@@ -159,7 +161,7 @@
Cpa32U hashSetupBlkSize = 0;
/* setup the offset in QuadWords into the hw blk */
- cd_ctrl->hash_cfg_offset = hwBlockOffsetInQuadWords;
+ cd_ctrl->hash_cfg_offset = (Cpa8U)hwBlockOffsetInQuadWords;
ICP_QAT_FW_COMN_NEXT_ID_SET(cd_ctrl, nextSlice);
ICP_QAT_FW_COMN_CURR_ID_SET(cd_ctrl, ICP_QAT_FW_SLICE_AUTH);
@@ -170,11 +172,19 @@
/* Hmac in mode 2 TLS */
if (IS_HASH_MODE_2(qatHashMode)) {
- /* Set bit for nested hashing.
- * Make sure not to overwrite other flags in hash_flags byte.
- */
- ICP_QAT_FW_HASH_FLAG_AUTH_HDR_NESTED_SET(
- cd_ctrl->hash_flags, ICP_QAT_FW_AUTH_HDR_FLAG_DO_NESTED);
+ if (isCyGen4x((sal_crypto_service_t *)instanceHandle)) {
+ /* CPM2.0 has a dedicated bit for HMAC mode2 */
+ ICP_QAT_FW_HASH_FLAG_MODE2_SET(cd_ctrl->hash_flags,
+ QAT_FW_LA_MODE2);
+ } else {
+ /* Set bit for nested hashing.
+ * Make sure not to overwrite other flags in hash_flags
+ * byte.
+ */
+ ICP_QAT_FW_HASH_FLAG_AUTH_HDR_NESTED_SET(
+ cd_ctrl->hash_flags,
+ ICP_QAT_FW_AUTH_HDR_FLAG_DO_NESTED);
+ }
}
/* Nested hash in mode 0 */
else if (CPA_CY_SYM_HASH_MODE_NESTED == pHashSetupData->hashMode) {
@@ -190,16 +200,32 @@
cd_ctrl->hash_flags, ICP_QAT_FW_AUTH_HDR_FLAG_NO_NESTED);
}
+ /* Set skip state load flags */
+ if (useStatefulSha3ContentDesc) {
+ /* Here both skip state load flags are set. FW reads them based
+ * on partial packet type. */
+ ICP_QAT_FW_HASH_FLAG_SKIP_INNER_STATE1_LOAD_SET(
+ cd_ctrl->hash_flags, QAT_FW_LA_SKIP_INNER_STATE1_LOAD);
+ ICP_QAT_FW_HASH_FLAG_SKIP_OUTER_STATE1_LOAD_SET(
+ cd_ctrl->hash_flags, QAT_FW_LA_SKIP_OUTER_STATE1_LOAD);
+ }
+
/* set the final digest size */
- cd_ctrl->final_sz = pHashSetupData->digestResultLenInBytes;
+ cd_ctrl->final_sz = (Cpa8U)pHashSetupData->digestResultLenInBytes;
/* set the state1 size */
- cd_ctrl->inner_state1_sz =
- LAC_ALIGN_POW2_ROUNDUP(pHashDefs->qatInfo->state1Length,
- LAC_QUAD_WORD_IN_BYTES);
+ if (useStatefulSha3ContentDesc) {
+ cd_ctrl->inner_state1_sz =
+ LAC_ALIGN_POW2_ROUNDUP(LAC_HASH_SHA3_STATEFUL_STATE_SIZE,
+ LAC_QUAD_WORD_IN_BYTES);
+ } else {
+ cd_ctrl->inner_state1_sz =
+ LAC_ALIGN_POW2_ROUNDUP(pHashDefs->qatInfo->state1Length,
+ LAC_QUAD_WORD_IN_BYTES);
+ }
/* set the inner result size to the digest length */
- cd_ctrl->inner_res_sz = pHashDefs->algInfo->digestLength;
+ cd_ctrl->inner_res_sz = (Cpa8U)pHashDefs->algInfo->digestLength;
/* set the state2 size - only for mode 1 Auth algos and AES CBC MAC */
if (IS_HASH_MODE_1(qatHashMode) ||
@@ -212,13 +238,22 @@
cd_ctrl->inner_state2_sz = 0;
}
- cd_ctrl->inner_state2_offset = cd_ctrl->hash_cfg_offset +
- LAC_BYTES_TO_QUADWORDS(sizeof(icp_qat_hw_auth_setup_t) +
- cd_ctrl->inner_state1_sz);
+ if (useSymConstantsTable) {
+ cd_ctrl->inner_state2_offset =
+ LAC_BYTES_TO_QUADWORDS(cd_ctrl->inner_state1_sz);
+
+ /* size of inner part of hash setup block */
+ hashSetupBlkSize =
+ cd_ctrl->inner_state1_sz + cd_ctrl->inner_state2_sz;
+ } else {
+ cd_ctrl->inner_state2_offset = cd_ctrl->hash_cfg_offset +
+ LAC_BYTES_TO_QUADWORDS(sizeof(icp_qat_hw_auth_setup_t) +
+ cd_ctrl->inner_state1_sz);
- /* size of inner part of hash setup block */
- hashSetupBlkSize = sizeof(icp_qat_hw_auth_setup_t) +
- cd_ctrl->inner_state1_sz + cd_ctrl->inner_state2_sz;
+ /* size of inner part of hash setup block */
+ hashSetupBlkSize = sizeof(icp_qat_hw_auth_setup_t) +
+ cd_ctrl->inner_state1_sz + cd_ctrl->inner_state2_sz;
+ }
/* For nested hashing - Fill in the outer fields */
if (CPA_CY_SYM_HASH_MODE_NESTED == pHashSetupData->hashMode ||
@@ -238,18 +273,24 @@
cd_ctrl->outer_config_offset = cd_ctrl->inner_state2_offset +
LAC_BYTES_TO_QUADWORDS(cd_ctrl->inner_state2_sz);
- cd_ctrl->outer_state1_sz =
- LAC_ALIGN_POW2_ROUNDUP(pOuterHashDefs->algInfo->stateSize,
- LAC_QUAD_WORD_IN_BYTES);
+ if (useStatefulSha3ContentDesc) {
+ cd_ctrl->outer_state1_sz = LAC_ALIGN_POW2_ROUNDUP(
+ LAC_HASH_SHA3_STATEFUL_STATE_SIZE,
+ LAC_QUAD_WORD_IN_BYTES);
+ } else {
+ cd_ctrl->outer_state1_sz = LAC_ALIGN_POW2_ROUNDUP(
+ pOuterHashDefs->algInfo->stateSize,
+ LAC_QUAD_WORD_IN_BYTES);
+ }
/* outer result size */
- cd_ctrl->outer_res_sz = pOuterHashDefs->algInfo->digestLength;
+ cd_ctrl->outer_res_sz =
+ (Cpa8U)pOuterHashDefs->algInfo->digestLength;
/* outer_prefix_offset will be the size of the inner prefix data
* plus the hash state storage size. */
/* The prefix buffer is part of the ReqParams, so this param
- * will be
- * setup where ReqParams are set up */
+ * will be setup where ReqParams are set up */
/* add on size of outer part of hash block */
hashSetupBlkSize +=
@@ -325,8 +366,9 @@
if (IS_HASH_MODE_2(qatHashMode)) {
/* Inner and outer prefixes are the block length */
pHashReqParams->u2.inner_prefix_sz =
- pHashDefs->algInfo->blockLength;
- cd_ctrl->outer_prefix_sz = pHashDefs->algInfo->blockLength;
+ (Cpa8U)pHashDefs->algInfo->blockLength;
+ cd_ctrl->outer_prefix_sz =
+ (Cpa8U)pHashDefs->algInfo->blockLength;
cd_ctrl->outer_prefix_offset = LAC_BYTES_TO_QUADWORDS(
LAC_ALIGN_POW2_ROUNDUP((pHashReqParams->u2.inner_prefix_sz),
LAC_QUAD_WORD_IN_BYTES));
@@ -336,9 +378,11 @@
/* set inner and outer prefixes */
pHashReqParams->u2.inner_prefix_sz =
- pHashSetupData->nestedModeSetupData.innerPrefixLenInBytes;
+ (Cpa8U)pHashSetupData->nestedModeSetupData
+ .innerPrefixLenInBytes;
cd_ctrl->outer_prefix_sz =
- pHashSetupData->nestedModeSetupData.outerPrefixLenInBytes;
+ (Cpa8U)pHashSetupData->nestedModeSetupData
+ .outerPrefixLenInBytes;
cd_ctrl->outer_prefix_offset = LAC_BYTES_TO_QUADWORDS(
LAC_ALIGN_POW2_ROUNDUP((pHashReqParams->u2.inner_prefix_sz),
LAC_QUAD_WORD_IN_BYTES));
@@ -363,8 +407,9 @@
* just need 2 bytes to store encoded length of
* 'a' */
aadDataSize += sizeof(Cpa16U);
- aadDataSize += pHashSetupData->authModeSetupData
- .aadLenInBytes;
+ aadDataSize +=
+ (Cpa16U)pHashSetupData->authModeSetupData
+ .aadLenInBytes;
}
/* round the aad size to the multiple of CCM block
@@ -375,7 +420,8 @@
} else if (CPA_CY_SYM_HASH_AES_GCM ==
pHashSetupData->hashAlgorithm) {
aadDataSize =
- pHashSetupData->authModeSetupData.aadLenInBytes;
+ (Cpa16U)
+ pHashSetupData->authModeSetupData.aadLenInBytes;
/* round the aad size to the multiple of GCM hash block
* size. */
@@ -406,7 +452,7 @@
/* auth result size in bytes to be read in for a verify
* operation */
pHashReqParams->auth_res_sz =
- pHashSetupData->digestResultLenInBytes;
+ (Cpa8U)pHashSetupData->digestResultLenInBytes;
} else {
pHashReqParams->auth_res_sz = 0;
}
@@ -453,7 +499,7 @@
{
Cpa32U innerConfig = 0;
lac_hash_blk_ptrs_t hashBlkPtrs = { 0 };
- Cpa32U aed_hash_cmp_length = 0;
+ Cpa32U aedHashCmpLength = 0;
LacSymQat_HashHwBlockPtrsInit(pHashControlBlock,
pHwBlockBase,
@@ -610,7 +656,7 @@
ICP_QAT_HW_CIPHER_ALGO_SNOW_3G_UEA2,
ICP_QAT_HW_CIPHER_KEY_CONVERT,
ICP_QAT_HW_CIPHER_ENCRYPT,
- aed_hash_cmp_length);
+ aedHashCmpLength);
pCipherConfig->reserved = 0;
@@ -633,7 +679,7 @@
ICP_QAT_HW_CIPHER_ALGO_ZUC_3G_128_EEA3,
ICP_QAT_HW_CIPHER_KEY_CONVERT,
ICP_QAT_HW_CIPHER_ENCRYPT,
- aed_hash_cmp_length);
+ aedHashCmpLength);
pCipherConfig->reserved = 0;
@@ -820,7 +866,7 @@
CpaBoolean digestVerify,
Cpa8U *pAuthResult,
CpaCySymHashAlgorithm alg,
- void *hkdf_secret)
+ void *pHKDFSecret)
{
Cpa64U authResultPhys = 0;
icp_qat_fw_la_auth_req_params_t *pHashReqParams;
@@ -833,11 +879,11 @@
pHashReqParams->auth_len = authLenInBytes;
/* Set the physical location of secret for HKDF */
- if (NULL != hkdf_secret) {
+ if (NULL != pHKDFSecret) {
LAC_MEM_SHARED_WRITE_VIRT_TO_PHYS_PTR_EXTERNAL(
- (*pService), pHashReqParams->u1.aad_adr, hkdf_secret);
+ (*pService), pHashReqParams->u1.aad_adr, pHKDFSecret);
- if (pHashReqParams->u1.aad_adr == 0) {
+ if (0 == pHashReqParams->u1.aad_adr) {
LAC_LOG_ERROR(
"Unable to get the physical address of the"
" HKDF secret\n");
@@ -868,7 +914,7 @@
if (CPA_TRUE == digestVerify) {
/* auth result size in bytes to be read in for a verify
* operation */
- pHashReqParams->auth_res_sz = hashResultSize;
+ pHashReqParams->auth_res_sz = (Cpa8U)hashResultSize;
} else {
pHashReqParams->auth_res_sz = 0;
}
Index: sys/dev/qat/qat_api/common/crypto/sym/qat/lac_sym_qat_hash_defs_lookup.c
===================================================================
--- sys/dev/qat/qat_api/common/crypto/sym/qat/lac_sym_qat_hash_defs_lookup.c
+++ sys/dev/qat/qat_api/common/crypto/sym/qat/lac_sym_qat_hash_defs_lookup.c
@@ -74,62 +74,70 @@
};
/* SHA 256 - 32 bytes - Initialiser state can be found in FIPS stds 180-2 */
-static Cpa8U sha256InitialState[LAC_HASH_SHA256_STATE_SIZE] =
- { 0x6a, 0x09, 0xe6, 0x67, 0xbb, 0x67, 0xae, 0x85, 0x3c, 0x6e, 0xf3,
- 0x72, 0xa5, 0x4f, 0xf5, 0x3a, 0x51, 0x0e, 0x52, 0x7f, 0x9b, 0x05,
- 0x68, 0x8c, 0x1f, 0x83, 0xd9, 0xab, 0x5b, 0xe0, 0xcd, 0x19 };
+static Cpa8U sha256InitialState[LAC_HASH_SHA256_STATE_SIZE] = {
+ 0x6a, 0x09, 0xe6, 0x67, 0xbb, 0x67, 0xae, 0x85, 0x3c, 0x6e, 0xf3,
+ 0x72, 0xa5, 0x4f, 0xf5, 0x3a, 0x51, 0x0e, 0x52, 0x7f, 0x9b, 0x05,
+ 0x68, 0x8c, 0x1f, 0x83, 0xd9, 0xab, 0x5b, 0xe0, 0xcd, 0x19
+};
/* SHA 384 - 64 bytes - Initialiser state can be found in FIPS stds 180-2 */
-static Cpa8U sha384InitialState[LAC_HASH_SHA384_STATE_SIZE] =
- { 0xcb, 0xbb, 0x9d, 0x5d, 0xc1, 0x05, 0x9e, 0xd8, 0x62, 0x9a, 0x29,
- 0x2a, 0x36, 0x7c, 0xd5, 0x07, 0x91, 0x59, 0x01, 0x5a, 0x30, 0x70,
- 0xdd, 0x17, 0x15, 0x2f, 0xec, 0xd8, 0xf7, 0x0e, 0x59, 0x39, 0x67,
- 0x33, 0x26, 0x67, 0xff, 0xc0, 0x0b, 0x31, 0x8e, 0xb4, 0x4a, 0x87,
- 0x68, 0x58, 0x15, 0x11, 0xdb, 0x0c, 0x2e, 0x0d, 0x64, 0xf9, 0x8f,
- 0xa7, 0x47, 0xb5, 0x48, 0x1d, 0xbe, 0xfa, 0x4f, 0xa4 };
+static Cpa8U sha384InitialState[LAC_HASH_SHA384_STATE_SIZE] = {
+ 0xcb, 0xbb, 0x9d, 0x5d, 0xc1, 0x05, 0x9e, 0xd8, 0x62, 0x9a, 0x29,
+ 0x2a, 0x36, 0x7c, 0xd5, 0x07, 0x91, 0x59, 0x01, 0x5a, 0x30, 0x70,
+ 0xdd, 0x17, 0x15, 0x2f, 0xec, 0xd8, 0xf7, 0x0e, 0x59, 0x39, 0x67,
+ 0x33, 0x26, 0x67, 0xff, 0xc0, 0x0b, 0x31, 0x8e, 0xb4, 0x4a, 0x87,
+ 0x68, 0x58, 0x15, 0x11, 0xdb, 0x0c, 0x2e, 0x0d, 0x64, 0xf9, 0x8f,
+ 0xa7, 0x47, 0xb5, 0x48, 0x1d, 0xbe, 0xfa, 0x4f, 0xa4
+};
/* SHA 512 - 64 bytes - Initialiser state can be found in FIPS stds 180-2 */
-static Cpa8U sha512InitialState[LAC_HASH_SHA512_STATE_SIZE] =
- { 0x6a, 0x09, 0xe6, 0x67, 0xf3, 0xbc, 0xc9, 0x08, 0xbb, 0x67, 0xae,
- 0x85, 0x84, 0xca, 0xa7, 0x3b, 0x3c, 0x6e, 0xf3, 0x72, 0xfe, 0x94,
- 0xf8, 0x2b, 0xa5, 0x4f, 0xf5, 0x3a, 0x5f, 0x1d, 0x36, 0xf1, 0x51,
- 0x0e, 0x52, 0x7f, 0xad, 0xe6, 0x82, 0xd1, 0x9b, 0x05, 0x68, 0x8c,
- 0x2b, 0x3e, 0x6c, 0x1f, 0x1f, 0x83, 0xd9, 0xab, 0xfb, 0x41, 0xbd,
- 0x6b, 0x5b, 0xe0, 0xcd, 0x19, 0x13, 0x7e, 0x21, 0x79 };
+static Cpa8U sha512InitialState[LAC_HASH_SHA512_STATE_SIZE] = {
+ 0x6a, 0x09, 0xe6, 0x67, 0xf3, 0xbc, 0xc9, 0x08, 0xbb, 0x67, 0xae,
+ 0x85, 0x84, 0xca, 0xa7, 0x3b, 0x3c, 0x6e, 0xf3, 0x72, 0xfe, 0x94,
+ 0xf8, 0x2b, 0xa5, 0x4f, 0xf5, 0x3a, 0x5f, 0x1d, 0x36, 0xf1, 0x51,
+ 0x0e, 0x52, 0x7f, 0xad, 0xe6, 0x82, 0xd1, 0x9b, 0x05, 0x68, 0x8c,
+ 0x2b, 0x3e, 0x6c, 0x1f, 0x1f, 0x83, 0xd9, 0xab, 0xfb, 0x41, 0xbd,
+ 0x6b, 0x5b, 0xe0, 0xcd, 0x19, 0x13, 0x7e, 0x21, 0x79
+};
/* SHA3 224 - 28 bytes */
-static Cpa8U sha3_224InitialState[LAC_HASH_SHA3_224_STATE_SIZE] =
- { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 };
+static Cpa8U sha3_224InitialState[LAC_HASH_SHA3_224_STATE_SIZE] = {
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+};
/* SHA3 256 - 32 bytes */
-static Cpa8U sha3_256InitialState[LAC_HASH_SHA3_256_STATE_SIZE] =
- { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 };
+static Cpa8U sha3_256InitialState[LAC_HASH_SHA3_256_STATE_SIZE] = {
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+};
/* SHA3 384 - 48 bytes */
-static Cpa8U sha3_384InitialState[LAC_HASH_SHA3_384_STATE_SIZE] =
- { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 };
+static Cpa8U sha3_384InitialState[LAC_HASH_SHA3_384_STATE_SIZE] = {
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+};
/* SHA3 512 - 64 bytes */
-static Cpa8U sha3_512InitialState[LAC_HASH_SHA3_512_STATE_SIZE] =
- { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 };
+static Cpa8U sha3_512InitialState[LAC_HASH_SHA3_512_STATE_SIZE] = {
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+};
/* SM3 - 32 bytes */
-static Cpa8U sm3InitialState[LAC_HASH_SM3_STATE_SIZE] =
- { 0x73, 0x80, 0x16, 0x6f, 0x49, 0x14, 0xb2, 0xb9, 0x17, 0x24, 0x42,
- 0xd7, 0xda, 0x8a, 0x06, 0x00, 0xa9, 0x6f, 0x30, 0xbc, 0x16, 0x31,
- 0x38, 0xaa, 0xe3, 0x8d, 0xee, 0x4d, 0xb0, 0xfb, 0x0e, 0x4e };
+static Cpa8U sm3InitialState[LAC_HASH_SM3_STATE_SIZE] = {
+ 0x73, 0x80, 0x16, 0x6f, 0x49, 0x14, 0xb2, 0xb9, 0x17, 0x24, 0x42,
+ 0xd7, 0xda, 0x8a, 0x06, 0x00, 0xa9, 0x6f, 0x30, 0xbc, 0x16, 0x31,
+ 0x38, 0xaa, 0xe3, 0x8d, 0xee, 0x4d, 0xb0, 0xfb, 0x0e, 0x4e
+};
/* Constants used in generating K1, K2, K3 from a Key for AES_XCBC_MAC
* State defined in RFC 3566 */
@@ -140,12 +148,10 @@
0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03,
};
-static Cpa8U aesCmacKeySeed[LAC_HASH_CMAC_BLOCK_SIZE] = { 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00,
- 0x00 };
+static Cpa8U aesCmacKeySeed[LAC_HASH_CMAC_BLOCK_SIZE] = {
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+};
/* Hash Algorithm specific structure */
@@ -179,57 +185,57 @@
sha512InitialState,
LAC_HASH_SHA512_STATE_SIZE };
-static lac_sym_qat_hash_alg_info_t sha3_224Info =
- { LAC_HASH_SHA3_224_DIGEST_SIZE,
- LAC_HASH_SHA3_224_BLOCK_SIZE,
- sha3_224InitialState,
- LAC_HASH_SHA3_224_STATE_SIZE };
-
-static lac_sym_qat_hash_alg_info_t sha3_256Info =
- { LAC_HASH_SHA3_256_DIGEST_SIZE,
- LAC_HASH_SHA3_256_BLOCK_SIZE,
- sha3_256InitialState,
- LAC_HASH_SHA3_256_STATE_SIZE };
-
-static lac_sym_qat_hash_alg_info_t sha3_384Info =
- { LAC_HASH_SHA3_384_DIGEST_SIZE,
- LAC_HASH_SHA3_384_BLOCK_SIZE,
- sha3_384InitialState,
- LAC_HASH_SHA3_384_STATE_SIZE };
-
-static lac_sym_qat_hash_alg_info_t sha3_512Info =
- { LAC_HASH_SHA3_512_DIGEST_SIZE,
- LAC_HASH_SHA3_512_BLOCK_SIZE,
- sha3_512InitialState,
- LAC_HASH_SHA3_512_STATE_SIZE };
+static lac_sym_qat_hash_alg_info_t sha3_224Info = {
+ LAC_HASH_SHA3_224_DIGEST_SIZE,
+ LAC_HASH_SHA3_224_BLOCK_SIZE,
+ sha3_224InitialState,
+ LAC_HASH_SHA3_224_STATE_SIZE
+};
-static lac_sym_qat_hash_alg_info_t polyInfo = { LAC_HASH_POLY_DIGEST_SIZE,
- LAC_HASH_POLY_BLOCK_SIZE,
- NULL, /* intial state */
- LAC_HASH_POLY_STATE_SIZE };
+static lac_sym_qat_hash_alg_info_t sha3_256Info = {
+ LAC_HASH_SHA3_256_DIGEST_SIZE,
+ LAC_HASH_SHA3_256_BLOCK_SIZE,
+ sha3_256InitialState,
+ LAC_HASH_SHA3_256_STATE_SIZE
+};
-static lac_sym_qat_hash_alg_info_t shake_128Info =
- { LAC_HASH_SHAKE_128_DIGEST_SIZE, LAC_HASH_SHAKE_128_BLOCK_SIZE, NULL, 0 };
+static lac_sym_qat_hash_alg_info_t sha3_384Info = {
+ LAC_HASH_SHA3_384_DIGEST_SIZE,
+ LAC_HASH_SHA3_384_BLOCK_SIZE,
+ sha3_384InitialState,
+ LAC_HASH_SHA3_384_STATE_SIZE
+};
-static lac_sym_qat_hash_alg_info_t shake_256Info =
- { LAC_HASH_SHAKE_256_DIGEST_SIZE, LAC_HASH_SHAKE_256_BLOCK_SIZE, NULL, 0 };
+static lac_sym_qat_hash_alg_info_t sha3_512Info = {
+ LAC_HASH_SHA3_512_DIGEST_SIZE,
+ LAC_HASH_SHA3_512_BLOCK_SIZE,
+ sha3_512InitialState,
+ LAC_HASH_SHA3_512_STATE_SIZE
+};
static lac_sym_qat_hash_alg_info_t sm3Info = { LAC_HASH_SM3_DIGEST_SIZE,
LAC_HASH_SM3_BLOCK_SIZE,
sm3InitialState,
LAC_HASH_SM3_STATE_SIZE };
-static lac_sym_qat_hash_alg_info_t xcbcMacInfo =
- { LAC_HASH_XCBC_MAC_128_DIGEST_SIZE,
- LAC_HASH_XCBC_MAC_BLOCK_SIZE,
- aesXcbcKeySeed,
- LAC_SYM_QAT_XCBC_STATE_SIZE };
+static lac_sym_qat_hash_alg_info_t polyInfo = { LAC_HASH_POLY_DIGEST_SIZE,
+ LAC_HASH_POLY_BLOCK_SIZE,
+ NULL, /* intial state */
+ LAC_HASH_POLY_STATE_SIZE };
-static lac_sym_qat_hash_alg_info_t aesCmacInfo =
- { LAC_HASH_CMAC_128_DIGEST_SIZE,
- LAC_HASH_CMAC_BLOCK_SIZE,
- aesCmacKeySeed,
- LAC_SYM_QAT_CMAC_STATE_SIZE };
+static lac_sym_qat_hash_alg_info_t xcbcMacInfo = {
+ LAC_HASH_XCBC_MAC_128_DIGEST_SIZE,
+ LAC_HASH_XCBC_MAC_BLOCK_SIZE,
+ aesXcbcKeySeed,
+ LAC_SYM_QAT_XCBC_STATE_SIZE
+};
+
+static lac_sym_qat_hash_alg_info_t aesCmacInfo = {
+ LAC_HASH_CMAC_128_DIGEST_SIZE,
+ LAC_HASH_CMAC_BLOCK_SIZE,
+ aesCmacKeySeed,
+ LAC_SYM_QAT_CMAC_STATE_SIZE
+};
static lac_sym_qat_hash_alg_info_t aesCcmInfo = {
LAC_HASH_AES_CCM_DIGEST_SIZE,
@@ -259,11 +265,12 @@
0 /* state size */
};
-static lac_sym_qat_hash_alg_info_t aesCbcMacInfo =
- { LAC_HASH_AES_CBC_MAC_DIGEST_SIZE,
- LAC_HASH_AES_CBC_MAC_BLOCK_SIZE,
- NULL,
- 0 };
+static lac_sym_qat_hash_alg_info_t aesCbcMacInfo = {
+ LAC_HASH_AES_CBC_MAC_DIGEST_SIZE,
+ LAC_HASH_AES_CBC_MAC_BLOCK_SIZE,
+ NULL,
+ 0
+};
static lac_sym_qat_hash_alg_info_t zucEia3Info = {
LAC_HASH_ZUC_EIA3_DIGEST_SIZE,
@@ -283,145 +290,154 @@
ICP_QAT_HW_SHA1_STATE1_SZ,
ICP_QAT_HW_SHA1_STATE2_SZ };
-static lac_sym_qat_hash_qat_info_t sha224Config =
- { ICP_QAT_HW_AUTH_ALGO_SHA224,
- LAC_HASH_SHA224_BLOCK_SIZE,
- ICP_QAT_HW_SHA224_STATE1_SZ,
- ICP_QAT_HW_SHA224_STATE2_SZ };
-
-static lac_sym_qat_hash_qat_info_t sha256Config =
- { ICP_QAT_HW_AUTH_ALGO_SHA256,
- LAC_HASH_SHA256_BLOCK_SIZE,
- ICP_QAT_HW_SHA256_STATE1_SZ,
- ICP_QAT_HW_SHA256_STATE2_SZ };
-
-static lac_sym_qat_hash_qat_info_t sha384Config =
- { ICP_QAT_HW_AUTH_ALGO_SHA384,
- LAC_HASH_SHA384_BLOCK_SIZE,
- ICP_QAT_HW_SHA384_STATE1_SZ,
- ICP_QAT_HW_SHA384_STATE2_SZ };
-
-static lac_sym_qat_hash_qat_info_t sha512Config =
- { ICP_QAT_HW_AUTH_ALGO_SHA512,
- LAC_HASH_SHA512_BLOCK_SIZE,
- ICP_QAT_HW_SHA512_STATE1_SZ,
- ICP_QAT_HW_SHA512_STATE2_SZ };
-
-static lac_sym_qat_hash_qat_info_t sha3_224Config =
- { ICP_QAT_HW_AUTH_ALGO_SHA3_224,
- LAC_HASH_SHA3_224_BLOCK_SIZE,
- ICP_QAT_HW_SHA3_224_STATE1_SZ,
- ICP_QAT_HW_SHA3_224_STATE2_SZ };
-
-static lac_sym_qat_hash_qat_info_t sha3_256Config =
- { ICP_QAT_HW_AUTH_ALGO_SHA3_256,
- LAC_HASH_SHA3_256_BLOCK_SIZE,
- ICP_QAT_HW_SHA3_256_STATE1_SZ,
- ICP_QAT_HW_SHA3_256_STATE2_SZ };
-
-static lac_sym_qat_hash_qat_info_t sha3_384Config =
- { ICP_QAT_HW_AUTH_ALGO_SHA3_384,
- LAC_HASH_SHA3_384_BLOCK_SIZE,
- ICP_QAT_HW_SHA3_384_STATE1_SZ,
- ICP_QAT_HW_SHA3_384_STATE2_SZ };
-
-static lac_sym_qat_hash_qat_info_t sha3_512Config =
- { ICP_QAT_HW_AUTH_ALGO_SHA3_512,
- LAC_HASH_SHA3_512_BLOCK_SIZE,
- ICP_QAT_HW_SHA3_512_STATE1_SZ,
- ICP_QAT_HW_SHA3_512_STATE2_SZ };
-
-static lac_sym_qat_hash_qat_info_t shake_128Config =
- { ICP_QAT_HW_AUTH_ALGO_SHAKE_128, LAC_HASH_SHAKE_128_BLOCK_SIZE, 0, 0 };
-
-static lac_sym_qat_hash_qat_info_t shake_256Config =
- { ICP_QAT_HW_AUTH_ALGO_SHAKE_256, LAC_HASH_SHAKE_256_BLOCK_SIZE, 0, 0 };
+static lac_sym_qat_hash_qat_info_t sha224Config = {
+ ICP_QAT_HW_AUTH_ALGO_SHA224,
+ LAC_HASH_SHA224_BLOCK_SIZE,
+ ICP_QAT_HW_SHA224_STATE1_SZ,
+ ICP_QAT_HW_SHA224_STATE2_SZ
+};
-static lac_sym_qat_hash_qat_info_t polyConfig = { ICP_QAT_HW_AUTH_ALGO_POLY,
- LAC_HASH_POLY_BLOCK_SIZE,
- 0,
- 0 };
+static lac_sym_qat_hash_qat_info_t sha256Config = {
+ ICP_QAT_HW_AUTH_ALGO_SHA256,
+ LAC_HASH_SHA256_BLOCK_SIZE,
+ ICP_QAT_HW_SHA256_STATE1_SZ,
+ ICP_QAT_HW_SHA256_STATE2_SZ
+};
+
+static lac_sym_qat_hash_qat_info_t sha384Config = {
+ ICP_QAT_HW_AUTH_ALGO_SHA384,
+ LAC_HASH_SHA384_BLOCK_SIZE,
+ ICP_QAT_HW_SHA384_STATE1_SZ,
+ ICP_QAT_HW_SHA384_STATE2_SZ
+};
+
+static lac_sym_qat_hash_qat_info_t sha512Config = {
+ ICP_QAT_HW_AUTH_ALGO_SHA512,
+ LAC_HASH_SHA512_BLOCK_SIZE,
+ ICP_QAT_HW_SHA512_STATE1_SZ,
+ ICP_QAT_HW_SHA512_STATE2_SZ
+};
+
+static lac_sym_qat_hash_qat_info_t sha3_224Config = {
+ ICP_QAT_HW_AUTH_ALGO_SHA3_224,
+ LAC_HASH_SHA3_224_BLOCK_SIZE,
+ ICP_QAT_HW_SHA3_224_STATE1_SZ,
+ ICP_QAT_HW_SHA3_224_STATE2_SZ
+};
+
+static lac_sym_qat_hash_qat_info_t sha3_256Config = {
+ ICP_QAT_HW_AUTH_ALGO_SHA3_256,
+ LAC_HASH_SHA3_256_BLOCK_SIZE,
+ ICP_QAT_HW_SHA3_256_STATE1_SZ,
+ ICP_QAT_HW_SHA3_256_STATE2_SZ
+};
+
+static lac_sym_qat_hash_qat_info_t sha3_384Config = {
+ ICP_QAT_HW_AUTH_ALGO_SHA3_384,
+ LAC_HASH_SHA3_384_BLOCK_SIZE,
+ ICP_QAT_HW_SHA3_384_STATE1_SZ,
+ ICP_QAT_HW_SHA3_384_STATE2_SZ
+};
+
+static lac_sym_qat_hash_qat_info_t sha3_512Config = {
+ ICP_QAT_HW_AUTH_ALGO_SHA3_512,
+ LAC_HASH_SHA3_512_BLOCK_SIZE,
+ ICP_QAT_HW_SHA3_512_STATE1_SZ,
+ ICP_QAT_HW_SHA3_512_STATE2_SZ
+};
static lac_sym_qat_hash_qat_info_t sm3Config = { ICP_QAT_HW_AUTH_ALGO_SM3,
LAC_HASH_SM3_BLOCK_SIZE,
ICP_QAT_HW_SM3_STATE1_SZ,
ICP_QAT_HW_SM3_STATE2_SZ };
-static lac_sym_qat_hash_qat_info_t xcbcMacConfig =
- { ICP_QAT_HW_AUTH_ALGO_AES_XCBC_MAC,
- 0,
- ICP_QAT_HW_AES_XCBC_MAC_STATE1_SZ,
- LAC_SYM_QAT_XCBC_STATE_SIZE };
-
-static lac_sym_qat_hash_qat_info_t aesCmacConfig =
- { ICP_QAT_HW_AUTH_ALGO_AES_XCBC_MAC,
- 0,
- ICP_QAT_HW_AES_XCBC_MAC_STATE1_SZ,
- LAC_SYM_QAT_CMAC_STATE_SIZE };
-
-static lac_sym_qat_hash_qat_info_t aesCcmConfig =
- { ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC,
- 0,
- ICP_QAT_HW_AES_CBC_MAC_STATE1_SZ,
- ICP_QAT_HW_AES_CBC_MAC_KEY_SZ + ICP_QAT_HW_AES_CCM_CBC_E_CTR0_SZ };
-
-static lac_sym_qat_hash_qat_info_t aesGcmConfig =
- { ICP_QAT_HW_AUTH_ALGO_GALOIS_128,
- 0,
- ICP_QAT_HW_GALOIS_128_STATE1_SZ,
- ICP_QAT_HW_GALOIS_H_SZ + ICP_QAT_HW_GALOIS_LEN_A_SZ +
- ICP_QAT_HW_GALOIS_E_CTR0_SZ };
-
-static lac_sym_qat_hash_qat_info_t kasumiF9Config =
- { ICP_QAT_HW_AUTH_ALGO_KASUMI_F9,
- 0,
- ICP_QAT_HW_KASUMI_F9_STATE1_SZ,
- ICP_QAT_HW_KASUMI_F9_STATE2_SZ };
-
-static lac_sym_qat_hash_qat_info_t snow3gUia2Config =
- { ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2,
- 0,
- ICP_QAT_HW_SNOW_3G_UIA2_STATE1_SZ,
- ICP_QAT_HW_SNOW_3G_UIA2_STATE2_SZ };
-
-static lac_sym_qat_hash_qat_info_t aesCbcMacConfig =
- { ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC,
- 0,
- ICP_QAT_HW_AES_CBC_MAC_STATE1_SZ,
- ICP_QAT_HW_AES_CBC_MAC_STATE1_SZ + ICP_QAT_HW_AES_CBC_MAC_STATE1_SZ };
-
-static lac_sym_qat_hash_qat_info_t zucEia3Config =
- { ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3,
- 0,
- ICP_QAT_HW_ZUC_3G_EIA3_STATE1_SZ,
- ICP_QAT_HW_ZUC_3G_EIA3_STATE2_SZ };
+static lac_sym_qat_hash_qat_info_t polyConfig = { ICP_QAT_HW_AUTH_ALGO_POLY,
+ LAC_HASH_POLY_BLOCK_SIZE,
+ 0,
+ 0 };
+
+static lac_sym_qat_hash_qat_info_t xcbcMacConfig = {
+ ICP_QAT_HW_AUTH_ALGO_AES_XCBC_MAC,
+ 0,
+ ICP_QAT_HW_AES_XCBC_MAC_STATE1_SZ,
+ LAC_SYM_QAT_XCBC_STATE_SIZE
+};
+
+static lac_sym_qat_hash_qat_info_t aesCmacConfig = {
+ ICP_QAT_HW_AUTH_ALGO_AES_XCBC_MAC,
+ 0,
+ ICP_QAT_HW_AES_XCBC_MAC_STATE1_SZ,
+ LAC_SYM_QAT_CMAC_STATE_SIZE
+};
+
+static lac_sym_qat_hash_qat_info_t aesCcmConfig = {
+ ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC,
+ 0,
+ ICP_QAT_HW_AES_CBC_MAC_STATE1_SZ,
+ ICP_QAT_HW_AES_CBC_MAC_KEY_SZ + ICP_QAT_HW_AES_CCM_CBC_E_CTR0_SZ
+};
+
+static lac_sym_qat_hash_qat_info_t aesGcmConfig = {
+ ICP_QAT_HW_AUTH_ALGO_GALOIS_128,
+ 0,
+ ICP_QAT_HW_GALOIS_128_STATE1_SZ,
+ ICP_QAT_HW_GALOIS_H_SZ + ICP_QAT_HW_GALOIS_LEN_A_SZ +
+ ICP_QAT_HW_GALOIS_E_CTR0_SZ
+};
+
+static lac_sym_qat_hash_qat_info_t kasumiF9Config = {
+ ICP_QAT_HW_AUTH_ALGO_KASUMI_F9,
+ 0,
+ ICP_QAT_HW_KASUMI_F9_STATE1_SZ,
+ ICP_QAT_HW_KASUMI_F9_STATE2_SZ
+};
+
+static lac_sym_qat_hash_qat_info_t snow3gUia2Config = {
+ ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2,
+ 0,
+ ICP_QAT_HW_SNOW_3G_UIA2_STATE1_SZ,
+ ICP_QAT_HW_SNOW_3G_UIA2_STATE2_SZ
+};
+
+static lac_sym_qat_hash_qat_info_t aesCbcMacConfig = {
+ ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC,
+ 0,
+ ICP_QAT_HW_AES_CBC_MAC_STATE1_SZ,
+ ICP_QAT_HW_AES_CBC_MAC_STATE1_SZ + ICP_QAT_HW_AES_CBC_MAC_STATE1_SZ
+};
+
+static lac_sym_qat_hash_qat_info_t zucEia3Config = {
+ ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3,
+ 0,
+ ICP_QAT_HW_ZUC_3G_EIA3_STATE1_SZ,
+ ICP_QAT_HW_ZUC_3G_EIA3_STATE2_SZ
+};
/* Array of mappings between algorithm and info structure
* This array is used to populate the lookup table */
-static lac_sym_qat_hash_def_map_t lacHashDefsMapping[] =
- { { CPA_CY_SYM_HASH_MD5, { &md5Info, &md5Config } },
- { CPA_CY_SYM_HASH_SHA1, { &sha1Info, &sha1Config } },
- { CPA_CY_SYM_HASH_SHA224, { &sha224Info, &sha224Config } },
- { CPA_CY_SYM_HASH_SHA256, { &sha256Info, &sha256Config } },
- { CPA_CY_SYM_HASH_SHA384, { &sha384Info, &sha384Config } },
- { CPA_CY_SYM_HASH_SHA512, { &sha512Info, &sha512Config } },
- { CPA_CY_SYM_HASH_SHA3_224, { &sha3_224Info, &sha3_224Config } },
- { CPA_CY_SYM_HASH_SHA3_256, { &sha3_256Info, &sha3_256Config } },
- { CPA_CY_SYM_HASH_SHA3_384, { &sha3_384Info, &sha3_384Config } },
- { CPA_CY_SYM_HASH_SHA3_512, { &sha3_512Info, &sha3_512Config } },
- { CPA_CY_SYM_HASH_SHAKE_128, { &shake_128Info, &shake_128Config } },
- { CPA_CY_SYM_HASH_SHAKE_256, { &shake_256Info, &shake_256Config } },
- { CPA_CY_SYM_HASH_POLY, { &polyInfo, &polyConfig } },
- { CPA_CY_SYM_HASH_SM3, { &sm3Info, &sm3Config } },
- { CPA_CY_SYM_HASH_AES_XCBC, { &xcbcMacInfo, &xcbcMacConfig } },
- { CPA_CY_SYM_HASH_AES_CMAC, { &aesCmacInfo, &aesCmacConfig } },
- { CPA_CY_SYM_HASH_AES_CCM, { &aesCcmInfo, &aesCcmConfig } },
- { CPA_CY_SYM_HASH_AES_GCM, { &aesGcmInfo, &aesGcmConfig } },
- { CPA_CY_SYM_HASH_KASUMI_F9, { &kasumiF9Info, &kasumiF9Config } },
- { CPA_CY_SYM_HASH_SNOW3G_UIA2, { &snow3gUia2Info, &snow3gUia2Config } },
- { CPA_CY_SYM_HASH_AES_GMAC, { &aesGcmInfo, &aesGcmConfig } },
- { CPA_CY_SYM_HASH_ZUC_EIA3, { &zucEia3Info, &zucEia3Config } },
- { CPA_CY_SYM_HASH_AES_CBC_MAC, { &aesCbcMacInfo, &aesCbcMacConfig } } };
+static lac_sym_qat_hash_def_map_t lacHashDefsMapping[] = {
+ { CPA_CY_SYM_HASH_MD5, { &md5Info, &md5Config } },
+ { CPA_CY_SYM_HASH_SHA1, { &sha1Info, &sha1Config } },
+ { CPA_CY_SYM_HASH_SHA224, { &sha224Info, &sha224Config } },
+ { CPA_CY_SYM_HASH_SHA256, { &sha256Info, &sha256Config } },
+ { CPA_CY_SYM_HASH_SHA384, { &sha384Info, &sha384Config } },
+ { CPA_CY_SYM_HASH_SHA512, { &sha512Info, &sha512Config } },
+ { CPA_CY_SYM_HASH_SHA3_224, { &sha3_224Info, &sha3_224Config } },
+ { CPA_CY_SYM_HASH_SHA3_256, { &sha3_256Info, &sha3_256Config } },
+ { CPA_CY_SYM_HASH_SHA3_384, { &sha3_384Info, &sha3_384Config } },
+ { CPA_CY_SYM_HASH_SHA3_512, { &sha3_512Info, &sha3_512Config } },
+ { CPA_CY_SYM_HASH_SM3, { &sm3Info, &sm3Config } },
+ { CPA_CY_SYM_HASH_POLY, { &polyInfo, &polyConfig } },
+ { CPA_CY_SYM_HASH_AES_XCBC, { &xcbcMacInfo, &xcbcMacConfig } },
+ { CPA_CY_SYM_HASH_AES_CMAC, { &aesCmacInfo, &aesCmacConfig } },
+ { CPA_CY_SYM_HASH_AES_CCM, { &aesCcmInfo, &aesCcmConfig } },
+ { CPA_CY_SYM_HASH_AES_GCM, { &aesGcmInfo, &aesGcmConfig } },
+ { CPA_CY_SYM_HASH_KASUMI_F9, { &kasumiF9Info, &kasumiF9Config } },
+ { CPA_CY_SYM_HASH_SNOW3G_UIA2, { &snow3gUia2Info, &snow3gUia2Config } },
+ { CPA_CY_SYM_HASH_AES_GMAC, { &aesGcmInfo, &aesGcmConfig } },
+ { CPA_CY_SYM_HASH_ZUC_EIA3, { &zucEia3Info, &zucEia3Config } },
+ { CPA_CY_SYM_HASH_AES_CBC_MAC, { &aesCbcMacInfo, &aesCbcMacConfig } }
+};
/*
* LacSymQat_HashLookupInit
@@ -434,33 +450,35 @@
Cpa32U arraySize = 0;
CpaStatus status = CPA_STATUS_SUCCESS;
CpaCySymHashAlgorithm hashAlg = CPA_CY_SYM_HASH_NONE;
- sal_crypto_service_t *pService = (sal_crypto_service_t *)instanceHandle;
+ sal_service_t *pService = (sal_service_t *)instanceHandle;
+ lac_sym_qat_hash_defs_t **pLacHashLookupDefs;
arraySize =
(CPA_CY_HASH_ALG_END + 1) * sizeof(lac_sym_qat_hash_defs_t *);
/* Size round up for performance */
arraySize = LAC_ALIGN_POW2_ROUNDUP(arraySize, LAC_64BYTE_ALIGNMENT);
- pService->pLacHashLookupDefs = LAC_OS_MALLOC(arraySize);
-
- if (NULL != pService->pLacHashLookupDefs) {
- LAC_OS_BZERO(pService->pLacHashLookupDefs, arraySize);
+ pLacHashLookupDefs = LAC_OS_MALLOC(arraySize);
+ if (NULL == pLacHashLookupDefs) {
+ return CPA_STATUS_RESOURCE;
+ }
- numEntries = sizeof(lacHashDefsMapping) /
- sizeof(lac_sym_qat_hash_def_map_t);
+ LAC_OS_BZERO(pLacHashLookupDefs, arraySize);
+ numEntries =
+ sizeof(lacHashDefsMapping) / sizeof(lac_sym_qat_hash_def_map_t);
- /* initialise the hash lookup definitions table so that the
- * algorithm
- * can be used to index into the table */
- for (entry = 0; entry < numEntries; entry++) {
- hashAlg = lacHashDefsMapping[entry].hashAlgorithm;
+ /* initialise the hash lookup definitions table so that the algorithm
+ * can be used to index into the table */
+ for (entry = 0; entry < numEntries; entry++) {
+ hashAlg = lacHashDefsMapping[entry].hashAlgorithm;
- pService->pLacHashLookupDefs[hashAlg] =
- &(lacHashDefsMapping[entry].hashDefs);
- }
- } else {
- status = CPA_STATUS_RESOURCE;
+ pLacHashLookupDefs[hashAlg] =
+ &(lacHashDefsMapping[entry].hashDefs);
}
+
+ ((sal_crypto_service_t *)pService)->pLacHashLookupDefs =
+ pLacHashLookupDefs;
+
return status;
}
@@ -472,9 +490,11 @@
CpaCySymHashAlgorithm hashAlgorithm,
lac_sym_qat_hash_alg_info_t **ppHashAlgInfo)
{
- sal_crypto_service_t *pService = (sal_crypto_service_t *)instanceHandle;
+ sal_service_t *pService = (sal_service_t *)instanceHandle;
+ lac_sym_qat_hash_defs_t **pLacHashLookupDefs =
+ ((sal_crypto_service_t *)pService)->pLacHashLookupDefs;
- *ppHashAlgInfo = pService->pLacHashLookupDefs[hashAlgorithm]->algInfo;
+ *ppHashAlgInfo = pLacHashLookupDefs[hashAlgorithm]->algInfo;
}
/*
@@ -485,7 +505,9 @@
CpaCySymHashAlgorithm hashAlgorithm,
lac_sym_qat_hash_defs_t **ppHashDefsInfo)
{
- sal_crypto_service_t *pService = (sal_crypto_service_t *)instanceHandle;
+ sal_service_t *pService = (sal_service_t *)instanceHandle;
+ lac_sym_qat_hash_defs_t **pLacHashLookupDefs =
+ ((sal_crypto_service_t *)pService)->pLacHashLookupDefs;
- *ppHashDefsInfo = pService->pLacHashLookupDefs[hashAlgorithm];
+ *ppHashDefsInfo = pLacHashLookupDefs[hashAlgorithm];
}
Index: sys/dev/qat/qat_api/common/ctrl/sal_compression.c
===================================================================
--- sys/dev/qat/qat_api/common/ctrl/sal_compression.c
+++ sys/dev/qat/qat_api/common/ctrl/sal_compression.c
@@ -47,6 +47,7 @@
#include "sal_service_state.h"
#include "lac_buffer_desc.h"
#include "icp_qat_fw_comp.h"
+#include "icp_qat_hw_20_comp_defs.h"
#include "icp_sal_versions.h"
/* C string null terminator size */
@@ -124,6 +125,10 @@
SalCtrl_CompressionInit_CompData(icp_accel_dev_t *device,
sal_compression_service_t *pCompService)
{
+ int level = 0;
+
+ pCompService->comp_device_data.uniqueCompressionLevels[0] = CPA_FALSE;
+
switch (device->deviceType) {
case DEVICE_DH895XCC:
case DEVICE_DH895XCCVF:
@@ -191,13 +196,36 @@
ICP_QAT_FW_COMP_ENABLE_SECURE_RAM_USED_AS_INTMD_BUF;
pCompService->comp_device_data.inflateContextSize =
DC_INFLATE_EH_CONTEXT_SIZE;
+ pCompService->comp_device_data.highestHwCompressionDepth =
+ ICP_QAT_HW_COMPRESSION_DEPTH_16;
pCompService->comp_device_data.windowSizeMask =
- (1 << DC_16K_WINDOW_SIZE | 1 << DC_32K_WINDOW_SIZE);
+ (1 << DC_4K_WINDOW_SIZE | 1 << DC_8K_WINDOW_SIZE |
+ 1 << DC_16K_WINDOW_SIZE | 1 << DC_32K_WINDOW_SIZE);
pCompService->comp_device_data.minOutputBuffSize =
DC_DEST_BUFFER_STA_MIN_SIZE;
+ pCompService->comp_device_data.minOutputBuffSizeDynamic =
+ pCompService->comp_device_data.minOutputBuffSize;
pCompService->comp_device_data.enableDmm =
ICP_QAT_HW_COMPRESSION_DELAYED_MATCH_ENABLED;
pCompService->comp_device_data.cnvnrSupported = CPA_TRUE;
+
+ for (level = CPA_DC_L1; level <= CPA_DC_L9; level++) {
+ switch (level) {
+ case CPA_DC_L1:
+ case CPA_DC_L2:
+ case CPA_DC_L3:
+ case CPA_DC_L4:
+ pCompService->comp_device_data
+ .uniqueCompressionLevels[level] = CPA_TRUE;
+ break;
+ default:
+ pCompService->comp_device_data
+ .uniqueCompressionLevels[level] = CPA_FALSE;
+ break;
+ }
+ }
+ pCompService->comp_device_data.numCompressionLevels =
+ DC_NUM_COMPRESSION_LEVELS;
break;
case DEVICE_C4XXX:
case DEVICE_C4XXXVF:
@@ -227,6 +255,46 @@
(1 << DC_16K_WINDOW_SIZE | 1 << DC_32K_WINDOW_SIZE);
pCompService->comp_device_data.cnvnrSupported = CPA_TRUE;
break;
+ case DEVICE_GEN4:
+ pCompService->generic_service_info.integrityCrcCheck = CPA_TRUE;
+ pCompService->numInterBuffs =
+ DC_QAT_MAX_NUM_INTER_BUFFERS_6COMP_SLICES;
+ pCompService->comp_device_data.minOutputBuffSize =
+ DC_DEST_BUFFER_STA_MIN_SIZE_GEN4;
+ pCompService->comp_device_data.minOutputBuffSizeDynamic =
+ DC_DEST_BUFFER_DYN_MIN_SIZE_GEN4;
+ pCompService->comp_device_data.oddByteDecompNobFinal = CPA_TRUE;
+ pCompService->comp_device_data.oddByteDecompInterim = CPA_FALSE;
+ pCompService->comp_device_data.translatorOverflow = CPA_TRUE;
+ pCompService->comp_device_data.useDevRam =
+ ICP_QAT_FW_COMP_ENABLE_SECURE_RAM_USED_AS_INTMD_BUF;
+ pCompService->comp_device_data.enableDmm =
+ ICP_QAT_HW_COMPRESSION_DELAYED_MATCH_ENABLED;
+
+ pCompService->comp_device_data.inflateContextSize =
+ DC_INFLATE_CONTEXT_SIZE;
+ pCompService->comp_device_data.highestHwCompressionDepth =
+ ICP_QAT_HW_COMP_20_SEARCH_DEPTH_LEVEL_9;
+ pCompService->comp_device_data.windowSizeMask =
+ (1 << DC_4K_WINDOW_SIZE | 1 << DC_8K_WINDOW_SIZE |
+ 1 << DC_16K_WINDOW_SIZE | 1 << DC_32K_WINDOW_SIZE);
+ for (level = CPA_DC_L1; level <= CPA_DC_L9; level++) {
+ switch (level) {
+ case CPA_DC_L1:
+ case CPA_DC_L6:
+ case CPA_DC_L9:
+ pCompService->comp_device_data
+ .uniqueCompressionLevels[level] = CPA_TRUE;
+ break;
+ default:
+ pCompService->comp_device_data
+ .uniqueCompressionLevels[level] = CPA_FALSE;
+ break;
+ }
+ }
+ pCompService->comp_device_data.numCompressionLevels =
+ DC_NUM_COMPRESSION_LEVELS;
+ break;
default:
QAT_UTILS_LOG("Unknown device type! - %d.\n",
device->deviceType);
Index: sys/dev/qat/qat_api/common/ctrl/sal_crypto.c
===================================================================
--- sys/dev/qat/qat_api/common/ctrl/sal_crypto.c
+++ sys/dev/qat/qat_api/common/ctrl/sal_crypto.c
@@ -59,7 +59,10 @@
#include "lac_sym_qat.h"
#include "icp_sal_versions.h"
#include "icp_sal_user.h"
+#include "sal_hw_gen.h"
+#define HMAC_MODE_1 1
+#define HMAC_MODE_2 2
#define TH_CY_RX_0 0
#define TH_CY_RX_1 1
#define MAX_CY_RX_RINGS 2
@@ -211,7 +214,7 @@
ICP_TRANS_TYPE_ETR,
section,
pCryptoService->acceleratorNum,
- pCryptoService->bankNum,
+ pCryptoService->bankNumSym,
temp_string,
lac_getRingType(SAL_RING_TYPE_A_SYM_HI),
NULL,
@@ -235,7 +238,7 @@
ICP_TRANS_TYPE_ETR,
section,
pCryptoService->acceleratorNum,
- pCryptoService->bankNum,
+ pCryptoService->bankNumSym,
temp_string,
lac_getRingType(SAL_RING_TYPE_NONE),
(icp_trans_callback)LacSymQat_SymRespHandler,
@@ -328,6 +331,7 @@
SalCtrl_SymInit(icp_accel_dev_t *device, sal_service_t *service)
{
CpaStatus status = CPA_STATUS_SUCCESS;
+ Cpa32U qatHmacMode = 0;
Cpa32U numSymConcurrentReq = 0;
char adfGetParam[ADF_CFG_MAX_VAL_LEN_IN_BYTES] = { 0 };
char temp_string[SAL_CFG_MAX_VAL_LEN_IN_BYTES] = { 0 };
@@ -344,6 +348,19 @@
* (Hash, Cipher, Algorithm-Chaining) (returns void)*/
LacSymCb_CallbacksRegister();
+ qatHmacMode = (Cpa32U)Sal_Strtoul(adfGetParam, NULL, SAL_CFG_BASE_DEC);
+ switch (qatHmacMode) {
+ case HMAC_MODE_1:
+ pCryptoService->qatHmacMode = ICP_QAT_HW_AUTH_MODE1;
+ break;
+ case HMAC_MODE_2:
+ pCryptoService->qatHmacMode = ICP_QAT_HW_AUTH_MODE2;
+ break;
+ default:
+ pCryptoService->qatHmacMode = ICP_QAT_HW_AUTH_MODE1;
+ break;
+ }
+
/* Get num concurrent requests from config file */
status =
Sal_StringParsing("Cy",
@@ -524,6 +541,32 @@
return status;
}
+static CpaStatus
+SalCtrl_GetBankNum(icp_accel_dev_t *device,
+ Cpa32U inst,
+ char *section,
+ char *bank_name,
+ Cpa16U *bank)
+{
+ char adfParamValue[ADF_CFG_MAX_VAL_LEN_IN_BYTES] = { 0 };
+ char adfParamName[SAL_CFG_MAX_VAL_LEN_IN_BYTES] = { 0 };
+ CpaStatus status = CPA_STATUS_SUCCESS;
+
+ status = Sal_StringParsing("Cy", inst, bank_name, adfParamName);
+ LAC_CHECK_STATUS(status);
+ status = icp_adf_cfgGetParamValue(device,
+ section,
+ adfParamName,
+ adfParamValue);
+ if (CPA_STATUS_SUCCESS != status) {
+ QAT_UTILS_LOG("Failed to get %s from configuration file\n",
+ adfParamName);
+ return status;
+ }
+ *bank = (Cpa16U)Sal_Strtoul(adfParamValue, NULL, SAL_CFG_BASE_DEC);
+ return status;
+}
+
static CpaStatus
SalCtr_InstInit(icp_accel_dev_t *device, sal_service_t *service)
{
@@ -545,21 +588,62 @@
pCryptoService->acceleratorNum = 0;
- status =
- Sal_StringParsing("Cy",
- pCryptoService->generic_service_info.instance,
- "BankNumber",
- temp_string);
- LAC_CHECK_STATUS(status);
- status =
- icp_adf_cfgGetParamValue(device, section, temp_string, adfGetParam);
- if (CPA_STATUS_SUCCESS != status) {
- QAT_UTILS_LOG("Failed to get %s from configuration file\n",
- temp_string);
- return status;
+ /* Gen4, a bank only has 2 rings (1 ring pair), only one type of service
+ can be assigned one time. asym and sym will be in different bank*/
+ if (isCyGen4x(pCryptoService)) {
+ switch (service->type) {
+ case SAL_SERVICE_TYPE_CRYPTO_ASYM:
+ status = SalCtrl_GetBankNum(
+ device,
+ pCryptoService->generic_service_info.instance,
+ section,
+ "BankNumberAsym",
+ &pCryptoService->bankNumAsym);
+ if (CPA_STATUS_SUCCESS != status)
+ return status;
+ break;
+ case SAL_SERVICE_TYPE_CRYPTO_SYM:
+ status = SalCtrl_GetBankNum(
+ device,
+ pCryptoService->generic_service_info.instance,
+ section,
+ "BankNumberSym",
+ &pCryptoService->bankNumSym);
+ if (CPA_STATUS_SUCCESS != status)
+ return status;
+ break;
+ case SAL_SERVICE_TYPE_CRYPTO:
+ status = SalCtrl_GetBankNum(
+ device,
+ pCryptoService->generic_service_info.instance,
+ section,
+ "BankNumberAsym",
+ &pCryptoService->bankNumAsym);
+ if (CPA_STATUS_SUCCESS != status)
+ return status;
+ status = SalCtrl_GetBankNum(
+ device,
+ pCryptoService->generic_service_info.instance,
+ section,
+ "BankNumberSym",
+ &pCryptoService->bankNumSym);
+ if (CPA_STATUS_SUCCESS != status)
+ return status;
+ break;
+ default:
+ return CPA_STATUS_FAIL;
+ }
+ } else {
+ status = SalCtrl_GetBankNum(
+ device,
+ pCryptoService->generic_service_info.instance,
+ section,
+ "BankNumber",
+ &pCryptoService->bankNumSym);
+ if (CPA_STATUS_SUCCESS != status)
+ return status;
+ pCryptoService->bankNumAsym = pCryptoService->bankNumSym;
}
- pCryptoService->bankNum =
- (Cpa16U)Sal_Strtoul(adfGetParam, NULL, SAL_CFG_BASE_DEC);
status =
Sal_StringParsing("Cy",
@@ -619,10 +703,18 @@
"",
temp_string2);
LAC_CHECK_STATUS(status);
- status = Sal_StringParsing("Bank",
- pCryptoService->bankNum,
- "CoreAffinity",
- temp_string);
+ if (service->type == SAL_SERVICE_TYPE_CRYPTO_ASYM)
+ status = Sal_StringParsing("Bank",
+ pCryptoService->bankNumAsym,
+ "CoreAffinity",
+ temp_string);
+ else
+ /* For cy service, asym bank and sym bank will set the
+ same core affinity. So Just read one*/
+ status = Sal_StringParsing("Bank",
+ pCryptoService->bankNumSym,
+ "CoreAffinity",
+ temp_string);
LAC_CHECK_STATUS(status);
} else {
strncpy(temp_string2, section, (strlen(section) + 1));
@@ -817,6 +909,9 @@
case CPA_STATUS_FATAL:
LAC_COPY_STRING(pStatusText, CPA_STATUS_STR_FATAL);
break;
+ case CPA_STATUS_UNSUPPORTED:
+ LAC_COPY_STRING(pStatusText, CPA_STATUS_STR_UNSUPPORTED);
+ break;
default:
status = CPA_STATUS_INVALID_PARAM;
break;
@@ -886,6 +981,10 @@
instanceHandle = instanceHandle_in;
}
LAC_CHECK_NULL_PARAM(instanceHandle);
+ SAL_CHECK_INSTANCE_TYPE(instanceHandle,
+ (SAL_SERVICE_TYPE_CRYPTO |
+ SAL_SERVICE_TYPE_CRYPTO_ASYM |
+ SAL_SERVICE_TYPE_CRYPTO_SYM));
pService = (sal_crypto_service_t *)instanceHandle;
@@ -930,6 +1029,10 @@
instanceHandle = instanceHandle_in;
}
LAC_CHECK_NULL_PARAM(instanceHandle);
+ SAL_CHECK_INSTANCE_TYPE(instanceHandle,
+ (SAL_SERVICE_TYPE_CRYPTO |
+ SAL_SERVICE_TYPE_CRYPTO_ASYM |
+ SAL_SERVICE_TYPE_CRYPTO_SYM));
status = cpaCyInstanceGetInfo2(instanceHandle, &info);
if (CPA_STATUS_SUCCESS != status) {
@@ -1431,23 +1534,31 @@
}
CPA_BITMAP_BIT_SET(pCapInfo->ciphers, CPA_CY_SYM_CIPHER_NULL);
- CPA_BITMAP_BIT_SET(pCapInfo->ciphers, CPA_CY_SYM_CIPHER_ARC4);
CPA_BITMAP_BIT_SET(pCapInfo->ciphers, CPA_CY_SYM_CIPHER_AES_ECB);
CPA_BITMAP_BIT_SET(pCapInfo->ciphers, CPA_CY_SYM_CIPHER_AES_CBC);
CPA_BITMAP_BIT_SET(pCapInfo->ciphers, CPA_CY_SYM_CIPHER_AES_CTR);
CPA_BITMAP_BIT_SET(pCapInfo->ciphers, CPA_CY_SYM_CIPHER_AES_CCM);
CPA_BITMAP_BIT_SET(pCapInfo->ciphers, CPA_CY_SYM_CIPHER_AES_GCM);
- CPA_BITMAP_BIT_SET(pCapInfo->ciphers, CPA_CY_SYM_CIPHER_DES_ECB);
- CPA_BITMAP_BIT_SET(pCapInfo->ciphers, CPA_CY_SYM_CIPHER_DES_CBC);
- CPA_BITMAP_BIT_SET(pCapInfo->ciphers, CPA_CY_SYM_CIPHER_3DES_ECB);
- CPA_BITMAP_BIT_SET(pCapInfo->ciphers, CPA_CY_SYM_CIPHER_3DES_CBC);
- CPA_BITMAP_BIT_SET(pCapInfo->ciphers, CPA_CY_SYM_CIPHER_3DES_CTR);
- CPA_BITMAP_BIT_SET(pCapInfo->ciphers, CPA_CY_SYM_CIPHER_KASUMI_F8);
- CPA_BITMAP_BIT_SET(pCapInfo->ciphers, CPA_CY_SYM_CIPHER_SNOW3G_UEA2);
- CPA_BITMAP_BIT_SET(pCapInfo->ciphers, CPA_CY_SYM_CIPHER_AES_F8);
CPA_BITMAP_BIT_SET(pCapInfo->ciphers, CPA_CY_SYM_CIPHER_AES_XTS);
+ if (isCyGen2x(pCryptoService)) {
+ CPA_BITMAP_BIT_SET(pCapInfo->ciphers, CPA_CY_SYM_CIPHER_ARC4);
+ CPA_BITMAP_BIT_SET(pCapInfo->ciphers,
+ CPA_CY_SYM_CIPHER_DES_ECB);
+ CPA_BITMAP_BIT_SET(pCapInfo->ciphers,
+ CPA_CY_SYM_CIPHER_DES_CBC);
+ CPA_BITMAP_BIT_SET(pCapInfo->ciphers,
+ CPA_CY_SYM_CIPHER_3DES_ECB);
+ CPA_BITMAP_BIT_SET(pCapInfo->ciphers,
+ CPA_CY_SYM_CIPHER_3DES_CBC);
+ CPA_BITMAP_BIT_SET(pCapInfo->ciphers,
+ CPA_CY_SYM_CIPHER_3DES_CTR);
+ CPA_BITMAP_BIT_SET(pCapInfo->ciphers,
+ CPA_CY_SYM_CIPHER_KASUMI_F8);
+ CPA_BITMAP_BIT_SET(pCapInfo->ciphers,
+ CPA_CY_SYM_CIPHER_SNOW3G_UEA2);
+ CPA_BITMAP_BIT_SET(pCapInfo->ciphers, CPA_CY_SYM_CIPHER_AES_F8);
+ }
- CPA_BITMAP_BIT_SET(pCapInfo->hashes, CPA_CY_SYM_HASH_MD5);
CPA_BITMAP_BIT_SET(pCapInfo->hashes, CPA_CY_SYM_HASH_SHA1);
CPA_BITMAP_BIT_SET(pCapInfo->hashes, CPA_CY_SYM_HASH_SHA224);
CPA_BITMAP_BIT_SET(pCapInfo->hashes, CPA_CY_SYM_HASH_SHA256);
@@ -1456,11 +1567,15 @@
CPA_BITMAP_BIT_SET(pCapInfo->hashes, CPA_CY_SYM_HASH_AES_XCBC);
CPA_BITMAP_BIT_SET(pCapInfo->hashes, CPA_CY_SYM_HASH_AES_CCM);
CPA_BITMAP_BIT_SET(pCapInfo->hashes, CPA_CY_SYM_HASH_AES_GCM);
- CPA_BITMAP_BIT_SET(pCapInfo->hashes, CPA_CY_SYM_HASH_KASUMI_F9);
- CPA_BITMAP_BIT_SET(pCapInfo->hashes, CPA_CY_SYM_HASH_SNOW3G_UIA2);
CPA_BITMAP_BIT_SET(pCapInfo->hashes, CPA_CY_SYM_HASH_AES_CMAC);
CPA_BITMAP_BIT_SET(pCapInfo->hashes, CPA_CY_SYM_HASH_AES_GMAC);
CPA_BITMAP_BIT_SET(pCapInfo->hashes, CPA_CY_SYM_HASH_AES_CBC_MAC);
+ if (isCyGen2x(pCryptoService)) {
+ CPA_BITMAP_BIT_SET(pCapInfo->hashes, CPA_CY_SYM_HASH_MD5);
+ CPA_BITMAP_BIT_SET(pCapInfo->hashes, CPA_CY_SYM_HASH_KASUMI_F9);
+ CPA_BITMAP_BIT_SET(pCapInfo->hashes,
+ CPA_CY_SYM_HASH_SNOW3G_UIA2);
+ }
if (pGenericService->capabilitiesMask &
ICP_ACCEL_CAPABILITIES_CRYPTO_ZUC) {
Index: sys/dev/qat/qat_api/common/ctrl/sal_ctrl_services.c
===================================================================
--- sys/dev/qat/qat_api/common/ctrl/sal_ctrl_services.c
+++ sys/dev/qat/qat_api/common/ctrl/sal_ctrl_services.c
@@ -458,6 +458,37 @@
return status;
}
+static CpaStatus
+selectGeneration(device_type_t deviceType, sal_service_t *pInst)
+{
+ switch (deviceType) {
+ case DEVICE_C62X:
+ case DEVICE_C62XVF:
+ case DEVICE_DH895XCC:
+ case DEVICE_DH895XCCVF:
+ case DEVICE_C3XXX:
+ case DEVICE_C3XXXVF:
+ case DEVICE_200XX:
+ case DEVICE_200XXVF:
+ pInst->gen = GEN2;
+ break;
+
+ case DEVICE_C4XXX:
+ case DEVICE_C4XXXVF:
+ pInst->gen = GEN3;
+ break;
+
+ case DEVICE_GEN4:
+ pInst->gen = GEN4;
+ break;
+
+ default:
+ QAT_UTILS_LOG("deviceType not initialised\n");
+ return CPA_STATUS_FAIL;
+ }
+ return CPA_STATUS_SUCCESS;
+}
+
/*************************************************************************
* @ingroup SalCtrl
* @description
@@ -523,7 +554,12 @@
}
pInst->debug_parent_dir = debug_dir;
pInst->capabilitiesMask = device->accelCapabilitiesMask;
- status = SalList_add(services, &tail_list, pInst);
+
+ status = selectGeneration(device->deviceType, pInst);
+ if (CPA_STATUS_SUCCESS == status) {
+ status =
+ SalList_add(services, &tail_list, pInst);
+ }
if (CPA_STATUS_SUCCESS != status) {
free(pInst, M_QAT);
}
Index: sys/dev/qat/qat_api/common/include/lac_common.h
===================================================================
--- sys/dev/qat/qat_api/common/include/lac_common.h
+++ sys/dev/qat/qat_api/common/include/lac_common.h
@@ -756,17 +756,14 @@
#define LAC_SPINLOCK(lock) \
({ \
(void)qatUtilsLock(lock); \
- CPA_STATUS_SUCCESS; \
})
#define LAC_SPINUNLOCK(lock) \
({ \
(void)qatUtilsUnlock(lock); \
- CPA_STATUS_SUCCESS; \
})
#define LAC_SPINLOCK_DESTROY(lock) \
({ \
(void)qatUtilsLockDestroy(lock); \
- CPA_STATUS_SUCCESS; \
})
#define LAC_CONST_PTR_CAST(castee) ((void *)(LAC_ARCH_UINT)(castee))
Index: sys/dev/qat/qat_api/common/include/lac_sal_types.h
===================================================================
--- sys/dev/qat/qat_api/common/include/lac_sal_types.h
+++ sys/dev/qat/qat_api/common/include/lac_sal_types.h
@@ -67,6 +67,17 @@
SAL_SERVICE_TYPE_QAT = 32
} sal_service_type_t;
+/**
+ *****************************************************************************
+ * @ingroup SalCtrl
+ * Device generations
+ *
+ * @description
+ * List in an enum all the QAT device generations.
+ *
+ *****************************************************************************/
+typedef enum { GEN2, GEN3, GEN4 } sal_generation_t;
+
/**
*****************************************************************************
* @ingroup SalCtrl
@@ -128,6 +139,9 @@
CpaBoolean integrityCrcCheck;
/** < True if the device supports end to end data integrity checks */
+
+ sal_generation_t gen;
+ /** Generation of devices */
} sal_service_t;
/**
Index: sys/dev/qat/qat_api/common/include/lac_sal_types_crypto.h
===================================================================
--- sys/dev/qat/qat_api/common/include/lac_sal_types_crypto.h
+++ sys/dev/qat/qat_api/common/include/lac_sal_types_crypto.h
@@ -16,6 +16,7 @@
#define LAC_SAL_TYPES_CRYPTO_H_
#include "lac_sym_qat_hash_defs_lookup.h"
+#include "lac_sym_qat_constants_table.h"
#include "lac_sym_key.h"
#include "cpa_cy_sym_dp.h"
@@ -84,6 +85,8 @@
QatUtilsAtomic *pLacDrbgStatsArr;
/**< pointer to an array of atomic stats for DRBG */
+ icp_qat_hw_auth_mode_t qatHmacMode;
+ /**< Hmac Mode */
Cpa32U pkeFlowId;
/**< Flow ID for all pke requests from this instance - identifies
@@ -105,6 +108,8 @@
Cpa16U acceleratorNum;
Cpa16U bankNum;
+ Cpa16U bankNumAsym;
+ Cpa16U bankNumSym;
Cpa16U pkgID;
Cpa8U isPolled;
Cpa8U executionEngine;
@@ -119,6 +124,9 @@
/**< table of pointers to standard defined information for all hash
algorithms. We support an extra hash algo that is not exported by
cy api which is why we need the extra +1 */
+
+ lac_sym_qat_constants_t constantsLookupTables;
+
Cpa8U **ppHmacContentDesc;
/**< table of pointers to CD for Hmac precomputes - used at session init
*/
Index: sys/dev/qat/qat_api/common/include/lac_sync.h
===================================================================
--- sys/dev/qat/qat_api/common/include/lac_sync.h
+++ sys/dev/qat/qat_api/common/include/lac_sync.h
@@ -61,7 +61,7 @@
* Timeout for wait for init messages response in msecs
*/
-#define DC_SYNC_CALLBACK_TIMEOUT (1000)
+#define DC_SYNC_CALLBACK_TIMEOUT (2000)
/**< @ingroup LacSyn
* Timeout for wait for compression response in msecs */
Index: sys/dev/qat/qat_api/common/include/sal_hw_gen.h
===================================================================
--- /dev/null
+++ sys/dev/qat/qat_api/common/include/sal_hw_gen.h
@@ -0,0 +1,88 @@
+/* SPDX-License-Identifier: BSD-3-Clause */
+/* Copyright(c) 2007-2022 Intel Corporation */
+/* $FreeBSD$ */
+/**
+ ***************************************************************************
+ * @file sal_hw_gen.h
+ *
+ * @ingroup SalHwGen
+ *
+ * @description
+ * Functions which return a value corresponding to qat device generation
+ *
+ ***************************************************************************/
+
+#ifndef SAL_HW_GEN_H
+#define SAL_HW_GEN_H
+
+#include "cpa.h"
+#include "sal_types_compression.h"
+#include "lac_sal_types_crypto.h"
+
+/**
+ ***************************************************************************
+ * @ingroup SalHwGen
+ *
+ * @description This function returns whether qat device is gen 4 or not
+ *
+ * @param[in] pService pointer to compression service
+ *
+ ***************************************************************************/
+
+static inline CpaBoolean
+isDcGen4x(const sal_compression_service_t *pService)
+{
+ return (pService->generic_service_info.gen == GEN4);
+}
+
+/**
+ ***************************************************************************
+ * @ingroup SalHwGen
+ *
+ * @description This function returns whether qat device is gen 2/3 or not
+ *
+ * @param[in] pService pointer to compression service
+ *
+ ***************************************************************************/
+
+static inline CpaBoolean
+isDcGen2x(const sal_compression_service_t *pService)
+{
+ return ((pService->generic_service_info.gen == GEN2) ||
+ (pService->generic_service_info.gen == GEN3));
+}
+
+/**
+ ***************************************************************************
+ * @ingroup SalHwGen
+ *
+ * @description This function returns whether qat device is gen 4 or not
+ *
+ * @param[in] pService pointer to crypto service
+ *
+ ***************************************************************************/
+
+static inline CpaBoolean
+isCyGen4x(const sal_crypto_service_t *pService)
+{
+ return (pService->generic_service_info.gen == GEN4);
+}
+
+/**
+ ***************************************************************************
+ * @ingroup SalHwGen
+ *
+ * @description This function returns whether qat device is gen 2/3 or not
+ *
+ * @param[in] pService pointer to crypto service
+ *
+ ***************************************************************************/
+
+static inline CpaBoolean
+isCyGen2x(const sal_crypto_service_t *pService)
+{
+ return ((pService->generic_service_info.gen == GEN2) ||
+ (pService->generic_service_info.gen == GEN3));
+}
+
+#endif /* SAL_HW_GEN_H */
Index: sys/dev/qat/qat_api/common/include/sal_types_compression.h
===================================================================
--- sys/dev/qat/qat_api/common/include/sal_types_compression.h
+++ sys/dev/qat/qat_api/common/include/sal_types_compression.h
@@ -23,6 +23,7 @@
#include "icp_adf_transport.h"
#define DC_NUM_RX_RINGS (1)
+#define DC_NUM_COMPRESSION_LEVELS (CPA_DC_L9)
/**
*****************************************************************************
@@ -37,6 +38,9 @@
/* Device specific minimum output buffer size for static compression */
Cpa32U minOutputBuffSize;
+ /* Device specific minimum output buffer size for dynamic compression */
+ Cpa32U minOutputBuffSizeDynamic;
+
/* Enable/disable secureRam/acceleratorRam for intermediate buffers*/
Cpa8U useDevRam;
@@ -62,6 +66,11 @@
/* Mask that reports supported window sizes for comp/decomp */
Cpa8U windowSizeMask;
+ /* List representing compression levels that are the first to have
+ a unique search depth. */
+ CpaBoolean uniqueCompressionLevels[DC_NUM_COMPRESSION_LEVELS + 1];
+ Cpa8U numCompressionLevels;
+
/* Flag to indicate CompressAndVerifyAndRecover feature support */
CpaBoolean cnvnrSupported;
} sal_compression_device_data_t;
Index: sys/dev/qat/qat_api/firmware/include/icp_qat_fw.h
===================================================================
--- sys/dev/qat/qat_api/firmware/include/icp_qat_fw.h
+++ sys/dev/qat/qat_api/firmware/include/icp_qat_fw.h
@@ -486,12 +486,12 @@
/* ========================================================================= */
/* Common QAT FW request header - structure of LW0
- * + ===== + ---- + ----------- + ----------- + ----------- + ----------- +
- * | Bit | 31 | 30 - 24 | 21 - 16 | 15 - 8 | 7 - 0 |
- * + ===== + ---- + ----------- + ----------- + ----------- + ----------- +
- * | Flags | V | Reserved | Serv Type | Serv Cmd Id | Reserved |
- * + ===== + ---- + ----------- + ----------- + ----------- + ----------- +
-*/
+ * + ===== + ------- + ----------- + ----------- + ----------- + -------- +
+ * | Bit | 31/30 | 29 - 24 | 21 - 16 | 15 - 8 | 7 - 0 |
+ * + ===== + ------- + ----------- + ----------- + ----------- + -------- +
+ * | Flags | V/Gen | Reserved | Serv Type | Serv Cmd Id | Rsv |
+ * + ===== + ------- + ----------- + ----------- + ----------- + -------- +
+ */
/**< @ingroup icp_qat_fw_comn
* Definition of the setting of the header's valid flag */
@@ -505,6 +505,20 @@
* hdr_flags field of LW0 (service request and response) */
#define ICP_QAT_FW_COMN_VALID_FLAG_BITPOS 7
#define ICP_QAT_FW_COMN_VALID_FLAG_MASK 0x1
+
+/**< @ingroup icp_qat_fw_comn
+ * Macros defining the bit position and mask of the 'generation' flag, within
+ * the hdr_flags field of LW0 (service request and response) */
+#define ICP_QAT_FW_COMN_GEN_FLAG_BITPOS 6
+#define ICP_QAT_FW_COMN_GEN_FLAG_MASK 0x1
+/**< @ingroup icp_qat_fw_comn
+ * The request is targeted for QAT2.0 */
+#define ICP_QAT_FW_COMN_GEN_2 1
+/**< @ingroup icp_qat_fw_comn
+* The request is targeted for QAT1.x. QAT2.0 FW will return
+ 'unsupported request' if GEN1 request type is sent to QAT2.0 FW */
+#define ICP_QAT_FW_COMN_GEN_1 0
+
#define ICP_QAT_FW_COMN_HDR_RESRVD_FLD_MASK 0x7F
/* Common QAT FW response header - structure of LW0
@@ -525,6 +539,13 @@
#define ICP_QAT_FW_COMN_CNVNR_FLAG_BITPOS 5
#define ICP_QAT_FW_COMN_CNVNR_FLAG_MASK 0x1
+/**< @ingroup icp_qat_fw_comn
+ * Macros defining the bit position and mask of Stored Blocks flag
+ * within the hdr_flags field of LW0 (service response only)
+ */
+#define ICP_QAT_FW_COMN_ST_BLK_FLAG_BITPOS 4
+#define ICP_QAT_FW_COMN_ST_BLK_FLAG_MASK 0x1
+
/**
******************************************************************************
* @ingroup icp_qat_fw_comn
@@ -660,6 +681,89 @@
ICP_QAT_FW_COMN_VALID_FLAG_BITPOS, \
ICP_QAT_FW_COMN_VALID_FLAG_MASK)
+/**
+ ******************************************************************************
+ * @ingroup icp_qat_fw_comn
+ *
+ * @description
+ * Extract the Stored Block flag from the header flags in the
+ * response only.
+ *
+ * @param hdr_flags Response 'hdr' structure to extract the
+ * Stored Block bit from the 'hdr_flags' field.
+ *
+ *****************************************************************************/
+#define ICP_QAT_FW_COMN_HDR_ST_BLK_FLAG_GET(hdr_flags) \
+ QAT_FIELD_GET(hdr_flags, \
+ ICP_QAT_FW_COMN_ST_BLK_FLAG_BITPOS, \
+ ICP_QAT_FW_COMN_ST_BLK_FLAG_MASK)
+
+/**
+ ******************************************************************************
+ * @ingroup icp_qat_fw_comn
+ *
+ * @description
+ * Set the Stored Block bit in the response's header flags.
+ *
+ * @param hdr_t Response 'hdr_t' structure to set the ST_BLK bit
+ * @param val Value of the ST_BLK bit flag.
+ *
+ *****************************************************************************/
+#define ICP_QAT_FW_COMN_HDR_ST_BLK_FLAG_SET(hdr_t, val) \
+ QAT_FIELD_SET((hdr_t.hdr_flags), \
+ (val), \
+ ICP_QAT_FW_COMN_ST_BLK_FLAG_BITPOS, \
+ ICP_QAT_FW_COMN_ST_BLK_FLAG_MASK)
+
+/**
+ ******************************************************************************
+ * @ingroup icp_qat_fw_comn
+ *
+ * @description
+ * Set the generation bit in the request's header flags.
+ *
+ * @param hdr_t Request or Response 'hdr_t' structure to set the gen bit
+ * @param val Value of the generation bit flag.
+ *
+ *****************************************************************************/
+#define ICP_QAT_FW_COMN_HDR_GENERATION_FLAG_SET(hdr_t, val) \
+ ICP_QAT_FW_COMN_GENERATION_FLAG_SET(hdr_t, val)
+
+/**
+******************************************************************************
+* @ingroup icp_qat_fw_comn
+*
+* @description
+* Common macro to set the generation bit in the common header
+*
+* @param hdr_t Structure (request or response) containing the header
+* flags field, to allow the generation bit to be set.
+* @param val Value of the generation bit flag.
+*
+*****************************************************************************/
+#define ICP_QAT_FW_COMN_GENERATION_FLAG_SET(hdr_t, val) \
+ QAT_FIELD_SET((hdr_t.hdr_flags), \
+ (val), \
+ ICP_QAT_FW_COMN_GEN_FLAG_BITPOS, \
+ ICP_QAT_FW_COMN_GEN_FLAG_MASK)
+
+/**
+******************************************************************************
+* @ingroup icp_qat_fw_comn
+*
+* @description
+* Common macro to extract the generation flag from the header flags field
+* within the header structure (request or response).
+*
+* @param hdr_t Structure (request or response) to extract the
+* generation bit from the 'hdr_flags' field.
+*
+*****************************************************************************/
+
+#define ICP_QAT_FW_COMN_HDR_GENERATION_FLAG_GET(hdr_flags) \
+ QAT_FIELD_GET(hdr_flags, \
+ ICP_QAT_FW_COMN_GEN_FLAG_BITPOS, \
+ ICP_QAT_FW_COMN_GEN_FLAG_MASK)
/**
******************************************************************************
* @ingroup icp_qat_fw_comn
Index: sys/dev/qat/qat_api/firmware/include/icp_qat_fw_comp.h
===================================================================
--- sys/dev/qat/qat_api/firmware/include/icp_qat_fw_comp.h
+++ sys/dev/qat/qat_api/firmware/include/icp_qat_fw_comp.h
@@ -61,6 +61,8 @@
* | | |as intmd| | | | | | | |
* | | | buf | | | | | | | |
* + ===== + ------ + ----- + --- + ------ + ----- + ----- + -- + ---- + --- +
+ * Note: For QAT 2.0 Disable Secure Ram, DisType0 Header and Enhanced ASB bits
+ * are don't care. i.e., these features are removed from QAT 2.0.
*/
/** Flag usage */
@@ -183,6 +185,28 @@
((secure_ram & ICP_QAT_FW_COMP_DISABLE_SECURE_RAM_AS_INTMD_BUF_MASK) \
<< ICP_QAT_FW_COMP_DISABLE_SECURE_RAM_AS_INTMD_BUF_BITPOS))
+/**
+******************************************************************************
+* @ingroup icp_qat_fw_comp
+*
+* @description
+* Macro used for the generation of the command flags for Compression Request.
+* This should always be used for the generation of the flags. No direct sets or
+* masks should be performed on the flags data
+*
+* @param sesstype Session Type
+* @param autoselect AutoSelectBest
+* Selects between compressed and uncompressed output.
+* No distinction made between static and dynamic
+* compressed data.
+*
+*********************************************************************************/
+#define ICP_QAT_FW_COMP_20_FLAGS_BUILD(sesstype, autoselect) \
+ (((sesstype & ICP_QAT_FW_COMP_SESSION_TYPE_MASK) \
+ << ICP_QAT_FW_COMP_SESSION_TYPE_BITPOS) | \
+ ((autoselect & ICP_QAT_FW_COMP_AUTO_SELECT_BEST_MASK) \
+ << ICP_QAT_FW_COMP_AUTO_SELECT_BEST_BITPOS))
+
/**
******************************************************************************
* @ingroup icp_qat_fw_comp
@@ -375,14 +399,16 @@
*
*****************************************************************************/
#define ICP_QAT_FW_COMP_REQ_PARAM_FLAGS_BUILD( \
- sop, eop, bfinal, cnv, cnvnr, crc) \
+ sop, eop, bfinal, cnv, cnvnr, cnvdfx, crc) \
(((sop & ICP_QAT_FW_COMP_SOP_MASK) << ICP_QAT_FW_COMP_SOP_BITPOS) | \
((eop & ICP_QAT_FW_COMP_EOP_MASK) << ICP_QAT_FW_COMP_EOP_BITPOS) | \
((bfinal & ICP_QAT_FW_COMP_BFINAL_MASK) \
<< ICP_QAT_FW_COMP_BFINAL_BITPOS) | \
((cnv & ICP_QAT_FW_COMP_CNV_MASK) << ICP_QAT_FW_COMP_CNV_BITPOS) | \
- ((cnvnr & ICP_QAT_FW_COMP_CNV_RECOVERY_MASK) \
- << ICP_QAT_FW_COMP_CNV_RECOVERY_BITPOS) | \
+ ((cnvnr & ICP_QAT_FW_COMP_CNVNR_MASK) \
+ << ICP_QAT_FW_COMP_CNVNR_BITPOS) | \
+ ((cnvdfx & ICP_QAT_FW_COMP_CNV_DFX_MASK) \
+ << ICP_QAT_FW_COMP_CNV_DFX_BITPOS) | \
((crc & ICP_QAT_FW_COMP_CRC_MODE_MASK) \
<< ICP_QAT_FW_COMP_CRC_MODE_BITPOS))
@@ -443,6 +469,14 @@
/**< @ingroup icp_qat_fw_comp
* Flag indicating that a cnv recovery is to be performed on the request */
+#define ICP_QAT_FW_COMP_NO_CNV_DFX 0
+/**< @ingroup icp_qat_fw_comp
+ * Flag indicating that NO CNV inject error is to be performed on the request */
+
+#define ICP_QAT_FW_COMP_CNV_DFX 1
+/**< @ingroup icp_qat_fw_comp
+ * Flag indicating that CNV inject error is to be performed on the request */
+
#define ICP_QAT_FW_COMP_CRC_MODE_LEGACY 0
/**< @ingroup icp_qat_fw_comp
* Flag representing to use the legacy CRC mode */
@@ -491,6 +525,22 @@
/**< @ingroup icp_qat_fw_comp
* Starting bit position for the CNV Recovery bit */
+#define ICP_QAT_FW_COMP_CNVNR_MASK 0x1
+/**< @ingroup icp_qat_fw_comp
+ * One bit mask for the CNV Recovery bit */
+
+#define ICP_QAT_FW_COMP_CNVNR_BITPOS 17
+/**< @ingroup icp_qat_fw_comp
+ * Starting bit position for the CNV Recovery bit */
+
+#define ICP_QAT_FW_COMP_CNV_DFX_BITPOS 18
+/**< @ingroup icp_qat_fw_comp
+ * Starting bit position for the CNV DFX bit */
+
+#define ICP_QAT_FW_COMP_CNV_DFX_MASK 0x1
+/**< @ingroup icp_qat_fw_comp
+ * One bit mask for the CNV DFX bit */
+
#define ICP_QAT_FW_COMP_CRC_MODE_BITPOS 19
/**< @ingroup icp_qat_fw_comp
* Starting bit position for CRC mode */
@@ -499,6 +549,14 @@
/**< @ingroup icp_qat_fw_comp
* One bit mask used to determine CRC mode */
+#define ICP_QAT_FW_COMP_XXHASH_ACC_MODE_BITPOS 20
+/**< @ingroup icp_qat_fw_comp
+ * Starting bit position for xxHash accumulate mode */
+
+#define ICP_QAT_FW_COMP_XXHASH_ACC_MODE_MASK 0x1
+/**< @ingroup icp_qat_fw_comp
+ * One bit mask used to determine xxHash accumulate mode */
+
/**
******************************************************************************
* @ingroup icp_qat_fw_comp
@@ -574,6 +632,38 @@
ICP_QAT_FW_COMP_CRC_MODE_BITPOS, \
ICP_QAT_FW_COMP_CRC_MODE_MASK)
+/**
+ ******************************************************************************
+ * @ingroup icp_qat_fw_comp
+ *
+ * @description
+ * Macro for extraction of the xxHash accumulate mode bit
+ *
+ * @param flags Flags to extract the xxHash accumulate mode bit from
+ *
+ *****************************************************************************/
+#define ICP_QAT_FW_COMP_XXHASH_ACC_MODE_GET(flags) \
+ QAT_FIELD_GET(flags, \
+ ICP_QAT_FW_COMP_XXHASH_ACC_MODE_BITPOS, \
+ ICP_QAT_FW_COMP_XXHASH_ACC_MODE_MASK)
+
+/**
+ ******************************************************************************
+ * @ingroup icp_qat_fw_comp
+ *
+ * @description
+ * Macro for setting of the xxHash accumulate mode bit
+ *
+ * @param flags Flags to set the xxHash accumulate mode bit to
+ * @param val xxHash accumulate mode to set
+ *
+ *****************************************************************************/
+#define ICP_QAT_FW_COMP_XXHASH_ACC_MODE_SET(flags, val) \
+ QAT_FIELD_SET(flags, \
+ val, \
+ ICP_QAT_FW_COMP_XXHASH_ACC_MODE_BITPOS, \
+ ICP_QAT_FW_COMP_XXHASH_ACC_MODE_MASK)
+
/**
******************************************************************************
* @ingroup icp_qat_fw_comp
@@ -589,10 +679,11 @@
/**< LWs 20-21 */
uint64_t inter_buff_ptr;
/**< This field specifies the physical address of an intermediate
- * buffer SGL array. The array contains a pair of 64-bit
- * intermediate buffer pointers to SGL buffer descriptors, one pair
- * per CPM. Please refer to the CPM1.6 Firmware Interface HLD
- * specification for more details. */
+ * buffer SGL array. The array contains a pair of 64-bit
+ * intermediate buffer pointers to SGL buffer descriptors, one pair
+ * per CPM. Please refer to the CPM1.6 Firmware Interface HLD
+ * specification for more details.
+ * Placeholder for QAT2.0. */
} icp_qat_fw_xlt_req_params_t;
/**
@@ -1026,4 +1117,31 @@
(((bank_a_enable)&QAT_FW_COMP_BANK_FLAG_MASK) \
<< QAT_FW_COMP_BANK_A_BITPOS))
+/**
+ *****************************************************************************
+ * @ingroup icp_qat_fw_comp
+ * Definition of the xxhash32 acc state buffer
+ * @description
+ * This is data structure used in stateful lite for xxhash32
+ *
+ *****************************************************************************/
+typedef struct xxhash_acc_state_buff_s {
+ /**< LW 0 */
+ uint32_t in_counter;
+ /**< Accumulated (total) consumed bytes. As oppose to the per request
+ * IBC in the response.*/
+
+ /**< LW 1 */
+ uint32_t out_counter;
+ /**< OBC as in the response.*/
+
+ /**< LW 2-5 */
+ uint32_t xxhash_state[4];
+ /**< Initial value is set by IA to the values stated in HAS.*/
+
+ /**< LW 6-9 */
+ uint32_t clear_txt[4];
+ /**< Set to 0 for the first request.*/
+} xxhash_acc_state_buff_t;
+
#endif /* _ICP_QAT_FW_COMP_H_ */
Index: sys/dev/qat/qat_api/firmware/include/icp_qat_fw_la.h
===================================================================
--- sys/dev/qat/qat_api/firmware/include/icp_qat_fw_la.h
+++ sys/dev/qat/qat_api/firmware/include/icp_qat_fw_la.h
@@ -92,6 +92,44 @@
/**< Delimiter type */
} icp_qat_fw_la_cmd_id_t;
+typedef struct icp_qat_fw_la_cipher_20_req_params_s {
+ /**< LW 14 */
+ uint32_t cipher_offset;
+ /**< Cipher offset long word. */
+
+ /**< LW 15 */
+ uint32_t cipher_length;
+ /**< Cipher length long word. */
+
+ /**< LWs 16-19 */
+ union {
+ uint32_t cipher_IV_array[ICP_QAT_FW_NUM_LONGWORDS_4];
+ /**< Cipher IV array */
+
+ struct {
+ uint64_t cipher_IV_ptr;
+ /**< Cipher IV pointer or Partial State Pointer */
+
+ uint64_t resrvd1;
+ /**< reserved */
+
+ } s;
+
+ } u;
+ /**< LW 20 */
+ uint32_t spc_aad_offset;
+ /**< LW 21 */
+ uint32_t spc_aad_sz;
+ /**< LW 22 - 23 */
+ uint64_t spc_aad_addr;
+ /**< LW 24 - 25 */
+ uint64_t spc_auth_res_addr;
+ /**< LW 26 */
+ uint8_t reserved[3];
+ uint8_t spc_auth_res_sz;
+
+} icp_qat_fw_la_cipher_20_req_params_t;
+
/* For the definitions of the bits in the status field of the common
* response, refer to icp_qat_fw.h.
* The return values specific to Lookaside service are given below.
@@ -165,6 +203,34 @@
/* Private defines */
+/* bits 15:14 */
+#define ICP_QAT_FW_LA_USE_WIRELESS_SLICE_TYPE 2
+/**< @ingroup icp_qat_fw_la
+ * FW Selects Wireless Cipher Slice
+ * Cipher Algorithms: AES-{F8}, Snow3G, ZUC
+ * Auth Algorithms : Snow3G, ZUC */
+
+#define ICP_QAT_FW_LA_USE_UCS_SLICE_TYPE 1
+/**< @ingroup icp_qat_fw_la
+ * FW Selects UCS Cipher Slice
+ * Cipher Algorithms: AES-{CTR/XTS}, Single Pass AES-GCM
+ * Auth Algorithms : SHA1/ SHA{2/3}-{224/256/384/512} */
+
+#define ICP_QAT_FW_LA_USE_LEGACY_SLICE_TYPE 0
+/**< @ingroup icp_qat_fw_la
+ * FW Selects Legacy Cipher/Auth Slice
+ * Cipher Algorithms: AES-{CBC/ECB}, SM4, Single Pass AES-CCM
+ * Auth Algorithms : SHA1/ SHA{2/3}-{224/256/384/512} */
+
+#define QAT_LA_SLICE_TYPE_BITPOS 14
+/**< @ingroup icp_qat_fw_la
+ * Starting bit position for the slice type selection.
+ * Refer to HAS for Slice type assignment details on QAT2.0 */
+
+#define QAT_LA_SLICE_TYPE_MASK 0x3
+/**< @ingroup icp_qat_fw_la
+ * Two bit mask used to determine the Slice type */
+
/* bit 11 */
#define ICP_QAT_FW_LA_GCM_IV_LEN_12_OCTETS 1
/**< @ingroup icp_qat_fw_la
@@ -482,7 +548,7 @@
*
* @description
* Macro for extraction of the Cipher IV field contents (bit 2)
- *
+ *
* @param flags Flags to extract the Cipher IV field contents
*
*****************************************************************************/
@@ -496,7 +562,7 @@
* @description
* Macro for extraction of the Cipher/Auth Config
* offset type (bit 3)
- *
+ *
* @param flags Flags to extract the Cipher/Auth Config
* offset type
*
@@ -640,6 +706,19 @@
QAT_LA_USE_EXTENDED_PROTOCOL_FLAGS_BITPOS, \
QAT_LA_USE_EXTENDED_PROTOCOL_FLAGS_MASK)
+/**
+ ******************************************************************************
+ * @ingroup icp_qat_fw_la
+ *
+ * @description
+ * Macro for extraction of the slice type information from the flags.
+ *
+ * @param flags Flags to extract the protocol state
+ *
+ *****************************************************************************/
+#define ICP_QAT_FW_LA_SLICE_TYPE_GET(flags) \
+ QAT_FIELD_GET(flags, QAT_LA_SLICE_TYPE_BITPOS, QAT_LA_SLICE_TYPE_MASK)
+
/* Macros for setting field bits */
/**
******************************************************************************
@@ -647,7 +726,7 @@
*
* @description
* Macro for setting the Cipher IV field contents
- *
+ *
* @param flags Flags to set with the Cipher IV field contents
* @param val Field contents indicator value
*
@@ -665,7 +744,7 @@
* @description
* Macro for setting the Cipher/Auth Config
* offset type
- *
+ *
* @param flags Flags to set the Cipher/Auth Config offset type
* @param val Offset type value
*
@@ -840,6 +919,23 @@
QAT_LA_USE_EXTENDED_PROTOCOL_FLAGS_BITPOS, \
QAT_LA_USE_EXTENDED_PROTOCOL_FLAGS_MASK)
+/**
+******************************************************************************
+* @ingroup icp_qat_fw_la
+*
+* @description
+* Macro for setting the "slice type" field in la flags
+*
+* @param flags Flags to set the slice type
+* @param val Value of the slice type to be set.
+*
+*****************************************************************************/
+#define ICP_QAT_FW_LA_SLICE_TYPE_SET(flags, val) \
+ QAT_FIELD_SET(flags, \
+ val, \
+ QAT_LA_SLICE_TYPE_BITPOS, \
+ QAT_LA_SLICE_TYPE_MASK)
+
/**
*****************************************************************************
* @ingroup icp_qat_fw_la
@@ -860,14 +956,10 @@
uint8_t content_desc_params_sz;
/**< Size of the content descriptor parameters in quad words.
- * These
- * parameters describe the session setup configuration info for
- * the
- * slices that this request relies upon i.e. the configuration
- * word and
- * cipher key needed by the cipher slice if there is a request
- * for
- * cipher processing. */
+ * These parameters describe the session setup configuration
+ * info for the slices that this request relies upon i.e. the
+ * configuration word and cipher key needed by the cipher slice
+ * if there is a request for cipher processing. */
uint8_t content_desc_hdr_resrvd2;
/**< Content descriptor reserved field */
@@ -916,14 +1008,10 @@
uint8_t content_desc_params_sz;
/**< Size of the content descriptor parameters in quad words.
- * These
- * parameters describe the session setup configuration info for
- * the
- * slices that this request relies upon i.e. the configuration
- * word and
- * cipher key needed by the cipher slice if there is a request
- * for
- * cipher processing. */
+ * These parameters describe the session setup configuration
+ * info for the slices that this request relies upon i.e. the
+ * configuration word and cipher key needed by the cipher slice
+ * if there is a request for cipher processing. */
uint8_t content_desc_hdr_resrvd2;
/**< Content descriptor reserved field */
@@ -955,8 +1043,7 @@
/**< LW 27 */
uint8_t cipher_state_sz;
/**< State size in quad words of the cipher algorithm used in this
- * session.
- * Set to zero if the algorithm doesnt provide any state */
+ * session. Set to zero if the algorithm doesnt provide any state */
uint8_t cipher_key_sz;
/**< Key size in quad words of the cipher algorithm used in this session
@@ -964,17 +1051,16 @@
uint8_t cipher_cfg_offset;
/**< Quad word offset from the content descriptor parameters address
- * i.e.
- * (content_address + (cd_hdr_sz << 3)) to the parameters for the cipher
- * processing */
+ * i.e. (content_address + (cd_hdr_sz << 3)) to the parameters for the
+ * cipher processing */
uint8_t next_curr_id;
/**< This field combines the next and current id (each four bits) -
- * the next id is the most significant nibble.
- * Next Id: Set to the next slice to pass the ciphered data through.
- * Set to ICP_QAT_FW_SLICE_DRAM_WR if the data is not to go through
- * any more slices after cipher.
- * Current Id: Initialised with the cipher slice type */
+ * the next id is the most significant nibble.
+ * Next Id: Set to the next slice to pass the ciphered data through.
+ * Set to ICP_QAT_FW_SLICE_DRAM_WR if the data is not to go through
+ * any more slices after cipher.
+ * Current Id: Initialised with the cipher slice type */
/**< LW 28 */
uint8_t cipher_padding_sz;
@@ -1021,17 +1107,15 @@
uint8_t hash_cfg_offset;
/**< Quad word offset from the content descriptor parameters address to
- * the
- * parameters for the auth processing */
+ * the parameters for the auth processing */
uint8_t next_curr_id;
/**< This field combines the next and current id (each four bits) -
- * the next id is the most significant nibble.
- * Next Id: Set to the next slice to pass the authentication data
- * through.
- * Set to ICP_QAT_FW_SLICE_DRAM_WR if the data is not to go through
- * any more slices after authentication.
- * Current Id: Initialised with the authentication slice type */
+ * the next id is the most significant nibble.
+ * Next Id: Set to the next slice to pass the authentication data
+ * through. Set to ICP_QAT_FW_SLICE_DRAM_WR if the data is not to go
+ * through any more slices after authentication.
+ * Current Id: Initialised with the authentication slice type */
/**< LW 29 */
uint8_t resrvd3;
@@ -1057,8 +1141,7 @@
uint8_t inner_state2_offset;
/**< Quad word offset from the content descriptor parameters pointer to
- * the
- * inner state2 value */
+ * the inner state2 value */
uint8_t inner_state2_sz;
/**< Size in bytes of inner hash state2 data. Must be a qword multiple
@@ -1067,8 +1150,7 @@
/**< LW 31 */
uint8_t outer_config_offset;
/**< Quad word offset from the content descriptor parameters pointer to
- * the
- * outer configuration information */
+ * the outer configuration information */
uint8_t outer_state1_sz;
/**< Size in bytes of the outer state1 value */
@@ -1078,10 +1160,8 @@
uint8_t outer_prefix_offset;
/**< Quad word offset from the start of the inner prefix data to the
- * outer
- * prefix information. Should equal the rounded inner prefix size,
- * converted
- * to qwords */
+ * outer prefix information. Should equal the rounded inner prefix size,
+ * converted to qwords */
} icp_qat_fw_auth_cd_ctrl_hdr_t;
@@ -1100,8 +1180,7 @@
/**< LW 27 */
uint8_t cipher_state_sz;
/**< State size in quad words of the cipher algorithm used in this
- * session.
- * Set to zero if the algorithm doesnt provide any state */
+ * session. Set to zero if the algorithm doesnt provide any state */
uint8_t cipher_key_sz;
/**< Key size in quad words of the cipher algorithm used in this session
@@ -1109,17 +1188,16 @@
uint8_t cipher_cfg_offset;
/**< Quad word offset from the content descriptor parameters address
- * i.e.
- * (content_address + (cd_hdr_sz << 3)) to the parameters for the cipher
- * processing */
+ * i.e. (content_address + (cd_hdr_sz << 3)) to the parameters for the
+ * cipher processing */
uint8_t next_curr_id_cipher;
/**< This field combines the next and current id (each four bits) -
- * the next id is the most significant nibble.
- * Next Id: Set to the next slice to pass the ciphered data through.
- * Set to ICP_QAT_FW_SLICE_DRAM_WR if the data is not to go through
- * any more slices after cipher.
- * Current Id: Initialised with the cipher slice type */
+ * the next id is the most significant nibble.
+ * Next Id: Set to the next slice to pass the ciphered data through.
+ * Set to ICP_QAT_FW_SLICE_DRAM_WR if the data is not to go through
+ * any more slices after cipher.
+ * Current Id: Initialised with the cipher slice type */
/**< LW 28 */
uint8_t cipher_padding_sz;
@@ -1134,16 +1212,14 @@
uint8_t hash_cfg_offset;
/**< Quad word offset from the content descriptor parameters address to
- * the
- * parameters for the auth processing */
+ * the parameters for the auth processing */
uint8_t next_curr_id_auth;
/**< This field combines the next and current id (each four bits) -
* the next id is the most significant nibble.
* Next Id: Set to the next slice to pass the authentication data
- * through.
- * Set to ICP_QAT_FW_SLICE_DRAM_WR if the data is not to go through
- * any more slices after authentication.
+ * through. Set to ICP_QAT_FW_SLICE_DRAM_WR if the data is not to go
+ * through any more slices after authentication.
* Current Id: Initialised with the authentication slice type */
/**< LW 29 */
@@ -1170,8 +1246,7 @@
uint8_t inner_state2_offset;
/**< Quad word offset from the content descriptor parameters pointer to
- * the
- * inner state2 value */
+ * the inner state2 value */
uint8_t inner_state2_sz;
/**< Size in bytes of inner hash state2 data. Must be a qword multiple
@@ -1180,8 +1255,7 @@
/**< LW 31 */
uint8_t outer_config_offset;
/**< Quad word offset from the content descriptor parameters pointer to
- * the
- * outer configuration information */
+ * the outer configuration information */
uint8_t outer_state1_sz;
/**< Size in bytes of the outer state1 value */
@@ -1191,10 +1265,8 @@
uint8_t outer_prefix_offset;
/**< Quad word offset from the start of the inner prefix data to the
- * outer
- * prefix information. Should equal the rounded inner prefix size,
- * converted
- * to qwords */
+ * outer prefix information. Should equal the rounded inner prefix size,
+ * converted to qwords */
} icp_qat_fw_cipher_auth_cd_ctrl_hdr_t;
@@ -1204,9 +1276,9 @@
* + ===== + --- + --- + --- + --- + --- + --- + --- + ---- +
* | Bit | 7 | 6 | 5 | 4 | 3 | 2 | 1 | 0 |
* + ===== + --- + --- + --- + --- + --- + --- + --- + ---- +
- * | Flags | Rsv | Rsv | Rsv | ZUC |SNOW | Rsv | Rsv |NESTED|
- * | | | | |EIA3 | 3G | | | |
- * | | | | | |UIA2 | | | |
+ * | Flags | Rsv | Rsv | Rsv | ZUC |SNOW |SKIP |SKIP |NESTED|
+ * | | | | |EIA3 | 3G |LOAD |LOAD | |
+ * | | | | | |UIA2 |OUTER|INNER| |
* + ===== + --- + --- + --- + --- + --- + --- + --- + ---- +
*/
@@ -1233,6 +1305,42 @@
* requires nested hashing
*/
+/* Bit 1 */
+
+#define QAT_FW_LA_SKIP_INNER_STATE1_LOAD_BITPOS 1
+/**< @ingroup icp_qat_fw_comn
+ * Bit position of the Skipping Inner State1 Load bit */
+
+#define QAT_FW_LA_SKIP_INNER_STATE1_LOAD 1
+/**< @ingroup icp_qat_fw_comn
+ * Value indicating the skipping of inner hash state load */
+
+#define QAT_FW_LA_NO_SKIP_INNER_STATE1_LOAD 0
+/**< @ingroup icp_qat_fw_comn
+ * Value indicating the no skipping of inner hash state load */
+
+#define QAT_FW_LA_SKIP_INNER_STATE1_LOAD_MASK 0x1
+/**< @ingroup icp_qat_fw_comn
+ * Bit mask of Skipping Inner State1 Load bit */
+
+/* Bit 2 */
+
+#define QAT_FW_LA_SKIP_OUTER_STATE1_LOAD_BITPOS 2
+/**< @ingroup icp_qat_fw_comn
+ * Bit position of the Skipping Outer State1 Load bit */
+
+#define QAT_FW_LA_SKIP_OUTER_STATE1_LOAD 1
+/**< @ingroup icp_qat_fw_comn
+ * Value indicating the skipping of outer hash state load */
+
+#define QAT_FW_LA_NO_SKIP_OUTER_STATE1_LOAD 0
+/**< @ingroup icp_qat_fw_comn
+ * Value indicating the no skipping of outer hash state load */
+
+#define QAT_FW_LA_SKIP_OUTER_STATE1_LOAD_MASK 0x1
+/**< @ingroup icp_qat_fw_comn
+ * Bit mask of Skipping Outer State1 Load bit */
+
/* Bit 3 */
#define QAT_FW_LA_SNOW3G_UIA2_BITPOS 3
@@ -1261,6 +1369,24 @@
/**< @ingroup icp_qat_fw_la
* One bit mask used to determine the use of hash algorithm ZUC-EIA3 */
+/* Bit 5 */
+
+#define QAT_FW_LA_MODE2_BITPOS 5
+/**< @ingroup icp_qat_fw_comn
+ * Bit position of the Mode 2 bit */
+
+#define QAT_FW_LA_MODE2 1
+/**< @ingroup icp_qat_fw_comn
+ * Value indicating the Mode 2*/
+
+#define QAT_FW_LA_NO_MODE2 0
+/**< @ingroup icp_qat_fw_comn
+ * Value indicating the no Mode 2*/
+
+#define QAT_FW_LA_MODE2_MASK 0x1
+/**< @ingroup icp_qat_fw_comn
+ * Bit mask of Mode 2 */
+
/* Macros for extracting hash flags */
/**
@@ -1279,6 +1405,67 @@
QAT_FW_LA_AUTH_HDR_NESTED_BITPOS, \
QAT_FW_LA_AUTH_HDR_NESTED_MASK)
+/**
+ ******************************************************************************
+ * @ingroup icp_qat_fw_la
+ *
+ * @description
+ * Macro for extraction of the "Skipping Inner State1 Load state" hash flag
+ *
+ * @param flags Hash Flags
+ *
+ *****************************************************************************/
+#define ICP_QAT_FW_HASH_FLAG_SKIP_INNER_STATE1_LOAD_GET(flags) \
+ QAT_FIELD_GET(flags, \
+ QAT_FW_LA_SKIP_INNER_STATE1_LOAD_BITPOS, \
+ QAT_FW_LA_INNER_STATE1_LOAD_MASK)
+
+/**
+ ******************************************************************************
+ * Macro for setting the "Skipping Inner State1 Load" hash flag
+ *
+ * @param flags Hash Flags
+ * @param val Value of the flag
+ *
+ *****************************************************************************/
+#define ICP_QAT_FW_HASH_FLAG_SKIP_INNER_STATE1_LOAD_SET(flags, val) \
+ QAT_FIELD_SET(flags, \
+ val, \
+ QAT_FW_LA_SKIP_INNER_STATE1_LOAD_BITPOS, \
+ QAT_FW_LA_SKIP_INNER_STATE1_LOAD_MASK)
+
+/**
+ ******************************************************************************
+ * @ingroup icp_qat_fw_la
+ *
+ * @description
+ * Macro for extraction of the "Skipping Outer State1 Load state" hash flag
+ *
+ * @param flags Hash Flags
+ *
+ *****************************************************************************/
+#define ICP_QAT_FW_HASH_FLAG_SKIP_OUTER_STATE1_LOAD_GET(flags) \
+ QAT_FIELD_GET(flags, \
+ QAT_FW_LA_SKIP_OUTER_STATE1_LOAD_BITPOS, \
+ QAT_FW_LA_SKIP_OUTER_STATE1_LOAD_MASK)
+
+/**
+ ******************************************************************************
+ * @ingroup icp_qat_fw_la
+ *
+ * @description
+ * Macro for setting the "Skipping Outer State1 Load" hash flag
+ *
+ * @param flags Hash Flags
+ * @param val Value of the flag
+ *
+ *****************************************************************************/
+#define ICP_QAT_FW_HASH_FLAG_SKIP_OUTER_STATE1_LOAD_SET(flags, val) \
+ QAT_FIELD_SET(flags, \
+ val, \
+ QAT_FW_LA_SKIP_OUTER_STATE1_LOAD_BITPOS, \
+ QAT_FW_LA_SKIP_OUTER_STATE1_LOAD_MASK)
+
/**
******************************************************************************
* @ingroup icp_qat_fw_la
@@ -1328,6 +1515,40 @@
QAT_FW_LA_AUTH_HDR_NESTED_BITPOS, \
QAT_FW_LA_AUTH_HDR_NESTED_MASK)
+/**
+ ******************************************************************************
+ * @ingroup icp_qat_fw_la
+ *
+ * @description
+ * Macro for setting the "Skipping Inner State1 Load" hash flag
+ *
+ * @param flags Hash Flags
+ * @param val Value of the flag
+ *
+ *****************************************************************************/
+#define ICP_QAT_FW_HASH_FLAG_SKIP_INNER_STATE1_LOAD_SET(flags, val) \
+ QAT_FIELD_SET(flags, \
+ val, \
+ QAT_FW_LA_SKIP_INNER_STATE1_LOAD_BITPOS, \
+ QAT_FW_LA_SKIP_INNER_STATE1_LOAD_MASK)
+
+/**
+ ******************************************************************************
+ * @ingroup icp_qat_fw_la
+ *
+ * @description
+ * Macro for setting the "Skipping Outer State1 Load" hash flag
+ *
+ * @param flags Hash Flags
+ * @param val Value of the flag
+ *
+ *****************************************************************************/
+#define ICP_QAT_FW_HASH_FLAG_SKIP_OUTER_STATE1_LOAD_SET(flags, val) \
+ QAT_FIELD_SET(flags, \
+ val, \
+ QAT_FW_LA_SKIP_OUTER_STATE1_LOAD_BITPOS, \
+ QAT_FW_LA_SKIP_OUTER_STATE1_LOAD_MASK)
+
/**
******************************************************************************
* @ingroup icp_qat_fw_la
@@ -1362,6 +1583,20 @@
QAT_FW_LA_ZUC_EIA3_BITPOS, \
QAT_FW_LA_ZUC_EIA3_MASK)
+/**
+ ******************************************************************************
+ * @ingroup icp_qat_fw_la
+ *
+ * @description
+ * Macro for setting the "Mode 2" hash flag
+ *
+ * @param flags Hash Flags
+ * @param val Value of the flag
+ *
+ *****************************************************************************/
+#define ICP_QAT_FW_HASH_FLAG_MODE2_SET(flags, val) \
+ QAT_FIELD_SET(flags, val, QAT_FW_LA_MODE2_BITPOS, QAT_FW_LA_MODE2_MASK)
+
#define ICP_QAT_FW_CCM_GCM_AAD_SZ_MAX 240
#define ICP_QAT_FW_SPC_AAD_SZ_MAX 0x3FFF
@@ -1494,8 +1729,7 @@
uint8_t aad_sz;
/**< Size in bytes of padded AAD data to prefix to the packet
- * for CCM
- * or GCM processing */
+ * for CCM or GCM processing */
} u2;
uint8_t resrvd1;
@@ -1535,8 +1769,7 @@
uint8_t aad_sz;
/**< Size in bytes of padded AAD data to prefix to the packet
- * for CCM
- * or GCM processing */
+ * for CCM or GCM processing */
} u2;
uint8_t resrvd1;
@@ -1565,8 +1798,7 @@
/**< SSL3 */
uint16_t secret_lgth_ssl;
/**< Length of Secret information for SSL. In the case of TLS
- * the
- * secret is supplied in the content descriptor */
+ * the secret is supplied in the content descriptor */
/**< MGF */
uint16_t mask_length;
@@ -1692,8 +1924,7 @@
uint8_t aad_sz;
/**< Size in bytes of padded AAD data to prefix to the packet
- * for CCM
- * or GCM processing */
+ * for CCM or GCM processing */
} u2;
uint8_t resrvd1;
@@ -1731,8 +1962,7 @@
uint8_t aad_sz;
/**< Size in bytes of padded AAD data to prefix to the packet
- * for CCM
- * or GCM processing */
+ * for CCM or GCM processing */
} u2;
uint8_t resrvd1;
@@ -1779,16 +2009,14 @@
/**< LWs 6-13 */
uint64_t opaque_data;
/**< Opaque data passed unmodified from the request to response messages
- * by
- * firmware (fw) */
+ * by firmware (fw) */
uint64_t resrvd1;
/**< Reserved, unused for TRNG */
uint64_t dest_data_addr;
/**< Generic definition of the destination data supplied to the QAT AE.
- * The
- * common flags are used to further describe the attributes of this
+ * The common flags are used to further describe the attributes of this
* field */
uint32_t resrvd2;
@@ -1796,7 +2024,7 @@
uint32_t entropy_length;
/**< Size of the data in bytes to process. Used by the get_random
- * command. Set to 0 for commands that dont need a length parameter */
+ * command. Set to 0 for commands that dont need a length parameter */
} icp_qat_fw_la_trng_req_mid_t;
@@ -2235,10 +2463,8 @@
union {
uint8_t label_flags;
/**< For first-level labels: each bit in [0..3] will trigger a
- * child
- * Expand-Label operation on the corresponding sublabel. Bits
- * [4..7]
- * are reserved.
+ * child Expand-Label operation on the corresponding sublabel.
+ * Bits [4..7] are reserved.
*/
uint8_t sublabel_flags;
Index: sys/dev/qat/qat_api/firmware/include/icp_qat_hw.h
===================================================================
--- sys/dev/qat/qat_api/firmware/include/icp_qat_hw.h
+++ sys/dev/qat/qat_api/firmware/include/icp_qat_hw.h
@@ -25,22 +25,22 @@
/* ========================================================================= */
typedef enum {
- ICP_QAT_HW_AE_0 = 0, /*!< ID of AE0 */
- ICP_QAT_HW_AE_1 = 1, /*!< ID of AE1 */
- ICP_QAT_HW_AE_2 = 2, /*!< ID of AE2 */
- ICP_QAT_HW_AE_3 = 3, /*!< ID of AE3 */
- ICP_QAT_HW_AE_4 = 4, /*!< ID of AE4 */
- ICP_QAT_HW_AE_5 = 5, /*!< ID of AE5 */
- ICP_QAT_HW_AE_6 = 6, /*!< ID of AE6 */
- ICP_QAT_HW_AE_7 = 7, /*!< ID of AE7 */
- ICP_QAT_HW_AE_8 = 8, /*!< ID of AE8 */
- ICP_QAT_HW_AE_9 = 9, /*!< ID of AE9 */
- ICP_QAT_HW_AE_10 = 10, /*!< ID of AE10 */
- ICP_QAT_HW_AE_11 = 11, /*!< ID of AE11 */
- ICP_QAT_HW_AE_12 = 12, /*!< ID of AE12 */
- ICP_QAT_HW_AE_13 = 13, /*!< ID of AE13 */
- ICP_QAT_HW_AE_14 = 14, /*!< ID of AE14 */
- ICP_QAT_HW_AE_15 = 15, /*!< ID of AE15 */
+ ICP_QAT_HW_AE_0 = 0, /*!< ID of AE0 */
+ ICP_QAT_HW_AE_1 = 1, /*!< ID of AE1 */
+ ICP_QAT_HW_AE_2 = 2, /*!< ID of AE2 */
+ ICP_QAT_HW_AE_3 = 3, /*!< ID of AE3 */
+ ICP_QAT_HW_AE_4 = 4, /*!< ID of AE4 */
+ ICP_QAT_HW_AE_5 = 5, /*!< ID of AE5 */
+ ICP_QAT_HW_AE_6 = 6, /*!< ID of AE6 */
+ ICP_QAT_HW_AE_7 = 7, /*!< ID of AE7 */
+ ICP_QAT_HW_AE_8 = 8, /*!< ID of AE8 */
+ ICP_QAT_HW_AE_9 = 9, /*!< ID of AE9 */
+ ICP_QAT_HW_AE_10 = 10, /*!< ID of AE10 */
+ ICP_QAT_HW_AE_11 = 11, /*!< ID of AE11 */
+ ICP_QAT_HW_AE_12 = 12, /*!< ID of AE12 */
+ ICP_QAT_HW_AE_13 = 13, /*!< ID of AE13 */
+ ICP_QAT_HW_AE_14 = 14, /*!< ID of AE14 */
+ ICP_QAT_HW_AE_15 = 15, /*!< ID of AE15 */
ICP_QAT_HW_AE_DELIMITER = 16 /**< Delimiter type */
} icp_qat_hw_ae_id_t;
@@ -49,12 +49,12 @@
/* ========================================================================= */
typedef enum {
- ICP_QAT_HW_QAT_0 = 0, /*!< ID of QAT0 */
- ICP_QAT_HW_QAT_1 = 1, /*!< ID of QAT1 */
- ICP_QAT_HW_QAT_2 = 2, /*!< ID of QAT2 */
- ICP_QAT_HW_QAT_3 = 3, /*!< ID of QAT3 */
- ICP_QAT_HW_QAT_4 = 4, /*!< ID of QAT4 */
- ICP_QAT_HW_QAT_5 = 5, /*!< ID of QAT5 */
+ ICP_QAT_HW_QAT_0 = 0, /*!< ID of QAT0 */
+ ICP_QAT_HW_QAT_1 = 1, /*!< ID of QAT1 */
+ ICP_QAT_HW_QAT_2 = 2, /*!< ID of QAT2 */
+ ICP_QAT_HW_QAT_3 = 3, /*!< ID of QAT3 */
+ ICP_QAT_HW_QAT_4 = 4, /*!< ID of QAT4 */
+ ICP_QAT_HW_QAT_5 = 5, /*!< ID of QAT5 */
ICP_QAT_HW_QAT_DELIMITER = 6 /**< Delimiter type */
} icp_qat_hw_qat_id_t;
@@ -79,24 +79,24 @@
ICP_QAT_HW_AUTH_ALGO_SHA256 = 4, /*!< SHA-256 hashing */
ICP_QAT_HW_AUTH_ALGO_SHA384 = 5, /*!< SHA-384 hashing */
ICP_QAT_HW_AUTH_ALGO_SHA512 = 6, /*!< SHA-512 hashing */
- ICP_QAT_HW_AUTH_ALGO_AES_XCBC_MAC = 7, /*!< AES-XCBC-MAC hashing */
- ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC = 8, /*!< AES-CBC-MAC hashing */
+ ICP_QAT_HW_AUTH_ALGO_AES_XCBC_MAC = 7, /*!< AES-XCBC-MAC hashing */
+ ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC = 8, /*!< AES-CBC-MAC hashing */
ICP_QAT_HW_AUTH_ALGO_AES_F9 = 9, /*!< AES F9 hashing */
- ICP_QAT_HW_AUTH_ALGO_GALOIS_128 = 10, /*!< Galois 128 bit hashing */
- ICP_QAT_HW_AUTH_ALGO_GALOIS_64 = 11, /*!< Galois 64 hashing */
- ICP_QAT_HW_AUTH_ALGO_KASUMI_F9 = 12, /*!< Kasumi F9 hashing */
+ ICP_QAT_HW_AUTH_ALGO_GALOIS_128 = 10, /*!< Galois 128 bit hashing */
+ ICP_QAT_HW_AUTH_ALGO_GALOIS_64 = 11, /*!< Galois 64 hashing */
+ ICP_QAT_HW_AUTH_ALGO_KASUMI_F9 = 12, /*!< Kasumi F9 hashing */
ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2 = 13, /*!< UIA2/SNOW_3G F9 hashing */
ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3 =
- 14, /*!< 128_EIA3/ZUC_3G hashing */
- ICP_QAT_HW_AUTH_ALGO_SM3 = 15, /*!< SM3 hashing */
- ICP_QAT_HW_AUTH_ALGO_SHA3_224 = 16, /*!< SHA3-224 hashing */
- ICP_QAT_HW_AUTH_ALGO_SHA3_256 = 17, /*!< SHA3-256 hashing */
- ICP_QAT_HW_AUTH_ALGO_SHA3_384 = 18, /*!< SHA3-384 hashing */
- ICP_QAT_HW_AUTH_ALGO_SHA3_512 = 19, /*!< SHA3-512 hashing */
- ICP_QAT_HW_AUTH_ALGO_SHAKE_128 = 20, /*!< SHAKE-128 hashing */
- ICP_QAT_HW_AUTH_ALGO_SHAKE_256 = 21, /*!< SHAKE-256 hashing */
- ICP_QAT_HW_AUTH_ALGO_POLY = 22, /*!< POLY hashing */
- ICP_QAT_HW_AUTH_ALGO_DELIMITER = 23 /**< Delimiter type */
+ 14, /*!< 128_EIA3/ZUC_3G hashing */
+ ICP_QAT_HW_AUTH_ALGO_SM3 = 15, /*!< SM3 hashing */
+ ICP_QAT_HW_AUTH_ALGO_SHA3_224 = 16, /*!< SHA3-224 hashing */
+ ICP_QAT_HW_AUTH_ALGO_SHA3_256 = 17, /*!< SHA3-256 hashing */
+ ICP_QAT_HW_AUTH_ALGO_SHA3_384 = 18, /*!< SHA3-384 hashing */
+ ICP_QAT_HW_AUTH_ALGO_SHA3_512 = 19, /*!< SHA3-512 hashing */
+ ICP_QAT_HW_AUTH_RESERVED_4 = 20, /*!< Reserved */
+ ICP_QAT_HW_AUTH_RESERVED_5 = 21, /*!< Reserved */
+ ICP_QAT_HW_AUTH_ALGO_POLY = 22, /*!< POLY hashing */
+ ICP_QAT_HW_AUTH_ALGO_DELIMITER = 23 /**< Delimiter type */
} icp_qat_hw_auth_algo_t;
/**
@@ -118,9 +118,9 @@
*****************************************************************************/
typedef enum {
- ICP_QAT_HW_AUTH_MODE0 = 0, /*!< QAT Auth Mode0 configuration */
- ICP_QAT_HW_AUTH_MODE1 = 1, /*!< QAT Auth Mode1 configuration */
- ICP_QAT_HW_AUTH_MODE2 = 2, /*!< QAT AuthMode2 configuration */
+ ICP_QAT_HW_AUTH_MODE0 = 0, /*!< QAT Auth Mode0 configuration */
+ ICP_QAT_HW_AUTH_MODE1 = 1, /*!< QAT Auth Mode1 configuration */
+ ICP_QAT_HW_AUTH_MODE2 = 2, /*!< QAT AuthMode2 configuration */
ICP_QAT_HW_AUTH_MODE_DELIMITER = 3 /**< Delimiter type */
} icp_qat_hw_auth_mode_t;
@@ -269,7 +269,7 @@
/**< Flag usage - see additional notes @description for
* ICP_QAT_HW_AUTH_CONFIG_BUILD and
* ICP_QAT_HW_AUTH_CONFIG_BUILD_UPPER macros.
-*/
+ */
#define QAT_AUTH_SHA3_HW_PADDING_ENABLE 0
/**< @ingroup icp_qat_hw_defs
@@ -461,7 +461,7 @@
#define QAT_HW_ROUND_UP(val, n) (((val) + ((n)-1)) & (~(n - 1)))
/* State1 */
-#define ICP_QAT_HW_NULL_STATE1_SZ 64
+#define ICP_QAT_HW_NULL_STATE1_SZ 32
/**< @ingroup icp_qat_hw_defs
* State1 block size for NULL hashing */
#define ICP_QAT_HW_MD5_STATE1_SZ 16
@@ -474,7 +474,7 @@
#define ICP_QAT_HW_SHA224_STATE1_SZ 32
/**< @ingroup icp_qat_hw_defs
* State1 block size for SHA24 */
-#define ICP_QAT_HW_SHA3_224_STATE1_SZ 32
+#define ICP_QAT_HW_SHA3_224_STATE1_SZ 28
/**< @ingroup icp_qat_hw_defs
* State1 block size for SHA3_224 */
#define ICP_QAT_HW_SHA256_STATE1_SZ 32
@@ -486,7 +486,7 @@
#define ICP_QAT_HW_SHA384_STATE1_SZ 64
/**< @ingroup icp_qat_hw_defs
* State1 block size for SHA384 */
-#define ICP_QAT_HW_SHA3_384_STATE1_SZ 64
+#define ICP_QAT_HW_SHA3_384_STATE1_SZ 48
/**< @ingroup icp_qat_hw_defs
* State1 block size for SHA3_384 */
#define ICP_QAT_HW_SHA512_STATE1_SZ 64
@@ -516,15 +516,15 @@
#define ICP_QAT_HW_ZUC_3G_EIA3_STATE1_SZ 8
/**< @ingroup icp_cpm_hw_defs
* State1 block size for EIA3 */
+#define ICP_QAT_HW_SM3_STATE1_SZ 32
+/**< @ingroup icp_qat_hw_defs
+ * State1 block size for SM3 */
#define ICP_QAT_HW_SHA3_STATEFUL_STATE1_SZ 200
/** <@ingroup icp_cpm_hw_defs
* State1 block size for stateful SHA3 processing*/
-#define ICP_QAT_HW_SM3_STATE1_SZ 32
-/**< @ingroup icp_cpm_hw_defs
- * State1 block size for SM3 */
/* State2 */
-#define ICP_QAT_HW_NULL_STATE2_SZ 64
+#define ICP_QAT_HW_NULL_STATE2_SZ 32
/**< @ingroup icp_qat_hw_defs
* State2 block size for NULL hashing */
#define ICP_QAT_HW_MD5_STATE2_SZ 16
@@ -537,25 +537,25 @@
#define ICP_QAT_HW_SHA224_STATE2_SZ 32
/**< @ingroup icp_qat_hw_defs
* State2 block size for SHA224 */
-#define ICP_QAT_HW_SHA3_224_STATE2_SZ 32
+#define ICP_QAT_HW_SHA3_224_STATE2_SZ 0
/**< @ingroup icp_qat_hw_defs
* State2 block size for SHA3_224 */
#define ICP_QAT_HW_SHA256_STATE2_SZ 32
/**< @ingroup icp_qat_hw_defs
* State2 block size for SHA256 */
-#define ICP_QAT_HW_SHA3_256_STATE2_SZ 32
+#define ICP_QAT_HW_SHA3_256_STATE2_SZ 0
/**< @ingroup icp_qat_hw_defs
* State2 block size for SHA3_256 */
#define ICP_QAT_HW_SHA384_STATE2_SZ 64
/**< @ingroup icp_qat_hw_defs
* State2 block size for SHA384 */
-#define ICP_QAT_HW_SHA3_384_STATE2_SZ 64
+#define ICP_QAT_HW_SHA3_384_STATE2_SZ 0
/**< @ingroup icp_qat_hw_defs
* State2 block size for SHA3_384 */
#define ICP_QAT_HW_SHA512_STATE2_SZ 64
/**< @ingroup icp_qat_hw_defs
* State2 block size for SHA512 */
-#define ICP_QAT_HW_SHA3_512_STATE2_SZ 64
+#define ICP_QAT_HW_SHA3_512_STATE2_SZ 0
/**< @ingroup icp_qat_hw_defs
* State2 block size for SHA3_512 */
#define ICP_QAT_HW_AES_XCBC_MAC_KEY_SZ 16
@@ -598,6 +598,9 @@
#define ICP_QAT_HW_SM3_STATE2_SZ 32
/**< @ingroup icp_qat_hw_defs
* State2 block size for SM3 */
+#define ICP_QAT_HW_SHA3_STATEFUL_STATE2_SZ 208
+/** <@ingroup icp_cpm_hw_defs
+ * State2 block size for stateful SHA3 processing*/
/* ************************************************************************* */
/* ************************************************************************* */
@@ -647,10 +650,33 @@
icp_qat_hw_auth_setup_t outer_setup;
/**< Outer configuration word for the slice */
- /* State2 size is zero - this may change for future implementations */
- uint8_t state2[ICP_QAT_HW_SHA3_512_STATE2_SZ];
} icp_qat_hw_auth_sha3_512_t;
+/**
+ *****************************************************************************
+ * @ingroup icp_qat_hw_defs
+ * Definition of stateful SHA3 auth algorithm processing struct
+ * @description
+ * This structs described the parameters to pass to the slice for
+ * configuring it for stateful SHA3 processing. This is the largest
+ * possible setup block for authentication
+ *
+ *****************************************************************************/
+typedef struct icp_qat_hw_auth_sha3_stateful_s {
+ icp_qat_hw_auth_setup_t inner_setup;
+ /**< Inner loop configuration word for the slice */
+
+ uint8_t inner_state1[ICP_QAT_HW_SHA3_STATEFUL_STATE1_SZ];
+ /**< Inner hash block */
+
+ icp_qat_hw_auth_setup_t outer_setup;
+ /**< Outer configuration word for the slice */
+
+ uint8_t outer_state1[ICP_QAT_HW_SHA3_STATEFUL_STATE1_SZ];
+ /**< Outer hash block */
+
+} icp_qat_hw_auth_sha3_stateful_t;
+
/**
*****************************************************************************
* @ingroup icp_qat_hw_defs
@@ -662,6 +688,8 @@
typedef union icp_qat_hw_auth_algo_blk_u {
icp_qat_hw_auth_sha512_t sha512;
/**< SHA512 Hashing */
+ icp_qat_hw_auth_sha3_stateful_t sha3_stateful;
+ /**< Stateful SHA3 Hashing */
} icp_qat_hw_auth_algo_blk_t;
@@ -691,11 +719,11 @@
ICP_QAT_HW_CIPHER_ALGO_NULL = 0, /*!< Null ciphering */
ICP_QAT_HW_CIPHER_ALGO_DES = 1, /*!< DES ciphering */
ICP_QAT_HW_CIPHER_ALGO_3DES = 2, /*!< 3DES ciphering */
- ICP_QAT_HW_CIPHER_ALGO_AES128 = 3, /*!< AES-128 ciphering */
- ICP_QAT_HW_CIPHER_ALGO_AES192 = 4, /*!< AES-192 ciphering */
- ICP_QAT_HW_CIPHER_ALGO_AES256 = 5, /*!< AES-256 ciphering */
+ ICP_QAT_HW_CIPHER_ALGO_AES128 = 3, /*!< AES-128 ciphering */
+ ICP_QAT_HW_CIPHER_ALGO_AES192 = 4, /*!< AES-192 ciphering */
+ ICP_QAT_HW_CIPHER_ALGO_AES256 = 5, /*!< AES-256 ciphering */
ICP_QAT_HW_CIPHER_ALGO_ARC4 = 6, /*!< ARC4 ciphering */
- ICP_QAT_HW_CIPHER_ALGO_KASUMI = 7, /*!< Kasumi */
+ ICP_QAT_HW_CIPHER_ALGO_KASUMI = 7, /*!< Kasumi */
ICP_QAT_HW_CIPHER_ALGO_SNOW_3G_UEA2 = 8, /*!< Snow_3G */
ICP_QAT_HW_CIPHER_ALGO_ZUC_3G_128_EEA3 = 9, /*!< ZUC_3G */
ICP_QAT_HW_CIPHER_ALGO_SM4 = 10, /*!< SM4 ciphering */
@@ -723,7 +751,7 @@
ICP_QAT_HW_CIPHER_CTR_MODE = 2, /*!< CTR mode */
ICP_QAT_HW_CIPHER_F8_MODE = 3, /*!< F8 mode */
ICP_QAT_HW_CIPHER_AEAD_MODE = 4, /*!< AES-GCM SPC AEAD mode */
- ICP_QAT_HW_CIPHER_RESERVED_MODE = 5, /*!< Reserved */
+ ICP_QAT_HW_CIPHER_CCM_MODE = 5, /*!< AES-CCM SPC AEAD mode */
ICP_QAT_HW_CIPHER_XTS_MODE = 6, /*!< XTS mode */
ICP_QAT_HW_CIPHER_MODE_DELIMITER = 7 /**< Delimiter type */
} icp_qat_hw_cipher_mode_t;
@@ -746,6 +774,23 @@
/**< Reserved */
} icp_qat_hw_cipher_config_t;
+/**
+ *****************************************************************************
+ * @ingroup icp_qat_hw_defs
+ * Cipher Configuration Struct
+ *
+ * @description
+ * Configuration data used for setting up the QAT UCS Cipher Slice
+ *
+ *****************************************************************************/
+typedef struct icp_qat_hw_ucs_cipher_config_s {
+ uint32_t val;
+ /**< Cipher slice configuration */
+
+ uint32_t reserved[3];
+ /**< Reserved */
+} icp_qat_hw_ucs_cipher_config_t;
+
/**
*****************************************************************************
* @ingroup icp_qat_hw_defs
@@ -851,6 +896,10 @@
/**< @ingroup icp_qat_hw_defs
* Define for the cipher XTS mode key size */
+#define QAT_CIPHER_MODE_UCS_XTS_KEY_SZ_MULT 1
+/**< @ingroup icp_qat_hw_defs
+ * Define for the UCS cipher XTS mode key size */
+
/**
******************************************************************************
* @ingroup icp_qat_hw_defs
@@ -931,6 +980,16 @@
#define ICP_QAT_HW_AES_256_KEY_SZ 32
/**< @ingroup icp_qat_hw_defs
* Define the key size for AES256 */
+/* AES UCS */
+#define ICP_QAT_HW_UCS_AES_128_KEY_SZ ICP_QAT_HW_AES_128_KEY_SZ
+/**< @ingroup icp_qat_hw_defs
+ * Define the key size for AES128 for UCS slice*/
+#define ICP_QAT_HW_UCS_AES_192_KEY_SZ 32
+/**< @ingroup icp_qat_hw_defs
+ * Define the key size for AES192 for UCS slice*/
+#define ICP_QAT_HW_UCS_AES_256_KEY_SZ ICP_QAT_HW_AES_256_KEY_SZ
+/**< @ingroup icp_qat_hw_defs
+ * Define the key size for AES256 for UCS slice*/
#define ICP_QAT_HW_AES_128_F8_KEY_SZ \
(ICP_QAT_HW_AES_128_KEY_SZ * QAT_CIPHER_MODE_F8_KEY_SZ_MULT)
/**< @ingroup icp_qat_hw_defs
@@ -951,6 +1010,14 @@
(ICP_QAT_HW_AES_256_KEY_SZ * QAT_CIPHER_MODE_XTS_KEY_SZ_MULT)
/**< @ingroup icp_qat_hw_defs
* Define the key size for AES256 XTS */
+#define ICP_QAT_HW_UCS_AES_128_XTS_KEY_SZ \
+ (ICP_QAT_HW_UCS_AES_128_KEY_SZ * QAT_CIPHER_MODE_UCS_XTS_KEY_SZ_MULT)
+/**< @ingroup icp_qat_hw_defs
+ * Define the key size for AES128 XTS for the UCS Slice*/
+#define ICP_QAT_HW_UCS_AES_256_XTS_KEY_SZ \
+ (ICP_QAT_HW_UCS_AES_256_KEY_SZ * QAT_CIPHER_MODE_UCS_XTS_KEY_SZ_MULT)
+/**< @ingroup icp_qat_hw_defs
+ * Define the key size for AES256 XTS for the UCS Slice*/
#define ICP_QAT_HW_KASUMI_KEY_SZ 16
/**< @ingroup icp_qat_hw_defs
* Define the key size for Kasumi */
@@ -1090,10 +1157,10 @@
*****************************************************************************/
typedef enum {
- ICP_QAT_HW_TRNG_NEG_0 = 0, /*!< TRNG Neg Zero Test */
- ICP_QAT_HW_TRNG_NEG_1 = 1, /*!< TRNG Neg One Test */
+ ICP_QAT_HW_TRNG_NEG_0 = 0, /*!< TRNG Neg Zero Test */
+ ICP_QAT_HW_TRNG_NEG_1 = 1, /*!< TRNG Neg One Test */
ICP_QAT_HW_TRNG_POS = 2, /*!< TRNG POS Test */
- ICP_QAT_HW_TRNG_POS_VNC = 3, /*!< TRNG POS VNC Test */
+ ICP_QAT_HW_TRNG_POS_VNC = 3, /*!< TRNG POS VNC Test */
ICP_QAT_HW_TRNG_KAT_DELIMITER = 4 /**< Delimiter type */
} icp_qat_hw_trng_kat_mode_t;
@@ -1388,7 +1455,7 @@
typedef enum {
ICP_QAT_HW_COMPRESSION_ALGO_DEFLATE = 0, /*!< Deflate compression */
- ICP_QAT_HW_COMPRESSION_DEPRECATED = 1, /*!< Deprecated */
+ ICP_QAT_HW_COMPRESSION_DEPRECATED = 1, /*!< Deprecated */
ICP_QAT_HW_COMPRESSION_ALGO_DELIMITER = 2 /**< Delimiter type */
} icp_qat_hw_compression_algo_t;
@@ -1470,11 +1537,11 @@
*****************************************************************************/
typedef struct icp_qat_hw_compression_config_s {
- uint32_t val;
- /**< Compression slice configuration */
+ uint32_t lower_val;
+ /**< Compression slice configuration lower LW */
- uint32_t reserved;
- /**< Reserved */
+ uint32_t upper_val;
+ /**< Compression slice configuration upper LW */
} icp_qat_hw_compression_config_t;
/* Private defines */
Index: sys/dev/qat/qat_api/firmware/include/icp_qat_hw_20_comp.h
===================================================================
--- /dev/null
+++ sys/dev/qat/qat_api/firmware/include/icp_qat_hw_20_comp.h
@@ -0,0 +1,292 @@
+/* SPDX-License-Identifier: BSD-3-Clause */
+/* Copyright(c) 2007-2022 Intel Corporation */
+/* $FreeBSD$ */
+/**
+ *****************************************************************************
+ * @file icp_qat_hw_2x_comp.h
+ * @defgroup ICP QAT HW accessors for using the for 2.x Compression Slice
+ * definitions
+ * @ingroup icp_qat_hw_2x_comp
+ * @description
+ * This file documents definitions for the QAT HW COMP SLICE
+ *
+ *****************************************************************************/
+
+#ifndef _ICP_QAT_HW_20_COMP_H_
+#define _ICP_QAT_HW_20_COMP_H_
+
+#include "icp_qat_hw_20_comp_defs.h" // For HW definitions
+#include "icp_qat_fw.h" //For Set Field Macros.
+
+#ifdef WIN32
+#include <stdlib.h> // built in support for _byteswap_ulong
+#define BYTE_SWAP_32 _byteswap_ulong
+#else
+#define BYTE_SWAP_32 __builtin_bswap32
+#endif
+
+/**
+*****************************************************************************
+* @ingroup icp_qat_fw_comn
+*
+* @description
+* Definition of the hw config csr. This representation has to be further
+* processed by the corresponding config build function.
+*
+*****************************************************************************/
+typedef struct icp_qat_hw_comp_20_config_csr_lower_s {
+ // Fields programmable directly by the SW.
+ icp_qat_hw_comp_20_extended_delay_match_mode_t edmm;
+ icp_qat_hw_comp_20_hw_comp_format_t algo;
+ icp_qat_hw_comp_20_search_depth_t sd;
+ icp_qat_hw_comp_20_hbs_control_t hbs;
+ // Fields programmable directly by the FW.
+ // Block Drop enable. (Set by FW)
+ icp_qat_hw_comp_20_abd_t abd;
+ icp_qat_hw_comp_20_lllbd_ctrl_t lllbd;
+ // Advanced HW control (Set to default vals)
+ icp_qat_hw_comp_20_skip_hash_collision_t hash_col;
+ icp_qat_hw_comp_20_skip_hash_update_t hash_update;
+ icp_qat_hw_comp_20_byte_skip_t skip_ctrl;
+
+} icp_qat_hw_comp_20_config_csr_lower_t;
+
+/**
+*****************************************************************************
+* @ingroup icp_qat_fw_comn
+*
+* @description
+* Build the longword as expected by the HW
+*
+*****************************************************************************/
+static inline uint32_t
+ICP_QAT_FW_COMP_20_BUILD_CONFIG_LOWER(icp_qat_hw_comp_20_config_csr_lower_t csr)
+{
+ uint32_t val32 = 0;
+ // Programmable values
+ QAT_FIELD_SET(val32,
+ csr.algo,
+ ICP_QAT_HW_COMP_20_CONFIG_CSR_HW_COMP_FORMAT_BITPOS,
+ ICP_QAT_HW_COMP_20_CONFIG_CSR_HW_COMP_FORMAT_MASK);
+
+ QAT_FIELD_SET(val32,
+ csr.sd,
+ ICP_QAT_HW_COMP_20_CONFIG_CSR_SEARCH_DEPTH_BITPOS,
+ ICP_QAT_HW_COMP_20_CONFIG_CSR_SEARCH_DEPTH_MASK);
+
+ QAT_FIELD_SET(
+ val32,
+ csr.edmm,
+ ICP_QAT_HW_COMP_20_CONFIG_CSR_EXTENDED_DELAY_MATCH_MODE_BITPOS,
+ ICP_QAT_HW_COMP_20_CONFIG_CSR_EXTENDED_DELAY_MATCH_MODE_MASK);
+
+ QAT_FIELD_SET(val32,
+ csr.hbs,
+ ICP_QAT_HW_COMP_20_CONFIG_CSR_HBS_CONTROL_BITPOS,
+ ICP_QAT_HW_COMP_20_CONFIG_CSR_HBS_CONTROL_MASK);
+
+ QAT_FIELD_SET(val32,
+ csr.lllbd,
+ ICP_QAT_HW_COMP_20_CONFIG_CSR_LLLBD_CTRL_BITPOS,
+ ICP_QAT_HW_COMP_20_CONFIG_CSR_LLLBD_CTRL_MASK);
+
+ QAT_FIELD_SET(val32,
+ csr.hash_col,
+ ICP_QAT_HW_COMP_20_CONFIG_CSR_SKIP_HASH_COLLISION_BITPOS,
+ ICP_QAT_HW_COMP_20_CONFIG_CSR_SKIP_HASH_COLLISION_MASK);
+
+ QAT_FIELD_SET(val32,
+ csr.hash_update,
+ ICP_QAT_HW_COMP_20_CONFIG_CSR_SKIP_HASH_UPDATE_BITPOS,
+ ICP_QAT_HW_COMP_20_CONFIG_CSR_SKIP_HASH_UPDATE_MASK);
+
+ QAT_FIELD_SET(val32,
+ csr.skip_ctrl,
+ ICP_QAT_HW_COMP_20_CONFIG_CSR_BYTE_SKIP_BITPOS,
+ ICP_QAT_HW_COMP_20_CONFIG_CSR_BYTE_SKIP_MASK);
+ // Default values.
+
+ QAT_FIELD_SET(val32,
+ csr.abd,
+ ICP_QAT_HW_COMP_20_CONFIG_CSR_ABD_BITPOS,
+ ICP_QAT_HW_COMP_20_CONFIG_CSR_ABD_MASK);
+
+ QAT_FIELD_SET(val32,
+ csr.lllbd,
+ ICP_QAT_HW_COMP_20_CONFIG_CSR_LLLBD_CTRL_BITPOS,
+ ICP_QAT_HW_COMP_20_CONFIG_CSR_LLLBD_CTRL_MASK);
+
+ return BYTE_SWAP_32(val32);
+}
+
+/**
+*****************************************************************************
+* @ingroup icp_qat_fw_comn
+*
+* @description
+* Definition of the hw config csr. This representation has to be further
+* processed by the corresponding config build function.
+*
+*****************************************************************************/
+typedef struct icp_qat_hw_comp_20_config_csr_upper_s {
+ icp_qat_hw_comp_20_scb_control_t scb_ctrl;
+ icp_qat_hw_comp_20_rmb_control_t rmb_ctrl;
+ icp_qat_hw_comp_20_som_control_t som_ctrl;
+ icp_qat_hw_comp_20_skip_hash_rd_control_t skip_hash_ctrl;
+ icp_qat_hw_comp_20_scb_unload_control_t scb_unload_ctrl;
+ icp_qat_hw_comp_20_disable_token_fusion_control_t
+ disable_token_fusion_ctrl;
+ icp_qat_hw_comp_20_scb_mode_reset_mask_t scb_mode_reset;
+ uint16_t lazy;
+ uint16_t nice;
+} icp_qat_hw_comp_20_config_csr_upper_t;
+
+/**
+*****************************************************************************
+* @ingroup icp_qat_fw_comn
+*
+* @description
+* Build the longword as expected by the HW
+*
+*****************************************************************************/
+static inline uint32_t
+ICP_QAT_FW_COMP_20_BUILD_CONFIG_UPPER(icp_qat_hw_comp_20_config_csr_upper_t csr)
+{
+ uint32_t val32 = 0;
+
+ QAT_FIELD_SET(val32,
+ csr.scb_ctrl,
+ ICP_QAT_HW_COMP_20_CONFIG_CSR_SCB_CONTROL_BITPOS,
+ ICP_QAT_HW_COMP_20_CONFIG_CSR_SCB_CONTROL_MASK);
+
+ QAT_FIELD_SET(val32,
+ csr.rmb_ctrl,
+ ICP_QAT_HW_COMP_20_CONFIG_CSR_RMB_CONTROL_BITPOS,
+ ICP_QAT_HW_COMP_20_CONFIG_CSR_RMB_CONTROL_MASK);
+
+ QAT_FIELD_SET(val32,
+ csr.som_ctrl,
+ ICP_QAT_HW_COMP_20_CONFIG_CSR_SOM_CONTROL_BITPOS,
+ ICP_QAT_HW_COMP_20_CONFIG_CSR_SOM_CONTROL_MASK);
+
+ QAT_FIELD_SET(val32,
+ csr.skip_hash_ctrl,
+ ICP_QAT_HW_COMP_20_CONFIG_CSR_SKIP_HASH_RD_CONTROL_BITPOS,
+ ICP_QAT_HW_COMP_20_CONFIG_CSR_SKIP_HASH_RD_CONTROL_MASK);
+
+ QAT_FIELD_SET(val32,
+ csr.scb_unload_ctrl,
+ ICP_QAT_HW_COMP_20_CONFIG_CSR_SCB_UNLOAD_CONTROL_BITPOS,
+ ICP_QAT_HW_COMP_20_CONFIG_CSR_SCB_UNLOAD_CONTROL_MASK);
+
+ QAT_FIELD_SET(
+ val32,
+ csr.disable_token_fusion_ctrl,
+ ICP_QAT_HW_COMP_20_CONFIG_CSR_DISABLE_TOKEN_FUSION_CONTROL_BITPOS,
+ ICP_QAT_HW_COMP_20_CONFIG_CSR_DISABLE_TOKEN_FUSION_CONTROL_MASK);
+
+ QAT_FIELD_SET(val32,
+ csr.scb_mode_reset,
+ ICP_QAT_HW_COMP_20_CONFIG_CSR_SCB_MODE_RESET_MASK_BITPOS,
+ ICP_QAT_HW_COMP_20_CONFIG_CSR_SCB_MODE_RESET_MASK_MASK);
+
+ QAT_FIELD_SET(val32,
+ csr.lazy,
+ ICP_QAT_HW_COMP_20_CONFIG_CSR_LAZY_PARAM_BITPOS,
+ ICP_QAT_HW_COMP_20_CONFIG_CSR_LAZY_PARAM_MASK);
+
+ QAT_FIELD_SET(val32,
+ csr.nice,
+ ICP_QAT_HW_COMP_20_CONFIG_CSR_NICE_PARAM_BITPOS,
+ ICP_QAT_HW_COMP_20_CONFIG_CSR_NICE_PARAM_MASK);
+
+ return BYTE_SWAP_32(val32);
+}
+
+/**
+*****************************************************************************
+* @ingroup icp_qat_fw_comn
+*
+* @description
+* Definition of the hw config csr. This representation has to be further
+* processed by the corresponding config build function.
+*
+*****************************************************************************/
+typedef struct icp_qat_hw_decomp_20_config_csr_lower_s {
+ /* Fields programmable directly by the SW. */
+ icp_qat_hw_decomp_20_hbs_control_t hbs;
+ /* Advanced HW control (Set to default vals) */
+ icp_qat_hw_decomp_20_hw_comp_format_t algo;
+} icp_qat_hw_decomp_20_config_csr_lower_t;
+
+/**
+*****************************************************************************
+* @ingroup icp_qat_fw_comn
+*
+* @description
+* Build the longword as expected by the HW
+*
+*****************************************************************************/
+static inline uint32_t
+ICP_QAT_FW_DECOMP_20_BUILD_CONFIG_LOWER(
+ icp_qat_hw_decomp_20_config_csr_lower_t csr)
+{
+ uint32_t val32 = 0;
+
+ QAT_FIELD_SET(val32,
+ csr.hbs,
+ ICP_QAT_HW_DECOMP_20_CONFIG_CSR_HBS_CONTROL_BITPOS,
+ ICP_QAT_HW_DECOMP_20_CONFIG_CSR_HBS_CONTROL_MASK);
+
+ QAT_FIELD_SET(val32,
+ csr.algo,
+ ICP_QAT_HW_DECOMP_20_CONFIG_CSR_HW_DECOMP_FORMAT_BITPOS,
+ ICP_QAT_HW_DECOMP_20_CONFIG_CSR_HW_DECOMP_FORMAT_MASK);
+
+ return BYTE_SWAP_32(val32);
+}
+
+/**
+*****************************************************************************
+* @ingroup icp_qat_fw_comn
+*
+* @description
+* Definition of the hw config csr. This representation has to be further
+* processed by the corresponding config build function.
+*
+*****************************************************************************/
+typedef struct icp_qat_hw_decomp_20_config_csr_upper_s {
+ /* Advanced HW control (Set to default vals) */
+ icp_qat_hw_decomp_20_speculative_decoder_control_t sdc;
+ icp_qat_hw_decomp_20_mini_cam_control_t mcc;
+} icp_qat_hw_decomp_20_config_csr_upper_t;
+
+/**
+*****************************************************************************
+* @ingroup icp_qat_fw_comn
+*
+* @description
+* Build the longword as expected by the HW
+*
+*****************************************************************************/
+static inline uint32_t
+ICP_QAT_FW_DECOMP_20_BUILD_CONFIG_UPPER(
+ icp_qat_hw_decomp_20_config_csr_upper_t csr)
+{
+ uint32_t val32 = 0;
+
+ QAT_FIELD_SET(
+ val32,
+ csr.sdc,
+ ICP_QAT_HW_DECOMP_20_CONFIG_CSR_SPECULATIVE_DECODER_CONTROL_BITPOS,
+ ICP_QAT_HW_DECOMP_20_CONFIG_CSR_SPECULATIVE_DECODER_CONTROL_MASK);
+
+ QAT_FIELD_SET(val32,
+ csr.mcc,
+ ICP_QAT_HW_DECOMP_20_CONFIG_CSR_MINI_CAM_CONTROL_BITPOS,
+ ICP_QAT_HW_DECOMP_20_CONFIG_CSR_MINI_CAM_CONTROL_MASK);
+
+ return BYTE_SWAP_32(val32);
+}
+
+#endif /* ICP_QAT_HW__2X_COMP_H_ */
Index: sys/dev/qat/qat_api/firmware/include/icp_qat_hw_20_comp_defs.h
===================================================================
--- /dev/null
+++ sys/dev/qat/qat_api/firmware/include/icp_qat_hw_20_comp_defs.h
@@ -0,0 +1,443 @@
+/* SPDX-License-Identifier: BSD-3-Clause */
+/* Copyright(c) 2007-2022 Intel Corporation */
+/* $FreeBSD$ */
+/*
+ ****************************************************************************
+ * @file icp_qat_hw_20_comp_defs.h, (autogenerated at 04-19-18 16:06)
+ * @defgroup icp_qat_hw_comp_20
+ * @ingroup icp_qat_hw_comp_20
+ * @description
+ * This file represents the HW configuration CSR definitions
+ ****************************************************************************
+ */
+
+#ifndef _ICP_QAT_HW_20_COMP_DEFS_H
+#define _ICP_QAT_HW_20_COMP_DEFS_H
+
+/*****************************************************************************/
+/* SCB Disabled - Set by FW, located in upper 32bit */
+#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SCB_CONTROL_BITPOS 31
+#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SCB_CONTROL_MASK 0x1
+/*
+ ****************************************************************************
+ * @ingroup icp_qat_hw_defs
+ * @description
+ * Enumeration of possible SCB_CONTROL field values
+ *****************************************************************************/
+typedef enum {
+ ICP_QAT_HW_COMP_20_SCB_CONTROL_ENABLE = 0x0,
+ /* Normal Mode using SCB (Default) */
+ ICP_QAT_HW_COMP_20_SCB_CONTROL_DISABLE = 0x1,
+ /* Legacy CPM1.x Mode with SCB disabled. */
+} icp_qat_hw_comp_20_scb_control_t;
+
+#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SCB_CONTROL_DEFAULT_VAL \
+ ICP_QAT_HW_COMP_20_SCB_CONTROL_DISABLE
+
+/*****************************************************************************/
+/* Reset Bit Mask Disabled - Set by FW , located in upper 32bit */
+#define ICP_QAT_HW_COMP_20_CONFIG_CSR_RMB_CONTROL_BITPOS 30
+#define ICP_QAT_HW_COMP_20_CONFIG_CSR_RMB_CONTROL_MASK 0x1
+/*
+ ****************************************************************************
+ * @ingroup icp_qat_hw_defs
+ * @description
+ * Enumeration of possible RMB_CONTROL field values
+ *****************************************************************************/
+typedef enum {
+ ICP_QAT_HW_COMP_20_RMB_CONTROL_RESET_ALL = 0x0,
+ /* Reset all data structures with a set_config command. (Set by FW) */
+ ICP_QAT_HW_COMP_20_RMB_CONTROL_RESET_FC_ONLY = 0x1,
+ /* Reset only the Frequency Counters (LFCT) with a set_config command.
+ */
+} icp_qat_hw_comp_20_rmb_control_t;
+
+#define ICP_QAT_HW_COMP_20_CONFIG_CSR_RMB_CONTROL_DEFAULT_VAL \
+ ICP_QAT_HW_COMP_20_RMB_CONTROL_RESET_ALL
+
+/*****************************************************************************/
+/* Slice Operation Mode (SOM) - Set By FW, located in upper 32bit */
+#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SOM_CONTROL_BITPOS 28
+#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SOM_CONTROL_MASK 0x3
+/*
+ ****************************************************************************
+ * @ingroup icp_qat_hw_defs
+ * @description
+ * Enumeration of possible SOM_CONTROL field values
+ *****************************************************************************/
+typedef enum {
+ ICP_QAT_HW_COMP_20_SOM_CONTROL_NORMAL_MODE = 0x0,
+ /* Normal mode. */
+ ICP_QAT_HW_COMP_20_SOM_CONTROL_REPLAY_MODE = 0x1,
+ /* Replay mode */
+ ICP_QAT_HW_COMP_20_SOM_CONTROL_INPUT_CRC = 0x2,
+ /* Input CRC Mode */
+ ICP_QAT_HW_COMP_20_SOM_CONTROL_RESERVED_MODE = 0x3,
+ /* Reserved. */
+} icp_qat_hw_comp_20_som_control_t;
+
+#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SOM_CONTROL_DEFAULT_VAL \
+ ICP_QAT_HW_COMP_20_SOM_CONTROL_NORMAL_MODE
+
+/*****************************************************************************/
+/* Skip Hash Read (Set By FW) , located in upper 32bit */
+#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SKIP_HASH_RD_CONTROL_BITPOS 27
+#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SKIP_HASH_RD_CONTROL_MASK 0x1
+/*
+ ****************************************************************************
+ * @ingroup icp_qat_hw_defs
+ * @description
+ * Enumeration of possible SKIP_HASH_RD_CONTROL field values
+ *****************************************************************************/
+typedef enum {
+ ICP_QAT_HW_COMP_20_SKIP_HASH_RD_CONTROL_NO_SKIP = 0x0,
+ /* When set to 0, hash reads are not skipped. */
+ ICP_QAT_HW_COMP_20_SKIP_HASH_RD_CONTROL_SKIP_HASH_READS = 0x1,
+ /* Hash reads are skipped. */
+} icp_qat_hw_comp_20_skip_hash_rd_control_t;
+
+#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SKIP_HASH_RD_CONTROL_DEFAULT_VAL \
+ ICP_QAT_HW_COMP_20_SKIP_HASH_RD_CONTROL_NO_SKIP
+
+/*****************************************************************************/
+/* SCB Unload Disable, located in upper 32bit */
+#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SCB_UNLOAD_CONTROL_BITPOS 26
+#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SCB_UNLOAD_CONTROL_MASK 0x1
+/*
+ ****************************************************************************
+ * @ingroup icp_qat_hw_defs
+ * @description
+ * Enumeration of possible SCB_UNLOAD_CONTROL field values
+ *****************************************************************************/
+typedef enum {
+ ICP_QAT_HW_COMP_20_SCB_UNLOAD_CONTROL_UNLOAD = 0x0,
+ /* Unloads the LFCT and flushes the State Registers. */
+ ICP_QAT_HW_COMP_20_SCB_UNLOAD_CONTROL_NO_UNLOAD = 0x1,
+ /* Does not unload the LFCT, but flushes the State Registers. */
+} icp_qat_hw_comp_20_scb_unload_control_t;
+
+#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SCB_UNLOAD_CONTROL_DEFAULT_VAL \
+ ICP_QAT_HW_COMP_20_SCB_UNLOAD_CONTROL_UNLOAD
+
+/*****************************************************************************/
+/* Disable token fusion, located in upper 32bit */
+#define ICP_QAT_HW_COMP_20_CONFIG_CSR_DISABLE_TOKEN_FUSION_CONTROL_BITPOS 21
+#define ICP_QAT_HW_COMP_20_CONFIG_CSR_DISABLE_TOKEN_FUSION_CONTROL_MASK 0x1
+/*
+ ****************************************************************************
+ * @ingroup icp_qat_hw_defs
+ * @description
+ * Enumeration of possible DISABLE_TOKEN_FUSION_CONTROL field values
+ *****************************************************************************/
+typedef enum {
+ ICP_QAT_HW_COMP_20_DISABLE_TOKEN_FUSION_CONTROL_ENABLE = 0x0,
+ /* Enables token fusion. */
+ ICP_QAT_HW_COMP_20_DISABLE_TOKEN_FUSION_CONTROL_DISABLE = 0x1,
+ /* Disables token fusion. */
+} icp_qat_hw_comp_20_disable_token_fusion_control_t;
+
+#define ICP_QAT_HW_COMP_20_CONFIG_CSR_DISABLE_TOKEN_FUSION_CONTROL_DEFAULT_VAL \
+ ICP_QAT_HW_COMP_20_DISABLE_TOKEN_FUSION_CONTROL_ENABLE
+
+/*****************************************************************************/
+/* SCB Mode Reset Mask (Set By FW) , located in upper 32bit */
+#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SCB_MODE_RESET_MASK_BITPOS 18
+#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SCB_MODE_RESET_MASK_MASK 0x1
+/*
+ ****************************************************************************
+ * @ingroup icp_qat_hw_defs
+ * @description
+ * Enumeration of possible SCB_MODE_RESET_MASK field values
+ *****************************************************************************/
+typedef enum {
+ ICP_QAT_HW_COMP_20_SCB_MODE_RESET_MASK_RESET_COUNTERS = 0x0,
+ /* iLZ77 mode: Reset LFCT, OBC */
+ ICP_QAT_HW_COMP_20_SCB_MODE_RESET_MASK_RESET_COUNTERS_AND_HISTORY = 0x1,
+ /* iLZ77 mode: Reset LFCT, OBC, HB, HT */
+} icp_qat_hw_comp_20_scb_mode_reset_mask_t;
+
+#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SCB_MODE_RESET_MASK_DEFAULT_VAL \
+ ICP_QAT_HW_COMP_20_SCB_MODE_RESET_MASK_RESET_COUNTERS
+
+/*****************************************************************************/
+/* Lazy - For iLZ77 and Static DEFLATE, Lazy = 102h , located in upper
+ * 32bit */
+#define ICP_QAT_HW_COMP_20_CONFIG_CSR_LAZY_PARAM_BITPOS 9
+#define ICP_QAT_HW_COMP_20_CONFIG_CSR_LAZY_PARAM_MASK 0x1ff
+#define ICP_QAT_HW_COMP_20_CONFIG_CSR_LAZY_PARAM_DEFAULT_VAL 258
+
+/*****************************************************************************/
+/* Nice - For iLZ77 and Static DEFLATE, Nice = 103h , located in upper
+ * 32bit */
+#define ICP_QAT_HW_COMP_20_CONFIG_CSR_NICE_PARAM_BITPOS 0
+#define ICP_QAT_HW_COMP_20_CONFIG_CSR_NICE_PARAM_MASK 0x1ff
+#define ICP_QAT_HW_COMP_20_CONFIG_CSR_NICE_PARAM_DEFAULT_VAL 259
+
+/*****************************************************************************/
+/* History Buffer Size (Set By the Driver/ Application), located in lower 32bit
+ */
+#define ICP_QAT_HW_COMP_20_CONFIG_CSR_HBS_CONTROL_BITPOS 14
+#define ICP_QAT_HW_COMP_20_CONFIG_CSR_HBS_CONTROL_MASK 0x7
+/*
+ ****************************************************************************
+ * @ingroup icp_qat_hw_defs
+ * @description
+ * Enumeration of possible HBS_CONTROL field values
+ *****************************************************************************/
+typedef enum {
+ ICP_QAT_HW_COMP_20_HBS_CONTROL_HBS_IS_32KB = 0x0,
+ /* 000b - 32KB */
+ ICP_QAT_HW_COMP_20_HBS_CONTROL_HBS_IS_64KB = 0x1,
+ /* 001b - 64KB */
+} icp_qat_hw_comp_20_hbs_control_t;
+
+#define ICP_QAT_HW_COMP_20_CONFIG_CSR_HBS_CONTROL_DEFAULT_VAL \
+ ICP_QAT_HW_COMP_20_HBS_CONTROL_HBS_IS_32KB
+
+/*****************************************************************************/
+/* Adaptive Block Drop (Set By FW if Dynamic), located in lower 32bit */
+#define ICP_QAT_HW_COMP_20_CONFIG_CSR_ABD_BITPOS 13
+#define ICP_QAT_HW_COMP_20_CONFIG_CSR_ABD_MASK 0x1
+/*
+ ****************************************************************************
+ * @ingroup icp_qat_hw_defs
+ * @description
+ * Enumeration of possible ABD field values
+ *****************************************************************************/
+typedef enum {
+ ICP_QAT_HW_COMP_20_ABD_ABD_ENABLED = 0x0,
+ /* 0b - Feature enabled. */
+ ICP_QAT_HW_COMP_20_ABD_ABD_DISABLED = 0x1,
+ /* 1b - Feature disabled. */
+} icp_qat_hw_comp_20_abd_t;
+
+#define ICP_QAT_HW_COMP_20_CONFIG_CSR_ABD_DEFAULT_VAL \
+ ICP_QAT_HW_COMP_20_ABD_ABD_ENABLED
+
+/*****************************************************************************/
+/* Literal+Length Limit Block Drop Block Drop, (Set By FW if Dynamic) , located
+ * in lower 32bit */
+#define ICP_QAT_HW_COMP_20_CONFIG_CSR_LLLBD_CTRL_BITPOS 12
+#define ICP_QAT_HW_COMP_20_CONFIG_CSR_LLLBD_CTRL_MASK 0x1
+/*
+ ****************************************************************************
+ * @ingroup icp_qat_hw_defs
+ * @description
+ * Enumeration of possible LLLBD_CTRL field values
+ *****************************************************************************/
+typedef enum {
+ ICP_QAT_HW_COMP_20_LLLBD_CTRL_LLLBD_ENABLED = 0x0,
+ /* 0b - Feature enabled. */
+ ICP_QAT_HW_COMP_20_LLLBD_CTRL_LLLBD_DISABLED = 0x1,
+ /* 1b - Feature disabled. */
+} icp_qat_hw_comp_20_lllbd_ctrl_t;
+
+#define ICP_QAT_HW_COMP_20_CONFIG_CSR_LLLBD_CTRL_DEFAULT_VAL \
+ ICP_QAT_HW_COMP_20_LLLBD_CTRL_LLLBD_ENABLED
+
+/*****************************************************************************/
+/* Search Depth (SD) (Set By Driver/Application), located in lower 32bit */
+#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SEARCH_DEPTH_BITPOS 8
+#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SEARCH_DEPTH_MASK 0xf
+/*
+ ****************************************************************************
+ * @ingroup icp_qat_hw_defs
+ * @description
+ * Enumeration of possible SEARCH_DEPTH field values
+ *****************************************************************************/
+typedef enum {
+ ICP_QAT_HW_COMP_20_SEARCH_DEPTH_LEVEL_1 = 0x1,
+ /* 0001b - Level 1 (search depth = 2^1 = 2) */
+ ICP_QAT_HW_COMP_20_SEARCH_DEPTH_LEVEL_6 = 0x3,
+ /* 0011b - Level 6 (search depth = 2^3 = 8) */
+ ICP_QAT_HW_COMP_20_SEARCH_DEPTH_LEVEL_9 = 0x4,
+ /* 0100b - Level 9 (search depth = 2^4 = 16) */
+} icp_qat_hw_comp_20_search_depth_t;
+
+#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SEARCH_DEPTH_DEFAULT_VAL \
+ ICP_QAT_HW_COMP_20_SEARCH_DEPTH_LEVEL_1
+
+/*****************************************************************************/
+/* Compression Format (Set By Driver/Application. Also See CMD ID), located in
+ * lower 32bit */
+#define ICP_QAT_HW_COMP_20_CONFIG_CSR_HW_COMP_FORMAT_BITPOS 5
+#define ICP_QAT_HW_COMP_20_CONFIG_CSR_HW_COMP_FORMAT_MASK 0x7
+/*
+ ****************************************************************************
+ * @ingroup icp_qat_hw_defs
+ * @description
+ * Enumeration of possible HW_COMP_FORMAT field values
+ *****************************************************************************/
+typedef enum {
+ ICP_QAT_HW_COMP_20_HW_COMP_FORMAT_ILZ77 = 0x0,
+ /* 000 - iLZ77. (Must set Min_Match = 3 bytes and HB size = 32KB.) */
+ ICP_QAT_HW_COMP_20_HW_COMP_FORMAT_DEFLATE = 0x1,
+ /* 001 - Static DEFLATE. (Must set Min_Match = 3 bytes and HB size =
+ 32KB.) */
+} icp_qat_hw_comp_20_hw_comp_format_t;
+
+#define ICP_QAT_HW_COMP_20_CONFIG_CSR_HW_COMP_FORMAT_DEFAULT_VAL \
+ ICP_QAT_HW_COMP_20_HW_COMP_FORMAT_DEFLATE
+
+/*****************************************************************************/
+/* Skip Hash Collision (Set By FW to default value), located in lower 32bit */
+#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SKIP_HASH_COLLISION_BITPOS 3
+#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SKIP_HASH_COLLISION_MASK 0x1
+/*
+ ****************************************************************************
+ * @ingroup icp_qat_hw_defs
+ * @description
+ * Enumeration of possible SKIP_HASH_COLLISION field values
+ *****************************************************************************/
+typedef enum {
+ ICP_QAT_HW_COMP_20_SKIP_HASH_COLLISION_ALLOW = 0x0,
+ /* When set to 0, hash collisions are allowed. */
+ ICP_QAT_HW_COMP_20_SKIP_HASH_COLLISION_DONT_ALLOW = 0x1,
+ /* When set to 0, hash collisions are allowed. */
+} icp_qat_hw_comp_20_skip_hash_collision_t;
+
+#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SKIP_HASH_COLLISION_DEFAULT_VAL \
+ ICP_QAT_HW_COMP_20_SKIP_HASH_COLLISION_ALLOW
+
+/*****************************************************************************/
+/* Skip Hash Update (Set By FW to default value) , located in lower 32bit */
+#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SKIP_HASH_UPDATE_BITPOS 2
+#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SKIP_HASH_UPDATE_MASK 0x1
+/*
+ ****************************************************************************
+ * @ingroup icp_qat_hw_defs
+ * @description
+ * Enumeration of possible SKIP_HASH_UPDATE field values
+ *****************************************************************************/
+typedef enum {
+ ICP_QAT_HW_COMP_20_SKIP_HASH_UPDATE_ALLOW = 0x0,
+ /* 0 - hash updates are not skipped. */
+ ICP_QAT_HW_COMP_20_SKIP_HASH_UPDATE_DONT_ALLOW = 0x1,
+ /* 1 - hash updates are skipped. */
+} icp_qat_hw_comp_20_skip_hash_update_t;
+
+#define ICP_QAT_HW_COMP_20_CONFIG_CSR_SKIP_HASH_UPDATE_DEFAULT_VAL \
+ ICP_QAT_HW_COMP_20_SKIP_HASH_UPDATE_ALLOW
+
+/*****************************************************************************/
+/* 3-Byte Match Skip (Set By FW to default value), located in lower 32bit */
+#define ICP_QAT_HW_COMP_20_CONFIG_CSR_BYTE_SKIP_BITPOS 1
+#define ICP_QAT_HW_COMP_20_CONFIG_CSR_BYTE_SKIP_MASK 0x1
+/*
+ ****************************************************************************
+ * @ingroup icp_qat_hw_defs
+ * @description
+ * Enumeration of possible BYTE_SKIP field values
+ *****************************************************************************/
+typedef enum {
+ ICP_QAT_HW_COMP_20_BYTE_SKIP_3BYTE_TOKEN = 0x0,
+ /* 0 - Use 3-byte token */
+ ICP_QAT_HW_COMP_20_BYTE_SKIP_3BYTE_LITERAL = 0x1,
+ /* 0 - Use 3-byte literal */
+} icp_qat_hw_comp_20_byte_skip_t;
+
+#define ICP_QAT_HW_COMP_20_CONFIG_CSR_BYTE_SKIP_DEFAULT_VAL \
+ ICP_QAT_HW_COMP_20_BYTE_SKIP_3BYTE_TOKEN
+
+/*****************************************************************************/
+/* Extended Delayed Match Mode enabled (Set By the Driver), located in lower
+ * 32bit */
+#define ICP_QAT_HW_COMP_20_CONFIG_CSR_EXTENDED_DELAY_MATCH_MODE_BITPOS 0
+#define ICP_QAT_HW_COMP_20_CONFIG_CSR_EXTENDED_DELAY_MATCH_MODE_MASK 0x1
+/*
+ ****************************************************************************
+ * @ingroup icp_qat_hw_defs
+ * @description
+ * Enumeration of possible EXTENDED_DELAY_MATCH_MODE field values
+ *****************************************************************************/
+typedef enum {
+ ICP_QAT_HW_COMP_20_EXTENDED_DELAY_MATCH_MODE_EDMM_DISABLED = 0x0,
+ /* 0 - EXTENDED_DELAY_MATCH_MODE disabled */
+ ICP_QAT_HW_COMP_20_EXTENDED_DELAY_MATCH_MODE_EDMM_ENABLED = 0x1,
+ /* 1 - EXTENDED_DELAY_MATCH_MODE enabled */
+} icp_qat_hw_comp_20_extended_delay_match_mode_t;
+
+#define ICP_QAT_HW_COMP_20_CONFIG_CSR_EXTENDED_DELAY_MATCH_MODE_DEFAULT_VAL \
+ ICP_QAT_HW_COMP_20_EXTENDED_DELAY_MATCH_MODE_EDMM_DISABLED
+
+/*****************************************************************************/
+/* Speculative Decoder Disable (Set By the Driver/ Application), located in
+ * upper 32bit */
+#define ICP_QAT_HW_DECOMP_20_CONFIG_CSR_SPECULATIVE_DECODER_CONTROL_BITPOS 31
+#define ICP_QAT_HW_DECOMP_20_CONFIG_CSR_SPECULATIVE_DECODER_CONTROL_MASK 0x1
+/*
+ ****************************************************************************
+ * @ingroup icp_qat_hw_defs
+ * @description
+ * Enumeration of possible SPECULATIVE_DECODER_CONTROL field values
+ *****************************************************************************/
+typedef enum {
+ ICP_QAT_HW_DECOMP_20_SPECULATIVE_DECODER_CONTROL_ENABLE = 0x0,
+ /* 0b - Enabled */
+ ICP_QAT_HW_DECOMP_20_SPECULATIVE_DECODER_CONTROL_DISABLE = 0x1,
+ /* 1b - Disabled */
+} icp_qat_hw_decomp_20_speculative_decoder_control_t;
+
+#define ICP_QAT_HW_DECOMP_20_CONFIG_CSR_SPECULATIVE_DECODER_CONTROL_DEFAULT_VAL \
+ ICP_QAT_HW_DECOMP_20_SPECULATIVE_DECODER_CONTROL_ENABLE
+
+/*****************************************************************************/
+/* Mini CAM Disable (Set By the Driver/ Application), located in upper 32bit */
+#define ICP_QAT_HW_DECOMP_20_CONFIG_CSR_MINI_CAM_CONTROL_BITPOS 30
+#define ICP_QAT_HW_DECOMP_20_CONFIG_CSR_MINI_CAM_CONTROL_MASK 0x1
+/*
+ ****************************************************************************
+ * @ingroup icp_qat_hw_defs
+ * @description
+ * Enumeration of possible MINI_CAM_CONTROL field values
+ *****************************************************************************/
+typedef enum {
+ ICP_QAT_HW_DECOMP_20_MINI_CAM_CONTROL_ENABLE = 0x0,
+ /* 0b - Enabled */
+ ICP_QAT_HW_DECOMP_20_MINI_CAM_CONTROL_DISABLE = 0x1,
+ /* 1b - Disabled */
+} icp_qat_hw_decomp_20_mini_cam_control_t;
+
+#define ICP_QAT_HW_DECOMP_20_CONFIG_CSR_MINI_CAM_CONTROL_DEFAULT_VAL \
+ ICP_QAT_HW_DECOMP_20_MINI_CAM_CONTROL_ENABLE
+
+/*****************************************************************************/
+/* History Buffer Size (Set By the Driver/ Application), located in lower 32bit
+ */
+#define ICP_QAT_HW_DECOMP_20_CONFIG_CSR_HBS_CONTROL_BITPOS 14
+#define ICP_QAT_HW_DECOMP_20_CONFIG_CSR_HBS_CONTROL_MASK 0x7
+/*
+ ****************************************************************************
+ * @ingroup icp_qat_hw_defs
+ * @description
+ * Enumeration of possible HBS_CONTROL field values
+ *****************************************************************************/
+typedef enum {
+ ICP_QAT_HW_DECOMP_20_HBS_CONTROL_HBS_IS_32KB = 0x0,
+ /* 000b - 32KB */
+} icp_qat_hw_decomp_20_hbs_control_t;
+
+#define ICP_QAT_HW_DECOMP_20_CONFIG_CSR_HBS_CONTROL_DEFAULT_VAL \
+ ICP_QAT_HW_DECOMP_20_HBS_CONTROL_HBS_IS_32KB
+
+/*****************************************************************************/
+/* Decompression Format (Set By Driver/Application. Also See CMD ID), located in
+ * lower 32bit */
+#define ICP_QAT_HW_DECOMP_20_CONFIG_CSR_HW_DECOMP_FORMAT_BITPOS 5
+#define ICP_QAT_HW_DECOMP_20_CONFIG_CSR_HW_DECOMP_FORMAT_MASK 0x7
+/*
+ ****************************************************************************
+ * @ingroup icp_qat_hw_defs
+ * @description
+ * Enumeration of possible HW_DECOMP_FORMAT field values
+ *****************************************************************************/
+typedef enum {
+ ICP_QAT_HW_DECOMP_20_HW_DECOMP_FORMAT_DEFLATE = 0x1,
+ /* 001 - Static DEFLATE. (Must set Min_Match = 3 bytes and HB size =
+ 32KB.) */
+} icp_qat_hw_decomp_20_hw_comp_format_t;
+
+#define ICP_QAT_HW_DECOMP_20_CONFIG_CSR_HW_DECOMP_FORMAT_DEFAULT_VAL \
+ ICP_QAT_HW_DECOMP_20_HW_DECOMP_FORMAT_DEFLATE
+
+#endif //_ICP_QAT_HW_20_COMP_DEFS_H
Index: sys/dev/qat/qat_api/include/icp_sal_versions.h
===================================================================
--- sys/dev/qat/qat_api/include/icp_sal_versions.h
+++ sys/dev/qat/qat_api/include/icp_sal_versions.h
@@ -26,8 +26,8 @@
/**< Max length of hardware version string */
/* Part name and number of the accelerator device */
-#define SAL_INFO2_DRIVER_SW_VERSION_MAJ_NUMBER 3
-#define SAL_INFO2_DRIVER_SW_VERSION_MIN_NUMBER 11
+#define SAL_INFO2_DRIVER_SW_VERSION_MAJ_NUMBER 3
+#define SAL_INFO2_DRIVER_SW_VERSION_MIN_NUMBER 12
#define SAL_INFO2_DRIVER_SW_VERSION_PATCH_NUMBER 0
/**
Index: sys/dev/qat/qat_api/qat_direct/include/icp_accel_devices.h
===================================================================
--- sys/dev/qat/qat_api/qat_direct/include/icp_accel_devices.h
+++ sys/dev/qat/qat_api/qat_direct/include/icp_accel_devices.h
@@ -97,7 +97,8 @@
DEVICE_200XX,
DEVICE_200XXVF,
DEVICE_C4XXX,
- DEVICE_C4XXXVF
+ DEVICE_C4XXXVF,
+ DEVICE_GEN4
} device_type_t;
/*
Index: sys/dev/qat/qat_api/qat_kernel/src/qat_transport.c
===================================================================
--- sys/dev/qat/qat_api/qat_kernel/src/qat_transport.c
+++ sys/dev/qat/qat_api/qat_kernel/src/qat_transport.c
@@ -369,13 +369,20 @@
icp_adf_updateQueueTail(icp_comms_trans_handle trans_handle)
{
struct adf_etr_ring_data *ring = trans_handle;
+ struct adf_hw_csr_ops *csr_ops;
ICP_CHECK_FOR_NULL_PARAM_VOID(ring);
+ ICP_CHECK_FOR_NULL_PARAM_VOID(ring->bank);
+ ICP_CHECK_FOR_NULL_PARAM_VOID(ring->bank->accel_dev);
- WRITE_CSR_RING_TAIL(ring->bank->csr_addr,
- ring->bank->bank_number,
- ring->ring_number,
- ring->tail);
+ csr_ops = GET_CSR_OPS(ring->bank->accel_dev);
+
+ ICP_CHECK_FOR_NULL_PARAM_VOID(csr_ops);
+
+ csr_ops->write_csr_ring_tail(ring->bank->csr_addr,
+ ring->bank->bank_number,
+ ring->ring_number,
+ ring->tail);
ring->csr_tail_offset = ring->tail;
}
Index: sys/dev/qat/qat_api/qat_utils/include/qat_utils.h
===================================================================
--- sys/dev/qat/qat_api/qat_utils/include/qat_utils.h
+++ sys/dev/qat/qat_api/qat_utils/include/qat_utils.h
@@ -848,4 +848,25 @@
uint32_t keyLenInBytes,
uint8_t *in,
uint8_t *out);
+
+/**
+ * @ingroup QatUtils
+ *
+ * @brief Converts AES forward key to reverse key
+ *
+ * @param key - pointer to symetric key.
+ * keyLenInBytes - key length
+ * out - pointer to output buffer for reversed key
+ * The in and out buffers need to be at least AES block size long
+ * as defined in rfc3686 (16 bytes)
+ *
+ * @li Reentrant: yes
+ * @li IRQ safe: yes
+ *
+ * @return - CPA_STATUS_SUCCESS/CPA_STATUS_FAIL
+ *
+ */
+CpaStatus qatUtilsAESKeyExpansionForward(uint8_t *key,
+ uint32_t keyLenInBytes,
+ uint32_t *out);
#endif
Index: sys/dev/qat/qat_api/qat_utils/src/QatUtilsCrypto.c
===================================================================
--- sys/dev/qat/qat_api/qat_utils/src/QatUtilsCrypto.c
+++ sys/dev/qat/qat_api/qat_utils/src/QatUtilsCrypto.c
@@ -3,6 +3,10 @@
/* $FreeBSD$ */
#include "qat_utils.h"
+#define AES_128_KEY_LEN_BYTES 16
+#define AES_192_KEY_LEN_BYTES 24
+#define AES_256_KEY_LEN_BYTES 32
+
CpaStatus
qatUtilsHashMD5(uint8_t *in, uint8_t *out)
{
@@ -150,3 +154,38 @@
return CPA_STATUS_SUCCESS;
}
+
+CpaStatus
+qatUtilsAESKeyExpansionForward(uint8_t *key,
+ uint32_t keyLenInBytes,
+ uint32_t *out)
+{
+ rijndael_ctx ctx;
+ uint32_t i = 0, j = 0;
+ uint32_t lw_per_round = 4;
+ int32_t lw_left_to_copy = keyLenInBytes / lw_per_round;
+ uint32_t *key_pointer = NULL;
+
+ /* Error check for wrong input key len */
+ if (AES_128_KEY_LEN_BYTES != keyLenInBytes &&
+ AES_192_KEY_LEN_BYTES != keyLenInBytes &&
+ AES_256_KEY_LEN_BYTES != keyLenInBytes) {
+ return CPA_STATUS_INVALID_PARAM;
+ }
+
+ rijndael_set_key(&ctx, key, keyLenInBytes << BYTE_TO_BITS_SHIFT);
+
+ /* Pointer to the last round of expanded key. */
+ key_pointer = &ctx.ek[lw_per_round * ctx.Nr];
+
+ while (lw_left_to_copy > 0) {
+ for (i = 0; i < MIN(lw_left_to_copy, lw_per_round); i++, j++) {
+ out[j] = __builtin_bswap32(key_pointer[i]);
+ }
+
+ lw_left_to_copy -= lw_per_round;
+ key_pointer -= lw_left_to_copy;
+ }
+
+ return CPA_STATUS_SUCCESS;
+}
Index: sys/dev/qat/qat_common/adf_accel_engine.c
===================================================================
--- sys/dev/qat/qat_common/adf_accel_engine.c
+++ sys/dev/qat/qat_common/adf_accel_engine.c
@@ -99,11 +99,24 @@
*/
if (hw_device->get_obj_name && hw_device->get_obj_cfg_ae_mask) {
unsigned long service_mask = hw_device->service_mask;
+ enum adf_accel_unit_services service_type =
+ ADF_ACCEL_SERVICE_NULL;
- if (hw_device->service_mask &&
- !(test_bit(i, &service_mask)))
+ if (hw_device->get_service_type)
+ service_type =
+ hw_device->get_service_type(accel_dev, i);
+ else
+ service_type = BIT(i);
+
+ if (service_mask && !(service_mask & service_type))
continue;
- obj_name = hw_device->get_obj_name(accel_dev, BIT(i));
+
+ obj_name =
+ hw_device->get_obj_name(accel_dev, service_type);
+ cfg_ae_mask =
+ hw_device->get_obj_cfg_ae_mask(accel_dev,
+ service_type);
+
if (!obj_name) {
device_printf(
GET_DEV(accel_dev),
@@ -111,10 +124,8 @@
BIT(i));
goto out_err;
}
- if (!hw_device->get_obj_cfg_ae_mask(accel_dev, BIT(i)))
+ if (!cfg_ae_mask)
continue;
- cfg_ae_mask =
- hw_device->get_obj_cfg_ae_mask(accel_dev, BIT(i));
if (qat_uclo_set_cfg_ae_mask(loader_data->fw_loader,
cfg_ae_mask)) {
device_printf(GET_DEV(accel_dev),
@@ -172,17 +183,12 @@
{
struct adf_fw_loader_data *loader_data = accel_dev->fw_loader;
struct adf_hw_device_data *hw_data = accel_dev->hw_device;
- uint32_t ae_ctr, ae, max_aes = GET_MAX_ACCELENGINES(accel_dev);
+ uint32_t ae_ctr;
if (!hw_data->fw_name)
return 0;
- for (ae = 0, ae_ctr = 0; ae < max_aes; ae++) {
- if (hw_data->ae_mask & (1 << ae)) {
- qat_hal_start(loader_data->fw_loader, ae, 0xFF);
- ae_ctr++;
- }
- }
+ ae_ctr = qat_hal_start(loader_data->fw_loader);
device_printf(GET_DEV(accel_dev),
"qat_dev%d started %d acceleration engines\n",
accel_dev->accel_id,
Index: sys/dev/qat/qat_common/adf_cfg_bundle.h
===================================================================
--- sys/dev/qat/qat_common/adf_cfg_bundle.h
+++ sys/dev/qat/qat_common/adf_cfg_bundle.h
@@ -49,7 +49,29 @@
struct adf_accel_dev *accel_dev);
void adf_cfg_init_ring2serv_mapping(struct adf_accel_dev *accel_dev,
- struct adf_cfg_bundle *bundle);
+ struct adf_cfg_bundle *bundle,
+ struct adf_cfg_device *device);
int adf_cfg_rel_ring2serv_mapping(struct adf_cfg_bundle *bundle);
+
+static inline void
+adf_get_ring_svc_map_data(struct adf_hw_device_data *hw_data,
+ int bundle_num,
+ int ring_pair_index,
+ u8 *serv_type,
+ int *ring_index,
+ int *num_rings_per_srv)
+{
+ if (hw_data->get_ring_svc_map_data)
+ return hw_data->get_ring_svc_map_data(ring_pair_index,
+ hw_data->ring_to_svc_map,
+ serv_type,
+ ring_index,
+ num_rings_per_srv,
+ bundle_num);
+ *serv_type = GET_SRV_TYPE(hw_data->ring_to_svc_map, ring_pair_index);
+ *num_rings_per_srv =
+ hw_data->num_rings_per_bank / (2 * ADF_CFG_NUM_SERVICES);
+ *ring_index = (*num_rings_per_srv) * ring_pair_index;
+}
#endif
Index: sys/dev/qat/qat_common/adf_cfg_bundle.c
===================================================================
--- sys/dev/qat/qat_common/adf_cfg_bundle.c
+++ sys/dev/qat/qat_common/adf_cfg_bundle.c
@@ -157,16 +157,23 @@
{
struct adf_cfg_instance *cfg_instance = NULL;
int ring_pair_index = 0;
+ int ring_index = 0;
int i = 0;
u8 serv_type;
- int num_req_rings = bundle->num_of_rings / 2;
- int num_rings_per_srv = num_req_rings / ADF_CFG_NUM_SERVICES;
+ int num_rings_per_srv = 0;
+ struct adf_hw_device_data *hw_data = accel_dev->hw_device;
u16 ring_to_svc_map = GET_HW_DATA(accel_dev)->ring_to_svc_map;
/* init the bundle with instance information */
- for (ring_pair_index = 0; ring_pair_index < ADF_CFG_NUM_SERVICES;
+ for (ring_pair_index = 0; ring_pair_index < bundle->max_cfg_svc_num;
ring_pair_index++) {
- serv_type = GET_SRV_TYPE(ring_to_svc_map, ring_pair_index);
+ adf_get_ring_svc_map_data(hw_data,
+ bundle->number,
+ ring_pair_index,
+ &serv_type,
+ &ring_index,
+ &num_rings_per_srv);
+
for (i = 0; i < num_rings_per_srv; i++) {
cfg_instance = malloc(sizeof(*cfg_instance),
M_QAT,
@@ -219,8 +226,9 @@
{
int i = 0;
+ bundle->number = bank_num;
/* init ring to service mapping for this bundle */
- adf_cfg_init_ring2serv_mapping(accel_dev, bundle);
+ adf_cfg_init_ring2serv_mapping(accel_dev, bundle, device);
/* init the bundle with instance information */
adf_cfg_init_and_insert_inst(bundle, device, bank_num, accel_dev);
@@ -229,7 +237,6 @@
bundle->type = FREE;
bundle->polling_mode = -1;
bundle->section_index = 0;
- bundle->number = bank_num;
bundle->sections = malloc(sizeof(char *) * bundle->max_section,
M_QAT,
@@ -262,18 +269,25 @@
}
static void
-adf_cfg_assign_serv_to_rings(struct adf_cfg_bundle *bundle, u16 ring_to_svc_map)
+adf_cfg_assign_serv_to_rings(struct adf_hw_device_data *hw_data,
+ struct adf_cfg_bundle *bundle,
+ struct adf_cfg_device *device)
{
int ring_pair_index = 0;
int ring_index = 0;
u8 serv_type = 0;
int num_req_rings = bundle->num_of_rings / 2;
- int num_rings_per_srv = num_req_rings / ADF_CFG_NUM_SERVICES;
+ int num_rings_per_srv = 0;
- for (ring_pair_index = 0; ring_pair_index < ADF_CFG_NUM_SERVICES;
+ for (ring_pair_index = 0; ring_pair_index < bundle->max_cfg_svc_num;
ring_pair_index++) {
- serv_type = GET_SRV_TYPE(ring_to_svc_map, ring_pair_index);
- ring_index = num_rings_per_srv * ring_pair_index;
+ adf_get_ring_svc_map_data(hw_data,
+ bundle->number,
+ ring_pair_index,
+ &serv_type,
+ &ring_index,
+ &num_rings_per_srv);
+
switch (serv_type) {
case CRYPTO:
ASSIGN_SERV_TO_RINGS(bundle,
@@ -283,7 +297,7 @@
num_rings_per_srv);
ring_pair_index++;
ring_index = num_rings_per_srv * ring_pair_index;
- if (ring_pair_index == ADF_CFG_NUM_SERVICES)
+ if (ring_pair_index == bundle->max_cfg_svc_num)
break;
ASSIGN_SERV_TO_RINGS(bundle,
ring_index,
@@ -324,7 +338,7 @@
/* unknown service type */
pr_err("Unknown service type %d, mask 0x%x.\n",
serv_type,
- ring_to_svc_map);
+ hw_data->ring_to_svc_map);
}
}
@@ -333,13 +347,18 @@
void
adf_cfg_init_ring2serv_mapping(struct adf_accel_dev *accel_dev,
- struct adf_cfg_bundle *bundle)
+ struct adf_cfg_bundle *bundle,
+ struct adf_cfg_device *device)
{
struct adf_hw_device_data *hw_data = accel_dev->hw_device;
struct adf_cfg_ring *ring_in_bundle;
int ring_num = 0;
bundle->num_of_rings = hw_data->num_rings_per_bank;
+ if (hw_data->num_rings_per_bank >= (2 * ADF_CFG_NUM_SERVICES))
+ bundle->max_cfg_svc_num = ADF_CFG_NUM_SERVICES;
+ else
+ bundle->max_cfg_svc_num = 1;
bundle->rings =
malloc(bundle->num_of_rings * sizeof(struct adf_cfg_ring *),
@@ -356,7 +375,7 @@
bundle->rings[ring_num] = ring_in_bundle;
}
- adf_cfg_assign_serv_to_rings(bundle, hw_data->ring_to_svc_map);
+ adf_cfg_assign_serv_to_rings(hw_data, bundle, device);
return;
}
Index: sys/dev/qat/qat_common/adf_cfg_device.c
===================================================================
--- sys/dev/qat/qat_common/adf_cfg_device.c
+++ sys/dev/qat/qat_common/adf_cfg_device.c
@@ -249,7 +249,6 @@
int i = 0;
int ret = EFAULT;
struct adf_cfg_instance *free_inst = NULL;
- struct adf_cfg_bundle *first_free_bundle = NULL;
enum adf_cfg_bundle_type free_bundle_type;
int first_user_bundle = 0;
@@ -304,29 +303,25 @@
return ret;
- } else if (!first_free_bundle &&
- adf_cfg_is_free(device->bundles[i])) {
- first_free_bundle = device->bundles[i];
}
}
+ for (i = 0; i < device->bundle_num; i++) {
+ if (adf_cfg_is_free(device->bundles[i])) {
+ free_inst = adf_cfg_get_free_instance(
+ device,
+ device->bundles[i],
+ inst,
+ process_name);
+ if (!free_inst)
+ continue;
- if (first_free_bundle) {
- free_inst = adf_cfg_get_free_instance(device,
- first_free_bundle,
- inst,
- process_name);
-
- if (!free_inst)
+ ret = adf_cfg_get_ring_pairs_from_bundle(
+ device->bundles[i],
+ inst,
+ process_name,
+ free_inst);
return ret;
-
- ret = adf_cfg_get_ring_pairs_from_bundle(
- first_free_bundle, inst, process_name, free_inst);
-
- if (free_bundle_type == KERNEL) {
- device->max_kernel_bundle_nr =
- first_free_bundle->number;
}
- return ret;
}
}
pr_err("Don't have enough rings for instance %s in process %s\n",
Index: sys/dev/qat/qat_common/adf_cfg_section.c
===================================================================
--- sys/dev/qat/qat_common/adf_cfg_section.c
+++ sys/dev/qat/qat_common/adf_cfg_section.c
@@ -286,10 +286,16 @@
key = malloc(ADF_CFG_MAX_KEY_LEN_IN_BYTES, M_QAT, M_WAITOK | M_ZERO);
- snprintf(key,
- ADF_CFG_MAX_KEY_LEN_IN_BYTES,
- ADF_CY_BANK_NUM_FORMAT,
- inst_index);
+ if (adf_cy_inst_cross_banks(accel_dev))
+ snprintf(key,
+ ADF_CFG_MAX_KEY_LEN_IN_BYTES,
+ ADF_CY_ASYM_BANK_NUM_FORMAT,
+ inst_index);
+ else
+ snprintf(key,
+ ADF_CFG_MAX_KEY_LEN_IN_BYTES,
+ ADF_CY_BANK_NUM_FORMAT,
+ inst_index);
bank_number = asym_inst->bundle;
adf_cfg_add_key_value_param(
accel_dev, derived_sec, key, (void *)&bank_number, ADF_DEC);
@@ -337,10 +343,17 @@
key = malloc(ADF_CFG_MAX_KEY_LEN_IN_BYTES, M_QAT, M_WAITOK | M_ZERO);
- snprintf(key,
- ADF_CFG_MAX_KEY_LEN_IN_BYTES,
- ADF_CY_BANK_NUM_FORMAT,
- inst_index);
+ if (adf_cy_inst_cross_banks(accel_dev))
+ snprintf(key,
+ ADF_CFG_MAX_KEY_LEN_IN_BYTES,
+ ADF_CY_SYM_BANK_NUM_FORMAT,
+ inst_index);
+ else
+ snprintf(key,
+ ADF_CFG_MAX_KEY_LEN_IN_BYTES,
+ ADF_CY_BANK_NUM_FORMAT,
+ inst_index);
+
bank_number = sym_inst->bundle;
adf_cfg_add_key_value_param(
accel_dev, derived_sec, key, (void *)&bank_number, ADF_DEC);
Index: sys/dev/qat/qat_common/adf_freebsd_admin.c
===================================================================
--- sys/dev/qat/qat_common/adf_freebsd_admin.c
+++ sys/dev/qat/qat_common/adf_freebsd_admin.c
@@ -27,10 +27,7 @@
#define ADF_CONST_TABLE_VERSION (1)
/* Admin Messages Registers */
-#define ADF_DH895XCC_ADMINMSGUR_OFFSET (0x3A000 + 0x574)
-#define ADF_DH895XCC_ADMINMSGLR_OFFSET (0x3A000 + 0x578)
-#define ADF_DH895XCC_MAILBOX_BASE_OFFSET 0x20970
-#define ADF_DH895XCC_MAILBOX_STRIDE 0x1000
+#define ADF_MAILBOX_STRIDE 0x1000
#define ADF_ADMINMSG_LEN 32
#define FREEBSD_ALLIGNMENT_SIZE 64
#define ADF_INIT_CONFIG_SIZE 1024
@@ -146,7 +143,7 @@
hw_data->get_admin_info(&admin_csrs_info);
int offset = ae * ADF_ADMINMSG_LEN * 2;
int mb_offset =
- ae * ADF_DH895XCC_MAILBOX_STRIDE + admin_csrs_info.mailbox_offset;
+ ae * ADF_MAILBOX_STRIDE + admin_csrs_info.mailbox_offset;
int times, received;
struct icp_qat_fw_init_admin_req *request = in;
@@ -294,7 +291,7 @@
struct icp_qat_fw_init_admin_req req;
struct icp_qat_fw_init_admin_resp resp;
struct adf_hw_device_data *hw_device = accel_dev->hw_device;
- u32 ae_mask = hw_device->ae_mask;
+ u32 ae_mask = hw_device->admin_ae_mask;
explicit_bzero(&req, sizeof(req));
req.cmd_id = ICP_QAT_FW_CONSTANTS_CFG;
Index: sys/dev/qat/qat_common/adf_freebsd_transport_debug.c
===================================================================
--- sys/dev/qat/qat_common/adf_freebsd_transport_debug.c
+++ sys/dev/qat/qat_common/adf_freebsd_transport_debug.c
@@ -20,6 +20,7 @@
{
struct adf_etr_ring_data *ring = arg1;
struct adf_etr_bank_data *bank = ring->bank;
+ struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(bank->accel_dev);
struct resource *csr = ring->bank->csr_addr;
struct sbuf sb;
int error, word;
@@ -29,13 +30,13 @@
{
int head, tail, empty;
- head = READ_CSR_RING_HEAD(csr,
- bank->bank_number,
- ring->ring_number);
- tail = READ_CSR_RING_TAIL(csr,
- bank->bank_number,
- ring->ring_number);
- empty = READ_CSR_E_STAT(csr, bank->bank_number);
+ head = csr_ops->read_csr_ring_head(csr,
+ bank->bank_number,
+ ring->ring_number);
+ tail = csr_ops->read_csr_ring_tail(csr,
+ bank->bank_number,
+ ring->ring_number);
+ empty = csr_ops->read_csr_e_stat(csr, bank->bank_number);
sbuf_cat(&sb, "\n------- Ring configuration -------\n");
sbuf_printf(&sb,
@@ -119,6 +120,7 @@
{
struct adf_etr_bank_data *bank;
struct adf_accel_dev *accel_dev = NULL;
+ struct adf_hw_csr_ops *csr_ops = NULL;
struct adf_hw_device_data *hw_data = NULL;
u8 num_rings_per_bank = 0;
struct sbuf sb;
@@ -127,6 +129,7 @@
sbuf_new_for_sysctl(&sb, NULL, 128, req);
bank = arg1;
accel_dev = bank->accel_dev;
+ csr_ops = GET_CSR_OPS(bank->accel_dev);
hw_data = accel_dev->hw_device;
num_rings_per_bank = hw_data->num_rings_per_bank;
sbuf_printf(&sb,
@@ -140,13 +143,13 @@
if (!(bank->ring_mask & 1 << ring_id))
continue;
- head = READ_CSR_RING_HEAD(csr,
- bank->bank_number,
- ring->ring_number);
- tail = READ_CSR_RING_TAIL(csr,
- bank->bank_number,
- ring->ring_number);
- empty = READ_CSR_E_STAT(csr, bank->bank_number);
+ head = csr_ops->read_csr_ring_head(csr,
+ bank->bank_number,
+ ring->ring_number);
+ tail = csr_ops->read_csr_ring_tail(csr,
+ bank->bank_number,
+ ring->ring_number);
+ empty = csr_ops->read_csr_e_stat(csr, bank->bank_number);
sbuf_printf(&sb,
"ring num %02d, head %04x, tail %04x, empty: %d\n",
Index: sys/dev/qat/qat_common/adf_gen2_hw_data.c
===================================================================
--- /dev/null
+++ sys/dev/qat/qat_common/adf_gen2_hw_data.c
@@ -0,0 +1,132 @@
+/* SPDX-License-Identifier: BSD-3-Clause */
+/* Copyright(c) 2021 Intel Corporation */
+/* $FreeBSD$ */
+#include "adf_gen2_hw_data.h"
+#include "icp_qat_hw.h"
+
+static u64
+build_csr_ring_base_addr(dma_addr_t addr, u32 size)
+{
+ return BUILD_RING_BASE_ADDR(addr, size);
+}
+
+static u32
+read_csr_ring_head(struct resource *csr_base_addr, u32 bank, u32 ring)
+{
+ return READ_CSR_RING_HEAD(csr_base_addr, bank, ring);
+}
+
+static void
+write_csr_ring_head(struct resource *csr_base_addr,
+ u32 bank,
+ u32 ring,
+ u32 value)
+{
+ WRITE_CSR_RING_HEAD(csr_base_addr, bank, ring, value);
+}
+
+static u32
+read_csr_ring_tail(struct resource *csr_base_addr, u32 bank, u32 ring)
+{
+ return READ_CSR_RING_TAIL(csr_base_addr, bank, ring);
+}
+
+static void
+write_csr_ring_tail(struct resource *csr_base_addr,
+ u32 bank,
+ u32 ring,
+ u32 value)
+{
+ WRITE_CSR_RING_TAIL(csr_base_addr, bank, ring, value);
+}
+
+static u32
+read_csr_e_stat(struct resource *csr_base_addr, u32 bank)
+{
+ return READ_CSR_E_STAT(csr_base_addr, bank);
+}
+
+static void
+write_csr_ring_config(struct resource *csr_base_addr,
+ u32 bank,
+ u32 ring,
+ u32 value)
+{
+ WRITE_CSR_RING_CONFIG(csr_base_addr, bank, ring, value);
+}
+
+static void
+write_csr_ring_base(struct resource *csr_base_addr,
+ u32 bank,
+ u32 ring,
+ dma_addr_t addr)
+{
+ WRITE_CSR_RING_BASE(csr_base_addr, bank, ring, addr);
+}
+
+static void
+write_csr_int_flag(struct resource *csr_base_addr, u32 bank, u32 value)
+{
+ WRITE_CSR_INT_FLAG(csr_base_addr, bank, value);
+}
+
+static void
+write_csr_int_srcsel(struct resource *csr_base_addr, u32 bank)
+{
+ WRITE_CSR_INT_SRCSEL(csr_base_addr, bank);
+}
+
+static void
+write_csr_int_col_en(struct resource *csr_base_addr, u32 bank, u32 value)
+{
+ WRITE_CSR_INT_COL_EN(csr_base_addr, bank, value);
+}
+
+static void
+write_csr_int_col_ctl(struct resource *csr_base_addr, u32 bank, u32 value)
+{
+ WRITE_CSR_INT_COL_CTL(csr_base_addr, bank, value);
+}
+
+static void
+write_csr_int_flag_and_col(struct resource *csr_base_addr, u32 bank, u32 value)
+{
+ WRITE_CSR_INT_FLAG_AND_COL(csr_base_addr, bank, value);
+}
+
+static u32
+read_csr_ring_srv_arb_en(struct resource *csr_base_addr, u32 bank)
+{
+ return READ_CSR_RING_SRV_ARB_EN(csr_base_addr, bank);
+}
+
+static void
+write_csr_ring_srv_arb_en(struct resource *csr_base_addr, u32 bank, u32 value)
+{
+ WRITE_CSR_RING_SRV_ARB_EN(csr_base_addr, bank, value);
+}
+
+void
+adf_gen2_init_hw_csr_info(struct adf_hw_csr_info *csr_info)
+{
+ struct adf_hw_csr_ops *csr_ops = &csr_info->csr_ops;
+
+ csr_info->arb_enable_mask = 0xFF;
+
+ csr_ops->build_csr_ring_base_addr = build_csr_ring_base_addr;
+ csr_ops->read_csr_ring_head = read_csr_ring_head;
+ csr_ops->write_csr_ring_head = write_csr_ring_head;
+ csr_ops->read_csr_ring_tail = read_csr_ring_tail;
+ csr_ops->write_csr_ring_tail = write_csr_ring_tail;
+ csr_ops->read_csr_e_stat = read_csr_e_stat;
+ csr_ops->write_csr_ring_config = write_csr_ring_config;
+ csr_ops->write_csr_ring_base = write_csr_ring_base;
+ csr_ops->write_csr_int_flag = write_csr_int_flag;
+ csr_ops->write_csr_int_srcsel = write_csr_int_srcsel;
+ csr_ops->write_csr_int_col_en = write_csr_int_col_en;
+ csr_ops->write_csr_int_col_ctl = write_csr_int_col_ctl;
+ csr_ops->write_csr_int_flag_and_col = write_csr_int_flag_and_col;
+ csr_ops->read_csr_ring_srv_arb_en = read_csr_ring_srv_arb_en;
+ csr_ops->write_csr_ring_srv_arb_en = write_csr_ring_srv_arb_en;
+}
+EXPORT_SYMBOL_GPL(adf_gen2_init_hw_csr_info);
Index: sys/dev/qat/qat_common/adf_gen4_hw_data.c
===================================================================
--- /dev/null
+++ sys/dev/qat/qat_common/adf_gen4_hw_data.c
@@ -0,0 +1,176 @@
+/* SPDX-License-Identifier: BSD-3-Clause */
+/* Copyright(c) 2021 Intel Corporation */
+/* $FreeBSD$ */
+#include "adf_accel_devices.h"
+#include "adf_gen4_hw_data.h"
+
+static u64
+build_csr_ring_base_addr(bus_addr_t addr, u32 size)
+{
+ return BUILD_RING_BASE_ADDR(addr, size);
+}
+
+static u32
+read_csr_ring_head(struct resource *csr_base_addr, u32 bank, u32 ring)
+{
+ return READ_CSR_RING_HEAD(csr_base_addr, bank, ring);
+}
+
+static void
+write_csr_ring_head(struct resource *csr_base_addr,
+ u32 bank,
+ u32 ring,
+ u32 value)
+{
+ WRITE_CSR_RING_HEAD(csr_base_addr, bank, ring, value);
+}
+
+static u32
+read_csr_ring_tail(struct resource *csr_base_addr, u32 bank, u32 ring)
+{
+ return READ_CSR_RING_TAIL(csr_base_addr, bank, ring);
+}
+
+static void
+write_csr_ring_tail(struct resource *csr_base_addr,
+ u32 bank,
+ u32 ring,
+ u32 value)
+{
+ WRITE_CSR_RING_TAIL(csr_base_addr, bank, ring, value);
+}
+
+static u32
+read_csr_e_stat(struct resource *csr_base_addr, u32 bank)
+{
+ return READ_CSR_E_STAT(csr_base_addr, bank);
+}
+
+static void
+write_csr_ring_config(struct resource *csr_base_addr,
+ u32 bank,
+ u32 ring,
+ u32 value)
+{
+ WRITE_CSR_RING_CONFIG(csr_base_addr, bank, ring, value);
+}
+
+static void
+write_csr_ring_base(struct resource *csr_base_addr,
+ u32 bank,
+ u32 ring,
+ bus_addr_t addr)
+{
+ WRITE_CSR_RING_BASE(csr_base_addr, bank, ring, addr);
+}
+
+static void
+write_csr_int_flag(struct resource *csr_base_addr, u32 bank, u32 value)
+{
+ WRITE_CSR_INT_FLAG(csr_base_addr, bank, value);
+}
+
+static void
+write_csr_int_srcsel(struct resource *csr_base_addr, u32 bank)
+{
+ WRITE_CSR_INT_SRCSEL(csr_base_addr, bank);
+}
+
+static void
+write_csr_int_col_en(struct resource *csr_base_addr, u32 bank, u32 value)
+{
+ WRITE_CSR_INT_COL_EN(csr_base_addr, bank, value);
+}
+
+static void
+write_csr_int_col_ctl(struct resource *csr_base_addr, u32 bank, u32 value)
+{
+ WRITE_CSR_INT_COL_CTL(csr_base_addr, bank, value);
+}
+
+static void
+write_csr_int_flag_and_col(struct resource *csr_base_addr, u32 bank, u32 value)
+{
+ WRITE_CSR_INT_FLAG_AND_COL(csr_base_addr, bank, value);
+}
+
+static u32
+read_csr_ring_srv_arb_en(struct resource *csr_base_addr, u32 bank)
+{
+ return READ_CSR_RING_SRV_ARB_EN(csr_base_addr, bank);
+}
+
+static void
+write_csr_ring_srv_arb_en(struct resource *csr_base_addr, u32 bank, u32 value)
+{
+ WRITE_CSR_RING_SRV_ARB_EN(csr_base_addr, bank, value);
+}
+
+void
+adf_gen4_init_hw_csr_info(struct adf_hw_csr_info *csr_info)
+{
+ struct adf_hw_csr_ops *csr_ops = &csr_info->csr_ops;
+
+ csr_info->arb_enable_mask = 0x1;
+
+ csr_ops->build_csr_ring_base_addr = build_csr_ring_base_addr;
+ csr_ops->read_csr_ring_head = read_csr_ring_head;
+ csr_ops->write_csr_ring_head = write_csr_ring_head;
+ csr_ops->read_csr_ring_tail = read_csr_ring_tail;
+ csr_ops->write_csr_ring_tail = write_csr_ring_tail;
+ csr_ops->read_csr_e_stat = read_csr_e_stat;
+ csr_ops->write_csr_ring_config = write_csr_ring_config;
+ csr_ops->write_csr_ring_base = write_csr_ring_base;
+ csr_ops->write_csr_int_flag = write_csr_int_flag;
+ csr_ops->write_csr_int_srcsel = write_csr_int_srcsel;
+ csr_ops->write_csr_int_col_en = write_csr_int_col_en;
+ csr_ops->write_csr_int_col_ctl = write_csr_int_col_ctl;
+ csr_ops->write_csr_int_flag_and_col = write_csr_int_flag_and_col;
+ csr_ops->read_csr_ring_srv_arb_en = read_csr_ring_srv_arb_en;
+ csr_ops->write_csr_ring_srv_arb_en = write_csr_ring_srv_arb_en;
+}
+EXPORT_SYMBOL_GPL(adf_gen4_init_hw_csr_info);
+
+static inline void
+adf_gen4_unpack_ssm_wdtimer(u64 value, u32 *upper, u32 *lower)
+{
+ *lower = lower_32_bits(value);
+ *upper = upper_32_bits(value);
+}
+
+int
+adf_gen4_set_ssm_wdtimer(struct adf_accel_dev *accel_dev)
+{
+ struct adf_hw_device_data *hw_data = accel_dev->hw_device;
+ u64 timer_val_pke = ADF_SSM_WDT_PKE_DEFAULT_VALUE;
+ u64 timer_val = ADF_SSM_WDT_DEFAULT_VALUE;
+ u32 ssm_wdt_pke_high = 0;
+ u32 ssm_wdt_pke_low = 0;
+ u32 ssm_wdt_high = 0;
+ u32 ssm_wdt_low = 0;
+ struct resource *pmisc_addr;
+ struct adf_bar *pmisc;
+ int pmisc_id;
+
+ pmisc_id = hw_data->get_misc_bar_id(hw_data);
+ pmisc = &GET_BARS(accel_dev)[pmisc_id];
+ pmisc_addr = pmisc->virt_addr;
+
+ /* Convert 64bit WDT timer value into 32bit values for
+ * mmio write to 32bit CSRs.
+ */
+ adf_gen4_unpack_ssm_wdtimer(timer_val, &ssm_wdt_high, &ssm_wdt_low);
+ adf_gen4_unpack_ssm_wdtimer(timer_val_pke,
+ &ssm_wdt_pke_high,
+ &ssm_wdt_pke_low);
+
+ /* Enable WDT for sym and dc */
+ ADF_CSR_WR(pmisc_addr, ADF_SSMWDTL_OFFSET, ssm_wdt_low);
+ ADF_CSR_WR(pmisc_addr, ADF_SSMWDTH_OFFSET, ssm_wdt_high);
+ /* Enable WDT for pke */
+ ADF_CSR_WR(pmisc_addr, ADF_SSMWDTPKEL_OFFSET, ssm_wdt_pke_low);
+ ADF_CSR_WR(pmisc_addr, ADF_SSMWDTPKEH_OFFSET, ssm_wdt_pke_high);
+
+ return 0;
+}
+EXPORT_SYMBOL_GPL(adf_gen4_set_ssm_wdtimer);
Index: sys/dev/qat/qat_common/adf_heartbeat.c
===================================================================
--- sys/dev/qat/qat_common/adf_heartbeat.c
+++ sys/dev/qat/qat_common/adf_heartbeat.c
@@ -68,10 +68,14 @@
unsigned int timer_val = ADF_CFG_HB_DEFAULT_VALUE;
u32 clk_per_sec = 0;
- if (!hw_data->get_ae_clock)
+ /* HB clock may be different than AE clock */
+ if (hw_data->get_hb_clock) {
+ clk_per_sec = (u32)hw_data->get_hb_clock(hw_data);
+ } else if (hw_data->get_ae_clock) {
+ clk_per_sec = (u32)hw_data->get_ae_clock(hw_data);
+ } else {
return EINVAL;
-
- clk_per_sec = (u32)hw_data->get_ae_clock(hw_data);
+ }
/* Get Heartbeat Timer value from the configuration */
if (!adf_cfg_get_param_value(accel_dev,
@@ -99,24 +103,19 @@
return 0;
}
-struct adf_hb_count {
- u16 ae_thread[ADF_NUM_HB_CNT_PER_AE];
-};
-
int
adf_get_heartbeat_status(struct adf_accel_dev *accel_dev)
{
+ struct icp_qat_fw_init_admin_hb_cnt *live_s, *last_s, *curr_s;
struct adf_hw_device_data *hw_device = accel_dev->hw_device;
- struct icp_qat_fw_init_admin_hb_stats *live_s =
- (struct icp_qat_fw_init_admin_hb_stats *)
- accel_dev->admin->virt_hb_addr;
const size_t max_aes = hw_device->get_num_aes(hw_device);
+ const size_t hb_ctrs = hw_device->heartbeat_ctr_num;
const size_t stats_size =
- max_aes * sizeof(struct icp_qat_fw_init_admin_hb_stats);
+ max_aes * hb_ctrs * sizeof(struct icp_qat_fw_init_admin_hb_cnt);
int ret = 0;
size_t ae, thr;
+ u16 *count_s;
unsigned long ae_mask = 0;
- int num_threads_per_ae = ADF_NUM_HB_CNT_PER_AE;
/*
* Memory layout of Heartbeat
@@ -127,13 +126,19 @@
* \_______________/\_______________/\________/
* ^ ^ ^
* | | |
- * | | max_aes * sizeof(adf_hb_count)
- * | max_aes * sizeof(icp_qat_fw_init_admin_hb_stats)
- * max_aes * sizeof(icp_qat_fw_init_admin_hb_stats)
+ * | | max_aes * hb_ctrs *
+ * | | sizeof(u16)
+ * | |
+ * | max_aes * hb_ctrs *
+ * | sizeof(icp_qat_fw_init_admin_hb_cnt)
+ * |
+ * max_aes * hb_ctrs *
+ * sizeof(icp_qat_fw_init_admin_hb_cnt)
*/
- struct icp_qat_fw_init_admin_hb_stats *curr_s;
- struct icp_qat_fw_init_admin_hb_stats *last_s = live_s + max_aes;
- struct adf_hb_count *count = (struct adf_hb_count *)(last_s + max_aes);
+ live_s = (struct icp_qat_fw_init_admin_hb_cnt *)
+ accel_dev->admin->virt_hb_addr;
+ last_s = live_s + (max_aes * hb_ctrs);
+ count_s = (u16 *)(last_s + (max_aes * hb_ctrs));
curr_s = malloc(stats_size, M_QAT, M_WAITOK | M_ZERO);
@@ -142,23 +147,25 @@
for_each_set_bit(ae, &ae_mask, max_aes)
{
- for (thr = 0; thr < num_threads_per_ae; ++thr) {
- struct icp_qat_fw_init_admin_hb_cnt *curr =
- &curr_s[ae].stats[thr];
- struct icp_qat_fw_init_admin_hb_cnt *prev =
- &last_s[ae].stats[thr];
- u16 req = curr->req_heartbeat_cnt;
- u16 resp = curr->resp_heartbeat_cnt;
- u16 last = prev->resp_heartbeat_cnt;
+ struct icp_qat_fw_init_admin_hb_cnt *curr =
+ curr_s + ae * hb_ctrs;
+ struct icp_qat_fw_init_admin_hb_cnt *prev =
+ last_s + ae * hb_ctrs;
+ u16 *count = count_s + ae * hb_ctrs;
+
+ for (thr = 0; thr < hb_ctrs; ++thr) {
+ u16 req = curr[thr].req_heartbeat_cnt;
+ u16 resp = curr[thr].resp_heartbeat_cnt;
+ u16 last = prev[thr].resp_heartbeat_cnt;
if ((thr == ADF_AE_ADMIN_THREAD || req != resp) &&
resp == last) {
- u16 retry = ++count[ae].ae_thread[thr];
+ u16 retry = ++count[thr];
if (retry >= ADF_CFG_HB_COUNT_THRESHOLD)
ret = EIO;
} else {
- count[ae].ae_thread[thr] = 0;
+ count[thr] = 0;
}
}
}
Index: sys/dev/qat/qat_common/adf_hw_arbiter.c
===================================================================
--- sys/dev/qat/qat_common/adf_hw_arbiter.c
+++ sys/dev/qat/qat_common/adf_hw_arbiter.c
@@ -97,14 +97,30 @@
void
adf_update_ring_arb(struct adf_etr_ring_data *ring)
{
- WRITE_CSR_ARB_RINGSRVARBEN(ring->bank->csr_addr,
- ring->bank->bank_number,
- ring->bank->ring_mask & 0xFF);
+ int shift;
+ u32 arben, arben_tx, arben_rx, arb_mask;
+ struct adf_accel_dev *accel_dev = ring->bank->accel_dev;
+ struct adf_hw_csr_info *csr_info = &accel_dev->hw_device->csr_info;
+ struct adf_hw_csr_ops *csr_ops = &csr_info->csr_ops;
+
+ arb_mask = csr_info->arb_enable_mask;
+ shift = hweight32(arb_mask);
+
+ arben_tx = ring->bank->ring_mask & arb_mask;
+ arben_rx = (ring->bank->ring_mask >> shift) & arb_mask;
+ arben = arben_tx & arben_rx;
+ csr_ops->write_csr_ring_srv_arb_en(ring->bank->csr_addr,
+ ring->bank->bank_number,
+ arben);
}
void
-adf_enable_ring_arb(void *csr_addr, unsigned int bank_nr, unsigned int mask)
+adf_enable_ring_arb(struct adf_accel_dev *accel_dev,
+ void *csr_addr,
+ unsigned int bank_nr,
+ unsigned int mask)
{
+ struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev);
struct resource *csr = csr_addr;
u32 arbenable;
@@ -112,16 +128,20 @@
return;
mutex_lock(&csr_arb_lock);
- arbenable = READ_CSR_ARB_RINGSRVARBEN(csr, bank_nr);
+ arbenable = csr_ops->read_csr_ring_srv_arb_en(csr, bank_nr);
arbenable |= mask & 0xFF;
- WRITE_CSR_ARB_RINGSRVARBEN(csr, bank_nr, arbenable);
+ csr_ops->write_csr_ring_srv_arb_en(csr, bank_nr, arbenable);
mutex_unlock(&csr_arb_lock);
}
void
-adf_disable_ring_arb(void *csr_addr, unsigned int bank_nr, unsigned int mask)
+adf_disable_ring_arb(struct adf_accel_dev *accel_dev,
+ void *csr_addr,
+ unsigned int bank_nr,
+ unsigned int mask)
{
+ struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev);
struct resource *csr = csr_addr;
u32 arbenable;
@@ -129,15 +149,16 @@
return;
mutex_lock(&csr_arb_lock);
- arbenable = READ_CSR_ARB_RINGSRVARBEN(csr, bank_nr);
+ arbenable = csr_ops->read_csr_ring_srv_arb_en(csr, bank_nr);
arbenable &= ~mask & 0xFF;
- WRITE_CSR_ARB_RINGSRVARBEN(csr, bank_nr, arbenable);
+ csr_ops->write_csr_ring_srv_arb_en(csr, bank_nr, arbenable);
mutex_unlock(&csr_arb_lock);
}
void
adf_exit_arb(struct adf_accel_dev *accel_dev)
{
+ struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev);
struct adf_hw_device_data *hw_data = accel_dev->hw_device;
struct arb_info info;
struct resource *csr;
@@ -166,12 +187,13 @@
/* Disable arbitration on all rings */
for (i = 0; i < GET_MAX_BANKS(accel_dev); i++)
- WRITE_CSR_ARB_RINGSRVARBEN(csr, i, 0);
+ csr_ops->write_csr_ring_srv_arb_en(csr, i, 0);
}
void
adf_disable_arb(struct adf_accel_dev *accel_dev)
{
+ struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev);
struct resource *csr;
unsigned int i;
@@ -182,5 +204,5 @@
/* Disable arbitration on all rings */
for (i = 0; i < GET_MAX_BANKS(accel_dev); i++)
- WRITE_CSR_ARB_RINGSRVARBEN(csr, i, 0);
+ csr_ops->write_csr_ring_srv_arb_en(csr, i, 0);
}
Index: sys/dev/qat/qat_common/adf_init.c
===================================================================
--- sys/dev/qat/qat_common/adf_init.c
+++ sys/dev/qat/qat_common/adf_init.c
@@ -213,7 +213,7 @@
unsigned int mask;
u32 clk_per_sec = hw_data->get_clock_speed(hw_data);
u32 timer_val = ADF_WDT_TIMER_SYM_COMP_MS * (clk_per_sec / 1000);
- u32 timer_val_pke = ADF_SSM_WDT_PKE_DEFAULT_VALUE;
+ u32 timer_val_pke = ADF_GEN2_SSM_WDT_PKE_DEFAULT_VALUE;
char timer_str[ADF_CFG_MAX_VAL_LEN_IN_BYTES] = { 0 };
/* Get Watch Dog Timer for CySym+Comp from the configuration */
@@ -289,6 +289,12 @@
return EFAULT;
}
+ if (hw_data->init_device && hw_data->init_device(accel_dev)) {
+ device_printf(GET_DEV(accel_dev),
+ "Failed to initialize device\n");
+ return EFAULT;
+ }
+
if (hw_data->init_accel_units && hw_data->init_accel_units(accel_dev)) {
device_printf(GET_DEV(accel_dev),
"Failed initialize accel_units\n");
@@ -343,7 +349,8 @@
hw_data->enable_error_correction(accel_dev);
- if (hw_data->enable_vf2pf_comms(accel_dev)) {
+ if (hw_data->enable_vf2pf_comms &&
+ hw_data->enable_vf2pf_comms(accel_dev)) {
device_printf(GET_DEV(accel_dev),
"QAT: Failed to enable vf2pf comms\n");
return EFAULT;
Index: sys/dev/qat/qat_common/adf_isr.c
===================================================================
--- sys/dev/qat/qat_common/adf_isr.c
+++ sys/dev/qat/qat_common/adf_isr.c
@@ -38,6 +38,9 @@
int num_vectors = 0;
u_int *vectors;
+ if (hw_data->set_msix_rttable)
+ hw_data->set_msix_rttable(accel_dev);
+
/* If SR-IOV is disabled, add entries for each bank */
if (!accel_dev->u1.pf.vf_info) {
msix_num_entries += hw_data->num_banks;
@@ -90,8 +93,11 @@
{
struct adf_etr_bank_data *bank = bank_ptr;
struct adf_etr_data *priv_data = bank->accel_dev->transport;
+ struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(bank->accel_dev);
- WRITE_CSR_INT_FLAG_AND_COL(bank->csr_addr, bank->bank_number, 0);
+ csr_ops->write_csr_int_flag_and_col(bank->csr_addr,
+ bank->bank_number,
+ 0);
adf_response_handler((uintptr_t)&priv_data->banks[bank->bank_number]);
return;
}
Index: sys/dev/qat/qat_common/adf_transport.c
===================================================================
--- sys/dev/qat/qat_common/adf_transport.c
+++ sys/dev/qat/qat_common/adf_transport.c
@@ -73,27 +73,36 @@
static void
adf_enable_ring_irq(struct adf_etr_bank_data *bank, u32 ring)
{
+ struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(bank->accel_dev);
+
mtx_lock(&bank->lock);
bank->irq_mask |= (1 << ring);
mtx_unlock(&bank->lock);
- WRITE_CSR_INT_COL_EN(bank->csr_addr, bank->bank_number, bank->irq_mask);
- WRITE_CSR_INT_COL_CTL(bank->csr_addr,
- bank->bank_number,
- bank->irq_coalesc_timer);
+ csr_ops->write_csr_int_col_en(bank->csr_addr,
+ bank->bank_number,
+ bank->irq_mask);
+ csr_ops->write_csr_int_col_ctl(bank->csr_addr,
+ bank->bank_number,
+ bank->irq_coalesc_timer);
}
static void
adf_disable_ring_irq(struct adf_etr_bank_data *bank, u32 ring)
{
+ struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(bank->accel_dev);
+
mtx_lock(&bank->lock);
bank->irq_mask &= ~(1 << ring);
mtx_unlock(&bank->lock);
- WRITE_CSR_INT_COL_EN(bank->csr_addr, bank->bank_number, bank->irq_mask);
+ csr_ops->write_csr_int_col_en(bank->csr_addr,
+ bank->bank_number,
+ bank->irq_mask);
}
int
adf_send_message(struct adf_etr_ring_data *ring, u32 *msg)
{
+ struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(ring->bank->accel_dev);
u32 msg_size = 0;
if (atomic_add_return(1, ring->inflights) > ring->max_inflights) {
@@ -110,10 +119,10 @@
ring->tail = adf_modulo(ring->tail + msg_size,
ADF_RING_SIZE_MODULO(ring->ring_size));
- WRITE_CSR_RING_TAIL(ring->bank->csr_addr,
- ring->bank->bank_number,
- ring->ring_number,
- ring->tail);
+ csr_ops->write_csr_ring_tail(ring->bank->csr_addr,
+ ring->bank->bank_number,
+ ring->ring_number,
+ ring->tail);
ring->csr_tail_offset = ring->tail;
mtx_unlock(&ring->lock);
return 0;
@@ -122,6 +131,7 @@
int
adf_handle_response(struct adf_etr_ring_data *ring, u32 quota)
{
+ struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(ring->bank->accel_dev);
u32 msg_counter = 0;
u32 *msg = (u32 *)((uintptr_t)ring->base_addr + ring->head);
@@ -139,10 +149,10 @@
msg = (u32 *)((uintptr_t)ring->base_addr + ring->head);
}
if (msg_counter > 0)
- WRITE_CSR_RING_HEAD(ring->bank->csr_addr,
- ring->bank->bank_number,
- ring->ring_number,
- ring->head);
+ csr_ops->write_csr_ring_head(ring->bank->csr_addr,
+ ring->bank->bank_number,
+ ring->ring_number,
+ ring->head);
return msg_counter;
}
@@ -154,6 +164,7 @@
struct adf_etr_data *trans_data;
struct adf_etr_bank_data *bank;
struct adf_etr_ring_data *ring;
+ struct adf_hw_csr_ops *csr_ops;
u32 rings_not_empty;
u32 ring_num;
u32 resp_total = 0;
@@ -168,12 +179,14 @@
return EINVAL;
}
+ csr_ops = GET_CSR_OPS(accel_dev);
trans_data = accel_dev->transport;
bank = &trans_data->banks[bank_num];
mtx_lock(&bank->lock);
/* Read the ring status CSR to determine which rings are empty. */
- rings_not_empty = READ_CSR_E_STAT(bank->csr_addr, bank->bank_number);
+ rings_not_empty =
+ csr_ops->read_csr_e_stat(bank->csr_addr, bank->bank_number);
/* Complement to find which rings have data to be processed. */
rings_not_empty = (~rings_not_empty) & bank->ring_mask;
@@ -265,25 +278,27 @@
static void
adf_configure_tx_ring(struct adf_etr_ring_data *ring)
{
+ struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(ring->bank->accel_dev);
u32 ring_config = BUILD_RING_CONFIG(ring->ring_size);
- WRITE_CSR_RING_CONFIG(ring->bank->csr_addr,
- ring->bank->bank_number,
- ring->ring_number,
- ring_config);
+ csr_ops->write_csr_ring_config(ring->bank->csr_addr,
+ ring->bank->bank_number,
+ ring->ring_number,
+ ring_config);
}
static void
adf_configure_rx_ring(struct adf_etr_ring_data *ring)
{
+ struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(ring->bank->accel_dev);
u32 ring_config = BUILD_RESP_RING_CONFIG(ring->ring_size,
ADF_RING_NEAR_WATERMARK_512,
ADF_RING_NEAR_WATERMARK_0);
- WRITE_CSR_RING_CONFIG(ring->bank->csr_addr,
- ring->bank->bank_number,
- ring->ring_number,
- ring_config);
+ csr_ops->write_csr_ring_config(ring->bank->csr_addr,
+ ring->bank->bank_number,
+ ring->ring_number,
+ ring_config);
}
static int
@@ -292,6 +307,7 @@
struct adf_etr_bank_data *bank = ring->bank;
struct adf_accel_dev *accel_dev = bank->accel_dev;
struct adf_hw_device_data *hw_data = accel_dev->hw_device;
+ struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev);
u64 ring_base;
u32 ring_size_bytes = ADF_SIZE_TO_RING_SIZE_IN_BYTES(ring->ring_size);
@@ -323,11 +339,12 @@
else
adf_configure_rx_ring(ring);
- ring_base = BUILD_RING_BASE_ADDR(ring->dma_addr, ring->ring_size);
- WRITE_CSR_RING_BASE(ring->bank->csr_addr,
- ring->bank->bank_number,
- ring->ring_number,
- ring_base);
+ ring_base =
+ csr_ops->build_csr_ring_base_addr(ring->dma_addr, ring->ring_size);
+ csr_ops->write_csr_ring_base(ring->bank->csr_addr,
+ ring->bank->bank_number,
+ ring->ring_number,
+ ring_base);
mtx_init(&ring->lock, "adf bank", NULL, MTX_DEF);
return 0;
}
@@ -443,19 +460,20 @@
adf_remove_ring(struct adf_etr_ring_data *ring)
{
struct adf_etr_bank_data *bank = ring->bank;
+ struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(bank->accel_dev);
/* Disable interrupts for the given ring */
adf_disable_ring_irq(bank, ring->ring_number);
/* Clear PCI config space */
- WRITE_CSR_RING_CONFIG(bank->csr_addr,
- bank->bank_number,
- ring->ring_number,
- 0);
- WRITE_CSR_RING_BASE(bank->csr_addr,
- bank->bank_number,
- ring->ring_number,
- 0);
+ csr_ops->write_csr_ring_config(bank->csr_addr,
+ bank->bank_number,
+ ring->ring_number,
+ 0);
+ csr_ops->write_csr_ring_base(bank->csr_addr,
+ bank->bank_number,
+ ring->ring_number,
+ 0);
adf_ring_debugfs_rm(ring);
adf_unreserve_ring(bank, ring->ring_number);
/* Disable HW arbitration for the given ring */
@@ -468,10 +486,12 @@
{
struct adf_accel_dev *accel_dev = bank->accel_dev;
struct adf_hw_device_data *hw_data = accel_dev->hw_device;
+ struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev);
u8 num_rings_per_bank = hw_data->num_rings_per_bank;
u32 empty_rings, i;
- empty_rings = READ_CSR_E_STAT(bank->csr_addr, bank->bank_number);
+ empty_rings =
+ csr_ops->read_csr_e_stat(bank->csr_addr, bank->bank_number);
empty_rings = ~empty_rings & bank->irq_mask;
for (i = 0; i < num_rings_per_bank; ++i) {
@@ -484,12 +504,13 @@
adf_response_handler(uintptr_t bank_addr)
{
struct adf_etr_bank_data *bank = (void *)bank_addr;
+ struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(bank->accel_dev);
/* Handle all the responses and re-enable IRQs */
adf_ring_response_handler(bank);
- WRITE_CSR_INT_FLAG_AND_COL(bank->csr_addr,
- bank->bank_number,
- bank->irq_mask);
+ csr_ops->write_csr_int_flag_and_col(bank->csr_addr,
+ bank->bank_number,
+ bank->irq_mask);
}
static inline int
@@ -548,10 +569,12 @@
struct resource *csr_addr)
{
struct adf_hw_device_data *hw_data = accel_dev->hw_device;
+ struct adf_hw_csr_ops *csr_ops = &hw_data->csr_info.csr_ops;
struct adf_etr_ring_data *ring;
struct adf_etr_ring_data *tx_ring;
u32 i, coalesc_enabled = 0;
u8 num_rings_per_bank = hw_data->num_rings_per_bank;
+ u32 irq_mask = BIT(num_rings_per_bank) - 1;
u32 size = 0;
explicit_bzero(bank, sizeof(*bank));
@@ -580,8 +603,8 @@
bank->irq_coalesc_timer = ADF_COALESCING_MIN_TIME;
for (i = 0; i < num_rings_per_bank; i++) {
- WRITE_CSR_RING_CONFIG(csr_addr, bank_num, i, 0);
- WRITE_CSR_RING_BASE(csr_addr, bank_num, i, 0);
+ csr_ops->write_csr_ring_config(csr_addr, bank_num, i, 0);
+ csr_ops->write_csr_ring_base(csr_addr, bank_num, i, 0);
ring = &bank->rings[i];
if (hw_data->tx_rings_mask & (1 << i)) {
ring->inflights =
@@ -605,8 +628,8 @@
goto err;
}
- WRITE_CSR_INT_FLAG(csr_addr, bank_num, ADF_BANK_INT_FLAG_CLEAR_MASK);
- WRITE_CSR_INT_SRCSEL(csr_addr, bank_num);
+ csr_ops->write_csr_int_flag(csr_addr, bank_num, irq_mask);
+ csr_ops->write_csr_int_srcsel(csr_addr, bank_num);
return 0;
err:
for (i = 0; i < num_rings_per_bank; i++) {
Index: sys/dev/qat/qat_common/qat_hal.c
===================================================================
--- sys/dev/qat/qat_common/qat_hal.c
+++ sys/dev/qat/qat_common/qat_hal.c
@@ -75,6 +75,29 @@
0x0D81581C010ull, 0x0E000010000ull, 0x0E000010000ull,
};
+static const uint64_t inst_CPM2X[] = {
+ 0x0F0000C0000ull, 0x0D802C00011ull, 0x0F0000C0001ull, 0x0FC066C0001ull,
+ 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F000500300ull,
+ 0x0A0610C0000ull, 0x0BAC0000301ull, 0x0D802000101ull, 0x0A0580C0000ull,
+ 0x0A0581C0000ull, 0x0A0582C0000ull, 0x0A0583C0000ull, 0x0A0584C0000ull,
+ 0x0A0585C0000ull, 0x0A0586C0000ull, 0x0A0587C0000ull, 0x0A0588C0000ull,
+ 0x0A0589C0000ull, 0x0A058AC0000ull, 0x0A058BC0000ull, 0x0A058CC0000ull,
+ 0x0A058DC0000ull, 0x0A058EC0000ull, 0x0A058FC0000ull, 0x0A05C0C0000ull,
+ 0x0A05C1C0000ull, 0x0A05C2C0000ull, 0x0A05C3C0000ull, 0x0A05C4C0000ull,
+ 0x0A05C5C0000ull, 0x0A05C6C0000ull, 0x0A05C7C0000ull, 0x0A05C8C0000ull,
+ 0x0A05C9C0000ull, 0x0A05CAC0000ull, 0x0A05CBC0000ull, 0x0A05CCC0000ull,
+ 0x0A05CDC0000ull, 0x0A05CEC0000ull, 0x0A05CFC0000ull, 0x0A0400C0000ull,
+ 0x0B0400C0000ull, 0x0A0401C0000ull, 0x0B0401C0000ull, 0x0A0402C0000ull,
+ 0x0B0402C0000ull, 0x0A0403C0000ull, 0x0B0403C0000ull, 0x0A0404C0000ull,
+ 0x0B0404C0000ull, 0x0A0405C0000ull, 0x0B0405C0000ull, 0x0A0406C0000ull,
+ 0x0B0406C0000ull, 0x0A0407C0000ull, 0x0B0407C0000ull, 0x0A0408C0000ull,
+ 0x0B0408C0000ull, 0x0A0409C0000ull, 0x0B0409C0000ull, 0x0A040AC0000ull,
+ 0x0B040AC0000ull, 0x0A040BC0000ull, 0x0B040BC0000ull, 0x0A040CC0000ull,
+ 0x0B040CC0000ull, 0x0A040DC0000ull, 0x0B040DC0000ull, 0x0A040EC0000ull,
+ 0x0B040EC0000ull, 0x0A040FC0000ull, 0x0B040FC0000ull, 0x0D81341C010ull,
+ 0x0E000000001ull, 0x0E000010000ull,
+};
+
void
qat_hal_set_live_ctx(struct icp_qat_fw_loader_handle *handle,
unsigned char ae,
@@ -206,6 +229,11 @@
{
unsigned int csr, new_csr;
+ if (IS_QAT_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) {
+ pr_err("QAT: No next neigh for CPM2X\n");
+ return EINVAL;
+ }
+
qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &csr);
csr &= IGNORE_W1C_MASK;
@@ -339,6 +367,21 @@
return reg_addr;
}
+static u32
+qat_hal_get_ae_mask_gen4(struct icp_qat_fw_loader_handle *handle)
+{
+ u32 tg = 0, ae;
+ u32 valid_ae_mask = 0;
+
+ for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) {
+ if (handle->hal_handle->ae_mask & (1 << ae)) {
+ tg = ae / 4;
+ valid_ae_mask |= (1 << (tg * 2));
+ }
+ }
+ return valid_ae_mask;
+}
+
void
qat_hal_reset(struct icp_qat_fw_loader_handle *handle)
{
@@ -353,15 +396,26 @@
ae_reset_csr[1] = ICP_RESET_CPP1;
if (handle->hal_handle->ae_mask > 0xffff)
++cpp_num;
+ } else if (IS_QAT_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) {
+ ae_reset_csr[0] = ICP_RESET_CPP0;
} else {
ae_reset_csr[0] = ICP_RESET;
}
for (i = 0; i < cpp_num; i++) {
if (i == 0) {
- valid_ae_mask = handle->hal_handle->ae_mask & 0xFFFF;
- valid_slice_mask =
- handle->hal_handle->slice_mask & 0x3F;
+ if (IS_QAT_GEN4(
+ pci_get_device(GET_DEV(handle->accel_dev)))) {
+ valid_ae_mask =
+ qat_hal_get_ae_mask_gen4(handle);
+ valid_slice_mask =
+ handle->hal_handle->slice_mask;
+ } else {
+ valid_ae_mask =
+ handle->hal_handle->ae_mask & 0xFFFF;
+ valid_slice_mask =
+ handle->hal_handle->slice_mask & 0x3F;
+ }
} else {
valid_ae_mask =
(handle->hal_handle->ae_mask >> AES_PER_CPP) &
@@ -509,7 +563,7 @@
unsigned long ae_mask = handle->hal_handle->ae_mask;
misc_ctl_csr =
- (IS_QAT_GEN3(pci_get_device(GET_DEV(handle->accel_dev)))) ?
+ (IS_QAT_GEN3_OR_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) ?
MISC_CONTROL_C4XXX :
MISC_CONTROL;
/* stop the timestamp timers */
@@ -586,6 +640,9 @@
clk_csr[1] = ICP_GLOBAL_CLK_ENABLE_CPP1;
if (handle->hal_handle->ae_mask > 0xffff)
++cpp_num;
+ } else if (IS_QAT_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) {
+ ae_reset_csr[0] = ICP_RESET_CPP0;
+ clk_csr[0] = ICP_GLOBAL_CLK_ENABLE_CPP0;
} else {
ae_reset_csr[0] = ICP_RESET;
clk_csr[0] = ICP_GLOBAL_CLK_ENABLE;
@@ -593,9 +650,18 @@
for (i = 0; i < cpp_num; i++) {
if (i == 0) {
- valid_ae_mask = handle->hal_handle->ae_mask & 0xFFFF;
- valid_slice_mask =
- handle->hal_handle->slice_mask & 0x3F;
+ if (IS_QAT_GEN4(
+ pci_get_device(GET_DEV(handle->accel_dev)))) {
+ valid_ae_mask =
+ qat_hal_get_ae_mask_gen4(handle);
+ valid_slice_mask =
+ handle->hal_handle->slice_mask;
+ } else {
+ valid_ae_mask =
+ handle->hal_handle->ae_mask & 0xFFFF;
+ valid_slice_mask =
+ handle->hal_handle->slice_mask & 0x3F;
+ }
} else {
valid_ae_mask =
(handle->hal_handle->ae_mask >> AES_PER_CPP) &
@@ -714,7 +780,24 @@
const uint64_t *uword)
{
unsigned int ustore_addr;
- unsigned int i;
+ unsigned int i, ae_in_group;
+
+ if (IS_QAT_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) {
+ ae_in_group = ae / 4 * 4;
+
+ for (i = 0; i < AE_TG_NUM_CPM2X; i++) {
+ if (ae_in_group + i == ae)
+ continue;
+ if (ae_in_group + i >= handle->hal_handle->ae_max_num)
+ break;
+ if (qat_hal_check_ae_active(handle, ae_in_group + i)) {
+ pr_err(
+ "ae%d in T_group is active, cannot write to ustore!\n",
+ ae_in_group + i);
+ return;
+ }
+ }
+ }
qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS, &ustore_addr);
uaddr |= UA_ECS;
@@ -826,10 +909,25 @@
qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, csr_val);
qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &csr_val);
csr_val &= IGNORE_W1C_MASK;
- csr_val |= CE_NN_MODE;
+ if (!IS_QAT_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) {
+ csr_val |= CE_NN_MODE;
+ }
qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, csr_val);
- qat_hal_wr_uwords(
- handle, ae, 0, ARRAY_SIZE(inst), (const uint64_t *)inst);
+
+ if (IS_QAT_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) {
+ if (ae % 4 == 0)
+ qat_hal_wr_uwords(handle,
+ ae,
+ 0,
+ ARRAY_SIZE(inst_CPM2X),
+ (const uint64_t *)inst_CPM2X);
+ } else {
+ qat_hal_wr_uwords(handle,
+ ae,
+ 0,
+ ARRAY_SIZE(inst),
+ (const uint64_t *)inst);
+ }
qat_hal_wr_indr_csr(handle,
ae,
ctx_mask,
@@ -971,6 +1069,7 @@
malloc(sizeof(*handle->hal_handle), M_QAT, M_WAITOK | M_ZERO);
handle->hal_handle->revision_id = accel_dev->accel_pci_dev.revid;
handle->hal_handle->ae_mask = hw_data->ae_mask;
+ handle->hal_handle->admin_ae_mask = hw_data->admin_ae_mask;
handle->hal_handle->slice_mask = hw_data->accel_mask;
handle->cfg_ae_mask = 0xFFFFFFFF;
/* create AE objects */
@@ -1038,17 +1137,23 @@
free(handle, M_QAT);
}
-void
-qat_hal_start(struct icp_qat_fw_loader_handle *handle,
- unsigned char ae,
- unsigned int ctx_mask)
+int
+qat_hal_start(struct icp_qat_fw_loader_handle *handle)
{
+ unsigned char ae = 0;
int retry = 0;
unsigned int fcu_sts = 0;
unsigned int fcu_ctl_csr, fcu_sts_csr;
+ unsigned long ae_mask = handle->hal_handle->ae_mask;
+ u32 ae_ctr = 0;
if (handle->fw_auth) {
- if (IS_QAT_GEN3(pci_get_device(GET_DEV(handle->accel_dev)))) {
+ for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num)
+ {
+ ae_ctr++;
+ }
+ if (IS_QAT_GEN3_OR_GEN4(
+ pci_get_device(GET_DEV(handle->accel_dev)))) {
fcu_ctl_csr = FCU_CONTROL_C4XXX;
fcu_sts_csr = FCU_STATUS_C4XXX;
@@ -1061,17 +1166,27 @@
pause_ms("adfstop", FW_AUTH_WAIT_PERIOD);
fcu_sts = GET_FCU_CSR(handle, fcu_sts_csr);
if (((fcu_sts >> FCU_STS_DONE_POS) & 0x1))
- return;
+ return ae_ctr;
} while (retry++ < FW_AUTH_MAX_RETRY);
pr_err("QAT: start error (AE 0x%x FCU_STS = 0x%x)\n",
ae,
fcu_sts);
+ return 0;
} else {
- qat_hal_put_wakeup_event(handle,
- ae,
- (~ctx_mask) & ICP_QAT_UCLO_AE_ALL_CTX,
- 0x10000);
- qat_hal_enable_ctx(handle, ae, ctx_mask);
+ for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num)
+ {
+ qat_hal_put_wakeup_event(handle,
+ ae,
+ 0,
+ IS_QAT_GEN4(
+ pci_get_device(GET_DEV(
+ handle->accel_dev))) ?
+ 0x80000000 :
+ 0x10000);
+ qat_hal_enable_ctx(handle, ae, ICP_QAT_UCLO_AE_ALL_CTX);
+ ae_ctr++;
+ }
+ return ae_ctr;
}
}
@@ -1193,7 +1308,7 @@
handle, ae, ctx, INDIRECT_LM_ADDR_0_BYTE_INDEX, &ind_lm_addr_byte0);
qat_hal_rd_indr_csr(
handle, ae, ctx, INDIRECT_LM_ADDR_1_BYTE_INDEX, &ind_lm_addr_byte1);
- if (IS_QAT_GEN3(pci_get_device(GET_DEV(handle->accel_dev)))) {
+ if (IS_QAT_GEN3_OR_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) {
qat_hal_rd_indr_csr(
handle, ae, ctx, LM_ADDR_2_INDIRECT, &ind_lm_addr2);
qat_hal_rd_indr_csr(
@@ -1286,7 +1401,7 @@
(1 << ctx),
INDIRECT_LM_ADDR_1_BYTE_INDEX,
ind_lm_addr_byte1);
- if (IS_QAT_GEN3(pci_get_device(GET_DEV(handle->accel_dev)))) {
+ if (IS_QAT_GEN3_OR_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) {
qat_hal_wr_indr_csr(
handle, ae, (1 << ctx), LM_ADDR_2_INDIRECT, ind_lm_addr2);
qat_hal_wr_indr_csr(
@@ -1832,6 +1947,11 @@
int stat = 0;
unsigned char ctx;
+ if (IS_QAT_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) {
+ pr_err("QAT: No next neigh for CPM2X\n");
+ return EINVAL;
+ }
+
if (ctx_mask == 0)
return EINVAL;
Index: sys/dev/qat/qat_common/qat_uclo.c
===================================================================
--- sys/dev/qat/qat_common/qat_uclo.c
+++ sys/dev/qat/qat_common/qat_uclo.c
@@ -46,9 +46,13 @@
encap_image->img_ptr->ctx_assigned;
ae_data->shareable_ustore =
ICP_QAT_SHARED_USTORE_MODE(encap_image->img_ptr->ae_mode);
- ae_data->eff_ustore_size = ae_data->shareable_ustore ?
- (obj_handle->ustore_phy_size << 1) :
- obj_handle->ustore_phy_size;
+ if (obj_handle->prod_type == ICP_QAT_AC_4XXX_A_DEV_TYPE)
+ ae_data->eff_ustore_size = obj_handle->ustore_phy_size;
+ else {
+ ae_data->eff_ustore_size = ae_data->shareable_ustore ?
+ (obj_handle->ustore_phy_size << 1) :
+ obj_handle->ustore_phy_size;
+ }
} else {
ae_slice->ctx_mask_assigned = 0;
}
@@ -324,9 +328,13 @@
{
struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
unsigned int ae;
+ unsigned int lmem;
+
+ lmem = IS_QAT_GEN4(pci_get_device(GET_DEV(handle->accel_dev))) ?
+ ICP_QAT_UCLO_MAX_LMEM_REG_2X :
+ ICP_QAT_UCLO_MAX_LMEM_REG;
- if (qat_uclo_fetch_initmem_ae(
- handle, init_mem, ICP_QAT_UCLO_MAX_LMEM_REG, &ae))
+ if (qat_uclo_fetch_initmem_ae(handle, init_mem, lmem, &ae))
return EINVAL;
if (qat_uclo_create_batch_init_list(
handle, init_mem, ae, &obj_handle->lm_init_tab[ae]))
@@ -411,6 +419,8 @@
{
unsigned long cfg_ae_mask = handle->cfg_ae_mask;
unsigned long ae_assigned = uof_image->ae_assigned;
+ const bool gen4 =
+ IS_QAT_GEN4(pci_get_device(GET_DEV(handle->accel_dev)));
if (!test_bit(ae, &cfg_ae_mask))
continue;
@@ -418,7 +428,8 @@
if (!test_bit(ae, &ae_assigned))
continue;
- if (obj_handle->ae_data[ae].shareable_ustore && (ae & 1)) {
+ if (obj_handle->ae_data[ae].shareable_ustore && (ae & 1) &&
+ !gen4) {
qat_hal_get_scs_neigh_ae(ae, &neigh_ae);
if (test_bit(neigh_ae, &ae_assigned))
@@ -427,7 +438,7 @@
ustore_size = obj_handle->ae_data[ae].eff_ustore_size;
patt_pos = page->beg_addr_p + page->micro_words_num;
- if (obj_handle->ae_data[ae].shareable_ustore) {
+ if (obj_handle->ae_data[ae].shareable_ustore && !gen4) {
qat_hal_get_scs_neigh_ae(ae, &neigh_ae);
if (init[ae] == 0 && page->beg_addr_p != 0) {
qat_hal_wr_coalesce_uwords(handle,
@@ -445,6 +456,9 @@
init[ae] = 1;
init[neigh_ae] = 1;
} else {
+ if (gen4 && (ae % 4 != 0))
+ continue;
+
qat_hal_wr_uwords(handle,
(unsigned char)ae,
0,
@@ -727,6 +741,61 @@
return 0;
}
+static int
+UcLo_checkTGroupList2X(struct icp_qat_fw_loader_handle *handle)
+{
+ int i;
+ unsigned int swAe = 0;
+ unsigned int ii, jj;
+ struct icp_qat_uclo_aedata *ae_data0, *ae_datax;
+ struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
+
+ for (i = 0; i < obj_handle->uimage_num; i++) {
+ struct icp_qat_uof_image *image =
+ obj_handle->ae_uimage[i].img_ptr;
+ if (image->numpages > 1) {
+ pr_err(
+ "Only 1 page is allowed in a UOF for CPM2X; We found %d in %s\n",
+ image->numpages,
+ qat_uclo_get_string(&obj_handle->str_table,
+ image->img_name));
+ return EINVAL;
+ }
+ }
+
+ for (swAe = 0;
+ (swAe < obj_handle->ae_num) && (swAe < ICP_QAT_UCLO_MAX_AE);
+ swAe += AE_TG_NUM_CPM2X) {
+ if (!qat_hal_check_ae_active(handle, swAe)) {
+ continue;
+ }
+
+ for (ii = swAe; ii < (swAe + AE_TG_NUM_CPM2X); ii++) {
+ ae_data0 = &obj_handle->ae_data[ii];
+ if (ae_data0->slice_num != 1) // not assigned
+ continue;
+
+ for (jj = ii + 1; jj < (swAe + AE_TG_NUM_CPM2X); jj++) {
+ ae_datax = &obj_handle->ae_data[jj];
+ if (ae_datax->slice_num != 1) // not assigned
+ continue;
+ if (ae_data0->ae_slices[0]
+ .encap_image->img_ptr !=
+ ae_datax->ae_slices[0]
+ .encap_image->img_ptr) {
+ pr_err("Only 1 list is allowed in a ");
+ pr_err("Tgroup for CPM2X;\n");
+ pr_err("ME%d, %d is assigned", ii, jj);
+ pr_err(" different list files\n");
+ return EINVAL;
+ }
+ }
+ }
+ }
+
+ return 0;
+}
+
static int
qat_uclo_map_ae(struct icp_qat_fw_loader_handle *handle, int max_ae)
{
@@ -752,6 +821,11 @@
return EINVAL;
}
}
+ if (IS_QAT_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) {
+ if (UcLo_checkTGroupList2X(handle)) {
+ return EINVAL;
+ }
+ }
if (!mflag) {
pr_err("QAT: uimage uses AE not set");
return EINVAL;
@@ -817,6 +891,8 @@
return ICP_QAT_AC_200XX_DEV_TYPE;
case ADF_C4XXX_PCI_DEVICE_ID:
return ICP_QAT_AC_C4XXX_DEV_TYPE;
+ case ADF_4XXX_PCI_DEVICE_ID:
+ return ICP_QAT_AC_4XXX_A_DEV_TYPE;
default:
pr_err("QAT: unsupported device 0x%x\n",
pci_get_device(GET_DEV(handle->accel_dev)));
@@ -1001,11 +1077,13 @@
ae_mode = (char)ICP_QAT_SHARED_USTORE_MODE(uof_image->ae_mode);
qat_hal_set_ae_scs_mode(handle, ae, ae_mode);
- nn_mode = ICP_QAT_NN_MODE(uof_image->ae_mode);
+ if (!IS_QAT_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) {
+ nn_mode = ICP_QAT_NN_MODE(uof_image->ae_mode);
- if (qat_hal_set_ae_nn_mode(handle, ae, nn_mode)) {
- pr_err("QAT: qat_hal_set_ae_nn_mode error\n");
- return EFAULT;
+ if (qat_hal_set_ae_nn_mode(handle, ae, nn_mode)) {
+ pr_err("QAT: qat_hal_set_ae_nn_mode error\n");
+ return EFAULT;
+ }
}
ae_mode = (char)ICP_QAT_LOC_MEM0_MODE(uof_image->ae_mode);
if (qat_hal_set_ae_lm_mode(handle, ae, ICP_LMEM0, ae_mode)) {
@@ -1017,7 +1095,7 @@
pr_err("QAT: qat_hal_set_ae_lm_mode LMEM1 error\n");
return EFAULT;
}
- if (obj_handle->prod_type == ICP_QAT_AC_C4XXX_DEV_TYPE) {
+ if (IS_QAT_GEN3_OR_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) {
ae_mode = (char)ICP_QAT_LOC_MEM2_MODE(uof_image->ae_mode);
if (qat_hal_set_ae_lm_mode(handle, ae, ICP_LMEM2, ae_mode)) {
pr_err("QAT: qat_hal_set_ae_lm_mode LMEM2 error\n");
@@ -1168,12 +1246,14 @@
}
static void
-qat_uclo_map_simg(struct icp_qat_suof_handle *suof_handle,
+qat_uclo_map_simg(struct icp_qat_fw_loader_handle *handle,
struct icp_qat_suof_img_hdr *suof_img_hdr,
struct icp_qat_suof_chunk_hdr *suof_chunk_hdr)
{
+ struct icp_qat_suof_handle *suof_handle = handle->sobj_handle;
const struct icp_qat_simg_ae_mode *ae_mode;
struct icp_qat_suof_objhdr *suof_objhdr;
+ unsigned int device_id = pci_get_device(GET_DEV(handle->accel_dev));
suof_img_hdr->simg_buf =
(suof_handle->suof_buf + suof_chunk_hdr->offset +
@@ -1187,9 +1267,10 @@
suof_img_hdr->css_key =
(suof_img_hdr->css_header + sizeof(struct icp_qat_css_hdr));
suof_img_hdr->css_signature = suof_img_hdr->css_key +
- ICP_QAT_CSS_FWSK_MODULUS_LEN + ICP_QAT_CSS_FWSK_EXPONENT_LEN;
+ ICP_QAT_CSS_FWSK_MODULUS_LEN(device_id) +
+ ICP_QAT_CSS_FWSK_EXPONENT_LEN(device_id);
suof_img_hdr->css_simg =
- suof_img_hdr->css_signature + ICP_QAT_CSS_SIGNATURE_LEN;
+ suof_img_hdr->css_signature + ICP_QAT_CSS_SIGNATURE_LEN(device_id);
ae_mode = (const struct icp_qat_simg_ae_mode *)(suof_img_hdr->css_simg);
suof_img_hdr->ae_mask = ae_mode->ae_mask;
@@ -1277,7 +1358,8 @@
struct icp_qat_suof_handle *suof_handle = handle->sobj_handle;
struct icp_qat_suof_chunk_hdr *suof_chunk_hdr = NULL;
struct icp_qat_suof_img_hdr *suof_img_hdr = NULL;
- int ret = 0, ae0_img = ICP_QAT_UCLO_MAX_AE;
+ int ret = 0, ae0_img = ICP_QAT_UCLO_MAX_AE,
+ aeMax_img = ICP_QAT_UCLO_MAX_AE;
unsigned int i = 0;
struct icp_qat_suof_img_hdr img_header;
@@ -1305,7 +1387,7 @@
}
for (i = 0; i < suof_handle->img_table.num_simgs; i++) {
- qat_uclo_map_simg(handle->sobj_handle,
+ qat_uclo_map_simg(handle,
&suof_img_hdr[i],
&suof_chunk_hdr[1 + i]);
ret = qat_uclo_check_simg_compat(handle, &suof_img_hdr[i]);
@@ -1315,9 +1397,29 @@
if ((suof_img_hdr[i].ae_mask & 0x1) != 0)
ae0_img = i;
}
- qat_uclo_tail_img(suof_img_hdr,
- ae0_img,
- suof_handle->img_table.num_simgs);
+
+ if (!IS_QAT_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) {
+ qat_uclo_tail_img(suof_img_hdr,
+ ae0_img,
+ suof_handle->img_table.num_simgs);
+ } else {
+ if (suof_handle->img_table.num_simgs == 1)
+ return 0;
+ qat_uclo_tail_img(suof_img_hdr,
+ ae0_img,
+ suof_handle->img_table.num_simgs - 1);
+ for (i = 0; i < suof_handle->img_table.num_simgs; i++) {
+ if ((suof_img_hdr[i].ae_mask &
+ (0x1 << (handle->hal_handle->ae_max_num - 1))) !=
+ 0) {
+ aeMax_img = i;
+ break;
+ }
+ }
+ qat_uclo_tail_img(suof_img_hdr,
+ aeMax_img,
+ suof_handle->img_table.num_simgs);
+ }
return 0;
}
@@ -1335,7 +1437,7 @@
bus_addr = ADD_ADDR(desc->css_hdr_high, desc->css_hdr_low) -
sizeof(struct icp_qat_auth_chunk);
- if (IS_QAT_GEN3(pci_get_device(GET_DEV(handle->accel_dev)))) {
+ if (IS_QAT_GEN3_OR_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) {
fcu_ctl_csr = FCU_CONTROL_C4XXX;
fcu_sts_csr = FCU_STATUS_C4XXX;
fcu_dram_hi_csr = FCU_DRAM_ADDR_HI_C4XXX;
@@ -1372,6 +1474,103 @@
return EINVAL;
}
+static int
+qat_uclo_is_broadcast(struct icp_qat_fw_loader_handle *handle, int imgid)
+{
+ struct icp_qat_suof_handle *sobj_handle;
+
+ if (!IS_QAT_GEN4(pci_get_device(GET_DEV(handle->accel_dev))))
+ return 0;
+
+ sobj_handle = (struct icp_qat_suof_handle *)handle->sobj_handle;
+ if (handle->hal_handle->admin_ae_mask &
+ sobj_handle->img_table.simg_hdr[imgid].ae_mask)
+ return 0;
+
+ return 1;
+}
+
+static int
+qat_uclo_broadcast_load_fw(struct icp_qat_fw_loader_handle *handle,
+ struct icp_qat_fw_auth_desc *desc)
+{
+ unsigned int i = 0;
+ unsigned int fcuSts = 0, fcuAeBroadcastMask = 0;
+ unsigned int retry = 0;
+ unsigned int fcuStsCsr = 0;
+ unsigned int fcuCtlCsr = 0;
+ unsigned int loadedAes = 0;
+ unsigned int device_id = pci_get_device(GET_DEV(handle->accel_dev));
+
+ if (IS_QAT_GEN4(device_id)) {
+ fcuCtlCsr = FCU_CONTROL_4XXX;
+ fcuStsCsr = FCU_STATUS_4XXX;
+ } else {
+ pr_err("Uclo_BroadcastLoadFW only applicable for CPM20\n");
+ return EINVAL;
+ }
+
+ for (i = 0; i < ICP_QAT_UCLO_MAX_AE; i++) {
+ if (!test_bit(i, (unsigned long *)&handle->hal_handle->ae_mask))
+ continue;
+
+ if (qat_hal_check_ae_active(handle, (unsigned char)i)) {
+ pr_err(
+ "Uclo_BroadcastLoadFW error (invalid AE status)\n");
+ return EINVAL;
+ }
+
+ if ((desc->ae_mask >> i) & 0x1) {
+ fcuAeBroadcastMask |= 1 << i;
+ }
+ }
+
+ if (fcuAeBroadcastMask) {
+ retry = 0;
+ SET_FCU_CSR(handle,
+ FCU_ME_BROADCAST_MASK_TYPE,
+ fcuAeBroadcastMask);
+ SET_FCU_CSR(handle, fcuCtlCsr, FCU_CTRL_CMD_LOAD);
+ do {
+ msleep(FW_AUTH_WAIT_PERIOD);
+ fcuSts = GET_FCU_CSR(handle, fcuStsCsr);
+
+ if ((fcuSts & FCU_AUTH_STS_MASK) == FCU_STS_LOAD_FAIL) {
+ pr_err(
+ "Uclo_BroadcastLoadFW fail (fcu_status = 0x%x)\n",
+ fcuSts & FCU_AUTH_STS_MASK);
+ return EINVAL;
+ } else if ((fcuSts & FCU_AUTH_STS_MASK) ==
+ FCU_STS_LOAD_DONE) {
+ if (IS_QAT_GEN4(device_id))
+ loadedAes =
+ GET_FCU_CSR(handle,
+ FCU_AE_LOADED_4XXX);
+ else
+ loadedAes =
+ (fcuSts >> FCU_LOADED_AE_POS);
+
+ if ((loadedAes & fcuAeBroadcastMask) ==
+ fcuAeBroadcastMask)
+ break;
+ } else if ((fcuSts & FCU_AUTH_STS_MASK) ==
+ FCU_STS_VERI_DONE) {
+ SET_FCU_CSR(handle,
+ fcuCtlCsr,
+ FCU_CTRL_CMD_LOAD);
+ }
+ } while (retry++ < FW_BROADCAST_MAX_RETRY);
+ if (retry > FW_BROADCAST_MAX_RETRY) {
+ pr_err(
+ "Uclo_BroadcastLoadFW fail(fcu_status = 0x%x),retry = %d\n",
+ fcuSts & FCU_AUTH_STS_MASK,
+ retry);
+ return EINVAL;
+ }
+ }
+ return 0;
+}
+
static int
qat_uclo_simg_alloc(struct icp_qat_fw_loader_handle *handle,
struct icp_firml_dram_desc *dram_desc,
@@ -1417,14 +1616,16 @@
struct icp_qat_auth_chunk *auth_chunk;
u64 virt_addr, bus_addr, virt_base;
unsigned int length, simg_offset = sizeof(*auth_chunk);
+ unsigned int device_id = pci_get_device(GET_DEV(handle->accel_dev));
- if (size > (ICP_QAT_AE_IMG_OFFSET + ICP_QAT_CSS_MAX_IMAGE_LEN)) {
+ if (size >
+ (ICP_QAT_AE_IMG_OFFSET(device_id) + ICP_QAT_CSS_MAX_IMAGE_LEN)) {
pr_err("QAT: error, input image size overflow %d\n", size);
return EINVAL;
}
length = (css_hdr->fw_type == CSS_AE_FIRMWARE) ?
- ICP_QAT_CSS_AE_SIMG_LEN + simg_offset :
- size + ICP_QAT_CSS_FWSK_PAD_LEN + simg_offset;
+ ICP_QAT_CSS_AE_SIMG_LEN(device_id) + simg_offset :
+ size + ICP_QAT_CSS_FWSK_PAD_LEN(device_id) + simg_offset;
if (qat_uclo_simg_alloc(handle, img_desc, length)) {
pr_err("QAT: error, allocate continuous dram fail\n");
return -ENOMEM;
@@ -1451,42 +1652,43 @@
memcpy((void *)(uintptr_t)virt_addr,
(const void *)(image + sizeof(*css_hdr)),
- ICP_QAT_CSS_FWSK_MODULUS_LEN);
+ ICP_QAT_CSS_FWSK_MODULUS_LEN(device_id));
/* padding */
- explicit_bzero((void *)(uintptr_t)(virt_addr +
- ICP_QAT_CSS_FWSK_MODULUS_LEN),
- ICP_QAT_CSS_FWSK_PAD_LEN);
+ explicit_bzero((void *)(uintptr_t)(
+ virt_addr + ICP_QAT_CSS_FWSK_MODULUS_LEN(device_id)),
+ ICP_QAT_CSS_FWSK_PAD_LEN(device_id));
/* exponent */
- memcpy((void *)(uintptr_t)(virt_addr + ICP_QAT_CSS_FWSK_MODULUS_LEN +
- ICP_QAT_CSS_FWSK_PAD_LEN),
+ memcpy((void *)(uintptr_t)(virt_addr +
+ ICP_QAT_CSS_FWSK_MODULUS_LEN(device_id) +
+ ICP_QAT_CSS_FWSK_PAD_LEN(device_id)),
(const void *)(image + sizeof(*css_hdr) +
- ICP_QAT_CSS_FWSK_MODULUS_LEN),
+ ICP_QAT_CSS_FWSK_MODULUS_LEN(device_id)),
sizeof(unsigned int));
/* signature */
bus_addr = ADD_ADDR(auth_desc->fwsk_pub_high, auth_desc->fwsk_pub_low) +
- ICP_QAT_CSS_FWSK_PUB_LEN;
- virt_addr = virt_addr + ICP_QAT_CSS_FWSK_PUB_LEN;
+ ICP_QAT_CSS_FWSK_PUB_LEN(device_id);
+ virt_addr = virt_addr + ICP_QAT_CSS_FWSK_PUB_LEN(device_id);
auth_desc->signature_high = (unsigned int)(bus_addr >> BITS_IN_DWORD);
auth_desc->signature_low = (unsigned int)bus_addr;
memcpy((void *)(uintptr_t)virt_addr,
(const void *)(image + sizeof(*css_hdr) +
- ICP_QAT_CSS_FWSK_MODULUS_LEN +
- ICP_QAT_CSS_FWSK_EXPONENT_LEN),
- ICP_QAT_CSS_SIGNATURE_LEN);
+ ICP_QAT_CSS_FWSK_MODULUS_LEN(device_id) +
+ ICP_QAT_CSS_FWSK_EXPONENT_LEN(device_id)),
+ ICP_QAT_CSS_SIGNATURE_LEN(device_id));
bus_addr =
ADD_ADDR(auth_desc->signature_high, auth_desc->signature_low) +
- ICP_QAT_CSS_SIGNATURE_LEN;
- virt_addr += ICP_QAT_CSS_SIGNATURE_LEN;
+ ICP_QAT_CSS_SIGNATURE_LEN(device_id);
+ virt_addr += ICP_QAT_CSS_SIGNATURE_LEN(device_id);
auth_desc->img_high = (unsigned int)(bus_addr >> BITS_IN_DWORD);
auth_desc->img_low = (unsigned int)bus_addr;
- auth_desc->img_len = size - ICP_QAT_AE_IMG_OFFSET;
+ auth_desc->img_len = size - ICP_QAT_AE_IMG_OFFSET(device_id);
memcpy((void *)(uintptr_t)virt_addr,
- (const void *)(image + ICP_QAT_AE_IMG_OFFSET),
+ (const void *)(image + ICP_QAT_AE_IMG_OFFSET(device_id)),
auth_desc->img_len);
virt_addr = virt_base;
/* AE firmware */
@@ -1506,7 +1708,8 @@
(unsigned int)(bus_addr >> BITS_IN_DWORD);
auth_desc->img_ae_insts_low = (unsigned int)bus_addr;
virt_addr += sizeof(struct icp_qat_css_hdr) +
- ICP_QAT_CSS_FWSK_PUB_LEN + ICP_QAT_CSS_SIGNATURE_LEN;
+ ICP_QAT_CSS_FWSK_PUB_LEN(device_id) +
+ ICP_QAT_CSS_SIGNATURE_LEN(device_id);
auth_desc->ae_mask =
((struct icp_qat_simg_ae_mode *)virt_addr)->ae_mask &
handle->cfg_ae_mask;
@@ -1528,7 +1731,7 @@
unsigned int loaded_aes = FCU_LOADED_AE_POS;
unsigned long ae_mask = handle->hal_handle->ae_mask;
- if (IS_QAT_GEN3(pci_get_device(GET_DEV(handle->accel_dev)))) {
+ if (IS_QAT_GEN3_OR_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) {
fcu_ctl_csr = FCU_CONTROL_C4XXX;
fcu_sts_csr = FCU_STATUS_C4XXX;
@@ -1549,14 +1752,19 @@
}
SET_FCU_CSR(handle,
fcu_ctl_csr,
- (FCU_CTRL_CMD_LOAD | (i << FCU_CTRL_AE_POS)));
+ (FCU_CTRL_CMD_LOAD |
+ (IS_QAT_GEN4(
+ pci_get_device(GET_DEV(handle->accel_dev))) ?
+ (1 << FCU_CTRL_BROADCAST_POS) :
+ 0) |
+ (i << FCU_CTRL_AE_POS)));
do {
pause_ms("adfstop", FW_AUTH_WAIT_PERIOD);
fcu_sts = GET_FCU_CSR(handle, fcu_sts_csr);
if ((fcu_sts & FCU_AUTH_STS_MASK) ==
FCU_STS_LOAD_DONE) {
- loaded_aes = IS_QAT_GEN3(pci_get_device(
+ loaded_aes = IS_QAT_GEN3_OR_GEN4(pci_get_device(
GET_DEV(handle->accel_dev))) ?
GET_FCU_CSR(handle, FCU_AE_LOADED_C4XXX) :
(fcu_sts >> FCU_LOADED_AE_POS);
@@ -1606,6 +1814,16 @@
qat_uclo_simg_free(handle, &img_desc);
} else {
+ if (IS_QAT_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) {
+ device_printf(
+ NULL, "QAT: PKE service is not allowed because ");
+ device_printf(NULL, "MMP fw will not be loaded for ");
+ device_printf(NULL,
+ "device 0x%x",
+ pci_get_device(
+ GET_DEV(handle->accel_dev)));
+ return status;
+ }
if (pci_get_device(GET_DEV(handle->accel_dev)) ==
ADF_C3XXX_PCI_DEVICE_ID) {
pr_err("QAT: C3XXX doesn't support unsigned MMP\n");
@@ -2044,7 +2262,8 @@
uw_relative_addr + i,
fill_pat);
- if (obj_handle->ae_data[ae].shareable_ustore)
+ if (obj_handle->ae_data[ae].shareable_ustore &&
+ !IS_QAT_GEN4(pci_get_device(GET_DEV(handle->accel_dev))))
/* copy the buffer to ustore */
qat_hal_wr_coalesce_uwords(handle,
(unsigned char)ae,
@@ -2140,10 +2359,16 @@
goto wr_err;
if (qat_uclo_auth_fw(handle, desc))
goto wr_err;
- if (qat_uclo_load_fw(handle, desc))
- goto wr_err;
+ if (qat_uclo_is_broadcast(handle, i)) {
+ if (qat_uclo_broadcast_load_fw(handle, desc))
+ goto wr_err;
+ } else {
+ if (qat_uclo_load_fw(handle, desc))
+ goto wr_err;
+ }
qat_uclo_simg_free(handle, &img_desc);
}
+
return 0;
wr_err:
qat_uclo_simg_free(handle, &img_desc);
Index: sys/dev/qat/qat_hw/qat_200xx/adf_200xx_hw_data.c
===================================================================
--- sys/dev/qat/qat_hw/qat_200xx/adf_200xx_hw_data.c
+++ sys/dev/qat/qat_hw/qat_200xx/adf_200xx_hw_data.c
@@ -6,6 +6,7 @@
#include <adf_cfg.h>
#include <adf_pf2vf_msg.h>
#include <adf_dev_err.h>
+#include <adf_gen2_hw_data.h>
#include "adf_200xx_hw_data.h"
#include "icp_qat_hw.h"
#include "adf_heartbeat.h"
@@ -495,6 +496,7 @@
hw_data->get_errsou_offset = get_errsou_offset;
hw_data->get_clock_speed = get_clock_speed;
hw_data->get_sku = get_sku;
+ hw_data->heartbeat_ctr_num = ADF_NUM_HB_CNT_PER_AE;
hw_data->fw_name = ADF_200XX_FW;
hw_data->fw_mmp_name = ADF_200XX_MMP;
hw_data->init_admin_comms = adf_init_admin_comms;
@@ -532,6 +534,8 @@
hw_data->ring_to_svc_map = ADF_DEFAULT_RING_TO_SRV_MAP;
hw_data->pre_reset = adf_dev_pre_reset;
hw_data->post_reset = adf_dev_post_reset;
+
+ adf_gen2_init_hw_csr_info(&hw_data->csr_info);
}
void
Index: sys/dev/qat/qat_hw/qat_200xx/adf_drv.c
===================================================================
--- sys/dev/qat/qat_hw/qat_200xx/adf_drv.c
+++ sys/dev/qat/qat_hw/qat_200xx/adf_drv.c
@@ -145,6 +145,7 @@
/* Get Accelerators and Accelerators Engines masks */
hw_data->accel_mask = hw_data->get_accel_mask(accel_dev);
hw_data->ae_mask = hw_data->get_ae_mask(accel_dev);
+ hw_data->admin_ae_mask = hw_data->ae_mask;
accel_pci_dev->sku = hw_data->get_sku(hw_data);
/* If the device has no acceleration engines then ignore it. */
Index: sys/dev/qat/qat_hw/qat_4xxx/adf_4xxx_hw_data.h
===================================================================
--- /dev/null
+++ sys/dev/qat/qat_hw/qat_4xxx/adf_4xxx_hw_data.h
@@ -0,0 +1,111 @@
+/* SPDX-License-Identifier: BSD-3-Clause */
+/* Copyright(c) 2007 - 2022 Intel Corporation */
+/* $FreeBSD$ */
+#ifndef ADF_4XXX_HW_DATA_H_
+#define ADF_4XXX_HW_DATA_H_
+
+#include <adf_accel_devices.h>
+
+/* PCIe configuration space */
+#define ADF_4XXX_SRAM_BAR 0
+#define ADF_4XXX_PMISC_BAR 1
+#define ADF_4XXX_ETR_BAR 2
+#define ADF_4XXX_RX_RINGS_OFFSET 1
+#define ADF_4XXX_TX_RINGS_MASK 0x1
+#define ADF_4XXX_MAX_ACCELERATORS 1
+#define ADF_4XXX_MAX_ACCELENGINES 9
+#define ADF_4XXX_BAR_MASK (BIT(0) | BIT(2) | BIT(4))
+
+/* 2 Accel units dedicated to services and */
+/* 1 Accel unit dedicated to Admin AE */
+#define ADF_4XXX_MAX_ACCELUNITS 3
+
+/* Physical function fuses */
+#define ADF_4XXX_FUSECTL0_OFFSET (0x2C8)
+#define ADF_4XXX_FUSECTL1_OFFSET (0x2CC)
+#define ADF_4XXX_FUSECTL2_OFFSET (0x2D0)
+#define ADF_4XXX_FUSECTL3_OFFSET (0x2D4)
+#define ADF_4XXX_FUSECTL4_OFFSET (0x2D8)
+#define ADF_4XXX_FUSECTL5_OFFSET (0x2DC)
+
+#define ADF_4XXX_ACCELERATORS_MASK (0x1)
+#define ADF_4XXX_ACCELENGINES_MASK (0x1FF)
+#define ADF_4XXX_ADMIN_AE_MASK (0x100)
+
+#define ADF_4XXX_ETR_MAX_BANKS 64
+
+/* MSIX interrupt */
+#define ADF_4XXX_SMIAPF_RP_X0_MASK_OFFSET (0x41A040)
+#define ADF_4XXX_SMIAPF_RP_X1_MASK_OFFSET (0x41A044)
+#define ADF_4XXX_SMIAPF_MASK_OFFSET (0x41A084)
+#define ADF_4XXX_MSIX_RTTABLE_OFFSET(i) (0x409000 + ((i)*0x04))
+
+/* Bank and ring configuration */
+#define ADF_4XXX_NUM_RINGS_PER_BANK 2
+
+/* Error source registers */
+#define ADF_4XXX_ERRSOU0 (0x41A200)
+#define ADF_4XXX_ERRSOU1 (0x41A204)
+#define ADF_4XXX_ERRSOU2 (0x41A208)
+#define ADF_4XXX_ERRSOU3 (0x41A20C)
+
+/* Error source mask registers */
+#define ADF_4XXX_ERRMSK0 (0x41A210)
+#define ADF_4XXX_ERRMSK1 (0x41A214)
+#define ADF_4XXX_ERRMSK2 (0x41A218)
+#define ADF_4XXX_ERRMSK3 (0x41A21C)
+
+#define ADF_4XXX_VFLNOTIFY BIT(7)
+
+/* Arbiter configuration */
+#define ADF_4XXX_ARB_CONFIG (BIT(31) | BIT(6) | BIT(0))
+#define ADF_4XXX_ARB_OFFSET (0x0)
+#define ADF_4XXX_ARB_WRK_2_SER_MAP_OFFSET (0x400)
+
+/* Admin Interface Reg Offset */
+#define ADF_4XXX_ADMINMSGUR_OFFSET (0x500574)
+#define ADF_4XXX_ADMINMSGLR_OFFSET (0x500578)
+#define ADF_4XXX_MAILBOX_BASE_OFFSET (0x600970)
+
+/* Power management */
+#define ADF_4XXX_PM_POLL_DELAY_US 20
+#define ADF_4XXX_PM_POLL_TIMEOUT_US USEC_PER_SEC
+#define ADF_4XXX_PM_STATUS (0x50A00C)
+#define ADF_4XXX_PM_INTERRUPT (0x50A028)
+#define ADF_4XXX_PM_DRV_ACTIVE BIT(20)
+#define ADF_4XXX_PM_INIT_STATE BIT(21)
+/* Power management source in ERRSOU2 and ERRMSK2 */
+#define ADF_4XXX_PM_SOU BIT(18)
+
+/* Firmware Binaries */
+#define ADF_4XXX_FW "qat_4xxx_fw"
+#define ADF_4XXX_MMP "qat_4xxx_mmp_fw"
+#define ADF_4XXX_DC_OBJ "qat_4xxx_dc.bin"
+#define ADF_4XXX_SYM_OBJ "qat_4xxx_sym.bin"
+#define ADF_4XXX_ASYM_OBJ "qat_4xxx_asym.bin"
+#define ADF_4XXX_ADMIN_OBJ "qat_4xxx_admin.bin"
+
+/* Only 3 types of images can be loaded including the admin image */
+#define ADF_4XXX_MAX_OBJ 3
+
+#define ADF_4XXX_AE_FREQ (1000 * 1000000)
+#define ADF_4XXX_KPT_COUNTER_FREQ (100 * 1000000)
+
+#define ADF_4XXX_MIN_AE_FREQ (9 * 1000000)
+#define ADF_4XXX_MAX_AE_FREQ (1100 * 1000000)
+
+/* qat_4xxx fuse bits are different from old GENs, redefine them */
+enum icp_qat_4xxx_slice_mask {
+ ICP_ACCEL_4XXX_MASK_CIPHER_SLICE = BIT(0),
+ ICP_ACCEL_4XXX_MASK_AUTH_SLICE = BIT(1),
+ ICP_ACCEL_4XXX_MASK_PKE_SLICE = BIT(2),
+ ICP_ACCEL_4XXX_MASK_COMPRESS_SLICE = BIT(3),
+ ICP_ACCEL_4XXX_MASK_UCS_SLICE = BIT(4),
+ ICP_ACCEL_4XXX_MASK_EIA3_SLICE = BIT(5),
+ ICP_ACCEL_4XXX_MASK_SMX_SLICE = BIT(6),
+};
+
+void adf_init_hw_data_4xxx(struct adf_hw_device_data *hw_data);
+void adf_clean_hw_data_4xxx(struct adf_hw_device_data *hw_data);
+
+#endif
Index: sys/dev/qat/qat_hw/qat_4xxx/adf_4xxx_hw_data.c
===================================================================
--- /dev/null
+++ sys/dev/qat/qat_hw/qat_4xxx/adf_4xxx_hw_data.c
@@ -0,0 +1,973 @@
+/* SPDX-License-Identifier: BSD-3-Clause */
+/* Copyright(c) 2007-2022 Intel Corporation */
+/* $FreeBSD$ */
+#include <linux/iopoll.h>
+#include <adf_accel_devices.h>
+#include <adf_cfg.h>
+#include <adf_common_drv.h>
+#include <adf_dev_err.h>
+#include <adf_pf2vf_msg.h>
+#include <adf_gen4_hw_data.h>
+#include "adf_4xxx_hw_data.h"
+#include "adf_heartbeat.h"
+#include "icp_qat_fw_init_admin.h"
+#include "icp_qat_hw.h"
+
+#define ADF_CONST_TABLE_SIZE 1024
+
+struct adf_fw_config {
+ u32 ae_mask;
+ char *obj_name;
+};
+
+/* Accel unit information */
+static const struct adf_accel_unit adf_4xxx_au_a_ae[] = {
+ { 0x1, 0x1, 0xF, 0x1B, 4, ADF_ACCEL_SERVICE_NULL },
+ { 0x2, 0x1, 0xF0, 0x6C0, 4, ADF_ACCEL_SERVICE_NULL },
+ { 0x4, 0x1, 0x100, 0xF000, 1, ADF_ACCEL_ADMIN },
+};
+
+/* Worker thread to service arbiter mappings */
+static u32 thrd_to_arb_map[ADF_4XXX_MAX_ACCELENGINES] = { 0x5555555, 0x5555555,
+ 0x5555555, 0x5555555,
+ 0xAAAAAAA, 0xAAAAAAA,
+ 0xAAAAAAA, 0xAAAAAAA,
+ 0x0 };
+
+/* Masks representing ME thread-service mappings.
+ * Thread 7 carries out Admin work and is thus
+ * left out.
+ */
+static u8 default_active_thd_mask = 0x7F;
+static u8 dc_me_active_thd_mask = 0x03;
+
+static u32 thrd_to_arb_map_gen[ADF_4XXX_MAX_ACCELENGINES] = { 0 };
+
+#define ADF_4XXX_ASYM_SYM \
+ (ASYM | SYM << ADF_CFG_SERV_RING_PAIR_1_SHIFT | \
+ ASYM << ADF_CFG_SERV_RING_PAIR_2_SHIFT | \
+ SYM << ADF_CFG_SERV_RING_PAIR_3_SHIFT)
+
+#define ADF_4XXX_DC \
+ (COMP | COMP << ADF_CFG_SERV_RING_PAIR_1_SHIFT | \
+ COMP << ADF_CFG_SERV_RING_PAIR_2_SHIFT | \
+ COMP << ADF_CFG_SERV_RING_PAIR_3_SHIFT)
+
+#define ADF_4XXX_SYM \
+ (SYM | SYM << ADF_CFG_SERV_RING_PAIR_1_SHIFT | \
+ SYM << ADF_CFG_SERV_RING_PAIR_2_SHIFT | \
+ SYM << ADF_CFG_SERV_RING_PAIR_3_SHIFT)
+
+#define ADF_4XXX_ASYM \
+ (ASYM | ASYM << ADF_CFG_SERV_RING_PAIR_1_SHIFT | \
+ ASYM << ADF_CFG_SERV_RING_PAIR_2_SHIFT | \
+ ASYM << ADF_CFG_SERV_RING_PAIR_3_SHIFT)
+
+#define ADF_4XXX_ASYM_DC \
+ (ASYM | ASYM << ADF_CFG_SERV_RING_PAIR_1_SHIFT | \
+ COMP << ADF_CFG_SERV_RING_PAIR_2_SHIFT | \
+ COMP << ADF_CFG_SERV_RING_PAIR_3_SHIFT)
+
+#define ADF_4XXX_SYM_DC \
+ (SYM | SYM << ADF_CFG_SERV_RING_PAIR_1_SHIFT | \
+ COMP << ADF_CFG_SERV_RING_PAIR_2_SHIFT | \
+ COMP << ADF_CFG_SERV_RING_PAIR_3_SHIFT)
+
+#define ADF_4XXX_NA \
+ (NA | NA << ADF_CFG_SERV_RING_PAIR_1_SHIFT | \
+ NA << ADF_CFG_SERV_RING_PAIR_2_SHIFT | \
+ NA << ADF_CFG_SERV_RING_PAIR_3_SHIFT)
+
+#define ADF_4XXX_DEFAULT_RING_TO_SRV_MAP ADF_4XXX_ASYM_SYM
+
+struct adf_enabled_services {
+ const char svcs_enabled[ADF_CFG_MAX_VAL_LEN_IN_BYTES];
+ u16 rng_to_svc_msk;
+};
+
+static struct adf_enabled_services adf_4xxx_svcs[] = {
+ { "dc", ADF_4XXX_DC },
+ { "sym", ADF_4XXX_SYM },
+ { "asym", ADF_4XXX_ASYM },
+ { "dc;asym", ADF_4XXX_ASYM_DC },
+ { "asym;dc", ADF_4XXX_ASYM_DC },
+ { "sym;dc", ADF_4XXX_SYM_DC },
+ { "dc;sym", ADF_4XXX_SYM_DC },
+ { "asym;sym", ADF_4XXX_ASYM_SYM },
+ { "sym;asym", ADF_4XXX_ASYM_SYM },
+};
+
+static struct adf_hw_device_class adf_4xxx_class = {
+ .name = ADF_4XXX_DEVICE_NAME,
+ .type = DEV_4XXX,
+ .instances = 0,
+};
+
+static u32
+get_accel_mask(struct adf_accel_dev *accel_dev)
+{
+ return ADF_4XXX_ACCELERATORS_MASK;
+}
+
+static u32
+get_ae_mask(struct adf_accel_dev *accel_dev)
+{
+ u32 fusectl4 = accel_dev->hw_device->fuses;
+
+ return ~fusectl4 & ADF_4XXX_ACCELENGINES_MASK;
+}
+
+static int
+get_ring_to_svc_map(struct adf_accel_dev *accel_dev, u16 *ring_to_svc_map)
+{
+ char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES];
+ char val[ADF_CFG_MAX_KEY_LEN_IN_BYTES];
+ u32 i = 0;
+
+ *ring_to_svc_map = 0;
+ /* Get the services enabled by user */
+ snprintf(key, sizeof(key), ADF_SERVICES_ENABLED);
+ if (adf_cfg_get_param_value(accel_dev, ADF_GENERAL_SEC, key, val))
+ return EFAULT;
+
+ for (i = 0; i < ARRAY_SIZE(adf_4xxx_svcs); i++) {
+ if (!strncmp(val,
+ adf_4xxx_svcs[i].svcs_enabled,
+ ADF_CFG_MAX_KEY_LEN_IN_BYTES)) {
+ *ring_to_svc_map = adf_4xxx_svcs[i].rng_to_svc_msk;
+ return 0;
+ }
+ }
+
+ device_printf(GET_DEV(accel_dev),
+ "Invalid services enabled: %s\n",
+ val);
+ return EFAULT;
+}
+
+static u32
+get_num_accels(struct adf_hw_device_data *self)
+{
+ return ADF_4XXX_MAX_ACCELERATORS;
+}
+
+static u32
+get_num_aes(struct adf_hw_device_data *self)
+{
+ if (!self || !self->ae_mask)
+ return 0;
+
+ return hweight32(self->ae_mask);
+}
+
+static u32
+get_misc_bar_id(struct adf_hw_device_data *self)
+{
+ return ADF_4XXX_PMISC_BAR;
+}
+
+static u32
+get_etr_bar_id(struct adf_hw_device_data *self)
+{
+ return ADF_4XXX_ETR_BAR;
+}
+
+static u32
+get_sram_bar_id(struct adf_hw_device_data *self)
+{
+ return ADF_4XXX_SRAM_BAR;
+}
+
+/*
+ * The vector routing table is used to select the MSI-X entry to use for each
+ * interrupt source.
+ * The first ADF_4XXX_ETR_MAX_BANKS entries correspond to ring interrupts.
+ * The final entry corresponds to VF2PF or error interrupts.
+ * This vector table could be used to configure one MSI-X entry to be shared
+ * between multiple interrupt sources.
+ *
+ * The default routing is set to have a one to one correspondence between the
+ * interrupt source and the MSI-X entry used.
+ */
+static void
+set_msix_default_rttable(struct adf_accel_dev *accel_dev)
+{
+ struct resource *csr;
+ int i;
+
+ csr = (&GET_BARS(accel_dev)[ADF_4XXX_PMISC_BAR])->virt_addr;
+ for (i = 0; i <= ADF_4XXX_ETR_MAX_BANKS; i++)
+ ADF_CSR_WR(csr, ADF_4XXX_MSIX_RTTABLE_OFFSET(i), i);
+}
+
+static u32
+adf_4xxx_get_hw_cap(struct adf_accel_dev *accel_dev)
+{
+ device_t pdev = accel_dev->accel_pci_dev.pci_dev;
+ u32 fusectl1;
+ u32 capabilities;
+
+ /* Read accelerator capabilities mask */
+ fusectl1 = pci_read_config(pdev, ADF_4XXX_FUSECTL1_OFFSET, 4);
+ capabilities = ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC |
+ ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC |
+ ICP_ACCEL_CAPABILITIES_CIPHER |
+ ICP_ACCEL_CAPABILITIES_AUTHENTICATION |
+ ICP_ACCEL_CAPABILITIES_COMPRESSION |
+ ICP_ACCEL_CAPABILITIES_LZ4_COMPRESSION |
+ ICP_ACCEL_CAPABILITIES_LZ4S_COMPRESSION |
+ ICP_ACCEL_CAPABILITIES_HKDF | ICP_ACCEL_CAPABILITIES_SHA3_EXT |
+ ICP_ACCEL_CAPABILITIES_SM3 | ICP_ACCEL_CAPABILITIES_SM4 |
+ ICP_ACCEL_CAPABILITIES_CHACHA_POLY |
+ ICP_ACCEL_CAPABILITIES_AESGCM_SPC |
+ ICP_ACCEL_CAPABILITIES_AES_V2 | ICP_ACCEL_CAPABILITIES_RL;
+
+ if (fusectl1 & ICP_ACCEL_4XXX_MASK_CIPHER_SLICE) {
+ capabilities &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC;
+ capabilities &= ~ICP_ACCEL_CAPABILITIES_CIPHER;
+ }
+ if (fusectl1 & ICP_ACCEL_4XXX_MASK_AUTH_SLICE)
+ capabilities &= ~ICP_ACCEL_CAPABILITIES_AUTHENTICATION;
+ if (fusectl1 & ICP_ACCEL_4XXX_MASK_PKE_SLICE)
+ capabilities &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC;
+ if (fusectl1 & ICP_ACCEL_4XXX_MASK_COMPRESS_SLICE) {
+ capabilities &= ~ICP_ACCEL_CAPABILITIES_COMPRESSION;
+ capabilities &= ~ICP_ACCEL_CAPABILITIES_CNV_INTEGRITY64;
+ }
+ if (fusectl1 & ICP_ACCEL_4XXX_MASK_SMX_SLICE) {
+ capabilities &= ~ICP_ACCEL_CAPABILITIES_SM3;
+ capabilities &= ~ICP_ACCEL_CAPABILITIES_SM4;
+ }
+ return capabilities;
+}
+
+static u32
+get_hb_clock(struct adf_hw_device_data *self)
+{
+ /*
+ * 4XXX uses KPT counter for HB
+ */
+ return ADF_4XXX_KPT_COUNTER_FREQ;
+}
+
+static u32
+get_ae_clock(struct adf_hw_device_data *self)
+{
+ /*
+ * Clock update interval is <16> ticks for qat_4xxx.
+ */
+ return self->clock_frequency / 16;
+}
+
+static int
+measure_clock(struct adf_accel_dev *accel_dev)
+{
+ u32 frequency;
+ int ret = 0;
+
+ ret = adf_dev_measure_clock(accel_dev,
+ &frequency,
+ ADF_4XXX_MIN_AE_FREQ,
+ ADF_4XXX_MAX_AE_FREQ);
+ if (ret)
+ return ret;
+
+ accel_dev->hw_device->clock_frequency = frequency;
+ return 0;
+}
+
+static int
+adf_4xxx_configure_accel_units(struct adf_accel_dev *accel_dev)
+{
+ char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES] = { 0 };
+ char val_str[ADF_CFG_MAX_VAL_LEN_IN_BYTES] = { 0 };
+
+ if (adf_cfg_section_add(accel_dev, ADF_GENERAL_SEC))
+ goto err;
+
+ snprintf(key, sizeof(key), ADF_SERVICES_ENABLED);
+ snprintf(val_str,
+ sizeof(val_str),
+ ADF_CFG_ASYM ADF_SERVICES_SEPARATOR ADF_CFG_SYM);
+
+ if (adf_cfg_add_key_value_param(
+ accel_dev, ADF_GENERAL_SEC, key, (void *)val_str, ADF_STR))
+ goto err;
+
+ return 0;
+err:
+ device_printf(GET_DEV(accel_dev), "Failed to configure accel units\n");
+ return EINVAL;
+}
+
+static u32
+get_num_accel_units(struct adf_hw_device_data *self)
+{
+ return ADF_4XXX_MAX_ACCELUNITS;
+}
+
+static void
+get_accel_unit(struct adf_hw_device_data *self,
+ struct adf_accel_unit **accel_unit)
+{
+ memcpy(*accel_unit, adf_4xxx_au_a_ae, sizeof(adf_4xxx_au_a_ae));
+}
+
+static void
+adf_exit_accel_unit_services(struct adf_accel_dev *accel_dev)
+{
+ if (accel_dev->au_info) {
+ kfree(accel_dev->au_info->au);
+ accel_dev->au_info->au = NULL;
+ kfree(accel_dev->au_info);
+ accel_dev->au_info = NULL;
+ }
+}
+
+static int
+get_accel_unit_config(struct adf_accel_dev *accel_dev,
+ u8 *num_sym_au,
+ u8 *num_dc_au,
+ u8 *num_asym_au)
+{
+ struct adf_hw_device_data *hw_data = accel_dev->hw_device;
+ char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES];
+ char val[ADF_CFG_MAX_VAL_LEN_IN_BYTES];
+ u32 num_au = hw_data->get_num_accel_units(hw_data);
+ /* One AU will be allocated by default if a service enabled */
+ u32 alloc_au = 1;
+ /* There's always one AU that is used for Admin AE */
+ u32 service_mask = ADF_ACCEL_ADMIN;
+ char *token, *cur_str;
+ u32 disabled_caps = 0;
+
+ /* Get the services enabled by user */
+ snprintf(key, sizeof(key), ADF_SERVICES_ENABLED);
+ if (adf_cfg_get_param_value(accel_dev, ADF_GENERAL_SEC, key, val))
+ return EFAULT;
+ cur_str = val;
+ token = strsep(&cur_str, ADF_SERVICES_SEPARATOR);
+ while (token) {
+ if (!strncmp(token, ADF_CFG_SYM, strlen(ADF_CFG_SYM)))
+ service_mask |= ADF_ACCEL_CRYPTO;
+ if (!strncmp(token, ADF_CFG_ASYM, strlen(ADF_CFG_ASYM)))
+ service_mask |= ADF_ACCEL_ASYM;
+
+ /* cy means both asym & crypto should be enabled
+ * Hardware resources allocation check will be done later
+ */
+ if (!strncmp(token, ADF_CFG_CY, strlen(ADF_CFG_CY)))
+ service_mask |= ADF_ACCEL_ASYM | ADF_ACCEL_CRYPTO;
+ if (!strncmp(token, ADF_SERVICE_DC, strlen(ADF_SERVICE_DC)))
+ service_mask |= ADF_ACCEL_COMPRESSION;
+
+ token = strsep(&cur_str, ADF_SERVICES_SEPARATOR);
+ }
+
+ /* Ensure the user won't enable more services than it can support */
+ if (hweight32(service_mask) > num_au) {
+ device_printf(GET_DEV(accel_dev),
+ "Can't enable more services than ");
+ device_printf(GET_DEV(accel_dev), "%d!\n", num_au);
+ return EFAULT;
+ } else if (hweight32(service_mask) == 2) {
+ /* Due to limitation, besides AU for Admin AE
+ * only 2 more AUs can be allocated
+ */
+ alloc_au = 2;
+ }
+
+ if (service_mask & ADF_ACCEL_CRYPTO)
+ *num_sym_au = alloc_au;
+ if (service_mask & ADF_ACCEL_ASYM)
+ *num_asym_au = alloc_au;
+ if (service_mask & ADF_ACCEL_COMPRESSION)
+ *num_dc_au = alloc_au;
+
+ /*update capability*/
+ if (!*num_sym_au || !(service_mask & ADF_ACCEL_CRYPTO)) {
+ disabled_caps = ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC |
+ ICP_ACCEL_CAPABILITIES_CIPHER |
+ ICP_ACCEL_CAPABILITIES_SHA3_EXT |
+ ICP_ACCEL_CAPABILITIES_SM3 | ICP_ACCEL_CAPABILITIES_SM4 |
+ ICP_ACCEL_CAPABILITIES_CHACHA_POLY |
+ ICP_ACCEL_CAPABILITIES_AESGCM_SPC |
+ ICP_ACCEL_CAPABILITIES_AES_V2;
+ }
+ if (!*num_asym_au || !(service_mask & ADF_ACCEL_ASYM)) {
+ disabled_caps |= ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC |
+ ICP_ACCEL_CAPABILITIES_AUTHENTICATION;
+ }
+ if (!*num_dc_au || !(service_mask & ADF_ACCEL_COMPRESSION)) {
+ disabled_caps |= ICP_ACCEL_CAPABILITIES_COMPRESSION |
+ ICP_ACCEL_CAPABILITIES_LZ4_COMPRESSION |
+ ICP_ACCEL_CAPABILITIES_LZ4S_COMPRESSION |
+ ICP_ACCEL_CAPABILITIES_CNV_INTEGRITY64;
+ accel_dev->hw_device->extended_dc_capabilities = 0;
+ }
+ accel_dev->hw_device->accel_capabilities_mask =
+ adf_4xxx_get_hw_cap(accel_dev) & ~disabled_caps;
+
+ hw_data->service_mask = service_mask;
+ hw_data->service_to_load_mask = service_mask;
+
+ return 0;
+}
+
+static int
+adf_init_accel_unit_services(struct adf_accel_dev *accel_dev)
+{
+ u8 num_sym_au = 0, num_dc_au = 0, num_asym_au = 0;
+ struct adf_hw_device_data *hw_data = accel_dev->hw_device;
+ u32 num_au = hw_data->get_num_accel_units(hw_data);
+ u32 au_size = num_au * sizeof(struct adf_accel_unit);
+ u8 i;
+
+ if (get_accel_unit_config(
+ accel_dev, &num_sym_au, &num_dc_au, &num_asym_au))
+ return EFAULT;
+
+ accel_dev->au_info = kzalloc(sizeof(*accel_dev->au_info), GFP_KERNEL);
+ if (!accel_dev->au_info)
+ return ENOMEM;
+
+ accel_dev->au_info->au = kzalloc(au_size, GFP_KERNEL);
+ if (!accel_dev->au_info->au) {
+ kfree(accel_dev->au_info);
+ accel_dev->au_info = NULL;
+ return ENOMEM;
+ }
+
+ accel_dev->au_info->num_cy_au = num_sym_au;
+ accel_dev->au_info->num_dc_au = num_dc_au;
+ accel_dev->au_info->num_asym_au = num_asym_au;
+
+ get_accel_unit(hw_data, &accel_dev->au_info->au);
+
+ /* Enable ASYM accel units */
+ for (i = 0; i < num_au && num_asym_au > 0; i++) {
+ if (accel_dev->au_info->au[i].services ==
+ ADF_ACCEL_SERVICE_NULL) {
+ accel_dev->au_info->au[i].services = ADF_ACCEL_ASYM;
+ num_asym_au--;
+ }
+ }
+ /* Enable SYM accel units */
+ for (i = 0; i < num_au && num_sym_au > 0; i++) {
+ if (accel_dev->au_info->au[i].services ==
+ ADF_ACCEL_SERVICE_NULL) {
+ accel_dev->au_info->au[i].services = ADF_ACCEL_CRYPTO;
+ num_sym_au--;
+ }
+ }
+ /* Enable compression accel units */
+ for (i = 0; i < num_au && num_dc_au > 0; i++) {
+ if (accel_dev->au_info->au[i].services ==
+ ADF_ACCEL_SERVICE_NULL) {
+ accel_dev->au_info->au[i].services =
+ ADF_ACCEL_COMPRESSION;
+ num_dc_au--;
+ }
+ }
+ accel_dev->au_info->dc_ae_msk |=
+ hw_data->get_obj_cfg_ae_mask(accel_dev, ADF_ACCEL_COMPRESSION);
+
+ return 0;
+}
+
+static int
+adf_init_accel_units(struct adf_accel_dev *accel_dev)
+{
+ return adf_init_accel_unit_services(accel_dev);
+}
+
+static void
+adf_exit_accel_units(struct adf_accel_dev *accel_dev)
+{
+ /* reset the AU service */
+ adf_exit_accel_unit_services(accel_dev);
+}
+
+static const char *
+get_obj_name(struct adf_accel_dev *accel_dev,
+ enum adf_accel_unit_services service)
+{
+ switch (service) {
+ case ADF_ACCEL_ASYM:
+ return ADF_4XXX_ASYM_OBJ;
+ case ADF_ACCEL_CRYPTO:
+ return ADF_4XXX_SYM_OBJ;
+ case ADF_ACCEL_COMPRESSION:
+ return ADF_4XXX_DC_OBJ;
+ case ADF_ACCEL_ADMIN:
+ return ADF_4XXX_ADMIN_OBJ;
+ default:
+ return NULL;
+ }
+}
+
+static uint32_t
+get_objs_num(struct adf_accel_dev *accel_dev)
+{
+ return ADF_4XXX_MAX_OBJ;
+}
+
+static uint32_t
+get_obj_cfg_ae_mask(struct adf_accel_dev *accel_dev,
+ enum adf_accel_unit_services service)
+{
+ u32 ae_mask = 0;
+ struct adf_hw_device_data *hw_data = accel_dev->hw_device;
+ u32 num_au = hw_data->get_num_accel_units(hw_data);
+ struct adf_accel_unit *accel_unit = accel_dev->au_info->au;
+ u32 i = 0;
+
+ if (service == ADF_ACCEL_SERVICE_NULL)
+ return 0;
+
+ for (i = 0; i < num_au; i++) {
+ if (accel_unit[i].services == service)
+ ae_mask |= accel_unit[i].ae_mask;
+ }
+
+ return ae_mask;
+}
+
+static enum adf_accel_unit_services
+adf_4xxx_get_service_type(struct adf_accel_dev *accel_dev, s32 obj_num)
+{
+ struct adf_accel_unit *accel_unit;
+ struct adf_hw_device_data *hw_data = accel_dev->hw_device;
+ u8 num_au = hw_data->get_num_accel_units(hw_data);
+ int i;
+
+ if (!hw_data->service_to_load_mask)
+ return ADF_ACCEL_SERVICE_NULL;
+
+ if (accel_dev->au_info && accel_dev->au_info->au)
+ accel_unit = accel_dev->au_info->au;
+ else
+ return ADF_ACCEL_SERVICE_NULL;
+
+ for (i = num_au - 2; i >= 0; i--) {
+ if (hw_data->service_to_load_mask & accel_unit[i].services) {
+ hw_data->service_to_load_mask &=
+ ~accel_unit[i].services;
+ return accel_unit[i].services;
+ }
+ }
+
+ /* admin AE should be loaded last */
+ if (hw_data->service_to_load_mask & accel_unit[num_au - 1].services) {
+ hw_data->service_to_load_mask &=
+ ~accel_unit[num_au - 1].services;
+ return accel_unit[num_au - 1].services;
+ }
+
+ return ADF_ACCEL_SERVICE_NULL;
+}
+
+static void
+get_ring_svc_map_data(int ring_pair_index,
+ u16 ring_to_svc_map,
+ u8 *serv_type,
+ int *ring_index,
+ int *num_rings_per_srv,
+ int bundle_num)
+{
+ *serv_type =
+ GET_SRV_TYPE(ring_to_svc_map, bundle_num % ADF_CFG_NUM_SERVICES);
+ *ring_index = 0;
+ *num_rings_per_srv = ADF_4XXX_NUM_RINGS_PER_BANK / 2;
+}
+
+static int
+adf_get_dc_extcapabilities(struct adf_accel_dev *accel_dev, u32 *capabilities)
+{
+ struct icp_qat_fw_init_admin_req req;
+ struct icp_qat_fw_init_admin_resp resp;
+ u8 i;
+ struct adf_hw_device_data *hw_data = accel_dev->hw_device;
+ u8 num_au = hw_data->get_num_accel_units(hw_data);
+ u32 first_dc_ae = 0;
+
+ for (i = 0; i < num_au; i++) {
+ if (accel_dev->au_info->au[i].services &
+ ADF_ACCEL_COMPRESSION) {
+ first_dc_ae = accel_dev->au_info->au[i].ae_mask;
+ first_dc_ae &= ~(first_dc_ae - 1);
+ }
+ }
+
+ memset(&req, 0, sizeof(req));
+ memset(&resp, 0, sizeof(resp));
+ req.cmd_id = ICP_QAT_FW_COMP_CAPABILITY_GET;
+
+ if (likely(first_dc_ae)) {
+ if (adf_send_admin(accel_dev, &req, &resp, first_dc_ae) ||
+ resp.status) {
+ *capabilities = 0;
+ return EFAULT;
+ }
+
+ *capabilities = resp.extended_features;
+ }
+
+ return 0;
+}
+
+static int
+adf_get_fw_status(struct adf_accel_dev *accel_dev,
+ u8 *major,
+ u8 *minor,
+ u8 *patch)
+{
+ struct icp_qat_fw_init_admin_req req;
+ struct icp_qat_fw_init_admin_resp resp;
+ u32 ae_mask = 1;
+
+ memset(&req, 0, sizeof(req));
+ memset(&resp, 0, sizeof(resp));
+ req.cmd_id = ICP_QAT_FW_STATUS_GET;
+
+ if (adf_send_admin(accel_dev, &req, &resp, ae_mask))
+ return EFAULT;
+
+ *major = resp.version_major_num;
+ *minor = resp.version_minor_num;
+ *patch = resp.version_patch_num;
+
+ return 0;
+}
+
+static int
+adf_4xxx_send_admin_init(struct adf_accel_dev *accel_dev)
+{
+ int ret = 0;
+ struct icp_qat_fw_init_admin_req req;
+ struct icp_qat_fw_init_admin_resp resp;
+ struct adf_hw_device_data *hw_data = accel_dev->hw_device;
+ u32 ae_mask = hw_data->ae_mask;
+ u32 admin_ae_mask = hw_data->admin_ae_mask;
+ u8 num_au = hw_data->get_num_accel_units(hw_data);
+ u8 i;
+ u32 dc_capabilities = 0;
+
+ for (i = 0; i < num_au; i++) {
+ if (accel_dev->au_info->au[i].services ==
+ ADF_ACCEL_SERVICE_NULL)
+ ae_mask &= ~accel_dev->au_info->au[i].ae_mask;
+
+ if (accel_dev->au_info->au[i].services != ADF_ACCEL_ADMIN)
+ admin_ae_mask &= ~accel_dev->au_info->au[i].ae_mask;
+ }
+
+ if (!accel_dev->admin) {
+ device_printf(GET_DEV(accel_dev), "adf_admin not available\n");
+ return EFAULT;
+ }
+
+ memset(&req, 0, sizeof(req));
+ memset(&resp, 0, sizeof(resp));
+
+ req.cmd_id = ICP_QAT_FW_CONSTANTS_CFG;
+ req.init_cfg_sz = ADF_CONST_TABLE_SIZE;
+ req.init_cfg_ptr = accel_dev->admin->const_tbl_addr;
+ if (adf_send_admin(accel_dev, &req, &resp, admin_ae_mask)) {
+ device_printf(GET_DEV(accel_dev),
+ "Error sending constants config message\n");
+ return EFAULT;
+ }
+
+ memset(&req, 0, sizeof(req));
+ memset(&resp, 0, sizeof(resp));
+ req.cmd_id = ICP_QAT_FW_INIT_ME;
+ if (adf_send_admin(accel_dev, &req, &resp, ae_mask)) {
+ device_printf(GET_DEV(accel_dev),
+ "Error sending init message\n");
+ return EFAULT;
+ }
+
+ memset(&req, 0, sizeof(req));
+ memset(&resp, 0, sizeof(resp));
+ req.cmd_id = ICP_QAT_FW_HEARTBEAT_TIMER_SET;
+ req.init_cfg_ptr = accel_dev->admin->phy_hb_addr;
+ if (adf_get_hb_timer(accel_dev, &req.heartbeat_ticks))
+ return EINVAL;
+
+ if (adf_send_admin(accel_dev, &req, &resp, ae_mask))
+ device_printf(GET_DEV(accel_dev),
+ "Heartbeat is not supported\n");
+
+ ret = adf_get_dc_extcapabilities(accel_dev, &dc_capabilities);
+ if (unlikely(ret)) {
+ device_printf(GET_DEV(accel_dev),
+ "Could not get FW ext. capabilities\n");
+ }
+
+ accel_dev->hw_device->extended_dc_capabilities = dc_capabilities;
+
+ adf_get_fw_status(accel_dev,
+ &accel_dev->fw_versions.fw_version_major,
+ &accel_dev->fw_versions.fw_version_minor,
+ &accel_dev->fw_versions.fw_version_patch);
+
+ device_printf(GET_DEV(accel_dev),
+ "FW version: %d.%d.%d\n",
+ accel_dev->fw_versions.fw_version_major,
+ accel_dev->fw_versions.fw_version_minor,
+ accel_dev->fw_versions.fw_version_patch);
+
+ return ret;
+}
+
+static enum dev_sku_info
+get_sku(struct adf_hw_device_data *self)
+{
+ return DEV_SKU_1;
+}
+
+static struct adf_accel_unit *
+get_au_by_ae(struct adf_accel_dev *accel_dev, int ae_num)
+{
+ int i = 0;
+ struct adf_accel_unit *accel_unit = accel_dev->au_info->au;
+
+ if (!accel_unit)
+ return NULL;
+
+ for (i = 0; i < ADF_4XXX_MAX_ACCELUNITS; i++)
+ if (accel_unit[i].ae_mask & BIT(ae_num))
+ return &accel_unit[i];
+
+ return NULL;
+}
+
+static bool
+check_accel_unit_service(enum adf_accel_unit_services au_srv,
+ enum adf_cfg_service_type ring_srv)
+{
+ if ((au_srv & ADF_ACCEL_SERVICE_NULL) && ring_srv == NA)
+ return true;
+ if ((au_srv & ADF_ACCEL_COMPRESSION) && ring_srv == COMP)
+ return true;
+ if ((au_srv & ADF_ACCEL_ASYM) && ring_srv == ASYM)
+ return true;
+ if ((au_srv & ADF_ACCEL_CRYPTO) && ring_srv == SYM)
+ return true;
+
+ return false;
+}
+
+static void
+adf_4xxx_cfg_gen_dispatch_arbiter(struct adf_accel_dev *accel_dev,
+ u32 *thrd_to_arb_map_gen)
+{
+ struct adf_accel_unit *au = NULL;
+ int engine = 0;
+ int thread = 0;
+ int service;
+ u16 ena_srv_mask;
+ u16 service_type;
+ u32 service_mask;
+ unsigned long thd_srv_mask = default_active_thd_mask;
+
+ ena_srv_mask = accel_dev->hw_device->ring_to_svc_map;
+ /* If ring_to_svc_map is not changed, return default arbiter value */
+ if (ena_srv_mask == ADF_4XXX_DEFAULT_RING_TO_SRV_MAP) {
+ memcpy(thrd_to_arb_map_gen,
+ thrd_to_arb_map,
+ sizeof(thrd_to_arb_map_gen[0]) *
+ ADF_4XXX_MAX_ACCELENGINES);
+ return;
+ }
+
+ for (engine = 0; engine < ADF_4XXX_MAX_ACCELENGINES - 1; engine++) {
+ thrd_to_arb_map_gen[engine] = 0;
+ service_mask = 0;
+ au = get_au_by_ae(accel_dev, engine);
+ if (!au)
+ continue;
+
+ for (service = 0; service < ADF_CFG_MAX_SERVICES; service++) {
+ service_type = GET_SRV_TYPE(ena_srv_mask, service);
+ if (check_accel_unit_service(au->services,
+ service_type))
+ service_mask |= BIT(service);
+ }
+
+ if (au->services == ADF_ACCEL_COMPRESSION)
+ thd_srv_mask = dc_me_active_thd_mask;
+ else
+ thd_srv_mask = default_active_thd_mask;
+
+ for_each_set_bit(thread, &thd_srv_mask, 8)
+ {
+ thrd_to_arb_map_gen[engine] |=
+ (service_mask << (ADF_CFG_MAX_SERVICES * thread));
+ }
+ }
+}
+
+static void
+adf_get_arbiter_mapping(struct adf_accel_dev *accel_dev,
+ u32 const **arb_map_config)
+{
+ int i;
+ struct adf_hw_device_data *hw_device = accel_dev->hw_device;
+
+ for (i = 1; i < ADF_4XXX_MAX_ACCELENGINES; i++) {
+ if (~hw_device->ae_mask & (1 << i))
+ thrd_to_arb_map[i] = 0;
+ }
+ adf_4xxx_cfg_gen_dispatch_arbiter(accel_dev, thrd_to_arb_map_gen);
+ *arb_map_config = thrd_to_arb_map_gen;
+}
+
+static void
+get_arb_info(struct arb_info *arb_info)
+{
+ arb_info->wrk_cfg_offset = ADF_4XXX_ARB_CONFIG;
+ arb_info->arbiter_offset = ADF_4XXX_ARB_OFFSET;
+ arb_info->wrk_thd_2_srv_arb_map = ADF_4XXX_ARB_WRK_2_SER_MAP_OFFSET;
+}
+
+static void
+get_admin_info(struct admin_info *admin_csrs_info)
+{
+ admin_csrs_info->mailbox_offset = ADF_4XXX_MAILBOX_BASE_OFFSET;
+ admin_csrs_info->admin_msg_ur = ADF_4XXX_ADMINMSGUR_OFFSET;
+ admin_csrs_info->admin_msg_lr = ADF_4XXX_ADMINMSGLR_OFFSET;
+}
+
+static void
+adf_enable_error_correction(struct adf_accel_dev *accel_dev)
+{
+ struct adf_bar *misc_bar = &GET_BARS(accel_dev)[ADF_4XXX_PMISC_BAR];
+ struct resource *csr = misc_bar->virt_addr;
+
+ /* Enable all in errsou3 except VFLR notification on host */
+ ADF_CSR_WR(csr, ADF_4XXX_ERRMSK3, ADF_4XXX_VFLNOTIFY);
+}
+
+static void
+adf_enable_ints(struct adf_accel_dev *accel_dev)
+{
+ struct resource *addr;
+
+ addr = (&GET_BARS(accel_dev)[ADF_4XXX_PMISC_BAR])->virt_addr;
+
+ /* Enable bundle interrupts */
+ ADF_CSR_WR(addr, ADF_4XXX_SMIAPF_RP_X0_MASK_OFFSET, 0);
+ ADF_CSR_WR(addr, ADF_4XXX_SMIAPF_RP_X1_MASK_OFFSET, 0);
+
+ /* Enable misc interrupts */
+ ADF_CSR_WR(addr, ADF_4XXX_SMIAPF_MASK_OFFSET, 0);
+}
+
+static int
+adf_init_device(struct adf_accel_dev *accel_dev)
+{
+ struct resource *addr;
+ u32 status;
+ u32 csr;
+ int ret;
+
+ addr = (&GET_BARS(accel_dev)[ADF_4XXX_PMISC_BAR])->virt_addr;
+
+ /* Temporarily mask PM interrupt */
+ csr = ADF_CSR_RD(addr, ADF_4XXX_ERRMSK2);
+ csr |= ADF_4XXX_PM_SOU;
+ ADF_CSR_WR(addr, ADF_4XXX_ERRMSK2, csr);
+
+ /* Set DRV_ACTIVE bit to power up the device */
+ ADF_CSR_WR(addr, ADF_4XXX_PM_INTERRUPT, ADF_4XXX_PM_DRV_ACTIVE);
+
+ /* Poll status register to make sure the device is powered up */
+ status = 0;
+ ret = read_poll_timeout(ADF_CSR_RD,
+ status,
+ status & ADF_4XXX_PM_INIT_STATE,
+ ADF_4XXX_PM_POLL_DELAY_US,
+ ADF_4XXX_PM_POLL_TIMEOUT_US,
+ true,
+ addr,
+ ADF_4XXX_PM_STATUS);
+ if (ret)
+ device_printf(GET_DEV(accel_dev),
+ "Failed to power up the device\n");
+
+ return ret;
+}
+
+void
+adf_init_hw_data_4xxx(struct adf_hw_device_data *hw_data)
+{
+ hw_data->dev_class = &adf_4xxx_class;
+ hw_data->instance_id = adf_4xxx_class.instances++;
+ hw_data->num_banks = ADF_4XXX_ETR_MAX_BANKS;
+ hw_data->num_rings_per_bank = ADF_4XXX_NUM_RINGS_PER_BANK;
+ hw_data->num_accel = ADF_4XXX_MAX_ACCELERATORS;
+ hw_data->num_engines = ADF_4XXX_MAX_ACCELENGINES;
+ hw_data->num_logical_accel = 1;
+ hw_data->tx_rx_gap = ADF_4XXX_RX_RINGS_OFFSET;
+ hw_data->tx_rings_mask = ADF_4XXX_TX_RINGS_MASK;
+ hw_data->alloc_irq = adf_isr_resource_alloc;
+ hw_data->free_irq = adf_isr_resource_free;
+ hw_data->enable_error_correction = adf_enable_error_correction;
+ hw_data->get_accel_mask = get_accel_mask;
+ hw_data->get_ae_mask = get_ae_mask;
+ hw_data->get_num_accels = get_num_accels;
+ hw_data->get_num_aes = get_num_aes;
+ hw_data->get_sram_bar_id = get_sram_bar_id;
+ hw_data->get_etr_bar_id = get_etr_bar_id;
+ hw_data->get_misc_bar_id = get_misc_bar_id;
+ hw_data->get_arb_info = get_arb_info;
+ hw_data->get_admin_info = get_admin_info;
+ hw_data->get_accel_cap = adf_4xxx_get_hw_cap;
+ hw_data->clock_frequency = ADF_4XXX_AE_FREQ;
+ hw_data->get_sku = get_sku;
+ hw_data->heartbeat_ctr_num = ADF_NUM_HB_CNT_PER_AE;
+ hw_data->fw_name = ADF_4XXX_FW;
+ hw_data->fw_mmp_name = ADF_4XXX_MMP;
+ hw_data->init_admin_comms = adf_init_admin_comms;
+ hw_data->exit_admin_comms = adf_exit_admin_comms;
+ hw_data->send_admin_init = adf_4xxx_send_admin_init;
+ hw_data->init_arb = adf_init_gen2_arb;
+ hw_data->exit_arb = adf_exit_arb;
+ hw_data->get_arb_mapping = adf_get_arbiter_mapping;
+ hw_data->enable_ints = adf_enable_ints;
+ hw_data->init_device = adf_init_device;
+ hw_data->reset_device = adf_reset_flr;
+ hw_data->restore_device = adf_dev_restore;
+ hw_data->init_accel_units = adf_init_accel_units;
+ hw_data->exit_accel_units = adf_exit_accel_units;
+ hw_data->get_num_accel_units = get_num_accel_units;
+ hw_data->configure_accel_units = adf_4xxx_configure_accel_units;
+ hw_data->get_ring_to_svc_map = get_ring_to_svc_map;
+ hw_data->get_ring_svc_map_data = get_ring_svc_map_data;
+ hw_data->admin_ae_mask = ADF_4XXX_ADMIN_AE_MASK;
+ hw_data->get_objs_num = get_objs_num;
+ hw_data->get_obj_name = get_obj_name;
+ hw_data->get_obj_cfg_ae_mask = get_obj_cfg_ae_mask;
+ hw_data->get_service_type = adf_4xxx_get_service_type;
+ hw_data->set_msix_rttable = set_msix_default_rttable;
+ hw_data->set_ssm_wdtimer = adf_gen4_set_ssm_wdtimer;
+ hw_data->disable_iov = adf_disable_sriov;
+ hw_data->min_iov_compat_ver = ADF_PFVF_COMPATIBILITY_VERSION;
+ hw_data->config_device = adf_config_device;
+ hw_data->set_asym_rings_mask = adf_cfg_set_asym_rings_mask;
+ hw_data->get_hb_clock = get_hb_clock;
+ hw_data->get_heartbeat_status = adf_get_heartbeat_status;
+ hw_data->get_ae_clock = get_ae_clock;
+ hw_data->measure_clock = measure_clock;
+ hw_data->query_storage_cap = 1;
+
+ adf_gen4_init_hw_csr_info(&hw_data->csr_info);
+}
+
+void
+adf_clean_hw_data_4xxx(struct adf_hw_device_data *hw_data)
+{
+ hw_data->dev_class->instances--;
+}
Index: sys/dev/qat/qat_hw/qat_4xxx/adf_drv.c
===================================================================
--- sys/dev/qat/qat_hw/qat_4xxx/adf_drv.c
+++ sys/dev/qat/qat_hw/qat_4xxx/adf_drv.c
@@ -1,11 +1,12 @@
/* SPDX-License-Identifier: BSD-3-Clause */
-/* Copyright(c) 2007-2022 Intel Corporation */
+/* Copyright(c) 2007 - 2022 Intel Corporation */
/* $FreeBSD$ */
#include "qat_freebsd.h"
#include "adf_cfg.h"
#include "adf_common_drv.h"
#include "adf_accel_devices.h"
-#include "adf_c62x_hw_data.h"
+#include "adf_4xxx_hw_data.h"
+#include "adf_gen4_hw_data.h"
#include "adf_fw_counters.h"
#include "adf_cfg_device.h"
#include <sys/types.h>
@@ -16,7 +17,7 @@
#include "adf_heartbeat_dbg.h"
#include "adf_cnvnr_freq_counters.h"
-static MALLOC_DEFINE(M_QAT_C62X, "qat_c62x", "qat_c62x");
+static MALLOC_DEFINE(M_QAT_4XXX, "qat_4xxx", "qat_4xxx");
#define ADF_SYSTEM_DEVICE(device_id) \
{ \
@@ -24,7 +25,7 @@
}
static const struct pci_device_id adf_pci_tbl[] = { ADF_SYSTEM_DEVICE(
- ADF_C62X_PCI_DEVICE_ID),
+ ADF_4XXX_PCI_DEVICE_ID),
{
0,
} };
@@ -38,7 +39,7 @@
if (pci_get_vendor(dev) == id->vendor &&
pci_get_device(dev) == id->device) {
device_set_desc(dev,
- "Intel " ADF_C62X_DEVICE_NAME
+ "Intel " ADF_4XXX_DEVICE_NAME
" QuickAssist");
return BUS_PROBE_GENERIC;
}
@@ -65,13 +66,13 @@
if (accel_dev->hw_device) {
switch (pci_get_device(accel_pci_dev->pci_dev)) {
- case ADF_C62X_PCI_DEVICE_ID:
- adf_clean_hw_data_c62x(accel_dev->hw_device);
+ case ADF_4XXX_PCI_DEVICE_ID:
+ adf_clean_hw_data_4xxx(accel_dev->hw_device);
break;
default:
break;
}
- free(accel_dev->hw_device, M_QAT_C62X);
+ free(accel_dev->hw_device, M_QAT_4XXX);
accel_dev->hw_device = NULL;
}
adf_cfg_dev_remove(accel_dev);
@@ -90,7 +91,8 @@
/* Set pci MaxPayLoad to 256. Implemented to avoid the issue of
* Pci-passthrough causing Maxpayload to be reset to 128 bytes
- * when the device is reset. */
+ * when the device is reset.
+ */
if (pci_get_max_payload(dev) != 256)
pci_set_max_payload(dev, 256);
@@ -103,22 +105,21 @@
if (bus_get_domain(dev, &accel_pci_dev->node) != 0)
accel_pci_dev->node = 0;
- /* XXX: Revisit if we actually need a devmgr table at all. */
-
/* Add accel device to accel table.
- * This should be called before adf_cleanup_accel is called */
+ * This should be called before adf_cleanup_accel is called
+ */
if (adf_devmgr_add_dev(accel_dev, NULL)) {
device_printf(dev, "Failed to add new accelerator device.\n");
return ENXIO;
}
/* Allocate and configure device configuration structure */
- hw_data = malloc(sizeof(*hw_data), M_QAT_C62X, M_WAITOK | M_ZERO);
+ hw_data = malloc(sizeof(*hw_data), M_QAT_4XXX, M_WAITOK | M_ZERO);
accel_dev->hw_device = hw_data;
- adf_init_hw_data_c62x(accel_dev->hw_device);
+ adf_init_hw_data_4xxx(accel_dev->hw_device);
accel_pci_dev->revid = pci_get_revid(dev);
- hw_data->fuses = pci_read_config(dev, ADF_DEVICE_FUSECTL_OFFSET, 4);
+ hw_data->fuses = pci_read_config(dev, ADF_4XXX_FUSECTL4_OFFSET, 4);
if (accel_pci_dev->revid == 0x00) {
device_printf(dev, "A0 stepping is not supported.\n");
ret = ENODEV;
@@ -133,10 +134,11 @@
/* Get Accelerators and Accelerators Engines masks */
hw_data->accel_mask = hw_data->get_accel_mask(accel_dev);
hw_data->ae_mask = hw_data->get_ae_mask(accel_dev);
+
accel_pci_dev->sku = hw_data->get_sku(hw_data);
/* If the device has no acceleration engines then ignore it. */
if (!hw_data->accel_mask || !hw_data->ae_mask ||
- ((~hw_data->ae_mask) & 0x01)) {
+ (~hw_data->ae_mask & 0x01)) {
device_printf(dev, "No acceleration units found\n");
ret = ENXIO;
goto out_err;
@@ -146,7 +148,6 @@
ret = adf_cfg_dev_add(accel_dev);
if (ret)
goto out_err;
-
ret = adf_clock_debugfs_add(accel_dev);
if (ret)
goto out_err;
@@ -176,16 +177,11 @@
}
/* Find and map all the device's BARS */
- i = (hw_data->fuses & ADF_DEVICE_FUSECTL_MASK) ? 1 : 0;
+ i = 0;
for (bar_nr = 0; i < ADF_PCI_MAX_BARS && bar_nr < PCIR_MAX_BAR_0;
bar_nr++) {
struct adf_bar *bar;
- /*
- * XXX: This isn't quite right as it will ignore a BAR
- * that wasn't assigned a valid resource range by the
- * firmware.
- */
rid = PCIR_BAR(bar_nr);
if (bus_get_resource(dev, SYS_RES_MEMORY, rid, NULL, NULL) != 0)
continue;
@@ -194,8 +190,7 @@
SYS_RES_MEMORY,
&rid,
RF_ACTIVE);
-
- if (bar->virt_addr == NULL) {
+ if (!bar->virt_addr) {
device_printf(dev, "Failed to map BAR %d\n", bar_nr);
ret = ENXIO;
goto out_err;
@@ -263,8 +258,8 @@
adf_methods,
sizeof(struct adf_accel_dev) };
-DRIVER_MODULE_ORDERED(qat_c62x, pci, adf_driver, NULL, NULL, SI_ORDER_THIRD);
-MODULE_VERSION(qat_c62x, 1);
-MODULE_DEPEND(qat_c62x, qat_common, 1, 1, 1);
-MODULE_DEPEND(qat_c62x, qat_api, 1, 1, 1);
-MODULE_DEPEND(qat_c62x, linuxkpi, 1, 1, 1);
+DRIVER_MODULE_ORDERED(qat_4xxx, pci, adf_driver, NULL, NULL, SI_ORDER_THIRD);
+MODULE_VERSION(qat_4xxx, 1);
+MODULE_DEPEND(qat_4xxx, qat_common, 1, 1, 1);
+MODULE_DEPEND(qat_4xxx, qat_api, 1, 1, 1);
+MODULE_DEPEND(qat_4xxx, linuxkpi, 1, 1, 1);
Index: sys/dev/qat/qat_hw/qat_c3xxx/adf_c3xxx_hw_data.c
===================================================================
--- sys/dev/qat/qat_hw/qat_c3xxx/adf_c3xxx_hw_data.c
+++ sys/dev/qat/qat_hw/qat_c3xxx/adf_c3xxx_hw_data.c
@@ -6,6 +6,7 @@
#include <adf_cfg.h>
#include <adf_pf2vf_msg.h>
#include <adf_dev_err.h>
+#include <adf_gen2_hw_data.h>
#include "adf_c3xxx_hw_data.h"
#include "icp_qat_hw.h"
#include "adf_heartbeat.h"
@@ -368,6 +369,7 @@
hw_data->get_errsou_offset = get_errsou_offset;
hw_data->get_clock_speed = get_clock_speed;
hw_data->get_sku = get_sku;
+ hw_data->heartbeat_ctr_num = ADF_NUM_HB_CNT_PER_AE;
hw_data->fw_name = ADF_C3XXX_FW;
hw_data->fw_mmp_name = ADF_C3XXX_MMP;
hw_data->init_admin_comms = adf_init_admin_comms;
@@ -406,6 +408,8 @@
hw_data->ring_to_svc_map = ADF_DEFAULT_RING_TO_SRV_MAP;
hw_data->pre_reset = adf_dev_pre_reset;
hw_data->post_reset = adf_dev_post_reset;
+
+ adf_gen2_init_hw_csr_info(&hw_data->csr_info);
}
void
Index: sys/dev/qat/qat_hw/qat_c3xxx/adf_drv.c
===================================================================
--- sys/dev/qat/qat_hw/qat_c3xxx/adf_drv.c
+++ sys/dev/qat/qat_hw/qat_c3xxx/adf_drv.c
@@ -133,6 +133,7 @@
/* Get Accelerators and Accelerators Engines masks */
hw_data->accel_mask = hw_data->get_accel_mask(accel_dev);
hw_data->ae_mask = hw_data->get_ae_mask(accel_dev);
+ hw_data->admin_ae_mask = hw_data->ae_mask;
accel_pci_dev->sku = hw_data->get_sku(hw_data);
/* If the device has no acceleration engines then ignore it. */
Index: sys/dev/qat/qat_hw/qat_c4xxx/adf_c4xxx_hw_data.c
===================================================================
--- sys/dev/qat/qat_hw/qat_c4xxx/adf_c4xxx_hw_data.c
+++ sys/dev/qat/qat_hw/qat_c4xxx/adf_c4xxx_hw_data.c
@@ -9,6 +9,7 @@
#include <adf_dev_err.h>
#include <adf_cfg.h>
#include <adf_fw_counters.h>
+#include <adf_gen2_hw_data.h>
#include "adf_c4xxx_hw_data.h"
#include "adf_c4xxx_reset.h"
#include "adf_c4xxx_inline.h"
@@ -754,7 +755,6 @@
ICP_ACCEL_CAPABILITIES_SM3 | ICP_ACCEL_CAPABILITIES_SM4 |
ICP_ACCEL_CAPABILITIES_CHACHA_POLY |
ICP_ACCEL_CAPABILITIES_AESGCM_SPC |
- ICP_ACCEL_CAPABILITIES_CNV_INTEGRITY |
ICP_ACCEL_CAPABILITIES_ECEDMONT;
if (legfuses & ICP_ACCEL_MASK_CIPHER_SLICE) {
@@ -2128,74 +2128,6 @@
}
}
-static int
-adf_get_heartbeat_status_c4xxx(struct adf_accel_dev *accel_dev)
-{
- struct adf_hw_device_data *hw_device = accel_dev->hw_device;
- struct icp_qat_fw_init_c4xxx_admin_hb_stats *live_s =
- (struct icp_qat_fw_init_c4xxx_admin_hb_stats *)
- accel_dev->admin->virt_hb_addr;
- const size_t max_aes = hw_device->get_num_aes(hw_device);
- const size_t stats_size =
- max_aes * sizeof(struct icp_qat_fw_init_c4xxx_admin_hb_stats);
- int ret = 0;
- size_t ae = 0, thr;
- unsigned long ae_mask = 0;
- int num_threads_per_ae = ADF_NUM_THREADS_PER_AE;
-
- /*
- * Memory layout of Heartbeat
- *
- * +----------------+----------------+---------+
- * | Live value | Last value | Count |
- * +----------------+----------------+---------+
- * \_______________/\_______________/\________/
- * ^ ^ ^
- * | | |
- * | | max_aes * sizeof(adf_hb_count)
- * | max_aes *
- * sizeof(icp_qat_fw_init_c4xxx_admin_hb_stats)
- * max_aes * sizeof(icp_qat_fw_init_c4xxx_admin_hb_stats)
- */
- struct icp_qat_fw_init_c4xxx_admin_hb_stats *curr_s;
- struct icp_qat_fw_init_c4xxx_admin_hb_stats *last_s = live_s + max_aes;
- struct adf_hb_count *count = (struct adf_hb_count *)(last_s + max_aes);
-
- curr_s = malloc(stats_size, M_QAT, M_WAITOK | M_ZERO);
-
- memcpy(curr_s, live_s, stats_size);
- ae_mask = hw_device->ae_mask;
-
- for_each_set_bit(ae, &ae_mask, max_aes)
- {
- for (thr = 0; thr < num_threads_per_ae; ++thr) {
- struct icp_qat_fw_init_admin_hb_cnt *curr =
- &curr_s[ae].stats[thr];
- struct icp_qat_fw_init_admin_hb_cnt *prev =
- &last_s[ae].stats[thr];
- u16 req = curr->req_heartbeat_cnt;
- u16 resp = curr->resp_heartbeat_cnt;
- u16 last = prev->resp_heartbeat_cnt;
-
- if ((thr == ADF_AE_ADMIN_THREAD || req != resp) &&
- resp == last) {
- u16 retry = ++count[ae].ae_thread[thr];
-
- if (retry >= ADF_CFG_HB_COUNT_THRESHOLD)
- ret = EIO;
- } else {
- count[ae].ae_thread[thr] = 0;
- }
- }
- }
-
- /* Copy current stats for the next iteration */
- memcpy(last_s, curr_s, stats_size);
- free(curr_s, M_QAT);
-
- return ret;
-}
-
void
adf_init_hw_data_c4xxx(struct adf_hw_device_data *hw_data)
{
@@ -2230,6 +2162,7 @@
hw_data->get_clock_speed = get_clock_speed;
hw_data->get_eth_doorbell_msg = get_eth_doorbell_msg;
hw_data->get_sku = get_sku;
+ hw_data->heartbeat_ctr_num = ADF_NUM_THREADS_PER_AE;
hw_data->check_prod_sku = c4xxx_check_prod_sku;
hw_data->fw_name = ADF_C4XXX_FW;
hw_data->fw_mmp_name = ADF_C4XXX_MMP;
@@ -2256,7 +2189,7 @@
hw_data->reset_hw_units = adf_c4xxx_reset_hw_units;
hw_data->exit_accel_units = adf_exit_accel_units;
hw_data->ring_to_svc_map = ADF_DEFAULT_RING_TO_SRV_MAP;
- hw_data->get_heartbeat_status = adf_get_heartbeat_status_c4xxx;
+ hw_data->get_heartbeat_status = adf_get_heartbeat_status;
hw_data->get_ae_clock = get_ae_clock;
hw_data->clock_frequency = ADF_C4XXX_AE_FREQ;
hw_data->measure_clock = measure_clock;
@@ -2275,6 +2208,9 @@
hw_data->count_ras_event = adf_fw_count_ras_event;
hw_data->config_device = adf_config_device;
hw_data->set_asym_rings_mask = adf_cfg_set_asym_rings_mask;
+
+ adf_gen2_init_hw_csr_info(&hw_data->csr_info);
+ hw_data->csr_info.arb_enable_mask = 0xF;
}
void
Index: sys/dev/qat/qat_hw/qat_c4xxx/adf_drv.c
===================================================================
--- sys/dev/qat/qat_hw/qat_c4xxx/adf_drv.c
+++ sys/dev/qat/qat_hw/qat_c4xxx/adf_drv.c
@@ -130,6 +130,7 @@
/* Get Accelerators and Accelerators Engines masks */
hw_data->accel_mask = hw_data->get_accel_mask(accel_dev);
hw_data->ae_mask = hw_data->get_ae_mask(accel_dev);
+ hw_data->admin_ae_mask = hw_data->ae_mask;
/* If the device has no acceleration engines then ignore it. */
if (!hw_data->accel_mask || !hw_data->ae_mask ||
Index: sys/dev/qat/qat_hw/qat_c62x/adf_c62x_hw_data.c
===================================================================
--- sys/dev/qat/qat_hw/qat_c62x/adf_c62x_hw_data.c
+++ sys/dev/qat/qat_hw/qat_c62x/adf_c62x_hw_data.c
@@ -6,6 +6,7 @@
#include <adf_cfg.h>
#include <adf_pf2vf_msg.h>
#include <adf_dev_err.h>
+#include <adf_gen2_hw_data.h>
#include "adf_c62x_hw_data.h"
#include "icp_qat_hw.h"
#include "adf_cfg.h"
@@ -373,6 +374,7 @@
hw_data->get_errsou_offset = get_errsou_offset;
hw_data->get_clock_speed = get_clock_speed;
hw_data->get_sku = get_sku;
+ hw_data->heartbeat_ctr_num = ADF_NUM_HB_CNT_PER_AE;
hw_data->fw_name = ADF_C62X_FW;
hw_data->fw_mmp_name = ADF_C62X_MMP;
hw_data->init_admin_comms = adf_init_admin_comms;
@@ -411,6 +413,8 @@
hw_data->ring_to_svc_map = ADF_DEFAULT_RING_TO_SRV_MAP;
hw_data->pre_reset = adf_dev_pre_reset;
hw_data->post_reset = adf_dev_post_reset;
+
+ adf_gen2_init_hw_csr_info(&hw_data->csr_info);
}
void
Index: sys/dev/qat/qat_hw/qat_c62x/adf_drv.c
===================================================================
--- sys/dev/qat/qat_hw/qat_c62x/adf_drv.c
+++ sys/dev/qat/qat_hw/qat_c62x/adf_drv.c
@@ -133,6 +133,7 @@
/* Get Accelerators and Accelerators Engines masks */
hw_data->accel_mask = hw_data->get_accel_mask(accel_dev);
hw_data->ae_mask = hw_data->get_ae_mask(accel_dev);
+ hw_data->admin_ae_mask = hw_data->ae_mask;
accel_pci_dev->sku = hw_data->get_sku(hw_data);
/* If the device has no acceleration engines then ignore it. */
if (!hw_data->accel_mask || !hw_data->ae_mask ||
Index: sys/dev/qat/qat_hw/qat_dh895xcc/adf_dh895xcc_hw_data.c
===================================================================
--- sys/dev/qat/qat_hw/qat_dh895xcc/adf_dh895xcc_hw_data.c
+++ sys/dev/qat/qat_hw/qat_dh895xcc/adf_dh895xcc_hw_data.c
@@ -7,6 +7,7 @@
#include <adf_pf2vf_msg.h>
#include <adf_common_drv.h>
#include <adf_dev_err.h>
+#include <adf_gen2_hw_data.h>
#include "adf_dh895xcc_hw_data.h"
#include "icp_qat_hw.h"
#include "adf_heartbeat.h"
@@ -361,6 +362,7 @@
hw_data->get_clock_speed = get_clock_speed;
hw_data->get_sram_bar_id = get_sram_bar_id;
hw_data->get_sku = get_sku;
+ hw_data->heartbeat_ctr_num = ADF_NUM_HB_CNT_PER_AE;
hw_data->fw_name = ADF_DH895XCC_FW;
hw_data->fw_mmp_name = ADF_DH895XCC_MMP;
hw_data->init_admin_comms = adf_init_admin_comms;
@@ -396,6 +398,8 @@
hw_data->ring_to_svc_map = ADF_DEFAULT_RING_TO_SRV_MAP;
hw_data->pre_reset = adf_dev_pre_reset;
hw_data->post_reset = adf_dev_post_reset;
+
+ adf_gen2_init_hw_csr_info(&hw_data->csr_info);
}
void
Index: sys/dev/qat/qat_hw/qat_dh895xcc/adf_drv.c
===================================================================
--- sys/dev/qat/qat_hw/qat_dh895xcc/adf_drv.c
+++ sys/dev/qat/qat_hw/qat_dh895xcc/adf_drv.c
@@ -126,6 +126,7 @@
/* Get Accelerators and Accelerators Engines masks */
hw_data->accel_mask = hw_data->get_accel_mask(accel_dev);
hw_data->ae_mask = hw_data->get_ae_mask(accel_dev);
+ hw_data->admin_ae_mask = hw_data->ae_mask;
accel_pci_dev->sku = hw_data->get_sku(hw_data);
/* If the device has no acceleration engines then ignore it. */
if (!hw_data->accel_mask || !hw_data->ae_mask ||
Index: sys/modules/qat/qat_api/Makefile
===================================================================
--- sys/modules/qat/qat_api/Makefile
+++ sys/modules/qat/qat_api/Makefile
@@ -44,6 +44,7 @@
SRCS+= common/crypto/sym/qat/lac_sym_qat_hash.c
SRCS+= common/crypto/sym/qat/lac_sym_qat_hash_defs_lookup.c
SRCS+= common/crypto/sym/qat/lac_sym_qat_cipher.c
+SRCS+= common/crypto/sym/qat/lac_sym_qat_constants_table.c
SRCS+= common/crypto/sym/qat/lac_sym_qat_key.c
SRCS+= common/crypto/sym/key/lac_sym_key.c
SRCS+= common/stubs/lac_stubs.c
Index: sys/modules/qat/qat_common/Makefile
===================================================================
--- sys/modules/qat/qat_common/Makefile
+++ sys/modules/qat/qat_common/Makefile
@@ -9,6 +9,8 @@
SRCS+= adf_heartbeat.c adf_freebsd_heartbeat_dbg.c
SRCS+= adf_dev_mgr.c adf_hw_arbiter.c
SRCS+= adf_init.c adf_transport.c adf_isr.c adf_fw_counters.c adf_dev_err.c
+SRCS+= adf_gen2_hw_data.c
+SRCS+= adf_gen4_hw_data.c
SRCS+= qat_freebsd.c
SRCS+= adf_freebsd_cfg_dev_dbg.c adf_freebsd_ver_dbg.c
SRCS+= adf_cfg_device.c adf_cfg_section.c adf_cfg_instance.c adf_cfg_bundle.c
Index: sys/modules/qat/qat_hw/Makefile
===================================================================
--- sys/modules/qat/qat_hw/Makefile
+++ sys/modules/qat/qat_hw/Makefile
@@ -6,6 +6,7 @@
KMOD= qat_hw
SRCS+= qat_c62x/adf_c62x_hw_data.c qat_c62x/adf_drv.c
SRCS+= qat_200xx/adf_200xx_hw_data.c qat_200xx/adf_drv.c
+SRCS+= qat_4xxx/adf_4xxx_hw_data.c qat_4xxx/adf_drv.c
SRCS+= qat_c3xxx/adf_c3xxx_hw_data.c qat_c3xxx/adf_drv.c
SRCS+= qat_dh895xcc/adf_dh895xcc_hw_data.c qat_dh895xcc/adf_drv.c
SRCS+= qat_c4xxx/adf_c4xxx_hw_data.c qat_c4xxx/adf_drv.c qat_c4xxx/adf_c4xxx_ae_config.c qat_c4xxx/adf_c4xxx_misc_error_stats.c
Index: sys/modules/qatfw/Makefile
===================================================================
--- sys/modules/qatfw/Makefile
+++ sys/modules/qatfw/Makefile
@@ -5,6 +5,7 @@
qat_200xx \
qat_c3xxx \
qat_c4xxx \
- qat_dh895xcc
+ qat_dh895xcc \
+ qat_4xxx
.include <bsd.subdir.mk>
Index: sys/modules/qatfw/qat_4xxx/Makefile
===================================================================
--- /dev/null
+++ sys/modules/qatfw/qat_4xxx/Makefile
@@ -0,0 +1,10 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2007-2022 Intel Corporation
+# $FreeBSD$
+.PATH: ${SRCTOP}/sys/contrib/dev/qat
+
+KMOD= qat_4xxx_fw
+
+FIRMWS= ${SRCTOP}/sys/contrib/dev/qat/qat_4xxx.bin:qat_4xxx_fw:111 ${SRCTOP}/sys/contrib/dev/qat/qat_4xxx_mmp.bin:qat_4xxx_mmp_fw:111
+
+.include <bsd.kmod.mk>

File Metadata

Mime Type
text/plain
Expires
Tue, Jul 28, 9:26 AM (11 h, 58 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
29282984
Default Alt Text
D36254.1785230803.diff (564 KB)

Event Timeline