/* * Copyright (c) 2017 Mellanox Technologies, Inc. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #ifndef _MLX5DV_H_ #define _MLX5DV_H_ #include #include #include /* For the __be64 type */ #include #include #if defined(__SSE3__) #include #include #include #endif /* defined(__SSE3__) */ #include #include #include #ifdef __cplusplus extern "C" { #endif /* Always inline the functions */ #ifdef __GNUC__ #define MLX5DV_ALWAYS_INLINE inline __attribute__((always_inline)) #else #define MLX5DV_ALWAYS_INLINE inline #endif #define MLX5DV_RES_TYPE_QP ((uint64_t)RDMA_DRIVER_MLX5 << 32 | 1) #define MLX5DV_RES_TYPE_RWQ ((uint64_t)RDMA_DRIVER_MLX5 << 32 | 2) #define MLX5DV_RES_TYPE_DBR ((uint64_t)RDMA_DRIVER_MLX5 << 32 | 3) #define MLX5DV_RES_TYPE_SRQ ((uint64_t)RDMA_DRIVER_MLX5 << 32 | 4) #define MLX5DV_RES_TYPE_CQ ((uint64_t)RDMA_DRIVER_MLX5 << 32 | 5) enum { MLX5_RCV_DBR = 0, MLX5_SND_DBR = 1, }; enum mlx5dv_context_comp_mask { MLX5DV_CONTEXT_MASK_CQE_COMPRESION = 1 << 0, MLX5DV_CONTEXT_MASK_SWP = 1 << 1, MLX5DV_CONTEXT_MASK_STRIDING_RQ = 1 << 2, MLX5DV_CONTEXT_MASK_TUNNEL_OFFLOADS = 1 << 3, MLX5DV_CONTEXT_MASK_DYN_BFREGS = 1 << 4, MLX5DV_CONTEXT_MASK_CLOCK_INFO_UPDATE = 1 << 5, MLX5DV_CONTEXT_MASK_FLOW_ACTION_FLAGS = 1 << 6, MLX5DV_CONTEXT_MASK_DC_ODP_CAPS = 1 << 7, MLX5DV_CONTEXT_MASK_HCA_CORE_CLOCK = 1 << 8, MLX5DV_CONTEXT_MASK_NUM_LAG_PORTS = 1 << 9, }; struct mlx5dv_cqe_comp_caps { uint32_t max_num; uint32_t supported_format; /* enum mlx5dv_cqe_comp_res_format */ }; struct mlx5dv_sw_parsing_caps { uint32_t sw_parsing_offloads; /* Use enum mlx5dv_sw_parsing_offloads */ uint32_t supported_qpts; }; struct mlx5dv_striding_rq_caps { uint32_t min_single_stride_log_num_of_bytes; uint32_t max_single_stride_log_num_of_bytes; uint32_t min_single_wqe_log_num_of_strides; uint32_t max_single_wqe_log_num_of_strides; uint32_t supported_qpts; }; enum mlx5dv_tunnel_offloads { MLX5DV_RAW_PACKET_CAP_TUNNELED_OFFLOAD_VXLAN = 1 << 0, MLX5DV_RAW_PACKET_CAP_TUNNELED_OFFLOAD_GRE = 1 << 1, MLX5DV_RAW_PACKET_CAP_TUNNELED_OFFLOAD_GENEVE = 1 << 2, MLX5DV_RAW_PACKET_CAP_TUNNELED_OFFLOAD_CW_MPLS_OVER_GRE = 1 << 3, MLX5DV_RAW_PACKET_CAP_TUNNELED_OFFLOAD_CW_MPLS_OVER_UDP = 1 << 4, }; enum mlx5dv_flow_action_cap_flags { MLX5DV_FLOW_ACTION_FLAGS_ESP_AES_GCM = 1 << 0, MLX5DV_FLOW_ACTION_FLAGS_ESP_AES_GCM_REQ_METADATA = 1 << 1, MLX5DV_FLOW_ACTION_FLAGS_ESP_AES_GCM_SPI_STEERING = 1 << 2, MLX5DV_FLOW_ACTION_FLAGS_ESP_AES_GCM_FULL_OFFLOAD = 1 << 3, MLX5DV_FLOW_ACTION_FLAGS_ESP_AES_GCM_TX_IV_IS_ESN = 1 << 4, }; struct mlx5dv_devx_port { uint64_t comp_mask; uint16_t vport_num; uint16_t vport_vhca_id; uint16_t esw_owner_vhca_id; uint64_t icm_addr_rx; uint64_t icm_addr_tx; struct mlx5dv_devx_reg_32 reg_c_0; }; /* * Direct verbs device-specific attributes */ struct mlx5dv_context { uint8_t version; uint64_t flags; uint64_t comp_mask; struct mlx5dv_cqe_comp_caps cqe_comp_caps; struct mlx5dv_sw_parsing_caps sw_parsing_caps; struct mlx5dv_striding_rq_caps striding_rq_caps; uint32_t tunnel_offloads_caps; uint32_t max_dynamic_bfregs; uint64_t max_clock_info_update_nsec; uint32_t flow_action_flags; /* use enum mlx5dv_flow_action_cap_flags */ uint32_t dc_odp_caps; /* use enum ibv_odp_transport_cap_bits */ void *hca_core_clock; uint8_t num_lag_ports; }; enum mlx5dv_context_flags { /* * This flag indicates if CQE version 0 or 1 is needed. */ MLX5DV_CONTEXT_FLAGS_CQE_V1 = (1 << 0), MLX5DV_CONTEXT_FLAGS_OBSOLETE = (1 << 1), /* Obsoleted, don't use */ MLX5DV_CONTEXT_FLAGS_MPW_ALLOWED = (1 << 2), MLX5DV_CONTEXT_FLAGS_ENHANCED_MPW = (1 << 3), MLX5DV_CONTEXT_FLAGS_CQE_128B_COMP = (1 << 4), /* Support CQE 128B compression */ MLX5DV_CONTEXT_FLAGS_CQE_128B_PAD = (1 << 5), /* Support CQE 128B padding */ MLX5DV_CONTEXT_FLAGS_PACKET_BASED_CREDIT_MODE = (1 << 6), }; enum mlx5dv_cq_init_attr_mask { MLX5DV_CQ_INIT_ATTR_MASK_COMPRESSED_CQE = 1 << 0, MLX5DV_CQ_INIT_ATTR_MASK_FLAGS = 1 << 1, MLX5DV_CQ_INIT_ATTR_MASK_CQE_SIZE = 1 << 2, }; enum mlx5dv_cq_init_attr_flags { MLX5DV_CQ_INIT_ATTR_FLAGS_CQE_PAD = 1 << 0, MLX5DV_CQ_INIT_ATTR_FLAGS_RESERVED = 1 << 1, }; struct mlx5dv_cq_init_attr { uint64_t comp_mask; /* Use enum mlx5dv_cq_init_attr_mask */ uint8_t cqe_comp_res_format; /* Use enum mlx5dv_cqe_comp_res_format */ uint32_t flags; /* Use enum mlx5dv_cq_init_attr_flags */ uint16_t cqe_size; /* when MLX5DV_CQ_INIT_ATTR_MASK_CQE_SIZE set */ }; struct ibv_cq_ex *mlx5dv_create_cq(struct ibv_context *context, struct ibv_cq_init_attr_ex *cq_attr, struct mlx5dv_cq_init_attr *mlx5_cq_attr); enum mlx5dv_qp_create_flags { MLX5DV_QP_CREATE_TUNNEL_OFFLOADS = 1 << 0, MLX5DV_QP_CREATE_TIR_ALLOW_SELF_LOOPBACK_UC = 1 << 1, MLX5DV_QP_CREATE_TIR_ALLOW_SELF_LOOPBACK_MC = 1 << 2, MLX5DV_QP_CREATE_DISABLE_SCATTER_TO_CQE = 1 << 3, MLX5DV_QP_CREATE_ALLOW_SCATTER_TO_CQE = 1 << 4, MLX5DV_QP_CREATE_PACKET_BASED_CREDIT_MODE = 1 << 5, }; enum mlx5dv_mkey_init_attr_flags { MLX5DV_MKEY_INIT_ATTR_FLAGS_INDIRECT = 1 << 0, }; struct mlx5dv_mkey_init_attr { struct ibv_pd *pd; uint32_t create_flags; /* Use enum mlx5dv_mkey_init_attr_flags */ uint16_t max_entries; /* Requested max number of pointed entries by this indirect mkey */ }; struct mlx5dv_mkey { uint32_t lkey; uint32_t rkey; }; struct mlx5dv_mkey *mlx5dv_create_mkey(struct mlx5dv_mkey_init_attr *mkey_init_attr); int mlx5dv_destroy_mkey(struct mlx5dv_mkey *mkey); enum mlx5dv_qp_init_attr_mask { MLX5DV_QP_INIT_ATTR_MASK_QP_CREATE_FLAGS = 1 << 0, MLX5DV_QP_INIT_ATTR_MASK_DC = 1 << 1, MLX5DV_QP_INIT_ATTR_MASK_SEND_OPS_FLAGS = 1 << 2, }; enum mlx5dv_dc_type { MLX5DV_DCTYPE_DCT = 1, MLX5DV_DCTYPE_DCI, }; struct mlx5dv_dc_init_attr { enum mlx5dv_dc_type dc_type; uint64_t dct_access_key; }; enum mlx5dv_qp_create_send_ops_flags { MLX5DV_QP_EX_WITH_MR_INTERLEAVED = 1 << 0, MLX5DV_QP_EX_WITH_MR_LIST = 1 << 1, }; struct mlx5dv_qp_init_attr { uint64_t comp_mask; /* Use enum mlx5dv_qp_init_attr_mask */ uint32_t create_flags; /* Use enum mlx5dv_qp_create_flags */ struct mlx5dv_dc_init_attr dc_init_attr; uint64_t send_ops_flags; /* Use enum mlx5dv_qp_create_send_ops_flags */ }; struct ibv_qp *mlx5dv_create_qp(struct ibv_context *context, struct ibv_qp_init_attr_ex *qp_attr, struct mlx5dv_qp_init_attr *mlx5_qp_attr); struct mlx5dv_mr_interleaved { uint64_t addr; uint32_t bytes_count; uint32_t bytes_skip; uint32_t lkey; }; enum mlx5dv_wc_opcode { MLX5DV_WC_UMR = IBV_WC_DRIVER1, }; struct mlx5dv_qp_ex { uint64_t comp_mask; /* * Available just for the MLX5 DC QP type with send opcodes of type: * rdma, atomic and send. */ void (*wr_set_dc_addr)(struct mlx5dv_qp_ex *mqp, struct ibv_ah *ah, uint32_t remote_dctn, uint64_t remote_dc_key); void (*wr_mr_interleaved)(struct mlx5dv_qp_ex *mqp, struct mlx5dv_mkey *mkey, uint32_t access_flags, /* use enum ibv_access_flags */ uint32_t repeat_count, uint16_t num_interleaved, struct mlx5dv_mr_interleaved *data); void (*wr_mr_list)(struct mlx5dv_qp_ex *mqp, struct mlx5dv_mkey *mkey, uint32_t access_flags, /* use enum ibv_access_flags */ uint16_t num_sges, struct ibv_sge *sge); }; struct mlx5dv_qp_ex *mlx5dv_qp_ex_from_ibv_qp_ex(struct ibv_qp_ex *qp); int mlx5dv_query_devx_port(struct ibv_context *ctx, uint32_t port_num, struct mlx5dv_devx_port *mlx5_devx_port); static inline void mlx5dv_wr_set_dc_addr(struct mlx5dv_qp_ex *mqp, struct ibv_ah *ah, uint32_t remote_dctn, uint64_t remote_dc_key) { mqp->wr_set_dc_addr(mqp, ah, remote_dctn, remote_dc_key); } static inline void mlx5dv_wr_mr_interleaved(struct mlx5dv_qp_ex *mqp, struct mlx5dv_mkey *mkey, uint32_t access_flags, uint32_t repeat_count, uint16_t num_interleaved, struct mlx5dv_mr_interleaved *data) { mqp->wr_mr_interleaved(mqp, mkey, access_flags, repeat_count, num_interleaved, data); } static inline void mlx5dv_wr_mr_list(struct mlx5dv_qp_ex *mqp, struct mlx5dv_mkey *mkey, uint32_t access_flags, uint16_t num_sges, struct ibv_sge *sge) { mqp->wr_mr_list(mqp, mkey, access_flags, num_sges, sge); } enum mlx5dv_flow_action_esp_mask { MLX5DV_FLOW_ACTION_ESP_MASK_FLAGS = 1 << 0, }; struct mlx5dv_flow_action_esp { uint64_t comp_mask; /* Use enum mlx5dv_flow_action_esp_mask */ uint32_t action_flags; /* Use enum mlx5dv_flow_action_flags */ }; struct mlx5dv_flow_match_parameters { size_t match_sz; uint64_t match_buf[]; /* Device spec format */ }; enum mlx5dv_flow_matcher_attr_mask { MLX5DV_FLOW_MATCHER_MASK_FT_TYPE = 1 << 0, }; struct mlx5dv_flow_matcher_attr { enum ibv_flow_attr_type type; uint32_t flags; /* From enum ibv_flow_flags */ uint16_t priority; uint8_t match_criteria_enable; /* Device spec format */ struct mlx5dv_flow_match_parameters *match_mask; uint64_t comp_mask; /* use mlx5dv_flow_matcher_attr_mask */ enum mlx5dv_flow_table_type ft_type; }; struct mlx5dv_flow_matcher; struct mlx5dv_flow_matcher * mlx5dv_create_flow_matcher(struct ibv_context *context, struct mlx5dv_flow_matcher_attr *matcher_attr); int mlx5dv_destroy_flow_matcher(struct mlx5dv_flow_matcher *matcher); enum mlx5dv_flow_action_type { MLX5DV_FLOW_ACTION_DEST_IBV_QP, MLX5DV_FLOW_ACTION_DROP, MLX5DV_FLOW_ACTION_IBV_COUNTER, MLX5DV_FLOW_ACTION_IBV_FLOW_ACTION, MLX5DV_FLOW_ACTION_TAG, MLX5DV_FLOW_ACTION_DEST_DEVX, MLX5DV_FLOW_ACTION_COUNTERS_DEVX, MLX5DV_FLOW_ACTION_DEFAULT_MISS, }; struct mlx5dv_flow_action_attr { enum mlx5dv_flow_action_type type; union { struct ibv_qp *qp; struct ibv_counters *counter; struct ibv_flow_action *action; uint32_t tag_value; struct mlx5dv_devx_obj *obj; }; }; struct ibv_flow * mlx5dv_create_flow(struct mlx5dv_flow_matcher *matcher, struct mlx5dv_flow_match_parameters *match_value, size_t num_actions, struct mlx5dv_flow_action_attr actions_attr[]); struct ibv_flow_action *mlx5dv_create_flow_action_esp(struct ibv_context *ctx, struct ibv_flow_action_esp_attr *esp, struct mlx5dv_flow_action_esp *mlx5_attr); /* * mlx5dv_create_flow_action_modify_header - Create a flow action which mutates * a packet. The flow action can be attached to steering rules via * ibv_create_flow(). * * @ctx: RDMA device context to create the action on. * @actions_sz: The size of *actions* buffer in bytes. * @actions: A buffer which contains modify actions provided in device spec * format. * @ft_type: Defines the flow table type to which the modify * header action will be attached. * * Return a valid ibv_flow_action if successful, NULL otherwise. */ struct ibv_flow_action * mlx5dv_create_flow_action_modify_header(struct ibv_context *ctx, size_t actions_sz, uint64_t actions[], enum mlx5dv_flow_table_type ft_type); /* * mlx5dv_create_flow_action_packet_reformat - Create flow action which can * encap/decap packets. */ struct ibv_flow_action * mlx5dv_create_flow_action_packet_reformat(struct ibv_context *ctx, size_t data_sz, void *data, enum mlx5dv_flow_action_packet_reformat_type reformat_type, enum mlx5dv_flow_table_type ft_type); /* * Most device capabilities are exported by ibv_query_device(...), * but there is HW device-specific information which is important * for data-path, but isn't provided. * * Return 0 on success. */ int mlx5dv_query_device(struct ibv_context *ctx_in, struct mlx5dv_context *attrs_out); enum mlx5dv_qp_comp_mask { MLX5DV_QP_MASK_UAR_MMAP_OFFSET = 1 << 0, MLX5DV_QP_MASK_RAW_QP_HANDLES = 1 << 1, MLX5DV_QP_MASK_RAW_QP_TIR_ADDR = 1 << 2, }; struct mlx5dv_qp { __be32 *dbrec; struct { void *buf; uint32_t wqe_cnt; uint32_t stride; } sq; struct { void *buf; uint32_t wqe_cnt; uint32_t stride; } rq; struct { void *reg; uint32_t size; } bf; uint64_t comp_mask; off_t uar_mmap_offset; uint32_t tirn; uint32_t tisn; uint32_t rqn; uint32_t sqn; uint64_t tir_icm_addr; }; struct mlx5dv_cq { void *buf; __be32 *dbrec; uint32_t cqe_cnt; uint32_t cqe_size; void *cq_uar; uint32_t cqn; uint64_t comp_mask; }; enum mlx5dv_srq_comp_mask { MLX5DV_SRQ_MASK_SRQN = 1 << 0, }; struct mlx5dv_srq { void *buf; __be32 *dbrec; uint32_t stride; uint32_t head; uint32_t tail; uint64_t comp_mask; uint32_t srqn; }; struct mlx5dv_rwq { void *buf; __be32 *dbrec; uint32_t wqe_cnt; uint32_t stride; uint64_t comp_mask; }; struct mlx5dv_alloc_dm_attr { enum mlx5dv_alloc_dm_type type; uint64_t comp_mask; }; enum mlx5dv_dm_comp_mask { MLX5DV_DM_MASK_REMOTE_VA = 1 << 0, }; struct mlx5dv_dm { void *buf; uint64_t length; uint64_t comp_mask; uint64_t remote_va; }; struct ibv_dm *mlx5dv_alloc_dm(struct ibv_context *context, struct ibv_alloc_dm_attr *dm_attr, struct mlx5dv_alloc_dm_attr *mlx5_dm_attr); struct mlx5_wqe_av; struct mlx5dv_ah { struct mlx5_wqe_av *av; uint64_t comp_mask; }; struct mlx5dv_pd { uint32_t pdn; uint64_t comp_mask; }; struct mlx5dv_obj { struct { struct ibv_qp *in; struct mlx5dv_qp *out; } qp; struct { struct ibv_cq *in; struct mlx5dv_cq *out; } cq; struct { struct ibv_srq *in; struct mlx5dv_srq *out; } srq; struct { struct ibv_wq *in; struct mlx5dv_rwq *out; } rwq; struct { struct ibv_dm *in; struct mlx5dv_dm *out; } dm; struct { struct ibv_ah *in; struct mlx5dv_ah *out; } ah; struct { struct ibv_pd *in; struct mlx5dv_pd *out; } pd; }; enum mlx5dv_obj_type { MLX5DV_OBJ_QP = 1 << 0, MLX5DV_OBJ_CQ = 1 << 1, MLX5DV_OBJ_SRQ = 1 << 2, MLX5DV_OBJ_RWQ = 1 << 3, MLX5DV_OBJ_DM = 1 << 4, MLX5DV_OBJ_AH = 1 << 5, MLX5DV_OBJ_PD = 1 << 6, }; enum mlx5dv_wq_init_attr_mask { MLX5DV_WQ_INIT_ATTR_MASK_STRIDING_RQ = 1 << 0, }; struct mlx5dv_striding_rq_init_attr { uint32_t single_stride_log_num_of_bytes; uint32_t single_wqe_log_num_of_strides; uint8_t two_byte_shift_en; }; struct mlx5dv_wq_init_attr { uint64_t comp_mask; /* Use enum mlx5dv_wq_init_attr_mask */ struct mlx5dv_striding_rq_init_attr striding_rq_attrs; }; /* * This function creates a work queue object with extra properties * defined by mlx5dv_wq_init_attr struct. * * For each bit in the comp_mask, a field in mlx5dv_wq_init_attr * should follow. * * MLX5DV_WQ_INIT_ATTR_MASK_STRIDING_RQ: Create a work queue with * striding RQ capabilities. * - single_stride_log_num_of_bytes represents the size of each stride in the * WQE and its value should be between min_single_stride_log_num_of_bytes * and max_single_stride_log_num_of_bytes that are reported in * mlx5dv_query_device. * - single_wqe_log_num_of_strides represents the number of strides in each WQE. * Its value should be between min_single_wqe_log_num_of_strides and * max_single_wqe_log_num_of_strides that are reported in mlx5dv_query_device. * - two_byte_shift_en: When enabled, hardware pads 2 bytes of zeroes * before writing the message to memory (e.g. for IP alignment) */ struct ibv_wq *mlx5dv_create_wq(struct ibv_context *context, struct ibv_wq_init_attr *wq_init_attr, struct mlx5dv_wq_init_attr *mlx5_wq_attr); /* * This function will initialize mlx5dv_xxx structs based on supplied type. * The information for initialization is taken from ibv_xx structs supplied * as part of input. * * Request information of CQ marks its owned by DV for all consumer index * related actions. * * The initialization type can be combination of several types together. * * Return: 0 in case of success. */ int mlx5dv_init_obj(struct mlx5dv_obj *obj, uint64_t obj_type); enum { MLX5_OPCODE_NOP = 0x00, MLX5_OPCODE_SEND_INVAL = 0x01, MLX5_OPCODE_RDMA_WRITE = 0x08, MLX5_OPCODE_RDMA_WRITE_IMM = 0x09, MLX5_OPCODE_SEND = 0x0a, MLX5_OPCODE_SEND_IMM = 0x0b, MLX5_OPCODE_TSO = 0x0e, MLX5_OPCODE_RDMA_READ = 0x10, MLX5_OPCODE_ATOMIC_CS = 0x11, MLX5_OPCODE_ATOMIC_FA = 0x12, MLX5_OPCODE_ATOMIC_MASKED_CS = 0x14, MLX5_OPCODE_ATOMIC_MASKED_FA = 0x15, MLX5_OPCODE_FMR = 0x19, MLX5_OPCODE_LOCAL_INVAL = 0x1b, MLX5_OPCODE_CONFIG_CMD = 0x1f, MLX5_OPCODE_UMR = 0x25, MLX5_OPCODE_TAG_MATCHING = 0x28, MLX5_OPCODE_FLOW_TBL_ACCESS = 0x2c, }; /* * CQE related part */ enum { MLX5_INLINE_SCATTER_32 = 0x4, MLX5_INLINE_SCATTER_64 = 0x8, }; enum { MLX5_CQE_SYNDROME_LOCAL_LENGTH_ERR = 0x01, MLX5_CQE_SYNDROME_LOCAL_QP_OP_ERR = 0x02, MLX5_CQE_SYNDROME_LOCAL_PROT_ERR = 0x04, MLX5_CQE_SYNDROME_WR_FLUSH_ERR = 0x05, MLX5_CQE_SYNDROME_MW_BIND_ERR = 0x06, MLX5_CQE_SYNDROME_BAD_RESP_ERR = 0x10, MLX5_CQE_SYNDROME_LOCAL_ACCESS_ERR = 0x11, MLX5_CQE_SYNDROME_REMOTE_INVAL_REQ_ERR = 0x12, MLX5_CQE_SYNDROME_REMOTE_ACCESS_ERR = 0x13, MLX5_CQE_SYNDROME_REMOTE_OP_ERR = 0x14, MLX5_CQE_SYNDROME_TRANSPORT_RETRY_EXC_ERR = 0x15, MLX5_CQE_SYNDROME_RNR_RETRY_EXC_ERR = 0x16, MLX5_CQE_SYNDROME_REMOTE_ABORTED_ERR = 0x22, }; enum { MLX5_CQE_VENDOR_SYNDROME_ODP_PFAULT = 0x93, }; enum { MLX5_CQE_L2_OK = 1 << 0, MLX5_CQE_L3_OK = 1 << 1, MLX5_CQE_L4_OK = 1 << 2, }; enum { MLX5_CQE_L3_HDR_TYPE_NONE = 0x0, MLX5_CQE_L3_HDR_TYPE_IPV6 = 0x1, MLX5_CQE_L3_HDR_TYPE_IPV4 = 0x2, }; enum { MLX5_CQE_OWNER_MASK = 1, MLX5_CQE_REQ = 0, MLX5_CQE_RESP_WR_IMM = 1, MLX5_CQE_RESP_SEND = 2, MLX5_CQE_RESP_SEND_IMM = 3, MLX5_CQE_RESP_SEND_INV = 4, MLX5_CQE_RESIZE_CQ = 5, MLX5_CQE_NO_PACKET = 6, MLX5_CQE_REQ_ERR = 13, MLX5_CQE_RESP_ERR = 14, MLX5_CQE_INVALID = 15, }; enum { MLX5_CQ_DOORBELL = 0x20 }; enum { MLX5_CQ_DB_REQ_NOT_SOL = 1 << 24, MLX5_CQ_DB_REQ_NOT = 0 << 24, }; struct mlx5_err_cqe { uint8_t rsvd0[32]; uint32_t srqn; uint8_t rsvd1[18]; uint8_t vendor_err_synd; uint8_t syndrome; uint32_t s_wqe_opcode_qpn; uint16_t wqe_counter; uint8_t signature; uint8_t op_own; }; struct mlx5_tm_cqe { __be32 success; __be16 hw_phase_cnt; uint8_t rsvd0[12]; }; struct mlx5_cqe64 { union { struct { uint8_t rsvd0[2]; __be16 wqe_id; uint8_t rsvd4[13]; uint8_t ml_path; uint8_t rsvd20[4]; __be16 slid; __be32 flags_rqpn; uint8_t hds_ip_ext; uint8_t l4_hdr_type_etc; __be16 vlan_info; }; struct mlx5_tm_cqe tm_cqe; /* TMH is scattered to CQE upon match */ struct ibv_tmh tmh; }; __be32 srqn_uidx; __be32 imm_inval_pkey; uint8_t app; uint8_t app_op; __be16 app_info; __be32 byte_cnt; __be64 timestamp; __be32 sop_drop_qpn; __be16 wqe_counter; uint8_t signature; uint8_t op_own; }; enum { MLX5_TMC_SUCCESS = 0x80000000U, }; enum mlx5dv_cqe_comp_res_format { MLX5DV_CQE_RES_FORMAT_HASH = 1 << 0, MLX5DV_CQE_RES_FORMAT_CSUM = 1 << 1, MLX5DV_CQE_RES_FORMAT_CSUM_STRIDX = 1 << 2, }; enum mlx5dv_sw_parsing_offloads { MLX5DV_SW_PARSING = 1 << 0, MLX5DV_SW_PARSING_CSUM = 1 << 1, MLX5DV_SW_PARSING_LSO = 1 << 2, }; static MLX5DV_ALWAYS_INLINE uint8_t mlx5dv_get_cqe_owner(struct mlx5_cqe64 *cqe) { return cqe->op_own & 0x1; } static MLX5DV_ALWAYS_INLINE void mlx5dv_set_cqe_owner(struct mlx5_cqe64 *cqe, uint8_t val) { cqe->op_own = (val & 0x1) | (cqe->op_own & ~0x1); } /* Solicited event */ static MLX5DV_ALWAYS_INLINE uint8_t mlx5dv_get_cqe_se(struct mlx5_cqe64 *cqe) { return (cqe->op_own >> 1) & 0x1; } static MLX5DV_ALWAYS_INLINE uint8_t mlx5dv_get_cqe_format(struct mlx5_cqe64 *cqe) { return (cqe->op_own >> 2) & 0x3; } static MLX5DV_ALWAYS_INLINE uint8_t mlx5dv_get_cqe_opcode(struct mlx5_cqe64 *cqe) { return cqe->op_own >> 4; } /* * WQE related part */ enum { MLX5_INVALID_LKEY = 0x100, }; enum { MLX5_EXTENDED_UD_AV = 0x80000000, }; enum { MLX5_WQE_CTRL_CQ_UPDATE = 2 << 2, MLX5_WQE_CTRL_SOLICITED = 1 << 1, MLX5_WQE_CTRL_FENCE = 4 << 5, MLX5_WQE_CTRL_INITIATOR_SMALL_FENCE = 1 << 5, }; enum { MLX5_SEND_WQE_BB = 64, MLX5_SEND_WQE_SHIFT = 6, }; enum { MLX5_INLINE_SEG = 0x80000000, }; enum { MLX5_ETH_WQE_L3_CSUM = (1 << 6), MLX5_ETH_WQE_L4_CSUM = (1 << 7), }; struct mlx5_wqe_srq_next_seg { uint8_t rsvd0[2]; __be16 next_wqe_index; uint8_t signature; uint8_t rsvd1[11]; }; struct mlx5_wqe_data_seg { __be32 byte_count; __be32 lkey; __be64 addr; }; struct mlx5_wqe_ctrl_seg { __be32 opmod_idx_opcode; __be32 qpn_ds; uint8_t signature; uint8_t rsvd[2]; uint8_t fm_ce_se; __be32 imm; }; struct mlx5_wqe_flow_update_ctrl_seg { __be32 flow_idx_update; __be32 dest_handle; uint8_t reserved0[40]; }; struct mlx5_wqe_header_modify_argument_update_seg { uint8_t argument_list[64]; }; struct mlx5_mprq_wqe { struct mlx5_wqe_srq_next_seg nseg; struct mlx5_wqe_data_seg dseg; }; struct mlx5_wqe_av { union { struct { __be32 qkey; __be32 reserved; } qkey; __be64 dc_key; } key; __be32 dqp_dct; uint8_t stat_rate_sl; uint8_t fl_mlid; __be16 rlid; uint8_t reserved0[4]; uint8_t rmac[6]; uint8_t tclass; uint8_t hop_limit; __be32 grh_gid_fl; uint8_t rgid[16]; }; struct mlx5_wqe_datagram_seg { struct mlx5_wqe_av av; }; struct mlx5_wqe_raddr_seg { __be64 raddr; __be32 rkey; __be32 reserved; }; struct mlx5_wqe_atomic_seg { __be64 swap_add; __be64 compare; }; struct mlx5_wqe_inl_data_seg { uint32_t byte_count; }; struct mlx5_wqe_eth_seg { __be32 rsvd0; uint8_t cs_flags; uint8_t rsvd1; __be16 mss; __be32 rsvd2; __be16 inline_hdr_sz; uint8_t inline_hdr_start[2]; uint8_t inline_hdr[16]; }; struct mlx5_wqe_tm_seg { uint8_t opcode; uint8_t flags; __be16 index; uint8_t rsvd0[2]; __be16 sw_cnt; uint8_t rsvd1[8]; __be64 append_tag; __be64 append_mask; }; enum { MLX5_WQE_UMR_CTRL_FLAG_INLINE = 1 << 7, MLX5_WQE_UMR_CTRL_FLAG_CHECK_FREE = 1 << 5, MLX5_WQE_UMR_CTRL_FLAG_TRNSLATION_OFFSET = 1 << 4, MLX5_WQE_UMR_CTRL_FLAG_CHECK_QPN = 1 << 3, }; enum { MLX5_WQE_UMR_CTRL_MKEY_MASK_LEN = 1 << 0, MLX5_WQE_UMR_CTRL_MKEY_MASK_START_ADDR = 1 << 6, MLX5_WQE_UMR_CTRL_MKEY_MASK_MKEY = 1 << 13, MLX5_WQE_UMR_CTRL_MKEY_MASK_QPN = 1 << 14, MLX5_WQE_UMR_CTRL_MKEY_MASK_ACCESS_LOCAL_WRITE = 1 << 18, MLX5_WQE_UMR_CTRL_MKEY_MASK_ACCESS_REMOTE_READ = 1 << 19, MLX5_WQE_UMR_CTRL_MKEY_MASK_ACCESS_REMOTE_WRITE = 1 << 20, MLX5_WQE_UMR_CTRL_MKEY_MASK_ACCESS_ATOMIC = 1 << 21, MLX5_WQE_UMR_CTRL_MKEY_MASK_FREE = 1 << 29, }; struct mlx5_wqe_umr_ctrl_seg { uint8_t flags; uint8_t rsvd0[3]; __be16 klm_octowords; __be16 translation_offset; __be64 mkey_mask; uint8_t rsvd1[32]; }; struct mlx5_wqe_umr_klm_seg { /* up to 2GB */ __be32 byte_count; __be32 mkey; __be64 address; }; union mlx5_wqe_umr_inline_seg { struct mlx5_wqe_umr_klm_seg klm; }; struct mlx5_wqe_umr_repeat_ent_seg { __be16 stride; __be16 byte_count; __be32 memkey; __be64 va; }; struct mlx5_wqe_umr_repeat_block_seg { __be32 byte_count; __be32 op; __be32 repeat_count; __be16 reserved; __be16 num_ent; struct mlx5_wqe_umr_repeat_ent_seg entries[0]; }; enum { MLX5_WQE_MKEY_CONTEXT_FREE = 1 << 6 }; enum { MLX5_WQE_MKEY_CONTEXT_ACCESS_FLAGS_ATOMIC = 1 << 6, MLX5_WQE_MKEY_CONTEXT_ACCESS_FLAGS_REMOTE_WRITE = 1 << 5, MLX5_WQE_MKEY_CONTEXT_ACCESS_FLAGS_REMOTE_READ = 1 << 4, MLX5_WQE_MKEY_CONTEXT_ACCESS_FLAGS_LOCAL_WRITE = 1 << 3, MLX5_WQE_MKEY_CONTEXT_ACCESS_FLAGS_LOCAL_READ = 1 << 2 }; struct mlx5_wqe_mkey_context_seg { uint8_t free; uint8_t reserved1; uint8_t access_flags; uint8_t sf; __be32 qpn_mkey; __be32 reserved2; __be32 flags_pd; __be64 start_addr; __be64 len; __be32 bsf_octword_size; __be32 reserved3[4]; __be32 translations_octword_size; uint8_t reserved4[3]; uint8_t log_page_size; __be32 reserved; union mlx5_wqe_umr_inline_seg inseg[0]; }; /* * Control segment - contains some control information for the current WQE. * * Output: * seg - control segment to be filled * Input: * pi - WQEBB number of the first block of this WQE. * This number should wrap at 0xffff, regardless of * size of the WQ. * opcode - Opcode of this WQE. Encodes the type of operation * to be executed on the QP. * opmod - Opcode modifier. * qp_num - QP/SQ number this WQE is posted to. * fm_ce_se - FM (fence mode), CE (completion and event mode) * and SE (solicited event). * ds - WQE size in octowords (16-byte units). DS accounts for all * the segments in the WQE as summarized in WQE construction. * signature - WQE signature. * imm - Immediate data/Invalidation key/UMR mkey. */ static MLX5DV_ALWAYS_INLINE void mlx5dv_set_ctrl_seg(struct mlx5_wqe_ctrl_seg *seg, uint16_t pi, uint8_t opcode, uint8_t opmod, uint32_t qp_num, uint8_t fm_ce_se, uint8_t ds, uint8_t signature, uint32_t imm) { seg->opmod_idx_opcode = htobe32(((uint32_t)opmod << 24) | ((uint32_t)pi << 8) | opcode); seg->qpn_ds = htobe32((qp_num << 8) | ds); seg->fm_ce_se = fm_ce_se; seg->signature = signature; /* * The caller should prepare "imm" in advance based on WR opcode. * For IBV_WR_SEND_WITH_IMM and IBV_WR_RDMA_WRITE_WITH_IMM, * the "imm" should be assigned as is. * For the IBV_WR_SEND_WITH_INV, it should be htobe32(imm). */ seg->imm = imm; } /* x86 optimized version of mlx5dv_set_ctrl_seg() * * This is useful when doing calculations on large data sets * for parallel calculations. * * It doesn't suit for serialized algorithms. */ #if defined(__SSE3__) static MLX5DV_ALWAYS_INLINE void mlx5dv_x86_set_ctrl_seg(struct mlx5_wqe_ctrl_seg *seg, uint16_t pi, uint8_t opcode, uint8_t opmod, uint32_t qp_num, uint8_t fm_ce_se, uint8_t ds, uint8_t signature, uint32_t imm) { __m128i val = _mm_set_epi32(imm, qp_num, (ds << 16) | pi, (signature << 24) | (opcode << 16) | (opmod << 8) | fm_ce_se); __m128i mask = _mm_set_epi8(15, 14, 13, 12, /* immediate */ 0, /* signal/fence_mode */ #if CHAR_MIN -128, -128, /* reserved */ #else 0x80, 0x80, /* reserved */ #endif 3, /* signature */ 6, /* data size */ 8, 9, 10, /* QP num */ 2, /* opcode */ 4, 5, /* sw_pi in BE */ 1 /* opmod */ ); *(__m128i *) seg = _mm_shuffle_epi8(val, mask); } #endif /* defined(__SSE3__) */ /* * Datagram Segment - contains address information required in order * to form a datagram message. * * Output: * seg - datagram segment to be filled. * Input: * key - Q_key/access key. * dqp_dct - Destination QP number for UD and DCT for DC. * ext - Address vector extension. * stat_rate_sl - Maximum static rate control, SL/ethernet priority. * fl_mlid - Force loopback and source LID for IB. * rlid - Remote LID * rmac - Remote MAC * tclass - GRH tclass/IPv6 tclass/IPv4 ToS * hop_limit - GRH hop limit/IPv6 hop limit/IPv4 TTL * grh_gid_fi - GRH, source GID address and IPv6 flow label. * rgid - Remote GID/IP address. */ static MLX5DV_ALWAYS_INLINE void mlx5dv_set_dgram_seg(struct mlx5_wqe_datagram_seg *seg, uint64_t key, uint32_t dqp_dct, uint8_t ext, uint8_t stat_rate_sl, uint8_t fl_mlid, uint16_t rlid, uint8_t *rmac, uint8_t tclass, uint8_t hop_limit, uint32_t grh_gid_fi, uint8_t *rgid) { /* Always put 64 bits, in q_key, the reserved part will be 0 */ seg->av.key.dc_key = htobe64(key); seg->av.dqp_dct = htobe32(((uint32_t)ext << 31) | dqp_dct); seg->av.stat_rate_sl = stat_rate_sl; seg->av.fl_mlid = fl_mlid; seg->av.rlid = htobe16(rlid); memcpy(seg->av.rmac, rmac, 6); seg->av.tclass = tclass; seg->av.hop_limit = hop_limit; seg->av.grh_gid_fl = htobe32(grh_gid_fi); memcpy(seg->av.rgid, rgid, 16); } /* * Data Segments - contain pointers and a byte count for the scatter/gather list. * They can optionally contain data, which will save a memory read access for * gather Work Requests. */ static MLX5DV_ALWAYS_INLINE void mlx5dv_set_data_seg(struct mlx5_wqe_data_seg *seg, uint32_t length, uint32_t lkey, uintptr_t address) { seg->byte_count = htobe32(length); seg->lkey = htobe32(lkey); seg->addr = htobe64(address); } /* * x86 optimized version of mlx5dv_set_data_seg() * * This is useful when doing calculations on large data sets * for parallel calculations. * * It doesn't suit for serialized algorithms. */ #if defined(__SSE3__) static MLX5DV_ALWAYS_INLINE void mlx5dv_x86_set_data_seg(struct mlx5_wqe_data_seg *seg, uint32_t length, uint32_t lkey, uintptr_t address) { uint64_t address64 = address; __m128i val = _mm_set_epi32((uint32_t)address64, (uint32_t)(address64 >> 32), lkey, length); __m128i mask = _mm_set_epi8(12, 13, 14, 15, /* local address low */ 8, 9, 10, 11, /* local address high */ 4, 5, 6, 7, /* l_key */ 0, 1, 2, 3 /* byte count */ ); *(__m128i *) seg = _mm_shuffle_epi8(val, mask); } #endif /* defined(__SSE3__) */ /* * Eth Segment - contains packet headers and information for stateless L2, L3, L4 offloading. * * Output: * seg - Eth segment to be filled. * Input: * cs_flags - l3cs/l3cs_inner/l4cs/l4cs_inner. * mss - Maximum segment size. For TSO WQEs, the number of bytes * in the TCP payload to be transmitted in each packet. Must * be 0 on non TSO WQEs. * inline_hdr_sz - Length of the inlined packet headers. * inline_hdr_start - Inlined packet header. */ static MLX5DV_ALWAYS_INLINE void mlx5dv_set_eth_seg(struct mlx5_wqe_eth_seg *seg, uint8_t cs_flags, uint16_t mss, uint16_t inline_hdr_sz, uint8_t *inline_hdr_start) { seg->cs_flags = cs_flags; seg->mss = htobe16(mss); seg->inline_hdr_sz = htobe16(inline_hdr_sz); memcpy(seg->inline_hdr_start, inline_hdr_start, inline_hdr_sz); } enum mlx5dv_set_ctx_attr_type { MLX5DV_CTX_ATTR_BUF_ALLOCATORS = 1, }; enum { MLX5_MMAP_GET_REGULAR_PAGES_CMD = 0, MLX5_MMAP_GET_NC_PAGES_CMD = 3, }; struct mlx5dv_ctx_allocators { void *(*alloc)(size_t size, void *priv_data); void (*free)(void *ptr, void *priv_data); void *data; }; /* * Generic context attributes set API * * Returns 0 on success, or the value of errno on failure * (which indicates the failure reason). */ int mlx5dv_set_context_attr(struct ibv_context *context, enum mlx5dv_set_ctx_attr_type type, void *attr); struct mlx5dv_clock_info { uint64_t nsec; uint64_t last_cycles; uint64_t frac; uint32_t mult; uint32_t shift; uint64_t mask; }; /* * Get mlx5 core clock info * * Output: * clock_info - clock info to be filled * Input: * context - device context * * Return: 0 on success, or the value of errno on failure */ int mlx5dv_get_clock_info(struct ibv_context *context, struct mlx5dv_clock_info *clock_info); /* * Translate device timestamp to nano-sec * * Input: * clock_info - clock info to be filled * device_timestamp - timestamp to translate * * Return: nano-sec */ static inline uint64_t mlx5dv_ts_to_ns(struct mlx5dv_clock_info *clock_info, uint64_t device_timestamp) { uint64_t delta, nsec; /* * device_timestamp & cycles are the free running 'mask' bit counters * from the hardware hca_core_clock clock. */ delta = (device_timestamp - clock_info->last_cycles) & clock_info->mask; nsec = clock_info->nsec; /* * Guess if the device_timestamp is more recent than * clock_info->last_cycles, if not (too far in the future) treat * it as old time stamp. This will break every max_clock_info_update_nsec. */ if (delta > clock_info->mask / 2) { delta = (clock_info->last_cycles - device_timestamp) & clock_info->mask; nsec -= ((delta * clock_info->mult) - clock_info->frac) >> clock_info->shift; } else { nsec += ((delta * clock_info->mult) + clock_info->frac) >> clock_info->shift; } return nsec; } enum mlx5dv_context_attr_flags { MLX5DV_CONTEXT_FLAGS_DEVX = 1 << 0, }; struct mlx5dv_context_attr { uint32_t flags; /* Use enum mlx5dv_context_attr_flags */ uint64_t comp_mask; }; bool mlx5dv_is_supported(struct ibv_device *device); struct ibv_context * mlx5dv_open_device(struct ibv_device *device, struct mlx5dv_context_attr *attr); struct mlx5dv_devx_obj; struct mlx5dv_devx_obj * mlx5dv_devx_obj_create(struct ibv_context *context, const void *in, size_t inlen, void *out, size_t outlen); int mlx5dv_devx_obj_query(struct mlx5dv_devx_obj *obj, const void *in, size_t inlen, void *out, size_t outlen); int mlx5dv_devx_obj_modify(struct mlx5dv_devx_obj *obj, const void *in, size_t inlen, void *out, size_t outlen); int mlx5dv_devx_obj_destroy(struct mlx5dv_devx_obj *obj); int mlx5dv_devx_general_cmd(struct ibv_context *context, const void *in, size_t inlen, void *out, size_t outlen); struct mlx5dv_devx_umem { uint32_t umem_id; }; struct mlx5dv_devx_umem * mlx5dv_devx_umem_reg(struct ibv_context *ctx, void *addr, size_t size, uint32_t access); int mlx5dv_devx_umem_dereg(struct mlx5dv_devx_umem *umem); struct mlx5dv_devx_uar { void *reg_addr; void *base_addr; uint32_t page_id; off_t mmap_off; uint64_t comp_mask; }; struct mlx5dv_devx_uar *mlx5dv_devx_alloc_uar(struct ibv_context *context, uint32_t flags); void mlx5dv_devx_free_uar(struct mlx5dv_devx_uar *devx_uar); struct mlx5dv_var { uint32_t page_id; uint32_t length; off_t mmap_off; uint64_t comp_mask; }; struct mlx5dv_var * mlx5dv_alloc_var(struct ibv_context *context, uint32_t flags); void mlx5dv_free_var(struct mlx5dv_var *dv_var); int mlx5dv_devx_query_eqn(struct ibv_context *context, uint32_t vector, uint32_t *eqn); int mlx5dv_devx_cq_query(struct ibv_cq *cq, const void *in, size_t inlen, void *out, size_t outlen); int mlx5dv_devx_cq_modify(struct ibv_cq *cq, const void *in, size_t inlen, void *out, size_t outlen); int mlx5dv_devx_qp_query(struct ibv_qp *qp, const void *in, size_t inlen, void *out, size_t outlen); int mlx5dv_devx_qp_modify(struct ibv_qp *qp, const void *in, size_t inlen, void *out, size_t outlen); int mlx5dv_devx_srq_query(struct ibv_srq *srq, const void *in, size_t inlen, void *out, size_t outlen); int mlx5dv_devx_srq_modify(struct ibv_srq *srq, const void *in, size_t inlen, void *out, size_t outlen); int mlx5dv_devx_wq_query(struct ibv_wq *wq, const void *in, size_t inlen, void *out, size_t outlen); int mlx5dv_devx_wq_modify(struct ibv_wq *wq, const void *in, size_t inlen, void *out, size_t outlen); int mlx5dv_devx_ind_tbl_query(struct ibv_rwq_ind_table *ind_tbl, const void *in, size_t inlen, void *out, size_t outlen); int mlx5dv_devx_ind_tbl_modify(struct ibv_rwq_ind_table *ind_tbl, const void *in, size_t inlen, void *out, size_t outlen); struct mlx5dv_devx_cmd_comp { int fd; }; struct mlx5dv_devx_cmd_comp * mlx5dv_devx_create_cmd_comp(struct ibv_context *context); void mlx5dv_devx_destroy_cmd_comp(struct mlx5dv_devx_cmd_comp *cmd_comp); int mlx5dv_devx_obj_query_async(struct mlx5dv_devx_obj *obj, const void *in, size_t inlen, size_t outlen, uint64_t wr_id, struct mlx5dv_devx_cmd_comp *cmd_comp); int mlx5dv_devx_get_async_cmd_comp(struct mlx5dv_devx_cmd_comp *cmd_comp, struct mlx5dv_devx_async_cmd_hdr *cmd_resp, size_t cmd_resp_len); struct mlx5dv_devx_event_channel { int fd; }; struct mlx5dv_devx_event_channel * mlx5dv_devx_create_event_channel(struct ibv_context *context, enum mlx5dv_devx_create_event_channel_flags flags); void mlx5dv_devx_destroy_event_channel(struct mlx5dv_devx_event_channel *event_channel); int mlx5dv_devx_subscribe_devx_event(struct mlx5dv_devx_event_channel *event_channel, struct mlx5dv_devx_obj *obj, /* can be NULL for unaffiliated events */ uint16_t events_sz, uint16_t events_num[], uint64_t cookie); int mlx5dv_devx_subscribe_devx_event_fd(struct mlx5dv_devx_event_channel *event_channel, int fd, struct mlx5dv_devx_obj *obj, /* can be NULL for unaffiliated events */ uint16_t event_num); /* return code: upon success number of bytes read, otherwise -1 and errno was set */ ssize_t mlx5dv_devx_get_event(struct mlx5dv_devx_event_channel *event_channel, struct mlx5dv_devx_async_event_hdr *event_data, size_t event_resp_len); #define __devx_nullp(typ) ((struct mlx5_ifc_##typ##_bits *)NULL) #define __devx_st_sz_bits(typ) sizeof(struct mlx5_ifc_##typ##_bits) #define __devx_bit_sz(typ, fld) sizeof(__devx_nullp(typ)->fld) #define __devx_bit_off(typ, fld) offsetof(struct mlx5_ifc_##typ##_bits, fld) #define __devx_dw_off(bit_off) ((bit_off) / 32) #define __devx_64_off(bit_off) ((bit_off) / 64) #define __devx_dw_bit_off(bit_sz, bit_off) (32 - (bit_sz) - ((bit_off) & 0x1f)) #define __devx_mask(bit_sz) ((uint32_t)((1ull << (bit_sz)) - 1)) #define __devx_dw_mask(bit_sz, bit_off) \ (__devx_mask(bit_sz) << __devx_dw_bit_off(bit_sz, bit_off)) #define DEVX_FLD_SZ_BYTES(typ, fld) (__devx_bit_sz(typ, fld) / 8) #define DEVX_ST_SZ_BYTES(typ) (sizeof(struct mlx5_ifc_##typ##_bits) / 8) #define DEVX_ST_SZ_DW(typ) (sizeof(struct mlx5_ifc_##typ##_bits) / 32) #define DEVX_ST_SZ_QW(typ) (sizeof(struct mlx5_ifc_##typ##_bits) / 64) #define DEVX_UN_SZ_BYTES(typ) (sizeof(union mlx5_ifc_##typ##_bits) / 8) #define DEVX_UN_SZ_DW(typ) (sizeof(union mlx5_ifc_##typ##_bits) / 32) #define DEVX_BYTE_OFF(typ, fld) (__devx_bit_off(typ, fld) / 8) #define DEVX_ADDR_OF(typ, p, fld) \ ((unsigned char *)(p) + DEVX_BYTE_OFF(typ, fld)) static inline void _devx_set(void *p, uint32_t value, size_t bit_off, size_t bit_sz) { __be32 *fld = (__be32 *)(p) + __devx_dw_off(bit_off); uint32_t dw_mask = __devx_dw_mask(bit_sz, bit_off); uint32_t mask = __devx_mask(bit_sz); *fld = htobe32((be32toh(*fld) & (~dw_mask)) | ((value & mask) << __devx_dw_bit_off(bit_sz, bit_off))); } #define DEVX_SET(typ, p, fld, v) \ _devx_set(p, v, __devx_bit_off(typ, fld), __devx_bit_sz(typ, fld)) static inline uint32_t _devx_get(const void *p, size_t bit_off, size_t bit_sz) { return ((be32toh(*((const __be32 *)(p) + __devx_dw_off(bit_off))) >> __devx_dw_bit_off(bit_sz, bit_off)) & __devx_mask(bit_sz)); } #define DEVX_GET(typ, p, fld) \ _devx_get(p, __devx_bit_off(typ, fld), __devx_bit_sz(typ, fld)) static inline void _devx_set64(void *p, uint64_t v, size_t bit_off) { *((__be64 *)(p) + __devx_64_off(bit_off)) = htobe64(v); } #define DEVX_SET64(typ, p, fld, v) _devx_set64(p, v, __devx_bit_off(typ, fld)) static inline uint64_t _devx_get64(const void *p, size_t bit_off) { return be64toh(*((const __be64 *)(p) + __devx_64_off(bit_off))); } #define DEVX_GET64(typ, p, fld) _devx_get64(p, __devx_bit_off(typ, fld)) struct mlx5dv_dr_domain; struct mlx5dv_dr_table; struct mlx5dv_dr_matcher; struct mlx5dv_dr_rule; struct mlx5dv_dr_action; enum mlx5dv_dr_domain_type { MLX5DV_DR_DOMAIN_TYPE_NIC_RX, MLX5DV_DR_DOMAIN_TYPE_NIC_TX, MLX5DV_DR_DOMAIN_TYPE_FDB, }; enum mlx5dv_dr_domain_sync_flags { MLX5DV_DR_DOMAIN_SYNC_FLAGS_SW = 1 << 0, MLX5DV_DR_DOMAIN_SYNC_FLAGS_HW = 1 << 1, MLX5DV_DR_DOMAIN_SYNC_FLAGS_MEM = 1 << 2, }; struct mlx5dv_dr_flow_meter_attr { struct mlx5dv_dr_table *next_table; uint8_t active; uint8_t reg_c_index; size_t flow_meter_parameter_sz; void *flow_meter_parameter; }; struct mlx5dv_dr_flow_sampler_attr { uint32_t sample_ratio; struct mlx5dv_dr_table *default_next_table; uint32_t num_sample_actions; struct mlx5dv_dr_action **sample_actions; __be64 action; }; struct mlx5dv_dr_domain * mlx5dv_dr_domain_create(struct ibv_context *ctx, enum mlx5dv_dr_domain_type type); int mlx5dv_dr_domain_destroy(struct mlx5dv_dr_domain *domain); int mlx5dv_dr_domain_sync(struct mlx5dv_dr_domain *domain, uint32_t flags); void mlx5dv_dr_domain_set_reclaim_device_memory(struct mlx5dv_dr_domain *dmn, bool enable); struct mlx5dv_dr_table * mlx5dv_dr_table_create(struct mlx5dv_dr_domain *domain, uint32_t level); int mlx5dv_dr_table_destroy(struct mlx5dv_dr_table *table); struct mlx5dv_dr_matcher * mlx5dv_dr_matcher_create(struct mlx5dv_dr_table *table, uint16_t priority, uint8_t match_criteria_enable, struct mlx5dv_flow_match_parameters *mask); int mlx5dv_dr_matcher_destroy(struct mlx5dv_dr_matcher *matcher); struct mlx5dv_dr_rule * mlx5dv_dr_rule_create(struct mlx5dv_dr_matcher *matcher, struct mlx5dv_flow_match_parameters *value, size_t num_actions, struct mlx5dv_dr_action *actions[]); int mlx5dv_dr_rule_destroy(struct mlx5dv_dr_rule *rule); enum mlx5dv_dr_action_flags { MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL = 1 << 0, }; struct mlx5dv_dr_action * mlx5dv_dr_action_create_dest_ibv_qp(struct ibv_qp *ibqp); struct mlx5dv_dr_action * mlx5dv_dr_action_create_dest_table(struct mlx5dv_dr_table *table); struct mlx5dv_dr_action * mlx5dv_dr_action_create_dest_vport(struct mlx5dv_dr_domain *domain, uint32_t vport); struct mlx5dv_dr_action * mlx5dv_dr_action_create_dest_devx_tir(struct mlx5dv_devx_obj *devx_obj); struct mlx5dv_dr_action * mlx5dv_dr_action_create_dest_ib_port(struct mlx5dv_dr_domain *domain, uint32_t ib_port); enum mlx5dv_dr_action_dest_type { MLX5DV_DR_ACTION_DEST, MLX5DV_DR_ACTION_DEST_REFORMAT, }; struct mlx5dv_dr_action_dest_reformat { struct mlx5dv_dr_action *reformat; struct mlx5dv_dr_action *dest; }; struct mlx5dv_dr_action_dest_attr { enum mlx5dv_dr_action_dest_type type; union { struct mlx5dv_dr_action *dest; struct mlx5dv_dr_action_dest_reformat *dest_reformat; }; }; struct mlx5dv_dr_action * mlx5dv_dr_action_create_dest_array(struct mlx5dv_dr_domain *domain, size_t num_dest, struct mlx5dv_dr_action_dest_attr *dests[]); struct mlx5dv_dr_action *mlx5dv_dr_action_create_drop(void); struct mlx5dv_dr_action *mlx5dv_dr_action_create_default_miss(void); struct mlx5dv_dr_action *mlx5dv_dr_action_create_tag(uint32_t tag_value); struct mlx5dv_dr_action * mlx5dv_dr_action_create_flow_counter(struct mlx5dv_devx_obj *devx_obj, uint32_t offset); enum mlx5dv_dr_action_aso_first_hit_flags { MLX5DV_DR_ACTION_FLAGS_ASO_FIRST_HIT_SET = 1 << 0, }; enum mlx5dv_dr_action_aso_flow_meter_flags { MLX5DV_DR_ACTION_FLAGS_ASO_FLOW_METER_RED = 1 << 0, MLX5DV_DR_ACTION_FLAGS_ASO_FLOW_METER_YELLOW = 1 << 1, MLX5DV_DR_ACTION_FLAGS_ASO_FLOW_METER_GREEN = 1 << 2, MLX5DV_DR_ACTION_FLAGS_ASO_FLOW_METER_UNDEFINED = 1 << 3, }; enum mlx5dv_dr_action_aso_ct_flags { MLX5DV_DR_ACTION_FLAGS_ASO_CT_DIRECTION_INITIATOR = 1 << 0, MLX5DV_DR_ACTION_FLAGS_ASO_CT_DIRECTION_RESPONDER = 1 << 1, }; struct mlx5dv_dr_action * mlx5dv_dr_action_create_aso(struct mlx5dv_dr_domain *domain, struct mlx5dv_devx_obj *devx_obj, uint32_t offset, uint32_t flags, uint8_t return_reg_c); int mlx5dv_dr_action_modify_aso(struct mlx5dv_dr_action *action, uint32_t offset, uint32_t flags, uint8_t return_reg_c); struct mlx5dv_dr_action * mlx5dv_dr_action_create_packet_reformat(struct mlx5dv_dr_domain *domain, uint32_t flags, enum mlx5dv_flow_action_packet_reformat_type reformat_type, size_t data_sz, void *data); struct mlx5dv_dr_action * mlx5dv_dr_action_create_modify_header(struct mlx5dv_dr_domain *domain, uint32_t flags, size_t actions_sz, __be64 actions[]); struct mlx5dv_dr_action * mlx5dv_dr_action_create_flow_meter(struct mlx5dv_dr_flow_meter_attr *attr); int mlx5dv_dr_action_modify_flow_meter(struct mlx5dv_dr_action *action, struct mlx5dv_dr_flow_meter_attr *attr, __be64 modify_field_select); struct mlx5dv_dr_action * mlx5dv_dr_action_create_flow_sampler(struct mlx5dv_dr_flow_sampler_attr *attr); struct mlx5dv_dr_action *mlx5dv_dr_action_create_pop_vlan(void); struct mlx5dv_dr_action *mlx5dv_dr_action_create_push_vlan(struct mlx5dv_dr_domain *domain, __be32 vlan_hdr); int mlx5dv_dr_action_destroy(struct mlx5dv_dr_action *action); int mlx5dv_dump_dr_domain(FILE *fout, struct mlx5dv_dr_domain *domain); int mlx5dv_dump_dr_table(FILE *fout, struct mlx5dv_dr_table *table); int mlx5dv_dump_dr_matcher(FILE *fout, struct mlx5dv_dr_matcher *matcher); int mlx5dv_dump_dr_rule(FILE *fout, struct mlx5dv_dr_rule *rule); struct mlx5dv_pp { uint16_t index; }; struct mlx5dv_pp *mlx5dv_pp_alloc(struct ibv_context *context, size_t pp_context_sz, const void *pp_context, uint32_t flags); void mlx5dv_pp_free(struct mlx5dv_pp *pp); int mlx5dv_query_qp_lag_port(struct ibv_qp *qp, uint8_t *port_num, uint8_t *active_port_num); int mlx5dv_modify_qp_lag_port(struct ibv_qp *qp, uint8_t port_num); int mlx5dv_modify_qp_udp_sport(struct ibv_qp *qp, uint16_t udp_sport); enum mlx5dv_sched_elem_attr_flags { MLX5DV_SCHED_ELEM_ATTR_FLAGS_BW_SHARE = 1 << 0, MLX5DV_SCHED_ELEM_ATTR_FLAGS_MAX_AVG_BW = 1 << 1, }; struct mlx5dv_sched_attr { struct mlx5dv_sched_node *parent; uint32_t flags; /* Use mlx5dv_sched_elem_attr_flags */ uint32_t bw_share; uint32_t max_avg_bw; uint64_t comp_mask; }; struct mlx5dv_sched_node; struct mlx5dv_sched_leaf; struct mlx5dv_sched_node * mlx5dv_sched_node_create(struct ibv_context *context, const struct mlx5dv_sched_attr *sched_attr); struct mlx5dv_sched_leaf * mlx5dv_sched_leaf_create(struct ibv_context *context, const struct mlx5dv_sched_attr *sched_attr); int mlx5dv_sched_node_modify(struct mlx5dv_sched_node *node, const struct mlx5dv_sched_attr *sched_attr); int mlx5dv_sched_leaf_modify(struct mlx5dv_sched_leaf *leaf, const struct mlx5dv_sched_attr *sched_attr); int mlx5dv_sched_node_destroy(struct mlx5dv_sched_node *node); int mlx5dv_sched_leaf_destroy(struct mlx5dv_sched_leaf *leaf); int mlx5dv_modify_qp_sched_elem(struct ibv_qp *qp, const struct mlx5dv_sched_leaf *requestor, const struct mlx5dv_sched_leaf *responder); int mlx5dv_reserved_qpn_alloc(struct ibv_context *ctx, uint32_t *qpn); int mlx5dv_reserved_qpn_dealloc(struct ibv_context *ctx, uint32_t qpn); #ifdef __cplusplus } #endif #endif /* _MLX5DV_H_ */