github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-07-27 09:36:22 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

Merge branch 'net-remove-tls_toe'

Browse Source 
Sabrina Dubroca says:

====================
net: remove tls_toe

This series removes the tls_toe feature, its single user (chtls), and
cleans up the EXPORT_SYMBOL()s that no other module requires.

Driver changes only compile-tested.
====================

Link: https://patch.msgid.link/cover.1781165969.git.sd@queasysnail.net
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
This commit is contained in:
Jakub Kicinski 2026-06-12 16:43:14 -07:00
commit 8d8fb5b5f5
28 changed files with 4 additions and 6374 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
Documentation/networking/tls-offload.rst
View File

@ -13,7 +13,7 @@ Layer Protocol (ULP) and install the cryptographic connection state.
For details regarding the user-facing interface refer to the TLS
documentation in :ref:`Documentation/networking/tls.rst <kernel_tls>`.
``ktls`` can operate in three modes:
``ktls`` can operate in two modes:
* Software crypto mode (``TLS_SW``) - CPU handles the cryptography.
In most basic cases only crypto operations synchronous with the CPU
@ -26,11 +26,6 @@ documentation in :ref:`Documentation/networking/tls.rst <kernel_tls>`.
This mode integrates best with the kernel stack and is described in detail
in the remaining part of this document
(``ethtool`` flags ``tls-hw-tx-offload`` and ``tls-hw-rx-offload``).
* Full TCP NIC offload mode (``TLS_HW_RECORD``) - mode of operation where
NIC driver and firmware replace the kernel networking stack
with its own TCP handling, it is not usable in production environments
making use of the Linux networking stack for example any firewalling
abilities or QoS and packet scheduling (``ethtool`` flag ``tls-hw-record``).
The operation mode is selected automatically based on device configuration,
offload opt-in or opt-out on per-connection basis is not currently supported.

1
arch/s390/configs/debug_defconfig
View File

@ -125,7 +125,6 @@ CONFIG_UNIX=y
CONFIG_UNIX_DIAG=m
CONFIG_TLS=m
CONFIG_TLS_DEVICE=y
CONFIG_TLS_TOE=y
CONFIG_XFRM_USER=m
CONFIG_NET_KEY=m
CONFIG_SMC=m

1
arch/s390/configs/defconfig
View File

@ -116,7 +116,6 @@ CONFIG_UNIX=y
CONFIG_UNIX_DIAG=m
CONFIG_TLS=m
CONFIG_TLS_DEVICE=y
CONFIG_TLS_TOE=y
CONFIG_XFRM_USER=m
CONFIG_NET_KEY=m
CONFIG_SMC=m

3
drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
View File

@ -6795,8 +6795,7 @@ static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
NETIF_F_TSO | NETIF_F_TSO6;
netdev->hw_features |= NETIF_F_GSO_UDP_TUNNEL |
NETIF_F_GSO_UDP_TUNNEL_CSUM |
NETIF_F_HW_TLS_RECORD;
NETIF_F_GSO_UDP_TUNNEL_CSUM;
if (adapter->rawf_cnt)
netdev->udp_tunnel_nic_info = &cxgb_udp_tunnels;

12
drivers/net/ethernet/chelsio/inline_crypto/Kconfig
View File

@ -13,18 +13,6 @@ config CHELSIO_INLINE_CRYPTO
if CHELSIO_INLINE_CRYPTO
config CRYPTO_DEV_CHELSIO_TLS
tristate "Chelsio Crypto Inline TLS Driver"
depends on CHELSIO_T4
depends on TLS
depends on TLS_TOE
help
Support Chelsio Inline TLS with Chelsio crypto accelerator.
Enable inline TLS support for Tx and Rx.
To compile this driver as a module, choose M here: the module
will be called chtls.
config CHELSIO_IPSEC_INLINE
tristate "Chelsio IPSec XFRM Tx crypto offload"
depends on CHELSIO_T4

1
drivers/net/ethernet/chelsio/inline_crypto/Makefile
View File

@ -1,4 +1,3 @@
# SPDX-License-Identifier: GPL-2.0-only
obj-$(CONFIG_CRYPTO_DEV_CHELSIO_TLS) += chtls/
obj-$(CONFIG_CHELSIO_IPSEC_INLINE) += ch_ipsec/
obj-$(CONFIG_CHELSIO_TLS_DEVICE) += ch_ktls/

6
drivers/net/ethernet/chelsio/inline_crypto/chtls/Makefile
View File

@ -1,6 +0,0 @@
# SPDX-License-Identifier: GPL-2.0-only
ccflags-y := -I $(srctree)/drivers/net/ethernet/chelsio/cxgb4 \
-I $(srctree)/drivers/crypto/chelsio
obj-$(CONFIG_CRYPTO_DEV_CHELSIO_TLS) += chtls.o
chtls-objs := chtls_main.o chtls_cm.o chtls_io.o chtls_hw.o

584
drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls.h
View File

@ -1,584 +0,0 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (c) 2018 Chelsio Communications, Inc.
*/
#ifndef __CHTLS_H__
#define __CHTLS_H__
#include <crypto/aes.h>
#include <crypto/hash.h>
#include <crypto/sha1.h>
#include <crypto/sha2.h>
#include <crypto/authenc.h>
#include <crypto/ctr.h>
#include <crypto/gf128mul.h>
#include <crypto/internal/aead.h>
#include <crypto/null.h>
#include <crypto/internal/skcipher.h>
#include <crypto/aead.h>
#include <crypto/scatterwalk.h>
#include <crypto/internal/hash.h>
#include <linux/tls.h>
#include <net/tls.h>
#include <net/tls_prot.h>
#include <net/tls_toe.h>
#include "t4fw_api.h"
#include "t4_msg.h"
#include "cxgb4.h"
#include "cxgb4_uld.h"
#include "l2t.h"
#include "chcr_algo.h"
#include "chcr_core.h"
#include "chcr_crypto.h"
#define CHTLS_DRV_VERSION "1.0.0.0-ko"
#define TLS_KEYCTX_RXFLIT_CNT_S 24
#define TLS_KEYCTX_RXFLIT_CNT_V(x) ((x) << TLS_KEYCTX_RXFLIT_CNT_S)
#define TLS_KEYCTX_RXPROT_VER_S 20
#define TLS_KEYCTX_RXPROT_VER_M 0xf
#define TLS_KEYCTX_RXPROT_VER_V(x) ((x) << TLS_KEYCTX_RXPROT_VER_S)
#define TLS_KEYCTX_RXCIPH_MODE_S 16
#define TLS_KEYCTX_RXCIPH_MODE_M 0xf
#define TLS_KEYCTX_RXCIPH_MODE_V(x) ((x) << TLS_KEYCTX_RXCIPH_MODE_S)
#define TLS_KEYCTX_RXAUTH_MODE_S 12
#define TLS_KEYCTX_RXAUTH_MODE_M 0xf
#define TLS_KEYCTX_RXAUTH_MODE_V(x) ((x) << TLS_KEYCTX_RXAUTH_MODE_S)
#define TLS_KEYCTX_RXCIAU_CTRL_S 11
#define TLS_KEYCTX_RXCIAU_CTRL_V(x) ((x) << TLS_KEYCTX_RXCIAU_CTRL_S)
#define TLS_KEYCTX_RX_SEQCTR_S 9
#define TLS_KEYCTX_RX_SEQCTR_M 0x3
#define TLS_KEYCTX_RX_SEQCTR_V(x) ((x) << TLS_KEYCTX_RX_SEQCTR_S)
#define TLS_KEYCTX_RX_VALID_S 8
#define TLS_KEYCTX_RX_VALID_V(x) ((x) << TLS_KEYCTX_RX_VALID_S)
#define TLS_KEYCTX_RXCK_SIZE_S 3
#define TLS_KEYCTX_RXCK_SIZE_M 0x7
#define TLS_KEYCTX_RXCK_SIZE_V(x) ((x) << TLS_KEYCTX_RXCK_SIZE_S)
#define TLS_KEYCTX_RXMK_SIZE_S 0
#define TLS_KEYCTX_RXMK_SIZE_M 0x7
#define TLS_KEYCTX_RXMK_SIZE_V(x) ((x) << TLS_KEYCTX_RXMK_SIZE_S)
#define KEYCTX_TX_WR_IV_S 55
#define KEYCTX_TX_WR_IV_M 0x1ffULL
#define KEYCTX_TX_WR_IV_V(x) ((x) << KEYCTX_TX_WR_IV_S)
#define KEYCTX_TX_WR_IV_G(x) \
(((x) >> KEYCTX_TX_WR_IV_S) & KEYCTX_TX_WR_IV_M)
#define KEYCTX_TX_WR_AAD_S 47
#define KEYCTX_TX_WR_AAD_M 0xffULL
#define KEYCTX_TX_WR_AAD_V(x) ((x) << KEYCTX_TX_WR_AAD_S)
#define KEYCTX_TX_WR_AAD_G(x) (((x) >> KEYCTX_TX_WR_AAD_S) & \
KEYCTX_TX_WR_AAD_M)
#define KEYCTX_TX_WR_AADST_S 39
#define KEYCTX_TX_WR_AADST_M 0xffULL
#define KEYCTX_TX_WR_AADST_V(x) ((x) << KEYCTX_TX_WR_AADST_S)
#define KEYCTX_TX_WR_AADST_G(x) \
(((x) >> KEYCTX_TX_WR_AADST_S) & KEYCTX_TX_WR_AADST_M)
#define KEYCTX_TX_WR_CIPHER_S 30
#define KEYCTX_TX_WR_CIPHER_M 0x1ffULL
#define KEYCTX_TX_WR_CIPHER_V(x) ((x) << KEYCTX_TX_WR_CIPHER_S)
#define KEYCTX_TX_WR_CIPHER_G(x) \
(((x) >> KEYCTX_TX_WR_CIPHER_S) & KEYCTX_TX_WR_CIPHER_M)
#define KEYCTX_TX_WR_CIPHERST_S 23
#define KEYCTX_TX_WR_CIPHERST_M 0x7f
#define KEYCTX_TX_WR_CIPHERST_V(x) ((x) << KEYCTX_TX_WR_CIPHERST_S)
#define KEYCTX_TX_WR_CIPHERST_G(x) \
(((x) >> KEYCTX_TX_WR_CIPHERST_S) & KEYCTX_TX_WR_CIPHERST_M)
#define KEYCTX_TX_WR_AUTH_S 14
#define KEYCTX_TX_WR_AUTH_M 0x1ff
#define KEYCTX_TX_WR_AUTH_V(x) ((x) << KEYCTX_TX_WR_AUTH_S)
#define KEYCTX_TX_WR_AUTH_G(x) \
(((x) >> KEYCTX_TX_WR_AUTH_S) & KEYCTX_TX_WR_AUTH_M)
#define KEYCTX_TX_WR_AUTHST_S 7
#define KEYCTX_TX_WR_AUTHST_M 0x7f
#define KEYCTX_TX_WR_AUTHST_V(x) ((x) << KEYCTX_TX_WR_AUTHST_S)
#define KEYCTX_TX_WR_AUTHST_G(x) \
(((x) >> KEYCTX_TX_WR_AUTHST_S) & KEYCTX_TX_WR_AUTHST_M)
#define KEYCTX_TX_WR_AUTHIN_S 0
#define KEYCTX_TX_WR_AUTHIN_M 0x7f
#define KEYCTX_TX_WR_AUTHIN_V(x) ((x) << KEYCTX_TX_WR_AUTHIN_S)
#define KEYCTX_TX_WR_AUTHIN_G(x) \
(((x) >> KEYCTX_TX_WR_AUTHIN_S) & KEYCTX_TX_WR_AUTHIN_M)
struct sge_opaque_hdr {
void *dev;
dma_addr_t addr[MAX_SKB_FRAGS + 1];
};
#define MAX_IVS_PAGE 256
#define TLS_KEY_CONTEXT_SZ 64
#define CIPHER_BLOCK_SIZE 16
#define GCM_TAG_SIZE 16
#define KEY_ON_MEM_SZ 16
#define AEAD_EXPLICIT_DATA_SIZE 8
#define TLS_HEADER_LENGTH 5
#define SCMD_CIPH_MODE_AES_GCM 2
/* Any MFS size should work and come from openssl */
#define TLS_MFS 16384
#define RSS_HDR sizeof(struct rss_header)
#define TLS_WR_CPL_LEN \
(sizeof(struct fw_tlstx_data_wr) + sizeof(struct cpl_tx_tls_sfo))
enum {
CHTLS_KEY_CONTEXT_DSGL,
CHTLS_KEY_CONTEXT_IMM,
CHTLS_KEY_CONTEXT_DDR,
};
enum {
CHTLS_LISTEN_START,
CHTLS_LISTEN_STOP,
};
/* Flags for return value of CPL message handlers */
enum {
CPL_RET_BUF_DONE = 1, /* buffer processing done */
CPL_RET_BAD_MSG = 2, /* bad CPL message */
CPL_RET_UNKNOWN_TID = 4 /* unexpected unknown TID */
};
#define LISTEN_INFO_HASH_SIZE 32
#define RSPQ_HASH_BITS 5
struct listen_info {
struct listen_info *next; /* Link to next entry */
struct sock *sk; /* The listening socket */
unsigned int stid; /* The server TID */
};
enum {
T4_LISTEN_START_PENDING,
T4_LISTEN_STARTED
};
enum csk_flags {
CSK_CALLBACKS_CHKD, /* socket callbacks have been sanitized */
CSK_ABORT_REQ_RCVD, /* received one ABORT_REQ_RSS message */
CSK_TX_MORE_DATA, /* sending ULP data; don't set SHOVE bit */
CSK_TX_WAIT_IDLE, /* suspend Tx until in-flight data is ACKed */
CSK_ABORT_SHUTDOWN, /* shouldn't send more abort requests */
CSK_ABORT_RPL_PENDING, /* expecting an abort reply */
CSK_CLOSE_CON_REQUESTED,/* we've sent a close_conn_req */
CSK_TX_DATA_SENT, /* sent a TX_DATA WR on this connection */
CSK_TX_FAILOVER, /* Tx traffic failing over */
CSK_UPDATE_RCV_WND, /* Need to update rcv window */
CSK_RST_ABORTED, /* outgoing RST was aborted */
CSK_TLS_HANDSHK, /* TLS Handshake */
CSK_CONN_INLINE, /* Connection on HW */
};
enum chtls_cdev_state {
CHTLS_CDEV_STATE_UP = 1
};
struct listen_ctx {
struct sock *lsk;
struct chtls_dev *cdev;
struct sk_buff_head synq;
u32 state;
};
struct key_map {
unsigned long *addr;
unsigned int start;
unsigned int available;
unsigned int size;
spinlock_t lock; /* lock for key id request from map */
} __packed;
struct tls_scmd {
u32 seqno_numivs;
u32 ivgen_hdrlen;
};
struct chtls_dev {
struct tls_toe_device tlsdev;
struct list_head list;
struct cxgb4_lld_info *lldi;
struct pci_dev *pdev;
struct listen_info *listen_hash_tab[LISTEN_INFO_HASH_SIZE];
spinlock_t listen_lock; /* lock for listen list */
struct net_device **ports;
struct tid_info *tids;
unsigned int pfvf;
const unsigned short *mtus;
struct idr hwtid_idr;
struct idr stid_idr;
spinlock_t idr_lock ____cacheline_aligned_in_smp;
struct net_device *egr_dev[NCHAN * 2];
struct sk_buff *rspq_skb_cache[1 << RSPQ_HASH_BITS];
struct sk_buff *askb;
struct sk_buff_head deferq;
struct work_struct deferq_task;
struct list_head list_node;
struct list_head rcu_node;
struct list_head na_node;
unsigned int send_page_order;
int max_host_sndbuf;
u32 round_robin_cnt;
struct key_map kmap;
unsigned int cdev_state;
};
struct chtls_listen {
struct chtls_dev *cdev;
struct sock *sk;
};
struct chtls_hws {
struct sk_buff_head sk_recv_queue;
u8 txqid;
u8 ofld;
u16 type;
u16 rstate;
u16 keyrpl;
u16 pldlen;
u16 rcvpld;
u16 compute;
u16 expansion;
u16 keylen;
u16 pdus;
u16 adjustlen;
u16 ivsize;
u16 txleft;
u32 mfs;
s32 txkey;
s32 rxkey;
u32 fcplenmax;
u32 copied_seq;
u64 tx_seq_no;
struct tls_scmd scmd;
union {
struct tls12_crypto_info_aes_gcm_128 aes_gcm_128;
struct tls12_crypto_info_aes_gcm_256 aes_gcm_256;
} crypto_info;
};
struct chtls_sock {
struct sock *sk;
struct chtls_dev *cdev;
struct l2t_entry *l2t_entry; /* pointer to the L2T entry */
struct net_device *egress_dev; /* TX_CHAN for act open retry */
struct sk_buff_head txq;
struct sk_buff *wr_skb_head;
struct sk_buff *wr_skb_tail;
struct sk_buff *ctrl_skb_cache;
struct sk_buff *txdata_skb_cache; /* abort path messages */
struct kref kref;
unsigned long flags;
u32 opt2;
u32 wr_credits;
u32 wr_unacked;
u32 wr_max_credits;
u32 wr_nondata;
u32 hwtid; /* TCP Control Block ID */
u32 txq_idx;
u32 rss_qid;
u32 tid;
u32 idr;
u32 mss;
u32 ulp_mode;
u32 tx_chan;
u32 rx_chan;
u32 sndbuf;
u32 txplen_max;
u32 mtu_idx; /* MTU table index */
u32 smac_idx;
u8 port_id;
u8 tos;
u16 resv2;
u32 delack_mode;
u32 delack_seq;
u32 snd_win;
u32 rcv_win;
void *passive_reap_next; /* placeholder for passive */
struct chtls_hws tlshws;
struct synq {
struct sk_buff *next;
struct sk_buff *prev;
} synq;
struct listen_ctx *listen_ctx;
};
struct tls_hdr {
u8 type;
u16 version;
u16 length;
} __packed;
struct tlsrx_cmp_hdr {
u8 type;
u16 version;
u16 length;
u64 tls_seq;
u16 reserved1;
u8 res_to_mac_error;
} __packed;
/* res_to_mac_error fields */
#define TLSRX_HDR_PKT_INT_ERROR_S 4
#define TLSRX_HDR_PKT_INT_ERROR_M 0x1
#define TLSRX_HDR_PKT_INT_ERROR_V(x) \
((x) << TLSRX_HDR_PKT_INT_ERROR_S)
#define TLSRX_HDR_PKT_INT_ERROR_G(x) \
(((x) >> TLSRX_HDR_PKT_INT_ERROR_S) & TLSRX_HDR_PKT_INT_ERROR_M)
#define TLSRX_HDR_PKT_INT_ERROR_F TLSRX_HDR_PKT_INT_ERROR_V(1U)
#define TLSRX_HDR_PKT_SPP_ERROR_S 3
#define TLSRX_HDR_PKT_SPP_ERROR_M 0x1
#define TLSRX_HDR_PKT_SPP_ERROR_V(x) ((x) << TLSRX_HDR_PKT_SPP_ERROR)
#define TLSRX_HDR_PKT_SPP_ERROR_G(x) \
(((x) >> TLSRX_HDR_PKT_SPP_ERROR_S) & TLSRX_HDR_PKT_SPP_ERROR_M)
#define TLSRX_HDR_PKT_SPP_ERROR_F TLSRX_HDR_PKT_SPP_ERROR_V(1U)
#define TLSRX_HDR_PKT_CCDX_ERROR_S 2
#define TLSRX_HDR_PKT_CCDX_ERROR_M 0x1
#define TLSRX_HDR_PKT_CCDX_ERROR_V(x) ((x) << TLSRX_HDR_PKT_CCDX_ERROR_S)
#define TLSRX_HDR_PKT_CCDX_ERROR_G(x) \
(((x) >> TLSRX_HDR_PKT_CCDX_ERROR_S) & TLSRX_HDR_PKT_CCDX_ERROR_M)
#define TLSRX_HDR_PKT_CCDX_ERROR_F TLSRX_HDR_PKT_CCDX_ERROR_V(1U)
#define TLSRX_HDR_PKT_PAD_ERROR_S 1
#define TLSRX_HDR_PKT_PAD_ERROR_M 0x1
#define TLSRX_HDR_PKT_PAD_ERROR_V(x) ((x) << TLSRX_HDR_PKT_PAD_ERROR_S)
#define TLSRX_HDR_PKT_PAD_ERROR_G(x) \
(((x) >> TLSRX_HDR_PKT_PAD_ERROR_S) & TLSRX_HDR_PKT_PAD_ERROR_M)
#define TLSRX_HDR_PKT_PAD_ERROR_F TLSRX_HDR_PKT_PAD_ERROR_V(1U)
#define TLSRX_HDR_PKT_MAC_ERROR_S 0
#define TLSRX_HDR_PKT_MAC_ERROR_M 0x1
#define TLSRX_HDR_PKT_MAC_ERROR_V(x) ((x) << TLSRX_HDR_PKT_MAC_ERROR)
#define TLSRX_HDR_PKT_MAC_ERROR_G(x) \
(((x) >> S_TLSRX_HDR_PKT_MAC_ERROR_S) & TLSRX_HDR_PKT_MAC_ERROR_M)
#define TLSRX_HDR_PKT_MAC_ERROR_F TLSRX_HDR_PKT_MAC_ERROR_V(1U)
#define TLSRX_HDR_PKT_ERROR_M 0x1F
#define CONTENT_TYPE_ERROR 0x7F
struct ulp_mem_rw {
__be32 cmd;
__be32 len16; /* command length */
__be32 dlen; /* data length in 32-byte units */
__be32 lock_addr;
};
struct tls_key_wr {
__be32 op_to_compl;
__be32 flowid_len16;
__be32 ftid;
u8 reneg_to_write_rx;
u8 protocol;
__be16 mfs;
};
struct tls_key_req {
struct tls_key_wr wr;
struct ulp_mem_rw req;
struct ulptx_idata sc_imm;
};
/*
* This lives in skb->cb and is used to chain WRs in a linked list.
*/
struct wr_skb_cb {
struct l2t_skb_cb l2t; /* reserve space for l2t CB */
struct sk_buff *next_wr; /* next write request */
};
/* Per-skb backlog handler. Run when a socket's backlog is processed. */
struct blog_skb_cb {
void (*backlog_rcv)(struct sock *sk, struct sk_buff *skb);
struct chtls_dev *cdev;
};
/*
* Similar to tcp_skb_cb but with ULP elements added to support TLS,
* etc.
*/
struct ulp_skb_cb {
struct wr_skb_cb wr; /* reserve space for write request */
u16 flags; /* TCP-like flags */
u8 psh;
u8 ulp_mode; /* ULP mode/submode of sk_buff */
u32 seq; /* TCP sequence number */
union { /* ULP-specific fields */
struct {
u8 type;
u8 ofld;
u8 iv;
} tls;
} ulp;
};
#define ULP_SKB_CB(skb) ((struct ulp_skb_cb *)&((skb)->cb[0]))
#define BLOG_SKB_CB(skb) ((struct blog_skb_cb *)(skb)->cb)
/*
* Flags for ulp_skb_cb.flags.
*/
enum {
ULPCB_FLAG_NEED_HDR = 1 << 0, /* packet needs a TX_DATA_WR header */
ULPCB_FLAG_NO_APPEND = 1 << 1, /* don't grow this skb */
ULPCB_FLAG_BARRIER = 1 << 2, /* set TX_WAIT_IDLE after sending */
ULPCB_FLAG_HOLD = 1 << 3, /* skb not ready for Tx yet */
ULPCB_FLAG_COMPL = 1 << 4, /* request WR completion */
ULPCB_FLAG_URG = 1 << 5, /* urgent data */
ULPCB_FLAG_TLS_HDR = 1 << 6, /* payload with tls hdr */
ULPCB_FLAG_NO_HDR = 1 << 7, /* not a ofld wr */
};
/* The ULP mode/submode of an skbuff */
#define skb_ulp_mode(skb) (ULP_SKB_CB(skb)->ulp_mode)
#define TCP_PAGE(sk) (sk->sk_frag.page)
#define TCP_OFF(sk) (sk->sk_frag.offset)
static inline struct chtls_dev *to_chtls_dev(struct tls_toe_device *tlsdev)
{
return container_of(tlsdev, struct chtls_dev, tlsdev);
}
static inline void csk_set_flag(struct chtls_sock *csk,
enum csk_flags flag)
{
__set_bit(flag, &csk->flags);
}
static inline void csk_reset_flag(struct chtls_sock *csk,
enum csk_flags flag)
{
__clear_bit(flag, &csk->flags);
}
static inline bool csk_conn_inline(const struct chtls_sock *csk)
{
return test_bit(CSK_CONN_INLINE, &csk->flags);
}
static inline int csk_flag(const struct sock *sk, enum csk_flags flag)
{
struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
if (!csk_conn_inline(csk))
return 0;
return test_bit(flag, &csk->flags);
}
static inline int csk_flag_nochk(const struct chtls_sock *csk,
enum csk_flags flag)
{
return test_bit(flag, &csk->flags);
}
static inline void *cplhdr(struct sk_buff *skb)
{
return skb->data;
}
static inline int is_neg_adv(unsigned int status)
{
return status == CPL_ERR_RTX_NEG_ADVICE ||
status == CPL_ERR_KEEPALV_NEG_ADVICE ||
status == CPL_ERR_PERSIST_NEG_ADVICE;
}
static inline void process_cpl_msg(void (*fn)(struct sock *, struct sk_buff *),
struct sock *sk,
struct sk_buff *skb)
{
skb_reset_mac_header(skb);
skb_reset_network_header(skb);
skb_reset_transport_header(skb);
bh_lock_sock(sk);
if (unlikely(sock_owned_by_user(sk))) {
BLOG_SKB_CB(skb)->backlog_rcv = fn;
__sk_add_backlog(sk, skb);
} else {
fn(sk, skb);
}
bh_unlock_sock(sk);
}
static inline void chtls_sock_free(struct kref *ref)
{
struct chtls_sock *csk = container_of(ref, struct chtls_sock,
kref);
kfree(csk);
}
static inline void __chtls_sock_put(const char *fn, struct chtls_sock *csk)
{
kref_put(&csk->kref, chtls_sock_free);
}
static inline void __chtls_sock_get(const char *fn,
struct chtls_sock *csk)
{
kref_get(&csk->kref);
}
static inline void send_or_defer(struct sock *sk, struct tcp_sock *tp,
struct sk_buff *skb, int through_l2t)
{
struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
if (through_l2t) {
/* send through L2T */
cxgb4_l2t_send(csk->egress_dev, skb, csk->l2t_entry);
} else {
/* send directly */
cxgb4_ofld_send(csk->egress_dev, skb);
}
}
typedef int (*chtls_handler_func)(struct chtls_dev *, struct sk_buff *);
extern chtls_handler_func chtls_handlers[NUM_CPL_CMDS];
void chtls_install_cpl_ops(struct sock *sk);
int chtls_init_kmap(struct chtls_dev *cdev, struct cxgb4_lld_info *lldi);
void chtls_listen_stop(struct chtls_dev *cdev, struct sock *sk);
int chtls_listen_start(struct chtls_dev *cdev, struct sock *sk);
void chtls_close(struct sock *sk, long timeout);
int chtls_disconnect(struct sock *sk, int flags);
void chtls_shutdown(struct sock *sk, int how);
void chtls_destroy_sock(struct sock *sk);
int chtls_sendmsg(struct sock *sk, struct msghdr *msg, size_t size);
int chtls_recvmsg(struct sock *sk, struct msghdr *msg,
size_t len, int flags);
void chtls_splice_eof(struct socket *sock);
int send_tx_flowc_wr(struct sock *sk, int compl,
u32 snd_nxt, u32 rcv_nxt);
void chtls_tcp_push(struct sock *sk, int flags);
int chtls_push_frames(struct chtls_sock *csk, int comp);
void chtls_set_tcb_field_rpl_skb(struct sock *sk, u16 word,
u64 mask, u64 val, u8 cookie,
int through_l2t);
int chtls_setkey(struct chtls_sock *csk, u32 keylen, u32 mode, int cipher_type);
void chtls_set_quiesce_ctrl(struct sock *sk, int val);
void skb_entail(struct sock *sk, struct sk_buff *skb, int flags);
unsigned int keyid_to_addr(int start_addr, int keyid);
void free_tls_keyid(struct sock *sk);
#endif

2336
drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_cm.c
View File

File diff suppressed because it is too large Load Diff

218
drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_cm.h
View File

@ -1,218 +0,0 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (c) 2018 Chelsio Communications, Inc.
*/
#ifndef __CHTLS_CM_H__
#define __CHTLS_CM_H__
/*
* TCB settings
*/
/* 3:0 */
#define TCB_ULP_TYPE_W 0
#define TCB_ULP_TYPE_S 0
#define TCB_ULP_TYPE_M 0xfULL
#define TCB_ULP_TYPE_V(x) ((x) << TCB_ULP_TYPE_S)
/* 11:4 */
#define TCB_ULP_RAW_W 0
#define TCB_ULP_RAW_S 4
#define TCB_ULP_RAW_M 0xffULL
#define TCB_ULP_RAW_V(x) ((x) << TCB_ULP_RAW_S)
#define TF_TLS_KEY_SIZE_S 7
#define TF_TLS_KEY_SIZE_V(x) ((x) << TF_TLS_KEY_SIZE_S)
#define TF_TLS_CONTROL_S 2
#define TF_TLS_CONTROL_V(x) ((x) << TF_TLS_CONTROL_S)
#define TF_TLS_ACTIVE_S 1
#define TF_TLS_ACTIVE_V(x) ((x) << TF_TLS_ACTIVE_S)
#define TF_TLS_ENABLE_S 0
#define TF_TLS_ENABLE_V(x) ((x) << TF_TLS_ENABLE_S)
#define TF_RX_QUIESCE_S 15
#define TF_RX_QUIESCE_V(x) ((x) << TF_RX_QUIESCE_S)
/*
* Max receive window supported by HW in bytes. Only a small part of it can
* be set through option0, the rest needs to be set through RX_DATA_ACK.
*/
#define MAX_RCV_WND ((1U << 27) - 1)
#define MAX_MSS 65536
/*
* Min receive window. We want it to be large enough to accommodate receive
* coalescing, handle jumbo frames, and not trigger sender SWS avoidance.
*/
#define MIN_RCV_WND (24 * 1024U)
#define LOOPBACK(x) (((x) & htonl(0xff000000)) == htonl(0x7f000000))
/* for TX: a skb must have a headroom of at least TX_HEADER_LEN bytes */
#define TX_HEADER_LEN \
(sizeof(struct fw_ofld_tx_data_wr) + sizeof(struct sge_opaque_hdr))
#define TX_TLSHDR_LEN \
(sizeof(struct fw_tlstx_data_wr) + sizeof(struct cpl_tx_tls_sfo) + \
sizeof(struct sge_opaque_hdr))
#define TXDATA_SKB_LEN 128
enum {
CPL_TX_TLS_SFO_TYPE_CCS,
CPL_TX_TLS_SFO_TYPE_ALERT,
CPL_TX_TLS_SFO_TYPE_HANDSHAKE,
CPL_TX_TLS_SFO_TYPE_DATA,
CPL_TX_TLS_SFO_TYPE_HEARTBEAT,
};
enum {
TLS_HDR_TYPE_CCS = 20,
TLS_HDR_TYPE_ALERT,
TLS_HDR_TYPE_HANDSHAKE,
TLS_HDR_TYPE_RECORD,
TLS_HDR_TYPE_HEARTBEAT,
};
typedef void (*defer_handler_t)(struct chtls_dev *dev, struct sk_buff *skb);
extern struct request_sock_ops chtls_rsk_ops;
extern struct request_sock_ops chtls_rsk_opsv6;
struct deferred_skb_cb {
defer_handler_t handler;
struct chtls_dev *dev;
};
#define DEFERRED_SKB_CB(skb) ((struct deferred_skb_cb *)(skb)->cb)
#define failover_flowc_wr_len offsetof(struct fw_flowc_wr, mnemval[3])
#define WR_SKB_CB(skb) ((struct wr_skb_cb *)(skb)->cb)
#define ACCEPT_QUEUE(sk) (&inet_csk(sk)->icsk_accept_queue.rskq_accept_head)
#define SND_WSCALE(tp) ((tp)->rx_opt.snd_wscale)
#define RCV_WSCALE(tp) ((tp)->rx_opt.rcv_wscale)
#define USER_MSS(tp) (READ_ONCE((tp)->rx_opt.user_mss))
#define TS_RECENT_STAMP(tp) ((tp)->rx_opt.ts_recent_stamp)
#define WSCALE_OK(tp) ((tp)->rx_opt.wscale_ok)
#define TSTAMP_OK(tp) ((tp)->rx_opt.tstamp_ok)
#define SACK_OK(tp) ((tp)->rx_opt.sack_ok)
/* TLS SKB */
#define skb_ulp_tls_inline(skb) (ULP_SKB_CB(skb)->ulp.tls.ofld)
#define skb_ulp_tls_iv_imm(skb) (ULP_SKB_CB(skb)->ulp.tls.iv)
void chtls_defer_reply(struct sk_buff *skb, struct chtls_dev *dev,
defer_handler_t handler);
/*
* Returns true if the socket is in one of the supplied states.
*/
static inline unsigned int sk_in_state(const struct sock *sk,
unsigned int states)
{
return states & (1 << sk->sk_state);
}
static void chtls_rsk_destructor(struct request_sock *req)
{
/* do nothing */
}
static inline void chtls_init_rsk_ops(struct proto *chtls_tcp_prot,
struct request_sock_ops *chtls_tcp_ops,
struct proto *tcp_prot, int family)
{
memset(chtls_tcp_ops, 0, sizeof(*chtls_tcp_ops));
chtls_tcp_ops->family = family;
chtls_tcp_ops->obj_size = sizeof(struct tcp_request_sock);
chtls_tcp_ops->destructor = chtls_rsk_destructor;
chtls_tcp_ops->slab = tcp_prot->rsk_prot->slab;
chtls_tcp_prot->rsk_prot = chtls_tcp_ops;
}
static inline void chtls_reqsk_free(struct request_sock *req)
{
if (req->rsk_listener)
sock_put(req->rsk_listener);
kmem_cache_free(req->rsk_ops->slab, req);
}
#define DECLARE_TASK_FUNC(task, task_param) \
static void task(struct work_struct *task_param)
static inline void sk_wakeup_sleepers(struct sock *sk, bool interruptable)
{
struct socket_wq *wq;
rcu_read_lock();
wq = rcu_dereference(sk->sk_wq);
if (skwq_has_sleeper(wq)) {
if (interruptable)
wake_up_interruptible(sk_sleep(sk));
else
wake_up_all(sk_sleep(sk));
}
rcu_read_unlock();
}
static inline void chtls_set_req_port(struct request_sock *oreq,
__be16 source, __be16 dest)
{
inet_rsk(oreq)->ir_rmt_port = source;
inet_rsk(oreq)->ir_num = ntohs(dest);
}
static inline void chtls_set_req_addr(struct request_sock *oreq,
__be32 local_ip, __be32 peer_ip)
{
inet_rsk(oreq)->ir_loc_addr = local_ip;
inet_rsk(oreq)->ir_rmt_addr = peer_ip;
}
static inline void chtls_free_skb(struct sock *sk, struct sk_buff *skb)
{
skb_dstref_steal(skb);
__skb_unlink(skb, &sk->sk_receive_queue);
__kfree_skb(skb);
}
static inline void chtls_kfree_skb(struct sock *sk, struct sk_buff *skb)
{
skb_dstref_steal(skb);
__skb_unlink(skb, &sk->sk_receive_queue);
kfree_skb(skb);
}
static inline void chtls_reset_wr_list(struct chtls_sock *csk)
{
csk->wr_skb_head = NULL;
csk->wr_skb_tail = NULL;
}
static inline void enqueue_wr(struct chtls_sock *csk, struct sk_buff *skb)
{
WR_SKB_CB(skb)->next_wr = NULL;
skb_get(skb);
if (!csk->wr_skb_head)
csk->wr_skb_head = skb;
else
WR_SKB_CB(csk->wr_skb_tail)->next_wr = skb;
csk->wr_skb_tail = skb;
}
static inline struct sk_buff *dequeue_wr(struct sock *sk)
{
struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
struct sk_buff *skb = NULL;
skb = csk->wr_skb_head;
if (likely(skb)) {
/* Don't bother clearing the tail */
csk->wr_skb_head = WR_SKB_CB(skb)->next_wr;
WR_SKB_CB(skb)->next_wr = NULL;
}
return skb;
}
#endif

462
drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_hw.c
View File

@ -1,462 +0,0 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (c) 2018 Chelsio Communications, Inc.
*
* Written by: Atul Gupta (atul.gupta@chelsio.com)
*/
#include <linux/module.h>
#include <linux/list.h>
#include <linux/workqueue.h>
#include <linux/skbuff.h>
#include <linux/timer.h>
#include <linux/notifier.h>
#include <linux/inetdevice.h>
#include <linux/ip.h>
#include <linux/tcp.h>
#include <linux/tls.h>
#include <net/tls.h>
#include "chtls.h"
#include "chtls_cm.h"
static void __set_tcb_field_direct(struct chtls_sock *csk,
struct cpl_set_tcb_field *req, u16 word,
u64 mask, u64 val, u8 cookie, int no_reply)
{
struct ulptx_idata *sc;
INIT_TP_WR_CPL(req, CPL_SET_TCB_FIELD, csk->tid);
req->wr.wr_mid |= htonl(FW_WR_FLOWID_V(csk->tid));
req->reply_ctrl = htons(NO_REPLY_V(no_reply) |
QUEUENO_V(csk->rss_qid));
req->word_cookie = htons(TCB_WORD_V(word) | TCB_COOKIE_V(cookie));
req->mask = cpu_to_be64(mask);
req->val = cpu_to_be64(val);
sc = (struct ulptx_idata *)(req + 1);
sc->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_NOOP));
sc->len = htonl(0);
}
static void __set_tcb_field(struct sock *sk, struct sk_buff *skb, u16 word,
u64 mask, u64 val, u8 cookie, int no_reply)
{
struct cpl_set_tcb_field *req;
struct chtls_sock *csk;
struct ulptx_idata *sc;
unsigned int wrlen;
wrlen = roundup(sizeof(*req) + sizeof(*sc), 16);
csk = rcu_dereference_sk_user_data(sk);
req = (struct cpl_set_tcb_field *)__skb_put(skb, wrlen);
__set_tcb_field_direct(csk, req, word, mask, val, cookie, no_reply);
set_wr_txq(skb, CPL_PRIORITY_CONTROL, csk->port_id);
}
/*
* Send control message to HW, message go as immediate data and packet
* is freed immediately.
*/
static int chtls_set_tcb_field(struct sock *sk, u16 word, u64 mask, u64 val)
{
struct cpl_set_tcb_field *req;
unsigned int credits_needed;
struct chtls_sock *csk;
struct ulptx_idata *sc;
struct sk_buff *skb;
unsigned int wrlen;
int ret;
wrlen = roundup(sizeof(*req) + sizeof(*sc), 16);
skb = alloc_skb(wrlen, GFP_ATOMIC);
if (!skb)
return -ENOMEM;
credits_needed = DIV_ROUND_UP(wrlen, 16);
csk = rcu_dereference_sk_user_data(sk);
__set_tcb_field(sk, skb, word, mask, val, 0, 1);
skb_set_queue_mapping(skb, (csk->txq_idx << 1) | CPL_PRIORITY_DATA);
csk->wr_credits -= credits_needed;
csk->wr_unacked += credits_needed;
enqueue_wr(csk, skb);
ret = cxgb4_ofld_send(csk->egress_dev, skb);
if (ret < 0)
kfree_skb(skb);
return ret < 0 ? ret : 0;
}
void chtls_set_tcb_field_rpl_skb(struct sock *sk, u16 word,
u64 mask, u64 val, u8 cookie,
int through_l2t)
{
struct sk_buff *skb;
unsigned int wrlen;
wrlen = sizeof(struct cpl_set_tcb_field) + sizeof(struct ulptx_idata);
wrlen = roundup(wrlen, 16);
skb = alloc_skb(wrlen, GFP_KERNEL | __GFP_NOFAIL);
if (!skb)
return;
__set_tcb_field(sk, skb, word, mask, val, cookie, 0);
send_or_defer(sk, tcp_sk(sk), skb, through_l2t);
}
static int chtls_set_tcb_keyid(struct sock *sk, int keyid)
{
return chtls_set_tcb_field(sk, 31, 0xFFFFFFFFULL, keyid);
}
static int chtls_set_tcb_seqno(struct sock *sk)
{
return chtls_set_tcb_field(sk, 28, ~0ULL, 0);
}
static int chtls_set_tcb_quiesce(struct sock *sk, int val)
{
return chtls_set_tcb_field(sk, 1, (1ULL << TF_RX_QUIESCE_S),
TF_RX_QUIESCE_V(val));
}
void chtls_set_quiesce_ctrl(struct sock *sk, int val)
{
struct chtls_sock *csk;
struct sk_buff *skb;
unsigned int wrlen;
int ret;
wrlen = sizeof(struct cpl_set_tcb_field) + sizeof(struct ulptx_idata);
wrlen = roundup(wrlen, 16);
skb = alloc_skb(wrlen, GFP_ATOMIC);
if (!skb)
return;
csk = rcu_dereference_sk_user_data(sk);
__set_tcb_field(sk, skb, 1, TF_RX_QUIESCE_V(1), 0, 0, 1);
set_wr_txq(skb, CPL_PRIORITY_CONTROL, csk->port_id);
ret = cxgb4_ofld_send(csk->egress_dev, skb);
if (ret < 0)
kfree_skb(skb);
}
/* TLS Key bitmap processing */
int chtls_init_kmap(struct chtls_dev *cdev, struct cxgb4_lld_info *lldi)
{
unsigned int num_key_ctx, bsize;
int ksize;
num_key_ctx = (lldi->vr->key.size / TLS_KEY_CONTEXT_SZ);
bsize = BITS_TO_LONGS(num_key_ctx);
cdev->kmap.size = num_key_ctx;
cdev->kmap.available = bsize;
ksize = sizeof(*cdev->kmap.addr) * bsize;
cdev->kmap.addr = kvzalloc(ksize, GFP_KERNEL);
if (!cdev->kmap.addr)
return -ENOMEM;
cdev->kmap.start = lldi->vr->key.start;
spin_lock_init(&cdev->kmap.lock);
return 0;
}
static int get_new_keyid(struct chtls_sock *csk, u32 optname)
{
struct net_device *dev = csk->egress_dev;
struct chtls_dev *cdev = csk->cdev;
struct chtls_hws *hws;
struct adapter *adap;
int keyid;
adap = netdev2adap(dev);
hws = &csk->tlshws;
spin_lock_bh(&cdev->kmap.lock);
keyid = find_first_zero_bit(cdev->kmap.addr, cdev->kmap.size);
if (keyid < cdev->kmap.size) {
__set_bit(keyid, cdev->kmap.addr);
if (optname == TLS_RX)
hws->rxkey = keyid;
else
hws->txkey = keyid;
atomic_inc(&adap->chcr_stats.tls_key);
} else {
keyid = -1;
}
spin_unlock_bh(&cdev->kmap.lock);
return keyid;
}
void free_tls_keyid(struct sock *sk)
{
struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
struct net_device *dev = csk->egress_dev;
struct chtls_dev *cdev = csk->cdev;
struct chtls_hws *hws;
struct adapter *adap;
if (!cdev->kmap.addr)
return;
adap = netdev2adap(dev);
hws = &csk->tlshws;
spin_lock_bh(&cdev->kmap.lock);
if (hws->rxkey >= 0) {
__clear_bit(hws->rxkey, cdev->kmap.addr);
atomic_dec(&adap->chcr_stats.tls_key);
hws->rxkey = -1;
}
if (hws->txkey >= 0) {
__clear_bit(hws->txkey, cdev->kmap.addr);
atomic_dec(&adap->chcr_stats.tls_key);
hws->txkey = -1;
}
spin_unlock_bh(&cdev->kmap.lock);
}
unsigned int keyid_to_addr(int start_addr, int keyid)
{
return (start_addr + (keyid * TLS_KEY_CONTEXT_SZ)) >> 5;
}
static void chtls_rxkey_ivauth(struct _key_ctx *kctx)
{
kctx->iv_to_auth = cpu_to_be64(KEYCTX_TX_WR_IV_V(6ULL) |
KEYCTX_TX_WR_AAD_V(1ULL) |
KEYCTX_TX_WR_AADST_V(5ULL) |
KEYCTX_TX_WR_CIPHER_V(14ULL) |
KEYCTX_TX_WR_CIPHERST_V(0ULL) |
KEYCTX_TX_WR_AUTH_V(14ULL) |
KEYCTX_TX_WR_AUTHST_V(16ULL) |
KEYCTX_TX_WR_AUTHIN_V(16ULL));
}
static int chtls_key_info(struct chtls_sock *csk,
struct _key_ctx *kctx,
u32 keylen, u32 optname,
int cipher_type)
{
unsigned char key[AES_MAX_KEY_SIZE];
unsigned char *key_p, *salt;
unsigned char ghash_h[AEAD_H_SIZE];
int ck_size, key_ctx_size, kctx_mackey_size, salt_size;
struct aes_enckey aes;
int ret;
key_ctx_size = sizeof(struct _key_ctx) +
roundup(keylen, 16) + AEAD_H_SIZE;
/* GCM mode of AES supports 128 and 256 bit encryption, so
* prepare key context base on GCM cipher type
*/
switch (cipher_type) {
case TLS_CIPHER_AES_GCM_128: {
struct tls12_crypto_info_aes_gcm_128 *gcm_ctx_128 =
(struct tls12_crypto_info_aes_gcm_128 *)
&csk->tlshws.crypto_info;
memcpy(key, gcm_ctx_128->key, keylen);
key_p = gcm_ctx_128->key;
salt = gcm_ctx_128->salt;
ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
salt_size = TLS_CIPHER_AES_GCM_128_SALT_SIZE;
kctx_mackey_size = CHCR_KEYCTX_MAC_KEY_SIZE_128;
break;
}
case TLS_CIPHER_AES_GCM_256: {
struct tls12_crypto_info_aes_gcm_256 *gcm_ctx_256 =
(struct tls12_crypto_info_aes_gcm_256 *)
&csk->tlshws.crypto_info;
memcpy(key, gcm_ctx_256->key, keylen);
key_p = gcm_ctx_256->key;
salt = gcm_ctx_256->salt;
ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
salt_size = TLS_CIPHER_AES_GCM_256_SALT_SIZE;
kctx_mackey_size = CHCR_KEYCTX_MAC_KEY_SIZE_256;
break;
}
default:
pr_err("GCM: Invalid key length %d\n", keylen);
return -EINVAL;
}
/* Calculate the H = CIPH(K, 0 repeated 16 times).
* It will go in key context
*/
ret = aes_prepareenckey(&aes, key, keylen);
if (ret)
return ret;
memset(ghash_h, 0, AEAD_H_SIZE);
aes_encrypt(&aes, ghash_h, ghash_h);
memzero_explicit(&aes, sizeof(aes));
csk->tlshws.keylen = key_ctx_size;
/* Copy the Key context */
if (optname == TLS_RX) {
int key_ctx;
key_ctx = ((key_ctx_size >> 4) << 3);
kctx->ctx_hdr = FILL_KEY_CRX_HDR(ck_size,
kctx_mackey_size,
0, 0, key_ctx);
chtls_rxkey_ivauth(kctx);
} else {
kctx->ctx_hdr = FILL_KEY_CTX_HDR(ck_size,
kctx_mackey_size,
0, 0, key_ctx_size >> 4);
}
memcpy(kctx->salt, salt, salt_size);
memcpy(kctx->key, key_p, keylen);
memcpy(kctx->key + keylen, ghash_h, AEAD_H_SIZE);
/* erase key info from driver */
memset(key_p, 0, keylen);
return 0;
}
static void chtls_set_scmd(struct chtls_sock *csk)
{
struct chtls_hws *hws = &csk->tlshws;
hws->scmd.seqno_numivs =
SCMD_SEQ_NO_CTRL_V(3) |
SCMD_PROTO_VERSION_V(0) |
SCMD_ENC_DEC_CTRL_V(0) |
SCMD_CIPH_AUTH_SEQ_CTRL_V(1) |
SCMD_CIPH_MODE_V(2) |
SCMD_AUTH_MODE_V(4) |
SCMD_HMAC_CTRL_V(0) |
SCMD_IV_SIZE_V(4) |
SCMD_NUM_IVS_V(1);
hws->scmd.ivgen_hdrlen =
SCMD_IV_GEN_CTRL_V(1) |
SCMD_KEY_CTX_INLINE_V(0) |
SCMD_TLS_FRAG_ENABLE_V(1);
}
int chtls_setkey(struct chtls_sock *csk, u32 keylen,
u32 optname, int cipher_type)
{
struct tls_key_req *kwr;
struct chtls_dev *cdev;
struct _key_ctx *kctx;
int wrlen, klen, len;
struct sk_buff *skb;
struct sock *sk;
int keyid;
int kaddr;
int ret;
cdev = csk->cdev;
sk = csk->sk;
klen = roundup((keylen + AEAD_H_SIZE) + sizeof(*kctx), 32);
wrlen = roundup(sizeof(*kwr), 16);
len = klen + wrlen;
/* Flush out-standing data before new key takes effect */
if (optname == TLS_TX) {
lock_sock(sk);
if (skb_queue_len(&csk->txq))
chtls_push_frames(csk, 0);
release_sock(sk);
}
skb = alloc_skb(len, GFP_KERNEL);
if (!skb)
return -ENOMEM;
keyid = get_new_keyid(csk, optname);
if (keyid < 0) {
ret = -ENOSPC;
goto out_nokey;
}
kaddr = keyid_to_addr(cdev->kmap.start, keyid);
kwr = (struct tls_key_req *)__skb_put_zero(skb, len);
kwr->wr.op_to_compl =
cpu_to_be32(FW_WR_OP_V(FW_ULPTX_WR) | FW_WR_COMPL_F |
FW_WR_ATOMIC_V(1U));
kwr->wr.flowid_len16 =
cpu_to_be32(FW_WR_LEN16_V(DIV_ROUND_UP(len, 16) |
FW_WR_FLOWID_V(csk->tid)));
kwr->wr.protocol = 0;
kwr->wr.mfs = htons(TLS_MFS);
kwr->wr.reneg_to_write_rx = optname;
/* ulptx command */
kwr->req.cmd = cpu_to_be32(ULPTX_CMD_V(ULP_TX_MEM_WRITE) |
T5_ULP_MEMIO_ORDER_V(1) |
T5_ULP_MEMIO_IMM_V(1));
kwr->req.len16 = cpu_to_be32((csk->tid << 8) |
DIV_ROUND_UP(len - sizeof(kwr->wr), 16));
kwr->req.dlen = cpu_to_be32(ULP_MEMIO_DATA_LEN_V(klen >> 5));
kwr->req.lock_addr = cpu_to_be32(ULP_MEMIO_ADDR_V(kaddr));
/* sub command */
kwr->sc_imm.cmd_more = cpu_to_be32(ULPTX_CMD_V(ULP_TX_SC_IMM));
kwr->sc_imm.len = cpu_to_be32(klen);
lock_sock(sk);
/* key info */
kctx = (struct _key_ctx *)(kwr + 1);
ret = chtls_key_info(csk, kctx, keylen, optname, cipher_type);
if (ret)
goto out_notcb;
if (unlikely(csk_flag(sk, CSK_ABORT_SHUTDOWN)))
goto out_notcb;
set_wr_txq(skb, CPL_PRIORITY_DATA, csk->tlshws.txqid);
csk->wr_credits -= DIV_ROUND_UP(len, 16);
csk->wr_unacked += DIV_ROUND_UP(len, 16);
enqueue_wr(csk, skb);
cxgb4_ofld_send(csk->egress_dev, skb);
skb = NULL;
chtls_set_scmd(csk);
/* Clear quiesce for Rx key */
if (optname == TLS_RX) {
ret = chtls_set_tcb_keyid(sk, keyid);
if (ret)
goto out_notcb;
ret = chtls_set_tcb_field(sk, 0,
TCB_ULP_RAW_V(TCB_ULP_RAW_M),
TCB_ULP_RAW_V((TF_TLS_KEY_SIZE_V(1) |
TF_TLS_CONTROL_V(1) |
TF_TLS_ACTIVE_V(1) |
TF_TLS_ENABLE_V(1))));
if (ret)
goto out_notcb;
ret = chtls_set_tcb_seqno(sk);
if (ret)
goto out_notcb;
ret = chtls_set_tcb_quiesce(sk, 0);
if (ret)
goto out_notcb;
csk->tlshws.rxkey = keyid;
} else {
csk->tlshws.tx_seq_no = 0;
csk->tlshws.txkey = keyid;
}
release_sock(sk);
return ret;
out_notcb:
release_sock(sk);
free_tls_keyid(sk);
out_nokey:
kfree_skb(skb);
return ret;
}

1836
drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_io.c
View File

File diff suppressed because it is too large Load Diff

642
drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_main.c
View File

@ -1,642 +0,0 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (c) 2018 Chelsio Communications, Inc.
*
* Written by: Atul Gupta (atul.gupta@chelsio.com)
*/
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/skbuff.h>
#include <linux/socket.h>
#include <linux/hash.h>
#include <linux/in.h>
#include <linux/net.h>
#include <linux/ip.h>
#include <linux/tcp.h>
#include <net/ipv6.h>
#include <net/transp_v6.h>
#include <net/tcp.h>
#include <net/tls.h>
#include "chtls.h"
#include "chtls_cm.h"
#define DRV_NAME "chtls"
/*
* chtls device management
* maintains a list of the chtls devices
*/
static LIST_HEAD(cdev_list);
static DEFINE_MUTEX(cdev_mutex);
static DEFINE_MUTEX(notify_mutex);
static RAW_NOTIFIER_HEAD(listen_notify_list);
static struct proto chtls_cpl_prot, chtls_cpl_protv6;
struct request_sock_ops chtls_rsk_ops, chtls_rsk_opsv6;
static uint send_page_order = (14 - PAGE_SHIFT < 0) ? 0 : 14 - PAGE_SHIFT;
static void register_listen_notifier(struct notifier_block *nb)
{
mutex_lock(&notify_mutex);
raw_notifier_chain_register(&listen_notify_list, nb);
mutex_unlock(&notify_mutex);
}
static void unregister_listen_notifier(struct notifier_block *nb)
{
mutex_lock(&notify_mutex);
raw_notifier_chain_unregister(&listen_notify_list, nb);
mutex_unlock(&notify_mutex);
}
static int listen_notify_handler(struct notifier_block *this,
unsigned long event, void *data)
{
struct chtls_listen *clisten;
int ret = NOTIFY_DONE;
clisten = (struct chtls_listen *)data;
switch (event) {
case CHTLS_LISTEN_START:
ret = chtls_listen_start(clisten->cdev, clisten->sk);
kfree(clisten);
break;
case CHTLS_LISTEN_STOP:
chtls_listen_stop(clisten->cdev, clisten->sk);
kfree(clisten);
break;
}
return ret;
}
static struct notifier_block listen_notifier = {
.notifier_call = listen_notify_handler
};
static int listen_backlog_rcv(struct sock *sk, struct sk_buff *skb)
{
if (likely(skb_transport_header(skb) != skb_network_header(skb)))
return tcp_v4_do_rcv(sk, skb);
BLOG_SKB_CB(skb)->backlog_rcv(sk, skb);
return 0;
}
static int chtls_start_listen(struct chtls_dev *cdev, struct sock *sk)
{
struct chtls_listen *clisten;
if (sk->sk_protocol != IPPROTO_TCP)
return -EPROTONOSUPPORT;
if (sk->sk_family == PF_INET &&
LOOPBACK(inet_sk(sk)->inet_rcv_saddr))
return -EADDRNOTAVAIL;
sk->sk_backlog_rcv = listen_backlog_rcv;
clisten = kmalloc_obj(*clisten);
if (!clisten)
return -ENOMEM;
clisten->cdev = cdev;
clisten->sk = sk;
mutex_lock(&notify_mutex);
raw_notifier_call_chain(&listen_notify_list,
CHTLS_LISTEN_START, clisten);
mutex_unlock(&notify_mutex);
return 0;
}
static void chtls_stop_listen(struct chtls_dev *cdev, struct sock *sk)
{
struct chtls_listen *clisten;
if (sk->sk_protocol != IPPROTO_TCP)
return;
clisten = kmalloc_obj(*clisten);
if (!clisten)
return;
clisten->cdev = cdev;
clisten->sk = sk;
mutex_lock(&notify_mutex);
raw_notifier_call_chain(&listen_notify_list,
CHTLS_LISTEN_STOP, clisten);
mutex_unlock(&notify_mutex);
}
static int chtls_inline_feature(struct tls_toe_device *dev)
{
struct net_device *netdev;
struct chtls_dev *cdev;
int i;
cdev = to_chtls_dev(dev);
for (i = 0; i < cdev->lldi->nports; i++) {
netdev = cdev->ports[i];
if (netdev->features & NETIF_F_HW_TLS_RECORD)
return 1;
}
return 0;
}
static int chtls_create_hash(struct tls_toe_device *dev, struct sock *sk)
{
struct chtls_dev *cdev = to_chtls_dev(dev);
if (sk->sk_state == TCP_LISTEN)
return chtls_start_listen(cdev, sk);
return 0;
}
static void chtls_destroy_hash(struct tls_toe_device *dev, struct sock *sk)
{
struct chtls_dev *cdev = to_chtls_dev(dev);
if (sk->sk_state == TCP_LISTEN)
chtls_stop_listen(cdev, sk);
}
static void chtls_free_uld(struct chtls_dev *cdev)
{
int i;
tls_toe_unregister_device(&cdev->tlsdev);
kvfree(cdev->kmap.addr);
idr_destroy(&cdev->hwtid_idr);
for (i = 0; i < (1 << RSPQ_HASH_BITS); i++)
kfree_skb(cdev->rspq_skb_cache[i]);
kfree(cdev->lldi);
kfree_skb(cdev->askb);
kfree(cdev);
}
static inline void chtls_dev_release(struct kref *kref)
{
struct tls_toe_device *dev;
struct chtls_dev *cdev;
struct adapter *adap;
dev = container_of(kref, struct tls_toe_device, kref);
cdev = to_chtls_dev(dev);
/* Reset tls rx/tx stats */
adap = pci_get_drvdata(cdev->pdev);
atomic_set(&adap->chcr_stats.tls_pdu_tx, 0);
atomic_set(&adap->chcr_stats.tls_pdu_rx, 0);
chtls_free_uld(cdev);
}
static void chtls_register_dev(struct chtls_dev *cdev)
{
struct tls_toe_device *tlsdev = &cdev->tlsdev;
strscpy(tlsdev->name, "chtls", TLS_TOE_DEVICE_NAME_MAX);
strlcat(tlsdev->name, cdev->lldi->ports[0]->name,
TLS_TOE_DEVICE_NAME_MAX);
tlsdev->feature = chtls_inline_feature;
tlsdev->hash = chtls_create_hash;
tlsdev->unhash = chtls_destroy_hash;
tlsdev->release = chtls_dev_release;
kref_init(&tlsdev->kref);
tls_toe_register_device(tlsdev);
cdev->cdev_state = CHTLS_CDEV_STATE_UP;
}
static void process_deferq(struct work_struct *task_param)
{
struct chtls_dev *cdev = container_of(task_param,
struct chtls_dev, deferq_task);
struct sk_buff *skb;
spin_lock_bh(&cdev->deferq.lock);
while ((skb = __skb_dequeue(&cdev->deferq)) != NULL) {
spin_unlock_bh(&cdev->deferq.lock);
DEFERRED_SKB_CB(skb)->handler(cdev, skb);
spin_lock_bh(&cdev->deferq.lock);
}
spin_unlock_bh(&cdev->deferq.lock);
}
static int chtls_get_skb(struct chtls_dev *cdev)
{
cdev->askb = alloc_skb(sizeof(struct tcphdr), GFP_KERNEL);
if (!cdev->askb)
return -ENOMEM;
skb_put(cdev->askb, sizeof(struct tcphdr));
skb_reset_transport_header(cdev->askb);
memset(cdev->askb->data, 0, cdev->askb->len);
return 0;
}
static void *chtls_uld_add(const struct cxgb4_lld_info *info)
{
struct cxgb4_lld_info *lldi;
struct chtls_dev *cdev;
int i, j;
cdev = kzalloc_obj(*cdev);
if (!cdev)
goto out;
lldi = kzalloc_obj(*lldi);
if (!lldi)
goto out_lldi;
if (chtls_get_skb(cdev))
goto out_skb;
*lldi = *info;
cdev->lldi = lldi;
cdev->pdev = lldi->pdev;
cdev->tids = lldi->tids;
cdev->ports = lldi->ports;
cdev->mtus = lldi->mtus;
cdev->tids = lldi->tids;
cdev->pfvf = FW_VIID_PFN_G(cxgb4_port_viid(lldi->ports[0]))
<< FW_VIID_PFN_S;
for (i = 0; i < (1 << RSPQ_HASH_BITS); i++) {
unsigned int size = 64 - sizeof(struct rsp_ctrl) - 8;
cdev->rspq_skb_cache[i] = __alloc_skb(size,
gfp_any(), 0,
lldi->nodeid);
if (unlikely(!cdev->rspq_skb_cache[i]))
goto out_rspq_skb;
}
idr_init(&cdev->hwtid_idr);
INIT_WORK(&cdev->deferq_task, process_deferq);
spin_lock_init(&cdev->listen_lock);
spin_lock_init(&cdev->idr_lock);
cdev->send_page_order = min_t(uint, get_order(32768),
send_page_order);
cdev->max_host_sndbuf = 48 * 1024;
if (lldi->vr->key.size)
if (chtls_init_kmap(cdev, lldi))
goto out_rspq_skb;
mutex_lock(&cdev_mutex);
list_add_tail(&cdev->list, &cdev_list);
mutex_unlock(&cdev_mutex);
return cdev;
out_rspq_skb:
for (j = 0; j < i; j++)
kfree_skb(cdev->rspq_skb_cache[j]);
kfree_skb(cdev->askb);
out_skb:
kfree(lldi);
out_lldi:
kfree(cdev);
out:
return NULL;
}
static void chtls_free_all_uld(void)
{
struct chtls_dev *cdev, *tmp;
mutex_lock(&cdev_mutex);
list_for_each_entry_safe(cdev, tmp, &cdev_list, list) {
if (cdev->cdev_state == CHTLS_CDEV_STATE_UP) {
list_del(&cdev->list);
kref_put(&cdev->tlsdev.kref, cdev->tlsdev.release);
}
}
mutex_unlock(&cdev_mutex);
}
static int chtls_uld_state_change(void *handle, enum cxgb4_state new_state)
{
struct chtls_dev *cdev = handle;
switch (new_state) {
case CXGB4_STATE_UP:
chtls_register_dev(cdev);
break;
case CXGB4_STATE_DOWN:
break;
case CXGB4_STATE_START_RECOVERY:
break;
case CXGB4_STATE_DETACH:
mutex_lock(&cdev_mutex);
list_del(&cdev->list);
mutex_unlock(&cdev_mutex);
kref_put(&cdev->tlsdev.kref, cdev->tlsdev.release);
break;
default:
break;
}
return 0;
}
static struct sk_buff *copy_gl_to_skb_pkt(const struct pkt_gl *gl,
const __be64 *rsp,
u32 pktshift)
{
struct sk_buff *skb;
/* Allocate space for cpl_pass_accept_req which will be synthesized by
* driver. Once driver synthesizes cpl_pass_accept_req the skb will go
* through the regular cpl_pass_accept_req processing in TOM.
*/
skb = alloc_skb(size_add(gl->tot_len,
sizeof(struct cpl_pass_accept_req)) -
pktshift, GFP_ATOMIC);
if (unlikely(!skb))
return NULL;
__skb_put(skb, gl->tot_len + sizeof(struct cpl_pass_accept_req)
- pktshift);
/* For now we will copy cpl_rx_pkt in the skb */
skb_copy_to_linear_data(skb, rsp, sizeof(struct cpl_rx_pkt));
skb_copy_to_linear_data_offset(skb, sizeof(struct cpl_pass_accept_req)
, gl->va + pktshift,
gl->tot_len - pktshift);
return skb;
}
static int chtls_recv_packet(struct chtls_dev *cdev,
const struct pkt_gl *gl, const __be64 *rsp)
{
unsigned int opcode = *(u8 *)rsp;
struct sk_buff *skb;
int ret;
skb = copy_gl_to_skb_pkt(gl, rsp, cdev->lldi->sge_pktshift);
if (!skb)
return -ENOMEM;
ret = chtls_handlers[opcode](cdev, skb);
if (ret & CPL_RET_BUF_DONE)
kfree_skb(skb);
return 0;
}
static int chtls_recv_rsp(struct chtls_dev *cdev, const __be64 *rsp)
{
unsigned long rspq_bin;
unsigned int opcode;
struct sk_buff *skb;
unsigned int len;
int ret;
len = 64 - sizeof(struct rsp_ctrl) - 8;
opcode = *(u8 *)rsp;
rspq_bin = hash_ptr((void *)rsp, RSPQ_HASH_BITS);
skb = cdev->rspq_skb_cache[rspq_bin];
if (skb && !skb_is_nonlinear(skb) &&
!skb_shared(skb) && !skb_cloned(skb)) {
refcount_inc(&skb->users);
if (refcount_read(&skb->users) == 2) {
__skb_trim(skb, 0);
if (skb_tailroom(skb) >= len)
goto copy_out;
}
refcount_dec(&skb->users);
}
skb = alloc_skb(len, GFP_ATOMIC);
if (unlikely(!skb))
return -ENOMEM;
copy_out:
__skb_put(skb, len);
skb_copy_to_linear_data(skb, rsp, len);
skb_reset_network_header(skb);
skb_reset_transport_header(skb);
ret = chtls_handlers[opcode](cdev, skb);
if (ret & CPL_RET_BUF_DONE)
kfree_skb(skb);
return 0;
}
static void chtls_recv(struct chtls_dev *cdev,
struct sk_buff **skbs, const __be64 *rsp)
{
struct sk_buff *skb = *skbs;
unsigned int opcode;
int ret;
opcode = *(u8 *)rsp;
__skb_push(skb, sizeof(struct rss_header));
skb_copy_to_linear_data(skb, rsp, sizeof(struct rss_header));
ret = chtls_handlers[opcode](cdev, skb);
if (ret & CPL_RET_BUF_DONE)
kfree_skb(skb);
}
static int chtls_uld_rx_handler(void *handle, const __be64 *rsp,
const struct pkt_gl *gl)
{
struct chtls_dev *cdev = handle;
unsigned int opcode;
struct sk_buff *skb;
opcode = *(u8 *)rsp;
if (unlikely(opcode == CPL_RX_PKT)) {
if (chtls_recv_packet(cdev, gl, rsp) < 0)
goto nomem;
return 0;
}
if (!gl)
return chtls_recv_rsp(cdev, rsp);
#define RX_PULL_LEN 128
skb = cxgb4_pktgl_to_skb(gl, RX_PULL_LEN, RX_PULL_LEN);
if (unlikely(!skb))
goto nomem;
chtls_recv(cdev, &skb, rsp);
return 0;
nomem:
return -ENOMEM;
}
static int do_chtls_getsockopt(struct sock *sk, char __user *optval,
int __user *optlen)
{
struct tls_crypto_info crypto_info = { 0 };
crypto_info.version = TLS_1_2_VERSION;
if (copy_to_user(optval, &crypto_info, sizeof(struct tls_crypto_info)))
return -EFAULT;
return 0;
}
static int chtls_getsockopt(struct sock *sk, int level, int optname,
char __user *optval, int __user *optlen)
{
struct tls_context *ctx = tls_get_ctx(sk);
if (level != SOL_TLS)
return ctx->sk_proto->getsockopt(sk, level,
optname, optval, optlen);
return do_chtls_getsockopt(sk, optval, optlen);
}
static int do_chtls_setsockopt(struct sock *sk, int optname,
sockptr_t optval, unsigned int optlen)
{
struct tls_crypto_info *crypto_info, tmp_crypto_info;
struct chtls_sock *csk;
int keylen;
int cipher_type;
int rc = 0;
csk = rcu_dereference_sk_user_data(sk);
if (sockptr_is_null(optval) || optlen < sizeof(*crypto_info)) {
rc = -EINVAL;
goto out;
}
rc = copy_from_sockptr(&tmp_crypto_info, optval, sizeof(*crypto_info));
if (rc) {
rc = -EFAULT;
goto out;
}
/* check version */
if (tmp_crypto_info.version != TLS_1_2_VERSION) {
rc = -ENOTSUPP;
goto out;
}
crypto_info = (struct tls_crypto_info *)&csk->tlshws.crypto_info;
/* GCM mode of AES supports 128 and 256 bit encryption, so
* copy keys from user based on GCM cipher type.
*/
switch (tmp_crypto_info.cipher_type) {
case TLS_CIPHER_AES_GCM_128: {
/* Obtain version and type from previous copy */
crypto_info[0] = tmp_crypto_info;
/* Now copy the following data */
rc = copy_from_sockptr_offset((char *)crypto_info +
sizeof(*crypto_info),
optval, sizeof(*crypto_info),
sizeof(struct tls12_crypto_info_aes_gcm_128)
- sizeof(*crypto_info));
if (rc) {
rc = -EFAULT;
goto out;
}
keylen = TLS_CIPHER_AES_GCM_128_KEY_SIZE;
cipher_type = TLS_CIPHER_AES_GCM_128;
break;
}
case TLS_CIPHER_AES_GCM_256: {
crypto_info[0] = tmp_crypto_info;
rc = copy_from_sockptr_offset((char *)crypto_info +
sizeof(*crypto_info),
optval, sizeof(*crypto_info),
sizeof(struct tls12_crypto_info_aes_gcm_256)
- sizeof(*crypto_info));
if (rc) {
rc = -EFAULT;
goto out;
}
keylen = TLS_CIPHER_AES_GCM_256_KEY_SIZE;
cipher_type = TLS_CIPHER_AES_GCM_256;
break;
}
default:
rc = -EINVAL;
goto out;
}
rc = chtls_setkey(csk, keylen, optname, cipher_type);
out:
return rc;
}
static int chtls_setsockopt(struct sock *sk, int level, int optname,
sockptr_t optval, unsigned int optlen)
{
struct tls_context *ctx = tls_get_ctx(sk);
if (level != SOL_TLS)
return ctx->sk_proto->setsockopt(sk, level,
optname, optval, optlen);
return do_chtls_setsockopt(sk, optname, optval, optlen);
}
static struct cxgb4_uld_info chtls_uld_info = {
.name = DRV_NAME,
.nrxq = MAX_ULD_QSETS,
.ntxq = MAX_ULD_QSETS,
.rxq_size = 1024,
.add = chtls_uld_add,
.state_change = chtls_uld_state_change,
.rx_handler = chtls_uld_rx_handler,
};
void chtls_install_cpl_ops(struct sock *sk)
{
if (sk->sk_family == AF_INET)
sk->sk_prot = &chtls_cpl_prot;
else
sk->sk_prot = &chtls_cpl_protv6;
}
static void __init chtls_init_ulp_ops(void)
{
chtls_cpl_prot = tcp_prot;
chtls_init_rsk_ops(&chtls_cpl_prot, &chtls_rsk_ops,
&tcp_prot, PF_INET);
chtls_cpl_prot.close = chtls_close;
chtls_cpl_prot.disconnect = chtls_disconnect;
chtls_cpl_prot.destroy = chtls_destroy_sock;
chtls_cpl_prot.shutdown = chtls_shutdown;
chtls_cpl_prot.sendmsg = chtls_sendmsg;
chtls_cpl_prot.splice_eof = chtls_splice_eof;
chtls_cpl_prot.recvmsg = chtls_recvmsg;
chtls_cpl_prot.setsockopt = chtls_setsockopt;
chtls_cpl_prot.getsockopt = chtls_getsockopt;
#if IS_ENABLED(CONFIG_IPV6)
chtls_cpl_protv6 = chtls_cpl_prot;
chtls_init_rsk_ops(&chtls_cpl_protv6, &chtls_rsk_opsv6,
&tcpv6_prot, PF_INET6);
#endif
}
static int __init chtls_register(void)
{
chtls_init_ulp_ops();
register_listen_notifier(&listen_notifier);
cxgb4_register_uld(CXGB4_ULD_TLS, &chtls_uld_info);
return 0;
}
static void __exit chtls_unregister(void)
{
unregister_listen_notifier(&listen_notifier);
chtls_free_all_uld();
cxgb4_unregister_uld(CXGB4_ULD_TLS);
}
module_init(chtls_register);
module_exit(chtls_unregister);
MODULE_DESCRIPTION("Chelsio TLS Inline driver");
MODULE_LICENSE("GPL");
MODULE_AUTHOR("Chelsio Communications");
MODULE_VERSION(CHTLS_DRV_VERSION);

3
include/linux/netdev_features.h
View File

@ -79,7 +79,7 @@ enum {
NETIF_F_HW_TLS_RX_BIT, /* Hardware TLS RX offload */
NETIF_F_GRO_HW_BIT, /* Hardware Generic receive offload */
NETIF_F_HW_TLS_RECORD_BIT, /* Offload TLS record */
__UNUSED_NETIF_F_56,
NETIF_F_GRO_FRAGLIST_BIT, /* Fraglist GRO */
NETIF_F_HW_MACSEC_BIT, /* Offload MACsec operations */
@ -153,7 +153,6 @@ enum {
#define NETIF_F_HW_ESP __NETIF_F(HW_ESP)
#define NETIF_F_HW_ESP_TX_CSUM __NETIF_F(HW_ESP_TX_CSUM)
#define NETIF_F_RX_UDP_TUNNEL_PORT __NETIF_F(RX_UDP_TUNNEL_PORT)
#define NETIF_F_HW_TLS_RECORD __NETIF_F(HW_TLS_RECORD)
#define NETIF_F_GSO_UDP_L4 __NETIF_F(GSO_UDP_L4)
#define NETIF_F_HW_TLS_TX __NETIF_F(HW_TLS_TX)
#define NETIF_F_HW_TLS_RX __NETIF_F(HW_TLS_RX)

1
include/net/tls.h
View File

@ -85,7 +85,6 @@ enum {
TLS_BASE,
TLS_SW,
TLS_HW,
TLS_HW_RECORD,
TLS_NUM_CONFIG,
};

77
include/net/tls_toe.h
View File

@ -1,77 +0,0 @@
/*
* Copyright (c) 2016-2017, Mellanox Technologies. All rights reserved.
* Copyright (c) 2016-2017, Dave Watson <davejwatson@fb.com>. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#include <linux/kref.h>
#include <linux/list.h>
struct sock;
#define TLS_TOE_DEVICE_NAME_MAX 32
/*
* This structure defines the routines for Inline TLS driver.
* The following routines are optional and filled with a
* null pointer if not defined.
*
* @name: Its the name of registered Inline tls device
* @dev_list: Inline tls device list
* int (*feature)(struct tls_toe_device *device);
* Called to return Inline TLS driver capability
*
* int (*hash)(struct tls_toe_device *device, struct sock *sk);
* This function sets Inline driver for listen and program
* device specific functioanlity as required
*
* void (*unhash)(struct tls_toe_device *device, struct sock *sk);
* This function cleans listen state set by Inline TLS driver
*
* void (*release)(struct kref *kref);
* Release the registered device and allocated resources
* @kref: Number of reference to tls_toe_device
*/
struct tls_toe_device {
char name[TLS_TOE_DEVICE_NAME_MAX];
struct list_head dev_list;
int (*feature)(struct tls_toe_device *device);
int (*hash)(struct tls_toe_device *device, struct sock *sk);
void (*unhash)(struct tls_toe_device *device, struct sock *sk);
void (*release)(struct kref *kref);
struct kref kref;
};
int tls_toe_bypass(struct sock *sk);
int tls_toe_hash(struct sock *sk);
void tls_toe_unhash(struct sock *sk);
void tls_toe_register_device(struct tls_toe_device *device);
void tls_toe_unregister_device(struct tls_toe_device *device);

2
include/uapi/linux/tls.h
View File

@ -203,6 +203,6 @@ enum {
#define TLS_CONF_BASE 1
#define TLS_CONF_SW 2
#define TLS_CONF_HW 3
#define TLS_CONF_HW_RECORD 4
#define TLS_CONF_HW_RECORD 4 /* unused */
#endif /* _UAPI_LINUX_TLS_H */

1
net/core/stream.c
View File

@ -112,7 +112,6 @@ void sk_stream_wait_close(struct sock *sk, long timeout)
remove_wait_queue(sk_sleep(sk), &wait);
}
}
EXPORT_SYMBOL(sk_stream_wait_close);
/**
* sk_stream_wait_memory - Wait for more memory for a socket

1
net/ethtool/common.c
View File

@ -67,7 +67,6 @@ const char netdev_features_strings[NETDEV_FEATURE_COUNT][ETH_GSTRING_LEN] = {
[NETIF_F_HW_ESP_BIT] = "esp-hw-offload",
[NETIF_F_HW_ESP_TX_CSUM_BIT] = "esp-tx-csum-hw-offload",
[NETIF_F_RX_UDP_TUNNEL_PORT_BIT] = "rx-udp_tunnel-port-offload",
[NETIF_F_HW_TLS_RECORD_BIT] = "tls-hw-record",
[NETIF_F_HW_TLS_TX_BIT] = "tls-hw-tx-offload",
[NETIF_F_HW_TLS_RX_BIT] = "tls-hw-rx-offload",
[NETIF_F_GRO_FRAGLIST_BIT] = "rx-gro-list",

6
net/ipv4/inet_connection_sock.c
View File

@ -824,7 +824,6 @@ struct dst_entry *inet_csk_route_child_sock(const struct sock *sk,
__IP_INC_STATS(net, IPSTATS_MIB_OUTNOROUTES);
return NULL;
}
EXPORT_SYMBOL_GPL(inet_csk_route_child_sock);
/* Decide when to expire the request and when to resend SYN-ACK */
static void syn_ack_recalc(struct request_sock *req,
@ -901,7 +900,6 @@ struct request_sock *inet_reqsk_alloc(const struct request_sock_ops *ops,
return req;
}
EXPORT_SYMBOL(inet_reqsk_alloc);
void __reqsk_free(struct request_sock *req)
{
@ -911,7 +909,6 @@ void __reqsk_free(struct request_sock *req)
kfree(req->saved_syn);
kmem_cache_free(req->rsk_ops->slab, req);
}
EXPORT_SYMBOL_GPL(__reqsk_free);
static struct request_sock *inet_reqsk_clone(struct request_sock *req,
struct sock *sk)
@ -1280,7 +1277,6 @@ void inet_csk_destroy_sock(struct sock *sk)
sock_put(sk);
}
EXPORT_SYMBOL(inet_csk_destroy_sock);
void inet_csk_prepare_for_destroy_sock(struct sock *sk)
{
@ -1301,7 +1297,6 @@ void inet_csk_prepare_forced_close(struct sock *sk)
inet_csk_prepare_for_destroy_sock(sk);
inet_sk(sk)->inet_num = 0;
}
EXPORT_SYMBOL(inet_csk_prepare_forced_close);
static int inet_ulp_can_listen(const struct sock *sk)
{
@ -1405,7 +1400,6 @@ struct sock *inet_csk_reqsk_queue_add(struct sock *sk,
spin_unlock(&queue->rskq_lock);
return child;
}
EXPORT_SYMBOL(inet_csk_reqsk_queue_add);
struct sock *inet_csk_complete_hashdance(struct sock *sk, struct sock *child,
struct request_sock *req, bool own_req)

2
net/ipv4/inet_hashtables.c
View File

@ -252,7 +252,6 @@ void inet_put_port(struct sock *sk)
__inet_put_port(sk);
local_bh_enable();
}
EXPORT_SYMBOL(inet_put_port);
int __inet_inherit_port(const struct sock *sk, struct sock *child)
{
@ -331,7 +330,6 @@ int __inet_inherit_port(const struct sock *sk, struct sock *child)
spin_unlock(&head->lock);
return -ENOMEM;
}
EXPORT_SYMBOL_GPL(__inet_inherit_port);
static struct inet_listen_hashbucket *
inet_lhash2_bucket_sk(struct inet_hashinfo *h, struct sock *sk)

4
net/ipv4/tcp.c
View File

@ -297,7 +297,6 @@ enum {
};
DEFINE_PER_CPU(unsigned int, tcp_orphan_count);
EXPORT_PER_CPU_SYMBOL_GPL(tcp_orphan_count);
long sysctl_tcp_mem[3] __read_mostly;
@ -3027,7 +3026,6 @@ void tcp_set_state(struct sock *sk, int state)
*/
inet_sk_state_store(sk, state);
}
EXPORT_SYMBOL_GPL(tcp_set_state);
/*
* State processing on a close. This implements the state shift for
@ -3506,7 +3504,6 @@ int tcp_disconnect(struct sock *sk, int flags)
sk_error_report(sk);
return 0;
}
EXPORT_SYMBOL(tcp_disconnect);
static inline bool tcp_can_repair_sock(const struct sock *sk)
{
@ -5104,7 +5101,6 @@ void tcp_done(struct sock *sk)
else
inet_csk_destroy_sock(sk);
}
EXPORT_SYMBOL_GPL(tcp_done);
int tcp_abort(struct sock *sk, int err)
{

1
net/ipv4/tcp_ipv4.c
View File

@ -1894,7 +1894,6 @@ int tcp_v4_do_rcv(struct sock *sk, struct sk_buff *skb)
TCP_INC_STATS(sock_net(sk), TCP_MIB_INERRS);
goto discard;
}
EXPORT_SYMBOL(tcp_v4_do_rcv);
enum skb_drop_reason tcp_add_backlog(struct sock *sk, struct sk_buff *skb)
{

2
net/ipv4/tcp_minisocks.c
View File

@ -392,7 +392,6 @@ void tcp_time_wait(struct sock *sk, int state, int timeo)
tcp_update_metrics(sk);
tcp_done(sk);
}
EXPORT_SYMBOL(tcp_time_wait);
void tcp_twsk_destructor(struct sock *sk)
{
@ -670,7 +669,6 @@ struct sock *tcp_create_openreq_child(const struct sock *sk,
return newsk;
}
EXPORT_SYMBOL(tcp_create_openreq_child);
/*
* Process an incoming packet for SYN_RECV sockets represented as a

10
net/tls/Kconfig
View File

@ -27,13 +27,3 @@ config TLS_DEVICE
Enable kernel support for HW offload of the TLS protocol.
If unsure, say N.
config TLS_TOE
bool "Transport Layer Security TCP stack bypass"
depends on TLS
default n
help
Enable kernel support for legacy HW offload of the TLS protocol,
which is incompatible with the Linux networking stack semantics.
If unsure, say N.

1
net/tls/Makefile
View File

@ -9,5 +9,4 @@ obj-$(CONFIG_TLS) += tls.o
tls-y := tls_main.o tls_sw.o tls_proc.o trace.o tls_strp.o
tls-$(CONFIG_TLS_TOE) += tls_toe.o
tls-$(CONFIG_TLS_DEVICE) += tls_device.o tls_device_fallback.o

17
net/tls/tls_main.c
View File

@ -43,8 +43,6 @@
#include <net/snmp.h>
#include <net/tls.h>
#include <net/tls_toe.h>
#include "tls.h"
MODULE_AUTHOR("Mellanox Technologies");
@ -963,9 +961,6 @@ static void build_proto_ops(struct proto_ops ops[TLS_NUM_CONFIG][TLS_NUM_CONFIG]
ops[TLS_HW ][TLS_HW ] = ops[TLS_HW ][TLS_SW ];
#endif
#ifdef CONFIG_TLS_TOE
ops[TLS_HW_RECORD][TLS_HW_RECORD] = *base;
#endif
}
static void tls_build_proto(struct sock *sk)
@ -1037,11 +1032,6 @@ static void build_protos(struct proto prot[TLS_NUM_CONFIG][TLS_NUM_CONFIG],
prot[TLS_HW][TLS_HW] = prot[TLS_HW][TLS_SW];
#endif
#ifdef CONFIG_TLS_TOE
prot[TLS_HW_RECORD][TLS_HW_RECORD] = *base;
prot[TLS_HW_RECORD][TLS_HW_RECORD].hash = tls_toe_hash;
prot[TLS_HW_RECORD][TLS_HW_RECORD].unhash = tls_toe_unhash;
#endif
}
static int tls_init(struct sock *sk)
@ -1051,11 +1041,6 @@ static int tls_init(struct sock *sk)
tls_build_proto(sk);
#ifdef CONFIG_TLS_TOE
if (tls_toe_bypass(sk))
return 0;
#endif
/* The TLS ulp is currently supported only for TCP sockets
* in ESTABLISHED state.
* Supporting sockets in LISTEN state will require us
@ -1111,8 +1096,6 @@ static u16 tls_user_config(struct tls_context *ctx, bool tx)
return TLS_CONF_SW;
case TLS_HW:
return TLS_CONF_HW;
case TLS_HW_RECORD:
return TLS_CONF_HW_RECORD;
}
return 0;
}

141
net/tls/tls_toe.c
View File

@ -1,141 +0,0 @@
/*
* Copyright (c) 2016-2017, Mellanox Technologies. All rights reserved.
* Copyright (c) 2016-2017, Dave Watson <davejwatson@fb.com>. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#include <linux/list.h>
#include <linux/rcupdate.h>
#include <linux/spinlock.h>
#include <net/inet_connection_sock.h>
#include <net/tls.h>
#include <net/tls_toe.h>
#include "tls.h"
static LIST_HEAD(device_list);
static DEFINE_SPINLOCK(device_spinlock);
static void tls_toe_sk_destruct(struct sock *sk)
{
struct inet_connection_sock *icsk = inet_csk(sk);
struct tls_context *ctx = tls_get_ctx(sk);
ctx->sk_destruct(sk);
/* Free ctx */
rcu_assign_pointer(icsk->icsk_ulp_data, NULL);
tls_ctx_free(sk, ctx);
}
int tls_toe_bypass(struct sock *sk)
{
struct tls_toe_device *dev;
struct tls_context *ctx;
int rc = 0;
spin_lock_bh(&device_spinlock);
list_for_each_entry(dev, &device_list, dev_list) {
if (dev->feature && dev->feature(dev)) {
ctx = tls_ctx_create(sk);
if (!ctx)
goto out;
ctx->sk_destruct = sk->sk_destruct;
sk->sk_destruct = tls_toe_sk_destruct;
ctx->rx_conf = TLS_HW_RECORD;
ctx->tx_conf = TLS_HW_RECORD;
update_sk_prot(sk, ctx);
rc = 1;
break;
}
}
out:
spin_unlock_bh(&device_spinlock);
return rc;
}
void tls_toe_unhash(struct sock *sk)
{
struct tls_context *ctx = tls_get_ctx(sk);
struct tls_toe_device *dev;
spin_lock_bh(&device_spinlock);
list_for_each_entry(dev, &device_list, dev_list) {
if (dev->unhash) {
kref_get(&dev->kref);
spin_unlock_bh(&device_spinlock);
dev->unhash(dev, sk);
kref_put(&dev->kref, dev->release);
spin_lock_bh(&device_spinlock);
}
}
spin_unlock_bh(&device_spinlock);
ctx->sk_proto->unhash(sk);
}
int tls_toe_hash(struct sock *sk)
{
struct tls_context *ctx = tls_get_ctx(sk);
struct tls_toe_device *dev;
int err;
err = ctx->sk_proto->hash(sk);
spin_lock_bh(&device_spinlock);
list_for_each_entry(dev, &device_list, dev_list) {
if (dev->hash) {
kref_get(&dev->kref);
spin_unlock_bh(&device_spinlock);
err |= dev->hash(dev, sk);
kref_put(&dev->kref, dev->release);
spin_lock_bh(&device_spinlock);
}
}
spin_unlock_bh(&device_spinlock);
if (err)
tls_toe_unhash(sk);
return err;
}
void tls_toe_register_device(struct tls_toe_device *device)
{
spin_lock_bh(&device_spinlock);
list_add_tail(&device->dev_list, &device_list);
spin_unlock_bh(&device_spinlock);
}
EXPORT_SYMBOL(tls_toe_register_device);
void tls_toe_unregister_device(struct tls_toe_device *device)
{
spin_lock_bh(&device_spinlock);
list_del(&device->dev_list);
spin_unlock_bh(&device_spinlock);
}
EXPORT_SYMBOL(tls_toe_unregister_device);