2 * Copyright (c) 2018 Mellanox Technologies. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
34 #include <linux/mlx5/device.h>
38 #include "fpga/core.h"
39 #include "accel/tls.h"
41 struct mlx5_fpga_tls_command_context;
43 typedef void (*mlx5_fpga_tls_command_complete)
44 (struct mlx5_fpga_conn *conn, struct mlx5_fpga_device *fdev,
45 struct mlx5_fpga_tls_command_context *ctx,
46 struct mlx5_fpga_dma_buf *resp);
48 struct mlx5_fpga_tls_command_context {
49 struct list_head list;
50 /* There is no guarantee on the order between the TX completion
51 * and the command response.
52 * The TX completion is going to touch cmd->buf even in
53 * the case of successful transmission.
54 * So instead of requiring separate allocations for cmd
55 * and cmd->buf we've decided to use a reference counter
58 struct mlx5_fpga_dma_buf buf;
59 mlx5_fpga_tls_command_complete complete;
63 mlx5_fpga_tls_put_command_ctx(struct mlx5_fpga_tls_command_context *ctx)
65 if (refcount_dec_and_test(&ctx->ref))
69 static void mlx5_fpga_tls_cmd_complete(struct mlx5_fpga_device *fdev,
70 struct mlx5_fpga_dma_buf *resp)
72 struct mlx5_fpga_conn *conn = fdev->tls->conn;
73 struct mlx5_fpga_tls_command_context *ctx;
74 struct mlx5_fpga_tls *tls = fdev->tls;
77 spin_lock_irqsave(&tls->pending_cmds_lock, flags);
78 ctx = list_first_entry(&tls->pending_cmds,
79 struct mlx5_fpga_tls_command_context, list);
81 spin_unlock_irqrestore(&tls->pending_cmds_lock, flags);
82 ctx->complete(conn, fdev, ctx, resp);
85 static void mlx5_fpga_cmd_send_complete(struct mlx5_fpga_conn *conn,
86 struct mlx5_fpga_device *fdev,
87 struct mlx5_fpga_dma_buf *buf,
90 struct mlx5_fpga_tls_command_context *ctx =
91 container_of(buf, struct mlx5_fpga_tls_command_context, buf);
93 mlx5_fpga_tls_put_command_ctx(ctx);
96 mlx5_fpga_tls_cmd_complete(fdev, NULL);
99 static void mlx5_fpga_tls_cmd_send(struct mlx5_fpga_device *fdev,
100 struct mlx5_fpga_tls_command_context *cmd,
101 mlx5_fpga_tls_command_complete complete)
103 struct mlx5_fpga_tls *tls = fdev->tls;
107 refcount_set(&cmd->ref, 2);
108 cmd->complete = complete;
109 cmd->buf.complete = mlx5_fpga_cmd_send_complete;
111 spin_lock_irqsave(&tls->pending_cmds_lock, flags);
112 /* mlx5_fpga_sbu_conn_sendmsg is called under pending_cmds_lock
113 * to make sure commands are inserted to the tls->pending_cmds list
114 * and the command QP in the same order.
116 ret = mlx5_fpga_sbu_conn_sendmsg(tls->conn, &cmd->buf);
118 list_add_tail(&cmd->list, &tls->pending_cmds);
120 complete(tls->conn, fdev, cmd, NULL);
121 spin_unlock_irqrestore(&tls->pending_cmds_lock, flags);
124 /* Start of context identifiers range (inclusive) */
126 /* End of context identifiers range (exclusive) */
127 #define SWID_END BIT(24)
129 static int mlx5_fpga_tls_alloc_swid(struct idr *idr, spinlock_t *idr_spinlock,
135 /* TLS metadata format is 1 byte for syndrome followed
136 * by 3 bytes of swid (software ID)
137 * swid must not exceed 3 bytes.
138 * See tls_rxtx.c:insert_pet() for details
140 BUILD_BUG_ON((SWID_END - 1) & 0xFF000000);
142 idr_preload(GFP_KERNEL);
143 spin_lock_irqsave(idr_spinlock, flags);
144 ret = idr_alloc(idr, ptr, SWID_START, SWID_END, GFP_ATOMIC);
145 spin_unlock_irqrestore(idr_spinlock, flags);
151 static void mlx5_fpga_tls_release_swid(struct idr *idr,
152 spinlock_t *idr_spinlock, u32 swid)
156 spin_lock_irqsave(idr_spinlock, flags);
157 idr_remove(idr, swid);
158 spin_unlock_irqrestore(idr_spinlock, flags);
161 static void mlx_tls_kfree_complete(struct mlx5_fpga_conn *conn,
162 struct mlx5_fpga_device *fdev,
163 struct mlx5_fpga_dma_buf *buf, u8 status)
168 struct mlx5_teardown_stream_context {
169 struct mlx5_fpga_tls_command_context cmd;
174 mlx5_fpga_tls_teardown_completion(struct mlx5_fpga_conn *conn,
175 struct mlx5_fpga_device *fdev,
176 struct mlx5_fpga_tls_command_context *cmd,
177 struct mlx5_fpga_dma_buf *resp)
179 struct mlx5_teardown_stream_context *ctx =
180 container_of(cmd, struct mlx5_teardown_stream_context, cmd);
183 u32 syndrome = MLX5_GET(tls_resp, resp->sg[0].data, syndrome);
187 "Teardown stream failed with syndrome = %d",
189 else if (MLX5_GET(tls_cmd, cmd->buf.sg[0].data, direction_sx))
190 mlx5_fpga_tls_release_swid(&fdev->tls->tx_idr,
191 &fdev->tls->tx_idr_spinlock,
194 mlx5_fpga_tls_release_swid(&fdev->tls->rx_idr,
195 &fdev->tls->rx_idr_spinlock,
198 mlx5_fpga_tls_put_command_ctx(cmd);
201 static void mlx5_fpga_tls_flow_to_cmd(void *flow, void *cmd)
203 memcpy(MLX5_ADDR_OF(tls_cmd, cmd, src_port), flow,
204 MLX5_BYTE_OFF(tls_flow, ipv6));
206 MLX5_SET(tls_cmd, cmd, ipv6, MLX5_GET(tls_flow, flow, ipv6));
207 MLX5_SET(tls_cmd, cmd, direction_sx,
208 MLX5_GET(tls_flow, flow, direction_sx));
211 int mlx5_fpga_tls_resync_rx(struct mlx5_core_dev *mdev, u32 handle, u32 seq,
214 struct mlx5_fpga_dma_buf *buf;
215 int size = sizeof(*buf) + MLX5_TLS_COMMAND_SIZE;
220 buf = kzalloc(size, GFP_ATOMIC);
227 flow = idr_find(&mdev->fpga->tls->rx_idr, ntohl(handle));
229 mlx5_fpga_tls_flow_to_cmd(flow, cmd);
231 MLX5_SET(tls_cmd, cmd, swid, ntohl(handle));
232 MLX5_SET64(tls_cmd, cmd, tls_rcd_sn, be64_to_cpu(rcd_sn));
233 MLX5_SET(tls_cmd, cmd, tcp_sn, seq);
234 MLX5_SET(tls_cmd, cmd, command_type, CMD_RESYNC_RX);
236 buf->sg[0].data = cmd;
237 buf->sg[0].size = MLX5_TLS_COMMAND_SIZE;
238 buf->complete = mlx_tls_kfree_complete;
240 ret = mlx5_fpga_sbu_conn_sendmsg(mdev->fpga->tls->conn, buf);
245 static void mlx5_fpga_tls_send_teardown_cmd(struct mlx5_core_dev *mdev,
246 void *flow, u32 swid, gfp_t flags)
248 struct mlx5_teardown_stream_context *ctx;
249 struct mlx5_fpga_dma_buf *buf;
252 ctx = kzalloc(sizeof(*ctx) + MLX5_TLS_COMMAND_SIZE, flags);
258 MLX5_SET(tls_cmd, cmd, command_type, CMD_TEARDOWN_STREAM);
259 MLX5_SET(tls_cmd, cmd, swid, swid);
261 mlx5_fpga_tls_flow_to_cmd(flow, cmd);
264 buf->sg[0].data = cmd;
265 buf->sg[0].size = MLX5_TLS_COMMAND_SIZE;
268 mlx5_fpga_tls_cmd_send(mdev->fpga, &ctx->cmd,
269 mlx5_fpga_tls_teardown_completion);
272 void mlx5_fpga_tls_del_flow(struct mlx5_core_dev *mdev, u32 swid,
273 gfp_t flags, bool direction_sx)
275 struct mlx5_fpga_tls *tls = mdev->fpga->tls;
280 flow = idr_find(&tls->tx_idr, swid);
282 flow = idr_find(&tls->rx_idr, swid);
287 mlx5_fpga_err(mdev->fpga, "No flow information for swid %u\n",
292 mlx5_fpga_tls_send_teardown_cmd(mdev, flow, swid, flags);
295 enum mlx5_fpga_setup_stream_status {
296 MLX5_FPGA_CMD_PENDING,
297 MLX5_FPGA_CMD_SEND_FAILED,
298 MLX5_FPGA_CMD_RESPONSE_RECEIVED,
299 MLX5_FPGA_CMD_ABANDONED,
302 struct mlx5_setup_stream_context {
303 struct mlx5_fpga_tls_command_context cmd;
306 struct completion comp;
310 mlx5_fpga_tls_setup_completion(struct mlx5_fpga_conn *conn,
311 struct mlx5_fpga_device *fdev,
312 struct mlx5_fpga_tls_command_context *cmd,
313 struct mlx5_fpga_dma_buf *resp)
315 struct mlx5_setup_stream_context *ctx =
316 container_of(cmd, struct mlx5_setup_stream_context, cmd);
317 int status = MLX5_FPGA_CMD_SEND_FAILED;
318 void *tls_cmd = ctx + 1;
320 /* If we failed to send to command resp == NULL */
322 ctx->syndrome = MLX5_GET(tls_resp, resp->sg[0].data, syndrome);
323 status = MLX5_FPGA_CMD_RESPONSE_RECEIVED;
326 status = atomic_xchg_release(&ctx->status, status);
327 if (likely(status != MLX5_FPGA_CMD_ABANDONED)) {
328 complete(&ctx->comp);
332 mlx5_fpga_err(fdev, "Command was abandoned, syndrome = %u\n",
335 if (!ctx->syndrome) {
336 /* The process was killed while waiting for the context to be
337 * added, and the add completed successfully.
338 * We need to destroy the HW context, and we can't can't reuse
339 * the command context because we might not have received
340 * the tx completion yet.
342 mlx5_fpga_tls_del_flow(fdev->mdev,
343 MLX5_GET(tls_cmd, tls_cmd, swid),
345 MLX5_GET(tls_cmd, tls_cmd,
349 mlx5_fpga_tls_put_command_ctx(cmd);
352 static int mlx5_fpga_tls_setup_stream_cmd(struct mlx5_core_dev *mdev,
353 struct mlx5_setup_stream_context *ctx)
355 struct mlx5_fpga_dma_buf *buf;
360 buf->sg[0].data = cmd;
361 buf->sg[0].size = MLX5_TLS_COMMAND_SIZE;
362 MLX5_SET(tls_cmd, cmd, command_type, CMD_SETUP_STREAM);
364 init_completion(&ctx->comp);
365 atomic_set(&ctx->status, MLX5_FPGA_CMD_PENDING);
368 mlx5_fpga_tls_cmd_send(mdev->fpga, &ctx->cmd,
369 mlx5_fpga_tls_setup_completion);
370 wait_for_completion_killable(&ctx->comp);
372 status = atomic_xchg_acquire(&ctx->status, MLX5_FPGA_CMD_ABANDONED);
373 if (unlikely(status == MLX5_FPGA_CMD_PENDING))
374 /* ctx is going to be released in mlx5_fpga_tls_setup_completion */
377 if (unlikely(ctx->syndrome))
380 mlx5_fpga_tls_put_command_ctx(&ctx->cmd);
384 static void mlx5_fpga_tls_hw_qp_recv_cb(void *cb_arg,
385 struct mlx5_fpga_dma_buf *buf)
387 struct mlx5_fpga_device *fdev = (struct mlx5_fpga_device *)cb_arg;
389 mlx5_fpga_tls_cmd_complete(fdev, buf);
392 bool mlx5_fpga_is_tls_device(struct mlx5_core_dev *mdev)
394 if (!mdev->fpga || !MLX5_CAP_GEN(mdev, fpga))
397 if (MLX5_CAP_FPGA(mdev, ieee_vendor_id) !=
398 MLX5_FPGA_CAP_SANDBOX_VENDOR_ID_MLNX)
401 if (MLX5_CAP_FPGA(mdev, sandbox_product_id) !=
402 MLX5_FPGA_CAP_SANDBOX_PRODUCT_ID_TLS)
405 if (MLX5_CAP_FPGA(mdev, sandbox_product_version) != 0)
411 static int mlx5_fpga_tls_get_caps(struct mlx5_fpga_device *fdev,
414 int err, cap_size = MLX5_ST_SZ_BYTES(tls_extended_cap);
418 buf = kzalloc(cap_size, GFP_KERNEL);
422 err = mlx5_fpga_get_sbu_caps(fdev, cap_size, buf);
426 if (MLX5_GET(tls_extended_cap, buf, tx))
427 caps |= MLX5_ACCEL_TLS_TX;
428 if (MLX5_GET(tls_extended_cap, buf, rx))
429 caps |= MLX5_ACCEL_TLS_RX;
430 if (MLX5_GET(tls_extended_cap, buf, tls_v12))
431 caps |= MLX5_ACCEL_TLS_V12;
432 if (MLX5_GET(tls_extended_cap, buf, tls_v13))
433 caps |= MLX5_ACCEL_TLS_V13;
434 if (MLX5_GET(tls_extended_cap, buf, lro))
435 caps |= MLX5_ACCEL_TLS_LRO;
436 if (MLX5_GET(tls_extended_cap, buf, ipv6))
437 caps |= MLX5_ACCEL_TLS_IPV6;
439 if (MLX5_GET(tls_extended_cap, buf, aes_gcm_128))
440 caps |= MLX5_ACCEL_TLS_AES_GCM128;
441 if (MLX5_GET(tls_extended_cap, buf, aes_gcm_256))
442 caps |= MLX5_ACCEL_TLS_AES_GCM256;
451 int mlx5_fpga_tls_init(struct mlx5_core_dev *mdev)
453 struct mlx5_fpga_device *fdev = mdev->fpga;
454 struct mlx5_fpga_conn_attr init_attr = {0};
455 struct mlx5_fpga_conn *conn;
456 struct mlx5_fpga_tls *tls;
459 if (!mlx5_fpga_is_tls_device(mdev) || !fdev)
462 tls = kzalloc(sizeof(*tls), GFP_KERNEL);
466 err = mlx5_fpga_tls_get_caps(fdev, &tls->caps);
470 if (!(tls->caps & (MLX5_ACCEL_TLS_V12 | MLX5_ACCEL_TLS_AES_GCM128))) {
475 init_attr.rx_size = SBU_QP_QUEUE_SIZE;
476 init_attr.tx_size = SBU_QP_QUEUE_SIZE;
477 init_attr.recv_cb = mlx5_fpga_tls_hw_qp_recv_cb;
478 init_attr.cb_arg = fdev;
479 conn = mlx5_fpga_sbu_conn_create(fdev, &init_attr);
482 mlx5_fpga_err(fdev, "Error creating TLS command connection %d\n",
488 spin_lock_init(&tls->pending_cmds_lock);
489 INIT_LIST_HEAD(&tls->pending_cmds);
491 idr_init(&tls->tx_idr);
492 idr_init(&tls->rx_idr);
493 spin_lock_init(&tls->tx_idr_spinlock);
494 spin_lock_init(&tls->rx_idr_spinlock);
503 void mlx5_fpga_tls_cleanup(struct mlx5_core_dev *mdev)
505 struct mlx5_fpga_device *fdev = mdev->fpga;
507 if (!fdev || !fdev->tls)
510 mlx5_fpga_sbu_conn_destroy(fdev->tls->conn);
515 static void mlx5_fpga_tls_set_aes_gcm128_ctx(void *cmd,
516 struct tls_crypto_info *info,
519 struct tls12_crypto_info_aes_gcm_128 *crypto_info =
520 (struct tls12_crypto_info_aes_gcm_128 *)info;
522 memcpy(MLX5_ADDR_OF(tls_cmd, cmd, tls_rcd_sn), crypto_info->rec_seq,
523 TLS_CIPHER_AES_GCM_128_REC_SEQ_SIZE);
525 memcpy(MLX5_ADDR_OF(tls_cmd, cmd, tls_implicit_iv),
526 crypto_info->salt, TLS_CIPHER_AES_GCM_128_SALT_SIZE);
527 memcpy(MLX5_ADDR_OF(tls_cmd, cmd, encryption_key),
528 crypto_info->key, TLS_CIPHER_AES_GCM_128_KEY_SIZE);
530 /* in AES-GCM 128 we need to write the key twice */
531 memcpy(MLX5_ADDR_OF(tls_cmd, cmd, encryption_key) +
532 TLS_CIPHER_AES_GCM_128_KEY_SIZE,
533 crypto_info->key, TLS_CIPHER_AES_GCM_128_KEY_SIZE);
535 MLX5_SET(tls_cmd, cmd, alg, MLX5_TLS_ALG_AES_GCM_128);
538 static int mlx5_fpga_tls_set_key_material(void *cmd, u32 caps,
539 struct tls_crypto_info *crypto_info)
543 switch (crypto_info->cipher_type) {
544 case TLS_CIPHER_AES_GCM_128:
545 if (!(caps & MLX5_ACCEL_TLS_AES_GCM128))
547 mlx5_fpga_tls_set_aes_gcm128_ctx(cmd, crypto_info, &rcd_sn);
556 static int _mlx5_fpga_tls_add_flow(struct mlx5_core_dev *mdev, void *flow,
557 struct tls_crypto_info *crypto_info,
558 u32 swid, u32 tcp_sn)
560 u32 caps = mlx5_fpga_tls_device_caps(mdev);
561 struct mlx5_setup_stream_context *ctx;
566 cmd_size = MLX5_TLS_COMMAND_SIZE + sizeof(*ctx);
567 ctx = kzalloc(cmd_size, GFP_KERNEL);
572 ret = mlx5_fpga_tls_set_key_material(cmd, caps, crypto_info);
576 mlx5_fpga_tls_flow_to_cmd(flow, cmd);
578 MLX5_SET(tls_cmd, cmd, swid, swid);
579 MLX5_SET(tls_cmd, cmd, tcp_sn, tcp_sn);
581 return mlx5_fpga_tls_setup_stream_cmd(mdev, ctx);
589 int mlx5_fpga_tls_add_flow(struct mlx5_core_dev *mdev, void *flow,
590 struct tls_crypto_info *crypto_info,
591 u32 start_offload_tcp_sn, u32 *p_swid,
594 struct mlx5_fpga_tls *tls = mdev->fpga->tls;
599 ret = mlx5_fpga_tls_alloc_swid(&tls->tx_idr,
600 &tls->tx_idr_spinlock, flow);
602 ret = mlx5_fpga_tls_alloc_swid(&tls->rx_idr,
603 &tls->rx_idr_spinlock, flow);
609 MLX5_SET(tls_flow, flow, direction_sx, direction_sx ? 1 : 0);
611 ret = _mlx5_fpga_tls_add_flow(mdev, flow, crypto_info, swid,
612 start_offload_tcp_sn);
613 if (ret && ret != -EINTR)
620 mlx5_fpga_tls_release_swid(&tls->tx_idr,
621 &tls->tx_idr_spinlock, swid);
623 mlx5_fpga_tls_release_swid(&tls->rx_idr,
624 &tls->rx_idr_spinlock, swid);