drivers/net/ethernet/mellanox/mlx5/core/accel/ipsec_offload.c

   1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIBt
   2 /* Copyright (c) 2020, Mellanox Technologies inc. All rights reserved. */
   3
   4 #include "mlx5_core.h"
   5 #include "ipsec_offload.h"
   6 #include "lib/mlx5.h"
   7 #include "en_accel/ipsec_fs.h"
   8
   9 #define MLX5_IPSEC_DEV_BASIC_CAPS (MLX5_ACCEL_IPSEC_CAP_DEVICE | MLX5_ACCEL_IPSEC_CAP_IPV6 | \
  10                                    MLX5_ACCEL_IPSEC_CAP_LSO)
  11
  12 struct mlx5_ipsec_sa_ctx {
  13         struct rhash_head hash;
  14         u32 enc_key_id;
  15         u32 ipsec_obj_id;
  16         /* hw ctx */
  17         struct mlx5_core_dev *dev;
  18         struct mlx5_ipsec_esp_xfrm *mxfrm;
  19 };
  20
  21 struct mlx5_ipsec_esp_xfrm {
  22         /* reference counter of SA ctx */
  23         struct mlx5_ipsec_sa_ctx *sa_ctx;
  24         struct mutex lock; /* protects mlx5_ipsec_esp_xfrm */
  25         struct mlx5_accel_esp_xfrm accel_xfrm;
  26 };
  27
  28 static u32 mlx5_ipsec_offload_device_caps(struct mlx5_core_dev *mdev)
  29 {
  30         u32 caps = MLX5_IPSEC_DEV_BASIC_CAPS;
  31
  32         if (!mlx5_is_ipsec_device(mdev))
  33                 return 0;
  34
  35         if (!MLX5_CAP_FLOWTABLE_NIC_TX(mdev, ipsec_encrypt) ||
  36             !MLX5_CAP_FLOWTABLE_NIC_RX(mdev, ipsec_decrypt))
  37                 return 0;
  38
  39         if (MLX5_CAP_IPSEC(mdev, ipsec_crypto_esp_aes_gcm_128_encrypt) &&
  40             MLX5_CAP_IPSEC(mdev, ipsec_crypto_esp_aes_gcm_128_decrypt))
  41                 caps |= MLX5_ACCEL_IPSEC_CAP_ESP;
  42
  43         if (MLX5_CAP_IPSEC(mdev, ipsec_esn)) {
  44                 caps |= MLX5_ACCEL_IPSEC_CAP_ESN;
  45                 caps |= MLX5_ACCEL_IPSEC_CAP_TX_IV_IS_ESN;
  46         }
  47
  48         /* We can accommodate up to 2^24 different IPsec objects
  49          * because we use up to 24 bit in flow table metadata
  50          * to hold the IPsec Object unique handle.
  51          */
  52         WARN_ON_ONCE(MLX5_CAP_IPSEC(mdev, log_max_ipsec_offload) > 24);
  53         return caps;
  54 }
  55
  56 static int
  57 mlx5_ipsec_offload_esp_validate_xfrm_attrs(struct mlx5_core_dev *mdev,
  58                                            const struct mlx5_accel_esp_xfrm_attrs *attrs)
  59 {
  60         if (attrs->replay_type != MLX5_ACCEL_ESP_REPLAY_NONE) {
  61                 mlx5_core_err(mdev, "Cannot offload xfrm states with anti replay (replay_type = %d)\n",
  62                               attrs->replay_type);
  63                 return -EOPNOTSUPP;
  64         }
  65
  66         if (attrs->keymat_type != MLX5_ACCEL_ESP_KEYMAT_AES_GCM) {
  67                 mlx5_core_err(mdev, "Only aes gcm keymat is supported (keymat_type = %d)\n",
  68                               attrs->keymat_type);
  69                 return -EOPNOTSUPP;
  70         }
  71
  72         if (attrs->keymat.aes_gcm.iv_algo !=
  73             MLX5_ACCEL_ESP_AES_GCM_IV_ALGO_SEQ) {
  74                 mlx5_core_err(mdev, "Only iv sequence algo is supported (iv_algo = %d)\n",
  75                               attrs->keymat.aes_gcm.iv_algo);
  76                 return -EOPNOTSUPP;
  77         }
  78
  79         if (attrs->keymat.aes_gcm.key_len != 128 &&
  80             attrs->keymat.aes_gcm.key_len != 256) {
  81                 mlx5_core_err(mdev, "Cannot offload xfrm states with key length other than 128/256 bit (key length = %d)\n",
  82                               attrs->keymat.aes_gcm.key_len);
  83                 return -EOPNOTSUPP;
  84         }
  85
  86         if ((attrs->flags & MLX5_ACCEL_ESP_FLAGS_ESN_TRIGGERED) &&
  87             !MLX5_CAP_IPSEC(mdev, ipsec_esn)) {
  88                 mlx5_core_err(mdev, "Cannot offload xfrm states with ESN triggered\n");
  89                 return -EOPNOTSUPP;
  90         }
  91
  92         return 0;
  93 }
  94
  95 static struct mlx5_accel_esp_xfrm *
  96 mlx5_ipsec_offload_esp_create_xfrm(struct mlx5_core_dev *mdev,
  97                                    const struct mlx5_accel_esp_xfrm_attrs *attrs,
  98                                    u32 flags)
  99 {
 100         struct mlx5_ipsec_esp_xfrm *mxfrm;
 101         int err = 0;
 102
 103         err = mlx5_ipsec_offload_esp_validate_xfrm_attrs(mdev, attrs);
 104         if (err)
 105                 return ERR_PTR(err);
 106
 107         mxfrm = kzalloc(sizeof(*mxfrm), GFP_KERNEL);
 108         if (!mxfrm)
 109                 return ERR_PTR(-ENOMEM);
 110
 111         mutex_init(&mxfrm->lock);
 112         memcpy(&mxfrm->accel_xfrm.attrs, attrs,
 113                sizeof(mxfrm->accel_xfrm.attrs));
 114
 115         return &mxfrm->accel_xfrm;
 116 }
 117
 118 static void mlx5_ipsec_offload_esp_destroy_xfrm(struct mlx5_accel_esp_xfrm *xfrm)
 119 {
 120         struct mlx5_ipsec_esp_xfrm *mxfrm = container_of(xfrm, struct mlx5_ipsec_esp_xfrm,
 121                                                          accel_xfrm);
 122
 123         /* assuming no sa_ctx are connected to this xfrm_ctx */
 124         WARN_ON(mxfrm->sa_ctx);
 125         kfree(mxfrm);
 126 }
 127
 128 struct mlx5_ipsec_obj_attrs {
 129         const struct aes_gcm_keymat *aes_gcm;
 130         u32 accel_flags;
 131         u32 esn_msb;
 132         u32 enc_key_id;
 133 };
 134
 135 static int mlx5_create_ipsec_obj(struct mlx5_core_dev *mdev,
 136                                  struct mlx5_ipsec_obj_attrs *attrs,
 137                                  u32 *ipsec_id)
 138 {
 139         const struct aes_gcm_keymat *aes_gcm = attrs->aes_gcm;
 140         u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
 141         u32 in[MLX5_ST_SZ_DW(create_ipsec_obj_in)] = {};
 142         void *obj, *salt_p, *salt_iv_p;
 143         int err;
 144
 145         obj = MLX5_ADDR_OF(create_ipsec_obj_in, in, ipsec_object);
 146
 147         /* salt and seq_iv */
 148         salt_p = MLX5_ADDR_OF(ipsec_obj, obj, salt);
 149         memcpy(salt_p, &aes_gcm->salt, sizeof(aes_gcm->salt));
 150
 151         switch (aes_gcm->icv_len) {
 152         case 64:
 153                 MLX5_SET(ipsec_obj, obj, icv_length,
 154                          MLX5_IPSEC_OBJECT_ICV_LEN_8B);
 155                 break;
 156         case 96:
 157                 MLX5_SET(ipsec_obj, obj, icv_length,
 158                          MLX5_IPSEC_OBJECT_ICV_LEN_12B);
 159                 break;
 160         case 128:
 161                 MLX5_SET(ipsec_obj, obj, icv_length,
 162                          MLX5_IPSEC_OBJECT_ICV_LEN_16B);
 163                 break;
 164         default:
 165                 return -EINVAL;
 166         }
 167         salt_iv_p = MLX5_ADDR_OF(ipsec_obj, obj, implicit_iv);
 168         memcpy(salt_iv_p, &aes_gcm->seq_iv, sizeof(aes_gcm->seq_iv));
 169         /* esn */
 170         if (attrs->accel_flags & MLX5_ACCEL_ESP_FLAGS_ESN_TRIGGERED) {
 171                 MLX5_SET(ipsec_obj, obj, esn_en, 1);
 172                 MLX5_SET(ipsec_obj, obj, esn_msb, attrs->esn_msb);
 173                 if (attrs->accel_flags & MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP)
 174                         MLX5_SET(ipsec_obj, obj, esn_overlap, 1);
 175         }
 176
 177         MLX5_SET(ipsec_obj, obj, dekn, attrs->enc_key_id);
 178
 179         /* general object fields set */
 180         MLX5_SET(general_obj_in_cmd_hdr, in, opcode,
 181                  MLX5_CMD_OP_CREATE_GENERAL_OBJECT);
 182         MLX5_SET(general_obj_in_cmd_hdr, in, obj_type,
 183                  MLX5_GENERAL_OBJECT_TYPES_IPSEC);
 184
 185         err = mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out));
 186         if (!err)
 187                 *ipsec_id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);
 188
 189         return err;
 190 }
 191
 192 static void mlx5_destroy_ipsec_obj(struct mlx5_core_dev *mdev, u32 ipsec_id)
 193 {
 194         u32 in[MLX5_ST_SZ_DW(general_obj_in_cmd_hdr)] = {};
 195         u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
 196
 197         MLX5_SET(general_obj_in_cmd_hdr, in, opcode,
 198                  MLX5_CMD_OP_DESTROY_GENERAL_OBJECT);
 199         MLX5_SET(general_obj_in_cmd_hdr, in, obj_type,
 200                  MLX5_GENERAL_OBJECT_TYPES_IPSEC);
 201         MLX5_SET(general_obj_in_cmd_hdr, in, obj_id, ipsec_id);
 202
 203         mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out));
 204 }
 205
 206 static void *mlx5_ipsec_offload_create_sa_ctx(struct mlx5_core_dev *mdev,
 207                                               struct mlx5_accel_esp_xfrm *accel_xfrm,
 208                                               const __be32 saddr[4], const __be32 daddr[4],
 209                                               const __be32 spi, bool is_ipv6, u32 *hw_handle)
 210 {
 211         struct mlx5_accel_esp_xfrm_attrs *xfrm_attrs = &accel_xfrm->attrs;
 212         struct aes_gcm_keymat *aes_gcm = &xfrm_attrs->keymat.aes_gcm;
 213         struct mlx5_ipsec_obj_attrs ipsec_attrs = {};
 214         struct mlx5_ipsec_esp_xfrm *mxfrm;
 215         struct mlx5_ipsec_sa_ctx *sa_ctx;
 216         int err;
 217
 218         /* alloc SA context */
 219         sa_ctx = kzalloc(sizeof(*sa_ctx), GFP_KERNEL);
 220         if (!sa_ctx)
 221                 return ERR_PTR(-ENOMEM);
 222
 223         sa_ctx->dev = mdev;
 224
 225         mxfrm = container_of(accel_xfrm, struct mlx5_ipsec_esp_xfrm, accel_xfrm);
 226         mutex_lock(&mxfrm->lock);
 227         sa_ctx->mxfrm = mxfrm;
 228
 229         /* key */
 230         err = mlx5_create_encryption_key(mdev, aes_gcm->aes_key,
 231                                          aes_gcm->key_len / BITS_PER_BYTE,
 232                                          MLX5_ACCEL_OBJ_IPSEC_KEY,
 233                                          &sa_ctx->enc_key_id);
 234         if (err) {
 235                 mlx5_core_dbg(mdev, "Failed to create encryption key (err = %d)\n", err);
 236                 goto err_sa_ctx;
 237         }
 238
 239         ipsec_attrs.aes_gcm = aes_gcm;
 240         ipsec_attrs.accel_flags = accel_xfrm->attrs.flags;
 241         ipsec_attrs.esn_msb = accel_xfrm->attrs.esn;
 242         ipsec_attrs.enc_key_id = sa_ctx->enc_key_id;
 243         err = mlx5_create_ipsec_obj(mdev, &ipsec_attrs,
 244                                     &sa_ctx->ipsec_obj_id);
 245         if (err) {
 246                 mlx5_core_dbg(mdev, "Failed to create IPsec object (err = %d)\n", err);
 247                 goto err_enc_key;
 248         }
 249
 250         *hw_handle = sa_ctx->ipsec_obj_id;
 251         mxfrm->sa_ctx = sa_ctx;
 252         mutex_unlock(&mxfrm->lock);
 253
 254         return sa_ctx;
 255
 256 err_enc_key:
 257         mlx5_destroy_encryption_key(mdev, sa_ctx->enc_key_id);
 258 err_sa_ctx:
 259         mutex_unlock(&mxfrm->lock);
 260         kfree(sa_ctx);
 261         return ERR_PTR(err);
 262 }
 263
 264 static void mlx5_ipsec_offload_delete_sa_ctx(void *context)
 265 {
 266         struct mlx5_ipsec_sa_ctx *sa_ctx = (struct mlx5_ipsec_sa_ctx *)context;
 267         struct mlx5_ipsec_esp_xfrm *mxfrm = sa_ctx->mxfrm;
 268
 269         mutex_lock(&mxfrm->lock);
 270         mlx5_destroy_ipsec_obj(sa_ctx->dev, sa_ctx->ipsec_obj_id);
 271         mlx5_destroy_encryption_key(sa_ctx->dev, sa_ctx->enc_key_id);
 272         kfree(sa_ctx);
 273         mxfrm->sa_ctx = NULL;
 274         mutex_unlock(&mxfrm->lock);
 275 }
 276
 277 static int mlx5_ipsec_offload_init(struct mlx5_core_dev *mdev)
 278 {
 279         return 0;
 280 }
 281
 282 static int mlx5_modify_ipsec_obj(struct mlx5_core_dev *mdev,
 283                                  struct mlx5_ipsec_obj_attrs *attrs,
 284                                  u32 ipsec_id)
 285 {
 286         u32 in[MLX5_ST_SZ_DW(modify_ipsec_obj_in)] = {};
 287         u32 out[MLX5_ST_SZ_DW(query_ipsec_obj_out)];
 288         u64 modify_field_select = 0;
 289         u64 general_obj_types;
 290         void *obj;
 291         int err;
 292
 293         if (!(attrs->accel_flags & MLX5_ACCEL_ESP_FLAGS_ESN_TRIGGERED))
 294                 return 0;
 295
 296         general_obj_types = MLX5_CAP_GEN_64(mdev, general_obj_types);
 297         if (!(general_obj_types & MLX5_HCA_CAP_GENERAL_OBJECT_TYPES_IPSEC))
 298                 return -EINVAL;
 299
 300         /* general object fields set */
 301         MLX5_SET(general_obj_in_cmd_hdr, in, opcode, MLX5_CMD_OP_QUERY_GENERAL_OBJECT);
 302         MLX5_SET(general_obj_in_cmd_hdr, in, obj_type, MLX5_GENERAL_OBJECT_TYPES_IPSEC);
 303         MLX5_SET(general_obj_in_cmd_hdr, in, obj_id, ipsec_id);
 304         err = mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out));
 305         if (err) {
 306                 mlx5_core_err(mdev, "Query IPsec object failed (Object id %d), err = %d\n",
 307                               ipsec_id, err);
 308                 return err;
 309         }
 310
 311         obj = MLX5_ADDR_OF(query_ipsec_obj_out, out, ipsec_object);
 312         modify_field_select = MLX5_GET64(ipsec_obj, obj, modify_field_select);
 313
 314         /* esn */
 315         if (!(modify_field_select & MLX5_MODIFY_IPSEC_BITMASK_ESN_OVERLAP) ||
 316             !(modify_field_select & MLX5_MODIFY_IPSEC_BITMASK_ESN_MSB))
 317                 return -EOPNOTSUPP;
 318
 319         obj = MLX5_ADDR_OF(modify_ipsec_obj_in, in, ipsec_object);
 320         MLX5_SET(ipsec_obj, obj, esn_msb, attrs->esn_msb);
 321         if (attrs->accel_flags & MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP)
 322                 MLX5_SET(ipsec_obj, obj, esn_overlap, 1);
 323
 324         /* general object fields set */
 325         MLX5_SET(general_obj_in_cmd_hdr, in, opcode, MLX5_CMD_OP_MODIFY_GENERAL_OBJECT);
 326
 327         return mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out));
 328 }
 329
 330 static int mlx5_ipsec_offload_esp_modify_xfrm(struct mlx5_accel_esp_xfrm *xfrm,
 331                                               const struct mlx5_accel_esp_xfrm_attrs *attrs)
 332 {
 333         struct mlx5_ipsec_obj_attrs ipsec_attrs = {};
 334         struct mlx5_core_dev *mdev = xfrm->mdev;
 335         struct mlx5_ipsec_esp_xfrm *mxfrm;
 336
 337         int err = 0;
 338
 339         if (!memcmp(&xfrm->attrs, attrs, sizeof(xfrm->attrs)))
 340                 return 0;
 341
 342         if (mlx5_ipsec_offload_esp_validate_xfrm_attrs(mdev, attrs))
 343                 return -EOPNOTSUPP;
 344
 345         mxfrm = container_of(xfrm, struct mlx5_ipsec_esp_xfrm, accel_xfrm);
 346
 347         mutex_lock(&mxfrm->lock);
 348
 349         if (!mxfrm->sa_ctx)
 350                 /* Not bound xfrm, change only sw attrs */
 351                 goto change_sw_xfrm_attrs;
 352
 353         /* need to add find and replace in ipsec_rhash_sa the sa_ctx */
 354         /* modify device with new hw_sa */
 355         ipsec_attrs.accel_flags = attrs->flags;
 356         ipsec_attrs.esn_msb = attrs->esn;
 357         err = mlx5_modify_ipsec_obj(mdev,
 358                                     &ipsec_attrs,
 359                                     mxfrm->sa_ctx->ipsec_obj_id);
 360
 361 change_sw_xfrm_attrs:
 362         if (!err)
 363                 memcpy(&xfrm->attrs, attrs, sizeof(xfrm->attrs));
 364
 365         mutex_unlock(&mxfrm->lock);
 366         return err;
 367 }
 368
 369 static const struct mlx5_accel_ipsec_ops ipsec_offload_ops = {
 370         .device_caps = mlx5_ipsec_offload_device_caps,
 371         .create_hw_context = mlx5_ipsec_offload_create_sa_ctx,
 372         .free_hw_context = mlx5_ipsec_offload_delete_sa_ctx,
 373         .init = mlx5_ipsec_offload_init,
 374         .esp_create_xfrm = mlx5_ipsec_offload_esp_create_xfrm,
 375         .esp_destroy_xfrm = mlx5_ipsec_offload_esp_destroy_xfrm,
 376         .esp_modify_xfrm = mlx5_ipsec_offload_esp_modify_xfrm,
 377 };
 378
 379 const struct mlx5_accel_ipsec_ops *mlx5_ipsec_offload_ops(struct mlx5_core_dev *mdev)
 380 {
 381         if (!mlx5_ipsec_offload_device_caps(mdev))
 382                 return NULL;
 383
 384         return &ipsec_offload_ops;
 385 }