diff options
Diffstat (limited to 'drivers')
20 files changed, 1542 insertions, 445 deletions
diff --git a/drivers/net/ethernet/chelsio/inline_crypto/ch_ipsec/chcr_ipsec.c b/drivers/net/ethernet/chelsio/inline_crypto/ch_ipsec/chcr_ipsec.c index 585590520076..ca21794281d6 100644 --- a/drivers/net/ethernet/chelsio/inline_crypto/ch_ipsec/chcr_ipsec.c +++ b/drivers/net/ethernet/chelsio/inline_crypto/ch_ipsec/chcr_ipsec.c @@ -283,6 +283,10 @@ static int ch_ipsec_xfrm_add_state(struct xfrm_state *x) pr_debug("Cannot offload xfrm states with geniv other than seqiv\n"); return -EINVAL; } + if (x->xso.type != XFRM_DEV_OFFLOAD_CRYPTO) { + pr_debug("Unsupported xfrm offload\n"); + return -EINVAL; + } sa_entry = kzalloc(sizeof(*sa_entry), GFP_KERNEL); if (!sa_entry) { diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_ipsec.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_ipsec.c index 774de63dd93a..53a969e34883 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_ipsec.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_ipsec.c @@ -585,6 +585,11 @@ static int ixgbe_ipsec_add_sa(struct xfrm_state *xs) return -EINVAL; } + if (xs->xso.type != XFRM_DEV_OFFLOAD_CRYPTO) { + netdev_err(dev, "Unsupported ipsec offload type\n"); + return -EINVAL; + } + if (xs->xso.dir == XFRM_DEV_OFFLOAD_IN) { struct rx_sa rsa; diff --git a/drivers/net/ethernet/intel/ixgbevf/ipsec.c b/drivers/net/ethernet/intel/ixgbevf/ipsec.c index 9984ebc62d78..c1cf540d162a 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ipsec.c +++ b/drivers/net/ethernet/intel/ixgbevf/ipsec.c @@ -280,6 +280,11 @@ static int ixgbevf_ipsec_add_sa(struct xfrm_state *xs) return -EINVAL; } + if (xs->xso.type != XFRM_DEV_OFFLOAD_CRYPTO) { + netdev_err(dev, "Unsupported ipsec offload type\n"); + return -EINVAL; + } + if (xs->xso.dir == XFRM_DEV_OFFLOAD_IN) { struct rx_sa rsa; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en.h b/drivers/net/ethernet/mellanox/mlx5/core/en.h index 65790ff58a74..2d77fb8a8a01 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en.h @@ -1245,4 +1245,5 @@ int mlx5e_set_vf_rate(struct net_device *dev, int vf, int min_tx_rate, int max_t int mlx5e_get_vf_config(struct net_device *dev, int vf, struct ifla_vf_info *ivi); int mlx5e_get_vf_stats(struct net_device *dev, int vf, struct ifla_vf_stats *vf_stats); #endif +int mlx5e_create_mkey(struct mlx5_core_dev *mdev, u32 pdn, u32 *mkey); #endif /* __MLX5_EN_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/fs.h b/drivers/net/ethernet/mellanox/mlx5/core/en/fs.h index bf2741eb7f9b..379c6dc9a3be 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/fs.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/fs.h @@ -84,7 +84,8 @@ enum { MLX5E_ARFS_FT_LEVEL = MLX5E_INNER_TTC_FT_LEVEL + 1, #endif #ifdef CONFIG_MLX5_EN_IPSEC - MLX5E_ACCEL_FS_ESP_FT_LEVEL = MLX5E_INNER_TTC_FT_LEVEL + 1, + MLX5E_ACCEL_FS_POL_FT_LEVEL = MLX5E_INNER_TTC_FT_LEVEL + 1, + MLX5E_ACCEL_FS_ESP_FT_LEVEL, MLX5E_ACCEL_FS_ESP_FT_ERR_LEVEL, #endif }; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/meter.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/meter.c index 9c1c24da9453..78af8a3175bf 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/meter.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/meter.c @@ -162,7 +162,6 @@ mlx5e_tc_meter_modify(struct mlx5_core_dev *mdev, MLX5_ACCESS_ASO_OPC_MOD_FLOW_METER); aso_ctrl = &aso_wqe->aso_ctrl; - memset(aso_ctrl, 0, sizeof(*aso_ctrl)); aso_ctrl->data_mask_mode = MLX5_ASO_DATA_MASK_MODE_BYTEWISE_64BYTE << 6; aso_ctrl->condition_1_0_operand = MLX5_ASO_ALWAYS_TRUE | MLX5_ASO_ALWAYS_TRUE << 4; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c index 1b03ab03fc5a..bb9023957f74 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c @@ -45,55 +45,9 @@ static struct mlx5e_ipsec_sa_entry *to_ipsec_sa_entry(struct xfrm_state *x) return (struct mlx5e_ipsec_sa_entry *)x->xso.offload_handle; } -struct xfrm_state *mlx5e_ipsec_sadb_rx_lookup(struct mlx5e_ipsec *ipsec, - unsigned int handle) +static struct mlx5e_ipsec_pol_entry *to_ipsec_pol_entry(struct xfrm_policy *x) { - struct mlx5e_ipsec_sa_entry *sa_entry; - struct xfrm_state *ret = NULL; - - rcu_read_lock(); - hash_for_each_possible_rcu(ipsec->sadb_rx, sa_entry, hlist, handle) - if (sa_entry->handle == handle) { - ret = sa_entry->x; - xfrm_state_hold(ret); - break; - } - rcu_read_unlock(); - - return ret; -} - -static int mlx5e_ipsec_sadb_rx_add(struct mlx5e_ipsec_sa_entry *sa_entry) -{ - unsigned int handle = sa_entry->ipsec_obj_id; - struct mlx5e_ipsec *ipsec = sa_entry->ipsec; - struct mlx5e_ipsec_sa_entry *_sa_entry; - unsigned long flags; - - rcu_read_lock(); - hash_for_each_possible_rcu(ipsec->sadb_rx, _sa_entry, hlist, handle) - if (_sa_entry->handle == handle) { - rcu_read_unlock(); - return -EEXIST; - } - rcu_read_unlock(); - - spin_lock_irqsave(&ipsec->sadb_rx_lock, flags); - sa_entry->handle = handle; - hash_add_rcu(ipsec->sadb_rx, &sa_entry->hlist, sa_entry->handle); - spin_unlock_irqrestore(&ipsec->sadb_rx_lock, flags); - - return 0; -} - -static void mlx5e_ipsec_sadb_rx_del(struct mlx5e_ipsec_sa_entry *sa_entry) -{ - struct mlx5e_ipsec *ipsec = sa_entry->ipsec; - unsigned long flags; - - spin_lock_irqsave(&ipsec->sadb_rx_lock, flags); - hash_del_rcu(&sa_entry->hlist); - spin_unlock_irqrestore(&ipsec->sadb_rx_lock, flags); + return (struct mlx5e_ipsec_pol_entry *)x->xdo.offload_handle; } static bool mlx5e_ipsec_update_esn_state(struct mlx5e_ipsec_sa_entry *sa_entry) @@ -129,9 +83,33 @@ static bool mlx5e_ipsec_update_esn_state(struct mlx5e_ipsec_sa_entry *sa_entry) return false; } -static void -mlx5e_ipsec_build_accel_xfrm_attrs(struct mlx5e_ipsec_sa_entry *sa_entry, - struct mlx5_accel_esp_xfrm_attrs *attrs) +static void mlx5e_ipsec_init_limits(struct mlx5e_ipsec_sa_entry *sa_entry, + struct mlx5_accel_esp_xfrm_attrs *attrs) +{ + struct xfrm_state *x = sa_entry->x; + + attrs->hard_packet_limit = x->lft.hard_packet_limit; + if (x->lft.soft_packet_limit == XFRM_INF) + return; + + /* Hardware decrements hard_packet_limit counter through + * the operation. While fires an event when soft_packet_limit + * is reached. It emans that we need substitute the numbers + * in order to properly count soft limit. + * + * As an example: + * XFRM user sets soft limit is 2 and hard limit is 9 and + * expects to see soft event after 2 packets and hard event + * after 9 packets. In our case, the hard limit will be set + * to 9 and soft limit is comparator to 7 so user gets the + * soft event after 2 packeta + */ + attrs->soft_packet_limit = + x->lft.hard_packet_limit - x->lft.soft_packet_limit; +} + +void mlx5e_ipsec_build_accel_xfrm_attrs(struct mlx5e_ipsec_sa_entry *sa_entry, + struct mlx5_accel_esp_xfrm_attrs *attrs) { struct xfrm_state *x = sa_entry->x; struct aes_gcm_keymat *aes_gcm = &attrs->aes_gcm; @@ -157,33 +135,31 @@ mlx5e_ipsec_build_accel_xfrm_attrs(struct mlx5e_ipsec_sa_entry *sa_entry, memcpy(&aes_gcm->salt, x->aead->alg_key + key_len, sizeof(aes_gcm->salt)); + attrs->authsize = crypto_aead_authsize(aead) / 4; /* in dwords */ + /* iv len */ aes_gcm->icv_len = x->aead->alg_icv_len; /* esn */ if (sa_entry->esn_state.trigger) { - attrs->flags |= MLX5_ACCEL_ESP_FLAGS_ESN_TRIGGERED; + attrs->esn_trigger = true; attrs->esn = sa_entry->esn_state.esn; - if (sa_entry->esn_state.overlap) - attrs->flags |= MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP; + attrs->esn_overlap = sa_entry->esn_state.overlap; + attrs->replay_window = x->replay_esn->replay_window; } - /* action */ - attrs->action = (x->xso.dir == XFRM_DEV_OFFLOAD_OUT) ? - MLX5_ACCEL_ESP_ACTION_ENCRYPT : - MLX5_ACCEL_ESP_ACTION_DECRYPT; - /* flags */ - attrs->flags |= (x->props.mode == XFRM_MODE_TRANSPORT) ? - MLX5_ACCEL_ESP_FLAGS_TRANSPORT : - MLX5_ACCEL_ESP_FLAGS_TUNNEL; - + attrs->dir = x->xso.dir; /* spi */ attrs->spi = be32_to_cpu(x->id.spi); /* source , destination ips */ memcpy(&attrs->saddr, x->props.saddr.a6, sizeof(attrs->saddr)); memcpy(&attrs->daddr, x->id.daddr.a6, sizeof(attrs->daddr)); - attrs->is_ipv6 = (x->props.family != AF_INET); + attrs->family = x->props.family; + attrs->type = x->xso.type; + attrs->reqid = x->props.reqid; + + mlx5e_ipsec_init_limits(sa_entry, attrs); } static inline int mlx5e_xfrm_validate_state(struct xfrm_state *x) @@ -215,11 +191,6 @@ static inline int mlx5e_xfrm_validate_state(struct xfrm_state *x) netdev_info(netdev, "Only IPv4/6 xfrm states may be offloaded\n"); return -EINVAL; } - if (x->props.mode != XFRM_MODE_TRANSPORT && - x->props.mode != XFRM_MODE_TUNNEL) { - dev_info(&netdev->dev, "Only transport and tunnel xfrm states may be offloaded\n"); - return -EINVAL; - } if (x->id.proto != IPPROTO_ESP) { netdev_info(netdev, "Only ESP xfrm state may be offloaded\n"); return -EINVAL; @@ -253,6 +224,67 @@ static inline int mlx5e_xfrm_validate_state(struct xfrm_state *x) netdev_info(netdev, "Cannot offload xfrm states with geniv other than seqiv\n"); return -EINVAL; } + switch (x->xso.type) { + case XFRM_DEV_OFFLOAD_CRYPTO: + if (!(mlx5_ipsec_device_caps(priv->mdev) & + MLX5_IPSEC_CAP_CRYPTO)) { + netdev_info(netdev, "Crypto offload is not supported\n"); + return -EINVAL; + } + + if (x->props.mode != XFRM_MODE_TRANSPORT && + x->props.mode != XFRM_MODE_TUNNEL) { + netdev_info(netdev, "Only transport and tunnel xfrm states may be offloaded\n"); + return -EINVAL; + } + break; + case XFRM_DEV_OFFLOAD_PACKET: + if (!(mlx5_ipsec_device_caps(priv->mdev) & + MLX5_IPSEC_CAP_PACKET_OFFLOAD)) { + netdev_info(netdev, "Packet offload is not supported\n"); + return -EINVAL; + } + + if (x->props.mode != XFRM_MODE_TRANSPORT) { + netdev_info(netdev, "Only transport xfrm states may be offloaded in packet mode\n"); + return -EINVAL; + } + + if (x->replay_esn && x->replay_esn->replay_window != 32 && + x->replay_esn->replay_window != 64 && + x->replay_esn->replay_window != 128 && + x->replay_esn->replay_window != 256) { + netdev_info(netdev, + "Unsupported replay window size %u\n", + x->replay_esn->replay_window); + return -EINVAL; + } + + if (!x->props.reqid) { + netdev_info(netdev, "Cannot offload without reqid\n"); + return -EINVAL; + } + + if (x->lft.hard_byte_limit != XFRM_INF || + x->lft.soft_byte_limit != XFRM_INF) { + netdev_info(netdev, + "Device doesn't support limits in bytes\n"); + return -EINVAL; + } + + if (x->lft.soft_packet_limit >= x->lft.hard_packet_limit && + x->lft.hard_packet_limit != XFRM_INF) { + /* XFRM stack doesn't prevent such configuration :(. */ + netdev_info(netdev, + "Hard packet limit must be greater than soft one\n"); + return -EINVAL; + } + break; + default: + netdev_info(netdev, "Unsupported xfrm offload type %d\n", + x->xso.type); + return -EINVAL; + } return 0; } @@ -270,6 +302,7 @@ static int mlx5e_xfrm_add_state(struct xfrm_state *x) { struct mlx5e_ipsec_sa_entry *sa_entry = NULL; struct net_device *netdev = x->xso.real_dev; + struct mlx5e_ipsec *ipsec; struct mlx5e_priv *priv; int err; @@ -277,6 +310,7 @@ static int mlx5e_xfrm_add_state(struct xfrm_state *x) if (!priv->ipsec) return -EOPNOTSUPP; + ipsec = priv->ipsec; err = mlx5e_xfrm_validate_state(x); if (err) return err; @@ -288,7 +322,7 @@ static int mlx5e_xfrm_add_state(struct xfrm_state *x) } sa_entry->x = x; - sa_entry->ipsec = priv->ipsec; + sa_entry->ipsec = ipsec; /* check esn */ mlx5e_ipsec_update_esn_state(sa_entry); @@ -299,25 +333,29 @@ static int mlx5e_xfrm_add_state(struct xfrm_state *x) if (err) goto err_xfrm; - err = mlx5e_accel_ipsec_fs_add_rule(priv, sa_entry); + err = mlx5e_accel_ipsec_fs_add_rule(sa_entry); if (err) goto err_hw_ctx; - if (x->xso.dir == XFRM_DEV_OFFLOAD_IN) { - err = mlx5e_ipsec_sadb_rx_add(sa_entry); - if (err) - goto err_add_rule; - } else { + /* We use *_bh() variant because xfrm_timer_handler(), which runs + * in softirq context, can reach our state delete logic and we need + * xa_erase_bh() there. + */ + err = xa_insert_bh(&ipsec->sadb, sa_entry->ipsec_obj_id, sa_entry, + GFP_KERNEL); + if (err) + goto err_add_rule; + + if (x->xso.dir == XFRM_DEV_OFFLOAD_OUT) sa_entry->set_iv_op = (x->props.flags & XFRM_STATE_ESN) ? mlx5e_ipsec_set_iv_esn : mlx5e_ipsec_set_iv; - } INIT_WORK(&sa_entry->modify_work.work, _update_xfrm_state); x->xso.offload_handle = (unsigned long)sa_entry; - goto out; + return 0; err_add_rule: - mlx5e_accel_ipsec_fs_del_rule(priv, sa_entry); + mlx5e_accel_ipsec_fs_del_rule(sa_entry); err_hw_ctx: mlx5_ipsec_free_sa_ctx(sa_entry); err_xfrm: @@ -329,18 +367,19 @@ out: static void mlx5e_xfrm_del_state(struct xfrm_state *x) { struct mlx5e_ipsec_sa_entry *sa_entry = to_ipsec_sa_entry(x); + struct mlx5e_ipsec *ipsec = sa_entry->ipsec; + struct mlx5e_ipsec_sa_entry *old; - if (x->xso.dir == XFRM_DEV_OFFLOAD_IN) - mlx5e_ipsec_sadb_rx_del(sa_entry); + old = xa_erase_bh(&ipsec->sadb, sa_entry->ipsec_obj_id); + WARN_ON(old != sa_entry); } static void mlx5e_xfrm_free_state(struct xfrm_state *x) { struct mlx5e_ipsec_sa_entry *sa_entry = to_ipsec_sa_entry(x); - struct mlx5e_priv *priv = netdev_priv(x->xso.dev); cancel_work_sync(&sa_entry->modify_work.work); - mlx5e_accel_ipsec_fs_del_rule(priv, sa_entry); + mlx5e_accel_ipsec_fs_del_rule(sa_entry); mlx5_ipsec_free_sa_ctx(sa_entry); kfree(sa_entry); } @@ -359,23 +398,33 @@ void mlx5e_ipsec_init(struct mlx5e_priv *priv) if (!ipsec) return; - hash_init(ipsec->sadb_rx); - spin_lock_init(&ipsec->sadb_rx_lock); + xa_init_flags(&ipsec->sadb, XA_FLAGS_ALLOC); ipsec->mdev = priv->mdev; ipsec->wq = alloc_ordered_workqueue("mlx5e_ipsec: %s", 0, priv->netdev->name); if (!ipsec->wq) goto err_wq; + if (mlx5_ipsec_device_caps(priv->mdev) & + MLX5_IPSEC_CAP_PACKET_OFFLOAD) { + ret = mlx5e_ipsec_aso_init(ipsec); + if (ret) + goto err_aso; + } + ret = mlx5e_accel_ipsec_fs_init(ipsec); if (ret) goto err_fs_init; + ipsec->fs = priv->fs; priv->ipsec = ipsec; netdev_dbg(priv->netdev, "IPSec attached to netdevice\n"); return; err_fs_init: + if (mlx5_ipsec_device_caps(priv->mdev) & MLX5_IPSEC_CAP_PACKET_OFFLOAD) + mlx5e_ipsec_aso_cleanup(ipsec); +err_aso: destroy_workqueue(ipsec->wq); err_wq: kfree(ipsec); @@ -391,6 +440,8 @@ void mlx5e_ipsec_cleanup(struct mlx5e_priv *priv) return; mlx5e_accel_ipsec_fs_cleanup(ipsec); + if (mlx5_ipsec_device_caps(priv->mdev) & MLX5_IPSEC_CAP_PACKET_OFFLOAD) + mlx5e_ipsec_aso_cleanup(ipsec); destroy_workqueue(ipsec->wq); kfree(ipsec); priv->ipsec = NULL; @@ -426,6 +477,122 @@ static void mlx5e_xfrm_advance_esn_state(struct xfrm_state *x) queue_work(sa_entry->ipsec->wq, &modify_work->work); } +static void mlx5e_xfrm_update_curlft(struct xfrm_state *x) +{ + struct mlx5e_ipsec_sa_entry *sa_entry = to_ipsec_sa_entry(x); + int err; + + lockdep_assert_held(&x->lock); + + if (sa_entry->attrs.soft_packet_limit == XFRM_INF) + /* Limits are not configured, as soft limit + * must be lowever than hard limit. + */ + return; + + err = mlx5e_ipsec_aso_query(sa_entry, NULL); + if (err) + return; + + mlx5e_ipsec_aso_update_curlft(sa_entry, &x->curlft.packets); +} + +static int mlx5e_xfrm_validate_policy(struct xfrm_policy *x) +{ + struct net_device *netdev = x->xdo.real_dev; + + if (x->type != XFRM_POLICY_TYPE_MAIN) { + netdev_info(netdev, "Cannot offload non-main policy types\n"); + return -EINVAL; + } + + /* Please pay attention that we support only one template */ + if (x->xfrm_nr > 1) { + netdev_info(netdev, "Cannot offload more than one template\n"); + return -EINVAL; + } + + if (x->xdo.dir != XFRM_DEV_OFFLOAD_IN && + x->xdo.dir != XFRM_DEV_OFFLOAD_OUT) { + netdev_info(netdev, "Cannot offload forward policy\n"); + return -EINVAL; + } + + if (!x->xfrm_vec[0].reqid) { + netdev_info(netdev, "Cannot offload policy without reqid\n"); + return -EINVAL; + } + + if (x->xdo.type != XFRM_DEV_OFFLOAD_PACKET) { + netdev_info(netdev, "Unsupported xfrm offload type\n"); + return -EINVAL; + } + + return 0; +} + +static void +mlx5e_ipsec_build_accel_pol_attrs(struct mlx5e_ipsec_pol_entry *pol_entry, + struct mlx5_accel_pol_xfrm_attrs *attrs) +{ + struct xfrm_policy *x = pol_entry->x; + struct xfrm_selector *sel; + + sel = &x->selector; + memset(attrs, 0, sizeof(*attrs)); + + memcpy(&attrs->saddr, sel->saddr.a6, sizeof(attrs->saddr)); + memcpy(&attrs->daddr, sel->daddr.a6, sizeof(attrs->daddr)); + attrs->family = sel->family; + attrs->dir = x->xdo.dir; + attrs->action = x->action; + attrs->type = XFRM_DEV_OFFLOAD_PACKET; + attrs->reqid = x->xfrm_vec[0].reqid; +} + +static int mlx5e_xfrm_add_policy(struct xfrm_policy *x) +{ + struct net_device *netdev = x->xdo.real_dev; + struct mlx5e_ipsec_pol_entry *pol_entry; + struct mlx5e_priv *priv; + int err; + + priv = netdev_priv(netdev); + if (!priv->ipsec) + return -EOPNOTSUPP; + + err = mlx5e_xfrm_validate_policy(x); + if (err) + return err; + + pol_entry = kzalloc(sizeof(*pol_entry), GFP_KERNEL); + if (!pol_entry) + return -ENOMEM; + + pol_entry->x = x; + pol_entry->ipsec = priv->ipsec; + + mlx5e_ipsec_build_accel_pol_attrs(pol_entry, &pol_entry->attrs); + err = mlx5e_accel_ipsec_fs_add_pol(pol_entry); + if (err) + goto err_fs; + + x->xdo.offload_handle = (unsigned long)pol_entry; + return 0; + +err_fs: + kfree(pol_entry); + return err; +} + +static void mlx5e_xfrm_free_policy(struct xfrm_policy *x) +{ + struct mlx5e_ipsec_pol_entry *pol_entry = to_ipsec_pol_entry(x); + + mlx5e_accel_ipsec_fs_del_pol(pol_entry); + kfree(pol_entry); +} + static const struct xfrmdev_ops mlx5e_ipsec_xfrmdev_ops = { .xdo_dev_state_add = mlx5e_xfrm_add_state, .xdo_dev_state_delete = mlx5e_xfrm_del_state, @@ -434,6 +601,18 @@ static const struct xfrmdev_ops mlx5e_ipsec_xfrmdev_ops = { .xdo_dev_state_advance_esn = mlx5e_xfrm_advance_esn_state, }; +static const struct xfrmdev_ops mlx5e_ipsec_packet_xfrmdev_ops = { + .xdo_dev_state_add = mlx5e_xfrm_add_state, + .xdo_dev_state_delete = mlx5e_xfrm_del_state, + .xdo_dev_state_free = mlx5e_xfrm_free_state, + .xdo_dev_offload_ok = mlx5e_ipsec_offload_ok, + .xdo_dev_state_advance_esn = mlx5e_xfrm_advance_esn_state, + + .xdo_dev_state_update_curlft = mlx5e_xfrm_update_curlft, + .xdo_dev_policy_add = mlx5e_xfrm_add_policy, + .xdo_dev_policy_free = mlx5e_xfrm_free_policy, +}; + void mlx5e_ipsec_build_netdev(struct mlx5e_priv *priv) { struct mlx5_core_dev *mdev = priv->mdev; @@ -443,7 +622,12 @@ void mlx5e_ipsec_build_netdev(struct mlx5e_priv *priv) return; mlx5_core_info(mdev, "mlx5e: IPSec ESP acceleration enabled\n"); - netdev->xfrmdev_ops = &mlx5e_ipsec_xfrmdev_ops; + + if (mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_PACKET_OFFLOAD) + netdev->xfrmdev_ops = &mlx5e_ipsec_packet_xfrmdev_ops; + else + netdev->xfrmdev_ops = &mlx5e_ipsec_xfrmdev_ops; + netdev->features |= NETIF_F_HW_ESP; netdev->hw_enc_features |= NETIF_F_HW_ESP; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h index 4c47347d0ee2..a92e19c4c499 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h @@ -34,27 +34,14 @@ #ifndef __MLX5E_IPSEC_H__ #define __MLX5E_IPSEC_H__ -#ifdef CONFIG_MLX5_EN_IPSEC - #include <linux/mlx5/device.h> #include <net/xfrm.h> #include <linux/idr.h> +#include "lib/aso.h" #define MLX5E_IPSEC_SADB_RX_BITS 10 #define MLX5E_IPSEC_ESN_SCOPE_MID 0x80000000L -enum mlx5_accel_esp_flags { - MLX5_ACCEL_ESP_FLAGS_TUNNEL = 0, /* Default */ - MLX5_ACCEL_ESP_FLAGS_TRANSPORT = 1UL << 0, - MLX5_ACCEL_ESP_FLAGS_ESN_TRIGGERED = 1UL << 1, - MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP = 1UL << 2, -}; - -enum mlx5_accel_esp_action { - MLX5_ACCEL_ESP_ACTION_DECRYPT, - MLX5_ACCEL_ESP_ACTION_ENCRYPT, -}; - struct aes_gcm_keymat { u64 seq_iv; @@ -66,7 +53,6 @@ struct aes_gcm_keymat { }; struct mlx5_accel_esp_xfrm_attrs { - enum mlx5_accel_esp_action action; u32 esn; u32 spi; u32 flags; @@ -82,16 +68,37 @@ struct mlx5_accel_esp_xfrm_attrs { __be32 a6[4]; } daddr; - u8 is_ipv6; + u8 dir : 2; + u8 esn_overlap : 1; + u8 esn_trigger : 1; + u8 type : 2; + u8 family; + u32 replay_window; + u32 authsize; + u32 reqid; + u64 hard_packet_limit; + u64 soft_packet_limit; }; enum mlx5_ipsec_cap { MLX5_IPSEC_CAP_CRYPTO = 1 << 0, MLX5_IPSEC_CAP_ESN = 1 << 1, + MLX5_IPSEC_CAP_PACKET_OFFLOAD = 1 << 2, }; struct mlx5e_priv; +struct mlx5e_ipsec_hw_stats { + u64 ipsec_rx_pkts; + u64 ipsec_rx_bytes; + u64 ipsec_rx_drop_pkts; + u64 ipsec_rx_drop_bytes; + u64 ipsec_tx_pkts; + u64 ipsec_tx_bytes; + u64 ipsec_tx_drop_pkts; + u64 ipsec_tx_drop_bytes; +}; + struct mlx5e_ipsec_sw_stats { atomic64_t ipsec_rx_drop_sp_alloc; atomic64_t ipsec_rx_drop_sadb_miss; @@ -102,17 +109,38 @@ struct mlx5e_ipsec_sw_stats { atomic64_t ipsec_tx_drop_trailer; }; -struct mlx5e_accel_fs_esp; +struct mlx5e_ipsec_rx; struct mlx5e_ipsec_tx; +struct mlx5e_ipsec_work { + struct work_struct work; + struct mlx5e_ipsec *ipsec; + u32 id; +}; + +struct mlx5e_ipsec_aso { + u8 ctx[MLX5_ST_SZ_BYTES(ipsec_aso)]; + dma_addr_t dma_addr; + struct mlx5_aso *aso; + /* IPsec ASO caches data on every query call, + * so in nested calls, we can use this boolean to save + * recursive calls to mlx5e_ipsec_aso_query() + */ + u8 use_cache : 1; +}; + struct mlx5e_ipsec { struct mlx5_core_dev *mdev; - DECLARE_HASHTABLE(sadb_rx, MLX5E_IPSEC_SADB_RX_BITS); - spinlock_t sadb_rx_lock; /* Protects sadb_rx */ + struct xarray sadb; struct mlx5e_ipsec_sw_stats sw_stats; + struct mlx5e_ipsec_hw_stats hw_stats; struct workqueue_struct *wq; - struct mlx5e_accel_fs_esp *rx_fs; - struct mlx5e_ipsec_tx *tx_fs; + struct mlx5e_flow_steering *fs; + struct mlx5e_ipsec_rx *rx_ipv4; + struct mlx5e_ipsec_rx *rx_ipv6; + struct mlx5e_ipsec_tx *tx; + struct mlx5e_ipsec_aso *aso; + struct notifier_block nb; }; struct mlx5e_ipsec_esn_state { @@ -123,7 +151,8 @@ struct mlx5e_ipsec_esn_state { struct mlx5e_ipsec_rule { struct mlx5_flow_handle *rule; - struct mlx5_modify_hdr *set_modify_hdr; + struct mlx5_modify_hdr *modify_hdr; + struct mlx5_pkt_reformat *pkt_reformat; }; struct mlx5e_ipsec_modify_state_work { @@ -132,9 +161,7 @@ struct mlx5e_ipsec_modify_state_work { }; struct mlx5e_ipsec_sa_entry { - struct hlist_node hlist; /* Item in SADB_RX hashtable */ struct mlx5e_ipsec_esn_state esn_state; - unsigned int handle; /* Handle in SADB_RX */ struct xfrm_state *x; struct mlx5e_ipsec *ipsec; struct mlx5_accel_esp_xfrm_attrs attrs; @@ -146,19 +173,43 @@ struct mlx5e_ipsec_sa_entry { struct mlx5e_ipsec_modify_state_work modify_work; }; +struct mlx5_accel_pol_xfrm_attrs { + union { + __be32 a4; + __be32 a6[4]; + } saddr; + + union { + __be32 a4; + __be32 a6[4]; + } daddr; + + u8 family; + u8 action; + u8 type : 2; + u8 dir : 2; + u32 reqid; +}; + +struct mlx5e_ipsec_pol_entry { + struct xfrm_policy *x; + struct mlx5e_ipsec *ipsec; + struct mlx5e_ipsec_rule ipsec_rule; + struct mlx5_accel_pol_xfrm_attrs attrs; +}; + +#ifdef CONFIG_MLX5_EN_IPSEC + void mlx5e_ipsec_init(struct mlx5e_priv *priv); void mlx5e_ipsec_cleanup(struct mlx5e_priv *priv); void mlx5e_ipsec_build_netdev(struct mlx5e_priv *priv); -struct xfrm_state *mlx5e_ipsec_sadb_rx_lookup(struct mlx5e_ipsec *dev, - unsigned int handle); - void mlx5e_accel_ipsec_fs_cleanup(struct mlx5e_ipsec *ipsec); int mlx5e_accel_ipsec_fs_init(struct mlx5e_ipsec *ipsec); -int mlx5e_accel_ipsec_fs_add_rule(struct mlx5e_priv *priv, - struct mlx5e_ipsec_sa_entry *sa_entry); -void mlx5e_accel_ipsec_fs_del_rule(struct mlx5e_priv *priv, - struct mlx5e_ipsec_sa_entry *sa_entry); +int mlx5e_accel_ipsec_fs_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry); +void mlx5e_accel_ipsec_fs_del_rule(struct mlx5e_ipsec_sa_entry *sa_entry); +int mlx5e_accel_ipsec_fs_add_pol(struct mlx5e_ipsec_pol_entry *pol_entry); +void mlx5e_accel_ipsec_fs_del_pol(struct mlx5e_ipsec_pol_entry *pol_entry); int mlx5_ipsec_create_sa_ctx(struct mlx5e_ipsec_sa_entry *sa_entry); void mlx5_ipsec_free_sa_ctx(struct mlx5e_ipsec_sa_entry *sa_entry); @@ -168,11 +219,30 @@ u32 mlx5_ipsec_device_caps(struct mlx5_core_dev *mdev); void mlx5_accel_esp_modify_xfrm(struct mlx5e_ipsec_sa_entry *sa_entry, const struct mlx5_accel_esp_xfrm_attrs *attrs); +int mlx5e_ipsec_aso_init(struct mlx5e_ipsec *ipsec); +void mlx5e_ipsec_aso_cleanup(struct mlx5e_ipsec *ipsec); + +int mlx5e_ipsec_aso_query(struct mlx5e_ipsec_sa_entry *sa_entry, + struct mlx5_wqe_aso_ctrl_seg *data); +void mlx5e_ipsec_aso_update_curlft(struct mlx5e_ipsec_sa_entry *sa_entry, + u64 *packets); + +void mlx5e_accel_ipsec_fs_read_stats(struct mlx5e_priv *priv, + void *ipsec_stats); + +void mlx5e_ipsec_build_accel_xfrm_attrs(struct mlx5e_ipsec_sa_entry *sa_entry, + struct mlx5_accel_esp_xfrm_attrs *attrs); static inline struct mlx5_core_dev * mlx5e_ipsec_sa2dev(struct mlx5e_ipsec_sa_entry *sa_entry) { return sa_entry->ipsec->mdev; } + +static inline struct mlx5_core_dev * +mlx5e_ipsec_pol2dev(struct mlx5e_ipsec_pol_entry *pol_entry) +{ + return pol_entry->ipsec->mdev; +} #else static inline void mlx5e_ipsec_init(struct mlx5e_priv *priv) { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c index b859e4a4c744..9f19f4b59a70 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c @@ -9,53 +9,67 @@ #define NUM_IPSEC_FTE BIT(15) -enum accel_fs_esp_type { - ACCEL_FS_ESP4, - ACCEL_FS_ESP6, - ACCEL_FS_ESP_NUM_TYPES, +struct mlx5e_ipsec_fc { + struct mlx5_fc *cnt; + struct mlx5_fc *drop; }; -struct mlx5e_ipsec_rx_err { - struct mlx5_flow_table *ft; - struct mlx5_flow_handle *rule; - struct mlx5_modify_hdr *copy_modify_hdr; +struct mlx5e_ipsec_ft { + struct mutex mutex; /* Protect changes to this struct */ + struct mlx5_flow_table *pol; + struct mlx5_flow_table *sa; + struct mlx5_flow_table *status; + u32 refcnt; }; -struct mlx5e_accel_fs_esp_prot { - struct mlx5_flow_table *ft; - struct mlx5_flow_group *miss_group; - struct mlx5_flow_handle *miss_rule; - struct mlx5_flow_destination default_dest; - struct mlx5e_ipsec_rx_err rx_err; - u32 refcnt; - struct mutex prot_mutex; /* protect ESP4/ESP6 protocol */ +struct mlx5e_ipsec_miss { + struct mlx5_flow_group *group; + struct mlx5_flow_handle *rule; }; -struct mlx5e_accel_fs_esp { - struct mlx5e_accel_fs_esp_prot fs_prot[ACCEL_FS_ESP_NUM_TYPES]; +struct mlx5e_ipsec_rx { + struct mlx5e_ipsec_ft ft; + struct mlx5e_ipsec_miss pol; + struct mlx5e_ipsec_miss sa; + struct mlx5e_ipsec_rule status; + struct mlx5e_ipsec_fc *fc; }; struct mlx5e_ipsec_tx { + struct mlx5e_ipsec_ft ft; + struct mlx5e_ipsec_miss pol; struct mlx5_flow_namespace *ns; - struct mlx5_flow_table *ft; - struct mutex mutex; /* Protect IPsec TX steering */ - u32 refcnt; + struct mlx5e_ipsec_fc *fc; }; /* IPsec RX flow steering */ -static enum mlx5_traffic_types fs_esp2tt(enum accel_fs_esp_type i) +static enum mlx5_traffic_types family2tt(u32 family) { - if (i == ACCEL_FS_ESP4) + if (family == AF_INET) return MLX5_TT_IPV4_IPSEC_ESP; return MLX5_TT_IPV6_IPSEC_ESP; } -static int rx_err_add_rule(struct mlx5e_priv *priv, - struct mlx5e_accel_fs_esp_prot *fs_prot, - struct mlx5e_ipsec_rx_err *rx_err) +static struct mlx5_flow_table *ipsec_ft_create(struct mlx5_flow_namespace *ns, + int level, int prio, + int max_num_groups) +{ + struct mlx5_flow_table_attr ft_attr = {}; + + ft_attr.autogroup.num_reserved_entries = 1; + ft_attr.autogro |