From 7c26c20da5d420cde55618263be4aa2f6de53056 Mon Sep 17 00:00:00 2001 From: Lorenzo Bianconi Date: Fri, 22 Jul 2022 09:19:37 +0200 Subject: [PATCH] net: ethernet: mtk_eth_soc: add basic XDP support Introduce basic XDP support to mtk_eth_soc driver. Supported XDP verdicts: - XDP_PASS - XDP_DROP - XDP_REDIRECT Signed-off-by: Lorenzo Bianconi Signed-off-by: David S. Miller --- drivers/net/ethernet/mediatek/mtk_eth_soc.c | 162 +++++++++++++++++--- drivers/net/ethernet/mediatek/mtk_eth_soc.h | 2 + 2 files changed, 145 insertions(+), 19 deletions(-) --- a/drivers/net/ethernet/mediatek/mtk_eth_soc.c +++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.c @@ -1389,6 +1389,11 @@ static void mtk_update_rx_cpu_idx(struct } } +static bool mtk_page_pool_enabled(struct mtk_eth *eth) +{ + return !eth->hwlro; +} + static struct page_pool *mtk_create_page_pool(struct mtk_eth *eth, struct xdp_rxq_info *xdp_q, int id, int size) @@ -1451,11 +1456,52 @@ static void mtk_rx_put_buff(struct mtk_r skb_free_frag(data); } +static u32 mtk_xdp_run(struct mtk_eth *eth, struct mtk_rx_ring *ring, + struct xdp_buff *xdp, struct net_device *dev) +{ + struct bpf_prog *prog; + u32 act = XDP_PASS; + + rcu_read_lock(); + + prog = rcu_dereference(eth->prog); + if (!prog) + goto out; + + act = bpf_prog_run_xdp(prog, xdp); + switch (act) { + case XDP_PASS: + goto out; + case XDP_REDIRECT: + if (unlikely(xdp_do_redirect(dev, xdp, prog))) { + act = XDP_DROP; + break; + } + goto out; + default: + bpf_warn_invalid_xdp_action(act); + fallthrough; + case XDP_ABORTED: + trace_xdp_exception(dev, prog, act); + fallthrough; + case XDP_DROP: + break; + } + + page_pool_put_full_page(ring->page_pool, + virt_to_head_page(xdp->data), true); +out: + rcu_read_unlock(); + + return act; +} + static int mtk_poll_rx(struct napi_struct *napi, int budget, struct mtk_eth *eth) { struct dim_sample dim_sample = {}; struct mtk_rx_ring *ring; + bool xdp_flush = false; int idx; struct sk_buff *skb; u8 *data, *new_data; @@ -1464,9 +1510,9 @@ static int mtk_poll_rx(struct napi_struc while (done < budget) { unsigned int pktlen, *rxdcsum; - u32 hash, reason, reserve_len; struct net_device *netdev; dma_addr_t dma_addr; + u32 hash, reason; int mac = 0; ring = mtk_get_rx_ring(eth); @@ -1496,8 +1542,14 @@ static int mtk_poll_rx(struct napi_struc if (unlikely(test_bit(MTK_RESETTING, ð->state))) goto release_desc; + pktlen = RX_DMA_GET_PLEN0(trxd.rxd2); + /* alloc new buffer */ if (ring->page_pool) { + struct page *page = virt_to_head_page(data); + struct xdp_buff xdp; + u32 ret; + new_data = mtk_page_pool_get_buff(ring->page_pool, &dma_addr, GFP_ATOMIC); @@ -1505,6 +1557,34 @@ static int mtk_poll_rx(struct napi_struc netdev->stats.rx_dropped++; goto release_desc; } + + dma_sync_single_for_cpu(eth->dma_dev, + page_pool_get_dma_addr(page) + MTK_PP_HEADROOM, + pktlen, page_pool_get_dma_dir(ring->page_pool)); + + xdp_init_buff(&xdp, PAGE_SIZE, &ring->xdp_q); + xdp_prepare_buff(&xdp, data, MTK_PP_HEADROOM, pktlen, + false); + xdp_buff_clear_frags_flag(&xdp); + + ret = mtk_xdp_run(eth, ring, &xdp, netdev); + if (ret == XDP_REDIRECT) + xdp_flush = true; + + if (ret != XDP_PASS) + goto skip_rx; + + skb = build_skb(data, PAGE_SIZE); + if (unlikely(!skb)) { + page_pool_put_full_page(ring->page_pool, + page, true); + netdev->stats.rx_dropped++; + goto skip_rx; + } + + skb_reserve(skb, xdp.data - xdp.data_hard_start); + skb_put(skb, xdp.data_end - xdp.data); + skb_mark_for_recycle(skb); } else { if (ring->frag_size <= PAGE_SIZE) new_data = napi_alloc_frag(ring->frag_size); @@ -1528,27 +1608,20 @@ static int mtk_poll_rx(struct napi_struc dma_unmap_single(eth->dma_dev, trxd.rxd1, ring->buf_size, DMA_FROM_DEVICE); - } - /* receive data */ - skb = build_skb(data, ring->frag_size); - if (unlikely(!skb)) { - mtk_rx_put_buff(ring, data, true); - netdev->stats.rx_dropped++; - goto skip_rx; - } + skb = build_skb(data, ring->frag_size); + if (unlikely(!skb)) { + netdev->stats.rx_dropped++; + skb_free_frag(data); + goto skip_rx; + } - if (ring->page_pool) { - reserve_len = MTK_PP_HEADROOM; - skb_mark_for_recycle(skb); - } else { - reserve_len = NET_SKB_PAD + NET_IP_ALIGN; + skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN); + skb_put(skb, pktlen); } - skb_reserve(skb, reserve_len); - pktlen = RX_DMA_GET_PLEN0(trxd.rxd2); skb->dev = netdev; - skb_put(skb, pktlen); + bytes += skb->len; if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) rxdcsum = &trxd.rxd3; @@ -1560,7 +1633,6 @@ static int mtk_poll_rx(struct napi_struc else skb_checksum_none_assert(skb); skb->protocol = eth_type_trans(skb, netdev); - bytes += pktlen; hash = trxd.rxd4 & MTK_RXD4_FOE_ENTRY; if (hash != MTK_RXD4_FOE_ENTRY) { @@ -1623,6 +1695,9 @@ rx_done: &dim_sample); net_dim(ð->rx_dim, dim_sample); + if (xdp_flush) + xdp_do_flush_map(); + return done; } @@ -1968,7 +2043,7 @@ static int mtk_rx_alloc(struct mtk_eth * if (!ring->data) return -ENOMEM; - if (!eth->hwlro) { + if (mtk_page_pool_enabled(eth)) { struct page_pool *pp; pp = mtk_create_page_pool(eth, &ring->xdp_q, ring_no, @@ -2713,6 +2788,48 @@ static int mtk_stop(struct net_device *d return 0; } +static int mtk_xdp_setup(struct net_device *dev, struct bpf_prog *prog, + struct netlink_ext_ack *extack) +{ + struct mtk_mac *mac = netdev_priv(dev); + struct mtk_eth *eth = mac->hw; + struct bpf_prog *old_prog; + bool need_update; + + if (eth->hwlro) { + NL_SET_ERR_MSG_MOD(extack, "XDP not supported with HWLRO"); + return -EOPNOTSUPP; + } + + if (dev->mtu > MTK_PP_MAX_BUF_SIZE) { + NL_SET_ERR_MSG_MOD(extack, "MTU too large for XDP"); + return -EOPNOTSUPP; + } + + need_update = !!eth->prog != !!prog; + if (netif_running(dev) && need_update) + mtk_stop(dev); + + old_prog = rcu_replace_pointer(eth->prog, prog, lockdep_rtnl_is_held()); + if (old_prog) + bpf_prog_put(old_prog); + + if (netif_running(dev) && need_update) + return mtk_open(dev); + + return 0; +} + +static int mtk_xdp(struct net_device *dev, struct netdev_bpf *xdp) +{ + switch (xdp->command) { + case XDP_SETUP_PROG: + return mtk_xdp_setup(dev, xdp->prog, xdp->extack); + default: + return -EINVAL; + } +} + static void ethsys_reset(struct mtk_eth *eth, u32 reset_bits) { regmap_update_bits(eth->ethsys, ETHSYS_RSTCTRL, @@ -2991,6 +3108,12 @@ static int mtk_change_mtu(struct net_dev struct mtk_eth *eth = mac->hw; u32 mcr_cur, mcr_new; + if (rcu_access_pointer(eth->prog) && + length > MTK_PP_MAX_BUF_SIZE) { + netdev_err(dev, "Invalid MTU for XDP mode\n"); + return -EINVAL; + } + if (!MTK_HAS_CAPS(eth->soc->caps, MTK_SOC_MT7628)) { mcr_cur = mtk_r32(mac->hw, MTK_MAC_MCR(mac->id)); mcr_new = mcr_cur & ~MAC_MCR_MAX_RX_MASK; @@ -3317,6 +3440,7 @@ static const struct net_device_ops mtk_n .ndo_poll_controller = mtk_poll_controller, #endif .ndo_setup_tc = mtk_eth_setup_tc, + .ndo_bpf = mtk_xdp, }; static int mtk_add_mac(struct mtk_eth *eth, struct device_node *np) --- a/drivers/net/ethernet/mediatek/mtk_eth_soc.h +++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.h @@ -1086,6 +1086,8 @@ struct mtk_eth { struct mtk_ppe *ppe; struct rhashtable flow_table; + + struct bpf_prog __rcu *prog; }; /* struct mtk_mac - the structure that holds the info about the MACs of the