| /**************************************************************************** |
| * Driver for Solarflare Solarstorm network controllers and boards |
| * Copyright 2005-2010 Solarflare Communications Inc. |
| * |
| * This program is free software; you can redistribute it and/or modify it |
| * under the terms of the GNU General Public License version 2 as published |
| * by the Free Software Foundation, incorporated herein by reference. |
| */ |
| |
| #include <linux/in.h> |
| #include <net/ip.h> |
| #include "efx.h" |
| #include "filter.h" |
| #include "io.h" |
| #include "nic.h" |
| #include "regs.h" |
| |
| /* "Fudge factors" - difference between programmed value and actual depth. |
| * Due to pipelined implementation we need to program H/W with a value that |
| * is larger than the hop limit we want. |
| */ |
| #define FILTER_CTL_SRCH_FUDGE_WILD 3 |
| #define FILTER_CTL_SRCH_FUDGE_FULL 1 |
| |
| /* Hard maximum hop limit. Hardware will time-out beyond 200-something. |
| * We also need to avoid infinite loops in efx_filter_search() when the |
| * table is full. |
| */ |
| #define FILTER_CTL_SRCH_MAX 200 |
| |
| /* Don't try very hard to find space for performance hints, as this is |
| * counter-productive. */ |
| #define FILTER_CTL_SRCH_HINT_MAX 5 |
| |
| enum efx_filter_table_id { |
| EFX_FILTER_TABLE_RX_IP = 0, |
| EFX_FILTER_TABLE_RX_MAC, |
| EFX_FILTER_TABLE_RX_DEF, |
| EFX_FILTER_TABLE_TX_MAC, |
| EFX_FILTER_TABLE_COUNT, |
| }; |
| |
| enum efx_filter_index { |
| EFX_FILTER_INDEX_UC_DEF, |
| EFX_FILTER_INDEX_MC_DEF, |
| EFX_FILTER_SIZE_RX_DEF, |
| }; |
| |
| struct efx_filter_table { |
| enum efx_filter_table_id id; |
| u32 offset; /* address of table relative to BAR */ |
| unsigned size; /* number of entries */ |
| unsigned step; /* step between entries */ |
| unsigned used; /* number currently used */ |
| unsigned long *used_bitmap; |
| struct efx_filter_spec *spec; |
| unsigned search_depth[EFX_FILTER_TYPE_COUNT]; |
| }; |
| |
| struct efx_filter_state { |
| spinlock_t lock; |
| struct efx_filter_table table[EFX_FILTER_TABLE_COUNT]; |
| #ifdef CONFIG_RFS_ACCEL |
| u32 *rps_flow_id; |
| unsigned rps_expire_index; |
| #endif |
| }; |
| |
| static void efx_filter_table_clear_entry(struct efx_nic *efx, |
| struct efx_filter_table *table, |
| unsigned int filter_idx); |
| |
| /* The filter hash function is LFSR polynomial x^16 + x^3 + 1 of a 32-bit |
| * key derived from the n-tuple. The initial LFSR state is 0xffff. */ |
| static u16 efx_filter_hash(u32 key) |
| { |
| u16 tmp; |
| |
| /* First 16 rounds */ |
| tmp = 0x1fff ^ key >> 16; |
| tmp = tmp ^ tmp >> 3 ^ tmp >> 6; |
| tmp = tmp ^ tmp >> 9; |
| /* Last 16 rounds */ |
| tmp = tmp ^ tmp << 13 ^ key; |
| tmp = tmp ^ tmp >> 3 ^ tmp >> 6; |
| return tmp ^ tmp >> 9; |
| } |
| |
| /* To allow for hash collisions, filter search continues at these |
| * increments from the first possible entry selected by the hash. */ |
| static u16 efx_filter_increment(u32 key) |
| { |
| return key * 2 - 1; |
| } |
| |
| static enum efx_filter_table_id |
| efx_filter_spec_table_id(const struct efx_filter_spec *spec) |
| { |
| BUILD_BUG_ON(EFX_FILTER_TABLE_RX_IP != (EFX_FILTER_TCP_FULL >> 2)); |
| BUILD_BUG_ON(EFX_FILTER_TABLE_RX_IP != (EFX_FILTER_TCP_WILD >> 2)); |
| BUILD_BUG_ON(EFX_FILTER_TABLE_RX_IP != (EFX_FILTER_UDP_FULL >> 2)); |
| BUILD_BUG_ON(EFX_FILTER_TABLE_RX_IP != (EFX_FILTER_UDP_WILD >> 2)); |
| BUILD_BUG_ON(EFX_FILTER_TABLE_RX_MAC != (EFX_FILTER_MAC_FULL >> 2)); |
| BUILD_BUG_ON(EFX_FILTER_TABLE_RX_MAC != (EFX_FILTER_MAC_WILD >> 2)); |
| BUILD_BUG_ON(EFX_FILTER_TABLE_TX_MAC != EFX_FILTER_TABLE_RX_MAC + 2); |
| EFX_BUG_ON_PARANOID(spec->type == EFX_FILTER_UNSPEC); |
| return (spec->type >> 2) + ((spec->flags & EFX_FILTER_FLAG_TX) ? 2 : 0); |
| } |
| |
| static struct efx_filter_table * |
| efx_filter_spec_table(struct efx_filter_state *state, |
| const struct efx_filter_spec *spec) |
| { |
| if (spec->type == EFX_FILTER_UNSPEC) |
| return NULL; |
| else |
| return &state->table[efx_filter_spec_table_id(spec)]; |
| } |
| |
| static void efx_filter_table_reset_search_depth(struct efx_filter_table *table) |
| { |
| memset(table->search_depth, 0, sizeof(table->search_depth)); |
| } |
| |
| static void efx_filter_push_rx_config(struct efx_nic *efx) |
| { |
| struct efx_filter_state *state = efx->filter_state; |
| struct efx_filter_table *table; |
| efx_oword_t filter_ctl; |
| |
| efx_reado(efx, &filter_ctl, FR_BZ_RX_FILTER_CTL); |
| |
| table = &state->table[EFX_FILTER_TABLE_RX_IP]; |
| EFX_SET_OWORD_FIELD(filter_ctl, FRF_BZ_TCP_FULL_SRCH_LIMIT, |
| table->search_depth[EFX_FILTER_TCP_FULL] + |
| FILTER_CTL_SRCH_FUDGE_FULL); |
| EFX_SET_OWORD_FIELD(filter_ctl, FRF_BZ_TCP_WILD_SRCH_LIMIT, |
| table->search_depth[EFX_FILTER_TCP_WILD] + |
| FILTER_CTL_SRCH_FUDGE_WILD); |
| EFX_SET_OWORD_FIELD(filter_ctl, FRF_BZ_UDP_FULL_SRCH_LIMIT, |
| table->search_depth[EFX_FILTER_UDP_FULL] + |
| FILTER_CTL_SRCH_FUDGE_FULL); |
| EFX_SET_OWORD_FIELD(filter_ctl, FRF_BZ_UDP_WILD_SRCH_LIMIT, |
| table->search_depth[EFX_FILTER_UDP_WILD] + |
| FILTER_CTL_SRCH_FUDGE_WILD); |
| |
| table = &state->table[EFX_FILTER_TABLE_RX_MAC]; |
| if (table->size) { |
| EFX_SET_OWORD_FIELD( |
| filter_ctl, FRF_CZ_ETHERNET_FULL_SEARCH_LIMIT, |
| table->search_depth[EFX_FILTER_MAC_FULL] + |
| FILTER_CTL_SRCH_FUDGE_FULL); |
| EFX_SET_OWORD_FIELD( |
| filter_ctl, FRF_CZ_ETHERNET_WILDCARD_SEARCH_LIMIT, |
| table->search_depth[EFX_FILTER_MAC_WILD] + |
| FILTER_CTL_SRCH_FUDGE_WILD); |
| } |
| |
| table = &state->table[EFX_FILTER_TABLE_RX_DEF]; |
| if (table->size) { |
| EFX_SET_OWORD_FIELD( |
| filter_ctl, FRF_CZ_UNICAST_NOMATCH_Q_ID, |
| table->spec[EFX_FILTER_INDEX_UC_DEF].dmaq_id); |
| EFX_SET_OWORD_FIELD( |
| filter_ctl, FRF_CZ_UNICAST_NOMATCH_RSS_ENABLED, |
| !!(table->spec[EFX_FILTER_INDEX_UC_DEF].flags & |
| EFX_FILTER_FLAG_RX_RSS)); |
| EFX_SET_OWORD_FIELD( |
| filter_ctl, FRF_CZ_MULTICAST_NOMATCH_Q_ID, |
| table->spec[EFX_FILTER_INDEX_MC_DEF].dmaq_id); |
| EFX_SET_OWORD_FIELD( |
| filter_ctl, FRF_CZ_MULTICAST_NOMATCH_RSS_ENABLED, |
| !!(table->spec[EFX_FILTER_INDEX_MC_DEF].flags & |
| EFX_FILTER_FLAG_RX_RSS)); |
| |
| /* There is a single bit to enable RX scatter for all |
| * unmatched packets. Only set it if scatter is |
| * enabled in both filter specs. |
| */ |
| EFX_SET_OWORD_FIELD( |
| filter_ctl, FRF_BZ_SCATTER_ENBL_NO_MATCH_Q, |
| !!(table->spec[EFX_FILTER_INDEX_UC_DEF].flags & |
| table->spec[EFX_FILTER_INDEX_MC_DEF].flags & |
| EFX_FILTER_FLAG_RX_SCATTER)); |
| } else if (efx_nic_rev(efx) >= EFX_REV_FALCON_B0) { |
| /* We don't expose 'default' filters because unmatched |
| * packets always go to the queue number found in the |
| * RSS table. But we still need to set the RX scatter |
| * bit here. |
| */ |
| EFX_SET_OWORD_FIELD( |
| filter_ctl, FRF_BZ_SCATTER_ENBL_NO_MATCH_Q, |
| efx->rx_scatter); |
| } |
| |
| efx_writeo(efx, &filter_ctl, FR_BZ_RX_FILTER_CTL); |
| } |
| |
| static void efx_filter_push_tx_limits(struct efx_nic *efx) |
| { |
| struct efx_filter_state *state = efx->filter_state; |
| struct efx_filter_table *table; |
| efx_oword_t tx_cfg; |
| |
| efx_reado(efx, &tx_cfg, FR_AZ_TX_CFG); |
| |
| table = &state->table[EFX_FILTER_TABLE_TX_MAC]; |
| if (table->size) { |
| EFX_SET_OWORD_FIELD( |
| tx_cfg, FRF_CZ_TX_ETH_FILTER_FULL_SEARCH_RANGE, |
| table->search_depth[EFX_FILTER_MAC_FULL] + |
| FILTER_CTL_SRCH_FUDGE_FULL); |
| EFX_SET_OWORD_FIELD( |
| tx_cfg, FRF_CZ_TX_ETH_FILTER_WILD_SEARCH_RANGE, |
| table->search_depth[EFX_FILTER_MAC_WILD] + |
| FILTER_CTL_SRCH_FUDGE_WILD); |
| } |
| |
| efx_writeo(efx, &tx_cfg, FR_AZ_TX_CFG); |
| } |
| |
| static inline void __efx_filter_set_ipv4(struct efx_filter_spec *spec, |
| __be32 host1, __be16 port1, |
| __be32 host2, __be16 port2) |
| { |
| spec->data[0] = ntohl(host1) << 16 | ntohs(port1); |
| spec->data[1] = ntohs(port2) << 16 | ntohl(host1) >> 16; |
| spec->data[2] = ntohl(host2); |
| } |
| |
| static inline void __efx_filter_get_ipv4(const struct efx_filter_spec *spec, |
| __be32 *host1, __be16 *port1, |
| __be32 *host2, __be16 *port2) |
| { |
| *host1 = htonl(spec->data[0] >> 16 | spec->data[1] << 16); |
| *port1 = htons(spec->data[0]); |
| *host2 = htonl(spec->data[2]); |
| *port2 = htons(spec->data[1] >> 16); |
| } |
| |
| /** |
| * efx_filter_set_ipv4_local - specify IPv4 host, transport protocol and port |
| * @spec: Specification to initialise |
| * @proto: Transport layer protocol number |
| * @host: Local host address (network byte order) |
| * @port: Local port (network byte order) |
| */ |
| int efx_filter_set_ipv4_local(struct efx_filter_spec *spec, u8 proto, |
| __be32 host, __be16 port) |
| { |
| __be32 host1; |
| __be16 port1; |
| |
| EFX_BUG_ON_PARANOID(!(spec->flags & EFX_FILTER_FLAG_RX)); |
| |
| /* This cannot currently be combined with other filtering */ |
| if (spec->type != EFX_FILTER_UNSPEC) |
| return -EPROTONOSUPPORT; |
| |
| if (port == 0) |
| return -EINVAL; |
| |
| switch (proto) { |
| case IPPROTO_TCP: |
| spec->type = EFX_FILTER_TCP_WILD; |
| break; |
| case IPPROTO_UDP: |
| spec->type = EFX_FILTER_UDP_WILD; |
| break; |
| default: |
| return -EPROTONOSUPPORT; |
| } |
| |
| /* Filter is constructed in terms of source and destination, |
| * with the odd wrinkle that the ports are swapped in a UDP |
| * wildcard filter. We need to convert from local and remote |
| * (= zero for wildcard) addresses. |
| */ |
| host1 = 0; |
| if (proto != IPPROTO_UDP) { |
| port1 = 0; |
| } else { |
| port1 = port; |
| port = 0; |
| } |
| |
| __efx_filter_set_ipv4(spec, host1, port1, host, port); |
| return 0; |
| } |
| |
| int efx_filter_get_ipv4_local(const struct efx_filter_spec *spec, |
| u8 *proto, __be32 *host, __be16 *port) |
| { |
| __be32 host1; |
| __be16 port1; |
| |
| switch (spec->type) { |
| case EFX_FILTER_TCP_WILD: |
| *proto = IPPROTO_TCP; |
| __efx_filter_get_ipv4(spec, &host1, &port1, host, port); |
| return 0; |
| case EFX_FILTER_UDP_WILD: |
| *proto = IPPROTO_UDP; |
| __efx_filter_get_ipv4(spec, &host1, port, host, &port1); |
| return 0; |
| default: |
| return -EINVAL; |
| } |
| } |
| |
| /** |
| * efx_filter_set_ipv4_full - specify IPv4 hosts, transport protocol and ports |
| * @spec: Specification to initialise |
| * @proto: Transport layer protocol number |
| * @host: Local host address (network byte order) |
| * @port: Local port (network byte order) |
| * @rhost: Remote host address (network byte order) |
| * @rport: Remote port (network byte order) |
| */ |
| int efx_filter_set_ipv4_full(struct efx_filter_spec *spec, u8 proto, |
| __be32 host, __be16 port, |
| __be32 rhost, __be16 rport) |
| { |
| EFX_BUG_ON_PARANOID(!(spec->flags & EFX_FILTER_FLAG_RX)); |
| |
| /* This cannot currently be combined with other filtering */ |
| if (spec->type != EFX_FILTER_UNSPEC) |
| return -EPROTONOSUPPORT; |
| |
| if (port == 0 || rport == 0) |
| return -EINVAL; |
| |
| switch (proto) { |
| case IPPROTO_TCP: |
| spec->type = EFX_FILTER_TCP_FULL; |
| break; |
| case IPPROTO_UDP: |
| spec->type = EFX_FILTER_UDP_FULL; |
| break; |
| default: |
| return -EPROTONOSUPPORT; |
| } |
| |
| __efx_filter_set_ipv4(spec, rhost, rport, host, port); |
| return 0; |
| } |
| |
| int efx_filter_get_ipv4_full(const struct efx_filter_spec *spec, |
| u8 *proto, __be32 *host, __be16 *port, |
| __be32 *rhost, __be16 *rport) |
| { |
| switch (spec->type) { |
| case EFX_FILTER_TCP_FULL: |
| *proto = IPPROTO_TCP; |
| break; |
| case EFX_FILTER_UDP_FULL: |
| *proto = IPPROTO_UDP; |
| break; |
| default: |
| return -EINVAL; |
| } |
| |
| __efx_filter_get_ipv4(spec, rhost, rport, host, port); |
| return 0; |
| } |
| |
| /** |
| * efx_filter_set_eth_local - specify local Ethernet address and optional VID |
| * @spec: Specification to initialise |
| * @vid: VLAN ID to match, or %EFX_FILTER_VID_UNSPEC |
| * @addr: Local Ethernet MAC address |
| */ |
| int efx_filter_set_eth_local(struct efx_filter_spec *spec, |
| u16 vid, const u8 *addr) |
| { |
| EFX_BUG_ON_PARANOID(!(spec->flags & |
| (EFX_FILTER_FLAG_RX | EFX_FILTER_FLAG_TX))); |
| |
| /* This cannot currently be combined with other filtering */ |
| if (spec->type != EFX_FILTER_UNSPEC) |
| return -EPROTONOSUPPORT; |
| |
| if (vid == EFX_FILTER_VID_UNSPEC) { |
| spec->type = EFX_FILTER_MAC_WILD; |
| spec->data[0] = 0; |
| } else { |
| spec->type = EFX_FILTER_MAC_FULL; |
| spec->data[0] = vid; |
| } |
| |
| spec->data[1] = addr[2] << 24 | addr[3] << 16 | addr[4] << 8 | addr[5]; |
| spec->data[2] = addr[0] << 8 | addr[1]; |
| return 0; |
| } |
| |
| /** |
| * efx_filter_set_uc_def - specify matching otherwise-unmatched unicast |
| * @spec: Specification to initialise |
| */ |
| int efx_filter_set_uc_def(struct efx_filter_spec *spec) |
| { |
| EFX_BUG_ON_PARANOID(!(spec->flags & |
| (EFX_FILTER_FLAG_RX | EFX_FILTER_FLAG_TX))); |
| |
| if (spec->type != EFX_FILTER_UNSPEC) |
| return -EINVAL; |
| |
| spec->type = EFX_FILTER_UC_DEF; |
| memset(spec->data, 0, sizeof(spec->data)); /* ensure equality */ |
| return 0; |
| } |
| |
| /** |
| * efx_filter_set_mc_def - specify matching otherwise-unmatched multicast |
| * @spec: Specification to initialise |
| */ |
| int efx_filter_set_mc_def(struct efx_filter_spec *spec) |
| { |
| EFX_BUG_ON_PARANOID(!(spec->flags & |
| (EFX_FILTER_FLAG_RX | EFX_FILTER_FLAG_TX))); |
| |
| if (spec->type != EFX_FILTER_UNSPEC) |
| return -EINVAL; |
| |
| spec->type = EFX_FILTER_MC_DEF; |
| memset(spec->data, 0, sizeof(spec->data)); /* ensure equality */ |
| return 0; |
| } |
| |
| static void efx_filter_reset_rx_def(struct efx_nic *efx, unsigned filter_idx) |
| { |
| struct efx_filter_state *state = efx->filter_state; |
| struct efx_filter_table *table = &state->table[EFX_FILTER_TABLE_RX_DEF]; |
| struct efx_filter_spec *spec = &table->spec[filter_idx]; |
| enum efx_filter_flags flags = 0; |
| |
| /* If there's only one channel then disable RSS for non VF |
| * traffic, thereby allowing VFs to use RSS when the PF can't. |
| */ |
| if (efx->n_rx_channels > 1) |
| flags |= EFX_FILTER_FLAG_RX_RSS; |
| |
| if (efx->rx_scatter) |
| flags |= EFX_FILTER_FLAG_RX_SCATTER; |
| |
| efx_filter_init_rx(spec, EFX_FILTER_PRI_MANUAL, flags, 0); |
| spec->type = EFX_FILTER_UC_DEF + filter_idx; |
| table->used_bitmap[0] |= 1 << filter_idx; |
| } |
| |
| int efx_filter_get_eth_local(const struct efx_filter_spec *spec, |
| u16 *vid, u8 *addr) |
| { |
| switch (spec->type) { |
| case EFX_FILTER_MAC_WILD: |
| *vid = EFX_FILTER_VID_UNSPEC; |
| break; |
| case EFX_FILTER_MAC_FULL: |
| *vid = spec->data[0]; |
| break; |
| default: |
| return -EINVAL; |
| } |
| |
| addr[0] = spec->data[2] >> 8; |
| addr[1] = spec->data[2]; |
| addr[2] = spec->data[1] >> 24; |
| addr[3] = spec->data[1] >> 16; |
| addr[4] = spec->data[1] >> 8; |
| addr[5] = spec->data[1]; |
| return 0; |
| } |
| |
| /* Build a filter entry and return its n-tuple key. */ |
| static u32 efx_filter_build(efx_oword_t *filter, struct efx_filter_spec *spec) |
| { |
| u32 data3; |
| |
| switch (efx_filter_spec_table_id(spec)) { |
| case EFX_FILTER_TABLE_RX_IP: { |
| bool is_udp = (spec->type == EFX_FILTER_UDP_FULL || |
| spec->type == EFX_FILTER_UDP_WILD); |
| EFX_POPULATE_OWORD_7( |
| *filter, |
| FRF_BZ_RSS_EN, |
| !!(spec->flags & EFX_FILTER_FLAG_RX_RSS), |
| FRF_BZ_SCATTER_EN, |
| !!(spec->flags & EFX_FILTER_FLAG_RX_SCATTER), |
| FRF_BZ_TCP_UDP, is_udp, |
| FRF_BZ_RXQ_ID, spec->dmaq_id, |
| EFX_DWORD_2, spec->data[2], |
| EFX_DWORD_1, spec->data[1], |
| EFX_DWORD_0, spec->data[0]); |
| data3 = is_udp; |
| break; |
| } |
| |
| case EFX_FILTER_TABLE_RX_MAC: { |
| bool is_wild = spec->type == EFX_FILTER_MAC_WILD; |
| EFX_POPULATE_OWORD_7( |
| *filter, |
| FRF_CZ_RMFT_RSS_EN, |
| !!(spec->flags & EFX_FILTER_FLAG_RX_RSS), |
| FRF_CZ_RMFT_SCATTER_EN, |
| !!(spec->flags & EFX_FILTER_FLAG_RX_SCATTER), |
| FRF_CZ_RMFT_RXQ_ID, spec->dmaq_id, |
| FRF_CZ_RMFT_WILDCARD_MATCH, is_wild, |
| FRF_CZ_RMFT_DEST_MAC_HI, spec->data[2], |
| FRF_CZ_RMFT_DEST_MAC_LO, spec->data[1], |
| FRF_CZ_RMFT_VLAN_ID, spec->data[0]); |
| data3 = is_wild; |
| break; |
| } |
| |
| case EFX_FILTER_TABLE_TX_MAC: { |
| bool is_wild = spec->type == EFX_FILTER_MAC_WILD; |
| EFX_POPULATE_OWORD_5(*filter, |
| FRF_CZ_TMFT_TXQ_ID, spec->dmaq_id, |
| FRF_CZ_TMFT_WILDCARD_MATCH, is_wild, |
| FRF_CZ_TMFT_SRC_MAC_HI, spec->data[2], |
| FRF_CZ_TMFT_SRC_MAC_LO, spec->data[1], |
| FRF_CZ_TMFT_VLAN_ID, spec->data[0]); |
| data3 = is_wild | spec->dmaq_id << 1; |
| break; |
| } |
| |
| default: |
| BUG(); |
| } |
| |
| return spec->data[0] ^ spec->data[1] ^ spec->data[2] ^ data3; |
| } |
| |
| static bool efx_filter_equal(const struct efx_filter_spec *left, |
| const struct efx_filter_spec *right) |
| { |
| if (left->type != right->type || |
| memcmp(left->data, right->data, sizeof(left->data))) |
| return false; |
| |
| if (left->flags & EFX_FILTER_FLAG_TX && |
| left->dmaq_id != right->dmaq_id) |
| return false; |
| |
| return true; |
| } |
| |
| /* |
| * Construct/deconstruct external filter IDs. At least the RX filter |
| * IDs must be ordered by matching priority, for RX NFC semantics. |
| * |
| * Deconstruction needs to be robust against invalid IDs so that |
| * efx_filter_remove_id_safe() and efx_filter_get_filter_safe() can |
| * accept user-provided IDs. |
| */ |
| |
| #define EFX_FILTER_MATCH_PRI_COUNT 5 |
| |
| static const u8 efx_filter_type_match_pri[EFX_FILTER_TYPE_COUNT] = { |
| [EFX_FILTER_TCP_FULL] = 0, |
| [EFX_FILTER_UDP_FULL] = 0, |
| [EFX_FILTER_TCP_WILD] = 1, |
| [EFX_FILTER_UDP_WILD] = 1, |
| [EFX_FILTER_MAC_FULL] = 2, |
| [EFX_FILTER_MAC_WILD] = 3, |
| [EFX_FILTER_UC_DEF] = 4, |
| [EFX_FILTER_MC_DEF] = 4, |
| }; |
| |
| static const enum efx_filter_table_id efx_filter_range_table[] = { |
| EFX_FILTER_TABLE_RX_IP, /* RX match pri 0 */ |
| EFX_FILTER_TABLE_RX_IP, |
| EFX_FILTER_TABLE_RX_MAC, |
| EFX_FILTER_TABLE_RX_MAC, |
| EFX_FILTER_TABLE_RX_DEF, /* RX match pri 4 */ |
| EFX_FILTER_TABLE_COUNT, /* TX match pri 0; invalid */ |
| EFX_FILTER_TABLE_COUNT, /* invalid */ |
| EFX_FILTER_TABLE_TX_MAC, |
| EFX_FILTER_TABLE_TX_MAC, /* TX match pri 3 */ |
| }; |
| |
| #define EFX_FILTER_INDEX_WIDTH 13 |
| #define EFX_FILTER_INDEX_MASK ((1 << EFX_FILTER_INDEX_WIDTH) - 1) |
| |
| static inline u32 |
| efx_filter_make_id(const struct efx_filter_spec *spec, unsigned int index) |
| { |
| unsigned int range; |
| |
| range = efx_filter_type_match_pri[spec->type]; |
| if (!(spec->flags & EFX_FILTER_FLAG_RX)) |
| range += EFX_FILTER_MATCH_PRI_COUNT; |
| |
| return range << EFX_FILTER_INDEX_WIDTH | index; |
| } |
| |
| static inline enum efx_filter_table_id efx_filter_id_table_id(u32 id) |
| { |
| unsigned int range = id >> EFX_FILTER_INDEX_WIDTH; |
| |
| if (range < ARRAY_SIZE(efx_filter_range_table)) |
| return efx_filter_range_table[range]; |
| else |
| return EFX_FILTER_TABLE_COUNT; /* invalid */ |
| } |
| |
| static inline unsigned int efx_filter_id_index(u32 id) |
| { |
| return id & EFX_FILTER_INDEX_MASK; |
| } |
| |
| static inline u8 efx_filter_id_flags(u32 id) |
| { |
| unsigned int range = id >> EFX_FILTER_INDEX_WIDTH; |
| |
| if (range < EFX_FILTER_MATCH_PRI_COUNT) |
| return EFX_FILTER_FLAG_RX; |
| else |
| return EFX_FILTER_FLAG_TX; |
| } |
| |
| u32 efx_filter_get_rx_id_limit(struct efx_nic *efx) |
| { |
| struct efx_filter_state *state = efx->filter_state; |
| unsigned int range = EFX_FILTER_MATCH_PRI_COUNT - 1; |
| enum efx_filter_table_id table_id; |
| |
| do { |
| table_id = efx_filter_range_table[range]; |
| if (state->table[table_id].size != 0) |
| return range << EFX_FILTER_INDEX_WIDTH | |
| state->table[table_id].size; |
| } while (range--); |
| |
| return 0; |
| } |
| |
| /** |
| * efx_filter_insert_filter - add or replace a filter |
| * @efx: NIC in which to insert the filter |
| * @spec: Specification for the filter |
| * @replace_equal: Flag for whether the specified filter may replace an |
| * existing filter with equal priority |
| * |
| * On success, return the filter ID. |
| * On failure, return a negative error code. |
| * |
| * If an existing filter has equal match values to the new filter |
| * spec, then the new filter might replace it, depending on the |
| * relative priorities. If the existing filter has lower priority, or |
| * if @replace_equal is set and it has equal priority, then it is |
| * replaced. Otherwise the function fails, returning -%EPERM if |
| * the existing filter has higher priority or -%EEXIST if it has |
| * equal priority. |
| */ |
| s32 efx_filter_insert_filter(struct efx_nic *efx, struct efx_filter_spec *spec, |
| bool replace_equal) |
| { |
| struct efx_filter_state *state = efx->filter_state; |
| struct efx_filter_table *table = efx_filter_spec_table(state, spec); |
| efx_oword_t filter; |
| int rep_index, ins_index; |
| unsigned int depth = 0; |
| int rc; |
| |
| if (!table || table->size == 0) |
| return -EINVAL; |
| |
| netif_vdbg(efx, hw, efx->net_dev, |
| "%s: type %d search_depth=%d", __func__, spec->type, |
| table->search_depth[spec->type]); |
| |
| if (table->id == EFX_FILTER_TABLE_RX_DEF) { |
| /* One filter spec per type */ |
| BUILD_BUG_ON(EFX_FILTER_INDEX_UC_DEF != 0); |
| BUILD_BUG_ON(EFX_FILTER_INDEX_MC_DEF != |
| EFX_FILTER_MC_DEF - EFX_FILTER_UC_DEF); |
| rep_index = spec->type - EFX_FILTER_INDEX_UC_DEF; |
| ins_index = rep_index; |
| |
| spin_lock_bh(&state->lock); |
| } else { |
| /* Search concurrently for |
| * (1) a filter to be replaced (rep_index): any filter |
| * with the same match values, up to the current |
| * search depth for this type, and |
| * (2) the insertion point (ins_index): (1) or any |
| * free slot before it or up to the maximum search |
| * depth for this priority |
| * We fail if we cannot find (2). |
| * |
| * We can stop once either |
| * (a) we find (1), in which case we have definitely |
| * found (2) as well; or |
| * (b) we have searched exhaustively for (1), and have |
| * either found (2) or searched exhaustively for it |
| */ |
| u32 key = efx_filter_build(&filter, spec); |
| unsigned int hash = efx_filter_hash(key); |
| unsigned int incr = efx_filter_increment(key); |
| unsigned int max_rep_depth = table->search_depth[spec->type]; |
| unsigned int max_ins_depth = |
| spec->priority <= EFX_FILTER_PRI_HINT ? |
| FILTER_CTL_SRCH_HINT_MAX : FILTER_CTL_SRCH_MAX; |
| unsigned int i = hash & (table->size - 1); |
| |
| ins_index = -1; |
| depth = 1; |
| |
| spin_lock_bh(&state->lock); |
| |
| for (;;) { |
| if (!test_bit(i, table->used_bitmap)) { |
| if (ins_index < 0) |
| ins_index = i; |
| } else if (efx_filter_equal(spec, &table->spec[i])) { |
| /* Case (a) */ |
| if (ins_index < 0) |
| ins_index = i; |
| rep_index = i; |
| break; |
| } |
| |
| if (depth >= max_rep_depth && |
| (ins_index >= 0 || depth >= max_ins_depth)) { |
| /* Case (b) */ |
| if (ins_index < 0) { |
| rc = -EBUSY; |
| goto out; |
| } |
| rep_index = -1; |
| break; |
| } |
| |
| i = (i + incr) & (table->size - 1); |
| ++depth; |
| } |
| } |
| |
| /* If we found a filter to be replaced, check whether we |
| * should do so |
| */ |
| if (rep_index >= 0) { |
| struct efx_filter_spec *saved_spec = &table->spec[rep_index]; |
| |
| if (spec->priority == saved_spec->priority && !replace_equal) { |
| rc = -EEXIST; |
| goto out; |
| } |
| if (spec->priority < saved_spec->priority) { |
| rc = -EPERM; |
| goto out; |
| } |
| } |
| |
| /* Insert the filter */ |
| if (ins_index != rep_index) { |
| __set_bit(ins_index, table->used_bitmap); |
| ++table->used; |
| } |
| table->spec[ins_index] = *spec; |
| |
| if (table->id == EFX_FILTER_TABLE_RX_DEF) { |
| efx_filter_push_rx_config(efx); |
| } else { |
| if (table->search_depth[spec->type] < depth) { |
| table->search_depth[spec->type] = depth; |
| if (spec->flags & EFX_FILTER_FLAG_TX) |
| efx_filter_push_tx_limits(efx); |
| else |
| efx_filter_push_rx_config(efx); |
| } |
| |
| efx_writeo(efx, &filter, |
| table->offset + table->step * ins_index); |
| |
| /* If we were able to replace a filter by inserting |
| * at a lower depth, clear the replaced filter |
| */ |
| if (ins_index != rep_index && rep_index >= 0) |
| efx_filter_table_clear_entry(efx, table, rep_index); |
| } |
| |
| netif_vdbg(efx, hw, efx->net_dev, |
| "%s: filter type %d index %d rxq %u set", |
| __func__, spec->type, ins_index, spec->dmaq_id); |
| rc = efx_filter_make_id(spec, ins_index); |
| |
| out: |
| spin_unlock_bh(&state->lock); |
| return rc; |
| } |
| |
| static void efx_filter_table_clear_entry(struct efx_nic *efx, |
| struct efx_filter_table *table, |
| unsigned int filter_idx) |
| { |
| static efx_oword_t filter; |
| |
| if (table->id == EFX_FILTER_TABLE_RX_DEF) { |
| /* RX default filters must always exist */ |
| efx_filter_reset_rx_def(efx, filter_idx); |
| efx_filter_push_rx_config(efx); |
| } else if (test_bit(filter_idx, table->used_bitmap)) { |
| __clear_bit(filter_idx, table->used_bitmap); |
| --table->used; |
| memset(&table->spec[filter_idx], 0, sizeof(table->spec[0])); |
| |
| efx_writeo(efx, &filter, |
| table->offset + table->step * filter_idx); |
| } |
| } |
| |
| /** |
| * efx_filter_remove_id_safe - remove a filter by ID, carefully |
| * @efx: NIC from which to remove the filter |
| * @priority: Priority of filter, as passed to @efx_filter_insert_filter |
| * @filter_id: ID of filter, as returned by @efx_filter_insert_filter |
| * |
| * This function will range-check @filter_id, so it is safe to call |
| * with a value passed from userland. |
| */ |
| int efx_filter_remove_id_safe(struct efx_nic *efx, |
| enum efx_filter_priority priority, |
| u32 filter_id) |
| { |
| struct efx_filter_state *state = efx->filter_state; |
| enum efx_filter_table_id table_id; |
| struct efx_filter_table *table; |
| unsigned int filter_idx; |
| struct efx_filter_spec *spec; |
| u8 filter_flags; |
| int rc; |
| |
| table_id = efx_filter_id_table_id(filter_id); |
| if ((unsigned int)table_id >= EFX_FILTER_TABLE_COUNT) |
| return -ENOENT; |
| table = &state->table[table_id]; |
| |
| filter_idx = efx_filter_id_index(filter_id); |
| if (filter_idx >= table->size) |
| return -ENOENT; |
| spec = &table->spec[filter_idx]; |
| |
| filter_flags = efx_filter_id_flags(filter_id); |
| |
| spin_lock_bh(&state->lock); |
| |
| if (test_bit(filter_idx, table->used_bitmap) && |
| spec->priority == priority) { |
| efx_filter_table_clear_entry(efx, table, filter_idx); |
| if (table->used == 0) |
| efx_filter_table_reset_search_depth(table); |
| rc = 0; |
| } else { |
| rc = -ENOENT; |
| } |
| |
| spin_unlock_bh(&state->lock); |
| |
| return rc; |
| } |
| |
| /** |
| * efx_filter_get_filter_safe - retrieve a filter by ID, carefully |
| * @efx: NIC from which to remove the filter |
| * @priority: Priority of filter, as passed to @efx_filter_insert_filter |
| * @filter_id: ID of filter, as returned by @efx_filter_insert_filter |
| * @spec: Buffer in which to store filter specification |
| * |
| * This function will range-check @filter_id, so it is safe to call |
| * with a value passed from userland. |
| */ |
| int efx_filter_get_filter_safe(struct efx_nic *efx, |
| enum efx_filter_priority priority, |
| u32 filter_id, struct efx_filter_spec *spec_buf) |
| { |
| struct efx_filter_state *state = efx->filter_state; |
| enum efx_filter_table_id table_id; |
| struct efx_filter_table *table; |
| struct efx_filter_spec *spec; |
| unsigned int filter_idx; |
| u8 filter_flags; |
| int rc; |
| |
| table_id = efx_filter_id_table_id(filter_id); |
| if ((unsigned int)table_id >= EFX_FILTER_TABLE_COUNT) |
| return -ENOENT; |
| table = &state->table[table_id]; |
| |
| filter_idx = efx_filter_id_index(filter_id); |
| if (filter_idx >= table->size) |
| return -ENOENT; |
| spec = &table->spec[filter_idx]; |
| |
| filter_flags = efx_filter_id_flags(filter_id); |
| |
| spin_lock_bh(&state->lock); |
| |
| if (test_bit(filter_idx, table->used_bitmap) && |
| spec->priority == priority) { |
| *spec_buf = *spec; |
| rc = 0; |
| } else { |
| rc = -ENOENT; |
| } |
| |
| spin_unlock_bh(&state->lock); |
| |
| return rc; |
| } |
| |
| static void efx_filter_table_clear(struct efx_nic *efx, |
| enum efx_filter_table_id table_id, |
| enum efx_filter_priority priority) |
| { |
| struct efx_filter_state *state = efx->filter_state; |
| struct efx_filter_table *table = &state->table[table_id]; |
| unsigned int filter_idx; |
| |
| spin_lock_bh(&state->lock); |
| |
| for (filter_idx = 0; filter_idx < table->size; ++filter_idx) |
| if (table->spec[filter_idx].priority <= priority) |
| efx_filter_table_clear_entry(efx, table, filter_idx); |
| if (table->used == 0) |
| efx_filter_table_reset_search_depth(table); |
| |
| spin_unlock_bh(&state->lock); |
| } |
| |
| /** |
| * efx_filter_clear_rx - remove RX filters by priority |
| * @efx: NIC from which to remove the filters |
| * @priority: Maximum priority to remove |
| */ |
| void efx_filter_clear_rx(struct efx_nic *efx, enum efx_filter_priority priority) |
| { |
| efx_filter_table_clear(efx, EFX_FILTER_TABLE_RX_IP, priority); |
| efx_filter_table_clear(efx, EFX_FILTER_TABLE_RX_MAC, priority); |
| } |
| |
| u32 efx_filter_count_rx_used(struct efx_nic *efx, |
| enum efx_filter_priority priority) |
| { |
| struct efx_filter_state *state = efx->filter_state; |
| enum efx_filter_table_id table_id; |
| struct efx_filter_table *table; |
| unsigned int filter_idx; |
| u32 count = 0; |
| |
| spin_lock_bh(&state->lock); |
| |
| for (table_id = EFX_FILTER_TABLE_RX_IP; |
| table_id <= EFX_FILTER_TABLE_RX_DEF; |
| table_id++) { |
| table = &state->table[table_id]; |
| for (filter_idx = 0; filter_idx < table->size; filter_idx++) { |
| if (test_bit(filter_idx, table->used_bitmap) && |
| table->spec[filter_idx].priority == priority) |
| ++count; |
| } |
| } |
| |
| spin_unlock_bh(&state->lock); |
| |
| return count; |
| } |
| |
| s32 efx_filter_get_rx_ids(struct efx_nic *efx, |
| enum efx_filter_priority priority, |
| u32 *buf, u32 size) |
| { |
| struct efx_filter_state *state = efx->filter_state; |
| enum efx_filter_table_id table_id; |
| struct efx_filter_table *table; |
| unsigned int filter_idx; |
| s32 count = 0; |
| |
| spin_lock_bh(&state->lock); |
| |
| for (table_id = EFX_FILTER_TABLE_RX_IP; |
| table_id <= EFX_FILTER_TABLE_RX_DEF; |
| table_id++) { |
| table = &state->table[table_id]; |
| for (filter_idx = 0; filter_idx < table->size; filter_idx++) { |
| if (test_bit(filter_idx, table->used_bitmap) && |
| table->spec[filter_idx].priority == priority) { |
| if (count == size) { |
| count = -EMSGSIZE; |
| goto out; |
| } |
| buf[count++] = efx_filter_make_id( |
| &table->spec[filter_idx], filter_idx); |
| } |
| } |
| } |
| out: |
| spin_unlock_bh(&state->lock); |
| |
| return count; |
| } |
| |
| /* Restore filter stater after reset */ |
| void efx_restore_filters(struct efx_nic *efx) |
| { |
| struct efx_filter_state *state = efx->filter_state; |
| enum efx_filter_table_id table_id; |
| struct efx_filter_table *table; |
| efx_oword_t filter; |
| unsigned int filter_idx; |
| |
| spin_lock_bh(&state->lock); |
| |
| for (table_id = 0; table_id < EFX_FILTER_TABLE_COUNT; table_id++) { |
| table = &state->table[table_id]; |
| |
| /* Check whether this is a regular register table */ |
| if (table->step == 0) |
| continue; |
| |
| for (filter_idx = 0; filter_idx < table->size; filter_idx++) { |
| if (!test_bit(filter_idx, table->used_bitmap)) |
| continue; |
| efx_filter_build(&filter, &table->spec[filter_idx]); |
| efx_writeo(efx, &filter, |
| table->offset + table->step * filter_idx); |
| } |
| } |
| |
| efx_filter_push_rx_config(efx); |
| efx_filter_push_tx_limits(efx); |
| |
| spin_unlock_bh(&state->lock); |
| } |
| |
| int efx_probe_filters(struct efx_nic *efx) |
| { |
| struct efx_filter_state *state; |
| struct efx_filter_table *table; |
| unsigned table_id; |
| |
| state = kzalloc(sizeof(*efx->filter_state), GFP_KERNEL); |
| if (!state) |
| return -ENOMEM; |
| efx->filter_state = state; |
| |
| spin_lock_init(&state->lock); |
| |
| if (efx_nic_rev(efx) >= EFX_REV_FALCON_B0) { |
| #ifdef CONFIG_RFS_ACCEL |
| state->rps_flow_id = kcalloc(FR_BZ_RX_FILTER_TBL0_ROWS, |
| sizeof(*state->rps_flow_id), |
| GFP_KERNEL); |
| if (!state->rps_flow_id) |
| goto fail; |
| #endif |
| table = &state->table[EFX_FILTER_TABLE_RX_IP]; |
| table->id = EFX_FILTER_TABLE_RX_IP; |
| table->offset = FR_BZ_RX_FILTER_TBL0; |
| table->size = FR_BZ_RX_FILTER_TBL0_ROWS; |
| table->step = FR_BZ_RX_FILTER_TBL0_STEP; |
| } |
| |
| if (efx_nic_rev(efx) >= EFX_REV_SIENA_A0) { |
| table = &state->table[EFX_FILTER_TABLE_RX_MAC]; |
| table->id = EFX_FILTER_TABLE_RX_MAC; |
| table->offset = FR_CZ_RX_MAC_FILTER_TBL0; |
| table->size = FR_CZ_RX_MAC_FILTER_TBL0_ROWS; |
| table->step = FR_CZ_RX_MAC_FILTER_TBL0_STEP; |
| |
| table = &state->table[EFX_FILTER_TABLE_RX_DEF]; |
| table->id = EFX_FILTER_TABLE_RX_DEF; |
| table->size = EFX_FILTER_SIZE_RX_DEF; |
| |
| table = &state->table[EFX_FILTER_TABLE_TX_MAC]; |
| table->id = EFX_FILTER_TABLE_TX_MAC; |
| table->offset = FR_CZ_TX_MAC_FILTER_TBL0; |
| table->size = FR_CZ_TX_MAC_FILTER_TBL0_ROWS; |
| table->step = FR_CZ_TX_MAC_FILTER_TBL0_STEP; |
| } |
| |
| for (table_id = 0; table_id < EFX_FILTER_TABLE_COUNT; table_id++) { |
| table = &state->table[table_id]; |
| if (table->size == 0) |
| continue; |
| table->used_bitmap = kcalloc(BITS_TO_LONGS(table->size), |
| sizeof(unsigned long), |
| GFP_KERNEL); |
| if (!table->used_bitmap) |
| goto fail; |
| table->spec = vzalloc(table->size * sizeof(*table->spec)); |
| if (!table->spec) |
| goto fail; |
| } |
| |
| if (state->table[EFX_FILTER_TABLE_RX_DEF].size) { |
| /* RX default filters must always exist */ |
| unsigned i; |
| for (i = 0; i < EFX_FILTER_SIZE_RX_DEF; i++) |
| efx_filter_reset_rx_def(efx, i); |
| } |
| |
| efx_filter_push_rx_config(efx); |
| |
| return 0; |
| |
| fail: |
| efx_remove_filters(efx); |
| return -ENOMEM; |
| } |
| |
| void efx_remove_filters(struct efx_nic *efx) |
| { |
| struct efx_filter_state *state = efx->filter_state; |
| enum efx_filter_table_id table_id; |
| |
| for (table_id = 0; table_id < EFX_FILTER_TABLE_COUNT; table_id++) { |
| kfree(state->table[table_id].used_bitmap); |
| vfree(state->table[table_id].spec); |
| } |
| #ifdef CONFIG_RFS_ACCEL |
| kfree(state->rps_flow_id); |
| #endif |
| kfree(state); |
| } |
| |
| /* Update scatter enable flags for filters pointing to our own RX queues */ |
| void efx_filter_update_rx_scatter(struct efx_nic *efx) |
| { |
| struct efx_filter_state *state = efx->filter_state; |
| enum efx_filter_table_id table_id; |
| struct efx_filter_table *table; |
| efx_oword_t filter; |
| unsigned int filter_idx; |
| |
| spin_lock_bh(&state->lock); |
| |
| for (table_id = EFX_FILTER_TABLE_RX_IP; |
| table_id <= EFX_FILTER_TABLE_RX_DEF; |
| table_id++) { |
| table = &state->table[table_id]; |
| |
| for (filter_idx = 0; filter_idx < table->size; filter_idx++) { |
| if (!test_bit(filter_idx, table->used_bitmap) || |
| table->spec[filter_idx].dmaq_id >= |
| efx->n_rx_channels) |
| continue; |
| |
| if (efx->rx_scatter) |
| table->spec[filter_idx].flags |= |
| EFX_FILTER_FLAG_RX_SCATTER; |
| else |
| table->spec[filter_idx].flags &= |
| ~EFX_FILTER_FLAG_RX_SCATTER; |
| |
| if (table_id == EFX_FILTER_TABLE_RX_DEF) |
| /* Pushed by efx_filter_push_rx_config() */ |
| continue; |
| |
| efx_filter_build(&filter, &table->spec[filter_idx]); |
| efx_writeo(efx, &filter, |
| table->offset + table->step * filter_idx); |
| } |
| } |
| |
| efx_filter_push_rx_config(efx); |
| |
| spin_unlock_bh(&state->lock); |
| } |
| |
| #ifdef CONFIG_RFS_ACCEL |
| |
| int efx_filter_rfs(struct net_device *net_dev, const struct sk_buff *skb, |
| u16 rxq_index, u32 flow_id) |
| { |
| struct efx_nic *efx = netdev_priv(net_dev); |
| struct efx_channel *channel; |
| struct efx_filter_state *state = efx->filter_state; |
| struct efx_filter_spec spec; |
| const struct iphdr *ip; |
| const __be16 *ports; |
| int nhoff; |
| int rc; |
| |
| nhoff = skb_network_offset(skb); |
| |
| if (skb->protocol == htons(ETH_P_8021Q)) { |
| EFX_BUG_ON_PARANOID(skb_headlen(skb) < |
| nhoff + sizeof(struct vlan_hdr)); |
| if (((const struct vlan_hdr *)skb->data + nhoff)-> |
| h_vlan_encapsulated_proto != htons(ETH_P_IP)) |
| return -EPROTONOSUPPORT; |
| |
| /* This is IP over 802.1q VLAN. We can't filter on the |
| * IP 5-tuple and the vlan together, so just strip the |
| * vlan header and filter on the IP part. |
| */ |
| nhoff += sizeof(struct vlan_hdr); |
| } else if (skb->protocol != htons(ETH_P_IP)) { |
| return -EPROTONOSUPPORT; |
| } |
| |
| /* RFS must validate the IP header length before calling us */ |
| EFX_BUG_ON_PARANOID(skb_headlen(skb) < nhoff + sizeof(*ip)); |
| ip = (const struct iphdr *)(skb->data + nhoff); |
| if (ip_is_fragment(ip)) |
| return -EPROTONOSUPPORT; |
| EFX_BUG_ON_PARANOID(skb_headlen(skb) < nhoff + 4 * ip->ihl + 4); |
| ports = (const __be16 *)(skb->data + nhoff + 4 * ip->ihl); |
| |
| efx_filter_init_rx(&spec, EFX_FILTER_PRI_HINT, |
| efx->rx_scatter ? EFX_FILTER_FLAG_RX_SCATTER : 0, |
| rxq_index); |
| rc = efx_filter_set_ipv4_full(&spec, ip->protocol, |
| ip->daddr, ports[1], ip->saddr, ports[0]); |
| if (rc) |
| return rc; |
| |
| rc = efx_filter_insert_filter(efx, &spec, true); |
| if (rc < 0) |
| return rc; |
| |
| /* Remember this so we can check whether to expire the filter later */ |
| state->rps_flow_id[rc] = flow_id; |
| channel = efx_get_channel(efx, skb_get_rx_queue(skb)); |
| ++channel->rfs_filters_added; |
| |
| netif_info(efx, rx_status, efx->net_dev, |
| "steering %s %pI4:%u:%pI4:%u to queue %u [flow %u filter %d]\n", |
| (ip->protocol == IPPROTO_TCP) ? "TCP" : "UDP", |
| &ip->saddr, ntohs(ports[0]), &ip->daddr, ntohs(ports[1]), |
| rxq_index, flow_id, rc); |
| |
| return rc; |
| } |
| |
| bool __efx_filter_rfs_expire(struct efx_nic *efx, unsigned quota) |
| { |
| struct efx_filter_state *state = efx->filter_state; |
| struct efx_filter_table *table = &state->table[EFX_FILTER_TABLE_RX_IP]; |
| unsigned mask = table->size - 1; |
| unsigned index; |
| unsigned stop; |
| |
| if (!spin_trylock_bh(&state->lock)) |
| return false; |
| |
| index = state->rps_expire_index; |
| stop = (index + quota) & mask; |
| |
| while (index != stop) { |
| if (test_bit(index, table->used_bitmap) && |
| table->spec[index].priority == EFX_FILTER_PRI_HINT && |
| rps_may_expire_flow(efx->net_dev, |
| table->spec[index].dmaq_id, |
| state->rps_flow_id[index], index)) { |
| netif_info(efx, rx_status, efx->net_dev, |
| "expiring filter %d [flow %u]\n", |
| index, state->rps_flow_id[index]); |
| efx_filter_table_clear_entry(efx, table, index); |
| } |
| index = (index + 1) & mask; |
| } |
| |
| state->rps_expire_index = stop; |
| if (table->used == 0) |
| efx_filter_table_reset_search_depth(table); |
| |
| spin_unlock_bh(&state->lock); |
| return true; |
| } |
| |
| #endif /* CONFIG_RFS_ACCEL */ |