1 /****************************************************************************
2 * Driver for Solarflare Solarstorm network controllers and boards
3 * Copyright 2005-2010 Solarflare Communications Inc.
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 as published
7 * by the Free Software Foundation, incorporated herein by reference.
18 /* "Fudge factors" - difference between programmed value and actual depth.
19 * Due to pipelined implementation we need to program H/W with a value that
20 * is larger than the hop limit we want.
22 #define FILTER_CTL_SRCH_FUDGE_WILD 3
23 #define FILTER_CTL_SRCH_FUDGE_FULL 1
25 /* Hard maximum hop limit. Hardware will time-out beyond 200-something.
26 * We also need to avoid infinite loops in efx_filter_search() when the
29 #define FILTER_CTL_SRCH_MAX 200
31 /* Don't try very hard to find space for performance hints, as this is
32 * counter-productive. */
33 #define FILTER_CTL_SRCH_HINT_MAX 5
35 enum efx_filter_table_id
{
36 EFX_FILTER_TABLE_RX_IP
= 0,
37 EFX_FILTER_TABLE_RX_MAC
,
38 EFX_FILTER_TABLE_RX_DEF
,
39 EFX_FILTER_TABLE_TX_MAC
,
40 EFX_FILTER_TABLE_COUNT
,
43 enum efx_filter_index
{
44 EFX_FILTER_INDEX_UC_DEF
,
45 EFX_FILTER_INDEX_MC_DEF
,
46 EFX_FILTER_SIZE_RX_DEF
,
49 struct efx_filter_table
{
50 enum efx_filter_table_id id
;
51 u32 offset
; /* address of table relative to BAR */
52 unsigned size
; /* number of entries */
53 unsigned step
; /* step between entries */
54 unsigned used
; /* number currently used */
55 unsigned long *used_bitmap
;
56 struct efx_filter_spec
*spec
;
57 unsigned search_depth
[EFX_FILTER_TYPE_COUNT
];
60 struct efx_filter_state
{
62 struct efx_filter_table table
[EFX_FILTER_TABLE_COUNT
];
63 #ifdef CONFIG_RFS_ACCEL
65 unsigned rps_expire_index
;
69 /* The filter hash function is LFSR polynomial x^16 + x^3 + 1 of a 32-bit
70 * key derived from the n-tuple. The initial LFSR state is 0xffff. */
71 static u16
efx_filter_hash(u32 key
)
76 tmp
= 0x1fff ^ key
>> 16;
77 tmp
= tmp
^ tmp
>> 3 ^ tmp
>> 6;
80 tmp
= tmp
^ tmp
<< 13 ^ key
;
81 tmp
= tmp
^ tmp
>> 3 ^ tmp
>> 6;
82 return tmp
^ tmp
>> 9;
85 /* To allow for hash collisions, filter search continues at these
86 * increments from the first possible entry selected by the hash. */
87 static u16
efx_filter_increment(u32 key
)
92 static enum efx_filter_table_id
93 efx_filter_spec_table_id(const struct efx_filter_spec
*spec
)
95 BUILD_BUG_ON(EFX_FILTER_TABLE_RX_IP
!= (EFX_FILTER_TCP_FULL
>> 2));
96 BUILD_BUG_ON(EFX_FILTER_TABLE_RX_IP
!= (EFX_FILTER_TCP_WILD
>> 2));
97 BUILD_BUG_ON(EFX_FILTER_TABLE_RX_IP
!= (EFX_FILTER_UDP_FULL
>> 2));
98 BUILD_BUG_ON(EFX_FILTER_TABLE_RX_IP
!= (EFX_FILTER_UDP_WILD
>> 2));
99 BUILD_BUG_ON(EFX_FILTER_TABLE_RX_MAC
!= (EFX_FILTER_MAC_FULL
>> 2));
100 BUILD_BUG_ON(EFX_FILTER_TABLE_RX_MAC
!= (EFX_FILTER_MAC_WILD
>> 2));
101 BUILD_BUG_ON(EFX_FILTER_TABLE_TX_MAC
!= EFX_FILTER_TABLE_RX_MAC
+ 2);
102 EFX_BUG_ON_PARANOID(spec
->type
== EFX_FILTER_UNSPEC
);
103 return (spec
->type
>> 2) + ((spec
->flags
& EFX_FILTER_FLAG_TX
) ? 2 : 0);
106 static struct efx_filter_table
*
107 efx_filter_spec_table(struct efx_filter_state
*state
,
108 const struct efx_filter_spec
*spec
)
110 if (spec
->type
== EFX_FILTER_UNSPEC
)
113 return &state
->table
[efx_filter_spec_table_id(spec
)];
116 static void efx_filter_table_reset_search_depth(struct efx_filter_table
*table
)
118 memset(table
->search_depth
, 0, sizeof(table
->search_depth
));
121 static void efx_filter_push_rx_config(struct efx_nic
*efx
)
123 struct efx_filter_state
*state
= efx
->filter_state
;
124 struct efx_filter_table
*table
;
125 efx_oword_t filter_ctl
;
127 efx_reado(efx
, &filter_ctl
, FR_BZ_RX_FILTER_CTL
);
129 table
= &state
->table
[EFX_FILTER_TABLE_RX_IP
];
130 EFX_SET_OWORD_FIELD(filter_ctl
, FRF_BZ_TCP_FULL_SRCH_LIMIT
,
131 table
->search_depth
[EFX_FILTER_TCP_FULL
] +
132 FILTER_CTL_SRCH_FUDGE_FULL
);
133 EFX_SET_OWORD_FIELD(filter_ctl
, FRF_BZ_TCP_WILD_SRCH_LIMIT
,
134 table
->search_depth
[EFX_FILTER_TCP_WILD
] +
135 FILTER_CTL_SRCH_FUDGE_WILD
);
136 EFX_SET_OWORD_FIELD(filter_ctl
, FRF_BZ_UDP_FULL_SRCH_LIMIT
,
137 table
->search_depth
[EFX_FILTER_UDP_FULL
] +
138 FILTER_CTL_SRCH_FUDGE_FULL
);
139 EFX_SET_OWORD_FIELD(filter_ctl
, FRF_BZ_UDP_WILD_SRCH_LIMIT
,
140 table
->search_depth
[EFX_FILTER_UDP_WILD
] +
141 FILTER_CTL_SRCH_FUDGE_WILD
);
143 table
= &state
->table
[EFX_FILTER_TABLE_RX_MAC
];
146 filter_ctl
, FRF_CZ_ETHERNET_FULL_SEARCH_LIMIT
,
147 table
->search_depth
[EFX_FILTER_MAC_FULL
] +
148 FILTER_CTL_SRCH_FUDGE_FULL
);
150 filter_ctl
, FRF_CZ_ETHERNET_WILDCARD_SEARCH_LIMIT
,
151 table
->search_depth
[EFX_FILTER_MAC_WILD
] +
152 FILTER_CTL_SRCH_FUDGE_WILD
);
155 table
= &state
->table
[EFX_FILTER_TABLE_RX_DEF
];
158 filter_ctl
, FRF_CZ_UNICAST_NOMATCH_Q_ID
,
159 table
->spec
[EFX_FILTER_INDEX_UC_DEF
].dmaq_id
);
161 filter_ctl
, FRF_CZ_UNICAST_NOMATCH_RSS_ENABLED
,
162 !!(table
->spec
[EFX_FILTER_INDEX_UC_DEF
].flags
&
163 EFX_FILTER_FLAG_RX_RSS
));
165 filter_ctl
, FRF_CZ_UNICAST_NOMATCH_IP_OVERRIDE
,
166 !!(table
->spec
[EFX_FILTER_INDEX_UC_DEF
].flags
&
167 EFX_FILTER_FLAG_RX_OVERRIDE_IP
));
169 filter_ctl
, FRF_CZ_MULTICAST_NOMATCH_Q_ID
,
170 table
->spec
[EFX_FILTER_INDEX_MC_DEF
].dmaq_id
);
172 filter_ctl
, FRF_CZ_MULTICAST_NOMATCH_RSS_ENABLED
,
173 !!(table
->spec
[EFX_FILTER_INDEX_MC_DEF
].flags
&
174 EFX_FILTER_FLAG_RX_RSS
));
176 filter_ctl
, FRF_CZ_MULTICAST_NOMATCH_IP_OVERRIDE
,
177 !!(table
->spec
[EFX_FILTER_INDEX_MC_DEF
].flags
&
178 EFX_FILTER_FLAG_RX_OVERRIDE_IP
));
181 efx_writeo(efx
, &filter_ctl
, FR_BZ_RX_FILTER_CTL
);
184 static void efx_filter_push_tx_limits(struct efx_nic
*efx
)
186 struct efx_filter_state
*state
= efx
->filter_state
;
187 struct efx_filter_table
*table
;
190 efx_reado(efx
, &tx_cfg
, FR_AZ_TX_CFG
);
192 table
= &state
->table
[EFX_FILTER_TABLE_TX_MAC
];
195 tx_cfg
, FRF_CZ_TX_ETH_FILTER_FULL_SEARCH_RANGE
,
196 table
->search_depth
[EFX_FILTER_MAC_FULL
] +
197 FILTER_CTL_SRCH_FUDGE_FULL
);
199 tx_cfg
, FRF_CZ_TX_ETH_FILTER_WILD_SEARCH_RANGE
,
200 table
->search_depth
[EFX_FILTER_MAC_WILD
] +
201 FILTER_CTL_SRCH_FUDGE_WILD
);
204 efx_writeo(efx
, &tx_cfg
, FR_AZ_TX_CFG
);
207 static inline void __efx_filter_set_ipv4(struct efx_filter_spec
*spec
,
208 __be32 host1
, __be16 port1
,
209 __be32 host2
, __be16 port2
)
211 spec
->data
[0] = ntohl(host1
) << 16 | ntohs(port1
);
212 spec
->data
[1] = ntohs(port2
) << 16 | ntohl(host1
) >> 16;
213 spec
->data
[2] = ntohl(host2
);
216 static inline void __efx_filter_get_ipv4(const struct efx_filter_spec
*spec
,
217 __be32
*host1
, __be16
*port1
,
218 __be32
*host2
, __be16
*port2
)
220 *host1
= htonl(spec
->data
[0] >> 16 | spec
->data
[1] << 16);
221 *port1
= htons(spec
->data
[0]);
222 *host2
= htonl(spec
->data
[2]);
223 *port2
= htons(spec
->data
[1] >> 16);
227 * efx_filter_set_ipv4_local - specify IPv4 host, transport protocol and port
228 * @spec: Specification to initialise
229 * @proto: Transport layer protocol number
230 * @host: Local host address (network byte order)
231 * @port: Local port (network byte order)
233 int efx_filter_set_ipv4_local(struct efx_filter_spec
*spec
, u8 proto
,
234 __be32 host
, __be16 port
)
239 EFX_BUG_ON_PARANOID(!(spec
->flags
& EFX_FILTER_FLAG_RX
));
241 /* This cannot currently be combined with other filtering */
242 if (spec
->type
!= EFX_FILTER_UNSPEC
)
243 return -EPROTONOSUPPORT
;
250 spec
->type
= EFX_FILTER_TCP_WILD
;
253 spec
->type
= EFX_FILTER_UDP_WILD
;
256 return -EPROTONOSUPPORT
;
259 /* Filter is constructed in terms of source and destination,
260 * with the odd wrinkle that the ports are swapped in a UDP
261 * wildcard filter. We need to convert from local and remote
262 * (= zero for wildcard) addresses.
265 if (proto
!= IPPROTO_UDP
) {
272 __efx_filter_set_ipv4(spec
, host1
, port1
, host
, port
);
276 int efx_filter_get_ipv4_local(const struct efx_filter_spec
*spec
,
277 u8
*proto
, __be32
*host
, __be16
*port
)
282 switch (spec
->type
) {
283 case EFX_FILTER_TCP_WILD
:
284 *proto
= IPPROTO_TCP
;
285 __efx_filter_get_ipv4(spec
, &host1
, &port1
, host
, port
);
287 case EFX_FILTER_UDP_WILD
:
288 *proto
= IPPROTO_UDP
;
289 __efx_filter_get_ipv4(spec
, &host1
, port
, host
, &port1
);
297 * efx_filter_set_ipv4_full - specify IPv4 hosts, transport protocol and ports
298 * @spec: Specification to initialise
299 * @proto: Transport layer protocol number
300 * @host: Local host address (network byte order)
301 * @port: Local port (network byte order)
302 * @rhost: Remote host address (network byte order)
303 * @rport: Remote port (network byte order)
305 int efx_filter_set_ipv4_full(struct efx_filter_spec
*spec
, u8 proto
,
306 __be32 host
, __be16 port
,
307 __be32 rhost
, __be16 rport
)
309 EFX_BUG_ON_PARANOID(!(spec
->flags
& EFX_FILTER_FLAG_RX
));
311 /* This cannot currently be combined with other filtering */
312 if (spec
->type
!= EFX_FILTER_UNSPEC
)
313 return -EPROTONOSUPPORT
;
315 if (port
== 0 || rport
== 0)
320 spec
->type
= EFX_FILTER_TCP_FULL
;
323 spec
->type
= EFX_FILTER_UDP_FULL
;
326 return -EPROTONOSUPPORT
;
329 __efx_filter_set_ipv4(spec
, rhost
, rport
, host
, port
);
333 int efx_filter_get_ipv4_full(const struct efx_filter_spec
*spec
,
334 u8
*proto
, __be32
*host
, __be16
*port
,
335 __be32
*rhost
, __be16
*rport
)
337 switch (spec
->type
) {
338 case EFX_FILTER_TCP_FULL
:
339 *proto
= IPPROTO_TCP
;
341 case EFX_FILTER_UDP_FULL
:
342 *proto
= IPPROTO_UDP
;
348 __efx_filter_get_ipv4(spec
, rhost
, rport
, host
, port
);
353 * efx_filter_set_eth_local - specify local Ethernet address and optional VID
354 * @spec: Specification to initialise
355 * @vid: VLAN ID to match, or %EFX_FILTER_VID_UNSPEC
356 * @addr: Local Ethernet MAC address
358 int efx_filter_set_eth_local(struct efx_filter_spec
*spec
,
359 u16 vid
, const u8
*addr
)
361 EFX_BUG_ON_PARANOID(!(spec
->flags
&
362 (EFX_FILTER_FLAG_RX
| EFX_FILTER_FLAG_TX
)));
364 /* This cannot currently be combined with other filtering */
365 if (spec
->type
!= EFX_FILTER_UNSPEC
)
366 return -EPROTONOSUPPORT
;
368 if (vid
== EFX_FILTER_VID_UNSPEC
) {
369 spec
->type
= EFX_FILTER_MAC_WILD
;
372 spec
->type
= EFX_FILTER_MAC_FULL
;
376 spec
->data
[1] = addr
[2] << 24 | addr
[3] << 16 | addr
[4] << 8 | addr
[5];
377 spec
->data
[2] = addr
[0] << 8 | addr
[1];
382 * efx_filter_set_uc_def - specify matching otherwise-unmatched unicast
383 * @spec: Specification to initialise
385 int efx_filter_set_uc_def(struct efx_filter_spec
*spec
)
387 EFX_BUG_ON_PARANOID(!(spec
->flags
&
388 (EFX_FILTER_FLAG_RX
| EFX_FILTER_FLAG_TX
)));
390 if (spec
->type
!= EFX_FILTER_UNSPEC
)
393 spec
->type
= EFX_FILTER_UC_DEF
;
394 memset(spec
->data
, 0, sizeof(spec
->data
)); /* ensure equality */
399 * efx_filter_set_mc_def - specify matching otherwise-unmatched multicast
400 * @spec: Specification to initialise
402 int efx_filter_set_mc_def(struct efx_filter_spec
*spec
)
404 EFX_BUG_ON_PARANOID(!(spec
->flags
&
405 (EFX_FILTER_FLAG_RX
| EFX_FILTER_FLAG_TX
)));
407 if (spec
->type
!= EFX_FILTER_UNSPEC
)
410 spec
->type
= EFX_FILTER_MC_DEF
;
411 memset(spec
->data
, 0, sizeof(spec
->data
)); /* ensure equality */
415 static void efx_filter_reset_rx_def(struct efx_nic
*efx
, unsigned filter_idx
)
417 struct efx_filter_state
*state
= efx
->filter_state
;
418 struct efx_filter_table
*table
= &state
->table
[EFX_FILTER_TABLE_RX_DEF
];
419 struct efx_filter_spec
*spec
= &table
->spec
[filter_idx
];
421 efx_filter_init_rx(spec
, EFX_FILTER_PRI_MANUAL
,
422 EFX_FILTER_FLAG_RX_RSS
, 0);
423 spec
->type
= EFX_FILTER_UC_DEF
+ filter_idx
;
424 table
->used_bitmap
[0] |= 1 << filter_idx
;
427 int efx_filter_get_eth_local(const struct efx_filter_spec
*spec
,
430 switch (spec
->type
) {
431 case EFX_FILTER_MAC_WILD
:
432 *vid
= EFX_FILTER_VID_UNSPEC
;
434 case EFX_FILTER_MAC_FULL
:
435 *vid
= spec
->data
[0];
441 addr
[0] = spec
->data
[2] >> 8;
442 addr
[1] = spec
->data
[2];
443 addr
[2] = spec
->data
[1] >> 24;
444 addr
[3] = spec
->data
[1] >> 16;
445 addr
[4] = spec
->data
[1] >> 8;
446 addr
[5] = spec
->data
[1];
450 /* Build a filter entry and return its n-tuple key. */
451 static u32
efx_filter_build(efx_oword_t
*filter
, struct efx_filter_spec
*spec
)
455 switch (efx_filter_spec_table_id(spec
)) {
456 case EFX_FILTER_TABLE_RX_IP
: {
457 bool is_udp
= (spec
->type
== EFX_FILTER_UDP_FULL
||
458 spec
->type
== EFX_FILTER_UDP_WILD
);
459 EFX_POPULATE_OWORD_7(
462 !!(spec
->flags
& EFX_FILTER_FLAG_RX_RSS
),
464 !!(spec
->flags
& EFX_FILTER_FLAG_RX_SCATTER
),
465 FRF_BZ_TCP_UDP
, is_udp
,
466 FRF_BZ_RXQ_ID
, spec
->dmaq_id
,
467 EFX_DWORD_2
, spec
->data
[2],
468 EFX_DWORD_1
, spec
->data
[1],
469 EFX_DWORD_0
, spec
->data
[0]);
474 case EFX_FILTER_TABLE_RX_DEF
:
475 /* One filter spec per type */
476 BUILD_BUG_ON(EFX_FILTER_INDEX_UC_DEF
!= 0);
477 BUILD_BUG_ON(EFX_FILTER_INDEX_MC_DEF
!=
478 EFX_FILTER_MC_DEF
- EFX_FILTER_UC_DEF
);
479 return spec
->type
- EFX_FILTER_UC_DEF
;
481 case EFX_FILTER_TABLE_RX_MAC
: {
482 bool is_wild
= spec
->type
== EFX_FILTER_MAC_WILD
;
483 EFX_POPULATE_OWORD_8(
486 !!(spec
->flags
& EFX_FILTER_FLAG_RX_RSS
),
487 FRF_CZ_RMFT_SCATTER_EN
,
488 !!(spec
->flags
& EFX_FILTER_FLAG_RX_SCATTER
),
489 FRF_CZ_RMFT_IP_OVERRIDE
,
490 !!(spec
->flags
& EFX_FILTER_FLAG_RX_OVERRIDE_IP
),
491 FRF_CZ_RMFT_RXQ_ID
, spec
->dmaq_id
,
492 FRF_CZ_RMFT_WILDCARD_MATCH
, is_wild
,
493 FRF_CZ_RMFT_DEST_MAC_HI
, spec
->data
[2],
494 FRF_CZ_RMFT_DEST_MAC_LO
, spec
->data
[1],
495 FRF_CZ_RMFT_VLAN_ID
, spec
->data
[0]);
500 case EFX_FILTER_TABLE_TX_MAC
: {
501 bool is_wild
= spec
->type
== EFX_FILTER_MAC_WILD
;
502 EFX_POPULATE_OWORD_5(*filter
,
503 FRF_CZ_TMFT_TXQ_ID
, spec
->dmaq_id
,
504 FRF_CZ_TMFT_WILDCARD_MATCH
, is_wild
,
505 FRF_CZ_TMFT_SRC_MAC_HI
, spec
->data
[2],
506 FRF_CZ_TMFT_SRC_MAC_LO
, spec
->data
[1],
507 FRF_CZ_TMFT_VLAN_ID
, spec
->data
[0]);
508 data3
= is_wild
| spec
->dmaq_id
<< 1;
516 return spec
->data
[0] ^ spec
->data
[1] ^ spec
->data
[2] ^ data3
;
519 static bool efx_filter_equal(const struct efx_filter_spec
*left
,
520 const struct efx_filter_spec
*right
)
522 if (left
->type
!= right
->type
||
523 memcmp(left
->data
, right
->data
, sizeof(left
->data
)))
526 if (left
->flags
& EFX_FILTER_FLAG_TX
&&
527 left
->dmaq_id
!= right
->dmaq_id
)
533 static int efx_filter_search(struct efx_filter_table
*table
,
534 struct efx_filter_spec
*spec
, u32 key
,
535 bool for_insert
, unsigned int *depth_required
)
537 unsigned hash
, incr
, filter_idx
, depth
, depth_max
;
539 hash
= efx_filter_hash(key
);
540 incr
= efx_filter_increment(key
);
542 filter_idx
= hash
& (table
->size
- 1);
544 depth_max
= (for_insert
?
545 (spec
->priority
<= EFX_FILTER_PRI_HINT
?
546 FILTER_CTL_SRCH_HINT_MAX
: FILTER_CTL_SRCH_MAX
) :
547 table
->search_depth
[spec
->type
]);
550 /* Return success if entry is used and matches this spec
551 * or entry is unused and we are trying to insert.
553 if (test_bit(filter_idx
, table
->used_bitmap
) ?
554 efx_filter_equal(spec
, &table
->spec
[filter_idx
]) :
556 *depth_required
= depth
;
560 /* Return failure if we reached the maximum search depth */
561 if (depth
== depth_max
)
562 return for_insert
? -EBUSY
: -ENOENT
;
564 filter_idx
= (filter_idx
+ incr
) & (table
->size
- 1);
570 * Construct/deconstruct external filter IDs. These must be ordered
571 * by matching priority, for RX NFC semantics.
573 * Each RX MAC filter entry has a flag for whether it can override an
574 * RX IP filter that also matches. So we assign locations for MAC
575 * filters with overriding behaviour, then for IP filters, then for
576 * MAC filters without overriding behaviour.
579 #define EFX_FILTER_MATCH_PRI_RX_MAC_OVERRIDE_IP 0
580 #define EFX_FILTER_MATCH_PRI_RX_DEF_OVERRIDE_IP 1
581 #define EFX_FILTER_MATCH_PRI_NORMAL_BASE 2
583 #define EFX_FILTER_INDEX_WIDTH 13
584 #define EFX_FILTER_INDEX_MASK ((1 << EFX_FILTER_INDEX_WIDTH) - 1)
586 static inline u32
efx_filter_make_id(enum efx_filter_table_id table_id
,
587 unsigned int index
, u8 flags
)
589 unsigned int match_pri
= EFX_FILTER_MATCH_PRI_NORMAL_BASE
+ table_id
;
591 if (flags
& EFX_FILTER_FLAG_RX_OVERRIDE_IP
) {
592 if (table_id
== EFX_FILTER_TABLE_RX_MAC
)
593 match_pri
= EFX_FILTER_MATCH_PRI_RX_MAC_OVERRIDE_IP
;
594 else if (table_id
== EFX_FILTER_TABLE_RX_DEF
)
595 match_pri
= EFX_FILTER_MATCH_PRI_RX_DEF_OVERRIDE_IP
;
598 return match_pri
<< EFX_FILTER_INDEX_WIDTH
| index
;
601 static inline enum efx_filter_table_id
efx_filter_id_table_id(u32 id
)
603 unsigned int match_pri
= id
>> EFX_FILTER_INDEX_WIDTH
;
606 case EFX_FILTER_MATCH_PRI_RX_MAC_OVERRIDE_IP
:
607 return EFX_FILTER_TABLE_RX_MAC
;
608 case EFX_FILTER_MATCH_PRI_RX_DEF_OVERRIDE_IP
:
609 return EFX_FILTER_TABLE_RX_DEF
;
611 return match_pri
- EFX_FILTER_MATCH_PRI_NORMAL_BASE
;
615 static inline unsigned int efx_filter_id_index(u32 id
)
617 return id
& EFX_FILTER_INDEX_MASK
;
620 static inline u8
efx_filter_id_flags(u32 id
)
622 unsigned int match_pri
= id
>> EFX_FILTER_INDEX_WIDTH
;
624 if (match_pri
< EFX_FILTER_MATCH_PRI_NORMAL_BASE
)
625 return EFX_FILTER_FLAG_RX
| EFX_FILTER_FLAG_RX_OVERRIDE_IP
;
626 else if (match_pri
<=
627 EFX_FILTER_MATCH_PRI_NORMAL_BASE
+ EFX_FILTER_TABLE_RX_DEF
)
628 return EFX_FILTER_FLAG_RX
;
630 return EFX_FILTER_FLAG_TX
;
633 u32
efx_filter_get_rx_id_limit(struct efx_nic
*efx
)
635 struct efx_filter_state
*state
= efx
->filter_state
;
636 unsigned int table_id
= EFX_FILTER_TABLE_RX_DEF
;
639 if (state
->table
[table_id
].size
!= 0)
640 return ((EFX_FILTER_MATCH_PRI_NORMAL_BASE
+ table_id
)
641 << EFX_FILTER_INDEX_WIDTH
) +
642 state
->table
[table_id
].size
;
643 } while (table_id
--);
649 * efx_filter_insert_filter - add or replace a filter
650 * @efx: NIC in which to insert the filter
651 * @spec: Specification for the filter
652 * @replace: Flag for whether the specified filter may replace a filter
653 * with an identical match expression and equal or lower priority
655 * On success, return the filter ID.
656 * On failure, return a negative error code.
658 s32
efx_filter_insert_filter(struct efx_nic
*efx
, struct efx_filter_spec
*spec
,
661 struct efx_filter_state
*state
= efx
->filter_state
;
662 struct efx_filter_table
*table
= efx_filter_spec_table(state
, spec
);
663 struct efx_filter_spec
*saved_spec
;
665 unsigned int filter_idx
, depth
;
669 if (!table
|| table
->size
== 0)
672 key
= efx_filter_build(&filter
, spec
);
674 netif_vdbg(efx
, hw
, efx
->net_dev
,
675 "%s: type %d search_depth=%d", __func__
, spec
->type
,
676 table
->search_depth
[spec
->type
]);
678 spin_lock_bh(&state
->lock
);
680 rc
= efx_filter_search(table
, spec
, key
, true, &depth
);
684 BUG_ON(filter_idx
>= table
->size
);
685 saved_spec
= &table
->spec
[filter_idx
];
687 if (test_bit(filter_idx
, table
->used_bitmap
)) {
688 /* Should we replace the existing filter? */
693 if (spec
->priority
< saved_spec
->priority
) {
698 __set_bit(filter_idx
, table
->used_bitmap
);
703 if (table
->id
== EFX_FILTER_TABLE_RX_DEF
) {
704 efx_filter_push_rx_config(efx
);
706 if (table
->search_depth
[spec
->type
] < depth
) {
707 table
->search_depth
[spec
->type
] = depth
;
708 if (spec
->flags
& EFX_FILTER_FLAG_TX
)
709 efx_filter_push_tx_limits(efx
);
711 efx_filter_push_rx_config(efx
);
714 efx_writeo(efx
, &filter
,
715 table
->offset
+ table
->step
* filter_idx
);
718 netif_vdbg(efx
, hw
, efx
->net_dev
,
719 "%s: filter type %d index %d rxq %u set",
720 __func__
, spec
->type
, filter_idx
, spec
->dmaq_id
);
721 rc
= efx_filter_make_id(table
->id
, filter_idx
, spec
->flags
);
724 spin_unlock_bh(&state
->lock
);
728 static void efx_filter_table_clear_entry(struct efx_nic
*efx
,
729 struct efx_filter_table
*table
,
730 unsigned int filter_idx
)
732 static efx_oword_t filter
;
734 if (table
->id
== EFX_FILTER_TABLE_RX_DEF
) {
735 /* RX default filters must always exist */
736 efx_filter_reset_rx_def(efx
, filter_idx
);
737 efx_filter_push_rx_config(efx
);
738 } else if (test_bit(filter_idx
, table
->used_bitmap
)) {
739 __clear_bit(filter_idx
, table
->used_bitmap
);
741 memset(&table
->spec
[filter_idx
], 0, sizeof(table
->spec
[0]));
743 efx_writeo(efx
, &filter
,
744 table
->offset
+ table
->step
* filter_idx
);
749 * efx_filter_remove_id_safe - remove a filter by ID, carefully
750 * @efx: NIC from which to remove the filter
751 * @priority: Priority of filter, as passed to @efx_filter_insert_filter
752 * @filter_id: ID of filter, as returned by @efx_filter_insert_filter
754 * This function will range-check @filter_id, so it is safe to call
755 * with a value passed from userland.
757 int efx_filter_remove_id_safe(struct efx_nic
*efx
,
758 enum efx_filter_priority priority
,
761 struct efx_filter_state
*state
= efx
->filter_state
;
762 enum efx_filter_table_id table_id
;
763 struct efx_filter_table
*table
;
764 unsigned int filter_idx
;
765 struct efx_filter_spec
*spec
;
769 table_id
= efx_filter_id_table_id(filter_id
);
770 if ((unsigned int)table_id
>= EFX_FILTER_TABLE_COUNT
)
772 table
= &state
->table
[table_id
];
774 filter_idx
= efx_filter_id_index(filter_id
);
775 if (filter_idx
>= table
->size
)
777 spec
= &table
->spec
[filter_idx
];
779 filter_flags
= efx_filter_id_flags(filter_id
);
781 spin_lock_bh(&state
->lock
);
783 if (test_bit(filter_idx
, table
->used_bitmap
) &&
784 spec
->priority
== priority
&&
785 !((spec
->flags
^ filter_flags
) & EFX_FILTER_FLAG_RX_OVERRIDE_IP
)) {
786 efx_filter_table_clear_entry(efx
, table
, filter_idx
);
787 if (table
->used
== 0)
788 efx_filter_table_reset_search_depth(table
);
794 spin_unlock_bh(&state
->lock
);
800 * efx_filter_get_filter_safe - retrieve a filter by ID, carefully
801 * @efx: NIC from which to remove the filter
802 * @priority: Priority of filter, as passed to @efx_filter_insert_filter
803 * @filter_id: ID of filter, as returned by @efx_filter_insert_filter
804 * @spec: Buffer in which to store filter specification
806 * This function will range-check @filter_id, so it is safe to call
807 * with a value passed from userland.
809 int efx_filter_get_filter_safe(struct efx_nic
*efx
,
810 enum efx_filter_priority priority
,
811 u32 filter_id
, struct efx_filter_spec
*spec_buf
)
813 struct efx_filter_state
*state
= efx
->filter_state
;
814 enum efx_filter_table_id table_id
;
815 struct efx_filter_table
*table
;
816 struct efx_filter_spec
*spec
;
817 unsigned int filter_idx
;
821 table_id
= efx_filter_id_table_id(filter_id
);
822 if ((unsigned int)table_id
>= EFX_FILTER_TABLE_COUNT
)
824 table
= &state
->table
[table_id
];
826 filter_idx
= efx_filter_id_index(filter_id
);
827 if (filter_idx
>= table
->size
)
829 spec
= &table
->spec
[filter_idx
];
831 filter_flags
= efx_filter_id_flags(filter_id
);
833 spin_lock_bh(&state
->lock
);
835 if (test_bit(filter_idx
, table
->used_bitmap
) &&
836 spec
->priority
== priority
&&
837 !((spec
->flags
^ filter_flags
) & EFX_FILTER_FLAG_RX_OVERRIDE_IP
)) {
844 spin_unlock_bh(&state
->lock
);
849 static void efx_filter_table_clear(struct efx_nic
*efx
,
850 enum efx_filter_table_id table_id
,
851 enum efx_filter_priority priority
)
853 struct efx_filter_state
*state
= efx
->filter_state
;
854 struct efx_filter_table
*table
= &state
->table
[table_id
];
855 unsigned int filter_idx
;
857 spin_lock_bh(&state
->lock
);
859 for (filter_idx
= 0; filter_idx
< table
->size
; ++filter_idx
)
860 if (table
->spec
[filter_idx
].priority
<= priority
)
861 efx_filter_table_clear_entry(efx
, table
, filter_idx
);
862 if (table
->used
== 0)
863 efx_filter_table_reset_search_depth(table
);
865 spin_unlock_bh(&state
->lock
);
869 * efx_filter_clear_rx - remove RX filters by priority
870 * @efx: NIC from which to remove the filters
871 * @priority: Maximum priority to remove
873 void efx_filter_clear_rx(struct efx_nic
*efx
, enum efx_filter_priority priority
)
875 efx_filter_table_clear(efx
, EFX_FILTER_TABLE_RX_IP
, priority
);
876 efx_filter_table_clear(efx
, EFX_FILTER_TABLE_RX_MAC
, priority
);
879 u32
efx_filter_count_rx_used(struct efx_nic
*efx
,
880 enum efx_filter_priority priority
)
882 struct efx_filter_state
*state
= efx
->filter_state
;
883 enum efx_filter_table_id table_id
;
884 struct efx_filter_table
*table
;
885 unsigned int filter_idx
;
888 spin_lock_bh(&state
->lock
);
890 for (table_id
= EFX_FILTER_TABLE_RX_IP
;
891 table_id
<= EFX_FILTER_TABLE_RX_DEF
;
893 table
= &state
->table
[table_id
];
894 for (filter_idx
= 0; filter_idx
< table
->size
; filter_idx
++) {
895 if (test_bit(filter_idx
, table
->used_bitmap
) &&
896 table
->spec
[filter_idx
].priority
== priority
)
901 spin_unlock_bh(&state
->lock
);
906 s32
efx_filter_get_rx_ids(struct efx_nic
*efx
,
907 enum efx_filter_priority priority
,
910 struct efx_filter_state
*state
= efx
->filter_state
;
911 enum efx_filter_table_id table_id
;
912 struct efx_filter_table
*table
;
913 unsigned int filter_idx
;
916 spin_lock_bh(&state
->lock
);
918 for (table_id
= EFX_FILTER_TABLE_RX_IP
;
919 table_id
<= EFX_FILTER_TABLE_RX_DEF
;
921 table
= &state
->table
[table_id
];
922 for (filter_idx
= 0; filter_idx
< table
->size
; filter_idx
++) {
923 if (test_bit(filter_idx
, table
->used_bitmap
) &&
924 table
->spec
[filter_idx
].priority
== priority
) {
929 buf
[count
++] = efx_filter_make_id(
930 table_id
, filter_idx
,
931 table
->spec
[filter_idx
].flags
);
936 spin_unlock_bh(&state
->lock
);
941 /* Restore filter stater after reset */
942 void efx_restore_filters(struct efx_nic
*efx
)
944 struct efx_filter_state
*state
= efx
->filter_state
;
945 enum efx_filter_table_id table_id
;
946 struct efx_filter_table
*table
;
948 unsigned int filter_idx
;
950 spin_lock_bh(&state
->lock
);
952 for (table_id
= 0; table_id
< EFX_FILTER_TABLE_COUNT
; table_id
++) {
953 table
= &state
->table
[table_id
];
955 /* Check whether this is a regular register table */
956 if (table
->step
== 0)
959 for (filter_idx
= 0; filter_idx
< table
->size
; filter_idx
++) {
960 if (!test_bit(filter_idx
, table
->used_bitmap
))
962 efx_filter_build(&filter
, &table
->spec
[filter_idx
]);
963 efx_writeo(efx
, &filter
,
964 table
->offset
+ table
->step
* filter_idx
);
968 efx_filter_push_rx_config(efx
);
969 efx_filter_push_tx_limits(efx
);
971 spin_unlock_bh(&state
->lock
);
974 int efx_probe_filters(struct efx_nic
*efx
)
976 struct efx_filter_state
*state
;
977 struct efx_filter_table
*table
;
980 state
= kzalloc(sizeof(*efx
->filter_state
), GFP_KERNEL
);
983 efx
->filter_state
= state
;
985 spin_lock_init(&state
->lock
);
987 if (efx_nic_rev(efx
) >= EFX_REV_FALCON_B0
) {
988 #ifdef CONFIG_RFS_ACCEL
989 state
->rps_flow_id
= kcalloc(FR_BZ_RX_FILTER_TBL0_ROWS
,
990 sizeof(*state
->rps_flow_id
),
992 if (!state
->rps_flow_id
)
995 table
= &state
->table
[EFX_FILTER_TABLE_RX_IP
];
996 table
->id
= EFX_FILTER_TABLE_RX_IP
;
997 table
->offset
= FR_BZ_RX_FILTER_TBL0
;
998 table
->size
= FR_BZ_RX_FILTER_TBL0_ROWS
;
999 table
->step
= FR_BZ_RX_FILTER_TBL0_STEP
;
1002 if (efx_nic_rev(efx
) >= EFX_REV_SIENA_A0
) {
1003 table
= &state
->table
[EFX_FILTER_TABLE_RX_MAC
];
1004 table
->id
= EFX_FILTER_TABLE_RX_MAC
;
1005 table
->offset
= FR_CZ_RX_MAC_FILTER_TBL0
;
1006 table
->size
= FR_CZ_RX_MAC_FILTER_TBL0_ROWS
;
1007 table
->step
= FR_CZ_RX_MAC_FILTER_TBL0_STEP
;
1009 table
= &state
->table
[EFX_FILTER_TABLE_RX_DEF
];
1010 table
->id
= EFX_FILTER_TABLE_RX_DEF
;
1011 table
->size
= EFX_FILTER_SIZE_RX_DEF
;
1013 table
= &state
->table
[EFX_FILTER_TABLE_TX_MAC
];
1014 table
->id
= EFX_FILTER_TABLE_TX_MAC
;
1015 table
->offset
= FR_CZ_TX_MAC_FILTER_TBL0
;
1016 table
->size
= FR_CZ_TX_MAC_FILTER_TBL0_ROWS
;
1017 table
->step
= FR_CZ_TX_MAC_FILTER_TBL0_STEP
;
1020 for (table_id
= 0; table_id
< EFX_FILTER_TABLE_COUNT
; table_id
++) {
1021 table
= &state
->table
[table_id
];
1022 if (table
->size
== 0)
1024 table
->used_bitmap
= kcalloc(BITS_TO_LONGS(table
->size
),
1025 sizeof(unsigned long),
1027 if (!table
->used_bitmap
)
1029 table
->spec
= vzalloc(table
->size
* sizeof(*table
->spec
));
1034 if (state
->table
[EFX_FILTER_TABLE_RX_DEF
].size
) {
1035 /* RX default filters must always exist */
1037 for (i
= 0; i
< EFX_FILTER_SIZE_RX_DEF
; i
++)
1038 efx_filter_reset_rx_def(efx
, i
);
1041 efx_filter_push_rx_config(efx
);
1046 efx_remove_filters(efx
);
1050 void efx_remove_filters(struct efx_nic
*efx
)
1052 struct efx_filter_state
*state
= efx
->filter_state
;
1053 enum efx_filter_table_id table_id
;
1055 for (table_id
= 0; table_id
< EFX_FILTER_TABLE_COUNT
; table_id
++) {
1056 kfree(state
->table
[table_id
].used_bitmap
);
1057 vfree(state
->table
[table_id
].spec
);
1059 #ifdef CONFIG_RFS_ACCEL
1060 kfree(state
->rps_flow_id
);
1065 #ifdef CONFIG_RFS_ACCEL
1067 int efx_filter_rfs(struct net_device
*net_dev
, const struct sk_buff
*skb
,
1068 u16 rxq_index
, u32 flow_id
)
1070 struct efx_nic
*efx
= netdev_priv(net_dev
);
1071 struct efx_channel
*channel
;
1072 struct efx_filter_state
*state
= efx
->filter_state
;
1073 struct efx_filter_spec spec
;
1074 const struct iphdr
*ip
;
1075 const __be16
*ports
;
1079 nhoff
= skb_network_offset(skb
);
1081 if (skb
->protocol
!= htons(ETH_P_IP
))
1082 return -EPROTONOSUPPORT
;
1084 /* RFS must validate the IP header length before calling us */
1085 EFX_BUG_ON_PARANOID(skb_headlen(skb
) < nhoff
+ sizeof(*ip
));
1086 ip
= (const struct iphdr
*)(skb
->data
+ nhoff
);
1087 if (ip_is_fragment(ip
))
1088 return -EPROTONOSUPPORT
;
1089 EFX_BUG_ON_PARANOID(skb_headlen(skb
) < nhoff
+ 4 * ip
->ihl
+ 4);
1090 ports
= (const __be16
*)(skb
->data
+ nhoff
+ 4 * ip
->ihl
);
1092 efx_filter_init_rx(&spec
, EFX_FILTER_PRI_HINT
, 0, rxq_index
);
1093 rc
= efx_filter_set_ipv4_full(&spec
, ip
->protocol
,
1094 ip
->daddr
, ports
[1], ip
->saddr
, ports
[0]);
1098 rc
= efx_filter_insert_filter(efx
, &spec
, true);
1102 /* Remember this so we can check whether to expire the filter later */
1103 state
->rps_flow_id
[rc
] = flow_id
;
1104 channel
= efx_get_channel(efx
, skb_get_rx_queue(skb
));
1105 ++channel
->rfs_filters_added
;
1107 netif_info(efx
, rx_status
, efx
->net_dev
,
1108 "steering %s %pI4:%u:%pI4:%u to queue %u [flow %u filter %d]\n",
1109 (ip
->protocol
== IPPROTO_TCP
) ? "TCP" : "UDP",
1110 &ip
->saddr
, ntohs(ports
[0]), &ip
->daddr
, ntohs(ports
[1]),
1111 rxq_index
, flow_id
, rc
);
1116 bool __efx_filter_rfs_expire(struct efx_nic
*efx
, unsigned quota
)
1118 struct efx_filter_state
*state
= efx
->filter_state
;
1119 struct efx_filter_table
*table
= &state
->table
[EFX_FILTER_TABLE_RX_IP
];
1120 unsigned mask
= table
->size
- 1;
1124 if (!spin_trylock_bh(&state
->lock
))
1127 index
= state
->rps_expire_index
;
1128 stop
= (index
+ quota
) & mask
;
1130 while (index
!= stop
) {
1131 if (test_bit(index
, table
->used_bitmap
) &&
1132 table
->spec
[index
].priority
== EFX_FILTER_PRI_HINT
&&
1133 rps_may_expire_flow(efx
->net_dev
,
1134 table
->spec
[index
].dmaq_id
,
1135 state
->rps_flow_id
[index
], index
)) {
1136 netif_info(efx
, rx_status
, efx
->net_dev
,
1137 "expiring filter %d [flow %u]\n",
1138 index
, state
->rps_flow_id
[index
]);
1139 efx_filter_table_clear_entry(efx
, table
, index
);
1141 index
= (index
+ 1) & mask
;
1144 state
->rps_expire_index
= stop
;
1145 if (table
->used
== 0)
1146 efx_filter_table_reset_search_depth(table
);
1148 spin_unlock_bh(&state
->lock
);
1152 #endif /* CONFIG_RFS_ACCEL */