5 * YOSHIFUJI Hideaki @USAGI
6 * Split up af-specific portion
10 #include <linux/slab.h>
11 #include <linux/module.h>
12 #include <linux/netdevice.h>
16 #include <net/ip_tunnels.h>
17 #include <net/ip6_tunnel.h>
19 static struct kmem_cache
*secpath_cachep __read_mostly
;
21 static DEFINE_SPINLOCK(xfrm_input_afinfo_lock
);
22 static struct xfrm_input_afinfo
const __rcu
*xfrm_input_afinfo
[AF_INET6
+ 1];
24 static struct gro_cells gro_cells
;
25 static struct net_device xfrm_napi_dev
;
27 int xfrm_input_register_afinfo(const struct xfrm_input_afinfo
*afinfo
)
31 if (WARN_ON(afinfo
->family
>= ARRAY_SIZE(xfrm_input_afinfo
)))
34 spin_lock_bh(&xfrm_input_afinfo_lock
);
35 if (unlikely(xfrm_input_afinfo
[afinfo
->family
] != NULL
))
38 rcu_assign_pointer(xfrm_input_afinfo
[afinfo
->family
], afinfo
);
39 spin_unlock_bh(&xfrm_input_afinfo_lock
);
42 EXPORT_SYMBOL(xfrm_input_register_afinfo
);
44 int xfrm_input_unregister_afinfo(const struct xfrm_input_afinfo
*afinfo
)
48 spin_lock_bh(&xfrm_input_afinfo_lock
);
49 if (likely(xfrm_input_afinfo
[afinfo
->family
] != NULL
)) {
50 if (unlikely(xfrm_input_afinfo
[afinfo
->family
] != afinfo
))
53 RCU_INIT_POINTER(xfrm_input_afinfo
[afinfo
->family
], NULL
);
55 spin_unlock_bh(&xfrm_input_afinfo_lock
);
59 EXPORT_SYMBOL(xfrm_input_unregister_afinfo
);
61 static const struct xfrm_input_afinfo
*xfrm_input_get_afinfo(unsigned int family
)
63 const struct xfrm_input_afinfo
*afinfo
;
65 if (WARN_ON_ONCE(family
>= ARRAY_SIZE(xfrm_input_afinfo
)))
69 afinfo
= rcu_dereference(xfrm_input_afinfo
[family
]);
70 if (unlikely(!afinfo
))
75 static int xfrm_rcv_cb(struct sk_buff
*skb
, unsigned int family
, u8 protocol
,
79 const struct xfrm_input_afinfo
*afinfo
= xfrm_input_get_afinfo(family
);
84 ret
= afinfo
->callback(skb
, protocol
, err
);
90 void __secpath_destroy(struct sec_path
*sp
)
93 for (i
= 0; i
< sp
->len
; i
++)
94 xfrm_state_put(sp
->xvec
[i
]);
95 kmem_cache_free(secpath_cachep
, sp
);
97 EXPORT_SYMBOL(__secpath_destroy
);
99 struct sec_path
*secpath_dup(struct sec_path
*src
)
103 sp
= kmem_cache_alloc(secpath_cachep
, GFP_ATOMIC
);
113 memcpy(sp
, src
, sizeof(*sp
));
114 for (i
= 0; i
< sp
->len
; i
++)
115 xfrm_state_hold(sp
->xvec
[i
]);
117 atomic_set(&sp
->refcnt
, 1);
120 EXPORT_SYMBOL(secpath_dup
);
122 int secpath_set(struct sk_buff
*skb
)
126 /* Allocate new secpath or COW existing one. */
127 if (!skb
->sp
|| atomic_read(&skb
->sp
->refcnt
) != 1) {
128 sp
= secpath_dup(skb
->sp
);
133 secpath_put(skb
->sp
);
138 EXPORT_SYMBOL(secpath_set
);
140 /* Fetch spi and seq from ipsec header */
142 int xfrm_parse_spi(struct sk_buff
*skb
, u8 nexthdr
, __be32
*spi
, __be32
*seq
)
144 int offset
, offset_seq
;
149 hlen
= sizeof(struct ip_auth_hdr
);
150 offset
= offsetof(struct ip_auth_hdr
, spi
);
151 offset_seq
= offsetof(struct ip_auth_hdr
, seq_no
);
154 hlen
= sizeof(struct ip_esp_hdr
);
155 offset
= offsetof(struct ip_esp_hdr
, spi
);
156 offset_seq
= offsetof(struct ip_esp_hdr
, seq_no
);
159 if (!pskb_may_pull(skb
, sizeof(struct ip_comp_hdr
)))
161 *spi
= htonl(ntohs(*(__be16
*)(skb_transport_header(skb
) + 2)));
168 if (!pskb_may_pull(skb
, hlen
))
171 *spi
= *(__be32
*)(skb_transport_header(skb
) + offset
);
172 *seq
= *(__be32
*)(skb_transport_header(skb
) + offset_seq
);
175 EXPORT_SYMBOL(xfrm_parse_spi
);
177 int xfrm_prepare_input(struct xfrm_state
*x
, struct sk_buff
*skb
)
179 struct xfrm_mode
*inner_mode
= x
->inner_mode
;
182 err
= x
->outer_mode
->afinfo
->extract_input(x
, skb
);
186 if (x
->sel
.family
== AF_UNSPEC
) {
187 inner_mode
= xfrm_ip2inner_mode(x
, XFRM_MODE_SKB_CB(skb
)->protocol
);
188 if (inner_mode
== NULL
)
189 return -EAFNOSUPPORT
;
192 skb
->protocol
= inner_mode
->afinfo
->eth_proto
;
193 return inner_mode
->input2(x
, skb
);
195 EXPORT_SYMBOL(xfrm_prepare_input
);
197 int xfrm_input(struct sk_buff
*skb
, int nexthdr
, __be32 spi
, int encap_type
)
199 struct net
*net
= dev_net(skb
->dev
);
203 struct xfrm_state
*x
= NULL
;
204 xfrm_address_t
*daddr
;
205 struct xfrm_mode
*inner_mode
;
206 u32 mark
= skb
->mark
;
210 struct xfrm_offload
*xo
;
211 bool xfrm_gro
= false;
213 if (encap_type
< 0) {
214 x
= xfrm_input_state(skb
);
215 family
= x
->outer_mode
->afinfo
->family
;
217 /* An encap_type of -1 indicates async resumption. */
218 if (encap_type
== -1) {
220 seq
= XFRM_SKB_CB(skb
)->seq
.input
.low
;
223 /* encap_type < -1 indicates a GRO call. */
225 seq
= XFRM_SPI_SKB_CB(skb
)->seq
;
229 daddr
= (xfrm_address_t
*)(skb_network_header(skb
) +
230 XFRM_SPI_SKB_CB(skb
)->daddroff
);
231 family
= XFRM_SPI_SKB_CB(skb
)->family
;
233 /* if tunnel is present override skb->mark value with tunnel i_key */
236 if (XFRM_TUNNEL_SKB_CB(skb
)->tunnel
.ip4
)
237 mark
= be32_to_cpu(XFRM_TUNNEL_SKB_CB(skb
)->tunnel
.ip4
->parms
.i_key
);
240 if (XFRM_TUNNEL_SKB_CB(skb
)->tunnel
.ip6
)
241 mark
= be32_to_cpu(XFRM_TUNNEL_SKB_CB(skb
)->tunnel
.ip6
->parms
.i_key
);
245 err
= secpath_set(skb
);
247 XFRM_INC_STATS(net
, LINUX_MIB_XFRMINERROR
);
252 if (!spi
&& (err
= xfrm_parse_spi(skb
, nexthdr
, &spi
, &seq
)) != 0) {
253 XFRM_INC_STATS(net
, LINUX_MIB_XFRMINHDRERROR
);
258 if (skb
->sp
->len
== XFRM_MAX_DEPTH
) {
259 XFRM_INC_STATS(net
, LINUX_MIB_XFRMINBUFFERERROR
);
263 x
= xfrm_state_lookup(net
, mark
, daddr
, spi
, nexthdr
, family
);
265 XFRM_INC_STATS(net
, LINUX_MIB_XFRMINNOSTATES
);
266 xfrm_audit_state_notfound(skb
, family
, spi
, seq
);
270 skb
->sp
->xvec
[skb
->sp
->len
++] = x
;
275 if (unlikely(x
->km
.state
!= XFRM_STATE_VALID
)) {
276 if (x
->km
.state
== XFRM_STATE_ACQ
)
277 XFRM_INC_STATS(net
, LINUX_MIB_XFRMACQUIREERROR
);
280 LINUX_MIB_XFRMINSTATEINVALID
);
284 if ((x
->encap
? x
->encap
->encap_type
: 0) != encap_type
) {
285 XFRM_INC_STATS(net
, LINUX_MIB_XFRMINSTATEMISMATCH
);
289 if (x
->repl
->check(x
, skb
, seq
)) {
290 XFRM_INC_STATS(net
, LINUX_MIB_XFRMINSTATESEQERROR
);
294 if (xfrm_state_check_expire(x
)) {
295 XFRM_INC_STATS(net
, LINUX_MIB_XFRMINSTATEEXPIRED
);
299 spin_unlock(&x
->lock
);
301 if (xfrm_tunnel_check(skb
, x
, family
)) {
302 XFRM_INC_STATS(net
, LINUX_MIB_XFRMINSTATEMODEERROR
);
306 seq_hi
= htonl(xfrm_replay_seqhi(x
, seq
));
308 XFRM_SKB_CB(skb
)->seq
.input
.low
= seq
;
309 XFRM_SKB_CB(skb
)->seq
.input
.hi
= seq_hi
;
314 nexthdr
= x
->type
->input(x
, skb
);
316 if (nexthdr
== -EINPROGRESS
)
323 if (nexthdr
== -EBADMSG
) {
324 xfrm_audit_state_icvfail(x
, skb
,
326 x
->stats
.integrity_failed
++;
328 XFRM_INC_STATS(net
, LINUX_MIB_XFRMINSTATEPROTOERROR
);
332 /* only the first xfrm gets the encap type */
335 if (async
&& x
->repl
->recheck(x
, skb
, seq
)) {
336 XFRM_INC_STATS(net
, LINUX_MIB_XFRMINSTATESEQERROR
);
340 x
->repl
->advance(x
, seq
);
342 x
->curlft
.bytes
+= skb
->len
;
345 spin_unlock(&x
->lock
);
347 XFRM_MODE_SKB_CB(skb
)->protocol
= nexthdr
;
349 inner_mode
= x
->inner_mode
;
351 if (x
->sel
.family
== AF_UNSPEC
) {
352 inner_mode
= xfrm_ip2inner_mode(x
, XFRM_MODE_SKB_CB(skb
)->protocol
);
353 if (inner_mode
== NULL
) {
354 XFRM_INC_STATS(net
, LINUX_MIB_XFRMINSTATEMODEERROR
);
359 if (inner_mode
->input(x
, skb
)) {
360 XFRM_INC_STATS(net
, LINUX_MIB_XFRMINSTATEMODEERROR
);
364 if (x
->outer_mode
->flags
& XFRM_MODE_FLAG_TUNNEL
) {
370 * We need the inner address. However, we only get here for
371 * transport mode so the outer address is identical.
373 daddr
= &x
->id
.daddr
;
374 family
= x
->outer_mode
->afinfo
->family
;
376 err
= xfrm_parse_spi(skb
, nexthdr
, &spi
, &seq
);
378 XFRM_INC_STATS(net
, LINUX_MIB_XFRMINHDRERROR
);
383 err
= xfrm_rcv_cb(skb
, family
, x
->type
->proto
, 0);
391 gro_cells_receive(&gro_cells
, skb
);
394 xo
= xfrm_offload(skb
);
396 xfrm_gro
= xo
->flags
& XFRM_GRO
;
398 err
= x
->inner_mode
->afinfo
->transport_finish(skb
, async
);
401 gro_cells_receive(&gro_cells
, skb
);
409 spin_unlock(&x
->lock
);
411 xfrm_rcv_cb(skb
, family
, x
&& x
->type
? x
->type
->proto
: nexthdr
, -1);
415 EXPORT_SYMBOL(xfrm_input
);
417 int xfrm_input_resume(struct sk_buff
*skb
, int nexthdr
)
419 return xfrm_input(skb
, nexthdr
, 0, -1);
421 EXPORT_SYMBOL(xfrm_input_resume
);
423 void __init
xfrm_input_init(void)
427 init_dummy_netdev(&xfrm_napi_dev
);
428 err
= gro_cells_init(&gro_cells
, &xfrm_napi_dev
);
430 gro_cells
.cells
= NULL
;
432 secpath_cachep
= kmem_cache_create("secpath_cache",
433 sizeof(struct sec_path
),
434 0, SLAB_HWCACHE_ALIGN
|SLAB_PANIC
,