2 * Connection oriented routing
3 * Copyright (C) 2007-2021 Michael Blizek
5 * This program is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU General Public License
7 * as published by the Free Software Foundation; either version 2
8 * of the License, or (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
21 #include <asm/atomic.h>
23 #include <linux/module.h>
24 #include <linux/types.h>
25 #include <linux/interrupt.h>
26 #include <linux/sched.h>
27 #include <linux/netdevice.h>
28 #include <linux/skbuff.h>
29 #include <linux/spinlock.h>
30 #include <linux/workqueue.h>
31 #include <linux/kref.h>
32 #include <linux/ktime.h>
33 #include <linux/rbtree.h>
35 #include <linux/socket.h>
38 #include <linux/math64.h>
43 #define ETH_P_COR 0x1022
47 #define PROTO_COR_RAW 0
48 #define PROTO_COR_RDEAMON 1
50 #define SOCKADDRTYPE_PORT 1
51 #define SOCKADDRTYPE_ADDRPORT 2
59 #define COR_PASS_ON_CLOSE 1
61 #define COR_PUBLISH_SERVICE 2
64 #define COR_TOS_DEFAULT 0
65 #define COR_TOS_LOW_LATENCY 1
66 #define COR_TOS_HIGH_LATENCY 2
68 #define COR_PRIORITY 4
70 #define MAX_CONN_CMD_LEN 64
72 #define PACKET_TYPE_NONE 0
73 #define PACKET_TYPE_ANNOUNCE 1
74 #define PACKET_TYPE_CMSG_NOACK 2
75 #define PACKET_TYPE_CMSG_ACKSLOW 3
76 #define PACKET_TYPE_CMSG_ACKFAST 4
77 #define PACKET_TYPE_CONNDATA 64
79 #define PACKET_TYPE_CONNDATA_FLAGS 63
80 #define PACKET_TYPE_CONNDATA_FLAGS_FLUSH 32
81 #define PACKET_TYPE_CONNDATA_FLAGS_WINDOWUSED 31
85 * Announce data format:
87 * is 0, may be increased if the protocol changes
89 * is 0, must be increased if a future version of the protocol is incompatible
90 * to the current version
93 * Data format of the announce packet "data" field:
94 *{command [2] commandlength [2] commanddata [commandlength]}[...]
99 /* ANNCMD_VERSION: version[2] minversion[2] */
100 #define ANNCMD_VERSION 1
102 /* ANNCMD_ADDR: addrlen[2] addr[addrlen] */
103 #define ANNCMD_ADDR 2
107 * Kernel packet data - these commands are sent by the neighbor
108 * The end nodes may cause these commands to be sent, but they see them beyond
112 #define KP_ACK_CONN 1
113 #define KP_CONN_DATA 2
116 #define KP_MISC_PADDING 0
119 * KP_INIT_SESSION[1] sessionid[4]
121 * finishes neighbor discovery and starts a session
123 * Before this is received all other commands are ignored. The sessionid is used
124 * to prevent usage of old neighbor discovery data (e.g. addresses)
126 #define KP_MISC_INIT_SESSION 1
128 #define KP_MISC_INIT_SESSION_CMDLEN 5
131 * KP_PING[1] cookie[4]
132 * KP_PONG[1] cookie[4] respdelay_full[4] respdelay_netonly[4]
134 * This is needed to find out whether the other node is reachable. After a new
135 * neighbor is seen, ping requests are sent and the neighbor is only reachable
136 * after a few pongs are received. These requests are also used to find out
137 * whether a neighber is gone.
140 * The receiver of a ping may delay the sending of the pong e.g. to create
141 * bigger packets. The respdelay is the time in microseconds the packet was
144 #define KP_MISC_PING 2
146 #define KP_MISC_PING_CMDLEN 5
148 #define KP_MISC_PONG 3
150 /* KP_ACK[1] seqno[6] */
151 #define KP_MISC_ACK 4
154 * NOTE on connection ids:
155 * connection ids we receive with most significant bit 0 have been generated by
157 * connection ids we receive with most significant bit 1 have been generated by
160 * ATTENTION: the priority seqno are reversed:
161 * priority seqnos we send are used when we send updates
162 * priority seqnos we received are used when we receive updates
166 * incoming connection
167 * seqno1... used to ack data sent from the side which initiated the connection
168 * seqno2... used to ack data sent to the side which initiated the connection
169 * KP_CONNECT[1] conn_id[4] seqno1[6] seqno2[6] window[2] priority_seqno[0.5]
170 * priority[1.5] is_highlatency[1]
172 #define KP_MISC_CONNECT 5
175 * incoming connection successful,
176 * KP_CONNECT_SUCCESS[1] conn_id[4] window[2]
178 #define KP_MISC_CONNECT_SUCCESS 6
181 * KP_RESET_CONN[1] conn_id[4]
182 * We send this, if there is an established connection we want to close.
184 #define KP_MISC_RESET_CONN 7
187 * KP_SET_MAX_CMSG_DELAY[1] cpacket_ack_fast_delay[4] cpacket_ack_slow_delay[4]
188 * data_ack_lowlatency_delay[4] data_ack_highlatency_delay[4] cmsg_delay[4]
189 * Sent after connecting and at any change
190 * delay in specifies in microsecs
192 #define KP_MISC_SET_MAX_CMSG_DELAY 8
194 #define KP_MISC_SET_MAX_CMSG_DELAY_CMDLEN 21
198 * KP_ACK_CONN[1] conn_id[4] delay_remaining[1] seqno[6] window[2]
199 * bufsize_changerate[1] seqno_ooo[6]
200 * length[1-4] priority_seqno[0.5] priority[1.5]
202 * conn_id is the conn_id we use if we sent something through this conn and
203 * *not* the conn_id that the neighbor used to send us the data
205 * delay_remaining = time the ack_conn could have remained in the queue
206 * 255 means the ack_conn has been sent immediately
207 * 0 means it has been delayed by as much the delay set by SET_MAX_CMSG_DELAY
209 * seqno = the seqno which is expected in the next non-out-of-order packet
211 * window = amount of data which can be sent without receiving the next ack
212 * packets with lower seqno do not overwrite the last window size
213 * The window may also be reduced. However, this only indicates a wish.
214 * Packets must be accepted if they exceed the new window, but not the old
219 * 1...255 = 64*2^((value-1)/7) end result is rounded down to an integer
221 * bufsize_changerate = if the next router(s) is increasing or decreasing its
223 * 0 = for every byte we can send, the end host will receive 2 bytes
224 * 64 = for every byte we can send, the end host will receive 1 byte
225 * 128 = for every 2 byte we can send, the end host will receive 1 byte
228 * seqno_ooo, length = This packet was received out of order. Maybe a previous
229 * packet has been lost. Out of order data should not be retransmitted.
230 * Multiple ooo packets may be merged into a single ack. Ooo packets may be
231 * partially accepted, so that the length does not cover the full packet and/
232 * or the seqno starts in the middle of a packet
234 #define KP_ACK_CONN_FLAGS_SEQNO 1
235 #define KP_ACK_CONN_FLAGS_WINDOW 2
236 #define KP_ACK_CONN_FLAGS_OOO 12 /* 4+8 */
237 #define KP_ACK_CONN_FLAGS_PRIORITY 16
239 static inline __u8
cor_ooolen_to_flags(__u32 len
)
250 static inline int cor_ooolen(__u8 flags
)
252 int len
= ((flags
& KP_ACK_CONN_FLAGS_OOO
) >> 2);
253 if (unlikely(len
== 3))
258 static inline int cor_ack_conn_len(__u8 flags
)
261 if ((flags
& KP_ACK_CONN_FLAGS_SEQNO
) != 0) {
263 if ((flags
& KP_ACK_CONN_FLAGS_WINDOW
) != 0)
267 if (cor_ooolen(flags
) != 0) {
269 len
+= cor_ooolen(flags
);
272 if ((flags
& KP_ACK_CONN_FLAGS_SEQNO
) != 0 ||
273 cor_ooolen(flags
) != 0)
276 if (flags
& KP_ACK_CONN_FLAGS_PRIORITY
)
282 /* KP_CONN_DATA[1] conn_id[4] seqno[6] length[1-2] data[length] */
283 #define KP_CONN_DATA_FLAGS_WINDOWUSED 31
284 #define KP_CONN_DATA_FLAGS_FLUSH 32
286 #define KP_CONN_DATA_MAXLEN (128+32767)
288 static inline __u32
get_kp_conn_data_length(__u32 datalen
)
297 static inline __u8
get_kp_code(__u8 maj
, __u8 min
)
299 BUILD_BUG_ON(maj
> 3);
301 return (maj
<< 6) + min
;
304 static inline __u8
kp_maj(__u8 code
)
309 static inline __u8
kp_min(__u8 code
)
316 * Connection data which in interpreted when connection has no target yet
317 * These commands are sent by the end node.
320 * cmd[2] length[1-4] parameter[length]
321 * unrecogniced commands are ignored
322 * parameters which are longer than expected are ignored as well
325 #define CD_CONTINUE_ON_ERROR_FLAG 32768
327 /* outgoing connection: CD_CONNECT_NB[2] length[1-4]
328 * addrlen[1-4] addr[addrlen] */
329 #define CD_CONNECT_NB 1
331 /* connection to local open part: CD_CONNECT_PORT[2] length[1-4] port[4] */
332 #define CD_CONNECT_PORT 2
335 * list connected neighbors: CD_LIST_NEIGH[2] length[1-4]
336 * responds with CDR_BINDATA if successful
341 * numfields[1-4] (field[2] fieldlen[1-4])[numfields]
342 * rows[responserows]:
343 * fieldlen[1-4], only if fieldlen in the header was "0"
344 * fielddata[fieldlen]
346 * Future versions may append data to field definition. Clients must silently
347 * discard fields they do not expect.
349 #define CD_LIST_NEIGH 3
354 #define LIST_NEIGH_FIELD_ADDR 1
357 * latency_in_microsecs[1] (64_11 encoding)
358 * Only raw network latency in measured. Delays caused by the priority queues
359 * are *not* included.
361 #define LIST_NEIGH_FIELD_LATENCY 2
364 * list services: CD_LIST_SERVICES[2] length[1-4]
365 * responds with CDR_BINDATA if successful
367 #define CD_LIST_SERVICES 4
370 * list services: CD_LIST_SERVICES[2] length[1-4]
371 * responds with CDR_BINDATA if successful
373 #define CD_LIST_L4PROTOCOLS 5
377 * Connection data response
378 * Format is the same as with connection data
387 * CDR_EXECFAILED[1] reasoncode[2]
389 #define CDR_EXECFAILED 2
390 #define CDR_EXECFAILED_INVALID_COMMAND 1
391 #define CDR_EXECFAILED_TEMPORARILY_OUT_OF_RESSOURCES 2
392 #define CDR_EXECFAILED_NB_DOESNTEXIST 3
393 #define CDR_EXECFAILED_UNKNOWN_L4PROTOCOL 4
394 #define CDR_EXECFAILED_PORTCLOSED 5
397 * must be sent after CDR_EXEC{OK|FAILED}
398 * CDR_EXEOK_BINDATA[1] bindatalen[1-4] bindata[bindatalen] */
399 #define CDR_BINDATA 3
401 #define L4PROTO_STREAM 42399
405 * routing daemon sock
407 * cmdcode[4] length[4] cmddata[length]
409 #define CRD_KTU_SUPPORTEDVERSIONS 1
411 * CRD_KTU_SUPPORTEDVERSIONS[4] length[4] min[4] max[4]
414 #define CRD_KTU_CONNECT 2
416 * CRD_KTU_KTOU_CONNECT[4] length[4] cookie[8] targetlen[4] target[targetlen]
419 #define CRD_UTK_VERSION 1
421 * CRD_UTK_VERSION[4] length[4] version[4]
425 #define CRD_UTK_UP_FLAGS_INTERFACES 1
427 * CRD_UTK_UP[4] length[4] flags[8] addrlen[4] addr[addrlen]
428 * if CRD_UTK_UP_FLAGS_INTERFACES:
429 * num_interfaces[4] (length[4] interface[length])[num_interfaces]
433 #define CRD_UTK_CONNECTERROR 3
435 * CRD_UTK_CONNECTERROR[4] length[4] cookie[8] error[4]
438 #define CRD_UTK_CONNECTERROR_ACCES 1
439 #define CRD_UTK_CONNECTERROR_NETUNREACH 2
440 #define CRD_UTK_CONNECTERROR_TIMEDOUT 3
441 #define CRD_UTK_CONNECTERROR_REFUSED 4
443 #define CONN_MNGD_HEADERLEN 2
444 #define CONN_MNGD_MAX_CTRL_DATALEN 8
445 #define CONN_MNGD_CHECKSUMLEN 4
447 #define CONN_MNGD_HASDATA (1 << 15)
448 #define CONN_MNGD_EOF (1 << 0)
449 #define CONN_MNGD_RCVEND (1 << 1)
450 #define CONN_MNGD_KEEPALIVE_REQ (1 << 2)
451 #define CONN_MNGD_KEEPALIVE_RESP (1 << 3)
452 #define CONN_MNGD_DATALEN 4095
454 #define CONN_MNGD_MAX_SEGMENT_SIZE (CONN_MNGD_DATALEN + 1)
457 struct cor_interface_config
{
462 #define CONGSTATUS_NONE 0
463 #define CONGSTATUS_CONNDATA 1
464 #define CONGSTATUS_ANNOUNCE 2
465 #define CONGSTATUS_RETRANS 3
466 #define CONGSTATUS_KPACKETS 4
468 struct cor_qos_queue
{
473 struct list_head queue_list
;
475 struct net_device
*dev
; /* may not change while queue is in list */
477 struct task_struct
*qos_resume_thread
;
478 wait_queue_head_t qos_resume_wq
;
479 atomic_t qos_resume_scheduled
;
480 unsigned long jiffies_lastprogress
;
482 struct list_head kpackets_waiting
;
483 struct list_head conn_retrans_waiting
;
484 struct list_head announce_waiting
;
485 struct list_head neighbors_waiting
;
486 struct list_head neighbors_waiting_nextpass
;
488 unsigned long jiffies_nb_pass_start
;
489 unsigned long jiffies_nb_lastduration
;
492 unsigned long jiffies_lastdrop
;
497 atomic_t cong_status
;
502 * switch to and from RB_INQUEUE_NBCONGWIN is only done with nbcongwin.lock
505 #define RB_INQUEUE_FALSE 0
506 #define RB_INQUEUE_TRUE 1
507 #define RB_INQUEUE_NBCONGWIN 2 /* only for nb->rb */
508 #define RB_INQUEUE_NBNOTACTIVE 3 /* only for nb->rb */
510 struct cor_resume_block
{
515 #define ANNOUNCE_TYPE_BROADCAST 1
516 #define ANNOUNCE_TYPE_UNICAST 2
518 struct cor_announce_data
{
524 struct net_device
*dev
;
525 char mac
[MAX_ADDR_LEN
];
526 struct delayed_work announce_work
;
527 struct cor_resume_block rb
;
530 struct cor_neighbor_discdata
{
532 unsigned long jiffies_created
;
536 struct net_device
*dev
;
537 char mac
[MAX_ADDR_LEN
];
551 struct cor_ping_cookie
{
552 ktime_t time_created
;
554 unsigned long jiffies_sent
;
557 __u8 pongs
; /* count of pongs for pings sent after this one */
560 #define NEIGHBOR_STATE_INITIAL 0
561 #define NEIGHBOR_STATE_ACTIVE 1
562 #define NEIGHBOR_STATE_STALLED 2
563 #define NEIGHBOR_STATE_KILLED 3
565 #define NBCONGWIN_SHIFT 16
566 #define NBCONGWIN_MUL (1 << NBCONGWIN_SHIFT)
569 struct list_head nb_list
;
574 struct net_device
*dev
;
575 char mac
[MAX_ADDR_LEN
];
576 struct cor_qos_queue
*queue
;
580 atomic_t sessionid_rcv_needed
;
581 atomic_t sessionid_snd_needed
;
586 atomic64_t cmsg_timer_timeout
;
587 struct timer_list cmsg_timer
;
588 spinlock_t cmsg_lock
;
589 struct list_head cmsg_queue_pong
;
590 struct list_head cmsg_queue_ack_fast
;
591 struct list_head cmsg_queue_ack_slow
;
592 struct list_head cmsg_queue_ackconn_urgent
;
593 struct list_head cmsg_queue_ackconn_lowlat
;
594 struct list_head cmsg_queue_ackconn_highlat
;
595 struct list_head cmsg_queue_conndata_lowlat
;
596 struct list_head cmsg_queue_conndata_highlat
;
597 struct list_head cmsg_queue_other
;
598 __u8 add_retrans_needed
;
599 __u64 kpacket_seqno
; /* not locked, only accessed by single tasklet */
601 struct rb_root pending_conn_resets_rb
;
603 __u32 cmsg_pongslength
;
604 __u32 cmsg_otherlength
;
606 __u32 cmsg_pongscnt
; /* size of queue only, protected by cmsg_lock */
607 atomic_t cmsg_pongs_retrans_cnt
; /* number of retransmits only */
608 atomic_t cmsg_othercnt
; /* size of queue + retransmits */
610 atomic_t cmsg_bulk_readds
;
612 atomic_t cmsg_delay_conndata
;
614 /* not locked, only accessed by single thread */
615 __u8 max_cmsg_delay_sent
;
617 /* procected by cor_qos_queue->qlock */
618 struct cor_resume_block rb_kp
;
619 struct cor_resume_block rb_cr
;
620 struct cor_resume_block rb
;
621 unsigned long cmsg_send_start_j
;
622 ktime_t cmsg_send_start_kt
;
627 struct list_head lh_nextpass
;
634 atomic64_t data_intransit
;
639 spinlock_t state_lock
;
640 unsigned long last_ping_time
;
642 struct cor_ping_cookie cookies
[PING_COOKIES_PER_NEIGH
];
643 __u32 ping_intransit
;
646 __u64 latency_variance_retrans_us
; /* microsecs */
647 atomic_t latency_retrans_us
; /* microsecs */
648 atomic_t latency_stddev_retrans_us
; /* microsecs */
649 atomic_t latency_advertised_us
; /* microsecs */
650 atomic_t max_remote_ack_fast_delay_us
; /* microsecs */
651 atomic_t max_remote_ack_slow_delay_us
; /* microsecs */
652 atomic_t max_remote_ackconn_lowlat_delay_us
; /* microsecs */
653 atomic_t max_remote_ackconn_highlat_delay_us
; /* microsecs */
654 atomic_t max_remote_pong_delay_us
; /* microsecs */
657 unsigned long initial_state_since
;/* initial state */
660 * time of the last sent packet which has been acked or
661 * otherwise responded to (e.g. pong)
663 unsigned long last_roundtrip
;/* active/stalled state */
665 ktime_t last_roundtrip_end
;
669 __u8 str_timer_pending
;
670 struct delayed_work stalltimeout_timer
;
672 spinlock_t connid_lock
;
673 struct rb_root connid_rb
;
675 spinlock_t connid_reuse_lock
;
676 struct rb_root connid_reuse_rb
;
677 struct list_head connid_reuse_list
;
678 __u16 connid_reuse_pingcnt
;
680 atomic64_t priority_sum
;
683 * connecions which receive data from/send data to this node
684 * used when terminating all connections of a neighbor and terminating
685 * inactive connections
687 spinlock_t conn_list_lock
;
688 struct list_head snd_conn_idle_list
;
689 struct list_head snd_conn_busy_list
;
692 * the timer has to be inited when adding the neighbor
694 * add_timer(struct timer_list * timer);
696 spinlock_t retrans_lock
;
697 struct timer_list retrans_timer
;
698 struct list_head retrans_fast_list
;
699 struct list_head retrans_slow_list
;
700 struct rb_root kp_retransmits_rb
;
702 spinlock_t retrans_conn_lock
;
703 struct timer_list retrans_conn_timer
;
704 struct list_head retrans_conn_lowlatency_list
;
705 struct list_head retrans_conn_highlatency_list
;
707 struct work_struct reset_neigh_work
;
710 static inline void cor_nb_kref_get(struct cor_neighbor
*nb
, char *reason
)
712 /* printk(KERN_ERR "cor_nb_kref_get %p %s\n", nb, reason); */
713 kref_get(&(nb
->ref
));
716 extern void cor_neighbor_free(struct kref
*ref
); /* neigh.c */
718 static inline void cor_nb_kref_put(struct cor_neighbor
*nb
, char *reason
)
720 /* printk(KERN_ERR "cor_nb_kref_put %p %s\n", nb, reason); */
721 kref_put(&(nb
->ref
), cor_neighbor_free
);
724 extern void cor_kreffree_bug(struct kref
*ref
); /* util.c */
726 static inline void cor_nb_kref_put_bug(struct cor_neighbor
*nb
, char *reason
)
728 /* printk(KERN_ERR "cor_nb_kref_put_bug %p %s\n", nb, reason); */
729 kref_put(&(nb
->ref
), cor_kreffree_bug
);
733 #define DATABUF_BUF 0
734 #define DATABUF_SKB 1
736 struct cor_data_buf_item
{
737 struct list_head buf_list
;
746 struct cor_connid_reuse_item
{
756 #define SNDSPEED_INIT 0
757 #define SNDSPEED_ACTIVE 1
758 struct cor_snd_speed
{
761 unsigned long jiffies_last_refresh
;
764 /* bytes per second */
771 /* This struct helps keep struct cor_conn small. */
772 struct cor_conn_src_sock_extradata
{
777 struct cor_snd_speed snd_speed
;
779 __be32 keepalive_req_cookie
;
780 __be32 keepalive_resp_cookie
;
782 * keepalive_intransit == 0... last resp received
783 * keepalive_intransit == 1... req sent
785 unsigned long jiffies_keepalive_lastact
;
788 char snd_hdr
[CONN_MNGD_HEADERLEN
];
789 char snd_data
[CONN_MNGD_MAX_CTRL_DATALEN
];
790 char snd_chksum
[CONN_MNGD_CHECKSUMLEN
];
795 char snd_hdr
[CONN_MNGD_HEADERLEN
];
796 char snd_chksum
[CONN_MNGD_CHECKSUMLEN
];
805 * There are 2 conn objects per bi-directional connection. They refer to each
806 * other with in the reversedir field.
812 * cn: conn we do not know what is inside
813 * src_in, trgt_unconn, trgt_out, ...: A conn with the specified source or
814 * targettype. In the unlocked case the types are only a guess, because they
815 * might have changed since the last access. After locking the
816 * source/destination parameters have to be checked whether they still are what
817 * we expect. This includes source/targettype, neighbor, conn_id
821 * no suffix: unlocked
823 * _l: this direction is locked
825 * _ll: both directions are locked
827 * _lx: this direction is locked, the other direction may be locked
829 * _o: unlocked, but source or target is known for sure, because an outside
830 * lock is taken; For variables on the heap this means that an outside lock must
831 * be taken before accessing the struct which points to the conn can be
835 * Most fields are protected by rcv_lock. Fields which which control
836 * source and destination of the data flow require both directions to
837 * to be locked and external references to be cleared before the change can
838 * happen. This includes fields like sourcetype, targettype, connid,
839 * list_heads, ???. In this case the side with is_client == 1 needs to be locked
842 * Some other fields are locked outside (e.g. at struct neighbor).
844 #define SOURCE_UNCONNECTED 0
846 #define SOURCE_SOCK 2
848 #define TARGET_UNCONNECTED 0
850 #define TARGET_SOCK 2
851 #define TARGET_DISCARD 3
853 #define BUFSIZE_NOACTION 0
854 #define BUFSIZE_DECR 1
855 #define BUFSIZE_DECR_FAST 2
856 #define BUFSIZE_INCR 3
857 #define BUFSIZE_INCR_FAST 4
859 #define JIFFIES_LAST_IDLE_SHIFT 8
860 #define BUFSIZE_SHIFT 5
862 #define SOCKTYPE_RAW 0
863 #define SOCKTYPE_MANAGED 1
865 #define RCV_BUF_STATE_OK 0
866 #define RCV_BUF_STATE_INCOMPLETE 1
867 #define RCV_BUF_STATE_RESET 2
869 #define SND_BUF_STATE_INCOMPLETE 0
870 #define SND_BUF_STATE_FILLED 1
877 __u8 is_client
; /* immutable after allocated */
881 * 0... connection active
882 * 1... connection is about to be reset, target does not need to be
884 * 2... connection is reset
895 struct cor_neighbor
*nb
;
897 struct list_head reorder_queue
;
898 __u32 reorder_memused
;
901 struct cor_connid_reuse_item
*cir
;
905 /* number of ack sent, not data seqno */
908 __u16 small_ooo_packets
;
912 __u8 inorder_ack_needed
;
916 __u64 window_seqnolimit
;
917 __u64 window_seqnolimit_remote
;
919 /* protected by nb->cmsg_lock */
920 struct list_head acks_pending
;
924 struct cor_conn_src_sock_extradata
*ed
;
927 * cl_list and in_cl_list is protected by cor_bindnodes
929 struct list_head cl_list
;
933 * keepalive_lh and in_keepalive_list is protected by
934 * cor_keepalive_req_lock
936 struct timer_list keepalive_timer
;
937 struct list_head keepalive_lh
;
939 __u8 in_keepalive_list
;
941 /* protected by flushtoconn_oom_lock */
942 struct list_head flushtoconn_oom_lh
;
943 /* protected by conn->rcv_lock */
944 __u8 in_flushtoconn_oom_list
;
947 __u8 keepalive_intransit
:1,
950 send_keepalive_req_needed
:1,
951 send_keepalive_resp_needed
:1,
953 send_rcvend_needed
:1;
955 __u8 last_windowused
;
968 char paramlen_buf
[4];
972 struct cor_neighbor
*nb
;
974 /* list of all connections to this neighbor */
975 struct list_head nb_list
;
976 unsigned long jiffies_last_act
;
977 __u32 nblist_busy_remaining
;
980 __u64 seqno_nextsend
;
982 __u64 seqno_windowlimit
;
984 /* protected by nb->retrans_conn_lock, sorted by seqno
986 struct list_head retrans_list
;
988 struct cor_resume_block rb
;
991 unsigned long jiffies_idle_since
;
998 __u8 lastsend_windowused
;
1000 __u8 remote_bufsize_changerate
;
1002 __u8 priority_send_allowed
:1,
1006 __u16 priority_last
:12,
1009 /* protected by nb->retrans_conn_lock */
1010 __u16 retrans_lowwindow
;
1014 __u8 waiting_for_userspace
;
1015 unsigned long waiting_for_userspace_since
;
1017 struct cor_sock
*cs
;
1022 char rcv_hdr
[CONN_MNGD_HEADERLEN
];
1023 char rcv_chksum
[CONN_MNGD_CHECKSUMLEN
];
1026 __u16 rcv_hdr_flags
;
1032 struct list_head items
;
1033 struct cor_data_buf_item
*nextread
;
1038 __u32 read_remaining
;
1040 __u16 next_read_offset
;
1043 __u32 bufspace_accounted
;
1046 __u32 bufsize
; /* 32 ==> 1 byte, see BUFSIZE_SHIFT */
1047 __u32 ignore_rcv_lowbuf
;
1068 static inline __u32
cor_get_connid_reverse(__u32 conn_id
)
1070 return conn_id
^ (1 << 31);
1073 struct cor_conn_bidir
{
1074 struct cor_conn cli
;
1075 struct cor_conn srv
;
1080 static inline struct cor_conn_bidir
* cor_get_conn_bidir(struct cor_conn
*cn
)
1082 if (cn
->is_client
) {
1083 return container_of(cn
, struct cor_conn_bidir
, cli
);
1085 return container_of(cn
, struct cor_conn_bidir
, srv
);
1089 static inline struct cor_conn
* cor_get_conn_reversedir(struct cor_conn
*cn
)
1091 if (cn
->is_client
) {
1092 struct cor_conn_bidir
*cnb
= container_of(cn
,
1093 struct cor_conn_bidir
, cli
);
1096 struct cor_conn_bidir
*cnb
= container_of(cn
,
1097 struct cor_conn_bidir
, srv
);
1102 static inline void cor_conn_kref_get(struct cor_conn
*cn
, char *reason
)
1104 /* printk(KERN_ERR "cor_conn_kref_get %p %s\n", cn, reason); */
1105 kref_get(&(cor_get_conn_bidir(cn
)->ref
));
1108 extern void cor_free_conn(struct kref
*ref
); /* conn.c */
1110 static inline void cor_conn_kref_put(struct cor_conn
*cn
, char *reason
)
1112 /* printk(KERN_ERR "cor_conn_kref_put %p %s\n", cn, reason); */
1113 kref_put(&(cor_get_conn_bidir(cn
)->ref
), cor_free_conn
);
1116 static inline void cor_conn_kref_put_bug(struct cor_conn
*cn
, char *reason
)
1118 /* printk(KERN_ERR "cor_conn_kref_put_bug %p %s\n", cn, reason); */
1119 kref_put(&(cor_get_conn_bidir(cn
)->ref
), cor_kreffree_bug
);
1124 #define CONN_RETRANS_INITIAL 0
1125 #define CONN_RETRANS_SCHEDULED 1
1126 #define CONN_RETRANS_LOWWINDOW 2
1127 #define CONN_RETRANS_SENDING 3
1128 #define CONN_RETRANS_ACKED 4
1129 struct cor_conn_retrans
{
1130 /* timeout_list and conn_list share a single ref */
1132 /* only in timeout_list if state == CONN_RETRANS_SCHEDULED */
1133 struct list_head timeout_list
;
1134 struct list_head conn_list
;
1135 struct cor_conn
*trgt_out_o
;
1141 unsigned long timeout
;
1144 #define RCVOOO_BUF 0
1145 #define RCVOOO_SKB 1
1147 struct list_head lh
;
1154 struct cor_rcvooo_buf
{
1155 struct cor_rcvooo r
;
1160 /* inside skb->cb */
1161 struct cor_skb_procstate
{
1164 struct work_struct work
;
1173 struct cor_rcvooo r
;
1177 struct cor_data_buf_item dbi
;
1182 #define CS_TYPE_UNCONNECTED 0
1183 #define CS_TYPE_LISTENER 1
1184 #define CS_TYPE_CONN_RAW 2
1185 #define CS_TYPE_CONN_MANAGED 3
1187 #define CS_CONNECTSTATE_UNCONNECTED 0
1188 #define CS_CONNECTSTATE_CONNECTING 1
1189 #define CS_CONNECTSTATE_CONNECTED 2
1190 #define CS_CONNECTSTATE_ERROR 3
1193 struct sock sk
; /* must be first */
1198 /* type may not change once it is set to != CS_TYPE_UNCONNECTED */
1202 __u8 publish_service
;
1204 __u8 is_highlatency
;
1210 /* listener is protected by cor_bindnodes */
1211 struct list_head lh
;
1213 __u8 publish_service
;
1216 struct list_head conn_queue
;
1220 struct cor_conn
*src_sock
;
1221 struct cor_conn
*trgt_sock
;
1223 struct cor_data_buf_item
*rcvitem
;
1226 struct cor_sock
*pass_on_close
;
1230 struct cor_sockaddr remoteaddr
;
1232 struct list_head rd_msgs
;
1233 struct list_head crd_lh
;
1252 struct cor_conn
*src_sock
;
1253 struct cor_conn
*trgt_sock
;
1257 __u16 snd_segment_size
;
1259 __u8 send_in_progress
;
1264 __u16 rcvbuf_consumed
;
1269 struct work_struct readfromconn_work
;
1270 atomic_t readfromconn_work_scheduled
;
1272 atomic_t ready_to_read
;
1273 atomic_t ready_to_write
;
1274 atomic_t ready_to_accept
;
1277 #define ACK_NEEDED_NO 0
1278 #define ACK_NEEDED_SLOW 1
1279 #define ACK_NEEDED_FAST 2
1282 extern spinlock_t cor_local_addr_lock
;
1283 extern char *cor_local_addr
;
1284 extern __u32 cor_local_addrlen
;
1285 extern __be32 cor_local_addr_sessionid
;
1287 extern int cor_is_device_configurated(struct net_device
*dev
);
1289 extern void cor_set_interface_config(struct cor_interface_config
*new_config
,
1290 __u32 new_num_interfaces
, int new_all_interfaces
);
1292 extern void cor_config_down(void);
1294 extern int cor_config_up(char *addr2
, __u32 addrlen2
);
1296 extern int cor_is_clientmode(void);
1299 extern void cor_qos_set_lastdrop(struct cor_qos_queue
*q
);
1301 #ifdef DEBUG_QOS_SLOWSEND
1302 extern int _cor_dev_queue_xmit(struct sk_buff
*skb
, int caller
);
1304 static inline int _cor_dev_queue_xmit(struct sk_buff
*skb
, int caller
)
1306 return dev_queue_xmit(skb
);
1310 static inline int cor_dev_queue_xmit(struct sk_buff
*skb
,
1311 struct cor_qos_queue
*q
, int caller
)
1313 int rc
= _cor_dev_queue_xmit(skb
, caller
);
1314 if (unlikely(rc
!= NET_XMIT_SUCCESS
))
1315 cor_qos_set_lastdrop(q
);
1319 extern void cor_free_qos(struct kref
*ref
);
1321 #ifdef COR_NBCONGWIN
1322 extern void cor_nbcongwin_data_retransmitted(struct cor_neighbor
*nb
,
1325 extern void cor_nbcongwin_data_acked(struct cor_neighbor
*nb
,
1328 extern void cor_nbcongwin_data_sent(struct cor_neighbor
*nb
, __u32 bytes_sent
);
1330 extern int cor_nbcongwin_send_allowed(struct cor_neighbor
*nb
);
1334 static inline void cor_nbcongwin_data_retransmitted(struct cor_neighbor
*nb
,
1339 static inline void cor_nbcongwin_data_acked(struct cor_neighbor
*nb
,
1344 static inline void cor_nbcongwin_data_sent(struct cor_neighbor
*nb
,
1349 static inline int cor_nbcongwin_send_allowed(struct cor_neighbor
*nb
)
1355 extern unsigned long cor_get_conn_idletime(struct cor_conn
*trgt_out_l
);
1357 extern struct cor_qos_queue
*cor_get_queue(struct net_device
*dev
);
1359 extern int cor_destroy_queue(struct net_device
*dev
);
1361 extern int cor_create_queue(struct net_device
*dev
);
1363 #define QOS_RESUME_DONE 0
1364 #define QOS_RESUME_CONG 1
1365 #define QOS_RESUME_NEXTNEIGHBOR 2 /* cor_resume_neighbors() internal */
1366 #define QOS_RESUME_EXIT 3
1368 #define QOS_CALLER_KPACKET 0
1369 #define QOS_CALLER_CONN_RETRANS 1
1370 #define QOS_CALLER_ANNOUNCE 2
1371 #define QOS_CALLER_NEIGHBOR 3
1373 static inline void cor_schedule_qos_resume(struct cor_qos_queue
*q
)
1375 if (atomic_cmpxchg(&(q
->qos_resume_scheduled
), 0, 1) == 0) {
1377 wake_up(&(q
->qos_resume_wq
));
1381 extern void cor_qos_enqueue(struct cor_qos_queue
*q
,
1382 struct cor_resume_block
*rb
, unsigned long cmsg_send_start_j
,
1383 ktime_t cmsg_send_start_kt
, int caller
,
1384 int from_nbnotactive_resume
);
1386 extern void cor_qos_remove_conn(struct cor_conn
*trgt_out_l
);
1388 extern int cor_may_send_announce(struct net_device
*dev
);
1390 extern struct sk_buff
*cor_create_packet_cmsg(struct cor_neighbor
*nb
, int size
,
1391 gfp_t alloc_flags
, __u64 seqno
);
1393 extern struct sk_buff
*cor_create_packet(struct cor_neighbor
*nb
, int size
,
1396 extern struct sk_buff
*cor_create_packet_conndata(struct cor_neighbor
*nb
,
1397 int size
, gfp_t alloc_flags
, __u32 conn_id
, __u64 seqno
,
1398 __u8 windowused
, __u8 flush
);
1400 extern void cor_qos_enqueue_conn(struct cor_conn
*trgt_out_lx
);
1402 extern void cor_dev_down(void);
1404 extern int cor_dev_up(void);
1406 extern void __exit
cor_dev_exit1(void);
1408 extern int __init
cor_dev_init(void);
1411 static inline __u16
cor_enc_priority(__u32 value
)
1417 while ((value
>> exponent
) > 255) {
1420 BUG_ON(exponent
> 15);
1422 mantissa
= (value
>> exponent
);
1423 ret
= (mantissa
<< 4) | exponent
;
1428 static inline __u32
cor_dec_priority(__u16 priority
)
1430 __u32 mantissa
= (__u32
) (priority
>> 4);
1431 __u16 exponent
= (priority
& 15);
1432 BUG_ON(priority
> 4095);
1433 return (mantissa
<< exponent
);
1436 static inline __u32
cor_priority_max(void)
1438 return cor_dec_priority(4095);
1441 static inline __u16
cor_enc_window(__u64 value
)
1447 while ((value
>> exponent
) > 2047) {
1450 if (unlikely(exponent
> 31))
1453 mantissa
= (value
>> exponent
);
1454 ret
= (mantissa
<< 5) | exponent
;
1455 BUG_ON(ret
> 65535);
1459 static inline __u32
cor_dec_window(__u16 value
)
1461 __u64 mantissa
= (__u64
) (value
>> 5);
1462 __u16 exponent
= (value
& 31);
1463 return (mantissa
<< exponent
);
1466 extern __u8
__attribute__((const)) cor_enc_log_64_11(__u32 value
);
1468 extern __u32
__attribute__((const)) cor_dec_log_64_11(__u8 value
);
1470 extern void cor_swap_list_items(struct list_head
*lh1
, struct list_head
*lh2
);
1472 extern int __init
cor_util_init(void);
1477 extern atomic_t cor_num_neighs
;
1479 extern int cor_is_from_nb(struct sk_buff
*skb
, struct cor_neighbor
*nb
);
1481 extern struct cor_neighbor
*_cor_get_neigh_by_mac(struct net_device
*dev
,
1484 extern struct cor_neighbor
*cor_get_neigh_by_mac(struct sk_buff
*skb
);
1486 extern struct cor_neighbor
*cor_find_neigh(char *addr
, __u16 addrlen
);
1488 extern __u32
cor_generate_neigh_list(char *buf
, __u32 buflen
);
1490 extern void cor_reset_neighbors(struct net_device
*dev
);
1492 extern int cor_get_neigh_state(struct cor_neighbor
*nb
);
1494 extern void cor_ping_resp(struct cor_neighbor
*nb
, __u32 cookie
,
1497 extern __u32
cor_add_ping_req(struct cor_neighbor
*nb
,
1498 unsigned long *last_ping_time
);
1500 extern void cor_ping_sent(struct cor_neighbor
*nb
, __u32 cookie
);
1502 extern void cor_unadd_ping_req(struct cor_neighbor
*nb
, __u32 cookie
,
1503 unsigned long last_ping_time
, int congested
);
1505 #define TIMETOSENDPING_NO 0
1506 #define TIMETOSENDPING_YES 1
1507 #define TIMETOSENDPING_FORCE 2
1508 extern int cor_time_to_send_ping(struct cor_neighbor
*nb
);
1510 extern unsigned long cor_get_next_ping_time(struct cor_neighbor
*nb
);
1512 extern void cor_add_neighbor(struct cor_neighbor_discdata
*nb_dd
);
1514 extern struct cor_conn
*cor_get_conn(struct cor_neighbor
*nb
, __u32 conn_id
);
1516 extern int cor_insert_connid(struct cor_neighbor
*nb
,
1517 struct cor_conn
*src_in_ll
);
1519 extern void cor_insert_connid_reuse(struct cor_neighbor
*nb
,
1520 struct cor_connid_reuse_item
*ins
);
1522 extern int cor_connid_alloc(struct cor_neighbor
*nb
,
1523 struct cor_conn
*src_in_ll
);
1525 extern int __init
cor_neighbor_init(void);
1527 extern void __exit
cor_neighbor_exit2(void);
1529 /* neigh_ann_rcv.c */
1530 extern int cor_rcv_announce(struct sk_buff
*skb
);
1532 extern int __init
cor_neigh_ann_rcv_init(void);
1534 extern void __exit
cor_neigh_ann_rcv_exit2(void);
1536 /* neigh_ann_snd.c */
1537 extern int _cor_send_announce(struct cor_announce_data
*ann
, int fromqos
,
1540 extern void cor_announce_data_free(struct kref
*ref
);
1542 extern void cor_announce_send_start(struct net_device
*dev
, char *mac
,
1545 extern void cor_announce_send_stop(struct net_device
*dev
, char *mac
, int type
);
1548 extern void cor_kernel_packet(struct cor_neighbor
*nb
, struct sk_buff
*skb
,
1552 struct cor_control_msg_out
;
1554 #define ACM_PRIORITY_LOW 1 /* oom recovery easy */
1555 #define ACM_PRIORITY_MED 2 /* oom may cause timeouts */
1556 #define ACM_PRIORITY_HIGH 3 /* cm acks - needed for freeing old cms */
1558 extern struct cor_control_msg_out
*cor_alloc_control_msg(
1559 struct cor_neighbor
*nb
, int priority
);
1561 extern void cor_free_control_msg(struct cor_control_msg_out
*cm
);
1563 extern void cor_retransmit_timerfunc(struct timer_list
*retrans_timer
);
1565 extern void cor_kern_ack_rcvd(struct cor_neighbor
*nb
, __u64 seqno
);
1567 extern int cor_send_messages(struct cor_neighbor
*nb
,
1568 unsigned long cmsg_send_start_j
, ktime_t cmsg_send_start_kt
,
1571 extern void cor_controlmsg_timerfunc(struct timer_list
*cmsg_timer
);
1573 extern void cor_schedule_controlmsg_timer(struct cor_neighbor
*nb_cmsglocked
);
1575 extern void cor_send_pong(struct cor_neighbor
*nb
, __u32 cookie
,
1576 ktime_t ping_rcvtime
);
1578 extern int cor_send_reset_conn(struct cor_neighbor
*nb
, __u32 conn_id
,
1581 extern void cor_send_ack(struct cor_neighbor
*nb
, __u64 seqno
, __u8 fast
);
1583 extern void cor_send_ack_conn_ifneeded(struct cor_conn
*src_in_l
, __u64 seqno_ooo
,
1586 extern void cor_send_priority(struct cor_conn
*trgt_out_ll
, __u16 priority
);
1588 extern void cor_free_ack_conns(struct cor_conn
*src_in_lx
);
1590 extern void cor_send_connect_success(struct cor_control_msg_out
*cm
,
1591 __u32 conn_id
, struct cor_conn
*src_in
);
1593 extern void cor_send_connect_nb(struct cor_control_msg_out
*cm
, __u32 conn_id
,
1594 __u64 seqno1
, __u64 seqno2
, struct cor_conn
*src_in_ll
);
1596 extern void cor_send_conndata(struct cor_control_msg_out
*cm
, __u32 conn_id
,
1597 __u64 seqno
, char *data_orig
, char *data
, __u32 datalen
,
1598 __u8 windowused
, __u8 flush
, __u8 highlatency
,
1599 struct cor_conn_retrans
*cr
);
1601 extern int __init
cor_kgen_init(void);
1603 extern void __exit
cor_kgen_exit2(void);
1606 extern struct kmem_cache
*cor_connid_reuse_slab
;
1608 extern atomic_t cor_num_conns
;
1610 extern spinlock_t cor_bindnodes
;
1612 extern int cor_new_incoming_conn_allowed(struct cor_neighbor
*nb
);
1614 extern __u32
_cor_conn_refresh_priority(struct cor_conn
*cn_lx
);
1616 extern __u32
cor_conn_refresh_priority(struct cor_conn
*cn
, int locked
);
1618 extern void cor_set_conn_in_priority(struct cor_neighbor
*nb
, __u32 conn_id
,
1619 struct cor_conn
*src_in
, __u8 priority_seqno
, __u16 priority
);
1621 extern void cor_conn_set_last_act(struct cor_conn
*trgt_out_lx
);
1623 extern int cor_conn_init_out(struct cor_conn
*trgt_unconn_ll
,
1624 struct cor_neighbor
*nb
, __u32 rcvd_connid
,
1625 int use_rcvd_connid
);
1627 extern int cor_conn_init_sock_source(struct cor_conn
*cn
);
1629 extern void cor_conn_init_sock_target(struct cor_conn
*cn
);
1631 extern __u32
cor_list_services(char *buf
, __u32 buflen
);
1633 extern void cor_set_publish_service(struct cor_sock
*cs
, __u8 value
);
1635 extern void cor_close_port(struct cor_sock
*cs
);
1637 extern int cor_open_port(struct cor_sock
*cs_l
, __be32 port
);
1639 #define CONNECT_PORT_OK 0
1640 #define CONNECT_PORT_PORTCLOSED 1
1641 #define CONNECT_PORT_TEMPORARILY_OUT_OF_RESSOURCES 2
1643 extern int cor_connect_port(struct cor_conn
*trgt_unconn_ll
, __be32 port
);
1645 extern int cor_connect_neigh(struct cor_conn
*trgt_unconn_ll
, char *addr
,
1648 extern struct cor_conn_bidir
* cor_alloc_conn(gfp_t allocflags
,
1649 __u8 is_highlatency
);
1651 extern void cor_reset_conn_locked(struct cor_conn_bidir
*cnb_ll
);
1653 extern void cor_reset_conn(struct cor_conn
*cn
);
1656 extern void cor_reset_ooo_queue(struct cor_conn
*src_in_lx
);
1658 extern void cor_drain_ooo_queue(struct cor_conn
*src_in_l
);
1660 extern void cor_conn_rcv(struct cor_neighbor
*nb
,
1661 struct sk_buff
*skb
, char *data
, __u32 len
,
1662 __u32 conn_id
, __u64 seqno
,
1663 __u8 windowused
, __u8 flush
);
1665 extern int __init
cor_rcv_init(void);
1667 extern void __exit
cor_rcv_exit2(void);
1669 /* conn_src_sock.c */
1670 extern void cor_update_src_sock_sndspeed(struct cor_conn
*src_sock_l
,
1673 extern int cor_sock_sndbufavailable(struct cor_conn
*src_sock_lx
,
1677 #define RC_FTC_OOM 1
1678 #define RC_FTC_ERR 2
1679 extern int _cor_mngdsocket_flushtoconn(struct cor_conn
*src_sock_l
);
1681 extern int cor_mngdsocket_flushtoconn_ctrl(struct cor_sock
*cs_m_l
,
1682 __u8 send_eof
, __u8 send_rcvend
,
1683 __u8 send_keepalive_resp
, __be32 keepalive_resp_cookie
);
1685 extern int cor_mngdsocket_flushtoconn_data(struct cor_sock
*cs_m_l
);
1687 extern void cor_keepalive_req_timerfunc(struct timer_list
*retrans_conn_timer
);
1689 extern void cor_keepalive_req_sched_timer(struct cor_conn
*src_sock_lx
);
1691 extern void cor_keepalive_resp_rcvd(struct cor_sock
*cs_m_l
, __be32 cookie
);
1693 extern int __init
cor_conn_src_sock_init1(void);
1695 extern void __exit
cor_conn_src_sock_exit1(void);
1697 /* conn_trgt_unconn.c */
1698 extern int cor_encode_len(char *buf
, int buflen
, __u32 len
);
1700 extern void cor_proc_cpacket(struct cor_conn
*trgt_unconn
);
1702 /* conn_trgt_out.c */
1703 extern void cor_free_connretrans(struct kref
*ref
);
1705 extern void cor_reschedule_conn_retrans_timer(
1706 struct cor_neighbor
*nb_retranslocked
);
1708 extern void cor_cancel_all_conn_retrans(struct cor_conn
*trgt_out_l
);
1710 extern int cor_send_retrans(struct cor_neighbor
*nb
, int *sent
);
1712 extern void cor_retransmit_conn_timerfunc(struct timer_list
*retrans_timer_conn
);
1714 extern void cor_conn_ack_ooo_rcvd(struct cor_neighbor
*nb
, __u32 conn_id
,
1715 struct cor_conn
*trgt_out
, __u64 seqno_ooo
, __u32 length
,
1716 __u64
*bytes_acked
);
1718 extern void cor_conn_ack_rcvd(struct cor_neighbor
*nb
, __u32 conn_id
,
1719 struct cor_conn
*trgt_out
, __u64 seqno
, int setwindow
,
1720 __u16 window
, __u8 bufsize_changerate
, __u64
*bytes_acked
);
1722 extern void cor_schedule_retransmit_conn(struct cor_conn_retrans
*cr
,
1723 int connlocked
, int nbretrans_locked
);
1725 extern int cor_srcin_buflimit_reached(struct cor_conn
*src_in_lx
);
1727 /* RC_FLUSH_CONN_OUT_SENT | RC_FLUSH_CONN_OUT_{^SENT} */
1728 #define RC_FLUSH_CONN_OUT_OK 1
1729 #define RC_FLUSH_CONN_OUT_SENT_CONG 2 /* cor_flush_out internal only */
1730 #define RC_FLUSH_CONN_OUT_NBNOTACTIVE 3
1731 #define RC_FLUSH_CONN_OUT_CONG 4
1732 #define RC_FLUSH_CONN_OUT_MAXSENT 5
1733 #define RC_FLUSH_CONN_OUT_OOM 6
1735 extern int _cor_flush_out(struct cor_conn
*trgt_out_lx
, __u32 maxsend
,
1736 __u32
*sent
, int from_qos
, int maxsend_forcedelay
);
1738 static inline int cor_flush_out(struct cor_conn
*trgt_out_lx
, __u32
*sent
)
1740 int rc
= _cor_flush_out(trgt_out_lx
, 1 << 30, sent
, 0, 0);
1742 if (rc
== RC_FLUSH_CONN_OUT_CONG
|| rc
== RC_FLUSH_CONN_OUT_MAXSENT
||
1743 rc
== RC_FLUSH_CONN_OUT_OOM
||
1744 rc
== RC_FLUSH_CONN_OUT_NBNOTACTIVE
)
1745 cor_qos_enqueue_conn(trgt_out_lx
);
1750 extern int __init
cor_snd_init(void);
1752 extern void __exit
cor_snd_exit2(void);
1754 /* conn_trgt_sock.c */
1755 extern void cor_flush_sock_managed(struct cor_conn
*trgt_sock_lx
,
1756 int from_recvmsg
, __u8
*do_wake_sender
);
1758 extern void cor_flush_sock(struct cor_conn
*trgt_sock_lx
);
1760 /* conn_databuf.c */
1761 extern struct kmem_cache
*cor_data_buf_item_slab
;
1763 extern void cor_databuf_init(struct cor_conn
*cn_init
);
1765 extern void cor_bufsize_init(struct cor_conn
*cn_l
, __u32 bufsize
);
1767 extern int cor_account_bufspace(struct cor_conn
*cn_lx
);
1769 extern int cor_conn_src_unconn_write_allowed(struct cor_conn
*src_unconn_lx
);
1771 extern void cor_update_windowlimit(struct cor_conn
*src_in_lx
);
1773 extern __u8
_cor_bufsize_update_get_changerate(struct cor_conn
*cn_lx
);
1775 static inline int cor_bufsize_initial_phase(struct cor_conn
*cn_lx
)
1777 return unlikely(cn_lx
->bufsize
.bytes_rcvd
!= (1 << 24) - 1 &&
1778 cn_lx
->bufsize
.bytes_rcvd
< cn_lx
->bufsize
.bufsize
);
1781 static inline int cor_ackconn_urgent(struct cor_conn
*cn_lx
)
1783 return cor_bufsize_initial_phase(cn_lx
) ||
1784 cn_lx
->bufsize
.state
== BUFSIZE_INCR_FAST
;
1787 extern void cor_bufsize_read_to_sock(struct cor_conn
*trgt_sock_lx
);
1789 extern void cor_databuf_ackdiscard(struct cor_conn
*cn_lx
);
1791 extern void cor_reset_seqno(struct cor_conn
*cn_l
, __u64 initseqno
);
1793 extern void cor_databuf_pull(struct cor_conn
*cn_lx
, char *dst
, __u32 len
);
1795 static inline __u32
cor_databuf_trypull(struct cor_conn
*cn_l
, char *dst
,
1798 if (len
> cn_l
->data_buf
.read_remaining
)
1799 len
= cn_l
->data_buf
.read_remaining
;
1800 cor_databuf_pull(cn_l
, dst
, len
);
1804 extern void cor_databuf_unpull_dpi(struct cor_conn
*trgt_sock
, struct cor_sock
*cs
,
1805 struct cor_data_buf_item
*item
, __u16 next_read_offset
);
1807 extern void cor_databuf_pull_dbi(struct cor_sock
*cs_rl
,
1808 struct cor_conn
*trgt_sock_l
);
1810 extern void cor_databuf_unpull(struct cor_conn
*trgt_out_l
, __u32 bytes
);
1812 extern void cor_databuf_pullold(struct cor_conn
*trgt_out_l
, __u64 startpos
,
1813 char *dst
, int len
);
1815 extern void cor_databuf_ack(struct cor_conn
*trgt_out_l
, __u64 pos
);
1817 extern void cor_databuf_ackread(struct cor_conn
*cn_lx
);
1819 extern __u32
_cor_receive_buf(struct cor_conn
*cn_lx
, char *buf
, __u32 datalen
,
1820 int from_sock
, __u8 windowused
, __u8 flush
);
1822 static inline __u32
cor_receive_buf(struct cor_conn
*cn_lx
, char *buf
,
1823 __u32 datalen
, __u8 windowused
, __u8 flush
)
1825 return _cor_receive_buf(cn_lx
, buf
, datalen
, 0, windowused
, flush
);
1828 static inline __u32
cor_receive_sock(struct cor_conn
*src_sock_l
, char *buf
,
1829 __u32 datalen
, __u8 flush
)
1833 BUG_ON(src_sock_l
->sourcetype
!= SOURCE_SOCK
);
1835 ret
= _cor_receive_buf(src_sock_l
, buf
, datalen
, 1,
1836 src_sock_l
->source
.sock
.last_windowused
, flush
);
1838 if (likely(ret
> 0)) {
1839 __u32 bufsize
= src_sock_l
->bufsize
.bufsize
>> BUFSIZE_SHIFT
;
1840 __u32 bufused
= src_sock_l
->data_buf
.read_remaining
;
1842 if (bufused
>= bufsize
)
1843 src_sock_l
->source
.sock
.last_windowused
= 31;
1844 else if (unlikely(bufused
* 31 > U32_MAX
))
1845 src_sock_l
->source
.sock
.last_windowused
=
1846 bufused
/(bufsize
/31);
1848 src_sock_l
->source
.sock
.last_windowused
=
1849 (bufused
*31)/bufsize
;
1855 extern __u32
cor_receive_skb(struct cor_conn
*src_in_l
, struct sk_buff
*skb
,
1856 __u8 windowused
, __u8 flush
);
1858 extern void cor_wake_sender(struct cor_conn
*cn
);
1860 extern int __init
cor_forward_init(void);
1862 extern void __exit
cor_forward_exit2(void);
1864 /* sock_rdaemon.c */
1865 extern int cor_is_device_configurated(struct net_device
*dev
);
1867 extern int cor_create_rdaemon_sock(struct net
*net
, struct socket
*sock
,
1868 int protocol
, int kern
);
1870 extern int cor_rdreq_connect(struct cor_sock
*cs
);
1872 extern void cor_usersock_release(struct cor_sock
*cs
);
1874 extern int __init
cor_rd_init1(void);
1876 extern int __init
cor_rd_init2(void);
1878 extern void __exit
cor_rd_exit1(void);
1880 extern void __exit
cor_rd_exit2(void);
1883 extern int cor_create_raw_sock(struct net
*net
, struct socket
*sock
,
1884 int protocol
, int kern
);
1886 /* sock_managed.c */
1887 extern struct cor_sock
*cor_get_sock_by_cookie(__be64 cookie
);
1889 extern void __cor_set_sock_connecterror(struct cor_sock
*cs_m_l
, int errorno
);
1891 extern void _cor_set_sock_connecterror(struct cor_sock
*cs
, int errorno
);
1893 extern void cor_mngdsocket_chksum(char *hdr
, __u32 hdrlen
,
1894 char *data
, __u32 datalen
,
1895 char *chksum
, __u32 chksum_len
);
1897 static inline void cor_set_sock_connecterror(__be64 cookie
, int errorno
)
1899 struct cor_sock
*cs
= cor_get_sock_by_cookie(cookie
);
1900 _cor_set_sock_connecterror(cs
, errorno
);
1903 extern void cor_mngdsocket_readfromconn_fromatomic(struct cor_sock
*cs
);
1905 extern void cor_mngdsocket_readfromconn_wq(struct work_struct
*work
);
1907 extern int cor_create_managed_sock(struct net
*net
, struct socket
*sock
,
1908 int protocol
, int kern
);
1910 extern int __init
cor_sock_managed_init1(void);
1913 extern void cor_free_sock(struct kref
*ref
);
1915 extern int cor_socket_setsockopt_tos(struct socket
*sock
,
1916 char __user
*optval
, unsigned int optlen
);
1918 extern int cor_socket_setsockopt_priority(struct socket
*sock
,
1919 char __user
*optval
, unsigned int optlen
);
1921 extern int cor_socket_socketpair(struct socket
*sock1
, struct socket
*sock2
);
1923 extern int cor_socket_getname(struct socket
*sock
, struct sockaddr
*addr
,
1926 extern int cor_socket_mmap(struct file
*file
, struct socket
*sock
,
1927 struct vm_area_struct
*vma
);
1929 extern int _cor_createsock(struct net
*net
, struct socket
*sock
, int protocol
,
1930 int kern
, __u8 is_client
);
1932 extern int __init
cor_sock_init1(void);
1934 extern int __init
cor_sock_init2(void);
1936 extern void __exit
cor_sock_exit1(void);
1939 static inline struct cor_skb_procstate
*cor_skb_pstate(struct sk_buff
*skb
)
1941 BUILD_BUG_ON(sizeof(struct cor_skb_procstate
) > sizeof(skb
->cb
));
1942 return (struct cor_skb_procstate
*) &(skb
->cb
[0]);
1945 static inline struct sk_buff
*cor_skb_from_pstate(struct cor_skb_procstate
*ps
)
1947 return (struct sk_buff
*) (((char *)ps
) - offsetof(struct sk_buff
,cb
));
1950 static inline int cor_qos_fastsend_allowed_conn_retrans(struct cor_neighbor
*nb
)
1952 return atomic_read(&(nb
->queue
->cong_status
)) < CONGSTATUS_RETRANS
;
1955 static inline int cor_qos_fastsend_allowed_announce(struct net_device
*dev
)
1958 struct cor_qos_queue
*q
= cor_get_queue(dev
);
1963 rc
= atomic_read(&(q
->cong_status
)) < CONGSTATUS_ANNOUNCE
;
1965 kref_put(&(q
->ref
), cor_free_qos
);
1970 static inline int cor_qos_fastsend_allowed_conn(struct cor_conn
*trgt_out_lx
)
1972 struct cor_qos_queue
*q
= trgt_out_lx
->target
.out
.nb
->queue
;
1973 return atomic_read(&(q
->cong_status
)) < CONGSTATUS_CONNDATA
;
1976 static inline __u32
cor_mss(struct cor_neighbor
*nb
, __u32 l3overhead
)
1978 return nb
->dev
->mtu
- LL_RESERVED_SPACE(nb
->dev
) - l3overhead
;
1981 static inline __u32
cor_mss_cmsg(struct cor_neighbor
*nb
)
1983 return cor_mss(nb
, 7);
1986 static inline __u32
cor_mss_conndata(struct cor_neighbor
*nb
, int highlatency
)
1988 __u32 mss_tmp
= cor_mss(nb
, 11);
1991 if (mss_tmp
< 256 || highlatency
|| LOWLATENCY_LOWERMTU
== 0)
1994 for (i
=256;i
<4096;i
*=2) {
1999 return mss_tmp
- mss_tmp
%4096;
2002 static inline __u32
cor_send_conndata_as_skb(struct cor_neighbor
*nb
,
2005 return size
>= cor_mss_conndata(nb
, 0)/2 || size
> KP_CONN_DATA_MAXLEN
;
2008 static inline long cor_calc_timeout(__u32 latency_us
, __u32 latency_stddev_us
,
2009 __u32 max_remote_ack_delay_us
)
2011 unsigned long addto
;
2012 if (unlikely(unlikely(latency_us
> 1000000000) ||
2013 unlikely(latency_stddev_us
> 500000000) ||
2014 unlikely(max_remote_ack_delay_us
> 1000000000))) {
2015 addto
= msecs_to_jiffies(latency_us
/1000 + latency_us
/4000 +
2016 latency_stddev_us
/333 +
2017 max_remote_ack_delay_us
/1000);
2019 addto
= usecs_to_jiffies(latency_us
+ latency_us
/4 +
2020 latency_stddev_us
*3 + max_remote_ack_delay_us
);
2024 * 2 is added because
2025 * 1) _to_jiffies rounds down, but should round up, so add 1 to
2027 * 2) even if latency is 0, we never want to schedule the retransmit
2028 * to run right now, so add 1 more
2030 return jiffies
+ 2 + addto
;
2033 static inline void cor_put_be64(char *dst
, __be64 value
)
2035 char *p_value
= (char *) &value
;
2037 dst
[0] = p_value
[0];
2038 dst
[1] = p_value
[1];
2039 dst
[2] = p_value
[2];
2040 dst
[3] = p_value
[3];
2041 dst
[4] = p_value
[4];
2042 dst
[5] = p_value
[5];
2043 dst
[6] = p_value
[6];
2044 dst
[7] = p_value
[7];
2047 static inline void cor_put_u64(char *dst
, __u64 value
)
2049 cor_put_be64(dst
, cpu_to_be64(value
));
2052 static inline void cor_put_u48(char *dst
, __u64 value
)
2054 char *p_value
= (char *) &value
;
2056 value
= cpu_to_be64(value
);
2058 dst
[0] = p_value
[2];
2059 dst
[1] = p_value
[3];
2060 dst
[2] = p_value
[4];
2061 dst
[3] = p_value
[5];
2062 dst
[4] = p_value
[6];
2063 dst
[5] = p_value
[7];
2066 static inline void cor_put_be32(char *dst
, __be32 value
)
2068 char *p_value
= (char *) &value
;
2069 dst
[0] = p_value
[0];
2070 dst
[1] = p_value
[1];
2071 dst
[2] = p_value
[2];
2072 dst
[3] = p_value
[3];
2075 static inline void cor_put_u32(char *dst
, __u32 value
)
2077 cor_put_be32(dst
, cpu_to_be32(value
));
2080 static inline void cor_put_be16(char *dst
, __be16 value
)
2082 char *p_value
= (char *) &value
;
2083 dst
[0] = p_value
[0];
2084 dst
[1] = p_value
[1];
2087 static inline void cor_put_u16(char *dst
, __u16 value
)
2089 cor_put_be16(dst
, cpu_to_be16(value
));
2092 static inline char *cor_pull_skb(struct sk_buff
*skb
, unsigned int len
)
2094 char *ptr
= skb_pull(skb
, len
);
2096 if (unlikely(ptr
== 0))
2102 static inline __be64
cor_parse_be64(char *buf
)
2108 ((char *)&ret
)[0] = buf
[0];
2109 ((char *)&ret
)[1] = buf
[1];
2110 ((char *)&ret
)[2] = buf
[2];
2111 ((char *)&ret
)[3] = buf
[3];
2112 ((char *)&ret
)[4] = buf
[4];
2113 ((char *)&ret
)[5] = buf
[5];
2114 ((char *)&ret
)[6] = buf
[6];
2115 ((char *)&ret
)[7] = buf
[7];
2120 static inline __u64
cor_parse_u64(char *buf
)
2122 return be64_to_cpu(cor_parse_be64(buf
));
2125 static inline __u64
cor_parse_u48(char *ptr
)
2129 ((char *)&ret
)[0] = 0;
2130 ((char *)&ret
)[1] = 0;
2131 ((char *)&ret
)[2] = ptr
[0];
2132 ((char *)&ret
)[3] = ptr
[1];
2133 ((char *)&ret
)[4] = ptr
[2];
2134 ((char *)&ret
)[5] = ptr
[3];
2135 ((char *)&ret
)[6] = ptr
[4];
2136 ((char *)&ret
)[7] = ptr
[5];
2138 return be64_to_cpu(ret
);
2141 static inline __be32
cor_parse_be32(char *ptr
)
2147 ((char *)&ret
)[0] = ptr
[0];
2148 ((char *)&ret
)[1] = ptr
[1];
2149 ((char *)&ret
)[2] = ptr
[2];
2150 ((char *)&ret
)[3] = ptr
[3];
2155 static inline __u32
cor_parse_u32(char *ptr
)
2157 return be32_to_cpu(cor_parse_be32(ptr
));
2160 static inline __be16
cor_parse_be16(char *ptr
)
2166 ((char *)&ret
)[0] = ptr
[0];
2167 ((char *)&ret
)[1] = ptr
[1];
2172 static inline __u16
cor_parse_u16(char *ptr
)
2174 return be16_to_cpu(cor_parse_be16(ptr
));
2177 static inline __u8
cor_parse_u8(char *ptr
)
2180 return (__u8
) ptr
[0];
2183 static inline __u64
cor_pull_u48(struct sk_buff
*skb
)
2185 return cor_parse_u48(cor_pull_skb(skb
, 6));
2188 static inline __be32
cor_pull_be32(struct sk_buff
*skb
)
2190 return cor_parse_be32(cor_pull_skb(skb
, 4));
2193 static inline __u32
cor_pull_u32(struct sk_buff
*skb
)
2195 return cor_parse_u32(cor_pull_skb(skb
, 4));
2198 static inline __u16
cor_pull_u16(struct sk_buff
*skb
)
2200 return cor_parse_u16(cor_pull_skb(skb
, 2));
2203 static inline __u8
cor_pull_u8(struct sk_buff
*skb
)
2205 char *ptr
= cor_pull_skb(skb
, 1);
2210 static inline int cor_is_conn_in(struct cor_conn
*cn_l
, struct cor_neighbor
*nb
,
2213 if (unlikely(unlikely(cn_l
->sourcetype
!= SOURCE_IN
) ||
2214 unlikely(cn_l
->source
.in
.nb
!= nb
) ||
2215 unlikely(cn_l
->source
.in
.conn_id
!= conn_id
) ||
2216 unlikely(cn_l
->isreset
!= 0)))
2221 static inline int cor_is_src_sock(struct cor_conn
*cn_l
, struct cor_sock
*cs
)
2223 if (unlikely(unlikely(cn_l
->sourcetype
!= SOURCE_SOCK
) ||
2224 unlikely(cn_l
->source
.sock
.ed
->cs
!= cs
)))
2229 static inline int cor_is_trgt_sock(struct cor_conn
*cn_l
, struct cor_sock
*cs
)
2231 if (unlikely(unlikely(cn_l
->targettype
!= TARGET_SOCK
) ||
2232 unlikely(cn_l
->target
.sock
.cs
!= cs
)))
2237 #define BUFLEN_MIN 128
2238 #define BUFLEN_MAX 4096
2239 #define PAGESIZE (1 << PAGE_SHIFT)
2241 static inline __u32
cor_buf_optlen(__u32 datalen
, int from_sock
)
2243 __u32 optlen
= BUFLEN_MIN
;
2248 while (optlen
< datalen
&& optlen
< PAGESIZE
&& optlen
< BUFLEN_MAX
)
2249 optlen
= (optlen
<< 1);
2254 inline static void cor_databuf_item_free(struct cor_data_buf_item
*item
)
2256 if (item
->type
== DATABUF_BUF
) {
2258 kmem_cache_free(cor_data_buf_item_slab
, item
);
2259 } else if (item
->type
== DATABUF_SKB
) {
2260 struct sk_buff
*skb
= cor_skb_from_pstate(container_of(item
,
2261 struct cor_skb_procstate
, funcstate
.rcv
.dbi
));
2268 static inline __u64
cor_seqno_clean(__u64 seqno
)
2270 return seqno
& ((1LL << 48) - 1);
2273 static inline int cor_seqno_eq(__u64 seqno1
, __u64 seqno2
)
2275 seqno1
= seqno1
<< 16;
2276 seqno2
= seqno2
<< 16;
2277 return seqno1
== seqno2
;
2280 static inline int cor_seqno_before(__u64 seqno1
, __u64 seqno2
)
2282 seqno1
= seqno1
<< 16;
2283 seqno2
= seqno2
<< 16;
2284 return (seqno1
- seqno2
) >= (1LL << 63);
2287 static inline int cor_seqno_before_eq(__u64 seqno1
, __u64 seqno2
)
2289 return cor_seqno_eq(seqno1
, seqno2
) || cor_seqno_before(seqno1
, seqno2
);
2292 static inline int cor_seqno_after(__u64 seqno1
, __u64 seqno2
)
2294 return cor_seqno_before_eq(seqno1
, seqno2
) ? 0 : 1;
2297 static inline int cor_seqno_after_eq(__u64 seqno1
, __u64 seqno2
)
2299 return cor_seqno_before(seqno1
, seqno2
) ? 0 : 1;
2302 static inline int ktime_before_eq(ktime_t time1
, ktime_t time2
)
2304 return ktime_after(time1
, time2
) ? 0 : 1;
2307 static inline int ktime_after_eq(ktime_t time1
, ktime_t time2
)
2309 return ktime_before(time1
, time2
) ? 0 : 1;
2312 static inline __u64
cor_update_atomic_sum(atomic64_t
*atomic_sum
, __u32 oldvalue
,
2315 __u64 sum_old
= atomic64_read(atomic_sum
);
2323 BUG_ON(sum
< oldvalue
);
2326 BUG_ON(sum
+ newvalue
< sum
);
2329 cmpxchg_ret
= atomic64_cmpxchg(atomic_sum
, sum_old
, sum
);
2331 if (likely(cmpxchg_ret
== sum_old
))
2334 sum_old
= cmpxchg_ret
;
2340 static inline void cor_sk_write_space(struct cor_sock
*cs
)
2342 atomic_set(&(cs
->ready_to_write
), 1);
2344 cs
->sk
.sk_write_space(&(cs
->sk
));
2347 static inline void cor_sk_data_ready(struct cor_sock
*cs
)
2349 atomic_set(&(cs
->ready_to_read
), 1);
2351 cs
->sk
.sk_data_ready(&(cs
->sk
));
2354 /* the other direction may be locked only if called from cor_proc_cpacket */
2355 static inline void cor_flush_buf(struct cor_conn
*cn_lx
)
2357 if (unlikely(cn_lx
->targettype
== TARGET_UNCONNECTED
)) {
2358 cor_proc_cpacket(cn_lx
);
2359 } else if (cn_lx
->targettype
== TARGET_SOCK
) {
2360 cor_flush_sock(cn_lx
);
2361 } else if (cn_lx
->targettype
== TARGET_OUT
) {
2362 __u32 bytessent
= 0;
2363 cor_flush_out(cn_lx
, &bytessent
);
2364 } else if (unlikely(cn_lx
->targettype
== TARGET_DISCARD
)) {
2365 cor_databuf_ackdiscard(cn_lx
);