2 * Connection oriented routing
3 * Copyright (C) 2007-2008 Michael Blizek
5 * This program is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU General Public License
7 * as published by the Free Software Foundation; either version 2
8 * of the License, or (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
21 #include <asm/atomic.h>
23 #include <linux/types.h>
24 #include <linux/netdevice.h>
25 #include <linux/skbuff.h>
26 #include <linux/spinlock.h>
27 #include <linux/workqueue.h>
28 #include <linux/kref.h>
34 #define PIDOUT_NEWCONN 16
35 #define PIDOUT_SENDDEF_THRES 8
36 #define PIDOUT_SENDDEF_COUNT 16
40 #define ETH_P_COR 0x1022
44 #define SOCKADDRTYPE_PORT 1
53 #define MAX_CONN_CMD_LEN 4096
56 #define PACKET_TYPE_ANNOUNCE 1
57 #define PACKET_TYPE_DATA 2
60 * Kernel packet data - these commands are sent by the neighbor
61 * The end nodes may cause these commands to be sent, but they see them beyond
69 * KP_PING[1] cookie[4]
70 * KP_PONG[1] cookie[4] respdelay[4]
72 * This is needed to find out whether the other node is reachable. After a new
73 * neighbor is seen, ping requests are sent and the neighbor is only reachable
74 * after a few pongs are received. These requests are also used to find out
75 * whether a neighber is gone.
78 * The receiver of a ping may delay the sending of the pong e.g. to create
79 * bigger kernel packets. The respdelay is the time in microseconds the packet
85 /* KP_ACK[1] seqno[4] */
89 * KP_ACK_CONN[1] conn_id[4] seqno[4] window[1]
90 * KP_ACK_CONN_OOO[1] conn_id[4] seqno[4] window[1] seqno_ooo[4] length[4]
92 * conn_id is the conn_id we use if we sent something through this conn and
93 * *not* the conn_id that the neighbor used to send us the data
95 * seqno = the seqno which is expected in the next non-out-of-order packet
96 * seqno_ooo, length = in case
98 * window = amount of data which can be sent without receiving the next ack
99 * packets with lower seqno do not overwrite the last window size
100 * note: the other side may also reduce the window size
103 * 1...255 = 64*2^((value-1)/11) end result is rounded down to an integer
106 #define KP_ACK_CONN 5
107 #define KP_ACK_CONN_OOO 6
110 * NOTE on connection ids:
111 * connection ids we send are used for the receive channel
112 * connection ids we receive are used for the send channel
116 * incoming connection
117 * KP_CONNECT[1] conn_id[4]
122 * incoming connection successful,
123 * the first conn_id is the same as previously sent/received in KP_CONNECT
124 * the second conn_id is generated by us and used for the other direction
125 * KP_CONNECT_SUCCESS[1] conn_id[4] conn_id[4]
127 #define KP_CONNECT_SUCCESS 8
129 /* KP_CONN_DATA[1] conn_id[4] seqno[4] length[2] data[length] */
130 #define KP_CONN_DATA 9
133 * KP_PING_CONN[1] conn_id[4]
135 * This is for querying the status of an open connection. The response is either
136 * KP_ACK_CONN or CONNID_UNKNOWN
138 #define KP_PING_CONN 10
141 * { KP_RESET_CONN[1] conn_id[4] }
142 * We send this, if there is an established connection we want to close.
144 #define KP_RESET_CONN 11
147 * KP_CONNID_UNKNOWN[1] sent_conn_id[4]
148 * We send this, if we receive an invalid conn_id
150 #define KP_CONNID_UNKNOWN 12
153 * KP_PING_ALL_CONNS[1]
154 * We send this, if we we lost a conn, but could not send reset_conn and
157 #define KP_PING_ALL_CONNS 13
160 * KP_SET_MAX_CMSG_DELAY[1] delay[4]
161 * Sent after connecting and at any change
162 * delay in specifies in microsecs
164 #define KP_SET_MAX_CMSG_DELAY 14
167 * KP_SET_CREDITS[1] credits[8] rate_initial[4] rate_earning[4] rate_spending[4]
169 #define KP_SET_CREDITS 15
172 * KP_SET_CONN_CREDITS[1] conn_id[4] credit_rate[4]
174 #define KP_SET_CONN_CREDITS 16
178 * Connection data which in interpreted when connection has no target yet
179 * These commands are sent by the end node.
182 * cmd[2] length[4] parameter[length]
183 * unrecogniced commands are ignored
184 * parameters which are longer than expected are ignored as well
187 /* outgoing connection: CD_CONNECT_NB[2] length[4]
188 * addrtypelen[2] addrlen[2] addrtype[addrtypelen] addr[addrlen] */
189 #define CD_CONNECT_NB 1
191 /* connection to local open part: CD_CONNECT_PORT[2] length[4] port[8] */
192 #define CD_CONNECT_PORT 2
195 * CD_LIST_NEIGH sends CDR_BINDATA if the command was successful. The response
198 * totalneighs[4] response_rows[4]
200 * numaddr[2] (addrtypelen[2] addrlen[2] addrtype[addrtypelen] addr[addrlen]
203 * Neighbors have to be sorted by uptime, new neighbors first. This is so that
204 * the routing daemon can easily find out whether there are new neighbors. It
205 * only needs to send a query with offset 0. If the totalneighs stays the same
206 * while new were added, a connection to another neighbor was lost.
209 /* list connected neighbors: CD_LIST_NEIGH[2] length[4] limit[4] offset[4] */
210 #define CD_LIST_NEIGH 3
213 * CD_SET_(FORWARD|BACKWARD)_TIMEOUT[2] length[4] timeout_ms[4]
215 * If there is no successful communication with the previous or neighbor for
216 * this period, the connection will be reset. This value must be between
217 * NB_STALL_TIME and NB_KILL_TIME. Otherwise it will silently behave as if it
218 * was set to exactly one of these limits.
220 #define CD_SET_FORWARD_TIMEOUT 4
221 #define CD_SET_BACKWARD_TIMEOUT 5
224 * Connection data response
225 * Format is the same as with connection data
229 * {CDR_EXECOK[2] || CDR_EXECFAILED[2]}
230 * reasoncode[2] reasontextlength[2] reasontext[reasontextlength]
231 * reasontextlength may be 0
233 #define CDR_EXECOK 32768
234 #define CDR_EXECOK_OK 33024
237 #define CDR_EXECFAILED 32769
238 #define CDR_EXECFAILED_UNKNOWN_COMMAND 33280
239 #define CDR_EXECFAILED_PERMISSION_DENIED 33281
240 #define CDR_EXECFAILED_TEMPORARILY_OUT_OF_RESSOURCES 33282
241 #define CDR_EXECFAILED_CMD_TOO_SHORT 33283
242 #define CDR_EXECFAILED_CMD_TOO_LONG 33284
243 #define CDR_EXECFAILED_TARGETADDRTYPE_UNKNOWN 33285
244 #define CDR_EXECFAILED_TARGETADDR_DOESNTEXIST 33286
245 #define CDR_EXECFAILED_TARGETADDR_PORTCLOSED 33287
246 #define CDR_EXECFAILED_LISTENERQUEUE_FULL 33288
247 #define CDR_EXECFAILED_ILLEGAL_COMMAND 33289
250 * must be sent after CDR_EXEC{OK|FAILED}
251 * CDR_EXEOK_BINDATA[2] bindatalen[4] bindata[bindatalen] */
252 #define CDR_BINDATA 32770
255 /* result codes for rcv.c/proc_packet */
257 #define RC_FINISHED 1
259 #define RC_RCV1_ANNOUNCE 2
260 #define RC_RCV1_KERNEL 3
261 #define RC_RCV1_CONN 4
264 /* start of next element, *not* next htab_entry */
269 struct htab_entry
**htable
;
274 int (*matches
)(void *htentry
, void *searcheditem
);
285 struct announce_data
{
289 struct net_device
*dev
;
290 struct delayed_work announce_work
;
291 struct announce
*ann
;
292 struct resume_block rb
;
294 __u32 curr_announce_msg_offset
;
295 __u64 scheduled_announce_timer
;
301 __u8 pongs
; /* count of pongs for pings sent after this one */
304 #define NEIGHBOR_STATE_INITIAL 0
305 #define NEIGHBOR_STATE_ACTIVE 1
306 #define NEIGHBOR_STATE_STALLED 2
307 #define NEIGHBOR_STATE_KILLED 3
310 struct list_head nb_list
;
314 struct net_device
*dev
;
315 char mac
[MAX_ADDR_LEN
];
320 struct delayed_work cmsg_timer
;
321 struct mutex cmsg_lock
;
322 struct list_head control_msgs_out
;
324 * urgent messages; These are sent even if the neighbor state is not
325 * active. If the queue gets full, the oldest ones are dropped. It thus
326 * may only contain messages which are allowed to be dropped.
328 struct list_head ucontrol_msgs_out
;
333 atomic_t cmcnt
; /* size of queue + retransmits */
334 atomic_t ucmcnt
; /* size of queue only */
338 __u8 max_cmsg_delay_sent
;
340 /* see snd.c/qos_queue */
341 /* protected by cmsg_lock */
344 /* procected by queues_lock */
345 struct resume_block rb_kp
;
346 struct resume_block rb_cr
;
348 struct mutex pingcookie_lock
;
349 unsigned long last_ping_time
;
350 __u32 ping_intransit
;
351 struct ping_cookie cookies
[PING_COOKIES_PER_NEIGH
];
353 atomic_t latency
; /* microsecs */
354 atomic_t max_remote_cmsg_delay
; /* microsecs */
356 spinlock_t state_lock
;
358 __u64 last_state_change
;/* initial state */
361 * time of the last sent packet which has been acked or
362 * otherwise responded to (e.g. pong)
364 unsigned long last_roundtrip
;/* active/stalled state */
369 struct delayed_work stalltimeout_timer
;
370 __u8 str_timer_pending
;
373 atomic_t kpacket_seqno
;
374 atomic_t ooo_packets
;
376 spinlock_t credits_lock
;
377 long jiffies_credit_update
;
379 /* all cretid rates are in credits/ms */
380 __u64 credits
; /* how much we can spend */
382 __u64 credits_diff
; /* diff between the neighbor's and our calc */
383 __u32 credits_diff_fract
;
384 __s32 creditrate_initial
;
385 __u32 creditrate_earning
;
386 __u32 creditrate_spending
;
387 __u32 creditrate_spending_expected
;
389 __s32 creditrate_spending_diff
;
391 __u64 debits
; /* how much the other side can spend */
393 __s32 debitrate_initial
;
394 __s32 debitrate_initial_adj
;
395 __u32 debitrate_earning
;
396 __u32 debitrate_spending
;
401 * connecions which receive data from/send data to this node
402 * used when terminating all connections of a neighbor
404 struct mutex conn_list_lock
;
405 struct list_head rcv_conn_list
;
406 struct list_head snd_conn_list
;
407 __u32 num_send_conns
;
410 * used for ping_all conns, if not zero this is the next conn we need to
411 * ping, protected by conn_list_lock
413 struct conn
*next_ping_conn
;
414 __u32 ping_conns_remaining
;
415 __u32 ping_conns_retrans_remaining
;
416 __u32 pong_conns_expected
;
417 unsigned long ping_conn_completed
; /* jiffies */
420 * the timer has to be inited when adding the neighbor
421 * init_timer(struct timer_list * timer);
422 * add_timer(struct timer_list * timer);
424 spinlock_t retrans_lock
;
425 struct delayed_work retrans_timer_conn
;
426 struct delayed_work retrans_timer
;
427 __u8 retrans_timer_conn_running
;
428 __u8 retrans_timer_running
;
430 struct list_head retrans_list
;
431 struct list_head retrans_list_conn
;
433 struct conn
*firstboundconn
;
436 struct cor_sched_data
{
438 struct list_head conn_list
;
439 struct sk_buff_head requeue_queue
;
445 struct data_buf_item
{
446 struct list_head buf_list
;
462 struct list_head items
;
463 struct data_buf_item
*lastread
;
467 __u32 read_remaining
;
469 __u16 last_read_offset
;
478 struct connlistener
*owner
;
482 #define SOCKSTATE_LISTENER 1
483 #define SOCKSTATE_CONN 2
486 /* The first member of connlistener/conn (see sock.c) */
490 struct connlistener
{
491 /* The first member has to be the same as in conn (see sock.c) */
497 struct list_head conn_queue
;
498 wait_queue_head_t wait
;
503 * There are 2 conn objects per bi-directional connection. They refer to each
504 * other with in the reversedir field. To distinguish them, the variables on
505 * the stack are usually called rconn and sconn. rconn refers to the conn object
506 * which has received a command. sconn is the other conn object. This means that
507 * in send functions rconn means the connection we want to send the command to.
511 /* The first member has to be the same as in connlistener (see sock.c)*/
514 #define SOURCE_NONE 0
516 #define SOURCE_SOCK 2
518 #define TARGET_UNCONNECTED 0
520 #define TARGET_SOCK 2
527 * 0... connection active
528 * 1... connection is about to be reset, target does not need to be
530 * 2... connection is reset
531 * 3... connection is reset + no pointers to "struct conn *reversedir"
532 * remaining except from this conn
536 struct list_head queue_list
;
542 * If one side is SOCK or NONE/UNCONNECTED and both directions
543 * need to be locked, the direction with TARGET_UNCONNECTED or
544 * TARGET_SOCK has to be locked first, the direction with
545 * SOURCE_NONE or SOURCE_SOCK afterwards. This is needed for changing
546 * source/targettype and credit flow.
547 * If data is forwarded, (both sides are IN/OUT), only one direction
550 struct mutex rcv_lock
;
552 long jiffies_credit_update
;
556 /* credit rates, locked by credit_lock (in credit.c) */
563 /* list of all connections from this neighbor */
564 struct list_head nb_list
;
566 struct sk_buff_head reorder_queue
;
568 struct htab_entry htab_entry
;
573 atomic_t pong_awaiting
;
580 struct list_head cl_list
;
581 wait_queue_head_t wait
;
596 __u32 stall_timeout_ms
;
600 /* has to be first (because it is first in target
604 /* list of all connections to this neighbor */
605 struct list_head nb_list
;
606 /* protected by nb->retrans_lock, sorted by seqno */
607 struct list_head retrans_list
;
609 /* reverse conn_id lookup */
610 struct htab_entry htab_entry
;
613 __u32 seqno_nextsend
;
615 __u32 seqno_windowlimit
;
616 __u32 kp_windowsetseqno
;
618 struct resume_block rb
;
620 __u32 stall_timeout_ms
;
627 wait_queue_head_t wait
;
635 struct conn
*reversedir
;
639 struct skb_procstate
{
642 struct work_struct work
;
657 extern __u8
enc_window(__u32 window_bytes
);
659 extern __u32
dec_window(__u8 window
);
661 extern char *htable_get(struct htable
*ht
, __u32 key
, void *searcheditem
);
663 extern int htable_delete(struct htable
*ht
, __u32 key
, void *searcheditem
,
664 void (*free
) (struct kref
*ref
));
666 extern void htable_insert(struct htable
*ht
, char *newelement
, __u32 key
);
668 extern void htable_init(struct htable
*ht
, int (*matches
)(void *htentry
,
669 void *searcheditem
), __u32 entry_offset
,
672 extern struct conn
*get_conn_reverse(struct neighbor
*nb
, __u32 conn_id
);
674 extern void insert_reverse_connid(struct conn
*rconn
);
676 extern struct conn
*get_conn(__u32 conn_id
);
678 extern void free_conn(struct kref
*ref
);
680 extern int conn_init_out(struct conn
*rconn
, struct neighbor
*nb
);
682 extern void conn_init_sock_source(struct conn
*conn
);
684 extern void conn_init_sock_target(struct conn
*conn
);
686 extern void close_port(struct connlistener
*listener
);
688 extern struct connlistener
*open_port(__be64 port
);
690 extern int connect_port(struct conn
*rconn
, __be64 port
);
692 extern int connect_neigh(struct conn
*rconn
,
693 __u16 addrtypelen
, __u8
*addrtype
,
694 __u16 addrlen
, __u8
*addr
);
696 extern struct conn
* alloc_conn(gfp_t allocflags
);
698 extern void reset_conn(struct conn
*conn
);
701 extern int refresh_credits_state(struct neighbor
*nb
);
703 extern void check_credit_state(struct neighbor
*nb
);
705 extern int debit_adj_needed(struct neighbor
*nb
);
707 extern void refresh_conn_credits(struct conn
*conn
);
709 extern void set_credits(struct neighbor
*nb
, __u64 credits
,
710 __s32 creditrate_initial
, __u32 creditrate_earning
,
711 __u32 creditrate_spending
);
713 extern void set_debitrate_initial(struct neighbor
*nb
, __u32 debitrate
);
715 extern void set_conn_in_crate(struct conn
*rconn
, __u32 crate_in
);
718 extern void neighbor_free(struct kref
*ref
);
720 extern struct neighbor
*get_neigh_by_mac(struct sk_buff
*skb
);
722 extern struct neighbor
*find_neigh(__u16 addrtypelen
, __u8
*addrtype
,
723 __u16 addrlen
, __u8
*addr
);
725 extern __u32
generate_neigh_list(char *buf
, __u32 buflen
, __u32 limit
,
728 extern int get_neigh_state(struct neighbor
*nb
);
730 extern void ping_resp(struct neighbor
*nb
, __u32 cookie
, __u32 respdelay
);
732 extern __u32
add_ping_req(struct neighbor
*nb
);
734 extern void unadd_ping_req(struct neighbor
*nb
, __u32 cookie
);
736 extern int time_to_send_ping(struct neighbor
*nb
);
738 extern int force_ping(struct neighbor
*nb
);
740 extern void rcv_announce(struct sk_buff
*skb
);
742 extern int send_announce_qos(struct announce_data
*ann
);
744 extern void announce_data_free(struct kref
*ref
);
746 extern int __init
cor_neighbor_init(void);
749 extern void drain_ooo_queue(struct conn
*rconn
);
751 extern void conn_rcv_buildskb(char *data
, __u32 datalen
, __u32 conn_id
,
754 extern int __init
cor_rcv_init(void);
756 /* kpacket_parse.c */
757 extern void kernel_packet(struct neighbor
*nb
, struct sk_buff
*skb
, __u32 seqno
);
760 extern void schedule_controlmsg_timerfunc(struct neighbor
*nb
);
762 struct control_msg_out
;
764 #define ACM_PRIORITY_LOW 1
765 #define ACM_PRIORITY_MED 2
766 #define ACM_PRIORITY_HIGH 3
768 extern struct control_msg_out
*alloc_control_msg(struct neighbor
*nb
,
771 extern void free_control_msg(struct control_msg_out
*cm
);
773 extern void retransmit_timerfunc(struct work_struct
*work
);
775 extern void kern_ack_rcvd(struct neighbor
*nb
, __u32 seqno
);
777 extern int resume_send_messages(struct neighbor
*nb
);
779 extern void send_pong(struct neighbor
*nb
,
782 extern void send_reset_conn(struct control_msg_out
*cm
, __u32 conn_id
);
784 extern void send_ack(struct neighbor
*nb
,
787 extern void send_ack_conn(struct control_msg_out
*cm
, __u32 conn_id
,
788 __u32 seqno
, __u8 window
);
790 extern void send_ack_conn_ooo(struct control_msg_out
*cm
, __u32 conn_id
,
791 __u32 seqno
, __u8 window
, __u32 seqno_ooo
, __u32 length
);
793 extern void send_connect_success(struct control_msg_out
*cm
, __u32 rcvd_conn_id
,
796 extern void send_connect_nb(struct control_msg_out
*cm
, __u32 conn_id
);
798 extern void send_conndata(struct control_msg_out
*cm
, __u32 conn_id
,
799 __u32 seqno
, char *data_orig
, char *data
, __u32 datalen
);
801 extern void send_ping_conn(struct control_msg_out
*cm
, __u32 conn_id
);
803 extern void send_connid_unknown(struct control_msg_out
*cm
, __u32 conn_id
);
805 extern void send_ping_all_conns(struct neighbor
*nb
);
807 extern void send_credits(struct neighbor
*nb
);
809 extern void cor_kgen_init(void);
811 /* cpacket_parse.c */
812 extern void parse(struct conn
*rconn
);
815 extern int destroy_queue(struct net_device
*dev
);
817 extern int create_queue(struct net_device
*dev
);
819 #define QOS_CALLER_KPACKET 0
820 #define QOS_CALLER_CONN_RETRANS 1
821 #define QOS_CALLER_ANNOUNCE 2
822 #define QOS_CALLER_CONN 3
824 extern void qos_enqueue(struct net_device
*dev
, struct resume_block
*rb
,
827 extern void qos_enqueue_kpacket(struct neighbor
*nb
);
829 extern struct sk_buff
*create_packet(struct neighbor
*nb
, int size
,
830 gfp_t alloc_flags
, __u32 conn_id
, __u32 seqno
);
832 extern void cancel_retrans(struct conn
*rconn
);
834 extern void retransmit_conn_timerfunc(struct work_struct
*work
);
836 extern void conn_ack_rcvd(__u32 kpacket_seqno
, struct conn
*rconn
, __u32 seqno
,
837 __u8 window
, __u32 seqno_ooo
, __u32 length
);
839 extern void flush_out(struct conn
*rconn
);
841 extern int __init
cor_snd_init(void);
844 extern void databuf_pull(struct data_buf
*data
, char *dst
, int len
);
846 extern size_t databuf_pulluser(struct conn
*sconn
, struct msghdr
*msg
);
848 extern void databuf_unpull(struct data_buf
*data
, __u32 bytes
);
850 extern void databuf_pullold(struct data_buf
*data
, __u32 startpos
, char *dst
,
853 extern void databuf_ack(struct data_buf
*buf
, __u32 pos
);
855 extern void databuf_ackread(struct data_buf
*buf
);
857 extern int databuf_maypush(struct data_buf
*buf
);
859 extern void reset_seqno(struct data_buf
*buf
);
861 extern void databuf_free(struct data_buf
*data
);
863 extern void databuf_init(struct data_buf
*data
);
865 extern int receive_userbuf(struct conn
*rconn
, struct msghdr
*msg
);
867 extern void receive_buf(struct conn
*rconn
, char *buf
, int len
);
869 extern int receive_skb(struct conn
*rconn
, struct sk_buff
*skb
);
871 extern void wake_sender(struct conn
*rconn
);
873 extern void forward_init(void);
876 static inline struct skb_procstate
*skb_pstate(struct sk_buff
*skb
)
878 return (struct skb_procstate
*) &(skb
->cb
[0]);
881 static inline struct sk_buff
*skb_from_pstate(struct skb_procstate
*ps
)
883 return (struct sk_buff
*) (((char *)ps
) - offsetof(struct sk_buff
,cb
));
887 static inline __u32
mss(struct neighbor
*nb
)
889 return nb
->dev
->mtu
- LL_RESERVED_SPACE(nb
->dev
) - 9;
893 static inline void put_u64(char *dst
, __u64 value
, int convbo
)
895 char *p_value
= (char *) &value
;
898 value
= cpu_to_be64(value
);
910 static inline void put_u32(char *dst
, __u32 value
, int convbo
)
912 char *p_value
= (char *) &value
;
915 value
= cpu_to_be32(value
);
923 static inline void put_u16(char *dst
, __u16 value
, int convbo
)
925 char *p_value
= (char *) &value
;
928 value
= cpu_to_be16(value
);
934 static inline char *cor_pull_skb(struct sk_buff
*skb
, unsigned int len
)
936 char *ptr
= skb_pull(skb
, len
);
938 if(unlikely(ptr
== 0))
945 static inline __u64
mul_saturated(__u64 a
, __u64 b
)
953 static inline int numdigits(__u64 value
)
956 for (;value
!= 0;value
= (value
>> 1)) {
962 /* approximate (a*b) / c without overflowing a*b */
963 static inline __u64
multiply_div(__u64 a
, __u64 b
, __u64 c
)
965 int alen
= numdigits(a
);
966 int blen
= numdigits(b
);
967 int clen
= numdigits(c
);
969 BUG_ON(alen
< 0 || alen
> 64);
970 BUG_ON(blen
< 0 || blen
> 64);
971 BUG_ON(clen
< 0 || clen
> 64);
973 BUG_ON((a
== 0 && alen
!= 0) || (a
!= 0 && alen
== 0));
974 BUG_ON((b
== 0 && blen
!= 0) || (b
!= 0 && blen
== 0));
975 BUG_ON((c
== 0 && clen
!= 0) || (c
!= 0 && clen
== 0));
977 BUG_ON(a
>= b
&& alen
< blen
);
978 BUG_ON(a
>= c
&& alen
< clen
);
979 BUG_ON(b
>= a
&& blen
< alen
);
980 BUG_ON(b
>= c
&& blen
< clen
);
981 BUG_ON(c
>= a
&& clen
< alen
);
982 BUG_ON(c
>= b
&& clen
< blen
);
984 if (alen
== 0 || blen
== 0)
990 if (alen
+ blen
<= 64)
993 if (a
>= b
&& alen
> clen
+ 16)
994 return mul_saturated(a
/c
, b
);
995 else if (a
< b
&& blen
> clen
+ 16)
996 return mul_saturated(b
/c
, a
);
998 while (alen
+ blen
> 64) {
999 if (alen
> blen
|| (alen
== blen
&& a
> b
)) {