blob: 95c425ca97b6100f7809019d8e47cde6174df3d3 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
|
/*
* Kernel Connection Multiplexor
*
* Copyright (c) 2016 Tom Herbert <tom@herbertland.com>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2
* as published by the Free Software Foundation.
*/
#ifndef __NET_KCM_H_
#define __NET_KCM_H_
#include <linux/skbuff.h>
#include <net/sock.h>
#include <uapi/linux/kcm.h>
extern unsigned int kcm_net_id;
#define KCM_STATS_ADD(stat, count) ((stat) += (count))
#define KCM_STATS_INCR(stat) ((stat)++)
struct kcm_psock_stats {
unsigned long long rx_msgs;
unsigned long long rx_bytes;
unsigned long long tx_msgs;
unsigned long long tx_bytes;
unsigned int rx_aborts;
unsigned int rx_mem_fail;
unsigned int rx_need_more_hdr;
unsigned int rx_msg_too_big;
unsigned int rx_msg_timeouts;
unsigned int rx_bad_hdr_len;
unsigned long long reserved;
unsigned long long unreserved;
unsigned int tx_aborts;
};
struct kcm_mux_stats {
unsigned long long rx_msgs;
unsigned long long rx_bytes;
unsigned long long tx_msgs;
unsigned long long tx_bytes;
unsigned int rx_ready_drops;
unsigned int tx_retries;
unsigned int psock_attach;
unsigned int psock_unattach_rsvd;
unsigned int psock_unattach;
};
struct kcm_stats {
unsigned long long rx_msgs;
unsigned long long rx_bytes;
unsigned long long tx_msgs;
unsigned long long tx_bytes;
};
struct kcm_tx_msg {
unsigned int sent;
unsigned int fragidx;
unsigned int frag_offset;
unsigned int msg_flags;
struct sk_buff *frag_skb;
struct sk_buff *last_skb;
};
struct kcm_rx_msg {
int full_len;
int accum_len;
int offset;
int early_eaten;
};
/* Socket structure for KCM client sockets */
struct kcm_sock {
struct sock sk;
struct kcm_mux *mux;
struct list_head kcm_sock_list;
int index;
u32 done : 1;
struct work_struct done_work;
struct kcm_stats stats;
/* Transmit */
struct kcm_psock *tx_psock;
struct work_struct tx_work;
struct list_head wait_psock_list;
struct sk_buff *seq_skb;
/* Don't use bit fields here, these are set under different locks */
bool tx_wait;
bool tx_wait_more;
/* Receive */
struct kcm_psock *rx_psock;
struct list_head wait_rx_list; /* KCMs waiting for receiving */
bool rx_wait;
u32 rx_disabled : 1;
};
struct bpf_prog;
/* Structure for an attached lower socket */
struct kcm_psock {
struct sock *sk;
struct kcm_mux *mux;
int index;
u32 tx_stopped : 1;
u32 rx_stopped : 1;
u32 done : 1;
u32 unattaching : 1;
void (*save_state_change)(struct sock *sk);
void (*save_data_ready)(struct sock *sk);
void (*save_write_space)(struct sock *sk);
struct list_head psock_list;
struct kcm_psock_stats stats;
/* Receive */
struct sk_buff *rx_skb_head;
struct sk_buff **rx_skb_nextp;
struct sk_buff *ready_rx_msg;
struct list_head psock_ready_list;
struct work_struct rx_work;
struct delayed_work rx_delayed_work;
struct bpf_prog *bpf_prog;
struct kcm_sock *rx_kcm;
unsigned long long saved_rx_bytes;
unsigned long long saved_rx_msgs;
struct timer_list rx_msg_timer;
unsigned int rx_need_bytes;
/* Transmit */
struct kcm_sock *tx_kcm;
struct list_head psock_avail_list;
unsigned long long saved_tx_bytes;
unsigned long long saved_tx_msgs;
};
/* Per net MUX list */
struct kcm_net {
struct mutex mutex;
struct kcm_psock_stats aggregate_psock_stats;
struct kcm_mux_stats aggregate_mux_stats;
struct list_head mux_list;
int count;
};
/* Structure for a MUX */
struct kcm_mux {
struct list_head kcm_mux_list;
struct rcu_head rcu;
struct kcm_net *knet;
struct list_head kcm_socks; /* All KCM sockets on MUX */
int kcm_socks_cnt; /* Total KCM socket count for MUX */
struct list_head psocks; /* List of all psocks on MUX */
int psocks_cnt; /* Total attached sockets */
struct kcm_mux_stats stats;
struct kcm_psock_stats aggregate_psock_stats;
/* Receive */
spinlock_t rx_lock ____cacheline_aligned_in_smp;
struct list_head kcm_rx_waiters; /* KCMs waiting for receiving */
struct list_head psocks_ready; /* List of psocks with a msg ready */
struct sk_buff_head rx_hold_queue;
/* Transmit */
spinlock_t lock ____cacheline_aligned_in_smp; /* TX and mux locking */
struct list_head psocks_avail; /* List of available psocks */
struct list_head kcm_tx_waiters; /* KCMs waiting for a TX psock */
};
#ifdef CONFIG_PROC_FS
int kcm_proc_init(void);
void kcm_proc_exit(void);
#else
static int kcm_proc_init(void) { return 0; }
static void kcm_proc_exit(void) { }
#endif
static inline void aggregate_psock_stats(struct kcm_psock_stats *stats,
struct kcm_psock_stats *agg_stats)
{
/* Save psock statistics in the mux when psock is being unattached. */
#define SAVE_PSOCK_STATS(_stat) (agg_stats->_stat += stats->_stat)
SAVE_PSOCK_STATS(rx_msgs);
SAVE_PSOCK_STATS(rx_bytes);
SAVE_PSOCK_STATS(rx_aborts);
SAVE_PSOCK_STATS(rx_mem_fail);
SAVE_PSOCK_STATS(rx_need_more_hdr);
SAVE_PSOCK_STATS(rx_msg_too_big);
SAVE_PSOCK_STATS(rx_msg_timeouts);
SAVE_PSOCK_STATS(rx_bad_hdr_len);
SAVE_PSOCK_STATS(tx_msgs);
SAVE_PSOCK_STATS(tx_bytes);
SAVE_PSOCK_STATS(reserved);
SAVE_PSOCK_STATS(unreserved);
SAVE_PSOCK_STATS(tx_aborts);
#undef SAVE_PSOCK_STATS
}
static inline void aggregate_mux_stats(struct kcm_mux_stats *stats,
struct kcm_mux_stats *agg_stats)
{
/* Save psock statistics in the mux when psock is being unattached. */
#define SAVE_MUX_STATS(_stat) (agg_stats->_stat += stats->_stat)
SAVE_MUX_STATS(rx_msgs);
SAVE_MUX_STATS(rx_bytes);
SAVE_MUX_STATS(tx_msgs);
SAVE_MUX_STATS(tx_bytes);
SAVE_MUX_STATS(rx_ready_drops);
SAVE_MUX_STATS(psock_attach);
SAVE_MUX_STATS(psock_unattach_rsvd);
SAVE_MUX_STATS(psock_unattach);
#undef SAVE_MUX_STATS
}
#endif /* __NET_KCM_H_ */
|
