AcoSail/kernel_optimize_test
8
0
Fork 0
forked from luck/tmp_suning_uos_patched
Code Pull Requests Activity

tcp: annotate sk->sk_sndbuf lockless reads

Browse Source 
For the sake of tcp_poll(), there are few places where we fetch
sk->sk_sndbuf while this field can change from IRQ or other cpu.

We need to add READ_ONCE() annotations, and also make sure write
sides use corresponding WRITE_ONCE() to avoid store-tearing.

Note that other transports probably need similar fixes.

Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
This commit is contained in:
Eric Dumazet 2019-10-10 20:17:45 -07:00 committed by David S. Miller
parent ebb3b78db7
commit e292f05e0d
5 changed files with 25 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
include/net/sock.h
View File

@ -883,7 +883,7 @@ static inline int sk_stream_min_wspace(const struct sock *sk)
static inline int sk_stream_wspace(const struct sock *sk) static inline int sk_stream_wspace(const struct sock *sk)
{ {
return sk->sk_sndbuf - sk->sk_wmem_queued; return READ_ONCE(sk->sk_sndbuf) - sk->sk_wmem_queued;
} }
void sk_stream_write_space(struct sock *sk); void sk_stream_write_space(struct sock *sk);
@ -1207,7 +1207,7 @@ static inline void sk_refcnt_debug_release(const struct sock *sk)
static inline bool __sk_stream_memory_free(const struct sock *sk, int wake) static inline bool __sk_stream_memory_free(const struct sock *sk, int wake)
{ {
if (sk->sk_wmem_queued >= sk->sk_sndbuf) if (sk->sk_wmem_queued >= READ_ONCE(sk->sk_sndbuf))
return false; return false;
return sk->sk_prot->stream_memory_free ? return sk->sk_prot->stream_memory_free ?
@ -2220,10 +2220,14 @@ static inline void sk_wake_async(const struct sock *sk, int how, int band)
static inline void sk_stream_moderate_sndbuf(struct sock *sk) static inline void sk_stream_moderate_sndbuf(struct sock *sk)
{ {
if (!(sk->sk_userlocks & SOCK_SNDBUF_LOCK)) { u32 val;
sk->sk_sndbuf = min(sk->sk_sndbuf, sk->sk_wmem_queued >> 1);
sk->sk_sndbuf = max_t(u32, sk->sk_sndbuf, SOCK_MIN_SNDBUF); if (sk->sk_userlocks & SOCK_SNDBUF_LOCK)
} return;
val = min(sk->sk_sndbuf, sk->sk_wmem_queued >> 1);
WRITE_ONCE(sk->sk_sndbuf, max_t(u32, val, SOCK_MIN_SNDBUF));
} }
struct sk_buff *sk_stream_alloc_skb(struct sock *sk, int size, gfp_t gfp, struct sk_buff *sk_stream_alloc_skb(struct sock *sk, int size, gfp_t gfp,
@ -2251,7 +2255,7 @@ bool sk_page_frag_refill(struct sock *sk, struct page_frag *pfrag);
*/ */
static inline bool sock_writeable(const struct sock *sk) static inline bool sock_writeable(const struct sock *sk)
{ {
return refcount_read(&sk->sk_wmem_alloc) < (sk->sk_sndbuf >> 1); return refcount_read(&sk->sk_wmem_alloc) < (READ_ONCE(sk->sk_sndbuf) >> 1);
} }
static inline gfp_t gfp_any(void) static inline gfp_t gfp_any(void)

3
net/core/filter.c
View File

@ -4258,7 +4258,8 @@ BPF_CALL_5(bpf_setsockopt, struct bpf_sock_ops_kern *, bpf_sock,
case SO_SNDBUF: case SO_SNDBUF:
val = min_t(u32, val, sysctl_wmem_max); val = min_t(u32, val, sysctl_wmem_max);
sk->sk_userlocks |= SOCK_SNDBUF_LOCK; sk->sk_userlocks |= SOCK_SNDBUF_LOCK;
sk->sk_sndbuf = max_t(int, val * 2, SOCK_MIN_SNDBUF); WRITE_ONCE(sk->sk_sndbuf,
max_t(int, val * 2, SOCK_MIN_SNDBUF));
break; break;
case SO_MAX_PACING_RATE: /* 32bit version */ case SO_MAX_PACING_RATE: /* 32bit version */
if (val != ~0U) if (val != ~0U)

15
net/core/sock.c
View File

@ -785,7 +785,8 @@ int sock_setsockopt(struct socket *sock, int level, int optname,
*/ */
val = min_t(int, val, INT_MAX / 2); val = min_t(int, val, INT_MAX / 2);
sk->sk_userlocks |= SOCK_SNDBUF_LOCK; sk->sk_userlocks |= SOCK_SNDBUF_LOCK;
sk->sk_sndbuf = max_t(int, val * 2, SOCK_MIN_SNDBUF); WRITE_ONCE(sk->sk_sndbuf,
max_t(int, val * 2, SOCK_MIN_SNDBUF));
/* Wake up sending tasks if we upped the value. */ /* Wake up sending tasks if we upped the value. */
sk->sk_write_space(sk); sk->sk_write_space(sk);
break; break;
@ -2089,8 +2090,10 @@ EXPORT_SYMBOL(sock_i_ino);
struct sk_buff *sock_wmalloc(struct sock *sk, unsigned long size, int force, struct sk_buff *sock_wmalloc(struct sock *sk, unsigned long size, int force,
gfp_t priority) gfp_t priority)
{ {
if (force || refcount_read(&sk->sk_wmem_alloc) < sk->sk_sndbuf) { if (force ||
refcount_read(&sk->sk_wmem_alloc) < READ_ONCE(sk->sk_sndbuf)) {
struct sk_buff *skb = alloc_skb(size, priority); struct sk_buff *skb = alloc_skb(size, priority);
if (skb) { if (skb) {
skb_set_owner_w(skb, sk); skb_set_owner_w(skb, sk);
return skb; return skb;
@ -2191,7 +2194,7 @@ static long sock_wait_for_wmem(struct sock *sk, long timeo)
break; break;
set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
prepare_to_wait(sk_sleep(sk), &wait, TASK_INTERRUPTIBLE); prepare_to_wait(sk_sleep(sk), &wait, TASK_INTERRUPTIBLE);
if (refcount_read(&sk->sk_wmem_alloc) < sk->sk_sndbuf) if (refcount_read(&sk->sk_wmem_alloc) < READ_ONCE(sk->sk_sndbuf))
break; break;
if (sk->sk_shutdown & SEND_SHUTDOWN) if (sk->sk_shutdown & SEND_SHUTDOWN)
break; break;
@ -2226,7 +2229,7 @@ struct sk_buff *sock_alloc_send_pskb(struct sock *sk, unsigned long header_len,
if (sk->sk_shutdown & SEND_SHUTDOWN) if (sk->sk_shutdown & SEND_SHUTDOWN)
goto failure; goto failure;
if (sk_wmem_alloc_get(sk) < sk->sk_sndbuf) if (sk_wmem_alloc_get(sk) < READ_ONCE(sk->sk_sndbuf))
break; break;
sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk); sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk);
@ -2807,7 +2810,7 @@ static void sock_def_write_space(struct sock *sk)
/* Do not wake up a writer until he can make "significant" /* Do not wake up a writer until he can make "significant"
* progress. --DaveM * progress. --DaveM
*/ */
if ((refcount_read(&sk->sk_wmem_alloc) << 1) <= sk->sk_sndbuf) { if ((refcount_read(&sk->sk_wmem_alloc) << 1) <= READ_ONCE(sk->sk_sndbuf)) {
wq = rcu_dereference(sk->sk_wq); wq = rcu_dereference(sk->sk_wq);
if (skwq_has_sleeper(wq)) if (skwq_has_sleeper(wq))
wake_up_interruptible_sync_poll(&wq->wait, EPOLLOUT | wake_up_interruptible_sync_poll(&wq->wait, EPOLLOUT |
@ -3207,7 +3210,7 @@ void sk_get_meminfo(const struct sock *sk, u32 *mem)
mem[SK_MEMINFO_RMEM_ALLOC] = sk_rmem_alloc_get(sk); mem[SK_MEMINFO_RMEM_ALLOC] = sk_rmem_alloc_get(sk);
mem[SK_MEMINFO_RCVBUF] = READ_ONCE(sk->sk_rcvbuf); mem[SK_MEMINFO_RCVBUF] = READ_ONCE(sk->sk_rcvbuf);
mem[SK_MEMINFO_WMEM_ALLOC] = sk_wmem_alloc_get(sk); mem[SK_MEMINFO_WMEM_ALLOC] = sk_wmem_alloc_get(sk);
mem[SK_MEMINFO_SNDBUF] = sk->sk_sndbuf; mem[SK_MEMINFO_SNDBUF] = READ_ONCE(sk->sk_sndbuf);
mem[SK_MEMINFO_FWD_ALLOC] = sk->sk_forward_alloc; mem[SK_MEMINFO_FWD_ALLOC] = sk->sk_forward_alloc;
mem[SK_MEMINFO_WMEM_QUEUED] = sk->sk_wmem_queued; mem[SK_MEMINFO_WMEM_QUEUED] = sk->sk_wmem_queued;
mem[SK_MEMINFO_OPTMEM] = atomic_read(&sk->sk_omem_alloc); mem[SK_MEMINFO_OPTMEM] = atomic_read(&sk->sk_omem_alloc);

2
net/ipv4/tcp.c
View File

@ -450,7 +450,7 @@ void tcp_init_sock(struct sock *sk)
icsk->icsk_sync_mss = tcp_sync_mss; icsk->icsk_sync_mss = tcp_sync_mss;
sk->sk_sndbuf = sock_net(sk)->ipv4.sysctl_tcp_wmem[1]; WRITE_ONCE(sk->sk_sndbuf, sock_net(sk)->ipv4.sysctl_tcp_wmem[1]);
WRITE_ONCE(sk->sk_rcvbuf, sock_net(sk)->ipv4.sysctl_tcp_rmem[1]); WRITE_ONCE(sk->sk_rcvbuf, sock_net(sk)->ipv4.sysctl_tcp_rmem[1]);
sk_sockets_allocated_inc(sk); sk_sockets_allocated_inc(sk);

3
net/ipv4/tcp_input.c
View File

@ -359,7 +359,8 @@ static void tcp_sndbuf_expand(struct sock *sk)
sndmem *= nr_segs * per_mss; sndmem *= nr_segs * per_mss;
if (sk->sk_sndbuf < sndmem) if (sk->sk_sndbuf < sndmem)
sk->sk_sndbuf = min(sndmem, sock_net(sk)->ipv4.sysctl_tcp_wmem[2]); WRITE_ONCE(sk->sk_sndbuf,
min(sndmem, sock_net(sk)->ipv4.sysctl_tcp_wmem[2]));
} }
/* 2. Tuning advertised window (window_clamp, rcv_ssthresh) /* 2. Tuning advertised window (window_clamp, rcv_ssthresh)