b56141ab34
This patch fixes race between inet_frag_lru_move() and inet_frag_lru_add()
which was introduced in commit 3ef0eb0db4
("net: frag, move LRU list maintenance outside of rwlock")
One cpu already added new fragment queue into hash but not into LRU.
Other cpu found it in hash and tries to move it to the end of LRU.
This leads to NULL pointer dereference inside of list_move_tail().
Another possible race condition is between inet_frag_lru_move() and
inet_frag_lru_del(): move can happens after deletion.
This patch initializes LRU list head before adding fragment into hash and
inet_frag_lru_move() doesn't touches it if it's empty.
I saw this kernel oops two times in a couple of days.
[119482.128853] BUG: unable to handle kernel NULL pointer dereference at (null)
[119482.132693] IP: [<ffffffff812ede89>] __list_del_entry+0x29/0xd0
[119482.136456] PGD 2148f6067 PUD 215ab9067 PMD 0
[119482.140221] Oops: 0000 [#1] SMP
[119482.144008] Modules linked in: vfat msdos fat 8021q fuse nfsd auth_rpcgss nfs_acl nfs lockd sunrpc ppp_async ppp_generic bridge slhc stp llc w83627ehf hwmon_vid snd_hda_codec_hdmi snd_hda_codec_realtek kvm_amd k10temp kvm snd_hda_intel snd_hda_codec edac_core radeon snd_hwdep ath9k snd_pcm ath9k_common snd_page_alloc ath9k_hw snd_timer snd soundcore drm_kms_helper ath ttm r8169 mii
[119482.152692] CPU 3
[119482.152721] Pid: 20, comm: ksoftirqd/3 Not tainted 3.9.0-zurg-00001-g9f95269 #132 To Be Filled By O.E.M. To Be Filled By O.E.M./RS880D
[119482.161478] RIP: 0010:[<ffffffff812ede89>] [<ffffffff812ede89>] __list_del_entry+0x29/0xd0
[119482.166004] RSP: 0018:ffff880216d5db58 EFLAGS: 00010207
[119482.170568] RAX: 0000000000000000 RBX: ffff88020882b9c0 RCX: dead000000200200
[119482.175189] RDX: 0000000000000000 RSI: 0000000000000880 RDI: ffff88020882ba00
[119482.179860] RBP: ffff880216d5db58 R08: ffffffff8155c7f0 R09: 0000000000000014
[119482.184570] R10: 0000000000000000 R11: 0000000000000000 R12: ffff88020882ba00
[119482.189337] R13: ffffffff81c8d780 R14: ffff880204357f00 R15: 00000000000005a0
[119482.194140] FS: 00007f58124dc700(0000) GS:ffff88021fcc0000(0000) knlGS:0000000000000000
[119482.198928] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b
[119482.203711] CR2: 0000000000000000 CR3: 00000002155f0000 CR4: 00000000000007e0
[119482.208533] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[119482.213371] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
[119482.218221] Process ksoftirqd/3 (pid: 20, threadinfo ffff880216d5c000, task ffff880216d3a9a0)
[119482.223113] Stack:
[119482.228004] ffff880216d5dbd8 ffffffff8155dcda 0000000000000000 ffff000200000001
[119482.233038] ffff8802153c1f00 ffff880000289440 ffff880200000014 ffff88007bc72000
[119482.238083] 00000000000079d5 ffff88007bc72f44 ffffffff00000002 ffff880204357f00
[119482.243090] Call Trace:
[119482.248009] [<ffffffff8155dcda>] ip_defrag+0x8fa/0xd10
[119482.252921] [<ffffffff815a8013>] ipv4_conntrack_defrag+0x83/0xe0
[119482.257803] [<ffffffff8154485b>] nf_iterate+0x8b/0xa0
[119482.262658] [<ffffffff8155c7f0>] ? inet_del_offload+0x40/0x40
[119482.267527] [<ffffffff815448e4>] nf_hook_slow+0x74/0x130
[119482.272412] [<ffffffff8155c7f0>] ? inet_del_offload+0x40/0x40
[119482.277302] [<ffffffff8155d068>] ip_rcv+0x268/0x320
[119482.282147] [<ffffffff81519992>] __netif_receive_skb_core+0x612/0x7e0
[119482.286998] [<ffffffff81519b78>] __netif_receive_skb+0x18/0x60
[119482.291826] [<ffffffff8151a650>] process_backlog+0xa0/0x160
[119482.296648] [<ffffffff81519f29>] net_rx_action+0x139/0x220
[119482.301403] [<ffffffff81053707>] __do_softirq+0xe7/0x220
[119482.306103] [<ffffffff81053868>] run_ksoftirqd+0x28/0x40
[119482.310809] [<ffffffff81074f5f>] smpboot_thread_fn+0xff/0x1a0
[119482.315515] [<ffffffff81074e60>] ? lg_local_lock_cpu+0x40/0x40
[119482.320219] [<ffffffff8106d870>] kthread+0xc0/0xd0
[119482.324858] [<ffffffff8106d7b0>] ? insert_kthread_work+0x40/0x40
[119482.329460] [<ffffffff816c32dc>] ret_from_fork+0x7c/0xb0
[119482.334057] [<ffffffff8106d7b0>] ? insert_kthread_work+0x40/0x40
[119482.338661] Code: 00 00 55 48 8b 17 48 b9 00 01 10 00 00 00 ad de 48 8b 47 08 48 89 e5 48 39 ca 74 29 48 b9 00 02 20 00 00 00 ad de 48 39 c8 74 7a <4c> 8b 00 4c 39 c7 75 53 4c 8b 42 08 4c 39 c7 75 2b 48 89 42 08
[119482.343787] RIP [<ffffffff812ede89>] __list_del_entry+0x29/0xd0
[119482.348675] RSP <ffff880216d5db58>
[119482.353493] CR2: 0000000000000000
Oops happened on this path:
ip_defrag() -> ip_frag_queue() -> inet_frag_lru_move() -> list_move_tail() -> __list_del_entry()
Signed-off-by: Konstantin Khlebnikov <khlebnikov@openvz.org>
Cc: Jesper Dangaard Brouer <brouer@redhat.com>
Cc: Florian Westphal <fw@strlen.de>
Cc: Eric Dumazet <edumazet@google.com>
Cc: David S. Miller <davem@davemloft.net>
Acked-by: Florian Westphal <fw@strlen.de>
Signed-off-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
366 lines
8.7 KiB
C
366 lines
8.7 KiB
C
/*
|
|
* inet fragments management
|
|
*
|
|
* This program is free software; you can redistribute it and/or
|
|
* modify it under the terms of the GNU General Public License
|
|
* as published by the Free Software Foundation; either version
|
|
* 2 of the License, or (at your option) any later version.
|
|
*
|
|
* Authors: Pavel Emelyanov <xemul@openvz.org>
|
|
* Started as consolidation of ipv4/ip_fragment.c,
|
|
* ipv6/reassembly. and ipv6 nf conntrack reassembly
|
|
*/
|
|
|
|
#include <linux/list.h>
|
|
#include <linux/spinlock.h>
|
|
#include <linux/module.h>
|
|
#include <linux/timer.h>
|
|
#include <linux/mm.h>
|
|
#include <linux/random.h>
|
|
#include <linux/skbuff.h>
|
|
#include <linux/rtnetlink.h>
|
|
#include <linux/slab.h>
|
|
|
|
#include <net/sock.h>
|
|
#include <net/inet_frag.h>
|
|
#include <net/inet_ecn.h>
|
|
|
|
/* Given the OR values of all fragments, apply RFC 3168 5.3 requirements
|
|
* Value : 0xff if frame should be dropped.
|
|
* 0 or INET_ECN_CE value, to be ORed in to final iph->tos field
|
|
*/
|
|
const u8 ip_frag_ecn_table[16] = {
|
|
/* at least one fragment had CE, and others ECT_0 or ECT_1 */
|
|
[IPFRAG_ECN_CE | IPFRAG_ECN_ECT_0] = INET_ECN_CE,
|
|
[IPFRAG_ECN_CE | IPFRAG_ECN_ECT_1] = INET_ECN_CE,
|
|
[IPFRAG_ECN_CE | IPFRAG_ECN_ECT_0 | IPFRAG_ECN_ECT_1] = INET_ECN_CE,
|
|
|
|
/* invalid combinations : drop frame */
|
|
[IPFRAG_ECN_NOT_ECT | IPFRAG_ECN_CE] = 0xff,
|
|
[IPFRAG_ECN_NOT_ECT | IPFRAG_ECN_ECT_0] = 0xff,
|
|
[IPFRAG_ECN_NOT_ECT | IPFRAG_ECN_ECT_1] = 0xff,
|
|
[IPFRAG_ECN_NOT_ECT | IPFRAG_ECN_ECT_0 | IPFRAG_ECN_ECT_1] = 0xff,
|
|
[IPFRAG_ECN_NOT_ECT | IPFRAG_ECN_CE | IPFRAG_ECN_ECT_0] = 0xff,
|
|
[IPFRAG_ECN_NOT_ECT | IPFRAG_ECN_CE | IPFRAG_ECN_ECT_1] = 0xff,
|
|
[IPFRAG_ECN_NOT_ECT | IPFRAG_ECN_CE | IPFRAG_ECN_ECT_0 | IPFRAG_ECN_ECT_1] = 0xff,
|
|
};
|
|
EXPORT_SYMBOL(ip_frag_ecn_table);
|
|
|
|
static void inet_frag_secret_rebuild(unsigned long dummy)
|
|
{
|
|
struct inet_frags *f = (struct inet_frags *)dummy;
|
|
unsigned long now = jiffies;
|
|
int i;
|
|
|
|
/* Per bucket lock NOT needed here, due to write lock protection */
|
|
write_lock(&f->lock);
|
|
|
|
get_random_bytes(&f->rnd, sizeof(u32));
|
|
for (i = 0; i < INETFRAGS_HASHSZ; i++) {
|
|
struct inet_frag_bucket *hb;
|
|
struct inet_frag_queue *q;
|
|
struct hlist_node *n;
|
|
|
|
hb = &f->hash[i];
|
|
hlist_for_each_entry_safe(q, n, &hb->chain, list) {
|
|
unsigned int hval = f->hashfn(q);
|
|
|
|
if (hval != i) {
|
|
struct inet_frag_bucket *hb_dest;
|
|
|
|
hlist_del(&q->list);
|
|
|
|
/* Relink to new hash chain. */
|
|
hb_dest = &f->hash[hval];
|
|
hlist_add_head(&q->list, &hb_dest->chain);
|
|
}
|
|
}
|
|
}
|
|
write_unlock(&f->lock);
|
|
|
|
mod_timer(&f->secret_timer, now + f->secret_interval);
|
|
}
|
|
|
|
void inet_frags_init(struct inet_frags *f)
|
|
{
|
|
int i;
|
|
|
|
for (i = 0; i < INETFRAGS_HASHSZ; i++) {
|
|
struct inet_frag_bucket *hb = &f->hash[i];
|
|
|
|
spin_lock_init(&hb->chain_lock);
|
|
INIT_HLIST_HEAD(&hb->chain);
|
|
}
|
|
rwlock_init(&f->lock);
|
|
|
|
f->rnd = (u32) ((num_physpages ^ (num_physpages>>7)) ^
|
|
(jiffies ^ (jiffies >> 6)));
|
|
|
|
setup_timer(&f->secret_timer, inet_frag_secret_rebuild,
|
|
(unsigned long)f);
|
|
f->secret_timer.expires = jiffies + f->secret_interval;
|
|
add_timer(&f->secret_timer);
|
|
}
|
|
EXPORT_SYMBOL(inet_frags_init);
|
|
|
|
void inet_frags_init_net(struct netns_frags *nf)
|
|
{
|
|
nf->nqueues = 0;
|
|
init_frag_mem_limit(nf);
|
|
INIT_LIST_HEAD(&nf->lru_list);
|
|
spin_lock_init(&nf->lru_lock);
|
|
}
|
|
EXPORT_SYMBOL(inet_frags_init_net);
|
|
|
|
void inet_frags_fini(struct inet_frags *f)
|
|
{
|
|
del_timer(&f->secret_timer);
|
|
}
|
|
EXPORT_SYMBOL(inet_frags_fini);
|
|
|
|
void inet_frags_exit_net(struct netns_frags *nf, struct inet_frags *f)
|
|
{
|
|
nf->low_thresh = 0;
|
|
|
|
local_bh_disable();
|
|
inet_frag_evictor(nf, f, true);
|
|
local_bh_enable();
|
|
|
|
percpu_counter_destroy(&nf->mem);
|
|
}
|
|
EXPORT_SYMBOL(inet_frags_exit_net);
|
|
|
|
static inline void fq_unlink(struct inet_frag_queue *fq, struct inet_frags *f)
|
|
{
|
|
struct inet_frag_bucket *hb;
|
|
unsigned int hash;
|
|
|
|
read_lock(&f->lock);
|
|
hash = f->hashfn(fq);
|
|
hb = &f->hash[hash];
|
|
|
|
spin_lock(&hb->chain_lock);
|
|
hlist_del(&fq->list);
|
|
spin_unlock(&hb->chain_lock);
|
|
|
|
read_unlock(&f->lock);
|
|
inet_frag_lru_del(fq);
|
|
}
|
|
|
|
void inet_frag_kill(struct inet_frag_queue *fq, struct inet_frags *f)
|
|
{
|
|
if (del_timer(&fq->timer))
|
|
atomic_dec(&fq->refcnt);
|
|
|
|
if (!(fq->last_in & INET_FRAG_COMPLETE)) {
|
|
fq_unlink(fq, f);
|
|
atomic_dec(&fq->refcnt);
|
|
fq->last_in |= INET_FRAG_COMPLETE;
|
|
}
|
|
}
|
|
EXPORT_SYMBOL(inet_frag_kill);
|
|
|
|
static inline void frag_kfree_skb(struct netns_frags *nf, struct inet_frags *f,
|
|
struct sk_buff *skb)
|
|
{
|
|
if (f->skb_free)
|
|
f->skb_free(skb);
|
|
kfree_skb(skb);
|
|
}
|
|
|
|
void inet_frag_destroy(struct inet_frag_queue *q, struct inet_frags *f,
|
|
int *work)
|
|
{
|
|
struct sk_buff *fp;
|
|
struct netns_frags *nf;
|
|
unsigned int sum, sum_truesize = 0;
|
|
|
|
WARN_ON(!(q->last_in & INET_FRAG_COMPLETE));
|
|
WARN_ON(del_timer(&q->timer) != 0);
|
|
|
|
/* Release all fragment data. */
|
|
fp = q->fragments;
|
|
nf = q->net;
|
|
while (fp) {
|
|
struct sk_buff *xp = fp->next;
|
|
|
|
sum_truesize += fp->truesize;
|
|
frag_kfree_skb(nf, f, fp);
|
|
fp = xp;
|
|
}
|
|
sum = sum_truesize + f->qsize;
|
|
if (work)
|
|
*work -= sum;
|
|
sub_frag_mem_limit(q, sum);
|
|
|
|
if (f->destructor)
|
|
f->destructor(q);
|
|
kfree(q);
|
|
|
|
}
|
|
EXPORT_SYMBOL(inet_frag_destroy);
|
|
|
|
int inet_frag_evictor(struct netns_frags *nf, struct inet_frags *f, bool force)
|
|
{
|
|
struct inet_frag_queue *q;
|
|
int work, evicted = 0;
|
|
|
|
if (!force) {
|
|
if (frag_mem_limit(nf) <= nf->high_thresh)
|
|
return 0;
|
|
}
|
|
|
|
work = frag_mem_limit(nf) - nf->low_thresh;
|
|
while (work > 0) {
|
|
spin_lock(&nf->lru_lock);
|
|
|
|
if (list_empty(&nf->lru_list)) {
|
|
spin_unlock(&nf->lru_lock);
|
|
break;
|
|
}
|
|
|
|
q = list_first_entry(&nf->lru_list,
|
|
struct inet_frag_queue, lru_list);
|
|
atomic_inc(&q->refcnt);
|
|
/* Remove q from list to avoid several CPUs grabbing it */
|
|
list_del_init(&q->lru_list);
|
|
|
|
spin_unlock(&nf->lru_lock);
|
|
|
|
spin_lock(&q->lock);
|
|
if (!(q->last_in & INET_FRAG_COMPLETE))
|
|
inet_frag_kill(q, f);
|
|
spin_unlock(&q->lock);
|
|
|
|
if (atomic_dec_and_test(&q->refcnt))
|
|
inet_frag_destroy(q, f, &work);
|
|
evicted++;
|
|
}
|
|
|
|
return evicted;
|
|
}
|
|
EXPORT_SYMBOL(inet_frag_evictor);
|
|
|
|
static struct inet_frag_queue *inet_frag_intern(struct netns_frags *nf,
|
|
struct inet_frag_queue *qp_in, struct inet_frags *f,
|
|
void *arg)
|
|
{
|
|
struct inet_frag_bucket *hb;
|
|
struct inet_frag_queue *qp;
|
|
#ifdef CONFIG_SMP
|
|
#endif
|
|
unsigned int hash;
|
|
|
|
read_lock(&f->lock); /* Protects against hash rebuild */
|
|
/*
|
|
* While we stayed w/o the lock other CPU could update
|
|
* the rnd seed, so we need to re-calculate the hash
|
|
* chain. Fortunatelly the qp_in can be used to get one.
|
|
*/
|
|
hash = f->hashfn(qp_in);
|
|
hb = &f->hash[hash];
|
|
spin_lock(&hb->chain_lock);
|
|
|
|
#ifdef CONFIG_SMP
|
|
/* With SMP race we have to recheck hash table, because
|
|
* such entry could be created on other cpu, while we
|
|
* released the hash bucket lock.
|
|
*/
|
|
hlist_for_each_entry(qp, &hb->chain, list) {
|
|
if (qp->net == nf && f->match(qp, arg)) {
|
|
atomic_inc(&qp->refcnt);
|
|
spin_unlock(&hb->chain_lock);
|
|
read_unlock(&f->lock);
|
|
qp_in->last_in |= INET_FRAG_COMPLETE;
|
|
inet_frag_put(qp_in, f);
|
|
return qp;
|
|
}
|
|
}
|
|
#endif
|
|
qp = qp_in;
|
|
if (!mod_timer(&qp->timer, jiffies + nf->timeout))
|
|
atomic_inc(&qp->refcnt);
|
|
|
|
atomic_inc(&qp->refcnt);
|
|
hlist_add_head(&qp->list, &hb->chain);
|
|
spin_unlock(&hb->chain_lock);
|
|
read_unlock(&f->lock);
|
|
inet_frag_lru_add(nf, qp);
|
|
return qp;
|
|
}
|
|
|
|
static struct inet_frag_queue *inet_frag_alloc(struct netns_frags *nf,
|
|
struct inet_frags *f, void *arg)
|
|
{
|
|
struct inet_frag_queue *q;
|
|
|
|
q = kzalloc(f->qsize, GFP_ATOMIC);
|
|
if (q == NULL)
|
|
return NULL;
|
|
|
|
q->net = nf;
|
|
f->constructor(q, arg);
|
|
add_frag_mem_limit(q, f->qsize);
|
|
|
|
setup_timer(&q->timer, f->frag_expire, (unsigned long)q);
|
|
spin_lock_init(&q->lock);
|
|
atomic_set(&q->refcnt, 1);
|
|
INIT_LIST_HEAD(&q->lru_list);
|
|
|
|
return q;
|
|
}
|
|
|
|
static struct inet_frag_queue *inet_frag_create(struct netns_frags *nf,
|
|
struct inet_frags *f, void *arg)
|
|
{
|
|
struct inet_frag_queue *q;
|
|
|
|
q = inet_frag_alloc(nf, f, arg);
|
|
if (q == NULL)
|
|
return NULL;
|
|
|
|
return inet_frag_intern(nf, q, f, arg);
|
|
}
|
|
|
|
struct inet_frag_queue *inet_frag_find(struct netns_frags *nf,
|
|
struct inet_frags *f, void *key, unsigned int hash)
|
|
__releases(&f->lock)
|
|
{
|
|
struct inet_frag_bucket *hb;
|
|
struct inet_frag_queue *q;
|
|
int depth = 0;
|
|
|
|
hb = &f->hash[hash];
|
|
|
|
spin_lock(&hb->chain_lock);
|
|
hlist_for_each_entry(q, &hb->chain, list) {
|
|
if (q->net == nf && f->match(q, key)) {
|
|
atomic_inc(&q->refcnt);
|
|
spin_unlock(&hb->chain_lock);
|
|
read_unlock(&f->lock);
|
|
return q;
|
|
}
|
|
depth++;
|
|
}
|
|
spin_unlock(&hb->chain_lock);
|
|
read_unlock(&f->lock);
|
|
|
|
if (depth <= INETFRAGS_MAXDEPTH)
|
|
return inet_frag_create(nf, f, key);
|
|
else
|
|
return ERR_PTR(-ENOBUFS);
|
|
}
|
|
EXPORT_SYMBOL(inet_frag_find);
|
|
|
|
void inet_frag_maybe_warn_overflow(struct inet_frag_queue *q,
|
|
const char *prefix)
|
|
{
|
|
static const char msg[] = "inet_frag_find: Fragment hash bucket"
|
|
" list length grew over limit " __stringify(INETFRAGS_MAXDEPTH)
|
|
". Dropping fragment.\n";
|
|
|
|
if (PTR_ERR(q) == -ENOBUFS)
|
|
LIMIT_NETDEBUG(KERN_WARNING "%s%s", prefix, msg);
|
|
}
|
|
EXPORT_SYMBOL(inet_frag_maybe_warn_overflow);
|