forked from luck/tmp_suning_uos_patched
3fc498f165
We have lots of infrastructure in place to partition multi-core systems such that we have a group of CPUs that are dedicated to specific task: cgroups, scheduler and interrupt affinity, and cpuisol= boot parameter. Still, kernel code will at times interrupt all CPUs in the system via IPIs for various needs. These IPIs are useful and cannot be avoided altogether, but in certain cases it is possible to interrupt only specific CPUs that have useful work to do and not the entire system. This patch set, inspired by discussions with Peter Zijlstra and Frederic Weisbecker when testing the nohz task patch set, is a first stab at trying to explore doing this by locating the places where such global IPI calls are being made and turning the global IPI into an IPI for a specific group of CPUs. The purpose of the patch set is to get feedback if this is the right way to go for dealing with this issue and indeed, if the issue is even worth dealing with at all. Based on the feedback from this patch set I plan to offer further patches that address similar issue in other code paths. This patch creates an on_each_cpu_mask() and on_each_cpu_cond() infrastructure API (the former derived from existing arch specific versions in Tile and Arm) and uses them to turn several global IPI invocation to per CPU group invocations. Core kernel: on_each_cpu_mask() calls a function on processors specified by cpumask, which may or may not include the local processor. You must not call this function with disabled interrupts or from a hardware interrupt handler or from a bottom half handler. arch/arm: Note that the generic version is a little different then the Arm one: 1. It has the mask as first parameter 2. It calls the function on the calling CPU with interrupts disabled, but this should be OK since the function is called on the other CPUs with interrupts disabled anyway. arch/tile: The API is the same as the tile private one, but the generic version also calls the function on the with interrupts disabled in UP case This is OK since the function is called on the other CPUs with interrupts disabled. Signed-off-by: Gilad Ben-Yossef <gilad@benyossef.com> Reviewed-by: Christoph Lameter <cl@linux.com> Acked-by: Chris Metcalf <cmetcalf@tilera.com> Acked-by: Peter Zijlstra <a.p.zijlstra@chello.nl> Cc: Frederic Weisbecker <fweisbec@gmail.com> Cc: Russell King <linux@arm.linux.org.uk> Cc: Pekka Enberg <penberg@kernel.org> Cc: Matt Mackall <mpm@selenic.com> Cc: Rik van Riel <riel@redhat.com> Cc: Andi Kleen <andi@firstfloor.org> Cc: Sasha Levin <levinsasha928@gmail.com> Cc: Mel Gorman <mel@csn.ul.ie> Cc: Alexander Viro <viro@zeniv.linux.org.uk> Cc: Avi Kivity <avi@redhat.com> Acked-by: Michal Nazarewicz <mina86@mina86.org> Cc: Kosaki Motohiro <kosaki.motohiro@gmail.com> Cc: Milton Miller <miltonm@bga.com> Cc: Russell King <linux@arm.linux.org.uk> Acked-by: Peter Zijlstra <a.p.zijlstra@chello.nl> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
130 lines
2.8 KiB
C
130 lines
2.8 KiB
C
/*
|
|
* linux/arch/arm/kernel/smp_tlb.c
|
|
*
|
|
* Copyright (C) 2002 ARM Limited, All Rights Reserved.
|
|
*
|
|
* This program is free software; you can redistribute it and/or modify
|
|
* it under the terms of the GNU General Public License version 2 as
|
|
* published by the Free Software Foundation.
|
|
*/
|
|
#include <linux/preempt.h>
|
|
#include <linux/smp.h>
|
|
|
|
#include <asm/smp_plat.h>
|
|
#include <asm/tlbflush.h>
|
|
|
|
/**********************************************************************/
|
|
|
|
/*
|
|
* TLB operations
|
|
*/
|
|
struct tlb_args {
|
|
struct vm_area_struct *ta_vma;
|
|
unsigned long ta_start;
|
|
unsigned long ta_end;
|
|
};
|
|
|
|
static inline void ipi_flush_tlb_all(void *ignored)
|
|
{
|
|
local_flush_tlb_all();
|
|
}
|
|
|
|
static inline void ipi_flush_tlb_mm(void *arg)
|
|
{
|
|
struct mm_struct *mm = (struct mm_struct *)arg;
|
|
|
|
local_flush_tlb_mm(mm);
|
|
}
|
|
|
|
static inline void ipi_flush_tlb_page(void *arg)
|
|
{
|
|
struct tlb_args *ta = (struct tlb_args *)arg;
|
|
|
|
local_flush_tlb_page(ta->ta_vma, ta->ta_start);
|
|
}
|
|
|
|
static inline void ipi_flush_tlb_kernel_page(void *arg)
|
|
{
|
|
struct tlb_args *ta = (struct tlb_args *)arg;
|
|
|
|
local_flush_tlb_kernel_page(ta->ta_start);
|
|
}
|
|
|
|
static inline void ipi_flush_tlb_range(void *arg)
|
|
{
|
|
struct tlb_args *ta = (struct tlb_args *)arg;
|
|
|
|
local_flush_tlb_range(ta->ta_vma, ta->ta_start, ta->ta_end);
|
|
}
|
|
|
|
static inline void ipi_flush_tlb_kernel_range(void *arg)
|
|
{
|
|
struct tlb_args *ta = (struct tlb_args *)arg;
|
|
|
|
local_flush_tlb_kernel_range(ta->ta_start, ta->ta_end);
|
|
}
|
|
|
|
void flush_tlb_all(void)
|
|
{
|
|
if (tlb_ops_need_broadcast())
|
|
on_each_cpu(ipi_flush_tlb_all, NULL, 1);
|
|
else
|
|
local_flush_tlb_all();
|
|
}
|
|
|
|
void flush_tlb_mm(struct mm_struct *mm)
|
|
{
|
|
if (tlb_ops_need_broadcast())
|
|
on_each_cpu_mask(mm_cpumask(mm), ipi_flush_tlb_mm, mm, 1);
|
|
else
|
|
local_flush_tlb_mm(mm);
|
|
}
|
|
|
|
void flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr)
|
|
{
|
|
if (tlb_ops_need_broadcast()) {
|
|
struct tlb_args ta;
|
|
ta.ta_vma = vma;
|
|
ta.ta_start = uaddr;
|
|
on_each_cpu_mask(mm_cpumask(vma->vm_mm), ipi_flush_tlb_page,
|
|
&ta, 1);
|
|
} else
|
|
local_flush_tlb_page(vma, uaddr);
|
|
}
|
|
|
|
void flush_tlb_kernel_page(unsigned long kaddr)
|
|
{
|
|
if (tlb_ops_need_broadcast()) {
|
|
struct tlb_args ta;
|
|
ta.ta_start = kaddr;
|
|
on_each_cpu(ipi_flush_tlb_kernel_page, &ta, 1);
|
|
} else
|
|
local_flush_tlb_kernel_page(kaddr);
|
|
}
|
|
|
|
void flush_tlb_range(struct vm_area_struct *vma,
|
|
unsigned long start, unsigned long end)
|
|
{
|
|
if (tlb_ops_need_broadcast()) {
|
|
struct tlb_args ta;
|
|
ta.ta_vma = vma;
|
|
ta.ta_start = start;
|
|
ta.ta_end = end;
|
|
on_each_cpu_mask(mm_cpumask(vma->vm_mm), ipi_flush_tlb_range,
|
|
&ta, 1);
|
|
} else
|
|
local_flush_tlb_range(vma, start, end);
|
|
}
|
|
|
|
void flush_tlb_kernel_range(unsigned long start, unsigned long end)
|
|
{
|
|
if (tlb_ops_need_broadcast()) {
|
|
struct tlb_args ta;
|
|
ta.ta_start = start;
|
|
ta.ta_end = end;
|
|
on_each_cpu(ipi_flush_tlb_kernel_range, &ta, 1);
|
|
} else
|
|
local_flush_tlb_kernel_range(start, end);
|
|
}
|
|
|