summaryrefslogtreecommitdiff
path: root/arch/x86/mm/tlb.c
diff options
context:
space:
mode:
authorSrinivasarao P <spathi@codeaurora.org>2018-01-09 16:10:13 +0530
committerSrinivasarao P <spathi@codeaurora.org>2018-01-18 12:50:06 +0530
commit3726391f0570b5c8889d1db5f02a6a0a622c65b2 (patch)
treec1c8da9447d46a22126ee5e8172cbfeda6847c46 /arch/x86/mm/tlb.c
parent2d309c994df3624f59114edea6c03dd7cf3f334d (diff)
parent55b3b8c2b57468ab16c2b04778074fe52406a03c (diff)
Merge android-4.4.108 (55b3b8c) into msm-4.4
* refs/heads/tmp-55b3b8c Linux 4.4.108 alpha: fix build failures ALSA: hda - Fix yet another i915 pointer leftover in error path ALSA: hda - Degrade i915 binding failure message ALSA: hda - Clear the leftover component assignment at snd_hdac_i915_exit() Revert "Bluetooth: btusb: driver to enable the usb-wakeup feature" MIPS: math-emu: Fix final emulation phase for certain instructions thermal: hisilicon: Handle return value of clk_prepare_enable cpuidle: fix broadcast control when broadcast can not be entered rtc: set the alarm to the next expiring timer tcp: fix under-evaluated ssthresh in TCP Vegas fm10k: ensure we process SM mbx when processing VF mbx scsi: lpfc: PLOGI failures during NPIV testing scsi: lpfc: Fix secure firmware updates PCI/AER: Report non-fatal errors only to the affected endpoint ixgbe: fix use of uninitialized padding igb: check memory allocation failure PCI: Create SR-IOV virtfn/physfn links before attaching driver scsi: mpt3sas: Fix IO error occurs on pulling out a drive from RAID1 volume created on two SATA drive scsi: cxgb4i: fix Tx skb leak PCI: Avoid bus reset if bridge itself is broken net: phy: at803x: Change error to EINVAL for invalid MAC rtc: pl031: make interrupt optional crypto: crypto4xx - increase context and scatter ring buffer elements backlight: pwm_bl: Fix overflow condition bnxt_en: Fix NULL pointer dereference in reopen failure path cpuidle: powernv: Pass correct drv->cpumask for registration ARM: dma-mapping: disallow dma_get_sgtable() for non-kernel managed memory netfilter: nfnetlink_queue: fix secctx memory leak xhci: plat: Register shutdown for xhci_plat isdn: kcapi: avoid uninitialized data KVM: pci-assign: do not map smm memory slot pages in vt-d page tables ARM: dts: am335x-evmsk: adjust mmc2 param to allow suspend netfilter: nf_nat_snmp: Fix panic when snmp_trap_helper fails to register netfilter: nfnl_cthelper: fix a race when walk the nf_ct_helper_hash table irda: vlsi_ir: fix check for DMA mapping errors RDMA/iser: Fix possible mr leak on device removal event i40e: Do not enable NAPI on q_vectors that have no rings net: Do not allow negative values for busy_read and busy_poll sysctl interfaces bna: avoid writing uninitialized data into hw registers s390/qeth: no ETH header for outbound AF_IUCV r8152: prevent the driver from transmitting packets with carrier off HID: xinmo: fix for out of range for THT 2P arcade controller. hwmon: (asus_atk0110) fix uninitialized data access ARM: dts: ti: fix PCI bus dtc warnings KVM: VMX: Fix enable VPID conditions KVM: x86: correct async page present tracepoint scsi: lpfc: Fix PT2PT PRLI reject pinctrl: st: add irq_request/release_resources callbacks inet: frag: release spinlock before calling icmp_send() netfilter: nfnl_cthelper: Fix memory leak netfilter: nfnl_cthelper: fix runtime expectation policy updates usb: gadget: udc: remove pointer dereference after free usb: gadget: f_uvc: Sanity check wMaxPacketSize for SuperSpeed net: qmi_wwan: Add USB IDs for MDM6600 modem on Motorola Droid 4 bna: integer overflow bug in debugfs sch_dsmark: fix invalid skb_cow() usage crypto: deadlock between crypto_alg_sem/rtnl_mutex/genl_mutex r8152: fix the list rx_done may be used without initialization cpuidle: Validate cpu_dev in cpuidle_add_sysfs() arm: kprobes: Align stack to 8-bytes in test code arm: kprobes: Fix the return address of multiple kretprobes ALSA: hda - add support for docking station for HP 840 G3 ALSA: hda - add support for docking station for HP 820 G2 x86/irq: Do not substract irq_tlb_count from irq_call_count sched/core: Idle_task_exit() shouldn't use switch_mm_irqs_off() ARM: Hide finish_arch_post_lock_switch() from modules x86/mm, sched/core: Turn off IRQs in switch_mm() x86/mm, sched/core: Uninline switch_mm() x86/mm: Build arch/x86/mm/tlb.c even on !SMP sched/core: Add switch_mm_irqs_off() and use it in the scheduler mm/mmu_context, sched/core: Fix mmu_context.h assumption mm/rmap: batched invalidations should use existing api x86/mm: If INVPCID is available, use it to flush global mappings x86/mm: Add a 'noinvpcid' boot option to turn off INVPCID x86/mm: Fix INVPCID asm constraint x86/mm: Add INVPCID helpers cxl: Check if vphb exists before iterating over AFU devices arm64: Initialise high_memory global variable earlier ANDROID: binder: Remove obsolete proc waitqueue. Change-Id: Ie954ccd1dbd861672345bb0ee879273be4d0a441 Signed-off-by: Srinivasarao P <spathi@codeaurora.org>
Diffstat (limited to 'arch/x86/mm/tlb.c')
-rw-r--r--arch/x86/mm/tlb.c118
1 files changed, 117 insertions, 1 deletions
diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c
index 5a760fd66bec..45ba87466e6a 100644
--- a/arch/x86/mm/tlb.c
+++ b/arch/x86/mm/tlb.c
@@ -28,6 +28,8 @@
* Implement flush IPI by CALL_FUNCTION_VECTOR, Alex Shi
*/
+#ifdef CONFIG_SMP
+
struct flush_tlb_info {
struct mm_struct *flush_mm;
unsigned long flush_start;
@@ -57,6 +59,118 @@ void leave_mm(int cpu)
}
EXPORT_SYMBOL_GPL(leave_mm);
+#endif /* CONFIG_SMP */
+
+void switch_mm(struct mm_struct *prev, struct mm_struct *next,
+ struct task_struct *tsk)
+{
+ unsigned long flags;
+
+ local_irq_save(flags);
+ switch_mm_irqs_off(prev, next, tsk);
+ local_irq_restore(flags);
+}
+
+void switch_mm_irqs_off(struct mm_struct *prev, struct mm_struct *next,
+ struct task_struct *tsk)
+{
+ unsigned cpu = smp_processor_id();
+
+ if (likely(prev != next)) {
+#ifdef CONFIG_SMP
+ this_cpu_write(cpu_tlbstate.state, TLBSTATE_OK);
+ this_cpu_write(cpu_tlbstate.active_mm, next);
+#endif
+ cpumask_set_cpu(cpu, mm_cpumask(next));
+
+ /*
+ * Re-load page tables.
+ *
+ * This logic has an ordering constraint:
+ *
+ * CPU 0: Write to a PTE for 'next'
+ * CPU 0: load bit 1 in mm_cpumask. if nonzero, send IPI.
+ * CPU 1: set bit 1 in next's mm_cpumask
+ * CPU 1: load from the PTE that CPU 0 writes (implicit)
+ *
+ * We need to prevent an outcome in which CPU 1 observes
+ * the new PTE value and CPU 0 observes bit 1 clear in
+ * mm_cpumask. (If that occurs, then the IPI will never
+ * be sent, and CPU 0's TLB will contain a stale entry.)
+ *
+ * The bad outcome can occur if either CPU's load is
+ * reordered before that CPU's store, so both CPUs must
+ * execute full barriers to prevent this from happening.
+ *
+ * Thus, switch_mm needs a full barrier between the
+ * store to mm_cpumask and any operation that could load
+ * from next->pgd. TLB fills are special and can happen
+ * due to instruction fetches or for no reason at all,
+ * and neither LOCK nor MFENCE orders them.
+ * Fortunately, load_cr3() is serializing and gives the
+ * ordering guarantee we need.
+ *
+ */
+ load_cr3(next->pgd);
+
+ trace_tlb_flush(TLB_FLUSH_ON_TASK_SWITCH, TLB_FLUSH_ALL);
+
+ /* Stop flush ipis for the previous mm */
+ cpumask_clear_cpu(cpu, mm_cpumask(prev));
+
+ /* Load per-mm CR4 state */
+ load_mm_cr4(next);
+
+#ifdef CONFIG_MODIFY_LDT_SYSCALL
+ /*
+ * Load the LDT, if the LDT is different.
+ *
+ * It's possible that prev->context.ldt doesn't match
+ * the LDT register. This can happen if leave_mm(prev)
+ * was called and then modify_ldt changed
+ * prev->context.ldt but suppressed an IPI to this CPU.
+ * In this case, prev->context.ldt != NULL, because we
+ * never set context.ldt to NULL while the mm still
+ * exists. That means that next->context.ldt !=
+ * prev->context.ldt, because mms never share an LDT.
+ */
+ if (unlikely(prev->context.ldt != next->context.ldt))
+ load_mm_ldt(next);
+#endif
+ }
+#ifdef CONFIG_SMP
+ else {
+ this_cpu_write(cpu_tlbstate.state, TLBSTATE_OK);
+ BUG_ON(this_cpu_read(cpu_tlbstate.active_mm) != next);
+
+ if (!cpumask_test_cpu(cpu, mm_cpumask(next))) {
+ /*
+ * On established mms, the mm_cpumask is only changed
+ * from irq context, from ptep_clear_flush() while in
+ * lazy tlb mode, and here. Irqs are blocked during
+ * schedule, protecting us from simultaneous changes.
+ */
+ cpumask_set_cpu(cpu, mm_cpumask(next));
+
+ /*
+ * We were in lazy tlb mode and leave_mm disabled
+ * tlb flush IPI delivery. We must reload CR3
+ * to make sure to use no freed page tables.
+ *
+ * As above, load_cr3() is serializing and orders TLB
+ * fills with respect to the mm_cpumask write.
+ */
+ load_cr3(next->pgd);
+ trace_tlb_flush(TLB_FLUSH_ON_TASK_SWITCH, TLB_FLUSH_ALL);
+ load_mm_cr4(next);
+ load_mm_ldt(next);
+ }
+ }
+#endif
+}
+
+#ifdef CONFIG_SMP
+
/*
* The flush IPI assumes that a thread switch happens in this order:
* [cpu0: the cpu that switches]
@@ -104,7 +218,7 @@ static void flush_tlb_func(void *info)
inc_irq_stat(irq_tlb_count);
- if (f->flush_mm != this_cpu_read(cpu_tlbstate.active_mm))
+ if (f->flush_mm && f->flush_mm != this_cpu_read(cpu_tlbstate.active_mm))
return;
count_vm_tlb_event(NR_TLB_REMOTE_FLUSH_RECEIVED);
@@ -351,3 +465,5 @@ static int __init create_tlb_single_page_flush_ceiling(void)
return 0;
}
late_initcall(create_tlb_single_page_flush_ceiling);
+
+#endif /* CONFIG_SMP */