From fc53662f13b889a5a1c069e79ee1e3d4534df132 Mon Sep 17 00:00:00 2001 From: Vitaly Kuznetsov <vkuznets@redhat.com> Date: Wed, 2 Aug 2017 18:09:14 +0200 Subject: x86/hyper-v: Make hv_do_hypercall() inline We have only three call sites for hv_do_hypercall() and we're going to change HVCALL_SIGNAL_EVENT to doing fast hypercall so we can inline this function for optimization. Hyper-V top level functional specification states that r9-r11 registers and flags may be clobbered by the hypervisor during hypercall and with inlining this is somewhat important, add the clobbers. Signed-off-by: Vitaly Kuznetsov <vkuznets@redhat.com> Reviewed-by: Andy Shevchenko <andy.shevchenko@gmail.com> Reviewed-by: Stephen Hemminger <sthemmin@microsoft.com> Cc: Andy Lutomirski <luto@kernel.org> Cc: Haiyang Zhang <haiyangz@microsoft.com> Cc: Jork Loeser <Jork.Loeser@microsoft.com> Cc: K. Y. Srinivasan <kys@microsoft.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Cc: Peter Zijlstra <peterz@infradead.org> Cc: Simon Xiao <sixiao@microsoft.com> Cc: Steven Rostedt <rostedt@goodmis.org> Cc: Thomas Gleixner <tglx@linutronix.de> Cc: devel@linuxdriverproject.org Link: http://lkml.kernel.org/r/20170802160921.21791-3-vkuznets@redhat.com Signed-off-by: Ingo Molnar <mingo@kernel.org> --- drivers/hv/connection.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'drivers/hv') diff --git a/drivers/hv/connection.c b/drivers/hv/connection.c index 59c11ff90d12..45e806e3112f 100644 --- a/drivers/hv/connection.c +++ b/drivers/hv/connection.c @@ -32,6 +32,8 @@ #include <linux/hyperv.h> #include <linux/export.h> #include <asm/hyperv.h> +#include <asm/mshyperv.h> + #include "hyperv_vmbus.h" -- cgit v1.2.3-70-g09d2 From 057841713cfff62b4485cdd2b245f05b7ea3ba16 Mon Sep 17 00:00:00 2001 From: Vitaly Kuznetsov <vkuznets@redhat.com> Date: Wed, 2 Aug 2017 18:09:16 +0200 Subject: hyper-v: Use fast hypercall for HVCALL_SIGNAL_EVENT We need to pass only 8 bytes of input for HvSignalEvent which makes it a perfect fit for fast hypercall. hv_input_signal_event_buffer is not needed any more and hv_input_signal_event is converted to union for convenience. Signed-off-by: Vitaly Kuznetsov <vkuznets@redhat.com> Reviewed-by: Andy Shevchenko <andy.shevchenko@gmail.com> Reviewed-by: Stephen Hemminger <sthemmin@microsoft.com> Cc: Andy Lutomirski <luto@kernel.org> Cc: Haiyang Zhang <haiyangz@microsoft.com> Cc: Jork Loeser <Jork.Loeser@microsoft.com> Cc: K. Y. Srinivasan <kys@microsoft.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Cc: Peter Zijlstra <peterz@infradead.org> Cc: Simon Xiao <sixiao@microsoft.com> Cc: Steven Rostedt <rostedt@goodmis.org> Cc: Thomas Gleixner <tglx@linutronix.de> Cc: devel@linuxdriverproject.org Link: http://lkml.kernel.org/r/20170802160921.21791-5-vkuznets@redhat.com Signed-off-by: Ingo Molnar <mingo@kernel.org> --- drivers/hv/channel_mgmt.c | 13 ++----------- drivers/hv/connection.c | 2 +- include/linux/hyperv.h | 15 +-------------- 3 files changed, 4 insertions(+), 26 deletions(-) (limited to 'drivers/hv') diff --git a/drivers/hv/channel_mgmt.c b/drivers/hv/channel_mgmt.c index 4bbb8dea4727..fd2b6c67f781 100644 --- a/drivers/hv/channel_mgmt.c +++ b/drivers/hv/channel_mgmt.c @@ -805,21 +805,12 @@ static void vmbus_onoffer(struct vmbus_channel_message_header *hdr) /* * Setup state for signalling the host. */ - newchannel->sig_event = (struct hv_input_signal_event *) - (ALIGN((unsigned long) - &newchannel->sig_buf, - HV_HYPERCALL_PARAM_ALIGN)); - - newchannel->sig_event->connectionid.asu32 = 0; - newchannel->sig_event->connectionid.u.id = VMBUS_EVENT_CONNECTION_ID; - newchannel->sig_event->flag_number = 0; - newchannel->sig_event->rsvdz = 0; + newchannel->sig_event = VMBUS_EVENT_CONNECTION_ID; if (vmbus_proto_version != VERSION_WS2008) { newchannel->is_dedicated_interrupt = (offer->is_dedicated_interrupt != 0); - newchannel->sig_event->connectionid.u.id = - offer->connection_id; + newchannel->sig_event = offer->connection_id; } memcpy(&newchannel->offermsg, offer, diff --git a/drivers/hv/connection.c b/drivers/hv/connection.c index 45e806e3112f..37ecf514189e 100644 --- a/drivers/hv/connection.c +++ b/drivers/hv/connection.c @@ -408,6 +408,6 @@ void vmbus_set_event(struct vmbus_channel *channel) if (!channel->is_dedicated_interrupt) vmbus_send_interrupt(child_relid); - hv_do_hypercall(HVCALL_SIGNAL_EVENT, channel->sig_event, NULL); + hv_do_fast_hypercall8(HVCALL_SIGNAL_EVENT, channel->sig_event); } EXPORT_SYMBOL_GPL(vmbus_set_event); diff --git a/include/linux/hyperv.h b/include/linux/hyperv.h index 6608a71e7d79..c472bd43bdd7 100644 --- a/include/linux/hyperv.h +++ b/include/linux/hyperv.h @@ -677,18 +677,6 @@ union hv_connection_id { } u; }; -/* Definition of the hv_signal_event hypercall input structure. */ -struct hv_input_signal_event { - union hv_connection_id connectionid; - u16 flag_number; - u16 rsvdz; -}; - -struct hv_input_signal_event_buffer { - u64 align8; - struct hv_input_signal_event event; -}; - enum hv_numa_policy { HV_BALANCED = 0, HV_LOCALIZED, @@ -770,8 +758,7 @@ struct vmbus_channel { } callback_mode; bool is_dedicated_interrupt; - struct hv_input_signal_event_buffer sig_buf; - struct hv_input_signal_event *sig_event; + u64 sig_event; /* * Starting with win8, this field will be used to specify -- cgit v1.2.3-70-g09d2 From 7415aea6072bab15969b6c3c5b2a193d88095326 Mon Sep 17 00:00:00 2001 From: Vitaly Kuznetsov <vkuznets@redhat.com> Date: Wed, 2 Aug 2017 18:09:18 +0200 Subject: hyper-v: Globalize vp_index To support implementing remote TLB flushing on Hyper-V with a hypercall we need to make vp_index available outside of vmbus module. Rename and globalize. Signed-off-by: Vitaly Kuznetsov <vkuznets@redhat.com> Reviewed-by: Andy Shevchenko <andy.shevchenko@gmail.com> Reviewed-by: Stephen Hemminger <sthemmin@microsoft.com> Cc: Andy Lutomirski <luto@kernel.org> Cc: Haiyang Zhang <haiyangz@microsoft.com> Cc: Jork Loeser <Jork.Loeser@microsoft.com> Cc: K. Y. Srinivasan <kys@microsoft.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Cc: Peter Zijlstra <peterz@infradead.org> Cc: Simon Xiao <sixiao@microsoft.com> Cc: Steven Rostedt <rostedt@goodmis.org> Cc: Thomas Gleixner <tglx@linutronix.de> Cc: devel@linuxdriverproject.org Link: http://lkml.kernel.org/r/20170802160921.21791-7-vkuznets@redhat.com Signed-off-by: Ingo Molnar <mingo@kernel.org> --- arch/x86/hyperv/hv_init.c | 34 +++++++++++++++++++++++++- arch/x86/include/asm/mshyperv.h | 24 ++++++++++++++++++ drivers/hv/channel_mgmt.c | 7 +++--- drivers/hv/connection.c | 3 ++- drivers/hv/hv.c | 9 ------- drivers/hv/hyperv_vmbus.h | 11 --------- drivers/hv/vmbus_drv.c | 17 ------------- drivers/pci/host/pci-hyperv.c | 54 +++-------------------------------------- include/linux/hyperv.h | 1 - 9 files changed, 65 insertions(+), 95 deletions(-) (limited to 'drivers/hv') diff --git a/arch/x86/hyperv/hv_init.c b/arch/x86/hyperv/hv_init.c index 691603ee9179..e93b9a0b1b10 100644 --- a/arch/x86/hyperv/hv_init.c +++ b/arch/x86/hyperv/hv_init.c @@ -26,6 +26,8 @@ #include <linux/mm.h> #include <linux/clockchips.h> #include <linux/hyperv.h> +#include <linux/slab.h> +#include <linux/cpuhotplug.h> #ifdef CONFIG_HYPERV_TSCPAGE @@ -80,6 +82,20 @@ EXPORT_SYMBOL_GPL(hv_hypercall_pg); struct clocksource *hyperv_cs; EXPORT_SYMBOL_GPL(hyperv_cs); +u32 *hv_vp_index; +EXPORT_SYMBOL_GPL(hv_vp_index); + +static int hv_cpu_init(unsigned int cpu) +{ + u64 msr_vp_index; + + hv_get_vp_index(msr_vp_index); + + hv_vp_index[smp_processor_id()] = msr_vp_index; + + return 0; +} + /* * This function is to be invoked early in the boot sequence after the * hypervisor has been detected. @@ -95,6 +111,16 @@ void hyperv_init(void) if (x86_hyper != &x86_hyper_ms_hyperv) return; + /* Allocate percpu VP index */ + hv_vp_index = kmalloc_array(num_possible_cpus(), sizeof(*hv_vp_index), + GFP_KERNEL); + if (!hv_vp_index) + return; + + if (cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "x86/hyperv_init:online", + hv_cpu_init, NULL) < 0) + goto free_vp_index; + /* * Setup the hypercall page and enable hypercalls. * 1. Register the guest ID @@ -106,7 +132,7 @@ void hyperv_init(void) hv_hypercall_pg = __vmalloc(PAGE_SIZE, GFP_KERNEL, PAGE_KERNEL_RX); if (hv_hypercall_pg == NULL) { wrmsrl(HV_X64_MSR_GUEST_OS_ID, 0); - return; + goto free_vp_index; } rdmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64); @@ -149,6 +175,12 @@ register_msr_cs: hyperv_cs = &hyperv_cs_msr; if (ms_hyperv.features & HV_X64_MSR_TIME_REF_COUNT_AVAILABLE) clocksource_register_hz(&hyperv_cs_msr, NSEC_PER_SEC/100); + + return; + +free_vp_index: + kfree(hv_vp_index); + hv_vp_index = NULL; } /* diff --git a/arch/x86/include/asm/mshyperv.h b/arch/x86/include/asm/mshyperv.h index efa1860276b5..efd2f80d3353 100644 --- a/arch/x86/include/asm/mshyperv.h +++ b/arch/x86/include/asm/mshyperv.h @@ -282,6 +282,30 @@ static inline u64 hv_do_rep_hypercall(u16 code, u16 rep_count, u16 varhead_size, return status; } +/* + * Hypervisor's notion of virtual processor ID is different from + * Linux' notion of CPU ID. This information can only be retrieved + * in the context of the calling CPU. Setup a map for easy access + * to this information. + */ +extern u32 *hv_vp_index; + +/** + * hv_cpu_number_to_vp_number() - Map CPU to VP. + * @cpu_number: CPU number in Linux terms + * + * This function returns the mapping between the Linux processor + * number and the hypervisor's virtual processor number, useful + * in making hypercalls and such that talk about specific + * processors. + * + * Return: Virtual processor number in Hyper-V terms + */ +static inline int hv_cpu_number_to_vp_number(int cpu_number) +{ + return hv_vp_index[cpu_number]; +} + void hyperv_init(void); void hyperv_report_panic(struct pt_regs *regs); bool hv_is_hypercall_page_setup(void); diff --git a/drivers/hv/channel_mgmt.c b/drivers/hv/channel_mgmt.c index fd2b6c67f781..dc590195a74e 100644 --- a/drivers/hv/channel_mgmt.c +++ b/drivers/hv/channel_mgmt.c @@ -599,7 +599,7 @@ static void init_vp_index(struct vmbus_channel *channel, u16 dev_type) */ channel->numa_node = 0; channel->target_cpu = 0; - channel->target_vp = hv_context.vp_index[0]; + channel->target_vp = hv_cpu_number_to_vp_number(0); return; } @@ -683,7 +683,7 @@ static void init_vp_index(struct vmbus_channel *channel, u16 dev_type) } channel->target_cpu = cur_cpu; - channel->target_vp = hv_context.vp_index[cur_cpu]; + channel->target_vp = hv_cpu_number_to_vp_number(cur_cpu); } static void vmbus_wait_for_unload(void) @@ -1219,8 +1219,7 @@ struct vmbus_channel *vmbus_get_outgoing_channel(struct vmbus_channel *primary) return outgoing_channel; } - cur_cpu = hv_context.vp_index[get_cpu()]; - put_cpu(); + cur_cpu = hv_cpu_number_to_vp_number(smp_processor_id()); list_for_each_safe(cur, tmp, &primary->sc_list) { cur_channel = list_entry(cur, struct vmbus_channel, sc_list); if (cur_channel->state != CHANNEL_OPENED_STATE) diff --git a/drivers/hv/connection.c b/drivers/hv/connection.c index 37ecf514189e..f41901f80b64 100644 --- a/drivers/hv/connection.c +++ b/drivers/hv/connection.c @@ -96,7 +96,8 @@ static int vmbus_negotiate_version(struct vmbus_channel_msginfo *msginfo, * the CPU attempting to connect may not be CPU 0. */ if (version >= VERSION_WIN8_1) { - msg->target_vcpu = hv_context.vp_index[smp_processor_id()]; + msg->target_vcpu = + hv_cpu_number_to_vp_number(smp_processor_id()); vmbus_connection.connect_cpu = smp_processor_id(); } else { msg->target_vcpu = 0; diff --git a/drivers/hv/hv.c b/drivers/hv/hv.c index 2ea12207caa0..8267439dd1ee 100644 --- a/drivers/hv/hv.c +++ b/drivers/hv/hv.c @@ -234,7 +234,6 @@ int hv_synic_init(unsigned int cpu) union hv_synic_siefp siefp; union hv_synic_sint shared_sint; union hv_synic_scontrol sctrl; - u64 vp_index; /* Setup the Synic's message page */ hv_get_simp(simp.as_uint64); @@ -275,14 +274,6 @@ int hv_synic_init(unsigned int cpu) hv_context.synic_initialized = true; - /* - * Setup the mapping between Hyper-V's notion - * of cpuid and Linux' notion of cpuid. - * This array will be indexed using Linux cpuid. - */ - hv_get_vp_index(vp_index); - hv_context.vp_index[cpu] = (u32)vp_index; - /* * Register the per-cpu clockevent source. */ diff --git a/drivers/hv/hyperv_vmbus.h b/drivers/hv/hyperv_vmbus.h index 1b6a5e0dfa75..49569f8fe038 100644 --- a/drivers/hv/hyperv_vmbus.h +++ b/drivers/hv/hyperv_vmbus.h @@ -228,17 +228,6 @@ struct hv_context { struct hv_per_cpu_context __percpu *cpu_context; - /* - * Hypervisor's notion of virtual processor ID is different from - * Linux' notion of CPU ID. This information can only be retrieved - * in the context of the calling CPU. Setup a map for easy access - * to this information: - * - * vp_index[a] is the Hyper-V's processor ID corresponding to - * Linux cpuid 'a'. - */ - u32 vp_index[NR_CPUS]; - /* * To manage allocations in a NUMA node. * Array indexed by numa node ID. diff --git a/drivers/hv/vmbus_drv.c b/drivers/hv/vmbus_drv.c index ed84e96715a0..c7e7d6db2d21 100644 --- a/drivers/hv/vmbus_drv.c +++ b/drivers/hv/vmbus_drv.c @@ -1451,23 +1451,6 @@ void vmbus_free_mmio(resource_size_t start, resource_size_t size) } EXPORT_SYMBOL_GPL(vmbus_free_mmio); -/** - * vmbus_cpu_number_to_vp_number() - Map CPU to VP. - * @cpu_number: CPU number in Linux terms - * - * This function returns the mapping between the Linux processor - * number and the hypervisor's virtual processor number, useful - * in making hypercalls and such that talk about specific - * processors. - * - * Return: Virtual processor number in Hyper-V terms - */ -int vmbus_cpu_number_to_vp_number(int cpu_number) -{ - return hv_context.vp_index[cpu_number]; -} -EXPORT_SYMBOL_GPL(vmbus_cpu_number_to_vp_number); - static int vmbus_acpi_add(struct acpi_device *device) { acpi_status result; diff --git a/drivers/pci/host/pci-hyperv.c b/drivers/pci/host/pci-hyperv.c index 415dcc69a502..aba041438566 100644 --- a/drivers/pci/host/pci-hyperv.c +++ b/drivers/pci/host/pci-hyperv.c @@ -562,52 +562,6 @@ static void put_pcichild(struct hv_pci_dev *hv_pcidev, static void get_hvpcibus(struct hv_pcibus_device *hv_pcibus); static void put_hvpcibus(struct hv_pcibus_device *hv_pcibus); - -/* - * Temporary CPU to vCPU mapping to address transitioning - * vmbus_cpu_number_to_vp_number() being migrated to - * hv_cpu_number_to_vp_number() in a separate patch. Once that patch - * has been picked up in the main line, remove this code here and use - * the official code. - */ -static struct hv_tmpcpumap -{ - bool initialized; - u32 vp_index[NR_CPUS]; -} hv_tmpcpumap; - -static void hv_tmpcpumap_init_cpu(void *_unused) -{ - int cpu = smp_processor_id(); - u64 vp_index; - - hv_get_vp_index(vp_index); - - hv_tmpcpumap.vp_index[cpu] = vp_index; -} - -static void hv_tmpcpumap_init(void) -{ - if (hv_tmpcpumap.initialized) - return; - - memset(hv_tmpcpumap.vp_index, -1, sizeof(hv_tmpcpumap.vp_index)); - on_each_cpu(hv_tmpcpumap_init_cpu, NULL, true); - hv_tmpcpumap.initialized = true; -} - -/** - * hv_tmp_cpu_nr_to_vp_nr() - Convert Linux CPU nr to Hyper-V vCPU nr - * - * Remove once vmbus_cpu_number_to_vp_number() has been converted to - * hv_cpu_number_to_vp_number() and replace callers appropriately. - */ -static u32 hv_tmp_cpu_nr_to_vp_nr(int cpu) -{ - return hv_tmpcpumap.vp_index[cpu]; -} - - /** * devfn_to_wslot() - Convert from Linux PCI slot to Windows * @devfn: The Linux representation of PCI slot @@ -971,7 +925,7 @@ static void hv_irq_unmask(struct irq_data *data) var_size = 1 + HV_VP_SET_BANK_COUNT_MAX; for_each_cpu_and(cpu, dest, cpu_online_mask) { - cpu_vmbus = hv_tmp_cpu_nr_to_vp_nr(cpu); + cpu_vmbus = hv_cpu_number_to_vp_number(cpu); if (cpu_vmbus >= HV_VP_SET_BANK_COUNT_MAX * 64) { dev_err(&hbus->hdev->device, @@ -986,7 +940,7 @@ static void hv_irq_unmask(struct irq_data *data) } else { for_each_cpu_and(cpu, dest, cpu_online_mask) { params->int_target.vp_mask |= - (1ULL << hv_tmp_cpu_nr_to_vp_nr(cpu)); + (1ULL << hv_cpu_number_to_vp_number(cpu)); } } @@ -1063,7 +1017,7 @@ static u32 hv_compose_msi_req_v2( */ cpu = cpumask_first_and(affinity, cpu_online_mask); int_pkt->int_desc.processor_array[0] = - hv_tmp_cpu_nr_to_vp_nr(cpu); + hv_cpu_number_to_vp_number(cpu); int_pkt->int_desc.processor_count = 1; return sizeof(*int_pkt); @@ -2490,8 +2444,6 @@ static int hv_pci_probe(struct hv_device *hdev, return -ENOMEM; hbus->state = hv_pcibus_init; - hv_tmpcpumap_init(); - /* * The PCI bus "domain" is what is called "segment" in ACPI and * other specs. Pull it from the instance ID, to get something diff --git a/include/linux/hyperv.h b/include/linux/hyperv.h index c472bd43bdd7..e2a4fa57f110 100644 --- a/include/linux/hyperv.h +++ b/include/linux/hyperv.h @@ -1173,7 +1173,6 @@ int vmbus_allocate_mmio(struct resource **new, struct hv_device *device_obj, resource_size_t size, resource_size_t align, bool fb_overlap_ok); void vmbus_free_mmio(resource_size_t start, resource_size_t size); -int vmbus_cpu_number_to_vp_number(int cpu_number); /* * GUID definitions of various offer types - services offered to the guest. -- cgit v1.2.3-70-g09d2 From 2ffd9e33ce4af4e8cfa3e17bf493defe8474e2eb Mon Sep 17 00:00:00 2001 From: Vitaly Kuznetsov <vkuznets@redhat.com> Date: Wed, 2 Aug 2017 18:09:19 +0200 Subject: x86/hyper-v: Use hypercall for remote TLB flush Hyper-V host can suggest us to use hypercall for doing remote TLB flush, this is supposed to work faster than IPIs. Implementation details: to do HvFlushVirtualAddress{Space,List} hypercalls we need to put the input somewhere in memory and we don't really want to have memory allocation on each call so we pre-allocate per cpu memory areas on boot. pv_ops patching is happening very early so we need to separate hyperv_setup_mmu_ops() and hyper_alloc_mmu(). It is possible and easy to implement local TLB flushing too and there is even a hint for that. However, I don't see a room for optimization on the host side as both hypercall and native tlb flush will result in vmexit. The hint is also not set on modern Hyper-V versions. Signed-off-by: Vitaly Kuznetsov <vkuznets@redhat.com> Reviewed-by: Andy Shevchenko <andy.shevchenko@gmail.com> Reviewed-by: Stephen Hemminger <sthemmin@microsoft.com> Cc: Andy Lutomirski <luto@kernel.org> Cc: Haiyang Zhang <haiyangz@microsoft.com> Cc: Jork Loeser <Jork.Loeser@microsoft.com> Cc: K. Y. Srinivasan <kys@microsoft.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Cc: Peter Zijlstra <peterz@infradead.org> Cc: Simon Xiao <sixiao@microsoft.com> Cc: Steven Rostedt <rostedt@goodmis.org> Cc: Thomas Gleixner <tglx@linutronix.de> Cc: devel@linuxdriverproject.org Link: http://lkml.kernel.org/r/20170802160921.21791-8-vkuznets@redhat.com Signed-off-by: Ingo Molnar <mingo@kernel.org> --- arch/x86/hyperv/Makefile | 2 +- arch/x86/hyperv/hv_init.c | 2 + arch/x86/hyperv/mmu.c | 138 +++++++++++++++++++++++++++++++++++++ arch/x86/include/asm/mshyperv.h | 3 + arch/x86/include/uapi/asm/hyperv.h | 7 ++ arch/x86/kernel/cpu/mshyperv.c | 1 + drivers/hv/Kconfig | 1 + 7 files changed, 153 insertions(+), 1 deletion(-) create mode 100644 arch/x86/hyperv/mmu.c (limited to 'drivers/hv') diff --git a/arch/x86/hyperv/Makefile b/arch/x86/hyperv/Makefile index 171ae09864d7..367a8203cfcf 100644 --- a/arch/x86/hyperv/Makefile +++ b/arch/x86/hyperv/Makefile @@ -1 +1 @@ -obj-y := hv_init.o +obj-y := hv_init.o mmu.o diff --git a/arch/x86/hyperv/hv_init.c b/arch/x86/hyperv/hv_init.c index e93b9a0b1b10..1a8eb550c40f 100644 --- a/arch/x86/hyperv/hv_init.c +++ b/arch/x86/hyperv/hv_init.c @@ -140,6 +140,8 @@ void hyperv_init(void) hypercall_msr.guest_physical_address = vmalloc_to_pfn(hv_hypercall_pg); wrmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64); + hyper_alloc_mmu(); + /* * Register Hyper-V specific clocksource. */ diff --git a/arch/x86/hyperv/mmu.c b/arch/x86/hyperv/mmu.c new file mode 100644 index 000000000000..9419a20b1d75 --- /dev/null +++ b/arch/x86/hyperv/mmu.c @@ -0,0 +1,138 @@ +#define pr_fmt(fmt) "Hyper-V: " fmt + +#include <linux/hyperv.h> +#include <linux/log2.h> +#include <linux/slab.h> +#include <linux/types.h> + +#include <asm/fpu/api.h> +#include <asm/mshyperv.h> +#include <asm/msr.h> +#include <asm/tlbflush.h> + +/* HvFlushVirtualAddressSpace, HvFlushVirtualAddressList hypercalls */ +struct hv_flush_pcpu { + u64 address_space; + u64 flags; + u64 processor_mask; + u64 gva_list[]; +}; + +/* Each gva in gva_list encodes up to 4096 pages to flush */ +#define HV_TLB_FLUSH_UNIT (4096 * PAGE_SIZE) + +static struct hv_flush_pcpu __percpu *pcpu_flush; + +/* + * Fills in gva_list starting from offset. Returns the number of items added. + */ +static inline int fill_gva_list(u64 gva_list[], int offset, + unsigned long start, unsigned long end) +{ + int gva_n = offset; + unsigned long cur = start, diff; + + do { + diff = end > cur ? end - cur : 0; + + gva_list[gva_n] = cur & PAGE_MASK; + /* + * Lower 12 bits encode the number of additional + * pages to flush (in addition to the 'cur' page). + */ + if (diff >= HV_TLB_FLUSH_UNIT) + gva_list[gva_n] |= ~PAGE_MASK; + else if (diff) + gva_list[gva_n] |= (diff - 1) >> PAGE_SHIFT; + + cur += HV_TLB_FLUSH_UNIT; + gva_n++; + + } while (cur < end); + + return gva_n - offset; +} + +static void hyperv_flush_tlb_others(const struct cpumask *cpus, + const struct flush_tlb_info *info) +{ + int cpu, vcpu, gva_n, max_gvas; + struct hv_flush_pcpu *flush; + u64 status = U64_MAX; + unsigned long flags; + + if (!pcpu_flush || !hv_hypercall_pg) + goto do_native; + + if (cpumask_empty(cpus)) + return; + + local_irq_save(flags); + + flush = this_cpu_ptr(pcpu_flush); + + if (info->mm) { + flush->address_space = virt_to_phys(info->mm->pgd); + flush->flags = 0; + } else { + flush->address_space = 0; + flush->flags = HV_FLUSH_ALL_VIRTUAL_ADDRESS_SPACES; + } + + flush->processor_mask = 0; + if (cpumask_equal(cpus, cpu_present_mask)) { + flush->flags |= HV_FLUSH_ALL_PROCESSORS; + } else { + for_each_cpu(cpu, cpus) { + vcpu = hv_cpu_number_to_vp_number(cpu); + if (vcpu >= 64) + goto do_native; + + __set_bit(vcpu, (unsigned long *) + &flush->processor_mask); + } + } + + /* + * We can flush not more than max_gvas with one hypercall. Flush the + * whole address space if we were asked to do more. + */ + max_gvas = (PAGE_SIZE - sizeof(*flush)) / sizeof(flush->gva_list[0]); + + if (info->end == TLB_FLUSH_ALL) { + flush->flags |= HV_FLUSH_NON_GLOBAL_MAPPINGS_ONLY; + status = hv_do_hypercall(HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE, + flush, NULL); + } else if (info->end && + ((info->end - info->start)/HV_TLB_FLUSH_UNIT) > max_gvas) { + status = hv_do_hypercall(HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE, + flush, NULL); + } else { + gva_n = fill_gva_list(flush->gva_list, 0, + info->start, info->end); + status = hv_do_rep_hypercall(HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST, + gva_n, 0, flush, NULL); + } + + local_irq_restore(flags); + + if (!(status & HV_HYPERCALL_RESULT_MASK)) + return; +do_native: + native_flush_tlb_others(cpus, info); +} + +void hyperv_setup_mmu_ops(void) +{ + if (ms_hyperv.hints & HV_X64_REMOTE_TLB_FLUSH_RECOMMENDED) { + pr_info("Using hypercall for remote TLB flush\n"); + pv_mmu_ops.flush_tlb_others = hyperv_flush_tlb_others; + setup_clear_cpu_cap(X86_FEATURE_PCID); + } +} + +void hyper_alloc_mmu(void) +{ + if (ms_hyperv.hints & HV_X64_REMOTE_TLB_FLUSH_RECOMMENDED) + pcpu_flush = __alloc_percpu(PAGE_SIZE, PAGE_SIZE); +} diff --git a/arch/x86/include/asm/mshyperv.h b/arch/x86/include/asm/mshyperv.h index efd2f80d3353..0d4b01c5e438 100644 --- a/arch/x86/include/asm/mshyperv.h +++ b/arch/x86/include/asm/mshyperv.h @@ -307,6 +307,8 @@ static inline int hv_cpu_number_to_vp_number(int cpu_number) } void hyperv_init(void); +void hyperv_setup_mmu_ops(void); +void hyper_alloc_mmu(void); void hyperv_report_panic(struct pt_regs *regs); bool hv_is_hypercall_page_setup(void); void hyperv_cleanup(void); @@ -314,6 +316,7 @@ void hyperv_cleanup(void); static inline void hyperv_init(void) {} static inline bool hv_is_hypercall_page_setup(void) { return false; } static inline void hyperv_cleanup(void) {} +static inline void hyperv_setup_mmu_ops(void) {} #endif /* CONFIG_HYPERV */ #ifdef CONFIG_HYPERV_TSCPAGE diff --git a/arch/x86/include/uapi/asm/hyperv.h b/arch/x86/include/uapi/asm/hyperv.h index 127ddadee1a5..a6fdd3b82b4a 100644 --- a/arch/x86/include/uapi/asm/hyperv.h +++ b/arch/x86/include/uapi/asm/hyperv.h @@ -242,6 +242,8 @@ (~((1ull << HV_X64_MSR_HYPERCALL_PAGE_ADDRESS_SHIFT) - 1)) /* Declare the various hypercall operations. */ +#define HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE 0x0002 +#define HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST 0x0003 #define HVCALL_NOTIFY_LONG_SPIN_WAIT 0x0008 #define HVCALL_POST_MESSAGE 0x005c #define HVCALL_SIGNAL_EVENT 0x005d @@ -259,6 +261,11 @@ #define HV_PROCESSOR_POWER_STATE_C2 2 #define HV_PROCESSOR_POWER_STATE_C3 3 +#define HV_FLUSH_ALL_PROCESSORS BIT(0) +#define HV_FLUSH_ALL_VIRTUAL_ADDRESS_SPACES BIT(1) +#define HV_FLUSH_NON_GLOBAL_MAPPINGS_ONLY BIT(2) +#define HV_FLUSH_USE_EXTENDED_RANGE_FORMAT BIT(3) + /* hypercall status code */ #define HV_STATUS_SUCCESS 0 #define HV_STATUS_INVALID_HYPERCALL_CODE 2 diff --git a/arch/x86/kernel/cpu/mshyperv.c b/arch/x86/kernel/cpu/mshyperv.c index 70e717fccdd6..daefd67a66c7 100644 --- a/arch/x86/kernel/cpu/mshyperv.c +++ b/arch/x86/kernel/cpu/mshyperv.c @@ -249,6 +249,7 @@ static void __init ms_hyperv_init_platform(void) * Setup the hook to get control post apic initialization. */ x86_platform.apic_post_init = hyperv_init; + hyperv_setup_mmu_ops(); #endif } diff --git a/drivers/hv/Kconfig b/drivers/hv/Kconfig index c29cd5387a35..50b89ea0e60f 100644 --- a/drivers/hv/Kconfig +++ b/drivers/hv/Kconfig @@ -3,6 +3,7 @@ menu "Microsoft Hyper-V guest support" config HYPERV tristate "Microsoft Hyper-V client drivers" depends on X86 && ACPI && PCI && X86_LOCAL_APIC && HYPERVISOR_GUEST + select PARAVIRT help Select this option to run Linux as a Hyper-V client operating system. -- cgit v1.2.3-70-g09d2