aboutsummaryrefslogtreecommitdiffstats
path: root/arch/loongarch/kernel
diff options
context:
space:
mode:
Diffstat (limited to 'arch/loongarch/kernel')
-rw-r--r--arch/loongarch/kernel/Makefile7
-rw-r--r--arch/loongarch/kernel/cacheinfo.c98
-rw-r--r--arch/loongarch/kernel/cpu-probe.c4
-rw-r--r--arch/loongarch/kernel/crash_dump.c23
-rw-r--r--arch/loongarch/kernel/head.S22
-rw-r--r--arch/loongarch/kernel/machine_kexec.c304
-rw-r--r--arch/loongarch/kernel/mem.c3
-rw-r--r--arch/loongarch/kernel/module-sections.c61
-rw-r--r--arch/loongarch/kernel/module.c105
-rw-r--r--arch/loongarch/kernel/perf_event.c887
-rw-r--r--arch/loongarch/kernel/perf_regs.c53
-rw-r--r--arch/loongarch/kernel/relocate_kernel.S112
-rw-r--r--arch/loongarch/kernel/setup.c76
-rw-r--r--arch/loongarch/kernel/smp.c5
-rw-r--r--arch/loongarch/kernel/sysrq.c65
-rw-r--r--arch/loongarch/kernel/topology.c3
-rw-r--r--arch/loongarch/kernel/traps.c33
-rw-r--r--arch/loongarch/kernel/vmlinux.lds.S4
18 files changed, 1754 insertions, 111 deletions
diff --git a/arch/loongarch/kernel/Makefile b/arch/loongarch/kernel/Makefile
index 6c33b5c45573..42be564278fa 100644
--- a/arch/loongarch/kernel/Makefile
+++ b/arch/loongarch/kernel/Makefile
@@ -23,7 +23,14 @@ obj-$(CONFIG_SMP) += smp.o
obj-$(CONFIG_NUMA) += numa.o
+obj-$(CONFIG_MAGIC_SYSRQ) += sysrq.o
+
+obj-$(CONFIG_KEXEC) += machine_kexec.o relocate_kernel.o
+obj-$(CONFIG_CRASH_DUMP) += crash_dump.o
+
obj-$(CONFIG_UNWINDER_GUESS) += unwind_guess.o
obj-$(CONFIG_UNWINDER_PROLOGUE) += unwind_prologue.o
+obj-$(CONFIG_PERF_EVENTS) += perf_event.o perf_regs.o
+
CPPFLAGS_vmlinux.lds := $(KBUILD_CFLAGS)
diff --git a/arch/loongarch/kernel/cacheinfo.c b/arch/loongarch/kernel/cacheinfo.c
index 4662b06269f4..c7988f757281 100644
--- a/arch/loongarch/kernel/cacheinfo.c
+++ b/arch/loongarch/kernel/cacheinfo.c
@@ -5,73 +5,34 @@
* Copyright (C) 2020-2022 Loongson Technology Corporation Limited
*/
#include <linux/cacheinfo.h>
+#include <linux/topology.h>
#include <asm/bootinfo.h>
#include <asm/cpu-info.h>
-/* Populates leaf and increments to next leaf */
-#define populate_cache(cache, leaf, c_level, c_type) \
-do { \
- leaf->type = c_type; \
- leaf->level = c_level; \
- leaf->coherency_line_size = c->cache.linesz; \
- leaf->number_of_sets = c->cache.sets; \
- leaf->ways_of_associativity = c->cache.ways; \
- leaf->size = c->cache.linesz * c->cache.sets * \
- c->cache.ways; \
- if (leaf->level > 2) \
- leaf->size *= nodes_per_package; \
- leaf++; \
-} while (0)
-
int init_cache_level(unsigned int cpu)
{
- struct cpuinfo_loongarch *c = &current_cpu_data;
+ int cache_present = current_cpu_data.cache_leaves_present;
struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu);
- int levels = 0, leaves = 0;
-
- /*
- * If Dcache is not set, we assume the cache structures
- * are not properly initialized.
- */
- if (c->dcache.waysize)
- levels += 1;
- else
- return -ENOENT;
-
-
- leaves += (c->icache.waysize) ? 2 : 1;
-
- if (c->vcache.waysize) {
- levels++;
- leaves++;
- }
- if (c->scache.waysize) {
- levels++;
- leaves++;
- }
+ this_cpu_ci->num_levels =
+ current_cpu_data.cache_leaves[cache_present - 1].level;
+ this_cpu_ci->num_leaves = cache_present;
- if (c->tcache.waysize) {
- levels++;
- leaves++;
- }
-
- this_cpu_ci->num_levels = levels;
- this_cpu_ci->num_leaves = leaves;
return 0;
}
static inline bool cache_leaves_are_shared(struct cacheinfo *this_leaf,
struct cacheinfo *sib_leaf)
{
- return !((this_leaf->level == 1) || (this_leaf->level == 2));
+ return (!(*(unsigned char *)(this_leaf->priv) & CACHE_PRIVATE)
+ && !(*(unsigned char *)(sib_leaf->priv) & CACHE_PRIVATE));
}
static void cache_cpumap_setup(unsigned int cpu)
{
- struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu);
- struct cacheinfo *this_leaf, *sib_leaf;
unsigned int index;
+ struct cacheinfo *this_leaf, *sib_leaf;
+ struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu);
for (index = 0; index < this_cpu_ci->num_leaves; index++) {
unsigned int i;
@@ -85,8 +46,10 @@ static void cache_cpumap_setup(unsigned int cpu)
for_each_online_cpu(i) {
struct cpu_cacheinfo *sib_cpu_ci = get_cpu_cacheinfo(i);
- if (i == cpu || !sib_cpu_ci->info_list)
- continue;/* skip if itself or no cacheinfo */
+ if (i == cpu || !sib_cpu_ci->info_list ||
+ (cpu_to_node(i) != cpu_to_node(cpu)))
+ continue;
+
sib_leaf = sib_cpu_ci->info_list + index;
if (cache_leaves_are_shared(this_leaf, sib_leaf)) {
cpumask_set_cpu(cpu, &sib_leaf->shared_cpu_map);
@@ -98,31 +61,24 @@ static void cache_cpumap_setup(unsigned int cpu)
int populate_cache_leaves(unsigned int cpu)
{
- int level = 1, nodes_per_package = 1;
- struct cpuinfo_loongarch *c = &current_cpu_data;
+ int i, cache_present = current_cpu_data.cache_leaves_present;
struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu);
struct cacheinfo *this_leaf = this_cpu_ci->info_list;
-
- if (loongson_sysconf.nr_nodes > 1)
- nodes_per_package = loongson_sysconf.cores_per_package
- / loongson_sysconf.cores_per_node;
-
- if (c->icache.waysize) {
- populate_cache(dcache, this_leaf, level, CACHE_TYPE_DATA);
- populate_cache(icache, this_leaf, level++, CACHE_TYPE_INST);
- } else {
- populate_cache(dcache, this_leaf, level++, CACHE_TYPE_UNIFIED);
+ struct cache_desc *cd, *cdesc = current_cpu_data.cache_leaves;
+
+ for (i = 0; i < cache_present; i++) {
+ cd = cdesc + i;
+
+ this_leaf->type = cd->type;
+ this_leaf->level = cd->level;
+ this_leaf->coherency_line_size = cd->linesz;
+ this_leaf->number_of_sets = cd->sets;
+ this_leaf->ways_of_associativity = cd->ways;
+ this_leaf->size = cd->linesz * cd->sets * cd->ways;
+ this_leaf->priv = &cd->flags;
+ this_leaf++;
}
- if (c->vcache.waysize)
- populate_cache(vcache, this_leaf, level++, CACHE_TYPE_UNIFIED);
-
- if (c->scache.waysize)
- populate_cache(scache, this_leaf, level++, CACHE_TYPE_UNIFIED);
-
- if (c->tcache.waysize)
- populate_cache(tcache, this_leaf, level++, CACHE_TYPE_UNIFIED);
-
cache_cpumap_setup(cpu);
this_cpu_ci->cpu_map_populated = true;
diff --git a/arch/loongarch/kernel/cpu-probe.c b/arch/loongarch/kernel/cpu-probe.c
index 529ab8f44ec6..255a09876ef2 100644
--- a/arch/loongarch/kernel/cpu-probe.c
+++ b/arch/loongarch/kernel/cpu-probe.c
@@ -187,7 +187,9 @@ static inline void cpu_probe_loongson(struct cpuinfo_loongarch *c, unsigned int
uint64_t *vendor = (void *)(&cpu_full_name[VENDOR_OFFSET]);
uint64_t *cpuname = (void *)(&cpu_full_name[CPUNAME_OFFSET]);
- __cpu_full_name[cpu] = cpu_full_name;
+ if (!__cpu_full_name[cpu])
+ __cpu_full_name[cpu] = cpu_full_name;
+
*vendor = iocsr_read64(LOONGARCH_IOCSR_VENDOR);
*cpuname = iocsr_read64(LOONGARCH_IOCSR_CPUNAME);
diff --git a/arch/loongarch/kernel/crash_dump.c b/arch/loongarch/kernel/crash_dump.c
new file mode 100644
index 000000000000..e559307c1092
--- /dev/null
+++ b/arch/loongarch/kernel/crash_dump.c
@@ -0,0 +1,23 @@
+// SPDX-License-Identifier: GPL-2.0
+#include <linux/crash_dump.h>
+#include <linux/io.h>
+#include <linux/uio.h>
+
+ssize_t copy_oldmem_page(struct iov_iter *iter, unsigned long pfn,
+ size_t csize, unsigned long offset)
+{
+ void *vaddr;
+
+ if (!csize)
+ return 0;
+
+ vaddr = memremap(__pfn_to_phys(pfn), PAGE_SIZE, MEMREMAP_WB);
+ if (!vaddr)
+ return -ENOMEM;
+
+ csize = copy_to_iter(vaddr + offset, csize, iter);
+
+ memunmap(vaddr);
+
+ return csize;
+}
diff --git a/arch/loongarch/kernel/head.S b/arch/loongarch/kernel/head.S
index 7e57ae8741b1..97425779ce9f 100644
--- a/arch/loongarch/kernel/head.S
+++ b/arch/loongarch/kernel/head.S
@@ -8,6 +8,7 @@
#include <asm/addrspace.h>
#include <asm/asm.h>
#include <asm/asmmacro.h>
+#include <asm/bug.h>
#include <asm/regdef.h>
#include <asm/loongarch.h>
#include <asm/stackframe.h>
@@ -20,7 +21,11 @@
_head:
.word MZ_MAGIC /* "MZ", MS-DOS header */
- .org 0x3c /* 0x04 ~ 0x3b reserved */
+ .org 0x8
+ .dword kernel_entry /* Kernel entry point */
+ .dword _end - _text /* Kernel image effective size */
+ .quad 0 /* Kernel image load offset from start of RAM */
+ .org 0x3c /* 0x20 ~ 0x3b reserved */
.long pe_header - _head /* Offset to the PE header */
pe_header:
@@ -57,19 +62,19 @@ SYM_CODE_START(kernel_entry) # kernel entry point
li.w t0, 0x00 # FPE=0, SXE=0, ASXE=0, BTE=0
csrwr t0, LOONGARCH_CSR_EUEN
- la t0, __bss_start # clear .bss
+ la.pcrel t0, __bss_start # clear .bss
st.d zero, t0, 0
- la t1, __bss_stop - LONGSIZE
+ la.pcrel t1, __bss_stop - LONGSIZE
1:
addi.d t0, t0, LONGSIZE
st.d zero, t0, 0
bne t0, t1, 1b
- la t0, fw_arg0
+ la.pcrel t0, fw_arg0
st.d a0, t0, 0 # firmware arguments
- la t0, fw_arg1
+ la.pcrel t0, fw_arg1
st.d a1, t0, 0
- la t0, fw_arg2
+ la.pcrel t0, fw_arg2
st.d a2, t0, 0
/* KSave3 used for percpu base, initialized as 0 */
@@ -77,7 +82,7 @@ SYM_CODE_START(kernel_entry) # kernel entry point
/* GPR21 used for percpu base (runtime), initialized as 0 */
move u0, zero
- la tp, init_thread_union
+ la.pcrel tp, init_thread_union
/* Set the SP after an empty pt_regs. */
PTR_LI sp, (_THREAD_SIZE - 32 - PT_SIZE)
PTR_ADD sp, sp, tp
@@ -85,6 +90,7 @@ SYM_CODE_START(kernel_entry) # kernel entry point
PTR_ADDI sp, sp, -4 * SZREG # init stack pointer
bl start_kernel
+ ASM_BUG()
SYM_CODE_END(kernel_entry)
@@ -116,6 +122,8 @@ SYM_CODE_START(smpboot_entry)
ld.d tp, t0, CPU_BOOT_TINFO
bl start_secondary
+ ASM_BUG()
+
SYM_CODE_END(smpboot_entry)
#endif /* CONFIG_SMP */
diff --git a/arch/loongarch/kernel/machine_kexec.c b/arch/loongarch/kernel/machine_kexec.c
new file mode 100644
index 000000000000..2dcb9e003657
--- /dev/null
+++ b/arch/loongarch/kernel/machine_kexec.c
@@ -0,0 +1,304 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * machine_kexec.c for kexec
+ *
+ * Copyright (C) 2022 Loongson Technology Corporation Limited
+ */
+#include <linux/compiler.h>
+#include <linux/cpu.h>
+#include <linux/kexec.h>
+#include <linux/crash_dump.h>
+#include <linux/delay.h>
+#include <linux/irq.h>
+#include <linux/libfdt.h>
+#include <linux/mm.h>
+#include <linux/of_fdt.h>
+#include <linux/reboot.h>
+#include <linux/sched.h>
+#include <linux/sched/task_stack.h>
+
+#include <asm/bootinfo.h>
+#include <asm/cacheflush.h>
+#include <asm/page.h>
+
+/* 0x100000 ~ 0x200000 is safe */
+#define KEXEC_CONTROL_CODE TO_CACHE(0x100000UL)
+#define KEXEC_CMDLINE_ADDR TO_CACHE(0x108000UL)
+
+static unsigned long reboot_code_buffer;
+static cpumask_t cpus_in_crash = CPU_MASK_NONE;
+
+#ifdef CONFIG_SMP
+static void (*relocated_kexec_smp_wait)(void *);
+atomic_t kexec_ready_to_reboot = ATOMIC_INIT(0);
+#endif
+
+static unsigned long efi_boot;
+static unsigned long cmdline_ptr;
+static unsigned long systable_ptr;
+static unsigned long start_addr;
+static unsigned long first_ind_entry;
+
+static void kexec_image_info(const struct kimage *kimage)
+{
+ unsigned long i;
+
+ pr_debug("kexec kimage info:\n");
+ pr_debug("\ttype: %d\n", kimage->type);
+ pr_debug("\tstart: %lx\n", kimage->start);
+ pr_debug("\thead: %lx\n", kimage->head);
+ pr_debug("\tnr_segments: %lu\n", kimage->nr_segments);
+
+ for (i = 0; i < kimage->nr_segments; i++) {
+ pr_debug("\t segment[%lu]: %016lx - %016lx", i,
+ kimage->segment[i].mem,
+ kimage->segment[i].mem + kimage->segment[i].memsz);
+ pr_debug("\t\t0x%lx bytes, %lu pages\n",
+ (unsigned long)kimage->segment[i].memsz,
+ (unsigned long)kimage->segment[i].memsz / PAGE_SIZE);
+ }
+}
+
+int machine_kexec_prepare(struct kimage *kimage)
+{
+ int i;
+ char *bootloader = "kexec";
+ void *cmdline_ptr = (void *)KEXEC_CMDLINE_ADDR;
+
+ kexec_image_info(kimage);
+
+ kimage->arch.efi_boot = fw_arg0;
+ kimage->arch.systable_ptr = fw_arg2;
+
+ /* Find the command line */
+ for (i = 0; i < kimage->nr_segments; i++) {
+ if (!strncmp(bootloader, (char __user *)kimage->segment[i].buf, strlen(bootloader))) {
+ if (!copy_from_user(cmdline_ptr, kimage->segment[i].buf, COMMAND_LINE_SIZE))
+ kimage->arch.cmdline_ptr = (unsigned long)cmdline_ptr;
+ break;
+ }
+ }
+
+ if (!kimage->arch.cmdline_ptr) {
+ pr_err("Command line not included in the provided image\n");
+ return -EINVAL;
+ }
+
+ /* kexec/kdump need a safe page to save reboot_code_buffer */
+ kimage->control_code_page = virt_to_page((void *)KEXEC_CONTROL_CODE);
+
+ reboot_code_buffer = (unsigned long)page_address(kimage->control_code_page);
+ memcpy((void *)reboot_code_buffer, relocate_new_kernel, relocate_new_kernel_size);
+
+#ifdef CONFIG_SMP
+ /* All secondary cpus now may jump to kexec_smp_wait cycle */
+ relocated_kexec_smp_wait = reboot_code_buffer + (void *)(kexec_smp_wait - relocate_new_kernel);
+#endif
+
+ return 0;
+}
+
+void machine_kexec_cleanup(struct kimage *kimage)
+{
+}
+
+void kexec_reboot(void)
+{
+ do_kexec_t do_kexec = NULL;
+
+ /*
+ * We know we were online, and there will be no incoming IPIs at
+ * this point. Mark online again before rebooting so that the crash
+ * analysis tool will see us correctly.
+ */
+ set_cpu_online(smp_processor_id(), true);
+
+ /* Ensure remote CPUs observe that we're online before rebooting. */
+ smp_mb__after_atomic();
+
+ /*
+ * Make sure we get correct instructions written by the
+ * machine_kexec_prepare() CPU.
+ */
+ __asm__ __volatile__ ("\tibar 0\n"::);
+
+#ifdef CONFIG_SMP
+ /* All secondary cpus go to kexec_smp_wait */
+ if (smp_processor_id() > 0) {
+ relocated_kexec_smp_wait(NULL);
+ unreachable();
+ }
+#endif
+
+ do_kexec = (void *)reboot_code_buffer;
+ do_kexec(efi_boot, cmdline_ptr, systable_ptr, start_addr, first_ind_entry);
+
+ unreachable();
+}
+
+
+#ifdef CONFIG_SMP
+static void kexec_shutdown_secondary(void *regs)
+{
+ int cpu = smp_processor_id();
+
+ if (!cpu_online(cpu))
+ return;
+
+ /* We won't be sent IPIs any more. */
+ set_cpu_online(cpu, false);
+
+ local_irq_disable();
+ while (!atomic_read(&kexec_ready_to_reboot))
+ cpu_relax();
+
+ kexec_reboot();
+}
+
+static void crash_shutdown_secondary(void *passed_regs)
+{
+ int cpu = smp_processor_id();
+ struct pt_regs *regs = passed_regs;
+
+ /*
+ * If we are passed registers, use those. Otherwise get the
+ * regs from the last interrupt, which should be correct, as
+ * we are in an interrupt. But if the regs are not there,
+ * pull them from the top of the stack. They are probably
+ * wrong, but we need something to keep from crashing again.
+ */
+ if (!regs)
+ regs = get_irq_regs();
+ if (!regs)
+ regs = task_pt_regs(current);
+
+ if (!cpu_online(cpu))
+ return;
+
+ /* We won't be sent IPIs any more. */
+ set_cpu_online(cpu, false);
+
+ local_irq_disable();
+ if (!cpumask_test_cpu(cpu, &cpus_in_crash))
+ crash_save_cpu(regs, cpu);
+ cpumask_set_cpu(cpu, &cpus_in_crash);
+
+ while (!atomic_read(&kexec_ready_to_reboot))
+ cpu_relax();
+
+ kexec_reboot();
+}
+
+void crash_smp_send_stop(void)
+{
+ unsigned int ncpus;
+ unsigned long timeout;
+ static int cpus_stopped;
+
+ /*
+ * This function can be called twice in panic path, but obviously
+ * we should execute this only once.
+ */
+ if (cpus_stopped)
+ return;
+
+ cpus_stopped = 1;
+
+ /* Excluding the panic cpu */
+ ncpus = num_online_cpus() - 1;
+
+ smp_call_function(crash_shutdown_secondary, NULL, 0);
+ smp_wmb();
+
+ /*
+ * The crash CPU sends an IPI and wait for other CPUs to
+ * respond. Delay of at least 10 seconds.
+ */
+ timeout = MSEC_PER_SEC * 10;
+ pr_emerg("Sending IPI to other cpus...\n");
+ while ((cpumask_weight(&cpus_in_crash) < ncpus) && timeout--) {
+ mdelay(1);
+ cpu_relax();
+ }
+}
+#endif /* defined(CONFIG_SMP) */
+
+void machine_shutdown(void)
+{
+ int cpu;
+
+ /* All CPUs go to reboot_code_buffer */
+ for_each_possible_cpu(cpu)
+ if (!cpu_online(cpu))
+ cpu_device_up(get_cpu_device(cpu));
+
+#ifdef CONFIG_SMP
+ smp_call_function(kexec_shutdown_secondary, NULL, 0);
+#endif
+}
+
+void machine_crash_shutdown(struct pt_regs *regs)
+{
+ int crashing_cpu;
+
+ local_irq_disable();
+
+ crashing_cpu = smp_processor_id();
+ crash_save_cpu(regs, crashing_cpu);
+
+#ifdef CONFIG_SMP
+ crash_smp_send_stop();
+#endif
+ cpumask_set_cpu(crashing_cpu, &cpus_in_crash);
+
+ pr_info("Starting crashdump kernel...\n");
+}
+
+void machine_kexec(struct kimage *image)
+{
+ unsigned long entry, *ptr;
+ struct kimage_arch *internal = &image->arch;
+
+ efi_boot = internal->efi_boot;
+ cmdline_ptr = internal->cmdline_ptr;
+ systable_ptr = internal->systable_ptr;
+
+ start_addr = (unsigned long)phys_to_virt(image->start);
+
+ first_ind_entry = (image->type == KEXEC_TYPE_DEFAULT) ?
+ (unsigned long)phys_to_virt(image->head & PAGE_MASK) : 0;
+
+ /*
+ * The generic kexec code builds a page list with physical
+ * addresses. they are directly accessible through XKPRANGE
+ * hence the phys_to_virt() call.
+ */
+ for (ptr = &image->head; (entry = *ptr) && !(entry & IND_DONE);
+ ptr = (entry & IND_INDIRECTION) ?
+ phys_to_virt(entry & PAGE_MASK) : ptr + 1) {
+ if (*ptr & IND_SOURCE || *ptr & IND_INDIRECTION ||
+ *ptr & IND_DESTINATION)
+ *ptr = (unsigned long) phys_to_virt(*ptr);
+ }
+
+ /* Mark offline before disabling local irq. */
+ set_cpu_online(smp_processor_id(), false);
+
+ /* We do not want to be bothered. */
+ local_irq_disable();
+
+ pr_notice("EFI boot flag 0x%lx\n", efi_boot);
+ pr_notice("Command line at 0x%lx\n", cmdline_ptr);
+ pr_notice("System table at 0x%lx\n", systable_ptr);
+ pr_notice("We will call new kernel at 0x%lx\n", start_addr);
+ pr_notice("Bye ...\n");
+
+ /* Make reboot code buffer available to the boot CPU. */
+ flush_cache_all();
+
+#ifdef CONFIG_SMP
+ atomic_set(&kexec_ready_to_reboot, 1);
+#endif
+
+ kexec_reboot();
+}
diff --git a/arch/loongarch/kernel/mem.c b/arch/loongarch/kernel/mem.c
index 7423361b0ebc..4a4107a6a965 100644
--- a/arch/loongarch/kernel/mem.c
+++ b/arch/loongarch/kernel/mem.c
@@ -58,7 +58,4 @@ void __init memblock_init(void)
/* Reserve the kernel text/data/bss */
memblock_reserve(__pa_symbol(&_text),
__pa_symbol(&_end) - __pa_symbol(&_text));
-
- /* Reserve the initrd */
- reserve_initrd_mem();
}
diff --git a/arch/loongarch/kernel/module-sections.c b/arch/loongarch/kernel/module-sections.c
index 6d498288977d..d296a70b758f 100644
--- a/arch/loongarch/kernel/module-sections.c
+++ b/arch/loongarch/kernel/module-sections.c
@@ -7,7 +7,33 @@
#include <linux/kernel.h>
#include <linux/module.h>
-Elf_Addr module_emit_plt_entry(struct module *mod, unsigned long val)
+Elf_Addr module_emit_got_entry(struct module *mod, Elf_Addr val)
+{
+ struct mod_section *got_sec = &mod->arch.got;
+ int i = got_sec->num_entries;
+ struct got_entry *got = get_got_entry(val, got_sec);
+
+ if (got)
+ return (Elf_Addr)got;
+
+ /* There is no GOT entry for val yet, create a new one. */
+ got = (struct got_entry *)got_sec->shdr->sh_addr;
+ got[i] = emit_got_entry(val);
+
+ got_sec->num_entries++;
+ if (got_sec->num_entries > got_sec->max_entries) {
+ /*
+ * This may happen when the module contains a GOT_HI20 without
+ * a paired GOT_LO12. Such a module is broken, reject it.
+ */
+ pr_err("%s: module contains bad GOT relocation\n", mod->name);
+ return 0;
+ }
+
+ return (Elf_Addr)&got[i];
+}
+
+Elf_Addr module_emit_plt_entry(struct module *mod, Elf_Addr val)
{
int nr;
struct mod_section *plt_sec = &mod->arch.plt;
@@ -50,15 +76,25 @@ static bool duplicate_rela(const Elf_Rela *rela, int idx)
return false;
}
-static void count_max_entries(Elf_Rela *relas, int num, unsigned int *plts)
+static void count_max_entries(Elf_Rela *relas, int num,
+ unsigned int *plts, unsigned int *gots)
{
unsigned int i, type;
for (i = 0; i < num; i++) {
type = ELF_R_TYPE(relas[i].r_info);
- if (type == R_LARCH_SOP_PUSH_PLT_PCREL) {
+ switch (type) {
+ case R_LARCH_SOP_PUSH_PLT_PCREL:
+ case R_LARCH_B26:
if (!duplicate_rela(relas, i))
(*plts)++;
+ break;
+ case R_LARCH_GOT_PC_HI20:
+ if (!duplicate_rela(relas, i))
+ (*gots)++;
+ break;
+ default:
+ break; /* Do nothing. */
}
}
}
@@ -66,18 +102,24 @@ static void count_max_entries(Elf_Rela *relas, int num, unsigned int *plts)
int module_frob_arch_sections(Elf_Ehdr *ehdr, Elf_Shdr *sechdrs,
char *secstrings, struct module *mod)
{
- unsigned int i, num_plts = 0;
+ unsigned int i, num_plts = 0, num_gots = 0;
/*
* Find the empty .plt sections.
*/
for (i = 0; i < ehdr->e_shnum; i++) {
- if (!strcmp(secstrings + sechdrs[i].sh_name, ".plt"))
+ if (!strcmp(secstrings + sechdrs[i].sh_name, ".got"))
+ mod->arch.got.shdr = sechdrs + i;
+ else if (!strcmp(secstrings + sechdrs[i].sh_name, ".plt"))
mod->arch.plt.shdr = sechdrs + i;
else if (!strcmp(secstrings + sechdrs[i].sh_name, ".plt.idx"))
mod->arch.plt_idx.shdr = sechdrs + i;
}
+ if (!mod->arch.got.shdr) {
+ pr_err("%s: module GOT section(s) missing\n", mod->name);
+ return -ENOEXEC;
+ }
if (!mod->arch.plt.shdr) {
pr_err("%s: module PLT section(s) missing\n", mod->name);
return -ENOEXEC;
@@ -100,9 +142,16 @@ int module_frob_arch_sections(Elf_Ehdr *ehdr, Elf_Shdr *sechdrs,
if (!(dst_sec->sh_flags & SHF_EXECINSTR))
continue;
- count_max_entries(relas, num_rela, &num_plts);
+ count_max_entries(relas, num_rela, &num_plts, &num_gots);
}
+ mod->arch.got.shdr->sh_type = SHT_NOBITS;
+ mod->arch.got.shdr->sh_flags = SHF_ALLOC;
+ mod->arch.got.shdr->sh_addralign = L1_CACHE_BYTES;
+ mod->arch.got.shdr->sh_size = (num_gots + 1) * sizeof(struct got_entry);
+ mod->arch.got.num_entries = 0;
+ mod->arch.got.max_entries = num_gots;
+
mod->arch.plt.shdr->sh_type = SHT_NOBITS;
mod->arch.plt.shdr->sh_flags = SHF_EXECINSTR | SHF_ALLOC;
mod->arch.plt.shdr->sh_addralign = L1_CACHE_BYTES;
diff --git a/arch/loongarch/kernel/module.c b/arch/loongarch/kernel/module.c
index 638427ff0d51..097595b2fc14 100644
--- a/arch/loongarch/kernel/module.c
+++ b/arch/loongarch/kernel/module.c
@@ -18,16 +18,6 @@
#include <linux/string.h>
#include <linux/kernel.h>
-static inline bool signed_imm_check(long val, unsigned int bit)
-{
- return -(1L << (bit - 1)) <= val && val < (1L << (bit - 1));
-}
-
-static inline bool unsigned_imm_check(unsigned long val, unsigned int bit)
-{
- return val < (1UL << bit);
-}
-
static int rela_stack_push(s64 stack_value, s64 *rela_stack, size_t *rela_stack_top)
{
if (*rela_stack_top >= RELA_STACK_DEPTH)
@@ -281,6 +271,96 @@ static int apply_r_larch_add_sub(struct module *mod, u32 *location, Elf_Addr v,
}
}
+static int apply_r_larch_b26(struct module *mod, u32 *location, Elf_Addr v,
+ s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
+{
+ ptrdiff_t offset = (void *)v - (void *)location;
+ union loongarch_instruction *insn = (union loongarch_instruction *)location;
+
+ if (offset >= SZ_128M)
+ v = module_emit_plt_entry(mod, v);
+
+ if (offset < -SZ_128M)
+ v = module_emit_plt_entry(mod, v);
+
+ offset = (void *)v - (void *)location;
+
+ if (offset & 3) {
+ pr_err("module %s: jump offset = 0x%llx unaligned! dangerous R_LARCH_B26 (%u) relocation\n",
+ mod->name, (long long)offset, type);
+ return -ENOEXEC;
+ }
+
+ if (!signed_imm_check(offset, 28)) {
+ pr_err("module %s: jump offset = 0x%llx overflow! dangerous R_LARCH_B26 (%u) relocation\n",
+ mod->name, (long long)offset, type);
+ return -ENOEXEC;
+ }
+
+ offset >>= 2;
+ insn->reg0i26_format.immediate_l = offset & 0xffff;
+ insn->reg0i26_format.immediate_h = (offset >> 16) & 0x3ff;
+
+ return 0;
+}
+
+static int apply_r_larch_pcala(struct module *mod, u32 *location, Elf_Addr v,
+ s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
+{
+ union loongarch_instruction *insn = (union loongarch_instruction *)location;
+ /* Use s32 for a sign-extension deliberately. */
+ s32 offset_hi20 = (void *)((v + 0x800) & ~0xfff) -
+ (void *)((Elf_Addr)location & ~0xfff);
+ Elf_Addr anchor = (((Elf_Addr)location) & ~0xfff) + offset_hi20;
+ ptrdiff_t offset_rem = (void *)v - (void *)anchor;
+
+ switch (type) {
+ case R_LARCH_PCALA_LO12:
+ insn->reg2i12_format.immediate = v & 0xfff;
+ break;
+ case R_LARCH_PCALA_HI20:
+ v = offset_hi20 >> 12;
+ insn->reg1i20_format.immediate = v & 0xfffff;
+ break;
+ case R_LARCH_PCALA64_LO20:
+ v = offset_rem >> 32;
+ insn->reg1i20_format.immediate = v & 0xfffff;
+ break;
+ case R_LARCH_PCALA64_HI12:
+ v = offset_rem >> 52;
+ insn->reg2i12_format.immediate = v & 0xfff;
+ break;
+ default:
+ pr_err("%s: Unsupport relocation type %u\n", mod->name, type);
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static int apply_r_larch_got_pc(struct module *mod, u32 *location, Elf_Addr v,
+ s64 *rela_stack, size_t *rela_stack_top, unsigned int type)
+{
+ Elf_Addr got = module_emit_got_entry(mod, v);
+
+ if (!got)
+ return -EINVAL;
+
+ switch (type) {
+ case R_LARCH_GOT_PC_LO12:
+ type = R_LARCH_PCALA_LO12;
+ break;
+ case R_LARCH_GOT_PC_HI20:
+ type = R_LARCH_PCALA_HI20;
+ break;
+ default:
+ pr_err("%s: Unsupport relocation type %u\n", mod->name, type);
+ return -EINVAL;
+ }
+
+ return apply_r_larch_pcala(mod, location, got, rela_stack, rela_stack_top, type);
+}
+
/*
* reloc_handlers_rela() - Apply a particular relocation to a module
* @mod: the module to apply the reloc to
@@ -296,7 +376,7 @@ typedef int (*reloc_rela_handler)(struct module *mod, u32 *location, Elf_Addr v,
/* The handlers for known reloc types */
static reloc_rela_handler reloc_rela_handlers[] = {
- [R_LARCH_NONE ... R_LARCH_SUB64] = apply_r_larch_error,
+ [R_LARCH_NONE ... R_LARCH_RELAX] = apply_r_larch_error,
[R_LARCH_NONE] = apply_r_larch_none,
[R_LARCH_32] = apply_r_larch_32,
@@ -310,6 +390,9 @@ static reloc_rela_handler reloc_rela_handlers[] = {
[R_LARCH_SOP_SUB ... R_LARCH_SOP_IF_ELSE] = apply_r_larch_sop,
[R_LARCH_SOP_POP_32_S_10_5 ... R_LARCH_SOP_POP_32_U] = apply_r_larch_sop_imm_field,
[R_LARCH_ADD32 ... R_LARCH_SUB64] = apply_r_larch_add_sub,
+ [R_LARCH_B26] = apply_r_larch_b26,
+ [R_LARCH_PCALA_HI20...R_LARCH_PCALA64_HI12] = apply_r_larch_pcala,
+ [R_LARCH_GOT_PC_HI20...R_LARCH_GOT_PC_LO12] = apply_r_larch_got_pc,
};
int apply_relocate_add(Elf_Shdr *sechdrs, const char *strtab,
diff --git a/arch/loongarch/kernel/perf_event.c b/arch/loongarch/kernel/perf_event.c
new file mode 100644
index 000000000000..707bd32e5c4f
--- /dev/null
+++ b/arch/loongarch/kernel/perf_event.c
@@ -0,0 +1,887 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Linux performance counter support for LoongArch.
+ *
+ * Copyright (C) 2022 Loongson Technology Corporation Limited
+ *
+ * Derived from MIPS:
+ * Copyright (C) 2010 MIPS Technologies, Inc.
+ * Copyright (C) 2011 Cavium Networks, Inc.
+ * Author: Deng-Cheng Zhu
+ */
+
+#include <linux/cpumask.h>
+#include <linux/interrupt.h>
+#include <linux/smp.h>
+#include <linux/kernel.h>
+#include <linux/perf_event.h>
+#include <linux/uaccess.h>
+#include <linux/sched/task_stack.h>
+
+#include <asm/irq.h>
+#include <asm/irq_regs.h>
+#include <asm/stacktrace.h>
+#include <asm/unwind.h>
+
+/*
+ * Get the return address for a single stackframe and return a pointer to the
+ * next frame tail.
+ */
+static unsigned long
+user_backtrace(struct perf_callchain_entry_ctx *entry, unsigned long fp)
+{
+ unsigned long err;
+ unsigned long __user *user_frame_tail;
+ struct stack_frame buftail;
+
+ user_frame_tail = (unsigned long __user *)(fp - sizeof(struct stack_frame));
+
+ /* Also check accessibility of one struct frame_tail beyond */
+ if (!access_ok(user_frame_tail, sizeof(buftail)))
+ return 0;
+
+ pagefault_disable();
+ err = __copy_from_user_inatomic(&buftail, user_frame_tail, sizeof(buftail));
+ pagefault_enable();
+
+ if (err || (unsigned long)user_frame_tail >= buftail.fp)
+ return 0;
+
+ perf_callchain_store(entry, buftail.ra);
+
+ return buftail.fp;
+}
+
+void perf_callchain_user(struct perf_callchain_entry_ctx *entry,
+ struct pt_regs *regs)
+{
+ unsigned long fp;
+
+ if (perf_guest_state()) {
+ /* We don't support guest os callchain now */
+ return;
+ }
+
+ perf_callchain_store(entry, regs->csr_era);
+
+ fp = regs->regs[22];
+
+ while (entry->nr < entry->max_stack && fp && !((unsigned long)fp & 0xf))
+ fp = user_backtrace(entry, fp);
+}
+
+void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry,
+ struct pt_regs *regs)
+{
+ struct unwind_state state;
+ unsigned long addr;
+
+ for (unwind_start(&state, current, regs);
+ !unwind_done(&state); unwind_next_frame(&state)) {
+ addr = unwind_get_return_address(&state);
+ if (!addr || perf_callchain_store(entry, addr))
+ return;
+ }
+}
+
+#define LOONGARCH_MAX_HWEVENTS 32
+
+struct cpu_hw_events {
+ /* Array of events on this cpu. */
+ struct perf_event *events[LOONGARCH_MAX_HWEVENTS];
+
+ /*
+ * Set the bit (indexed by the counter number) when the counter
+ * is used for an event.
+ */
+ unsigned long used_mask[BITS_TO_LONGS(LOONGARCH_MAX_HWEVENTS)];
+
+ /*
+ * Software copy of the control register for each performance counter.
+ */
+ unsigned int saved_ctrl[LOONGARCH_MAX_HWEVENTS];
+};
+static DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = {
+ .saved_ctrl = {0},
+};
+
+/* The description of LoongArch performance events. */
+struct loongarch_perf_event {
+ unsigned int event_id;
+};
+
+static struct loongarch_perf_event raw_event;
+static DEFINE_MUTEX(raw_event_mutex);
+
+#define C(x) PERF_COUNT_HW_CACHE_##x
+#define HW_OP_UNSUPPORTED 0xffffffff
+#define CACHE_OP_UNSUPPORTED 0xffffffff
+
+#define PERF_MAP_ALL_UNSUPPORTED \
+ [0 ... PERF_COUNT_HW_MAX - 1] = {HW_OP_UNSUPPORTED}
+
+#define PERF_CACHE_MAP_ALL_UNSUPPORTED \
+[0 ... C(MAX) - 1] = { \
+ [0 ... C(OP_MAX) - 1] = { \
+ [0 ... C(RESULT_MAX) - 1] = {CACHE_OP_UNSUPPORTED}, \
+ }, \
+}
+
+struct loongarch_pmu {
+ u64 max_period;
+ u64 valid_count;
+ u64 overflow;
+ const char *name;
+ unsigned int num_counters;
+ u64 (*read_counter)(unsigned int idx);
+ void (*write_counter)(unsigned int idx, u64 val);
+ const struct loongarch_perf_event *(*map_raw_event)(u64 config);
+ const struct loongarch_perf_event (*general_event_map)[PERF_COUNT_HW_MAX];
+ const struct loongarch_perf_event (*cache_event_map)
+ [PERF_COUNT_HW_CACHE_MAX]
+ [PERF_COUNT_HW_CACHE_OP_MAX]
+ [PERF_COUNT_HW_CACHE_RESULT_MAX];
+};
+
+static struct loongarch_pmu loongarch_pmu;
+
+#define M_PERFCTL_EVENT(event) (event & CSR_PERFCTRL_EVENT)
+
+#define M_PERFCTL_COUNT_EVENT_WHENEVER (CSR_PERFCTRL_PLV0 | \
+ CSR_PERFCTRL_PLV1 | \
+ CSR_PERFCTRL_PLV2 | \
+ CSR_PERFCTRL_PLV3 | \
+ CSR_PERFCTRL_IE)
+
+#define M_PERFCTL_CONFIG_MASK 0x1f0000
+
+static void pause_local_counters(void);
+static void resume_local_counters(void);
+
+static u64 loongarch_pmu_read_counter(unsigned int idx)
+{
+ u64 val = -1;
+
+ switch (idx) {
+ case 0:
+ val = read_csr_perfcntr0();
+ break;
+ case 1:
+ val = read_csr_perfcntr1();
+ break;
+ case 2:
+ val = read_csr_perfcntr2();
+ break;
+ case 3:
+ val = read_csr_perfcntr3();
+ break;
+ default:
+ WARN_ONCE(1, "Invalid performance counter number (%d)\n", idx);
+ return 0;
+ }
+
+ return val;
+}
+
+static void loongarch_pmu_write_counter(unsigned int idx, u64 val)
+{
+ switch (idx) {
+ case 0:
+ write_csr_perfcntr0(val);
+ return;
+ case 1:
+ write_csr_perfcntr1(val);
+ return;
+ case 2:
+ write_csr_perfcntr2(val);
+ return;
+ case 3:
+ write_csr_perfcntr3(val);
+ return;
+ default:
+ WARN_ONCE(1, "Invalid performance counter number (%d)\n", idx);
+ return;
+ }
+}
+
+static unsigned int loongarch_pmu_read_control(unsigned int idx)
+{
+ unsigned int val = -1;
+
+ switch (idx) {
+ case 0:
+ val = read_csr_perfctrl0();
+ break;
+ case 1:
+ val = read_csr_perfctrl1();
+ break;
+ case 2:
+ val = read_csr_perfctrl2();
+ break;
+ case 3:
+ val = read_csr_perfctrl3();
+ break;
+ default:
+ WARN_ONCE(1, "Invalid performance counter number (%d)\n", idx);
+ return 0;
+ }
+
+ return val;
+}
+
+static void loongarch_pmu_write_control(unsigned int idx, unsigned int val)
+{
+ switch (idx) {
+ case 0:
+ write_csr_perfctrl0(val);
+ return;
+ case 1:
+ write_csr_perfctrl1(val);
+ return;
+ case 2:
+ write_csr_perfctrl2(val);
+ return;
+ case 3:
+ write_csr_perfctrl3(val);
+ return;
+ default:
+ WARN_ONCE(1, "Invalid performance counter number (%d)\n", idx);
+ return;
+ }
+}
+
+static int loongarch_pmu_alloc_counter(struct cpu_hw_events *cpuc, struct hw_perf_event *hwc)
+{
+ int i;
+
+ for (i = 0; i < loongarch_pmu.num_counters; i++) {
+ if (!test_and_set_bit(i, cpuc->used_mask))
+ return i;
+ }
+
+ return -EAGAIN;
+}
+
+static void loongarch_pmu_enable_event(struct hw_perf_event *evt, int idx)
+{
+ unsigned int cpu;
+ struct perf_event *event = container_of(evt, struct perf_event, hw);
+ struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
+
+ WARN_ON(idx < 0 || idx >= loongarch_pmu.num_counters);
+
+ /* Make sure interrupt enabled. */
+ cpuc->saved_ctrl[idx] = M_PERFCTL_EVENT(evt->event_base & 0xff) |
+ (evt->config_base & M_PERFCTL_CONFIG_MASK) | CSR_PERFCTRL_IE;
+
+ cpu = (event->cpu >= 0) ? event->cpu : smp_processor_id();
+
+ /*
+ * We do not actually let the counter run. Leave it until start().
+ */
+ pr_debug("Enabling perf counter for CPU%d\n", cpu);
+}
+
+static void loongarch_pmu_disable_event(int idx)
+{
+ unsigned long flags;
+ struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
+
+ WARN_ON(idx < 0 || idx >= loongarch_pmu.num_counters);
+
+ local_irq_save(flags);
+ cpuc->saved_ctrl[idx] = loongarch_pmu_read_control(idx) &
+ ~M_PERFCTL_COUNT_EVENT_WHENEVER;
+ loongarch_pmu_write_control(idx, cpuc->saved_ctrl[idx]);
+ local_irq_restore(flags);
+}
+
+static int loongarch_pmu_event_set_period(struct perf_event *event,
+ struct hw_perf_event *hwc,
+ int idx)
+{
+ int ret = 0;
+ u64 left = local64_read(&hwc->period_left);
+ u64 period = hwc->sample_period;
+
+ if (unlikely((left + period) & (1ULL << 63))) {
+ /* left underflowed by more than period. */
+ left = period;
+ local64_set(&hwc->period_left, left);
+ hwc->last_period = period;
+ ret = 1;
+ } else if (unlikely((left + period) <= period)) {
+ /* left underflowed by less than period. */
+ left += period;
+ local64_set(&hwc->period_left, left);
+ hwc->last_period = period;
+ ret = 1;
+ }
+
+ if (left > loongarch_pmu.max_period) {
+ left = loongarch_pmu.max_period;
+ local64_set(&hwc->period_left, left);
+ }
+
+ local64_set(&hwc->prev_count, loongarch_pmu.overflow - left);
+
+ loongarch_pmu.write_counter(idx, loongarch_pmu.overflow - left);
+
+ perf_event_update_userpage(event);
+
+ return ret;
+}
+
+static void loongarch_pmu_event_update(struct perf_event *event,
+ struct hw_perf_event *hwc,
+ int idx)
+{
+ u64 delta;
+ u64 prev_raw_count, new_raw_count;
+
+again:
+ prev_raw_count = local64_read(&hwc->prev_count);
+ new_raw_count = loongarch_pmu.read_counter(idx);
+
+ if (local64_cmpxchg(&hwc->prev_count, prev_raw_count,
+ new_raw_count) != prev_raw_count)
+ goto again;
+
+ delta = new_raw_count - prev_raw_count;
+
+ local64_add(delta, &event->count);
+ local64_sub(delta, &hwc->period_left);
+}
+
+static void loongarch_pmu_start(struct perf_event *event, int flags)
+{
+ struct hw_perf_event *hwc = &event->hw;
+
+ if (flags & PERF_EF_RELOAD)
+ WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE));
+
+ hwc->state = 0;
+
+ /* Set the period for the event. */
+ loongarch_pmu_event_set_period(event, hwc, hwc->idx);
+
+ /* Enable the event. */
+ loongarch_pmu_enable_event(hwc, hwc->idx);
+}
+
+static void loongarch_pmu_stop(struct perf_event *event, int flags)
+{
+ struct hw_perf_event *hwc = &event->hw;
+
+ if (!(hwc->state & PERF_HES_STOPPED)) {
+ /* We are working on a local event. */
+ loongarch_pmu_disable_event(hwc->idx);
+ barrier();
+ loongarch_pmu_event_update(event, hwc, hwc->idx);
+ hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;
+ }
+}
+
+static int loongarch_pmu_add(struct perf_event *event, int flags)
+{
+ int idx, err = 0;
+ struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
+ struct hw_perf_event *hwc = &event->hw;
+
+ perf_pmu_disable(event->pmu);
+
+ /* To look for a free counter for this event. */
+ idx = loongarch_pmu_alloc_counter(cpuc, hwc);
+ if (idx < 0) {
+ err = idx;
+ goto out;
+ }
+
+ /*
+ * If there is an event in the counter we are going to use then
+ * make sure it is disabled.
+ */
+ event->hw.idx = idx;
+ loongarch_pmu_disable_event(idx);
+ cpuc->events[idx] = event;
+
+ hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
+ if (flags & PERF_EF_START)
+ loongarch_pmu_start(event, PERF_EF_RELOAD);
+
+ /* Propagate our changes to the userspace mapping. */
+ perf_event_update_userpage(event);
+
+out:
+ perf_pmu_enable(event->pmu);
+ return err;
+}
+
+static void loongarch_pmu_del(struct perf_event *event, int flags)
+{
+ struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
+ struct hw_perf_event *hwc = &event->hw;
+ int idx = hwc->idx;
+
+ WARN_ON(idx < 0 || idx >= loongarch_pmu.num_counters);
+
+ loongarch_pmu_stop(event, PERF_EF_UPDATE);
+ cpuc->events[idx] = NULL;
+ clear_bit(idx, cpuc->used_mask);
+
+ perf_event_update_userpage(event);
+}
+
+static void loongarch_pmu_read(struct perf_event *event)
+{
+ struct hw_perf_event *hwc = &event->hw;
+
+ /* Don't read disabled counters! */
+ if (hwc->idx < 0)
+ return;
+
+ loongarch_pmu_event_update(event, hwc, hwc->idx);
+}
+
+static void loongarch_pmu_enable(struct pmu *pmu)
+{
+ resume_local_counters();
+}
+
+static void loongarch_pmu_disable(struct pmu *pmu)
+{
+ pause_local_counters();
+}
+
+static DEFINE_MUTEX(pmu_reserve_mutex);
+static atomic_t active_events = ATOMIC_INIT(0);
+
+static int get_pmc_irq(void)
+{
+ struct irq_domain *d = irq_find_matching_fwnode(cpuintc_handle, DOMAIN_BUS_ANY);
+
+ if (d)
+ return irq_create_mapping(d, EXCCODE_PMC - EXCCODE_INT_START);
+
+ return -EINVAL;
+}
+
+static void reset_counters(void *arg);
+static int __hw_perf_event_init(struct perf_event *event);
+
+static void hw_perf_event_destroy(struct perf_event *event)
+{
+ if (atomic_dec_and_mutex_lock(&active_events, &pmu_reserve_mutex)) {
+ on_each_cpu(reset_counters, NULL, 1);
+ free_irq(get_pmc_irq(), &loongarch_pmu);
+ mutex_unlock(&pmu_reserve_mutex);
+ }
+}
+
+static void handle_associated_event(struct cpu_hw_events *cpuc, int idx,
+ struct perf_sample_data *data, struct pt_regs *regs)
+{
+ struct perf_event *event = cpuc->events[idx];
+ struct hw_perf_event *hwc = &event->hw;
+
+ loongarch_pmu_event_update(event, hwc, idx);
+ data->period = event->hw.last_period;
+ if (!loongarch_pmu_event_set_period(event, hwc, idx))
+ return;
+
+ if (perf_event_overflow(event, data, regs))
+ loongarch_pmu_disable_event(idx);
+}
+
+static irqreturn_t pmu_handle_irq(int irq, void *dev)
+{
+ int n;
+ int handled = IRQ_NONE;
+ uint64_t counter;
+ struct pt_regs *regs;
+ struct perf_sample_data data;
+ struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
+
+ /*
+ * First we pause the local counters, so that when we are locked
+ * here, the counters are all paused. When it gets locked due to
+ * perf_disable(), the timer interrupt handler will be delayed.
+ *
+ * See also loongarch_pmu_start().
+ */
+ pause_local_counters();
+
+ regs = get_irq_regs();
+
+ perf_sample_data_init(&data, 0, 0);
+
+ for (n = 0; n < loongarch_pmu.num_counters; n++) {
+ if (test_bit(n, cpuc->used_mask)) {
+ counter = loongarch_pmu.read_counter(n);
+ if (counter & loongarch_pmu.overflow) {
+ handle_associated_event(cpuc, n, &data, regs);
+ handled = IRQ_HANDLED;
+ }
+ }
+ }
+
+ resume_local_counters();
+
+ /*
+ * Do all the work for the pending perf events. We can do this
+ * in here because the performance counter interrupt is a regular
+ * interrupt, not NMI.
+ */
+ if (handled == IRQ_HANDLED)
+ irq_work_run();
+
+ return handled;
+}
+
+static int loongarch_pmu_event_init(struct perf_event *event)
+{
+ int r, irq;
+ unsigned long flags;
+
+ /* does not support taken branch sampling */
+ if (has_branch_stack(event))
+ return -EOPNOTSUPP;
+
+ switch (event->attr.type) {
+ case PERF_TYPE_RAW:
+ case PERF_TYPE_HARDWARE:
+ case PERF_TYPE_HW_CACHE:
+ break;
+
+ default:
+ /* Init it to avoid false validate_group */
+ event->hw.event_base = 0xffffffff;
+ return -ENOENT;
+ }
+
+ if (event->cpu >= 0 && !cpu_online(event->cpu))
+ return -ENODEV;
+
+ irq = get_pmc_irq();
+ flags = IRQF_PERCPU | IRQF_NOBALANCING | IRQF_NO_THREAD | IRQF_NO_SUSPEND | IRQF_SHARED;
+ if (!atomic_inc_not_zero(&active_events)) {
+ mutex_lock(&pmu_reserve_mutex);
+ if (atomic_read(&active_events) == 0) {
+ r = request_irq(irq, pmu_handle_irq, flags, "Perf_PMU", &loongarch_pmu);
+ if (r < 0) {
+ mutex_unlock(&pmu_reserve_mutex);
+ pr_warn("PMU IRQ request failed\n");
+ return -ENODEV;
+ }
+ }
+ atomic_inc(&active_events);
+ mutex_unlock(&pmu_reserve_mutex);
+ }
+
+ return __hw_perf_event_init(event);
+}
+
+static struct pmu pmu = {
+ .pmu_enable = loongarch_pmu_enable,
+ .pmu_disable = loongarch_pmu_disable,
+ .event_init = loongarch_pmu_event_init,
+ .add = loongarch_pmu_add,
+ .del = loongarch_pmu_del,
+ .start = loongarch_pmu_start,
+ .stop = loongarch_pmu_stop,
+ .read = loongarch_pmu_read,
+};
+
+static unsigned int loongarch_pmu_perf_event_encode(const struct loongarch_perf_event *pev)
+{
+ return (pev->event_id & 0xff);
+}
+
+static const struct loongarch_perf_event *loongarch_pmu_map_general_event(int idx)
+{
+ const struct loongarch_perf_event *pev;
+
+ pev = &(*loongarch_pmu.general_event_map)[idx];
+
+ if (pev->event_id == HW_OP_UNSUPPORTED)
+ return ERR_PTR(-ENOENT);
+
+ return pev;
+}
+
+static const struct loongarch_perf_event *loongarch_pmu_map_cache_event(u64 config)
+{
+ unsigned int cache_type, cache_op, cache_result;
+ const struct loongarch_perf_event *pev;
+
+ cache_type = (config >> 0) & 0xff;
+ if (cache_type >= PERF_COUNT_HW_CACHE_MAX)
+ return ERR_PTR(-EINVAL);
+
+ cache_op = (config >> 8) & 0xff;
+ if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX)
+ return ERR_PTR(-EINVAL);
+
+ cache_result = (config >> 16) & 0xff;
+ if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
+ return ERR_PTR(-EINVAL);
+
+ pev = &((*loongarch_pmu.cache_event_map)
+ [cache_type]
+ [cache_op]
+ [cache_result]);
+
+ if (pev->event_id == CACHE_OP_UNSUPPORTED)
+ return ERR_PTR(-ENOENT);
+
+ return pev;
+}
+
+static int validate_group(struct perf_event *event)
+{
+ struct cpu_hw_events fake_cpuc;
+ struct perf_event *sibling, *leader = event->group_leader;
+
+ memset(&fake_cpuc, 0, sizeof(fake_cpuc));
+
+ if (loongarch_pmu_alloc_counter(&fake_cpuc, &leader->hw) < 0)
+ return -EINVAL;
+
+ for_each_sibling_event(sibling, leader) {
+ if (loongarch_pmu_alloc_counter(&fake_cpuc, &sibling->hw) < 0)
+ return -EINVAL;
+ }
+
+ if (loongarch_pmu_alloc_counter(&fake_cpuc, &event->hw) < 0)
+ return -EINVAL;
+
+ return 0;
+}
+
+static void reset_counters(void *arg)
+{
+ int n;
+ int counters = loongarch_pmu.num_counters;
+
+ for (n = 0; n < counters; n++) {
+ loongarch_pmu_write_control(n, 0);
+ loongarch_pmu.write_counter(n, 0);
+ }
+}
+
+static const struct loongarch_perf_event loongson_event_map[PERF_COUNT_HW_MAX] = {
+ PERF_MAP_ALL_UNSUPPORTED,
+ [PERF_COUNT_HW_CPU_CYCLES] = { 0x00 },
+ [PERF_COUNT_HW_INSTRUCTIONS] = { 0x01 },
+ [PERF_COUNT_HW_CACHE_REFERENCES] = { 0x08 },
+ [PERF_COUNT_HW_CACHE_MISSES] = { 0x09 },
+ [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = { 0x02 },
+ [PERF_COUNT_HW_BRANCH_MISSES] = { 0x03 },
+};
+
+static const struct loongarch_perf_event loongson_cache_map
+ [PERF_COUNT_HW_CACHE_MAX]
+ [PERF_COUNT_HW_CACHE_OP_MAX]
+ [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
+PERF_CACHE_MAP_ALL_UNSUPPORTED,
+[C(L1D)] = {
+ /*
+ * Like some other architectures (e.g. ARM), the performance
+ * counters don't differentiate between read and write
+ * accesses/misses, so this isn't strictly correct, but it's the
+ * best we can do. Writes and reads get combined.
+ */
+ [C(OP_READ)] = {
+ [C(RESULT_ACCESS)] = { 0x8 },
+ [C(RESULT_MISS)] = { 0x9 },
+ },
+ [C(OP_WRITE)] = {
+ [C(RESULT_ACCESS)] = { 0x8 },
+ [C(RESULT_MISS)] = { 0x9 },
+ },
+ [C(OP_PREFETCH)] = {
+ [C(RESULT_ACCESS)] = { 0xaa },
+ [C(RESULT_MISS)] = { 0xa9 },
+ },
+},
+[C(L1I)] = {
+ [C(OP_READ)] = {
+ [C(RESULT_ACCESS)] = { 0x6 },
+ [C(RESULT_MISS)] = { 0x7 },
+ },
+},
+[C(LL)] = {
+ [C(OP_READ)] = {
+ [C(RESULT_ACCESS)] = { 0xc },
+ [C(RESULT_MISS)] = { 0xd },
+ },
+ [C(OP_WRITE)] = {
+ [C(RESULT_ACCESS)] = { 0xc },
+ [C(RESULT_MISS)] = { 0xd },
+ },
+},
+[C(ITLB)] = {
+ [C(OP_READ)] = {
+ [C(RESULT_MISS)] = { 0x3b },
+ },
+},
+[C(DTLB)] = {
+ [C(OP_READ)] = {
+ [C(RESULT_ACCESS)] = { 0x4 },
+ [C(RESULT_MISS)] = { 0x3c },
+ },
+ [C(OP_WRITE)] = {
+ [C(RESULT_ACCESS)] = { 0x4 },
+ [C(RESULT_MISS)] = { 0x3c },
+ },
+},
+[C(BPU)] = {
+ /* Using the same code for *HW_BRANCH* */
+ [C(OP_READ)] = {
+ [C(RESULT_ACCESS)] = { 0x02 },
+ [C(RESULT_MISS)] = { 0x03 },
+ },
+},
+};
+
+static int __hw_perf_event_init(struct perf_event *event)
+{
+ int err;
+ struct hw_perf_event *hwc = &event->hw;
+ struct perf_event_attr *attr = &event->attr;
+ const struct loongarch_perf_event *pev;
+
+ /* Returning LoongArch event descriptor for generic perf event. */
+ if (PERF_TYPE_HARDWARE == event->attr.type) {
+ if (event->attr.config >= PERF_COUNT_HW_MAX)
+ return -EINVAL;
+ pev = loongarch_pmu_map_general_event(event->attr.config);
+ } else if (PERF_TYPE_HW_CACHE == event->attr.type) {
+ pev = loongarch_pmu_map_cache_event(event->attr.config);
+ } else if (PERF_TYPE_RAW == event->attr.type) {
+ /* We are working on the global raw event. */
+ mutex_lock(&raw_event_mutex);
+ pev = loongarch_pmu.map_raw_event(event->attr.config);
+ } else {
+ /* The event type is not (yet) supported. */
+ return -EOPNOTSUPP;
+ }
+
+ if (IS_ERR(pev)) {
+ if (PERF_TYPE_RAW == event->attr.type)
+ mutex_unlock(&raw_event_mutex);
+ return PTR_ERR(pev);
+ }
+
+ /*
+ * We allow max flexibility on how each individual counter shared
+ * by the single CPU operates (the mode exclusion and the range).
+ */
+ hwc->config_base = CSR_PERFCTRL_IE;
+
+ hwc->event_base = loongarch_pmu_perf_event_encode(pev);
+ if (PERF_TYPE_RAW == event->attr.type)
+ mutex_unlock(&raw_event_mutex);
+
+ if (!attr->exclude_user) {
+ hwc->config_base |= CSR_PERFCTRL_PLV3;
+ hwc->config_base |= CSR_PERFCTRL_PLV2;
+ }
+ if (!attr->exclude_kernel) {
+ hwc->config_base |= CSR_PERFCTRL_PLV0;
+ }
+ if (!attr->exclude_hv) {
+ hwc->config_base |= CSR_PERFCTRL_PLV1;
+ }
+
+ hwc->config_base &= M_PERFCTL_CONFIG_MASK;
+ /*
+ * The event can belong to another cpu. We do not assign a local
+ * counter for it for now.
+ */
+ hwc->idx = -1;
+ hwc->config = 0;
+
+ if (!hwc->sample_period) {
+ hwc->sample_period = loongarch_pmu.max_period;
+ hwc->last_period = hwc->sample_period;
+ local64_set(&hwc->period_left, hwc->sample_period);
+ }
+
+ err = 0;
+ if (event->group_leader != event)
+ err = validate_group(event);
+
+ event->destroy = hw_perf_event_destroy;
+
+ if (err)
+ event->destroy(event);
+
+ return err;
+}
+
+static void pause_local_counters(void)
+{
+ unsigned long flags;
+ int ctr = loongarch_pmu.num_counters;
+ struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
+
+ local_irq_save(flags);
+ do {
+ ctr--;
+ cpuc->saved_ctrl[ctr] = loongarch_pmu_read_control(ctr);
+ loongarch_pmu_write_control(ctr, cpuc->saved_ctrl[ctr] &
+ ~M_PERFCTL_COUNT_EVENT_WHENEVER);
+ } while (ctr > 0);
+ local_irq_restore(flags);
+}
+
+static void resume_local_counters(void)
+{
+ int ctr = loongarch_pmu.num_counters;
+ struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
+
+ do {
+ ctr--;
+ loongarch_pmu_write_control(ctr, cpuc->saved_ctrl[ctr]);
+ } while (ctr > 0);
+}
+
+static const struct loongarch_perf_event *loongarch_pmu_map_raw_event(u64 config)
+{
+ raw_event.event_id = config & 0xff;
+
+ return &raw_event;
+}
+
+static int __init init_hw_perf_events(void)
+{
+ int counters;
+
+ if (!cpu_has_pmp)
+ return -ENODEV;
+
+ pr_info("Performance counters: ");
+ counters = ((read_cpucfg(LOONGARCH_CPUCFG6) & CPUCFG6_PMNUM) >> 4) + 1;
+
+ loongarch_pmu.num_counters = counters;
+ loongarch_pmu.max_period = (1ULL << 63) - 1;
+ loongarch_pmu.valid_count = (1ULL << 63) - 1;
+ loongarch_pmu.overflow = 1ULL << 63;
+ loongarch_pmu.name = "loongarch/loongson64";
+ loongarch_pmu.read_counter = loongarch_pmu_read_counter;
+ loongarch_pmu.write_counter = loongarch_pmu_write_counter;
+ loongarch_pmu.map_raw_event = loongarch_pmu_map_raw_event;
+ loongarch_pmu.general_event_map = &loongson_event_map;
+ loongarch_pmu.cache_event_map = &loongson_cache_map;
+
+ on_each_cpu(reset_counters, NULL, 1);
+
+ pr_cont("%s PMU enabled, %d %d-bit counters available to each CPU.\n",
+ loongarch_pmu.name, counters, 64);
+
+ perf_pmu_register(&pmu, "cpu", PERF_TYPE_RAW);
+
+ return 0;
+}
+early_initcall(init_hw_perf_events);
diff --git a/arch/loongarch/kernel/perf_regs.c b/arch/loongarch/kernel/perf_regs.c
new file mode 100644
index 000000000000..263ac4ab5af6
--- /dev/null
+++ b/arch/loongarch/kernel/perf_regs.c
@@ -0,0 +1,53 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2022 Loongson Technology Corporation Limited
+ *
+ * Derived from MIPS:
+ * Copyright (C) 2013 Cavium, Inc.
+ */
+
+#include <linux/perf_event.h>
+
+#include <asm/ptrace.h>
+
+#ifdef CONFIG_32BIT
+u64 perf_reg_abi(struct task_struct *tsk)
+{
+ return PERF_SAMPLE_REGS_ABI_32;
+}
+#else /* Must be CONFIG_64BIT */
+u64 perf_reg_abi(struct task_struct *tsk)
+{
+ if (test_tsk_thread_flag(tsk, TIF_32BIT_REGS))
+ return PERF_SAMPLE_REGS_ABI_32;
+ else
+ return PERF_SAMPLE_REGS_ABI_64;
+}
+#endif /* CONFIG_32BIT */
+
+int perf_reg_validate(u64 mask)
+{
+ if (!mask)
+ return -EINVAL;
+ if (mask & ~((1ull << PERF_REG_LOONGARCH_MAX) - 1))
+ return -EINVAL;
+ return 0;
+}
+
+u64 perf_reg_value(struct pt_regs *regs, int idx)
+{
+ if (WARN_ON_ONCE((u32)idx >= PERF_REG_LOONGARCH_MAX))
+ return 0;
+
+ if ((u32)idx == PERF_REG_LOONGARCH_PC)
+ return regs->csr_era;
+
+ return regs->regs[idx];
+}
+
+void perf_get_regs_user(struct perf_regs *regs_user,
+ struct pt_regs *regs)
+{
+ regs_user->regs = task_pt_regs(current);
+ regs_user->abi = perf_reg_abi(current);
+}
diff --git a/arch/loongarch/kernel/relocate_kernel.S b/arch/loongarch/kernel/relocate_kernel.S
new file mode 100644
index 000000000000..d13252553a7c
--- /dev/null
+++ b/arch/loongarch/kernel/relocate_kernel.S
@@ -0,0 +1,112 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * relocate_kernel.S for kexec
+ *
+ * Copyright (C) 2022 Loongson Technology Corporation Limited
+ */
+
+#include <linux/kexec.h>
+
+#include <asm/asm.h>
+#include <asm/asmmacro.h>
+#include <asm/regdef.h>
+#include <asm/loongarch.h>
+#include <asm/stackframe.h>
+#include <asm/addrspace.h>
+
+SYM_CODE_START(relocate_new_kernel)
+ /*
+ * a0: EFI boot flag for the new kernel
+ * a1: Command line pointer for the new kernel
+ * a2: System table pointer for the new kernel
+ * a3: Start address to jump to after relocation
+ * a4: Pointer to the current indirection page entry
+ */
+ move s0, a4
+
+ /*
+ * In case of a kdump/crash kernel, the indirection page is not
+ * populated as the kernel is directly copied to a reserved location
+ */
+ beqz s0, done
+
+process_entry:
+ PTR_L s1, s0, 0
+ PTR_ADDI s0, s0, SZREG
+
+ /* destination page */
+ andi s2, s1, IND_DESTINATION
+ beqz s2, 1f
+ li.w t0, ~0x1
+ and s3, s1, t0 /* store destination addr in s3 */
+ b process_entry
+
+1:
+ /* indirection page, update s0 */
+ andi s2, s1, IND_INDIRECTION
+ beqz s2, 1f
+ li.w t0, ~0x2
+ and s0, s1, t0
+ b process_entry
+
+1:
+ /* done page */
+ andi s2, s1, IND_DONE
+ beqz s2, 1f
+ b done
+
+1:
+ /* source page */
+ andi s2, s1, IND_SOURCE
+ beqz s2, process_entry
+ li.w t0, ~0x8
+ and s1, s1, t0
+ li.w s5, (1 << _PAGE_SHIFT) / SZREG
+
+copy_word:
+ /* copy page word by word */
+ REG_L s4, s1, 0
+ REG_S s4, s3, 0
+ PTR_ADDI s3, s3, SZREG
+ PTR_ADDI s1, s1, SZREG
+ LONG_ADDI s5, s5, -1
+ beqz s5, process_entry
+ b copy_word
+ b process_entry
+
+done:
+ ibar 0
+ dbar 0
+
+ /*
+ * Jump to the new kernel,
+ * make sure the values of a0, a1, a2 and a3 are not changed.
+ */
+ jr a3
+SYM_CODE_END(relocate_new_kernel)
+
+#ifdef CONFIG_SMP
+/*
+ * Other CPUs should wait until code is relocated and
+ * then start at the entry point from LOONGARCH_IOCSR_MBUF0.
+ */
+SYM_CODE_START(kexec_smp_wait)
+1: li.w t0, 0x100 /* wait for init loop */
+2: addi.w t0, t0, -1 /* limit mailbox access */
+ bnez t0, 2b
+ li.w t1, LOONGARCH_IOCSR_MBUF0
+ iocsrrd.w s0, t1 /* check PC as an indicator */
+ beqz s0, 1b
+ iocsrrd.d s0, t1 /* get PC via mailbox */
+
+ li.d t0, CACHE_BASE
+ or s0, s0, t0 /* s0 = TO_CACHE(s0) */
+ jr s0 /* jump to initial PC */
+SYM_CODE_END(kexec_smp_wait)
+#endif
+
+relocate_new_kernel_end:
+
+SYM_DATA_START(relocate_new_kernel_size)
+ PTR relocate_new_kernel_end - relocate_new_kernel
+SYM_DATA_END(relocate_new_kernel_size)
diff --git a/arch/loongarch/kernel/setup.c b/arch/loongarch/kernel/setup.c
index 5b49c78c23f4..1eb63fa9bc81 100644
--- a/arch/loongarch/kernel/setup.c
+++ b/arch/loongarch/kernel/setup.c
@@ -19,6 +19,8 @@
#include <linux/memblock.h>
#include <linux/initrd.h>
#include <linux/ioport.h>
+#include <linux/kexec.h>
+#include <linux/crash_dump.h>
#include <linux/root_dev.h>
#include <linux/console.h>
#include <linux/pfn.h>
@@ -185,8 +187,70 @@ static int __init early_parse_mem(char *p)
}
early_param("mem", early_parse_mem);
+static void __init arch_reserve_vmcore(void)
+{
+#ifdef CONFIG_PROC_VMCORE
+ u64 i;
+ phys_addr_t start, end;
+
+ if (!is_kdump_kernel())
+ return;
+
+ if (!elfcorehdr_size) {
+ for_each_mem_range(i, &start, &end) {
+ if (elfcorehdr_addr >= start && elfcorehdr_addr < end) {
+ /*
+ * Reserve from the elf core header to the end of
+ * the memory segment, that should all be kdump
+ * reserved memory.
+ */
+ elfcorehdr_size = end - elfcorehdr_addr;
+ break;
+ }
+ }
+ }
+
+ if (memblock_is_region_reserved(elfcorehdr_addr, elfcorehdr_size)) {
+ pr_warn("elfcorehdr is overlapped\n");
+ return;
+ }
+
+ memblock_reserve(elfcorehdr_addr, elfcorehdr_size);
+
+ pr_info("Reserving %llu KiB of memory at 0x%llx for elfcorehdr\n",
+ elfcorehdr_size >> 10, elfcorehdr_addr);
+#endif
+}
+
+static void __init arch_parse_crashkernel(void)
+{
+#ifdef CONFIG_KEXEC
+ int ret;
+ unsigned long long start;
+ unsigned long long total_mem;
+ unsigned long long crash_base, crash_size;
+
+ total_mem = memblock_phys_mem_size();
+ ret = parse_crashkernel(boot_command_line, total_mem, &crash_size, &crash_base);
+ if (ret < 0 || crash_size <= 0)
+ return;
+
+ start = memblock_phys_alloc_range(crash_size, 1, crash_base, crash_base + crash_size);
+ if (start != crash_base) {
+ pr_warn("Invalid memory region reserved for crash kernel\n");
+ return;
+ }
+
+ crashk_res.start = crash_base;
+ crashk_res.end = crash_base + crash_size - 1;
+#endif
+}
+
void __init platform_init(void)
{
+ arch_reserve_vmcore();
+ arch_parse_crashkernel();
+
#ifdef CONFIG_ACPI_TABLE_UPGRADE
acpi_table_upgrade();
#endif
@@ -289,6 +353,15 @@ static void __init resource_init(void)
request_resource(res, &data_resource);
request_resource(res, &bss_resource);
}
+
+#ifdef CONFIG_KEXEC
+ if (crashk_res.start < crashk_res.end) {
+ insert_resource(&iomem_resource, &crashk_res);
+ pr_info("Reserving %ldMB of memory at %ldMB for crashkernel\n",
+ (unsigned long)((crashk_res.end - crashk_res.start + 1) >> 20),
+ (unsigned long)(crashk_res.start >> 20));
+ }
+#endif
}
static int __init reserve_memblock_reserved_regions(void)
@@ -348,10 +421,11 @@ void __init setup_arch(char **cmdline_p)
init_environ();
efi_init();
memblock_init();
+ pagetable_init();
parse_early_param();
+ reserve_initrd_mem();
platform_init();
- pagetable_init();
arch_mem_init(cmdline_p);
resource_init();
diff --git a/arch/loongarch/kernel/smp.c b/arch/loongarch/kernel/smp.c
index b5fab308dcf2..781a4d4bdddc 100644
--- a/arch/loongarch/kernel/smp.c
+++ b/arch/loongarch/kernel/smp.c
@@ -240,11 +240,6 @@ void loongson3_smp_finish(void)
#ifdef CONFIG_HOTPLUG_CPU
-static bool io_master(int cpu)
-{
- return test_bit(cpu, &loongson_sysconf.cores_io_master);
-}
-
int loongson3_cpu_disable(void)
{
unsigned long flags;
diff --git a/arch/loongarch/kernel/sysrq.c b/arch/loongarch/kernel/sysrq.c
new file mode 100644
index 000000000000..366baef72d29
--- /dev/null
+++ b/arch/loongarch/kernel/sysrq.c
@@ -0,0 +1,65 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * LoongArch specific sysrq operations.
+ *
+ * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
+ */
+#include <linux/init.h>
+#include <linux/smp.h>
+#include <linux/spinlock.h>
+#include <linux/sysrq.h>
+#include <linux/workqueue.h>
+
+#include <asm/cpu-features.h>
+#include <asm/tlb.h>
+
+/*
+ * Dump TLB entries on all CPUs.
+ */
+
+static DEFINE_SPINLOCK(show_lock);
+
+static void sysrq_tlbdump_single(void *dummy)
+{
+ unsigned long flags;
+
+ spin_lock_irqsave(&show_lock, flags);
+
+ pr_info("CPU%d:\n", smp_processor_id());
+ dump_tlb_regs();
+ pr_info("\n");
+ dump_tlb_all();
+ pr_info("\n");
+
+ spin_unlock_irqrestore(&show_lock, flags);
+}
+
+#ifdef CONFIG_SMP
+static void sysrq_tlbdump_othercpus(struct work_struct *dummy)
+{
+ smp_call_function(sysrq_tlbdump_single, NULL, 0);
+}
+
+static DECLARE_WORK(sysrq_tlbdump, sysrq_tlbdump_othercpus);
+#endif
+
+static void sysrq_handle_tlbdump(int key)
+{
+ sysrq_tlbdump_single(NULL);
+#ifdef CONFIG_SMP
+ schedule_work(&sysrq_tlbdump);
+#endif
+}
+
+static struct sysrq_key_op sysrq_tlbdump_op = {
+ .handler = sysrq_handle_tlbdump,
+ .help_msg = "show-tlbs(x)",
+ .action_msg = "Show TLB entries",
+ .enable_mask = SYSRQ_ENABLE_DUMP,
+};
+
+static int __init loongarch_sysrq_init(void)
+{
+ return register_sysrq_key('x', &sysrq_tlbdump_op);
+}
+arch_initcall(loongarch_sysrq_init);
diff --git a/arch/loongarch/kernel/topology.c b/arch/loongarch/kernel/topology.c
index ab1a75c0b5a6..caa7cd859078 100644
--- a/arch/loongarch/kernel/topology.c
+++ b/arch/loongarch/kernel/topology.c
@@ -5,6 +5,7 @@
#include <linux/node.h>
#include <linux/nodemask.h>
#include <linux/percpu.h>
+#include <asm/bootinfo.h>
static DEFINE_PER_CPU(struct cpu, cpu_devices);
@@ -40,7 +41,7 @@ static int __init topology_init(void)
for_each_present_cpu(i) {
struct cpu *c = &per_cpu(cpu_devices, i);
- c->hotpluggable = !!i;
+ c->hotpluggable = !io_master(i);
ret = register_cpu(c, i);
if (ret < 0)
pr_warn("topology_init: register_cpu %d failed (%d)\n", i, ret);
diff --git a/arch/loongarch/kernel/traps.c b/arch/loongarch/kernel/traps.c
index 5010e95cef84..1a4dce84ebc6 100644
--- a/arch/loongarch/kernel/traps.c
+++ b/arch/loongarch/kernel/traps.c
@@ -10,6 +10,7 @@
#include <linux/entry-common.h>
#include <linux/init.h>
#include <linux/kernel.h>
+#include <linux/kexec.h>
#include <linux/module.h>
#include <linux/extable.h>
#include <linux/mm.h>
@@ -246,6 +247,9 @@ void __noreturn die(const char *str, struct pt_regs *regs)
oops_exit();
+ if (regs && kexec_should_crash(current))
+ crash_kexec(regs);
+
if (in_interrupt())
panic("Fatal exception in interrupt");
@@ -374,6 +378,29 @@ asmlinkage void noinstr do_ale(struct pt_regs *regs)
irqentry_exit(regs, state);
}
+#ifdef CONFIG_GENERIC_BUG
+int is_valid_bugaddr(unsigned long addr)
+{
+ return 1;
+}
+#endif /* CONFIG_GENERIC_BUG */
+
+static void bug_handler(struct pt_regs *regs)
+{
+ switch (report_bug(regs->csr_era, regs)) {
+ case BUG_TRAP_TYPE_BUG:
+ case BUG_TRAP_TYPE_NONE:
+ die_if_kernel("Oops - BUG", regs);
+ force_sig(SIGTRAP);
+ break;
+
+ case BUG_TRAP_TYPE_WARN:
+ /* Skip the BUG instruction and continue */
+ regs->csr_era += LOONGARCH_INSN_SIZE;
+ break;
+ }
+}
+
asmlinkage void noinstr do_bp(struct pt_regs *regs)
{
bool user = user_mode(regs);
@@ -427,8 +454,7 @@ asmlinkage void noinstr do_bp(struct pt_regs *regs)
switch (bcode) {
case BRK_BUG:
- die_if_kernel("Kernel bug detected", regs);
- force_sig(SIGTRAP);
+ bug_handler(regs);
break;
case BRK_DIVZERO:
die_if_kernel("Break instruction in kernel code", regs);
@@ -620,9 +646,6 @@ asmlinkage void noinstr do_vint(struct pt_regs *regs, unsigned long sp)
irqentry_exit(regs, state);
}
-extern void tlb_init(int cpu);
-extern void cache_error_setup(void);
-
unsigned long eentry;
unsigned long tlbrentry;
diff --git a/arch/loongarch/kernel/vmlinux.lds.S b/arch/loongarch/kernel/vmlinux.lds.S
index e5890bec2bf6..b3309a5e695b 100644
--- a/arch/loongarch/kernel/vmlinux.lds.S
+++ b/arch/loongarch/kernel/vmlinux.lds.S
@@ -55,6 +55,10 @@ SECTIONS
EXCEPTION_TABLE(16)
+ .got : ALIGN(16) { *(.got) }
+ .plt : ALIGN(16) { *(.plt) }
+ .got.plt : ALIGN(16) { *(.got.plt) }
+
. = ALIGN(PECOFF_SEGMENT_ALIGN);
__init_begin = .;
__inittext_begin = .;
generated by cgit (git 2.34.1) at 2025-02-25 01:28:14 +0000