Page MenuHomeFreeBSD
Files F149197468

D37428.1789656868.diff
No OneTemporary
Actions

Size
307 KB
Referenced Files
None
Subscribers
None

D37428.1789656868.diff
View Options

This file is larger than 256 KB, so syntax highlighting was skipped.
diff --git a/sys/arm64/arm64/genassym.c b/sys/arm64/arm64/genassym.c
--- a/sys/arm64/arm64/genassym.c
+++ b/sys/arm64/arm64/genassym.c
@@ -74,6 +74,7 @@
ASSYM(TF_SIZE, sizeof(struct trapframe));
ASSYM(TF_SP, offsetof(struct trapframe, tf_sp));
+ASSYM(TF_LR, offsetof(struct trapframe, tf_lr));
ASSYM(TF_ELR, offsetof(struct trapframe, tf_elr));
ASSYM(TF_SPSR, offsetof(struct trapframe, tf_spsr));
ASSYM(TF_ESR, offsetof(struct trapframe, tf_esr));
diff --git a/sys/arm64/arm64/identcpu.c b/sys/arm64/arm64/identcpu.c
--- a/sys/arm64/arm64/identcpu.c
+++ b/sys/arm64/arm64/identcpu.c
@@ -105,8 +105,6 @@
SYSCTL_STRING(_hw, HW_MODEL, model, CTLFLAG_RD,
cpu_model, sizeof(cpu_model), "Machine model");
-#define MAX_CACHES 8 /* Maximum number of caches supported
- architecturally. */
/*
* Per-CPU affinity as provided in MPIDR_EL1
* Indexed by CPU number in logical order selected by the system.
@@ -120,32 +118,6 @@
uint64_t __cpu_affinity[MAXCPU];
static u_int cpu_aff_levels;
-struct cpu_desc {
- uint64_t mpidr;
- uint64_t id_aa64afr0;
- uint64_t id_aa64afr1;
- uint64_t id_aa64dfr0;
- uint64_t id_aa64dfr1;
- uint64_t id_aa64isar0;
- uint64_t id_aa64isar1;
- uint64_t id_aa64isar2;
- uint64_t id_aa64mmfr0;
- uint64_t id_aa64mmfr1;
- uint64_t id_aa64mmfr2;
- uint64_t id_aa64pfr0;
- uint64_t id_aa64pfr1;
- uint64_t id_aa64zfr0;
- uint64_t ctr;
-#ifdef COMPAT_FREEBSD32
- uint64_t id_isar5;
- uint64_t mvfr0;
- uint64_t mvfr1;
-#endif
- uint64_t clidr;
- uint32_t ccsidr[MAX_CACHES][2]; /* 2 possible types. */
- bool have_sve;
-};
-
static struct cpu_desc cpu_desc[MAXCPU];
static struct cpu_desc kern_cpu_desc;
static struct cpu_desc user_cpu_desc;
@@ -1833,6 +1805,27 @@
}
}
+void
+update_cpu_desc(struct cpu_desc *desc)
+{
+ struct mrs_field *fields;
+ uint64_t desc_val, kern_val;
+ int i, j;
+
+ for (i = 0; i < nitems(user_regs); i++) {
+ kern_val = CPU_DESC_FIELD(kern_cpu_desc, i);
+ desc_val = CPU_DESC_FIELD(*desc, i);
+
+ fields = user_regs[i].fields;
+ for (j = 0; fields[j].type != 0; j++) {
+ desc_val = update_lower_register(desc_val, kern_val,
+ fields[j].shift, 4, fields[j].sign);
+ }
+
+ CPU_DESC_FIELD(*desc, i) = desc_val;
+ }
+}
+
/* HWCAP */
bool __read_frequently lse_supported = false;
diff --git a/sys/arm64/arm64/vfp.c b/sys/arm64/arm64/vfp.c
--- a/sys/arm64/arm64/vfp.c
+++ b/sys/arm64/arm64/vfp.c
@@ -37,6 +37,9 @@
#include <sys/malloc.h>
#include <sys/pcpu.h>
#include <sys/proc.h>
+#include <sys/queue.h>
+
+#include <vm/uma.h>
#include <vm/uma.h>
diff --git a/sys/arm64/include/armreg.h b/sys/arm64/include/armreg.h
--- a/sys/arm64/include/armreg.h
+++ b/sys/arm64/include/armreg.h
@@ -342,6 +342,12 @@
#define ISS_MSR_REG_MASK \
(ISS_MSR_OP0_MASK | ISS_MSR_OP2_MASK | ISS_MSR_OP1_MASK | \
ISS_MSR_CRn_MASK | ISS_MSR_CRm_MASK)
+#define ISS_MSR_REG(reg) \
+ (((reg ## _op0) << ISS_MSR_OP0_SHIFT) | \
+ ((reg ## _op1) << ISS_MSR_OP1_SHIFT) | \
+ ((reg ## _CRn) << ISS_MSR_CRn_SHIFT) | \
+ ((reg ## _CRm) << ISS_MSR_CRm_SHIFT) | \
+ ((reg ## _op2) << ISS_MSR_OP2_SHIFT))
#define ISS_DATA_ISV_SHIFT 24
#define ISS_DATA_ISV (0x01 << ISS_DATA_ISV_SHIFT)
diff --git a/sys/arm64/include/cpu.h b/sys/arm64/include/cpu.h
--- a/sys/arm64/include/cpu.h
+++ b/sys/arm64/include/cpu.h
@@ -183,6 +183,36 @@
#define CPU_MATCH_ERRATA_CAVIUM_THUNDERX_1_1 0
#endif
+#define MAX_CACHES 8 /* Maximum number of caches supported
+ architecturally. */
+
+struct cpu_desc {
+ uint64_t mpidr;
+ uint64_t id_aa64afr0;
+ uint64_t id_aa64afr1;
+ uint64_t id_aa64dfr0;
+ uint64_t id_aa64dfr1;
+ uint64_t id_aa64isar0;
+ uint64_t id_aa64isar1;
+ uint64_t id_aa64isar2;
+ uint64_t id_aa64mmfr0;
+ uint64_t id_aa64mmfr1;
+ uint64_t id_aa64mmfr2;
+ uint64_t id_aa64pfr0;
+ uint64_t id_aa64pfr1;
+ uint64_t id_aa64zfr0;
+ uint64_t ctr;
+#ifdef COMPAT_FREEBSD32
+ uint64_t id_isar5;
+ uint64_t mvfr0;
+ uint64_t mvfr1;
+#endif
+ uint64_t clidr;
+ uint32_t ccsidr[MAX_CACHES][2]; /* 2 possible types. */
+ bool have_sve;
+};
+
+
extern char btext[];
extern char etext[];
@@ -211,6 +241,7 @@
/* Functions to read the sanitised view of the special registers */
void update_special_regs(u_int);
+void update_cpu_desc(struct cpu_desc *desc);
bool extract_user_id_field(u_int, u_int, uint8_t *);
bool get_kernel_reg(u_int, uint64_t *);
diff --git a/sys/arm64/include/hypervisor.h b/sys/arm64/include/hypervisor.h
--- a/sys/arm64/include/hypervisor.h
+++ b/sys/arm64/include/hypervisor.h
@@ -171,6 +171,7 @@
#define TCR_EL2_TG0_64K (0x1UL << TCR_EL2_TG0_SHIFT)
#define TCR_EL2_TG0_16K (0x2UL << TCR_EL2_TG0_SHIFT)
#define TCR_EL2_PS_SHIFT 16
+#define TCR_EL2_PS_MASK (0xfUL << TCR_EL2_PS_SHIFT)
#define TCR_EL2_PS_32BITS (0UL << TCR_EL2_PS_SHIFT)
#define TCR_EL2_PS_36BITS (1UL << TCR_EL2_PS_SHIFT)
#define TCR_EL2_PS_40BITS (2UL << TCR_EL2_PS_SHIFT)
diff --git a/sys/arm64/include/pcpu.h b/sys/arm64/include/pcpu.h
--- a/sys/arm64/include/pcpu.h
+++ b/sys/arm64/include/pcpu.h
@@ -50,6 +50,7 @@
/* Store as two u_int values to preserve KBI */ \
uint64_t pc_mpidr; \
u_int pc_bcast_tlbi_workaround; \
+ void *pc_vcpu; \
char __pad[197]
#ifdef _KERNEL
diff --git a/sys/arm64/include/vmm.h b/sys/arm64/include/vmm.h
new file mode 100644
--- /dev/null
+++ b/sys/arm64/include/vmm.h
@@ -0,0 +1,376 @@
+/*
+ * Copyright (C) 2015 Mihai Carabas <mihai.carabas@gmail.com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef _VMM_H_
+#define _VMM_H_
+
+#include <sys/param.h>
+#include <vm/vm.h>
+#include <vm/pmap.h>
+
+#include "pte.h"
+#include "pmap.h"
+
+struct vcpu;
+
+enum vm_suspend_how {
+ VM_SUSPEND_NONE,
+ VM_SUSPEND_RESET,
+ VM_SUSPEND_POWEROFF,
+ VM_SUSPEND_HALT,
+ VM_SUSPEND_TRIPLEFAULT,
+ VM_SUSPEND_LAST
+};
+
+/*
+ * Identifiers for architecturally defined registers.
+ */
+enum vm_reg_name {
+ VM_REG_GUEST_X0,
+ VM_REG_GUEST_X1,
+ VM_REG_GUEST_X2,
+ VM_REG_GUEST_X3,
+ VM_REG_GUEST_X4,
+ VM_REG_GUEST_X5,
+ VM_REG_GUEST_X6,
+ VM_REG_GUEST_X7,
+ VM_REG_GUEST_X8,
+ VM_REG_GUEST_X9,
+ VM_REG_GUEST_X10,
+ VM_REG_GUEST_X11,
+ VM_REG_GUEST_X12,
+ VM_REG_GUEST_X13,
+ VM_REG_GUEST_X14,
+ VM_REG_GUEST_X15,
+ VM_REG_GUEST_X16,
+ VM_REG_GUEST_X17,
+ VM_REG_GUEST_X18,
+ VM_REG_GUEST_X19,
+ VM_REG_GUEST_X20,
+ VM_REG_GUEST_X21,
+ VM_REG_GUEST_X22,
+ VM_REG_GUEST_X23,
+ VM_REG_GUEST_X24,
+ VM_REG_GUEST_X25,
+ VM_REG_GUEST_X26,
+ VM_REG_GUEST_X27,
+ VM_REG_GUEST_X28,
+ VM_REG_GUEST_X29,
+ VM_REG_GUEST_LR,
+ VM_REG_GUEST_SP,
+ VM_REG_GUEST_ELR,
+ VM_REG_GUEST_SPSR,
+ VM_REG_ELR_EL2,
+ VM_REG_LAST
+};
+
+#define VM_INTINFO_VECTOR(info) ((info) & 0xff)
+#define VM_INTINFO_DEL_ERRCODE 0x800
+#define VM_INTINFO_RSVD 0x7ffff000
+#define VM_INTINFO_VALID 0x80000000
+#define VM_INTINFO_TYPE 0x700
+#define VM_INTINFO_HWINTR (0 << 8)
+#define VM_INTINFO_NMI (2 << 8)
+#define VM_INTINFO_HWEXCEPTION (3 << 8)
+#define VM_INTINFO_SWINTR (4 << 8)
+
+#define VM_MAX_SUFFIXLEN 15
+
+#define VM_GUEST_BASE_IPA 0x80000000UL /* Guest kernel start ipa */
+
+#ifdef _KERNEL
+
+#define VM_MAX_NAMELEN 32
+
+struct vm;
+struct vm_exception;
+struct vm_exit;
+struct vm_run;
+struct vm_object;
+struct vm_vgic_descr;
+struct pmap;
+
+struct vm_eventinfo {
+ void *rptr; /* rendezvous cookie */
+ int *sptr; /* suspend cookie */
+ int *iptr; /* reqidle cookie */
+};
+
+int vm_create(const char *name, struct vm **retvm);
+struct vcpu *vm_alloc_vcpu(struct vm *vm, int vcpuid);
+void vm_slock_vcpus(struct vm *vm);
+void vm_unlock_vcpus(struct vm *vm);
+void vm_destroy(struct vm *vm);
+int vm_reinit(struct vm *vm);
+const char *vm_name(struct vm *vm);
+
+/*
+ * APIs that modify the guest memory map require all vcpus to be frozen.
+ */
+void vm_slock_memsegs(struct vm *vm);
+void vm_xlock_memsegs(struct vm *vm);
+void vm_unlock_memsegs(struct vm *vm);
+int vm_mmap_memseg(struct vm *vm, vm_paddr_t gpa, int segid, vm_ooffset_t off,
+ size_t len, int prot, int flags);
+int vm_alloc_memseg(struct vm *vm, int ident, size_t len, bool sysmem);
+void vm_free_memseg(struct vm *vm, int ident);
+int vm_map_mmio(struct vm *vm, vm_paddr_t gpa, size_t len, vm_paddr_t hpa);
+int vm_unmap_mmio(struct vm *vm, vm_paddr_t gpa, size_t len);
+int vmm_map_gpa(struct vm *vm, vm_offset_t va, vm_paddr_t gpa, int pages,
+ vm_page_t *ma);
+void vmm_unmap_gpa(struct vm *vm, vm_offset_t va, size_t pages, vm_page_t *ma);
+
+/*
+ * APIs that inspect the guest memory map require only a *single* vcpu to
+ * be frozen. This acts like a read lock on the guest memory map since any
+ * modification requires *all* vcpus to be frozen.
+ */
+int vm_mmap_getnext(struct vm *vm, vm_paddr_t *gpa, int *segid,
+ vm_ooffset_t *segoff, size_t *len, int *prot, int *flags);
+int vm_get_memseg(struct vm *vm, int ident, size_t *len, bool *sysmem,
+ struct vm_object **objptr);
+vm_paddr_t vmm_sysmem_maxaddr(struct vm *vm);
+void *vm_gpa_hold(struct vcpu *vcpu, vm_paddr_t gpa, size_t len,
+ int prot, void **cookie);
+void *vm_gpa_hold_global(struct vm *vm, vm_paddr_t gpa, size_t len,
+ int prot, void **cookie);
+void vm_gpa_release(void *cookie);
+bool vm_mem_allocated(struct vcpu *vcpu, vm_paddr_t gpa);
+
+int vm_gla2gpa_nofault(struct vcpu *vcpu, uint64_t gla, int prot,
+ uint64_t *gpa, int *is_fault);
+
+uint16_t vm_get_maxcpus(struct vm *vm);
+void vm_get_topology(struct vm *vm, uint16_t *sockets, uint16_t *cores,
+ uint16_t *threads, uint16_t *maxcpus);
+int vm_set_topology(struct vm *vm, uint16_t sockets, uint16_t cores,
+ uint16_t threads, uint16_t maxcpus);
+int vm_get_register(struct vcpu *vcpu, int reg, uint64_t *retval);
+int vm_set_register(struct vcpu *vcpu, int reg, uint64_t val);
+int vm_run(struct vcpu *vcpu, struct vm_exit *vme_user);
+int vm_suspend(struct vm *vm, enum vm_suspend_how how);
+void* vm_get_cookie(struct vm *vm);
+int vcpu_vcpuid(struct vcpu *vcpu);
+void *vcpu_get_cookie(struct vcpu *vcpu);
+struct vm *vcpu_vm(struct vcpu *vcpu);
+struct vcpu *vm_vcpu(struct vm *vm, int cpu);
+int vm_get_capability(struct vcpu *vcpu, int type, int *val);
+int vm_set_capability(struct vcpu *vcpu, int type, int val);
+int vm_activate_cpu(struct vcpu *vcpu);
+int vm_suspend_cpu(struct vm *vm, struct vcpu *vcpu);
+int vm_resume_cpu(struct vm *vm, struct vcpu *vcpu);
+int vm_inject_exception(struct vcpu *vcpu, uint64_t esr, uint64_t far);
+int vm_attach_vgic(struct vm *vm, struct vm_vgic_descr *descr);
+int vm_assert_irq(struct vm *vm, uint32_t irq);
+int vm_deassert_irq(struct vm *vm, uint32_t irq);
+int vm_raise_msi(struct vm *vm, uint64_t msg, uint64_t addr, int bus, int slot,
+ int func);
+struct vm_exit *vm_exitinfo(struct vcpu *vcpu);
+void vm_exit_suspended(struct vcpu *vcpu, uint64_t rip);
+void vm_exit_debug(struct vcpu *vcpu, uint64_t rip);
+void vm_exit_rendezvous(struct vcpu *vcpu, uint64_t rip);
+void vm_exit_astpending(struct vcpu *vcpu, uint64_t rip);
+
+#ifdef _SYS__CPUSET_H_
+cpuset_t vm_active_cpus(struct vm *vm);
+cpuset_t vm_debug_cpus(struct vm *vm);
+cpuset_t vm_suspended_cpus(struct vm *vm);
+#endif /* _SYS__CPUSET_H_ */
+
+static __inline bool
+virt_enabled(void)
+{
+
+ return (has_hyp());
+}
+
+static __inline int
+vcpu_rendezvous_pending(struct vm_eventinfo *info)
+{
+
+ return (*((uintptr_t *)(info->rptr)) != 0);
+}
+
+static __inline int
+vcpu_suspended(struct vm_eventinfo *info)
+{
+
+ return (*info->sptr);
+}
+
+int vcpu_debugged(struct vcpu *vcpu);
+
+enum vcpu_state {
+ VCPU_IDLE,
+ VCPU_FROZEN,
+ VCPU_RUNNING,
+ VCPU_SLEEPING,
+};
+
+int vcpu_set_state(struct vcpu *vcpu, enum vcpu_state state, bool from_idle);
+enum vcpu_state vcpu_get_state(struct vcpu *vcpu, int *hostcpu);
+
+static int __inline
+vcpu_is_running(struct vcpu *vcpu, int *hostcpu)
+{
+ return (vcpu_get_state(vcpu, hostcpu) == VCPU_RUNNING);
+}
+
+#ifdef _SYS_PROC_H_
+static int __inline
+vcpu_should_yield(struct vcpu *vcpu)
+{
+ struct thread *td;
+
+ td = curthread;
+ return (td->td_ast != 0 || td->td_owepreempt != 0);
+}
+#endif
+
+void *vcpu_stats(struct vcpu *vcpu);
+void vcpu_notify_event(struct vcpu *vcpu, bool lapic_intr);
+
+enum vm_reg_name vm_segment_name(int seg_encoding);
+
+struct vm_copyinfo {
+ uint64_t gpa;
+ size_t len;
+ void *hva;
+ void *cookie;
+};
+
+#endif /* _KERNEL */
+
+#define VM_DIR_READ 0
+#define VM_DIR_WRITE 1
+
+#define VM_GP_M_MASK 0x1f
+#define VM_GP_MMU_ENABLED (1 << 5)
+
+struct vm_guest_paging {
+ uint64_t far;
+ uint64_t ttbr0_el1;
+ uint64_t ttbr1_el1;
+ int flags;
+ int padding;
+};
+
+struct vie {
+ uint8_t access_size:4, sign_extend:1, dir:1, unused:2;
+ enum vm_reg_name reg;
+};
+
+struct vre {
+ uint32_t inst_syndrome;
+ uint8_t dir:1, unused:7;
+ enum vm_reg_name reg;
+};
+
+/*
+ * Identifiers for optional vmm capabilities
+ */
+enum vm_cap_type {
+ VM_CAP_HALT_EXIT,
+ VM_CAP_MTRAP_EXIT,
+ VM_CAP_PAUSE_EXIT,
+ VM_CAP_UNRESTRICTED_GUEST,
+ VM_CAP_MAX
+};
+
+enum vm_exitcode {
+ VM_EXITCODE_BOGUS,
+ VM_EXITCODE_INST_EMUL,
+ VM_EXITCODE_REG_EMUL,
+ VM_EXITCODE_HVC,
+ VM_EXITCODE_SUSPENDED,
+ VM_EXITCODE_HYP,
+ VM_EXITCODE_WFI,
+ VM_EXITCODE_PAGING,
+ VM_EXITCODE_SMCCC,
+ VM_EXITCODE_DEBUG,
+ VM_EXITCODE_MAX
+};
+
+enum task_switch_reason {
+ TSR_CALL,
+ TSR_IRET,
+ TSR_JMP,
+ TSR_IDT_GATE, /* task gate in IDT */
+};
+
+struct vm_task_switch {
+ uint16_t tsssel; /* new TSS selector */
+ int ext; /* task switch due to external event */
+ uint32_t errcode;
+ int errcode_valid; /* push 'errcode' on the new stack */
+ enum task_switch_reason reason;
+};
+
+struct vm_exit {
+ enum vm_exitcode exitcode;
+ int inst_length;
+ uint64_t pc;
+ union {
+ /*
+ * ARM specific payload.
+ */
+ struct {
+ uint32_t exception_nr;
+ uint32_t pad;
+ uint64_t esr_el2; /* Exception Syndrome Register */
+ uint64_t far_el2; /* Fault Address Register */
+ uint64_t hpfar_el2; /* Hypervisor IPA Fault Address Register */
+ } hyp;
+ struct {
+ struct vre vre;
+ } reg_emul;
+ struct {
+ uint64_t gpa;
+ uint64_t esr;
+ } paging;
+ struct {
+ uint64_t gpa;
+ struct vm_guest_paging paging;
+ struct vie vie;
+ } inst_emul;
+
+ /*
+ * A SMCCC call, e.g. starting a core via PSCI.
+ * Further arguments can be read by asking the kernel for
+ * all register values.
+ */
+ struct {
+ uint64_t func_id;
+ uint64_t args[7];
+ } smccc_call;
+
+ struct {
+ enum vm_suspend_how how;
+ } suspended;
+ } u;
+};
+
+#endif /* _VMM_H_ */
diff --git a/sys/arm64/include/vmm_dev.h b/sys/arm64/include/vmm_dev.h
new file mode 100644
--- /dev/null
+++ b/sys/arm64/include/vmm_dev.h
@@ -0,0 +1,263 @@
+/*
+ * Copyright (C) 2015 Mihai Carabas <mihai.carabas@gmail.com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef _VMM_DEV_H_
+#define _VMM_DEV_H_
+
+#ifdef _KERNEL
+void vmmdev_init(void);
+int vmmdev_cleanup(void);
+#endif
+
+struct vm_memmap {
+ vm_paddr_t gpa;
+ int segid; /* memory segment */
+ vm_ooffset_t segoff; /* offset into memory segment */
+ size_t len; /* mmap length */
+ int prot; /* RWX */
+ int flags;
+};
+#define VM_MEMMAP_F_WIRED 0x01
+
+#define VM_MEMSEG_NAME(m) ((m)->name[0] != '\0' ? (m)->name : NULL)
+struct vm_memseg {
+ int segid;
+ size_t len;
+ char name[VM_MAX_SUFFIXLEN + 1];
+};
+
+struct vm_register {
+ int cpuid;
+ int regnum; /* enum vm_reg_name */
+ uint64_t regval;
+};
+
+struct vm_register_set {
+ int cpuid;
+ unsigned int count;
+ const int *regnums; /* enum vm_reg_name */
+ uint64_t *regvals;
+};
+
+struct vm_run {
+ int cpuid;
+ uint64_t pc;
+ struct vm_exit vm_exit;
+
+};
+
+struct vm_exception {
+ int cpuid;
+ uint64_t esr;
+ uint64_t far;
+};
+
+struct vm_msi {
+ uint64_t msg;
+ uint64_t addr;
+ int bus;
+ int slot;
+ int func;
+};
+
+struct vm_capability {
+ int cpuid;
+ enum vm_cap_type captype;
+ int capval;
+ int allcpus;
+};
+
+#define MAX_VM_STATS 64
+struct vm_stats {
+ int cpuid; /* in */
+ int index; /* in */
+ int num_entries; /* out */
+ struct timeval tv;
+ uint64_t statbuf[MAX_VM_STATS];
+};
+struct vm_stat_desc {
+ int index; /* in */
+ char desc[128]; /* out */
+};
+
+struct vm_suspend {
+ enum vm_suspend_how how;
+};
+
+struct vm_gla2gpa {
+ int vcpuid; /* inputs */
+ int prot; /* PROT_READ or PROT_WRITE */
+ uint64_t gla;
+ int fault; /* outputs */
+ uint64_t gpa;
+};
+
+struct vm_activate_cpu {
+ int vcpuid;
+};
+
+struct vm_cpuset {
+ int which;
+ int cpusetsize;
+ cpuset_t *cpus;
+};
+#define VM_ACTIVE_CPUS 0
+#define VM_SUSPENDED_CPUS 1
+#define VM_DEBUG_CPUS 2
+
+struct vm_vgic_version {
+ u_int version;
+ u_int flags;
+};
+
+struct vm_vgic_descr {
+ struct vm_vgic_version ver;
+ union {
+ struct {
+ uint64_t dist_start;
+ uint64_t dist_size;
+ uint64_t redist_start;
+ uint64_t redist_size;
+ } v3_regs;
+ };
+};
+
+struct vm_irq {
+ uint32_t irq;
+};
+
+struct vm_cpu_topology {
+ uint16_t sockets;
+ uint16_t cores;
+ uint16_t threads;
+ uint16_t maxcpus;
+};
+
+enum {
+ /* general routines */
+ IOCNUM_ABIVERS = 0,
+ IOCNUM_RUN = 1,
+ IOCNUM_SET_CAPABILITY = 2,
+ IOCNUM_GET_CAPABILITY = 3,
+ IOCNUM_SUSPEND = 4,
+ IOCNUM_REINIT = 5,
+
+ /* memory apis */
+ IOCNUM_GET_GPA_PMAP = 12,
+ IOCNUM_GLA2GPA_NOFAULT = 13,
+ IOCNUM_ALLOC_MEMSEG = 14,
+ IOCNUM_GET_MEMSEG = 15,
+ IOCNUM_MMAP_MEMSEG = 16,
+ IOCNUM_MMAP_GETNEXT = 17,
+
+ /* register/state accessors */
+ IOCNUM_SET_REGISTER = 20,
+ IOCNUM_GET_REGISTER = 21,
+ IOCNUM_SET_REGISTER_SET = 24,
+ IOCNUM_GET_REGISTER_SET = 25,
+
+ /* statistics */
+ IOCNUM_VM_STATS = 50,
+ IOCNUM_VM_STAT_DESC = 51,
+
+ /* CPU Topology */
+ IOCNUM_SET_TOPOLOGY = 63,
+ IOCNUM_GET_TOPOLOGY = 64,
+
+ /* interrupt injection */
+ IOCNUM_ASSERT_IRQ = 80,
+ IOCNUM_DEASSERT_IRQ = 81,
+ IOCNUM_RAISE_MSI = 82,
+ IOCNUM_INJECT_EXCEPTION = 83,
+
+ /* vm_cpuset */
+ IOCNUM_ACTIVATE_CPU = 90,
+ IOCNUM_GET_CPUSET = 91,
+ IOCNUM_SUSPEND_CPU = 92,
+ IOCNUM_RESUME_CPU = 93,
+
+ /* vm_attach_vgic */
+ IOCNUM_GET_VGIC_VERSION = 110,
+ IOCNUM_ATTACH_VGIC = 111,
+};
+
+#define VM_RUN \
+ _IOWR('v', IOCNUM_RUN, struct vm_run)
+#define VM_SUSPEND \
+ _IOW('v', IOCNUM_SUSPEND, struct vm_suspend)
+#define VM_REINIT \
+ _IO('v', IOCNUM_REINIT)
+#define VM_ALLOC_MEMSEG \
+ _IOW('v', IOCNUM_ALLOC_MEMSEG, struct vm_memseg)
+#define VM_GET_MEMSEG \
+ _IOWR('v', IOCNUM_GET_MEMSEG, struct vm_memseg)
+#define VM_MMAP_MEMSEG \
+ _IOW('v', IOCNUM_MMAP_MEMSEG, struct vm_memmap)
+#define VM_MMAP_GETNEXT \
+ _IOWR('v', IOCNUM_MMAP_GETNEXT, struct vm_memmap)
+#define VM_SET_REGISTER \
+ _IOW('v', IOCNUM_SET_REGISTER, struct vm_register)
+#define VM_GET_REGISTER \
+ _IOWR('v', IOCNUM_GET_REGISTER, struct vm_register)
+#define VM_SET_REGISTER_SET \
+ _IOW('v', IOCNUM_SET_REGISTER_SET, struct vm_register_set)
+#define VM_GET_REGISTER_SET \
+ _IOWR('v', IOCNUM_GET_REGISTER_SET, struct vm_register_set)
+#define VM_SET_CAPABILITY \
+ _IOW('v', IOCNUM_SET_CAPABILITY, struct vm_capability)
+#define VM_GET_CAPABILITY \
+ _IOWR('v', IOCNUM_GET_CAPABILITY, struct vm_capability)
+#define VM_STATS \
+ _IOWR('v', IOCNUM_VM_STATS, struct vm_stats)
+#define VM_STAT_DESC \
+ _IOWR('v', IOCNUM_VM_STAT_DESC, struct vm_stat_desc)
+#define VM_ASSERT_IRQ \
+ _IOW('v', IOCNUM_ASSERT_IRQ, struct vm_irq)
+#define VM_DEASSERT_IRQ \
+ _IOW('v', IOCNUM_DEASSERT_IRQ, struct vm_irq)
+#define VM_RAISE_MSI \
+ _IOW('v', IOCNUM_RAISE_MSI, struct vm_msi)
+#define VM_INJECT_EXCEPTION \
+ _IOW('v', IOCNUM_INJECT_EXCEPTION, struct vm_exception)
+#define VM_SET_TOPOLOGY \
+ _IOW('v', IOCNUM_SET_TOPOLOGY, struct vm_cpu_topology)
+#define VM_GET_TOPOLOGY \
+ _IOR('v', IOCNUM_GET_TOPOLOGY, struct vm_cpu_topology)
+#define VM_GLA2GPA_NOFAULT \
+ _IOWR('v', IOCNUM_GLA2GPA_NOFAULT, struct vm_gla2gpa)
+#define VM_ACTIVATE_CPU \
+ _IOW('v', IOCNUM_ACTIVATE_CPU, struct vm_activate_cpu)
+#define VM_GET_CPUS \
+ _IOW('v', IOCNUM_GET_CPUSET, struct vm_cpuset)
+#define VM_SUSPEND_CPU \
+ _IOW('v', IOCNUM_SUSPEND_CPU, struct vm_activate_cpu)
+#define VM_RESUME_CPU \
+ _IOW('v', IOCNUM_RESUME_CPU, struct vm_activate_cpu)
+#define VM_GET_VGIC_VERSION \
+ _IOR('v', IOCNUM_GET_VGIC_VERSION, struct vm_vgic_version)
+#define VM_ATTACH_VGIC \
+ _IOW('v', IOCNUM_ATTACH_VGIC, struct vm_vgic_descr)
+#endif
diff --git a/sys/arm64/include/vmm_instruction_emul.h b/sys/arm64/include/vmm_instruction_emul.h
new file mode 100644
--- /dev/null
+++ b/sys/arm64/include/vmm_instruction_emul.h
@@ -0,0 +1,83 @@
+/*
+ * Copyright (C) 2015 Mihai Carabas <mihai.carabas@gmail.com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef _VMM_INSTRUCTION_EMUL_H_
+#define _VMM_INSTRUCTION_EMUL_H_
+
+/*
+ * Callback functions to read and write memory regions.
+ */
+typedef int (*mem_region_read_t)(struct vcpu *vcpu, uint64_t gpa,
+ uint64_t *rval, int rsize, void *arg);
+typedef int (*mem_region_write_t)(struct vcpu *vcpu, uint64_t gpa,
+ uint64_t wval, int wsize, void *arg);
+
+/*
+ * Callback functions to read and write registers.
+ */
+typedef int (*reg_read_t)(struct vcpu *vcpu, uint64_t *rval, void *arg);
+typedef int (*reg_write_t)(struct vcpu *vcpu, uint64_t wval, void *arg);
+
+/*
+ * Emulate the decoded 'vie' instruction when it contains a memory operation.
+ *
+ * The callbacks 'mrr' and 'mrw' emulate reads and writes to the memory region
+ * containing 'gpa'. 'mrarg' is an opaque argument that is passed into the
+ * callback functions.
+ *
+ * 'void *vm' should be 'struct vm *' when called from kernel context and
+ * 'struct vmctx *' when called from user context.
+ *
+ */
+int vmm_emulate_instruction(struct vcpu *vcpu, uint64_t gpa, struct vie *vie,
+ struct vm_guest_paging *paging, mem_region_read_t mrr,
+ mem_region_write_t mrw, void *mrarg);
+
+/*
+ * Emulate the decoded 'vre' instruction when it contains a register access.
+ *
+ * The callbacks 'regread' and 'regwrite' emulate reads and writes to the
+ * register from 'vie'. 'regarg' is an opaque argument that is passed into the
+ * callback functions.
+ *
+ * 'void *vm' should be 'struct vm *' when called from kernel context and
+ * 'struct vmctx *' when called from user context.
+ *
+ */
+int vmm_emulate_register(struct vcpu *vcpu, struct vre *vre, reg_read_t regread,
+ reg_write_t regwrite, void *regarg);
+
+#ifdef _KERNEL
+void vm_register_reg_handler(struct vm *vm, uint64_t iss, uint64_t mask,
+ reg_read_t reg_read, reg_write_t reg_write, void *arg);
+void vm_deregister_reg_handler(struct vm *vm, uint64_t iss, uint64_t mask);
+
+void vm_register_inst_handler(struct vm *vm, uint64_t start, uint64_t size,
+ mem_region_read_t mmio_read, mem_region_write_t mmio_write);
+void vm_deregister_inst_handler(struct vm *vm, uint64_t start, uint64_t size);
+#endif
+
+#endif /* _VMM_INSTRUCTION_EMUL_H_ */
diff --git a/sys/arm64/include/vmm_snapshot.h b/sys/arm64/include/vmm_snapshot.h
new file mode 100644
--- /dev/null
+++ b/sys/arm64/include/vmm_snapshot.h
@@ -0,0 +1 @@
+/* $FreeBSD$ */
diff --git a/sys/arm64/vmm/arm64.h b/sys/arm64/vmm/arm64.h
new file mode 100644
--- /dev/null
+++ b/sys/arm64/vmm/arm64.h
@@ -0,0 +1,186 @@
+/*
+ * Copyright (C) 2015 Mihai Carabas <mihai.carabas@gmail.com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+#ifndef _VMM_ARM64_H_
+#define _VMM_ARM64_H_
+
+#include <machine/reg.h>
+#include <machine/hypervisor.h>
+#include <machine/pcpu.h>
+
+#include "mmu.h"
+#include "io/vgic_v3.h"
+#include "io/vtimer.h"
+
+struct vgic_v3;
+struct vgic_v3_cpu;
+
+struct hypctx {
+ struct trapframe tf;
+
+ /*
+ * EL1 control registers.
+ * Be careful changing the layout of these as we access them from
+ * assembly when switching between the host and guest.
+ */
+ uint64_t elr_el1; /* Exception Link Register */
+ uint64_t sp_el0; /* Stack pointer */
+ uint64_t tpidr_el0; /* EL0 Software ID Register */
+ uint64_t tpidrro_el0; /* Read-only Thread ID Register */
+ uint64_t tpidr_el1; /* EL1 Software ID Register */
+ uint64_t vbar_el1; /* Vector Base Address Register */
+
+ uint64_t actlr_el1; /* Auxiliary Control Register */
+ uint64_t afsr0_el1; /* Auxiliary Fault Status Register 0 */
+ uint64_t afsr1_el1; /* Auxiliary Fault Status Register 1 */
+ uint64_t amair_el1; /* Auxiliary Memory Attribute Indirection Register */
+ uint64_t contextidr_el1; /* Current Process Identifier */
+ uint64_t cpacr_el1; /* Architectural Feature Access Control Register */
+ uint64_t csselr_el1; /* Cache Size Selection Register */
+ uint64_t esr_el1; /* Exception Syndrome Register */
+ uint64_t far_el1; /* Fault Address Register */
+ uint64_t mair_el1; /* Memory Attribute Indirection Register */
+ uint64_t mdccint_el1; /* Monitor DCC Interrupt Enable Register */
+ uint64_t mdscr_el1; /* Monitor Debug System Control Register */
+ uint64_t par_el1; /* Physical Address Register */
+ uint64_t sctlr_el1; /* System Control Register */
+ uint64_t tcr_el1; /* Translation Control Register */
+ uint64_t ttbr0_el1; /* Translation Table Base Register 0 */
+ uint64_t ttbr1_el1; /* Translation Table Base Register 1 */
+ uint64_t spsr_el1; /* Saved Program Status Register */
+
+ uint64_t pmcr_el0; /* Performance Monitors Control Register */
+ uint64_t pmccntr_el0;
+ uint64_t pmccfiltr_el0;
+ uint64_t pmcntenset_el0;
+ uint64_t pmintenset_el1;
+ uint64_t pmovsset_el0;
+ uint64_t pmselr_el0;
+ uint64_t pmuserenr_el0;
+ uint64_t pmevcntr_el0[31];
+ uint64_t pmevtyper_el0[31];
+
+ uint64_t dbgbcr_el1[16]; /* Debug Breakpoint Control Registers */
+ uint64_t dbgbvr_el1[16]; /* Debug Breakpoint Value Registers */
+ uint64_t dbgwcr_el1[16]; /* Debug Watchpoint Control Registers */
+ uint64_t dbgwvr_el1[16]; /* Debug Watchpoint Value Registers */
+
+ /* EL2 control registers */
+ uint64_t cptr_el2; /* Architectural Feature Trap Register */
+ uint64_t hcr_el2; /* Hypervisor Configuration Register */
+ uint64_t mdcr_el2; /* Monitor Debug Configuration Register */
+ uint64_t vpidr_el2; /* Virtualization Processor ID Register */
+ uint64_t vmpidr_el2; /* Virtualization Multiprocessor ID Register */
+ uint64_t el2_addr; /* The address of this in el2 space */
+ struct hyp *hyp;
+ struct vcpu *vcpu;
+ struct {
+ uint64_t far_el2; /* Fault Address Register */
+ uint64_t hpfar_el2; /* Hypervisor IPA Fault Address Register */
+ } exit_info;
+
+ struct vtimer_cpu vtimer_cpu;
+
+ struct vgic_v3_regs vgic_v3_regs;
+ struct vgic_v3_cpu *vgic_cpu;
+ bool has_exception;
+};
+
+struct hyp {
+ struct vm *vm;
+ struct vtimer vtimer;
+ uint64_t vmid_generation;
+ uint64_t vttbr_el2;
+ uint64_t el2_addr; /* The address of this in el2 space */
+ bool vgic_attached;
+ struct vgic_v3 *vgic;
+ struct hypctx *ctx[];
+};
+
+#define DEFINE_VMMOPS_IFUNC(ret_type, opname, args) \
+ ret_type vmmops_##opname args;
+
+DEFINE_VMMOPS_IFUNC(int, modinit, (int ipinum))
+DEFINE_VMMOPS_IFUNC(int, modcleanup, (void))
+DEFINE_VMMOPS_IFUNC(void *, init, (struct vm *vm, struct pmap *pmap))
+DEFINE_VMMOPS_IFUNC(void, gla2gpa, (void *vcpui, uint64_t gla, int prot,
+ uint64_t *gpa, int *is_fault))
+DEFINE_VMMOPS_IFUNC(int, run, (void *vcpui, register_t rip, struct pmap *pmap,
+ struct vm_eventinfo *info))
+DEFINE_VMMOPS_IFUNC(void, cleanup, (void *vmi))
+DEFINE_VMMOPS_IFUNC(void *, vcpu_init, (void *vmi, struct vcpu *vcpu,
+ int vcpu_id))
+DEFINE_VMMOPS_IFUNC(void, vcpu_cleanup, (void *vcpui))
+DEFINE_VMMOPS_IFUNC(int, exception, (void *vcpui, uint64_t esr, uint64_t far))
+DEFINE_VMMOPS_IFUNC(int, getreg, (void *vcpui, int num, uint64_t *retval))
+DEFINE_VMMOPS_IFUNC(int, setreg, (void *vcpui, int num, uint64_t val))
+DEFINE_VMMOPS_IFUNC(int, getcap, (void *vcpui, int num, int *retval))
+DEFINE_VMMOPS_IFUNC(int, setcap, (void *vcpui, int num, int val))
+DEFINE_VMMOPS_IFUNC(struct vmspace *, vmspace_alloc, (vm_offset_t min,
+ vm_offset_t max))
+DEFINE_VMMOPS_IFUNC(void, vmspace_free, (struct vmspace *vmspace))
+#ifdef notyet
+#ifdef BHYVE_SNAPSHOT
+DEFINE_VMMOPS_IFUNC(int, snapshot, (void *vmi, struct vm_snapshot_meta *meta))
+DEFINE_VMMOPS_IFUNC(int, vcpu_snapshot, (void *vcpui,
+ struct vm_snapshot_meta *meta))
+DEFINE_VMMOPS_IFUNC(int, restore_tsc, (void *vcpui, uint64_t now))
+#endif
+#endif
+
+uint64_t vmm_call_hyp(uint64_t, ...);
+void vmm_cleanup(void *hyp_stub_vectors);
+uint64_t vmm_enter_guest(struct hypctx *hypctx);
+uint64_t vmm_read_ich_vtr_el2(void);
+uint64_t vmm_read_cnthctl_el2(void);
+uint64_t vmm_read_tcr_el2(void);
+
+#define eprintf(fmt, ...) printf("%s:%d " fmt, __func__, __LINE__, ##__VA_ARGS__)
+//#define eprintf(fmt, ...) do {} while(0)
+
+#define VMID_GENERATION_MASK ((1UL<<8) - 1)
+#define build_vttbr(vmid, ptaddr) \
+ ((((vmid) & VMID_GENERATION_MASK) << VTTBR_VMID_SHIFT) | \
+ (uint64_t)(ptaddr))
+
+#define MPIDR_SMP_MASK (0x3 << 30)
+#define MPIDR_AFF1_LEVEL(x) (((x) >> 2) << 8)
+#define MPIDR_AFF0_LEVEL(x) (((x) & 0x3) << 0)
+
+/*
+ * Return true if the exception was caused by a translation fault in the stage 2
+ * translation regime. The DFSC encoding for a translation fault has the format
+ * 0b0001LL, where LL (bits [1:0]) represents the level where the fault occured
+ * (page D7-2280 of the ARMv8 Architecture Manual).
+ */
+#define ISS_DATA_DFSC_TF(esr_iss) \
+ (!((esr_iss) & 0b111000) && ((esr_iss) & 0b000100))
+#define FAR_EL2_PAGE_OFFSET(x) ((x) & PAGE_MASK)
+
+#define DEBUG_ME 0
+
+#define arm64_get_active_vcpu() ((struct hypctx *)PCPU_GET(vcpu))
+
+#endif /* !_VMM_ARM64_H_ */
diff --git a/sys/arm64/vmm/hyp.h b/sys/arm64/vmm/hyp.h
new file mode 100644
--- /dev/null
+++ b/sys/arm64/vmm/hyp.h
@@ -0,0 +1,113 @@
+/*
+ * Copyright (C) 2017 Alexandru Elisei <alexandru.elisei@gmail.com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef _VMM_HYP_H_
+#define _VMM_HYP_H_
+
+/*
+ * The translation tables for the hypervisor mode will hold mappings for kernel
+ * virtual addresses and an identity mapping (VA == PA) necessary when
+ * enabling/disabling the MMU.
+ *
+ * When in EL2 exception level the translation table base register is TTBR0_EL2
+ * and the virtual addresses generated by the CPU must be at the bottom of the
+ * memory, with the first 16 bits all set to zero:
+ *
+ * 0x0000ffffffffffff End hyp address space
+ * 0x0000000000000000 Start of hyp address space
+ *
+ * To run code in hyp mode we need to convert kernel virtual addresses to
+ * addreses that fit into this address space.
+ *
+ * The kernel virtual address range is:
+ *
+ * 0xffff007fffffffff End of KVA
+ * 0xffff000000000000 Kernel base address & start of KVA
+ *
+ * (see /sys/arm64/include/vmparam.h).
+ *
+ * We could convert the kernel virtual addresses to valid EL2 addresses by
+ * setting the first 16 bits to zero and thus mapping the kernel addresses in
+ * the bottom half of the EL2 address space, but then they might clash with the
+ * identity mapping addresses. Instead we map the kernel addresses in the upper
+ * half of the EL2 address space.
+ *
+ * The hypervisor address space will look like this:
+ *
+ * 0x0000807fffffffff End of KVA mapping
+ * 0x0000800000000000 Start of KVA mapping
+ *
+ * 0x00007fffffffffff End of identity mapping
+ * 0x0000000000000000 Start of identity mapping
+ *
+ * With the scheme we have 47 bits at our disposable for the identity map and
+ * another 47 bits for the kernel virtual addresses. For a maximum physical
+ * memory size of 128TB we are guaranteed to not have any clashes between
+ * addresses.
+ */
+#define HYP_VM_MIN_ADDRESS 0x0000000000000000
+#define HYP_VM_MAX_ADDRESS 0x0001000000000000
+
+/*
+ * When the vmm code is installed the following handles can be used by
+ * the host to call into EL2.
+ */
+#define HYP_CLEANUP 0x00000001
+#define HYP_ENTER_GUEST 0x00000002
+#define HYP_READ_REGISTER 0x00000003
+#define HYP_REG_ICH_VTR 0x1
+#define HYP_REG_CNTHCTL 0x2
+#define HYP_CLEAN_S2_TLBI 0x00000004
+#define HYP_DC_CIVAC 0x00000005
+#define HYP_EL2_TLBI 0x00000006
+#define HYP_EL2_TLBI_ALL 0x1
+#define HYP_EL2_TLBI_VA 0x2
+#define HYP_S2_TLBI_RANGE 0x00000010
+#define HYP_S2_TLBI_ALL 0x00000011
+
+/*
+ * When taking asynchronous exceptions, or interrupts, with the exception of the
+ * SError interrupt, the exception syndrome register is not updated with the
+ * exception code. We need to differentiate between the different exception
+ * types taken to EL2.
+ */
+#define EXCP_TYPE_EL1_SYNC 0
+#define EXCP_TYPE_EL1_IRQ 1
+#define EXCP_TYPE_EL1_FIQ 2
+#define EXCP_TYPE_EL1_ERROR 3
+
+#define EXCP_TYPE_EL2_SYNC 4
+#define EXCP_TYPE_EL2_IRQ 5
+#define EXCP_TYPE_EL2_FIQ 6
+#define EXCP_TYPE_EL2_ERROR 7
+
+#define EXCP_TYPE_MAINT_IRQ 8
+/* Used internally in vmm_hyp.c */
+#define EXCP_TYPE_REENTER 9
+
+#define HYP_GET_VECTOR_TABLE -1
+
+#endif /* !_VMM_HYP_H_ */
diff --git a/sys/arm64/vmm/io/vgic.h b/sys/arm64/vmm/io/vgic.h
new file mode 100644
--- /dev/null
+++ b/sys/arm64/vmm/io/vgic.h
@@ -0,0 +1,44 @@
+/*-
+ * Copyright (c) 2023 Arm Ltd
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+struct hyp;
+struct hypctx;
+struct vm_vgic_descr;
+
+extern device_t vgic_dev;
+
+bool vgic_present(void);
+void vgic_init(void);
+int vgic_attach_to_vm(struct hyp *hyp, struct vm_vgic_descr *descr);
+void vgic_detach_from_vm(struct hyp *hyp);
+void vgic_vminit(struct hyp *hyp);
+void vgic_cpuinit(struct hypctx *hypctx);
+void vgic_cpucleanup(struct hypctx *hypctx);
+void vgic_vmcleanup(struct hyp *hyp);
+bool vgic_has_pending_irq(struct hypctx *hypctx);
+int vgic_inject_irq(struct hyp *hyp, int vcpuid, uint32_t irqid, bool level);
+int vgic_inject_msi(struct hyp *hyp, uint64_t msg, uint64_t addr);
+void vgic_flush_hwstate(struct hypctx *hypctx);
+void vgic_sync_hwstate(struct hypctx *hypctx);
diff --git a/sys/arm64/vmm/io/vgic.c b/sys/arm64/vmm/io/vgic.c
new file mode 100644
--- /dev/null
+++ b/sys/arm64/vmm/io/vgic.c
@@ -0,0 +1,114 @@
+/*-
+ * Copyright (c) 2023 Arm Ltd
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+#include <sys/types.h>
+#include <sys/systm.h>
+#include <sys/bus.h>
+
+#include "vgic.h"
+#include "vgic_if.h"
+
+device_t vgic_dev;
+
+bool
+vgic_present(void)
+{
+ return (vgic_dev != NULL);
+}
+
+void
+vgic_init(void)
+{
+ VGIC_INIT(vgic_dev);
+}
+
+int
+vgic_attach_to_vm(struct hyp *hyp, struct vm_vgic_descr *descr)
+{
+ return (VGIC_ATTACH_TO_VM(vgic_dev, hyp, descr));
+}
+
+void
+vgic_detach_from_vm(struct hyp *hyp)
+{
+ VGIC_DETACH_FROM_VM(vgic_dev, hyp);
+}
+
+void
+vgic_vminit(struct hyp *hyp)
+{
+ VGIC_VMINIT(vgic_dev, hyp);
+}
+
+void
+vgic_cpuinit(struct hypctx *hypctx)
+{
+ VGIC_CPUINIT(vgic_dev, hypctx);
+}
+
+void
+vgic_cpucleanup(struct hypctx *hypctx)
+{
+ VGIC_CPUCLEANUP(vgic_dev, hypctx);
+}
+
+void
+vgic_vmcleanup(struct hyp *hyp)
+{
+ VGIC_VMCLEANUP(vgic_dev, hyp);
+}
+
+bool
+vgic_has_pending_irq(struct hypctx *hypctx)
+{
+ return (VGIC_HAS_PENDING_IRQ(vgic_dev, hypctx));
+}
+
+/* TODO: vcpuid -> hypctx ? */
+/* TODO: Add a vgic interface */
+int
+vgic_inject_irq(struct hyp *hyp, int vcpuid, uint32_t irqid, bool level)
+{
+ return (VGIC_INJECT_IRQ(vgic_dev, hyp, vcpuid, irqid, level));
+}
+
+int
+vgic_inject_msi(struct hyp *hyp, uint64_t msg, uint64_t addr)
+{
+ return (VGIC_INJECT_MSI(vgic_dev, hyp, msg, addr));
+}
+
+void
+vgic_flush_hwstate(struct hypctx *hypctx)
+{
+ VGIC_FLUSH_HWSTATE(vgic_dev, hypctx);
+}
+
+void
+vgic_sync_hwstate(struct hypctx *hypctx)
+{
+ VGIC_SYNC_HWSTATE(vgic_dev, hypctx);
+}
diff --git a/sys/arm64/vmm/io/vgic_if.m b/sys/arm64/vmm/io/vgic_if.m
new file mode 100644
--- /dev/null
+++ b/sys/arm64/vmm/io/vgic_if.m
@@ -0,0 +1,97 @@
+#-
+# Copyright (c) 2023 Arm Ltd
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+#
+
+INTERFACE vgic;
+
+HEADER {
+ struct hyp;
+ struct hypctx;
+ struct vm_vgic_descr;
+};
+
+METHOD void init {
+ device_t dev;
+}
+
+METHOD int attach_to_vm {
+ device_t dev;
+ struct hyp *hyp;
+ struct vm_vgic_descr *descr;
+};
+
+METHOD void detach_from_vm {
+ device_t dev;
+ struct hyp *hyp;
+}
+
+METHOD void vminit {
+ device_t dev;
+ struct hyp *hyp;
+}
+
+METHOD void cpuinit {
+ device_t dev;
+ struct hypctx *hypctx;
+}
+
+METHOD void cpucleanup {
+ device_t dev;
+ struct hypctx *hypctx;
+}
+
+METHOD void vmcleanup {
+ device_t dev;
+ struct hyp *hyp;
+}
+
+METHOD bool has_pending_irq {
+ device_t dev;
+ struct hypctx *hypctx;
+}
+
+METHOD int inject_irq {
+ device_t dev;
+ struct hyp *hyp;
+ int vcpuid;
+ uint32_t irqid;
+ bool level;
+}
+
+METHOD int inject_msi {
+ device_t dev;
+ struct hyp *hyp;
+ uint64_t msg;
+ uint64_t addr;
+}
+
+METHOD void flush_hwstate {
+ device_t dev;
+ struct hypctx *hypctx;
+}
+
+METHOD void sync_hwstate {
+ device_t dev;
+ struct hypctx *hypctx;
+}
diff --git a/sys/arm64/vmm/io/vgic_v3.h b/sys/arm64/vmm/io/vgic_v3.h
new file mode 100644
--- /dev/null
+++ b/sys/arm64/vmm/io/vgic_v3.h
@@ -0,0 +1,55 @@
+/*
+ * Copyright (C) 2015 Mihai Carabas <mihai.carabas@gmail.com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef _VMM_VGIC_V3_H_
+#define _VMM_VGIC_V3_H_
+
+#define VGIC_ICH_LR_NUM_MAX 16
+#define VGIC_ICH_APR_NUM_MAX 4
+
+/* Registers accessed by EL2 */
+struct vgic_v3_regs {
+ uint32_t ich_eisr_el2; /* End of Interrupt Status Register */
+ uint32_t ich_elrsr_el2; /* Empty List register Status Register (ICH_ELRSR_EL2) */
+ uint32_t ich_hcr_el2; /* Hyp Control Register */
+ uint32_t ich_misr_el2; /* Maintenance Interrupt State Register */
+ uint32_t ich_vmcr_el2; /* Virtual Machine Control Register */
+
+ /*
+ * The List Registers are part of the VM context and are modified on a
+ * world switch. They need to be allocated statically so they are
+ * mapped in the EL2 translation tables when struct hypctx is mapped.
+ */
+ uint64_t ich_lr_el2[VGIC_ICH_LR_NUM_MAX];
+ uint16_t ich_lr_num;
+
+ /* Active Priorities Registers for Group 0 and 1 interrupts */
+ uint16_t ich_apr_num;
+ uint32_t ich_ap0r_el2[VGIC_ICH_APR_NUM_MAX];
+ uint32_t ich_ap1r_el2[VGIC_ICH_APR_NUM_MAX];
+};
+
+#endif /* !_VMM_VGIC_V3_H_ */
diff --git a/sys/arm64/vmm/io/vgic_v3.c b/sys/arm64/vmm/io/vgic_v3.c
new file mode 100644
--- /dev/null
+++ b/sys/arm64/vmm/io/vgic_v3.c
@@ -0,0 +1,2206 @@
+/*
+ * Copyright (C) 2018 Alexandru Elisei <alexandru.elisei@gmail.com>
+ * All rights reserved.
+ * Copyright (C) 2020-2022 Andrew Turner
+ * Copyright (C) 2023 Arm Ltd
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+
+#include <sys/types.h>
+#include <sys/errno.h>
+#include <sys/systm.h>
+#include <sys/bitstring.h>
+#include <sys/bus.h>
+#include <sys/kernel.h>
+#include <sys/lock.h>
+#include <sys/malloc.h>
+#include <sys/module.h>
+#include <sys/mutex.h>
+#include <sys/rman.h>
+#include <sys/smp.h>
+
+#include <vm/vm.h>
+#include <vm/pmap.h>
+
+#include <dev/ofw/openfirm.h>
+
+#include <machine/armreg.h>
+#include <machine/atomic.h>
+#include <machine/bus.h>
+#include <machine/cpufunc.h>
+#include <machine/cpu.h>
+#include <machine/machdep.h>
+#include <machine/param.h>
+#include <machine/pmap.h>
+#include <machine/vmparam.h>
+#include <machine/intr.h>
+#include <machine/vmm.h>
+#include <machine/vmm_dev.h>
+#include <machine/vmm_instruction_emul.h>
+
+#include <arm/arm/gic_common.h>
+#include <arm64/arm64/gic_v3_reg.h>
+#include <arm64/arm64/gic_v3_var.h>
+
+#include <arm64/vmm/hyp.h>
+#include <arm64/vmm/mmu.h>
+#include <arm64/vmm/arm64.h>
+
+#include "vgic.h"
+#include "vgic_v3.h"
+#include "vgic_v3_reg.h"
+
+#include "vgic_if.h"
+
+#define VGIC_SGI_NUM (GIC_LAST_SGI - GIC_FIRST_SGI + 1)
+#define VGIC_PPI_NUM (GIC_LAST_PPI - GIC_FIRST_PPI + 1)
+#define VGIC_SPI_NUM (GIC_LAST_SPI - GIC_FIRST_SPI + 1)
+#define VGIC_PRV_I_NUM (VGIC_SGI_NUM + VGIC_PPI_NUM)
+#define VGIC_SHR_I_NUM (VGIC_SPI_NUM)
+
+MALLOC_DEFINE(M_VGIC_V3, "ARM VMM VGIC V3", "ARM VMM VGIC V3");
+
+/* TODO: Move to softc */
+struct vgic_v3_virt_features {
+ uint8_t min_prio;
+ size_t ich_lr_num;
+ size_t ich_apr_num;
+};
+
+struct vgic_v3_irq {
+ /* List of IRQs that are active or pending */
+ TAILQ_ENTRY(vgic_v3_irq) act_pend_list;
+ struct mtx irq_spinmtx;
+ uint64_t mpidr;
+ int target_vcpu;
+ uint32_t irq;
+ bool active;
+ bool pending;
+ bool enabled;
+ bool level;
+ bool on_aplist;
+ uint8_t priority;
+ uint8_t config;
+#define VGIC_CONFIG_MASK 0x2
+#define VGIC_CONFIG_LEVEL 0x0
+#define VGIC_CONFIG_EDGE 0x2
+};
+
+/* Global data not needed by EL2 */
+struct vgic_v3 {
+ struct mtx dist_mtx;
+ uint64_t dist_start;
+ size_t dist_end;
+
+ uint64_t redist_start;
+ size_t redist_end;
+
+ uint32_t gicd_ctlr; /* Distributor Control Register */
+
+ struct vgic_v3_irq *irqs;
+};
+
+/* Per-CPU data not needed by EL2 */
+struct vgic_v3_cpu {
+ /*
+ * We need a mutex for accessing the list registers because they are
+ * modified asynchronously by the virtual timer.
+ *
+ * Note that the mutex *MUST* be a spin mutex because an interrupt can
+ * be injected by a callout callback function, thereby modifying the
+ * list registers from a context where sleeping is forbidden.
+ */
+ struct mtx lr_mtx;
+
+ struct vgic_v3_irq private_irqs[VGIC_PRV_I_NUM];
+ TAILQ_HEAD(, vgic_v3_irq) irq_act_pend;
+ u_int ich_lr_used;
+
+ uint64_t gicr_typer; /* Redistributor Type Register */
+};
+
+/* How many IRQs we support (SGIs + PPIs + SPIs). Not including LPIs */
+#define VGIC_NIRQS 1023
+/* Pretend to be an Arm design */
+#define VGIC_IIDR 0x43b
+
+static vgic_inject_irq_t vgic_v3_inject_irq;
+static vgic_inject_msi_t vgic_v3_inject_msi;
+
+#define INJECT_IRQ(hyp, vcpuid, irqid, level) \
+ vgic_v3_inject_irq(NULL, (hyp), (vcpuid), (irqid), (level))
+
+typedef void (register_read)(struct hypctx *, u_int, uint64_t *, void *);
+typedef void (register_write)(struct hypctx *, u_int, u_int, u_int,
+ uint64_t, void *);
+
+#define VGIC_8_BIT (1 << 0)
+/* (1 << 1) is reserved for 16 bit accesses */
+#define VGIC_32_BIT (1 << 2)
+#define VGIC_64_BIT (1 << 3)
+
+struct vgic_register {
+ u_int start; /* Start within a memory region */
+ u_int end;
+ u_int size;
+ u_int flags;
+ register_read *read;
+ register_write *write;
+};
+
+#define VGIC_REGISTER_RANGE(reg_start, reg_end, reg_size, reg_flags, readf, \
+ writef) \
+{ \
+ .start = (reg_start), \
+ .end = (reg_end), \
+ .size = (reg_size), \
+ .flags = (reg_flags), \
+ .read = (readf), \
+ .write = (writef), \
+}
+
+#define VGIC_REGISTER_RANGE_RAZ_WI(reg_start, reg_end, reg_size, reg_flags) \
+ VGIC_REGISTER_RANGE(reg_start, reg_end, reg_size, reg_flags, \
+ gic_zero_read, gic_ignore_write)
+
+#define VGIC_REGISTER(start_addr, reg_size, reg_flags, readf, writef) \
+ VGIC_REGISTER_RANGE(start_addr, (start_addr) + (reg_size), \
+ reg_size, reg_flags, readf, writef)
+
+#define VGIC_REGISTER_RAZ_WI(start_addr, reg_size, reg_flags) \
+ VGIC_REGISTER_RANGE_RAZ_WI(start_addr, \
+ (start_addr) + (reg_size), reg_size, reg_flags)
+
+static register_read gic_pidr2_read;
+static register_read gic_zero_read;
+static register_write gic_ignore_write;
+
+/* GICD_CTLR */
+static register_read dist_ctlr_read;
+static register_write dist_ctlr_write;
+/* GICD_TYPER */
+static register_read dist_typer_read;
+/* GICD_IIDR */
+static register_read dist_iidr_read;
+/* GICD_STATUSR - RAZ/WI as we don't report errors (yet) */
+/* GICD_SETSPI_NSR & GICD_CLRSPI_NSR */
+static register_write dist_setclrspi_nsr_write;
+/* GICD_SETSPI_SR - RAZ/WI */
+/* GICD_CLRSPI_SR - RAZ/WI */
+/* GICD_IGROUPR - RAZ/WI as GICD_CTLR.ARE == 1 */
+/* GICD_ISENABLER */
+static register_read dist_isenabler_read;
+static register_write dist_isenabler_write;
+/* GICD_ICENABLER */
+static register_read dist_icenabler_read;
+static register_write dist_icenabler_write;
+/* GICD_ISPENDR */
+static register_read dist_ispendr_read;
+static register_write dist_ispendr_write;
+/* GICD_ICPENDR */
+static register_read dist_icpendr_read;
+static register_write dist_icpendr_write;
+/* GICD_ISACTIVER */
+static register_read dist_isactiver_read;
+static register_write dist_isactiver_write;
+/* GICD_ICACTIVER */
+static register_read dist_icactiver_read;
+static register_write dist_icactiver_write;
+/* GICD_IPRIORITYR */
+static register_read dist_ipriorityr_read;
+static register_write dist_ipriorityr_write;
+/* GICD_ITARGETSR - RAZ/WI as GICD_CTLR.ARE == 1 */
+/* GICD_ICFGR */
+static register_read dist_icfgr_read;
+static register_write dist_icfgr_write;
+/* GICD_IGRPMODR - RAZ/WI from non-secure mode */
+/* GICD_NSACR - RAZ/WI from non-secure mode */
+/* GICD_SGIR - RAZ/WI as GICD_CTLR.ARE == 1 */
+/* GICD_CPENDSGIR - RAZ/WI as GICD_CTLR.ARE == 1 */
+/* GICD_SPENDSGIR - RAZ/WI as GICD_CTLR.ARE == 1 */
+/* GICD_IROUTER */
+static register_read dist_irouter_read;
+static register_write dist_irouter_write;
+
+static struct vgic_register dist_registers[] = {
+ VGIC_REGISTER(GICD_CTLR, 4, VGIC_32_BIT, dist_ctlr_read,
+ dist_ctlr_write),
+ VGIC_REGISTER(GICD_TYPER, 4, VGIC_32_BIT, dist_typer_read,
+ gic_ignore_write),
+ VGIC_REGISTER(GICD_IIDR, 4, VGIC_32_BIT, dist_iidr_read,
+ gic_ignore_write),
+ VGIC_REGISTER_RAZ_WI(GICD_STATUSR, 4, VGIC_32_BIT),
+ VGIC_REGISTER(GICD_SETSPI_NSR, 4, VGIC_32_BIT, gic_zero_read,
+ dist_setclrspi_nsr_write),
+ VGIC_REGISTER(GICD_CLRSPI_NSR, 4, VGIC_32_BIT, gic_zero_read,
+ dist_setclrspi_nsr_write),
+ VGIC_REGISTER_RAZ_WI(GICD_SETSPI_SR, 4, VGIC_32_BIT),
+ VGIC_REGISTER_RAZ_WI(GICD_CLRSPI_SR, 4, VGIC_32_BIT),
+ VGIC_REGISTER_RANGE_RAZ_WI(GICD_IGROUPR(0), GICD_IGROUPR(1024), 4,
+ VGIC_32_BIT),
+
+ VGIC_REGISTER_RAZ_WI(GICD_ISENABLER(0), 4, VGIC_32_BIT),
+ VGIC_REGISTER_RANGE(GICD_ISENABLER(32), GICD_ISENABLER(1024), 4,
+ VGIC_32_BIT, dist_isenabler_read, dist_isenabler_write),
+
+ VGIC_REGISTER_RAZ_WI(GICD_ICENABLER(0), 4, VGIC_32_BIT),
+ VGIC_REGISTER_RANGE(GICD_ICENABLER(32), GICD_ICENABLER(1024), 4,
+ VGIC_32_BIT, dist_icenabler_read, dist_icenabler_write),
+
+ VGIC_REGISTER_RAZ_WI(GICD_ISPENDR(0), 4, VGIC_32_BIT),
+ VGIC_REGISTER_RANGE(GICD_ISPENDR(32), GICD_ISPENDR(1024), 4,
+ VGIC_32_BIT, dist_ispendr_read, dist_ispendr_write),
+
+ VGIC_REGISTER_RAZ_WI(GICD_ICPENDR(0), 4, VGIC_32_BIT),
+ VGIC_REGISTER_RANGE(GICD_ICPENDR(32), GICD_ICPENDR(1024), 4,
+ VGIC_32_BIT, dist_icpendr_read, dist_icpendr_write),
+
+ VGIC_REGISTER_RAZ_WI(GICD_ISACTIVER(0), 4, VGIC_32_BIT),
+ VGIC_REGISTER_RANGE(GICD_ISACTIVER(32), GICD_ISACTIVER(1024), 4,
+ VGIC_32_BIT, dist_isactiver_read, dist_isactiver_write),
+
+ VGIC_REGISTER_RAZ_WI(GICD_ICACTIVER(0), 4, VGIC_32_BIT),
+ VGIC_REGISTER_RANGE(GICD_ICACTIVER(32), GICD_ICACTIVER(1024), 4,
+ VGIC_32_BIT, dist_icactiver_read, dist_icactiver_write),
+
+ VGIC_REGISTER_RANGE_RAZ_WI(GICD_IPRIORITYR(0), GICD_IPRIORITYR(32), 4,
+ VGIC_32_BIT | VGIC_8_BIT),
+ VGIC_REGISTER_RANGE(GICD_IPRIORITYR(32), GICD_IPRIORITYR(1024), 4,
+ VGIC_32_BIT | VGIC_8_BIT, dist_ipriorityr_read,
+ dist_ipriorityr_write),
+
+ VGIC_REGISTER_RANGE_RAZ_WI(GICD_ITARGETSR(0), GICD_ITARGETSR(1024), 4,
+ VGIC_32_BIT | VGIC_8_BIT),
+
+ VGIC_REGISTER_RANGE_RAZ_WI(GICD_ICFGR(0), GICD_ICFGR(32), 4,
+ VGIC_32_BIT),
+ VGIC_REGISTER_RANGE(GICD_ICFGR(32), GICD_ICFGR(1024), 4,
+ VGIC_32_BIT, dist_icfgr_read, dist_icfgr_write),
+/*
+ VGIC_REGISTER_RANGE(GICD_IGRPMODR(0), GICD_IGRPMODR(1024), 4,
+ VGIC_32_BIT, dist_igrpmodr_read, dist_igrpmodr_write),
+ VGIC_REGISTER_RANGE(GICD_NSACR(0), GICD_NSACR(1024), 4,
+ VGIC_32_BIT, dist_nsacr_read, dist_nsacr_write),
+*/
+ VGIC_REGISTER_RAZ_WI(GICD_SGIR, 4, VGIC_32_BIT),
+/*
+ VGIC_REGISTER_RANGE(GICD_CPENDSGIR(0), GICD_CPENDSGIR(1024), 4,
+ VGIC_32_BIT | VGIC_8_BIT, dist_cpendsgir_read,
+ dist_cpendsgir_write),
+ VGIC_REGISTER_RANGE(GICD_SPENDSGIR(0), GICD_SPENDSGIR(1024), 4,
+ VGIC_32_BIT | VGIC_8_BIT, dist_spendsgir_read,
+ dist_spendsgir_write),
+*/
+ VGIC_REGISTER_RANGE(GICD_IROUTER(32), GICD_IROUTER(1024), 8,
+ VGIC_64_BIT | VGIC_32_BIT, dist_irouter_read, dist_irouter_write),
+
+ VGIC_REGISTER_RANGE_RAZ_WI(GICD_PIDR4, GICD_PIDR2, 4, VGIC_32_BIT),
+ VGIC_REGISTER(GICD_PIDR2, 4, VGIC_32_BIT, gic_pidr2_read,
+ gic_ignore_write),
+ VGIC_REGISTER_RANGE_RAZ_WI(GICD_PIDR2 + 4, GICD_SIZE, 4, VGIC_32_BIT),
+};
+
+/* GICR_CTLR - Ignore writes as no bits can be set */
+static register_read redist_ctlr_read;
+/* GICR_IIDR */
+static register_read redist_iidr_read;
+/* GICR_TYPER */
+static register_read redist_typer_read;
+/* GICR_STATUSR - RAZ/WI as we don't report errors (yet) */
+/* GICR_WAKER - RAZ/WI from non-secure mode */
+/* GICR_SETLPIR - RAZ/WI as no LPIs are supported */
+/* GICR_CLRLPIR - RAZ/WI as no LPIs are supported */
+/* GICR_PROPBASER - RAZ/WI as no LPIs are supported */
+/* GICR_PENDBASER - RAZ/WI as no LPIs are supported */
+/* GICR_INVLPIR - RAZ/WI as no LPIs are supported */
+/* GICR_INVALLR - RAZ/WI as no LPIs are supported */
+/* GICR_SYNCR - RAZ/WI as no LPIs are supported */
+
+static struct vgic_register redist_rd_registers[] = {
+ VGIC_REGISTER(GICR_CTLR, 4, VGIC_32_BIT, redist_ctlr_read,
+ gic_ignore_write),
+ VGIC_REGISTER(GICR_IIDR, 4, VGIC_32_BIT, redist_iidr_read,
+ gic_ignore_write),
+ VGIC_REGISTER(GICR_TYPER, 8, VGIC_64_BIT | VGIC_32_BIT,
+ redist_typer_read, gic_ignore_write),
+ VGIC_REGISTER_RAZ_WI(GICR_STATUSR, 4, VGIC_32_BIT),
+ VGIC_REGISTER_RAZ_WI(GICR_WAKER, 4, VGIC_32_BIT),
+ VGIC_REGISTER_RAZ_WI(GICR_SETLPIR, 8, VGIC_64_BIT | VGIC_32_BIT),
+ VGIC_REGISTER_RAZ_WI(GICR_CLRLPIR, 8, VGIC_64_BIT | VGIC_32_BIT),
+ VGIC_REGISTER_RAZ_WI(GICR_PROPBASER, 8, VGIC_64_BIT | VGIC_32_BIT),
+ VGIC_REGISTER_RAZ_WI(GICR_PENDBASER, 8, VGIC_64_BIT | VGIC_32_BIT),
+ VGIC_REGISTER_RAZ_WI(GICR_INVLPIR, 8, VGIC_64_BIT | VGIC_32_BIT),
+ VGIC_REGISTER_RAZ_WI(GICR_INVALLR, 8, VGIC_64_BIT | VGIC_32_BIT),
+ VGIC_REGISTER_RAZ_WI(GICR_SYNCR, 4, VGIC_32_BIT),
+
+ /* These are identical to the dist registers */
+ VGIC_REGISTER_RANGE_RAZ_WI(GICD_PIDR4, GICD_PIDR2, 4, VGIC_32_BIT),
+ VGIC_REGISTER(GICD_PIDR2, 4, VGIC_32_BIT, gic_pidr2_read,
+ gic_ignore_write),
+ VGIC_REGISTER_RANGE_RAZ_WI(GICD_PIDR2 + 4, GICD_SIZE, 4,
+ VGIC_32_BIT),
+};
+
+/* GICR_IGROUPR0 - RAZ/WI from non-secure mode */
+/* GICR_ISENABLER0 */
+static register_read redist_ienabler0_read;
+static register_write redist_isenabler0_write;
+/* GICR_ICENABLER0 */
+static register_write redist_icenabler0_write;
+/* GICR_ISPENDR0 */
+static register_read redist_ipendr0_read;
+static register_write redist_ispendr0_write;
+/* GICR_ICPENDR0 */
+static register_write redist_icpendr0_write;
+/* GICR_ISACTIVER0 */
+static register_read redist_iactiver0_read;
+static register_write redist_isactiver0_write;
+/* GICR_ICACTIVER0 */
+static register_write redist_icactiver0_write;
+/* GICR_IPRIORITYR */
+static register_read redist_ipriorityr_read;
+static register_write redist_ipriorityr_write;
+/* GICR_ICFGR0 - RAZ/WI from non-secure mode */
+/* GICR_ICFGR1 */
+static register_read redist_icfgr1_read;
+static register_write redist_icfgr1_write;
+/* GICR_IGRPMODR0 - RAZ/WI from non-secure mode */
+/* GICR_NSCAR - RAZ/WI from non-secure mode */
+
+static struct vgic_register redist_sgi_registers[] = {
+ VGIC_REGISTER_RAZ_WI(GICR_IGROUPR0, 4, VGIC_32_BIT),
+ VGIC_REGISTER(GICR_ISENABLER0, 4, VGIC_32_BIT, redist_ienabler0_read,
+ redist_isenabler0_write),
+ VGIC_REGISTER(GICR_ICENABLER0, 4, VGIC_32_BIT, redist_ienabler0_read,
+ redist_icenabler0_write),
+ VGIC_REGISTER(GICR_ISPENDR0, 4, VGIC_32_BIT, redist_ipendr0_read,
+ redist_ispendr0_write),
+ VGIC_REGISTER(GICR_ICPENDR0, 4, VGIC_32_BIT, redist_ipendr0_read,
+ redist_icpendr0_write),
+ VGIC_REGISTER(GICR_ISACTIVER0, 4, VGIC_32_BIT, redist_iactiver0_read,
+ redist_isactiver0_write),
+ VGIC_REGISTER(GICR_ICACTIVER0, 4, VGIC_32_BIT, redist_iactiver0_read,
+ redist_icactiver0_write),
+ VGIC_REGISTER_RANGE(GICR_IPRIORITYR(0), GICR_IPRIORITYR(32), 4,
+ VGIC_32_BIT | VGIC_8_BIT, redist_ipriorityr_read,
+ redist_ipriorityr_write),
+ VGIC_REGISTER_RAZ_WI(GICR_ICFGR0, 4, VGIC_32_BIT),
+ VGIC_REGISTER(GICR_ICFGR1, 4, VGIC_32_BIT, redist_icfgr1_read,
+ redist_icfgr1_write),
+ VGIC_REGISTER_RAZ_WI(GICR_IGRPMODR0, 4, VGIC_32_BIT),
+ VGIC_REGISTER_RAZ_WI(GICR_NSACR, 4, VGIC_32_BIT),
+};
+
+static struct vgic_v3_virt_features virt_features;
+
+static struct vgic_v3_irq *vgic_v3_get_irq(struct hyp *, int, uint32_t);
+static void vgic_v3_release_irq(struct vgic_v3_irq *);
+
+/* TODO: Move to a common file */
+static int
+mpidr_to_vcpu(struct hyp *hyp, uint64_t mpidr)
+{
+ struct vm *vm;
+ struct hypctx *hypctx;
+
+ vm = hyp->vm;
+ for (int i = 0; i < vm_get_maxcpus(vm); i++) {
+ hypctx = hyp->ctx[i];
+ if (hypctx != NULL && (hypctx->vmpidr_el2 & GICD_AFF) == mpidr)
+ return (i);
+ }
+ return (-1);
+}
+
+static void
+vgic_v3_vminit(device_t dev, struct hyp *hyp)
+{
+ struct vgic_v3 *vgic;
+
+ hyp->vgic = malloc(sizeof(*hyp->vgic), M_VGIC_V3,
+ M_WAITOK | M_ZERO);
+ vgic = hyp->vgic;
+
+ /*
+ * Configure the Distributor control register. The register resets to an
+ * architecturally UNKNOWN value, so we reset to 0 to disable all
+ * functionality controlled by the register.
+ *
+ * The exception is GICD_CTLR.DS, which is RA0/WI when the Distributor
+ * supports one security state (ARM GIC Architecture Specification for
+ * GICv3 and GICv4, p. 4-464)
+ */
+ vgic->gicd_ctlr = 0;
+
+ mtx_init(&vgic->dist_mtx, "VGICv3 Distributor lock", NULL,
+ MTX_SPIN);
+}
+
+static void
+vgic_v3_cpuinit(device_t dev, struct hypctx *hypctx)
+{
+ struct vgic_v3_cpu *vgic_cpu;
+ struct vgic_v3_irq *irq;
+ uint64_t aff, vmpidr_el2;
+ int i, irqid;
+ bool last_vcpu;
+
+ hypctx->vgic_cpu = malloc(sizeof(*hypctx->vgic_cpu),
+ M_VGIC_V3, M_WAITOK | M_ZERO);
+ vgic_cpu = hypctx->vgic_cpu;
+
+ last_vcpu =
+ vcpu_vcpuid(hypctx->vcpu) == (vm_get_maxcpus(hypctx->hyp->vm) - 1);
+
+ vmpidr_el2 = hypctx->vmpidr_el2;
+ KASSERT(vmpidr_el2 != 0,
+ ("Trying to init this CPU's vGIC before the vCPU"));
+ /*
+ * Get affinity for the current CPU. The guest CPU affinity is taken
+ * from VMPIDR_EL2. The Redistributor corresponding to this CPU is
+ * the Redistributor with the same affinity from GICR_TYPER.
+ */
+ aff = (CPU_AFF3(vmpidr_el2) << 24) | (CPU_AFF2(vmpidr_el2) << 16) |
+ (CPU_AFF1(vmpidr_el2) << 8) | CPU_AFF0(vmpidr_el2);
+
+ /* Set up GICR_TYPER. */
+ vgic_cpu->gicr_typer = aff << GICR_TYPER_AFF_SHIFT;
+ /* Set the vcpu as the processsor ID */
+ vgic_cpu->gicr_typer |=
+ (uint64_t)vcpu_vcpuid(hypctx->vcpu) << GICR_TYPER_CPUNUM_SHIFT;
+
+ if (last_vcpu)
+ /* Mark the last Redistributor */
+ vgic_cpu->gicr_typer |= GICR_TYPER_LAST;
+
+ mtx_init(&vgic_cpu->lr_mtx, "VGICv3 ICH_LR_EL2 lock", NULL, MTX_SPIN);
+
+ /* Set the SGI and PPI state */
+ for (irqid = 0; irqid < VGIC_PRV_I_NUM; irqid++) {
+ irq = &vgic_cpu->private_irqs[irqid];
+
+ mtx_init(&irq->irq_spinmtx, "VGIC IRQ spinlock", NULL,
+ MTX_SPIN);
+ irq->irq = irqid;
+ irq->mpidr = hypctx->vmpidr_el2 & GICD_AFF;
+ irq->target_vcpu = vcpu_vcpuid(hypctx->vcpu);
+ MPASS(irq->target_vcpu >= 0);
+
+ if (irqid < VGIC_SGI_NUM) {
+ /* SGIs */
+ irq->enabled = true;
+ irq->config = VGIC_CONFIG_EDGE;
+ } else {
+ /* PPIs */
+ irq->config = VGIC_CONFIG_LEVEL;
+ }
+ irq->priority = 0;
+ }
+
+ /*
+ * Configure the Interrupt Controller Hyp Control Register.
+ *
+ * ICH_HCR_EL2_En: enable virtual CPU interface.
+ *
+ * Maintenance interrupts are disabled.
+ */
+ hypctx->vgic_v3_regs.ich_hcr_el2 = ICH_HCR_EL2_En;
+
+ /*
+ * Configure the Interrupt Controller Virtual Machine Control Register.
+ *
+ * ICH_VMCR_EL2_VPMR: lowest priority mask for the VCPU interface
+ * ICH_VMCR_EL2_VBPR1_NO_PREEMPTION: disable interrupt preemption for
+ * Group 1 interrupts
+ * ICH_VMCR_EL2_VBPR0_NO_PREEMPTION: disable interrupt preemption for
+ * Group 0 interrupts
+ * ~ICH_VMCR_EL2_VEOIM: writes to EOI registers perform priority drop
+ * and interrupt deactivation.
+ * ICH_VMCR_EL2_VENG0: virtual Group 0 interrupts enabled.
+ * ICH_VMCR_EL2_VENG1: virtual Group 1 interrupts enabled.
+ */
+ hypctx->vgic_v3_regs.ich_vmcr_el2 = \
+ (virt_features.min_prio << ICH_VMCR_EL2_VPMR_SHIFT) | \
+ ICH_VMCR_EL2_VBPR1_NO_PREEMPTION | ICH_VMCR_EL2_VBPR0_NO_PREEMPTION;
+ hypctx->vgic_v3_regs.ich_vmcr_el2 &= ~ICH_VMCR_EL2_VEOIM;
+ hypctx->vgic_v3_regs.ich_vmcr_el2 |= ICH_VMCR_EL2_VENG0 |
+ ICH_VMCR_EL2_VENG1;
+
+ hypctx->vgic_v3_regs.ich_lr_num = virt_features.ich_lr_num;
+ for (i = 0; i < hypctx->vgic_v3_regs.ich_lr_num; i++)
+ hypctx->vgic_v3_regs.ich_lr_el2[i] = 0UL;
+ vgic_cpu->ich_lr_used = 0;
+ TAILQ_INIT(&vgic_cpu->irq_act_pend);
+
+ hypctx->vgic_v3_regs.ich_apr_num = virt_features.ich_apr_num;
+}
+
+static void
+vgic_v3_cpucleanup(device_t dev, struct hypctx *hypctx)
+{
+ struct vgic_v3_cpu *vgic_cpu;
+ struct vgic_v3_irq *irq;
+ int irqid;
+
+ vgic_cpu = hypctx->vgic_cpu;
+ for (irqid = 0; irqid < VGIC_PRV_I_NUM; irqid++) {
+ irq = &vgic_cpu->private_irqs[irqid];
+ mtx_destroy(&irq->irq_spinmtx);
+ }
+
+ mtx_destroy(&vgic_cpu->lr_mtx);
+ free(hypctx->vgic_cpu, M_VGIC_V3);
+}
+
+static void
+vgic_v3_vmcleanup(device_t dev, struct hyp *hyp)
+{
+ mtx_destroy(&hyp->vgic->dist_mtx);
+ free(hyp->vgic, M_VGIC_V3);
+}
+
+static bool
+vgic_v3_irq_pending(struct vgic_v3_irq *irq)
+{
+ if ((irq->config & VGIC_CONFIG_MASK) == VGIC_CONFIG_LEVEL) {
+ return (irq->pending || irq->level);
+ } else {
+ return (irq->pending);
+ }
+}
+
+static bool
+vgic_v3_queue_irq(struct hyp *hyp, struct vgic_v3_cpu *vgic_cpu,
+ int vcpuid, struct vgic_v3_irq *irq)
+{
+ MPASS(vcpuid >= 0);
+ MPASS(vcpuid < vm_get_maxcpus(hyp->vm));
+
+ mtx_assert(&vgic_cpu->lr_mtx, MA_OWNED);
+ mtx_assert(&irq->irq_spinmtx, MA_OWNED);
+
+ /* No need to queue the IRQ */
+ if (!irq->level && !irq->pending)
+ return (false);
+
+ if (!irq->on_aplist) {
+ irq->on_aplist = true;
+ TAILQ_INSERT_TAIL(&vgic_cpu->irq_act_pend, irq, act_pend_list);
+ }
+ return (true);
+}
+
+static uint64_t
+gic_reg_value_64(uint64_t field, uint64_t val, u_int offset, u_int size)
+{
+ uint32_t mask;
+
+ if (offset != 0 || size != 8) {
+ mask = ((1ul << (size * 8)) - 1) << (offset * 8);
+ /* Shift the new bits to the correct place */
+ val <<= (offset * 8);
+ /* Keep only the interesting bits */
+ val &= mask;
+ /* Add the bits we are keeping from the old value */
+ val |= field & ~mask;
+ }
+
+ return (val);
+}
+
+static void
+gic_pidr2_read(struct hypctx *hypctx, u_int reg, uint64_t *rval,
+ void *arg)
+{
+ *rval = GICR_PIDR2_ARCH_GICv3 << GICR_PIDR2_ARCH_SHIFT;
+}
+
+/* Common read-only/write-ignored helpers */
+static void
+gic_zero_read(struct hypctx *hypctx, u_int reg, uint64_t *rval,
+ void *arg)
+{
+ *rval = 0;
+}
+
+static void
+gic_ignore_write(struct hypctx *hypctx, u_int reg, u_int offset, u_int size,
+ uint64_t wval, void *arg)
+{
+ /* Nothing to do */
+}
+
+static uint64_t
+read_enabler(struct hypctx *hypctx, int n)
+{
+ struct vgic_v3_irq *irq;
+ uint64_t ret;
+ uint32_t irq_base;
+ int i;
+
+ ret = 0;
+ irq_base = n * 32;
+ for (i = 0; i < 32; i++) {
+ irq = vgic_v3_get_irq(hypctx->hyp, vcpu_vcpuid(hypctx->vcpu),
+ irq_base + i);
+ if (irq == NULL)
+ continue;
+
+ if (!irq->enabled)
+ ret |= 1u << i;
+ vgic_v3_release_irq(irq);
+ }
+
+ return (ret);
+}
+
+static void
+write_enabler(struct hypctx *hypctx,int n, bool set, uint64_t val)
+{
+ struct vgic_v3_irq *irq;
+ uint32_t irq_base;
+ int i;
+
+ irq_base = n * 32;
+ for (i = 0; i < 32; i++) {
+ /* We only change interrupts when the appropriate bit is set */
+ if ((val & (1u << i)) == 0)
+ continue;
+
+ /* Find the interrupt this bit represents */
+ irq = vgic_v3_get_irq(hypctx->hyp, vcpu_vcpuid(hypctx->vcpu),
+ irq_base + i);
+ if (irq == NULL)
+ continue;
+
+ irq->enabled = set;
+ vgic_v3_release_irq(irq);
+ }
+}
+
+static uint64_t
+read_pendr(struct hypctx *hypctx, int n)
+{
+ struct vgic_v3_irq *irq;
+ uint64_t ret;
+ uint32_t irq_base;
+ int i;
+
+ ret = 0;
+ irq_base = n * 32;
+ for (i = 0; i < 32; i++) {
+ irq = vgic_v3_get_irq(hypctx->hyp, vcpu_vcpuid(hypctx->vcpu),
+ irq_base + i);
+ if (irq == NULL)
+ continue;
+
+ if (vgic_v3_irq_pending(irq))
+ ret |= 1u << i;
+ vgic_v3_release_irq(irq);
+ }
+
+ return (ret);
+}
+
+static uint64_t
+write_pendr(struct hypctx *hypctx, int n, bool set, uint64_t val)
+{
+ struct vgic_v3_cpu *vgic_cpu;
+ struct vgic_v3_irq *irq;
+ struct hyp *hyp;
+ struct hypctx *target_hypctx;
+ uint64_t ret;
+ uint32_t irq_base;
+ int target_vcpu, i;
+ bool notify;
+
+ hyp = hypctx->hyp;
+ ret = 0;
+ irq_base = n * 32;
+ for (i = 0; i < 32; i++) {
+ /* We only change interrupts when the appropriate bit is set */
+ if ((val & (1u << i)) == 0)
+ continue;
+
+ irq = vgic_v3_get_irq(hypctx->hyp, vcpu_vcpuid(hypctx->vcpu),
+ irq_base + i);
+ if (irq == NULL)
+ continue;
+
+ notify = false;
+ target_vcpu = irq->target_vcpu;
+ if (target_vcpu < 0)
+ goto next_irq;
+ target_hypctx = hyp->ctx[target_vcpu];
+ if (target_hypctx == NULL)
+ goto next_irq;
+ vgic_cpu = target_hypctx->vgic_cpu;
+
+ if (!set) {
+ /* pending -> not pending */
+ irq->pending = false;
+ } else {
+ irq->pending = true;
+ mtx_lock_spin(&vgic_cpu->lr_mtx);
+ notify = vgic_v3_queue_irq(hyp, vgic_cpu, target_vcpu,
+ irq);
+ mtx_unlock_spin(&vgic_cpu->lr_mtx);
+ }
+next_irq:
+ vgic_v3_release_irq(irq);
+
+ if (notify)
+ vcpu_notify_event(vm_vcpu(hyp->vm, target_vcpu), false);
+ }
+
+ return (ret);
+}
+
+static uint64_t
+read_activer(struct hypctx *hypctx, int n)
+{
+ struct vgic_v3_irq *irq;
+ uint64_t ret;
+ uint32_t irq_base;
+ int i;
+
+ ret = 0;
+ irq_base = n * 32;
+ for (i = 0; i < 32; i++) {
+ irq = vgic_v3_get_irq(hypctx->hyp, vcpu_vcpuid(hypctx->vcpu),
+ irq_base + i);
+ if (irq == NULL)
+ continue;
+
+ if (irq->active)
+ ret |= 1u << i;
+ vgic_v3_release_irq(irq);
+ }
+
+ return (ret);
+}
+
+static void
+write_activer(struct hypctx *hypctx, u_int n, bool set, uint64_t val)
+{
+ struct vgic_v3_cpu *vgic_cpu;
+ struct vgic_v3_irq *irq;
+ struct hyp *hyp;
+ struct hypctx *target_hypctx;
+ uint32_t irq_base;
+ int target_vcpu, i;
+ bool notify;
+
+ hyp = hypctx->hyp;
+ irq_base = n * 32;
+ for (i = 0; i < 32; i++) {
+ /* We only change interrupts when the appropriate bit is set */
+ if ((val & (1u << i)) == 0)
+ continue;
+
+ irq = vgic_v3_get_irq(hypctx->hyp, vcpu_vcpuid(hypctx->vcpu),
+ irq_base + i);
+ if (irq == NULL)
+ continue;
+
+ notify = false;
+ target_vcpu = irq->target_vcpu;
+ if (target_vcpu < 0)
+ goto next_irq;
+ target_hypctx = hyp->ctx[target_vcpu];
+ if (target_hypctx == NULL)
+ goto next_irq;
+ vgic_cpu = target_hypctx->vgic_cpu;
+
+ if (!set) {
+ /* active -> not active */
+ irq->active = false;
+ } else {
+ /* not active -> active */
+ irq->active = true;
+ mtx_lock_spin(&vgic_cpu->lr_mtx);
+ notify = vgic_v3_queue_irq(hyp, vgic_cpu, target_vcpu,
+ irq);
+ mtx_unlock_spin(&vgic_cpu->lr_mtx);
+ }
+next_irq:
+ vgic_v3_release_irq(irq);
+
+ if (notify)
+ vcpu_notify_event(vm_vcpu(hyp->vm, target_vcpu), false);
+ }
+}
+
+static uint64_t
+read_priorityr(struct hypctx *hypctx, int n)
+{
+ struct vgic_v3_irq *irq;
+ uint64_t ret;
+ uint32_t irq_base;
+ int i;
+
+ ret = 0;
+ irq_base = n * 4;
+ for (i = 0; i < 4; i++) {
+ irq = vgic_v3_get_irq(hypctx->hyp, vcpu_vcpuid(hypctx->vcpu),
+ irq_base + i);
+ if (irq == NULL)
+ continue;
+
+ ret |= ((uint64_t)irq->priority) << (i * 8);
+ vgic_v3_release_irq(irq);
+ }
+
+ return (ret);
+}
+
+static void
+write_priorityr(struct hypctx *hypctx, u_int irq_base, u_int size, uint64_t val)
+{
+ struct vgic_v3_irq *irq;
+ int i;
+
+ for (i = 0; i < size; i++) {
+ irq = vgic_v3_get_irq(hypctx->hyp, vcpu_vcpuid(hypctx->vcpu),
+ irq_base + i);
+ if (irq == NULL)
+ continue;
+
+ /* Set the priority. We support 32 priority steps (5 bits) */
+ irq->priority = (val >> (i * 8)) & 0xf8;
+ vgic_v3_release_irq(irq);
+ }
+}
+
+static uint64_t
+read_config(struct hypctx *hypctx, int n)
+{
+ struct vgic_v3_irq *irq;
+ uint64_t ret;
+ uint32_t irq_base;
+ int i;
+
+ ret = 0;
+ irq_base = n * 16;
+ for (i = 0; i < 16; i++) {
+ irq = vgic_v3_get_irq(hypctx->hyp, vcpu_vcpuid(hypctx->vcpu),
+ irq_base + i);
+ if (irq == NULL)
+ continue;
+
+ ret |= ((uint64_t)irq->config) << (i * 2);
+ vgic_v3_release_irq(irq);
+ }
+
+ return (ret);
+}
+
+static void
+write_config(struct hypctx *hypctx, int n, uint64_t val)
+{
+ struct vgic_v3_irq *irq;
+ uint32_t irq_base;
+ int i;
+
+ irq_base = n * 16;
+ for (i = 0; i < 16; i++) {
+ /*
+ * The config can't be changed for SGIs and PPIs. SGIs have
+ * an edge-triggered behaviour, and the register is
+ * implementation defined to be read-only for PPIs.
+ */
+ if (irq_base + i < VGIC_PRV_I_NUM)
+ continue;
+
+ irq = vgic_v3_get_irq(hypctx->hyp, vcpu_vcpuid(hypctx->vcpu),
+ irq_base + i);
+ if (irq == NULL)
+ continue;
+
+ /* Bit 0 is RES0 */
+ irq->config = (val >> (i * 2)) & VGIC_CONFIG_MASK;
+ vgic_v3_release_irq(irq);
+ }
+}
+
+static uint64_t
+read_route(struct hypctx *hypctx, int n)
+{
+ struct vgic_v3_irq *irq;
+ uint64_t mpidr;
+
+ irq = vgic_v3_get_irq(hypctx->hyp, vcpu_vcpuid(hypctx->vcpu), n);
+ if (irq == NULL)
+ return (0);
+
+ mpidr = irq->mpidr;
+ vgic_v3_release_irq(irq);
+
+ return (mpidr);
+}
+
+static void
+write_route(struct hypctx *hypctx, int n, uint64_t val, u_int offset,
+ u_int size)
+{
+ struct vgic_v3_irq *irq;
+
+ irq = vgic_v3_get_irq(hypctx->hyp, vcpu_vcpuid(hypctx->vcpu), n);
+ if (irq == NULL)
+ return;
+
+ irq->mpidr = gic_reg_value_64(irq->mpidr, val, offset, size) & GICD_AFF;
+ irq->target_vcpu = mpidr_to_vcpu(hypctx->hyp, irq->mpidr);
+ /*
+ * If the interrupt is pending we can either use the old mpidr, or
+ * the new mpidr. To simplify this code we use the old value so we
+ * don't need to move the interrupt until the next time it is
+ * moved to the pending state.
+ */
+ vgic_v3_release_irq(irq);
+}
+
+/*
+ * Distributor register handlers.
+ */
+/* GICD_CTLR */
+static void
+dist_ctlr_read(struct hypctx *hypctx, u_int reg, uint64_t *rval,
+ void *arg)
+{
+ struct hyp *hyp;
+ struct vgic_v3 *vgic;
+
+ hyp = hypctx->hyp;
+ vgic = hyp->vgic;
+
+ mtx_lock_spin(&vgic->dist_mtx);
+ *rval = vgic->gicd_ctlr;
+ mtx_unlock_spin(&vgic->dist_mtx);
+
+ /* Writes are never pending */
+ *rval &= ~GICD_CTLR_RWP;
+}
+
+static void
+dist_ctlr_write(struct hypctx *hypctx, u_int reg, u_int offset, u_int size,
+ uint64_t wval, void *arg)
+{
+ struct vgic_v3 *vgic;
+
+ MPASS(offset == 0);
+ MPASS(size == 4);
+ vgic = hypctx->hyp->vgic;
+
+ /*
+ * GICv2 backwards compatibility is not implemented so
+ * ARE_NS is RAO/WI. This means EnableGrp1 is RES0.
+ *
+ * EnableGrp1A is supported, and RWP is read-only.
+ *
+ * All other bits are RES0 from non-secure mode as we
+ * implement as if we are in a system with two security
+ * states.
+ */
+ wval &= GICD_CTLR_G1A;
+ wval |= GICD_CTLR_ARE_NS;
+ mtx_lock_spin(&vgic->dist_mtx);
+ vgic->gicd_ctlr = wval;
+ /* TODO: Wake any vcpus that have interrupts pending */
+ mtx_unlock_spin(&vgic->dist_mtx);
+}
+
+/* GICD_TYPER */
+static void
+dist_typer_read(struct hypctx *hypctx, u_int reg, uint64_t *rval,
+ void *arg)
+{
+ uint32_t typer;
+
+ typer = (10 - 1) << GICD_TYPER_IDBITS_SHIFT;
+ typer |= GICD_TYPER_MBIS;
+ /* ITLinesNumber: */
+ typer |= howmany(VGIC_NIRQS + 1, 32) - 1;
+
+ *rval = typer;
+}
+
+/* GICD_IIDR */
+static void
+dist_iidr_read(struct hypctx *hypctx, u_int reg, uint64_t *rval, void *arg)
+{
+ *rval = VGIC_IIDR;
+}
+
+/* GICD_SETSPI_NSR & GICD_CLRSPI_NSR */
+static void
+dist_setclrspi_nsr_write(struct hypctx *hypctx, u_int reg, u_int offset,
+ u_int size, uint64_t wval, void *arg)
+{
+ uint32_t irqid;
+
+ MPASS(offset == 0);
+ MPASS(size == 4);
+ irqid = wval & GICD_SPI_INTID_MASK;
+ INJECT_IRQ(hypctx->hyp, vcpu_vcpuid(hypctx->vcpu), irqid,
+ reg == GICD_SETSPI_NSR);
+}
+
+/* GICD_ISENABLER */
+static void
+dist_isenabler_read(struct hypctx *hypctx, u_int reg, uint64_t *rval, void *arg)
+{
+ int n;
+
+ n = (reg - GICD_ISENABLER(0)) / 4;
+ /* GICD_ISENABLER0 is RAZ/WI so handled separately */
+ MPASS(n > 0);
+ *rval = read_enabler(hypctx, n);
+}
+
+static void
+dist_isenabler_write(struct hypctx *hypctx, u_int reg, u_int offset, u_int size,
+ uint64_t wval, void *arg)
+{
+ int n;
+
+ MPASS(offset == 0);
+ MPASS(size == 4);
+ n = (reg - GICD_ISENABLER(0)) / 4;
+ /* GICD_ISENABLER0 is RAZ/WI so handled separately */
+ MPASS(n > 0);
+ write_enabler(hypctx, n, true, wval);
+}
+
+/* GICD_ICENABLER */
+static void
+dist_icenabler_read(struct hypctx *hypctx, u_int reg, uint64_t *rval, void *arg)
+{
+ int n;
+
+ n = (reg - GICD_ICENABLER(0)) / 4;
+ /* GICD_ICENABLER0 is RAZ/WI so handled separately */
+ MPASS(n > 0);
+ *rval = read_enabler(hypctx, n);
+}
+
+static void
+dist_icenabler_write(struct hypctx *hypctx, u_int reg, u_int offset, u_int size,
+ uint64_t wval, void *arg)
+{
+ int n;
+
+ MPASS(offset == 0);
+ MPASS(size == 4);
+ n = (reg - GICD_ISENABLER(0)) / 4;
+ /* GICD_ICENABLER0 is RAZ/WI so handled separately */
+ MPASS(n > 0);
+ write_enabler(hypctx, n, false, wval);
+}
+
+/* GICD_ISPENDR */
+static void
+dist_ispendr_read(struct hypctx *hypctx, u_int reg, uint64_t *rval, void *arg)
+{
+ int n;
+
+ n = (reg - GICD_ISPENDR(0)) / 4;
+ /* GICD_ISPENDR0 is RAZ/WI so handled separately */
+ MPASS(n > 0);
+ *rval = read_pendr(hypctx, n);
+}
+
+static void
+dist_ispendr_write(struct hypctx *hypctx, u_int reg, u_int offset, u_int size,
+ uint64_t wval, void *arg)
+{
+ int n;
+
+ MPASS(offset == 0);
+ MPASS(size == 4);
+ n = (reg - GICD_ISPENDR(0)) / 4;
+ /* GICD_ISPENDR0 is RAZ/WI so handled separately */
+ MPASS(n > 0);
+ write_pendr(hypctx, n, true, wval);
+}
+
+/* GICD_ICPENDR */
+static void
+dist_icpendr_read(struct hypctx *hypctx, u_int reg, uint64_t *rval, void *arg)
+{
+ int n;
+
+ n = (reg - GICD_ICPENDR(0)) / 4;
+ /* GICD_ICPENDR0 is RAZ/WI so handled separately */
+ MPASS(n > 0);
+ *rval = read_pendr(hypctx, n);
+}
+
+static void
+dist_icpendr_write(struct hypctx *hypctx, u_int reg, u_int offset, u_int size,
+ uint64_t wval, void *arg)
+{
+ int n;
+
+ MPASS(offset == 0);
+ MPASS(size == 4);
+ n = (reg - GICD_ICPENDR(0)) / 4;
+ /* GICD_ICPENDR0 is RAZ/WI so handled separately */
+ MPASS(n > 0);
+ write_pendr(hypctx, n, false, wval);
+}
+
+/* GICD_ISACTIVER */
+/* Affinity routing is enabled so isactiver0 is RAZ/WI */
+static void
+dist_isactiver_read(struct hypctx *hypctx, u_int reg, uint64_t *rval, void *arg)
+{
+ int n;
+
+ n = (reg - GICD_ISACTIVER(0)) / 4;
+ /* GICD_ISACTIVER0 is RAZ/WI so handled separately */
+ MPASS(n > 0);
+ *rval = read_activer(hypctx, n);
+}
+
+static void
+dist_isactiver_write(struct hypctx *hypctx, u_int reg, u_int offset, u_int size,
+ uint64_t wval, void *arg)
+{
+ int n;
+
+ MPASS(offset == 0);
+ MPASS(size == 4);
+ n = (reg - GICD_ISACTIVER(0)) / 4;
+ /* GICD_ISACTIVE0 is RAZ/WI so handled separately */
+ MPASS(n > 0);
+ write_activer(hypctx, n, true, wval);
+}
+
+/* GICD_ICACTIVER */
+static void
+dist_icactiver_read(struct hypctx *hypctx, u_int reg, uint64_t *rval,
+ void *arg)
+{
+ int n;
+
+ n = (reg - GICD_ICACTIVER(0)) / 4;
+ /* GICD_ICACTIVE0 is RAZ/WI so handled separately */
+ MPASS(n > 0);
+ *rval = read_activer(hypctx, n);
+}
+
+static void
+dist_icactiver_write(struct hypctx *hypctx, u_int reg, u_int offset, u_int size,
+ uint64_t wval, void *arg)
+{
+ int n;
+
+ MPASS(offset == 0);
+ MPASS(size == 4);
+ n = (reg - GICD_ICACTIVER(0)) / 4;
+ /* GICD_ICACTIVE0 is RAZ/WI so handled separately */
+ MPASS(n > 0);
+ write_activer(hypctx, n, false, wval);
+}
+
+/* GICD_IPRIORITYR */
+/* Affinity routing is enabled so ipriorityr0-7 is RAZ/WI */
+static void
+dist_ipriorityr_read(struct hypctx *hypctx, u_int reg, uint64_t *rval,
+ void *arg)
+{
+ int n;
+
+ n = (reg - GICD_IPRIORITYR(0)) / 4;
+ /* GICD_IPRIORITY0-7 is RAZ/WI so handled separately */
+ MPASS(n > 7);
+ *rval = read_priorityr(hypctx, n);
+}
+
+static void
+dist_ipriorityr_write(struct hypctx *hypctx, u_int reg, u_int offset,
+ u_int size, uint64_t wval, void *arg)
+{
+ u_int irq_base;
+
+ irq_base = (reg - GICD_IPRIORITYR(0)) + offset;
+ /* GICD_IPRIORITY0-7 is RAZ/WI so handled separately */
+ MPASS(irq_base > 31);
+ write_priorityr(hypctx, irq_base, size, wval);
+}
+
+/* GICD_ICFGR */
+static void
+dist_icfgr_read(struct hypctx *hypctx, u_int reg, uint64_t *rval, void *arg)
+{
+ int n;
+
+ n = (reg - GICD_ICFGR(0)) / 4;
+ /* GICD_ICFGR0-1 are RAZ/WI so handled separately */
+ MPASS(n > 1);
+ *rval = read_config(hypctx, n);
+}
+
+static void
+dist_icfgr_write(struct hypctx *hypctx, u_int reg, u_int offset, u_int size,
+ uint64_t wval, void *arg)
+{
+ int n;
+
+ MPASS(offset == 0);
+ MPASS(size == 4);
+ n = (reg - GICD_ICFGR(0)) / 4;
+ /* GICD_ICFGR0-1 are RAZ/WI so handled separately */
+ MPASS(n > 1);
+ write_config(hypctx, n, wval);
+}
+
+/* GICD_IROUTER */
+static void
+dist_irouter_read(struct hypctx *hypctx, u_int reg, uint64_t *rval, void *arg)
+{
+ int n;
+
+ n = (reg - GICD_IROUTER(0)) / 8;
+ /* GICD_IROUTER0-31 don't exist */
+ MPASS(n > 31);
+ *rval = read_route(hypctx, n);
+}
+
+static void
+dist_irouter_write(struct hypctx *hypctx, u_int reg, u_int offset, u_int size,
+ uint64_t wval, void *arg)
+{
+ int n;
+
+ n = (reg - GICD_IROUTER(0)) / 8;
+ /* GICD_IROUTER0-31 don't exist */
+ MPASS(n > 31);
+ write_route(hypctx, n, wval, offset, size);
+}
+
+static bool
+vgic_register_read(struct hypctx *hypctx, struct vgic_register *reg_list,
+ u_int reg_list_size, u_int reg, u_int size, uint64_t *rval, void *arg)
+{
+ u_int i, offset;
+
+ for (i = 0; i < reg_list_size; i++) {
+ if (reg_list[i].start <= reg && reg_list[i].end >= reg + size) {
+ offset = reg & reg_list[i].size - 1;
+ reg -= offset;
+ if ((reg_list[i].flags & size) != 0) {
+ reg_list[i].read(hypctx, reg, rval, NULL);
+
+ /* Move the bits into the correct place */
+ *rval >>= (offset * 8);
+ if (size < 8) {
+ *rval &= (1ul << (size * 8)) - 1;
+ }
+ } else {
+ /*
+ * The access is an invalid size. Section
+ * 12.1.3 "GIC memory-mapped register access"
+ * of the GICv3 and GICv4 spec issue H
+ * (IHI0069) lists the options. For a read
+ * the controller returns unknown data, in
+ * this case it is zero.
+ */
+ *rval = 0;
+ }
+ return (true);
+ }
+ }
+ return (false);
+}
+
+static bool
+vgic_register_write(struct hypctx *hypctx, struct vgic_register *reg_list,
+ u_int reg_list_size, u_int reg, u_int size, uint64_t wval, void *arg)
+{
+ u_int i, offset;
+
+ for (i = 0; i < reg_list_size; i++) {
+ if (reg_list[i].start <= reg && reg_list[i].end >= reg + size) {
+ offset = reg & reg_list[i].size - 1;
+ reg -= offset;
+ if ((reg_list[i].flags & size) != 0) {
+ reg_list[i].write(hypctx, reg, offset,
+ size, wval, NULL);
+ } else {
+ /*
+ * See the comment in vgic_register_read.
+ * For writes the controller ignores the
+ * operation.
+ */
+ }
+ return (true);
+ }
+ }
+ return (false);
+}
+
+static int
+dist_read(struct vcpu *vcpu, uint64_t fault_ipa, uint64_t *rval,
+ int size, void *arg)
+{
+ struct hyp *hyp;
+ struct hypctx *hypctx;
+ struct vgic_v3 *vgic;
+ uint64_t reg;
+
+ hypctx = vcpu_get_cookie(vcpu);
+ hyp = hypctx->hyp;
+ vgic = hyp->vgic;
+
+ /* Check the register is one of ours and is the correct size */
+ if (fault_ipa < vgic->dist_start || fault_ipa + size > vgic->dist_end) {
+ return (EINVAL);
+ }
+
+ reg = fault_ipa - vgic->dist_start;
+ /* Check the register is correctly aligned */
+ if ((reg & (size - 1)) != 0)
+ return (EINVAL);
+
+ if (vgic_register_read(hypctx, dist_registers, nitems(dist_registers),
+ reg, size, rval, NULL))
+ return (0);
+
+ /* Reserved register addresses are RES0 so we can hardware it to 0 */
+ *rval = 0;
+
+ return (0);
+}
+
+static int
+dist_write(struct vcpu *vcpu, uint64_t fault_ipa, uint64_t wval,
+ int size, void *arg)
+{
+ struct hyp *hyp;
+ struct hypctx *hypctx;
+ struct vgic_v3 *vgic;
+ uint64_t reg;
+
+ hypctx = vcpu_get_cookie(vcpu);
+ hyp = hypctx->hyp;
+ vgic = hyp->vgic;
+
+ /* Check the register is one of ours and is the correct size */
+ if (fault_ipa < vgic->dist_start || fault_ipa + size > vgic->dist_end) {
+ return (EINVAL);
+ }
+
+ reg = fault_ipa - vgic->dist_start;
+ /* Check the register is correctly aligned */
+ if ((reg & (size - 1)) != 0)
+ return (EINVAL);
+
+ if (vgic_register_write(hypctx, dist_registers, nitems(dist_registers),
+ reg, size, wval, NULL))
+ return (0);
+
+ /* Reserved register addresses are RES0 so we can ignore the write */
+ return (0);
+}
+
+/*
+ * Redistributor register handlers.
+ *
+ * RD_base:
+ */
+/* GICR_CTLR */
+static void
+redist_ctlr_read(struct hypctx *hypctx, u_int reg, uint64_t *rval, void *arg)
+{
+ /* LPIs not supported */
+ *rval = 0;
+}
+
+/* GICR_IIDR */
+static void
+redist_iidr_read(struct hypctx *hypctx, u_int reg, uint64_t *rval, void *arg)
+{
+ *rval = VGIC_IIDR;
+}
+
+/* GICR_TYPER */
+static void
+redist_typer_read(struct hypctx *hypctx, u_int reg, uint64_t *rval, void *arg)
+{
+ *rval = hypctx->vgic_cpu->gicr_typer;
+}
+
+/*
+ * SGI_base:
+ */
+/* GICR_ISENABLER0 */
+static void
+redist_ienabler0_read(struct hypctx *hypctx, u_int reg, uint64_t *rval,
+ void *arg)
+{
+ *rval = read_enabler(hypctx, 0);
+}
+
+static void
+redist_isenabler0_write(struct hypctx *hypctx, u_int reg, u_int offset,
+ u_int size, uint64_t wval, void *arg)
+{
+ MPASS(offset == 0);
+ MPASS(size == 4);
+ write_enabler(hypctx, 0, true, wval);
+}
+
+/* GICR_ICENABLER0 */
+static void
+redist_icenabler0_write(struct hypctx *hypctx, u_int reg, u_int offset,
+ u_int size, uint64_t wval, void *arg)
+{
+ MPASS(offset == 0);
+ MPASS(size == 4);
+ write_enabler(hypctx, 0, false, wval);
+}
+
+/* GICR_ISPENDR0 */
+static void
+redist_ipendr0_read(struct hypctx *hypctx, u_int reg, uint64_t *rval,
+ void *arg)
+{
+ *rval = read_pendr(hypctx, 0);
+}
+
+static void
+redist_ispendr0_write(struct hypctx *hypctx, u_int reg, u_int offset,
+ u_int size, uint64_t wval, void *arg)
+{
+ MPASS(offset == 0);
+ MPASS(size == 4);
+ write_pendr(hypctx, 0, true, wval);
+}
+
+/* GICR_ICPENDR0 */
+static void
+redist_icpendr0_write(struct hypctx *hypctx, u_int reg, u_int offset,
+ u_int size, uint64_t wval, void *arg)
+{
+ MPASS(offset == 0);
+ MPASS(size == 4);
+ write_pendr(hypctx, 0, false, wval);
+}
+
+/* GICR_ISACTIVER0 */
+static void
+redist_iactiver0_read(struct hypctx *hypctx, u_int reg, uint64_t *rval,
+ void *arg)
+{
+ *rval = read_activer(hypctx, 0);
+}
+
+static void
+redist_isactiver0_write(struct hypctx *hypctx, u_int reg, u_int offset,
+ u_int size, uint64_t wval, void *arg)
+{
+ write_activer(hypctx, 0, true, wval);
+}
+
+/* GICR_ICACTIVER0 */
+static void
+redist_icactiver0_write(struct hypctx *hypctx, u_int reg, u_int offset,
+ u_int size, uint64_t wval, void *arg)
+{
+ write_activer(hypctx, 0, false, wval);
+}
+
+/* GICR_IPRIORITYR */
+static void
+redist_ipriorityr_read(struct hypctx *hypctx, u_int reg, uint64_t *rval,
+ void *arg)
+{
+ int n;
+
+ n = (reg - GICR_IPRIORITYR(0)) / 4;
+ *rval = read_priorityr(hypctx, n);
+}
+
+static void
+redist_ipriorityr_write(struct hypctx *hypctx, u_int reg, u_int offset,
+ u_int size, uint64_t wval, void *arg)
+{
+ u_int irq_base;
+
+ irq_base = (reg - GICR_IPRIORITYR(0)) + offset;
+ write_priorityr(hypctx, irq_base, size, wval);
+}
+
+/* GICR_ICFGR1 */
+static void
+redist_icfgr1_read(struct hypctx *hypctx, u_int reg, uint64_t *rval, void *arg)
+{
+ *rval = read_config(hypctx, 0);
+}
+
+static void
+redist_icfgr1_write(struct hypctx *hypctx, u_int reg, u_int offset, u_int size,
+ uint64_t wval, void *arg)
+{
+ MPASS(offset == 0);
+ MPASS(size == 4);
+ write_config(hypctx, 0, wval);
+}
+
+static int
+redist_read(struct vcpu *vcpu, uint64_t fault_ipa, uint64_t *rval,
+ int size, void *arg)
+{
+ struct hyp *hyp;
+ struct hypctx *hypctx;
+ struct vgic_v3 *vgic;
+ uint64_t reg;
+
+ hypctx = vcpu_get_cookie(vcpu);
+ hyp = hypctx->hyp;
+ vgic = hyp->vgic;
+
+ /* Check the register is one of ours and is the correct size */
+ if (fault_ipa < vgic->redist_start ||
+ fault_ipa + size > vgic->redist_end) {
+ return (EINVAL);
+ }
+
+ reg = fault_ipa - vgic->redist_start;
+ /* Check the register is correctly aligned */
+ if ((reg & (size - 1)) != 0)
+ return (EINVAL);
+
+ if (reg < GICR_RD_BASE_SIZE) {
+ if (vgic_register_read(hypctx, redist_rd_registers,
+ nitems(redist_rd_registers), reg, size, rval, NULL))
+ return (0);
+ } else if (reg < (GICR_SGI_BASE + GICR_SGI_BASE_SIZE)) {
+ if (vgic_register_read(hypctx, redist_sgi_registers,
+ nitems(redist_sgi_registers), reg - GICR_SGI_BASE, size,
+ rval, NULL))
+ return (0);
+ }
+
+ /* Reserved register addresses are RES0 so we can hardware it to 0 */
+ *rval = 0;
+ return (0);
+}
+
+static int
+redist_write(struct vcpu *vcpu, uint64_t fault_ipa, uint64_t wval,
+ int size, void *arg)
+{
+ struct hyp *hyp;
+ struct hypctx *hypctx;
+ struct vgic_v3 *vgic;
+ uint64_t reg;
+
+ hypctx = vcpu_get_cookie(vcpu);
+ hyp = hypctx->hyp;
+ vgic = hyp->vgic;
+
+ /* Check the register is one of ours and is the correct size */
+ if (fault_ipa < vgic->redist_start ||
+ fault_ipa + size > vgic->redist_end) {
+ return (EINVAL);
+ }
+
+ reg = fault_ipa - vgic->redist_start;
+ /* Check the register is correctly aligned */
+ if ((reg & (size - 1)) != 0)
+ return (EINVAL);
+
+ if (reg < GICR_RD_BASE_SIZE) {
+ if (vgic_register_write(hypctx, redist_rd_registers,
+ nitems(redist_rd_registers), reg, size, wval, NULL))
+ return (0);
+ } else if (reg < (GICR_SGI_BASE + GICR_SGI_BASE_SIZE)) {
+ if (vgic_register_write(hypctx, redist_sgi_registers,
+ nitems(redist_sgi_registers), reg - GICR_SGI_BASE, size,
+ wval, NULL))
+ return (0);
+ }
+
+ /* Reserved register addresses are RES0 so we can ignore the write */
+ return (0);
+}
+
+static int
+vgic_v3_icc_sgi1r_read(struct vcpu *vcpu, uint64_t *rval, void *arg)
+{
+ /*
+ * TODO: Inject an unknown exception.
+ */
+ *rval = 0;
+ return (0);
+}
+
+static int
+vgic_v3_icc_sgi1r_write(struct vcpu *vcpu, uint64_t rval, void *arg)
+{
+ struct vm *vm;
+ struct hyp *hyp;
+ cpuset_t active_cpus;
+ uint64_t mpidr, aff1, aff2, aff3;
+ uint32_t irqid;
+ int cpus, cpu_off, target_vcpuid, vcpuid;
+
+ vm = vcpu_vm(vcpu);
+ hyp = vm_get_cookie(vm);
+ active_cpus = vm_active_cpus(vm);
+ vcpuid = vcpu_vcpuid(vcpu);
+
+ irqid = ICC_SGI1R_EL1_SGIID_VAL(rval) >> ICC_SGI1R_EL1_SGIID_SHIFT;
+ if ((rval & ICC_SGI1R_EL1_IRM) == 0) {
+ /* Non-zero points at no vcpus */
+ if (ICC_SGI1R_EL1_RS_VAL(rval) != 0)
+ return (0);
+
+ aff1 = ICC_SGI1R_EL1_AFF1_VAL(rval) >> ICC_SGI1R_EL1_AFF1_SHIFT;
+ aff2 = ICC_SGI1R_EL1_AFF2_VAL(rval) >> ICC_SGI1R_EL1_AFF2_SHIFT;
+ aff3 = ICC_SGI1R_EL1_AFF3_VAL(rval) >> ICC_SGI1R_EL1_AFF3_SHIFT;
+ mpidr = aff3 << MPIDR_AFF3_SHIFT |
+ aff2 << MPIDR_AFF2_SHIFT | aff1 << MPIDR_AFF1_SHIFT;
+
+ cpus = ICC_SGI1R_EL1_TL_VAL(rval) >> ICC_SGI1R_EL1_TL_SHIFT;
+ cpu_off = 0;
+ while (cpus > 0) {
+ if (cpus & 1) {
+ target_vcpuid = mpidr_to_vcpu(hyp,
+ mpidr | (cpu_off << MPIDR_AFF0_SHIFT));
+ if (target_vcpuid >= 0 &&
+ CPU_ISSET(target_vcpuid, &active_cpus)) {
+ INJECT_IRQ(hyp, target_vcpuid, irqid,
+ true);
+ }
+ }
+ cpu_off++;
+ cpus >>= 1;
+ }
+ } else {
+ /* Send an IPI to all CPUs other than the current CPU */
+ for (target_vcpuid = 0; target_vcpuid < vm_get_maxcpus(vm);
+ target_vcpuid++) {
+ if (CPU_ISSET(target_vcpuid, &active_cpus) &&
+ target_vcpuid != vcpuid) {
+ INJECT_IRQ(hyp, target_vcpuid, irqid, true);
+ }
+ }
+ }
+
+ return (0);
+}
+
+static void
+vgic_v3_mmio_init(struct hyp *hyp)
+{
+ struct vgic_v3 *vgic;
+ struct vgic_v3_irq *irq;
+ int i;
+
+ /* Allocate memory for the SPIs */
+ vgic = hyp->vgic;
+ vgic->irqs = malloc((VGIC_NIRQS - VGIC_PRV_I_NUM) *
+ sizeof(*vgic->irqs), M_VGIC_V3, M_WAITOK | M_ZERO);
+
+ for (i = 0; i < VGIC_NIRQS - VGIC_PRV_I_NUM; i++) {
+ irq = &vgic->irqs[i];
+
+ mtx_init(&irq->irq_spinmtx, "VGIC IRQ spinlock", NULL,
+ MTX_SPIN);
+
+ irq->irq = i + VGIC_PRV_I_NUM;
+ }
+}
+
+static void
+vgic_v3_mmio_destroy(struct hyp *hyp)
+{
+ struct vgic_v3 *vgic;
+ struct vgic_v3_irq *irq;
+ int i;
+
+ vgic = hyp->vgic;
+ for (i = 0; i < VGIC_NIRQS - VGIC_PRV_I_NUM; i++) {
+ irq = &vgic->irqs[i];
+
+ mtx_destroy(&irq->irq_spinmtx);
+ }
+
+ free(vgic->irqs, M_VGIC_V3);
+}
+
+static int
+vgic_v3_attach_to_vm(device_t dev, struct hyp *hyp, struct vm_vgic_descr *descr)
+{
+ struct vm *vm;
+ struct vgic_v3 *vgic;
+
+ if (descr->ver.version != 3)
+ return (EINVAL);
+
+ /* The register bases need to be 64k aligned */
+ if (!__is_aligned(descr->v3_regs.dist_start, PAGE_SIZE_64K) ||
+ !__is_aligned(descr->v3_regs.redist_start, PAGE_SIZE_64K))
+ return (EINVAL);
+
+ /* The dist register space is 1 64k block */
+ if (descr->v3_regs.dist_size != PAGE_SIZE_64K)
+ return (EINVAL);
+
+ /* The redist register space is 2 64k blocks */
+ if (descr->v3_regs.redist_size != PAGE_SIZE_64K * 2)
+ return (EINVAL);
+
+ vm = hyp->vm;
+ vgic = hyp->vgic;
+
+ /* Set the distributor address and size for trapping guest access. */
+ vgic->dist_start = descr->v3_regs.dist_start;
+ vgic->dist_end = descr->v3_regs.dist_start + descr->v3_regs.dist_size;
+
+ vgic->redist_start = descr->v3_regs.redist_start;
+ vgic->redist_end = descr->v3_regs.redist_start +
+ descr->v3_regs.redist_size;
+
+ vm_register_inst_handler(vm, descr->v3_regs.dist_start,
+ descr->v3_regs.dist_size, dist_read, dist_write);
+ vm_register_inst_handler(vm, descr->v3_regs.redist_start,
+ descr->v3_regs.redist_size, redist_read, redist_write);
+
+ vm_register_reg_handler(vm, ISS_MSR_REG(ICC_SGI1R_EL1),
+ ISS_MSR_REG_MASK, vgic_v3_icc_sgi1r_read, vgic_v3_icc_sgi1r_write,
+ NULL);
+
+ vgic_v3_mmio_init(hyp);
+
+ hyp->vgic_attached = true;
+
+ return (0);
+}
+
+static void
+vgic_v3_detach_from_vm(device_t dev, struct hyp *hyp)
+{
+ if (hyp->vgic_attached) {
+ hyp->vgic_attached = false;
+ vgic_v3_mmio_destroy(hyp);
+ }
+}
+
+static struct vgic_v3_irq *
+vgic_v3_get_irq(struct hyp *hyp, int vcpuid, uint32_t irqid)
+{
+ struct vgic_v3_cpu *vgic_cpu;
+ struct vgic_v3_irq *irq;
+ struct hypctx *hypctx;
+
+ if (irqid < VGIC_PRV_I_NUM) {
+ if (vcpuid < 0 || vcpuid >= vm_get_maxcpus(hyp->vm))
+ return (NULL);
+ hypctx = hyp->ctx[vcpuid];
+ if (hypctx == NULL)
+ return (NULL);
+ vgic_cpu = hypctx->vgic_cpu;
+ irq = &vgic_cpu->private_irqs[irqid];
+ } else if (irqid <= GIC_LAST_SPI) {
+ irqid -= VGIC_PRV_I_NUM;
+ if (irqid >= VGIC_NIRQS)
+ return (NULL);
+ irq = &hyp->vgic->irqs[irqid];
+ } else if (irqid < GIC_FIRST_LPI) {
+ return (NULL);
+ } else {
+ /* No support for LPIs */
+ return (NULL);
+ }
+
+ mtx_lock_spin(&irq->irq_spinmtx);
+ return (irq);
+}
+
+static void
+vgic_v3_release_irq(struct vgic_v3_irq *irq)
+{
+
+ mtx_unlock_spin(&irq->irq_spinmtx);
+}
+
+static bool
+vgic_v3_has_pending_irq(device_t dev, struct hypctx *hypctx)
+{
+ struct vgic_v3_cpu *vgic_cpu;
+ bool empty;
+
+ vgic_cpu = hypctx->vgic_cpu;
+ mtx_lock_spin(&vgic_cpu->lr_mtx);
+ empty = TAILQ_EMPTY(&vgic_cpu->irq_act_pend);
+ mtx_unlock_spin(&vgic_cpu->lr_mtx);
+
+ return (!empty);
+}
+
+static bool
+vgic_v3_check_irq(struct vgic_v3_irq *irq, bool level)
+{
+ /*
+ * Only inject if:
+ * - Level-triggered IRQ: level changes low -> high
+ * - Edge-triggered IRQ: level is high
+ */
+ switch (irq->config & VGIC_CONFIG_MASK) {
+ case VGIC_CONFIG_LEVEL:
+ return (level != irq->level);
+ case VGIC_CONFIG_EDGE:
+ return (level);
+ default:
+ break;
+ }
+
+ return (false);
+}
+
+static int
+vgic_v3_inject_irq(device_t dev, struct hyp *hyp, int vcpuid, uint32_t irqid,
+ bool level)
+{
+ struct vgic_v3_cpu *vgic_cpu;
+ struct vgic_v3_irq *irq;
+ struct hypctx *hypctx;
+ int target_vcpu;
+ bool notify;
+
+ KASSERT(vcpuid == -1 || irqid < VGIC_PRV_I_NUM,
+ ("%s: SPI/LPI with vcpuid set: irq %u vcpuid %u", __func__, irqid,
+ vcpuid));
+
+ irq = vgic_v3_get_irq(hyp, vcpuid, irqid);
+ if (irq == NULL) {
+ eprintf("Malformed IRQ %u.\n", irqid);
+ return (1);
+ }
+
+ target_vcpu = irq->target_vcpu;
+ KASSERT(vcpuid == -1 || vcpuid == target_vcpu,
+ ("%s: Interrupt %u has bad cpu affinity: vcpu %d target vcpu %d",
+ __func__, irqid, vcpuid, target_vcpu));
+ KASSERT(target_vcpu >= 0 && target_vcpu < vm_get_maxcpus(hyp->vm),
+ ("%s: Interrupt %u sent to invalid vcpu %d", __func__, irqid,
+ target_vcpu));
+
+ if (vcpuid == -1)
+ vcpuid = target_vcpu;
+ /* TODO: Check from 0 to vm->maxcpus */
+ if (vcpuid < 0 || vcpuid >= vm_get_maxcpus(hyp->vm)) {
+ vgic_v3_release_irq(irq);
+ return (1);
+ }
+
+ hypctx = hyp->ctx[vcpuid];
+ if (hypctx == NULL) {
+ vgic_v3_release_irq(irq);
+ return (1);
+ }
+
+ notify = false;
+ vgic_cpu = hypctx->vgic_cpu;
+
+ mtx_lock_spin(&vgic_cpu->lr_mtx);
+
+ if (!vgic_v3_check_irq(irq, level)) {
+ goto out;
+ }
+
+ if ((irq->config & VGIC_CONFIG_MASK) == VGIC_CONFIG_LEVEL)
+ irq->level = level;
+ else /* VGIC_CONFIG_EDGE */
+ irq->pending = true;
+
+ notify = vgic_v3_queue_irq(hyp, vgic_cpu, vcpuid, irq);
+
+out:
+ mtx_unlock_spin(&vgic_cpu->lr_mtx);
+ vgic_v3_release_irq(irq);
+
+ if (notify)
+ vcpu_notify_event(vm_vcpu(hyp->vm, vcpuid), false);
+
+ return (0);
+}
+
+static int
+vgic_v3_inject_msi(device_t dev, struct hyp *hyp, uint64_t msg, uint64_t addr)
+{
+ struct vgic_v3 *vgic;
+ uint64_t reg;
+
+ vgic = hyp->vgic;
+
+ /* This is a 4 byte register */
+ if (addr < vgic->dist_start || addr + 4 > vgic->dist_end) {
+ return (EINVAL);
+ }
+
+ reg = addr - vgic->dist_start;
+ if (reg != GICD_SETSPI_NSR)
+ return (EINVAL);
+
+ return (INJECT_IRQ(hyp, -1, msg, true));
+}
+
+static void
+vgic_v3_flush_hwstate(device_t dev, struct hypctx *hypctx)
+{
+ struct vgic_v3_cpu *vgic_cpu;
+ struct vgic_v3_irq *irq;
+ int i;
+
+ vgic_cpu = hypctx->vgic_cpu;
+
+ /*
+ * All Distributor writes have been executed at this point, do not
+ * protect Distributor reads with a mutex.
+ *
+ * This is callled with all interrupts disabled, so there is no need for
+ * a List Register spinlock either.
+ */
+ mtx_lock_spin(&vgic_cpu->lr_mtx);
+
+ hypctx->vgic_v3_regs.ich_hcr_el2 &= ~ICH_HCR_EL2_UIE;
+
+ /* Exit early if there are no buffered interrupts */
+ if (TAILQ_EMPTY(&vgic_cpu->irq_act_pend))
+ goto out;
+
+ KASSERT(vgic_cpu->ich_lr_used == 0, ("%s: Used LR count not zero %u",
+ __func__, vgic_cpu->ich_lr_used));
+
+ i = 0;
+ hypctx->vgic_v3_regs.ich_elrsr_el2 =
+ (1u << hypctx->vgic_v3_regs.ich_lr_num) - 1;
+ TAILQ_FOREACH(irq, &vgic_cpu->irq_act_pend, act_pend_list) {
+ /* No free list register, stop searching for IRQs */
+ if (i == hypctx->vgic_v3_regs.ich_lr_num)
+ break;
+
+ if (!irq->enabled)
+ continue;
+
+ hypctx->vgic_v3_regs.ich_lr_el2[i] = ICH_LR_EL2_GROUP1 |
+ ((uint64_t)irq->priority << ICH_LR_EL2_PRIO_SHIFT) |
+ irq->irq;
+
+ if (irq->active) {
+ hypctx->vgic_v3_regs.ich_lr_el2[i] |=
+ ICH_LR_EL2_STATE_ACTIVE;
+ }
+
+#ifdef notyet
+ /* TODO: Check why this is needed */
+ if ((irq->config & _MASK) == LEVEL)
+ hypctx->vgic_v3_regs.ich_lr_el2[i] |= ICH_LR_EL2_EOI;
+#endif
+
+ if (!irq->active && vgic_v3_irq_pending(irq)) {
+ hypctx->vgic_v3_regs.ich_lr_el2[i] |=
+ ICH_LR_EL2_STATE_PENDING;
+
+ /*
+ * This IRQ is now pending on the guest. Allow for
+ * another edge that could cause the interrupt to
+ * be raised again.
+ */
+ if ((irq->config & VGIC_CONFIG_MASK) ==
+ VGIC_CONFIG_EDGE) {
+ irq->pending = false;
+ }
+ }
+
+ i++;
+ }
+ vgic_cpu->ich_lr_used = i;
+
+out:
+ mtx_unlock_spin(&vgic_cpu->lr_mtx);
+}
+
+static void
+vgic_v3_sync_hwstate(device_t dev, struct hypctx *hypctx)
+{
+ struct vgic_v3_cpu *vgic_cpu;
+ struct vgic_v3_irq *irq;
+ uint64_t lr;
+ int i;
+
+ vgic_cpu = hypctx->vgic_cpu;
+
+ /* Exit early if there are no buffered interrupts */
+ if (vgic_cpu->ich_lr_used == 0)
+ return;
+
+ /*
+ * Check on the IRQ state after running the guest. ich_lr_used and
+ * ich_lr_el2 are only ever used within this thread so is safe to
+ * access unlocked.
+ */
+ for (i = 0; i < vgic_cpu->ich_lr_used; i++) {
+ lr = hypctx->vgic_v3_regs.ich_lr_el2[i];
+ hypctx->vgic_v3_regs.ich_lr_el2[i] = 0;
+
+ irq = vgic_v3_get_irq(hypctx->hyp, vcpu_vcpuid(hypctx->vcpu),
+ ICH_LR_EL2_VINTID(lr));
+ if (irq == NULL)
+ continue;
+
+ irq->active = (lr & ICH_LR_EL2_STATE_ACTIVE) != 0;
+
+ if ((irq->config & VGIC_CONFIG_MASK) == VGIC_CONFIG_EDGE) {
+ /*
+ * If we have an edge triggered IRQ preserve the
+ * pending bit until the IRQ has been handled.
+ */
+ if ((lr & ICH_LR_EL2_STATE_PENDING) != 0) {
+ irq->pending = true;
+ }
+ } else {
+ /*
+ * If we have a level triggerend IRQ remove the
+ * pending bit if the IRQ has been handled.
+ * The level is separate, so may still be high
+ * triggering another IRQ.
+ */
+ if ((lr & ICH_LR_EL2_STATE_PENDING) == 0) {
+ irq->pending = false;
+ }
+ }
+
+ /* Lock to update irq_act_pend */
+ mtx_lock_spin(&vgic_cpu->lr_mtx);
+ if (irq->active) {
+ /* Ensure the active IRQ is at the head of the list */
+ TAILQ_REMOVE(&vgic_cpu->irq_act_pend, irq,
+ act_pend_list);
+ TAILQ_INSERT_HEAD(&vgic_cpu->irq_act_pend, irq,
+ act_pend_list);
+ } else if (!vgic_v3_irq_pending(irq)) {
+ /* If pending or active remove from the list */
+ TAILQ_REMOVE(&vgic_cpu->irq_act_pend, irq,
+ act_pend_list);
+ irq->on_aplist = false;
+ }
+ mtx_unlock_spin(&vgic_cpu->lr_mtx);
+ vgic_v3_release_irq(irq);
+ }
+
+ hypctx->vgic_v3_regs.ich_hcr_el2 &= ~ICH_HCR_EL2_EOICOUNT_MASK;
+ vgic_cpu->ich_lr_used = 0;
+}
+
+static void
+vgic_v3_init(device_t dev)
+{
+ uint64_t ich_vtr_el2;
+ uint32_t pribits, prebits;
+
+ ich_vtr_el2 = vmm_call_hyp(HYP_READ_REGISTER, HYP_REG_ICH_VTR);
+
+ /* TODO: These fields are common with the vgicv2 driver */
+ pribits = ICH_VTR_EL2_PRIBITS(ich_vtr_el2);
+ switch (pribits) {
+ case 5:
+ virt_features.min_prio = 0xf8;
+ case 6:
+ virt_features.min_prio = 0xfc;
+ case 7:
+ virt_features.min_prio = 0xfe;
+ case 8:
+ virt_features.min_prio = 0xff;
+ }
+
+ prebits = ICH_VTR_EL2_PREBITS(ich_vtr_el2);
+ switch (prebits) {
+ case 5:
+ virt_features.ich_apr_num = 1;
+ case 6:
+ virt_features.ich_apr_num = 2;
+ case 7:
+ virt_features.ich_apr_num = 4;
+ }
+
+ virt_features.ich_lr_num = ICH_VTR_EL2_LISTREGS(ich_vtr_el2);
+}
+
+static int
+vgic_v3_probe(device_t dev)
+{
+ if (!gic_get_vgic(dev))
+ return (EINVAL);
+
+ /* We currently only support the GICv3 */
+ if (gic_get_hw_rev(dev) < 3)
+ return (EINVAL);
+
+ device_set_desc(dev, "Virtual GIC v3");
+ return (BUS_PROBE_DEFAULT);
+}
+
+static int
+vgic_v3_attach(device_t dev)
+{
+ vgic_dev = dev;
+ return (0);
+}
+
+static device_method_t vgic_v3_methods[] = {
+ /* Device interface */
+ DEVMETHOD(device_probe, vgic_v3_probe),
+ DEVMETHOD(device_attach, vgic_v3_attach),
+
+ /* VGIC interface */
+ DEVMETHOD(vgic_init, vgic_v3_init),
+ DEVMETHOD(vgic_attach_to_vm, vgic_v3_attach_to_vm),
+ DEVMETHOD(vgic_detach_from_vm, vgic_v3_detach_from_vm),
+ DEVMETHOD(vgic_vminit, vgic_v3_vminit),
+ DEVMETHOD(vgic_cpuinit, vgic_v3_cpuinit),
+ DEVMETHOD(vgic_cpucleanup, vgic_v3_cpucleanup),
+ DEVMETHOD(vgic_vmcleanup, vgic_v3_vmcleanup),
+
+ DEVMETHOD(vgic_has_pending_irq, vgic_v3_has_pending_irq),
+ DEVMETHOD(vgic_inject_irq, vgic_v3_inject_irq),
+ DEVMETHOD(vgic_inject_msi, vgic_v3_inject_msi),
+ DEVMETHOD(vgic_flush_hwstate, vgic_v3_flush_hwstate),
+ DEVMETHOD(vgic_sync_hwstate, vgic_v3_sync_hwstate),
+
+ /* End */
+ DEVMETHOD_END
+};
+
+/* TODO: Create a vgic base class? */
+DEFINE_CLASS_0(vgic, vgic_v3_driver, vgic_v3_methods, 0);
+
+DRIVER_MODULE(vgic_v3, gic, vgic_v3_driver, 0, 0);
diff --git a/sys/arm64/vmm/io/vgic_v3_reg.h b/sys/arm64/vmm/io/vgic_v3_reg.h
new file mode 100644
--- /dev/null
+++ b/sys/arm64/vmm/io/vgic_v3_reg.h
@@ -0,0 +1,99 @@
+#ifndef _VGIC_V3_REG_H_
+#define _VGIC_V3_REG_H_
+
+/* Interrupt Controller End of Interrupt Status Register */
+#define ICH_EISR_EL2_STATUS_MASK 0xffff
+#define ICH_EISR_EL2_EOI_NOT_HANDLED(lr) ((1 << lr) & ICH_EISR_EL2_STATUS_MASK)
+
+/* Interrupt Controller Empty List Register Status Register */
+#define ICH_ELSR_EL2_STATUS_MASK 0xffff
+#define ICH_ELSR_EL2_LR_EMPTY(x) ((1 << x) & ICH_ELSR_EL2_STATUS_MASK)
+
+/* Interrupt Controller Hyp Control Register */
+#define ICH_HCR_EL2_EOICOUNT_SHIFT 27
+#define ICH_HCR_EL2_EOICOUNT_MASK (0x1f << ICH_HCR_EL2_EOICOUNT_SHIFT)
+#define ICH_HCR_EL2_TDIR (1 << 14) /* Trap non-secure EL1 writes to IC{C, V}_DIR_EL1 */
+#define ICH_HCR_EL2_TSEI (1 << 14) /* Trap System Error Interupts (SEI) to EL2 */
+#define ICH_HCR_EL2_TALL1 (1 << 12) /* Trap non-secure EL1 accesses to IC{C, V}_* for Group 1 interrupts */
+#define ICH_HCR_EL2_TALL0 (1 << 11) /* Trap non-secure EL1 accesses to IC{C, V}_* for Group 0 interrupts */
+#define ICH_HCR_EL2_TC (1 << 10) /* Trap non-secure EL1 accesses to common IC{C, V}_* registers */
+#define ICH_HCR_EL2_VGRP1DIE (1 << 7) /* VM Group 1 Disabled Interrupt Enable */
+#define ICH_HCR_EL2_VGRP1EIE (1 << 6) /* VM Group 1 Enabled Interrupt Enable */
+#define ICH_HCR_EL2_VGRP0DIE (1 << 5) /* VM Group 0 Disabled Interrupt Enable */
+#define ICH_HCR_EL2_VGRP0EIE (1 << 4) /* VM Group 0 Enabled Interrupt Enable */
+#define ICH_HCR_EL2_NPIE (1 << 3) /* No Pending Interrupt Enable */
+#define ICH_HCR_EL2_LRENPIE (1 << 2) /* List Register Entry Not Present Interrupt Enable */
+#define ICH_HCR_EL2_UIE (1 << 1) /* Underflow Interrupt Enable */
+#define ICH_HCR_EL2_En (1 << 0) /* Global enable for the virtual CPU interface */
+
+/* Interrupt Controller List Registers */
+#define ICH_LR_EL2_VINTID_MASK 0xffffffff
+#define ICH_LR_EL2_VINTID(x) ((x) & ICH_LR_EL2_VINTID_MASK)
+#define ICH_LR_EL2_PINTID_SHIFT 32
+#define ICH_LR_EL2_PINTID_MASK (0x3fUL << ICH_LR_EL2_PINTID_SHIFT)
+/* Raise a maintanance IRQ when deactivated (only non-HW virqs) */
+#define ICH_LR_EL2_EOI (1UL << 41)
+#define ICH_LR_EL2_PRIO_SHIFT 48
+#define ICH_LR_EL2_PRIO_MASK (0xffUL << ICH_LR_EL2_PRIO_SHIFT)
+#define ICH_LR_EL2_GROUP_SHIFT 60
+#define ICH_LR_EL2_GROUP1 (1UL << ICH_LR_EL2_GROUP_SHIFT)
+#define ICH_LR_EL2_HW (1UL << 61)
+#define ICH_LR_EL2_STATE_SHIFT 62
+#define ICH_LR_EL2_STATE_MASK (0x3UL << ICH_LR_EL2_STATE_SHIFT)
+#define ICH_LR_EL2_STATE(x) ((x) & ICH_LR_EL2_STATE_MASK)
+#define ICH_LR_EL2_STATE_INACTIVE (0x0UL << ICH_LR_EL2_STATE_SHIFT)
+#define ICH_LR_EL2_STATE_PENDING (0x1UL << ICH_LR_EL2_STATE_SHIFT)
+#define ICH_LR_EL2_STATE_ACTIVE (0x2UL << ICH_LR_EL2_STATE_SHIFT)
+#define ICH_LR_EL2_STATE_PENDING_ACTIVE (0x3UL << ICH_LR_EL2_STATE_SHIFT)
+
+/* Interrupt Controller Maintenance Interrupt State Register */
+#define ICH_MISR_EL2_VGRP1D (1 << 7) /* vPE Group 1 Disabled */
+#define ICH_MISR_EL2_VGRP1E (1 << 6) /* vPE Group 1 Enabled */
+#define ICH_MISR_EL2_VGRP0D (1 << 5) /* vPE Group 0 Disabled */
+#define ICH_MISR_EL2_VGRP0E (1 << 4) /* vPE Group 0 Enabled */
+#define ICH_MISR_EL2_NP (1 << 3) /* No Pending */
+#define ICH_MISR_EL2_LRENP (1 << 2) /* List Register Entry Not Present */
+#define ICH_MISR_EL2_U (1 << 1) /* Underflow */
+#define ICH_MISR_EL2_EOI (1 << 0) /* End Of Interrupt */
+
+/* Interrupt Controller Virtual Machine Control Register */
+#define ICH_VMCR_EL2_VPMR_SHIFT 24
+#define ICH_VMCR_EL2_VPMR_MASK (0xff << ICH_VMCR_EL2_VPMR_SHIFT)
+#define ICH_VMCR_EL2_VPMR_PRIO_LOWEST (0xff << ICH_VMCR_EL2_VPMR_SHIFT)
+#define ICH_VMCR_EL2_VPMR_PRIO_HIGHEST (0x00 << ICH_VMCR_EL2_VPMR_SHIFT)
+#define ICH_VMCR_EL2_VBPR0_SHIFT 21
+#define ICH_VMCR_EL2_VBPR0_MASK (0x7 << ICH_VMCR_EL2_VBPR0_SHIFT)
+#define ICH_VMCR_EL2_VBPR0_NO_PREEMPTION \
+ (0x7 << ICH_VMCR_EL2_VBPR0_SHIFT)
+#define ICH_VMCR_EL2_VBPR1_SHIFT 18
+#define ICH_VMCR_EL2_VBPR1_MASK (0x7 << ICH_VMCR_EL2_VBPR1_SHIFT)
+#define ICH_VMCR_EL2_VBPR1_NO_PREEMPTION \
+ (0x7 << ICH_VMCR_EL2_VBPR1_SHIFT)
+#define ICH_VMCR_EL2_VEOIM (1 << 9) /* Virtual EOI mode */
+#define ICH_VMCR_EL2_VCBPR (1 << 4) /* Virtual Common binary Point Register */
+#define ICH_VMCR_EL2_VFIQEN (1 << 3) /* Virtual FIQ enable */
+#define ICH_VMCR_EL2_VACKCTL (1 << 2) /* Virtual AckCtl */
+#define ICH_VMCR_EL2_VENG1 (1 << 1) /* Virtual Group 1 Interrupt Enable */
+#define ICH_VMCR_EL2_VENG0 (1 << 0) /* Virtual Group 0 Interrupt Enable */
+
+/* Interrupt Controller VGIC Type Register */
+#define ICH_VTR_EL2_PRIBITS_SHIFT 29
+#define ICH_VTR_EL2_PRIBITS_MASK (0x7 << ICH_VTR_EL2_PRIBITS_SHIFT)
+#define ICH_VTR_EL2_PRIBITS(x) \
+ ((((x) & ICH_VTR_EL2_PRIBITS_MASK) >> ICH_VTR_EL2_PRIBITS_SHIFT) + 1)
+#define ICH_VTR_EL2_PREBITS_SHIFT 26
+#define ICH_VTR_EL2_PREBITS_MASK (0x7 << ICH_VTR_EL2_PREBITS_SHIFT)
+#define ICH_VTR_EL2_PREBITS(x) \
+ (((x) & ICH_VTR_EL2_PREBITS_MASK) >> ICH_VTR_EL2_PREBITS_SHIFT)
+#define ICH_VTR_EL2_SEIS (1 << 22) /* System Error Interrupt (SEI) Support */
+#define ICH_VTR_EL2_A3V (1 << 21) /* Affinity 3 Valid */
+#define ICH_VTR_EL2_NV4 (1 << 20) /* Direct injection of virtual interrupts. RES1 for GICv3 */
+#define ICH_VTR_EL2_TDS (1 << 19) /* Implementation supports ICH_HCR_EL2.TDIR */
+#define ICH_VTR_EL2_LISTREGS_MASK 0x1f
+/*
+ * ICH_VTR_EL2.ListRegs holds the number of list registers, minus one. Add one
+ * to get the actual number of list registers.
+ */
+#define ICH_VTR_EL2_LISTREGS(x) (((x) & ICH_VTR_EL2_LISTREGS_MASK) + 1)
+
+#endif /* !_VGIC_V3_REG_H_ */
diff --git a/sys/arm64/vmm/io/vtimer.h b/sys/arm64/vmm/io/vtimer.h
new file mode 100644
--- /dev/null
+++ b/sys/arm64/vmm/io/vtimer.h
@@ -0,0 +1,86 @@
+/*-
+ * Copyright (c) 2017 The FreeBSD Foundation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * 3. The name of the company nor the name of the author may be used to
+ * endorse or promote products derived from this software without specific
+ * prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef _VMM_VTIMER_H_
+#define _VMM_VTIMER_H_
+
+#define GT_PHYS_NS_IRQ 30
+#define GT_VIRT_IRQ 27
+
+struct hyp;
+struct hypctx;
+
+struct vtimer
+{
+ uint64_t cnthctl_el2;
+ uint64_t cntvoff_el2;
+};
+
+struct vtimer_timer
+{
+ struct callout callout;
+ struct mtx mtx;
+
+ uint32_t irqid;
+
+ /*
+ * These registers are either emulated for the physical timer, or
+ * the guest has full access to them for the virtual timer.
+
+ * CNTx_CTL_EL0: Counter-timer Timer Control Register
+ * CNTx_CVAL_EL0: Counter-timer Timer CompareValue Register
+ */
+ uint32_t cntx_cval_el0;
+ uint32_t cntx_ctl_el0;
+};
+
+struct vtimer_cpu
+{
+ struct vtimer_timer phys_timer;
+ struct vtimer_timer virt_timer;
+
+ uint32_t cntkctl_el1;
+};
+
+int vtimer_init(uint64_t cnthctl_el2);
+void vtimer_vminit(struct hyp *);
+void vtimer_cpuinit(struct hypctx *);
+void vtimer_cpucleanup(struct hypctx *);
+void vtimer_vmcleanup(struct hyp *);
+void vtimer_cleanup(void);
+void vtimer_sync_hwstate(struct hypctx *hypctx);
+
+int vtimer_phys_ctl_read(struct vcpu *vcpu, uint64_t *rval, void *arg);
+int vtimer_phys_ctl_write(struct vcpu *vcpu, uint64_t wval, void *arg);
+int vtimer_phys_cnt_read(struct vcpu *vcpu, uint64_t *rval, void *arg);
+int vtimer_phys_cnt_write(struct vcpu *vcpu, uint64_t wval, void *arg);
+int vtimer_phys_cval_read(struct vcpu *vcpu, uint64_t *rval, void *arg);
+int vtimer_phys_cval_write(struct vcpu *vcpu, uint64_t wval, void *arg);
+int vtimer_phys_tval_read(struct vcpu *vcpu, uint64_t *rval, void *arg);
+int vtimer_phys_tval_write(struct vcpu *vcpu, uint64_t wval, void *arg);
+#endif
diff --git a/sys/arm64/vmm/io/vtimer.c b/sys/arm64/vmm/io/vtimer.c
new file mode 100644
--- /dev/null
+++ b/sys/arm64/vmm/io/vtimer.c
@@ -0,0 +1,510 @@
+/*-
+ * Copyright (c) 2017 The FreeBSD Foundation
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * 3. The name of the company nor the name of the author may be used to
+ * endorse or promote products derived from this software without specific
+ * prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+#include <sys/types.h>
+#include <sys/systm.h>
+#include <sys/bus.h>
+#include <sys/kernel.h>
+#include <sys/module.h>
+#include <sys/mutex.h>
+#include <sys/rman.h>
+#include <sys/time.h>
+#include <sys/timeet.h>
+#include <sys/timetc.h>
+
+#include <machine/bus.h>
+#include <machine/machdep.h>
+#include <machine/vmm.h>
+#include <machine/armreg.h>
+
+#include <arm64/vmm/arm64.h>
+
+#include "vgic.h"
+#include "vtimer.h"
+
+#define RES1 0xffffffffffffffffUL
+
+#define timer_enabled(ctl) \
+ (!((ctl) & CNTP_CTL_IMASK) && ((ctl) & CNTP_CTL_ENABLE))
+
+static uint64_t cnthctl_el2_reg;
+static uint32_t tmr_frq;
+static bool have_vtimer = false;
+
+#define timer_condition_met(ctl) ((ctl) & CNTP_CTL_ISTATUS)
+
+static void vtimer_schedule_irq(struct hypctx *hypctx, bool phys);
+
+static int
+vtimer_virtual_timer_intr(void *arg)
+{
+ struct hypctx *hypctx;
+ uint64_t cntpct_el0;
+ uint32_t cntv_ctl;
+
+ /*
+ * TODO everything here is very strange. The relantionship between the
+ * hardware value and the value in memory is not clear at all.
+ */
+
+ hypctx = arm64_get_active_vcpu();
+ cntv_ctl = READ_SPECIALREG(cntv_ctl_el0);
+
+ if (!hypctx) {
+ /* vm_destroy() was called. */
+ eprintf("No active vcpu\n");
+ cntv_ctl = READ_SPECIALREG(cntv_ctl_el0);
+ goto out;
+ }
+ if (!timer_enabled(cntv_ctl)) {
+ eprintf("Timer not enabled\n");
+ goto out;
+ }
+ if (!timer_condition_met(cntv_ctl)) {
+ eprintf("Timer condition not met\n");
+ goto out;
+ }
+
+ cntpct_el0 = READ_SPECIALREG(cntpct_el0) -
+ hypctx->hyp->vtimer.cntvoff_el2;
+ if (hypctx->vtimer_cpu.virt_timer.cntx_cval_el0 < cntpct_el0)
+ vgic_inject_irq(hypctx->hyp, vcpu_vcpuid(hypctx->vcpu),
+ GT_VIRT_IRQ, true);
+
+ //hypctx->vtimer_cpu.virt_timer.cntx_ctl_el0 &= ~CNTP_CTL_ENABLE;
+ cntv_ctl = hypctx->vtimer_cpu.virt_timer.cntx_ctl_el0;
+
+out:
+ /*
+ * Disable the timer interrupt. This will prevent the interrupt from
+ * being reasserted as soon as we exit the handler and getting stuck
+ * in an infinite loop.
+ *
+ * This is safe to do because the guest disabled the timer, and then
+ * enables it as part of the interrupt handling routine.
+ */
+ cntv_ctl &= ~CNTP_CTL_ENABLE;
+ WRITE_SPECIALREG(cntv_ctl_el0, cntv_ctl);
+
+ return (FILTER_HANDLED);
+}
+
+int
+vtimer_init(uint64_t cnthctl_el2)
+{
+ cnthctl_el2_reg = cnthctl_el2;
+ /*
+ * The guest *MUST* use the same timer frequency as the host. The
+ * register CNTFRQ_EL0 is accessible to the guest and a different value
+ * in the guest dts file might have unforseen consequences.
+ */
+ tmr_frq = READ_SPECIALREG(cntfrq_el0);
+
+ return (0);
+}
+
+void
+vtimer_vminit(struct hyp *hyp)
+{
+ uint64_t now;
+
+ /*
+ * Configure the Counter-timer Hypervisor Control Register for the VM.
+ *
+ * CNTHCTL_EL1PCEN: trap access to CNTP_{CTL, CVAL, TVAL}_EL0 from EL1
+ * CNTHCTL_EL1PCTEN: trap access to CNTPCT_EL0
+ */
+ hyp->vtimer.cnthctl_el2 = cnthctl_el2_reg & ~CNTHCTL_EL1PCEN;
+ hyp->vtimer.cnthctl_el2 &= ~CNTHCTL_EL1PCTEN;
+
+ now = READ_SPECIALREG(cntpct_el0);
+ hyp->vtimer.cntvoff_el2 = now;
+
+ return;
+}
+
+void
+vtimer_cpuinit(struct hypctx *hypctx)
+{
+ struct vtimer_cpu *vtimer_cpu;
+
+ vtimer_cpu = &hypctx->vtimer_cpu;
+ /*
+ * Configure physical timer interrupts for the VCPU.
+ *
+ * CNTP_CTL_IMASK: mask interrupts
+ * ~CNTP_CTL_ENABLE: disable the timer
+ */
+ vtimer_cpu->phys_timer.cntx_ctl_el0 = CNTP_CTL_IMASK & ~CNTP_CTL_ENABLE;
+
+ mtx_init(&vtimer_cpu->phys_timer.mtx, "vtimer phys callout mutex", NULL,
+ MTX_DEF);
+ callout_init_mtx(&vtimer_cpu->phys_timer.callout,
+ &vtimer_cpu->phys_timer.mtx, 0);
+ vtimer_cpu->phys_timer.irqid = GT_PHYS_NS_IRQ;
+
+ mtx_init(&vtimer_cpu->virt_timer.mtx, "vtimer virt callout mutex", NULL,
+ MTX_DEF);
+ callout_init_mtx(&vtimer_cpu->virt_timer.callout,
+ &vtimer_cpu->virt_timer.mtx, 0);
+ vtimer_cpu->virt_timer.irqid = GT_VIRT_IRQ;
+}
+
+void
+vtimer_cpucleanup(struct hypctx *hypctx)
+{
+ struct vtimer_cpu *vtimer_cpu;
+
+ vtimer_cpu = &hypctx->vtimer_cpu;
+ callout_drain(&vtimer_cpu->phys_timer.callout);
+ callout_drain(&vtimer_cpu->virt_timer.callout);
+ mtx_destroy(&vtimer_cpu->phys_timer.mtx);
+ mtx_destroy(&vtimer_cpu->virt_timer.mtx);
+}
+
+void
+vtimer_vmcleanup(struct hyp *hyp)
+{
+ struct hypctx *hypctx;
+ uint32_t cntv_ctl;
+
+ hypctx = arm64_get_active_vcpu();
+ if (!hypctx) {
+ /* The active VM was destroyed, stop the timer. */
+ cntv_ctl = READ_SPECIALREG(cntv_ctl_el0);
+ cntv_ctl &= ~CNTP_CTL_ENABLE;
+ WRITE_SPECIALREG(cntv_ctl_el0, cntv_ctl);
+ }
+}
+
+void
+vtimer_cleanup(void)
+{
+}
+
+void
+vtimer_sync_hwstate(struct hypctx *hypctx)
+{
+ struct vtimer_timer *timer;
+ uint64_t cntpct_el0;
+
+ timer = &hypctx->vtimer_cpu.virt_timer;
+ cntpct_el0 = READ_SPECIALREG(cntpct_el0) -
+ hypctx->hyp->vtimer.cntvoff_el2;
+ if (!timer_enabled(timer->cntx_ctl_el0)) {
+ vgic_inject_irq(hypctx->hyp, vcpu_vcpuid(hypctx->vcpu),
+ timer->irqid, false);
+ } else if (timer->cntx_cval_el0 < cntpct_el0) {
+ vgic_inject_irq(hypctx->hyp, vcpu_vcpuid(hypctx->vcpu),
+ timer->irqid, true);
+ } else {
+ vgic_inject_irq(hypctx->hyp, vcpu_vcpuid(hypctx->vcpu),
+ timer->irqid, false);
+ vtimer_schedule_irq(hypctx, false);
+ }
+}
+
+static void
+vtimer_inject_irq_callout_phys(void *context)
+{
+ struct hypctx *hypctx;
+
+ hypctx = context;
+ vgic_inject_irq(hypctx->hyp, vcpu_vcpuid(hypctx->vcpu),
+ hypctx->vtimer_cpu.phys_timer.irqid, true);
+}
+
+static void
+vtimer_inject_irq_callout_virt(void *context)
+{
+ struct hypctx *hypctx;
+
+ hypctx = context;
+ vgic_inject_irq(hypctx->hyp, vcpu_vcpuid(hypctx->vcpu),
+ hypctx->vtimer_cpu.virt_timer.irqid, true);
+}
+
+static void
+vtimer_schedule_irq(struct hypctx *hypctx, bool phys)
+{
+ sbintime_t time;
+ struct vtimer_timer *timer;
+ uint64_t cntpct_el0;
+ uint64_t diff;
+
+ if (phys)
+ timer = &hypctx->vtimer_cpu.phys_timer;
+ else
+ timer = &hypctx->vtimer_cpu.virt_timer;
+ cntpct_el0 = READ_SPECIALREG(cntpct_el0) -
+ hypctx->hyp->vtimer.cntvoff_el2;
+ if (timer->cntx_cval_el0 < cntpct_el0) {
+ /* Timer set in the past, trigger interrupt */
+ vgic_inject_irq(hypctx->hyp, vcpu_vcpuid(hypctx->vcpu),
+ timer->irqid, true);
+ } else {
+ diff = timer->cntx_cval_el0 - cntpct_el0;
+ time = diff * SBT_1S / tmr_frq;
+ if (phys)
+ callout_reset_sbt(&timer->callout, time, 0,
+ vtimer_inject_irq_callout_phys, hypctx, 0);
+ else
+ callout_reset_sbt(&timer->callout, time, 0,
+ vtimer_inject_irq_callout_virt, hypctx, 0);
+ }
+}
+
+static void
+vtimer_remove_irq(struct hypctx *hypctx, struct vcpu *vcpu)
+{
+ struct vtimer_cpu *vtimer_cpu;
+ struct vtimer_timer *timer;
+
+ vtimer_cpu = &hypctx->vtimer_cpu;
+ timer = &vtimer_cpu->phys_timer;
+
+ callout_drain(&timer->callout);
+ /*
+ * The interrupt needs to be deactivated here regardless of the callout
+ * function having been executed. The timer interrupt can be masked with
+ * the CNTP_CTL_EL0.IMASK bit instead of reading the IAR register.
+ * Masking the interrupt doesn't remove it from the list registers.
+ */
+ vgic_inject_irq(hypctx->hyp, vcpu_vcpuid(vcpu), timer->irqid, false);
+}
+
+/*
+ * Timer emulation functions.
+ *
+ * The guest should use the virtual timer, however some software, e.g. u-boot,
+ * used the physical timer. Emulate this in software for the guest to use.
+ *
+ * Adjust for cntvoff_el2 so the physical and virtual timers are at similar
+ * times. This simplifies interrupt handling in the virtual timer as the
+ * adjustment will have already happened.
+ */
+
+int
+vtimer_phys_ctl_read(struct vcpu *vcpu, uint64_t *rval, void *arg)
+{
+ struct hyp *hyp;
+ struct hypctx *hypctx;
+ struct vtimer_cpu *vtimer_cpu;
+ uint64_t cntpct_el0;
+
+ hypctx = vcpu_get_cookie(vcpu);
+ hyp = hypctx->hyp;
+ vtimer_cpu = &hypctx->vtimer_cpu;
+
+ cntpct_el0 = READ_SPECIALREG(cntpct_el0) - hyp->vtimer.cntvoff_el2;
+ if (vtimer_cpu->phys_timer.cntx_cval_el0 < cntpct_el0)
+ /* Timer condition met */
+ *rval = vtimer_cpu->phys_timer.cntx_ctl_el0 | CNTP_CTL_ISTATUS;
+ else
+ *rval = vtimer_cpu->phys_timer.cntx_ctl_el0 & ~CNTP_CTL_ISTATUS;
+
+ return (0);
+}
+
+int
+vtimer_phys_ctl_write(struct vcpu *vcpu, uint64_t wval, void *arg)
+{
+ struct hypctx *hypctx;
+ struct vtimer_cpu *vtimer_cpu;
+ uint64_t ctl_el0;
+ bool timer_toggled_on;
+
+ hypctx = vcpu_get_cookie(vcpu);
+ vtimer_cpu = &hypctx->vtimer_cpu;
+
+ timer_toggled_on = false;
+ ctl_el0 = vtimer_cpu->phys_timer.cntx_ctl_el0;
+
+ if (!timer_enabled(ctl_el0) && timer_enabled(wval))
+ timer_toggled_on = true;
+ else if (timer_enabled(ctl_el0) && !timer_enabled(wval))
+ vtimer_remove_irq(hypctx, vcpu);
+
+ vtimer_cpu->phys_timer.cntx_ctl_el0 = wval;
+
+ if (timer_toggled_on)
+ vtimer_schedule_irq(hypctx, true);
+
+ return (0);
+}
+
+int
+vtimer_phys_cnt_read(struct vcpu *vcpu, uint64_t *rval, void *arg)
+{
+ struct vm *vm;
+ struct hyp *hyp;
+
+ vm = vcpu_vm(vcpu);
+ hyp = vm_get_cookie(vm);
+ *rval = READ_SPECIALREG(cntpct_el0) - hyp->vtimer.cntvoff_el2;
+ return (0);
+}
+
+int
+vtimer_phys_cnt_write(struct vcpu *vcpu, uint64_t wval, void *arg)
+{
+ return (0);
+}
+
+int
+vtimer_phys_cval_read(struct vcpu *vcpu, uint64_t *rval, void *arg)
+{
+ struct hypctx *hypctx;
+ struct vtimer_cpu *vtimer_cpu;
+
+ hypctx = vcpu_get_cookie(vcpu);
+ vtimer_cpu = &hypctx->vtimer_cpu;
+
+ *rval = vtimer_cpu->phys_timer.cntx_cval_el0;
+
+ return (0);
+}
+
+int
+vtimer_phys_cval_write(struct vcpu *vcpu, uint64_t wval, void *arg)
+{
+ struct hypctx *hypctx;
+ struct vtimer_cpu *vtimer_cpu;
+
+ hypctx = vcpu_get_cookie(vcpu);
+ vtimer_cpu = &hypctx->vtimer_cpu;
+
+ vtimer_cpu->phys_timer.cntx_cval_el0 = wval;
+
+ vtimer_remove_irq(hypctx, vcpu);
+ if (timer_enabled(vtimer_cpu->phys_timer.cntx_ctl_el0)) {
+ vtimer_schedule_irq(hypctx, true);
+ }
+
+ return (0);
+}
+
+int
+vtimer_phys_tval_read(struct vcpu *vcpu, uint64_t *rval, void *arg)
+{
+ struct hyp *hyp;
+ struct hypctx *hypctx;
+ struct vtimer_cpu *vtimer_cpu;
+ uint32_t cntpct_el0;
+
+ hypctx = vcpu_get_cookie(vcpu);
+ hyp = hypctx->hyp;
+ vtimer_cpu = &hypctx->vtimer_cpu;
+
+ if (!(vtimer_cpu->phys_timer.cntx_ctl_el0 & CNTP_CTL_ENABLE)) {
+ /*
+ * ARMv8 Architecture Manual, p. D7-2702: the result of reading
+ * TVAL when the timer is disabled is UNKNOWN. I have chosen to
+ * return the maximum value possible on 32 bits which means the
+ * timer will fire very far into the future.
+ */
+ *rval = (uint32_t)RES1;
+ } else {
+ cntpct_el0 = READ_SPECIALREG(cntpct_el0) -
+ hyp->vtimer.cntvoff_el2;
+ *rval = vtimer_cpu->phys_timer.cntx_cval_el0 - cntpct_el0;
+ }
+
+ return (0);
+}
+
+int
+vtimer_phys_tval_write(struct vcpu *vcpu, uint64_t wval, void *arg)
+{
+ struct hyp *hyp;
+ struct hypctx *hypctx;
+ struct vtimer_cpu *vtimer_cpu;
+ uint64_t cntpct_el0;
+
+ hypctx = vcpu_get_cookie(vcpu);
+ hyp = hypctx->hyp;
+ vtimer_cpu = &hypctx->vtimer_cpu;
+
+ cntpct_el0 = READ_SPECIALREG(cntpct_el0) - hyp->vtimer.cntvoff_el2;
+ vtimer_cpu->phys_timer.cntx_cval_el0 = (int32_t)wval + cntpct_el0;
+
+ vtimer_remove_irq(hypctx, vcpu);
+ if (timer_enabled(vtimer_cpu->phys_timer.cntx_ctl_el0)) {
+ vtimer_schedule_irq(hypctx, true);
+ }
+
+ return (0);
+}
+
+struct vtimer_softc {
+ struct resource *res;
+ void *ihl;
+ int rid;
+};
+
+static int
+vtimer_probe(device_t dev)
+{
+ device_set_desc(dev, "Virtual timer");
+ return (BUS_PROBE_DEFAULT);
+}
+
+static int
+vtimer_attach(device_t dev)
+{
+ struct vtimer_softc *sc;
+
+ sc = device_get_softc(dev);
+
+ sc->rid = 0;
+ sc->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &sc->rid, RF_ACTIVE);
+ if (sc->res == NULL)
+ return (ENXIO);
+
+ bus_setup_intr(dev, sc->res, INTR_TYPE_CLK, vtimer_virtual_timer_intr,
+ NULL, NULL, &sc->ihl);
+
+ have_vtimer = true;
+ return (0);
+}
+
+static device_method_t vtimer_methods[] = {
+ /* Device interface */
+ DEVMETHOD(device_probe, vtimer_probe),
+ DEVMETHOD(device_attach, vtimer_attach),
+
+ /* End */
+ DEVMETHOD_END
+};
+
+DEFINE_CLASS_0(vtimer, vtimer_driver, vtimer_methods,
+ sizeof(struct vtimer_softc));
+
+DRIVER_MODULE(vtimer, generic_timer, vtimer_driver, 0, 0);
diff --git a/sys/arm64/vmm/mmu.h b/sys/arm64/vmm/mmu.h
new file mode 100644
--- /dev/null
+++ b/sys/arm64/vmm/mmu.h
@@ -0,0 +1,51 @@
+/*
+ * Copyright (C) 2017 Alexandru Elisei <alexandru.elisei@gmail.com>
+ * All rights reserved.
+ *
+ * This software was developed by Alexandru Elisei under sponsorship
+ * from the FreeBSD Foundation.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef _VMM_MMU_H_
+#define _VMM_MMU_H_
+
+#include <machine/machdep.h>
+#include <machine/vmparam.h>
+#include <machine/vmm.h>
+
+#include "hyp.h"
+
+extern char vmm_hyp_code;
+extern char vmm_hyp_code_end;
+
+extern char _vmm_start;
+extern char _vmm_end;
+
+bool vmmpmap_init(void);
+void vmmpmap_fini(void);
+uint64_t vmmpmap_to_ttbr0(void);
+bool vmmpmap_enter(vm_offset_t, vm_size_t, vm_paddr_t, vm_prot_t);
+void vmmpmap_remove(vm_offset_t, vm_size_t, bool);
+
+#endif
diff --git a/sys/arm64/vmm/reset.h b/sys/arm64/vmm/reset.h
new file mode 100644
--- /dev/null
+++ b/sys/arm64/vmm/reset.h
@@ -0,0 +1,32 @@
+/*
+ * Copyright (C) 2018 Alexandru Elisei <alexandru.elisei@gmail.com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+#ifndef _VMM_RESET_H_
+#define _VMM_RESET_H_
+
+void reset_vm_el01_regs(void *vcpu);
+void reset_vm_el2_regs(void *vcpu);
+
+#endif
diff --git a/sys/arm64/vmm/vmm.c b/sys/arm64/vmm/vmm.c
new file mode 100644
--- /dev/null
+++ b/sys/arm64/vmm/vmm.c
@@ -0,0 +1,1795 @@
+/*
+ * Copyright (C) 2015 Mihai Carabas <mihai.carabas@gmail.com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/kernel.h>
+#include <sys/module.h>
+#include <sys/sysctl.h>
+#include <sys/malloc.h>
+#include <sys/pcpu.h>
+#include <sys/lock.h>
+#include <sys/mutex.h>
+#include <sys/proc.h>
+#include <sys/rwlock.h>
+#include <sys/sched.h>
+#include <sys/smp.h>
+#include <sys/cpuset.h>
+
+#include <vm/vm.h>
+#include <vm/vm_object.h>
+#include <vm/vm_page.h>
+#include <vm/pmap.h>
+#include <vm/vm_map.h>
+#include <vm/vm_extern.h>
+#include <vm/vm_param.h>
+
+#include <machine/armreg.h>
+#include <machine/cpu.h>
+#include <machine/fpu.h>
+#include <machine/machdep.h>
+#include <machine/pcb.h>
+#include <machine/smp.h>
+#include <machine/vm.h>
+#include <machine/vmparam.h>
+#include <machine/vmm.h>
+#include <machine/vmm_dev.h>
+#include <machine/vmm_instruction_emul.h>
+
+#include <dev/pci/pcireg.h>
+
+#include "vmm_ktr.h"
+#include "vmm_stat.h"
+#include "vmm_mem.h"
+#include "arm64.h"
+#include "mmu.h"
+
+#include "io/vgic.h"
+#include "io/vtimer.h"
+
+struct vcpu {
+ int flags;
+ enum vcpu_state state;
+ struct mtx mtx;
+ int hostcpu; /* host cpuid this vcpu last ran on */
+ int vcpuid;
+ void *stats;
+ struct vm_exit exitinfo;
+ uint64_t nextpc; /* (x) next instruction to execute */
+ struct vm *vm; /* (o) */
+ void *cookie; /* (i) cpu-specific data */
+ struct vfpstate *guestfpu; /* (a,i) guest fpu state */
+};
+
+#define vcpu_lock_initialized(v) mtx_initialized(&((v)->mtx))
+#define vcpu_lock_init(v) mtx_init(&((v)->mtx), "vcpu lock", 0, MTX_SPIN)
+#define vcpu_lock_destroy(v) mtx_destroy(&((v)->mtx))
+#define vcpu_lock(v) mtx_lock_spin(&((v)->mtx))
+#define vcpu_unlock(v) mtx_unlock_spin(&((v)->mtx))
+#define vcpu_assert_locked(v) mtx_assert(&((v)->mtx), MA_OWNED)
+
+struct mem_seg {
+ uint64_t gpa;
+ size_t len;
+ bool wired;
+ bool sysmem;
+ vm_object_t object;
+};
+#define VM_MAX_MEMSEGS 3
+
+struct mem_map {
+ vm_paddr_t gpa;
+ size_t len;
+ vm_ooffset_t segoff;
+ int segid;
+ int prot;
+ int flags;
+};
+#define VM_MAX_MEMMAPS 4
+
+struct vmm_mmio_region {
+ uint64_t start;
+ uint64_t end;
+ mem_region_read_t read;
+ mem_region_write_t write;
+};
+#define VM_MAX_MMIO_REGIONS 4
+
+struct vmm_special_reg {
+ uint32_t esr_iss;
+ uint32_t esr_mask;
+ reg_read_t reg_read;
+ reg_write_t reg_write;
+ void *arg;
+};
+#define VM_MAX_SPECIAL_REGS 16
+
+/*
+ * Initialization:
+ * (o) initialized the first time the VM is created
+ * (i) initialized when VM is created and when it is reinitialized
+ * (x) initialized before use
+ */
+struct vm {
+ void *cookie; /* (i) cpu-specific data */
+ volatile cpuset_t active_cpus; /* (i) active vcpus */
+ volatile cpuset_t debug_cpus; /* (i) vcpus stopped for debug */
+ int suspend; /* (i) stop VM execution */
+ volatile cpuset_t suspended_cpus; /* (i) suspended vcpus */
+ volatile cpuset_t halted_cpus; /* (x) cpus in a hard halt */
+ struct mem_map mem_maps[VM_MAX_MEMMAPS]; /* (i) guest address space */
+ struct mem_seg mem_segs[VM_MAX_MEMSEGS]; /* (o) guest memory regions */
+ struct vmspace *vmspace; /* (o) guest's address space */
+ char name[VM_MAX_NAMELEN]; /* (o) virtual machine name */
+ struct vcpu **vcpu; /* (i) guest vcpus */
+ struct vmm_mmio_region mmio_region[VM_MAX_MMIO_REGIONS];
+ /* (o) guest MMIO regions */
+ struct vmm_special_reg special_reg[VM_MAX_SPECIAL_REGS];
+ /* The following describe the vm cpu topology */
+ uint16_t sockets; /* (o) num of sockets */
+ uint16_t cores; /* (o) num of cores/socket */
+ uint16_t threads; /* (o) num of threads/core */
+ uint16_t maxcpus; /* (o) max pluggable cpus */
+ struct sx mem_segs_lock; /* (o) */
+ struct sx vcpus_init_lock; /* (o) */
+};
+
+static bool vmm_initialized = false;
+
+static int vm_handle_wfi(struct vcpu *vcpu,
+ struct vm_exit *vme, bool *retu);
+
+static MALLOC_DEFINE(M_VMM, "vmm", "vmm");
+
+/* statistics */
+static VMM_STAT(VCPU_TOTAL_RUNTIME, "vcpu total runtime");
+
+SYSCTL_NODE(_hw, OID_AUTO, vmm, CTLFLAG_RW, NULL, NULL);
+
+static int vmm_ipinum;
+SYSCTL_INT(_hw_vmm, OID_AUTO, ipinum, CTLFLAG_RD, &vmm_ipinum, 0,
+ "IPI vector used for vcpu notifications");
+
+static struct cpu_desc vmm_desc = {
+ .id_aa64afr0 = 0,
+ .id_aa64afr1 = 0,
+ .id_aa64dfr0 =
+ (0xful << ID_AA64DFR0_CTX_CMPs_SHIFT) |
+ (0xful << ID_AA64DFR0_WRPs_SHIFT) |
+ (0xful << ID_AA64DFR0_BRPs_SHIFT) |
+ ID_AA64DFR0_PMUVer_3 |
+ ID_AA64DFR0_DebugVer_8,
+ .id_aa64dfr1 = 0,
+ .id_aa64isar0 =
+ ID_AA64ISAR0_TLB_TLBIOSR |
+ ID_AA64ISAR0_SHA3_IMPL |
+ ID_AA64ISAR0_RDM_IMPL |
+ ID_AA64ISAR0_Atomic_IMPL |
+ ID_AA64ISAR0_CRC32_BASE |
+ ID_AA64ISAR0_SHA2_512 |
+ ID_AA64ISAR0_SHA1_BASE |
+ ID_AA64ISAR0_AES_PMULL,
+ .id_aa64isar1 = 0,
+ .id_aa64mmfr0 =
+ ID_AA64MMFR0_TGran4_IMPL |
+ ID_AA64MMFR0_TGran64_IMPL |
+ ID_AA64MMFR0_TGran16_IMPL |
+ ID_AA64MMFR0_ASIDBits_16 |
+ ID_AA64MMFR0_PARange_4P,
+ .id_aa64mmfr1 =
+ ID_AA64MMFR1_SpecSEI_IMPL |
+ ID_AA64MMFR1_PAN_ATS1E1 |
+ ID_AA64MMFR1_HAFDBS_AF,
+ .id_aa64mmfr2 = 0,
+ .id_aa64pfr0 =
+ ID_AA64PFR0_GIC_CPUIF_NONE |
+ ID_AA64PFR0_AdvSIMD_HP |
+ ID_AA64PFR0_FP_HP |
+ ID_AA64PFR0_EL3_64 |
+ ID_AA64PFR0_EL2_64 |
+ ID_AA64PFR0_EL1_64 |
+ ID_AA64PFR0_EL0_64,
+ .id_aa64pfr1 = 0,
+};
+
+u_int vm_maxcpu;
+SYSCTL_UINT(_hw_vmm, OID_AUTO, maxcpu, CTLFLAG_RDTUN | CTLFLAG_NOFETCH,
+ &vm_maxcpu, 0, "Maximum number of vCPUs");
+
+static void vm_free_memmap(struct vm *vm, int ident);
+static bool sysmem_mapping(struct vm *vm, struct mem_map *mm);
+static void vcpu_notify_event_locked(struct vcpu *vcpu, bool lapic_intr);
+
+/*
+ * Upper limit on vm_maxcpu. We could increase this to 28 bits, but this
+ * is a safe value for now.
+ */
+#define VM_MAXCPU MIN(0xffff - 1, CPU_SETSIZE)
+
+static void
+vcpu_cleanup(struct vcpu *vcpu, bool destroy)
+{
+ vmmops_vcpu_cleanup(vcpu->cookie);
+ vcpu->cookie = NULL;
+ if (destroy) {
+ vmm_stat_free(vcpu->stats);
+ //fpu_save_area_free(vcpu->guestfpu);
+ vcpu_lock_destroy(vcpu);
+ }
+}
+
+static struct vcpu *
+vcpu_alloc(struct vm *vm, int vcpu_id)
+{
+ struct vcpu *vcpu;
+
+ KASSERT(vcpu_id >= 0 && vcpu_id < vm->maxcpus,
+ ("vcpu_alloc: invalid vcpu %d", vcpu_id));
+
+ vcpu = malloc(sizeof(*vcpu), M_VMM, M_WAITOK | M_ZERO);
+ vcpu_lock_init(vcpu);
+ vcpu->state = VCPU_IDLE;
+ vcpu->hostcpu = NOCPU;
+ vcpu->vcpuid = vcpu_id;
+ vcpu->vm = vm;
+ vcpu->guestfpu = fpu_save_area_alloc();
+ vcpu->stats = vmm_stat_alloc();
+ return (vcpu);
+}
+
+static void
+vcpu_init(struct vcpu *vcpu)
+{
+ vcpu->cookie = vmmops_vcpu_init(vcpu->vm->cookie, vcpu, vcpu->vcpuid);
+ MPASS(vcpu->cookie != NULL);
+ fpu_save_area_reset(vcpu->guestfpu);
+ vmm_stat_init(vcpu->stats);
+}
+
+struct vm_exit *
+vm_exitinfo(struct vcpu *vcpu)
+{
+ return (&vcpu->exitinfo);
+}
+
+static int
+vmm_init(void)
+{
+ update_cpu_desc(&vmm_desc);
+
+ vm_maxcpu = mp_ncpus;
+ TUNABLE_INT_FETCH("hw.vmm.maxcpu", &vm_maxcpu);
+
+ if (vm_maxcpu > VM_MAXCPU) {
+ printf("vmm: vm_maxcpu clamped to %u\n", VM_MAXCPU);
+ vm_maxcpu = VM_MAXCPU;
+ }
+ if (vm_maxcpu == 0)
+ vm_maxcpu = 1;
+
+#if 0
+ error = vmm_mem_init();
+ if (error)
+ return (error);
+#endif
+
+ return (vmmops_modinit(0));
+}
+
+static int
+vmm_handler(module_t mod, int what, void *arg)
+{
+ int error;
+
+ switch (what) {
+ case MOD_LOAD:
+ /* TODO: if (vmm_is_hw_supported()) { */
+ vmmdev_init();
+ error = vmm_init();
+ if (error == 0)
+ vmm_initialized = true;
+ break;
+ case MOD_UNLOAD:
+ /* TODO: if (vmm_is_hw_supported()) { */
+ error = vmmdev_cleanup();
+ if (error == 0 && vmm_initialized) {
+ error = vmmops_modcleanup();
+ if (error)
+ vmm_initialized = false;
+ }
+ break;
+ default:
+ error = 0;
+ break;
+ }
+ return (error);
+}
+
+static moduledata_t vmm_kmod = {
+ "vmm",
+ vmm_handler,
+ NULL
+};
+
+/*
+ * vmm initialization has the following dependencies:
+ *
+ * - HYP initialization requires smp_rendezvous() and therefore must happen
+ * after SMP is fully functional (after SI_SUB_SMP).
+ */
+DECLARE_MODULE(vmm, vmm_kmod, SI_SUB_SMP + 1, SI_ORDER_ANY);
+MODULE_VERSION(vmm, 1);
+
+static void
+vm_init(struct vm *vm, bool create)
+{
+ int i;
+
+ vm->cookie = vmmops_init(vm, vmspace_pmap(vm->vmspace));
+ MPASS(vm->cookie != NULL);
+
+ CPU_ZERO(&vm->active_cpus);
+ CPU_ZERO(&vm->debug_cpus);
+ //CPU_ZERO(&vm->startup_cpus);
+
+ vm->suspend = 0;
+ CPU_ZERO(&vm->suspended_cpus);
+
+ memset(vm->mmio_region, 0, sizeof(vm->mmio_region));
+ memset(vm->special_reg, 0, sizeof(vm->special_reg));
+
+ if (!create) {
+ for (i = 0; i < vm->maxcpus; i++) {
+ if (vm->vcpu[i] != NULL)
+ vcpu_init(vm->vcpu[i]);
+ }
+ }
+}
+
+struct vcpu *
+vm_alloc_vcpu(struct vm *vm, int vcpuid)
+{
+ struct vcpu *vcpu;
+
+ if (vcpuid < 0 || vcpuid >= vm_get_maxcpus(vm))
+ return (NULL);
+
+ vcpu = atomic_load_ptr(&vm->vcpu[vcpuid]);
+ if (__predict_true(vcpu != NULL))
+ return (vcpu);
+
+ sx_xlock(&vm->vcpus_init_lock);
+ vcpu = vm->vcpu[vcpuid];
+ if (vcpu == NULL/* && !vm->dying*/) {
+ vcpu = vcpu_alloc(vm, vcpuid);
+ vcpu_init(vcpu);
+
+ /*
+ * Ensure vCPU is fully created before updating pointer
+ * to permit unlocked reads above.
+ */
+ atomic_store_rel_ptr((uintptr_t *)&vm->vcpu[vcpuid],
+ (uintptr_t)vcpu);
+ }
+ sx_xunlock(&vm->vcpus_init_lock);
+ return (vcpu);
+}
+
+void
+vm_slock_vcpus(struct vm *vm)
+{
+ sx_slock(&vm->vcpus_init_lock);
+}
+
+void
+vm_unlock_vcpus(struct vm *vm)
+{
+ sx_unlock(&vm->vcpus_init_lock);
+}
+
+int
+vm_create(const char *name, struct vm **retvm)
+{
+ struct vm *vm;
+ struct vmspace *vmspace;
+
+ /*
+ * If vmm.ko could not be successfully initialized then don't attempt
+ * to create the virtual machine.
+ */
+ if (!vmm_initialized)
+ return (ENXIO);
+
+ if (name == NULL || strlen(name) >= VM_MAX_NAMELEN)
+ return (EINVAL);
+
+ vmspace = vmmops_vmspace_alloc(0, 1ul << 39);
+ if (vmspace == NULL)
+ return (ENOMEM);
+
+ vm = malloc(sizeof(struct vm), M_VMM, M_WAITOK | M_ZERO);
+ strcpy(vm->name, name);
+ vm->vmspace = vmspace;
+ sx_init(&vm->mem_segs_lock, "vm mem_segs");
+ sx_init(&vm->vcpus_init_lock, "vm vcpus");
+
+ vm->sockets = 1;
+ vm->cores = 1; /* XXX backwards compatibility */
+ vm->threads = 1; /* XXX backwards compatibility */
+ vm->maxcpus = VM_MAXCPU; /* XXX temp to keep code working */
+
+ vm->vcpu = malloc(sizeof(*vm->vcpu) * vm->maxcpus, M_VMM,
+ M_WAITOK | M_ZERO);
+
+ vm_init(vm, true);
+
+ *retvm = vm;
+ return (0);
+}
+
+void
+vm_get_topology(struct vm *vm, uint16_t *sockets, uint16_t *cores,
+ uint16_t *threads, uint16_t *maxcpus)
+{
+ *sockets = vm->sockets;
+ *cores = vm->cores;
+ *threads = vm->threads;
+ *maxcpus = vm->maxcpus;
+}
+
+uint16_t
+vm_get_maxcpus(struct vm *vm)
+{
+ return (vm->maxcpus);
+}
+
+int
+vm_set_topology(struct vm *vm, uint16_t sockets, uint16_t cores,
+ uint16_t threads, uint16_t maxcpus)
+{
+ if (maxcpus != 0)
+ return (EINVAL); /* XXX remove when supported */
+ if ((sockets * cores * threads) > vm->maxcpus)
+ return (EINVAL);
+ /* XXX need to check sockets * cores * threads == vCPU, how? */
+ vm->sockets = sockets;
+ vm->cores = cores;
+ vm->threads = threads;
+ vm->maxcpus = VM_MAXCPU; /* XXX temp to keep code working */
+ return(0);
+}
+
+static void
+vm_cleanup(struct vm *vm, bool destroy)
+{
+ struct mem_map *mm;
+ pmap_t pmap;
+ int i;
+
+ if (destroy) {
+ pmap = vmspace_pmap(vm->vmspace);
+ sched_pin();
+ PCPU_SET(curvmpmap, NULL);
+ sched_unpin();
+ CPU_FOREACH(i) {
+ MPASS(cpuid_to_pcpu[i]->pc_curvmpmap != pmap);
+ }
+ }
+
+ vgic_detach_from_vm(vm->cookie);
+
+ for (i = 0; i < vm->maxcpus; i++) {
+ if (vm->vcpu[i] != NULL)
+ vcpu_cleanup(vm->vcpu[i], destroy);
+ }
+
+ vmmops_cleanup(vm->cookie);
+
+ /*
+ * System memory is removed from the guest address space only when
+ * the VM is destroyed. This is because the mapping remains the same
+ * across VM reset.
+ *
+ * Device memory can be relocated by the guest (e.g. using PCI BARs)
+ * so those mappings are removed on a VM reset.
+ */
+ if (!destroy) {
+ for (i = 0; i < VM_MAX_MEMMAPS; i++) {
+ mm = &vm->mem_maps[i];
+ if (destroy || !sysmem_mapping(vm, mm))
+ vm_free_memmap(vm, i);
+ }
+ }
+
+ if (destroy) {
+ for (i = 0; i < VM_MAX_MEMSEGS; i++)
+ vm_free_memseg(vm, i);
+
+ vmmops_vmspace_free(vm->vmspace);
+ vm->vmspace = NULL;
+
+ for (i = 0; i < vm->maxcpus; i++)
+ free(vm->vcpu[i], M_VMM);
+ free(vm->vcpu, M_VMM);
+ sx_destroy(&vm->vcpus_init_lock);
+ sx_destroy(&vm->mem_segs_lock);
+ }
+}
+
+void
+vm_destroy(struct vm *vm)
+{
+ vm_cleanup(vm, true);
+ free(vm, M_VMM);
+}
+
+int
+vm_reinit(struct vm *vm)
+{
+ int error;
+
+ /*
+ * A virtual machine can be reset only if all vcpus are suspended.
+ */
+ if (CPU_CMP(&vm->suspended_cpus, &vm->active_cpus) == 0) {
+ vm_cleanup(vm, false);
+ vm_init(vm, false);
+ error = 0;
+ } else {
+ error = EBUSY;
+ }
+
+ return (error);
+}
+
+const char *
+vm_name(struct vm *vm)
+{
+ return (vm->name);
+}
+
+void
+vm_slock_memsegs(struct vm *vm)
+{
+ sx_slock(&vm->mem_segs_lock);
+}
+
+void
+vm_xlock_memsegs(struct vm *vm)
+{
+ sx_xlock(&vm->mem_segs_lock);
+}
+
+void
+vm_unlock_memsegs(struct vm *vm)
+{
+ sx_unlock(&vm->mem_segs_lock);
+}
+
+int
+vm_map_mmio(struct vm *vm, vm_paddr_t gpa, size_t len, vm_paddr_t hpa)
+{
+ vm_object_t obj;
+
+ if ((obj = vmm_mmio_alloc(vm->vmspace, gpa, len, hpa)) == NULL)
+ return (ENOMEM);
+ else
+ return (0);
+}
+
+int
+vm_unmap_mmio(struct vm *vm, vm_paddr_t gpa, size_t len)
+{
+
+ vmm_mmio_free(vm->vmspace, gpa, len);
+ return (0);
+}
+
+/*
+ * Return 'true' if 'gpa' is allocated in the guest address space.
+ *
+ * This function is called in the context of a running vcpu which acts as
+ * an implicit lock on 'vm->mem_maps[]'.
+ */
+bool
+vm_mem_allocated(struct vcpu *vcpu, vm_paddr_t gpa)
+{
+ struct vm *vm = vcpu->vm;
+ struct mem_map *mm;
+ int i;
+
+#ifdef INVARIANTS
+ int hostcpu, state;
+ state = vcpu_get_state(vcpu, &hostcpu);
+ KASSERT(state == VCPU_RUNNING && hostcpu == curcpu,
+ ("%s: invalid vcpu state %d/%d", __func__, state, hostcpu));
+#endif
+
+ for (i = 0; i < VM_MAX_MEMMAPS; i++) {
+ mm = &vm->mem_maps[i];
+ if (mm->len != 0 && gpa >= mm->gpa && gpa < mm->gpa + mm->len)
+ return (true); /* 'gpa' is sysmem or devmem */
+ }
+
+ return (false);
+}
+
+int
+vm_alloc_memseg(struct vm *vm, int ident, size_t len, bool sysmem)
+{
+ struct mem_seg *seg;
+ vm_object_t obj;
+
+ sx_assert(&vm->mem_segs_lock, SX_XLOCKED);
+
+ if (ident < 0 || ident >= VM_MAX_MEMSEGS)
+ return (EINVAL);
+
+ if (len == 0 || (len & PAGE_MASK))
+ return (EINVAL);
+
+ seg = &vm->mem_segs[ident];
+ if (seg->object != NULL) {
+ if (seg->len == len && seg->sysmem == sysmem)
+ return (EEXIST);
+ else
+ return (EINVAL);
+ }
+
+ obj = vm_object_allocate(OBJT_DEFAULT, len >> PAGE_SHIFT);
+ if (obj == NULL)
+ return (ENOMEM);
+
+ seg->len = len;
+ seg->object = obj;
+ seg->sysmem = sysmem;
+ return (0);
+}
+
+int
+vm_get_memseg(struct vm *vm, int ident, size_t *len, bool *sysmem,
+ vm_object_t *objptr)
+{
+ struct mem_seg *seg;
+
+ sx_assert(&vm->mem_segs_lock, SX_LOCKED);
+
+ if (ident < 0 || ident >= VM_MAX_MEMSEGS)
+ return (EINVAL);
+
+ seg = &vm->mem_segs[ident];
+ if (len)
+ *len = seg->len;
+ if (sysmem)
+ *sysmem = seg->sysmem;
+ if (objptr)
+ *objptr = seg->object;
+ return (0);
+}
+
+void
+vm_free_memseg(struct vm *vm, int ident)
+{
+ struct mem_seg *seg;
+
+ KASSERT(ident >= 0 && ident < VM_MAX_MEMSEGS,
+ ("%s: invalid memseg ident %d", __func__, ident));
+
+ seg = &vm->mem_segs[ident];
+ if (seg->object != NULL) {
+ vm_object_deallocate(seg->object);
+ bzero(seg, sizeof(struct mem_seg));
+ }
+}
+
+int
+vm_mmap_memseg(struct vm *vm, vm_paddr_t gpa, int segid, vm_ooffset_t first,
+ size_t len, int prot, int flags)
+{
+ struct mem_seg *seg;
+ struct mem_map *m, *map;
+ vm_ooffset_t last;
+ int i, error;
+
+ if (prot == 0 || (prot & ~(VM_PROT_ALL)) != 0)
+ return (EINVAL);
+
+ if (flags & ~VM_MEMMAP_F_WIRED)
+ return (EINVAL);
+
+ if (segid < 0 || segid >= VM_MAX_MEMSEGS)
+ return (EINVAL);
+
+ seg = &vm->mem_segs[segid];
+ if (seg->object == NULL)
+ return (EINVAL);
+
+ last = first + len;
+ if (first < 0 || first >= last || last > seg->len)
+ return (EINVAL);
+
+ if ((gpa | first | last) & PAGE_MASK)
+ return (EINVAL);
+
+ map = NULL;
+ for (i = 0; i < VM_MAX_MEMMAPS; i++) {
+ m = &vm->mem_maps[i];
+ if (m->len == 0) {
+ map = m;
+ break;
+ }
+ }
+
+ if (map == NULL)
+ return (ENOSPC);
+
+ error = vm_map_find(&vm->vmspace->vm_map, seg->object, first, &gpa,
+ len, 0, VMFS_NO_SPACE, prot, prot, 0);
+ if (error != KERN_SUCCESS)
+ return (EFAULT);
+
+ vm_object_reference(seg->object);
+
+ if (flags & VM_MEMMAP_F_WIRED) {
+ error = vm_map_wire(&vm->vmspace->vm_map, gpa, gpa + len,
+ VM_MAP_WIRE_USER | VM_MAP_WIRE_NOHOLES);
+ if (error != KERN_SUCCESS) {
+ vm_map_remove(&vm->vmspace->vm_map, gpa, gpa + len);
+ return (error == KERN_RESOURCE_SHORTAGE ? ENOMEM :
+ EFAULT);
+ }
+ }
+
+ map->gpa = gpa;
+ map->len = len;
+ map->segoff = first;
+ map->segid = segid;
+ map->prot = prot;
+ map->flags = flags;
+ return (0);
+}
+
+int
+vm_mmap_getnext(struct vm *vm, vm_paddr_t *gpa, int *segid,
+ vm_ooffset_t *segoff, size_t *len, int *prot, int *flags)
+{
+ struct mem_map *mm, *mmnext;
+ int i;
+
+ mmnext = NULL;
+ for (i = 0; i < VM_MAX_MEMMAPS; i++) {
+ mm = &vm->mem_maps[i];
+ if (mm->len == 0 || mm->gpa < *gpa)
+ continue;
+ if (mmnext == NULL || mm->gpa < mmnext->gpa)
+ mmnext = mm;
+ }
+
+ if (mmnext != NULL) {
+ *gpa = mmnext->gpa;
+ if (segid)
+ *segid = mmnext->segid;
+ if (segoff)
+ *segoff = mmnext->segoff;
+ if (len)
+ *len = mmnext->len;
+ if (prot)
+ *prot = mmnext->prot;
+ if (flags)
+ *flags = mmnext->flags;
+ return (0);
+ } else {
+ return (ENOENT);
+ }
+}
+
+static void
+vm_free_memmap(struct vm *vm, int ident)
+{
+ struct mem_map *mm;
+ int error __diagused;
+
+ mm = &vm->mem_maps[ident];
+ if (mm->len) {
+ error = vm_map_remove(&vm->vmspace->vm_map, mm->gpa,
+ mm->gpa + mm->len);
+ KASSERT(error == KERN_SUCCESS, ("%s: vm_map_remove error %d",
+ __func__, error));
+ bzero(mm, sizeof(struct mem_map));
+ }
+}
+
+static __inline bool
+sysmem_mapping(struct vm *vm, struct mem_map *mm)
+{
+
+ if (mm->len != 0 && vm->mem_segs[mm->segid].sysmem)
+ return (true);
+ else
+ return (false);
+}
+
+vm_paddr_t
+vmm_sysmem_maxaddr(struct vm *vm)
+{
+ struct mem_map *mm;
+ vm_paddr_t maxaddr;
+ int i;
+
+ maxaddr = 0;
+ for (i = 0; i < VM_MAX_MEMMAPS; i++) {
+ mm = &vm->mem_maps[i];
+ if (sysmem_mapping(vm, mm)) {
+ if (maxaddr < mm->gpa + mm->len)
+ maxaddr = mm->gpa + mm->len;
+ }
+ }
+ return (maxaddr);
+}
+
+int
+vm_gla2gpa_nofault(struct vcpu *vcpu, uint64_t gla, int prot, uint64_t *gpa,
+ int *is_fault)
+{
+
+ vmmops_gla2gpa(vcpu->cookie, gla, prot, gpa, is_fault);
+ return (0);
+}
+
+static int
+vmm_reg_raz(struct vcpu *vcpu, uint64_t *rval, void *arg)
+{
+ *rval = 0;
+ return (0);
+}
+
+static int
+vmm_reg_read_arg(struct vcpu *vcpu, uint64_t *rval, void *arg)
+{
+ *rval = *(uint64_t *)arg;
+ return (0);
+}
+
+static int
+vmm_reg_wi(struct vcpu *vcpu, uint64_t wval, void *arg)
+{
+ return (0);
+}
+
+
+#include <sys/queue.h>
+#include <sys/linker.h>
+
+struct vmm_special_reg vmm_special_regs[] = {
+#define SPECIAL_REG(_reg, _read, _write) \
+ { \
+ .esr_iss = ((_reg ## _op0) << ISS_MSR_OP0_SHIFT) | \
+ ((_reg ## _op1) << ISS_MSR_OP1_SHIFT) | \
+ ((_reg ## _CRn) << ISS_MSR_CRn_SHIFT) | \
+ ((_reg ## _CRm) << ISS_MSR_CRm_SHIFT) | \
+ ((_reg ## _op2) << ISS_MSR_OP2_SHIFT), \
+ .esr_mask = ISS_MSR_REG_MASK, \
+ .reg_read = (_read), \
+ .reg_write = (_write), \
+ .arg = NULL, \
+ }
+#define ID_SPECIAL_REG(_reg, _name) \
+ { \
+ .esr_iss = ((_reg ## _op0) << ISS_MSR_OP0_SHIFT) | \
+ ((_reg ## _op1) << ISS_MSR_OP1_SHIFT) | \
+ ((_reg ## _CRn) << ISS_MSR_CRn_SHIFT) | \
+ ((_reg ## _CRm) << ISS_MSR_CRm_SHIFT) | \
+ ((_reg ## _op2) << ISS_MSR_OP2_SHIFT), \
+ .esr_mask = ISS_MSR_REG_MASK, \
+ .reg_read = vmm_reg_read_arg, \
+ .reg_write = vmm_reg_wi, \
+ .arg = &(vmm_desc._name), \
+ }
+
+ /* ID registers */
+ ID_SPECIAL_REG(ID_AA64PFR0_EL1, id_aa64pfr0),
+ ID_SPECIAL_REG(ID_AA64PFR1_EL1, id_aa64pfr1),
+
+ ID_SPECIAL_REG(ID_AA64DFR0_EL1, id_aa64dfr0),
+ ID_SPECIAL_REG(ID_AA64DFR1_EL1, id_aa64dfr1),
+
+ ID_SPECIAL_REG(ID_AA64ISAR0_EL1, id_aa64isar0),
+ ID_SPECIAL_REG(ID_AA64ISAR1_EL1, id_aa64isar1),
+
+ ID_SPECIAL_REG(ID_AA64MMFR0_EL1, id_aa64mmfr0),
+ ID_SPECIAL_REG(ID_AA64MMFR1_EL1, id_aa64mmfr1),
+
+ /*
+ * All other ID registers are read as zero.
+ * They are all in the op0=3, op1=0, CRn=0, CRm={0..7} space.
+ */
+ {
+ .esr_iss = (3 << ISS_MSR_OP0_SHIFT) |
+ (0 << ISS_MSR_OP1_SHIFT) |
+ (0 << ISS_MSR_CRn_SHIFT) |
+ (0 << ISS_MSR_CRm_SHIFT),
+ .esr_mask = ISS_MSR_OP0_MASK | ISS_MSR_OP1_MASK |
+ ISS_MSR_CRn_MASK | (0x8 << ISS_MSR_CRm_SHIFT),
+ .reg_read = vmm_reg_raz,
+ .reg_write = vmm_reg_wi,
+ .arg = NULL,
+ },
+
+ /* Counter physical registers */
+ SPECIAL_REG(CNTP_CTL_EL0, vtimer_phys_ctl_read, vtimer_phys_ctl_write),
+ SPECIAL_REG(CNTP_CVAL_EL0, vtimer_phys_cval_read,
+ vtimer_phys_cval_write),
+ SPECIAL_REG(CNTP_TVAL_EL0, vtimer_phys_tval_read,
+ vtimer_phys_tval_write),
+ SPECIAL_REG(CNTPCT_EL0, vtimer_phys_cnt_read, vtimer_phys_cnt_write),
+#undef SPECIAL_REG
+};
+
+void
+vm_register_reg_handler(struct vm *vm, uint64_t iss, uint64_t mask,
+ reg_read_t reg_read, reg_write_t reg_write, void *arg)
+{
+ int i;
+
+ for (i = 0; i < nitems(vm->special_reg); i++) {
+ if (vm->special_reg[i].esr_iss == 0 &&
+ vm->special_reg[i].esr_mask == 0) {
+ vm->special_reg[i].esr_iss = iss;
+ vm->special_reg[i].esr_mask = mask;
+ vm->special_reg[i].reg_read = reg_read;
+ vm->special_reg[i].reg_write = reg_write;
+ vm->special_reg[i].arg = arg;
+ return;
+ }
+ }
+
+ panic("%s: No free special register slot", __func__);
+}
+
+void
+vm_deregister_reg_handler(struct vm *vm, uint64_t iss, uint64_t mask)
+{
+ int i;
+
+ for (i = 0; i < nitems(vm->special_reg); i++) {
+ if (vm->special_reg[i].esr_iss == iss &&
+ vm->special_reg[i].esr_mask == mask) {
+ memset(&vm->special_reg[i], 0,
+ sizeof(vm->special_reg[i]));
+ return;
+ }
+ }
+
+ panic("%s: Invalid special register: iss %lx mask %lx", __func__, iss,
+ mask);
+}
+
+static int
+vm_handle_reg_emul(struct vcpu *vcpu, bool *retu)
+{
+ struct vm *vm;
+ struct vm_exit *vme;
+ struct vre *vre;
+ int i, rv;
+
+ vm = vcpu->vm;
+ vme = &vcpu->exitinfo;
+ vre = &vme->u.reg_emul.vre;
+
+ for (i = 0; i < nitems(vm->special_reg); i++) {
+ if (vm->special_reg[i].esr_iss == 0 &&
+ vm->special_reg[i].esr_mask == 0)
+ continue;
+
+ if ((vre->inst_syndrome & vm->special_reg[i].esr_mask) ==
+ vm->special_reg[i].esr_iss) {
+ rv = vmm_emulate_register(vcpu, vre,
+ vm->special_reg[i].reg_read,
+ vm->special_reg[i].reg_write,
+ vm->special_reg[i].arg);
+ if (rv == 0) {
+ *retu = false;
+ }
+ return (rv);
+ }
+ }
+ for (i = 0; i < nitems(vmm_special_regs); i++) {
+ if ((vre->inst_syndrome & vmm_special_regs[i].esr_mask) ==
+ vmm_special_regs[i].esr_iss) {
+ rv = vmm_emulate_register(vcpu, vre,
+ vmm_special_regs[i].reg_read,
+ vmm_special_regs[i].reg_write,
+ vmm_special_regs[i].arg);
+ if (rv == 0) {
+ *retu = false;
+ }
+ return (rv);
+ }
+ }
+
+
+ *retu = true;
+ return (0);
+}
+
+void
+vm_register_inst_handler(struct vm *vm, uint64_t start, uint64_t size,
+ mem_region_read_t mmio_read, mem_region_write_t mmio_write)
+{
+ int i;
+
+ for (i = 0; i < nitems(vm->mmio_region); i++) {
+ if (vm->mmio_region[i].start == 0 &&
+ vm->mmio_region[i].end == 0) {
+ vm->mmio_region[i].start = start;
+ vm->mmio_region[i].end = start + size;
+ vm->mmio_region[i].read = mmio_read;
+ vm->mmio_region[i].write = mmio_write;
+ return;
+ }
+ }
+
+ panic("%s: No free MMIO region", __func__);
+}
+
+void
+vm_deregister_inst_handler(struct vm *vm, uint64_t start, uint64_t size)
+{
+ int i;
+
+ for (i = 0; i < nitems(vm->mmio_region); i++) {
+ if (vm->mmio_region[i].start == start &&
+ vm->mmio_region[i].end == start + size) {
+ memset(&vm->mmio_region[i], 0,
+ sizeof(vm->mmio_region[i]));
+ return;
+ }
+ }
+
+ panic("%s: Invalid MMIO region: %lx - %lx", __func__, start,
+ start + size);
+}
+
+static int
+vm_handle_inst_emul(struct vcpu *vcpu, bool *retu)
+{
+ struct vm *vm;
+ struct vm_exit *vme;
+ struct vie *vie;
+ struct hyp *hyp;
+ uint64_t fault_ipa;
+ struct vm_guest_paging *paging;
+ struct vmm_mmio_region *vmr;
+ int error, i;
+
+ vm = vcpu->vm;
+ hyp = vm->cookie;
+ if (!hyp->vgic_attached)
+ goto out_user;
+
+ vme = &vcpu->exitinfo;
+ vie = &vme->u.inst_emul.vie;
+ paging = &vme->u.inst_emul.paging;
+
+ fault_ipa = vme->u.inst_emul.gpa;
+
+ vmr = NULL;
+ for (i = 0; i < nitems(vm->mmio_region); i++) {
+ if (vm->mmio_region[i].start <= fault_ipa &&
+ vm->mmio_region[i].end > fault_ipa) {
+ vmr = &vm->mmio_region[i];
+ break;
+ }
+ }
+ if (vmr == NULL)
+ goto out_user;
+
+ error = vmm_emulate_instruction(vcpu, fault_ipa, vie, paging,
+ vmr->read, vmr->write, retu);
+ return (error);
+
+out_user:
+ *retu = true;
+ return (0);
+}
+
+int
+vm_suspend(struct vm *vm, enum vm_suspend_how how)
+{
+ int i;
+
+ if (how <= VM_SUSPEND_NONE || how >= VM_SUSPEND_LAST)
+ return (EINVAL);
+
+ if (atomic_cmpset_int(&vm->suspend, 0, how) == 0) {
+ VM_CTR2(vm, "virtual machine already suspended %d/%d",
+ vm->suspend, how);
+ return (EALREADY);
+ }
+
+ VM_CTR1(vm, "virtual machine successfully suspended %d", how);
+
+ /*
+ * Notify all active vcpus that they are now suspended.
+ */
+ for (i = 0; i < vm->maxcpus; i++) {
+ if (CPU_ISSET(i, &vm->active_cpus))
+ vcpu_notify_event(vm_vcpu(vm, i), false);
+ }
+
+ return (0);
+}
+
+void
+vm_exit_suspended(struct vcpu *vcpu, uint64_t pc)
+{
+ struct vm *vm = vcpu->vm;
+ struct vm_exit *vmexit;
+
+ KASSERT(vm->suspend > VM_SUSPEND_NONE && vm->suspend < VM_SUSPEND_LAST,
+ ("vm_exit_suspended: invalid suspend type %d", vm->suspend));
+
+ vmexit = vm_exitinfo(vcpu);
+ vmexit->pc = pc;
+ vmexit->inst_length = 4;
+ vmexit->exitcode = VM_EXITCODE_SUSPENDED;
+ vmexit->u.suspended.how = vm->suspend;
+}
+
+void
+vm_exit_debug(struct vcpu *vcpu, uint64_t pc)
+{
+ struct vm_exit *vmexit;
+
+ vmexit = vm_exitinfo(vcpu);
+ vmexit->pc = pc;
+ vmexit->inst_length = 4;
+ vmexit->exitcode = VM_EXITCODE_DEBUG;
+}
+
+int
+vm_activate_cpu(struct vcpu *vcpu)
+{
+ struct vm *vm = vcpu->vm;
+
+ if (CPU_ISSET(vcpu->vcpuid, &vm->active_cpus))
+ return (EBUSY);
+
+ CPU_SET_ATOMIC(vcpu->vcpuid, &vm->active_cpus);
+ return (0);
+
+}
+
+int
+vm_suspend_cpu(struct vm *vm, struct vcpu *vcpu)
+{
+ if (vcpu == NULL) {
+ vm->debug_cpus = vm->active_cpus;
+ for (int i = 0; i < vm->maxcpus; i++) {
+ if (CPU_ISSET(i, &vm->active_cpus))
+ vcpu_notify_event(vm_vcpu(vm, i), false);
+ }
+ } else {
+ if (!CPU_ISSET(vcpu->vcpuid, &vm->active_cpus))
+ return (EINVAL);
+
+ CPU_SET_ATOMIC(vcpu->vcpuid, &vm->debug_cpus);
+ vcpu_notify_event(vcpu, false);
+ }
+ return (0);
+}
+
+int
+vm_resume_cpu(struct vm *vm, struct vcpu *vcpu)
+{
+
+ if (vcpu == NULL) {
+ CPU_ZERO(&vm->debug_cpus);
+ } else {
+ if (!CPU_ISSET(vcpu->vcpuid, &vm->debug_cpus))
+ return (EINVAL);
+
+ CPU_CLR_ATOMIC(vcpu->vcpuid, &vm->debug_cpus);
+ }
+ return (0);
+}
+
+int
+vcpu_debugged(struct vcpu *vcpu)
+{
+
+ return (CPU_ISSET(vcpu->vcpuid, &vcpu->vm->debug_cpus));
+}
+
+cpuset_t
+vm_active_cpus(struct vm *vm)
+{
+
+ return (vm->active_cpus);
+}
+
+cpuset_t
+vm_debug_cpus(struct vm *vm)
+{
+
+ return (vm->debug_cpus);
+}
+
+cpuset_t
+vm_suspended_cpus(struct vm *vm)
+{
+
+ return (vm->suspended_cpus);
+}
+
+
+void *
+vcpu_stats(struct vcpu *vcpu)
+{
+
+ return (vcpu->stats);
+}
+
+/*
+ * This function is called to ensure that a vcpu "sees" a pending event
+ * as soon as possible:
+ * - If the vcpu thread is sleeping then it is woken up.
+ * - If the vcpu is running on a different host_cpu then an IPI will be directed
+ * to the host_cpu to cause the vcpu to trap into the hypervisor.
+ */
+static void
+vcpu_notify_event_locked(struct vcpu *vcpu, bool lapic_intr)
+{
+ int hostcpu;
+
+ KASSERT(lapic_intr == false, ("%s: lapic_intr != false", __func__));
+ hostcpu = vcpu->hostcpu;
+ if (vcpu->state == VCPU_RUNNING) {
+ KASSERT(hostcpu != NOCPU, ("vcpu running on invalid hostcpu"));
+ if (hostcpu != curcpu) {
+#if 0
+ if (lapic_intr) {
+ vlapic_post_intr(vcpu->vlapic, hostcpu,
+ vmm_ipinum);
+ } else
+#endif
+ {
+ ipi_cpu(hostcpu, vmm_ipinum);
+ }
+ } else {
+ /*
+ * If the 'vcpu' is running on 'curcpu' then it must
+ * be sending a notification to itself (e.g. SELF_IPI).
+ * The pending event will be picked up when the vcpu
+ * transitions back to guest context.
+ */
+ }
+ } else {
+ KASSERT(hostcpu == NOCPU, ("vcpu state %d not consistent "
+ "with hostcpu %d", vcpu->state, hostcpu));
+ if (vcpu->state == VCPU_SLEEPING)
+ wakeup_one(vcpu);
+ }
+}
+
+void
+vcpu_notify_event(struct vcpu *vcpu, bool lapic_intr)
+{
+ vcpu_lock(vcpu);
+ vcpu_notify_event_locked(vcpu, lapic_intr);
+ vcpu_unlock(vcpu);
+}
+
+static void
+restore_guest_fpustate(struct vcpu *vcpu)
+{
+
+ /* flush host state to the pcb */
+ vfp_save_state(curthread, curthread->td_pcb);
+ /* Ensure the VFP state will be re-loaded when exiting the guest */
+ PCPU_SET(fpcurthread, NULL);
+
+ /* restore guest FPU state */
+ vfp_enable();
+ vfp_restore(vcpu->guestfpu);
+
+ /*
+ * The FPU is now "dirty" with the guest's state so turn on emulation
+ * to trap any access to the FPU by the host.
+ */
+ vfp_disable();
+}
+
+static void
+save_guest_fpustate(struct vcpu *vcpu)
+{
+ if ((READ_SPECIALREG(cpacr_el1) & CPACR_FPEN_MASK) !=
+ CPACR_FPEN_TRAP_ALL1)
+ panic("VFP not enabled in host!");
+
+ /* save guest FPU state */
+ vfp_enable();
+ vfp_store(vcpu->guestfpu);
+ vfp_disable();
+
+ KASSERT(PCPU_GET(fpcurthread) == NULL,
+ ("%s: fpcurthread set with guest registers", __func__));
+}
+static int
+vcpu_set_state_locked(struct vcpu *vcpu, enum vcpu_state newstate,
+ bool from_idle)
+{
+ int error;
+
+ vcpu_assert_locked(vcpu);
+
+ /*
+ * State transitions from the vmmdev_ioctl() must always begin from
+ * the VCPU_IDLE state. This guarantees that there is only a single
+ * ioctl() operating on a vcpu at any point.
+ */
+ if (from_idle) {
+ while (vcpu->state != VCPU_IDLE) {
+ vcpu_notify_event_locked(vcpu, false);
+ msleep_spin(&vcpu->state, &vcpu->mtx, "vmstat", hz);
+ }
+ } else {
+ KASSERT(vcpu->state != VCPU_IDLE, ("invalid transition from "
+ "vcpu idle state"));
+ }
+
+ if (vcpu->state == VCPU_RUNNING) {
+ KASSERT(vcpu->hostcpu == curcpu, ("curcpu %d and hostcpu %d "
+ "mismatch for running vcpu", curcpu, vcpu->hostcpu));
+ } else {
+ KASSERT(vcpu->hostcpu == NOCPU, ("Invalid hostcpu %d for a "
+ "vcpu that is not running", vcpu->hostcpu));
+ }
+
+ /*
+ * The following state transitions are allowed:
+ * IDLE -> FROZEN -> IDLE
+ * FROZEN -> RUNNING -> FROZEN
+ * FROZEN -> SLEEPING -> FROZEN
+ */
+ switch (vcpu->state) {
+ case VCPU_IDLE:
+ case VCPU_RUNNING:
+ case VCPU_SLEEPING:
+ error = (newstate != VCPU_FROZEN);
+ break;
+ case VCPU_FROZEN:
+ error = (newstate == VCPU_FROZEN);
+ break;
+ default:
+ error = 1;
+ break;
+ }
+
+ if (error)
+ return (EBUSY);
+
+ vcpu->state = newstate;
+ if (newstate == VCPU_RUNNING)
+ vcpu->hostcpu = curcpu;
+ else
+ vcpu->hostcpu = NOCPU;
+
+ if (newstate == VCPU_IDLE)
+ wakeup(&vcpu->state);
+
+ return (0);
+}
+
+static void
+vcpu_require_state(struct vcpu *vcpu, enum vcpu_state newstate)
+{
+ int error;
+
+ if ((error = vcpu_set_state(vcpu, newstate, false)) != 0)
+ panic("Error %d setting state to %d\n", error, newstate);
+}
+
+static void
+vcpu_require_state_locked(struct vcpu *vcpu, enum vcpu_state newstate)
+{
+ int error;
+
+ if ((error = vcpu_set_state_locked(vcpu, newstate, false)) != 0)
+ panic("Error %d setting state to %d", error, newstate);
+}
+
+int
+vm_get_capability(struct vcpu *vcpu, int type, int *retval)
+{
+ if (type < 0 || type >= VM_CAP_MAX)
+ return (EINVAL);
+
+ return (vmmops_getcap(vcpu->cookie, type, retval));
+}
+
+int
+vm_set_capability(struct vcpu *vcpu, int type, int val)
+{
+ if (type < 0 || type >= VM_CAP_MAX)
+ return (EINVAL);
+
+ return (vmmops_setcap(vcpu->cookie, type, val));
+}
+
+struct vm *
+vcpu_vm(struct vcpu *vcpu)
+{
+ return (vcpu->vm);
+}
+
+int
+vcpu_vcpuid(struct vcpu *vcpu)
+{
+ return (vcpu->vcpuid);
+}
+
+void *
+vcpu_get_cookie(struct vcpu *vcpu)
+{
+ return (vcpu->cookie);
+}
+
+struct vcpu *
+vm_vcpu(struct vm *vm, int vcpuid)
+{
+ return (vm->vcpu[vcpuid]);
+}
+
+int
+vcpu_set_state(struct vcpu *vcpu, enum vcpu_state newstate, bool from_idle)
+{
+ int error;
+
+ vcpu_lock(vcpu);
+ error = vcpu_set_state_locked(vcpu, newstate, from_idle);
+ vcpu_unlock(vcpu);
+
+ return (error);
+}
+
+enum vcpu_state
+vcpu_get_state(struct vcpu *vcpu, int *hostcpu)
+{
+ enum vcpu_state state;
+
+ vcpu_lock(vcpu);
+ state = vcpu->state;
+ if (hostcpu != NULL)
+ *hostcpu = vcpu->hostcpu;
+ vcpu_unlock(vcpu);
+
+ return (state);
+}
+
+static void *
+_vm_gpa_hold(struct vm *vm, vm_paddr_t gpa, size_t len, int reqprot,
+ void **cookie)
+{
+ int i, count, pageoff;
+ struct mem_map *mm;
+ vm_page_t m;
+
+ pageoff = gpa & PAGE_MASK;
+ if (len > PAGE_SIZE - pageoff)
+ panic("vm_gpa_hold: invalid gpa/len: 0x%016lx/%lu", gpa, len);
+
+ count = 0;
+ for (i = 0; i < VM_MAX_MEMMAPS; i++) {
+ mm = &vm->mem_maps[i];
+ if (sysmem_mapping(vm, mm) && gpa >= mm->gpa &&
+ gpa < mm->gpa + mm->len) {
+ count = vm_fault_quick_hold_pages(&vm->vmspace->vm_map,
+ trunc_page(gpa), PAGE_SIZE, reqprot, &m, 1);
+ break;
+ }
+ }
+
+ if (count == 1) {
+ *cookie = m;
+ return ((void *)(PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)) + pageoff));
+ } else {
+ *cookie = NULL;
+ return (NULL);
+ }
+}
+
+void *
+vm_gpa_hold(struct vcpu *vcpu, vm_paddr_t gpa, size_t len, int reqprot,
+ void **cookie)
+{
+#ifdef INVARIANTS
+ /*
+ * The current vcpu should be frozen to ensure 'vm_memmap[]'
+ * stability.
+ */
+ int state = vcpu_get_state(vcpu, NULL);
+ KASSERT(state == VCPU_FROZEN, ("%s: invalid vcpu state %d",
+ __func__, state));
+#endif
+ return (_vm_gpa_hold(vcpu->vm, gpa, len, reqprot, cookie));
+}
+
+void *
+vm_gpa_hold_global(struct vm *vm, vm_paddr_t gpa, size_t len, int reqprot,
+ void **cookie)
+{
+ sx_assert(&vm->mem_segs_lock, SX_LOCKED);
+ return (_vm_gpa_hold(vm, gpa, len, reqprot, cookie));
+}
+
+void
+vm_gpa_release(void *cookie)
+{
+ vm_page_t m = cookie;
+
+ vm_page_unwire(m, PQ_ACTIVE);
+}
+
+int
+vm_get_register(struct vcpu *vcpu, int reg, uint64_t *retval)
+{
+
+ if (reg >= VM_REG_LAST)
+ return (EINVAL);
+
+ return (vmmops_getreg(vcpu->cookie, reg, retval));
+}
+
+int
+vm_set_register(struct vcpu *vcpu, int reg, uint64_t val)
+{
+ int error;
+
+ if (reg >= VM_REG_LAST)
+ return (EINVAL);
+ error = vmmops_setreg(vcpu->cookie, reg, val);
+ if (error || reg != VM_REG_ELR_EL2)
+ return (error);
+
+ vcpu->nextpc = val;
+
+ return(0);
+}
+
+void *
+vm_get_cookie(struct vm *vm)
+{
+ return vm->cookie;
+}
+
+int
+vm_inject_exception(struct vcpu *vcpu, uint64_t esr, uint64_t far)
+{
+ return (vmmops_exception(vcpu->cookie, esr, far));
+}
+
+int
+vm_attach_vgic(struct vm *vm, struct vm_vgic_descr *descr)
+{
+ return (vgic_attach_to_vm(vm->cookie, descr));
+}
+
+int
+vm_assert_irq(struct vm *vm, uint32_t irq)
+{
+ return (vgic_inject_irq(vm->cookie, -1, irq, true));
+}
+
+int
+vm_deassert_irq(struct vm *vm, uint32_t irq)
+{
+ struct hyp *hyp = (struct hyp *)vm->cookie;
+ int error;
+
+ error = vgic_inject_irq(hyp, -1, irq, false);
+
+ return (error);
+}
+
+int
+vm_raise_msi(struct vm *vm, uint64_t msg, uint64_t addr, int bus, int slot,
+ int func)
+{
+ /* TODO: Should we raise an SError? */
+ return (vgic_inject_msi(vm->cookie, msg, addr));
+}
+
+static int
+vm_handle_smccc_call(struct vcpu *vcpu, struct vm_exit *vme, bool *retu)
+{
+ struct hypctx *hypctx;
+ int i;
+
+ hypctx = vcpu_get_cookie(vcpu);
+
+ if ((hypctx->tf.tf_esr & ESR_ELx_ISS_MASK) != 0)
+ return (1);
+
+ vme->exitcode = VM_EXITCODE_SMCCC;
+ vme->u.smccc_call.func_id = hypctx->tf.tf_x[0];
+ for (i = 0; i < nitems(vme->u.smccc_call.args); i++)
+ vme->u.smccc_call.args[i] = hypctx->tf.tf_x[i + 1];
+
+ *retu = true;
+ return (0);
+}
+
+static int
+vm_handle_wfi(struct vcpu *vcpu, struct vm_exit *vme, bool *retu)
+{
+ vcpu_lock(vcpu);
+ while (1) {
+ if (vgic_has_pending_irq(vcpu->cookie))
+ break;
+
+ if (vcpu_should_yield(vcpu))
+ break;
+
+ vcpu_require_state_locked(vcpu, VCPU_SLEEPING);
+ /*
+ * XXX msleep_spin() cannot be interrupted by signals so
+ * wake up periodically to check pending signals.
+ */
+ msleep_spin(vcpu, &vcpu->mtx, "vmidle", hz);
+ vcpu_require_state_locked(vcpu, VCPU_FROZEN);
+ }
+ vcpu_unlock(vcpu);
+
+ *retu = false;
+ return (0);
+}
+
+static int
+vm_handle_paging(struct vcpu *vcpu, bool *retu)
+{
+ struct vm *vm = vcpu->vm;
+ struct vm_exit *vme;
+ struct vm_map *map;
+ uint64_t addr, esr;
+ pmap_t pmap;
+ int ftype, rv;
+
+ vme = &vcpu->exitinfo;
+
+ pmap = vmspace_pmap(vcpu->vm->vmspace);
+ addr = vme->u.paging.gpa;
+ esr = vme->u.paging.esr;
+
+ /* The page exists, but the page table needs to be upddated */
+ if (pmap_fault(pmap, esr, addr) == KERN_SUCCESS)
+ return (0);
+
+ switch (ESR_ELx_EXCEPTION(esr)) {
+ case EXCP_INSN_ABORT_L:
+ case EXCP_DATA_ABORT_L:
+ ftype = VM_PROT_EXECUTE | VM_PROT_READ | VM_PROT_WRITE;
+ break;
+ default:
+ panic("%s: Invalid exception (esr = %lx)", __func__, esr);
+ }
+
+ map = &vm->vmspace->vm_map;
+ rv = vm_fault(map, vme->u.paging.gpa, ftype, VM_FAULT_NORMAL, NULL);
+ if (rv != KERN_SUCCESS)
+ return (EFAULT);
+
+ return (0);
+}
+
+int
+vm_run(struct vcpu *vcpu, struct vm_exit *vme_user)
+{
+ struct vm *vm = vcpu->vm;
+ struct vm_eventinfo evinfo;
+ int error, vcpuid;
+ struct vm_exit *vme;
+ bool retu;
+ pmap_t pmap;
+
+ vcpuid = vcpu->vcpuid;
+
+ if (!CPU_ISSET(vcpuid, &vm->active_cpus))
+ return (EINVAL);
+
+ if (CPU_ISSET(vcpuid, &vm->suspended_cpus))
+ return (EINVAL);
+
+ pmap = vmspace_pmap(vm->vmspace);
+ vme = &vcpu->exitinfo;
+ evinfo.rptr = NULL;
+ evinfo.sptr = &vm->suspend;
+ evinfo.iptr = NULL;
+restart:
+ critical_enter();
+
+ restore_guest_fpustate(vcpu);
+
+ vcpu_require_state(vcpu, VCPU_RUNNING);
+ error = vmmops_run(vcpu->cookie, vcpu->nextpc, pmap, &evinfo);
+ vcpu_require_state(vcpu, VCPU_FROZEN);
+
+ save_guest_fpustate(vcpu);
+
+ critical_exit();
+
+ if (error == 0) {
+ retu = false;
+ switch (vme->exitcode) {
+ case VM_EXITCODE_INST_EMUL:
+ vcpu->nextpc = vme->pc + vme->inst_length;
+ error = vm_handle_inst_emul(vcpu, &retu);
+ break;
+
+ case VM_EXITCODE_REG_EMUL:
+ vcpu->nextpc = vme->pc + vme->inst_length;
+ error = vm_handle_reg_emul(vcpu, &retu);
+ break;
+
+ case VM_EXITCODE_HVC:
+ /*
+ * The HVC instruction saves the address for the
+ * next instruction as the return address.
+ */
+ vcpu->nextpc = vme->pc;
+ /*
+ * The PSCI call can change the exit information in the
+ * case of suspend/reset/poweroff/cpu off/cpu on.
+ */
+ error = vm_handle_smccc_call(vcpu, vme, &retu);
+ break;
+
+ case VM_EXITCODE_WFI:
+ vcpu->nextpc = vme->pc + vme->inst_length;
+ error = vm_handle_wfi(vcpu, vme, &retu);
+ break;
+
+ case VM_EXITCODE_PAGING:
+ vcpu->nextpc = vme->pc;
+ error = vm_handle_paging(vcpu, &retu);
+ break;
+
+ default:
+ /* Handle in userland */
+ vcpu->nextpc = vme->pc;
+ retu = true;
+ break;
+ }
+ }
+
+ if (error == 0 && retu == false)
+ goto restart;
+
+ /* copy the exit information */
+ *vme_user = *vme;
+ return (error);
+}
diff --git a/sys/arm64/vmm/vmm_arm64.c b/sys/arm64/vmm/vmm_arm64.c
new file mode 100644
--- /dev/null
+++ b/sys/arm64/vmm/vmm_arm64.c
@@ -0,0 +1,1301 @@
+/*
+ * Copyright (C) 2015 Mihai Carabas <mihai.carabas@gmail.com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/smp.h>
+#include <sys/kernel.h>
+#include <sys/malloc.h>
+#include <sys/pcpu.h>
+#include <sys/proc.h>
+#include <sys/sysctl.h>
+#include <sys/lock.h>
+#include <sys/mutex.h>
+#include <sys/vmem.h>
+
+#include <vm/vm.h>
+#include <vm/pmap.h>
+#include <vm/vm_extern.h>
+#include <vm/vm_map.h>
+#include <vm/vm_page.h>
+#include <vm/vm_param.h>
+
+#include <machine/armreg.h>
+#include <machine/vm.h>
+#include <machine/cpufunc.h>
+#include <machine/cpu.h>
+#include <machine/machdep.h>
+#include <machine/vmm.h>
+#include <machine/vmm_dev.h>
+#include <machine/atomic.h>
+#include <machine/hypervisor.h>
+#include <machine/pmap.h>
+
+#include "mmu.h"
+#include "arm64.h"
+#include "hyp.h"
+#include "reset.h"
+#include "io/vgic.h"
+#include "io/vgic_v3.h"
+#include "io/vtimer.h"
+#include "vmm_stat.h"
+
+#define HANDLED 1
+#define UNHANDLED 0
+
+#define UNUSED 0
+
+/* Number of bits in an EL2 virtual address */
+#define EL2_VIRT_BITS 48
+CTASSERT((1ul << EL2_VIRT_BITS) >= HYP_VM_MAX_ADDRESS);
+
+/* TODO: Move the host hypctx off the stack */
+#define VMM_STACK_PAGES 4
+#define VMM_STACK_SIZE (VMM_STACK_PAGES * PAGE_SIZE)
+
+static int vmm_pmap_levels, vmm_virt_bits, vmm_max_ipa_bits;
+
+/* Register values passed to arm_setup_vectors to set in the hypervisor */
+struct vmm_init_regs {
+ uint64_t tcr_el2;
+ uint64_t vtcr_el2;
+};
+
+MALLOC_DEFINE(M_HYP, "ARM VMM HYP", "ARM VMM HYP");
+
+extern char hyp_init_vectors[];
+extern char hyp_vectors[];
+extern char hyp_stub_vectors[];
+
+static vm_paddr_t hyp_code_base;
+static size_t hyp_code_len;
+
+static char *stack[MAXCPU];
+static vm_offset_t stack_hyp_va[MAXCPU];
+
+static vmem_t *el2_mem_alloc;
+
+static void arm_setup_vectors(void *arg);
+static void vmm_pmap_clean_stage2_tlbi(void);
+static void vmm_pmap_invalidate_range(uint64_t, vm_offset_t, vm_offset_t, bool);
+static void vmm_pmap_invalidate_all(uint64_t);
+
+static inline void
+arm64_set_active_vcpu(struct hypctx *hypctx)
+{
+
+ PCPU_SET(vcpu, hypctx);
+}
+
+static void
+arm_setup_vectors(void *arg)
+{
+ struct vmm_init_regs *el2_regs;
+ char *stack_top;
+ uint32_t sctlr_el2;
+ register_t daif;
+
+ el2_regs = arg;
+ arm64_set_active_vcpu(NULL);
+
+ daif = intr_disable();
+
+ /*
+ * Install the temporary vectors which will be responsible for
+ * initializing the VMM when we next trap into EL2.
+ *
+ * x0: the exception vector table responsible for hypervisor
+ * initialization on the next call.
+ */
+ vmm_call_hyp(vtophys(&vmm_hyp_code));
+
+ /* Create and map the hypervisor stack */
+ stack_top = (char *)stack_hyp_va[PCPU_GET(cpuid)] + VMM_STACK_SIZE;
+
+ /*
+ * Configure the system control register for EL2:
+ *
+ * SCTLR_EL2_M: MMU on
+ * SCTLR_EL2_C: Data cacheability not affected
+ * SCTLR_EL2_I: Instruction cacheability not affected
+ * SCTLR_EL2_A: Instruction alignment check
+ * SCTLR_EL2_SA: Stack pointer alignment check
+ * SCTLR_EL2_WXN: Treat writable memory as execute never
+ * ~SCTLR_EL2_EE: Data accesses are little-endian
+ */
+ sctlr_el2 = SCTLR_EL2_RES1;
+ sctlr_el2 |= SCTLR_EL2_M | SCTLR_EL2_C | SCTLR_EL2_I;
+ sctlr_el2 |= SCTLR_EL2_A | SCTLR_EL2_SA;
+ sctlr_el2 |= SCTLR_EL2_WXN;
+ sctlr_el2 &= ~SCTLR_EL2_EE;
+
+ /* Special call to initialize EL2 */
+ vmm_call_hyp(vmmpmap_to_ttbr0(), stack_top, el2_regs->tcr_el2,
+ sctlr_el2, el2_regs->vtcr_el2);
+
+ intr_restore(daif);
+}
+
+static void
+arm_teardown_vectors(void *arg)
+{
+ register_t daif;
+
+ /*
+ * vmm_cleanup() will disable the MMU. For the next few instructions,
+ * before the hardware disables the MMU, one of the following is
+ * possible:
+ *
+ * a. The instruction addresses are fetched with the MMU disabled,
+ * and they must represent the actual physical addresses. This will work
+ * because we call the vmm_cleanup() function by its physical address.
+ *
+ * b. The instruction addresses are fetched using the old translation
+ * tables. This will work because we have an identity mapping in place
+ * in the translation tables and vmm_cleanup() is called by its physical
+ * address.
+ */
+ daif = intr_disable();
+ /* TODO: Invalidate the cache */
+ vmm_call_hyp(HYP_CLEANUP, vtophys(hyp_stub_vectors));
+ intr_restore(daif);
+
+ arm64_set_active_vcpu(NULL);
+}
+
+static uint64_t
+vmm_vtcr_el2_sl(u_int levels)
+{
+#if PAGE_SIZE == PAGE_SIZE_4K
+ switch(levels) {
+ case 2:
+ return (VTCR_EL2_SL0_4K_LVL2);
+ case 3:
+ return (VTCR_EL2_SL0_4K_LVL1);
+ case 4:
+ return (VTCR_EL2_SL0_4K_LVL0);
+ default:
+ panic("%s: Invalid number of page table levels %u", __func__,
+ levels);
+ }
+#elif PAGE_SIZE == PAGE_SIZE_16K
+ switch(levels) {
+ case 2:
+ return (VTCR_EL2_SL0_16K_LVL2);
+ case 3:
+ return (VTCR_EL2_SL0_16K_LVL1);
+ case 4:
+ return (VTCR_EL2_SL0_16K_LVL0);
+ default:
+ panic("%s: Invalid number of page table levels %u", __func__,
+ levels);
+ }
+#else
+#error Unsupported page size
+#endif
+}
+
+int
+vmmops_modinit(int ipinum)
+{
+ struct vmm_init_regs el2_regs;
+ vm_offset_t next_hyp_va;
+ vm_paddr_t vmm_base;
+ uint64_t id_aa64mmfr0_el1, pa_range_bits, pa_range_field;
+ uint64_t cnthctl_el2;
+ register_t daif;
+ int cpu, i;
+ bool rv __diagused;
+
+ if (!virt_enabled()) {
+ printf("arm_init: Processor doesn't have support for virtualization.\n");
+ return (ENXIO);
+ }
+
+ if (!vgic_present()) {
+ printf("arm_init: No vgic found\n");
+ return (ENODEV);
+ }
+
+ if (!get_kernel_reg(ID_AA64MMFR0_EL1, &id_aa64mmfr0_el1)) {
+ printf("arm_init: Unable to read ID_AA64MMFR0_EL1\n");
+ return (ENXIO);
+ }
+ pa_range_field = ID_AA64MMFR0_PARange_VAL(id_aa64mmfr0_el1);
+ /*
+ * Use 3 levels to give us up to 39 bits with 4k pages, or
+ * 47 bits with 16k pages.
+ */
+ /* TODO: Check the number of levels for 64k pages */
+ vmm_pmap_levels = 3;
+ switch (pa_range_field) {
+ case ID_AA64MMFR0_PARange_4G:
+ printf("arm_init: Not enough physical address bits\n");
+ return (ENXIO);
+ case ID_AA64MMFR0_PARange_64G:
+ vmm_virt_bits = 36;
+#if PAGE_SIZE == PAGE_SIZE_16K
+ /* TODO: Test */
+ vmm_pmap_levels = 2;
+#endif
+ break;
+ default:
+ vmm_virt_bits = 39;
+ break;
+ }
+ pa_range_bits = pa_range_field >> ID_AA64MMFR0_PARange_SHIFT;
+
+ /* Initialise the EL2 MMU */
+ if (!vmmpmap_init()) {
+ printf("arm_init: Failed to init the EL2 MMU\n");
+ return (ENOMEM);
+ }
+
+ /* Set up the stage 2 pmap callbacks */
+ MPASS(pmap_clean_stage2_tlbi == NULL);
+ pmap_clean_stage2_tlbi = vmm_pmap_clean_stage2_tlbi;
+ pmap_stage2_invalidate_range = vmm_pmap_invalidate_range;
+ pmap_stage2_invalidate_all = vmm_pmap_invalidate_all;
+
+ /* Create the vmem allocator */
+ el2_mem_alloc = vmem_create("VMM EL2", 0, 0, PAGE_SIZE, 0, M_WAITOK);
+
+ /* Create the mappings for the hypervisor translation table. */
+ hyp_code_len = roundup2(&vmm_hyp_code_end - &vmm_hyp_code, PAGE_SIZE);
+
+ /* We need an physical identity mapping for when we activate the MMU */
+ hyp_code_base = vmm_base = vtophys(&vmm_hyp_code);
+ rv = vmmpmap_enter(vmm_base, hyp_code_len, vtophys(&vmm_hyp_code),
+ VM_PROT_READ | VM_PROT_EXECUTE);
+ MPASS(rv);
+
+ next_hyp_va = roundup2(vtophys(&vmm_hyp_code) + hyp_code_len, L2_SIZE);
+
+ /* Create a per-CPU hypervisor stack */
+ CPU_FOREACH(cpu) {
+ stack[cpu] = malloc(VMM_STACK_SIZE, M_HYP, M_WAITOK | M_ZERO);
+ stack_hyp_va[cpu] = next_hyp_va;
+
+ for (i = 0; i < VMM_STACK_PAGES; i++) {
+ rv = vmmpmap_enter(stack_hyp_va[cpu] + (i * PAGE_SIZE),
+ PAGE_SIZE, vtophys(stack[cpu] + (i * PAGE_SIZE)),
+ VM_PROT_READ | VM_PROT_WRITE);
+ MPASS(rv);
+ }
+ next_hyp_va += L2_SIZE;
+ }
+
+ el2_regs.tcr_el2 = TCR_EL2_RES1;
+ el2_regs.tcr_el2 |= min(pa_range_bits << TCR_EL2_PS_SHIFT,
+ TCR_EL2_PS_52BITS);
+ el2_regs.tcr_el2 |= TCR_EL2_T0SZ(64 - EL2_VIRT_BITS);
+ el2_regs.tcr_el2 |= TCR_EL2_IRGN0_WBWA | TCR_EL2_ORGN0_WBWA;
+#if PAGE_SIZE == PAGE_SIZE_4K
+ el2_regs.tcr_el2 |= TCR_EL2_TG0_4K;
+#elif PAGE_SIZE == PAGE_SIZE_16K
+ el2_regs.tcr_el2 |= TCR_EL2_TG0_16K;
+#else
+#error Unsupported page size
+#endif
+#ifdef SMP
+ el2_regs.tcr_el2 |= TCR_EL2_SH0_IS;
+#endif
+
+ switch (el2_regs.tcr_el2 & TCR_EL2_PS_MASK) {
+ case TCR_EL2_PS_32BITS:
+ vmm_max_ipa_bits = 32;
+ break;
+ case TCR_EL2_PS_36BITS:
+ vmm_max_ipa_bits = 36;
+ break;
+ case TCR_EL2_PS_40BITS:
+ vmm_max_ipa_bits = 40;
+ break;
+ case TCR_EL2_PS_42BITS:
+ vmm_max_ipa_bits = 42;
+ break;
+ case TCR_EL2_PS_44BITS:
+ vmm_max_ipa_bits = 44;
+ break;
+ case TCR_EL2_PS_48BITS:
+ vmm_max_ipa_bits = 48;
+ break;
+ case TCR_EL2_PS_52BITS:
+ default:
+ vmm_max_ipa_bits = 52;
+ break;
+ }
+
+ /*
+ * Configure the Stage 2 translation control register:
+ *
+ * VTCR_IRGN0_WBWA: Translation table walks access inner cacheable
+ * normal memory
+ * VTCR_ORGN0_WBWA: Translation table walks access outer cacheable
+ * normal memory
+ * VTCR_EL2_TG0_4K/16K: Stage 2 uses the same page size as the kernel
+ * VTCR_EL2_SL0_4K_LVL1: Stage 2 uses concatenated level 1 tables
+ * VTCR_EL2_SH0_IS: Memory associated with Stage 2 walks is inner
+ * shareable
+ */
+ el2_regs.vtcr_el2 = VTCR_EL2_RES1;
+ el2_regs.vtcr_el2 |=
+ min(pa_range_bits << VTCR_EL2_PS_SHIFT, VTCR_EL2_PS_48BIT);
+ el2_regs.vtcr_el2 |= VTCR_EL2_IRGN0_WBWA | VTCR_EL2_ORGN0_WBWA;
+ el2_regs.vtcr_el2 |= VTCR_EL2_T0SZ(64 - vmm_virt_bits);
+ el2_regs.vtcr_el2 |= vmm_vtcr_el2_sl(vmm_pmap_levels);
+#if PAGE_SIZE == PAGE_SIZE_4K
+ el2_regs.vtcr_el2 |= VTCR_EL2_TG0_4K;
+#elif PAGE_SIZE == PAGE_SIZE_16K
+ el2_regs.vtcr_el2 |= VTCR_EL2_TG0_16K;
+#else
+#error Unsupported page size
+#endif
+#ifdef SMP
+ el2_regs.vtcr_el2 |= VTCR_EL2_SH0_IS;
+#endif
+
+ smp_rendezvous(NULL, arm_setup_vectors, NULL, &el2_regs);
+
+ /* Add memory to the vmem allocator (checking there is space) */
+ if (vmm_base > L2_SIZE) {
+ /*
+ * Ensure there is an L2 block before the vmm code to check
+ * for buffer overflows on earlier data. Include the PAGE_SIZE
+ * of the minimum we can allocate.
+ */
+ vmm_base -= L2_SIZE + PAGE_SIZE;
+ vmm_base = rounddown2(vmm_base, L2_SIZE);
+
+ /*
+ * Check there is memory before the vmm code to add.
+ *
+ * Reserve the L2 block at address 0 so NULL dereference will
+ * raise an exception
+ */
+ if (vmm_base > L2_SIZE)
+ vmem_add(el2_mem_alloc, L2_SIZE, next_hyp_va - L2_SIZE,
+ M_WAITOK);
+ }
+
+ /*
+ * Add the memory after the stacks. There is most of an L2 block
+ * between the last stack and the first allocation so this should
+ * be safe without adding more padding.
+ */
+ if (next_hyp_va < HYP_VM_MAX_ADDRESS - PAGE_SIZE)
+ vmem_add(el2_mem_alloc, next_hyp_va,
+ HYP_VM_MAX_ADDRESS - next_hyp_va, M_WAITOK);
+
+
+ daif = intr_disable();
+ cnthctl_el2 = vmm_call_hyp(HYP_READ_REGISTER, HYP_REG_CNTHCTL);
+ intr_restore(daif);
+
+ vgic_init();
+ vtimer_init(cnthctl_el2);
+
+ return (0);
+}
+
+int
+vmmops_modcleanup(void)
+{
+ int cpu;
+
+ smp_rendezvous(NULL, arm_teardown_vectors, NULL, NULL);
+
+#ifdef INVARIANTS
+ CPU_FOREACH(cpu) {
+ vmmpmap_remove(stack_hyp_va[cpu], VMM_STACK_PAGES * PAGE_SIZE,
+ false);
+ }
+
+ vmmpmap_remove(hyp_code_base, hyp_code_len, false);
+#endif
+
+ vtimer_cleanup();
+
+ vmmpmap_fini();
+ for (cpu = 0; cpu < nitems(stack); cpu++)
+ free(stack[cpu], M_HYP);
+
+ pmap_clean_stage2_tlbi = NULL;
+
+ return (0);
+}
+
+void *
+vmmops_init(struct vm *vm, pmap_t pmap)
+{
+ struct hyp *hyp;
+ vmem_addr_t vm_addr;
+ vm_size_t size;
+ bool rv __diagused;
+ int err __diagused;
+
+ /*
+ * Allocate space for the common hyp struct and
+ * a hypctx pointer per vcpu.
+ */
+ size = sizeof(struct hyp) +
+ sizeof(struct hypctx *) * vm_get_maxcpus(vm);
+ /* Ensure this is the only data on the page */
+ size = roundup2(size, PAGE_SIZE);
+ hyp = malloc_aligned(size, PAGE_SIZE, M_HYP, M_WAITOK | M_ZERO);
+ MPASS(((vm_offset_t)hyp & PAGE_MASK) == 0);
+
+ hyp->vm = vm;
+ hyp->vgic_attached = false;
+
+ vtimer_vminit(hyp);
+ vgic_vminit(hyp);
+
+ /* XXX: Can this fail? */
+ err = vmem_alloc(el2_mem_alloc, size, M_NEXTFIT | M_WAITOK,
+ &vm_addr);
+ MPASS(err == 0);
+ MPASS((vm_addr & PAGE_MASK) == 0);
+ hyp->el2_addr = vm_addr;
+
+ rv = vmmpmap_enter(hyp->el2_addr, size, vtophys(hyp),
+ VM_PROT_READ | VM_PROT_WRITE);
+ MPASS(rv);
+
+ return (hyp);
+}
+
+void *
+vmmops_vcpu_init(void *vmi, struct vcpu *vcpu1, int vcpuid)
+{
+ struct hyp *hyp = vmi;
+ struct hypctx *hypctx;
+ vmem_addr_t vm_addr;
+ vm_size_t size;
+ bool rv __diagused;
+ int err __diagused;
+
+ /*
+ * Allocate space for the common hyp struct and
+ * a hypctx pointer per vcpu.
+ */
+ size = sizeof(struct hypctx);
+ /* Ensure this is the only data on the page */
+ size = roundup2(size, PAGE_SIZE);
+ hypctx = malloc_aligned(size, PAGE_SIZE, M_HYP, M_WAITOK | M_ZERO);
+ MPASS(((vm_offset_t)hyp & PAGE_MASK) == 0);
+
+ KASSERT(vcpuid >= 0 && vcpuid < vm_get_maxcpus(hyp->vm),
+ ("%s: Invalid vcpuid %d", __func__, vcpuid));
+ hyp->ctx[vcpuid] = hypctx;
+
+ hypctx->hyp = hyp;
+ hypctx->vcpu = vcpu1;
+
+ reset_vm_el01_regs(hypctx);
+ reset_vm_el2_regs(hypctx);
+
+ vtimer_cpuinit(hypctx);
+
+ vgic_cpuinit(hypctx);
+
+ /* XXX: Can this fail? */
+ err = vmem_alloc(el2_mem_alloc, size, M_NEXTFIT | M_WAITOK,
+ &vm_addr);
+ MPASS(err == 0);
+ MPASS((vm_addr & PAGE_MASK) == 0);
+ hypctx->el2_addr = vm_addr;
+
+ rv = vmmpmap_enter(hypctx->el2_addr, size, vtophys(hypctx),
+ VM_PROT_READ | VM_PROT_WRITE);
+ MPASS(rv);
+
+ return (hypctx);
+}
+
+static int
+arm_vmm_pinit(pmap_t pmap)
+{
+
+ pmap_pinit_stage(pmap, PM_STAGE2, vmm_pmap_levels);
+ return (1);
+}
+
+struct vmspace *
+vmmops_vmspace_alloc(vm_offset_t min, vm_offset_t max)
+{
+ return (vmspace_alloc(min, max, arm_vmm_pinit));
+}
+
+void
+vmmops_vmspace_free(struct vmspace *vmspace)
+{
+
+ pmap_remove_pages(vmspace_pmap(vmspace));
+ vmspace_free(vmspace);
+}
+
+static void
+vmm_pmap_clean_stage2_tlbi(void)
+{
+ vmm_call_hyp(HYP_CLEAN_S2_TLBI);
+}
+
+static void
+vmm_pmap_invalidate_range(uint64_t vttbr, vm_offset_t sva, vm_offset_t eva,
+ bool final_only)
+{
+ MPASS(eva > sva);
+ vmm_call_hyp(HYP_S2_TLBI_RANGE, vttbr, sva, eva, final_only);
+}
+
+static void
+vmm_pmap_invalidate_all(uint64_t vttbr)
+{
+ vmm_call_hyp(HYP_S2_TLBI_ALL, vttbr);
+}
+
+static enum vm_reg_name
+get_vm_reg_name(uint32_t reg_nr, uint32_t mode __attribute__((unused)))
+{
+ switch(reg_nr) {
+ case 0:
+ return VM_REG_GUEST_X0;
+ case 1:
+ return VM_REG_GUEST_X1;
+ case 2:
+ return VM_REG_GUEST_X2;
+ case 3:
+ return VM_REG_GUEST_X3;
+ case 4:
+ return VM_REG_GUEST_X4;
+ case 5:
+ return VM_REG_GUEST_X5;
+ case 6:
+ return VM_REG_GUEST_X6;
+ case 7:
+ return VM_REG_GUEST_X7;
+ case 8:
+ return VM_REG_GUEST_X8;
+ case 9:
+ return VM_REG_GUEST_X9;
+ case 10:
+ return VM_REG_GUEST_X10;
+ case 11:
+ return VM_REG_GUEST_X11;
+ case 12:
+ return VM_REG_GUEST_X12;
+ case 13:
+ return VM_REG_GUEST_X13;
+ case 14:
+ return VM_REG_GUEST_X14;
+ case 15:
+ return VM_REG_GUEST_X15;
+ case 16:
+ return VM_REG_GUEST_X16;
+ case 17:
+ return VM_REG_GUEST_X17;
+ case 18:
+ return VM_REG_GUEST_X18;
+ case 19:
+ return VM_REG_GUEST_X19;
+ case 20:
+ return VM_REG_GUEST_X20;
+ case 21:
+ return VM_REG_GUEST_X21;
+ case 22:
+ return VM_REG_GUEST_X22;
+ case 23:
+ return VM_REG_GUEST_X23;
+ case 24:
+ return VM_REG_GUEST_X24;
+ case 25:
+ return VM_REG_GUEST_X25;
+ case 26:
+ return VM_REG_GUEST_X26;
+ case 27:
+ return VM_REG_GUEST_X27;
+ case 28:
+ return VM_REG_GUEST_X28;
+ case 29:
+ return VM_REG_GUEST_X29;
+ case 30:
+ return VM_REG_GUEST_LR;
+ case 31:
+ return VM_REG_GUEST_SP;
+ case 32:
+ return VM_REG_GUEST_ELR;
+ case 33:
+ return VM_REG_GUEST_SPSR;
+ case 34:
+ return VM_REG_ELR_EL2;
+ default:
+ break;
+ }
+
+ return (VM_REG_LAST);
+}
+
+static inline void
+arm64_print_hyp_regs(struct vm_exit *vme)
+{
+ printf("esr_el2: 0x%016lx\n", vme->u.hyp.esr_el2);
+ printf("far_el2: 0x%016lx\n", vme->u.hyp.far_el2);
+ printf("hpfar_el2: 0x%016lx\n", vme->u.hyp.hpfar_el2);
+ printf("elr_el2: 0x%016lx\n", vme->pc);
+}
+
+static void
+arm64_gen_inst_emul_data(struct hypctx *hypctx, uint32_t esr_iss,
+ struct vm_exit *vme_ret)
+{
+ struct vm_guest_paging *paging;
+ struct vie *vie;
+ uint32_t esr_sas, reg_num;
+ uint64_t page_off;
+
+ /*
+ * Get the page address from HPFAR_EL2.
+ */
+ vme_ret->u.inst_emul.gpa =
+ HPFAR_EL2_FIPA_ADDR(hypctx->exit_info.hpfar_el2);
+ /* Bits [11:0] are the same as bits [11:0] from the virtual address. */
+ page_off = FAR_EL2_PAGE_OFFSET(hypctx->exit_info.far_el2);
+ vme_ret->u.inst_emul.gpa += page_off;
+
+ esr_sas = (esr_iss & ISS_DATA_SAS_MASK) >> ISS_DATA_SAS_SHIFT;
+ reg_num = (esr_iss & ISS_DATA_SRT_MASK) >> ISS_DATA_SRT_SHIFT;
+
+ vie = &vme_ret->u.inst_emul.vie;
+ vie->access_size = 1 << esr_sas;
+ vie->sign_extend = (esr_iss & ISS_DATA_SSE) ? 1 : 0;
+ vie->dir = (esr_iss & ISS_DATA_WnR) ? VM_DIR_WRITE : VM_DIR_READ;
+ vie->reg = get_vm_reg_name(reg_num, UNUSED);
+
+ paging = &vme_ret->u.inst_emul.paging;
+ paging->far = hypctx->exit_info.far_el2;
+ paging->ttbr0_el1 = hypctx->ttbr0_el1;
+ paging->ttbr1_el1 = hypctx->ttbr1_el1;
+ paging->flags = hypctx->tf.tf_spsr & (PSR_M_MASK | PSR_M_32);
+ if ((hypctx->sctlr_el1 & SCTLR_M) != 0)
+ paging->flags |= VM_GP_MMU_ENABLED;
+}
+
+static void
+arm64_gen_reg_emul_data(uint32_t esr_iss, struct vm_exit *vme_ret)
+{
+ uint32_t reg_num;
+ struct vre *vre;
+
+ /* u.hyp member will be replaced by u.reg_emul */
+ vre = &vme_ret->u.reg_emul.vre;
+
+ vre->inst_syndrome = esr_iss;
+ /* ARMv8 Architecture Manual, p. D7-2273: 1 means read */
+ vre->dir = (esr_iss & ISS_MSR_DIR) ? VM_DIR_READ : VM_DIR_WRITE;
+ reg_num = ISS_MSR_Rt(esr_iss);
+ vre->reg = get_vm_reg_name(reg_num, UNUSED);
+}
+
+static void
+raise_data_insn_abort(struct hypctx *hypctx, uint64_t far, bool dabort, int fsc)
+{
+ uint64_t esr;
+
+ if ((hypctx->tf.tf_spsr & PSR_M_MASK) == PSR_M_EL0t)
+ esr = EXCP_INSN_ABORT_L << ESR_ELx_EC_SHIFT;
+ else
+ esr = EXCP_INSN_ABORT << ESR_ELx_EC_SHIFT;
+ /* Set the bit that changes from insn -> data abort */
+ if (dabort)
+ esr |= EXCP_DATA_ABORT_L << ESR_ELx_EC_SHIFT;
+ /* Set the IL bit if set by hardware */
+ esr |= hypctx->tf.tf_esr & ESR_ELx_IL;
+
+ vmmops_exception(hypctx, esr | fsc, far);
+}
+
+static int
+handle_el1_sync_excp(struct hypctx *hypctx, struct vm_exit *vme_ret,
+ pmap_t pmap)
+{
+ uint64_t gpa;
+ uint32_t esr_ec, esr_iss;
+
+ esr_ec = ESR_ELx_EXCEPTION(hypctx->tf.tf_esr);
+ esr_iss = hypctx->tf.tf_esr & ESR_ELx_ISS_MASK;
+
+ switch(esr_ec) {
+ case EXCP_UNKNOWN:
+ vmm_stat_incr(hypctx->vcpu, VMEXIT_UNKNOWN, 1);
+ arm64_print_hyp_regs(vme_ret);
+ vme_ret->exitcode = VM_EXITCODE_HYP;
+ break;
+ case EXCP_TRAP_WFI_WFE:
+ if ((hypctx->tf.tf_esr & 0x3) == 0) { /* WFI */
+ vmm_stat_incr(hypctx->vcpu, VMEXIT_WFI, 1);
+ vme_ret->exitcode = VM_EXITCODE_WFI;
+ } else {
+ vmm_stat_incr(hypctx->vcpu, VMEXIT_WFE, 1);
+ vme_ret->exitcode = VM_EXITCODE_HYP;
+ }
+ break;
+ case EXCP_HVC:
+ vmm_stat_incr(hypctx->vcpu, VMEXIT_HVC, 1);
+ vme_ret->exitcode = VM_EXITCODE_HVC;
+ break;
+ case EXCP_MSR:
+ vmm_stat_incr(hypctx->vcpu, VMEXIT_MSR, 1);
+ arm64_gen_reg_emul_data(esr_iss, vme_ret);
+ vme_ret->exitcode = VM_EXITCODE_REG_EMUL;
+ break;
+
+ case EXCP_INSN_ABORT_L:
+ case EXCP_DATA_ABORT_L:
+ vmm_stat_incr(hypctx->vcpu, esr_ec == EXCP_DATA_ABORT_L ?
+ VMEXIT_DATA_ABORT : VMEXIT_INSN_ABORT, 1);
+ switch (hypctx->tf.tf_esr & ISS_DATA_DFSC_MASK) {
+ case ISS_DATA_DFSC_TF_L0:
+ case ISS_DATA_DFSC_TF_L1:
+ case ISS_DATA_DFSC_TF_L2:
+ case ISS_DATA_DFSC_TF_L3:
+ case ISS_DATA_DFSC_AFF_L1:
+ case ISS_DATA_DFSC_AFF_L2:
+ case ISS_DATA_DFSC_AFF_L3:
+ case ISS_DATA_DFSC_PF_L1:
+ case ISS_DATA_DFSC_PF_L2:
+ case ISS_DATA_DFSC_PF_L3:
+ gpa = HPFAR_EL2_FIPA_ADDR(hypctx->exit_info.hpfar_el2);
+ /* Check the IPA is valid */
+ if (gpa >= (1ul << vmm_max_ipa_bits)) {
+ raise_data_insn_abort(hypctx,
+ hypctx->exit_info.far_el2,
+ esr_ec == EXCP_DATA_ABORT_L,
+ ISS_DATA_DFSC_ASF_L0);
+ vme_ret->inst_length = 0;
+ return (HANDLED);
+ }
+
+ if (vm_mem_allocated(hypctx->vcpu, gpa)) {
+ vme_ret->exitcode = VM_EXITCODE_PAGING;
+ vme_ret->inst_length = 0;
+ vme_ret->u.paging.esr = hypctx->tf.tf_esr;
+ vme_ret->u.paging.gpa = gpa;
+ } else if (esr_ec == EXCP_INSN_ABORT_L) {
+ /*
+ * Raise an external abort. Device memory is
+ * not executable
+ */
+ raise_data_insn_abort(hypctx,
+ hypctx->exit_info.far_el2, false,
+ ISS_DATA_DFSC_EXT);
+ vme_ret->inst_length = 0;
+ return (HANDLED);
+ } else {
+ arm64_gen_inst_emul_data(hypctx, esr_iss,
+ vme_ret);
+ vme_ret->exitcode = VM_EXITCODE_INST_EMUL;
+ }
+ break;
+ default:
+ arm64_print_hyp_regs(vme_ret);
+ vme_ret->exitcode = VM_EXITCODE_HYP;
+ break;
+ }
+
+ break;
+
+ default:
+ vmm_stat_incr(hypctx->vcpu, VMEXIT_UNHANDLED_SYNC, 1);
+ arm64_print_hyp_regs(vme_ret);
+ vme_ret->exitcode = VM_EXITCODE_HYP;
+ break;
+ }
+
+ /* We don't don't do any instruction emulation here */
+ return (UNHANDLED);
+}
+
+static int
+arm64_handle_world_switch(struct hypctx *hypctx, int excp_type,
+ struct vm_exit *vme, pmap_t pmap)
+{
+ int handled;
+
+ switch (excp_type) {
+ case EXCP_TYPE_EL1_SYNC:
+ /* The exit code will be set by handle_el1_sync_excp(). */
+ handled = handle_el1_sync_excp(hypctx, vme, pmap);
+ break;
+
+ case EXCP_TYPE_EL1_IRQ:
+ case EXCP_TYPE_EL1_FIQ:
+ /* The host kernel will handle IRQs and FIQs. */
+ vmm_stat_incr(hypctx->vcpu,
+ excp_type == EXCP_TYPE_EL1_IRQ ? VMEXIT_IRQ : VMEXIT_FIQ,1);
+ vme->exitcode = VM_EXITCODE_BOGUS;
+ handled = UNHANDLED;
+ break;
+
+ case EXCP_TYPE_EL1_ERROR:
+ case EXCP_TYPE_EL2_SYNC:
+ case EXCP_TYPE_EL2_IRQ:
+ case EXCP_TYPE_EL2_FIQ:
+ case EXCP_TYPE_EL2_ERROR:
+ vmm_stat_incr(hypctx->vcpu, VMEXIT_UNHANDLED_EL2, 1);
+ vme->exitcode = VM_EXITCODE_BOGUS;
+ handled = UNHANDLED;
+ break;
+
+ default:
+ vmm_stat_incr(hypctx->vcpu, VMEXIT_UNHANDLED, 1);
+ vme->exitcode = VM_EXITCODE_BOGUS;
+ handled = UNHANDLED;
+ break;
+ }
+
+ return (handled);
+}
+
+static void
+ptp_release(void **cookie)
+{
+ if (*cookie != NULL) {
+ vm_gpa_release(*cookie);
+ *cookie = NULL;
+ }
+}
+
+static void *
+ptp_hold(struct vcpu *vcpu, vm_paddr_t ptpphys, size_t len, void **cookie)
+{
+ void *ptr;
+
+ ptp_release(cookie);
+ ptr = vm_gpa_hold(vcpu, ptpphys, len, VM_PROT_RW, cookie);
+ return (ptr);
+}
+
+void
+vmmops_gla2gpa(void *vcpui, uint64_t gla, int prot, uint64_t *gpa,
+ int *is_fault)
+{
+ struct hypctx *hypctx;
+ void *cookie;
+ uint64_t *ptep, pte;
+ uint64_t ttbr;
+
+ hypctx = (struct hypctx *)vcpui;
+
+ /* TODO: Handle TBI/MTE/etc */
+ /* TODO: Support non-native granule size */
+ /* TODO: Support non-4 level page tables */
+ /* TODO: Support > 48-bit address space */
+ /* TODO: Check prot */
+
+ /* Check if the MMU is off */
+ if ((hypctx->sctlr_el1 & SCTLR_M) == 0) {
+ *is_fault = 0;
+ *gpa = gla;
+ return;
+ }
+
+ if (ADDR_IS_KERNEL(gla)) {
+ ttbr = hypctx->ttbr1_el1;
+ } else {
+ ttbr = hypctx->ttbr0_el1;
+ }
+
+ cookie = NULL;
+ ptep = ptp_hold(hypctx->vcpu, ttbr, PAGE_SIZE, &cookie);
+ pte = ptep[pmap_l0_index(gla)];
+ if ((pte & ATTR_DESCR_MASK) != L0_TABLE)
+ goto fault;
+
+ ptep = ptp_hold(hypctx->vcpu, pte & ~ATTR_MASK, PAGE_SIZE, &cookie);
+ pte = ptep[pmap_l1_index(gla)];
+ if ((pte & ATTR_DESCR_MASK) == L1_BLOCK) {
+ *gpa = (pte & ~ATTR_MASK) | (gla & L1_OFFSET);
+ goto done;
+ }
+ if ((pte & ATTR_DESCR_MASK) != L1_TABLE)
+ goto fault;
+
+ ptep = ptp_hold(hypctx->vcpu, pte & ~ATTR_MASK, PAGE_SIZE, &cookie);
+ pte = ptep[pmap_l2_index(gla)];
+ if ((pte & ATTR_DESCR_MASK) == L2_BLOCK) {
+ *gpa = (pte & ~ATTR_MASK) | (gla & L2_OFFSET);
+ goto done;
+ }
+ if ((pte & ATTR_DESCR_MASK) != L2_TABLE)
+ goto fault;
+
+ ptep = ptp_hold(hypctx->vcpu, pte & ~ATTR_MASK, PAGE_SIZE, &cookie);
+ pte = ptep[pmap_l3_index(gla)];
+ if ((pte & ATTR_DESCR_MASK) == L3_PAGE) {
+ *gpa = (pte & ~ATTR_MASK) | (gla & L3_OFFSET);
+ } else
+ goto fault;
+
+done:
+ *is_fault = 0;
+ ptp_release(&cookie);
+ return;
+
+fault:
+ *is_fault = 1;
+ ptp_release(&cookie);
+}
+
+int
+vmmops_run(void *vcpui, register_t pc, pmap_t pmap, struct vm_eventinfo *evinfo)
+{
+ uint64_t excp_type;
+ int handled;
+ register_t daif;
+ struct hyp *hyp;
+ struct hypctx *hypctx;
+ struct vcpu *vcpu;
+ struct vm_exit *vme;
+ int mode;
+
+ hypctx = (struct hypctx *)vcpui;
+ hyp = hypctx->hyp;
+ vcpu = hypctx->vcpu;
+ vme = vm_exitinfo(vcpu);
+
+ hypctx->tf.tf_elr = (uint64_t)pc;
+
+ for (;;) {
+ if (hypctx->has_exception) {
+ hypctx->has_exception = false;
+ hypctx->elr_el1 = hypctx->tf.tf_elr;
+
+ mode = hypctx->tf.tf_spsr & (PSR_M_MASK | PSR_M_32);
+
+ if (mode == PSR_M_EL1t) {
+ hypctx->tf.tf_elr = hypctx->vbar_el1 + 0x0;
+ } else if (mode == PSR_M_EL1h) {
+ hypctx->tf.tf_elr = hypctx->vbar_el1 + 0x200;
+ } else if ((mode & PSR_M_32) == PSR_M_64) {
+ /* 64-bit EL0 */
+ hypctx->tf.tf_elr = hypctx->vbar_el1 + 0x400;
+ } else {
+ /* 32-bit EL0 */
+ hypctx->tf.tf_elr = hypctx->vbar_el1 + 0x600;
+ }
+
+ /* Set the new spsr */
+ hypctx->spsr_el1 = hypctx->tf.tf_spsr;
+
+ /* Set the new cpsr */
+ hypctx->tf.tf_spsr = hypctx->spsr_el1 & PSR_FLAGS;
+ /* TODO: DIT, PAN, SSBS */
+ hypctx->tf.tf_spsr |= PSR_DAIF | PSR_M_EL1h;
+ }
+
+ daif = intr_disable();
+
+ /* Check if the vcpu is suspended */
+ if (vcpu_suspended(evinfo)) {
+ intr_restore(daif);
+ vm_exit_suspended(vcpu, pc);
+ break;
+ }
+
+ if (vcpu_debugged(vcpu)) {
+ intr_restore(daif);
+ vm_exit_debug(vcpu, pc);
+ break;
+ }
+
+ /* Activate the stage2 pmap so the vmid is valid */
+ pmap_activate_vm(pmap);
+ hyp->vttbr_el2 = pmap_to_ttbr0(pmap);
+
+ /*
+ * TODO: What happens if a timer interrupt is asserted exactly
+ * here, but for the previous VM?
+ */
+ arm64_set_active_vcpu(hypctx);
+ vgic_flush_hwstate(hypctx);
+
+ /* Call into EL2 to switch to the guest */
+ excp_type = vmm_call_hyp(HYP_ENTER_GUEST,
+ hyp->el2_addr, hypctx->el2_addr);
+
+ vgic_sync_hwstate(hypctx);
+ vtimer_sync_hwstate(hypctx);
+
+ /*
+ * Deactivate the stage2 pmap. vmm_pmap_clean_stage2_tlbi
+ * depends on this meaning we activate the VM before entering
+ * the vm again
+ */
+ PCPU_SET(curvmpmap, NULL);
+ intr_restore(daif);
+
+ vmm_stat_incr(vcpu, VMEXIT_COUNT, 1);
+ if (excp_type == EXCP_TYPE_MAINT_IRQ)
+ continue;
+
+ vme->pc = hypctx->tf.tf_elr;
+ vme->inst_length = INSN_SIZE;
+ vme->u.hyp.exception_nr = excp_type;
+ vme->u.hyp.esr_el2 = hypctx->tf.tf_esr;
+ vme->u.hyp.far_el2 = hypctx->exit_info.far_el2;
+ vme->u.hyp.hpfar_el2 = hypctx->exit_info.hpfar_el2;
+
+ handled = arm64_handle_world_switch(hypctx, excp_type, vme,
+ pmap);
+ if (handled == UNHANDLED)
+ /* Exit loop to emulate instruction. */
+ break;
+ else
+ /* Resume guest execution from the next instruction. */
+ hypctx->tf.tf_elr += vme->inst_length;
+ }
+
+ return (0);
+}
+
+static void
+arm_pcpu_vmcleanup(void *arg)
+{
+ struct hyp *hyp;
+ int i, maxcpus;
+
+ hyp = arg;
+ maxcpus = vm_get_maxcpus(hyp->vm);
+ for (i = 0; i < maxcpus; i++) {
+ if (arm64_get_active_vcpu() == hyp->ctx[i]) {
+ arm64_set_active_vcpu(NULL);
+ break;
+ }
+ }
+}
+
+void
+vmmops_vcpu_cleanup(void *vcpui)
+{
+ struct hypctx *hypctx = vcpui;
+
+ vtimer_cpucleanup(hypctx);
+ vgic_cpucleanup(hypctx);
+ free(hypctx, M_HYP);
+}
+
+void
+vmmops_cleanup(void *vmi)
+{
+ struct hyp *hyp = vmi;
+
+ vtimer_vmcleanup(hyp);
+ vgic_vmcleanup(hyp);
+
+ smp_rendezvous(NULL, arm_pcpu_vmcleanup, NULL, hyp);
+
+ /* Unmap the VM hyp struct from the hyp mode translation table */
+ vmmpmap_remove(hyp->el2_addr, roundup2(sizeof(*hyp), PAGE_SIZE),
+ true);
+
+ free(hyp, M_HYP);
+}
+
+/*
+ * Return register value. Registers have different sizes and an explicit cast
+ * must be made to ensure proper conversion.
+ */
+static void *
+hypctx_regptr(struct hypctx *hypctx, int reg)
+{
+ switch (reg) {
+ case VM_REG_GUEST_X0:
+ return (&hypctx->tf.tf_x[0]);
+ case VM_REG_GUEST_X1:
+ return (&hypctx->tf.tf_x[1]);
+ case VM_REG_GUEST_X2:
+ return (&hypctx->tf.tf_x[2]);
+ case VM_REG_GUEST_X3:
+ return (&hypctx->tf.tf_x[3]);
+ case VM_REG_GUEST_X4:
+ return (&hypctx->tf.tf_x[4]);
+ case VM_REG_GUEST_X5:
+ return (&hypctx->tf.tf_x[5]);
+ case VM_REG_GUEST_X6:
+ return (&hypctx->tf.tf_x[6]);
+ case VM_REG_GUEST_X7:
+ return (&hypctx->tf.tf_x[7]);
+ case VM_REG_GUEST_X8:
+ return (&hypctx->tf.tf_x[8]);
+ case VM_REG_GUEST_X9:
+ return (&hypctx->tf.tf_x[9]);
+ case VM_REG_GUEST_X10:
+ return (&hypctx->tf.tf_x[10]);
+ case VM_REG_GUEST_X11:
+ return (&hypctx->tf.tf_x[11]);
+ case VM_REG_GUEST_X12:
+ return (&hypctx->tf.tf_x[12]);
+ case VM_REG_GUEST_X13:
+ return (&hypctx->tf.tf_x[13]);
+ case VM_REG_GUEST_X14:
+ return (&hypctx->tf.tf_x[14]);
+ case VM_REG_GUEST_X15:
+ return (&hypctx->tf.tf_x[15]);
+ case VM_REG_GUEST_X16:
+ return (&hypctx->tf.tf_x[16]);
+ case VM_REG_GUEST_X17:
+ return (&hypctx->tf.tf_x[17]);
+ case VM_REG_GUEST_X18:
+ return (&hypctx->tf.tf_x[18]);
+ case VM_REG_GUEST_X19:
+ return (&hypctx->tf.tf_x[19]);
+ case VM_REG_GUEST_X20:
+ return (&hypctx->tf.tf_x[20]);
+ case VM_REG_GUEST_X21:
+ return (&hypctx->tf.tf_x[21]);
+ case VM_REG_GUEST_X22:
+ return (&hypctx->tf.tf_x[22]);
+ case VM_REG_GUEST_X23:
+ return (&hypctx->tf.tf_x[23]);
+ case VM_REG_GUEST_X24:
+ return (&hypctx->tf.tf_x[24]);
+ case VM_REG_GUEST_X25:
+ return (&hypctx->tf.tf_x[25]);
+ case VM_REG_GUEST_X26:
+ return (&hypctx->tf.tf_x[26]);
+ case VM_REG_GUEST_X27:
+ return (&hypctx->tf.tf_x[27]);
+ case VM_REG_GUEST_X28:
+ return (&hypctx->tf.tf_x[28]);
+ case VM_REG_GUEST_X29:
+ return (&hypctx->tf.tf_x[29]);
+ case VM_REG_GUEST_LR:
+ return (&hypctx->tf.tf_lr);
+ case VM_REG_GUEST_SP:
+ return (&hypctx->tf.tf_sp);
+ case VM_REG_GUEST_ELR: /* This is bogus */
+ return (&hypctx->tf.tf_elr);
+ case VM_REG_GUEST_SPSR: /* This is bogus */
+ return (&hypctx->tf.tf_spsr);
+ case VM_REG_ELR_EL2:
+ return (&hypctx->tf.tf_elr);
+ default:
+ break;
+ }
+ return (NULL);
+}
+
+int
+vmmops_getreg(void *vcpui, int reg, uint64_t *retval)
+{
+ void *regp;
+ int running, hostcpu;
+ struct hypctx *hypctx = vcpui;
+
+ running = vcpu_is_running(hypctx->vcpu, &hostcpu);
+ if (running && hostcpu != curcpu)
+ panic("arm_getreg: %s%d is running", vm_name(hypctx->hyp->vm),
+ vcpu_vcpuid(hypctx->vcpu));
+
+ if ((regp = hypctx_regptr(hypctx, reg)) != NULL) {
+ if (reg == VM_REG_GUEST_SPSR)
+ *retval = *(uint32_t *)regp;
+ else
+ *retval = *(uint64_t *)regp;
+ return (0);
+ } else {
+ return (EINVAL);
+ }
+}
+
+int
+vmmops_setreg(void *vcpui, int reg, uint64_t val)
+{
+ void *regp;
+ struct hypctx *hypctx = vcpui;
+ int running, hostcpu;
+
+ running = vcpu_is_running(hypctx->vcpu, &hostcpu);
+ if (running && hostcpu != curcpu)
+ panic("arm_setreg: %s%d is running", vm_name(hypctx->hyp->vm),
+ vcpu_vcpuid(hypctx->vcpu));
+
+ if ((regp = hypctx_regptr(hypctx, reg)) != NULL) {
+ if (reg == VM_REG_GUEST_SPSR)
+ *(uint32_t *)regp = (uint32_t)val;
+ else
+ *(uint64_t *)regp = val;
+ return (0);
+ } else {
+ return (EINVAL);
+ }
+}
+
+int
+vmmops_exception(void *vcpui, uint64_t esr, uint64_t far)
+{
+ struct hypctx *hypctx = vcpui;
+ int running, hostcpu;
+
+ running = vcpu_is_running(hypctx->vcpu, &hostcpu);
+ if (running && hostcpu != curcpu)
+ panic("%s: %s%d is running", __func__, vm_name(hypctx->hyp->vm),
+ vcpu_vcpuid(hypctx->vcpu));
+
+ hypctx->far_el1 = far;
+ hypctx->esr_el1 = esr;
+ hypctx->has_exception = true;
+
+ return (0);
+}
+
+int
+vmmops_getcap(void *vcpui, int num, int *retval)
+{
+ int ret;
+
+ ret = ENOENT;
+
+ switch (num) {
+ case VM_CAP_UNRESTRICTED_GUEST:
+ *retval = 1;
+ ret = 0;
+ break;
+ default:
+ break;
+ }
+
+ return (ret);
+}
+
+int
+vmmops_setcap(void *vcpui, int num, int val)
+{
+
+ return (ENOENT);
+}
diff --git a/sys/arm64/vmm/vmm_call.S b/sys/arm64/vmm/vmm_call.S
new file mode 100644
--- /dev/null
+++ b/sys/arm64/vmm/vmm_call.S
@@ -0,0 +1,38 @@
+/*
+ * Copyright (C) 2017 Alexandru Elisei <alexandru.elisei@gmail.com>
+ * All rights reserved.
+ *
+ * This software was developed by Alexandru Elisei under sponsorship
+ * from the FreeBSD Foundation.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+
+#include <machine/asm.h>
+
+ .text
+
+ENTRY(vmm_call_hyp)
+ hvc #0
+ ret
+END(vmm_call_hyp)
diff --git a/sys/arm64/vmm/vmm_dev.c b/sys/arm64/vmm/vmm_dev.c
new file mode 100644
--- /dev/null
+++ b/sys/arm64/vmm/vmm_dev.c
@@ -0,0 +1,1033 @@
+/*
+ * Copyright (C) 2015 Mihai Carabas <mihai.carabas@gmail.com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/kernel.h>
+#include <sys/jail.h>
+#include <sys/queue.h>
+#include <sys/lock.h>
+#include <sys/mutex.h>
+#include <sys/malloc.h>
+#include <sys/conf.h>
+#include <sys/sysctl.h>
+#include <sys/libkern.h>
+#include <sys/ioccom.h>
+#include <sys/mman.h>
+#include <sys/uio.h>
+#include <sys/proc.h>
+
+#include <vm/vm.h>
+#include <vm/pmap.h>
+#include <vm/vm_map.h>
+#include <vm/vm_object.h>
+
+#include <machine/machdep.h>
+#include <machine/vmparam.h>
+#include <machine/vmm.h>
+#include <machine/vmm_dev.h>
+
+#include "vmm_stat.h"
+
+#include "io/vgic.h"
+
+struct devmem_softc {
+ int segid;
+ char *name;
+ struct cdev *cdev;
+ struct vmmdev_softc *sc;
+ SLIST_ENTRY(devmem_softc) link;
+};
+
+struct vmmdev_softc {
+ struct vm *vm; /* vm instance cookie */
+ struct cdev *cdev;
+ SLIST_ENTRY(vmmdev_softc) link;
+ SLIST_HEAD(, devmem_softc) devmem;
+ int flags;
+};
+#define VSC_LINKED 0x01
+
+static SLIST_HEAD(, vmmdev_softc) head;
+
+static unsigned pr_allow_flag;
+static struct mtx vmmdev_mtx;
+
+static MALLOC_DEFINE(M_VMMDEV, "vmmdev", "vmmdev");
+
+SYSCTL_DECL(_hw_vmm);
+
+static int vmm_priv_check(struct ucred *ucred);
+static int devmem_create_cdev(const char *vmname, int id, char *devmem);
+static void devmem_destroy(void *arg);
+
+static int
+vmm_priv_check(struct ucred *ucred)
+{
+
+ if (jailed(ucred) &&
+ !(ucred->cr_prison->pr_allow & pr_allow_flag))
+ return (EPERM);
+
+ return (0);
+}
+
+static int
+vcpu_lock_one(struct vcpu *vcpu)
+{
+ int error;
+
+ error = vcpu_set_state(vcpu, VCPU_FROZEN, true);
+ return (error);
+}
+
+static void
+vcpu_unlock_one(struct vcpu *vcpu)
+{
+ enum vcpu_state state;
+
+ state = vcpu_get_state(vcpu, NULL);
+ if (state != VCPU_FROZEN) {
+ panic("vcpu %s(%d) has invalid state %d",
+ vm_name(vcpu_vm(vcpu)), vcpu_vcpuid(vcpu), state);
+ }
+
+ vcpu_set_state(vcpu, VCPU_IDLE, false);
+}
+
+static int
+vcpu_lock_all(struct vmmdev_softc *sc)
+{
+ struct vcpu *vcpu;
+ int error;
+ uint16_t i, maxcpus;
+
+ vm_slock_vcpus(sc->vm);
+ maxcpus = vm_get_maxcpus(sc->vm);
+ for (i = 0; i < maxcpus; i++) {
+ vcpu = vm_vcpu(sc->vm, i);
+ if (vcpu == NULL)
+ continue;
+ error = vcpu_lock_one(vcpu);
+ if (error)
+ break;
+ }
+
+ if (error) {
+ while (--i >= 0) {
+ vcpu = vm_vcpu(sc->vm, i);
+ if (vcpu == NULL)
+ continue;
+ vcpu_unlock_one(vcpu);
+ }
+ vm_unlock_vcpus(sc->vm);
+ }
+
+ return (error);
+}
+
+static void
+vcpu_unlock_all(struct vmmdev_softc *sc)
+{
+ struct vcpu *vcpu;
+ uint16_t i, maxcpus;
+
+ maxcpus = vm_get_maxcpus(sc->vm);
+ for (i = 0; i < maxcpus; i++) {
+ vcpu = vm_vcpu(sc->vm, i);
+ if (vcpu == NULL)
+ continue;
+ vcpu_unlock_one(vcpu);
+ }
+ vm_unlock_vcpus(sc->vm);
+}
+
+static struct vmmdev_softc *
+vmmdev_lookup(const char *name)
+{
+ struct vmmdev_softc *sc;
+
+#ifdef notyet /* XXX kernel is not compiled with invariants */
+ mtx_assert(&vmmdev_mtx, MA_OWNED);
+#endif
+
+ SLIST_FOREACH(sc, &head, link) {
+ if (strcmp(name, vm_name(sc->vm)) == 0)
+ break;
+ }
+
+ return (sc);
+}
+
+static struct vmmdev_softc *
+vmmdev_lookup2(struct cdev *cdev)
+{
+
+ return (cdev->si_drv1);
+}
+
+static int
+vmmdev_rw(struct cdev *cdev, struct uio *uio, int flags)
+{
+ int error, off, c, prot;
+ vm_paddr_t gpa, maxaddr;
+ void *hpa, *cookie;
+ struct vmmdev_softc *sc;
+
+ error = vmm_priv_check(curthread->td_ucred);
+ if (error)
+ return (error);
+
+ sc = vmmdev_lookup2(cdev);
+ if (sc == NULL)
+ return (ENXIO);
+
+ /*
+ * Get a read lock on the guest memory map.
+ */
+ vm_slock_memsegs(sc->vm);
+
+ prot = (uio->uio_rw == UIO_WRITE ? VM_PROT_WRITE : VM_PROT_READ);
+ maxaddr = vmm_sysmem_maxaddr(sc->vm);
+ while (uio->uio_resid > 0 && error == 0) {
+ gpa = uio->uio_offset;
+ off = gpa & PAGE_MASK;
+ c = min(uio->uio_resid, PAGE_SIZE - off);
+
+ /*
+ * The VM has a hole in its physical memory map. If we want to
+ * use 'dd' to inspect memory beyond the hole we need to
+ * provide bogus data for memory that lies in the hole.
+ *
+ * Since this device does not support lseek(2), dd(1) will
+ * read(2) blocks of data to simulate the lseek(2).
+ */
+ hpa = vm_gpa_hold_global(sc->vm, gpa, c, prot, &cookie);
+ if (hpa == NULL) {
+ if (uio->uio_rw == UIO_READ && gpa < maxaddr)
+ error = uiomove(__DECONST(void *, zero_region),
+ c, uio);
+ else
+ error = EFAULT;
+ } else {
+ error = uiomove(hpa, c, uio);
+ vm_gpa_release(cookie);
+ }
+ }
+ vm_unlock_memsegs(sc->vm);
+ return (error);
+}
+
+static int
+get_memseg(struct vmmdev_softc *sc, struct vm_memseg *mseg)
+{
+ struct devmem_softc *dsc;
+ int error;
+ bool sysmem;
+
+ error = vm_get_memseg(sc->vm, mseg->segid, &mseg->len, &sysmem, NULL);
+ if (error || mseg->len == 0)
+ return (error);
+
+ if (!sysmem) {
+ SLIST_FOREACH(dsc, &sc->devmem, link) {
+ if (dsc->segid == mseg->segid)
+ break;
+ }
+ KASSERT(dsc != NULL, ("%s: devmem segment %d not found",
+ __func__, mseg->segid));
+ error = copystr(dsc->name, mseg->name, sizeof(mseg->name),
+ NULL);
+ } else {
+ bzero(mseg->name, sizeof(mseg->name));
+ }
+
+ return (error);
+}
+
+static int
+alloc_memseg(struct vmmdev_softc *sc, struct vm_memseg *mseg)
+{
+ char *name;
+ int error;
+ bool sysmem;
+
+ error = 0;
+ name = NULL;
+ sysmem = true;
+
+ /*
+ * The allocation is lengthened by 1 to hold a terminating NUL. It'll
+ * by stripped off when devfs processes the full string.
+ */
+ if (VM_MEMSEG_NAME(mseg)) {
+ sysmem = false;
+ name = malloc(sizeof(mseg->name), M_VMMDEV, M_WAITOK);
+ error = copystr(mseg->name, name, sizeof(mseg->name), NULL);
+ if (error)
+ goto done;
+ }
+
+ error = vm_alloc_memseg(sc->vm, mseg->segid, mseg->len, sysmem);
+ if (error)
+ goto done;
+
+ if (VM_MEMSEG_NAME(mseg)) {
+ error = devmem_create_cdev(vm_name(sc->vm), mseg->segid, name);
+ if (error)
+ vm_free_memseg(sc->vm, mseg->segid);
+ else
+ name = NULL; /* freed when 'cdev' is destroyed */
+ }
+done:
+ free(name, M_VMMDEV);
+ return (error);
+}
+
+static int
+vm_get_register_set(struct vcpu *vcpu, unsigned int count, int *regnum,
+ uint64_t *regval)
+{
+ int error, i;
+
+ error = 0;
+ for (i = 0; i < count; i++) {
+ error = vm_get_register(vcpu, regnum[i], &regval[i]);
+ if (error)
+ break;
+ }
+ return (error);
+}
+
+static int
+vm_set_register_set(struct vcpu *vcpu, unsigned int count, int *regnum,
+ uint64_t *regval)
+{
+ int error, i;
+
+ error = 0;
+ for (i = 0; i < count; i++) {
+ error = vm_set_register(vcpu, regnum[i], regval[i]);
+ if (error)
+ break;
+ }
+ return (error);
+}
+
+static int
+vmmdev_ioctl(struct cdev *cdev, u_long cmd, caddr_t data, int fflag,
+ struct thread *td)
+{
+ int error, vcpuid, size;
+ cpuset_t *cpuset;
+ struct vmmdev_softc *sc;
+ struct vcpu *vcpu;
+ struct vm_register *vmreg;
+ struct vm_register_set *vmregset;
+ struct vm_run *vmrun;
+ struct vm_vgic_version *vgv;
+ struct vm_vgic_descr *vgic;
+ struct vm_cpuset *vm_cpuset;
+ struct vm_irq *vi;
+ struct vm_capability *vmcap;
+ struct vm_stats *vmstats;
+ struct vm_stat_desc *statdesc;
+ struct vm_suspend *vmsuspend;
+ struct vm_exception *vmexc;
+ struct vm_gla2gpa *gg;
+ struct vm_memmap *mm;
+ struct vm_msi *vmsi;
+ struct vm_cpu_topology *topology;
+ uint64_t *regvals;
+ int *regnums;
+ enum { NONE, SINGLE, ALL } vcpus_locked;
+ bool memsegs_locked;
+
+ error = vmm_priv_check(curthread->td_ucred);
+ if (error)
+ return (error);
+
+ sc = vmmdev_lookup2(cdev);
+ if (sc == NULL)
+ return (ENXIO);
+
+ error = 0;
+ vcpuid = -1;
+ vcpu = NULL;
+ vcpus_locked = NONE;
+ memsegs_locked = false;
+
+ /*
+ * Some VMM ioctls can operate only on vcpus that are not running.
+ */
+ switch (cmd) {
+ case VM_RUN:
+ case VM_GET_REGISTER:
+ case VM_SET_REGISTER:
+ case VM_GET_REGISTER_SET:
+ case VM_SET_REGISTER_SET:
+ case VM_INJECT_EXCEPTION:
+ case VM_GET_CAPABILITY:
+ case VM_SET_CAPABILITY:
+ case VM_GLA2GPA_NOFAULT:
+ case VM_ACTIVATE_CPU:
+ /*
+ * ioctls that can operate only on vcpus that are not running.
+ */
+ vcpuid = *(int *)data;
+ vcpu = vm_alloc_vcpu(sc->vm, vcpuid);
+ if (vcpu == NULL) {
+ error = EINVAL;
+ goto done;
+ }
+ error = vcpu_lock_one(vcpu);
+ if (error)
+ goto done;
+ vcpus_locked = SINGLE;
+ break;
+
+ case VM_ALLOC_MEMSEG:
+ case VM_MMAP_MEMSEG:
+ /* TODO: VM_MUNMAP_MEMSEG */
+ case VM_REINIT:
+ case VM_ATTACH_VGIC:
+ /*
+ * ioctls that modify the memory map must lock memory
+ * segments exclusively.
+ */
+ vm_xlock_memsegs(sc->vm);
+ memsegs_locked = true;
+ /* FALLTHROUGH */
+ /* TODO: Any non-memseg ioctls? */
+ /*
+ * ioctls that operate on the entire virtual machine must
+ * prevent all vcpus from running.
+ */
+ error = vcpu_lock_all(sc);
+ if (error)
+ goto done;
+ vcpus_locked = ALL;
+ break;
+ case VM_GET_MEMSEG:
+ case VM_MMAP_GETNEXT:
+ /*
+ * Lock the memory map while it is being inspected.
+ */
+ vm_slock_memsegs(sc->vm);
+ memsegs_locked = true;
+ break;
+
+ case VM_STATS:
+ /*
+ * These do not need the vCPU locked but do operate on
+ * a specific vCPU.
+ */
+ vcpuid = *(int *)data;
+ vcpu = vm_alloc_vcpu(sc->vm, vcpuid);
+ if (vcpu == NULL) {
+ error = EINVAL;
+ goto done;
+ }
+ break;
+
+ case VM_SUSPEND_CPU:
+ case VM_RESUME_CPU:
+ /*
+ * These can either operate on all CPUs via a vcpuid of
+ * -1 or on a specific vCPU.
+ */
+ vcpuid = *(int *)data;
+ if (vcpuid == -1)
+ break;
+ vcpu = vm_alloc_vcpu(sc->vm, vcpuid);
+ if (vcpu == NULL) {
+ error = EINVAL;
+ goto done;
+ }
+ break;
+
+ case VM_ASSERT_IRQ:
+ vi =(struct vm_irq *)data;
+ error = vm_assert_irq(sc->vm, vi->irq);
+ break;
+ case VM_DEASSERT_IRQ:
+ vi = (struct vm_irq *)data;
+ error = vm_deassert_irq(sc->vm, vi->irq);
+ break;
+ default:
+ break;
+ }
+
+ switch(cmd) {
+ case VM_RUN:
+ vmrun = (struct vm_run *)data;
+ error = vm_run(vcpu, &vmrun->vm_exit);
+ break;
+ case VM_SUSPEND:
+ vmsuspend = (struct vm_suspend *)data;
+ error = vm_suspend(sc->vm, vmsuspend->how);
+ break;
+ case VM_REINIT:
+ error = vm_reinit(sc->vm);
+ break;
+ case VM_STAT_DESC: {
+ statdesc = (struct vm_stat_desc *)data;
+ error = vmm_stat_desc_copy(statdesc->index,
+ statdesc->desc, sizeof(statdesc->desc));
+ break;
+ }
+ case VM_STATS: {
+ CTASSERT(MAX_VM_STATS >= MAX_VMM_STAT_ELEMS);
+ vmstats = (struct vm_stats *)data;
+ getmicrotime(&vmstats->tv);
+ error = vmm_stat_copy(vcpu, vmstats->index,
+ nitems(vmstats->statbuf),
+ &vmstats->num_entries, vmstats->statbuf);
+ break;
+ }
+ case VM_MMAP_GETNEXT:
+ mm = (struct vm_memmap *)data;
+ error = vm_mmap_getnext(sc->vm, &mm->gpa, &mm->segid,
+ &mm->segoff, &mm->len, &mm->prot, &mm->flags);
+ break;
+ case VM_MMAP_MEMSEG:
+ mm = (struct vm_memmap *)data;
+ error = vm_mmap_memseg(sc->vm, mm->gpa, mm->segid, mm->segoff,
+ mm->len, mm->prot, mm->flags);
+ break;
+ case VM_ALLOC_MEMSEG:
+ error = alloc_memseg(sc, (struct vm_memseg *)data);
+ break;
+ case VM_GET_MEMSEG:
+ error = get_memseg(sc, (struct vm_memseg *)data);
+ break;
+ case VM_GET_REGISTER:
+ vmreg = (struct vm_register *)data;
+ error = vm_get_register(vcpu, vmreg->regnum, &vmreg->regval);
+ break;
+ case VM_SET_REGISTER:
+ vmreg = (struct vm_register *)data;
+ error = vm_set_register(vcpu, vmreg->regnum, vmreg->regval);
+ break;
+ case VM_GET_REGISTER_SET:
+ vmregset = (struct vm_register_set *)data;
+ if (vmregset->count > VM_REG_LAST) {
+ error = EINVAL;
+ break;
+ }
+ regvals = malloc(sizeof(regvals[0]) * vmregset->count, M_VMMDEV,
+ M_WAITOK);
+ regnums = malloc(sizeof(regnums[0]) * vmregset->count, M_VMMDEV,
+ M_WAITOK);
+ error = copyin(vmregset->regnums, regnums, sizeof(regnums[0]) *
+ vmregset->count);
+ if (error == 0)
+ error = vm_get_register_set(vcpu, vmregset->count,
+ regnums, regvals);
+ if (error == 0)
+ error = copyout(regvals, vmregset->regvals,
+ sizeof(regvals[0]) * vmregset->count);
+ free(regvals, M_VMMDEV);
+ free(regnums, M_VMMDEV);
+ break;
+ case VM_SET_REGISTER_SET:
+ vmregset = (struct vm_register_set *)data;
+ if (vmregset->count > VM_REG_LAST) {
+ error = EINVAL;
+ break;
+ }
+ regvals = malloc(sizeof(regvals[0]) * vmregset->count, M_VMMDEV,
+ M_WAITOK);
+ regnums = malloc(sizeof(regnums[0]) * vmregset->count, M_VMMDEV,
+ M_WAITOK);
+ error = copyin(vmregset->regnums, regnums, sizeof(regnums[0]) *
+ vmregset->count);
+ if (error == 0)
+ error = copyin(vmregset->regvals, regvals,
+ sizeof(regvals[0]) * vmregset->count);
+ if (error == 0)
+ error = vm_set_register_set(vcpu, vmregset->count,
+ regnums, regvals);
+ free(regvals, M_VMMDEV);
+ free(regnums, M_VMMDEV);
+ break;
+ case VM_GET_CAPABILITY:
+ vmcap = (struct vm_capability *)data;
+ error = vm_get_capability(vcpu,
+ vmcap->captype,
+ &vmcap->capval);
+ break;
+ case VM_SET_CAPABILITY:
+ vmcap = (struct vm_capability *)data;
+ error = vm_set_capability(vcpu,
+ vmcap->captype,
+ vmcap->capval);
+ break;
+ case VM_INJECT_EXCEPTION:
+ vmexc = (struct vm_exception *)data;
+ error = vm_inject_exception(vcpu, vmexc->esr, vmexc->far);
+ break;
+ case VM_GLA2GPA_NOFAULT:
+ gg = (struct vm_gla2gpa *)data;
+ error = vm_gla2gpa_nofault(vcpu, gg->gla, gg->prot, &gg->gpa,
+ &gg->fault);
+ KASSERT(error == 0 || error == EFAULT,
+ ("%s: vm_gla2gpa unknown error %d", __func__, error));
+ break;
+ case VM_ACTIVATE_CPU:
+ error = vm_activate_cpu(vcpu);
+ break;
+ case VM_GET_CPUS:
+ error = 0;
+ vm_cpuset = (struct vm_cpuset *)data;
+ size = vm_cpuset->cpusetsize;
+ if (size < sizeof(cpuset_t) || size > CPU_MAXSIZE / NBBY) {
+ error = ERANGE;
+ break;
+ }
+ cpuset = malloc(size, M_TEMP, M_WAITOK | M_ZERO);
+ if (vm_cpuset->which == VM_ACTIVE_CPUS)
+ *cpuset = vm_active_cpus(sc->vm);
+ else if (vm_cpuset->which == VM_SUSPENDED_CPUS)
+ *cpuset = vm_suspended_cpus(sc->vm);
+ else if (vm_cpuset->which == VM_DEBUG_CPUS)
+ *cpuset = vm_debug_cpus(sc->vm);
+ else
+ error = EINVAL;
+ if (error == 0)
+ error = copyout(cpuset, vm_cpuset->cpus, size);
+ free(cpuset, M_TEMP);
+ break;
+ case VM_SUSPEND_CPU:
+ error = vm_suspend_cpu(sc->vm, vcpu);
+ break;
+ case VM_RESUME_CPU:
+ error = vm_resume_cpu(sc->vm, vcpu);
+ break;
+ case VM_GET_VGIC_VERSION:
+ vgv = (struct vm_vgic_version *)data;
+ /* TODO: Query the vgic driver for this */
+ vgv->version = 3;
+ vgv->flags = 0;
+ error = 0;
+ break;
+ case VM_ATTACH_VGIC:
+ vgic = (struct vm_vgic_descr *)data;
+ error = vm_attach_vgic(sc->vm, vgic);
+ break;
+ case VM_RAISE_MSI:
+ vmsi = (struct vm_msi *)data;
+ error = vm_raise_msi(sc->vm, vmsi->msg, vmsi->addr, vmsi->bus,
+ vmsi->slot, vmsi->func);
+ break;
+ case VM_SET_TOPOLOGY:
+ topology = (struct vm_cpu_topology *)data;
+ error = vm_set_topology(sc->vm, topology->sockets,
+ topology->cores, topology->threads, topology->maxcpus);
+ break;
+ case VM_GET_TOPOLOGY:
+ topology = (struct vm_cpu_topology *)data;
+ vm_get_topology(sc->vm, &topology->sockets, &topology->cores,
+ &topology->threads, &topology->maxcpus);
+ error = 0;
+ break;
+ default:
+ error = ENOTTY;
+ break;
+ }
+
+ if (vcpus_locked == SINGLE)
+ vcpu_unlock_one(vcpu);
+ else if (vcpus_locked == ALL)
+ vcpu_unlock_all(sc);
+ if (memsegs_locked)
+ vm_unlock_memsegs(sc->vm);
+
+done:
+ /*
+ * Make sure that no handler returns a kernel-internal
+ * error value to userspace.
+ */
+ KASSERT(error == ERESTART || error >= 0,
+ ("vmmdev_ioctl: invalid error return %d", error));
+ return (error);
+}
+
+static int
+vmmdev_mmap_single(struct cdev *cdev, vm_ooffset_t *offset, vm_size_t mapsize,
+ struct vm_object **objp, int nprot)
+{
+ struct vmmdev_softc *sc;
+ vm_paddr_t gpa;
+ size_t len;
+ vm_ooffset_t segoff, first, last;
+ int error, found, segid;
+ bool sysmem;
+
+ error = vmm_priv_check(curthread->td_ucred);
+ if (error)
+ return (error);
+
+ first = *offset;
+ last = first + mapsize;
+ if ((nprot & PROT_EXEC) || first < 0 || first >= last)
+ return (EINVAL);
+
+ sc = vmmdev_lookup2(cdev);
+ if (sc == NULL) {
+ /* virtual machine is in the process of being created */
+ return (EINVAL);
+ }
+
+ /*
+ * Get a read lock on the guest memory map.
+ */
+ vm_slock_memsegs(sc->vm);
+
+ gpa = 0;
+ found = 0;
+ while (!found) {
+ error = vm_mmap_getnext(sc->vm, &gpa, &segid, &segoff, &len,
+ NULL, NULL);
+ if (error)
+ break;
+
+ if (first >= gpa && last <= gpa + len)
+ found = 1;
+ else
+ gpa += len;
+ }
+
+ if (found) {
+ error = vm_get_memseg(sc->vm, segid, &len, &sysmem, objp);
+ KASSERT(error == 0 && *objp != NULL,
+ ("%s: invalid memory segment %d", __func__, segid));
+ if (sysmem) {
+ vm_object_reference(*objp);
+ *offset = segoff + (first - gpa);
+ } else {
+ error = EINVAL;
+ }
+ }
+ vm_unlock_memsegs(sc->vm);
+ return (error);
+}
+
+static void
+vmmdev_destroy(void *arg)
+{
+ struct vmmdev_softc *sc = arg;
+ struct devmem_softc *dsc;
+ int error __diagused;
+
+ //vm_disable_vcpu_creation(sc->vm);
+ error = vcpu_lock_all(sc);
+ KASSERT(error == 0, ("%s: error %d freezing vcpus", __func__, error));
+ vm_unlock_vcpus(sc->vm);
+
+ while ((dsc = SLIST_FIRST(&sc->devmem)) != NULL) {
+ KASSERT(dsc->cdev == NULL, ("%s: devmem not free", __func__));
+ SLIST_REMOVE_HEAD(&sc->devmem, link);
+ free(dsc->name, M_VMMDEV);
+ free(dsc, M_VMMDEV);
+ }
+
+ if (sc->cdev != NULL)
+ destroy_dev(sc->cdev);
+
+ if (sc->vm != NULL)
+ vm_destroy(sc->vm);
+
+ if ((sc->flags & VSC_LINKED) != 0) {
+ mtx_lock(&vmmdev_mtx);
+ SLIST_REMOVE(&head, sc, vmmdev_softc, link);
+ mtx_unlock(&vmmdev_mtx);
+ }
+
+ free(sc, M_VMMDEV);
+}
+
+static int
+sysctl_vmm_destroy(SYSCTL_HANDLER_ARGS)
+{
+ struct devmem_softc *dsc;
+ struct vmmdev_softc *sc;
+ struct cdev *cdev;
+ char *buf;
+ int error, buflen;
+
+ error = vmm_priv_check(req->td->td_ucred);
+ if (error)
+ return (error);
+
+ buflen = VM_MAX_NAMELEN + 1;
+ buf = malloc(buflen, M_VMMDEV, M_WAITOK | M_ZERO);
+ strlcpy(buf, "beavis", buflen);
+ error = sysctl_handle_string(oidp, buf, buflen, req);
+ if (error != 0 || req->newptr == NULL)
+ goto out;
+
+ mtx_lock(&vmmdev_mtx);
+ sc = vmmdev_lookup(buf);
+ if (sc == NULL || sc->cdev == NULL) {
+ mtx_unlock(&vmmdev_mtx);
+ error = EINVAL;
+ goto out;
+ }
+
+ /*
+ * The 'cdev' will be destroyed asynchronously when 'si_threadcount'
+ * goes down to 0 so we should not do it again in the callback.
+ *
+ * Setting 'sc->cdev' to NULL is also used to indicate that the VM
+ * is scheduled for destruction.
+ */
+ cdev = sc->cdev;
+ sc->cdev = NULL;
+ mtx_unlock(&vmmdev_mtx);
+
+ /*
+ * Schedule all cdevs to be destroyed:
+ *
+ * - any new operations on the 'cdev' will return an error (ENXIO).
+ *
+ * - when the 'si_threadcount' dwindles down to zero the 'cdev' will
+ * be destroyed and the callback will be invoked in a taskqueue
+ * context.
+ *
+ * - the 'devmem' cdevs are destroyed before the virtual machine 'cdev'
+ */
+ SLIST_FOREACH(dsc, &sc->devmem, link) {
+ KASSERT(dsc->cdev != NULL, ("devmem cdev already destroyed"));
+ destroy_dev_sched_cb(dsc->cdev, devmem_destroy, dsc);
+ }
+ destroy_dev_sched_cb(cdev, vmmdev_destroy, sc);
+ error = 0;
+
+out:
+ free(buf, M_VMMDEV);
+ return (error);
+}
+SYSCTL_PROC(_hw_vmm, OID_AUTO, destroy,
+ CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_PRISON | CTLFLAG_MPSAFE,
+ NULL, 0, sysctl_vmm_destroy, "A",
+ NULL);
+
+static struct cdevsw vmmdevsw = {
+ .d_name = "vmmdev",
+ .d_version = D_VERSION,
+ .d_ioctl = vmmdev_ioctl,
+ .d_mmap_single = vmmdev_mmap_single,
+ .d_read = vmmdev_rw,
+ .d_write = vmmdev_rw,
+};
+
+static int
+sysctl_vmm_create(SYSCTL_HANDLER_ARGS)
+{
+ struct vm *vm;
+ struct cdev *cdev;
+ struct vmmdev_softc *sc, *sc2;
+ char *buf;
+ int error, buflen;
+
+ error = vmm_priv_check(req->td->td_ucred);
+ if (error)
+ return (error);
+
+ buflen = VM_MAX_NAMELEN + 1;
+ buf = malloc(buflen, M_VMMDEV, M_WAITOK | M_ZERO);
+ strlcpy(buf, "beavis", buflen);
+ error = sysctl_handle_string(oidp, buf, buflen, req);
+ if (error != 0 || req->newptr == NULL)
+ goto out;
+
+ mtx_lock(&vmmdev_mtx);
+ sc = vmmdev_lookup(buf);
+ mtx_unlock(&vmmdev_mtx);
+ if (sc != NULL) {
+ error = EEXIST;
+ goto out;
+ }
+
+ error = vm_create(buf, &vm);
+ if (error != 0)
+ goto out;
+
+ sc = malloc(sizeof(struct vmmdev_softc), M_VMMDEV, M_WAITOK | M_ZERO);
+ sc->vm = vm;
+ SLIST_INIT(&sc->devmem);
+
+ /*
+ * Lookup the name again just in case somebody sneaked in when we
+ * dropped the lock.
+ */
+ mtx_lock(&vmmdev_mtx);
+ sc2 = vmmdev_lookup(buf);
+ if (sc2 == NULL) {
+ SLIST_INSERT_HEAD(&head, sc, link);
+ sc->flags |= VSC_LINKED;
+ }
+ mtx_unlock(&vmmdev_mtx);
+
+ if (sc2 != NULL) {
+ vmmdev_destroy(sc);
+ error = EEXIST;
+ goto out;
+ }
+
+ error = make_dev_p(MAKEDEV_CHECKNAME, &cdev, &vmmdevsw, NULL,
+ UID_ROOT, GID_WHEEL, 0600, "vmm/%s", buf);
+ if (error != 0) {
+ vmmdev_destroy(sc);
+ goto out;
+ }
+
+ mtx_lock(&vmmdev_mtx);
+ sc->cdev = cdev;
+ sc->cdev->si_drv1 = sc;
+ mtx_unlock(&vmmdev_mtx);
+
+out:
+ free(buf, M_VMMDEV);
+ return (error);
+}
+SYSCTL_PROC(_hw_vmm, OID_AUTO, create,
+ CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_PRISON | CTLFLAG_MPSAFE,
+ NULL, 0, sysctl_vmm_create, "A",
+ NULL);
+
+void
+vmmdev_init(void)
+{
+ mtx_init(&vmmdev_mtx, "vmm device mutex", NULL, MTX_DEF);
+ pr_allow_flag = prison_add_allow(NULL, "vmm", NULL,
+ "Allow use of vmm in a jail.");
+}
+
+int
+vmmdev_cleanup(void)
+{
+ int error;
+
+ if (SLIST_EMPTY(&head))
+ error = 0;
+ else
+ error = EBUSY;
+
+ return (error);
+}
+
+static int
+devmem_mmap_single(struct cdev *cdev, vm_ooffset_t *offset, vm_size_t len,
+ struct vm_object **objp, int nprot)
+{
+ struct devmem_softc *dsc;
+ vm_ooffset_t first, last;
+ size_t seglen;
+ int error;
+ bool sysmem;
+
+ dsc = cdev->si_drv1;
+ if (dsc == NULL) {
+ /* 'cdev' has been created but is not ready for use */
+ return (ENXIO);
+ }
+
+ first = *offset;
+ last = *offset + len;
+ if ((nprot & PROT_EXEC) || first < 0 || first >= last)
+ return (EINVAL);
+
+ vm_slock_memsegs(dsc->sc->vm);
+
+ error = vm_get_memseg(dsc->sc->vm, dsc->segid, &seglen, &sysmem, objp);
+ KASSERT(error == 0 && !sysmem && *objp != NULL,
+ ("%s: invalid devmem segment %d", __func__, dsc->segid));
+
+ if (seglen >= last)
+ vm_object_reference(*objp);
+ else
+ error = 0;
+ vm_unlock_memsegs(dsc->sc->vm);
+ return (error);
+}
+
+static struct cdevsw devmemsw = {
+ .d_name = "devmem",
+ .d_version = D_VERSION,
+ .d_mmap_single = devmem_mmap_single,
+};
+
+static int
+devmem_create_cdev(const char *vmname, int segid, char *devname)
+{
+ struct devmem_softc *dsc;
+ struct vmmdev_softc *sc;
+ struct cdev *cdev;
+ int error;
+
+ error = make_dev_p(MAKEDEV_CHECKNAME, &cdev, &devmemsw, NULL,
+ UID_ROOT, GID_WHEEL, 0600, "vmm.io/%s.%s", vmname, devname);
+ if (error)
+ return (error);
+
+ dsc = malloc(sizeof(struct devmem_softc), M_VMMDEV, M_WAITOK | M_ZERO);
+
+ mtx_lock(&vmmdev_mtx);
+ sc = vmmdev_lookup(vmname);
+ KASSERT(sc != NULL, ("%s: vm %s softc not found", __func__, vmname));
+ if (sc->cdev == NULL) {
+ /* virtual machine is being created or destroyed */
+ mtx_unlock(&vmmdev_mtx);
+ free(dsc, M_VMMDEV);
+ destroy_dev_sched_cb(cdev, NULL, 0);
+ return (ENODEV);
+ }
+
+ dsc->segid = segid;
+ dsc->name = devname;
+ dsc->cdev = cdev;
+ dsc->sc = sc;
+ SLIST_INSERT_HEAD(&sc->devmem, dsc, link);
+ mtx_unlock(&vmmdev_mtx);
+
+ /* The 'cdev' is ready for use after 'si_drv1' is initialized */
+ cdev->si_drv1 = dsc;
+ return (0);
+}
+
+static void
+devmem_destroy(void *arg)
+{
+ struct devmem_softc *dsc = arg;
+
+ KASSERT(dsc->cdev, ("%s: devmem cdev already destroyed", __func__));
+ dsc->cdev = NULL;
+ dsc->sc = NULL;
+}
diff --git a/sys/arm64/vmm/vmm_hyp.c b/sys/arm64/vmm/vmm_hyp.c
new file mode 100644
--- /dev/null
+++ b/sys/arm64/vmm/vmm_hyp.c
@@ -0,0 +1,750 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2021 Andrew Turner
+ *
+ * This work was supported by Innovate UK project 105694, "Digital Security
+ * by Design (DSbD) Technology Platform Prototype".
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+#include <sys/types.h>
+#include <sys/proc.h>
+
+#include <machine/armreg.h>
+
+#include "arm64.h"
+#include "hyp.h"
+
+struct hypctx;
+
+uint64_t vmm_hyp_enter(uint64_t, uint64_t, uint64_t, uint64_t, uint64_t,
+ uint64_t, uint64_t, uint64_t);
+uint64_t vmm_enter_guest(struct hypctx *);
+
+static void
+vmm_hyp_reg_store(struct hypctx *hypctx, struct hyp *hyp, bool guest)
+{
+ uint64_t dfr0;
+
+ /* Store the guest VFP registers */
+ if (guest) {
+ /* Store the timer registers */
+ hypctx->vtimer_cpu.cntkctl_el1 = READ_SPECIALREG(cntkctl_el1);
+ hypctx->vtimer_cpu.virt_timer.cntx_cval_el0 =
+ READ_SPECIALREG(cntv_cval_el0);
+ hypctx->vtimer_cpu.virt_timer.cntx_ctl_el0 =
+ READ_SPECIALREG(cntv_ctl_el0);
+
+ /* Store the GICv3 registers */
+ hypctx->vgic_v3_regs.ich_eisr_el2 =
+ READ_SPECIALREG(ich_eisr_el2);
+ hypctx->vgic_v3_regs.ich_elrsr_el2 =
+ READ_SPECIALREG(ich_elrsr_el2);
+ hypctx->vgic_v3_regs.ich_hcr_el2 =
+ READ_SPECIALREG(ich_hcr_el2);
+ hypctx->vgic_v3_regs.ich_misr_el2 =
+ READ_SPECIALREG(ich_misr_el2);
+ hypctx->vgic_v3_regs.ich_vmcr_el2 =
+ READ_SPECIALREG(ich_vmcr_el2);
+ switch(hypctx->vgic_v3_regs.ich_lr_num - 1) {
+#define STORE_LR(x) \
+ case x: \
+ hypctx->vgic_v3_regs.ich_lr_el2[x] = \
+ READ_SPECIALREG(ich_lr ## x ##_el2)
+ STORE_LR(15);
+ STORE_LR(14);
+ STORE_LR(13);
+ STORE_LR(12);
+ STORE_LR(11);
+ STORE_LR(10);
+ STORE_LR(9);
+ STORE_LR(8);
+ STORE_LR(7);
+ STORE_LR(6);
+ STORE_LR(5);
+ STORE_LR(4);
+ STORE_LR(3);
+ STORE_LR(2);
+ STORE_LR(1);
+ default:
+ STORE_LR(0);
+#undef STORE_LR
+ }
+
+ switch(hypctx->vgic_v3_regs.ich_apr_num - 1) {
+#define STORE_APR(x) \
+ case x: \
+ hypctx->vgic_v3_regs.ich_ap0r_el2[x] = \
+ READ_SPECIALREG(ich_ap0r ## x ##_el2); \
+ hypctx->vgic_v3_regs.ich_ap1r_el2[x] = \
+ READ_SPECIALREG(ich_ap1r ## x ##_el2)
+ STORE_APR(3);
+ STORE_APR(2);
+ STORE_APR(1);
+ default:
+ STORE_APR(0);
+#undef STORE_APR
+ }
+ }
+
+ dfr0 = READ_SPECIALREG(id_aa64dfr0_el1);
+ switch(ID_AA64DFR0_BRPs_VAL(dfr0) - 1) {
+#define STORE_DBG_BRP(x) \
+ case x: \
+ hypctx->dbgbcr_el1[x] = \
+ READ_SPECIALREG(dbgbcr ## x ## _el1); \
+ hypctx->dbgbvr_el1[x] = \
+ READ_SPECIALREG(dbgbvr ## x ## _el1)
+ STORE_DBG_BRP(15);
+ STORE_DBG_BRP(14);
+ STORE_DBG_BRP(13);
+ STORE_DBG_BRP(12);
+ STORE_DBG_BRP(11);
+ STORE_DBG_BRP(10);
+ STORE_DBG_BRP(9);
+ STORE_DBG_BRP(8);
+ STORE_DBG_BRP(7);
+ STORE_DBG_BRP(6);
+ STORE_DBG_BRP(5);
+ STORE_DBG_BRP(4);
+ STORE_DBG_BRP(3);
+ STORE_DBG_BRP(2);
+ STORE_DBG_BRP(1);
+ default:
+ STORE_DBG_BRP(0);
+#undef STORE_DBG_BRP
+ }
+
+ switch(ID_AA64DFR0_WRPs_VAL(dfr0) - 1) {
+#define STORE_DBG_WRP(x) \
+ case x: \
+ hypctx->dbgwcr_el1[x] = \
+ READ_SPECIALREG(dbgwcr ## x ## _el1); \
+ hypctx->dbgwvr_el1[x] = \
+ READ_SPECIALREG(dbgwvr ## x ## _el1)
+ STORE_DBG_WRP(15);
+ STORE_DBG_WRP(14);
+ STORE_DBG_WRP(13);
+ STORE_DBG_WRP(12);
+ STORE_DBG_WRP(11);
+ STORE_DBG_WRP(10);
+ STORE_DBG_WRP(9);
+ STORE_DBG_WRP(8);
+ STORE_DBG_WRP(7);
+ STORE_DBG_WRP(6);
+ STORE_DBG_WRP(5);
+ STORE_DBG_WRP(4);
+ STORE_DBG_WRP(3);
+ STORE_DBG_WRP(2);
+ STORE_DBG_WRP(1);
+ default:
+ STORE_DBG_WRP(0);
+#undef STORE_DBG_WRP
+ }
+
+ /* Store the PMU registers */
+ hypctx->pmcr_el0 = READ_SPECIALREG(pmcr_el0);
+ hypctx->pmccntr_el0 = READ_SPECIALREG(pmccntr_el0);
+ hypctx->pmccfiltr_el0 = READ_SPECIALREG(pmccfiltr_el0);
+ hypctx->pmcntenset_el0 = READ_SPECIALREG(pmcntenset_el0);
+ hypctx->pmintenset_el1 = READ_SPECIALREG(pmintenset_el1);
+ hypctx->pmovsset_el0 = READ_SPECIALREG(pmovsset_el0);
+ hypctx->pmuserenr_el0 = READ_SPECIALREG(pmuserenr_el0);
+ switch ((hypctx->pmcr_el0 & PMCR_N_MASK) >> PMCR_N_SHIFT) {
+#define STORE_PMU(x) \
+ case (x + 1): \
+ hypctx->pmevcntr_el0[x] = \
+ READ_SPECIALREG(pmevcntr ## x ## _el0); \
+ hypctx->pmevtyper_el0[x] = \
+ READ_SPECIALREG(pmevtyper ## x ## _el0)
+ STORE_PMU(30);
+ STORE_PMU(29);
+ STORE_PMU(28);
+ STORE_PMU(27);
+ STORE_PMU(26);
+ STORE_PMU(25);
+ STORE_PMU(24);
+ STORE_PMU(23);
+ STORE_PMU(22);
+ STORE_PMU(21);
+ STORE_PMU(20);
+ STORE_PMU(19);
+ STORE_PMU(18);
+ STORE_PMU(17);
+ STORE_PMU(16);
+ STORE_PMU(15);
+ STORE_PMU(14);
+ STORE_PMU(13);
+ STORE_PMU(12);
+ STORE_PMU(11);
+ STORE_PMU(10);
+ STORE_PMU(9);
+ STORE_PMU(8);
+ STORE_PMU(7);
+ STORE_PMU(6);
+ STORE_PMU(5);
+ STORE_PMU(4);
+ STORE_PMU(3);
+ STORE_PMU(2);
+ STORE_PMU(1);
+ STORE_PMU(0);
+ default: /* N == 0 when only PMCCNTR_EL0 is available */
+ break;
+#undef STORE_PMU
+ }
+
+ /* Store the special to from the trapframe */
+ hypctx->tf.tf_sp = READ_SPECIALREG(sp_el1);
+ hypctx->tf.tf_elr = READ_SPECIALREG(elr_el2);
+ hypctx->tf.tf_spsr = READ_SPECIALREG(spsr_el2);
+ if (guest) {
+ hypctx->tf.tf_esr = READ_SPECIALREG(esr_el2);
+ }
+
+ /* Store the guest special registers */
+ hypctx->elr_el1 = READ_SPECIALREG(elr_el1);
+ hypctx->sp_el0 = READ_SPECIALREG(sp_el0);
+ hypctx->tpidr_el0 = READ_SPECIALREG(tpidr_el0);
+ hypctx->tpidrro_el0 = READ_SPECIALREG(tpidrro_el0);
+ hypctx->tpidr_el1 = READ_SPECIALREG(tpidr_el1);
+ hypctx->vbar_el1 = READ_SPECIALREG(vbar_el1);
+
+ hypctx->actlr_el1 = READ_SPECIALREG(actlr_el1);
+ hypctx->afsr0_el1 = READ_SPECIALREG(afsr0_el1);
+ hypctx->afsr1_el1 = READ_SPECIALREG(afsr1_el1);
+ hypctx->amair_el1 = READ_SPECIALREG(amair_el1);
+ hypctx->contextidr_el1 = READ_SPECIALREG(contextidr_el1);
+ hypctx->cpacr_el1 = READ_SPECIALREG(cpacr_el1);
+ hypctx->csselr_el1 = READ_SPECIALREG(csselr_el1);
+ hypctx->esr_el1 = READ_SPECIALREG(esr_el1);
+ hypctx->far_el1 = READ_SPECIALREG(far_el1);
+ hypctx->mair_el1 = READ_SPECIALREG(mair_el1);
+ hypctx->mdccint_el1 = READ_SPECIALREG(mdccint_el1);
+ hypctx->mdscr_el1 = READ_SPECIALREG(mdscr_el1);
+ hypctx->par_el1 = READ_SPECIALREG(par_el1);
+ hypctx->sctlr_el1 = READ_SPECIALREG(sctlr_el1);
+ hypctx->spsr_el1 = READ_SPECIALREG(spsr_el1);
+ hypctx->tcr_el1 = READ_SPECIALREG(tcr_el1);
+ hypctx->ttbr0_el1 = READ_SPECIALREG(ttbr0_el1);
+ hypctx->ttbr1_el1 = READ_SPECIALREG(ttbr1_el1);
+
+ hypctx->cptr_el2 = READ_SPECIALREG(cptr_el2);
+ hypctx->hcr_el2 = READ_SPECIALREG(hcr_el2);
+ hypctx->vpidr_el2 = READ_SPECIALREG(vpidr_el2);
+ hypctx->vmpidr_el2 = READ_SPECIALREG(vmpidr_el2);
+}
+
+static void
+vmm_hyp_reg_restore(struct hypctx *hypctx, struct hyp *hyp, bool guest)
+{
+ uint64_t dfr0;
+
+ /* Restore the special registers */
+ WRITE_SPECIALREG(elr_el1, hypctx->elr_el1);
+ WRITE_SPECIALREG(sp_el0, hypctx->sp_el0);
+ WRITE_SPECIALREG(tpidr_el0, hypctx->tpidr_el0);
+ WRITE_SPECIALREG(tpidrro_el0, hypctx->tpidrro_el0);
+ WRITE_SPECIALREG(tpidr_el1, hypctx->tpidr_el1);
+ WRITE_SPECIALREG(vbar_el1, hypctx->vbar_el1);
+
+ WRITE_SPECIALREG(actlr_el1, hypctx->actlr_el1);
+ WRITE_SPECIALREG(afsr0_el1, hypctx->afsr0_el1);
+ WRITE_SPECIALREG(afsr1_el1, hypctx->afsr1_el1);
+ WRITE_SPECIALREG(amair_el1, hypctx->amair_el1);
+ WRITE_SPECIALREG(contextidr_el1, hypctx->contextidr_el1);
+ WRITE_SPECIALREG(cpacr_el1, hypctx->cpacr_el1);
+ WRITE_SPECIALREG(csselr_el1, hypctx->csselr_el1);
+ WRITE_SPECIALREG(esr_el1, hypctx->esr_el1);
+ WRITE_SPECIALREG(far_el1, hypctx->far_el1);
+ WRITE_SPECIALREG(mdccint_el1, hypctx->mdccint_el1);
+ WRITE_SPECIALREG(mdscr_el1, hypctx->mdscr_el1);
+ WRITE_SPECIALREG(mair_el1, hypctx->mair_el1);
+ WRITE_SPECIALREG(par_el1, hypctx->par_el1);
+ WRITE_SPECIALREG(sctlr_el1, hypctx->sctlr_el1);
+ WRITE_SPECIALREG(tcr_el1, hypctx->tcr_el1);
+ WRITE_SPECIALREG(ttbr0_el1, hypctx->ttbr0_el1);
+ WRITE_SPECIALREG(ttbr1_el1, hypctx->ttbr1_el1);
+ WRITE_SPECIALREG(spsr_el1, hypctx->spsr_el1);
+
+ WRITE_SPECIALREG(cptr_el2, hypctx->cptr_el2);
+ WRITE_SPECIALREG(hcr_el2, hypctx->hcr_el2);
+ WRITE_SPECIALREG(vpidr_el2, hypctx->vpidr_el2);
+ WRITE_SPECIALREG(vmpidr_el2, hypctx->vmpidr_el2);
+
+ /* Load the special regs from the trapframe */
+ WRITE_SPECIALREG(sp_el1, hypctx->tf.tf_sp);
+ WRITE_SPECIALREG(elr_el2, hypctx->tf.tf_elr);
+ WRITE_SPECIALREG(spsr_el2, hypctx->tf.tf_spsr);
+
+ /* Restore the PMU registers */
+ WRITE_SPECIALREG(pmcr_el0, hypctx->pmcr_el0);
+ WRITE_SPECIALREG(pmccntr_el0, hypctx->pmccntr_el0);
+ WRITE_SPECIALREG(pmccfiltr_el0, hypctx->pmccfiltr_el0);
+ /* Clear all events/interrupts then enable them */
+ WRITE_SPECIALREG(pmcntenclr_el0, 0xfffffffful);
+ WRITE_SPECIALREG(pmcntenset_el0, hypctx->pmcntenset_el0);
+ WRITE_SPECIALREG(pmintenclr_el1, 0xfffffffful);
+ WRITE_SPECIALREG(pmintenset_el1, hypctx->pmintenset_el1);
+ WRITE_SPECIALREG(pmovsclr_el0, 0xfffffffful);
+ WRITE_SPECIALREG(pmovsset_el0, hypctx->pmovsset_el0);
+
+ switch ((hypctx->pmcr_el0 & PMCR_N_MASK) >> PMCR_N_SHIFT) {
+#define LOAD_PMU(x) \
+ case (x + 1): \
+ WRITE_SPECIALREG(pmevcntr ## x ## _el0, \
+ hypctx->pmevcntr_el0[x]); \
+ WRITE_SPECIALREG(pmevtyper ## x ## _el0, \
+ hypctx->pmevtyper_el0[x])
+ LOAD_PMU(30);
+ LOAD_PMU(29);
+ LOAD_PMU(28);
+ LOAD_PMU(27);
+ LOAD_PMU(26);
+ LOAD_PMU(25);
+ LOAD_PMU(24);
+ LOAD_PMU(23);
+ LOAD_PMU(22);
+ LOAD_PMU(21);
+ LOAD_PMU(20);
+ LOAD_PMU(19);
+ LOAD_PMU(18);
+ LOAD_PMU(17);
+ LOAD_PMU(16);
+ LOAD_PMU(15);
+ LOAD_PMU(14);
+ LOAD_PMU(13);
+ LOAD_PMU(12);
+ LOAD_PMU(11);
+ LOAD_PMU(10);
+ LOAD_PMU(9);
+ LOAD_PMU(8);
+ LOAD_PMU(7);
+ LOAD_PMU(6);
+ LOAD_PMU(5);
+ LOAD_PMU(4);
+ LOAD_PMU(3);
+ LOAD_PMU(2);
+ LOAD_PMU(1);
+ LOAD_PMU(0);
+ default: /* N == 0 when only PMCCNTR_EL0 is available */
+ break;
+#undef LOAD_PMU
+ }
+
+ dfr0 = READ_SPECIALREG(id_aa64dfr0_el1);
+ switch(ID_AA64DFR0_BRPs_VAL(dfr0) - 1) {
+#define LOAD_DBG_BRP(x) \
+ case x: \
+ WRITE_SPECIALREG(dbgbcr ## x ## _el1, \
+ hypctx->dbgbcr_el1[x]); \
+ WRITE_SPECIALREG(dbgbvr ## x ## _el1, \
+ hypctx->dbgbvr_el1[x])
+ LOAD_DBG_BRP(15);
+ LOAD_DBG_BRP(14);
+ LOAD_DBG_BRP(13);
+ LOAD_DBG_BRP(12);
+ LOAD_DBG_BRP(11);
+ LOAD_DBG_BRP(10);
+ LOAD_DBG_BRP(9);
+ LOAD_DBG_BRP(8);
+ LOAD_DBG_BRP(7);
+ LOAD_DBG_BRP(6);
+ LOAD_DBG_BRP(5);
+ LOAD_DBG_BRP(4);
+ LOAD_DBG_BRP(3);
+ LOAD_DBG_BRP(2);
+ LOAD_DBG_BRP(1);
+ default:
+ LOAD_DBG_BRP(0);
+#undef LOAD_DBG_BRP
+ }
+
+ switch(ID_AA64DFR0_WRPs_VAL(dfr0) - 1) {
+#define LOAD_DBG_WRP(x) \
+ case x: \
+ WRITE_SPECIALREG(dbgwcr ## x ## _el1, \
+ hypctx->dbgwcr_el1[x]); \
+ WRITE_SPECIALREG(dbgwvr ## x ## _el1, \
+ hypctx->dbgwvr_el1[x])
+ LOAD_DBG_WRP(15);
+ LOAD_DBG_WRP(14);
+ LOAD_DBG_WRP(13);
+ LOAD_DBG_WRP(12);
+ LOAD_DBG_WRP(11);
+ LOAD_DBG_WRP(10);
+ LOAD_DBG_WRP(9);
+ LOAD_DBG_WRP(8);
+ LOAD_DBG_WRP(7);
+ LOAD_DBG_WRP(6);
+ LOAD_DBG_WRP(5);
+ LOAD_DBG_WRP(4);
+ LOAD_DBG_WRP(3);
+ LOAD_DBG_WRP(2);
+ LOAD_DBG_WRP(1);
+ default:
+ LOAD_DBG_WRP(0);
+#undef LOAD_DBG_WRP
+ }
+
+ if (guest) {
+ /* Load the timer registers */
+ WRITE_SPECIALREG(cntkctl_el1, hypctx->vtimer_cpu.cntkctl_el1);
+ WRITE_SPECIALREG(cntv_cval_el0,
+ hypctx->vtimer_cpu.virt_timer.cntx_cval_el0);
+ WRITE_SPECIALREG(cntv_ctl_el0,
+ hypctx->vtimer_cpu.virt_timer.cntx_ctl_el0);
+ WRITE_SPECIALREG(cnthctl_el2, hyp->vtimer.cnthctl_el2);
+ WRITE_SPECIALREG(cntvoff_el2, hyp->vtimer.cntvoff_el2);
+
+ /* Load the GICv3 registers */
+ WRITE_SPECIALREG(ich_hcr_el2, hypctx->vgic_v3_regs.ich_hcr_el2);
+ WRITE_SPECIALREG(ich_vmcr_el2,
+ hypctx->vgic_v3_regs.ich_vmcr_el2);
+ switch(hypctx->vgic_v3_regs.ich_lr_num - 1) {
+#define LOAD_LR(x) \
+ case x: \
+ WRITE_SPECIALREG(ich_lr ## x ##_el2, \
+ hypctx->vgic_v3_regs.ich_lr_el2[x])
+ LOAD_LR(15);
+ LOAD_LR(14);
+ LOAD_LR(13);
+ LOAD_LR(12);
+ LOAD_LR(11);
+ LOAD_LR(10);
+ LOAD_LR(9);
+ LOAD_LR(8);
+ LOAD_LR(7);
+ LOAD_LR(6);
+ LOAD_LR(5);
+ LOAD_LR(4);
+ LOAD_LR(3);
+ LOAD_LR(2);
+ LOAD_LR(1);
+ default:
+ LOAD_LR(0);
+#undef LOAD_LR
+ }
+
+ switch(hypctx->vgic_v3_regs.ich_apr_num - 1) {
+#define LOAD_APR(x) \
+ case x: \
+ WRITE_SPECIALREG(ich_ap0r ## x ##_el2, \
+ hypctx->vgic_v3_regs.ich_ap0r_el2[x]); \
+ WRITE_SPECIALREG(ich_ap1r ## x ##_el2, \
+ hypctx->vgic_v3_regs.ich_ap1r_el2[x])
+ LOAD_APR(3);
+ LOAD_APR(2);
+ LOAD_APR(1);
+ default:
+ LOAD_APR(0);
+#undef LOAD_APR
+ }
+ }
+}
+
+static uint64_t
+vmm_hyp_call_guest(struct hyp *hyp, struct hypctx *hypctx)
+{
+ struct hypctx host_hypctx;
+ uint64_t cntvoff_el2;
+ uint64_t ich_hcr_el2, ich_vmcr_el2, cnthctl_el2, cntkctl_el1;
+ uint64_t ret;
+ uint64_t s1e1r, hpfar_el2;
+ bool hpfar_valid;
+
+ vmm_hyp_reg_store(&host_hypctx, NULL, false);
+
+ /* Save the host special registers */
+ cnthctl_el2 = READ_SPECIALREG(cnthctl_el2);
+ cntkctl_el1 = READ_SPECIALREG(cntkctl_el1);
+ cntvoff_el2 = READ_SPECIALREG(cntvoff_el2);
+
+ ich_hcr_el2 = READ_SPECIALREG(ich_hcr_el2);
+ ich_vmcr_el2 = READ_SPECIALREG(ich_vmcr_el2);
+
+ vmm_hyp_reg_restore(hypctx, hyp, true);
+
+ /* Load the common hypervisor registers */
+ WRITE_SPECIALREG(vttbr_el2, hyp->vttbr_el2);
+
+ host_hypctx.mdcr_el2 = READ_SPECIALREG(mdcr_el2);
+ WRITE_SPECIALREG(mdcr_el2, hypctx->mdcr_el2);
+
+ /* Call into the guest */
+ ret = vmm_enter_guest(hypctx);
+
+ WRITE_SPECIALREG(mdcr_el2, host_hypctx.mdcr_el2);
+ isb();
+
+ /* Store the exit info */
+ hypctx->exit_info.far_el2 = READ_SPECIALREG(far_el2);
+ hpfar_valid = true;
+ if (ret == EXCP_TYPE_EL1_SYNC) {
+ switch(ESR_ELx_EXCEPTION(hypctx->tf.tf_esr)) {
+ case EXCP_INSN_ABORT_L:
+ case EXCP_DATA_ABORT_L:
+ /*
+ * The hpfar_el2 register is valid for:
+ * - Translaation and Access faults.
+ * - Translaation, Access, and permission faults on
+ * the translation table walk on the stage 1 tables.
+ * - A stage 2 Address size fault.
+ *
+ * As we only need it in the first 2 cases we can just
+ * exclude it on permission faults that are not from
+ * the stage 1 table walk.
+ *
+ * TODO: Add a case for Arm erratum 834220.
+ */
+ if ((hypctx->tf.tf_esr & ISS_DATA_S1PTW) != 0)
+ break;
+ switch(hypctx->tf.tf_esr & ISS_DATA_DFSC_MASK) {
+ case ISS_DATA_DFSC_PF_L1:
+ case ISS_DATA_DFSC_PF_L2:
+ case ISS_DATA_DFSC_PF_L3:
+ hpfar_valid = false;
+ break;
+ }
+ break;
+ }
+ }
+ if (hpfar_valid) {
+ hypctx->exit_info.hpfar_el2 = READ_SPECIALREG(hpfar_el2);
+ } else {
+ /*
+ * TODO: There is a risk the at instruction could cause an
+ * exception here. We should handle it & return a failure.
+ */
+ s1e1r =
+ arm64_address_translate_s1e1r(hypctx->exit_info.far_el2);
+ if (PAR_SUCCESS(s1e1r)) {
+ hpfar_el2 = (s1e1r & PAR_PA_MASK) >> PAR_PA_SHIFT;
+ hpfar_el2 <<= HPFAR_EL2_FIPA_SHIFT;
+ hypctx->exit_info.hpfar_el2 = hpfar_el2;
+ } else {
+ ret = EXCP_TYPE_REENTER;
+ }
+ }
+
+ vmm_hyp_reg_store(hypctx, hyp, true);
+
+ vmm_hyp_reg_restore(&host_hypctx, NULL, false);
+
+ /* Restore the host special registers */
+ WRITE_SPECIALREG(ich_hcr_el2, ich_hcr_el2);
+ WRITE_SPECIALREG(ich_vmcr_el2, ich_vmcr_el2);
+
+ WRITE_SPECIALREG(cnthctl_el2, cnthctl_el2);
+ WRITE_SPECIALREG(cntkctl_el1, cntkctl_el1);
+ WRITE_SPECIALREG(cntvoff_el2, cntvoff_el2);
+
+ return (ret);
+}
+
+static uint64_t
+vmm_hyp_read_reg(uint64_t reg)
+{
+ switch(reg) {
+ case HYP_REG_ICH_VTR:
+ return (READ_SPECIALREG(ich_vtr_el2));
+ case HYP_REG_CNTHCTL:
+ return (READ_SPECIALREG(cnthctl_el2));
+ }
+
+ return (0);
+}
+
+static bool
+vmm_is_vpipt_cache(void)
+{
+ /* TODO: Implement */
+ return (0);
+}
+
+static int
+vmm_clean_s2_tlbi(void)
+{
+ dsb(ishst);
+ __asm __volatile("tlbi alle1is");
+
+ /*
+ * If we have a VPIPT icache it will use the VMID to tag cachelines.
+ * As we are changing the allocated VMIDs we need to invalidate the
+ * icache lines containing all old values.
+ */
+ if (vmm_is_vpipt_cache())
+ __asm __volatile("ic ialluis");
+ dsb(ish);
+
+ return (0);
+}
+
+static int
+vm_s2_tlbi_range(uint64_t vttbr, vm_offset_t sva, vm_size_t eva,
+ bool final_only)
+{
+ uint64_t end, r, start;
+ uint64_t host_vttbr;
+
+#define TLBI_VA_SHIFT 12
+#define TLBI_VA_MASK ((1ul << 44) - 1)
+#define TLBI_VA(addr) (((addr) >> TLBI_VA_SHIFT) & TLBI_VA_MASK)
+#define TLBI_VA_L3_INCR (L3_SIZE >> TLBI_VA_SHIFT)
+
+ /* Switch to the guest vttbr */
+ /* TODO: Handle Cortex-A57/A72 erratum 131936 */
+ host_vttbr = READ_SPECIALREG(vttbr_el2);
+ WRITE_SPECIALREG(vttbr_el2, vttbr);
+ isb();
+
+ /*
+ * The CPU can cache the stage 1 + 2 combination so we need to ensure
+ * the stage 2 is invalidated first, then when this has completed we
+ * invalidate the stage 1 TLB. As we don't know which stage 1 virtual
+ * addresses point at the stage 2 IPA we need to invalidate the entire
+ * stage 1 TLB.
+ */
+
+ start = TLBI_VA(sva);
+ end = TLBI_VA(eva);
+ for (r = start; r < end; r += TLBI_VA_L3_INCR) {
+ /* Invalidate the stage 2 TLB entry */
+ if (final_only)
+ __asm __volatile("tlbi ipas2le1is, %0" : : "r"(r));
+ else
+ __asm __volatile("tlbi ipas2e1is, %0" : : "r"(r));
+ }
+ /* Ensure the entry has been invalidated */
+ dsb(ish);
+ /* Invalidate the stage 1 TLB. */
+ __asm __volatile("tlbi vmalle1is");
+ dsb(ish);
+ isb();
+
+ /* Switch back t othe host vttbr */
+ WRITE_SPECIALREG(vttbr_el2, host_vttbr);
+ isb();
+
+ return (0);
+}
+
+static int
+vm_s2_tlbi_all(uint64_t vttbr)
+{
+ uint64_t host_vttbr;
+
+ /* Switch to the guest vttbr */
+ /* TODO: Handle Cortex-A57/A72 erratum 131936 */
+ host_vttbr = READ_SPECIALREG(vttbr_el2);
+ WRITE_SPECIALREG(vttbr_el2, vttbr);
+ isb();
+
+ __asm __volatile("tlbi vmalls12e1is");
+ dsb(ish);
+ isb();
+
+ /* Switch back t othe host vttbr */
+ WRITE_SPECIALREG(vttbr_el2, host_vttbr);
+ isb();
+
+ return (0);
+}
+
+static int
+vmm_dc_civac(uint64_t start, uint64_t len)
+{
+ size_t line_size, end;
+ uint64_t ctr;
+
+ ctr = READ_SPECIALREG(ctr_el0);
+ line_size = sizeof(int) << CTR_DLINE_SIZE(ctr);
+ end = start + len;
+ dsb(ishst);
+ /* Clean and Invalidate the D-cache */
+ for (; start < end; start += line_size)
+ __asm __volatile("dc civac, %0" :: "r" (start) : "memory");
+ dsb(ish);
+ return (0);
+}
+
+static int
+vmm_el2_tlbi(uint64_t type, uint64_t start, uint64_t len)
+{
+ uint64_t end, r;
+
+ dsb(ishst);
+ switch (type) {
+ default:
+ case HYP_EL2_TLBI_ALL:
+ __asm __volatile("tlbi alle2" ::: "memory");
+ break;
+ case HYP_EL2_TLBI_VA:
+ end = (start + len) >> 12;
+ start >>= 12;
+ while (start < end) {
+ /* TODO: Use new macros when merged past them */
+ r = start & 0xffffffffffful;
+ __asm __volatile("tlbi vae2is, %0" :: "r"(r));
+ start += PAGE_SIZE;
+ }
+ break;
+ }
+ dsb(ish);
+
+ return (0);
+}
+
+uint64_t
+vmm_hyp_enter(uint64_t handle, uint64_t x1, uint64_t x2, uint64_t x3,
+ uint64_t x4, uint64_t x5, uint64_t x6, uint64_t x7)
+{
+ uint64_t ret;
+
+ switch (handle) {
+ case HYP_ENTER_GUEST:
+ do {
+ ret = vmm_hyp_call_guest((struct hyp *)x1,
+ (struct hypctx *)x2);
+ } while (ret == EXCP_TYPE_REENTER);
+ return (ret);
+ case HYP_READ_REGISTER:
+ return (vmm_hyp_read_reg(x1));
+ case HYP_CLEAN_S2_TLBI:
+ return (vmm_clean_s2_tlbi());
+ case HYP_DC_CIVAC:
+ return (vmm_dc_civac(x1, x2));
+ case HYP_EL2_TLBI:
+ return (vmm_el2_tlbi(x1, x2, x3));
+ case HYP_S2_TLBI_RANGE:
+ return (vm_s2_tlbi_range(x1, x2, x3, x4));
+ case HYP_S2_TLBI_ALL:
+ return (vm_s2_tlbi_all(x1));
+ case HYP_CLEANUP: /* Handled in vmm_hyp_exception.S */
+ default:
+ break;
+ }
+
+ return (0);
+}
diff --git a/sys/arm64/vmm/vmm_hyp_el2.S b/sys/arm64/vmm/vmm_hyp_el2.S
new file mode 100644
--- /dev/null
+++ b/sys/arm64/vmm/vmm_hyp_el2.S
@@ -0,0 +1,39 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2021 Andrew Turner
+ *
+ * This work was supported by Innovate UK project 105694, "Digital Security
+ * by Design (DSbD) Technology Platform Prototype".
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <machine/param.h>
+
+ .rodata
+ .align PAGE_SHIFT
+ .globl vmm_hyp_code
+vmm_hyp_code:
+ .incbin "vmm_hyp_blob.bin"
+ .globl vmm_hyp_code_end
+vmm_hyp_code_end:
diff --git a/sys/arm64/vmm/vmm_hyp_exception.S b/sys/arm64/vmm/vmm_hyp_exception.S
new file mode 100644
--- /dev/null
+++ b/sys/arm64/vmm/vmm_hyp_exception.S
@@ -0,0 +1,383 @@
+/*
+ * Copyright (C) 2017 Alexandru Elisei <alexandru.elisei@gmail.com>
+ * All rights reserved.
+ * Copyright (c) 2021 Andrew Turner
+ *
+ * This software was developed by Alexandru Elisei under sponsorship
+ * from the FreeBSD Foundation.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+
+#include <machine/asm.h>
+#include <machine/hypervisor.h>
+
+#include "assym.inc"
+#include "hyp.h"
+
+.macro save_host_registers
+ /* TODO: Only store callee saved registers */
+ sub sp, sp, #(32 * 8)
+ str x30, [sp, #(30 * 8)]
+ stp x28, x29, [sp, #(28 * 8)]
+ stp x26, x27, [sp, #(26 * 8)]
+ stp x24, x25, [sp, #(24 * 8)]
+ stp x22, x23, [sp, #(22 * 8)]
+ stp x20, x21, [sp, #(20 * 8)]
+ stp x18, x19, [sp, #(18 * 8)]
+ stp x16, x17, [sp, #(16 * 8)]
+ stp x14, x15, [sp, #(14 * 8)]
+ stp x12, x13, [sp, #(12 * 8)]
+ stp x10, x11, [sp, #(10 * 8)]
+ stp x8, x9, [sp, #(8 * 8)]
+ stp x6, x7, [sp, #(6 * 8)]
+ stp x4, x5, [sp, #(4 * 8)]
+ stp x2, x3, [sp, #(2 * 8)]
+ stp x0, x1, [sp, #(0 * 8)]
+.endm
+
+.macro restore_host_registers
+ /* TODO: Only restore callee saved registers */
+ ldp x0, x1, [sp, #(0 * 8)]
+ ldp x2, x3, [sp, #(2 * 8)]
+ ldp x4, x5, [sp, #(4 * 8)]
+ ldp x6, x7, [sp, #(6 * 8)]
+ ldp x8, x9, [sp, #(8 * 8)]
+ ldp x10, x11, [sp, #(10 * 8)]
+ ldp x12, x13, [sp, #(12 * 8)]
+ ldp x14, x15, [sp, #(14 * 8)]
+ ldp x16, x17, [sp, #(16 * 8)]
+ ldp x18, x19, [sp, #(18 * 8)]
+ ldp x20, x21, [sp, #(20 * 8)]
+ ldp x22, x23, [sp, #(22 * 8)]
+ ldp x24, x25, [sp, #(24 * 8)]
+ ldp x26, x27, [sp, #(26 * 8)]
+ ldp x28, x29, [sp, #(28 * 8)]
+ ldr x30, [sp, #(30 * 8)]
+ add sp, sp, #(32 * 8)
+.endm
+
+.macro save_guest_registers
+ /* Back up x0 so we can use it as a temporary register */
+ stp x0, x1, [sp, #-(2 * 8)]!
+
+ /* Restore the hypctx pointer */
+ mrs x0, tpidr_el2
+
+ stp x2, x3, [x0, #(TF_X + 2 * 8)]
+ stp x4, x5, [x0, #(TF_X + 4 * 8)]
+ stp x6, x7, [x0, #(TF_X + 6 * 8)]
+ stp x8, x9, [x0, #(TF_X + 8 * 8)]
+ stp x10, x11, [x0, #(TF_X + 10 * 8)]
+ stp x12, x13, [x0, #(TF_X + 12 * 8)]
+ stp x14, x15, [x0, #(TF_X + 14 * 8)]
+ stp x16, x17, [x0, #(TF_X + 16 * 8)]
+ stp x18, x19, [x0, #(TF_X + 18 * 8)]
+ stp x20, x21, [x0, #(TF_X + 20 * 8)]
+ stp x22, x23, [x0, #(TF_X + 22 * 8)]
+ stp x24, x25, [x0, #(TF_X + 24 * 8)]
+ stp x26, x27, [x0, #(TF_X + 26 * 8)]
+ stp x28, x29, [x0, #(TF_X + 28 * 8)]
+
+ str lr, [x0, #(TF_LR)]
+
+ /* Restore the saved x0 & x1 and save them */
+ ldp x2, x3, [sp], #(2 * 8)
+ stp x2, x3, [x0, #(TF_X + 0 * 8)]
+.endm
+
+.macro restore_guest_registers
+ /*
+ * Copy the guest x0 and x1 to the stack so we can restore them
+ * after loading the other registers.
+ */
+ ldp x2, x3, [x0, #(TF_X + 0 * 8)]
+ stp x2, x3, [sp, #-(2 * 8)]!
+
+ ldr lr, [x0, #(TF_LR)]
+
+ ldp x28, x29, [x0, #(TF_X + 28 * 8)]
+ ldp x26, x27, [x0, #(TF_X + 26 * 8)]
+ ldp x24, x25, [x0, #(TF_X + 24 * 8)]
+ ldp x22, x23, [x0, #(TF_X + 22 * 8)]
+ ldp x20, x21, [x0, #(TF_X + 20 * 8)]
+ ldp x18, x19, [x0, #(TF_X + 18 * 8)]
+ ldp x16, x17, [x0, #(TF_X + 16 * 8)]
+ ldp x14, x15, [x0, #(TF_X + 14 * 8)]
+ ldp x12, x13, [x0, #(TF_X + 12 * 8)]
+ ldp x10, x11, [x0, #(TF_X + 10 * 8)]
+ ldp x8, x9, [x0, #(TF_X + 8 * 8)]
+ ldp x6, x7, [x0, #(TF_X + 6 * 8)]
+ ldp x4, x5, [x0, #(TF_X + 4 * 8)]
+ ldp x2, x3, [x0, #(TF_X + 2 * 8)]
+
+ ldp x0, x1, [sp], #(2 * 8)
+.endm
+
+.macro vempty
+ .align 7
+ 1: b 1b
+.endm
+
+.macro vector name
+ .align 7
+ b handle_\name
+.endm
+
+ .section ".vmm_vectors","ax"
+ .align 11
+hyp_init_vectors:
+ vempty /* Synchronous EL2t */
+ vempty /* IRQ EL2t */
+ vempty /* FIQ EL2t */
+ vempty /* Error EL2t */
+
+ vempty /* Synchronous EL2h */
+ vempty /* IRQ EL2h */
+ vempty /* FIQ EL2h */
+ vempty /* Error EL2h */
+
+ vector hyp_init /* Synchronous 64-bit EL1 */
+ vempty /* IRQ 64-bit EL1 */
+ vempty /* FIQ 64-bit EL1 */
+ vempty /* Error 64-bit EL1 */
+
+ vempty /* Synchronous 32-bit EL1 */
+ vempty /* IRQ 32-bit EL1 */
+ vempty /* FIQ 32-bit EL1 */
+ vempty /* Error 32-bit EL1 */
+
+ .text
+ .align 11
+hyp_vectors:
+ vempty /* Synchronous EL2t */
+ vempty /* IRQ EL2t */
+ vempty /* FIQ EL2t */
+ vempty /* Error EL2t */
+
+ vector el2_el2h_sync /* Synchronous EL2h */
+ vector el2_el2h_irq /* IRQ EL2h */
+ vector el2_el2h_fiq /* FIQ EL2h */
+ vector el2_el2h_error /* Error EL2h */
+
+ vector el2_el1_sync64 /* Synchronous 64-bit EL1 */
+ vector el2_el1_irq64 /* IRQ 64-bit EL1 */
+ vector el2_el1_fiq64 /* FIQ 64-bit EL1 */
+ vector el2_el1_error64 /* Error 64-bit EL1 */
+
+ vempty /* Synchronous 32-bit EL1 */
+ vempty /* IRQ 32-bit EL1 */
+ vempty /* FIQ 32-bit EL1 */
+ vempty /* Error 32-bit EL1 */
+
+/*
+ * Initialize the hypervisor mode with a new exception vector table, translation
+ * table and stack.
+ *
+ * Expecting:
+ * x0 - translation tables physical address
+ * x1 - stack top virtual address
+ * x2 - TCR_EL2 value
+ * x3 - SCTLR_EL2 value
+ * x4 - VTCR_EL2 value
+ */
+LENTRY(handle_hyp_init)
+ /* Install the new exception vectors */
+ adrp x6, hyp_vectors
+ add x6, x6, :lo12:hyp_vectors
+ msr vbar_el2, x6
+ /* Set the stack top address */
+ mov sp, x1
+ /* Use the host VTTBR_EL2 to tell the host and the guests apart */
+ mov x9, #VTTBR_HOST
+ msr vttbr_el2, x9
+ /* Load the base address for the translation tables */
+ msr ttbr0_el2, x0
+ /* Invalidate the TLB */
+ tlbi alle2
+ /* Use the same memory attributes as EL1 */
+ mrs x9, mair_el1
+ msr mair_el2, x9
+ /* Configure address translation */
+ msr tcr_el2, x2
+ isb
+ /* Set the system control register for EL2 */
+ msr sctlr_el2, x3
+ /* Set the Stage 2 translation control register */
+ msr vtcr_el2, x4
+ /* Return success */
+ mov x0, #0
+ /* MMU is up and running */
+ ERET
+LEND(handle_hyp_init)
+
+.macro do_world_switch_to_host
+ save_guest_registers
+ restore_host_registers
+
+ /* Restore host VTTBR */
+ mov x9, #VTTBR_HOST
+ msr vttbr_el2, x9
+.endm
+
+
+.macro handle_el2_excp type
+ /* Save registers before modifying so we can restore them */
+ str x9, [sp, #-16]!
+
+ /* Test if the exception happened when the host was running */
+ mrs x9, vttbr_el2
+ cmp x9, #VTTBR_HOST
+ beq 1f
+
+ /* We got the exception while the guest was running */
+ ldr x9, [sp], #16
+ do_world_switch_to_host
+ mov x0, \type
+ ret
+
+1:
+ /* We got the exception while the host was running */
+ ldr x9, [sp], #16
+ mov x0, \type
+ eret
+.endm
+
+
+LENTRY(handle_el2_el2h_sync)
+ handle_el2_excp #EXCP_TYPE_EL2_SYNC
+LEND(handle_el2_el2h_sync)
+
+LENTRY(handle_el2_el2h_irq)
+ handle_el2_excp #EXCP_TYPE_EL2_IRQ
+LEND(handle_el2_el2h_irq)
+
+LENTRY(handle_el2_el2h_fiq)
+ handle_el2_excp #EXCP_TYPE_EL2_FIQ
+LEND(handle_el2_el2h_fiq)
+
+LENTRY(handle_el2_el2h_error)
+ handle_el2_excp #EXCP_TYPE_EL2_ERROR
+LEND(handle_el2_el2h_error)
+
+
+LENTRY(handle_el2_el1_sync64)
+ /* Save registers before modifying so we can restore them */
+ str x9, [sp, #-16]!
+
+ /* Check for host hypervisor call */
+ mrs x9, vttbr_el2
+ cmp x9, #VTTBR_HOST
+ ldr x9, [sp], #16 /* Restore the temp register */
+ bne 1f
+
+ /*
+ * Called from the host
+ */
+
+ /* Check if this is a cleanup call and handle in a controlled state */
+ cmp x0, #(HYP_CLEANUP)
+ b.eq vmm_cleanup
+
+ str lr, [sp, #-16]!
+ bl vmm_hyp_enter
+ ldr lr, [sp], #16
+ ERET
+
+1: /* Guest exception taken to EL2 */
+ do_world_switch_to_host
+ mov x0, #EXCP_TYPE_EL1_SYNC
+ ret
+LEND(handle_el2_el1_sync64)
+
+/*
+ * We only trap IRQ, FIQ and SError exceptions when a guest is running. Do a
+ * world switch to host to handle these exceptions.
+ */
+
+LENTRY(handle_el2_el1_irq64)
+ do_world_switch_to_host
+ str x9, [sp, #-16]!
+ mrs x9, ich_misr_el2
+ cmp x9, xzr
+ beq 1f
+ mov x0, #EXCP_TYPE_MAINT_IRQ
+ b 2f
+1:
+ mov x0, #EXCP_TYPE_EL1_IRQ
+2:
+ ldr x9, [sp], #16
+ ret
+LEND(handle_el2_el1_irq)
+
+LENTRY(handle_el2_el1_fiq64)
+ do_world_switch_to_host
+ mov x0, #EXCP_TYPE_EL1_FIQ
+ ret
+LEND(handle_el2_el1_fiq64)
+
+LENTRY(handle_el2_el1_error64)
+ do_world_switch_to_host
+ mov x0, #EXCP_TYPE_EL1_ERROR
+ ret
+LEND(handle_el2_el1_error64)
+
+
+/*
+ * Usage:
+ * uint64_t vmm_enter_guest(struct hypctx *hypctx)
+ *
+ * Expecting:
+ * x0 - hypctx address
+ */
+ENTRY(vmm_enter_guest)
+ /* Save hypctx address */
+ msr tpidr_el2, x0
+
+ save_host_registers
+ restore_guest_registers
+
+ /* Enter guest */
+ eret
+END(vmm_enter_guest)
+
+/*
+ * Usage:
+ * void vmm_cleanup(uint64_t handle, void *hyp_stub_vectors)
+ *
+ * Expecting:
+ * x1 - physical address of hyp_stub_vectors
+ */
+LENTRY(vmm_cleanup)
+ /* Restore the stub vectors */
+ msr vbar_el2, x1
+
+ /* Disable the MMU */
+ dsb sy
+ mrs x2, sctlr_el2
+ bic x2, x2, #SCTLR_EL2_M
+ msr sctlr_el2, x2
+ isb
+
+ ERET
+LEND(vmm_cleanup)
diff --git a/sys/arm64/vmm/vmm_instruction_emul.c b/sys/arm64/vmm/vmm_instruction_emul.c
new file mode 100644
--- /dev/null
+++ b/sys/arm64/vmm/vmm_instruction_emul.c
@@ -0,0 +1,98 @@
+/*
+ * Copyright (C) 2015 Mihai Carabas <mihai.carabas@gmail.com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifdef _KERNEL
+#include <sys/param.h>
+#include <sys/pcpu.h>
+#include <sys/systm.h>
+#include <sys/proc.h>
+
+#include <vm/vm.h>
+
+#include <machine/machdep.h>
+#include <machine/vmm.h>
+
+#else
+#include <sys/types.h>
+#include <sys/errno.h>
+#include <sys/_iovec.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#include <machine/vmm.h>
+
+#include <assert.h>
+#include <vmmapi.h>
+#endif
+
+#include <machine/vmm_instruction_emul.h>
+
+int
+vmm_emulate_instruction(struct vcpu *vcpu, uint64_t gpa, struct vie *vie,
+ struct vm_guest_paging *paging __unused, mem_region_read_t memread,
+ mem_region_write_t memwrite, void *memarg)
+{
+ uint64_t val;
+ int error;
+
+ if (vie->dir == VM_DIR_READ) {
+ error = memread(vcpu, gpa, &val, vie->access_size, memarg);
+ if (error)
+ goto out;
+ error = vm_set_register(vcpu, vie->reg, val);
+ } else {
+ error = vm_get_register(vcpu, vie->reg, &val);
+ if (error)
+ goto out;
+ error = memwrite(vcpu, gpa, val, vie->access_size, memarg);
+ }
+
+out:
+ return (error);
+}
+
+int
+vmm_emulate_register(struct vcpu *vcpu, struct vre *vre, reg_read_t regread,
+ reg_write_t regwrite, void *regarg)
+{
+ uint64_t val;
+ int error;
+
+ if (vre->dir == VM_DIR_READ) {
+ error = regread(vcpu, &val, regarg);
+ if (error)
+ goto out;
+ error = vm_set_register(vcpu, vre->reg, val);
+ } else {
+ error = vm_get_register(vcpu, vre->reg, &val);
+ if (error)
+ goto out;
+ error = regwrite(vcpu, val, regarg);
+ }
+
+out:
+ return (error);
+}
diff --git a/sys/arm64/vmm/vmm_ktr.h b/sys/arm64/vmm/vmm_ktr.h
new file mode 100644
--- /dev/null
+++ b/sys/arm64/vmm/vmm_ktr.h
@@ -0,0 +1,71 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
+ *
+ * Copyright (c) 2011 NetApp, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _VMM_KTR_H_
+#define _VMM_KTR_H_
+
+#include <sys/ktr.h>
+#include <sys/pcpu.h>
+
+#ifndef KTR_VMM
+#define KTR_VMM KTR_GEN
+#endif
+
+#define VCPU_CTR0(vm, vcpuid, format) \
+CTR2(KTR_VMM, "vm %s[%d]: " format, vm_name((vm)), (vcpuid))
+
+#define VCPU_CTR1(vm, vcpuid, format, p1) \
+CTR3(KTR_VMM, "vm %s[%d]: " format, vm_name((vm)), (vcpuid), (p1))
+
+#define VCPU_CTR2(vm, vcpuid, format, p1, p2) \
+CTR4(KTR_VMM, "vm %s[%d]: " format, vm_name((vm)), (vcpuid), (p1), (p2))
+
+#define VCPU_CTR3(vm, vcpuid, format, p1, p2, p3) \
+CTR5(KTR_VMM, "vm %s[%d]: " format, vm_name((vm)), (vcpuid), (p1), (p2), (p3))
+
+#define VCPU_CTR4(vm, vcpuid, format, p1, p2, p3, p4) \
+CTR6(KTR_VMM, "vm %s[%d]: " format, vm_name((vm)), (vcpuid), \
+ (p1), (p2), (p3), (p4))
+
+#define VM_CTR0(vm, format) \
+CTR1(KTR_VMM, "vm %s: " format, vm_name((vm)))
+
+#define VM_CTR1(vm, format, p1) \
+CTR2(KTR_VMM, "vm %s: " format, vm_name((vm)), (p1))
+
+#define VM_CTR2(vm, format, p1, p2) \
+CTR3(KTR_VMM, "vm %s: " format, vm_name((vm)), (p1), (p2))
+
+#define VM_CTR3(vm, format, p1, p2, p3) \
+CTR4(KTR_VMM, "vm %s: " format, vm_name((vm)), (p1), (p2), (p3))
+
+#define VM_CTR4(vm, format, p1, p2, p3, p4) \
+CTR5(KTR_VMM, "vm %s: " format, vm_name((vm)), (p1), (p2), (p3), (p4))
+#endif
diff --git a/sys/arm64/vmm/vmm_mem.h b/sys/arm64/vmm/vmm_mem.h
new file mode 100644
--- /dev/null
+++ b/sys/arm64/vmm/vmm_mem.h
@@ -0,0 +1,43 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
+ *
+ * Copyright (c) 2011 NetApp, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _VMM_MEM_H_
+#define _VMM_MEM_H_
+
+struct vmspace;
+struct vm_object;
+
+int vmm_mem_init(void);
+struct vm_object *vmm_mmio_alloc(struct vmspace *, vm_paddr_t gpa, size_t len,
+ vm_paddr_t hpa);
+void vmm_mmio_free(struct vmspace *, vm_paddr_t gpa, size_t size);
+vm_paddr_t vmm_mem_maxaddr(void);
+
+#endif
diff --git a/sys/arm64/vmm/vmm_mem.c b/sys/arm64/vmm/vmm_mem.c
new file mode 100644
--- /dev/null
+++ b/sys/arm64/vmm/vmm_mem.c
@@ -0,0 +1,124 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
+ *
+ * Copyright (c) 2011 NetApp, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/malloc.h>
+#include <sys/sglist.h>
+#include <sys/lock.h>
+#include <sys/rwlock.h>
+
+#include <vm/vm.h>
+#include <vm/vm_param.h>
+#include <vm/pmap.h>
+#include <vm/vm_map.h>
+#include <vm/vm_object.h>
+#include <vm/vm_page.h>
+#include <vm/vm_pager.h>
+
+#include <machine/md_var.h>
+
+#include "vmm_mem.h"
+
+int
+vmm_mem_init(void)
+{
+
+ return (0);
+}
+
+vm_object_t
+vmm_mmio_alloc(struct vmspace *vmspace, vm_paddr_t gpa, size_t len,
+ vm_paddr_t hpa)
+{
+ int error;
+ vm_object_t obj;
+ struct sglist *sg;
+
+ sg = sglist_alloc(1, M_WAITOK);
+ error = sglist_append_phys(sg, hpa, len);
+ KASSERT(error == 0, ("error %d appending physaddr to sglist", error));
+
+ obj = vm_pager_allocate(OBJT_SG, sg, len, VM_PROT_RW, 0, NULL);
+ if (obj != NULL) {
+ /*
+ * VT-x ignores the MTRR settings when figuring out the
+ * memory type for translations obtained through EPT.
+ *
+ * Therefore we explicitly force the pages provided by
+ * this object to be mapped as uncacheable.
+ */
+ VM_OBJECT_WLOCK(obj);
+ error = vm_object_set_memattr(obj, VM_MEMATTR_UNCACHEABLE);
+ VM_OBJECT_WUNLOCK(obj);
+ if (error != KERN_SUCCESS) {
+ panic("vmm_mmio_alloc: vm_object_set_memattr error %d",
+ error);
+ }
+ error = vm_map_find(&vmspace->vm_map, obj, 0, &gpa, len, 0,
+ VMFS_NO_SPACE, VM_PROT_RW, VM_PROT_RW, 0);
+ if (error != KERN_SUCCESS) {
+ vm_object_deallocate(obj);
+ obj = NULL;
+ }
+ }
+
+ /*
+ * Drop the reference on the sglist.
+ *
+ * If the scatter/gather object was successfully allocated then it
+ * has incremented the reference count on the sglist. Dropping the
+ * initial reference count ensures that the sglist will be freed
+ * when the object is deallocated.
+ *
+ * If the object could not be allocated then we end up freeing the
+ * sglist.
+ */
+ sglist_free(sg);
+
+ return (obj);
+}
+
+void
+vmm_mmio_free(struct vmspace *vmspace, vm_paddr_t gpa, size_t len)
+{
+
+ vm_map_remove(&vmspace->vm_map, gpa, gpa + len);
+}
+
+vm_paddr_t
+vmm_mem_maxaddr(void)
+{
+
+ return (ptoa(Maxmem));
+}
diff --git a/sys/arm64/vmm/vmm_mmu.c b/sys/arm64/vmm/vmm_mmu.c
new file mode 100644
--- /dev/null
+++ b/sys/arm64/vmm/vmm_mmu.c
@@ -0,0 +1,432 @@
+/*
+ * Copyright (C) 2017 Alexandru Elisei <alexandru.elisei@gmail.com>
+ * All rights reserved.
+ *
+ * This software was developed by Alexandru Elisei under sponsorship
+ * from the FreeBSD Foundation.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+
+#include <sys/types.h>
+#include <sys/malloc.h>
+#include <sys/lock.h>
+#include <sys/mutex.h>
+#include <vm/vm.h>
+#include <vm/pmap.h>
+#include <vm/vm_page.h>
+#include <vm/vm_param.h>
+#include <vm/vm_phys.h>
+
+#include <machine/atomic.h>
+#include <machine/machdep.h>
+#include <machine/vm.h>
+#include <machine/vmm.h>
+#include <machine/vmparam.h>
+
+#include "mmu.h"
+#include "arm64.h"
+
+MALLOC_DECLARE(M_HYP);
+
+static struct mtx vmmpmap_mtx;
+static pt_entry_t *l0;
+static vm_paddr_t l0_paddr;
+
+bool
+vmmpmap_init(void)
+{
+ vm_page_t m;
+
+ m = vm_page_alloc_noobj(VM_ALLOC_WIRED | VM_ALLOC_ZERO);
+ if (m == NULL)
+ return (false);
+
+ l0_paddr = VM_PAGE_TO_PHYS(m);
+ l0 = (pd_entry_t *)PHYS_TO_DMAP(l0_paddr);
+ memset(l0, 0, PAGE_SIZE);
+
+ mtx_init(&vmmpmap_mtx, "vmm pmap", NULL, MTX_DEF);
+
+ return (true);
+}
+
+static void
+vmmpmap_release_l3(pd_entry_t l2e)
+{
+ pt_entry_t *l3 __diagused;
+ vm_page_t m;
+ int i;
+
+ l3 = (pd_entry_t *)PHYS_TO_DMAP(l2e & ~ATTR_MASK);
+ for (i = 0; i < Ln_ENTRIES; i++) {
+ KASSERT(l3[i] == 0, ("%s: l3 still mapped: %p %lx", __func__,
+ &l3[i], l3[i]));
+ }
+
+ m = PHYS_TO_VM_PAGE(l2e & ~ATTR_MASK);
+ vm_page_unwire_noq(m);
+ vm_page_free(m);
+}
+
+static void
+vmmpmap_release_l2(pd_entry_t l1e)
+{
+ pt_entry_t *l2;
+ vm_page_t m;
+ int i;
+
+ l2 = (pd_entry_t *)PHYS_TO_DMAP(l1e & ~ATTR_MASK);
+ for (i = 0; i < Ln_ENTRIES; i++) {
+ if (l2[i] != 0) {
+ vmmpmap_release_l3(l2[i]);
+ }
+ }
+
+ m = PHYS_TO_VM_PAGE(l1e & ~ATTR_MASK);
+ vm_page_unwire_noq(m);
+ vm_page_free(m);
+}
+
+static void
+vmmpmap_release_l1(pd_entry_t l0e)
+{
+ pt_entry_t *l1;
+ vm_page_t m;
+ int i;
+
+ l1 = (pd_entry_t *)PHYS_TO_DMAP(l0e & ~ATTR_MASK);
+ for (i = 0; i < Ln_ENTRIES; i++) {
+ if (l1[i] != 0) {
+ vmmpmap_release_l2(l1[i]);
+ }
+ }
+
+ m = PHYS_TO_VM_PAGE(l0e & ~ATTR_MASK);
+ vm_page_unwire_noq(m);
+ vm_page_free(m);
+}
+
+void
+vmmpmap_fini(void)
+{
+ vm_page_t m;
+ int i;
+
+ /* Remove the remaining entries */
+ for (i = 0; i < L0_ENTRIES; i++) {
+ if (l0[i] != 0) {
+ vmmpmap_release_l1(l0[i]);
+ }
+ }
+
+ m = PHYS_TO_VM_PAGE(l0_paddr);
+ vm_page_unwire_noq(m);
+ vm_page_free(m);
+
+ mtx_destroy(&vmmpmap_mtx);
+}
+
+uint64_t
+vmmpmap_to_ttbr0(void)
+{
+
+ return (l0_paddr);
+}
+
+/* Returns a pointer to the level 1 table, allocating if needed. */
+static pt_entry_t *
+vmmpmap_l1_table(vm_offset_t va)
+{
+ pt_entry_t new_l0e, l0e, *l1;
+ vm_page_t m;
+ int rv;
+
+ m = NULL;
+again:
+ l0e = atomic_load_64(&l0[pmap_l0_index(va)]);
+ if ((l0e & ATTR_DESCR_VALID) == 0) {
+ /* Allocate a page for the level 1 table */
+ if (m == NULL) {
+ m = vm_page_alloc_noobj(VM_ALLOC_WIRED | VM_ALLOC_ZERO);
+ if (m == NULL)
+ return (NULL);
+ }
+
+ new_l0e = VM_PAGE_TO_PHYS(m) | L0_TABLE;
+
+ mtx_lock(&vmmpmap_mtx);
+ rv = atomic_cmpset_64(&l0[pmap_l0_index(va)], l0e, new_l0e);
+ mtx_unlock(&vmmpmap_mtx);
+ /* We may have raced another thread, try again */
+ if (rv == 0)
+ goto again;
+
+ /* The cmpset succeeded */
+ l0e = new_l0e;
+ } else if (m != NULL) {
+ /* We allocated a page that wasn't used */
+ vm_page_unwire_noq(m);
+ vm_page_free_zero(m);
+ }
+
+ l1 = (pd_entry_t *)PHYS_TO_DMAP(l0e & ~ATTR_MASK);
+ return (l1);
+}
+
+static pt_entry_t *
+vmmpmap_l2_table(vm_offset_t va)
+{
+ pt_entry_t new_l1e, l1e, *l1, *l2;
+ vm_page_t m;
+ int rv;
+
+ l1 = vmmpmap_l1_table(va);
+ if (l1 == NULL)
+ return (NULL);
+
+ m = NULL;
+again:
+ l1e = atomic_load_64(&l1[pmap_l1_index(va)]);
+ if ((l1e & ATTR_DESCR_VALID) == 0) {
+ /* Allocate a page for the level 2 table */
+ if (m == NULL) {
+ m = vm_page_alloc_noobj(VM_ALLOC_WIRED | VM_ALLOC_ZERO);
+ if (m == NULL)
+ return (NULL);
+ }
+
+ new_l1e = VM_PAGE_TO_PHYS(m) | L1_TABLE;
+
+ mtx_lock(&vmmpmap_mtx);
+ rv = atomic_cmpset_64(&l1[pmap_l1_index(va)], l1e, new_l1e);
+ mtx_unlock(&vmmpmap_mtx);
+ /* We may have raced another thread, try again */
+ if (rv == 0)
+ goto again;
+
+ /* The cmpset succeeded */
+ l1e = new_l1e;
+ } else if (m != NULL) {
+ /* We allocated a page that wasn't used */
+ vm_page_unwire_noq(m);
+ vm_page_free_zero(m);
+ }
+
+ l2 = (pd_entry_t *)PHYS_TO_DMAP(l1e & ~ATTR_MASK);
+ return (l2);
+}
+
+static pd_entry_t *
+vmmpmap_l3_table(vm_offset_t va)
+{
+ pt_entry_t new_l2e, l2e, *l2, *l3;
+ vm_page_t m;
+ int rv;
+
+ l2 = vmmpmap_l2_table(va);
+ if (l2 == NULL)
+ return (NULL);
+
+ m = NULL;
+again:
+ l2e = atomic_load_64(&l2[pmap_l2_index(va)]);
+ if ((l2e & ATTR_DESCR_VALID) == 0) {
+ /* Allocate a page for the level 3 table */
+ if (m == NULL) {
+ m = vm_page_alloc_noobj(VM_ALLOC_WIRED | VM_ALLOC_ZERO);
+ if (m == NULL)
+ return (NULL);
+ }
+
+ new_l2e = VM_PAGE_TO_PHYS(m) | L2_TABLE;
+
+ mtx_lock(&vmmpmap_mtx);
+ rv = atomic_cmpset_64(&l2[pmap_l2_index(va)], l2e, new_l2e);
+ mtx_unlock(&vmmpmap_mtx);
+ /* We may have raced another thread, try again */
+ if (rv == 0)
+ goto again;
+
+ /* The cmpset succeeded */
+ l2e = new_l2e;
+ } else if (m != NULL) {
+ /* We allocated a page that wasn't used */
+ vm_page_unwire_noq(m);
+ vm_page_free_zero(m);
+ }
+
+ l3 = (pt_entry_t *)PHYS_TO_DMAP(l2e & ~ATTR_MASK);
+ return (l3);
+}
+
+/*
+ * Creates an EL2 entry in the hyp_pmap. Similar to pmap_kenter.
+ */
+bool
+vmmpmap_enter(vm_offset_t va, vm_size_t size, vm_paddr_t pa, vm_prot_t prot)
+{
+ pd_entry_t l3e, *l3;
+
+ KASSERT((pa & L3_OFFSET) == 0,
+ ("%s: Invalid physical address", __func__));
+ KASSERT((va & L3_OFFSET) == 0,
+ ("%s: Invalid virtual address", __func__));
+ KASSERT((size & PAGE_MASK) == 0,
+ ("%s: Mapping is not page-sized", __func__));
+
+ l3e = ATTR_DEFAULT | L3_PAGE;
+ /* This bit is res1 at EL2 */
+ l3e |= ATTR_S1_AP(ATTR_S1_AP_USER);
+ /* Only normal memory is used at EL2 */
+ l3e |= ATTR_S1_IDX(VM_MEMATTR_DEFAULT);
+
+ if ((prot & VM_PROT_EXECUTE) == 0) {
+ /* PXN is res0 at EL2. UXN is XN */
+ l3e |= ATTR_S1_UXN;
+ }
+ if ((prot & VM_PROT_WRITE) == 0) {
+ l3e |= ATTR_S1_AP(ATTR_S1_AP_RO);
+ }
+
+ while (size > 0) {
+ l3 = vmmpmap_l3_table(va);
+ if (l3 == NULL)
+ return (false);
+
+#ifdef INVARIANTS
+ /*
+ * Ensure no other threads can write to l3 between the KASSERT
+ * and store.
+ */
+ mtx_lock(&vmmpmap_mtx);
+#endif
+ KASSERT(atomic_load_64(&l3[pmap_l3_index(va)]) == 0,
+ ("%s: VA already mapped", __func__));
+
+ atomic_store_64(&l3[pmap_l3_index(va)], l3e | pa);
+#ifdef INVARIANTS
+ mtx_unlock(&vmmpmap_mtx);
+#endif
+
+ size -= PAGE_SIZE;
+ pa += PAGE_SIZE;
+ va += PAGE_SIZE;
+ }
+
+ return (true);
+}
+
+void
+vmmpmap_remove(vm_offset_t va, vm_size_t size, bool invalidate)
+{
+ pt_entry_t l0e, *l1, l1e, *l2, l2e;
+ pd_entry_t *l3, l3e, **l3_list;
+ vm_offset_t eva, va_next, sva;
+ size_t i;
+
+ KASSERT((va & L3_OFFSET) == 0,
+ ("%s: Invalid virtual address", __func__));
+ KASSERT((size & PAGE_MASK) == 0,
+ ("%s: Mapping is not page-sized", __func__));
+
+ if (invalidate) {
+ l3_list = malloc((size / PAGE_SIZE) * sizeof(l3_list[0]),
+ M_TEMP, M_WAITOK | M_ZERO);
+ }
+
+ sva = va;
+ eva = va + size;
+ mtx_lock(&vmmpmap_mtx);
+ for (i = 0; va < eva; va = va_next) {
+ l0e = atomic_load_64(&l0[pmap_l0_index(va)]);
+ if (l0e == 0) {
+ va_next = (va + L0_SIZE) & ~L0_OFFSET;
+ if (va_next < va)
+ va_next = eva;
+ continue;
+ }
+ MPASS((l0e & ATTR_DESCR_MASK) == L0_TABLE);
+
+ l1 = (pd_entry_t *)PHYS_TO_DMAP(l0e & ~ATTR_MASK);
+ l1e = atomic_load_64(&l1[pmap_l1_index(va)]);
+ if (l1e == 0) {
+ va_next = (va + L1_SIZE) & ~L1_OFFSET;
+ if (va_next < va)
+ va_next = eva;
+ continue;
+ }
+ MPASS((l1e & ATTR_DESCR_MASK) == L1_TABLE);
+
+ l2 = (pd_entry_t *)PHYS_TO_DMAP(l1e & ~ATTR_MASK);
+ l2e = atomic_load_64(&l2[pmap_l2_index(va)]);
+ if (l2e == 0) {
+ va_next = (va + L2_SIZE) & ~L2_OFFSET;
+ if (va_next < va)
+ va_next = eva;
+ continue;
+ }
+ MPASS((l2e & ATTR_DESCR_MASK) == L2_TABLE);
+
+ l3 = (pd_entry_t *)PHYS_TO_DMAP(l2e & ~ATTR_MASK);
+ if (invalidate) {
+ l3e = atomic_load_64(&l3[pmap_l3_index(va)]);
+ MPASS(l3e != 0);
+ /*
+ * Mark memory as read-only so we can invalidate
+ * the cache.
+ */
+ l3e &= ~ATTR_S1_AP_MASK;
+ l3e |= ATTR_S1_AP(ATTR_S1_AP_RO);
+ atomic_store_64(&l3[pmap_l3_index(va)], l3e);
+
+ l3_list[i] = &l3[pmap_l3_index(va)];
+ i++;
+ } else {
+ /*
+ * The caller is responsible for clearing the cache &
+ * handling the TLB
+ */
+ atomic_store_64(&l3[pmap_l3_index(va)], 0);
+ }
+
+ va_next = (va + L3_SIZE) & ~L3_OFFSET;
+ if (va_next < va)
+ va_next = eva;
+ }
+ mtx_unlock(&vmmpmap_mtx);
+
+ if (invalidate) {
+ /* Invalidate the memory from the D-cache */
+ vmm_call_hyp(HYP_DC_CIVAC, sva, size);
+
+ for (i = 0; i < (size / PAGE_SIZE); i++) {
+ atomic_store_64(l3_list[i], 0);
+ }
+
+ vmm_call_hyp(HYP_EL2_TLBI, HYP_EL2_TLBI_VA, sva, size);
+
+ free(l3_list, M_TEMP);
+ }
+}
diff --git a/sys/arm64/vmm/vmm_reset.c b/sys/arm64/vmm/vmm_reset.c
new file mode 100644
--- /dev/null
+++ b/sys/arm64/vmm/vmm_reset.c
@@ -0,0 +1,176 @@
+/*
+ * Copyright (C) 2018 Alexandru Elisei <alexandru.elisei@gmail.com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+#include <sys/types.h>
+#include <sys/systm.h>
+#include <sys/kernel.h>
+#include <sys/lock.h>
+
+#include <machine/armreg.h>
+#include <machine/cpu.h>
+#include <machine/hypervisor.h>
+
+#include "arm64.h"
+#include "reset.h"
+
+/*
+ * Make the architecturally UNKNOWN value 0. As a bonus, we don't have to
+ * manually set all those RES0 fields.
+ */
+#define ARCH_UNKNOWN 0
+#define set_arch_unknown(reg) (memset(&(reg), ARCH_UNKNOWN, sizeof(reg)))
+
+void
+reset_vm_el01_regs(void *vcpu)
+{
+ struct hypctx *el2ctx;
+
+ el2ctx = vcpu;
+
+ set_arch_unknown(el2ctx->tf);
+
+ set_arch_unknown(el2ctx->actlr_el1);
+ set_arch_unknown(el2ctx->afsr0_el1);
+ set_arch_unknown(el2ctx->afsr1_el1);
+ set_arch_unknown(el2ctx->amair_el1);
+ set_arch_unknown(el2ctx->contextidr_el1);
+ set_arch_unknown(el2ctx->cpacr_el1);
+ set_arch_unknown(el2ctx->csselr_el1);
+ set_arch_unknown(el2ctx->elr_el1);
+ set_arch_unknown(el2ctx->esr_el1);
+ set_arch_unknown(el2ctx->far_el1);
+ set_arch_unknown(el2ctx->mair_el1);
+ set_arch_unknown(el2ctx->mdccint_el1);
+ set_arch_unknown(el2ctx->mdscr_el1);
+ set_arch_unknown(el2ctx->par_el1);
+
+ /*
+ * Guest starts with:
+ * ~SCTLR_M: MMU off
+ * ~SCTLR_C: data cache off
+ * SCTLR_CP15BEN: memory barrier instruction enable from EL0; RAO/WI
+ * ~SCTLR_I: instruction cache off
+ */
+ el2ctx->sctlr_el1 = SCTLR_RES1;
+ el2ctx->sctlr_el1 &= ~SCTLR_M & ~SCTLR_C & ~SCTLR_I;
+ el2ctx->sctlr_el1 |= SCTLR_CP15BEN;
+
+ set_arch_unknown(el2ctx->sp_el0);
+ set_arch_unknown(el2ctx->tcr_el1);
+ set_arch_unknown(el2ctx->tpidr_el0);
+ set_arch_unknown(el2ctx->tpidr_el1);
+ set_arch_unknown(el2ctx->tpidrro_el0);
+ set_arch_unknown(el2ctx->ttbr0_el1);
+ set_arch_unknown(el2ctx->ttbr1_el1);
+ set_arch_unknown(el2ctx->vbar_el1);
+ set_arch_unknown(el2ctx->spsr_el1);
+
+ set_arch_unknown(el2ctx->dbgbcr_el1);
+ set_arch_unknown(el2ctx->dbgbvr_el1);
+ set_arch_unknown(el2ctx->dbgwcr_el1);
+ set_arch_unknown(el2ctx->dbgwvr_el1);
+
+ el2ctx->pmcr_el0 = READ_SPECIALREG(pmcr_el0) & PMCR_N_MASK;
+ /* PMCR_LC is unknown when AArch32 is supported or RES1 otherwise */
+ el2ctx->pmcr_el0 |= PMCR_LC;
+ set_arch_unknown(el2ctx->pmccntr_el0);
+ set_arch_unknown(el2ctx->pmccfiltr_el0);
+ set_arch_unknown(el2ctx->pmcntenset_el0);
+ set_arch_unknown(el2ctx->pmintenset_el1);
+ set_arch_unknown(el2ctx->pmovsset_el0);
+ set_arch_unknown(el2ctx->pmuserenr_el0);
+ memset(el2ctx->pmevcntr_el0, 0, sizeof(el2ctx->pmevcntr_el0));
+ memset(el2ctx->pmevtyper_el0, 0, sizeof(el2ctx->pmevtyper_el0));
+}
+
+void
+reset_vm_el2_regs(void *vcpu)
+{
+ struct hypctx *el2ctx;
+ uint64_t cpu_aff, vcpuid;
+
+ el2ctx = vcpu;
+ vcpuid = vcpu_vcpuid(el2ctx->vcpu);
+
+ /*
+ * Set the Hypervisor Configuration Register:
+ *
+ * HCR_RW: use AArch64 for EL1
+ * HCR_TID3: handle ID registers in the vmm to privide a common
+ * set of featers on all vcpus
+ * HCR_TWI: Trap WFI to the hypervisor
+ * HCR_BSU_IS: barrier instructions apply to the inner shareable
+ * domain
+ * HCR_FB: broadcast maintenance operations
+ * HCR_AMO: route physical SError interrupts to EL2
+ * HCR_IMO: route physical IRQ interrupts to EL2
+ * HCR_FMO: route physical FIQ interrupts to EL2
+ * HCR_SWIO: turn set/way invalidate into set/way clean and
+ * invalidate
+ * HCR_VM: use stage 2 translation
+ */
+ el2ctx->hcr_el2 = HCR_RW | HCR_TID3 | HCR_TWI | HCR_BSU_IS | HCR_FB |
+ HCR_AMO | HCR_IMO | HCR_FMO | HCR_SWIO | HCR_VM;
+
+ /* TODO: Trap all extensions we don't support */
+ el2ctx->mdcr_el2 = 0;
+ /* PMCR_EL0.N is read from MDCR_EL2.HPMN */
+ el2ctx->mdcr_el2 |= (el2ctx->pmcr_el0 & PMCR_N_MASK) >> PMCR_N_SHIFT;
+
+ el2ctx->vmpidr_el2 = VMPIDR_EL2_RES1;
+ /* The guest will detect a multi-core, single-threaded CPU */
+ el2ctx->vmpidr_el2 &= ~VMPIDR_EL2_U & ~VMPIDR_EL2_MT;
+ /*
+ * Generate the guest MPIDR value. We only support 16 CPUs at affinity
+ * level 0 to simplify the vgicv3 driver (see writing sgi1r_el1).
+ */
+ cpu_aff = (vcpuid & 0xf) << MPIDR_AFF0_SHIFT |
+ ((vcpuid >> 4) & 0xff) << MPIDR_AFF1_SHIFT |
+ ((vcpuid >> 12) & 0xff) << MPIDR_AFF2_SHIFT |
+ ((vcpuid >> 20) & 0xff) << MPIDR_AFF3_SHIFT;
+ el2ctx->vmpidr_el2 |= cpu_aff;
+
+ /* Use the same CPU identification information as the host */
+ el2ctx->vpidr_el2 = CPU_IMPL_TO_MIDR(CPU_IMPL_ARM);
+ el2ctx->vpidr_el2 |= CPU_VAR_TO_MIDR(0);
+ el2ctx->vpidr_el2 |= CPU_ARCH_TO_MIDR(0xf);
+ el2ctx->vpidr_el2 |= CPU_PART_TO_MIDR(CPU_PART_FOUNDATION);
+ el2ctx->vpidr_el2 |= CPU_REV_TO_MIDR(0);
+
+ /*
+ * Don't trap accesses to CPACR_EL1, trace, SVE, Advanced SIMD
+ * and floating point functionality to EL2.
+ */
+ el2ctx->cptr_el2 = CPTR_RES1;
+ /*
+ * Disable interrupts in the guest. The guest OS will re-enable
+ * them.
+ */
+ el2ctx->tf.tf_spsr = PSR_D | PSR_A | PSR_I | PSR_F;
+ /* Use the EL1 stack when taking exceptions to EL1 */
+ el2ctx->tf.tf_spsr |= PSR_M_EL1h;
+}
diff --git a/sys/arm64/vmm/vmm_stat.h b/sys/arm64/vmm/vmm_stat.h
new file mode 100644
--- /dev/null
+++ b/sys/arm64/vmm/vmm_stat.h
@@ -0,0 +1,147 @@
+/*-
+ * SPDX-License-Identifier: BSD-3-Clause
+ *
+ * Copyright (c) 2011 NetApp, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * 3. Neither the name of the University nor the names of its contributors
+ * may be used to endorse or promote products derived from this software
+ * without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _VMM_STAT_H_
+#define _VMM_STAT_H_
+
+struct vm;
+
+#define MAX_VMM_STAT_ELEMS 64 /* arbitrary */
+
+enum vmm_stat_scope {
+ VMM_STAT_SCOPE_ANY,
+};
+
+struct vmm_stat_type;
+typedef void (*vmm_stat_func_t)(struct vcpu *vcpu,
+ struct vmm_stat_type *stat);
+
+struct vmm_stat_type {
+ int index; /* position in the stats buffer */
+ int nelems; /* standalone or array */
+ const char *desc; /* description of statistic */
+ vmm_stat_func_t func;
+ enum vmm_stat_scope scope;
+};
+
+void vmm_stat_register(void *arg);
+
+#define VMM_STAT_FDEFINE(type, nelems, desc, func, scope) \
+ struct vmm_stat_type type[1] = { \
+ { -1, nelems, desc, func, scope } \
+ }; \
+ SYSINIT(type##_stat, SI_SUB_KLD, SI_ORDER_ANY, vmm_stat_register, type)
+
+#define VMM_STAT_DEFINE(type, nelems, desc, scope) \
+ VMM_STAT_FDEFINE(type, nelems, desc, NULL, scope)
+
+#define VMM_STAT_DECLARE(type) \
+ extern struct vmm_stat_type type[1]
+
+#define VMM_STAT(type, desc) \
+ VMM_STAT_DEFINE(type, 1, desc, VMM_STAT_SCOPE_ANY)
+
+#define VMM_STAT_FUNC(type, desc, func) \
+ VMM_STAT_FDEFINE(type, 1, desc, func, VMM_STAT_SCOPE_ANY)
+
+#define VMM_STAT_ARRAY(type, nelems, desc) \
+ VMM_STAT_DEFINE(type, nelems, desc, VMM_STAT_SCOPE_ANY)
+
+void *vmm_stat_alloc(void);
+void vmm_stat_init(void *vp);
+void vmm_stat_free(void *vp);
+
+int vmm_stat_copy(struct vcpu *vcpu, int index, int count,
+ int *num_stats, uint64_t *buf);
+int vmm_stat_desc_copy(int index, char *buf, int buflen);
+
+static void __inline
+vmm_stat_array_incr(struct vcpu *vcpu, struct vmm_stat_type *vst, int statidx,
+ uint64_t x)
+{
+#ifdef VMM_KEEP_STATS
+ uint64_t *stats;
+
+ stats = vcpu_stats(vcpu);
+
+ if (vst->index >= 0 && statidx < vst->nelems)
+ stats[vst->index + statidx] += x;
+#endif
+}
+
+static void __inline
+vmm_stat_array_set(struct vcpu *vcpu, struct vmm_stat_type *vst, int statidx,
+ uint64_t val)
+{
+#ifdef VMM_KEEP_STATS
+ uint64_t *stats;
+
+ stats = vcpu_stats(vcpu);
+
+ if (vst->index >= 0 && statidx < vst->nelems)
+ stats[vst->index + statidx] = val;
+#endif
+}
+
+static void __inline
+vmm_stat_incr(struct vcpu *vcpu, struct vmm_stat_type *vst, uint64_t x)
+{
+
+#ifdef VMM_KEEP_STATS
+ vmm_stat_array_incr(vcpu, vst, 0, x);
+#endif
+}
+
+static void __inline
+vmm_stat_set(struct vcpu *vcpu, struct vmm_stat_type *vst, uint64_t val)
+{
+
+#ifdef VMM_KEEP_STATS
+ vmm_stat_array_set(vcpu, vst, 0, val);
+#endif
+}
+
+VMM_STAT_DECLARE(VMEXIT_COUNT);
+VMM_STAT_DECLARE(VMEXIT_UNKNOWN);
+VMM_STAT_DECLARE(VMEXIT_WFI);
+VMM_STAT_DECLARE(VMEXIT_WFE);
+VMM_STAT_DECLARE(VMEXIT_HVC);
+VMM_STAT_DECLARE(VMEXIT_MSR);
+VMM_STAT_DECLARE(VMEXIT_DATA_ABORT);
+VMM_STAT_DECLARE(VMEXIT_INSN_ABORT);
+VMM_STAT_DECLARE(VMEXIT_UNHANDLED_SYNC);
+VMM_STAT_DECLARE(VMEXIT_IRQ);
+VMM_STAT_DECLARE(VMEXIT_FIQ);
+VMM_STAT_DECLARE(VMEXIT_UNHANDLED_EL2);
+VMM_STAT_DECLARE(VMEXIT_UNHANDLED);
+#endif
diff --git a/sys/arm64/vmm/vmm_stat.c b/sys/arm64/vmm/vmm_stat.c
new file mode 100644
--- /dev/null
+++ b/sys/arm64/vmm/vmm_stat.c
@@ -0,0 +1,165 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2011 NetApp, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+
+#include <sys/param.h>
+#include <sys/kernel.h>
+#include <sys/systm.h>
+#include <sys/malloc.h>
+
+#include <machine/machdep.h>
+#include <machine/vmm.h>
+#include "vmm_stat.h"
+
+/*
+ * 'vst_num_elems' is the total number of addressable statistic elements
+ * 'vst_num_types' is the number of unique statistic types
+ *
+ * It is always true that 'vst_num_elems' is greater than or equal to
+ * 'vst_num_types'. This is because a stat type may represent more than
+ * one element (for e.g. VMM_STAT_ARRAY).
+ */
+static int vst_num_elems, vst_num_types;
+static struct vmm_stat_type *vsttab[MAX_VMM_STAT_ELEMS];
+
+static MALLOC_DEFINE(M_VMM_STAT, "vmm stat", "vmm stat");
+
+#define vst_size ((size_t)vst_num_elems * sizeof(uint64_t))
+
+void
+vmm_stat_register(void *arg)
+{
+ struct vmm_stat_type *vst = arg;
+
+ /* We require all stats to identify themselves with a description */
+ if (vst->desc == NULL)
+ return;
+
+ if (vst_num_elems + vst->nelems >= MAX_VMM_STAT_ELEMS) {
+ printf("Cannot accommodate vmm stat type \"%s\"!\n", vst->desc);
+ return;
+ }
+
+ vst->index = vst_num_elems;
+ vst_num_elems += vst->nelems;
+
+ vsttab[vst_num_types++] = vst;
+}
+
+int
+vmm_stat_copy(struct vcpu *vcpu, int index, int count, int *num_stats,
+ uint64_t *buf)
+{
+ struct vmm_stat_type *vst;
+ uint64_t *stats;
+ int i, tocopy;
+
+ if (index < 0 || count < 0)
+ return (EINVAL);
+
+ if (index > vst_num_elems)
+ return (ENOENT);
+
+ if (index == vst_num_elems) {
+ *num_stats = 0;
+ return (0);
+ }
+
+ tocopy = min(vst_num_elems - index, count);
+
+ /* Let stats functions update their counters */
+ for (i = 0; i < vst_num_types; i++) {
+ vst = vsttab[i];
+ if (vst->func != NULL)
+ (*vst->func)(vcpu, vst);
+ }
+
+ /* Copy over the stats */
+ stats = vcpu_stats(vcpu);
+ memcpy(buf, stats + index, tocopy * sizeof(stats[0]));
+ *num_stats = tocopy;
+ return (0);
+}
+
+void *
+vmm_stat_alloc(void)
+{
+
+ return (malloc(vst_size, M_VMM_STAT, M_WAITOK));
+}
+
+void
+vmm_stat_init(void *vp)
+{
+
+ bzero(vp, vst_size);
+}
+
+void
+vmm_stat_free(void *vp)
+{
+ free(vp, M_VMM_STAT);
+}
+
+int
+vmm_stat_desc_copy(int index, char *buf, int bufsize)
+{
+ int i;
+ struct vmm_stat_type *vst;
+
+ for (i = 0; i < vst_num_types; i++) {
+ vst = vsttab[i];
+ if (index >= vst->index && index < vst->index + vst->nelems) {
+ if (vst->nelems > 1) {
+ snprintf(buf, bufsize, "%s[%d]",
+ vst->desc, index - vst->index);
+ } else {
+ strlcpy(buf, vst->desc, bufsize);
+ }
+ return (0); /* found it */
+ }
+ }
+
+ return (EINVAL);
+}
+
+/* global statistics */
+VMM_STAT(VMEXIT_COUNT, "total number of vm exits");
+VMM_STAT(VMEXIT_UNKNOWN, "number of vmexits for the unknown exception");
+VMM_STAT(VMEXIT_WFI, "number of times wfi was intercepted");
+VMM_STAT(VMEXIT_WFE, "number of times wfe was intercepted");
+VMM_STAT(VMEXIT_HVC, "number of times hvc was intercepted");
+VMM_STAT(VMEXIT_MSR, "number of times msr/mrs was intercepted");
+VMM_STAT(VMEXIT_DATA_ABORT, "number of vmexits for a data abort");
+VMM_STAT(VMEXIT_INSN_ABORT, "number of vmexits for an instruction abort");
+VMM_STAT(VMEXIT_UNHANDLED_SYNC, "number of vmexits for an unhandled synchronous exception");
+VMM_STAT(VMEXIT_IRQ, "number of vmexits for an irq");
+VMM_STAT(VMEXIT_FIQ, "number of vmexits for an interrupt");
+VMM_STAT(VMEXIT_UNHANDLED_EL2, "number of vmexits for an unhandled EL2 exception");
+VMM_STAT(VMEXIT_UNHANDLED, "number of vmexits for an unhandled exception");
diff --git a/sys/conf/files.arm64 b/sys/conf/files.arm64
--- a/sys/conf/files.arm64
+++ b/sys/conf/files.arm64
@@ -116,6 +116,38 @@
dev/iommu/busdma_iommu.c optional iommu
dev/iommu/iommu_gas.c optional iommu
+arm64/vmm/vmm.c optional vmm
+arm64/vmm/vmm_dev.c optional vmm
+arm64/vmm/vmm_instruction_emul.c optional vmm
+arm64/vmm/vmm_mem.c optional vmm
+arm64/vmm/vmm_stat.c optional vmm
+arm64/vmm/vmm_arm64.c optional vmm
+arm64/vmm/vmm_reset.c optional vmm
+arm64/vmm/vmm_call.S optional vmm
+arm64/vmm/vmm_hyp_exception.S optional vmm \
+ compile-with "${NORMAL_C} -fpie" \
+ no-obj
+arm64/vmm/vmm_hyp.c optional vmm \
+ compile-with "${NORMAL_C} -fpie" \
+ no-obj
+vmm_hyp_blob.elf.full optional vmm \
+ dependency "vmm_hyp.o vmm_hyp_exception.o" \
+ compile-with "${CC} -o ${.TARGET} ${.ALLSRC} -fPIE -nostdlib -T ${LDSCRIPT} -Wl,--defsym=text_start='0x0'" \
+ no-obj no-implicit-rule
+vmm_hyp_blob.elf optional vmm \
+ dependency "vmm_hyp_blob.elf.full" \
+ compile-with "${OBJCOPY} --strip-debug ${.ALLSRC} ${.TARGET}" \
+ no-obj no-implicit-rule
+vmm_hyp_blob.bin optional vmm \
+ dependency vmm_hyp_blob.elf \
+ compile-with "${OBJCOPY} --output-target=binary ${.ALLSRC} ${.TARGET}" \
+ no-obj no-implicit-rule
+arm64/vmm/vmm_hyp_el2.S optional vmm \
+ dependency vmm_hyp_blob.bin
+arm64/vmm/vmm_mmu.c optional vmm
+arm64/vmm/io/vgic_v3.c optional vmm
+arm64/vmm/io/vtimer.c optional vmm
+
crypto/armv8/armv8_crypto.c optional armv8crypto
armv8_crypto_wrap.o optional armv8crypto \
dependency "$S/crypto/armv8/armv8_crypto_wrap.c" \
diff --git a/sys/conf/ldscript.arm64 b/sys/conf/ldscript.arm64
--- a/sys/conf/ldscript.arm64
+++ b/sys/conf/ldscript.arm64
@@ -7,6 +7,7 @@
{
/* Read-only sections, merged into text segment: */
. = text_start; /* This is set using --defsym= on the command line. */
+ .vmm_vectors : { *(.vmm_vectors) }
.text :
{
*(.text)
@@ -17,6 +18,7 @@
} =0x9090
_etext = .;
PROVIDE (etext = .);
+
.fini : { *(.fini) } =0x9090
.rodata : { *(.rodata*) *(.gnu.linkonce.r*) }
.rodata1 : { *(.rodata1) }
diff --git a/sys/conf/options.arm64 b/sys/conf/options.arm64
--- a/sys/conf/options.arm64
+++ b/sys/conf/options.arm64
@@ -20,6 +20,9 @@
# EFI Runtime services support
EFIRT opt_efirt.h
+# Bhyve
+VMM opt_global.h
+
# SoC Support
SOC_ALLWINNER_A64 opt_soc.h
SOC_ALLWINNER_H5 opt_soc.h
diff --git a/sys/modules/Makefile b/sys/modules/Makefile
--- a/sys/modules/Makefile
+++ b/sys/modules/Makefile
@@ -821,7 +821,9 @@
_sgx_linux= sgx_linux
_smartpqi= smartpqi
_p2sb= p2sb
+.endif
+.if ${MACHINE_CPUARCH} == "aarch64" || ${MACHINE_CPUARCH} == "amd64"
.if ${MK_BHYVE} != "no" || defined(ALL_MODULES)
.if ${KERN_OPTS:MSMP}
_vmm= vmm
diff --git a/sys/modules/vmm/Makefile b/sys/modules/vmm/Makefile
--- a/sys/modules/vmm/Makefile
+++ b/sys/modules/vmm/Makefile
@@ -4,31 +4,72 @@
KMOD= vmm
-SRCS= opt_acpi.h opt_bhyve_snapshot.h opt_ddb.h
-SRCS+= device_if.h bus_if.h pci_if.h pcib_if.h acpi_if.h vnode_if.h
-DPSRCS+= vmx_assym.h svm_assym.h
-DPSRCS+= vmx_genassym.c svm_genassym.c offset.inc
+SRCS= opt_acpi.h opt_ddb.h device_if.h bus_if.h pci_if.h pcib_if.h acpi_if.h
CFLAGS+= -DVMM_KEEP_STATS
-CFLAGS+= -I${SRCTOP}/sys/amd64/vmm
-CFLAGS+= -I${SRCTOP}/sys/amd64/vmm/io
-CFLAGS+= -I${SRCTOP}/sys/amd64/vmm/intel
-CFLAGS+= -I${SRCTOP}/sys/amd64/vmm/amd
+CFLAGS+= -I${SRCTOP}/sys/${MACHINE}/vmm
+CFLAGS+= -I${SRCTOP}/sys/${MACHINE}/vmm/io
# generic vmm support
-.PATH: ${SRCTOP}/sys/amd64/vmm
+.PATH: ${SRCTOP}/sys/${MACHINE}/vmm
SRCS+= vmm.c \
vmm_dev.c \
- vmm_host.c \
vmm_instruction_emul.c \
+ vmm_mem.c \
+ vmm_stat.c
+
+.if ${MACHINE_CPUARCH} == "aarch64"
+# TODO: Add the new EL2 code
+SRCS+= vmm_arm64.c \
+ vmm_reset.c \
+ vmm_call.S \
+ vmm_mmu.c \
+ vmm_hyp_el2.S
+
+.PATH: ${SRCTOP}/sys/${MACHINE}/vmm/io
+SRCS+= vgic.c \
+ vgic_if.h \
+ vgic_if.c \
+ vgic_v3.c \
+ vtimer.c
+
+SRCS+= vmm_hyp_exception.S vmm_hyp.c
+
+CLEANFILES+= vmm_hyp_blob.elf.full
+CLEANFILES+= vmm_hyp_blob.elf vmm_hyp_blob.bin
+
+CFLAGS.vmm_hyp_exception.S += -fpie
+CFLAGS.vmm_hyp.c += -fpie
+vmm_hyp_exception.o: vmm_hyp_exception.S
+vmm_hyp.o: vmm_hyp.c
+
+vmm_hyp_blob.elf.full: vmm_hyp_exception.o vmm_hyp.o
+ ${CC} -o ${.TARGET} ${.ALLSRC} -fPIE -nostdlib \
+ -T ${SYSDIR}/conf/ldscript.arm64 \
+ -Wl,--defsym=text_start='0x0'
+
+vmm_hyp_blob.elf: vmm_hyp_blob.elf.full
+ ${OBJCOPY} --strip-debug ${.ALLSRC} ${.TARGET}
+
+vmm_hyp_blob.bin: vmm_hyp_blob.elf
+ ${OBJCOPY} --output-target=binary ${.ALLSRC} ${.TARGET}
+
+vmm_hyp_el2.o: vmm_hyp_blob.bin
+
+.elif ${MACHINE_CPUARCH} == "amd64"
+DPSRCS+= vmx_assym.h svm_assym.h
+DPSRCS+= vmx_genassym.c svm_genassym.c offset.inc
+
+CFLAGS+= -I${SRCTOP}/sys/amd64/vmm/intel
+CFLAGS+= -I${SRCTOP}/sys/amd64/vmm/amd
+
+SRCS+= vmm_host.c \
vmm_ioport.c \
vmm_lapic.c \
- vmm_mem.c \
- vmm_stat.c \
vmm_util.c \
x86.c
-.PATH: ${SRCTOP}/sys/amd64/vmm/io
+.PATH: ${SRCTOP}/sys/${MACHINE}/vmm/io
SRCS+= iommu.c \
ppt.c \
vatpic.c \
@@ -63,10 +104,11 @@
SRCS.BHYVE_SNAPSHOT= vmm_snapshot.c
-CLEANFILES= vmx_assym.h vmx_genassym.o svm_assym.h svm_genassym.o
+CLEANFILES+= vmx_assym.h vmx_genassym.o svm_assym.h svm_genassym.o
OBJS_DEPEND_GUESS.vmx_support.o+= vmx_assym.h
OBJS_DEPEND_GUESS.svm_support.o+= svm_assym.h
+.endif
vmx_assym.h: vmx_genassym.o
sh ${SYSDIR}/kern/genassym.sh vmx_genassym.o > ${.TARGET}
@@ -82,6 +124,9 @@
${CC} -c -x assembler-with-cpp -DLOCORE ${CFLAGS} \
${.IMPSRC} -o ${.TARGET}
+hyp_genassym.o: offset.inc
+ ${CC} -c ${CFLAGS:N-flto:N-fno-common} -fcommon ${.IMPSRC}
+
vmx_genassym.o: offset.inc
${CC} -c ${CFLAGS:N-flto:N-fno-common} -fcommon ${.IMPSRC}

File Metadata

Mime Type
text/plain
Expires
Thu, Sep 17, 2:54 PM (18 h, 59 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
29733731
Default Alt Text
D37428.1789656868.diff (307 KB)

Event Timeline