Page MenuHomeFreeBSD
Files F146298907

D46379.1779798474.diff
No OneTemporary
Actions

Size
43 KB
Referenced Files
None
Subscribers
None

D46379.1779798474.diff
View Options

diff --git a/sys/amd64/amd64/zcond_machdep.c b/sys/amd64/amd64/zcond_machdep.c
new file mode 100644
--- /dev/null
+++ b/sys/amd64/amd64/zcond_machdep.c
@@ -0,0 +1,109 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2024 Marko Vlaić <mvlaic@freebsd.org>
+ *
+ * This code was developed as a Google Summer of Code 2024. project
+ * under the guidance of Bojan Novković <bnovkov@freebsdorg>.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/types.h>
+#include <sys/kassert.h>
+#include <sys/pcpu.h>
+#include <sys/zcond.h>
+
+#include <machine/cpufunc.h>
+#include <machine/md_var.h>
+#include <machine/zcond.h>
+
+static uint8_t insn[ZCOND_MAX_INSN_SIZE];
+
+static uint8_t *
+insn_nop(size_t size)
+{
+ if (size == ZCOND_INSN_SHORT_SIZE) {
+ return &nop_short_bytes[0];
+ }
+ return &nop_long_bytes[0];
+}
+
+static uint8_t *
+insn_jmp(size_t size, vm_offset_t patch_addr, vm_offset_t lbl_true_addr)
+{
+ int i;
+ vm_offset_t offset;
+
+ offset = lbl_true_addr - patch_addr - size;
+
+ if (size == ZCOND_INSN_SHORT_SIZE) {
+ insn[0] = ZCOND_JMP_SHORT_OPCODE;
+ insn[1] = offset;
+ } else {
+ insn[0] = ZCOND_JMP_LONG_OPCODE;
+ for (i = 0; i < ZCOND_INSN_LONG_SIZE - 1; i++) {
+ insn[i + 1] = (offset >> (i * 8)) & 0xFF;
+ }
+ }
+
+ return &insn[0];
+}
+
+void
+zcond_before_patch(struct zcond_md_ctxt *ctxt)
+{
+ ctxt->wp = disable_wp();
+}
+
+void
+zcond_after_patch(struct zcond_md_ctxt *ctxt)
+{
+ restore_wp(ctxt->wp);
+}
+
+uint8_t *
+zcond_get_patch_insn(vm_offset_t patch_addr, vm_offset_t lbl_true_addr,
+ size_t *size)
+{
+ uint8_t *pa;
+
+ pa = (uint8_t *)patch_addr;
+ if (*pa == nop_short_bytes[0]) {
+ /* two byte nop */
+ *size = ZCOND_INSN_SHORT_SIZE;
+ return insn_jmp(*size, patch_addr, lbl_true_addr);
+ } else if (*pa == nop_long_bytes[0]) {
+ *size = ZCOND_INSN_LONG_SIZE;
+ return insn_jmp(*size, patch_addr, lbl_true_addr);
+ } else if (*pa == ZCOND_JMP_SHORT_OPCODE) {
+ /* two byte jump */
+ *size = ZCOND_INSN_SHORT_SIZE;
+ return insn_nop(*size);
+ } else if (*pa == ZCOND_JMP_LONG_OPCODE) {
+ /* five byte jump */
+ *size = ZCOND_INSN_LONG_SIZE;
+ return insn_nop(*size);
+ } else {
+ panic("%s: unexpected opcode: %02hhx", __func__, *pa);
+ }
+}
diff --git a/sys/amd64/include/sdt_machdep.h b/sys/amd64/include/sdt_machdep.h
deleted file mode 100644
--- a/sys/amd64/include/sdt_machdep.h
+++ /dev/null
@@ -1,12 +0,0 @@
-/*-
- * SPDX-License-Identifier: BSD-2-Clause
- *
- * Copyright (c) 2024 Mark Johnston <markj@FreeBSD.org>
- */
-
-#ifndef _SYS_SDT_MACHDEP_H_
-#define _SYS_SDT_MACHDEP_H_
-
-#define _SDT_ASM_PATCH_INSTR "nopw 0(%%rax,%%rax,1)"
-
-#endif
diff --git a/sys/amd64/include/zcond.h b/sys/amd64/include/zcond.h
new file mode 100644
--- /dev/null
+++ b/sys/amd64/include/zcond.h
@@ -0,0 +1,64 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2024 Marko Vlaić <mvlaic@freebsd.org>
+ *
+ * This code was developed as a Google Summer of Code 2024. project
+ * under the guidance of Bojan Novković <bnovkov@freebsdorg>.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifdef _KERNEL
+#ifndef _MACHINE_ZCOND_H
+#define _MACHINE_ZCOND_H
+
+// #include <sys/zcond.h>
+#include <sys/types.h>
+
+/* from Intel® 64 and IA-32 Architectures Software Developer’s Manual, Volume 2B
+ * 4-165 */
+static uint8_t nop_short_bytes[] = { 0x66, 0x90 };
+static uint8_t nop_long_bytes[] = { 0x0f, 0x1f, 0x44, 0x00, 0x00 };
+
+#define ZCOND_NOP_ASM \
+ ".byte 0x0f \n\t" \
+ ".byte 0x1f \n\t" \
+ ".byte 0x44 \n\t" \
+ ".byte 0x00 \n\t" \
+ ".byte 0x00 \n\t"
+
+#define ZCOND_JMP_ASM "jmp"
+
+#define ZCOND_JMP_SHORT_OPCODE 0xeb
+#define ZCOND_JMP_LONG_OPCODE 0xe9
+
+#define ZCOND_INSN_SHORT_SIZE 2
+#define ZCOND_INSN_LONG_SIZE 5
+#define ZCOND_MAX_INSN_SIZE 5
+
+struct zcond_md_ctxt {
+ bool wp;
+};
+
+#endif /*_MACHINE_ZCOND_H*/
+#endif /* _KERNEL */
diff --git a/sys/arm/arm/sdt_machdep.c b/sys/arm/arm/sdt_machdep.c
deleted file mode 100644
--- a/sys/arm/arm/sdt_machdep.c
+++ /dev/null
@@ -1,63 +0,0 @@
-/*-
- * SPDX-License-Identifier: BSD-2-Clause
- *
- * Copyright (c) 2024 Mark Johnston <markj@FreeBSD.org>
- */
-
-#include <sys/systm.h>
-#include <sys/sdt.h>
-
-#include <machine/cpu.h>
-
-/*
- * Return true if we can overwrite a nop at "patchpoint" with a jump to the
- * target address.
- */
-bool
-sdt_tracepoint_valid(uintptr_t patchpoint, uintptr_t target)
-{
- int32_t offset;
-
- if (patchpoint == target ||
- (patchpoint & (INSN_SIZE - 1)) != 0 ||
- (target & (INSN_SIZE - 1)) != 0 ||
- patchpoint + 2 * INSN_SIZE < patchpoint)
- return (false);
- offset = target - (patchpoint + 2 * INSN_SIZE);
- if (offset < -(1 << 24) || offset > (1 >> 24))
- return (false);
- return (true);
-}
-
-/*
- * Overwrite the copy of _SDT_ASM_PATCH_INSTR at the tracepoint with a jump to
- * the target address.
- */
-void
-sdt_tracepoint_patch(uintptr_t patchpoint, uintptr_t target)
-{
- uint32_t instr;
-
- KASSERT(sdt_tracepoint_valid(patchpoint, target),
- ("%s: invalid tracepoint %#x -> %#x",
- __func__, patchpoint, target));
-
- instr =
- (((target - (patchpoint + 2 * INSN_SIZE)) >> 2) & ((1 << 24) - 1)) |
- 0xea000000;
- memcpy((void *)patchpoint, &instr, sizeof(instr));
- icache_sync(patchpoint, sizeof(instr));
-}
-
-/*
- * Overwrite the patchpoint with a nop instruction.
- */
-void
-sdt_tracepoint_restore(uintptr_t patchpoint)
-{
- uint32_t instr;
-
- instr = 0xe320f000u;
- memcpy((void *)patchpoint, &instr, sizeof(instr));
- icache_sync(patchpoint, sizeof(instr));
-}
diff --git a/sys/arm/include/sdt_machdep.h b/sys/arm/include/sdt_machdep.h
deleted file mode 100644
--- a/sys/arm/include/sdt_machdep.h
+++ /dev/null
@@ -1,12 +0,0 @@
-/*-
- * SPDX-License-Identifier: BSD-2-Clause
- *
- * Copyright (c) 2024 Mark Johnston <markj@FreeBSD.org>
- */
-
-#ifndef _SYS_SDT_MACHDEP_H_
-#define _SYS_SDT_MACHDEP_H_
-
-#define _SDT_ASM_PATCH_INSTR "nop"
-
-#endif /* _SYS_SDT_MACHDEP_H_ */
diff --git a/sys/arm64/arm64/sdt_machdep.c b/sys/arm64/arm64/sdt_machdep.c
deleted file mode 100644
--- a/sys/arm64/arm64/sdt_machdep.c
+++ /dev/null
@@ -1,77 +0,0 @@
-/*-
- * SPDX-License-Identifier: BSD-2-Clause
- *
- * Copyright (c) 2024 Mark Johnston <markj@FreeBSD.org>
- */
-
-#include <sys/systm.h>
-#include <sys/sdt.h>
-
-#include <vm/vm.h>
-#include <vm/pmap.h>
-
-#include <machine/cpufunc.h>
-#include <machine/md_var.h>
-#include <machine/vmparam.h>
-
-/*
- * Return true if we can overwrite a nop at "patchpoint" with a jump to the
- * target address.
- */
-bool
-sdt_tracepoint_valid(uintptr_t patchpoint, uintptr_t target)
-{
- void *addr;
- int64_t offset;
-
- if (!arm64_get_writable_addr((void *)patchpoint, &addr))
- return (false);
-
- if (patchpoint == target ||
- (patchpoint & (INSN_SIZE - 1)) != 0 ||
- (target & (INSN_SIZE - 1)) != 0)
- return (false);
- offset = target - patchpoint;
- if (offset < -(1 << 26) || offset > (1 << 26))
- return (false);
- return (true);
-}
-
-/*
- * Overwrite the copy of _SDT_ASM_PATCH_INSTR at the tracepoint with a jump to the
- * target address.
- */
-void
-sdt_tracepoint_patch(uintptr_t patchpoint, uintptr_t target)
-{
- void *addr;
- uint32_t instr;
-
- KASSERT(sdt_tracepoint_valid(patchpoint, target),
- ("%s: invalid tracepoint %#lx -> %#lx",
- __func__, patchpoint, target));
-
- if (!arm64_get_writable_addr((void *)patchpoint, &addr))
- panic("%s: Unable to write new instruction", __func__);
-
- instr = (((target - patchpoint) >> 2) & 0x3fffffful) | 0x14000000;
- memcpy(addr, &instr, sizeof(instr));
- cpu_icache_sync_range((void *)patchpoint, INSN_SIZE);
-}
-
-/*
- * Overwrite the patchpoint with a nop instruction.
- */
-void
-sdt_tracepoint_restore(uintptr_t patchpoint)
-{
- void *addr;
- uint32_t instr;
-
- if (!arm64_get_writable_addr((void *)patchpoint, &addr))
- panic("%s: Unable to write new instruction", __func__);
-
- instr = 0xd503201f;
- memcpy(addr, &instr, sizeof(instr));
- cpu_icache_sync_range((void *)patchpoint, INSN_SIZE);
-}
diff --git a/sys/arm64/include/sdt_machdep.h b/sys/arm64/include/sdt_machdep.h
deleted file mode 100644
--- a/sys/arm64/include/sdt_machdep.h
+++ /dev/null
@@ -1,12 +0,0 @@
-/*-
- * SPDX-License-Identifier: BSD-2-Clause
- *
- * Copyright (c) 2024 Mark Johnston <markj@FreeBSD.org>
- */
-
-#ifndef _SYS_SDT_MACHDEP_H_
-#define _SYS_SDT_MACHDEP_H_
-
-#define _SDT_ASM_PATCH_INSTR "nop"
-
-#endif /* _SYS_SDT_MACHDEP_H_ */
diff --git a/sys/cddl/dev/sdt/sdt.c b/sys/cddl/dev/sdt/sdt.c
--- a/sys/cddl/dev/sdt/sdt.c
+++ b/sys/cddl/dev/sdt/sdt.c
@@ -54,6 +54,7 @@
#include <sys/mutex.h>
#include <sys/queue.h>
#include <sys/sdt.h>
+#include <sys/zcond.h>
#include <sys/dtrace.h>
#include <sys/dtrace_bsd.h>
@@ -214,55 +215,6 @@
{
}
-struct sdt_enable_cb_arg {
- struct sdt_probe *probe;
- int cpu;
- int arrived;
- int done;
- bool enable;
-};
-
-static void
-sdt_probe_update_cb(void *_arg)
-{
- struct sdt_enable_cb_arg *arg;
- struct sdt_tracepoint *tp;
-
- arg = _arg;
- if (arg->cpu != curcpu) {
- atomic_add_rel_int(&arg->arrived, 1);
- while (atomic_load_acq_int(&arg->done) == 0)
- cpu_spinwait();
- return;
- } else {
- while (atomic_load_acq_int(&arg->arrived) != mp_ncpus - 1)
- cpu_spinwait();
- }
-
- STAILQ_FOREACH(tp, &arg->probe->tracepoint_list, tracepoint_entry) {
- if (arg->enable)
- sdt_tracepoint_patch(tp->patchpoint, tp->target);
- else
- sdt_tracepoint_restore(tp->patchpoint);
- }
-
- atomic_store_rel_int(&arg->done, 1);
-}
-
-static void
-sdt_probe_update(struct sdt_probe *probe, bool enable)
-{
- struct sdt_enable_cb_arg cbarg;
-
- sched_pin();
- cbarg.probe = probe;
- cbarg.cpu = curcpu;
- atomic_store_rel_int(&cbarg.arrived, 0);
- atomic_store_rel_int(&cbarg.done, 0);
- cbarg.enable = enable;
- smp_rendezvous(NULL, sdt_probe_update_cb, NULL, &cbarg);
- sched_unpin();
-}
static void
sdt_enable(void *arg __unused, dtrace_id_t id, void *parg)
@@ -282,7 +234,7 @@
if (sdt_probes_enabled_count == 1)
sdt_probes_enabled = true;
- sdt_probe_update(probe, true);
+ zcond_enable(probe->enabled);
}
static void
@@ -293,7 +245,7 @@
probe = parg;
KASSERT(probe->sdtp_lf->nenabled > 0, ("no probes enabled"));
- sdt_probe_update(probe, false);
+ zcond_disable(probe->enabled);
sdt_probes_enabled_count--;
if (sdt_probes_enabled_count == 0)
@@ -305,6 +257,7 @@
}
probe->id = 0;
probe->sdtp_lf->nenabled--;
+
}
static void
@@ -374,7 +327,6 @@
{
struct sdt_probe **p_begin, **p_end;
struct sdt_argtype **a_begin, **a_end;
- struct sdt_tracepoint *tp_begin, *tp_end;
if (linker_file_lookup_set(lf, "sdt_probes_set", &p_begin, &p_end,
NULL) == 0) {
@@ -383,7 +335,6 @@
(*probe)->sdtp_lf = lf;
sdt_create_probe(*probe);
TAILQ_INIT(&(*probe)->argtype_list);
- STAILQ_INIT(&(*probe)->tracepoint_list);
}
}
@@ -396,23 +347,6 @@
*argtype, argtype_entry);
}
}
-
- if (linker_file_lookup_set(lf, __XSTRING(_SDT_TRACEPOINT_SET),
- &tp_begin, &tp_end, NULL) == 0) {
- for (struct sdt_tracepoint *tp = tp_begin; tp < tp_end; tp++) {
- if (!sdt_tracepoint_valid(tp->patchpoint, tp->target)) {
- printf(
- "invalid tracepoint %#jx->%#jx for %s:%s:%s:%s\n",
- (uintmax_t)tp->patchpoint,
- (uintmax_t)tp->target,
- tp->probe->prov->name, tp->probe->mod,
- tp->probe->func, tp->probe->name);
- continue;
- }
- STAILQ_INSERT_TAIL(&tp->probe->tracepoint_list, tp,
- tracepoint_entry);
- }
- }
}
/*
@@ -494,7 +428,8 @@
TAILQ_INIT(&sdt_prov_list);
- sdt_probe_func = sdt_dtrace_probe;
+ sdt_probe_func = dtrace_probe;
+ sdt_probe6_func = (sdt_probe6_func_t)sdt_dtrace_probe;
sdt_kld_load_tag = EVENTHANDLER_REGISTER(kld_load, sdt_kld_load, NULL,
EVENTHANDLER_PRI_ANY);
@@ -520,6 +455,7 @@
EVENTHANDLER_DEREGISTER(kld_unload_try, sdt_kld_unload_try_tag);
sdt_probe_func = sdt_probe_stub;
+ sdt_probe6_func = (sdt_probe6_func_t)sdt_probe_stub;
TAILQ_FOREACH_SAFE(prov, &sdt_prov_list, prov_entry, tmp) {
ret = dtrace_unregister(prov->id);
diff --git a/sys/conf/files b/sys/conf/files
--- a/sys/conf/files
+++ b/sys/conf/files
@@ -3865,6 +3865,7 @@
kern/kern_uuid.c standard
kern/kern_vnodedumper.c standard
kern/kern_xxx.c standard
+kern/kern_zcond.c standard
kern/link_elf.c standard
kern/linker_if.m standard
kern/md4c.c optional netsmb
diff --git a/sys/conf/files.amd64 b/sys/conf/files.amd64
--- a/sys/conf/files.amd64
+++ b/sys/conf/files.amd64
@@ -93,6 +93,7 @@
amd64/amd64/trap.c standard
amd64/amd64/uio_machdep.c standard
amd64/amd64/vm_machdep.c standard
+amd64/amd64/zcond_machdep.c standard
amd64/pci/pci_cfgreg.c optional pci
cddl/dev/dtrace/amd64/dtrace_asm.S optional dtrace compile-with "${DTRACE_S}"
cddl/dev/dtrace/amd64/dtrace_subr.c optional dtrace compile-with "${DTRACE_C}"
diff --git a/sys/conf/files.arm b/sys/conf/files.arm
--- a/sys/conf/files.arm
+++ b/sys/conf/files.arm
@@ -58,7 +58,6 @@
arm/arm/pmu_fdt.c optional fdt pmu | fdt hwpmc
arm/arm/ptrace_machdep.c standard
arm/arm/sc_machdep.c optional sc
-arm/arm/sdt_machdep.c optional kdtrace_hooks
arm/arm/setcpsr.S standard
arm/arm/setstack.S standard
arm/arm/stack_machdep.c optional ddb | stack
diff --git a/sys/conf/files.arm64 b/sys/conf/files.arm64
--- a/sys/conf/files.arm64
+++ b/sys/conf/files.arm64
@@ -68,7 +68,6 @@
compile-with "${NORMAL_C:N-mbranch-protection*} -mbranch-protection=bti"
arm64/arm64/pmap.c standard
arm64/arm64/ptrace_machdep.c standard
-arm64/arm64/sdt_machdep.c optional kdtrace_hooks
arm64/arm64/sigtramp.S standard
arm64/arm64/stack_machdep.c optional ddb | stack
arm64/arm64/strcmp.S standard
@@ -84,6 +83,7 @@
compile-with "${NORMAL_C:N-fsanitize*:N-fno-sanitize*}"
arm64/arm64/vfp.c standard
arm64/arm64/vm_machdep.c standard
+arm64/arm64/zcond_machdep.c standard
arm64/coresight/coresight.c standard
arm64/coresight/coresight_acpi.c optional acpi
diff --git a/sys/conf/files.powerpc b/sys/conf/files.powerpc
--- a/sys/conf/files.powerpc
+++ b/sys/conf/files.powerpc
@@ -379,7 +379,6 @@
powerpc/powerpc/platform_if.m standard
powerpc/powerpc/ptrace_machdep.c standard
powerpc/powerpc/sc_machdep.c optional sc
-powerpc/powerpc/sdt_machdep.c optional powerpc64 kdtrace_hooks
powerpc/powerpc/setjmp.S standard
powerpc/powerpc/sigcode32.S optional powerpc | powerpcspe | compat_freebsd32
powerpc/powerpc/sigcode64.S optional powerpc64 | powerpc64le
diff --git a/sys/conf/files.riscv b/sys/conf/files.riscv
--- a/sys/conf/files.riscv
+++ b/sys/conf/files.riscv
@@ -62,7 +62,6 @@
riscv/riscv/sigtramp.S standard
riscv/riscv/sbi.c standard
riscv/riscv/sbi_ipi.c optional smp
-riscv/riscv/sdt_machdep.c optional kdtrace_hooks
riscv/riscv/stack_machdep.c optional ddb | stack
riscv/riscv/support.S standard
riscv/riscv/swtch.S standard
diff --git a/sys/conf/files.x86 b/sys/conf/files.x86
--- a/sys/conf/files.x86
+++ b/sys/conf/files.x86
@@ -377,7 +377,6 @@
x86/x86/mp_x86.c optional smp
x86/x86/nexus.c standard
x86/x86/pvclock.c optional kvm_clock | xenhvm
-x86/x86/sdt_machdep.c optional kdtrace_hooks
x86/x86/stack_machdep.c optional ddb | stack
x86/x86/tsc.c standard
x86/x86/ucode.c standard
diff --git a/sys/i386/include/sdt_machdep.h b/sys/i386/include/sdt_machdep.h
deleted file mode 100644
--- a/sys/i386/include/sdt_machdep.h
+++ /dev/null
@@ -1,19 +0,0 @@
-/*-
- * SPDX-License-Identifier: BSD-2-Clause
- *
- * Copyright (c) 2024 Mark Johnston <markj@FreeBSD.org>
- */
-
-#ifndef _SYS_SDT_MACHDEP_H_
-#define _SYS_SDT_MACHDEP_H_
-
-#define _SDT_ASM_PATCH_INSTR "nop; nop; nop; nop; nop"
-
-/*
- * Work around an apparent clang bug or limitation which prevents the use of the
- * "i" (immediate) constraint with the probe structure.
- */
-#define _SDT_ASM_PROBE_CONSTRAINT "Ws"
-#define _SDT_ASM_PROBE_OPERAND "p"
-
-#endif
diff --git a/sys/kern/kern_sdt.c b/sys/kern/kern_sdt.c
--- a/sys/kern/kern_sdt.c
+++ b/sys/kern/kern_sdt.c
@@ -11,7 +11,7 @@
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
- *
+ *
* THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
@@ -38,6 +38,7 @@
* dtrace_probe() when it loads.
*/
sdt_probe_func_t sdt_probe_func = sdt_probe_stub;
+sdt_probe6_func_t sdt_probe6_func = (sdt_probe6_func_t) sdt_probe_stub;
volatile bool __read_frequently sdt_probes_enabled;
/*
@@ -48,7 +49,7 @@
void
sdt_probe_stub(uint32_t id __unused, uintptr_t arg0 __unused,
uintptr_t arg1 __unused, uintptr_t arg2 __unused, uintptr_t arg3 __unused,
- uintptr_t arg4 __unused, uintptr_t arg5 __unused)
+ uintptr_t arg4 __unused)
{
printf("sdt_probe_stub: unexpectedly called\n");
kdb_backtrace();
@@ -58,12 +59,12 @@
sdt_probe(uint32_t id, uintptr_t arg0, uintptr_t arg1,
uintptr_t arg2, uintptr_t arg3, uintptr_t arg4)
{
- sdt_probe_func(id, arg0, arg1, arg2, arg3, arg4, 0);
+ sdt_probe_func(id, arg0, arg1, arg2, arg3, arg4);
}
void
sdt_probe6(uint32_t id, uintptr_t arg0, uintptr_t arg1,
uintptr_t arg2, uintptr_t arg3, uintptr_t arg4, uintptr_t arg5)
{
- sdt_probe_func(id, arg0, arg1, arg2, arg3, arg4, arg5);
+ sdt_probe6_func(id, arg0, arg1, arg2, arg3, arg4, arg5);
}
diff --git a/sys/kern/kern_zcond.c b/sys/kern/kern_zcond.c
new file mode 100644
--- /dev/null
+++ b/sys/kern/kern_zcond.c
@@ -0,0 +1,156 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2024 Marko Vlaić <mvlaic@freebsd.org>
+ *
+ * This code was developed as a Google Summer of Code 2024. project
+ * under the guidance of Bojan Novković <bnovkov@freebsdorg>.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+#include <sys/types.h>
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/eventhandler.h>
+#include <sys/kernel.h>
+#include <sys/linker.h>
+#include <sys/linker_set.h>
+#include <sys/refcount.h>
+#include <sys/sbuf.h>
+#include <sys/smp.h>
+#include <sys/sysctl.h>
+#include <sys/zcond.h>
+
+struct patch_point {
+ vm_offset_t patch_addr;
+ vm_offset_t lbl_true_addr;
+ struct zcond *zcond;
+ SLIST_ENTRY(patch_point) next;
+} __attribute__((packed));
+
+struct zcond_patch_arg {
+ int patching_cpu;
+ struct zcond *cond;
+ struct zcond_md_ctxt *md_ctxt;
+};
+
+static void
+zcond_load_patch_points(linker_file_t lf)
+{
+ struct patch_point *begin, *end;
+ struct patch_point *ins_p;
+ struct zcond *owning_zcond;
+
+ if (linker_file_lookup_set(lf, __XSTRING(ZCOND_LINKER_SET), &begin,
+ &end, NULL) == 0) {
+ for (ins_p = begin; ins_p < end; ins_p++) {
+ owning_zcond = ins_p->zcond;
+ owning_zcond->refcnt = 1;
+
+ if (owning_zcond->patch_points.slh_first == NULL) {
+ SLIST_INIT(&owning_zcond->patch_points);
+ }
+
+ SLIST_INSERT_HEAD(&owning_zcond->patch_points, ins_p,
+ next);
+ }
+ }
+}
+
+static void
+zcond_kld_load(void *arg __unused, struct linker_file *lf)
+{
+ zcond_load_patch_points(lf);
+}
+
+static int
+zcond_load_patch_points_cb(linker_file_t lf, void *arg __unused)
+{
+ zcond_load_patch_points(lf);
+ return (0);
+}
+
+static void
+zcond_init(const void *unused)
+{
+ EVENTHANDLER_REGISTER(kld_load, zcond_kld_load, NULL,
+ EVENTHANDLER_PRI_ANY);
+ linker_file_foreach(zcond_load_patch_points_cb, NULL);
+}
+SYSINIT(zcond, SI_SUB_ZCOND, SI_ORDER_SECOND, zcond_init, NULL);
+
+/*
+ * Patch all patch_points belonging to cond.
+ */
+static void
+zcond_patch(struct zcond *cond, struct zcond_md_ctxt *ctxt)
+{
+ struct patch_point *p;
+ uint8_t *insn;
+ size_t insn_size;
+
+ SLIST_FOREACH(p, &cond->patch_points, next) {
+ insn = zcond_get_patch_insn(p->patch_addr, p->lbl_true_addr,
+ &insn_size);
+
+ zcond_before_patch(ctxt);
+ memcpy((void *)p->patch_addr, insn, insn_size);
+ zcond_after_patch(ctxt);
+ }
+}
+
+static void
+rendezvous_action(void *arg)
+{
+ struct zcond_patch_arg *data;
+
+ data = (struct zcond_patch_arg *)arg;
+
+ if (data->patching_cpu == curcpu) {
+ zcond_patch(data->cond, data->md_ctxt);
+ }
+}
+
+void
+__zcond_toggle(struct zcond *cond, bool enable)
+{
+ struct zcond_md_ctxt ctxt;
+
+ if (enable && refcount_acquire(&cond->refcnt) > 1) {
+ return;
+ } else if (!enable) {
+ if (!refcount_release_if_not_last(&cond->refcnt) ||
+ refcount_load(&cond->refcnt) != 1) {
+ return;
+ }
+ }
+
+ struct zcond_patch_arg arg = {
+ .patching_cpu = curcpu,
+ .cond = cond,
+ .md_ctxt = &ctxt,
+ };
+
+ smp_rendezvous(NULL, rendezvous_action, NULL, &arg);
+}
diff --git a/sys/powerpc/include/sdt_machdep.h b/sys/powerpc/include/sdt_machdep.h
deleted file mode 100644
--- a/sys/powerpc/include/sdt_machdep.h
+++ /dev/null
@@ -1,12 +0,0 @@
-/*-
- * SPDX-License-Identifier: BSD-2-Clause
- *
- * Copyright (c) 2024 Mark Johnston <markj@FreeBSD.org>
- */
-
-#ifndef _SYS_SDT_MACHDEP_H_
-#define _SYS_SDT_MACHDEP_H_
-
-#define _SDT_ASM_PATCH_INSTR "nop"
-
-#endif
diff --git a/sys/powerpc/powerpc/sdt_machdep.c b/sys/powerpc/powerpc/sdt_machdep.c
deleted file mode 100644
--- a/sys/powerpc/powerpc/sdt_machdep.c
+++ /dev/null
@@ -1,59 +0,0 @@
-/*-
- * SPDX-License-Identifier: BSD-2-Clause
- *
- * Copyright (c) 2024 Mark Johnston <markj@FreeBSD.org>
- */
-
-#include <sys/systm.h>
-#include <sys/sdt.h>
-
-#include <machine/md_var.h>
-
-/*
- * Return true if we can overwrite a nop at "patchpoint" with a jump to the
- * target address.
- */
-bool
-sdt_tracepoint_valid(uintptr_t patchpoint, uintptr_t target)
-{
- int64_t offset;
-
- if (patchpoint == target ||
- (patchpoint & 3) != 0 || (target & 3) != 0)
- return (false);
- offset = target - patchpoint;
- if (offset < -(1 << 26) || offset > (1 << 26))
- return (false);
- return (true);
-}
-
-/*
- * Overwrite the copy of _SDT_ASM_PATCH_INSTR at the tracepoint with a jump to
- * the target address.
- */
-void
-sdt_tracepoint_patch(uintptr_t patchpoint, uintptr_t target)
-{
- uint32_t instr;
-
- KASSERT(sdt_tracepoint_valid(patchpoint, target),
- ("%s: invalid tracepoint %#lx -> %#lx",
- __func__, patchpoint, target));
-
- instr = ((target - patchpoint) & 0x7fffffful) | 0x48000000;
- memcpy((void *)patchpoint, &instr, sizeof(instr));
- __syncicache((void *)patchpoint, sizeof(instr));
-}
-
-/*
- * Overwrite the patchpoint with a nop instruction.
- */
-void
-sdt_tracepoint_restore(uintptr_t patchpoint)
-{
- uint32_t instr;
-
- instr = 0x60000000;
- memcpy((void *)patchpoint, &instr, sizeof(instr));
- __syncicache((void *)patchpoint, sizeof(instr));
-}
diff --git a/sys/riscv/include/sdt_machdep.h b/sys/riscv/include/sdt_machdep.h
deleted file mode 100644
--- a/sys/riscv/include/sdt_machdep.h
+++ /dev/null
@@ -1,12 +0,0 @@
-/*-
- * SPDX-License-Identifier: BSD-2-Clause
- *
- * Copyright (c) 2024 Mark Johnston <markj@FreeBSD.org>
- */
-
-#ifndef _SYS_SDT_MACHDEP_H_
-#define _SYS_SDT_MACHDEP_H_
-
-#define _SDT_ASM_PATCH_INSTR ".option push; .option norvc; nop; .option pop"
-
-#endif /* _SYS_SDT_MACHDEP_H_ */
diff --git a/sys/riscv/riscv/sdt_machdep.c b/sys/riscv/riscv/sdt_machdep.c
deleted file mode 100644
--- a/sys/riscv/riscv/sdt_machdep.c
+++ /dev/null
@@ -1,68 +0,0 @@
-/*-
- * SPDX-License-Identifier: BSD-2-Clause
- *
- * Copyright (c) 2024 Mark Johnston <markj@FreeBSD.org>
- */
-
-#include <sys/systm.h>
-#include <sys/sdt.h>
-
-#include <machine/encoding.h>
-
-/*
- * Return true if we can overwrite a nop at "patchpoint" with a jump to the
- * target address.
- */
-bool
-sdt_tracepoint_valid(uintptr_t patchpoint, uintptr_t target)
-{
- int64_t offset;
-
- if (patchpoint == target ||
- (patchpoint & (INSN_C_SIZE - 1)) != 0 ||
- (target & (INSN_C_SIZE - 1)) != 0)
- return (false);
- offset = target - patchpoint;
- if (offset < -(1 << 19) || offset > (1 << 19))
- return (false);
- return (true);
-}
-
-/*
- * Overwrite the copy of _SDT_ASM_PATCH_INSTR at the tracepoint with a jump to
- * the target address.
- */
-void
-sdt_tracepoint_patch(uintptr_t patchpoint, uintptr_t target)
-{
- int32_t imm;
- uint32_t instr;
-
- KASSERT(sdt_tracepoint_valid(patchpoint, target),
- ("%s: invalid tracepoint %#lx -> %#lx",
- __func__, patchpoint, target));
-
- imm = target - patchpoint;
- imm = (imm & 0x100000) |
- ((imm & 0x7fe) << 8) |
- ((imm & 0x800) >> 2) |
- ((imm & 0xff000) >> 12);
- instr = (imm << 12) | MATCH_JAL;
-
- memcpy((void *)patchpoint, &instr, sizeof(instr));
- fence_i();
-}
-
-/*
- * Overwrite the patchpoint with a nop instruction.
- */
-void
-sdt_tracepoint_restore(uintptr_t patchpoint)
-{
- uint32_t instr;
-
- instr = 0x13; /* uncompressed nop */
-
- memcpy((void *)patchpoint, &instr, sizeof(instr));
- fence_i();
-}
diff --git a/sys/sys/kernel.h b/sys/sys/kernel.h
--- a/sys/sys/kernel.h
+++ b/sys/sys/kernel.h
@@ -107,6 +107,7 @@
SI_SUB_RACCT = 0x2110000, /* resource accounting */
SI_SUB_KDTRACE = 0x2140000, /* Kernel dtrace hooks */
SI_SUB_RANDOM = 0x2160000, /* random number generator */
+ SI_SUB_ZCOND = 0x2170000,
SI_SUB_MAC = 0x2180000, /* TrustedBSD MAC subsystem */
SI_SUB_MAC_POLICY = 0x21C0000, /* TrustedBSD MAC policies */
SI_SUB_MAC_LATE = 0x21D0000, /* TrustedBSD MAC subsystem */
diff --git a/sys/sys/sdt.h b/sys/sys/sdt.h
--- a/sys/sys/sdt.h
+++ b/sys/sys/sdt.h
@@ -77,8 +77,9 @@
#else /* _KERNEL */
+#include <sys/cdefs.h>
#include <sys/linker_set.h>
-#include <machine/sdt_machdep.h>
+#include <sys/zcond.h>
extern volatile bool sdt_probes_enabled;
@@ -177,6 +178,7 @@
.mod = #_mod, \
.func = #_func, \
.name = #_name, \
+ .enabled = ZCOND_INIT() \
}, \
}; \
DATA_SET(sdt_probes_set, _SDT_PROBE_NAME(_prov, _mod, _func, _name))
@@ -186,58 +188,17 @@
#define SDT_PROBES_ENABLED() __predict_false(sdt_probes_enabled)
-#ifdef _ILP32
-#define _SDT_ASM_WORD ".long"
-#else
-#define _SDT_ASM_WORD ".quad"
-#endif
-
-#ifndef _SDT_ASM_PROBE_CONSTRAINT
-#define _SDT_ASM_PROBE_CONSTRAINT "i"
-#endif
-#ifndef _SDT_ASM_PROBE_OPERAND
-#define _SDT_ASM_PROBE_OPERAND "c"
-#endif
+#define _SDT_PROBE(prov, mod, func, name, f, ...) do { \
+ if(SDT_PROBES_ENABLED()) { \
+ if (zcond_true(_SDT_PROBE_NAME(prov, mod, func, name)->enabled)) \
+ f(_SDT_PROBE_NAME(prov, mod, func, name)->id, __VA_ARGS__); \
+ } \
+ } while(0)
-/*
- * The asm below generates records corresponding to the structure's layout, so
- * the two must be kept in sync.
- */
-struct sdt_tracepoint {
- struct sdt_probe *probe;
- uintptr_t patchpoint;
- uintptr_t target;
- STAILQ_ENTRY(sdt_tracepoint) tracepoint_entry;
-};
-#define __SDT_PROBE(prov, mod, func, name, uniq, f, ...) do { \
- __WEAK(__CONCAT(__start_set_, _SDT_TRACEPOINT_SET)); \
- __WEAK(__CONCAT(__stop_set_, _SDT_TRACEPOINT_SET)); \
- asm goto( \
- "0:\n" \
- _SDT_ASM_PATCH_INSTR "\n" \
- ".pushsection " _SDT_TRACEPOINT_SECTION ", \"aw\"\n" \
- _SDT_ASM_WORD " %" _SDT_ASM_PROBE_OPERAND "0\n" \
- _SDT_ASM_WORD " 0b\n" \
- _SDT_ASM_WORD " %l1\n" \
- _SDT_ASM_WORD " 0\n" \
- ".popsection\n" \
- : \
- : _SDT_ASM_PROBE_CONSTRAINT (_SDT_PROBE_NAME(prov, mod, \
- func, name)) \
- : \
- : __sdt_probe##uniq); \
- if (0) { \
-__sdt_probe##uniq:; \
- f(_SDT_PROBE_NAME(prov, mod, func, name)->id, __VA_ARGS__); \
- } \
-} while (0)
-#define _SDT_PROBE(prov, mod, func, name, uniq, f, ...) \
- __SDT_PROBE(prov, mod, func, name, uniq, f, __VA_ARGS__)
#define SDT_PROBE(prov, mod, func, name, arg0, arg1, arg2, arg3, arg4) \
- _SDT_PROBE(prov, mod, func, name, __COUNTER__, sdt_probe, \
- (uintptr_t)arg0, (uintptr_t)arg1, (uintptr_t)arg2, \
- (uintptr_t)arg3, (uintptr_t)arg4)
+ _SDT_PROBE(prov, mod, func, name, sdt_probe, \
+ (uintptr_t)arg0, (uintptr_t)arg1, (uintptr_t)arg2, (uintptr_t)arg3, (uintptr_t)arg4)
#define SDT_PROBE_ARGTYPE(_prov, _mod, _func, _name, _num, _type, _xtype) \
static struct sdt_argtype \
@@ -355,9 +316,7 @@
#define SDT_PROBE5(prov, mod, func, name, arg0, arg1, arg2, arg3, arg4) \
SDT_PROBE(prov, mod, func, name, arg0, arg1, arg2, arg3, arg4)
#define SDT_PROBE6(prov, mod, func, name, arg0, arg1, arg2, arg3, arg4, arg5) \
- _SDT_PROBE(prov, mod, func, name, __COUNTER__, sdt_probe6, \
- (uintptr_t)arg0, (uintptr_t)arg1, (uintptr_t)arg2, \
- (uintptr_t)arg3, (uintptr_t)arg4, (uintptr_t)arg5)
+ _SDT_PROBE(prov, mod, func, name, sdt_probe6, (uintptr_t)arg0, (uintptr_t)arg1, (uintptr_t)arg2, (uintptr_t)arg3, (uintptr_t)arg4, (uintptr_t)arg5)
#ifndef KDTRACE_NO_MIB_SDT
#define MIB_SDT_PROBE1(...) SDT_PROBE1(mib, __VA_ARGS__)
@@ -419,12 +378,15 @@
* way to avoid having to rely on CDDL code.
*/
typedef void (*sdt_probe_func_t)(uint32_t, uintptr_t arg0, uintptr_t arg1,
- uintptr_t arg2, uintptr_t arg3, uintptr_t arg4, uintptr_t arg5);
+ uintptr_t arg2, uintptr_t arg3, uintptr_t arg4);
+typedef void (*sdt_probe6_func_t)(uint32_t, uintptr_t arg0, uintptr_t arg1,
+ uintptr_t arg2, uintptr_t arg3, uintptr_t arg4, uintptr_t arg5);
/*
* The 'sdt' provider will set it to dtrace_probe when it loads.
*/
extern sdt_probe_func_t sdt_probe_func;
+extern sdt_probe6_func_t sdt_probe6_func;
struct sdt_probe;
struct sdt_provider;
@@ -452,6 +414,7 @@
id_t id; /* DTrace probe ID. */
int n_args; /* Number of arguments. */
struct linker_file *sdtp_lf; /* Module in which we're defined. */
+ DECLARE_ZCOND_FALSE(enabled);
};
struct sdt_provider {
@@ -463,7 +426,7 @@
};
void sdt_probe_stub(uint32_t, uintptr_t, uintptr_t, uintptr_t, uintptr_t,
- uintptr_t, uintptr_t);
+ uintptr_t);
SDT_PROVIDER_DECLARE(sdt);
diff --git a/sys/sys/zcond.h b/sys/sys/zcond.h
new file mode 100644
--- /dev/null
+++ b/sys/sys/zcond.h
@@ -0,0 +1,249 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2024 Marko Vlaić <mvlaic@freebsd.org>
+ *
+ * This code was developed as a Google Summer of Code 2024. project
+ * under the guidance of Bojan Novković <bnovkov@freebsdorg>.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifdef _KERNEL
+#ifndef _SYS_ZCOND_H
+#define _SYS_ZCOND_H
+
+#include <sys/cdefs.h>
+#include <sys/types.h>
+#include <sys/param.h>
+#include <sys/queue.h>
+
+#include <machine/zcond.h>
+
+/*
+ * The zcond interface provides a low-cost mechanism for conditional execution.
+ * It is applicable in situations where branch selection is performed by
+ * inspecting the state of a single boolean flag i.e blocks of the following
+ * form: if(flag) {
+ * // do something
+ * }
+ *
+ * This kind of block is compiled into some sequence of load, test, jump
+ * assembly instructions. The low cost provided by zcond is achieved by "baking
+ * in" a single branch direction at compile time. This means outputting either
+ * an unconditional jump or a nop, while the memory access is avoided.
+ *
+ * When the time comes to switch the branch direction, the current instruction
+ * (jump or nop) is patched at runtime to a corresponding instruction (nop or
+ * jump). Keep in mind that this is an expensive operation, since all cpus
+ * except the one performing the patch need to be halted.
+ *
+ *
+ * To use a zcond, first define it with: ZCOND_DEFINE_TRUE(name) or
+ * ZCOND_DEFINE_FALSE(name) Alternatively, declare it with
+ * ZCOND_DECLARE_TRUE(name) or ZCOND_DECLARE_FALSE(name). Then initialize it
+ * with ZCOND_INIT(true) or ZCOND_INIT(false).
+ *
+ * Use zcond_false(cond) or zcond_true(cond) to inspect the state of a zcond.
+ *
+ * To flip the state of a zcond, use zcond_enable(cond) and zcond_disable(cond).
+ *
+ * This header includes the interface intended to be used by consumers, as well
+ * as some MI code. MD support can be found in sys/<arch>/include/zcond.h and
+ * sys/<arch>/<arch>/zcond_machdep.c
+ */
+
+/*
+ * Describes a single inspection of the zcond state (performed with an if
+ * statement). Holds all the data neccessary to perform a safe instruction
+ * patch.
+ */
+
+/*
+ * A single optimized boolean.
+ */
+struct zcond {
+ int refcnt;
+ SLIST_HEAD(, patch_point) patch_points;
+};
+
+/*
+ * Wrapper types are needed for compile time decision making.
+ */
+struct zcond_true {
+ struct zcond cond;
+};
+
+struct zcond_false {
+ struct zcond cond;
+};
+
+#define ZCOND_ELF_SECTION "set_zcond_patch_points_set"
+#define ZCOND_LINKER_SET zcond_patch_points_set
+
+/*
+ * __zcond_table is an ELF section which keeps
+ * all the data related to the zcond mechanism.
+ * A single entry describes a single patch_point.
+ */
+#define ZCOND_TABLE_ENTRY \
+ ".pushsection " ZCOND_ELF_SECTION ", \"aw\" \n\t" \
+ ".quad 1b \n\t" \
+ ".quad %l[l_true] \n\t" \
+ ".quad %c0 \n\t" \
+ ".quad 0 \n\t" \
+ ".popsection \n\t"
+
+#define ZCOND_SET_START_STOP \
+ do { \
+ __WEAK(__CONCAT(__start_set_, ZCOND_LINKER_SET)); \
+ __WEAK(__CONCAT(__stop_set_, ZCOND_LINKER_SET)); \
+ } while (0);
+
+/*
+ * Emits a __zcond_table entry, describing one patch_point.
+ * Bakes in a nop instruction instruction, so the return value is initially
+ * false.
+ */
+static __always_inline bool
+zcond_nop(struct zcond *const zcond_p)
+{
+ ZCOND_SET_START_STOP
+ asm goto("1: " ZCOND_NOP_ASM ZCOND_TABLE_ENTRY
+ :
+ : "i"(zcond_p)
+ :
+ : l_true);
+
+ return (false);
+l_true:
+ return (true);
+}
+
+/*
+ * Emits a __zcond_table entry, describing one patch_point.
+ * Bakes in a jmp instruction instruction, so the return value is initially
+ * true.
+ */
+static __always_inline bool
+zcond_jmp(struct zcond *const zcond_p)
+{
+ ZCOND_SET_START_STOP
+ asm goto("1:" ZCOND_JMP_ASM " %[l_true] \n\t" ZCOND_TABLE_ENTRY
+ :
+ : "i"(zcond_p)
+ :
+ : l_true);
+ return (false);
+l_true:
+ return (true);
+}
+
+/*
+ * These macros declare and initialize a new zcond.
+ */
+
+#define ZCOND_INIT() \
+ { \
+ { \
+ .patch_points = SLIST_HEAD_INITIALIZER() \
+ } \
+ }
+
+#define DEFINE_ZCOND_TRUE(name) struct zcond_true name = ZCOND_INIT()
+
+#define DEFINE_ZCOND_FALSE(name) struct zcond_false name = ZCOND_INIT()
+
+#define DECLARE_ZCOND_TRUE(name) struct zcond_true name;
+
+#define DECLARE_ZCOND_FALSE(name) struct zcond_false name;
+
+/*
+ * These macros inspect the state of a zcond (is it true or false)
+ * thus instatiating a patch_point.
+ */
+#define zcond_true(cond_wrapped) \
+ ({ \
+ bool branch; \
+ if (__builtin_types_compatible_p(typeof(cond_wrapped), \
+ struct zcond_true)) { \
+ branch = zcond_jmp(&(cond_wrapped.cond)); \
+ } else if (__builtin_types_compatible_p(typeof(cond_wrapped), \
+ struct zcond_false)) { \
+ branch = zcond_nop(&(cond_wrapped.cond)); \
+ } \
+ \
+ branch; \
+ })
+
+#define zcond_false(cond_wrapped) \
+ ({ \
+ bool branch; \
+ if (__builtin_types_compatible_p(typeof(cond_wrapped), \
+ struct zcond_true)) { \
+ branch = zcond_nop(&(cond_wrapped.cond)); \
+ } else if (__builtin_types_compatible_p(typeof(cond_wrapped), \
+ struct zcond_false)) { \
+ branch = zcond_jmp(&(cond_wrapped.cond)); \
+ } \
+ \
+ branch; \
+ })
+
+/*
+ * These macros change the state of a zcond.
+ */
+#define zcond_enable(cond_wrapped) __zcond_toggle(&cond_wrapped.cond, true)
+#define zcond_disable(cond_wrapped) __zcond_toggle(&cond_wrapped.cond, false)
+
+/*
+ * Forward declaration of a struct, defined separately for each architecture in
+ * <machine/zcond.h>
+ */
+struct zcond_md_ctxt;
+
+/*
+ * Change the state of a zcond by safely patching all of its
+ * inspection points with appropriate instructions.
+ */
+void __zcond_toggle(struct zcond *cond, bool enable);
+
+/*
+ * Called before a single patch_point is patched.
+ */
+void zcond_before_patch(struct zcond_md_ctxt *);
+
+/*
+ * Called after a single patch_point was patched.
+ */
+void zcond_after_patch(struct zcond_md_ctxt *);
+
+/*
+ * Calculates the bytes of instruction with which the ins_p inspection point is
+ * to be patched with. insn[] is populated with the instruction bytes and size
+ * is set to the number of instruction bytes.
+ */
+uint8_t *zcond_get_patch_insn(vm_offset_t patch_addr, vm_offset_t lbl_true_addr,
+ size_t *size);
+
+#endif
+#endif
diff --git a/sys/x86/x86/sdt_machdep.c b/sys/x86/x86/sdt_machdep.c
deleted file mode 100644
--- a/sys/x86/x86/sdt_machdep.c
+++ /dev/null
@@ -1,84 +0,0 @@
-/*-
- * SPDX-License-Identifier: BSD-2-Clause
- *
- * Copyright (c) 2024 Mark Johnston <markj@FreeBSD.org>
- */
-
-#include <sys/systm.h>
-#include <sys/sdt.h>
-
-#include <vm/vm.h>
-#include <vm/pmap.h>
-
-#include <machine/md_var.h>
-#include <machine/vmparam.h>
-
-#define SDT_PATCH_SIZE 5
-
-/*
- * Return true if we can overwrite a nop at "patchpoint" with a jump to the
- * target address.
- */
-bool
-sdt_tracepoint_valid(uintptr_t patchpoint, uintptr_t target)
-{
-#ifdef __amd64__
- if (patchpoint < KERNSTART || target < KERNSTART)
- return (false);
-#endif
- if (patchpoint == target ||
- patchpoint + SDT_PATCH_SIZE < patchpoint)
- return (false);
-#ifdef __amd64__
- int64_t offset = target - (patchpoint + SDT_PATCH_SIZE);
- if (offset < -(1l << 31) || offset > (1l << 31))
- return (false);
-#endif
- return (true);
-}
-
-/*
- * Overwrite the copy of _SDT_ASM_PATCH_INSTR at the tracepoint with a jump to
- * the target address.
- */
-void
-sdt_tracepoint_patch(uintptr_t patchpoint, uintptr_t target)
-{
- uint8_t instr[SDT_PATCH_SIZE];
- int32_t disp;
- bool old_wp;
-
- KASSERT(sdt_tracepoint_valid(patchpoint, target),
- ("%s: invalid tracepoint %#jx -> %#jx",
- __func__, (uintmax_t)patchpoint, (uintmax_t)target));
-
- instr[0] = 0xe9;
- disp = target - (patchpoint + SDT_PATCH_SIZE);
- memcpy(&instr[1], &disp, sizeof(disp));
-
- old_wp = disable_wp();
- memcpy((void *)patchpoint, instr, sizeof(instr));
- restore_wp(old_wp);
-}
-
-/*
- * Overwrite the patchpoint with a nop instruction.
- */
-void
-sdt_tracepoint_restore(uintptr_t patchpoint)
-{
- uint8_t instr[SDT_PATCH_SIZE];
- bool old_wp;
-
-#ifdef __amd64__
- KASSERT(patchpoint >= KERNSTART,
- ("%s: invalid patchpoint %#lx", __func__, patchpoint));
-#endif
-
- for (int i = 0; i < SDT_PATCH_SIZE; i++)
- instr[i] = 0x90;
-
- old_wp = disable_wp();
- memcpy((void *)patchpoint, instr, sizeof(instr));
- restore_wp(old_wp);
-}

File Metadata

Mime Type
text/plain
Expires
Tue, May 26, 12:27 PM (11 h, 3 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
28778010
Default Alt Text
D46379.1779798474.diff (43 KB)

Event Timeline