From fae6e9ade46a52976ad5d099643d51cc20876448 Mon Sep 17 00:00:00 2001 From: linfeng Date: Fri, 25 Oct 2024 15:59:57 +0800 Subject: [PATCH] feat(ebpf):[WIP] add eBPF support (#948) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(kprobe): Add basic kprobe support for x86_64 * feat: add ebpf support (#912) - 实现bpf()一部分命令，包括几种基本map，相关的helper函数 - 实现部分perf相关的数据结构 - 暂时为文件实现简单mmap - 实现一个使用kprobe统计syscall 调用次数的ebpf程序 对eBPF支持程度(基本)： - 简单的eBPF程序(没有指定特殊的Map) - 使用内核已经实现的Map的eBPF程序 - 可以和kprobe配合使用 - 内核Map相关的接口定义已经实现，添加新的Map较为简单 不支持的功能: - 区分不同的eBPF程序类型(Network/Cgroup)并限定可调用的helper函数集 - 与内核其它跟踪机制配合(tracepoint) - 其它helper和Map todo - [ ] 修改mmap，需要讨论，因为这个和块缓存层相关 - [x] 添加文档 - [x] 修复可能的错误 - [x] 增加rbpf版本信息 * feat: add /sys/devices/system/cpu/possible file * feat: add /sys/devices/system/cpu/online --- .github/workflows/cache-toolchain.yml | 94 + .github/workflows/makefile.yml | 2 +- docs/index.rst | 1 + docs/kernel/trace/eBPF.md | 324 ++ docs/kernel/trace/ebpf_flow.png | Bin 0 -> 66484 bytes docs/kernel/trace/index.rst | 11 + docs/kernel/trace/kprobe.md | 57 + docs/kernel/trace/kprobe_flow.png | Bin 0 -> 47130 bytes kernel/Cargo.toml | 6 + kernel/Makefile | 4 +- kernel/crates/kprobe/Cargo.toml | 11 + .../crates/kprobe/src/arch/loongarch64/mod.rs | 112 + kernel/crates/kprobe/src/arch/mod.rs | 211 ++ kernel/crates/kprobe/src/arch/rv64/mod.rs | 157 + kernel/crates/kprobe/src/arch/x86/mod.rs | 135 + kernel/crates/kprobe/src/lib.rs | 7 + kernel/crates/rbpf/.appveyor.yml | 21 + kernel/crates/rbpf/.gitignore | 2 + kernel/crates/rbpf/Cargo.toml | 78 + kernel/crates/rbpf/LICENSE-APACHE | 202 ++ kernel/crates/rbpf/LICENSE-MIT | 25 + kernel/crates/rbpf/README.md | 743 +++++ kernel/crates/rbpf/clippy.toml | 1 + kernel/crates/rbpf/examples/disassemble.rs | 26 + kernel/crates/rbpf/examples/helper.rs | 3 + kernel/crates/rbpf/examples/load_elf.rs | 115 + .../rbpf/examples/load_elf__block_a_port.c | 43 + kernel/crates/rbpf/examples/rbpf_plugin.rs | 126 + kernel/crates/rbpf/examples/to_json.rs | 74 + kernel/crates/rbpf/examples/uptime.rs | 78 + kernel/crates/rbpf/mk/appveyor.bat | 72 + kernel/crates/rbpf/rustfmt.toml | 3 + kernel/crates/rbpf/src/asm_parser.rs | 642 ++++ kernel/crates/rbpf/src/assembler.rs | 277 ++ kernel/crates/rbpf/src/cranelift.rs | 1230 +++++++ kernel/crates/rbpf/src/disassembler.rs | 807 +++++ kernel/crates/rbpf/src/ebpf.rs | 635 ++++ kernel/crates/rbpf/src/helpers.rs | 488 +++ kernel/crates/rbpf/src/insn_builder.rs | 2199 +++++++++++++ kernel/crates/rbpf/src/interpreter.rs | 708 ++++ kernel/crates/rbpf/src/jit.rs | 1054 ++++++ kernel/crates/rbpf/src/lib.rs | 1782 ++++++++++ kernel/crates/rbpf/src/no_std_error.rs | 41 + kernel/crates/rbpf/src/stack.rs | 75 + kernel/crates/rbpf/src/verifier.rs | 386 +++ kernel/crates/rbpf/tests/assembler.rs | 655 ++++ kernel/crates/rbpf/tests/common.rs | 97 + kernel/crates/rbpf/tests/cranelift.rs | 2257 +++++++++++++ kernel/crates/rbpf/tests/disassembler.rs | 377 +++ kernel/crates/rbpf/tests/misc.rs | 571 ++++ kernel/crates/rbpf/tests/ubpf_jit_x86_64.rs | 2891 +++++++++++++++++ kernel/crates/rbpf/tests/ubpf_verifier.rs | 177 + kernel/crates/rbpf/tests/ubpf_vm.rs | 2670 +++++++++++++++ kernel/src/arch/riscv64/interrupt/handle.rs | 15 +- kernel/src/arch/riscv64/interrupt/mod.rs | 19 + kernel/src/arch/riscv64/kprobe.rs | 85 + kernel/src/arch/riscv64/mod.rs | 1 + kernel/src/arch/x86_64/interrupt/mod.rs | 20 +- kernel/src/arch/x86_64/interrupt/trap.rs | 25 +- kernel/src/arch/x86_64/kprobe.rs | 65 + kernel/src/arch/x86_64/mod.rs | 1 + kernel/src/bpf/helper/consts.rs | 11 + kernel/src/bpf/helper/mod.rs | 340 ++ kernel/src/bpf/helper/print.rs | 25 + kernel/src/bpf/map/array_map.rs | 283 ++ kernel/src/bpf/map/hash_map.rs | 156 + kernel/src/bpf/map/lru.rs | 151 + kernel/src/bpf/map/mod.rs | 416 +++ kernel/src/bpf/map/queue.rs | 154 + kernel/src/bpf/map/util.rs | 100 + kernel/src/bpf/mod.rs | 50 + kernel/src/bpf/prog/mod.rs | 123 + kernel/src/bpf/prog/util.rs | 112 + kernel/src/bpf/prog/verifier.rs | 131 + kernel/src/common/string.h | 9 + kernel/src/debug/kprobe/args.rs | 66 + kernel/src/debug/kprobe/mod.rs | 183 ++ kernel/src/debug/kprobe/test.rs | 84 + kernel/src/debug/mod.rs | 1 + kernel/src/debug/traceback/traceback.c | 13 + kernel/src/debug/traceback/traceback.h | 3 +- kernel/src/driver/base/cpu.rs | 96 +- kernel/src/driver/base/init.rs | 4 +- kernel/src/exception/debug.rs | 33 + kernel/src/exception/ebreak.rs | 37 + kernel/src/exception/mod.rs | 2 + kernel/src/filesystem/vfs/file.rs | 31 +- kernel/src/filesystem/vfs/mod.rs | 3 + kernel/src/include/bindings/linux_bpf.rs | 2430 ++++++++++++++ kernel/src/include/bindings/mod.rs | 10 +- kernel/src/init/init.rs | 6 +- kernel/src/lib.rs | 4 +- kernel/src/mm/fault.rs | 6 +- kernel/src/mm/ucontext.rs | 10 +- kernel/src/perf/bpf.rs | 333 ++ kernel/src/perf/kprobe.rs | 159 + kernel/src/perf/mod.rs | 337 ++ kernel/src/perf/util.rs | 123 + kernel/src/smp/cpu/mod.rs | 1 - kernel/src/syscall/mod.rs | 14 + kernel/src/time/syscall.rs | 4 +- user/apps/test_ebpf/.gitignore | 3 + user/apps/test_ebpf/Cargo.toml | 16 + user/apps/test_ebpf/Makefile | 61 + user/apps/test_ebpf/src/main.rs | 60 + .../test_ebpf/syscall_ebpf/.cargo/config.toml | 2 + user/apps/test_ebpf/syscall_ebpf/.gitignore | 9 + .../syscall_ebpf/.vscode/settings.json | 3 + user/apps/test_ebpf/syscall_ebpf/Cargo.toml | 3 + user/apps/test_ebpf/syscall_ebpf/README.md | 32 + .../syscall_ebpf-common/Cargo.toml | 14 + .../syscall_ebpf-common/src/lib.rs | 1 + .../syscall_ebpf-ebpf/.cargo/config.toml | 6 + .../syscall_ebpf-ebpf/.helix/config.toml | 2 + .../syscall_ebpf-ebpf/.vim/coc-settings.json | 4 + .../syscall_ebpf-ebpf/.vscode/settings.json | 4 + .../syscall_ebpf/syscall_ebpf-ebpf/Cargo.toml | 33 + .../syscall_ebpf-ebpf/rust-toolchain.toml | 13 + .../syscall_ebpf-ebpf/src/main.rs | 44 + .../test_ebpf/syscall_ebpf/xtask/Cargo.toml | 8 + .../test_ebpf/syscall_ebpf/xtask/src/build.rs | 42 + .../syscall_ebpf/xtask/src/build_ebpf.rs | 67 + .../test_ebpf/syscall_ebpf/xtask/src/main.rs | 36 + .../test_ebpf/syscall_ebpf/xtask/src/run.rs | 55 + user/apps/test_tokio/src/main.rs | 2 - user/dadk/config/test_ebpf_0_1_0.dadk | 23 + 126 files changed, 29529 insertions(+), 62 deletions(-) create mode 100644 .github/workflows/cache-toolchain.yml create mode 100644 docs/kernel/trace/eBPF.md create mode 100644 docs/kernel/trace/ebpf_flow.png create mode 100644 docs/kernel/trace/index.rst create mode 100644 docs/kernel/trace/kprobe.md create mode 100644 docs/kernel/trace/kprobe_flow.png create mode 100644 kernel/crates/kprobe/Cargo.toml create mode 100644 kernel/crates/kprobe/src/arch/loongarch64/mod.rs create mode 100644 kernel/crates/kprobe/src/arch/mod.rs create mode 100644 kernel/crates/kprobe/src/arch/rv64/mod.rs create mode 100644 kernel/crates/kprobe/src/arch/x86/mod.rs create mode 100644 kernel/crates/kprobe/src/lib.rs create mode 100644 kernel/crates/rbpf/.appveyor.yml create mode 100644 kernel/crates/rbpf/.gitignore create mode 100644 kernel/crates/rbpf/Cargo.toml create mode 100644 kernel/crates/rbpf/LICENSE-APACHE create mode 100644 kernel/crates/rbpf/LICENSE-MIT create mode 100644 kernel/crates/rbpf/README.md create mode 100644 kernel/crates/rbpf/clippy.toml create mode 100644 kernel/crates/rbpf/examples/disassemble.rs create mode 100644 kernel/crates/rbpf/examples/helper.rs create mode 100644 kernel/crates/rbpf/examples/load_elf.rs create mode 100644 kernel/crates/rbpf/examples/load_elf__block_a_port.c create mode 100644 kernel/crates/rbpf/examples/rbpf_plugin.rs create mode 100644 kernel/crates/rbpf/examples/to_json.rs create mode 100644 kernel/crates/rbpf/examples/uptime.rs create mode 100644 kernel/crates/rbpf/mk/appveyor.bat create mode 100644 kernel/crates/rbpf/rustfmt.toml create mode 100644 kernel/crates/rbpf/src/asm_parser.rs create mode 100644 kernel/crates/rbpf/src/assembler.rs create mode 100644 kernel/crates/rbpf/src/cranelift.rs create mode 100644 kernel/crates/rbpf/src/disassembler.rs create mode 100644 kernel/crates/rbpf/src/ebpf.rs create mode 100644 kernel/crates/rbpf/src/helpers.rs create mode 100644 kernel/crates/rbpf/src/insn_builder.rs create mode 100644 kernel/crates/rbpf/src/interpreter.rs create mode 100644 kernel/crates/rbpf/src/jit.rs create mode 100644 kernel/crates/rbpf/src/lib.rs create mode 100644 kernel/crates/rbpf/src/no_std_error.rs create mode 100644 kernel/crates/rbpf/src/stack.rs create mode 100644 kernel/crates/rbpf/src/verifier.rs create mode 100644 kernel/crates/rbpf/tests/assembler.rs create mode 100644 kernel/crates/rbpf/tests/common.rs create mode 100644 kernel/crates/rbpf/tests/cranelift.rs create mode 100644 kernel/crates/rbpf/tests/disassembler.rs create mode 100644 kernel/crates/rbpf/tests/misc.rs create mode 100644 kernel/crates/rbpf/tests/ubpf_jit_x86_64.rs create mode 100644 kernel/crates/rbpf/tests/ubpf_verifier.rs create mode 100644 kernel/crates/rbpf/tests/ubpf_vm.rs create mode 100644 kernel/src/arch/riscv64/kprobe.rs create mode 100644 kernel/src/arch/x86_64/kprobe.rs create mode 100644 kernel/src/bpf/helper/consts.rs create mode 100644 kernel/src/bpf/helper/mod.rs create mode 100644 kernel/src/bpf/helper/print.rs create mode 100644 kernel/src/bpf/map/array_map.rs create mode 100644 kernel/src/bpf/map/hash_map.rs create mode 100644 kernel/src/bpf/map/lru.rs create mode 100644 kernel/src/bpf/map/mod.rs create mode 100644 kernel/src/bpf/map/queue.rs create mode 100644 kernel/src/bpf/map/util.rs create mode 100644 kernel/src/bpf/mod.rs create mode 100644 kernel/src/bpf/prog/mod.rs create mode 100644 kernel/src/bpf/prog/util.rs create mode 100644 kernel/src/bpf/prog/verifier.rs create mode 100644 kernel/src/debug/kprobe/args.rs create mode 100644 kernel/src/debug/kprobe/mod.rs create mode 100644 kernel/src/debug/kprobe/test.rs create mode 100644 kernel/src/exception/debug.rs create mode 100644 kernel/src/exception/ebreak.rs create mode 100644 kernel/src/include/bindings/linux_bpf.rs create mode 100644 kernel/src/perf/bpf.rs create mode 100644 kernel/src/perf/kprobe.rs create mode 100644 kernel/src/perf/mod.rs create mode 100644 kernel/src/perf/util.rs create mode 100644 user/apps/test_ebpf/.gitignore create mode 100644 user/apps/test_ebpf/Cargo.toml create mode 100644 user/apps/test_ebpf/Makefile create mode 100644 user/apps/test_ebpf/src/main.rs create mode 100644 user/apps/test_ebpf/syscall_ebpf/.cargo/config.toml create mode 100644 user/apps/test_ebpf/syscall_ebpf/.gitignore create mode 100644 user/apps/test_ebpf/syscall_ebpf/.vscode/settings.json create mode 100644 user/apps/test_ebpf/syscall_ebpf/Cargo.toml create mode 100644 user/apps/test_ebpf/syscall_ebpf/README.md create mode 100644 user/apps/test_ebpf/syscall_ebpf/syscall_ebpf-common/Cargo.toml create mode 100644 user/apps/test_ebpf/syscall_ebpf/syscall_ebpf-common/src/lib.rs create mode 100644 user/apps/test_ebpf/syscall_ebpf/syscall_ebpf-ebpf/.cargo/config.toml create mode 100644 user/apps/test_ebpf/syscall_ebpf/syscall_ebpf-ebpf/.helix/config.toml create mode 100644 user/apps/test_ebpf/syscall_ebpf/syscall_ebpf-ebpf/.vim/coc-settings.json create mode 100644 user/apps/test_ebpf/syscall_ebpf/syscall_ebpf-ebpf/.vscode/settings.json create mode 100644 user/apps/test_ebpf/syscall_ebpf/syscall_ebpf-ebpf/Cargo.toml create mode 100644 user/apps/test_ebpf/syscall_ebpf/syscall_ebpf-ebpf/rust-toolchain.toml create mode 100644 user/apps/test_ebpf/syscall_ebpf/syscall_ebpf-ebpf/src/main.rs create mode 100644 user/apps/test_ebpf/syscall_ebpf/xtask/Cargo.toml create mode 100644 user/apps/test_ebpf/syscall_ebpf/xtask/src/build.rs create mode 100644 user/apps/test_ebpf/syscall_ebpf/xtask/src/build_ebpf.rs create mode 100644 user/apps/test_ebpf/syscall_ebpf/xtask/src/main.rs create mode 100644 user/apps/test_ebpf/syscall_ebpf/xtask/src/run.rs create mode 100644 user/dadk/config/test_ebpf_0_1_0.dadk diff --git a/.github/workflows/cache-toolchain.yml b/.github/workflows/cache-toolchain.yml new file mode 100644 index 00000000..4b6099c5 --- /dev/null +++ b/.github/workflows/cache-toolchain.yml @@ -0,0 +1,94 @@ +name: Cache toolchain + +on: workflow_call + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + + - name: Cache DragonOS GCC + id: cache-dragonos-gcc + uses: actions/cache@v3 + env: + cache-name: cache-dragonos-gcc + with: + path: | + ~/opt + ~/.bashrc + key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('tools/build_gcc_toolchain.sh') }} + + - if: ${{ steps.cache-dragonos-gcc.outputs.cache-hit != 'true' }} + name: build dragonos-gcc + continue-on-error: true + run: | + bash tools/build_gcc_toolchain.sh -f + + - uses: ./.github/actions/install-apt-packages + + - name: Cache build tools + id: cache-build-tools + uses: actions/cache@v3 + env: + cache-name: cache-build-tools + dadk_version: 0.1.11 + with: + path: | + ~/.cargo + ~/.rustup + ~/.bashrc + ~/opt + key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ env.dadk_version }}-${{ hashFiles('.github/workflows/cache-toolchain.yml') }}-${{ hashFiles('tools/install_musl_gcc.sh') }} + + - if: ${{ steps.cache-build-tools.outputs.cache-hit != 'true' }} + name: Install toolchain + continue-on-error: false + run: | + USE_GITHUB=1 bash tools/install_musl_gcc.sh + + cargo install cargo-binutils + rustup toolchain install nightly-x86_64-unknown-linux-gnu + rustup toolchain install nightly-2024-07-23-x86_64-unknown-linux-gnu + rustup toolchain install nightly-2023-08-15-x86_64-unknown-linux-gnu + rustup component add rust-src --toolchain nightly-2024-07-23-x86_64-unknown-linux-gnu + rustup component add rust-src --toolchain nightly-2023-08-15-x86_64-unknown-linux-gnu + rustup target add x86_64-unknown-none --toolchain nightly-2024-07-23-x86_64-unknown-linux-gnu + rustup target add x86_64-unknown-none --toolchain nightly-2023-08-15-x86_64-unknown-linux-gnu + + rustup toolchain install nightly-2024-07-23-riscv64gc-unknown-linux-gnu --force-non-host + rustup toolchain install nightly-2023-08-15-riscv64gc-unknown-linux-gnu --force-non-host + rustup target add riscv64gc-unknown-none-elf --toolchain nightly-2024-07-23-riscv64gc-unknown-linux-gnu + rustup target add riscv64imac-unknown-none-elf --toolchain nightly-2024-07-23-riscv64gc-unknown-linux-gnu + rustup target add riscv64gc-unknown-none-elf --toolchain nightly-2023-08-15-riscv64gc-unknown-linux-gnu + rustup target add riscv64imac-unknown-none-elf --toolchain nightly-2023-08-15-riscv64gc-unknown-linux-gnu + + rustup component add rust-src --toolchain nightly-x86_64-unknown-linux-gnu + rustup component add rust-src + rustup component add llvm-tools-preview + + rustup component add rustfmt + rustup component add rustfmt --toolchain nightly-x86_64-unknown-linux-gnu + rustup component add rustfmt --toolchain nightly-2024-07-23-x86_64-unknown-linux-gnu + rustup component add rustfmt --toolchain nightly-2023-08-15-x86_64-unknown-linux-gnu + rustup component add rustfmt --toolchain nightly-2024-07-23-riscv64gc-unknown-linux-gnu + rustup component add rustfmt --toolchain nightly-2023-08-15-riscv64gc-unknown-linux-gnu + + rustup default nightly-2024-07-23 + + cargo install dadk --version 0.1.11 + + + userapp_musl_toolchain="nightly-2023-08-15-x86_64-unknown-linux-gnu" + + rustup toolchain install ${userapp_musl_toolchain} + rustup component add --toolchain ${userapp_musl_toolchain} rust-src + rustup target add --toolchain ${userapp_musl_toolchain} x86_64-unknown-linux-musl + + rustup target add x86_64-unknown-linux-musl --toolchain nightly-2024-07-23-x86_64-unknown-linux-gnu + rustup component add rust-src --toolchain nightly-2024-07-23-x86_64-unknown-linux-gnu + cargo install bpf-linker + + diff --git a/.github/workflows/makefile.yml b/.github/workflows/makefile.yml index 03ddb7ec..cfb55083 100644 --- a/.github/workflows/makefile.yml +++ b/.github/workflows/makefile.yml @@ -51,7 +51,7 @@ jobs: env: ARCH: ${{ matrix.arch }} HOME: /root - run: bash -c "source /root/.cargo/env && cd kernel && make test" + run: bash -c "source /root/.cargo/env && cd kernel && make test && make test-rbpf" build-x86_64: diff --git a/docs/index.rst b/docs/index.rst index be25163d..5d2aa82b 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -31,6 +31,7 @@ kernel/ktest/index kernel/cpu_arch/index kernel/libs/index + kernel/trace/index .. toctree:: diff --git a/docs/kernel/trace/eBPF.md b/docs/kernel/trace/eBPF.md new file mode 100644 index 00000000..b9a1bf1a --- /dev/null +++ b/docs/kernel/trace/eBPF.md @@ -0,0 +1,324 @@ +# eBPF + +> 作者: 陈林峰 +> +> Email: chenlinfeng25@outlook.com + +## 概述 + +eBPF 是一项革命性的技术，起源于 Linux 内核，它可以在特权上下文中（如操作系统内核）运行沙盒程序。它用于安全有效地扩展内核的功能，而无需通过更改内核源代码或加载内核模块的方式来实现。 + +从历史上看，由于内核具有监督和控制整个系统的特权，操作系统一直是实现可观测性、安全性和网络功能的理想场所。同时，由于操作系统内核的核心地位和对稳定性和安全性的高要求，操作系统内核很难快速迭代发展。因此在传统意义上，与在操作系统本身之外实现的功能相比，操作系统级别的创新速度要慢一些。 + +eBPF 从根本上改变了这个方式。通过允许在操作系统中运行沙盒程序的方式，应用程序开发人员可以运行 eBPF 程序，以便在运行时向操作系统添加额外的功能。然后在 JIT 编译器和验证引擎的帮助下，操作系统确保它像本地编译的程序一样具备安全性和执行效率。这引发了一股基于 eBPF 的项目热潮，它们涵盖了广泛的用例，包括下一代网络实现、可观测性和安全功能等领域。 + +## eBPF In DragonOS + +在一个新的OS上添加eBPF的支持需要了解eBPF的运行过程，通常，eBPF需要用户态工具和内核相关基础设施配合才能发挥其功能。而新的OS通常会兼容Linux上的应用程序，这可以进一步简化对用户态工具的移植工作，只要内核实现相关的系统调用和功能，就可以配合已有的工具完成eBPF的支持。 + +## eBPF的运行流程 + +![image-20240909165945192](./ebpf_flow.png) + +如图所示，eBPF程序的运行过程分为三个主要步骤： + +1. 源代码->二进制 + 1. 用户可以使用python/C/Rust编写eBPF程序，并使用相关的工具链编译源代码到二进制程序 + 2. 这个步骤中，用户需要合理使用helper函数丰富eBPF程序功能 +2. 加载eBPF程序 + 1. 用户态的工具库会封装内核提供的系统调用接口，以简化用户的工作。用户态工具对eBPF程序经过预处理后发出系统调用，请求内核加载eBPF程序。 + 1. 内核首先会对eBPF程序进行验证，检查程序的正确性和合法性，同时也会对程序做进一步的处理 + 1. 内核会根据用户请求，将eBPF程序附加到内核的挂载点上(kprobe/uprobe/trace_point) + 1. 在内核运行期间，当这些挂载点被特定的事件触发， eBPF程序就会被执行 +3. 数据交互 + 1. eBPF程序可以收集内核的信息，用户工具可以选择性的获取这些信息 + 2. eBPF程序可以直接将信息输出到文件中，用户工具通过读取和解析文件中的内容拿到信息 + 3. eBPF程序通过Map在内核和用户态之间共享和交换数据 + + + +## 用户态支持 + +用户态的eBPF工具库有很多，比如C的libbpf，python的bcc, Rust的Aya，总体来说，这些工具的处理流程都大致相同。DragonOS当前支持[Aya](https://github.com/aya-rs/aya)框架编写的eBPF程序，以Aya为例，用户态的工具的处理过程如下: + +1. 提供eBPF使用的helper函数和Map抽象，方便实现eBPF程序 +2. 处理编译出来的eBPF程序，调用系统调用创建Map，获得对应的文件描述符 +3. 根据需要，更新Map的值(.data) +4. 根据重定位信息，对eBPF程序的相关指令做修改 +5. 根据内核版本，对eBPF程序中的bpf to bpf call进行处理 +6. 加载eBPF程序到内核中 +7. 对系统调用封装，提供大量的函数帮助访问eBPF的信息并与内核交互 + +DragonOS对Aya 库的支持并不完整。通过对Aya库的删减，我们实现了一个较小的[tiny-aya](https://github.com/DragonOS-Community/tiny-aya)。为了确保后期对Aya的兼容，tiny-aya只对Aya中的核心工具aya做了修改**，其中一些函数被禁用，因为这些函数的所需的系统调用或者文件在DragonOS中还未实现**。 + +### Tokio + +Aya需要使用异步运行时，通过增加一些系统调用和修复一些错误DragonOS现在已经支持基本的tokio运行时。 + +### 使用Aya创建eBPF程序 + +与Aya官方提供的[文档](https://aya-rs.dev/book/start/development/)所述，只需要根据其流程安装对应的Rust工具链，就可以按照模板创建eBPF项目。以当前实现的`syscall_ebf`为例，这个程序的功能是统计系统调用的次数，并将其存储在一个HashMap中。 + +``` +├── Cargo.toml +├── README.md +├── syscall_ebpf +├── syscall_ebpf-common +├── syscall_ebpf-ebpf +└── xtask +``` + +在user/app目录中，项目结构如上所示： + +- `syscall_ebpf-ebpf`是 eBPF代码的实现目录，其会被编译到字节码 +- `syscall_ebpf-common` 是公共库，方便内核和用户态进行信息交互 +- `syscall_ebpf` 是用户态程序，其负责加载eBPF程序并获取eBPF程序产生的数据 +- `xtask` 是一个命令行工具，方便用户编译和运行用户态程序 + +为了在DragonOS中运行用户态程序，暂时还不能直接使用模板创建的项目： + +1. 这个项目不符合DragonOS对用户程序的项目结构要求，当然这可以通过稍加修改完成 +2. 因为DragonOS对tokio运行时的支持还不是完整体，需要稍微修改一下使用方式 + +``` +#[tokio::main(flavor = "current_thread")] +async fn main() -> Result<(), Box> { +``` + +3. 因为对Aya支持不是完整体，因此项目依赖的aya和aya-log需要换成tiny-aya中的实现。 + +``` +[dependencies] +aya = { git = "https://github.com/DragonOS-Community/tiny-aya.git" } +aya-log = { git = "https://github.com/DragonOS-Community/tiny-aya.git" } +``` + +只需要稍加修改，就可以利用Aya现有的工具完成eBPF程序的实现。 + +## 内核态支持 + +内核态支持主要为三个部分： + +1. kprobe实现：位于目录`kernel/crates/kprobe` +2. rbpf运行时：位于目录`kernel/crates/rbpf` +3. 系统调用支持 +4. helper函数支持 + +### rbpf + +由于rbpf之前只是用于运行一些简单的eBPF程序，其需要通过一些修改才能运行更复杂的程序。 + +1. 增加bpf to bpf call 的支持：通过增加新的栈抽象和保存和恢复必要的寄存器数据 +2. 关闭内部不必要的内存检查，这通常由内核的验证器完成 +3. 增加带所有权的数据结构避免生命周期的限制 + + + +### 系统调用 + +eBPF相关的系统调用都集中在`bpf()` 上，通过参数cmd来进一步区分功能，目前对其支持如下: + +```rust +pub fn bpf(cmd: bpf_cmd, attr: &bpf_attr) -> Result { + let res = match cmd { + // Map related commands + bpf_cmd::BPF_MAP_CREATE => map::bpf_map_create(attr), + bpf_cmd::BPF_MAP_UPDATE_ELEM => map::bpf_map_update_elem(attr), + bpf_cmd::BPF_MAP_LOOKUP_ELEM => map::bpf_lookup_elem(attr), + bpf_cmd::BPF_MAP_GET_NEXT_KEY => map::bpf_map_get_next_key(attr), + bpf_cmd::BPF_MAP_DELETE_ELEM => map::bpf_map_delete_elem(attr), + bpf_cmd::BPF_MAP_LOOKUP_AND_DELETE_ELEM => map::bpf_map_lookup_and_delete_elem(attr), + bpf_cmd::BPF_MAP_LOOKUP_BATCH => map::bpf_map_lookup_batch(attr), + bpf_cmd::BPF_MAP_FREEZE => map::bpf_map_freeze(attr), + // Program related commands + bpf_cmd::BPF_PROG_LOAD => prog::bpf_prog_load(attr), + // Object creation commands + bpf_cmd::BPF_BTF_LOAD => { + error!("bpf cmd {:?} not implemented", cmd); + return Err(SystemError::ENOSYS); + } + ty => { + unimplemented!("bpf cmd {:?} not implemented", ty) + } + }; + res +} +``` + +其中对创建Map命令会再次细分，以确定具体的Map类型，目前我们对通用的Map基本添加了支持: + +```rust +bpf_map_type::BPF_MAP_TYPE_ARRAY +bpf_map_type::BPF_MAP_TYPE_PERCPU_ARRAY +bpf_map_type::BPF_MAP_TYPE_PERF_EVENT_ARRAY +bpf_map_type::BPF_MAP_TYPE_HASH +bpf_map_type::BPF_MAP_TYPE_PERCPU_HASH +bpf_map_type::BPF_MAP_TYPE_QUEUE +bpf_map_type::BPF_MAP_TYPE_STACK +bpf_map_type::BPF_MAP_TYPE_LRU_HASH +bpf_map_type::BPF_MAP_TYPE_LRU_PERCPU_HASH + +bpf_map_type::BPF_MAP_TYPE_CPUMAP +| bpf_map_type::BPF_MAP_TYPE_DEVMAP +| bpf_map_type::BPF_MAP_TYPE_DEVMAP_HASH => { + error!("bpf map type {:?} not implemented", map_meta.map_type); + Err(SystemError::EINVAL)? +} +``` + +所有的Map都会实现定义好的接口，这个接口参考Linux的实现定义: + +```rust +pub trait BpfMapCommonOps: Send + Sync + Debug + CastFromSync { + /// Lookup an element in the map. + /// + /// See https://ebpf-docs.dylanreimerink.nl/linux/helper-function/bpf_map_lookup_elem/ + fn lookup_elem(&mut self, _key: &[u8]) -> Result> { + Err(SystemError::ENOSYS) + } + /// Update an element in the map. + /// + /// See https://ebpf-docs.dylanreimerink.nl/linux/helper-function/bpf_map_update_elem/ + fn update_elem(&mut self, _key: &[u8], _value: &[u8], _flags: u64) -> Result<()> { + Err(SystemError::ENOSYS) + } + /// Delete an element from the map. + /// + /// See https://ebpf-docs.dylanreimerink.nl/linux/helper-function/bpf_map_delete_elem/ + fn delete_elem(&mut self, _key: &[u8]) -> Result<()> { + Err(SystemError::ENOSYS) + } + /// For each element in map, call callback_fn function with map, + /// callback_ctx and other map-specific parameters. + /// + /// See https://ebpf-docs.dylanreimerink.nl/linux/helper-function/bpf_for_each_map_elem/ + fn for_each_elem(&mut self, _cb: BpfCallBackFn, _ctx: *const u8, _flags: u64) -> Result { + Err(SystemError::ENOSYS) + } + /// Look up an element with the given key in the map referred to by the file descriptor fd, + /// and if found, delete the element. + fn lookup_and_delete_elem(&mut self, _key: &[u8], _value: &mut [u8]) -> Result<()> { + Err(SystemError::ENOSYS) + } + /// perform a lookup in percpu map for an entry associated to key on cpu. + fn lookup_percpu_elem(&mut self, _key: &[u8], cpu: u32) -> Result> { + Err(SystemError::ENOSYS) + } + /// Get the next key in the map. If key is None, get the first key. + /// + /// Called from syscall + fn get_next_key(&self, _key: Option<&[u8]>, _next_key: &mut [u8]) -> Result<()> { + Err(SystemError::ENOSYS) + } + /// Push an element value in map. + fn push_elem(&mut self, _value: &[u8], _flags: u64) -> Result<()> { + Err(SystemError::ENOSYS) + } + /// Pop an element value from map. + fn pop_elem(&mut self, _value: &mut [u8]) -> Result<()> { + Err(SystemError::ENOSYS) + } + /// Peek an element value from map. + fn peek_elem(&self, _value: &mut [u8]) -> Result<()> { + Err(SystemError::ENOSYS) + } + /// Freeze the map. + /// + /// It's useful for .rodata maps. + fn freeze(&self) -> Result<()> { + Err(SystemError::ENOSYS) + } + /// Get the first value pointer. + fn first_value_ptr(&self) -> *const u8 { + panic!("value_ptr not implemented") + } +} +``` + +联通eBPF和kprobe的系统调用是[`perf_event_open`](https://man7.org/linux/man-pages/man2/perf_event_open.2.html)，这个系统调用在Linux中非常复杂，因此Dragon中并没有按照Linux进行实现，目前只支持其中两个功能: + + + +```rust +match args.type_ { + // Kprobe + // See /sys/bus/event_source/devices/kprobe/type + perf_type_id::PERF_TYPE_MAX => { + let kprobe_event = kprobe::perf_event_open_kprobe(args); + Box::new(kprobe_event) + } + perf_type_id::PERF_TYPE_SOFTWARE => { + // For bpf prog output + assert_eq!(args.config, perf_sw_ids::PERF_COUNT_SW_BPF_OUTPUT); + assert_eq!( + args.sample_type, + Some(perf_event_sample_format::PERF_SAMPLE_RAW) + ); + let bpf_event = bpf::perf_event_open_bpf(args); + Box::new(bpf_event) + } +} +``` + +- 其中一个`PERF_TYPE_SOFTWARE`是用来创建软件定义的事件，`PERF_COUNT_SW_BPF_OUTPUT` 确保这个事件用来采集bpf的输出。 +- `PERF_TYPE_MAX` 通常指示创建kprobe/uprobe事件，也就是用户程序使用kprobe的途径之一，用户程序可以将eBPF程序绑定在这个事件上 + +同样的，perf不同的事件也实现定义的接口: + +```rust +pub trait PerfEventOps: Send + Sync + Debug + CastFromSync + CastFrom { + fn mmap(&self, _start: usize, _len: usize, _offset: usize) -> Result<()> { + panic!("mmap not implemented for PerfEvent"); + } + fn set_bpf_prog(&self, _bpf_prog: Arc) -> Result<()> { + panic!("set_bpf_prog not implemented for PerfEvent"); + } + fn enable(&self) -> Result<()> { + panic!("enable not implemented"); + } + fn disable(&self) -> Result<()> { + panic!("disable not implemented"); + } + fn readable(&self) -> bool { + panic!("readable not implemented"); + } +} +``` + +这个接口目前并不稳定。 + +### helper函数支持 + +用户态工具通过系统调用和内核进行通信，完成eBPF数据的设置、交换。在内核中，eBPF程序的运行也需要内核的帮助，单独的eBPF程序并没有什么太大的用处，因此其会调用内核提供的`helper` 函数完成对内核资源的访问。 + +目前已经支持的大多数`helper` 函数是与Map操作相关: + +```rust +/// Initialize the helper functions. +pub fn init_helper_functions() { + let mut map = BTreeMap::new(); + unsafe { + // Map helpers::Generic map helpers + map.insert(1, define_func!(raw_map_lookup_elem)); + map.insert(2, define_func!(raw_map_update_elem)); + map.insert(3, define_func!(raw_map_delete_elem)); + map.insert(164, define_func!(raw_map_for_each_elem)); + map.insert(195, define_func!(raw_map_lookup_percpu_elem)); + // map.insert(93,define_func!(raw_bpf_spin_lock); + // map.insert(94,define_func!(raw_bpf_spin_unlock); + // Map helpers::Perf event array helpers + map.insert(25, define_func!(raw_perf_event_output)); + // Probe and trace helpers::Memory helpers + map.insert(4, define_func!(raw_bpf_probe_read)); + // Print helpers + map.insert(6, define_func!(trace_printf)); + + // Map helpers::Queue and stack helpers + map.insert(87, define_func!(raw_map_push_elem)); + map.insert(88, define_func!(raw_map_pop_elem)); + map.insert(89, define_func!(raw_map_peek_elem)); + } + BPF_HELPER_FUN_SET.init(map); +} +``` + diff --git a/docs/kernel/trace/ebpf_flow.png b/docs/kernel/trace/ebpf_flow.png new file mode 100644 index 0000000000000000000000000000000000000000..fc46be5d2db7d7dcac4a8b21f449252cc4f8cc32 GIT binary patch literal 66484 zcmeFZWk8hc_CBoJKt-ffI-~^wX=%}+OG1!VVCay}fvu>3gaSjC(v5&L45Bc=(A^*m zIdsDS!+W#0dXD>?^Z)#Q`0Y>Z!FgicE3S2|YZ3TJMV8W|EC!9OhVqt)Ugo%=QGM z6<0SaMz;M_!vu3=GCfaZ{1n0lXde;YDxW z@^VCl`0h^%Lm&+?aJF+I?5O1!fu*|MOk!eU6ZMqf2qI{n6dYWPiW&9iHMNVx#I|rY zG!wWJ5tN9yTPBeA!^0oHzIPcsDTD$7`K*?5b>Zi)RUW{LF8=u5IguZ~{=drt8~y*$ z2;yIYM$2qtc%jELEt57QX`KvJ#SjYCTc{^~Y<>X&f(r1nG$~h8@W_cI5l2#?G@`pJ z-~4wj1aNVvA?1Nz)1y9gC;N;Ulv)jYA2?pW6H^VVeuMpEaW!!Z6~!4YWWPJzxYe1! z12Oup9!lMfZWmJ2%6g=)ldiDnxif(o&)q@p^x}@^KO-(PAY&jDUAIslWJM6F$;Jnp zv+YT>Hd9}#%{m3!X^o0f99zPT%kCHG6Xa;sQc>ot)z=1-vW6Z>-5k4eZGeovD(}~W z%edJkr>;t%o8Dz|zrRK?ro-vAH!ieup#WV@R+jZJAQP$D4BhRb^+U9=JcN<2y%B*&AT>IAvrY4a=cUO}Nul1ijhv(c^ zF5N)>9>fxaVq1;|68nx)P*bDGxN$@TR)we>m4{31Mg9z-t}Jue*r6g5-;r{INeMdF z)xOV~*)Og1RT3pCwI->FQ}up4PRJdUO{|1L2)=~%q~YWM4KwP)p#t22BqdlNfY+k0 zBsI9Ou+YJ({wT~!p{r)N8ao$2K28F?@6YB(YxzoTZtlgoMfFTAbff}&e{V(}rsoD{ z^IQG5Q;Q~n(vC@T*s0L%9eq$qL_N-d+f6KwXKP#>tDwzWlXailuA=D0b6f73St)OI z+0QC^;=$k?iCErD7f07Ei+VrVE6rsMQa!DB+lh$f z;@t+9YKtkdX_4!bL8wSY0c&hWCT!I9iO}OZi{5^=qq(EGKvHafzMk~G=l7cwC70EM z?8mPXlW`&&?eGXOpMBRJ!2AO`6#w#d&opA0SegEu?0V;H4WscA*~v%lhwD=Uq^#Fg zhl&g%*}uM^5xGMrw7U162sWEeBZ@ehiFjsEU66EF${Rc1oyu6{lB=7cl&BEdM+lE0 z!AI__9`|-6PWRS5d|0lOjFVp)5SVaJ_1c;Xk_mNw9ygUmclmek>X3~^H)+ka#~ZY> zldxRf-~Bvsjjv%}Yi zc+t(0QZ%1DXB*gPk!$6l-d#Bb$;pM3Lh7F_Bzk$s3uQJI!Lc-i(u`KfzdXMIZ6`j- zP>3A6$Go3+B1^cM++W8AF=zU#Xo#zeqt1uW=F3bs1eVz2ykQ*An!}lg`Jn0?H*|V4 zB|klVVY%^MC2__^%=cvS;RA&m?%4b3cC#qQ5KjnCjrAa1xjlCjcd{cOL?Q=my13w>WD@A3`O42 zG2RjhX$+#Rqv~@a{I(ww@E&+_wJ*04&7Zp5y8|HHItyS*DUcVbGQ0eZl#O36qy`%i zr)>R*zwFy6{fSZzo$1VXR}c%s8k3NU=M9(8C#V>ry+OSsagO!xp)|Aby5z~~8yu8m zgYIa8N*VDlMEL5=ie~z$S^C9k5wZ_{Pa6?xW-|e_Ry97gIjj$~VNO0FBMwwhgkHr|hV@OsnciO|kbrE=>_WV$LHj_Ogd37q0kl1gE_-cL+LcrOidp1MGPANza0T>iy63Q>>Wlntr+$rbt=(ZwgYl859`U<r zP+}E+#4fQis?t8!95(FE9!0Q}L0R2) z?wY|+ipNS<>bvBc)aAYnSn1;vpOyZKMXR|$x{|QOotSr$f+~@>HRY6BdYj~w z)sT~&_Q(%XLf5D-8Gc>>Xworc@vKZ|6epBlnELv4qLamvKV@ClR@=gepj|L#)DKk0 zk1>T?j%B?89?-HxrC{Dco)WclxE%}iJ@I%@tI{+$s4d>!ph+66n&ObR$XQfJzEku` zZ*VSMQ4)4$YHMLw1gSAkNhfoiBg%+6(z5)vh6Jpja3h;d-6TQX9^ zirXlVxK$?eH_jq6ZAU3HM+OQDGphJ)bC5>g<-nB?B+!IcY$gOLS1Db1QNdJ)fs|y| z+1`6$wf#=^SF$Xf(2FjI-aW2|13EHJv#1<1Zl7|4x(4h{4#xj_ERv0sVwywT&CJh% z1O4sNv3G6VvFDTaw~-lpCL~b=tSvN{cPB@) zn-DGOUuo){EMb)+)t}Qr9q6YhF!;rHN*UnLbV)? z%wkJrQ)-b4NF78&`T4R_s?xW;9N0-+)1s`UH~Ge4K4Zr4H~*-dVdq8lC9*27d9hM@ z7*@N_z0y&-&m91sH;KAl*qISZ0aR{1Ko?Yd=>5BCd#g#hwT{LsBa!NvD#TO69;xI| zE?`;ZMK$DpLkbAbcr)U}Uuj|_2g=91F&IwY( zt2C`&pR&qPNmuyKS6iK|el!k4zP-`9jp#!mU=o_Gar_0A2~|Ej_xp2HRrNF7S4wS! zd@n~?za{ha^fbV003|dL(IBP#JnuH@6yiSmC?;S9U+bw{+~+<-@#QRt6x_S^CLZB3h2#H@{bvsaE~HJhv&BgxVZiWy_d8J zv#M2JEJ}-`fD`5jOs{#v>R-TuCw={avRcQDkwP=TZmL(ez!$%eOVe!z-t9I;Qv3zIi*1H8RS|OGOks5z2-ZALIfZ_e$1b0|t3c;zPt z6S0$~&h#7CJ0j*M0!bZnv~x36xjNv2E4T~z!J;)D-@hD9eG?V1G^P&DVB7jbcKW+I z5s0XtUA6(*I+Xgope?(`?dD|3%+6FfwF3PxEzTuD@4`49gm)hzOjj5iWT@&kgO0fS z?0%8)LGEJ?R-AGie-ZWaaB8IP+6R_%qM+fQY$`0uA^3f!&6T3dj5 zzebTv80RI|%yC+avn&4TGY!aq*ooAtdd(4qoZRsps)aQ%nuC z^J!%I%ykD6OG-|LF?y2Y2R-k7Q`ok|FLQN`cIR7+Sa#jB-pn+GNHHS>^`^wukza;$a#9VQDZQjkY>6&n<1E%k~eEZ3g2}DfyZW8T1U?<;!cmHs}uu9w3HUOQ2 zXhh4Z9^MF0ErGW3+hV?);KWny9YGxc-CFhuXBd-8!YBJ=!v4c&RO1ysx~pS_lw^uM z>5Ah;4@uFjAIt#wFp~{k__)T-+xaG6^*ude?5@{tQ(>}a0cKXRC7d~Q8CL(gZsJ}a zfaWh=T<$Bn+|U@wZKR}sBcpLz4)jp(tsQ4kBEAy=I(+^ru}3=P1?`P5vfI!7L6Ws3 z$nL+<|5?hiCrvXx2=9J>Zk@x7-k4de+|P=GK-t zaTSU`=oc0n$x)&4_;&MM#vB2aS)6#4bG8Dl0~#riR6dJ*an5#W@EzS+toc_IbT;Uw zfuL)igCyTY0(E>D=OweM>y}7<%bAy^futuc%P|gCOWTX=8O|1c&7VJ}|1;m4fU|W0 z=zwZaRj!F+XQ*0ECh-9lYj|W=?V#Y@f(eN|Ki_E^j#d-9x}i+0{XORfDL}OrVxpae zLf0-UpW1BmZI4hGZg=Tf(Z>q$1&v>DCCB&GEB)~j*5U^aCw2UYhYcoxP}sZuH|r zES#5Jc6uMgzRs`M4?cSWC?(GhwKrESl}b*IV7v~~uN<}RR39Iz!3YDA*#FA?0Ra}N zzj7t{If)A2fqkx#hZo%>K#K!XwDA$>qt+0I8F6o?y4pZ`XwYoU8{Uoo0nE#rawHj`0n zvYwBVJ@vg0zDva2d`v&v&mThtcP01%cBBZuj#^gL(X5NIh)c*3;Q}P#L z|NbYg{MV!+I+D$Af#6rb{&M<-ViG7(t<Y$_YZ^jf^- z-CKK2Ggjw57kmpaMU*GdNzQn)<#BPNgTX{JmIW|&6)szC#NVplQ608`5rNO)&L@Nn zTgE_stmFJUlg-}N=-swtA>)PjT%g2}dUXv_9q%RgW~$`Hikk2L*(q{IM-01ELUnSu z7mQd&&DMghON_Wm!Kcc)X$;`ZprA*LDEkikB=M!KdOr~Y>Mk{EKezmw>h}WP0np;xg|(DoCJY zZQK+pZu;C@gTE4>S>bAai{+KccLN0mEfs-vb(4S`+eU}2P6%@wRP`ZuUeiU4}z)UWm|*7aTVHvoMp zJK1UHY8!tsgi@Mo)#~9_>K(l3Oyy<3wTCm;>JtI~ zq9*fV)8(H8#8<)NTx0C^lpD#*s}w!t$yWex=Hoz&Kuiq=(i@8TZe-u3r>_S@W*@KO z;5p-~rv*(rh}W7$RalnFerPc?8`J+rt0Av?`)^W72pH{0m4FE-X^WoG3W3af?WvN3uT z){mNh8)(f|WAsARzr5fS&vi=b71zPJjoS7}GW`z8G698dxA`5XBfzB>tfUB_Wz>Zb zKm?!qo|ju^G`{OfO#D5EuYfQ$JoZQ)!6etImRT6;(fTcB-zQ9cu0rwe?1Y?MEq-=}TZscY0ZR zd~u=QG1wjC#|j7ZTY`ANlf*evL-Py?19<2`qSNO@U2lz6rBy|lwg51@D=Qa(KxXYh zVrgUtTIT%`kGGV2`?~G-E}p-*FXoB0>JS@tnr~mE>ok&y)pjx^Mo)(>-yaf$C7*WV z|DN|KjO^gTpX|fNMx&JZlr&s0n!6smpYMDH10vK8==c}BmH=&eS8RHg$ME9=2lR)U?Y7w7 z@oFyxxBXA6j`kS+8g|rY+ukpqeW!Xp6gdi7w2%vAiFD`1KHLc+Vv!@7obTrN*vG<- zIy&k#JXWXXI0+!8Q6F6XV>s#g zmr=pP2$umcek5U4VUr1GR}mN@`|@QMD5uSW+?*eCCJwSH!PE)wW!vWN@Ez5VGj z+t*t&VNK{55R8Jp^NH48dq^d?@b*r}?X4yiBVL|jva^X5?$5E)mwRvecwm~I!bXQ% zGq#r}E7!V`@CVxN?bg*|=f0kMHpT}JS)}p&yf{$qM@h_OD5*z{#C^)& z)Ns%V8fd?s;mMp{ago=&olUZT+{>}VC$NW&HYtY0C~=>+u}Ky&-0SPUR%W0Z8grMe zm}4eg?w|Icg6=NIvAW*iPC5yD4?R9CEz~Ww0pm)LNKUC2Vu@BD2>8Y=F7_wJN_QWkH4 zr(eJ{QKas~2aV~UWH+d4wl0J!CrenSKsI5o^TO$UjwtPGZb^m?u<;pxrT4gPaOj=Z zd1ch)_&(PY1QbCjwqvsH)s8_qNG}f+HL7f<+U}@n@Elr&-kC3;%^%#SDV0D+kTC=6 zJ__ZxF;oo=PPq4k(@6;gQ4Ci{&s260&T+1JR*#HHF-$Z6#2EDm*2%5-pFzcE@szbF zms*&2NrQN#O!=J0BH+XZ%w3G=QTmlN+SjxS^c5zV9wbvifiK@NYmMSeQI)<4%}}AB z->Y;yI1l6aWrXM(P@J&O(Wr8KgUZ}h8j-Ar{sTLhS_3emq81Kkm1o$aQ#NfU!l7uA zEkb$yxtF8n$y%fMawWW`TJQtZYs1xPkP{K{v!iNo|-CmUU+EB;H{I_Jv>>&W`mPYjky6ABG~SW2+r?U;eqjLD~?324V*ix_G## zd<hod_qD}mt30rNlbGjd)gcG8|EmiM4U>YQkzWVw8@ zK=uS{mh4`X|23}4{qbEes5=5&DlC0lu9>&ekQxb^a_k=w+(ghr|9r1>8a4Nr~M)hwt&xad0Q| z<-^+$aeTAQnj&Ok^G)>O!t2jx&t67$QoF2Tx

5%#XiKwxA}n_&tF)y)}A0kPY=j zM9igobA2KYi$wJ5br=d4Ta4#DoOg}eJtFzNgmlk<@!z$}*VJ!<;n_qF5J%9Ds^ue}9GszHoZSNwHNEgYc%B z+t&1_A3w>D^a?IvfO>STPu+k2Zdubn|%;VGzd^sry-QJs`v1cr*`O^;j~U^@8l z7~uBCl;GzheJ?i9l0F?}ovC{-6ygOVy*(>uGU?oM8-dY4I!A(;#9+X=j!2H&HXgtR zXB!pfs6y1{K2FKYzvXV)Q7efm_yO)S*qk`-xp$6Lk{<5bY7`n~|8aqY)f>yvN)q_T zxvd(kRit`)T-}z+WSMQ_wdfFThkgT78>+QD89aZ%@o?$otpk@SJ4 zCvaid;-mXd|5SyniE$e3M2HXOTBSL(VmGrweW>a^oZ`#u7Pc1pWu8S7=1Sdwj({{c zIrCL9gjb6YL~eLmAmJ<9UZFsi-vKB80T}GhD7T(H+`aQ?3BZ3cqWAQ`G=MN0*fVnk zE#Ri`ZG&olujeOu>xH6C_WFbqwN8e0s-rF0?i)wStj9N6S%OdaH8PqCRkTf8qp}B^ z=(1SLE&&D6D;seDE(W;`5_{cry6(YtNAdM0e5w=nb%V)nN!;+u)e>_`A{+kL&<)Du zkh#qwH3_5CN4UJ8UdCCn3OvDtOdB)7ke|(c zB=Sa6jrLnc&d2*{XDe&M)$9vQH<@K9a{V*8s-${Fu+os^-w2B?mm0UdJ}_zMZ*l># z{CLblF_L{BG#)Um#17ZJj)2aU&H0tCu2haPn>Yu6#tw?<#{t26GCNX#Y42pco8r0M z_|@*F5JT*0Fj76)84d|c{35OY>K-|4Y^FbXZ)LRPJ7j?!jgM#u3$j)+x>I|g z_lmx%7~+0SksJZcNo%6F;6ZJnv@LfSeybGhg5 zC4vIDnAMQhmCvWXE)SQCezuGdvY%i}3LBj4EPT2;wlgqw$w1ynVH3w z-9nivC;RNazqO~5&=q~#gclVM^nPAXK3<~zVCkbV;Xr?4aP4q zA#{>SB0-!Cr+n1fRNV;e{7FW`Ic^x55{X>+uv3`_pYBrJ)*f z+d8%S`<8=z>B^$8?_YdF=?2MSf?L=aZ9v)aaFC*!4^VBa+w8q%a!3{3QqpGH9;Gw+ z&=hI<-wn8$C1S1s(mx_j(4w=z1E`#aKG|UaNbbv4pKGTnun!0*U{OPscs zj5dZ$u`#v*7*kOJ&sI+WO)?@bTOmY9*?PiSOPe;*`+>41upCJL)cVe!e@7hSgFu?# zx;BtvSX-_T$sQx=+X2wx!nzvTQUebMbR`IKVgrqo;HKw$tK;qQLZ=}h8%n+TQQDMA zF)GijJ#M;wwcZc!$f)(!?^^;Fe(+ZhxoIihFGgr$NU|-`6r-#ABDQ);q(jd27 z+J!81WsT^?h|2PxZ~S4U1EQk`Di?8A?E_`@3|mxmc?O=8@Lft;25`%vMVKn~FHMnI$NV+*<8C)Lr8_&zCg zrs&DCk7d$`x@s#@de7ag+QGq}*4CT0G3*rypI62e&y5$V$Ex)z_|h=?)>CcC}S zHtMLzp|CqR6BPU-CA~1291Tolur?J{{2mwVq*`+ys~ub(<4+dj#@w zHTUOg%f6#M#x51M%3c!j*qS?DF6jhC`q{)LGm!QFgAGqPHwlXD(b>zFcM&pS)^4N;vrUJv^f6(oWTYxwN?AQ!p zK-!d7zfYYD#3(R;`;Hp&#y)tKYPYo{2I(tkdok%yYT=7l$>|4Tgn|VB+iRU5hIIi+ zc)-Wd0>8ade;l+E8L)3m0=1c$sP$5ex^r85bwlG1WiY7DQH1|tSVH@h4Dl z4_9hv?A+D-zO;1MnDq`!qI0=rAb2Fu6hGiNp>&cGK$C$q(s&7Y4+PkZX?jN|WdGf` z!VA8^PIi^9!0zcY>wyJ)BI0qCU*jC_XF>(ZQM^#zou`_QDjl=ItOmnD%mWBz?&!n4 zHPv`QPJ=3Eq>=K=KmYu*z@XZ&%DJWHh2jIa17rVX=krb4d0k}m3{{E0XuXACZQqH# z0X=9RT21Qhb@{=<#GFD%!zE@^%^g|^?+WL|9-HA}3SY*9SIOaG4s|}q`s)om*XU6d zVH}7f_3@zxQFprkamh5Gnx^5aDhD!aPfhlPp^q&<#N0*32)dOv#i=hZ&NaTh!H2uK z=>a#nc#V30)V?lOyIKrFA>_D{)?`o2IaK%b1>y{GH<--)P;E9uPXqa=(_C8$ZOoO; zxprDtt9(SyjHGXJA0Bb7EHKZ}KKdO2CI4OA=7z_%aXJEaSY;9lq@QEW%DoxcR1*cm zC1AZHz>f9jM2Q}8=cI;a1ncLCDar<<{Ygs%c8Mc$^2FZjV6*X__+8HuNG#c0vM-K; z*Axr}E(h1Sf`(ModExoWh`^Rr`Aasm%-glFssr0Z2?H=9TX6j~jhk+Le8k4A9QQ>0 z!HvE=ohjtbQ`jsBQ)b1RMEFq`L;B5$Yat*H7dg#$PzyVf`}SV~%I)KJp~;x&s#~Xq zSgY~C+^l$px(;$z3Rt!HIkn>+2Qq(8bVv$DDu-S<4nA7m!OOX{XDKQi&nPj- zf04GiV?eDh9dr{pdH3a@kuR(>NqjEd4qpu!a@Fv&-1s#>2z=na4{;X#6&5@qN1zV} zO;0FTKJmiMrYd2sj~$V61=5$&L=PZ z9$~>}$=c{*?>_s}bBOZRZ4H)cFbjj}K{}(@&@ujmbfJ_2?0@A%GSE!aIn9T9WaCMJ z=&Bq4EI-a`ew8xqA7(W>0!ou0vd%)%$cg((xsv)2BHL-2gx$;G3td)^{<=t{67PVbomp&$i-ofr3j zO)GCGl0&;KNQ~z6T?UIxFmr$i#QWzJo|an8_L&UK1C~5t4KbdvcD|dD8AZN0Sy#7F zE(K8c+C;6j;mn0o7xGn2(kvhHq4UY&zOVg!G?;>hQUIIv>>;GH%zm`&bTo2u>B+Bs zCZg2Q4Wr%1giliCTEZ`!22~n_;&pbj^UBE1)HnY*>4~#y$l73_Xgl)RvFcYyme#&& zfaz29tvut-k3QV)_C25XYpp4udqq`pb)v=r8`E8@=liG-tVYR4NQirwn+D)jfSF3K z+v^1n5-8JjSn@{nm+$p{?Ho0iQ-Zw~=a)F@jK<>xdd<6UD220y?O#l9fiv3|sRtzG z6ecHEBog{ku$D9YN|P~rBaz3|u5F28*}g#7%C2=Mj73@Dz{Fx^+-^ih6G@)W|&Nm>*D#w|^JAk(UyeC+O3}%s} z8xa6HrpkS)SAQkv`WS{qa=ZVhHd zmO7(+OkX#6btcq@Q{@89UyFH9`E^nnmZn`Fml<=#{{eJ+ten_k`0Z5`(efxM^c34Of3_Rh$aQEa`R zUA~>_F&gG|tGxH=<{=b?DMocZn_N%ca~mNfPmX$PzUtjVRYb@3rbkQ`?oIgUfkS`K z=65PmaH0Qev9Oy7pqFq6#ny2|CMhK)C1{X6$f(K5-KV}@*t){~h7E0K)(MyQsl#X$ z}i&+M&m}S^}zG;t-0x;`;^b$;S1g zP}BlAVxV^bGK`9Af9<{AjblR51F&a;@ZHpJ5Vy}@<=$K0+dEmUUTfp1p#=`8!*6d0 z?x>ma>i>1ePAGWB%_QwE&|YuxgJ5e{zi_2_O}p~z>z)5JHPEMU%ENgkB|W%+3L$#0 z=Iq(CLUvulA-WQ&4t9!4_AhkHKGsnPiO zptKmu5q(DoVbKk4b#<{%ev%Lg|XF11W&@+j`e8r3AND zfzs-7E|j4uinDjmkLS++GA3cVd1sO;PHRUPhW+e8NefavwDmfBn%%d~*BU2P0ss4! z`l=I|bESiZe4X{AAdHr2zj^tKlNCLgJcq3LpWhlisirLG{N>01pYiI-r`A_y7DVg{bTD=~qF^&bq}f z_JR$K9q?V^bTJ8i|3{_cqTWY?|LHOi&+UaHh<;e zKJvA-`LbxKbh0DkU-6)t2HM3<2_b-s=Y;RxhdbbN+l|hhW>yUrGf=M!WvO<)vz5#? zNzUXKXa$vF^I9<({wqM3{3|;EV7JlG=(mYi@@&f!dnc|fj&fA)_QKr zit3fw{CyL$YJOYxIt>}HEdUeZ*E9+i#MG;qZ4uVBQW>_-*>3AZ_bU^Qf*Jd-Safb@ z)KrlA_6VdZZS+VtsDbt&_u)Ii9wg^dbp8|e)2f6g)yWJP*!LP2-p<&M7geqw4-jxX z&L!gZ5fi2*djIqw>VViTT-3hfqVm!Lm?^7C3a0W9K`*|;X&N@<&;)Jk?1i(T2Y@!> z;9x#^mN4<@`uF-d%Tsq@w7-DY^loqen%e1N4vb&~?5XrS_kME)dquy>nt?D10H`K@ z#IwoG2Kldag_m9mcJQcVf9*l&S)<(-_6}!_x?di>-jsnxWKo?{%Q*RDxXBP@7nj>v z6%C;%AMi?Pn{Es`y)hV${(WO^NvokpK6OXOM+K|HsoeKRfwRPc061Ng`56FdUO)uj zlfD3vm59`cO$rt>*ir6SsrDr@m7cRNmf1g;4*oU>G*E) zNqBOH`12g}youf}?Oo%Ik)26g%9>FHW_h-AzA&MbRdeN|N2gD|Bu$D;uwAXQ*GSoS zT9b^ouxl|zPnLb>`>V0M$DhpC#Btrm`(%PPTW9Hm?eJQb+9>bbzqN0?BHAdVi0TF z6JDO>%=aOO+IMt?N<2oJeCe)H_se>-ivz+OvGTFdt;0|--Tb5j=D(}|$qZR^63acF zc5lCN$vkNMZ7R*~?4RRJLKtg!^#dWLSjjT$hLOqk`s;B%KNgpz zQ7Qc?%&U7~o~BiIUhnUS464InIqu$lgzI`-pg$}|&^i5BZ4Q$p?thcp)y)gQ-1(*I z({2z|D@J{NhG)K^l8Um*W_btq}0P{iH$cnpuNu}nTmy*^9EJ%Re9==i9XcxT~8 zymn%(ewCH-LH2a*weh{N3j3-{JZu@fv(2?j*jg)P-JHh7;utaiVK4ADSz3lpEkmCj1l8 z4I$S0i%awKVdbxCAr#s#f24amGAJTJSRHrA`5>*bZ5@r|I#@r zsa?;|qV9ikfh&6;Eq#$;;xH?oyb~@See`9%``qOX`#2??zPQL|(G5N`dF~0Z;zloC zLOGZ&DeapUN`SaA9?KLVDZ1l=4VHK}tyb_-IwHgS=(@SRe{buqUw~p3i3Esbf z=tWnx46A&FP4j2%9n46Frq|NqlcsN`{tg;;6r4?i-xkY1eeT2^cW?y}B{tiO_asEO zPh;twPxWBur0P{ORJhb+{9Saa&4$8ApOKMhRNbSA{mat_SP7gxX^}Y)K}y~CMpFdr zC5;^c#fhVGl6NJTW|Ye$<}C8Pk$g$4nWe5$2L~CsSw{-!F|Lqt$1qtMA7l>=RBQrW zWY#&kCLP3B^=*1WPCMvY0N@k*kFhKHPB`7*;fTRjs_XWkksV7a-DXXpv{c)?u{ z{)`Jiz=YGFMFT}YQmE$3moKB`_VLl)dy$?X z39nM~i&_lsc;$8&CVlcL*VV{gMe}h4!t4$<2!$)=k;iBGqMHdC{W@RYnr9FVXD5hj z*1i#_jCfMk-8pF#+BxLX=gzd{*V&9y|D@!B&VA2fKaqUT;|fobxUbZXQu1SSpK#Ia z!LkSk*4{E244-A0Lxah`fp_OrHc~J!@nn!6msyJuba?ax$SvbI&9#K9C2g!V1p`yq zQF0U6M#o))Q-w>yzT(b6X+CUpKG%&)D_+xU&!OaQ?G5U&nn_vuYPg9>4 z^IzaRAo7i54WQMYCS4@p4(8>HQW|qGr5Ul#G+gl><3mwjzJmQEb?#V48we~yavqaP z%F2_xk%WGuN@=UIFEDRQ>Saw~FwtM$T8LE07*=JG@KG=+MREqCk*v+Z29^DvBcCG+ zQXQpQ*!y?1C1HZFumo7DnSp?XSU5;{g0`C*6@@dJAsKY!O|wh+e~>Z{sQUK0G@P$& zna9^^2@8{2nhjW7j zzhjwR-rAZ-)ft-Sn#8B-nk+jR>f#lZ3<@4h*&XB%~X-wp2aJfqMI$emm`=2LUTb+ ztw#q-{#s5Y5RK1bJC@tITIIuiz%kY7d6V#7W0T|hVi>$IkFI4vvHX~}w%0o)t)=h# z$d>jj%Z|iyo)PNJ$@qJ)5MMV!yXl_hWdy601)R!H7GQB&jpYFl@mYvuqb@vrW;s@j z8+b49H9dar8lB`P)8VFA`e!L$YFUM3E96ZVc)q5+<(($n*&2MPT*B&mZ-I{c$Rl4q zoJGqINZ}$27vz(8WA~qg1hx03;nx|pL<;#kumUmdzVU3HkNUHZ`(;=$49l}t4?_P(19R%B#27 zw)KQbb{K~}7t5$m0Ri=*__Mi=TsIdqvmCt{$ABIEp#8}Kw~NGg>`GOhxbNe->x2>i zhOr~TD{JVBt%vW@_T@MFEBBG|fVm_Ynk5upalj_F@ei;nsO$|x1L<>&QaJxcK^FnG zh@yWI){>_1fYsOCQ)O&`hVDeRz3qd%)m;lTweXPF_g)QRBX>ji?Hw5ho=6Fk5k5(} zftBvUh%r~Eb@oahpl)(BZI1E5P6X$;Rbuv*_1hqt7L)Dvp3eJ4PL-`E;FOS2Cxvzf!0~`!e&~w`Y*y#MJjap@5y;xQ968RSsuiTiy zF9=CfiptX~F*6@1=r8BR5U6Fy^b)?UJJ|0I^NG6IJKd}DT-*NNlqJjT$bW|Hrf z>@U3DEX8D(OOwI#cMCR{>XVp$1Mw77>l^20-#zTtjqLlterMWx*vU9z{Zz?aCSyX6 z;LAqN+K5!7{?iA@3hESTJb7-WyyWc$??>V0Dd++am z9X@;S70-Isv-Vzb0Ny%U(*;se2Y-h%<`OK>w3v_KiR#?WnV9+cr7G((2OJHTMx0*1 zblyQa*`dY7D&1h=B*kp;O*l=I~Iy@JIYd_TK}VPaRsBcy}}g%$={}^18@& zBK)EP*4tkK-S|Ic54T9Hb9QChBOfqxApeP3?v>_S+;C76M3 z9~xN5rliGY+xYPzB6N#5kyTyanO^V=bA{rF&1mzX^RQt(t(RUyp8RRvHlR5yUt}8N z&$oSP7n*Ma2Xb)m@BK*t?b~)cLfod{M^_V7R*U#|IbRZ%2_rs3UBNi?$H6b`lAhDd zF?U@VT!!l|n$5=d^GTMehq_eAGWLH zX$;&OEOXl=e2jTll~{Xt!`4q4r^k3+ycJ}H8qH}|iJ!Ss9vsMnp8%~1d67-s{axwP z>vn%q1f=N!@M|m2G8O;Wj@|sA>(_5c%^|hIJ$E5mRe8r{@AkH@^gk2uCyk^UWzNx2 z%9h~rzu*D8Lfb>kaP(j_Ry#tAk%(C}uKPp!n|iz0H`_0K^euI#zP4Im=Ez)_KpTJ- zN3uxc7h9VMhI>tCn}V-|6rMVtTJNF1M?+c+epwdsAOyyWXvzEtyXEjls@eX9x1qa{(rf9P^Ic~PGGjPInLX7Y z)BjE0Y(CT*+>>ZZZZ+OUMeo2c)m9LWMeHsOJHmq~*XB*@JPKFFk#jA{qFXhN4VWLw zsPfpJzUNGvDuQUa^PHZ7Sk@$a_wrRMo?f2IoeD`L>s!SLxOA&5so420+1?s^t3C9O zWJvoV22(#26k1ph<(_Eq9pnmyGDt?r^E!ZuYSC$W=RoA-1!Hf^K>E}ZIa_#`GK zTD|c8X2*2y(sOqFrspL2GdH`|V$#Y`vTfH`P5AUwks0ln%rM<649VX7=uqbRJ4Eca zg!`$VOw4p(%iEIcW#EbtesGVthU(cb9_tA+pdSafh26L^fL1+yQuXk$TR4=SaBq@k z;`7Xr?Z!~3ZT`qNU(&T=3*HU0(E`HS;zibp^b(6LotpYoZHGWe zCrbrB;?&AmHLo|Vfe*GaDLdz*P|vL<8%H*t1hM3S9iL9F#gHBc0a&(YlPpb zn|rsJ?_7<@6`wy6db4$y)G*MjYs}n7C}`T=`EW(#Qr<91K-ov)&>5AmKpHO>SPGa@VegtDR zm-^^f&v?bwV7P4Ag21`-sU(99R$!y*=3ew04!4L<8FIZQupYd-3Q^Gc3yanlc0uLi zQNgj$&OsjMFt#wF@~=CEIT09*Vg3}?IUqvQU(GhmqVf_6>?j{UZHT^n{5b<`S}ejv z!r7rO?;9kaf}&E7{&R>`?DS2ENQn5RI|Gi+5!onN3oJ?zkrca{ZF>=SE&C{&r~3`T zXh8Y6Mtr3?)ahK{@;YZndVYHC`=Pvnofo>yPC;9%6Y0E0c?$VZRr_PzEG7y^cKhyc zKcBfElC}KAu{CP-a*D-IvqkXrWu|SC7z|;lp+AU%7YNKn!n~N(o<$RpIDgug#Ky?z zl-_}*ydu~0+>(`u0S;EGk}u^hrT}zlK!S+oA3|~yd7$>qECRx&@g-V|tn|~^d zPq}o|*jwo6jLKJKeGw;Y=#J^w7ZrLvHl`UGeY5#My@J6c@uZc?MXJ*u=F^0x5{e74 z+TKm1$BuQiN=sE7?>+A?$`SB*uMVBEac;L%c|%R0qgX24nm&RrtyZcWpS>O67aJG- zq)RJW3x4DEXr0AkPV4IGY9?HjPCo029afg$iI}fAY#=HfdpGoy1tZ(!N8+yn0!>!_>gZSD}Lq3)tuAyqxjTE_!9BDUNfbzO(WZGBJ9 z`1>j_7wH23n)GOhx*yD#L%e6Y`p$z93pv>W)sP?Q8pn~P(K17W{qqhdmE|^N1hB4q zPfkwvBohJ^+P|&dw9|o4eE1}ixy8(&XIYGaep5l@_%g%P{$MrolhA2R!1D(W9#oo7 z-hV*BiIIURhh`Z9oemEds7OL^d)3a<*Cv!(%B`(epPSgwtuVp-t|=~k)phbit3C+Z zSU_{V)cd)AGO>`X1id^~swv!gv${?Z1-|@J#^jc3m(| zLBAzUeBwavTZqMq3Vz4I^7v4P-81f{t3kfkY&KILD^zKKP5YBR{t0h<6&O&tS}R(i z^^I_}+(2|nyeSLjfP#e6v7d9sVNAZ$NQCM_&HIl}n1CNf{@rhcrS#y9@y95Lqh%OL#y@KCu zAah2mIZ3#hBM%D4nZ7;su|)&#GjF~(<{Q`jGJ}^+$9v4{@39p`9yC{U%s16mstfiw zcs7S&={Rz>T|Xh1q8YEEcyA0*wi+90c2Z!1DueUdoh zuuF2(k=1~n5nz2CLdg^YO>qm!ifAW|-b%zx;M-HQWO*sje_<7Ee&txTHexa}CCYmt zyAgg8PJJ!I`;sU(t*ah!*zJeGPKo2a&PP%mZTf6nl7iTZ>im1qr%l}(v*s_0P_N(L zY$8D$tGbF&?TnPGi0%m_xRcFHHomcS&xX@;BeFdqP+=cmVBGPhsC<nHZ$Vfo7QR8}5X8;6*rjuXf(-DPQ7;7PKkO|Y1Ml@+5h z0YNm7AE2#;Ox!~Lg$eOk_N8vKq-U+}>_hC)t88p*2Hj3JhflaAx&`2J4au3D&&@SK zCXQfLAJt4*|39l?G1R>R9%i>ZMl-WXSV186K#XwaESzc&Tq%r2%w}j}abjD&IqXNV@g%2I9(l{`3Z(8PVOPCvm_ci9TpZ;1+zOl zP{qQ++O%1@&kH=M!y(%0!eqd;!>UH>ECU01k;BLCNBhXdIsxP*K&nOSS4ufPBC zqd_2Ar+6Z~3kiV=Q0P+YnJ06w7#wdyA3T`wM+yA;)p+d|meb?}A1zhUZB+#jKGRVC zb)?q>@P66(op~V1%vH(n9~d}WVtC~MjMDaGSMMDptPp@f^cUoM;3tmrG$-LK7tLyS zB{BS~C`BgjVCeI6NC3=1m|h*(`OPL*Hz z&D)Xdxe0nm|B1CtK@PYa{hQ&LtQx_7hv z#j+$zMu7GnkRFCI&S>q#HTCwspRBfVy}GC@B7zhly-^?&lfRma3y6PSZcKHNaW6Db z3Bi9V5-NNXO0EcjDxm#NO=*rtDpyV!FYZ!h$Ff>+oSD*3h`oeoWM)=XRmH}|p$Cag z*EuKi*gwt_LFYi8*mkBAqodf+xPApN-*+tpSPJ#FH zp*CHIgtrR7eSuG*2>X$0O0Fm-WoW=?rt7ncSW((=Q6&g>v6oj?)KUdeH^pB7wPLIW z%=Ti-^$HhTU>^UMkylf(p`?7d_U8{VCx_?dcMKTtehyb$>v_ans)y2s1(17L*>gL) zquqroXQtkmn3!+X)h?IkY8)t`7Dy{`1J5$Z{j(1#Up`#s1!rZ^IM0R;<{8<|y+%{N z<>nMp8exBSn4g=Q>*a-dgqPBXKqLdfHPFODNZjZiBNq-%5(R{0d(2;_g(7#8qQ?%n z-{}NxUtgbQh4pN7kFYr#E;;Hvjo#)YjQzvxOCcel%~09&>}+sNX}p(~lF!XHj~A$^MeJyRmm4`x^8A}y)Ec8v}V zT?X!Sy#1rGG2P){)#Em&I8cm5F{&RPZm7>Zd`X1#&}$$_2)IYI>-+^y-O@Uzrwa0u#iZldm50|jpAAn=)~e*c_7H`B z1~Dg|&CKJ|Csz=eNW8_E|I5~+&>d|hk@0>l3<_d%mj2n=*{M74%NIK1_;e1l&2E=T zug>*~jD%z&SC!e!j4)V}k}@6WI5g{BG&nFr-y@BJZbD(vO0XeYG8%<$!U@ESN9R>2 z3N*ogMRydGy`WakuUCIVC>y%bWBE@3X)`XKYz@uK*PiYq_H2UCE8rR zfoo$N&ki>p@w?z*3sF7#1t5weR>o6r9F6&sdvTmrR~yvU{@WW-ikzvTZ6hr zFot^{K3wMHEI-_RvwQpwkDJ6TAshn|9uva>WTxo&^k|Qw30YWJ2w9Dl1O+|bKTP96 z9$6~D+u9-`6aG{lc$?p3659Ulc;v)Myr0!z`ud4nLzjEDN7G8V`p=AJ-F}h>AGtP{ zN}`ZETGHu(`2S(1!He*jzS37E@#DLlzHxD{!2w{CaJY#(VSvC5HmgaY1q3YXOXuZ& zUKbQRl>q53B~a>vY=kWJQRY3zT@&8Sa8+s667Ay)=?@MKk&?Q`a_WqH{<*O=CD`jm z5?TvxfjTfTG+bWpOYew)P&$3T2nQnv)m2rz_PZ}R9?Fj)1ErjACMK_<{w!R&ukh+t zN|8dElo|*JMt>n76qK}&=TG4}YYjo9@hhvVy{b5|@`{SNY6Iq-%Pwa}`E`zGcrX$a zI%sC+K(;Ix{u)ge#fgnB&nhl2Pvm#K1gY=%MrgyYEDF3w&iEg`0%bl3L=+$Ik1}x_ zDX&?HPoZJb5)n_=CaRjln&;X>5X?QLk~f4O08hF&Wrad57dvCIv9Z5?{o2sl%b%l= z2?qpWczC$imVp=rIx>*}7pG(WI}M|t>~@#ZvVe~y<7yfmtLAYQZ|74{(Mw2p&8z{X z;Aop`PdBu6Px0M8s?3dKP!IKa>4lEJe{kT16XBZn_3LX;hUs-g&D6v^zW>|#|NZ&Q zvSh+%r+I|jiI4ac{3r6t8Pfqy}%{jx>7+HXap1qM1(f zHJ*WPe^j#{A;u%D8W50eHYeR-Q#8BmQ+7Ex9%2VqMlF-|s_O&8I<@L|b$x~}Rw6A9hDt*I6tF9*gqPb+NJyl>rd zXb`%b9vK@0BO#YSMf5F5Ar$MsZPt8Ceu89@mWGBJqX8}< zACAJ;oCm&FS-Gk~&X*r!HBdv)vCF0QK#^!x0%hBBJ|IaUc)5m zb>2z)a8{$5$ZhqrJ$?DgSym5$iXI2{a<==-!Bm68=~W?@%~ufr(1Rb~;^L-vC#nOq z?=g|YW)`=7aR<`yp9q@?V)btXiQ@M1_JHLz8BnX{OkP`pHzQ^~t+n$`q>B<0$M5p<5_o5U z57y)zx0rZJ3)yJ{@%yrEHf%)f9?pN;VXBU7oX?-5&yTj$d0nTLr)tL#c}CfVSK%vS zCBdPg?ui zysInh^!T09I%o2yIuq5xI5vIjdJrN;$JN(XSJAd|Ow%(n#j)W8M{d7DnNgZ*M17Z3%61Boh5s7;RLm6aJ)I=?AoT7nym zOoqO?ms)`fDH5_}2y1kAj08jfrNaB|b%kVYWK{cpMtMb6j$O4Z3&Le9`PJ9;S$m<%TISgDXt7OTk^KC^jT zBd6RhwXw0WxA*CT2a6!;tZ(Flxt6gn^7&DW#MbL0o$Wa`(|n}2v6uOPEwiMbNyl-# zJuRasH3*F1x7eQ*o{M5k{MKNz`!!N=Qbks4j!kDHE_Q8LU+@yJit6hAm3beZpNU70z)xSFX19#(en3Tq$^u*kw64fF(|T5w2ZD|*B*d|w_;!Q{{g2H<(cJ|hmpM6E zQu>bH=UwdW@#QM#9XP(CK3I*Zp{&7D1rVcuQSPYIZY}|T=ER1~?_5V)Th~NA)qW2Q zJln=`>1|h+KWlgV;WhH9C#k8;?2zPt8N9&M$v(x-%G@$BLXfG26rI0(&G7r8^HeT~ zT-NHm&V3CZ2C}gTpH)w%xXU;@*e&-jAlS@^SdA32%_mpEY$(voeRVqN&y>JtfA);! z`5>aNPeoNVo4F$mWC@9uxR{5b!pdKj3PiBRy|GBgD$Lf`Cv4k(l7-V^y(#&4z9_wi zy>5qraUfAMf3^H#KDs9?W*vCJf8BT#@)vc^RbhvamANxdJBv9G)VCLv*K-M$wD|as zW=GYoSG~AATUz7jKU)!+1vzE?egx|OS~&pWAy72HXCRTQTLP!U#R_!G4a@1@#fA!PoURo~CF9fS?S)t;>2VdN^Am+@ zJP5*KMB}J~t?>1@O7pPupFG?Wd?4I&4mft|o}#~>AH%I6@CCfae*!Q-;D9Oy3#W9A zJ0A-%zeCK*T5ev;O+CTe7-T&BwYcas+0fV+^AZ0xU`#o+?ses0xUc)o-^joWc~&bb zn}&1dHrQ&)d>7;Ik{G-VhKKemtcY9=S}2K9>%g~J6usq7u~tXt5#GQBtubI6c5TLZ z?LYgNgj%u5uegrZjLp$*1F(Jb!P(aWcr{N}AH)g$C)0bs862UbBz z5g^yQ%N7FN^{KvZV`KAzd-SPJ`2N-&RoxGmZqVbZ!zW48bwiGcSy{I>#x_eKv_csu z$gd88p8WTZ1*JdQz@lslF{i=k^VoDHUBU+nQZTTy5seh+w-s)+5sJm0|M$Hs359ZF zR}TcP^Ops8%F~x%O1_$qAJbDSu*UYGBca!O&iee+cKKHtL0pnPy(}}xsE_R{1o%KN0U7TP=P0c>X!-VPjtNv7> z_G{M=pF99bv9rCuVjwSX(Vgh%I@1u{^zx8}9PCuaoeZfa-n+k z>T1g66}CmCxvI4ebFa%^59T5&X?$o1(bWa`3v#=o-i>ie+fAOi4d)*S_P(Ai zWidzKh6=36QBN^L{HXXaQ-C2d(vcq^X=4dRP%~=si61D zoA*vh46I zX*|7E2jm`eU6>h-zfJ6Oll6X9zy_DFdZGLQ4N>Us{_^~Q^L7z<*;>D!`F!8|N54!d z>R7$6ELSAon3E*!{XC@}UQ7e69=2wkkX`VY@y5R3pz<0Ud8Gkt zi;Z3%J}FGqEUCW9nf&+_9;rfIiAIxuA*l;-6Ml4#+-I!^mrwGikBuhwN8YqS5u}{d zuk=r~?G?Ny!@?Hw4x2S}MDMZP^d?|d_STY8=qrc={WbL(7P@};u%_Ib9OFP8*P6Hf zk$=M5f7TER(NiGPddO=}HwlPxm_Dn7aMU|Bm>K8B>cTA2CwwFbNzFFwDMh=%)a9<* zwKblUokE54N*vJ8EtmZzy?X&)NKeN0Rv zjCRmSxvAd9m@icCzAw^VvcPbq;6K1cv)F8xD1JYHsQf*(vo&EHp2=W=)e%Ly3F}3q zc}-B#REy5NbM0hF*Jb?#TH%TejpXUGfW-c^+|s&tr#+*(4v+stw0{It4%J=BjnxRh zVU>LovXfJG<9@kV)vw*~qL&)jkazM4L6feXl8bjM^E-yk)tSiYPVd4VRE2KgUbx80 zTt0ks#m06hEY&&Z>kFfmox;SH;dFQo+{B}Bi_l`rH<253pl~ehjU0FMaU%AY4>MP&$Y-4J=gpB{X_i)TZ!G)4;dN2zR3vWhZKWk~H(VLxq_(A=Usd$QHWCT({Jb%<>|5%&2suMOE# zDeA^H5{c>#)+DE6m(Kf@4eX#!`cDhx&n)x4;H7>ZeUy5Ur3zcROacxEq22T2faKP? z1D))^UIZe2z_VxRky z3K~w^39xkpeB2qz(}hF^ETL9)F%9!ld6TlfU41jg{jT}e*nJRk%|4K^B<1tFoZ)XQ zSY+s|VP^v@4sy=T>x%EaP832^1D(zMv}Cf*s91Xh`JKQES)F zw73h4YPk7_mB>3uOCCGNjOk5wZN9^PDq|_zRtDp! z&y`LlP&W{jZ#CFNmKKx3OgBECxfbU#-2B5>+Bm_L0YxSEH9bOG1_wDf+CGs!Mi+~H z{P)K(>uDEw_HX1VRbA(n5Ds|P-hB6$2>^DI({kg^hS^xLBeqY1%W3SP{xh47FT%0O z)KTl4ovixxBk73b{wW&L>b|IVR?+|4lL>0w=JXoLbMRl6C&^EilqvcD? z%5~;Pwt;Z$e|FX_VW7-t(Psq|Pn`)Yrh(%DKR!7aniyyf|LE+b3SR0Efa3w?>=#0# zb%a=*OmS8xMlEy{r7JZ(jCz9^N(_$uHm%+&}04_ET# zGU>-7t2|l4SMGy0>g9OMhW1ecOwcjWCpew4bi%GL3N`8W$m#kWkf}#CU42@*R_=?b z9FypD{EMyO%m$nb=rO{Qh&nB##Ic`0ZL8^R$%}0qmFK*zkcIX)G?I)39qxe*)4_>Q z(#*##Mf2Le6QEtk7WajT@ZDjX_#blf*pEQy$ZkM^TN-z+i!_to*<@vhNW z`)4M1*lXMUo(C|e>3d_``k#r-MoCDd)kPTJv=1+Z*pDCf`5UX|&$S)W|H+71b^{8+ zl=9tWP2YYbvOi#I(GB{N9)Fd&^RjFTpmi4uw_S-|r2a5j6!Pb{i99dwmqp}HmSv{! zt_~el)M_k$Vq}0kLo~)~yv-$ZHe_~vJ+Z!VX9lrZYxo0zi;*di?|b|El<3uL86Nmt_8hmiYQQeQ_=4Qntb( zsE5Ao)sd+afJecGm)z7PEPo=(bSnb1Tz5SYM3sb~a2_dgv6cHdyd>LGq(=f63I}RP zFCa@wGI1#zkDM+)p4`rmx_09?IvtGsl9RK+vD{s3JH87lhS<35E2d8w05c)00MQ&@ z7NmNLom?*>v}8DPMd;|~0{0u8!Q8FHH{QLXaT3D{A{rVR+uY=ZZIXjY5jk?=i{-hc zVNl{=;oj?KV+W%~)-WtWVMo;54t+`6MDes}1$r~(P;hhaxNVwH8xg8WsQAyWCDU}F z@CUkg78u5h|IY?ktuH?~mFWNW!K{LP34{d@x&hhe>9m@L`sN9vq<8XIGt(+y`6<1R zhKHoD44Z?Kk~YCw^)x%rTFUPNhU#2fQE{&KuB=$>b-*qDD!WLC5t-^?%0#SoHOHb6 zZ+$qBg|*liVpb;1cjJx+4i>jV(YVat=|PU#K!LFTDd1ts zymVK>?sy278x8o{Mk2|=%Y3b+mOv9L83S;$kH;xsk7#IClGVh^%FEm64T}LjWUpn( zka~gg+hm?Skk-zxObvBcZnoq-wLRFK?8K%xAFCjQ3=9oaU7p_`X9)i*)WF5sYWW!% z^e=0IlHv7(yJ-HImB2Q-DO;|%-J%bjZ+r4fhI3L8|JFNQg~o$bPHboqvGF=B)V)R1 zFva)bx{=j=VfT>P%<+K0^v*kYT>e;f^_xI>7R9}=%#*cUNqYeX9O}shBCG&u_~^(g z(Zk-WFmA@~iFJPSLnE&~y~zda~o&Q6ovcv@KK*~K2p=i(f-X!piO zpoRewq`J?AJ2(r7J**DD8}8H{YJFE>;D+ra{(9!yku>v-x729HJMLYccYI&jJ0VD> zA(s0~qV!nHMQ2TerbnfHn|1y$jMt}qtjZl!F8a^nx7?58gYw7LR+jnGXR@a|%^R^~mf@ zMqMC@RiL5Z+v7q5fnDcO+U-MF-xL||*!Fn6lgxc3g>T9e)r4@?q|Z~L#EA*7F;?4m zSL_?;L!hXDw)hX-nYp+>(33PnVE2-G+x0OXC^C1r3sECI1~kQLRp&FyFKL4{UExeo zl-rAYpmZ$VfCByFMpSPp*}LL*!F)xvB)UVo#WOwi@2{xGWh-y;Tu@rWrYKS>f?jOg4H zP|0wXD5>Z^Zgr`u-Pj?_M%Z5!a3kZ`8`_)J+Q zq8ISN>%&1Lq4qgmz5}pP3IvOL%JO=&FVZ~cm)@kXR6X-gQs3benFa0lY zEar(ogCsAE<@QrYnr@so&N*)G5idsmYAYzdbc=xG{f01)Ng+C~I|r@Sm{J5Vu?CjD zXjR(MI0m4Ev}Xmz?)4Al869n!@u-tOh*fCenKRCt&*fH^1&?GL z9wHeRElBBWhu&S(Vd-1c9w zi6$XgAMv^Z>w5baph5P*@$Ql9p)>r$*Q4kiiJ@3Ua$51#h{$ z7#5?P*F$mn*;27QzCjWaJc8<%m$s2esi|GzVYbZXH$adB9AK0jhJ8|yzTpG`4{x8s zz;lzCqyPWEs}I{MP%0M*yd^F8s7A-pefJ+!{u4C%IjMHS>We>|22X7#YqFx0qwj0 zc*?)pU$-+b+wLF~Klu}|E4Rx%1mFG^W7k6pG5}L`9DDe&DUoz{wb`DO)U4mcMBUwF zqR54f^uCL6bc$wH#7q`CpNna3O48GxN9JV&3R&cCf1Ay}x>3kw@GndyD(b;sg|vS5 z_7;&#!F2s$x!LK?St{zVr2*M%5V@`^5&OlLx2)LZf@tgNz6>pogOb+R8mK}{tfzT` z*{Ye6JDD%ndi()%ouc6c!TsRCZf4q-b_sa&VvpQw5rxFxSub z^(XEsD=Rys8!NA6G*Rxe{hdJml12PMaeTi9xlyvzmHWzpB4d4SelU(7JgB;uQLHPz z2UiY4g8Qp2!xfI?tHL7mHBeg z8??I(bbJAh3-_A-_MpLnDn6p?ZV18u-g?l&22Ai@jedn7pBuHyP z3E>9^7BBcjPa%lxqIo!Klk{^`7EP`DGrbIyt5r$%PoH3AhCfIFHu$U`74+zeg)fOW z55-OJ*5B$Nw;Mp4L+*+^7%ye`x1Pk@^n7W%z>Kfu<^W+JHL6Q+g&G%B-z@Gg~8Ib6`owiKn2B3i1jfbnI|a zayC)5*zC81qeXe!2cxcT-Tf8{9`--tsXxv{SAoX*>`|7$3qeEyHx>Cg3uz@22oVtC%7OcKOZ1QPLn4wy$L z*P{|H*&%RygsJ6B+~ zr{G?l_H#h)iti{wgzik%)qX8!wbK>PgGX?d*)po%L6HMG{QrpK{#lp2Eo6_E(|%)r z<($)bgI+~ye@};j0(=p6X6G~^ZkK5s?|5gECwCK-v@s%x|6>T_EfhM^e^g;0bfUPO zBOw7@c|?`-?8{PMwTA-%>yu|dk88Rx9Rj^ixuoE|d%8Zlb8%*}O%n%N%B1V_O;=t- zC<26kOR!pcmcAUxBs;Pv_g;y3Cu%%*9UT;ZcDTtC5@+q7zqGe~xVPOKgJm+2EtODb z6K){&aeWTPJX(_BpTk|b#fyJm_)2n^!{~gFP;u!!<9(#Q>n|IEGw4-xCl?mxO0p+q zc;>A1Nym>s!Dow8sKDibBzZVh9CtG%9Ee}@W1fpE;C9vc`*W{tINx$>=pOnmaJZ9< z@q6PSvkHrc<+rVmoNF$^n6fI2hBlLr208;s*cRJsRo4YP>Mh>pgk|15P&VDrZqPxE z)TPMiXsO;!=Vm@VJPl;1px1KO$h(e|5dAXDK&V9Q?QEHHvq4`+f7%b`1B=2^nCk7f z6+{%~W#DpxJUr|4D)am(^sio@=49Aow9$T$#H1IJiO#FrxF*mUYbgsQu!vRYbh ztK@5U0JSBRux1A4jHLFgw_0gi-xAp!s;<>RaSo>##;9k_0RhMAUf^R3aA`|quND6t zYZ@KVbFJ5ABS>5WZArQArsRg(-?B99HM;z=Jg}7GYBZAaL1I?A;3DSgj-ukTe$85A ztjiPD!Dd4vqfpm_1FQ;*(UawFe{k~Gd_0c*m4P($KcD#+vL`AgCMhafUH+Ujp3Rx$ z-VYTxP*W)Ynp93Rk(awdi~saRC{Xa&d@0>i^pbgwjON~*udTs!28tJcX}}{EkbnOQ z=8d&_K_qM?Pg-!3Vp1*bYa8LFtvCKey4vfl&7RvbAB8o&@BLzD}$7*)s=M7^BLP#7(_M$A?e zi3&6;f5?7??CM?XubDEZerU`p`w1++jJlWKvm#HUeCJ2L!ISE0hm1;-rOgRJ4nXF>$APnq4Z3K#C&% zA~PV={fDf9VK=ANeOyFW3`Ia4IZ0GUCJPsuV`|%cj1l z1M_$z$wOc-ER}a<;8Yq?da5(!Hb<@^m$;RYYooJB;^lV~2#bVXt|8PTA&su3m$fE{ zXoK#eNKT|>0DW)VhbdggqTTEEAs|EeZ0!OynfKB9$rE}%MkavG{uHfOpQ(uxMo-`B z_t@@zI0m&Edo!kOXb9ToG~u(&l6h|yTNtST1G+4LvVrf(&k*SJcNz-3nb}#c(|S|2 zrr@m+VYM^2It1wH#rs?hA8j6+$FTbIYT8pREcdZt5nJZUwR12S0Q)gQ+;)e>}+sB(7qIR0w6pW^7FMzh>uXIc-E*@+5 z;lsc!oZ-RVl=zovsi`cWS9C(^uKI^BkU_ZA#L+}M$rq8l@R+x%>UNj zTOZ}B=D#q;TJC87@@S+9MgD`j(XG(AeI#`iPfD6BFrj;NB$GQNx_I7r`r4{+^& z>~&-9z2O|?BYilpi$x`qOqod|k)4argc$*du2^A{D<#D1bJJ<7KJD z0_pjgtYD}Bhk&`H*&b2mv*pwx1_vZa}5Wr7Q!xIJ4S$=-4ZmNmpIqTRZ zIP%_<0o)duiUQs6u=k&GeL#Y7%R|;%bP{l|F0%2$Tgb;irem{|n1-rpS%0DHib(LJX zvkyAM{0j@s50(JDM}EnC?*#gt88Vz8R%0-Uq>H0<85R{%XnZ()nUmtoP~a^dJ(R<=Q^4ZK8)izuB#FiT8h(oYAR>T*hOh@!=&z*b@^2 zDEo3xyLoxnsO&kb=>Dn=^OJylW<#(JR0ch?Xl&BQ!FL^ExY*3#WKGu>fJPk@MuC`^ z-+UOv$3+4`5uxV~(&oPxg9F*a11_KPt8PZ6&`S~(Wq!yT=fb*>mY&bMSvgIXz|}n5 zaT^Kp6ybM=yJCLu>JQ5#caybxfXZr6&cNWu%V$8s;utxH3;VyZEd(k{0sPtz1;mr^ zdIcJA#d%S9ctz9$4Vu8Dg3bABHj8?~T_l>=6@~-FK3Bw}0*zyxtMof!3@k86@Pu(- z+T|ONChB#dw1HuxyKMFmP%u1<=t+RYP((I0bei_Xuy9^A`;h$ps|^rlW+aPHbIW=I z{-7YVr+pJoVOZSfo8PsR*#3gv3 zKtFkSIPBn1J>LycuTbiU!g9j?RvanOm0UQ7iS>+sT)DthIo&%yE^Z9ye@7)yb^*KY zDP53>d7{Aeqgi-cFES&l_5ytPylv`QLZf(WdueOIMA<{SsDfIqueywSXrN6_xA!5qO}dl8_XQk}#);|TMX+OOH|Sxcfq|Fb!<(RV^X>us z>!({>7kiwdF`xkodd&Kw5J*R~blO?D{Q%F{?(FC&Qf4k8zO#E$6*2!EDO%Rp_2f}1 z(BO4hQh7LzY6%+04I+6FCk#y8R()yStR|L){m%i==6zimG*%l=`uJYhjq5kZ1L$~~ zOQMirnbc%3MbG>6v{F!TgoqPNAEoNstZr9M4~~2U3M(yUt;J|@HTcV#=(KsE`PBC?`fY$kUoa+wX`BCDiL%~K#8ZUs!C8nuMC{thm*Th)Nk(aGwuZHSCDU# z!o_VDzJtWfa1JuxnM939pim>OJ4tu)fS)ZkdSf}rT;w6kKp9t>UitBl6lkN>tJ zf;nvELQT#mgOa`7&NtZEnOlMs6i7&2DI9fm6q7c>pjlZhy2wYDnruNtAztkDX*wz8 z`_DR(C1QQDWtl-ns(!=yx}}G?q9T5&Q~oDUo^aIQcL6Z#c87_{RF$F;(G}=eBTdW` z!zhQA%%t&}?VX2v64};hNAuRZ<-;^{Uw29N8~0Wu`MkAGpBzTagE>y#3DA zS(357zF;yd5QDtKh`496wySH03!k+F4h;?(UL)8| zw6ad-+qsQXlAQ1Ev^CZ%{d3dL41g-g00R%TYVWq~8cmF$KHwle zr=|inH$?bd^%ce6cT<3BpVQODhpNhM32!{!3Jf*z&DO81M(XU>C-YH}?Q6i^{}^H1 z+ylbvb1;X+?Kw9d&Cnhlyg9W{{r2)<@(pu90+)Ub#+(LS(I}iPrqFQ<&1Um44m%)~ zhuJa##ty2cV?D1a2na_dXm>>F74`WESCB@;E515eU+%57VjGH4NJ%K9-5u&mc2K{S zlig@bhBa~yB()XMrYAc(>YqOG^z%{oMFHm-2+Rn_MF3KB+;p@1JTw<}G;@Ilod>;; z3V(sk_x zl-hi{m2otXDY>3)OwN8E=g}gzM7Dm{h=*Xy83qA>paMsThR$_0B|P zd#MCXg3$MeLmj&{Mjzs>@M!Z)wzcO@w!1=7ai%`#5npfAIC_-H*lJN9_f31w0$0Ct ztmfJJj_74SvV*&5r`>sb+9{=})t2~pfJ9JX<%?#w%%;^rIYGSfo-QRdU>BGE?=Hn*uUXmdyY5mn3rAz$nA?7+kJ^u%HcG&y2JD zcqMAk$z6K6yaF*-Z*%6hT_I{;{b%AD1$5|TjxSk-MMdK$kM0TO;Ybm6(7 zT=>LP>3C$)(U0c-5~`~+{^Yi8UclLJ`-^{7F0D!;dh+Fbd-MJV?=tLAaQ(+@vvifi zRxN|wmgT}FLt9N%`pCgzLZMMRP@o8rPn5)%DCZqFQ zK0RT)tZ%6AqzCs2CH@mefBp$*D4<({_Mb_(N0?P$(jy9Z~b_ z+M-IGwbACcFJiLrhjDmw7G(V4DrQ-%;DGz0yttis8?>dNe0lwylhx6~H>W-h>-~Ih zrf&`*<*fxGni8l7)S!dx2Ib7zM}Ko(a5>6+v=`9D*?+HBolK)he(&9>*(nhrk!m z?~cJ7`OdM|X-)+6F2NT&9$LxaXURps1)Xv^##lf4fy()hR7FFZr|~SDPY=wFfQYHR zo8hj@d9a3v;h3?j?kv%h94gRI*3d}2L<|ldTMD|p9R$-2E7LIWV#_8N`vT(yE*C?g z*nI;7fqo;6%q~}|JQ{fRQ6E%1s>8rohOE!VAkf#To~g7SKO?G=9-S$VnEXis2Kbb~ z#N9Hq-5a#{5NX2Bn_=TrlF?E>5!1Da5WOr};sz$O9;N6p3rAE zR`Mr~@rNDRMHiHRKDiqh>NZ)8xC}l9~mRt9XH9rT?H>g z%F4>XJ5&AH`0-#C5Xetuwniv^Y<(Agz!i^q^;G)F^=BP-*}CJJH+ENZ6i2Un?XmE= zw14|>dNqQcBTfi=ePF%6|NPRPc&NXR;%cb<<0Fj~;+G~e6IcF+K`~o!PV7^0{faa5 z*NT8uL5ulOPk!y+nk1XruPDp&h_7`UAlTu$J&Ws_On zLQB^g#Vo*8DP@`O#$`O356Kc;I3ys&k9c@eu zLE%7+4Y=;+RO^0+->^^k*3MehnWrfZ%ob`p?}5J$$uW5=mnxyWS@6YB3n+i=3ixP` z>U%Qzt)J;v3_p8nidZ`0)5$JdUd^+J;S=ry;o#UL`k!%l6l}5B^^;@1ckQz~`@|nl zb~7JM1-~_U`6B;XU{1~lw`q~F%fl`&FuP*t2xc>ng#c)gnMTC37ar}Na28xl#NJq3 zTyfoe|GwyK`beoDF&z5(^-BC?W=CQ0BN68_OKKW;{FVCu$JSTIRk>|nTL=n@ih{JX zv~Idpx&;KJMK&ef4T^+8r<5QmA|Tx$puncPOKCRU%{#Z}oO{px|6V@n2iVVE&suZM zIp!E+euB}J3@xp>e1-=*3sUOC8BW+M>Vn8DcaQ$gwmTKppgLA(S8KLLie$*Kt?1?{ z`+4&jVrX}dHo+Tvtiow`X{hI(!3OR_xqtx3C|%uK2;XWiU0nNAfLc=#Gm4_JH|_FX z$xrud5ZBWyesEdvGJb}xMEb|ayh9zPb(e730ye?zXZ3gD#=WEcXmF`1)!W4w*Z|?w z>_~LKp_bz)Imh#W(g#dy{6u7@*zF7+)9M&ujE0mukxM`lH=>j6vl%y zDe`-23fNyyVensVCL!l^9P+e1;XIF(f~?dt5zh0B-#^les)#8>xm+huX`tT3{t4jG zFVPQmQ z869**&MQki1V%w^C0J9e{i=U`@k;*Q+{o8E2gF_z^n7_1q=y>BQkQo)iDQ%P^5yp0 z6XK&5C(1aq?qFSh6>*=*h-s%c0Hg}@gIp({w5K2wFfowL7+<<3Ja#cSHL`KF(#XvX zPxkcA!>dAimuxhVcGjE%$j$()LZ?c!>#!}P92S{lg>+hqA1a3v2=U^D6U~kQwJUSv ze-#8|IPaaMRjBnk@aL@R#eH~dy$r-h4aci`UD+p9gr z{W_Zn%lHa%G3>)#t6^C+d99{~&M5Fgs7a?Q=rmPx-F?x)>JyFt=^s+PHZ)kdp=+He zGGO0wkgXY{U-scZJ4s>1Vl}jN-aP>_=Cu%nIIW21k(9Kwjh&rYS27k4LHM1S=15Y| z6|7G+u1?e_YF&u_aZ#Bjs|RWFt_g+k0gnx9x2l1@NY=-^fRccK`|e1bP|MCiPoC@! z*(}MxL1Jp4%7E)P(Y;>N{&x|PEXEjavEg3jj6L6YUgqdEh4)Qcl5s(jUE>f`I|)nb zsS162IL}>Wm`K4o(z|#3q`663;22BAC5{QRQl}X9w{D9J_ox^L#zN+Asa4t?x^*Mu z;1INBhH~65D9>hutS(w+W%Oj=q4H9+24myHtGM>28ZHfwK%gAXH#43c@!ByRucW}> z*m#uvVDuy@EJMs+bY(xb%~5H#9d%Tp^7J#2h0>b{{nc^G5X9wk%IVf~2is_n%`SjX z$olV7T@GJLFb#njO<$kpQPa`+lG2sZa)+>JB*4kESBTno-M6ie1 zQ&v9&aeC_-OZBA^kiJYLXW3n8Y>2v*&C1e39A|jK8pv)~YNx11hJz%k%Rmlo?H(r@ zP37TOO6_POPk)jYMhi&$lU*OC{t)L!bF@#S`8ZowH0OTSA?;z2I%GWdp$Rt%jlE7K ztB?9Wdk|?Rh26*P7&J6PUHlfQ{9guPT9VlM7D1&}iFu3T4B&x$K8G{1plAkmENYxY z(Cvn(TZNOAxE4qE z%r@^Q;C(chf-lc+pJIrpk479b80kv%9RDD8RC1p;d<#^Jz_!+cx^%zur(u0VW}z4x zyYY1LiXK8sgFPe3GYL7R5RY=`{w&FyLbI50YaNG6k-#=Nr^HV=BIo^Yk)SsF&$dbW zAbl`|-4<8;tYYK3R1De1{KMJQU2HXlHA_vw z5N$~9^*0sD&ZX4F9gCN*C_V1S{#URz=!trGK4I|S&-%#T7MSH3-UHdTzy%S5m~b%C_@|s+{w(Wtnh;`4rD{RKmTTc$%cj@7~k?h zD|)syzQ*JP;WhD{RDI?0<#WfPj^-l-4vS<0QMLzSoQyd5f%+Am<<^%Mh6@Z%m(B3! zy$IM-%{5C58+!Q;9rwfNaNb#^jpOs)6uWot-W?~UHa)U=uS-Mb$Q>E1 zTsV)vzSat_s+wD2IILEXlv}vKwO3qdITn%~{q`tLZdG@Af#iH}X+c%4ZagS;>8py1 zZ|yBZ#u%1m*mBYP0Q273b-_(6!Kuu3yV=|E-C#xZH+PNM4B{PcdFi*3-~jOiDIJro zgA?vhn$AbbT~oqheYM3FDs) zyOJumvs)v#$f1M!M6Hx~T z7gIm#G*POrE|{Q0yb0U1W}pA^85yAH9Jg2zuuL1bnqb>BqY5l8-kIH|I69!i zYiMna*NcJP*xFE43z!xfmkr@N7{kqn~rBNYXHhZhR|V&Y5sL``r&M7aZUFswS{viEpb%0aho?TK2Bd0N=| zR}5M-um@O;sQ=L+gQQw>$Q3nI=`K>?Y&aBc-#vNwks!17c6C(?ddI>@;7n`R{2ds3 zm&+e007Pv2QPAbawQKhb8UhGeHE)xUbo;xi8Z|n~AcP9&sr|Z`Pm&H8r2ztgEkSbSIXw!EXxfR^FMusWRvFhho=SuuX55qq$ zCU5ACU4s!n<)q-Hl= zNc9T7tMciFm+4h`%}%QKjt7^cib0m+N^Zui<#rPCbtN2}*x+ja9@jA>QQr1P8}RP$ zyFEJjjv?Dm-MqXUIi~{GJ`#Aj^W#O58ZBE^pO{V)gq(hS>ngN4j7nIt<<6dvaZ_ir zYx!B1XcWKyXvDbVgZfYI<3rlFMMZ(8_f=fuegIcC$1^&kBqxXBm1Ah`*R2{KSBJ#{ ztq;&|-2*aWxMF;QK*`#+hTUOsI2eBDRq;~VxbEiG@ORfgtWFd0y#13hzut^-`F96- z+?9AP=o*(!HS7cn&255g{ozE>Xi*6`KuqHUJR^6?pPijuq<_2w5-yWNaLZ`W;}taC zX*w-%qJPGf=0z6#*p$wVic#q{UH+{hCxe#ij={$2Tq=RrOBE~{$;q7`Kd1VJT|T$MYZ9odD-)Xx`2LNVZQ8kb6}wZ1SzC)rv_bC~grzS@D{MGva>pn{h?e+3 z8`jFXNBoOg_L3`7XV~9t+QoKm$K@e$(_k^*XF;zMZQgh?@I~S)6Or>vmm+5o9V~x#cDE!qmOqm9uJaET!>fK1Nr&%_ zl64EwZziVjNrlb_iH)aM@m<2Xs*v#Hi>3#<>^+gL=Ou+>H-EiSn}qU09bdkSK0Vjc zdiyf6$6Q%v*)(u9iY*cYPWcg;hq&rP<5cL%Sy%2hCv1s}g7PWL#w(RiD@Te<582K$ zbo>r(HN76$x=K;yl~_8)(<-o#HZ^Ww$c)r3dW}3dSUwVW+|!v|lw^Opa>Fsf{1mn4 zk6TRH+a6#2(#QOsS7`jsTXSWJx)Y~M!Ro9^$#b9AM-6MvCuZ0Yp^Q3+RDeD68@xj8 z880iRs|N?1nyag*j8xhP3^c&9mJ;{8WEOse}Zbtu}`qyiW zcBdajhR%=fd`9YSL{(TA8z0X<40#q>GcMLmtaTwTA5mkw&s)_d6Vv%#(vzjbS-Vts z$Z@rw71!4x=5(USbV@KEUo}4*;ZtnEQ9Zv_U{kMMjE4kkfODG=jNZHfiLDb@^i*pU z7>NiG`c+1ea+=&TYW)DiQ|V}Fr9OUD{|MeWG3+)rZqaOGGxJT2E-TtAF5OucS`XQ& zer=gLety)No2_M+8~Y|ja>)CfM60sjv$1^E-J?(Y1|YSJ41YP%ZkEws=lJnja=RJ zy1jk9tMr1CL=)8?<>nlh2dfKOUvH+#dHKKc4v&i+sUmrSrm z<+m1kQ9%keb-tr?fh!DNANpyR>$wgp=|)XmA$i8H~*%BTW4wYr6VPbB|Yvj|b* z^7B1Wu1ZFFPd-ZTw7RyRV7&4jc=>Sgyw0z#9tQ|hxVlEhkxL`0aBqJ}KYF;L@vT6F zU#gw4K#uI}nA_1{VkDCxxs1x!r}vqe?Yt(cQWKgJjM`X+#3@9|JFAbq^oAj+f8>py zZ+J%w7~=>=Mn+v-T?GXN*b3RIS%e-o(hA#)edJ_h{UCT>=uQJOzF%+l0h&ykTAulO zx~z^$(GzQKZ|Qto|4u_v;x;Ks*l~UTO=P176O7>=G-6Kc%3#_HC*^$Ymkw%U@OlMnWjpK=;nD<74-Iqk9k!v zSUFu+gp5Y^Q#6$B0jXY-*`;pgtDk=nb2x`i=-{sirm@-?8IjvGi){FQGxg{%ynfX4 zK9xK^9KmX#B~*5*{I~|$Wjl=atds4;0z=o6UG?6g+fEakZq#+b$6K1Ri6TF%+?eB~ z=ol_UV;}6${d%#*xN&&8-y>^oQ!MCu?dD`j@X<+4q&0U&7P*JJ7uUAShe>;?v&aHH zLEK}3WBw632BejbKSdS9fFZo%XgVQQZvGA0pX!f^n-x|a4`2wRh zFoVTh7{yCo8O^gdH)0H*Kkv>^Oay~67Z;bAWohX$Fxc%r%UFM-(o1Ac2-|+8mGI9i zznNQ+IpY4vY0vji(7n-WSJuUge|_IKT5nK%Z%<@@{su)&ZT)F$0?{e!;oj@rDs}Vw zjb(>c#qn2cb}kFwI(@S|Gj%xcRq;Bxzbs+ombGuqWnpgv)h6!MOF<`_5xH~O8RqwK zGet-|j9*tIYK?jnM=L5E`f!FVBhi;0vI{3x9|s62_|s$@E3TF9BMWwY9T++&gG!}$ zI4ZtMz5MUHb!62HtIZ-&b93G$ZmAs7$Y9dOtCU02h!&(HiR7?*hK1|d%*@Ot6y2#e zC*vcC1+70k6SyQ$vXWLMP`q^A9(w39w0Z?_%{u1hpJ5DVa5>mAm6B=#V@O!Tp(-~g z4i1i$QVtY2LA)3)To}wVD6yKLprAPURge9v^Bq`%!uo}?zHc;^SCh)0K)q$Xdk=V& zH_y=~KDe*}O{#U^Ta@2)WfW!qywy`mDN@h+yyva~z50ejl)%Z1k@k&*n+7lI@3>ry zW-$z+;Fose$r+)Z2X#fncamx5!K&tSsR4!3^72*g`x~{du>s2*%+(W-m6gr*Qrzf} z2M1R;(Ir^8gr#L=W8jwxsY2bYH$Z`vfwow&d$nU*vA$@z7TCL3y13+Vfxb+x_6%bR zr29YM^e-sSr{NvvK7z(^zTviA@<;LG)HwVyaW(dia+Z~YzBm)yve2UFAlGIN-c-Cv z8e21KzNLA-)r7L?qamFk-Du5MZ`^m!0a_pbNH}SAv}|f>s=xu=ztEEb(0GC8@qzL_ z(Z)IwHeQ-+v}sTJ@@N?cE$uYS1+VLSgZS$y6$IBdQy%&J$X7@x{BXNC{S3&MZi8+e zE;?t3{5qGbB0i~SaA%q}JI;tbTGDx;r8!)j%sPFsOe@XyR;}2*?yFZB&+ok%BB^wX za~Mh=om>0FwZwZ_thIt@?Nx-!`fqW2`@)Kf!*uz$bj8G#$w?r35S%~%dbG8!MR9*) zCXUBS`1y}FptuU_x^d%%u-6HTc1#$&xCJN)zM}$3%Mv(TnywWm{ePY;&%#pj;y~x! zHb?0s(Fz5onA67Fly(9qFQ}_d+I{lEFBtyybfsiZHEOD3AeY6N^M&*8nj*0+C!aMJ zrdbqFCmNhTb@$r+Ro(9ycco%rhmBXdaNM~w+4)IynshvZQ3Vh)V^Av<7|8=Jd(h6V zy0x|SH9H&oDp{%X`fnP^ATbGv6HIkq#AFBRV<0P{0+t7YM?`eBO7gMxufR*cXN7(h zOSp~ZDKeaE#=?RY+&_y8r{ebU`Hp!0j-A;bITnZW%31OKIU0E%$FFWw(cTSrw2n!W z+uyh)<(4o!yQJ&_{f{$AlNbsXKSeUws4bS5 zl+6Kg=sG=GOS~R=$9+4RO`~ey%}n zl(ax;t8zM|bCIo9bIAHZm~#!>t_QU=3XGf3R#eZ`9V@eApr-x}PLTfil#cLYZy(QT z+6`0A1;7*rwg&uwO#7gk)!5$No`r>l-(isxh7~a|Fo5~>tx1|Uf)xO>q%JTXpC|N8qTx%GG7^^susc2FY_tOPoy5~gB>m?jpaBe+w z*ZDGNGGOb1)fiL;;5z#1)vJ}26)-p*`KDhFNFjsbM?r9t)$=;;8y_FQ6U~n&PERB~b0#XBR$-s(>+1^( z3o|Ih55oT&0!gF6kRAM0?;AAWk@0>5KRi~g;==rV1Q%EK)4b=9%3xW2a-p!WSO%X` zZos*~kq^#8Jcn^R{4tocK95Gra&N;r)rnjT1xMbr^UCzmTb2VsD|8!y_m|f6i|-@? zqY~q7{a38UVia|@Jx%KgQ}y#BW`E-O;x@73Z2?BMMf!>+5R+-98x?+qlWzVB?={1O zr(7m#}#Yp4MNm)aT7hNy+&N7goLe-j`%dPSXzpCVD6Bpy2Il~LN z=S1Z@=cD<$gps-2pQZjfw%fa7Zo>v>r2z-GU0&kbPiR*K0WFFX_Ben!+J0BbhhZs# z$OVRnhPccJzrh4~@EY$M7>Krbq|K+2EcSL|rX?;Wh6|>+$wsrm)cY7gmyIwwS#GOw z_vxnVl$43!w;T}>0cy)%{>;?iQ3eqrMtg&avru!U*Fge#VF_s>cN)OazkP>d+J zL~I=?KmL(vkWnw1JgB>mg7D4nCk%1v%w10ReX`RNovp0iwZ>pJ+OhR2n`1fqX_PFL z|FQc{{LY{&xt+&7N^+!XD5RTJrV(wJkNft=^h_CtaPQwLsn#A0k{Af ztWA4A>%%_av#^ji%|*xOQjvm>voR70qVv}n^W^z1gZa)+;q>yFrPjY-N9?YQxou3N z0DFTqfy=VSZC7D0^zBU$T4EMu)NXz9;YwiFCj-giJ0kzHFSr=b(}s{T)(_n01VZXG zC1mpJ3kFN4C+}SMsgH)j-v?7%7%TG-7GANJ>qV_~ra;~%Jm;9WK052@KJ`n~Y-h!0 z7`|7-6+1A$thPr{;?4#q@EP=`iueT(k6FRjJAdE=T@i~61;?Eo?zUJBO^ukixVYP- z>ekBk67?$Md_aH=6Nj>{|x0A9H6 zc2+06Hm0TZ^d{EU)<}6Q1z|!uw5`EsmB{llHHI<>=?l#*S$eri$AOPvM(!>6HwE|4 zIfqcgODjvP*4PPT>PO#8*e^la>#?WTs%J%=c_QU|<9b15I79 zudd(?viaYu6}$1f?HnIRnnu#mp$=@=YAZuk=7+^U(2zlf}h<){1t@UikF9rRQ<2A!hO-5EG|kKcc7*QqcLJ*WBTrK=e#g^n)&=E zXnhu6m`19pFh|bWwJ`~T$M5iP?6lQarTWggx>ThU*KAELde+n|%dv6-dbtlUO*bWF zYPiU(#A3v8^7~mHo)Wvcc3&SKF>&#+N*7E;ovT|VBP&bb8Ajy`BQPL=3V3ovPCC*( z9J`X!O+s=hL$QF%>0xNZYNva;SnJb{Vc=|VrPj;uI_TVL&0hZ1X*Y=pWtXt&pAQV(Z1pEGq6|Ehmxjd z5Eaa`k#eluq`y&d{3t}$Ith1wn_{wr-HNK_1}y&G$BvM5bCpgXw}*du7*z}`EE@}! zgieHfT+tF#uy7PvaBpslsU#Z>vJ$j0uU|Lt3e)=_sWVVUq2V;WHeB!DPsU$*kkrBy zeV!SANl)6-Yh|RueU&0(tRiYlrN9^$-$X) zwPu@~D}NV2$fn$hPol1P6ulmkFLNM-I?j(|S|tWmxs|LA6|nBRj8?g!b9A&ER?G+T zQY_%y!71ramsfFfivx4vks*x|ldf;3Jr}vTw^8gb0D;&zUuvnNVrNIdQ9zQO-iHdd z0D}maKcJqe{0ZQCs6yd(qN%BOum>WoEV%VUx}gmA(`)4UpDG!3D@*yeCmGv&qi9_9$<9p~SJR6}EU& zPo!WVswYlx6UFRZ>71)AOyShVu@2T}sIwi3x(8@^q{2Bmh~5~-2TX#7nEbWWx?K47 zJa#^oOOq!_(CKOKUsIaNIt3@7!yl`FQ=&dNVgj26fHm9xT{`s;_=!8R~4qYVH~V5(aK!z z_Pv)QRv+Vd3b4*#476#C0Rw?a29uMia*w~C6|E0&qP`GH1!xqzMt*8}c`SS}9i3%> zE;HNjD1KS&D+rKC;O9d3bJ5i_0Q7A#TI$)8eg!~;!7r*XJSDam-i``Gh2_|i*Kv{j z>i9DGfg=fT5;~-I%pV`k_X4>JqWwThp2^_1uQTK*_ze!4Fae&&skIJkDvE92VCJ3a z6BqNLO6x|)MhIKlSCLPH2^zVbm9S6IJhqq$!^$+ej}fP)mrxt|6{3NCqWE6YB(4R7-JNDwV!0#aT8POHAW9!>ElfhZKJ`q1$j$Hkco%cP zDf{iF-;CD;E^@%G!@LO65wE(vS6*^tf-c=5m29DjM+ z08aVz_g-UuT6mOzyhffolS|)M4IzWEVGZ;^8$JF|df+Cv0Q+FR3zbVRAl{=zb_**y zvb&Ja)L8kpyo&U{n&T*xzp|GnG8SZIFS4+-Q2i6VwNs!g;NHHxvpYsQUvv5>q~#~B ze;r$y?$s84>}Ab>!|lNyo}9!ofXLwJrV3P}q)+fyvC{?dfR!{2KpMNAPmQO`elAbw z=^^tX^iC3m8%`2Ds~%G{p2a{b7!yKF`oh7Zkl6oNJ)@$%+GeT-M#sS`Kl`s&{%j7W zhf8cIaeSC3I&%kR9%1Z_G1m!&BOkx5o7E7d*EcK}gaNMaFtSJr<_3MQl_7qIAH?7xV4vissH(b+(afAZlXT}_m(4V-;a$E=ki?yNDfpK)8m0{Ab0H^GtCG1@ zejatjHah$4nVYvm5Vr71<={ng6{YM~_vQco%KjxGdnwR8uha`KkHA5Gc@7@+p9c#i z1=Gu{|MM#`g0-gIw3a+8+La$z!P^+`zaJ4aDocD>l>^czbD}e6reP?--(M!20h8Jc zH~6hcTyfZTR#Ys$c;@k$2hCt1OuT@}`%)tj5$uUmLw^dj|Ew8(uGA$w7WH(zv2cJd zL?k2&xDb=oraehHI@pd2=N|krs(X2sT~u@^`eot2{umT}AUXW+($nMfRT<)za5V0& zb`Ul_fGeZ){`FLAcMPX7Y8L3@NBKJ?x|AI{yX~cqy4CIws$sN?SonXe zUoh|tN{jvqMeDNSF3980nRfMbQHl0_w&+ULF$R;1wBFv0MOK`DRxieUsM>{Rj1_yg zEWgSZ23Guu3>e?x>l}K&e-i>*(5?5bu!{fHG<;&|EQ!DiRX+2eFZ}dCA@%1E&d&+! z<5J3M>bsw6M6_sb8N&FAh`u;cT@NgnfKi+D1%{{bV&JMfhWUmq4dnJEOA{V%mRZ4Q z2pm@_{#y8^7+&i;9xrWU5d8dB?0QCxXK$3c%*vTwxl(?9nS}dDfIBK1CG@WeHwyIn~zHnsivCC|EGGDfB33uB2@!^MWUwxCV_*XPdK z*uDr^P20}mX^SbF?9J;zl~9Tt$3$GJxK^gWJgd<&*ZzUuZPU+UoTA#>yW$f5sG{?A zsFIrQT=C3Hb-F}!RDa>8bP1>+?YnJi^Q8$g*zB}jtP|1O^IA(XxL}j~pADgO+1BnG z-qmmvK~u1tWwfPMX$UJuN%UV#hnmfh4NLs?z-N?4duT)#^OS2RD+6PJ@!?tA&GIAH z*oJ=YSHeVP`G!HvzPC;#^lHe|2`TN|nv#BzENZBr?YcVe3K`FON4T*MUTIhV_pa6m zasTtt%+)`B7Fl%rtf)G074bSb6-xkK`rijgW-+W3a(8gBUr5YNPgfgA)?NSo6Se&O z=R2-EQwd-*HMETRb3;|c4P<@El8@xW4queZE?dx!c&$2K&Zd<$HHo0_WJ9o2$(op3 z)L$8FZvRO5Ke!2rLwK3rhlTsLq9^hN+~a z4a1ghax?ws8&pj1vO~eX{68-&?XE1bS(e0>)v3V|a#7{Cfn`Tx!J8gxCJ~vKa7iHt zdwV0f1acyWa5H!7In$bIjZ^vQ(q}QxZiRu_)AOFM8a;%KR})=rPw58cr^ug{7pBY1 z-WR}M6*h=&7Wwyry=_`}ModN@KId#T(y%;!|LRrqR~P=h@x=jy`6wuRy4Fr!ak;#h zZxwRUv?}kOXo^vn=Bo}IY;P65_oD}@z1dL8^dnt32A-6P0~8cDe(uZyj(6sz_g{uO zDPq(1f8Gp!`c?AgW&M&Xq=Vk^|AP@|Fml+hz4vnfWGcgMDJm>vVP$O$B;DOwP|;^phDfNyMuBx9f=MGx7|F>i zM}JznyqujG>Ja#bP)#i{SI58%8V7%j1^%34SlAD-z0}NHUvytDyD}*?3sxSQcU+A= zd-e5_R#D!)T(f7-q>?h&QS^q?Y(M1^5m{Q%6x@Naz4s9GDUqyMiUR=LpSimQf^Q!` zKa$(GJ9)Go!ilY{tn|UXgM~rs4D2VWN13HW<*)dorkSy$*ynb3>hcVR&cUESj5PGG zHeYBYVyj$sn`N;;N6{_{fxSbPK1)3!#S9a^6c&0v6YPJb7%j37^#^Jzo>IGKh1naHL}f#%{Vo?sKjK)RKXOP#iCP5J;=^UwX<^k zgWTcs4(cQhh8C2n>KzI6yZWl@+Bp0d-V{T`W5J@OK{IL4*i4=J(X}P5kmWH?P9;2=i^JhM>{|E4vX8C zO^BfNXYwK{{Sf=1*lj>!mcj|89+)c|?bmRagpt+>n})^4@;fdKcBlESs>jC0GOK-k zmizq}x^|2z=@CckH@K(keFffsfesE+O!zgsFyBNB!PZ^}Vn;e_YQR<7-#^~&u1@SO z_PrO$FE2j`rIGX`o2mhmeJMr7U_CvcazpR0%JW$BX{-n%zcT$!FXM@)#E)x260io`Gbdn%sAiV!YW<-OrrwW~0m{n8% zy{Quh4*ggiaR_lt5Sdb}z`K(5QP^PxbsQZWuF~Op(8rx6I-+{V(DtQF%uvsq??t_{ z4+OW+arYB3?bkOlnCW-LFqIGKCKGWtG(MhXDTu^QvaqlKtiY5; zK)=oB5`l?2#1!Cb@mr4aLjT!zpuIha5QYN!6OswL?=^~Z*v@xS4OhVQ(BX}fW4Gb_ z7kyvg&v%W$076D}wkeU@CJG+~_owa=6AQU=F4HAirhYUpuJ-KbItx9&yI^_Bzi@koZ&5J&O#Yvu@6*+g-NfIvI`T`*E(@N0YeU{}|ab$K|d->_Nz zv(kt+>v0`Cg>{!OlfqA*SRy7 zky}<%nE7*(g37bak<8rO+*=;bL-~evmW(|h>vMB+^YS_cJ%n8#w6C5VpwZ~?4!s=A zCbX{#XHc9aIYkK?E_{D=erpG!o(X0-_qaG=0;MWSFn<=Ng%$dSBeqh$c%7a=&oNa8 zT0?0%YHDhyS3+X|;ONc)6w6NY-O;7SAo48bG@0V8_F!1GDUUAJ``b*9S>CI6s48(Y zit?7wn9+-cO&o0L<5oXvo_?v*X9PI6kwG z?A@@PL@My?1bpf+rDc=fxwKZ`dBFWJ4up8Hzk=26+10nR+dMwp#g$F#nCnRNm5+%z zStYv$%|Y#{GCPfnBDAzV>dbf!5_B>V*R*pEp##75`y~(tW=Z%MMG>~sO%T-9qnfNG z3en?LZrW3DytBW4m7>|d;C{4sKUa_@SFa{0Fz^=VtO{nh)njEw(GT~ZU+|y~CDSSi zVuezSF!~$XJ|j@~Tl_Msx13&0!b-^4t5EaYy?@9!*P{^;P|qa@EITpY^`7`(kM6?I za_&>HEvEa&m7JF0!f7mPARSl%7X1}0nHX4*9>oW2FxYo#tYR0wL5=%9_1E^4($dmShyBAtyFeu+r9zup zZ>&m9{bH+*gL1(9jXVX0o(-=oETFh-mD3@fAEh{$05x06ja|30iA&gd?IrhBz|_>U zUSh0clPZ;GRpbJSe6c*(7G9W5Y_4AiT0gRdzl4RV+ImjExC95`(nnt|TDi9PU87 zz_Vx17TOHt=u~QAL4|ks!VDOB{wOdmI!Vx|)OdXn^0=wpNATR<_>*B1tca?F3-$< z-S>B*4Js+&$yGaxnZEZhGY#)oXLID%f*8VT%fvH!Ftt^pRn${ix{cB`A3!B$+Zx+c0B#&tC=Z?a^+ z#iXTnjkI02Ry4gWrNV5>duE#LRaUhB_ql~bHr==ihtoBU_k^pCLiT)AyxUfuXZH!B zC~Z4Rk9MZ;mwsPJ^j~c{(Qx?@UM0M%h&^{lz(3%DU$T?pPm$P~Hs7VQ`MUXMy`RJW zpYB-YFuE~D)zfd8CgAfu@WCb5^_ZpB1$6P17=EYZ9#^G>e@HIXX>eEL76T&% zRPWhHM9n)xcZ89XO|mvUD=S9FF?w^u{DQI{EhA$lymhF^=Uz17U=o;tcpHk zh+=DcIu@1yJ;v7&F+A+(U1?`$-hb9}C=E|vYIkLeSbW@lkDXFUns|dI(Q>*xd~S?@ zf@gIwaC`;`oQ1gTHQ#U#(kh9xt7CmCje}xKB5n~NuXSrxsU;*4B?{mlShbVCCgGB{ zN_!!pjVl6W<*N?iysUPiI0fveT=P4(`%4wZM575xW6QL5x*YRomT-KiR5(7qqZK%+ zOCyi3TShikHMWcFwdBYuo=^iaL%78IBX^*mxt@K}crlh$JElPWTWCyKQc}A3F^7{b z4mZZa_;1AFA|z0M;G=vh2-q7VRyAFU5-}xj-Li7UBizumbkdNXJ42gLlD4PNeTo`*Wbs* z=g}Q-wRm$cTQ@;C6MPode*YF9W@BQsJE_&P$7NvAQqNJo6)7kAc%-}B{3i>QLbppN zPOY-#0OgZhRsWsX{*oo`B^Y&ZXlI$G;h*Du;5n5MwzHUCv~VR-&O`JhTf6Gwf`!R| zM3wzQSd2<)7Si%enQT(Cgi!-uPCA>eR<7J;(3oowiIw3%EJdr-MJHhbv6jk}^e<6& zACd&NwhleSL2^j35AnGVMYPm1jw(=8S^ENCdy9LBwA^re!6Gc}FxYO};VVh_Lm8y* zoKI>*IsX_HNo;IZuuE5LN|zHSB3D;gQ?|n3^wTkeLdj~Eq>8{)W>IZ@VvVCXAgUO` z4F(4N&y9nV77!jzaE!dEoy5Nj5*Q}(ht@J(B7Cik^|Nr36c-m4!i=uy2`*r13{#Mk zA5WtQ4mb?DlEsf=y^vvu(PBOUfz21Uz~nFK8Yn`k#3%&R5S!Q{(j{uyCMmpWc=B<& z$AXr^pL9q^kAe7Ua+vdy`6z98hdAjG{`#P2h}o9cMySALBS@i-To<2zcI@ix*34gh z+WEd?1xWU{L|u~ zrKiQ!lYif1n1%{e{SPSa7$QY|l+z&r*$NM3XGwimP{2V>b@#4RXQV8n=nV=A1!gb= zX|xIs4$fFs$xxVKOSYeDC--a17BJDd4IdWit{w4Q=JQ&SlcIx!rk>2a%ir}l1==OeIQgmW0V<} zAzsudw6`BJwMqZleDL_GXH!Bk?~0IwmY-Z4!(9qqM)a>KMQfVANh*;QWIh8{hSPC3 zi`nx-*bjTlVQF#}EdhK3ELw%nE5sn_{X-6eni(7F(GBP%kzNi(fbsxxDxyvR8u{u*m~|dEaf^bY`dO`ay+3||`UgTJc&~32IKW8wqa8?6KnEV>xSrOE9>}fn zIBZ~UZGtdnI`fu*;eyT;Qf}I1Al`u->OM>zvIFNS$8wkie zH~-E%?s6*6%4%z85VFpzWIaKJ-M6&-9f9a)9m%(A+)Jy>z-}=?NB$!5BIn~qN7}C5 z5_q;c!SJQi)c#{S#%Pil;lNgbLx-Q%x-il)|Jb8J?zvl4xb9`{hShcdZcXCw3R03d zoOcU+8eA{5ngpG3yckMeG_Fj)Z8=S?I-2r(O%93DpBQBu;tP9!BJ_VxAk z0EHvOHKaH+ky4fcEF;Fo#>U0PRaYMX{toEyB9L(HHe;j=VPSQu-R9$!F{J)Jqkk_Q zOhd=B80hRAtoCrTwM7v7A6uanEocAV%iO`fFaOo8G8IC0uqiChB>?I+Vq)Fk%Q*y7yAY<4R4$( z^jNQq@_DKk6Z6~O<%xau>S39e7rA^quM*JpWTY|+pEdcAidXA2z_yZ*{bcs+%VuiB z`K|l3{xp`|Q8KCpM(j~!C&-dh>6rKo0hB0Xz0PB4+r7CHbL$RQ$`uhC{(&ZoV&E|Y zjkhvuyFr=%Z-jl6+A0#WnGrz3^#veY zjR`gm46zINZ-;w(I9VJ$*;);Z$8bQe#O6W|1Cu3igXotI=Mpbb(=7{rN>_(kp1%e`A|~H*@n56(~?NXe4cOpWm7ZMU43er(;3Rl)}38 zIL{KxhqU zV_XrJCT2sSP{0#BIXQv)#yiDeuo94lLE8Ip^+T47clk2Z(zgcopp-N(x~k!(tsS43 zs3#^SR*YS>;8Y6-H8UtJkRQ1be14Trvym3dd#o!X~B_=lSC!f-HWuuk;WBboP zD|Cj&O?hhKnYoVCd%n+uw@s%Z}lLw_2%Co1dOB(1&JB(m54)=w-vY++tC|`nCa2=_z z-StC(;>pk@TpTXe#VjPnt-2!4LNig#UfF1)_jbuI(^{hnsO&%61ZBaK4HA&aM=u^9 zirJQcI)^DHdY!Pw`l?lA;70chX`~Rvq?M;UxoPjp9j-H=>rFy@FY*J$ zDm`o>-~;7DRwG*a_SNgxumAIyYK)@S=IHXISxC5qoKCOQl*s3L7Mj5u)Sgl}#Thmk z%eR2+V(9}57KTi)EJpzvJ^56O-ES(Fodx*=-|>Q9vUcoE|@kuGQcm2-h2c1bjXE#p=B` zj(r8D)dKerlcppVB6f2>e~;zJk2n$Yx)9b>Po}kgYPqzf{ZaYFi*Pl|tVIO`JM3qhx`bts-Nvz72E+j7$Wl^i|hznmB$UA9Y2dq?oMGTsOsX&83u z%FF3pC4-^y#IW4g%v=bm80am-P;(oeF8^gFe#`>+`1k;k5jd+yzzwgfuDbzctG+*e zh1Kwn;yD^EduI2P$8C)025h8HOp*a-j-G8 zV(|0;QDp=b;1D|UqOV726A*?is$U*i;I@I%a7lSPZrB!U%URAc#DaR_y)j3k)zo2h-xU>BuaeNe1^krsb` zX2rm8sB`#BXo)}j$f3%HgLRq&6QihuTw82k6Sb21=%9BpX1I@pOImAJkr~QJt3NYb z7B~i&Yw+^LYWB72-7h^a^|x+c3sYWPgp4L^;&e%wDrUgo;~&jbpXq%fQG@VT;YPOD z!H)sgOA1@Cgt8!CR4HCtq5C>_>0w?~{x6cq!Bh)|EAsLugkcq__eVCvaNawT1GD^T zLx0rWyn6O{))0MbjE2kM2S&ry_{-?3ZvN_rB=nP`6p5YXvRV|)@1!{=iN67j2MZ8>8cb142Z$}(U41vhI~~~5;g~mD7J+cbupd*vRt0IyAK@tGHDaf>Dxx;e z?oXp9W!e~yUc}}4du$!y2t{C zcCvz*@2<9S+ELe8&P%*$M?;aVzXNm;+^!(?qUl$F55cHs)?Npx+rioDGLOf zmxtYku}YT0@2PRamdOMRd%=w_I8oqx|1|4gBdVGh{+qf*T%`*`tT0r*h>7irzD)!j zE}opRPgt(yuHL$A%+cAu8iopvU;4V}==;8tF))jJkof+HOXhY+Tj=@5%@<}vI2Rdy zNYEEr98YL8a{_=mT;`DbB#ufmKN1n>5 zEZb&LEO;9xa7(Q%h&a-mHsi?4dwsKbu*W7P2~&-MR78>*9qOb$NyNzH;imZH|+j zI0GZfZl5*O5x8pN&cnk4AXam5VqdN*C13$vAIGL-ub-yZ2b7&RM)W9s03_7QR5^_e zm*#h_79rUDYB*mr5EEuPb1nQ_5}+2m9Q~nZ#|A4B>T(4$_pL{8U3{>*XyY$owAuaY zAmAOMnex6iSGwC88Y1HQK3^IBG-&604g6LHCyqe@4WP4(HdDitR<9>O%!+Ti5-B8H{wFX@?2x3O}X(azIKCwoWk z8?*ykX-%<+b1md2|4?Jo;Z^W$bPNWDhA6K-EqzRFeuRW-D?T+*OiWB5<8?@2IbKMr zbgAg_`+ZIr{$$f5c%r$X>ZH}hDrFNJo05A+V|Q5>WEJ`aJTkYT&na< zQ#TgaxdsEua{3yKG3&lVUU_GC79vfE%DXp2RWp_KZ$&3EloI31-wM68+{EnG53+3g zuIl0i`JKcu&nfG0xReFYtyfF5=VQE?=leMzixJd5eF1Xmvlss;O!#uFN3dOTc7wv~ zcckdi&uFYki|HPwQ+<)p9t$o!`_K2B&=8;@AGitBhLdJPd(;LZ^XUtVR@Vy|Ztu>? z7diBWhRv{mXho`WGid1bwz>adE-Xl0M#m+;Tcu&)Deg53gX+lXV#i0Uvs048 zHTSSn28N-uabrE+AGg3!8if>U`Z0|lH{Mj8pfZ*vCY?Zj?!{QB%*Fa3#@*oK;60{# z@{EyPxSmnm>@bDLMz`XeV0t`m?@qb&c#}4-njPMc`PkXpDwbbpnx;m!nK3iNV?LIV zmJOn~t$J>A4D9l+x2w$8s8eMw|6fHg8K5keJ^at zx}K!xM&=9P3SeQ2RI~b?dW2Zdf{o&952tGqvT{7yU>0cB&b&`-(+g6mehQTlAJdmOX1UVtXi$0UmzrJ*pfV5`+Y)SQ@&%&vIa=qHAUA(V3+T~7w}YMA zWOhBSwzjT8A>d5TW1->}jntHk63?-=1A2PyFX6FjQmXn*DummW_3+wk{dv-^dC6q9 zgN3oP*p$NE*qFFQdbWJYFjol*SBIxQ+!o!mtRq$xA-3jpcnH#8i%)&mazs#IA&K)y zN4#iS;qv8E>_Qi!PaArQeAs7%F}R_5a>%2^!mkzOC%c+U(0|-3bmq*dH$;s~3N54W zdd$V4)EinryeGf+PA*;OX8U1g?*PxNSbRG|tJEVty3Hb;VXv`lqy>tzwwTYXtTnW2 z3N`&!VXMXiPe+-sB`5SX=(cW*Q{LF7vXuZ{)NZ*H(XGB>TipuVTrS_pgUdR~hM4YL z*Pemu4lE_XaoWu(E@am@4d2&rEwP)NpQ6Hg2)3hlw7+jjAeD%OU}hNYj*;Dl_qa#~ zMOyjYu`bsAtelT>L=PY=5EIsc(SQiISS6!Yik;!tI)tvS5*nk9@K$($eiJHj5i978 zu(e!23q_meoypHBr?;)j(=WG1ryE-X1#;VOC53pa)$+aPk&#k!TkO4{_vZxNv0xl! z(#V?#=2ZXIR{(|Od>EbxNOOTfcaXrSCB>|G3twoCD6kkNoHMfjmG{DF;elz*oBMqg z0BId=n>JJPd90`H%Q6ma{6|%WP`_b`6o+Ryn&p%cMm{XZ{QH7rEe)0@oGqN#OM|ZA zSGx4GR2tV?Ey8(f!jGLuqLMvRhHLhtMn+Cf{}TC=tm=QmI)#Q>!o^=(QXMg)a;JNtZuc|Rmvr}g5rLPp{%QPgbB|r@(P>=-bFwG z2WvT>i?Hm3ikj<>V6Wdf{uq2Hk()cAt@t7&g)d#S31IdS^WR{)K2t0K%)o{@@{ zyk;kdUKqa#$5zrG^pcw5)>F>OM_<%;iQ+aMn4o$ElQ=gct_= z%rG14p|?@im08L>o+Ec9{U%%O7aJ^RST;63osZ57nFFQV)~e(HB~%d3GULFWqI{ey zg3r!vcbTseHwHJv0bS@fOVghROQp@;%)=sS*>+Au{{E^#2>==$O-hL%EQ>%BNB4rI=fTqFl7ol5og+Nf4X0a;kl%U}<2g1E!rC<5Tp zE`FXtF9p(YD*_LL)lH@+ZuCRXi+eg#z1uz|1HvztB;152$Le99J}R4bbYpJ7S(5-g zOIsPCD<51ivDWL{r_H|m8%Ae-ZFVR@_3MLnzp&}CzMRT&=wT^?f+4O(XZ*sjROG4i zXu@>SpPi8nrVnL$tM7TH2S*;_`s=2nPfrq=q_PwlN$`}b)L=r%DT&}w-tfP?^ci=C z-Bg*Q_-h`=I8g5Mrk*=tP!;QO@H}L5w>j!|OYp8^QinFnHiok;%6J)!ZLS_=-6+%0u2Q_{!appLc33sW zf{r(S=tX0FKl)BxyARRted9IcKe^Zu6zNbZLcro_x}MTUD0nFB*k|`V?4b3+JMM+t znVB*bXCdb=M@=q$j?o#i^klTt67ztn-=ehZ2M0&4-kT$Y z1Yj#XZ;Xg1sq~N(9U#(JG`oJ|c>Ln%PAl|xGa65edp8JiMOG5hd-d@gy%0 zeZTMT?;jZ01XQMu#iZI}la`UO2GUV9+`6H!CJD^&=I$z^iR)C*oiG8Tl7H<>=g~~+ z2n?2$-Ued&35y9pb%0sN4fgb(!-4Dcm>aAr=mpHld(WkU!SCqkGDrf4Zlb}_0+3tl z$u5IO*X$7|*5_9md-E?7ubcv$q}U33>A~Z`>gwwJe7>|Ggg*lHcz{L!%kYF*<0Hf> z;RwT)O0V5SPv7v*HX9oo%fbnKBS6P0VrOG(92|^yF)%O)Mhln^*WLvK)?^+?q;5Lk zyu_Sv<#Gjg@9ry;yg6@bWMpJZ5dtJ5(kWIJmX-*7slwTS^78WZbT%GdlggW|Q3f|0 zIf8%00kB1x(~>sds*-91u5>sAn%f`jwZ@tral!6v zv~agXEp--OMmjpWa$YYluRMpr(jW^4WqbR1uk>AD8wx?qUBx15aNJRVtD>L~+jp%D z=ygc^K?d3nIH@_bab0y=(e>Vt<({WnBcEr0KgU(R2AnV-ReppM+6Z0-LiFfYMH0H_xdJY%Y8F`t5ffP;3OFhj9=GGP^n0Vf5Z;=9gF6C5qRU6%(7^0IU8 zsu7%8Qsept1|Dnkca2W%u{)gDW39oU6}Zc0`}2I@1{d}+P9Wm>C2&EXYXGk^Yie7t zk?{ZqoXwv~6zi+A^pdOrX@EWh-v}US* z-ZU`+9xXXc?^RL$eo|E@5UyUPy=qK}&A>CW*Xt}ZIA&n>q!WShfbN^f*?4|%%HWFW z9Z&~jw0GvnMeViCTko!zFmv3wf)-~Rky}9q=`Z&!n@;ZSk91J&(sXjmOyUn8KGapu ze4Lq4o4)7mz2CTBB?f@1uBJkZjzz0N{@ZKz1TqY(;UAcL$mZ*Bm+H)k`9%3X@vwp# zTIhuc!v5tP)Mn@6dhnW}JI}ZRAXd_f80&$OV2V=c3Z9!lj+>AbL8J!5!={8jFv^1$ zHy^cTF*g`=^eklg6k4<2ya5%?Aghl*Gf!@AzY-}*GY#VDj{ya0v!PN?cBwf*)Rht4 zy8e8x_ofIF+Q+CRlpM`}zo+P0Cks3K*wT1M(Khr9=xx)4;^}zN8>JtgKIJsDHZ!B9 zrCkM19uRkop-{F-m3t8aQ_5>+sxuIzOl$8A%_<*^ksT!y27aR7#v2NWgKuD#??^1Z*6AIUWo8n9$Kep(?e+E9m+#kB zfU^9loSd8-_Lxd`8hhxdPZ&QC!m`0(`Q>?lwpR<x1|*3& zKXB5L9hshP&F3h)^(7BD*uK6b#tpT#rZaudf$cdi+|kjYQQ(h5#B~tCeoiLH5maMc z=jcwU1R_3SWE3*92%gzMFoVdhsH9ZmzM|6CuIlc-0h|-)L2Fo;-`qD>dGe?HLMfMV zI9x-6f+wp%>iu&Hz_LPG3Z{r(+D!rIP~nWoR0=Wt=VZEfgs5oY6l(Bv`nCw%JNjB> z9BGxmx6!wH@+g>|E`Dlt2S^Hg(AADtRM@}Ad@wu9{!I$P@Qs2f_PM#1UXb^M3GW~${h-LfWbM? zT<1Xg2fx$|1*1nuzkSUFtG6&6@MiLV0(i%Ky)qhdXZy7vl$rsq_clu_)p!1&Kk^U6 zk73Llkd6;!iSI~~h7GgH+G+S#+0fsD{QUgpmk#1VIGFYttf=JV<@uPAOR`YW({pc` z$vwEOt)t_kL5*A*^Ayw<@@x3}qhg@R^hU0{_eBb>Oy7S-PW}z<=jUG!JOKjhFNfZo z&}o013qSn-=|5WIziGOy8U}G%c=@Pfqjr)(l0HXJs9I!=fw(>c3g*-V^@Bad=8hJDT literal 0 HcmV?d00001 diff --git a/docs/kernel/trace/index.rst b/docs/kernel/trace/index.rst new file mode 100644 index 00000000..23b4c8fa --- /dev/null +++ b/docs/kernel/trace/index.rst @@ -0,0 +1,11 @@ +内核跟踪机制 +==================================== + + 内核跟踪机制由很多功能构成， 比如kprobe/uprobe/tracepoint/ftrace等， 以及用于扩展内核可观测性的eBPF，内核当前支持kprobe和eBPF， 本章将介绍这两种机制。 + +.. toctree:: + :maxdepth: 1 + :caption: 目录 + + eBPF + kprobe diff --git a/docs/kernel/trace/kprobe.md b/docs/kernel/trace/kprobe.md new file mode 100644 index 00000000..53bd3aec --- /dev/null +++ b/docs/kernel/trace/kprobe.md @@ -0,0 +1,57 @@ +# kprobe + +> 作者: 陈林峰 +> +> Email: chenlinfeng25@outlook.com + +## 概述 + +Linux kprobes调试技术是内核开发者们专门为了便于跟踪内核函数执行状态所设计的一种轻量级内核调试技术。利用kprobes技术，内核开发人员可以在内核的绝大多数指定函数中动态的插入探测点来收集所需的调试状态信息而基本不影响内核原有的执行流程。 + +kprobes技术依赖硬件架构相关的支持，主要包括CPU的异常处理和单步调试机制，前者用于让程序的执行流程陷入到用户注册的回调函数中去，而后者则用于单步执行被探测点指令。需要注意的是，在一些架构上硬件并不支持单步调试机制，这可以通过一些软件模拟的方法解决(比如riscv)。 + + + +## kprobe工作流程 + +xxx + + + +1. 注册kprobe后，注册的每一个kprobe对应一个kprobe结构体，该结构中记录着探测点的位置，以及该探测点本来对应的指令。 +2. 探测点的位置被替换成了一条异常的指令，这样当CPU执行到探测点位置时会陷入到异常态，在x86_64上指令是int3（如果kprobe经过优化后，指令是jmp） +3. 当执行到异常指令时，系统换检查是否是kprobe 安装的异常，如果是，就执行kprobe的pre_handler,然后利用CPU提供的单步调试（single-step）功能，设置好相应的寄存器，将下一条指令设置为插入点处本来的指令，从异常态返回； +4. 再次陷入异常态。上一步骤中设置了single-step相关的寄存器，所以原指令刚一执行，便会再次陷入异常态，此时将single-step清除，并且执行post_handler，然后从异常态安全返回. +5. 当卸载kprobe时，探测点原来的指令会被恢复回去。 + + + +内核目前对x86和riscv64都进行了支持，由于 riscv64 没有单步执行模式，因此我们使用 break 异常来进行模拟，在保存探测点指令时，我们会额外填充一条 break 指令，这样就可以使得在riscv64架构上，在执行完原指令后，会再次触发break陷入异常。 + +## kprobe的接口 + +```rust +pub fn register_kprobe(kprobe_info: KprobeInfo) -> Result; +pub fn unregister_kprobe(kprobe: LockKprobe) -> Result<(), SystemError>; + +impl KprobeBasic { + pub fn call_pre_handler(&self, trap_frame: &dyn ProbeArgs) + pub fn call_post_handler(&self, trap_frame: &dyn ProbeArgs) + pub fn call_fault_handler(&self, trap_frame: &dyn ProbeArgs) + pub fn call_event_callback(&self, trap_frame: &dyn ProbeArgs) + pub fn update_event_callback(&mut self, callback: Box) + pub fn disable(&mut self) + pub fn enable(&mut self) + pub fn is_enabled(&self) -> bool + pub fn symbol(&self) -> Option<&str> +} +``` + +- `call_pre_handler` 在探测点指令被执行前调用用户定义的回调函数 +- `call_post_handler` 在单步执行完探测点指令后调用用户定义的回调函数 +- `call_fault_handler` 在调用前两种回调函数发生失败时调用 +- `call_event_callback` 用于调用eBPF相关的回调函数，通常与`call_post_handler` 一样在单步执行探测点指令会调用 +- `update_event_callback`用于运行过程中更新回调函数 +- `disable` 和 `enable` 用于动态关闭kprobe，在`disable`调用后，kprobe被触发时不执行回调函数 +- `symbol` 返回探测点的函数名称 + diff --git a/docs/kernel/trace/kprobe_flow.png b/docs/kernel/trace/kprobe_flow.png new file mode 100644 index 0000000000000000000000000000000000000000..884ec42f97a8c7e88885bfa9693a59c1840485ec GIT binary patch literal 47130 zcmbTeby$>L+dfLD2na}tC?VY-gGi?!HGq^fh%g{h($XRV64DF|AxejIx1!|G10vGh z-M!b~^SwUr_uI$b``G`Da^Lr=tIzXVK~GfV2=MOUVPIeoD9FpGV_;yrVqjotUdICe zBSa%_gMsk|LqSGL^R>}xGOj0??D0YAM#F`np_IQi*%kDp>kl)qQW9ksrHwomw7NMM}kEsDcD31IR~M_4b+vq+V?|tM?BF&LqBJEeHY3s z#{Yfe#7*3jd2mo`5T$`K{wU5&PXvYsLrao=zIFKvy{Becm5f>i|%fe2N#6*KvZOx}EhkHZD+J#2-X6+=- zm>8e0F%yCZ9q)HTmU_!=Mk*Fke8lPLpU<>>Se|~fJ!04BwbLcQrC*WW@cjAnfo6<% zgz+E{i_Za_t`QB$UL?9t`u5n{SjUauhX=~U* zcQWaRFS@1k^NDuzvS2AET(s`rm<~oAdp?SJWZwa^vi|2?I#Yy$ifoo~qxZ$}P-7Lb zyOfWE8(0((m0%IrjbL&P2V7e5omWLyOs-t+6gDpsva`y=lPKiKOwn($5yI2(iGiM4 zYN<`T#%byOsf?1ldrh|I?qXj8zm>xoA+_)Wo8K|U0+#)=r$;-fcer`5O|UP>|_E zta~*j9de7S>Q!evZ<0nkvhnv9t-PlO;?B#Oeb=?#hlR~MU{b?kYw6Jw`1_iu+qPM< zuyfUYUS72VT1v|HIcgD?)1!qH*&`x})2}@{q@j2Cm$&DT{keiVZ#OYXae$Oqz6*eL z_`8%@qUQTElpHj`){?vOiJl+MAP|V{eo-n;soTCm*ccdJA2hW>Vo!*?Pu41zP+1`v z8A@>lK5qp{|ZsXwr3lb`m>d)s?^>P9w%X z^z9qTJA!{?3vpX4sbmZz+78r0plKa*iLg9$N&3AG58RgRW@c^h$NGE|tz5Cwx zK%yOWjLs|QZIzUNmIO0u@N_%eoZ80xVpQ+$RF)p9x)TM$!@p^c*Ep{LjR-2oxPSBV;OM08bNw+5?>LH};3P3HZodBiXi6LwqJ=4r@1Qqu|Bh{>{3OF5-HN? zn7@7#WO}a%dNcJEcQhy8!Gh!UcQi%N$y@*W%X@>&0*dQ0+?=G~5GZM$P2axGN&1ep z4Mv}Y4aXhDLf#;{Z-3(o99NpG*1&xGcdV+|-jAHb+uIogtxx=z-o?Gt|l{b_L%q~*)U;K#2Xk^0}QS=czcgDX{`N)(OM z51qcwsr^`YRGig?YS0$QWdJsE`jdx z`kG?Toh{*8l++(uh{y+0&wC5uVEpt&;{EfsNmI|?t9C4 z`RRB~kl}2>koGv!nqm$kE6EFS+`L;7Pj@1Lg{mK|nu2$=Cl$=NFsmHS)^k%MV{509 zE9{cH#J#6ZP>u`G!mVe{4hF|@@ET&d_3XV7uADTB6t9G#40xScCrU#p${%4!$_%TBf<+#vsevSR1iFdYkvO^9i4W{Mr~P*nn&eB(^tp`BnMq1uO1GYPx^m|^X6?taj|sUC z^bX=GEg8FyH5jgdjofS53zOGU;I`;Kbeh|)5=dx#Tp`CCr~e9~_dsN&l&#!qFx|vj z(J(r(5%J*K9g{#pn-{X{Wv~$Q(WTGWwQW}|Eup)eM?2&TM&9lHkOeRCkl#~} zvzZ4!e_I_cN7R1$_M~)|Il^jvtbQ(ww$x*9Rq?L(Rv>CSpx5K%2WfdY?O36mY5P?m z_*ef5{;kHdef`IeTYi^Y8NVF;@eQ&x0oG?eq*>PA5AhEYo_N;%qfie@^H{u6*q*KM zyvHs)@}l^=J$_fh!*+4Un7Amirv;uH;A~TC*DP0Z-og6lbPm@WYDoqcNJ#km%ZVfw zV|P%xkY;e0dg220R&wlG^CZ8aca}f-p2*sTPT9 zOW`?-w%Ug8ubhbEGP&}aoW6)m7lm)}Bi1eWSVHLRt;fsY;{|(cxQcE;*Zylw@wg*q z4i4L^!=*TcRKIey@`Rn2M_op28c)VHboO$!3P0;J8=EO8330R=?JcFt3)SYZhz#S$ zN(;$X&pu%Bh>top-|d?ngD%I9qY)lGO^Eu(dW7inFwHY_E#C?(XYz zHHgG9#r>KysJ4n~0=5IHR}bZ!BRM{>-rt%pZ2X;Pp$(BOj``Wo75$V)dp#gWqa`F# zqs+UOX^4y<_;FjnM%@l!^}CtPZ)4%22H$6_j7 zqmEvt;eMacuJc?Juwlvfy`6xHzCfKFw-0{>k9E zE-f5^gM2{w*Ibj`d1%cV(?TZ=lO`$xmp7H2l@ z#wbM~%dp{=+4>-xwqJIeHkL;Oko?*X=%y@uIW70^3Aa~%gjE&VCf zW|Sy=q^QX)5xedNv+{1Ih*qMyO=;{g$kBQVFzsZO-%Oen=>h{SAoTgmt_=M4jB`mBfJg&JmrwCV|XENv|LZ-vp$$+VOu#W6W z@!GevM^@M(%*@wD%9$@#ewLF`F1*`Hvy?3nb`YuO5lZ561bzjN+%}`Y5(CJT^YXy@ z{)in1uxkSFc0IOmiPXss&0$Ep!Ger(TBIWnenUL6$9g((WskcALbNTmwEc8;e!|o! zr$5;}tW@T|z3AKbv-d42vZ*oLUU|IdmGTa{A&+Z;WV{O1DUaVc{QS`4V7WrV%OkFv z`p{Rg28*5P0fzT8k*oE$#a?FHBC3y46;zI<@q41I`r!@x%gE`Y9d`;(H;e@*Srgmy zrNiUB6j6k;JU8dke|zP7(<^Q}AWJ#}iZQ*9B=xMPsmz{5655uMCOMH(G( zNw%X!dRr=XSpOAiyzN%vsb@S%qIO>S89Rjc1$Ip4TLM+7${k@di3{NuZEG{O9lgKu zqmc)9GUXc;pLtB5Ul|YNc(-A`m#c9969*G68;tW+;Uv#=F%>pQev#nHaeR)FxKC$ojBuW zV0ffU4G(%n8Q@v{4bt`3^}G&`YDyo{oPO0<{n|b9hSzgq2dDMXjNS4fwZ)K|%ULY9 zzy8O%CAnuvpyQPa~~SR-)V<471FUn=b7R z+@YlZ6MABhP*WRrk}+@I{m)9`pHxDlkZ$@c5gIkPB)#cp_7*;(X%!^LAUKx4tutS} z>t%^8A#+H!o0+p=x+RInxA=Fj-W9;sg(HQ9+};#K**;%d7h2$f<#7c-K!iv_1E;*` zP`aK%Ujz>#%f$WHOR2DrM|6DdJ33?3ULOV+wqBy>@R`GW-$lc0m$qn9+pd3s(feBx z{@n*U3xr6xU9VT)#(U*NC$@vs-*VUAf?$8j3}JDrd2V$y?A3Mm5Udvnd6PR=|L)NX zUGr^UrW!T}nSynG9vg?I^Je|9{0T>gp&^TYl9bfr##}F1D46OgM(b&smW+A%1q_J=re&2Wa?rzxwz&+NeqZ?-+)_|{^)?l!7&9a1PA@{NhN626ViYGI z7wPnDxHYeam7pb4SzUyX`ZDW5Xhg`F)jN`Y@vC7D&KOAWBV@N!f*C$l7=|2hhvup0 zL?sRFiUgaYx0&!iGY&n1A0g+GwiLX`84QX{1J=XDn={kcOi4W0y8_k?|4g?Nw*poo zQ$kziyBbZUY4=n4&;ooV;L6`Kp4T_`bpMjzH2P4h^0CBQ8I7J70gH|gc+u$?LIVrV z_O26i7&2!3O2}hTjz*tZJ^nM+-#JXd5nfa2&wbGc3B*KRxCm!(y?ba57DJWHN^M86%sh821ts71%O3PfiT)*6R)#j0N{hn$b7=XiExIZd`oA&P>hv%RvVf9N!)--evU zt2f-hv8Sof3~3-{dYnwoe#55_pM!bGwE{|<9_xUmb)-QUs3hctcYq$f?O&spz|K#R zOwAX!)`R=``yl|)9!RH%cuE;C=gtgllqB@QJY zpDJ>Hrr2Hh8b<9sUi=~1L3%=bHz5uW2~7}6UUhsC*DcNKW*Ms(8!U=l5RyiR9{D}i z-*uP3Y$f3I$O#}&=_%uRR;e&+@v5BhQq`JbC|;CybsCV3!OypefR2BKIO0*+njP3S zF)TF)u8mu69-3wi<;U{HQ5isrjVEw8v@>6pJa-^-TTfI=DrqYR6$vJPA=h<-@@x0W z|D%Vuqq+DV+C*{saN8XZyS}t*`gsUn)9V*_tArdylq;{lr(f|55WZ9T4V{;ptN6m$ z!|_rjXbbe_dV^su@b(8^^aZTCUe+s1NO!~7dyu>L;=r1+vH!g$c7Zo`f>G)CsU*Bg zY0cVh@2rsNPrTnkW@#+np%uK7$^-djPjW8}H*b}*6t&W|*gRG_ao&C+CPSk6?tDg! ziZH8r?2A!nQ7O>H?-+k4Z_1@nU7HlDq5VkgsjXh$*>_vA{0XnxQRTT&{dr)G99)gp z;xh@`tpCxqz*lRBOC3Y%MbE;R%TaFrX?1NM?$`h(g5V#q;Vf-A zI~elW?Q=_e+)X9A=zFs6^;u<&G>_Er%g1j8F`kq{nFhlpj&e8RMh2QAzjMbe6~nws z?XiKSc>hFUjGv238#Ma)esjC;h_g8G9 z3{;Lk8E1^J!|T_77jTW=bnBw$iViLMZdEA0-Kf>hNT|iQ&EWKR3*`UYlKT7TYpTWNRB@e3L zoE1UgdXF1>QCTxb*hst_X?cL&%=>L51nIGr#gC3I8be zMCauA&sxlZYjtpS6!@TqWA^aUKh>p5`$Q36mtE~gxcq&IXU-_5Jp8%E&xd1#Yfak7 zq_~H=>m1@@PpKn@FHO=;FrJS2uSxObAel@$5CS&qAQ>~i++|Fh=-9nlL;c&o5KdWL z6E~`T-2TA%!KHM9AVA-}E|3-S3^C<<;>1jX2mm1#1+WlPD_2X7j&`59q;T>9UtP@* ztW-@g*cXqMC=hJWkK%Ch-FOwz;56YA)$8<>E4>^o>Z8w3{q>I@Zz}_mwDZ-C%pQO2vQ+ke(afk*9b+gwShML$CD}a@5V!FZB&_zk5ho?Z?|6SHpk0 z#iSm8A1q{ZCb(KZ85N{Px_eAE6Mfvirm8~oiIAS`vhoN@pT>-h5PKC1`bbMM^!bcJ zf-Kn{k@bzfVpNEe4U5m{TaJ^$WRN)1DU@(UzRuV(WKka^euwoZmT-<}s%t7%Lce_n z{q_WeD2aPFPn0Z$arc*lb$vl-L(5xgz8H9#K)6;d5%Hy6qU^utj|+}h-Ynr75b(}Q z==0|B+lCLQAqlqhE+d*5%E5dRX1QL0*Z&Nn7SVJCOANhO;03yv7&)U4t!J(q;w2aM zm4%eeB$`_ny|7u>x_NFWcqw=R^ztTHlz(5)?*F?Yiln{5-1 zw{p3UYseH%W(K;Y3I2?G6O1b&&v9|sDzR;W9R%g`g(WCm*EKitevH~4sD8htaAuz6 zEiHXC7uIuYHJ-TCnI1Pg*}x&4CP=#$SL)9cNWc_=(?N~@bPK2~6o?mz9=v@~0Q-mn z93PEvsho}2gG=|6d34X;5*M8Tm01ln{C5V#M8*XGgssQNjt{(o4|jhVb1!7C1#n(& z=mX01`_!KpuzZ||uP=a#IXZEpdx_}mp1B*7^2v!5xi4d3ESOV4K7FZKD4mEVr)z(6 zF{X5`Xd&X(;tqMmbxHF2+O_uBhKDF-BISJPg`Of2*bLl{! zC`mjNQxn9|M?6q+5nV^dWkJ?O{aL@zJ^v-rBIrBEuMl5-pv0J$2A=?v4QlNBzYFt} zP7zJJL^{k(LjP5q`gWBK+W3Dy%o)+d$!xAS|EI*x{&IgngM~<9Z-BJjN_*Kk`_<%| z26v(Sk$U|FFOQ1lkeE-xd&`66c}oq{LXX4mEEnM)^ck3(!iN`xG6Z8+qfV4={>9?COn$Hua9kerHBH^-7r_G(Tt)hbT5dEV+=C z-C6%ts&K!d9`D=K58(@2HHDe=AiG@SKvfOy4?Um};fRXm6)+z1EP73AahkI#X#B4N&X2pu zx)=}MGc!F;_tjBl(-xLJ>f#bq1Z?Am;kdVxbS1Db4ZnH0_T)W(9^G&*FT!T)#CUFr zpekWomNvIU_&c}VqBB`FTYm1L^D%h$zOO@rPGu*%7CT2CtJ{;`k|YNGwlG8892kN%+UKrpRS)d=IFlDXV-E!4KgbP}4T3Rlr$VLIXQ=}sFB!exntZku`de|)kIek5 zeGybh-Zkbl$rj-*C{G?3RT|^xUdGu|zO1()&nrPV21yQFC@S!SIaE1)Qy^^>)jCwa z(if5p6UI5nyKZIwry3J;1Q1D(2s58B@!#0K? z)7X#xF~@G>j;H!Jbb)~32A4ril!9f8^|sazKN+hEJZAddJ!_3{$xY{oTMvQHYkMN0 zBQ+`tKI%M69;n?0&hG_sH2J^ASx>AaL!6X#=6$|>>oMCcGoMi1x^s-0 z>w4h+OCk1p96U!8f0EL%c50175W)SWTbbMO**AV{mne%N2D?aWgoCj|IXxHO5c`%v z5?a1qn<7BT{GPLh!#gyB&a9H8190%E?|}=mvuz>4xCjFoUe7STghv54`l!{U%mi|n z^QAfZt@<^hG))ljIZPaoG-KKRADR05H)Q5Psrt-;N}tu5=x|vguC{HLL6C53z4oI6 zMSLMtYvxE|64Hr}{pv%#Y6hB~tZ4edTw-ERqzl+696-$F9cYVyCuZ(4accNXkkQV4GKqHIez7>I=FRNNcVtBNM);2v7eVoyi3C3mr_WCB>$6;1Z4+|^_(6^7D62cN zX_o{PrhV9_Q<1b3ddyR+>`CmN*K{kOPb)!XS-mkBzT~n9iL2(hc&Vh8UG_aM!VlD* zrK@xa#`KwaV!aIOu!CX2t_t4yc2(+9yV;GF-Z%h2YBoiRf+TaK!bV{OzzwkGPUHbs zRP)Qh-1>|2lc1oWvy%fBFDm|*19Ql@VF;**J{B(fqF4U1FI86J;uP=>d{oGVofaiy z(^6AuJU69CP~eRPa-I1ufr(Lo5bE8bq*UN?T^wuHHZS&97ZX z;{4E*L^&5qJ#0zKNqQ(F`Tta{d5D{SU#ukqR&vo^LuwMs<|N;QYsNH9`XhLu_?>+? zvHwO;AGK1Mshjo(^}tc3hoWwG-EyK5$jO;~gk5&CKbL7LtO)DRYvVu9Zh%x|HxZd; zIlR<0Qo?Ok-0S7i@V*;Di(^I3GKCupWn zZ1taPG^PkoOIR|(iMUG2pyOrI6LGR8!>@TkJPOKC%>SJ^z*Da$W_hl(%os==X{a<6_q6LN#2VZ5XFJS! zT$~WBV5xjt4n}a;%zQn?XLYtTM})CfeT3vrwtgGa$^7-887M)406G6$`JwD*g5KWf zj$;q;qD`B57rYf_bckW3$J}*A{rj(9zW@uU!g_c+kk&h5Wzn|PO3kYR8cD_o=hWS< zY&^38grro6?ovFj>GD91qf zrk)|DM^xp>YxUm6M%c?7MIp&rm=iv*wiyJ=WU{t9YceeuWTplpDf%0)QI6*E+BPx5JkV1u5l8(&T&v3~UY0F|`5?oNbL~OiYeiCIXa`tA6P+ z=S4*+&XWA(R8lVP%p2CanIn@ysKmlUA|&s^ATsPw0Tp}4sYfU&qbWtBePcvj2V}L{ zj_YAgPs;8;5x!fX&s-?q_1mWQR00$B=ruXFX28PAkrTj3LgL_T5Fa5SAu;#eY+%~J zG-asArgV_<=U6a=*U5f{DS2XE*~j29G0EfWbF%P<3UjGEI`grQdAS^iKfch#4CLO3 zaPf@h4r#_cGrvdb{g#BS^OLD1VwN>cjsQk%!WP8pnfjEhgPS$9mC3GtP}-Gx%r~YX z=9r3`D_R!OM;hotXRT`dgU*Vokw)j$)hw8C(r0uy;378+sV1K5Jd|8w6b$*Y!@OA} zM9watH9OCG86OA3NaVt4@^wqizXuTXBpvmreSnJ}FQ&DIQIk5q&Sz4BFTyQdTj~H$ zX?S?pv&Hjdf9*M23U7F5=p`4;bJ1TXSJH^}faz6HC*9Ge)$o7LqZQ(FJ{L9ZgYPQ4 zlu$+Ib(Y=Ekm9z5&_8#;r+#L8v0`Ia;o}IZhV_}SRk3n?w>q`{b3tcTnzv+`bO77E z{u6?eXrCrR^R5&YoRs31CHyYQKIS_lHs0PW#WSwPaI+S{cS0-lWdGZ^B{a4pGqXStgRDPwW#ujyQ ze3;gm-#29|IrXZ`RRT^vn4w#L>xe(2fH?5MH|HT^0r%y_UDSzF{(#@C*7^%ibz^5% zBVV#iAwhj+`cr9@gc@8RJxVvZqI(jfUqso?lk1odm%wm{XrN68=O-HzuV+4(1ik0NGx%*e!~c4X)}Hdt&b=lv-&Sy7l% zU8NU4QM0>FUys?Ye7pM)o$rvfLQGN2?b{Y^DAmsX7ss2TBa6d}OpdM=>-721Uw2e} z)Tay&#^xYt)qGZoqr|f4i2DH($OJMmkurj47sm7+j?jKJp0v8z`}q;?k+v4MZb>?@ z-|6A1;zzT~gLy}BzD~$aq^e7YDr{qS8D&J97e^0WA zTyVQ8tng`m!fSJpkWXV;iEwzkX|(p|5BbK8=YZxpd&Po=bEhLs5S}xN(J&<+#r@v+ zN9*J6+ocO_mYIRUUhe(m(UA%z z{o|r?>mdi%`9>d9U7JoF%j5O+JZjVcX6y}sO(?bL&gV2qU?KDmD{eoMZtyZ{5NOMq zv=Zhu-+}PV< z8`a4?EuyQxx)o5{IF$A7U#Krw&%f%OWg*mZm5u$UFk!+-t7RoF)4#RPt7Pf-yITLr z^vD{Vb2o{?7WBl{)En|;jB-Uy)kQs`@?^=pUsoNL-6v-sJ?=pA3QVF~}h?z-_4(K?d?Tb|K|3%^d^Q7`zUli9FFbxb`Ws;8!WggQZ+ zg?LlMCqYoDcs~K^&qck3+n&`ke1-&ErglF+cGrubq*fyJ!oQ3}f6*?&uNJ{`mh`a}fT_8e~${oR>r`CzAdMW&RTdYSM7AJ)t>fcpvGlA@}TH!0GW)Q;9$Q z)MIJZ%3zRx0eT;R=Z23sX-&;2)k0EjkC0qLF*W7`vyt2>aSFu?v)t9S#2g|y*z6#I z2)HGy$DH`&-Tug&L-4wvw^((U$1lXFj(NALg~aJ6intZRh0xKGB4};hr_~?9avuS) zM0H~w9Hk7L#29#h<3hCTV4o3pH_c~oDoOrB>Po%Ku0VU zRT+0t(&y?!Z{)iqn6M~+vOS>2?xc(sguf$p4H*#|YeOMY>8Mb^BUZYi6A)p13l8r&;53y^%<1X~YUEl`NXnovjfpDeZ$&^&R-gl2fwzEjEvm{Id>G_kBr|wUw;A&m`z1aJSlP8o}~&L#dBLQ08wn zdX@(q0&5M-f%W4oB#AmKl1af;Dgy;rc`QD{@CWN@bZ0)ty!soF%}PC+?Fl-a2J`66 z)~s@No>nh-{OIMGi_$Q}k`&H;E^XV91*#=dsxH4{DOyj?uA}1%0XW;U9u!}s=5qwU zCTDc+QhyGklru7>5jk6Mw6zp$`>;|#p~wfu_+nV1|X>Ly$LBizI1z21YzYKYs>uPD>%6mQd6WY>u9CW97Z#sKXc78 z>2zhPZ|4s#HM7jt^6AI^o^KCs)~HhfYHZ6>tL;r8EtG_ZqrCvJ#-Ea{<5!W`v}XdY^p_vt}(pv+@Z zkk_A9K%aNiNR=GrLybR;(6x?#5gU_OeVnq-?o(J|{aw9YI+FNrU0(bx0Mll`nHF-E ziE0_18e}?z9|1;`!X&rh>C<)DD}~agpD4^sQ`u;OA|z;Xi=fVuB+toB5TjRvGLsa> z>MYkD@zVg@tXt|anRSkznB0}=;iNyLS0YZ0IEjFP4`ffcy5!nY+F;8RPQML5u4>`Z)Sk;jZ?1UDCVzAL0?o3{XnhI%Irw!4AG@|{NVeZb1RwZc#veQ!hs+(~~Rn?P>{e z{|E6hJmdShp@BTOgY`eJpd)@kxZCL{o$R?fAq)FkRl@!-}b`cb0+N>+yqrlz?pVbPDf5 zJ0#hUV};u7Ci>KSIHNVfr^9F?Da^bWdrYY{xBa&P$&Avme{l*g1 z=Oq7M+a&#yXo^L1G4%QI@#m5YWn&2_@sD%b1D?$11wuNDTN{r~#SFv9%;VM4VNutL zua*qjr*bzj=P+B~aXrd;LWN}1a4Oz9kgBplZ#IuEPg%7o^%GAqm ze-D=<-?M2lf&z~^^e<`>M;+-?IO-dj;{7EtgX%9#(sUJA0a`*ho4DbzuAx{+`;;zs zv`ER_YXZ&G?QD|c&B0P;mQYQy0zUR1zX;Iby7befk6&(+U$PjlC$VGtz za;yy1zB2*rF10TMmn#!O_^Xj)Z#dXE1 z6V~0NAHa~ET@+osX@Su%;9MG+fEW!cz3~O+u$kBslB7{VbLEfMa#G>$SZEbOfrP-o zn@ewA^u_b%siTo&V-kwhuQYHtM|iKGi=Y+P55Hwn#vd8hbXLvRJ|_zgsD_x1p<`5L@=13%f>vgk+z~Kmvj1PW!@zp@&XaLg9JrzB)EHg0a92;h!m&uoO zlYi3zdKvOgK!V6UOr82-_!7KjRx5H86UjhnQ}d((ldZD|L_M;vY*nkjKPLRiABg8m zaP*Oe&u@bFNURvC`#Tm%$HMPN{>PSSP$5=}%AnQn?5oD~E1~7#SYV)|U_@Y$DI;bv zbh`8dgA#{zQ=H0n(AIkUy?sl%e4|u{gcT|8@_|!!j0^Y-aLs4Z| zY@Hl9{{vJ20@D|-q-J!V6)IDiT!S+VHcvh}+Z}RP{?9}K=Wn``2mwzOV!ZHO<3Lde zo?e*?U{EL?$@H$v4Qb$(1u0lL9|u6{Qs4Yf5M2xlmU&MNTIR-7-99U+ql> z3^Hy0DdI`zCC>N>Ha}F2B{-l zMS)^#Ri}Ob=$HC;JgAIzo zN%`Q#%^rMb%=_4XSbsAgVCLrLpbDf}b!Xy>Y2X*9ZhlHxOfe9jj=LcT>oI}5tt;z% z{+5gxgYYkF4wUEJuHMm}O@7d=n3Xm(lw!unz8HH|TsL!DGm~DR0~^eUQInGt44L$o zgSgcR+1XAZ&}{Vf+6tvpa>{UgT)5E!5x6BPqwyj`70%YXlbH&;iiBfd2U%#m9Ux_4 zP5z7K$isyq2<{(NNDTErlJ!h7(lao?@-;a%Zap{V(#TnzEX2G5rhB6jFg-9|{bjL* zPQi5<9lZ91rXq=tRGqX%wQ`_JdrA@tkCVBgX@3=bA&>M~7Ukt7hU4BgDTx2Xm9?KV+QvdwQ93`NK6C zY`^kZH#;G2*1a5r;O`)oaJKIaZIp_b`8wq03bAz?Izkdzt{4-`O6r(cZ+4t$Pj%j-;fdOa4(q zoxm?iV;+rD(?s2-AqSEbV+86L6hO*?8y1Znj4$WHR0U{T%ox7^+Gk1RXuJ=zo_%ZG zTP;Daw(%UK@kcYntmhh@gkH8hGL*IIpiL;QJ=PS@)-=vB)djsB_oUFwLd@%+(dvI# zs^j|1_3`3fH6BQ_=n&RJI=SN1Ijuyu{s_ANQRE=zXF=*m9N-;SMv{y9aDxr!Z!kcr}nNGw(TY1LSYp^i4I;I zF28B?S3pP$;O*K_y>SV%AWyN332pUlf_bSeJ+!r?LkklIpcVfQ!59oX=izEFJx;iG zQ?;w?k=tbCU8K`P?P7#fwFK@TG!DJXWPj%cB2%Sb?PP=aWF3CPuwP`(#DK+hL>%wp zrfB595tb$F!mQ_j4q={g@d*nbx7@Xx?V$tv^ZKuDLl1SC6=;;wy$zX9QqR*FjnZ|@ zeRXp8_HbN+aq5snePyc|vuKwfd8s&8DE{i&{Zi7LdM)1kyV`8wH{GEMMC&YR7M|y@Jq~zl=J6nX08>H}_ zPzQfb5g>XyoJXAvenw$jMy;1Zx*1w*+8U+keK6*_{MIwLtaEmDHh|#n?|%{Ct_(qp zJ=XY5EPZ9WpE#U21lf8kl4v&E((u(M)m1l10<|dR#D`i@37$X4{YueXR&}b&W8k5a zs#~vFDQG-*R~iC)xQzq_z}{3}Y@A%=8*QD{+aiD^9zUtaEJW!M?=7MdDG;<0Q9AB( zadxUFk;9dk@xov?PbZU4)Z8#^ts_Tf;_FxR#Lxfi+0}lTk4`mKdU~0gN^fsyxoS)T zn9dQm0ES0rx`ryWy>Yz;2Iv_5jbL)lO9}AaTe6QI!=2{U(HQJ$(WQpoO6r~+rcnFE zPmYW^i;|+}t3q>(ZX!E5%z`+ifUEHmcG+1j{YQ@70LC%$UUgOoBSH zo2nUK3W8$4fhB|p!P{?8LKr8td4(4s)vZ5%{Gi`BSL^Uk6i%zuAVC*! zJT5ng^hrPBZC*i*B3G0oJ(@G#)%*Z?gUtG!HM|7&TlY(%N9S4x*Hj0Smry3UEpb!_ zt%tY$KXyA^3xbk@p_D+WS@5XAYN#=My$RBO`l1hYMD$R-0CSK_fZ zbaS^fP{?Te6f}eECYs`74m^Y5rmOEBcIzB+#N%Rm>qza-sEpl7LOUHCh1n^I z@81s@`-pLH-~-p!*{Y15h2!Qo$U>`kI}TE}bfI^PM6JK?i<^>r<;07liW(Nh+eDlw zCf=*6;w0KHf6sBQb-;Fa_QpY+gL5Xu@>b$>is%#zd<@$1&&OIJ#6~UAs{EiycQfw8 zQKL#u$|oK*pi7(IaA>nz=|Oykd7pLx&6j%&Sj3l&xSXh)9dSH`G+}t46@k)u0TZM1 z`yYDRp(*0H+E7N}ig6q+@`iJ2qk)U-7&BQ<#roc|haa(2P}an2ZLwRr#dZ_o&KKzS zxY0~6%R@`VOE^Xh2IypX!?{zqr+4q|-Ac_uFvORL4FMdC+aZSNwWGiZpu0(K!iPVt z;k&ibpE^teg)A|;MD6L!5 zIawdo52!G{TF;_)#2gr8V7KUcT`};EN^g(hrFOw5sM8u}-G1)X7^{yftd)K+I?N*F;>V zuz96r{72Ta&nFq&kH`T`_XJSr9VeYKKY`l@)(uJFme0OIqyRZvrV8{XvtsFFs=as0 zEG)o9J^<}}_hd|v)#dYB+)-)Chd)@JAs&BD=esIf z!_0~L`UW&P%j%e~_blAJ4g^a)IyqVF)#O*uQ0M%s$ONB)+Zq7v)edu+swv*BjRTa2BB0D5s(0*D5a1Ruy-@$0aJzoRV6!2fU&jZk*-UIfhU$TYGF$w>( zs6U{>5+7h8)GuWwzT1ybffy(~53U*)&F zL6MUVkCSwY%ET5i;kly>xY)K`ED(qTK$YrZ{5Qbe0kpzSFAxyeN1yYf!p3vG2hzZ_ z#;|KCR*iwA3qW0C0K%~wsrYb<|3&XJG}^pfWb*yv>**i{xv=*nN@PgJpmeppjfsnt zi&K^}xAoC#z}?ZnwZS@5@Js*vH1Da+`RUPsASbELbVN6q)7oz>Z-+!bnD|lWPX5GN z%a(hd>v}_B$DmJ*d2cqZL-H*f;n0twN-~AM-`lUt@{i#wsLLrQxM>{eFY~{iLLTnb zJhKl)knBC_(!bKffdKqn?VaH_vC}h%P=*+ii7y@Ej(8T$AA=x#`xTU3~AtBrv zG1n1m)t}xTXY5T$O|49GWSr6=&{SbFBEH-EsMd8uJcTD&6LB89a?_$a>GNPt*Ec_$ zQcmGdI?RJvy;9?G$?U!M&^P``^u#WRafE*4M8ebqR zKDzA>TPXAzdu}zy8M!ab#TjP`>;U2gfDd(d=DYOEUTlGDMH=>g>I2S7rfZ2|tqZt( zV|5@$BaYip-W}Y_pb|Whr(G!Sb-cSWwDFxtf=ck!z52VfUc1C>C+`8c7e*!IzPre4 z+7gPswc~IL5MPqOjUM0&S*j`G0IUS)t*oqUSF%Xlep{W}&WiR@BSXLIr! zxV8f2qXe2i-NB$BtOj?l^k+?j=EV##S&=zFuV{QV)f5g$B`8&ix!Kv%Z6r?|kMa3h z<-|ZwMkN`d+YH1l$5MXRimilo%u~M6B9}>P=@Fe9{&9)i#us+&rC%6wBW1XCYh|~j z)jKh1!xcE#^AI1ITK>ukC&>~?L zh8cl1%VYDz{f)g@iHY@18h-h%j{L0^xXlDLzoq%y#hHkWN|&&z3gU#RomGoDGGnnf zU%>T&%dx?o_s2YKlyulfbWz3LeE7JJCnPO{lznQn{7Q1?2?#kDiCG@e ztJ3S!ThKew`_PBf%!G;9)~;3Hk+Yx9@%YfS(xd1n=vU|u=!fWt7zApLm-EmJ?DL+B zQzG3v)d`}tlI>erRt43c`>|#9=5q(ezPBjnyarTI@jEc>KtIt!BR>inpVafz^sXj+ zio)U1`iaREWQ@&@Z6*0#7uLM@pMDN(kxP0he+bjyjK<5;^!Q|P@%bz;Jm>fTC+}&Y z+DiMeazf?t>!#g#yUN;Q)uAU@K}T@i{Qw8oG5unru19j*kR5A3w*C6jor9%nMXox5u=(4uUQVX7~O~J=h;o1T!5+Wj{C|$asG$^5TBMmAnE#0XI5)x92OA68mNJ&aavj|cmNQlxc zAq{sH%+L4Uf8hFy`+Bkaj&tVBnKSb|&+GzvEj2ZDexS?&$m@I#bNv-=e1VqWHq+D7 z$*-GTBK8Ne@b0x0P5adS223pn zTp%qi4cuY7$CU}0I0Uj81b}$0zSe$++k*21)ufjIcj2)f0l)eLT)KYaGw@IC9vp&O z3pKElufQekbbDiMqZLn{kIBw;~ZA{8<3*etyHcEi{rGz9ii=`ES40iE${QW#~^+oI3f>c)pYUYcS+l~~%Dd~OwJ94`Gq$&eSk z5^uV8Yx;+>IO{SMr*OYe((UE1UhQFq3egTjs@^-vJ&$k66sKB8tU90ejk!LKQDH1a zwgzawn+{c1+|25;qq!){@=RNBRu+}cuJ|!|I52*_L@)`Pn)YHqp(Jhzr_i`I$|^OY zfrB|C?N7wxH5iqrOv=p@UXYx}mYMX*ieFiK(l*eBn*W%S-FTqlpwljk(h$2~T?7(6 zLSfrtZ-LhJ*uF+NW=uMjia|EoiR1XY9!Wu_UY5R0(d9ilxwu59ZV=WJmFx7(F`e)5 z5!|v=gQ;h__!QjFUJVSFN@L1A24~xYwcodjS}=J?jq6ZbzYEoVhG$$SJHKlBDM=lI z)92@jQ(Yn`^io8vm^c+mS8f;3X{K5+dXI9DJ zgy3=`QfGQn+J+_nJs;)H{O*>V=+2T=SL{JgubmS&cuAP==vR)YP_QU9DTQ)Ztjp4e zidT3klr)rs#{g%rK1q81y=K(qJNM#WsvL9WM-{L;E4*`4$>@HylvFb?dS-TLx_9~g z^%%|wi^CxQp}VB9ae6E2VrF!< zmDrQGTZJu>OdsPiPjy(cI@vi%r8nf9t!<<|WJsw>W5yb$(@z@vf_!kxU#9wX@H9Aenrrl~##i4So$AAlhKm{EtfSCSZx zXvu4t&SobL+07T3YCTZ>qEVD?eH4^aA`1I84eq-eQ+Jy-u zH}8#!S?cWVKg1EuJsh@LYRdw0C=PuUh4-Cei~^@b+hP?ewq%r211fF5YtCWRCB}x+sKm}(LrirYAG5uR5}=P%*Y2?H7CO~&n5C?QAlx`j>b9+HVJjq(5GMpLYN^u zW>po(csSF>b45ne=Fb3F6+-G`_!Nz)UFvwOZE?)8izSQT0CoT)wLA`yB{`?ji+C=s zX2kARB;Hm#prGcvw-53+3M2l@C>qu!*9s$iYQdO*XQHnm$P#XiVGVQ51=+iZG5L;)9Or+opvbG-cB`LyJI^r&ahU#E^^JZE?XScGYTtwwW{Wt1Q+p%-9yhh zq6QHxN>8@(WMg?PD%Mj<4?GM7=y^ycao#%<1`iqBGZuUYc@4)3~uyl^u#Ywq+HR%e=g@k)JIzZIK4 z0fE%4>&?>^i?ISwoK#y-{!uociP*%9mbO_fIN7v^Q#`&cN3c5DSa~2E2V;ADlh)(d zGWD6GFR&_EBKC|R1Qs$(7r8!HHm|j|Q;@dM+tTEbbH~nO!~QzQz+h@ zs5uX6CGn3smeshN+1hwJc=$Y*X^C}>><%$%GGHe12obu<1vo&V=fobz5bKp9o2yfj zImII&Ha1>LDZ~<>UB8#SYI*D&?d~;W@4q!^|D5?P$D$((A@hb?eT+}nTTR0|u9)v1 zn3QP-rNwA?I9Rz(U``_7JPn7TVJTLUUk7$>%<3_9lxoZv3RuOZDNVrc9WkEvaaLrq!#s(6G@Phc7KDu0dWIWN^_lC=_rf&DP`gSaA0+uXHSp4YagkN~IpsKa*PsdChI zkq_k$K<?9Z^`1WR@@>NLdr$Syg;Y5_Hub4Xq2m?$gb1}WO4nZ;Ie62^Uh*6*@o9;R2L`U=un>#y+IcN2Z+=DqD z=RZJ*E!@9pp^rCNSgqWR9h(Z16HK2YBEW+>$w@v8O9z%Vee*RG^+PE2H)ywr>^YtUmZ=0vNlj!32@$+tf}IA0BBtf3ptOq9VX3kCctD zC0uz90r|y1YLV{T(U}~ZZJST^@=UV zb~2JsT;KBru$Gt&tgDUHwUt;N7fNhfe!k1-S$7ge&o{eNgK%0RV^io?$iD0M7{9bY zI$@KL&Hb#R5yM*oK7|}QlPL>dq}OX?(2hRKX}6CvJUFKl)t6Aq<~>e{I{{=IYx%|4 zTxrgMZ$w6YQ zs*-bje;hVP+dJ_QqQJ=Vw`fV-pD8tsBAo2pEhx`_r+Zh8P5xdA?(Ti-!sX#=_&zFm zj^r-?$kneH%ICN0KF`TNeR!`5fwaYlUKjy1f$NDpY3C)W2qv+iH{ev|SMjb;siib2 z7th)k;+#9iqJ-s*1_6%Tj&Yih3Ua{e<9-{5wr>*L_aF2#0m9RvaKidH3(WPtN`a8rglJx>eAd!0#_wu7Pq{kqoXtD&fUG< znYufk*Jt&O)%`fhk)wU2!D(adgacOWHt3)M=r|AzAjl%K{piV@4zP3*9mnX8lY>5#BTEqNVwhVO z@&``7EijLA0+zC_6LthM&>T94waLANm172+NbU?=tpcImWA_cw(jnP?wkgM}5Ujv8 z)U`NPR6fp|@n!Yo%EpEUelT7hMro0WzxC14%G_*Kk+nT##$JS{N{bdV#@1aYxB}HEiD*8soJwf5nHaT5Pjdl_iBw?8 zzgP2wGY}$H$GcOjz#Ln8Oon6Ua&}H?_<=2Zd zrG3@^H13<|Esodw*Md`B0)K9Pe*TGhvb&t%=3uiANH(R3RZoN$gPrcOT+-o}uiSAy zvz@^dgx6`Y(lU-C>>|!2Y{{OEmmZH&{semhk4Mvv3D*HHx&4cB2nxnk+Ni!#Q*dlM z@DlruIZ%JczN6nEOaa6IgspjLJ~<^&8~>iFQFg#y(p<}aDB#kj&Vvp$36sT~v%@n^ z-?1~82tq?PRIpJs|V|zn;05l3w=QNqYCL(ElNMx>kv`hDKCr%N59u&XtT)ZFoBFtdA!T z9FjhcGR9)tp8UQ+EHY1hYOucgPO4)PclVqR6xVXD8_#rHV}1Oj`LuQi=|z9ydr0){ zAPT3ixT&0F_>hgn^B69xOU7o={dtMH&!olN?kKX*wL{;VKtWzt>ijwcW0z;o-UT%p zEcgmR@M7;>fVw4mYt0C)474@(B342aBkxy|^-}3V`-Aq6Z-yH|VqW9jlrgrGk z?5|e>A+B%0_tF%s9JP!XV$zb9N9(7hz@26{mU)El0~RlA;i#qTP8^q7KP)-NVAApE zwfR74^YAm97_h|DM-!+LrzL~IPvD89T#lU!G)Hse5@2;Bz=mOdLpkvHhQEQIURUPM z8UzEWxS*(Jkk>u0(u{A?ldfHYv_xtJk)=IEZaj8ZPN^9IO%m4AsPU!X$Yup}{mayQ zC)bN)Fo9|sX0b0tGOc46 zCi7&6LyR?h1XLFI1n!Fgy@_k`)2Und3tDhU#K4Z+Aki~u$v@q$GOC3Z&|!!gG4G7lat-N8h? zRqvTMk-s{TRkE7F%UgjoGi?>?KGv_pJFnoqV_bj8L~bZ89T1nFfJZn+$<4|Nv5G8b z7q;}tc7%>t6jP3nJ5+oSWh@A_caVFs9v#^_uG5EWluZ}qu1VKWb#Pa(Z;{6NRNYBO zEk=~YV7(8`HbI@_II*j|GE2UA+9lYDktOHeXr0qy*-Nr+#erV42;1>`B2kQfc0t$y zb10udd5{PW`w9|7eD3&!Id!9p0=&-WLo<*Ry$v!XO93SlIQy@V!kv_>bgXu;wmMXU zxwZQFE%Di;q@=VHCqa!5^2=?+jzD0u=DBho+u=drq8idF}wG>)2G>&KpJSO@f1xuaNjiulae3XQ{MD&g9bhZOy z_g)}1WC<{f^8C11&=&Sykc-!X8yzox@B2=u(9B8RmVxxhREmFSeKq2C;Q6|tN+9e` zA?}{L_y9dFvT*q!Y^~VE?A0}l7o7X8KC@2`(8xkZMS;sx0tf&PA3^08D!it>@Q#-ONxKMrFf1~~d zP)+(ie#Ee8uW;OoD$#4ekJEtijzZ+Yu!qR=IC~aVb?Wi4Q(xQsN)J4Lbeiwh8)>iO zAns7xt*R@dlnE{Yo&}Ozz<`3KN{safSZTI4CqmxDQII zLOdqIM?oy-5T#MMyFI_VlH-w%d3Y6~WvFC>W&T3!c_CY)1a1qxeTw7AX*w7>ILoC` zku~RcXsj*1E#DN9w=nT1-ywEaJu(H3wLJWBbpDK5N-iMQ!3eN8tlnHbQq3{m{_StL zuG$8hLE2pc3RNMOZadn`Jil^N`&htDHM%RHhWFP`9wD8jLr@ao?7DFVs$dpfN2|^? z>aC^Pg#a=E8^zs4*MnFzc&I(p5^D0QIm;5HU{ibTpuCcH;^9?*jNo8H#sUXW5pOvF z!2kr|KNS*zLjat+7UG%y5*v8-V}QSb$~-k7JD=GJg8IRhDqjjH9U^!WK7SaU^#S$6 z+Gbh6eYXy9(g_M&-`4K~k;EyJ>_3<(AXPA{W{=>WrKZkdJ8ZBo8NLJvQ%;aQ3@Z={ zQVxyjkZJkM`^Z-5LEX_ms4#OD4nDqU`cGIW!9PD_0L?=zn{`#3>ne{^RW@14nx+4vUGuk;UUrXc)7FZHV=_@$(=WcAQSj`pAYV(rOvsmaT z;J@|^hohhvBsoC(EIYux0SjXmeD!l=L2wc%KM(+a<3WXlB(NPm_UGLQdP!wvK`$tp zrXk?erL*&X5#_O}=M#I%Iy-oO zgw?)vAOJhz6tpq}qT0t!?hBHG13HPzHmU=t*Lw5l($y|=Z3fEw@2g2*Dko)*>A?R{yl^)LtJ#O6l*PA1JFrm}e8suO5w2IV7Ldzby5Ke`t8HgfC}b@oQ(#)#;V z>sSom-pU?(Iiw&+mpcxshFCiK3HuiLD9ZS*RV-e4b$#s~`g=~fmnFX17|A0SjM=iz zgE#|9`tn!OvZ8kngV^@|PR9*q!mJxioinBAS~uc5a9zciA1i!yQ}A_5C=T|wOq><; z(h-Hux8W0p7l`ndL#;1-JKf1yHooJsT;k8*4yvl4mIk9LLuc#}GvjgTTiKLD`271m z;(X512a#`7GF3;l*S}mky-cyT`FQ_#tecgg{jD@2Vrx}1p+mokmEJbFh^3i~-hx7ip z1kzTn`I@<(+31=^m!N0M3U-Ht`Vp4354#4)^G^JjX>HReSfqC7DT@v=e%hdsG(VKM zo7^ozcI-^#RT<-xEF7K(F#&9zc`YL?`V}36=ND7l6KLMSwHx*W9JO%VfU%^&u3dvr zWS%Y)X59U}jdsLz#(?~Lq=P|Uf__Q1LS`3xt>o*LoKCLQb@5zLq*I@w{_!P}T(+Zd8AQ_D48pS@;bGZZOtmBX%B zYhW;pZ?s-H2$`45`1VQCbNzT{1H9&2=-w@EVBXpsOa82gj~JMId`ySQ0*x}5p}^XN z8fIp@mE1LVl8cex@7L1{bH?;VI=M!}>mgv$%7x}`GurNI(bc_spfo!5hEkoKD@B-9 z$@cg$(Gund#^@?jgc#bh(+l;hxYLm zLuyvmPIVgAYsxosr6W&kX_$K>s}L`r-etA<}J2jB3oB zKr-#wK%un%=N(7#Se3x29Tl*F;Uj-+;1aP*b6=$0{m30ICH#A6(b!{lbam4Gc4nz@ z@7cne0pwtuYC601lrA752|r*kE{+4$|F-x7N2Dy{zT-7bkE}rl6%$O zd3&;li3ZYTzt%m{&OYXq+f#(Wga^m0=Mb)_PoZjMoO1>h-%=6APC<)-2aJY@B@E+G z5B$tqy*R87BI2{Um`^cEfK-unj;gR3M>P(+ zYVv=Xngh?T?PY-8LVYTm8cf#B_I>up{h?0Kjw6>|it{{6+wCm~2oU+KQREVUKTUpE zw?}D3K1E-yS^X6!n0qI}7Sr_@(HQINk5tSUaGgX&8Ns2dBo4uG_c4rayOuW#d@+ za$d}E$seDAHJuwnF?0Wr)KL_V*j0AmOj?{&)dM&--?qeB%3zz zUa_mZWKoJ+7oQI;IH7~71{Dra8?BID-J7X5Epu*VOX?LXH%jBq1NzkYw`GsnO!ZsU z9cBFbrmbt&s}%d7rT4S@-^yo|1vD2)pWbt(y@*uy=dG&RE%%>#Fz286QJbT=IKm;f zpdAq9(1ms5#yrL}kIm5q&5H#{ZJoLMCC&ctnb#hfKZ;GTZqiST&(`k$ntQfKr6dS1 z>3)xyoT4?IVfJiyWlyZ#+{mGoTh<@^`{D`Z_|$j_)GBZxAh`9nDvP!g!?_G4-q-idCJKXOhxz6E1t ztQlCD(;4;%Mo!J0+V=TTxm&&IwiEJLc`Pwv_ro*^ZDq$MlBnW|%y~rxExv08?(g_R z0tGvXGP3Q;8qx|Pha4w=L>0CB%TSv$332QMlk8q7OYTl5Feh=#cG1<2*GG4)fvx1X z+RyaeB1---F;-^ixUBy<{w3qhwDOuJ_uso5XJPjiw2kS`jx|tw@Pq^pH z^N@=Dwip@P?rWyK(lEhQAIE~P=k&ZD+|ZtO!gczTX6(K`z0@2>EVLsg&tfy2cQnZ; zDFK%pwHPWu?+_e)ExYejcT}OP{4#rKiC!!1=GM~p#lv5eH(SK071tBfmbW#<4;nX> zC+{w{f_sXaT2AU`3Yndce?N-8=`e11D)vhF?Q3|Lk$Nt3``tcDQ%AqXkA*I(T&EL2 zMDnNm>>+5`ZX`oKXoa?DvFU*UXHxJ8&*zw}o6xZAxFp|DWZ`sQ>Jfo~jPrsFwmiK$dn925= zl~7n0M(WJp>ZNOot(Af-87?hPqE1)Ay3(Yob?+v1c|4QKnP-Yfw-Mc@JG}usZz%iw zQrt{K0wpK5?$hVpD|xA`zq$ROa+m+wO`SR5Y3~O=V}*5NBAIqFXKt2CFqK$>pfTD; z@O_Nb)O!1v6-d-g@rbnyaCVfChCM>wg0vv2*I& z;N5li3%v-58`FcTk)DDKz2XB5Jo+tnk?S#XV;>f&EPlK;JAJ(EJ7z=N zpm(;Ax=y+Mw*ToA*9&=B1G!__r49owyKgrR$8HbJ-|nq{e?FC^=)|wcV=3xeRr)Ik zs%>>oUZi-k$>61j#^SyeM7dF9(F@-$oV(xA?#UG3lrmNLT*yoa|-%T;F$( zUCXCA($tLP$dtbIYIz=gGVO8$E7(#7wCCq^wp~J0FyB8Z1^b(s|Mt%1XG>g1JEms1 z2hI;Pyy{A@4?C+4jhkpF;CMv>kIQGnvHs2_x95apDB^BRx}7TPRsGvV(sjGs-0Ani zolJC#kU7p!-V(_U;;GuP8j9(j9X*1 zi?5EB-sEk7ZH9Gd?%PS0e43v(JL8eDO<%fjp3~$7y;aM$tvQ5&?k=A`^LuX2IXf%` z2tz`D8R>r+D#$})s7(adoWz3-OQrbTqs5m}gS0}odOsf4YzjcSC7n+n1f*aNcfRrn z95sKQZ^}QGIIh17>hCHJFn)RGZ32Y zv*jtUDn|}FYK%q*=JczNxB6Wr9vQ`lR4NquAm4Kc%gLknJ^Yv*BS|NX3D7O_%6U6X z?BnNs0#&!uCxJ>#Om@YT$K4TzJnSEmNps#4q-qN%+*qT3lk-vML@m^wh*djCWy{RRnhQX!E zB-&L0%OQUmj1w+0Ma4+$yVt|$-%-X7ZM-InI@9lPe+VVOF*5$xh$!Jg#eZ@!J6ZjTuHzzw0v$7^+n*}-j*!4k*!_Z50z zY|@@-ICRt0*toB@!SNma<3w}MtTbu} zJ(%f{GtyGX-f6zX4a4;1S8 zVj8giU}Y%J%6uSxXDhR9=^Y(wRHM_m@A3x8-g7yGr?cN}+fK9nUR-;bOj&~C$8=E< z0aBRD(XTdkGq&`DMlXI-CXKk6Il-pVPIwbT!DnLzuWaDKFQwC{*OoaTFH;dg9rAyj zWJrlMJwjdfxML7(ZVc;uAeo7m#t3bdDM%OXn$a1yHI_hOtR@G!gT{G@VDbO!s6#dP z%q6k(8HO4G&{W<;OKDVGC z17brwy@{|>m+(y_oW3Jpu!JSbY+gs=lb!PknIs`4ENG$Foq3)w7}pCi)fQVb#9&>= ziNu(GMG(>#jM@{ux3)=G02yrJ_aYCH7gDcL5(xoE^1fy8hdap}{}rj@9!c^Wt*rHY zcv6KGUm2*iHn`OY%3Ft;v?6>EInb8r=K<@HbmIVeqOrUbauBpfFfCSGL`JnE!=>UE z!E68Ybw>~hc7}@tkcHy^ZZr^PZLh(~?V#D|#ry@YY}7u=Sk!{S8o=CCByhL5M)* zbpPm0JW)K4{qsfa1*9=a&q;SfoJWTfTAuxRP*m~E@2|LGk|u--DqV-+DIHt^c*hd=*uSOW1>L2`ZaQtt=N&{ z!~s)_ z*!^b2iujR3q}cW!P-Ml8X)ohw-)HoJw{Ybqh=8Pz1!VoHp@*h#N337aUYX=6IVOqU zw;2)~k$agi;v=GS*zrNbRlMKRy+CGUKEk^p4)r|p{pLl;}tET)5UGc6O0mV?6;dYYA-%MK7k>O!y=>JfvgTRK7`)Hpiw|fO3DSeKTrURX0Ze~ zwTa-VH!aZA7StE1`0S-+J~A*dC8~%^0Bflk<+QOp+11r0_Vld6;devNUi^765O|`O zqi~%_NYQIJnQ9gu?@%aPWlcZ_>V@1;%8cW+%Wzqp3+7++{m@)cPyp;5Hy}Taa%fbc zB!s$;FfT4H-U>MPeJOr7<_;mM>4TvL(b$tCFrxeO%n126pOaJsAMgi8(kHlITl6I1 zz8olcbp16{b6xV@OiF|VhOhV~(qx2-#@K4L*=GT;o|`ZQd6tBP_VQ&;^Zw^M(n27y zne><@ViQMCPmi$>$YfaLtTzGRJ^)-2_$m|xU0<334Pghs!+dsAoF-iiAv{2Mg6YUN zcuyoX^NcnB#fvXc_KUEH^?-883UJ3hW|2QQx-B{TN-?<^C}%wGTaIdUzTA6}2p)|& z`XGn0Y_SETnIU;{i2hwci}wlA&@{W5Pgey6-D`abSv3lby3#^kOG}mlU5#G)`Hk$7 z2cNGi;E0fkGYhSKIjpFHlJyj9bYftnr%wf370@_)p-Xs-zF$YA1oUVbi8|^&aaf(M zgf117o3U*gfn0ad!yD=83!W2_tyLX}brjb;&6A`+VI7)h@s&w!EOKWqJE^uO-bIsO zK}`_MOWaxmmQ=;UyA$i{>oHN!o>>Cx3J0^iP91Vt=}aaDs;L{mWQ4AwpOuu9nyyWX z0Qx58kxDS$`Zwy`VuSbRd7ALIgPf$LrGaJz%DKYAE!0UfLSXGA`=ypcP{epWjX9&y zL{^)luOK|rpGjWU^CVnldvj%{uFV76$vTL7^X7N-HyV+h@7py#Z>2Dkj@OK>cj>K` z*kgA6!|-IO&U1f}ku9xMsQI*>U&*6cmBnOpH@?GNpg7$$4E6T=MrPN;X!DF6`&~h0 zM5>~;vfk3z={s2(-lpmVd_Lq=Y*ZYEysYV}MfhCo#wzcU=UaWp6cm)9w4eU1CtKaX z5MA)LpnK!GH3>l0UA7CUx-+&Hlg-;=8CyEJhBNl8DhL^RW@q@chi`B&X*y`OPJc*v zY=&+P7N_F2e8|57Of*=)8z2-)LQUJPS!!!+W@ZMw0Sh1vVYfGC!M0;Q2~t#@7h37It4<*T81gG`J!&jRX3XLW{Iale#r>N zCBvtrsPG2{+s>=c9=n%MT*BK}3HB&=Qynvq6i~5TYohOa#vQ6lSL?CW(mBfr;*p%C z14<9((`~qLnY)GU6avcwwT{ll_c+_>?yTR(OL(mchsP>EOO`PGAQq*$ha->pfZ~89 z^a023L%I^Z&&6RUCnul{U8YuCWoroLMI!^0H+dU*q)Ux{6n~ECK>I0~5 z8K4T8W5d{VxMe!(>28RhCnCdZH5>5O8ZW&M9-SLiHYb|Tl>0gsR+6oq9}t$Na=8NdJO=&6 z-*rz@HWJAs3dm=Ej5i`L@lWM@^AhE&ECO<&F}|;maiFW>Fh4lQnxB?t)R80+E>o$U zeMPVSB$v`%jXS0KO`%NgEEbWZkgcrSw~MN3gipADFFH>!#4c5CUdMVq1!+hx6+Pf) z^5aTLbeN!OmWG&)HO=k~FnWCQ4Yu@d;s=_mF4LzjI-S{EoqGjlUqW|6*XM_vfw>&< z>IbnSAhT~);-Z9BvQURT+)eGpi`%!I=1+}pE+8WbW~`! zKfi!PzZ;gCmHbpz(N3JZ|DhF>*2lly5s?Vq$=Oxw0%mJ)!!~0sxQId$h^lHdTKzJhD-^iTRi* z+P}jp_sng4WSCxvd^0W1cbX#2EeRGOjAN%vcl>l8dTS;+|7Z*=14jA$3zm!C0@g;S zNd&F77%CtMHI2uBYD6$8b{4(L(VeHN11TUua+1dU6{Z&KhDZZ~c8plwI*@~FC*om; z1v=V5GN`#&QmM9fo|B20VCO`3#oF^FB~lWEiKSFZ=lWS}RfgG-c_ zKDjH&tTD*tgY@*@he6$xM=^Fi@b#!qyyWx4c1b9*39;N-Rm6~Afuca6)~9uco>d^< zs=ae)X$T^lvutoRZ#kJfa$>wPio+#}fr$pyWVbTkv$?EV5@VZkC8v{P_OM3ECf}IW zi%=zYdAU+3Mjny%GDcs1y4+I&OH1G4vM0z>597pA{Lb?2JjJ(oHrVA0;fTyb z3R?i)9%SW!pzi|IbO7marno?Q=amb&CLLNS-$3S5Jd`v5x@rsV+KEK&a_!PuICral zxQDAp^vA5Zbeq-iCcPdy)@XjKlSpEv3xSjzF}7Ficg(mr$a>Ys9=Sn>zal}ET+5p+ zQ?*=&zi>v(F17#19;W-;?UD zX@1PM&4a#^>|0b?6TlKU_(h#~ZqmLpc)GD_;Bm3&|3 zBlDD|P>k$1wTQ8%rS%jwc|eoy-s%^F8*9}{Y)4SI@Ag5CkCKs3h(RmXhhpARw|C7p z1P@bw3((0i3(t`{oxg*9PhU-g*9r+J#!`*hD@z8;*RXU3D9`+f>mL=dE_FB<;>l48 zOva-+5dN~%ojtm;4el?s zN06zbs@xusp>QS=_Hvd~**h#7p@VIW+DnzD*#>lODBJRHyBhX=KJ)7YmGY1!tOADs zWhFPT;Tks~fkQaOsNx5n25@oX7Gzwu>r&L_jzZ(g*Ao->S@cGUCOb3usphwKF(ZNJagYO;rix8iH4m8uNaam#VUG|4Q!f&81+MTy z=2agY!tc5s?m6g`2;Vgn+C(RVE(2XG&uRXJ1GJ!ECMt6{!n^KK?qNte{X0$%IYZ=* z{oUFy4M1CCa6J7pD;K`ZNefcDj{fXN2)PjL|M|+LQV!@b2z>GDyBV9wrzz3LAeaCF z9G*YQg@YmQXY^F zo0uC-L6;Z2%^z!p97h<1cdhTTILpPbZ`*V)r(>gPo)8o?u@R}VBshyn0ckc_=0V`4 zx!DhF;wxRkm`n5ZL=LD)v+VmE#;PCi+c~g6r2o7TK5D)9t9KG(UhOm7CQ3as9)IL@ z(~n-Uw00S*QDsACrh3W@*IqlEkfkCbnXEQw>gLf*L0X2L>3*t=Q`ZID+a9qG z<_=b&pE&Hd!^`o2D4}PV2q+t3Zpg9yS+szbMj54uR}g(bJe~yw+7`n&bD{fdcL^!` zgwzX7BF0PY1-DhQ2gL~s*tpslFiCW|qEl_i#d;>ndK!sb>D;04t>&AI?e^)5rb*af z1XjQwn*i0@(GT4+MbelL*TnHb)S$V1c>qn;NgryIEvuCg`QkW9;wnDx?W5``EKg~X zg$w5TR$ZnssRF2~*k)A6E9yhp!b8J^^X*>$ojeeywts6b?AI7`f4uGN{ShLKfk33< z%+7t&EDev2Nak(VgY8^5*>1B7eZ9!_Ns)p_U!Yq>(QC{1P|hgxXP1K4aAH}kU#zxH z;{?*ke=Gw6#P_cZjKwrH!h1MQ2v>H`@JQBwj`a82x*i^ z@r>3P9g+_Sb~pr@FT**`&Z^*qBC~cFRMZqIpIdN%^8Fb*+B;d2{f{!?Z<4KB?$iA)a0~pSy*sk z+dEbLvwmO-Xs^FDbad`WV8N%qcXFjKgNT)i8v`x1!ahN~ zTyE%#(P4_`d+I7R-rVuh&KP~adZosR)GE-A8HH1=g71ZcEV1#(zrjwwm)H9DF)iT6 zc#Y0UYcamQ%g85Y!YH@Q+0O#Gsb-~;Wudby9E#P^O`5ElDY13U0u39Fo?*HoHIEtM z?^;R!47mPsHgM9{^3nH|F^b#WvH0pwJ76~WV4`0k)zJ3^GVgZZ0P(nT*%OsCq<{20 z+2`xw;w}%bqeIo2BMhcdj7PMqWuh--Kc|HS|IhAv6so!o;Vj|&i2A(a*&|acNb9%u zSxk&WgqgSN)amBgO7yjB>)xA(z)E`i0EH=GP-kEkz6v~%Dqw#6%E=o)wgv>ME0y z6SMp)zy3S=z?)zZ`y>WayW|uaUU29C@vTE#we|IjPS#(l4qiKT4tSqhS{ih?1c^kj zx&gZaXRjmX(9@7Ho*ra9P0n9|mfR%(&ljbIO3<$o0?^-h1?3c#l)5?ZLU%qJU=BdR z_-Ap#QsmaHWAXGrAW-gncn}r_#MBZF1gf-Mmk4pResUA&ENeilYn|Wk=~JcO->NP= z^b1Z!Ts*w?{hMnX$h)N(S7DnuAHpmj`z!d)H*8yULKt^k{`nHv5G%+D(FOUJchKe>oAG)iwKF@k6WAKIv%Q&#J8YFO;`+lcpu5*9SF;`(246ws zj(-?sKc)M<`BTy$Aj8~IJf=$t-RpI?ySi^U{m-L>#&6FU+QuzUw2R2#kc48yKBr9B z1gIUuHCl|6YISvyWIehDllh-VFfW~?4Fto6?z zXjy|EUc1xc2a(h1xY@nv$dt|djf9a`XN8-b3WJOwuKCaPjoTBC#?r&7CMW7k5Eki_ z(s=dWgGwD+3$pAUz-0;^Ho>IPJB$`1fLApyQMYHf=UZ4M&T@bx%N^J(+ z-fm9vJBAJEu|b~=r#_w&er7Z!5Y^5fO!q&Ng|M4a7F2xi&r z4TViS$-Q9w?ZH1Q3-Y0~t{F4HM3w0*c3k)T>hZm7Y%(gXHt5o4Kn{gu7!4)68JMTU z-Siuvuar;WlH9TO>kFTN&34rqW35}W z)?#PY>$1%!T(GU^s}R3lLQa0_KqC5uED7>}BS8DL3B;b=-`UPDsazkhh!X4%FXMnV zmfnka%@Z*lIR88}aX9Zoqx$Xi#DKj{S@JKKF7W9qSKG}GSU8r}u9uYNO1$kp zCU)@O`Cf(BrdUmFu0PC-oXvMhzTBp13nYj7mUFS%uYE|L+(pCM9G!ZS<0$q({>;Ba zZIy$6q1nnKBWaF(^-i1|$M}?qIMMo4v_M(`K;Y(E+lVlb@xv?O#Vfq2fZY}T_>zXV zybl=~w$t<0KbSyIr_`&Uqe4NlV%rnM7i1P0>2nvYPH*4T9Vk7M{{MVwd&WdbCvJ^dQ0o&mWstbE_u}&0BUqZ;#%8*kIp>h zf9PK?JlB0tlxI#Z0*l$6efbHkJNKqo#0#>j3iT&Tvca>PJWj;!V}DF2*v0OEf~SiH0*!L_{; z^yDea;d5BRrm*~Rf>gdvV`JaW~2rD4)nOwxz6W)ODoG{^rQHty&SI{S&34(nDe4jH9Xg1;9%D zW&QLNU@2nHQzcCIs- z`mVeiuTg%*ObJxDmNE&{@?HuFR9EgE+bt!>duZC#$)LRGkAHcTWyi9{rBvSkT;Wt9 z03Z3RmDqJEUSywc#(9T0cmVz-`oJ0rj_WO&{xtU9=T7RJZpHVi<_EVw&I^B{ZUDS z8EwbiA3PF_N`bT`$^PKHxDT*p`u#CfW&vgvB;nz=5B+KLP)?u^oR~->*^I$_Gb`C? zV3UjE$Eq1qyRY4BHCxr9Q>{M1u3Mp{rsfFvh>|pmiuGGex4iwgLFuuU>4;m3uu%zM z%eA{4`wri?AXrADK3LloB3k2xq!9uH4>8ddGgGe zGtaL1Z^6;e>za^|E^FQCrkJm{=#uhOgw~ zJ=3|6;TMi{2l+|itCNRuM~#K7RjC!!)EMqqy^&_~5qdVRpdfjgcAM$WJ%(WMC9fv zS^qtw^8kGlsktI#Zxf%eKWa}qcItyv+tPa<9XWJ`UMR1up}C=IUatsvgeZm*HPmpKxDmPe1Vme>+BYMU_<*%AGv_iDVq?OFXrhaoryzLLP}cj`Y` zI)a<(%ATSiFv-b7Y<#!(3<^Xc!G@VbxzIet!booK-wz`K52KfnNOz{vR>_p~{3`gX zQ?SVvj>Tf);**yhBh60pu)FV9KI-2SOZgnG>39gPVnoPwRHhWd5R;q4We|Yvu0%7j z!$IPGSeQ58-bEh(4l^AcT}#1jjd}36K7-Ow`O)UVVSbU0Qo_d_q{DCOb3T6n(U+jX zbi}{a*Z<#n8kd8?nrA5oBtAm=1&P#D8)9<7yM&UT>3PWNu|dI|EDdJ$ikTg;-(zo~ zHVS|B&n&h)Vl)(Pue8Nz3<<6qlM>lhii#Hyj5-GNKV95 zEs-lu4~nCQX0T(-`~DtN3S0MKFeemC&BCV8CFWA1ab!Ga3y9L`+P9&}olHRI=d zt*JF9*Bcb$PJ5*5rposr&tSx)73%G=C(*NA|IR@C-77_bVp2mA;VR3qIV$fyyyy1d zO@YD=uSsDqa9l+tzhRyYqLHdSR)iS|Tt@CwZSA%_ZfBFFZXSOhm~TxLL?J4pm^gaXu z#E)n8|L=-x%Tx|9p04lc>ME8hv#v#o{0=~#nm1y0AykU%<&GS0dmW=P-jWDBY2)d} z0t)O>ZTUUyv;4@fnYY_hHYI9<`X{9^*E%8}SrU7pmI0SZJGF;%E0m!OpBJM5%)cWY=`YzB?hT1r}JhTO9Pb#1? z24pxxKt^@5I^j%bPyR&|5~=AvCClJ3nHJW+k47b79qi4vf+SzvjpbPEFlyI5T_?Yn zKm$|dkmQdfe_%WSm9G<=0En)Ig`oJOtcOT6Zuu})1j5D*EcE_Ts{oKL@sfCI7q-@y zrWY#GFJ*z8?}S(VbEl8#fFE&7S8iC!7nfzHOvADuRx5^y@D%?e7PMdd=}#z4psjt~^XmK78azA25b@MT~>oa@UWo_klbzR6OU= zO`VM_QDNbSU^-B8ZtYhSE4eyV;lgGt3)38-pnzZYBfVP6wekr$4=rzh!=bGm0n87wvx$466IGR2Z z`4yPkW~>MkK-jvL0&^`Qy|gHmH$#+1|4YKhbXk;q8+ZLT-b`ki2zRh0rLlj@l^a4N z0XdoV0EgIBPFIkJdgcsQu`>i%no5^^bVEm#Fp|We`E9T?!(_%XQwkIBCN6h(z6(j^ zTY{JL?5B{RV9Xbg5C;3xsmr?cyL9K=epSgVzH7kUxYjI@F5{|+;Ad@^dpYcKf?WE& z?}E!SmID~=Rg6x-jHzcqQ?7Nj=U-6gniFTB^{T zz^V}R3d8vqSmlYlKu1phJ zO!@DL6kla?JR$DoF=I>lC_m=6F!~6qdlA6Zc?B_hE&D+N)3}PVwhqxb18lQICzxg2 z5<#_h#&w1sVB$xu>t1u6yFdh|&};OSP@k9t?*+3>i*3V9g=D*E1{fe0tn@=1!1mc- zJ94Bcetq^u7Y7GNJJBNAog~$!?b5iu#Y*+~RG!@llU*FU;ITKLiS4!DD`G(Eljz-NQjJqWKBhA5Wy7M}1Bh^=IEnHz>@`i?HOr-}l1JbVX~w~I;P zt$Sl8!Y+-*E6>;P@$p5b{W_@{a?5Xi#B#1rG#@~;@}q&*nk^@fQxC7Fk)sc0tHfjC zot?v}n`31*buVW>?c4*+*Qu*og8FJ`CU|8^e_k0BtK)YOH>7o+AjC!xC{(-DHfI4G zUtkx$c+alVeaI6SQ0iO1A~P*R@;Ny;!2VzZ{2PigCuGJIwX-e*Awo?hmZeZ_SHY{b zgKZj2OMRxY&PAWN?Z{eD1;e{ZtH74GDF_+LN7tm#RC2 z!55#*b6TDoPBUKs3pf{B&3HYAXt|rPdTxCuzTLHSX9ll_o=vf!Qvy9t+RKHuU2M(E z)oW*4N)<>2+xV=kEH=fpr#OT($iTq!VA;LQ_BFc7P5Yt;7C2d-gqlTdzT(I zw5VtYLWKj+hGycY_6H(*W!w3N>7~ITGDv$zX8~|Xa5ib3+7MUwnb0MU^+Lt>gRF>* z*Jub>o8&>mJJ?kSxSL6#_JctA@;(E`C@>tf2V-rf&+EWSZ!QBV{#C02BG_fwjNkRHOtLNzG zIC7(fdweVR-R05)tU)}zf=>qO@@r~7&8uhcWnhtUAG|LY0#Wa%Sp0UK&X3lz`*QhrXzIJ~mi z(OF&3zGPi~l{Pf3k5f8S4;8aOnq)1mqGvs2ed*G9a!N`Hzkt9%snyU_^%)9{_{A$l zU1MXria`(%nua}o^yN)-D{klR+mmKT*>#(HHQo42b)TS|$SgiYVCW8zqrw0DLdF}8 zR@r#&@l$H&E0S?;eJ{J&Mo(rp7AL5Ys$YyjZ4`8G&t)lFiJ3;~71GX~%X+NbOrN;8 zb$oGwh452c+-HwC=)`s@t+k6+K}Bum{^)?nDh4YoVfE6`MVRWWx%u;;`tt2gD<$vv zhNk~Mce$aDRPIAOy&#{X!isptN))AlT@t&^l+i*e|vhF%WNlD3iE!_ba+(ah79vs_CdTu4>1zM{Z zk5_bzV*Q2)W%bEW*MMq}8v zOKO)o)!G)WT(q4_O-@94GAvk;8|sYUvhA4X^0oDLuh+?iJ8Uosn}<8Qoqc4R8`&-W$_;y5-D>l=d5;}4S>;m?ZT&vp zToh*~Ve<-b_U~aY*mlP>e(Xy;gDWc9`DxKOK5C~_u?r-to4X@ul>V!Uetgl0mM{;V zKx7Al?cfM>A&)i5qdsK4f_$NH&r8Et+65)}?!32emUqC`)x(k-uK3NIdSq9D(>l1v4&DZh{?4(tXy3QV`&Emgal2p zMylK`vJ)nz_f>Yx&dx&2$fFi=W!bu75mdY?YpQZ``06`MJmkh^ug^H{6il!2G3d-& z5GTtfZOCC;aBnRT>R|@CXXYQGngEq5Y?!>t&qJm^$kly219~q&L+xsDb(xjMlyNs2 z__`3h?fx)Ud-JJIR|?wl4)ZDoj?@<_O9rg~m9hiu^vTd?pqx~UO1}&zIiG+)^vIXD zc{RS3w|@T|QMCltB{(5Gn6g9*s7uMN?8TIn=5nr*oK60T5}S70!EVAOGycdq)O~g z^EmO5V@h$n{lACAyH2S%h`1ZPJcl@CufCzdbOaK^K`TD`I`oS#;Z^!RK!%8l&b%D< zTypKgfZY)4qlju+8*zGi`f`=MIPLQn!iy(`*c)Kk0j90rLM}1Os!+vWKSSF7-M+9r z8ZuUCDh7UgQCH)5q#W;la;?vI{0`8xSr0HF5o*HQtOHe=zZ=Q~VC2-#<6H3hOPQT| z(0U0a^6h>l`@mafN;MmQIg>n&s{X+LRZ__A+cNU@F!6Ig^Tz5mxFnT$y^QVU|1TKx zE;ro3lO?~Z&Cwd!J^}9&=;lfpTstgTW3{aK*|LZvIq9IC>yEdZ16BF=Z)(f8!&G4{ z?`2i_x+I!X_zn&1VQ4!d+BG(Kn^V5S;S6m+clFKrrR3!UAf702sjOQ|yy=KQmDH;A zKN1_U2s?)d%i>$hF0kEVEX?IHwcDpyYQ4W8APdNe28DWCR~ILV?&h#?QCWo_>ZM`m zau@ime5v*_p?@mILv~nZ&DQ0wnfSO+zr^y$Cq+y1EWPn(xaR*StYyJliWprtzEcI4 zw`IKm@cFDY^Ybtb+4;*DA@~nKx+))&N0@x^EHxHE2b4a`8JYark|K! z=?%)?c9Bken@DC>mT@pPU4OH^S1Rbk?`%OGN>iW7C}1@CLnxFKip2D{Yg4}-6gJ8E zu-+P=pz@FhT)X;?r;cZz$vfrI^=+RfQ3^Ko0SEka=ORP;_HhDo&4 zjXKY>n3B~kYxmhbs$q$hMm=Qh+*s!8+O6PMOG`@`GOq7lohMjol>CC`xXGOW`cdNR zg`Qri+O95Vs@Ca+gY(uCdeh_^rpZ+u;8M5Nz9}Bp?_Ty=kn0UNU8q-X0EF`3v2zid zTnGfJ#fK`)_??q_>W9MheDdD(ku<}Lgf8TK|IsJ+sL>MvdA+q-oAEp3wNa{$sMNLf zkxOE@;ybW!TZDwDun$9HpQKSy2YAW}+1`MC{GZQDi&)CWNc1e~d6j*uIu*Ly13o;h zc1>#3Iv(eahrDC+e&rp=#<5SYTQWz)rKM4wZLf4V-kI}W5UF(=5Ft-0xJn)gaa**6FiHG{8stSgT0|qS{;FxR!82eq~*?4AKO!K`}S$ z`V3_TH%+^#O7YaMOTlj7DFhv9c_HBkZ1~!sp^;0`hd+F^5AA5P#iq-Rz89ajC+NDu zHou~vyXgmM$$0JU7uSDwKVFd!Hz3VpadufyKLB$jbNZJI7=qdN2fPJl$FlN=HFO*KB~rMaQ9?TAET)6HX!>0@dWm{)W9lJky4W-q78qO0KruOMveuWSW3 zc{xp;-T1v@Dvu|gR|F)vV5i5wLwphe`p(d1%7~ukkGHy@7@hXrgcEx; z?ri2U1`_{RdwaW5*wz|B>OCoTLP(izOOuZ`$jkt;k`VLi9Rvb*y{>YWMY&2^Rn=y~ z;!hN`6C^C3_#vt3nD00gkoZ|=#u}-xM@)3ij*N^L?eNM%KUl2#_A+Yp34X!bL3gqw zXyitxv7QG9*T!w8x zUSOl4X@%gShAFT^>IfU#7490}lzoLYH3}f--!z{S96bDB8#1vWix|qJ9|IJ*hH*>< zHz=wBw5wgJzxz@B^dv)hsoLt)!g%L9W|It;#88!r!ZjB&TZZmo z4LnAh{9X*Gq$@f0N;ItHJyeSL)*=(S6L2pLrLJD-di^>0N8zT#b0QU>uPTUT=+|P0 z01Zt!TuHpS{N@_|ta9wJSMQTH6b;$LaTyuIP!mB1bmwU3Gi1EL{PaCP3K|kgau|QOQ?e}A;fy5se@cMRxJtD)-jawx_y~N$%gaqPc!4BckJ+6 z)L<-X56znp$m*==RGfBiz9OLSBI3zzb&w`v>i_%ZYe)T4{kIvz53D8dWHfr(##+xb Huip6|(pA=V literal 0 HcmV?d00001 diff --git a/kernel/Cargo.toml b/kernel/Cargo.toml index 18781dbd..ebe69b99 100644 --- a/kernel/Cargo.toml +++ b/kernel/Cargo.toml @@ -24,6 +24,8 @@ kvm = [] fatfs = [] fatfs-secure = ["fatfs"] +# kprobe +kprobe_test = [] # 运行时依赖项 [dependencies] @@ -58,8 +60,12 @@ wait_queue_macros = { path = "crates/wait_queue_macros" } paste = "=1.0.14" slabmalloc = { path = "crates/rust-slabmalloc" } log = "0.4.21" +kprobe = { path = "crates/kprobe" } +xarray = "0.1.0" lru = "0.12.3" +rbpf = { path = "crates/rbpf" } +printf-compat = { version = "0.1.1", default-features = false } # target为x86_64时，使用下面的依赖 [target.'cfg(target_arch = "x86_64")'.dependencies] mini-backtrace = { git = "https://git.mirrors.dragonos.org.cn/DragonOS-Community/mini-backtrace.git", rev = "e0b1d90940" } diff --git a/kernel/Makefile b/kernel/Makefile index be85d3e7..90543aaf 100644 --- a/kernel/Makefile +++ b/kernel/Makefile @@ -43,5 +43,7 @@ endif test: # 测试内核库 - RUSTFLAGS="$(RUSTFLAGS)" cargo +nightly-2024-07-23 test --workspace --exclude dragonos_kernel + RUSTFLAGS="$(RUSTFLAGS)" cargo +nightly-2024-07-23 test --workspace --exclude dragonos_kernel rbpf +test-rbpf: + cd crates/rbpf && RUSTFLAGS="$(RUSTFLAGS)" cargo +nightly-2024-07-23 test --features=std,user,cranelift \ No newline at end of file diff --git a/kernel/crates/kprobe/Cargo.toml b/kernel/crates/kprobe/Cargo.toml new file mode 100644 index 00000000..063591aa --- /dev/null +++ b/kernel/crates/kprobe/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "kprobe" +version = "0.1.0" +edition = "2021" + +[dependencies] +log = "0.4.21" + +[target.'cfg(target_arch = "x86_64")'.dependencies] +yaxpeax-x86 = { version = "2", default-features = false, features = ["fmt"] } +yaxpeax-arch = { version = "0", default-features = false } diff --git a/kernel/crates/kprobe/src/arch/loongarch64/mod.rs b/kernel/crates/kprobe/src/arch/loongarch64/mod.rs new file mode 100644 index 00000000..263e7cd7 --- /dev/null +++ b/kernel/crates/kprobe/src/arch/loongarch64/mod.rs @@ -0,0 +1,112 @@ +use alloc::sync::Arc; +use core::ops::{Deref, DerefMut}; + +use crate::{KprobeBasic, KprobeBuilder, KprobeOps}; + +const BRK_KPROBE_BP: u64 = 10; +const BRK_KPROBE_SSTEPBP: u64 = 11; +const EBREAK_INST: u32 = 0x002a0000; + +#[derive(Debug)] +pub struct Kprobe { + basic: KprobeBasic, + point: Arc, +} +#[derive(Debug)] +pub struct LA64KprobePoint { + addr: usize, + inst_tmp: [u8; 8], +} + +impl Deref for Kprobe { + type Target = KprobeBasic; + + fn deref(&self) -> &Self::Target { + &self.basic + } +} + +impl DerefMut for Kprobe { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.basic + } +} + +impl Kprobe { + pub fn probe_point(&self) -> &Arc { + &self.point + } +} + +impl Drop for LA64KprobePoint { + fn drop(&mut self) { + let address = self.addr; + let inst_tmp_ptr = self.inst_tmp.as_ptr() as usize; + let inst_32 = unsafe { core::ptr::read(inst_tmp_ptr as *const u32) }; + unsafe { + core::ptr::write(address as *mut u32, inst_32); + } + log::trace!( + "Kprobe::uninstall: address: {:#x}, old_instruction: {:?}", + address, + inst_32 + ); + } +} + +impl KprobeBuilder { + pub fn install(self) -> (Kprobe, Arc) { + let probe_point = match &self.probe_point { + Some(point) => point.clone(), + None => self.replace_inst(), + }; + let kprobe = Kprobe { + basic: KprobeBasic::from(self), + point: probe_point.clone(), + }; + (kprobe, probe_point) + } + /// # 安装kprobe + /// + /// 不同的架构下需要保存原指令，然后替换为断点指令 + fn replace_inst(&self) -> Arc { + let address = self.symbol_addr + self.offset; + let point = LA64KprobePoint { + addr: address, + inst_tmp: [0u8; 8], + }; + let inst_tmp_ptr = point.inst_tmp.as_ptr() as usize; + let inst_32 = unsafe { core::ptr::read(address as *const u32) }; + unsafe { + core::ptr::write(address as *mut u32, EBREAK_INST); + // inst_32 :0-32 + // ebreak :32-64 + core::ptr::write(inst_tmp_ptr as *mut u32, inst_32); + core::ptr::write((inst_tmp_ptr + 4) as *mut u32, EBREAK_INST); + } + log::trace!( + "Kprobe::install: address: {:#x}, func_name: {:?}, opcode: {:x?}", + address, + self.symbol, + inst_32 + ); + } +} + +impl KprobeOps for LA64KprobePoint { + fn return_address(&self) -> usize { + self.addr + 4 + } + + fn single_step_address(&self) -> usize { + self.inst_tmp.as_ptr() as usize + } + + fn debug_address(&self) -> usize { + self.inst_tmp.as_ptr() as usize + 4 + } + + fn break_address(&self) -> usize { + self.addr + } +} diff --git a/kernel/crates/kprobe/src/arch/mod.rs b/kernel/crates/kprobe/src/arch/mod.rs new file mode 100644 index 00000000..27abd559 --- /dev/null +++ b/kernel/crates/kprobe/src/arch/mod.rs @@ -0,0 +1,211 @@ +use alloc::boxed::Box; +use alloc::string::String; +use alloc::sync::Arc; +use core::{any::Any, fmt::Debug}; + +#[cfg(target_arch = "loongarch64")] +mod loongarch64; +#[cfg(target_arch = "riscv64")] +mod rv64; +#[cfg(target_arch = "x86_64")] +mod x86; + +#[cfg(target_arch = "loongarch64")] +pub use loongarch64::*; +#[cfg(target_arch = "riscv64")] +pub use rv64::*; +#[cfg(target_arch = "x86_64")] +pub use x86::*; + +#[cfg(target_arch = "x86_64")] +pub type KprobePoint = X86KprobePoint; +#[cfg(target_arch = "riscv64")] +pub type KprobePoint = Rv64KprobePoint; +#[cfg(target_arch = "loongarch64")] +pub type KprobePoint = LA64KprobePoint; + +pub trait ProbeArgs: Send { + /// 用于使用者转换到特定架构下的TrapFrame + fn as_any(&self) -> &dyn Any; + /// 返回导致break异常的地址 + fn break_address(&self) -> usize; + /// 返回导致单步执行异常的地址 + fn debug_address(&self) -> usize; +} + +pub trait KprobeOps: Send { + /// # 返回探测点的下一条指令地址 + /// + /// 执行流需要回到正常的路径中，在执行完探测点的指令后，需要返回到下一条指令 + fn return_address(&self) -> usize; + /// # 返回单步执行的指令地址 + /// + /// 通常探测点的处的原指令被保存在一个数组当中。根据架构的不同, 在保存的指令后面，可能会填充必要的指令。 + /// 例如x86架构下支持单步执行的特性， 而其它架构下通常没有，因此我们使用break异常来进行模拟，所以会填充 + /// 一条断点指令。 + fn single_step_address(&self) -> usize; + /// # 返回单步执行指令触发异常的地址 + /// + /// 其值等于`single_step_address`的值加上探测点指令的长度 + fn debug_address(&self) -> usize; + /// # 返回设置break断点的地址 + /// + /// 其值与探测点地址相等 + fn break_address(&self) -> usize; +} + +struct ProbeHandler { + func: fn(&dyn ProbeArgs), +} + +impl ProbeHandler { + pub fn new(func: fn(&dyn ProbeArgs)) -> Self { + ProbeHandler { func } + } + /// 调用探测点处理函数 + pub fn call(&self, trap_frame: &dyn ProbeArgs) { + (self.func)(trap_frame); + } +} + +pub struct KprobeBuilder { + symbol: Option, + symbol_addr: usize, + offset: usize, + pre_handler: ProbeHandler, + post_handler: ProbeHandler, + fault_handler: Option, + event_callback: Option>, + probe_point: Option>, + enable: bool, +} + +pub trait EventCallback: Send { + fn call(&self, trap_frame: &dyn ProbeArgs); +} + +impl KprobeBuilder { + pub fn new( + symbol: Option, + symbol_addr: usize, + offset: usize, + pre_handler: fn(&dyn ProbeArgs), + post_handler: fn(&dyn ProbeArgs), + enable: bool, + ) -> Self { + KprobeBuilder { + symbol, + symbol_addr, + offset, + pre_handler: ProbeHandler::new(pre_handler), + post_handler: ProbeHandler::new(post_handler), + event_callback: None, + fault_handler: None, + probe_point: None, + enable, + } + } + + pub fn with_fault_handler(mut self, func: fn(&dyn ProbeArgs)) -> Self { + self.fault_handler = Some(ProbeHandler::new(func)); + self + } + + pub fn with_probe_point(mut self, point: Arc) -> Self { + self.probe_point = Some(point); + self + } + + pub fn with_event_callback(mut self, event_callback: Box) -> Self { + self.event_callback = Some(event_callback); + self + } + + /// 获取探测点的地址 + /// + /// 探测点的地址 == break指令的地址 + pub fn probe_addr(&self) -> usize { + self.symbol_addr + self.offset + } +} + +pub struct KprobeBasic { + symbol: Option, + symbol_addr: usize, + offset: usize, + pre_handler: ProbeHandler, + post_handler: ProbeHandler, + fault_handler: ProbeHandler, + event_callback: Option>, + enable: bool, +} + +pub trait CallBackFunc: Send + Sync { + fn call(&self, trap_frame: &dyn ProbeArgs); +} + +impl Debug for KprobeBasic { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("Kprobe") + .field("symbol", &self.symbol) + .field("symbol_addr", &self.symbol_addr) + .field("offset", &self.offset) + .finish() + } +} + +impl KprobeBasic { + pub fn call_pre_handler(&self, trap_frame: &dyn ProbeArgs) { + self.pre_handler.call(trap_frame); + } + + pub fn call_post_handler(&self, trap_frame: &dyn ProbeArgs) { + self.post_handler.call(trap_frame); + } + + pub fn call_fault_handler(&self, trap_frame: &dyn ProbeArgs) { + self.fault_handler.call(trap_frame); + } + + pub fn call_event_callback(&self, trap_frame: &dyn ProbeArgs) { + if let Some(ref call_back) = self.event_callback { + call_back.call(trap_frame); + } + } + + pub fn update_event_callback(&mut self, callback: Box) { + self.event_callback = Some(callback); + } + + pub fn disable(&mut self) { + self.enable = false; + } + + pub fn enable(&mut self) { + self.enable = true; + } + + pub fn is_enabled(&self) -> bool { + self.enable + } + /// 返回探测点的函数名称 + pub fn symbol(&self) -> Option<&str> { + self.symbol.as_deref() + } +} + +impl From for KprobeBasic { + fn from(value: KprobeBuilder) -> Self { + let fault_handler = value.fault_handler.unwrap_or(ProbeHandler::new(|_| {})); + KprobeBasic { + symbol: value.symbol, + symbol_addr: value.symbol_addr, + offset: value.offset, + pre_handler: value.pre_handler, + post_handler: value.post_handler, + event_callback: value.event_callback, + fault_handler, + enable: value.enable, + } + } +} diff --git a/kernel/crates/kprobe/src/arch/rv64/mod.rs b/kernel/crates/kprobe/src/arch/rv64/mod.rs new file mode 100644 index 00000000..b73deac6 --- /dev/null +++ b/kernel/crates/kprobe/src/arch/rv64/mod.rs @@ -0,0 +1,157 @@ +use alloc::sync::Arc; +use core::{ + arch::riscv64::sfence_vma_all, + fmt::Debug, + ops::{Deref, DerefMut}, +}; + +use crate::{KprobeBasic, KprobeBuilder, KprobeOps}; +const EBREAK_INST: u32 = 0x00100073; // ebreak +const C_EBREAK_INST: u32 = 0x9002; // c.ebreak +const INSN_LENGTH_MASK: u16 = 0x3; +const INSN_LENGTH_32: u16 = 0x3; + +#[derive(Debug)] +pub struct Kprobe { + basic: KprobeBasic, + point: Arc, +} + +#[derive(Debug)] +enum OpcodeTy { + Inst16(u16), + Inst32(u32), +} +#[derive(Debug)] +pub struct Rv64KprobePoint { + addr: usize, + old_instruction: OpcodeTy, + inst_tmp: [u8; 8], +} + +impl Deref for Kprobe { + type Target = KprobeBasic; + + fn deref(&self) -> &Self::Target { + &self.basic + } +} + +impl DerefMut for Kprobe { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.basic + } +} + +impl Kprobe { + pub fn probe_point(&self) -> &Arc { + &self.point + } +} + +impl Drop for Rv64KprobePoint { + fn drop(&mut self) { + let address = self.addr; + match self.old_instruction { + OpcodeTy::Inst16(inst_16) => unsafe { + core::ptr::write(address as *mut u16, inst_16); + }, + OpcodeTy::Inst32(inst_32) => unsafe { + core::ptr::write(address as *mut u32, inst_32); + }, + } + unsafe { + sfence_vma_all(); + } + log::trace!( + "Kprobe::uninstall: address: {:#x}, old_instruction: {:?}", + address, + self.old_instruction + ); + } +} + +impl KprobeBuilder { + pub fn install(self) -> (Kprobe, Arc) { + let probe_point = match &self.probe_point { + Some(point) => point.clone(), + None => self.replace_inst(), + }; + let kprobe = Kprobe { + basic: KprobeBasic::from(self), + point: probe_point.clone(), + }; + (kprobe, probe_point) + } + /// # 安装kprobe + /// + /// 不同的架构下需要保存原指令，然后替换为断点指令 + fn replace_inst(&self) -> Arc { + let address = self.symbol_addr + self.offset; + let inst_16 = unsafe { core::ptr::read(address as *const u16) }; + // See https://elixir.bootlin.com/linux/v6.10.2/source/arch/riscv/kernel/probes/kprobes.c#L68 + let is_inst_16 = if (inst_16 & INSN_LENGTH_MASK) == INSN_LENGTH_32 { + false + } else { + true + }; + let mut point = Rv64KprobePoint { + old_instruction: OpcodeTy::Inst16(0), + inst_tmp: [0; 8], + addr: address, + }; + let inst_tmp_ptr = point.inst_tmp.as_ptr() as usize; + if is_inst_16 { + point.old_instruction = OpcodeTy::Inst16(inst_16); + unsafe { + core::ptr::write(address as *mut u16, C_EBREAK_INST as u16); + // inst_16 :0-16 + // c.ebreak:16-32 + core::ptr::write(inst_tmp_ptr as *mut u16, inst_16); + core::ptr::write((inst_tmp_ptr + 2) as *mut u16, C_EBREAK_INST as u16); + } + } else { + let inst_32 = unsafe { core::ptr::read(address as *const u32) }; + point.old_instruction = OpcodeTy::Inst32(inst_32); + unsafe { + core::ptr::write(address as *mut u32, EBREAK_INST); + // inst_32 :0-32 + // ebreak :32-64 + core::ptr::write(inst_tmp_ptr as *mut u32, inst_32); + core::ptr::write((inst_tmp_ptr + 4) as *mut u32, EBREAK_INST); + } + } + unsafe { + sfence_vma_all(); + } + log::trace!( + "Kprobe::install: address: {:#x}, func_name: {:?}, opcode: {:x?}", + address, + self.symbol, + point.old_instruction + ); + Arc::new(point) + } +} + +impl KprobeOps for Rv64KprobePoint { + fn return_address(&self) -> usize { + let address = self.addr; + match self.old_instruction { + OpcodeTy::Inst16(_) => address + 2, + OpcodeTy::Inst32(_) => address + 4, + } + } + fn single_step_address(&self) -> usize { + self.inst_tmp.as_ptr() as usize + } + fn debug_address(&self) -> usize { + match self.old_instruction { + OpcodeTy::Inst16(_) => self.inst_tmp.as_ptr() as usize + 2, + OpcodeTy::Inst32(_) => self.inst_tmp.as_ptr() as usize + 4, + } + } + fn break_address(&self) -> usize { + self.addr + } +} diff --git a/kernel/crates/kprobe/src/arch/x86/mod.rs b/kernel/crates/kprobe/src/arch/x86/mod.rs new file mode 100644 index 00000000..e8fb83f6 --- /dev/null +++ b/kernel/crates/kprobe/src/arch/x86/mod.rs @@ -0,0 +1,135 @@ +use crate::{KprobeBasic, KprobeBuilder, KprobeOps}; +use alloc::string::ToString; +use alloc::sync::Arc; +use core::{ + fmt::Debug, + ops::{Deref, DerefMut}, +}; +use yaxpeax_arch::LengthedInstruction; + +const EBREAK_INST: u8 = 0xcc; // x86_64: 0xcc +const MAX_INSTRUCTION_SIZE: usize = 15; // x86_64 max instruction length + +pub struct Kprobe { + basic: KprobeBasic, + point: Arc, +} + +#[derive(Debug)] +pub struct X86KprobePoint { + addr: usize, + old_instruction: [u8; MAX_INSTRUCTION_SIZE], + old_instruction_len: usize, +} + +impl Drop for X86KprobePoint { + fn drop(&mut self) { + let address = self.addr; + unsafe { + core::ptr::copy( + self.old_instruction.as_ptr(), + address as *mut u8, + self.old_instruction_len, + ); + core::arch::x86_64::_mm_mfence(); + } + let decoder = yaxpeax_x86::amd64::InstDecoder::default(); + let inst = decoder.decode_slice(&self.old_instruction).unwrap(); + log::trace!( + "Kprobe::uninstall: address: {:#x}, old_instruction: {:?}", + address, + inst.to_string() + ); + } +} + +impl Debug for Kprobe { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("Kprobe") + .field("basic", &self.basic) + .field("point", &self.point) + .finish() + } +} + +impl Deref for Kprobe { + type Target = KprobeBasic; + + fn deref(&self) -> &Self::Target { + &self.basic + } +} + +impl DerefMut for Kprobe { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.basic + } +} + +impl KprobeBuilder { + pub fn install(self) -> (Kprobe, Arc) { + let probe_point = match &self.probe_point { + Some(point) => point.clone(), + None => self.replace_inst(), + }; + let kprobe = Kprobe { + basic: KprobeBasic::from(self), + point: probe_point.clone(), + }; + (kprobe, probe_point) + } + /// # 安装kprobe + /// + /// 不同的架构下需要保存原指令，然后替换为断点指令 + fn replace_inst(&self) -> Arc { + let address = self.symbol_addr + self.offset; + let mut inst_tmp = [0u8; MAX_INSTRUCTION_SIZE]; + unsafe { + core::ptr::copy( + address as *const u8, + inst_tmp.as_mut_ptr(), + MAX_INSTRUCTION_SIZE, + ); + } + let decoder = yaxpeax_x86::amd64::InstDecoder::default(); + let inst = decoder.decode_slice(&inst_tmp).unwrap(); + let len = inst.len().to_const(); + log::trace!("inst: {:?}, len: {:?}", inst.to_string(), len); + let point = Arc::new(X86KprobePoint { + addr: address, + old_instruction: inst_tmp, + old_instruction_len: len as usize, + }); + unsafe { + core::ptr::write_volatile(address as *mut u8, EBREAK_INST); + core::arch::x86_64::_mm_mfence(); + } + log::trace!( + "Kprobe::install: address: {:#x}, func_name: {:?}", + address, + self.symbol + ); + point + } +} + +impl Kprobe { + pub fn probe_point(&self) -> &Arc { + &self.point + } +} + +impl KprobeOps for X86KprobePoint { + fn return_address(&self) -> usize { + self.addr + self.old_instruction_len + } + fn single_step_address(&self) -> usize { + self.old_instruction.as_ptr() as usize + } + fn debug_address(&self) -> usize { + self.old_instruction.as_ptr() as usize + self.old_instruction_len + } + fn break_address(&self) -> usize { + self.addr + } +} diff --git a/kernel/crates/kprobe/src/lib.rs b/kernel/crates/kprobe/src/lib.rs new file mode 100644 index 00000000..7e871c6d --- /dev/null +++ b/kernel/crates/kprobe/src/lib.rs @@ -0,0 +1,7 @@ +#![cfg_attr(target_arch = "riscv64", feature(riscv_ext_intrinsics))] +#![no_std] +extern crate alloc; + +mod arch; + +pub use arch::*; diff --git a/kernel/crates/rbpf/.appveyor.yml b/kernel/crates/rbpf/.appveyor.yml new file mode 100644 index 00000000..c8ea114a --- /dev/null +++ b/kernel/crates/rbpf/.appveyor.yml @@ -0,0 +1,21 @@ +version: 1.0.{build} +branches: + only: + - main +os: + - Visual Studio 2015 +clone_depth: 1 +configuration: + - Debug +platform: + - x64 +environment: + matrix: + - TOOLCHAIN_VERSION: 14.0 + RUST: 1.76.0 + - TOOLCHAIN_VERSION: 14.0 + RUST: beta + - TOOLCHAIN_VERSION: 14.0 + RUST: nightly + +build_script: mk/appveyor.bat diff --git a/kernel/crates/rbpf/.gitignore b/kernel/crates/rbpf/.gitignore new file mode 100644 index 00000000..a9d37c56 --- /dev/null +++ b/kernel/crates/rbpf/.gitignore @@ -0,0 +1,2 @@ +target +Cargo.lock diff --git a/kernel/crates/rbpf/Cargo.toml b/kernel/crates/rbpf/Cargo.toml new file mode 100644 index 00000000..1a711ac9 --- /dev/null +++ b/kernel/crates/rbpf/Cargo.toml @@ -0,0 +1,78 @@ +[package] + +# Project metadata +name = "rbpf" +version = "0.2.0" +authors = ["Quentin "] + +# Additional metadata for packaging +description = "Virtual machine and JIT compiler for eBPF programs" +repository = "https://github.com/qmonnet/rbpf" +readme = "README.md" +keywords = ["BPF", "eBPF", "interpreter", "JIT", "filtering"] +license = "Apache-2.0/MIT" +edition = "2021" + +# Packaging directives +include = [ + "src/**", + "examples/**", + "tests/**", + "bench/**", + "LICENSE*", + "Cargo.toml", +] + +[dependencies] + +# Default features (std) are disabled so that the dependencies don't pull in the +# standard library when the crate is compiled for no_std +byteorder = { version = "1.2", default-features = false } +log = {version = "0.4.21", default-features = false } +combine = { version = "4.6", default-features = false } + +# Optional Dependencies when using the standard library +libc = { version = "0.2", optional = true } +time = { version = "0.2", optional = true } + +# Optional Dependencies for the CraneLift JIT +cranelift-codegen = { version = "0.99", optional = true } +cranelift-frontend = { version = "0.99", optional = true } +cranelift-jit = { version = "0.99", optional = true } +cranelift-native = { version = "0.99", optional = true } +cranelift-module = { version = "0.99", optional = true } + +[dev-dependencies] + +elf = "0.0.10" +json = "0.11" +hex = "0.4.3" + +[features] +#default = ["std", "user", "cranelift"] +cargo-clippy = [] +std = ["dep:time", "dep:libc", "combine/std"] +cranelift = [ + "dep:cranelift-codegen", + "dep:cranelift-frontend", + "dep:cranelift-jit", + "dep:cranelift-native", + "dep:cranelift-module", +] +user = [] + +# Examples that depend on the standard library should be disabled when +# testing the `no_std` configuration. +[[example]] +name = "disassemble" +required-features = ["std"] + +[[example]] +name = "uptime" +required-features = ["std"] + +[[example]] +name = "to_json" + +[[example]] +name = "rbpf_plugin" diff --git a/kernel/crates/rbpf/LICENSE-APACHE b/kernel/crates/rbpf/LICENSE-APACHE new file mode 100644 index 00000000..d6456956 --- /dev/null +++ b/kernel/crates/rbpf/LICENSE-APACHE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/kernel/crates/rbpf/LICENSE-MIT b/kernel/crates/rbpf/LICENSE-MIT new file mode 100644 index 00000000..661a705a --- /dev/null +++ b/kernel/crates/rbpf/LICENSE-MIT @@ -0,0 +1,25 @@ +Copyright (c) 2016 6WIND S.A. + +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. diff --git a/kernel/crates/rbpf/README.md b/kernel/crates/rbpf/README.md new file mode 100644 index 00000000..e2dc8ce7 --- /dev/null +++ b/kernel/crates/rbpf/README.md @@ -0,0 +1,743 @@ +# rbpf + + + + + + +Rust (user-space) virtual machine for eBPF + +[![Build Status](https://github.com/qmonnet/rbpf/actions/workflows/test.yaml/badge.svg)](https://github.com/qmonnet/rbpf/actions/workflows/test.yaml) +[![Build status](https://ci.appveyor.com/api/projects/status/ia74coeuhxtrcvsk/branch/main?svg=true)](https://ci.appveyor.com/project/qmonnet/rbpf/branch/main) +[![Coverage Status](https://coveralls.io/repos/github/qmonnet/rbpf/badge.svg?branch=main)](https://coveralls.io/github/qmonnet/rbpf?branch=main) +[![Crates.io](https://img.shields.io/crates/v/rbpf.svg)](https://crates.io/crates/rbpf) + +* [Description](#description) +* [Link to the crate](#link-to-the-crate) +* [API](#api) +* [Example uses](#example-uses) +* [Building eBPF programs](#building-ebpf-programs) +* [Build Features](#build-features) +* [Feedback welcome!](#feedback-welcome) +* [Questions / Answers](#questions--answers) +* [Caveats](#caveats) +* [_To do_ list](#to-do-list) +* [License](#license) +* [Inspired by](#inspired-by) +* [Other resources](#other-resources) + +## Description + +This crate contains a virtual machine for eBPF program execution. BPF, as in +_Berkeley Packet Filter_, is an assembly-like language initially developed for +BSD systems, in order to filter packets in the kernel with tools such as +tcpdump so as to avoid useless copies to user-space. It was ported to Linux, +where it evolved into eBPF (_extended_ BPF), a faster version with more +features. While BPF programs are originally intended to run in the kernel, the +virtual machine of this crate enables running it in user-space applications; +it contains an interpreter, an x86_64 JIT-compiler for eBPF programs, as well as +a disassembler. + +It is based on Rich Lane's [uBPF software](https://github.com/iovisor/ubpf/), +which does nearly the same, but is written in C. + +The crate is supposed to compile and run on Linux, MacOS X, and Windows, +although the JIT-compiler does not work with Windows at this time. + +## Link to the crate + +This crate is available from [crates.io](https://crates.io/crates/rbpf), so it +should work out of the box by adding it as a dependency in your `Cargo.toml` +file: + +```toml +[dependencies] +rbpf = "0.2.0" +``` + +You can also use the development version from this GitHub repository. This +should be as simple as putting this inside your `Cargo.toml`: + +```toml +[dependencies] +rbpf = { git = "https://github.com/qmonnet/rbpf" } +``` + +Of course, if you prefer, you can clone it locally, possibly hack the crate, +and then indicate the path of your local version in `Cargo.toml`: + +```toml +[dependencies] +rbpf = { path = "path/to/rbpf" } +``` + +Then indicate in your source code that you want to use the crate: + +```rust,ignore +extern crate rbpf; +``` + +## API + +The API is pretty well documented inside the source code. You should also be +able to access [an online version of the documentation from +here](https://docs.rs/rbpf/), automatically generated from the +[crates.io](https://crates.io/crates/rbpf) version (may not be up-to-date with +the main branch). [Examples](../../tree/main/examples) and [unit +tests](../../tree/main/tests) should also prove helpful. Here is a summary of +how to use the crate. + +Here are the steps to follow to run an eBPF program with rbpf: + +1. Create a virtual machine. There are several kinds of machines, we will come + back on this later. When creating the VM, pass the eBPF program as an + argument to the constructor. +2. If you want to use some helper functions, register them into the virtual + machine. +3. If you want a JIT-compiled program, compile it. +4. Execute your program: either run the interpreter or call the JIT-compiled + function. + +eBPF has been initially designed to filter packets (now it has some other hooks +in the Linux kernel, such as kprobes, but this is not covered by rbpf). As a +consequence, most of the load and store instructions of the program are +performed on a memory area representing the packet data. However, in the Linux +kernel, the eBPF program does not immediately access this data area: initially, +it has access to a C `struct sk_buff` instead, which is a buffer containing +metadata about the packet—including memory addresses of the beginning and of +the end of the packet data area. So the program first loads those pointers from +the `sk_buff`, and then can access the packet data. + +This behavior can be replicated with rbpf, but it is not mandatory. For this +reason, we have several structs representing different kinds of virtual +machines: + +* `struct EbpfVmMbuffer` mimics the kernel. When the program is run, the + address provided to its first eBPF register will be the address of a metadata + buffer provided by the user, and that is expected to contain pointers to the + start and the end of the packet data memory area. + +* `struct EbpfVmFixedMbuff` has one purpose: enabling the execution of programs + created to be compatible with the kernel, while saving the effort to manually + handle the metadata buffer for the user. In fact, this struct has a static + internal buffer that is passed to the program. The user has to indicate the + offset values at which the eBPF program expects to find the start and the end + of packet data in the buffer. On calling the function that runs the program + (JITted or not), the struct automatically updates the addresses in this + static buffer, at the appointed offsets, for the start and the end of the + packet data the program is called upon. + +* `struct EbpfVmRaw` is for programs that want to run directly on packet data. + No metadata buffer is involved, the eBPF program directly receives the + address of the packet data in its first register. This is the behavior of + uBPF. + +* `struct EbpfVmNoData` does not take any data. The eBPF program takes no + argument whatsoever and its return value is deterministic. Not so sure there + is a valid use case for that, but if nothing else, this is very useful for + unit tests. + +All these structs implement the same public functions: + +```rust,ignore +// called with EbpfVmMbuff:: prefix +pub fn new(prog: &'a [u8]) -> Result, Error> + +// called with EbpfVmFixedMbuff:: prefix +pub fn new(prog: &'a [u8], + data_offset: usize, + data_end_offset: usize) -> Result, Error> + +// called with EbpfVmRaw:: prefix +pub fn new(prog: &'a [u8]) -> Result, Error> + +// called with EbpfVmNoData:: prefix +pub fn new(prog: &'a [u8]) -> Result, Error> +``` + +This is used to create a new instance of a VM. The return type is dependent of +the struct from which the function is called. For instance, +`rbpf::EbpfVmRaw::new(Some(my_program))` would return an instance of `struct +rbpf::EbpfVmRaw` (wrapped in a `Result`). When a program is loaded, it is +checked with a very simple verifier (nothing close to the one for Linux +kernel). Users are also able to replace it with a custom verifier. + +For `struct EbpfVmFixedMbuff`, two additional arguments must be passed to the +constructor: `data_offset` and `data_end_offset`. They are the offset (byte +number) at which the pointers to the beginning and to the end, respectively, of +the memory area of packet data are to be stored in the internal metadata buffer +each time the program is executed. Other structs do not use this mechanism and +do not need those offsets. + +```rust,ignore +// for struct EbpfVmMbuff, struct EbpfVmRaw and struct EbpfVmRawData +pub fn set_program(&mut self, prog: &'a [u8]) -> Result<(), Error> + +// for struct EbpfVmFixedMbuff +pub fn set_program(&mut self, prog: &'a [u8], + data_offset: usize, + data_end_offset: usize) -> Result<(), Error> +``` + +You can use for example `my_vm.set_program(my_program);` to change the loaded +program after the VM instance creation. This program is checked with the +verifier attached to the VM. The verifying function of the VM can be changed at +any moment. + +```rust,ignore +pub type Verifier = fn(prog: &[u8]) -> Result<(), Error>; + +pub fn set_verifier(&mut self, + verifier: Verifier) -> Result<(), Error> +``` + +Note that if a program has already been loaded into the VM, setting a new +verifier also immediately runs it on the loaded program. However, the verifier +is not run if no program has been loaded (if `None` was passed to the `new()` +method when creating the VM). + +```rust,ignore +pub type Helper = fn (u64, u64, u64, u64, u64) -> u64; + +pub fn register_helper(&mut self, + key: u32, + function: Helper) -> Result<(), Error> +``` + +This function is used to register a helper function. The VM stores its +registers in a hashmap, so the key can be any `u32` value you want. It may be +useful for programs that should be compatible with the Linux kernel and +therefore must use specific helper numbers. + +```rust,ignore +// for struct EbpfVmMbuff +pub fn execute_program(&self, + mem: &'a mut [u8], + mbuff: &'a mut [u8]) -> Result<(u64), Error> + +// for struct EbpfVmFixedMbuff and struct EbpfVmRaw +pub fn execute_program(&self, + mem: &'a mut [u8]) -> Result<(u64), Error> + +// for struct EbpfVmNoData +pub fn execute_program(&self) -> Result<(u64), Error> +``` + +Interprets the loaded program. The function takes a reference to the packet +data and the metadata buffer, or only to the packet data, or nothing at all, +depending on the kind of the VM used. The value returned is the result of the +eBPF program. + +```rust,ignore +pub fn jit_compile(&mut self) -> Result<(), Error> +``` + +JIT-compile the loaded program, for x86_64 architecture. If the program is to +use helper functions, they must be registered into the VM before this function +is called. The generated assembly function is internally stored in the VM. + +```rust,ignore +// for struct EbpfVmMbuff +pub unsafe fn execute_program_jit(&self, mem: &'a mut [u8], + mbuff: &'a mut [u8]) -> Result<(u64), Error> + +// for struct EbpfVmFixedMbuff and struct EbpfVmRaw +pub unsafe fn execute_program_jit(&self, mem: &'a mut [u8]) -> Result<(u64), Error> + +// for struct EbpfVmNoData +pub unsafe fn execute_program_jit(&self) -> Result<(u64), Error> +``` + +Calls the JIT-compiled program. The arguments to provide are the same as for +`execute_program()`, again depending on the kind of VM that is used. The result of +the JIT-compiled program should be the same as with the interpreter, but it +should run faster. Note that if errors occur during the program execution, the +JIT-compiled version does not handle it as well as the interpreter, and the +program may crash. For this reason, the functions are marked as `unsafe`. + +## Example uses + +### Simple example + +This comes from the unit test `test_vm_add`. + +```rust +extern crate rbpf; + +fn main() { + + // This is the eBPF program, in the form of bytecode instructions. + let prog = &[ + 0xb4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov32 r0, 0 + 0xb4, 0x01, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, // mov32 r1, 2 + 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, // add32 r0, 1 + 0x0c, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // add32 r0, r1 + 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit + ]; + + // Instantiate a struct EbpfVmNoData. This is an eBPF VM for programs that + // takes no packet data in argument. + // The eBPF program is passed to the constructor. + let vm = rbpf::EbpfVmNoData::new(Some(prog)).unwrap(); + + // Execute (interpret) the program. No argument required for this VM. + assert_eq!(vm.execute_program().unwrap(), 0x3); +} +``` + +### With JIT, on packet data + +This comes from the unit test `test_jit_ldxh`. + +```rust +extern crate rbpf; + +fn main() { + let prog = &[ + 0x71, 0x10, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, // ldxh r0, [r1+2] + 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit + ]; + + // Let's use some data. + let mem = &mut [ + 0xaa, 0xbb, 0x11, 0xcc, 0xdd + ]; + + // This is an eBPF VM for programs reading from a given memory area (it + // directly reads from packet data) + let mut vm = rbpf::EbpfVmRaw::new(Some(prog)).unwrap(); + + #[cfg(any(windows, not(feature = "std")))] { + assert_eq!(vm.execute_program(mem).unwrap(), 0x11); + } + #[cfg(all(not(windows), feature = "std"))] { + // This time we JIT-compile the program. + vm.jit_compile().unwrap(); + + // Then we execute it. For this kind of VM, a reference to the packet + // data must be passed to the function that executes the program. + unsafe { assert_eq!(vm.execute_program_jit(mem).unwrap(), 0x11); } + } +} +``` +### Using a metadata buffer + +This comes from the unit test `test_jit_mbuff` and derives from the unit test +`test_jit_ldxh`. + +```rust +extern crate rbpf; + +fn main() { + let prog = &[ + // Load mem from mbuff at offset 8 into R1 + 0x79, 0x11, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, + // ldhx r1[2], r0 + 0x69, 0x10, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 + ]; + let mem = &mut [ + 0xaa, 0xbb, 0x11, 0x22, 0xcc, 0xdd + ]; + + // Just for the example we create our metadata buffer from scratch, and + // we store the pointers to packet data start and end in it. + let mut mbuff = &mut [0u8; 32]; + unsafe { + let mut data = mbuff.as_ptr().offset(8) as *mut u64; + let mut data_end = mbuff.as_ptr().offset(24) as *mut u64; + *data = mem.as_ptr() as u64; + *data_end = mem.as_ptr() as u64 + mem.len() as u64; + } + + // This eBPF VM is for program that use a metadata buffer. + let mut vm = rbpf::EbpfVmMbuff::new(Some(prog)).unwrap(); + + #[cfg(any(windows, not(feature = "std")))] { + assert_eq!(vm.execute_program(mem, mbuff).unwrap(), 0x2211); + } + #[cfg(all(not(windows), feature = "std"))] { + // Here again we JIT-compile the program. + vm.jit_compile().unwrap(); + + // Here we must provide both a reference to the packet data, and to the + // metadata buffer we use. + unsafe { + assert_eq!(vm.execute_program_jit(mem, mbuff).unwrap(), 0x2211); + } + } +} +``` + +### Loading code from an object file; and using a virtual metadata buffer + +This comes from unit test `test_vm_block_port`. + +This example requires the following additional crates, you may have to add them +to your `Cargo.toml` file. + +```toml +[dependencies] +rbpf = "0.2.0" +elf = "0.0.10" +``` + +It also uses a kind of VM that uses an internal buffer used to simulate the +`sk_buff` used by eBPF programs in the kernel, without having to manually +create a new buffer for each packet. It may be useful for programs compiled for +the kernel and that assumes the data they receive is a `sk_buff` pointing to +the packet data start and end addresses. So here we just provide the offsets at +which the eBPF program expects to find those pointers, and the VM handles the +buffer update so that we only have to provide a reference to the packet data +for each run of the program. + +```rust +extern crate elf; +use std::path::PathBuf; + +extern crate rbpf; +use rbpf::helpers; + +fn main() { + // Load a program from an ELF file, e.g. compiled from C to eBPF with + // clang/LLVM. Some minor modification to the bytecode may be required. + let filename = "examples/load_elf__block_a_port.elf"; + + let path = PathBuf::from(filename); + let file = match elf::File::open_path(&path) { + Ok(f) => f, + Err(e) => panic!("Error: {:?}", e), + }; + + // Here we assume the eBPF program is in the ELF section called + // ".classifier". + let text_scn = match file.get_section(".classifier") { + Some(s) => s, + None => panic!("Failed to look up .classifier section"), + }; + + let prog = &text_scn.data; + + // This is our data: a real packet, starting with Ethernet header + let packet = &mut [ + 0x01, 0x23, 0x45, 0x67, 0x89, 0xab, + 0xfe, 0xdc, 0xba, 0x98, 0x76, 0x54, + 0x08, 0x00, // ethertype + 0x45, 0x00, 0x00, 0x3b, // start ip_hdr + 0xa6, 0xab, 0x40, 0x00, + 0x40, 0x06, 0x96, 0x0f, + 0x7f, 0x00, 0x00, 0x01, + 0x7f, 0x00, 0x00, 0x01, + 0x99, 0x99, 0xc6, 0xcc, // start tcp_hdr + 0xd1, 0xe5, 0xc4, 0x9d, + 0xd4, 0x30, 0xb5, 0xd2, + 0x80, 0x18, 0x01, 0x56, + 0xfe, 0x2f, 0x00, 0x00, + 0x01, 0x01, 0x08, 0x0a, // start data + 0x00, 0x23, 0x75, 0x89, + 0x00, 0x23, 0x63, 0x2d, + 0x71, 0x64, 0x66, 0x73, + 0x64, 0x66, 0x0a + ]; + + // This is an eBPF VM for programs using a virtual metadata buffer, similar + // to the sk_buff that eBPF programs use with tc and in Linux kernel. + // We must provide the offsets at which the pointers to packet data start + // and end must be stored: these are the offsets at which the program will + // load the packet data from the metadata buffer. + let mut vm = rbpf::EbpfVmFixedMbuff::new(Some(prog), 0x40, 0x50).unwrap(); + + // We register a helper function, that can be called by the program, into + // the VM. The `bpf_trace_printf` is only available when we have access to + // the standard library. + #[cfg(feature = "std")] { + vm.register_helper(helpers::BPF_TRACE_PRINTK_IDX, + helpers::bpf_trace_printf).unwrap(); + } + + // This kind of VM takes a reference to the packet data, but does not need + // any reference to the metadata buffer: a fixed buffer is handled + // internally by the VM. + let res = vm.execute_program(packet).unwrap(); + println!("Program returned: {:?} ({:#x})", res, res); +} +``` + +## Building eBPF programs + +Besides passing the raw hexadecimal codes for building eBPF programs, two other +methods are available. + +### Assembler + +The first method consists in using the assembler provided by the crate. + +```rust +extern crate rbpf; +use rbpf::assembler::assemble; + +let prog = assemble("add64 r1, 0x605 + mov64 r2, 0x32 + mov64 r1, r0 + be16 r0 + neg64 r2 + exit").unwrap(); + +#[cfg(feature = "std")] { + println!("{:?}", prog); +} +``` + +The above snippet will produce: + +```rust,ignore +Ok([0x07, 0x01, 0x00, 0x00, 0x05, 0x06, 0x00, 0x00, + 0xb7, 0x02, 0x00, 0x00, 0x32, 0x00, 0x00, 0x00, + 0xbf, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xdc, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, + 0x87, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]) +``` + +Conversely, a disassembler is also available to dump instruction names from +bytecode in a human-friendly format. + +```rust +extern crate rbpf; +use rbpf::disassembler::disassemble; + +let prog = &[ + 0x07, 0x01, 0x00, 0x00, 0x05, 0x06, 0x00, 0x00, + 0xb7, 0x02, 0x00, 0x00, 0x32, 0x00, 0x00, 0x00, + 0xbf, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xdc, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, + 0x87, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +]; + +disassemble(prog); +``` + +This will produce the following output: + +```txt +add64 r1, 0x605 +mov64 r2, 0x32 +mov64 r1, r0 +be16 r0 +neg64 r2 +exit +``` + +Please refer to [source code](src/assembler.rs) and [tests](tests/assembler.rs) +for the syntax and the list of instruction names. + +### Building API + +The other way to build programs is to chain commands from the instruction +builder API. It looks less like assembly, maybe more like high-level functions. +What's sure is that the result is more verbose, but if you prefer to build +programs this way, it works just as well. If we take again the same sample as +above, it would be constructed as follows. + +```rust +extern crate rbpf; +use rbpf::insn_builder::*; + +let mut program = BpfCode::new(); +program.add(Source::Imm, Arch::X64).set_dst(1).set_imm(0x605).push() + .mov(Source::Imm, Arch::X64).set_dst(2).set_imm(0x32).push() + .mov(Source::Reg, Arch::X64).set_src(0).set_dst(1).push() + .swap_bytes(Endian::Big).set_dst(0).set_imm(0x10).push() + .negate(Arch::X64).set_dst(2).push() + .exit().push(); +``` + +Again, please refer to [the source and related tests](src/insn_builder.rs) to +get more information and examples on how to use it. + +## Build features + +### `no_std` + +The `rbpf` crate has a Cargo feature named "std" that is enabled by default. To +use `rbpf` in `no_std` environments this feature needs to be disabled. To do +this, you need to modify your dependency on `rbpf` in Cargo.toml to disable the +enabled-by-default features. + +```toml +[dependencies] +rbpf = { version = "1.0", default-features = false } +``` + +Note that when using this crate in `no_std` environments, the `jit` module +isn't available. This is because it depends on functions provided by `libc` +(`libc::posix_memalign()`, `libc::mprotect()`) which aren't available on +`no_std`. + +The `assembler` module is available, albeit with reduced debugging features. It +depends on the `combine` crate providing parser combinators. Under `no_std` +this crate only provides simple parsers which generate less descriptive error +messages. + +## Feedback welcome! + +This is the author's first try at writing Rust code. He learned a lot in the +process, but there remains a feeling that this crate has a kind of C-ish style +in some places instead of the Rusty look the author would like it to have. So +feedback (or PRs) are welcome, including about ways you might see to take +better advantage of Rust features. + +Note that the project expects new commits to be covered by the +[Developer's Certificate of Origin](https://wiki.linuxfoundation.org/dco). +When contributing Pull Requests, please sign off your commits accordingly. + +## Questions / Answers + +### Why implementing an eBPF virtual machine in Rust? + +As of this writing, there is no particular use case for this crate at the best +of the author's knowledge. The author happens to work with BPF on Linux and to +know how uBPF works, and he wanted to learn and experiment with Rust—no more +than that. + +### What are the differences with uBPF? + +Other than the language, obviously? Well, there are some differences: + +* Some constants, such as the maximum length for programs or the length for the + stack, differs between uBPF and rbpf. The latter uses the same values as the + Linux kernel, while uBPF has its own values. + +* When an error occurs while a program is run by uBPF, the function running the + program silently returns the maximum value as an error code, while rbpf + returns Rust type `Error`. + +* The registration of helper functions, that can be called from within an eBPF + program, is not handled in the same way. + +* The distinct structs permitting to run program either on packet data, or with + a metadata buffer (simulated or not) is a specificity of rbpf. + +* As for performance: theoretically the JITted programs are expected to run at + the same speed, while the C interpreter of uBPF should go slightly faster + than rbpf. But this has not been asserted yet. Benchmarking both programs + would be an interesting thing to do. + +### Can I use it with the “classic” BPF (a.k.a cBPF) version? + +No. This crate only works with extended BPF (eBPF) programs. For cBPF programs, +such as used by tcpdump (as of this writing) for example, you may be interested +in the [bpfjit crate](https://crates.io/crates/bpfjit) written by Alexander +Polakov instead. + +### What functionalities are implemented? + +Running and JIT-compiling eBPF programs work. There is also a mechanism to +register user-defined helper functions. The eBPF implementation of the Linux +kernel comes with [some additional +features](https://github.com/iovisor/bcc/blob/master/docs/kernel-versions.md): +a high number of helpers, several kinds of maps, tail calls. + +* Additional helpers should be easy to add, but very few of the existing Linux + helpers have been replicated in rbpf so far. + +* Tail calls (“long jumps” from an eBPF program into another) are not + implemented. This is probably not trivial to design and implement. + +* The interaction with maps is done through the use of specific helpers, so + this should not be difficult to add. The maps themselves can reuse the maps + in the kernel (if on Linux), to communicate with in-kernel eBPF programs for + instance; or they can be handled in user space. Rust has arrays and hashmaps, + so their implementation should be pretty straightforward (and may be added to + rbpf in the future). + +### What about program validation? + +The ”verifier” of this crate is very short and has nothing to do with the +kernel verifier, which means that it accepts programs that may not be safe. On +the other hand, you probably do not run this in a kernel here, so it will not +crash your system. Implementing a verifier similar to the one in the kernel is +not trivial, and we cannot “copy” it since it is under GPL license. + +### What about safety then? + +Rust has a strong emphasis on safety. Yet to have the eBPF VM work, some +`unsafe` blocks of code are used. The VM, taken as an eBPF interpreter, can +return an error but should not crash. Please file an issue otherwise. + +As for the JIT-compiler, it is a different story, since runtime memory checks +are more complicated to implement in assembly. It _will_ crash if your +JIT-compiled program tries to perform unauthorized memory accesses. Usually, it +could be a good idea to test your program with the interpreter first. + +Oh, and if your program has infinite loops, even with the interpreter, you're +on your own. + +## Caveats + +* This crate is **under development** and the API may be subject to change. + +* The JIT compiler produces an unsafe program: memory access are not tested at + runtime (yet). Use with caution. + +* A small number of eBPF instructions have not been implemented yet. This + should not be a problem for the majority of eBPF programs. + +* Beware of turnips. Turnips are disgusting. + +## _To do_ list + +* Implement some traits (`Clone`, `Drop`, `Debug` are good candidates). +* Provide built-in support for user-space array and hash BPF maps. +* Improve safety of JIT-compiled programs with runtime memory checks. +* Add helpers (some of those supported in the kernel, such as checksum update, + could be helpful). +* Improve verifier. Could we find a way to directly support programs compiled + with clang? +* Maybe one day, tail calls? +* JIT-compilers for other architectures? +* … + +## License + +Following the effort of the Rust language project itself in order to ease +integration with other projects, the rbpf crate is distributed under the terms +of both the MIT license and the Apache License (Version 2.0). + +See +[LICENSE-APACHE](https://github.com/qmonnet/rbpf/blob/main/LICENSE-APACHE) +and [LICENSE-MIT](https://github.com/qmonnet/rbpf/blob/main/LICENSE-MIT) for +details. + +## Version +[The last commit](https://github.com/qmonnet/rbpf/commit/fe7021b07b08a43b836743a77796d07ce1f4902e) + + +## Inspired by + +* [uBPF](https://github.com/iovisor/ubpf), a C user-space implementation of an + eBPF virtual machine, with a JIT-compiler and disassembler (and also + including the assembler from the human-readable form of the instructions, + such as in `mov r0, 0x1337`), by Rich Lane for Big Switch Networks (2015) + +* [_Building a simple JIT in + Rust_](https://www.sophiajt.com/building-a-simple-jit-in-rust), + by Sophia Turner (2015) + +* [bpfjit](https://github.com/polachok/bpfjit) (also [on + crates.io](https://crates.io/crates/bpfjit)), a Rust crate exporting the cBPF + JIT compiler from FreeBSD 10 tree to Rust, by Alexander Polakov (2016) + +## Other resources + +* Cilium project documentation about BPF: [_BPF and XDP Reference + Guide_](http://docs.cilium.io/en/latest/bpf/) + +* [Kernel documentation about BPF](https://docs.kernel.org/bpf/) + +* [_Dive into BPF: a list of reading + material_](https://qmonnet.github.io/whirl-offload/2016/09/01/dive-into-bpf), + a blog article listing documentation for BPF and related technologies (2016) + +* [The Rust programming language](https://www.rust-lang.org) diff --git a/kernel/crates/rbpf/clippy.toml b/kernel/crates/rbpf/clippy.toml new file mode 100644 index 00000000..1d4a2968 --- /dev/null +++ b/kernel/crates/rbpf/clippy.toml @@ -0,0 +1 @@ +doc-valid-idents = ["eBPF", "uBPF"] diff --git a/kernel/crates/rbpf/examples/disassemble.rs b/kernel/crates/rbpf/examples/disassemble.rs new file mode 100644 index 00000000..4dbcc277 --- /dev/null +++ b/kernel/crates/rbpf/examples/disassemble.rs @@ -0,0 +1,26 @@ +// SPDX-License-Identifier: (Apache-2.0 OR MIT) +// Copyright 2017 6WIND S.A. + +extern crate rbpf; +use rbpf::disassembler; + +// Simply disassemble a program into human-readable instructions. +fn main() { + let prog = &[ + 0xb7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x79, 0x12, 0x50, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x79, 0x11, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0xbf, 0x13, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x07, 0x03, 0x00, 0x00, 0x36, 0x00, 0x00, 0x00, 0x2d, 0x23, 0x12, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x69, 0x12, 0x0c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x55, 0x02, 0x10, 0x00, + 0x08, 0x00, 0x00, 0x00, 0x71, 0x12, 0x17, 0x00, 0x00, 0x00, 0x00, 0x00, 0x55, 0x02, 0x0e, + 0x00, 0x06, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x79, 0x11, 0x22, 0x00, 0x00, 0x00, 0x00, 0x00, 0xbf, + 0x12, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x57, 0x02, 0x00, 0x00, 0xff, 0xff, 0x00, 0x00, + 0x15, 0x02, 0x08, 0x00, 0x99, 0x99, 0x00, 0x00, 0x18, 0x02, 0x00, 0x00, 0x00, 0x00, 0xff, + 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5f, 0x21, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xb7, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x18, 0x02, 0x00, 0x00, 0x00, + 0x00, 0x99, 0x99, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1d, 0x21, 0x01, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xb7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x95, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, + ]; + disassembler::disassemble(prog); +} diff --git a/kernel/crates/rbpf/examples/helper.rs b/kernel/crates/rbpf/examples/helper.rs new file mode 100644 index 00000000..ace3dfef --- /dev/null +++ b/kernel/crates/rbpf/examples/helper.rs @@ -0,0 +1,3 @@ +fn main() { + rbpf::helpers::show_helper(); +} diff --git a/kernel/crates/rbpf/examples/load_elf.rs b/kernel/crates/rbpf/examples/load_elf.rs new file mode 100644 index 00000000..9d794140 --- /dev/null +++ b/kernel/crates/rbpf/examples/load_elf.rs @@ -0,0 +1,115 @@ +// SPDX-License-Identifier: (Apache-2.0 OR MIT) +// Copyright 2016 6WIND S.A. + +#![allow(clippy::unreadable_literal)] + +extern crate elf; +use std::path::PathBuf; + +extern crate rbpf; +use rbpf::helpers; + +// The following example uses an ELF file that has been compiled from the C program available in +// `load_elf__block_a_port.c` in the same directory. +// +// It was compiled with the following command: +// +// ```bash +// clang -O2 -emit-llvm -c load_elf__block_a_port.c -o - | \ +// llc -march=bpf -filetype=obj -o load_elf__block_a_port.o +// ``` +// +// Once compiled, this program can be injected into Linux kernel, with tc for instance. Sadly, we +// need to bring some modifications to the generated bytecode in order to run it: the three +// instructions with opcode 0x61 load data from a packet area as 4-byte words, where we need to +// load it as 8-bytes double words (0x79). The kernel does the same kind of translation before +// running the program, but rbpf does not implement this. +// +// In addition, the offset at which the pointer to the packet data is stored must be changed: since +// we use 8 bytes instead of 4 for the start and end addresses of the data packet, we cannot use +// the offsets produced by clang (0x4c and 0x50), the addresses would overlap. Instead we can use, +// for example, 0x40 and 0x50. +// +// These change were applied with the following script: +// +// ```bash +// xxd load_elf__block_a_port.o | sed ' +// s/6112 5000 0000 0000/7912 5000 0000 0000/ ; +// s/6111 4c00 0000 0000/7911 4000 0000 0000/ ; +// s/6111 2200 0000 0000/7911 2200 0000 0000/' | xxd -r > load_elf__block_a_port.tmp + +// mv load_elf__block_a_port.tmp load_elf__block_a_port.o +// ``` +// +// The eBPF program was placed into the `.classifier` ELF section (see C code above), which means +// that you can retrieve the raw bytecode with `readelf -x .classifier load_elf__block_a_port.o` or +// with `objdump -s -j .classifier load_elf__block_a_port.o`. +// +// Once the bytecode has been edited, we can load the bytecode directly from the ELF object file. + +fn main() { + let filename = "examples/load_elf__block_a_port.elf"; + + let path = PathBuf::from(filename); + let file = match elf::File::open_path(path) { + Ok(f) => f, + Err(e) => panic!("Error: {:?}", e), + }; + + let text_scn = match file.get_section(".classifier") { + Some(s) => s, + None => panic!("Failed to look up .classifier section"), + }; + + let prog = &text_scn.data; + + let packet1 = &mut [ + 0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xfe, 0xdc, 0xba, 0x98, 0x76, 0x54, 0x08, + 0x00, // ethertype + 0x45, 0x00, 0x00, 0x3b, // start ip_hdr + 0xa6, 0xab, 0x40, 0x00, 0x40, 0x06, 0x96, 0x0f, 0x7f, 0x00, 0x00, 0x01, 0x7f, 0x00, 0x00, + 0x01, + // Program matches the next two bytes: 0x9999 returns 0xffffffff, else return 0. + 0x99, 0x99, 0xc6, 0xcc, // start tcp_hdr + 0xd1, 0xe5, 0xc4, 0x9d, 0xd4, 0x30, 0xb5, 0xd2, 0x80, 0x18, 0x01, 0x56, 0xfe, 0x2f, 0x00, + 0x00, 0x01, 0x01, 0x08, 0x0a, // start data + 0x00, 0x23, 0x75, 0x89, 0x00, 0x23, 0x63, 0x2d, 0x71, 0x64, 0x66, 0x73, 0x64, 0x66, 0x0au8, + ]; + + let packet2 = &mut [ + 0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xfe, 0xdc, 0xba, 0x98, 0x76, 0x54, 0x08, + 0x00, // ethertype + 0x45, 0x00, 0x00, 0x3b, // start ip_hdr + 0xa6, 0xab, 0x40, 0x00, 0x40, 0x06, 0x96, 0x0f, 0x7f, 0x00, 0x00, 0x01, 0x7f, 0x00, 0x00, + 0x01, + // Program matches the next two bytes: 0x9999 returns 0xffffffff, else return 0. + 0x98, 0x76, 0xc6, 0xcc, // start tcp_hdr + 0xd1, 0xe5, 0xc4, 0x9d, 0xd4, 0x30, 0xb5, 0xd2, 0x80, 0x18, 0x01, 0x56, 0xfe, 0x2f, 0x00, + 0x00, 0x01, 0x01, 0x08, 0x0a, // start data + 0x00, 0x23, 0x75, 0x89, 0x00, 0x23, 0x63, 0x2d, 0x71, 0x64, 0x66, 0x73, 0x64, 0x66, 0x0au8, + ]; + + let mut vm = rbpf::EbpfVmFixedMbuff::new(Some(prog), 0x40, 0x50).unwrap(); + vm.register_helper(helpers::BPF_TRACE_PRINTK_IDX, helpers::bpf_trace_printf) + .unwrap(); + + let res = vm.execute_program(packet1).unwrap(); + println!("Packet #1, program returned: {res:?} ({res:#x})"); + assert_eq!(res, 0xffffffff); + + #[cfg(not(windows))] + { + vm.jit_compile().unwrap(); + + let res = unsafe { vm.execute_program_jit(packet2).unwrap() }; + println!("Packet #2, program returned: {res:?} ({res:#x})"); + assert_eq!(res, 0); + } + + #[cfg(windows)] + { + let res = vm.execute_program(packet2).unwrap(); + println!("Packet #2, program returned: {:?} ({:#x})", res, res); + assert_eq!(res, 0); + } +} diff --git a/kernel/crates/rbpf/examples/load_elf__block_a_port.c b/kernel/crates/rbpf/examples/load_elf__block_a_port.c new file mode 100644 index 00000000..0a51117e --- /dev/null +++ b/kernel/crates/rbpf/examples/load_elf__block_a_port.c @@ -0,0 +1,43 @@ +// SPDX-License-Identifier: (APACHE-2.0 OR MIT) +// Copyright 2016 6WIND S.A. + +// Block TCP packets on source or destination port 0x9999. + +#include +#include +#include +#include + +#define ETH_ALEN 6 +#define ETH_P_IP 0x0008 /* htons(0x0800) */ +#define TCP_HDR_LEN 20 + +#define BLOCKED_TCP_PORT 0x9999 + +struct eth_hdr { + unsigned char h_dest[ETH_ALEN]; + unsigned char h_source[ETH_ALEN]; + unsigned short h_proto; +}; + +#define SEC(NAME) __attribute__((section(NAME), used)) +SEC(".classifier") +int handle_ingress(struct __sk_buff *skb) +{ + void *data = (void *)(long)skb->data; + void *data_end = (void *)(long)skb->data_end; + struct eth_hdr *eth = data; + struct iphdr *iph = data + sizeof(*eth); + struct tcphdr *tcp = data + sizeof(*eth) + sizeof(*iph); + + /* single length check */ + if (data + sizeof(*eth) + sizeof(*iph) + sizeof(*tcp) > data_end) + return 0; + if (eth->h_proto != ETH_P_IP) + return 0; + if (iph->protocol != IPPROTO_TCP) + return 0; + if (tcp->source == BLOCKED_TCP_PORT || tcp->dest == BLOCKED_TCP_PORT) + return -1; + return 0; +} diff --git a/kernel/crates/rbpf/examples/rbpf_plugin.rs b/kernel/crates/rbpf/examples/rbpf_plugin.rs new file mode 100644 index 00000000..7a7ace6e --- /dev/null +++ b/kernel/crates/rbpf/examples/rbpf_plugin.rs @@ -0,0 +1,126 @@ +// Copyright Microsoft Corporation +// SPDX-License-Identifier: (Apache-2.0 OR MIT) + +// Path: examples/rbpf_plugin.rs +use std::io::Read; + +// Helper function used by https://github.com/Alan-Jowett/bpf_conformance/blob/main/tests/call_unwind_fail.data +fn _unwind(a: u64, _b: u64, _c: u64, _d: u64, _e: u64) -> u64 { + a +} + +// This is a plugin for the bpf_conformance test suite (https://github.com/Alan-Jowett/bpf_conformance) +// It accepts a single argument, the memory contents to pass to the VM. +// It reads the program from stdin. +fn main() { + let mut args: Vec = std::env::args().collect(); + #[allow(unused_mut)] // In no_std the jit variable isn't mutated. + let mut jit: bool = false; + let mut cranelift: bool = false; + let mut program_text = String::new(); + let mut memory_text = String::new(); + + args.remove(0); + + // Memory is always the first argument. + if !args.is_empty() { + memory_text.clone_from(&args[0]); + // Strip whitespace + memory_text.retain(|c| !c.is_whitespace()); + args.remove(0); + } + + // Process the rest of the arguments. + while !args.is_empty() { + match args[0].as_str() { + "--help" => { + println!("Usage: rbpf_plugin [memory] < program"); + return; + } + "--jit" => { + #[cfg(any(windows, not(feature = "std")))] + { + println!("JIT not supported"); + return; + } + #[cfg(all(not(windows), feature = "std"))] + { + jit = true; + } + } + "--cranelift" => { + cranelift = true; + + #[cfg(not(feature = "cranelift"))] + { + let _ = cranelift; + println!("Cranelift is not enabled"); + return; + } + } + "--program" => { + if args.len() < 2 { + println!("Missing argument to --program"); + return; + } + args.remove(0); + if !args.is_empty() { + program_text.clone_from(&args[0]); + args.remove(0); + } + } + _ => panic!("Unknown argument {}", args[0]), + } + args.remove(0); + } + + if program_text.is_empty() { + // Read program text from stdin + std::io::stdin().read_to_string(&mut program_text).unwrap(); + } + + // Strip whitespace + program_text.retain(|c| !c.is_whitespace()); + + // Convert program from hex to bytecode + let bytecode = hex::decode(program_text).unwrap(); + + // Convert memory from hex to bytes + let mut memory: Vec = hex::decode(memory_text).unwrap(); + + // Create rbpf vm + let mut vm = rbpf::EbpfVmRaw::new(Some(&bytecode)).unwrap(); + + // Register the helper function used by call_unwind_fail.data test. + vm.register_helper(5, _unwind).unwrap(); + + let result: u64; + if jit { + #[cfg(any(windows, not(feature = "std")))] + { + println!("JIT not supported"); + return; + } + #[cfg(all(not(windows), feature = "std"))] + { + unsafe { + vm.jit_compile().unwrap(); + result = vm.execute_program_jit(&mut memory).unwrap(); + } + } + } else if cranelift { + #[cfg(not(feature = "cranelift"))] + { + println!("Cranelift is not enabled"); + return; + } + #[cfg(feature = "cranelift")] + { + vm.cranelift_compile().unwrap(); + result = vm.execute_program_cranelift(&mut memory).unwrap(); + } + } else { + result = vm.execute_program(&mut memory).unwrap(); + } + println!("{result:x}"); +} diff --git a/kernel/crates/rbpf/examples/to_json.rs b/kernel/crates/rbpf/examples/to_json.rs new file mode 100644 index 00000000..6b86047d --- /dev/null +++ b/kernel/crates/rbpf/examples/to_json.rs @@ -0,0 +1,74 @@ +// SPDX-License-Identifier: (Apache-2.0 OR MIT) +// Copyright 2017 6WIND S.A. + +#[macro_use] +extern crate json; + +extern crate elf; +use std::path::PathBuf; + +extern crate rbpf; +use rbpf::disassembler; + +// Turn a program into a JSON string. +// +// Relies on `json` crate. +// +// You may copy this function and adapt it according to your needs. For instance, you may want to: +// +// * Remove the "desc" (description) attributes from the output. +// * Print integers as integers, and not as strings containing their hexadecimal representation +// (just replace the relevant `format!()` calls by the commented values. +fn to_json(prog: &[u8]) -> String { + // This call returns a high-level representation of the instructions, with the two parts of + // `LD_DW_IMM` instructions merged, and name and descriptions of the instructions. + // If you prefer to use a lower-level representation, use `ebpf::to_insn_vec()` function + // instead. + let insns = disassembler::to_insn_vec(prog); + let mut json_insns = vec![]; + for insn in insns { + json_insns.push(object!( + "opc" => format!("{:#x}", insn.opc), // => insn.opc, + "dst" => format!("{:#x}", insn.dst), // => insn.dst, + "src" => format!("{:#x}", insn.src), // => insn.src, + "off" => format!("{:#x}", insn.off), // => insn.off, + // Warning: for imm we use a i64 instead of a i32 (to have correct values for + // `lddw` operation. If we print a number in the JSON this is not a problem, the + // internal i64 has the same value with extended sign on 32 most significant bytes. + // If we print the hexadecimal value as a string however, we want to cast as a i32 + // to prevent all other instructions to print spurious `ffffffff` prefix if the + // number is negative. When values takes more than 32 bits with `lddw`, the cast + // has no effect and the complete value is printed anyway. + "imm" => format!("{:#x}", insn.imm as i32), // => insn.imm, + "desc" => insn.desc + )); + } + json::stringify_pretty( + object!( + "size" => json_insns.len(), + "insns" => json_insns + ), + 4, + ) +} + +// Load a program from an object file, and prints it to standard output as a JSON string. +fn main() { + // Let's reuse this file from `load_elf/example`. + let filename = "examples/load_elf__block_a_port.elf"; + + let path = PathBuf::from(filename); + let file = match elf::File::open_path(path) { + Ok(f) => f, + Err(e) => panic!("Error: {:?}", e), + }; + + let text_scn = match file.get_section(".classifier") { + Some(s) => s, + None => panic!("Failed to look up .classifier section"), + }; + + let prog = &text_scn.data; + + println!("{}", to_json(prog)); +} diff --git a/kernel/crates/rbpf/examples/uptime.rs b/kernel/crates/rbpf/examples/uptime.rs new file mode 100644 index 00000000..49b1642a --- /dev/null +++ b/kernel/crates/rbpf/examples/uptime.rs @@ -0,0 +1,78 @@ +// SPDX-License-Identifier: (Apache-2.0 OR MIT) +// Copyright 2017 6WIND S.A. + +extern crate rbpf; +use rbpf::helpers; + +// The main objectives of this example is to show: +// +// * the use of EbpfVmNoData function, +// * and the use of a helper. +// +// The two eBPF programs are independent and are not related to one another. +fn main() { + let prog1 = &[ + 0xb4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov32 r0, 0 + 0xb4, 0x01, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, // mov32 r1, 2 + 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, // add32 r0, 1 + 0x0c, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // add32 r0, r1 + 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // exit and return r0 + ]; + + // We use helper `bpf_time_getns()`, which is similar to helper `bpf_ktime_getns()` from Linux + // kernel. Hence rbpf::helpers module provides the index of this in-kernel helper as a + // constant, so that we can remain compatible with programs for the kernel. Here we also cast + // it to a u8 so as to use it directly in program instructions. + let hkey = helpers::BPF_KTIME_GETNS_IDX as u8; + let prog2 = &[ + 0xb7, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov64 r1, 0 + 0xb7, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov64 r1, 0 + 0xb7, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov64 r1, 0 + 0xb7, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov64 r1, 0 + 0xb7, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov64 r1, 0 + 0x85, 0x00, 0x00, 0x00, hkey, 0x00, 0x00, 0x00, // call helper + 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // exit and return r0 + ]; + + // Create a VM: this one takes no data. Load prog1 in it. + let mut vm = rbpf::EbpfVmNoData::new(Some(prog1)).unwrap(); + // Execute prog1. + assert_eq!(vm.execute_program().unwrap(), 0x3); + + // As struct EbpfVmNoData does not takes any memory area, its return value is mostly + // deterministic. So we know prog1 will always return 3. There is an exception: when it uses + // helpers, the latter may have non-deterministic values, and all calls may not return the same + // value. + // + // In the following example we use a helper to get the elapsed time since boot time: we + // reimplement uptime in eBPF, in Rust. Because why not. + + vm.set_program(prog2).unwrap(); + vm.register_helper(helpers::BPF_KTIME_GETNS_IDX, helpers::bpf_time_getns) + .unwrap(); + + let time; + + #[cfg(all(not(windows), feature = "std"))] + { + vm.jit_compile().unwrap(); + + time = unsafe { vm.execute_program_jit().unwrap() }; + } + + #[cfg(any(windows, not(feature = "std")))] + { + time = vm.execute_program().unwrap(); + } + + let days = time / 10u64.pow(9) / 60 / 60 / 24; + let hours = (time / 10u64.pow(9) / 60 / 60) % 24; + let minutes = (time / 10u64.pow(9) / 60) % 60; + let seconds = (time / 10u64.pow(9)) % 60; + let nanosec = time % 10u64.pow(9); + + println!( + "Uptime: {:#x} ns == {} days {:02}:{:02}:{:02}, {} ns", + time, days, hours, minutes, seconds, nanosec + ); +} diff --git a/kernel/crates/rbpf/mk/appveyor.bat b/kernel/crates/rbpf/mk/appveyor.bat new file mode 100644 index 00000000..06f61d39 --- /dev/null +++ b/kernel/crates/rbpf/mk/appveyor.bat @@ -0,0 +1,72 @@ +echo on +SetLocal EnableDelayedExpansion + +REM This is the recommended way to choose the toolchain version, according to +REM Appveyor's documentation. +SET PATH=C:\Program Files (x86)\MSBuild\%TOOLCHAIN_VERSION%\Bin;%PATH% + +set VCVARSALL="C:\Program Files (x86)\Microsoft Visual Studio %TOOLCHAIN_VERSION%\VC\vcvarsall.bat" + +if [%Platform%] NEQ [x64] goto win32 +set TARGET_ARCH=x86_64 +set TARGET_PROGRAM_FILES=%ProgramFiles% +call %VCVARSALL% amd64 +if %ERRORLEVEL% NEQ 0 exit 1 +goto download + +:win32 +echo on +if [%Platform%] NEQ [Win32] exit 1 +set TARGET_ARCH=i686 +set TARGET_PROGRAM_FILES=%ProgramFiles(x86)% +call %VCVARSALL% amd64_x86 +if %ERRORLEVEL% NEQ 0 exit 1 +goto download + +:download +REM vcvarsall turns echo off +echo on + +mkdir windows_build_tools +mkdir windows_build_tools\ +echo Downloading Yasm... +powershell -Command "(New-Object Net.WebClient).DownloadFile('http://www.tortall.net/projects/yasm/releases/yasm-1.3.0-win64.exe', 'windows_build_tools\yasm.exe')" +if %ERRORLEVEL% NEQ 0 ( + echo ...downloading Yasm failed. + exit 1 +) + +set RUST_URL=https://static.rust-lang.org/dist/rust-%RUST%-%TARGET_ARCH%-pc-windows-msvc.msi +echo Downloading %RUST_URL%... +mkdir build +powershell -Command "(New-Object Net.WebClient).DownloadFile('%RUST_URL%', 'build\rust-%RUST%-%TARGET_ARCH%-pc-windows-msvc.msi')" +if %ERRORLEVEL% NEQ 0 ( + echo ...downloading Rust failed. + exit 1 +) + +start /wait msiexec /i build\rust-%RUST%-%TARGET_ARCH%-pc-windows-msvc.msi INSTALLDIR="%TARGET_PROGRAM_FILES%\Rust %RUST%" /quiet /qn /norestart +if %ERRORLEVEL% NEQ 0 exit 1 + +set PATH="%TARGET_PROGRAM_FILES%\Rust %RUST%\bin";%cd%\windows_build_tools;%PATH% + +if [%Configuration%] == [Release] set CARGO_MODE=--release + +set + +link /? +cl /? +rustc --version +cargo --version + +cargo test --all-features -vv %CARGO_MODE% +if %ERRORLEVEL% NEQ 0 exit 1 + +REM Verify that `cargo build`, independent from `cargo test`, works; i.e. +REM verify that non-test builds aren't trying to use test-only features. +cargo build -vv %CARGO_MODE% +if %ERRORLEVEL% NEQ 0 exit 1 + +REM Verify that we can build with all features +cargo build --all-features -vv %CARGO_MODE% +if %ERRORLEVEL% NEQ 0 exit 1 diff --git a/kernel/crates/rbpf/rustfmt.toml b/kernel/crates/rbpf/rustfmt.toml new file mode 100644 index 00000000..8a0a3841 --- /dev/null +++ b/kernel/crates/rbpf/rustfmt.toml @@ -0,0 +1,3 @@ +group_imports="StdExternalCrate" +reorder_imports=true +imports_granularity="Crate" \ No newline at end of file diff --git a/kernel/crates/rbpf/src/asm_parser.rs b/kernel/crates/rbpf/src/asm_parser.rs new file mode 100644 index 00000000..41969218 --- /dev/null +++ b/kernel/crates/rbpf/src/asm_parser.rs @@ -0,0 +1,642 @@ +// SPDX-License-Identifier: (Apache-2.0 OR MIT) +// Copyright 2017 Rich Lane + +// Rust-doc comments were left in the module, but it is no longer publicly exposed from the root +// file of the crate. Do not expect to find those comments in the documentation of the crate. + +//! This module parses eBPF assembly language source code. + +use alloc::{ + string::{String, ToString}, + vec::Vec, +}; + +#[cfg(feature = "std")] +use combine::EasyParser; +use combine::{ + attempt, between, eof, many, many1, one_of, optional, + parser::char::{alpha_num, char, digit, hex_digit, spaces, string}, + sep_by, + stream::position::{self}, + ParseError, Parser, Stream, +}; + +/// Operand of an instruction. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Operand { + /// Register number. + Register(i64), + /// Jump offset or immediate. + Integer(i64), + /// Register number and offset. + Memory(i64, i64), + /// Used for pattern matching. + Nil, +} + +/// Parsed instruction. +#[derive(Debug, PartialEq, Eq)] +pub struct Instruction { + /// Instruction name. + pub name: String, + /// Operands. + pub operands: Vec, +} + +fn ident() -> impl Parser +where + I: Stream, + I::Error: ParseError, +{ + many1(alpha_num()) +} + +fn integer() -> impl Parser +where + I: Stream, + I::Error: ParseError, +{ + let sign = optional(one_of("-+".chars())).map(|x| match x { + Some('-') => -1, + _ => 1, + }); + let hex = string("0x") + .with(many1(hex_digit())) + .map(|x: String| u64::from_str_radix(&x, 16).unwrap() as i64); + let dec = many1(digit()).map(|x: String| x.parse::().unwrap()); + (sign, attempt(hex).or(dec)).map(|(s, x)| s * x) +} + +fn register() -> impl Parser +where + I: Stream, + I::Error: ParseError, +{ + char('r') + .with(many1(digit())) + .map(|x: String| x.parse::().unwrap()) +} + +fn operand() -> impl Parser +where + I: Stream, + I::Error: ParseError, +{ + let register_operand = register().map(Operand::Register); + let immediate = integer().map(Operand::Integer); + let memory = between(char('['), char(']'), (register(), optional(integer()))) + .map(|t| Operand::Memory(t.0, t.1.unwrap_or(0))); + register_operand.or(immediate).or(memory) +} + +fn instruction() -> impl Parser +where + I: Stream, + I::Error: ParseError, +{ + let operands = sep_by(operand(), char(',').skip(spaces())); + (ident().skip(spaces()), operands, spaces()).map(|t| Instruction { + name: t.0, + operands: t.1, + }) +} + +/// Parse a string into a list of instructions. +/// +/// The instructions are not validated and may have invalid names and operand types. +pub fn parse(input: &str) -> Result, String> { + let mut with = spaces().with(many(instruction()).skip(eof())); + + #[cfg(feature = "std")] + { + match with.easy_parse(position::Stream::new(input)) { + Ok((insts, _)) => Ok(insts), + Err(err) => Err(err.to_string()), + } + } + #[cfg(not(feature = "std"))] + { + match with.parse(position::Stream::new(input)) { + Ok((insts, _)) => Ok(insts), + Err(err) => Err(err.to_string()), + } + } +} + +#[cfg(test)] +mod tests { + use alloc::{string::ToString, vec}; + + use combine::Parser; + + use super::{ident, instruction, integer, operand, parse, register, Instruction, Operand}; + + // Unit tests for the different kinds of parsers. + + #[test] + fn test_ident() { + assert_eq!(ident().parse("nop"), Ok(("nop".to_string(), ""))); + assert_eq!(ident().parse("add32"), Ok(("add32".to_string(), ""))); + assert_eq!(ident().parse("add32*"), Ok(("add32".to_string(), "*"))); + } + + #[test] + fn test_integer() { + assert_eq!(integer().parse("0"), Ok((0, ""))); + assert_eq!(integer().parse("42"), Ok((42, ""))); + assert_eq!(integer().parse("+42"), Ok((42, ""))); + assert_eq!(integer().parse("-42"), Ok((-42, ""))); + assert_eq!(integer().parse("0x0"), Ok((0, ""))); + assert_eq!( + integer().parse("0x123456789abcdef0"), + Ok((0x123456789abcdef0, "")) + ); + assert_eq!(integer().parse("-0x1f"), Ok((-31, ""))); + } + + #[test] + fn test_register() { + assert_eq!(register().parse("r0"), Ok((0, ""))); + assert_eq!(register().parse("r15"), Ok((15, ""))); + } + + #[test] + fn test_operand() { + assert_eq!(operand().parse("r0"), Ok((Operand::Register(0), ""))); + assert_eq!(operand().parse("r15"), Ok((Operand::Register(15), ""))); + assert_eq!(operand().parse("0"), Ok((Operand::Integer(0), ""))); + assert_eq!(operand().parse("42"), Ok((Operand::Integer(42), ""))); + assert_eq!(operand().parse("[r1]"), Ok((Operand::Memory(1, 0), ""))); + assert_eq!(operand().parse("[r3+5]"), Ok((Operand::Memory(3, 5), ""))); + assert_eq!( + operand().parse("[r3+0x1f]"), + Ok((Operand::Memory(3, 31), "")) + ); + assert_eq!( + operand().parse("[r3-0x1f]"), + Ok((Operand::Memory(3, -31), "")) + ); + } + + #[test] + fn test_instruction() { + assert_eq!( + instruction().parse("exit"), + Ok(( + Instruction { + name: "exit".to_string(), + operands: vec![], + }, + "" + )) + ); + + assert_eq!( + instruction().parse("call 2"), + Ok(( + Instruction { + name: "call".to_string(), + operands: vec![Operand::Integer(2)], + }, + "" + )) + ); + + assert_eq!( + instruction().parse("addi r1, 2"), + Ok(( + Instruction { + name: "addi".to_string(), + operands: vec![Operand::Register(1), Operand::Integer(2)], + }, + "" + )) + ); + + assert_eq!( + instruction().parse("ldxb r2, [r1+12]"), + Ok(( + Instruction { + name: "ldxb".to_string(), + operands: vec![Operand::Register(2), Operand::Memory(1, 12)], + }, + "" + )) + ); + + assert_eq!( + instruction().parse("lsh r3, 0x8"), + Ok(( + Instruction { + name: "lsh".to_string(), + operands: vec![Operand::Register(3), Operand::Integer(8)], + }, + "" + )) + ); + + assert_eq!( + instruction().parse("jne r3, 0x8, +37"), + Ok(( + Instruction { + name: "jne".to_string(), + operands: vec![ + Operand::Register(3), + Operand::Integer(8), + Operand::Integer(37) + ], + }, + "" + )) + ); + + // Whitespace between operands is optional. + assert_eq!( + instruction().parse("jne r3,0x8,+37"), + Ok(( + Instruction { + name: "jne".to_string(), + operands: vec![ + Operand::Register(3), + Operand::Integer(8), + Operand::Integer(37) + ], + }, + "" + )) + ); + } + + // Other unit tests: try to parse various set of instructions. + + #[test] + fn test_empty() { + assert_eq!(parse(""), Ok(vec![])); + } + + #[test] + fn test_exit() { + // No operands. + assert_eq!( + parse("exit"), + Ok(vec![Instruction { + name: "exit".to_string(), + operands: vec![], + }]) + ); + } + + #[test] + fn test_lsh() { + // Register and immediate operands. + assert_eq!( + parse("lsh r3, 0x20"), + Ok(vec![Instruction { + name: "lsh".to_string(), + operands: vec![Operand::Register(3), Operand::Integer(0x20)], + }]) + ); + } + + #[test] + fn test_ja() { + // Jump offset operand. + assert_eq!( + parse("ja +1"), + Ok(vec![Instruction { + name: "ja".to_string(), + operands: vec![Operand::Integer(1)], + }]) + ); + } + + #[test] + fn test_ldxh() { + // Register and memory operands. + assert_eq!( + parse("ldxh r4, [r1+12]"), + Ok(vec![Instruction { + name: "ldxh".to_string(), + operands: vec![Operand::Register(4), Operand::Memory(1, 12)], + }]) + ); + } + + #[test] + fn test_tcp_sack() { + // Sample program from ubpf. + // We could technically indent the instructions since the parser support white spaces at + // the beginning, but there is another test for that. + let src = "\ +ldxb r2, [r1+12] +ldxb r3, [r1+13] +lsh r3, 0x8 +or r3, r2 +mov r0, 0x0 +jne r3, 0x8, +37 +ldxb r2, [r1+23] +jne r2, 0x6, +35 +ldxb r2, [r1+14] +add r1, 0xe +and r2, 0xf +lsh r2, 0x2 +add r1, r2 +mov r0, 0x0 +ldxh r4, [r1+12] +add r1, 0x14 +rsh r4, 0x2 +and r4, 0x3c +mov r2, r4 +add r2, 0xffffffec +mov r5, 0x15 +mov r3, 0x0 +jgt r5, r4, +20 +mov r5, r3 +lsh r5, 0x20 +arsh r5, 0x20 +mov r4, r1 +add r4, r5 +ldxb r5, [r4] +jeq r5, 0x1, +4 +jeq r5, 0x0, +12 +mov r6, r3 +jeq r5, 0x5, +9 +ja +2 +add r3, 0x1 +mov r6, r3 +ldxb r3, [r4+1] +add r3, r6 +lsh r3, 0x20 +arsh r3, 0x20 +jsgt r2, r3, -18 +ja +1 +mov r0, 0x1 +exit +"; + + assert_eq!( + parse(src), + Ok(vec![ + Instruction { + name: "ldxb".to_string(), + operands: vec![Operand::Register(2), Operand::Memory(1, 12)], + }, + Instruction { + name: "ldxb".to_string(), + operands: vec![Operand::Register(3), Operand::Memory(1, 13)], + }, + Instruction { + name: "lsh".to_string(), + operands: vec![Operand::Register(3), Operand::Integer(8)], + }, + Instruction { + name: "or".to_string(), + operands: vec![Operand::Register(3), Operand::Register(2)], + }, + Instruction { + name: "mov".to_string(), + operands: vec![Operand::Register(0), Operand::Integer(0)], + }, + Instruction { + name: "jne".to_string(), + operands: vec![ + Operand::Register(3), + Operand::Integer(8), + Operand::Integer(37) + ], + }, + Instruction { + name: "ldxb".to_string(), + operands: vec![Operand::Register(2), Operand::Memory(1, 23)], + }, + Instruction { + name: "jne".to_string(), + operands: vec![ + Operand::Register(2), + Operand::Integer(6), + Operand::Integer(35) + ], + }, + Instruction { + name: "ldxb".to_string(), + operands: vec![Operand::Register(2), Operand::Memory(1, 14)], + }, + Instruction { + name: "add".to_string(), + operands: vec![Operand::Register(1), Operand::Integer(14)], + }, + Instruction { + name: "and".to_string(), + operands: vec![Operand::Register(2), Operand::Integer(15)], + }, + Instruction { + name: "lsh".to_string(), + operands: vec![Operand::Register(2), Operand::Integer(2)], + }, + Instruction { + name: "add".to_string(), + operands: vec![Operand::Register(1), Operand::Register(2)], + }, + Instruction { + name: "mov".to_string(), + operands: vec![Operand::Register(0), Operand::Integer(0)], + }, + Instruction { + name: "ldxh".to_string(), + operands: vec![Operand::Register(4), Operand::Memory(1, 12)], + }, + Instruction { + name: "add".to_string(), + operands: vec![Operand::Register(1), Operand::Integer(20)], + }, + Instruction { + name: "rsh".to_string(), + operands: vec![Operand::Register(4), Operand::Integer(2)], + }, + Instruction { + name: "and".to_string(), + operands: vec![Operand::Register(4), Operand::Integer(60)], + }, + Instruction { + name: "mov".to_string(), + operands: vec![Operand::Register(2), Operand::Register(4)], + }, + Instruction { + name: "add".to_string(), + operands: vec![Operand::Register(2), Operand::Integer(4294967276)], + }, + Instruction { + name: "mov".to_string(), + operands: vec![Operand::Register(5), Operand::Integer(21)], + }, + Instruction { + name: "mov".to_string(), + operands: vec![Operand::Register(3), Operand::Integer(0)], + }, + Instruction { + name: "jgt".to_string(), + operands: vec![ + Operand::Register(5), + Operand::Register(4), + Operand::Integer(20) + ], + }, + Instruction { + name: "mov".to_string(), + operands: vec![Operand::Register(5), Operand::Register(3)], + }, + Instruction { + name: "lsh".to_string(), + operands: vec![Operand::Register(5), Operand::Integer(32)], + }, + Instruction { + name: "arsh".to_string(), + operands: vec![Operand::Register(5), Operand::Integer(32)], + }, + Instruction { + name: "mov".to_string(), + operands: vec![Operand::Register(4), Operand::Register(1)], + }, + Instruction { + name: "add".to_string(), + operands: vec![Operand::Register(4), Operand::Register(5)], + }, + Instruction { + name: "ldxb".to_string(), + operands: vec![Operand::Register(5), Operand::Memory(4, 0)], + }, + Instruction { + name: "jeq".to_string(), + operands: vec![ + Operand::Register(5), + Operand::Integer(1), + Operand::Integer(4) + ], + }, + Instruction { + name: "jeq".to_string(), + operands: vec![ + Operand::Register(5), + Operand::Integer(0), + Operand::Integer(12) + ], + }, + Instruction { + name: "mov".to_string(), + operands: vec![Operand::Register(6), Operand::Register(3)], + }, + Instruction { + name: "jeq".to_string(), + operands: vec![ + Operand::Register(5), + Operand::Integer(5), + Operand::Integer(9) + ], + }, + Instruction { + name: "ja".to_string(), + operands: vec![Operand::Integer(2)], + }, + Instruction { + name: "add".to_string(), + operands: vec![Operand::Register(3), Operand::Integer(1)], + }, + Instruction { + name: "mov".to_string(), + operands: vec![Operand::Register(6), Operand::Register(3)], + }, + Instruction { + name: "ldxb".to_string(), + operands: vec![Operand::Register(3), Operand::Memory(4, 1)], + }, + Instruction { + name: "add".to_string(), + operands: vec![Operand::Register(3), Operand::Register(6)], + }, + Instruction { + name: "lsh".to_string(), + operands: vec![Operand::Register(3), Operand::Integer(32)], + }, + Instruction { + name: "arsh".to_string(), + operands: vec![Operand::Register(3), Operand::Integer(32)], + }, + Instruction { + name: "jsgt".to_string(), + operands: vec![ + Operand::Register(2), + Operand::Register(3), + Operand::Integer(-18) + ], + }, + Instruction { + name: "ja".to_string(), + operands: vec![Operand::Integer(1)], + }, + Instruction { + name: "mov".to_string(), + operands: vec![Operand::Register(0), Operand::Integer(1)], + }, + Instruction { + name: "exit".to_string(), + operands: vec![], + } + ]) + ); + } + + /// When running without `std` the `EasyParser` provided by `combine` + /// cannot be used. Because of this we need to use the `Parser` and the + /// error messages are different. + #[test] + fn test_error_eof() { + let expected_error; + #[cfg(feature = "std")] + { + expected_error = Err( + "Parse error at line: 1, column: 6\nUnexpected end of input\nExpected digit\n" + .to_string(), + ); + } + #[cfg(not(feature = "std"))] + { + expected_error = Err("unexpected parse".to_string()); + } + // Unexpected end of input in a register name. + assert_eq!(parse("lsh r"), expected_error); + } + + /// When running without `std` the `EasyParser` provided by `combine` + /// cannot be used. Because of this we need to use the `Parser` and the + /// error messages are different. + #[test] + fn test_error_unexpected_character() { + let expected_error; + #[cfg(feature = "std")] + { + expected_error = Err( + "Parse error at line: 2, column: 1\nUnexpected `^`\nExpected letter or digit, whitespaces, `r`, `-`, `+`, `[` or end of input\n".to_string() + ); + } + #[cfg(not(feature = "std"))] + { + expected_error = Err("unexpected parse".to_string()); + } + // Unexpected character at end of input. + assert_eq!(parse("exit\n^"), expected_error); + } + + #[test] + fn test_initial_whitespace() { + assert_eq!( + parse( + " + exit" + ), + Ok(vec![Instruction { + name: "exit".to_string(), + operands: vec![], + }]) + ); + } +} diff --git a/kernel/crates/rbpf/src/assembler.rs b/kernel/crates/rbpf/src/assembler.rs new file mode 100644 index 00000000..bf452b1e --- /dev/null +++ b/kernel/crates/rbpf/src/assembler.rs @@ -0,0 +1,277 @@ +// SPDX-License-Identifier: (Apache-2.0 OR MIT) +// Copyright 2017 Rich Lane + +//! This module translates eBPF assembly language to binary. + +use alloc::{ + collections::BTreeMap, + format, + string::{String, ToString}, + vec, + vec::Vec, +}; + +use self::InstructionType::{ + AluBinary, AluUnary, Call, Endian, JumpConditional, JumpUnconditional, LoadAbs, LoadImm, + LoadInd, LoadReg, NoOperand, StoreImm, StoreReg, +}; +use crate::{ + asm_parser::{ + parse, Instruction, Operand, + Operand::{Integer, Memory, Nil, Register}, + }, + ebpf::{self, Insn}, +}; + +#[derive(Clone, Copy, Debug, PartialEq)] +enum InstructionType { + AluBinary, + AluUnary, + LoadImm, + LoadAbs, + LoadInd, + LoadReg, + StoreImm, + StoreReg, + JumpUnconditional, + JumpConditional, + Call, + Endian(i64), + NoOperand, +} + +fn make_instruction_map() -> BTreeMap { + let mut result = BTreeMap::new(); + + let alu_binary_ops = [ + ("add", ebpf::BPF_ADD), + ("sub", ebpf::BPF_SUB), + ("mul", ebpf::BPF_MUL), + ("div", ebpf::BPF_DIV), + ("or", ebpf::BPF_OR), + ("and", ebpf::BPF_AND), + ("lsh", ebpf::BPF_LSH), + ("rsh", ebpf::BPF_RSH), + ("mod", ebpf::BPF_MOD), + ("xor", ebpf::BPF_XOR), + ("mov", ebpf::BPF_MOV), + ("arsh", ebpf::BPF_ARSH), + ]; + + let mem_sizes = [ + ("w", ebpf::BPF_W), + ("h", ebpf::BPF_H), + ("b", ebpf::BPF_B), + ("dw", ebpf::BPF_DW), + ]; + + let jump_conditions = [ + ("jeq", ebpf::BPF_JEQ), + ("jgt", ebpf::BPF_JGT), + ("jge", ebpf::BPF_JGE), + ("jlt", ebpf::BPF_JLT), + ("jle", ebpf::BPF_JLE), + ("jset", ebpf::BPF_JSET), + ("jne", ebpf::BPF_JNE), + ("jsgt", ebpf::BPF_JSGT), + ("jsge", ebpf::BPF_JSGE), + ("jslt", ebpf::BPF_JSLT), + ("jsle", ebpf::BPF_JSLE), + ]; + + { + let mut entry = |name: &str, inst_type: InstructionType, opc: u8| { + result.insert(name.to_string(), (inst_type, opc)) + }; + + // Miscellaneous. + entry("exit", NoOperand, ebpf::EXIT); + entry("ja", JumpUnconditional, ebpf::JA); + entry("call", Call, ebpf::CALL); + entry("lddw", LoadImm, ebpf::LD_DW_IMM); + + // AluUnary. + entry("neg", AluUnary, ebpf::NEG64); + entry("neg32", AluUnary, ebpf::NEG32); + entry("neg64", AluUnary, ebpf::NEG64); + + // AluBinary. + for &(name, opc) in &alu_binary_ops { + entry(name, AluBinary, ebpf::BPF_ALU64 | opc); + entry(&format!("{name}32"), AluBinary, ebpf::BPF_ALU | opc); + entry(&format!("{name}64"), AluBinary, ebpf::BPF_ALU64 | opc); + } + + // LoadAbs, LoadInd, LoadReg, StoreImm, and StoreReg. + for &(suffix, size) in &mem_sizes { + entry( + &format!("ldabs{suffix}"), + LoadAbs, + ebpf::BPF_ABS | ebpf::BPF_LD | size, + ); + entry( + &format!("ldind{suffix}"), + LoadInd, + ebpf::BPF_IND | ebpf::BPF_LD | size, + ); + entry( + &format!("ldx{suffix}"), + LoadReg, + ebpf::BPF_MEM | ebpf::BPF_LDX | size, + ); + entry( + &format!("st{suffix}"), + StoreImm, + ebpf::BPF_MEM | ebpf::BPF_ST | size, + ); + entry( + &format!("stx{suffix}"), + StoreReg, + ebpf::BPF_MEM | ebpf::BPF_STX | size, + ); + } + + // JumpConditional. + for &(name, condition) in &jump_conditions { + entry(name, JumpConditional, ebpf::BPF_JMP | condition); + entry( + &format!("{name}32"), + JumpConditional, + ebpf::BPF_JMP32 | condition, + ); + } + + // Endian. + for &size in &[16, 32, 64] { + entry(&format!("be{size}"), Endian(size), ebpf::BE); + entry(&format!("le{size}"), Endian(size), ebpf::LE); + } + } + + result +} + +fn insn(opc: u8, dst: i64, src: i64, off: i64, imm: i64) -> Result { + if !(0..16).contains(&dst) { + return Err(format!("Invalid destination register {dst}")); + } + if dst < 0 || src >= 16 { + return Err(format!("Invalid source register {src}")); + } + if !(-32768..32768).contains(&off) { + return Err(format!("Invalid offset {off}")); + } + if !(-2147483648..2147483648).contains(&imm) { + return Err(format!("Invalid immediate {imm}")); + } + Ok(Insn { + opc, + dst: dst as u8, + src: src as u8, + off: off as i16, + imm: imm as i32, + }) +} + +// TODO Use slice patterns when available and remove this function. +fn operands_tuple(operands: &[Operand]) -> Result<(Operand, Operand, Operand), String> { + match operands.len() { + 0 => Ok((Nil, Nil, Nil)), + 1 => Ok((operands[0], Nil, Nil)), + 2 => Ok((operands[0], operands[1], Nil)), + 3 => Ok((operands[0], operands[1], operands[2])), + _ => Err("Too many operands".to_string()), + } +} + +fn encode(inst_type: InstructionType, opc: u8, operands: &[Operand]) -> Result { + let (a, b, c) = (operands_tuple(operands))?; + match (inst_type, a, b, c) { + (AluBinary, Register(dst), Register(src), Nil) => insn(opc | ebpf::BPF_X, dst, src, 0, 0), + (AluBinary, Register(dst), Integer(imm), Nil) => insn(opc | ebpf::BPF_K, dst, 0, 0, imm), + (AluUnary, Register(dst), Nil, Nil) => insn(opc, dst, 0, 0, 0), + (LoadAbs, Integer(imm), Nil, Nil) => insn(opc, 0, 0, 0, imm), + (LoadInd, Register(src), Integer(imm), Nil) => insn(opc, 0, src, 0, imm), + (LoadReg, Register(dst), Memory(src, off), Nil) + | (StoreReg, Memory(dst, off), Register(src), Nil) => insn(opc, dst, src, off, 0), + (StoreImm, Memory(dst, off), Integer(imm), Nil) => insn(opc, dst, 0, off, imm), + (NoOperand, Nil, Nil, Nil) => insn(opc, 0, 0, 0, 0), + (JumpUnconditional, Integer(off), Nil, Nil) => insn(opc, 0, 0, off, 0), + (JumpConditional, Register(dst), Register(src), Integer(off)) => { + insn(opc | ebpf::BPF_X, dst, src, off, 0) + } + (JumpConditional, Register(dst), Integer(imm), Integer(off)) => { + insn(opc | ebpf::BPF_K, dst, 0, off, imm) + } + (Call, Integer(imm), Nil, Nil) => insn(opc, 0, 0, 0, imm), + (Endian(size), Register(dst), Nil, Nil) => insn(opc, dst, 0, 0, size), + (LoadImm, Register(dst), Integer(imm), Nil) => insn(opc, dst, 0, 0, (imm << 32) >> 32), + _ => Err(format!("Unexpected operands: {operands:?}")), + } +} + +fn assemble_internal(parsed: &[Instruction]) -> Result, String> { + let instruction_map = make_instruction_map(); + let mut result: Vec = vec![]; + for instruction in parsed { + let name = instruction.name.as_str(); + match instruction_map.get(name) { + Some(&(inst_type, opc)) => { + match encode(inst_type, opc, &instruction.operands) { + Ok(insn) => result.push(insn), + Err(msg) => return Err(format!("Failed to encode {name}: {msg}")), + } + // Special case for lddw. + if let LoadImm = inst_type { + if let Integer(imm) = instruction.operands[1] { + result.push(insn(0, 0, 0, 0, imm >> 32).unwrap()); + } + } + } + None => return Err(format!("Invalid instruction {name:?}")), + } + } + Ok(result) +} + +/// Parse assembly source and translate to binary. +/// +/// # Examples +/// +/// ``` +/// use rbpf::assembler::assemble; +/// let prog = assemble("add64 r1, 0x605 +/// mov64 r2, 0x32 +/// mov64 r1, r0 +/// be16 r0 +/// neg64 r2 +/// exit"); +/// println!("{:?}", prog); +/// # assert_eq!(prog, +/// # Ok(vec![0x07, 0x01, 0x00, 0x00, 0x05, 0x06, 0x00, 0x00, +/// # 0xb7, 0x02, 0x00, 0x00, 0x32, 0x00, 0x00, 0x00, +/// # 0xbf, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +/// # 0xdc, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, +/// # 0x87, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +/// # 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00])); +/// ``` +/// +/// This will produce the following output: +/// +/// ```test +/// Ok([0x07, 0x01, 0x00, 0x00, 0x05, 0x06, 0x00, 0x00, +/// 0xb7, 0x02, 0x00, 0x00, 0x32, 0x00, 0x00, 0x00, +/// 0xbf, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +/// 0xdc, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, +/// 0x87, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +/// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]) +/// ``` +pub fn assemble(src: &str) -> Result, String> { + let parsed = (parse(src))?; + let insns = (assemble_internal(&parsed))?; + let mut result: Vec = vec![]; + for insn in insns { + result.extend_from_slice(&insn.to_array()); + } + Ok(result) +} diff --git a/kernel/crates/rbpf/src/cranelift.rs b/kernel/crates/rbpf/src/cranelift.rs new file mode 100644 index 00000000..10031e5b --- /dev/null +++ b/kernel/crates/rbpf/src/cranelift.rs @@ -0,0 +1,1230 @@ +// SPDX-License-Identifier: (Apache-2.0 OR MIT) + +use alloc::{collections::BTreeMap, format, vec, vec::Vec}; +use core::{mem, mem::ManuallyDrop}; +use std::io::ErrorKind; + +use cranelift_codegen::{ + entity::EntityRef, + ir::{ + condcodes::IntCC, + types::{I16, I32, I64, I8}, + AbiParam, Block, Endianness, FuncRef, Function, InstBuilder, MemFlags, Signature, + SourceLoc, StackSlotData, StackSlotKind, TrapCode, Type, UserFuncName, Value, + }, + isa::OwnedTargetIsa, + settings::{self, Configurable}, +}; +use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext, Variable}; +use cranelift_jit::{JITBuilder, JITModule}; +use cranelift_module::{FuncId, Linkage, Module}; + +use super::{Error, HashMap, HashSet}; +use crate::ebpf::{ + self, Insn, BPF_ALU_OP_MASK, BPF_IND, BPF_JEQ, BPF_JGE, BPF_JGT, BPF_JLE, BPF_JLT, BPF_JMP32, + BPF_JNE, BPF_JSET, BPF_JSGE, BPF_JSGT, BPF_JSLE, BPF_JSLT, BPF_X, STACK_SIZE, +}; + +pub type JittedFunction = extern "C" fn( + *mut u8, // mem_ptr + usize, // mem_len + *mut u8, // mbuff_ptr + usize, // mbuff_len +) -> u64; + +pub(crate) struct CraneliftCompiler { + isa: OwnedTargetIsa, + module: JITModule, + + helpers: HashMap, + helper_func_refs: HashMap, + + /// List of blocks corresponding to each instruction. + /// We only store the first instruction that observes a new block + insn_blocks: BTreeMap, + /// Map of block targets for each jump/branching instruction. + insn_targets: BTreeMap, + filled_blocks: HashSet, + + /// Map of register numbers to Cranelift variables. + registers: [Variable; 11], + /// Other usefull variables used throughout the program. + mem_start: Variable, + mem_end: Variable, + mbuf_start: Variable, + mbuf_end: Variable, + stack_start: Variable, + stack_end: Variable, +} + +impl CraneliftCompiler { + pub(crate) fn new(helpers: HashMap) -> Self { + let mut flag_builder = settings::builder(); + + flag_builder.set("opt_level", "speed").unwrap(); + + // Enable stack probes + flag_builder.enable("enable_probestack").unwrap(); + flag_builder.set("probestack_strategy", "inline").unwrap(); + + let isa_builder = cranelift_native::builder().unwrap_or_else(|msg| { + panic!("host machine is not supported: {}", msg); + }); + let isa = isa_builder + .finish(settings::Flags::new(flag_builder)) + .unwrap(); + + let mut jit_builder = + JITBuilder::with_isa(isa.clone(), cranelift_module::default_libcall_names()); + // Register all the helpers + for (k, v) in helpers.iter() { + let name = format!("helper_{}", k); + jit_builder.symbol(name, (*v) as usize as *const u8); + } + + let mut module = JITModule::new(jit_builder); + + let registers = (0..11) + .map(|i| Variable::new(i)) + .collect::>() + .try_into() + .unwrap(); + + Self { + isa, + module, + helpers, + helper_func_refs: HashMap::new(), + insn_blocks: BTreeMap::new(), + insn_targets: BTreeMap::new(), + filled_blocks: HashSet::new(), + registers, + mem_start: Variable::new(11), + mem_end: Variable::new(12), + mbuf_start: Variable::new(13), + mbuf_end: Variable::new(14), + stack_start: Variable::new(15), + stack_end: Variable::new(16), + } + } + + pub(crate) fn compile_function(mut self, prog: &[u8]) -> Result { + let name = "main"; + // This is not a standard eBPF function! We use an informal ABI with just 4 parameters. + // See [JittedFunction] which is the signature of this function. + // + // Since this function only serves as the entrypoint for the JITed program, it doesen't + // really matter. + let sig = Signature { + params: vec![ + AbiParam::new(I64), + AbiParam::new(I64), + AbiParam::new(I64), + AbiParam::new(I64), + ], + returns: vec![AbiParam::new(I64)], + call_conv: self.isa.default_call_conv(), + }; + + let func_id = self + .module + .declare_function(name, Linkage::Local, &sig) + .unwrap(); + + let mut ctx = self.module.make_context(); + ctx.func = Function::with_name_signature(UserFuncName::testcase(name.as_bytes()), sig); + let mut func_ctx = FunctionBuilderContext::new(); + + { + let mut builder: FunctionBuilder = FunctionBuilder::new(&mut ctx.func, &mut func_ctx); + + let entry = builder.create_block(); + builder.append_block_params_for_function_params(entry); + builder.switch_to_block(entry); + + self.build_cfg(&mut builder, prog)?; + self.build_function_prelude(&mut builder, entry)?; + self.translate_program(&mut builder, prog)?; + + builder.seal_all_blocks(); + builder.finalize(); + } + + self.module.define_function(func_id, &mut ctx).unwrap(); + self.module.finalize_definitions().unwrap(); + self.module.clear_context(&mut ctx); + + Ok(CraneliftProgram::new(self.module, func_id)) + } + + fn build_function_prelude( + &mut self, + bcx: &mut FunctionBuilder, + entry: Block, + ) -> Result<(), Error> { + // Register the VM registers as variables + for var in self.registers.iter() { + bcx.declare_var(*var, I64); + } + + // Register the bounds check variables + bcx.declare_var(self.mem_start, I64); + bcx.declare_var(self.mem_end, I64); + bcx.declare_var(self.mbuf_start, I64); + bcx.declare_var(self.mbuf_end, I64); + bcx.declare_var(self.stack_start, I64); + bcx.declare_var(self.stack_end, I64); + + // Register the helpers + for (k, _) in self.helpers.iter() { + let name = format!("helper_{}", k); + let sig = Signature { + params: vec![ + AbiParam::new(I64), + AbiParam::new(I64), + AbiParam::new(I64), + AbiParam::new(I64), + AbiParam::new(I64), + ], + returns: vec![AbiParam::new(I64)], + call_conv: self.isa.default_call_conv(), + }; + let func_id = self + .module + .declare_function(&name, Linkage::Import, &sig) + .unwrap(); + + let func_ref = self.module.declare_func_in_func(func_id, bcx.func); + self.helper_func_refs.insert(*k, func_ref); + } + + // Register the stack + let ss = bcx.create_sized_stack_slot(StackSlotData { + kind: StackSlotKind::ExplicitSlot, + size: STACK_SIZE as u32, + }); + let addr_ty = self.isa.pointer_type(); + let stack_addr = bcx.ins().stack_addr(addr_ty, ss, STACK_SIZE as i32); + bcx.def_var(self.registers[10], stack_addr); + + // Initialize the bounds check variables + let stack_start = bcx.ins().stack_addr(addr_ty, ss, 0); + bcx.def_var(self.stack_start, stack_start); + let stack_end = bcx.ins().stack_addr(addr_ty, ss, STACK_SIZE as i32); + bcx.def_var(self.stack_end, stack_end); + + // This is our internal ABI where the first 2 params are the memory + let mem_start = bcx.block_params(entry)[0]; + let mem_len = bcx.block_params(entry)[1]; + let mem_end = bcx.ins().iadd(mem_start, mem_len); + bcx.def_var(self.mem_start, mem_start); + bcx.def_var(self.mem_end, mem_end); + + // And the next 2 are the mbuf + let mbuf_start = bcx.block_params(entry)[2]; + let mbuf_len = bcx.block_params(entry)[3]; + let mbuf_end = bcx.ins().iadd(mbuf_start, mbuf_len); + bcx.def_var(self.mbuf_start, mbuf_start); + bcx.def_var(self.mbuf_end, mbuf_end); + + // The ABI for eBPF specifies that R1 must contain either the memory, or mbuff pointer + // If the mbuf length is non-zero, then we use that, otherwise we use the memory pointer + let mbuf_exists = bcx.ins().icmp_imm(IntCC::NotEqual, mbuf_len, 0); + let mem_or_mbuf = bcx.ins().select(mbuf_exists, mbuf_start, mem_start); + bcx.def_var(self.registers[1], mem_or_mbuf); + + // R2 should contain the length of the memory or mbuf + // At least ebpf-conformance tests expect this + let mem_or_mbuf_len = bcx.ins().select(mbuf_exists, mbuf_len, mem_len); + bcx.def_var(self.registers[2], mem_or_mbuf_len); + + // Insert the *actual* initial block + let program_entry = bcx.create_block(); + bcx.ins().jump(program_entry, &[]); + self.filled_blocks.insert(bcx.current_block().unwrap()); + self.insn_blocks.insert(0, program_entry); + + Ok(()) + } + + fn translate_program(&mut self, bcx: &mut FunctionBuilder, prog: &[u8]) -> Result<(), Error> { + let mut insn_ptr: usize = 0; + while insn_ptr * ebpf::INSN_SIZE < prog.len() { + let insn = ebpf::get_insn(prog, insn_ptr); + + // If this instruction is on a new block switch to it. + if let Some(block) = self.insn_blocks.get(&(insn_ptr as u32)) { + // Blocks must have a terminator instruction at the end before we switch away from them + let current_block = bcx.current_block().unwrap(); + if !self.filled_blocks.contains(¤t_block) { + bcx.ins().jump(*block, &[]); + } + + bcx.switch_to_block(*block); + } + + // Set the source location for the instruction + bcx.set_srcloc(SourceLoc::new(insn_ptr as u32)); + + match insn.opc { + // BPF_LD class + // LD_ABS_* and LD_IND_* are supposed to load pointer to data from metadata buffer. + // Since this pointer is constant, and since we already know it (mem), do not + // bother re-fetching it, just use mem already. + ebpf::LD_ABS_B + | ebpf::LD_ABS_H + | ebpf::LD_ABS_W + | ebpf::LD_ABS_DW + | ebpf::LD_IND_B + | ebpf::LD_IND_H + | ebpf::LD_IND_W + | ebpf::LD_IND_DW => { + let ty = match insn.opc { + ebpf::LD_ABS_B | ebpf::LD_IND_B => I8, + ebpf::LD_ABS_H | ebpf::LD_IND_H => I16, + ebpf::LD_ABS_W | ebpf::LD_IND_W => I32, + ebpf::LD_ABS_DW | ebpf::LD_IND_DW => I64, + _ => unreachable!(), + }; + + // Both instructions add the imm part of the instruction to the pointer + let ptr = bcx.use_var(self.mem_start); + let offset = bcx + .ins() + .iconst(self.isa.pointer_type(), insn.imm as u32 as i64); + let addr = bcx.ins().iadd(ptr, offset); + + // IND instructions additionally add the value of the source register + let is_ind = (insn.opc & BPF_IND) != 0; + let addr = if is_ind { + let src_reg = self.insn_src(bcx, &insn); + bcx.ins().iadd(addr, src_reg) + } else { + addr + }; + + // The offset here has already been added to the pointer, so we pass 0 + let loaded = self.reg_load(bcx, ty, addr, 0); + + let ext = if ty != I64 { + bcx.ins().uextend(I64, loaded) + } else { + loaded + }; + + self.set_dst(bcx, &insn, ext); + } + ebpf::LD_DW_IMM => { + insn_ptr += 1; + let next_insn = ebpf::get_insn(prog, insn_ptr); + + let imm = (((insn.imm as u32) as u64) + ((next_insn.imm as u64) << 32)) as i64; + let iconst = bcx.ins().iconst(I64, imm); + self.set_dst(bcx, &insn, iconst); + } + + // BPF_LDX class + ebpf::LD_B_REG | ebpf::LD_H_REG | ebpf::LD_W_REG | ebpf::LD_DW_REG => { + let ty = match insn.opc { + ebpf::LD_B_REG => I8, + ebpf::LD_H_REG => I16, + ebpf::LD_W_REG => I32, + ebpf::LD_DW_REG => I64, + _ => unreachable!(), + }; + + let base = self.insn_src(bcx, &insn); + let loaded = self.reg_load(bcx, ty, base, insn.off); + + let ext = if ty != I64 { + bcx.ins().uextend(I64, loaded) + } else { + loaded + }; + + self.set_dst(bcx, &insn, ext); + } + + // BPF_ST and BPF_STX class + ebpf::ST_B_IMM + | ebpf::ST_H_IMM + | ebpf::ST_W_IMM + | ebpf::ST_DW_IMM + | ebpf::ST_B_REG + | ebpf::ST_H_REG + | ebpf::ST_W_REG + | ebpf::ST_DW_REG => { + let ty = match insn.opc { + ebpf::ST_B_IMM | ebpf::ST_B_REG => I8, + ebpf::ST_H_IMM | ebpf::ST_H_REG => I16, + ebpf::ST_W_IMM | ebpf::ST_W_REG => I32, + ebpf::ST_DW_IMM | ebpf::ST_DW_REG => I64, + _ => unreachable!(), + }; + let is_imm = match insn.opc { + ebpf::ST_B_IMM | ebpf::ST_H_IMM | ebpf::ST_W_IMM | ebpf::ST_DW_IMM => true, + ebpf::ST_B_REG | ebpf::ST_H_REG | ebpf::ST_W_REG | ebpf::ST_DW_REG => false, + _ => unreachable!(), + }; + + let value = if is_imm { + self.insn_imm64(bcx, &insn) + } else { + self.insn_src(bcx, &insn) + }; + + let narrow = if ty != I64 { + bcx.ins().ireduce(ty, value) + } else { + value + }; + + let base = self.insn_dst(bcx, &insn); + self.reg_store(bcx, ty, base, insn.off, narrow); + } + + ebpf::ST_W_XADD => unimplemented!(), + ebpf::ST_DW_XADD => unimplemented!(), + + // BPF_ALU class + // TODO Check how overflow works in kernel. Should we &= U32MAX all src register value + // before we do the operation? + // Cf ((0x11 << 32) - (0x1 << 32)) as u32 VS ((0x11 << 32) as u32 - (0x1 << 32) as u32 + ebpf::ADD32_IMM => { + let src = self.insn_dst32(bcx, &insn); + let imm = self.insn_imm32(bcx, &insn); + let res = bcx.ins().iadd(src, imm); + self.set_dst32(bcx, &insn, res); + } + ebpf::ADD32_REG => { + //((reg[_dst] & U32MAX) + (reg[_src] & U32MAX)) & U32MAX, + let lhs = self.insn_dst32(bcx, &insn); + let rhs = self.insn_src32(bcx, &insn); + let res = bcx.ins().iadd(lhs, rhs); + self.set_dst32(bcx, &insn, res); + } + ebpf::SUB32_IMM => { + // reg[_dst] = (reg[_dst] as i32).wrapping_sub(insn.imm) as u64, + let src = self.insn_dst32(bcx, &insn); + let imm = self.insn_imm32(bcx, &insn); + let res = bcx.ins().isub(src, imm); + self.set_dst32(bcx, &insn, res); + } + ebpf::SUB32_REG => { + // reg[_dst] = (reg[_dst] as i32).wrapping_sub(reg[_src] as i32) as u64, + let lhs = self.insn_dst32(bcx, &insn); + let rhs = self.insn_src32(bcx, &insn); + let res = bcx.ins().isub(lhs, rhs); + self.set_dst32(bcx, &insn, res); + } + ebpf::MUL32_IMM => { + // reg[_dst] = (reg[_dst] as i32).wrapping_mul(insn.imm) as u64, + let src = self.insn_dst32(bcx, &insn); + let imm = self.insn_imm32(bcx, &insn); + let res = bcx.ins().imul(src, imm); + self.set_dst32(bcx, &insn, res); + } + ebpf::MUL32_REG => { + // reg[_dst] = (reg[_dst] as i32).wrapping_mul(reg[_src] as i32) as u64, + let lhs = self.insn_dst32(bcx, &insn); + let rhs = self.insn_src32(bcx, &insn); + let res = bcx.ins().imul(lhs, rhs); + self.set_dst32(bcx, &insn, res); + } + ebpf::DIV32_IMM => { + // reg[_dst] = (reg[_dst] as u32 / insn.imm as u32) as u64, + let res = if insn.imm == 0 { + bcx.ins().iconst(I32, 0) + } else { + let imm = self.insn_imm32(bcx, &insn); + let src = self.insn_dst32(bcx, &insn); + bcx.ins().udiv(src, imm) + }; + self.set_dst32(bcx, &insn, res); + } + ebpf::DIV32_REG => { + // reg[_dst] = (reg[_dst] as u32 / reg[_src] as u32) as u64, + let zero = bcx.ins().iconst(I32, 0); + let one = bcx.ins().iconst(I32, 1); + + let lhs = self.insn_dst32(bcx, &insn); + let rhs = self.insn_src32(bcx, &insn); + + let rhs_is_zero = bcx.ins().icmp(IntCC::Equal, rhs, zero); + let safe_rhs = bcx.ins().select(rhs_is_zero, one, rhs); + let div_res = bcx.ins().udiv(lhs, safe_rhs); + + let res = bcx.ins().select(rhs_is_zero, zero, div_res); + self.set_dst32(bcx, &insn, res); + } + ebpf::OR32_IMM => { + // reg[_dst] = (reg[_dst] as u32 | insn.imm as u32) as u64, + let src = self.insn_dst32(bcx, &insn); + let imm = self.insn_imm32(bcx, &insn); + let res = bcx.ins().bor(src, imm); + self.set_dst32(bcx, &insn, res); + } + ebpf::OR32_REG => { + // reg[_dst] = (reg[_dst] as u32 | reg[_src] as u32) as u64, + let lhs = self.insn_dst32(bcx, &insn); + let rhs = self.insn_src32(bcx, &insn); + let res = bcx.ins().bor(lhs, rhs); + self.set_dst32(bcx, &insn, res); + } + ebpf::AND32_IMM => { + // reg[_dst] = (reg[_dst] as u32 & insn.imm as u32) as u64, + let src = self.insn_dst32(bcx, &insn); + let imm = self.insn_imm32(bcx, &insn); + let res = bcx.ins().band(src, imm); + self.set_dst32(bcx, &insn, res); + } + ebpf::AND32_REG => { + // reg[_dst] = (reg[_dst] as u32 & reg[_src] as u32) as u64, + let lhs = self.insn_dst32(bcx, &insn); + let rhs = self.insn_src32(bcx, &insn); + let res = bcx.ins().band(lhs, rhs); + self.set_dst32(bcx, &insn, res); + } + ebpf::LSH32_IMM => { + // reg[_dst] = (reg[_dst] as u32).wrapping_shl(insn.imm as u32) as u64, + let src = self.insn_dst32(bcx, &insn); + let imm = self.insn_imm32(bcx, &insn); + let res = bcx.ins().ishl(src, imm); + self.set_dst32(bcx, &insn, res); + } + ebpf::LSH32_REG => { + // reg[_dst] = (reg[_dst] as u32).wrapping_shl(reg[_src] as u32) as u64, + let lhs = self.insn_dst32(bcx, &insn); + let rhs = self.insn_src32(bcx, &insn); + let res = bcx.ins().ishl(lhs, rhs); + self.set_dst32(bcx, &insn, res); + } + ebpf::RSH32_IMM => { + // reg[_dst] = (reg[_dst] as u32).wrapping_shr(insn.imm as u32) as u64, + let src = self.insn_dst32(bcx, &insn); + let imm = self.insn_imm32(bcx, &insn); + let res = bcx.ins().ushr(src, imm); + self.set_dst32(bcx, &insn, res); + } + ebpf::RSH32_REG => { + // reg[_dst] = (reg[_dst] as u32).wrapping_shr(reg[_src] as u32) as u64, + let lhs = self.insn_dst32(bcx, &insn); + let rhs = self.insn_src32(bcx, &insn); + let res = bcx.ins().ushr(lhs, rhs); + self.set_dst32(bcx, &insn, res); + } + ebpf::NEG32 => { + // { reg[_dst] = (reg[_dst] as i32).wrapping_neg() as u64; reg[_dst] &= U32MAX; }, + let src = self.insn_dst32(bcx, &insn); + let res = bcx.ins().ineg(src); + // TODO: Do we need to mask the result? + self.set_dst32(bcx, &insn, res); + } + ebpf::MOD32_IMM => { + // reg[_dst] = (reg[_dst] as u32 % insn.imm as u32) as u64, + + if insn.imm != 0 { + let imm = self.insn_imm32(bcx, &insn); + let src = self.insn_dst32(bcx, &insn); + let res = bcx.ins().urem(src, imm); + self.set_dst32(bcx, &insn, res); + } + } + ebpf::MOD32_REG => { + // reg[_dst] = (reg[_dst] as u32 % reg[_src] as u32) as u64, + let zero = bcx.ins().iconst(I32, 0); + let one = bcx.ins().iconst(I32, 1); + + let lhs = self.insn_dst32(bcx, &insn); + let rhs = self.insn_src32(bcx, &insn); + + let rhs_is_zero = bcx.ins().icmp(IntCC::Equal, rhs, zero); + let safe_rhs = bcx.ins().select(rhs_is_zero, one, rhs); + let div_res = bcx.ins().urem(lhs, safe_rhs); + + let res = bcx.ins().select(rhs_is_zero, lhs, div_res); + self.set_dst32(bcx, &insn, res); + } + ebpf::XOR32_IMM => { + // reg[_dst] = (reg[_dst] as u32 ^ insn.imm as u32) as u64, + let src = self.insn_dst32(bcx, &insn); + let imm = self.insn_imm32(bcx, &insn); + let res = bcx.ins().bxor(src, imm); + self.set_dst32(bcx, &insn, res); + } + ebpf::XOR32_REG => { + // reg[_dst] = (reg[_dst] as u32 ^ reg[_src] as u32) as u64, + let lhs = self.insn_dst32(bcx, &insn); + let rhs = self.insn_src32(bcx, &insn); + let res = bcx.ins().bxor(lhs, rhs); + self.set_dst32(bcx, &insn, res); + } + ebpf::MOV32_IMM => { + let imm = self.insn_imm32(bcx, &insn); + self.set_dst32(bcx, &insn, imm); + } + ebpf::MOV32_REG => { + // reg[_dst] = (reg[_src] as u32) as u64, + let src = self.insn_src32(bcx, &insn); + self.set_dst32(bcx, &insn, src); + } + ebpf::ARSH32_IMM => { + // { reg[_dst] = (reg[_dst] as i32).wrapping_shr(insn.imm as u32) as u64; reg[_dst] &= U32MAX; }, + let src = self.insn_dst32(bcx, &insn); + let imm = self.insn_imm32(bcx, &insn); + let res = bcx.ins().sshr(src, imm); + self.set_dst32(bcx, &insn, res); + } + ebpf::ARSH32_REG => { + // { reg[_dst] = (reg[_dst] as i32).wrapping_shr(reg[_src] as u32) as u64; reg[_dst] &= U32MAX; }, + let lhs = self.insn_dst32(bcx, &insn); + let rhs = self.insn_src32(bcx, &insn); + let res = bcx.ins().sshr(lhs, rhs); + self.set_dst32(bcx, &insn, res); + } + + ebpf::BE | ebpf::LE => { + let should_swap = match insn.opc { + ebpf::BE => self.isa.endianness() == Endianness::Little, + ebpf::LE => self.isa.endianness() == Endianness::Big, + _ => unreachable!(), + }; + + let ty: Type = match insn.imm { + 16 => I16, + 32 => I32, + 64 => I64, + _ => unreachable!(), + }; + + if should_swap { + let src = self.insn_dst(bcx, &insn); + let src_narrow = if ty != I64 { + bcx.ins().ireduce(ty, src) + } else { + src + }; + + let res = bcx.ins().bswap(src_narrow); + let res_wide = if ty != I64 { + bcx.ins().uextend(I64, res) + } else { + res + }; + + self.set_dst(bcx, &insn, res_wide); + } + } + + // BPF_ALU64 class + ebpf::ADD64_IMM => { + // reg[_dst] = reg[_dst].wrapping_add(insn.imm as u64), + let imm = self.insn_imm64(bcx, &insn); + let src = self.insn_dst(bcx, &insn); + let res = bcx.ins().iadd(src, imm); + self.set_dst(bcx, &insn, res); + } + ebpf::ADD64_REG => { + // reg[_dst] = reg[_dst].wrapping_add(reg[_src]), + let lhs = self.insn_dst(bcx, &insn); + let rhs = self.insn_src(bcx, &insn); + let res = bcx.ins().iadd(lhs, rhs); + self.set_dst(bcx, &insn, res); + } + ebpf::SUB64_IMM => { + // reg[_dst] = reg[_dst].wrapping_sub(insn.imm as u64), + let imm = self.insn_imm64(bcx, &insn); + let src = self.insn_dst(bcx, &insn); + let res = bcx.ins().isub(src, imm); + self.set_dst(bcx, &insn, res); + } + ebpf::SUB64_REG => { + // reg[_dst] = reg[_dst].wrapping_sub(reg[_src]), + let lhs = self.insn_dst(bcx, &insn); + let rhs = self.insn_src(bcx, &insn); + let res = bcx.ins().isub(lhs, rhs); + self.set_dst(bcx, &insn, res); + } + ebpf::MUL64_IMM => { + // reg[_dst] = reg[_dst].wrapping_mul(insn.imm as u64), + let imm = self.insn_imm64(bcx, &insn); + let src = self.insn_dst(bcx, &insn); + let res = bcx.ins().imul(src, imm); + self.set_dst(bcx, &insn, res); + } + ebpf::MUL64_REG => { + // reg[_dst] = reg[_dst].wrapping_mul(reg[_src]), + let lhs = self.insn_dst(bcx, &insn); + let rhs = self.insn_src(bcx, &insn); + let res = bcx.ins().imul(lhs, rhs); + self.set_dst(bcx, &insn, res); + } + ebpf::DIV64_IMM => { + // reg[_dst] /= insn.imm as u64, + let res = if insn.imm == 0 { + bcx.ins().iconst(I64, 0) + } else { + let imm = self.insn_imm64(bcx, &insn); + let src = self.insn_dst(bcx, &insn); + bcx.ins().udiv(src, imm) + }; + self.set_dst(bcx, &insn, res); + } + ebpf::DIV64_REG => { + // reg[_dst] /= reg[_src], if reg[_src] != 0 + // reg[_dst] = 0, if reg[_src] == 0 + let zero = bcx.ins().iconst(I64, 0); + let one = bcx.ins().iconst(I64, 1); + + let lhs = self.insn_dst(bcx, &insn); + let rhs = self.insn_src(bcx, &insn); + + let rhs_is_zero = bcx.ins().icmp(IntCC::Equal, rhs, zero); + let safe_rhs = bcx.ins().select(rhs_is_zero, one, rhs); + let div_res = bcx.ins().udiv(lhs, safe_rhs); + + let res = bcx.ins().select(rhs_is_zero, zero, div_res); + self.set_dst(bcx, &insn, res); + } + ebpf::MOD64_IMM => { + // reg[_dst] %= insn.imm as u64, + + if insn.imm != 0 { + let imm = self.insn_imm64(bcx, &insn); + let src = self.insn_dst(bcx, &insn); + let res = bcx.ins().urem(src, imm); + self.set_dst(bcx, &insn, res); + }; + } + ebpf::MOD64_REG => { + // reg[_dst] %= reg[_src], if reg[_src] != 0 + + let zero = bcx.ins().iconst(I64, 0); + let one = bcx.ins().iconst(I64, 1); + + let lhs = self.insn_dst(bcx, &insn); + let rhs = self.insn_src(bcx, &insn); + + let rhs_is_zero = bcx.ins().icmp(IntCC::Equal, rhs, zero); + let safe_rhs = bcx.ins().select(rhs_is_zero, one, rhs); + let div_res = bcx.ins().urem(lhs, safe_rhs); + + let res = bcx.ins().select(rhs_is_zero, lhs, div_res); + self.set_dst(bcx, &insn, res); + } + ebpf::OR64_IMM => { + // reg[_dst] |= insn.imm as u64, + let imm = self.insn_imm64(bcx, &insn); + let src = self.insn_dst(bcx, &insn); + let res = bcx.ins().bor(src, imm); + self.set_dst(bcx, &insn, res); + } + ebpf::OR64_REG => { + // reg[_dst] |= reg[_src], + let lhs = self.insn_dst(bcx, &insn); + let rhs = self.insn_src(bcx, &insn); + let res = bcx.ins().bor(lhs, rhs); + self.set_dst(bcx, &insn, res); + } + ebpf::AND64_IMM => { + // reg[_dst] &= insn.imm as u64, + let imm = self.insn_imm64(bcx, &insn); + let src = self.insn_dst(bcx, &insn); + let res = bcx.ins().band(src, imm); + self.set_dst(bcx, &insn, res); + } + ebpf::AND64_REG => { + // reg[_dst] &= reg[_src], + let lhs = self.insn_dst(bcx, &insn); + let rhs = self.insn_src(bcx, &insn); + let res = bcx.ins().band(lhs, rhs); + self.set_dst(bcx, &insn, res); + } + ebpf::LSH64_IMM => { + // reg[_dst] <<= insn.imm as u64, + let imm = self.insn_imm64(bcx, &insn); + let src = self.insn_dst(bcx, &insn); + let res = bcx.ins().ishl(src, imm); + self.set_dst(bcx, &insn, res); + } + ebpf::LSH64_REG => { + // reg[_dst] <<= reg[_src], + let lhs = self.insn_dst(bcx, &insn); + let rhs = self.insn_src(bcx, &insn); + let res = bcx.ins().ishl(lhs, rhs); + self.set_dst(bcx, &insn, res); + } + ebpf::RSH64_IMM => { + // reg[_dst] >>= insn.imm as u64, + let imm = self.insn_imm64(bcx, &insn); + let src = self.insn_dst(bcx, &insn); + let res = bcx.ins().ushr(src, imm); + self.set_dst(bcx, &insn, res); + } + ebpf::RSH64_REG => { + // reg[_dst] >>= reg[_src], + let lhs = self.insn_dst(bcx, &insn); + let rhs = self.insn_src(bcx, &insn); + let res = bcx.ins().ushr(lhs, rhs); + self.set_dst(bcx, &insn, res); + } + ebpf::NEG64 => { + // reg[_dst] = -(reg[_dst] as i64) as u64, + let src = self.insn_dst(bcx, &insn); + let res = bcx.ins().ineg(src); + self.set_dst(bcx, &insn, res); + } + ebpf::XOR64_IMM => { + // reg[_dst] ^= insn.imm as u64, + let imm = self.insn_imm64(bcx, &insn); + let src = self.insn_dst(bcx, &insn); + let res = bcx.ins().bxor(src, imm); + self.set_dst(bcx, &insn, res); + } + ebpf::XOR64_REG => { + // reg[_dst] ^= reg[_src], + let lhs = self.insn_dst(bcx, &insn); + let rhs = self.insn_src(bcx, &insn); + let res = bcx.ins().bxor(lhs, rhs); + self.set_dst(bcx, &insn, res); + } + ebpf::MOV64_IMM => { + // reg[_dst] = insn.imm as u64, + let imm = self.insn_imm64(bcx, &insn); + bcx.def_var(self.registers[insn.dst as usize], imm); + } + ebpf::MOV64_REG => { + // reg[_dst] = reg[_src], + let src = self.insn_src(bcx, &insn); + bcx.def_var(self.registers[insn.dst as usize], src); + } + ebpf::ARSH64_IMM => { + // reg[_dst] = (reg[_dst] as i64 >> insn.imm) as u64, + let imm = self.insn_imm64(bcx, &insn); + let src = self.insn_dst(bcx, &insn); + let res = bcx.ins().sshr(src, imm); + self.set_dst(bcx, &insn, res); + } + ebpf::ARSH64_REG => { + // reg[_dst] = (reg[_dst] as i64 >> reg[_src]) as u64, + let lhs = self.insn_dst(bcx, &insn); + let rhs = self.insn_src(bcx, &insn); + let res = bcx.ins().sshr(lhs, rhs); + self.set_dst(bcx, &insn, res); + } + + // BPF_JMP & BPF_JMP32 class + ebpf::JA => { + let (_, target_block) = self.insn_targets[&(insn_ptr as u32)]; + + bcx.ins().jump(target_block, &[]); + self.filled_blocks.insert(bcx.current_block().unwrap()); + } + ebpf::JEQ_IMM + | ebpf::JEQ_REG + | ebpf::JGT_IMM + | ebpf::JGT_REG + | ebpf::JGE_IMM + | ebpf::JGE_REG + | ebpf::JLT_IMM + | ebpf::JLT_REG + | ebpf::JLE_IMM + | ebpf::JLE_REG + | ebpf::JNE_IMM + | ebpf::JNE_REG + | ebpf::JSGT_IMM + | ebpf::JSGT_REG + | ebpf::JSGE_IMM + | ebpf::JSGE_REG + | ebpf::JSLT_IMM + | ebpf::JSLT_REG + | ebpf::JSLE_IMM + | ebpf::JSLE_REG + | ebpf::JSET_IMM + | ebpf::JSET_REG + | ebpf::JEQ_IMM32 + | ebpf::JEQ_REG32 + | ebpf::JGT_IMM32 + | ebpf::JGT_REG32 + | ebpf::JGE_IMM32 + | ebpf::JGE_REG32 + | ebpf::JLT_IMM32 + | ebpf::JLT_REG32 + | ebpf::JLE_IMM32 + | ebpf::JLE_REG32 + | ebpf::JNE_IMM32 + | ebpf::JNE_REG32 + | ebpf::JSGT_IMM32 + | ebpf::JSGT_REG32 + | ebpf::JSGE_IMM32 + | ebpf::JSGE_REG32 + | ebpf::JSLT_IMM32 + | ebpf::JSLT_REG32 + | ebpf::JSLE_IMM32 + | ebpf::JSLE_REG32 + | ebpf::JSET_IMM32 + | ebpf::JSET_REG32 => { + let (fallthrough, target) = self.insn_targets[&(insn_ptr as u32)]; + + let is_reg = (insn.opc & BPF_X) != 0; + let is_32 = (insn.opc & BPF_JMP32) != 0; + let intcc = match insn.opc { + c if (c & BPF_ALU_OP_MASK) == BPF_JEQ => IntCC::Equal, + c if (c & BPF_ALU_OP_MASK) == BPF_JNE => IntCC::NotEqual, + c if (c & BPF_ALU_OP_MASK) == BPF_JGT => IntCC::UnsignedGreaterThan, + c if (c & BPF_ALU_OP_MASK) == BPF_JGE => IntCC::UnsignedGreaterThanOrEqual, + c if (c & BPF_ALU_OP_MASK) == BPF_JLT => IntCC::UnsignedLessThan, + c if (c & BPF_ALU_OP_MASK) == BPF_JLE => IntCC::UnsignedLessThanOrEqual, + c if (c & BPF_ALU_OP_MASK) == BPF_JSGT => IntCC::SignedGreaterThan, + c if (c & BPF_ALU_OP_MASK) == BPF_JSGE => IntCC::SignedGreaterThanOrEqual, + c if (c & BPF_ALU_OP_MASK) == BPF_JSLT => IntCC::SignedLessThan, + c if (c & BPF_ALU_OP_MASK) == BPF_JSLE => IntCC::SignedLessThanOrEqual, + // JSET is handled specially below + c if (c & BPF_ALU_OP_MASK) == BPF_JSET => IntCC::NotEqual, + _ => unreachable!(), + }; + + let lhs = if is_32 { + self.insn_dst32(bcx, &insn) + } else { + self.insn_dst(bcx, &insn) + }; + let rhs = match (is_reg, is_32) { + (true, false) => self.insn_src(bcx, &insn), + (true, true) => self.insn_src32(bcx, &insn), + (false, false) => self.insn_imm64(bcx, &insn), + (false, true) => self.insn_imm32(bcx, &insn), + }; + + let cmp_res = if (insn.opc & BPF_ALU_OP_MASK) == BPF_JSET { + bcx.ins().band(lhs, rhs) + } else { + bcx.ins().icmp(intcc, lhs, rhs) + }; + bcx.ins().brif(cmp_res, target, &[], fallthrough, &[]); + self.filled_blocks.insert(bcx.current_block().unwrap()); + } + + // Do not delegate the check to the verifier, since registered functions can be + // changed after the program has been verified. + ebpf::CALL => { + let func_ref = self + .helper_func_refs + .get(&(insn.imm as u32)) + .copied() + .ok_or_else(|| { + Error::new( + ErrorKind::Other, + format!( + "[CRANELIFT] Error: unknown helper function (id: {:#x})", + insn.imm as u32 + ), + ) + })?; + + let arg0 = bcx.use_var(self.registers[1]); + let arg1 = bcx.use_var(self.registers[2]); + let arg2 = bcx.use_var(self.registers[3]); + let arg3 = bcx.use_var(self.registers[4]); + let arg4 = bcx.use_var(self.registers[5]); + + let call = bcx.ins().call(func_ref, &[arg0, arg1, arg2, arg3, arg4]); + let ret = bcx.inst_results(call)[0]; + self.set_dst(bcx, &insn, ret); + } + ebpf::TAIL_CALL => unimplemented!(), + ebpf::EXIT => { + let ret = bcx.use_var(self.registers[0]); + bcx.ins().return_(&[ret]); + self.filled_blocks.insert(bcx.current_block().unwrap()); + } + _ => unimplemented!("inst: {:?}", insn), + } + + insn_ptr += 1; + } + + Ok(()) + } + + fn insn_imm64(&mut self, bcx: &mut FunctionBuilder, insn: &Insn) -> Value { + bcx.ins().iconst(I64, insn.imm as u64 as i64) + } + fn insn_imm32(&mut self, bcx: &mut FunctionBuilder, insn: &Insn) -> Value { + bcx.ins().iconst(I32, insn.imm as u32 as u64 as i64) + } + + fn insn_dst(&mut self, bcx: &mut FunctionBuilder, insn: &Insn) -> Value { + bcx.use_var(self.registers[insn.dst as usize]) + } + fn insn_dst32(&mut self, bcx: &mut FunctionBuilder, insn: &Insn) -> Value { + let dst = self.insn_dst(bcx, insn); + bcx.ins().ireduce(I32, dst) + } + + fn insn_src(&mut self, bcx: &mut FunctionBuilder, insn: &Insn) -> Value { + bcx.use_var(self.registers[insn.src as usize]) + } + fn insn_src32(&mut self, bcx: &mut FunctionBuilder, insn: &Insn) -> Value { + let src = self.insn_src(bcx, insn); + bcx.ins().ireduce(I32, src) + } + + fn set_dst(&mut self, bcx: &mut FunctionBuilder, insn: &Insn, val: Value) { + bcx.def_var(self.registers[insn.dst as usize], val); + } + fn set_dst32(&mut self, bcx: &mut FunctionBuilder, insn: &Insn, val: Value) { + let val32 = bcx.ins().uextend(I64, val); + self.set_dst(bcx, insn, val32); + } + + fn reg_load(&mut self, bcx: &mut FunctionBuilder, ty: Type, base: Value, offset: i16) -> Value { + self.insert_bounds_check(bcx, ty, base, offset); + + let mut flags = MemFlags::new(); + flags.set_endianness(Endianness::Little); + + bcx.ins().load(ty, flags, base, offset as i32) + } + fn reg_store( + &mut self, + bcx: &mut FunctionBuilder, + ty: Type, + base: Value, + offset: i16, + val: Value, + ) { + self.insert_bounds_check(bcx, ty, base, offset); + + let mut flags = MemFlags::new(); + flags.set_endianness(Endianness::Little); + + bcx.ins().store(flags, val, base, offset as i32); + } + + /// Inserts a bounds check for a memory access + /// + /// This emits a conditional trap if the access is out of bounds for any of the known + /// valid memory regions. These are the stack, the memory, and the mbuf. + fn insert_bounds_check( + &mut self, + bcx: &mut FunctionBuilder, + ty: Type, + base: Value, + offset: i16, + ) { + let access_size = bcx.ins().iconst(I64, ty.bytes() as i64); + + let offset = bcx.ins().iconst(I64, offset as i64); + let start_addr = bcx.ins().iadd(base, offset); + let end_addr = bcx.ins().iadd(start_addr, access_size); + + let does_not_overflow = + bcx.ins() + .icmp(IntCC::UnsignedGreaterThanOrEqual, end_addr, start_addr); + + // Check if it's a valid stack access + let stack_start = bcx.use_var(self.stack_start); + let stack_end = bcx.use_var(self.stack_end); + let stack_start_valid = + bcx.ins() + .icmp(IntCC::UnsignedGreaterThanOrEqual, start_addr, stack_start); + let stack_end_valid = bcx + .ins() + .icmp(IntCC::UnsignedLessThanOrEqual, end_addr, stack_end); + let stack_valid = bcx.ins().band(stack_start_valid, stack_end_valid); + + // Check if it's a valid memory access + let mem_start = bcx.use_var(self.mem_start); + let mem_end = bcx.use_var(self.mem_end); + let has_mem = bcx.ins().icmp_imm(IntCC::NotEqual, mem_start, 0); + let mem_start_valid = + bcx.ins() + .icmp(IntCC::UnsignedGreaterThanOrEqual, start_addr, mem_start); + let mem_end_valid = bcx + .ins() + .icmp(IntCC::UnsignedLessThanOrEqual, end_addr, mem_end); + + let mem_valid = bcx.ins().band(mem_start_valid, mem_end_valid); + let mem_valid = bcx.ins().band(mem_valid, has_mem); + + // Check if it's a valid mbuf access + let mbuf_start = bcx.use_var(self.mbuf_start); + let mbuf_end = bcx.use_var(self.mbuf_end); + let has_mbuf = bcx.ins().icmp_imm(IntCC::NotEqual, mbuf_start, 0); + let mbuf_start_valid = + bcx.ins() + .icmp(IntCC::UnsignedGreaterThanOrEqual, start_addr, mbuf_start); + let mbuf_end_valid = bcx + .ins() + .icmp(IntCC::UnsignedLessThanOrEqual, end_addr, mbuf_end); + let mbuf_valid = bcx.ins().band(mbuf_start_valid, mbuf_end_valid); + let mbuf_valid = bcx.ins().band(mbuf_valid, has_mbuf); + + // Join all of these checks together and trap if any of them fails + + // We need it to be valid to at least one region of memory + let valid_region = bcx.ins().bor(stack_valid, mem_valid); + let valid_region = bcx.ins().bor(valid_region, mbuf_valid); + + // And that it does not overflow + let valid = bcx.ins().band(does_not_overflow, valid_region); + + // TODO: We can potentially throw a custom trap code here to indicate + // which check failed. + bcx.ins().trapz(valid, TrapCode::HeapOutOfBounds); + } + + /// Analyze the program and build the CFG + /// + /// We do this because cranelift does not allow us to switch back to a previously + /// filled block and add instructions to it. So we can't split the program as we + /// translate it. + fn build_cfg(&mut self, bcx: &mut FunctionBuilder, prog: &[u8]) -> Result<(), Error> { + let mut insn_ptr: usize = 0; + while insn_ptr * ebpf::INSN_SIZE < prog.len() { + let insn = ebpf::get_insn(prog, insn_ptr); + + match insn.opc { + // This instruction consumes two opcodes + ebpf::LD_DW_IMM => { + insn_ptr += 1; + } + + ebpf::JA + | ebpf::JEQ_IMM + | ebpf::JEQ_REG + | ebpf::JGT_IMM + | ebpf::JGT_REG + | ebpf::JGE_IMM + | ebpf::JGE_REG + | ebpf::JLT_IMM + | ebpf::JLT_REG + | ebpf::JLE_IMM + | ebpf::JLE_REG + | ebpf::JNE_IMM + | ebpf::JNE_REG + | ebpf::JSGT_IMM + | ebpf::JSGT_REG + | ebpf::JSGE_IMM + | ebpf::JSGE_REG + | ebpf::JSLT_IMM + | ebpf::JSLT_REG + | ebpf::JSLE_IMM + | ebpf::JSLE_REG + | ebpf::JSET_IMM + | ebpf::JSET_REG + | ebpf::JEQ_IMM32 + | ebpf::JEQ_REG32 + | ebpf::JGT_IMM32 + | ebpf::JGT_REG32 + | ebpf::JGE_IMM32 + | ebpf::JGE_REG32 + | ebpf::JLT_IMM32 + | ebpf::JLT_REG32 + | ebpf::JLE_IMM32 + | ebpf::JLE_REG32 + | ebpf::JNE_IMM32 + | ebpf::JNE_REG32 + | ebpf::JSGT_IMM32 + | ebpf::JSGT_REG32 + | ebpf::JSGE_IMM32 + | ebpf::JSGE_REG32 + | ebpf::JSLT_IMM32 + | ebpf::JSLT_REG32 + | ebpf::JSLE_IMM32 + | ebpf::JSLE_REG32 + | ebpf::JSET_IMM32 + | ebpf::JSET_REG32 + | ebpf::EXIT + | ebpf::TAIL_CALL => { + self.prepare_jump_blocks(bcx, insn_ptr, &insn); + } + _ => {} + } + + insn_ptr += 1; + } + + Ok(()) + } + + fn prepare_jump_blocks(&mut self, bcx: &mut FunctionBuilder, insn_ptr: usize, insn: &Insn) { + let insn_ptr = insn_ptr as u32; + let next_pc: u32 = insn_ptr + 1; + let target_pc: u32 = (insn_ptr as isize + insn.off as isize + 1) + .try_into() + .unwrap(); + + // This is the fallthrough block + let fallthrough_block = *self + .insn_blocks + .entry(next_pc) + .or_insert_with(|| bcx.create_block()); + + // Jump Target + let target_block = *self + .insn_blocks + .entry(target_pc) + .or_insert_with(|| bcx.create_block()); + + // Mark the blocks for this instruction + self.insn_targets + .insert(insn_ptr, (fallthrough_block, target_block)); + } +} + +/// Contains the backing memory for a previously compiled function. +/// +/// Currently this will allways just contain code for a single function, but +/// in the future we might want to support multiple functions per module. +/// +/// Ensures that the backing memory is freed when dropped. +pub struct CraneliftProgram { + module: ManuallyDrop, + + main_id: FuncId, +} + +impl CraneliftProgram { + pub(crate) fn new(module: JITModule, main_id: FuncId) -> Self { + Self { + module: ManuallyDrop::new(module), + main_id, + } + } + + /// We shouldn't allow this function pointer to be exposed outside of this + /// module, since it's not guaranteed to be valid after the module is dropped. + pub(crate) fn get_main_function(&self) -> JittedFunction { + let function_ptr = self.module.get_finalized_function(self.main_id); + unsafe { mem::transmute(function_ptr) } + } + + /// Execute this module by calling the main function + pub fn execute( + &self, + mem_ptr: *mut u8, + mem_len: usize, + mbuff_ptr: *mut u8, + mbuff_len: usize, + ) -> u64 { + let main = self.get_main_function(); + + main(mem_ptr, mem_len, mbuff_ptr, mbuff_len) + } +} + +impl Drop for CraneliftProgram { + fn drop(&mut self) { + // We need to have an owned version of `JITModule` to be able to free + // it's memory. Use `ManuallyDrop` to get the owned `JITModule`. + // + // We can no longer use `module` after this, but since we are `Drop` + // it should be safe. + unsafe { + let module = ManuallyDrop::take(&mut self.module); + module.free_memory() + }; + } +} diff --git a/kernel/crates/rbpf/src/disassembler.rs b/kernel/crates/rbpf/src/disassembler.rs new file mode 100644 index 00000000..6d6572c5 --- /dev/null +++ b/kernel/crates/rbpf/src/disassembler.rs @@ -0,0 +1,807 @@ +// SPDX-License-Identifier: (Apache-2.0 OR MIT) +// Copyright 2017 6WIND S.A. + +//! Functions in this module are used to handle eBPF programs with a higher level representation, +//! for example to disassemble the code into a human-readable format. + +use alloc::{ + format, + string::{String, ToString}, + vec, + vec::Vec, +}; + +use log::warn; + +use crate::ebpf; + +#[inline] +fn alu_imm_str(name: &str, insn: &ebpf::Insn) -> String { + format!("{name} r{}, {:#x}", insn.dst, insn.imm) +} + +#[inline] +fn alu_reg_str(name: &str, insn: &ebpf::Insn) -> String { + format!("{name} r{}, r{}", insn.dst, insn.src) +} + +#[inline] +fn byteswap_str(name: &str, insn: &ebpf::Insn) -> String { + match insn.imm { + 16 | 32 | 64 => {} + _ => warn!("[Disassembler] Warning: Invalid offset value for {name} insn"), + } + format!("{name}{} r{}", insn.imm, insn.dst) +} + +#[inline] +fn ld_st_imm_str(name: &str, insn: &ebpf::Insn) -> String { + if insn.off >= 0 { + format!("{name} [r{}+{:#x}], {:#x}", insn.dst, insn.off, insn.imm) + } else { + format!( + "{name} [r{}-{:#x}], {:#x}", + insn.dst, + -(insn.off as isize), + insn.imm + ) + } +} + +#[inline] +fn ld_reg_str(name: &str, insn: &ebpf::Insn) -> String { + if insn.off >= 0 { + format!("{name} r{}, [r{}+{:#x}]", insn.dst, insn.src, insn.off) + } else { + format!( + "{name} r{}, [r{}-{:#x}]", + insn.dst, + insn.src, + -(insn.off as isize) + ) + } +} + +#[inline] +fn st_reg_str(name: &str, insn: &ebpf::Insn) -> String { + if insn.off >= 0 { + format!("{name} [r{}+{:#x}], r{}", insn.dst, insn.off, insn.src) + } else { + format!( + "{name} [r{}-{:#x}], r{}", + insn.dst, + -(insn.off as isize), + insn.src + ) + } +} + +#[inline] +fn ldabs_str(name: &str, insn: &ebpf::Insn) -> String { + format!("{name} {:#x}", insn.imm) +} + +#[inline] +fn ldind_str(name: &str, insn: &ebpf::Insn) -> String { + format!("{name} r{}, {:#x}", insn.src, insn.imm) +} + +#[inline] +fn jmp_imm_str(name: &str, insn: &ebpf::Insn) -> String { + if insn.off >= 0 { + format!("{name} r{}, {:#x}, +{:#x}", insn.dst, insn.imm, insn.off) + } else { + format!( + "{name} r{}, {:#x}, -{:#x}", + insn.dst, + insn.imm, + -(insn.off as isize) + ) + } +} + +#[inline] +fn jmp_reg_str(name: &str, insn: &ebpf::Insn) -> String { + if insn.off >= 0 { + format!("{name} r{}, r{}, +{:#x}", insn.dst, insn.src, insn.off) + } else { + format!( + "{name} r{}, r{}, -{:#x}", + insn.dst, + insn.src, + -(insn.off as isize) + ) + } +} + +/// High-level representation of an eBPF instruction. +/// +/// In addition to standard operation code and various operand, this struct has the following +/// properties: +/// +/// * It stores a name, corresponding to a mnemonic for the operation code. +/// * It also stores a description, which is a mnemonic for the full instruction, using the actual +/// values of the relevant operands, and that can be used for disassembling the eBPF program for +/// example. +/// * Immediate values are stored in an `i64` instead of a traditional i32, in order to merge the +/// two parts of (otherwise double-length) `LD_DW_IMM` instructions. +/// +/// See for the Linux kernel +/// documentation about eBPF, or for a +/// more concise version. +#[derive(Debug, PartialEq, Eq)] +pub struct HLInsn { + /// Operation code. + pub opc: u8, + /// Name (mnemonic). This name is not canon. + pub name: String, + /// Description of the instruction. This is not canon. + pub desc: String, + /// Destination register operand. + pub dst: u8, + /// Source register operand. + pub src: u8, + /// Offset operand. + pub off: i16, + /// Immediate value operand. For `LD_DW_IMM` instructions, contains the whole value merged from + /// the two 8-bytes parts of the instruction. + pub imm: i64, +} + +/// Return a vector of `struct HLInsn` built from an eBPF program. +/// +/// This is made public to provide a way to manipulate a program as a vector of instructions, in a +/// high-level format, for example for dumping the program instruction after instruction with a +/// custom format. +/// +/// Note that the two parts of `LD_DW_IMM` instructions (that have the size of two standard +/// instructions) are considered as making a single immediate value. As a consequence, the number +/// of instructions stored in the vector may not be equal to the size in bytes of the program +/// divided by the length of an instructions. +/// +/// To do so, the immediate value operand is stored as an `i64` instead as an i32, so be careful +/// when you use it (see example `examples/to_json.rs`). +/// +/// This is to oppose to `ebpf::to_insn_vec()` function, that treats instructions on a low-level +/// ground and do not merge the parts of `LD_DW_IMM`. Also, the version in `ebpf` module does not +/// use names or descriptions when storing the instructions. +/// +/// # Examples +/// +/// ``` +/// use rbpf::disassembler; +/// +/// let prog = &[ +/// 0x18, 0x00, 0x00, 0x00, 0x88, 0x77, 0x66, 0x55, +/// 0x00, 0x00, 0x00, 0x00, 0x44, 0x33, 0x22, 0x11, +/// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +/// ]; +/// +/// let v = disassembler::to_insn_vec(prog); +/// assert_eq!(v, vec![ +/// disassembler::HLInsn { +/// opc: 0x18, +/// name: "lddw".to_string(), +/// desc: "lddw r0, 0x1122334455667788".to_string(), +/// dst: 0, +/// src: 0, +/// off: 0, +/// imm: 0x1122334455667788 +/// }, +/// disassembler::HLInsn { +/// opc: 0x95, +/// name: "exit".to_string(), +/// desc: "exit".to_string(), +/// dst: 0, +/// src: 0, +/// off: 0, +/// imm: 0 +/// }, +/// ]); +/// ``` +pub fn to_insn_vec(prog: &[u8]) -> Vec { + if prog.len() % ebpf::INSN_SIZE != 0 { + panic!( + "[Disassembler] Error: eBPF program length must be a multiple of {:?} octets", + ebpf::INSN_SIZE + ); + } + if prog.is_empty() { + return vec![]; + } + + let mut res = vec![]; + let mut insn_ptr: usize = 0; + + while insn_ptr * ebpf::INSN_SIZE < prog.len() { + let insn = ebpf::get_insn(prog, insn_ptr); + + let name; + let desc; + let mut imm = insn.imm as i64; + match insn.opc { + // BPF_LD class + ebpf::LD_ABS_B => { + name = "ldabsb"; + desc = ldabs_str(name, &insn); + } + ebpf::LD_ABS_H => { + name = "ldabsh"; + desc = ldabs_str(name, &insn); + } + ebpf::LD_ABS_W => { + name = "ldabsw"; + desc = ldabs_str(name, &insn); + } + ebpf::LD_ABS_DW => { + name = "ldabsdw"; + desc = ldabs_str(name, &insn); + } + ebpf::LD_IND_B => { + name = "ldindb"; + desc = ldind_str(name, &insn); + } + ebpf::LD_IND_H => { + name = "ldindh"; + desc = ldind_str(name, &insn); + } + ebpf::LD_IND_W => { + name = "ldindw"; + desc = ldind_str(name, &insn); + } + ebpf::LD_IND_DW => { + name = "ldinddw"; + desc = ldind_str(name, &insn); + } + + ebpf::LD_DW_IMM => { + insn_ptr += 1; + let next_insn = ebpf::get_insn(prog, insn_ptr); + imm = ((insn.imm as u32) as u64 + ((next_insn.imm as u64) << 32)) as i64; + name = "lddw"; + desc = format!("{name} r{:}, {imm:#x}", insn.dst); + } + + // BPF_LDX class + ebpf::LD_B_REG => { + name = "ldxb"; + desc = ld_reg_str(name, &insn); + } + ebpf::LD_H_REG => { + name = "ldxh"; + desc = ld_reg_str(name, &insn); + } + ebpf::LD_W_REG => { + name = "ldxw"; + desc = ld_reg_str(name, &insn); + } + ebpf::LD_DW_REG => { + name = "ldxdw"; + desc = ld_reg_str(name, &insn); + } + + // BPF_ST class + ebpf::ST_B_IMM => { + name = "stb"; + desc = ld_st_imm_str(name, &insn); + } + ebpf::ST_H_IMM => { + name = "sth"; + desc = ld_st_imm_str(name, &insn); + } + ebpf::ST_W_IMM => { + name = "stw"; + desc = ld_st_imm_str(name, &insn); + } + ebpf::ST_DW_IMM => { + name = "stdw"; + desc = ld_st_imm_str(name, &insn); + } + + // BPF_STX class + ebpf::ST_B_REG => { + name = "stxb"; + desc = st_reg_str(name, &insn); + } + ebpf::ST_H_REG => { + name = "stxh"; + desc = st_reg_str(name, &insn); + } + ebpf::ST_W_REG => { + name = "stxw"; + desc = st_reg_str(name, &insn); + } + ebpf::ST_DW_REG => { + name = "stxdw"; + desc = st_reg_str(name, &insn); + } + ebpf::ST_W_XADD => { + name = "stxxaddw"; + desc = st_reg_str(name, &insn); + } + ebpf::ST_DW_XADD => { + name = "stxxadddw"; + desc = st_reg_str(name, &insn); + } + + // BPF_ALU class + ebpf::ADD32_IMM => { + name = "add32"; + desc = alu_imm_str(name, &insn); + } + ebpf::ADD32_REG => { + name = "add32"; + desc = alu_reg_str(name, &insn); + } + ebpf::SUB32_IMM => { + name = "sub32"; + desc = alu_imm_str(name, &insn); + } + ebpf::SUB32_REG => { + name = "sub32"; + desc = alu_reg_str(name, &insn); + } + ebpf::MUL32_IMM => { + name = "mul32"; + desc = alu_imm_str(name, &insn); + } + ebpf::MUL32_REG => { + name = "mul32"; + desc = alu_reg_str(name, &insn); + } + ebpf::DIV32_IMM => { + name = "div32"; + desc = alu_imm_str(name, &insn); + } + ebpf::DIV32_REG => { + name = "div32"; + desc = alu_reg_str(name, &insn); + } + ebpf::OR32_IMM => { + name = "or32"; + desc = alu_imm_str(name, &insn); + } + ebpf::OR32_REG => { + name = "or32"; + desc = alu_reg_str(name, &insn); + } + ebpf::AND32_IMM => { + name = "and32"; + desc = alu_imm_str(name, &insn); + } + ebpf::AND32_REG => { + name = "and32"; + desc = alu_reg_str(name, &insn); + } + ebpf::LSH32_IMM => { + name = "lsh32"; + desc = alu_imm_str(name, &insn); + } + ebpf::LSH32_REG => { + name = "lsh32"; + desc = alu_reg_str(name, &insn); + } + ebpf::RSH32_IMM => { + name = "rsh32"; + desc = alu_imm_str(name, &insn); + } + ebpf::RSH32_REG => { + name = "rsh32"; + desc = alu_reg_str(name, &insn); + } + ebpf::NEG32 => { + name = "neg32"; + desc = format!("{name} r{:}", insn.dst); + } + ebpf::MOD32_IMM => { + name = "mod32"; + desc = alu_imm_str(name, &insn); + } + ebpf::MOD32_REG => { + name = "mod32"; + desc = alu_reg_str(name, &insn); + } + ebpf::XOR32_IMM => { + name = "xor32"; + desc = alu_imm_str(name, &insn); + } + ebpf::XOR32_REG => { + name = "xor32"; + desc = alu_reg_str(name, &insn); + } + ebpf::MOV32_IMM => { + name = "mov32"; + desc = alu_imm_str(name, &insn); + } + ebpf::MOV32_REG => { + name = "mov32"; + desc = alu_reg_str(name, &insn); + } + ebpf::ARSH32_IMM => { + name = "arsh32"; + desc = alu_imm_str(name, &insn); + } + ebpf::ARSH32_REG => { + name = "arsh32"; + desc = alu_reg_str(name, &insn); + } + ebpf::LE => { + name = "le"; + desc = byteswap_str(name, &insn); + } + ebpf::BE => { + name = "be"; + desc = byteswap_str(name, &insn); + } + + // BPF_ALU64 class + ebpf::ADD64_IMM => { + name = "add64"; + desc = alu_imm_str(name, &insn); + } + ebpf::ADD64_REG => { + name = "add64"; + desc = alu_reg_str(name, &insn); + } + ebpf::SUB64_IMM => { + name = "sub64"; + desc = alu_imm_str(name, &insn); + } + ebpf::SUB64_REG => { + name = "sub64"; + desc = alu_reg_str(name, &insn); + } + ebpf::MUL64_IMM => { + name = "mul64"; + desc = alu_imm_str(name, &insn); + } + ebpf::MUL64_REG => { + name = "mul64"; + desc = alu_reg_str(name, &insn); + } + ebpf::DIV64_IMM => { + name = "div64"; + desc = alu_imm_str(name, &insn); + } + ebpf::DIV64_REG => { + name = "div64"; + desc = alu_reg_str(name, &insn); + } + ebpf::OR64_IMM => { + name = "or64"; + desc = alu_imm_str(name, &insn); + } + ebpf::OR64_REG => { + name = "or64"; + desc = alu_reg_str(name, &insn); + } + ebpf::AND64_IMM => { + name = "and64"; + desc = alu_imm_str(name, &insn); + } + ebpf::AND64_REG => { + name = "and64"; + desc = alu_reg_str(name, &insn); + } + ebpf::LSH64_IMM => { + name = "lsh64"; + desc = alu_imm_str(name, &insn); + } + ebpf::LSH64_REG => { + name = "lsh64"; + desc = alu_reg_str(name, &insn); + } + ebpf::RSH64_IMM => { + name = "rsh64"; + desc = alu_imm_str(name, &insn); + } + ebpf::RSH64_REG => { + name = "rsh64"; + desc = alu_reg_str(name, &insn); + } + ebpf::NEG64 => { + name = "neg64"; + desc = format!("{name} r{:}", insn.dst); + } + ebpf::MOD64_IMM => { + name = "mod64"; + desc = alu_imm_str(name, &insn); + } + ebpf::MOD64_REG => { + name = "mod64"; + desc = alu_reg_str(name, &insn); + } + ebpf::XOR64_IMM => { + name = "xor64"; + desc = alu_imm_str(name, &insn); + } + ebpf::XOR64_REG => { + name = "xor64"; + desc = alu_reg_str(name, &insn); + } + ebpf::MOV64_IMM => { + name = "mov64"; + desc = alu_imm_str(name, &insn); + } + ebpf::MOV64_REG => { + name = "mov64"; + desc = alu_reg_str(name, &insn); + } + ebpf::ARSH64_IMM => { + name = "arsh64"; + desc = alu_imm_str(name, &insn); + } + ebpf::ARSH64_REG => { + name = "arsh64"; + desc = alu_reg_str(name, &insn); + } + + // BPF_JMP class + ebpf::JA => { + name = "ja"; + desc = if insn.off >= 0 { + format!("{name} +{:#x}", insn.off) + } else { + format!("{name} -{:#x}", -insn.off) + } + } + ebpf::JEQ_IMM => { + name = "jeq"; + desc = jmp_imm_str(name, &insn); + } + ebpf::JEQ_REG => { + name = "jeq"; + desc = jmp_reg_str(name, &insn); + } + ebpf::JGT_IMM => { + name = "jgt"; + desc = jmp_imm_str(name, &insn); + } + ebpf::JGT_REG => { + name = "jgt"; + desc = jmp_reg_str(name, &insn); + } + ebpf::JGE_IMM => { + name = "jge"; + desc = jmp_imm_str(name, &insn); + } + ebpf::JGE_REG => { + name = "jge"; + desc = jmp_reg_str(name, &insn); + } + ebpf::JLT_IMM => { + name = "jlt"; + desc = jmp_imm_str(name, &insn); + } + ebpf::JLT_REG => { + name = "jlt"; + desc = jmp_reg_str(name, &insn); + } + ebpf::JLE_IMM => { + name = "jle"; + desc = jmp_imm_str(name, &insn); + } + ebpf::JLE_REG => { + name = "jle"; + desc = jmp_reg_str(name, &insn); + } + ebpf::JSET_IMM => { + name = "jset"; + desc = jmp_imm_str(name, &insn); + } + ebpf::JSET_REG => { + name = "jset"; + desc = jmp_reg_str(name, &insn); + } + ebpf::JNE_IMM => { + name = "jne"; + desc = jmp_imm_str(name, &insn); + } + ebpf::JNE_REG => { + name = "jne"; + desc = jmp_reg_str(name, &insn); + } + ebpf::JSGT_IMM => { + name = "jsgt"; + desc = jmp_imm_str(name, &insn); + } + ebpf::JSGT_REG => { + name = "jsgt"; + desc = jmp_reg_str(name, &insn); + } + ebpf::JSGE_IMM => { + name = "jsge"; + desc = jmp_imm_str(name, &insn); + } + ebpf::JSGE_REG => { + name = "jsge"; + desc = jmp_reg_str(name, &insn); + } + ebpf::JSLT_IMM => { + name = "jslt"; + desc = jmp_imm_str(name, &insn); + } + ebpf::JSLT_REG => { + name = "jslt"; + desc = jmp_reg_str(name, &insn); + } + ebpf::JSLE_IMM => { + name = "jsle"; + desc = jmp_imm_str(name, &insn); + } + ebpf::JSLE_REG => { + name = "jsle"; + desc = jmp_reg_str(name, &insn); + } + ebpf::CALL => { + name = "call"; + desc = format!("{name} {:#x}", insn.imm); + } + ebpf::TAIL_CALL => { + name = "tail_call"; + desc = name.to_string(); + } + ebpf::EXIT => { + name = "exit"; + desc = name.to_string(); + } + + // BPF_JMP32 class + ebpf::JEQ_IMM32 => { + name = "jeq32"; + desc = jmp_imm_str(name, &insn); + } + ebpf::JEQ_REG32 => { + name = "jeq32"; + desc = jmp_reg_str(name, &insn); + } + ebpf::JGT_IMM32 => { + name = "jgt32"; + desc = jmp_imm_str(name, &insn); + } + ebpf::JGT_REG32 => { + name = "jgt32"; + desc = jmp_reg_str(name, &insn); + } + ebpf::JGE_IMM32 => { + name = "jge32"; + desc = jmp_imm_str(name, &insn); + } + ebpf::JGE_REG32 => { + name = "jge32"; + desc = jmp_reg_str(name, &insn); + } + ebpf::JLT_IMM32 => { + name = "jlt32"; + desc = jmp_imm_str(name, &insn); + } + ebpf::JLT_REG32 => { + name = "jlt32"; + desc = jmp_reg_str(name, &insn); + } + ebpf::JLE_IMM32 => { + name = "jle32"; + desc = jmp_imm_str(name, &insn); + } + ebpf::JLE_REG32 => { + name = "jle32"; + desc = jmp_reg_str(name, &insn); + } + ebpf::JSET_IMM32 => { + name = "jset32"; + desc = jmp_imm_str(name, &insn); + } + ebpf::JSET_REG32 => { + name = "jset32"; + desc = jmp_reg_str(name, &insn); + } + ebpf::JNE_IMM32 => { + name = "jne32"; + desc = jmp_imm_str(name, &insn); + } + ebpf::JNE_REG32 => { + name = "jne32"; + desc = jmp_reg_str(name, &insn); + } + ebpf::JSGT_IMM32 => { + name = "jsgt32"; + desc = jmp_imm_str(name, &insn); + } + ebpf::JSGT_REG32 => { + name = "jsgt32"; + desc = jmp_reg_str(name, &insn); + } + ebpf::JSGE_IMM32 => { + name = "jsge32"; + desc = jmp_imm_str(name, &insn); + } + ebpf::JSGE_REG32 => { + name = "jsge32"; + desc = jmp_reg_str(name, &insn); + } + ebpf::JSLT_IMM32 => { + name = "jslt32"; + desc = jmp_imm_str(name, &insn); + } + ebpf::JSLT_REG32 => { + name = "jslt32"; + desc = jmp_reg_str(name, &insn); + } + ebpf::JSLE_IMM32 => { + name = "jsle32"; + desc = jmp_imm_str(name, &insn); + } + ebpf::JSLE_REG32 => { + name = "jsle32"; + desc = jmp_reg_str(name, &insn); + } + + _ => { + panic!( + "[Disassembler] Error: unknown eBPF opcode {:#2x} (insn #{:?})", + insn.opc, insn_ptr + ); + } + }; + + let hl_insn = HLInsn { + opc: insn.opc, + name: name.to_string(), + desc, + dst: insn.dst, + src: insn.src, + off: insn.off, + imm, + }; + + res.push(hl_insn); + + insn_ptr += 1; + } + res +} + +/// Disassemble an eBPF program into human-readable instructions and prints it to standard output. +/// +/// The program is not checked for errors or inconsistencies. +/// +/// # Examples +/// +/// ``` +/// use rbpf::disassembler; +/// let prog = &[ +/// 0x07, 0x01, 0x00, 0x00, 0x05, 0x06, 0x00, 0x00, +/// 0xb7, 0x02, 0x00, 0x00, 0x32, 0x00, 0x00, 0x00, +/// 0xbf, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +/// 0xdc, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, +/// 0x87, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +/// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +/// ]; +/// disassembler::disassemble(prog); +/// # // "\nadd64 r1, 0x605\nmov64 r2, 0x32\nmov64 r1, r0\nbe16 r0\nneg64 r2\nexit" +/// ``` +/// +/// This will produce the following output: +/// +/// ```test +/// add64 r1, 0x605 +/// mov64 r2, 0x32 +/// mov64 r1, r0 +/// be16 r0 +/// neg64 r2 +/// exit +/// ``` +pub fn disassemble(prog: &[u8]) { + #[cfg(feature = "std")] + { + for insn in to_insn_vec(prog) { + println!("{}", insn.desc); + } + } + #[cfg(not(feature = "std"))] + { + for insn in to_insn_vec(prog) { + log::info!("{}", insn.desc); + } + } +} diff --git a/kernel/crates/rbpf/src/ebpf.rs b/kernel/crates/rbpf/src/ebpf.rs new file mode 100644 index 00000000..33018b06 --- /dev/null +++ b/kernel/crates/rbpf/src/ebpf.rs @@ -0,0 +1,635 @@ +// SPDX-License-Identifier: (Apache-2.0 OR MIT) +// Copyright 2016 6WIND S.A. + +//! This module contains all the definitions related to eBPF, and some functions permitting to +//! manipulate eBPF instructions. +//! +//! The number of bytes in an instruction, the maximum number of instructions in a program, and +//! also all operation codes are defined here as constants. +//! +//! The structure for an instruction used by this crate, as well as the function to extract it from +//! a program, is also defined in the module. +//! +//! To learn more about these instructions, see the Linux kernel documentation: +//! , or for a shorter version of +//! the list of the operation codes: + +use alloc::{vec, vec::Vec}; + +use byteorder::{ByteOrder, LittleEndian}; + +/// The maximum call depth is 8 +pub const RBPF_MAX_CALL_DEPTH: usize = 8; + +/// Maximum number of instructions in an eBPF program. +pub const PROG_MAX_INSNS: usize = 1000000; +/// Size of an eBPF instructions, in bytes. +pub const INSN_SIZE: usize = 8; +/// Maximum size of an eBPF program, in bytes. +pub const PROG_MAX_SIZE: usize = PROG_MAX_INSNS * INSN_SIZE; +/// Stack for the eBPF stack, in bytes. +pub const STACK_SIZE: usize = 512; + +// eBPF op codes. +// See also https://www.kernel.org/doc/Documentation/networking/filter.txt + +// Three least significant bits are operation class: +/// BPF operation class: load from immediate. +pub const BPF_LD: u8 = 0x00; +/// BPF operation class: load from register. +pub const BPF_LDX: u8 = 0x01; +/// BPF operation class: store immediate. +pub const BPF_ST: u8 = 0x02; +/// BPF operation class: store value from register. +pub const BPF_STX: u8 = 0x03; +/// BPF operation class: 32 bits arithmetic operation. +pub const BPF_ALU: u8 = 0x04; +/// BPF operation class: jump (64-bit wide operands for comparisons). +pub const BPF_JMP: u8 = 0x05; +/// BPF operation class: jump (32-bit wide operands for comparisons). +pub const BPF_JMP32: u8 = 0x06; +// [ class 6 unused, reserved for future use ] +/// BPF operation class: 64 bits arithmetic operation. +pub const BPF_ALU64: u8 = 0x07; + +// For load and store instructions: +// +------------+--------+------------+ +// | 3 bits | 2 bits | 3 bits | +// | mode | size | insn class | +// +------------+--------+------------+ +// (MSB) (LSB) + +// Size modifiers: +/// BPF size modifier: word (4 bytes). +pub const BPF_W: u8 = 0x00; +/// BPF size modifier: half-word (2 bytes). +pub const BPF_H: u8 = 0x08; +/// BPF size modifier: byte (1 byte). +pub const BPF_B: u8 = 0x10; +/// BPF size modifier: double word (8 bytes). +pub const BPF_DW: u8 = 0x18; + +// Mode modifiers: +/// BPF mode modifier: immediate value. +pub const BPF_IMM: u8 = 0x00; +/// BPF mode modifier: absolute load. +pub const BPF_ABS: u8 = 0x20; +/// BPF mode modifier: indirect load. +pub const BPF_IND: u8 = 0x40; +/// BPF mode modifier: load from / store to memory. +pub const BPF_MEM: u8 = 0x60; +// [ 0x80 reserved ] +// [ 0xa0 reserved ] +/// BPF mode modifier: exclusive add. +pub const BPF_XADD: u8 = 0xc0; + +// For arithmetic (BPF_ALU/BPF_ALU64) and jump (BPF_JMP) instructions: +// +----------------+--------+--------+ +// | 4 bits |1 b.| 3 bits | +// | operation code | src| insn class | +// +----------------+----+------------+ +// (MSB) (LSB) + +// Source modifiers: +/// BPF source operand modifier: 32-bit immediate value. +pub const BPF_K: u8 = 0x00; +/// BPF source operand modifier: `src` register. +pub const BPF_X: u8 = 0x08; + +// Operation codes -- BPF_ALU or BPF_ALU64 classes: +/// BPF ALU/ALU64 operation code: addition. +pub const BPF_ADD: u8 = 0x00; +/// BPF ALU/ALU64 operation code: subtraction. +pub const BPF_SUB: u8 = 0x10; +/// BPF ALU/ALU64 operation code: multiplication. +pub const BPF_MUL: u8 = 0x20; +/// BPF ALU/ALU64 operation code: division. +pub const BPF_DIV: u8 = 0x30; +/// BPF ALU/ALU64 operation code: or. +pub const BPF_OR: u8 = 0x40; +/// BPF ALU/ALU64 operation code: and. +pub const BPF_AND: u8 = 0x50; +/// BPF ALU/ALU64 operation code: left shift. +pub const BPF_LSH: u8 = 0x60; +/// BPF ALU/ALU64 operation code: right shift. +pub const BPF_RSH: u8 = 0x70; +/// BPF ALU/ALU64 operation code: negation. +pub const BPF_NEG: u8 = 0x80; +/// BPF ALU/ALU64 operation code: modulus. +pub const BPF_MOD: u8 = 0x90; +/// BPF ALU/ALU64 operation code: exclusive or. +pub const BPF_XOR: u8 = 0xa0; +/// BPF ALU/ALU64 operation code: move. +pub const BPF_MOV: u8 = 0xb0; +/// BPF ALU/ALU64 operation code: sign extending right shift. +pub const BPF_ARSH: u8 = 0xc0; +/// BPF ALU/ALU64 operation code: endianness conversion. +pub const BPF_END: u8 = 0xd0; + +// Operation codes -- BPF_JMP or BPF_JMP32 classes: +/// BPF JMP operation code: jump. +pub const BPF_JA: u8 = 0x00; +/// BPF JMP operation code: jump if equal. +pub const BPF_JEQ: u8 = 0x10; +/// BPF JMP operation code: jump if greater than. +pub const BPF_JGT: u8 = 0x20; +/// BPF JMP operation code: jump if greater or equal. +pub const BPF_JGE: u8 = 0x30; +/// BPF JMP operation code: jump if `src` & `reg`. +pub const BPF_JSET: u8 = 0x40; +/// BPF JMP operation code: jump if not equal. +pub const BPF_JNE: u8 = 0x50; +/// BPF JMP operation code: jump if greater than (signed). +pub const BPF_JSGT: u8 = 0x60; +/// BPF JMP operation code: jump if greater or equal (signed). +pub const BPF_JSGE: u8 = 0x70; +/// BPF JMP operation code: helper function call. +pub const BPF_CALL: u8 = 0x80; +/// BPF JMP operation code: return from program. +pub const BPF_EXIT: u8 = 0x90; +/// BPF JMP operation code: jump if lower than. +pub const BPF_JLT: u8 = 0xa0; +/// BPF JMP operation code: jump if lower or equal. +pub const BPF_JLE: u8 = 0xb0; +/// BPF JMP operation code: jump if lower than (signed). +pub const BPF_JSLT: u8 = 0xc0; +/// BPF JMP operation code: jump if lower or equal (signed). +pub const BPF_JSLE: u8 = 0xd0; + +// Op codes +// (Following operation names are not “official”, but may be proper to rbpf; Linux kernel only +// combines above flags and does not attribute a name per operation.) + +/// BPF opcode: `ldabsb src, dst, imm`. +pub const LD_ABS_B: u8 = BPF_LD | BPF_ABS | BPF_B; +/// BPF opcode: `ldabsh src, dst, imm`. +pub const LD_ABS_H: u8 = BPF_LD | BPF_ABS | BPF_H; +/// BPF opcode: `ldabsw src, dst, imm`. +pub const LD_ABS_W: u8 = BPF_LD | BPF_ABS | BPF_W; +/// BPF opcode: `ldabsdw src, dst, imm`. +pub const LD_ABS_DW: u8 = BPF_LD | BPF_ABS | BPF_DW; +/// BPF opcode: `ldindb src, dst, imm`. +pub const LD_IND_B: u8 = BPF_LD | BPF_IND | BPF_B; +/// BPF opcode: `ldindh src, dst, imm`. +pub const LD_IND_H: u8 = BPF_LD | BPF_IND | BPF_H; +/// BPF opcode: `ldindw src, dst, imm`. +pub const LD_IND_W: u8 = BPF_LD | BPF_IND | BPF_W; +/// BPF opcode: `ldinddw src, dst, imm`. +pub const LD_IND_DW: u8 = BPF_LD | BPF_IND | BPF_DW; + +#[allow(unknown_lints)] +#[allow(clippy::eq_op)] +/// BPF opcode: `lddw dst, imm` /// `dst = imm`. +pub const LD_DW_IMM: u8 = BPF_LD | BPF_IMM | BPF_DW; +/// BPF opcode: `ldxb dst, [src + off]` /// `dst = (src + off) as u8`. +pub const LD_B_REG: u8 = BPF_LDX | BPF_MEM | BPF_B; +/// BPF opcode: `ldxh dst, [src + off]` /// `dst = (src + off) as u16`. +pub const LD_H_REG: u8 = BPF_LDX | BPF_MEM | BPF_H; +/// BPF opcode: `ldxw dst, [src + off]` /// `dst = (src + off) as u32`. +pub const LD_W_REG: u8 = BPF_LDX | BPF_MEM | BPF_W; +/// BPF opcode: `ldxdw dst, [src + off]` /// `dst = (src + off) as u64`. +pub const LD_DW_REG: u8 = BPF_LDX | BPF_MEM | BPF_DW; +/// BPF opcode: `stb [dst + off], imm` /// `(dst + offset) as u8 = imm`. +pub const ST_B_IMM: u8 = BPF_ST | BPF_MEM | BPF_B; +/// BPF opcode: `sth [dst + off], imm` /// `(dst + offset) as u16 = imm`. +pub const ST_H_IMM: u8 = BPF_ST | BPF_MEM | BPF_H; +/// BPF opcode: `stw [dst + off], imm` /// `(dst + offset) as u32 = imm`. +pub const ST_W_IMM: u8 = BPF_ST | BPF_MEM | BPF_W; +/// BPF opcode: `stdw [dst + off], imm` /// `(dst + offset) as u64 = imm`. +pub const ST_DW_IMM: u8 = BPF_ST | BPF_MEM | BPF_DW; +/// BPF opcode: `stxb [dst + off], src` /// `(dst + offset) as u8 = src`. +pub const ST_B_REG: u8 = BPF_STX | BPF_MEM | BPF_B; +/// BPF opcode: `stxh [dst + off], src` /// `(dst + offset) as u16 = src`. +pub const ST_H_REG: u8 = BPF_STX | BPF_MEM | BPF_H; +/// BPF opcode: `stxw [dst + off], src` /// `(dst + offset) as u32 = src`. +pub const ST_W_REG: u8 = BPF_STX | BPF_MEM | BPF_W; +/// BPF opcode: `stxdw [dst + off], src` /// `(dst + offset) as u64 = src`. +pub const ST_DW_REG: u8 = BPF_STX | BPF_MEM | BPF_DW; + +/// BPF opcode: `stxxaddw [dst + off], src`. +pub const ST_W_XADD: u8 = BPF_STX | BPF_XADD | BPF_W; +/// BPF opcode: `stxxadddw [dst + off], src`. +pub const ST_DW_XADD: u8 = BPF_STX | BPF_XADD | BPF_DW; + +/// BPF opcode: `add32 dst, imm` /// `dst += imm`. +pub const ADD32_IMM: u8 = BPF_ALU | BPF_K | BPF_ADD; +/// BPF opcode: `add32 dst, src` /// `dst += src`. +pub const ADD32_REG: u8 = BPF_ALU | BPF_X | BPF_ADD; +/// BPF opcode: `sub32 dst, imm` /// `dst -= imm`. +pub const SUB32_IMM: u8 = BPF_ALU | BPF_K | BPF_SUB; +/// BPF opcode: `sub32 dst, src` /// `dst -= src`. +pub const SUB32_REG: u8 = BPF_ALU | BPF_X | BPF_SUB; +/// BPF opcode: `mul32 dst, imm` /// `dst *= imm`. +pub const MUL32_IMM: u8 = BPF_ALU | BPF_K | BPF_MUL; +/// BPF opcode: `mul32 dst, src` /// `dst *= src`. +pub const MUL32_REG: u8 = BPF_ALU | BPF_X | BPF_MUL; +/// BPF opcode: `div32 dst, imm` /// `dst /= imm`. +pub const DIV32_IMM: u8 = BPF_ALU | BPF_K | BPF_DIV; +/// BPF opcode: `div32 dst, src` /// `dst /= src`. +pub const DIV32_REG: u8 = BPF_ALU | BPF_X | BPF_DIV; +/// BPF opcode: `or32 dst, imm` /// `dst |= imm`. +pub const OR32_IMM: u8 = BPF_ALU | BPF_K | BPF_OR; +/// BPF opcode: `or32 dst, src` /// `dst |= src`. +pub const OR32_REG: u8 = BPF_ALU | BPF_X | BPF_OR; +/// BPF opcode: `and32 dst, imm` /// `dst &= imm`. +pub const AND32_IMM: u8 = BPF_ALU | BPF_K | BPF_AND; +/// BPF opcode: `and32 dst, src` /// `dst &= src`. +pub const AND32_REG: u8 = BPF_ALU | BPF_X | BPF_AND; +/// BPF opcode: `lsh32 dst, imm` /// `dst <<= imm`. +pub const LSH32_IMM: u8 = BPF_ALU | BPF_K | BPF_LSH; +/// BPF opcode: `lsh32 dst, src` /// `dst <<= src`. +pub const LSH32_REG: u8 = BPF_ALU | BPF_X | BPF_LSH; +/// BPF opcode: `rsh32 dst, imm` /// `dst >>= imm`. +pub const RSH32_IMM: u8 = BPF_ALU | BPF_K | BPF_RSH; +/// BPF opcode: `rsh32 dst, src` /// `dst >>= src`. +pub const RSH32_REG: u8 = BPF_ALU | BPF_X | BPF_RSH; +/// BPF opcode: `neg32 dst` /// `dst = -dst`. +pub const NEG32: u8 = BPF_ALU | BPF_NEG; +/// BPF opcode: `mod32 dst, imm` /// `dst %= imm`. +pub const MOD32_IMM: u8 = BPF_ALU | BPF_K | BPF_MOD; +/// BPF opcode: `mod32 dst, src` /// `dst %= src`. +pub const MOD32_REG: u8 = BPF_ALU | BPF_X | BPF_MOD; +/// BPF opcode: `xor32 dst, imm` /// `dst ^= imm`. +pub const XOR32_IMM: u8 = BPF_ALU | BPF_K | BPF_XOR; +/// BPF opcode: `xor32 dst, src` /// `dst ^= src`. +pub const XOR32_REG: u8 = BPF_ALU | BPF_X | BPF_XOR; +/// BPF opcode: `mov32 dst, imm` /// `dst = imm`. +pub const MOV32_IMM: u8 = BPF_ALU | BPF_K | BPF_MOV; +/// BPF opcode: `mov32 dst, src` /// `dst = src`. +pub const MOV32_REG: u8 = BPF_ALU | BPF_X | BPF_MOV; +/// BPF opcode: `arsh32 dst, imm` /// `dst >>= imm (arithmetic)`. +/// +/// +pub const ARSH32_IMM: u8 = BPF_ALU | BPF_K | BPF_ARSH; +/// BPF opcode: `arsh32 dst, src` /// `dst >>= src (arithmetic)`. +/// +/// +pub const ARSH32_REG: u8 = BPF_ALU | BPF_X | BPF_ARSH; + +/// BPF opcode: `le dst` /// `dst = htole(dst), with imm in {16, 32, 64}`. +pub const LE: u8 = BPF_ALU | BPF_K | BPF_END; +/// BPF opcode: `be dst` /// `dst = htobe(dst), with imm in {16, 32, 64}`. +pub const BE: u8 = BPF_ALU | BPF_X | BPF_END; + +/// BPF opcode: `add64 dst, imm` /// `dst += imm`. +pub const ADD64_IMM: u8 = BPF_ALU64 | BPF_K | BPF_ADD; +/// BPF opcode: `add64 dst, src` /// `dst += src`. +pub const ADD64_REG: u8 = BPF_ALU64 | BPF_X | BPF_ADD; +/// BPF opcode: `sub64 dst, imm` /// `dst -= imm`. +pub const SUB64_IMM: u8 = BPF_ALU64 | BPF_K | BPF_SUB; +/// BPF opcode: `sub64 dst, src` /// `dst -= src`. +pub const SUB64_REG: u8 = BPF_ALU64 | BPF_X | BPF_SUB; +/// BPF opcode: `div64 dst, imm` /// `dst /= imm`. +pub const MUL64_IMM: u8 = BPF_ALU64 | BPF_K | BPF_MUL; +/// BPF opcode: `div64 dst, src` /// `dst /= src`. +pub const MUL64_REG: u8 = BPF_ALU64 | BPF_X | BPF_MUL; +/// BPF opcode: `div64 dst, imm` /// `dst /= imm`. +pub const DIV64_IMM: u8 = BPF_ALU64 | BPF_K | BPF_DIV; +/// BPF opcode: `div64 dst, src` /// `dst /= src`. +pub const DIV64_REG: u8 = BPF_ALU64 | BPF_X | BPF_DIV; +/// BPF opcode: `or64 dst, imm` /// `dst |= imm`. +pub const OR64_IMM: u8 = BPF_ALU64 | BPF_K | BPF_OR; +/// BPF opcode: `or64 dst, src` /// `dst |= src`. +pub const OR64_REG: u8 = BPF_ALU64 | BPF_X | BPF_OR; +/// BPF opcode: `and64 dst, imm` /// `dst &= imm`. +pub const AND64_IMM: u8 = BPF_ALU64 | BPF_K | BPF_AND; +/// BPF opcode: `and64 dst, src` /// `dst &= src`. +pub const AND64_REG: u8 = BPF_ALU64 | BPF_X | BPF_AND; +/// BPF opcode: `lsh64 dst, imm` /// `dst <<= imm`. +pub const LSH64_IMM: u8 = BPF_ALU64 | BPF_K | BPF_LSH; +/// BPF opcode: `lsh64 dst, src` /// `dst <<= src`. +pub const LSH64_REG: u8 = BPF_ALU64 | BPF_X | BPF_LSH; +/// BPF opcode: `rsh64 dst, imm` /// `dst >>= imm`. +pub const RSH64_IMM: u8 = BPF_ALU64 | BPF_K | BPF_RSH; +/// BPF opcode: `rsh64 dst, src` /// `dst >>= src`. +pub const RSH64_REG: u8 = BPF_ALU64 | BPF_X | BPF_RSH; +/// BPF opcode: `neg64 dst, imm` /// `dst = -dst`. +pub const NEG64: u8 = BPF_ALU64 | BPF_NEG; +/// BPF opcode: `mod64 dst, imm` /// `dst %= imm`. +pub const MOD64_IMM: u8 = BPF_ALU64 | BPF_K | BPF_MOD; +/// BPF opcode: `mod64 dst, src` /// `dst %= src`. +pub const MOD64_REG: u8 = BPF_ALU64 | BPF_X | BPF_MOD; +/// BPF opcode: `xor64 dst, imm` /// `dst ^= imm`. +pub const XOR64_IMM: u8 = BPF_ALU64 | BPF_K | BPF_XOR; +/// BPF opcode: `xor64 dst, src` /// `dst ^= src`. +pub const XOR64_REG: u8 = BPF_ALU64 | BPF_X | BPF_XOR; +/// BPF opcode: `mov64 dst, imm` /// `dst = imm`. +pub const MOV64_IMM: u8 = BPF_ALU64 | BPF_K | BPF_MOV; +/// BPF opcode: `mov64 dst, src` /// `dst = src`. +pub const MOV64_REG: u8 = BPF_ALU64 | BPF_X | BPF_MOV; +/// BPF opcode: `arsh64 dst, imm` /// `dst >>= imm (arithmetic)`. +/// +/// +pub const ARSH64_IMM: u8 = BPF_ALU64 | BPF_K | BPF_ARSH; +/// BPF opcode: `arsh64 dst, src` /// `dst >>= src (arithmetic)`. +/// +/// +pub const ARSH64_REG: u8 = BPF_ALU64 | BPF_X | BPF_ARSH; + +/// BPF opcode: `ja +off` /// `PC += off`. +pub const JA: u8 = BPF_JMP | BPF_JA; +/// BPF opcode: `jeq dst, imm, +off` /// `PC += off if dst == imm`. +pub const JEQ_IMM: u8 = BPF_JMP | BPF_K | BPF_JEQ; +/// BPF opcode: `jeq dst, src, +off` /// `PC += off if dst == src`. +pub const JEQ_REG: u8 = BPF_JMP | BPF_X | BPF_JEQ; +/// BPF opcode: `jgt dst, imm, +off` /// `PC += off if dst > imm`. +pub const JGT_IMM: u8 = BPF_JMP | BPF_K | BPF_JGT; +/// BPF opcode: `jgt dst, src, +off` /// `PC += off if dst > src`. +pub const JGT_REG: u8 = BPF_JMP | BPF_X | BPF_JGT; +/// BPF opcode: `jge dst, imm, +off` /// `PC += off if dst >= imm`. +pub const JGE_IMM: u8 = BPF_JMP | BPF_K | BPF_JGE; +/// BPF opcode: `jge dst, src, +off` /// `PC += off if dst >= src`. +pub const JGE_REG: u8 = BPF_JMP | BPF_X | BPF_JGE; +/// BPF opcode: `jlt dst, imm, +off` /// `PC += off if dst < imm`. +pub const JLT_IMM: u8 = BPF_JMP | BPF_K | BPF_JLT; +/// BPF opcode: `jlt dst, src, +off` /// `PC += off if dst < src`. +pub const JLT_REG: u8 = BPF_JMP | BPF_X | BPF_JLT; +/// BPF opcode: `jle dst, imm, +off` /// `PC += off if dst <= imm`. +pub const JLE_IMM: u8 = BPF_JMP | BPF_K | BPF_JLE; +/// BPF opcode: `jle dst, src, +off` /// `PC += off if dst <= src`. +pub const JLE_REG: u8 = BPF_JMP | BPF_X | BPF_JLE; +/// BPF opcode: `jset dst, imm, +off` /// `PC += off if dst & imm`. +pub const JSET_IMM: u8 = BPF_JMP | BPF_K | BPF_JSET; +/// BPF opcode: `jset dst, src, +off` /// `PC += off if dst & src`. +pub const JSET_REG: u8 = BPF_JMP | BPF_X | BPF_JSET; +/// BPF opcode: `jne dst, imm, +off` /// `PC += off if dst != imm`. +pub const JNE_IMM: u8 = BPF_JMP | BPF_K | BPF_JNE; +/// BPF opcode: `jne dst, src, +off` /// `PC += off if dst != src`. +pub const JNE_REG: u8 = BPF_JMP | BPF_X | BPF_JNE; +/// BPF opcode: `jsgt dst, imm, +off` /// `PC += off if dst > imm (signed)`. +pub const JSGT_IMM: u8 = BPF_JMP | BPF_K | BPF_JSGT; +/// BPF opcode: `jsgt dst, src, +off` /// `PC += off if dst > src (signed)`. +pub const JSGT_REG: u8 = BPF_JMP | BPF_X | BPF_JSGT; +/// BPF opcode: `jsge dst, imm, +off` /// `PC += off if dst >= imm (signed)`. +pub const JSGE_IMM: u8 = BPF_JMP | BPF_K | BPF_JSGE; +/// BPF opcode: `jsge dst, src, +off` /// `PC += off if dst >= src (signed)`. +pub const JSGE_REG: u8 = BPF_JMP | BPF_X | BPF_JSGE; +/// BPF opcode: `jslt dst, imm, +off` /// `PC += off if dst < imm (signed)`. +pub const JSLT_IMM: u8 = BPF_JMP | BPF_K | BPF_JSLT; +/// BPF opcode: `jslt dst, src, +off` /// `PC += off if dst < src (signed)`. +pub const JSLT_REG: u8 = BPF_JMP | BPF_X | BPF_JSLT; +/// BPF opcode: `jsle dst, imm, +off` /// `PC += off if dst <= imm (signed)`. +pub const JSLE_IMM: u8 = BPF_JMP | BPF_K | BPF_JSLE; +/// BPF opcode: `jsle dst, src, +off` /// `PC += off if dst <= src (signed)`. +pub const JSLE_REG: u8 = BPF_JMP | BPF_X | BPF_JSLE; + +/// BPF opcode: `jeq dst, imm, +off` /// `PC += off if (dst as u32) == imm`. +pub const JEQ_IMM32: u8 = BPF_JMP32 | BPF_K | BPF_JEQ; +/// BPF opcode: `jeq dst, src, +off` /// `PC += off if (dst as u32) == (src as u32)`. +pub const JEQ_REG32: u8 = BPF_JMP32 | BPF_X | BPF_JEQ; +/// BPF opcode: `jgt dst, imm, +off` /// `PC += off if (dst as u32) > imm`. +pub const JGT_IMM32: u8 = BPF_JMP32 | BPF_K | BPF_JGT; +/// BPF opcode: `jgt dst, src, +off` /// `PC += off if (dst as u32) > (src as u32)`. +pub const JGT_REG32: u8 = BPF_JMP32 | BPF_X | BPF_JGT; +/// BPF opcode: `jge dst, imm, +off` /// `PC += off if (dst as u32) >= imm`. +pub const JGE_IMM32: u8 = BPF_JMP32 | BPF_K | BPF_JGE; +/// BPF opcode: `jge dst, src, +off` /// `PC += off if (dst as u32) >= (src as u32)`. +pub const JGE_REG32: u8 = BPF_JMP32 | BPF_X | BPF_JGE; +/// BPF opcode: `jlt dst, imm, +off` /// `PC += off if (dst as u32) < imm`. +pub const JLT_IMM32: u8 = BPF_JMP32 | BPF_K | BPF_JLT; +/// BPF opcode: `jlt dst, src, +off` /// `PC += off if (dst as u32) < (src as u32)`. +pub const JLT_REG32: u8 = BPF_JMP32 | BPF_X | BPF_JLT; +/// BPF opcode: `jle dst, imm, +off` /// `PC += off if (dst as u32) <= imm`. +pub const JLE_IMM32: u8 = BPF_JMP32 | BPF_K | BPF_JLE; +/// BPF opcode: `jle dst, src, +off` /// `PC += off if (dst as u32) <= (src as u32)`. +pub const JLE_REG32: u8 = BPF_JMP32 | BPF_X | BPF_JLE; +/// BPF opcode: `jset dst, imm, +off` /// `PC += off if (dst as u32) & imm`. +pub const JSET_IMM32: u8 = BPF_JMP32 | BPF_K | BPF_JSET; +/// BPF opcode: `jset dst, src, +off` /// `PC += off if (dst as u32) & (src as u32)`. +pub const JSET_REG32: u8 = BPF_JMP32 | BPF_X | BPF_JSET; +/// BPF opcode: `jne dst, imm, +off` /// `PC += off if (dst as u32) != imm`. +pub const JNE_IMM32: u8 = BPF_JMP32 | BPF_K | BPF_JNE; +/// BPF opcode: `jne dst, src, +off` /// `PC += off if (dst as u32) != (src as u32)`. +pub const JNE_REG32: u8 = BPF_JMP32 | BPF_X | BPF_JNE; +/// BPF opcode: `jsgt dst, imm, +off` /// `PC += off if (dst as i32) > imm (signed)`. +pub const JSGT_IMM32: u8 = BPF_JMP32 | BPF_K | BPF_JSGT; +/// BPF opcode: `jsgt dst, src, +off` /// `PC += off if (dst as i32) > (src as i32) (signed)`. +pub const JSGT_REG32: u8 = BPF_JMP32 | BPF_X | BPF_JSGT; +/// BPF opcode: `jsge dst, imm, +off` /// `PC += off if (dst as i32) >= imm (signed)`. +pub const JSGE_IMM32: u8 = BPF_JMP32 | BPF_K | BPF_JSGE; +/// BPF opcode: `jsge dst, src, +off` /// `PC += off if (dst as i32) >= (src as i32) (signed)`. +pub const JSGE_REG32: u8 = BPF_JMP32 | BPF_X | BPF_JSGE; +/// BPF opcode: `jslt dst, imm, +off` /// `PC += off if (dst as i32) < imm (signed)`. +pub const JSLT_IMM32: u8 = BPF_JMP32 | BPF_K | BPF_JSLT; +/// BPF opcode: `jslt dst, src, +off` /// `PC += off if (dst as i32) < (src as i32) (signed)`. +pub const JSLT_REG32: u8 = BPF_JMP32 | BPF_X | BPF_JSLT; +/// BPF opcode: `jsle dst, imm, +off` /// `PC += off if (dst as i32) <= imm (signed)`. +pub const JSLE_IMM32: u8 = BPF_JMP32 | BPF_K | BPF_JSLE; +/// BPF opcode: `jsle dst, src, +off` /// `PC += off if (dst as i32) <= (src as i32) (signed)`. +pub const JSLE_REG32: u8 = BPF_JMP32 | BPF_X | BPF_JSLE; + +/// BPF opcode: `call imm` /// helper function call to helper with key `imm`. +pub const CALL: u8 = BPF_JMP | BPF_CALL; +/// BPF opcode: tail call. +pub const TAIL_CALL: u8 = BPF_JMP | BPF_X | BPF_CALL; +/// BPF opcode: `exit` /// `return r0`. +pub const EXIT: u8 = BPF_JMP | BPF_EXIT; + +// Used in JIT +/// Mask to extract the operation class from an operation code. +pub const BPF_CLS_MASK: u8 = 0x07; +/// Mask to extract the arithmetic operation code from an instruction operation code. +pub const BPF_ALU_OP_MASK: u8 = 0xf0; + +/// Prototype of an eBPF helper function. +pub type Helper = fn(u64, u64, u64, u64, u64) -> u64; + +/// An eBPF instruction. +/// +/// See for the Linux kernel +/// documentation about eBPF, or for a +/// more concise version. +#[derive(Debug, PartialEq, Eq, Clone)] +pub struct Insn { + /// Operation code. + pub opc: u8, + /// Destination register operand. + pub dst: u8, + /// Source register operand. + pub src: u8, + /// Offset operand. + pub off: i16, + /// Immediate value operand. + pub imm: i32, +} + +impl Insn { + /// Turn an `Insn` back into an array of bytes. + /// + /// # Examples + /// + /// ``` + /// use rbpf::ebpf; + /// + /// let prog: &[u8] = &[ + /// 0xb7, 0x12, 0x56, 0x34, 0xde, 0xbc, 0x9a, 0x78, + /// ]; + /// let insn = ebpf::Insn { + /// opc: 0xb7, + /// dst: 2, + /// src: 1, + /// off: 0x3456, + /// imm: 0x789abcde + /// }; + /// assert_eq!(insn.to_array(), prog); + /// ``` + pub fn to_array(&self) -> [u8; INSN_SIZE] { + [ + self.opc, + self.src.wrapping_shl(4) | self.dst, + (self.off & 0xff) as u8, + self.off.wrapping_shr(8) as u8, + (self.imm & 0xff) as u8, + (self.imm & 0xff_00).wrapping_shr(8) as u8, + (self.imm as u32 & 0xff_00_00).wrapping_shr(16) as u8, + (self.imm as u32 & 0xff_00_00_00).wrapping_shr(24) as u8, + ] + } + + /// Turn an `Insn` into an vector of bytes. + /// + /// # Examples + /// + /// ``` + /// use rbpf::ebpf; + /// + /// let prog: Vec = vec![ + /// 0xb7, 0x12, 0x56, 0x34, 0xde, 0xbc, 0x9a, 0x78, + /// ]; + /// let insn = ebpf::Insn { + /// opc: 0xb7, + /// dst: 2, + /// src: 1, + /// off: 0x3456, + /// imm: 0x789abcde + /// }; + /// assert_eq!(insn.to_vec(), prog); + /// ``` + pub fn to_vec(&self) -> Vec { + vec![ + self.opc, + self.src.wrapping_shl(4) | self.dst, + (self.off & 0xff) as u8, + self.off.wrapping_shr(8) as u8, + (self.imm & 0xff) as u8, + (self.imm & 0xff_00).wrapping_shr(8) as u8, + (self.imm as u32 & 0xff_00_00).wrapping_shr(16) as u8, + (self.imm as u32 & 0xff_00_00_00).wrapping_shr(24) as u8, + ] + } +} + +/// Get the instruction at `idx` of an eBPF program. `idx` is the index (number) of the +/// instruction (not a byte offset). The first instruction has index 0. +/// +/// # Panics +/// +/// Panics if it is not possible to get the instruction (if idx is too high, or last instruction is +/// incomplete). +/// +/// # Examples +/// +/// ``` +/// use rbpf::ebpf; +/// +/// let prog = &[ +/// 0xb7, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +/// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +/// ]; +/// let insn = ebpf::get_insn(prog, 1); +/// assert_eq!(insn.opc, 0x95); +/// ``` +/// +/// The example below will panic, since the last instruction is not complete and cannot be loaded. +/// +/// ```rust,should_panic +/// use rbpf::ebpf; +/// +/// let prog = &[ +/// 0xb7, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +/// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00 // two bytes missing +/// ]; +/// let insn = ebpf::get_insn(prog, 1); +/// ``` +pub fn get_insn(prog: &[u8], idx: usize) -> Insn { + // This guard should not be needed in most cases, since the verifier already checks the program + // size, and indexes should be fine in the interpreter/JIT. But this function is publicly + // available and user can call it with any `idx`, so we have to check anyway. + if (idx + 1) * INSN_SIZE > prog.len() { + panic!( + "Error: cannot reach instruction at index {:?} in program containing {:?} bytes", + idx, + prog.len() + ); + } + Insn { + opc: prog[INSN_SIZE * idx], + dst: prog[INSN_SIZE * idx + 1] & 0x0f, + src: (prog[INSN_SIZE * idx + 1] & 0xf0) >> 4, + off: LittleEndian::read_i16(&prog[(INSN_SIZE * idx + 2)..]), + imm: LittleEndian::read_i32(&prog[(INSN_SIZE * idx + 4)..]), + } +} + +/// Return a vector of `struct Insn` built from a program. +/// +/// This is provided as a convenience for users wishing to manipulate a vector of instructions, for +/// example for dumping the program instruction after instruction with a custom format. +/// +/// Note that the two parts of `LD_DW_IMM` instructions (spanning on 64 bits) are considered as two +/// distinct instructions. +/// +/// # Examples +/// +/// ``` +/// use rbpf::ebpf; +/// +/// let prog = &[ +/// 0x18, 0x00, 0x00, 0x00, 0x88, 0x77, 0x66, 0x55, +/// 0x00, 0x00, 0x00, 0x00, 0x44, 0x33, 0x22, 0x11, +/// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +/// ]; +/// +/// let v = ebpf::to_insn_vec(prog); +/// assert_eq!(v, vec![ +/// ebpf::Insn { +/// opc: 0x18, +/// dst: 0, +/// src: 0, +/// off: 0, +/// imm: 0x55667788 +/// }, +/// ebpf::Insn { +/// opc: 0, +/// dst: 0, +/// src: 0, +/// off: 0, +/// imm: 0x11223344 +/// }, +/// ebpf::Insn { +/// opc: 0x95, +/// dst: 0, +/// src: 0, +/// off: 0, +/// imm: 0 +/// }, +/// ]); +/// ``` +pub fn to_insn_vec(prog: &[u8]) -> Vec { + if prog.len() % INSN_SIZE != 0 { + panic!( + "Error: eBPF program length must be a multiple of {:?} octets", + INSN_SIZE + ); + } + + let mut res = vec![]; + let mut insn_ptr: usize = 0; + + while insn_ptr * INSN_SIZE < prog.len() { + let insn = get_insn(prog, insn_ptr); + res.push(insn); + insn_ptr += 1; + } + res +} diff --git a/kernel/crates/rbpf/src/helpers.rs b/kernel/crates/rbpf/src/helpers.rs new file mode 100644 index 00000000..834bf8db --- /dev/null +++ b/kernel/crates/rbpf/src/helpers.rs @@ -0,0 +1,488 @@ +// SPDX-License-Identifier: (Apache-2.0 OR MIT) +// Copyright 2015 Big Switch Networks, Inc +// (Algorithms for uBPF helpers, originally in C) +// Copyright 2016 6WIND S.A. +// (Translation to Rust, other helpers) + +//! This module implements some built-in helpers that can be called from within an eBPF program. +//! +//! These helpers may originate from several places: +//! +//! * Some of them mimic the helpers available in the Linux kernel. +//! * Some of them were proposed as example helpers in uBPF and they were adapted here. +//! * Other helpers may be specific to rbpf. +//! +//! The prototype for helpers is always the same: five `u64` as arguments, and a `u64` as a return +//! value. Hence some helpers have unused arguments, or return a 0 value in all cases, in order to +//! respect this convention. + +// Helpers associated to kernel helpers +// See also linux/include/uapi/linux/bpf.h in Linux kernel sources. + +// bpf_ktime_getns() + +/// Index of helper `bpf_ktime_getns()`, equivalent to `bpf_time_getns()`, in Linux kernel, see +/// . +pub const BPF_KTIME_GETNS_IDX: u32 = 5; + +/// Get monotonic time (since boot time) in nanoseconds. All arguments are unused. +/// +/// # Examples +/// +/// ``` +/// use rbpf::helpers; +/// +/// let t = helpers::bpf_time_getns(0, 0, 0, 0, 0); +/// let d = t / 10u64.pow(9) / 60 / 60 / 24; +/// let h = (t / 10u64.pow(9) / 60 / 60) % 24; +/// let m = (t / 10u64.pow(9) / 60 ) % 60; +/// let s = (t / 10u64.pow(9)) % 60; +/// let ns = t % 10u64.pow(9); +/// println!("Uptime: {:#x} == {} days {}:{}:{}, {} ns", t, d, h, m, s, ns); +/// ``` +#[allow(dead_code)] +#[allow(unused_variables)] +#[allow(deprecated)] +#[cfg(feature = "std")] +pub fn bpf_time_getns(unused1: u64, unused2: u64, unused3: u64, unused4: u64, unused5: u64) -> u64 { + time::precise_time_ns() +} + +// bpf_trace_printk() + +/// Index of helper `bpf_trace_printk()`, equivalent to `bpf_trace_printf()`, in Linux kernel, see +/// . +pub const BPF_TRACE_PRINTK_IDX: u32 = 6; + +/// Prints its **last three** arguments to standard output. The **first two** arguments are +/// **unused**. Returns the number of bytes written. +/// +/// By ignoring the first two arguments, it creates a helper that will have a behavior similar to +/// the one of the equivalent helper `bpf_trace_printk()` from Linux kernel. +/// +/// # Examples +/// +/// ``` +/// use rbpf::helpers; +/// +/// let res = helpers::bpf_trace_printf(0, 0, 1, 15, 32); +/// assert_eq!(res as usize, "bpf_trace_printf: 0x1, 0xf, 0x20\n".len()); +/// ``` +/// +/// This will print `bpf_trace_printf: 0x1, 0xf, 0x20`. +/// +/// The eBPF code needed to perform the call in this example would be nearly identical to the code +/// obtained by compiling the following code from C to eBPF with clang: +/// +/// ```c +/// #include +/// #include "path/to/linux/samples/bpf/bpf_helpers.h" +/// +/// int main(struct __sk_buff *skb) +/// { +/// // Only %d %u %x %ld %lu %lx %lld %llu %llx %p %s conversion specifiers allowed. +/// // See . +/// char *fmt = "bpf_trace_printk %llx, %llx, %llx\n"; +/// return bpf_trace_printk(fmt, sizeof(fmt), 1, 15, 32); +/// } +/// ``` +/// +/// This would equally print the three numbers in `/sys/kernel/debug/tracing` file each time the +/// program is run. +#[allow(dead_code)] +#[allow(unused_variables)] +#[cfg(feature = "std")] +pub fn bpf_trace_printf(unused1: u64, unused2: u64, arg3: u64, arg4: u64, arg5: u64) -> u64 { + println!("bpf_trace_printf: {arg3:#x}, {arg4:#x}, {arg5:#x}"); + let size_arg = |x| { + if x == 0 { + 1 + } else { + (x as f64).log(16.0).floor() as u64 + 1 + } + }; + "bpf_trace_printf: 0x, 0x, 0x\n".len() as u64 + size_arg(arg3) + size_arg(arg4) + size_arg(arg5) +} + +// Helpers coming from uBPF + +/// The idea is to assemble five bytes into a single `u64`. For compatibility with the helpers API, +/// each argument must be a `u64`. +/// +/// # Examples +/// +/// ``` +/// use rbpf::helpers; +/// +/// let gathered = helpers::gather_bytes(0x11, 0x22, 0x33, 0x44, 0x55); +/// assert_eq!(gathered, 0x1122334455); +/// ``` +pub fn gather_bytes(arg1: u64, arg2: u64, arg3: u64, arg4: u64, arg5: u64) -> u64 { + arg1.wrapping_shl(32) + | arg2.wrapping_shl(24) + | arg3.wrapping_shl(16) + | arg4.wrapping_shl(8) + | arg5 +} + +/// Same as `void *memfrob(void *s, size_t n);` in `string.h` in C. See the GNU manual page (in +/// section 3) for `memfrob`. The memory is directly modified, and the helper returns 0 in all +/// cases. Arguments 3 to 5 are unused. +/// +/// # Examples +/// +/// ``` +/// use rbpf::helpers; +/// +/// let val: u64 = 0x112233; +/// let val_ptr = &val as *const u64; +/// +/// helpers::memfrob(val_ptr as u64, 8, 0, 0, 0); +/// assert_eq!(val, 0x2a2a2a2a2a3b0819); +/// helpers::memfrob(val_ptr as u64, 8, 0, 0, 0); +/// assert_eq!(val, 0x112233); +/// ``` +#[allow(unused_variables)] +pub fn memfrob(ptr: u64, len: u64, unused3: u64, unused4: u64, unused5: u64) -> u64 { + for i in 0..len { + unsafe { + let mut p = (ptr + i) as *mut u8; + *p ^= 0b101010; + } + } + 0 +} + +// TODO: Try again when asm!() is available in stable Rust. +// #![feature(asm)] +// #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +// #[allow(unused_variables)] +// pub fn memfrob (ptr: u64, len: u64, arg3: u64, arg4: u64, arg5: u64) -> u64 { +// unsafe { +// asm!( +// "mov $0xf0, %rax" +// ::: "mov $0xf1, %rcx" +// ::: "mov $0xf2, %rdx" +// ::: "mov $0xf3, %rsi" +// ::: "mov $0xf4, %rdi" +// ::: "mov $0xf5, %r8" +// ::: "mov $0xf6, %r9" +// ::: "mov $0xf7, %r10" +// ::: "mov $0xf8, %r11" +// ); +// } +// 0 +// } + +/// Compute and return the square root of argument 1, cast as a float. Arguments 2 to 5 are +/// unused. +/// +/// # Examples +/// +/// ``` +/// use rbpf::helpers; +/// +/// let x = helpers::sqrti(9, 0, 0, 0, 0); +/// assert_eq!(x, 3); +/// ``` +#[allow(dead_code)] +#[allow(unused_variables)] +#[cfg(feature = "std")] // sqrt is only available when using `std` +pub fn sqrti(arg1: u64, unused2: u64, unused3: u64, unused4: u64, unused5: u64) -> u64 { + (arg1 as f64).sqrt() as u64 +} + +/// C-like `strcmp`, return 0 if the strings are equal, and a non-null value otherwise. +/// +/// # Examples +/// +/// ``` +/// use rbpf::helpers; +/// +/// let foo = "This is a string.\0".as_ptr() as u64; +/// let bar = "This is another sting.\0".as_ptr() as u64; +/// +/// assert!(helpers::strcmp(foo, foo, 0, 0, 0) == 0); +/// assert!(helpers::strcmp(foo, bar, 0, 0, 0) != 0); +/// ``` +#[allow(dead_code)] +#[allow(unused_variables)] +pub fn strcmp(arg1: u64, arg2: u64, arg3: u64, unused4: u64, unused5: u64) -> u64 { + // C-like strcmp, maybe shorter than converting the bytes to string and comparing? + if arg1 == 0 || arg2 == 0 { + return u64::MAX; + } + let mut a = arg1; + let mut b = arg2; + unsafe { + let mut a_val = *(a as *const u8); + let mut b_val = *(b as *const u8); + while a_val == b_val && a_val != 0 && b_val != 0 { + a += 1; + b += 1; + a_val = *(a as *const u8); + b_val = *(b as *const u8); + } + if a_val >= b_val { + (a_val - b_val) as u64 + } else { + (b_val - a_val) as u64 + } + } +} + +// Some additional helpers + +/// Returns a random u64 value comprised between `min` and `max` values (inclusive). Arguments 3 to +/// 5 are unused. +/// +/// Relies on `rand()` function from libc, so `libc::srand()` should be called once before this +/// helper is used. +/// +/// # Examples +/// +/// ``` +/// extern crate libc; +/// extern crate rbpf; +/// extern crate time; +/// +/// unsafe { +/// libc::srand(time::precise_time_ns() as u32) +/// } +/// +/// let n = rbpf::helpers::rand(3, 6, 0, 0, 0); +/// assert!(3 <= n && n <= 6); +/// ``` +#[allow(dead_code)] +#[allow(unused_variables)] +#[cfg(feature = "std")] +pub fn rand(min: u64, max: u64, unused3: u64, unused4: u64, unused5: u64) -> u64 { + let mut n = unsafe { (libc::rand() as u64).wrapping_shl(32) + libc::rand() as u64 }; + if min < max { + n = n % (max + 1 - min) + min; + }; + n +} +/// Prints the helper functions name and it's index. +#[cfg(feature = "std")] +pub fn show_helper() { + for (index, name) in BPF_FUNC_MAPPER.iter().enumerate() { + println!("{}:{}", index, name); + } +} + +/// See https://github.com/torvalds/linux/blob/master/include/uapi/linux/bpf.h +pub const BPF_FUNC_MAPPER: &[&str] = &[ + "unspec", + "map_lookup_elem", + "map_update_elem", + "map_delete_elem", + "probe_read", + "ktime_get_ns", + "trace_printk", + "get_prandom_u32", + "get_smp_processor_id", + "skb_store_bytes", + "l3_csum_replace", + "l4_csum_replace", + "tail_call", + "clone_redirect", + "get_current_pid_tgid", + "get_current_uid_gid", + "get_current_comm", + "get_cgroup_classid", + "skb_vlan_push", + "skb_vlan_pop", + "skb_get_tunnel_key", + "skb_set_tunnel_key", + "perf_event_read", + "redirect", + "get_route_realm", + "perf_event_output", + "skb_load_bytes", + "get_stackid", + "csum_diff", + "skb_get_tunnel_opt", + "skb_set_tunnel_opt", + "skb_change_proto", + "skb_change_type", + "skb_under_cgroup", + "get_hash_recalc", + "get_current_task", + "probe_write_user", + "current_task_under_cgroup", + "skb_change_tail", + "skb_pull_data", + "csum_update", + "set_hash_invalid", + "get_numa_node_id", + "skb_change_head", + "xdp_adjust_head", + "probe_read_str", + "get_socket_cookie", + "get_socket_uid", + "set_hash", + "setsockopt", + "skb_adjust_room", + "redirect_map", + "sk_redirect_map", + "sock_map_update", + "xdp_adjust_meta", + "perf_event_read_value", + "perf_prog_read_value", + "getsockopt", + "override_return", + "sock_ops_cb_flags_set", + "msg_redirect_map", + "msg_apply_bytes", + "msg_cork_bytes", + "msg_pull_data", + "bind", + "xdp_adjust_tail", + "skb_get_xfrm_state", + "get_stack", + "skb_load_bytes_relative", + "fib_lookup", + "sock_hash_update", + "msg_redirect_hash", + "sk_redirect_hash", + "lwt_push_encap", + "lwt_seg6_store_bytes", + "lwt_seg6_adjust_srh", + "lwt_seg6_action", + "rc_repeat", + "rc_keydown", + "skb_cgroup_id", + "get_current_cgroup_id", + "get_local_storage", + "sk_select_reuseport", + "skb_ancestor_cgroup_id", + "sk_lookup_tcp", + "sk_lookup_udp", + "sk_release", + "map_push_elem", + "map_pop_elem", + "map_peek_elem", + "msg_push_data", + "msg_pop_data", + "rc_pointer_rel", + "spin_lock", + "spin_unlock", + "sk_fullsock", + "tcp_sock", + "skb_ecn_set_ce", + "get_listener_sock", + "skc_lookup_tcp", + "tcp_check_syncookie", + "sysctl_get_name", + "sysctl_get_current_value", + "sysctl_get_new_value", + "sysctl_set_new_value", + "strtol", + "strtoul", + "sk_storage_get", + "sk_storage_delete", + "send_signal", + "tcp_gen_syncookie", + "skb_output", + "probe_read_user", + "probe_read_kernel", + "probe_read_user_str", + "probe_read_kernel_str", + "tcp_send_ack", + "send_signal_thread", + "jiffies64", + "read_branch_records", + "get_ns_current_pid_tgid", + "xdp_output", + "get_netns_cookie", + "get_current_ancestor_cgroup_id", + "sk_assign", + "ktime_get_boot_ns", + "seq_printf", + "seq_write", + "sk_cgroup_id", + "sk_ancestor_cgroup_id", + "ringbuf_output", + "ringbuf_reserve", + "ringbuf_submit", + "ringbuf_discard", + "ringbuf_query", + "csum_level", + "skc_to_tcp6_sock", + "skc_to_tcp_sock", + "skc_to_tcp_timewait_sock", + "skc_to_tcp_request_sock", + "skc_to_udp6_sock", + "get_task_stack", + "load_hdr_opt", + "store_hdr_opt", + "reserve_hdr_opt", + "inode_storage_get", + "inode_storage_delete", + "d_path", + "copy_from_user", + "snprintf_btf", + "seq_printf_btf", + "skb_cgroup_classid", + "redirect_neigh", + "per_cpu_ptr", + "this_cpu_ptr", + "redirect_peer", + "task_storage_get", + "task_storage_delete", + "get_current_task_btf", + "bprm_opts_set", + "ktime_get_coarse_ns", + "ima_inode_hash", + "sock_from_file", + "check_mtu", + "for_each_map_elem", + "snprintf", + "sys_bpf", + "btf_find_by_name_kind", + "sys_close", + "timer_init", + "timer_set_callback", + "timer_start", + "timer_cancel", + "get_func_ip", + "get_attach_cookie", + "task_pt_regs", + "get_branch_snapshot", + "trace_vprintk", + "skc_to_unix_sock", + "kallsyms_lookup_name", + "find_vma", + "loop", + "strncmp", + "get_func_arg", + "get_func_ret", + "get_func_arg_cnt", + "get_retval", + "set_retval", + "xdp_get_buff_len", + "xdp_load_bytes", + "xdp_store_bytes", + "copy_from_user_task", + "skb_set_tstamp", + "ima_file_hash", + "kptr_xchg", + "map_lookup_percpu_elem", + "skc_to_mptcp_sock", + "dynptr_from_mem", + "ringbuf_reserve_dynptr", + "ringbuf_submit_dynptr", + "ringbuf_discard_dynptr", + "dynptr_read", + "dynptr_write", + "dynptr_data", + "tcp_raw_gen_syncookie_ipv4", + "tcp_raw_gen_syncookie_ipv6", + "tcp_raw_check_syncookie_ipv4", + "tcp_raw_check_syncookie_ipv6", + "ktime_get_tai_ns", + "user_ringbuf_drain", + "cgrp_storage_get", + "cgrp_storage_delete", +]; diff --git a/kernel/crates/rbpf/src/insn_builder.rs b/kernel/crates/rbpf/src/insn_builder.rs new file mode 100644 index 00000000..2b8b6c0d --- /dev/null +++ b/kernel/crates/rbpf/src/insn_builder.rs @@ -0,0 +1,2199 @@ +// SPDX-License-Identifier: (Apache-2.0 OR MIT) +// Copyright 2017 Alex Dukhno + +//! Module provides API to create eBPF programs by Rust programming language + +use alloc::{vec, vec::Vec}; + +use crate::ebpf::*; + +/// Represents single eBPF instruction +pub trait Instruction: Sized { + /// returns instruction opt code + fn opt_code_byte(&self) -> u8; + + /// returns destination register + fn get_dst(&self) -> u8 { + self.get_insn().dst + } + + /// returns source register + fn get_src(&self) -> u8 { + self.get_insn().src + } + + /// returns offset bytes + fn get_off(&self) -> i16 { + self.get_insn().off + } + + /// returns immediate value + fn get_imm(&self) -> i32 { + self.get_insn().imm + } + + /// sets destination register + fn set_dst(mut self, dst: u8) -> Self { + self.get_insn_mut().dst = dst; + self + } + + /// sets source register + fn set_src(mut self, src: u8) -> Self { + self.get_insn_mut().src = src; + self + } + + /// sets offset bytes + fn set_off(mut self, offset: i16) -> Self { + self.get_insn_mut().off = offset; + self + } + + /// sets immediate value + fn set_imm(mut self, imm: i32) -> Self { + self.get_insn_mut().imm = imm; + self + } + + /// get `ebpf::Insn` struct + fn get_insn(&self) -> &Insn; + + /// get mutable `ebpf::Insn` struct + fn get_insn_mut(&mut self) -> &mut Insn; +} + +/// General trait for `Instruction`s and `BpfCode`. +/// Provides functionality to transform `struct` into collection of bytes +pub trait IntoBytes { + /// type of targeted transformation + type Bytes; + + /// consume `Self` with transformation into `Self::Bytes` + fn into_bytes(self) -> Self::Bytes; +} + +/// General implementation of `IntoBytes` for `Instruction` +impl<'i, I: Instruction> IntoBytes for &'i I { + type Bytes = Vec; + + /// transform immutable reference of `Instruction` into `Vec` with size of 8 + /// [ 1 byte , 1 byte , 2 bytes, 4 bytes ] + /// [ OP_CODE, SRC_REG | DST_REG, OFFSET , IMMEDIATE ] + fn into_bytes(self) -> Self::Bytes { + let buffer = vec![ + self.opt_code_byte(), + self.get_src() << 4 | self.get_dst(), + self.get_off() as u8, + (self.get_off() >> 8) as u8, + self.get_imm() as u8, + (self.get_imm() >> 8) as u8, + (self.get_imm() >> 16) as u8, + (self.get_imm() >> 24) as u8, + ]; + buffer + } +} + +/// BPF instruction stack in byte representation +#[derive(Default)] +pub struct BpfCode { + instructions: Vec, +} + +impl BpfCode { + /// creates new empty BPF instruction stack + pub fn new() -> Self { + BpfCode { + instructions: vec![], + } + } + + /// create ADD instruction + pub fn add(&mut self, source: Source, arch: Arch) -> Move { + self.mov_internal(source, arch, OpBits::Add) + } + + /// create SUB instruction + pub fn sub(&mut self, source: Source, arch: Arch) -> Move { + self.mov_internal(source, arch, OpBits::Sub) + } + + /// create MUL instruction + pub fn mul(&mut self, source: Source, arch: Arch) -> Move { + self.mov_internal(source, arch, OpBits::Mul) + } + + /// create DIV instruction + pub fn div(&mut self, source: Source, arch: Arch) -> Move { + self.mov_internal(source, arch, OpBits::Div) + } + + /// create OR instruction + pub fn bit_or(&mut self, source: Source, arch: Arch) -> Move { + self.mov_internal(source, arch, OpBits::BitOr) + } + + /// create AND instruction + pub fn bit_and(&mut self, source: Source, arch: Arch) -> Move { + self.mov_internal(source, arch, OpBits::BitAnd) + } + + /// create LSHIFT instruction + pub fn left_shift(&mut self, source: Source, arch: Arch) -> Move { + self.mov_internal(source, arch, OpBits::LShift) + } + + /// create RSHIFT instruction + pub fn right_shift(&mut self, source: Source, arch: Arch) -> Move { + self.mov_internal(source, arch, OpBits::RShift) + } + + /// create NEGATE instruction + pub fn negate(&mut self, arch: Arch) -> Move { + self.mov_internal(Source::Imm, arch, OpBits::Negate) + } + + /// create MOD instruction + pub fn modulo(&mut self, source: Source, arch: Arch) -> Move { + self.mov_internal(source, arch, OpBits::Mod) + } + + /// create XOR instruction + pub fn bit_xor(&mut self, source: Source, arch: Arch) -> Move { + self.mov_internal(source, arch, OpBits::BitXor) + } + + /// create MOV instruction + pub fn mov(&mut self, source: Source, arch: Arch) -> Move { + self.mov_internal(source, arch, OpBits::Mov) + } + + /// create SIGNED RSHIFT instruction + pub fn signed_right_shift(&mut self, source: Source, arch: Arch) -> Move { + self.mov_internal(source, arch, OpBits::SignRShift) + } + + #[inline] + fn mov_internal(&mut self, source: Source, arch_bits: Arch, op_bits: OpBits) -> Move { + Move { + bpf_code: self, + src_bit: source, + op_bits, + arch_bits, + insn: Insn { + opc: 0x00, + dst: 0x00, + src: 0x00, + off: 0x00_00, + imm: 0x00_00_00_00, + }, + } + } + + /// create byte swap instruction + pub fn swap_bytes(&mut self, endian: Endian) -> SwapBytes { + SwapBytes { + bpf_code: self, + endian, + insn: Insn { + opc: 0x00, + dst: 0x00, + src: 0x00, + off: 0x00_00, + imm: 0x00_00_00_00, + }, + } + } + + /// create LOAD instruction, IMMEDIATE is the source + pub fn load(&mut self, mem_size: MemSize) -> Load { + self.load_internal(mem_size, Addressing::Imm, BPF_LD) + } + + /// create ABSOLUTE LOAD instruction + pub fn load_abs(&mut self, mem_size: MemSize) -> Load { + self.load_internal(mem_size, Addressing::Abs, BPF_LD) + } + + /// create INDIRECT LOAD instruction + pub fn load_ind(&mut self, mem_size: MemSize) -> Load { + self.load_internal(mem_size, Addressing::Ind, BPF_LD) + } + + /// create LOAD instruction, MEMORY is the source + pub fn load_x(&mut self, mem_size: MemSize) -> Load { + self.load_internal(mem_size, Addressing::Mem, BPF_LDX) + } + + #[inline] + fn load_internal(&mut self, mem_size: MemSize, addressing: Addressing, source: u8) -> Load { + Load { + bpf_code: self, + addressing, + mem_size, + source, + insn: Insn { + opc: 0x00, + dst: 0x00, + src: 0x00, + off: 0x00_00, + imm: 0x00_00_00_00, + }, + } + } + + /// creates STORE instruction, IMMEDIATE is the source + pub fn store(&mut self, mem_size: MemSize) -> Store { + self.store_internal(mem_size, BPF_IMM) + } + + /// creates STORE instruction, MEMORY is the source + pub fn store_x(&mut self, mem_size: MemSize) -> Store { + self.store_internal(mem_size, BPF_MEM | BPF_STX) + } + + #[inline] + fn store_internal(&mut self, mem_size: MemSize, source: u8) -> Store { + Store { + bpf_code: self, + mem_size, + source, + insn: Insn { + opc: 0x00, + dst: 0x00, + src: 0x00, + off: 0x00_00, + imm: 0x00_00_00_00, + }, + } + } + + /// create unconditional JMP instruction + pub fn jump_unconditional(&mut self) -> Jump { + self.jump_conditional(Cond::Abs, Source::Imm) + } + + /// create conditional JMP instruction + pub fn jump_conditional(&mut self, cond: Cond, src_bit: Source) -> Jump { + Jump { + bpf_code: self, + cond, + src_bit, + insn: Insn { + opc: 0x00, + dst: 0x00, + src: 0x00, + off: 0x00_00, + imm: 0x00_00_00_00, + }, + } + } + + /// create CALL instruction + pub fn call(&mut self) -> FunctionCall { + FunctionCall { + bpf_code: self, + insn: Insn { + opc: 0x00, + dst: 0x00, + src: 0x00, + off: 0x00_00, + imm: 0x00_00_00_00, + }, + } + } + + /// create EXIT instruction + pub fn exit(&mut self) -> Exit { + Exit { + bpf_code: self, + insn: Insn { + opc: 0x00, + dst: 0x00, + src: 0x00, + off: 0x00_00, + imm: 0x00_00_00_00, + }, + } + } +} + +/// Transform `BpfCode` into assemble representation +impl<'a> IntoBytes for &'a BpfCode { + type Bytes = &'a [u8]; + + /// returns `BpfCode` instruction stack as `&[u8]` + fn into_bytes(self) -> Self::Bytes { + self.instructions.as_slice() + } +} + +/// struct to represent `MOV ALU` instructions +pub struct Move<'i> { + bpf_code: &'i mut BpfCode, + src_bit: Source, + op_bits: OpBits, + arch_bits: Arch, + insn: Insn, +} + +impl<'i> Move<'i> { + /// push MOV instruction into BpfCode instruction stack + pub fn push(self) -> &'i mut BpfCode { + let mut asm = self.into_bytes(); + self.bpf_code.instructions.append(&mut asm); + self.bpf_code + } +} + +impl<'i> Instruction for Move<'i> { + fn opt_code_byte(&self) -> u8 { + let op_bits = self.op_bits as u8; + let src_bit = self.src_bit as u8; + let arch_bits = self.arch_bits as u8; + op_bits | src_bit | arch_bits + } + + fn get_insn_mut(&mut self) -> &mut Insn { + &mut self.insn + } + + fn get_insn(&self) -> &Insn { + &self.insn + } +} + +#[derive(Copy, Clone, PartialEq, Eq)] +/// The source of ALU and JMP instructions +pub enum Source { + /// immediate field will be used as a source + Imm = BPF_IMM as isize, + /// src register will be used as a source + Reg = BPF_X as isize, +} + +#[derive(Copy, Clone)] +enum OpBits { + Add = BPF_ADD as isize, + Sub = BPF_SUB as isize, + Mul = BPF_MUL as isize, + Div = BPF_DIV as isize, + BitOr = BPF_OR as isize, + BitAnd = BPF_AND as isize, + LShift = BPF_LSH as isize, + RShift = BPF_RSH as isize, + Negate = BPF_NEG as isize, + Mod = BPF_MOD as isize, + BitXor = BPF_XOR as isize, + Mov = BPF_MOV as isize, + SignRShift = BPF_ARSH as isize, +} + +#[derive(Copy, Clone)] +/// Architecture of instructions +pub enum Arch { + /// 64-bit instructions + X64 = BPF_ALU64 as isize, + /// 32-bit instructions + X32 = BPF_ALU as isize, +} + +/// struct representation of byte swap operation +pub struct SwapBytes<'i> { + bpf_code: &'i mut BpfCode, + endian: Endian, + insn: Insn, +} + +impl<'i> SwapBytes<'i> { + /// push bytes swap instruction into BpfCode instruction stack + pub fn push(self) -> &'i mut BpfCode { + let mut asm = self.into_bytes(); + self.bpf_code.instructions.append(&mut asm); + self.bpf_code + } +} + +impl<'i> Instruction for SwapBytes<'i> { + fn opt_code_byte(&self) -> u8 { + self.endian as u8 + } + + fn get_insn_mut(&mut self) -> &mut Insn { + &mut self.insn + } + + fn get_insn(&self) -> &Insn { + &self.insn + } +} + +#[derive(Copy, Clone)] +/// Bytes endian +pub enum Endian { + /// Little endian + Little = LE as isize, + /// Big endian + Big = BE as isize, +} + +/// struct representation of LOAD instructions +pub struct Load<'i> { + bpf_code: &'i mut BpfCode, + addressing: Addressing, + mem_size: MemSize, + source: u8, + insn: Insn, +} + +impl<'i> Load<'i> { + /// push LOAD instruction into BpfCode instruction stack + pub fn push(self) -> &'i mut BpfCode { + let mut asm = self.into_bytes(); + self.bpf_code.instructions.append(&mut asm); + self.bpf_code + } +} + +impl<'i> Instruction for Load<'i> { + fn opt_code_byte(&self) -> u8 { + let size = self.mem_size as u8; + let addressing = self.addressing as u8; + addressing | size | self.source + } + + fn get_insn_mut(&mut self) -> &mut Insn { + &mut self.insn + } + + fn get_insn(&self) -> &Insn { + &self.insn + } +} + +/// struct representation of STORE instructions +pub struct Store<'i> { + bpf_code: &'i mut BpfCode, + mem_size: MemSize, + source: u8, + insn: Insn, +} + +impl<'i> Store<'i> { + /// push STORE instruction into BpfCode instruction stack + pub fn push(self) -> &'i mut BpfCode { + let mut asm = self.into_bytes(); + self.bpf_code.instructions.append(&mut asm); + self.bpf_code + } +} + +impl<'i> Instruction for Store<'i> { + fn opt_code_byte(&self) -> u8 { + let size = self.mem_size as u8; + BPF_MEM | BPF_ST | size | self.source + } + + fn get_insn_mut(&mut self) -> &mut Insn { + &mut self.insn + } + + fn get_insn(&self) -> &Insn { + &self.insn + } +} + +#[derive(Copy, Clone)] +/// Memory size for LOAD and STORE instructions +pub enum MemSize { + /// 8-bit size + Byte = BPF_B as isize, + /// 16-bit size + HalfWord = BPF_H as isize, + /// 32-bit size + Word = BPF_W as isize, + /// 64-bit size + DoubleWord = BPF_DW as isize, +} + +#[derive(Copy, Clone)] +enum Addressing { + Imm = BPF_IMM as isize, + Abs = BPF_ABS as isize, + Ind = BPF_IND as isize, + Mem = BPF_MEM as isize, +} + +/// struct representation of JMP instructions +pub struct Jump<'i> { + bpf_code: &'i mut BpfCode, + cond: Cond, + src_bit: Source, + insn: Insn, +} + +impl<'i> Jump<'i> { + /// push JMP instruction into BpfCode instruction stack + pub fn push(self) -> &'i mut BpfCode { + let mut asm = self.into_bytes(); + self.bpf_code.instructions.append(&mut asm); + self.bpf_code + } +} + +impl<'i> Instruction for Jump<'i> { + fn opt_code_byte(&self) -> u8 { + let cmp: u8 = self.cond as u8; + let src_bit = self.src_bit as u8; + cmp | src_bit | BPF_JMP + } + + fn get_insn_mut(&mut self) -> &mut Insn { + &mut self.insn + } + + fn get_insn(&self) -> &Insn { + &self.insn + } +} + +#[derive(Copy, Clone, PartialEq, Eq)] +/// Conditions for JMP instructions +pub enum Cond { + /// Absolute or unconditional + Abs = BPF_JA as isize, + /// Jump if `==` + Equals = BPF_JEQ as isize, + /// Jump if `>` + Greater = BPF_JGT as isize, + /// Jump if `>=` + GreaterEquals = BPF_JGE as isize, + /// Jump if `<` + Lower = BPF_JLT as isize, + /// Jump if `<=` + LowerEquals = BPF_JLE as isize, + /// Jump if `src` & `dst` + BitAnd = BPF_JSET as isize, + /// Jump if `!=` + NotEquals = BPF_JNE as isize, + /// Jump if `>` (signed) + GreaterSigned = BPF_JSGT as isize, + /// Jump if `>=` (signed) + GreaterEqualsSigned = BPF_JSGE as isize, + /// Jump if `<` (signed) + LowerSigned = BPF_JSLT as isize, + /// Jump if `<=` (signed) + LowerEqualsSigned = BPF_JSLE as isize, +} + +/// struct representation of CALL instruction +pub struct FunctionCall<'i> { + bpf_code: &'i mut BpfCode, + insn: Insn, +} + +impl<'i> FunctionCall<'i> { + /// push CALL instruction into BpfCode instruction stack + pub fn push(self) -> &'i mut BpfCode { + let mut asm = self.into_bytes(); + self.bpf_code.instructions.append(&mut asm); + self.bpf_code + } +} + +impl<'i> Instruction for FunctionCall<'i> { + fn opt_code_byte(&self) -> u8 { + BPF_CALL | BPF_JMP + } + + fn get_insn_mut(&mut self) -> &mut Insn { + &mut self.insn + } + + fn get_insn(&self) -> &Insn { + &self.insn + } +} + +/// struct representation of EXIT instruction +pub struct Exit<'i> { + bpf_code: &'i mut BpfCode, + insn: Insn, +} + +impl<'i> Exit<'i> { + /// push EXIT instruction into BpfCode instruction stack + pub fn push(self) -> &'i mut BpfCode { + let mut asm = self.into_bytes(); + self.bpf_code.instructions.append(&mut asm); + self.bpf_code + } +} + +impl<'i> Instruction for Exit<'i> { + fn opt_code_byte(&self) -> u8 { + BPF_EXIT | BPF_JMP + } + + fn get_insn_mut(&mut self) -> &mut Insn { + &mut self.insn + } + + fn get_insn(&self) -> &Insn { + &self.insn + } +} + +#[cfg(test)] +mod tests { + #[cfg(test)] + mod special { + use super::super::*; + + #[test] + fn call_immediate() { + let mut program = BpfCode::new(); + program.call().set_imm(0x11_22_33_44).push(); + + assert_eq!( + program.into_bytes(), + &[0x85, 0x00, 0x00, 0x00, 0x44, 0x33, 0x22, 0x11] + ); + } + + #[test] + fn exit_operation() { + let mut program = BpfCode::new(); + program.exit().push(); + + assert_eq!( + program.into_bytes(), + &[0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + } + + #[cfg(test)] + mod jump_instructions { + #[cfg(test)] + mod register { + use super::super::super::*; + + #[test] + fn jump_on_dst_equals_src() { + let mut program = BpfCode::new(); + program + .jump_conditional(Cond::Equals, Source::Reg) + .set_dst(0x01) + .set_src(0x02) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x1d, 0x21, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn jump_on_dst_greater_than_src() { + let mut program = BpfCode::new(); + program + .jump_conditional(Cond::Greater, Source::Reg) + .set_dst(0x03) + .set_src(0x02) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x2d, 0x23, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn jump_on_dst_greater_or_equals_to_src() { + let mut program = BpfCode::new(); + program + .jump_conditional(Cond::GreaterEquals, Source::Reg) + .set_dst(0x04) + .set_src(0x01) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x3d, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn jump_on_dst_lower_than_src() { + let mut program = BpfCode::new(); + program + .jump_conditional(Cond::Lower, Source::Reg) + .set_dst(0x03) + .set_src(0x02) + .push(); + + assert_eq!( + program.into_bytes(), + &[0xad, 0x23, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn jump_on_dst_lower_or_equals_to_src() { + let mut program = BpfCode::new(); + program + .jump_conditional(Cond::LowerEquals, Source::Reg) + .set_dst(0x04) + .set_src(0x01) + .push(); + + assert_eq!( + program.into_bytes(), + &[0xbd, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn jump_on_dst_bit_and_with_src_not_equal_zero() { + let mut program = BpfCode::new(); + program + .jump_conditional(Cond::BitAnd, Source::Reg) + .set_dst(0x05) + .set_src(0x02) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x4d, 0x25, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn jump_on_dst_not_equals_src() { + let mut program = BpfCode::new(); + program + .jump_conditional(Cond::NotEquals, Source::Reg) + .set_dst(0x03) + .set_src(0x05) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x5d, 0x53, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn jump_on_dst_greater_than_src_signed() { + let mut program = BpfCode::new(); + program + .jump_conditional(Cond::GreaterSigned, Source::Reg) + .set_dst(0x04) + .set_src(0x01) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x6d, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn jump_on_dst_greater_or_equals_src_signed() { + let mut program = BpfCode::new(); + program + .jump_conditional(Cond::GreaterEqualsSigned, Source::Reg) + .set_dst(0x01) + .set_src(0x03) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x7d, 0x31, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn jump_on_dst_lower_than_src_signed() { + let mut program = BpfCode::new(); + program + .jump_conditional(Cond::LowerSigned, Source::Reg) + .set_dst(0x04) + .set_src(0x01) + .push(); + + assert_eq!( + program.into_bytes(), + &[0xcd, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn jump_on_dst_lower_or_equals_src_signed() { + let mut program = BpfCode::new(); + program + .jump_conditional(Cond::LowerEqualsSigned, Source::Reg) + .set_dst(0x01) + .set_src(0x03) + .push(); + + assert_eq!( + program.into_bytes(), + &[0xdd, 0x31, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + } + + #[cfg(test)] + mod immediate { + use super::super::super::*; + + #[test] + fn jump_to_label() { + let mut program = BpfCode::new(); + program.jump_unconditional().set_off(0x00_11).push(); + + assert_eq!( + program.into_bytes(), + &[0x05, 0x00, 0x11, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn jump_on_dst_equals_const() { + let mut program = BpfCode::new(); + program + .jump_conditional(Cond::Equals, Source::Imm) + .set_dst(0x01) + .set_imm(0x00_11_22_33) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x15, 0x01, 0x00, 0x00, 0x33, 0x22, 0x11, 0x00] + ); + } + + #[test] + fn jump_on_dst_greater_than_const() { + let mut program = BpfCode::new(); + program + .jump_conditional(Cond::Greater, Source::Imm) + .set_dst(0x02) + .set_imm(0x00_11_00_11) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x25, 0x02, 0x00, 0x00, 0x11, 0x00, 0x11, 0x00] + ); + } + + #[test] + fn jump_on_dst_greater_or_equals_to_const() { + let mut program = BpfCode::new(); + program + .jump_conditional(Cond::GreaterEquals, Source::Imm) + .set_dst(0x04) + .set_imm(0x00_22_11_00) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x35, 0x04, 0x00, 0x00, 0x00, 0x11, 0x22, 0x00] + ); + } + + #[test] + fn jump_on_dst_lower_than_const() { + let mut program = BpfCode::new(); + program + .jump_conditional(Cond::Lower, Source::Imm) + .set_dst(0x02) + .set_imm(0x00_11_00_11) + .push(); + + assert_eq!( + program.into_bytes(), + &[0xa5, 0x02, 0x00, 0x00, 0x11, 0x00, 0x11, 0x00] + ); + } + + #[test] + fn jump_on_dst_lower_or_equals_to_const() { + let mut program = BpfCode::new(); + program + .jump_conditional(Cond::LowerEquals, Source::Imm) + .set_dst(0x04) + .set_imm(0x00_22_11_00) + .push(); + + assert_eq!( + program.into_bytes(), + &[0xb5, 0x04, 0x00, 0x00, 0x00, 0x11, 0x22, 0x00] + ); + } + + #[test] + fn jump_on_dst_bit_and_with_const_not_equal_zero() { + let mut program = BpfCode::new(); + program + .jump_conditional(Cond::BitAnd, Source::Imm) + .set_dst(0x05) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x45, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn jump_on_dst_not_equals_const() { + let mut program = BpfCode::new(); + program + .jump_conditional(Cond::NotEquals, Source::Imm) + .set_dst(0x03) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x55, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn jump_on_dst_greater_than_const_signed() { + let mut program = BpfCode::new(); + program + .jump_conditional(Cond::GreaterSigned, Source::Imm) + .set_dst(0x04) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x65, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn jump_on_dst_greater_or_equals_src_signed() { + let mut program = BpfCode::new(); + program + .jump_conditional(Cond::GreaterEqualsSigned, Source::Imm) + .set_dst(0x01) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x75, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn jump_on_dst_lower_than_const_signed() { + let mut program = BpfCode::new(); + program + .jump_conditional(Cond::LowerSigned, Source::Imm) + .set_dst(0x04) + .push(); + + assert_eq!( + program.into_bytes(), + &[0xc5, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn jump_on_dst_lower_or_equals_src_signed() { + let mut program = BpfCode::new(); + program + .jump_conditional(Cond::LowerEqualsSigned, Source::Imm) + .set_dst(0x01) + .push(); + + assert_eq!( + program.into_bytes(), + &[0xd5, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + } + } + + #[cfg(test)] + mod store_instructions { + use super::super::*; + + #[test] + fn store_word_from_dst_into_immediate_address() { + let mut program = BpfCode::new(); + program + .store(MemSize::Word) + .set_dst(0x01) + .set_off(0x00_11) + .set_imm(0x11_22_33_44) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x62, 0x01, 0x11, 0x00, 0x44, 0x33, 0x22, 0x11] + ); + } + + #[test] + fn store_half_word_from_dst_into_immediate_address() { + let mut program = BpfCode::new(); + program + .store(MemSize::HalfWord) + .set_dst(0x02) + .set_off(0x11_22) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x6a, 0x02, 0x22, 0x11, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn store_byte_from_dst_into_immediate_address() { + let mut program = BpfCode::new(); + program.store(MemSize::Byte).push(); + + assert_eq!( + program.into_bytes(), + &[0x72, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn store_double_word_from_dst_into_immediate_address() { + let mut program = BpfCode::new(); + program.store(MemSize::DoubleWord).push(); + + assert_eq!( + program.into_bytes(), + &[0x7a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn store_word_from_dst_into_src_address() { + let mut program = BpfCode::new(); + program + .store_x(MemSize::Word) + .set_dst(0x01) + .set_src(0x02) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x63, 0x21, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn store_half_word_from_dst_into_src_address() { + let mut program = BpfCode::new(); + program.store_x(MemSize::HalfWord).push(); + + assert_eq!( + program.into_bytes(), + &[0x6b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn store_byte_from_dst_into_src_address() { + let mut program = BpfCode::new(); + program.store_x(MemSize::Byte).push(); + + assert_eq!( + program.into_bytes(), + &[0x73, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn store_double_word_from_dst_into_src_address() { + let mut program = BpfCode::new(); + program.store_x(MemSize::DoubleWord).push(); + + assert_eq!( + program.into_bytes(), + &[0x7b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + } + + #[cfg(test)] + mod load_instructions { + #[cfg(test)] + mod register { + use super::super::super::*; + + #[test] + fn load_word_from_set_src_with_offset() { + let mut program = BpfCode::new(); + program + .load_x(MemSize::Word) + .set_dst(0x01) + .set_src(0x02) + .set_off(0x00_02) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x61, 0x21, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn load_half_word_from_set_src_with_offset() { + let mut program = BpfCode::new(); + program + .load_x(MemSize::HalfWord) + .set_dst(0x02) + .set_src(0x01) + .set_off(0x11_22) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x69, 0x12, 0x22, 0x11, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn load_byte_from_set_src_with_offset() { + let mut program = BpfCode::new(); + program + .load_x(MemSize::Byte) + .set_dst(0x01) + .set_src(0x04) + .set_off(0x00_11) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x71, 0x41, 0x11, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn load_double_word_from_set_src_with_offset() { + let mut program = BpfCode::new(); + program + .load_x(MemSize::DoubleWord) + .set_dst(0x04) + .set_src(0x05) + .set_off(0x44_55) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x79, 0x54, 0x55, 0x44, 0x00, 0x00, 0x00, 0x00] + ); + } + } + + #[cfg(test)] + mod immediate { + use super::super::super::*; + + #[test] + fn load_double_word() { + let mut program = BpfCode::new(); + program + .load(MemSize::DoubleWord) + .set_dst(0x01) + .set_imm(0x00_01_02_03) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x18, 0x01, 0x00, 0x00, 0x03, 0x02, 0x01, 0x00] + ); + } + + #[test] + fn load_abs_word() { + let mut program = BpfCode::new(); + program.load_abs(MemSize::Word).push(); + + assert_eq!( + program.into_bytes(), + &[0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn load_abs_half_word() { + let mut program = BpfCode::new(); + program.load_abs(MemSize::HalfWord).set_dst(0x05).push(); + + assert_eq!( + program.into_bytes(), + &[0x28, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn load_abs_byte() { + let mut program = BpfCode::new(); + program.load_abs(MemSize::Byte).set_dst(0x01).push(); + + assert_eq!( + program.into_bytes(), + &[0x30, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn load_abs_double_word() { + let mut program = BpfCode::new(); + program + .load_abs(MemSize::DoubleWord) + .set_dst(0x01) + .set_imm(0x01_02_03_04) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x38, 0x01, 0x00, 0x00, 0x04, 0x03, 0x02, 0x01] + ); + } + + #[test] + fn load_indirect_word() { + let mut program = BpfCode::new(); + program.load_ind(MemSize::Word).push(); + + assert_eq!( + program.into_bytes(), + &[0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn load_indirect_half_word() { + let mut program = BpfCode::new(); + program.load_ind(MemSize::HalfWord).push(); + + assert_eq!( + program.into_bytes(), + &[0x48, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn load_indirect_byte() { + let mut program = BpfCode::new(); + program.load_ind(MemSize::Byte).push(); + + assert_eq!( + program.into_bytes(), + &[0x50, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn load_indirect_double_word() { + let mut program = BpfCode::new(); + program.load_ind(MemSize::DoubleWord).push(); + + assert_eq!( + program.into_bytes(), + &[0x58, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + } + } + + #[cfg(test)] + mod byte_swap_instructions { + use super::super::*; + + #[test] + fn convert_host_to_little_endian_16bits() { + let mut program = BpfCode::new(); + program + .swap_bytes(Endian::Little) + .set_dst(0x01) + .set_imm(0x00_00_00_10) + .push(); + + assert_eq!( + program.into_bytes(), + &[0xd4, 0x01, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn convert_host_to_little_endian_32bits() { + let mut program = BpfCode::new(); + program + .swap_bytes(Endian::Little) + .set_dst(0x02) + .set_imm(0x00_00_00_20) + .push(); + + assert_eq!( + program.into_bytes(), + &[0xd4, 0x02, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn convert_host_to_little_endian_64bit() { + let mut program = BpfCode::new(); + program + .swap_bytes(Endian::Little) + .set_dst(0x03) + .set_imm(0x00_00_00_40) + .push(); + + assert_eq!( + program.into_bytes(), + &[0xd4, 0x03, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn convert_host_to_big_endian_16bits() { + let mut program = BpfCode::new(); + program + .swap_bytes(Endian::Big) + .set_dst(0x01) + .set_imm(0x00_00_00_10) + .push(); + + assert_eq!( + program.into_bytes(), + &[0xdc, 0x01, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn convert_host_to_big_endian_32bits() { + let mut program = BpfCode::new(); + program + .swap_bytes(Endian::Big) + .set_dst(0x02) + .set_imm(0x00_00_00_20) + .push(); + + assert_eq!( + program.into_bytes(), + &[0xdc, 0x02, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn convert_host_to_big_endian_64bit() { + let mut program = BpfCode::new(); + program + .swap_bytes(Endian::Big) + .set_dst(0x03) + .set_imm(0x00_00_00_40) + .push(); + + assert_eq!( + program.into_bytes(), + &[0xdc, 0x03, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00] + ); + } + } + + #[cfg(test)] + mod moves_instructions { + #[cfg(test)] + mod arch_x64 { + #[cfg(test)] + mod immediate { + use super::super::super::super::*; + + #[test] + fn move_and_add_const_to_register() { + let mut program = BpfCode::new(); + program + .add(Source::Imm, Arch::X64) + .set_dst(0x02) + .set_imm(0x01_02_03_04) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x07, 0x02, 0x00, 0x00, 0x04, 0x03, 0x02, 0x01] + ); + } + + #[test] + fn move_sub_const_to_register() { + let mut program = BpfCode::new(); + program + .sub(Source::Imm, Arch::X64) + .set_dst(0x04) + .set_imm(0x00_01_02_03) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x17, 0x04, 0x00, 0x00, 0x03, 0x02, 0x01, 0x00] + ); + } + + #[test] + fn move_mul_const_to_register() { + let mut program = BpfCode::new(); + program + .mul(Source::Imm, Arch::X64) + .set_dst(0x05) + .set_imm(0x04_03_02_01) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x27, 0x05, 0x00, 0x00, 0x01, 0x02, 0x03, 0x04] + ); + } + + #[test] + fn move_div_constant_to_register() { + let mut program = BpfCode::new(); + program + .div(Source::Imm, Arch::X64) + .set_dst(0x02) + .set_imm(0x00_ff_00_ff) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x37, 0x02, 0x00, 0x00, 0xff, 0x00, 0xff, 0x00] + ); + } + + #[test] + fn move_bit_or_const_to_register() { + let mut program = BpfCode::new(); + program + .bit_or(Source::Imm, Arch::X64) + .set_dst(0x02) + .set_imm(0x00_11_00_22) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x47, 0x02, 0x00, 0x00, 0x22, 0x00, 0x11, 0x00] + ); + } + + #[test] + fn move_bit_and_const_to_register() { + let mut program = BpfCode::new(); + program + .bit_and(Source::Imm, Arch::X64) + .set_dst(0x02) + .set_imm(0x11_22_33_44) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x57, 0x02, 0x00, 0x00, 0x44, 0x33, 0x22, 0x11] + ); + } + + #[test] + fn move_left_shift_const_to_register() { + let mut program = BpfCode::new(); + program + .left_shift(Source::Imm, Arch::X64) + .set_dst(0x01) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x67, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn move_logical_right_shift_const_to_register() { + let mut program = BpfCode::new(); + program + .right_shift(Source::Imm, Arch::X64) + .set_dst(0x01) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x77, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn move_negate_register() { + let mut program = BpfCode::new(); + program.negate(Arch::X64).set_dst(0x02).push(); + + assert_eq!( + program.into_bytes(), + &[0x87, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn move_mod_const_to_register() { + let mut program = BpfCode::new(); + program.modulo(Source::Imm, Arch::X64).set_dst(0x02).push(); + + assert_eq!( + program.into_bytes(), + &[0x97, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn move_bit_xor_const_to_register() { + let mut program = BpfCode::new(); + program.bit_xor(Source::Imm, Arch::X64).set_dst(0x03).push(); + + assert_eq!( + program.into_bytes(), + &[0xa7, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn move_const_to_register() { + let mut program = BpfCode::new(); + program + .mov(Source::Imm, Arch::X64) + .set_dst(0x01) + .set_imm(0x00_00_00_FF) + .push(); + + assert_eq!( + program.into_bytes(), + &[0xb7, 0x01, 0x00, 0x00, 0xff, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn move_signed_right_shift_const_to_register() { + let mut program = BpfCode::new(); + program + .signed_right_shift(Source::Imm, Arch::X64) + .set_dst(0x05) + .push(); + + assert_eq!( + program.into_bytes(), + &[0xc7, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + } + + #[cfg(test)] + mod register { + use super::super::super::super::*; + + #[test] + fn move_and_add_from_register() { + let mut program = BpfCode::new(); + program + .add(Source::Reg, Arch::X64) + .set_dst(0x03) + .set_src(0x02) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x0f, 0x23, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn move_sub_from_register_to_register() { + let mut program = BpfCode::new(); + program + .sub(Source::Reg, Arch::X64) + .set_dst(0x03) + .set_src(0x04) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x1f, 0x43, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn move_mul_from_register_to_register() { + let mut program = BpfCode::new(); + program + .mul(Source::Reg, Arch::X64) + .set_dst(0x04) + .set_src(0x03) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x2f, 0x34, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn move_div_from_register_to_register() { + let mut program = BpfCode::new(); + program + .div(Source::Reg, Arch::X64) + .set_dst(0x01) + .set_src(0x00) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x3f, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn move_bit_or_from_register_to_register() { + let mut program = BpfCode::new(); + program + .bit_or(Source::Reg, Arch::X64) + .set_dst(0x03) + .set_src(0x01) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x4f, 0x13, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn move_bit_and_from_register_to_register() { + let mut program = BpfCode::new(); + program + .bit_and(Source::Reg, Arch::X64) + .set_dst(0x03) + .set_src(0x02) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x5f, 0x23, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn move_left_shift_from_register_to_register() { + let mut program = BpfCode::new(); + program + .left_shift(Source::Reg, Arch::X64) + .set_dst(0x02) + .set_src(0x03) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x6f, 0x32, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn move_logical_right_shift_from_register_to_register() { + let mut program = BpfCode::new(); + program + .right_shift(Source::Reg, Arch::X64) + .set_dst(0x02) + .set_src(0x04) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x7f, 0x42, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn move_mod_from_register_to_register() { + let mut program = BpfCode::new(); + program + .modulo(Source::Reg, Arch::X64) + .set_dst(0x01) + .set_src(0x02) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x9f, 0x21, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn move_bit_xor_from_register_to_register() { + let mut program = BpfCode::new(); + program + .bit_xor(Source::Reg, Arch::X64) + .set_dst(0x02) + .set_src(0x04) + .push(); + + assert_eq!( + program.into_bytes(), + &[0xaf, 0x42, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn move_from_register_to_another_register() { + let mut program = BpfCode::new(); + program.mov(Source::Reg, Arch::X64).set_src(0x01).push(); + + assert_eq!( + program.into_bytes(), + &[0xbf, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn move_signed_right_shift_from_register_to_register() { + let mut program = BpfCode::new(); + program + .signed_right_shift(Source::Reg, Arch::X64) + .set_dst(0x02) + .set_src(0x03) + .push(); + + assert_eq!( + program.into_bytes(), + &[0xcf, 0x32, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + } + } + + #[cfg(test)] + mod arch_x32 { + #[cfg(test)] + mod immediate { + use super::super::super::super::*; + + #[test] + fn move_and_add_const_to_register() { + let mut program = BpfCode::new(); + program + .add(Source::Imm, Arch::X32) + .set_dst(0x02) + .set_imm(0x01_02_03_04) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x04, 0x02, 0x00, 0x00, 0x04, 0x03, 0x02, 0x01] + ); + } + + #[test] + fn move_sub_const_to_register() { + let mut program = BpfCode::new(); + program + .sub(Source::Imm, Arch::X32) + .set_dst(0x04) + .set_imm(0x00_01_02_03) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x14, 0x04, 0x00, 0x00, 0x03, 0x02, 0x01, 0x00] + ); + } + + #[test] + fn move_mul_const_to_register() { + let mut program = BpfCode::new(); + program + .mul(Source::Imm, Arch::X32) + .set_dst(0x05) + .set_imm(0x04_03_02_01) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x24, 0x05, 0x00, 0x00, 0x01, 0x02, 0x03, 0x04] + ); + } + + #[test] + fn move_div_constant_to_register() { + let mut program = BpfCode::new(); + program + .div(Source::Imm, Arch::X32) + .set_dst(0x02) + .set_imm(0x00_ff_00_ff) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x34, 0x02, 0x00, 0x00, 0xff, 0x00, 0xff, 0x00] + ); + } + + #[test] + fn move_bit_or_const_to_register() { + let mut program = BpfCode::new(); + program + .bit_or(Source::Imm, Arch::X32) + .set_dst(0x02) + .set_imm(0x00_11_00_22) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x44, 0x02, 0x00, 0x00, 0x22, 0x00, 0x11, 0x00] + ); + } + + #[test] + fn move_bit_and_const_to_register() { + let mut program = BpfCode::new(); + program + .bit_and(Source::Imm, Arch::X32) + .set_dst(0x02) + .set_imm(0x11_22_33_44) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x54, 0x02, 0x00, 0x00, 0x44, 0x33, 0x22, 0x11] + ); + } + + #[test] + fn move_left_shift_const_to_register() { + let mut program = BpfCode::new(); + program + .left_shift(Source::Imm, Arch::X32) + .set_dst(0x01) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x64, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn move_logical_right_shift_const_to_register() { + let mut program = BpfCode::new(); + program + .right_shift(Source::Imm, Arch::X32) + .set_dst(0x01) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x74, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn move_negate_register() { + let mut program = BpfCode::new(); + program.negate(Arch::X32).set_dst(0x02).push(); + + assert_eq!( + program.into_bytes(), + &[0x84, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn move_mod_const_to_register() { + let mut program = BpfCode::new(); + program.modulo(Source::Imm, Arch::X32).set_dst(0x02).push(); + + assert_eq!( + program.into_bytes(), + &[0x94, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn move_bit_xor_const_to_register() { + let mut program = BpfCode::new(); + program.bit_xor(Source::Imm, Arch::X32).set_dst(0x03).push(); + + assert_eq!( + program.into_bytes(), + &[0xa4, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn move_const_to_register() { + let mut program = BpfCode::new(); + program + .mov(Source::Imm, Arch::X32) + .set_dst(0x01) + .set_imm(0x00_00_00_FF) + .push(); + + assert_eq!( + program.into_bytes(), + &[0xb4, 0x01, 0x00, 0x00, 0xff, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn move_signed_right_shift_const_to_register() { + let mut program = BpfCode::new(); + program + .signed_right_shift(Source::Imm, Arch::X32) + .set_dst(0x05) + .push(); + + assert_eq!( + program.into_bytes(), + &[0xc4, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + } + + #[cfg(test)] + mod register { + use super::super::super::super::*; + + #[test] + fn move_and_add_from_register() { + let mut program = BpfCode::new(); + program + .add(Source::Reg, Arch::X32) + .set_dst(0x03) + .set_src(0x02) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x0c, 0x23, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn move_sub_from_register_to_register() { + let mut program = BpfCode::new(); + program + .sub(Source::Reg, Arch::X32) + .set_dst(0x03) + .set_src(0x04) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x1c, 0x43, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn move_mul_from_register_to_register() { + let mut program = BpfCode::new(); + program + .mul(Source::Reg, Arch::X32) + .set_dst(0x04) + .set_src(0x03) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x2c, 0x34, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn move_div_from_register_to_register() { + let mut program = BpfCode::new(); + program + .div(Source::Reg, Arch::X32) + .set_dst(0x01) + .set_src(0x00) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x3c, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn move_bit_or_from_register_to_register() { + let mut program = BpfCode::new(); + program + .bit_or(Source::Reg, Arch::X32) + .set_dst(0x03) + .set_src(0x01) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x4c, 0x13, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn move_bit_and_from_register_to_register() { + let mut program = BpfCode::new(); + program + .bit_and(Source::Reg, Arch::X32) + .set_dst(0x03) + .set_src(0x02) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x5c, 0x23, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn move_left_shift_from_register_to_register() { + let mut program = BpfCode::new(); + program + .left_shift(Source::Reg, Arch::X32) + .set_dst(0x02) + .set_src(0x03) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x6c, 0x32, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn move_logical_right_shift_from_register_to_register() { + let mut program = BpfCode::new(); + program + .right_shift(Source::Reg, Arch::X32) + .set_dst(0x02) + .set_src(0x04) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x7c, 0x42, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn move_mod_from_register_to_register() { + let mut program = BpfCode::new(); + program + .modulo(Source::Reg, Arch::X32) + .set_dst(0x01) + .set_src(0x02) + .push(); + + assert_eq!( + program.into_bytes(), + &[0x9c, 0x21, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn move_bit_xor_from_register_to_register() { + let mut program = BpfCode::new(); + program + .bit_xor(Source::Reg, Arch::X32) + .set_dst(0x02) + .set_src(0x04) + .push(); + + assert_eq!( + program.into_bytes(), + &[0xac, 0x42, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn move_from_register_to_another_register() { + let mut program = BpfCode::new(); + program + .mov(Source::Reg, Arch::X32) + .set_dst(0x00) + .set_src(0x01) + .push(); + + assert_eq!( + program.into_bytes(), + &[0xbc, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + + #[test] + fn move_signed_right_shift_from_register_to_register() { + let mut program = BpfCode::new(); + program + .signed_right_shift(Source::Reg, Arch::X32) + .set_dst(0x02) + .set_src(0x03) + .push(); + + assert_eq!( + program.into_bytes(), + &[0xcc, 0x32, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + ); + } + } + } + } + + #[cfg(test)] + mod programs { + use super::super::*; + + #[test] + fn example_from_assembler() { + let mut program = BpfCode::new(); + program + .add(Source::Imm, Arch::X64) + .set_dst(1) + .set_imm(0x605) + .push() + .mov(Source::Imm, Arch::X64) + .set_dst(2) + .set_imm(0x32) + .push() + .mov(Source::Reg, Arch::X64) + .set_src(0) + .set_dst(1) + .push() + .swap_bytes(Endian::Big) + .set_dst(0) + .set_imm(0x10) + .push() + .negate(Arch::X64) + .set_dst(2) + .push() + .exit() + .push(); + + let bytecode = program.into_bytes(); + let ref_prog = &[ + 0x07, 0x01, 0x00, 0x00, 0x05, 0x06, 0x00, 0x00, 0xb7, 0x02, 0x00, 0x00, 0x32, 0x00, + 0x00, 0x00, 0xbf, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xdc, 0x00, 0x00, 0x00, + 0x10, 0x00, 0x00, 0x00, 0x87, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x95, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + ]; + // cargo says: "`[{integer}; 48]` cannot be formatted using `{:?}` + // because it doesn't implement `std::fmt::Debug`" + // So let's check in two steps. + assert_eq!(bytecode[..32], ref_prog[..32]); + assert_eq!(bytecode[33..], ref_prog[33..]); + } + } +} diff --git a/kernel/crates/rbpf/src/interpreter.rs b/kernel/crates/rbpf/src/interpreter.rs new file mode 100644 index 00000000..cb4bddf3 --- /dev/null +++ b/kernel/crates/rbpf/src/interpreter.rs @@ -0,0 +1,708 @@ +// SPDX-License-Identifier: (Apache-2.0 OR MIT) +// Derived from uBPF +// Copyright 2015 Big Switch Networks, Inc +// (uBPF: VM architecture, parts of the interpreter, originally in C) +// Copyright 2016 6WIND S.A. +// (Translation to Rust, MetaBuff/multiple classes addition, hashmaps for helpers) + +use crate::{ + ebpf::{self, Insn}, + helpers::BPF_FUNC_MAPPER, + stack::StackFrame, + *, +}; + +#[cfg(not(feature = "user"))] +#[allow(unused)] +fn check_mem( + addr: u64, + len: usize, + access_type: &str, + insn_ptr: usize, + mbuff: &[u8], + mem: &[u8], + stack: &[u8], +) -> Result<(), Error> { + log::trace!( + "check_mem: addr {:#x}, len {}, access_type {}, insn_ptr {}", + addr, + len, + access_type, + insn_ptr + ); + log::trace!( + "check_mem: mbuff: {:#x}/{:#x}, mem: {:#x}/{:#x}, stack: {:#x}/{:#x}", + mbuff.as_ptr() as u64, + mbuff.len(), + mem.as_ptr() as u64, + mem.len(), + stack.as_ptr() as u64, + stack.len() + ); + Ok(()) +} + +#[cfg(feature = "user")] +fn check_mem( + addr: u64, + len: usize, + access_type: &str, + insn_ptr: usize, + mbuff: &[u8], + mem: &[u8], + stack: &[u8], +) -> Result<(), Error> { + if let Some(addr_end) = addr.checked_add(len as u64) { + if mbuff.as_ptr() as u64 <= addr && addr_end <= mbuff.as_ptr() as u64 + mbuff.len() as u64 { + return Ok(()); + } + if mem.as_ptr() as u64 <= addr && addr_end <= mem.as_ptr() as u64 + mem.len() as u64 { + return Ok(()); + } + if stack.as_ptr() as u64 <= addr && addr_end <= stack.as_ptr() as u64 + stack.len() as u64 { + return Ok(()); + } + } + + Err(Error::new(ErrorKind::Other, format!( + "Error: out of bounds memory {} (insn #{:?}), addr {:#x}, size {:?}\nmbuff: {:#x}/{:#x}, mem: {:#x}/{:#x}, stack: {:#x}/{:#x}", + access_type, insn_ptr, addr, len, + mbuff.as_ptr() as u64, mbuff.len(), + mem.as_ptr() as u64, mem.len(), + stack.as_ptr() as u64, stack.len() + ))) +} + +#[inline] +fn do_jump(insn_ptr: &mut usize, insn: &Insn) { + *insn_ptr = (*insn_ptr as i16 + insn.off) as usize; +} + +#[allow(unknown_lints)] +#[allow(cyclomatic_complexity)] +pub fn execute_program( + prog_: Option<&[u8]>, + mem: &[u8], + mbuff: &[u8], + helpers: &HashMap, +) -> Result { + const U32MAX: u64 = u32::MAX as u64; + const SHIFT_MASK_64: u64 = 0x3f; + + let prog = match prog_ { + Some(prog) => prog, + None => Err(Error::new( + ErrorKind::Other, + "Error: No program set, call prog_set() to load one", + ))?, + }; + let mut stacks = Vec::new(); + let stack = StackFrame::new(); + // R1 points to beginning of memory area, R10 to stack + let mut reg: [u64; 11] = [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + stack.as_ptr() as u64 + stack.len() as u64, + ]; + stacks.push(stack); + if !mbuff.is_empty() { + reg[1] = mbuff.as_ptr() as u64; + } else if !mem.is_empty() { + reg[1] = mem.as_ptr() as u64; + } + let check_mem_load = + |stack: &[u8], addr: u64, len: usize, insn_ptr: usize| -> Result<(), Error> { + check_mem(addr, len, "load", insn_ptr, mbuff, mem, stack) + }; + let check_mem_store = + |stack: &[u8], addr: u64, len: usize, insn_ptr: usize| -> Result<(), Error> { + check_mem(addr, len, "store", insn_ptr, mbuff, mem, stack) + }; + + // Loop on instructions + let mut insn_ptr: usize = 0; + while insn_ptr * ebpf::INSN_SIZE < prog.len() { + let insn = ebpf::get_insn(prog, insn_ptr); + insn_ptr += 1; + let _dst = insn.dst as usize; + let _src = insn.src as usize; + + match insn.opc { + // BPF_LD class + // LD_ABS_* and LD_IND_* are supposed to load pointer to data from metadata buffer. + // Since this pointer is constant, and since we already know it (mem), do not + // bother re-fetching it, just use mem already. + ebpf::LD_ABS_B => { + reg[0] = unsafe { + let x = (mem.as_ptr() as u64 + (insn.imm as u32) as u64) as *const u8; + check_mem_load(stacks.last().unwrap().as_slice(), x as u64, 8, insn_ptr)?; + x.read_unaligned() as u64 + } + } + ebpf::LD_ABS_H => { + reg[0] = unsafe { + let x = (mem.as_ptr() as u64 + (insn.imm as u32) as u64) as *const u16; + check_mem_load(stacks.last().unwrap().as_slice(), x as u64, 8, insn_ptr)?; + x.read_unaligned() as u64 + } + } + ebpf::LD_ABS_W => { + reg[0] = unsafe { + let x = (mem.as_ptr() as u64 + (insn.imm as u32) as u64) as *const u32; + check_mem_load(stacks.last().unwrap().as_slice(), x as u64, 8, insn_ptr)?; + x.read_unaligned() as u64 + } + } + ebpf::LD_ABS_DW => { + log::info!("executing LD_ABS_DW, set reg[{}] to {:#x}", _dst, insn.imm); + reg[0] = unsafe { + let x = (mem.as_ptr() as u64 + (insn.imm as u32) as u64) as *const u64; + check_mem_load(stacks.last().unwrap().as_slice(), x as u64, 8, insn_ptr)?; + x.read_unaligned() + } + } + ebpf::LD_IND_B => { + reg[0] = unsafe { + let x = + (mem.as_ptr() as u64 + reg[_src] + (insn.imm as u32) as u64) as *const u8; + check_mem_load(stacks.last().unwrap().as_slice(), x as u64, 8, insn_ptr)?; + x.read_unaligned() as u64 + } + } + ebpf::LD_IND_H => { + reg[0] = unsafe { + let x = + (mem.as_ptr() as u64 + reg[_src] + (insn.imm as u32) as u64) as *const u16; + check_mem_load(stacks.last().unwrap().as_slice(), x as u64, 8, insn_ptr)?; + x.read_unaligned() as u64 + } + } + ebpf::LD_IND_W => { + reg[0] = unsafe { + let x = + (mem.as_ptr() as u64 + reg[_src] + (insn.imm as u32) as u64) as *const u32; + check_mem_load(stacks.last().unwrap().as_slice(), x as u64, 8, insn_ptr)?; + x.read_unaligned() as u64 + } + } + ebpf::LD_IND_DW => { + reg[0] = unsafe { + let x = + (mem.as_ptr() as u64 + reg[_src] + (insn.imm as u32) as u64) as *const u64; + check_mem_load(stacks.last().unwrap().as_slice(), x as u64, 8, insn_ptr)?; + x.read_unaligned() + } + } + + ebpf::LD_DW_IMM => { + let next_insn = ebpf::get_insn(prog, insn_ptr); + insn_ptr += 1; + // log::warn!( + // "executing LD_DW_IMM, set reg[{}] to {:#x}", + // _dst, + // ((insn.imm as u32) as u64) + ((next_insn.imm as u64) << 32) + // ); + reg[_dst] = ((insn.imm as u32) as u64) + ((next_insn.imm as u64) << 32); + } + + // BPF_LDX class + ebpf::LD_B_REG => { + reg[_dst] = unsafe { + #[allow(clippy::cast_ptr_alignment)] + let x = (reg[_src] as *const u8).offset(insn.off as isize); + check_mem_load(stacks.last().unwrap().as_slice(), x as u64, 1, insn_ptr)?; + x.read_unaligned() as u64 + } + } + ebpf::LD_H_REG => { + reg[_dst] = unsafe { + #[allow(clippy::cast_ptr_alignment)] + let x = (reg[_src] as *const u8).offset(insn.off as isize) as *const u16; + check_mem_load(stacks.last().unwrap().as_slice(), x as u64, 2, insn_ptr)?; + x.read_unaligned() as u64 + } + } + ebpf::LD_W_REG => { + reg[_dst] = unsafe { + #[allow(clippy::cast_ptr_alignment)] + let x = (reg[_src] as *const u8).offset(insn.off as isize) as *const u32; + check_mem_load(stacks.last().unwrap().as_slice(), x as u64, 4, insn_ptr)?; + // log::warn!( + // "executing LD_W_REG, the ptr is REG:{} -> [{:#x}] + {:#x}", + // _src, + // reg[_src], + // insn.off + // ); + x.read_unaligned() as u64 + } + } + ebpf::LD_DW_REG => { + reg[_dst] = unsafe { + #[allow(clippy::cast_ptr_alignment)] + let x = (reg[_src] as *const u8).offset(insn.off as isize) as *const u64; + check_mem_load(stacks.last().unwrap().as_slice(), x as u64, 8, insn_ptr)?; + x.read_unaligned() + } + } + + // BPF_ST class + ebpf::ST_B_IMM => unsafe { + let x = (reg[_dst] as *const u8).offset(insn.off as isize) as *mut u8; + check_mem_store(stacks.last().unwrap().as_slice(), x as u64, 1, insn_ptr)?; + x.write_unaligned(insn.imm as u8); + }, + ebpf::ST_H_IMM => unsafe { + #[allow(clippy::cast_ptr_alignment)] + let x = (reg[_dst] as *const u8).offset(insn.off as isize) as *mut u16; + check_mem_store(stacks.last().unwrap().as_slice(), x as u64, 2, insn_ptr)?; + x.write_unaligned(insn.imm as u16); + }, + ebpf::ST_W_IMM => unsafe { + #[allow(clippy::cast_ptr_alignment)] + let x = (reg[_dst] as *const u8).offset(insn.off as isize) as *mut u32; + check_mem_store(stacks.last().unwrap().as_slice(), x as u64, 4, insn_ptr)?; + x.write_unaligned(insn.imm as u32); + }, + ebpf::ST_DW_IMM => unsafe { + #[allow(clippy::cast_ptr_alignment)] + let x = (reg[_dst] as *const u8).offset(insn.off as isize) as *mut u64; + check_mem_store(stacks.last().unwrap().as_slice(), x as u64, 8, insn_ptr)?; + x.write_unaligned(insn.imm as u64); + }, + + // BPF_STX class + ebpf::ST_B_REG => unsafe { + let x = (reg[_dst] as *const u8).offset(insn.off as isize) as *mut u8; + check_mem_store(stacks.last().unwrap().as_slice(), x as u64, 1, insn_ptr)?; + x.write_unaligned(reg[_src] as u8); + }, + ebpf::ST_H_REG => unsafe { + #[allow(clippy::cast_ptr_alignment)] + let x = (reg[_dst] as *const u8).offset(insn.off as isize) as *mut u16; + check_mem_store(stacks.last().unwrap().as_slice(), x as u64, 2, insn_ptr)?; + x.write_unaligned(reg[_src] as u16); + }, + ebpf::ST_W_REG => unsafe { + #[allow(clippy::cast_ptr_alignment)] + let x = (reg[_dst] as *const u8).offset(insn.off as isize) as *mut u32; + check_mem_store(stacks.last().unwrap().as_slice(), x as u64, 4, insn_ptr)?; + x.write_unaligned(reg[_src] as u32); + }, + ebpf::ST_DW_REG => unsafe { + #[allow(clippy::cast_ptr_alignment)] + let x = (reg[_dst] as *const u8).offset(insn.off as isize) as *mut u64; + check_mem_store(stacks.last().unwrap().as_slice(), x as u64, 8, insn_ptr)?; + x.write_unaligned(reg[_src]); + }, + ebpf::ST_W_XADD => unimplemented!(), + ebpf::ST_DW_XADD => unimplemented!(), + + // BPF_ALU class + // TODO Check how overflow works in kernel. Should we &= U32MAX all src register value + // before we do the operation? + // Cf ((0x11 << 32) - (0x1 << 32)) as u32 VS ((0x11 << 32) as u32 - (0x1 << 32) as u32 + ebpf::ADD32_IMM => reg[_dst] = (reg[_dst] as i32).wrapping_add(insn.imm) as u64, //((reg[_dst] & U32MAX) + insn.imm as u64) & U32MAX, + ebpf::ADD32_REG => reg[_dst] = (reg[_dst] as i32).wrapping_add(reg[_src] as i32) as u64, //((reg[_dst] & U32MAX) + (reg[_src] & U32MAX)) & U32MAX, + ebpf::SUB32_IMM => reg[_dst] = (reg[_dst] as i32).wrapping_sub(insn.imm) as u64, + ebpf::SUB32_REG => reg[_dst] = (reg[_dst] as i32).wrapping_sub(reg[_src] as i32) as u64, + ebpf::MUL32_IMM => reg[_dst] = (reg[_dst] as i32).wrapping_mul(insn.imm) as u64, + ebpf::MUL32_REG => reg[_dst] = (reg[_dst] as i32).wrapping_mul(reg[_src] as i32) as u64, + ebpf::DIV32_IMM if insn.imm as u32 == 0 => reg[_dst] = 0, + ebpf::DIV32_IMM => reg[_dst] = (reg[_dst] as u32 / insn.imm as u32) as u64, + ebpf::DIV32_REG if reg[_src] as u32 == 0 => reg[_dst] = 0, + ebpf::DIV32_REG => reg[_dst] = (reg[_dst] as u32 / reg[_src] as u32) as u64, + ebpf::OR32_IMM => reg[_dst] = (reg[_dst] as u32 | insn.imm as u32) as u64, + ebpf::OR32_REG => reg[_dst] = (reg[_dst] as u32 | reg[_src] as u32) as u64, + ebpf::AND32_IMM => reg[_dst] = (reg[_dst] as u32 & insn.imm as u32) as u64, + ebpf::AND32_REG => reg[_dst] = (reg[_dst] as u32 & reg[_src] as u32) as u64, + // As for the 64-bit version, we should mask the number of bits to shift with + // 0x1f, but .wrappping_shr() already takes care of it for us. + ebpf::LSH32_IMM => reg[_dst] = (reg[_dst] as u32).wrapping_shl(insn.imm as u32) as u64, + ebpf::LSH32_REG => reg[_dst] = (reg[_dst] as u32).wrapping_shl(reg[_src] as u32) as u64, + ebpf::RSH32_IMM => reg[_dst] = (reg[_dst] as u32).wrapping_shr(insn.imm as u32) as u64, + ebpf::RSH32_REG => reg[_dst] = (reg[_dst] as u32).wrapping_shr(reg[_src] as u32) as u64, + ebpf::NEG32 => { + reg[_dst] = (reg[_dst] as i32).wrapping_neg() as u64; + reg[_dst] &= U32MAX; + } + ebpf::MOD32_IMM if insn.imm as u32 == 0 => (), + ebpf::MOD32_IMM => reg[_dst] = (reg[_dst] as u32 % insn.imm as u32) as u64, + ebpf::MOD32_REG if reg[_src] as u32 == 0 => (), + ebpf::MOD32_REG => reg[_dst] = (reg[_dst] as u32 % reg[_src] as u32) as u64, + ebpf::XOR32_IMM => reg[_dst] = (reg[_dst] as u32 ^ insn.imm as u32) as u64, + ebpf::XOR32_REG => reg[_dst] = (reg[_dst] as u32 ^ reg[_src] as u32) as u64, + ebpf::MOV32_IMM => reg[_dst] = insn.imm as u32 as u64, + ebpf::MOV32_REG => reg[_dst] = (reg[_src] as u32) as u64, + // As for the 64-bit version, we should mask the number of bits to shift with + // 0x1f, but .wrappping_shr() already takes care of it for us. + ebpf::ARSH32_IMM => { + reg[_dst] = (reg[_dst] as i32).wrapping_shr(insn.imm as u32) as u64; + reg[_dst] &= U32MAX; + } + ebpf::ARSH32_REG => { + reg[_dst] = (reg[_dst] as i32).wrapping_shr(reg[_src] as u32) as u64; + reg[_dst] &= U32MAX; + } + ebpf::LE => { + reg[_dst] = match insn.imm { + 16 => (reg[_dst] as u16).to_le() as u64, + 32 => (reg[_dst] as u32).to_le() as u64, + 64 => reg[_dst].to_le(), + _ => unreachable!(), + }; + } + ebpf::BE => { + reg[_dst] = match insn.imm { + 16 => (reg[_dst] as u16).to_be() as u64, + 32 => (reg[_dst] as u32).to_be() as u64, + 64 => reg[_dst].to_be(), + _ => unreachable!(), + }; + } + + // BPF_ALU64 class + ebpf::ADD64_IMM => reg[_dst] = reg[_dst].wrapping_add(insn.imm as u64), + ebpf::ADD64_REG => reg[_dst] = reg[_dst].wrapping_add(reg[_src]), + ebpf::SUB64_IMM => reg[_dst] = reg[_dst].wrapping_sub(insn.imm as u64), + ebpf::SUB64_REG => reg[_dst] = reg[_dst].wrapping_sub(reg[_src]), + ebpf::MUL64_IMM => reg[_dst] = reg[_dst].wrapping_mul(insn.imm as u64), + ebpf::MUL64_REG => reg[_dst] = reg[_dst].wrapping_mul(reg[_src]), + ebpf::DIV64_IMM if insn.imm == 0 => reg[_dst] = 0, + ebpf::DIV64_IMM => reg[_dst] /= insn.imm as u64, + ebpf::DIV64_REG if reg[_src] == 0 => reg[_dst] = 0, + ebpf::DIV64_REG => reg[_dst] /= reg[_src], + ebpf::OR64_IMM => reg[_dst] |= insn.imm as u64, + ebpf::OR64_REG => reg[_dst] |= reg[_src], + ebpf::AND64_IMM => reg[_dst] &= insn.imm as u64, + ebpf::AND64_REG => reg[_dst] &= reg[_src], + ebpf::LSH64_IMM => reg[_dst] <<= insn.imm as u64 & SHIFT_MASK_64, + ebpf::LSH64_REG => reg[_dst] <<= reg[_src] & SHIFT_MASK_64, + ebpf::RSH64_IMM => reg[_dst] >>= insn.imm as u64 & SHIFT_MASK_64, + ebpf::RSH64_REG => reg[_dst] >>= reg[_src] & SHIFT_MASK_64, + ebpf::NEG64 => reg[_dst] = -(reg[_dst] as i64) as u64, + ebpf::MOD64_IMM if insn.imm == 0 => (), + ebpf::MOD64_IMM => reg[_dst] %= insn.imm as u64, + ebpf::MOD64_REG if reg[_src] == 0 => (), + ebpf::MOD64_REG => reg[_dst] %= reg[_src], + ebpf::XOR64_IMM => reg[_dst] ^= insn.imm as u64, + ebpf::XOR64_REG => reg[_dst] ^= reg[_src], + ebpf::MOV64_IMM => reg[_dst] = insn.imm as u64, + ebpf::MOV64_REG => reg[_dst] = reg[_src], + ebpf::ARSH64_IMM => { + reg[_dst] = (reg[_dst] as i64 >> (insn.imm as u64 & SHIFT_MASK_64)) as u64 + } + ebpf::ARSH64_REG => { + reg[_dst] = (reg[_dst] as i64 >> (reg[_src] as u64 & SHIFT_MASK_64)) as u64 + } + + // BPF_JMP class + // TODO: check this actually works as expected for signed / unsigned ops + ebpf::JA => do_jump(&mut insn_ptr, &insn), + ebpf::JEQ_IMM => { + if reg[_dst] == insn.imm as u64 { + do_jump(&mut insn_ptr, &insn); + } + } + ebpf::JEQ_REG => { + if reg[_dst] == reg[_src] { + do_jump(&mut insn_ptr, &insn); + } + } + ebpf::JGT_IMM => { + if reg[_dst] > insn.imm as u64 { + do_jump(&mut insn_ptr, &insn); + } + } + ebpf::JGT_REG => { + if reg[_dst] > reg[_src] { + do_jump(&mut insn_ptr, &insn); + } + } + ebpf::JGE_IMM => { + if reg[_dst] >= insn.imm as u64 { + do_jump(&mut insn_ptr, &insn); + } + } + ebpf::JGE_REG => { + if reg[_dst] >= reg[_src] { + do_jump(&mut insn_ptr, &insn); + } + } + ebpf::JLT_IMM => { + if reg[_dst] < insn.imm as u64 { + do_jump(&mut insn_ptr, &insn); + } + } + ebpf::JLT_REG => { + if reg[_dst] < reg[_src] { + do_jump(&mut insn_ptr, &insn); + } + } + ebpf::JLE_IMM => { + if reg[_dst] <= insn.imm as u64 { + do_jump(&mut insn_ptr, &insn); + } + } + ebpf::JLE_REG => { + if reg[_dst] <= reg[_src] { + do_jump(&mut insn_ptr, &insn); + } + } + ebpf::JSET_IMM => { + if reg[_dst] & insn.imm as u64 != 0 { + do_jump(&mut insn_ptr, &insn); + } + } + ebpf::JSET_REG => { + if reg[_dst] & reg[_src] != 0 { + do_jump(&mut insn_ptr, &insn); + } + } + ebpf::JNE_IMM => { + if reg[_dst] != insn.imm as u64 { + do_jump(&mut insn_ptr, &insn); + } + } + ebpf::JNE_REG => { + if reg[_dst] != reg[_src] { + do_jump(&mut insn_ptr, &insn); + } + } + ebpf::JSGT_IMM => { + if reg[_dst] as i64 > insn.imm as i64 { + do_jump(&mut insn_ptr, &insn); + } + } + ebpf::JSGT_REG => { + if reg[_dst] as i64 > reg[_src] as i64 { + do_jump(&mut insn_ptr, &insn); + } + } + ebpf::JSGE_IMM => { + if reg[_dst] as i64 >= insn.imm as i64 { + do_jump(&mut insn_ptr, &insn); + } + } + ebpf::JSGE_REG => { + if reg[_dst] as i64 >= reg[_src] as i64 { + do_jump(&mut insn_ptr, &insn); + } + } + ebpf::JSLT_IMM => { + if (reg[_dst] as i64) < insn.imm as i64 { + do_jump(&mut insn_ptr, &insn); + } + } + ebpf::JSLT_REG => { + if (reg[_dst] as i64) < reg[_src] as i64 { + do_jump(&mut insn_ptr, &insn); + } + } + ebpf::JSLE_IMM => { + if reg[_dst] as i64 <= insn.imm as i64 { + do_jump(&mut insn_ptr, &insn); + } + } + ebpf::JSLE_REG => { + if reg[_dst] as i64 <= reg[_src] as i64 { + do_jump(&mut insn_ptr, &insn); + } + } + + // BPF_JMP32 class + ebpf::JEQ_IMM32 => { + if reg[_dst] as u32 == insn.imm as u32 { + do_jump(&mut insn_ptr, &insn); + } + } + ebpf::JEQ_REG32 => { + if reg[_dst] as u32 == reg[_src] as u32 { + do_jump(&mut insn_ptr, &insn); + } + } + ebpf::JGT_IMM32 => { + if reg[_dst] as u32 > insn.imm as u32 { + do_jump(&mut insn_ptr, &insn); + } + } + ebpf::JGT_REG32 => { + if reg[_dst] as u32 > reg[_src] as u32 { + do_jump(&mut insn_ptr, &insn); + } + } + ebpf::JGE_IMM32 => { + if reg[_dst] as u32 >= insn.imm as u32 { + do_jump(&mut insn_ptr, &insn); + } + } + ebpf::JGE_REG32 => { + if reg[_dst] as u32 >= reg[_src] as u32 { + do_jump(&mut insn_ptr, &insn); + } + } + ebpf::JLT_IMM32 => { + if (reg[_dst] as u32) < insn.imm as u32 { + do_jump(&mut insn_ptr, &insn); + } + } + ebpf::JLT_REG32 => { + if (reg[_dst] as u32) < reg[_src] as u32 { + do_jump(&mut insn_ptr, &insn); + } + } + ebpf::JLE_IMM32 => { + if reg[_dst] as u32 <= insn.imm as u32 { + do_jump(&mut insn_ptr, &insn); + } + } + ebpf::JLE_REG32 => { + if reg[_dst] as u32 <= reg[_src] as u32 { + do_jump(&mut insn_ptr, &insn); + } + } + ebpf::JSET_IMM32 => { + if reg[_dst] as u32 & insn.imm as u32 != 0 { + do_jump(&mut insn_ptr, &insn); + } + } + ebpf::JSET_REG32 => { + if reg[_dst] as u32 & reg[_src] as u32 != 0 { + do_jump(&mut insn_ptr, &insn); + } + } + ebpf::JNE_IMM32 => { + if reg[_dst] as u32 != insn.imm as u32 { + do_jump(&mut insn_ptr, &insn); + } + } + ebpf::JNE_REG32 => { + if reg[_dst] as u32 != reg[_src] as u32 { + do_jump(&mut insn_ptr, &insn); + } + } + ebpf::JSGT_IMM32 => { + if reg[_dst] as i32 > insn.imm { + do_jump(&mut insn_ptr, &insn); + } + } + ebpf::JSGT_REG32 => { + if reg[_dst] as i32 > reg[_src] as i32 { + do_jump(&mut insn_ptr, &insn); + } + } + ebpf::JSGE_IMM32 => { + if reg[_dst] as i32 >= insn.imm { + do_jump(&mut insn_ptr, &insn); + } + } + ebpf::JSGE_REG32 => { + if reg[_dst] as i32 >= reg[_src] as i32 { + do_jump(&mut insn_ptr, &insn); + } + } + ebpf::JSLT_IMM32 => { + if (reg[_dst] as i32) < insn.imm { + do_jump(&mut insn_ptr, &insn); + } + } + ebpf::JSLT_REG32 => { + if (reg[_dst] as i32) < reg[_src] as i32 { + do_jump(&mut insn_ptr, &insn); + } + } + ebpf::JSLE_IMM32 => { + if reg[_dst] as i32 <= insn.imm { + do_jump(&mut insn_ptr, &insn); + } + } + ebpf::JSLE_REG32 => { + if reg[_dst] as i32 <= reg[_src] as i32 { + do_jump(&mut insn_ptr, &insn); + } + } + + // Do not delegate the check to the verifier, since registered functions can be + // changed after the program has been verified. + ebpf::CALL => { + // See https://www.kernel.org/doc/html/latest/bpf/standardization/instruction-set.html#id16 + let src_reg = _src; + let call_func_res = match src_reg { + 0 => { + // Handle call by address to external function. + if let Some(function) = helpers.get(&(insn.imm as u32)) { + reg[0] = function(reg[1], reg[2], reg[3], reg[4], reg[5]); + Ok(()) + }else { + Err(format!( + "Error: unknown helper function (id: {:#x}) [{}], (instruction #{})", + insn.imm as u32,BPF_FUNC_MAPPER[insn.imm as usize],insn_ptr + )) + } + } + 1 => { + // bpf to bpf call + // The function is in the same program, so we can just jump to the address + if stacks.len() >= ebpf::RBPF_MAX_CALL_DEPTH{ + Err(format!( + "Error: bpf to bpf call stack limit reached (instruction #{}) max depth: {}", + insn_ptr, ebpf::RBPF_MAX_CALL_DEPTH + )) + }else { + let mut pre_stack = stacks.last_mut().unwrap(); + // Save the callee saved registers + pre_stack.save_registers(®[6..=9]); + // Save the return address + pre_stack.save_return_address(insn_ptr as u16); + // save the stack pointer + pre_stack.save_sp(reg[10] as u16); + let mut stack = StackFrame::new(); + log::trace!("BPF TO BPF CALL: new pc: {} + {} = {}",insn_ptr ,insn.imm,insn_ptr + insn.imm as usize); + reg[10] = stack.as_ptr() as u64 + stack.len() as u64; + stacks.push(stack); + insn_ptr += insn.imm as usize; + Ok(()) + } + } + _ =>{ + Err(format!( + "Error: the function call type (id: {:#x}) [{}], (instruction #{}) not supported", + insn.imm as u32,BPF_FUNC_MAPPER[insn.imm as usize],insn_ptr + )) + } + }; + if let Err(e) = call_func_res { + Err(Error::new(ErrorKind::Other, e))?; + } + } + ebpf::TAIL_CALL => unimplemented!(), + ebpf::EXIT => { + if stacks.len() == 1 { + return Ok(reg[0]); + } else { + // Pop the stack + stacks.pop(); + let stack = stacks.last().unwrap(); + // Restore the callee saved registers + reg[6..=9].copy_from_slice(&stack.get_registers()); + // Restore the return address + insn_ptr = stack.get_return_address() as usize; + // Restore the stack pointer + reg[10] = stack.get_sp() as u64; + log::trace!("EXIT: new pc: {}", insn_ptr); + } + } + + _ => unreachable!(), + } + } + + unreachable!() +} diff --git a/kernel/crates/rbpf/src/jit.rs b/kernel/crates/rbpf/src/jit.rs new file mode 100644 index 00000000..df509ce0 --- /dev/null +++ b/kernel/crates/rbpf/src/jit.rs @@ -0,0 +1,1054 @@ +// SPDX-License-Identifier: (Apache-2.0 OR MIT) +// Derived from uBPF +// Copyright 2015 Big Switch Networks, Inc +// (uBPF: JIT algorithm, originally in C) +// Copyright 2016 6WIND S.A. +// (Translation to Rust, MetaBuff addition) + +use std::{ + fmt::{Error as FormatterError, Formatter}, + io::{Error, ErrorKind}, + mem, + ops::{Index, IndexMut}, +}; + +use crate::{ebpf, HashMap}; + +extern crate libc; + +type MachineCode = unsafe fn(*mut u8, usize, *mut u8, usize, usize, usize) -> u64; + +const PAGE_SIZE: usize = 4096; +// TODO: check how long the page must be to be sure to support an eBPF program of maximum possible +// length +const NUM_PAGES: usize = 1; + +// Special values for target_pc in struct Jump +const TARGET_OFFSET: isize = ebpf::PROG_MAX_INSNS as isize; +const TARGET_PC_EXIT: isize = TARGET_OFFSET + 1; + +#[derive(Copy, Clone)] +enum OperandSize { + S8 = 8, + S16 = 16, + S32 = 32, + S64 = 64, +} + +// Registers +const RAX: u8 = 0; +const RCX: u8 = 1; +const RDX: u8 = 2; +const RBX: u8 = 3; +const RSP: u8 = 4; +const RBP: u8 = 5; +const RSI: u8 = 6; +const RDI: u8 = 7; +const R8: u8 = 8; +const R9: u8 = 9; +const R10: u8 = 10; +const R11: u8 = 11; +//const R12: u8 = 12; +const R13: u8 = 13; +const R14: u8 = 14; +const R15: u8 = 15; + +const REGISTER_MAP_SIZE: usize = 11; +const REGISTER_MAP: [u8; REGISTER_MAP_SIZE] = [ + RAX, // 0 return value + RDI, // 1 arg 1 + RSI, // 2 arg 2 + RDX, // 3 arg 3 + R9, // 4 arg 4 + R8, // 5 arg 5 + RBX, // 6 callee-saved + R13, // 7 callee-saved + R14, // 8 callee-saved + R15, // 9 callee-saved + RBP, // 10 stack pointer + // R10 and R11 are used to compute store a constant pointer to mem and to compute offset for + // LD_ABS_* and LD_IND_* operations, so they are not mapped to any eBPF register. +]; + +// Return the x86 register for the given eBPF register +fn map_register(r: u8) -> u8 { + assert!(r < REGISTER_MAP_SIZE as u8); + REGISTER_MAP[(r % REGISTER_MAP_SIZE as u8) as usize] +} + +macro_rules! emit_bytes { + ( $mem:ident, $data:tt, $t:ty ) => {{ + let size = mem::size_of::<$t>() as usize; + assert!($mem.offset + size <= $mem.contents.len()); + unsafe { + let mut ptr = $mem.contents.as_ptr().add($mem.offset) as *mut $t; + ptr.write_unaligned($data); + } + $mem.offset += size; + }}; +} + +#[derive(Debug)] +struct Jump { + offset_loc: usize, + target_pc: isize, +} + +#[derive(Debug)] +struct JitCompiler { + pc_locs: Vec, + special_targets: HashMap, + jumps: Vec, +} + +impl JitCompiler { + fn new() -> JitCompiler { + JitCompiler { + pc_locs: vec![], + jumps: vec![], + special_targets: HashMap::new(), + } + } + + fn emit1(&self, mem: &mut JitMemory, data: u8) { + emit_bytes!(mem, data, u8); + } + + fn emit2(&self, mem: &mut JitMemory, data: u16) { + emit_bytes!(mem, data, u16); + } + + fn emit4(&self, mem: &mut JitMemory, data: u32) { + emit_bytes!(mem, data, u32); + } + + fn emit8(&self, mem: &mut JitMemory, data: u64) { + emit_bytes!(mem, data, u64); + } + + fn emit_modrm(&self, mem: &mut JitMemory, modrm: u8, r: u8, m: u8) { + assert_eq!((modrm | 0xc0), 0xc0); + self.emit1(mem, (modrm & 0xc0) | ((r & 0b111) << 3) | (m & 0b111)); + } + + fn emit_modrm_reg2reg(&self, mem: &mut JitMemory, r: u8, m: u8) { + self.emit_modrm(mem, 0xc0, r, m); + } + + fn emit_modrm_and_displacement(&self, mem: &mut JitMemory, r: u8, m: u8, d: i32) { + if d == 0 && (m & 0b111) != RBP { + self.emit_modrm(mem, 0x00, r, m); + } else if (-128..=127).contains(&d) { + self.emit_modrm(mem, 0x40, r, m); + self.emit1(mem, d as u8); + } else { + self.emit_modrm(mem, 0x80, r, m); + self.emit4(mem, d as u32); + } + } + + fn basix_rex_would_set_bits(&self, w: u8, src: u8, dst: u8) -> bool { + w != 0 || (src & 0b1000) != 0 || (dst & 0b1000) != 0 + } + + fn emit_rex(&self, mem: &mut JitMemory, w: u8, r: u8, x: u8, b: u8) { + assert_eq!((w | 1), 1); + assert_eq!((r | 1), 1); + assert_eq!((x | 1), 1); + assert_eq!((b | 1), 1); + self.emit1(mem, 0x40 | (w << 3) | (r << 2) | (x << 1) | b); + } + + // Emits a REX prefix with the top bit of src and dst. + // Skipped if no bits would be set. + fn emit_basic_rex(&self, mem: &mut JitMemory, w: u8, src: u8, dst: u8) { + if self.basix_rex_would_set_bits(w, src, dst) { + let is_masked = |val, mask| match val & mask { + 0 => 0, + _ => 1, + }; + self.emit_rex(mem, w, is_masked(src, 8), 0, is_masked(dst, 8)); + } + } + + fn emit_push(&self, mem: &mut JitMemory, r: u8) { + self.emit_basic_rex(mem, 0, 0, r); + self.emit1(mem, 0x50 | (r & 0b111)); + } + + fn emit_pop(&self, mem: &mut JitMemory, r: u8) { + self.emit_basic_rex(mem, 0, 0, r); + self.emit1(mem, 0x58 | (r & 0b111)); + } + + // REX prefix and ModRM byte + // We use the MR encoding when there is a choice + // 'src' is often used as an opcode extension + fn emit_alu32(&self, mem: &mut JitMemory, op: u8, src: u8, dst: u8) { + self.emit_basic_rex(mem, 0, src, dst); + self.emit1(mem, op); + self.emit_modrm_reg2reg(mem, src, dst); + } + + // REX prefix, ModRM byte, and 32-bit immediate + fn emit_alu32_imm32(&self, mem: &mut JitMemory, op: u8, src: u8, dst: u8, imm: i32) { + self.emit_alu32(mem, op, src, dst); + self.emit4(mem, imm as u32); + } + + // REX prefix, ModRM byte, and 8-bit immediate + fn emit_alu32_imm8(&self, mem: &mut JitMemory, op: u8, src: u8, dst: u8, imm: i8) { + self.emit_alu32(mem, op, src, dst); + self.emit1(mem, imm as u8); + } + + // REX.W prefix and ModRM byte + // We use the MR encoding when there is a choice + // 'src' is often used as an opcode extension + fn emit_alu64(&self, mem: &mut JitMemory, op: u8, src: u8, dst: u8) { + self.emit_basic_rex(mem, 1, src, dst); + self.emit1(mem, op); + self.emit_modrm_reg2reg(mem, src, dst); + } + + // REX.W prefix, ModRM byte, and 32-bit immediate + fn emit_alu64_imm32(&self, mem: &mut JitMemory, op: u8, src: u8, dst: u8, imm: i32) { + self.emit_alu64(mem, op, src, dst); + self.emit4(mem, imm as u32); + } + + // REX.W prefix, ModRM byte, and 8-bit immediate + fn emit_alu64_imm8(&self, mem: &mut JitMemory, op: u8, src: u8, dst: u8, imm: i8) { + self.emit_alu64(mem, op, src, dst); + self.emit1(mem, imm as u8); + } + + // Register to register mov + fn emit_mov(&self, mem: &mut JitMemory, src: u8, dst: u8) { + self.emit_alu64(mem, 0x89, src, dst); + } + + fn emit_cmp_imm32(&self, mem: &mut JitMemory, dst: u8, imm: i32) { + self.emit_alu64_imm32(mem, 0x81, 7, dst, imm); + } + + fn emit_cmp(&self, mem: &mut JitMemory, src: u8, dst: u8) { + self.emit_alu64(mem, 0x39, src, dst); + } + + fn emit_cmp32_imm32(&self, mem: &mut JitMemory, dst: u8, imm: i32) { + self.emit_alu32_imm32(mem, 0x81, 7, dst, imm); + } + + fn emit_cmp32(&self, mem: &mut JitMemory, src: u8, dst: u8) { + self.emit_alu32(mem, 0x39, src, dst); + } + + // Load [src + offset] into dst + fn emit_load(&self, mem: &mut JitMemory, size: OperandSize, src: u8, dst: u8, offset: i32) { + let data = match size { + OperandSize::S64 => 1, + _ => 0, + }; + self.emit_basic_rex(mem, data, dst, src); + + match size { + OperandSize::S8 => { + // movzx + self.emit1(mem, 0x0f); + self.emit1(mem, 0xb6); + } + OperandSize::S16 => { + // movzx + self.emit1(mem, 0x0f); + self.emit1(mem, 0xb7); + } + OperandSize::S32 | OperandSize::S64 => { + // mov + self.emit1(mem, 0x8b); + } + } + + self.emit_modrm_and_displacement(mem, dst, src, offset); + } + + // Load sign-extended immediate into register + fn emit_load_imm(&self, mem: &mut JitMemory, dst: u8, imm: i64) { + if imm >= i32::MIN as i64 && imm <= i32::MAX as i64 { + self.emit_alu64_imm32(mem, 0xc7, 0, dst, imm as i32); + } else { + // movabs $imm,dst + self.emit_basic_rex(mem, 1, 0, dst); + self.emit1(mem, 0xb8 | (dst & 0b111)); + self.emit8(mem, imm as u64); + } + } + + // Store register src to [dst + offset] + fn emit_store(&self, mem: &mut JitMemory, size: OperandSize, src: u8, dst: u8, offset: i32) { + match size { + OperandSize::S16 => self.emit1(mem, 0x66), // 16-bit override + _ => {} + }; + let (is_s8, is_u64, rexw) = match size { + OperandSize::S8 => (true, false, 0), + OperandSize::S64 => (false, true, 1), + _ => (false, false, 0), + }; + if is_u64 || (src & 0b1000) != 0 || (dst & 0b1000) != 0 || is_s8 { + let is_masked = |val, mask| match val & mask { + 0 => 0, + _ => 1, + }; + self.emit_rex(mem, rexw, is_masked(src, 8), 0, is_masked(dst, 8)); + } + match size { + OperandSize::S8 => self.emit1(mem, 0x88), + _ => self.emit1(mem, 0x89), + }; + self.emit_modrm_and_displacement(mem, src, dst, offset); + } + + // Store immediate to [dst + offset] + fn emit_store_imm32( + &self, + mem: &mut JitMemory, + size: OperandSize, + dst: u8, + offset: i32, + imm: i32, + ) { + match size { + OperandSize::S16 => self.emit1(mem, 0x66), // 16-bit override + _ => {} + }; + match size { + OperandSize::S64 => self.emit_basic_rex(mem, 1, 0, dst), + _ => self.emit_basic_rex(mem, 0, 0, dst), + }; + match size { + OperandSize::S8 => self.emit1(mem, 0xc6), + _ => self.emit1(mem, 0xc7), + }; + self.emit_modrm_and_displacement(mem, 0, dst, offset); + match size { + OperandSize::S8 => self.emit1(mem, imm as u8), + OperandSize::S16 => self.emit2(mem, imm as u16), + _ => self.emit4(mem, imm as u32), + }; + } + + fn emit_direct_jcc(&self, mem: &mut JitMemory, code: u8, offset: u32) { + self.emit1(mem, 0x0f); + self.emit1(mem, code); + emit_bytes!(mem, offset, u32); + } + + fn emit_call(&self, mem: &mut JitMemory, target: usize) { + // TODO use direct call when possible + self.emit_load_imm(mem, RAX, target as i64); + // callq *%rax + self.emit1(mem, 0xff); + self.emit1(mem, 0xd0); + } + + fn emit_jump_offset(&mut self, mem: &mut JitMemory, target_pc: isize) { + let jump = Jump { + offset_loc: mem.offset, + target_pc, + }; + self.jumps.push(jump); + self.emit4(mem, 0); + } + + fn emit_jcc(&mut self, mem: &mut JitMemory, code: u8, target_pc: isize) { + self.emit1(mem, 0x0f); + self.emit1(mem, code); + self.emit_jump_offset(mem, target_pc); + } + + fn emit_jmp(&mut self, mem: &mut JitMemory, target_pc: isize) { + self.emit1(mem, 0xe9); + self.emit_jump_offset(mem, target_pc); + } + + fn set_anchor(&mut self, mem: &mut JitMemory, target: isize) { + self.special_targets.insert(target, mem.offset); + } + + fn emit_muldivmod( + &mut self, + mem: &mut JitMemory, + pc: u16, + opc: u8, + src: u8, + dst: u8, + imm: i32, + ) { + let mul = (opc & ebpf::BPF_ALU_OP_MASK) == (ebpf::MUL32_IMM & ebpf::BPF_ALU_OP_MASK); + let div = (opc & ebpf::BPF_ALU_OP_MASK) == (ebpf::DIV32_IMM & ebpf::BPF_ALU_OP_MASK); + let modrm = (opc & ebpf::BPF_ALU_OP_MASK) == (ebpf::MOD32_IMM & ebpf::BPF_ALU_OP_MASK); + let is64 = (opc & ebpf::BPF_CLS_MASK) == ebpf::BPF_ALU64; + let is_reg = (opc & ebpf::BPF_X) == ebpf::BPF_X; + + if (div || mul) && !is_reg && imm == 0 { + // Division by zero returns 0 + // Set register to 0: xor with itself + self.emit_alu32(mem, 0x31, dst, dst); + return; + } + if modrm && !is_reg && imm == 0 { + // Modulo remainder of division by zero keeps destination register unchanged + return; + } + if (div || modrm) && is_reg { + self.emit_load_imm(mem, RCX, pc as i64); + + // test src,src + if is64 { + self.emit_alu64(mem, 0x85, src, src); + } else { + self.emit_alu32(mem, 0x85, src, src); + } + + if div { + // No division by 0: skip next instructions + // Jump offset: emit_alu32 adds 2 to 3 bytes, emit_jmp adds 5 + let offset = match self.basix_rex_would_set_bits(0, dst, dst) { + true => 3 + 5, + false => 2 + 5, + }; + self.emit_direct_jcc(mem, 0x85, offset); + // Division by 0: set dst to 0 then go to next instruction + // Set register to 0: xor with itself + self.emit_alu32(mem, 0x31, dst, dst); + self.emit_jmp(mem, (pc + 1) as isize); + } + if modrm { + // Modulo by zero: keep destination register unchanged + self.emit_jcc(mem, 0x84, (pc + 1) as isize); + } + } + + if dst != RAX { + self.emit_push(mem, RAX); + } + if dst != RDX { + self.emit_push(mem, RDX); + } + if imm != 0 { + self.emit_load_imm(mem, RCX, imm as i64); + } else { + self.emit_mov(mem, src, RCX); + } + + self.emit_mov(mem, dst, RAX); + + if div || modrm { + // Set register to 0: xor %edx,%edx + self.emit_alu32(mem, 0x31, RDX, RDX); + } + + if is64 { + self.emit_rex(mem, 1, 0, 0, 0); + } + + // mul %ecx or div %ecx + self.emit_alu32(mem, 0xf7, if mul { 4 } else { 6 }, RCX); + + if dst != RDX { + if modrm { + self.emit_mov(mem, RDX, dst); + } + self.emit_pop(mem, RDX); + } + if dst != RAX { + if div || mul { + self.emit_mov(mem, RAX, dst); + } + self.emit_pop(mem, RAX); + } + } + + fn jit_compile( + &mut self, + mem: &mut JitMemory, + prog: &[u8], + use_mbuff: bool, + update_data_ptr: bool, + helpers: &HashMap, + ) -> Result<(), Error> { + self.emit_push(mem, RBP); + self.emit_push(mem, RBX); + self.emit_push(mem, R13); + self.emit_push(mem, R14); + self.emit_push(mem, R15); + + // RDI: mbuff + // RSI: mbuff_len + // RDX: mem + // RCX: mem_len + // R8: mem_offset + // R9: mem_end_offset + + // Save mem pointer for use with LD_ABS_* and LD_IND_* instructions + self.emit_mov(mem, RDX, R10); + + match (use_mbuff, update_data_ptr) { + (false, _) => { + // We do not use any mbuff. Move mem pointer into register 1. + if map_register(1) != RDX { + self.emit_mov(mem, RDX, map_register(1)); + } + } + (true, false) => { + // We use a mbuff already pointing to mem and mem_end: move it to register 1. + if map_register(1) != RDI { + self.emit_mov(mem, RDI, map_register(1)); + } + } + (true, true) => { + // We have a fixed (simulated) mbuff: update mem and mem_end offset values in it. + // Store mem at mbuff + mem_offset. Trash R8. + self.emit_alu64(mem, 0x01, RDI, R8); // add mbuff to mem_offset in R8 + self.emit_store(mem, OperandSize::S64, RDX, R8, 0); // set mem at mbuff + mem_offset + // Store mem_end at mbuff + mem_end_offset. Trash R9. + self.emit_load(mem, OperandSize::S64, RDX, R8, 0); // load mem into R8 + self.emit_alu64(mem, 0x01, RCX, R8); // add mem_len to mem (= mem_end) + self.emit_alu64(mem, 0x01, RDI, R9); // add mbuff to mem_end_offset + self.emit_store(mem, OperandSize::S64, R8, R9, 0); // store mem_end + + // Move rdi into register 1 + if map_register(1) != RDI { + self.emit_mov(mem, RDI, map_register(1)); + } + } + } + + // Copy stack pointer to R10 + self.emit_mov(mem, RSP, map_register(10)); + + // Allocate stack space + self.emit_alu64_imm32(mem, 0x81, 5, RSP, ebpf::STACK_SIZE as i32); + + self.pc_locs = vec![0; prog.len() / ebpf::INSN_SIZE + 1]; + + let mut insn_ptr: usize = 0; + while insn_ptr * ebpf::INSN_SIZE < prog.len() { + let insn = ebpf::get_insn(prog, insn_ptr); + + self.pc_locs[insn_ptr] = mem.offset; + + let dst = map_register(insn.dst); + let src = map_register(insn.src); + let target_pc = insn_ptr as isize + insn.off as isize + 1; + + match insn.opc { + // BPF_LD class + // R10 is a constant pointer to mem. + ebpf::LD_ABS_B => self.emit_load(mem, OperandSize::S8, R10, RAX, insn.imm), + ebpf::LD_ABS_H => self.emit_load(mem, OperandSize::S16, R10, RAX, insn.imm), + ebpf::LD_ABS_W => self.emit_load(mem, OperandSize::S32, R10, RAX, insn.imm), + ebpf::LD_ABS_DW => self.emit_load(mem, OperandSize::S64, R10, RAX, insn.imm), + ebpf::LD_IND_B => { + self.emit_mov(mem, R10, R11); // load mem into R11 + self.emit_alu64(mem, 0x01, src, R11); // add src to R11 + self.emit_load(mem, OperandSize::S8, R11, RAX, insn.imm); // ld R0, mem[src+imm] + } + ebpf::LD_IND_H => { + self.emit_mov(mem, R10, R11); // load mem into R11 + self.emit_alu64(mem, 0x01, src, R11); // add src to R11 + self.emit_load(mem, OperandSize::S16, R11, RAX, insn.imm); // ld R0, mem[src+imm] + } + ebpf::LD_IND_W => { + self.emit_mov(mem, R10, R11); // load mem into R11 + self.emit_alu64(mem, 0x01, src, R11); // add src to R11 + self.emit_load(mem, OperandSize::S32, R11, RAX, insn.imm); // ld R0, mem[src+imm] + } + ebpf::LD_IND_DW => { + self.emit_mov(mem, R10, R11); // load mem into R11 + self.emit_alu64(mem, 0x01, src, R11); // add src to R11 + self.emit_load(mem, OperandSize::S64, R11, RAX, insn.imm); // ld R0, mem[src+imm] + } + + ebpf::LD_DW_IMM => { + insn_ptr += 1; + let second_part = ebpf::get_insn(prog, insn_ptr).imm as u64; + let imm = (insn.imm as u32) as u64 | second_part.wrapping_shl(32); + self.emit_load_imm(mem, dst, imm as i64); + } + + // BPF_LDX class + ebpf::LD_B_REG => self.emit_load(mem, OperandSize::S8, src, dst, insn.off as i32), + ebpf::LD_H_REG => self.emit_load(mem, OperandSize::S16, src, dst, insn.off as i32), + ebpf::LD_W_REG => self.emit_load(mem, OperandSize::S32, src, dst, insn.off as i32), + ebpf::LD_DW_REG => self.emit_load(mem, OperandSize::S64, src, dst, insn.off as i32), + + // BPF_ST class + ebpf::ST_B_IMM => { + self.emit_store_imm32(mem, OperandSize::S8, dst, insn.off as i32, insn.imm) + } + ebpf::ST_H_IMM => { + self.emit_store_imm32(mem, OperandSize::S16, dst, insn.off as i32, insn.imm) + } + ebpf::ST_W_IMM => { + self.emit_store_imm32(mem, OperandSize::S32, dst, insn.off as i32, insn.imm) + } + ebpf::ST_DW_IMM => { + self.emit_store_imm32(mem, OperandSize::S64, dst, insn.off as i32, insn.imm) + } + + // BPF_STX class + ebpf::ST_B_REG => self.emit_store(mem, OperandSize::S8, src, dst, insn.off as i32), + ebpf::ST_H_REG => self.emit_store(mem, OperandSize::S16, src, dst, insn.off as i32), + ebpf::ST_W_REG => self.emit_store(mem, OperandSize::S32, src, dst, insn.off as i32), + ebpf::ST_DW_REG => { + self.emit_store(mem, OperandSize::S64, src, dst, insn.off as i32) + } + ebpf::ST_W_XADD => unimplemented!(), + ebpf::ST_DW_XADD => unimplemented!(), + + // BPF_ALU class + ebpf::ADD32_IMM => self.emit_alu32_imm32(mem, 0x81, 0, dst, insn.imm), + ebpf::ADD32_REG => self.emit_alu32(mem, 0x01, src, dst), + ebpf::SUB32_IMM => self.emit_alu32_imm32(mem, 0x81, 5, dst, insn.imm), + ebpf::SUB32_REG => self.emit_alu32(mem, 0x29, src, dst), + ebpf::MUL32_IMM + | ebpf::MUL32_REG + | ebpf::DIV32_IMM + | ebpf::DIV32_REG + | ebpf::MOD32_IMM + | ebpf::MOD32_REG => { + self.emit_muldivmod(mem, insn_ptr as u16, insn.opc, src, dst, insn.imm) + } + ebpf::OR32_IMM => self.emit_alu32_imm32(mem, 0x81, 1, dst, insn.imm), + ebpf::OR32_REG => self.emit_alu32(mem, 0x09, src, dst), + ebpf::AND32_IMM => self.emit_alu32_imm32(mem, 0x81, 4, dst, insn.imm), + ebpf::AND32_REG => self.emit_alu32(mem, 0x21, src, dst), + ebpf::LSH32_IMM => self.emit_alu32_imm8(mem, 0xc1, 4, dst, insn.imm as i8), + ebpf::LSH32_REG => { + self.emit_mov(mem, src, RCX); + self.emit_alu32(mem, 0xd3, 4, dst); + } + ebpf::RSH32_IMM => self.emit_alu32_imm8(mem, 0xc1, 5, dst, insn.imm as i8), + ebpf::RSH32_REG => { + self.emit_mov(mem, src, RCX); + self.emit_alu32(mem, 0xd3, 5, dst); + } + ebpf::NEG32 => self.emit_alu32(mem, 0xf7, 3, dst), + ebpf::XOR32_IMM => self.emit_alu32_imm32(mem, 0x81, 6, dst, insn.imm), + ebpf::XOR32_REG => self.emit_alu32(mem, 0x31, src, dst), + ebpf::MOV32_IMM => self.emit_alu32_imm32(mem, 0xc7, 0, dst, insn.imm), + ebpf::MOV32_REG => self.emit_mov(mem, src, dst), + ebpf::ARSH32_IMM => self.emit_alu32_imm8(mem, 0xc1, 7, dst, insn.imm as i8), + ebpf::ARSH32_REG => { + self.emit_mov(mem, src, RCX); + self.emit_alu32(mem, 0xd3, 7, dst); + } + ebpf::LE => {} // No-op + ebpf::BE => { + match insn.imm { + 16 => { + // rol + self.emit1(mem, 0x66); // 16-bit override + self.emit_alu32_imm8(mem, 0xc1, 0, dst, 8); + // and + self.emit_alu32_imm32(mem, 0x81, 4, dst, 0xffff); + } + 32 | 64 => { + // bswap + let bit = match insn.imm { + 64 => 1, + _ => 0, + }; + self.emit_basic_rex(mem, bit, 0, dst); + self.emit1(mem, 0x0f); + self.emit1(mem, 0xc8 | (dst & 0b111)); + } + _ => unreachable!(), // Should have been caught by verifier + } + } + + // BPF_ALU64 class + ebpf::ADD64_IMM => self.emit_alu64_imm32(mem, 0x81, 0, dst, insn.imm), + ebpf::ADD64_REG => self.emit_alu64(mem, 0x01, src, dst), + ebpf::SUB64_IMM => self.emit_alu64_imm32(mem, 0x81, 5, dst, insn.imm), + ebpf::SUB64_REG => self.emit_alu64(mem, 0x29, src, dst), + ebpf::MUL64_IMM + | ebpf::MUL64_REG + | ebpf::DIV64_IMM + | ebpf::DIV64_REG + | ebpf::MOD64_IMM + | ebpf::MOD64_REG => { + self.emit_muldivmod(mem, insn_ptr as u16, insn.opc, src, dst, insn.imm) + } + ebpf::OR64_IMM => self.emit_alu64_imm32(mem, 0x81, 1, dst, insn.imm), + ebpf::OR64_REG => self.emit_alu64(mem, 0x09, src, dst), + ebpf::AND64_IMM => self.emit_alu64_imm32(mem, 0x81, 4, dst, insn.imm), + ebpf::AND64_REG => self.emit_alu64(mem, 0x21, src, dst), + ebpf::LSH64_IMM => self.emit_alu64_imm8(mem, 0xc1, 4, dst, insn.imm as i8), + ebpf::LSH64_REG => { + self.emit_mov(mem, src, RCX); + self.emit_alu64(mem, 0xd3, 4, dst); + } + ebpf::RSH64_IMM => self.emit_alu64_imm8(mem, 0xc1, 5, dst, insn.imm as i8), + ebpf::RSH64_REG => { + self.emit_mov(mem, src, RCX); + self.emit_alu64(mem, 0xd3, 5, dst); + } + ebpf::NEG64 => self.emit_alu64(mem, 0xf7, 3, dst), + ebpf::XOR64_IMM => self.emit_alu64_imm32(mem, 0x81, 6, dst, insn.imm), + ebpf::XOR64_REG => self.emit_alu64(mem, 0x31, src, dst), + ebpf::MOV64_IMM => self.emit_load_imm(mem, dst, insn.imm as i64), + ebpf::MOV64_REG => self.emit_mov(mem, src, dst), + ebpf::ARSH64_IMM => self.emit_alu64_imm8(mem, 0xc1, 7, dst, insn.imm as i8), + ebpf::ARSH64_REG => { + self.emit_mov(mem, src, RCX); + self.emit_alu64(mem, 0xd3, 7, dst); + } + + // BPF_JMP class + ebpf::JA => self.emit_jmp(mem, target_pc), + ebpf::JEQ_IMM => { + self.emit_cmp_imm32(mem, dst, insn.imm); + self.emit_jcc(mem, 0x84, target_pc); + } + ebpf::JEQ_REG => { + self.emit_cmp(mem, src, dst); + self.emit_jcc(mem, 0x84, target_pc); + } + ebpf::JGT_IMM => { + self.emit_cmp_imm32(mem, dst, insn.imm); + self.emit_jcc(mem, 0x87, target_pc); + } + ebpf::JGT_REG => { + self.emit_cmp(mem, src, dst); + self.emit_jcc(mem, 0x87, target_pc); + } + ebpf::JGE_IMM => { + self.emit_cmp_imm32(mem, dst, insn.imm); + self.emit_jcc(mem, 0x83, target_pc); + } + ebpf::JGE_REG => { + self.emit_cmp(mem, src, dst); + self.emit_jcc(mem, 0x83, target_pc); + } + ebpf::JLT_IMM => { + self.emit_cmp_imm32(mem, dst, insn.imm); + self.emit_jcc(mem, 0x82, target_pc); + } + ebpf::JLT_REG => { + self.emit_cmp(mem, src, dst); + self.emit_jcc(mem, 0x82, target_pc); + } + ebpf::JLE_IMM => { + self.emit_cmp_imm32(mem, dst, insn.imm); + self.emit_jcc(mem, 0x86, target_pc); + } + ebpf::JLE_REG => { + self.emit_cmp(mem, src, dst); + self.emit_jcc(mem, 0x86, target_pc); + } + ebpf::JSET_IMM => { + self.emit_alu64_imm32(mem, 0xf7, 0, dst, insn.imm); + self.emit_jcc(mem, 0x85, target_pc); + } + ebpf::JSET_REG => { + self.emit_alu64(mem, 0x85, src, dst); + self.emit_jcc(mem, 0x85, target_pc); + } + ebpf::JNE_IMM => { + self.emit_cmp_imm32(mem, dst, insn.imm); + self.emit_jcc(mem, 0x85, target_pc); + } + ebpf::JNE_REG => { + self.emit_cmp(mem, src, dst); + self.emit_jcc(mem, 0x85, target_pc); + } + ebpf::JSGT_IMM => { + self.emit_cmp_imm32(mem, dst, insn.imm); + self.emit_jcc(mem, 0x8f, target_pc); + } + ebpf::JSGT_REG => { + self.emit_cmp(mem, src, dst); + self.emit_jcc(mem, 0x8f, target_pc); + } + ebpf::JSGE_IMM => { + self.emit_cmp_imm32(mem, dst, insn.imm); + self.emit_jcc(mem, 0x8d, target_pc); + } + ebpf::JSGE_REG => { + self.emit_cmp(mem, src, dst); + self.emit_jcc(mem, 0x8d, target_pc); + } + ebpf::JSLT_IMM => { + self.emit_cmp_imm32(mem, dst, insn.imm); + self.emit_jcc(mem, 0x8c, target_pc); + } + ebpf::JSLT_REG => { + self.emit_cmp(mem, src, dst); + self.emit_jcc(mem, 0x8c, target_pc); + } + ebpf::JSLE_IMM => { + self.emit_cmp_imm32(mem, dst, insn.imm); + self.emit_jcc(mem, 0x8e, target_pc); + } + ebpf::JSLE_REG => { + self.emit_cmp(mem, src, dst); + self.emit_jcc(mem, 0x8e, target_pc); + } + + // BPF_JMP32 class + ebpf::JEQ_IMM32 => { + self.emit_cmp32_imm32(mem, dst, insn.imm); + self.emit_jcc(mem, 0x84, target_pc); + } + ebpf::JEQ_REG32 => { + self.emit_cmp32(mem, src, dst); + self.emit_jcc(mem, 0x84, target_pc); + } + ebpf::JGT_IMM32 => { + self.emit_cmp32_imm32(mem, dst, insn.imm); + self.emit_jcc(mem, 0x87, target_pc); + } + ebpf::JGT_REG32 => { + self.emit_cmp32(mem, src, dst); + self.emit_jcc(mem, 0x87, target_pc); + } + ebpf::JGE_IMM32 => { + self.emit_cmp32_imm32(mem, dst, insn.imm); + self.emit_jcc(mem, 0x83, target_pc); + } + ebpf::JGE_REG32 => { + self.emit_cmp32(mem, src, dst); + self.emit_jcc(mem, 0x83, target_pc); + } + ebpf::JLT_IMM32 => { + self.emit_cmp32_imm32(mem, dst, insn.imm); + self.emit_jcc(mem, 0x82, target_pc); + } + ebpf::JLT_REG32 => { + self.emit_cmp32(mem, src, dst); + self.emit_jcc(mem, 0x82, target_pc); + } + ebpf::JLE_IMM32 => { + self.emit_cmp32_imm32(mem, dst, insn.imm); + self.emit_jcc(mem, 0x86, target_pc); + } + ebpf::JLE_REG32 => { + self.emit_cmp32(mem, src, dst); + self.emit_jcc(mem, 0x86, target_pc); + } + ebpf::JSET_IMM32 => { + self.emit_alu32_imm32(mem, 0xf7, 0, dst, insn.imm); + self.emit_jcc(mem, 0x85, target_pc); + } + ebpf::JSET_REG32 => { + self.emit_alu32(mem, 0x85, src, dst); + self.emit_jcc(mem, 0x85, target_pc); + } + ebpf::JNE_IMM32 => { + self.emit_cmp32_imm32(mem, dst, insn.imm); + self.emit_jcc(mem, 0x85, target_pc); + } + ebpf::JNE_REG32 => { + self.emit_cmp32(mem, src, dst); + self.emit_jcc(mem, 0x85, target_pc); + } + ebpf::JSGT_IMM32 => { + self.emit_cmp32_imm32(mem, dst, insn.imm); + self.emit_jcc(mem, 0x8f, target_pc); + } + ebpf::JSGT_REG32 => { + self.emit_cmp32(mem, src, dst); + self.emit_jcc(mem, 0x8f, target_pc); + } + ebpf::JSGE_IMM32 => { + self.emit_cmp32_imm32(mem, dst, insn.imm); + self.emit_jcc(mem, 0x8d, target_pc); + } + ebpf::JSGE_REG32 => { + self.emit_cmp32(mem, src, dst); + self.emit_jcc(mem, 0x8d, target_pc); + } + ebpf::JSLT_IMM32 => { + self.emit_cmp32_imm32(mem, dst, insn.imm); + self.emit_jcc(mem, 0x8c, target_pc); + } + ebpf::JSLT_REG32 => { + self.emit_cmp32(mem, src, dst); + self.emit_jcc(mem, 0x8c, target_pc); + } + ebpf::JSLE_IMM32 => { + self.emit_cmp32_imm32(mem, dst, insn.imm); + self.emit_jcc(mem, 0x8e, target_pc); + } + ebpf::JSLE_REG32 => { + self.emit_cmp32(mem, src, dst); + self.emit_jcc(mem, 0x8e, target_pc); + } + + ebpf::CALL => { + // For JIT, helpers in use MUST be registered at compile time. They can be + // updated later, but not created after compiling (we need the address of the + // helper function in the JIT-compiled program). + if let Some(helper) = helpers.get(&(insn.imm as u32)) { + // We reserve RCX for shifts + self.emit_mov(mem, R9, RCX); + self.emit_call(mem, *helper as usize); + } else { + Err(Error::new( + ErrorKind::Other, + format!( + "[JIT] Error: unknown helper function (id: {:#x})", + insn.imm as u32 + ), + ))?; + }; + } + ebpf::TAIL_CALL => { + unimplemented!() + } + ebpf::EXIT => { + if insn_ptr != prog.len() / ebpf::INSN_SIZE - 1 { + self.emit_jmp(mem, TARGET_PC_EXIT); + }; + } + + _ => { + Err(Error::new( + ErrorKind::Other, + format!( + "[JIT] Error: unknown eBPF opcode {:#2x} (insn #{insn_ptr:?})", + insn.opc + ), + ))?; + } + } + + insn_ptr += 1; + } + + // Epilogue + self.set_anchor(mem, TARGET_PC_EXIT); + + // Move register 0 into rax + if map_register(0) != RAX { + self.emit_mov(mem, map_register(0), RAX); + } + + // Deallocate stack space + self.emit_alu64_imm32(mem, 0x81, 0, RSP, ebpf::STACK_SIZE as i32); + + self.emit_pop(mem, R15); + self.emit_pop(mem, R14); + self.emit_pop(mem, R13); + self.emit_pop(mem, RBX); + self.emit_pop(mem, RBP); + + self.emit1(mem, 0xc3); // ret + + Ok(()) + } + + fn resolve_jumps(&mut self, mem: &mut JitMemory) -> Result<(), Error> { + for jump in &self.jumps { + let target_loc = match self.special_targets.get(&jump.target_pc) { + Some(target) => *target, + None => self.pc_locs[jump.target_pc as usize], + }; + + // Assumes jump offset is at end of instruction + unsafe { + let offset_loc = jump.offset_loc as i32 + std::mem::size_of::() as i32; + let rel = &(target_loc as i32 - offset_loc) as *const i32; + + let offset_ptr = mem.contents.as_ptr().add(jump.offset_loc); + + libc::memcpy( + offset_ptr as *mut libc::c_void, + rel as *const libc::c_void, + std::mem::size_of::(), + ); + } + } + Ok(()) + } +} // impl JitCompiler + +pub struct JitMemory<'a> { + contents: &'a mut [u8], + offset: usize, +} + +impl<'a> JitMemory<'a> { + pub fn new( + prog: &[u8], + helpers: &HashMap, + use_mbuff: bool, + update_data_ptr: bool, + ) -> Result, Error> { + let contents: &mut [u8]; + let mut raw: mem::MaybeUninit<*mut libc::c_void> = mem::MaybeUninit::uninit(); + unsafe { + let size = NUM_PAGES * PAGE_SIZE; + libc::posix_memalign(raw.as_mut_ptr(), PAGE_SIZE, size); + libc::mprotect( + *raw.as_mut_ptr(), + size, + libc::PROT_EXEC | libc::PROT_READ | libc::PROT_WRITE, + ); + std::ptr::write_bytes(*raw.as_mut_ptr(), 0xc3, size); // for now, prepopulate with 'RET' calls + contents = + std::slice::from_raw_parts_mut(*raw.as_mut_ptr() as *mut u8, NUM_PAGES * PAGE_SIZE); + raw.assume_init(); + } + + let mut mem = JitMemory { + contents, + offset: 0, + }; + + let mut jit = JitCompiler::new(); + jit.jit_compile(&mut mem, prog, use_mbuff, update_data_ptr, helpers)?; + jit.resolve_jumps(&mut mem)?; + + Ok(mem) + } + + pub fn get_prog(&self) -> MachineCode { + unsafe { mem::transmute(self.contents.as_ptr()) } + } +} + +impl<'a> Index for JitMemory<'a> { + type Output = u8; + + fn index(&self, _index: usize) -> &u8 { + &self.contents[_index] + } +} + +impl<'a> IndexMut for JitMemory<'a> { + fn index_mut(&mut self, _index: usize) -> &mut u8 { + &mut self.contents[_index] + } +} + +impl<'a> Drop for JitMemory<'a> { + fn drop(&mut self) { + unsafe { + libc::free(self.contents.as_mut_ptr() as *mut libc::c_void); + } + } +} + +impl<'a> std::fmt::Debug for JitMemory<'a> { + fn fmt(&self, fmt: &mut Formatter) -> Result<(), FormatterError> { + fmt.write_str("JIT contents: [")?; + fmt.write_str(" ] | ")?; + fmt.debug_struct("JIT memory") + .field("offset", &self.offset) + .finish() + } +} diff --git a/kernel/crates/rbpf/src/lib.rs b/kernel/crates/rbpf/src/lib.rs new file mode 100644 index 00000000..0d4fa283 --- /dev/null +++ b/kernel/crates/rbpf/src/lib.rs @@ -0,0 +1,1782 @@ +// SPDX-License-Identifier: (Apache-2.0 OR MIT) +// Derived from uBPF +// Copyright 2016 6WIND S.A. +// Copyright 2023 Isovalent, Inc. + +//! Virtual machine and JIT compiler for eBPF programs. +#![doc( + html_logo_url = "https://raw.githubusercontent.com/qmonnet/rbpf/main/misc/rbpf.png", + html_favicon_url = "https://raw.githubusercontent.com/qmonnet/rbpf/main/misc/rbpf.ico" +)] +#![warn(missing_docs)] +// There are unused mut warnings due to unsafe code. +#![allow(unused_mut)] +// Allows old-style clippy +#![allow(renamed_and_removed_lints)] +#![cfg_attr( + clippy, + allow( + redundant_field_names, + single_match, + cast_lossless, + doc_markdown, + match_same_arms, + unreadable_literal + ) +)] +// Configures the crate to be `no_std` when `std` feature is disabled. +#![cfg_attr(not(feature = "std"), no_std)] +extern crate alloc; +use alloc::{collections::BTreeMap, format, vec, vec::Vec}; + +use byteorder::{ByteOrder, LittleEndian}; + +type HashMap = BTreeMap; +#[cfg(feature = "cranelift")] +type HashSet = alloc::collections::BTreeSet; +mod asm_parser; +pub mod assembler; +#[cfg(feature = "cranelift")] +mod cranelift; +pub mod disassembler; +pub mod ebpf; +pub mod helpers; +pub mod insn_builder; +mod interpreter; +#[cfg(all(not(windows), feature = "std"))] +mod jit; +#[cfg(not(feature = "std"))] +mod no_std_error; +mod stack; +mod verifier; + +#[cfg(feature = "std")] +pub use std::io::{Error, ErrorKind}; + +/// In no_std we use a custom implementation of the error which acts as a +/// replacement for the io Error. +#[cfg(not(feature = "std"))] +pub use crate::no_std_error::{Error, ErrorKind}; + +/// eBPF verification function that returns an error if the program does not meet its requirements. +/// +/// Some examples of things the verifier may reject the program for: +/// +/// - Program does not terminate. +/// - Unknown instructions. +/// - Bad formed instruction. +/// - Unknown eBPF helper index. +pub type Verifier = fn(prog: &[u8]) -> Result<(), Error>; + +/// eBPF helper function. +pub type Helper = fn(u64, u64, u64, u64, u64) -> u64; + +// A metadata buffer with two offset indications. It can be used in one kind of eBPF VM to simulate +// the use of a metadata buffer each time the program is executed, without the user having to +// actually handle it. The offsets are used to tell the VM where in the buffer the pointers to +// packet data start and end should be stored each time the program is run on a new packet. +struct MetaBuff { + data_offset: usize, + data_end_offset: usize, + buffer: Vec, +} + +/// A virtual machine to run eBPF program. This kind of VM is used for programs expecting to work +/// on a metadata buffer containing pointers to packet data. +/// +/// # Examples +/// +/// ``` +/// let prog = &[ +/// 0x79, 0x11, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, // Load mem from mbuff at offset 8 into R1. +/// 0x69, 0x10, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, // ldhx r1[2], r0 +/// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit +/// ]; +/// let mem = &mut [ +/// 0xaa, 0xbb, 0x11, 0x22, 0xcc, 0xdd +/// ]; +/// +/// // Just for the example we create our metadata buffer from scratch, and we store the pointers +/// // to packet data start and end in it. +/// let mut mbuff = [0u8; 32]; +/// unsafe { +/// let mut data = mbuff.as_ptr().offset(8) as *mut u64; +/// let mut data_end = mbuff.as_ptr().offset(24) as *mut u64; +/// *data = mem.as_ptr() as u64; +/// *data_end = mem.as_ptr() as u64 + mem.len() as u64; +/// } +/// +/// // Instantiate a VM. +/// let mut vm = rbpf::EbpfVmMbuff::new(Some(prog)).unwrap(); +/// +/// // Provide both a reference to the packet data, and to the metadata buffer. +/// let res = vm.execute_program(mem, &mut mbuff).unwrap(); +/// assert_eq!(res, 0x2211); +/// ``` +pub struct EbpfVmMbuff<'a> { + prog: Option<&'a [u8]>, + verifier: Verifier, + #[cfg(all(not(windows), feature = "std"))] + jit: Option>, + #[cfg(feature = "cranelift")] + cranelift_prog: Option, + helpers: HashMap, +} + +impl<'a> EbpfVmMbuff<'a> { + /// Create a new virtual machine instance, and load an eBPF program into that instance. + /// When attempting to load the program, it passes through a simple verifier. + /// + /// # Examples + /// + /// ``` + /// let prog = &[ + /// 0x79, 0x11, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, // Load mem from mbuff into R1. + /// 0x69, 0x10, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, // ldhx r1[2], r0 + /// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit + /// ]; + /// + /// // Instantiate a VM. + /// let mut vm = rbpf::EbpfVmMbuff::new(Some(prog)).unwrap(); + /// ``` + pub fn new(prog: Option<&'a [u8]>) -> Result, Error> { + if let Some(prog) = prog { + verifier::check(prog)?; + } + + Ok(EbpfVmMbuff { + prog, + verifier: verifier::check, + #[cfg(all(not(windows), feature = "std"))] + jit: None, + #[cfg(feature = "cranelift")] + cranelift_prog: None, + helpers: HashMap::new(), + }) + } + + /// Load a new eBPF program into the virtual machine instance. + /// + /// # Examples + /// + /// ``` + /// let prog1 = &[ + /// 0xb7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov r0, 0 + /// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit + /// ]; + /// let prog2 = &[ + /// 0x79, 0x11, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, // Load mem from mbuff into R1. + /// 0x69, 0x10, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, // ldhx r1[2], r0 + /// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit + /// ]; + /// + /// // Instantiate a VM. + /// let mut vm = rbpf::EbpfVmMbuff::new(Some(prog1)).unwrap(); + /// vm.set_program(prog2).unwrap(); + /// ``` + pub fn set_program(&mut self, prog: &'a [u8]) -> Result<(), Error> { + (self.verifier)(prog)?; + self.prog = Some(prog); + Ok(()) + } + + /// Set a new verifier function. The function should return an `Error` if the program should be + /// rejected by the virtual machine. If a program has been loaded to the VM already, the + /// verifier is immediately run. + /// + /// # Examples + /// + /// ``` + /// use rbpf::{Error, ErrorKind}; + /// use rbpf::ebpf; + /// + /// // Define a simple verifier function. + /// fn verifier(prog: &[u8]) -> Result<(), Error> { + /// let last_insn = ebpf::get_insn(prog, (prog.len() / ebpf::INSN_SIZE) - 1); + /// if last_insn.opc != ebpf::EXIT { + /// return Err(Error::new(ErrorKind::Other, + /// "[Verifier] Error: program does not end with “EXIT” instruction")); + /// } + /// Ok(()) + /// } + /// + /// let prog1 = &[ + /// 0xb7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov r0, 0 + /// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit + /// ]; + /// + /// // Instantiate a VM. + /// let mut vm = rbpf::EbpfVmMbuff::new(Some(prog1)).unwrap(); + /// // Change the verifier. + /// vm.set_verifier(verifier).unwrap(); + /// ``` + pub fn set_verifier(&mut self, verifier: Verifier) -> Result<(), Error> { + if let Some(prog) = self.prog { + verifier(prog)?; + } + self.verifier = verifier; + Ok(()) + } + + /// Register a built-in or user-defined helper function in order to use it later from within + /// the eBPF program. The helper is registered into a hashmap, so the `key` can be any `u32`. + /// + /// If using JIT-compiled eBPF programs, be sure to register all helpers before compiling the + /// program. You should be able to change registered helpers after compiling, but not to add + /// new ones (i.e. with new keys). + /// + /// # Examples + /// + /// ``` + /// use rbpf::helpers; + /// + /// // This program was compiled with clang, from a C program containing the following single + /// // instruction: `return bpf_trace_printk("foo %c %c %c\n", 10, 1, 2, 3);` + /// let prog = &[ + /// 0x18, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // load 0 as u64 into r1 (That would be + /// 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // replaced by tc by the address of + /// // the format string, in the .map + /// // section of the ELF file). + /// 0xb7, 0x02, 0x00, 0x00, 0x0a, 0x00, 0x00, 0x00, // mov r2, 10 + /// 0xb7, 0x03, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, // mov r3, 1 + /// 0xb7, 0x04, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, // mov r4, 2 + /// 0xb7, 0x05, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, // mov r5, 3 + /// 0x85, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, // call helper with key 6 + /// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit + /// ]; + /// + /// // Instantiate a VM. + /// let mut vm = rbpf::EbpfVmMbuff::new(Some(prog)).unwrap(); + /// + /// // Register a helper. + /// // On running the program this helper will print the content of registers r3, r4 and r5 to + /// // standard output. + /// # #[cfg(feature = "std")] + /// vm.register_helper(6, helpers::bpf_trace_printf).unwrap(); + /// ``` + pub fn register_helper(&mut self, key: u32, function: Helper) -> Result<(), Error> { + self.helpers.insert(key, function); + Ok(()) + } + + /// Execute the program loaded, with the given packet data and metadata buffer. + /// + /// If the program is made to be compatible with Linux kernel, it is expected to load the + /// address of the beginning and of the end of the memory area used for packet data from the + /// metadata buffer, at some appointed offsets. It is up to the user to ensure that these + /// pointers are correctly stored in the buffer. + /// + /// # Examples + /// + /// ``` + /// let prog = &[ + /// 0x79, 0x11, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, // Load mem from mbuff into R1. + /// 0x69, 0x10, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, // ldhx r1[2], r0 + /// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit + /// ]; + /// let mem = &mut [ + /// 0xaa, 0xbb, 0x11, 0x22, 0xcc, 0xdd + /// ]; + /// + /// // Just for the example we create our metadata buffer from scratch, and we store the + /// // pointers to packet data start and end in it. + /// let mut mbuff = [0u8; 32]; + /// unsafe { + /// let mut data = mbuff.as_ptr().offset(8) as *mut u64; + /// let mut data_end = mbuff.as_ptr().offset(24) as *mut u64; + /// *data = mem.as_ptr() as u64; + /// *data_end = mem.as_ptr() as u64 + mem.len() as u64; + /// } + /// + /// // Instantiate a VM. + /// let mut vm = rbpf::EbpfVmMbuff::new(Some(prog)).unwrap(); + /// + /// // Provide both a reference to the packet data, and to the metadata buffer. + /// let res = vm.execute_program(mem, &mut mbuff).unwrap(); + /// assert_eq!(res, 0x2211); + /// ``` + pub fn execute_program(&self, mem: &[u8], mbuff: &[u8]) -> Result { + interpreter::execute_program(self.prog, mem, mbuff, &self.helpers) + } + + /// JIT-compile the loaded program. No argument required for this. + /// + /// If using helper functions, be sure to register them into the VM before calling this + /// function. + /// + /// # Examples + /// + /// ``` + /// let prog = &[ + /// 0x79, 0x11, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, // Load mem from mbuff into R1. + /// 0x69, 0x10, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, // ldhx r1[2], r0 + /// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit + /// ]; + /// + /// // Instantiate a VM. + /// let mut vm = rbpf::EbpfVmMbuff::new(Some(prog)).unwrap(); + /// + /// vm.jit_compile(); + /// ``` + #[cfg(all(not(windows), feature = "std"))] + pub fn jit_compile(&mut self) -> Result<(), Error> { + let prog = match self.prog { + Some(prog) => prog, + None => Err(Error::new( + ErrorKind::Other, + "Error: No program set, call prog_set() to load one", + ))?, + }; + self.jit = Some(jit::JitMemory::new(prog, &self.helpers, true, false)?); + Ok(()) + } + + /// Execute the previously JIT-compiled program, with the given packet data and metadata + /// buffer, in a manner very similar to `execute_program()`. + /// + /// If the program is made to be compatible with Linux kernel, it is expected to load the + /// address of the beginning and of the end of the memory area used for packet data from the + /// metadata buffer, at some appointed offsets. It is up to the user to ensure that these + /// pointers are correctly stored in the buffer. + /// + /// # Safety + /// + /// **WARNING:** JIT-compiled assembly code is not safe, in particular there is no runtime + /// check for memory access; so if the eBPF program attempts erroneous accesses, this may end + /// very bad (program may segfault). It may be wise to check that the program works with the + /// interpreter before running the JIT-compiled version of it. + /// + /// For this reason the function should be called from within an `unsafe` bloc. + /// + /// # Examples + /// + /// ``` + /// let prog = &[ + /// 0x79, 0x11, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, // Load mem from mbuff into r1. + /// 0x69, 0x10, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, // ldhx r1[2], r0 + /// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit + /// ]; + /// let mem = &mut [ + /// 0xaa, 0xbb, 0x11, 0x22, 0xcc, 0xdd + /// ]; + /// + /// // Just for the example we create our metadata buffer from scratch, and we store the + /// // pointers to packet data start and end in it. + /// let mut mbuff = [0u8; 32]; + /// unsafe { + /// let mut data = mbuff.as_ptr().offset(8) as *mut u64; + /// let mut data_end = mbuff.as_ptr().offset(24) as *mut u64; + /// *data = mem.as_ptr() as u64; + /// *data_end = mem.as_ptr() as u64 + mem.len() as u64; + /// } + /// + /// // Instantiate a VM. + /// let mut vm = rbpf::EbpfVmMbuff::new(Some(prog)).unwrap(); + /// + /// # #[cfg(all(not(windows), feature = "std"))] + /// vm.jit_compile(); + /// + /// // Provide both a reference to the packet data, and to the metadata buffer. + /// # #[cfg(all(not(windows), feature = "std"))] + /// unsafe { + /// let res = vm.execute_program_jit(mem, &mut mbuff).unwrap(); + /// assert_eq!(res, 0x2211); + /// } + /// ``` + #[cfg(all(not(windows), feature = "std"))] + pub unsafe fn execute_program_jit( + &self, + mem: &mut [u8], + mbuff: &'a mut [u8], + ) -> Result { + // If packet data is empty, do not send the address of an empty slice; send a null pointer + // as first argument instead, as this is uBPF's behavior (empty packet should not happen + // in the kernel; anyway the verifier would prevent the use of uninitialized registers). + // See `mul_loop` test. + let mem_ptr = match mem.len() { + 0 => std::ptr::null_mut(), + _ => mem.as_ptr() as *mut u8, + }; + // The last two arguments are not used in this function. They would be used if there was a + // need to indicate to the JIT at which offset in the mbuff mem_ptr and mem_ptr + mem.len() + // should be stored; this is what happens with struct EbpfVmFixedMbuff. + match &self.jit { + Some(jit) => Ok(jit.get_prog()( + mbuff.as_ptr() as *mut u8, + mbuff.len(), + mem_ptr, + mem.len(), + 0, + 0, + )), + None => Err(Error::new( + ErrorKind::Other, + "Error: program has not been JIT-compiled", + )), + } + } + + /// Compile the loaded program using the Cranelift JIT. + /// + /// If using helper functions, be sure to register them into the VM before calling this + /// function. + /// + /// # Examples + /// + /// ``` + /// let prog = &[ + /// 0x79, 0x11, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, // Load mem from mbuff into R1. + /// 0x69, 0x10, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, // ldhx r1[2], r0 + /// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit + /// ]; + /// + /// // Instantiate a VM. + /// let mut vm = rbpf::EbpfVmMbuff::new(Some(prog)).unwrap(); + /// + /// vm.cranelift_compile(); + /// ``` + #[cfg(feature = "cranelift")] + pub fn cranelift_compile(&mut self) -> Result<(), Error> { + use crate::cranelift::CraneliftCompiler; + + let prog = match self.prog { + Some(prog) => prog, + None => Err(Error::new( + ErrorKind::Other, + "Error: No program set, call prog_set() to load one", + ))?, + }; + + let mut compiler = CraneliftCompiler::new(self.helpers.clone()); + let program = compiler.compile_function(prog)?; + + self.cranelift_prog = Some(program); + Ok(()) + } + + /// Execute the previously compiled program, with the given packet data and metadata + /// buffer, in a manner very similar to `execute_program()`. + /// + /// If the program is made to be compatible with Linux kernel, it is expected to load the + /// address of the beginning and of the end of the memory area used for packet data from the + /// metadata buffer, at some appointed offsets. It is up to the user to ensure that these + /// pointers are correctly stored in the buffer. + /// + /// + /// # Examples + /// + /// ``` + /// let prog = &[ + /// 0x79, 0x11, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, // Load mem from mbuff into r1. + /// 0x69, 0x10, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, // ldhx r1[2], r0 + /// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit + /// ]; + /// let mem = &mut [ + /// 0xaa, 0xbb, 0x11, 0x22, 0xcc, 0xdd + /// ]; + /// + /// // Just for the example we create our metadata buffer from scratch, and we store the + /// // pointers to packet data start and end in it. + /// let mut mbuff = [0u8; 32]; + /// unsafe { + /// let mut data = mbuff.as_ptr().offset(8) as *mut u64; + /// let mut data_end = mbuff.as_ptr().offset(24) as *mut u64; + /// *data = mem.as_ptr() as u64; + /// *data_end = mem.as_ptr() as u64 + mem.len() as u64; + /// } + /// + /// // Instantiate a VM. + /// let mut vm = rbpf::EbpfVmMbuff::new(Some(prog)).unwrap(); + /// + /// vm.cranelift_compile(); + /// + /// // Provide both a reference to the packet data, and to the metadata buffer. + /// let res = vm.execute_program_cranelift(mem, &mut mbuff).unwrap(); + /// assert_eq!(res, 0x2211); + /// ``` + #[cfg(feature = "cranelift")] + pub fn execute_program_cranelift( + &self, + mem: &mut [u8], + mbuff: &'a mut [u8], + ) -> Result { + // If packet data is empty, do not send the address of an empty slice; send a null pointer + // as first argument instead, as this is uBPF's behavior (empty packet should not happen + // in the kernel; anyway the verifier would prevent the use of uninitialized registers). + // See `mul_loop` test. + let mem_ptr = match mem.len() { + 0 => core::ptr::null_mut(), + _ => mem.as_ptr() as *mut u8, + }; + + // The last two arguments are not used in this function. They would be used if there was a + // need to indicate to the JIT at which offset in the mbuff mem_ptr and mem_ptr + mem.len() + // should be stored; this is what happens with struct EbpfVmFixedMbuff. + match &self.cranelift_prog { + Some(prog) => { + Ok(prog.execute(mem_ptr, mem.len(), mbuff.as_ptr() as *mut u8, mbuff.len())) + } + None => Err(Error::new( + ErrorKind::Other, + "Error: program has not been compiled with cranelift", + )), + } + } +} + +/// A virtual machine to run eBPF program. This kind of VM is used for programs expecting to work +/// on a metadata buffer containing pointers to packet data, but it internally handles the buffer +/// so as to save the effort to manually handle the metadata buffer for the user. +/// +/// This struct implements a static internal buffer that is passed to the program. The user has to +/// indicate the offset values at which the eBPF program expects to find the start and the end of +/// packet data in the buffer. On calling the `execute_program()` or `execute_program_jit()` functions, the +/// struct automatically updates the addresses in this static buffer, at the appointed offsets, for +/// the start and the end of the packet data the program is called upon. +/// +/// # Examples +/// +/// This was compiled with clang from the following program, in C: +/// +/// ```c +/// #include +/// #include "path/to/linux/samples/bpf/bpf_helpers.h" +/// +/// SEC(".classifier") +/// int classifier(struct __sk_buff *skb) +/// { +/// void *data = (void *)(long)skb->data; +/// void *data_end = (void *)(long)skb->data_end; +/// +/// // Check program is long enough. +/// if (data + 5 > data_end) +/// return 0; +/// +/// return *((char *)data + 5); +/// } +/// ``` +/// +/// Some small modifications have been brought to have it work, see comments. +/// +/// ``` +/// let prog = &[ +/// 0xb7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov r0, 0 +/// // Here opcode 0x61 had to be replace by 0x79 so as to load a 8-bytes long address. +/// // Also, offset 0x4c had to be replace with e.g. 0x40 so as to prevent the two pointers +/// // from overlapping in the buffer. +/// 0x79, 0x12, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, // load pointer to mem from r1[0x40] to r2 +/// 0x07, 0x02, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, // add r2, 5 +/// // Here opcode 0x61 had to be replace by 0x79 so as to load a 8-bytes long address. +/// 0x79, 0x11, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, // load ptr to mem_end from r1[0x50] to r1 +/// 0x2d, 0x12, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, // if r2 > r1 skip 3 instructions +/// 0x71, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // load r2 (= *(mem + 5)) into r0 +/// 0x67, 0x00, 0x00, 0x00, 0x38, 0x00, 0x00, 0x00, // r0 >>= 56 +/// 0xc7, 0x00, 0x00, 0x00, 0x38, 0x00, 0x00, 0x00, // r0 <<= 56 (arsh) extend byte sign to u64 +/// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit +/// ]; +/// let mem1 = &mut [ +/// 0xaa, 0xbb, 0x11, 0x22, 0xcc, 0xdd +/// ]; +/// let mem2 = &mut [ +/// 0xaa, 0xbb, 0x11, 0x22, 0xcc, 0x27 +/// ]; +/// +/// // Instantiate a VM. Note that we provide the start and end offsets for mem pointers. +/// let mut vm = rbpf::EbpfVmFixedMbuff::new(Some(prog), 0x40, 0x50).unwrap(); +/// +/// // Provide only a reference to the packet data. We do not manage the metadata buffer. +/// let res = vm.execute_program(mem1).unwrap(); +/// assert_eq!(res, 0xffffffffffffffdd); +/// +/// let res = vm.execute_program(mem2).unwrap(); +/// assert_eq!(res, 0x27); +/// ``` +pub struct EbpfVmFixedMbuff<'a> { + parent: EbpfVmMbuff<'a>, + mbuff: MetaBuff, +} + +impl<'a> EbpfVmFixedMbuff<'a> { + /// Create a new virtual machine instance, and load an eBPF program into that instance. + /// When attempting to load the program, it passes through a simple verifier. + /// + /// # Examples + /// + /// ``` + /// let prog = &[ + /// 0xb7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov r0, 0 + /// 0x79, 0x12, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, // load mem from r1[0x40] to r2 + /// 0x07, 0x02, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, // add r2, 5 + /// 0x79, 0x11, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, // load mem_end from r1[0x50] to r1 + /// 0x2d, 0x12, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, // if r2 > r1 skip 3 instructions + /// 0x71, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // load r2 (= *(mem + 5)) into r0 + /// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit + /// ]; + /// + /// // Instantiate a VM. Note that we provide the start and end offsets for mem pointers. + /// let mut vm = rbpf::EbpfVmFixedMbuff::new(Some(prog), 0x40, 0x50).unwrap(); + /// ``` + pub fn new( + prog: Option<&'a [u8]>, + data_offset: usize, + data_end_offset: usize, + ) -> Result, Error> { + let parent = EbpfVmMbuff::new(prog)?; + let get_buff_len = |x: usize, y: usize| if x >= y { x + 8 } else { y + 8 }; + let buffer = vec![0u8; get_buff_len(data_offset, data_end_offset)]; + let mbuff = MetaBuff { + data_offset, + data_end_offset, + buffer, + }; + Ok(EbpfVmFixedMbuff { parent, mbuff }) + } + + /// Load a new eBPF program into the virtual machine instance. + /// + /// At the same time, load new offsets for storing pointers to start and end of packet data in + /// the internal metadata buffer. + /// + /// # Examples + /// + /// ``` + /// let prog1 = &[ + /// 0xb7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov r0, 0 + /// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit + /// ]; + /// let prog2 = &[ + /// 0xb7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov r0, 0 + /// 0x79, 0x12, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, // load mem from r1[0x40] to r2 + /// 0x07, 0x02, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, // add r2, 5 + /// 0x79, 0x11, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, // load mem_end from r1[0x50] to r1 + /// 0x2d, 0x12, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, // if r2 > r1 skip 3 instructions + /// 0x71, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // load r2 (= *(mem + 5)) into r0 + /// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit + /// ]; + /// + /// let mem = &mut [ + /// 0xaa, 0xbb, 0x11, 0x22, 0xcc, 0x27, + /// ]; + /// + /// let mut vm = rbpf::EbpfVmFixedMbuff::new(Some(prog1), 0, 0).unwrap(); + /// vm.set_program(prog2, 0x40, 0x50); + /// + /// let res = vm.execute_program(mem).unwrap(); + /// assert_eq!(res, 0x27); + /// ``` + pub fn set_program( + &mut self, + prog: &'a [u8], + data_offset: usize, + data_end_offset: usize, + ) -> Result<(), Error> { + let get_buff_len = |x: usize, y: usize| if x >= y { x + 8 } else { y + 8 }; + let buffer = vec![0u8; get_buff_len(data_offset, data_end_offset)]; + self.mbuff.buffer = buffer; + self.mbuff.data_offset = data_offset; + self.mbuff.data_end_offset = data_end_offset; + self.parent.set_program(prog)?; + Ok(()) + } + + /// Set a new verifier function. The function should return an `Error` if the program should be + /// rejected by the virtual machine. If a program has been loaded to the VM already, the + /// verifier is immediately run. + /// + /// # Examples + /// + /// ``` + /// use rbpf::{Error, ErrorKind}; + /// use rbpf::ebpf; + /// + /// // Define a simple verifier function. + /// fn verifier(prog: &[u8]) -> Result<(), Error> { + /// let last_insn = ebpf::get_insn(prog, (prog.len() / ebpf::INSN_SIZE) - 1); + /// if last_insn.opc != ebpf::EXIT { + /// return Err(Error::new(ErrorKind::Other, + /// "[Verifier] Error: program does not end with “EXIT” instruction")); + /// } + /// Ok(()) + /// } + /// + /// let prog1 = &[ + /// 0xb7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov r0, 0 + /// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit + /// ]; + /// + /// // Instantiate a VM. + /// let mut vm = rbpf::EbpfVmMbuff::new(Some(prog1)).unwrap(); + /// // Change the verifier. + /// vm.set_verifier(verifier).unwrap(); + /// ``` + pub fn set_verifier(&mut self, verifier: Verifier) -> Result<(), Error> { + self.parent.set_verifier(verifier) + } + + /// Register a built-in or user-defined helper function in order to use it later from within + /// the eBPF program. The helper is registered into a hashmap, so the `key` can be any `u32`. + /// + /// If using JIT-compiled eBPF programs, be sure to register all helpers before compiling the + /// program. You should be able to change registered helpers after compiling, but not to add + /// new ones (i.e. with new keys). + /// + /// # Examples + /// + /// ``` + /// #[cfg(feature = "std")] { + /// use rbpf::helpers; + /// + /// // This program was compiled with clang, from a C program containing the following single + /// // instruction: `return bpf_trace_printk("foo %c %c %c\n", 10, 1, 2, 3);` + /// let prog = &[ + /// 0xb7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov r0, 0 + /// 0x79, 0x12, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, // load mem from r1[0x40] to r2 + /// 0x07, 0x02, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, // add r2, 5 + /// 0x79, 0x11, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, // load mem_end from r1[0x50] to r1 + /// 0x2d, 0x12, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, // if r2 > r1 skip 6 instructions + /// 0x71, 0x21, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // load r2 (= *(mem + 5)) into r1 + /// 0xb7, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov r2, 0 + /// 0xb7, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov r3, 0 + /// 0xb7, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov r4, 0 + /// 0xb7, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov r5, 0 + /// 0x85, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, // call helper with key 1 + /// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit + /// ]; + /// + /// let mem = &mut [ + /// 0xaa, 0xbb, 0x11, 0x22, 0xcc, 0x09, + /// ]; + /// + /// // Instantiate a VM. + /// let mut vm = rbpf::EbpfVmFixedMbuff::new(Some(prog), 0x40, 0x50).unwrap(); + /// + /// // Register a helper. This helper will store the result of the square root of r1 into r0. + /// vm.register_helper(1, helpers::sqrti); + /// + /// let res = vm.execute_program(mem).unwrap(); + /// assert_eq!(res, 3); + /// } + /// ``` + pub fn register_helper( + &mut self, + key: u32, + function: fn(u64, u64, u64, u64, u64) -> u64, + ) -> Result<(), Error> { + self.parent.register_helper(key, function) + } + + /// Execute the program loaded, with the given packet data. + /// + /// If the program is made to be compatible with Linux kernel, it is expected to load the + /// address of the beginning and of the end of the memory area used for packet data from some + /// metadata buffer, which in the case of this VM is handled internally. The offsets at which + /// the addresses should be placed should have be set at the creation of the VM. + /// + /// # Examples + /// + /// ``` + /// let prog = &[ + /// 0xb7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov r0, 0 + /// 0x79, 0x12, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, // load mem from r1[0x40] to r2 + /// 0x07, 0x02, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, // add r2, 5 + /// 0x79, 0x11, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, // load mem_end from r1[0x50] to r1 + /// 0x2d, 0x12, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, // if r2 > r1 skip 3 instructions + /// 0x71, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // load r2 (= *(mem + 5)) into r0 + /// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit + /// ]; + /// let mem = &mut [ + /// 0xaa, 0xbb, 0x11, 0x22, 0xcc, 0xdd + /// ]; + /// + /// // Instantiate a VM. Note that we provide the start and end offsets for mem pointers. + /// let mut vm = rbpf::EbpfVmFixedMbuff::new(Some(prog), 0x40, 0x50).unwrap(); + /// + /// // Provide only a reference to the packet data. We do not manage the metadata buffer. + /// let res = vm.execute_program(mem).unwrap(); + /// assert_eq!(res, 0xdd); + /// ``` + pub fn execute_program(&mut self, mem: &'a mut [u8]) -> Result { + let l = self.mbuff.buffer.len(); + // Can this ever happen? Probably not, should be ensured at mbuff creation. + if self.mbuff.data_offset + 8 > l || self.mbuff.data_end_offset + 8 > l { + Err(Error::new(ErrorKind::Other, format!("Error: buffer too small ({:?}), cannot use data_offset {:?} and data_end_offset {:?}", + l, self.mbuff.data_offset, self.mbuff.data_end_offset)))?; + } + LittleEndian::write_u64( + &mut self.mbuff.buffer[(self.mbuff.data_offset)..], + mem.as_ptr() as u64, + ); + LittleEndian::write_u64( + &mut self.mbuff.buffer[(self.mbuff.data_end_offset)..], + mem.as_ptr() as u64 + mem.len() as u64, + ); + self.parent.execute_program(mem, &self.mbuff.buffer) + } + + /// JIT-compile the loaded program. No argument required for this. + /// + /// If using helper functions, be sure to register them into the VM before calling this + /// function. + /// + /// # Examples + /// + /// ``` + /// let prog = &[ + /// 0xb7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov r0, 0 + /// 0x79, 0x12, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, // load mem from r1[0x40] to r2 + /// 0x07, 0x02, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, // add r2, 5 + /// 0x79, 0x11, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, // load mem_end from r1[0x50] to r1 + /// 0x2d, 0x12, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, // if r2 > r1 skip 3 instructions + /// 0x71, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // load r2 (= *(mem + 5)) into r0 + /// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit + /// ]; + /// + /// // Instantiate a VM. Note that we provide the start and end offsets for mem pointers. + /// let mut vm = rbpf::EbpfVmFixedMbuff::new(Some(prog), 0x40, 0x50).unwrap(); + /// + /// vm.jit_compile(); + /// ``` + #[cfg(all(not(windows), feature = "std"))] + pub fn jit_compile(&mut self) -> Result<(), Error> { + let prog = match self.parent.prog { + Some(prog) => prog, + None => Err(Error::new( + ErrorKind::Other, + "Error: No program set, call prog_set() to load one", + ))?, + }; + self.parent.jit = Some(jit::JitMemory::new(prog, &self.parent.helpers, true, true)?); + Ok(()) + } + + /// Execute the previously JIT-compiled program, with the given packet data, in a manner very + /// similar to `execute_program()`. + /// + /// If the program is made to be compatible with Linux kernel, it is expected to load the + /// address of the beginning and of the end of the memory area used for packet data from some + /// metadata buffer, which in the case of this VM is handled internally. The offsets at which + /// the addresses should be placed should have be set at the creation of the VM. + /// + /// # Safety + /// + /// **WARNING:** JIT-compiled assembly code is not safe, in particular there is no runtime + /// check for memory access; so if the eBPF program attempts erroneous accesses, this may end + /// very bad (program may segfault). It may be wise to check that the program works with the + /// interpreter before running the JIT-compiled version of it. + /// + /// For this reason the function should be called from within an `unsafe` bloc. + /// + /// # Examples + /// + /// ``` + /// let prog = &[ + /// 0xb7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov r0, 0 + /// 0x79, 0x12, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, // load mem from r1[0x40] to r2 + /// 0x07, 0x02, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, // add r2, 5 + /// 0x79, 0x11, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, // load mem_end from r1[0x50] to r1 + /// 0x2d, 0x12, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, // if r2 > r1 skip 3 instructions + /// 0x71, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // load r2 (= *(mem + 5)) into r0 + /// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit + /// ]; + /// let mem = &mut [ + /// 0xaa, 0xbb, 0x11, 0x22, 0xcc, 0xdd + /// ]; + /// + /// // Instantiate a VM. Note that we provide the start and end offsets for mem pointers. + /// let mut vm = rbpf::EbpfVmFixedMbuff::new(Some(prog), 0x40, 0x50).unwrap(); + /// + /// # #[cfg(all(not(windows), feature = "std"))] + /// vm.jit_compile(); + /// + /// // Provide only a reference to the packet data. We do not manage the metadata buffer. + /// # #[cfg(all(not(windows), feature = "std"))] + /// unsafe { + /// let res = vm.execute_program_jit(mem).unwrap(); + /// assert_eq!(res, 0xdd); + /// } + /// ``` + // This struct redefines the `execute_program_jit()` function, in order to pass the offsets + // associated with the fixed mbuff. + #[cfg(all(not(windows), feature = "std"))] + pub unsafe fn execute_program_jit(&mut self, mem: &'a mut [u8]) -> Result { + // If packet data is empty, do not send the address of an empty slice; send a null pointer + // as first argument instead, as this is uBPF's behavior (empty packet should not happen + // in the kernel; anyway the verifier would prevent the use of uninitialized registers). + // See `mul_loop` test. + let mem_ptr = match mem.len() { + 0 => core::ptr::null_mut(), + _ => mem.as_ptr() as *mut u8, + }; + + match &self.parent.jit { + Some(jit) => Ok(jit.get_prog()( + self.mbuff.buffer.as_ptr() as *mut u8, + self.mbuff.buffer.len(), + mem_ptr, + mem.len(), + self.mbuff.data_offset, + self.mbuff.data_end_offset, + )), + None => Err(Error::new( + ErrorKind::Other, + "Error: program has not been JIT-compiled", + )), + } + } + + /// Compile the loaded program using the Cranelift JIT. + /// + /// If using helper functions, be sure to register them into the VM before calling this + /// function. + /// + /// # Examples + /// + /// ``` + /// let prog = &[ + /// 0xb7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov r0, 0 + /// 0x79, 0x12, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, // load mem from r1[0x40] to r2 + /// 0x07, 0x02, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, // add r2, 5 + /// 0x79, 0x11, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, // load mem_end from r1[0x50] to r1 + /// 0x2d, 0x12, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, // if r2 > r1 skip 3 instructions + /// 0x71, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // load r2 (= *(mem + 5)) into r0 + /// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit + /// ]; + /// + /// // Instantiate a VM. Note that we provide the start and end offsets for mem pointers. + /// let mut vm = rbpf::EbpfVmFixedMbuff::new(Some(prog), 0x40, 0x50).unwrap(); + /// + /// vm.cranelift_compile(); + /// ``` + #[cfg(feature = "cranelift")] + pub fn cranelift_compile(&mut self) -> Result<(), Error> { + use crate::cranelift::CraneliftCompiler; + + let prog = match self.parent.prog { + Some(prog) => prog, + None => Err(Error::new( + ErrorKind::Other, + "Error: No program set, call prog_set() to load one", + ))?, + }; + + let mut compiler = CraneliftCompiler::new(self.parent.helpers.clone()); + let program = compiler.compile_function(prog)?; + + self.parent.cranelift_prog = Some(program); + Ok(()) + } + + /// Execute the previously compiled program, with the given packet data and metadata + /// buffer, in a manner very similar to `execute_program()`. + /// + /// If the program is made to be compatible with Linux kernel, it is expected to load the + /// address of the beginning and of the end of the memory area used for packet data from some + /// metadata buffer, which in the case of this VM is handled internally. The offsets at which + /// the addresses should be placed should have be set at the creation of the VM. + /// + /// # Examples + /// + /// ``` + /// let prog = &[ + /// 0xb7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov r0, 0 + /// 0x79, 0x12, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, // load mem from r1[0x40] to r2 + /// 0x07, 0x02, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, // add r2, 5 + /// 0x79, 0x11, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, // load mem_end from r1[0x50] to r1 + /// 0x2d, 0x12, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, // if r2 > r1 skip 3 instructions + /// 0x71, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // load r2 (= *(mem + 5)) into r0 + /// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit + /// ]; + /// let mem = &mut [ + /// 0xaa, 0xbb, 0x11, 0x22, 0xcc, 0xdd + /// ]; + /// + /// // Instantiate a VM. Note that we provide the start and end offsets for mem pointers. + /// let mut vm = rbpf::EbpfVmFixedMbuff::new(Some(prog), 0x40, 0x50).unwrap(); + /// + /// vm.cranelift_compile(); + /// + /// // Provide only a reference to the packet data. We do not manage the metadata buffer. + /// let res = vm.execute_program_cranelift(mem).unwrap(); + /// assert_eq!(res, 0xdd); + /// ``` + #[cfg(feature = "cranelift")] + pub fn execute_program_cranelift(&mut self, mem: &'a mut [u8]) -> Result { + // If packet data is empty, do not send the address of an empty slice; send a null pointer + // as first argument instead, as this is uBPF's behavior (empty packet should not happen + // in the kernel; anyway the verifier would prevent the use of uninitialized registers). + // See `mul_loop` test. + let mem_ptr = match mem.len() { + 0 => core::ptr::null_mut(), + _ => mem.as_ptr() as *mut u8, + }; + + let l = self.mbuff.buffer.len(); + // Can this ever happen? Probably not, should be ensured at mbuff creation. + if self.mbuff.data_offset + 8 > l || self.mbuff.data_end_offset + 8 > l { + Err(Error::new(ErrorKind::Other, format!("Error: buffer too small ({:?}), cannot use data_offset {:?} and data_end_offset {:?}", + l, self.mbuff.data_offset, self.mbuff.data_end_offset)))?; + } + LittleEndian::write_u64( + &mut self.mbuff.buffer[(self.mbuff.data_offset)..], + mem.as_ptr() as u64, + ); + LittleEndian::write_u64( + &mut self.mbuff.buffer[(self.mbuff.data_end_offset)..], + mem.as_ptr() as u64 + mem.len() as u64, + ); + + match &self.parent.cranelift_prog { + Some(prog) => Ok(prog.execute( + mem_ptr, + mem.len(), + self.mbuff.buffer.as_ptr() as *mut u8, + self.mbuff.buffer.len(), + )), + None => Err(Error::new( + ErrorKind::Other, + "Error: program has not been compiled with cranelift", + )), + } + } +} + +/// A virtual machine to run eBPF program. This kind of VM is used for programs expecting to work +/// directly on the memory area representing packet data. +/// +/// # Examples +/// +/// ``` +/// let prog = &[ +/// 0x71, 0x11, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, // ldxb r1[0x04], r1 +/// 0x07, 0x01, 0x00, 0x00, 0x00, 0x22, 0x00, 0x00, // add r1, 0x22 +/// 0xbf, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov r0, r1 +/// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit +/// ]; +/// let mem = &mut [ +/// 0xaa, 0xbb, 0x11, 0x22, 0xcc, 0xdd +/// ]; +/// +/// // Instantiate a VM. +/// let vm = rbpf::EbpfVmRaw::new(Some(prog)).unwrap(); +/// +/// // Provide only a reference to the packet data. +/// let res = vm.execute_program(mem).unwrap(); +/// assert_eq!(res, 0x22cc); +/// ``` +pub struct EbpfVmRaw<'a> { + parent: EbpfVmMbuff<'a>, +} + +impl<'a> EbpfVmRaw<'a> { + /// Create a new virtual machine instance, and load an eBPF program into that instance. + /// When attempting to load the program, it passes through a simple verifier. + /// + /// # Examples + /// + /// ``` + /// let prog = &[ + /// 0x71, 0x11, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, // ldxb r1[0x04], r1 + /// 0x07, 0x01, 0x00, 0x00, 0x00, 0x22, 0x00, 0x00, // add r1, 0x22 + /// 0xbf, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov r0, r1 + /// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit + /// ]; + /// + /// // Instantiate a VM. + /// let vm = rbpf::EbpfVmRaw::new(Some(prog)).unwrap(); + /// ``` + pub fn new(prog: Option<&'a [u8]>) -> Result, Error> { + let parent = EbpfVmMbuff::new(prog)?; + Ok(EbpfVmRaw { parent }) + } + + /// Load a new eBPF program into the virtual machine instance. + /// + /// # Examples + /// + /// ``` + /// let prog1 = &[ + /// 0xb7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov r0, 0 + /// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit + /// ]; + /// let prog2 = &[ + /// 0x71, 0x11, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, // ldxb r1[0x04], r1 + /// 0x07, 0x01, 0x00, 0x00, 0x00, 0x22, 0x00, 0x00, // add r1, 0x22 + /// 0xbf, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov r0, r1 + /// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit + /// ]; + /// + /// let mem = &mut [ + /// 0xaa, 0xbb, 0x11, 0x22, 0xcc, 0x27, + /// ]; + /// + /// let mut vm = rbpf::EbpfVmRaw::new(Some(prog1)).unwrap(); + /// vm.set_program(prog2); + /// + /// let res = vm.execute_program(mem).unwrap(); + /// assert_eq!(res, 0x22cc); + /// ``` + pub fn set_program(&mut self, prog: &'a [u8]) -> Result<(), Error> { + self.parent.set_program(prog)?; + Ok(()) + } + + /// Set a new verifier function. The function should return an `Error` if the program should be + /// rejected by the virtual machine. If a program has been loaded to the VM already, the + /// verifier is immediately run. + /// + /// # Examples + /// + /// ``` + /// use rbpf::{Error, ErrorKind}; + /// use rbpf::ebpf; + /// + /// // Define a simple verifier function. + /// fn verifier(prog: &[u8]) -> Result<(), Error> { + /// let last_insn = ebpf::get_insn(prog, (prog.len() / ebpf::INSN_SIZE) - 1); + /// if last_insn.opc != ebpf::EXIT { + /// return Err(Error::new(ErrorKind::Other, + /// "[Verifier] Error: program does not end with “EXIT” instruction")); + /// } + /// Ok(()) + /// } + /// + /// let prog1 = &[ + /// 0xb7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov r0, 0 + /// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit + /// ]; + /// + /// // Instantiate a VM. + /// let mut vm = rbpf::EbpfVmMbuff::new(Some(prog1)).unwrap(); + /// // Change the verifier. + /// vm.set_verifier(verifier).unwrap(); + /// ``` + pub fn set_verifier(&mut self, verifier: Verifier) -> Result<(), Error> { + self.parent.set_verifier(verifier) + } + + /// Register a built-in or user-defined helper function in order to use it later from within + /// the eBPF program. The helper is registered into a hashmap, so the `key` can be any `u32`. + /// + /// If using JIT-compiled eBPF programs, be sure to register all helpers before compiling the + /// program. You should be able to change registered helpers after compiling, but not to add + /// new ones (i.e. with new keys). + /// + /// # Examples + /// + /// ``` + /// #[cfg(feature = "std")] { + /// use rbpf::helpers; + /// + /// let prog = &[ + /// 0x79, 0x11, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // ldxdw r1, r1[0x00] + /// 0xb7, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov r2, 0 + /// 0xb7, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov r3, 0 + /// 0xb7, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov r4, 0 + /// 0xb7, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov r5, 0 + /// 0x85, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, // call helper with key 1 + /// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit + /// ]; + /// + /// let mem = &mut [ + /// 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01 + /// ]; + /// + /// // Instantiate a VM. + /// let mut vm = rbpf::EbpfVmRaw::new(Some(prog)).unwrap(); + /// + /// // Register a helper. This helper will store the result of the square root of r1 into r0. + /// vm.register_helper(1, helpers::sqrti); + /// + /// let res = vm.execute_program(mem).unwrap(); + /// assert_eq!(res, 0x10000000); + /// } + /// ``` + pub fn register_helper( + &mut self, + key: u32, + function: fn(u64, u64, u64, u64, u64) -> u64, + ) -> Result<(), Error> { + self.parent.register_helper(key, function) + } + + /// Register a set of built-in or user-defined helper functions in order to use them later from + /// within the eBPF program. The helpers are registered into a hashmap, so the `key` can be any + /// `u32`. + #[allow(clippy::type_complexity)] + pub fn register_helper_set( + &mut self, + helpers: &HashMap u64>, + ) -> Result<(), Error> { + for (key, function) in helpers { + self.parent.register_helper(*key, *function)?; + } + Ok(()) + } + + /// Execute the program loaded, with the given packet data. + /// + /// # Examples + /// + /// ``` + /// let prog = &[ + /// 0x71, 0x11, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, // ldxb r1[0x04], r1 + /// 0x07, 0x01, 0x00, 0x00, 0x00, 0x22, 0x00, 0x00, // add r1, 0x22 + /// 0xbf, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov r0, r1 + /// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit + /// ]; + /// + /// let mem = &mut [ + /// 0xaa, 0xbb, 0x11, 0x22, 0xcc, 0x27 + /// ]; + /// + /// let mut vm = rbpf::EbpfVmRaw::new(Some(prog)).unwrap(); + /// + /// let res = vm.execute_program(mem).unwrap(); + /// assert_eq!(res, 0x22cc); + /// ``` + pub fn execute_program(&self, mem: &'a mut [u8]) -> Result { + self.parent.execute_program(mem, &[]) + } + + /// JIT-compile the loaded program. No argument required for this. + /// + /// If using helper functions, be sure to register them into the VM before calling this + /// function. + /// + /// # Examples + /// + /// ``` + /// let prog = &[ + /// 0x71, 0x11, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, // ldxb r1[0x04], r1 + /// 0x07, 0x01, 0x00, 0x00, 0x00, 0x22, 0x00, 0x00, // add r1, 0x22 + /// 0xbf, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov r0, r1 + /// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit + /// ]; + /// + /// let mut vm = rbpf::EbpfVmRaw::new(Some(prog)).unwrap(); + /// + /// vm.jit_compile(); + /// ``` + #[cfg(all(not(windows), feature = "std"))] + pub fn jit_compile(&mut self) -> Result<(), Error> { + let prog = match self.parent.prog { + Some(prog) => prog, + None => Err(Error::new( + ErrorKind::Other, + "Error: No program set, call prog_set() to load one", + ))?, + }; + self.parent.jit = Some(jit::JitMemory::new( + prog, + &self.parent.helpers, + false, + false, + )?); + Ok(()) + } + + /// Execute the previously JIT-compiled program, with the given packet data, in a manner very + /// similar to `execute_program()`. + /// + /// # Safety + /// + /// **WARNING:** JIT-compiled assembly code is not safe, in particular there is no runtime + /// check for memory access; so if the eBPF program attempts erroneous accesses, this may end + /// very bad (program may segfault). It may be wise to check that the program works with the + /// interpreter before running the JIT-compiled version of it. + /// + /// For this reason the function should be called from within an `unsafe` bloc. + /// + /// # Examples + /// + /// ``` + /// let prog = &[ + /// 0x71, 0x11, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, // ldxb r1[0x04], r1 + /// 0x07, 0x01, 0x00, 0x00, 0x00, 0x22, 0x00, 0x00, // add r1, 0x22 + /// 0xbf, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov r0, r1 + /// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit + /// ]; + /// + /// let mem = &mut [ + /// 0xaa, 0xbb, 0x11, 0x22, 0xcc, 0x27 + /// ]; + /// + /// let mut vm = rbpf::EbpfVmRaw::new(Some(prog)).unwrap(); + /// + /// # #[cfg(all(not(windows), feature = "std"))] + /// vm.jit_compile(); + /// + /// # #[cfg(all(not(windows), feature = "std"))] + /// unsafe { + /// let res = vm.execute_program_jit(mem).unwrap(); + /// assert_eq!(res, 0x22cc); + /// } + /// ``` + #[cfg(all(not(windows), feature = "std"))] + pub unsafe fn execute_program_jit(&self, mem: &'a mut [u8]) -> Result { + let mut mbuff = vec![]; + self.parent.execute_program_jit(mem, &mut mbuff) + } + + /// Compile the loaded program using the Cranelift JIT. + /// + /// If using helper functions, be sure to register them into the VM before calling this + /// function. + /// + /// # Examples + /// + /// ``` + /// let prog = &[ + /// 0x71, 0x11, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, // ldxb r1[0x04], r1 + /// 0x07, 0x01, 0x00, 0x00, 0x00, 0x22, 0x00, 0x00, // add r1, 0x22 + /// 0xbf, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov r0, r1 + /// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit + /// ]; + /// + /// let mut vm = rbpf::EbpfVmRaw::new(Some(prog)).unwrap(); + /// + /// vm.cranelift_compile(); + /// ``` + #[cfg(feature = "cranelift")] + pub fn cranelift_compile(&mut self) -> Result<(), Error> { + use crate::cranelift::CraneliftCompiler; + + let prog = match self.parent.prog { + Some(prog) => prog, + None => Err(Error::new( + ErrorKind::Other, + "Error: No program set, call prog_set() to load one", + ))?, + }; + + let mut compiler = CraneliftCompiler::new(self.parent.helpers.clone()); + let program = compiler.compile_function(prog)?; + + self.parent.cranelift_prog = Some(program); + Ok(()) + } + + /// Execute the previously compiled program, with the given packet data, in a manner very + /// similar to `execute_program()`. + /// + /// # Examples + /// + /// ``` + /// let prog = &[ + /// 0x71, 0x11, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, // ldxb r1[0x04], r1 + /// 0x07, 0x01, 0x00, 0x00, 0x00, 0x22, 0x00, 0x00, // add r1, 0x22 + /// 0xbf, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov r0, r1 + /// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit + /// ]; + /// + /// let mem = &mut [ + /// 0xaa, 0xbb, 0x11, 0x22, 0xcc, 0x27 + /// ]; + /// + /// let mut vm = rbpf::EbpfVmRaw::new(Some(prog)).unwrap(); + /// + /// vm.cranelift_compile(); + /// + /// let res = vm.execute_program_cranelift(mem).unwrap(); + /// assert_eq!(res, 0x22cc); + /// ``` + #[cfg(feature = "cranelift")] + pub fn execute_program_cranelift(&self, mem: &'a mut [u8]) -> Result { + let mut mbuff = vec![]; + self.parent.execute_program_cranelift(mem, &mut mbuff) + } +} + +/// A virtual machine to run eBPF program. This kind of VM is used for programs that do not work +/// with any memory area—no metadata buffer, no packet data either. +/// +/// # Examples +/// +/// ``` +/// let prog = &[ +/// 0xb7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov r0, 0 +/// 0xb7, 0x01, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, // mov r1, 1 +/// 0xb7, 0x02, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, // mov r2, 2 +/// 0xb7, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, // mov r3, 3 +/// 0xb7, 0x04, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, // mov r4, 4 +/// 0xb7, 0x05, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, // mov r5, 5 +/// 0xb7, 0x06, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, // mov r6, 6 +/// 0xb7, 0x07, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, // mov r7, 7 +/// 0xb7, 0x08, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, // mov r8, 8 +/// 0x4f, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // or r0, r5 +/// 0x47, 0x00, 0x00, 0x00, 0xa0, 0x00, 0x00, 0x00, // or r0, 0xa0 +/// 0x57, 0x00, 0x00, 0x00, 0xa3, 0x00, 0x00, 0x00, // and r0, 0xa3 +/// 0xb7, 0x09, 0x00, 0x00, 0x91, 0x00, 0x00, 0x00, // mov r9, 0x91 +/// 0x5f, 0x90, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // and r0, r9 +/// 0x67, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, // lsh r0, 32 +/// 0x67, 0x00, 0x00, 0x00, 0x16, 0x00, 0x00, 0x00, // lsh r0, 22 +/// 0x6f, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // lsh r0, r8 +/// 0x77, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, // rsh r0, 32 +/// 0x77, 0x00, 0x00, 0x00, 0x13, 0x00, 0x00, 0x00, // rsh r0, 19 +/// 0x7f, 0x70, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // rsh r0, r7 +/// 0xa7, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, // xor r0, 0x03 +/// 0xaf, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // xor r0, r2 +/// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit +/// ]; +/// +/// // Instantiate a VM. +/// let vm = rbpf::EbpfVmNoData::new(Some(prog)).unwrap(); +/// +/// // Provide only a reference to the packet data. +/// let res = vm.execute_program().unwrap(); +/// assert_eq!(res, 0x11); +/// ``` +pub struct EbpfVmNoData<'a> { + parent: EbpfVmRaw<'a>, +} + +impl<'a> EbpfVmNoData<'a> { + /// Create a new virtual machine instance, and load an eBPF program into that instance. + /// When attempting to load the program, it passes through a simple verifier. + /// + /// # Examples + /// + /// ``` + /// let prog = &[ + /// 0xb7, 0x00, 0x00, 0x00, 0x11, 0x22, 0x00, 0x00, // mov r0, 0x2211 + /// 0xdc, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, // be16 r0 + /// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit + /// ]; + /// + /// // Instantiate a VM. + /// let vm = rbpf::EbpfVmNoData::new(Some(prog)); + /// ``` + pub fn new(prog: Option<&'a [u8]>) -> Result, Error> { + let parent = EbpfVmRaw::new(prog)?; + Ok(EbpfVmNoData { parent }) + } + + /// Load a new eBPF program into the virtual machine instance. + /// + /// # Examples + /// + /// ``` + /// let prog1 = &[ + /// 0xb7, 0x00, 0x00, 0x00, 0x11, 0x22, 0x00, 0x00, // mov r0, 0x2211 + /// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit + /// ]; + /// let prog2 = &[ + /// 0xb7, 0x00, 0x00, 0x00, 0x11, 0x22, 0x00, 0x00, // mov r0, 0x2211 + /// 0xdc, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, // be16 r0 + /// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit + /// ]; + /// + /// let mut vm = rbpf::EbpfVmNoData::new(Some(prog1)).unwrap(); + /// + /// let res = vm.execute_program().unwrap(); + /// assert_eq!(res, 0x2211); + /// + /// vm.set_program(prog2); + /// + /// let res = vm.execute_program().unwrap(); + /// assert_eq!(res, 0x1122); + /// ``` + pub fn set_program(&mut self, prog: &'a [u8]) -> Result<(), Error> { + self.parent.set_program(prog)?; + Ok(()) + } + + /// Set a new verifier function. The function should return an `Error` if the program should be + /// rejected by the virtual machine. If a program has been loaded to the VM already, the + /// verifier is immediately run. + /// + /// # Examples + /// + /// ``` + /// use rbpf::{Error, ErrorKind}; + /// use rbpf::ebpf; + /// + /// // Define a simple verifier function. + /// fn verifier(prog: &[u8]) -> Result<(), Error> { + /// let last_insn = ebpf::get_insn(prog, (prog.len() / ebpf::INSN_SIZE) - 1); + /// if last_insn.opc != ebpf::EXIT { + /// return Err(Error::new(ErrorKind::Other, + /// "[Verifier] Error: program does not end with “EXIT” instruction")); + /// } + /// Ok(()) + /// } + /// + /// let prog1 = &[ + /// 0xb7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov r0, 0 + /// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit + /// ]; + /// + /// // Instantiate a VM. + /// let mut vm = rbpf::EbpfVmMbuff::new(Some(prog1)).unwrap(); + /// // Change the verifier. + /// vm.set_verifier(verifier).unwrap(); + /// ``` + pub fn set_verifier(&mut self, verifier: Verifier) -> Result<(), Error> { + self.parent.set_verifier(verifier) + } + + /// Register a built-in or user-defined helper function in order to use it later from within + /// the eBPF program. The helper is registered into a hashmap, so the `key` can be any `u32`. + /// + /// If using JIT-compiled eBPF programs, be sure to register all helpers before compiling the + /// program. You should be able to change registered helpers after compiling, but not to add + /// new ones (i.e. with new keys). + /// + /// # Examples + /// + /// ``` + /// #[cfg(feature = "std")] { + /// use rbpf::helpers; + /// + /// let prog = &[ + /// 0xb7, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, // mov r1, 0x010000000 + /// 0xb7, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov r2, 0 + /// 0xb7, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov r3, 0 + /// 0xb7, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov r4, 0 + /// 0xb7, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov r5, 0 + /// 0x85, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, // call helper with key 1 + /// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit + /// ]; + /// + /// let mut vm = rbpf::EbpfVmNoData::new(Some(prog)).unwrap(); + /// + /// // Register a helper. This helper will store the result of the square root of r1 into r0. + /// vm.register_helper(1, helpers::sqrti).unwrap(); + /// + /// let res = vm.execute_program().unwrap(); + /// assert_eq!(res, 0x1000); + /// } + /// ``` + pub fn register_helper( + &mut self, + key: u32, + function: fn(u64, u64, u64, u64, u64) -> u64, + ) -> Result<(), Error> { + self.parent.register_helper(key, function) + } + + /// JIT-compile the loaded program. No argument required for this. + /// + /// If using helper functions, be sure to register them into the VM before calling this + /// function. + /// + /// # Examples + /// + /// ``` + /// let prog = &[ + /// 0xb7, 0x00, 0x00, 0x00, 0x11, 0x22, 0x00, 0x00, // mov r0, 0x2211 + /// 0xdc, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, // be16 r0 + /// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit + /// ]; + /// + /// let mut vm = rbpf::EbpfVmNoData::new(Some(prog)).unwrap(); + /// + /// + /// vm.jit_compile(); + /// ``` + #[cfg(all(not(windows), feature = "std"))] + pub fn jit_compile(&mut self) -> Result<(), Error> { + self.parent.jit_compile() + } + + /// Execute the program loaded, without providing pointers to any memory area whatsoever. + /// + /// # Examples + /// + /// ``` + /// let prog = &[ + /// 0xb7, 0x00, 0x00, 0x00, 0x11, 0x22, 0x00, 0x00, // mov r0, 0x2211 + /// 0xdc, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, // be16 r0 + /// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit + /// ]; + /// + /// let vm = rbpf::EbpfVmNoData::new(Some(prog)).unwrap(); + /// + /// // For this kind of VM, the `execute_program()` function needs no argument. + /// let res = vm.execute_program().unwrap(); + /// assert_eq!(res, 0x1122); + /// ``` + pub fn execute_program(&self) -> Result { + self.parent.execute_program(&mut []) + } + + /// Execute the previously JIT-compiled program, without providing pointers to any memory area + /// whatsoever, in a manner very similar to `execute_program()`. + /// + /// # Safety + /// + /// **WARNING:** JIT-compiled assembly code is not safe, in particular there is no runtime + /// check for memory access; so if the eBPF program attempts erroneous accesses, this may end + /// very bad (program may segfault). It may be wise to check that the program works with the + /// interpreter before running the JIT-compiled version of it. + /// + /// For this reason the function should be called from within an `unsafe` bloc. + /// + /// # Examples + /// + /// ``` + /// let prog = &[ + /// 0xb7, 0x00, 0x00, 0x00, 0x11, 0x22, 0x00, 0x00, // mov r0, 0x2211 + /// 0xdc, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, // be16 r0 + /// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit + /// ]; + /// + /// let mut vm = rbpf::EbpfVmNoData::new(Some(prog)).unwrap(); + /// + /// # #[cfg(all(not(windows), feature = "std"))] + /// vm.jit_compile(); + /// + /// # #[cfg(all(not(windows), feature = "std"))] + /// unsafe { + /// let res = vm.execute_program_jit().unwrap(); + /// assert_eq!(res, 0x1122); + /// } + /// ``` + #[cfg(all(not(windows), feature = "std"))] + pub unsafe fn execute_program_jit(&self) -> Result { + self.parent.execute_program_jit(&mut []) + } + + /// Compile the loaded program using the Cranelift JIT. + /// + /// If using helper functions, be sure to register them into the VM before calling this + /// function. + /// + /// # Examples + /// + /// ``` + /// let prog = &[ + /// 0xb7, 0x00, 0x00, 0x00, 0x11, 0x22, 0x00, 0x00, // mov r0, 0x2211 + /// 0xdc, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, // be16 r0 + /// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit + /// ]; + /// + /// let mut vm = rbpf::EbpfVmNoData::new(Some(prog)).unwrap(); + /// + /// + /// vm.cranelift_compile(); + /// ``` + #[cfg(feature = "cranelift")] + pub fn cranelift_compile(&mut self) -> Result<(), Error> { + self.parent.cranelift_compile() + } + + /// Execute the previously JIT-compiled program, without providing pointers to any memory area + /// whatsoever, in a manner very similar to `execute_program()`. + /// + /// # Examples + /// + /// ``` + /// let prog = &[ + /// 0xb7, 0x00, 0x00, 0x00, 0x11, 0x22, 0x00, 0x00, // mov r0, 0x2211 + /// 0xdc, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, // be16 r0 + /// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // exit + /// ]; + /// + /// let mut vm = rbpf::EbpfVmNoData::new(Some(prog)).unwrap(); + /// + /// vm.cranelift_compile(); + /// + /// let res = vm.execute_program_cranelift().unwrap(); + /// assert_eq!(res, 0x1122); + /// ``` + #[cfg(feature = "cranelift")] + pub fn execute_program_cranelift(&self) -> Result { + self.parent.execute_program_cranelift(&mut []) + } +} + +/// EbpfVm with Owned data +pub struct EbpfVmRawOwned { + parent: EbpfVmRaw<'static>, + data_len: usize, + data_cap: usize, +} + +impl EbpfVmRawOwned { + /// Create a new virtual machine instance, and load an eBPF program into that instance. + /// When attempting to load the program, it passes through a simple verifier. + pub fn new(prog: Option>) -> Result { + let (prog, data_len, data_cap) = match prog { + Some(prog) => { + let data_len = prog.len(); + let data_cap = prog.capacity(); + let slice = prog.leak(); + let slice = unsafe { core::slice::from_raw_parts(slice.as_ptr(), data_len) }; + (Some(slice), data_len, data_cap) + } + None => (None, 0, 0), + }; + let parent = EbpfVmRaw::new(prog)?; + Ok(Self { + parent, + data_len, + data_cap, + }) + } + /// Load a new eBPF program into the virtual machine instance + pub fn set_program(&mut self, prog: Vec) -> Result<(), Error> { + self.data_len = prog.len(); + self.data_cap = prog.capacity(); + let slice = prog.leak(); + self.parent.set_program(slice)?; + Ok(()) + } + + /// Set a new verifier function. The function should return an Error if the program should be rejected by the virtual machine. + /// If a program has been loaded to the VM already, the verifier is immediately run. + pub fn set_verifier(&mut self, verifier: Verifier) -> Result<(), Error> { + self.parent.set_verifier(verifier) + } + + /// Register a built-in or user-defined helper function in order to use it later from within the eBPF program. + /// The helper is registered into a hashmap, so the key can be any u32. + /// If using JIT-compiled eBPF programs, be sure to register all helpers before compiling the program. + /// You should be able to change registered helpers after compiling, but not to add new ones (i. e. with new keys). + pub fn register_helper( + &mut self, + key: u32, + function: fn(u64, u64, u64, u64, u64) -> u64, + ) -> Result<(), Error> { + self.parent.register_helper(key, function) + } + + /// Register a set of built-in or user-defined helper functions in order to use them later from + /// within the eBPF program. The helpers are registered into a hashmap, so the `key` can be any + /// `u32`. + #[allow(clippy::type_complexity)] + pub fn register_helper_set( + &mut self, + helpers: &HashMap u64>, + ) -> Result<(), Error> { + for (key, function) in helpers { + self.parent.register_helper(*key, *function)?; + } + Ok(()) + } + + /// Execute the previously JIT-compiled program, with the given packet data, in a manner very similar to execute_program(). + /// + /// Safety + /// + /// **WARNING:** JIT-compiled assembly code is not safe, in particular there is no runtime check for memory access; + /// so if the eBPF program attempts erroneous accesses, this may end very bad (program may segfault). + /// It may be wise to check that the program works with the interpreter before running the JIT-compiled version of it. + /// + /// For this reason the function should be called from within an unsafe bloc. + pub fn execute_program(&self, mem: &mut [u8]) -> Result { + self.parent.execute_program(mem) + } +} + +impl Drop for EbpfVmRawOwned { + fn drop(&mut self) { + match self.parent.parent.prog { + Some(prog) => unsafe { + let ptr = prog.as_ptr(); + let _prog = Vec::from_raw_parts(ptr as *mut u8, self.data_len, self.data_cap); + }, + None => {} + }; + } +} diff --git a/kernel/crates/rbpf/src/no_std_error.rs b/kernel/crates/rbpf/src/no_std_error.rs new file mode 100644 index 00000000..f502eecf --- /dev/null +++ b/kernel/crates/rbpf/src/no_std_error.rs @@ -0,0 +1,41 @@ +// SPDX-License-Identifier: (Apache-2.0 OR MIT) + +//! This module provides a simple implementation of the Error struct that is +//! used as a drop-in replacement for `std::io::Error` when using `rbpf` in `no_std`. + +use alloc::string::String; + +/// Implementation of Error for no_std applications. +/// Ensures that the existing code can use it with the same interface +/// as the Error from std::io::Error. +#[derive(Debug)] +pub struct Error { + #[allow(dead_code)] + kind: ErrorKind, + #[allow(dead_code)] + error: String, +} + +impl Error { + /// New function exposing the same signature as `std::io::Error::new`. + #[allow(dead_code)] + pub fn new>(kind: ErrorKind, error: S) -> Error { + Error { + kind, + error: error.into(), + } + } +} + +/// The current version of `rbpf` only uses the [`Other`](ErrorKind::Other) variant +/// from the [std::io::ErrorKind] enum. If a dependency on other variants were +/// introduced in the future, this enum needs to be updated accordingly to maintain +/// compatibility with the real `ErrorKind`. The reason all available variants +/// aren't included in the first place is that [std::io::ErrorKind] exposes +/// 40 variants, and not all of them are meaningful under `no_std`. +#[derive(Debug)] +pub enum ErrorKind { + /// The no_std code only uses this variant. + #[allow(dead_code)] + Other, +} diff --git a/kernel/crates/rbpf/src/stack.rs b/kernel/crates/rbpf/src/stack.rs new file mode 100644 index 00000000..be173263 --- /dev/null +++ b/kernel/crates/rbpf/src/stack.rs @@ -0,0 +1,75 @@ +use crate::{ebpf::STACK_SIZE, vec, Vec}; + +pub struct StackFrame { + return_address: u16, + saved_registers: [u64; 4], + sp: u16, + frame: Vec, +} + +impl StackFrame { + /// Create a new stack frame + /// + /// The stack frame is created with a capacity of `STACK_SIZE` == 512 bytes + pub fn new() -> Self { + Self { + sp: 0, + return_address: 0, + saved_registers: [0; 4], + frame: vec![0; STACK_SIZE], + } + } + + /// Create a new stack frame with a given capacity + #[allow(unused)] + pub fn with_capacity(capacity: usize) -> Self { + Self { + sp: 0, + return_address: 0, + saved_registers: [0; 4], + frame: vec![0; capacity], + } + } + + /// The capacity of the stack frame + pub fn len(&self) -> usize { + self.frame.len() + } + + pub fn as_ptr(&self) -> *const u8 { + self.frame.as_ptr() + } + + pub fn as_slice(&self) -> &[u8] { + self.frame.as_slice() + } + /// Save the callee-saved registers + pub fn save_registers(&mut self, regs: &[u64]) { + self.saved_registers.copy_from_slice(regs); + } + + /// Get the callee-saved registers + pub fn get_registers(&self) -> [u64; 4] { + self.saved_registers + } + + /// Save the return address + pub fn save_return_address(&mut self, address: u16) { + self.return_address = address; + } + + /// Get the return address + pub fn get_return_address(&self) -> u16 { + self.return_address + } + + /// Save the stack pointer + pub fn save_sp(&mut self, sp: u16) { + self.sp = sp; + } + + /// Get the stack pointer + pub fn get_sp(&self) -> u16 { + self.sp + } +} diff --git a/kernel/crates/rbpf/src/verifier.rs b/kernel/crates/rbpf/src/verifier.rs new file mode 100644 index 00000000..4c9cf250 --- /dev/null +++ b/kernel/crates/rbpf/src/verifier.rs @@ -0,0 +1,386 @@ +// SPDX-License-Identifier: (Apache-2.0 OR MIT) +// Derived from uBPF +// Copyright 2015 Big Switch Networks, Inc +// (uBPF: safety checks, originally in C) +// Copyright 2016 6WIND S.A. +// (Translation to Rust) + +// This “verifier” performs simple checks when the eBPF program is loaded into the VM (before it is +// interpreted or JIT-compiled). It has nothing to do with the much more elaborated verifier inside +// Linux kernel. There is no verification regarding the program flow control (should be a Direct +// Acyclic Graph) or the consistency for registers usage (the verifier of the kernel assigns types +// to the registers and is much stricter). +// +// On the other hand, rbpf is not expected to run in kernel space. +// +// Improving the verifier would be nice, but this is not trivial (and Linux kernel is under GPL +// license, so we cannot copy it). +// +// Contrary to the verifier of the Linux kernel, this one does not modify the bytecode at all. + +use alloc::format; + +use crate::{ebpf, Error, ErrorKind}; + +fn reject>(msg: S) -> Result<(), Error> { + let full_msg = format!("[Verifier] Error: {}", msg.as_ref()); + Err(Error::new(ErrorKind::Other, full_msg)) +} + +fn check_prog_len(prog: &[u8]) -> Result<(), Error> { + if prog.len() % ebpf::INSN_SIZE != 0 { + reject(format!( + "eBPF program length must be a multiple of {:?} octets", + ebpf::INSN_SIZE + ))?; + } + if prog.len() > ebpf::PROG_MAX_SIZE { + reject(format!( + "eBPF program length limited to {:?}, here {:?}", + ebpf::PROG_MAX_INSNS, + prog.len() / ebpf::INSN_SIZE + ))?; + } + + if prog.is_empty() { + reject("no program set, call set_program() to load one")?; + } + let last_opc = ebpf::get_insn(prog, (prog.len() / ebpf::INSN_SIZE) - 1).opc; + if last_opc & ebpf::BPF_CLS_MASK != ebpf::BPF_JMP { + reject("program does not end with “EXIT” instruction")?; + } + + Ok(()) +} + +fn check_imm_endian(insn: &ebpf::Insn, insn_ptr: usize) -> Result<(), Error> { + match insn.imm { + 16 | 32 | 64 => Ok(()), + _ => reject(format!( + "unsupported argument for LE/BE (insn #{insn_ptr:?})" + )), + } +} + +fn check_load_dw(prog: &[u8], insn_ptr: usize) -> Result<(), Error> { + // We know we can reach next insn since we enforce an EXIT insn at the end of program, while + // this function should be called only for LD_DW insn, that cannot be last in program. + let next_insn = ebpf::get_insn(prog, insn_ptr + 1); + if next_insn.opc != 0 { + reject(format!("incomplete LD_DW instruction (insn #{insn_ptr:?})"))?; + } + + Ok(()) +} + +fn check_jmp_offset(prog: &[u8], insn_ptr: usize) -> Result<(), Error> { + let insn = ebpf::get_insn(prog, insn_ptr); + if insn.off == -1 { + reject(format!("infinite loop (insn #{insn_ptr:?})"))?; + } + + let dst_insn_ptr = insn_ptr as isize + 1 + insn.off as isize; + if dst_insn_ptr < 0 || dst_insn_ptr as usize >= (prog.len() / ebpf::INSN_SIZE) { + reject(format!( + "jump out of code to #{dst_insn_ptr:?} (insn #{insn_ptr:?})" + ))?; + } + + let dst_insn = ebpf::get_insn(prog, dst_insn_ptr as usize); + if dst_insn.opc == 0 { + reject(format!( + "jump to middle of LD_DW at #{dst_insn_ptr:?} (insn #{insn_ptr:?})" + ))?; + } + + Ok(()) +} + +fn check_registers(insn: &ebpf::Insn, store: bool, insn_ptr: usize) -> Result<(), Error> { + if insn.src > 10 { + reject(format!("invalid source register (insn #{insn_ptr:?})"))?; + } + + match (insn.dst, store) { + (0..=9, _) | (10, true) => Ok(()), + (10, false) => reject(format!( + "cannot write into register r10 (insn #{insn_ptr:?})" + )), + (_, _) => reject(format!("invalid destination register (insn #{insn_ptr:?})")), + } +} + +pub fn check(prog: &[u8]) -> Result<(), Error> { + check_prog_len(prog)?; + + let mut insn_ptr: usize = 0; + while insn_ptr * ebpf::INSN_SIZE < prog.len() { + let insn = ebpf::get_insn(prog, insn_ptr); + let mut store = false; + + match insn.opc { + // BPF_LD class + ebpf::LD_ABS_B => {} + ebpf::LD_ABS_H => {} + ebpf::LD_ABS_W => {} + ebpf::LD_ABS_DW => {} + ebpf::LD_IND_B => {} + ebpf::LD_IND_H => {} + ebpf::LD_IND_W => {} + ebpf::LD_IND_DW => {} + + ebpf::LD_DW_IMM => { + store = true; + check_load_dw(prog, insn_ptr)?; + insn_ptr += 1; + } + + // BPF_LDX class + ebpf::LD_B_REG => {} + ebpf::LD_H_REG => {} + ebpf::LD_W_REG => {} + ebpf::LD_DW_REG => {} + + // BPF_ST class + ebpf::ST_B_IMM => store = true, + ebpf::ST_H_IMM => store = true, + ebpf::ST_W_IMM => store = true, + ebpf::ST_DW_IMM => store = true, + + // BPF_STX class + ebpf::ST_B_REG => store = true, + ebpf::ST_H_REG => store = true, + ebpf::ST_W_REG => store = true, + ebpf::ST_DW_REG => store = true, + ebpf::ST_W_XADD => { + unimplemented!(); + } + ebpf::ST_DW_XADD => { + unimplemented!(); + } + + // BPF_ALU class + ebpf::ADD32_IMM => {} + ebpf::ADD32_REG => {} + ebpf::SUB32_IMM => {} + ebpf::SUB32_REG => {} + ebpf::MUL32_IMM => {} + ebpf::MUL32_REG => {} + ebpf::DIV32_IMM => {} + ebpf::DIV32_REG => {} + ebpf::OR32_IMM => {} + ebpf::OR32_REG => {} + ebpf::AND32_IMM => {} + ebpf::AND32_REG => {} + ebpf::LSH32_IMM => {} + ebpf::LSH32_REG => {} + ebpf::RSH32_IMM => {} + ebpf::RSH32_REG => {} + ebpf::NEG32 => {} + ebpf::MOD32_IMM => {} + ebpf::MOD32_REG => {} + ebpf::XOR32_IMM => {} + ebpf::XOR32_REG => {} + ebpf::MOV32_IMM => {} + ebpf::MOV32_REG => {} + ebpf::ARSH32_IMM => {} + ebpf::ARSH32_REG => {} + ebpf::LE => { + check_imm_endian(&insn, insn_ptr)?; + } + ebpf::BE => { + check_imm_endian(&insn, insn_ptr)?; + } + + // BPF_ALU64 class + ebpf::ADD64_IMM => {} + ebpf::ADD64_REG => {} + ebpf::SUB64_IMM => {} + ebpf::SUB64_REG => {} + ebpf::MUL64_IMM => {} + ebpf::MUL64_REG => {} + ebpf::DIV64_IMM => {} + ebpf::DIV64_REG => {} + ebpf::OR64_IMM => {} + ebpf::OR64_REG => {} + ebpf::AND64_IMM => {} + ebpf::AND64_REG => {} + ebpf::LSH64_IMM => {} + ebpf::LSH64_REG => {} + ebpf::RSH64_IMM => {} + ebpf::RSH64_REG => {} + ebpf::NEG64 => {} + ebpf::MOD64_IMM => {} + ebpf::MOD64_REG => {} + ebpf::XOR64_IMM => {} + ebpf::XOR64_REG => {} + ebpf::MOV64_IMM => {} + ebpf::MOV64_REG => {} + ebpf::ARSH64_IMM => {} + ebpf::ARSH64_REG => {} + + // BPF_JMP class + ebpf::JA => { + check_jmp_offset(prog, insn_ptr)?; + } + ebpf::JEQ_IMM => { + check_jmp_offset(prog, insn_ptr)?; + } + ebpf::JEQ_REG => { + check_jmp_offset(prog, insn_ptr)?; + } + ebpf::JGT_IMM => { + check_jmp_offset(prog, insn_ptr)?; + } + ebpf::JGT_REG => { + check_jmp_offset(prog, insn_ptr)?; + } + ebpf::JGE_IMM => { + check_jmp_offset(prog, insn_ptr)?; + } + ebpf::JGE_REG => { + check_jmp_offset(prog, insn_ptr)?; + } + ebpf::JLT_IMM => { + check_jmp_offset(prog, insn_ptr)?; + } + ebpf::JLT_REG => { + check_jmp_offset(prog, insn_ptr)?; + } + ebpf::JLE_IMM => { + check_jmp_offset(prog, insn_ptr)?; + } + ebpf::JLE_REG => { + check_jmp_offset(prog, insn_ptr)?; + } + ebpf::JSET_IMM => { + check_jmp_offset(prog, insn_ptr)?; + } + ebpf::JSET_REG => { + check_jmp_offset(prog, insn_ptr)?; + } + ebpf::JNE_IMM => { + check_jmp_offset(prog, insn_ptr)?; + } + ebpf::JNE_REG => { + check_jmp_offset(prog, insn_ptr)?; + } + ebpf::JSGT_IMM => { + check_jmp_offset(prog, insn_ptr)?; + } + ebpf::JSGT_REG => { + check_jmp_offset(prog, insn_ptr)?; + } + ebpf::JSGE_IMM => { + check_jmp_offset(prog, insn_ptr)?; + } + ebpf::JSGE_REG => { + check_jmp_offset(prog, insn_ptr)?; + } + ebpf::JSLT_IMM => { + check_jmp_offset(prog, insn_ptr)?; + } + ebpf::JSLT_REG => { + check_jmp_offset(prog, insn_ptr)?; + } + ebpf::JSLE_IMM => { + check_jmp_offset(prog, insn_ptr)?; + } + ebpf::JSLE_REG => { + check_jmp_offset(prog, insn_ptr)?; + } + + // BPF_JMP32 class + ebpf::JEQ_IMM32 => { + check_jmp_offset(prog, insn_ptr)?; + } + ebpf::JEQ_REG32 => { + check_jmp_offset(prog, insn_ptr)?; + } + ebpf::JGT_IMM32 => { + check_jmp_offset(prog, insn_ptr)?; + } + ebpf::JGT_REG32 => { + check_jmp_offset(prog, insn_ptr)?; + } + ebpf::JGE_IMM32 => { + check_jmp_offset(prog, insn_ptr)?; + } + ebpf::JGE_REG32 => { + check_jmp_offset(prog, insn_ptr)?; + } + ebpf::JLT_IMM32 => { + check_jmp_offset(prog, insn_ptr)?; + } + ebpf::JLT_REG32 => { + check_jmp_offset(prog, insn_ptr)?; + } + ebpf::JLE_IMM32 => { + check_jmp_offset(prog, insn_ptr)?; + } + ebpf::JLE_REG32 => { + check_jmp_offset(prog, insn_ptr)?; + } + ebpf::JSET_IMM32 => { + check_jmp_offset(prog, insn_ptr)?; + } + ebpf::JSET_REG32 => { + check_jmp_offset(prog, insn_ptr)?; + } + ebpf::JNE_IMM32 => { + check_jmp_offset(prog, insn_ptr)?; + } + ebpf::JNE_REG32 => { + check_jmp_offset(prog, insn_ptr)?; + } + ebpf::JSGT_IMM32 => { + check_jmp_offset(prog, insn_ptr)?; + } + ebpf::JSGT_REG32 => { + check_jmp_offset(prog, insn_ptr)?; + } + ebpf::JSGE_IMM32 => { + check_jmp_offset(prog, insn_ptr)?; + } + ebpf::JSGE_REG32 => { + check_jmp_offset(prog, insn_ptr)?; + } + ebpf::JSLT_IMM32 => { + check_jmp_offset(prog, insn_ptr)?; + } + ebpf::JSLT_REG32 => { + check_jmp_offset(prog, insn_ptr)?; + } + ebpf::JSLE_IMM32 => { + check_jmp_offset(prog, insn_ptr)?; + } + ebpf::JSLE_REG32 => { + check_jmp_offset(prog, insn_ptr)?; + } + + ebpf::CALL => {} + ebpf::TAIL_CALL => { + unimplemented!() + } + ebpf::EXIT => {} + + _ => { + reject(format!( + "unknown eBPF opcode {:#2x} (insn #{insn_ptr:?})", + insn.opc + ))?; + } + } + + check_registers(&insn, store, insn_ptr)?; + + insn_ptr += 1; + } + + // insn_ptr should now be equal to number of instructions. + if insn_ptr != prog.len() / ebpf::INSN_SIZE { + reject(format!("jumped out of code to #{insn_ptr:?}"))?; + } + + Ok(()) +} diff --git a/kernel/crates/rbpf/tests/assembler.rs b/kernel/crates/rbpf/tests/assembler.rs new file mode 100644 index 00000000..0640772c --- /dev/null +++ b/kernel/crates/rbpf/tests/assembler.rs @@ -0,0 +1,655 @@ +// SPDX-License-Identifier: (Apache-2.0 OR MIT) +// Copyright 2017 Rich Lane + +#![allow(clippy::unreadable_literal)] + +extern crate rbpf; +mod common; + +use common::{TCP_SACK_ASM, TCP_SACK_BIN}; +use rbpf::{assembler::assemble, ebpf}; + +fn asm(src: &str) -> Result, String> { + Ok(ebpf::to_insn_vec(&(assemble(src))?)) +} + +fn insn(opc: u8, dst: u8, src: u8, off: i16, imm: i32) -> ebpf::Insn { + ebpf::Insn { + opc, + dst, + src, + off, + imm, + } +} + +#[test] +fn test_empty() { + assert_eq!(asm(""), Ok(vec![])); +} + +// Example for InstructionType::NoOperand. +#[test] +fn test_exit() { + assert_eq!(asm("exit"), Ok(vec![insn(ebpf::EXIT, 0, 0, 0, 0)])); +} + +// Example for InstructionType::AluBinary. +#[test] +fn test_add64() { + assert_eq!( + asm("add64 r1, r3"), + Ok(vec![insn(ebpf::ADD64_REG, 1, 3, 0, 0)]) + ); + assert_eq!( + asm("add64 r1, 5"), + Ok(vec![insn(ebpf::ADD64_IMM, 1, 0, 0, 5)]) + ); +} + +// Example for InstructionType::AluUnary. +#[test] +fn test_neg64() { + assert_eq!(asm("neg64 r1"), Ok(vec![insn(ebpf::NEG64, 1, 0, 0, 0)])); +} + +// Example for InstructionType::LoadReg. +#[test] +fn test_ldxw() { + assert_eq!( + asm("ldxw r1, [r2+5]"), + Ok(vec![insn(ebpf::LD_W_REG, 1, 2, 5, 0)]) + ); +} + +// Example for InstructionType::StoreImm. +#[test] +fn test_stw() { + assert_eq!( + asm("stw [r2+5], 7"), + Ok(vec![insn(ebpf::ST_W_IMM, 2, 0, 5, 7)]) + ); +} + +// Example for InstructionType::StoreReg. +#[test] +fn test_stxw() { + assert_eq!( + asm("stxw [r2+5], r8"), + Ok(vec![insn(ebpf::ST_W_REG, 2, 8, 5, 0)]) + ); +} + +// Example for InstructionType::JumpUnconditional. +#[test] +fn test_ja() { + assert_eq!(asm("ja +8"), Ok(vec![insn(ebpf::JA, 0, 0, 8, 0)])); + assert_eq!(asm("ja -3"), Ok(vec![insn(ebpf::JA, 0, 0, -3, 0)])); +} + +// Example for InstructionType::JumpConditional. +#[test] +fn test_jeq() { + assert_eq!( + asm("jeq r1, 4, +8"), + Ok(vec![insn(ebpf::JEQ_IMM, 1, 0, 8, 4)]) + ); + assert_eq!( + asm("jeq r1, r3, +8"), + Ok(vec![insn(ebpf::JEQ_REG, 1, 3, 8, 0)]) + ); +} + +// Example for InstructionType::Call. +#[test] +fn test_call() { + assert_eq!(asm("call 300"), Ok(vec![insn(ebpf::CALL, 0, 0, 0, 300)])); +} + +// Example for InstructionType::Endian. +#[test] +fn test_be32() { + assert_eq!(asm("be32 r1"), Ok(vec![insn(ebpf::BE, 1, 0, 0, 32)])); +} + +// Example for InstructionType::LoadImm. +#[test] +fn test_lddw() { + assert_eq!( + asm("lddw r1, 0x1234abcd5678eeff"), + Ok(vec![ + insn(ebpf::LD_DW_IMM, 1, 0, 0, 0x5678eeff), + insn(0, 0, 0, 0, 0x1234abcd) + ]) + ); + assert_eq!( + asm("lddw r1, 0xff11ee22dd33cc44"), + Ok(vec![ + insn(ebpf::LD_DW_IMM, 1, 0, 0, 0xdd33cc44u32 as i32), + insn(0, 0, 0, 0, 0xff11ee22u32 as i32) + ]) + ); +} + +// Example for InstructionType::LoadAbs. +#[test] +fn test_ldabsw() { + assert_eq!(asm("ldabsw 1"), Ok(vec![insn(ebpf::LD_ABS_W, 0, 0, 0, 1)])); +} + +// Example for InstructionType::LoadInd. +#[test] +fn test_ldindw() { + assert_eq!( + asm("ldindw r1, 2"), + Ok(vec![insn(ebpf::LD_IND_W, 0, 1, 0, 2)]) + ); +} + +// Example for InstructionType::LoadReg. +#[test] +fn test_ldxdw() { + assert_eq!( + asm("ldxdw r1, [r2+3]"), + Ok(vec![insn(ebpf::LD_DW_REG, 1, 2, 3, 0)]) + ); +} + +// Example for InstructionType::StoreImm. +#[test] +fn test_sth() { + assert_eq!( + asm("sth [r1+2], 3"), + Ok(vec![insn(ebpf::ST_H_IMM, 1, 0, 2, 3)]) + ); +} + +// Example for InstructionType::StoreReg. +#[test] +fn test_stxh() { + assert_eq!( + asm("stxh [r1+2], r3"), + Ok(vec![insn(ebpf::ST_H_REG, 1, 3, 2, 0)]) + ); +} + +// Test all supported AluBinary mnemonics. +#[test] +fn test_alu_binary() { + assert_eq!( + asm("add r1, r2 + sub r1, r2 + mul r1, r2 + div r1, r2 + or r1, r2 + and r1, r2 + lsh r1, r2 + rsh r1, r2 + mod r1, r2 + xor r1, r2 + mov r1, r2 + arsh r1, r2"), + Ok(vec![ + insn(ebpf::ADD64_REG, 1, 2, 0, 0), + insn(ebpf::SUB64_REG, 1, 2, 0, 0), + insn(ebpf::MUL64_REG, 1, 2, 0, 0), + insn(ebpf::DIV64_REG, 1, 2, 0, 0), + insn(ebpf::OR64_REG, 1, 2, 0, 0), + insn(ebpf::AND64_REG, 1, 2, 0, 0), + insn(ebpf::LSH64_REG, 1, 2, 0, 0), + insn(ebpf::RSH64_REG, 1, 2, 0, 0), + insn(ebpf::MOD64_REG, 1, 2, 0, 0), + insn(ebpf::XOR64_REG, 1, 2, 0, 0), + insn(ebpf::MOV64_REG, 1, 2, 0, 0), + insn(ebpf::ARSH64_REG, 1, 2, 0, 0) + ]) + ); + + assert_eq!( + asm("add r1, 2 + sub r1, 2 + mul r1, 2 + div r1, 2 + or r1, 2 + and r1, 2 + lsh r1, 2 + rsh r1, 2 + mod r1, 2 + xor r1, 2 + mov r1, 2 + arsh r1, 2"), + Ok(vec![ + insn(ebpf::ADD64_IMM, 1, 0, 0, 2), + insn(ebpf::SUB64_IMM, 1, 0, 0, 2), + insn(ebpf::MUL64_IMM, 1, 0, 0, 2), + insn(ebpf::DIV64_IMM, 1, 0, 0, 2), + insn(ebpf::OR64_IMM, 1, 0, 0, 2), + insn(ebpf::AND64_IMM, 1, 0, 0, 2), + insn(ebpf::LSH64_IMM, 1, 0, 0, 2), + insn(ebpf::RSH64_IMM, 1, 0, 0, 2), + insn(ebpf::MOD64_IMM, 1, 0, 0, 2), + insn(ebpf::XOR64_IMM, 1, 0, 0, 2), + insn(ebpf::MOV64_IMM, 1, 0, 0, 2), + insn(ebpf::ARSH64_IMM, 1, 0, 0, 2) + ]) + ); + + assert_eq!( + asm("add64 r1, r2 + sub64 r1, r2 + mul64 r1, r2 + div64 r1, r2 + or64 r1, r2 + and64 r1, r2 + lsh64 r1, r2 + rsh64 r1, r2 + mod64 r1, r2 + xor64 r1, r2 + mov64 r1, r2 + arsh64 r1, r2"), + Ok(vec![ + insn(ebpf::ADD64_REG, 1, 2, 0, 0), + insn(ebpf::SUB64_REG, 1, 2, 0, 0), + insn(ebpf::MUL64_REG, 1, 2, 0, 0), + insn(ebpf::DIV64_REG, 1, 2, 0, 0), + insn(ebpf::OR64_REG, 1, 2, 0, 0), + insn(ebpf::AND64_REG, 1, 2, 0, 0), + insn(ebpf::LSH64_REG, 1, 2, 0, 0), + insn(ebpf::RSH64_REG, 1, 2, 0, 0), + insn(ebpf::MOD64_REG, 1, 2, 0, 0), + insn(ebpf::XOR64_REG, 1, 2, 0, 0), + insn(ebpf::MOV64_REG, 1, 2, 0, 0), + insn(ebpf::ARSH64_REG, 1, 2, 0, 0) + ]) + ); + + assert_eq!( + asm("add64 r1, 2 + sub64 r1, 2 + mul64 r1, 2 + div64 r1, 2 + or64 r1, 2 + and64 r1, 2 + lsh64 r1, 2 + rsh64 r1, 2 + mod64 r1, 2 + xor64 r1, 2 + mov64 r1, 2 + arsh64 r1, 2"), + Ok(vec![ + insn(ebpf::ADD64_IMM, 1, 0, 0, 2), + insn(ebpf::SUB64_IMM, 1, 0, 0, 2), + insn(ebpf::MUL64_IMM, 1, 0, 0, 2), + insn(ebpf::DIV64_IMM, 1, 0, 0, 2), + insn(ebpf::OR64_IMM, 1, 0, 0, 2), + insn(ebpf::AND64_IMM, 1, 0, 0, 2), + insn(ebpf::LSH64_IMM, 1, 0, 0, 2), + insn(ebpf::RSH64_IMM, 1, 0, 0, 2), + insn(ebpf::MOD64_IMM, 1, 0, 0, 2), + insn(ebpf::XOR64_IMM, 1, 0, 0, 2), + insn(ebpf::MOV64_IMM, 1, 0, 0, 2), + insn(ebpf::ARSH64_IMM, 1, 0, 0, 2) + ]) + ); + + assert_eq!( + asm("add32 r1, r2 + sub32 r1, r2 + mul32 r1, r2 + div32 r1, r2 + or32 r1, r2 + and32 r1, r2 + lsh32 r1, r2 + rsh32 r1, r2 + mod32 r1, r2 + xor32 r1, r2 + mov32 r1, r2 + arsh32 r1, r2"), + Ok(vec![ + insn(ebpf::ADD32_REG, 1, 2, 0, 0), + insn(ebpf::SUB32_REG, 1, 2, 0, 0), + insn(ebpf::MUL32_REG, 1, 2, 0, 0), + insn(ebpf::DIV32_REG, 1, 2, 0, 0), + insn(ebpf::OR32_REG, 1, 2, 0, 0), + insn(ebpf::AND32_REG, 1, 2, 0, 0), + insn(ebpf::LSH32_REG, 1, 2, 0, 0), + insn(ebpf::RSH32_REG, 1, 2, 0, 0), + insn(ebpf::MOD32_REG, 1, 2, 0, 0), + insn(ebpf::XOR32_REG, 1, 2, 0, 0), + insn(ebpf::MOV32_REG, 1, 2, 0, 0), + insn(ebpf::ARSH32_REG, 1, 2, 0, 0) + ]) + ); + + assert_eq!( + asm("add32 r1, 2 + sub32 r1, 2 + mul32 r1, 2 + div32 r1, 2 + or32 r1, 2 + and32 r1, 2 + lsh32 r1, 2 + rsh32 r1, 2 + mod32 r1, 2 + xor32 r1, 2 + mov32 r1, 2 + arsh32 r1, 2"), + Ok(vec![ + insn(ebpf::ADD32_IMM, 1, 0, 0, 2), + insn(ebpf::SUB32_IMM, 1, 0, 0, 2), + insn(ebpf::MUL32_IMM, 1, 0, 0, 2), + insn(ebpf::DIV32_IMM, 1, 0, 0, 2), + insn(ebpf::OR32_IMM, 1, 0, 0, 2), + insn(ebpf::AND32_IMM, 1, 0, 0, 2), + insn(ebpf::LSH32_IMM, 1, 0, 0, 2), + insn(ebpf::RSH32_IMM, 1, 0, 0, 2), + insn(ebpf::MOD32_IMM, 1, 0, 0, 2), + insn(ebpf::XOR32_IMM, 1, 0, 0, 2), + insn(ebpf::MOV32_IMM, 1, 0, 0, 2), + insn(ebpf::ARSH32_IMM, 1, 0, 0, 2) + ]) + ); +} + +// Test all supported AluUnary mnemonics. +#[test] +fn test_alu_unary() { + assert_eq!( + asm("neg r1 + neg64 r1 + neg32 r1"), + Ok(vec![ + insn(ebpf::NEG64, 1, 0, 0, 0), + insn(ebpf::NEG64, 1, 0, 0, 0), + insn(ebpf::NEG32, 1, 0, 0, 0) + ]) + ); +} + +// Test all supported LoadAbs mnemonics. +#[test] +fn test_load_abs() { + assert_eq!( + asm("ldabsw 1 + ldabsh 1 + ldabsb 1 + ldabsdw 1"), + Ok(vec![ + insn(ebpf::LD_ABS_W, 0, 0, 0, 1), + insn(ebpf::LD_ABS_H, 0, 0, 0, 1), + insn(ebpf::LD_ABS_B, 0, 0, 0, 1), + insn(ebpf::LD_ABS_DW, 0, 0, 0, 1) + ]) + ); +} + +// Test all supported LoadInd mnemonics. +#[test] +fn test_load_ind() { + assert_eq!( + asm("ldindw r1, 2 + ldindh r1, 2 + ldindb r1, 2 + ldinddw r1, 2"), + Ok(vec![ + insn(ebpf::LD_IND_W, 0, 1, 0, 2), + insn(ebpf::LD_IND_H, 0, 1, 0, 2), + insn(ebpf::LD_IND_B, 0, 1, 0, 2), + insn(ebpf::LD_IND_DW, 0, 1, 0, 2) + ]) + ); +} + +// Test all supported LoadReg mnemonics. +#[test] +fn test_load_reg() { + assert_eq!( + asm("ldxw r1, [r2+3] + ldxh r1, [r2+3] + ldxb r1, [r2+3] + ldxdw r1, [r2+3]"), + Ok(vec![ + insn(ebpf::LD_W_REG, 1, 2, 3, 0), + insn(ebpf::LD_H_REG, 1, 2, 3, 0), + insn(ebpf::LD_B_REG, 1, 2, 3, 0), + insn(ebpf::LD_DW_REG, 1, 2, 3, 0) + ]) + ); +} + +// Test all supported StoreImm mnemonics. +#[test] +fn test_store_imm() { + assert_eq!( + asm("stw [r1+2], 3 + sth [r1+2], 3 + stb [r1+2], 3 + stdw [r1+2], 3"), + Ok(vec![ + insn(ebpf::ST_W_IMM, 1, 0, 2, 3), + insn(ebpf::ST_H_IMM, 1, 0, 2, 3), + insn(ebpf::ST_B_IMM, 1, 0, 2, 3), + insn(ebpf::ST_DW_IMM, 1, 0, 2, 3) + ]) + ); +} + +// Test all supported StoreReg mnemonics. +#[test] +fn test_store_reg() { + assert_eq!( + asm("stxw [r1+2], r3 + stxh [r1+2], r3 + stxb [r1+2], r3 + stxdw [r1+2], r3"), + Ok(vec![ + insn(ebpf::ST_W_REG, 1, 3, 2, 0), + insn(ebpf::ST_H_REG, 1, 3, 2, 0), + insn(ebpf::ST_B_REG, 1, 3, 2, 0), + insn(ebpf::ST_DW_REG, 1, 3, 2, 0) + ]) + ); +} + +// Test all supported JumpConditional mnemonics. +#[test] +fn test_jump_conditional() { + assert_eq!( + asm("jeq r1, r2, +3 + jgt r1, r2, +3 + jge r1, r2, +3 + jlt r1, r2, +3 + jle r1, r2, +3 + jset r1, r2, +3 + jne r1, r2, +3 + jsgt r1, r2, +3 + jsge r1, r2, +3 + jslt r1, r2, +3 + jsle r1, r2, +3"), + Ok(vec![ + insn(ebpf::JEQ_REG, 1, 2, 3, 0), + insn(ebpf::JGT_REG, 1, 2, 3, 0), + insn(ebpf::JGE_REG, 1, 2, 3, 0), + insn(ebpf::JLT_REG, 1, 2, 3, 0), + insn(ebpf::JLE_REG, 1, 2, 3, 0), + insn(ebpf::JSET_REG, 1, 2, 3, 0), + insn(ebpf::JNE_REG, 1, 2, 3, 0), + insn(ebpf::JSGT_REG, 1, 2, 3, 0), + insn(ebpf::JSGE_REG, 1, 2, 3, 0), + insn(ebpf::JSLT_REG, 1, 2, 3, 0), + insn(ebpf::JSLE_REG, 1, 2, 3, 0) + ]) + ); + + assert_eq!( + asm("jeq r1, 2, +3 + jgt r1, 2, +3 + jge r1, 2, +3 + jlt r1, 2, +3 + jle r1, 2, +3 + jset r1, 2, +3 + jne r1, 2, +3 + jsgt r1, 2, +3 + jsge r1, 2, +3 + jslt r1, 2, +3 + jsle r1, 2, +3"), + Ok(vec![ + insn(ebpf::JEQ_IMM, 1, 0, 3, 2), + insn(ebpf::JGT_IMM, 1, 0, 3, 2), + insn(ebpf::JGE_IMM, 1, 0, 3, 2), + insn(ebpf::JLT_IMM, 1, 0, 3, 2), + insn(ebpf::JLE_IMM, 1, 0, 3, 2), + insn(ebpf::JSET_IMM, 1, 0, 3, 2), + insn(ebpf::JNE_IMM, 1, 0, 3, 2), + insn(ebpf::JSGT_IMM, 1, 0, 3, 2), + insn(ebpf::JSGE_IMM, 1, 0, 3, 2), + insn(ebpf::JSLT_IMM, 1, 0, 3, 2), + insn(ebpf::JSLE_IMM, 1, 0, 3, 2) + ]) + ); + + assert_eq!( + asm("jeq32 r1, r2, +3 + jgt32 r1, r2, +3 + jge32 r1, r2, +3 + jlt32 r1, r2, +3 + jle32 r1, r2, +3 + jset32 r1, r2, +3 + jne32 r1, r2, +3 + jsgt32 r1, r2, +3 + jsge32 r1, r2, +3 + jslt32 r1, r2, +3 + jsle32 r1, r2, +3"), + Ok(vec![ + insn(ebpf::JEQ_REG32, 1, 2, 3, 0), + insn(ebpf::JGT_REG32, 1, 2, 3, 0), + insn(ebpf::JGE_REG32, 1, 2, 3, 0), + insn(ebpf::JLT_REG32, 1, 2, 3, 0), + insn(ebpf::JLE_REG32, 1, 2, 3, 0), + insn(ebpf::JSET_REG32, 1, 2, 3, 0), + insn(ebpf::JNE_REG32, 1, 2, 3, 0), + insn(ebpf::JSGT_REG32, 1, 2, 3, 0), + insn(ebpf::JSGE_REG32, 1, 2, 3, 0), + insn(ebpf::JSLT_REG32, 1, 2, 3, 0), + insn(ebpf::JSLE_REG32, 1, 2, 3, 0) + ]) + ); + + assert_eq!( + asm("jeq32 r1, 2, +3 + jgt32 r1, 2, +3 + jge32 r1, 2, +3 + jlt32 r1, 2, +3 + jle32 r1, 2, +3 + jset32 r1, 2, +3 + jne32 r1, 2, +3 + jsgt32 r1, 2, +3 + jsge32 r1, 2, +3 + jslt32 r1, 2, +3 + jsle32 r1, 2, +3"), + Ok(vec![ + insn(ebpf::JEQ_IMM32, 1, 0, 3, 2), + insn(ebpf::JGT_IMM32, 1, 0, 3, 2), + insn(ebpf::JGE_IMM32, 1, 0, 3, 2), + insn(ebpf::JLT_IMM32, 1, 0, 3, 2), + insn(ebpf::JLE_IMM32, 1, 0, 3, 2), + insn(ebpf::JSET_IMM32, 1, 0, 3, 2), + insn(ebpf::JNE_IMM32, 1, 0, 3, 2), + insn(ebpf::JSGT_IMM32, 1, 0, 3, 2), + insn(ebpf::JSGE_IMM32, 1, 0, 3, 2), + insn(ebpf::JSLT_IMM32, 1, 0, 3, 2), + insn(ebpf::JSLE_IMM32, 1, 0, 3, 2) + ]) + ); +} + +// Test all supported Endian mnemonics. +#[test] +fn test_endian() { + assert_eq!( + asm("be16 r1 + be32 r1 + be64 r1 + le16 r1 + le32 r1 + le64 r1"), + Ok(vec![ + insn(ebpf::BE, 1, 0, 0, 16), + insn(ebpf::BE, 1, 0, 0, 32), + insn(ebpf::BE, 1, 0, 0, 64), + insn(ebpf::LE, 1, 0, 0, 16), + insn(ebpf::LE, 1, 0, 0, 32), + insn(ebpf::LE, 1, 0, 0, 64) + ]) + ); +} + +#[test] +fn test_large_immediate() { + assert_eq!( + asm("add64 r1, 2147483647"), + Ok(vec![insn(ebpf::ADD64_IMM, 1, 0, 0, 2147483647)]) + ); + assert_eq!( + asm("add64 r1, -2147483648"), + Ok(vec![insn(ebpf::ADD64_IMM, 1, 0, 0, -2147483648)]) + ); +} + +#[test] +fn test_tcp_sack() { + assert_eq!(assemble(TCP_SACK_ASM), Ok(TCP_SACK_BIN.to_vec())); +} + +#[test] +fn test_error_invalid_instruction() { + assert_eq!(asm("abcd"), Err("Invalid instruction \"abcd\"".to_string())); +} + +#[test] +fn test_error_unexpected_operands() { + assert_eq!( + asm("add 1, 2"), + Err("Failed to encode add: Unexpected operands: [Integer(1), Integer(2)]".to_string()) + ); +} + +#[test] +fn test_error_too_many_operands() { + assert_eq!( + asm("add 1, 2, 3, 4"), + Err("Failed to encode add: Too many operands".to_string()) + ); +} + +#[test] +fn test_error_operands_out_of_range() { + assert_eq!( + asm("add r16, r2"), + Err("Failed to encode add: Invalid destination register 16".to_string()) + ); + assert_eq!( + asm("add r1, r16"), + Err("Failed to encode add: Invalid source register 16".to_string()) + ); + assert_eq!( + asm("ja -32769"), + Err("Failed to encode ja: Invalid offset -32769".to_string()) + ); + assert_eq!( + asm("ja 32768"), + Err("Failed to encode ja: Invalid offset 32768".to_string()) + ); + assert_eq!( + asm("add r1, 4294967296"), + Err("Failed to encode add: Invalid immediate 4294967296".to_string()) + ); + assert_eq!( + asm("add r1, 2147483648"), + Err("Failed to encode add: Invalid immediate 2147483648".to_string()) + ); + assert_eq!( + asm("add r1, -2147483649"), + Err("Failed to encode add: Invalid immediate -2147483649".to_string()) + ); +} diff --git a/kernel/crates/rbpf/tests/common.rs b/kernel/crates/rbpf/tests/common.rs new file mode 100644 index 00000000..e3783991 --- /dev/null +++ b/kernel/crates/rbpf/tests/common.rs @@ -0,0 +1,97 @@ +// SPDX-License-Identifier: (Apache-2.0 OR MIT) +// Converted from the tests for uBPF +// Copyright 2015 Big Switch Networks, Inc +// Copyright 2016 6WIND S.A. + +// Assembly code and data for tcp_sack testcases. + +#[allow(dead_code)] +pub const TCP_SACK_ASM: &str = " + ldxb r2, [r1+12] + ldxb r3, [r1+13] + lsh r3, 0x8 + or r3, r2 + mov r0, 0x0 + jne r3, 0x8, +37 + ldxb r2, [r1+23] + jne r2, 0x6, +35 + ldxb r2, [r1+14] + add r1, 0xe + and r2, 0xf + lsh r2, 0x2 + add r1, r2 + mov r0, 0x0 + ldxh r4, [r1+12] + add r1, 0x14 + rsh r4, 0x2 + and r4, 0x3c + mov r2, r4 + add r2, -20 + mov r5, 0x15 + mov r3, 0x0 + jgt r5, r4, +20 + mov r5, r3 + lsh r5, 0x20 + arsh r5, 0x20 + mov r4, r1 + add r4, r5 + ldxb r5, [r4] + jeq r5, 0x1, +4 + jeq r5, 0x0, +12 + mov r6, r3 + jeq r5, 0x5, +9 + ja +2 + add r3, 0x1 + mov r6, r3 + ldxb r3, [r4+1] + add r3, r6 + lsh r3, 0x20 + arsh r3, 0x20 + jsgt r2, r3, -18 + ja +1 + mov r0, 0x1 + exit"; + +#[allow(dead_code)] +pub const TCP_SACK_BIN: [u8; 352] = [ + 0x71, 0x12, 0x0c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x71, 0x13, 0x0d, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x67, 0x03, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x4f, 0x23, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xb7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x55, 0x03, 0x25, 0x00, 0x08, 0x00, 0x00, 0x00, + 0x71, 0x12, 0x17, 0x00, 0x00, 0x00, 0x00, 0x00, 0x55, 0x02, 0x23, 0x00, 0x06, 0x00, 0x00, 0x00, + 0x71, 0x12, 0x0e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0x01, 0x00, 0x00, 0x0e, 0x00, 0x00, 0x00, + 0x57, 0x02, 0x00, 0x00, 0x0f, 0x00, 0x00, 0x00, 0x67, 0x02, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x0f, 0x21, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x69, 0x14, 0x0c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0x01, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, + 0x77, 0x04, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x57, 0x04, 0x00, 0x00, 0x3c, 0x00, 0x00, 0x00, + 0xbf, 0x42, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0x02, 0x00, 0x00, 0xec, 0xff, 0xff, 0xff, + 0xb7, 0x05, 0x00, 0x00, 0x15, 0x00, 0x00, 0x00, 0xb7, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x2d, 0x45, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0xbf, 0x35, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x67, 0x05, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0xc7, 0x05, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, + 0xbf, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0f, 0x54, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x71, 0x45, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x15, 0x05, 0x04, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x15, 0x05, 0x0c, 0x00, 0x00, 0x00, 0x00, 0x00, 0xbf, 0x36, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x15, 0x05, 0x09, 0x00, 0x05, 0x00, 0x00, 0x00, 0x05, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x07, 0x03, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0xbf, 0x36, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x71, 0x43, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0f, 0x63, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x67, 0x03, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0xc7, 0x03, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, + 0x6d, 0x32, 0xee, 0xff, 0x00, 0x00, 0x00, 0x00, 0x05, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xb7, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +]; + +#[allow(dead_code)] +pub const TCP_SACK_MATCH: [u8; 78] = [ + 0x00, 0x26, 0x62, 0x2f, 0x47, 0x87, 0x00, 0x1d, 0x60, 0xb3, 0x01, 0x84, 0x08, 0x00, 0x45, 0x00, + 0x00, 0x40, 0xa8, 0xde, 0x40, 0x00, 0x40, 0x06, 0x9d, 0x58, 0xc0, 0xa8, 0x01, 0x03, 0x3f, 0x74, + 0xf3, 0x61, 0xe5, 0xc0, 0x00, 0x50, 0xe5, 0x94, 0x3f, 0x77, 0xa3, 0xc4, 0xc4, 0x80, 0xb0, 0x10, + 0x01, 0x3e, 0x34, 0xb6, 0x00, 0x00, 0x01, 0x01, 0x08, 0x0a, 0x00, 0x17, 0x95, 0x6f, 0x8d, 0x9d, + 0x9e, 0x27, 0x01, 0x01, 0x05, 0x0a, 0xa3, 0xc4, 0xca, 0x28, 0xa3, 0xc4, 0xcf, 0xd0, +]; + +#[allow(dead_code)] +pub const TCP_SACK_NOMATCH: [u8; 66] = [ + 0x00, 0x26, 0x62, 0x2f, 0x47, 0x87, 0x00, 0x1d, 0x60, 0xb3, 0x01, 0x84, 0x08, 0x00, 0x45, 0x00, + 0x00, 0x40, 0xa8, 0xde, 0x40, 0x00, 0x40, 0x06, 0x9d, 0x58, 0xc0, 0xa8, 0x01, 0x03, 0x3f, 0x74, + 0xf3, 0x61, 0xe5, 0xc0, 0x00, 0x50, 0xe5, 0x94, 0x3f, 0x77, 0xa3, 0xc4, 0xc4, 0x80, 0x80, 0x10, + 0x01, 0x3e, 0x34, 0xb6, 0x00, 0x00, 0x01, 0x01, 0x08, 0x0a, 0x00, 0x17, 0x95, 0x6f, 0x8d, 0x9d, + 0x9e, 0x27, +]; diff --git a/kernel/crates/rbpf/tests/cranelift.rs b/kernel/crates/rbpf/tests/cranelift.rs new file mode 100644 index 00000000..19aca973 --- /dev/null +++ b/kernel/crates/rbpf/tests/cranelift.rs @@ -0,0 +1,2257 @@ +// SPDX-License-Identifier: (Apache-2.0 OR MIT) + +#![allow(clippy::unreadable_literal)] +#![cfg(feature = "cranelift")] + +extern crate rbpf; +mod common; + +use rbpf::{assembler::assemble, helpers}; + +use crate::common::{TCP_SACK_ASM, TCP_SACK_MATCH, TCP_SACK_NOMATCH}; + +macro_rules! test_cranelift { + ($name:ident, $prog:expr, $expected:expr) => { + #[test] + fn $name() { + let prog = assemble($prog).unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.cranelift_compile().unwrap(); + assert_eq!(vm.execute_program_cranelift().unwrap(), $expected); + } + }; + ($name:ident, $prog:expr, $mem:expr, $expected:expr) => { + #[test] + fn $name() { + let prog = assemble($prog).unwrap(); + let mem = &mut $mem; + let mut vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + vm.cranelift_compile().unwrap(); + assert_eq!(vm.execute_program_cranelift(mem).unwrap(), $expected); + } + }; +} + +test_cranelift!( + test_cranelift_add, + " + mov32 r0, 0 + mov32 r1, 2 + add32 r0, 1 + add32 r0, r1 + exit + ", + 0x3 +); + +test_cranelift!( + test_cranelift_alu64_arith, + " + mov r0, 0 + mov r1, 1 + mov r2, 2 + mov r3, 3 + mov r4, 4 + mov r5, 5 + mov r6, 6 + mov r7, 7 + mov r8, 8 + mov r9, 9 + add r0, 23 + add r0, r7 + sub r0, 13 + sub r0, r1 + mul r0, 7 + mul r0, r3 + div r0, 2 + div r0, r4 + exit + ", + 0x2a +); + +test_cranelift!( + test_cranelift_alu64_bit, + " + mov r0, 0 + mov r1, 1 + mov r2, 2 + mov r3, 3 + mov r4, 4 + mov r5, 5 + mov r6, 6 + mov r7, 7 + mov r8, 8 + or r0, r5 + or r0, 0xa0 + and r0, 0xa3 + mov r9, 0x91 + and r0, r9 + lsh r0, 32 + lsh r0, 22 + lsh r0, r8 + rsh r0, 32 + rsh r0, 19 + rsh r0, r7 + xor r0, 0x03 + xor r0, r2 + exit + ", + 0x11 +); + +test_cranelift!( + test_cranelift_alu_arith, + " + mov32 r0, 0 + mov32 r1, 1 + mov32 r2, 2 + mov32 r3, 3 + mov32 r4, 4 + mov32 r5, 5 + mov32 r6, 6 + mov32 r7, 7 + mov32 r8, 8 + mov32 r9, 9 + add32 r0, 23 + add32 r0, r7 + sub32 r0, 13 + sub32 r0, r1 + mul32 r0, 7 + mul32 r0, r3 + div32 r0, 2 + div32 r0, r4 + exit + ", + 0x2a +); + +test_cranelift!( + test_cranelift_alu_bit, + " + mov32 r0, 0 + mov32 r1, 1 + mov32 r2, 2 + mov32 r3, 3 + mov32 r4, 4 + mov32 r5, 5 + mov32 r6, 6 + mov32 r7, 7 + mov32 r8, 8 + or32 r0, r5 + or32 r0, 0xa0 + and32 r0, 0xa3 + mov32 r9, 0x91 + and32 r0, r9 + lsh32 r0, 22 + lsh32 r0, r8 + rsh32 r0, 19 + rsh32 r0, r7 + xor32 r0, 0x03 + xor32 r0, r2 + exit + ", + 0x11 +); + +test_cranelift!( + test_cranelift_arsh32_high_shift, + " + mov r0, 8 + lddw r1, 0x100000001 + arsh32 r0, r1 + exit + ", + 0x4 +); + +test_cranelift!( + test_cranelift_arsh, + " + mov32 r0, 0xf8 + lsh32 r0, 28 + arsh32 r0, 16 + exit + ", + 0xffff8000 +); + +test_cranelift!( + test_cranelift_arsh64, + " + mov32 r0, 1 + lsh r0, 63 + arsh r0, 55 + mov32 r1, 5 + arsh r0, r1 + exit + ", + 0xfffffffffffffff8 +); + +test_cranelift!( + test_cranelift_arsh_reg, + " + mov32 r0, 0xf8 + mov32 r1, 16 + lsh32 r0, 28 + arsh32 r0, r1 + exit + ", + 0xffff8000 +); + +test_cranelift!( + test_cranelift_be16, + " + ldxh r0, [r1] + be16 r0 + exit + ", + [0x11, 0x22], + 0x1122 +); + +test_cranelift!( + test_cranelift_be16_high, + " + ldxdw r0, [r1] + be16 r0 + exit + ", + [0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88], + 0x1122 +); + +test_cranelift!( + test_cranelift_be32, + " + ldxw r0, [r1] + be32 r0 + exit + ", + [0x11, 0x22, 0x33, 0x44], + 0x11223344 +); + +test_cranelift!( + test_cranelift_be32_high, + " + ldxdw r0, [r1] + be32 r0 + exit + ", + [0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88], + 0x11223344 +); + +test_cranelift!( + test_cranelift_be64, + " + ldxdw r0, [r1] + be64 r0 + exit + ", + [0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88], + 0x1122334455667788 +); + +#[test] +fn test_cranelift_call() { + let prog = assemble( + " + mov r1, 1 + mov r2, 2 + mov r3, 3 + mov r4, 4 + mov r5, 5 + call 0 + exit", + ) + .unwrap(); + + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.register_helper(0, helpers::gather_bytes).unwrap(); + vm.cranelift_compile().unwrap(); + assert_eq!(vm.execute_program_cranelift().unwrap(), 0x0102030405); +} + +#[test] +#[should_panic(expected = "[CRANELIFT] Error: unknown helper function (id: 0x3f)")] +fn test_cranelift_err_call_unreg() { + let prog = assemble( + " + mov r1, 1 + mov r2, 2 + mov r3, 3 + mov r4, 4 + mov r5, 5 + call 63 + exit + ", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.cranelift_compile().unwrap(); +} + +#[test] +fn test_cranelift_call_memfrob() { + let prog = assemble( + " + mov r6, r1 + add r1, 2 + mov r2, 4 + call 1 + ldxdw r0, [r6] + be64 r0 + exit", + ) + .unwrap(); + + let mut vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + vm.register_helper(1, helpers::memfrob).unwrap(); + let mem = &mut [0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08]; + vm.cranelift_compile().unwrap(); + assert_eq!( + vm.execute_program_cranelift(mem).unwrap(), + 0x102292e2f2c0708 + ); +} + +test_cranelift!( + test_cranelift_div32_high_divisor, + " + mov r0, 12 + lddw r1, 0x100000004 + div32 r0, r1 + exit + ", + 0x3 +); + +test_cranelift!( + test_cranelift_div32_imm, + " + lddw r0, 0x10000000c + div32 r0, 4 + exit + ", + 0x3 +); + +test_cranelift!( + test_cranelift_div32_reg, + " + lddw r0, 0x10000000c + mov r1, 4 + div32 r0, r1 + exit + ", + 0x3 +); + +test_cranelift!( + test_cranelift_div64_imm, + " + mov r0, 0xc + lsh r0, 32 + div r0, 4 + exit + ", + 0x300000000 +); + +test_cranelift!( + test_cranelift_div64_reg, + " + mov r0, 0xc + lsh r0, 32 + mov r1, 4 + div r0, r1 + exit + ", + 0x300000000 +); + +test_cranelift!( + test_cranelift_early_exit, + " + mov r0, 3 + exit + mov r0, 4 + exit + ", + 0x3 +); + +test_cranelift!( + test_cranelift_div64_by_zero_imm, + " + mov32 r0, 1 + div r0, 0 + exit + ", + 0x0 +); + +test_cranelift!( + test_cranelift_div_by_zero_imm, + " + mov32 r0, 1 + div32 r0, 0 + exit + ", + 0x0 +); + +test_cranelift!( + test_cranelift_mod64_by_zero_imm, + " + mov32 r0, 1 + mod r0, 0 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_mod_by_zero_imm, + " + mov32 r0, 1 + mod32 r0, 0 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_div64_by_zero_reg, + " + mov32 r0, 1 + mov32 r1, 0 + div r0, r1 + exit + ", + 0x0 +); + +test_cranelift!( + test_cranelift_div_by_zero_reg, + " + mov32 r0, 1 + mov32 r1, 0 + div32 r0, r1 + exit + ", + 0x0 +); + +test_cranelift!( + test_cranelift_mod64_by_zero_reg, + " + mov32 r0, 1 + mov32 r1, 0 + mod r0, r1 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_mod_by_zero_reg, + " + mov32 r0, 1 + mov32 r1, 0 + mod32 r0, r1 + exit + ", + 0x1 +); + +#[test] +// #[should_panic(expected = "Error: out of bounds memory store (insn #1)")] +#[ignore = "We have stack OOB checks, but we don't yet catch the trap code and convert it into a panic"] +fn test_cranelift_err_stack_out_of_bound() { + let prog = [ + 0x72, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, + ]; + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.cranelift_compile().unwrap(); + vm.execute_program_cranelift().unwrap(); +} + +test_cranelift!( + test_cranelift_exit, + " + mov r0, 0 + exit + ", + 0x0 +); + +test_cranelift!( + test_cranelift_ja, + " + mov r0, 1 + ja +1 + mov r0, 2 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_jeq_imm, + " + mov32 r0, 0 + mov32 r1, 0xa + jeq r1, 0xb, +4 + mov32 r0, 1 + mov32 r1, 0xb + jeq r1, 0xb, +1 + mov32 r0, 2 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_jeq_reg, + " + mov32 r0, 0 + mov32 r1, 0xa + mov32 r2, 0xb + jeq r1, r2, +4 + mov32 r0, 1 + mov32 r1, 0xb + jeq r1, r2, +1 + mov32 r0, 2 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_jge_imm, + " + mov32 r0, 0 + mov32 r1, 0xa + jge r1, 0xb, +4 + mov32 r0, 1 + mov32 r1, 0xc + jge r1, 0xb, +1 + mov32 r0, 2 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_jle_imm, + " + mov32 r0, 0 + mov32 r1, 5 + jle r1, 4, +1 + jle r1, 6, +1 + exit + jle r1, 5, +1 + exit + mov32 r0, 1 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_jle_reg, + " + mov r0, 0 + mov r1, 5 + mov r2, 4 + mov r3, 6 + jle r1, r2, +2 + jle r1, r1, +1 + exit + jle r1, r3, +1 + exit + mov r0, 1 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_jgt_imm, + " + mov32 r0, 0 + mov32 r1, 5 + jgt r1, 6, +2 + jgt r1, 5, +1 + jgt r1, 4, +1 + exit + mov32 r0, 1 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_jgt_reg, + " + mov r0, 0 + mov r1, 5 + mov r2, 6 + mov r3, 4 + jgt r1, r2, +2 + jgt r1, r1, +1 + jgt r1, r3, +1 + exit + mov r0, 1 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_jlt_imm, + " + mov32 r0, 0 + mov32 r1, 5 + jlt r1, 4, +2 + jlt r1, 5, +1 + jlt r1, 6, +1 + exit + mov32 r0, 1 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_jlt_reg, + " + mov r0, 0 + mov r1, 5 + mov r2, 4 + mov r3, 6 + jlt r1, r2, +2 + jlt r1, r1, +1 + jlt r1, r3, +1 + exit + mov r0, 1 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_jit_bounce, + " + mov r0, 1 + mov r6, r0 + mov r7, r6 + mov r8, r7 + mov r9, r8 + mov r0, r9 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_jne_reg, + " + mov32 r0, 0 + mov32 r1, 0xb + mov32 r2, 0xb + jne r1, r2, +4 + mov32 r0, 1 + mov32 r1, 0xa + jne r1, r2, +1 + mov32 r0, 2 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_jset_imm, + " + mov32 r0, 0 + mov32 r1, 0x7 + jset r1, 0x8, +4 + mov32 r0, 1 + mov32 r1, 0x9 + jset r1, 0x8, +1 + mov32 r0, 2 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_jset_reg, + " + mov32 r0, 0 + mov32 r1, 0x7 + mov32 r2, 0x8 + jset r1, r2, +4 + mov32 r0, 1 + mov32 r1, 0x9 + jset r1, r2, +1 + mov32 r0, 2 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_jsge_imm, + " + mov32 r0, 0 + mov r1, -2 + jsge r1, -1, +5 + jsge r1, 0, +4 + mov32 r0, 1 + mov r1, -1 + jsge r1, -1, +1 + mov32 r0, 2 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_jsge_reg, + " + mov32 r0, 0 + mov r1, -2 + mov r2, -1 + mov32 r3, 0 + jsge r1, r2, +5 + jsge r1, r3, +4 + mov32 r0, 1 + mov r1, r2 + jsge r1, r2, +1 + mov32 r0, 2 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_jsle_imm, + " + mov32 r0, 0 + mov r1, -2 + jsle r1, -3, +1 + jsle r1, -1, +1 + exit + mov32 r0, 1 + jsle r1, -2, +1 + mov32 r0, 2 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_jsle_reg, + " + mov32 r0, 0 + mov r1, -1 + mov r2, -2 + mov32 r3, 0 + jsle r1, r2, +1 + jsle r1, r3, +1 + exit + mov32 r0, 1 + mov r1, r2 + jsle r1, r2, +1 + mov32 r0, 2 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_jsgt_imm, + " + mov32 r0, 0 + mov r1, -2 + jsgt r1, -1, +4 + mov32 r0, 1 + mov32 r1, 0 + jsgt r1, -1, +1 + mov32 r0, 2 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_jsgt_reg, + " + mov32 r0, 0 + mov r1, -2 + mov r2, -1 + jsgt r1, r2, +4 + mov32 r0, 1 + mov32 r1, 0 + jsgt r1, r2, +1 + mov32 r0, 2 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_jslt_imm, + " + mov32 r0, 0 + mov r1, -2 + jslt r1, -3, +2 + jslt r1, -2, +1 + jslt r1, -1, +1 + exit + mov32 r0, 1 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_jslt_reg, + " + mov32 r0, 0 + mov r1, -2 + mov r2, -3 + mov r3, -1 + jslt r1, r1, +2 + jslt r1, r2, +1 + jslt r1, r3, +1 + exit + mov32 r0, 1 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_jeq32_imm, + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0x0 + mov32 r1, 0xa + jeq32 r1, 0xb, +5 + mov32 r0, 1 + mov r1, 0xb + or r1, r9 + jeq32 r1, 0xb, +1 + mov32 r0, 2 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_jeq32_reg, + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, 0xa + mov32 r2, 0xb + jeq32 r1, r2, +5 + mov32 r0, 1 + mov32 r1, 0xb + or r1, r9 + jeq32 r1, r2, +1 + mov32 r0, 2 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_jge32_imm, + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, 0xa + jge32 r1, 0xb, +5 + mov32 r0, 1 + or r1, r9 + mov32 r1, 0xc + jge32 r1, 0xb, +1 + mov32 r0, 2 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_jge32_reg, + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, 0xa + mov32 r2, 0xb + jge32 r1, r2, +5 + mov32 r0, 1 + or r1, r9 + mov32 r1, 0xc + jge32 r1, r2, +1 + mov32 r0, 2 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_jgt32_imm, + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, 5 + or r1, r9 + jgt32 r1, 6, +4 + jgt32 r1, 5, +3 + jgt32 r1, 4, +1 + exit + mov32 r0, 1 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_jgt32_reg, + " + mov r9, 1 + lsh r9, 32 + mov r0, 0 + mov r1, 5 + mov32 r1, 5 + or r1, r9 + mov r2, 6 + mov r3, 4 + jgt32 r1, r2, +4 + jgt32 r1, r1, +3 + jgt32 r1, r3, +1 + exit + mov r0, 1 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_jle32_imm, + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, 5 + or r1, r9 + jle32 r1, 4, +5 + jle32 r1, 6, +1 + exit + jle32 r1, 5, +1 + exit + mov32 r0, 1 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_jle32_reg, + " + mov r9, 1 + lsh r9, 32 + mov r0, 0 + mov r1, 5 + mov r2, 4 + mov r3, 6 + or r1, r9 + jle32 r1, r2, +5 + jle32 r1, r1, +1 + exit + jle32 r1, r3, +1 + exit + mov r0, 1 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_jlt32_imm, + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, 5 + or r1, r9 + jlt32 r1, 4, +4 + jlt32 r1, 5, +3 + jlt32 r1, 6, +1 + exit + mov32 r0, 1 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_jlt32_reg, + " + mov r9, 1 + lsh r9, 32 + mov r0, 0 + mov r1, 5 + mov r2, 4 + mov r3, 6 + or r1, r9 + jlt32 r1, r2, +4 + jlt32 r1, r1, +3 + jlt32 r1, r3, +1 + exit + mov r0, 1 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_jne32_imm, + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, 0xb + or r1, r9 + jne32 r1, 0xb, +4 + mov32 r0, 1 + mov32 r1, 0xa + or r1, r9 + jne32 r1, 0xb, +1 + mov32 r0, 2 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_jne32_reg, + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, 0xb + or r1, r9 + mov32 r2, 0xb + jne32 r1, r2, +4 + mov32 r0, 1 + mov32 r1, 0xa + or r1, r9 + jne32 r1, r2, +1 + mov32 r0, 2 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_jset32_imm, + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, 0x7 + or r1, r9 + jset32 r1, 0x8, +4 + mov32 r0, 1 + mov32 r1, 0x9 + jset32 r1, 0x8, +1 + mov32 r0, 2 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_jset32_reg, + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, 0x7 + or r1, r9 + mov32 r2, 0x8 + jset32 r1, r2, +4 + mov32 r0, 1 + mov32 r1, 0x9 + jset32 r1, r2, +1 + mov32 r0, 2 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_jsge32_imm, + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, -2 + or r1, r9 + jsge32 r1, -1, +5 + jsge32 r1, 0, +4 + mov32 r0, 1 + mov r1, -1 + jsge32 r1, -1, +1 + mov32 r0, 2 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_jsge32_reg, + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, -2 + or r1, r9 + mov r2, -1 + mov32 r3, 0 + jsge32 r1, r2, +5 + jsge32 r1, r3, +4 + mov32 r0, 1 + mov r1, r2 + jsge32 r1, r2, +1 + mov32 r0, 2 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_jsgt32_imm, + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, -2 + or r1, r9 + jsgt32 r1, -1, +4 + mov32 r0, 1 + mov32 r1, 0 + jsgt32 r1, -1, +1 + mov32 r0, 2 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_jsgt32_reg, + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, -2 + or r1, r9 + mov r2, -1 + jsgt32 r1, r2, +4 + mov32 r0, 1 + mov32 r1, 0 + jsgt32 r1, r2, +1 + mov32 r0, 2 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_jsle32_imm, + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, -2 + or r1, r9 + jsle32 r1, -3, +5 + jsle32 r1, -1, +1 + exit + mov32 r0, 1 + jsle32 r1, -2, +1 + mov32 r0, 2 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_jsle32_reg, + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, -2 + or r1, r9 + mov r2, -3 + mov32 r3, 0 + jsle32 r1, r2, +6 + jsle32 r1, r3, +1 + exit + mov32 r0, 1 + mov r1, r2 + jsle32 r1, r2, +1 + mov32 r0, 2 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_jslt32_imm, + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, -2 + or r1, r9 + jslt32 r1, -3, +4 + jslt32 r1, -2, +3 + jslt32 r1, -1, +1 + exit + mov32 r0, 1 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_jslt32_reg, + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, -2 + or r1, r9 + mov r2, -3 + mov r3, -1 + jslt32 r1, r1, +4 + jslt32 r1, r2, +3 + jslt32 r1, r3, +1 + exit + mov32 r0, 1 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_lddw, + " + lddw r0, 0x1122334455667788 + exit + ", + 0x1122334455667788 +); + +test_cranelift!( + test_cranelift_lddw2, + " + lddw r0, 0x0000000080000000 + exit + ", + 0x80000000 +); + +test_cranelift!( + test_cranelift_ldxb_all, + " + mov r0, r1 + ldxb r9, [r0+0] + lsh r9, 0 + ldxb r8, [r0+1] + lsh r8, 4 + ldxb r7, [r0+2] + lsh r7, 8 + ldxb r6, [r0+3] + lsh r6, 12 + ldxb r5, [r0+4] + lsh r5, 16 + ldxb r4, [r0+5] + lsh r4, 20 + ldxb r3, [r0+6] + lsh r3, 24 + ldxb r2, [r0+7] + lsh r2, 28 + ldxb r1, [r0+8] + lsh r1, 32 + ldxb r0, [r0+9] + lsh r0, 36 + or r0, r1 + or r0, r2 + or r0, r3 + or r0, r4 + or r0, r5 + or r0, r6 + or r0, r7 + or r0, r8 + or r0, r9 + exit + ", + [0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09], + 0x9876543210 +); + +test_cranelift!( + test_cranelift_ldxb, + " + ldxb r0, [r1+2] + exit + ", + [0xaa, 0xbb, 0x11, 0xcc, 0xdd], + 0x11 +); + +test_cranelift!( + test_cranelift_ldxdw, + " + ldxdw r0, [r1+2] + exit + ", + [0xaa, 0xbb, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0xcc, 0xdd], + 0x8877665544332211 +); + +test_cranelift!( + test_cranelift_ldxh_all, + " + mov r0, r1 + ldxh r9, [r0+0] + be16 r9 + lsh r9, 0 + ldxh r8, [r0+2] + be16 r8 + lsh r8, 4 + ldxh r7, [r0+4] + be16 r7 + lsh r7, 8 + ldxh r6, [r0+6] + be16 r6 + lsh r6, 12 + ldxh r5, [r0+8] + be16 r5 + lsh r5, 16 + ldxh r4, [r0+10] + be16 r4 + lsh r4, 20 + ldxh r3, [r0+12] + be16 r3 + lsh r3, 24 + ldxh r2, [r0+14] + be16 r2 + lsh r2, 28 + ldxh r1, [r0+16] + be16 r1 + lsh r1, 32 + ldxh r0, [r0+18] + be16 r0 + lsh r0, 36 + or r0, r1 + or r0, r2 + or r0, r3 + or r0, r4 + or r0, r5 + or r0, r6 + or r0, r7 + or r0, r8 + or r0, r9 + exit + ", + [ + 0x00, 0x00, 0x00, 0x01, 0x00, 0x02, 0x00, 0x03, 0x00, 0x04, 0x00, 0x05, 0x00, 0x06, 0x00, + 0x07, 0x00, 0x08, 0x00, 0x09 + ], + 0x9876543210 +); + +test_cranelift!( + test_cranelift_ldxh_all2, + " + mov r0, r1 + ldxh r9, [r0+0] + be16 r9 + ldxh r8, [r0+2] + be16 r8 + ldxh r7, [r0+4] + be16 r7 + ldxh r6, [r0+6] + be16 r6 + ldxh r5, [r0+8] + be16 r5 + ldxh r4, [r0+10] + be16 r4 + ldxh r3, [r0+12] + be16 r3 + ldxh r2, [r0+14] + be16 r2 + ldxh r1, [r0+16] + be16 r1 + ldxh r0, [r0+18] + be16 r0 + or r0, r1 + or r0, r2 + or r0, r3 + or r0, r4 + or r0, r5 + or r0, r6 + or r0, r7 + or r0, r8 + or r0, r9 + exit + ", + [ + 0x00, 0x01, 0x00, 0x02, 0x00, 0x04, 0x00, 0x08, 0x00, 0x10, 0x00, 0x20, 0x00, 0x40, 0x00, + 0x80, 0x01, 0x00, 0x02, 0x00 + ], + 0x3ff +); + +test_cranelift!( + test_cranelift_ldxh, + " + ldxh r0, [r1+2] + exit + ", + [0xaa, 0xbb, 0x11, 0x22, 0xcc, 0xdd], + 0x2211 +); + +test_cranelift!( + test_cranelift_ldxh_same_reg, + " + mov r0, r1 + sth [r0], 0x1234 + ldxh r0, [r0] + exit + ", + [0xff, 0xff], + 0x1234 +); + +test_cranelift!( + test_cranelift_ldxw_all, + " + mov r0, r1 + ldxw r9, [r0+0] + be32 r9 + ldxw r8, [r0+4] + be32 r8 + ldxw r7, [r0+8] + be32 r7 + ldxw r6, [r0+12] + be32 r6 + ldxw r5, [r0+16] + be32 r5 + ldxw r4, [r0+20] + be32 r4 + ldxw r3, [r0+24] + be32 r3 + ldxw r2, [r0+28] + be32 r2 + ldxw r1, [r0+32] + be32 r1 + ldxw r0, [r0+36] + be32 r0 + or r0, r1 + or r0, r2 + or r0, r3 + or r0, r4 + or r0, r5 + or r0, r6 + or r0, r7 + or r0, r8 + or r0, r9 + exit + ", + [ + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x08, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x08, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00 + ], + 0x030f0f +); + +test_cranelift!( + test_cranelift_ldxw, + " + ldxw r0, [r1+2] + exit + ", + [0xaa, 0xbb, 0x11, 0x22, 0x33, 0x44, 0xcc, 0xdd], + 0x44332211 +); + +test_cranelift!( + test_cranelift_le16, + " + ldxh r0, [r1] + le16 r0 + exit + ", + [0x22, 0x11], + 0x1122 +); + +test_cranelift!( + test_cranelift_le32, + " + ldxw r0, [r1] + le32 r0 + exit + ", + [0x44, 0x33, 0x22, 0x11], + 0x11223344 +); + +test_cranelift!( + test_cranelift_le64, + " + ldxdw r0, [r1] + le64 r0 + exit + ", + [0x88, 0x77, 0x66, 0x55, 0x44, 0x33, 0x22, 0x11], + 0x1122334455667788 +); + +test_cranelift!( + test_cranelift_lsh_reg, + " + mov r0, 0x1 + mov r7, 4 + lsh r0, r7 + exit + ", + 0x10 +); + +test_cranelift!( + test_cranelift_mod, + " + mov32 r0, 5748 + mod32 r0, 92 + mov32 r1, 13 + mod32 r0, r1 + exit + ", + 0x5 +); + +test_cranelift!( + test_cranelift_mod32, + " + lddw r0, 0x100000003 + mod32 r0, 3 + exit + ", + 0x0 +); + +test_cranelift!( + test_cranelift_mod64, + " + mov32 r0, -1316649930 + lsh r0, 32 + or r0, 0x100dc5c8 + mov32 r1, 0xdde263e + lsh r1, 32 + or r1, 0x3cbef7f3 + mod r0, r1 + mod r0, 0x658f1778 + exit + ", + 0x30ba5a04 +); + +test_cranelift!( + test_cranelift_mov, + " + mov32 r1, 1 + mov32 r0, r1 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_mul32_imm, + " + mov r0, 3 + mul32 r0, 4 + exit + ", + 0xc +); + +test_cranelift!( + test_cranelift_mul32_reg, + " + mov r0, 3 + mov r1, 4 + mul32 r0, r1 + exit + ", + 0xc +); + +test_cranelift!( + test_cranelift_mul32_reg_overflow, + " + mov r0, 0x40000001 + mov r1, 4 + mul32 r0, r1 + exit + ", + 0x4 +); + +test_cranelift!( + test_cranelift_mul64_imm, + " + mov r0, 0x40000001 + mul r0, 4 + exit + ", + 0x100000004 +); + +test_cranelift!( + test_cranelift_mul64_reg, + " + mov r0, 0x40000001 + mov r1, 4 + mul r0, r1 + exit + ", + 0x100000004 +); + +test_cranelift!( + test_cranelift_mul_loop, + " + mov r0, 0x7 + add r1, 0xa + lsh r1, 0x20 + rsh r1, 0x20 + jeq r1, 0x0, +4 + mov r0, 0x7 + mul r0, 0x7 + add r1, -1 + jne r1, 0x0, -3 + exit + ", + 0x75db9c97 +); + +test_cranelift!( + test_cranelift_neg64, + " + mov32 r0, 2 + neg r0 + exit + ", + 0xfffffffffffffffe +); + +test_cranelift!( + test_cranelift_neg, + " + mov32 r0, 2 + neg32 r0 + exit + ", + 0xfffffffe +); + +test_cranelift!( + test_cranelift_prime, + " + mov r1, 67 + mov r0, 0x1 + mov r2, 0x2 + jgt r1, 0x2, +4 + ja +10 + add r2, 0x1 + mov r0, 0x1 + jge r2, r1, +7 + mov r3, r1 + div r3, r2 + mul r3, r2 + mov r4, r1 + sub r4, r3 + mov r0, 0x0 + jne r4, 0x0, -10 + exit + ", + 1 +); + +test_cranelift!( + test_cranelift_rhs32, + " + xor r0, r0 + sub r0, 1 + rsh32 r0, 8 + exit + ", + 0x00ffffff +); + +test_cranelift!( + test_cranelift_rsh_reg, + " + mov r0, 0x10 + mov r7, 4 + rsh r0, r7 + exit + ", + 0x1 +); + +test_cranelift!( + test_cranelift_stack, + " + mov r1, 51 + stdw [r10-16], 0xab + stdw [r10-8], 0xcd + and r1, 1 + lsh r1, 3 + mov r2, r10 + add r2, r1 + ldxdw r0, [r2-16] + exit + ", + 0xcd +); + +#[test] +fn test_cranelift_stack2() { + let prog = assemble( + " + stb [r10-4], 0x01 + stb [r10-3], 0x02 + stb [r10-2], 0x03 + stb [r10-1], 0x04 + mov r1, r10 + mov r2, 0x4 + sub r1, r2 + call 1 + mov r1, 0 + ldxb r2, [r10-4] + ldxb r3, [r10-3] + ldxb r4, [r10-2] + ldxb r5, [r10-1] + call 0 + xor r0, 0x2a2a2a2a + exit", + ) + .unwrap(); + + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.register_helper(0, helpers::gather_bytes).unwrap(); + vm.register_helper(1, helpers::memfrob).unwrap(); + vm.cranelift_compile().unwrap(); + assert_eq!(vm.execute_program_cranelift().unwrap(), 0x01020304); +} + +test_cranelift!( + test_cranelift_stb, + " + stb [r1+2], 0x11 + ldxb r0, [r1+2] + exit + ", + [0xaa, 0xbb, 0xff, 0xcc, 0xdd], + 0x11 +); + +test_cranelift!( + test_cranelift_stdw, + " + stdw [r1+2], 0x44332211 + ldxdw r0, [r1+2] + exit + ", + [0xaa, 0xbb, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xcc, 0xdd], + 0x44332211 +); + +test_cranelift!( + test_cranelift_sth, + " + sth [r1+2], 0x2211 + ldxh r0, [r1+2] + exit + ", + [0xaa, 0xbb, 0xff, 0xff, 0xcc, 0xdd], + 0x2211 +); + +#[test] +#[ignore] +fn test_cranelift_string_stack() { + let prog = assemble( + " + mov r1, 0x78636261 + stxw [r10-8], r1 + mov r6, 0x0 + stxb [r10-4], r6 + stxb [r10-12], r6 + mov r1, 0x79636261 + stxw [r10-16], r1 + mov r1, r10 + add r1, -8 + mov r2, r1 + call 0x4 + mov r1, r0 + mov r0, 0x1 + lsh r1, 0x20 + rsh r1, 0x20 + jne r1, 0x0, +11 + mov r1, r10 + add r1, -8 + mov r2, r10 + add r2, -16 + call 0x4 + mov r1, r0 + lsh r1, 0x20 + rsh r1, 0x20 + mov r0, 0x1 + jeq r1, r6, +1 + mov r0, 0x0 + exit", + ) + .unwrap(); + + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.register_helper(4, helpers::strcmp).unwrap(); + vm.cranelift_compile().unwrap(); + assert_eq!(vm.execute_program_cranelift().unwrap(), 0x0); +} + +test_cranelift!( + test_cranelift_stw, + " + stw [r1+2], 0x44332211 + ldxw r0, [r1+2] + exit + ", + [0xaa, 0xbb, 0xff, 0xff, 0xff, 0xff, 0xcc, 0xdd], + 0x44332211 +); + +test_cranelift!( + test_cranelift_stxb, + " + mov32 r2, 0x11 + stxb [r1+2], r2 + ldxb r0, [r1+2] + exit + ", + [0xaa, 0xbb, 0xff, 0xcc, 0xdd], + 0x11 +); + +test_cranelift!( + test_cranelift_stxb_all, + " + mov r0, 0xf0 + mov r2, 0xf2 + mov r3, 0xf3 + mov r4, 0xf4 + mov r5, 0xf5 + mov r6, 0xf6 + mov r7, 0xf7 + mov r8, 0xf8 + stxb [r1], r0 + stxb [r1+1], r2 + stxb [r1+2], r3 + stxb [r1+3], r4 + stxb [r1+4], r5 + stxb [r1+5], r6 + stxb [r1+6], r7 + stxb [r1+7], r8 + ldxdw r0, [r1] + be64 r0 + exit + ", + [0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff], + 0xf0f2f3f4f5f6f7f8 +); + +test_cranelift!( + test_cranelift_stxb_all2, + " + mov r0, r1 + mov r1, 0xf1 + mov r9, 0xf9 + stxb [r0], r1 + stxb [r0+1], r9 + ldxh r0, [r0] + be16 r0 + exit + ", + [0xff, 0xff], + 0xf1f9 +); + +test_cranelift!( + test_cranelift_stxb_chain, + " + mov r0, r1 + ldxb r9, [r0+0] + stxb [r0+1], r9 + ldxb r8, [r0+1] + stxb [r0+2], r8 + ldxb r7, [r0+2] + stxb [r0+3], r7 + ldxb r6, [r0+3] + stxb [r0+4], r6 + ldxb r5, [r0+4] + stxb [r0+5], r5 + ldxb r4, [r0+5] + stxb [r0+6], r4 + ldxb r3, [r0+6] + stxb [r0+7], r3 + ldxb r2, [r0+7] + stxb [r0+8], r2 + ldxb r1, [r0+8] + stxb [r0+9], r1 + ldxb r0, [r0+9] + exit + ", + [0x2a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00], + 0x2a +); + +test_cranelift!( + test_cranelift_stxdw, + " + mov r2, -2005440939 + lsh r2, 32 + or r2, 0x44332211 + stxdw [r1+2], r2 + ldxdw r0, [r1+2] + exit + ", + [0xaa, 0xbb, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xcc, 0xdd], + 0x8877665544332211 +); + +test_cranelift!( + test_cranelift_stxh, + " + mov32 r2, 0x2211 + stxh [r1+2], r2 + ldxh r0, [r1+2] + exit + ", + [0xaa, 0xbb, 0xff, 0xff, 0xcc, 0xdd], + 0x2211 +); + +test_cranelift!( + test_cranelift_stxw, + " + mov32 r2, 0x44332211 + stxw [r1+2], r2 + ldxw r0, [r1+2] + exit + ", + [0xaa, 0xbb, 0xff, 0xff, 0xff, 0xff, 0xcc, 0xdd], + 0x44332211 +); + +test_cranelift!( + test_cranelift_subnet, + " + mov r2, 0xe + ldxh r3, [r1+12] + jne r3, 0x81, +2 + mov r2, 0x12 + ldxh r3, [r1+16] + and r3, 0xffff + jne r3, 0x8, +5 + add r1, r2 + mov r0, 0x1 + ldxw r1, [r1+16] + and r1, 0xffffff + jeq r1, 0x1a8c0, +1 + mov r0, 0x0 + exit + ", + [ + 0x00, 0x00, 0xc0, 0x9f, 0xa0, 0x97, 0x00, 0xa0, 0xcc, 0x3b, 0xbf, 0xfa, 0x08, 0x00, 0x45, + 0x10, 0x00, 0x3c, 0x46, 0x3c, 0x40, 0x00, 0x40, 0x06, 0x73, 0x1c, 0xc0, 0xa8, 0x01, 0x02, + 0xc0, 0xa8, 0x01, 0x01, 0x06, 0x0e, 0x00, 0x17, 0x99, 0xc5, 0xa0, 0xec, 0x00, 0x00, 0x00, + 0x00, 0xa0, 0x02, 0x7d, 0x78, 0xe0, 0xa3, 0x00, 0x00, 0x02, 0x04, 0x05, 0xb4, 0x04, 0x02, + 0x08, 0x0a, 0x00, 0x9c, 0x27, 0x24, 0x00, 0x00, 0x00, 0x00, 0x01, 0x03, 0x03, 0x00, + ], + 0x1 +); + +const PROG_TCP_PORT_80: [u8; 152] = [ + 0x71, 0x12, 0x0c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x71, 0x13, 0x0d, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x67, 0x03, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x4f, 0x23, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xb7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x55, 0x03, 0x0c, 0x00, 0x08, 0x00, 0x00, 0x00, + 0x71, 0x12, 0x17, 0x00, 0x00, 0x00, 0x00, 0x00, 0x55, 0x02, 0x0a, 0x00, 0x06, 0x00, 0x00, 0x00, + 0x71, 0x12, 0x0e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0x01, 0x00, 0x00, 0x0e, 0x00, 0x00, 0x00, + 0x57, 0x02, 0x00, 0x00, 0x0f, 0x00, 0x00, 0x00, 0x67, 0x02, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x0f, 0x21, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x69, 0x12, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x15, 0x02, 0x02, 0x00, 0x00, 0x50, 0x00, 0x00, 0x69, 0x11, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x55, 0x01, 0x01, 0x00, 0x00, 0x50, 0x00, 0x00, 0xb7, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +]; + +#[test] +fn test_cranelift_tcp_port80_match() { + let mem = &mut [ + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x00, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x08, 0x00, 0x45, + 0x00, 0x00, 0x56, 0x00, 0x01, 0x00, 0x00, 0x40, 0x06, 0xf9, 0x4d, 0xc0, 0xa8, 0x00, 0x01, + 0xc0, 0xa8, 0x00, 0x02, 0x27, 0x10, 0x00, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x50, 0x02, 0x20, 0x00, 0xc5, 0x18, 0x00, 0x00, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + ]; + let prog = &PROG_TCP_PORT_80; + let mut vm = rbpf::EbpfVmRaw::new(Some(prog)).unwrap(); + vm.cranelift_compile().unwrap(); + assert_eq!(vm.execute_program_cranelift(mem).unwrap(), 0x1); +} + +#[test] +fn test_cranelift_tcp_port80_nomatch() { + let mem = &mut [ + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x00, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x08, 0x00, 0x45, + 0x00, 0x00, 0x56, 0x00, 0x01, 0x00, 0x00, 0x40, 0x06, 0xf9, 0x4d, 0xc0, 0xa8, 0x00, 0x01, + 0xc0, 0xa8, 0x00, 0x02, 0x00, 0x16, 0x27, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x51, 0x02, 0x20, 0x00, 0xc5, 0x18, 0x00, 0x00, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + ]; + let prog = &PROG_TCP_PORT_80; + let mut vm = rbpf::EbpfVmRaw::new(Some(prog)).unwrap(); + vm.cranelift_compile().unwrap(); + assert_eq!(vm.execute_program_cranelift(mem).unwrap(), 0x0); +} + +#[test] +fn test_cranelift_tcp_port80_nomatch_ethertype() { + let mem = &mut [ + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x00, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x08, 0x01, 0x45, + 0x00, 0x00, 0x56, 0x00, 0x01, 0x00, 0x00, 0x40, 0x06, 0xf9, 0x4d, 0xc0, 0xa8, 0x00, 0x01, + 0xc0, 0xa8, 0x00, 0x02, 0x27, 0x10, 0x00, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x50, 0x02, 0x20, 0x00, 0xc5, 0x18, 0x00, 0x00, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + ]; + let prog = &PROG_TCP_PORT_80; + let mut vm = rbpf::EbpfVmRaw::new(Some(prog)).unwrap(); + vm.cranelift_compile().unwrap(); + assert_eq!(vm.execute_program_cranelift(mem).unwrap(), 0x0); +} + +#[test] +fn test_cranelift_tcp_port80_nomatch_proto() { + let mem = &mut [ + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x00, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x08, 0x00, 0x45, + 0x00, 0x00, 0x56, 0x00, 0x01, 0x00, 0x00, 0x40, 0x11, 0xf9, 0x4d, 0xc0, 0xa8, 0x00, 0x01, + 0xc0, 0xa8, 0x00, 0x02, 0x27, 0x10, 0x00, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x50, 0x02, 0x20, 0x00, 0xc5, 0x18, 0x00, 0x00, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + ]; + let prog = &PROG_TCP_PORT_80; + let mut vm = rbpf::EbpfVmRaw::new(Some(prog)).unwrap(); + vm.cranelift_compile().unwrap(); + assert_eq!(vm.execute_program_cranelift(mem).unwrap(), 0x0); +} + +#[test] +fn test_cranelift_tcp_sack_match() { + let mut mem = TCP_SACK_MATCH.to_vec(); + let prog = assemble(TCP_SACK_ASM).unwrap(); + let mut vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + vm.cranelift_compile().unwrap(); + assert_eq!( + vm.execute_program_cranelift(mem.as_mut_slice()).unwrap(), + 0x1 + ); +} + +#[test] +fn test_cranelift_tcp_sack_nomatch() { + let mut mem = TCP_SACK_NOMATCH.to_vec(); + let prog = assemble(TCP_SACK_ASM).unwrap(); + let mut vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + vm.cranelift_compile().unwrap(); + assert_eq!( + vm.execute_program_cranelift(mem.as_mut_slice()).unwrap(), + 0x0 + ); +} + +#[test] +fn test_cranelift_ldabsb() { + let prog = &[ + 0x30, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, + ]; + let mem = &mut [ + 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, + 0xff, + ]; + let mut vm = rbpf::EbpfVmFixedMbuff::new(Some(prog), 0x00, 0x08).unwrap(); + + vm.cranelift_compile().unwrap(); + assert_eq!(vm.execute_program_cranelift(mem).unwrap(), 0x33); +} + +#[test] +fn test_cranelift_ldabsh() { + let prog = &[ + 0x28, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, + ]; + let mem = &mut [ + 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, + 0xff, + ]; + let mut vm = rbpf::EbpfVmFixedMbuff::new(Some(prog), 0x00, 0x08).unwrap(); + + vm.cranelift_compile().unwrap(); + assert_eq!(vm.execute_program_cranelift(mem).unwrap(), 0x4433); +} + +#[test] +fn test_cranelift_ldabsw() { + let prog = &[ + 0x20, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, + ]; + let mem = &mut [ + 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, + 0xff, + ]; + let mut vm = rbpf::EbpfVmFixedMbuff::new(Some(prog), 0x00, 0x08).unwrap(); + + vm.cranelift_compile().unwrap(); + assert_eq!(vm.execute_program_cranelift(mem).unwrap(), 0x66554433); +} + +#[test] +fn test_cranelift_ldabsdw() { + let prog = &[ + 0x38, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, + ]; + let mem = &mut [ + 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, + 0xff, + ]; + let mut vm = rbpf::EbpfVmFixedMbuff::new(Some(prog), 0x00, 0x08).unwrap(); + + vm.cranelift_compile().unwrap(); + assert_eq!( + vm.execute_program_cranelift(mem).unwrap(), + 0xaa99887766554433 + ); +} + +#[test] +fn test_cranelift_ldindb() { + let prog = &[ + 0xb7, 0x01, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x50, 0x10, 0x00, 0x00, 0x03, 0x00, 0x00, + 0x00, 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + ]; + let mem = &mut [ + 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, + 0xff, + ]; + let mut vm = rbpf::EbpfVmFixedMbuff::new(Some(prog), 0x00, 0x08).unwrap(); + + vm.cranelift_compile().unwrap(); + assert_eq!(vm.execute_program_cranelift(mem).unwrap(), 0x88); +} + +#[test] +fn test_cranelift_ldindh() { + let prog = &[ + 0xb7, 0x01, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x48, 0x10, 0x00, 0x00, 0x03, 0x00, 0x00, + 0x00, 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + ]; + let mem = &mut [ + 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, + 0xff, + ]; + let mut vm = rbpf::EbpfVmFixedMbuff::new(Some(prog), 0x00, 0x08).unwrap(); + + vm.cranelift_compile().unwrap(); + assert_eq!(vm.execute_program_cranelift(mem).unwrap(), 0x9988); +} + +#[test] +fn test_cranelift_ldindw() { + let prog = &[ + 0xb7, 0x01, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x40, 0x10, 0x00, 0x00, 0x01, 0x00, 0x00, + 0x00, 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + ]; + let mem = &mut [ + 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, + 0xff, + ]; + let mut vm = rbpf::EbpfVmFixedMbuff::new(Some(prog), 0x00, 0x08).unwrap(); + + vm.cranelift_compile().unwrap(); + assert_eq!(vm.execute_program_cranelift(mem).unwrap(), 0x88776655); +} + +#[test] +fn test_cranelift_ldinddw() { + let prog = &[ + 0xb7, 0x01, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x58, 0x10, 0x00, 0x00, 0x03, 0x00, 0x00, + 0x00, 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + ]; + let mem = &mut [ + 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, + 0xff, + ]; + let mut vm = rbpf::EbpfVmFixedMbuff::new(Some(prog), 0x00, 0x08).unwrap(); + + vm.cranelift_compile().unwrap(); + assert_eq!( + vm.execute_program_cranelift(mem).unwrap(), + 0xccbbaa9988776655 + ); +} diff --git a/kernel/crates/rbpf/tests/disassembler.rs b/kernel/crates/rbpf/tests/disassembler.rs new file mode 100644 index 00000000..90f7ebe4 --- /dev/null +++ b/kernel/crates/rbpf/tests/disassembler.rs @@ -0,0 +1,377 @@ +// SPDX-License-Identifier: (Apache-2.0 OR MIT) +// Copyright 2017 Jan-Erik Rediger +// +// Adopted from tests in `tests/assembler.rs` + +extern crate rbpf; +mod common; + +use rbpf::{assembler::assemble, disassembler::to_insn_vec}; + +// Using a macro to keep actual line numbers in failure output +macro_rules! disasm { + ($src:expr) => {{ + let src = $src; + let asm = assemble(src).expect("Can't assemble from string"); + let insn = to_insn_vec(&asm); + let reasm = insn + .into_iter() + .map(|ins| ins.desc) + .collect::>() + .join("\n"); + + assert_eq!(src, reasm); + }}; +} + +#[test] +fn test_empty() { + disasm!(""); +} + +// Example for InstructionType::NoOperand. +#[test] +fn test_exit() { + disasm!("exit"); +} + +// Example for InstructionType::AluBinary. +#[test] +fn test_add64() { + disasm!("add64 r1, r3"); + disasm!("add64 r1, 0x5"); +} + +// Example for InstructionType::AluUnary. +#[test] +fn test_neg64() { + disasm!("neg64 r1"); +} + +// Example for InstructionType::LoadReg. +#[test] +fn test_ldxw() { + disasm!("ldxw r1, [r2+0x5]"); +} + +// Example for InstructionType::StoreImm. +#[test] +fn test_stw() { + disasm!("stw [r2+0x5], 0x7"); +} + +// Example for InstructionType::StoreReg. +#[test] +fn test_stxw() { + disasm!("stxw [r2+0x5], r8"); +} + +// Example for InstructionType::JumpUnconditional. +#[test] +fn test_ja() { + disasm!("ja +0x8"); +} + +// Example for InstructionType::JumpConditional. +#[test] +fn test_jeq() { + disasm!("jeq r1, 0x4, +0x8"); + disasm!("jeq r1, r3, +0x8"); +} + +// Example for InstructionType::Call. +#[test] +fn test_call() { + disasm!("call 0x3"); +} + +// Example for InstructionType::Endian. +#[test] +fn test_be32() { + disasm!("be32 r1"); +} + +// Example for InstructionType::LoadImm. +#[test] +fn test_lddw() { + disasm!("lddw r1, 0x1234abcd5678eeff"); + disasm!("lddw r1, 0xff11ee22dd33cc44"); +} + +// Example for InstructionType::LoadAbs. +#[test] +fn test_ldabsw() { + disasm!("ldabsw 0x1"); +} + +// Example for InstructionType::LoadInd. +#[test] +fn test_ldindw() { + disasm!("ldindw r1, 0x2"); +} + +// Example for InstructionType::LoadReg. +#[test] +fn test_ldxdw() { + disasm!("ldxdw r1, [r2+0x3]"); +} + +// Example for InstructionType::StoreImm. +#[test] +fn test_sth() { + disasm!("sth [r1+0x2], 0x3"); +} + +// Example for InstructionType::StoreReg. +#[test] +fn test_stxh() { + disasm!("stxh [r1+0x2], r3"); +} + +// Test all supported AluBinary mnemonics. +#[test] +fn test_alu_binary() { + disasm!( + "add64 r1, r2 +sub64 r1, r2 +mul64 r1, r2 +div64 r1, r2 +or64 r1, r2 +and64 r1, r2 +lsh64 r1, r2 +rsh64 r1, r2 +mod64 r1, r2 +xor64 r1, r2 +mov64 r1, r2 +arsh64 r1, r2" + ); + + disasm!( + "add64 r1, 0x2 +sub64 r1, 0x2 +mul64 r1, 0x2 +div64 r1, 0x2 +or64 r1, 0x2 +and64 r1, 0x2 +lsh64 r1, 0x2 +rsh64 r1, 0x2 +mod64 r1, 0x2 +xor64 r1, 0x2 +mov64 r1, 0x2 +arsh64 r1, 0x2" + ); + + disasm!( + "add32 r1, r2 +sub32 r1, r2 +mul32 r1, r2 +div32 r1, r2 +or32 r1, r2 +and32 r1, r2 +lsh32 r1, r2 +rsh32 r1, r2 +mod32 r1, r2 +xor32 r1, r2 +mov32 r1, r2 +arsh32 r1, r2" + ); + + disasm!( + "add32 r1, 0x2 +sub32 r1, 0x2 +mul32 r1, 0x2 +div32 r1, 0x2 +or32 r1, 0x2 +and32 r1, 0x2 +lsh32 r1, 0x2 +rsh32 r1, 0x2 +mod32 r1, 0x2 +xor32 r1, 0x2 +mov32 r1, 0x2 +arsh32 r1, 0x2" + ); +} + +// Test all supported AluUnary mnemonics. +#[test] +fn test_alu_unary() { + disasm!( + "neg64 r1 +neg32 r1" + ); +} + +// Test all supported LoadAbs mnemonics. +#[test] +fn test_load_abs() { + disasm!( + "ldabsw 0x1 +ldabsh 0x1 +ldabsb 0x1 +ldabsdw 0x1" + ); +} + +// Test all supported LoadInd mnemonics. +#[test] +fn test_load_ind() { + disasm!( + "ldindw r1, 0x2 +ldindh r1, 0x2 +ldindb r1, 0x2 +ldinddw r1, 0x2" + ); +} + +// Test all supported LoadReg mnemonics. +#[test] +fn test_load_reg() { + disasm!( + r"ldxw r1, [r2+0x3] +ldxh r1, [r2+0x3] +ldxb r1, [r2+0x3] +ldxdw r1, [r2+0x3]" + ); +} + +// Test all supported StoreImm mnemonics. +#[test] +fn test_store_imm() { + disasm!( + "stw [r1+0x2], 0x3 +sth [r1+0x2], 0x3 +stb [r1+0x2], 0x3 +stdw [r1+0x2], 0x3" + ); +} + +// Test all supported StoreReg mnemonics. +#[test] +fn test_store_reg() { + disasm!( + "stxw [r1+0x2], r3 +stxh [r1+0x2], r3 +stxb [r1+0x2], r3 +stxdw [r1+0x2], r3" + ); +} + +// Test all supported JumpConditional mnemonics. +#[test] +fn test_jump_conditional() { + disasm!( + "jeq r1, r2, +0x3 +jgt r1, r2, +0x3 +jge r1, r2, +0x3 +jlt r1, r2, +0x3 +jle r1, r2, +0x3 +jset r1, r2, +0x3 +jne r1, r2, +0x3 +jsgt r1, r2, +0x3 +jsge r1, r2, -0x3 +jslt r1, r2, +0x3 +jsle r1, r2, -0x3" + ); + + disasm!( + "jeq r1, 0x2, +0x3 +jgt r1, 0x2, +0x3 +jge r1, 0x2, +0x3 +jlt r1, 0x2, +0x3 +jle r1, 0x2, +0x3 +jset r1, 0x2, +0x3 +jne r1, 0x2, +0x3 +jsgt r1, 0x2, +0x3 +jsge r1, 0x2, -0x3 +jslt r1, 0x2, +0x3 +jsle r1, 0x2, -0x3" + ); + + disasm!( + "jeq32 r1, r2, +0x3 +jgt32 r1, r2, +0x3 +jge32 r1, r2, +0x3 +jlt32 r1, r2, +0x3 +jle32 r1, r2, +0x3 +jset32 r1, r2, +0x3 +jne32 r1, r2, +0x3 +jsgt32 r1, r2, +0x3 +jsge32 r1, r2, -0x3 +jslt32 r1, r2, +0x3 +jsle32 r1, r2, -0x3" + ); + + disasm!( + "jeq32 r1, 0x2, +0x3 +jgt32 r1, 0x2, +0x3 +jge32 r1, 0x2, +0x3 +jlt32 r1, 0x2, +0x3 +jle32 r1, 0x2, +0x3 +jset32 r1, 0x2, +0x3 +jne32 r1, 0x2, +0x3 +jsgt32 r1, 0x2, +0x3 +jsge32 r1, 0x2, -0x3 +jslt32 r1, 0x2, +0x3 +jsle32 r1, 0x2, -0x3" + ); +} + +// Test all supported Endian mnemonics. +#[test] +fn test_endian() { + disasm!( + "be16 r1 +be32 r1 +be64 r1 +le16 r1 +le32 r1 +le64 r1" + ); +} + +#[test] +fn test_large_immediate() { + disasm!("add64 r1, 0x7fffffff"); + disasm!("add64 r1, 0x7fffffff"); +} + +// Non-regression tests for overflow when trying to negate offset 0x8000i16. +#[test] +fn test_offset_overflow() { + let insns = [ + 0x62, 0x01, 0x00, 0x80, 0x01, 0x00, 0x00, 0x00, // stw + 0x6a, 0x01, 0x00, 0x80, 0x01, 0x00, 0x00, 0x00, // sth + 0x72, 0x01, 0x00, 0x80, 0x01, 0x00, 0x00, 0x00, // stb + 0x7a, 0x01, 0x00, 0x80, 0x01, 0x00, 0x00, 0x00, // stdw + 0x61, 0x01, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, // ldxw + 0x69, 0x01, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, // ldxh + 0x71, 0x01, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, // ldxb + 0x79, 0x01, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, // ldxdw + 0x15, 0x01, 0x00, 0x80, 0x02, 0x00, 0x00, 0x00, // jeq (imm) + 0x1d, 0x21, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, // jeq (reg) + 0x16, 0x01, 0x00, 0x80, 0x02, 0x00, 0x00, 0x00, // jeq32 (imm) + 0x1e, 0x21, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, // jeq32 (reg) + ]; + + let expected_output = "stw [r1-0x8000], 0x1 +sth [r1-0x8000], 0x1 +stb [r1-0x8000], 0x1 +stdw [r1-0x8000], 0x1 +ldxw r1, [r0-0x8000] +ldxh r1, [r0-0x8000] +ldxb r1, [r0-0x8000] +ldxdw r1, [r0-0x8000] +jeq r1, 0x2, -0x8000 +jeq r1, r2, -0x8000 +jeq32 r1, 0x2, -0x8000 +jeq32 r1, r2, -0x8000"; + + let prog = to_insn_vec(&insns); + let asm = prog + .into_iter() + .map(|ins| ins.desc) + .collect::>() + .join("\n"); + + assert_eq!(asm, expected_output); +} diff --git a/kernel/crates/rbpf/tests/misc.rs b/kernel/crates/rbpf/tests/misc.rs new file mode 100644 index 00000000..10bb053d --- /dev/null +++ b/kernel/crates/rbpf/tests/misc.rs @@ -0,0 +1,571 @@ +// SPDX-License-Identifier: (Apache-2.0 OR MIT) +// Copyright 2016 6WIND S.A. + +// There are unused mut warnings due to unsafe code. +#![allow(unused_mut)] +#![allow(clippy::unreadable_literal)] + +// This crate would be needed to load bytecode from a BPF-compiled object file. Since the crate +// is not used anywhere else in the library, it is deactivated: we do not want to load and compile +// it just for the tests. If you want to use it, do not forget to add the following +// dependency to your Cargo.toml file: +// +// --- +// elf = "0.0.10" +// --- +// +// extern crate elf; +// use std::path::PathBuf; + +extern crate rbpf; + +#[cfg(feature = "std")] +use rbpf::helpers; +use rbpf::{assembler::assemble, Error, ErrorKind}; + +// The following two examples have been compiled from C with the following command: +// +// ```bash +// clang -O2 -emit-llvm -c -o - | llc -march=bpf -filetype=obj -o +// ``` +// +// The C source code was the following: +// +// ```c +// #include +// #include +// #include +// #include +// +// #define ETH_ALEN 6 +// #define ETH_P_IP 0x0008 /* htons(0x0800) */ +// #define TCP_HDR_LEN 20 +// +// #define BLOCKED_TCP_PORT 0x9999 +// +// struct eth_hdr { +// unsigned char h_dest[ETH_ALEN]; +// unsigned char h_source[ETH_ALEN]; +// unsigned short h_proto; +// }; +// +// #define SEC(NAME) __attribute__((section(NAME), used)) +// SEC(".classifier") +// int handle_ingress(struct __sk_buff *skb) +// { +// void *data = (void *)(long)skb->data; +// void *data_end = (void *)(long)skb->data_end; +// struct eth_hdr *eth = data; +// struct iphdr *iph = data + sizeof(*eth); +// struct tcphdr *tcp = data + sizeof(*eth) + sizeof(*iph); +// +// /* single length check */ +// if (data + sizeof(*eth) + sizeof(*iph) + sizeof(*tcp) > data_end) +// return 0; +// if (eth->h_proto != ETH_P_IP) +// return 0; +// if (iph->protocol != IPPROTO_TCP) +// return 0; +// if (tcp->source == BLOCKED_TCP_PORT || tcp->dest == BLOCKED_TCP_PORT) +// return -1; +// return 0; +// } +// char _license[] SEC(".license") = "GPL"; +// ``` +// +// This program, once compiled, can be injected into Linux kernel, with tc for instance. Sadly, we +// need to bring some modifications to the generated bytecode in order to run it: the three +// instructions with opcode 0x61 load data from a packet area as 4-byte words, where we need to +// load it as 8-bytes double words (0x79). The kernel does the same kind of translation before +// running the program, but rbpf does not implement this. +// +// In addition, the offset at which the pointer to the packet data is stored must be changed: since +// we use 8 bytes instead of 4 for the start and end addresses of the data packet, we cannot use +// the offsets produced by clang (0x4c and 0x50), the addresses would overlap. Instead we can use, +// for example, 0x40 and 0x50. See comments on the bytecode below to see the modifications. +// +// Once the bytecode has been (manually, in our case) edited, we can load the bytecode directly +// from the ELF object file. This is easy to do, but requires the addition of two crates in the +// Cargo.toml file (see comments above), so here we use just the hardcoded bytecode instructions +// instead. + +#[test] +#[cfg(feature = "std")] +fn test_vm_block_port() { + // To load the bytecode from an object file instead of using the hardcoded instructions, + // use the additional crates commented at the beginning of this file (and also add them to your + // Cargo.toml). See comments above. + // + // --- + // let filename = "my_ebpf_object_file.o"; + // + // let path = PathBuf::from(filename); + // let file = match elf::File::open_path(&path) { + // Ok(f) => f, + // Err(e) => panic!("Error: {:?}", e), + // }; + // + // let text_scn = match file.get_section(".classifier") { + // Some(s) => s, + // None => panic!("Failed to look up .classifier section"), + // }; + // + // let prog = &text_scn.data; + // --- + + let prog = &[ + 0xb7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x79, 0x12, 0x50, 0x00, 0x00, 0x00, 0x00, + 0x00, // 0x79 instead of 0x61 + 0x79, 0x11, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, // 0x79 instead of 0x61, 0x40 i.o. 0x4c + 0xbf, 0x13, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0x03, 0x00, 0x00, 0x36, 0x00, 0x00, + 0x00, 0x2d, 0x23, 0x12, 0x00, 0x00, 0x00, 0x00, 0x00, 0x69, 0x12, 0x0c, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x55, 0x02, 0x10, 0x00, 0x08, 0x00, 0x00, 0x00, 0x71, 0x12, 0x17, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x55, 0x02, 0x0e, 0x00, 0x06, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x79, 0x11, 0x22, + 0x00, 0x00, 0x00, 0x00, 0x00, // 0x79 instead of 0x61 + 0xbf, 0x12, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x57, 0x02, 0x00, 0x00, 0xff, 0xff, 0x00, + 0x00, 0x15, 0x02, 0x08, 0x00, 0x99, 0x99, 0x00, 0x00, 0x18, 0x02, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5f, 0x21, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xb7, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x18, 0x02, 0x00, 0x00, + 0x00, 0x00, 0x99, 0x99, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1d, 0x21, 0x01, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xb7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x95, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + ]; + + let packet = &mut [ + 0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xfe, 0xdc, 0xba, 0x98, 0x76, 0x54, 0x08, + 0x00, // ethertype + 0x45, 0x00, 0x00, 0x3b, // start ip_hdr + 0xa6, 0xab, 0x40, 0x00, 0x40, 0x06, 0x96, 0x0f, 0x7f, 0x00, 0x00, 0x01, 0x7f, 0x00, 0x00, + 0x01, + // Program matches the next two bytes: 0x9999 returns 0xffffffff, else return 0. + 0x99, 0x99, 0xc6, 0xcc, // start tcp_hdr + 0xd1, 0xe5, 0xc4, 0x9d, 0xd4, 0x30, 0xb5, 0xd2, 0x80, 0x18, 0x01, 0x56, 0xfe, 0x2f, 0x00, + 0x00, 0x01, 0x01, 0x08, 0x0a, // start data + 0x00, 0x23, 0x75, 0x89, 0x00, 0x23, 0x63, 0x2d, 0x71, 0x64, 0x66, 0x73, 0x64, 0x66, 0x0au8, + ]; + + let mut vm = rbpf::EbpfVmFixedMbuff::new(Some(prog), 0x40, 0x50).unwrap(); + vm.register_helper(helpers::BPF_TRACE_PRINTK_IDX, helpers::bpf_trace_printf) + .unwrap(); + + let res = vm.execute_program(packet).unwrap(); + println!("Program returned: {res:?} ({res:#x})"); + assert_eq!(res, 0xffffffff); +} + +#[test] +#[cfg(all(not(windows), feature = "std"))] +fn test_jit_block_port() { + // To load the bytecode from an object file instead of using the hardcoded instructions, + // use the additional crates commented at the beginning of this file (and also add them to your + // Cargo.toml). See comments above. + // + // --- + // let filename = "my_ebpf_object_file.o"; + // + // let path = PathBuf::from(filename); + // let file = match elf::File::open_path(&path) { + // Ok(f) => f, + // Err(e) => panic!("Error: {:?}", e), + // }; + // + // let text_scn = match file.get_section(".classifier") { + // Some(s) => s, + // None => panic!("Failed to look up .classifier section"), + // }; + // + // let prog = &text_scn.data; + // --- + + let prog = &[ + 0xb7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x79, 0x12, 0x50, 0x00, 0x00, 0x00, 0x00, + 0x00, // 0x79 instead of 0x61 + 0x79, 0x11, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, // 0x79 instead of 0x61, 0x40 i.o. 0x4c + 0xbf, 0x13, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0x03, 0x00, 0x00, 0x36, 0x00, 0x00, + 0x00, 0x2d, 0x23, 0x12, 0x00, 0x00, 0x00, 0x00, 0x00, 0x69, 0x12, 0x0c, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x55, 0x02, 0x10, 0x00, 0x08, 0x00, 0x00, 0x00, 0x71, 0x12, 0x17, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x55, 0x02, 0x0e, 0x00, 0x06, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x79, 0x11, 0x22, + 0x00, 0x00, 0x00, 0x00, 0x00, // 0x79 instead of 0x61 + 0xbf, 0x12, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x57, 0x02, 0x00, 0x00, 0xff, 0xff, 0x00, + 0x00, 0x15, 0x02, 0x08, 0x00, 0x99, 0x99, 0x00, 0x00, 0x18, 0x02, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5f, 0x21, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xb7, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x18, 0x02, 0x00, 0x00, + 0x00, 0x00, 0x99, 0x99, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1d, 0x21, 0x01, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xb7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x95, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + ]; + + let packet = &mut [ + 0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xfe, 0xdc, 0xba, 0x98, 0x76, 0x54, 0x08, + 0x00, // ethertype + 0x45, 0x00, 0x00, 0x3b, // start ip_hdr + 0xa6, 0xab, 0x40, 0x00, 0x40, 0x06, 0x96, 0x0f, 0x7f, 0x00, 0x00, 0x01, 0x7f, 0x00, 0x00, + 0x01, + // Program matches the next two bytes: 0x9999 returns 0xffffffff, else return 0. + 0x99, 0x99, 0xc6, 0xcc, // start tcp_hdr + 0xd1, 0xe5, 0xc4, 0x9d, 0xd4, 0x30, 0xb5, 0xd2, 0x80, 0x18, 0x01, 0x56, 0xfe, 0x2f, 0x00, + 0x00, 0x01, 0x01, 0x08, 0x0a, // start data + 0x00, 0x23, 0x75, 0x89, 0x00, 0x23, 0x63, 0x2d, 0x71, 0x64, 0x66, 0x73, 0x64, 0x66, 0x0au8, + ]; + + let mut vm = rbpf::EbpfVmFixedMbuff::new(Some(prog), 0x40, 0x50).unwrap(); + vm.register_helper(helpers::BPF_TRACE_PRINTK_IDX, helpers::bpf_trace_printf) + .unwrap(); + vm.jit_compile().unwrap(); + + unsafe { + let res = vm.execute_program_jit(packet).unwrap(); + println!("Program returned: {res:?} ({res:#x})"); + assert_eq!(res, 0xffffffff); + } +} + +// Program and memory come from uBPF test ldxh. +#[test] +fn test_vm_mbuff() { + let prog = &[ + // Load mem from mbuff into R1 + 0x79, 0x11, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, // ldhx r1[2], r0 + 0x69, 0x10, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, + ]; + let mem = &[0xaa, 0xbb, 0x11, 0x22, 0xcc, 0xdd]; + + let mbuff = [0u8; 32]; + unsafe { + let mut data = mbuff.as_ptr().offset(8) as *mut u64; + let mut data_end = mbuff.as_ptr().offset(24) as *mut u64; + data.write_unaligned(mem.as_ptr() as u64); + data_end.write_unaligned(mem.as_ptr() as u64 + mem.len() as u64); + } + + let vm = rbpf::EbpfVmMbuff::new(Some(prog)).unwrap(); + assert_eq!(vm.execute_program(mem, &mbuff).unwrap(), 0x2211); +} + +// Program and memory come from uBPF test ldxh. +#[test] +fn test_vm_mbuff_with_rust_api() { + use rbpf::insn_builder::*; + + let mut program = BpfCode::new(); + program + .load_x(MemSize::DoubleWord) + .set_dst(0x01) + .set_src(0x01) + .set_off(0x00_08) + .push() + .load_x(MemSize::HalfWord) + .set_dst(0x00) + .set_src(0x01) + .set_off(0x00_02) + .push() + .exit() + .push(); + + let mem = &[0xaa, 0xbb, 0x11, 0x22, 0xcc, 0xdd]; + + let mbuff = [0u8; 32]; + unsafe { + let mut data = mbuff.as_ptr().offset(8) as *mut u64; + let mut data_end = mbuff.as_ptr().offset(24) as *mut u64; + *data = mem.as_ptr() as u64; + *data_end = mem.as_ptr() as u64 + mem.len() as u64; + } + + let vm = rbpf::EbpfVmMbuff::new(Some(program.into_bytes())).unwrap(); + assert_eq!(vm.execute_program(mem, &mbuff).unwrap(), 0x2211); +} + +// Program and memory come from uBPF test ldxh. +#[test] +#[cfg(all(not(windows), feature = "std"))] +fn test_jit_mbuff() { + let prog = &[ + // Load mem from mbuff into R1 + 0x79, 0x11, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, // ldhx r1[2], r0 + 0x69, 0x10, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, + ]; + let mem = &mut [0xaa, 0xbb, 0x11, 0x22, 0xcc, 0xdd]; + + let mut mbuff = [0u8; 32]; + unsafe { + let mut data = mbuff.as_ptr().offset(8) as *mut u64; + let mut data_end = mbuff.as_ptr().offset(24) as *mut u64; + *data = mem.as_ptr() as u64; + *data_end = mem.as_ptr() as u64 + mem.len() as u64; + } + + unsafe { + let mut vm = rbpf::EbpfVmMbuff::new(Some(prog)).unwrap(); + vm.jit_compile().unwrap(); + assert_eq!(vm.execute_program_jit(mem, &mut mbuff).unwrap(), 0x2211); + } +} + +#[cfg(all(not(windows), feature = "std"))] +#[test] +fn test_vm_jit_ldabsb() { + let prog = &[ + 0x30, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, + ]; + let mem = &mut [ + 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, + 0xff, + ]; + let mut vm = rbpf::EbpfVmRaw::new(Some(prog)).unwrap(); + assert_eq!(vm.execute_program(mem).unwrap(), 0x33); + + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit(mem).unwrap(), 0x33); + }; +} + +#[cfg(all(not(windows), feature = "std"))] +#[test] +fn test_vm_jit_ldabsh() { + let prog = &[ + 0x28, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, + ]; + let mem = &mut [ + 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, + 0xff, + ]; + let mut vm = rbpf::EbpfVmRaw::new(Some(prog)).unwrap(); + assert_eq!(vm.execute_program(mem).unwrap(), 0x4433); + + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit(mem).unwrap(), 0x4433); + }; +} + +#[cfg(all(not(windows), feature = "std"))] +#[test] +fn test_vm_jit_ldabsw() { + let prog = &[ + 0x20, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, + ]; + let mem = &mut [ + 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, + 0xff, + ]; + let mut vm = rbpf::EbpfVmRaw::new(Some(prog)).unwrap(); + assert_eq!(vm.execute_program(mem).unwrap(), 0x66554433); + vm.jit_compile().unwrap(); + + unsafe { + assert_eq!(vm.execute_program_jit(mem).unwrap(), 0x66554433); + }; +} + +#[cfg(all(not(windows), feature = "std"))] +#[test] +fn test_vm_jit_ldabsdw() { + let prog = &[ + 0x38, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, + ]; + let mem = &mut [ + 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, + 0xff, + ]; + let mut vm = rbpf::EbpfVmRaw::new(Some(prog)).unwrap(); + assert_eq!(vm.execute_program(mem).unwrap(), 0xaa99887766554433); + vm.jit_compile().unwrap(); + + unsafe { + assert_eq!(vm.execute_program_jit(mem).unwrap(), 0xaa99887766554433); + }; +} + +#[test] +#[should_panic(expected = "Error: out of bounds memory load (insn #1),")] +fn test_vm_err_ldabsb_oob() { + let prog = &[ + 0x38, 0x00, 0x00, 0x00, 0x33, 0x00, 0x00, 0x00, 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, + ]; + let mem = &mut [ + 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, + 0xff, + ]; + let vm = rbpf::EbpfVmRaw::new(Some(prog)).unwrap(); + vm.execute_program(mem).unwrap(); + + // Memory check not implemented for JIT yet. +} + +#[test] +#[should_panic(expected = "Error: out of bounds memory load (insn #1),")] +fn test_vm_err_ldabsb_nomem() { + let prog = &[ + 0x38, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, + ]; + let vm = rbpf::EbpfVmNoData::new(Some(prog)).unwrap(); + vm.execute_program().unwrap(); + + // Memory check not implemented for JIT yet. +} + +#[cfg(all(not(windows), feature = "std"))] +#[test] +fn test_vm_jit_ldindb() { + let prog = &[ + 0xb7, 0x01, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x50, 0x10, 0x00, 0x00, 0x03, 0x00, 0x00, + 0x00, 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + ]; + let mem = &mut [ + 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, + 0xff, + ]; + let mut vm = rbpf::EbpfVmRaw::new(Some(prog)).unwrap(); + assert_eq!(vm.execute_program(mem).unwrap(), 0x88); + + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit(mem).unwrap(), 0x88); + }; +} + +#[cfg(all(not(windows), feature = "std"))] +#[test] +fn test_vm_jit_ldindh() { + let prog = &[ + 0xb7, 0x01, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x48, 0x10, 0x00, 0x00, 0x03, 0x00, 0x00, + 0x00, 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + ]; + let mem = &mut [ + 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, + 0xff, + ]; + let mut vm = rbpf::EbpfVmRaw::new(Some(prog)).unwrap(); + assert_eq!(vm.execute_program(mem).unwrap(), 0x9988); + + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit(mem).unwrap(), 0x9988); + }; +} + +#[cfg(all(not(windows), feature = "std"))] +#[test] +fn test_vm_jit_ldindw() { + let prog = &[ + 0xb7, 0x01, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x40, 0x10, 0x00, 0x00, 0x01, 0x00, 0x00, + 0x00, 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + ]; + let mem = &mut [ + 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, + 0xff, + ]; + let mut vm = rbpf::EbpfVmRaw::new(Some(prog)).unwrap(); + assert_eq!(vm.execute_program(mem).unwrap(), 0x88776655); + vm.jit_compile().unwrap(); + + unsafe { + assert_eq!(vm.execute_program_jit(mem).unwrap(), 0x88776655); + }; +} + +#[cfg(all(not(windows), feature = "std"))] +#[test] +fn test_vm_jit_ldinddw() { + let prog = &[ + 0xb7, 0x01, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x58, 0x10, 0x00, 0x00, 0x03, 0x00, 0x00, + 0x00, 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + ]; + let mem = &mut [ + 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, + 0xff, + ]; + let mut vm = rbpf::EbpfVmRaw::new(Some(prog)).unwrap(); + assert_eq!(vm.execute_program(mem).unwrap(), 0xccbbaa9988776655); + vm.jit_compile().unwrap(); + + unsafe { + assert_eq!(vm.execute_program_jit(mem).unwrap(), 0xccbbaa9988776655); + }; +} + +#[test] +#[should_panic(expected = "Error: out of bounds memory load (insn #2),")] +fn test_vm_err_ldindb_oob() { + let prog = &[ + 0xb7, 0x01, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x38, 0x10, 0x00, 0x00, 0x33, 0x00, 0x00, + 0x00, 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + ]; + let mem = &mut [ + 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, + 0xff, + ]; + let vm = rbpf::EbpfVmRaw::new(Some(prog)).unwrap(); + vm.execute_program(mem).unwrap(); + + // Memory check not implemented for JIT yet. +} + +#[test] +#[should_panic(expected = "Error: out of bounds memory load (insn #2),")] +fn test_vm_err_ldindb_nomem() { + let prog = &[ + 0xb7, 0x01, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x38, 0x10, 0x00, 0x00, 0x03, 0x00, 0x00, + 0x00, 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + ]; + let vm = rbpf::EbpfVmNoData::new(Some(prog)).unwrap(); + vm.execute_program().unwrap(); + + // Memory check not implemented for JIT yet. +} + +#[test] +#[should_panic(expected = "Error: No program set, call prog_set() to load one")] +fn test_vm_exec_no_program() { + let vm = rbpf::EbpfVmNoData::new(None).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0xBEE); +} + +fn verifier_success(_prog: &[u8]) -> Result<(), Error> { + Ok(()) +} + +fn verifier_fail(_prog: &[u8]) -> Result<(), Error> { + Err(Error::new(ErrorKind::Other, "Gaggablaghblagh!")) +} + +#[test] +fn test_verifier_success() { + let prog = assemble( + "mov32 r0, 0xBEE + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(None).unwrap(); + vm.set_verifier(verifier_success).unwrap(); + vm.set_program(&prog).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0xBEE); +} + +#[test] +#[should_panic(expected = "Gaggablaghblagh!")] +fn test_verifier_fail() { + let prog = assemble( + "mov32 r0, 0xBEE + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(None).unwrap(); + vm.set_verifier(verifier_fail).unwrap(); + vm.set_program(&prog).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0xBEE); +} diff --git a/kernel/crates/rbpf/tests/ubpf_jit_x86_64.rs b/kernel/crates/rbpf/tests/ubpf_jit_x86_64.rs new file mode 100644 index 00000000..2740a0ae --- /dev/null +++ b/kernel/crates/rbpf/tests/ubpf_jit_x86_64.rs @@ -0,0 +1,2891 @@ +// SPDX-License-Identifier: (Apache-2.0 OR MIT) +// Converted from the tests for uBPF +// Copyright 2015 Big Switch Networks, Inc +// Copyright 2016 6WIND S.A. + +// The tests contained in this file are extracted from the unit tests of uBPF software. Each test +// in this file has a name in the form `test_jit_`, and corresponds to the (human-readable) +// code in `ubpf/tree/master/tests/`, available at +// (hyphen had to be replaced with underscores +// as Rust will not accept them in function names). It is strongly advised to refer to the uBPF +// version to understand what these program do. +// +// Each program was assembled from the uBPF version with the assembler provided by uBPF itself, and +// available at . +// The very few modifications that have been realized should be indicated. + +// These are unit tests for the eBPF JIT compiler. + +#![allow(clippy::unreadable_literal)] +#![cfg(all(not(windows), feature = "std"))] + +extern crate rbpf; +mod common; + +use common::{TCP_SACK_ASM, TCP_SACK_MATCH, TCP_SACK_NOMATCH}; +use rbpf::{assembler::assemble, helpers}; + +#[test] +fn test_jit_add() { + let prog = assemble( + " + mov32 r0, 0 + mov32 r1, 2 + add32 r0, 1 + add32 r0, r1 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x3); + } +} + +#[test] +fn test_jit_alu64_arith() { + let prog = assemble( + " + mov r0, 0 + mov r1, 1 + mov r2, 2 + mov r3, 3 + mov r4, 4 + mov r5, 5 + mov r6, 6 + mov r7, 7 + mov r8, 8 + mov r9, 9 + add r0, 23 + add r0, r7 + sub r0, 13 + sub r0, r1 + mul r0, 7 + mul r0, r3 + div r0, 2 + div r0, r4 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x2a); + } +} + +#[test] +fn test_jit_alu64_bit() { + let prog = assemble( + " + mov r0, 0 + mov r1, 1 + mov r2, 2 + mov r3, 3 + mov r4, 4 + mov r5, 5 + mov r6, 6 + mov r7, 7 + mov r8, 8 + or r0, r5 + or r0, 0xa0 + and r0, 0xa3 + mov r9, 0x91 + and r0, r9 + lsh r0, 32 + lsh r0, 22 + lsh r0, r8 + rsh r0, 32 + rsh r0, 19 + rsh r0, r7 + xor r0, 0x03 + xor r0, r2 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x11); + } +} + +#[test] +fn test_jit_alu_arith() { + let prog = assemble( + " + mov32 r0, 0 + mov32 r1, 1 + mov32 r2, 2 + mov32 r3, 3 + mov32 r4, 4 + mov32 r5, 5 + mov32 r6, 6 + mov32 r7, 7 + mov32 r8, 8 + mov32 r9, 9 + add32 r0, 23 + add32 r0, r7 + sub32 r0, 13 + sub32 r0, r1 + mul32 r0, 7 + mul32 r0, r3 + div32 r0, 2 + div32 r0, r4 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x2a); + } +} + +#[test] +fn test_jit_alu_bit() { + let prog = assemble( + " + mov32 r0, 0 + mov32 r1, 1 + mov32 r2, 2 + mov32 r3, 3 + mov32 r4, 4 + mov32 r5, 5 + mov32 r6, 6 + mov32 r7, 7 + mov32 r8, 8 + or32 r0, r5 + or32 r0, 0xa0 + and32 r0, 0xa3 + mov32 r9, 0x91 + and32 r0, r9 + lsh32 r0, 22 + lsh32 r0, r8 + rsh32 r0, 19 + rsh32 r0, r7 + xor32 r0, 0x03 + xor32 r0, r2 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x11); + } +} + +#[test] +fn test_jit_arsh32_high_shift() { + let prog = assemble( + " + mov r0, 8 + lddw r1, 0x100000001 + arsh32 r0, r1 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x4); + } +} + +#[test] +fn test_jit_arsh() { + let prog = assemble( + " + mov32 r0, 0xf8 + lsh32 r0, 28 + arsh32 r0, 16 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0xffff8000); + } +} + +#[test] +fn test_jit_arsh64() { + let prog = assemble( + " + mov32 r0, 1 + lsh r0, 63 + arsh r0, 55 + mov32 r1, 5 + arsh r0, r1 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0xfffffffffffffff8); + } +} + +#[test] +fn test_jit_arsh_reg() { + let prog = assemble( + " + mov32 r0, 0xf8 + mov32 r1, 16 + lsh32 r0, 28 + arsh32 r0, r1 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0xffff8000); + } +} + +#[test] +fn test_jit_be16() { + let prog = assemble( + " + ldxh r0, [r1] + be16 r0 + exit", + ) + .unwrap(); + let mem = &mut [0x11, 0x22]; + let mut vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit(mem).unwrap(), 0x1122); + } +} + +#[test] +fn test_jit_be16_high() { + let prog = assemble( + " + ldxdw r0, [r1] + be16 r0 + exit", + ) + .unwrap(); + let mem = &mut [0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88]; + let mut vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit(mem).unwrap(), 0x1122); + } +} + +#[test] +fn test_jit_be32() { + let prog = assemble( + " + ldxw r0, [r1] + be32 r0 + exit", + ) + .unwrap(); + let mem = &mut [0x11, 0x22, 0x33, 0x44]; + let mut vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit(mem).unwrap(), 0x11223344); + } +} + +#[test] +fn test_jit_be32_high() { + let prog = assemble( + " + ldxdw r0, [r1] + be32 r0 + exit", + ) + .unwrap(); + let mem = &mut [0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88]; + let mut vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit(mem).unwrap(), 0x11223344); + } +} + +#[test] +fn test_jit_be64() { + let prog = assemble( + " + ldxdw r0, [r1] + be64 r0 + exit", + ) + .unwrap(); + let mem = &mut [0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88]; + let mut vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit(mem).unwrap(), 0x1122334455667788); + } +} + +#[test] +fn test_jit_call() { + let prog = assemble( + " + mov r1, 1 + mov r2, 2 + mov r3, 3 + mov r4, 4 + mov r5, 5 + call 0 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.register_helper(0, helpers::gather_bytes).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x0102030405); + } +} + +#[test] +fn test_jit_call_memfrob() { + let prog = assemble( + " + mov r6, r1 + add r1, 2 + mov r2, 4 + call 1 + ldxdw r0, [r6] + be64 r0 + exit", + ) + .unwrap(); + let mem = &mut [0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08]; + let mut vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + vm.register_helper(1, helpers::memfrob).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit(mem).unwrap(), 0x102292e2f2c0708); + } +} + +// TODO: helpers::trash_registers needs asm!(). +// Try this again once asm!() is available in stable. +//#[test] +//fn test_jit_call_save() { +//let prog = &[ +//0xb7, 0x06, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, +//0xb7, 0x07, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, +//0xb7, 0x08, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, +//0xb7, 0x09, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, +//0x85, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, +//0xb7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +//0x4f, 0x60, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +//0x4f, 0x70, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +//0x4f, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +//0x4f, 0x90, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +//0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +//]; +//let mut vm = rbpf::EbpfVmNoData::new(Some(prog)).unwrap(); +//vm.register_helper(2, helpers::trash_registers); +//vm.jit_compile().unwrap(); +//unsafe { assert_eq!(vm.execute_program_jit().unwrap(), 0x4321); } +//} + +#[test] +fn test_jit_div32_high_divisor() { + let prog = assemble( + " + mov r0, 12 + lddw r1, 0x100000004 + div32 r0, r1 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x3); + } +} + +#[test] +fn test_jit_div32_imm() { + let prog = assemble( + " + lddw r0, 0x10000000c + div32 r0, 4 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x3); + } +} + +#[test] +fn test_jit_div32_reg() { + let prog = assemble( + " + lddw r0, 0x10000000c + mov r1, 4 + div32 r0, r1 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x3); + } +} + +#[test] +fn test_jit_div64_imm() { + let prog = assemble( + " + mov r0, 0xc + lsh r0, 32 + div r0, 4 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x300000000); + } +} + +#[test] +fn test_jit_div64_reg() { + let prog = assemble( + " + mov r0, 0xc + lsh r0, 32 + mov r1, 4 + div r0, r1 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x300000000); + } +} + +// For some register numbers, we don't emit the same instructions for handling divisions by zero, +// which means we don't use the same offset to skip these instructions when the divisor is not +// zero. We've had a regression because of this before, make sure we test it. +#[test] +fn test_jit_div32_highreg() { + let prog = assemble( + " + mov r0, 2 + mov r7, 4 + div32 r7, r0 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x2); + } +} + +#[test] +fn test_jit_div64_highreg() { + let prog = assemble( + " + mov r0, 2 + mov r7, 4 + div r7, r0 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x2); + } +} + +#[test] +fn test_jit_early_exit() { + let prog = assemble( + " + mov r0, 3 + exit + mov r0, 4 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x3); + } +} + +// uBPF limits the number of user functions at 64. We don't. +//#[test] +//fn test_jit_err_call_bad_imm() { +//} + +#[test] +#[should_panic(expected = "[JIT] Error: unknown helper function (id: 0x3f)")] +fn test_jit_err_call_unreg() { + let prog = assemble( + " + mov r1, 1 + mov r2, 2 + mov r3, 3 + mov r4, 4 + mov r5, 5 + call 63 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + vm.execute_program_jit().unwrap(); + } +} + +#[test] +fn test_jit_div64_by_zero_imm() { + let prog = assemble( + " + mov32 r0, 1 + div r0, 0 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x0); + } +} + +#[test] +fn test_jit_div_by_zero_imm() { + let prog = assemble( + " + mov32 r0, 1 + div32 r0, 0 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x0); + } +} + +#[test] +fn test_jit_mod64_by_zero_imm() { + let prog = assemble( + " + mov32 r0, 1 + mod r0, 0 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_mod_by_zero_imm() { + let prog = assemble( + " + mov32 r0, 1 + mod32 r0, 0 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_div64_by_zero_reg() { + let prog = assemble( + " + mov32 r0, 1 + mov32 r1, 0 + div r0, r1 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x0); + } +} + +#[test] +fn test_jit_div_by_zero_reg() { + let prog = assemble( + " + mov32 r0, 1 + mov32 r1, 0 + div32 r0, r1 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x0); + } +} + +#[test] +fn test_jit_mod64_by_zero_reg() { + let prog = assemble( + " + mov32 r0, 1 + mov32 r1, 0 + mod r0, r1 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_mod_by_zero_reg() { + let prog = assemble( + " + mov32 r0, 1 + mov32 r1, 0 + mod32 r0, r1 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +// TODO SKIP: JIT disabled for this testcase (stack oob check not implemented) +// #[test] +// #[should_panic(expected = "Error: out of bounds memory store (insn #1)")] +// fn test_jit_err_stack_out_of_bound() { +// let prog = &[ +// 0x72, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +// 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +// ]; +// let mut vm = rbpf::EbpfVmNoData::new(Some(prog)).unwrap(); +// vm.jit_compile().unwrap(); +// unsafe { vm.execute_program_jit().unwrap(); } +// } + +#[test] +fn test_jit_exit() { + let prog = assemble( + " + mov r0, 0 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x0); + } +} + +#[test] +fn test_jit_ja() { + let prog = assemble( + " + mov r0, 1 + ja +1 + mov r0, 2 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_jeq_imm() { + let prog = assemble( + " + mov32 r0, 0 + mov32 r1, 0xa + jeq r1, 0xb, +4 + mov32 r0, 1 + mov32 r1, 0xb + jeq r1, 0xb, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_jeq_reg() { + let prog = assemble( + " + mov32 r0, 0 + mov32 r1, 0xa + mov32 r2, 0xb + jeq r1, r2, +4 + mov32 r0, 1 + mov32 r1, 0xb + jeq r1, r2, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_jge_imm() { + let prog = assemble( + " + mov32 r0, 0 + mov32 r1, 0xa + jge r1, 0xb, +4 + mov32 r0, 1 + mov32 r1, 0xc + jge r1, 0xb, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_jle_imm() { + let prog = assemble( + " + mov32 r0, 0 + mov32 r1, 5 + jle r1, 4, +1 + jle r1, 6, +1 + exit + jle r1, 5, +1 + exit + mov32 r0, 1 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_jle_reg() { + let prog = assemble( + " + mov r0, 0 + mov r1, 5 + mov r2, 4 + mov r3, 6 + jle r1, r2, +2 + jle r1, r1, +1 + exit + jle r1, r3, +1 + exit + mov r0, 1 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_jgt_imm() { + let prog = assemble( + " + mov32 r0, 0 + mov32 r1, 5 + jgt r1, 6, +2 + jgt r1, 5, +1 + jgt r1, 4, +1 + exit + mov32 r0, 1 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_jgt_reg() { + let prog = assemble( + " + mov r0, 0 + mov r1, 5 + mov r2, 6 + mov r3, 4 + jgt r1, r2, +2 + jgt r1, r1, +1 + jgt r1, r3, +1 + exit + mov r0, 1 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_jlt_imm() { + let prog = assemble( + " + mov32 r0, 0 + mov32 r1, 5 + jlt r1, 4, +2 + jlt r1, 5, +1 + jlt r1, 6, +1 + exit + mov32 r0, 1 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_jlt_reg() { + let prog = assemble( + " + mov r0, 0 + mov r1, 5 + mov r2, 4 + mov r3, 6 + jlt r1, r2, +2 + jlt r1, r1, +1 + jlt r1, r3, +1 + exit + mov r0, 1 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_jit_bounce() { + let prog = assemble( + " + mov r0, 1 + mov r6, r0 + mov r7, r6 + mov r8, r7 + mov r9, r8 + mov r0, r9 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_jne_reg() { + let prog = assemble( + " + mov32 r0, 0 + mov32 r1, 0xb + mov32 r2, 0xb + jne r1, r2, +4 + mov32 r0, 1 + mov32 r1, 0xa + jne r1, r2, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_jset_imm() { + let prog = assemble( + " + mov32 r0, 0 + mov32 r1, 0x7 + jset r1, 0x8, +4 + mov32 r0, 1 + mov32 r1, 0x9 + jset r1, 0x8, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_jset_reg() { + let prog = assemble( + " + mov32 r0, 0 + mov32 r1, 0x7 + mov32 r2, 0x8 + jset r1, r2, +4 + mov32 r0, 1 + mov32 r1, 0x9 + jset r1, r2, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_jsge_imm() { + let prog = assemble( + " + mov32 r0, 0 + mov r1, -2 + jsge r1, -1, +5 + jsge r1, 0, +4 + mov32 r0, 1 + mov r1, -1 + jsge r1, -1, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_jsge_reg() { + let prog = assemble( + " + mov32 r0, 0 + mov r1, -2 + mov r2, -1 + mov32 r3, 0 + jsge r1, r2, +5 + jsge r1, r3, +4 + mov32 r0, 1 + mov r1, r2 + jsge r1, r2, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_jsle_imm() { + let prog = assemble( + " + mov32 r0, 0 + mov r1, -2 + jsle r1, -3, +1 + jsle r1, -1, +1 + exit + mov32 r0, 1 + jsle r1, -2, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_jsle_reg() { + let prog = assemble( + " + mov32 r0, 0 + mov r1, -1 + mov r2, -2 + mov32 r3, 0 + jsle r1, r2, +1 + jsle r1, r3, +1 + exit + mov32 r0, 1 + mov r1, r2 + jsle r1, r2, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_jsgt_imm() { + let prog = assemble( + " + mov32 r0, 0 + mov r1, -2 + jsgt r1, -1, +4 + mov32 r0, 1 + mov32 r1, 0 + jsgt r1, -1, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_jsgt_reg() { + let prog = assemble( + " + mov32 r0, 0 + mov r1, -2 + mov r2, -1 + jsgt r1, r2, +4 + mov32 r0, 1 + mov32 r1, 0 + jsgt r1, r2, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_jslt_imm() { + let prog = assemble( + " + mov32 r0, 0 + mov r1, -2 + jslt r1, -3, +2 + jslt r1, -2, +1 + jslt r1, -1, +1 + exit + mov32 r0, 1 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_jslt_reg() { + let prog = assemble( + " + mov32 r0, 0 + mov r1, -2 + mov r2, -3 + mov r3, -1 + jslt r1, r1, +2 + jslt r1, r2, +1 + jslt r1, r3, +1 + exit + mov32 r0, 1 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_jeq32_imm() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0x0 + mov32 r1, 0xa + jeq32 r1, 0xb, +5 + mov32 r0, 1 + mov r1, 0xb + or r1, r9 + jeq32 r1, 0xb, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_jeq32_reg() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, 0xa + mov32 r2, 0xb + jeq32 r1, r2, +5 + mov32 r0, 1 + mov32 r1, 0xb + or r1, r9 + jeq32 r1, r2, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_jge32_imm() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, 0xa + jge32 r1, 0xb, +5 + mov32 r0, 1 + or r1, r9 + mov32 r1, 0xc + jge32 r1, 0xb, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_jge32_reg() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, 0xa + mov32 r2, 0xb + jge32 r1, r2, +5 + mov32 r0, 1 + or r1, r9 + mov32 r1, 0xc + jge32 r1, r2, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_jgt32_imm() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, 5 + or r1, r9 + jgt32 r1, 6, +4 + jgt32 r1, 5, +3 + jgt32 r1, 4, +1 + exit + mov32 r0, 1 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_jgt32_reg() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov r0, 0 + mov r1, 5 + mov32 r1, 5 + or r1, r9 + mov r2, 6 + mov r3, 4 + jgt32 r1, r2, +4 + jgt32 r1, r1, +3 + jgt32 r1, r3, +1 + exit + mov r0, 1 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_jle32_imm() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, 5 + or r1, r9 + jle32 r1, 4, +5 + jle32 r1, 6, +1 + exit + jle32 r1, 5, +1 + exit + mov32 r0, 1 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_jle32_reg() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov r0, 0 + mov r1, 5 + mov r2, 4 + mov r3, 6 + or r1, r9 + jle32 r1, r2, +5 + jle32 r1, r1, +1 + exit + jle32 r1, r3, +1 + exit + mov r0, 1 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_jlt32_imm() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, 5 + or r1, r9 + jlt32 r1, 4, +4 + jlt32 r1, 5, +3 + jlt32 r1, 6, +1 + exit + mov32 r0, 1 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_jlt32_reg() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov r0, 0 + mov r1, 5 + mov r2, 4 + mov r3, 6 + or r1, r9 + jlt32 r1, r2, +4 + jlt32 r1, r1, +3 + jlt32 r1, r3, +1 + exit + mov r0, 1 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_jne32_imm() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, 0xb + or r1, r9 + jne32 r1, 0xb, +4 + mov32 r0, 1 + mov32 r1, 0xa + or r1, r9 + jne32 r1, 0xb, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_jne32_reg() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, 0xb + or r1, r9 + mov32 r2, 0xb + jne32 r1, r2, +4 + mov32 r0, 1 + mov32 r1, 0xa + or r1, r9 + jne32 r1, r2, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_jset32_imm() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, 0x7 + or r1, r9 + jset32 r1, 0x8, +4 + mov32 r0, 1 + mov32 r1, 0x9 + jset32 r1, 0x8, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_jset32_reg() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, 0x7 + or r1, r9 + mov32 r2, 0x8 + jset32 r1, r2, +4 + mov32 r0, 1 + mov32 r1, 0x9 + jset32 r1, r2, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_jsge32_imm() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, -2 + or r1, r9 + jsge32 r1, -1, +5 + jsge32 r1, 0, +4 + mov32 r0, 1 + mov r1, -1 + jsge32 r1, -1, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_jsge32_reg() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, -2 + or r1, r9 + mov r2, -1 + mov32 r3, 0 + jsge32 r1, r2, +5 + jsge32 r1, r3, +4 + mov32 r0, 1 + mov r1, r2 + jsge32 r1, r2, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_jsgt32_imm() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, -2 + or r1, r9 + jsgt32 r1, -1, +4 + mov32 r0, 1 + mov32 r1, 0 + jsgt32 r1, -1, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_jsgt32_reg() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, -2 + or r1, r9 + mov r2, -1 + jsgt32 r1, r2, +4 + mov32 r0, 1 + mov32 r1, 0 + jsgt32 r1, r2, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_jsle32_imm() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, -2 + or r1, r9 + jsle32 r1, -3, +5 + jsle32 r1, -1, +1 + exit + mov32 r0, 1 + jsle32 r1, -2, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_jsle32_reg() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, -2 + or r1, r9 + mov r2, -3 + mov32 r3, 0 + jsle32 r1, r2, +6 + jsle32 r1, r3, +1 + exit + mov32 r0, 1 + mov r1, r2 + jsle32 r1, r2, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_jslt32_imm() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, -2 + or r1, r9 + jslt32 r1, -3, +4 + jslt32 r1, -2, +3 + jslt32 r1, -1, +1 + exit + mov32 r0, 1 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_jslt32_reg() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, -2 + or r1, r9 + mov r2, -3 + mov r3, -1 + jslt32 r1, r1, +4 + jslt32 r1, r2, +3 + jslt32 r1, r3, +1 + exit + mov32 r0, 1 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_lddw() { + let prog = assemble( + " + lddw r0, 0x1122334455667788 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1122334455667788); + } +} + +#[test] +fn test_jit_lddw2() { + let prog = assemble( + " + lddw r0, 0x0000000080000000 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x80000000); + } +} + +#[test] +fn test_jit_ldxb_all() { + let prog = assemble( + " + mov r0, r1 + ldxb r9, [r0+0] + lsh r9, 0 + ldxb r8, [r0+1] + lsh r8, 4 + ldxb r7, [r0+2] + lsh r7, 8 + ldxb r6, [r0+3] + lsh r6, 12 + ldxb r5, [r0+4] + lsh r5, 16 + ldxb r4, [r0+5] + lsh r4, 20 + ldxb r3, [r0+6] + lsh r3, 24 + ldxb r2, [r0+7] + lsh r2, 28 + ldxb r1, [r0+8] + lsh r1, 32 + ldxb r0, [r0+9] + lsh r0, 36 + or r0, r1 + or r0, r2 + or r0, r3 + or r0, r4 + or r0, r5 + or r0, r6 + or r0, r7 + or r0, r8 + or r0, r9 + exit", + ) + .unwrap(); + let mem = &mut [0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09]; + let mut vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit(mem).unwrap(), 0x9876543210); + } +} + +#[test] +fn test_jit_ldxb() { + let prog = assemble( + " + ldxb r0, [r1+2] + exit", + ) + .unwrap(); + let mem = &mut [0xaa, 0xbb, 0x11, 0xcc, 0xdd]; + let mut vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit(mem).unwrap(), 0x11); + } +} + +#[test] +fn test_jit_ldxdw() { + let prog = assemble( + " + ldxdw r0, [r1+2] + exit", + ) + .unwrap(); + let mem = &mut [ + 0xaa, 0xbb, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0xcc, 0xdd, + ]; + let mut vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit(mem).unwrap(), 0x8877665544332211); + } +} + +#[test] +fn test_jit_ldxh_all() { + let prog = assemble( + " + mov r0, r1 + ldxh r9, [r0+0] + be16 r9 + lsh r9, 0 + ldxh r8, [r0+2] + be16 r8 + lsh r8, 4 + ldxh r7, [r0+4] + be16 r7 + lsh r7, 8 + ldxh r6, [r0+6] + be16 r6 + lsh r6, 12 + ldxh r5, [r0+8] + be16 r5 + lsh r5, 16 + ldxh r4, [r0+10] + be16 r4 + lsh r4, 20 + ldxh r3, [r0+12] + be16 r3 + lsh r3, 24 + ldxh r2, [r0+14] + be16 r2 + lsh r2, 28 + ldxh r1, [r0+16] + be16 r1 + lsh r1, 32 + ldxh r0, [r0+18] + be16 r0 + lsh r0, 36 + or r0, r1 + or r0, r2 + or r0, r3 + or r0, r4 + or r0, r5 + or r0, r6 + or r0, r7 + or r0, r8 + or r0, r9 + exit", + ) + .unwrap(); + let mem = &mut [ + 0x00, 0x00, 0x00, 0x01, 0x00, 0x02, 0x00, 0x03, 0x00, 0x04, 0x00, 0x05, 0x00, 0x06, 0x00, + 0x07, 0x00, 0x08, 0x00, 0x09, + ]; + let mut vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit(mem).unwrap(), 0x9876543210); + } +} + +#[test] +fn test_jit_ldxh_all2() { + let prog = assemble( + " + mov r0, r1 + ldxh r9, [r0+0] + be16 r9 + ldxh r8, [r0+2] + be16 r8 + ldxh r7, [r0+4] + be16 r7 + ldxh r6, [r0+6] + be16 r6 + ldxh r5, [r0+8] + be16 r5 + ldxh r4, [r0+10] + be16 r4 + ldxh r3, [r0+12] + be16 r3 + ldxh r2, [r0+14] + be16 r2 + ldxh r1, [r0+16] + be16 r1 + ldxh r0, [r0+18] + be16 r0 + or r0, r1 + or r0, r2 + or r0, r3 + or r0, r4 + or r0, r5 + or r0, r6 + or r0, r7 + or r0, r8 + or r0, r9 + exit", + ) + .unwrap(); + let mem = &mut [ + 0x00, 0x01, 0x00, 0x02, 0x00, 0x04, 0x00, 0x08, 0x00, 0x10, 0x00, 0x20, 0x00, 0x40, 0x00, + 0x80, 0x01, 0x00, 0x02, 0x00, + ]; + let mut vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit(mem).unwrap(), 0x3ff); + } +} + +#[test] +fn test_jit_ldxh() { + let prog = assemble( + " + ldxh r0, [r1+2] + exit", + ) + .unwrap(); + let mem = &mut [0xaa, 0xbb, 0x11, 0x22, 0xcc, 0xdd]; + let mut vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit(mem).unwrap(), 0x2211); + } +} + +#[test] +fn test_jit_ldxh_same_reg() { + let prog = assemble( + " + mov r0, r1 + sth [r0], 0x1234 + ldxh r0, [r0] + exit", + ) + .unwrap(); + let mem = &mut [0xff, 0xff]; + let mut vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit(mem).unwrap(), 0x1234); + } +} + +#[test] +fn test_jit_ldxw_all() { + let prog = assemble( + " + mov r0, r1 + ldxw r9, [r0+0] + be32 r9 + ldxw r8, [r0+4] + be32 r8 + ldxw r7, [r0+8] + be32 r7 + ldxw r6, [r0+12] + be32 r6 + ldxw r5, [r0+16] + be32 r5 + ldxw r4, [r0+20] + be32 r4 + ldxw r3, [r0+24] + be32 r3 + ldxw r2, [r0+28] + be32 r2 + ldxw r1, [r0+32] + be32 r1 + ldxw r0, [r0+36] + be32 r0 + or r0, r1 + or r0, r2 + or r0, r3 + or r0, r4 + or r0, r5 + or r0, r6 + or r0, r7 + or r0, r8 + or r0, r9 + exit", + ) + .unwrap(); + let mem = &mut [ + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x08, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x08, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, + ]; + let mut vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit(mem).unwrap(), 0x030f0f); + } +} + +#[test] +fn test_jit_ldxw() { + let prog = assemble( + " + ldxw r0, [r1+2] + exit", + ) + .unwrap(); + let mem = &mut [0xaa, 0xbb, 0x11, 0x22, 0x33, 0x44, 0xcc, 0xdd]; + let mut vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit(mem).unwrap(), 0x44332211); + } +} + +#[test] +fn test_jit_le16() { + let prog = assemble( + " + ldxh r0, [r1] + le16 r0 + exit", + ) + .unwrap(); + let mem = &mut [0x22, 0x11]; + let mut vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit(mem).unwrap(), 0x1122); + } +} + +#[test] +fn test_jit_le32() { + let prog = assemble( + " + ldxw r0, [r1] + le32 r0 + exit", + ) + .unwrap(); + let mem = &mut [0x44, 0x33, 0x22, 0x11]; + let mut vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit(mem).unwrap(), 0x11223344); + } +} + +#[test] +fn test_jit_le64() { + let prog = assemble( + " + ldxdw r0, [r1] + le64 r0 + exit", + ) + .unwrap(); + let mem = &mut [0x88, 0x77, 0x66, 0x55, 0x44, 0x33, 0x22, 0x11]; + let mut vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit(mem).unwrap(), 0x1122334455667788); + } +} + +#[test] +fn test_jit_lsh_reg() { + let prog = assemble( + " + mov r0, 0x1 + mov r7, 4 + lsh r0, r7 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x10); + } +} + +#[test] +fn test_jit_mod() { + let prog = assemble( + " + mov32 r0, 5748 + mod32 r0, 92 + mov32 r1, 13 + mod32 r0, r1 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x5); + } +} + +#[test] +fn test_jit_mod32() { + let prog = assemble( + " + lddw r0, 0x100000003 + mod32 r0, 3 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x0); + } +} + +#[test] +fn test_jit_mod64() { + let prog = assemble( + " + mov32 r0, -1316649930 + lsh r0, 32 + or r0, 0x100dc5c8 + mov32 r1, 0xdde263e + lsh r1, 32 + or r1, 0x3cbef7f3 + mod r0, r1 + mod r0, 0x658f1778 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x30ba5a04); + } +} + +#[test] +fn test_jit_mov() { + let prog = assemble( + " + mov32 r1, 1 + mov32 r0, r1 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_mul32_imm() { + let prog = assemble( + " + mov r0, 3 + mul32 r0, 4 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0xc); + } +} + +#[test] +fn test_jit_mul32_reg() { + let prog = assemble( + " + mov r0, 3 + mov r1, 4 + mul32 r0, r1 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0xc); + } +} + +#[test] +fn test_jit_mul32_reg_overflow() { + let prog = assemble( + " + mov r0, 0x40000001 + mov r1, 4 + mul32 r0, r1 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x4); + } +} + +#[test] +fn test_jit_mul64_imm() { + let prog = assemble( + " + mov r0, 0x40000001 + mul r0, 4 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x100000004); + } +} + +#[test] +fn test_jit_mul64_reg() { + let prog = assemble( + " + mov r0, 0x40000001 + mov r1, 4 + mul r0, r1 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x100000004); + } +} + +#[test] +fn test_jit_mul_loop() { + let prog = assemble( + " + mov r0, 0x7 + add r1, 0xa + lsh r1, 0x20 + rsh r1, 0x20 + jeq r1, 0x0, +4 + mov r0, 0x7 + mul r0, 0x7 + add r1, -1 + jne r1, 0x0, -3 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x75db9c97); + } +} + +#[test] +fn test_jit_neg64() { + let prog = assemble( + " + mov32 r0, 2 + neg r0 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0xfffffffffffffffe); + } +} + +#[test] +fn test_jit_neg() { + let prog = assemble( + " + mov32 r0, 2 + neg32 r0 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0xfffffffe); + } +} + +#[test] +fn test_jit_prime() { + let prog = assemble( + " + mov r1, 67 + mov r0, 0x1 + mov r2, 0x2 + jgt r1, 0x2, +4 + ja +10 + add r2, 0x1 + mov r0, 0x1 + jge r2, r1, +7 + mov r3, r1 + div r3, r2 + mul r3, r2 + mov r4, r1 + sub r4, r3 + mov r0, 0x0 + jne r4, 0x0, -10 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_rhs32() { + let prog = assemble( + " + xor r0, r0 + sub r0, 1 + rsh32 r0, 8 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x00ffffff); + } +} + +#[test] +fn test_jit_rsh_reg() { + let prog = assemble( + " + mov r0, 0x10 + mov r7, 4 + rsh r0, r7 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x1); + } +} + +#[test] +fn test_jit_stack() { + let prog = assemble( + " + mov r1, 51 + stdw [r10-16], 0xab + stdw [r10-8], 0xcd + and r1, 1 + lsh r1, 3 + mov r2, r10 + add r2, r1 + ldxdw r0, [r2-16] + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0xcd); + } +} + +#[test] +fn test_jit_stack2() { + let prog = assemble( + " + stb [r10-4], 0x01 + stb [r10-3], 0x02 + stb [r10-2], 0x03 + stb [r10-1], 0x04 + mov r1, r10 + mov r2, 0x4 + sub r1, r2 + call 1 + mov r1, 0 + ldxb r2, [r10-4] + ldxb r3, [r10-3] + ldxb r4, [r10-2] + ldxb r5, [r10-1] + call 0 + xor r0, 0x2a2a2a2a + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.register_helper(0, helpers::gather_bytes).unwrap(); + vm.register_helper(1, helpers::memfrob).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x01020304); + } +} + +#[test] +fn test_jit_stb() { + let prog = assemble( + " + stb [r1+2], 0x11 + ldxb r0, [r1+2] + exit", + ) + .unwrap(); + let mem = &mut [0xaa, 0xbb, 0xff, 0xcc, 0xdd]; + let mut vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit(mem).unwrap(), 0x11); + } +} + +#[test] +fn test_jit_stdw() { + let prog = assemble( + " + stdw [r1+2], 0x44332211 + ldxdw r0, [r1+2] + exit", + ) + .unwrap(); + let mem = &mut [ + 0xaa, 0xbb, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xcc, 0xdd, + ]; + let mut vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit(mem).unwrap(), 0x44332211); + } +} + +#[test] +fn test_jit_sth() { + let prog = assemble( + " + sth [r1+2], 0x2211 + ldxh r0, [r1+2] + exit", + ) + .unwrap(); + let mem = &mut [0xaa, 0xbb, 0xff, 0xff, 0xcc, 0xdd]; + let mut vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit(mem).unwrap(), 0x2211); + } +} + +#[test] +fn test_jit_string_stack() { + let prog = assemble( + " + mov r1, 0x78636261 + stxw [r10-8], r1 + mov r6, 0x0 + stxb [r10-4], r6 + stxb [r10-12], r6 + mov r1, 0x79636261 + stxw [r10-16], r1 + mov r1, r10 + add r1, -8 + mov r2, r1 + call 0x4 + mov r1, r0 + mov r0, 0x1 + lsh r1, 0x20 + rsh r1, 0x20 + jne r1, 0x0, +11 + mov r1, r10 + add r1, -8 + mov r2, r10 + add r2, -16 + call 0x4 + mov r1, r0 + lsh r1, 0x20 + rsh r1, 0x20 + mov r0, 0x1 + jeq r1, r6, +1 + mov r0, 0x0 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.register_helper(4, helpers::strcmp).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit().unwrap(), 0x0); + } +} + +#[test] +fn test_jit_stw() { + let prog = assemble( + " + stw [r1+2], 0x44332211 + ldxw r0, [r1+2] + exit", + ) + .unwrap(); + let mem = &mut [0xaa, 0xbb, 0xff, 0xff, 0xff, 0xff, 0xcc, 0xdd]; + let mut vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit(mem).unwrap(), 0x44332211); + } +} + +#[test] +fn test_jit_stxb() { + let prog = assemble( + " + mov32 r2, 0x11 + stxb [r1+2], r2 + ldxb r0, [r1+2] + exit", + ) + .unwrap(); + let mem = &mut [0xaa, 0xbb, 0xff, 0xcc, 0xdd]; + let mut vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit(mem).unwrap(), 0x11); + } +} + +#[test] +fn test_jit_stxb_all() { + let prog = assemble( + " + mov r0, 0xf0 + mov r2, 0xf2 + mov r3, 0xf3 + mov r4, 0xf4 + mov r5, 0xf5 + mov r6, 0xf6 + mov r7, 0xf7 + mov r8, 0xf8 + stxb [r1], r0 + stxb [r1+1], r2 + stxb [r1+2], r3 + stxb [r1+3], r4 + stxb [r1+4], r5 + stxb [r1+5], r6 + stxb [r1+6], r7 + stxb [r1+7], r8 + ldxdw r0, [r1] + be64 r0 + exit", + ) + .unwrap(); + let mem = &mut [0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff]; + let mut vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit(mem).unwrap(), 0xf0f2f3f4f5f6f7f8); + } +} + +#[test] +fn test_jit_stxb_all2() { + let prog = assemble( + " + mov r0, r1 + mov r1, 0xf1 + mov r9, 0xf9 + stxb [r0], r1 + stxb [r0+1], r9 + ldxh r0, [r0] + be16 r0 + exit", + ) + .unwrap(); + let mem = &mut [0xff, 0xff]; + let mut vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit(mem).unwrap(), 0xf1f9); + } +} + +#[test] +fn test_jit_stxb_chain() { + let prog = assemble( + " + mov r0, r1 + ldxb r9, [r0+0] + stxb [r0+1], r9 + ldxb r8, [r0+1] + stxb [r0+2], r8 + ldxb r7, [r0+2] + stxb [r0+3], r7 + ldxb r6, [r0+3] + stxb [r0+4], r6 + ldxb r5, [r0+4] + stxb [r0+5], r5 + ldxb r4, [r0+5] + stxb [r0+6], r4 + ldxb r3, [r0+6] + stxb [r0+7], r3 + ldxb r2, [r0+7] + stxb [r0+8], r2 + ldxb r1, [r0+8] + stxb [r0+9], r1 + ldxb r0, [r0+9] + exit", + ) + .unwrap(); + let mem = &mut [0x2a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]; + let mut vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit(mem).unwrap(), 0x2a); + } +} + +#[test] +fn test_jit_stxdw() { + let prog = assemble( + " + mov r2, -2005440939 + lsh r2, 32 + or r2, 0x44332211 + stxdw [r1+2], r2 + ldxdw r0, [r1+2] + exit", + ) + .unwrap(); + let mem = &mut [ + 0xaa, 0xbb, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xcc, 0xdd, + ]; + let mut vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit(mem).unwrap(), 0x8877665544332211); + } +} + +#[test] +fn test_jit_stxh() { + let prog = assemble( + " + mov32 r2, 0x2211 + stxh [r1+2], r2 + ldxh r0, [r1+2] + exit", + ) + .unwrap(); + let mem = &mut [0xaa, 0xbb, 0xff, 0xff, 0xcc, 0xdd]; + let mut vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit(mem).unwrap(), 0x2211); + } +} + +#[test] +fn test_jit_stxw() { + let prog = assemble( + " + mov32 r2, 0x44332211 + stxw [r1+2], r2 + ldxw r0, [r1+2] + exit", + ) + .unwrap(); + let mem = &mut [0xaa, 0xbb, 0xff, 0xff, 0xff, 0xff, 0xcc, 0xdd]; + let mut vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit(mem).unwrap(), 0x44332211); + } +} + +#[test] +fn test_jit_subnet() { + let prog = assemble( + " + mov r2, 0xe + ldxh r3, [r1+12] + jne r3, 0x81, +2 + mov r2, 0x12 + ldxh r3, [r1+16] + and r3, 0xffff + jne r3, 0x8, +5 + add r1, r2 + mov r0, 0x1 + ldxw r1, [r1+16] + and r1, 0xffffff + jeq r1, 0x1a8c0, +1 + mov r0, 0x0 + exit", + ) + .unwrap(); + let mem = &mut [ + 0x00, 0x00, 0xc0, 0x9f, 0xa0, 0x97, 0x00, 0xa0, 0xcc, 0x3b, 0xbf, 0xfa, 0x08, 0x00, 0x45, + 0x10, 0x00, 0x3c, 0x46, 0x3c, 0x40, 0x00, 0x40, 0x06, 0x73, 0x1c, 0xc0, 0xa8, 0x01, 0x02, + 0xc0, 0xa8, 0x01, 0x01, 0x06, 0x0e, 0x00, 0x17, 0x99, 0xc5, 0xa0, 0xec, 0x00, 0x00, 0x00, + 0x00, 0xa0, 0x02, 0x7d, 0x78, 0xe0, 0xa3, 0x00, 0x00, 0x02, 0x04, 0x05, 0xb4, 0x04, 0x02, + 0x08, 0x0a, 0x00, 0x9c, 0x27, 0x24, 0x00, 0x00, 0x00, 0x00, 0x01, 0x03, 0x03, 0x00, + ]; + let mut vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit(mem).unwrap(), 0x1); + } +} + +const PROG_TCP_PORT_80: [u8; 152] = [ + 0x71, 0x12, 0x0c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x71, 0x13, 0x0d, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x67, 0x03, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x4f, 0x23, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xb7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x55, 0x03, 0x0c, 0x00, 0x08, 0x00, 0x00, 0x00, + 0x71, 0x12, 0x17, 0x00, 0x00, 0x00, 0x00, 0x00, 0x55, 0x02, 0x0a, 0x00, 0x06, 0x00, 0x00, 0x00, + 0x71, 0x12, 0x0e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0x01, 0x00, 0x00, 0x0e, 0x00, 0x00, 0x00, + 0x57, 0x02, 0x00, 0x00, 0x0f, 0x00, 0x00, 0x00, 0x67, 0x02, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x0f, 0x21, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x69, 0x12, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x15, 0x02, 0x02, 0x00, 0x00, 0x50, 0x00, 0x00, 0x69, 0x11, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x55, 0x01, 0x01, 0x00, 0x00, 0x50, 0x00, 0x00, 0xb7, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +]; + +#[test] +fn test_jit_tcp_port80_match() { + let mem = &mut [ + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x00, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x08, 0x00, 0x45, + 0x00, 0x00, 0x56, 0x00, 0x01, 0x00, 0x00, 0x40, 0x06, 0xf9, 0x4d, 0xc0, 0xa8, 0x00, 0x01, + 0xc0, 0xa8, 0x00, 0x02, 0x27, 0x10, 0x00, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x50, 0x02, 0x20, 0x00, 0xc5, 0x18, 0x00, 0x00, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + ]; + let prog = &PROG_TCP_PORT_80; + let mut vm = rbpf::EbpfVmRaw::new(Some(prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit(mem).unwrap(), 0x1); + } +} + +#[test] +fn test_jit_tcp_port80_nomatch() { + let mem = &mut [ + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x00, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x08, 0x00, 0x45, + 0x00, 0x00, 0x56, 0x00, 0x01, 0x00, 0x00, 0x40, 0x06, 0xf9, 0x4d, 0xc0, 0xa8, 0x00, 0x01, + 0xc0, 0xa8, 0x00, 0x02, 0x00, 0x16, 0x27, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x51, 0x02, 0x20, 0x00, 0xc5, 0x18, 0x00, 0x00, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + ]; + let prog = &PROG_TCP_PORT_80; + let mut vm = rbpf::EbpfVmRaw::new(Some(prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit(mem).unwrap(), 0x0); + } +} + +#[test] +fn test_jit_tcp_port80_nomatch_ethertype() { + let mem = &mut [ + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x00, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x08, 0x01, 0x45, + 0x00, 0x00, 0x56, 0x00, 0x01, 0x00, 0x00, 0x40, 0x06, 0xf9, 0x4d, 0xc0, 0xa8, 0x00, 0x01, + 0xc0, 0xa8, 0x00, 0x02, 0x27, 0x10, 0x00, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x50, 0x02, 0x20, 0x00, 0xc5, 0x18, 0x00, 0x00, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + ]; + let prog = &PROG_TCP_PORT_80; + let mut vm = rbpf::EbpfVmRaw::new(Some(prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit(mem).unwrap(), 0x0); + } +} + +#[test] +fn test_jit_tcp_port80_nomatch_proto() { + let mem = &mut [ + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x00, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x08, 0x00, 0x45, + 0x00, 0x00, 0x56, 0x00, 0x01, 0x00, 0x00, 0x40, 0x11, 0xf9, 0x4d, 0xc0, 0xa8, 0x00, 0x01, + 0xc0, 0xa8, 0x00, 0x02, 0x27, 0x10, 0x00, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x50, 0x02, 0x20, 0x00, 0xc5, 0x18, 0x00, 0x00, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + ]; + let prog = &PROG_TCP_PORT_80; + let mut vm = rbpf::EbpfVmRaw::new(Some(prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit(mem).unwrap(), 0x0); + } +} + +#[test] +fn test_jit_tcp_sack_match() { + let mut mem = TCP_SACK_MATCH.to_vec(); + let prog = assemble(TCP_SACK_ASM).unwrap(); + let mut vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit(mem.as_mut_slice()).unwrap(), 0x1); + } +} + +#[test] +fn test_jit_tcp_sack_nomatch() { + let mut mem = TCP_SACK_NOMATCH.to_vec(); + let prog = assemble(TCP_SACK_ASM).unwrap(); + let mut vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + vm.jit_compile().unwrap(); + unsafe { + assert_eq!(vm.execute_program_jit(mem.as_mut_slice()).unwrap(), 0x0); + } +} diff --git a/kernel/crates/rbpf/tests/ubpf_verifier.rs b/kernel/crates/rbpf/tests/ubpf_verifier.rs new file mode 100644 index 00000000..8e6b03af --- /dev/null +++ b/kernel/crates/rbpf/tests/ubpf_verifier.rs @@ -0,0 +1,177 @@ +// SPDX-License-Identifier: (Apache-2.0 OR MIT) +// Converted from the tests for uBPF +// Copyright 2015 Big Switch Networks, Inc +// Copyright 2016 6WIND S.A. + +// The tests contained in this file are extracted from the unit tests of uBPF software. Each test +// in this file has a name in the form `test_verifier_`, and corresponds to the +// (human-readable) code in `ubpf/tree/master/tests/`, available at +// (hyphen had to be replaced with underscores +// as Rust will not accept them in function names). It is strongly advised to refer to the uBPF +// version to understand what these program do. +// +// Each program was assembled from the uBPF version with the assembler provided by uBPF itself, and +// available at . +// The very few modifications that have been realized should be indicated. + +// These are unit tests for the eBPF “verifier”. + +extern crate rbpf; + +use rbpf::{assembler::assemble, ebpf}; + +#[test] +#[should_panic(expected = "[Verifier] Error: unsupported argument for LE/BE (insn #0)")] +fn test_verifier_err_endian_size() { + let prog = &[ + 0xdc, 0x01, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0xb7, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + ]; + let vm = rbpf::EbpfVmNoData::new(Some(prog)).unwrap(); + vm.execute_program().unwrap(); +} + +#[test] +#[should_panic(expected = "[Verifier] Error: incomplete LD_DW instruction (insn #0)")] +fn test_verifier_err_incomplete_lddw() { + // Note: ubpf has test-err-incomplete-lddw2, which is the same + let prog = &[ + 0x18, 0x00, 0x00, 0x00, 0x88, 0x77, 0x66, 0x55, 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, + ]; + let vm = rbpf::EbpfVmNoData::new(Some(prog)).unwrap(); + vm.execute_program().unwrap(); +} + +#[test] +#[should_panic(expected = "[Verifier] Error: infinite loop")] +fn test_verifier_err_infinite_loop() { + let prog = assemble( + " + ja -1 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.execute_program().unwrap(); +} + +#[test] +#[should_panic(expected = "[Verifier] Error: invalid destination register (insn #0)")] +fn test_verifier_err_invalid_reg_dst() { + let prog = assemble( + " + mov r11, 1 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.execute_program().unwrap(); +} + +#[test] +#[should_panic(expected = "[Verifier] Error: invalid source register (insn #0)")] +fn test_verifier_err_invalid_reg_src() { + let prog = assemble( + " + mov r0, r11 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.execute_program().unwrap(); +} + +#[test] +#[should_panic(expected = "[Verifier] Error: jump to middle of LD_DW at #2 (insn #0)")] +fn test_verifier_err_jmp_lddw() { + let prog = assemble( + " + ja +1 + lddw r0, 0x1122334455667788 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.execute_program().unwrap(); +} + +#[test] +#[should_panic(expected = "[Verifier] Error: jump out of code to #3 (insn #0)")] +fn test_verifier_err_jmp_out() { + let prog = assemble( + " + ja +2 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.execute_program().unwrap(); +} + +#[test] +#[should_panic(expected = "[Verifier] Error: program does not end with “EXIT” instruction")] +fn test_verifier_err_no_exit() { + let prog = assemble( + " + mov32 r0, 0", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.execute_program().unwrap(); +} + +#[test] +fn test_verifier_err_no_exit_backward_jump() { + let prog = assemble( + " + ja +1 + exit + ja -2", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.execute_program().unwrap(); +} + +#[test] +#[should_panic(expected = "[Verifier] Error: eBPF program length limited to 1000000, here 1000001")] +fn test_verifier_err_too_many_instructions() { + // uBPF uses 65637 instructions, because it sets its limit at 65636. + // We use the classic 4096 limit from kernel, so no need to produce as many instructions. + let mut prog = (0..(1_000_000 * ebpf::INSN_SIZE)) + .map(|x| match x % 8 { + 0 => 0xb7, + 1 => 0x01, + _ => 0, + }) + .collect::>(); + prog.append(&mut vec![0x95, 0, 0, 0, 0, 0, 0, 0]); + + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.execute_program().unwrap(); +} + +#[test] +#[should_panic(expected = "[Verifier] Error: unknown eBPF opcode 0x6 (insn #0)")] +fn test_verifier_err_unknown_opcode() { + let prog = &[ + 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, + ]; + let vm = rbpf::EbpfVmNoData::new(Some(prog)).unwrap(); + vm.execute_program().unwrap(); +} + +#[test] +#[should_panic(expected = "[Verifier] Error: cannot write into register r10 (insn #0)")] +fn test_verifier_err_write_r10() { + let prog = assemble( + " + mov r10, 1 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.execute_program().unwrap(); +} diff --git a/kernel/crates/rbpf/tests/ubpf_vm.rs b/kernel/crates/rbpf/tests/ubpf_vm.rs new file mode 100644 index 00000000..05d8a9ba --- /dev/null +++ b/kernel/crates/rbpf/tests/ubpf_vm.rs @@ -0,0 +1,2670 @@ +// SPDX-License-Identifier: (Apache-2.0 OR MIT) +// Converted from the tests for uBPF +// Copyright 2015 Big Switch Networks, Inc +// Copyright 2016 6WIND S.A. + +// The tests contained in this file are extracted from the unit tests of uBPF software. Each test +// in this file has a name in the form `test_vm_`, and corresponds to the (human-readable) +// code in `ubpf/tree/master/tests/`, available at +// (hyphen had to be replaced with underscores +// as Rust will not accept them in function names). It is strongly advised to refer to the uBPF +// version to understand what these program do. +// +// Each program was assembled from the uBPF version with the assembler provided by uBPF itself, and +// available at . +// The very few modifications that have been realized should be indicated. + +// These are unit tests for the eBPF interpreter. + +#![allow(clippy::unreadable_literal)] + +extern crate rbpf; +mod common; + +use common::{TCP_SACK_ASM, TCP_SACK_MATCH, TCP_SACK_NOMATCH}; +use rbpf::{assembler::assemble, helpers}; + +#[test] +fn test_vm_add() { + let prog = assemble( + " + mov32 r0, 0 + mov32 r1, 2 + add32 r0, 1 + add32 r0, r1 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x3); +} + +#[test] +fn test_vm_alu64_arith() { + let prog = assemble( + " + mov r0, 0 + mov r1, 1 + mov r2, 2 + mov r3, 3 + mov r4, 4 + mov r5, 5 + mov r6, 6 + mov r7, 7 + mov r8, 8 + mov r9, 9 + add r0, 23 + add r0, r7 + sub r0, 13 + sub r0, r1 + mul r0, 7 + mul r0, r3 + div r0, 2 + div r0, r4 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x2a); +} + +#[test] +fn test_vm_alu64_bit() { + let prog = assemble( + " + mov r0, 0 + mov r1, 1 + mov r2, 2 + mov r3, 3 + mov r4, 4 + mov r5, 5 + mov r6, 6 + mov r7, 7 + mov r8, 8 + or r0, r5 + or r0, 0xa0 + and r0, 0xa3 + mov r9, 0x91 + and r0, r9 + lsh r0, 32 + lsh r0, 22 + lsh r0, r8 + rsh r0, 32 + rsh r0, 19 + rsh r0, r7 + xor r0, 0x03 + xor r0, r2 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x11); +} + +#[test] +fn test_vm_alu_arith() { + let prog = assemble( + " + mov32 r0, 0 + mov32 r1, 1 + mov32 r2, 2 + mov32 r3, 3 + mov32 r4, 4 + mov32 r5, 5 + mov32 r6, 6 + mov32 r7, 7 + mov32 r8, 8 + mov32 r9, 9 + add32 r0, 23 + add32 r0, r7 + sub32 r0, 13 + sub32 r0, r1 + mul32 r0, 7 + mul32 r0, r3 + div32 r0, 2 + div32 r0, r4 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x2a); +} + +#[test] +fn test_vm_alu_bit() { + let prog = assemble( + " + mov32 r0, 0 + mov32 r1, 1 + mov32 r2, 2 + mov32 r3, 3 + mov32 r4, 4 + mov32 r5, 5 + mov32 r6, 6 + mov32 r7, 7 + mov32 r8, 8 + or32 r0, r5 + or32 r0, 0xa0 + and32 r0, 0xa3 + mov32 r9, 0x91 + and32 r0, r9 + lsh32 r0, 22 + lsh32 r0, r8 + rsh32 r0, 19 + rsh32 r0, r7 + xor32 r0, 0x03 + xor32 r0, r2 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x11); +} + +#[test] +fn test_vm_arsh32_high_shift() { + let prog = assemble( + " + mov r0, 8 + lddw r1, 0x100000001 + arsh32 r0, r1 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x4); +} + +#[test] +fn test_vm_arsh() { + let prog = assemble( + " + mov32 r0, 0xf8 + lsh32 r0, 28 + arsh32 r0, 16 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0xffff8000); +} + +#[test] +fn test_vm_arsh64() { + let prog = assemble( + " + mov32 r0, 1 + lsh r0, 63 + arsh r0, 55 + mov32 r1, 5 + arsh r0, r1 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0xfffffffffffffff8); +} + +#[test] +fn test_vm_arsh_reg() { + let prog = assemble( + " + mov32 r0, 0xf8 + mov32 r1, 16 + lsh32 r0, 28 + arsh32 r0, r1 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0xffff8000); +} + +#[test] +fn test_vm_arsh_imm_overflow() { + let prog = assemble( + " + mov r0, 1 + lsh r0, 63 + arsh r0, 0xff20 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0xffffffff80000000); +} + +#[test] +fn test_vm_arsh_reg_overflow() { + let prog = assemble( + " + mov r0, 1 + lsh r0, 63 + mov r1, 0xff04 + arsh r0, r1 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0xf800000000000000); +} + +#[test] +fn test_vm_arsh32_imm_overflow() { + let prog = assemble( + " + mov32 r0, 1 + lsh32 r0, 31 + arsh32 r0, 0xff10 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0xffff8000); +} + +#[test] +fn test_vm_arsh32_reg_overflow() { + let prog = assemble( + " + mov32 r0, 1 + lsh32 r0, 31 + mov32 r1, 32 + arsh32 r0, r1 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x80000000); +} + +#[test] +fn test_vm_be16() { + let prog = assemble( + " + ldxh r0, [r1] + be16 r0 + exit", + ) + .unwrap(); + let mem = &mut [0x11, 0x22]; + let vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program(mem).unwrap(), 0x1122); +} + +#[test] +fn test_vm_be16_high() { + let prog = assemble( + " + ldxdw r0, [r1] + be16 r0 + exit", + ) + .unwrap(); + let mem = &mut [0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88]; + let vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program(mem).unwrap(), 0x1122); +} + +#[test] +fn test_vm_be32() { + let prog = assemble( + " + ldxw r0, [r1] + be32 r0 + exit", + ) + .unwrap(); + let mem = &mut [0x11, 0x22, 0x33, 0x44]; + let vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program(mem).unwrap(), 0x11223344); +} + +#[test] +fn test_vm_be32_high() { + let prog = assemble( + " + ldxdw r0, [r1] + be32 r0 + exit", + ) + .unwrap(); + let mem = &mut [0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88]; + let vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program(mem).unwrap(), 0x11223344); +} + +#[test] +fn test_vm_be64() { + let prog = assemble( + " + ldxdw r0, [r1] + be64 r0 + exit", + ) + .unwrap(); + let mem = &mut [0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88]; + let vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program(mem).unwrap(), 0x1122334455667788); +} + +#[test] +fn test_vm_call() { + let prog = assemble( + " + mov r1, 1 + mov r2, 2 + mov r3, 3 + mov r4, 4 + mov r5, 5 + call 0 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.register_helper(0, helpers::gather_bytes).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x0102030405); +} + +#[test] +fn test_vm_call_memfrob() { + let prog = assemble( + " + mov r6, r1 + add r1, 2 + mov r2, 4 + call 1 + ldxdw r0, [r6] + be64 r0 + exit", + ) + .unwrap(); + let mem = &mut [0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08]; + let mut vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + vm.register_helper(1, helpers::memfrob).unwrap(); + assert_eq!(vm.execute_program(mem).unwrap(), 0x102292e2f2c0708); +} + +// TODO: helpers::trash_registers needs asm!(). +// Try this again once asm!() is available in stable. +//#[test] +//fn test_vm_call_save() { +//let prog = &[ +//0xb7, 0x06, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, +//0xb7, 0x07, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, +//0xb7, 0x08, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, +//0xb7, 0x09, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, +//0x85, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, +//0xb7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +//0x4f, 0x60, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +//0x4f, 0x70, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +//0x4f, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +//0x4f, 0x90, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +//0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +//]; +//let mut vm = rbpf::EbpfVmNoData::new(Some(prog)).unwrap(); +//vm.register_helper(2, helpers::trash_registers); +//assert_eq!(vm.execute_program().unwrap(), 0x4321); +//} + +#[test] +fn test_vm_div32_high_divisor() { + let prog = assemble( + " + mov r0, 12 + lddw r1, 0x100000004 + div32 r0, r1 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x3); +} + +#[test] +fn test_vm_div32_imm() { + let prog = assemble( + " + lddw r0, 0x10000000c + div32 r0, 4 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x3); +} + +#[test] +fn test_vm_div32_reg() { + let prog = assemble( + " + lddw r0, 0x10000000c + mov r1, 4 + div32 r0, r1 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x3); +} + +#[test] +fn test_vm_div64_imm() { + let prog = assemble( + " + mov r0, 0xc + lsh r0, 32 + div r0, 4 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x300000000); +} + +#[test] +fn test_vm_div64_reg() { + let prog = assemble( + " + mov r0, 0xc + lsh r0, 32 + mov r1, 4 + div r0, r1 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x300000000); +} + +#[test] +fn test_vm_early_exit() { + let prog = assemble( + " + mov r0, 3 + exit + mov r0, 4 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x3); +} + +// uBPF limits the number of user functions at 64. We don't. +//#[test] +//fn test_vm_err_call_bad_imm() { +//} + +#[test] +#[should_panic(expected = "Error: unknown helper function (id: 0x3f)")] +fn test_vm_err_call_unreg() { + let prog = assemble( + " + mov r1, 1 + mov r2, 2 + mov r3, 3 + mov r4, 4 + mov r5, 5 + call 63 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.execute_program().unwrap(); +} + +#[test] +fn test_vm_div64_by_zero_imm() { + let prog = assemble( + " + mov32 r0, 1 + div r0, 0 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x0); +} + +#[test] +fn test_vm_div_by_zero_imm() { + let prog = assemble( + " + mov32 r0, 1 + div32 r0, 0 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x0); +} + +#[test] +fn test_vm_mod64_by_zero_imm() { + let prog = assemble( + " + mov32 r0, 1 + mod r0, 0 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_mod_by_zero_imm() { + let prog = assemble( + " + mov32 r0, 1 + mod32 r0, 0 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +// Make sure we only consider the last 32 bits of the divisor. +#[test] +fn test_vm_mod_by_zero_reg_long() { + let prog = assemble( + " + lddw r1, 0x100000000 + mod32 r0, r1 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x0); +} + +#[test] +fn test_vm_div64_by_zero_reg() { + let prog = assemble( + " + mov32 r0, 1 + mov32 r1, 0 + div r0, r1 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x0); +} + +#[test] +fn test_vm_div_by_zero_reg() { + let prog = assemble( + " + mov32 r0, 1 + mov32 r1, 0 + div32 r0, r1 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x0); +} + +// Make sure we only consider the last 32 bits of the divisor. +#[test] +fn test_vm_div_by_zero_reg_long() { + let prog = assemble( + " + lddw r1, 0x100000000 + div32 r0, r1 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x0); +} + +#[test] +fn test_vm_mod64_by_zero_reg() { + let prog = assemble( + " + mov32 r0, 1 + mov32 r1, 0 + mod r0, r1 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_mod_by_zero_reg() { + let prog = assemble( + " + mov32 r0, 1 + mov32 r1, 0 + mod32 r0, r1 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +#[should_panic(expected = "Error: out of bounds memory store (insn #1)")] +fn test_vm_err_stack_out_of_bound() { + let prog = assemble( + " + stb [r10], 0 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.execute_program().unwrap(); +} + +#[test] +fn test_vm_exit() { + let prog = assemble( + " + mov r0, 0 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x0); +} + +#[test] +fn test_vm_ja() { + let prog = assemble( + " + mov r0, 1 + ja +1 + mov r0, 2 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_jeq_imm() { + let prog = assemble( + " + mov32 r0, 0 + mov32 r1, 0xa + jeq r1, 0xb, +4 + mov32 r0, 1 + mov32 r1, 0xb + jeq r1, 0xb, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_jeq_reg() { + let prog = assemble( + " + mov32 r0, 0 + mov32 r1, 0xa + mov32 r2, 0xb + jeq r1, r2, +4 + mov32 r0, 1 + mov32 r1, 0xb + jeq r1, r2, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_jge_imm() { + let prog = assemble( + " + mov32 r0, 0 + mov32 r1, 0xa + jge r1, 0xb, +4 + mov32 r0, 1 + mov32 r1, 0xc + jge r1, 0xb, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_jle_imm() { + let prog = assemble( + " + mov32 r0, 0 + mov32 r1, 5 + jle r1, 4, +1 + jle r1, 6, +1 + exit + jle r1, 5, +1 + exit + mov32 r0, 1 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_jle_reg() { + let prog = assemble( + " + mov r0, 0 + mov r1, 5 + mov r2, 4 + mov r3, 6 + jle r1, r2, +2 + jle r1, r1, +1 + exit + jle r1, r3, +1 + exit + mov r0, 1 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_jgt_imm() { + let prog = assemble( + " + mov32 r0, 0 + mov32 r1, 5 + jgt r1, 6, +2 + jgt r1, 5, +1 + jgt r1, 4, +1 + exit + mov32 r0, 1 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_jgt_reg() { + let prog = assemble( + " + mov r0, 0 + mov r1, 5 + mov r2, 6 + mov r3, 4 + jgt r1, r2, +2 + jgt r1, r1, +1 + jgt r1, r3, +1 + exit + mov r0, 1 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_jlt_imm() { + let prog = assemble( + " + mov32 r0, 0 + mov32 r1, 5 + jlt r1, 4, +2 + jlt r1, 5, +1 + jlt r1, 6, +1 + exit + mov32 r0, 1 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_jlt_reg() { + let prog = assemble( + " + mov r0, 0 + mov r1, 5 + mov r2, 4 + mov r3, 6 + jlt r1, r2, +2 + jlt r1, r1, +1 + jlt r1, r3, +1 + exit + mov r0, 1 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_jit_bounce() { + let prog = assemble( + " + mov r0, 1 + mov r6, r0 + mov r7, r6 + mov r8, r7 + mov r9, r8 + mov r0, r9 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_jne_reg() { + let prog = assemble( + " + mov32 r0, 0 + mov32 r1, 0xb + mov32 r2, 0xb + jne r1, r2, +4 + mov32 r0, 1 + mov32 r1, 0xa + jne r1, r2, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_jset_imm() { + let prog = assemble( + " + mov32 r0, 0 + mov32 r1, 0x7 + jset r1, 0x8, +4 + mov32 r0, 1 + mov32 r1, 0x9 + jset r1, 0x8, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_jset_reg() { + let prog = assemble( + " + mov32 r0, 0 + mov32 r1, 0x7 + mov32 r2, 0x8 + jset r1, r2, +4 + mov32 r0, 1 + mov32 r1, 0x9 + jset r1, r2, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_jsge_imm() { + let prog = assemble( + " + mov32 r0, 0 + mov r1, -2 + jsge r1, -1, +5 + jsge r1, 0, +4 + mov32 r0, 1 + mov r1, -1 + jsge r1, -1, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_jsge_reg() { + let prog = assemble( + " + mov32 r0, 0 + mov r1, -2 + mov r2, -1 + mov32 r3, 0 + jsge r1, r2, +5 + jsge r1, r3, +4 + mov32 r0, 1 + mov r1, r2 + jsge r1, r2, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_jsle_imm() { + let prog = assemble( + " + mov32 r0, 0 + mov r1, -2 + jsle r1, -3, +1 + jsle r1, -1, +1 + exit + mov32 r0, 1 + jsle r1, -2, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_jsle_reg() { + let prog = assemble( + " + mov32 r0, 0 + mov r1, -1 + mov r2, -2 + mov32 r3, 0 + jsle r1, r2, +1 + jsle r1, r3, +1 + exit + mov32 r0, 1 + mov r1, r2 + jsle r1, r2, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_jsgt_imm() { + let prog = assemble( + " + mov32 r0, 0 + mov r1, -2 + jsgt r1, -1, +4 + mov32 r0, 1 + mov32 r1, 0 + jsgt r1, -1, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_jsgt_reg() { + let prog = assemble( + " + mov32 r0, 0 + mov r1, -2 + mov r2, -1 + jsgt r1, r2, +4 + mov32 r0, 1 + mov32 r1, 0 + jsgt r1, r2, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_jslt_imm() { + let prog = assemble( + " + mov32 r0, 0 + mov r1, -2 + jslt r1, -3, +2 + jslt r1, -2, +1 + jslt r1, -1, +1 + exit + mov32 r0, 1 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_jslt_reg() { + let prog = assemble( + " + mov32 r0, 0 + mov r1, -2 + mov r2, -3 + mov r3, -1 + jslt r1, r1, +2 + jslt r1, r2, +1 + jslt r1, r3, +1 + exit + mov32 r0, 1 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_jeq32_imm() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0x0 + mov32 r1, 0xa + jeq32 r1, 0xb, +5 + mov32 r0, 1 + mov r1, 0xb + or r1, r9 + jeq32 r1, 0xb, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_jeq32_reg() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, 0xa + mov32 r2, 0xb + jeq32 r1, r2, +5 + mov32 r0, 1 + mov32 r1, 0xb + or r1, r9 + jeq32 r1, r2, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_jge32_imm() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, 0xa + jge32 r1, 0xb, +5 + mov32 r0, 1 + or r1, r9 + mov32 r1, 0xc + jge32 r1, 0xb, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_jge32_reg() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, 0xa + mov32 r2, 0xb + jge32 r1, r2, +5 + mov32 r0, 1 + or r1, r9 + mov32 r1, 0xc + jge32 r1, r2, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_jgt32_imm() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, 5 + or r1, r9 + jgt32 r1, 6, +4 + jgt32 r1, 5, +3 + jgt32 r1, 4, +1 + exit + mov32 r0, 1 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_jgt32_reg() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov r0, 0 + mov r1, 5 + mov32 r1, 5 + or r1, r9 + mov r2, 6 + mov r3, 4 + jgt32 r1, r2, +4 + jgt32 r1, r1, +3 + jgt32 r1, r3, +1 + exit + mov r0, 1 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_jle32_imm() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, 5 + or r1, r9 + jle32 r1, 4, +5 + jle32 r1, 6, +1 + exit + jle32 r1, 5, +1 + exit + mov32 r0, 1 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_jle32_reg() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov r0, 0 + mov r1, 5 + mov r2, 4 + mov r3, 6 + or r1, r9 + jle32 r1, r2, +5 + jle32 r1, r1, +1 + exit + jle32 r1, r3, +1 + exit + mov r0, 1 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_jlt32_imm() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, 5 + or r1, r9 + jlt32 r1, 4, +4 + jlt32 r1, 5, +3 + jlt32 r1, 6, +1 + exit + mov32 r0, 1 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_jlt32_reg() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov r0, 0 + mov r1, 5 + mov r2, 4 + mov r3, 6 + or r1, r9 + jlt32 r1, r2, +4 + jlt32 r1, r1, +3 + jlt32 r1, r3, +1 + exit + mov r0, 1 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_jne32_imm() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, 0xb + or r1, r9 + jne32 r1, 0xb, +4 + mov32 r0, 1 + mov32 r1, 0xa + or r1, r9 + jne32 r1, 0xb, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_jne32_reg() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, 0xb + or r1, r9 + mov32 r2, 0xb + jne32 r1, r2, +4 + mov32 r0, 1 + mov32 r1, 0xa + or r1, r9 + jne32 r1, r2, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_jset32_imm() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, 0x7 + or r1, r9 + jset32 r1, 0x8, +4 + mov32 r0, 1 + mov32 r1, 0x9 + jset32 r1, 0x8, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_jset32_reg() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, 0x7 + or r1, r9 + mov32 r2, 0x8 + jset32 r1, r2, +4 + mov32 r0, 1 + mov32 r1, 0x9 + jset32 r1, r2, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_jsge32_imm() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, -2 + or r1, r9 + jsge32 r1, -1, +5 + jsge32 r1, 0, +4 + mov32 r0, 1 + mov r1, -1 + jsge32 r1, -1, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_jsge32_reg() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, -2 + or r1, r9 + mov r2, -1 + mov32 r3, 0 + jsge32 r1, r2, +5 + jsge32 r1, r3, +4 + mov32 r0, 1 + mov r1, r2 + jsge32 r1, r2, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_jsgt32_imm() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, -2 + or r1, r9 + jsgt32 r1, -1, +4 + mov32 r0, 1 + mov32 r1, 0 + jsgt32 r1, -1, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_jsgt32_reg() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, -2 + or r1, r9 + mov r2, -1 + jsgt32 r1, r2, +4 + mov32 r0, 1 + mov32 r1, 0 + jsgt32 r1, r2, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_jsle32_imm() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, -2 + or r1, r9 + jsle32 r1, -3, +5 + jsle32 r1, -1, +1 + exit + mov32 r0, 1 + jsle32 r1, -2, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_jsle32_reg() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, -2 + or r1, r9 + mov r2, -3 + mov32 r3, 0 + jsle32 r1, r2, +6 + jsle32 r1, r3, +1 + exit + mov32 r0, 1 + mov r1, r2 + jsle32 r1, r2, +1 + mov32 r0, 2 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_jslt32_imm() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, -2 + or r1, r9 + jslt32 r1, -3, +4 + jslt32 r1, -2, +3 + jslt32 r1, -1, +1 + exit + mov32 r0, 1 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_jslt32_reg() { + let prog = assemble( + " + mov r9, 1 + lsh r9, 32 + mov32 r0, 0 + mov32 r1, -2 + or r1, r9 + mov r2, -3 + mov r3, -1 + jslt32 r1, r1, +4 + jslt32 r1, r2, +3 + jslt32 r1, r3, +1 + exit + mov32 r0, 1 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_lddw() { + let prog = assemble( + "lddw r0, 0x1122334455667788 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1122334455667788); +} + +#[test] +fn test_vm_lddw2() { + let prog = assemble( + " + lddw r0, 0x0000000080000000 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x80000000); +} + +#[test] +fn test_vm_ldxb_all() { + let prog = assemble( + " + mov r0, r1 + ldxb r9, [r0+0] + lsh r9, 0 + ldxb r8, [r0+1] + lsh r8, 4 + ldxb r7, [r0+2] + lsh r7, 8 + ldxb r6, [r0+3] + lsh r6, 12 + ldxb r5, [r0+4] + lsh r5, 16 + ldxb r4, [r0+5] + lsh r4, 20 + ldxb r3, [r0+6] + lsh r3, 24 + ldxb r2, [r0+7] + lsh r2, 28 + ldxb r1, [r0+8] + lsh r1, 32 + ldxb r0, [r0+9] + lsh r0, 36 + or r0, r1 + or r0, r2 + or r0, r3 + or r0, r4 + or r0, r5 + or r0, r6 + or r0, r7 + or r0, r8 + or r0, r9 + exit", + ) + .unwrap(); + let mem = &mut [0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09]; + let vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program(mem).unwrap(), 0x9876543210); +} + +#[test] +fn test_vm_ldxb() { + let prog = assemble( + " + ldxb r0, [r1+2] + exit", + ) + .unwrap(); + let mem = &mut [0xaa, 0xbb, 0x11, 0xcc, 0xdd]; + let vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program(mem).unwrap(), 0x11); +} + +#[test] +fn test_vm_ldxdw() { + let prog = assemble( + " + ldxdw r0, [r1+2] + exit", + ) + .unwrap(); + let mem = &mut [ + 0xaa, 0xbb, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0xcc, 0xdd, + ]; + let vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program(mem).unwrap(), 0x8877665544332211); +} + +#[test] +fn test_vm_ldxh_all() { + let prog = assemble( + " + mov r0, r1 + ldxh r9, [r0+0] + be16 r9 + lsh r9, 0 + ldxh r8, [r0+2] + be16 r8 + lsh r8, 4 + ldxh r7, [r0+4] + be16 r7 + lsh r7, 8 + ldxh r6, [r0+6] + be16 r6 + lsh r6, 12 + ldxh r5, [r0+8] + be16 r5 + lsh r5, 16 + ldxh r4, [r0+10] + be16 r4 + lsh r4, 20 + ldxh r3, [r0+12] + be16 r3 + lsh r3, 24 + ldxh r2, [r0+14] + be16 r2 + lsh r2, 28 + ldxh r1, [r0+16] + be16 r1 + lsh r1, 32 + ldxh r0, [r0+18] + be16 r0 + lsh r0, 36 + or r0, r1 + or r0, r2 + or r0, r3 + or r0, r4 + or r0, r5 + or r0, r6 + or r0, r7 + or r0, r8 + or r0, r9 + exit", + ) + .unwrap(); + let mem = &mut [ + 0x00, 0x00, 0x00, 0x01, 0x00, 0x02, 0x00, 0x03, 0x00, 0x04, 0x00, 0x05, 0x00, 0x06, 0x00, + 0x07, 0x00, 0x08, 0x00, 0x09, + ]; + let vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program(mem).unwrap(), 0x9876543210); +} + +#[test] +fn test_vm_ldxh_all2() { + let prog = assemble( + " + mov r0, r1 + ldxh r9, [r0+0] + be16 r9 + ldxh r8, [r0+2] + be16 r8 + ldxh r7, [r0+4] + be16 r7 + ldxh r6, [r0+6] + be16 r6 + ldxh r5, [r0+8] + be16 r5 + ldxh r4, [r0+10] + be16 r4 + ldxh r3, [r0+12] + be16 r3 + ldxh r2, [r0+14] + be16 r2 + ldxh r1, [r0+16] + be16 r1 + ldxh r0, [r0+18] + be16 r0 + or r0, r1 + or r0, r2 + or r0, r3 + or r0, r4 + or r0, r5 + or r0, r6 + or r0, r7 + or r0, r8 + or r0, r9 + exit", + ) + .unwrap(); + let mem = &mut [ + 0x00, 0x01, 0x00, 0x02, 0x00, 0x04, 0x00, 0x08, 0x00, 0x10, 0x00, 0x20, 0x00, 0x40, 0x00, + 0x80, 0x01, 0x00, 0x02, 0x00, + ]; + let vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program(mem).unwrap(), 0x3ff); +} + +#[test] +fn test_vm_ldxh() { + let prog = assemble( + " + ldxh r0, [r1+2] + exit", + ) + .unwrap(); + let mem = &mut [0xaa, 0xbb, 0x11, 0x22, 0xcc, 0xdd]; + let vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program(mem).unwrap(), 0x2211); +} + +#[test] +fn test_vm_ldxh_same_reg() { + let prog = assemble( + " + mov r0, r1 + sth [r0], 0x1234 + ldxh r0, [r0] + exit", + ) + .unwrap(); + let mem = &mut [0xff, 0xff]; + let vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program(mem).unwrap(), 0x1234); +} + +#[test] +fn test_vm_ldxw_all() { + let prog = assemble( + " + mov r0, r1 + ldxw r9, [r0+0] + be32 r9 + ldxw r8, [r0+4] + be32 r8 + ldxw r7, [r0+8] + be32 r7 + ldxw r6, [r0+12] + be32 r6 + ldxw r5, [r0+16] + be32 r5 + ldxw r4, [r0+20] + be32 r4 + ldxw r3, [r0+24] + be32 r3 + ldxw r2, [r0+28] + be32 r2 + ldxw r1, [r0+32] + be32 r1 + ldxw r0, [r0+36] + be32 r0 + or r0, r1 + or r0, r2 + or r0, r3 + or r0, r4 + or r0, r5 + or r0, r6 + or r0, r7 + or r0, r8 + or r0, r9 + exit", + ) + .unwrap(); + let mem = &mut [ + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x08, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x08, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, + ]; + let vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program(mem).unwrap(), 0x030f0f); +} + +#[test] +fn test_vm_ldxw() { + let prog = assemble( + " + ldxw r0, [r1+2] + exit", + ) + .unwrap(); + let mem = &mut [0xaa, 0xbb, 0x11, 0x22, 0x33, 0x44, 0xcc, 0xdd]; + let vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program(mem).unwrap(), 0x44332211); +} + +#[test] +fn test_vm_le16() { + let prog = assemble( + " + ldxh r0, [r1] + le16 r0 + exit", + ) + .unwrap(); + let mem = &mut [0x22, 0x11]; + let vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program(mem).unwrap(), 0x1122); +} + +#[test] +fn test_vm_le32() { + let prog = assemble( + " + ldxw r0, [r1] + le32 r0 + exit", + ) + .unwrap(); + let mem = &mut [0x44, 0x33, 0x22, 0x11]; + let vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program(mem).unwrap(), 0x11223344); +} + +#[test] +fn test_vm_le64() { + let prog = assemble( + " + ldxdw r0, [r1] + le64 r0 + exit", + ) + .unwrap(); + let mem = &mut [0x88, 0x77, 0x66, 0x55, 0x44, 0x33, 0x22, 0x11]; + let vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program(mem).unwrap(), 0x1122334455667788); +} + +#[test] +fn test_vm_lsh_imm() { + let prog = assemble( + " + mov r0, 1 + lsh r0, 4 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x10); +} + +#[test] +fn test_vm_lsh_reg() { + let prog = assemble( + " + mov r0, 1 + mov r7, 4 + lsh r0, r7 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x10); +} + +#[test] +fn test_vm_lsh32_imm() { + let prog = assemble( + " + mov32 r0, 1 + lsh32 r0, 4 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x10); +} + +#[test] +fn test_vm_lsh32_reg() { + let prog = assemble( + " + mov32 r0, 1 + mov32 r7, 4 + lsh32 r0, r7 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x10); +} + +#[test] +fn test_vm_lsh_imm_overflow() { + let prog = assemble( + " + mov r0, 1 + lsh r0, 64 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_lsh_reg_overflow() { + let prog = assemble( + " + mov r0, 1 + mov r7, 64 + lsh r0, r7 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_lsh32_imm_overflow() { + let prog = assemble( + " + mov32 r0, 1 + lsh32 r0, 32 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_lsh32_reg_overflow() { + let prog = assemble( + " + mov32 r0, 1 + mov32 r7, 32 + lsh32 r0, r7 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_mod() { + let prog = assemble( + " + mov32 r0, 5748 + mod32 r0, 92 + mov32 r1, 13 + mod32 r0, r1 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x5); +} + +#[test] +fn test_vm_mod32() { + let prog = assemble( + " + lddw r0, 0x100000003 + mod32 r0, 3 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x0); +} + +#[test] +fn test_vm_mod64() { + let prog = assemble( + " + mov32 r0, -1316649930 + lsh r0, 32 + or r0, 0x100dc5c8 + mov32 r1, 0xdde263e + lsh r1, 32 + or r1, 0x3cbef7f3 + mod r0, r1 + mod r0, 0x658f1778 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x30ba5a04); +} + +#[test] +fn test_vm_mov() { + let prog = assemble( + " + mov32 r1, 1 + mov32 r0, r1 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_mul32_imm() { + let prog = assemble( + " + mov r0, 3 + mul32 r0, 4 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0xc); +} + +#[test] +fn test_vm_mul32_reg() { + let prog = assemble( + " + mov r0, 3 + mov r1, 4 + mul32 r0, r1 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0xc); +} + +#[test] +fn test_vm_mul32_reg_overflow() { + let prog = assemble( + " + mov r0, 0x40000001 + mov r1, 4 + mul32 r0, r1 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x4); +} + +#[test] +fn test_vm_mul64_imm() { + let prog = assemble( + " + mov r0, 0x40000001 + mul r0, 4 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x100000004); +} + +#[test] +fn test_vm_mul64_reg() { + let prog = assemble( + " + mov r0, 0x40000001 + mov r1, 4 + mul r0, r1 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x100000004); +} + +#[test] +fn test_vm_mul_loop() { + let prog = assemble( + " + mov r0, 0x7 + add r1, 0xa + lsh r1, 0x20 + rsh r1, 0x20 + jeq r1, 0x0, +4 + mov r0, 0x7 + mul r0, 0x7 + add r1, -1 + jne r1, 0x0, -3 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x75db9c97); +} + +#[test] +fn test_vm_neg64() { + let prog = assemble( + " + mov32 r0, 2 + neg r0 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0xfffffffffffffffe); +} + +#[test] +fn test_vm_neg() { + let prog = assemble( + " + mov32 r0, 2 + neg32 r0 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0xfffffffe); +} + +#[test] +fn test_vm_prime() { + let prog = assemble( + " + mov r1, 67 + mov r0, 0x1 + mov r2, 0x2 + jgt r1, 0x2, +4 + ja +10 + add r2, 0x1 + mov r0, 0x1 + jge r2, r1, +7 + mov r3, r1 + div r3, r2 + mul r3, r2 + mov r4, r1 + sub r4, r3 + mov r0, 0x0 + jne r4, 0x0, -10 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_rhs32() { + let prog = assemble( + " + xor r0, r0 + sub r0, 1 + rsh32 r0, 8 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x00ffffff); +} + +#[test] +fn test_vm_rsh_reg() { + let prog = assemble( + " + mov r0, 0x10 + mov r7, 4 + rsh r0, r7 + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x1); +} + +#[test] +fn test_vm_stack() { + let prog = assemble( + " + mov r1, 51 + stdw [r10-16], 0xab + stdw [r10-8], 0xcd + and r1, 1 + lsh r1, 3 + mov r2, r10 + add r2, r1 + ldxdw r0, [r2-16] + exit", + ) + .unwrap(); + let vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0xcd); +} + +#[test] +fn test_vm_stack2() { + let prog = assemble( + " + stb [r10-4], 0x01 + stb [r10-3], 0x02 + stb [r10-2], 0x03 + stb [r10-1], 0x04 + mov r1, r10 + mov r2, 0x4 + sub r1, r2 + call 1 + mov r1, 0 + ldxb r2, [r10-4] + ldxb r3, [r10-3] + ldxb r4, [r10-2] + ldxb r5, [r10-1] + call 0 + xor r0, 0x2a2a2a2a + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.register_helper(0, helpers::gather_bytes).unwrap(); + vm.register_helper(1, helpers::memfrob).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x01020304); +} + +#[test] +fn test_vm_stb() { + let prog = assemble( + " + stb [r1+2], 0x11 + ldxb r0, [r1+2] + exit", + ) + .unwrap(); + let mem = &mut [0xaa, 0xbb, 0xff, 0xcc, 0xdd]; + let vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program(mem).unwrap(), 0x11); +} + +#[test] +fn test_vm_stdw() { + let prog = assemble( + " + stdw [r1+2], 0x44332211 + ldxdw r0, [r1+2] + exit", + ) + .unwrap(); + let mem = &mut [ + 0xaa, 0xbb, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xcc, 0xdd, + ]; + let vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program(mem).unwrap(), 0x44332211); +} + +// If this case is not handled properly in check_mem(), then we may overflow when adding the +// context address and the offset, and make the thread panic with "attempt to add with overflow". +// Check that we panic with the expected out-of-bounds error. +#[test] +#[should_panic(expected = "Error: out of bounds memory store (insn #1)")] +fn test_vm_stdw_add_overflow() { + let prog = assemble( + " + stdw [r2-0x1], 0x44332211 + ldxw r0, [r1+2] + exit", + ) + .unwrap(); + let mem = &mut [ + 0xaa, 0xbb, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xcc, 0xdd, + ]; + let mut vm = rbpf::EbpfVmFixedMbuff::new(Some(&prog), 0x00, 0x10).unwrap(); + _ = vm.execute_program(mem).unwrap(); +} + +#[test] +fn test_vm_sth() { + let prog = assemble( + " + sth [r1+2], 0x2211 + ldxh r0, [r1+2] + exit", + ) + .unwrap(); + let mem = &mut [0xaa, 0xbb, 0xff, 0xff, 0xcc, 0xdd]; + let vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program(mem).unwrap(), 0x2211); +} + +#[test] +fn test_vm_string_stack() { + let prog = assemble( + " + mov r1, 0x78636261 + stxw [r10-8], r1 + mov r6, 0x0 + stxb [r10-4], r6 + stxb [r10-12], r6 + mov r1, 0x79636261 + stxw [r10-16], r1 + mov r1, r10 + add r1, -8 + mov r2, r1 + call 0x4 + mov r1, r0 + mov r0, 0x1 + lsh r1, 0x20 + rsh r1, 0x20 + jne r1, 0x0, +11 + mov r1, r10 + add r1, -8 + mov r2, r10 + add r2, -16 + call 0x4 + mov r1, r0 + lsh r1, 0x20 + rsh r1, 0x20 + mov r0, 0x1 + jeq r1, r6, +1 + mov r0, 0x0 + exit", + ) + .unwrap(); + let mut vm = rbpf::EbpfVmNoData::new(Some(&prog)).unwrap(); + vm.register_helper(4, helpers::strcmp).unwrap(); + assert_eq!(vm.execute_program().unwrap(), 0x0); +} + +#[test] +fn test_vm_stw() { + let prog = assemble( + " + stw [r1+2], 0x44332211 + ldxw r0, [r1+2] + exit", + ) + .unwrap(); + let mem = &mut [0xaa, 0xbb, 0xff, 0xff, 0xff, 0xff, 0xcc, 0xdd]; + let vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program(mem).unwrap(), 0x44332211); +} + +#[test] +fn test_vm_stxb() { + let prog = assemble( + " + mov32 r2, 0x11 + stxb [r1+2], r2 + ldxb r0, [r1+2] + exit", + ) + .unwrap(); + let mem = &mut [0xaa, 0xbb, 0xff, 0xcc, 0xdd]; + let vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program(mem).unwrap(), 0x11); +} + +#[test] +fn test_vm_stxb_all() { + let prog = assemble( + " + mov r0, 0xf0 + mov r2, 0xf2 + mov r3, 0xf3 + mov r4, 0xf4 + mov r5, 0xf5 + mov r6, 0xf6 + mov r7, 0xf7 + mov r8, 0xf8 + stxb [r1], r0 + stxb [r1+1], r2 + stxb [r1+2], r3 + stxb [r1+3], r4 + stxb [r1+4], r5 + stxb [r1+5], r6 + stxb [r1+6], r7 + stxb [r1+7], r8 + ldxdw r0, [r1] + be64 r0 + exit", + ) + .unwrap(); + let mem = &mut [0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff]; + let vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program(mem).unwrap(), 0xf0f2f3f4f5f6f7f8); +} + +#[test] +fn test_vm_stxb_all2() { + let prog = assemble( + " + mov r0, r1 + mov r1, 0xf1 + mov r9, 0xf9 + stxb [r0], r1 + stxb [r0+1], r9 + ldxh r0, [r0] + be16 r0 + exit", + ) + .unwrap(); + let mem = &mut [0xff, 0xff]; + let vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program(mem).unwrap(), 0xf1f9); +} + +#[test] +fn test_vm_stxb_chain() { + let prog = assemble( + " + mov r0, r1 + ldxb r9, [r0+0] + stxb [r0+1], r9 + ldxb r8, [r0+1] + stxb [r0+2], r8 + ldxb r7, [r0+2] + stxb [r0+3], r7 + ldxb r6, [r0+3] + stxb [r0+4], r6 + ldxb r5, [r0+4] + stxb [r0+5], r5 + ldxb r4, [r0+5] + stxb [r0+6], r4 + ldxb r3, [r0+6] + stxb [r0+7], r3 + ldxb r2, [r0+7] + stxb [r0+8], r2 + ldxb r1, [r0+8] + stxb [r0+9], r1 + ldxb r0, [r0+9] + exit", + ) + .unwrap(); + let mem = &mut [0x2a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]; + let vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program(mem).unwrap(), 0x2a); +} + +#[test] +fn test_vm_stxdw() { + let prog = assemble( + " + mov r2, -2005440939 + lsh r2, 32 + or r2, 0x44332211 + stxdw [r1+2], r2 + ldxdw r0, [r1+2] + exit", + ) + .unwrap(); + let mem = &mut [ + 0xaa, 0xbb, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xcc, 0xdd, + ]; + let vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program(mem).unwrap(), 0x8877665544332211); +} + +#[test] +fn test_vm_stxh() { + let prog = assemble( + " + mov32 r2, 0x2211 + stxh [r1+2], r2 + ldxh r0, [r1+2] + exit", + ) + .unwrap(); + let mem = &mut [0xaa, 0xbb, 0xff, 0xff, 0xcc, 0xdd]; + let vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program(mem).unwrap(), 0x2211); +} + +#[test] +fn test_vm_stxw() { + let prog = assemble( + " + mov32 r2, 0x44332211 + stxw [r1+2], r2 + ldxw r0, [r1+2] + exit", + ) + .unwrap(); + let mem = &mut [0xaa, 0xbb, 0xff, 0xff, 0xff, 0xff, 0xcc, 0xdd]; + let vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program(mem).unwrap(), 0x44332211); +} + +#[test] +fn test_vm_subnet() { + let prog = assemble( + " + mov r2, 0xe + ldxh r3, [r1+12] + jne r3, 0x81, +2 + mov r2, 0x12 + ldxh r3, [r1+16] + and r3, 0xffff + jne r3, 0x8, +5 + add r1, r2 + mov r0, 0x1 + ldxw r1, [r1+16] + and r1, 0xffffff + jeq r1, 0x1a8c0, +1 + mov r0, 0x0 + exit", + ) + .unwrap(); + let mem = &mut [ + 0x00, 0x00, 0xc0, 0x9f, 0xa0, 0x97, 0x00, 0xa0, 0xcc, 0x3b, 0xbf, 0xfa, 0x08, 0x00, 0x45, + 0x10, 0x00, 0x3c, 0x46, 0x3c, 0x40, 0x00, 0x40, 0x06, 0x73, 0x1c, 0xc0, 0xa8, 0x01, 0x02, + 0xc0, 0xa8, 0x01, 0x01, 0x06, 0x0e, 0x00, 0x17, 0x99, 0xc5, 0xa0, 0xec, 0x00, 0x00, 0x00, + 0x00, 0xa0, 0x02, 0x7d, 0x78, 0xe0, 0xa3, 0x00, 0x00, 0x02, 0x04, 0x05, 0xb4, 0x04, 0x02, + 0x08, 0x0a, 0x00, 0x9c, 0x27, 0x24, 0x00, 0x00, 0x00, 0x00, 0x01, 0x03, 0x03, 0x00, + ]; + let vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program(mem).unwrap(), 0x1); +} + +const PROG_TCP_PORT_80: [u8; 152] = [ + 0x71, 0x12, 0x0c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x71, 0x13, 0x0d, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x67, 0x03, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x4f, 0x23, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xb7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x55, 0x03, 0x0c, 0x00, 0x08, 0x00, 0x00, 0x00, + 0x71, 0x12, 0x17, 0x00, 0x00, 0x00, 0x00, 0x00, 0x55, 0x02, 0x0a, 0x00, 0x06, 0x00, 0x00, 0x00, + 0x71, 0x12, 0x0e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0x01, 0x00, 0x00, 0x0e, 0x00, 0x00, 0x00, + 0x57, 0x02, 0x00, 0x00, 0x0f, 0x00, 0x00, 0x00, 0x67, 0x02, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x0f, 0x21, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x69, 0x12, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x15, 0x02, 0x02, 0x00, 0x00, 0x50, 0x00, 0x00, 0x69, 0x11, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x55, 0x01, 0x01, 0x00, 0x00, 0x50, 0x00, 0x00, 0xb7, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +]; + +#[test] +fn test_vm_tcp_port80_match() { + let mem = &mut [ + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x00, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x08, 0x00, 0x45, + 0x00, 0x00, 0x56, 0x00, 0x01, 0x00, 0x00, 0x40, 0x06, 0xf9, 0x4d, 0xc0, 0xa8, 0x00, 0x01, + 0xc0, 0xa8, 0x00, 0x02, 0x27, 0x10, 0x00, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x50, 0x02, 0x20, 0x00, 0xc5, 0x18, 0x00, 0x00, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + ]; + let prog = &PROG_TCP_PORT_80; + let vm = rbpf::EbpfVmRaw::new(Some(prog)).unwrap(); + assert_eq!(vm.execute_program(mem).unwrap(), 0x1); +} + +#[test] +fn test_vm_tcp_port80_nomatch() { + let mem = &mut [ + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x00, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x08, 0x00, 0x45, + 0x00, 0x00, 0x56, 0x00, 0x01, 0x00, 0x00, 0x40, 0x06, 0xf9, 0x4d, 0xc0, 0xa8, 0x00, 0x01, + 0xc0, 0xa8, 0x00, 0x02, 0x00, 0x16, 0x27, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x51, 0x02, 0x20, 0x00, 0xc5, 0x18, 0x00, 0x00, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + ]; + let prog = &PROG_TCP_PORT_80; + let vm = rbpf::EbpfVmRaw::new(Some(prog)).unwrap(); + assert_eq!(vm.execute_program(mem).unwrap(), 0x0); +} + +#[test] +fn test_vm_tcp_port80_nomatch_ethertype() { + let mem = &mut [ + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x00, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x08, 0x01, 0x45, + 0x00, 0x00, 0x56, 0x00, 0x01, 0x00, 0x00, 0x40, 0x06, 0xf9, 0x4d, 0xc0, 0xa8, 0x00, 0x01, + 0xc0, 0xa8, 0x00, 0x02, 0x27, 0x10, 0x00, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x50, 0x02, 0x20, 0x00, 0xc5, 0x18, 0x00, 0x00, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + ]; + let prog = &PROG_TCP_PORT_80; + let vm = rbpf::EbpfVmRaw::new(Some(prog)).unwrap(); + assert_eq!(vm.execute_program(mem).unwrap(), 0x0); +} + +#[test] +fn test_vm_tcp_port80_nomatch_proto() { + let mem = &mut [ + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x00, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x08, 0x00, 0x45, + 0x00, 0x00, 0x56, 0x00, 0x01, 0x00, 0x00, 0x40, 0x11, 0xf9, 0x4d, 0xc0, 0xa8, 0x00, 0x01, + 0xc0, 0xa8, 0x00, 0x02, 0x27, 0x10, 0x00, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x50, 0x02, 0x20, 0x00, 0xc5, 0x18, 0x00, 0x00, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + ]; + let prog = &PROG_TCP_PORT_80; + let vm = rbpf::EbpfVmRaw::new(Some(prog)).unwrap(); + assert_eq!(vm.execute_program(mem).unwrap(), 0x0); +} + +#[test] +fn test_vm_tcp_sack_match() { + let mut mem = TCP_SACK_MATCH.to_vec(); + let prog = assemble(TCP_SACK_ASM).unwrap(); + let vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program(mem.as_mut_slice()).unwrap(), 0x1); +} + +#[test] +fn test_vm_tcp_sack_nomatch() { + let mut mem = TCP_SACK_NOMATCH.to_vec(); + let prog = assemble(TCP_SACK_ASM).unwrap(); + let vm = rbpf::EbpfVmRaw::new(Some(&prog)).unwrap(); + assert_eq!(vm.execute_program(mem.as_mut_slice()).unwrap(), 0x0); +} diff --git a/kernel/src/arch/riscv64/interrupt/handle.rs b/kernel/src/arch/riscv64/interrupt/handle.rs index faff690f..fc971679 100644 --- a/kernel/src/arch/riscv64/interrupt/handle.rs +++ b/kernel/src/arch/riscv64/interrupt/handle.rs @@ -3,12 +3,12 @@ //! 架构相关的处理逻辑参考： https://code.dragonos.org.cn/xref/linux-6.6.21/arch/riscv/kernel/traps.c use core::hint::spin_loop; -use log::error; +use log::{error, trace}; use system_error::SystemError; -use crate::{arch::syscall::syscall_handler, driver::irqchip::riscv_intc::riscv_intc_irq}; - use super::TrapFrame; +use crate::exception::ebreak::EBreak; +use crate::{arch::syscall::syscall_handler, driver::irqchip::riscv_intc::riscv_intc_irq}; type ExceptionHandler = fn(&mut TrapFrame) -> Result<(), SystemError>; @@ -93,11 +93,10 @@ fn do_trap_insn_illegal(_trap_frame: &mut TrapFrame) -> Result<(), SystemError> } /// 处理断点异常 #3 -fn do_trap_break(_trap_frame: &mut TrapFrame) -> Result<(), SystemError> { - error!("riscv64_do_irq: do_trap_break"); - loop { - spin_loop(); - } +fn do_trap_break(trap_frame: &mut TrapFrame) -> Result<(), SystemError> { + trace!("riscv64_do_irq: do_trap_break"); + // handle breakpoint + EBreak::handle(trap_frame) } /// 处理加载地址不对齐异常 #4 diff --git a/kernel/src/arch/riscv64/interrupt/mod.rs b/kernel/src/arch/riscv64/interrupt/mod.rs index dc6d7893..bc366098 100644 --- a/kernel/src/arch/riscv64/interrupt/mod.rs +++ b/kernel/src/arch/riscv64/interrupt/mod.rs @@ -1,3 +1,5 @@ +use core::any::Any; +use kprobe::ProbeArgs; use riscv::register::{scause::Scause, sstatus::Sstatus}; use system_error::SystemError; @@ -160,4 +162,21 @@ impl TrapFrame { pub fn set_return_value(&mut self, value: usize) { self.a0 = value; } + + /// 设置当前的程序计数器 + pub fn set_pc(&mut self, pc: usize) { + self.epc = pc; + } +} + +impl ProbeArgs for TrapFrame { + fn as_any(&self) -> &dyn Any { + self + } + fn break_address(&self) -> usize { + self.epc + } + fn debug_address(&self) -> usize { + self.epc + } } diff --git a/kernel/src/arch/riscv64/kprobe.rs b/kernel/src/arch/riscv64/kprobe.rs new file mode 100644 index 00000000..960b06cd --- /dev/null +++ b/kernel/src/arch/riscv64/kprobe.rs @@ -0,0 +1,85 @@ +use crate::arch::interrupt::TrapFrame; + +pub fn setup_single_step(frame: &mut TrapFrame, step_addr: usize) { + frame.set_pc(step_addr); +} + +pub fn clear_single_step(frame: &mut TrapFrame, return_addr: usize) { + frame.set_pc(return_addr); +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct KProbeContext { + pub pc: usize, + pub ra: usize, + pub sp: usize, + pub gp: usize, + pub tp: usize, + pub t0: usize, + pub t1: usize, + pub t2: usize, + pub s0: usize, + pub s1: usize, + pub a0: usize, + pub a1: usize, + pub a2: usize, + pub a3: usize, + pub a4: usize, + pub a5: usize, + pub a6: usize, + pub a7: usize, + pub s2: usize, + pub s3: usize, + pub s4: usize, + pub s5: usize, + pub s6: usize, + pub s7: usize, + pub s8: usize, + pub s9: usize, + pub s10: usize, + pub s11: usize, + pub t3: usize, + pub t4: usize, + pub t5: usize, + pub t6: usize, +} + +impl From<&TrapFrame> for KProbeContext { + fn from(trap_frame: &TrapFrame) -> Self { + Self { + pc: trap_frame.epc, + ra: trap_frame.ra, + sp: trap_frame.sp, + gp: trap_frame.gp, + tp: trap_frame.tp, + t0: trap_frame.t0, + t1: trap_frame.t1, + t2: trap_frame.t2, + s0: trap_frame.s0, + s1: trap_frame.s1, + a0: trap_frame.a0, + a1: trap_frame.a1, + a2: trap_frame.a2, + a3: trap_frame.a3, + a4: trap_frame.a4, + a5: trap_frame.a5, + a6: trap_frame.a6, + a7: trap_frame.a7, + s2: trap_frame.s2, + s3: trap_frame.s3, + s4: trap_frame.s4, + s5: trap_frame.s5, + s6: trap_frame.s6, + s7: trap_frame.s7, + s8: trap_frame.s8, + s9: trap_frame.s9, + s10: trap_frame.s10, + s11: trap_frame.s11, + t3: trap_frame.t3, + t4: trap_frame.t4, + t5: trap_frame.t5, + t6: trap_frame.t6, + } + } +} diff --git a/kernel/src/arch/riscv64/mod.rs b/kernel/src/arch/riscv64/mod.rs index e7c2fe22..a34b99a7 100644 --- a/kernel/src/arch/riscv64/mod.rs +++ b/kernel/src/arch/riscv64/mod.rs @@ -5,6 +5,7 @@ pub mod elf; pub mod init; pub mod interrupt; pub mod ipc; +pub mod kprobe; mod kvm; pub mod mm; pub mod msi; diff --git a/kernel/src/arch/x86_64/interrupt/mod.rs b/kernel/src/arch/x86_64/interrupt/mod.rs index 0eb4a88a..e83566ca 100644 --- a/kernel/src/arch/x86_64/interrupt/mod.rs +++ b/kernel/src/arch/x86_64/interrupt/mod.rs @@ -4,11 +4,12 @@ pub mod ipi; pub mod msi; pub mod trap; +use core::any::Any; use core::{ arch::asm, sync::atomic::{compiler_fence, Ordering}, }; - +use kprobe::ProbeArgs; use log::error; use system_error::SystemError; @@ -177,4 +178,21 @@ impl TrapFrame { pub fn is_from_user(&self) -> bool { return (self.cs & 0x3) != 0; } + /// 设置当前的程序计数器 + pub fn set_pc(&mut self, pc: usize) { + self.rip = pc as u64; + } +} + +impl ProbeArgs for TrapFrame { + fn as_any(&self) -> &dyn Any { + self + } + fn break_address(&self) -> usize { + (self.rip - 1) as usize + } + + fn debug_address(&self) -> usize { + self.rip as usize + } } diff --git a/kernel/src/arch/x86_64/interrupt/trap.rs b/kernel/src/arch/x86_64/interrupt/trap.rs index 9cd31fa9..6cb3971a 100644 --- a/kernel/src/arch/x86_64/interrupt/trap.rs +++ b/kernel/src/arch/x86_64/interrupt/trap.rs @@ -1,6 +1,12 @@ -use log::{error, warn}; +use log::{error, trace, warn}; use system_error::SystemError; +use super::{ + entry::{set_intr_gate, set_system_trap_gate}, + TrapFrame, +}; +use crate::exception::debug::DebugException; +use crate::exception::ebreak::EBreak; use crate::{ arch::{CurrentIrqArch, MMArch}, exception::InterruptArch, @@ -9,11 +15,6 @@ use crate::{ smp::core::smp_get_processor_id, }; -use super::{ - entry::{set_intr_gate, set_system_trap_gate}, - TrapFrame, -}; - extern "C" { fn trap_divide_error(); fn trap_debug(); @@ -125,8 +126,8 @@ unsafe extern "C" fn do_divide_error(regs: &'static TrapFrame, error_code: u64) /// 处理调试异常 1 #DB #[no_mangle] -unsafe extern "C" fn do_debug(regs: &'static TrapFrame, error_code: u64) { - error!( +unsafe extern "C" fn do_debug(regs: &'static mut TrapFrame, error_code: u64) { + trace!( "do_debug(1), \tError code: {:#x},\trsp: {:#x},\trip: {:#x},\t CPU: {}, \tpid: {:?}", error_code, regs.rsp, @@ -134,7 +135,7 @@ unsafe extern "C" fn do_debug(regs: &'static TrapFrame, error_code: u64) { smp_get_processor_id().data(), ProcessManager::current_pid() ); - panic!("Debug Exception"); + DebugException::handle(regs).unwrap(); } /// 处理NMI中断 2 NMI @@ -153,8 +154,8 @@ unsafe extern "C" fn do_nmi(regs: &'static TrapFrame, error_code: u64) { /// 处理断点异常 3 #BP #[no_mangle] -unsafe extern "C" fn do_int3(regs: &'static TrapFrame, error_code: u64) { - error!( +unsafe extern "C" fn do_int3(regs: &'static mut TrapFrame, error_code: u64) { + trace!( "do_int3(3), \tError code: {:#x},\trsp: {:#x},\trip: {:#x},\t CPU: {}, \tpid: {:?}", error_code, regs.rsp, @@ -162,7 +163,7 @@ unsafe extern "C" fn do_int3(regs: &'static TrapFrame, error_code: u64) { smp_get_processor_id().data(), ProcessManager::current_pid() ); - panic!("Int3"); + EBreak::handle(regs).unwrap(); } /// 处理溢出异常 4 #OF diff --git a/kernel/src/arch/x86_64/kprobe.rs b/kernel/src/arch/x86_64/kprobe.rs new file mode 100644 index 00000000..e998aa99 --- /dev/null +++ b/kernel/src/arch/x86_64/kprobe.rs @@ -0,0 +1,65 @@ +use crate::arch::interrupt::TrapFrame; + +pub fn setup_single_step(frame: &mut TrapFrame, step_addr: usize) { + frame.rflags |= 0x100; + frame.set_pc(step_addr); +} + +pub fn clear_single_step(frame: &mut TrapFrame, return_addr: usize) { + frame.rflags &= !0x100; + frame.set_pc(return_addr); +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct KProbeContext { + pub r15: ::core::ffi::c_ulong, + pub r14: ::core::ffi::c_ulong, + pub r13: ::core::ffi::c_ulong, + pub r12: ::core::ffi::c_ulong, + pub rbp: ::core::ffi::c_ulong, + pub rbx: ::core::ffi::c_ulong, + pub r11: ::core::ffi::c_ulong, + pub r10: ::core::ffi::c_ulong, + pub r9: ::core::ffi::c_ulong, + pub r8: ::core::ffi::c_ulong, + pub rax: ::core::ffi::c_ulong, + pub rcx: ::core::ffi::c_ulong, + pub rdx: ::core::ffi::c_ulong, + pub rsi: ::core::ffi::c_ulong, + pub rdi: ::core::ffi::c_ulong, + pub orig_rax: ::core::ffi::c_ulong, + pub rip: ::core::ffi::c_ulong, + pub cs: ::core::ffi::c_ulong, + pub eflags: ::core::ffi::c_ulong, + pub rsp: ::core::ffi::c_ulong, + pub ss: ::core::ffi::c_ulong, +} + +impl From<&TrapFrame> for KProbeContext { + fn from(trap_frame: &TrapFrame) -> Self { + Self { + r15: trap_frame.r15, + r14: trap_frame.r14, + r13: trap_frame.r13, + r12: trap_frame.r12, + rbp: trap_frame.rbp, + rbx: trap_frame.rbx, + r11: trap_frame.r11, + r10: trap_frame.r10, + r9: trap_frame.r9, + r8: trap_frame.r8, + rax: trap_frame.rax, + rcx: trap_frame.rcx, + rdx: trap_frame.rdx, + rsi: trap_frame.rsi, + rdi: trap_frame.rdi, + orig_rax: 0, + rip: trap_frame.rip, + cs: trap_frame.cs, + eflags: trap_frame.rflags, + rsp: trap_frame.rsp, + ss: trap_frame.ss, + } + } +} diff --git a/kernel/src/arch/x86_64/mod.rs b/kernel/src/arch/x86_64/mod.rs index ff95ab1d..2bd97fe0 100644 --- a/kernel/src/arch/x86_64/mod.rs +++ b/kernel/src/arch/x86_64/mod.rs @@ -8,6 +8,7 @@ pub mod fpu; pub mod init; pub mod interrupt; pub mod ipc; +pub mod kprobe; pub mod kvm; pub mod libs; pub mod mm; diff --git a/kernel/src/bpf/helper/consts.rs b/kernel/src/bpf/helper/consts.rs new file mode 100644 index 00000000..69bce7d6 --- /dev/null +++ b/kernel/src/bpf/helper/consts.rs @@ -0,0 +1,11 @@ +pub const HELPER_MAP_LOOKUP_ELEM: u32 = 1; +pub const HELPER_MAP_UPDATE_ELEM: u32 = 2; +pub const HELPER_MAP_DELETE_ELEM: u32 = 3; +pub const HELPER_MAP_FOR_EACH_ELEM: u32 = 164; +pub const HELPER_MAP_LOOKUP_PERCPU_ELEM: u32 = 195; +pub const HELPER_PERF_EVENT_OUTPUT: u32 = 25; +pub const HELPER_BPF_PROBE_READ: u32 = 4; +pub const HELPER_TRACE_PRINTF: u32 = 6; +pub const HELPER_MAP_PUSH_ELEM: u32 = 87; +pub const HELPER_MAP_POP_ELEM: u32 = 88; +pub const HELPER_MAP_PEEK_ELEM: u32 = 89; diff --git a/kernel/src/bpf/helper/mod.rs b/kernel/src/bpf/helper/mod.rs new file mode 100644 index 00000000..2f951793 --- /dev/null +++ b/kernel/src/bpf/helper/mod.rs @@ -0,0 +1,340 @@ +mod consts; +mod print; + +use crate::bpf::helper::print::trace_printf; +use crate::bpf::map::{BpfCallBackFn, BpfMap}; +use crate::include::bindings::linux_bpf::BPF_F_CURRENT_CPU; +use crate::libs::lazy_init::Lazy; +use crate::smp::core::smp_get_processor_id; +use alloc::{collections::BTreeMap, sync::Arc}; +use core::ffi::c_void; +use system_error::SystemError; + +type RawBPFHelperFn = fn(u64, u64, u64, u64, u64) -> u64; +type Result = core::result::Result; +macro_rules! define_func { + ($name:ident) => { + core::mem::transmute::($name as usize) + }; +} + +/// See https://ebpf-docs.dylanreimerink.nl/linux/helper-function/bpf_map_lookup_elem/ +unsafe fn raw_map_lookup_elem(map: *mut c_void, key: *const c_void) -> *const c_void { + let map = Arc::from_raw(map as *const BpfMap); + let key_size = map.key_size(); + let key = core::slice::from_raw_parts(key as *const u8, key_size); + let value = map_lookup_elem(&map, key); + // log::info!(": {:x?}", value); + // warning: We need to keep the map alive, so we don't drop it here. + let _ = Arc::into_raw(map); + match value { + Ok(Some(value)) => value as *const c_void, + _ => core::ptr::null_mut(), + } +} + +pub fn map_lookup_elem(map: &Arc, key: &[u8]) -> Result> { + let mut binding = map.inner_map().lock(); + let value = binding.lookup_elem(key); + match value { + Ok(Some(value)) => Ok(Some(value.as_ptr())), + _ => Ok(None), + } +} + +/// See https://ebpf-docs.dylanreimerink.nl/linux/helper-function/bpf_perf_event_output/ +/// +/// See https://man7.org/linux/man-pages/man7/bpf-helpers.7.html +unsafe fn raw_perf_event_output( + ctx: *mut c_void, + map: *mut c_void, + flags: u64, + data: *mut c_void, + size: u64, +) -> i64 { + // log::info!(": {:x?}", data); + let map = Arc::from_raw(map as *const BpfMap); + let data = core::slice::from_raw_parts(data as *const u8, size as usize); + let res = perf_event_output(ctx, &map, flags, data); + // warning: We need to keep the map alive, so we don't drop it here. + let _ = Arc::into_raw(map); + match res { + Ok(_) => 0, + Err(e) => e as i64, + } +} + +pub fn perf_event_output( + ctx: *mut c_void, + map: &Arc, + flags: u64, + data: &[u8], +) -> Result<()> { + let mut binding = map.inner_map().lock(); + let index = flags as u32; + let flags = (flags >> 32) as u32; + let key = if index == BPF_F_CURRENT_CPU as u32 { + smp_get_processor_id().data() + } else { + index + }; + let fd = binding + .lookup_elem(&key.to_ne_bytes())? + .ok_or(SystemError::ENOENT)?; + let fd = u32::from_ne_bytes(fd.try_into().map_err(|_| SystemError::EINVAL)?); + crate::perf::perf_event_output(ctx, fd as usize, flags, data)?; + Ok(()) +} + +/// See https://ebpf-docs.dylanreimerink.nl/linux/helper-function/bpf_probe_read/ +fn raw_bpf_probe_read(dst: *mut c_void, size: u32, unsafe_ptr: *const c_void) -> i64 { + log::info!( + "raw_bpf_probe_read, dst:{:x}, size:{}, unsafe_ptr: {:x}", + dst as usize, + size, + unsafe_ptr as usize + ); + let (dst, src) = unsafe { + let dst = core::slice::from_raw_parts_mut(dst as *mut u8, size as usize); + let src = core::slice::from_raw_parts(unsafe_ptr as *const u8, size as usize); + (dst, src) + }; + let res = bpf_probe_read(dst, src); + match res { + Ok(_) => 0, + Err(e) => e as i64, + } +} + +/// For tracing programs, safely attempt to read size +/// bytes from kernel space address unsafe_ptr and +/// store the data in dst. +pub fn bpf_probe_read(dst: &mut [u8], src: &[u8]) -> Result<()> { + log::info!("bpf_probe_read: len: {}", dst.len()); + dst.copy_from_slice(src); + Ok(()) +} + +unsafe fn raw_map_update_elem( + map: *mut c_void, + key: *const c_void, + value: *const c_void, + flags: u64, +) -> i64 { + let map = Arc::from_raw(map as *const BpfMap); + let key_size = map.key_size(); + let value_size = map.value_size(); + // log::info!(": flags: {:x?}", flags); + let key = core::slice::from_raw_parts(key as *const u8, key_size); + let value = core::slice::from_raw_parts(value as *const u8, value_size); + let res = map_update_elem(&map, key, value, flags); + let _ = Arc::into_raw(map); + match res { + Ok(_) => 0, + Err(e) => e as _, + } +} + +pub fn map_update_elem(map: &Arc, key: &[u8], value: &[u8], flags: u64) -> Result<()> { + let mut binding = map.inner_map().lock(); + let value = binding.update_elem(key, value, flags); + value +} + +/// Delete entry with key from map. +/// +/// The delete map element helper call is used to delete values from maps. +unsafe fn raw_map_delete_elem(map: *mut c_void, key: *const c_void) -> i64 { + let map = Arc::from_raw(map as *const BpfMap); + let key_size = map.key_size(); + let key = core::slice::from_raw_parts(key as *const u8, key_size); + let res = map_delete_elem(&map, key); + let _ = Arc::into_raw(map); + match res { + Ok(_) => 0, + Err(e) => e as i64, + } +} + +pub fn map_delete_elem(map: &Arc, key: &[u8]) -> Result<()> { + let mut binding = map.inner_map().lock(); + let value = binding.delete_elem(key); + value +} + +/// For each element in map, call callback_fn function with map, callback_ctx and other map-specific +/// parameters. The callback_fn should be a static function and the callback_ctx should be a pointer +/// to the stack. The flags is used to control certain aspects of the helper. Currently, the flags must +/// be 0. +/// +/// The following are a list of supported map types and their respective expected callback signatures: +/// - BPF_MAP_TYPE_HASH +/// - BPF_MAP_TYPE_PERCPU_HASH +/// - BPF_MAP_TYPE_LRU_HASH +/// - BPF_MAP_TYPE_LRU_PERCPU_HASH +/// - BPF_MAP_TYPE_ARRAY +/// - BPF_MAP_TYPE_PERCPU_ARRAY +/// +/// `long (*callback_fn)(struct bpf_map *map, const void key, void *value, void *ctx);` +/// +/// For per_cpu maps, the map_value is the value on the cpu where the bpf_prog is running. +unsafe fn raw_map_for_each_elem( + map: *mut c_void, + cb: *const c_void, + ctx: *const c_void, + flags: u64, +) -> i64 { + let map = Arc::from_raw(map as *const BpfMap); + let cb = *core::mem::transmute::<*const c_void, *const BpfCallBackFn>(cb); + let res = map_for_each_elem(&map, cb, ctx as _, flags); + let _ = Arc::into_raw(map); + match res { + Ok(v) => v as i64, + Err(e) => e as i64, + } +} + +pub fn map_for_each_elem( + map: &Arc, + cb: BpfCallBackFn, + ctx: *const u8, + flags: u64, +) -> Result { + let mut binding = map.inner_map().lock(); + let value = binding.for_each_elem(cb, ctx, flags); + value +} + +/// Perform a lookup in percpu map for an entry associated to key on cpu. +/// +/// See https://ebpf-docs.dylanreimerink.nl/linux/helper-function/bpf_map_lookup_percpu_elem/ +unsafe fn raw_map_lookup_percpu_elem( + map: *mut c_void, + key: *const c_void, + cpu: u32, +) -> *const c_void { + let map = Arc::from_raw(map as *const BpfMap); + let key_size = map.key_size(); + let key = core::slice::from_raw_parts(key as *const u8, key_size); + let value = map_lookup_percpu_elem(&map, key, cpu); + // warning: We need to keep the map alive, so we don't drop it here. + let _ = Arc::into_raw(map); + match value { + Ok(Some(value)) => value as *const c_void, + _ => core::ptr::null_mut(), + } +} + +pub fn map_lookup_percpu_elem( + map: &Arc, + key: &[u8], + cpu: u32, +) -> Result> { + let mut binding = map.inner_map().lock(); + let value = binding.lookup_percpu_elem(key, cpu); + match value { + Ok(Some(value)) => Ok(Some(value.as_ptr())), + _ => Ok(None), + } +} +/// Push an element value in map. +/// +/// See https://ebpf-docs.dylanreimerink.nl/linux/helper-function/bpf_map_push_elem/ +unsafe fn raw_map_push_elem(map: *mut c_void, value: *const c_void, flags: u64) -> i64 { + let map = Arc::from_raw(map as *const BpfMap); + let value_size = map.value_size(); + let value = core::slice::from_raw_parts(value as *const u8, value_size); + let res = map_push_elem(&map, value, flags); + let _ = Arc::into_raw(map); + match res { + Ok(_) => 0, + Err(e) => e as i64, + } +} + +pub fn map_push_elem(map: &Arc, value: &[u8], flags: u64) -> Result<()> { + let mut binding = map.inner_map().lock(); + let value = binding.push_elem(value, flags); + value +} + +/// Pop an element from map. +/// +/// See https://ebpf-docs.dylanreimerink.nl/linux/helper-function/bpf_map_pop_elem/ +unsafe fn raw_map_pop_elem(map: *mut c_void, value: *mut c_void) -> i64 { + let map = Arc::from_raw(map as *const BpfMap); + let value_size = map.value_size(); + let value = core::slice::from_raw_parts_mut(value as *mut u8, value_size); + let res = map_pop_elem(&map, value); + let _ = Arc::into_raw(map); + match res { + Ok(_) => 0, + Err(e) => e as i64, + } +} + +pub fn map_pop_elem(map: &Arc, value: &mut [u8]) -> Result<()> { + let mut binding = map.inner_map().lock(); + let value = binding.pop_elem(value); + value +} + +/// Get an element from map without removing it. +/// +/// See https://ebpf-docs.dylanreimerink.nl/linux/helper-function/bpf_map_peek_elem/ +unsafe fn raw_map_peek_elem(map: *mut c_void, value: *mut c_void) -> i64 { + let map = Arc::from_raw(map as *const BpfMap); + let value_size = map.value_size(); + let value = core::slice::from_raw_parts_mut(value as *mut u8, value_size); + let res = map_peek_elem(&map, value); + let _ = Arc::into_raw(map); + match res { + Ok(_) => 0, + Err(e) => e as i64, + } +} + +pub fn map_peek_elem(map: &Arc, value: &mut [u8]) -> Result<()> { + let binding = map.inner_map().lock(); + let value = binding.peek_elem(value); + value +} + +pub static BPF_HELPER_FUN_SET: Lazy> = Lazy::new(); + +/// Initialize the helper functions. +pub fn init_helper_functions() { + use consts::*; + let mut map = BTreeMap::new(); + unsafe { + // Map helpers::Generic map helpers + map.insert(HELPER_MAP_LOOKUP_ELEM, define_func!(raw_map_lookup_elem)); + map.insert(HELPER_MAP_UPDATE_ELEM, define_func!(raw_map_update_elem)); + map.insert(HELPER_MAP_DELETE_ELEM, define_func!(raw_map_delete_elem)); + map.insert( + HELPER_MAP_FOR_EACH_ELEM, + define_func!(raw_map_for_each_elem), + ); + map.insert( + HELPER_MAP_LOOKUP_PERCPU_ELEM, + define_func!(raw_map_lookup_percpu_elem), + ); + // map.insert(93,define_func!(raw_bpf_spin_lock); + // map.insert(94,define_func!(raw_bpf_spin_unlock); + // Map helpers::Perf event array helpers + map.insert( + HELPER_PERF_EVENT_OUTPUT, + define_func!(raw_perf_event_output), + ); + // Probe and trace helpers::Memory helpers + map.insert(HELPER_BPF_PROBE_READ, define_func!(raw_bpf_probe_read)); + // Print helpers + map.insert(HELPER_TRACE_PRINTF, define_func!(trace_printf)); + + // Map helpers::Queue and stack helpers + map.insert(HELPER_MAP_PUSH_ELEM, define_func!(raw_map_push_elem)); + map.insert(HELPER_MAP_POP_ELEM, define_func!(raw_map_pop_elem)); + map.insert(HELPER_MAP_PEEK_ELEM, define_func!(raw_map_peek_elem)); + } + BPF_HELPER_FUN_SET.init(map); +} diff --git a/kernel/src/bpf/helper/print.rs b/kernel/src/bpf/helper/print.rs new file mode 100644 index 00000000..1e6c4e73 --- /dev/null +++ b/kernel/src/bpf/helper/print.rs @@ -0,0 +1,25 @@ +use core::{ + ffi::{c_char, c_int}, + fmt::Write, +}; + +use printf_compat::{format, output}; + +/// Printf according to the format string, function will return the number of bytes written(including '\0') +pub unsafe extern "C" fn printf(w: &mut impl Write, str: *const c_char, mut args: ...) -> c_int { + let bytes_written = format(str as _, args.as_va_list(), output::fmt_write(w)); + bytes_written + 1 +} + +struct TerminalOut; +impl Write for TerminalOut { + fn write_str(&mut self, s: &str) -> core::fmt::Result { + print!("{}", s); + Ok(()) + } +} + +/// See https://ebpf-docs.dylanreimerink.nl/linux/helper-function/bpf_trace_printk/ +pub fn trace_printf(fmt_ptr: u64, _fmt_len: u64, arg3: u64, arg4: u64, arg5: u64) -> u64 { + unsafe { printf(&mut TerminalOut, fmt_ptr as _, arg3, arg4, arg5) as u64 } +} diff --git a/kernel/src/bpf/map/array_map.rs b/kernel/src/bpf/map/array_map.rs new file mode 100644 index 00000000..a5b48d64 --- /dev/null +++ b/kernel/src/bpf/map/array_map.rs @@ -0,0 +1,283 @@ +//! BPF_MAP_TYPE_ARRAY and BPF_MAP_TYPE_PERCPU_ARRAY +//! +//! +//! See https://docs.kernel.org/bpf/map_array.html + +use super::super::Result; +use crate::bpf::map::util::round_up; +use crate::bpf::map::{BpfCallBackFn, BpfMapCommonOps, BpfMapMeta}; +use crate::mm::percpu::{PerCpu, PerCpuVar}; +use crate::smp::cpu::{smp_cpu_manager, ProcessorId}; +use alloc::{vec, vec::Vec}; +use core::{ + fmt::{Debug, Formatter}, + ops::{Index, IndexMut}, +}; +use log::info; +use system_error::SystemError; + +/// The array map type is a generic map type with no restrictions on the structure of the value. +/// Like a normal array, the array map has a numeric key starting at 0 and incrementing. +/// +/// See https://ebpf-docs.dylanreimerink.nl/linux/map-type/BPF_MAP_TYPE_ARRAY/ +#[derive(Debug)] +pub struct ArrayMap { + max_entries: u32, + data: ArrayMapData, +} + +struct ArrayMapData { + elem_size: u32, + /// The data is stored in a Vec with the size of elem_size * max_entries. + data: Vec, +} + +impl Debug for ArrayMapData { + fn fmt(&self, f: &mut Formatter<'_>) -> core::fmt::Result { + f.debug_struct("ArrayMapData") + .field("elem_size", &self.elem_size) + .field("data_len", &self.data.len()) + .finish() + } +} + +impl ArrayMapData { + pub fn new(elem_size: u32, max_entries: u32) -> Self { + debug_assert!(elem_size % 8 == 0); + let total_size = elem_size * max_entries; + let data = vec![0; total_size as usize]; + ArrayMapData { elem_size, data } + } +} + +impl Index for ArrayMapData { + type Output = [u8]; + fn index(&self, index: u32) -> &Self::Output { + let start = index * self.elem_size; + &self.data[start as usize..(start + self.elem_size) as usize] + } +} + +impl IndexMut for ArrayMapData { + fn index_mut(&mut self, index: u32) -> &mut Self::Output { + let start = index * self.elem_size; + &mut self.data[start as usize..(start + self.elem_size) as usize] + } +} + +impl ArrayMap { + pub fn new(attr: &BpfMapMeta) -> Result { + if attr.value_size == 0 || attr.max_entries == 0 || attr.key_size != 4 { + return Err(SystemError::EINVAL); + } + let elem_size = round_up(attr.value_size as usize, 8); + let data = ArrayMapData::new(elem_size as u32, attr.max_entries); + Ok(ArrayMap { + max_entries: attr.max_entries, + data, + }) + } +} + +impl BpfMapCommonOps for ArrayMap { + fn lookup_elem(&mut self, key: &[u8]) -> Result> { + if key.len() != 4 { + return Err(SystemError::EINVAL); + } + let index = u32::from_ne_bytes(key.try_into().map_err(|_| SystemError::EINVAL)?); + if index >= self.max_entries { + return Err(SystemError::EINVAL); + } + let val = self.data.index(index); + Ok(Some(val)) + } + fn update_elem(&mut self, key: &[u8], value: &[u8], _flags: u64) -> Result<()> { + if key.len() != 4 { + return Err(SystemError::EINVAL); + } + let index = u32::from_ne_bytes(key.try_into().map_err(|_| SystemError::EINVAL)?); + if index >= self.max_entries { + return Err(SystemError::EINVAL); + } + if value.len() > self.data.elem_size as usize { + return Err(SystemError::EINVAL); + } + let old_value = self.data.index_mut(index); + old_value[..value.len()].copy_from_slice(value); + Ok(()) + } + /// For ArrayMap, delete_elem is not supported. + fn delete_elem(&mut self, _key: &[u8]) -> Result<()> { + Err(SystemError::EINVAL) + } + fn for_each_elem(&mut self, cb: BpfCallBackFn, ctx: *const u8, flags: u64) -> Result { + if flags != 0 { + return Err(SystemError::EINVAL); + } + let mut total_used = 0; + for i in 0..self.max_entries { + let key = i.to_ne_bytes(); + let value = self.data.index(i); + total_used += 1; + let res = cb(&key, value, ctx); + // return value: 0 - continue, 1 - stop and return + if res != 0 { + break; + } + } + Ok(total_used) + } + + fn lookup_and_delete_elem(&mut self, _key: &[u8], _value: &mut [u8]) -> Result<()> { + Err(SystemError::EINVAL) + } + + fn get_next_key(&self, key: Option<&[u8]>, next_key: &mut [u8]) -> Result<()> { + if let Some(key) = key { + if key.len() != 4 { + return Err(SystemError::EINVAL); + } + let index = u32::from_ne_bytes(key.try_into().map_err(|_| SystemError::EINVAL)?); + if index == self.max_entries - 1 { + return Err(SystemError::ENOENT); + } + let next_index = index + 1; + next_key.copy_from_slice(&next_index.to_ne_bytes()); + } else { + next_key.copy_from_slice(&0u32.to_ne_bytes()); + } + Ok(()) + } + + fn freeze(&self) -> Result<()> { + info!("fake freeze done for ArrayMap"); + Ok(()) + } + fn first_value_ptr(&self) -> Result<*const u8> { + Ok(self.data.data.as_ptr()) + } +} + +/// This is the per-CPU variant of the [ArrayMap] map type. +/// +/// See https://ebpf-docs.dylanreimerink.nl/linux/map-type/BPF_MAP_TYPE_PERCPU_ARRAY/ +pub struct PerCpuArrayMap { + per_cpu_data: PerCpuVar, +} + +impl Debug for PerCpuArrayMap { + fn fmt(&self, f: &mut Formatter<'_>) -> core::fmt::Result { + f.debug_struct("PerCpuArrayMap") + .field("data", &self.per_cpu_data) + .finish() + } +} + +impl PerCpuArrayMap { + pub fn new(attr: &BpfMapMeta) -> Result { + let num_cpus = PerCpu::MAX_CPU_NUM; + let mut data = Vec::with_capacity(num_cpus as usize); + for _ in 0..num_cpus { + let array_map = ArrayMap::new(attr)?; + data.push(array_map); + } + let per_cpu_data = PerCpuVar::new(data).ok_or(SystemError::EINVAL)?; + Ok(PerCpuArrayMap { per_cpu_data }) + } +} + +impl BpfMapCommonOps for PerCpuArrayMap { + fn lookup_elem(&mut self, key: &[u8]) -> Result> { + self.per_cpu_data.get_mut().lookup_elem(key) + } + fn update_elem(&mut self, key: &[u8], value: &[u8], flags: u64) -> Result<()> { + self.per_cpu_data.get_mut().update_elem(key, value, flags) + } + fn delete_elem(&mut self, key: &[u8]) -> Result<()> { + self.per_cpu_data.get_mut().delete_elem(key) + } + fn for_each_elem(&mut self, cb: BpfCallBackFn, ctx: *const u8, flags: u64) -> Result { + self.per_cpu_data.get_mut().for_each_elem(cb, ctx, flags) + } + fn lookup_and_delete_elem(&mut self, _key: &[u8], _value: &mut [u8]) -> Result<()> { + Err(SystemError::EINVAL) + } + fn lookup_percpu_elem(&mut self, key: &[u8], cpu: u32) -> Result> { + unsafe { + self.per_cpu_data + .force_get_mut(ProcessorId::new(cpu)) + .lookup_elem(key) + } + } + fn get_next_key(&self, key: Option<&[u8]>, next_key: &mut [u8]) -> Result<()> { + self.per_cpu_data.get_mut().get_next_key(key, next_key) + } + fn first_value_ptr(&self) -> Result<*const u8> { + self.per_cpu_data.get_mut().first_value_ptr() + } +} + +/// See https://ebpf-docs.dylanreimerink.nl/linux/map-type/BPF_MAP_TYPE_PERF_EVENT_ARRAY/ +pub struct PerfEventArrayMap { + // The value is the file descriptor of the perf event. + fds: ArrayMapData, +} + +impl Debug for PerfEventArrayMap { + fn fmt(&self, f: &mut Formatter<'_>) -> core::fmt::Result { + f.debug_struct("PerfEventArrayMap") + .field("fds", &self.fds) + .finish() + } +} + +impl PerfEventArrayMap { + pub fn new(attr: &BpfMapMeta) -> Result { + let num_cpus = smp_cpu_manager().possible_cpus_count(); + if attr.key_size != 4 || attr.value_size != 4 || attr.max_entries != num_cpus { + return Err(SystemError::EINVAL); + } + let fds = ArrayMapData::new(4, num_cpus); + Ok(PerfEventArrayMap { fds }) + } +} + +impl BpfMapCommonOps for PerfEventArrayMap { + fn lookup_elem(&mut self, key: &[u8]) -> Result> { + let cpu_id = u32::from_ne_bytes(key.try_into().map_err(|_| SystemError::EINVAL)?); + let value = self.fds.index(cpu_id); + Ok(Some(value)) + } + fn update_elem(&mut self, key: &[u8], value: &[u8], _flags: u64) -> Result<()> { + assert_eq!(value.len(), 4); + let cpu_id = u32::from_ne_bytes(key.try_into().map_err(|_| SystemError::EINVAL)?); + let old_value = self.fds.index_mut(cpu_id); + old_value.copy_from_slice(value); + Ok(()) + } + fn delete_elem(&mut self, key: &[u8]) -> Result<()> { + let cpu_id = u32::from_ne_bytes(key.try_into().map_err(|_| SystemError::EINVAL)?); + self.fds.index_mut(cpu_id).copy_from_slice(&[0; 4]); + Ok(()) + } + fn for_each_elem(&mut self, cb: BpfCallBackFn, ctx: *const u8, _flags: u64) -> Result { + let mut total_used = 0; + let num_cpus = smp_cpu_manager().possible_cpus_count(); + for i in 0..num_cpus { + let key = i.to_ne_bytes(); + let value = self.fds.index(i); + total_used += 1; + let res = cb(&key, value, ctx); + if res != 0 { + break; + } + } + Ok(total_used) + } + fn lookup_and_delete_elem(&mut self, _key: &[u8], _value: &mut [u8]) -> Result<()> { + Err(SystemError::EINVAL) + } + fn first_value_ptr(&self) -> Result<*const u8> { + Ok(self.fds.data.as_ptr()) + } +} diff --git a/kernel/src/bpf/map/hash_map.rs b/kernel/src/bpf/map/hash_map.rs new file mode 100644 index 00000000..f3a740ba --- /dev/null +++ b/kernel/src/bpf/map/hash_map.rs @@ -0,0 +1,156 @@ +use super::Result; +use crate::bpf::map::util::{round_up, BpfMapUpdateElemFlags}; +use crate::bpf::map::{BpfCallBackFn, BpfMapCommonOps, BpfMapMeta}; +use crate::mm::percpu::{PerCpu, PerCpuVar}; +use crate::smp::cpu::ProcessorId; +use alloc::{collections::BTreeMap, vec::Vec}; +use core::fmt::Debug; +use system_error::SystemError; + +type BpfHashMapKey = Vec; +type BpfHashMapValue = Vec; + +/// The hash map type is a generic map type with no restrictions on the structure of the key and value. +/// Hash-maps are implemented using a hash table, allowing for lookups with arbitrary keys. +/// +/// See https://ebpf-docs.dylanreimerink.nl/linux/map-type/BPF_MAP_TYPE_HASH/ +#[derive(Debug)] +pub struct BpfHashMap { + _max_entries: u32, + _key_size: u32, + _value_size: u32, + data: BTreeMap, +} + +impl BpfHashMap { + pub fn new(attr: &BpfMapMeta) -> Result { + if attr.value_size == 0 || attr.max_entries == 0 { + return Err(SystemError::EINVAL); + } + let value_size = round_up(attr.value_size as usize, 8); + Ok(Self { + _max_entries: attr.max_entries, + _key_size: attr.key_size, + _value_size: value_size as u32, + data: BTreeMap::new(), + }) + } +} + +impl BpfMapCommonOps for BpfHashMap { + fn lookup_elem(&mut self, key: &[u8]) -> Result> { + let value = self.data.get(key).map(|v| v.as_slice()); + Ok(value) + } + fn update_elem(&mut self, key: &[u8], value: &[u8], flags: u64) -> Result<()> { + let _flags = BpfMapUpdateElemFlags::from_bits_truncate(flags); + self.data.insert(key.to_vec(), value.to_vec()); + Ok(()) + } + fn delete_elem(&mut self, key: &[u8]) -> Result<()> { + self.data.remove(key); + Ok(()) + } + fn for_each_elem(&mut self, cb: BpfCallBackFn, ctx: *const u8, flags: u64) -> Result { + if flags != 0 { + return Err(SystemError::EINVAL); + } + let mut total_used = 0; + for (key, value) in self.data.iter() { + let res = cb(key, value, ctx); + // return value: 0 - continue, 1 - stop and return + if res != 0 { + break; + } + total_used += 1; + } + Ok(total_used) + } + fn lookup_and_delete_elem(&mut self, key: &[u8], value: &mut [u8]) -> Result<()> { + let v = self + .data + .get(key) + .map(|v| v.as_slice()) + .ok_or(SystemError::ENOENT)?; + value.copy_from_slice(v); + self.data.remove(key); + Ok(()) + } + fn get_next_key(&self, key: Option<&[u8]>, next_key: &mut [u8]) -> Result<()> { + let mut iter = self.data.iter(); + if let Some(key) = key { + for (k, _) in iter.by_ref() { + if k.as_slice() == key { + break; + } + } + } + let res = iter.next(); + match res { + Some((k, _)) => { + next_key.copy_from_slice(k.as_slice()); + Ok(()) + } + None => Err(SystemError::ENOENT), + } + } +} + +/// This is the per-CPU variant of the [BpfHashMap] map type. +/// +/// See https://ebpf-docs.dylanreimerink.nl/linux/map-type/BPF_MAP_TYPE_PERCPU_HASH/ +pub struct PerCpuHashMap { + per_cpu_maps: PerCpuVar, +} + +impl Debug for PerCpuHashMap { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("PerCpuHashMap") + .field("maps", &self.per_cpu_maps) + .finish() + } +} +impl PerCpuHashMap { + pub fn new(attr: &BpfMapMeta) -> Result { + let num_cpus = PerCpu::MAX_CPU_NUM; + let mut data = Vec::with_capacity(num_cpus as usize); + for _ in 0..num_cpus { + let array_map = BpfHashMap::new(attr)?; + data.push(array_map); + } + let per_cpu_maps = PerCpuVar::new(data).ok_or(SystemError::EINVAL)?; + Ok(PerCpuHashMap { per_cpu_maps }) + } +} +impl BpfMapCommonOps for PerCpuHashMap { + fn lookup_elem(&mut self, key: &[u8]) -> Result> { + self.per_cpu_maps.get_mut().lookup_elem(key) + } + fn update_elem(&mut self, key: &[u8], value: &[u8], flags: u64) -> Result<()> { + self.per_cpu_maps.get_mut().update_elem(key, value, flags) + } + fn delete_elem(&mut self, key: &[u8]) -> Result<()> { + self.per_cpu_maps.get_mut().delete_elem(key) + } + fn for_each_elem(&mut self, cb: BpfCallBackFn, ctx: *const u8, flags: u64) -> Result { + self.per_cpu_maps.get_mut().for_each_elem(cb, ctx, flags) + } + fn lookup_and_delete_elem(&mut self, key: &[u8], value: &mut [u8]) -> Result<()> { + self.per_cpu_maps + .get_mut() + .lookup_and_delete_elem(key, value) + } + fn lookup_percpu_elem(&mut self, key: &[u8], cpu: u32) -> Result> { + unsafe { + self.per_cpu_maps + .force_get_mut(ProcessorId::new(cpu)) + .lookup_elem(key) + } + } + fn get_next_key(&self, key: Option<&[u8]>, next_key: &mut [u8]) -> Result<()> { + self.per_cpu_maps.get_mut().get_next_key(key, next_key) + } + fn first_value_ptr(&self) -> Result<*const u8> { + self.per_cpu_maps.get_mut().first_value_ptr() + } +} diff --git a/kernel/src/bpf/map/lru.rs b/kernel/src/bpf/map/lru.rs new file mode 100644 index 00000000..d4927aa4 --- /dev/null +++ b/kernel/src/bpf/map/lru.rs @@ -0,0 +1,151 @@ +use super::{BpfCallBackFn, BpfMapCommonOps, Result}; +use crate::bpf::map::util::BpfMapMeta; +use crate::mm::percpu::{PerCpu, PerCpuVar}; +use crate::smp::cpu::ProcessorId; +use alloc::vec::Vec; +use core::fmt::Debug; +use core::num::NonZero; +use lru::LruCache; +use system_error::SystemError; + +type BpfHashMapKey = Vec; +type BpfHashMapValue = Vec; +/// This map is the LRU (Least Recently Used) variant of the BPF_MAP_TYPE_HASH. +/// It is a generic map type that stores a fixed maximum number of key/value pairs. +/// When the map starts to get at capacity, the approximately least recently +/// used elements is removed to make room for new elements. +/// +/// See https://docs.ebpf.io/linux/map-type/BPF_MAP_TYPE_LRU_HASH/ +#[derive(Debug)] +pub struct LruMap { + _max_entries: u32, + data: LruCache, +} + +impl LruMap { + pub fn new(attr: &BpfMapMeta) -> Result { + if attr.value_size == 0 || attr.max_entries == 0 { + return Err(SystemError::EINVAL); + } + Ok(Self { + _max_entries: attr.max_entries, + data: LruCache::new( + NonZero::new(attr.max_entries as usize).ok_or(SystemError::EINVAL)?, + ), + }) + } +} + +impl BpfMapCommonOps for LruMap { + fn lookup_elem(&mut self, key: &[u8]) -> Result> { + let value = self.data.get(key).map(|v| v.as_slice()); + Ok(value) + } + fn update_elem(&mut self, key: &[u8], value: &[u8], _flags: u64) -> Result<()> { + self.data.put(key.to_vec(), value.to_vec()); + Ok(()) + } + fn delete_elem(&mut self, key: &[u8]) -> Result<()> { + self.data.pop(key); + Ok(()) + } + fn for_each_elem(&mut self, cb: BpfCallBackFn, ctx: *const u8, flags: u64) -> Result { + if flags != 0 { + return Err(SystemError::EINVAL); + } + let mut total_used = 0; + for (key, value) in self.data.iter() { + let res = cb(key, value, ctx); + // return value: 0 - continue, 1 - stop and return + if res != 0 { + break; + } + total_used += 1; + } + Ok(total_used) + } + fn lookup_and_delete_elem(&mut self, key: &[u8], value: &mut [u8]) -> Result<()> { + let v = self + .data + .get(key) + .map(|v| v.as_slice()) + .ok_or(SystemError::ENOENT)?; + value.copy_from_slice(v); + self.data.pop(key); + Ok(()) + } + fn get_next_key(&self, key: Option<&[u8]>, next_key: &mut [u8]) -> Result<()> { + let mut iter = self.data.iter(); + if let Some(key) = key { + for (k, _) in iter.by_ref() { + if k.as_slice() == key { + break; + } + } + } + let res = iter.next(); + match res { + Some((k, _)) => { + next_key.copy_from_slice(k.as_slice()); + Ok(()) + } + None => Err(SystemError::ENOENT), + } + } +} + +/// See https://ebpf-docs.dylanreimerink.nl/linux/map-type/BPF_MAP_TYPE_LRU_PERCPU_HASH/ +pub struct PerCpuLruMap { + per_cpu_maps: PerCpuVar, +} + +impl Debug for PerCpuLruMap { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("PerCpuLruMap") + .field("maps", &self.per_cpu_maps) + .finish() + } +} + +impl PerCpuLruMap { + pub fn new(attr: &BpfMapMeta) -> Result { + let num_cpus = PerCpu::MAX_CPU_NUM; + let mut data = Vec::with_capacity(num_cpus as usize); + for _ in 0..num_cpus { + let array_map = LruMap::new(attr)?; + data.push(array_map); + } + let per_cpu_maps = PerCpuVar::new(data).ok_or(SystemError::EINVAL)?; + Ok(PerCpuLruMap { per_cpu_maps }) + } +} + +impl BpfMapCommonOps for PerCpuLruMap { + fn lookup_elem(&mut self, key: &[u8]) -> Result> { + self.per_cpu_maps.get_mut().lookup_elem(key) + } + fn update_elem(&mut self, key: &[u8], value: &[u8], flags: u64) -> Result<()> { + self.per_cpu_maps.get_mut().update_elem(key, value, flags) + } + fn delete_elem(&mut self, key: &[u8]) -> Result<()> { + self.per_cpu_maps.get_mut().delete_elem(key) + } + fn for_each_elem(&mut self, cb: BpfCallBackFn, ctx: *const u8, flags: u64) -> Result { + self.per_cpu_maps.get_mut().for_each_elem(cb, ctx, flags) + } + fn lookup_and_delete_elem(&mut self, key: &[u8], value: &mut [u8]) -> Result<()> { + self.per_cpu_maps + .get_mut() + .lookup_and_delete_elem(key, value) + } + fn lookup_percpu_elem(&mut self, key: &[u8], cpu: u32) -> Result> { + unsafe { + self.per_cpu_maps + .force_get_mut(ProcessorId::new(cpu)) + .lookup_elem(key) + } + } + fn get_next_key(&self, key: Option<&[u8]>, next_key: &mut [u8]) -> Result<()> { + self.per_cpu_maps.get_mut().get_next_key(key, next_key) + } +} diff --git a/kernel/src/bpf/map/mod.rs b/kernel/src/bpf/map/mod.rs new file mode 100644 index 00000000..cc8eeffe --- /dev/null +++ b/kernel/src/bpf/map/mod.rs @@ -0,0 +1,416 @@ +mod array_map; +mod hash_map; +mod lru; +mod queue; +mod util; + +use super::Result; +use crate::bpf::map::array_map::{ArrayMap, PerCpuArrayMap, PerfEventArrayMap}; +use crate::bpf::map::hash_map::PerCpuHashMap; +use crate::bpf::map::util::{BpfMapGetNextKeyArg, BpfMapMeta, BpfMapUpdateArg}; +use crate::filesystem::vfs::file::{File, FileMode}; +use crate::filesystem::vfs::syscall::ModeType; +use crate::filesystem::vfs::{FilePrivateData, FileSystem, FileType, IndexNode, Metadata}; +use crate::include::bindings::linux_bpf::{bpf_attr, bpf_map_type}; +use crate::libs::casting::DowncastArc; +use crate::libs::spinlock::{SpinLock, SpinLockGuard}; +use crate::process::ProcessManager; +use crate::syscall::user_access::{UserBufferReader, UserBufferWriter}; +use alloc::boxed::Box; +use alloc::string::String; +use alloc::sync::Arc; +use alloc::vec::Vec; +use core::any::Any; +use core::fmt::Debug; +use intertrait::CastFromSync; +use log::{error, info}; +use system_error::SystemError; + +#[derive(Debug)] +pub struct BpfMap { + inner_map: SpinLock>, + meta: BpfMapMeta, +} + +pub type BpfCallBackFn = fn(key: &[u8], value: &[u8], ctx: *const u8) -> i32; + +pub trait BpfMapCommonOps: Send + Sync + Debug + CastFromSync { + /// Lookup an element in the map. + /// + /// See https://ebpf-docs.dylanreimerink.nl/linux/helper-function/bpf_map_lookup_elem/ + fn lookup_elem(&mut self, _key: &[u8]) -> Result> { + Err(SystemError::ENOSYS) + } + /// Update an element in the map. + /// + /// See https://ebpf-docs.dylanreimerink.nl/linux/helper-function/bpf_map_update_elem/ + fn update_elem(&mut self, _key: &[u8], _value: &[u8], _flags: u64) -> Result<()> { + Err(SystemError::ENOSYS) + } + /// Delete an element from the map. + /// + /// See https://ebpf-docs.dylanreimerink.nl/linux/helper-function/bpf_map_delete_elem/ + fn delete_elem(&mut self, _key: &[u8]) -> Result<()> { + Err(SystemError::ENOSYS) + } + /// For each element in map, call callback_fn function with map, + /// callback_ctx and other map-specific parameters. + /// + /// See https://ebpf-docs.dylanreimerink.nl/linux/helper-function/bpf_for_each_map_elem/ + fn for_each_elem(&mut self, _cb: BpfCallBackFn, _ctx: *const u8, _flags: u64) -> Result { + Err(SystemError::ENOSYS) + } + /// Look up an element with the given key in the map referred to by the file descriptor fd, + /// and if found, delete the element. + fn lookup_and_delete_elem(&mut self, _key: &[u8], _value: &mut [u8]) -> Result<()> { + Err(SystemError::ENOSYS) + } + + /// erform a lookup in percpu map for an entry associated to key on cpu. + fn lookup_percpu_elem(&mut self, _key: &[u8], _cpu: u32) -> Result> { + Err(SystemError::ENOSYS) + } + /// Get the next key in the map. If key is None, get the first key. + /// + /// Called from syscall + fn get_next_key(&self, _key: Option<&[u8]>, _next_key: &mut [u8]) -> Result<()> { + Err(SystemError::ENOSYS) + } + + /// Push an element value in map. + fn push_elem(&mut self, _value: &[u8], _flags: u64) -> Result<()> { + Err(SystemError::ENOSYS) + } + + /// Pop an element value from map. + fn pop_elem(&mut self, _value: &mut [u8]) -> Result<()> { + Err(SystemError::ENOSYS) + } + + /// Peek an element value from map. + fn peek_elem(&self, _value: &mut [u8]) -> Result<()> { + Err(SystemError::ENOSYS) + } + + /// Freeze the map. + /// + /// It's useful for .rodata maps. + fn freeze(&self) -> Result<()> { + Err(SystemError::ENOSYS) + } + + /// Get the first value pointer. + fn first_value_ptr(&self) -> Result<*const u8> { + Err(SystemError::ENOSYS) + } +} +impl DowncastArc for dyn BpfMapCommonOps { + fn as_any_arc(self: Arc) -> Arc { + self + } +} +impl BpfMap { + pub fn new(map: Box, meta: BpfMapMeta) -> Self { + assert_ne!(meta.key_size, 0); + BpfMap { + inner_map: SpinLock::new(map), + meta, + } + } + + pub fn inner_map(&self) -> &SpinLock> { + &self.inner_map + } + + pub fn key_size(&self) -> usize { + self.meta.key_size as usize + } + + pub fn value_size(&self) -> usize { + self.meta.value_size as usize + } +} + +impl IndexNode for BpfMap { + fn open(&self, _data: SpinLockGuard, _mode: &FileMode) -> Result<()> { + Ok(()) + } + fn close(&self, _data: SpinLockGuard) -> Result<()> { + Ok(()) + } + fn read_at( + &self, + _offset: usize, + _len: usize, + _buf: &mut [u8], + _data: SpinLockGuard, + ) -> Result { + Err(SystemError::ENOSYS) + } + + fn write_at( + &self, + _offset: usize, + _len: usize, + _buf: &[u8], + _data: SpinLockGuard, + ) -> Result { + Err(SystemError::ENOSYS) + } + + fn metadata(&self) -> Result { + let meta = Metadata { + mode: ModeType::from_bits_truncate(0o755), + file_type: FileType::File, + ..Default::default() + }; + Ok(meta) + } + + fn resize(&self, _len: usize) -> Result<()> { + Ok(()) + } + + fn fs(&self) -> Arc { + todo!("BpfMap does not have a filesystem") + } + + fn as_any_ref(&self) -> &dyn Any { + self + } + + fn list(&self) -> Result> { + Err(SystemError::ENOSYS) + } +} + +/// Create a map and return a file descriptor that refers to +/// the map. The close-on-exec file descriptor flag +/// is automatically enabled for the new file descriptor. +/// +/// See https://ebpf-docs.dylanreimerink.nl/linux/syscall/BPF_MAP_CREATE/ +pub fn bpf_map_create(attr: &bpf_attr) -> Result { + let map_meta = BpfMapMeta::try_from(attr)?; + info!("The map attr is {:#?}", map_meta); + let map: Box = match map_meta.map_type { + bpf_map_type::BPF_MAP_TYPE_ARRAY => { + let array_map = ArrayMap::new(&map_meta)?; + Box::new(array_map) + } + bpf_map_type::BPF_MAP_TYPE_PERCPU_ARRAY => { + let per_cpu_array_map = PerCpuArrayMap::new(&map_meta)?; + Box::new(per_cpu_array_map) + } + bpf_map_type::BPF_MAP_TYPE_PERF_EVENT_ARRAY => { + let perf_event_array_map = PerfEventArrayMap::new(&map_meta)?; + Box::new(perf_event_array_map) + } + + bpf_map_type::BPF_MAP_TYPE_CPUMAP + | bpf_map_type::BPF_MAP_TYPE_DEVMAP + | bpf_map_type::BPF_MAP_TYPE_DEVMAP_HASH => { + error!("bpf map type {:?} not implemented", map_meta.map_type); + Err(SystemError::EINVAL)? + } + bpf_map_type::BPF_MAP_TYPE_HASH => { + let hash_map = hash_map::BpfHashMap::new(&map_meta)?; + Box::new(hash_map) + } + bpf_map_type::BPF_MAP_TYPE_PERCPU_HASH => { + let per_cpu_hash_map = PerCpuHashMap::new(&map_meta)?; + Box::new(per_cpu_hash_map) + } + bpf_map_type::BPF_MAP_TYPE_QUEUE => { + let queue_map = queue::QueueMap::new(&map_meta)?; + Box::new(queue_map) + } + bpf_map_type::BPF_MAP_TYPE_STACK => { + let stack_map = queue::StackMap::new(&map_meta)?; + Box::new(stack_map) + } + bpf_map_type::BPF_MAP_TYPE_LRU_HASH => { + let lru_hash_map = lru::LruMap::new(&map_meta)?; + Box::new(lru_hash_map) + } + bpf_map_type::BPF_MAP_TYPE_LRU_PERCPU_HASH => { + let lru_per_cpu_hash_map = lru::PerCpuLruMap::new(&map_meta)?; + Box::new(lru_per_cpu_hash_map) + } + _ => { + unimplemented!("bpf map type {:?} not implemented", map_meta.map_type) + } + }; + let bpf_map = BpfMap::new(map, map_meta); + let fd_table = ProcessManager::current_pcb().fd_table(); + let file = File::new(Arc::new(bpf_map), FileMode::O_RDWR | FileMode::O_CLOEXEC)?; + let fd = fd_table.write().alloc_fd(file, None).map(|x| x as usize)?; + info!("create map with fd: [{}]", fd); + Ok(fd) +} + +/// Create or update an element (key/value pair) in a specified map. +/// +/// See https://ebpf-docs.dylanreimerink.nl/linux/syscall/BPF_MAP_UPDATE_ELEM/ +pub fn bpf_map_update_elem(attr: &bpf_attr) -> Result { + let arg = BpfMapUpdateArg::from(attr); + info!(": {:#x?}", arg); + let map = get_map_file(arg.map_fd as i32)?; + let meta = &map.meta; + let key_size = meta.key_size as usize; + let value_size = meta.value_size as usize; + + let key_buf = UserBufferReader::new(arg.key as *const u8, key_size, true)?; + let value_buf = UserBufferReader::new(arg.value as *const u8, value_size, true)?; + + let key = key_buf.read_from_user(0)?; + let value = value_buf.read_from_user(0)?; + map.inner_map.lock().update_elem(key, value, arg.flags)?; + info!("bpf_map_update_elem ok"); + Ok(0) +} + +pub fn bpf_map_freeze(attr: &bpf_attr) -> Result { + let arg = BpfMapUpdateArg::from(attr); + let map_fd = arg.map_fd; + info!(": map_fd: {:}", map_fd); + let map = get_map_file(map_fd as i32)?; + map.inner_map.lock().freeze()?; + Ok(0) +} + +/// Look up an element by key in a specified map and return its value. +/// +/// See https://ebpf-docs.dylanreimerink.nl/linux/syscall/BPF_MAP_LOOKUP_ELEM/ +pub fn bpf_lookup_elem(attr: &bpf_attr) -> Result { + let arg = BpfMapUpdateArg::from(attr); + // info!(": {:#x?}", arg); + let map = get_map_file(arg.map_fd as _)?; + let meta = &map.meta; + let key_size = meta.key_size as usize; + let value_size = meta.value_size as usize; + + let key_buf = UserBufferReader::new(arg.key as *const u8, key_size, true)?; + let mut value_buf = UserBufferWriter::new(arg.value as *mut u8, value_size, true)?; + + let key = key_buf.read_from_user(0)?; + + let mut inner = map.inner_map.lock(); + let r_value = inner.lookup_elem(key)?; + if let Some(r_value) = r_value { + value_buf.copy_to_user(r_value, 0)?; + Ok(0) + } else { + Err(SystemError::ENOENT) + } +} +/// Look up an element by key in a specified map and return the key of the next element. +/// +/// - If key is `None`, the operation returns zero and sets the next_key pointer to the key of the first element. +/// - If key is `Some(T)`, the operation returns zero and sets the next_key pointer to the key of the next element. +/// - If key is the last element, returns -1 and errno is set to ENOENT. +/// +/// See https://ebpf-docs.dylanreimerink.nl/linux/syscall/BPF_MAP_GET_NEXT_KEY/ +pub fn bpf_map_get_next_key(attr: &bpf_attr) -> Result { + let arg = BpfMapGetNextKeyArg::from(attr); + // info!(": {:#x?}", arg); + let map = get_map_file(arg.map_fd as i32)?; + let meta = &map.meta; + let key_size = meta.key_size as usize; + + let key = if let Some(key_ptr) = arg.key { + let key_buf = UserBufferReader::new(key_ptr as *const u8, key_size, true)?; + let key = key_buf.read_from_user(0)?.to_vec(); + Some(key) + } else { + None + }; + let key = key.as_deref(); + let mut next_key_buf = UserBufferWriter::new(arg.next_key as *mut u8, key_size, true)?; + let inner = map.inner_map.lock(); + let next_key = next_key_buf.buffer(0)?; + inner.get_next_key(key, next_key)?; + // info!("next_key: {:?}", next_key); + Ok(0) +} + +/// Look up and delete an element by key in a specified map. +/// +/// # WARN +/// +/// Not all map types (particularly array maps) support this operation, +/// instead a zero value can be written to the map value. Check the map types page to check for support. +/// +/// See https://ebpf-docs.dylanreimerink.nl/linux/syscall/BPF_MAP_DELETE_ELEM/ +pub fn bpf_map_delete_elem(attr: &bpf_attr) -> Result { + let arg = BpfMapUpdateArg::from(attr); + // info!(": {:#x?}", arg); + let map = get_map_file(arg.map_fd as i32)?; + let meta = &map.meta; + let key_size = meta.key_size as usize; + + let key_buf = UserBufferReader::new(arg.key as *const u8, key_size, true)?; + let key = key_buf.read_from_user(0)?; + map.inner_map.lock().delete_elem(key)?; + Ok(0) +} + +/// Iterate and fetch multiple elements in a map. +/// +/// See https://ebpf-docs.dylanreimerink.nl/linux/syscall/BPF_MAP_LOOKUP_BATCH/ +pub fn bpf_map_lookup_batch(_attr: &bpf_attr) -> Result { + todo!() +} + +/// Look up an element with the given key in the map referred to by the file descriptor fd, +/// and if found, delete the element. +/// +/// For BPF_MAP_TYPE_QUEUE and BPF_MAP_TYPE_STACK map types, the flags argument needs to be set to 0, +/// but for other map types, it may be specified as: +/// - BPF_F_LOCK : If this flag is set, the command will acquire the spin-lock of the map value we are looking up. +/// +/// If the map contains no spin-lock in its value, -EINVAL will be returned by the command. +/// +/// The BPF_MAP_TYPE_QUEUE and BPF_MAP_TYPE_STACK map types implement this command as a “pop” operation, +/// deleting the top element rather than one corresponding to key. +/// The key and key_len parameters should be zeroed when issuing this operation for these map types. +/// +/// This command is only valid for the following map types: +/// - BPF_MAP_TYPE_QUEUE +/// - BPF_MAP_TYPE_STACK +/// - BPF_MAP_TYPE_HASH +/// - BPF_MAP_TYPE_PERCPU_HASH +/// - BPF_MAP_TYPE_LRU_HASH +/// - BPF_MAP_TYPE_LRU_PERCPU_HASH +/// +/// +/// See https://ebpf-docs.dylanreimerink.nl/linux/syscall/BPF_MAP_LOOKUP_AND_DELETE_ELEM/ +pub fn bpf_map_lookup_and_delete_elem(attr: &bpf_attr) -> Result { + let arg = BpfMapUpdateArg::from(attr); + // info!(": {:#x?}", arg); + let map = get_map_file(arg.map_fd as i32)?; + let meta = &map.meta; + let key_size = meta.key_size as usize; + let value_size = meta.value_size as usize; + + let key_buf = UserBufferReader::new(arg.key as *const u8, key_size, true)?; + let mut value_buf = UserBufferWriter::new(arg.value as *mut u8, value_size, true)?; + + let value = value_buf.buffer(0)?; + let key = key_buf.read_from_user(0)?; + let mut inner = map.inner_map.lock(); + inner.lookup_and_delete_elem(key, value)?; + Ok(0) +} + +fn get_map_file(fd: i32) -> Result> { + let fd_table = ProcessManager::current_pcb().fd_table(); + let map = fd_table + .read() + .get_file_by_fd(fd) + .ok_or(SystemError::EBADF)?; + let map = map + .inode() + .downcast_arc::() + .ok_or(SystemError::EINVAL)?; + Ok(map) +} diff --git a/kernel/src/bpf/map/queue.rs b/kernel/src/bpf/map/queue.rs new file mode 100644 index 00000000..2a97edff --- /dev/null +++ b/kernel/src/bpf/map/queue.rs @@ -0,0 +1,154 @@ +use super::{BpfMapCommonOps, Result}; +use crate::bpf::map::util::{BpfMapMeta, BpfMapUpdateElemFlags}; +use alloc::vec::Vec; +use core::fmt::Debug; +use core::ops::Deref; +use core::ops::DerefMut; +use system_error::SystemError; + +type BpfQueueValue = Vec; +/// BPF_MAP_TYPE_QUEUE provides FIFO storage and BPF_MAP_TYPE_STACK provides LIFO storage for BPF programs. +/// These maps support peek, pop and push operations that are exposed to BPF programs through the respective helpers. +/// These operations are exposed to userspace applications using the existing bpf syscall in the following way: +/// - `BPF_MAP_LOOKUP_ELEM` -> `peek` +/// - `BPF_MAP_UPDATE_ELEM` -> `push` +/// - `BPF_MAP_LOOKUP_AND_DELETE_ELEM ` -> `pop` +/// +/// See https://docs.kernel.org/bpf/map_queue_stack.html +pub trait SpecialMap: Debug + Send + Sync + 'static { + /// Returns the number of elements the queue can hold. + fn push(&mut self, value: BpfQueueValue, flags: BpfMapUpdateElemFlags) -> Result<()>; + /// Removes the first element and returns it. + fn pop(&mut self) -> Option; + /// Returns the first element without removing it. + fn peek(&self) -> Option<&BpfQueueValue>; +} + +/// The queue map type is a generic map type, resembling a FIFO (First-In First-Out) queue. +/// +/// This map type has no keys, only values. The size and type of the values can be specified by the user +/// to fit a large variety of use cases. The typical use-case for this map type is to keep track of +/// a pool of elements such as available network ports when implementing NAT (network address translation). +/// +/// As apposed to most map types, this map type uses a custom set of helpers to pop, peek and push elements. +/// +/// See https://ebpf-docs.dylanreimerink.nl/linux/map-type/BPF_MAP_TYPE_QUEUE/ +#[derive(Debug)] +pub struct QueueMap { + max_entries: u32, + data: Vec, +} + +impl QueueMap { + pub fn new(attr: &BpfMapMeta) -> Result { + if attr.value_size == 0 || attr.max_entries == 0 || attr.key_size != 0 { + return Err(SystemError::EINVAL); + } + let data = Vec::with_capacity(attr.max_entries as usize); + Ok(Self { + max_entries: attr.max_entries, + data, + }) + } +} + +impl SpecialMap for QueueMap { + fn push(&mut self, value: BpfQueueValue, flags: BpfMapUpdateElemFlags) -> Result<()> { + if self.data.len() == self.max_entries as usize { + if flags.contains(BpfMapUpdateElemFlags::BPF_EXIST) { + // remove the first element + self.data.remove(0); + } else { + return Err(SystemError::ENOSPC); + } + } + self.data.push(value); + Ok(()) + } + fn pop(&mut self) -> Option { + if self.data.is_empty() { + return None; + } + Some(self.data.remove(0)) + } + fn peek(&self) -> Option<&BpfQueueValue> { + self.data.first() + } +} +/// The stack map type is a generic map type, resembling a stack data structure. +/// +/// See https://ebpf-docs.dylanreimerink.nl/linux/map-type/BPF_MAP_TYPE_STACK/ +#[derive(Debug)] +pub struct StackMap(QueueMap); + +impl StackMap { + pub fn new(attr: &BpfMapMeta) -> Result { + QueueMap::new(attr).map(StackMap) + } +} + +impl Deref for StackMap { + type Target = QueueMap; + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl DerefMut for StackMap { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +impl SpecialMap for StackMap { + fn push(&mut self, value: BpfQueueValue, flags: BpfMapUpdateElemFlags) -> Result<()> { + if self.data.len() == self.max_entries as usize { + if flags.contains(BpfMapUpdateElemFlags::BPF_EXIST) { + // remove the last element + self.data.pop(); + } else { + return Err(SystemError::ENOSPC); + } + } + self.data.push(value); + Ok(()) + } + fn pop(&mut self) -> Option { + self.data.pop() + } + fn peek(&self) -> Option<&BpfQueueValue> { + self.data.last() + } +} + +impl BpfMapCommonOps for T { + /// Equal to [QueueMap::peek] + fn lookup_elem(&mut self, _key: &[u8]) -> Result> { + Ok(self.peek().map(|v| v.as_slice())) + } + /// Equal to [QueueMap::push] + fn update_elem(&mut self, _key: &[u8], value: &[u8], flags: u64) -> Result<()> { + let flag = BpfMapUpdateElemFlags::from_bits_truncate(flags); + self.push(value.to_vec(), flag) + } + /// Equal to [QueueMap::pop] + fn lookup_and_delete_elem(&mut self, _key: &[u8], value: &mut [u8]) -> Result<()> { + if let Some(v) = self.pop() { + value.copy_from_slice(&v); + Ok(()) + } else { + Err(SystemError::ENOENT) + } + } + fn push_elem(&mut self, value: &[u8], flags: u64) -> Result<()> { + self.update_elem(&[], value, flags) + } + fn pop_elem(&mut self, value: &mut [u8]) -> Result<()> { + self.lookup_and_delete_elem(&[], value) + } + fn peek_elem(&self, value: &mut [u8]) -> Result<()> { + self.peek() + .map(|v| value.copy_from_slice(v)) + .ok_or(SystemError::ENOENT) + } +} diff --git a/kernel/src/bpf/map/util.rs b/kernel/src/bpf/map/util.rs new file mode 100644 index 00000000..fdfc594e --- /dev/null +++ b/kernel/src/bpf/map/util.rs @@ -0,0 +1,100 @@ +use crate::include::bindings::linux_bpf::{bpf_attr, bpf_map_type}; +use alloc::string::{String, ToString}; +use core::ffi::CStr; +use num_traits::FromPrimitive; +use system_error::SystemError; + +#[derive(Debug, Clone)] +pub struct BpfMapMeta { + pub map_type: bpf_map_type, + pub key_size: u32, + pub value_size: u32, + pub max_entries: u32, + pub _map_flags: u32, + pub _map_name: String, +} + +impl TryFrom<&bpf_attr> for BpfMapMeta { + type Error = SystemError; + fn try_from(value: &bpf_attr) -> Result { + let u = unsafe { &value.__bindgen_anon_1 }; + let map_name_slice = unsafe { + core::slice::from_raw_parts(u.map_name.as_ptr() as *const u8, u.map_name.len()) + }; + let map_name = CStr::from_bytes_until_nul(map_name_slice) + .map_err(|_| SystemError::EINVAL)? + .to_str() + .map_err(|_| SystemError::EINVAL)? + .to_string(); + let map_type = bpf_map_type::from_u32(u.map_type).ok_or(SystemError::EINVAL)?; + Ok(BpfMapMeta { + map_type, + key_size: u.key_size, + value_size: u.value_size, + max_entries: u.max_entries, + _map_flags: u.map_flags, + _map_name: map_name, + }) + } +} + +#[derive(Debug)] +pub struct BpfMapUpdateArg { + pub map_fd: u32, + pub key: u64, + pub value: u64, + pub flags: u64, +} + +impl From<&bpf_attr> for BpfMapUpdateArg { + fn from(value: &bpf_attr) -> Self { + unsafe { + let u = &value.__bindgen_anon_2; + BpfMapUpdateArg { + map_fd: u.map_fd, + key: u.key, + value: u.__bindgen_anon_1.value, + flags: u.flags, + } + } + } +} +#[derive(Debug)] +pub struct BpfMapGetNextKeyArg { + pub map_fd: u32, + pub key: Option, + pub next_key: u64, +} + +impl From<&bpf_attr> for BpfMapGetNextKeyArg { + fn from(value: &bpf_attr) -> Self { + unsafe { + let u = &value.__bindgen_anon_2; + BpfMapGetNextKeyArg { + map_fd: u.map_fd, + key: if u.key != 0 { Some(u.key) } else { None }, + next_key: u.__bindgen_anon_1.next_key, + } + } + } +} + +#[inline] +/// Round up `x` to the nearest multiple of `align`. +pub fn round_up(x: usize, align: usize) -> usize { + (x + align - 1) & !(align - 1) +} + +bitflags! { + /// flags for BPF_MAP_UPDATE_ELEM command + pub struct BpfMapUpdateElemFlags: u64 { + /// create new element or update existing + const BPF_ANY = 0; + /// create new element if it didn't exist + const BPF_NOEXIST = 1; + /// update existing element + const BPF_EXIST = 2; + /// spin_lock-ed map_lookup/map_update + const BPF_F_LOCK = 4; + } +} diff --git a/kernel/src/bpf/mod.rs b/kernel/src/bpf/mod.rs new file mode 100644 index 00000000..8e84f205 --- /dev/null +++ b/kernel/src/bpf/mod.rs @@ -0,0 +1,50 @@ +pub mod helper; +pub mod map; +pub mod prog; +use crate::include::bindings::linux_bpf::{bpf_attr, bpf_cmd}; +use crate::syscall::user_access::UserBufferReader; +use crate::syscall::Syscall; +use log::error; +use num_traits::FromPrimitive; +use system_error::SystemError; + +type Result = core::result::Result; + +impl Syscall { + pub fn sys_bpf(cmd: u32, attr: *mut u8, size: u32) -> Result { + let buf = UserBufferReader::new(attr, size as usize, true)?; + let attr = buf.read_one_from_user::(0)?; + let cmd = bpf_cmd::from_u32(cmd).ok_or(SystemError::EINVAL)?; + bpf(cmd, attr) + } +} + +pub fn bpf(cmd: bpf_cmd, attr: &bpf_attr) -> Result { + let res = match cmd { + // Map related commands + bpf_cmd::BPF_MAP_CREATE => map::bpf_map_create(attr), + bpf_cmd::BPF_MAP_UPDATE_ELEM => map::bpf_map_update_elem(attr), + bpf_cmd::BPF_MAP_LOOKUP_ELEM => map::bpf_lookup_elem(attr), + bpf_cmd::BPF_MAP_GET_NEXT_KEY => map::bpf_map_get_next_key(attr), + bpf_cmd::BPF_MAP_DELETE_ELEM => map::bpf_map_delete_elem(attr), + bpf_cmd::BPF_MAP_LOOKUP_AND_DELETE_ELEM => map::bpf_map_lookup_and_delete_elem(attr), + bpf_cmd::BPF_MAP_LOOKUP_BATCH => map::bpf_map_lookup_batch(attr), + bpf_cmd::BPF_MAP_FREEZE => map::bpf_map_freeze(attr), + // Program related commands + bpf_cmd::BPF_PROG_LOAD => prog::bpf_prog_load(attr), + // Object creation commands + bpf_cmd::BPF_BTF_LOAD => { + error!("bpf cmd {:?} not implemented", cmd); + return Err(SystemError::ENOSYS); + } + ty => { + unimplemented!("bpf cmd {:?} not implemented", ty) + } + }; + res +} + +/// Initialize the BPF system +pub fn init_bpf_system() { + helper::init_helper_functions(); +} diff --git a/kernel/src/bpf/prog/mod.rs b/kernel/src/bpf/prog/mod.rs new file mode 100644 index 00000000..569af42f --- /dev/null +++ b/kernel/src/bpf/prog/mod.rs @@ -0,0 +1,123 @@ +mod util; +mod verifier; + +use super::Result; +use crate::bpf::map::BpfMap; +use crate::bpf::prog::util::{BpfProgMeta, BpfProgVerifierInfo}; +use crate::bpf::prog::verifier::BpfProgVerifier; +use crate::filesystem::vfs::file::{File, FileMode}; +use crate::filesystem::vfs::syscall::ModeType; +use crate::filesystem::vfs::{FilePrivateData, FileSystem, FileType, IndexNode, Metadata}; +use crate::include::bindings::linux_bpf::bpf_attr; +use crate::libs::spinlock::SpinLockGuard; +use crate::process::ProcessManager; +use alloc::string::String; +use alloc::sync::Arc; +use alloc::vec::Vec; +use core::any::Any; +use system_error::SystemError; + +#[derive(Debug)] +pub struct BpfProg { + meta: BpfProgMeta, + raw_file_ptr: Vec, +} + +impl BpfProg { + pub fn new(meta: BpfProgMeta) -> Self { + Self { + meta, + raw_file_ptr: Vec::new(), + } + } + + pub fn insns(&self) -> &[u8] { + &self.meta.insns + } + + pub fn insns_mut(&mut self) -> &mut [u8] { + &mut self.meta.insns + } + + pub fn insert_map(&mut self, map_ptr: usize) { + self.raw_file_ptr.push(map_ptr); + } +} + +impl IndexNode for BpfProg { + fn open(&self, _data: SpinLockGuard, _mode: &FileMode) -> Result<()> { + Ok(()) + } + fn close(&self, _data: SpinLockGuard) -> Result<()> { + Ok(()) + } + fn read_at( + &self, + _offset: usize, + _len: usize, + _buf: &mut [u8], + _data: SpinLockGuard, + ) -> Result { + Err(SystemError::ENOSYS) + } + + fn write_at( + &self, + _offset: usize, + _len: usize, + _buf: &[u8], + _data: SpinLockGuard, + ) -> Result { + Err(SystemError::ENOSYS) + } + + fn metadata(&self) -> Result { + let meta = Metadata { + mode: ModeType::from_bits_truncate(0o755), + file_type: FileType::File, + ..Default::default() + }; + Ok(meta) + } + + fn resize(&self, _len: usize) -> Result<()> { + Ok(()) + } + + fn fs(&self) -> Arc { + panic!("BpfProg does not have a filesystem") + } + + fn as_any_ref(&self) -> &dyn Any { + self + } + + fn list(&self) -> Result> { + Err(SystemError::ENOSYS) + } +} + +impl Drop for BpfProg { + fn drop(&mut self) { + unsafe { + for ptr in self.raw_file_ptr.iter() { + let file = Arc::from_raw(*ptr as *const u8 as *const BpfMap); + drop(file) + } + } + } +} +/// Load a BPF program into the kernel. +/// +/// See https://ebpf-docs.dylanreimerink.nl/linux/syscall/BPF_PROG_LOAD/ +pub fn bpf_prog_load(attr: &bpf_attr) -> Result { + let args = BpfProgMeta::try_from(attr)?; + // info!("bpf_prog_load: {:#?}", args); + let log_info = BpfProgVerifierInfo::from(attr); + let prog = BpfProg::new(args); + let fd_table = ProcessManager::current_pcb().fd_table(); + let prog = BpfProgVerifier::new(prog, log_info.log_level, &mut []).verify(&fd_table)?; + let file = File::new(Arc::new(prog), FileMode::O_RDWR)?; + let fd = fd_table.write().alloc_fd(file, None).map(|x| x as usize)?; + Ok(fd) +} diff --git a/kernel/src/bpf/prog/util.rs b/kernel/src/bpf/prog/util.rs new file mode 100644 index 00000000..04aa47f3 --- /dev/null +++ b/kernel/src/bpf/prog/util.rs @@ -0,0 +1,112 @@ +use crate::include::bindings::linux_bpf::{bpf_attach_type, bpf_attr, bpf_prog_type}; +use crate::syscall::user_access::{check_and_clone_cstr, UserBufferReader}; +use alloc::string::{String, ToString}; +use alloc::vec::Vec; +use core::ffi::CStr; +use core::fmt::Debug; +use num_traits::FromPrimitive; +use system_error::SystemError; + +bitflags::bitflags! { + + pub struct VerifierLogLevel: u32 { + /// Sets no verifier logging. + const DISABLE = 0; + /// Enables debug verifier logging. + const DEBUG = 1; + /// Enables verbose verifier logging. + const VERBOSE = 2 | Self::DEBUG.bits(); + /// Enables verifier stats. + const STATS = 4; + } +} + +#[derive(Debug)] +pub struct BpfProgVerifierInfo { + /// This attribute specifies the level/detail of the log output. Valid values are. + pub log_level: VerifierLogLevel, + /// This attributes indicates the size of the memory region in bytes + /// indicated by `log_buf` which can safely be written to by the kernel. + pub _log_buf_size: u32, + /// This attributes can be set to a pointer to a memory region + /// allocated/reservedby the loader process where the verifier log will + /// be written to. + /// The detail of the log is set by log_level. The verifier log + /// is often the only indication in addition to the error code of + /// why the syscall command failed to load the program. + /// + /// The log is also written to on success. If the kernel runs out of + /// space in the buffer while loading, the loading process will fail + /// and the command will return with an error code of -ENOSPC. So it + /// is important to correctly size the buffer when enabling logging. + pub _log_buf_ptr: usize, +} + +impl From<&bpf_attr> for BpfProgVerifierInfo { + fn from(attr: &bpf_attr) -> Self { + unsafe { + let u = &attr.__bindgen_anon_3; + Self { + log_level: VerifierLogLevel::from_bits_truncate(u.log_level), + _log_buf_size: u.log_size, + _log_buf_ptr: u.log_buf as usize, + } + } + } +} + +pub struct BpfProgMeta { + pub prog_flags: u32, + pub prog_type: bpf_prog_type, + pub expected_attach_type: bpf_attach_type, + pub insns: Vec, + pub license: String, + pub kern_version: u32, + pub name: String, +} + +impl Debug for BpfProgMeta { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("BpfProgMeta") + .field("prog_flags", &self.prog_flags) + .field("prog_type", &self.prog_type) + .field("expected_attach_type", &self.expected_attach_type) + .field("insns_len", &(self.insns.len() / 8)) + .field("license", &self.license) + .field("kern_version", &self.kern_version) + .field("name", &self.name) + .finish() + } +} + +impl TryFrom<&bpf_attr> for BpfProgMeta { + type Error = SystemError; + fn try_from(attr: &bpf_attr) -> Result { + let u = unsafe { &attr.__bindgen_anon_3 }; + let prog_type = bpf_prog_type::from_u32(u.prog_type).ok_or(SystemError::EINVAL)?; + let expected_attach_type = + bpf_attach_type::from_u32(u.expected_attach_type).ok_or(SystemError::EINVAL)?; + unsafe { + let insns_buf = + UserBufferReader::new(u.insns as *mut u8, u.insn_cnt as usize * 8, true)?; + let insns = insns_buf.read_from_user::(0)?.to_vec(); + let name_slice = + core::slice::from_raw_parts(u.prog_name.as_ptr() as *const u8, u.prog_name.len()); + let prog_name = CStr::from_bytes_until_nul(name_slice) + .map_err(|_| SystemError::EINVAL)? + .to_str() + .map_err(|_| SystemError::EINVAL)? + .to_string(); + let license = check_and_clone_cstr(u.license as *const u8, None)?; + Ok(Self { + prog_flags: u.prog_flags, + prog_type, + expected_attach_type, + insns, + license: license.into_string().map_err(|_| SystemError::EINVAL)?, + kern_version: u.kern_version, + name: prog_name, + }) + } + } +} diff --git a/kernel/src/bpf/prog/verifier.rs b/kernel/src/bpf/prog/verifier.rs new file mode 100644 index 00000000..97cb1542 --- /dev/null +++ b/kernel/src/bpf/prog/verifier.rs @@ -0,0 +1,131 @@ +use super::super::Result; +use crate::bpf::map::BpfMap; +use crate::bpf::prog::util::VerifierLogLevel; +use crate::bpf::prog::BpfProg; +use crate::filesystem::vfs::file::FileDescriptorVec; +use crate::include::bindings::linux_bpf::*; +use crate::libs::casting::DowncastArc; +use crate::libs::rwlock::RwLock; +use alloc::{sync::Arc, vec::Vec}; +use log::{error, info}; +use rbpf::ebpf; +use rbpf::ebpf::to_insn_vec; +use system_error::SystemError; + +/// The BPF program verifier. +/// +/// See https://docs.kernel.org/bpf/verifier.html +#[derive(Debug)] +pub struct BpfProgVerifier<'a> { + prog: BpfProg, + _log_level: VerifierLogLevel, + _log_buf: &'a mut [u8], +} + +impl<'a> BpfProgVerifier<'a> { + pub fn new(prog: BpfProg, log_level: VerifierLogLevel, log_buf: &'a mut [u8]) -> Self { + Self { + prog, + _log_level: log_level, + _log_buf: log_buf, + } + } + /// Relocate the program. + /// + /// This function will relocate the program, and update the program's instructions. + fn relocation(&mut self, fd_table: &Arc>) -> Result<()> { + let instructions = self.prog.insns_mut(); + let mut fmt_insn = to_insn_vec(instructions); + let mut index = 0; + let mut raw_file_ptr = vec![]; + loop { + if index >= fmt_insn.len() { + break; + } + let mut insn = fmt_insn[index].clone(); + if insn.opc == ebpf::LD_DW_IMM { + // relocate the instruction + let mut next_insn = fmt_insn[index + 1].clone(); + // the imm is the map_fd because user lib has already done the relocation + let map_fd = insn.imm as usize; + let src_reg = insn.src; + // See https://www.kernel.org/doc/html/latest/bpf/standardization/instruction-set.html#id23 + let ptr = match src_reg as u32 { + BPF_PSEUDO_MAP_VALUE => { + // dst = map_val(map_by_fd(imm)) + next_imm + // map_val(map) gets the address of the first value in a given map + let file = fd_table + .read() + .get_file_by_fd(map_fd as i32) + .ok_or(SystemError::EBADF)?; + let bpf_map = file + .inode() + .downcast_arc::() + .ok_or(SystemError::EINVAL)?; + let first_value_ptr = + bpf_map.inner_map().lock().first_value_ptr()? as usize; + let offset = next_insn.imm as usize; + info!( + "Relocate for BPF_PSEUDO_MAP_VALUE, instruction index: {}, map_fd: {}", + index, map_fd + ); + Some(first_value_ptr + offset) + } + BPF_PSEUDO_MAP_FD => { + // dst = map_by_fd(imm) + // map_by_fd(imm) means to convert a 32-bit file descriptor into an address of a map + let bpf_map = fd_table + .read() + .get_file_by_fd(map_fd as i32) + .ok_or(SystemError::EBADF)? + .inode() + .downcast_arc::() + .ok_or(SystemError::EINVAL)?; + // todo!(warning: We need release after prog unload) + let map_ptr = Arc::into_raw(bpf_map) as usize; + info!( + "Relocate for BPF_PSEUDO_MAP_FD, instruction index: {}, map_fd: {}, ptr: {:#x}", + index, map_fd, map_ptr + ); + raw_file_ptr.push(map_ptr); + Some(map_ptr) + } + ty => { + error!( + "relocation for ty: {} not implemented, instruction index: {}", + ty, index + ); + None + } + }; + if let Some(ptr) = ptr { + // The current ins store the map_data_ptr low 32 bits, + // the next ins store the map_data_ptr high 32 bits + insn.imm = ptr as i32; + next_insn.imm = (ptr >> 32) as i32; + fmt_insn[index] = insn; + fmt_insn[index + 1] = next_insn; + index += 2; + } else { + index += 1; + } + } else { + index += 1; + } + } + let fmt_insn = fmt_insn + .iter() + .flat_map(|ins| ins.to_vec()) + .collect::>(); + instructions.copy_from_slice(&fmt_insn); + for ptr in raw_file_ptr { + self.prog.insert_map(ptr); + } + Ok(()) + } + + pub fn verify(mut self, fd_table: &Arc>) -> Result { + self.relocation(fd_table)?; + Ok(self.prog) + } +} diff --git a/kernel/src/common/string.h b/kernel/src/common/string.h index 81201624..cd7032ee 100644 --- a/kernel/src/common/string.h +++ b/kernel/src/common/string.h @@ -11,3 +11,12 @@ static inline int strlen(const char *s) { } return __res; } + + +static inline int strcmp(const char *s1, const char *s2) { + while (*s1 && *s2 && *s1 == *s2) { + ++s1; + ++s2; + } + return *s1 - *s2; +} \ No newline at end of file diff --git a/kernel/src/debug/kprobe/args.rs b/kernel/src/debug/kprobe/args.rs new file mode 100644 index 00000000..33ca4549 --- /dev/null +++ b/kernel/src/debug/kprobe/args.rs @@ -0,0 +1,66 @@ +use alloc::boxed::Box; +use alloc::string::String; +use kprobe::{CallBackFunc, KprobeBuilder, ProbeArgs}; +use log::warn; +use system_error::SystemError; + +pub struct KprobeInfo { + pub pre_handler: fn(&dyn ProbeArgs), + pub post_handler: fn(&dyn ProbeArgs), + pub fault_handler: Option, + pub event_callback: Option>, + pub symbol: Option, + pub addr: Option, + pub offset: usize, + pub enable: bool, +} + +extern "C" { + fn addr_from_symbol(symbol: *const u8) -> usize; +} + +impl TryFrom for KprobeBuilder { + type Error = SystemError; + fn try_from(kprobe_info: KprobeInfo) -> Result { + // 检查参数: symbol和addr必须有一个但不能同时有 + if kprobe_info.symbol.is_none() && kprobe_info.addr.is_none() { + return Err(SystemError::EINVAL); + } + if kprobe_info.symbol.is_some() && kprobe_info.addr.is_some() { + return Err(SystemError::EINVAL); + } + let func_addr = if let Some(symbol) = kprobe_info.symbol.clone() { + let mut symbol_sting = symbol; + if !symbol_sting.ends_with("\0") { + symbol_sting.push('\0'); + } + let symbol = symbol_sting.as_ptr(); + let func_addr = unsafe { addr_from_symbol(symbol) }; + if func_addr == 0 { + warn!( + "register_kprobe: the symbol: {:?} not found", + kprobe_info.symbol + ); + return Err(SystemError::ENXIO); + } + func_addr + } else { + kprobe_info.addr.unwrap() + }; + let mut builder = KprobeBuilder::new( + kprobe_info.symbol, + func_addr, + kprobe_info.offset, + kprobe_info.pre_handler, + kprobe_info.post_handler, + kprobe_info.enable, + ); + if let Some(fault_handler) = kprobe_info.fault_handler { + builder = builder.with_fault_handler(fault_handler); + } + if let Some(event_callback) = kprobe_info.event_callback { + builder = builder.with_event_callback(event_callback); + } + Ok(builder) + } +} diff --git a/kernel/src/debug/kprobe/mod.rs b/kernel/src/debug/kprobe/mod.rs new file mode 100644 index 00000000..7dc554f8 --- /dev/null +++ b/kernel/src/debug/kprobe/mod.rs @@ -0,0 +1,183 @@ +use crate::debug::kprobe::args::KprobeInfo; +use crate::libs::rwlock::RwLock; +use crate::libs::spinlock::SpinLock; +use alloc::collections::BTreeMap; +use alloc::sync::Arc; +use alloc::vec::Vec; +use kprobe::{Kprobe, KprobeBuilder, KprobeOps, KprobePoint}; +use system_error::SystemError; + +pub mod args; +#[cfg(feature = "kprobe_test")] +mod test; + +pub type LockKprobe = Arc>; +pub static KPROBE_MANAGER: SpinLock = SpinLock::new(KprobeManager::new()); +static KPROBE_POINT_LIST: SpinLock>> = + SpinLock::new(BTreeMap::new()); + +/// 管理所有的kprobe探测点 +#[derive(Debug, Default)] +pub struct KprobeManager { + break_list: BTreeMap>, + debug_list: BTreeMap>, +} + +impl KprobeManager { + pub const fn new() -> Self { + KprobeManager { + break_list: BTreeMap::new(), + debug_list: BTreeMap::new(), + } + } + /// # 插入一个kprobe + /// + /// ## 参数 + /// - `kprobe`: kprobe的实例 + pub fn insert_kprobe(&mut self, kprobe: LockKprobe) { + let probe_point = kprobe.read().probe_point().clone(); + self.insert_break_point(probe_point.break_address(), kprobe.clone()); + self.insert_debug_point(probe_point.debug_address(), kprobe); + } + + /// # 向break_list中插入一个kprobe + /// + /// ## 参数 + /// - `address`: kprobe的地址, 由`KprobePoint::break_address()`或者`KprobeBuilder::probe_addr()`返回 + /// - `kprobe`: kprobe的实例 + fn insert_break_point(&mut self, address: usize, kprobe: LockKprobe) { + let list = self.break_list.entry(address).or_default(); + list.push(kprobe); + } + + /// # 向debug_list中插入一个kprobe + /// + /// ## 参数 + /// - `address`: kprobe的单步执行地址，由`KprobePoint::debug_address()`返回 + /// - `kprobe`: kprobe的实例 + fn insert_debug_point(&mut self, address: usize, kprobe: LockKprobe) { + let list = self.debug_list.entry(address).or_default(); + list.push(kprobe); + } + + pub fn get_break_list(&self, address: usize) -> Option<&Vec> { + self.break_list.get(&address) + } + + pub fn get_debug_list(&self, address: usize) -> Option<&Vec> { + self.debug_list.get(&address) + } + + /// # 返回一个地址上注册的kprobe数量 + /// + /// ## 参数 + /// - `address`: kprobe的地址, 由`KprobePoint::break_address()`或者`KprobeBuilder::probe_addr()`返回 + pub fn kprobe_num(&self, address: usize) -> usize { + self.break_list_len(address) + } + + #[inline] + fn break_list_len(&self, address: usize) -> usize { + self.break_list + .get(&address) + .map(|list| list.len()) + .unwrap_or(0) + } + #[inline] + fn debug_list_len(&self, address: usize) -> usize { + self.debug_list + .get(&address) + .map(|list| list.len()) + .unwrap_or(0) + } + + /// # 移除一个kprobe + /// + /// ## 参数 + /// - `kprobe`: kprobe的实例 + pub fn remove_kprobe(&mut self, kprobe: &LockKprobe) { + let probe_point = kprobe.read().probe_point().clone(); + self.remove_one_break(probe_point.break_address(), kprobe); + self.remove_one_debug(probe_point.debug_address(), kprobe); + } + + /// # 从break_list中移除一个kprobe + /// + /// 如果没有其他kprobe注册在这个地址上，则删除列表 + /// + /// ## 参数 + /// - `address`: kprobe的地址, 由`KprobePoint::break_address()`或者`KprobeBuilder::probe_addr()`返回 + /// - `kprobe`: kprobe的实例 + fn remove_one_break(&mut self, address: usize, kprobe: &LockKprobe) { + if let Some(list) = self.break_list.get_mut(&address) { + list.retain(|x| !Arc::ptr_eq(x, kprobe)); + } + if self.break_list_len(address) == 0 { + self.break_list.remove(&address); + } + } + + /// # 从debug_list中移除一个kprobe + /// + /// 如果没有其他kprobe注册在这个地址上，则删除列表 + /// + /// ## 参数 + /// - `address`: kprobe的单步执行地址，由`KprobePoint::debug_address()`返回 + /// - `kprobe`: kprobe的实例 + fn remove_one_debug(&mut self, address: usize, kprobe: &LockKprobe) { + if let Some(list) = self.debug_list.get_mut(&address) { + list.retain(|x| !Arc::ptr_eq(x, kprobe)); + } + if self.debug_list_len(address) == 0 { + self.debug_list.remove(&address); + } + } +} + +#[cfg(feature = "kprobe_test")] +#[allow(unused)] +/// This function is only used for testing kprobe +pub fn kprobe_test() { + test::kprobe_test(); +} + +/// # 注册一个kprobe +/// +/// 该函数会根据`symbol`查找对应的函数地址，如果找不到则返回错误。 +/// +/// ## 参数 +/// - `kprobe_info`: kprobe的信息 +pub fn register_kprobe(kprobe_info: KprobeInfo) -> Result { + let kprobe_builder = KprobeBuilder::try_from(kprobe_info)?; + let address = kprobe_builder.probe_addr(); + let existed_point = KPROBE_POINT_LIST.lock().get(&address).map(Clone::clone); + let kprobe = match existed_point { + Some(existed_point) => { + kprobe_builder + .with_probe_point(existed_point.clone()) + .install() + .0 + } + None => { + let (kprobe, probe_point) = kprobe_builder.install(); + KPROBE_POINT_LIST.lock().insert(address, probe_point); + kprobe + } + }; + let kprobe = Arc::new(RwLock::new(kprobe)); + KPROBE_MANAGER.lock().insert_kprobe(kprobe.clone()); + Ok(kprobe) +} + +/// # 注销一个kprobe +/// +/// ## 参数 +/// - `kprobe`: 已安装的kprobe +pub fn unregister_kprobe(kprobe: LockKprobe) { + let kprobe_addr = kprobe.read().probe_point().break_address(); + KPROBE_MANAGER.lock().remove_kprobe(&kprobe); + // 如果没有其他kprobe注册在这个地址上，则删除探测点 + if KPROBE_MANAGER.lock().kprobe_num(kprobe_addr) == 0 { + KPROBE_POINT_LIST.lock().remove(&kprobe_addr); + } +} diff --git a/kernel/src/debug/kprobe/test.rs b/kernel/src/debug/kprobe/test.rs new file mode 100644 index 00000000..f23d0e98 --- /dev/null +++ b/kernel/src/debug/kprobe/test.rs @@ -0,0 +1,84 @@ +use crate::arch::interrupt::TrapFrame; +use crate::debug::kprobe::{register_kprobe, unregister_kprobe, KprobeInfo}; +use alloc::string::ToString; +use kprobe::ProbeArgs; +use log::info; + +#[inline(never)] +fn detect_func(x: usize, y: usize) -> usize { + let hart = 0; + info!("detect_func: hart_id: {}, x: {}, y:{}", hart, x, y); + hart +} + +fn pre_handler(regs: &dyn ProbeArgs) { + let pt_regs = regs.as_any().downcast_ref::().unwrap(); + info!( + "call pre_handler, the sp is {:#x}", + pt_regs as *const _ as usize + ); +} + +fn post_handler(regs: &dyn ProbeArgs) { + let pt_regs = regs.as_any().downcast_ref::().unwrap(); + info!( + "call post_handler, the sp is {:#x}", + pt_regs as *const _ as usize + ); +} + +fn fault_handler(regs: &dyn ProbeArgs) { + let pt_regs = regs.as_any().downcast_ref::().unwrap(); + info!( + "call fault_handler, the sp is {:#x}", + pt_regs as *const _ as usize + ); +} + +pub fn kprobe_test() { + info!("kprobe test for [detect_func]: {:#x}", detect_func as usize); + let kprobe_info = KprobeInfo { + pre_handler, + post_handler, + fault_handler: Some(fault_handler), + event_callback: None, + symbol: None, + addr: Some(detect_func as usize), + offset: 0, + enable: true, + }; + let kprobe = register_kprobe(kprobe_info).unwrap(); + + let new_pre_handler = |regs: &dyn ProbeArgs| { + let pt_regs = regs.as_any().downcast_ref::().unwrap(); + info!( + "call new pre_handler, the sp is {:#x}", + pt_regs as *const _ as usize + ); + }; + + let kprobe_info = KprobeInfo { + pre_handler: new_pre_handler, + post_handler, + fault_handler: Some(fault_handler), + event_callback: None, + symbol: Some("dragonos_kernel::debug::kprobe::test::detect_func".to_string()), + addr: None, + offset: 0, + enable: true, + }; + let kprobe2 = register_kprobe(kprobe_info).unwrap(); + info!( + "install 2 kprobes at [detect_func]: {:#x}", + detect_func as usize + ); + detect_func(1, 2); + unregister_kprobe(kprobe); + unregister_kprobe(kprobe2); + info!( + "uninstall 2 kprobes at [detect_func]: {:#x}", + detect_func as usize + ); + detect_func(1, 2); + info!("kprobe test end"); +} diff --git a/kernel/src/debug/mod.rs b/kernel/src/debug/mod.rs index 5516f607..b24c48bc 100644 --- a/kernel/src/debug/mod.rs +++ b/kernel/src/debug/mod.rs @@ -1 +1,2 @@ pub mod klog; +pub mod kprobe; diff --git a/kernel/src/debug/traceback/traceback.c b/kernel/src/debug/traceback/traceback.c index d9474eb6..9a2fb7e7 100644 --- a/kernel/src/debug/traceback/traceback.c +++ b/kernel/src/debug/traceback/traceback.c @@ -1,5 +1,6 @@ #include "traceback.h" #include +#include #include int lookup_kallsyms(uint64_t addr, int level) @@ -26,6 +27,18 @@ int lookup_kallsyms(uint64_t addr, int level) return -1; } +uint64_t addr_from_symbol(const char *symbol) +{ + const char *str = (const char *)&kallsyms_names; + for (uint64_t i = 0; i < kallsyms_num; ++i) + { + if (strcmp(&str[kallsyms_names_index[i]], symbol) == 0) + return kallsyms_address[i]; + } + return 0; + +} + /** * @brief 追溯内核栈调用情况 * diff --git a/kernel/src/debug/traceback/traceback.h b/kernel/src/debug/traceback/traceback.h index 837cb502..d2b6274c 100644 --- a/kernel/src/debug/traceback/traceback.h +++ b/kernel/src/debug/traceback/traceback.h @@ -14,4 +14,5 @@ extern const char *kallsyms_names __attribute__((weak)); * * @param regs 内核栈结构体 */ -void traceback(struct pt_regs *regs); \ No newline at end of file +void traceback(struct pt_regs *regs); +uint64_t addr_from_symbol(const char *symbol); \ No newline at end of file diff --git a/kernel/src/driver/base/cpu.rs b/kernel/src/driver/base/cpu.rs index 08486730..29e64d7d 100644 --- a/kernel/src/driver/base/cpu.rs +++ b/kernel/src/driver/base/cpu.rs @@ -11,8 +11,6 @@ use crate::{ libs::rwlock::{RwLock, RwLockReadGuard, RwLockWriteGuard}, }; -use system_error::SystemError; - use super::{ class::Class, device::{ @@ -24,29 +22,36 @@ use super::{ kset::KSet, subsys::SubSysPrivate, }; +use crate::filesystem::sysfs::file::sysfs_emit_str; +use crate::filesystem::sysfs::{Attribute, AttributeGroup, SysFSOpsSupport}; +use crate::filesystem::vfs::syscall::ModeType; +use crate::libs::lazy_init::Lazy; +use system_error::SystemError; -#[inline(always)] -pub fn cpu_device_manager() -> &'static CpuDeviceManager { - return &CpuDeviceManager; -} +static CPU_DEVICE_MANAGER: Lazy = Lazy::new(); #[derive(Debug)] -pub struct CpuDeviceManager; +pub struct CpuDeviceManager { + _root_device: Arc, +} impl CpuDeviceManager { /// 初始化设备驱动模型的CPU子系统 /// /// 参考 https://code.dragonos.org.cn/xref/linux-6.1.9/drivers/base/cpu.c?fi=get_cpu_device#622 - pub fn init(&self) -> Result<(), SystemError> { + pub fn init() -> Result<(), SystemError> { let cpu_subsys = CpuSubSystem::new(); let root_device = CpuSubSystemFakeRootDevice::new(); subsystem_manager() .subsys_system_register( &(cpu_subsys as Arc), - &(root_device as Arc), + &(root_device.clone() as Arc), ) .expect("register cpu subsys failed"); - + let manager = Self { + _root_device: root_device, + }; + CPU_DEVICE_MANAGER.init(manager); return Ok(()); } } @@ -190,6 +195,10 @@ impl Device for CpuSubSystemFakeRootDevice { fn set_dev_parent(&self, dev_parent: Option>) { self.inner.write().device_common.parent = dev_parent; } + + fn attribute_groups(&self) -> Option<&'static [&'static dyn AttributeGroup]> { + Some(&[&AttrGroupCpu]) + } } impl KObject for CpuSubSystemFakeRootDevice { @@ -249,3 +258,70 @@ impl KObject for CpuSubSystemFakeRootDevice { *self.kobj_state_mut() = state; } } + +#[derive(Debug)] +pub struct AttrGroupCpu; + +impl AttributeGroup for AttrGroupCpu { + fn name(&self) -> Option<&str> { + None + } + fn attrs(&self) -> &[&'static dyn Attribute] { + &[&AttrCpuPossible, &AttrCpuOnline] + } + fn is_visible( + &self, + _kobj: Arc, + _attr: &'static dyn Attribute, + ) -> Option { + None + } +} + +#[derive(Debug)] +pub struct AttrCpuPossible; + +impl Attribute for AttrCpuPossible { + fn name(&self) -> &str { + "possible" + } + + fn mode(&self) -> ModeType { + ModeType::S_IRUGO + } + + fn support(&self) -> SysFSOpsSupport { + SysFSOpsSupport::ATTR_SHOW + } + + fn show(&self, _kobj: Arc, buf: &mut [u8]) -> Result { + let cpu_manager = crate::smp::cpu::smp_cpu_manager(); + let cpus = cpu_manager.possible_cpus_count(); + let data = format!("0-{}", cpus - 1); + sysfs_emit_str(buf, &data) + } +} + +#[derive(Debug)] +pub struct AttrCpuOnline; + +impl Attribute for AttrCpuOnline { + fn name(&self) -> &str { + "online" + } + + fn mode(&self) -> ModeType { + ModeType::S_IRUGO + } + + fn support(&self) -> SysFSOpsSupport { + SysFSOpsSupport::ATTR_SHOW + } + + fn show(&self, _kobj: Arc, buf: &mut [u8]) -> Result { + let cpu_manager = crate::smp::cpu::smp_cpu_manager(); + let cpus = cpu_manager.present_cpus_count(); + let data = format!("0-{}", cpus - 1); + sysfs_emit_str(buf, &data) + } +} diff --git a/kernel/src/driver/base/init.rs b/kernel/src/driver/base/init.rs index c28119ef..93a32d81 100644 --- a/kernel/src/driver/base/init.rs +++ b/kernel/src/driver/base/init.rs @@ -3,7 +3,7 @@ use system_error::SystemError; use super::{ class::classes_init, - cpu::cpu_device_manager, + cpu::CpuDeviceManager, device::{bus::buses_init, init::devices_init}, firmware::firmware_init, hypervisor::hypervisor_init, @@ -20,7 +20,7 @@ pub fn driver_init() -> Result<(), SystemError> { hypervisor_init()?; platform_bus_init()?; serio_bus_init()?; - cpu_device_manager().init()?; + CpuDeviceManager::init()?; // 至此，已完成设备驱动模型的初始化 return Ok(()); diff --git a/kernel/src/exception/debug.rs b/kernel/src/exception/debug.rs new file mode 100644 index 00000000..b970d537 --- /dev/null +++ b/kernel/src/exception/debug.rs @@ -0,0 +1,33 @@ +use crate::arch::interrupt::TrapFrame; +use crate::arch::kprobe::clear_single_step; +use crate::debug::kprobe::KPROBE_MANAGER; +use kprobe::{KprobeOps, ProbeArgs}; +use log::debug; +use system_error::SystemError; + +#[derive(Debug)] +pub struct DebugException; + +impl DebugException { + pub fn handle(frame: &mut TrapFrame) -> Result<(), SystemError> { + Self::post_kprobe_handler(frame) + } + + fn post_kprobe_handler(frame: &mut TrapFrame) -> Result<(), SystemError> { + let pc = frame.debug_address(); + if let Some(kprobe_list) = KPROBE_MANAGER.lock().get_debug_list(pc) { + for kprobe in kprobe_list { + let guard = kprobe.read(); + if guard.is_enabled() { + guard.call_post_handler(frame); + guard.call_event_callback(frame); + } + } + let return_address = kprobe_list[0].read().probe_point().return_address(); + clear_single_step(frame, return_address); + } else { + debug!("There is no kprobe on pc {:#x}", pc); + } + Ok(()) + } +} diff --git a/kernel/src/exception/ebreak.rs b/kernel/src/exception/ebreak.rs new file mode 100644 index 00000000..24adfa13 --- /dev/null +++ b/kernel/src/exception/ebreak.rs @@ -0,0 +1,37 @@ +use crate::arch::interrupt::TrapFrame; +use crate::arch::kprobe::setup_single_step; +use crate::debug::kprobe::KPROBE_MANAGER; +use crate::exception::debug::DebugException; +use kprobe::{KprobeOps, ProbeArgs}; +use system_error::SystemError; + +#[derive(Debug)] +pub struct EBreak; + +impl EBreak { + pub fn handle(frame: &mut TrapFrame) -> Result<(), SystemError> { + Self::kprobe_handler(frame) + } + fn kprobe_handler(frame: &mut TrapFrame) -> Result<(), SystemError> { + let break_addr = frame.break_address(); + let guard = KPROBE_MANAGER.lock(); + let kprobe_list = guard.get_break_list(break_addr); + if let Some(kprobe_list) = kprobe_list { + for kprobe in kprobe_list { + let guard = kprobe.read(); + if guard.is_enabled() { + guard.call_pre_handler(frame); + } + } + let single_step_address = kprobe_list[0].read().probe_point().single_step_address(); + // setup_single_step + setup_single_step(frame, single_step_address); + } else { + // For some architectures, they do not support single step execution, + // and we need to use breakpoint exceptions to simulate + drop(guard); + DebugException::handle(frame)?; + } + Ok(()) + } +} diff --git a/kernel/src/exception/mod.rs b/kernel/src/exception/mod.rs index c7df823e..8eb14dd6 100644 --- a/kernel/src/exception/mod.rs +++ b/kernel/src/exception/mod.rs @@ -4,7 +4,9 @@ use system_error::SystemError; use crate::arch::CurrentIrqArch; +pub mod debug; pub mod dummychip; +pub mod ebreak; pub mod handle; pub mod init; pub mod ipi; diff --git a/kernel/src/filesystem/vfs/file.rs b/kernel/src/filesystem/vfs/file.rs index d15167ba..4b3d75b6 100644 --- a/kernel/src/filesystem/vfs/file.rs +++ b/kernel/src/filesystem/vfs/file.rs @@ -11,6 +11,8 @@ use system_error::SystemError; use super::{Dirent, FileType, IndexNode, InodeId, Metadata, SpecialNodeData}; use crate::filesystem::eventfd::EventFdInode; +use crate::libs::lazy_init::Lazy; +use crate::perf::PerfEventInode; use crate::{ arch::MMArch, driver::{ @@ -125,7 +127,7 @@ impl FileMode { /// 页面缓存 pub struct PageCache { xarray: SpinLock>>, - inode: Option>, + inode: Lazy>, } impl core::fmt::Debug for PageCache { @@ -148,13 +150,19 @@ impl PageCache { pub fn new(inode: Option>) -> Arc { let page_cache = Self { xarray: SpinLock::new(XArray::new()), - inode, + inode: { + let v: Lazy> = Lazy::new(); + if let Some(inode) = inode { + v.init(inode); + } + v + }, }; Arc::new(page_cache) } pub fn inode(&self) -> Option> { - self.inode.clone() + self.inode.try_get().cloned() } pub fn add_page(&self, offset: usize, page: &Arc) { @@ -176,8 +184,12 @@ impl PageCache { cursor.remove(); } - pub fn set_inode(&mut self, inode: Weak) { - self.inode = Some(inode) + pub fn set_inode(&self, inode: Weak) -> Result<(), SystemError> { + if self.inode.initialized() { + return Err(SystemError::EINVAL); + } + self.inode.init(inode); + Ok(()) } } @@ -603,11 +615,15 @@ impl File { inode.inner().lock().remove_epoll(epoll) } _ => { + let inode = self.inode.downcast_ref::(); + if let Some(inode) = inode { + return inode.remove_epoll(epoll); + } let inode = self .inode - .downcast_ref::() + .downcast_ref::() .ok_or(SystemError::ENOSYS)?; - inode.remove_epoll(epoll) + return inode.remove_epoll(epoll); } } } @@ -745,7 +761,6 @@ impl FileDescriptorVec { // 把文件描述符数组对应位置设置为空 let file = self.fds[fd as usize].take().unwrap(); - return Ok(file); } diff --git a/kernel/src/filesystem/vfs/mod.rs b/kernel/src/filesystem/vfs/mod.rs index 00ce6ba5..51d2c422 100644 --- a/kernel/src/filesystem/vfs/mod.rs +++ b/kernel/src/filesystem/vfs/mod.rs @@ -125,6 +125,9 @@ bitflags! { } pub trait IndexNode: Any + Sync + Send + Debug + CastFromSync { + fn mmap(&self, _start: usize, _len: usize, _offset: usize) -> Result<(), SystemError> { + return Err(SystemError::ENOSYS); + } /// @brief 打开文件 /// /// @return 成功：Ok() diff --git a/kernel/src/include/bindings/linux_bpf.rs b/kernel/src/include/bindings/linux_bpf.rs new file mode 100644 index 00000000..e3b2b79d --- /dev/null +++ b/kernel/src/include/bindings/linux_bpf.rs @@ -0,0 +1,2430 @@ +/* automatically generated by rust-bindgen 0.69.4 */ + +#[repr(C)] +#[derive(Copy, Clone, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct __BindgenBitfieldUnit { + storage: Storage, +} +impl __BindgenBitfieldUnit { + #[inline] + pub const fn new(storage: Storage) -> Self { + Self { storage } + } +} +impl __BindgenBitfieldUnit +where + Storage: AsRef<[u8]> + AsMut<[u8]>, +{ + #[inline] + pub fn get_bit(&self, index: usize) -> bool { + debug_assert!(index / 8 < self.storage.as_ref().len()); + let byte_index = index / 8; + let byte = self.storage.as_ref()[byte_index]; + let bit_index = if cfg!(target_endian = "big") { + 7 - (index % 8) + } else { + index % 8 + }; + let mask = 1 << bit_index; + byte & mask == mask + } + #[inline] + pub fn set_bit(&mut self, index: usize, val: bool) { + debug_assert!(index / 8 < self.storage.as_ref().len()); + let byte_index = index / 8; + let byte = &mut self.storage.as_mut()[byte_index]; + let bit_index = if cfg!(target_endian = "big") { + 7 - (index % 8) + } else { + index % 8 + }; + let mask = 1 << bit_index; + if val { + *byte |= mask; + } else { + *byte &= !mask; + } + } + #[inline] + pub fn get(&self, bit_offset: usize, bit_width: u8) -> u64 { + debug_assert!(bit_width <= 64); + debug_assert!(bit_offset / 8 < self.storage.as_ref().len()); + debug_assert!((bit_offset + (bit_width as usize)) / 8 <= self.storage.as_ref().len()); + let mut val = 0; + for i in 0..(bit_width as usize) { + if self.get_bit(i + bit_offset) { + let index = if cfg!(target_endian = "big") { + bit_width as usize - 1 - i + } else { + i + }; + val |= 1 << index; + } + } + val + } + #[inline] + pub fn set(&mut self, bit_offset: usize, bit_width: u8, val: u64) { + debug_assert!(bit_width <= 64); + debug_assert!(bit_offset / 8 < self.storage.as_ref().len()); + debug_assert!((bit_offset + (bit_width as usize)) / 8 <= self.storage.as_ref().len()); + for i in 0..(bit_width as usize) { + let mask = 1 << i; + let val_bit_is_set = val & mask == mask; + let index = if cfg!(target_endian = "big") { + bit_width as usize - 1 - i + } else { + i + }; + self.set_bit(index + bit_offset, val_bit_is_set); + } + } +} +#[repr(C)] +#[derive(Default)] +pub struct __IncompleteArrayField(::core::marker::PhantomData, [T; 0]); +impl __IncompleteArrayField { + #[inline] + pub const fn new() -> Self { + __IncompleteArrayField(::core::marker::PhantomData, []) + } + #[inline] + pub fn as_ptr(&self) -> *const T { + self as *const _ as *const T + } + #[inline] + pub fn as_mut_ptr(&mut self) -> *mut T { + self as *mut _ as *mut T + } + #[inline] + pub unsafe fn as_slice(&self, len: usize) -> &[T] { + ::core::slice::from_raw_parts(self.as_ptr(), len) + } + #[inline] + pub unsafe fn as_mut_slice(&mut self, len: usize) -> &mut [T] { + ::core::slice::from_raw_parts_mut(self.as_mut_ptr(), len) + } +} +impl ::core::fmt::Debug for __IncompleteArrayField { + fn fmt(&self, fmt: &mut ::core::fmt::Formatter<'_>) -> ::core::fmt::Result { + fmt.write_str("__IncompleteArrayField") + } +} +pub const SO_ATTACH_BPF: u32 = 50; +pub const SO_DETACH_BPF: u32 = 27; +pub const BPF_LD: u32 = 0; +pub const BPF_LDX: u32 = 1; +pub const BPF_ST: u32 = 2; +pub const BPF_STX: u32 = 3; +pub const BPF_ALU: u32 = 4; +pub const BPF_JMP: u32 = 5; +pub const BPF_W: u32 = 0; +pub const BPF_H: u32 = 8; +pub const BPF_B: u32 = 16; +pub const BPF_K: u32 = 0; +pub const BPF_ALU64: u32 = 7; +pub const BPF_DW: u32 = 24; +pub const BPF_CALL: u32 = 128; +pub const BPF_F_ALLOW_OVERRIDE: u32 = 1; +pub const BPF_F_ALLOW_MULTI: u32 = 2; +pub const BPF_F_REPLACE: u32 = 4; +pub const BPF_F_BEFORE: u32 = 8; +pub const BPF_F_AFTER: u32 = 16; +pub const BPF_F_ID: u32 = 32; +pub const BPF_F_STRICT_ALIGNMENT: u32 = 1; +pub const BPF_F_ANY_ALIGNMENT: u32 = 2; +pub const BPF_F_TEST_RND_HI32: u32 = 4; +pub const BPF_F_TEST_STATE_FREQ: u32 = 8; +pub const BPF_F_SLEEPABLE: u32 = 16; +pub const BPF_F_XDP_HAS_FRAGS: u32 = 32; +pub const BPF_F_XDP_DEV_BOUND_ONLY: u32 = 64; +pub const BPF_F_TEST_REG_INVARIANTS: u32 = 128; +pub const BPF_F_NETFILTER_IP_DEFRAG: u32 = 1; +pub const BPF_PSEUDO_MAP_FD: u32 = 1; +pub const BPF_PSEUDO_MAP_IDX: u32 = 5; +pub const BPF_PSEUDO_MAP_VALUE: u32 = 2; +pub const BPF_PSEUDO_MAP_IDX_VALUE: u32 = 6; +pub const BPF_PSEUDO_BTF_ID: u32 = 3; +pub const BPF_PSEUDO_FUNC: u32 = 4; +pub const BPF_PSEUDO_CALL: u32 = 1; +pub const BPF_PSEUDO_KFUNC_CALL: u32 = 2; +pub const BPF_F_QUERY_EFFECTIVE: u32 = 1; +pub const BPF_F_TEST_RUN_ON_CPU: u32 = 1; +pub const BPF_F_TEST_XDP_LIVE_FRAMES: u32 = 2; +pub const BTF_INT_SIGNED: u32 = 1; +pub const BTF_INT_CHAR: u32 = 2; +pub const BTF_INT_BOOL: u32 = 4; +pub const NLMSG_ALIGNTO: u32 = 4; +pub const XDP_FLAGS_UPDATE_IF_NOEXIST: u32 = 1; +pub const XDP_FLAGS_SKB_MODE: u32 = 2; +pub const XDP_FLAGS_DRV_MODE: u32 = 4; +pub const XDP_FLAGS_HW_MODE: u32 = 8; +pub const XDP_FLAGS_REPLACE: u32 = 16; +pub const XDP_FLAGS_MODES: u32 = 14; +pub const XDP_FLAGS_MASK: u32 = 31; +pub const PERF_MAX_STACK_DEPTH: u32 = 127; +pub const PERF_MAX_CONTEXTS_PER_STACK: u32 = 8; +pub const PERF_FLAG_FD_NO_GROUP: u32 = 1; +pub const PERF_FLAG_FD_OUTPUT: u32 = 2; +pub const PERF_FLAG_PID_CGROUP: u32 = 4; +pub const PERF_FLAG_FD_CLOEXEC: u32 = 8; +pub const TC_H_MAJ_MASK: u32 = 4294901760; +pub const TC_H_MIN_MASK: u32 = 65535; +pub const TC_H_UNSPEC: u32 = 0; +pub const TC_H_ROOT: u32 = 4294967295; +pub const TC_H_INGRESS: u32 = 4294967281; +pub const TC_H_CLSACT: u32 = 4294967281; +pub const TC_H_MIN_PRIORITY: u32 = 65504; +pub const TC_H_MIN_INGRESS: u32 = 65522; +pub const TC_H_MIN_EGRESS: u32 = 65523; +pub const TCA_BPF_FLAG_ACT_DIRECT: u32 = 1; +pub type __u8 = ::core::ffi::c_uchar; +pub type __s16 = ::core::ffi::c_short; +pub type __u16 = ::core::ffi::c_ushort; +pub type __s32 = ::core::ffi::c_int; +pub type __u32 = ::core::ffi::c_uint; +pub type __s64 = ::core::ffi::c_longlong; +pub type __u64 = ::core::ffi::c_ulonglong; +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct bpf_insn { + pub code: __u8, + pub _bitfield_align_1: [u8; 0], + pub _bitfield_1: __BindgenBitfieldUnit<[u8; 1usize]>, + pub off: __s16, + pub imm: __s32, +} +impl bpf_insn { + #[inline] + pub fn dst_reg(&self) -> __u8 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(0usize, 4u8) as u8) } + } + #[inline] + pub fn set_dst_reg(&mut self, val: __u8) { + unsafe { + let val: u8 = ::core::mem::transmute(val); + self._bitfield_1.set(0usize, 4u8, val as u64) + } + } + #[inline] + pub fn src_reg(&self) -> __u8 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(4usize, 4u8) as u8) } + } + #[inline] + pub fn set_src_reg(&mut self, val: __u8) { + unsafe { + let val: u8 = ::core::mem::transmute(val); + self._bitfield_1.set(4usize, 4u8, val as u64) + } + } + #[inline] + pub fn new_bitfield_1(dst_reg: __u8, src_reg: __u8) -> __BindgenBitfieldUnit<[u8; 1usize]> { + let mut __bindgen_bitfield_unit: __BindgenBitfieldUnit<[u8; 1usize]> = Default::default(); + __bindgen_bitfield_unit.set(0usize, 4u8, { + let dst_reg: u8 = unsafe { ::core::mem::transmute(dst_reg) }; + dst_reg as u64 + }); + __bindgen_bitfield_unit.set(4usize, 4u8, { + let src_reg: u8 = unsafe { ::core::mem::transmute(src_reg) }; + src_reg as u64 + }); + __bindgen_bitfield_unit + } +} +#[repr(C)] +#[derive(Debug)] +pub struct bpf_lpm_trie_key { + pub prefixlen: __u32, + pub data: __IncompleteArrayField<__u8>, +} +impl bpf_cmd { + pub const BPF_PROG_RUN: bpf_cmd = bpf_cmd::BPF_PROG_TEST_RUN; +} +#[repr(u32)] +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq, FromPrimitive)] +pub enum bpf_cmd { + BPF_MAP_CREATE = 0, + BPF_MAP_LOOKUP_ELEM = 1, + BPF_MAP_UPDATE_ELEM = 2, + BPF_MAP_DELETE_ELEM = 3, + BPF_MAP_GET_NEXT_KEY = 4, + BPF_PROG_LOAD = 5, + BPF_OBJ_PIN = 6, + BPF_OBJ_GET = 7, + BPF_PROG_ATTACH = 8, + BPF_PROG_DETACH = 9, + BPF_PROG_TEST_RUN = 10, + BPF_PROG_GET_NEXT_ID = 11, + BPF_MAP_GET_NEXT_ID = 12, + BPF_PROG_GET_FD_BY_ID = 13, + BPF_MAP_GET_FD_BY_ID = 14, + BPF_OBJ_GET_INFO_BY_FD = 15, + BPF_PROG_QUERY = 16, + BPF_RAW_TRACEPOINT_OPEN = 17, + BPF_BTF_LOAD = 18, + BPF_BTF_GET_FD_BY_ID = 19, + BPF_TASK_FD_QUERY = 20, + BPF_MAP_LOOKUP_AND_DELETE_ELEM = 21, + BPF_MAP_FREEZE = 22, + BPF_BTF_GET_NEXT_ID = 23, + BPF_MAP_LOOKUP_BATCH = 24, + BPF_MAP_LOOKUP_AND_DELETE_BATCH = 25, + BPF_MAP_UPDATE_BATCH = 26, + BPF_MAP_DELETE_BATCH = 27, + BPF_LINK_CREATE = 28, + BPF_LINK_UPDATE = 29, + BPF_LINK_GET_FD_BY_ID = 30, + BPF_LINK_GET_NEXT_ID = 31, + BPF_ENABLE_STATS = 32, + BPF_ITER_CREATE = 33, + BPF_LINK_DETACH = 34, + BPF_PROG_BIND_MAP = 35, + BPF_TOKEN_CREATE = 36, + __MAX_BPF_CMD = 37, +} +impl bpf_map_type { + pub const BPF_MAP_TYPE_CGROUP_STORAGE: bpf_map_type = + bpf_map_type::BPF_MAP_TYPE_CGROUP_STORAGE_DEPRECATED; +} +impl bpf_map_type { + pub const BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE: bpf_map_type = + bpf_map_type::BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE_DEPRECATED; +} +#[repr(u32)] +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq, FromPrimitive)] +pub enum bpf_map_type { + BPF_MAP_TYPE_UNSPEC = 0, + BPF_MAP_TYPE_HASH = 1, + BPF_MAP_TYPE_ARRAY = 2, + BPF_MAP_TYPE_PROG_ARRAY = 3, + BPF_MAP_TYPE_PERF_EVENT_ARRAY = 4, + BPF_MAP_TYPE_PERCPU_HASH = 5, + BPF_MAP_TYPE_PERCPU_ARRAY = 6, + BPF_MAP_TYPE_STACK_TRACE = 7, + BPF_MAP_TYPE_CGROUP_ARRAY = 8, + BPF_MAP_TYPE_LRU_HASH = 9, + BPF_MAP_TYPE_LRU_PERCPU_HASH = 10, + BPF_MAP_TYPE_LPM_TRIE = 11, + BPF_MAP_TYPE_ARRAY_OF_MAPS = 12, + BPF_MAP_TYPE_HASH_OF_MAPS = 13, + BPF_MAP_TYPE_DEVMAP = 14, + BPF_MAP_TYPE_SOCKMAP = 15, + BPF_MAP_TYPE_CPUMAP = 16, + BPF_MAP_TYPE_XSKMAP = 17, + BPF_MAP_TYPE_SOCKHASH = 18, + BPF_MAP_TYPE_CGROUP_STORAGE_DEPRECATED = 19, + BPF_MAP_TYPE_REUSEPORT_SOCKARRAY = 20, + BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE_DEPRECATED = 21, + BPF_MAP_TYPE_QUEUE = 22, + BPF_MAP_TYPE_STACK = 23, + BPF_MAP_TYPE_SK_STORAGE = 24, + BPF_MAP_TYPE_DEVMAP_HASH = 25, + BPF_MAP_TYPE_STRUCT_OPS = 26, + BPF_MAP_TYPE_RINGBUF = 27, + BPF_MAP_TYPE_INODE_STORAGE = 28, + BPF_MAP_TYPE_TASK_STORAGE = 29, + BPF_MAP_TYPE_BLOOM_FILTER = 30, + BPF_MAP_TYPE_USER_RINGBUF = 31, + BPF_MAP_TYPE_CGRP_STORAGE = 32, + BPF_MAP_TYPE_ARENA = 33, + __MAX_BPF_MAP_TYPE = 34, +} +#[repr(u32)] +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq, FromPrimitive)] +pub enum bpf_prog_type { + BPF_PROG_TYPE_UNSPEC = 0, + BPF_PROG_TYPE_SOCKET_FILTER = 1, + BPF_PROG_TYPE_KPROBE = 2, + BPF_PROG_TYPE_SCHED_CLS = 3, + BPF_PROG_TYPE_SCHED_ACT = 4, + BPF_PROG_TYPE_TRACEPOINT = 5, + BPF_PROG_TYPE_XDP = 6, + BPF_PROG_TYPE_PERF_EVENT = 7, + BPF_PROG_TYPE_CGROUP_SKB = 8, + BPF_PROG_TYPE_CGROUP_SOCK = 9, + BPF_PROG_TYPE_LWT_IN = 10, + BPF_PROG_TYPE_LWT_OUT = 11, + BPF_PROG_TYPE_LWT_XMIT = 12, + BPF_PROG_TYPE_SOCK_OPS = 13, + BPF_PROG_TYPE_SK_SKB = 14, + BPF_PROG_TYPE_CGROUP_DEVICE = 15, + BPF_PROG_TYPE_SK_MSG = 16, + BPF_PROG_TYPE_RAW_TRACEPOINT = 17, + BPF_PROG_TYPE_CGROUP_SOCK_ADDR = 18, + BPF_PROG_TYPE_LWT_SEG6LOCAL = 19, + BPF_PROG_TYPE_LIRC_MODE2 = 20, + BPF_PROG_TYPE_SK_REUSEPORT = 21, + BPF_PROG_TYPE_FLOW_DISSECTOR = 22, + BPF_PROG_TYPE_CGROUP_SYSCTL = 23, + BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE = 24, + BPF_PROG_TYPE_CGROUP_SOCKOPT = 25, + BPF_PROG_TYPE_TRACING = 26, + BPF_PROG_TYPE_STRUCT_OPS = 27, + BPF_PROG_TYPE_EXT = 28, + BPF_PROG_TYPE_LSM = 29, + BPF_PROG_TYPE_SK_LOOKUP = 30, + BPF_PROG_TYPE_SYSCALL = 31, + BPF_PROG_TYPE_NETFILTER = 32, + __MAX_BPF_PROG_TYPE = 33, +} +#[repr(u32)] +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq, FromPrimitive)] +pub enum bpf_attach_type { + BPF_CGROUP_INET_INGRESS = 0, + BPF_CGROUP_INET_EGRESS = 1, + BPF_CGROUP_INET_SOCK_CREATE = 2, + BPF_CGROUP_SOCK_OPS = 3, + BPF_SK_SKB_STREAM_PARSER = 4, + BPF_SK_SKB_STREAM_VERDICT = 5, + BPF_CGROUP_DEVICE = 6, + BPF_SK_MSG_VERDICT = 7, + BPF_CGROUP_INET4_BIND = 8, + BPF_CGROUP_INET6_BIND = 9, + BPF_CGROUP_INET4_CONNECT = 10, + BPF_CGROUP_INET6_CONNECT = 11, + BPF_CGROUP_INET4_POST_BIND = 12, + BPF_CGROUP_INET6_POST_BIND = 13, + BPF_CGROUP_UDP4_SENDMSG = 14, + BPF_CGROUP_UDP6_SENDMSG = 15, + BPF_LIRC_MODE2 = 16, + BPF_FLOW_DISSECTOR = 17, + BPF_CGROUP_SYSCTL = 18, + BPF_CGROUP_UDP4_RECVMSG = 19, + BPF_CGROUP_UDP6_RECVMSG = 20, + BPF_CGROUP_GETSOCKOPT = 21, + BPF_CGROUP_SETSOCKOPT = 22, + BPF_TRACE_RAW_TP = 23, + BPF_TRACE_FENTRY = 24, + BPF_TRACE_FEXIT = 25, + BPF_MODIFY_RETURN = 26, + BPF_LSM_MAC = 27, + BPF_TRACE_ITER = 28, + BPF_CGROUP_INET4_GETPEERNAME = 29, + BPF_CGROUP_INET6_GETPEERNAME = 30, + BPF_CGROUP_INET4_GETSOCKNAME = 31, + BPF_CGROUP_INET6_GETSOCKNAME = 32, + BPF_XDP_DEVMAP = 33, + BPF_CGROUP_INET_SOCK_RELEASE = 34, + BPF_XDP_CPUMAP = 35, + BPF_SK_LOOKUP = 36, + BPF_XDP = 37, + BPF_SK_SKB_VERDICT = 38, + BPF_SK_REUSEPORT_SELECT = 39, + BPF_SK_REUSEPORT_SELECT_OR_MIGRATE = 40, + BPF_PERF_EVENT = 41, + BPF_TRACE_KPROBE_MULTI = 42, + BPF_LSM_CGROUP = 43, + BPF_STRUCT_OPS = 44, + BPF_NETFILTER = 45, + BPF_TCX_INGRESS = 46, + BPF_TCX_EGRESS = 47, + BPF_TRACE_UPROBE_MULTI = 48, + BPF_CGROUP_UNIX_CONNECT = 49, + BPF_CGROUP_UNIX_SENDMSG = 50, + BPF_CGROUP_UNIX_RECVMSG = 51, + BPF_CGROUP_UNIX_GETPEERNAME = 52, + BPF_CGROUP_UNIX_GETSOCKNAME = 53, + BPF_NETKIT_PRIMARY = 54, + BPF_NETKIT_PEER = 55, + __MAX_BPF_ATTACH_TYPE = 56, +} +#[repr(u32)] +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub enum bpf_link_type { + BPF_LINK_TYPE_UNSPEC = 0, + BPF_LINK_TYPE_RAW_TRACEPOINT = 1, + BPF_LINK_TYPE_TRACING = 2, + BPF_LINK_TYPE_CGROUP = 3, + BPF_LINK_TYPE_ITER = 4, + BPF_LINK_TYPE_NETNS = 5, + BPF_LINK_TYPE_XDP = 6, + BPF_LINK_TYPE_PERF_EVENT = 7, + BPF_LINK_TYPE_KPROBE_MULTI = 8, + BPF_LINK_TYPE_STRUCT_OPS = 9, + BPF_LINK_TYPE_NETFILTER = 10, + BPF_LINK_TYPE_TCX = 11, + BPF_LINK_TYPE_UPROBE_MULTI = 12, + BPF_LINK_TYPE_NETKIT = 13, + __MAX_BPF_LINK_TYPE = 14, +} +pub const BPF_F_KPROBE_MULTI_RETURN: _bindgen_ty_2 = 1; +pub type _bindgen_ty_2 = ::core::ffi::c_uint; +pub const BPF_F_UPROBE_MULTI_RETURN: _bindgen_ty_3 = 1; +pub type _bindgen_ty_3 = ::core::ffi::c_uint; +pub const BPF_ANY: _bindgen_ty_4 = 0; +pub const BPF_NOEXIST: _bindgen_ty_4 = 1; +pub const BPF_EXIST: _bindgen_ty_4 = 2; +pub const BPF_F_LOCK: _bindgen_ty_4 = 4; +pub type _bindgen_ty_4 = ::core::ffi::c_uint; +pub const BPF_F_NO_PREALLOC: _bindgen_ty_5 = 1; +pub const BPF_F_NO_COMMON_LRU: _bindgen_ty_5 = 2; +pub const BPF_F_NUMA_NODE: _bindgen_ty_5 = 4; +pub const BPF_F_RDONLY: _bindgen_ty_5 = 8; +pub const BPF_F_WRONLY: _bindgen_ty_5 = 16; +pub const BPF_F_STACK_BUILD_ID: _bindgen_ty_5 = 32; +pub const BPF_F_ZERO_SEED: _bindgen_ty_5 = 64; +pub const BPF_F_RDONLY_PROG: _bindgen_ty_5 = 128; +pub const BPF_F_WRONLY_PROG: _bindgen_ty_5 = 256; +pub const BPF_F_CLONE: _bindgen_ty_5 = 512; +pub const BPF_F_MMAPABLE: _bindgen_ty_5 = 1024; +pub const BPF_F_PRESERVE_ELEMS: _bindgen_ty_5 = 2048; +pub const BPF_F_INNER_MAP: _bindgen_ty_5 = 4096; +pub const BPF_F_LINK: _bindgen_ty_5 = 8192; +pub const BPF_F_PATH_FD: _bindgen_ty_5 = 16384; +pub const BPF_F_VTYPE_BTF_OBJ_FD: _bindgen_ty_5 = 32768; +pub const BPF_F_TOKEN_FD: _bindgen_ty_5 = 65536; +pub const BPF_F_SEGV_ON_FAULT: _bindgen_ty_5 = 131072; +pub const BPF_F_NO_USER_CONV: _bindgen_ty_5 = 262144; +pub type _bindgen_ty_5 = ::core::ffi::c_uint; +#[repr(u32)] +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub enum bpf_stats_type { + BPF_STATS_RUN_TIME = 0, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub union bpf_attr { + pub __bindgen_anon_1: bpf_attr__bindgen_ty_1, + pub __bindgen_anon_2: bpf_attr__bindgen_ty_2, + pub batch: bpf_attr__bindgen_ty_3, + pub __bindgen_anon_3: bpf_attr__bindgen_ty_4, + pub __bindgen_anon_4: bpf_attr__bindgen_ty_5, + pub __bindgen_anon_5: bpf_attr__bindgen_ty_6, + pub test: bpf_attr__bindgen_ty_7, + pub __bindgen_anon_6: bpf_attr__bindgen_ty_8, + pub info: bpf_attr__bindgen_ty_9, + pub query: bpf_attr__bindgen_ty_10, + pub raw_tracepoint: bpf_attr__bindgen_ty_11, + pub __bindgen_anon_7: bpf_attr__bindgen_ty_12, + pub task_fd_query: bpf_attr__bindgen_ty_13, + pub link_create: bpf_attr__bindgen_ty_14, + pub link_update: bpf_attr__bindgen_ty_15, + pub link_detach: bpf_attr__bindgen_ty_16, + pub enable_stats: bpf_attr__bindgen_ty_17, + pub iter_create: bpf_attr__bindgen_ty_18, + pub prog_bind_map: bpf_attr__bindgen_ty_19, + pub token_create: bpf_attr__bindgen_ty_20, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct bpf_attr__bindgen_ty_1 { + pub map_type: __u32, + pub key_size: __u32, + pub value_size: __u32, + pub max_entries: __u32, + pub map_flags: __u32, + pub inner_map_fd: __u32, + pub numa_node: __u32, + pub map_name: [::core::ffi::c_char; 16usize], + pub map_ifindex: __u32, + pub btf_fd: __u32, + pub btf_key_type_id: __u32, + pub btf_value_type_id: __u32, + pub btf_vmlinux_value_type_id: __u32, + pub map_extra: __u64, + pub value_type_btf_obj_fd: __s32, + pub map_token_fd: __s32, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub struct bpf_attr__bindgen_ty_2 { + pub map_fd: __u32, + pub key: __u64, + pub __bindgen_anon_1: bpf_attr__bindgen_ty_2__bindgen_ty_1, + pub flags: __u64, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub union bpf_attr__bindgen_ty_2__bindgen_ty_1 { + pub value: __u64, + pub next_key: __u64, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct bpf_attr__bindgen_ty_3 { + pub in_batch: __u64, + pub out_batch: __u64, + pub keys: __u64, + pub values: __u64, + pub count: __u32, + pub map_fd: __u32, + pub elem_flags: __u64, + pub flags: __u64, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub struct bpf_attr__bindgen_ty_4 { + pub prog_type: __u32, + pub insn_cnt: __u32, + pub insns: __u64, + pub license: __u64, + pub log_level: __u32, + pub log_size: __u32, + pub log_buf: __u64, + pub kern_version: __u32, + pub prog_flags: __u32, + pub prog_name: [::core::ffi::c_char; 16usize], + pub prog_ifindex: __u32, + pub expected_attach_type: __u32, + pub prog_btf_fd: __u32, + pub func_info_rec_size: __u32, + pub func_info: __u64, + pub func_info_cnt: __u32, + pub line_info_rec_size: __u32, + pub line_info: __u64, + pub line_info_cnt: __u32, + pub attach_btf_id: __u32, + pub __bindgen_anon_1: bpf_attr__bindgen_ty_4__bindgen_ty_1, + pub core_relo_cnt: __u32, + pub fd_array: __u64, + pub core_relos: __u64, + pub core_relo_rec_size: __u32, + pub log_true_size: __u32, + pub prog_token_fd: __s32, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub union bpf_attr__bindgen_ty_4__bindgen_ty_1 { + pub attach_prog_fd: __u32, + pub attach_btf_obj_fd: __u32, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct bpf_attr__bindgen_ty_5 { + pub pathname: __u64, + pub bpf_fd: __u32, + pub file_flags: __u32, + pub path_fd: __s32, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub struct bpf_attr__bindgen_ty_6 { + pub __bindgen_anon_1: bpf_attr__bindgen_ty_6__bindgen_ty_1, + pub attach_bpf_fd: __u32, + pub attach_type: __u32, + pub attach_flags: __u32, + pub replace_bpf_fd: __u32, + pub __bindgen_anon_2: bpf_attr__bindgen_ty_6__bindgen_ty_2, + pub expected_revision: __u64, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub union bpf_attr__bindgen_ty_6__bindgen_ty_1 { + pub target_fd: __u32, + pub target_ifindex: __u32, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub union bpf_attr__bindgen_ty_6__bindgen_ty_2 { + pub relative_fd: __u32, + pub relative_id: __u32, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct bpf_attr__bindgen_ty_7 { + pub prog_fd: __u32, + pub retval: __u32, + pub data_size_in: __u32, + pub data_size_out: __u32, + pub data_in: __u64, + pub data_out: __u64, + pub repeat: __u32, + pub duration: __u32, + pub ctx_size_in: __u32, + pub ctx_size_out: __u32, + pub ctx_in: __u64, + pub ctx_out: __u64, + pub flags: __u32, + pub cpu: __u32, + pub batch_size: __u32, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub struct bpf_attr__bindgen_ty_8 { + pub __bindgen_anon_1: bpf_attr__bindgen_ty_8__bindgen_ty_1, + pub next_id: __u32, + pub open_flags: __u32, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub union bpf_attr__bindgen_ty_8__bindgen_ty_1 { + pub start_id: __u32, + pub prog_id: __u32, + pub map_id: __u32, + pub btf_id: __u32, + pub link_id: __u32, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct bpf_attr__bindgen_ty_9 { + pub bpf_fd: __u32, + pub info_len: __u32, + pub info: __u64, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub struct bpf_attr__bindgen_ty_10 { + pub __bindgen_anon_1: bpf_attr__bindgen_ty_10__bindgen_ty_1, + pub attach_type: __u32, + pub query_flags: __u32, + pub attach_flags: __u32, + pub prog_ids: __u64, + pub __bindgen_anon_2: bpf_attr__bindgen_ty_10__bindgen_ty_2, + pub _bitfield_align_1: [u8; 0], + pub _bitfield_1: __BindgenBitfieldUnit<[u8; 4usize]>, + pub prog_attach_flags: __u64, + pub link_ids: __u64, + pub link_attach_flags: __u64, + pub revision: __u64, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub union bpf_attr__bindgen_ty_10__bindgen_ty_1 { + pub target_fd: __u32, + pub target_ifindex: __u32, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub union bpf_attr__bindgen_ty_10__bindgen_ty_2 { + pub prog_cnt: __u32, + pub count: __u32, +} +impl bpf_attr__bindgen_ty_10 { + #[inline] + pub fn new_bitfield_1() -> __BindgenBitfieldUnit<[u8; 4usize]> { + let mut __bindgen_bitfield_unit: __BindgenBitfieldUnit<[u8; 4usize]> = Default::default(); + __bindgen_bitfield_unit + } +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct bpf_attr__bindgen_ty_11 { + pub name: __u64, + pub prog_fd: __u32, + pub _bitfield_align_1: [u8; 0], + pub _bitfield_1: __BindgenBitfieldUnit<[u8; 4usize]>, + pub cookie: __u64, +} +impl bpf_attr__bindgen_ty_11 { + #[inline] + pub fn new_bitfield_1() -> __BindgenBitfieldUnit<[u8; 4usize]> { + let mut __bindgen_bitfield_unit: __BindgenBitfieldUnit<[u8; 4usize]> = Default::default(); + __bindgen_bitfield_unit + } +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct bpf_attr__bindgen_ty_12 { + pub btf: __u64, + pub btf_log_buf: __u64, + pub btf_size: __u32, + pub btf_log_size: __u32, + pub btf_log_level: __u32, + pub btf_log_true_size: __u32, + pub btf_flags: __u32, + pub btf_token_fd: __s32, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct bpf_attr__bindgen_ty_13 { + pub pid: __u32, + pub fd: __u32, + pub flags: __u32, + pub buf_len: __u32, + pub buf: __u64, + pub prog_id: __u32, + pub fd_type: __u32, + pub probe_offset: __u64, + pub probe_addr: __u64, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub struct bpf_attr__bindgen_ty_14 { + pub __bindgen_anon_1: bpf_attr__bindgen_ty_14__bindgen_ty_1, + pub __bindgen_anon_2: bpf_attr__bindgen_ty_14__bindgen_ty_2, + pub attach_type: __u32, + pub flags: __u32, + pub __bindgen_anon_3: bpf_attr__bindgen_ty_14__bindgen_ty_3, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub union bpf_attr__bindgen_ty_14__bindgen_ty_1 { + pub prog_fd: __u32, + pub map_fd: __u32, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub union bpf_attr__bindgen_ty_14__bindgen_ty_2 { + pub target_fd: __u32, + pub target_ifindex: __u32, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub union bpf_attr__bindgen_ty_14__bindgen_ty_3 { + pub target_btf_id: __u32, + pub __bindgen_anon_1: bpf_attr__bindgen_ty_14__bindgen_ty_3__bindgen_ty_1, + pub perf_event: bpf_attr__bindgen_ty_14__bindgen_ty_3__bindgen_ty_2, + pub kprobe_multi: bpf_attr__bindgen_ty_14__bindgen_ty_3__bindgen_ty_3, + pub tracing: bpf_attr__bindgen_ty_14__bindgen_ty_3__bindgen_ty_4, + pub netfilter: bpf_attr__bindgen_ty_14__bindgen_ty_3__bindgen_ty_5, + pub tcx: bpf_attr__bindgen_ty_14__bindgen_ty_3__bindgen_ty_6, + pub uprobe_multi: bpf_attr__bindgen_ty_14__bindgen_ty_3__bindgen_ty_7, + pub netkit: bpf_attr__bindgen_ty_14__bindgen_ty_3__bindgen_ty_8, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct bpf_attr__bindgen_ty_14__bindgen_ty_3__bindgen_ty_1 { + pub iter_info: __u64, + pub iter_info_len: __u32, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct bpf_attr__bindgen_ty_14__bindgen_ty_3__bindgen_ty_2 { + pub bpf_cookie: __u64, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct bpf_attr__bindgen_ty_14__bindgen_ty_3__bindgen_ty_3 { + pub flags: __u32, + pub cnt: __u32, + pub syms: __u64, + pub addrs: __u64, + pub cookies: __u64, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct bpf_attr__bindgen_ty_14__bindgen_ty_3__bindgen_ty_4 { + pub target_btf_id: __u32, + pub cookie: __u64, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct bpf_attr__bindgen_ty_14__bindgen_ty_3__bindgen_ty_5 { + pub pf: __u32, + pub hooknum: __u32, + pub priority: __s32, + pub flags: __u32, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub struct bpf_attr__bindgen_ty_14__bindgen_ty_3__bindgen_ty_6 { + pub __bindgen_anon_1: bpf_attr__bindgen_ty_14__bindgen_ty_3__bindgen_ty_6__bindgen_ty_1, + pub expected_revision: __u64, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub union bpf_attr__bindgen_ty_14__bindgen_ty_3__bindgen_ty_6__bindgen_ty_1 { + pub relative_fd: __u32, + pub relative_id: __u32, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct bpf_attr__bindgen_ty_14__bindgen_ty_3__bindgen_ty_7 { + pub path: __u64, + pub offsets: __u64, + pub ref_ctr_offsets: __u64, + pub cookies: __u64, + pub cnt: __u32, + pub flags: __u32, + pub pid: __u32, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub struct bpf_attr__bindgen_ty_14__bindgen_ty_3__bindgen_ty_8 { + pub __bindgen_anon_1: bpf_attr__bindgen_ty_14__bindgen_ty_3__bindgen_ty_8__bindgen_ty_1, + pub expected_revision: __u64, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub union bpf_attr__bindgen_ty_14__bindgen_ty_3__bindgen_ty_8__bindgen_ty_1 { + pub relative_fd: __u32, + pub relative_id: __u32, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub struct bpf_attr__bindgen_ty_15 { + pub link_fd: __u32, + pub __bindgen_anon_1: bpf_attr__bindgen_ty_15__bindgen_ty_1, + pub flags: __u32, + pub __bindgen_anon_2: bpf_attr__bindgen_ty_15__bindgen_ty_2, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub union bpf_attr__bindgen_ty_15__bindgen_ty_1 { + pub new_prog_fd: __u32, + pub new_map_fd: __u32, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub union bpf_attr__bindgen_ty_15__bindgen_ty_2 { + pub old_prog_fd: __u32, + pub old_map_fd: __u32, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct bpf_attr__bindgen_ty_16 { + pub link_fd: __u32, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct bpf_attr__bindgen_ty_17 { + pub type_: __u32, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct bpf_attr__bindgen_ty_18 { + pub link_fd: __u32, + pub flags: __u32, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct bpf_attr__bindgen_ty_19 { + pub prog_fd: __u32, + pub map_fd: __u32, + pub flags: __u32, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct bpf_attr__bindgen_ty_20 { + pub flags: __u32, + pub bpffs_fd: __u32, +} +pub const BPF_F_RECOMPUTE_CSUM: _bindgen_ty_6 = 1; +pub const BPF_F_INVALIDATE_HASH: _bindgen_ty_6 = 2; +pub type _bindgen_ty_6 = ::core::ffi::c_uint; +pub const BPF_F_HDR_FIELD_MASK: _bindgen_ty_7 = 15; +pub type _bindgen_ty_7 = ::core::ffi::c_uint; +pub const BPF_F_PSEUDO_HDR: _bindgen_ty_8 = 16; +pub const BPF_F_MARK_MANGLED_0: _bindgen_ty_8 = 32; +pub const BPF_F_MARK_ENFORCE: _bindgen_ty_8 = 64; +pub type _bindgen_ty_8 = ::core::ffi::c_uint; +pub const BPF_F_INGRESS: _bindgen_ty_9 = 1; +pub type _bindgen_ty_9 = ::core::ffi::c_uint; +pub const BPF_F_TUNINFO_IPV6: _bindgen_ty_10 = 1; +pub type _bindgen_ty_10 = ::core::ffi::c_uint; +pub const BPF_F_SKIP_FIELD_MASK: _bindgen_ty_11 = 255; +pub const BPF_F_USER_STACK: _bindgen_ty_11 = 256; +pub const BPF_F_FAST_STACK_CMP: _bindgen_ty_11 = 512; +pub const BPF_F_REUSE_STACKID: _bindgen_ty_11 = 1024; +pub const BPF_F_USER_BUILD_ID: _bindgen_ty_11 = 2048; +pub type _bindgen_ty_11 = ::core::ffi::c_uint; +pub const BPF_F_ZERO_CSUM_TX: _bindgen_ty_12 = 2; +pub const BPF_F_DONT_FRAGMENT: _bindgen_ty_12 = 4; +pub const BPF_F_SEQ_NUMBER: _bindgen_ty_12 = 8; +pub const BPF_F_NO_TUNNEL_KEY: _bindgen_ty_12 = 16; +pub type _bindgen_ty_12 = ::core::ffi::c_uint; +pub const BPF_F_TUNINFO_FLAGS: _bindgen_ty_13 = 16; +pub type _bindgen_ty_13 = ::core::ffi::c_uint; +pub const BPF_F_INDEX_MASK: _bindgen_ty_14 = 4294967295; +pub const BPF_F_CURRENT_CPU: _bindgen_ty_14 = 4294967295; +pub const BPF_F_CTXLEN_MASK: _bindgen_ty_14 = 4503595332403200; +pub type _bindgen_ty_14 = ::core::ffi::c_ulong; +pub const BPF_F_CURRENT_NETNS: _bindgen_ty_15 = -1; +pub type _bindgen_ty_15 = ::core::ffi::c_int; +pub const BPF_F_ADJ_ROOM_FIXED_GSO: _bindgen_ty_17 = 1; +pub const BPF_F_ADJ_ROOM_ENCAP_L3_IPV4: _bindgen_ty_17 = 2; +pub const BPF_F_ADJ_ROOM_ENCAP_L3_IPV6: _bindgen_ty_17 = 4; +pub const BPF_F_ADJ_ROOM_ENCAP_L4_GRE: _bindgen_ty_17 = 8; +pub const BPF_F_ADJ_ROOM_ENCAP_L4_UDP: _bindgen_ty_17 = 16; +pub const BPF_F_ADJ_ROOM_NO_CSUM_RESET: _bindgen_ty_17 = 32; +pub const BPF_F_ADJ_ROOM_ENCAP_L2_ETH: _bindgen_ty_17 = 64; +pub const BPF_F_ADJ_ROOM_DECAP_L3_IPV4: _bindgen_ty_17 = 128; +pub const BPF_F_ADJ_ROOM_DECAP_L3_IPV6: _bindgen_ty_17 = 256; +pub type _bindgen_ty_17 = ::core::ffi::c_uint; +pub const BPF_F_SYSCTL_BASE_NAME: _bindgen_ty_19 = 1; +pub type _bindgen_ty_19 = ::core::ffi::c_uint; +pub const BPF_F_GET_BRANCH_RECORDS_SIZE: _bindgen_ty_21 = 1; +pub type _bindgen_ty_21 = ::core::ffi::c_uint; +pub const BPF_RINGBUF_BUSY_BIT: _bindgen_ty_24 = 2147483648; +pub const BPF_RINGBUF_DISCARD_BIT: _bindgen_ty_24 = 1073741824; +pub const BPF_RINGBUF_HDR_SZ: _bindgen_ty_24 = 8; +pub type _bindgen_ty_24 = ::core::ffi::c_uint; +pub const BPF_F_BPRM_SECUREEXEC: _bindgen_ty_26 = 1; +pub type _bindgen_ty_26 = ::core::ffi::c_uint; +pub const BPF_F_BROADCAST: _bindgen_ty_27 = 8; +pub const BPF_F_EXCLUDE_INGRESS: _bindgen_ty_27 = 16; +pub type _bindgen_ty_27 = ::core::ffi::c_uint; +#[repr(C)] +#[derive(Copy, Clone)] +pub struct bpf_devmap_val { + pub ifindex: __u32, + pub bpf_prog: bpf_devmap_val__bindgen_ty_1, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub union bpf_devmap_val__bindgen_ty_1 { + pub fd: ::core::ffi::c_int, + pub id: __u32, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub struct bpf_cpumap_val { + pub qsize: __u32, + pub bpf_prog: bpf_cpumap_val__bindgen_ty_1, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub union bpf_cpumap_val__bindgen_ty_1 { + pub fd: ::core::ffi::c_int, + pub id: __u32, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct bpf_prog_info { + pub type_: __u32, + pub id: __u32, + pub tag: [__u8; 8usize], + pub jited_prog_len: __u32, + pub xlated_prog_len: __u32, + pub jited_prog_insns: __u64, + pub xlated_prog_insns: __u64, + pub load_time: __u64, + pub created_by_uid: __u32, + pub nr_map_ids: __u32, + pub map_ids: __u64, + pub name: [::core::ffi::c_char; 16usize], + pub ifindex: __u32, + pub _bitfield_align_1: [u8; 0], + pub _bitfield_1: __BindgenBitfieldUnit<[u8; 4usize]>, + pub netns_dev: __u64, + pub netns_ino: __u64, + pub nr_jited_ksyms: __u32, + pub nr_jited_func_lens: __u32, + pub jited_ksyms: __u64, + pub jited_func_lens: __u64, + pub btf_id: __u32, + pub func_info_rec_size: __u32, + pub func_info: __u64, + pub nr_func_info: __u32, + pub nr_line_info: __u32, + pub line_info: __u64, + pub jited_line_info: __u64, + pub nr_jited_line_info: __u32, + pub line_info_rec_size: __u32, + pub jited_line_info_rec_size: __u32, + pub nr_prog_tags: __u32, + pub prog_tags: __u64, + pub run_time_ns: __u64, + pub run_cnt: __u64, + pub recursion_misses: __u64, + pub verified_insns: __u32, + pub attach_btf_obj_id: __u32, + pub attach_btf_id: __u32, +} +impl bpf_prog_info { + #[inline] + pub fn gpl_compatible(&self) -> __u32 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(0usize, 1u8) as u32) } + } + #[inline] + pub fn set_gpl_compatible(&mut self, val: __u32) { + unsafe { + let val: u32 = ::core::mem::transmute(val); + self._bitfield_1.set(0usize, 1u8, val as u64) + } + } + #[inline] + pub fn new_bitfield_1(gpl_compatible: __u32) -> __BindgenBitfieldUnit<[u8; 4usize]> { + let mut __bindgen_bitfield_unit: __BindgenBitfieldUnit<[u8; 4usize]> = Default::default(); + __bindgen_bitfield_unit.set(0usize, 1u8, { + let gpl_compatible: u32 = unsafe { ::core::mem::transmute(gpl_compatible) }; + gpl_compatible as u64 + }); + __bindgen_bitfield_unit + } +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct bpf_map_info { + pub type_: __u32, + pub id: __u32, + pub key_size: __u32, + pub value_size: __u32, + pub max_entries: __u32, + pub map_flags: __u32, + pub name: [::core::ffi::c_char; 16usize], + pub ifindex: __u32, + pub btf_vmlinux_value_type_id: __u32, + pub netns_dev: __u64, + pub netns_ino: __u64, + pub btf_id: __u32, + pub btf_key_type_id: __u32, + pub btf_value_type_id: __u32, + pub btf_vmlinux_id: __u32, + pub map_extra: __u64, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct bpf_btf_info { + pub btf: __u64, + pub btf_size: __u32, + pub id: __u32, + pub name: __u64, + pub name_len: __u32, + pub kernel_btf: __u32, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub struct bpf_link_info { + pub type_: __u32, + pub id: __u32, + pub prog_id: __u32, + pub __bindgen_anon_1: bpf_link_info__bindgen_ty_1, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub union bpf_link_info__bindgen_ty_1 { + pub raw_tracepoint: bpf_link_info__bindgen_ty_1__bindgen_ty_1, + pub tracing: bpf_link_info__bindgen_ty_1__bindgen_ty_2, + pub cgroup: bpf_link_info__bindgen_ty_1__bindgen_ty_3, + pub iter: bpf_link_info__bindgen_ty_1__bindgen_ty_4, + pub netns: bpf_link_info__bindgen_ty_1__bindgen_ty_5, + pub xdp: bpf_link_info__bindgen_ty_1__bindgen_ty_6, + pub struct_ops: bpf_link_info__bindgen_ty_1__bindgen_ty_7, + pub netfilter: bpf_link_info__bindgen_ty_1__bindgen_ty_8, + pub kprobe_multi: bpf_link_info__bindgen_ty_1__bindgen_ty_9, + pub uprobe_multi: bpf_link_info__bindgen_ty_1__bindgen_ty_10, + pub perf_event: bpf_link_info__bindgen_ty_1__bindgen_ty_11, + pub tcx: bpf_link_info__bindgen_ty_1__bindgen_ty_12, + pub netkit: bpf_link_info__bindgen_ty_1__bindgen_ty_13, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct bpf_link_info__bindgen_ty_1__bindgen_ty_1 { + pub tp_name: __u64, + pub tp_name_len: __u32, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct bpf_link_info__bindgen_ty_1__bindgen_ty_2 { + pub attach_type: __u32, + pub target_obj_id: __u32, + pub target_btf_id: __u32, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct bpf_link_info__bindgen_ty_1__bindgen_ty_3 { + pub cgroup_id: __u64, + pub attach_type: __u32, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub struct bpf_link_info__bindgen_ty_1__bindgen_ty_4 { + pub target_name: __u64, + pub target_name_len: __u32, + pub __bindgen_anon_1: bpf_link_info__bindgen_ty_1__bindgen_ty_4__bindgen_ty_1, + pub __bindgen_anon_2: bpf_link_info__bindgen_ty_1__bindgen_ty_4__bindgen_ty_2, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub union bpf_link_info__bindgen_ty_1__bindgen_ty_4__bindgen_ty_1 { + pub map: bpf_link_info__bindgen_ty_1__bindgen_ty_4__bindgen_ty_1__bindgen_ty_1, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct bpf_link_info__bindgen_ty_1__bindgen_ty_4__bindgen_ty_1__bindgen_ty_1 { + pub map_id: __u32, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub union bpf_link_info__bindgen_ty_1__bindgen_ty_4__bindgen_ty_2 { + pub cgroup: bpf_link_info__bindgen_ty_1__bindgen_ty_4__bindgen_ty_2__bindgen_ty_1, + pub task: bpf_link_info__bindgen_ty_1__bindgen_ty_4__bindgen_ty_2__bindgen_ty_2, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct bpf_link_info__bindgen_ty_1__bindgen_ty_4__bindgen_ty_2__bindgen_ty_1 { + pub cgroup_id: __u64, + pub order: __u32, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct bpf_link_info__bindgen_ty_1__bindgen_ty_4__bindgen_ty_2__bindgen_ty_2 { + pub tid: __u32, + pub pid: __u32, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct bpf_link_info__bindgen_ty_1__bindgen_ty_5 { + pub netns_ino: __u32, + pub attach_type: __u32, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct bpf_link_info__bindgen_ty_1__bindgen_ty_6 { + pub ifindex: __u32, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct bpf_link_info__bindgen_ty_1__bindgen_ty_7 { + pub map_id: __u32, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct bpf_link_info__bindgen_ty_1__bindgen_ty_8 { + pub pf: __u32, + pub hooknum: __u32, + pub priority: __s32, + pub flags: __u32, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct bpf_link_info__bindgen_ty_1__bindgen_ty_9 { + pub addrs: __u64, + pub count: __u32, + pub flags: __u32, + pub missed: __u64, + pub cookies: __u64, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct bpf_link_info__bindgen_ty_1__bindgen_ty_10 { + pub path: __u64, + pub offsets: __u64, + pub ref_ctr_offsets: __u64, + pub cookies: __u64, + pub path_size: __u32, + pub count: __u32, + pub flags: __u32, + pub pid: __u32, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub struct bpf_link_info__bindgen_ty_1__bindgen_ty_11 { + pub type_: __u32, + pub _bitfield_align_1: [u8; 0], + pub _bitfield_1: __BindgenBitfieldUnit<[u8; 4usize]>, + pub __bindgen_anon_1: bpf_link_info__bindgen_ty_1__bindgen_ty_11__bindgen_ty_1, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub union bpf_link_info__bindgen_ty_1__bindgen_ty_11__bindgen_ty_1 { + pub uprobe: bpf_link_info__bindgen_ty_1__bindgen_ty_11__bindgen_ty_1__bindgen_ty_1, + pub kprobe: bpf_link_info__bindgen_ty_1__bindgen_ty_11__bindgen_ty_1__bindgen_ty_2, + pub tracepoint: bpf_link_info__bindgen_ty_1__bindgen_ty_11__bindgen_ty_1__bindgen_ty_3, + pub event: bpf_link_info__bindgen_ty_1__bindgen_ty_11__bindgen_ty_1__bindgen_ty_4, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct bpf_link_info__bindgen_ty_1__bindgen_ty_11__bindgen_ty_1__bindgen_ty_1 { + pub file_name: __u64, + pub name_len: __u32, + pub offset: __u32, + pub cookie: __u64, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct bpf_link_info__bindgen_ty_1__bindgen_ty_11__bindgen_ty_1__bindgen_ty_2 { + pub func_name: __u64, + pub name_len: __u32, + pub offset: __u32, + pub addr: __u64, + pub missed: __u64, + pub cookie: __u64, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct bpf_link_info__bindgen_ty_1__bindgen_ty_11__bindgen_ty_1__bindgen_ty_3 { + pub tp_name: __u64, + pub name_len: __u32, + pub _bitfield_align_1: [u8; 0], + pub _bitfield_1: __BindgenBitfieldUnit<[u8; 4usize]>, + pub cookie: __u64, +} +impl bpf_link_info__bindgen_ty_1__bindgen_ty_11__bindgen_ty_1__bindgen_ty_3 { + #[inline] + pub fn new_bitfield_1() -> __BindgenBitfieldUnit<[u8; 4usize]> { + let mut __bindgen_bitfield_unit: __BindgenBitfieldUnit<[u8; 4usize]> = Default::default(); + __bindgen_bitfield_unit + } +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct bpf_link_info__bindgen_ty_1__bindgen_ty_11__bindgen_ty_1__bindgen_ty_4 { + pub config: __u64, + pub type_: __u32, + pub _bitfield_align_1: [u8; 0], + pub _bitfield_1: __BindgenBitfieldUnit<[u8; 4usize]>, + pub cookie: __u64, +} +impl bpf_link_info__bindgen_ty_1__bindgen_ty_11__bindgen_ty_1__bindgen_ty_4 { + #[inline] + pub fn new_bitfield_1() -> __BindgenBitfieldUnit<[u8; 4usize]> { + let mut __bindgen_bitfield_unit: __BindgenBitfieldUnit<[u8; 4usize]> = Default::default(); + __bindgen_bitfield_unit + } +} +impl bpf_link_info__bindgen_ty_1__bindgen_ty_11 { + #[inline] + pub fn new_bitfield_1() -> __BindgenBitfieldUnit<[u8; 4usize]> { + let mut __bindgen_bitfield_unit: __BindgenBitfieldUnit<[u8; 4usize]> = Default::default(); + __bindgen_bitfield_unit + } +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct bpf_link_info__bindgen_ty_1__bindgen_ty_12 { + pub ifindex: __u32, + pub attach_type: __u32, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct bpf_link_info__bindgen_ty_1__bindgen_ty_13 { + pub ifindex: __u32, + pub attach_type: __u32, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct bpf_func_info { + pub insn_off: __u32, + pub type_id: __u32, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct bpf_line_info { + pub insn_off: __u32, + pub file_name_off: __u32, + pub line_off: __u32, + pub line_col: __u32, +} +pub const BPF_F_TIMER_ABS: _bindgen_ty_41 = 1; +pub const BPF_F_TIMER_CPU_PIN: _bindgen_ty_41 = 2; +pub type _bindgen_ty_41 = ::core::ffi::c_uint; +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct btf_header { + pub magic: __u16, + pub version: __u8, + pub flags: __u8, + pub hdr_len: __u32, + pub type_off: __u32, + pub type_len: __u32, + pub str_off: __u32, + pub str_len: __u32, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub struct btf_type { + pub name_off: __u32, + pub info: __u32, + pub __bindgen_anon_1: btf_type__bindgen_ty_1, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub union btf_type__bindgen_ty_1 { + pub size: __u32, + pub type_: __u32, +} +pub const BTF_KIND_UNKN: _bindgen_ty_42 = 0; +pub const BTF_KIND_INT: _bindgen_ty_42 = 1; +pub const BTF_KIND_PTR: _bindgen_ty_42 = 2; +pub const BTF_KIND_ARRAY: _bindgen_ty_42 = 3; +pub const BTF_KIND_STRUCT: _bindgen_ty_42 = 4; +pub const BTF_KIND_UNION: _bindgen_ty_42 = 5; +pub const BTF_KIND_ENUM: _bindgen_ty_42 = 6; +pub const BTF_KIND_FWD: _bindgen_ty_42 = 7; +pub const BTF_KIND_TYPEDEF: _bindgen_ty_42 = 8; +pub const BTF_KIND_VOLATILE: _bindgen_ty_42 = 9; +pub const BTF_KIND_CONST: _bindgen_ty_42 = 10; +pub const BTF_KIND_RESTRICT: _bindgen_ty_42 = 11; +pub const BTF_KIND_FUNC: _bindgen_ty_42 = 12; +pub const BTF_KIND_FUNC_PROTO: _bindgen_ty_42 = 13; +pub const BTF_KIND_VAR: _bindgen_ty_42 = 14; +pub const BTF_KIND_DATASEC: _bindgen_ty_42 = 15; +pub const BTF_KIND_FLOAT: _bindgen_ty_42 = 16; +pub const BTF_KIND_DECL_TAG: _bindgen_ty_42 = 17; +pub const BTF_KIND_TYPE_TAG: _bindgen_ty_42 = 18; +pub const BTF_KIND_ENUM64: _bindgen_ty_42 = 19; +pub const NR_BTF_KINDS: _bindgen_ty_42 = 20; +pub const BTF_KIND_MAX: _bindgen_ty_42 = 19; +pub type _bindgen_ty_42 = ::core::ffi::c_uint; +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct btf_enum { + pub name_off: __u32, + pub val: __s32, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct btf_array { + pub type_: __u32, + pub index_type: __u32, + pub nelems: __u32, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct btf_member { + pub name_off: __u32, + pub type_: __u32, + pub offset: __u32, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct btf_param { + pub name_off: __u32, + pub type_: __u32, +} +pub const BTF_VAR_STATIC: _bindgen_ty_43 = 0; +pub const BTF_VAR_GLOBAL_ALLOCATED: _bindgen_ty_43 = 1; +pub const BTF_VAR_GLOBAL_EXTERN: _bindgen_ty_43 = 2; +pub type _bindgen_ty_43 = ::core::ffi::c_uint; +#[repr(u32)] +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub enum btf_func_linkage { + BTF_FUNC_STATIC = 0, + BTF_FUNC_GLOBAL = 1, + BTF_FUNC_EXTERN = 2, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct btf_var { + pub linkage: __u32, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct btf_var_secinfo { + pub type_: __u32, + pub offset: __u32, + pub size: __u32, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct btf_decl_tag { + pub component_idx: __s32, +} +pub const IFLA_XDP_UNSPEC: _bindgen_ty_92 = 0; +pub const IFLA_XDP_FD: _bindgen_ty_92 = 1; +pub const IFLA_XDP_ATTACHED: _bindgen_ty_92 = 2; +pub const IFLA_XDP_FLAGS: _bindgen_ty_92 = 3; +pub const IFLA_XDP_PROG_ID: _bindgen_ty_92 = 4; +pub const IFLA_XDP_DRV_PROG_ID: _bindgen_ty_92 = 5; +pub const IFLA_XDP_SKB_PROG_ID: _bindgen_ty_92 = 6; +pub const IFLA_XDP_HW_PROG_ID: _bindgen_ty_92 = 7; +pub const IFLA_XDP_EXPECTED_FD: _bindgen_ty_92 = 8; +pub const __IFLA_XDP_MAX: _bindgen_ty_92 = 9; +pub type _bindgen_ty_92 = ::core::ffi::c_uint; +#[repr(u32)] +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq, FromPrimitive)] +pub enum perf_type_id { + PERF_TYPE_HARDWARE = 0, + PERF_TYPE_SOFTWARE = 1, + PERF_TYPE_TRACEPOINT = 2, + PERF_TYPE_HW_CACHE = 3, + PERF_TYPE_RAW = 4, + PERF_TYPE_BREAKPOINT = 5, + PERF_TYPE_MAX = 6, +} +#[repr(u32)] +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub enum perf_hw_id { + PERF_COUNT_HW_CPU_CYCLES = 0, + PERF_COUNT_HW_INSTRUCTIONS = 1, + PERF_COUNT_HW_CACHE_REFERENCES = 2, + PERF_COUNT_HW_CACHE_MISSES = 3, + PERF_COUNT_HW_BRANCH_INSTRUCTIONS = 4, + PERF_COUNT_HW_BRANCH_MISSES = 5, + PERF_COUNT_HW_BUS_CYCLES = 6, + PERF_COUNT_HW_STALLED_CYCLES_FRONTEND = 7, + PERF_COUNT_HW_STALLED_CYCLES_BACKEND = 8, + PERF_COUNT_HW_REF_CPU_CYCLES = 9, + PERF_COUNT_HW_MAX = 10, +} +#[repr(u32)] +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub enum perf_hw_cache_id { + PERF_COUNT_HW_CACHE_L1D = 0, + PERF_COUNT_HW_CACHE_L1I = 1, + PERF_COUNT_HW_CACHE_LL = 2, + PERF_COUNT_HW_CACHE_DTLB = 3, + PERF_COUNT_HW_CACHE_ITLB = 4, + PERF_COUNT_HW_CACHE_BPU = 5, + PERF_COUNT_HW_CACHE_NODE = 6, + PERF_COUNT_HW_CACHE_MAX = 7, +} +#[repr(u32)] +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub enum perf_hw_cache_op_id { + PERF_COUNT_HW_CACHE_OP_READ = 0, + PERF_COUNT_HW_CACHE_OP_WRITE = 1, + PERF_COUNT_HW_CACHE_OP_PREFETCH = 2, + PERF_COUNT_HW_CACHE_OP_MAX = 3, +} +#[repr(u32)] +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub enum perf_hw_cache_op_result_id { + PERF_COUNT_HW_CACHE_RESULT_ACCESS = 0, + PERF_COUNT_HW_CACHE_RESULT_MISS = 1, + PERF_COUNT_HW_CACHE_RESULT_MAX = 2, +} +#[repr(u32)] +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq, FromPrimitive)] +pub enum perf_sw_ids { + PERF_COUNT_SW_CPU_CLOCK = 0, + PERF_COUNT_SW_TASK_CLOCK = 1, + PERF_COUNT_SW_PAGE_FAULTS = 2, + PERF_COUNT_SW_CONTEXT_SWITCHES = 3, + PERF_COUNT_SW_CPU_MIGRATIONS = 4, + PERF_COUNT_SW_PAGE_FAULTS_MIN = 5, + PERF_COUNT_SW_PAGE_FAULTS_MAJ = 6, + PERF_COUNT_SW_ALIGNMENT_FAULTS = 7, + PERF_COUNT_SW_EMULATION_FAULTS = 8, + PERF_COUNT_SW_DUMMY = 9, + PERF_COUNT_SW_BPF_OUTPUT = 10, + PERF_COUNT_SW_CGROUP_SWITCHES = 11, + PERF_COUNT_SW_MAX = 12, +} +#[repr(u32)] +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq, FromPrimitive)] +pub enum perf_event_sample_format { + PERF_SAMPLE_IP = 1, + PERF_SAMPLE_TID = 2, + PERF_SAMPLE_TIME = 4, + PERF_SAMPLE_ADDR = 8, + PERF_SAMPLE_READ = 16, + PERF_SAMPLE_CALLCHAIN = 32, + PERF_SAMPLE_ID = 64, + PERF_SAMPLE_CPU = 128, + PERF_SAMPLE_PERIOD = 256, + PERF_SAMPLE_STREAM_ID = 512, + PERF_SAMPLE_RAW = 1024, + PERF_SAMPLE_BRANCH_STACK = 2048, + PERF_SAMPLE_REGS_USER = 4096, + PERF_SAMPLE_STACK_USER = 8192, + PERF_SAMPLE_WEIGHT = 16384, + PERF_SAMPLE_DATA_SRC = 32768, + PERF_SAMPLE_IDENTIFIER = 65536, + PERF_SAMPLE_TRANSACTION = 131072, + PERF_SAMPLE_REGS_INTR = 262144, + PERF_SAMPLE_PHYS_ADDR = 524288, + PERF_SAMPLE_AUX = 1048576, + PERF_SAMPLE_CGROUP = 2097152, + PERF_SAMPLE_DATA_PAGE_SIZE = 4194304, + PERF_SAMPLE_CODE_PAGE_SIZE = 8388608, + PERF_SAMPLE_WEIGHT_STRUCT = 16777216, + PERF_SAMPLE_MAX = 33554432, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub struct perf_event_attr { + pub type_: __u32, + pub size: __u32, + pub config: __u64, + pub __bindgen_anon_1: perf_event_attr__bindgen_ty_1, + pub sample_type: __u64, + pub read_format: __u64, + pub _bitfield_align_1: [u32; 0], + pub _bitfield_1: __BindgenBitfieldUnit<[u8; 8usize]>, + pub __bindgen_anon_2: perf_event_attr__bindgen_ty_2, + pub bp_type: __u32, + pub __bindgen_anon_3: perf_event_attr__bindgen_ty_3, + pub __bindgen_anon_4: perf_event_attr__bindgen_ty_4, + pub branch_sample_type: __u64, + pub sample_regs_user: __u64, + pub sample_stack_user: __u32, + pub clockid: __s32, + pub sample_regs_intr: __u64, + pub aux_watermark: __u32, + pub sample_max_stack: __u16, + pub __reserved_2: __u16, + pub aux_sample_size: __u32, + pub __reserved_3: __u32, + pub sig_data: __u64, + pub config3: __u64, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub union perf_event_attr__bindgen_ty_1 { + pub sample_period: __u64, + pub sample_freq: __u64, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub union perf_event_attr__bindgen_ty_2 { + pub wakeup_events: __u32, + pub wakeup_watermark: __u32, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub union perf_event_attr__bindgen_ty_3 { + pub bp_addr: __u64, + pub kprobe_func: __u64, + pub uprobe_path: __u64, + pub config1: __u64, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub union perf_event_attr__bindgen_ty_4 { + pub bp_len: __u64, + pub kprobe_addr: __u64, + pub probe_offset: __u64, + pub config2: __u64, +} +impl perf_event_attr { + #[inline] + pub fn disabled(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(0usize, 1u8) as u64) } + } + #[inline] + pub fn set_disabled(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(0usize, 1u8, val as u64) + } + } + #[inline] + pub fn inherit(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(1usize, 1u8) as u64) } + } + #[inline] + pub fn set_inherit(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(1usize, 1u8, val as u64) + } + } + #[inline] + pub fn pinned(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(2usize, 1u8) as u64) } + } + #[inline] + pub fn set_pinned(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(2usize, 1u8, val as u64) + } + } + #[inline] + pub fn exclusive(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(3usize, 1u8) as u64) } + } + #[inline] + pub fn set_exclusive(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(3usize, 1u8, val as u64) + } + } + #[inline] + pub fn exclude_user(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(4usize, 1u8) as u64) } + } + #[inline] + pub fn set_exclude_user(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(4usize, 1u8, val as u64) + } + } + #[inline] + pub fn exclude_kernel(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(5usize, 1u8) as u64) } + } + #[inline] + pub fn set_exclude_kernel(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(5usize, 1u8, val as u64) + } + } + #[inline] + pub fn exclude_hv(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(6usize, 1u8) as u64) } + } + #[inline] + pub fn set_exclude_hv(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(6usize, 1u8, val as u64) + } + } + #[inline] + pub fn exclude_idle(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(7usize, 1u8) as u64) } + } + #[inline] + pub fn set_exclude_idle(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(7usize, 1u8, val as u64) + } + } + #[inline] + pub fn mmap(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(8usize, 1u8) as u64) } + } + #[inline] + pub fn set_mmap(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(8usize, 1u8, val as u64) + } + } + #[inline] + pub fn comm(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(9usize, 1u8) as u64) } + } + #[inline] + pub fn set_comm(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(9usize, 1u8, val as u64) + } + } + #[inline] + pub fn freq(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(10usize, 1u8) as u64) } + } + #[inline] + pub fn set_freq(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(10usize, 1u8, val as u64) + } + } + #[inline] + pub fn inherit_stat(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(11usize, 1u8) as u64) } + } + #[inline] + pub fn set_inherit_stat(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(11usize, 1u8, val as u64) + } + } + #[inline] + pub fn enable_on_exec(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(12usize, 1u8) as u64) } + } + #[inline] + pub fn set_enable_on_exec(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(12usize, 1u8, val as u64) + } + } + #[inline] + pub fn task(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(13usize, 1u8) as u64) } + } + #[inline] + pub fn set_task(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(13usize, 1u8, val as u64) + } + } + #[inline] + pub fn watermark(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(14usize, 1u8) as u64) } + } + #[inline] + pub fn set_watermark(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(14usize, 1u8, val as u64) + } + } + #[inline] + pub fn precise_ip(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(15usize, 2u8) as u64) } + } + #[inline] + pub fn set_precise_ip(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(15usize, 2u8, val as u64) + } + } + #[inline] + pub fn mmap_data(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(17usize, 1u8) as u64) } + } + #[inline] + pub fn set_mmap_data(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(17usize, 1u8, val as u64) + } + } + #[inline] + pub fn sample_id_all(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(18usize, 1u8) as u64) } + } + #[inline] + pub fn set_sample_id_all(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(18usize, 1u8, val as u64) + } + } + #[inline] + pub fn exclude_host(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(19usize, 1u8) as u64) } + } + #[inline] + pub fn set_exclude_host(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(19usize, 1u8, val as u64) + } + } + #[inline] + pub fn exclude_guest(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(20usize, 1u8) as u64) } + } + #[inline] + pub fn set_exclude_guest(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(20usize, 1u8, val as u64) + } + } + #[inline] + pub fn exclude_callchain_kernel(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(21usize, 1u8) as u64) } + } + #[inline] + pub fn set_exclude_callchain_kernel(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(21usize, 1u8, val as u64) + } + } + #[inline] + pub fn exclude_callchain_user(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(22usize, 1u8) as u64) } + } + #[inline] + pub fn set_exclude_callchain_user(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(22usize, 1u8, val as u64) + } + } + #[inline] + pub fn mmap2(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(23usize, 1u8) as u64) } + } + #[inline] + pub fn set_mmap2(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(23usize, 1u8, val as u64) + } + } + #[inline] + pub fn comm_exec(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(24usize, 1u8) as u64) } + } + #[inline] + pub fn set_comm_exec(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(24usize, 1u8, val as u64) + } + } + #[inline] + pub fn use_clockid(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(25usize, 1u8) as u64) } + } + #[inline] + pub fn set_use_clockid(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(25usize, 1u8, val as u64) + } + } + #[inline] + pub fn context_switch(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(26usize, 1u8) as u64) } + } + #[inline] + pub fn set_context_switch(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(26usize, 1u8, val as u64) + } + } + #[inline] + pub fn write_backward(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(27usize, 1u8) as u64) } + } + #[inline] + pub fn set_write_backward(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(27usize, 1u8, val as u64) + } + } + #[inline] + pub fn namespaces(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(28usize, 1u8) as u64) } + } + #[inline] + pub fn set_namespaces(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(28usize, 1u8, val as u64) + } + } + #[inline] + pub fn ksymbol(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(29usize, 1u8) as u64) } + } + #[inline] + pub fn set_ksymbol(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(29usize, 1u8, val as u64) + } + } + #[inline] + pub fn bpf_event(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(30usize, 1u8) as u64) } + } + #[inline] + pub fn set_bpf_event(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(30usize, 1u8, val as u64) + } + } + #[inline] + pub fn aux_output(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(31usize, 1u8) as u64) } + } + #[inline] + pub fn set_aux_output(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(31usize, 1u8, val as u64) + } + } + #[inline] + pub fn cgroup(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(32usize, 1u8) as u64) } + } + #[inline] + pub fn set_cgroup(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(32usize, 1u8, val as u64) + } + } + #[inline] + pub fn text_poke(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(33usize, 1u8) as u64) } + } + #[inline] + pub fn set_text_poke(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(33usize, 1u8, val as u64) + } + } + #[inline] + pub fn build_id(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(34usize, 1u8) as u64) } + } + #[inline] + pub fn set_build_id(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(34usize, 1u8, val as u64) + } + } + #[inline] + pub fn inherit_thread(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(35usize, 1u8) as u64) } + } + #[inline] + pub fn set_inherit_thread(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(35usize, 1u8, val as u64) + } + } + #[inline] + pub fn remove_on_exec(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(36usize, 1u8) as u64) } + } + #[inline] + pub fn set_remove_on_exec(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(36usize, 1u8, val as u64) + } + } + #[inline] + pub fn sigtrap(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(37usize, 1u8) as u64) } + } + #[inline] + pub fn set_sigtrap(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(37usize, 1u8, val as u64) + } + } + #[inline] + pub fn __reserved_1(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(38usize, 26u8) as u64) } + } + #[inline] + pub fn set___reserved_1(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(38usize, 26u8, val as u64) + } + } + #[inline] + pub fn new_bitfield_1( + disabled: __u64, + inherit: __u64, + pinned: __u64, + exclusive: __u64, + exclude_user: __u64, + exclude_kernel: __u64, + exclude_hv: __u64, + exclude_idle: __u64, + mmap: __u64, + comm: __u64, + freq: __u64, + inherit_stat: __u64, + enable_on_exec: __u64, + task: __u64, + watermark: __u64, + precise_ip: __u64, + mmap_data: __u64, + sample_id_all: __u64, + exclude_host: __u64, + exclude_guest: __u64, + exclude_callchain_kernel: __u64, + exclude_callchain_user: __u64, + mmap2: __u64, + comm_exec: __u64, + use_clockid: __u64, + context_switch: __u64, + write_backward: __u64, + namespaces: __u64, + ksymbol: __u64, + bpf_event: __u64, + aux_output: __u64, + cgroup: __u64, + text_poke: __u64, + build_id: __u64, + inherit_thread: __u64, + remove_on_exec: __u64, + sigtrap: __u64, + __reserved_1: __u64, + ) -> __BindgenBitfieldUnit<[u8; 8usize]> { + let mut __bindgen_bitfield_unit: __BindgenBitfieldUnit<[u8; 8usize]> = Default::default(); + __bindgen_bitfield_unit.set(0usize, 1u8, { + let disabled: u64 = unsafe { ::core::mem::transmute(disabled) }; + disabled as u64 + }); + __bindgen_bitfield_unit.set(1usize, 1u8, { + let inherit: u64 = unsafe { ::core::mem::transmute(inherit) }; + inherit as u64 + }); + __bindgen_bitfield_unit.set(2usize, 1u8, { + let pinned: u64 = unsafe { ::core::mem::transmute(pinned) }; + pinned as u64 + }); + __bindgen_bitfield_unit.set(3usize, 1u8, { + let exclusive: u64 = unsafe { ::core::mem::transmute(exclusive) }; + exclusive as u64 + }); + __bindgen_bitfield_unit.set(4usize, 1u8, { + let exclude_user: u64 = unsafe { ::core::mem::transmute(exclude_user) }; + exclude_user as u64 + }); + __bindgen_bitfield_unit.set(5usize, 1u8, { + let exclude_kernel: u64 = unsafe { ::core::mem::transmute(exclude_kernel) }; + exclude_kernel as u64 + }); + __bindgen_bitfield_unit.set(6usize, 1u8, { + let exclude_hv: u64 = unsafe { ::core::mem::transmute(exclude_hv) }; + exclude_hv as u64 + }); + __bindgen_bitfield_unit.set(7usize, 1u8, { + let exclude_idle: u64 = unsafe { ::core::mem::transmute(exclude_idle) }; + exclude_idle as u64 + }); + __bindgen_bitfield_unit.set(8usize, 1u8, { + let mmap: u64 = unsafe { ::core::mem::transmute(mmap) }; + mmap as u64 + }); + __bindgen_bitfield_unit.set(9usize, 1u8, { + let comm: u64 = unsafe { ::core::mem::transmute(comm) }; + comm as u64 + }); + __bindgen_bitfield_unit.set(10usize, 1u8, { + let freq: u64 = unsafe { ::core::mem::transmute(freq) }; + freq as u64 + }); + __bindgen_bitfield_unit.set(11usize, 1u8, { + let inherit_stat: u64 = unsafe { ::core::mem::transmute(inherit_stat) }; + inherit_stat as u64 + }); + __bindgen_bitfield_unit.set(12usize, 1u8, { + let enable_on_exec: u64 = unsafe { ::core::mem::transmute(enable_on_exec) }; + enable_on_exec as u64 + }); + __bindgen_bitfield_unit.set(13usize, 1u8, { + let task: u64 = unsafe { ::core::mem::transmute(task) }; + task as u64 + }); + __bindgen_bitfield_unit.set(14usize, 1u8, { + let watermark: u64 = unsafe { ::core::mem::transmute(watermark) }; + watermark as u64 + }); + __bindgen_bitfield_unit.set(15usize, 2u8, { + let precise_ip: u64 = unsafe { ::core::mem::transmute(precise_ip) }; + precise_ip as u64 + }); + __bindgen_bitfield_unit.set(17usize, 1u8, { + let mmap_data: u64 = unsafe { ::core::mem::transmute(mmap_data) }; + mmap_data as u64 + }); + __bindgen_bitfield_unit.set(18usize, 1u8, { + let sample_id_all: u64 = unsafe { ::core::mem::transmute(sample_id_all) }; + sample_id_all as u64 + }); + __bindgen_bitfield_unit.set(19usize, 1u8, { + let exclude_host: u64 = unsafe { ::core::mem::transmute(exclude_host) }; + exclude_host as u64 + }); + __bindgen_bitfield_unit.set(20usize, 1u8, { + let exclude_guest: u64 = unsafe { ::core::mem::transmute(exclude_guest) }; + exclude_guest as u64 + }); + __bindgen_bitfield_unit.set(21usize, 1u8, { + let exclude_callchain_kernel: u64 = + unsafe { ::core::mem::transmute(exclude_callchain_kernel) }; + exclude_callchain_kernel as u64 + }); + __bindgen_bitfield_unit.set(22usize, 1u8, { + let exclude_callchain_user: u64 = + unsafe { ::core::mem::transmute(exclude_callchain_user) }; + exclude_callchain_user as u64 + }); + __bindgen_bitfield_unit.set(23usize, 1u8, { + let mmap2: u64 = unsafe { ::core::mem::transmute(mmap2) }; + mmap2 as u64 + }); + __bindgen_bitfield_unit.set(24usize, 1u8, { + let comm_exec: u64 = unsafe { ::core::mem::transmute(comm_exec) }; + comm_exec as u64 + }); + __bindgen_bitfield_unit.set(25usize, 1u8, { + let use_clockid: u64 = unsafe { ::core::mem::transmute(use_clockid) }; + use_clockid as u64 + }); + __bindgen_bitfield_unit.set(26usize, 1u8, { + let context_switch: u64 = unsafe { ::core::mem::transmute(context_switch) }; + context_switch as u64 + }); + __bindgen_bitfield_unit.set(27usize, 1u8, { + let write_backward: u64 = unsafe { ::core::mem::transmute(write_backward) }; + write_backward as u64 + }); + __bindgen_bitfield_unit.set(28usize, 1u8, { + let namespaces: u64 = unsafe { ::core::mem::transmute(namespaces) }; + namespaces as u64 + }); + __bindgen_bitfield_unit.set(29usize, 1u8, { + let ksymbol: u64 = unsafe { ::core::mem::transmute(ksymbol) }; + ksymbol as u64 + }); + __bindgen_bitfield_unit.set(30usize, 1u8, { + let bpf_event: u64 = unsafe { ::core::mem::transmute(bpf_event) }; + bpf_event as u64 + }); + __bindgen_bitfield_unit.set(31usize, 1u8, { + let aux_output: u64 = unsafe { ::core::mem::transmute(aux_output) }; + aux_output as u64 + }); + __bindgen_bitfield_unit.set(32usize, 1u8, { + let cgroup: u64 = unsafe { ::core::mem::transmute(cgroup) }; + cgroup as u64 + }); + __bindgen_bitfield_unit.set(33usize, 1u8, { + let text_poke: u64 = unsafe { ::core::mem::transmute(text_poke) }; + text_poke as u64 + }); + __bindgen_bitfield_unit.set(34usize, 1u8, { + let build_id: u64 = unsafe { ::core::mem::transmute(build_id) }; + build_id as u64 + }); + __bindgen_bitfield_unit.set(35usize, 1u8, { + let inherit_thread: u64 = unsafe { ::core::mem::transmute(inherit_thread) }; + inherit_thread as u64 + }); + __bindgen_bitfield_unit.set(36usize, 1u8, { + let remove_on_exec: u64 = unsafe { ::core::mem::transmute(remove_on_exec) }; + remove_on_exec as u64 + }); + __bindgen_bitfield_unit.set(37usize, 1u8, { + let sigtrap: u64 = unsafe { ::core::mem::transmute(sigtrap) }; + sigtrap as u64 + }); + __bindgen_bitfield_unit.set(38usize, 26u8, { + let __reserved_1: u64 = unsafe { ::core::mem::transmute(__reserved_1) }; + __reserved_1 as u64 + }); + __bindgen_bitfield_unit + } +} +#[repr(C)] +#[derive(Copy, Clone)] +pub struct perf_event_mmap_page { + pub version: __u32, + pub compat_version: __u32, + pub lock: __u32, + pub index: __u32, + pub offset: __s64, + pub time_enabled: __u64, + pub time_running: __u64, + pub __bindgen_anon_1: perf_event_mmap_page__bindgen_ty_1, + pub pmc_width: __u16, + pub time_shift: __u16, + pub time_mult: __u32, + pub time_offset: __u64, + pub time_zero: __u64, + pub size: __u32, + pub __reserved_1: __u32, + pub time_cycles: __u64, + pub time_mask: __u64, + pub __reserved: [__u8; 928usize], + pub data_head: __u64, + pub data_tail: __u64, + pub data_offset: __u64, + pub data_size: __u64, + pub aux_head: __u64, + pub aux_tail: __u64, + pub aux_offset: __u64, + pub aux_size: __u64, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub union perf_event_mmap_page__bindgen_ty_1 { + pub capabilities: __u64, + pub __bindgen_anon_1: perf_event_mmap_page__bindgen_ty_1__bindgen_ty_1, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct perf_event_mmap_page__bindgen_ty_1__bindgen_ty_1 { + pub _bitfield_align_1: [u64; 0], + pub _bitfield_1: __BindgenBitfieldUnit<[u8; 8usize]>, +} +impl perf_event_mmap_page__bindgen_ty_1__bindgen_ty_1 { + #[inline] + pub fn cap_bit0(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(0usize, 1u8) as u64) } + } + #[inline] + pub fn set_cap_bit0(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(0usize, 1u8, val as u64) + } + } + #[inline] + pub fn cap_bit0_is_deprecated(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(1usize, 1u8) as u64) } + } + #[inline] + pub fn set_cap_bit0_is_deprecated(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(1usize, 1u8, val as u64) + } + } + #[inline] + pub fn cap_user_rdpmc(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(2usize, 1u8) as u64) } + } + #[inline] + pub fn set_cap_user_rdpmc(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(2usize, 1u8, val as u64) + } + } + #[inline] + pub fn cap_user_time(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(3usize, 1u8) as u64) } + } + #[inline] + pub fn set_cap_user_time(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(3usize, 1u8, val as u64) + } + } + #[inline] + pub fn cap_user_time_zero(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(4usize, 1u8) as u64) } + } + #[inline] + pub fn set_cap_user_time_zero(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(4usize, 1u8, val as u64) + } + } + #[inline] + pub fn cap_user_time_short(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(5usize, 1u8) as u64) } + } + #[inline] + pub fn set_cap_user_time_short(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(5usize, 1u8, val as u64) + } + } + #[inline] + pub fn cap_____res(&self) -> __u64 { + unsafe { ::core::mem::transmute(self._bitfield_1.get(6usize, 58u8) as u64) } + } + #[inline] + pub fn set_cap_____res(&mut self, val: __u64) { + unsafe { + let val: u64 = ::core::mem::transmute(val); + self._bitfield_1.set(6usize, 58u8, val as u64) + } + } + #[inline] + pub fn new_bitfield_1( + cap_bit0: __u64, + cap_bit0_is_deprecated: __u64, + cap_user_rdpmc: __u64, + cap_user_time: __u64, + cap_user_time_zero: __u64, + cap_user_time_short: __u64, + cap_____res: __u64, + ) -> __BindgenBitfieldUnit<[u8; 8usize]> { + let mut __bindgen_bitfield_unit: __BindgenBitfieldUnit<[u8; 8usize]> = Default::default(); + __bindgen_bitfield_unit.set(0usize, 1u8, { + let cap_bit0: u64 = unsafe { ::core::mem::transmute(cap_bit0) }; + cap_bit0 as u64 + }); + __bindgen_bitfield_unit.set(1usize, 1u8, { + let cap_bit0_is_deprecated: u64 = + unsafe { ::core::mem::transmute(cap_bit0_is_deprecated) }; + cap_bit0_is_deprecated as u64 + }); + __bindgen_bitfield_unit.set(2usize, 1u8, { + let cap_user_rdpmc: u64 = unsafe { ::core::mem::transmute(cap_user_rdpmc) }; + cap_user_rdpmc as u64 + }); + __bindgen_bitfield_unit.set(3usize, 1u8, { + let cap_user_time: u64 = unsafe { ::core::mem::transmute(cap_user_time) }; + cap_user_time as u64 + }); + __bindgen_bitfield_unit.set(4usize, 1u8, { + let cap_user_time_zero: u64 = unsafe { ::core::mem::transmute(cap_user_time_zero) }; + cap_user_time_zero as u64 + }); + __bindgen_bitfield_unit.set(5usize, 1u8, { + let cap_user_time_short: u64 = unsafe { ::core::mem::transmute(cap_user_time_short) }; + cap_user_time_short as u64 + }); + __bindgen_bitfield_unit.set(6usize, 58u8, { + let cap_____res: u64 = unsafe { ::core::mem::transmute(cap_____res) }; + cap_____res as u64 + }); + __bindgen_bitfield_unit + } +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct perf_event_header { + pub type_: __u32, + pub misc: __u16, + pub size: __u16, +} +#[repr(u32)] +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq, ToPrimitive)] +pub enum perf_event_type { + PERF_RECORD_MMAP = 1, + PERF_RECORD_LOST = 2, + PERF_RECORD_COMM = 3, + PERF_RECORD_EXIT = 4, + PERF_RECORD_THROTTLE = 5, + PERF_RECORD_UNTHROTTLE = 6, + PERF_RECORD_FORK = 7, + PERF_RECORD_READ = 8, + PERF_RECORD_SAMPLE = 9, + PERF_RECORD_MMAP2 = 10, + PERF_RECORD_AUX = 11, + PERF_RECORD_ITRACE_START = 12, + PERF_RECORD_LOST_SAMPLES = 13, + PERF_RECORD_SWITCH = 14, + PERF_RECORD_SWITCH_CPU_WIDE = 15, + PERF_RECORD_NAMESPACES = 16, + PERF_RECORD_KSYMBOL = 17, + PERF_RECORD_BPF_EVENT = 18, + PERF_RECORD_CGROUP = 19, + PERF_RECORD_TEXT_POKE = 20, + PERF_RECORD_AUX_OUTPUT_HW_ID = 21, + PERF_RECORD_MAX = 22, +} +pub const TCA_BPF_UNSPEC: _bindgen_ty_152 = 0; +pub const TCA_BPF_ACT: _bindgen_ty_152 = 1; +pub const TCA_BPF_POLICE: _bindgen_ty_152 = 2; +pub const TCA_BPF_CLASSID: _bindgen_ty_152 = 3; +pub const TCA_BPF_OPS_LEN: _bindgen_ty_152 = 4; +pub const TCA_BPF_OPS: _bindgen_ty_152 = 5; +pub const TCA_BPF_FD: _bindgen_ty_152 = 6; +pub const TCA_BPF_NAME: _bindgen_ty_152 = 7; +pub const TCA_BPF_FLAGS: _bindgen_ty_152 = 8; +pub const TCA_BPF_FLAGS_GEN: _bindgen_ty_152 = 9; +pub const TCA_BPF_TAG: _bindgen_ty_152 = 10; +pub const TCA_BPF_ID: _bindgen_ty_152 = 11; +pub const __TCA_BPF_MAX: _bindgen_ty_152 = 12; +pub type _bindgen_ty_152 = ::core::ffi::c_uint; +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct ifinfomsg { + pub ifi_family: ::core::ffi::c_uchar, + pub __ifi_pad: ::core::ffi::c_uchar, + pub ifi_type: ::core::ffi::c_ushort, + pub ifi_index: ::core::ffi::c_int, + pub ifi_flags: ::core::ffi::c_uint, + pub ifi_change: ::core::ffi::c_uint, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct tcmsg { + pub tcm_family: ::core::ffi::c_uchar, + pub tcm__pad1: ::core::ffi::c_uchar, + pub tcm__pad2: ::core::ffi::c_ushort, + pub tcm_ifindex: ::core::ffi::c_int, + pub tcm_handle: __u32, + pub tcm_parent: __u32, + pub tcm_info: __u32, +} +pub const TCA_UNSPEC: _bindgen_ty_170 = 0; +pub const TCA_KIND: _bindgen_ty_170 = 1; +pub const TCA_OPTIONS: _bindgen_ty_170 = 2; +pub const TCA_STATS: _bindgen_ty_170 = 3; +pub const TCA_XSTATS: _bindgen_ty_170 = 4; +pub const TCA_RATE: _bindgen_ty_170 = 5; +pub const TCA_FCNT: _bindgen_ty_170 = 6; +pub const TCA_STATS2: _bindgen_ty_170 = 7; +pub const TCA_STAB: _bindgen_ty_170 = 8; +pub const TCA_PAD: _bindgen_ty_170 = 9; +pub const TCA_DUMP_INVISIBLE: _bindgen_ty_170 = 10; +pub const TCA_CHAIN: _bindgen_ty_170 = 11; +pub const TCA_HW_OFFLOAD: _bindgen_ty_170 = 12; +pub const TCA_INGRESS_BLOCK: _bindgen_ty_170 = 13; +pub const TCA_EGRESS_BLOCK: _bindgen_ty_170 = 14; +pub const __TCA_MAX: _bindgen_ty_170 = 15; +pub type _bindgen_ty_170 = ::core::ffi::c_uint; +pub const AYA_PERF_EVENT_IOC_ENABLE: ::core::ffi::c_int = 9216; +pub const AYA_PERF_EVENT_IOC_DISABLE: ::core::ffi::c_int = 9217; +pub const AYA_PERF_EVENT_IOC_SET_BPF: ::core::ffi::c_int = 1074013192; diff --git a/kernel/src/include/bindings/mod.rs b/kernel/src/include/bindings/mod.rs index ee999761..4ab2c21e 100644 --- a/kernel/src/include/bindings/mod.rs +++ b/kernel/src/include/bindings/mod.rs @@ -1,2 +1,10 @@ -#[allow(clippy::module_inception)] +#![allow( + dead_code, + non_camel_case_types, + non_snake_case, + clippy::all, + missing_docs, + clippy::module_inception +)] pub mod bindings; +pub mod linux_bpf; diff --git a/kernel/src/init/init.rs b/kernel/src/init/init.rs index 953f0974..653e2f4d 100644 --- a/kernel/src/init/init.rs +++ b/kernel/src/init/init.rs @@ -1,5 +1,3 @@ -use log::warn; - use crate::{ arch::{ init::{early_setup_arch, setup_arch, setup_arch_post}, @@ -30,6 +28,7 @@ use crate::{ clocksource::clocksource_boot_finish, timekeeping::timekeeping_init, timer::timer_init, }, }; +use log::warn; use super::{ boot::{boot_callback_except_early, boot_callbacks}, @@ -89,9 +88,8 @@ fn do_start_kernel() { kthread_init(); setup_arch_post().expect("setup_arch_post failed"); clocksource_boot_finish(); - Futex::init(); - + crate::bpf::init_bpf_system(); #[cfg(all(target_arch = "x86_64", feature = "kvm"))] crate::virt::kvm::kvm_init(); } diff --git a/kernel/src/lib.rs b/kernel/src/lib.rs index aa67c0a7..5a7e0953 100644 --- a/kernel/src/lib.rs +++ b/kernel/src/lib.rs @@ -21,6 +21,7 @@ #![feature(slice_ptr_get)] #![feature(sync_unsafe_cell)] #![feature(vec_into_raw_parts)] +#![feature(c_variadic)] #![cfg_attr(target_os = "none", no_std)] #![allow(internal_features)] // clippy的配置 @@ -46,6 +47,7 @@ mod arch; mod libs; #[macro_use] mod include; +mod bpf; mod debug; mod driver; // 如果driver依赖了libs，应该在libs后面导出 mod exception; @@ -55,12 +57,12 @@ mod ipc; mod misc; mod mm; mod net; +mod perf; mod process; mod sched; mod smp; mod syscall; mod time; - #[cfg(target_arch = "x86_64")] mod virt; diff --git a/kernel/src/mm/fault.rs b/kernel/src/mm/fault.rs index 457b911f..e0aa1622 100644 --- a/kernel/src/mm/fault.rs +++ b/kernel/src/mm/fault.rs @@ -272,16 +272,16 @@ impl PageFaultHandler { /// - VmFaultReason: 页面错误处理信息标志 pub unsafe fn do_fault(pfm: &mut PageFaultMessage) -> VmFaultReason { if !pfm.flags().contains(FaultFlags::FAULT_FLAG_WRITE) { - return Self::do_read_fault(pfm); + Self::do_read_fault(pfm) } else if !pfm .vma() .lock_irqsave() .vm_flags() .contains(VmFlags::VM_SHARED) { - return Self::do_cow_fault(pfm); + Self::do_cow_fault(pfm) } else { - return Self::do_shared_fault(pfm); + Self::do_shared_fault(pfm) } } diff --git a/kernel/src/mm/ucontext.rs b/kernel/src/mm/ucontext.rs index 8892bf2c..81abd4b0 100644 --- a/kernel/src/mm/ucontext.rs +++ b/kernel/src/mm/ucontext.rs @@ -377,7 +377,7 @@ impl InnerAddressSpace { PageFrameCount::from_bytes(len).unwrap(), prot_flags, map_flags, - move |page, count, vm_flags, flags, mapper, flusher| { + |page, count, vm_flags, flags, mapper, flusher| { if allocate_at_once { VMA::zeroed( page, @@ -386,7 +386,7 @@ impl InnerAddressSpace { flags, mapper, flusher, - file, + file.clone(), Some(pgoff), ) } else { @@ -394,13 +394,17 @@ impl InnerAddressSpace { VirtRegion::new(page.virt_address(), count.data() * MMArch::PAGE_SIZE), vm_flags, flags, - file, + file.clone(), Some(pgoff), false, ))) } }, )?; + // todo!(impl mmap for other file) + // https://github.com/DragonOS-Community/DragonOS/pull/912#discussion_r1765334272 + let file = file.unwrap(); + let _ = file.inode().mmap(start_vaddr.data(), len, offset); return Ok(start_page); } diff --git a/kernel/src/perf/bpf.rs b/kernel/src/perf/bpf.rs new file mode 100644 index 00000000..84af5436 --- /dev/null +++ b/kernel/src/perf/bpf.rs @@ -0,0 +1,333 @@ +use super::{PerfEventOps, Result}; +use crate::arch::mm::LockedFrameAllocator; +use crate::arch::MMArch; +use crate::filesystem::vfs::file::PageCache; +use crate::filesystem::vfs::{FilePrivateData, FileSystem, IndexNode}; +use crate::include::bindings::linux_bpf::{ + perf_event_header, perf_event_mmap_page, perf_event_type, +}; +use crate::libs::spinlock::{SpinLock, SpinLockGuard}; +use crate::mm::allocator::page_frame::{FrameAllocator, PageFrameCount, PhysPageFrame}; +use crate::mm::page::{page_manager_lock_irqsave, Page}; +use crate::mm::{MemoryManagementArch, PhysAddr}; +use crate::perf::util::{LostSamples, PerfProbeArgs, PerfSample, SampleHeader}; +use alloc::string::String; +use alloc::sync::Arc; +use alloc::vec::Vec; +use core::any::Any; +use core::fmt::Debug; +use system_error::SystemError; +const PAGE_SIZE: usize = MMArch::PAGE_SIZE; +#[derive(Debug)] +pub struct BpfPerfEvent { + _args: PerfProbeArgs, + data: SpinLock, +} + +#[derive(Debug)] +pub struct BpfPerfEventData { + enabled: bool, + mmap_page: RingPage, + page_cache: Arc, + offset: usize, +} + +#[derive(Debug)] +pub struct RingPage { + size: usize, + ptr: usize, + data_region_size: usize, + lost: usize, + phys_addr: PhysAddr, +} + +impl RingPage { + pub fn empty() -> Self { + RingPage { + ptr: 0, + size: 0, + data_region_size: 0, + lost: 0, + phys_addr: PhysAddr::new(0), + } + } + + pub fn new_init(start: usize, len: usize, phys_addr: PhysAddr) -> Self { + Self::init(start as _, len, phys_addr) + } + + fn init(ptr: *mut u8, size: usize, phys_addr: PhysAddr) -> Self { + assert_eq!(size % PAGE_SIZE, 0); + assert!(size / PAGE_SIZE >= 2); + // The first page will be filled with perf_event_mmap_page + unsafe { + let perf_event_mmap_page = &mut *(ptr as *mut perf_event_mmap_page); + perf_event_mmap_page.data_offset = PAGE_SIZE as u64; + perf_event_mmap_page.data_size = (size - PAGE_SIZE) as u64; + // user will read sample or lost record from data_tail + perf_event_mmap_page.data_tail = 0; + // kernel will write sample or lost record from data_head + perf_event_mmap_page.data_head = 0; + // It is a ring buffer. + } + RingPage { + ptr: ptr as usize, + size, + data_region_size: size - PAGE_SIZE, + lost: 0, + phys_addr, + } + } + + fn can_write(&self, data_size: usize, data_tail: usize, data_head: usize) -> bool { + if (data_head + 1) % self.data_region_size == data_tail { + // The buffer is full + return false; + } + let capacity = if data_head >= data_tail { + self.data_region_size - data_head + data_tail + } else { + data_tail - data_head + }; + data_size <= capacity + } + + pub fn write_event(&mut self, data: &[u8]) -> Result<()> { + let data_tail = unsafe { &mut (*(self.ptr as *mut perf_event_mmap_page)).data_tail }; + let data_head = unsafe { &mut (*(self.ptr as *mut perf_event_mmap_page)).data_head }; + // data_tail..data_head is the region that can be written + // check if there is enough space to write the event + let sample_size = PerfSample::calculate_size(data.len()); + + let can_write_sample = + self.can_write(sample_size, *data_tail as usize, *data_head as usize); + // log::error!( + // "can_write_sample: {}, data_tail: {}, data_head: {}, data.len(): {}, region_size: {}", + // can_write_sample, + // *data_tail, + // *data_head, + // data.len(), + // self.data_region_size + // ); + if !can_write_sample { + //we need record it to the lost record + self.lost += 1; + // log::error!( + // "Lost record: {}, data_tail: {}, data_head: {}", + // self.lost, + // *data_tail, + // *data_head + // ); + Ok(()) + } else { + // we can write the sample to the page + // If the lost record is not zero, we need to write the lost record first. + let can_write_lost_record = self.can_write( + size_of::(), + *data_tail as usize, + *data_head as usize, + ); + if self.lost > 0 && can_write_lost_record { + let new_data_head = self.write_lost(*data_head as usize)?; + *data_head = new_data_head as u64; + // log::info!( + // "Write lost record: {}, data_tail: {}, new_data_head: {}", + // self.lost, + // *data_tail, + // *data_head + // ); + self.lost = 0; + self.write_event(data) + } else { + let new_data_head = self.write_sample(data, *data_head as usize)?; + *data_head = new_data_head as u64; + // log::info!( + // "Write sample record, data_tail: {}, new_data_head: {}", + // *data_tail, + // *data_head + // ); + Ok(()) + } + } + } + + /// Write any data to the page. + /// + /// Return the new data_head + fn write_any(&mut self, data: &[u8], data_head: usize) -> Result { + let data_region_len = self.data_region_size; + let data_region = self.as_mut_slice()[PAGE_SIZE..].as_mut(); + let data_len = data.len(); + let end = (data_head + data_len) % data_region_len; + let start = data_head; + if start < end { + data_region[start..end].copy_from_slice(data); + } else { + let first_len = data_region_len - start; + data_region[start..start + first_len].copy_from_slice(&data[..first_len]); + data_region[0..end].copy_from_slice(&data[first_len..]); + } + Ok(end) + } + + /// Write a sample to the page. + fn write_sample(&mut self, data: &[u8], data_head: usize) -> Result { + let perf_sample = PerfSample { + s_hdr: SampleHeader { + header: perf_event_header { + type_: perf_event_type::PERF_RECORD_SAMPLE as u32, + misc: 0, + size: size_of::() as u16 + data.len() as u16, + }, + size: data.len() as u32, + }, + value: data, + }; + let new_head = self.write_any(perf_sample.s_hdr.as_bytes(), data_head)?; + self.write_any(perf_sample.value, new_head) + } + + /// Write a lost record to the page. + /// + /// Return the new data_head + fn write_lost(&mut self, data_head: usize) -> Result { + let lost = LostSamples { + header: perf_event_header { + type_: perf_event_type::PERF_RECORD_LOST as u32, + misc: 0, + size: size_of::() as u16, + }, + id: 0, + count: self.lost as u64, + }; + self.write_any(lost.as_bytes(), data_head) + } + + pub fn readable(&self) -> bool { + let data_tail = unsafe { &(*(self.ptr as *mut perf_event_mmap_page)).data_tail }; + let data_head = unsafe { &(*(self.ptr as *mut perf_event_mmap_page)).data_head }; + data_tail != data_head + } + pub fn as_slice(&self) -> &[u8] { + unsafe { core::slice::from_raw_parts(self.ptr as *const u8, self.size) } + } + pub fn as_mut_slice(&mut self) -> &mut [u8] { + unsafe { core::slice::from_raw_parts_mut(self.ptr as *mut u8, self.size) } + } +} + +impl BpfPerfEvent { + pub fn new(args: PerfProbeArgs) -> Self { + BpfPerfEvent { + _args: args, + data: SpinLock::new(BpfPerfEventData { + enabled: false, + mmap_page: RingPage::empty(), + page_cache: PageCache::new(None), + offset: 0, + }), + } + } + pub fn do_mmap(&self, _start: usize, len: usize, offset: usize) -> Result<()> { + let mut data = self.data.lock(); + // alloc page frame + let (phy_addr, page_count) = + unsafe { LockedFrameAllocator.allocate(PageFrameCount::new(len / PAGE_SIZE)) } + .ok_or(SystemError::ENOSPC)?; + let mut page_manager_guard = page_manager_lock_irqsave(); + let mut cur_phys = PhysPageFrame::new(phy_addr); + for i in 0..page_count.data() { + let page = Arc::new(Page::new(true, cur_phys.phys_address())); + let paddr = cur_phys.phys_address(); + page_manager_guard.insert(paddr, &page); + data.page_cache.add_page(i, &page); + cur_phys = cur_phys.next(); + } + let virt_addr = unsafe { MMArch::phys_2_virt(phy_addr) }.ok_or(SystemError::EFAULT)?; + // create mmap page + let mmap_page = RingPage::new_init(virt_addr.data(), len, phy_addr); + data.mmap_page = mmap_page; + data.offset = offset; + Ok(()) + } + + pub fn write_event(&self, data: &[u8]) -> Result<()> { + let mut inner_data = self.data.lock(); + inner_data.mmap_page.write_event(data)?; + Ok(()) + } +} + +impl Drop for BpfPerfEvent { + fn drop(&mut self) { + let mut page_manager_guard = page_manager_lock_irqsave(); + let data = self.data.lock(); + let phy_addr = data.mmap_page.phys_addr; + let len = data.mmap_page.size; + let page_count = PageFrameCount::new(len / PAGE_SIZE); + let mut cur_phys = PhysPageFrame::new(phy_addr); + for _ in 0..page_count.data() { + page_manager_guard.remove_page(&cur_phys.phys_address()); + cur_phys = cur_phys.next(); + } + } +} + +impl IndexNode for BpfPerfEvent { + fn mmap(&self, start: usize, len: usize, offset: usize) -> Result<()> { + self.do_mmap(start, len, offset) + } + + fn read_at( + &self, + _offset: usize, + _len: usize, + _buf: &mut [u8], + _data: SpinLockGuard, + ) -> Result { + panic!("PerfEventInode does not support read") + } + + fn write_at( + &self, + _offset: usize, + _len: usize, + _buf: &[u8], + _data: SpinLockGuard, + ) -> Result { + panic!("PerfEventInode does not support write") + } + + fn fs(&self) -> Arc { + panic!("PerfEventInode does not have a filesystem") + } + + fn as_any_ref(&self) -> &dyn Any { + self + } + fn list(&self) -> Result> { + Err(SystemError::ENOSYS) + } + + fn page_cache(&self) -> Option> { + Some(self.data.lock().page_cache.clone()) + } +} + +impl PerfEventOps for BpfPerfEvent { + fn enable(&self) -> Result<()> { + self.data.lock().enabled = true; + Ok(()) + } + fn disable(&self) -> Result<()> { + self.data.lock().enabled = false; + Ok(()) + } + fn readable(&self) -> bool { + self.data.lock().mmap_page.readable() + } +} + +pub fn perf_event_open_bpf(args: PerfProbeArgs) -> BpfPerfEvent { + BpfPerfEvent::new(args) +} diff --git a/kernel/src/perf/kprobe.rs b/kernel/src/perf/kprobe.rs new file mode 100644 index 00000000..54ac829b --- /dev/null +++ b/kernel/src/perf/kprobe.rs @@ -0,0 +1,159 @@ +use super::Result; +use crate::arch::interrupt::TrapFrame; +use crate::arch::kprobe::KProbeContext; +use crate::bpf::helper::BPF_HELPER_FUN_SET; +use crate::bpf::prog::BpfProg; +use crate::debug::kprobe::args::KprobeInfo; +use crate::debug::kprobe::{register_kprobe, unregister_kprobe, LockKprobe}; +use crate::filesystem::vfs::file::{File, PageCache}; +use crate::filesystem::vfs::{FilePrivateData, FileSystem, IndexNode}; +use crate::libs::casting::DowncastArc; +use crate::libs::spinlock::SpinLockGuard; +use crate::perf::util::PerfProbeArgs; +use crate::perf::PerfEventOps; +use alloc::boxed::Box; +use alloc::string::String; +use alloc::sync::Arc; +use alloc::vec::Vec; +use core::any::Any; +use core::fmt::Debug; +use kprobe::{CallBackFunc, ProbeArgs}; +use rbpf::EbpfVmRawOwned; +use system_error::SystemError; +#[derive(Debug)] +pub struct KprobePerfEvent { + _args: PerfProbeArgs, + kprobe: LockKprobe, +} + +impl Drop for KprobePerfEvent { + fn drop(&mut self) { + unregister_kprobe(self.kprobe.clone()); + } +} + +impl KprobePerfEvent { + pub fn do_set_bpf_prog(&self, prog_file: Arc) -> Result<()> { + let file = prog_file + .inode() + .downcast_arc::() + .ok_or(SystemError::EINVAL)?; + let prog_slice = file.insns(); + let mut vm = + EbpfVmRawOwned::new(Some(prog_slice.to_vec())).map_err(|_| SystemError::EINVAL)?; + vm.register_helper_set(BPF_HELPER_FUN_SET.get()) + .map_err(|_| SystemError::EINVAL)?; + // create a callback to execute the ebpf prog + let callback = Box::new(KprobePerfCallBack::new(file, vm)); + // update callback for kprobe + self.kprobe.write().update_event_callback(callback); + Ok(()) + } +} + +pub struct KprobePerfCallBack { + _bpf_prog_file: Arc, + vm: EbpfVmRawOwned, +} + +impl KprobePerfCallBack { + fn new(bpf_prog_file: Arc, vm: EbpfVmRawOwned) -> Self { + Self { + _bpf_prog_file: bpf_prog_file, + vm, + } + } +} + +impl CallBackFunc for KprobePerfCallBack { + fn call(&self, trap_frame: &dyn ProbeArgs) { + let trap_frame = trap_frame.as_any().downcast_ref::().unwrap(); + let pt_regs = KProbeContext::from(trap_frame); + let probe_context = unsafe { + core::slice::from_raw_parts_mut( + &pt_regs as *const KProbeContext as *mut u8, + size_of::(), + ) + }; + let _res = self + .vm + .execute_program(probe_context) + .map_err(|_| SystemError::EINVAL); + } +} + +impl IndexNode for KprobePerfEvent { + fn read_at( + &self, + _offset: usize, + _len: usize, + _buf: &mut [u8], + _data: SpinLockGuard, + ) -> Result { + panic!("read_at not implemented for PerfEvent"); + } + + fn write_at( + &self, + _offset: usize, + _len: usize, + _buf: &[u8], + _data: SpinLockGuard, + ) -> Result { + panic!("write_at not implemented for PerfEvent"); + } + + fn fs(&self) -> Arc { + panic!("fs not implemented for PerfEvent"); + } + + fn as_any_ref(&self) -> &dyn Any { + self + } + + fn list(&self) -> Result> { + Err(SystemError::ENOSYS) + } + + fn page_cache(&self) -> Option> { + None + } +} + +impl PerfEventOps for KprobePerfEvent { + fn set_bpf_prog(&self, bpf_prog: Arc) -> Result<()> { + self.do_set_bpf_prog(bpf_prog) + } + fn enable(&self) -> Result<()> { + self.kprobe.write().enable(); + Ok(()) + } + fn disable(&self) -> Result<()> { + self.kprobe.write().disable(); + Ok(()) + } + + fn readable(&self) -> bool { + true + } +} + +pub fn perf_event_open_kprobe(args: PerfProbeArgs) -> KprobePerfEvent { + let symbol = args.name.clone(); + log::info!("create kprobe for symbol: {symbol}"); + let kprobe_info = KprobeInfo { + pre_handler: |_| {}, + post_handler: |_| {}, + fault_handler: None, + event_callback: None, + symbol: Some(symbol), + addr: None, + offset: 0, + enable: false, + }; + let kprobe = register_kprobe(kprobe_info).expect("create kprobe failed"); + KprobePerfEvent { + _args: args, + kprobe, + } +} diff --git a/kernel/src/perf/mod.rs b/kernel/src/perf/mod.rs new file mode 100644 index 00000000..0b6de2ad --- /dev/null +++ b/kernel/src/perf/mod.rs @@ -0,0 +1,337 @@ +mod bpf; +mod kprobe; +mod util; + +use crate::filesystem::vfs::file::{File, FileMode, PageCache}; +use crate::filesystem::vfs::syscall::ModeType; +use crate::filesystem::vfs::{ + FilePrivateData, FileSystem, FileType, FsInfo, IndexNode, Metadata, SuperBlock, +}; +use crate::include::bindings::linux_bpf::{ + perf_event_attr, perf_event_sample_format, perf_sw_ids, perf_type_id, +}; +use crate::libs::casting::DowncastArc; +use crate::libs::spinlock::{SpinLock, SpinLockGuard}; +use crate::mm::fault::{PageFaultHandler, PageFaultMessage}; +use crate::mm::VmFaultReason; +use crate::net::event_poll::{EPollEventType, EPollItem, EventPoll, KernelIoctlData}; +use crate::perf::bpf::BpfPerfEvent; +use crate::perf::util::{PerfEventIoc, PerfEventOpenFlags, PerfProbeArgs}; +use crate::process::ProcessManager; +use crate::syscall::user_access::UserBufferReader; +use crate::syscall::Syscall; +use alloc::boxed::Box; +use alloc::collections::LinkedList; +use alloc::string::String; +use alloc::sync::{Arc, Weak}; +use alloc::vec::Vec; +use core::any::Any; +use core::ffi::c_void; +use core::fmt::Debug; +use core::ops::Deref; +use intertrait::{CastFrom, CastFromSync}; +use log::info; +use num_traits::FromPrimitive; +use system_error::SystemError; + +type Result = core::result::Result; + +pub trait PerfEventOps: Send + Sync + Debug + CastFromSync + CastFrom + IndexNode { + /// Set the bpf program for the perf event + fn set_bpf_prog(&self, _bpf_prog: Arc) -> Result<()> { + Err(SystemError::ENOSYS) + } + /// Enable the perf event + fn enable(&self) -> Result<()> { + Err(SystemError::ENOSYS) + } + /// Disable the perf event + fn disable(&self) -> Result<()> { + Err(SystemError::ENOSYS) + } + /// Whether the perf event is readable + fn readable(&self) -> bool; +} + +#[derive(Debug)] +pub struct PerfEventInode { + event: Box, + epitems: SpinLock>>, +} + +impl PerfEventInode { + pub fn new(event: Box) -> Self { + Self { + event, + epitems: SpinLock::new(LinkedList::new()), + } + } + pub fn remove_epoll( + &self, + epoll: &Weak>, + ) -> core::result::Result<(), SystemError> { + let is_remove = !self + .epitems + .lock_irqsave() + .extract_if(|x| x.epoll().ptr_eq(epoll)) + .collect::>() + .is_empty(); + if is_remove { + return Ok(()); + } + Err(SystemError::ENOENT) + } + fn do_poll(&self) -> Result { + let mut events = EPollEventType::empty(); + if self.event.readable() { + events |= EPollEventType::EPOLLIN | EPollEventType::EPOLLRDNORM; + } + return Ok(events.bits() as usize); + } + fn epoll_callback(&self) -> Result<()> { + let pollflag = EPollEventType::from_bits_truncate(self.do_poll()? as u32); + // 唤醒epoll中等待的进程 + EventPoll::wakeup_epoll(&self.epitems, Some(pollflag)) + } +} + +impl Deref for PerfEventInode { + type Target = Box; + + fn deref(&self) -> &Self::Target { + &self.event + } +} + +impl IndexNode for PerfEventInode { + fn mmap(&self, start: usize, len: usize, offset: usize) -> Result<()> { + self.event.mmap(start, len, offset) + } + fn open(&self, _data: SpinLockGuard, _mode: &FileMode) -> Result<()> { + Ok(()) + } + fn close(&self, _data: SpinLockGuard) -> Result<()> { + Ok(()) + } + fn read_at( + &self, + _offset: usize, + _len: usize, + _buf: &mut [u8], + _data: SpinLockGuard, + ) -> Result { + panic!("read_at not implemented for PerfEvent"); + } + + fn write_at( + &self, + _offset: usize, + _len: usize, + _buf: &[u8], + _data: SpinLockGuard, + ) -> Result { + panic!("write_at not implemented for PerfEvent"); + } + + fn poll(&self, _private_data: &FilePrivateData) -> Result { + self.do_poll() + } + + fn metadata(&self) -> Result { + let meta = Metadata { + mode: ModeType::from_bits_truncate(0o755), + file_type: FileType::File, + ..Default::default() + }; + Ok(meta) + } + + fn resize(&self, _len: usize) -> Result<()> { + Ok(()) + } + + fn ioctl(&self, cmd: u32, data: usize, _private_data: &FilePrivateData) -> Result { + let req = PerfEventIoc::from_u32(cmd).ok_or(SystemError::EINVAL)?; + info!("perf_event_ioctl: request: {:?}, arg: {}", req, data); + match req { + PerfEventIoc::Enable => { + self.event.enable()?; + Ok(0) + } + PerfEventIoc::Disable => { + self.event.disable()?; + Ok(0) + } + PerfEventIoc::SetBpf => { + info!("perf_event_ioctl: PERF_EVENT_IOC_SET_BPF, arg: {}", data); + let bpf_prog_fd = data; + let fd_table = ProcessManager::current_pcb().fd_table(); + let file = fd_table + .read() + .get_file_by_fd(bpf_prog_fd as _) + .ok_or(SystemError::EBADF)?; + self.event.set_bpf_prog(file)?; + Ok(0) + } + } + } + + fn kernel_ioctl( + &self, + arg: Arc, + _data: &FilePrivateData, + ) -> core::result::Result { + let epitem = arg + .arc_any() + .downcast::() + .map_err(|_| SystemError::EFAULT)?; + self.epitems.lock().push_back(epitem); + Ok(0) + } + + fn fs(&self) -> Arc { + // panic!("PerfEvent does not have a filesystem") + Arc::new(PerfFakeFs) + } + fn as_any_ref(&self) -> &dyn Any { + self + } + fn list(&self) -> Result> { + Err(SystemError::ENOSYS) + } + fn page_cache(&self) -> Option> { + self.event.page_cache() + } +} + +#[derive(Debug)] +struct PerfFakeFs; + +impl FileSystem for PerfFakeFs { + fn root_inode(&self) -> Arc { + panic!("PerfFakeFs does not have a root inode") + } + + fn info(&self) -> FsInfo { + panic!("PerfFakeFs does not have a filesystem info") + } + + fn as_any_ref(&self) -> &dyn Any { + self + } + + fn name(&self) -> &str { + "perf" + } + + fn super_block(&self) -> SuperBlock { + panic!("PerfFakeFs does not have a super block") + } + unsafe fn fault(&self, pfm: &mut PageFaultMessage) -> VmFaultReason { + let res = PageFaultHandler::filemap_fault(pfm); + res + } + unsafe fn map_pages( + &self, + pfm: &mut PageFaultMessage, + start_pgoff: usize, + end_pgoff: usize, + ) -> VmFaultReason { + PageFaultHandler::filemap_map_pages(pfm, start_pgoff, end_pgoff) + } +} + +impl Syscall { + pub fn sys_perf_event_open( + attr: *const u8, + pid: i32, + cpu: i32, + group_fd: i32, + flags: u32, + ) -> Result { + let buf = UserBufferReader::new( + attr as *const perf_event_attr, + size_of::(), + true, + )?; + let attr = buf.read_one_from_user(0)?; + perf_event_open(attr, pid, cpu, group_fd, flags) + } +} + +pub fn perf_event_open( + attr: &perf_event_attr, + pid: i32, + cpu: i32, + group_fd: i32, + flags: u32, +) -> Result { + let args = PerfProbeArgs::try_from(attr, pid, cpu, group_fd, flags)?; + log::info!("perf_event_process: {:#?}", args); + let file_mode = if args + .flags + .contains(PerfEventOpenFlags::PERF_FLAG_FD_CLOEXEC) + { + FileMode::O_RDWR | FileMode::O_CLOEXEC + } else { + FileMode::O_RDWR + }; + + let event: Box = match args.type_ { + // Kprobe + // See /sys/bus/event_source/devices/kprobe/type + perf_type_id::PERF_TYPE_MAX => { + let kprobe_event = kprobe::perf_event_open_kprobe(args); + Box::new(kprobe_event) + } + perf_type_id::PERF_TYPE_SOFTWARE => { + // For bpf prog output + assert_eq!(args.config, perf_sw_ids::PERF_COUNT_SW_BPF_OUTPUT); + assert_eq!( + args.sample_type, + Some(perf_event_sample_format::PERF_SAMPLE_RAW) + ); + let bpf_event = bpf::perf_event_open_bpf(args); + Box::new(bpf_event) + } + _ => { + unimplemented!("perf_event_process: unknown type: {:?}", args); + } + }; + + let page_cache = event.page_cache(); + let perf_event = Arc::new(PerfEventInode::new(event)); + if let Some(cache) = page_cache { + cache.set_inode(Arc::downgrade(&(perf_event.clone() as _)))?; + } + let file = File::new(perf_event, file_mode)?; + let fd_table = ProcessManager::current_pcb().fd_table(); + let fd = fd_table.write().alloc_fd(file, None).map(|x| x as usize)?; + Ok(fd) +} + +pub fn perf_event_output(_ctx: *mut c_void, fd: usize, _flags: u32, data: &[u8]) -> Result<()> { + let file = get_perf_event_file(fd)?; + let bpf_event_file = file.deref().deref(); + let bpf_event_file = bpf_event_file + .deref() + .ref_any() + .downcast_ref::() + .ok_or(SystemError::EINVAL)?; + bpf_event_file.write_event(data)?; + file.epoll_callback()?; + Ok(()) +} + +fn get_perf_event_file(fd: usize) -> Result> { + let fd_table = ProcessManager::current_pcb().fd_table(); + let file = fd_table + .read() + .get_file_by_fd(fd as _) + .ok_or(SystemError::EBADF)?; + let event = file + .inode() + .downcast_arc::() + .ok_or(SystemError::EINVAL)?; + Ok(event) +} diff --git a/kernel/src/perf/util.rs b/kernel/src/perf/util.rs new file mode 100644 index 00000000..b1cc43eb --- /dev/null +++ b/kernel/src/perf/util.rs @@ -0,0 +1,123 @@ +use crate::include::bindings::linux_bpf::{ + perf_event_attr, perf_event_header, perf_event_sample_format, perf_sw_ids, perf_type_id, +}; +use crate::syscall::user_access::check_and_clone_cstr; +use alloc::string::String; +use num_traits::FromPrimitive; +use system_error::SystemError; + +bitflags! { + pub struct PerfEventOpenFlags: u32 { + const PERF_FLAG_FD_NO_GROUP = 1; + const PERF_FLAG_FD_OUTPUT = 2; + const PERF_FLAG_PID_CGROUP = 4; + const PERF_FLAG_FD_CLOEXEC = 8; + } +} + +/// The `PerfEventIoc` enum is used to define the ioctl commands for perf events. +/// +/// See https://elixir.bootlin.com/linux/v6.1/source/include/uapi/linux/perf_event.h#L544 +#[repr(u32)] +#[derive(Debug, Copy, Clone, FromPrimitive)] +pub enum PerfEventIoc { + /// Equivalent to [crate::include::bindings::linux_bpf::AYA_PERF_EVENT_IOC_ENABLE]. + Enable = 9216, + /// Equivalent to [crate::include::bindings::linux_bpf::AYA_PERF_EVENT_IOC_DISABLE]. + Disable = 9217, + /// Equivalent to [crate::include::bindings::linux_bpf::AYA_PERF_EVENT_IOC_SET_BPF]. + SetBpf = 1074013192, +} + +#[derive(Debug, Clone)] +#[allow(unused)] +/// `perf_event_open` syscall arguments. +pub struct PerfProbeArgs { + pub config: perf_sw_ids, + pub name: String, + pub offset: u64, + pub size: u32, + pub type_: perf_type_id, + pub pid: i32, + pub cpu: i32, + pub group_fd: i32, + pub flags: PerfEventOpenFlags, + pub sample_type: Option, +} + +impl PerfProbeArgs { + pub fn try_from( + attr: &perf_event_attr, + pid: i32, + cpu: i32, + group_fd: i32, + flags: u32, + ) -> Result { + let ty = perf_type_id::from_u32(attr.type_).ok_or(SystemError::EINVAL)?; + let config = perf_sw_ids::from_u32(attr.config as u32).ok_or(SystemError::EINVAL)?; + let name = if ty == perf_type_id::PERF_TYPE_MAX { + let name_ptr = unsafe { attr.__bindgen_anon_3.config1 } as *const u8; + let name = check_and_clone_cstr(name_ptr, None)?; + name.into_string().map_err(|_| SystemError::EINVAL)? + } else { + String::new() + }; + let sample_ty = perf_event_sample_format::from_u32(attr.sample_type as u32); + let args = PerfProbeArgs { + config, + name, + offset: unsafe { attr.__bindgen_anon_4.config2 }, + size: attr.size, + type_: ty, + pid, + cpu, + group_fd, + flags: PerfEventOpenFlags::from_bits_truncate(flags), + sample_type: sample_ty, + }; + Ok(args) + } +} + +/// The event type in our particular use case will be `PERF_RECORD_SAMPLE` or `PERF_RECORD_LOST`. +/// `PERF_RECORD_SAMPLE` indicating that there is an actual sample after this header. +/// And `PERF_RECORD_LOST` indicating that there is a record lost header following the perf event header. +#[repr(C)] +#[derive(Debug)] +pub struct LostSamples { + pub header: perf_event_header, + pub id: u64, + pub count: u64, +} + +impl LostSamples { + pub fn as_bytes(&self) -> &[u8] { + unsafe { core::slice::from_raw_parts(self as *const Self as *const u8, size_of::()) } + } +} + +#[repr(C)] +#[derive(Debug)] +pub struct SampleHeader { + pub header: perf_event_header, + pub size: u32, +} + +impl SampleHeader { + pub fn as_bytes(&self) -> &[u8] { + unsafe { core::slice::from_raw_parts(self as *const Self as *const u8, size_of::()) } + } +} + +#[repr(C)] +#[derive(Debug)] +pub struct PerfSample<'a> { + pub s_hdr: SampleHeader, + pub value: &'a [u8], +} + +impl<'a> PerfSample<'a> { + pub fn calculate_size(value_size: usize) -> usize { + size_of::() + value_size + } +} diff --git a/kernel/src/smp/cpu/mod.rs b/kernel/src/smp/cpu/mod.rs index 186184ef..d3b36002 100644 --- a/kernel/src/smp/cpu/mod.rs +++ b/kernel/src/smp/cpu/mod.rs @@ -134,7 +134,6 @@ impl SmpCpuManager { &self.possible_cpus } - #[allow(dead_code)] pub fn possible_cpus_count(&self) -> u32 { self.possible_cnt.load(core::sync::atomic::Ordering::SeqCst) } diff --git a/kernel/src/syscall/mod.rs b/kernel/src/syscall/mod.rs index c61c2db5..6e7e9f7b 100644 --- a/kernel/src/syscall/mod.rs +++ b/kernel/src/syscall/mod.rs @@ -1159,6 +1159,20 @@ impl Syscall { let flags = args[1] as u32; Self::sys_eventfd(initval, flags) } + SYS_BPF => { + let cmd = args[0] as u32; + let attr = args[1] as *mut u8; + let size = args[2] as u32; + Self::sys_bpf(cmd, attr, size) + } + SYS_PERF_EVENT_OPEN => { + let attr = args[0] as *const u8; + let pid = args[1] as i32; + let cpu = args[2] as i32; + let group_fd = args[3] as i32; + let flags = args[4] as u32; + Self::sys_perf_event_open(attr, pid, cpu, group_fd, flags) + } _ => panic!("Unsupported syscall ID: {}", syscall_num), }; diff --git a/kernel/src/time/syscall.rs b/kernel/src/time/syscall.rs index 76f9349e..ec8c0a15 100644 --- a/kernel/src/time/syscall.rs +++ b/kernel/src/time/syscall.rs @@ -2,8 +2,6 @@ use core::{ ffi::{c_int, c_longlong}, time::Duration, }; - -use log::warn; use num_traits::FromPrimitive; use system_error::SystemError; @@ -139,7 +137,7 @@ impl Syscall { pub fn clock_gettime(clock_id: c_int, tp: *mut PosixTimeSpec) -> Result { let clock_id = PosixClockID::try_from(clock_id)?; if clock_id != PosixClockID::Realtime { - warn!("clock_gettime: currently only support Realtime clock, but got {:?}. Defaultly return realtime!!!\n", clock_id); + // warn!("clock_gettime: currently only support Realtime clock, but got {:?}. Defaultly return realtime!!!\n", clock_id); } if tp.is_null() { return Err(SystemError::EFAULT); diff --git a/user/apps/test_ebpf/.gitignore b/user/apps/test_ebpf/.gitignore new file mode 100644 index 00000000..1ac35461 --- /dev/null +++ b/user/apps/test_ebpf/.gitignore @@ -0,0 +1,3 @@ +/target +Cargo.lock +/install/ \ No newline at end of file diff --git a/user/apps/test_ebpf/Cargo.toml b/user/apps/test_ebpf/Cargo.toml new file mode 100644 index 00000000..ab4a3b7f --- /dev/null +++ b/user/apps/test_ebpf/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "test_ebpf" +version = "0.1.0" +edition = "2021" + +[dependencies] +aya = { git = "https://git.mirrors.dragonos.org.cn/DragonOS-Community/tiny-aya.git", rev = "0689f13" } +aya-log = { git = "https://git.mirrors.dragonos.org.cn/DragonOS-Community/tiny-aya.git", rev = "0689f13" } + +log = "0.4.22" +env_logger = "0.11.5" +tokio = { version = "1.25", features = ["macros", "rt", "rt-multi-thread", "net", "signal", "time"] } + +[profile.release] +lto = true +strip = true diff --git a/user/apps/test_ebpf/Makefile b/user/apps/test_ebpf/Makefile new file mode 100644 index 00000000..f2770a32 --- /dev/null +++ b/user/apps/test_ebpf/Makefile @@ -0,0 +1,61 @@ +TOOLCHAIN="+nightly-2024-07-23-x86_64-unknown-linux-gnu" +RUSTFLAGS+="" + +ifdef DADK_CURRENT_BUILD_DIR +# 如果是在dadk中编译，那么安装到dadk的安装目录中 + INSTALL_DIR = $(DADK_CURRENT_BUILD_DIR) +else +# 如果是在本地编译，那么安装到当前目录下的install目录中 + INSTALL_DIR = ./install +endif + +ifeq ($(ARCH), x86_64) + export RUST_TARGET=x86_64-unknown-linux-musl +else ifeq ($(ARCH), riscv64) + export RUST_TARGET=riscv64gc-unknown-linux-gnu +else +# 默认为x86_86，用于本地编译 + export RUST_TARGET=x86_64-unknown-linux-musl +endif + +run: + RUSTFLAGS=$(RUSTFLAGS) cargo $(TOOLCHAIN) run --target $(RUST_TARGET) + +build:build-ebpf + RUSTFLAGS=$(RUSTFLAGS) cargo $(TOOLCHAIN) build --target $(RUST_TARGET) + +clean:clean-ebpf + RUSTFLAGS=$(RUSTFLAGS) cargo $(TOOLCHAIN) clean --target $(RUST_TARGET) + +test: + RUSTFLAGS=$(RUSTFLAGS) cargo $(TOOLCHAIN) test --target $(RUST_TARGET) + +doc: + RUSTFLAGS=$(RUSTFLAGS) cargo $(TOOLCHAIN) doc --target $(RUST_TARGET) + +fmt: + RUSTFLAGS=$(RUSTFLAGS) cargo $(TOOLCHAIN) fmt + +fmt-check: + RUSTFLAGS=$(RUSTFLAGS) cargo $(TOOLCHAIN) fmt --check + +run-release: + RUSTFLAGS=$(RUSTFLAGS) cargo $(TOOLCHAIN) run --target $(RUST_TARGET) --release + +build-release:build-ebpf + RUSTFLAGS=$(RUSTFLAGS) cargo $(TOOLCHAIN) build --target $(RUST_TARGET) --release + +clean-release:clean-ebpf + RUSTFLAGS=$(RUSTFLAGS) cargo $(TOOLCHAIN) clean --target $(RUST_TARGET) --release + +test-release: + RUSTFLAGS=$(RUSTFLAGS) cargo $(TOOLCHAIN) test --target $(RUST_TARGET) --release + +build-ebpf: + cd ./syscall_ebpf && RUST_LOG=debug cargo xtask build --release +clean-ebpf: + cd ./syscall_ebpf && cargo clean + +.PHONY: install +install:build-ebpf + RUSTFLAGS=$(RUSTFLAGS) cargo $(TOOLCHAIN) install --target $(RUST_TARGET) --path . --no-track --root $(INSTALL_DIR) --force diff --git a/user/apps/test_ebpf/src/main.rs b/user/apps/test_ebpf/src/main.rs new file mode 100644 index 00000000..1909aeeb --- /dev/null +++ b/user/apps/test_ebpf/src/main.rs @@ -0,0 +1,60 @@ +use aya::maps::HashMap; +use aya::programs::KProbe; +use aya::{include_bytes_aligned, Ebpf}; +use aya_log::EbpfLogger; +use log::{info, warn}; +use std::error::Error; +use tokio::task::yield_now; +use tokio::{signal, time}; + +#[tokio::main(flavor = "current_thread")] +async fn main() -> Result<(), Box> { + env_logger::builder() + .filter_level(log::LevelFilter::Warn) + .format_timestamp(None) + .init(); + + let mut bpf = Ebpf::load(include_bytes_aligned!( + "../syscall_ebpf/target/bpfel-unknown-none/release/syscall_ebpf" + ))?; + + // create a async task to read the log + if let Err(e) = EbpfLogger::init(&mut bpf) { + // This can happen if you remove all log statements from your eBPF program. + warn!("failed to initialize eBPF logger: {}", e); + } + + let program: &mut KProbe = bpf.program_mut("syscall_ebpf").unwrap().try_into()?; + program.load()?; + program.attach("dragonos_kernel::syscall::Syscall::handle", 0)?; + + info!("attacch the kprobe to dragonos_kernel::syscall::Syscall::handle"); + + // print the value of the blocklist per 5 seconds + tokio::spawn(async move { + let blocklist: HashMap<_, u32, u32> = + HashMap::try_from(bpf.map("SYSCALL_LIST").unwrap()).unwrap(); + let mut now = time::Instant::now(); + loop { + let new_now = time::Instant::now(); + let duration = new_now.duration_since(now); + if duration.as_secs() >= 5 { + println!("------------SYSCALL_LIST----------------"); + let iter = blocklist.iter(); + for item in iter { + if let Ok((key, value)) = item { + println!("syscall: {:?}, count: {:?}", key, value); + } + } + println!("----------------------------------------"); + now = new_now; + } + yield_now().await; + } + }); + + info!("Waiting for Ctrl-C..."); + signal::ctrl_c().await?; + info!("Exiting..."); + Ok(()) +} diff --git a/user/apps/test_ebpf/syscall_ebpf/.cargo/config.toml b/user/apps/test_ebpf/syscall_ebpf/.cargo/config.toml new file mode 100644 index 00000000..35049cbc --- /dev/null +++ b/user/apps/test_ebpf/syscall_ebpf/.cargo/config.toml @@ -0,0 +1,2 @@ +[alias] +xtask = "run --package xtask --" diff --git a/user/apps/test_ebpf/syscall_ebpf/.gitignore b/user/apps/test_ebpf/syscall_ebpf/.gitignore new file mode 100644 index 00000000..9db7029f --- /dev/null +++ b/user/apps/test_ebpf/syscall_ebpf/.gitignore @@ -0,0 +1,9 @@ +### https://raw.github.com/github/gitignore/master/Rust.gitignore + +# Generated by Cargo +# will have compiled files and executables +debug/ +target/ + +# These are backup files generated by rustfmt +**/*.rs.bk diff --git a/user/apps/test_ebpf/syscall_ebpf/.vscode/settings.json b/user/apps/test_ebpf/syscall_ebpf/.vscode/settings.json new file mode 100644 index 00000000..0c82ac97 --- /dev/null +++ b/user/apps/test_ebpf/syscall_ebpf/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "rust-analyzer.linkedProjects": ["Cargo.toml", "syscall_ebpf-ebpf/Cargo.toml"] +} diff --git a/user/apps/test_ebpf/syscall_ebpf/Cargo.toml b/user/apps/test_ebpf/syscall_ebpf/Cargo.toml new file mode 100644 index 00000000..6eb4e632 --- /dev/null +++ b/user/apps/test_ebpf/syscall_ebpf/Cargo.toml @@ -0,0 +1,3 @@ +[workspace] +resolver = "2" +members = ["xtask", "syscall_ebpf-common"] diff --git a/user/apps/test_ebpf/syscall_ebpf/README.md b/user/apps/test_ebpf/syscall_ebpf/README.md new file mode 100644 index 00000000..fe5ed32d --- /dev/null +++ b/user/apps/test_ebpf/syscall_ebpf/README.md @@ -0,0 +1,32 @@ +# syscall_ebpf + +## Prerequisites + +1. Install bpf-linker: `cargo install bpf-linker` + +## Build eBPF + +```bash +cargo xtask build-ebpf +``` + +To perform a release build you can use the `--release` flag. +You may also change the target architecture with the `--target` flag. + +## Build Userspace + +```bash +cargo build +``` + +## Build eBPF and Userspace + +```bash +cargo xtask build +``` + +## Run + +```bash +RUST_LOG=info cargo xtask run +``` diff --git a/user/apps/test_ebpf/syscall_ebpf/syscall_ebpf-common/Cargo.toml b/user/apps/test_ebpf/syscall_ebpf/syscall_ebpf-common/Cargo.toml new file mode 100644 index 00000000..7acc25d4 --- /dev/null +++ b/user/apps/test_ebpf/syscall_ebpf/syscall_ebpf-common/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "syscall_ebpf-common" +version = "0.1.0" +edition = "2021" + +[features] +default = [] +user = ["aya"] + +[dependencies] +aya = { git = "https://git.mirrors.dragonos.org.cn/DragonOS-Community/tiny-aya.git", rev = "0689f13", optional = true } + +[lib] +path = "src/lib.rs" diff --git a/user/apps/test_ebpf/syscall_ebpf/syscall_ebpf-common/src/lib.rs b/user/apps/test_ebpf/syscall_ebpf/syscall_ebpf-common/src/lib.rs new file mode 100644 index 00000000..0c9ac1ac --- /dev/null +++ b/user/apps/test_ebpf/syscall_ebpf/syscall_ebpf-common/src/lib.rs @@ -0,0 +1 @@ +#![no_std] diff --git a/user/apps/test_ebpf/syscall_ebpf/syscall_ebpf-ebpf/.cargo/config.toml b/user/apps/test_ebpf/syscall_ebpf/syscall_ebpf-ebpf/.cargo/config.toml new file mode 100644 index 00000000..4302a7f1 --- /dev/null +++ b/user/apps/test_ebpf/syscall_ebpf/syscall_ebpf-ebpf/.cargo/config.toml @@ -0,0 +1,6 @@ +[build] +target-dir = "../target" +target = "bpfel-unknown-none" + +[unstable] +build-std = ["core"] diff --git a/user/apps/test_ebpf/syscall_ebpf/syscall_ebpf-ebpf/.helix/config.toml b/user/apps/test_ebpf/syscall_ebpf/syscall_ebpf-ebpf/.helix/config.toml new file mode 100644 index 00000000..da5424f1 --- /dev/null +++ b/user/apps/test_ebpf/syscall_ebpf/syscall_ebpf-ebpf/.helix/config.toml @@ -0,0 +1,2 @@ +[editor] +workspace-lsp-roots = [] diff --git a/user/apps/test_ebpf/syscall_ebpf/syscall_ebpf-ebpf/.vim/coc-settings.json b/user/apps/test_ebpf/syscall_ebpf/syscall_ebpf-ebpf/.vim/coc-settings.json new file mode 100644 index 00000000..e2211a64 --- /dev/null +++ b/user/apps/test_ebpf/syscall_ebpf/syscall_ebpf-ebpf/.vim/coc-settings.json @@ -0,0 +1,4 @@ +{ + "rust-analyzer.cargo.target": "bpfel-unknown-none", + "rust-analyzer.checkOnSave.allTargets": false +} diff --git a/user/apps/test_ebpf/syscall_ebpf/syscall_ebpf-ebpf/.vscode/settings.json b/user/apps/test_ebpf/syscall_ebpf/syscall_ebpf-ebpf/.vscode/settings.json new file mode 100644 index 00000000..e2211a64 --- /dev/null +++ b/user/apps/test_ebpf/syscall_ebpf/syscall_ebpf-ebpf/.vscode/settings.json @@ -0,0 +1,4 @@ +{ + "rust-analyzer.cargo.target": "bpfel-unknown-none", + "rust-analyzer.checkOnSave.allTargets": false +} diff --git a/user/apps/test_ebpf/syscall_ebpf/syscall_ebpf-ebpf/Cargo.toml b/user/apps/test_ebpf/syscall_ebpf/syscall_ebpf-ebpf/Cargo.toml new file mode 100644 index 00000000..1911fa43 --- /dev/null +++ b/user/apps/test_ebpf/syscall_ebpf/syscall_ebpf-ebpf/Cargo.toml @@ -0,0 +1,33 @@ +[package] +name = "syscall_ebpf-ebpf" +version = "0.1.0" +edition = "2021" + +[dependencies] +aya-ebpf = { git = "https://git.mirrors.dragonos.org.cn/DragonOS-Community/aya.git", rev = "3d57d35" } +aya-log-ebpf = { git = "https://git.mirrors.dragonos.org.cn/DragonOS-Community/aya.git", rev = "3d57d35" } + +syscall_ebpf-common = { path = "../syscall_ebpf-common" } + +[[bin]] +name = "syscall_ebpf" +path = "src/main.rs" + +[profile.dev] +opt-level = 3 +debug = false +debug-assertions = false +overflow-checks = false +lto = true +panic = "abort" +incremental = false +codegen-units = 1 +rpath = false + +[profile.release] +lto = true +panic = "abort" +codegen-units = 1 + +[workspace] +members = [] diff --git a/user/apps/test_ebpf/syscall_ebpf/syscall_ebpf-ebpf/rust-toolchain.toml b/user/apps/test_ebpf/syscall_ebpf/syscall_ebpf-ebpf/rust-toolchain.toml new file mode 100644 index 00000000..447b8f04 --- /dev/null +++ b/user/apps/test_ebpf/syscall_ebpf/syscall_ebpf-ebpf/rust-toolchain.toml @@ -0,0 +1,13 @@ +[toolchain] +channel = "nightly-2024-07-23" +# The source code of rustc, provided by the rust-src component, is needed for +# building eBPF programs. +components = [ + "cargo", + "clippy", + "rust-docs", + "rust-src", + "rust-std", + "rustc", + "rustfmt", +] diff --git a/user/apps/test_ebpf/syscall_ebpf/syscall_ebpf-ebpf/src/main.rs b/user/apps/test_ebpf/syscall_ebpf/syscall_ebpf-ebpf/src/main.rs new file mode 100644 index 00000000..7f9b79b6 --- /dev/null +++ b/user/apps/test_ebpf/syscall_ebpf/syscall_ebpf-ebpf/src/main.rs @@ -0,0 +1,44 @@ +#![no_std] +#![no_main] + +use aya_ebpf::{macros::kprobe, programs::ProbeContext}; +use aya_ebpf::macros::map; +use aya_ebpf::maps::HashMap; +use aya_log_ebpf::info; + +#[kprobe] +pub fn syscall_ebpf(ctx: ProbeContext) -> u32 { + try_syscall_ebpf(ctx).unwrap_or_else(|ret| ret) +} + +fn try_syscall_ebpf(ctx: ProbeContext) -> Result { + let pt_regs = unsafe { + &*ctx.regs + }; + // first arg -> rdi + // second arg -> rsi + // third arg -> rdx + // four arg -> rcx + let syscall_num = pt_regs.rsi as usize; + if syscall_num != 1 { + unsafe { + if let Some(v) = SYSCALL_LIST.get(&(syscall_num as u32)){ + let new_v = *v + 1; + SYSCALL_LIST.insert(&(syscall_num as u32), &new_v,0).unwrap(); + }else { + SYSCALL_LIST.insert(&(syscall_num as u32), &1,0).unwrap(); + } + } + info!(&ctx, "invoke syscall {}", syscall_num); + } + Ok(0) +} + +#[map] // +static SYSCALL_LIST: HashMap = + HashMap::::with_max_entries(1024, 0); + +#[panic_handler] +fn panic(_info: &core::panic::PanicInfo) -> ! { + unsafe { core::hint::unreachable_unchecked() } +} diff --git a/user/apps/test_ebpf/syscall_ebpf/xtask/Cargo.toml b/user/apps/test_ebpf/syscall_ebpf/xtask/Cargo.toml new file mode 100644 index 00000000..c4dea5d1 --- /dev/null +++ b/user/apps/test_ebpf/syscall_ebpf/xtask/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "xtask" +version = "0.1.0" +edition = "2021" + +[dependencies] +anyhow = "1" +clap = { version = "4.1", features = ["derive"] } diff --git a/user/apps/test_ebpf/syscall_ebpf/xtask/src/build.rs b/user/apps/test_ebpf/syscall_ebpf/xtask/src/build.rs new file mode 100644 index 00000000..ddeee449 --- /dev/null +++ b/user/apps/test_ebpf/syscall_ebpf/xtask/src/build.rs @@ -0,0 +1,42 @@ +use std::process::Command; + +use anyhow::Context as _; +use clap::Parser; + +use crate::build_ebpf::{build_ebpf, Architecture, Options as BuildOptions}; + +#[derive(Debug, Parser)] +pub struct Options { + /// Set the endianness of the BPF target + #[clap(default_value = "bpfel-unknown-none", long)] + pub bpf_target: Architecture, + /// Build and run the release target + #[clap(long)] + pub release: bool, +} + +/// Build the project +fn build_project(opts: &Options) -> Result<(), anyhow::Error> { + let mut args = vec!["build"]; + if opts.release { + args.push("--release") + } + let status = Command::new("cargo") + .args(&args) + .status() + .expect("failed to build userspace"); + assert!(status.success()); + Ok(()) +} + +/// Build our ebpf program and the project +pub fn build(opts: Options) -> Result<(), anyhow::Error> { + // build our ebpf program followed by our application + build_ebpf(BuildOptions { + target: opts.bpf_target, + release: opts.release, + }) + .context("Error while building eBPF program")?; + build_project(&opts).context("Error while building userspace application")?; + Ok(()) +} \ No newline at end of file diff --git a/user/apps/test_ebpf/syscall_ebpf/xtask/src/build_ebpf.rs b/user/apps/test_ebpf/syscall_ebpf/xtask/src/build_ebpf.rs new file mode 100644 index 00000000..8c6e323f --- /dev/null +++ b/user/apps/test_ebpf/syscall_ebpf/xtask/src/build_ebpf.rs @@ -0,0 +1,67 @@ +use std::{path::PathBuf, process::Command}; + +use clap::Parser; + +#[derive(Debug, Copy, Clone)] +pub enum Architecture { + BpfEl, + BpfEb, +} + +impl std::str::FromStr for Architecture { + type Err = String; + + fn from_str(s: &str) -> Result { + Ok(match s { + "bpfel-unknown-none" => Architecture::BpfEl, + "bpfeb-unknown-none" => Architecture::BpfEb, + _ => return Err("invalid target".to_owned()), + }) + } +} + +impl std::fmt::Display for Architecture { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + Architecture::BpfEl => "bpfel-unknown-none", + Architecture::BpfEb => "bpfeb-unknown-none", + }) + } +} + +#[derive(Debug, Parser)] +pub struct Options { + /// Set the endianness of the BPF target + #[clap(default_value = "bpfel-unknown-none", long)] + pub target: Architecture, + /// Build the release target + #[clap(long)] + pub release: bool, +} + +pub fn build_ebpf(opts: Options) -> Result<(), anyhow::Error> { + let dir = PathBuf::from("syscall_ebpf-ebpf"); + let target = format!("--target={}", opts.target); + let mut args = vec![ + "build", + target.as_str(), + "-Z", + "build-std=core", + ]; + if opts.release { + args.push("--release") + } + + // Command::new creates a child process which inherits all env variables. This means env + // vars set by the cargo xtask command are also inherited. RUSTUP_TOOLCHAIN is removed + // so the rust-toolchain.toml file in the -ebpf folder is honored. + + let status = Command::new("cargo") + .current_dir(dir) + .env_remove("RUSTUP_TOOLCHAIN") + .args(&args) + .status() + .expect("failed to build bpf program"); + assert!(status.success()); + Ok(()) +} diff --git a/user/apps/test_ebpf/syscall_ebpf/xtask/src/main.rs b/user/apps/test_ebpf/syscall_ebpf/xtask/src/main.rs new file mode 100644 index 00000000..50794589 --- /dev/null +++ b/user/apps/test_ebpf/syscall_ebpf/xtask/src/main.rs @@ -0,0 +1,36 @@ +mod build_ebpf; +mod build; +mod run; + +use std::process::exit; + +use clap::Parser; + +#[derive(Debug, Parser)] +pub struct Options { + #[clap(subcommand)] + command: Command, +} + +#[derive(Debug, Parser)] +enum Command { + BuildEbpf(build_ebpf::Options), + Build(build::Options), + Run(run::Options), +} + +fn main() { + let opts = Options::parse(); + + use Command::*; + let ret = match opts.command { + BuildEbpf(opts) => build_ebpf::build_ebpf(opts), + Run(opts) => run::run(opts), + Build(opts) => build::build(opts), + }; + + if let Err(e) = ret { + eprintln!("{e:#}"); + exit(1); + } +} diff --git a/user/apps/test_ebpf/syscall_ebpf/xtask/src/run.rs b/user/apps/test_ebpf/syscall_ebpf/xtask/src/run.rs new file mode 100644 index 00000000..19af11c4 --- /dev/null +++ b/user/apps/test_ebpf/syscall_ebpf/xtask/src/run.rs @@ -0,0 +1,55 @@ +use std::process::Command; + +use anyhow::Context as _; +use clap::Parser; + +use crate::{build::{build, Options as BuildOptions}, build_ebpf::Architecture}; + +#[derive(Debug, Parser)] +pub struct Options { + /// Set the endianness of the BPF target + #[clap(default_value = "bpfel-unknown-none", long)] + pub bpf_target: Architecture, + /// Build and run the release target + #[clap(long)] + pub release: bool, + /// The command used to wrap your application + #[clap(short, long, default_value = "sudo -E")] + pub runner: String, + /// Arguments to pass to your application + #[clap(name = "args", last = true)] + pub run_args: Vec, +} + + +/// Build and run the project +pub fn run(opts: Options) -> Result<(), anyhow::Error> { + // Build our ebpf program and the project + build(BuildOptions{ + bpf_target: opts.bpf_target, + release: opts.release, + }).context("Error while building project")?; + + // profile we are building (release or debug) + let profile = if opts.release { "release" } else { "debug" }; + let bin_path = format!("target/{profile}/syscall_ebpf"); + + // arguments to pass to the application + let mut run_args: Vec<_> = opts.run_args.iter().map(String::as_str).collect(); + + // configure args + let mut args: Vec<_> = opts.runner.trim().split_terminator(' ').collect(); + args.push(bin_path.as_str()); + args.append(&mut run_args); + + // run the command + let status = Command::new(args.first().expect("No first argument")) + .args(args.iter().skip(1)) + .status() + .expect("failed to run the command"); + + if !status.success() { + anyhow::bail!("Failed to run `{}`", args.join(" ")); + } + Ok(()) +} diff --git a/user/apps/test_tokio/src/main.rs b/user/apps/test_tokio/src/main.rs index 91b06613..cded478d 100644 --- a/user/apps/test_tokio/src/main.rs +++ b/user/apps/test_tokio/src/main.rs @@ -1,5 +1,3 @@ -use tokio::signal; - async fn say_world() { println!("world"); } diff --git a/user/dadk/config/test_ebpf_0_1_0.dadk b/user/dadk/config/test_ebpf_0_1_0.dadk new file mode 100644 index 00000000..250952d3 --- /dev/null +++ b/user/dadk/config/test_ebpf_0_1_0.dadk @@ -0,0 +1,23 @@ +{ + "name": "test_ebpf", + "version": "0.1.0", + "description": "to test eBPF", + "task_type": { + "BuildFromSource": { + "Local": { + "path": "apps/test_ebpf" + } + } + }, + "depends": [], + "build": { + "build_command": "make install" + }, + "install": { + "in_dragonos_path": "/" + }, + "clean": { + "clean_command": "make clean" + }, + "target_arch": ["x86_64"] +}