diff --git a/.gitignore b/.gitignore index f0351b8a74d0..f3efefe2db77 100644 --- a/.gitignore +++ b/.gitignore @@ -73,6 +73,7 @@ modules.order /vmlinux.symvers /vmlinux.thinlto-index /vmlinux.unstripped +/vmlinux.BTF.inline /vmlinux-gdb.py /vmlinuz /System.map diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 1b2ae71def21..ba01c39d6bb5 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -657,6 +657,22 @@ Kernel parameters See Documentation/admin-guide/bootconfig.rst + bpf.keyring_unsealed= + [BPF] When set to 1, leave the bpf keyring unsealed + after initialization so that userspace can provision + keys. Once the keyring is restricted it becomes active + and can be used for BPF program signature verification. + + Setting this also means that the bpf keyring is the + only non-system keyring a loader may select for the + rest of the boot: caller-supplied user/session + keyrings are refused with -EPERM, whether or not + provisioning actually completed. The system keyrings + stay selectable. Leaving it unset keeps the prior + behaviour, where a caller-supplied keyring is allowed. + + See Documentation/bpf/signing.rst + bttv.card= [HW,V4L] bttv (bt848 + bt878 based grabber cards) bttv.radio= Most important insmod options are available as kernel args too. diff --git a/Documentation/bpf/bpf_design_QA.rst b/Documentation/bpf/bpf_design_QA.rst index eb19c945f4d5..98d70b8a2533 100644 --- a/Documentation/bpf/bpf_design_QA.rst +++ b/Documentation/bpf/bpf_design_QA.rst @@ -221,9 +221,14 @@ newer kernels. BPF programs need to change accordingly when this happens. Q: How much stack space a BPF program uses? ------------------------------------------- -A: Currently all program types are limited to 512 bytes of stack -space, but the verifier computes the actual amount of stack used -and both interpreter and most JITed code consume necessary amount. +A: A program may use up to 2 KiB of stack, combined over its call +chain, when the JIT of the architecture reports support for large +stacks (currently x86-64 and arm64); a single function may use all of +it, and every frame of a program running on a private stack gets the +whole amount. Elsewhere, and for programs verified for the interpreter, +the limit is 512 bytes; the interpreter itself never runs a frame +larger than that. The verifier computes the actual amount of stack +used and both interpreter and most JITed code consume necessary amount. Q: Can BPF be offloaded to HW? ------------------------------ diff --git a/Documentation/bpf/bpf_devel_QA.rst b/Documentation/bpf/bpf_devel_QA.rst index edf8107a7beb..4729efd657c3 100644 --- a/Documentation/bpf/bpf_devel_QA.rst +++ b/Documentation/bpf/bpf_devel_QA.rst @@ -374,9 +374,10 @@ developers in order to get the feature implemented in a timely manner. Please refer to the git log (``arch/*/net/``) to locate the necessary people for helping out. -Also always make sure to add BPF test cases (e.g. test_bpf.c and -test_verifier.c) for new instructions, so that they can receive -broad test coverage and help run-time testing the various BPF JITs. +Also always make sure to add BPF test cases (e.g. the jit_insns test +in test_progs and test_verifier.c) for new instructions, so that they +can receive broad test coverage and help run-time testing the various +BPF JITs. In case of new BPF instructions, once the changes have been accepted into the Linux kernel, please implement support into LLVM's BPF back @@ -514,10 +515,11 @@ A: If you run a kernel ``xyz``, then always run the BPF kernel selftests from that kernel ``xyz`` as well. Do not expect that the BPF selftest from the latest mainline tree will pass all the time. -In particular, test_bpf.c and test_verifier.c have a large number of -test cases and are constantly updated with new BPF test sequences, or -existing ones are adapted to verifier changes e.g. due to verifier -becoming smarter and being able to better track certain things. +In particular, test_verifier.c and the instruction tests in test_progs +have a large number of test cases and are constantly updated with new +BPF test sequences, or existing ones are adapted to verifier changes +e.g. due to verifier becoming smarter and being able to better track +certain things. LLVM ==== diff --git a/Documentation/bpf/btf.rst b/Documentation/bpf/btf.rst index 3f05f17990ad..29de1222c3e7 100644 --- a/Documentation/bpf/btf.rst +++ b/Documentation/bpf/btf.rst @@ -88,6 +88,9 @@ sequentially and type id is assigned to each recognized type starting from id #define BTF_KIND_DECL_TAG 17 /* Decl Tag */ #define BTF_KIND_TYPE_TAG 18 /* Type Tag */ #define BTF_KIND_ENUM64 19 /* Enumeration up to 64-bit values */ + #define BTF_KIND_LOC_PARAM 20 /* Location description (register, const etc) */ + #define BTF_KIND_LOC_PROTO 21 /* Set of location parameters for site */ + #define BTF_KIND_LOCSEC 22 /* Section with site descriptions */ Note that the type section encodes debug info, not just pure types. ``BTF_KIND_FUNC`` is not a type, and it represents a defined subprogram. @@ -104,11 +107,13 @@ Each type contains the following common data:: * decl_tag and type_tag */ __u32 info; - /* "size" is used by INT, ENUM, STRUCT, UNION and ENUM64. + /* "size" is used by INT, ENUM, STRUCT, UNION, ENUM64 and + * LOC_PARAM. * "size" tells the size of the type it is describing. * * "type" is used by PTR, TYPEDEF, VOLATILE, CONST, RESTRICT, - * FUNC, FUNC_PROTO, DECL_TAG and TYPE_TAG. + * FUNC, FUNC_PROTO, DECL_TAG and TYPE_TAG. It is unused by + * LOC_PROTO and LOCSEC. * "type" is a type_id referring to another type. */ union { @@ -416,7 +421,7 @@ If the function has variable arguments, the last parameter is encoded with * ``info.vlen``: 0 * ``type``: the type of the variable -``btf_type`` is followed by a single ``struct btf_variable`` with the +``btf_type`` is followed by a single ``struct btf_var`` with the following data:: struct btf_var { @@ -563,6 +568,80 @@ The ``btf_enum64`` encoding: If the original enum value is signed and the size is less than 8, that value will be sign extended into 8 bytes. +2.2.20 BTF_KIND_LOC_PARAM +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``struct btf_type`` encoding requirement: + * ``name_off``: 0 + * ``info.kind_flag``: 0 + * ``info.kind``: BTF_KIND_LOC_PARAM + * ``info.vlen``: number of 32-bit location value words + * ``size``: size in bytes of the represented parameter: 1 through 16 + +``btf_type`` is followed by a ``struct btf_loc_param`` and ``info.vlen`` +number of 32-bit value words.:: + + struct btf_loc_param { + __u32 flags; + __u32 values[]; + }; + +The ``flags`` field describes how to interpret ``values``: + + * ``BTF_LOC_PARAM_CONST`` describes a constant; the value is stored in + low-word, high-word order when it requires 64 bits. + * ``BTF_LOC_PARAM_ADDR | BTF_LOC_PARAM_CONST`` describes an address offset + relative to the runtime base address of the kernel or module image. + * ``BTF_LOC_PARAM_REG`` with one word describes a register number; with two + words it describes a multi-register parameter. + * ``BTF_LOC_PARAM_REG | BTF_LOC_PARAM_OFFSET`` describes an address held in + a register plus an offset. Adding ``BTF_LOC_PARAM_DEREF`` dereferences + that address. ``BTF_LOC_PARAM_REG | BTF_LOC_PARAM_DEREF`` with one word + dereferences the value held in the register. + * ``BTF_LOC_PARAM_SIGNED`` makes a constant or offset signed. A constant's + signed width is ``size``. For a register-relative offset, its signed + width is the number of offset value words times 32 bits. + +2.2.21 BTF_KIND_LOC_PROTO +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``struct btf_type`` encoding requirement: + * ``name_off``: 0 + * ``info.kind_flag``: 0 + * ``info.kind``: BTF_KIND_LOC_PROTO + * ``info.vlen``: number of function parameter locations + * ``type``: 0 + +``btf_type`` is followed by ``info.vlen`` number of ``__u32`` BTF type IDs. +Each entry corresponds to a function parameter at an inline site. An entry is +either 0, meaning that no location information is available, or the type ID +of a ``BTF_KIND_LOC_PARAM``. + +2.2.22 BTF_KIND_LOCSEC +~~~~~~~~~~~~~~~~~~~~~~ + +``struct btf_type`` encoding requirement: + * ``name_off``: offset to a valid ELF section name + * ``info.kind_flag``: 0 + * ``info.kind``: BTF_KIND_LOCSEC + * ``info.vlen``: number of inline sites in the section + * ``type``: 0 + +``btf_type`` is followed by ``info.vlen`` number of ``struct btf_loc``.:: + + struct btf_loc { + __u32 func; + __u32 loc_proto; + __u32 offset; + }; + +The ``func`` field is the non-zero type ID of the ``BTF_KIND_FUNC`` being +described. ``loc_proto`` is the non-zero type ID of the associated +``BTF_KIND_LOC_PROTO``. ``offset`` is the location address offset relative to +the runtime base address of the ELF section associated with the LOCSEC. +For example, a LOCSEC named ``inline.text`` contains records for ``.text`` +whose offsets are relative to the runtime base of that section. + 2.3 Constant Values ------------------- diff --git a/Documentation/bpf/clang-notes.rst b/Documentation/bpf/clang-notes.rst index 2c872a1ee08e..3ccc7b09d19e 100644 --- a/Documentation/bpf/clang-notes.rst +++ b/Documentation/bpf/clang-notes.rst @@ -23,8 +23,11 @@ For CPU versions prior to 3, Clang v7.0 and later can enable ``BPF_ALU`` support Jump instructions ================= -If ``-O0`` is used, Clang will generate the ``BPF_CALL | BPF_X | BPF_JMP`` (0x8d) -instruction, which is not supported by the Linux kernel verifier. +Clang generates the ``BPF_CALL | BPF_X | BPF_JMP`` (0x8d) instruction for calls +through a function pointer. The Linux kernel verifier accepts it only when it +can prove that the register holds the address of a static BPF function, see +Documentation/bpf/linux-notes.rst. If ``-O0`` is used, Clang will generate this +instruction for helper calls as well, which is not supported. Atomic operations ================= diff --git a/Documentation/bpf/kfuncs.rst b/Documentation/bpf/kfuncs.rst index 6691fe8a32c3..73578a4c3b1f 100644 --- a/Documentation/bpf/kfuncs.rst +++ b/Documentation/bpf/kfuncs.rst @@ -164,19 +164,32 @@ suffix should be used. 2.3.3 __uninit Annotation ------------------------- -This annotation is used to indicate that the argument will be treated as -uninitialized. +Use ``__uninit`` on a pointer parameter for an output buffer whose incoming +contents the kfunc does not read. -An example is given below:: +For generic memory buffers, the kfunc may leave bytes untouched, including +on error returns. The writable range is determined by the pointed-to type +or the associated ``__sz`` or ``__szk`` size argument. - __bpf_kfunc int bpf_dynptr_from_skb(..., struct bpf_dynptr_kern *ptr__uninit) +The annotation does not change the accepted pointer types. A stack-backed +struct passed as a generic memory buffer must still be scalar-only. + +Generic stack buffers may be uninitialized before the call. The call does +not make previously uninitialized bytes readable unless the program is +allowed to read uninitialized stack memory (normally requiring +``CAP_PERFMON``). Other callers must initialize those bytes themselves +before reading them. Stack bounds and variable-offset restrictions still +apply. + +For dynptr parameters, ``__uninit`` indicates that the kfunc constructs a +dynptr in the supplied storage. For example:: + + __bpf_kfunc int bpf_dynptr_from_skb(..., struct bpf_dynptr *ptr__uninit) { ... } -Here, the dynptr will be treated as an uninitialized dynptr. Without this -annotation, the verifier will reject the program if the dynptr passed in is -not initialized. +Without this annotation, a dynptr argument must already be initialized. 2.3.4 __nullable Annotation --------------------------- @@ -315,6 +328,29 @@ However, there is no obligation to prove to the verifier that such a pointer is non-NULL before use, in-line with existing semantics of arena pointers used in a program (or obtained from any other source). +2.3.9 __ctx_out Annotation +-------------------------- + +This annotation is used to indicate that the argument is an output +parameter of the attached hook, passed through from the program's +context. The verifier requires the register to be a trusted read-only +pointer to fixed-size memory, which can only be produced by loading an +argument described by the program's ctx_arg_info from the context. The +program itself cannot write through the pointer; the kfunc may. + +An example is given below:: + + __bpf_kfunc int bpf_inode_init_xattr(struct xattr *xattrs, + int *xattr_count__ctx_out, + ...) + { + ... + } + +In this case, a program attached to the ``inode_init_security`` LSM hook +can pass the hook's own ``xattr_count`` argument through to the kfunc, +which claims xattr slots by writing through it. + .. _BPF_kfunc_nodef: 2.4 Using an existing kernel function @@ -546,7 +582,7 @@ For example, for the following type definition: The verifier would allow a ``struct bpf_cpumask *`` to be passed to a kfunc taking a ``cpumask_t *`` (which is a typedef of ``struct cpumask *``). For -instance, both ``struct cpumask *`` and ``struct bpf_cpmuask *`` can be passed +instance, both ``struct cpumask *`` and ``struct bpf_cpumask *`` can be passed to bpf_cpumask_test_cpu(). In some cases, this type-aliasing behavior is not desired. ``struct @@ -585,6 +621,80 @@ is also covered by this recovery. A kfunc handed an arena pointer may therefore access up to ``GUARD_SZ / 2`` past it without bounds-checking against the arena. Larger accesses must verify the range explicitly. +2.9 kfunc Return Values +----------------------- + +A kfunc may return a scalar, a pointer, or a small struct or union by +value. A scalar or pointer of up to 8 bytes is returned in R0, as usual. + +A struct or union returned by value must be composed only of scalars and arena +pointers, where a scalar is an integer or an enum and an arena pointer is one +carrying the ``btf_type_tag("arena")`` attribute. Those may be nested in +structs and unions and in arrays of any number of dimensions, in any +combination, as long as what the nesting bottoms out in is a scalar or an arena +pointer. Its bytes are handed back to the program as the raw contents of R0 +(and R2), so a member of any other pointer type would be laundered into a +scalar and escape the verifier's pointer provenance and reference tracking. A +struct or union with such a member is therefore rejected at load time, and so +is one with a floating-point member, which the ABI may not return in R0:R2 at +all. + +An arena pointer member is handed back as a scalar too, but nothing is lost by +that. The program must ``cast_kern()`` the value before it can be used, and the +verifier allows that cast on any scalar, so a laundered arena address gives the +program no reach it did not already have. The result is confined to the +program's arena in either case. + +A kfunc may also return a value larger than 8 bytes and up to 16 bytes -- a +struct or union of scalars and arena pointers, or an ``__int128``. Such a +value is returned in the register pair R0:R2, matching the convention LLVM +uses for the BPF target: the first 8 bytes in R0 and the second 8 bytes in R2. +A struct or union of 8 bytes or less is returned in R0 alone. + +:: + + struct bpf_pair { __u64 a, b; }; /* 16 bytes */ + + __bpf_kfunc struct bpf_pair bpf_kfunc_get_pair(void) + { + struct bpf_pair p = { .a = 1, .b = 2 }; + + return p; /* p.a in R0, p.b in R2 */ + } + +Returning a value in the R0:R2 pair requires the JIT to place the second +half of the return value into R2, which not every architecture supports +right now. A kfunc with a return value larger than 8 bytes is therefore +rejected at load time on a JIT that does not advertise this capability (see +``bpf_jit_supports_kfunc_ret_reg_pair()``), and such a program is never run +by the interpreter. A return value larger than 16 bytes is not supported. + +The same R0:R2 convention applies to a BPF subprogram, global or static, that +returns an ``__int128`` or a struct or union larger than 8 bytes. It is only +used when the program is JITed, since the interpreter propagates only R0 out of +a subprogram: without a JIT the return value stays in R0 alone, and a caller +reading R2 is rejected for reading an uninitialized register. A global +subprogram is verified in isolation, so its by-value struct or union return is +restricted to scalars and arena pointers just like a kfunc's; a static +subprogram is verified inline and has no such restriction. The main program is +not covered: its return value is the program's exit code, read out of R0 +alone, so a declared upper half is never looked at. + +A global subprogram must leave a scalar in *every* register of the pair, so +both halves of the returned value have to be assigned. Leaving the upper half +uninitialized is not merely untidy: the compiler is then free to leave R2 +holding whatever it happened to hold, which for a subprogram taking a pointer +argument is typically that pointer. Handing the caller an unknown scalar built +from a pointer is a leak, so the verifier rejects it with:: + + At subprogram exit the register R2 is not a scalar value (...) + +Initialize the whole return value, for example ``struct pair p = {};``, to +avoid this. A static subprogram is exempt from the scalar-only rule: it is +verified inline, so an unassigned R2 is simply passed back to the caller as +uninitialized and only a caller that reads it fails. A stack pointer left in +R2 is still rejected there, just as one in R0 is. + .. _BPF_kfunc_lifecycle_expectations: 3. kfunc lifecycle expectations diff --git a/Documentation/bpf/linux-notes.rst b/Documentation/bpf/linux-notes.rst index 00d2693de025..6c036b54a29f 100644 --- a/Documentation/bpf/linux-notes.rst +++ b/Documentation/bpf/linux-notes.rst @@ -15,10 +15,33 @@ Byte swap instructions Jump instructions ================= -``BPF_CALL | BPF_X | BPF_JMP`` (0x8d), where the helper function -integer would be read from a specified register, is not currently supported -by the verifier. Any programs with this instruction will fail to load -until such support is added. +``BPF_CALL | BPF_X | BPF_JMP`` (0x8d), ``callx dst``, performs an indirect +call of a BPF function whose address is held in the ``dst`` register. The +``src``, ``offset`` and ``imm`` fields are reserved and must be zero. + +The address of a BPF function gets into a register in one of two ways: + +* it is loaded by a 64-bit immediate instruction with ``src`` = + ``BPF_PSEUDO_FUNC``; +* it is read, with a 64-bit load, from a frozen read-only array map, that no + other program uses, that holds its read-only data: tables of functions, + structures of operations, vtables, where pointers to functions may be mixed + with other data. In the map a pointer to a function is the offset in bytes + of its first instruction in the program, and that is how the verifier + recognizes it. It is replaced with the address of the function when + the program is loaded. The program reads it from there, which requires + ``CAP_PERFMON``. + +In both cases only static functions can be referenced. Therefore all functions +that can be called indirectly are known to the verifier before it starts to +analyze the program, and ``callx`` is verified as a direct call of every +function that ``dst`` may point to at that instruction. The same rules apply: +the calls can not be recursive, and the depth of the call chain and its +combined stack size are limited. + +Calling helper or kernel functions through a register, indirect calls of global +functions, and tail calls in functions that are called via ``callx`` are not +supported. ``callx`` requires the BPF JIT. Maps ==== diff --git a/Documentation/bpf/signing.rst b/Documentation/bpf/signing.rst index e73eaaebd8b1..27fdad3df96f 100644 --- a/Documentation/bpf/signing.rst +++ b/Documentation/bpf/signing.rst @@ -254,21 +254,24 @@ returned. Only after the program has fully loaded, at the next hook (``security_bpf_prog()``), does ``BPF_SIG_VERIFIED`` carry its full meaning: validly signed *and* fully verified. -A more realistic admission policy than "is it signed at all": accept programs -signed by a system keyring, accept a user-keyring signature only if the -key/keyring it was verified against is on an explicit allowlist, and emit a -tamper-evident record of every decision so that even denied attempts are -auditable. (Illustrative - error checking elided.) +A more realistic admission policy than "is it signed at all": base trust in +the bpf keyring or an explicit allowlist of a caller-supplied staging key/ +keyring, and emit a record of every decision including denied attempts. +(Illustrative - error checking elided.) .. code-block:: c - /* Serials of user keys/keyrings we additionally trust. */ + /* + * Serials of caller-supplied keyrings we are willing to stage. Empty + * on a system that has committed to the bpf keyring, where the kernel + * refuses them anyway. + */ struct { __uint(type, BPF_MAP_TYPE_HASH); __type(key, __s32); /* keyring_serial */ __type(value, __u8); __uint(max_entries, 64); - } trusted_user_keys SEC(".maps"); + } staging_keys SEC(".maps"); /* Audit stream consumed by a userspace logger. */ struct { @@ -291,11 +294,18 @@ auditable. (Illustrative - error checking elided.) if (kernel) return 0; /* trust in-kernel loads */ - if (verdict != BPF_SIG_VERIFIED) + if (verdict != BPF_SIG_VERIFIED) { ret = -EPERM; /* must be validly signed */ - else if (ktype == BPF_SIG_KEYRING_USER && - !bpf_map_lookup_elem(&trusted_user_keys, &serial)) - ret = -EPERM; /* key/keyring not allowlisted */ + } else switch (ktype) { + case BPF_SIG_KEYRING_BPF: + break; + case BPF_SIG_KEYRING_USER: + if (!bpf_map_lookup_elem(&staging_keys, &serial)) + ret = -EPERM; + break; + default: + ret = -EPERM; /* keyring not in policy */ + } d = bpf_ringbuf_reserve(&audit, sizeof(*d), 0); if (d) { @@ -309,6 +319,11 @@ auditable. (Illustrative - error checking elided.) return ret; } +Such a policy is what makes a caller-supplied keyring usable at all on a system +that does not boot with ``bpf.keyring_unsealed=1``: the allowlist bounds which +staged keys count, and the LSM itself has to protect them from being tampered +with. + Observing a verified load: ``security_bpf_prog()`` -------------------------------------------------- @@ -381,8 +396,9 @@ that verdict covered all of its exclusive maps, rejecting any that did not - so a deny-by-default admission policy needs no second enforcement point. Use ``security_bpf_prog()`` to record or finally gate the verified programs once they carry an id. The ``verdict``, ``keyring_type`` and ``keyring_serial`` fields -let a policy distinguish, for example, "verified and signed by a builtin key" -from "verified by a user key". A policy LSM such as IPE could consume the same +let a policy distinguish "verified against the operator's bpf keyring" from +"verified against a keyring the loader supplied itself", which is the +distinction that matters most. A policy LSM such as IPE could consume the same hooks to enforce system policy without writing any BPF, though none implements this today. @@ -390,33 +406,158 @@ Keyrings ======== ``keyring_id`` selects the trusted keyring the PKCS#7 signature is verified -against. The well-known ids ``0`` (builtin), ``VERIFY_USE_SECONDARY_KEYRING`` -and ``VERIFY_USE_PLATFORM_KEYRING`` select the corresponding system keyrings; -any other value is treated as the serial of a user/session key or keyring. -The keyring is looked up first, before the signature bytes are examined, so a -signature naming a non-existent keyring is rejected up front, and a failed -verification aborts the load - so a program that loads successfully with a -signature always has consistent keyring fields recorded. +against. Four values are well-known; anything else is resolved as a caller- +supplied key or keyring, named either by its serial or by one of the other +``KEY_SPEC_*`` ids: + +.. list-table:: + :header-rows: 1 + + * - ``keyring_id`` + - Keyring + * - ``0`` + - builtin trusted keyring + * - ``VERIFY_USE_SECONDARY_KEYRING`` (``1``) + - secondary trusted keyring + * - ``VERIFY_USE_PLATFORM_KEYRING`` (``2``) + - platform keyring + * - ``KEY_SPEC_BPF_KEYRING`` (``-9``) + - the bpf keyring + * - anything else + - a caller-supplied user/session key or keyring + +The keyring is resolved first, before the signature bytes are examined, so a +signature naming a keyring that cannot be used is rejected up front, and a +failed verification aborts the load - a program that loads successfully with +a signature therefore always has consistent keyring fields recorded. + +The bpf keyring +--------------- + +A system keyring needs a kernel rebuild or a vouched-for enrollment to rotate a +key, and grants BPF-signing trust to keys trusted for everything else in the +kernel too. A caller-supplied keyring, at the other extreme, is filled by the +very process that loads the program and so carries no trust of its own. + +The bpf keyring fills that gap and is the trust anchor which a signed BPF +deployment should be built on top of: a keyring named ``.bpf``, selected with +``KEY_SPEC_BPF_KEYRING``, that an operator provisions at boot with a key +scoped to BPF program loading and nothing else in the kernel's trust hierarchy. +It is owned by the operator rather than by the loader, and rotatable across a +reboot without touching the kernel image. It is modelled after the dm-verity +keyring (see ``dm_verity.keyring_unsealed=``) and provisioned the same way: an +initrd runs the ``keyctl`` steps below before handing off to the rootfs. + +Provisioning +~~~~~~~~~~~~ + +The keyring is created during ``late_initcall`` and is **sealed empty** by +default: it carries a reject-all restriction, so no key can ever be added and +``KEY_SPEC_BPF_KEYRING`` fails with ``-ENOKEY`` for the whole boot. + +``bpf.keyring_unsealed=1`` leaves it unrestricted at init so the initrd can +provision it. The keyring is not linked into any process keyring, but the +``KEY_SPEC_BPF_KEYRING`` special key id addresses it directly, so its serial +does not have to be looked up first. Steps would be as follows:: + + keyctl padd asymmetric "" -9 < signing_key.der + keyctl restrict_keyring -9 + +Both steps are required: the keyring is consulted only once it is **non-empty +and restricted**. An unrestricted keyring is ignored even when it holds keys, +so a half-provisioned keyring is inert rather than a weaker trust anchor, and a +load naming it fails with ``-ENOKEY`` and a verifier log. Restricting cannot +be undone. + +More than one key is enrolled by repeating the ``keyctl padd`` step; the +restriction is applied once, after the last of them:: + + for key in /etc/bpf/keys/*.der; do + keyctl padd asymmetric "" -9 < $key + done + + keyctl restrict_keyring -9 + keyctl show -9 + +The restriction bounds what can be added, never what can be taken away. A key +that is already enrolled can still be unlinked, and the keyring cleared or +revoked, by anything running as root. That does not weaken the anchor, since +a keyring left empty is no longer consulted and a load naming it fails with +``-ENOKEY``, but it does take signed loading out until the next boot. Dropping +the user permissions the keyring no longer needs would close that; as a third +step in the initrd:: + + keyctl setperm -9 0x08030000 + +What remains is ``KEY_POS_SEARCH`` for the in-kernel search during verification, +plus ``KEY_USR_VIEW`` and ``KEY_USR_READ`` so the keyring stays visible in +``/proc/keys``. ``KEY_SPEC_BPF_KEYRING`` names the keyring but conveys no rights +of its own - the keyring is not possessed by anyone, so the ``KEY_POS_*`` bits +are only ever reached by the in-kernel search, and userspace is left with what +the ``KEY_USR_*`` bits grant. Once they are dropped, addressing it by the serial +``/proc/keys`` reports gets no further than the special id does. + +Provisioning has to complete before control passes to the rootfs. The keyring +takes any number of keys for as long as it is unrestricted, so it is the +restriction that bounds the enrolled set, not the first enrollment. Before +the initrd hands off control, it must therefore restrict the keyring after +the last enrollment. + +Enforcement +~~~~~~~~~~~ + +``bpf.keyring_unsealed=1`` states that the bpf keyring is *the* trust anchor for +this boot, so it does more than unseal. From the first program load onwards a +caller-supplied user/session keyring is refused with ``-EPERM`` and a verifier +log message, whether or not provisioning ever completed. The system keyrings +stay selectable. + +Enforcement is readable at ``/sys/module/bpf/parameters/keyring_unsealed``, and +read-only there: the flag is ``__ro_after_init`` behind a 0444 parameter. It is +also derived from the boot flag rather than from the keyring's runtime state, +so there is no window early in boot during which a caller-supplied keyring is +still accepted. + +Caller-supplied keyrings are for staging +---------------------------------------- + +A ``keyring_id`` naming a user or session key or keyring is a *staging* +mechanism, not a trust anchor: it is filled by the same userspace that loads the +program, so verifying against it establishes only that the loader signed what it +loaded. Its purpose is to let software installed onto a running system - whose +signing key is not enrolled anywhere yet - run signed until that key reaches the +bpf keyring on the next boot. + +A system that has committed to the bpf keyring refuses this path outright (see +`Enforcement`_). A system that has not can still allow it, but a policy must +never treat ``BPF_SIG_KEYRING_USER`` as equivalent to the bpf or system +keyrings; it should allowlist the specific serials it is willing to stage and +pair that with a BPF LSM policy protecting those keys from tampering, as in +`Enforcement via LSMs`_. + +Recorded fields +--------------- Two fields are recorded in ``prog->aux->sig`` for an LSM to inspect: ``keyring_type`` (``enum bpf_sig_keyring``) Classified purely from ``keyring_id`` whenever the program is signed: ``BPF_SIG_KEYRING_BUILTIN``, ``_SECONDARY``, ``_PLATFORM`` for the system - keyrings, or ``_USER`` for a user/session keyring. It is - ``BPF_SIG_KEYRING_NONE`` for an unsigned program. + keyrings, ``_BPF`` for the bpf keyring, or ``_USER`` for a caller-supplied + user/session keyring. It is ``BPF_SIG_KEYRING_NONE`` for an unsigned + program. ``keyring_serial`` (``s32``) Set **only** on a successful verification, to the serial of the - **user/session key or keyring** that ``keyring_id`` resolved to - the + **caller-supplied key or keyring** that ``keyring_id`` resolved to - the object the signature was verified against, not the individual asymmetric key inside it that matched the signer. Passing ``KEY_SPEC_SESSION_KEYRING``, for example, records the session keyring's - serial. The system keyrings are trusted as a whole and expose no serial - here, so the serial is ``0`` for builtin, secondary and platform - signatures, and ``0`` for unsigned programs. In other words, a non-zero - ``keyring_serial`` is exactly "verified against the user key/keyring with - this serial". + serial. The system keyrings and the bpf keyring are trusted as a whole and + expose no serial here, so the serial is ``0`` for them, and ``0`` for + unsigned programs. A non-zero ``keyring_serial`` is therefore exactly + "verified against the caller-supplied key/keyring with this serial", which + is exactly the case a policy has to scrutinise. .. list-table:: :header-rows: 1 @@ -436,16 +577,51 @@ Two fields are recorded in ``prog->aux->sig`` for an LSM to inspect: * - ``VERIFY_USE_PLATFORM_KEYRING`` - ``BPF_SIG_KEYRING_PLATFORM`` - ``0`` - * - other (a user/session key serial) + * - ``KEY_SPEC_BPF_KEYRING`` + - ``BPF_SIG_KEYRING_BPF`` + - ``0`` + * - other (a caller-supplied key serial) - ``BPF_SIG_KEYRING_USER`` - serial of the resolved key/keyring -Producing a signed object -========================== +Producing and loading a signed object +===================================== + +Generating a signing key +------------------------ + +Signing is algorithm agnostic: the algorithm comes from the X.509 certificate +and the PKCS#7 ``SignerInfo``. Anything the X.509 and PKCS#7 parsers understand +works with no BPF-side change. Both examples below read the certificate request +config from ``x509.genkey``, for which ``certs/x509.genkey`` serves as a +template (see Documentation/admin-guide/module-signing.rst). RSA:: + + openssl req -new -nodes -utf8 -sha256 -days 36500 -batch -x509 \ + -config x509.genkey -outform PEM \ + -out signing_key.pem -keyout signing_key.pem + openssl x509 -in signing_key.pem -outform der -out signing_key.der + +ML-DSA-87 (FIPS-204), which needs openssl 3.5 or later, and both +``CONFIG_CRYPTO_MLDSA`` and ``CONFIG_CRYPTO_SHA512`` in the kernel - the latter +for the signedAttrs digest described below, which ``CONFIG_CRYPTO_MLDSA`` does +not select. Note the absence of a digest option: ML-DSA hashes the message +itself, so openssl ignores an explicit digest for it:: + + openssl req -new -nodes -utf8 -days 36500 -batch -x509 \ + -newkey ML-DSA-87 -config x509.genkey -outform PEM \ + -out signing_key.pem -keyout signing_key.pem + openssl x509 -in signing_key.pem -outform der -out signing_key.der + +``bpftool`` handles the following internally: openssl before 4.0 cannot combine +ML-DSA with ``CMS_NOATTR``, so it falls back to signedAttrs, where only SHA-512 +is permitted. This mirrors what module signing does as well. + +Signing +------- ``bpftool`` generates and signs a light skeleton in one step:: - bpftool gen skeleton -L -S -k -i \ + bpftool gen skeleton -L -S -k signing_key.pem -i signing_key.der \ obj.bpf.o > obj.lskel.h ``-L`` selects the light-skeleton (``gen_loader``) backend and ``-S`` enables @@ -454,12 +630,36 @@ signing; ``-k`` and ``-i`` supply the signing key and its X.509 certificate. reconstructs - and also computes ``excl_prog_hash`` as the digest of the loader instructions so the metadata map can be bound to the loader. The signature and hash are embedded in the generated header; the certificate is used only for -signing and is not included. Loading the skeleton performs the -create/populate/freeze/load sequence described above. +signing and is not included. -At runtime the trusted public key must be present in the chosen keyring (for -example added to the session keyring, or built into the kernel's builtin trusted -keyring) for verification to succeed. +Loading +------- + +The generated skeleton exposes ``keyring_id``, which selects the keyring the +kernel verifies against. Set it between open and load; loading then performs +the create/populate/freeze/load sequence described above:: + + struct obj *skel = obj__open(); + + skel->keyring_id = KEY_SPEC_BPF_KEYRING; + err = obj__load(skel); + +For the staging case the same object is loaded against a keyring the caller +populated itself, which only works on a system that has not set +``bpf.keyring_unsealed=1``:: + + /* + * Staging only: this keyring is under the loader's own control and + * carries no trust of its own. See "Caller-supplied keyrings are for + * staging". + */ + key_id = add_key("asymmetric", "", der, der_sz, KEY_SPEC_SESSION_KEYRING); + skel->keyring_id = KEY_SPEC_SESSION_KEYRING; + err = obj__load(skel); + +Either way the trusted public key must already be in the chosen keyring for +verification to succeed. For the bpf keyring that enrollment happens once at +boot, see `Provisioning`_. UAPI reference ============== @@ -487,6 +687,13 @@ UAPI reference The map content is not hashed separately at all - it is covered, as bytes, by the program signature. +Kernel command line: + +``bpf.keyring_unsealed=`` + Set to ``1`` to leave the bpf keyring unsealed for provisioning, and to make + it the only non-system keyring a loader may select for the rest of the boot + (see `The bpf keyring`_). + Notes and limitations ====================== @@ -495,3 +702,5 @@ Notes and limitations exceed it. - The metadata container is a single-element array map, accessed through ``map_direct_value_addr``. +- The bpf keyring needs ``CONFIG_KEYS``; without it there is no bpf keyring + and ``KEY_SPEC_BPF_KEYRING`` never resolves. diff --git a/Documentation/networking/filter.rst b/Documentation/networking/filter.rst index 8eb9a5d40f31..7f95c3199748 100644 --- a/Documentation/networking/filter.rst +++ b/Documentation/networking/filter.rst @@ -183,12 +183,9 @@ access to BPF code as well. BPF engine and instruction set ------------------------------ -Under tools/bpf/ there's a small helper tool called bpf_asm which can -be used to write low-level filters for example scenarios mentioned in the -previous section. Asm-like syntax mentioned here has been implemented in -bpf_asm and will be used for further explanations (instead of dealing with -less readable opcodes directly, principles are the same). The syntax is -closely modelled after Steven McCanne's and Van Jacobson's BPF paper. +The examples below use an assembly notation instead of less readable +opcodes. The notation is closely modelled after Steven McCanne's and +Van Jacobson's BPF paper. The BPF architecture consists of the following basic elements: @@ -201,8 +198,8 @@ The BPF architecture consists of the following basic elements: store", addressable from 0 to 15 ======= ==================================================== -A program, that is translated by bpf_asm into "opcodes" is an array that -consists of the following elements (as already mentioned):: +A BPF program is an array of instructions with the following elements +(as already mentioned):: op:16, jt:8, jf:8, k:32 @@ -213,9 +210,8 @@ contains a miscellaneous argument that can be interpreted in different ways depending on the given instruction in op. The instruction set consists of load, store, branch, alu, miscellaneous -and return instructions that are also represented in bpf_asm syntax. This -table lists all bpf_asm instructions available resp. what their underlying -opcodes as defined in linux/filter.h stand for: +and return instructions. This table lists the assembly notation and what +the corresponding opcodes defined in linux/filter.h do: =========== =================== ===================== Instruction Addressing mode Description @@ -374,141 +370,14 @@ Examples for low-level BPF extension: ret #-1 drop: ret #0 -The above example code can be placed into a file (here called "foo"), and -then be passed to the bpf_asm tool for generating opcodes, output that xt_bpf -and cls_bpf understands and can directly be loaded with. Example with above -ARP code:: +The ARP filter shown above corresponds to the following sock_filter +entries:: - $ ./bpf_asm foo - 4,40 0 0 12,21 0 1 2054,6 0 0 4294967295,6 0 0 0, - -In copy and paste C-like output:: - - $ ./bpf_asm -c foo { 0x28, 0, 0, 0x0000000c }, { 0x15, 0, 1, 0x00000806 }, { 0x06, 0, 0, 0xffffffff }, { 0x06, 0, 0, 0000000000 }, -In particular, as usage with xt_bpf or cls_bpf can result in more complex BPF -filters that might not be obvious at first, it's good to test filters before -attaching to a live system. For that purpose, there's a small tool called -bpf_dbg under tools/bpf/ in the kernel source directory. This debugger allows -for testing BPF filters against given pcap files, single stepping through the -BPF code on the pcap's packets and to do BPF machine register dumps. - -Starting bpf_dbg is trivial and just requires issuing:: - - # ./bpf_dbg - -In case input and output do not equal stdin/stdout, bpf_dbg takes an -alternative stdin source as a first argument, and an alternative stdout -sink as a second one, e.g. `./bpf_dbg test_in.txt test_out.txt`. - -Other than that, a particular libreadline configuration can be set via -file "~/.bpf_dbg_init" and the command history is stored in the file -"~/.bpf_dbg_history". - -Interaction in bpf_dbg happens through a shell that also has auto-completion -support (follow-up example commands starting with '>' denote bpf_dbg shell). -The usual workflow would be to ... - -* load bpf 6,40 0 0 12,21 0 3 2048,48 0 0 23,21 0 1 1,6 0 0 65535,6 0 0 0 - Loads a BPF filter from standard output of bpf_asm, or transformed via - e.g. ``tcpdump -iem1 -ddd port 22 | tr '\n' ','``. Note that for JIT - debugging (next section), this command creates a temporary socket and - loads the BPF code into the kernel. Thus, this will also be useful for - JIT developers. - -* load pcap foo.pcap - - Loads standard tcpdump pcap file. - -* run [] - -bpf passes:1 fails:9 - Runs through all packets from a pcap to account how many passes and fails - the filter will generate. A limit of packets to traverse can be given. - -* disassemble:: - - l0: ldh [12] - l1: jeq #0x800, l2, l5 - l2: ldb [23] - l3: jeq #0x1, l4, l5 - l4: ret #0xffff - l5: ret #0 - - Prints out BPF code disassembly. - -* dump:: - - /* { op, jt, jf, k }, */ - { 0x28, 0, 0, 0x0000000c }, - { 0x15, 0, 3, 0x00000800 }, - { 0x30, 0, 0, 0x00000017 }, - { 0x15, 0, 1, 0x00000001 }, - { 0x06, 0, 0, 0x0000ffff }, - { 0x06, 0, 0, 0000000000 }, - - Prints out C-style BPF code dump. - -* breakpoint 0:: - - breakpoint at: l0: ldh [12] - -* breakpoint 1:: - - breakpoint at: l1: jeq #0x800, l2, l5 - - ... - - Sets breakpoints at particular BPF instructions. Issuing a `run` command - will walk through the pcap file continuing from the current packet and - break when a breakpoint is being hit (another `run` will continue from - the currently active breakpoint executing next instructions): - - * run:: - - -- register dump -- - pc: [0] <-- program counter - code: [40] jt[0] jf[0] k[12] <-- plain BPF code of current instruction - curr: l0: ldh [12] <-- disassembly of current instruction - A: [00000000][0] <-- content of A (hex, decimal) - X: [00000000][0] <-- content of X (hex, decimal) - M[0,15]: [00000000][0] <-- folded content of M (hex, decimal) - -- packet dump -- <-- Current packet from pcap (hex) - len: 42 - 0: 00 19 cb 55 55 a4 00 14 a4 43 78 69 08 06 00 01 - 16: 08 00 06 04 00 01 00 14 a4 43 78 69 0a 3b 01 26 - 32: 00 00 00 00 00 00 0a 3b 01 01 - (breakpoint) - > - - * breakpoint:: - - breakpoints: 0 1 - - Prints currently set breakpoints. - -* step [-, +] - - Performs single stepping through the BPF program from the current pc - offset. Thus, on each step invocation, above register dump is issued. - This can go forwards and backwards in time, a plain `step` will break - on the next BPF instruction, thus +1. (No `run` needs to be issued here.) - -* select - - Selects a given packet from the pcap file to continue from. Thus, on - the next `run` or `step`, the BPF program is being evaluated against - the user pre-selected packet. Numbering starts just as in Wireshark - with index 1. - -* quit - - Exits bpf_dbg. - JIT compiler ------------ @@ -540,83 +409,6 @@ setting bpf_jit_enable to 2, since dumping the final JIT image into the kernel l is discouraged and introspection through bpftool (under tools/bpf/bpftool/) is the generally recommended approach instead. -In the kernel source tree under tools/bpf/, there's bpf_jit_disasm for -generating disassembly out of the kernel log's hexdump:: - - # ./bpf_jit_disasm - 70 bytes emitted from JIT compiler (pass:3, flen:6) - ffffffffa0069c8f + : - 0: push %rbp - 1: mov %rsp,%rbp - 4: sub $0x60,%rsp - 8: mov %rbx,-0x8(%rbp) - c: mov 0x68(%rdi),%r9d - 10: sub 0x6c(%rdi),%r9d - 14: mov 0xd8(%rdi),%r8 - 1b: mov $0xc,%esi - 20: callq 0xffffffffe0ff9442 - 25: cmp $0x800,%eax - 2a: jne 0x0000000000000042 - 2c: mov $0x17,%esi - 31: callq 0xffffffffe0ff945e - 36: cmp $0x1,%eax - 39: jne 0x0000000000000042 - 3b: mov $0xffff,%eax - 40: jmp 0x0000000000000044 - 42: xor %eax,%eax - 44: leaveq - 45: retq - - Issuing option `-o` will "annotate" opcodes to resulting assembler - instructions, which can be very useful for JIT developers: - - # ./bpf_jit_disasm -o - 70 bytes emitted from JIT compiler (pass:3, flen:6) - ffffffffa0069c8f + : - 0: push %rbp - 55 - 1: mov %rsp,%rbp - 48 89 e5 - 4: sub $0x60,%rsp - 48 83 ec 60 - 8: mov %rbx,-0x8(%rbp) - 48 89 5d f8 - c: mov 0x68(%rdi),%r9d - 44 8b 4f 68 - 10: sub 0x6c(%rdi),%r9d - 44 2b 4f 6c - 14: mov 0xd8(%rdi),%r8 - 4c 8b 87 d8 00 00 00 - 1b: mov $0xc,%esi - be 0c 00 00 00 - 20: callq 0xffffffffe0ff9442 - e8 1d 94 ff e0 - 25: cmp $0x800,%eax - 3d 00 08 00 00 - 2a: jne 0x0000000000000042 - 75 16 - 2c: mov $0x17,%esi - be 17 00 00 00 - 31: callq 0xffffffffe0ff945e - e8 28 94 ff e0 - 36: cmp $0x1,%eax - 83 f8 01 - 39: jne 0x0000000000000042 - 75 07 - 3b: mov $0xffff,%eax - b8 ff ff 00 00 - 40: jmp 0x0000000000000044 - eb 02 - 42: xor %eax,%eax - 31 c0 - 44: leaveq - c9 - 45: retq - c3 - -For BPF JIT developers, bpf_jit_disasm, bpf_asm and bpf_dbg provides a useful -toolchain for developing and testing the kernel's JIT compiler. - BPF kernel internals -------------------- Internally, for the kernel interpreter, a different instruction set @@ -656,16 +448,11 @@ from eBPF instruction set. Testing ------- -Next to the BPF toolchain, the kernel also ships a test module that contains -various test cases for classic and eBPF that can be executed against -the BPF interpreter and JIT compiler. It can be found in lib/test_bpf.c and -enabled via Kconfig:: - - CONFIG_TEST_BPF=m - -After the module has been built and installed, the test suite can be executed -via insmod or modprobe against 'test_bpf' module. Results of the test cases -including timings in nsec can be found in the kernel log (dmesg). +The BPF selftests under tools/testing/selftests/bpf ship instruction +tests in test_progs, see jit_insns, which runs the eBPF +instructions over a range of operand magnitudes and signs, register +combinations, memory offsets and jump offsets against the BPF interpreter +or the JIT compiler, depending on the net.core.bpf_jit_enable setting. Misc ---- diff --git a/Documentation/networking/net_cachelines/tcp_sock.rst b/Documentation/networking/net_cachelines/tcp_sock.rst index 103328dc409e..93c85b621ec2 100644 --- a/Documentation/networking/net_cachelines/tcp_sock.rst +++ b/Documentation/networking/net_cachelines/tcp_sock.rst @@ -152,6 +152,7 @@ u32 urg_seq unsigned_int keepalive_time unsigned_int keepalive_intvl int linger2 +u32 bpf_tcp_ops_flags read_mostly read_mostly bpf_tcp_ops_hdr_opt_len,bpf_skops_write_hdr_opt(tx);bpf_tcp_ops_parse_hdr,tcp_bpf_rtt,tcp_cleanup_rbuf,tcp_queue_rcv,tcp_ofo_queue,tcp_fastopen_add_skb(rx); u8 bpf_sock_ops_cb_flags u8:1 bpf_chg_cc_inprogress u16 timeout_rehash diff --git a/MAINTAINERS b/MAINTAINERS index 4929b0a956df..8e940a241dcd 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -4895,6 +4895,7 @@ F: arch/powerpc/net/ BPF JIT for RISC-V (32-bit) M: Luke Nelson M: Xi Wang +M: Pu Lehui L: bpf@vger.kernel.org S: Maintained F: arch/riscv/net/ @@ -4902,7 +4903,7 @@ X: arch/riscv/net/bpf_jit_comp64.c BPF JIT for RISC-V (64-bit) M: Björn Töpel -R: Pu Lehui +M: Pu Lehui R: Puranjay Mohan L: bpf@vger.kernel.org S: Maintained @@ -4959,8 +4960,6 @@ BPF [CRYPTO] M: Vadim Fedorenko L: bpf@vger.kernel.org S: Maintained -F: crypto/bpf_crypto_skcipher.c -F: include/linux/bpf_crypto.h F: kernel/bpf/crypto.c BPF [DOCUMENTATION] (Related to Standardization) @@ -5005,7 +5004,6 @@ F: include/uapi/linux/filter.h F: kernel/bpf/ F: kernel/trace/bpf_trace.c F: lib/buildid.c -F: lib/test_bpf.c F: net/bpf/ F: net/core/filter.c F: net/sched/act_bpf.c diff --git a/Makefile b/Makefile index 0319cd716d73..77caa3b0e992 100644 --- a/Makefile +++ b/Makefile @@ -1800,6 +1800,7 @@ endif # CONFIG_MODULES CLEAN_FILES += vmlinux.symvers modules-only.symvers \ modules.builtin modules.builtin.modinfo modules.nsdeps \ modules.builtin.ranges vmlinux.o.map vmlinux.unstripped \ + vmlinux.BTF.inline \ vmlinux.thinlto-index builtin.order \ compile_commands.json rust/test \ rust-project.json .vmlinux.objs .vmlinux.export.c \ diff --git a/arch/arc/Makefile b/arch/arc/Makefile index 868805ffcfea..4ea705a331c7 100644 --- a/arch/arc/Makefile +++ b/arch/arc/Makefile @@ -86,7 +86,14 @@ core-$(CONFIG_ARC_PLAT_TB10X) += arch/arc/plat-tb10x/ core-$(CONFIG_ARC_PLAT_AXS10X) += arch/arc/plat-axs10x/ core-$(CONFIG_ARC_SOC_HSDK) += arch/arc/plat-hsdk/ -libs-y += arch/arc/lib/ $(LIBGCC) +libs-y += arch/arc/lib/ + +ifdef CONFIG_DEBUG_INFO_BTF +export LIBGCC := $(LIBGCC) +libs-y += arch/arc/lib/libgcc.a +else +libs-y += $(LIBGCC) +endif boot := arch/arc/boot diff --git a/arch/arc/include/asm/cmpxchg.h b/arch/arc/include/asm/cmpxchg.h index b30b20933f32..dd5214a097c5 100644 --- a/arch/arc/include/asm/cmpxchg.h +++ b/arch/arc/include/asm/cmpxchg.h @@ -62,6 +62,8 @@ _prev_; \ }) +#define arch_cmpxchg_local arch_cmpxchg_relaxed + #else #define arch_cmpxchg(ptr, old, new) \ @@ -83,6 +85,8 @@ _prev_; \ }) +#define arch_cmpxchg_local arch_cmpxchg + #endif /* diff --git a/arch/arc/lib/Makefile b/arch/arc/lib/Makefile index 30158ae69fd4..2807ee24f27e 100644 --- a/arch/arc/lib/Makefile +++ b/arch/arc/lib/Makefile @@ -13,3 +13,11 @@ lib-$(CONFIG_ISA_ARCV2) +=memcpy-archs-unaligned.o else lib-$(CONFIG_ISA_ARCV2) +=memcpy-archs.o endif + +# libgcc can contain complex floating-point debug types unsupported by BTF. +# Strip only the debug information from a local copy of the selected archive. +always-$(CONFIG_DEBUG_INFO_BTF) += libgcc.a +OBJCOPYFLAGS_libgcc.a := --strip-debug + +$(obj)/libgcc.a: $(LIBGCC) FORCE + $(call if_changed,objcopy) diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index 8108012cd7e4..aaa34c739004 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -22,7 +23,6 @@ #include #include #include -#include #include "bpf_jit.h" @@ -1045,8 +1045,9 @@ static void __maybe_unused build_bhb_mitigation(struct jit_ctx *ctx) } } -static void build_epilogue(struct jit_ctx *ctx, bool was_classic) +static void build_epilogue(struct jit_ctx *ctx, bool was_classic, bool unwind) { + const u8 tmp = bpf2a64[TMP_REG_1]; const u8 r0 = bpf2a64[BPF_REG_0]; const u8 ptr = bpf2a64[TCCNT_PTR]; @@ -1074,6 +1075,10 @@ static void build_epilogue(struct jit_ctx *ctx, bool was_classic) if (IS_ENABLED(CONFIG_ARM64_PTR_AUTH_KERNEL)) emit(A64_AUTIASP, ctx); + /* tmp is the offset of the pad, see emit_unwind_exit() */ + if (unwind) + emit(A64_ADD(1, A64_LR, A64_LR, tmp), ctx); + emit(A64_RET(A64_LR), ctx); } @@ -1222,6 +1227,82 @@ static int add_exception_handler(const struct bpf_insn *insn, return 0; } +/* + * There is no branch for unwind_goto, only an extable entry + * { .insn = ret_addr - 1, .fixup = pad - ret_addr } + * where ret_addr is the return address of the call in front of it. unwind_exit + * adds .fixup to its LR. + * ret_addr - 1, because what follows the call can have an entry of its own. + * No insn starts there, so a fault cannot hit the entry. + * 'bl foo; mov x8, x0' is in front of unwind_goto. The pad doesn't miss the mov: + * r0 is x8 and the epilogue of the callee copies it to x0 and leaves x8 as it is. + */ +static int add_unwind_exentry(struct jit_ctx *ctx, int pad) +{ + int ret_addr = ctx->idx - 1; + struct exception_table_entry *ex; + off_t ins_offset; + + if (!ctx->image) + /* First pass */ + return 0; + + if (!ctx->prog->aux->extable || + WARN_ON_ONCE(ctx->exentry_idx >= ctx->prog->aux->num_exentries)) + return -EINVAL; + + ex = &ctx->prog->aux->extable[ctx->exentry_idx]; + ins_offset = (long)&ctx->ro_image[ret_addr] - 1 - (long)&ex->insn; + if (WARN_ON_ONCE(ins_offset >= 0 || ins_offset < INT_MIN)) + return -ERANGE; + + /* switch ex to rw buffer for writing */ + ex = (void *)ctx->image + ((void *)ex - (void *)ctx->ro_image); + ex->insn = ins_offset; + ex->fixup = (ctx->offset[pad] - ret_addr) * AARCH64_INSN_SIZE; + ex->type = EX_TYPE_BPF; + + ctx->exentry_idx++; + return 0; +} + +static long bpf_unwind_fixup(u64 lr) +{ + const struct exception_table_entry *e; + + e = search_bpf_extables(ptrauth_strip_kernel_insn_pac(lr) - 1); + /* + * The verifier made sure that unwind_goto follows a call in a prog. When + * the kernel called an async callback, it's exit. + */ + return e ? e->fixup : 0; +} + +/* + * unwind_exit: return to the pad. LR in the frame record is signed. Instead of + * signing the address of the pad, let the epilogue authenticate LR as it is and + * add the offset of the pad to it. r0 and r2 are returned. + */ +static void emit_unwind_exit(struct jit_ctx *ctx) +{ + const u8 tmp = bpf2a64[TMP_REG_1]; + const u8 r0 = bpf2a64[BPF_REG_0]; + const u8 r2 = bpf2a64[BPF_REG_2]; + + emit(A64_PUSH(r0, r2, A64_SP), ctx); + emit(A64_LDR64I(A64_R(0), A64_FP, 8), ctx); + emit_call((u64)bpf_unwind_fixup, ctx); + emit(A64_MOV(1, tmp, A64_R(0)), ctx); + emit(A64_POP(r0, r2, A64_SP), ctx); + build_epilogue(ctx, false, true); +} + +static const struct bpf_jit_arg_abi arm64_arg_abi = { + .nr_arg_regs = 8, + .even_reg_align = true, + .even_stack_align = true, +}; + static const u8 stack_arg_reg[] = { A64_R(5), A64_R(6), A64_R(7) }; #define NR_STACK_ARG_REGS ARRAY_SIZE(stack_arg_reg) @@ -1264,19 +1345,20 @@ static void emit_stack_arg_store_imm(s32 imm, s16 bpf_off, const u8 tmp, struct * kern_vm_start. A nullable arg preserves NULL by skipping the add, tested * on the truncated value as arena NULL is offset 0. */ -static int emit_kfunc_arena_args(struct jit_ctx *ctx, const struct bpf_insn *insn) +static int emit_kfunc_arena_args(struct jit_ctx *ctx, const struct btf_func_model *fm) { const u8 arena_vm_base = bpf2a64[ARENA_VM_START]; - const struct btf_func_model *fm; - int i; + int i, slot; - fm = bpf_jit_find_kfunc_model(ctx->prog, insn); - if (!fm) - return -EINVAL; - - for (i = 0; i < min_t(int, fm->nr_args, MAX_BPF_FUNC_REG_ARGS); i++) { - const u8 reg = bpf2a64[BPF_REG_1 + i]; + for (i = 0, slot = 0; i < fm->nr_args; i++) { + u32 arg_regs = (fm->arg_size[i] + 7) / 8; u8 flags = fm->arg_flags[i]; + u8 reg; + + if (slot + arg_regs > MAX_BPF_FUNC_REG_ARGS) + break; + reg = bpf2a64[BPF_REG_1 + slot]; + slot += arg_regs; if (!(flags & BTF_FMODEL_ARENA_ARG)) continue; @@ -1295,6 +1377,52 @@ static int emit_kfunc_arena_args(struct jit_ctx *ctx, const struct bpf_insn *ins return 0; } +static bool a64_arg_on_stack(u8 slot) +{ + return slot >= arm64_arg_abi.nr_arg_regs; +} + +static s32 a64_arg_stack_off(u8 slot) +{ + return (slot - arm64_arg_abi.nr_arg_regs) * sizeof(u64); +} + +/* + * Move the arguments AAPCS64 places somewhere other than the argument slot the + * BPF calling convention gave them. Slot N is X(N) up to the eighth, and the + * outgoing stack argument area from SP beyond it, both for the slot an + * argument comes from and for the one it goes to. + * + * AAPCS64 only ever moves an argument to a higher slot, so no move here ever + * takes BPF_JIT_ARG_TMP: bpf_jit_plan_arg_moves() hands out the scratch only + * for a convention that moves one down, which needs a register to carry the + * value past its own destination. + */ +static void emit_kfunc_arg_moves(struct jit_ctx *ctx, const struct btf_func_model *fm) +{ + struct bpf_jit_arg_move moves[BPF_JIT_MAX_ARG_MOVES]; + const u8 tmp = bpf2a64[TMP_REG_1]; + u32 i, n; + + n = bpf_jit_plan_arg_moves(&arm64_arg_abi, fm, moves); + + for (i = 0; i < n; i++) { + u8 dst = moves[i].dst, src = moves[i].src, reg; + + if (a64_arg_on_stack(src)) { + reg = tmp; + emit(A64_LDR64I(reg, A64_SP, a64_arg_stack_off(src)), ctx); + } else { + reg = src; + } + + if (a64_arg_on_stack(dst)) + emit(A64_STR64I(reg, A64_SP, a64_arg_stack_off(dst)), ctx); + else if (reg != dst) + emit(A64_MOV(1, dst, reg), ctx); + } +} + /* JITs an eBPF instruction. * Returns: * 0 - successfully JITed an 8-byte eBPF instruction. @@ -1713,9 +1841,15 @@ static int build_insn(const struct bpf_verifier_env *env, const struct bpf_insn if (ret < 0) return ret; if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) { - ret = emit_kfunc_arena_args(ctx, insn); + const struct btf_func_model *fm; + + fm = bpf_jit_find_kfunc_model(ctx->prog, insn); + if (!fm) + return -EINVAL; + ret = emit_kfunc_arena_args(ctx, fm); if (ret < 0) return ret; + emit_kfunc_arg_moves(ctx, fm); } emit_call(func_addr, ctx); /* @@ -1727,13 +1861,34 @@ static int build_insn(const struct bpf_verifier_env *env, const struct bpf_insn emit(A64_MOV(1, r0, A64_R(0)), ctx); break; } + /* indirect call of a bpf subprog, dst holds its address */ + case BPF_JMP | BPF_CALL | BPF_X: + /* + * It's the same as a direct call of a subprog that is out of + * range of BL: the subprog starts with BTI JC, the arguments + * are in place, and the registers that hold the tail call + * counter and the private stack are callee saved. + */ + emit(A64_BLR(dst), ctx); + emit(A64_MOV(1, bpf2a64[BPF_REG_0], A64_R(0)), ctx); + break; /* tail call */ case BPF_JMP | BPF_TAIL_CALL: if (emit_bpf_tail_call(ctx)) return -EFAULT; break; + /* unwind_goto. The verifier expanded may_goto. */ + case BPF_JMP | BPF_JCOND: + ret = add_unwind_exentry(ctx, i + 1 + off); + if (ret) + return ret; + break; /* function return */ case BPF_JMP | BPF_EXIT: + if (bpf_is_unwind_exit(insn) && bpf_prog_can_unwind(ctx->prog)) { + emit_unwind_exit(ctx); + break; + } /* Optimization: when last instruction is EXIT, simply fallthrough to epilogue. */ if (i == ctx->prog->len - 1) @@ -2210,8 +2365,10 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_pr if (ctx.offset == NULL) goto out_off; - ctx.user_vm_start = bpf_arena_get_user_vm_start(prog->aux->arena); - ctx.arena_vm_start = bpf_arena_get_kern_vm_start(prog->aux->arena); + if (bpf_prog_uses_arena_insns(prog)) { + ctx.user_vm_start = bpf_arena_get_user_vm_start(prog->aux->arena); + ctx.arena_vm_start = bpf_arena_get_kern_vm_start(prog->aux->arena); + } out_cnt = bpf_out_stack_arg_cnt(env, prog); if (out_cnt) { @@ -2236,7 +2393,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_pr goto out_off; ctx.epilogue_offset = ctx.idx; - build_epilogue(&ctx, was_classic); + build_epilogue(&ctx, was_classic, false); build_plt(&ctx); extable_align = __alignof__(struct exception_table_entry); @@ -2293,7 +2450,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_pr WARN_ON_ONCE(ctx.idx != ctx.epilogue_offset)) goto out_free_hdr; - build_epilogue(&ctx, was_classic); + build_epilogue(&ctx, was_classic, false); build_plt(&ctx); /* Extra pass to validate JITed code. */ @@ -2385,6 +2542,21 @@ bool bpf_jit_supports_kfunc_call(void) return true; } +bool bpf_jit_supports_kfunc_ret_reg_pair(void) +{ + return true; +} + +bool bpf_jit_supports_unwind(void) +{ + return true; +} + +const struct bpf_jit_arg_abi *bpf_jit_arg_abi(void) +{ + return &arm64_arg_abi; +} + bool bpf_jit_supports_stack_args(void) { return true; @@ -2413,6 +2585,16 @@ bool bpf_jit_supports_subprog_tailcalls(void) return true; } +bool bpf_jit_supports_callx(void) +{ + return true; +} + +bool bpf_jit_supports_large_stack(void) +{ + return true; +} + static void invoke_bpf_prog(struct jit_ctx *ctx, struct bpf_tramp_image *im, struct bpf_tramp_node *node, int bargs_off, int retval_off, int run_ctx_off, bool save_ret) @@ -2528,33 +2710,41 @@ struct arg_aux { * arguments to be properly aligned) */ int ostack_for_args; + /* where AAPCS64 puts each argument slot: an argument register below + * the eighth, an on-stack argument slot from it up + */ + u8 pos_of_slot[MAX_BPF_FUNC_ARG_SLOTS]; }; static int calc_arg_aux(const struct btf_func_model *m, struct arg_aux *a) { - int stack_slots, nregs, slots, i; + int slots, i, slot, total; + + total = bpf_jit_place_args(&arm64_arg_abi, m, a->pos_of_slot); + if (total > MAX_BPF_FUNC_ARGS) + return -ENOTSUPP; /* verifier ensures m->nr_args <= MAX_BPF_FUNC_ARGS */ - for (i = 0, nregs = 0; i < m->nr_args; i++) { + for (i = 0, slot = 0; i < m->nr_args; i++) { slots = (m->arg_size[i] + 7) / 8; - if (nregs + slots <= 8) /* passed through register ? */ - nregs += slots; - else + if (a64_arg_on_stack(a->pos_of_slot[slot])) /* passed through register ? */ break; + slot += slots; } a->args_in_regs = i; - a->regs_for_args = nregs; + a->regs_for_args = slot; a->ostack_for_args = 0; a->bstack_for_args = 0; /* the rest arguments are passed through stack */ - for (; i < m->nr_args; i++) { - stack_slots = (m->arg_size[i] + 7) / 8; - a->bstack_for_args += stack_slots * 8; - a->ostack_for_args = a->ostack_for_args + stack_slots * 8; - } + for (; i < m->nr_args; i++) + a->bstack_for_args += ((m->arg_size[i] + 7) / 8) * 8; + + /* the outgoing area reaches the last slot, over any alignment hole */ + if (a->bstack_for_args) + a->ostack_for_args = a64_arg_stack_off(a->pos_of_slot[total - 1]) + 8; return 0; } @@ -2595,7 +2785,7 @@ static void save_args(struct jit_ctx *ctx, int bargs_off, int oargs_off, { u8 tmp = bpf2a64[TMP_REG_1]; u8 base_lo = bpf2a64[TMP_REG_2]; - int i, reg, doff, soff, slots; + int i, reg, slot, soff, slots; /* only the low 32 bits of the base take part in the subtraction */ if (arena_base) @@ -2604,12 +2794,13 @@ static void save_args(struct jit_ctx *ctx, int bargs_off, int oargs_off, /* store arguments to the stack for the bpf program, or restore * arguments from stack for the original function */ - for (i = 0, reg = 0; i < a->args_in_regs; i++) { + for (i = 0, slot = 0; i < a->args_in_regs; i++) { bool arena_arg = arena_base && (m->arg_flags[i] & BTF_FMODEL_ARENA_ARG); bool nullable = m->arg_flags[i] & BTF_FMODEL_NULLABLE_ARG; slots = (m->arg_size[i] + 7) / 8; while (slots-- > 0) { + reg = a->pos_of_slot[slot++]; if (for_call_origin) { emit(A64_LDR64I(reg, A64_SP, bargs_off), ctx); } else if (arena_arg) { @@ -2618,7 +2809,6 @@ static void save_args(struct jit_ctx *ctx, int bargs_off, int oargs_off, } else { emit(A64_STR64I(reg, A64_SP, bargs_off), ctx); } - reg++; bargs_off += 8; } } @@ -2630,9 +2820,11 @@ static void save_args(struct jit_ctx *ctx, int bargs_off, int oargs_off, * (FP/LR) frames, so the arguments start at FP + 32. A struct_ops * callback is called indirectly and only the FP/LR frame is saved, so * they start at FP + 16. + * + * The outgoing area mirrors the incoming one, hole and all; only the + * bpf program takes the arguments packed. */ soff = is_struct_ops ? 16 : 32; - doff = (for_call_origin ? oargs_off : bargs_off); /* save on stack arguments */ for (i = a->args_in_regs; i < m->nr_args; i++) { @@ -2642,7 +2834,9 @@ static void save_args(struct jit_ctx *ctx, int bargs_off, int oargs_off, slots = (m->arg_size[i] + 7) / 8; /* verifier ensures arg_size <= 16, so slots equals 1 or 2 */ while (slots-- > 0) { - emit(A64_LDR64I(tmp, A64_FP, soff), ctx); + int off = a64_arg_stack_off(a->pos_of_slot[slot++]); + + emit(A64_LDR64I(tmp, A64_FP, soff + off), ctx); /* if there is unused space in the last slot, clear * the garbage contained in the space. */ @@ -2657,19 +2851,21 @@ static void save_args(struct jit_ctx *ctx, int bargs_off, int oargs_off, */ if (arena_arg) emit_arena_arg_conv(ctx, tmp, tmp, nullable, base_lo); - emit(A64_STR64I(tmp, A64_SP, doff), ctx); - soff += 8; - doff += 8; + if (for_call_origin) + emit(A64_STR64I(tmp, A64_SP, oargs_off + off), ctx); + else + emit(A64_STR64I(tmp, A64_SP, bargs_off), ctx); + bargs_off += 8; } } } -static void restore_args(struct jit_ctx *ctx, int bargs_off, int nregs) +static void restore_args(struct jit_ctx *ctx, int bargs_off, const struct arg_aux *a) { - int reg; + int slot; - for (reg = 0; reg < nregs; reg++) { - emit(A64_LDR64I(reg, A64_SP, bargs_off), ctx); + for (slot = 0; slot < a->regs_for_args; slot++) { + emit(A64_LDR64I(a->pos_of_slot[slot], A64_SP, bargs_off), ctx); bargs_off += 8; } } @@ -2937,7 +3133,7 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, } if (flags & BPF_TRAMP_F_RESTORE_REGS) - restore_args(ctx, bargs_off, a->regs_for_args); + restore_args(ctx, bargs_off, a); /* restore callee saved register x19 and x20 */ emit(A64_LDR64I(A64_R(19), A64_SP, regs_off), ctx); @@ -3226,21 +3422,22 @@ int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type old_t, */ plt_target = (u64)&dummy_tramp; - if (plt_target) { - /* non-zero plt_target indicates we're patching a bpf prog, - * which is read only. - */ - if (set_memory_rw(PAGE_MASK & ((uintptr_t)&plt->target), 1)) - return -EFAULT; - WRITE_ONCE(plt->target, plt_target); - set_memory_ro(PAGE_MASK & ((uintptr_t)&plt->target), 1); - /* since plt target points to either the new trampoline - * or dummy_tramp, even if another CPU reads the old plt - * target value before fetching the bl instruction to plt, - * it will be brought back by dummy_tramp, so no barrier is - * required here. - */ - } + /* + * non-zero plt_target indicates we're patching a bpf prog, + * which is read only. The prog may share its page in the bpf + * prog pack with other progs, so write the aligned target + * through the text patching fixmap without flipping the + * page permissions to avoid racing with concurrent pokers. + * + * since plt target points to either the new trampoline + * or dummy_tramp, even if another CPU reads the old plt + * target value before fetching the bl instruction to plt, + * it will be brought back by dummy_tramp, so no barrier is + * required here. + */ + if (plt_target && + aarch64_insn_write_literal_u64(&plt->target, plt_target)) + return -EFAULT; /* if the old target and the new target are both long jumps, no * patching is required @@ -3305,6 +3502,11 @@ bool bpf_jit_supports_arena(void) return true; } +bool bpf_jit_supports_arena_scalar(void) +{ + return true; +} + bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena) { if (!in_arena) diff --git a/arch/hexagon/include/asm/cmpxchg.h b/arch/hexagon/include/asm/cmpxchg.h index 9c58fb81f7fd..e6114e713db9 100644 --- a/arch/hexagon/include/asm/cmpxchg.h +++ b/arch/hexagon/include/asm/cmpxchg.h @@ -8,6 +8,8 @@ #ifndef _ASM_CMPXCHG_H #define _ASM_CMPXCHG_H +#include + /* * __arch_xchg - atomically exchange a register and a memory location * @x: value to swap @@ -72,4 +74,8 @@ __arch_xchg(unsigned long x, volatile void *ptr, int size) __oldval; \ }) +#define arch_cmpxchg_local(ptr, o, n) \ + ((__typeof__(*(ptr)))__generic_cmpxchg_local((ptr), \ + (unsigned long)(o), (unsigned long)(n), sizeof(*(ptr)))) + #endif /* _ASM_CMPXCHG_H */ diff --git a/arch/loongarch/net/bpf_jit.c b/arch/loongarch/net/bpf_jit.c index e78eb582c300..9393e4b0db8e 100644 --- a/arch/loongarch/net/bpf_jit.c +++ b/arch/loongarch/net/bpf_jit.c @@ -2245,8 +2245,10 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_pr memset(&ctx, 0, sizeof(ctx)); ctx.prog = prog; - ctx.arena_vm_start = bpf_arena_get_kern_vm_start(prog->aux->arena); - ctx.user_vm_start = bpf_arena_get_user_vm_start(prog->aux->arena); + if (bpf_prog_uses_arena_insns(prog)) { + ctx.arena_vm_start = bpf_arena_get_kern_vm_start(prog->aux->arena); + ctx.user_vm_start = bpf_arena_get_user_vm_start(prog->aux->arena); + } ctx.offset = kvcalloc(prog->len + 1, sizeof(u32), GFP_KERNEL); if (ctx.offset == NULL) diff --git a/arch/m68k/configs/amiga_defconfig b/arch/m68k/configs/amiga_defconfig index fefcaa5ad44c..e2d474e8cb8f 100644 --- a/arch/m68k/configs/amiga_defconfig +++ b/arch/m68k/configs/amiga_defconfig @@ -579,7 +579,6 @@ CONFIG_TEST_IDA=m CONFIG_TEST_BITOPS=m CONFIG_TEST_VMALLOC=m CONFIG_TEST_WORKQUEUE=m -CONFIG_TEST_BPF=m CONFIG_FIND_BIT_BENCHMARK=m CONFIG_REGION_ALLOC_BENCHMARK=m CONFIG_TEST_FIRMWARE=m diff --git a/arch/m68k/configs/apollo_defconfig b/arch/m68k/configs/apollo_defconfig index 84d4a6ed1707..77669e31bb5a 100644 --- a/arch/m68k/configs/apollo_defconfig +++ b/arch/m68k/configs/apollo_defconfig @@ -533,7 +533,6 @@ CONFIG_TEST_IDA=m CONFIG_TEST_BITOPS=m CONFIG_TEST_VMALLOC=m CONFIG_TEST_WORKQUEUE=m -CONFIG_TEST_BPF=m CONFIG_FIND_BIT_BENCHMARK=m CONFIG_REGION_ALLOC_BENCHMARK=m CONFIG_TEST_FIRMWARE=m diff --git a/arch/m68k/configs/atari_defconfig b/arch/m68k/configs/atari_defconfig index 22d8a8581601..43ed1e317ae2 100644 --- a/arch/m68k/configs/atari_defconfig +++ b/arch/m68k/configs/atari_defconfig @@ -558,7 +558,6 @@ CONFIG_TEST_IDA=m CONFIG_TEST_BITOPS=m CONFIG_TEST_VMALLOC=m CONFIG_TEST_WORKQUEUE=m -CONFIG_TEST_BPF=m CONFIG_FIND_BIT_BENCHMARK=m CONFIG_REGION_ALLOC_BENCHMARK=m CONFIG_TEST_FIRMWARE=m diff --git a/arch/m68k/configs/bvme6000_defconfig b/arch/m68k/configs/bvme6000_defconfig index 0f79c6382d35..44c1a2228d5f 100644 --- a/arch/m68k/configs/bvme6000_defconfig +++ b/arch/m68k/configs/bvme6000_defconfig @@ -525,7 +525,6 @@ CONFIG_TEST_IDA=m CONFIG_TEST_BITOPS=m CONFIG_TEST_VMALLOC=m CONFIG_TEST_WORKQUEUE=m -CONFIG_TEST_BPF=m CONFIG_FIND_BIT_BENCHMARK=m CONFIG_REGION_ALLOC_BENCHMARK=m CONFIG_TEST_FIRMWARE=m diff --git a/arch/m68k/configs/hp300_defconfig b/arch/m68k/configs/hp300_defconfig index 2bf412a1713c..7ab25377bfe8 100644 --- a/arch/m68k/configs/hp300_defconfig +++ b/arch/m68k/configs/hp300_defconfig @@ -535,7 +535,6 @@ CONFIG_TEST_IDA=m CONFIG_TEST_BITOPS=m CONFIG_TEST_VMALLOC=m CONFIG_TEST_WORKQUEUE=m -CONFIG_TEST_BPF=m CONFIG_FIND_BIT_BENCHMARK=m CONFIG_REGION_ALLOC_BENCHMARK=m CONFIG_TEST_FIRMWARE=m diff --git a/arch/m68k/configs/mac_defconfig b/arch/m68k/configs/mac_defconfig index 5496f1b60531..13f642884a94 100644 --- a/arch/m68k/configs/mac_defconfig +++ b/arch/m68k/configs/mac_defconfig @@ -554,7 +554,6 @@ CONFIG_TEST_IDA=m CONFIG_TEST_BITOPS=m CONFIG_TEST_VMALLOC=m CONFIG_TEST_WORKQUEUE=m -CONFIG_TEST_BPF=m CONFIG_FIND_BIT_BENCHMARK=m CONFIG_REGION_ALLOC_BENCHMARK=m CONFIG_TEST_FIRMWARE=m diff --git a/arch/m68k/configs/multi_defconfig b/arch/m68k/configs/multi_defconfig index a8f593b4fc6a..1bbd2aa653c0 100644 --- a/arch/m68k/configs/multi_defconfig +++ b/arch/m68k/configs/multi_defconfig @@ -644,7 +644,6 @@ CONFIG_TEST_IDA=m CONFIG_TEST_BITOPS=m CONFIG_TEST_VMALLOC=m CONFIG_TEST_WORKQUEUE=m -CONFIG_TEST_BPF=m CONFIG_FIND_BIT_BENCHMARK=m CONFIG_REGION_ALLOC_BENCHMARK=m CONFIG_TEST_FIRMWARE=m diff --git a/arch/m68k/configs/mvme147_defconfig b/arch/m68k/configs/mvme147_defconfig index bbce43b8f7aa..3bfc36faf8d5 100644 --- a/arch/m68k/configs/mvme147_defconfig +++ b/arch/m68k/configs/mvme147_defconfig @@ -525,7 +525,6 @@ CONFIG_TEST_IDA=m CONFIG_TEST_BITOPS=m CONFIG_TEST_VMALLOC=m CONFIG_TEST_WORKQUEUE=m -CONFIG_TEST_BPF=m CONFIG_FIND_BIT_BENCHMARK=m CONFIG_REGION_ALLOC_BENCHMARK=m CONFIG_TEST_FIRMWARE=m diff --git a/arch/m68k/configs/mvme16x_defconfig b/arch/m68k/configs/mvme16x_defconfig index a63d09dc8dbd..41ceed87c78a 100644 --- a/arch/m68k/configs/mvme16x_defconfig +++ b/arch/m68k/configs/mvme16x_defconfig @@ -526,7 +526,6 @@ CONFIG_TEST_IDA=m CONFIG_TEST_BITOPS=m CONFIG_TEST_VMALLOC=m CONFIG_TEST_WORKQUEUE=m -CONFIG_TEST_BPF=m CONFIG_FIND_BIT_BENCHMARK=m CONFIG_REGION_ALLOC_BENCHMARK=m CONFIG_TEST_FIRMWARE=m diff --git a/arch/m68k/configs/q40_defconfig b/arch/m68k/configs/q40_defconfig index d6b7cbab1700..af4b5c6e4655 100644 --- a/arch/m68k/configs/q40_defconfig +++ b/arch/m68k/configs/q40_defconfig @@ -544,7 +544,6 @@ CONFIG_TEST_IDA=m CONFIG_TEST_BITOPS=m CONFIG_TEST_VMALLOC=m CONFIG_TEST_WORKQUEUE=m -CONFIG_TEST_BPF=m CONFIG_FIND_BIT_BENCHMARK=m CONFIG_REGION_ALLOC_BENCHMARK=m CONFIG_TEST_FIRMWARE=m diff --git a/arch/m68k/configs/sun3_defconfig b/arch/m68k/configs/sun3_defconfig index bee5cdd20637..1140951fefe7 100644 --- a/arch/m68k/configs/sun3_defconfig +++ b/arch/m68k/configs/sun3_defconfig @@ -522,7 +522,6 @@ CONFIG_TEST_IDA=m CONFIG_TEST_BITOPS=m CONFIG_TEST_VMALLOC=m CONFIG_TEST_WORKQUEUE=m -CONFIG_TEST_BPF=m CONFIG_FIND_BIT_BENCHMARK=m CONFIG_REGION_ALLOC_BENCHMARK=m CONFIG_TEST_FIRMWARE=m diff --git a/arch/m68k/configs/sun3x_defconfig b/arch/m68k/configs/sun3x_defconfig index ef2b76a06439..bc9358e71ee0 100644 --- a/arch/m68k/configs/sun3x_defconfig +++ b/arch/m68k/configs/sun3x_defconfig @@ -523,7 +523,6 @@ CONFIG_TEST_IDA=m CONFIG_TEST_BITOPS=m CONFIG_TEST_VMALLOC=m CONFIG_TEST_WORKQUEUE=m -CONFIG_TEST_BPF=m CONFIG_FIND_BIT_BENCHMARK=m CONFIG_REGION_ALLOC_BENCHMARK=m CONFIG_TEST_FIRMWARE=m diff --git a/arch/mips/include/asm/uasm.h b/arch/mips/include/asm/uasm.h index b43bfd445252..adea35df0f8d 100644 --- a/arch/mips/include/asm/uasm.h +++ b/arch/mips/include/asm/uasm.h @@ -85,15 +85,20 @@ Ip_u1u2(_ctc1); Ip_u2u1(_ctcmsa); Ip_u2u1s3(_daddiu); Ip_u3u1u2(_daddu); +Ip_u1u2(_ddiv); +Ip_u3u1u2(_ddiv_r6); Ip_u1u2(_ddivu); Ip_u3u1u2(_ddivu_r6); Ip_u1(_di); Ip_u2u1msbu3(_dins); Ip_u2u1msbu3(_dinsm); Ip_u2u1msbu3(_dinsu); +Ip_u1u2(_div); +Ip_u3u1u2(_div_r6); Ip_u1u2(_divu); Ip_u3u1u2(_divu_r6); Ip_u1u2u3(_dmfc0); +Ip_u3u1u2(_dmod); Ip_u3u1u2(_dmodu); Ip_u1u2u3(_dmtc0); Ip_u1u2(_dmultu); @@ -135,6 +140,7 @@ Ip_u1u2u3(_mfc0); Ip_u1u2u3(_mfhc0); Ip_u1(_mfhi); Ip_u1(_mflo); +Ip_u3u1u2(_mod); Ip_u3u1u2(_modu); Ip_u3u1u2(_movn); Ip_u3u1u2(_movz); @@ -156,6 +162,8 @@ Ip_u2s3u1(_sb); Ip_u2s3u1(_sc); Ip_u2s3u1(_scd); Ip_u2s3u1(_sd); +Ip_u2u1(_seb); +Ip_u2u1(_seh); Ip_u3u1u2(_seleqz); Ip_u3u1u2(_selnez); Ip_u2s3u1(_sh); diff --git a/arch/mips/mm/uasm-mips.c b/arch/mips/mm/uasm-mips.c index e15c6700cd08..2d1668b58ddd 100644 --- a/arch/mips/mm/uasm-mips.c +++ b/arch/mips/mm/uasm-mips.c @@ -75,6 +75,9 @@ static const struct insn insn_table[insn_invalid] = { [insn_ctcmsa] = {M(msa_op, 0, msa_ctc_op, 0, 0, msa_elm_op), RD | RE}, [insn_daddiu] = {M(daddiu_op, 0, 0, 0, 0, 0), RS | RT | SIMM}, [insn_daddu] = {M(spec_op, 0, 0, 0, 0, daddu_op), RS | RT | RD}, + [insn_ddiv] = {M(spec_op, 0, 0, 0, 0, ddiv_op), RS | RT}, + [insn_ddiv_r6] = {M(spec_op, 0, 0, 0, ddiv_ddiv6_op, ddiv_op), + RS | RT | RD}, [insn_ddivu] = {M(spec_op, 0, 0, 0, 0, ddivu_op), RS | RT}, [insn_ddivu_r6] = {M(spec_op, 0, 0, 0, ddivu_ddivu6_op, ddivu_op), RS | RT | RD}, @@ -82,10 +85,15 @@ static const struct insn insn_table[insn_invalid] = { [insn_dins] = {M(spec3_op, 0, 0, 0, 0, dins_op), RS | RT | RD | RE}, [insn_dinsm] = {M(spec3_op, 0, 0, 0, 0, dinsm_op), RS | RT | RD | RE}, [insn_dinsu] = {M(spec3_op, 0, 0, 0, 0, dinsu_op), RS | RT | RD | RE}, + [insn_div] = {M(spec_op, 0, 0, 0, 0, div_op), RS | RT}, + [insn_div_r6] = {M(spec_op, 0, 0, 0, div_div6_op, div_op), + RS | RT | RD}, [insn_divu] = {M(spec_op, 0, 0, 0, 0, divu_op), RS | RT}, [insn_divu_r6] = {M(spec_op, 0, 0, 0, divu_divu6_op, divu_op), RS | RT | RD}, [insn_dmfc0] = {M(cop0_op, dmfc_op, 0, 0, 0, 0), RT | RD | SET}, + [insn_dmod] = {M(spec_op, 0, 0, 0, ddiv_dmod_op, ddiv_op), + RS | RT | RD}, [insn_dmodu] = {M(spec_op, 0, 0, 0, ddivu_dmodu_op, ddivu_op), RS | RT | RD}, [insn_dmtc0] = {M(cop0_op, dmtc_op, 0, 0, 0, 0), RT | RD | SET}, @@ -140,6 +148,8 @@ static const struct insn insn_table[insn_invalid] = { [insn_mfhc0] = {M(cop0_op, mfhc0_op, 0, 0, 0, 0), RT | RD | SET}, [insn_mfhi] = {M(spec_op, 0, 0, 0, 0, mfhi_op), RD}, [insn_mflo] = {M(spec_op, 0, 0, 0, 0, mflo_op), RD}, + [insn_mod] = {M(spec_op, 0, 0, 0, div_mod_op, div_op), + RS | RT | RD}, [insn_modu] = {M(spec_op, 0, 0, 0, divu_modu_op, divu_op), RS | RT | RD}, [insn_movn] = {M(spec_op, 0, 0, 0, 0, movn_op), RS | RT | RD}, @@ -177,6 +187,8 @@ static const struct insn insn_table[insn_invalid] = { [insn_scd] = {M6(spec3_op, 0, 0, 0, scd6_op), RS | RT | SIMM9}, #endif [insn_sd] = {M(sd_op, 0, 0, 0, 0, 0), RS | RT | SIMM}, + [insn_seb] = {M(spec3_op, 0, 0, 0, seb_op, bshfl_op), RT | RD}, + [insn_seh] = {M(spec3_op, 0, 0, 0, seh_op, bshfl_op), RT | RD}, [insn_seleqz] = {M(spec_op, 0, 0, 0, 0, seleqz_op), RS | RT | RD}, [insn_selnez] = {M(spec_op, 0, 0, 0, 0, selnez_op), RS | RT | RD}, [insn_sh] = {M(sh_op, 0, 0, 0, 0, 0), RS | RT | SIMM}, diff --git a/arch/mips/mm/uasm.c b/arch/mips/mm/uasm.c index 125140979d62..139afb0e14cd 100644 --- a/arch/mips/mm/uasm.c +++ b/arch/mips/mm/uasm.c @@ -49,19 +49,21 @@ enum opcode { insn_addiu, insn_addu, insn_and, insn_andi, insn_bbit0, insn_bbit1, insn_beq, insn_beql, insn_bgez, insn_bgezl, insn_bgtz, insn_blez, insn_bltz, insn_bltzl, insn_bne, insn_break, insn_cache, insn_cfc1, - insn_cfcmsa, insn_ctc1, insn_ctcmsa, insn_daddiu, insn_daddu, insn_ddivu, - insn_ddivu_r6, insn_di, insn_dins, insn_dinsm, insn_dinsu, insn_divu, - insn_divu_r6, insn_dmfc0, insn_dmodu, insn_dmtc0, insn_dmultu, + insn_cfcmsa, insn_ctc1, insn_ctcmsa, insn_daddiu, insn_daddu, insn_ddiv, + insn_ddiv_r6, insn_ddivu, insn_ddivu_r6, insn_di, insn_dins, insn_dinsm, + insn_dinsu, insn_div, insn_div_r6, insn_divu, insn_divu_r6, insn_dmfc0, + insn_dmod, insn_dmodu, insn_dmtc0, insn_dmultu, insn_dmulu, insn_drotr, insn_drotr32, insn_dsbh, insn_dshd, insn_dsll, insn_dsll32, insn_dsllv, insn_dsra, insn_dsra32, insn_dsrav, insn_dsrl, insn_dsrl32, insn_dsrlv, insn_dsubu, insn_eret, insn_ext, insn_ins, insn_j, insn_jal, insn_jalr, insn_jr, insn_lb, insn_lbu, insn_ld, insn_lddir, insn_ldpte, insn_ldx, insn_lh, insn_lhu, insn_ll, insn_lld, insn_lui, insn_lw, insn_lwu, insn_lwx, insn_mfc0, insn_mfhc0, insn_mfhi, - insn_mflo, insn_modu, insn_movn, insn_movz, insn_mtc0, insn_mthc0, - insn_mthi, insn_mtlo, insn_mul, insn_multu, insn_mulu, insn_muhu, insn_nor, - insn_or, insn_ori, insn_pref, insn_rfe, insn_rotr, insn_sb, insn_sc, - insn_scd, insn_seleqz, insn_selnez, insn_sd, insn_sh, insn_sll, + insn_mflo, insn_mod, insn_modu, insn_movn, insn_movz, insn_mtc0, + insn_mthc0, insn_mthi, insn_mtlo, insn_mul, insn_multu, insn_mulu, + insn_muhu, insn_nor, insn_or, insn_ori, insn_pref, insn_rfe, insn_rotr, + insn_sb, insn_sc, insn_scd, insn_seb, insn_seh, + insn_seleqz, insn_selnez, insn_sd, insn_sh, insn_sll, insn_sllv, insn_slt, insn_slti, insn_sltiu, insn_sltu, insn_sra, insn_srav, insn_srl, insn_srlv, insn_subu, insn_sw, insn_sync, insn_syscall, insn_tlbp, insn_tlbr, insn_tlbwi, insn_tlbwr, insn_wait, @@ -287,9 +289,12 @@ I_u1u2(_cfc1) I_u2u1(_cfcmsa) I_u1u2(_ctc1) I_u2u1(_ctcmsa) +I_u1u2(_ddiv) +I_u3u1u2(_ddiv_r6) I_u1u2(_ddivu) I_u3u1u2(_ddivu_r6) I_u1u2u3(_dmfc0) +I_u3u1u2(_dmod) I_u3u1u2(_dmodu) I_u1u2u3(_dmtc0) I_u1u2(_dmultu) @@ -297,6 +302,8 @@ I_u3u1u2(_dmulu) I_u2u1s3(_daddiu) I_u3u1u2(_daddu) I_u1(_di); +I_u1u2(_div) +I_u3u1u2(_div_r6) I_u1u2(_divu) I_u3u1u2(_divu_r6) I_u2u1(_dsbh); @@ -332,6 +339,7 @@ I_u2s3u1(_lw) I_u2s3u1(_lwu) I_u1u2u3(_mfc0) I_u1u2u3(_mfhc0) +I_u3u1u2(_mod) I_u3u1u2(_modu) I_u3u1u2(_movn) I_u3u1u2(_movz) @@ -353,6 +361,8 @@ I_u2s3u1(_sb) I_u2s3u1(_sc) I_u2s3u1(_scd) I_u2s3u1(_sd) +I_u2u1(_seb) +I_u2u1(_seh) I_u3u1u2(_seleqz) I_u3u1u2(_selnez) I_u2s3u1(_sh) diff --git a/arch/mips/net/bpf_jit_comp.c b/arch/mips/net/bpf_jit_comp.c index 6ee4abe6a1f7..4682dbb3a059 100644 --- a/arch/mips/net/bpf_jit_comp.c +++ b/arch/mips/net/bpf_jit_comp.c @@ -208,7 +208,7 @@ void emit_mov_r(struct jit_context *ctx, u8 dst, u8 src) } /* Validate ALU immediate range */ -bool valid_alu_i(u8 op, s32 imm) +bool valid_alu_i(u8 op, s32 imm, s16 off) { switch (BPF_OP(op)) { case BPF_NEG: @@ -237,6 +237,9 @@ bool valid_alu_i(u8 op, s32 imm) return imm == 0 || (imm > 0 && is_power_of_2(imm)); case BPF_DIV: case BPF_MOD: + /* Do not use unsigned shift/mask rewrites for signed div/mod. */ + if (off == 1) + return false; /* imm must be an 17-bit power of two */ return (u32)imm <= 0x10000 && is_power_of_2((u32)imm); } @@ -338,8 +341,42 @@ void emit_alu_i(struct jit_context *ctx, u8 dst, s32 imm, u8 op) clobber_reg(ctx, dst); } +/* ALU division operation (32-bit) */ +static void emit_div(struct jit_context *ctx, u8 dst, u8 src, s16 off) +{ + if (cpu_has_mips32r6) { + if (off == 1) + emit(ctx, div_r6, dst, dst, src); + else + emit(ctx, divu_r6, dst, dst, src); + } else { + if (off == 1) + emit(ctx, div, dst, src); + else + emit(ctx, divu, dst, src); + emit(ctx, mflo, dst); + } +} + +/* ALU modulo operation (32-bit) */ +static void emit_mod(struct jit_context *ctx, u8 dst, u8 src, s16 off) +{ + if (cpu_has_mips32r6) { + if (off == 1) + emit(ctx, mod, dst, dst, src); + else + emit(ctx, modu, dst, dst, src); + } else { + if (off == 1) + emit(ctx, div, dst, src); + else + emit(ctx, divu, dst, src); + emit(ctx, mfhi, dst); + } +} + /* ALU register operation (32-bit) */ -void emit_alu_r(struct jit_context *ctx, u8 dst, u8 src, u8 op) +void emit_alu_r(struct jit_context *ctx, u8 dst, u8 src, u8 op, s16 off) { switch (BPF_OP(op)) { /* dst = dst & src */ @@ -385,21 +422,11 @@ void emit_alu_r(struct jit_context *ctx, u8 dst, u8 src, u8 op) break; /* dst = dst / src */ case BPF_DIV: - if (cpu_has_mips32r6) { - emit(ctx, divu_r6, dst, dst, src); - } else { - emit(ctx, divu, dst, src); - emit(ctx, mflo, dst); - } + emit_div(ctx, dst, src, off); break; /* dst = dst % src */ case BPF_MOD: - if (cpu_has_mips32r6) { - emit(ctx, modu, dst, dst, src); - } else { - emit(ctx, divu, dst, src); - emit(ctx, mfhi, dst); - } + emit_mod(ctx, dst, src, off); break; } clobber_reg(ctx, dst); @@ -823,7 +850,7 @@ void emit_jmp_r(struct jit_context *ctx, u8 dst, u8 src, s32 off, u8 op) } /* Jump always */ -int emit_ja(struct jit_context *ctx, s16 off) +int emit_ja(struct jit_context *ctx, s32 off) { int target = get_target(ctx, ctx->bpf_index + off + 1); diff --git a/arch/mips/net/bpf_jit_comp.h b/arch/mips/net/bpf_jit_comp.h index a37fe20818eb..d0ab5c9a0c46 100644 --- a/arch/mips/net/bpf_jit_comp.h +++ b/arch/mips/net/bpf_jit_comp.h @@ -163,7 +163,7 @@ void emit_mov_i(struct jit_context *ctx, u8 dst, s32 imm); void emit_mov_r(struct jit_context *ctx, u8 dst, u8 src); /* Validate ALU/ALU64 immediate range */ -bool valid_alu_i(u8 op, s32 imm); +bool valid_alu_i(u8 op, s32 imm, s16 off); /* Rewrite ALU/ALU64 immediate operation */ bool rewrite_alu_i(u8 op, s32 imm, u8 *alu, s32 *val); @@ -172,7 +172,7 @@ bool rewrite_alu_i(u8 op, s32 imm, u8 *alu, s32 *val); void emit_alu_i(struct jit_context *ctx, u8 dst, s32 imm, u8 op); /* ALU register operation (32-bit) */ -void emit_alu_r(struct jit_context *ctx, u8 dst, u8 src, u8 op); +void emit_alu_r(struct jit_context *ctx, u8 dst, u8 src, u8 op, s16 off); /* Atomic read-modify-write (32-bit) */ void emit_atomic_r(struct jit_context *ctx, u8 dst, u8 src, s16 off, u8 code); @@ -204,7 +204,7 @@ void emit_jmp_i(struct jit_context *ctx, u8 dst, s32 imm, s32 off, u8 op); void emit_jmp_r(struct jit_context *ctx, u8 dst, u8 src, s32 off, u8 op); /* Jump always */ -int emit_ja(struct jit_context *ctx, s16 off); +int emit_ja(struct jit_context *ctx, s32 off); /* Jump to epilogue */ int emit_exit(struct jit_context *ctx); diff --git a/arch/mips/net/bpf_jit_comp32.c b/arch/mips/net/bpf_jit_comp32.c index 15a2a153dc87..84d9c7036e4f 100644 --- a/arch/mips/net/bpf_jit_comp32.c +++ b/arch/mips/net/bpf_jit_comp32.c @@ -190,6 +190,69 @@ static void emit_zext_ver(struct jit_context *ctx, const u8 dst[]) } } +/* Sign-extend an 8- or 16-bit field into a native register. */ +static void emit_movsx_r(struct jit_context *ctx, u8 dst, u8 src, s16 off) +{ + int shift; + + if (cpu_has_mips32r2 || cpu_has_mips32r6) { + if (off == 8) + emit(ctx, seb, dst, src); + else + emit(ctx, seh, dst, src); + } else { + shift = 32 - off; + emit(ctx, sll, dst, src, shift); + emit(ctx, sra, dst, dst, shift); + } +} + +/* Register move operation (32-bit), optionally with sign extension */ +static void emit_mov_r32(struct jit_context *ctx, const u8 dst[], + const u8 src[], s16 off) +{ + switch (off) { + case 8: + case 16: + emit_movsx_r(ctx, lo(dst), lo(src), off); + break; + default: + /* off == 0 is MOV; the verifier rejects other offsets. */ + emit_mov_r(ctx, lo(dst), lo(src)); + break; + } + clobber_reg(ctx, lo(dst)); + emit_zext_ver(ctx, dst); +} + +/* Register move operation (64-bit), optionally with sign extension */ +static void emit_mov_r64(struct jit_context *ctx, const u8 dst[], + const u8 src[], s16 off) +{ + switch (off) { + case 8: + case 16: + emit_movsx_r(ctx, lo(dst), lo(src), off); + emit(ctx, sra, hi(dst), lo(dst), 31); + break; + case 32: + emit(ctx, move, lo(dst), lo(src)); + emit(ctx, sra, hi(dst), lo(dst), 31); + break; + default: + /* + * off == 0 is ordinary MOV. The verifier rejects other + * offsets; defined exceptions require + * bpf_jit_supports_percpu_insn() or bpf_jit_supports_arena(), + * neither implemented by MIPS. + */ + emit_mov_r(ctx, lo(dst), lo(src)); + emit_mov_r(ctx, hi(dst), hi(src)); + break; + } + clobber_reg64(ctx, dst); +} + /* Load delay slot, if ISA mandates it */ static void emit_load_delay(struct jit_context *ctx) { @@ -512,7 +575,7 @@ static void emit_mul_r64(struct jit_context *ctx, clobber_reg64(ctx, dst); } -/* Helper function for 64-bit modulo */ +/* Helper function for unsigned 64-bit modulo */ static u64 jit_mod64(u64 a, u64 b) { u64 rem; @@ -521,13 +584,22 @@ static u64 jit_mod64(u64 a, u64 b) return rem; } +/* Helper function for signed 64-bit modulo */ +static s64 jit_smod64(s64 a, s64 b) +{ + s64 quot = div64_s64(a, b); + + return a - quot * b; +} + /* ALU div/mod register (64-bit) */ -static void emit_divmod_r64(struct jit_context *ctx, - const u8 dst[], const u8 src[], u8 op) +static void emit_divmod_r64(struct jit_context *ctx, const u8 dst[], + const u8 src[], u8 op, s16 off) { const u8 *r0 = bpf2mips32[BPF_REG_0]; /* Mapped to v0-v1 */ const u8 *r1 = bpf2mips32[BPF_REG_1]; /* Mapped to a0-a1 */ const u8 *r2 = bpf2mips32[BPF_REG_2]; /* Mapped to a2-a3 */ + bool is_signed = off == 1; int exclude, k; u32 addr = 0; @@ -546,11 +618,11 @@ static void emit_divmod_r64(struct jit_context *ctx, switch (BPF_OP(op)) { /* dst = dst / src */ case BPF_DIV: - addr = (u32)&div64_u64; + addr = is_signed ? (u32)&div64_s64 : (u32)&div64_u64; break; /* dst = dst % src */ case BPF_MOD: - addr = (u32)&jit_mod64; + addr = is_signed ? (u32)&jit_smod64 : (u32)&jit_mod64; break; } emit_mov_i(ctx, MIPS_R_T9, addr); @@ -1485,8 +1557,7 @@ int build_insn(const struct bpf_insn *insn, struct jit_context *ctx) /* Special mov32 for zext */ emit_mov_i(ctx, hi(dst), 0); } else { - emit_mov_r(ctx, lo(dst), lo(src)); - emit_zext_ver(ctx, dst); + emit_mov_r32(ctx, dst, src, off); } break; /* dst = -dst */ @@ -1516,9 +1587,9 @@ int build_insn(const struct bpf_insn *insn, struct jit_context *ctx) case BPF_ALU | BPF_MUL | BPF_K: case BPF_ALU | BPF_DIV | BPF_K: case BPF_ALU | BPF_MOD | BPF_K: - if (!valid_alu_i(BPF_OP(code), imm)) { + if (!valid_alu_i(BPF_OP(code), imm, off)) { emit_mov_i(ctx, MIPS_R_T6, imm); - emit_alu_r(ctx, lo(dst), MIPS_R_T6, BPF_OP(code)); + emit_alu_r(ctx, lo(dst), MIPS_R_T6, BPF_OP(code), off); } else if (rewrite_alu_i(BPF_OP(code), imm, &alu, &val)) { emit_alu_i(ctx, lo(dst), val, alu); } @@ -1546,7 +1617,7 @@ int build_insn(const struct bpf_insn *insn, struct jit_context *ctx) case BPF_ALU | BPF_MUL | BPF_X: case BPF_ALU | BPF_DIV | BPF_X: case BPF_ALU | BPF_MOD | BPF_X: - emit_alu_r(ctx, lo(dst), lo(src), BPF_OP(code)); + emit_alu_r(ctx, lo(dst), lo(src), BPF_OP(code), off); emit_zext_ver(ctx, dst); break; /* dst = imm (64-bit) */ @@ -1555,8 +1626,7 @@ int build_insn(const struct bpf_insn *insn, struct jit_context *ctx) break; /* dst = src (64-bit) */ case BPF_ALU64 | BPF_MOV | BPF_X: - emit_mov_r(ctx, lo(dst), lo(src)); - emit_mov_r(ctx, hi(dst), hi(src)); + emit_mov_r64(ctx, dst, src, off); break; /* dst = -dst (64-bit) */ case BPF_ALU64 | BPF_NEG: @@ -1599,7 +1669,7 @@ int build_insn(const struct bpf_insn *insn, struct jit_context *ctx) * and then do the operation on this register. */ emit_mov_se_i64(ctx, tmp, imm); - emit_divmod_r64(ctx, dst, tmp, BPF_OP(code)); + emit_divmod_r64(ctx, dst, tmp, BPF_OP(code), off); break; /* dst = dst & src (64-bit) */ /* dst = dst | src (64-bit) */ @@ -1629,7 +1699,11 @@ int build_insn(const struct bpf_insn *insn, struct jit_context *ctx) /* dst = dst % src (64-bit) */ case BPF_ALU64 | BPF_DIV | BPF_X: case BPF_ALU64 | BPF_MOD | BPF_X: - emit_divmod_r64(ctx, dst, src, BPF_OP(code)); + emit_divmod_r64(ctx, dst, src, BPF_OP(code), off); + break; + /* dst = bswap(dst) */ + case BPF_ALU64 | BPF_END: + emit_bswap_r64(ctx, dst, imm); break; /* dst = htole(dst) */ /* dst = htobe(dst) */ @@ -1803,6 +1877,13 @@ int build_insn(const struct bpf_insn *insn, struct jit_context *ctx) if (finish_jmp(ctx, jmp, off) < 0) goto toofar; break; + /* PC += imm */ + case BPF_JMP32 | BPF_JA: + if (imm == 0) + break; + if (emit_ja(ctx, imm) < 0) + goto toofar; + break; /* PC += off if dst == src */ /* PC += off if dst != src */ /* PC += off if dst & src */ diff --git a/arch/mips/net/bpf_jit_comp64.c b/arch/mips/net/bpf_jit_comp64.c index 6681ccac9dd9..994589c3de46 100644 --- a/arch/mips/net/bpf_jit_comp64.c +++ b/arch/mips/net/bpf_jit_comp64.c @@ -120,6 +120,58 @@ static void emit_zext_ver(struct jit_context *ctx, u8 dst) emit_zext(ctx, dst); } +/* Sign-extend an 8- or 16-bit field into a native register. */ +static void emit_movsx_r(struct jit_context *ctx, u8 dst, u8 src, s16 off) +{ + /* seb/seh would need a sign-extended 32-bit source first. */ + int shift = 32 - off; + + /* dsll32 and dsra32 add 32 to the shift argument. */ + emit(ctx, dsll32, dst, src, shift); + emit(ctx, dsra32, dst, dst, shift); +} + +/* Register move operation (32-bit), optionally with sign extension */ +static void emit_mov_r32(struct jit_context *ctx, u8 dst, u8 src, s16 off) +{ + switch (off) { + case 8: + case 16: + emit_movsx_r(ctx, dst, src, off); + break; + default: + /* off == 0 is MOV; the verifier rejects other offsets. */ + emit_mov_r(ctx, dst, src); + break; + } + clobber_reg(ctx, dst); + emit_zext_ver(ctx, dst); +} + +/* Register move operation (64-bit), optionally with sign extension */ +static void emit_mov_r64(struct jit_context *ctx, u8 dst, u8 src, s16 off) +{ + switch (off) { + case 8: + case 16: + emit_movsx_r(ctx, dst, src, off); + break; + case 32: + emit_sext(ctx, dst, src); + break; + default: + /* + * off == 0 is ordinary MOV. The verifier rejects other + * offsets; defined exceptions require + * bpf_jit_supports_percpu_insn() or bpf_jit_supports_arena(), + * neither implemented by MIPS. + */ + emit_mov_r(ctx, dst, src); + break; + } + clobber_reg(ctx, dst); +} + /* dst = imm (64-bit) */ static void emit_mov_i64(struct jit_context *ctx, u8 dst, u64 imm64) { @@ -197,8 +249,43 @@ static void emit_alu_i64(struct jit_context *ctx, u8 dst, s32 imm, u8 op) clobber_reg(ctx, dst); } +/* ALU division operation (64-bit) */ +static void emit_div64(struct jit_context *ctx, u8 dst, u8 src, s16 off) +{ + if (cpu_has_mips64r6) { + if (off == 1) + emit(ctx, ddiv_r6, dst, dst, src); + else + emit(ctx, ddivu_r6, dst, dst, src); + } else { + if (off == 1) + emit(ctx, ddiv, dst, src); + else + emit(ctx, ddivu, dst, src); + emit(ctx, mflo, dst); + } +} + +/* ALU modulo operation (64-bit) */ +static void emit_mod64(struct jit_context *ctx, u8 dst, u8 src, s16 off) +{ + if (cpu_has_mips64r6) { + if (off == 1) + emit(ctx, dmod, dst, dst, src); + else + emit(ctx, dmodu, dst, dst, src); + } else { + if (off == 1) + emit(ctx, ddiv, dst, src); + else + emit(ctx, ddivu, dst, src); + emit(ctx, mfhi, dst); + } +} + /* ALU register operation (64-bit) */ -static void emit_alu_r64(struct jit_context *ctx, u8 dst, u8 src, u8 op) +static void emit_alu_r64(struct jit_context *ctx, u8 dst, u8 src, u8 op, + s16 off) { switch (BPF_OP(op)) { /* dst = dst << src */ @@ -235,25 +322,15 @@ static void emit_alu_r64(struct jit_context *ctx, u8 dst, u8 src, u8 op) break; /* dst = dst / src */ case BPF_DIV: - if (cpu_has_mips64r6) { - emit(ctx, ddivu_r6, dst, dst, src); - } else { - emit(ctx, ddivu, dst, src); - emit(ctx, mflo, dst); - } + emit_div64(ctx, dst, src, off); break; /* dst = dst % src */ case BPF_MOD: - if (cpu_has_mips64r6) { - emit(ctx, dmodu, dst, dst, src); - } else { - emit(ctx, ddivu, dst, src); - emit(ctx, mfhi, dst); - } + emit_mod64(ctx, dst, src, off); break; default: /* Width-generic operations */ - emit_alu_r(ctx, dst, src, op); + emit_alu_r(ctx, dst, src, op, off); } clobber_reg(ctx, dst); } @@ -655,8 +732,7 @@ int build_insn(const struct bpf_insn *insn, struct jit_context *ctx) /* Special mov32 for zext */ emit_zext(ctx, dst); } else { - emit_mov_r(ctx, dst, src); - emit_zext_ver(ctx, dst); + emit_mov_r32(ctx, dst, src, off); } break; /* dst = -dst */ @@ -673,9 +749,9 @@ int build_insn(const struct bpf_insn *insn, struct jit_context *ctx) case BPF_ALU | BPF_AND | BPF_K: case BPF_ALU | BPF_XOR | BPF_K: case BPF_ALU | BPF_LSH | BPF_K: - if (!valid_alu_i(BPF_OP(code), imm)) { + if (!valid_alu_i(BPF_OP(code), imm, off)) { emit_mov_i(ctx, MIPS_R_T4, imm); - emit_alu_r(ctx, dst, MIPS_R_T4, BPF_OP(code)); + emit_alu_r(ctx, dst, MIPS_R_T4, BPF_OP(code), off); } else if (rewrite_alu_i(BPF_OP(code), imm, &alu, &val)) { emit_alu_i(ctx, dst, val, alu); } @@ -695,10 +771,10 @@ int build_insn(const struct bpf_insn *insn, struct jit_context *ctx) case BPF_ALU | BPF_MUL | BPF_K: case BPF_ALU | BPF_DIV | BPF_K: case BPF_ALU | BPF_MOD | BPF_K: - if (!valid_alu_i(BPF_OP(code), imm)) { + if (!valid_alu_i(BPF_OP(code), imm, off)) { emit_sext(ctx, dst, dst); emit_mov_i(ctx, MIPS_R_T4, imm); - emit_alu_r(ctx, dst, MIPS_R_T4, BPF_OP(code)); + emit_alu_r(ctx, dst, MIPS_R_T4, BPF_OP(code), off); } else if (rewrite_alu_i(BPF_OP(code), imm, &alu, &val)) { emit_sext(ctx, dst, dst); emit_alu_i(ctx, dst, val, alu); @@ -713,7 +789,7 @@ int build_insn(const struct bpf_insn *insn, struct jit_context *ctx) case BPF_ALU | BPF_OR | BPF_X: case BPF_ALU | BPF_XOR | BPF_X: case BPF_ALU | BPF_LSH | BPF_X: - emit_alu_r(ctx, dst, src, BPF_OP(code)); + emit_alu_r(ctx, dst, src, BPF_OP(code), off); emit_zext_ver(ctx, dst); break; /* dst = dst >> src */ @@ -732,7 +808,7 @@ int build_insn(const struct bpf_insn *insn, struct jit_context *ctx) case BPF_ALU | BPF_MOD | BPF_X: emit_sext(ctx, dst, dst); emit_sext(ctx, MIPS_R_T4, src); - emit_alu_r(ctx, dst, MIPS_R_T4, BPF_OP(code)); + emit_alu_r(ctx, dst, MIPS_R_T4, BPF_OP(code), off); emit_zext_ver(ctx, dst); break; /* dst = imm (64-bit) */ @@ -741,7 +817,7 @@ int build_insn(const struct bpf_insn *insn, struct jit_context *ctx) break; /* dst = src (64-bit) */ case BPF_ALU64 | BPF_MOV | BPF_X: - emit_mov_r(ctx, dst, src); + emit_mov_r64(ctx, dst, src, off); break; /* dst = -dst (64-bit) */ case BPF_ALU64 | BPF_NEG: @@ -769,9 +845,9 @@ int build_insn(const struct bpf_insn *insn, struct jit_context *ctx) case BPF_ALU64 | BPF_MUL | BPF_K: case BPF_ALU64 | BPF_DIV | BPF_K: case BPF_ALU64 | BPF_MOD | BPF_K: - if (!valid_alu_i(BPF_OP(code), imm)) { + if (!valid_alu_i(BPF_OP(code), imm, off)) { emit_mov_i(ctx, MIPS_R_T4, imm); - emit_alu_r64(ctx, dst, MIPS_R_T4, BPF_OP(code)); + emit_alu_r64(ctx, dst, MIPS_R_T4, BPF_OP(code), off); } else if (rewrite_alu_i(BPF_OP(code), imm, &alu, &val)) { emit_alu_i64(ctx, dst, val, alu); } @@ -798,7 +874,11 @@ int build_insn(const struct bpf_insn *insn, struct jit_context *ctx) case BPF_ALU64 | BPF_MUL | BPF_X: case BPF_ALU64 | BPF_DIV | BPF_X: case BPF_ALU64 | BPF_MOD | BPF_X: - emit_alu_r64(ctx, dst, src, BPF_OP(code)); + emit_alu_r64(ctx, dst, src, BPF_OP(code), off); + break; + /* dst = bswap(dst) */ + case BPF_ALU64 | BPF_END: + emit_bswap_r64(ctx, dst, imm); break; /* dst = htole(dst) */ /* dst = htobe(dst) */ @@ -961,6 +1041,13 @@ int build_insn(const struct bpf_insn *insn, struct jit_context *ctx) if (finish_jmp(ctx, jmp, off) < 0) goto toofar; break; + /* PC += imm */ + case BPF_JMP32 | BPF_JA: + if (imm == 0) + break; + if (emit_ja(ctx, imm) < 0) + goto toofar; + break; /* PC += off if dst == src */ /* PC += off if dst != src */ /* PC += off if dst & src */ diff --git a/arch/openrisc/include/asm/cmpxchg.h b/arch/openrisc/include/asm/cmpxchg.h index 8ee151c072e4..fcf1bc8db8fe 100644 --- a/arch/openrisc/include/asm/cmpxchg.h +++ b/arch/openrisc/include/asm/cmpxchg.h @@ -140,6 +140,8 @@ static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old, sizeof(*(ptr))); \ }) +#define arch_cmpxchg_local arch_cmpxchg + /* * This function doesn't exist, so you'll get a linker error if * something tries to do an invalidly-sized xchg(). diff --git a/arch/parisc/net/bpf_jit.h b/arch/parisc/net/bpf_jit.h index 8b8896959f04..074115c93c90 100644 --- a/arch/parisc/net/bpf_jit.h +++ b/arch/parisc/net/bpf_jit.h @@ -467,6 +467,8 @@ static inline u32 hppa_t21_insn(u8 opcode, u8 r2, u8 r1, u8 ext8, u8 t) u64 hppa_div64(u64 div, u64 divisor); u64 hppa_div64_rem(u64 div, u64 divisor); +u64 hppa_sdiv64(u64 div, u64 divisor); +u64 hppa_sdiv64_rem(u64 div, u64 divisor); /* Helper functions that emit HPPA instructions when possible. */ diff --git a/arch/parisc/net/bpf_jit_comp32.c b/arch/parisc/net/bpf_jit_comp32.c index 5ff0cf925fe9..eea0a34a3bfe 100644 --- a/arch/parisc/net/bpf_jit_comp32.c +++ b/arch/parisc/net/bpf_jit_comp32.c @@ -321,6 +321,8 @@ static void bpf_put_reg32(const s8 *reg, const s8 *src, extern void $$mulI(void); extern void $$divU(void); extern void $$remU(void); +extern void $$divI(void); +extern void $$remI(void); static void emit_call_millicode(void *func, const s8 arg0, const s8 arg1, u8 opcode, struct hppa_jit_context *ctx) @@ -562,12 +564,14 @@ static void emit_alu_i32(const s8 *dst, s32 imm, } static void emit_alu_r64(const s8 *dst, const s8 *src, - struct hppa_jit_context *ctx, const u8 op) + struct hppa_jit_context *ctx, const u8 op, + bool is_signed) { const s8 *tmp1 = regmap[TMP_REG_1]; const s8 *tmp2 = regmap[TMP_REG_2]; const s8 *rd; const s8 *rs = bpf_get_reg64(src, tmp2, ctx); + void *func; if (op == BPF_MOV) rd = bpf_get_reg64_ref(dst, tmp1, false, ctx); @@ -604,10 +608,12 @@ static void emit_alu_r64(const s8 *dst, const s8 *src, emit_call_libgcc_ll(__muldi3, rd, rs, op, ctx); break; case BPF_DIV: - emit_call_libgcc_ll(&hppa_div64, rd, rs, op, ctx); + func = is_signed ? &hppa_sdiv64 : &hppa_div64; + emit_call_libgcc_ll(func, rd, rs, op, ctx); break; case BPF_MOD: - emit_call_libgcc_ll(&hppa_div64_rem, rd, rs, op, ctx); + func = is_signed ? &hppa_sdiv64_rem : &hppa_div64_rem; + emit_call_libgcc_ll(func, rd, rs, op, ctx); break; case BPF_LSH: emit_call_libgcc_ll(__ashldi3, rd, rs, op, ctx); @@ -630,7 +636,8 @@ static void emit_alu_r64(const s8 *dst, const s8 *src, } static void emit_alu_r32(const s8 *dst, const s8 *src, - struct hppa_jit_context *ctx, const u8 op) + struct hppa_jit_context *ctx, const u8 op, + bool is_signed) { const s8 *tmp1 = regmap[TMP_REG_1]; const s8 *tmp2 = regmap[TMP_REG_2]; @@ -666,10 +673,12 @@ static void emit_alu_r32(const s8 *dst, const s8 *src, emit_call_millicode($$mulI, lo(rd), lo(rs), op, ctx); break; case BPF_DIV: - emit_call_millicode($$divU, lo(rd), lo(rs), op, ctx); + emit_call_millicode(is_signed ? $$divI : $$divU, + lo(rd), lo(rs), op, ctx); break; case BPF_MOD: - emit_call_millicode($$remU, lo(rd), lo(rs), op, ctx); + emit_call_millicode(is_signed ? $$remI : $$remU, + lo(rd), lo(rs), op, ctx); break; case BPF_LSH: emit(hppa_subi(0x1f, lo(rs), HPPA_REG_T0), ctx); @@ -1163,12 +1172,12 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct hppa_jit_context *ctx, emit_imm32(tmp2, imm, ctx); src = tmp2; } - emit_alu_r64(dst, src, ctx, BPF_OP(code)); + emit_alu_r64(dst, src, ctx, BPF_OP(code), off == 1); break; /* dst = -dst */ case BPF_ALU64 | BPF_NEG: - emit_alu_r64(dst, tmp2, ctx, BPF_OP(code)); + emit_alu_r64(dst, tmp2, ctx, BPF_OP(code), false); break; case BPF_ALU64 | BPF_MOV | BPF_K: @@ -1211,7 +1220,7 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct hppa_jit_context *ctx, emit_imm32(tmp2, imm, ctx); src = tmp2; } - emit_alu_r32(dst, src, ctx, BPF_OP(code)); + emit_alu_r32(dst, src, ctx, BPF_OP(code), off == 1); break; /* dst = dst OP imm */ @@ -1236,7 +1245,7 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct hppa_jit_context *ctx, * src is ignored---choose tmp2 as a dummy register since it * is not on the stack. */ - emit_alu_r32(dst, tmp2, ctx, BPF_OP(code)); + emit_alu_r32(dst, tmp2, ctx, BPF_OP(code), false); break; /* dst = BSWAP##imm(dst) */ diff --git a/arch/parisc/net/bpf_jit_comp64.c b/arch/parisc/net/bpf_jit_comp64.c index 54b0d5e25e02..c326fa737ec8 100644 --- a/arch/parisc/net/bpf_jit_comp64.c +++ b/arch/parisc/net/bpf_jit_comp64.c @@ -502,14 +502,19 @@ static void emit_call(u64 addr, bool fixed, struct hppa_jit_context *ctx) emit_hppa_copy(HPPA_REG_RET0, regmap[BPF_REG_0], ctx); } -static void emit_call_libgcc_ll(void *func, const s8 arg0, - const s8 arg1, u8 opcode, struct hppa_jit_context *ctx) +static void emit_call_libgcc_ll(void *func, const s8 arg0, const s8 arg1, + u8 opcode, bool is_signed, struct hppa_jit_context *ctx) { u64 func_addr; if (BPF_CLASS(opcode) == BPF_ALU) { - emit_hppa64_zext32(arg0, HPPA_REG_ARG0, ctx); - emit_hppa64_zext32(arg1, HPPA_REG_ARG1, ctx); + if (is_signed) { + emit_hppa64_sext32(arg0, HPPA_REG_ARG0, ctx); + emit_hppa64_sext32(arg1, HPPA_REG_ARG1, ctx); + } else { + emit_hppa64_zext32(arg0, HPPA_REG_ARG0, ctx); + emit_hppa64_zext32(arg1, HPPA_REG_ARG1, ctx); + } } else { emit_hppa_copy(arg0, HPPA_REG_ARG0, ctx); emit_hppa_copy(arg1, HPPA_REG_ARG1, ctx); @@ -600,6 +605,8 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct hppa_jit_context *ctx, u8 rd = -1, rs = -1, code = insn->code; s16 off = insn->off; s32 imm = insn->imm; + bool is_signed; + void *func; init_regs(&rd, &rs, insn, ctx); @@ -656,29 +663,39 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct hppa_jit_context *ctx, fallthrough; case BPF_ALU | BPF_MUL | BPF_X: case BPF_ALU64 | BPF_MUL | BPF_X: - emit_call_libgcc_ll(__muldi3, rd, rs, code, ctx); + emit_call_libgcc_ll(__muldi3, rd, rs, code, false, ctx); if (!is64 && !aux->verifier_zext) emit_zext_32(rd, ctx); break; case BPF_ALU | BPF_DIV | BPF_K: case BPF_ALU64 | BPF_DIV | BPF_K: - emit_imm(HPPA_REG_T1, is64 ? (s64)(s32)imm : (u32)imm, HPPA_REG_T2, ctx); + is_signed = (off == 1); + emit_imm(HPPA_REG_T1, + is64 || is_signed ? (s64)(s32)imm : (u32)imm, + HPPA_REG_T2, ctx); rs = HPPA_REG_T1; fallthrough; case BPF_ALU | BPF_DIV | BPF_X: case BPF_ALU64 | BPF_DIV | BPF_X: - emit_call_libgcc_ll(&hppa_div64, rd, rs, code, ctx); + is_signed = (off == 1); + func = is_signed ? &hppa_sdiv64 : &hppa_div64; + emit_call_libgcc_ll(func, rd, rs, code, is_signed, ctx); if (!is64 && !aux->verifier_zext) emit_zext_32(rd, ctx); break; case BPF_ALU | BPF_MOD | BPF_K: case BPF_ALU64 | BPF_MOD | BPF_K: - emit_imm(HPPA_REG_T1, is64 ? (s64)(s32)imm : (u32)imm, HPPA_REG_T2, ctx); + is_signed = (off == 1); + emit_imm(HPPA_REG_T1, + is64 || is_signed ? (s64)(s32)imm : (u32)imm, + HPPA_REG_T2, ctx); rs = HPPA_REG_T1; fallthrough; case BPF_ALU | BPF_MOD | BPF_X: case BPF_ALU64 | BPF_MOD | BPF_X: - emit_call_libgcc_ll(&hppa_div64_rem, rd, rs, code, ctx); + is_signed = (off == 1); + func = is_signed ? &hppa_sdiv64_rem : &hppa_div64_rem; + emit_call_libgcc_ll(func, rd, rs, code, is_signed, ctx); if (!is64 && !aux->verifier_zext) emit_zext_32(rd, ctx); break; diff --git a/arch/parisc/net/bpf_jit_core.c b/arch/parisc/net/bpf_jit_core.c index 172770132440..a2f93202a2ef 100644 --- a/arch/parisc/net/bpf_jit_core.c +++ b/arch/parisc/net/bpf_jit_core.c @@ -190,3 +190,17 @@ u64 hppa_div64_rem(u64 div, u64 divisor) div64_u64_rem(div, divisor, &rem); return rem; } + +u64 hppa_sdiv64(u64 div, u64 divisor) +{ + s64 sdiv = div64_s64((s64)div, (s64)divisor); + + return (u64)sdiv; +} + +u64 hppa_sdiv64_rem(u64 div, u64 divisor) +{ + s64 sdiv = div64_s64((s64)div, (s64)divisor); + + return (u64)((s64)div - sdiv * (s64)divisor); +} diff --git a/arch/powerpc/configs/ppc64_defconfig b/arch/powerpc/configs/ppc64_defconfig index 1eb8e3457e8b..e0b7901add11 100644 --- a/arch/powerpc/configs/ppc64_defconfig +++ b/arch/powerpc/configs/ppc64_defconfig @@ -442,7 +442,6 @@ CONFIG_TEST_IDA=m CONFIG_TEST_BITOPS=m CONFIG_TEST_VMALLOC=m CONFIG_TEST_USER_COPY=m -CONFIG_TEST_BPF=m CONFIG_TEST_BLACKHOLE_DEV=m CONFIG_FIND_BIT_BENCHMARK=m CONFIG_TEST_FIRMWARE=m diff --git a/arch/powerpc/net/bpf_jit_comp.c b/arch/powerpc/net/bpf_jit_comp.c index cfe01c3544db..e0b812d0c77e 100644 --- a/arch/powerpc/net/bpf_jit_comp.c +++ b/arch/powerpc/net/bpf_jit_comp.c @@ -267,8 +267,10 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_pr /* Make sure that the stack is quadword aligned. */ cgctx.stack_size = round_up(fp->aux->stack_depth, 16); - cgctx.arena_vm_start = bpf_arena_get_kern_vm_start(fp->aux->arena); - cgctx.user_vm_start = bpf_arena_get_user_vm_start(fp->aux->arena); + if (bpf_prog_uses_arena_insns(fp)) { + cgctx.arena_vm_start = bpf_arena_get_kern_vm_start(fp->aux->arena); + cgctx.user_vm_start = bpf_arena_get_user_vm_start(fp->aux->arena); + } cgctx.is_subprog = bpf_is_subprog(fp); cgctx.exception_boundary = fp->aux->exception_boundary; cgctx.exception_cb = fp->aux->exception_cb; diff --git a/arch/riscv/net/bpf_jit.h b/arch/riscv/net/bpf_jit.h index 419b9d795f2a..5c4f53060c91 100644 --- a/arch/riscv/net/bpf_jit.h +++ b/arch/riscv/net/bpf_jit.h @@ -12,6 +12,9 @@ #include #include +#define RV_MAX_REG_ARGS 8 +#define RV_EXTRA_STK_ARGS (RV_MAX_REG_ARGS - MAX_BPF_FUNC_REG_ARGS) + /* verify runtime detection extension status */ #define rv_ext_enabled(ext) \ (IS_ENABLED(CONFIG_RISCV_ISA_##ext) && riscv_has_extension_likely(RISCV_ISA_EXT_##ext)) @@ -82,6 +85,7 @@ struct rv_jit_context { unsigned long flags; int stack_size; int tcc_offset; + int stack_arg_sz; u64 arena_vm_start; u64 user_vm_start; }; diff --git a/arch/riscv/net/bpf_jit_comp64.c b/arch/riscv/net/bpf_jit_comp64.c index 01fe66774f02..788fcc21fd5b 100644 --- a/arch/riscv/net/bpf_jit_comp64.c +++ b/arch/riscv/net/bpf_jit_comp64.c @@ -15,7 +15,6 @@ #include #include "bpf_jit.h" -#define RV_MAX_REG_ARGS 8 #define RV_FENTRY_NINSNS 2 #define RV_FENTRY_NBYTES (RV_FENTRY_NINSNS * 4) /* imm that allows emit_imm to emit max count insns */ @@ -23,11 +22,11 @@ /* fentry and TCC init insns will be skipped on tailcall */ #define RV_TAILCALL_OFFSET ((RV_FENTRY_NINSNS + 1) * 4) -#define RV_REG_TCC RV_REG_A6 +#define RV_REG_TCC RV_REG_T5 #define RV_REG_ARENA RV_REG_S7 /* For storing arena_vm_start */ static const int regmap[] = { - [BPF_REG_0] = RV_REG_A5, + [BPF_REG_0] = RV_REG_T6, [BPF_REG_1] = RV_REG_A0, [BPF_REG_2] = RV_REG_A1, [BPF_REG_3] = RV_REG_A2, @@ -47,13 +46,13 @@ static const int pt_regmap[] = { [RV_REG_A2] = offsetof(struct pt_regs, a2), [RV_REG_A3] = offsetof(struct pt_regs, a3), [RV_REG_A4] = offsetof(struct pt_regs, a4), - [RV_REG_A5] = offsetof(struct pt_regs, a5), [RV_REG_S1] = offsetof(struct pt_regs, s1), [RV_REG_S2] = offsetof(struct pt_regs, s2), [RV_REG_S3] = offsetof(struct pt_regs, s3), [RV_REG_S4] = offsetof(struct pt_regs, s4), [RV_REG_S5] = offsetof(struct pt_regs, s5), [RV_REG_T0] = offsetof(struct pt_regs, t0), + [RV_REG_T6] = offsetof(struct pt_regs, t6), }; enum { @@ -239,7 +238,7 @@ static void __build_epilogue(bool is_tail_call, struct rv_jit_context *ctx) emit_addi(RV_REG_SP, RV_REG_SP, stack_adjust, ctx); /* Set return value. */ if (!is_tail_call) - emit_addiw(RV_REG_A0, RV_REG_A5, 0, ctx); + emit_addiw(RV_REG_A0, regmap[BPF_REG_0], 0, ctx); emit_jalr(RV_REG_ZERO, is_tail_call ? RV_REG_T3 : RV_REG_RA, is_tail_call ? RV_TAILCALL_OFFSET : 0, ctx); } @@ -498,6 +497,18 @@ static void emit_ldx(u8 rd, s16 off, u8 rs, u8 size, bool sign_ext, ctx->ex_jmp_off = ctx->ninsns; } +static void emit_stack_arg_ldx(u8 rd, s16 off, struct rv_jit_context *ctx) +{ + int idx = off / 8 - 1; + + if (idx < RV_EXTRA_STK_ARGS) { + emit_mv(rd, RV_REG_A5 + idx, ctx); + return; + } + + emit_ldx_insn(rd, (idx - RV_EXTRA_STK_ARGS) * 8, RV_REG_FP, BPF_DW, false, ctx); +} + static void emit_st(u8 rd, s16 off, s32 imm, u8 size, struct rv_jit_context *ctx) { emit_imm(RV_REG_T1, imm, ctx); @@ -515,6 +526,19 @@ static void emit_st(u8 rd, s16 off, s32 imm, u8 size, struct rv_jit_context *ctx ctx->ex_jmp_off = ctx->ninsns; } +static void emit_stack_arg_st(s16 off, s32 imm, struct rv_jit_context *ctx) +{ + int idx = -off / 8 - 1; + + if (idx < RV_EXTRA_STK_ARGS) { + emit_imm(RV_REG_A5 + idx, imm, ctx); + return; + } + + emit_imm(RV_REG_T1, imm, ctx); + emit_stx_insn(RV_REG_SP, (idx - RV_EXTRA_STK_ARGS) * 8, RV_REG_T1, BPF_DW, ctx); +} + static void emit_stx(u8 rd, s16 off, u8 rs, u8 size, struct rv_jit_context *ctx) { if (is_12b_int(off)) { @@ -531,6 +555,18 @@ static void emit_stx(u8 rd, s16 off, u8 rs, u8 size, struct rv_jit_context *ctx) ctx->ex_jmp_off = ctx->ninsns; } +static void emit_stack_arg_stx(s16 off, u8 rs, struct rv_jit_context *ctx) +{ + int idx = -off / 8 - 1; + + if (idx < RV_EXTRA_STK_ARGS) { + emit_mv(RV_REG_A5 + idx, rs, ctx); + return; + } + + emit_stx_insn(RV_REG_SP, (idx - RV_EXTRA_STK_ARGS) * 8, rs, BPF_DW, ctx); +} + static int emit_atomic_ld_st(u8 rd, u8 rs, const struct bpf_insn *insn, struct rv_jit_context *ctx) { @@ -747,6 +783,8 @@ static int add_exception_handler(const struct bpf_insn *insn, int dst_reg, if (BPF_MODE(insn->code) != BPF_PROBE_MEM && BPF_MODE(insn->code) != BPF_PROBE_MEMSX && BPF_MODE(insn->code) != BPF_PROBE_MEM32 && + !(BPF_MODE(insn->code) == BPF_PROBE_MEM32SX && + BPF_CLASS(insn->code) == BPF_LDX) && BPF_MODE(insn->code) != BPF_PROBE_ATOMIC) return 0; @@ -875,7 +913,8 @@ int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type old_t, return ret; } -static void store_args(int nr_arg_slots, int args_off, struct rv_jit_context *ctx) +static void store_args(int nr_arg_slots, int args_off, int stack_args_off, + struct rv_jit_context *ctx) { int i; @@ -883,8 +922,8 @@ static void store_args(int nr_arg_slots, int args_off, struct rv_jit_context *ct if (i < RV_MAX_REG_ARGS) { emit_sd(RV_REG_FP, -args_off, RV_REG_A0 + i, ctx); } else { - /* skip slots for T0 and FP of traced function */ - emit_ld(RV_REG_T1, 16 + (i - RV_MAX_REG_ARGS) * 8, RV_REG_FP, ctx); + emit_ld(RV_REG_T1, stack_args_off + + (i - RV_MAX_REG_ARGS) * 8, RV_REG_FP, ctx); emit_sd(RV_REG_FP, -args_off, RV_REG_T1, ctx); } args_off -= 8; @@ -1100,7 +1139,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, save_ret = flags & (BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_RET_FENTRY_RET); if (save_ret) - stack_size += 16; /* Save both A5 (BPF R0) and A0 */ + stack_size += 16; /* Save both RV_REG_T6 (BPF R0) and RV_REG_A0 */ retval_off = stack_size; stack_size += nr_arg_slots * 8; @@ -1179,7 +1218,12 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, func_meta = nr_arg_slots; emit_store_stack_imm64(RV_REG_T1, -func_meta_off, func_meta, ctx); - store_args(nr_arg_slots, args_off, ctx); + /* + * A direct struct_ops call has its first stack argument at the incoming + * SP, which the trampoline keeps as FP. The fentry path pushes the + * parent frame first, so its incoming stack arguments start at FP + 16. + */ + store_args(nr_arg_slots, args_off, is_struct_ops ? 0 : 16, ctx); if (bpf_fsession_cnt(tnodes)) { /* clear all session cookies' value */ @@ -1675,6 +1719,11 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, emit_zextw(rd, rd, ctx); break; + /* JUMP reg */ + case BPF_JMP | BPF_JA | BPF_X: + emit_jalr(RV_REG_ZERO, rd, 0, ctx); + break; + /* JUMP off */ case BPF_JMP | BPF_JA: case BPF_JMP32 | BPF_JA: @@ -1844,11 +1893,21 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, return -EINVAL; for (idx = 0; idx < fm->nr_args; idx++) { - u8 reg = bpf_to_rv_reg(BPF_REG_1 + idx, ctx); bool sign = fm->arg_flags[idx] & BTF_FMODEL_SIGNED_ARG; + u8 arg_sz = fm->arg_size[idx]; - if (sign_extend(reg, reg, fm->arg_size[idx], sign, ctx)) - return -EINVAL; + if (arg_sz == 8 || (arg_sz != 4 && !sign)) + continue; + + if (idx < RV_MAX_REG_ARGS) { + if (sign_extend(RV_REG_A0 + idx, RV_REG_A0 + idx, arg_sz, sign, ctx)) + return -EINVAL; + } else { + emit_ld(RV_REG_T1, (idx - RV_MAX_REG_ARGS) * 8, RV_REG_SP, ctx); + if (sign_extend(RV_REG_T1, RV_REG_T1, arg_sz, sign, ctx)) + return -EINVAL; + emit_sd(RV_REG_SP, (idx - RV_MAX_REG_ARGS) * 8, RV_REG_T1, ctx); + } } } @@ -1873,6 +1932,13 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, emit_mv(bpf_to_rv_reg(BPF_REG_0, ctx), RV_REG_A0, ctx); break; } + + /* indirect call to a bpf subprog via the address in rd */ + case BPF_JMP | BPF_CALL | BPF_X: + mark_call(ctx); + emit_jalr(RV_REG_RA, rd, 0, ctx); + break; + /* tail call */ case BPF_JMP | BPF_TAIL_CALL: if (emit_bpf_tail_call(i, ctx)) @@ -1930,18 +1996,27 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, case BPF_LDX | BPF_PROBE_MEM32 | BPF_H: case BPF_LDX | BPF_PROBE_MEM32 | BPF_W: case BPF_LDX | BPF_PROBE_MEM32 | BPF_DW: + /* LDX | PROBE_MEM32SX: dst = *(signed size *)(src + RV_REG_ARENA + off) */ + case BPF_LDX | BPF_PROBE_MEM32SX | BPF_B: + case BPF_LDX | BPF_PROBE_MEM32SX | BPF_H: + case BPF_LDX | BPF_PROBE_MEM32SX | BPF_W: { bool sign_ext; sign_ext = BPF_MODE(insn->code) == BPF_MEMSX || - BPF_MODE(insn->code) == BPF_PROBE_MEMSX; + BPF_MODE(insn->code) == BPF_PROBE_MEMSX || + BPF_MODE(insn->code) == BPF_PROBE_MEM32SX; - if (BPF_MODE(insn->code) == BPF_PROBE_MEM32) { + if (BPF_MODE(insn->code) == BPF_PROBE_MEM32 || + BPF_MODE(insn->code) == BPF_PROBE_MEM32SX) { emit_add(RV_REG_T2, rs, RV_REG_ARENA, ctx); rs = RV_REG_T2; } - emit_ldx(rd, off, rs, BPF_SIZE(code), sign_ext, ctx); + if (is_stack_arg_ldx(insn)) + emit_stack_arg_ldx(rd, off, ctx); + else + emit_ldx(rd, off, rs, BPF_SIZE(code), sign_ext, ctx); ret = add_exception_handler(insn, rd, ctx); if (ret) @@ -1971,7 +2046,10 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, rd = RV_REG_T3; } - emit_st(rd, off, imm, BPF_SIZE(code), ctx); + if (is_stack_arg_st(insn)) + emit_stack_arg_st(off, imm, ctx); + else + emit_st(rd, off, imm, BPF_SIZE(code), ctx); ret = add_exception_handler(insn, REG_DONT_CLEAR_MARKER, ctx); if (ret) @@ -1993,7 +2071,10 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, rd = RV_REG_T2; } - emit_stx(rd, off, rs, BPF_SIZE(code), ctx); + if (is_stack_arg_stx(insn)) + emit_stack_arg_stx(off, rs, ctx); + else + emit_stx(rd, off, rs, BPF_SIZE(code), ctx); ret = add_exception_handler(insn, REG_DONT_CLEAR_MARKER, ctx); if (ret) @@ -2043,9 +2124,9 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, void bpf_jit_build_prologue(struct rv_jit_context *ctx, bool is_subprog) { - int i, stack_adjust = 0, store_offset, bpf_stack_adjust; + int i, stack_adjust = 0, store_offset, bpf_stack_adjust = ctx->stack_arg_sz; - bpf_stack_adjust = round_up(ctx->prog->aux->stack_depth, STACK_ALIGN); + bpf_stack_adjust += round_up(ctx->prog->aux->stack_depth, STACK_ALIGN); if (bpf_stack_adjust) mark_fp(ctx); @@ -2143,6 +2224,11 @@ bool bpf_jit_supports_kfunc_call(void) return true; } +bool bpf_jit_supports_kfunc_ret_reg_pair(void) +{ + return true; +} + bool bpf_jit_supports_ptr_xchg(void) { return true; @@ -2170,10 +2256,6 @@ bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena) if (insn->imm == BPF_CMPXCHG) return rv_ext_enabled(ZACAS); break; - case BPF_LDX | BPF_MEMSX | BPF_B: - case BPF_LDX | BPF_MEMSX | BPF_H: - case BPF_LDX | BPF_MEMSX | BPF_W: - return false; } } @@ -2211,3 +2293,13 @@ bool bpf_jit_supports_timed_may_goto(void) { return true; } + +bool bpf_jit_supports_stack_args(void) +{ + return true; +} + +bool bpf_jit_supports_callx(void) +{ + return true; +} diff --git a/arch/riscv/net/bpf_jit_core.c b/arch/riscv/net/bpf_jit_core.c index cbfcd287ea16..f2497ec7d378 100644 --- a/arch/riscv/net/bpf_jit_core.c +++ b/arch/riscv/net/bpf_jit_core.c @@ -26,10 +26,13 @@ static int build_body(struct rv_jit_context *ctx, bool extra_pass, int *offset) int ret; ret = bpf_jit_emit_insn(insn, ctx, extra_pass); - if (ret > 0) - i++; /* skip the next instruction */ if (offset) offset[i] = ctx->ninsns; + if (ret > 0) { + i++; /* skip the next instruction */ + if (offset) + offset[i] = ctx->ninsns; + } if (ret < 0) return ret; } @@ -48,6 +51,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_pr int pass = 0, prev_ninsns = 0, i; struct rv_jit_data *jit_data; struct rv_jit_context *ctx; + u16 stack_arg_cnt; if (!prog->jit_requested) return prog; @@ -69,8 +73,16 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_pr goto skip_init_ctx; } - ctx->arena_vm_start = bpf_arena_get_kern_vm_start(prog->aux->arena); - ctx->user_vm_start = bpf_arena_get_user_vm_start(prog->aux->arena); + if (bpf_prog_uses_arena_insns(prog)) { + ctx->arena_vm_start = bpf_arena_get_kern_vm_start(prog->aux->arena); + ctx->user_vm_start = bpf_arena_get_user_vm_start(prog->aux->arena); + } + + stack_arg_cnt = bpf_out_stack_arg_cnt(env, prog); + /* First 3 stack args in regs, rest on stack */ + ctx->stack_arg_sz = stack_arg_cnt > RV_EXTRA_STK_ARGS ? + round_up((stack_arg_cnt - RV_EXTRA_STK_ARGS) * 8, STACK_ALIGN) : 0; + ctx->prog = prog; ctx->offset = kvzalloc_objs(int, prog->len); if (!ctx->offset) @@ -169,6 +181,15 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_pr for (i = 0; i < prog->len; i++) ctx->offset[i] = ninsns_rvoff(ctx->offset[i]); bpf_prog_fill_jited_linfo(prog, ctx->offset); + + /* + * bpf_prog_update_insn_ptrs() wants the start of each insn, so + * shift ctx->offset[] by one and get insn 0 from the prologue. + */ + for (i = prog->len - 1; i > 0; i--) + ctx->offset[i] = ctx->offset[i - 1]; + ctx->offset[0] = ninsns_rvoff(ctx->prologue_len); + bpf_prog_update_insn_ptrs(prog, ctx->offset, jit_data->ro_image); out_offset: kvfree(ctx->offset); kfree(jit_data); @@ -237,6 +258,7 @@ void bpf_jit_free(struct bpf_prog *prog) kvfree(jit_data->ctx.offset); kfree(jit_data); } + prog->bpf_func = (void *)prog->bpf_func - cfi_get_offset(); hdr = bpf_jit_binary_pack_hdr(prog); bpf_jit_binary_pack_free(hdr, NULL); WARN_ON_ONCE(!bpf_prog_kallsyms_verify_off(prog)); diff --git a/arch/riscv/net/bpf_timed_may_goto.S b/arch/riscv/net/bpf_timed_may_goto.S index 02c637d87420..fcf850f244c9 100644 --- a/arch/riscv/net/bpf_timed_may_goto.S +++ b/arch/riscv/net/bpf_timed_may_goto.S @@ -10,7 +10,7 @@ * - output: updated count in BPF_REG_AX (t0) * * Calls bpf_check_timed_may_goto(ptr) with the standard RISC-V ABI, where - * ptr = BPF_REG_FP (s5) + BPF_REG_AX (t0). BPF R0-R5 (a5, a0-a4) are saved + * ptr = BPF_REG_FP (s5) + BPF_REG_AX (t0). BPF R0-R5 (t6, a0-a4) are saved * across the call; BPF_REG_FP (s5) is callee-saved and needs no saving. */ @@ -20,8 +20,8 @@ SYM_FUNC_START(arch_bpf_timed_may_goto) REG_S s0, 6*SZREG(sp) addi s0, sp, 8*SZREG - /* Save BPF registers R0-R5 (a5, a0-a4) */ - REG_S a5, 5*SZREG(sp) + /* Save BPF registers R0-R5 (t6, a0-a4) */ + REG_S t6, 5*SZREG(sp) REG_S a0, 4*SZREG(sp) REG_S a1, 3*SZREG(sp) REG_S a2, 2*SZREG(sp) @@ -38,7 +38,7 @@ SYM_FUNC_START(arch_bpf_timed_may_goto) REG_L a2, 2*SZREG(sp) REG_L a1, 3*SZREG(sp) REG_L a0, 4*SZREG(sp) - REG_L a5, 5*SZREG(sp) + REG_L t6, 5*SZREG(sp) REG_L s0, 6*SZREG(sp) REG_L ra, 7*SZREG(sp) diff --git a/arch/s390/configs/debug_defconfig b/arch/s390/configs/debug_defconfig index 3892d9dd75d9..ac30bf5d0b31 100644 --- a/arch/s390/configs/debug_defconfig +++ b/arch/s390/configs/debug_defconfig @@ -941,6 +941,5 @@ CONFIG_INTERVAL_TREE_TEST=m CONFIG_PERCPU_TEST=m CONFIG_ATOMIC64_SELFTEST=y CONFIG_TEST_BITOPS=m -CONFIG_TEST_BPF=m CONFIG_PAGE_TABLE_CHECK=y CONFIG_PAGE_TABLE_CHECK_ENFORCED=y diff --git a/arch/s390/configs/defconfig b/arch/s390/configs/defconfig index 95720e8ad80d..a02d8bf57297 100644 --- a/arch/s390/configs/defconfig +++ b/arch/s390/configs/defconfig @@ -865,4 +865,3 @@ CONFIG_LKDTM=m CONFIG_KPROBES_SANITY_TEST=m CONFIG_PERCPU_TEST=m CONFIG_ATOMIC64_SELFTEST=y -CONFIG_TEST_BPF=m diff --git a/arch/s390/net/bpf_jit_comp.c b/arch/s390/net/bpf_jit_comp.c index 51db92c3329d..56ec49894dda 100644 --- a/arch/s390/net/bpf_jit_comp.c +++ b/arch/s390/net/bpf_jit_comp.c @@ -828,6 +828,14 @@ static int bpf_jit_probe_post(struct bpf_jit *jit, struct bpf_prog *fp, return 0; } +static void emit_shift_count32(struct bpf_jit *jit, u32 src_reg) +{ + /* risbg %w1,%src,59,128+63,0 */ + _EMIT6(0xec000000 | reg(REG_W1, src_reg) << 16 | 59 << 8 | (0x80 | 63), 0x0055); + REG_SET_SEEN(REG_W1); + REG_SET_SEEN(src_reg); +} + static int emit_ldx(struct bpf_jit *jit, struct bpf_prog *fp, struct bpf_insn *insn) { struct bpf_jit_probe probe; @@ -1400,9 +1408,10 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, /* * BPF_LSH */ - case BPF_ALU | BPF_LSH | BPF_X: /* dst = (u32) dst << (u32) src */ - /* sll %dst,0(%src) */ - EMIT4_DISP(0x89000000, dst_reg, src_reg, 0); + case BPF_ALU | BPF_LSH | BPF_X: /* dst = (u32) dst << ((u32) src & 31) */ + emit_shift_count32(jit, src_reg); + /* sll %dst,0(%w1) */ + EMIT4_DISP(0x89000000, dst_reg, REG_W1, 0); EMIT_ZERO(dst_reg); break; case BPF_ALU64 | BPF_LSH | BPF_X: /* dst = dst << src */ @@ -1425,9 +1434,10 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, /* * BPF_RSH */ - case BPF_ALU | BPF_RSH | BPF_X: /* dst = (u32) dst >> (u32) src */ - /* srl %dst,0(%src) */ - EMIT4_DISP(0x88000000, dst_reg, src_reg, 0); + case BPF_ALU | BPF_RSH | BPF_X: /* dst = (u32) dst >> ((u32) src & 31) */ + emit_shift_count32(jit, src_reg); + /* srl %dst,0(%w1) */ + EMIT4_DISP(0x88000000, dst_reg, REG_W1, 0); EMIT_ZERO(dst_reg); break; case BPF_ALU64 | BPF_RSH | BPF_X: /* dst = dst >> src */ @@ -1450,9 +1460,10 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, /* * BPF_ARSH */ - case BPF_ALU | BPF_ARSH | BPF_X: /* ((s32) dst) >>= src */ - /* sra %dst,%dst,0(%src) */ - EMIT4_DISP(0x8a000000, dst_reg, src_reg, 0); + case BPF_ALU | BPF_ARSH | BPF_X: /* ((s32) dst) >>= (u32) src & 31 */ + emit_shift_count32(jit, src_reg); + /* sra %dst,0(%w1) */ + EMIT4_DISP(0x8a000000, dst_reg, REG_W1, 0); EMIT_ZERO(dst_reg); break; case BPF_ALU64 | BPF_ARSH | BPF_X: /* ((s64) dst) >>= src */ @@ -2260,10 +2271,11 @@ static int bpf_jit_prog(struct bpf_jit *jit, struct bpf_prog *fp, else jit->frame_off = 0; - kern_arena = bpf_arena_get_kern_vm_start(fp->aux->arena); - if (kern_arena) + if (bpf_prog_uses_arena_insns(fp)) { + kern_arena = bpf_arena_get_kern_vm_start(fp->aux->arena); jit->kern_arena = _EMIT_CONST_U64(kern_arena); - jit->user_arena = bpf_arena_get_user_vm_start(fp->aux->arena); + jit->user_arena = bpf_arena_get_user_vm_start(fp->aux->arena); + } bpf_jit_prologue(jit, fp); if (bpf_set_addr(jit, 0) < 0) diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index a02d953628a8..a5e29ffd5ddc 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -234,6 +234,7 @@ config X86 select HAVE_SAMPLE_FTRACE_DIRECT if X86_64 select HAVE_SAMPLE_FTRACE_DIRECT_MULTI if X86_64 select HAVE_EBPF_JIT + select HAVE_EBPF_JIT_KASAN if X86_64 select HAVE_EFFICIENT_UNALIGNED_ACCESS select HAVE_EISA if X86_32 select HAVE_EXIT_THREAD diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index 6bca87457e87..6a2fe3d0039d 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -9,11 +9,13 @@ #include #include #include +#include #include #include #include #include #include +#include #include #include #include @@ -22,8 +24,34 @@ #include #include +#if IS_ENABLED(CONFIG_BPF_JIT_KASAN) +void __asan_load1(void *p); +void __asan_store1(void *p); +void __asan_load2(void *p); +void __asan_store2(void *p); +void __asan_load4(void *p); +void __asan_store4(void *p); +void __asan_load8(void *p); +void __asan_store8(void *p); +#endif + static bool all_callee_regs_used[4] = {true, true, true, true}; +static void *trampoline_size_image; + +static int __init init_trampoline_size_image(void) +{ + /* + * The generated trampoline contains calls and jumps with 32bit relative + * offsets, so the scratch image must be in the execmem range. + * On x86, module data and executable memory share the same address range, + * so using EXECMEM_MODULE_DATA to get writable memory. + */ + trampoline_size_image = execmem_alloc(EXECMEM_MODULE_DATA, PAGE_SIZE); + return trampoline_size_image ? 0 : -ENOMEM; +} +late_initcall(init_trampoline_size_image); + static u8 *emit_code(u8 *ptr, u32 bytes, unsigned int len) { if (len == 1) @@ -209,8 +237,16 @@ static const int reg2pt_regs[] = { [BPF_REG_7] = offsetof(struct pt_regs, r13), [BPF_REG_8] = offsetof(struct pt_regs, r14), [BPF_REG_9] = offsetof(struct pt_regs, r15), + [BPF_REG_AX] = offsetof(struct pt_regs, r10), }; +static bool is_atomic_fetch_op(const struct bpf_insn *insn) +{ + return insn->imm == (BPF_AND | BPF_FETCH) || + insn->imm == (BPF_OR | BPF_FETCH) || + insn->imm == (BPF_XOR | BPF_FETCH); +} + /* * is_ereg() == true if BPF register 'reg' maps to x86-64 r8..r15 * which need extra byte of encoding. @@ -393,7 +429,7 @@ static void pop_callee_regs(u8 **pprog, bool *callee_regs_used) } /* add rsp, depth */ -static void emit_add_rsp(u8 **pprog, u16 depth) +static void emit_add_rsp(u8 **pprog, u32 depth) { u8 *prog = *pprog; @@ -407,7 +443,7 @@ static void emit_add_rsp(u8 **pprog, u16 depth) } /* sub rsp, depth */ -static void emit_sub_rsp(u8 **pprog, u16 depth) +static void emit_sub_rsp(u8 **pprog, u32 depth) { u8 *prog = *pprog; @@ -444,6 +480,9 @@ static void emit_nops(u8 **pprog, int len) * in arch/x86/kernel/alternative.c */ static int emit_call(u8 **prog, void *func, void *ip); +static void emit_ldx(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, int off); +static void emit_mov_reg(u8 **pprog, bool is64, u32 dst_reg, u32 src_reg); +static void emit_alu_imm(u8 **pprog, u32 op, u32 dst_reg, s32 imm, bool is64); static void emit_fineibt(u8 **pprog, u8 *ip, u32 hash, int arity) { @@ -579,8 +618,7 @@ static void emit_prologue(u8 **pprog, u8 *ip, u32 stack_depth, bool ebpf_from_cb EMIT_ENDBR(); /* sub rsp, rounded_stack_depth */ - if (stack_depth) - EMIT3_off32(0x48, 0x81, 0xEC, round_up(stack_depth, 8)); + emit_sub_rsp(&prog, round_up(stack_depth, 8)); if (tail_call_reachable) emit_prologue_tail_call(&prog, is_subprog); *pprog = prog; @@ -722,6 +760,46 @@ static void emit_indirect_jump(u8 **pprog, int bpf_reg, u8 *ip) *pprog = prog; } +static void __emit_indirect_call(u8 **pprog, int reg, bool ereg) +{ + u8 *prog = *pprog; + + if (ereg) + EMIT1(0x41); + + EMIT2(0xFF, 0xD0 + reg); + + *pprog = prog; +} + +/* call *bpf_reg */ +static int emit_indirect_call(u8 **pprog, int bpf_reg, u8 *ip) +{ + u8 *prog = *pprog; + int reg = reg2hex[bpf_reg]; + bool ereg = is_ereg(bpf_reg); + int err = 0; + + if (cpu_feature_enabled(X86_FEATURE_INDIRECT_THUNK_ITS)) { + OPTIMIZER_HIDE_VAR(reg); + err = emit_call(&prog, its_static_thunk(reg + 8*ereg), ip); + } else if (cpu_feature_enabled(X86_FEATURE_RETPOLINE_LFENCE)) { + EMIT_LFENCE(); + __emit_indirect_call(&prog, reg, ereg); + } else if (cpu_feature_enabled(X86_FEATURE_RETPOLINE)) { + OPTIMIZER_HIDE_VAR(reg); + if (cpu_feature_enabled(X86_FEATURE_CALL_DEPTH)) + err = emit_call(&prog, &__x86_indirect_call_thunk_array[reg + 8*ereg], ip); + else + err = emit_call(&prog, &__x86_indirect_thunk_array[reg + 8*ereg], ip); + } else { + __emit_indirect_call(&prog, reg, ereg); + } + + *pprog = prog; + return err; +} + static void emit_return(u8 **pprog, u8 *ip) { u8 *prog = *pprog; @@ -772,7 +850,7 @@ static void emit_bpf_tail_call_indirect(struct bpf_prog *bpf_prog, * if (index >= array->map.max_entries) * goto out; */ - EMIT2(0x89, 0xD2); /* mov edx, edx */ + emit_mov_reg(&prog, false, BPF_REG_3, BPF_REG_3); /* mov edx, edx */ EMIT3(0x39, 0x56, /* cmp dword ptr [rsi + 16], edx */ offsetof(struct bpf_array, map.max_entries)); @@ -783,7 +861,8 @@ static void emit_bpf_tail_call_indirect(struct bpf_prog *bpf_prog, * if ((*tcc_ptr)++ >= MAX_TAIL_CALL_CNT) * goto out; */ - EMIT3_off32(0x48, 0x8B, 0x85, tcc_ptr_off); /* mov rax, qword ptr [rbp - tcc_ptr_off] */ + /* mov rax, qword ptr [rbp - tcc_ptr_off] */ + emit_ldx(&prog, BPF_DW, BPF_REG_0, BPF_REG_FP, tcc_ptr_off); EMIT4(0x48, 0x83, 0x38, MAX_TAIL_CALL_CNT); /* cmp qword ptr [rax], MAX_TAIL_CALL_CNT */ offset = ctx->tail_call_indirect_label - (prog + 2 - start); @@ -810,7 +889,7 @@ static void emit_bpf_tail_call_indirect(struct bpf_prog *bpf_prog, pop_r12(&prog); } else { pop_callee_regs(&prog, callee_regs_used); - if (bpf_arena_get_kern_vm_start(bpf_prog->aux->arena)) + if (bpf_prog_uses_arena_insns(bpf_prog)) pop_r12(&prog); } @@ -820,15 +899,13 @@ static void emit_bpf_tail_call_indirect(struct bpf_prog *bpf_prog, * Pop tail_call_cnt_ptr, if it's subprog. */ EMIT1(0x58); /* pop rax */ - if (stack_depth) - EMIT3_off32(0x48, 0x81, 0xC4, /* add rsp, sd */ - round_up(stack_depth, 8)); + emit_add_rsp(&prog, round_up(stack_depth, 8)); /* add rsp, sd */ /* goto *(prog->bpf_func + X86_TAIL_CALL_OFFSET); */ EMIT4(0x48, 0x8B, 0x49, /* mov rcx, qword ptr [rcx + 32] */ offsetof(struct bpf_prog, bpf_func)); - EMIT4(0x48, 0x83, 0xC1, /* add rcx, X86_TAIL_CALL_OFFSET */ - X86_TAIL_CALL_OFFSET); + /* add rcx, X86_TAIL_CALL_OFFSET */ + emit_alu_imm(&prog, BPF_ADD, BPF_REG_4, X86_TAIL_CALL_OFFSET, true); /* * Now we're ready to jump into next BPF program * rdi == ctx (1st arg) @@ -855,7 +932,8 @@ static void emit_bpf_tail_call_direct(struct bpf_prog *bpf_prog, * if ((*tcc_ptr)++ >= MAX_TAIL_CALL_CNT) * goto out; */ - EMIT3_off32(0x48, 0x8B, 0x85, tcc_ptr_off); /* mov rax, qword ptr [rbp - tcc_ptr_off] */ + /* mov rax, qword ptr [rbp - tcc_ptr_off] */ + emit_ldx(&prog, BPF_DW, BPF_REG_0, BPF_REG_FP, tcc_ptr_off); EMIT4(0x48, 0x83, 0x38, MAX_TAIL_CALL_CNT); /* cmp qword ptr [rax], MAX_TAIL_CALL_CNT */ offset = ctx->tail_call_direct_label - (prog + 2 - start); @@ -877,7 +955,7 @@ static void emit_bpf_tail_call_direct(struct bpf_prog *bpf_prog, pop_r12(&prog); } else { pop_callee_regs(&prog, callee_regs_used); - if (bpf_arena_get_kern_vm_start(bpf_prog->aux->arena)) + if (bpf_prog_uses_arena_insns(bpf_prog)) pop_r12(&prog); } @@ -887,8 +965,7 @@ static void emit_bpf_tail_call_direct(struct bpf_prog *bpf_prog, * Pop tail_call_cnt_ptr, if it's subprog. */ EMIT1(0x58); /* pop rax */ - if (stack_depth) - EMIT3_off32(0x48, 0x81, 0xC4, round_up(stack_depth, 8)); + emit_add_rsp(&prog, round_up(stack_depth, 8)); emit_nops(&prog, X86_PATCH_SIZE); @@ -1038,8 +1115,9 @@ static void emit_movsx_reg(u8 **pprog, int num_bits, bool is64, u32 dst_reg, } else { /* movs[b,w]l dst, src */ if (num_bits == 8) { - EMIT4(add_2mod(0x40, src_reg, dst_reg), 0x0f, 0xbe, - add_2reg(0xC0, src_reg, dst_reg)); + if (is_ereg(dst_reg) || is_ereg_8l(src_reg)) + EMIT1(add_2mod(0x40, src_reg, dst_reg)); + EMIT3(0x0f, 0xbe, add_2reg(0xC0, src_reg, dst_reg)); } else if (num_bits == 16) { if (is_ereg(dst_reg) || is_ereg(src_reg)) EMIT1(add_2mod(0x40, src_reg, dst_reg)); @@ -1051,18 +1129,40 @@ static void emit_movsx_reg(u8 **pprog, int num_bits, bool is64, u32 dst_reg, *pprog = prog; } +/* + * rbp and r13 as base register always need a displacement, since mod 00 + * with r/m 101 encodes a RIP-relative disp32 and with SIB base 101 a + * disp32 without base instead. + */ +static bool base_needs_disp(u32 reg) +{ + return reg2hex[reg] == 5; +} + +static void emit_movzx_reg(u8 **pprog, int num_bits, u32 dst_reg, u32 src_reg) +{ + u8 *prog = *pprog; + + /* movz[b,w]l dst, src */ + if (is_ereg(dst_reg) || + (num_bits == 8 ? is_ereg_8l(src_reg) : is_ereg(src_reg))) + EMIT1(add_2mod(0x40, src_reg, dst_reg)); + EMIT3(0x0F, num_bits == 8 ? 0xB6 : 0xB7, + add_2reg(0xC0, src_reg, dst_reg)); + + *pprog = prog; +} + /* Emit the suffix (ModR/M etc) for addressing *(ptr_reg + off) and val_reg */ static void emit_insn_suffix(u8 **pprog, u32 ptr_reg, u32 val_reg, int off) { u8 *prog = *pprog; - if (is_imm8(off)) { - /* 1-byte signed displacement. - * - * If off == 0 we could skip this and save one extra byte, but - * special case of x86 R13 which always needs an offset is not - * worth the hassle - */ + if (!off && !base_needs_disp(ptr_reg)) { + /* No displacement */ + EMIT1(add_2reg(0x00, ptr_reg, val_reg)); + } else if (is_imm8(off)) { + /* 1-byte signed displacement */ EMIT2(add_2reg(0x40, ptr_reg, val_reg), off); } else { /* 4-byte signed displacement */ @@ -1071,14 +1171,22 @@ static void emit_insn_suffix(u8 **pprog, u32 ptr_reg, u32 val_reg, int off) *pprog = prog; } -static void emit_insn_suffix_SIB(u8 **pprog, u32 ptr_reg, u32 val_reg, u32 index_reg, int off) +/* + * Emit the suffix for addressing *(ptr_reg + (index_reg << scale) + off) + * and val_reg, with a SIB byte + */ +static void emit_insn_suffix_SIB(u8 **pprog, u32 ptr_reg, u32 val_reg, u32 index_reg, int off, + u8 scale) { + u8 sib = (scale << 6) | add_2reg(0, ptr_reg, index_reg); u8 *prog = *pprog; - if (is_imm8(off)) { - EMIT3(add_2reg(0x44, BPF_REG_0, val_reg), add_2reg(0, ptr_reg, index_reg) /* SIB */, off); + if (!off && !base_needs_disp(ptr_reg)) { + EMIT2(add_2reg(0x04, BPF_REG_0, val_reg), sib); + } else if (is_imm8(off)) { + EMIT3(add_2reg(0x44, BPF_REG_0, val_reg), sib, off); } else { - EMIT2_off32(add_2reg(0x84, BPF_REG_0, val_reg), add_2reg(0, ptr_reg, index_reg) /* SIB */, off); + EMIT2_off32(add_2reg(0x84, BPF_REG_0, val_reg), sib, off); } *pprog = prog; } @@ -1111,6 +1219,165 @@ static void maybe_emit_1mod(u8 **pprog, u32 reg, bool is64) *pprog = prog; } +/* add/sub/and/or/xor dst_reg, imm with the imm8 and the rax short forms */ +static void emit_alu_imm(u8 **pprog, u32 op, u32 dst_reg, s32 imm, bool is64) +{ + /* ModR/M opcode extension and rax short form per operation */ + static const u8 modrm[] = { + [BPF_ADD] = 0xC0, [BPF_SUB] = 0xE8, [BPF_AND] = 0xE0, + [BPF_OR] = 0xC8, [BPF_XOR] = 0xF0, + }; + static const u8 short_op[] = { + [BPF_ADD] = 0x05, [BPF_SUB] = 0x2D, [BPF_AND] = 0x25, + [BPF_OR] = 0x0D, [BPF_XOR] = 0x35, + }; + u8 *prog = *pprog; + bool inc; + + /* A non-negative mask clears the upper half like the 32-bit and does */ + if (op == BPF_AND && imm >= 0) + is64 = false; + /* add 128 is sub -128 and the other way round, which fit into an imm8 */ + if ((op == BPF_ADD || op == BPF_SUB) && imm == 128) { + op = op == BPF_ADD ? BPF_SUB : BPF_ADD; + imm = -128; + } + inc = op == BPF_ADD ? imm == 1 : imm == -1; + maybe_emit_1mod(&prog, dst_reg, is64); + /* inc or dec for +-1, the flags are not read across instructions */ + if ((op == BPF_ADD || op == BPF_SUB) && (imm == 1 || imm == -1)) + EMIT2(0xFF, add_1reg(inc ? 0xC0 : 0xC8, dst_reg)); + else if (is_imm8(imm)) + EMIT3(0x83, add_1reg(modrm[op], dst_reg), imm); + else if (is_axreg(dst_reg)) + EMIT1_off32(short_op[op], imm); + else + EMIT2_off32(0x81, add_1reg(modrm[op], dst_reg), imm); + + *pprog = prog; +} + +/* add/sub/and/or/xor dst_reg, src_reg */ +static void emit_alu_reg(u8 **pprog, u32 op, u32 dst_reg, u32 src_reg, bool is64) +{ + u8 *prog = *pprog; + + maybe_emit_mod(&prog, dst_reg, src_reg, is64); + EMIT2(simple_alu_opcodes[op], add_2reg(0xC0, dst_reg, src_reg)); + + *pprog = prog; +} + +/* shl/shr/sar reg, imm */ +static void emit_shift_imm(u8 **pprog, u32 op, u32 reg, u8 imm, bool is64) +{ + u8 *prog = *pprog; + + /* The shift left by one is an add of the register to itself */ + if (op == BPF_LSH && imm == 1) { + emit_alu_reg(&prog, BPF_ADD, reg, reg, is64); + } else { + maybe_emit_1mod(&prog, reg, is64); + if (imm == 1) + EMIT2(0xD1, add_1reg(simple_alu_opcodes[op], reg)); + else + EMIT3(0xC1, add_1reg(simple_alu_opcodes[op], reg), imm); + } + + *pprog = prog; +} + +/* neg dst_reg */ +static void emit_neg(u8 **pprog, u32 dst_reg, bool is64) +{ + u8 *prog = *pprog; + + maybe_emit_1mod(&prog, dst_reg, is64); + EMIT2(0xF7, add_1reg(0xD8, dst_reg)); + + *pprog = prog; +} + +static int emit_kasan_check(struct bpf_verifier_env *env, u8 **pprog, + u32 addr_reg, s32 off, u32 bpf_size, u8 *ip, + bool is_write) +{ +#ifdef CONFIG_BPF_JIT_KASAN + u8 *prog = *pprog; + void *kasan_func; + + if (!env) + return 0; + + if (addr_reg == BPF_REG_FP || addr_reg == BPF_REG_PARAMS) + return 0; + + /* Derive KASAN check function from access type and size */ + switch (bpf_size) { + case BPF_B: + kasan_func = is_write ? __asan_store1 : __asan_load1; + break; + case BPF_H: + kasan_func = is_write ? __asan_store2 : __asan_load2; + break; + case BPF_W: + kasan_func = is_write ? __asan_store4 : __asan_load4; + break; + case BPF_DW: + kasan_func = is_write ? __asan_store8 : __asan_load8; + break; + default: + return -EINVAL; + } + + /* Save rax */ + EMIT1(0x50); + /* Save rcx */ + EMIT1(0x51); + /* Save rdx */ + EMIT1(0x52); + /* Save rsi */ + EMIT1(0x56); + /* Save rdi */ + EMIT1(0x57); + /* Save r8 */ + EMIT2(0x41, 0x50); + /* Save r9 */ + EMIT2(0x41, 0x51); + /* Save r10 */ + EMIT2(0x41, 0x52); + /* Save r11 */ + EMIT2(0x41, 0x53); + /* mov rdi, addr_reg */ + EMIT_mov(BPF_REG_1, addr_reg); + + /* add rdi, off (if offset is non-zero) */ + if (off) + emit_alu_imm(&prog, BPF_ADD, BPF_REG_1, off, true); + + /* Adjust ip to account for the instrumentation generated so far */ + ip += (prog - *pprog); + /* We emit a call, so update call depth counting */ + ip += x86_call_depth_emit_accounting(&prog, kasan_func, ip); + /* call kasan_func */ + if (emit_call(&prog, kasan_func, ip)) + return -ERANGE; + + EMIT2(0x41, 0x5B); + EMIT2(0x41, 0x5A); + EMIT2(0x41, 0x59); + EMIT2(0x41, 0x58); + EMIT1(0x5F); + EMIT1(0x5E); + EMIT1(0x5A); + EMIT1(0x59); + EMIT1(0x58); + + *pprog = prog; +#endif /* CONFIG_BPF_JIT_KASAN */ + return 0; +} + /* LDX: dst_reg = *(u8*)(src_reg + off) */ static void emit_ldx(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, int off) { @@ -1118,12 +1385,16 @@ static void emit_ldx(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, int off) switch (size) { case BPF_B: - /* Emit 'movzx rax, byte ptr [rax + off]' */ - EMIT3(add_2mod(0x48, src_reg, dst_reg), 0x0F, 0xB6); + /* Emit 'movzx eax, byte ptr [rax + off]' */ + if (is_ereg(dst_reg) || is_ereg(src_reg)) + EMIT1(add_2mod(0x40, src_reg, dst_reg)); + EMIT2(0x0F, 0xB6); break; case BPF_H: - /* Emit 'movzx rax, word ptr [rax + off]' */ - EMIT3(add_2mod(0x48, src_reg, dst_reg), 0x0F, 0xB7); + /* Emit 'movzx eax, word ptr [rax + off]' */ + if (is_ereg(dst_reg) || is_ereg(src_reg)) + EMIT1(add_2mod(0x40, src_reg, dst_reg)); + EMIT2(0x0F, 0xB7); break; case BPF_W: /* Emit 'mov eax, dword ptr [rax+0x14]' */ @@ -1186,7 +1457,7 @@ static void emit_ldx_index(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, u32 i EMIT2(add_3mod(0x48, src_reg, dst_reg, index_reg), 0x8B); break; } - emit_insn_suffix_SIB(&prog, src_reg, dst_reg, index_reg, off); + emit_insn_suffix_SIB(&prog, src_reg, dst_reg, index_reg, off, 0); *pprog = prog; } @@ -1208,7 +1479,7 @@ static void emit_ldsx_index(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, u32 EMIT2(add_3mod(0x48, src_reg, dst_reg, index_reg), 0x63); break; } - emit_insn_suffix_SIB(&prog, src_reg, dst_reg, index_reg, off); + emit_insn_suffix_SIB(&prog, src_reg, dst_reg, index_reg, off, 0); *pprog = prog; } @@ -1279,7 +1550,7 @@ static void emit_stx_index(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, u32 i EMIT2(add_3mod(0x48, dst_reg, src_reg, index_reg), 0x89); break; } - emit_insn_suffix_SIB(&prog, dst_reg, src_reg, index_reg, off); + emit_insn_suffix_SIB(&prog, dst_reg, src_reg, index_reg, off, 0); *pprog = prog; } @@ -1311,11 +1582,47 @@ static void emit_st_index(u8 **pprog, u32 size, u32 dst_reg, u32 index_reg, int EMIT2(add_3mod(0x48, dst_reg, 0, index_reg), 0xC7); break; } - emit_insn_suffix_SIB(&prog, dst_reg, 0, index_reg, off); + emit_insn_suffix_SIB(&prog, dst_reg, 0, index_reg, off, 0); EMIT(imm, bpf_size_to_x86_bytes(size)); *pprog = prog; } +/* ST: *(u8*)(dst_reg + off) = imm */ +static void emit_st(u8 **pprog, struct bpf_insn *insn, u32 dst_reg, + s32 insn_off) +{ + s32 imm32 = insn->imm; + u8 *prog = *pprog; + + switch (BPF_SIZE(insn->code)) { + case BPF_B: + if (is_ereg(dst_reg)) + EMIT2(0x41, 0xC6); + else + EMIT1(0xC6); + break; + case BPF_H: + if (is_ereg(dst_reg)) + EMIT3(0x66, 0x41, 0xC7); + else + EMIT2(0x66, 0xC7); + break; + case BPF_W: + if (is_ereg(dst_reg)) + EMIT2(0x41, 0xC7); + else + EMIT1(0xC7); + break; + case BPF_DW: + EMIT2(add_1mod(0x48, dst_reg), 0xC7); + break; + } + + emit_insn_suffix(&prog, dst_reg, BPF_REG_0, insn_off); + EMIT(imm32, bpf_size_to_x86_bytes(BPF_SIZE(insn->code))); + *pprog = prog; +} + static void emit_st_r12(u8 **pprog, u32 size, u32 dst_reg, int off, int imm) { emit_st_index(pprog, size, dst_reg, X86_REG_R12, off, imm); @@ -1419,22 +1726,40 @@ static int emit_atomic_rmw_index(u8 **pprog, u32 atomic_op, u32 size, pr_err("bpf_jit: unknown atomic opcode %02x\n", atomic_op); return -EFAULT; } - emit_insn_suffix_SIB(&prog, dst_reg, src_reg, index_reg, off); + emit_insn_suffix_SIB(&prog, dst_reg, src_reg, index_reg, off, 0); *pprog = prog; return 0; } -static int emit_atomic_ld_st(u8 **pprog, u32 atomic_op, u32 dst_reg, - u32 src_reg, s16 off, u8 bpf_size) +static int emit_atomic_ld_st(struct bpf_verifier_env *env, u8 **pprog, + struct bpf_insn *insn, u8 *ip, u32 dst_reg, + u32 src_reg, bool accesses_stack_only) { + u32 atomic_op = insn->imm; + int err; + switch (atomic_op) { case BPF_LOAD_ACQ: + if (!accesses_stack_only) { + err = emit_kasan_check(env, pprog, src_reg, insn->off, + BPF_SIZE(insn->code), ip, false); + if (err) + return err; + } /* dst_reg = smp_load_acquire(src_reg + off16) */ - emit_ldx(pprog, bpf_size, dst_reg, src_reg, off); + emit_ldx(pprog, BPF_SIZE(insn->code), dst_reg, src_reg, + insn->off); break; case BPF_STORE_REL: + if (!accesses_stack_only) { + err = emit_kasan_check(env, pprog, dst_reg, insn->off, + BPF_SIZE(insn->code), ip, true); + if (err) + return err; + } /* smp_store_release(dst_reg + off16, src_reg) */ - emit_stx(pprog, bpf_size, dst_reg, src_reg, off); + emit_stx(pprog, BPF_SIZE(insn->code), dst_reg, src_reg, + insn->off); break; default: pr_err("bpf_jit: unknown atomic load/store opcode %02x\n", @@ -1467,6 +1792,93 @@ static int emit_atomic_ld_st_index(u8 **pprog, u32 atomic_op, u32 size, return 0; } +/* + * A fetching AND/OR/XOR can't be implemented with a single x86 insn, so do a + * CMPXCHG loop. @index_reg is X86_REG_R12 for an arena access or -1 otherwise. + * Unless @r0_on_stack, the caller has substituted BPF_REG_AX for R0 in + * @dst_reg and @src_reg. + * + * Both the load and the CMPXCHG can fault when accessing the arena, so their + * addresses are handed back in @fault[0] and @fault[1]. A fault has to resume + * at @resume, which is past the loop and past the move that delivers the old + * value, but before R0 is restored. Both are NULL for the non-arena case, + * which needs no fixup. + * + * R0 is expected to be saved in BPF_REG_AX by the caller, or pushed on the + * stack if @r0_on_stack. In that case an R0 operand stays in the stack slot, + * RAX ends up holding the old value that R0 is to return, and the slot is + * dropped rather than popped. + */ +static int emit_atomic_fetch_rmw(u8 **pprog, struct bpf_insn *insn, u32 dst_reg, + u32 src_reg, int index_reg, bool r0_on_stack, + u8 **fault, u8 **resume) +{ + bool is64 = BPF_SIZE(insn->code) == BPF_DW; + bool operand_on_stack = r0_on_stack && src_reg == BPF_REG_0; + u8 *branch_target, *prog = *pprog; + int err; + + branch_target = prog; + + /* Load old value */ + if (fault) + fault[0] = prog; + if (index_reg < 0) + emit_ldx(&prog, BPF_SIZE(insn->code), BPF_REG_0, dst_reg, insn->off); + else + emit_ldx_index(&prog, BPF_SIZE(insn->code), BPF_REG_0, dst_reg, + index_reg, insn->off); + + /* + * Perform the (commutative) operation locally, put the result in + * the AUX_REG. + */ + if (operand_on_stack) { + /* mov r11, [rsp]: the operand is the R0 saved there, RAX is the old value */ + if (is64) + EMIT4(0x4C, 0x8B, 0x1C, 0x24); + else + EMIT4(0x44, 0x8B, 0x1C, 0x24); + } else { + emit_mov_reg(&prog, is64, AUX_REG, BPF_REG_0); + } + maybe_emit_mod(&prog, AUX_REG, src_reg, is64); + EMIT2(simple_alu_opcodes[BPF_OP(insn->imm)], + add_2reg(0xC0, AUX_REG, src_reg)); + + /* Attempt to swap in new value */ + if (fault) + fault[1] = prog; + if (index_reg < 0) + err = emit_atomic_rmw(&prog, BPF_CMPXCHG, dst_reg, AUX_REG, + insn->off, BPF_SIZE(insn->code)); + else + err = emit_atomic_rmw_index(&prog, BPF_CMPXCHG, BPF_SIZE(insn->code), + dst_reg, AUX_REG, index_reg, insn->off); + if (WARN_ON(err)) + return err; + + /* ZF tells us whether we won the race. If it's cleared we need to try again. */ + EMIT2(X86_JNE, -(prog - branch_target) - 2); + /* Return the pre-modification value, which RAX already is for R0 */ + if (!operand_on_stack) + emit_mov_reg(&prog, is64, src_reg, BPF_REG_0); + + if (resume) + *resume = prog; + + /* Restore R0 after clobbering RAX */ + if (!r0_on_stack) + emit_mov_reg(&prog, true, BPF_REG_0, BPF_REG_AX); + else if (operand_on_stack) + EMIT2(0x41, 0x5B); /* pop r11, dropping the operand */ + else + EMIT1(0x58); /* pop rax */ + + *pprog = prog; + return 0; +} + /* * Metadata encoding for exception handling in JITed code. * @@ -1480,7 +1892,12 @@ static int emit_atomic_ld_st_index(u8 **pprog, u32 atomic_op, u32 size, * | ARENA_ACC | ARENA_WRITE | Unused | ARENA_REG | DST_REG | INSN_LEN | * +-----------+-------------+--------+-----------+---------+----------+ * - * - INSN_LEN (8 bits): Length of faulting insn (max x86 insn = 15 bytes (fits in 8 bits)). + * - INSN_LEN (8 bits): How far past the faulting insn to resume. That is its own length + * for a single-insn access, but the distance to the end of the whole + * sequence where one BPF insn became several, as for the CMPXCHG loop + * of a fetching AND/OR/XOR. A sequence long enough to overflow this + * field would first have to exceed BPF_MAX_INSN_SIZE, which do_jit() + * rejects with -EFAULT before the image is used. * - DST_REG (8 bits): Offset of dst_reg from reg2pt_regs[] (max offset = 112 (fits in 8 bits)). * This is set to DONT_CLEAR if the insn does not read into a register. * - ARENA_REG (8 bits): Offset of the register that is used to calculate the @@ -1527,7 +1944,7 @@ bool ex_handler_bpf(const struct exception_table_entry *x, struct pt_regs *regs) bpf_prog_report_arena_violation(is_write, addr, regs->ip); } - /* jump over faulting load and clear dest register */ + /* jump over the faulting access, or the sequence it is part of, and clear dest register */ if (reg != DONT_CLEAR) *(unsigned long *)((void *)regs + reg) = 0; regs->ip += insn_len; @@ -1535,6 +1952,44 @@ bool ex_handler_bpf(const struct exception_table_entry *x, struct pt_regs *regs) return true; } +/* + * Record an arena access that may fault. @fault_ip is the address of the + * faulting insn in the RO image, @resume_off how far past it execution has to + * resume: for a multi-insn lowering that is the end of the whole sequence, not + * the end of the one insn. + */ +static int emit_arena_exentry(struct bpf_prog *bpf_prog, u8 *image, u8 *rw_image, + int *excnt, u8 *fault_ip, u32 resume_off, + u32 fixup_reg, u32 arena_reg, bool is_write, s16 off) +{ + struct exception_table_entry *ex; + s64 delta; + + if (!bpf_prog->aux->extable) + return 0; + + if (*excnt >= bpf_prog->aux->num_exentries) { + pr_err("arena extable bug\n"); + return -EFAULT; + } + ex = &bpf_prog->aux->extable[(*excnt)++]; + + delta = fault_ip - (u8 *)&ex->insn; + /* switch ex to rw buffer for writes */ + ex = (void *)rw_image + ((void *)ex - (void *)image); + + ex->insn = delta; + ex->data = EX_TYPE_BPF | FIELD_PREP(DATA_ARENA_OFFSET_MASK, off); + ex->fixup = FIELD_PREP(FIXUP_INSN_LEN_MASK, resume_off) | + FIELD_PREP(FIXUP_ARENA_REG_MASK, arena_reg) | + FIELD_PREP(FIXUP_REG_MASK, fixup_reg) | + FIXUP_ARENA_ACCESS; + if (is_write) + ex->fixup |= FIXUP_ARENA_WRITE; + + return 0; +} + static void detect_reg_usage(struct bpf_insn *insn, int insn_cnt, bool *regs_used) { @@ -1611,6 +2066,218 @@ static void emit_shiftx(u8 **pprog, u32 dst_reg, u8 src_reg, bool is64, u8 op) *pprog = prog; } +struct bpf_div_magic { + u64 m; /* multiplier, sign extended for signed division */ + int s; /* shift of the high half of the product */ + bool add; /* multiplier has 65 bits, use the add sequence */ +}; + +/* + * Multiplier and shift for an unsigned division by the constant d, with + * 2 <= d < 2^64 and a dividend below 2^(64 - lz): n / d == (mulhu(n, m) + * >> s), or with the add sequence when m needs 65 bits, (((n - t) >> 1) + * + t) >> (s - 1) with t = mulhu(n, m). Hacker's Delight, 2nd ed., figure + * 10-2. + */ +static void bpf_magic_udiv(u64 d, int lz, struct bpf_div_magic *mag) +{ + u64 mask = ~0ULL >> (lz & 63); + u64 nc, delta, q1, r1, q2, r2; + int p; + + mag->add = false; + /* Largest value with nc mod d == d - 1 */ + nc = mask - (mask % d + 1) % d; + p = 63; + q1 = (1ULL << 63) / nc; /* 2^p / nc */ + r1 = (1ULL << 63) - q1 * nc; /* 2^p mod nc */ + q2 = (~0ULL >> 1) / d; /* (2^p - 1) / d */ + r2 = (~0ULL >> 1) - q2 * d; /* (2^p - 1) mod d */ + do { + p++; + if (r1 >= nc - r1) { + q1 = 2 * q1 + 1; + r1 = 2 * r1 - nc; + } else { + q1 = 2 * q1; + r1 = 2 * r1; + } + if (r2 + 1 >= d - r2) { + if (q2 >= (~0ULL >> 1)) + mag->add = true; + q2 = 2 * q2 + 1; + r2 = 2 * r2 + 1 - d; + } else { + if (q2 >= (1ULL << 63)) + mag->add = true; + q2 = 2 * q2; + r2 = 2 * r2 + 1; + } + delta = d - 1 - r2; + } while (p < 128 && (q1 < delta || (q1 == delta && r1 == 0))); + + mag->m = q2 + 1; + mag->s = p - 64; +} + +/* + * Multiplier and shift for a signed division by the constant d, with + * 2 <= |d| < 2^63: q = mulhs(n, m), q += n if d > 0 and m < 0, q -= n + * if d < 0 and m > 0, q >>= s (arithmetic), n / d == q + (q >>> 63). + * Hacker's Delight, 2nd ed., figure 10-1. + */ +static void bpf_magic_sdiv(s64 d, struct bpf_div_magic *mag) +{ + u64 ad = d < 0 ? -(u64)d : (u64)d; + u64 t, anc, delta, q1, r1, q2, r2; + int p; + + t = (1ULL << 63) + (d < 0); + anc = t - 1 - t % ad; /* |nc| */ + p = 63; + q1 = (1ULL << 63) / anc; /* 2^p / |nc| */ + r1 = (1ULL << 63) - q1 * anc; /* 2^p mod |nc| */ + q2 = (1ULL << 63) / ad; /* 2^p / |d| */ + r2 = (1ULL << 63) - q2 * ad; /* 2^p mod |d| */ + do { + p++; + q1 = 2 * q1; + r1 = 2 * r1; + if (r1 >= anc) { + q1++; + r1 -= anc; + } + q2 = 2 * q2; + r2 = 2 * r2; + if (r2 >= ad) { + q2++; + r2 -= ad; + } + delta = ad - r2; + } while (q1 < delta || (q1 == delta && r1 == 0)); + + mag->m = q2 + 1; + if (d < 0) + mag->m = -mag->m; + mag->s = p - 64; + mag->add = false; +} + +/* + * Divide dst_reg by the constant in insn->imm of a 64-bit division through + * a multiplication with the magic number from bpf_magic_udiv() or + * bpf_magic_sdiv(). The dividend goes through rax and the multiplier + * through r11, the caller saves rax and rdx unless they are dst_reg, and + * r10 is scratch. Returns false for the divisors left to div/idiv. + */ +static bool emit_div_const(u8 **pprog, const struct bpf_insn *insn, u32 dst_reg) +{ + bool is_mod = BPF_OP(insn->code) == BPF_MOD; + bool is_signed = insn->off == 1; + u32 n_reg = dst_reg, q_reg; + s32 imm32 = insn->imm; + struct bpf_div_magic mag; + u8 *prog = *pprog; + int corr = 0, pre = 0, s; + + if (is_signed) { + /* |d| >= 2, the verifier already rewrote d == -1 */ + if (imm32 >= -1 && imm32 <= 1) + return false; + bpf_magic_sdiv(imm32, &mag); + /* q += n for d > 0 and m < 0, q -= n for d < 0 and m > 0 */ + if (imm32 > 0 && (s64)mag.m < 0) + corr = 1; + else if (imm32 < 0 && (s64)mag.m > 0) + corr = -1; + } else { + /* d >= 2, a negative imm32 is a divisor above 2^63 */ + if (imm32 < 2) + return false; + bpf_magic_udiv(imm32, 0, &mag); + /* + * Shift the dividend by the trailing zeros of an even divisor + * first, the magic number of the odd part then fits without + * the add sequence. + */ + if (mag.add && !(imm32 & 1)) { + pre = __ffs(imm32) & 31; + bpf_magic_udiv(imm32 >> pre, pre, &mag); + if (mag.add) + return false; + } + } + s = mag.s; + /* + * The shift is at most 63, or 64 with the add sequence which shifts + * by s - 1 and needs s >= 1. Never hit for d >= 2, kept as a guard + * against an unencodable count. + */ + if (s > 64 - !mag.add || (mag.add && !s)) + return false; + + /* Keep the dividend when the multiplication clobbers it */ + if ((is_mod || mag.add || corr) && + (dst_reg == BPF_REG_0 || dst_reg == BPF_REG_3)) { + emit_mov_reg(&prog, true, BPF_REG_AX, dst_reg); + n_reg = BPF_REG_AX; + } + + /* mov rax, dst_reg; shr rax, pre; mov r11, m; mul r11 or imul r11 */ + if (dst_reg != BPF_REG_0) + emit_mov_reg(&prog, true, BPF_REG_0, dst_reg); + if (pre) + emit_shift_imm(&prog, BPF_RSH, BPF_REG_0, pre, true); + emit_mov_imm64(&prog, AUX_REG, mag.m >> 32, (u32)mag.m); + EMIT3(0x49, 0xF7, is_signed ? 0xEB : 0xE3); + if (is_signed) { + /* add rdx, n_reg or sub rdx, n_reg */ + if (corr) + emit_alu_reg(&prog, corr > 0 ? BPF_ADD : BPF_SUB, + BPF_REG_3, n_reg, true); + /* sar rdx, s */ + if (s) + emit_shift_imm(&prog, BPF_ARSH, BPF_REG_3, s, true); + /* mov rax, rdx; shr rax, 63; add rdx, rax */ + emit_mov_reg(&prog, true, BPF_REG_0, BPF_REG_3); + emit_shift_imm(&prog, BPF_RSH, BPF_REG_0, 63, true); + emit_alu_reg(&prog, BPF_ADD, BPF_REG_3, BPF_REG_0, true); + q_reg = BPF_REG_3; + } else if (!mag.add) { + /* shr rdx, s */ + if (s) + emit_shift_imm(&prog, BPF_RSH, BPF_REG_3, s, true); + q_reg = BPF_REG_3; + } else { + /* mov rax, n_reg; sub rax, rdx; shr rax, 1 */ + emit_mov_reg(&prog, true, BPF_REG_0, n_reg); + emit_alu_reg(&prog, BPF_SUB, BPF_REG_0, BPF_REG_3, true); + emit_shift_imm(&prog, BPF_RSH, BPF_REG_0, 1, true); + /* add rax, rdx; shr rax, s - 1 */ + emit_alu_reg(&prog, BPF_ADD, BPF_REG_0, BPF_REG_3, true); + if (s > 1) + emit_shift_imm(&prog, BPF_RSH, BPF_REG_0, s - 1, true); + q_reg = BPF_REG_0; + } + + if (is_mod) { + /* imul q_reg, q_reg, imm32 */ + maybe_emit_mod(&prog, q_reg, q_reg, true); + if (is_imm8(imm32)) + EMIT3(0x6B, add_2reg(0xC0, q_reg, q_reg), imm32); + else + EMIT2_off32(0x69, add_2reg(0xC0, q_reg, q_reg), imm32); + /* sub n_reg, q_reg */ + emit_alu_reg(&prog, BPF_SUB, n_reg, q_reg, true); + q_reg = n_reg; + } + if (q_reg != dst_reg) + emit_mov_reg(&prog, true, dst_reg, q_reg); + + *pprog = prog; + return true; +} + static void emit_priv_frame_ptr(u8 **pprog, void __percpu *priv_frame_ptr) { u8 *prog = *pprog; @@ -1632,7 +2299,7 @@ static void emit_priv_frame_ptr(u8 **pprog, void __percpu *priv_frame_ptr) #define INSN_SZ_DIFF (((addrs[i] - addrs[i - 1]) - (prog - temp))) #define __LOAD_TCC_PTR(off) \ - EMIT3_off32(0x48, 0x8B, 0x85, off) + emit_ldx(&prog, BPF_DW, BPF_REG_0, BPF_REG_FP, off) /* mov rax, qword ptr [rbp - rounded_stack_depth - 16] */ #define LOAD_TAIL_CALL_CNT_PTR(stack) \ __LOAD_TCC_PTR(BPF_TAIL_CALL_CNT_PTR_STACK_OFF(stack)) @@ -1683,6 +2350,75 @@ static int emit_spectre_bhb_barrier(u8 **pprog, u8 *ip, return 0; } +static const struct bpf_jit_arg_abi x86_arg_abi = { + .nr_arg_regs = 6, + .backfill_after_stack = true, + .even_stack_align = true, +}; + +static const u8 x86_arg_reg[] = { + BPF_REG_1, BPF_REG_2, BPF_REG_3, BPF_REG_4, BPF_REG_5, X86_REG_R9, +}; + +static bool x86_arg_on_stack(u8 pos) +{ + return pos >= x86_arg_abi.nr_arg_regs; +} + +static s32 x86_arg_stack_off(u8 pos) +{ + return (pos - x86_arg_abi.nr_arg_regs) * 8; +} + +/* + * Move the arguments the x86-64 ABI places somewhere other than the argument + * slot the BPF calling convention gave them. @stack_base addresses the + * outgoing stack argument area from RBP. Return the number of emitted bytes. + */ +static int emit_kfunc_arg_moves(const struct btf_func_model *fm, s32 stack_base, u8 **pprog) +{ + struct bpf_jit_arg_move moves[BPF_JIT_MAX_ARG_MOVES]; + const u8 nreg = x86_arg_abi.nr_arg_regs; + u8 *prog = *pprog, *start = prog; + u32 i, n; + + n = bpf_jit_plan_arg_moves(&x86_arg_abi, fm, moves); + + for (i = 0; i < n; i++) { + u8 dst = moves[i].dst, src = moves[i].src, reg; + bool dst_mem = dst != BPF_JIT_ARG_TMP && dst >= nreg; + bool src_mem = src != BPF_JIT_ARG_TMP && src >= nreg; + + /* + * Take the value into a register: the one it belongs in, the + * scratch when it is carried past its own destination, and + * BPF_REG_AX only to pass one stack slot to another. + */ + if (src == BPF_JIT_ARG_TMP) { + reg = AUX_REG; + } else if (src_mem) { + reg = dst == BPF_JIT_ARG_TMP ? AUX_REG : + dst_mem ? BPF_REG_AX : x86_arg_reg[dst]; + emit_ldx(&prog, BPF_DW, reg, BPF_REG_FP, + stack_base + (src - nreg) * 8); + } else { + reg = x86_arg_reg[src]; + } + + /* And leave it where the argument belongs. */ + if (dst == BPF_JIT_ARG_TMP) + emit_mov_reg(&prog, true, AUX_REG, reg); + else if (dst_mem) + emit_stx(&prog, BPF_DW, BPF_REG_FP, reg, + stack_base + (dst - nreg) * 8); + else if (reg != x86_arg_reg[dst]) + emit_mov_reg(&prog, true, x86_arg_reg[dst], reg); + } + + *pprog = prog; + return prog - start; +} + /* * Rebase the __arena args of a kfunc call to arena kernel addresses, * rN = kern_vm_start + (u32)rN, with R12 holding kern_vm_start. A nullable @@ -1690,24 +2426,25 @@ static int emit_spectre_bhb_barrier(u8 **pprog, u8 *ip, * arena NULL is offset 0. Return the number of emitted bytes. */ static int emit_kfunc_arena_args(struct bpf_prog *bpf_prog, - const struct bpf_insn *insn, u8 **pprog) + const struct btf_func_model *fm, u8 **pprog) { - const struct btf_func_model *fm; u8 *prog = *pprog; u8 *start = prog; - int i; + int i, slot; - fm = bpf_jit_find_kfunc_model(bpf_prog, insn); - if (!fm) - return -EINVAL; - - for (i = 0; i < min_t(int, fm->nr_args, MAX_BPF_FUNC_REG_ARGS); i++) { + for (i = 0, slot = 0; i < fm->nr_args; i++) { + u32 arg_regs = (fm->arg_size[i] + 7) / 8; u8 flags = fm->arg_flags[i]; - u32 reg = BPF_REG_1 + i; + u32 reg; + + if (slot + arg_regs > MAX_BPF_FUNC_REG_ARGS) + break; + reg = BPF_REG_1 + slot; + slot += arg_regs; if (!(flags & BTF_FMODEL_ARENA_ARG)) continue; - if (WARN_ON_ONCE(!bpf_prog->aux->arena)) + if (WARN_ON_ONCE(!bpf_prog_uses_arena_insns(bpf_prog))) return -EINVAL; /* mov eN, eN: truncate and clear the upper 32 bits */ @@ -1727,6 +2464,116 @@ static int emit_kfunc_arena_args(struct bpf_prog *bpf_prog, return prog - start; } +/* + * A strict compare against 1 or 128 is the non-strict one against 0 or + * 127, a strict one against -129 or, with a signed condition, -1 the + * non-strict one against -128 or 0: the compare against 0 is a test + * without immediate, 127 and -128 fit into an imm8. + */ +static void adjust_cmp_imm(u32 *op, s32 *imm) +{ + if (*imm == 1 || *imm == 128) { + switch (*op) { + case BPF_JLT: + *op = BPF_JLE; + break; + case BPF_JGE: + *op = BPF_JGT; + break; + case BPF_JSLT: + *op = BPF_JSLE; + break; + case BPF_JSGE: + *op = BPF_JSGT; + break; + default: + return; + } + *imm -= 1; + } else if (*imm == -1 || *imm == -129) { + switch (*op) { + case BPF_JGT: + case BPF_JLE: + /* -1 is the largest unsigned value */ + if (*imm == -1) + return; + *op = *op == BPF_JGT ? BPF_JGE : BPF_JLT; + break; + case BPF_JSGT: + *op = BPF_JSGE; + break; + case BPF_JSLE: + *op = BPF_JSLT; + break; + default: + return; + } + *imm += 1; + } +} + +/* + * There is no branch for unwind_goto, only an extable entry + * { .insn = ret_addr - 1, .fixup = target - ret_addr } + * where ret_addr is the return address of the call in front of it. unwind_exit + * replaces the return address of its frame with bpf_unwind_target() of it. + * ret_addr - 1, because what follows the call can have an entry of its own. + * No insn starts there, so a fault cannot hit the entry. + */ +static int add_unwind_exentry(struct bpf_prog *prog, u8 *image, u8 *rw_image, int *excnt, + int ret_addr, int target) +{ + struct exception_table_entry *ex; + s64 delta; + + if (!prog->aux->extable) + return 0; + if (*excnt >= prog->aux->num_exentries) { + pr_err("unwind extable bug\n"); + return -EFAULT; + } + ex = &prog->aux->extable[(*excnt)++]; + delta = image + ret_addr - 1 - (u8 *)&ex->insn; + if (!is_simm32(delta)) { + pr_err("extable->insn doesn't fit into 32-bit\n"); + return -EFAULT; + } + /* switch ex to rw buffer for writes */ + ex = (void *)rw_image + ((void *)ex - (void *)image); + ex->insn = delta; + ex->fixup = target - ret_addr; + ex->data = EX_TYPE_BPF; + return 0; +} + +static u64 bpf_unwind_target(u64 ret_addr) +{ + const struct exception_table_entry *e = search_bpf_extables(ret_addr - 1); + + /* + * The verifier made sure that unwind_goto follows a call in a prog. When + * the kernel called an async callback, it's exit. + */ + return e ? ret_addr + e->fixup : ret_addr; +} + +/* unwind_exit: fix the return address of this frame. r0 and r2 are returned. */ +static int emit_unwind_ret_addr(u8 **pprog, u8 *ip) +{ + u8 *prog = *pprog, *start = prog; + + EMIT2(0x50, 0x56); /* push rax; push rsi */ + EMIT4(0x48, 0x8B, 0x7D, 0x08); /* mov rdi, [rbp + 8] */ + ip += prog - start; + ip += x86_call_depth_emit_accounting(&prog, bpf_unwind_target, ip); + if (emit_call(&prog, bpf_unwind_target, ip)) + return -EINVAL; + EMIT4(0x48, 0x89, 0x45, 0x08); /* mov [rbp + 8], rax */ + EMIT2(0x5E, 0x58); /* pop rsi; pop rax */ + *pprog = prog; + return 0; +} + static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int *addrs, u8 *image, u8 *rw_image, int oldproglen, struct jit_context *ctx, bool jmp_padding) { @@ -1738,7 +2585,7 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * u8 temp[BPF_MAX_INSN_SIZE + BPF_INSN_SAFETY]; void __percpu *priv_frame_ptr = NULL; u16 out_stack_arg_cnt, outgoing_rsp; - u64 arena_vm_start, user_vm_start; + u64 arena_vm_start = 0, user_vm_start = 0; void __percpu *priv_stack_ptr; int i, excnt = 0; int ilen, proglen = 0; @@ -1778,8 +2625,10 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * * [below] callee-saved regs * [below] outgoing args 7+ (= rsp) */ - arena_vm_start = bpf_arena_get_kern_vm_start(bpf_prog->aux->arena); - user_vm_start = bpf_arena_get_user_vm_start(bpf_prog->aux->arena); + if (bpf_prog_uses_arena_insns(bpf_prog)) { + arena_vm_start = bpf_arena_get_kern_vm_start(bpf_prog->aux->arena); + user_vm_start = bpf_arena_get_user_vm_start(bpf_prog->aux->arena); + } detect_reg_usage(insn, insn_cnt, callee_regs_used); @@ -1860,13 +2709,17 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * const s32 imm32 = insn->imm; u32 dst_reg = insn->dst_reg; u32 src_reg = insn->src_reg; - u8 b2 = 0, b3 = 0; + bool accesses_stack_only; + u8 b3 = 0; u8 *start_of_ldx; s64 jmp_offset; s32 insn_off; + int insn_idx; u8 jmp_cond; u8 *func; int nops; + u32 jmp_op = BPF_OP(insn->code); + s32 cmp_imm; if (priv_frame_ptr) { if (src_reg == BPF_REG_FP) @@ -1880,6 +2733,10 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * EMIT_ENDBR(); ip = image + addrs[i - 1] + (prog - temp); + insn_idx = i - 1 + bpf_prog->aux->subprog_start; + accesses_stack_only = + env ? !env->insn_aux_data[insn_idx].non_stack_access : + false; switch (insn->code) { /* ALU */ @@ -1893,43 +2750,24 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * case BPF_ALU64 | BPF_AND | BPF_X: case BPF_ALU64 | BPF_OR | BPF_X: case BPF_ALU64 | BPF_XOR | BPF_X: - maybe_emit_mod(&prog, dst_reg, src_reg, - BPF_CLASS(insn->code) == BPF_ALU64); - b2 = simple_alu_opcodes[BPF_OP(insn->code)]; - EMIT2(b2, add_2reg(0xC0, dst_reg, src_reg)); + emit_alu_reg(&prog, BPF_OP(insn->code), dst_reg, src_reg, + BPF_CLASS(insn->code) == BPF_ALU64); break; case BPF_ALU64 | BPF_MOV | BPF_X: if (insn_is_cast_user(insn)) { - if (dst_reg != src_reg) - /* 32-bit mov */ - emit_mov_reg(&prog, false, dst_reg, src_reg); - /* shl dst_reg, 32 */ - maybe_emit_1mod(&prog, dst_reg, true); - EMIT3(0xC1, add_1reg(0xE0, dst_reg), 32); - - /* or dst_reg, user_vm_start */ - maybe_emit_1mod(&prog, dst_reg, true); - if (is_axreg(dst_reg)) - EMIT1_off32(0x0D, user_vm_start >> 32); - else - EMIT2_off32(0x81, add_1reg(0xC8, dst_reg), user_vm_start >> 32); - - /* rol dst_reg, 32 */ - maybe_emit_1mod(&prog, dst_reg, true); - EMIT3(0xC1, add_1reg(0xC0, dst_reg), 32); - - /* xor r11, r11 */ - EMIT3(0x4D, 0x31, 0xDB); - - /* test dst_reg32, dst_reg32; check if lower 32-bit are zero */ + /* mov dst_reg32, src_reg32 to zero extend the low half */ + emit_mov_reg(&prog, false, dst_reg, src_reg); + /* mov r11, the high half of user_vm_start; or r11, dst_reg */ + emit_mov_imm64(&prog, AUX_REG, user_vm_start >> 32, 0); + emit_alu_reg(&prog, BPF_OR, AUX_REG, dst_reg, true); + /* test dst_reg32, dst_reg32; a NULL pointer stays NULL */ maybe_emit_mod(&prog, dst_reg, dst_reg, false); EMIT2(0x85, add_2reg(0xC0, dst_reg, dst_reg)); - - /* cmove r11, dst_reg; if so, set dst_reg to zero */ + /* cmovne dst_reg, r11 */ /* WARNING: Intel swapped src/dst register encoding in CMOVcc !!! */ maybe_emit_mod(&prog, AUX_REG, dst_reg, true); - EMIT3(0x0F, 0x44, add_2reg(0xC0, AUX_REG, dst_reg)); + EMIT3(0x0F, 0x45, add_2reg(0xC0, AUX_REG, dst_reg)); break; } else if (insn_is_mov_percpu_addr(insn)) { /* mov , (if necessary) */ @@ -1957,9 +2795,7 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * /* neg dst */ case BPF_ALU | BPF_NEG: case BPF_ALU64 | BPF_NEG: - maybe_emit_1mod(&prog, dst_reg, - BPF_CLASS(insn->code) == BPF_ALU64); - EMIT2(0xF7, add_1reg(0xD8, dst_reg)); + emit_neg(&prog, dst_reg, BPF_CLASS(insn->code) == BPF_ALU64); break; case BPF_ALU | BPF_ADD | BPF_K: @@ -1972,42 +2808,23 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * case BPF_ALU64 | BPF_AND | BPF_K: case BPF_ALU64 | BPF_OR | BPF_K: case BPF_ALU64 | BPF_XOR | BPF_K: - maybe_emit_1mod(&prog, dst_reg, - BPF_CLASS(insn->code) == BPF_ALU64); - - /* - * b3 holds 'normal' opcode, b2 short form only valid - * in case dst is eax/rax. - */ - switch (BPF_OP(insn->code)) { - case BPF_ADD: - b3 = 0xC0; - b2 = 0x05; + if (BPF_OP(insn->code) == BPF_AND && + (imm32 == 0xff || imm32 == 0xffff)) { + /* movzx dst_reg, dst_reg8/16 */ + emit_movzx_reg(&prog, imm32 == 0xff ? 8 : 16, + dst_reg, dst_reg); break; - case BPF_SUB: - b3 = 0xE8; - b2 = 0x2D; - break; - case BPF_AND: - b3 = 0xE0; - b2 = 0x25; - break; - case BPF_OR: - b3 = 0xC8; - b2 = 0x0D; - break; - case BPF_XOR: - b3 = 0xF0; - b2 = 0x35; + } + if (BPF_OP(insn->code) == BPF_XOR && imm32 == -1) { + /* not dst_reg */ + maybe_emit_1mod(&prog, dst_reg, + BPF_CLASS(insn->code) == BPF_ALU64); + EMIT2(0xF7, add_1reg(0xD0, dst_reg)); break; } - if (is_imm8(imm32)) - EMIT3(0x83, add_1reg(b3, dst_reg), imm32); - else if (is_axreg(dst_reg)) - EMIT1_off32(b2, imm32); - else - EMIT2_off32(0x81, add_1reg(b3, dst_reg), imm32); + emit_alu_imm(&prog, BPF_OP(insn->code), dst_reg, imm32, + BPF_CLASS(insn->code) == BPF_ALU64); break; case BPF_ALU64 | BPF_MOV | BPF_K: @@ -2038,6 +2855,11 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * if (dst_reg != BPF_REG_3) EMIT1(0x52); /* push rdx */ + /* LLVM strength-reduces the 32-bit ones itself */ + if (is64 && BPF_SRC(insn->code) == BPF_K && + emit_div_const(&prog, insn, dst_reg)) + goto div_done; + if (BPF_SRC(insn->code) == BPF_X) { if (src_reg == BPF_REG_0 || src_reg == BPF_REG_3) { @@ -2060,7 +2882,7 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * * xor edx, edx * equivalent to 'xor rdx, rdx', but one byte less */ - EMIT2(0x31, 0xd2); + emit_mov_imm32(&prog, false, BPF_REG_3, 0); /* div src_reg */ maybe_emit_1mod(&prog, src_reg, is64); @@ -2084,7 +2906,7 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * dst_reg != BPF_REG_0) /* mov dst_reg, rax */ emit_mov_reg(&prog, is64, dst_reg, BPF_REG_0); - +div_done: if (dst_reg != BPF_REG_3) EMIT1(0x5A); /* pop rdx */ if (dst_reg != BPF_REG_0) @@ -2094,6 +2916,25 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * case BPF_ALU | BPF_MUL | BPF_K: case BPF_ALU64 | BPF_MUL | BPF_K: + /* + * lea for 3, 5 and 9, which LLVM leaves as a + * multiplication, but not for r13, which needs a + * displacement as base and makes the lea a slow three + * operand form on Intel. + */ + if ((imm32 == 3 || imm32 == 5 || imm32 == 9) && + !base_needs_disp(dst_reg)) { + /* lea dst_reg, [dst_reg + dst_reg * (imm32 - 1)] */ + if (BPF_CLASS(insn->code) == BPF_ALU64) + EMIT1(add_3mod(0x48, dst_reg, dst_reg, dst_reg)); + else if (is_ereg(dst_reg)) + EMIT1(add_3mod(0x40, dst_reg, dst_reg, dst_reg)); + EMIT1(0x8D); + emit_insn_suffix_SIB(&prog, dst_reg, dst_reg, dst_reg, 0, + ilog2(imm32 - 1)); + break; + } + maybe_emit_mod(&prog, dst_reg, dst_reg, BPF_CLASS(insn->code) == BPF_ALU64); @@ -2124,14 +2965,8 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * case BPF_ALU64 | BPF_LSH | BPF_K: case BPF_ALU64 | BPF_RSH | BPF_K: case BPF_ALU64 | BPF_ARSH | BPF_K: - maybe_emit_1mod(&prog, dst_reg, - BPF_CLASS(insn->code) == BPF_ALU64); - - b3 = simple_alu_opcodes[BPF_OP(insn->code)]; - if (imm32 == 1) - EMIT2(0xD1, add_1reg(b3, dst_reg)); - else - EMIT3(0xC1, add_1reg(b3, dst_reg), imm32); + emit_shift_imm(&prog, BPF_OP(insn->code), dst_reg, + imm32, BPF_CLASS(insn->code) == BPF_ALU64); break; case BPF_ALU | BPF_LSH | BPF_X: @@ -2140,8 +2975,12 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * case BPF_ALU64 | BPF_LSH | BPF_X: case BPF_ALU64 | BPF_RSH | BPF_X: case BPF_ALU64 | BPF_ARSH | BPF_X: - /* BMI2 shifts aren't better when shift count is already in rcx */ - if (boot_cpu_has(X86_FEATURE_BMI2) && src_reg != BPF_REG_4) { + /* + * BMI2 shifts take the count from any register and, + * unlike the cl forms, are a single uop on Intel, so + * prefer them also when the count already is in rcx. + */ + if (boot_cpu_has(X86_FEATURE_BMI2)) { /* shrx/sarx/shlx dst_reg, dst_reg, src_reg */ bool w = (BPF_CLASS(insn->code) == BPF_ALU64); u8 op; @@ -2197,18 +3036,16 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * case BPF_ALU64 | BPF_END | BPF_FROM_LE: switch (imm32) { case 16: - /* Emit 'ror %ax, 8' to swap lower 2 bytes */ - EMIT1(0x66); + /* Emit 'bswap eax' to swap lower 4 bytes */ if (is_ereg(dst_reg)) EMIT1(0x41); - EMIT3(0xC1, add_1reg(0xC8, dst_reg), 8); + EMIT2(0x0F, add_1reg(0xC8, dst_reg)); - /* Emit 'movzwl eax, ax' */ - if (is_ereg(dst_reg)) - EMIT3(0x45, 0x0F, 0xB7); - else - EMIT2(0x0F, 0xB7); - EMIT1(add_2reg(0xC0, dst_reg, dst_reg)); + /* + * Emit 'shr eax, 16' to move the swapped lower + * 2 bytes back down and zero extend + */ + emit_shift_imm(&prog, BPF_RSH, dst_reg, 16, false); break; case 32: /* Emit 'bswap eax' to swap lower 4 bytes */ @@ -2233,11 +3070,7 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * * Emit 'movzwl eax, ax' to zero extend 16-bit * into 64 bit */ - if (is_ereg(dst_reg)) - EMIT3(0x45, 0x0F, 0xB7); - else - EMIT2(0x0F, 0xB7); - EMIT1(add_2reg(0xC0, dst_reg, dst_reg)); + emit_movzx_reg(&prog, 16, dst_reg, dst_reg); break; case 32: /* Emit 'mov eax, eax' to clear upper 32-bits */ @@ -2256,49 +3089,37 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * EMIT_LFENCE(); break; - /* ST: *(u8*)(dst_reg + off) = imm */ case BPF_ST | BPF_MEM | BPF_B: - if (is_ereg(dst_reg)) - EMIT2(0x41, 0xC6); - else - EMIT1(0xC6); - goto st; case BPF_ST | BPF_MEM | BPF_H: - if (is_ereg(dst_reg)) - EMIT3(0x66, 0x41, 0xC7); - else - EMIT2(0x66, 0xC7); - goto st; case BPF_ST | BPF_MEM | BPF_W: - if (is_ereg(dst_reg)) - EMIT2(0x41, 0xC7); - else - EMIT1(0xC7); - goto st; case BPF_ST | BPF_MEM | BPF_DW: if (dst_reg == BPF_REG_PARAMS && insn->off == -8) { /* Arg 6: store immediate in r9 register */ - emit_mov_imm64(&prog, X86_REG_R9, imm32 >> 31, (u32)imm32); + emit_mov_imm64(&prog, X86_REG_R9, imm32 >> 31, + imm32); break; } - EMIT2(add_1mod(0x48, dst_reg), 0xC7); - -st: insn_off = insn->off; + insn_off = insn->off; if (dst_reg == BPF_REG_PARAMS) { /* * Args 7+: reverse BPF negative offsets to * x86 positive rsp offsets. * BPF off=-16 → [rsp+0], off=-24 → [rsp+8], ... */ - insn_off = outgoing_arg_base - outgoing_rsp - insn_off - 16; + insn_off = outgoing_arg_base - outgoing_rsp - + insn_off - 16; dst_reg = BPF_REG_FP; } - if (is_imm8(insn_off)) - EMIT2(add_1reg(0x40, dst_reg), insn_off); - else - EMIT1_off32(add_1reg(0x80, dst_reg), insn_off); + if (!accesses_stack_only) { + err = emit_kasan_check(env, &prog, dst_reg, + insn_off, + BPF_SIZE(insn->code), ip, + true); + if (err) + return err; + } - EMIT(imm32, bpf_size_to_x86_bytes(BPF_SIZE(insn->code))); + emit_st(&prog, insn, dst_reg, insn_off); break; /* STX: *(u8*)(dst_reg + off) = src_reg */ @@ -2316,6 +3137,14 @@ st: insn_off = insn->off; insn_off = outgoing_arg_base - outgoing_rsp - insn_off - 16; dst_reg = BPF_REG_FP; } + if (!accesses_stack_only) { + err = emit_kasan_check(env, &prog, dst_reg, + insn_off, + BPF_SIZE(insn->code), ip, + true); + if (err) + return err; + } emit_stx(&prog, BPF_SIZE(insn->code), dst_reg, src_reg, insn_off); break; @@ -2350,28 +3179,8 @@ st: insn_off = insn->off; } populate_extable: { - struct exception_table_entry *ex; - u8 *_insn = image + proglen + (start_of_ldx - temp); u32 arena_reg, fixup_reg; bool is_write; - s64 delta; - - if (!bpf_prog->aux->extable) - break; - - if (excnt >= bpf_prog->aux->num_exentries) { - pr_err("mem32 extable bug\n"); - return -EFAULT; - } - ex = &bpf_prog->aux->extable[excnt++]; - - delta = _insn - (u8 *)&ex->insn; - /* switch ex to rw buffer for writes */ - ex = (void *)rw_image + ((void *)ex - (void *)image); - - ex->insn = delta; - - ex->data = EX_TYPE_BPF; /* * src_reg/dst_reg holds the address in the arena region with upper @@ -2407,14 +3216,12 @@ st: insn_off = insn->off; is_write = true; } - ex->fixup = FIELD_PREP(FIXUP_INSN_LEN_MASK, prog - start_of_ldx) | - FIELD_PREP(FIXUP_ARENA_REG_MASK, arena_reg) | - FIELD_PREP(FIXUP_REG_MASK, fixup_reg); - ex->fixup |= FIXUP_ARENA_ACCESS; - if (is_write) - ex->fixup |= FIXUP_ARENA_WRITE; - - ex->data |= FIELD_PREP(DATA_ARENA_OFFSET_MASK, insn->off); + err = emit_arena_exentry(bpf_prog, image, rw_image, &excnt, + image + proglen + (start_of_ldx - temp), + prog - start_of_ldx, fixup_reg, + arena_reg, is_write, insn->off); + if (err) + return err; } break; @@ -2455,38 +3262,32 @@ st: insn_off = insn->off; * src_reg + insn->off > TASK_SIZE_MAX + PAGE_SIZE * and * src_reg + insn->off < VSYSCALL_ADDR + * + * The check is done on src_reg + insn->off - VSYSCALL_ADDR + * against the next power of two above the limit. Addresses + * between the limit and that power of two are non-canonical + * and would fault anyway, so they get zeroed a bit earlier. */ u64 limit = TASK_SIZE_MAX + PAGE_SIZE - VSYSCALL_ADDR; u8 *end_of_jmp; - /* movabsq r10, VSYSCALL_ADDR */ - emit_mov_imm64(&prog, BPF_REG_AX, (long)VSYSCALL_ADDR >> 32, - (u32)(long)VSYSCALL_ADDR); + /* + * lea r11, [src_reg + insn->off - VSYSCALL_ADDR] + * + * VSYSCALL_ADDR is a small negative number, so the + * whole offset fits into the 32-bit displacement. + */ + BUILD_BUG_ON((long)(s32)VSYSCALL_ADDR != (long)VSYSCALL_ADDR); + EMIT2(add_2mod(0x48, src_reg, AUX_REG), 0x8D); + emit_insn_suffix(&prog, src_reg, AUX_REG, + insn->off - (s32)VSYSCALL_ADDR); - /* mov src_reg, r11 */ - EMIT_mov(AUX_REG, src_reg); + /* shr r11, fls64(limit) */ + emit_shift_imm(&prog, BPF_RSH, AUX_REG, fls64(limit), true); - if (insn->off) { - /* add r11, insn->off */ - maybe_emit_1mod(&prog, AUX_REG, true); - EMIT2_off32(0x81, add_1reg(0xC0, AUX_REG), insn->off); - } - - /* sub r11, r10 */ - maybe_emit_mod(&prog, AUX_REG, BPF_REG_AX, true); - EMIT2(0x29, add_2reg(0xC0, AUX_REG, BPF_REG_AX)); - - /* movabsq r10, limit */ - emit_mov_imm64(&prog, BPF_REG_AX, (long)limit >> 32, - (u32)(long)limit); - - /* cmp r10, r11 */ - maybe_emit_mod(&prog, AUX_REG, BPF_REG_AX, true); - EMIT2(0x39, add_2reg(0xC0, AUX_REG, BPF_REG_AX)); - - /* if unsigned '>', goto load */ - EMIT2(X86_JA, 0); + /* if not zero, goto load */ + EMIT2(X86_JNE, 0); end_of_jmp = prog; /* xor dst_reg, dst_reg */ @@ -2494,9 +3295,16 @@ st: insn_off = insn->off; /* jmp byte_after_ldx */ EMIT2(0xEB, 0); - /* populate jmp_offset for JAE above to jump to start_of_ldx */ + /* populate jmp_offset for JNE above to jump to start_of_ldx */ start_of_ldx = prog; end_of_jmp[-1] = start_of_ldx - end_of_jmp; + } else if (!accesses_stack_only) { + err = emit_kasan_check(env, &prog, src_reg, + insn_off, + BPF_SIZE(insn->code), ip, + false); + if (err) + return err; } if (BPF_MODE(insn->code) == BPF_PROBE_MEMSX || BPF_MODE(insn->code) == BPF_MEMSX) @@ -2558,68 +3366,52 @@ st: insn_off = insn->off; } fallthrough; case BPF_STX | BPF_ATOMIC | BPF_W: - case BPF_STX | BPF_ATOMIC | BPF_DW: - if (insn->imm == (BPF_AND | BPF_FETCH) || - insn->imm == (BPF_OR | BPF_FETCH) || - insn->imm == (BPF_XOR | BPF_FETCH)) { - bool is64 = BPF_SIZE(insn->code) == BPF_DW; - u32 real_src_reg = src_reg; - u32 real_dst_reg = dst_reg; - u8 *branch_target; - - /* - * Can't be implemented with a single x86 insn. - * Need to do a CMPXCHG loop. - */ + case BPF_STX | BPF_ATOMIC | BPF_DW: { + u32 real_src_reg = src_reg; + u32 real_dst_reg = dst_reg; + bool is_atomic_fetch = is_atomic_fetch_op(insn); + u8 *old_prog; + if (is_atomic_fetch) { /* Will need RAX as a CMPXCHG operand so save R0 */ + old_prog = prog; emit_mov_reg(&prog, true, BPF_REG_AX, BPF_REG_0); if (src_reg == BPF_REG_0) real_src_reg = BPF_REG_AX; if (dst_reg == BPF_REG_0) real_dst_reg = BPF_REG_AX; - - branch_target = prog; - /* Load old value */ - emit_ldx(&prog, BPF_SIZE(insn->code), - BPF_REG_0, real_dst_reg, insn->off); - /* - * Perform the (commutative) operation locally, - * put the result in the AUX_REG. - */ - emit_mov_reg(&prog, is64, AUX_REG, BPF_REG_0); - maybe_emit_mod(&prog, AUX_REG, real_src_reg, is64); - EMIT2(simple_alu_opcodes[BPF_OP(insn->imm)], - add_2reg(0xC0, AUX_REG, real_src_reg)); - /* Attempt to swap in new value */ - err = emit_atomic_rmw(&prog, BPF_CMPXCHG, - real_dst_reg, AUX_REG, - insn->off, - BPF_SIZE(insn->code)); - if (WARN_ON(err)) + ip += (prog - old_prog); + } + if (!bpf_atomic_is_load_store(insn)) { + if (!accesses_stack_only) { + err = emit_kasan_check( + env, &prog, real_dst_reg, + insn->off, BPF_SIZE(insn->code), + ip, true); + if (err) + return err; + } + } + if (is_atomic_fetch) { + err = emit_atomic_fetch_rmw(&prog, insn, real_dst_reg, + real_src_reg, -1, false, + NULL, NULL); + if (err) return err; - /* - * ZF tells us whether we won the race. If it's - * cleared we need to try again. - */ - EMIT2(X86_JNE, -(prog - branch_target) - 2); - /* Return the pre-modification value */ - emit_mov_reg(&prog, is64, real_src_reg, BPF_REG_0); - /* Restore R0 after clobbering RAX */ - emit_mov_reg(&prog, true, BPF_REG_0, BPF_REG_AX); break; } if (bpf_atomic_is_load_store(insn)) - err = emit_atomic_ld_st(&prog, insn->imm, dst_reg, src_reg, - insn->off, BPF_SIZE(insn->code)); + err = emit_atomic_ld_st(env, &prog, insn, ip, + dst_reg, src_reg, + accesses_stack_only); else err = emit_atomic_rmw(&prog, insn->imm, dst_reg, src_reg, insn->off, BPF_SIZE(insn->code)); if (err) return err; break; - + } case BPF_STX | BPF_PROBE_ATOMIC | BPF_B: case BPF_STX | BPF_PROBE_ATOMIC | BPF_H: if (!bpf_atomic_is_load_store(insn)) { @@ -2629,6 +3421,53 @@ st: insn_off = insn->off; fallthrough; case BPF_STX | BPF_PROBE_ATOMIC | BPF_W: case BPF_STX | BPF_PROBE_ATOMIC | BPF_DW: + if (is_atomic_fetch_op(insn)) { + u32 real_src_reg = src_reg, real_dst_reg = dst_reg; + /* + * An access through a number has its address in + * BPF_REG_AX (see arena_scalar_access()), so R0 is + * saved on the stack instead, as div and mod do. + */ + bool r0_on_stack = dst_reg == BPF_REG_AX; + u8 *fault[2], *resume; + int j; + + /* Will need RAX as a CMPXCHG operand so save R0 */ + if (r0_on_stack) { + EMIT1(0x50); /* push rax */ + } else { + emit_mov_reg(&prog, true, BPF_REG_AX, BPF_REG_0); + if (src_reg == BPF_REG_0) + real_src_reg = BPF_REG_AX; + if (dst_reg == BPF_REG_0) + real_dst_reg = BPF_REG_AX; + } + + err = emit_atomic_fetch_rmw(&prog, insn, real_dst_reg, + real_src_reg, X86_REG_R12, + r0_on_stack, fault, &resume); + if (err) + return err; + + /* + * The page may be unmapped between the load and the + * CMPXCHG, so both need an entry. Both report a write, + * since the BPF insn is a read-modify-write. + */ + for (j = 0; j < 2; j++) { + err = emit_arena_exentry(bpf_prog, image, rw_image, + &excnt, + image + proglen + + (fault[j] - temp), + resume - fault[j], + reg2pt_regs[real_src_reg], + reg2pt_regs[real_dst_reg], + true, insn->off); + if (err) + return err; + } + break; + } start_of_ldx = prog; if (bpf_atomic_is_load_store(insn)) @@ -2645,18 +3484,39 @@ st: insn_off = insn->off; /* call */ case BPF_JMP | BPF_CALL: { + const struct btf_func_model *fm = NULL; + func = (u8 *) __bpf_call_base + imm32; if (src_reg == BPF_PSEUDO_CALL && tail_call_reachable) { + u8 *tcc_load = prog; + LOAD_TAIL_CALL_CNT_PTR(stack_depth); - ip += 7; + ip += prog - tcc_load; } if (!imm32) return -EINVAL; if (src_reg == BPF_PSEUDO_KFUNC_CALL) { - err = emit_kfunc_arena_args(bpf_prog, insn, &prog); + fm = bpf_jit_find_kfunc_model(bpf_prog, insn); + if (!fm) + return -EINVAL; + err = emit_kfunc_arena_args(bpf_prog, fm, &prog); if (err < 0) return err; ip += err; + ip += emit_kfunc_arg_moves(fm, outgoing_arg_base - + outgoing_rsp, &prog); + } + if (func == (u8 *)arch_bpf_timed_may_goto) { + u32 fp = priv_frame_ptr ? X86_REG_R9 : BPF_REG_FP; + + /* + * AX has the offset of count and timestamp + * in the stack. Turn it into a pointer. + * add r10, rbp or add r10, r9 + */ + maybe_emit_mod(&prog, BPF_REG_AX, fp, true); + EMIT2(0x01, add_2reg(0xC0, BPF_REG_AX, fp)); + ip += 3; } if (priv_frame_ptr) { push_r9(&prog); @@ -2667,9 +3527,63 @@ st: insn_off = insn->off; return -EINVAL; if (priv_frame_ptr) pop_r9(&prog); + /* + * A kfunc returning more than 8 bytes hands the second + * half back in RDX (the native ABI's second return reg), + * but BPF expects it in R0:R2. BPF R0 is RAX (no move + * needed), while BPF R2 is RSI, so copy RDX into RSI. + */ + if (fm && fm->ret_size > 8) + emit_mov_reg(&prog, true, BPF_REG_2, BPF_REG_3); break; } + /* callx: call of a bpf subprog whose address is in dst_reg */ + case BPF_JMP | BPF_CALL | BPF_X: + /* + * The verifier makes sure that callees of callx are + * not tail call reachable, hence unlike a direct call + * of a subprog there is no need to pass + * tail_call_cnt_ptr in rax. + */ + if (priv_frame_ptr) { + push_r9(&prog); + ip += 2; + } + if (emit_indirect_call(&prog, insn->dst_reg, ip)) + return -EINVAL; + if (priv_frame_ptr) + pop_r9(&prog); + break; + + /* unwind_goto. The verifier expanded may_goto. */ + case BPF_JMP | BPF_JCOND: + if (!priv_frame_ptr) { + err = add_unwind_exentry(bpf_prog, image, rw_image, &excnt, + addrs[i - 1], addrs[i + insn->off]); + if (err) + return err; + break; + } + /* + * 'pop r9' after the call has to be done on the way to + * the pad too: + * call foo + * pop r9 <- exit returns here + * jmp 1f + * pop r9 <- unwind_exit returns here + * jmp pad + * 1: + */ + err = add_unwind_exentry(bpf_prog, image, rw_image, &excnt, + addrs[i - 1] - 2, addrs[i - 1] + 2); + if (err) + return err; + EMIT2(0xEB, 7); + pop_r9(&prog); + EMIT1_off32(0xE9, addrs[i + insn->off] - (addrs[i - 1] + 9)); + break; + case BPF_JMP | BPF_TAIL_CALL: if (imm32) emit_bpf_tail_call_direct(bpf_prog, @@ -2725,10 +3639,26 @@ st: insn_off = insn->off; case BPF_JMP | BPF_JSET | BPF_K: case BPF_JMP32 | BPF_JSET | BPF_K: + if ((u32)imm32 <= U8_MAX) { + /* + * test dst_reg8, imm8: with the upper bits of the + * mask clear the result only depends on the low byte. + */ + if (is_ereg_8l(dst_reg)) + EMIT1(add_1mod(0x40, dst_reg)); + if (is_axreg(dst_reg)) + EMIT2(0xA8, imm32); + else + EMIT3(0xF6, add_1reg(0xC0, dst_reg), imm32); + goto emit_cond_jmp; + } /* test dst_reg, imm32 */ maybe_emit_1mod(&prog, dst_reg, BPF_CLASS(insn->code) == BPF_JMP); - EMIT2_off32(0xF7, add_1reg(0xC0, dst_reg), imm32); + if (is_axreg(dst_reg)) + EMIT1_off32(0xA9, imm32); + else + EMIT2_off32(0xF7, add_1reg(0xC0, dst_reg), imm32); goto emit_cond_jmp; case BPF_JMP | BPF_JEQ | BPF_K: @@ -2751,8 +3681,11 @@ st: insn_off = insn->off; case BPF_JMP32 | BPF_JSLT | BPF_K: case BPF_JMP32 | BPF_JSGE | BPF_K: case BPF_JMP32 | BPF_JSLE | BPF_K: + cmp_imm = imm32; + adjust_cmp_imm(&jmp_op, &cmp_imm); + /* test dst_reg, dst_reg to save one extra byte */ - if (imm32 == 0) { + if (cmp_imm == 0) { maybe_emit_mod(&prog, dst_reg, dst_reg, BPF_CLASS(insn->code) == BPF_JMP); EMIT2(0x85, add_2reg(0xC0, dst_reg, dst_reg)); @@ -2763,13 +3696,15 @@ st: insn_off = insn->off; maybe_emit_1mod(&prog, dst_reg, BPF_CLASS(insn->code) == BPF_JMP); - if (is_imm8(imm32)) - EMIT3(0x83, add_1reg(0xF8, dst_reg), imm32); + if (is_imm8(cmp_imm)) + EMIT3(0x83, add_1reg(0xF8, dst_reg), cmp_imm); + else if (is_axreg(dst_reg)) + EMIT1_off32(0x3D, cmp_imm); else - EMIT2_off32(0x81, add_1reg(0xF8, dst_reg), imm32); + EMIT2_off32(0x81, add_1reg(0xF8, dst_reg), cmp_imm); emit_cond_jmp: /* Convert BPF opcode to x86 */ - switch (BPF_OP(insn->code)) { + switch (jmp_op) { case BPF_JEQ: jmp_cond = X86_JE; break; @@ -2930,13 +3865,16 @@ st: insn_off = insn->off; break; case BPF_JMP | BPF_EXIT: + if (src_reg == BPF_UNWIND_EXIT && bpf_prog_can_unwind(bpf_prog) && + emit_unwind_ret_addr(&prog, ip)) + return -EINVAL; if (seen_exit) { jmp_offset = ctx->cleanup_addr - addrs[i]; goto emit_jmp; } seen_exit = true; /* Update cleanup_addr */ - ctx->cleanup_addr = proglen; + ctx->cleanup_addr = proglen + (prog - temp); if (bpf_prog_was_classic(bpf_prog) && !ns_capable_noaudit(&init_user_ns, CAP_SYS_ADMIN)) { if (emit_spectre_bhb_barrier(&prog, ip, bpf_prog)) @@ -3051,21 +3989,17 @@ static void clean_stack_garbage(const struct btf_func_model *m, } } -/* get the count of the regs that are used to pass arguments */ -static int get_nr_used_regs(const struct btf_func_model *m) +/* the eightbytes taken by the on-stack arguments, alignment holes included */ +static int get_nr_stack_slots(const u8 *pos_of_slot, int nr_slots) { - int i, arg_regs, nr_used_regs = 0; + int slot; - for (i = 0; i < min_t(int, m->nr_args, MAX_BPF_FUNC_ARGS); i++) { - arg_regs = (m->arg_size[i] + 7) / 8; - if (nr_used_regs + arg_regs <= 6) - nr_used_regs += arg_regs; - - if (nr_used_regs >= 6) - break; + for (slot = nr_slots - 1; slot >= 0; slot--) { + if (x86_arg_on_stack(pos_of_slot[slot])) + return x86_arg_stack_off(pos_of_slot[slot]) / 8 + 1; } - return nr_used_regs; + return 0; } /* @@ -3095,20 +4029,21 @@ static void emit_arena_arg_conv(u8 **pprog, u32 src_reg, bool nullable, u32 base } static void save_args(const struct btf_func_model *m, u8 **prog, - int stack_size, bool for_call_origin, u32 flags, - u64 arena_base) + const u8 *pos_of_slot, int stack_size, + bool for_call_origin, u32 flags, u64 arena_base) { - int arg_regs, first_off = 0, nr_regs = 0, nr_stack_slots = 0; + int arg_regs, first_off = 0, nr_stack_slots = 0; bool use_jmp = bpf_trampoline_use_jmp(flags); int stack_args_off = (use_jmp || (flags & BPF_TRAMP_F_INDIRECT)) ? 16 : 24; - int i, j; + int bargs_off = stack_size; + int i, j, slot, off, doff; /* Store function arguments to stack. * For a function that accepts two pointers the sequence will be: * mov QWORD PTR [rbp-0x10],rdi * mov QWORD PTR [rbp-0x8],rsi */ - for (i = 0; i < min_t(int, m->nr_args, MAX_BPF_FUNC_ARGS); i++) { + for (i = 0, slot = 0; i < m->nr_args; i++) { bool arena_arg = arena_base && (m->arg_flags[i] & BTF_FMODEL_ARENA_ARG); bool nullable = m->arg_flags[i] & BTF_FMODEL_NULLABLE_ARG; @@ -3130,8 +4065,36 @@ static void save_args(const struct btf_func_model *m, u8 **prog, * * the arg1-5,arg7 will be passed by regs, and arg6 will * by stack. + * + * An argument aligned to 16 bytes, an __int128 or a struct + * holding one, also starts at an even stack slot, which can + * leave a hole before it. bpf_jit_place_args() worked out + * where each eightbyte is. */ - if (nr_regs + arg_regs > 6) { + for (j = 0; j < arg_regs; j++, slot++, bargs_off -= 8) { + u8 pos = pos_of_slot[slot]; + u32 src; + + if (!x86_arg_on_stack(pos)) { + /* + * Only copy the arguments on-stack to + * current 'stack_size' and ignore the regs, + * used to prepare the arguments on-stack for + * origin call. + */ + if (for_call_origin) + continue; + + /* copy the arguments from regs into stack */ + src = x86_arg_reg[pos]; + if (arena_arg) { + emit_arena_arg_conv(prog, src, nullable, (u32)arena_base); + src = BPF_REG_0; + } + emit_stx(prog, BPF_DW, BPF_REG_FP, src, -bargs_off); + continue; + } + /* copy function arguments from origin stack frame * into current stack frame. * @@ -3141,53 +4104,33 @@ static void save_args(const struct btf_func_model *m, u8 **prog, * entered through the fentry call, so rbp + 24, and * a single one when it is entered with a jmp or * called indirectly, so rbp + 16. + * + * The area for the origin call mirrors the incoming + * one, hole and all; only the bpf program takes the + * arguments packed. */ - for (j = 0; j < arg_regs; j++) { - emit_ldx(prog, BPF_DW, BPF_REG_0, BPF_REG_FP, - nr_stack_slots * 8 + stack_args_off); - if (arena_arg) - emit_arena_arg_conv(prog, BPF_REG_0, nullable, - (u32)arena_base); - emit_stx(prog, BPF_DW, BPF_REG_FP, BPF_REG_0, - -stack_size); + off = x86_arg_stack_off(pos); + doff = for_call_origin ? stack_size - off : bargs_off; + emit_ldx(prog, BPF_DW, BPF_REG_0, BPF_REG_FP, + stack_args_off + off); + if (arena_arg) + emit_arena_arg_conv(prog, BPF_REG_0, nullable, + (u32)arena_base); + emit_stx(prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -doff); - if (!nr_stack_slots) - first_off = stack_size; - stack_size -= 8; - nr_stack_slots++; - } - } else { - /* Only copy the arguments on-stack to current - * 'stack_size' and ignore the regs, used to - * prepare the arguments on-stack for origin call. - */ - if (for_call_origin) { - nr_regs += arg_regs; - continue; - } - - /* copy the arguments from regs into stack */ - for (j = 0; j < arg_regs; j++) { - u32 src = nr_regs == 5 ? X86_REG_R9 : BPF_REG_1 + nr_regs; - - if (arena_arg) { - emit_arena_arg_conv(prog, src, nullable, (u32)arena_base); - src = BPF_REG_0; - } - emit_stx(prog, BPF_DW, BPF_REG_FP, src, -stack_size); - stack_size -= 8; - nr_regs++; - } + if (!nr_stack_slots) + first_off = doff; + nr_stack_slots++; } } clean_stack_garbage(m, prog, nr_stack_slots, first_off); } -static void restore_regs(const struct btf_func_model *m, u8 **prog, +static void restore_regs(u8 **prog, const u8 *pos_of_slot, int nr_slots, int stack_size) { - int i, j, arg_regs, nr_regs = 0; + int slot; /* Restore function arguments from stack. * For a function that accepts two pointers the sequence will be: @@ -3196,23 +4139,10 @@ static void restore_regs(const struct btf_func_model *m, u8 **prog, * * The logic here is similar to what we do in save_args() */ - for (i = 0; i < min_t(int, m->nr_args, MAX_BPF_FUNC_ARGS); i++) { - arg_regs = (m->arg_size[i] + 7) / 8; - if (nr_regs + arg_regs <= 6) { - for (j = 0; j < arg_regs; j++) { - emit_ldx(prog, BPF_DW, - nr_regs == 5 ? X86_REG_R9 : BPF_REG_1 + nr_regs, - BPF_REG_FP, - -stack_size); - stack_size -= 8; - nr_regs++; - } - } else { - stack_size -= 8 * arg_regs; - } - - if (nr_regs >= 6) - break; + for (slot = 0; slot < nr_slots; slot++, stack_size -= 8) { + if (!x86_arg_on_stack(pos_of_slot[slot])) + emit_ldx(prog, BPF_DW, x86_arg_reg[pos_of_slot[slot]], + BPF_REG_FP, -stack_size); } } @@ -3401,9 +4331,9 @@ static int invoke_bpf_mod_ret(const struct btf_func_model *m, u8 **pprog, return 0; } -/* mov rax, qword ptr [rbp - rounded_stack_depth - 8] */ -#define LOAD_TRAMP_TAIL_CALL_CNT_PTR(stack) \ - __LOAD_TCC_PTR(-round_up(stack, 8) - 8) +/* mov rax, qword ptr [rbp - off] */ +#define LOAD_TRAMP_TAIL_CALL_CNT_PTR(off) \ + __LOAD_TCC_PTR(-(off)) /* Example: * __be16 eth_type_trans(struct sk_buff *skb, struct net_device *dev); @@ -3471,8 +4401,10 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im struct bpf_tramp_nodes *tnodes, void *func_addr) { - int i, ret, nr_regs = m->nr_args, stack_size = 0; + int i, ret, nr_regs, stack_size = 0; + u8 pos_of_slot[MAX_BPF_FUNC_ARG_SLOTS]; int regs_off, func_meta_off, ip_off, run_ctx_off, arg_stack_off, rbx_off; + int tcc_off = 0; struct bpf_tramp_nodes *fentry = &tnodes[BPF_TRAMP_FENTRY]; struct bpf_tramp_nodes *fexit = &tnodes[BPF_TRAMP_FEXIT]; struct bpf_tramp_nodes *fmod_ret = &tnodes[BPF_TRAMP_MODIFY_RETURN]; @@ -3494,12 +4426,12 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im arena_base = bpf_tramp_arena_base(m, tnodes, flags); - for (i = 0; i < m->nr_args; i++) - nr_regs += (m->arg_size[i] + 7) / 8 - 1; - - /* x86-64 supports up to MAX_BPF_FUNC_ARGS arguments. 1-6 - * are passed through regs, the remains are through stack. + /* + * x86-64 supports up to MAX_BPF_FUNC_ARGS argument slots, six in + * registers and the rest on the stack, where bpf_jit_place_args() + * puts them. */ + nr_regs = bpf_jit_place_args(&x86_arg_abi, m, pos_of_slot); if (nr_regs > MAX_BPF_FUNC_ARGS) return -ENOTSUPP; @@ -3511,11 +4443,11 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im * RBP - 8 [ return value ] BPF_TRAMP_F_CALL_ORIG or * BPF_TRAMP_F_RET_FENTRY_RET flags * - * [ reg_argN ] always - * [ ... ] - * RBP - regs_off [ reg_arg1 ] program's ctx pointer + * [ arg slotN ] always, packed, whether + * [ ... ] in a register or on the + * RBP - regs_off [ arg slot1 ] stack; program's ctx pointer * - * RBP - func_meta_off [ regs count, etc ] always + * RBP - func_meta_off [ slot count, etc ] always * * RBP - ip_off [ traced function ] BPF_TRAMP_F_IP_ARG flag * @@ -3523,11 +4455,12 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im * * RBP - run_ctx_off [ bpf_tramp_run_ctx ] * - * [ stack_argN ] BPF_TRAMP_F_CALL_ORIG - * [ ... ] - * [ stack_arg2 ] - * RBP - arg_stack_off [ stack_arg1 ] - * RSP [ tail_call_cnt_ptr ] BPF_TRAMP_F_TAIL_CALL_CTX + * RBP - tcc_off [ tail_call_cnt_ptr ] BPF_TRAMP_F_TAIL_CALL_CTX + * + * [ stack slotN ] BPF_TRAMP_F_CALL_ORIG, + * [ ... ] the incoming stack + * [ stack slot2 ] arguments, alignment + * RBP - arg_stack_off [ stack slot1 ] holes included */ /* room for return value of orig_call or fentry prog */ @@ -3538,7 +4471,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im stack_size += nr_regs * 8; regs_off = stack_size; - /* function matedata, such as regs count */ + /* function matedata, such as the argument slot count */ stack_size += 8; func_meta_off = stack_size; @@ -3558,9 +4491,14 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im stack_size += (sizeof(struct bpf_tramp_run_ctx) + 7) & ~0x7; run_ctx_off = stack_size; + if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) { + stack_size += 8; + tcc_off = stack_size; + } + if (nr_regs > 6 && (flags & BPF_TRAMP_F_CALL_ORIG)) { /* the space that used to pass arguments on-stack */ - stack_size += (nr_regs - get_nr_used_regs(m)) * 8; + stack_size += get_nr_stack_slots(pos_of_slot, nr_regs) * 8; /* make sure the stack pointer is 16-byte aligned if we * need pass arguments on stack, which means * [stack_size + 8(rbp) + 8(rip) + 8(origin rip)] @@ -3607,20 +4545,16 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im if (im) im->ksym.fp_start = prog - (u8 *)rw_image; - if (!is_imm8(stack_size)) { - /* sub rsp, stack_size */ - EMIT3_off32(0x48, 0x81, 0xEC, stack_size); - } else { - /* sub rsp, stack_size */ - EMIT4(0x48, 0x83, 0xEC, stack_size); - } + /* sub rsp, stack_size */ + emit_sub_rsp(&prog, stack_size); if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) - EMIT1(0x50); /* push rax */ + /* mov QWORD PTR [rbp - tcc_off], rax */ + emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -tcc_off); /* mov QWORD PTR [rbp - rbx_off], rbx */ emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_6, -rbx_off); func_meta = nr_regs; - /* Store number of argument registers of the traced function */ + /* Store number of argument slots of the traced function */ emit_store_stack_imm64(&prog, BPF_REG_0, -func_meta_off, func_meta); if (flags & BPF_TRAMP_F_IP_ARG) { @@ -3628,7 +4562,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im emit_store_stack_imm64(&prog, BPF_REG_0, -ip_off, (long)func_addr); } - save_args(m, &prog, regs_off, false, flags, arena_base); + save_args(m, &prog, pos_of_slot, regs_off, false, flags, arena_base); if (flags & BPF_TRAMP_F_CALL_ORIG) { /* arg1: mov rdi, im */ @@ -3669,14 +4603,14 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im } if (flags & BPF_TRAMP_F_CALL_ORIG) { - restore_regs(m, &prog, regs_off); - save_args(m, &prog, arg_stack_off, true, flags, 0); + restore_regs(&prog, pos_of_slot, nr_regs, regs_off); + save_args(m, &prog, pos_of_slot, arg_stack_off, true, flags, 0); if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) { /* Before calling the original function, load the * tail_call_cnt_ptr from stack to rax. */ - LOAD_TRAMP_TAIL_CALL_CNT_PTR(stack_size); + LOAD_TRAMP_TAIL_CALL_CNT_PTR(tcc_off); } if (flags & BPF_TRAMP_F_ORIG_STACK) { @@ -3723,7 +4657,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im } if (flags & BPF_TRAMP_F_RESTORE_REGS) - restore_regs(m, &prog, regs_off); + restore_regs(&prog, pos_of_slot, nr_regs, regs_off); /* This needs to be done regardless. If there were fmod_ret programs, * the return value is only updated on the stack and still needs to be @@ -3740,7 +4674,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im /* Before running the original function, load the * tail_call_cnt_ptr from stack to rax. */ - LOAD_TRAMP_TAIL_CALL_CNT_PTR(stack_size); + LOAD_TRAMP_TAIL_CALL_CNT_PTR(tcc_off); } /* restore return value of orig_call or fentry prog back into RAX */ @@ -3755,7 +4689,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im if (flags & BPF_TRAMP_F_SKIP_FRAME) { /* skip our return address and return to parent */ - EMIT4(0x48, 0x83, 0xC4, 8); /* add rsp, 8 */ + emit_add_rsp(&prog, 8); /* add rsp, 8 */ } emit_return(&prog, image + (prog - (u8 *)rw_image)); /* Make sure the trampoline generation logic doesn't overflow */ @@ -3818,24 +4752,14 @@ int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, struct bpf_tramp_nodes *tnodes, void *func_addr) { struct bpf_tramp_image im = {}; - void *image; - int ret; - /* Allocate a temporary buffer for __arch_prepare_bpf_trampoline(). - * - * We cannot use kvmalloc here, because we need image to be in - * module memory range. - * Since it must be writable use execmem_alloc(EXECMEM_MODULE_DATA) - * that returns writable memory in the module address space. - */ - image = execmem_alloc(EXECMEM_MODULE_DATA, PAGE_SIZE); - if (!image) + if (!trampoline_size_image) return -ENOMEM; - ret = __arch_prepare_bpf_trampoline(&im, image, image + PAGE_SIZE, image, - m, flags, tnodes, func_addr); - execmem_free(image); - return ret; + return __arch_prepare_bpf_trampoline(&im, trampoline_size_image, + trampoline_size_image + PAGE_SIZE, + trampoline_size_image, m, flags, + tnodes, func_addr); } static int emit_bpf_dispatcher(u8 **pprog, int a, int b, s64 *progs, u8 *image, u8 *buf) @@ -4021,6 +4945,22 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_pr padding = true; goto skip_init_addrs; } + + /* + * An arena fetching AND/OR/XOR is lowered to a CMPXCHG loop whose load + * and CMPXCHG can both fault, one more entry than the verifier reserved. + * The extra pass of a program with subprograms returned above, so the + * count is adjusted once. + */ + for (i = 0; prog->aux->arena && i < prog->len; i++) { + const struct bpf_insn *insn = &prog->insnsi[i]; + + if (BPF_CLASS(insn->code) == BPF_STX && + BPF_MODE(insn->code) == BPF_PROBE_ATOMIC && + is_atomic_fetch_op(insn)) + prog->aux->num_exentries++; + } + addrs = kvmalloc_objs(*addrs, prog->len + 1); if (!addrs) goto out_addrs; @@ -4075,7 +5015,9 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_pr * The number of entries in extable is the number of BPF_LDX * insns that access kernel memory via "pointer to BTF type". * The verifier changed their opcode from LDX|MEM|size - * to LDX|PROBE_MEM|size to make JITing easier. + * to LDX|PROBE_MEM|size to make JITing easier. Arena + * accesses add one each, and an arena fetching AND/OR/XOR + * one more, see the adjustment of num_exentries above. */ u32 align = __alignof__(struct exception_table_entry); u32 extable_size = prog->aux->num_exentries * @@ -4159,11 +5101,36 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_pr return prog; } +bool bpf_jit_supports_callx(void) +{ + /* + * FineIBT poisons ENDBR at the entry of a JITed function and expects + * indirect callers to go through the CFI preamble instead. + * callx doesn't do that yet. + */ + return cfi_mode != CFI_FINEIBT; +} + bool bpf_jit_supports_kfunc_call(void) { return true; } +bool bpf_jit_supports_kfunc_ret_reg_pair(void) +{ + return true; +} + +bool bpf_jit_supports_unwind(void) +{ + return true; +} + +const struct bpf_jit_arg_abi *bpf_jit_arg_abi(void) +{ + return &x86_arg_abi; +} + bool bpf_jit_supports_stack_args(void) { return true; @@ -4187,6 +5154,18 @@ bool bpf_jit_supports_subprog_tailcalls(void) return true; } +/* + * Frame sizes are 32-bit immediates in the prologue, epilogue and tail call + * sequences, a tail call pops the caller's frame and lands in the target's + * prologue before the target allocates its own, and private stacks are + * allocated from the program's own depth, so MAX_BPF_STACK_JIT frames need + * nothing special. + */ +bool bpf_jit_supports_large_stack(void) +{ + return true; +} + bool bpf_jit_supports_percpu_insn(void) { return true; @@ -4310,18 +5289,13 @@ bool bpf_jit_supports_arena(void) return true; } +bool bpf_jit_supports_arena_scalar(void) +{ + return true; +} + bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena) { - if (!in_arena) - return true; - switch (insn->code) { - case BPF_STX | BPF_ATOMIC | BPF_W: - case BPF_STX | BPF_ATOMIC | BPF_DW: - if (insn->imm == (BPF_AND | BPF_FETCH) || - insn->imm == (BPF_OR | BPF_FETCH) || - insn->imm == (BPF_XOR | BPF_FETCH)) - return false; - } return true; } diff --git a/arch/x86/net/bpf_timed_may_goto.S b/arch/x86/net/bpf_timed_may_goto.S index 54c690cae190..6c6e03f725a3 100644 --- a/arch/x86/net/bpf_timed_may_goto.S +++ b/arch/x86/net/bpf_timed_may_goto.S @@ -11,12 +11,6 @@ SYM_FUNC_START(arch_bpf_timed_may_goto) ANNOTATE_NOENDBR - /* - * r10 passes us stack depth, load the pointer to count and timestamp - * into r10 by adding it to BPF frame pointer. - */ - leaq (%rbp, %r10, 1), %r10 - /* Setup frame. */ pushq %rbp movq %rsp, %rbp @@ -30,8 +24,8 @@ SYM_FUNC_START(arch_bpf_timed_may_goto) pushq %r8 /* - * r10 has the pointer to count and timestamp, pass it as first - * argument. + * r10 has the pointer to count and timestamp. JIT added the frame + * pointer to the offset. Pass it as first argument. */ movq %r10, %rdi diff --git a/crypto/Makefile b/crypto/Makefile index 8386d55a9755..33bb5ad595e8 100644 --- a/crypto/Makefile +++ b/crypto/Makefile @@ -22,9 +22,6 @@ crypto_skcipher-y += lskcipher.o crypto_skcipher-y += skcipher.o obj-$(CONFIG_CRYPTO_SKCIPHER2) += crypto_skcipher.o -ifeq ($(CONFIG_BPF_SYSCALL),y) -obj-$(CONFIG_CRYPTO_SKCIPHER2) += bpf_crypto_skcipher.o -endif obj-$(CONFIG_CRYPTO_SEQIV) += seqiv.o obj-$(CONFIG_CRYPTO_ECHAINIV) += echainiv.o diff --git a/crypto/bpf_crypto_skcipher.c b/crypto/bpf_crypto_skcipher.c deleted file mode 100644 index a88798d3e8c8..000000000000 --- a/crypto/bpf_crypto_skcipher.c +++ /dev/null @@ -1,83 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* Copyright (c) 2024 Meta, Inc */ -#include -#include -#include -#include - -static void *bpf_crypto_lskcipher_alloc_tfm(const char *algo) -{ - return crypto_alloc_lskcipher(algo, 0, 0); -} - -static void bpf_crypto_lskcipher_free_tfm(void *tfm) -{ - crypto_free_lskcipher(tfm); -} - -static int bpf_crypto_lskcipher_has_algo(const char *algo) -{ - return crypto_has_skcipher(algo, CRYPTO_ALG_TYPE_LSKCIPHER, CRYPTO_ALG_TYPE_MASK); -} - -static int bpf_crypto_lskcipher_setkey(void *tfm, const u8 *key, unsigned int keylen) -{ - return crypto_lskcipher_setkey(tfm, key, keylen); -} - -static u32 bpf_crypto_lskcipher_get_flags(void *tfm) -{ - return crypto_lskcipher_get_flags(tfm); -} - -static unsigned int bpf_crypto_lskcipher_ivsize(void *tfm) -{ - return crypto_lskcipher_ivsize(tfm); -} - -static unsigned int bpf_crypto_lskcipher_statesize(void *tfm) -{ - return crypto_lskcipher_statesize(tfm); -} - -static int bpf_crypto_lskcipher_encrypt(void *tfm, const u8 *src, u8 *dst, - unsigned int len, u8 *siv) -{ - return crypto_lskcipher_encrypt(tfm, src, dst, len, siv); -} - -static int bpf_crypto_lskcipher_decrypt(void *tfm, const u8 *src, u8 *dst, - unsigned int len, u8 *siv) -{ - return crypto_lskcipher_decrypt(tfm, src, dst, len, siv); -} - -static const struct bpf_crypto_type bpf_crypto_lskcipher_type = { - .alloc_tfm = bpf_crypto_lskcipher_alloc_tfm, - .free_tfm = bpf_crypto_lskcipher_free_tfm, - .has_algo = bpf_crypto_lskcipher_has_algo, - .setkey = bpf_crypto_lskcipher_setkey, - .encrypt = bpf_crypto_lskcipher_encrypt, - .decrypt = bpf_crypto_lskcipher_decrypt, - .ivsize = bpf_crypto_lskcipher_ivsize, - .statesize = bpf_crypto_lskcipher_statesize, - .get_flags = bpf_crypto_lskcipher_get_flags, - .owner = THIS_MODULE, - .name = "skcipher", -}; - -static int __init bpf_crypto_skcipher_init(void) -{ - return bpf_crypto_register_type(&bpf_crypto_lskcipher_type); -} - -static void __exit bpf_crypto_skcipher_exit(void) -{ - int err = bpf_crypto_unregister_type(&bpf_crypto_lskcipher_type); - WARN_ON_ONCE(err); -} - -module_init(bpf_crypto_skcipher_init); -module_exit(bpf_crypto_skcipher_exit); -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("Symmetric key cipher support for BPF"); diff --git a/drivers/net/ethernet/netronome/nfp/bpf/verifier.c b/drivers/net/ethernet/netronome/nfp/bpf/verifier.c index 1caa87da72b5..9ddbfe9d6019 100644 --- a/drivers/net/ethernet/netronome/nfp/bpf/verifier.c +++ b/drivers/net/ethernet/netronome/nfp/bpf/verifier.c @@ -105,7 +105,7 @@ static bool nfp_bpf_map_update_value_ok(struct bpf_verifier_env *env) unsigned int soff; soff = -(off + i) - 1; - stack_entry = &state->stack[soff / BPF_REG_SIZE]; + stack_entry = bpf_stack_slot(state, soff / BPF_REG_SIZE); if (stack_entry->slot_type[soff % BPF_REG_SIZE] == STACK_ZERO) continue; diff --git a/fs/bpf_fs_kfuncs.c b/fs/bpf_fs_kfuncs.c index abdfbd83dc57..35ccd98a4de6 100644 --- a/fs/bpf_fs_kfuncs.c +++ b/fs/bpf_fs_kfuncs.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -328,6 +329,89 @@ __bpf_kfunc int bpf_remove_dentry_xattr(struct dentry *dentry, const char *name_ return ret; } +static int bpf_inode_init_xattrs_claimed(const struct xattr *xattrs, int xattr_count) +{ + const size_t suffix_len = sizeof(XATTR_BPF_LSM_SUFFIX) - 1; + int i, claimed = 0; + + for (i = 0; i < xattr_count; i++) { + const char *name = xattrs[i].name; + + if (name && !strncmp(name, XATTR_BPF_LSM_SUFFIX, suffix_len)) + claimed++; + } + return claimed; +} + +/** + * bpf_inode_init_xattr - attach a xattr to an inode that is being created + * @xattrs: xattr array the inode_init_security hook was handed + * @xattr_count__ctx_out: xattr count the inode_init_security hook was handed + * @name__str: name of the xattr + * @value_p: xattr value + * + * Claim one of the slots the BPF LSM reserved in the xattr array, so that + * the xattr is written out as part of the transaction creating the inode. + * + * For security reasons, only *name__str* with prefix "security.bpf." is + * allowed. The slot stores the name without that "security." prefix, which + * the filesystem's initxattrs() callback puts back. + * + * At most BPF_LSM_INODE_INIT_XATTRS xattrs can be attached to one inode, + * matching the number of slots the BPF LSM reserves. + * + * Return: 0 on success, a negative value on error. + */ +__bpf_kfunc int bpf_inode_init_xattr(struct xattr *xattrs, + int *xattr_count__ctx_out, + const char *name__str, + const struct bpf_dynptr *value_p) +{ + const struct bpf_dynptr_kern *value_ptr = (struct bpf_dynptr_kern *)value_p; + int *xattr_count = xattr_count__ctx_out; + const char *suffix; + struct xattr *slot; + const void *value; + size_t name_len; + u32 value_len; + char *buf; + + if (!match_security_bpf_prefix(name__str)) + return -EPERM; + if (bpf_inode_init_xattrs_claimed(xattrs, *xattr_count) >= + BPF_LSM_INODE_INIT_XATTRS) + return -ENOSPC; + + suffix = name__str + XATTR_SECURITY_PREFIX_LEN; + name_len = strlen(suffix); + if (name_len <= sizeof(XATTR_BPF_LSM_SUFFIX) - 1 || + name_len > XATTR_NAME_MAX - XATTR_SECURITY_PREFIX_LEN) + return -EINVAL; + + value_len = __bpf_dynptr_size(value_ptr); + value = __bpf_dynptr_data(value_ptr, value_len); + if (!value) + return -EINVAL; + if (value_len > XATTR_SIZE_MAX) + return -E2BIG; + + buf = kmalloc(value_len + name_len + 1, GFP_NOWAIT | __GFP_NOWARN); + if (!buf) + return -ENOMEM; + memcpy(buf, value, value_len); + memcpy(buf + value_len, suffix, name_len + 1); + + slot = lsm_get_xattr_slot(xattrs, xattr_count); + if (!slot) { + kfree(buf); + return -ENOSPC; + } + slot->value = buf; + slot->value_len = value_len; + slot->name = buf + value_len; + return 0; +} + #ifdef CONFIG_CGROUPS /** * bpf_cgroup_read_xattr - read xattr of a cgroup's node in cgroupfs @@ -425,6 +509,7 @@ BTF_ID_FLAGS(func, bpf_get_file_xattr, KF_SLEEPABLE) BTF_ID_FLAGS(func, bpf_set_dentry_xattr, KF_SLEEPABLE) BTF_ID_FLAGS(func, bpf_remove_dentry_xattr, KF_SLEEPABLE) BTF_ID_FLAGS(func, bpf_real_data_inode, KF_SLEEPABLE | KF_RET_NULL) +BTF_ID_FLAGS(func, bpf_inode_init_xattr) #ifdef CONFIG_NET BTF_ID_FLAGS(func, bpf_sock_read_xattr, KF_RCU) #endif @@ -436,10 +521,24 @@ BTF_ID(func, bpf_set_dentry_xattr) BTF_ID(func, bpf_remove_dentry_xattr) BTF_SET_END(bpf_fs_kfunc_lsm_only_ids) +BTF_ID_LIST_SINGLE(bpf_inode_init_xattr_ids, func, bpf_inode_init_xattr) + +BTF_SET_START(bpf_inode_init_xattr_hooks) +BTF_ID(func, bpf_lsm_inode_init_security) +BTF_SET_END(bpf_inode_init_xattr_hooks) + static int bpf_fs_kfuncs_filter(const struct bpf_prog *prog, u32 kfunc_id) { if (!btf_id_set8_contains(&bpf_fs_kfunc_set_ids, kfunc_id)) return 0; + if (kfunc_id == bpf_inode_init_xattr_ids[0]) { + if (prog->type != BPF_PROG_TYPE_LSM || + prog->expected_attach_type != BPF_LSM_MAC || + !btf_id_set_contains(&bpf_inode_init_xattr_hooks, + prog->aux->attach_btf_id)) + return -EACCES; + return 0; + } if (prog->type == BPF_PROG_TYPE_LSM) return 0; if (prog->type != BPF_PROG_TYPE_STRUCT_OPS) diff --git a/fs/exec.c b/fs/exec.c index 5d7b0fe32dbf..8d2b14e1d455 100644 --- a/fs/exec.c +++ b/fs/exec.c @@ -1471,9 +1471,12 @@ void bprm_drop_loader(struct linux_binprm *bprm) static void free_bprm(struct linux_binprm *bprm) { - if (bprm->mm) { + struct mm_struct *mm = bprm->mm; + + if (mm) { acct_arg_size(bprm, 0); - mmput(bprm->mm); + bprm->mm = NULL; + mmput(mm); } if (bprm->user_ns) put_user_ns(bprm->user_ns); diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h index 7a3e9f00d628..3eb732942d69 100644 --- a/include/asm-generic/vmlinux.lds.h +++ b/include/asm-generic/vmlinux.lds.h @@ -675,12 +675,43 @@ /* * .BTF */ +#ifdef CONFIG_DEBUG_INFO_BTF_INLINE +#define BTF_INLINE \ + . = ALIGN(PAGE_SIZE); \ + .BTF.inline : AT(ADDR(.BTF.inline) - LOAD_OFFSET) { \ + BOUNDED_SECTION_BY(.BTF.inline, _BTF_inline) \ + } +#else +#define BTF_INLINE +#endif + +#ifdef CONFIG_DEBUG_INFO_BTF_INLINE_MODULE +/* __MODULE_NAME_LEN + SHA-256 digest + u32 BTF size */ +#ifdef CONFIG_64BIT +#define BTF_LINK_SIZE (64 - 8 + 32 + 4) +#else +#define BTF_LINK_SIZE (64 - 4 + 32 + 4) +#endif +#define BTF_INLINE_LINK \ + . = ALIGN(PAGE_SIZE); \ + .BTF.inline.link : AT(ADDR(.BTF.inline.link) - LOAD_OFFSET) { \ + __start_BTF_inline_link = .; \ + BYTE(0) \ + . += BTF_LINK_SIZE - 1; \ + __stop_BTF_inline_link = .; \ + } +#else +#define BTF_INLINE_LINK +#endif + #ifdef CONFIG_DEBUG_INFO_BTF #define BTF \ . = ALIGN(PAGE_SIZE); \ .BTF : AT(ADDR(.BTF) - LOAD_OFFSET) { \ BOUNDED_SECTION_BY(.BTF, _BTF) \ } \ + BTF_INLINE \ + BTF_INLINE_LINK \ . = ALIGN(PAGE_SIZE); \ .BTF_ids : AT(ADDR(.BTF_ids) - LOAD_OFFSET) { \ *(.BTF_ids) \ diff --git a/include/linux/bpf-cgroup-defs.h b/include/linux/bpf-cgroup-defs.h index c9e6b26abab6..0147b8bec973 100644 --- a/include/linux/bpf-cgroup-defs.h +++ b/include/linux/bpf-cgroup-defs.h @@ -47,6 +47,7 @@ enum cgroup_bpf_attach_type { CGROUP_INET6_GETSOCKNAME, CGROUP_UNIX_GETSOCKNAME, CGROUP_INET_SOCK_RELEASE, + CGROUP_TCP_SOCK_OPS, CGROUP_LSM_START, CGROUP_LSM_END = CGROUP_LSM_START + CGROUP_LSM_NUM - 1, MAX_CGROUP_BPF_ATTACH_TYPE diff --git a/include/linux/bpf-cgroup.h b/include/linux/bpf-cgroup.h index 4d0cc65976a1..02ef7f899b3d 100644 --- a/include/linux/bpf-cgroup.h +++ b/include/linux/bpf-cgroup.h @@ -100,6 +100,8 @@ struct bpf_cgroup_storage { struct bpf_cgroup_link { struct bpf_link link; struct cgroup *cgroup; + struct bpf_map *map; + wait_queue_head_t wait_hup; }; struct bpf_prog_list { @@ -110,6 +112,18 @@ struct bpf_prog_list { u32 flags; }; +#define bpf_cgroup_struct_ops_foreach(var, item, cgrp, atype) \ + for (item = rcu_dereference((cgrp)->bpf.effective[atype])->items;\ + ((var) = READ_ONCE(item->kdata)); \ + item++) + +static inline bool cgroup_bpf_is_struct_ops_atype(enum cgroup_bpf_attach_type atype) +{ + return atype == CGROUP_TCP_SOCK_OPS; +} +void cgroup_bpf_struct_ops_register(int atype, u32 type_id, void *cfi_stubs, bool mult_trace); +int cgroup_bpf_struct_ops_attach(struct bpf_map *map, const union bpf_attr *attr); + void __init cgroup_bpf_lifetime_notifier_init(void); int __cgroup_bpf_run_filter_skb(struct sock *sk, @@ -331,27 +345,19 @@ static inline bool cgroup_bpf_sock_enabled(struct sock *sk, * calling bpf_setsockopt on listener-sk will not make sense anyway, * so passing 'sock_ops->sk == req_sk' to the bpf prog is appropriate here. */ -#define BPF_CGROUP_RUN_PROG_SOCK_OPS_SK(sock_ops, sk) \ -({ \ - int __ret = 0; \ - if (cgroup_bpf_enabled(CGROUP_SOCK_OPS)) \ - __ret = __cgroup_bpf_run_filter_sock_ops(sk, \ - sock_ops, \ - CGROUP_SOCK_OPS); \ - __ret; \ -}) +#define __BPF_CGROUP_RUN_PROG_SOCK_OPS_SK(sock_ops, sk) \ + __cgroup_bpf_run_filter_sock_ops(sk, sock_ops, \ + CGROUP_SOCK_OPS) -#define BPF_CGROUP_RUN_PROG_SOCK_OPS(sock_ops) \ -({ \ - int __ret = 0; \ - if (cgroup_bpf_enabled(CGROUP_SOCK_OPS) && (sock_ops)->sk) { \ - typeof(sk) __sk = sk_to_full_sk((sock_ops)->sk); \ - if (__sk && sk_fullsock(__sk)) \ - __ret = __cgroup_bpf_run_filter_sock_ops(__sk, \ - sock_ops, \ - CGROUP_SOCK_OPS); \ - } \ - __ret; \ +#define __BPF_CGROUP_RUN_PROG_SOCK_OPS(sock_ops) \ +({ \ + int __ret = 0; \ + typeof(sk) __sk = sk_to_full_sk((sock_ops)->sk); \ + if (__sk && sk_fullsock(__sk)) \ + __ret = __cgroup_bpf_run_filter_sock_ops(__sk, \ + sock_ops, \ + CGROUP_SOCK_OPS); \ + __ret; \ }) #define BPF_CGROUP_RUN_PROG_DEVICE_CGROUP(atype, major, minor, access) \ @@ -479,6 +485,20 @@ static inline int bpf_percpu_cgroup_storage_update(struct bpf_map *map, return 0; } +static inline bool cgroup_bpf_is_struct_ops_atype(int atype) +{ + return false; +} +static inline void cgroup_bpf_struct_ops_register(int atype, u32 type_id, void *cfi_stubs, + bool mult_trace) +{ +} +static inline int cgroup_bpf_struct_ops_attach(struct bpf_map *map, + const union bpf_attr *attr) +{ + return -EOPNOTSUPP; +} + #define cgroup_bpf_enabled(atype) (0) #define BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, uaddrlen, atype, t_ctx) ({ 0; }) #define BPF_CGROUP_RUN_SA_PROG(sk, uaddr, uaddrlen, atype) ({ 0; }) @@ -501,7 +521,7 @@ static inline int bpf_percpu_cgroup_storage_update(struct bpf_map *map, #define BPF_CGROUP_RUN_PROG_UDP4_RECVMSG_LOCK(sk, uaddr, uaddrlen) ({ 0; }) #define BPF_CGROUP_RUN_PROG_UDP6_RECVMSG_LOCK(sk, uaddr, uaddrlen) ({ 0; }) #define BPF_CGROUP_RUN_PROG_UNIX_RECVMSG_LOCK(sk, uaddr, uaddrlen) ({ 0; }) -#define BPF_CGROUP_RUN_PROG_SOCK_OPS(sock_ops) ({ 0; }) +#define __BPF_CGROUP_RUN_PROG_SOCK_OPS(sock_ops) ({ 0; }) #define BPF_CGROUP_RUN_PROG_DEVICE_CGROUP(atype, major, minor, access) ({ 0; }) #define BPF_CGROUP_RUN_PROG_SYSCTL(head,table,write,buf,count,pos) ({ 0; }) #define BPF_CGROUP_RUN_PROG_GETSOCKOPT(sock, level, optname, optval, \ diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 0ecb9418dfbd..580f89c450b5 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -90,6 +91,7 @@ struct bpf_map_ops { struct bpf_map *(*map_alloc)(union bpf_attr *attr); void (*map_release)(struct bpf_map *map, struct file *map_file); void (*map_free)(struct bpf_map *map); + void (*map_free_pre_rcu)(struct bpf_map *map); int (*map_get_next_key)(struct bpf_map *map, void *key, void *next_key); void (*map_release_uref)(struct bpf_map *map); void *(*map_lookup_elem_sys_only)(struct bpf_map *map, void *key); @@ -214,6 +216,7 @@ enum btf_field_type { BPF_UPTR = (1 << 11), BPF_RES_SPIN_LOCK = (1 << 12), BPF_TASK_WORK = (1 << 13), + BPF_RCU_HEAD = (1 << 14), }; enum bpf_cgroup_storage_type { @@ -268,6 +271,7 @@ struct btf_record { int wq_off; int refcount_off; int task_work_off; + int rcu_head_off; struct btf_field fields[]; }; @@ -337,12 +341,23 @@ struct bpf_map { bool frozen; /* write-once; write-protected by freeze_mutex */ bool free_after_mult_rcu_gp; bool free_after_rcu_gp; + bool was_inner_map; atomic64_t sleepable_refcnt; s64 __percpu *elem_count; u64 cookie; /* write-once */ char *excl_prog_sha; + /* + * Which programs use the map, see bpf_map_claim(): 0 - none so far, + * aux of the program - only that one, the same with BPF_MAP_USER_PATCHED + * set - only that one and it stored the addresses of its functions into + * the map, BPF_MAP_USER_MANY - more than one. + */ + unsigned long user; }; +#define BPF_MAP_USER_MANY 1UL +#define BPF_MAP_USER_PATCHED 1UL + static inline const char *btf_field_type_name(enum btf_field_type type) { switch (type) { @@ -373,6 +388,8 @@ static inline const char *btf_field_type_name(enum btf_field_type type) return "bpf_refcount"; case BPF_TASK_WORK: return "bpf_task_work"; + case BPF_RCU_HEAD: + return "bpf_rcu_head"; default: WARN_ON_ONCE(1); return "unknown"; @@ -413,6 +430,8 @@ static inline u32 btf_field_type_size(enum btf_field_type type) return sizeof(struct bpf_refcount); case BPF_TASK_WORK: return sizeof(struct bpf_task_work); + case BPF_RCU_HEAD: + return sizeof(struct bpf_rcu_head); default: WARN_ON_ONCE(1); return 0; @@ -447,6 +466,8 @@ static inline u32 btf_field_type_align(enum btf_field_type type) return __alignof__(struct bpf_refcount); case BPF_TASK_WORK: return __alignof__(struct bpf_task_work); + case BPF_RCU_HEAD: + return __alignof__(struct bpf_rcu_head); default: WARN_ON_ONCE(1); return 0; @@ -479,6 +500,7 @@ static inline void bpf_obj_init_field(const struct btf_field *field, void *addr) case BPF_KPTR_PERCPU: case BPF_UPTR: case BPF_TASK_WORK: + case BPF_RCU_HEAD: break; default: WARN_ON_ONCE(1); @@ -572,7 +594,7 @@ static inline void bpf_obj_memcpy(struct btf_record *rec, if (long_memcpy) bpf_long_memcpy(dst, src, size); else - memcpy(dst, src, size); + data_race(memcpy(dst, src, size)); return; } @@ -580,10 +602,10 @@ static inline void bpf_obj_memcpy(struct btf_record *rec, u32 next_off = rec->fields[i].offset; u32 sz = next_off - curr_off; - memcpy(dst + curr_off, src + curr_off, sz); + data_race(memcpy(dst + curr_off, src + curr_off, sz)); curr_off += rec->fields[i].size + sz; } - memcpy(dst + curr_off, src + curr_off, size - curr_off); + data_race(memcpy(dst + curr_off, src + curr_off, size - curr_off)); } static inline void copy_map_value(struct bpf_map *map, void *dst, void *src) @@ -780,7 +802,10 @@ enum bpf_type_flag { */ PTR_UNTRUSTED = BIT(6 + BPF_BASE_TYPE_BITS), - /* MEM can be uninitialized. */ + /* + * MEM can be uninitialized. Generic memory outputs need not be fully + * initialized by the callee. + */ MEM_UNINIT = BIT(7 + BPF_BASE_TYPE_BITS), /* DYNPTR points to memory local to the bpf program. */ @@ -874,7 +899,7 @@ enum bpf_type_flag { /* function argument constraints */ enum bpf_arg_type { - ARG_DONTCARE = 0, /* unused argument in helper function */ + ARG_UNUSED = 0, /* unused argument; terminates argument iteration */ /* the following constraints used to prototype * bpf_map_lookup/update/delete_elem() functions @@ -909,6 +934,24 @@ enum bpf_arg_type { ARG_PTR_TO_TIMER, /* pointer to bpf_timer */ ARG_KPTR_XCHG_DEST, /* pointer to destination that kptrs are bpf_kptr_xchg'd into */ ARG_PTR_TO_DYNPTR, /* pointer to bpf_dynptr. See bpf_type_flag for dynptr type */ + + ARG_CONST_SCALAR, /* scalar known at verification time */ + ARG_CONST_MEM_SIZE, /* ARG_MEM_SIZE that must be constant */ + ARG_PTR_TO_ALLOC_BTF_ID, /* pointer to an allocated object */ + ARG_PTR_TO_REFCOUNTED_KPTR, /* pointer to a refcounted local kptr */ + ARG_PTR_TO_ITER, /* pointer to an iterator */ + ARG_PTR_TO_LIST_HEAD, /* pointer to bpf_list_head */ + ARG_PTR_TO_LIST_NODE, /* pointer to bpf_list_node */ + ARG_PTR_TO_RB_ROOT, /* pointer to bpf_rb_root */ + ARG_PTR_TO_RB_NODE, /* pointer to bpf_rb_node */ + ARG_PTR_TO_WORKQUEUE, /* pointer to bpf_wq */ + ARG_PTR_TO_TASK_WORK, /* pointer to bpf_task_work */ + ARG_PTR_TO_RCU_HEAD, /* pointer to bpf_rcu_head */ + ARG_PTR_TO_IRQ_FLAG, /* pointer to saved IRQ flags on the stack */ + ARG_PTR_TO_RES_SPIN_LOCK, /* pointer to bpf_res_spin_lock */ + ARG_PTR_TO_CTX_OUT, /* hook output argument passed through from ctx */ + ARG_PTR_TO_PROG_AUX, /* pointer to the caller's bpf_prog_aux */ + ARG_IGNORE, /* argument the verifier does not check at all */ __BPF_ARG_TYPE_MAX, /* Extended arg_types. */ @@ -977,6 +1020,13 @@ static_assert(__BPF_RET_TYPE_MAX <= BPF_BASE_TYPE_LIMIT); */ #define MAX_BPF_FUNC_REG_ARGS 5 +/* A by-value argument takes two eightbytes at most, so the maximum number of + * argument slots of any function is 2 * MAX_BPF_FUNC_ARGS. A local array may + * need that size for processing, although eventually the maximum slots will + * be capped at MAX_BPF_FUNC_ARGS. + */ +#define MAX_BPF_FUNC_ARG_SLOTS (2 * MAX_BPF_FUNC_ARGS) + /* eBPF function prototype used by verifier to allow BPF_CALLs from eBPF programs * to in-kernel helper functions and for adjusting imm32 field in BPF_CALL * instructions after verifying @@ -1005,13 +1055,13 @@ struct bpf_func_proto { }; union { struct { - u32 *arg1_btf_id; - u32 *arg2_btf_id; - u32 *arg3_btf_id; - u32 *arg4_btf_id; - u32 *arg5_btf_id; + const u32 *arg1_btf_id; + const u32 *arg2_btf_id; + const u32 *arg3_btf_id; + const u32 *arg4_btf_id; + const u32 *arg5_btf_id; }; - u32 *arg_btf_id[MAX_BPF_FUNC_ARGS]; + const u32 *arg_btf_id[MAX_BPF_FUNC_ARGS]; struct { size_t arg1_size; size_t arg2_size; @@ -1114,6 +1164,7 @@ struct bpf_insn_access_aux { u32 ref_id; }; }; + u32 mem_size; struct bpf_verifier_log *log; /* for verbose logs */ bool is_retval; /* is accessing function return value ? */ }; @@ -1194,6 +1245,9 @@ struct bpf_prog_offload { u32 jited_len; }; +/* The argument is aligned to 16 bytes. */ +#define BTF_FMODEL_ALIGN16_ARG BIT(0) + /* The argument is signed. */ #define BTF_FMODEL_SIGNED_ARG BIT(1) @@ -1211,6 +1265,11 @@ struct btf_func_model { u8 arg_flags[MAX_BPF_FUNC_ARGS]; }; +static inline u32 btf_func_model_arg_slots(const struct btf_func_model *m, u32 arg) +{ + return (m->arg_size[arg] + sizeof(u64) - 1) / sizeof(u64); +} + /* Restore arguments before returning from trampoline to let original function * continue executing. This flag is used for fentry progs when there are no * fexit progs. @@ -1684,12 +1743,21 @@ static inline void bpf_trampoline_multi_detach(struct bpf_prog *prog, static inline void bpf_trampoline_set_flags(struct bpf_trampoline *tr, u32 flags) {} #endif +/* The contexts that a global function is called in, and is verified for. */ +enum bpf_func_context { + BPF_FUNC_CTX_NON_SLEEPABLE, + BPF_FUNC_CTX_SLEEPABLE, + /* Sleepable code has disabled preemption or interrupts: no RCU read lock. */ + BPF_FUNC_CTX_NO_RCU_LOCK, + BPF_FUNC_CTX_CNT, +}; + struct bpf_func_info_aux { u16 linkage; bool unreliable; - /* Indexed by in_sleepable. */ - bool called[2]; - bool verified[2]; + bool called[BPF_FUNC_CTX_CNT]; + bool verified[BPF_FUNC_CTX_CNT]; + bool called_with_pkt_ptrs; }; enum bpf_jit_poke_reason { @@ -1721,6 +1789,7 @@ struct bpf_ctx_arg_aux { struct btf *btf; u32 btf_id; u32 ref_id; + u32 mem_size; bool refcounted; }; @@ -1749,12 +1818,18 @@ enum { }; struct bpf_stream { - atomic_t capacity; + refcount_t refcnt; + atomic_t capacity; /* bytes reserved against the stream limit */ + atomic_t readable; /* published bytes available to readers */ struct llist_head log; /* list of in-flight stream elements in LIFO order */ struct mutex lock; /* lock protecting backlog_{head,tail} */ struct llist_node *backlog_head; /* list of in-flight stream elements in FIFO order */ struct llist_node *backlog_tail; /* tail of the list above */ + wait_queue_head_t waitq; + struct irq_work notify_work; + bool notify_used; /* notify_work was queued at least once */ + bool dead; }; struct bpf_stream_stage { @@ -1773,6 +1848,7 @@ enum bpf_sig_keyring { BPF_SIG_KEYRING_SECONDARY, BPF_SIG_KEYRING_PLATFORM, BPF_SIG_KEYRING_USER, + BPF_SIG_KEYRING_BPF, }; struct bpf_prog_aux { @@ -1792,6 +1868,9 @@ struct bpf_prog_aux { u32 ctx_arg_info_size; u32 max_rdonly_access; u32 max_rdwr_access; + /* [min, max): the bytes of a rdwr buffer that may be written */ + u32 min_rdwr_write; + u32 max_rdwr_write; u32 subprog_start; struct btf *attach_btf; struct bpf_ctx_arg_aux *ctx_arg_info; @@ -1807,24 +1886,28 @@ struct bpf_prog_aux { bool attach_btf_trace; /* true if attaching to BTF-enabled raw tp */ bool attach_tracing_prog; /* true if tracing another tracing program */ bool tramp_linked; /* true if it was ever linked to a trampoline */ + bool sleepable_async_cb; /* has a bpf_wq or bpf_task_work callback */ bool func_proto_unreliable; bool tail_call_reachable; bool xdp_has_frags; bool exception_cb; bool exception_boundary; - bool is_extended; /* true if extended by freplace program */ bool jits_use_priv_stack; bool priv_stack_requested; bool changes_pkt_data; + bool has_prologue; bool might_sleep; + bool can_unwind; bool kprobe_write_ctx; + /* has insns that work with the arena base, see bpf_prog_has_arena_insns() */ + bool uses_arena_insns; struct { s32 keyring_serial; u8 keyring_type; u8 verdict; } sig; u64 prog_array_member_cnt; /* counts how many times as member of prog_array */ - struct mutex ext_mutex; /* mutex for is_extended and prog_array_member_cnt */ + struct mutex ext_mutex; /* mutex for freplace_link_cnt and prog_array_member_cnt */ struct bpf_arena *arena; void (*recursion_detected)(struct bpf_prog *prog); /* callback if recursion is detected */ /* BTF_KIND_FUNC_PROTO for valid attach_btf_id */ @@ -1856,6 +1939,7 @@ struct bpf_prog_aux { char name[BPF_OBJ_NAME_LEN]; u64 (*bpf_exception_cb)(u64 cookie, u64 sp, u64 bp, u64, u64); u16 stack_arg_sp_adjust; + u16 freplace_link_cnt; /* counts freplace links extending this prog */ #ifdef CONFIG_SECURITY void *security; #endif @@ -1892,8 +1976,9 @@ struct bpf_prog_aux { union { struct work_struct work; struct rcu_head rcu; + struct rcu_work rwork; }; - struct bpf_stream stream[2]; + struct bpf_stream *stream[2]; struct mutex st_ops_assoc_mutex; struct bpf_map __rcu *st_ops_assoc; }; @@ -1940,6 +2025,15 @@ struct bpf_prog { }; }; +/* + * Does the program, or a callback of it, run under RCU Tasks Trace protection + * only, outside of any RCU read side critical section? + */ +static inline bool bpf_prog_runs_sleepable(const struct bpf_prog *prog) +{ + return prog->sleepable || prog->aux->sleepable_async_cb; +} + struct bpf_array_aux { /* Programs with direct jumps into programs part of this array. */ struct list_head poke_progs; @@ -2137,6 +2231,18 @@ struct btf_member; * unloaded while in use. * @name: The name of the struct bpf_struct_ops object. * @func_models: Func models + * @cgroup_atype: A value in enum cgroup_bpf_attach_type for cgroup attachment. + * 0 means the struct_ops type does not support cgroup attachment. + * If cgroup_atype is non-zero, the @reg and @unreg must be NULL + * because the attachment/detachment will be handled by the bpf core. + * @free_after_tasks_rcu_gp: Set to true if it needs the bpf core to wait for + * a tasks_rcu gp before freeing the struct_ops map + * and its progs. It is unnecessary if the @unreg + * has waited for the correct rcu gp or the @unreg + * has ensured all struct_ops prog has finished running. + * @free_after_mult_rcu_gp: Same as @free_after_tasks_rcu_gp but waiting for + * both tasks_trace_rcu and regular rcu grace period. + * It is usually needed if the struct_ops has sleepable prog. */ struct bpf_struct_ops { const struct bpf_verifier_ops *verifier_ops; @@ -2155,6 +2261,9 @@ struct bpf_struct_ops { struct module *owner; const char *name; struct btf_func_model func_models[BPF_STRUCT_OPS_MAX_NR_MEMBERS]; + int cgroup_atype; + bool free_after_tasks_rcu_gp; + bool free_after_mult_rcu_gp; }; /* Every member of a struct_ops type has an instance even a member is not @@ -2289,6 +2398,12 @@ u32 bpf_struct_ops_id(const void *kdata); int bpf_struct_ops_for_each_prog(const void *kdata, int (*cb)(struct bpf_prog *prog, void *data), void *data); +void *bpf_struct_ops_map_kdata(struct bpf_map *map); +void *bpf_struct_ops_map_cfi_stubs(struct bpf_map *map); +bool bpf_struct_ops_valid_to_reg(struct bpf_map *map); +int bpf_struct_ops_link_update_check(struct bpf_map *new_map, struct bpf_map *old_map, + struct bpf_map *expected_old_map); +int bpf_struct_ops_map_cgroup_atype(struct bpf_map *map); #ifdef CONFIG_NET /* Define it here to avoid the use of forward declaration */ @@ -2353,6 +2468,32 @@ static inline void bpf_map_struct_ops_info_fill(struct bpf_map_info *info, struc static inline void bpf_struct_ops_desc_release(struct bpf_struct_ops_desc *st_ops_desc) { } +static inline u32 bpf_struct_ops_id(void *kdata) +{ + return 0; +} +static inline void *bpf_struct_ops_map_kdata(struct bpf_map *map) +{ + return NULL; +} +static inline int bpf_struct_ops_map_cgroup_atype(struct bpf_map *map) +{ + return 0; +} +static inline void *bpf_struct_ops_map_cfi_stubs(struct bpf_map *map) +{ + return NULL; +} +static inline bool bpf_struct_ops_valid_to_reg(struct bpf_map *map) +{ + return false; +} +static inline int bpf_struct_ops_link_update_check(struct bpf_map *new_map, + struct bpf_map *old_map, + struct bpf_map *expected_old_map) +{ + return -EOPNOTSUPP; +} #endif @@ -2528,7 +2669,10 @@ u64 bpf_event_output(struct bpf_map *map, u64 flags, void *meta, u64 meta_size, * since other cpus are walking the array of pointers in parallel. */ struct bpf_prog_array_item { - struct bpf_prog *prog; + union { + struct bpf_prog *prog; + void *kdata; + }; union { struct bpf_cgroup_storage *cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE]; u64 bpf_cookie; @@ -2570,6 +2714,7 @@ int bpf_prog_array_copy(struct bpf_prog_array *old_array, struct bpf_prog *include_prog, u64 bpf_cookie, struct bpf_prog_array **new_array); +struct bpf_prog *bpf_prog_dummy(void); struct bpf_run_ctx {}; @@ -2588,6 +2733,7 @@ struct bpf_trace_run_ctx { struct bpf_tramp_run_ctx { struct bpf_run_ctx run_ctx; u64 bpf_cookie; + int retval; struct bpf_run_ctx *saved_run_ctx; }; @@ -2821,8 +2967,10 @@ struct bpf_map *bpf_map_get_curr_or_next(u32 *id); struct bpf_prog *bpf_prog_get_curr_or_next(u32 *id); -int bpf_map_alloc_pages(const struct bpf_map *map, int nid, - unsigned long nr_pages, struct page **page_array); +struct page *bpf_alloc_page(int nid, bool sleepable); +int bpf_alloc_pages(int nid, unsigned long nr_pages, + struct llist_head *pages, bool sleepable); +void bpf_free_pages(struct llist_head *pages); #ifdef CONFIG_MEMCG void bpf_map_memcg_enter(const struct bpf_map *map, struct mem_cgroup **old_memcg, struct mem_cgroup **new_memcg); @@ -3008,6 +3156,8 @@ typedef int (*bpf_iter_attach_target_t)(struct bpf_prog *prog, union bpf_iter_link_info *linfo, struct bpf_iter_aux_info *aux); typedef void (*bpf_iter_detach_target_t)(struct bpf_iter_aux_info *aux); +typedef int (*bpf_iter_check_prog_t)(const struct bpf_prog *prog, + const struct bpf_iter_aux_info *aux); typedef void (*bpf_iter_show_fdinfo_t) (const struct bpf_iter_aux_info *aux, struct seq_file *seq); typedef int (*bpf_iter_fill_link_info_t)(const struct bpf_iter_aux_info *aux, @@ -3017,7 +3167,8 @@ typedef const struct bpf_func_proto * const struct bpf_prog *prog); enum bpf_iter_feature { - BPF_ITER_RESCHED = BIT(0), + BPF_ITER_RESCHED = BIT(0), /* may cond_resched() between objects */ + BPF_ITER_NOSLEEP = BIT(1), /* ->show() can't run sleepable progs */ }; #define BPF_ITER_CTX_ARG_MAX 2 @@ -3025,6 +3176,8 @@ struct bpf_iter_reg { const char *target; bpf_iter_attach_target_t attach_target; bpf_iter_detach_target_t detach_target; + /* may the program run on what the link is attached to? */ + bpf_iter_check_prog_t check_prog; bpf_iter_show_fdinfo_t show_fdinfo; bpf_iter_fill_link_info_t fill_link_info; bpf_iter_get_func_proto_t get_func_proto; @@ -3061,6 +3214,8 @@ void bpf_iter_map_show_fdinfo(const struct bpf_iter_aux_info *aux, struct seq_file *seq); int bpf_iter_map_fill_link_info(const struct bpf_iter_aux_info *aux, struct bpf_link_info *info); +int bpf_iter_map_check_value_write(const struct bpf_prog *prog, + const struct bpf_map *map); int map_set_for_each_callback_args(struct bpf_verifier_env *env, struct bpf_func_state *caller, @@ -3218,7 +3373,7 @@ const struct btf_func_model * bpf_jit_find_kfunc_model(const struct bpf_prog *prog, const struct bpf_insn *insn); int bpf_get_kfunc_addr(const struct bpf_prog *prog, u32 func_id, - u16 btf_fd_idx, u8 **func_addr); + u16 desc_idx, u8 **func_addr); struct bpf_core_ctx { struct bpf_verifier_log *log; @@ -3561,7 +3716,7 @@ bpf_jit_find_kfunc_model(const struct bpf_prog *prog, static inline int bpf_get_kfunc_addr(const struct bpf_prog *prog, u32 func_id, - u16 btf_fd_idx, u8 **func_addr) + u16 desc_idx, u8 **func_addr) { return -ENOTSUPP; } @@ -3667,6 +3822,7 @@ int bpf_dev_bound_kfunc_check(struct bpf_verifier_log *log, void *bpf_dev_bound_resolve_kfunc(struct bpf_prog *prog, u32 func_id); int bpf_prog_dev_bound_init(struct bpf_prog *prog, union bpf_attr *attr); int bpf_prog_dev_bound_inherit(struct bpf_prog *new_prog, struct bpf_prog *old_prog); +void bpf_prog_dev_bound_moved(struct bpf_prog *prog); void bpf_dev_bound_netdev_unregister(struct net_device *dev); static inline bool bpf_prog_is_dev_bound(const struct bpf_prog_aux *aux) @@ -3728,6 +3884,10 @@ static inline int bpf_prog_dev_bound_inherit(struct bpf_prog *new_prog, return -EOPNOTSUPP; } +static inline void bpf_prog_dev_bound_moved(struct bpf_prog *prog) +{ +} + static inline void bpf_dev_bound_netdev_unregister(struct net_device *dev) { } @@ -3858,6 +4018,8 @@ struct bpf_key { #if defined(CONFIG_KEYS) && defined(CONFIG_BPF_SYSCALL) struct bpf_key *bpf_lookup_user_key(s32 serial, u64 flags); struct bpf_key *bpf_lookup_system_key(u64 id); +struct bpf_key *bpf_lookup_keyring(void); +bool bpf_keyring_enforced(void); void bpf_key_put(struct bpf_key *bkey); int bpf_verify_pkcs7_signature(const struct bpf_dynptr *data_p, const struct bpf_dynptr *sig_p, @@ -3878,6 +4040,16 @@ static inline struct bpf_key *bpf_lookup_system_key(u64 id) return NULL; } +static inline struct bpf_key *bpf_lookup_keyring(void) +{ + return NULL; +} + +static inline bool bpf_keyring_enforced(void) +{ + return false; +} + static inline void bpf_key_put(struct bpf_key *bkey) { } @@ -4138,9 +4310,10 @@ void bpf_bprintf_cleanup(struct bpf_bprintf_data *data); int bpf_try_get_buffers(struct bpf_bprintf_buffers **bufs); void bpf_put_buffers(void); -void bpf_prog_stream_init(struct bpf_prog *prog); +int bpf_prog_stream_init(struct bpf_prog *prog, gfp_t gfp_extra_flags); void bpf_prog_stream_free(struct bpf_prog *prog); -int bpf_prog_stream_read(struct bpf_prog *prog, enum bpf_stream_id stream_id, void __user *buf, int len); +int bpf_prog_stream_read(struct bpf_prog *prog, enum bpf_stream_id stream_id, void __user *buf, u32 len); +int bpf_prog_stream_new_fd(struct bpf_prog *prog, enum bpf_stream_id stream_id, u32 flags); void bpf_stream_stage_init(struct bpf_stream_stage *ss); void bpf_stream_stage_free(struct bpf_stream_stage *ss); __printf(2, 3) @@ -4203,7 +4376,7 @@ struct bpf_prog *bpf_prog_find_from_stack(void); int bpf_insn_array_init(struct bpf_map *map, const struct bpf_prog *prog); int bpf_insn_array_ready(struct bpf_map *map); void bpf_insn_array_release(struct bpf_map *map); -void bpf_insn_array_adjust(struct bpf_map *map, u32 off, u32 len); +void bpf_insn_array_adjust(struct bpf_map *map, u32 first, u32 len); void bpf_insn_array_adjust_after_remove(struct bpf_map *map, u32 off, u32 len); #ifdef CONFIG_BPF_SYSCALL @@ -4228,6 +4401,9 @@ static inline bool bpf_map_is_percpu_map(enum bpf_map_type map_type) } } +#define BPF_EXIST_FLAGS (BPF_EXIST | BPF_NOEXIST) +#define BPF_CPU_FLAGS (BPF_F_CPU | BPF_F_ALL_CPUS) + static inline int bpf_map_check_op_flags(struct bpf_map *map, u64 flags, u64 allowed_flags) { u32 cpu; diff --git a/include/linux/bpf_compiler_types.h b/include/linux/bpf_compiler_types.h new file mode 100644 index 000000000000..5c9e71acfdac --- /dev/null +++ b/include/linux/bpf_compiler_types.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __LINUX_BPF_COMPILER_TYPES_H +#define __LINUX_BPF_COMPILER_TYPES_H + +#include + +/* + * Qualifiers for the BPF verifier rather than address spaces: __arena marks a + * pointer into a BPF arena, __nullable a pointer that may be NULL. They reach + * BPF programs as BTF type tags on the members of kernel structs shared with + * them; to kernel code and to sparse the pointers are ordinary. + */ +#ifdef __CHECKER__ +# define __arena +# define __nullable +#else +# define __arena BTF_TYPE_TAG(arena) +# define __nullable BTF_TYPE_TAG(nullable) +#endif + +#endif /* __LINUX_BPF_COMPILER_TYPES_H */ diff --git a/include/linux/bpf_crypto.h b/include/linux/bpf_crypto.h deleted file mode 100644 index a41e71d4e2d9..000000000000 --- a/include/linux/bpf_crypto.h +++ /dev/null @@ -1,24 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* Copyright (c) 2024 Meta Platforms, Inc. and affiliates. */ -#ifndef _BPF_CRYPTO_H -#define _BPF_CRYPTO_H - -struct bpf_crypto_type { - void *(*alloc_tfm)(const char *algo); - void (*free_tfm)(void *tfm); - int (*has_algo)(const char *algo); - int (*setkey)(void *tfm, const u8 *key, unsigned int keylen); - int (*setauthsize)(void *tfm, unsigned int authsize); - int (*encrypt)(void *tfm, const u8 *src, u8 *dst, unsigned int len, u8 *iv); - int (*decrypt)(void *tfm, const u8 *src, u8 *dst, unsigned int len, u8 *iv); - unsigned int (*ivsize)(void *tfm); - unsigned int (*statesize)(void *tfm); - u32 (*get_flags)(void *tfm); - struct module *owner; - char name[14]; -}; - -int bpf_crypto_register_type(const struct bpf_crypto_type *type); -int bpf_crypto_unregister_type(const struct bpf_crypto_type *type); - -#endif /* _BPF_CRYPTO_H */ diff --git a/include/linux/bpf_local_storage.h b/include/linux/bpf_local_storage.h index 9e4f5c45c974..6c643ac51079 100644 --- a/include/linux/bpf_local_storage.h +++ b/include/linux/bpf_local_storage.h @@ -104,6 +104,19 @@ struct bpf_local_storage { refcount_t owner_refcnt;/* Used to pin owner when map_free is uncharging */ }; +/* + * Drop a reference got by refcount_inc_not_zero(&local_storage->owner_refcnt). + * bpf_local_storage_destroy() may have dropped the owner's reference meanwhile + * and wait for the count to reach zero, so this one can be the last. + */ +static inline void +bpf_local_storage_unpin_owner(struct bpf_local_storage *local_storage) +{ + bool last __maybe_unused; + + last = refcount_dec_and_test(&local_storage->owner_refcnt); +} + /* U16_MAX is much more than enough for sk local storage * considering a tcp_sock is ~2k. */ diff --git a/include/linux/bpf_lsm.h b/include/linux/bpf_lsm.h index dda272d78f01..eb4a38eef87e 100644 --- a/include/linux/bpf_lsm.h +++ b/include/linux/bpf_lsm.h @@ -21,6 +21,13 @@ extern bool bpf_lsm_initialized __ro_after_init; #include #undef LSM_HOOK +/* + * Number of xattr slots the BPF LSM reserves in the array handed to + * security_inode_init_security(), i.e. the maximum number of labels + * a policy may attach to an inode while it is being created. + */ +#define BPF_LSM_INODE_INIT_XATTRS 2 + struct bpf_storage_blob { struct bpf_local_storage __rcu *storage; }; @@ -28,7 +35,7 @@ struct bpf_storage_blob { extern struct lsm_blob_sizes bpf_lsm_blob_sizes; int bpf_lsm_verify_prog(struct bpf_verifier_log *vlog, - const struct bpf_prog *prog); + struct bpf_prog *prog); bool bpf_lsm_is_sleepable_hook(u32 btf_id); bool bpf_lsm_is_trusted(const struct bpf_prog *prog); @@ -71,7 +78,7 @@ static inline bool bpf_lsm_is_trusted(const struct bpf_prog *prog) } static inline int bpf_lsm_verify_prog(struct bpf_verifier_log *vlog, - const struct bpf_prog *prog) + struct bpf_prog *prog) { return -EOPNOTSUPP; } diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index b83ec99f1a13..5926f13ff43c 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -9,6 +9,20 @@ #include /* for MAX_BPF_STACK */ #include #include +#include /* for div_s64_rem() */ + +/* + * Mathematical modulo, the residue of 'v' modulo 'step', normalized to [0, step). + * This differs from C's '%' operator, which truncates the quotient toward zero + * and so returns a remainder with the sign of the dividend (e.g. -1 % 3 == -1, not 2). + */ +static inline u16 imod(s64 v, u16 step) +{ + s32 rem; + + div_s64_rem(v, step, &rem); + return rem < 0 ? rem + step : rem; +} /* Maximum variable offset umax_value permitted when resolving memory accesses. * In practice this is far bigger than any realistic pointer offset; this limit @@ -19,11 +33,12 @@ * that converting umax_value to int cannot overflow. */ #define BPF_MAX_VAR_SIZ (1 << 29) -/* size of tmp_str_buf in bpf_verifier. - * we need at least 306 bytes to fit full stack mask representation - * (in the "-8,-16,...,-512" form) +/* + * size of tmp_str_buf in bpf_verifier. + * we need at least 1399 bytes to fit full stack mask representation + * (in the "-8,-16,...,-2048" form) */ -#define TMP_STR_BUF_LEN 320 +#define TMP_STR_BUF_LEN 1408 /* Patch buffer size */ #define INSN_BUF_SIZE 32 @@ -124,6 +139,13 @@ struct bpf_reg_state { */ struct cnum64 r64; /* 64-bit range as circular number */ struct cnum32 r32; /* 32-bit range as circular number */ + /* + * The value described by this register, interpreted as s64, lies on + * a line described by a linear equation base + step * k. + * Invariant: base < step. + */ + u16 base; + u16 step; /* For PTR_TO_PACKET, used to find other pointers with the same variable * offset, so they can share range knowledge. * For PTR_TO_MAP_VALUE_OR_NULL this is used to share which map value we @@ -162,9 +184,9 @@ struct bpf_reg_state { * other registers. Kept outside the metadata union for ID remapping * during state comparisons. */ - u32 map_uid; + u32 map_uid:31; /* if (!precise && SCALAR_VALUE) min/max/tnum don't affect safety */ - bool precise; + u32 precise:1; }; static inline s64 reg_smin(const struct bpf_reg_state *reg) @@ -243,55 +265,18 @@ enum bpf_stack_slot_type { #define BPF_REG_SIZE 8 /* size of eBPF register in bytes */ +/* + * Largest number of BPF_REG_SIZE stack slots a single frame can have, sized + * for the largest stack budget any JIT supports. A frame may use any part of + * its program's budget; check_max_stack_depth() enforces the budget on the + * combined depth of frames sharing the kernel stack and on each frame using + * a private stack. + */ +#define MAX_BPF_STACK_SLOTS (MAX_BPF_STACK_JIT / BPF_REG_SIZE) + /* 4-byte stack slot granularity for liveness analysis */ #define BPF_HALF_REG_SIZE 4 #define STACK_SLOT_SZ 4 -#define STACK_SLOTS (MAX_BPF_STACK / BPF_HALF_REG_SIZE) /* 128 */ - -typedef struct { - u64 v[2]; -} spis_t; - -#define SPIS_ZERO ((spis_t){}) -#define SPIS_ALL ((spis_t){{ U64_MAX, U64_MAX }}) - -static inline bool spis_is_zero(spis_t s) -{ - return s.v[0] == 0 && s.v[1] == 0; -} - -static inline bool spis_equal(spis_t a, spis_t b) -{ - return a.v[0] == b.v[0] && a.v[1] == b.v[1]; -} - -static inline spis_t spis_or(spis_t a, spis_t b) -{ - return (spis_t){{ a.v[0] | b.v[0], a.v[1] | b.v[1] }}; -} - -static inline spis_t spis_and(spis_t a, spis_t b) -{ - return (spis_t){{ a.v[0] & b.v[0], a.v[1] & b.v[1] }}; -} - -static inline spis_t spis_not(spis_t s) -{ - return (spis_t){{ ~s.v[0], ~s.v[1] }}; -} - -static inline bool spis_test_bit(spis_t s, u32 slot) -{ - return s.v[slot / 64] & BIT_ULL(slot % 64); -} - -static inline void spis_or_range(spis_t *mask, u32 lo, u32 hi) -{ - u32 w; - - for (w = lo; w <= hi && w < STACK_SLOTS; w++) - mask->v[w / 64] |= BIT_ULL(w % 64); -} #define BPF_REGMASK_ARGS ((1 << BPF_REG_1) | (1 << BPF_REG_2) | \ (1 << BPF_REG_3) | (1 << BPF_REG_4) | \ @@ -343,6 +328,25 @@ struct bpf_retval_range { bool return_32bit; }; +/* SCEV/widening register space: r0..r10 plus every stack slot of a frame. */ +#define BPF_SCEV_REGS_NUM (MAX_BPF_REG + MAX_BPF_STACK_SLOTS) + +struct bpf_loop_iters { + u32 min_header_count; /* min number of times header is executed */ + u32 max_header_count; /* max number of times header is executed */ + unsigned long precise_regs[BITS_TO_LONGS(BPF_SCEV_REGS_NUM)]; +}; + +struct loop_stack_entry { + struct bpf_verifier_state *entry_state; + u32 min_header_count; + u32 max_header_count; + u32 loop_id:31; + u32 terminates:1; +}; + +#define LOOP_STACK_SIZE 16 + /* state of the program: * type of all registers and stack info */ @@ -375,6 +379,11 @@ struct bpf_func_state { bool in_async_callback_fn; bool in_exception_callback_fn; bool no_stack_arg_load; + /* + * The helper invokes this callback under rcu_read_lock(), so the + * callback and its callees run in an RCU read-side critical section. + */ + bool in_rcu_callback_fn; /* For callback calling functions that limit number of possible * callback executions (e.g. bpf_loop) keeps track of current * simulated iteration number. @@ -388,6 +397,11 @@ struct bpf_func_state { u32 callback_depth; /* Instructions processed in this frame and callees on the current path. */ u32 insns_subtotal; + /* + * Control-flow loop nesting at the current insn within this frame's + * subprogram (loops never cross subprogram boundaries). + */ + u32 loop_stack_cnt; /* The following fields should be last. See copy_func_state() */ /* The state of the stack. Each element of the array describes BPF_REG_SIZE @@ -405,6 +419,7 @@ struct bpf_func_state { u16 out_stack_arg_cnt; /* Number of outgoing on-stack argument slots */ struct bpf_reg_state *stack_arg_regs; /* Outgoing on-stack arguments */ + struct loop_stack_entry *loop_stack; }; #define MAX_CALL_FRAMES 16 @@ -421,30 +436,37 @@ enum { INSN_F_STACK_ARG_ACCESS = BIT(3), }; +/* Registers linked to one jump condition that a history entry can record */ +#define BPF_LINKED_REGS_MAX 5 + struct bpf_jmp_history_entry { /* insn idx can't be bigger than 1 million */ u32 idx : 20; u32 frame : 4; /* stack access frame number */ - u32 spi : 6; /* stack slot index (0..63) */ - u32 : 2; - u32 prev_idx : 20; /* special INSN_F_xxx flags */ u32 flags : 4; - u32 : 8; + u32 : 4; + u32 prev_idx : 20; + u32 spi : 12; /* stack slot index */ /* - * additional registers that need precision tracking when this - * jump is backtracked, vector of five 11-bit records + * Scalar registers and spilled scalars linked to the condition of + * this jump, which need precision tracking together when the jump is + * backtracked. Each is packed as 4 bits of frame number, one bit + * telling a register from a stack slot and 11 bits of register or + * slot index, see linked_regs_pack(). */ - u64 linked_regs; + u16 linked_regs[BPF_LINKED_REGS_MAX]; + u8 linked_regs_cnt; }; static_assert(MAX_CALL_FRAMES <= (1 << 4)); -static_assert(MAX_BPF_STACK / 8 <= (1 << 6)); +static_assert(MAX_BPF_STACK_SLOTS <= (1 << 12)); /* Maximum number of bpf_reg_state objects that can exist at once */ #define MAX_STACK_ARG_SLOTS (MAX_BPF_FUNC_ARGS - MAX_BPF_FUNC_REG_ARGS) -#define BPF_ID_MAP_SIZE ((MAX_BPF_REG + MAX_BPF_STACK / BPF_REG_SIZE + \ - MAX_STACK_ARG_SLOTS) * MAX_CALL_FRAMES) +#define BPF_ID_MAP_SIZE ((MAX_BPF_REG + MAX_BPF_STACK_SLOTS + MAX_STACK_ARG_SLOTS) * \ + MAX_CALL_FRAMES) + struct bpf_verifier_state { /* call stack tracking */ struct bpf_func_state *frame[MAX_CALL_FRAMES]; @@ -510,8 +532,13 @@ struct bpf_verifier_state { bool speculative; bool in_sleepable; + /* The callee left with unwind_exit. Next insn is unwind_goto. */ + bool unwound; - /* first and last insn idx of this verifier state */ + /* + * First and last insn idx of this verifier state. + * last_insn_idx is -1 if no instructions have been executed yet. + */ u32 first_insn_idx; u32 last_insn_idx; /* if this state is a backedge state then equal_state @@ -530,12 +557,31 @@ struct bpf_verifier_state { u32 may_goto_depth; }; +/* Number of BPF_REG_SIZE stack slots tracked for the frame so far. */ +static inline u32 bpf_stack_nr_slots(const struct bpf_func_state *frame) +{ + return frame->allocated_stack / BPF_REG_SIZE; +} + +/* + * Stack slot @spi of @frame, covering bytes [fp - (spi + 1) * 8, fp - spi * 8). + * The caller must ensure spi < bpf_stack_nr_slots(frame), see bpf_grow_stack_state(). + */ +static inline struct bpf_stack_state *bpf_stack_slot(const struct bpf_func_state *frame, u32 spi) +{ + return &frame->stack[spi]; +} + static inline struct bpf_reg_state * bpf_get_spilled_reg(int slot, struct bpf_func_state *frame, u32 mask) { - if (slot < frame->allocated_stack / BPF_REG_SIZE && - (1 << frame->stack[slot].slot_type[BPF_REG_SIZE - 1]) & mask) - return &frame->stack[slot].spilled_ptr; + struct bpf_stack_state *ss; + + if (slot >= bpf_stack_nr_slots(frame)) + return NULL; + ss = bpf_stack_slot(frame, slot); + if ((1 << ss->slot_type[BPF_REG_SIZE - 1]) & mask) + return &ss->spilled_ptr; return NULL; } @@ -551,7 +597,7 @@ bpf_get_spilled_stack_arg(int slot, struct bpf_func_state *frame) /* Iterate over 'frame', setting 'reg' to either NULL or a spilled register. */ #define bpf_for_each_spilled_reg(iter, frame, reg, mask) \ for (iter = 0, reg = bpf_get_spilled_reg(iter, frame, mask); \ - iter < frame->allocated_stack / BPF_REG_SIZE; \ + iter < bpf_stack_nr_slots(frame); \ iter++, reg = bpf_get_spilled_reg(iter, frame, mask)) /* Iterate over 'frame', setting 'reg' to either NULL or a spilled stack arg. */ @@ -576,7 +622,7 @@ bpf_get_spilled_stack_arg(int slot, struct bpf_func_state *frame) bpf_for_each_spilled_reg(___j, __state, __reg, __mask) { \ if (!__reg) \ continue; \ - __stack = &__state->stack[___j]; \ + __stack = bpf_stack_slot(__state, ___j); \ (void)(__expr); \ } \ __stack = NULL; \ @@ -640,6 +686,52 @@ struct bpf_iarray { u32 items[]; }; +#define iarray_for_each(item, arr) \ + for (int ___idx = 0; \ + ___idx < (arr)->cnt && ({ item = (arr)->items[___idx]; 1; }); \ + ___idx++) + +#define MAX_BACKEDGES 16 +#define MAX_LOOP_EXITS 256 + +/* Number of exit conditions dominating a backedge. */ +#define MAX_LATCHES 4 + +struct bpf_backedge { + int from; + /* + * Conditional jumps within the loop that dominate 'from' and have + * a successor outside the loop, nearest first. latches[0] is the + * latch; the remaining entries are additional exit conditions. + * Each usable condition gives an upper bound on the iteration count, + * so keeping only a subset is sound. + */ + int latches[MAX_LATCHES]; + u8 latches_cnt; +}; + +struct bpf_loop_exit { + int from; /* instruction inside the loop */ + int to; /* instruction outside the loop */ +}; + +struct bpf_loop { + struct bpf_backedge backedges[MAX_BACKEDGES]; + /* edges exiting from this loop, includes edges from nested loops */ + struct bpf_loop_exit *exits; + int backedges_cnt; + int exits_cnt; + bool irreducible; + bool backedges_overflow; + bool exits_overflow; + /* + * Loop-entry registers used to compute stack addresses for loads, + * stores and calls requiring fixed stack offsets, in this and nested + * loops. + */ + unsigned long store_base_regs[BITS_TO_LONGS(BPF_SCEV_REGS_NUM)]; +}; + struct bpf_insn_aux_data { union { enum bpf_reg_type ptr_type; /* pointer type for load/store insns */ @@ -671,45 +763,64 @@ struct bpf_insn_aux_data { /* remember the offset of node field within type to rewrite */ u64 insert_off; }; - struct bpf_iarray *jt; /* jump table for gotox or bpf_tailcall call instruction */ struct btf_struct_meta *kptr_struct_meta; u64 map_key_state; /* constant (32 bit) key tracking for maps */ + /* + * Per-instruction summary of stack slots in the current frame + * that this instruction may write to. + */ + DECLARE_BITMAP(may_write_mask, MAX_BPF_STACK_SLOTS); + DECLARE_BITMAP(live_stack_before, MAX_BPF_STACK_SLOTS); + u16 stack_ptrs; /* bitmask of regs that may hold a frame pointer here (arg_track) */ int ctx_field_size; /* the ctx field size for load insn, maybe 0 */ u32 seen; /* this insn was processed by the verifier at env->pass_cnt */ - bool nospec; /* do not execute this instruction speculatively */ - bool nospec_result; /* result is unsafe under speculation, nospec must follow */ - bool zext_dst; /* this insn zero extends dst reg */ - bool needs_zext; /* alu op needs to clear upper bits */ - bool prevent_zext; /* alu op cannot be zext (already used with 64-bit scalars) */ - bool non_sleepable; /* helper/kfunc may be called from non-sleepable context */ - bool is_iter_next; /* bpf_iter__next() kfunc call */ - bool call_with_percpu_alloc_ptr; /* {this,per}_cpu_ptr() with prog percpu alloc */ - u8 alu_state; /* used in combination with alu_limit */ + u64 nospec:1; /* do not execute this instruction speculatively */ + u64 nospec_result:1; /* result is unsafe under speculation, nospec must follow */ + u64 zext_dst:1; /* this insn zero extends dst reg */ + u64 needs_zext:1; /* alu op needs to clear upper bits */ + u64 prevent_zext:1; /* alu op cannot be zext (already used with 64-bit scalars) */ + u64 non_sleepable:1; /* helper/kfunc may be called from non-sleepable context */ + u64 is_iter_next:1; /* bpf_iter__next() kfunc call */ + u64 call_with_percpu_alloc_ptr:1; /* {this,per}_cpu_ptr() with prog percpu alloc */ + u64 arena_scalar:1; /* ldx/stx/st/atomic through a number, it's an address in arena */ + u64 bb_end:1; /* last instruction of a basic block */ + u64 need_scev:1; + u64 alu_state:5; /* used in combination with alu_limit */ /* true if STX or LDX instruction is a part of a spill/fill * pattern for a bpf_fastcall call. */ - u8 fastcall_pattern:1; + u64 fastcall_pattern:1; /* for CALL instructions, a number of spill/fill pairs in the * bpf_fastcall pattern. */ - u8 fastcall_spills_num:3; - u8 arg_prog:4; + u64 fastcall_spills_num:3; + u64 arg_prog:4; + /* + * What a load found, on any path: an __arena pointer member of a + * kernel struct, which arena_member_fixup() rewrites, one that is + * __nullable as well, or anything else through any pointer. + */ + u64 arena_member:1; + u64 arena_member_nullable:1; + u64 non_arena_load:1; - /* below fields are initialized once */ - unsigned int orig_idx; /* original instruction index */ - u32 jmp_point:1; - u32 prune_point:1; + /* below flags are initialized once */ + u64 jmp_point:1; + u64 prune_point:1; /* ensure we check state equivalence and save state checkpoint and * this instruction, regardless of any heuristics */ - u32 force_checkpoint:1; + u64 force_checkpoint:1; /* true if instruction is a call to a helper function that * accepts callback function as a parameter. */ - u32 calls_callback:1; - u32 indirect_target:1; /* if it is an indirect jump target */ + u64 calls_callback:1; + u64 indirect_target:1; /* if it is an indirect jump target */ + u64 non_stack_access:1; /* instruction can access non-stack memory */ /* true if some jump or call instruction targets this instruction */ - u32 jump_target:1; + u64 jump_target:1; + + unsigned int orig_idx; /* original instruction index, initialized once */ /* * CFG strongly connected component this instruction belongs to, * zero if it is a singleton SCC. @@ -729,12 +840,23 @@ struct bpf_insn_aux_data { u16 const_reg_map_mask; u16 const_reg_subprog_mask; u32 const_reg_vals[10]; + /* + * Header index of the innermost loop containing this instruction, -1 if none. + * For a loop header, identifies the parent loop instead. + */ + s32 loop_header; + /* additional information about the loop if this instruction is a loop header */ + struct bpf_loop *loop; }; #define MAX_USED_MAPS 64 /* max number of maps accessed by one eBPF program */ #define MAX_USED_BTFS 64 /* max number of BTFs accessed by one BPF program */ -#define BPF_VERIFIER_TMP_LOG_SIZE 1024 +/* + * Longest line the verifier log can carry: a full stack mask of + * MAX_BPF_STACK_SLOTS slots, see TMP_STR_BUF_LEN, plus its prefix. + */ +#define BPF_VERIFIER_TMP_LOG_SIZE 2048 struct bpf_verifier_log { /* Logical start and end positions of a "log window" of the verifier log. @@ -787,6 +909,21 @@ int bpf_log_attr_finalize(struct bpf_log_attr *attr, struct bpf_verifier_log *lo #define BPF_MAX_SUBPROGS 256 +/* + * A pointer to a static subprog in the value of a frozen read-only array map: + * a 64-bit value that is the offset in bytes of the first instruction of + * the subprog in the program. + */ +struct bpf_func_ptr { + struct bpf_map *map; + u32 map_off; /* offset of the pointer in the value of the map */ + u32 orig_off; /* what the map has: the first instruction of the subprog */ + u32 xlated_off; /* the same after instructions were patched and removed */ +}; + +/* the subprog that a bpf_func_ptr pointed to was removed as dead code */ +#define BPF_FUNC_PTR_DELETED ((u32)-1) + struct bpf_subprog_arg_info { enum bpf_arg_type arg_type; union { @@ -806,7 +943,12 @@ struct bpf_subprog_info { u32 start; /* insn idx of function entry point */ u32 linfo_idx; /* The idx to the main_prog->aux->linfo */ u32 postorder_start; /* The idx to the env->cfg.insn_postorder */ - u32 exit_idx; /* Index of one of the BPF_EXIT instructions in this subprogram */ + /* + * Index of one of the BPF_EXIT instructions in this subprogram, or + * U32_MAX when it has none. + */ + u32 exit_idx; + struct bpf_iarray *jt; /* jump table shared by all gotox of this subprogram */ u16 stack_depth; /* max. stack depth used by this function */ u16 stack_extra; u32 insns_total; @@ -817,17 +959,20 @@ struct bpf_subprog_info { s16 fastcall_stack_off; bool has_tail_call: 1; bool might_throw: 1; + bool can_unwind: 1; /* has unwind_exit */ bool tail_call_reachable: 1; bool has_ld_abs: 1; bool is_cb: 1; bool is_async_cb: 1; bool is_exception_cb: 1; bool args_cached: 1; + /* true if the return value is passed in the R0:R2 register pair */ + bool ret_reg_pair: 1; /* true if bpf_fastcall stack region is used by functions that can't be inlined */ bool keep_fastcall_stack: 1; bool changes_pkt_data: 1; bool might_sleep: 1; - u8 arg_cnt:4; + u8 arg_slot_cnt:4; enum priv_stack_mode priv_stack_mode; struct bpf_subprog_arg_info args[MAX_BPF_FUNC_ARGS]; @@ -837,8 +982,8 @@ struct bpf_subprog_info { static inline u16 bpf_in_stack_arg_cnt(const struct bpf_subprog_info *sub) { - if (sub->arg_cnt > MAX_BPF_FUNC_REG_ARGS) - return sub->arg_cnt - MAX_BPF_FUNC_REG_ARGS; + if (sub->arg_slot_cnt > MAX_BPF_FUNC_REG_ARGS) + return sub->arg_slot_cnt - MAX_BPF_FUNC_REG_ARGS; return 0; } @@ -849,7 +994,7 @@ struct backtrack_state { struct bpf_verifier_env *env; u32 frame; u32 reg_masks[MAX_CALL_FRAMES]; - u64 stack_masks[MAX_CALL_FRAMES]; + unsigned long stack_masks[MAX_CALL_FRAMES][BITS_TO_LONGS(MAX_BPF_STACK_SLOTS)]; u8 stack_arg_masks[MAX_CALL_FRAMES]; }; @@ -905,6 +1050,7 @@ struct bpf_scc_info { }; struct bpf_liveness; +struct scev; struct bpf_fd_array { union { @@ -928,6 +1074,7 @@ struct bpf_verifier_env { bool strict_alignment; /* perform strict pointer alignment checks */ bool test_state_freq; /* test verifier with different pruning frequency */ bool test_reg_invariants; /* fail verification on register invariants violations */ + bool arena_scalar; /* load and store through a number is arena access */ struct bpf_verifier_state *cur_state; /* current verifier state */ /* Search pruning optimization, array of list_heads for * lists of struct bpf_verifier_state_list. @@ -954,6 +1101,8 @@ struct bpf_verifier_env { bool bypass_spec_v1; bool bypass_spec_v4; bool seen_direct_write; + /* a global function is verified for BPF_FUNC_CTX_NO_RCU_LOCK */ + bool no_rcu_read_lock; bool seen_exception; bool signature; u32 insn_aux_data_len; @@ -961,9 +1110,24 @@ struct bpf_verifier_env { const struct bpf_line_info *prev_linfo; struct bpf_verifier_log log; struct bpf_diag *diag; + struct bpf_func_proto bpf_subprog_scratch; struct bpf_subprog_info subprog_info[BPF_MAX_SUBPROGS + 2]; /* max + 2 for the fake and exception subprogs */ /* subprog indices sorted in topological order: leaves first, callers last */ int subprog_topo_order[BPF_MAX_SUBPROGS + 2]; + /* + * Pointers to static subprogs found in frozen read-only maps of the + * program, see resolve_func_ptrs(). Sorted by map and map_off. + */ + struct bpf_func_ptr *func_ptrs; + u32 func_ptr_cnt; + bool has_callx; + /* + * Call graph edges created by callx instructions. A bitmap of + * subprog_cnt * subprog_cnt bits, where bit (caller * subprog_cnt + callee) + * is set when the main verification pass sees 'caller' calling 'callee' + * via callx. Allocated when the first such edge is recorded. + */ + unsigned long *callx_edges; union { struct bpf_idmap idmap_scratch; struct bpf_idset idset_scratch; @@ -976,14 +1140,17 @@ struct bpf_verifier_env { * see bpf_subprog_info->postorder_start. */ int *insn_postorder; + int *postorder_nums; int cur_stack; /* current position in the insn_postorder vector */ int cur_postorder; + u32 gotox_edges; + bool subprog_jts_ready; } cfg; struct backtrack_state bt; struct bpf_jmp_history_entry *cur_hist_ent; - /* Per-callsite copy of parent's converged at_stack_in for cross-frame fills. */ - struct arg_track **callsite_at_stack; + /* Per-callsite snapshot of the parent's tracked spill slots for cross-frame fills. */ + struct spill_snapshot **callsite_at_stack; u32 pass_cnt; /* number of times do_check() was called */ u32 subprog_cnt; /* number of instructions analyzed by the verifier */ @@ -992,6 +1159,8 @@ struct bpf_verifier_env { u32 prev_jmps_processed, jmps_processed; /* maximum combined stack depth */ u32 max_stack_depth; + /* stack budget of the program, see bpf_prog_stack_limit() */ + u32 stack_limit; /* total verification time */ u64 verification_time; /* maximum number of verifier states kept in 'branching' instructions */ @@ -1027,7 +1196,7 @@ struct bpf_verifier_env { */ u32 scratched_regs; /* Same as scratched_regs but for stack slots */ - u64 scratched_stack_slots; + DECLARE_BITMAP(scratched_stack_slots, MAX_BPF_STACK_SLOTS); u64 prev_log_pos, prev_insn_print_pos; /* buffer used to temporary hold constants as scalar registers */ struct bpf_reg_state fake_reg[1]; @@ -1047,6 +1216,8 @@ struct bpf_verifier_env { u32 scc_cnt; struct bpf_iarray *succ; struct bpf_iarray *gotox_tmp_buf; + int *idoms; + struct scev *scev; }; static inline struct bpf_func_info_aux *subprog_aux(struct bpf_verifier_env *env, int subprog) @@ -1059,8 +1230,13 @@ static inline struct bpf_subprog_info *subprog_info(struct bpf_verifier_env *env return &env->subprog_info[subprog]; } +static inline bool bpf_ret_reg_pair(struct bpf_verifier_env *env, int subprog) +{ + return subprog_info(env, subprog)->ret_reg_pair; +} + struct bpf_call_summary { - u8 num_params; + u8 arg_slot_cnt; bool is_void; bool fastcall; }; @@ -1083,6 +1259,12 @@ static inline bool bpf_pseudo_kfunc_call(const struct bpf_insn *insn) insn->src_reg == BPF_PSEUDO_KFUNC_CALL; } +/* callx: indirect call of a bpf subprog whose address is in insn->dst_reg */ +static inline bool bpf_is_callx(const struct bpf_insn *insn) +{ + return insn->code == (BPF_JMP | BPF_CALL | BPF_X); +} + __printf(2, 0) void bpf_verifier_vlog(struct bpf_verifier_log *log, const char *fmt, va_list args); __printf(2, 3) void bpf_verifier_log_write(struct bpf_verifier_env *env, @@ -1097,6 +1279,7 @@ int bpf_vlog_finalize(struct bpf_verifier_log *log, u32 *log_size_actual); __printf(3, 4) void verbose_linfo(struct bpf_verifier_env *env, u32 insn_off, const char *prefix_fmt, ...); +void bpf_verbose_snum(struct bpf_verifier_env *env, s64 num); #define verifier_bug_if(cond, env, fmt, args...) \ ({ \ @@ -1209,11 +1392,14 @@ int bpf_check_attach_target(struct bpf_verifier_log *log, const struct bpf_prog *tgt_prog, u32 btf_id, struct bpf_attach_target_info *tgt_info); +int bpf_check_ext_prologue(struct bpf_verifier_log *log, + const struct bpf_prog *tgt_prog, u32 btf_id); void bpf_free_kfunc_btf_tab(struct bpf_kfunc_btf_tab *tab); int mark_chain_precision(struct bpf_verifier_env *env, int regno); int bpf_is_state_visited(struct bpf_verifier_env *env, int insn_idx); +int bpf_split_cur_state(struct bpf_verifier_env *env); int bpf_update_branch_counts(struct bpf_verifier_env *env, struct bpf_verifier_state *st); void bpf_clear_jmp_history(struct bpf_verifier_state *state); @@ -1223,11 +1409,14 @@ struct list_head *bpf_explored_state(struct bpf_verifier_env *env, int idx); void bpf_free_verifier_state(struct bpf_verifier_state *state, bool free_self); void bpf_free_backedges(struct bpf_scc_visit *visit); int bpf_push_jmp_history(struct bpf_verifier_env *env, struct bpf_verifier_state *cur, - int insn_flags, int spi, int frame, u64 linked_regs); + int insn_flags, int spi, int frame, const u16 *linked_regs, + u8 linked_regs_cnt); void bpf_bt_sync_linked_regs(struct backtrack_state *bt, struct bpf_jmp_history_entry *hist); void bpf_mark_reg_not_init(const struct bpf_verifier_env *env, struct bpf_reg_state *reg); +void bpf_mark_reg_known_scalar(struct bpf_reg_state *reg, u64 imm); void bpf_mark_reg_unknown_imprecise(struct bpf_reg_state *reg); +int bpf_grow_stack_state(struct bpf_verifier_env *env, struct bpf_func_state *state, int size); void bpf_mark_all_scalars_precise(struct bpf_verifier_env *env, struct bpf_verifier_state *st); void bpf_clear_singular_ids(struct bpf_verifier_env *env, struct bpf_verifier_state *st); @@ -1240,6 +1429,24 @@ static inline int bpf_get_spi(s32 off) return (-off - 1) / BPF_REG_SIZE; } +/* + * Stack a program may use in total: combined over the frames of a call + * chain on the kernel stack, or per frame on a private stack. Any single + * frame may reach that deep. Only a JIT that lays out such frames may go + * beyond MAX_BPF_STACK, the interpreter's frame size, and only one whose + * tail calls let the target set up its own frame: without subprogram + * tail calls, do_misc_fixups() gives every program with tail calls a + * MAX_BPF_STACK frame, which a deeper frame would overrun. + */ +static inline u32 bpf_prog_stack_limit(const struct bpf_prog *prog) +{ + /* an offloaded program never runs on the host JIT, whatever it supports */ + if (prog->jit_requested && !bpf_prog_is_offloaded(prog->aux) && + bpf_jit_supports_large_stack() && bpf_jit_supports_subprog_tailcalls()) + return MAX_BPF_STACK_JIT; + return MAX_BPF_STACK; +} + /* * Return the function state a stack pointer register refers to. frameno * shares storage with other pointer metadata, so return NULL for any @@ -1273,6 +1480,15 @@ static inline bool bpf_is_spilled_reg(const struct bpf_stack_state *stack) return stack->slot_type[BPF_REG_SIZE - 1] == STACK_SPILL; } +static inline int bpf_spill_size(const struct bpf_stack_state *stack) +{ + int i, spill_size = 0; + + for (i = BPF_REG_SIZE - 1; i >= 0 && stack->slot_type[i] == STACK_SPILL; i--) + spill_size++; + return spill_size; +} + static inline bool bpf_is_spilled_scalar_reg(const struct bpf_stack_state *stack) { return bpf_is_spilled_reg(stack) && stack->spilled_ptr.type == SCALAR_VALUE; @@ -1290,12 +1506,7 @@ static inline void bpf_bt_set_frame_reg(struct backtrack_state *bt, u32 frame, u static inline void bpf_bt_set_frame_slot(struct backtrack_state *bt, u32 frame, u32 slot) { - bt->stack_masks[frame] |= 1ull << slot; -} - -static inline void bpf_bt_set_frame_slot_mask(struct backtrack_state *bt, u32 frame, u64 mask) -{ - bt->stack_masks[frame] |= mask; + __set_bit(slot, bt->stack_masks[frame]); } static inline void bt_set_frame_stack_arg_slot(struct backtrack_state *bt, u32 frame, u32 slot) @@ -1310,10 +1521,17 @@ static inline bool bt_is_frame_reg_set(struct backtrack_state *bt, u32 frame, u3 static inline bool bt_is_frame_slot_set(struct backtrack_state *bt, u32 frame, u32 slot) { - return bt->stack_masks[frame] & (1ull << slot); + return test_bit(slot, bt->stack_masks[frame]); } bool bpf_map_is_rdonly(const struct bpf_map *map); +struct bpf_func_ptr *bpf_map_func_ptrs(struct bpf_verifier_env *env, + const struct bpf_map *map, u32 *cnt); +struct bpf_func_ptr *bpf_map_range_func_ptrs(struct bpf_verifier_env *env, + const struct bpf_map *map, + u64 off, u64 size, u32 *cnt); +void bpf_adjust_func_ptrs(struct bpf_verifier_env *env, u32 off, u32 len); +void bpf_adjust_func_ptrs_after_remove(struct bpf_verifier_env *env, u32 off, u32 len); int bpf_map_direct_read(struct bpf_map *map, int off, int size, u64 *val, bool is_ldsx); @@ -1441,7 +1659,7 @@ static inline void mark_reg_scratched(struct bpf_verifier_env *env, u32 regno) static inline void mark_stack_slot_scratched(struct bpf_verifier_env *env, u32 spi) { - env->scratched_stack_slots |= 1ULL << spi; + __set_bit(spi, env->scratched_stack_slots); } static inline bool reg_scratched(const struct bpf_verifier_env *env, u32 regno) @@ -1449,27 +1667,28 @@ static inline bool reg_scratched(const struct bpf_verifier_env *env, u32 regno) return (env->scratched_regs >> regno) & 1; } -static inline bool stack_slot_scratched(const struct bpf_verifier_env *env, u64 regno) +static inline bool stack_slot_scratched(const struct bpf_verifier_env *env, u32 spi) { - return (env->scratched_stack_slots >> regno) & 1; + return test_bit(spi, env->scratched_stack_slots); } static inline bool verifier_state_scratched(const struct bpf_verifier_env *env) { - return env->scratched_regs || env->scratched_stack_slots; + return env->scratched_regs || + !bitmap_empty(env->scratched_stack_slots, MAX_BPF_STACK_SLOTS); } static inline void mark_verifier_state_clean(struct bpf_verifier_env *env) { env->scratched_regs = 0U; - env->scratched_stack_slots = 0ULL; + bitmap_zero(env->scratched_stack_slots, MAX_BPF_STACK_SLOTS); } /* Used for printing the entire verifier state. */ static inline void mark_verifier_state_scratched(struct bpf_verifier_env *env) { env->scratched_regs = ~0U; - env->scratched_stack_slots = ~0ULL; + bitmap_fill(env->scratched_stack_slots, MAX_BPF_STACK_SLOTS); } static inline bool bpf_stack_narrow_access_ok(int off, int fill_size, int spill_size) @@ -1504,14 +1723,25 @@ struct bpf_subprog_info *bpf_find_containing_subprog(struct bpf_verifier_env *en const char *bpf_subprog_name(const struct bpf_verifier_env *env, int subprog); int bpf_jmp_offset(struct bpf_insn *insn); struct bpf_iarray *bpf_insn_successors(struct bpf_verifier_env *env, u32 idx); -void bpf_fmt_stack_mask(char *buf, ssize_t buf_sz, u64 stack_mask); +void bpf_fmt_stack_mask(char *buf, ssize_t buf_sz, const unsigned long *stack_mask); bool bpf_subprog_is_global(const struct bpf_verifier_env *env, int subprog); +/* Kinds of member a by-value struct or union may be composed of. */ +enum btf_member_kind { + BTF_MEMBER_SCALAR = BIT(0), /* an int or an enum */ + BTF_MEMBER_ARENA_PTR = BIT(1), /* a pointer carrying the "arena" type tag */ +}; + +bool btf_struct_is_composed_of(struct bpf_verifier_env *env, const struct btf *btf, + const struct btf_type *t, u32 member_kinds); +u32 btf_func_arg_align(const struct btf *btf, const struct btf_type *t); + int bpf_find_subprog(struct bpf_verifier_env *env, int off); bool bpf_is_throw_kfunc(struct bpf_insn *insn); int bpf_compute_const_regs(struct bpf_verifier_env *env); int bpf_prune_dead_branches(struct bpf_verifier_env *env); int bpf_check_cfg(struct bpf_verifier_env *env); +void bpf_free_subprog_jts(struct bpf_verifier_env *env); int bpf_compute_postorder(struct bpf_verifier_env *env); int bpf_compute_scc(struct bpf_verifier_env *env); @@ -1540,12 +1770,15 @@ struct ref_obj_desc { }; /* - * A memory argument a call fills in. The verifier allows the stack to be uninitialized if - * the range is a known constant. Stack slots are marked as STACK_MISC by check_mem_access(). + * Generic MEM_UNINIT arguments, indexed by ABI slot. var_size_mask excludes + * variable-sized buffers from raw mode without losing the output annotation. + * size records constant ranges to mark initialized after checking all arguments, + * only when the caller is allowed to read uninitialized stack memory. */ struct arg_raw_mem_desc { - u8 regno; - int size; + u16 mask; + u16 var_size_mask; + int size[MAX_BPF_FUNC_ARGS]; }; /* Size of PTR_TO_MEM returned, taken from a constant allocation-size argument */ @@ -1565,6 +1798,7 @@ struct bpf_call_arg_meta { struct btf *btf; u32 func_id; const struct bpf_func_proto *fn; + const struct btf_type *func_proto; u8 release_regno; u32 ret_btf_id; u32 subprogno; @@ -1572,20 +1806,25 @@ struct bpf_call_arg_meta { struct bpf_dynptr_desc dynptr; struct ref_obj_desc ref_obj; struct ret_mem_desc ret_mem; + struct arg_raw_mem_desc arg_raw_mem; + + /* Only set by subprog */ + bool subprog_may_change_pkt; /* Only set by kfunc */ bool r0_rdonly; u32 kfunc_flags; - const struct btf_type *func_proto; const char *func_name; struct arg_constant_desc arg_constant; + /* The type of the dynptr that the kfunc writes through */ + enum bpf_dynptr_type dynptr_write_type; /* arg_{btf,btf_id,owning_ref} are used by kfunc-specific handling, * generally to pass info about user-defined local kptr types to later * verification logic * bpf_obj_drop/bpf_percpu_obj_drop * Record the local kptr type to be drop'd - * bpf_refcount_acquire (via KF_ARG_PTR_TO_REFCOUNTED_KPTR arg type) + * bpf_refcount_acquire (via ARG_PTR_TO_REFCOUNTED_KPTR arg type) * Record the local kptr type to be refcount_incr'd and use * arg_owning_ref to determine whether refcount_acquire should be * fallible @@ -1593,7 +1832,6 @@ struct bpf_call_arg_meta { struct btf *arg_btf; u32 arg_btf_id; bool arg_owning_ref; - bool arg_prog; struct { struct btf_field *field; @@ -1611,7 +1849,6 @@ struct bpf_call_arg_meta { s64 const_map_key; struct btf *ret_btf; struct btf_field *kptr_field; - struct arg_raw_mem_desc arg_raw_mem; }; int bpf_get_helper_proto(struct bpf_verifier_env *env, int func_id, @@ -1638,17 +1875,37 @@ bool bpf_is_may_goto_insn(struct bpf_insn *insn); void bpf_verbose_insn(struct bpf_verifier_env *env, struct bpf_insn *insn); bool bpf_get_call_summary(struct bpf_verifier_env *env, struct bpf_insn *call, struct bpf_call_summary *cs); -s64 bpf_helper_stack_access_bytes(struct bpf_verifier_env *env, - struct bpf_insn *insn, int arg, - int insn_idx); -s64 bpf_kfunc_stack_access_bytes(struct bpf_verifier_env *env, - struct bpf_insn *insn, int arg, - int insn_idx); +/* Stack effects used for state pruning; must_write implies may_write. */ +struct arg_access_info { + u32 size; /* Maximum extent; U32_MAX if unknown. */ + u8 may_read:1; /* Incoming contents or initialization may be needed. */ + u8 may_write:1; + u8 must_write:1; /* Prior verifier state is destroyed throughout size. */ +}; + +struct arg_access_info +bpf_helper_stack_access_bytes(struct bpf_verifier_env *env, + struct bpf_insn *insn, int arg, int insn_idx); +struct arg_access_info +bpf_kfunc_stack_access_bytes(struct bpf_verifier_env *env, + struct bpf_insn *insn, int arg, int insn_idx); +struct arg_access_info +bpf_global_subprog_stack_access_bytes(struct bpf_verifier_env *env, + struct bpf_insn *insn, int arg, int insn_idx); int bpf_compute_subprog_arg_access(struct bpf_verifier_env *env); +bool bpf_same_memory_origin(const struct bpf_reg_state *reg_a, + const struct bpf_reg_state *reg_b); +int bpf_set_reg_range(struct bpf_verifier_env *env, struct bpf_reg_state *reg, + struct cnum64 r64, struct cnum32 r32, u16 base, u16 step); +bool bpf_reg_union_compatible(const struct bpf_reg_state *a, const struct bpf_reg_state *b); +int bpf_reg_union(struct bpf_verifier_env *env, struct bpf_reg_state *acc, + const struct bpf_reg_state *src); int bpf_stack_liveness_init(struct bpf_verifier_env *env); void bpf_stack_liveness_free(struct bpf_verifier_env *env); int bpf_live_stack_query_init(struct bpf_verifier_env *env, struct bpf_verifier_state *st); +const unsigned long *bpf_may_write_mask(struct bpf_verifier_env *env, u32 insn_idx); +bool bpf_needs_fixed_stack_off(struct bpf_verifier_env *env, int insn_idx); bool bpf_stack_slot_alive(struct bpf_verifier_env *env, u32 frameno, u32 spi); int bpf_compute_live_registers(struct bpf_verifier_env *env); @@ -1680,6 +1937,21 @@ static inline u64 bpf_map_key_immediate(const struct bpf_insn_aux_data *aux) return aux->map_key_state & ~(BPF_MAP_KEY_SEEN | BPF_MAP_KEY_POISON); } +static inline bool bpf_is_mem_insn(struct bpf_insn *insn) +{ + if (BPF_CLASS(insn->code) != BPF_ST && + BPF_CLASS(insn->code) != BPF_STX && + BPF_CLASS(insn->code) != BPF_LDX) + return false; + + if (insn->code == (BPF_ST | BPF_NOSPEC)) + return false; + + return (BPF_MODE(insn->code) == BPF_MEM || + BPF_MODE(insn->code) == BPF_MEMSX || + BPF_MODE(insn->code) == BPF_ATOMIC); +} + #define MAX_PACKET_OFF 0xffff #define CALLER_SAVED_REGS 6 @@ -1690,30 +1962,34 @@ enum bpf_reg_arg_type { }; #define MAX_KFUNC_DESCS 256 +/* Each kfunc can have its canonical and one specialized call target. */ +#define MAX_KFUNC_CALL_DESCS (MAX_KFUNC_DESCS * 2) +static_assert(MAX_KFUNC_CALL_DESCS <= S16_MAX + 1); struct bpf_kfunc_desc { struct btf_func_model func_model; struct bpf_func_proto proto; u32 func_id; - s32 imm; u16 offset; unsigned long addr; }; struct bpf_kfunc_desc_tab { u32 nr_descs; + u32 nr_base_descs; /* Sorted by func_id (BTF ID) and offset (fd_array offset) during - * verification. JITs do lookups by bpf_insn, where func_id may not be - * available, therefore at the end of verification do_misc_fixups() - * sorts this by imm and offset. + * verification. JITs use the descriptor index stored in the finalized + * call's off field. The first nr_base_descs entries are the canonical + * descriptors used for verifier lookups. Call specialization may append + * immutable descriptors for additional targets. * - * Grown one entry at a time by bpf_add_kfunc_call(). + * Grown one entry at a time by bpf_add_kfunc_call() and during + * call specialization. */ struct bpf_kfunc_desc descs[]; }; /* Functions exported from verifier.c, used by fixups.c */ -void bpf_clear_insn_aux_data(struct bpf_verifier_env *env, int start, int len); void bpf_mark_subprog_exc_cb(struct bpf_verifier_env *env, int subprog); bool bpf_allow_tail_call_in_subprogs(struct bpf_verifier_env *env); bool bpf_verifier_inlines_helper_call(struct bpf_verifier_env *env, s32 imm); @@ -1726,6 +2002,7 @@ int bpf_check_attach_btf_id_multi(struct btf *btf, struct bpf_prog *prog, u32 bt struct bpf_attach_target_info *tgt_info); /* Functions in fixups.c, called from bpf_check() */ +void bpf_clear_insn_aux_data(struct bpf_verifier_env *env, int start, int len); int bpf_remove_fastcall_spills_fills(struct bpf_verifier_env *env); int bpf_optimize_bpf_loop(struct bpf_verifier_env *env); void bpf_opt_hard_wire_dead_code_branches(struct bpf_verifier_env *env); @@ -1738,4 +2015,23 @@ int bpf_fixup_call_args(struct bpf_verifier_env *env); int bpf_do_misc_fixups(struct bpf_verifier_env *env); int bpf_insn_def32(struct bpf_prog *prog, struct bpf_insn *insn); +int bpf_flip_opcode(u32 opcode); +u8 bpf_rev_opcode(u8 opcode); + +int bpf_compute_idoms(struct bpf_verifier_env *env); +int bpf_compute_loops(struct bpf_verifier_env *env); +int bpf_loop_at_index(struct bpf_verifier_env *env, u32 idx); +bool bpf_is_nested_loop(struct bpf_verifier_env *env, int inner_header, int outer_header); + +int bpf_init_scev(struct bpf_verifier_env *env); +void bpf_free_scev(struct bpf_verifier_env *env); +int bpf_compute_scev(struct bpf_verifier_env *env); + +int bpf_compute_loop_iters(struct bpf_verifier_env *env, struct bpf_verifier_state *st, + struct bpf_loop_iters *iters); +int bpf_widen_scev_regs(struct bpf_verifier_env *env, struct bpf_verifier_state *st, + struct bpf_verifier_state *loop_entry, struct bpf_loop_iters *iters); +int bpf_clamp_scev_regs(struct bpf_verifier_env *env, struct bpf_func_state *st, u32 insn_idx, + struct bpf_verifier_state *entry_state, u32 max_header_count); + #endif /* _LINUX_BPF_VERIFIER_H */ diff --git a/include/linux/btf.h b/include/linux/btf.h index 7c62ea17b116..b66e20a18c54 100644 --- a/include/linux/btf.h +++ b/include/linux/btf.h @@ -5,6 +5,7 @@ #define _LINUX_BTF_H 1 #include +#include #include #include #include @@ -118,6 +119,19 @@ struct btf_show; struct btf_id_set; struct bpf_prog; +#define BTF_LINK_SHA256_LEN 32 + +/* + * A .BTF.link or .BTF.inline.link section identifies the module which + * carries a vmlinux BTF section. The module name is NUL terminated and the + * field is padded to __MODULE_NAME_LEN. + */ +struct btf_link { + char module_name[__MODULE_NAME_LEN]; + u8 sha256[BTF_LINK_SHA256_LEN]; + u32 btf_size; +} __packed; + typedef int (*btf_kfunc_filter_t)(const struct bpf_prog *prog, u32 kfunc_id); struct btf_kfunc_id_set { @@ -236,6 +250,7 @@ struct btf_record *btf_parse_fields(const struct btf *btf, const struct btf_type u32 field_mask, u32 value_size); int btf_check_and_fixup_fields(const struct btf *btf, struct btf_record *rec); bool btf_type_is_void(const struct btf_type *t); +bool btf_type_is_arena_ptr(const struct btf *btf, const struct btf_type *t); s32 btf_find_by_name_kind(const struct btf *btf, const char *name, u8 kind); s32 bpf_find_btf_id(const char *name, u32 kind, struct btf **btf_p); struct btf *btf_get_module_btf(const struct module *module); @@ -261,6 +276,11 @@ const char *btf_type_str(const struct btf_type *t); i < btf_type_vlen(datasec_type); \ i++, member++) +#define for_each_loc(i, locsec_type, member) \ + for (i = 0, member = btf_type_loc_secinfo(locsec_type); \ + i < btf_type_vlen(locsec_type); \ + i++, member++) + static inline bool btf_type_is_ptr(const struct btf_type *t) { return BTF_INFO_KIND(t->info) == BTF_KIND_PTR; @@ -329,6 +349,21 @@ static inline u64 btf_enum64_value(const struct btf_enum64 *e) return ((u64)e->val_hi32 << 32) | e->val_lo32; } +static inline struct btf_loc_param *btf_loc_param(const struct btf_type *t) +{ + return (struct btf_loc_param *)(t + 1); +} + +static inline __u32 *btf_loc_proto_params(const struct btf_type *t) +{ + return (__u32 *)(t + 1); +} + +static inline struct btf_loc *btf_type_loc_secinfo(const struct btf_type *t) +{ + return (struct btf_loc *)(t + 1); +} + static inline bool btf_is_composite(const struct btf_type *t) { u16 kind = btf_kind(t); @@ -559,7 +594,7 @@ struct btf_field_desc { /* member struct size, or zero, if no members */ int m_sz; /* repeated per-member offsets */ - int m_off_cnt, m_offs[1]; + int m_off_cnt, m_offs[2]; }; struct btf_field_iter { @@ -573,7 +608,8 @@ struct btf_field_iter { #ifdef CONFIG_BPF_SYSCALL const struct btf_type *btf_type_by_id(const struct btf *btf, u32 type_id); void btf_set_base_btf(struct btf *btf, const struct btf *base_btf); -int btf_relocate(struct btf *btf, const struct btf *base_btf, __u32 **map_ids); +int btf_relocate(struct btf *btf, const struct btf *base_btf, __u32 **map_ids, + __u32 **map_strs); int btf_field_iter_init(struct btf_field_iter *it, struct btf_type *t, enum btf_field_iter_kind iter_kind); __u32 *btf_field_iter_next(struct btf_field_iter *it); @@ -603,6 +639,29 @@ bool btf_types_are_same(const struct btf *btf1, u32 id1, const struct btf *btf2, u32 id2); int btf_check_iter_arg(struct btf *btf, const struct btf_type *func, int arg_idx); +#if IS_ENABLED(CONFIG_SYSFS) +struct bin_attribute *sysfs_btf_add(const char *name, void *data, size_t data_size, + bool mmap, const char *lazy_module_name); +bool sysfs_btf_update(struct bin_attribute *attr, void *data); +void sysfs_btf_remove(struct bin_attribute *attr); +#else +static inline struct bin_attribute * +sysfs_btf_add(const char *name, void *data, size_t data_size, bool mmap, + const char *lazy_module_name) +{ + return NULL; +} + +static inline bool sysfs_btf_update(struct bin_attribute *attr, void *data) +{ + return false; +} + +static inline void sysfs_btf_remove(struct bin_attribute *attr) +{ +} +#endif + static inline bool btf_type_is_struct_ptr(struct btf *btf, const struct btf_type *t) { if (!btf_type_is_ptr(t)) @@ -624,7 +683,7 @@ static inline void btf_set_base_btf(struct btf *btf, const struct btf *base_btf) } static inline int btf_relocate(void *log, struct btf *btf, const struct btf *base_btf, - __u32 **map_ids) + __u32 **map_ids, __u32 **map_strs) { return -EOPNOTSUPP; } diff --git a/include/linux/cnum.h b/include/linux/cnum.h index 49b7d0c7645d..49160318adf9 100644 --- a/include/linux/cnum.h +++ b/include/linux/cnum.h @@ -39,6 +39,7 @@ u32 cnum32_umin(struct cnum32 cnum); u32 cnum32_umax(struct cnum32 cnum); s32 cnum32_smin(struct cnum32 cnum); s32 cnum32_smax(struct cnum32 cnum); +struct cnum32 cnum32_union(struct cnum32 a, struct cnum32 b); struct cnum32 cnum32_intersect(struct cnum32 a, struct cnum32 b); void cnum32_intersect_with(struct cnum32 *dst, struct cnum32 src); void cnum32_intersect_with_urange(struct cnum32 *dst, u32 min, u32 max); @@ -65,6 +66,7 @@ u64 cnum64_umin(struct cnum64 cnum); u64 cnum64_umax(struct cnum64 cnum); s64 cnum64_smin(struct cnum64 cnum); s64 cnum64_smax(struct cnum64 cnum); +struct cnum64 cnum64_union(struct cnum64 a, struct cnum64 b); struct cnum64 cnum64_intersect(struct cnum64 a, struct cnum64 b); void cnum64_intersect_with(struct cnum64 *dst, struct cnum64 src); void cnum64_intersect_with_urange(struct cnum64 *dst, u64 min, u64 max); @@ -78,5 +80,6 @@ bool cnum64_is_subset(struct cnum64 outer, struct cnum64 inner); struct cnum32 cnum32_from_cnum64(struct cnum64 cnum); struct cnum64 cnum64_cnum32_intersect(struct cnum64 a, struct cnum32 b); +struct cnum64 cnum64_intersect_linear(struct cnum64 in, u16 base, u16 step); #endif /* _LINUX_CNUM_H */ diff --git a/include/linux/filter.h b/include/linux/filter.h index 39decde7fc73..e0218e677a1d 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -98,6 +98,11 @@ struct ctl_table_header; /* BPF program can access up to 512 bytes of stack space. */ #define MAX_BPF_STACK 512 +/* + * Stack budget of a program on a JIT that lays out frames of that size. + * The interpreter and JITs without such support keep MAX_BPF_STACK. + */ +#define MAX_BPF_STACK_JIT 2048 /* Helper macros for filter block array initializers. */ @@ -249,6 +254,29 @@ static inline bool insn_is_mov_percpu_addr(const struct bpf_insn *insn) .off = 0, \ .imm = 1 }) +static inline bool bpf_is_unwind_goto(const struct bpf_insn *insn) +{ + return insn->code == (BPF_JMP | BPF_JCOND) && insn->src_reg == BPF_UNWIND_GOTO; +} + +static inline bool bpf_is_unwind_exit(const struct bpf_insn *insn) +{ + return insn->code == (BPF_JMP | BPF_EXIT) && insn->src_reg == BPF_UNWIND_EXIT; +} + +/* + * The caller of the main prog doesn't tell unwind_exit from exit. An extension + * is different: its caller is the caller of the subprog that it replaced. + * An exception callback returns from the frame of the main prog. After freplace + * or a tail call it returns to a prog that wasn't verified with the callback. + */ +static inline bool bpf_prog_can_unwind(const struct bpf_prog *prog) +{ + if (prog->aux->exception_cb) + return false; + return bpf_is_subprog(prog) || prog->type == BPF_PROG_TYPE_EXT; +} + static inline bool insn_is_zext(const struct bpf_insn *insn) { return insn->code == (BPF_ALU | BPF_MOV | BPF_X) && insn->imm == 1; @@ -1105,6 +1133,19 @@ static inline unsigned int bpf_prog_size(unsigned int proglen) offsetof(struct bpf_prog, insns[proglen])); } +bool bpf_prog_has_arena_insns(const struct bpf_prog *prog); + +/* + * Whether a JIT has to set up the arena base for @prog. A program bound to + * an arena need not touch it, libbpf binds the arena of an object to every + * program in it. The flag caches bpf_prog_has_arena_insns() for the function + * about to be JITed, see bpf_prog_jit_compile() and jit_subprogs(). + */ +static inline bool bpf_prog_uses_arena_insns(const struct bpf_prog *prog) +{ + return prog->aux->uses_arena_insns; +} + static inline bool bpf_prog_was_classic(const struct bpf_prog *prog) { /* When classic BPF programs have been loaded and the arch @@ -1237,16 +1278,53 @@ bool bpf_jit_inlines_helper_call(s32 imm); bool bpf_jit_supports_subprog_tailcalls(void); bool bpf_jit_supports_percpu_insn(void); bool bpf_jit_supports_kfunc_call(void); +bool bpf_jit_supports_callx(void); +bool bpf_jit_supports_kfunc_ret_reg_pair(void); +bool bpf_jit_supports_unwind(void); bool bpf_jit_supports_stack_args(void); bool bpf_jit_supports_arena_args(void); bool bpf_jit_supports_far_kfunc_call(void); bool bpf_jit_supports_exceptions(void); bool bpf_jit_supports_ptr_xchg(void); bool bpf_jit_supports_arena(void); +bool bpf_jit_supports_arena_scalar(void); bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena); bool bpf_jit_supports_private_stack(void); +bool bpf_jit_supports_large_stack(void); bool bpf_jit_supports_timed_may_goto(void); bool bpf_jit_supports_fsession(void); + +struct bpf_jit_arg_abi { + /* Argument registers of the kernel convention. */ + u8 nr_arg_regs; + /* Round the register number up to an even one for 16-byte alignment. */ + bool even_reg_align; + /* Round the stack slot up to an even one for 16-byte alignment. */ + bool even_stack_align; + /* An argument may straddle the last register and the stack. */ + bool split_at_boundary; + /* A later argument may reuse a register a stack-passed one skipped. */ + bool backfill_after_stack; +}; + +const struct bpf_jit_arg_abi *bpf_jit_arg_abi(void); +u32 bpf_jit_place_args(const struct bpf_jit_arg_abi *abi, + const struct btf_func_model *fm, u8 *pos_of_slot); + +/* The JIT's scratch register, in place of an argument slot. */ +#define BPF_JIT_ARG_TMP 0xff + +/* Every argument slot moves at most once, and the scratch goes out and back. */ +#define BPF_JIT_MAX_ARG_MOVES (MAX_BPF_FUNC_ARG_SLOTS + 2) + +struct bpf_jit_arg_move { + u8 dst; + u8 src; +}; + +u32 bpf_jit_plan_arg_moves(const struct bpf_jit_arg_abi *abi, + const struct btf_func_model *fm, + struct bpf_jit_arg_move *moves); u64 bpf_arch_uaddress_limit(void); void arch_bpf_stack_walk(bool (*consume_fn)(void *cookie, u64 ip, u64 sp, u64 bp), void *cookie); u64 arch_bpf_timed_may_goto(void); @@ -1897,6 +1975,11 @@ static __always_inline long __bpf_xdp_redirect_map(struct bpf_map *map, u64 inde return XDP_REDIRECT; } +int __bpf_sock_ops_load_hdr_opt(struct bpf_sock_ops_kern *bpf_sock, + void *search_res, u32 len, u64 flags); +int __bpf_sock_ops_store_hdr_opt(struct bpf_sock_ops_kern *bpf_sock, + const void *from, u32 len, u64 flags); + #ifdef CONFIG_NET int __bpf_skb_load_bytes(const struct sk_buff *skb, u32 offset, void *to, u32 len); int __bpf_skb_store_bytes(struct sk_buff *skb, u32 offset, const void *from, diff --git a/include/linux/ftrace.h b/include/linux/ftrace.h index bd76a16a63af..b041d9142d2b 100644 --- a/include/linux/ftrace.h +++ b/include/linux/ftrace.h @@ -416,6 +416,7 @@ void free_ftrace_hash(struct ftrace_hash *hash); struct ftrace_func_entry *add_ftrace_hash_entry_direct(struct ftrace_hash *hash, unsigned long ip, unsigned long direct); void add_ftrace_hash_entry(struct ftrace_hash *hash, struct ftrace_func_entry *entry); +struct ftrace_func_entry *ftrace_lookup_ip(struct ftrace_hash *hash, unsigned long ip); void ftrace_hash_remove(struct ftrace_hash *hash); /* The hash used to know what functions callbacks trace */ diff --git a/include/linux/key.h b/include/linux/key.h index e40b5267658f..dae5ba255ad1 100644 --- a/include/linux/key.h +++ b/include/linux/key.h @@ -445,6 +445,8 @@ extern key_ref_t keyring_search(key_ref_t keyring, extern int keyring_restrict(key_ref_t keyring, const char *type, const char *restriction); +extern void key_register_bpf_keyring(struct key *keyring); + extern struct key *key_lookup(key_serial_t id); static inline key_serial_t key_serial(const struct key *key) diff --git a/include/linux/mm.h b/include/linux/mm.h index 0a2a7fc4a442..6882ad400935 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -3314,10 +3314,10 @@ extern int access_process_vm(struct task_struct *tsk, unsigned long addr, extern int access_remote_vm(struct mm_struct *mm, unsigned long addr, void *buf, int len, unsigned int gup_flags); -#ifdef CONFIG_BPF_SYSCALL -extern int copy_remote_vm_str(struct task_struct *tsk, unsigned long addr, - void *buf, int len, unsigned int gup_flags); -#endif +int copy_remote_mm_str(struct mm_struct *mm, unsigned long addr, + void *buf, int len, unsigned int gup_flags); +int copy_remote_vm_str(struct task_struct *tsk, unsigned long addr, + void *buf, int len, unsigned int gup_flags); long get_user_pages_remote(struct mm_struct *mm, unsigned long start, unsigned long nr_pages, diff --git a/include/linux/module.h b/include/linux/module.h index 2c5a02621c19..76f4f3596477 100644 --- a/include/linux/module.h +++ b/include/linux/module.h @@ -503,6 +503,10 @@ struct module { void *btf_data; void *btf_base_data; #endif +#if IS_ENABLED(CONFIG_DEBUG_INFO_BTF_INLINE) + unsigned int btf_inline_data_size; + void *btf_inline_data; +#endif #ifdef CONFIG_JUMP_LABEL struct jump_entry *jump_entries; unsigned int num_jump_entries; diff --git a/include/linux/tcp.h b/include/linux/tcp.h index e4d1720f49f0..d18f7bc1e5d0 100644 --- a/include/linux/tcp.h +++ b/include/linux/tcp.h @@ -233,6 +233,13 @@ struct tcp_sock { is_sack_reneg:1, /* in recovery from loss with SACK reneg? */ is_cwnd_limited:1,/* forward progress limited by snd_cwnd? */ recvmsg_inq : 1;/* Indicate # of bytes in queue upon recvmsg */ +#ifdef CONFIG_BPF + u32 bpf_tcp_ops_flags; +#define BPF_TCP_OPS_TEST_FLAG(TP, ARG) \ + (READ_ONCE((TP)->bpf_tcp_ops_flags) & BPF_TCP_OPS_FLAG_ ## ARG) +#else +#define BPF_TCP_OPS_TEST_FLAG(TP, ARG) (0) +#endif __cacheline_group_end(tcp_sock_read_txrx); /* RX read-mostly hotpath cache lines */ diff --git a/include/linux/tnum.h b/include/linux/tnum.h index ca2cfec8de08..16682248f963 100644 --- a/include/linux/tnum.h +++ b/include/linux/tnum.h @@ -134,4 +134,6 @@ static inline bool tnum_subreg_is_const(struct tnum a) /* Returns the smallest member of t larger than z */ u64 tnum_step(struct tnum t, u64 z); +u32 tnum_alignment(struct tnum a); + #endif /* _LINUX_TNUM_H */ diff --git a/include/net/tcp.h b/include/net/tcp.h index 0265c27a5fd7..3560adddf821 100644 --- a/include/net/tcp.h +++ b/include/net/tcp.h @@ -512,6 +512,7 @@ void tcp_set_keepalive(struct sock *sk, int val); void tcp_syn_ack_timeout(const struct request_sock *req); int tcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int flags); +int __tcp_set_rcvlowat(struct sock *sk, int val, bool wakeup); int tcp_set_rcvlowat(struct sock *sk, int val); void tcp_set_rcvbuf(struct sock *sk, int val); int tcp_set_window_clamp(struct sock *sk, int val); @@ -2930,7 +2931,7 @@ static inline void bpf_skops_init_skb(struct bpf_sock_ops_kern *skops, * program loaded). */ #ifdef CONFIG_BPF -static inline int tcp_call_bpf(struct sock *sk, int op, u32 nargs, u32 *args) +static inline int __tcp_call_bpf(struct sock *sk, int op, u32 nargs, u32 *args) { struct bpf_sock_ops_kern sock_ops; int ret; @@ -2947,7 +2948,7 @@ static inline int tcp_call_bpf(struct sock *sk, int op, u32 nargs, u32 *args) if (nargs > 0) memcpy(sock_ops.args, args, nargs * sizeof(*args)); - ret = BPF_CGROUP_RUN_PROG_SOCK_OPS(&sock_ops); + ret = __BPF_CGROUP_RUN_PROG_SOCK_OPS(&sock_ops); if (ret == 0) ret = sock_ops.reply; else @@ -2955,42 +2956,37 @@ static inline int tcp_call_bpf(struct sock *sk, int op, u32 nargs, u32 *args) return ret; } -static inline int tcp_call_bpf_2arg(struct sock *sk, int op, u32 arg1, u32 arg2) -{ - u32 args[2] = {arg1, arg2}; +#define tcp_call_bpf(sk, op) \ +({ \ + int __ret = 0; \ + if (cgroup_bpf_enabled(CGROUP_SOCK_OPS)) { \ + __ret = __tcp_call_bpf(sk, op, 0, NULL); \ + } \ + __ret; \ +}) - return tcp_call_bpf(sk, op, 2, args); -} - -static inline int tcp_call_bpf_3arg(struct sock *sk, int op, u32 arg1, u32 arg2, - u32 arg3) -{ - u32 args[3] = {arg1, arg2, arg3}; - - return tcp_call_bpf(sk, op, 3, args); -} +#define tcp_call_bpf_flag(sk, op, ...) \ +do { \ + if (cgroup_bpf_enabled(CGROUP_SOCK_OPS) && \ + BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk), op ## _FLAG)) { \ + u32 __args[] = { __VA_ARGS__ }; \ + __tcp_call_bpf(sk, op, ARRAY_SIZE(__args), __args); \ + } \ +} while (0) static inline void tcp_clear_sock_ops_cb_flags(struct sock *sk) { tcp_sk(sk)->bpf_sock_ops_cb_flags = 0; + WRITE_ONCE(tcp_sk(sk)->bpf_tcp_ops_flags, 0); } #else -static inline int tcp_call_bpf(struct sock *sk, int op, u32 nargs, u32 *args) +static inline int tcp_call_bpf(struct sock *sk, int op) { return -EPERM; } -static inline int tcp_call_bpf_2arg(struct sock *sk, int op, u32 arg1, u32 arg2) -{ - return -EPERM; -} - -static inline int tcp_call_bpf_3arg(struct sock *sk, int op, u32 arg1, u32 arg2, - u32 arg3) -{ - return -EPERM; -} +#define tcp_call_bpf_flag(sk, op, ...) do { } while (0) static inline void tcp_clear_sock_ops_cb_flags(struct sock *sk) { @@ -2998,12 +2994,196 @@ static inline void tcp_clear_sock_ops_cb_flags(struct sock *sk) #endif +#if defined(CONFIG_BPF_JIT) && defined(CONFIG_CGROUP_BPF) + +struct bpf_tcp_ops { + /* Should return the initial SYN (active open) or SYN-ACK (passive open) + * retransmission timeout. Return the timeout in jiffies, or <= 0 for + * the kernel default. + * + * @req: request_sock on the passive (synack) path; NULL otherwise. + */ + int (*timeout_init)(struct sock *sk, struct request_sock *req); + + /* Should return the initial advertised receive window, in packets, + * or < 0 for the kernel default. @req as in timeout_init(). + */ + int (*rwnd_init)(struct sock *sk, struct request_sock *req); + + /* Called when an active connection becomes established. + * @skb is the SYNACK that completed the 3WHS, or NULL for a + * TCP_REPAIR socket (tcp_finish_connect() with no skb). + */ + void (*active_established)(struct sock *sk, struct sk_buff *skb__nullable); + + /* Called when a passive connection becomes established. + * @skb is the ACK that completed the 3WHS. + */ + void (*passive_established)(struct sock *sk, struct sk_buff *skb); + + /* Called when the retransmission timer fires. */ + void (*rto)(struct sock *sk); + + /* Called on every RTT sample if BPF_TCP_OPS_FLAG_RTT is enabled. + * @mrtt: the measured RTT, in microseconds. + * @srtt: the updated smoothed RTT. + */ + void (*rtt)(struct sock *sk, long mrtt, u32 srtt); + + /* Called when the connection changes TCP state. + * @state: the new state (one of the TCP_* states). + */ + void (*set_state)(struct sock *sk, int state); + + /* Called when an skb is retransmitted. + * @skb: the retransmitted skb. + * @err: tcp_transmit_skb() return value (0 on success). + */ + void (*retrans)(struct sock *sk, struct sk_buff *skb, int err); + + /* Called right before an active connection is initialized. */ + void (*connect)(struct sock *sk); + + /* Called on listen(2), right after the socket enters TCP_LISTEN. */ + void (*listen)(struct sock *sk); + + /* + * Parse the TCP header options of an incoming skb received on an + * established connection. Use bpf_dynptr_from_skb()/bpf_skb_load_bytes() + * to access the options. + * + * Called if BPF_TCP_OPS_FLAG_PARSE_HDR_OPT_ALL is enabled, or if + * BPF_TCP_OPS_FLAG_PARSE_HDR_OPT_UNKNOWN is enabled and an unknown + * option is received. + */ + void (*parse_hdr)(struct sock *sk, struct sk_buff *skb); + + /* + * Reserve space in the outgoing TCP header for options to be written + * later by write_hdr_opt(). Call bpf_reserve_hdr_opt() to reserve bytes. + * + * Called if BPF_TCP_OPS_FLAG_WRITE_HDR_OPT is enabled. + * + * @skb: outgoing packet. NULL when called from tcp_current_mss() + * (MSS sizing). + * @req: request_sock on the synack path; NULL otherwise. + * @syn_skb: incoming SYN on the synack path; NULL otherwise. + * @synack_type: TCP_SYNACK_COOKIE indicates a stateless syncookie. + * @remaining: pointer to the size of space still available; cast it + * using bpf_rdonly_cast() before dereferencing. + */ + void (*hdr_opt_len)(struct sock *sk, struct sk_buff *skb, + struct request_sock *req, struct sk_buff *syn_skb, + enum tcp_synack_type synack_type, + unsigned int *remaining); + + /* + * Write header options into the space reserved earlier by hdr_opt_len(). + * Use bpf_store_hdr_opt() to write; it appends within the reserved window + * shared with legacy SOCKOPS. + * + * Called if BPF_TCP_OPS_FLAG_WRITE_HDR_OPT is enabled. + * + * @skb: outgoing packet. + * @req: request_sock on the synack path; NULL otherwise. + * @syn_skb: incoming SYN on the synack path; NULL otherwise. + * @synack_type: TCP_SYNACK_COOKIE indicates a stateless syncookie. + * @opt_off: offset in the outgoing @skb's TCP header where the + * bpf_tcp_ops portion of the reserved window begins, i.e. after + * the kernel and legacy options. + */ + void (*write_hdr_opt)(struct sock *sk, struct sk_buff *skb, + struct request_sock *req, struct sk_buff *syn_skb, + enum tcp_synack_type synack_type, + u32 opt_off); + + /* + * Called when an incoming skb is enqueued to sk->sk_receive_queue + * if BPF_TCP_OPS_FLAG_RCVQ is enabled. + */ + void (*enqueue_rcvq)(struct sock *sk, struct sk_buff *skb); + + /* + * Called after data is dequeued from sk->sk_receive_queue + * if BPF_TCP_OPS_FLAG_RCVQ is enabled. + */ + void (*dequeue_rcvq)(struct sock *sk); +}; + +#define __bpf_tcp_ops_call(op, sk, ...) \ +do { \ + const struct bpf_prog_array_item *item; \ + const struct bpf_tcp_ops *tcp_ops; \ + struct cgroup *cgrp; \ + \ + cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data); \ + rcu_read_lock_dont_migrate(); \ + bpf_cgroup_struct_ops_foreach(tcp_ops, item, cgrp, \ + CGROUP_TCP_SOCK_OPS) { \ + if (tcp_ops->op) \ + tcp_ops->op(sk, ##__VA_ARGS__); \ + } \ + rcu_read_unlock_migrate(); \ +} while (0) + +#define bpf_tcp_ops_call(op, sk, ...) \ +do { \ + if (cgroup_bpf_enabled(CGROUP_TCP_SOCK_OPS)) \ + __bpf_tcp_ops_call(op, sk, ##__VA_ARGS__); \ +} while (0) + +#define bpf_tcp_ops_call_flag(op, flag, sk, ...) \ +do { \ + if (cgroup_bpf_enabled(CGROUP_TCP_SOCK_OPS) && \ + BPF_TCP_OPS_TEST_FLAG(tcp_sk(sk), flag)) \ + __bpf_tcp_ops_call(op, sk, ##__VA_ARGS__); \ +} while (0) + +#define bpf_tcp_ops_call_int(op, init_retval, sk, ...) \ +({ \ + int __retval = (init_retval); \ + if (cgroup_bpf_enabled(CGROUP_TCP_SOCK_OPS)) { \ + const struct bpf_prog_array_item *item; \ + const struct bpf_tcp_ops *tcp_ops; \ + struct bpf_tramp_run_ctx run_ctx; \ + struct bpf_run_ctx *old_run_ctx; \ + struct sock *__sk = sk_to_full_sk(sk); \ + struct request_sock *req = NULL; \ + struct cgroup *cgrp; \ + \ + if (__sk) { \ + run_ctx.retval = (init_retval); \ + cgrp = sock_cgroup_ptr(&__sk->sk_cgrp_data); \ + if (!sk_fullsock(sk)) \ + req = (struct request_sock *)sk; \ + rcu_read_lock_dont_migrate(); \ + old_run_ctx = bpf_set_run_ctx(&run_ctx.run_ctx);\ + bpf_cgroup_struct_ops_foreach(tcp_ops, item, cgrp, \ + CGROUP_TCP_SOCK_OPS) { \ + if (tcp_ops->op) \ + run_ctx.retval = tcp_ops->op(__sk, req, ##__VA_ARGS__); \ + } \ + bpf_reset_run_ctx(old_run_ctx); \ + rcu_read_unlock_migrate(); \ + __retval = run_ctx.retval; \ + } \ + } \ + __retval; \ +}) + +#else +#define __bpf_tcp_ops_call(op, sk, ...) do { } while (0) +#define bpf_tcp_ops_call(op, sk, ...) do { } while (0) +#define bpf_tcp_ops_call_flag(op, flag, sk, ...) do { } while (0) +#define bpf_tcp_ops_call_int(op, init_retval, sk, ...) (init_retval) +#endif + static inline u32 tcp_timeout_init(struct sock *sk) { int timeout; - timeout = tcp_call_bpf(sk, BPF_SOCK_OPS_TIMEOUT_INIT, 0, NULL); - + timeout = tcp_call_bpf(sk, BPF_SOCK_OPS_TIMEOUT_INIT); + timeout = bpf_tcp_ops_call_int(timeout_init, timeout, sk); if (timeout <= 0) timeout = TCP_TIMEOUT_INIT; return min_t(int, timeout, TCP_RTO_MAX); @@ -3013,8 +3193,8 @@ static inline u32 tcp_rwnd_init_bpf(struct sock *sk) { int rwnd; - rwnd = tcp_call_bpf(sk, BPF_SOCK_OPS_RWND_INIT, 0, NULL); - + rwnd = tcp_call_bpf(sk, BPF_SOCK_OPS_RWND_INIT); + rwnd = bpf_tcp_ops_call_int(rwnd_init, rwnd, sk); if (rwnd < 0) rwnd = 0; return rwnd; @@ -3022,13 +3202,13 @@ static inline u32 tcp_rwnd_init_bpf(struct sock *sk) static inline bool tcp_bpf_ca_needs_ecn(struct sock *sk) { - return (tcp_call_bpf(sk, BPF_SOCK_OPS_NEEDS_ECN, 0, NULL) == 1); + return (tcp_call_bpf(sk, BPF_SOCK_OPS_NEEDS_ECN) == 1); } static inline void tcp_bpf_rtt(struct sock *sk, long mrtt, u32 srtt) { - if (BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk), BPF_SOCK_OPS_RTT_CB_FLAG)) - tcp_call_bpf_2arg(sk, BPF_SOCK_OPS_RTT_CB, mrtt, srtt); + tcp_call_bpf_flag(sk, BPF_SOCK_OPS_RTT_CB, mrtt, srtt); + bpf_tcp_ops_call_flag(rtt, RTT, sk, mrtt, srtt); } #if IS_ENABLED(CONFIG_SMC) diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index 732b35cc08d1..f29b0aad440c 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -56,6 +56,19 @@ enum bpf_cond_pseudo_jmp { BPF_MAY_GOTO = 0, + /* + * Follows a call of a bpf subprog. Taken when the subprog left with + * BPF_EXIT that has src_reg == BPF_UNWIND_EXIT. + */ + BPF_UNWIND_GOTO = 1, +}; + +enum bpf_pseudo_exit { + /* + * When bpf_exit->src_reg == BPF_UNWIND_EXIT, it's the same as bpf_exit, + * except that BPF_UNWIND_GOTO after the call is taken. + */ + BPF_UNWIND_EXIT = 1, }; /* Register numbers */ @@ -936,6 +949,33 @@ union bpf_iter_link_info { * 0 on success or -1 if an error occurred (in which case, * *errno* is set appropriately). * + * BPF_PROG_STREAM_OPEN + * Description + * Open a file descriptor for one of the BPF streams associated + * with the program identified by *prog_fd*. The stream is selected + * by *stream_id*. + * + * The returned file descriptor supports **read**\ (2) and + * **poll**\ (2). Reads block while the stream is empty unless + * **BPF_F_STREAM_NONBLOCK** is specified in *flags*. A non-blocking + * read of an empty stream fails with **EAGAIN**. + * + * **poll**\ (2) reports **POLLIN** when data is available and + * **POLLHUP** once the program has been freed, that is, after every + * reference to it, including links and other file descriptors, has + * been dropped. Hangup may lag the final release because program + * teardown is deferred. Buffered data remains readable after + * **POLLHUP** and a read returns zero after all such data has been + * consumed. + * + * The file descriptor is read-only and has the close-on-exec flag + * set. It is not seekable and **lseek**\ (2) fails with **ESPIPE**. + * *flags* may only contain **BPF_F_STREAM_NONBLOCK**. + * + * Return + * A new file descriptor (a nonnegative integer), or -1 if an + * error occurred (in which case, *errno* is set appropriately). + * * NOTES * eBPF objects (maps and programs) can be shared between processes. * @@ -993,6 +1033,7 @@ enum bpf_cmd { BPF_TOKEN_CREATE, BPF_PROG_STREAM_READ_BY_FD, BPF_PROG_ASSOC_STRUCT_OPS, + BPF_PROG_STREAM_OPEN, __MAX_BPF_CMD, BPF_COMMON_ATTRS = 1 << 16, /* Indicate carrying syscall common attrs. */ }; @@ -1316,6 +1357,12 @@ enum bpf_perf_event_type { /* The verifier internal test flag. Behavior is undefined */ #define BPF_F_TEST_REG_INVARIANTS (1U << 7) +/* + * Load and store through a number is an access to the arena of the program + * at the low 32 bits of the number. It's for programs written in Rust. + */ +#define BPF_F_ARENA_SCALAR (1U << 8) + /* link_create.kprobe_multi.flags used in LINK_CREATE command for * BPF_TRACE_KPROBE_MULTI attach type to create return probe. */ @@ -1524,6 +1571,11 @@ enum { BPF_STREAM_STDERR = 2, }; +/* flags for BPF_PROG_STREAM_OPEN command */ +enum { + BPF_F_STREAM_NONBLOCK = (1U << 0), +}; + union bpf_attr { struct { /* anonymous struct used by BPF_MAP_CREATE command */ __u32 map_type; /* one of enum bpf_map_type */ @@ -1756,7 +1808,7 @@ union bpf_attr { __u32 prog_cnt; __u32 count; }; - __u32 :32; + __u32 type_id; /* output: per-program attach_flags. * not allowed to be set during effective query. */ @@ -1950,6 +2002,12 @@ union bpf_attr { __u32 flags; } prog_assoc_struct_ops; + struct { + __u32 prog_fd; + __u32 stream_id; + __u32 flags; + } prog_stream_open; + } __attribute__((aligned(8))); /* The description below is an attempt at providing documentation to eBPF @@ -4867,15 +4925,18 @@ union bpf_attr { * The non-negative copied *buf* length equal to or less than * *size* on success, or a negative error in case of failure. * - * long bpf_load_hdr_opt(struct bpf_sock_ops *skops, void *searchby_res, u32 len, u64 flags) + * long bpf_load_hdr_opt(void *ctx, void *searchby_res, u32 len, u64 flags) * Description * Load header option. Support reading a particular TCP header - * option for bpf program (**BPF_PROG_TYPE_SOCK_OPS**). + * option for bpf program (**BPF_PROG_TYPE_SOCK_OPS**). For the + * **bpf_tcp_ops** struct_ops, this helper can be called from the + * **parse_hdr**\ () and **write_hdr_opt**\ () operators. * - * If *flags* is 0, it will search the option from the - * *skops*\ **->skb_data**. The comment in **struct bpf_sock_ops** - * has details on what skb_data contains under different - * *skops*\ **->op**. + * If *flags* is 0, it will search the option from the packet + * associated with the current operation. For + * **BPF_PROG_TYPE_SOCK_OPS**, the comment in + * **struct bpf_sock_ops** has details on what skb_data + * contains under different *op*. * * The first byte of the *searchby_res* specifies the * kind that it wants to search. @@ -4908,6 +4969,8 @@ union bpf_attr { * * * **BPF_LOAD_HDR_OPT_TCP_SYN** to search from the * saved_syn packet or the just-received syn packet. + * Not supported by the **bpf_tcp_ops** struct_ops, which + * rejects all flags. * * Return * > 0 when found, the header option is copied to *searchby_res*. @@ -4928,9 +4991,9 @@ union bpf_attr { * packet. * * **-EPERM** if the helper cannot be used under the current - * *skops*\ **->op**. + * operation. * - * long bpf_store_hdr_opt(struct bpf_sock_ops *skops, const void *from, u32 len, u64 flags) + * long bpf_store_hdr_opt(void *ctx, const void *from, u32 len, u64 flags) * Description * Store header option. The data will be copied * from buffer *from* with length *len* to the TCP header. @@ -4946,7 +5009,9 @@ union bpf_attr { * by searching the same option in the outgoing skb. * * This helper can only be called during - * **BPF_SOCK_OPS_WRITE_HDR_OPT_CB**. + * **BPF_SOCK_OPS_WRITE_HDR_OPT_CB**, or from the + * **write_hdr_opt**\ () operator of the **bpf_tcp_ops** + * struct_ops. * * Return * 0 on success, or negative error in case of failure: @@ -4961,9 +5026,9 @@ union bpf_attr { * **-EFAULT** on failure to parse the existing header options. * * **-EPERM** if the helper cannot be used under the current - * *skops*\ **->op**. + * operation. * - * long bpf_reserve_hdr_opt(struct bpf_sock_ops *skops, u32 len, u64 flags) + * long bpf_reserve_hdr_opt(void *ctx, u32 len, u64 flags) * Description * Reserve *len* bytes for the bpf header option. The * space will be used by **bpf_store_hdr_opt**\ () later in @@ -4973,7 +5038,9 @@ union bpf_attr { * the total number of bytes will be reserved. * * This helper can only be called during - * **BPF_SOCK_OPS_HDR_OPT_LEN_CB**. + * **BPF_SOCK_OPS_HDR_OPT_LEN_CB**, or from the + * **hdr_opt_len**\ () operator of the **bpf_tcp_ops** + * struct_ops. * * Return * 0 on success, or negative error in case of failure: @@ -4983,7 +5050,7 @@ union bpf_attr { * **-ENOSPC** if there is not enough space in the header. * * **-EPERM** if the helper cannot be used under the current - * *skops*\ **->op**. + * operation. * * void *bpf_inode_storage_get(struct bpf_map *map, void *inode, void *value, u64 flags) * Description @@ -6890,6 +6957,8 @@ struct bpf_link_info { } xdp; struct { __u32 map_id; + __u32 :32; + __u64 cgroup_id; } struct_ops; struct { __u32 pf; @@ -7284,6 +7353,31 @@ enum { */ }; +/* + * Most callbacks of struct bpf_tcp_ops are placed in the slow + * path (e.g., one-shot connection setup or unlikely events like + * timers) and are invoked simply by defining non-NULL callbacks. + * + * Callbacks in the fast path, however, would incur noticeable + * overhead even when set to NULL, so they are disabled by default + * and must be explicitly enabled per socket via bpf_tcp_ops_set_flags(). + * + * The flags are per-socket and shared by all effective bpf_tcp_ops + * programs. + */ +enum { + /* .rtt() */ + BPF_TCP_OPS_FLAG_RTT = (1 << 0), + /* .parse_hdr() */ + BPF_TCP_OPS_FLAG_PARSE_HDR_OPT_ALL = (1 << 1), + BPF_TCP_OPS_FLAG_PARSE_HDR_OPT_UNKNOWN = (1 << 2), + /* .hdr_opt_len() and .write_hdr_opt() */ + BPF_TCP_OPS_FLAG_WRITE_HDR_OPT = (1 << 3), + /* .enqueue_rcvq() and .dequeue_rcvq() */ + BPF_TCP_OPS_FLAG_RCVQ = (1 << 4), + BPF_TCP_OPS_FLAG_ALL = (1 << 5) - 1, +}; + /* List of TCP states. There is a build check in net/ipv4/tcp.c to detect * changes between the TCP and BPF versions. Ideally this should never happen. * If it does, we need to add code to convert them before calling @@ -7600,6 +7694,10 @@ struct bpf_task_work { __u64 __opaque; } __attribute__((aligned(8))); +struct bpf_rcu_head { + __u64 __opaque[6]; +} __attribute__((aligned(8))); + struct bpf_wq { __u64 __opaque[2]; } __attribute__((aligned(8))); diff --git a/include/uapi/linux/btf.h b/include/uapi/linux/btf.h index 618167cab4e6..11d65871d163 100644 --- a/include/uapi/linux/btf.h +++ b/include/uapi/linux/btf.h @@ -54,8 +54,8 @@ struct btf_type { * decl_tag and type_tag */ __u32 info; - /* "size" is used by INT, ENUM, STRUCT, UNION, DATASEC and ENUM64. - * "size" tells the size of the type it is describing. + /* "size" is used by INT, ENUM, STRUCT, UNION, DATASEC, ENUM64 + * and LOC_PARAM. "size" tells the size of the type it is describing. * * "type" is used by PTR, TYPEDEF, VOLATILE, CONST, RESTRICT, * FUNC, FUNC_PROTO, VAR, DECL_TAG and TYPE_TAG. @@ -92,7 +92,9 @@ enum { BTF_KIND_DECL_TAG = 17, /* Decl Tag */ BTF_KIND_TYPE_TAG = 18, /* Type Tag */ BTF_KIND_ENUM64 = 19, /* Enumeration up to 64-bit values */ - + BTF_KIND_LOC_PARAM = 20, /* Location parameter information */ + BTF_KIND_LOC_PROTO = 21, /* Location prototype for site */ + BTF_KIND_LOCSEC = 22, /* Location section */ NR_BTF_KINDS, BTF_KIND_MAX = NR_BTF_KINDS - 1, }; @@ -212,4 +214,79 @@ struct btf_enum64 { __u32 val_hi32; }; +/* + * BTF_KIND_LOC_PARAM is followed by a single "struct btf_loc_param" + * that contains flags specifying the contents of the vlen-specified + * number of 4-byte values that follow. + */ +struct btf_loc_param { + __u32 flags; + __u32 values[]; +}; + +/* + * The combination of size, vlen and flags gives us the means to interpret + * the following vlen-specified set of 4-byte values: + * + * - a BTF_LOC_PARAM_CONST is a constant value; combination + * of size, vlen and _SIGNED flag determines it. If the value requires + * 64 bits it is stored in {lo,hi} order. + * - a BTF_LOC_PARAM_ADDR|BTF_LOC_PARAM_CONST is an address that should be + * normalized with respect to kernel/module base address. + * - a BTF_LOC_PARAM_REG with vlen 1 is a simple register number; + * with vlen 2 it is a multi-register parameter. Register numbers are + * numbers derived from DW_OP_reg values, i.e. 0 is DW_OP_reg0; since + * DW_OP_fbreg has its own special DW_OP_value and DW_OP_regx can refer + * to an arbitrary register number, we reserve BTF_LOC_PARAM_FBREG for + * the frame base register to avoid collisions. + * - a _REG | OFFSET describes an address without dereferencing it. + * - a _REG | DEREF with vlen 1 dereferences the value in the register + * number specified. + * - a REG | DEREF | OFFSET with vlen > 1 specifies the register number + * in the first 4-byte value and the offset in the remainder. + * In the case of REG and OFFSET combinations, the OFFSET has the width + * of the value words while the type size describes the represented parameter, + * so for example a REG | DEREF | OFFSET with size 8 and vlen 2 would be + * an 8-byte register dereference with signed 4-byte offset, since the vlen 2 + * values consist of a register value and the signed value. + */ +enum btf_loc_param_flags { + BTF_LOC_PARAM_SIGNED = 0x1, + BTF_LOC_PARAM_CONST = 0x2, + BTF_LOC_PARAM_ADDR = 0x4, + BTF_LOC_PARAM_REG = 0x8, + BTF_LOC_PARAM_DEREF = 0x10, + BTF_LOC_PARAM_OFFSET = 0x20, +}; + +enum { + BTF_LOC_PARAM_FBREG = 0xffffffff +}; + +/* + * BTF_KIND_LOC_PROTO specifies location prototypes; i.e. how locations relate + * to parameters; a struct btf_type of BTF_KIND_LOC_PROTO is followed by a + * vlen-specified number of __u32 BTF type ids which specify the associated + * BTF_KIND_LOC_PARAM for each function parameter associated with the + * location. The type should either be 0 (no location info) or point at + * a BTF_KIND_LOC_PARAM. + */ + +/* + * BTF_KIND_LOCSEC consists of vlen-specified number of "struct btf_loc" + * containing location site-specific information for a specific ELF section; + * for example locations in ".text" are in a LOCSEC named "inline.text". + * + * - function (func) + * - location prototype type id (loc_proto) + * - address offset (offset) relative to the runtime base address of the + * ELF section associated with the LOCSEC + */ + +struct btf_loc { + __u32 func; + __u32 loc_proto; + __u32 offset; +}; + #endif /* _UAPI__LINUX_BTF_H__ */ diff --git a/include/uapi/linux/keyctl.h b/include/uapi/linux/keyctl.h index 4c8884eea808..fa85b9760391 100644 --- a/include/uapi/linux/keyctl.h +++ b/include/uapi/linux/keyctl.h @@ -24,6 +24,7 @@ #define KEY_SPEC_GROUP_KEYRING -6 /* - key ID for GID-specific keyring */ #define KEY_SPEC_REQKEY_AUTH_KEY -7 /* - key ID for assumed request_key auth key */ #define KEY_SPEC_REQUESTOR_KEYRING -8 /* - key ID for request_key() dest keyring */ +#define KEY_SPEC_BPF_KEYRING -9 /* - key ID for the BPF-specific keyring */ /* request-key default keyrings */ #define KEY_REQKEY_DEFL_NO_CHANGE -1 diff --git a/kernel/bpf/Kconfig b/kernel/bpf/Kconfig index eb3de35734f0..98493e32db2a 100644 --- a/kernel/bpf/Kconfig +++ b/kernel/bpf/Kconfig @@ -17,6 +17,10 @@ config HAVE_CBPF_JIT config HAVE_EBPF_JIT bool +# KASAN support for JIT compiler +config HAVE_EBPF_JIT_KASAN + bool + # Used by archs to tell that they want the BPF JIT compiler enabled by # default for kernels that were compiled with BPF JIT support. config ARCH_WANT_DEFAULT_BPF_JIT @@ -87,6 +91,15 @@ config BPF_UNPRIV_DEFAULT_OFF If you are unsure how to answer this question, answer Y. +config BPF_CRYPTO + def_bool y + depends on BPF_SYSCALL + select CRYPTO_LIB_AES_CBC + select CRYPTO_LIB_AES_ECB + help + Provide the kfuncs needed for BPF programs to encrypt and decrypt + data. The supported algorithms are AES-CBC and AES-ECB. + source "kernel/bpf/preload/Kconfig" config BPF_LSM @@ -101,4 +114,17 @@ config BPF_LSM If you are unsure how to answer this question, answer N. +config BPF_JIT_KASAN + bool + depends on HAVE_EBPF_JIT_KASAN + depends on KASAN_GENERIC + depends on KASAN_VMALLOC + depends on BPF_JIT + default y + help + Makes JIT compiler insert generic outline KASAN checks in BPF + programs when they are inserted in the kernel. This feature is + automatically enabled if the needed set of KASAN and BPF + configuration options is enabled. + endmenu # "BPF subsystem" diff --git a/kernel/bpf/Makefile b/kernel/bpf/Makefile index 90255d80e5be..269810b128c4 100644 --- a/kernel/bpf/Makefile +++ b/kernel/bpf/Makefile @@ -6,7 +6,7 @@ cflags-nogcse-$(CONFIG_X86)$(CONFIG_CC_IS_GCC) := -fno-gcse endif CFLAGS_core.o += -Wno-override-init $(cflags-nogcse-yy) -obj-$(CONFIG_BPF_SYSCALL) += syscall.o verifier.o inode.o helpers.o tnum.o cnum.o log.o token.o liveness.o const_fold.o diagnostics.o +obj-$(CONFIG_BPF_SYSCALL) += syscall.o verifier.o inode.o helpers.o tnum.o cnum.o log.o token.o liveness.o const_fold.o diagnostics.o loops.o scev.o obj-$(CONFIG_BPF_SYSCALL) += bpf_iter.o map_iter.o task_iter.o prog_iter.o link_iter.o obj-$(CONFIG_BPF_SYSCALL) += hashtab.o arraymap.o percpu_freelist.o bpf_lru_list.o lpm_trie.o map_in_map.o bloom_filter.o obj-$(CONFIG_BPF_SYSCALL) += local_storage.o queue_stack_maps.o ringbuf.o bpf_insn_array.o @@ -27,6 +27,9 @@ obj-$(CONFIG_BPF_SYSCALL) += offload.o obj-$(CONFIG_BPF_SYSCALL) += net_namespace.o obj-$(CONFIG_BPF_SYSCALL) += tcx.o endif +ifeq ($(CONFIG_KEYS),y) +obj-$(CONFIG_BPF_SYSCALL) += keys.o +endif ifeq ($(CONFIG_PERF_EVENTS),y) obj-$(CONFIG_BPF_SYSCALL) += stackmap.o endif @@ -39,6 +42,7 @@ obj-$(CONFIG_BPF_SYSCALL) += reuseport_array.o endif ifeq ($(CONFIG_SYSFS),y) obj-$(CONFIG_DEBUG_INFO_BTF) += sysfs_btf.o +obj-$(CONFIG_DEBUG_INFO_BTF_INLINE) += btf_vmlinux_inline.o endif ifeq ($(CONFIG_BPF_JIT),y) obj-$(CONFIG_BPF_SYSCALL) += bpf_struct_ops.o @@ -55,9 +59,7 @@ obj-$(CONFIG_BPF_SYSCALL) += cpumask.o # semantics within pahole are revisited accordingly. obj-${CONFIG_BPF_LSM} += bpf_lsm_proto.o bpf_lsm.o endif -ifneq ($(CONFIG_CRYPTO),) -obj-$(CONFIG_BPF_SYSCALL) += crypto.o -endif +obj-$(CONFIG_BPF_CRYPTO) += crypto.o obj-$(CONFIG_BPF_PRELOAD) += preload/ obj-$(CONFIG_BPF_SYSCALL) += relo_core.o diff --git a/kernel/bpf/arena.c b/kernel/bpf/arena.c index 7b6847200b43..868589f07330 100644 --- a/kernel/bpf/arena.c +++ b/kernel/bpf/arena.c @@ -146,7 +146,7 @@ static long compute_pgoff(struct bpf_arena *arena, long uaddr) struct apply_range_data { struct bpf_arena *arena; - struct page **pages; + struct llist_head *pages; int i; }; @@ -158,13 +158,17 @@ struct clear_range_data { static int apply_range_set_cb(pte_t *pte, unsigned long addr, void *data) { struct apply_range_data *d = data; + struct llist_node *node; struct page *page; pte_t pteval; if (!data) return 0; - page = d->pages[d->i]; + node = READ_ONCE(d->pages->first); + if (WARN_ON_ONCE(!node)) + return -EINVAL; + page = llist_entry(node, struct page, pcp_llist); /* paranoia, similar to vmap_pages_pte_range() */ if (WARN_ON_ONCE(!pfn_valid(page_to_pfn(page)))) return -EINVAL; @@ -197,6 +201,7 @@ static int apply_range_set_cb(pte_t *pte, unsigned long addr, void *data) return -EBUSY; set_pte_at(&init_mm, addr, pte, pteval); #endif + WARN_ON_ONCE(llist_del_first(d->pages) != node); d->i++; WRITE_ONCE(d->arena->nr_pages, d->arena->nr_pages + 1); return 0; @@ -312,8 +317,8 @@ static struct bpf_map *arena_map_alloc(union bpf_attr *attr) INIT_WORK(&arena->free_work, arena_free_worker); bpf_map_init_from_attr(&arena->map, attr); - err = bpf_map_alloc_pages(&arena->map, NUMA_NO_NODE, 1, &arena->scratch_page); - if (err) + arena->scratch_page = bpf_alloc_page(NUMA_NO_NODE, true); + if (!arena->scratch_page) goto err_free_arena; range_tree_init(&arena->rt); @@ -481,7 +486,10 @@ static vm_fault_t arena_vm_fault(struct vm_fault *vmf) struct bpf_map *map = vmf->vma->vm_file->private_data; struct bpf_arena *arena = container_of(map, struct bpf_arena, map); struct mem_cgroup *new_memcg, *old_memcg; - struct page *page; + LLIST_HEAD(pages); + struct page *page, *new_page = NULL; + struct apply_range_data data; + vm_fault_t fault_ret; long kbase, kaddr; unsigned long flags; int ret; @@ -489,59 +497,123 @@ static vm_fault_t arena_vm_fault(struct vm_fault *vmf) kbase = bpf_arena_get_kern_vm_start(arena); kaddr = kbase + (u32)(vmf->address); - if (raw_res_spin_lock_irqsave(&arena->spinlock, flags)) + page = vmalloc_to_page((void *)kaddr); + if (!page && !(arena->map.map_flags & BPF_F_SEGV_ON_FAULT)) { + /* + * Preallocate outside the lock so the allocation can reclaim; + * __GFP_RETRY_MAYFAIL keeps the OOM killer out of it. + */ + bpf_map_memcg_enter(&arena->map, &old_memcg, &new_memcg); + new_page = alloc_pages_node(map->numa_node, + GFP_KERNEL | __GFP_ZERO | + __GFP_ACCOUNT | __GFP_NOWARN | + __GFP_RETRY_MAYFAIL, 0); + bpf_map_memcg_exit(old_memcg, new_memcg); + } + + if (raw_res_spin_lock_irqsave(&arena->spinlock, flags)) { /* * A failed lock means a possible deadlock was detected. Don't * return VM_FAULT_RETRY: this handler never took mmap_lock, but * the fault path would re-take it on retry and deadlock. Fail. */ + if (new_page) + free_pages_nolock(new_page, 0); return VM_FAULT_SIGBUS; + } page = vmalloc_to_page((void *)kaddr); if (page) { - if (page == arena->scratch_page) + if (page == arena->scratch_page) { /* BPF triggered scratch here; don't lazy-alloc over it */ - goto out_sigsegv; + fault_ret = (arena->map.map_flags & BPF_F_SEGV_ON_FAULT) + ? VM_FAULT_SIGSEGV : VM_FAULT_SIGBUS; + goto out_err_locked; + } /* already have a page vmap-ed */ goto out; } + if (arena->map.map_flags & BPF_F_SEGV_ON_FAULT) { + /* + * User space requested to segfault when page is not allocated + * by bpf prog + */ + fault_ret = VM_FAULT_SIGSEGV; + goto out_err_locked; + } + + if (is_range_tree_set(&arena->rt, vmf->pgoff, 1)) { + /* Allocated without a page: reserved or being freed */ + fault_ret = VM_FAULT_SIGBUS; + goto out_err_locked; + } + bpf_map_memcg_enter(&arena->map, &old_memcg, &new_memcg); - if (arena->map.map_flags & BPF_F_SEGV_ON_FAULT) - /* User space requested to segfault when page is not allocated by bpf prog */ - goto out_sigsegv_memcg; + if (!new_page) { + /* + * The probed page was freed meanwhile or preallocation failed; + * try the non-blocking allocator, we cannot sleep here. + */ + new_page = bpf_alloc_page(map->numa_node, false); + if (!new_page) { + fault_ret = VM_FAULT_SIGBUS; + goto out_err_locked_memcg; + } + } ret = range_tree_clear(&arena->rt, vmf->pgoff, 1); - if (ret) - goto out_sigsegv_memcg; - - struct apply_range_data data = { .arena = arena, .pages = &page, .i = 0 }; - /* Account into memcg of the process that created bpf_arena */ - ret = bpf_map_alloc_pages(map, NUMA_NO_NODE, 1, &page); if (ret) { - range_tree_set(&arena->rt, vmf->pgoff, 1); - goto out_sigsegv_memcg; + fault_ret = VM_FAULT_SIGBUS; + goto out_err_locked_memcg; } + llist_add(&new_page->pcp_llist, &pages); + data.arena = arena; + data.pages = &pages; + data.i = 0; ret = apply_to_page_range(&init_mm, kaddr, PAGE_SIZE, apply_range_set_cb, &data); if (ret) { + llist_del_first(&pages); range_tree_set(&arena->rt, vmf->pgoff, 1); - free_pages_nolock(page, 0); - goto out_sigsegv_memcg; + fault_ret = VM_FAULT_SIGBUS; + goto out_err_locked_memcg; } flush_vmap_cache(kaddr, PAGE_SIZE); bpf_map_memcg_exit(old_memcg, new_memcg); + /* new_page was consumed */ + page = new_page; + new_page = NULL; out: page_ref_add(page, 1); raw_res_spin_unlock_irqrestore(&arena->spinlock, flags); - vmf->page = page; - return 0; -out_sigsegv_memcg: + if (new_page) + free_pages_nolock(new_page, 0); + + /* + * A free clears the kernel pte and then takes the page lock before it + * zaps user vma-s. Either it waits for this fault and zaps its pte, or + * the page is gone from here. + */ + lock_page(page); + if (vmalloc_to_page((void *)kaddr) == page && !PageHWPoison(page)) { + vmf->page = page; + return VM_FAULT_LOCKED; + } + /* The page was freed meanwhile. Let the fault start over. */ + fault_ret = PageHWPoison(page) ? VM_FAULT_HWPOISON : VM_FAULT_NOPAGE; + unlock_page(page); + put_page(page); + return fault_ret; + +out_err_locked_memcg: bpf_map_memcg_exit(old_memcg, new_memcg); -out_sigsegv: +out_err_locked: raw_res_spin_unlock_irqrestore(&arena->spinlock, flags); - return VM_FAULT_SIGSEGV; + if (new_page) + free_pages_nolock(new_page, 0); + return fault_ret; } static const struct vm_operations_struct arena_vm_ops = { @@ -661,6 +733,27 @@ static u64 clear_lo32(u64 val) return val & ~(u64)~0U; } +static int arena_adjust_tree(struct bpf_arena *arena, long uaddr, long page_cnt, long *pgoff) +{ + int ret; + + /* Special case where user is requesting specific range. */ + if (uaddr) { + ret = is_range_tree_set(&arena->rt, *pgoff, page_cnt); + if (ret) + return ret; + return range_tree_clear(&arena->rt, *pgoff, page_cnt); + } + + ret = range_tree_find(&arena->rt, page_cnt); + if (ret < 0) + return ret; + + *pgoff = ret; + + return range_tree_clear(&arena->rt, *pgoff, page_cnt); +} + /* * Allocate pages and vmap them into kernel vmalloc area. * Later the pages will be mmaped into user space vma. @@ -673,13 +766,13 @@ static long arena_alloc_pages(struct bpf_arena *arena, long uaddr, long page_cnt u64 kern_vm_start = bpf_arena_get_kern_vm_start(arena); struct mem_cgroup *new_memcg, *old_memcg; struct apply_range_data data; - struct page **pages = NULL; - long remaining, mapped = 0; - long alloc_pages; + LLIST_HEAD(pages); + long mapped = 0; unsigned long flags; long pgoff = 0; u32 uaddr32; - int ret, i; + long addr = 0; + int ret; if (node_id != NUMA_NO_NODE && ((unsigned int)node_id >= nr_node_ids || !node_online(node_id))) @@ -696,89 +789,63 @@ static long arena_alloc_pages(struct bpf_arena *arena, long uaddr, long page_cnt /* requested address will be outside of user VMA */ return 0; } - bpf_map_memcg_enter(&arena->map, &old_memcg, &new_memcg); - /* Cap allocation size to KMALLOC_MAX_CACHE_SIZE so kmalloc_nolock() can succeed. */ - alloc_pages = min(page_cnt, KMALLOC_MAX_CACHE_SIZE / sizeof(struct page *)); - pages = kmalloc_nolock(alloc_pages * sizeof(struct page *), __GFP_ACCOUNT, NUMA_NO_NODE); - if (!pages) { - bpf_map_memcg_exit(old_memcg, new_memcg); - return 0; - } + + ret = bpf_alloc_pages(node_id, page_cnt, &pages, sleepable); + if (ret) + goto out_memcg; + data.arena = arena; - data.pages = pages; + data.pages = &pages; + data.i = 0; if (raw_res_spin_lock_irqsave(&arena->spinlock, flags)) goto out_free_pages; - if (uaddr) { - ret = is_range_tree_set(&arena->rt, pgoff, page_cnt); - if (ret) - goto out_unlock_free_pages; - ret = range_tree_clear(&arena->rt, pgoff, page_cnt); - } else { - ret = pgoff = range_tree_find(&arena->rt, page_cnt); - if (pgoff >= 0) - ret = range_tree_clear(&arena->rt, pgoff, page_cnt); + ret = arena_adjust_tree(arena, uaddr, page_cnt, &pgoff); + if (ret) { + raw_res_spin_unlock_irqrestore(&arena->spinlock, flags); + goto out_free_pages; } - if (ret) - goto out_unlock_free_pages; - remaining = page_cnt; uaddr32 = (u32)(arena->user_vm_start + pgoff * PAGE_SIZE); - while (remaining) { - long this_batch = min(remaining, alloc_pages); + /* + * Earlier checks made sure that uaddr32 + page_cnt * PAGE_SIZE - 1 + * will not overflow 32-bit. Lower 32-bit need to represent + * contiguous user address range. + * Map these pages at kern_vm_start base. + * kern_vm_start + uaddr32 + page_cnt * PAGE_SIZE - 1 can overflow + * lower 32-bit and it's ok. + */ + ret = apply_to_page_range(&init_mm, kern_vm_start + uaddr32, + page_cnt << PAGE_SHIFT, apply_range_set_cb, &data); + mapped = data.i; + if (ret) + goto out_unmap; - /* zeroing is needed, since alloc_pages_bulk() only fills in non-zero entries */ - memset(pages, 0, this_batch * sizeof(struct page *)); - - ret = bpf_map_alloc_pages(&arena->map, node_id, this_batch, pages); - if (ret) - goto out; - - /* - * Earlier checks made sure that uaddr32 + page_cnt * PAGE_SIZE - 1 - * will not overflow 32-bit. Lower 32-bit need to represent - * contiguous user address range. - * Map these pages at kern_vm_start base. - * kern_vm_start + uaddr32 + page_cnt * PAGE_SIZE - 1 can overflow - * lower 32-bit and it's ok. - */ - data.i = 0; - ret = apply_to_page_range(&init_mm, - kern_vm_start + uaddr32 + (mapped << PAGE_SHIFT), - this_batch << PAGE_SHIFT, apply_range_set_cb, &data); - if (ret) { - /* data.i pages were mapped, account them and free the remaining */ - mapped += data.i; - for (i = data.i; i < this_batch; i++) - free_pages_nolock(pages[i], 0); - goto out; - } - - mapped += this_batch; - remaining -= this_batch; - } flush_vmap_cache(kern_vm_start + uaddr32, mapped << PAGE_SHIFT); raw_res_spin_unlock_irqrestore(&arena->spinlock, flags); - kfree_nolock(pages); - bpf_map_memcg_exit(old_memcg, new_memcg); - return clear_lo32(arena->user_vm_start) + uaddr32; -out: + + addr = clear_lo32(arena->user_vm_start) + uaddr32; + goto out_memcg; + +out_unmap: + /* Error handling: Undo partial mappings. */ + flush_vmap_cache(kern_vm_start + uaddr32, mapped << PAGE_SHIFT); range_tree_set(&arena->rt, pgoff + mapped, page_cnt - mapped); raw_res_spin_unlock_irqrestore(&arena->spinlock, flags); - if (mapped) { - flush_vmap_cache(kern_vm_start + uaddr32, mapped << PAGE_SHIFT); + if (mapped) arena_free_pages(arena, uaddr32, mapped, sleepable); - } - goto out_free_pages; -out_unlock_free_pages: - raw_res_spin_unlock_irqrestore(&arena->spinlock, flags); + out_free_pages: - kfree_nolock(pages); + /* Error handling: Free back any unmapped pages. */ + bpf_free_pages(&pages); + +out_memcg: bpf_map_memcg_exit(old_memcg, new_memcg); - return 0; + + return addr; } /* @@ -807,7 +874,7 @@ static void zap_pages(struct bpf_arena *arena, long uaddr, long page_cnt) * otherwise call B's mark would make call A skip a vml that A has * not yet zapped for A's uaddr range. */ - mutex_lock(&arena->zap_mutex); + lockdep_assert_held(&arena->zap_mutex); mutex_lock(&arena->lock); my_gen = ++arena->zap_gen; for (;;) { @@ -841,49 +908,41 @@ static void zap_pages(struct bpf_arena *arena, long uaddr, long page_cnt) mutex_lock(&arena->lock); } mutex_unlock(&arena->lock); - mutex_unlock(&arena->zap_mutex); } -static void arena_free_pages(struct bpf_arena *arena, long uaddr, long page_cnt, bool sleepable) +/* + * The range stays allocated in the range tree until the pages are unmapped + * from vmalloc area and from all user space vma-s, so that nobody can + * populate it while stale TLB entries or user ptes are still around. + */ +static int arena_free_range(struct bpf_arena *arena, long uaddr, long page_cnt) { - struct mem_cgroup *new_memcg, *old_memcg; - u64 full_uaddr, uaddr_end; - long kaddr, pgoff; - struct page *page; - struct llist_head free_pages; - struct llist_node *pos, *t; - struct arena_free_span *s; + u64 full_uaddr = clear_lo32(arena->user_vm_start) + uaddr; + long kaddr = bpf_arena_get_kern_vm_start(arena) + uaddr; + long pgoff = compute_pgoff(arena, uaddr); struct clear_range_data cdata; + struct llist_node *pos, *t; + LLIST_HEAD(free_pages); unsigned long flags; - int ret = 0; + struct page *page; + int ret; - /* only aligned lower 32-bit are relevant */ - uaddr = (u32)uaddr; - uaddr &= PAGE_MASK; - kaddr = bpf_arena_get_kern_vm_start(arena) + uaddr; - full_uaddr = clear_lo32(arena->user_vm_start) + uaddr; - if (full_uaddr < arena->user_vm_start) - return; - uaddr_end = min(arena->user_vm_end, full_uaddr + (page_cnt << PAGE_SHIFT)); - if (full_uaddr >= uaddr_end) - return; - - page_cnt = (uaddr_end - full_uaddr) >> PAGE_SHIFT; - pgoff = compute_pgoff(arena, uaddr); - bpf_map_memcg_enter(&arena->map, &old_memcg, &new_memcg); - - if (!sleepable) - goto defer; + /* An overlapping free must not release the range before this one is done */ + guard(mutex)(&arena->zap_mutex); ret = raw_res_spin_lock_irqsave(&arena->spinlock, flags); - - /* Can't proceed without holding the spinlock so defer the free */ if (ret) - goto defer; + return ret; - range_tree_set(&arena->rt, pgoff, page_cnt); + /* + * Take the free pages of the range too. It allocates, and can fail, + * only when the whole range is free. Then there is nothing to do. + */ + if (range_tree_clear(&arena->rt, pgoff, page_cnt)) { + raw_res_spin_unlock_irqrestore(&arena->spinlock, flags); + return 0; + } - init_llist_head(&free_pages); cdata.arena = arena; cdata.free_pages = &free_pages; /* clear ptes and collect struct pages */ @@ -893,6 +952,12 @@ static void arena_free_pages(struct bpf_arena *arena, long uaddr, long page_cnt, /* drop the lock to do the tlb flush and zap pages */ raw_res_spin_unlock_irqrestore(&arena->spinlock, flags); + /* wait for arena_vm_fault() that found the page before its pte was cleared */ + llist_for_each_entry(page, free_pages.first, pcp_llist) { + lock_page(page); + unlock_page(page); + } + /* ensure no stale TLB entries */ flush_tlb_kernel_range(kaddr, kaddr + (page_cnt * PAGE_SIZE)); @@ -911,8 +976,39 @@ static void arena_free_pages(struct bpf_arena *arena, long uaddr, long page_cnt, zap_pages(arena, full_uaddr, 1); __free_page(page); } - bpf_map_memcg_exit(old_memcg, new_memcg); + /* The callers hold no other rqspinlock, so this one can only time out */ + while (raw_res_spin_lock_irqsave(&arena->spinlock, flags)) + cond_resched(); + range_tree_set(&arena->rt, pgoff, page_cnt); + raw_res_spin_unlock_irqrestore(&arena->spinlock, flags); + return 0; +} + +static void arena_free_pages(struct bpf_arena *arena, long uaddr, long page_cnt, bool sleepable) +{ + struct mem_cgroup *new_memcg, *old_memcg; + u64 full_uaddr, uaddr_end; + struct arena_free_span *s; + + /* only aligned lower 32-bit are relevant */ + uaddr = (u32)uaddr; + uaddr &= PAGE_MASK; + full_uaddr = clear_lo32(arena->user_vm_start) + uaddr; + if (full_uaddr < arena->user_vm_start) + return; + uaddr_end = min(arena->user_vm_end, full_uaddr + (page_cnt << PAGE_SHIFT)); + if (full_uaddr >= uaddr_end) + return; + + page_cnt = (uaddr_end - full_uaddr) >> PAGE_SHIFT; + bpf_map_memcg_enter(&arena->map, &old_memcg, &new_memcg); + + /* Can't proceed without holding the spinlock so defer the free */ + if (!sleepable || arena_free_range(arena, uaddr, page_cnt)) + goto defer; + + bpf_map_memcg_exit(old_memcg, new_memcg); return; defer: @@ -974,66 +1070,18 @@ static void arena_free_worker(struct work_struct *work) { struct bpf_arena *arena = container_of(work, struct bpf_arena, free_work); struct mem_cgroup *new_memcg, *old_memcg; - struct llist_node *list, *pos, *t; + struct llist_node *pos, *t; struct arena_free_span *s; - u64 arena_vm_start, user_vm_start; - struct llist_head free_pages; - struct clear_range_data cdata; - struct page *page; - unsigned long full_uaddr; - long kaddr, page_cnt, pgoff; - unsigned long flags; - - if (raw_res_spin_lock_irqsave(&arena->spinlock, flags)) { - schedule_work(work); - return; - } bpf_map_memcg_enter(&arena->map, &old_memcg, &new_memcg); - init_llist_head(&free_pages); - cdata.arena = arena; - cdata.free_pages = &free_pages; - arena_vm_start = bpf_arena_get_kern_vm_start(arena); - user_vm_start = bpf_arena_get_user_vm_start(arena); - - list = llist_del_all(&arena->free_spans); - llist_for_each(pos, list) { + llist_for_each_safe(pos, t, llist_del_all(&arena->free_spans)) { s = llist_entry(pos, struct arena_free_span, node); - page_cnt = s->page_cnt; - kaddr = arena_vm_start + s->uaddr; - pgoff = compute_pgoff(arena, s->uaddr); - - /* clear ptes and collect pages in free_pages llist */ - apply_to_existing_page_range(&init_mm, kaddr, page_cnt << PAGE_SHIFT, - apply_range_clear_cb, &cdata); - - range_tree_set(&arena->rt, pgoff, page_cnt); - } - raw_res_spin_unlock_irqrestore(&arena->spinlock, flags); - - /* Iterate the list again without holding spinlock to do the tlb flush and zap_pages */ - llist_for_each_safe(pos, t, list) { - s = llist_entry(pos, struct arena_free_span, node); - page_cnt = s->page_cnt; - full_uaddr = clear_lo32(user_vm_start) + s->uaddr; - kaddr = arena_vm_start + s->uaddr; - - /* ensure no stale TLB entries */ - flush_tlb_kernel_range(kaddr, kaddr + (page_cnt * PAGE_SIZE)); - - /* remove pages from user vmas */ - zap_pages(arena, full_uaddr, page_cnt); - + while (arena_free_range(arena, s->uaddr, s->page_cnt)) + cond_resched(); kfree_nolock(s); } - /* free all pages collected by apply_to_existing_page_range() in the first loop */ - llist_for_each_safe(pos, t, __llist_del_all(&free_pages)) { - page = llist_entry(pos, struct page, pcp_llist); - __free_page(page); - } - bpf_map_memcg_exit(old_memcg, new_memcg); } diff --git a/kernel/bpf/arraymap.c b/kernel/bpf/arraymap.c index 0ce26b538075..133f189bf6d2 100644 --- a/kernel/bpf/arraymap.c +++ b/kernel/bpf/arraymap.c @@ -438,15 +438,15 @@ int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value, u32 size; int cpu, off = 0; - if (unlikely((map_flags & BPF_F_LOCK) || (u32)map_flags > BPF_F_ALL_CPUS)) - /* unknown flags */ + if (unlikely(((map_flags & BPF_EXIST_FLAGS) == BPF_EXIST_FLAGS) || + ((u32)map_flags & ~(BPF_EXIST_FLAGS | BPF_CPU_FLAGS)))) return -EINVAL; if (unlikely(index >= array->map.max_entries)) /* all elements were pre-allocated, cannot insert a new one */ return -E2BIG; - if (unlikely(map_flags == BPF_NOEXIST)) + if (unlikely(map_flags & BPF_NOEXIST)) /* all elements already exist */ return -EEXIST; @@ -1035,7 +1035,7 @@ static void *prog_fd_array_get_ptr(struct bpf_map *map, } mutex_lock(&prog->aux->ext_mutex); - is_extended = prog->aux->is_extended; + is_extended = prog->aux->freplace_link_cnt > 0; if (!is_extended) prog->aux->prog_array_member_cnt++; mutex_unlock(&prog->aux->ext_mutex); diff --git a/kernel/bpf/backtrack.c b/kernel/bpf/backtrack.c index 47282ffeeaf9..43d5740034c0 100644 --- a/kernel/bpf/backtrack.c +++ b/kernel/bpf/backtrack.c @@ -9,7 +9,8 @@ /* for any branch, call, exit record the history of jmps in the given state */ int bpf_push_jmp_history(struct bpf_verifier_env *env, struct bpf_verifier_state *cur, - int insn_flags, int spi, int frame, u64 linked_regs) + int insn_flags, int spi, int frame, const u16 *linked_regs, + u8 linked_regs_cnt) { u32 cnt = cur->jmp_history_cnt; struct bpf_jmp_history_entry *p; @@ -27,10 +28,13 @@ int bpf_push_jmp_history(struct bpf_verifier_env *env, struct bpf_verifier_state env->cur_hist_ent->flags |= insn_flags; env->cur_hist_ent->spi = spi; env->cur_hist_ent->frame = frame; - verifier_bug_if(env->cur_hist_ent->linked_regs != 0, env, - "insn history: insn_idx %d linked_regs: %#llx", - env->insn_idx, env->cur_hist_ent->linked_regs); - env->cur_hist_ent->linked_regs = linked_regs; + verifier_bug_if(env->cur_hist_ent->linked_regs_cnt != 0, env, + "insn history: insn_idx %d has %u linked regs", + env->insn_idx, env->cur_hist_ent->linked_regs_cnt); + if (linked_regs_cnt) + memcpy(env->cur_hist_ent->linked_regs, linked_regs, + linked_regs_cnt * sizeof(*linked_regs)); + env->cur_hist_ent->linked_regs_cnt = linked_regs_cnt; return 0; } @@ -47,7 +51,9 @@ int bpf_push_jmp_history(struct bpf_verifier_env *env, struct bpf_verifier_state p->flags = insn_flags; p->spi = spi; p->frame = frame; - p->linked_regs = linked_regs; + if (linked_regs_cnt) + memcpy(p->linked_regs, linked_regs, linked_regs_cnt * sizeof(*linked_regs)); + p->linked_regs_cnt = linked_regs_cnt; cur->jmp_history_cnt = cnt; env->cur_hist_ent = p; @@ -123,13 +129,26 @@ static inline void bt_reset(struct backtrack_state *bt) bt->env = env; } -static inline u32 bt_empty(struct backtrack_state *bt) +static inline bool bt_frame_stack_empty(struct backtrack_state *bt, u32 frame) { - u64 mask = 0; + return bitmap_empty(bt->stack_masks[frame], MAX_BPF_STACK_SLOTS); +} + +static inline bool bt_stack_empty(struct backtrack_state *bt) +{ + return bt_frame_stack_empty(bt, bt->frame); +} + +static inline bool bt_empty(struct backtrack_state *bt) +{ + u32 mask = 0; int i; - for (i = 0; i <= bt->frame; i++) - mask |= bt->reg_masks[i] | bt->stack_masks[i] | bt->stack_arg_masks[i]; + for (i = 0; i <= bt->frame; i++) { + mask |= bt->reg_masks[i] | bt->stack_arg_masks[i]; + if (!bt_frame_stack_empty(bt, i)) + return false; + } return mask == 0; } @@ -181,7 +200,7 @@ static inline void bt_clear_reg(struct backtrack_state *bt, u32 reg) static inline void bt_clear_frame_slot(struct backtrack_state *bt, u32 frame, u32 slot) { - bt->stack_masks[frame] &= ~(1ull << slot); + __clear_bit(slot, bt->stack_masks[frame]); } static inline u32 bt_frame_reg_mask(struct backtrack_state *bt, u32 frame) @@ -194,14 +213,14 @@ static inline u32 bt_reg_mask(struct backtrack_state *bt) return bt->reg_masks[bt->frame]; } -static inline u64 bt_frame_stack_mask(struct backtrack_state *bt, u32 frame) +static inline unsigned long *bt_frame_stack_mask(struct backtrack_state *bt, u32 frame) { return bt->stack_masks[frame]; } -static inline u64 bt_stack_mask(struct backtrack_state *bt) +static inline unsigned long *bt_stack_mask(struct backtrack_state *bt) { - return bt->stack_masks[bt->frame]; + return bt_frame_stack_mask(bt, bt->frame); } static inline u8 bt_stack_arg_mask(struct backtrack_state *bt) @@ -233,17 +252,16 @@ static void fmt_reg_mask(char *buf, ssize_t buf_sz, u32 reg_mask) break; } } -/* format stack slots bitmask, e.g., "-8,-24,-40" for 0x15 mask */ -void bpf_fmt_stack_mask(char *buf, ssize_t buf_sz, u64 stack_mask) + +/* format stack slots bitmask, e.g., "-8,-24,-40" for slots 0, 2 and 4 */ +void bpf_fmt_stack_mask(char *buf, ssize_t buf_sz, const unsigned long *stack_mask) { - DECLARE_BITMAP(mask, 64); bool first = true; int i, n; buf[0] = '\0'; - bitmap_from_u64(mask, stack_mask); - for_each_set_bit(i, mask, 64) { + for_each_set_bit(i, stack_mask, MAX_BPF_STACK_SLOTS) { n = snprintf(buf, buf_sz, "%s%d", first ? "" : ",", -(i + 1) * 8); first = false; buf += n; @@ -406,15 +424,18 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, if (class == BPF_STX) bt_set_reg(bt, sreg); } else if (class == BPF_JMP || class == BPF_JMP32) { - if (bpf_pseudo_call(insn)) { - int subprog_insn_idx, subprog; + if (bpf_pseudo_call(insn) || bpf_is_callx(insn)) { + int subprog_insn_idx, subprog = -1; - subprog_insn_idx = idx + insn->imm + 1; - subprog = bpf_find_subprog(env, subprog_insn_idx); - if (subprog < 0) - return -EFAULT; + if (bpf_pseudo_call(insn)) { + subprog_insn_idx = idx + insn->imm + 1; + subprog = bpf_find_subprog(env, subprog_insn_idx); + if (subprog < 0) + return -EFAULT; + } - if (bpf_subprog_is_global(env, subprog)) { + /* callx calls static subprogs only */ + if (subprog >= 0 && bpf_subprog_is_global(env, subprog)) { /* check that jump history doesn't have any * extra instructions from subprog; the next * instruction after call to global subprog @@ -423,6 +444,10 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, */ verifier_bug_if(idx + 1 != subseq_idx, env, "extra insn from subprog"); + /* global subprog always sets R0 */ + bt_clear_reg(bt, BPF_REG_0); + /* and if it does not set R2, main pass would catch it */ + bt_clear_reg(bt, BPF_REG_2); /* r1-r5 are invalidated after subprog call, * so for global func call it shouldn't be set * anymore @@ -432,8 +457,6 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, bt_reg_mask(bt)); return -EFAULT; } - /* global subprog always sets R0 */ - bt_clear_reg(bt, BPF_REG_0); return 0; } else { /* static subprog call instruction, which @@ -450,10 +473,11 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, /* we are now tracking register spills correctly, * so any instance of leftover slots is a bug */ - if (bt_stack_mask(bt) != 0) { - verifier_bug(env, - "static subprog leftover stack slots %llx", - bt_stack_mask(bt)); + if (!bt_stack_empty(bt)) { + bpf_fmt_stack_mask(env->tmp_str_buf, TMP_STR_BUF_LEN, + bt_stack_mask(bt)); + verifier_bug(env, "static subprog leftover stack slots %s", + env->tmp_str_buf); return -EFAULT; } /* propagate r1-r5 to the caller */ @@ -486,9 +510,11 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, bt_reg_mask(bt)); return -EFAULT; } - if (bt_stack_mask(bt) != 0) { - verifier_bug(env, "callback leftover stack slots %llx", - bt_stack_mask(bt)); + if (!bt_stack_empty(bt)) { + bpf_fmt_stack_mask(env->tmp_str_buf, TMP_STR_BUF_LEN, + bt_stack_mask(bt)); + verifier_bug(env, "callback leftover stack slots %s", + env->tmp_str_buf); return -EFAULT; } /* clear r1-r5 in callback subprog's mask */ @@ -506,6 +532,8 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, return -ENOTSUPP; /* regular helper call sets R0 */ bt_clear_reg(bt, BPF_REG_0); + /* kfunc might also set R2 */ + bt_clear_reg(bt, BPF_REG_2); if (bt_reg_mask(bt) & BPF_REGMASK_ARGS) { /* if backtracking was looking for registers R1-R5 * they should have been found already. @@ -520,40 +548,51 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, return -EFAULT; } } else if (opcode == BPF_EXIT) { - bool from_subprog_call, r0_precise; + bool from_subprog_call, r0_precise, r2_precise; /* BPF_EXIT in subprog or callback always returns * right after the call instruction, so by checking * whether the instruction at subseq_idx-1 is subprog * call or not we can distinguish actual exit from * *subprog* from exit from *callback*. In the former - * case, we need to propagate r0 precision, if - * necessary. In the former we never do that. + * case, we need to propagate the precision of the + * return registers, if necessary. In the latter we + * never do that. */ from_subprog_call = subseq_idx - 1 >= 0 && - bpf_pseudo_call(&env->prog->insnsi[subseq_idx - 1]); + (bpf_pseudo_call(&env->prog->insnsi[subseq_idx - 1]) || + bpf_is_callx(&env->prog->insnsi[subseq_idx - 1])); r0_precise = from_subprog_call && bt_is_reg_set(bt, BPF_REG_0); + r2_precise = from_subprog_call && bt_is_reg_set(bt, BPF_REG_2); /* Backtracking to a nested function call, 'idx' is a part of * the inner frame 'subseq_idx' is a part of the outer frame. - * In case of a regular function call, instructions giving - * precision to registers R1-R5 should have been found already. + * In case of a regular function call, the callee defines the + * return registers R0 and R2, so clear them before checking + * that instructions giving precision to registers R1-R5 have + * been found already. R2 is also an argument register, hence + * it has to be cleared before that check. * In case of a callback from bpf_loop(), R{1,4} in the calling - * frame would be set as precise and that is correct. + * frame would be set as precise and that is correct, and R2 + * might be a precise helper argument as well, so leave it set. */ + bt_clear_reg(bt, BPF_REG_0); + if (from_subprog_call) + bt_clear_reg(bt, BPF_REG_2); if (from_subprog_call && (bt_reg_mask(bt) & BPF_REGMASK_ARGS)) { verifier_bug(env, "backtracking exit unexpected regs %x", bt_reg_mask(bt)); return -EFAULT; } - bt_clear_reg(bt, BPF_REG_0); if (bt_subprog_enter(bt)) return -EFAULT; if (r0_precise) bt_set_reg(bt, BPF_REG_0); + if (r2_precise) + bt_set_reg(bt, BPF_REG_2); /* r6-r9 and stack slots will stay set in caller frame * bitmasks until we return back from callee(s) */ @@ -693,10 +732,12 @@ void bpf_mark_all_scalars_precise(struct bpf_verifier_env *env, i, j); } } - for (j = 0; j < func->allocated_stack / BPF_REG_SIZE; j++) { - if (!bpf_is_spilled_reg(&func->stack[j])) + for (j = 0; j < bpf_stack_nr_slots(func); j++) { + struct bpf_stack_state *ss = bpf_stack_slot(func, j); + + if (!bpf_is_spilled_reg(ss)) continue; - reg = &func->stack[j].spilled_ptr; + reg = &ss->spilled_ptr; if (reg->type != SCALAR_VALUE || reg->precise) continue; reg->precise = true; @@ -808,6 +849,7 @@ int bpf_mark_chain_precision(struct bpf_verifier_env *env, int subseq_idx = -1; struct bpf_func_state *func; bool tmp, skip_first = true; + struct bpf_stack_state *ss; struct bpf_reg_state *reg; int i, fr, err; @@ -846,6 +888,10 @@ int bpf_mark_chain_precision(struct bpf_verifier_env *env, } if (last_idx < 0) { + /* Consecutive checkpoints can have no instructions between them. */ + if (st->parent) + goto parent; + /* we are at the entry into subprog, which * is expected for global funcs, but only if * requested precise registers are R1-R5 @@ -854,7 +900,7 @@ int bpf_mark_chain_precision(struct bpf_verifier_env *env, if (st->curframe == 0 && st->frame[0]->subprogno > 0 && st->frame[0]->callsite == BPF_MAIN_FUNC && - bt_stack_mask(bt) == 0 && + bt_stack_empty(bt) && (bt_reg_mask(bt) & ~BPF_REGMASK_ARGS) == 0) { bitmap_from_u64(mask, bt_reg_mask(bt)); for_each_set_bit(i, mask, 32) { @@ -868,8 +914,9 @@ int bpf_mark_chain_precision(struct bpf_verifier_env *env, return 0; } - verifier_bug(env, "backtracking func entry subprog %d reg_mask %x stack_mask %llx", - st->frame[0]->subprogno, bt_reg_mask(bt), bt_stack_mask(bt)); + bpf_fmt_stack_mask(env->tmp_str_buf, TMP_STR_BUF_LEN, bt_stack_mask(bt)); + verifier_bug(env, "backtracking func entry subprog %d reg_mask %x stack_mask %s", + st->frame[0]->subprogno, bt_reg_mask(bt), env->tmp_str_buf); return -EFAULT; } @@ -909,6 +956,7 @@ int bpf_mark_chain_precision(struct bpf_verifier_env *env, return -EFAULT; } } +parent: st = st->parent; if (!st) break; @@ -930,18 +978,18 @@ int bpf_mark_chain_precision(struct bpf_verifier_env *env, } } - bitmap_from_u64(mask, bt_frame_stack_mask(bt, fr)); - for_each_set_bit(i, mask, 64) { - if (verifier_bug_if(i >= func->allocated_stack / BPF_REG_SIZE, + for_each_set_bit(i, bt_frame_stack_mask(bt, fr), MAX_BPF_STACK_SLOTS) { + if (verifier_bug_if(i >= bpf_stack_nr_slots(func), env, "stack slot %d, total slots %d", - i, func->allocated_stack / BPF_REG_SIZE)) + i, bpf_stack_nr_slots(func))) return -EFAULT; - if (!bpf_is_spilled_scalar_reg(&func->stack[i])) { + ss = bpf_stack_slot(func, i); + if (!bpf_is_spilled_scalar_reg(ss)) { bt_clear_frame_slot(bt, fr, i); continue; } - reg = &func->stack[i].spilled_ptr; + reg = &ss->spilled_ptr; if (reg->precise) { bt_clear_frame_slot(bt, fr, i); } else { diff --git a/kernel/bpf/bpf_insn_array.c b/kernel/bpf/bpf_insn_array.c index a2f84afe6f7c..92bbb71576ae 100644 --- a/kernel/bpf/bpf_insn_array.c +++ b/kernel/bpf/bpf_insn_array.c @@ -199,12 +199,16 @@ int bpf_insn_array_init(struct bpf_map *map, const struct bpf_prog *prog) return -EBUSY; /* - * Reset all the map indexes to the original values. This is needed, - * e.g., when a replay of verification with different log level should - * be performed. + * Reset the map to its pre-verification state. The xlated and jitted + * offsets and the jitted target pointers are recomputed by the verifier + * and the JIT for this program, so any values left by a previous owner + * must be cleared here. */ - for (i = 0; i < map->max_entries; i++) + for (i = 0; i < map->max_entries; i++) { values[i].xlated_off = values[i].orig_off; + values[i].jitted_off = 0; + insn_array->ips[i] = 0; + } return 0; } @@ -228,10 +232,11 @@ void bpf_insn_array_release(struct bpf_map *map) { struct bpf_insn_array *insn_array = cast_insn_array(map); - atomic_set(&insn_array->used, 0); + /* Paired with atomic_xchg() in bpf_insn_array_init(). */ + atomic_set_release(&insn_array->used, 0); } -void bpf_insn_array_adjust(struct bpf_map *map, u32 off, u32 len) +void bpf_insn_array_adjust(struct bpf_map *map, u32 first, u32 len) { struct bpf_insn_array *insn_array = cast_insn_array(map); int i; @@ -240,7 +245,7 @@ void bpf_insn_array_adjust(struct bpf_map *map, u32 off, u32 len) return; for (i = 0; i < map->max_entries; i++) { - if (insn_array->values[i].xlated_off <= off) + if (insn_array->values[i].xlated_off < first) continue; if (insn_array->values[i].xlated_off == INSN_DELETED) continue; diff --git a/kernel/bpf/bpf_iter.c b/kernel/bpf/bpf_iter.c index d9191df5de22..f0b29a170935 100644 --- a/kernel/bpf/bpf_iter.c +++ b/kernel/bpf/bpf_iter.c @@ -72,6 +72,15 @@ static inline bool bpf_iter_target_support_resched(const struct bpf_iter_target_ return tinfo->reg_info->feature & BPF_ITER_RESCHED; } +/* Only allow sleepable program for resched-able iterator that can run one */ +static bool bpf_iter_target_support_prog(const struct bpf_iter_target_info *tinfo, + const struct bpf_prog *prog) +{ + return !prog->sleepable || + (bpf_iter_target_support_resched(tinfo) && + !(tinfo->reg_info->feature & BPF_ITER_NOSLEEP)); +} + static bool bpf_iter_support_resched(struct seq_file *seq) { struct bpf_iter_priv_data *iter_priv; @@ -342,6 +351,13 @@ int bpf_iter_prog_supported(struct bpf_prog *prog) const char *prefix = BPF_ITER_FUNC_PREFIX; int prefix_len = strlen(prefix); + /* + * The attach function of a program that has a target program is a + * function of that program, whatever its name is. + */ + if (prog->aux->dst_prog) + return -EINVAL; + if (strncmp(attach_fname, prefix, prefix_len)) return -EINVAL; @@ -409,6 +425,8 @@ static int bpf_iter_link_replace(struct bpf_link *link, struct bpf_prog *new_prog, struct bpf_prog *old_prog) { + struct bpf_iter_link *iter_link = + container_of(link, struct bpf_iter_link, link); int ret = 0; mutex_lock(&link_mutex); @@ -419,11 +437,19 @@ static int bpf_iter_link_replace(struct bpf_link *link, if (link->prog->type != new_prog->type || link->prog->expected_attach_type != new_prog->expected_attach_type || - link->prog->aux->attach_btf_id != new_prog->aux->attach_btf_id) { + link->prog->aux->attach_btf_id != new_prog->aux->attach_btf_id || + !bpf_iter_target_support_prog(iter_link->tinfo, new_prog)) { ret = -EINVAL; goto out_unlock; } + if (iter_link->tinfo->reg_info->check_prog) { + ret = iter_link->tinfo->reg_info->check_prog(new_prog, + &iter_link->aux); + if (ret) + goto out_unlock; + } + old_prog = xchg(&link->prog, new_prog); bpf_prog_put(old_prog); @@ -544,8 +570,7 @@ int bpf_iter_link_attach(const union bpf_attr *attr, bpfptr_t uattr, if (!tinfo) return -ENOENT; - /* Only allow sleepable program for resched-able iterator */ - if (prog->sleepable && !bpf_iter_target_support_resched(tinfo)) + if (!bpf_iter_target_support_prog(tinfo, prog)) return -EINVAL; link = kzalloc_obj(*link, GFP_USER | __GFP_NOWARN); diff --git a/kernel/bpf/bpf_local_storage.c b/kernel/bpf/bpf_local_storage.c index 6fc6a4b672b5..b12fcec47ffa 100644 --- a/kernel/bpf/bpf_local_storage.c +++ b/kernel/bpf/bpf_local_storage.c @@ -199,7 +199,7 @@ static void bpf_selem_unlink_storage_nolock_misc(struct bpf_local_storage_elem * } if (pin_owner) - refcount_dec(&local_storage->owner_refcnt); + bpf_local_storage_unpin_owner(local_storage); } /* local_storage->lock must be held and selem->local_storage == local_storage. @@ -240,24 +240,26 @@ void bpf_selem_link_storage_nolock(struct bpf_local_storage *local_storage, hlist_add_head_rcu(&selem->snode, &local_storage->list); } +/* Must be called with the owning local_storage->lock held. */ static int bpf_selem_unlink_map(struct bpf_local_storage_elem *selem) { struct bpf_local_storage *local_storage; struct bpf_local_storage_map *smap; struct bpf_local_storage_map_bucket *b; - unsigned long flags; int err; + lockdep_assert_irqs_disabled(); + local_storage = rcu_dereference_check(selem->local_storage, bpf_rcu_lock_held()); smap = rcu_dereference_check(SDATA(selem)->smap, bpf_rcu_lock_held()); b = select_bucket(smap, local_storage); - err = raw_res_spin_lock_irqsave(&b->lock, flags); + err = raw_res_spin_lock(&b->lock); if (err) return err; hlist_del_init_rcu(&selem->map_node); - raw_res_spin_unlock_irqrestore(&b->lock, flags); + raw_res_spin_unlock(&b->lock); return 0; } @@ -350,7 +352,8 @@ int bpf_selem_unlink(struct bpf_local_storage_elem *selem) static void bpf_selem_unlink_nofail(struct bpf_local_storage_elem *selem, struct bpf_local_storage_map_bucket *b) { - bool in_map_free = !!b, free_storage = false; + bool in_map_free = !!b, free_storage = false, sleepable_users = false; + bool keep_fields = false; struct bpf_local_storage *local_storage; struct bpf_local_storage_map *smap; unsigned long flags; @@ -360,17 +363,26 @@ static void bpf_selem_unlink_nofail(struct bpf_local_storage_elem *selem, smap = rcu_dereference_check(SDATA(selem)->smap, bpf_rcu_lock_held()); if (smap) { + /* A map that is freed, or NULL here, has no program left. */ + sleepable_users = !in_map_free && + (atomic64_read(&smap->map.sleepable_refcnt) || + READ_ONCE(smap->map.was_inner_map)); b = b ? : select_bucket(smap, local_storage); err = raw_res_spin_lock_irqsave(&b->lock, flags); if (!err) { /* * Call bpf_obj_free_fields() under b->lock to make sure it is done * exactly once for an selem. Safe to free special fields immediately - * as no BPF program should be referencing the selem. + * as no BPF program should be referencing the selem, unless it + * is sleepable: see below. */ if (likely(selem_linked_to_map(selem))) { hlist_del_init_rcu(&selem->map_node); - bpf_obj_free_fields(smap->map.record, SDATA(selem)->data); + if (sleepable_users) + keep_fields = true; + else + bpf_obj_free_fields(smap->map.record, + SDATA(selem)->data); unlink++; } raw_res_spin_unlock_irqrestore(&b->lock, flags); @@ -385,7 +397,15 @@ static void bpf_selem_unlink_nofail(struct bpf_local_storage_elem *selem, * with selem1 as raw_res_spin_lock_irqsave() returns -ETIMEDOUT. */ WARN_ON_ONCE(err && in_map_free); - if (!err || in_map_free) + /* + * A sleepable program can still use, and fill, the fields of a value + * that it got before it let go of the owner. As for bpf_selem_unlink(), + * bpf_selem_free_trace_rcu() frees them, and takes smap for that: + * map_free() waits for this RCU read side critical section and then + * for the callback. It must not take smap away from an selem that + * destroy() has unlinked from the map. + */ + if (in_map_free ? err || unlink : !err && !keep_fields) RCU_INIT_POINTER(SDATA(selem)->smap, NULL); } @@ -429,13 +449,18 @@ static void bpf_selem_unlink_nofail(struct bpf_local_storage_elem *selem, * then freed after an RCU grace period. However, if destroy() and map_free() are * racing or rqspinlock returns errors in unlikely situations (unlink != 2), free * the selem only after both map_free() and destroy() see the selem. + * + * The owner that map_free() takes an selem from is alive: sleepable code, which + * is in no RCU read side critical section, may be walking its list for another + * map. And it can still use a value that it got before it let go of an owner + * that destroy() is called for. */ if (unlink == 2 || atomic_cmpxchg(&selem->state, SELEM_UNLINKED, SELEM_TOFREE) == SELEM_UNLINKED) - bpf_selem_free(selem, true); + bpf_selem_free(selem, !in_map_free && !sleepable_users); if (free_storage) - bpf_local_storage_free(local_storage, true); + bpf_local_storage_free(local_storage, !in_map_free); } void __bpf_local_storage_insert_cache(struct bpf_local_storage *local_storage, @@ -507,8 +532,6 @@ int bpf_local_storage_alloc(void *owner, if (err) goto uncharge; - bpf_selem_link_map_nolock(b, first_selem); - owner_storage_ptr = (struct bpf_local_storage **)owner_storage(smap, owner); /* Publish storage to the owner. @@ -523,11 +546,17 @@ int bpf_local_storage_alloc(void *owner, */ prev_storage = cmpxchg(owner_storage_ptr, NULL, storage); if (unlikely(prev_storage)) { - bpf_selem_unlink_map_nolock(first_selem); raw_res_spin_unlock_irqrestore(&b->lock, flags); err = -EAGAIN; goto uncharge; } + /* + * Not before the storage is published: the owner does not wait for + * users of a storage that it knows nothing of, so that nobody may find + * such a storage through the map. b->lock holds off those who find + * first_selem through the owner and want to unlink it. + */ + bpf_selem_link_map_nolock(b, first_selem); raw_res_spin_unlock_irqrestore(&b->lock, flags); return 0; @@ -552,7 +581,7 @@ bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap, struct bpf_local_storage *local_storage; struct bpf_local_storage_map_bucket *b; HLIST_HEAD(old_selem_free_list); - unsigned long flags, b_flags; + unsigned long flags; int err; /* BPF_EXIST and BPF_NOEXIST cannot be both set */ @@ -637,7 +666,8 @@ bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap, b = select_bucket(smap, local_storage); - err = raw_res_spin_lock_irqsave(&b->lock, b_flags); + /* local_storage->lock is held, so IRQs are already disabled. */ + err = raw_res_spin_lock(&b->lock); if (err) goto unlock; @@ -655,7 +685,7 @@ bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap, &old_selem_free_list); } - raw_res_spin_unlock_irqrestore(&b->lock, b_flags); + raw_res_spin_unlock(&b->lock); unlock: raw_res_spin_unlock_irqrestore(&local_storage->lock, flags); free_selem: @@ -750,8 +780,8 @@ u32 bpf_local_storage_destroy(struct bpf_local_storage *local_storage) while (refcount_read(&local_storage->owner_refcnt)) cpu_relax(); /* - * Paired with refcount_dec() in bpf_selem_unlink_nofail() - * to make sure destroy() sees the correct local_storage->mem_charge. + * Paired with bpf_local_storage_unpin_owner() to make sure + * destroy() sees the correct local_storage->mem_charge. */ smp_mb(); } diff --git a/kernel/bpf/bpf_lru_list.c b/kernel/bpf/bpf_lru_list.c index 5ed7cb4b98c0..3858886b0232 100644 --- a/kernel/bpf/bpf_lru_list.c +++ b/kernel/bpf/bpf_lru_list.c @@ -71,7 +71,7 @@ static void __bpf_lru_node_move_in(struct bpf_lru_list *l, return; bpf_lru_list_count_inc(l, tgt_type); - node->type = tgt_type; + WRITE_ONCE(node->type, tgt_type); bpf_lru_node_clear_ref(node); /* Reset pending_free only when moving to the free list */ if (tgt_type == BPF_LRU_LIST_T_FREE) @@ -94,7 +94,7 @@ static void __bpf_lru_node_move(struct bpf_lru_list *l, if (node->type != tgt_type) { bpf_lru_list_count_dec(l, node->type); bpf_lru_list_count_inc(l, tgt_type); - node->type = tgt_type; + WRITE_ONCE(node->type, tgt_type); } bpf_lru_node_clear_ref(node); @@ -298,9 +298,6 @@ static void bpf_lru_list_push_free(struct bpf_lru_list *l, { unsigned long flags; - if (WARN_ON_ONCE(IS_LOCAL_LIST_TYPE(node->type))) - return; - if (raw_res_spin_lock_irqsave(&l->lock, flags)) { WRITE_ONCE(node->pending_free, 1); return; @@ -542,7 +539,7 @@ static void bpf_common_lru_push_free(struct bpf_lru *lru, return; } - if (unlikely(node->type != BPF_LRU_LOCAL_LIST_T_PENDING)) { + if (unlikely(READ_ONCE(node->type) != BPF_LRU_LOCAL_LIST_T_PENDING)) { raw_res_spin_unlock_irqrestore(&loc_l->lock, flags); goto check_lru_list; diff --git a/kernel/bpf/bpf_lsm.c b/kernel/bpf/bpf_lsm.c index 82c5988417a0..f58dce888ff4 100644 --- a/kernel/bpf/bpf_lsm.c +++ b/kernel/bpf/bpf_lsm.c @@ -116,8 +116,11 @@ void bpf_lsm_find_cgroup_shim(const struct bpf_prog *prog, } #endif +BTF_ID_LIST_SINGLE(bpf_lsm_inode_init_security_btf_id, func, + bpf_lsm_inode_init_security) + int bpf_lsm_verify_prog(struct bpf_verifier_log *vlog, - const struct bpf_prog *prog) + struct bpf_prog *prog) { u32 btf_id = prog->aux->attach_btf_id; const char *func_name = prog->aux->attach_func_name; @@ -140,6 +143,23 @@ int bpf_lsm_verify_prog(struct bpf_verifier_log *vlog, return -EINVAL; } + if (btf_id == bpf_lsm_inode_init_security_btf_id[0]) { + /* + * inode, dir, qstr, xattrs, xattr_count + * + * The trusted pointer this hands the program is only as + * trustworthy as the context it is loaded from, which + * lsm_verifier_ops keeps read-only. + */ + static const struct bpf_ctx_arg_aux xattr_count_arg_info = { + .offset = 4 * sizeof(u64), + .reg_type = PTR_TO_MEM | MEM_RDONLY | PTR_TRUSTED, + .mem_size = sizeof(int), + }; + + return bpf_prog_ctx_arg_info_init(prog, &xattr_count_arg_info, 1); + } + return 0; } @@ -414,8 +434,8 @@ const struct bpf_prog_ops lsm_prog_ops = { }; const struct bpf_verifier_ops lsm_verifier_ops = { - .get_func_proto = bpf_lsm_func_proto, - .is_valid_access = btf_ctx_access, + .get_func_proto = bpf_lsm_func_proto, + .is_valid_access = bpf_tracing_btf_ctx_access, }; /* hooks return 0 or 1 */ diff --git a/kernel/bpf/bpf_lsm_proto.c b/kernel/bpf/bpf_lsm_proto.c index 44a54fd8045e..b81179ae2e08 100644 --- a/kernel/bpf/bpf_lsm_proto.c +++ b/kernel/bpf/bpf_lsm_proto.c @@ -4,6 +4,7 @@ */ #include +#include #include /* @@ -12,8 +13,22 @@ * explicitly enforces that BPF LSM programs check for NULL before attempting to * dereference it. */ -int bpf_lsm_mmap_file(struct file *file__nullable, unsigned long reqprot, - unsigned long prot, unsigned long flags) +noinline int bpf_lsm_mmap_file(struct file *file__nullable, unsigned long reqprot, + unsigned long prot, unsigned long flags) { return 0; } + +/* + * Strong definition of the inode_init_security() BPF LSM hook. Both the + * qstr and the xattr array are NULL for some callers, so the __nullable + * suffix marks it as PTR_MAYBE_NULL. BPF LSM programs have to check before + * dereferencing them or handing them to bpf_inode_init_xattr(). + */ +noinline int bpf_lsm_inode_init_security(struct inode *inode, struct inode *dir, + const struct qstr *qstr__nullable, + struct xattr *xattrs__nullable, + int *xattr_count) +{ + return -EOPNOTSUPP; +} diff --git a/kernel/bpf/bpf_struct_ops.c b/kernel/bpf/bpf_struct_ops.c index d7c3030bc63b..1178acd72296 100644 --- a/kernel/bpf/bpf_struct_ops.c +++ b/kernel/bpf/bpf_struct_ops.c @@ -13,6 +13,7 @@ #include #include #include +#include struct bpf_struct_ops_value { struct bpf_struct_ops_common_value common; @@ -57,7 +58,7 @@ struct bpf_struct_ops_map { struct bpf_struct_ops_link { struct bpf_link link; - struct bpf_map __rcu *map; + struct bpf_map *map; wait_queue_head_t wait_hup; }; @@ -1024,9 +1025,18 @@ static void __bpf_struct_ops_map_free(struct bpf_map *map) bpf_map_area_free(st_map); } +static void bpf_struct_ops_map_free_pre_rcu(struct bpf_map *map) +{ + struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map; + + bpf_struct_ops_map_del_ksyms(st_map); +} + static void bpf_struct_ops_map_free(struct bpf_map *map) { struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map; + struct bpf_struct_ops *st_ops = st_map->st_ops_desc->st_ops; + bool tasks_rcu = st_ops->free_after_tasks_rcu_gp; /* st_ops->owner was acquired during map_alloc to implicitly holds * the btf's refcnt. The acquire was only done when btf_is_module() @@ -1037,24 +1047,8 @@ static void bpf_struct_ops_map_free(struct bpf_map *map) bpf_struct_ops_map_dissoc_progs(st_map); - bpf_struct_ops_map_del_ksyms(st_map); - - /* The struct_ops's function may switch to another struct_ops. - * - * For example, bpf_tcp_cc_x->init() may switch to - * another tcp_cc_y by calling - * setsockopt(TCP_CONGESTION, "tcp_cc_y"). - * During the switch, bpf_struct_ops_put(tcp_cc_x) is called - * and its refcount may reach 0 which then free its - * trampoline image while tcp_cc_x is still running. - * - * A vanilla rcu gp is to wait for all bpf-tcp-cc prog - * to finish. bpf-tcp-cc prog is non sleepable. - * A rcu_tasks gp is to wait for the last few insn - * in the tramopline image to finish before releasing - * the trampoline image. - */ - synchronize_rcu_mult(call_rcu, call_rcu_tasks); + if (tasks_rcu && IS_ENABLED(CONFIG_TASKS_RCU)) + synchronize_rcu_tasks(); __bpf_struct_ops_map_free(map); } @@ -1123,6 +1117,11 @@ static struct bpf_map *bpf_struct_ops_map_alloc(union bpf_attr *attr) goto errout; } + if (st_ops_desc->st_ops->cgroup_atype && !(attr->map_flags & BPF_F_LINK)) { + ret = -EOPNOTSUPP; + goto errout; + } + vt = st_ops_desc->value_type; if (attr->value_size != vt->size) { ret = -EINVAL; @@ -1163,6 +1162,8 @@ static struct bpf_map *bpf_struct_ops_map_alloc(union bpf_attr *attr) mutex_init(&st_map->lock); bpf_map_init_from_attr(map, attr); + map->free_after_mult_rcu_gp = st_ops_desc->st_ops->free_after_mult_rcu_gp; + map->free_after_rcu_gp = true; return map; @@ -1195,6 +1196,7 @@ const struct bpf_map_ops bpf_struct_ops_map_ops = { .map_alloc_check = bpf_struct_ops_map_alloc_check, .map_alloc = bpf_struct_ops_map_alloc, .map_free = bpf_struct_ops_map_free, + .map_free_pre_rcu = bpf_struct_ops_map_free_pre_rcu, .map_get_next_key = bpf_struct_ops_map_get_next_key, .map_lookup_elem = bpf_struct_ops_map_lookup_elem, .map_delete_elem = bpf_struct_ops_map_delete_elem, @@ -1281,7 +1283,31 @@ int bpf_struct_ops_for_each_prog(const void *kdata, } EXPORT_SYMBOL_GPL(bpf_struct_ops_for_each_prog); -static bool bpf_struct_ops_valid_to_reg(struct bpf_map *map) +void *bpf_struct_ops_map_kdata(struct bpf_map *map) +{ + struct bpf_struct_ops_map *st_map; + + st_map = container_of(map, struct bpf_struct_ops_map, map); + return st_map->kvalue.data; +} + +int bpf_struct_ops_map_cgroup_atype(struct bpf_map *map) +{ + struct bpf_struct_ops_map *st_map; + + st_map = container_of(map, struct bpf_struct_ops_map, map); + return st_map->st_ops_desc->st_ops->cgroup_atype; +} + +void *bpf_struct_ops_map_cfi_stubs(struct bpf_map *map) +{ + struct bpf_struct_ops_map *st_map; + + st_map = container_of(map, struct bpf_struct_ops_map, map); + return st_map->st_ops_desc->st_ops->cfi_stubs; +} + +bool bpf_struct_ops_valid_to_reg(struct bpf_map *map) { struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map; @@ -1297,8 +1323,7 @@ static void bpf_struct_ops_map_link_dealloc(struct bpf_link *link) struct bpf_struct_ops_map *st_map; st_link = container_of(link, struct bpf_struct_ops_link, link); - st_map = (struct bpf_struct_ops_map *) - rcu_dereference_protected(st_link->map, true); + st_map = (struct bpf_struct_ops_map *)st_link->map; if (st_map) { st_map->st_ops_desc->st_ops->unreg(&st_map->kvalue.data, link); bpf_map_put(&st_map->map); @@ -1313,11 +1338,11 @@ static void bpf_struct_ops_map_link_show_fdinfo(const struct bpf_link *link, struct bpf_map *map; st_link = container_of(link, struct bpf_struct_ops_link, link); - rcu_read_lock(); - map = rcu_dereference(st_link->map); + mutex_lock(&update_mutex); + map = st_link->map; if (map) seq_printf(seq, "map_id:\t%d\n", map->id); - rcu_read_unlock(); + mutex_unlock(&update_mutex); } static int bpf_struct_ops_map_link_fill_link_info(const struct bpf_link *link, @@ -1327,11 +1352,31 @@ static int bpf_struct_ops_map_link_fill_link_info(const struct bpf_link *link, struct bpf_map *map; st_link = container_of(link, struct bpf_struct_ops_link, link); - rcu_read_lock(); - map = rcu_dereference(st_link->map); + mutex_lock(&update_mutex); + map = st_link->map; if (map) info->struct_ops.map_id = map->id; - rcu_read_unlock(); + mutex_unlock(&update_mutex); + return 0; +} + +int bpf_struct_ops_link_update_check(struct bpf_map *new_map, + struct bpf_map *old_map, + struct bpf_map *expected_old_map) +{ + struct bpf_struct_ops_map *st_map, *old_st_map; + + if (!old_map) + return -ENOLINK; + if (expected_old_map && old_map != expected_old_map) + return -EPERM; + + st_map = container_of(new_map, struct bpf_struct_ops_map, map); + old_st_map = container_of(old_map, struct bpf_struct_ops_map, map); + /* The new and old struct_ops must be the same type. */ + if (st_map->st_ops_desc != old_st_map->st_ops_desc) + return -EINVAL; + return 0; } @@ -1353,30 +1398,19 @@ static int bpf_struct_ops_map_link_update(struct bpf_link *link, struct bpf_map return -EOPNOTSUPP; mutex_lock(&update_mutex); - - old_map = rcu_dereference_protected(st_link->map, lockdep_is_held(&update_mutex)); - if (!old_map) { - err = -ENOLINK; + old_map = st_link->map; + err = bpf_struct_ops_link_update_check(new_map, old_map, expected_old_map); + if (err) goto err_out; - } - if (expected_old_map && old_map != expected_old_map) { - err = -EPERM; - goto err_out; - } old_st_map = container_of(old_map, struct bpf_struct_ops_map, map); - /* The new and old struct_ops must be the same type. */ - if (st_map->st_ops_desc != old_st_map->st_ops_desc) { - err = -EINVAL; - goto err_out; - } err = st_map->st_ops_desc->st_ops->update(st_map->kvalue.data, old_st_map->kvalue.data, link); if (err) goto err_out; bpf_map_inc(new_map); - rcu_assign_pointer(st_link->map, new_map); + WRITE_ONCE(st_link->map, new_map); bpf_map_put(old_map); err_out: @@ -1393,7 +1427,7 @@ static int bpf_struct_ops_map_link_detach(struct bpf_link *link) mutex_lock(&update_mutex); - map = rcu_dereference_protected(st_link->map, lockdep_is_held(&update_mutex)); + map = st_link->map; if (!map) { mutex_unlock(&update_mutex); return 0; @@ -1402,7 +1436,7 @@ static int bpf_struct_ops_map_link_detach(struct bpf_link *link) st_map->st_ops_desc->st_ops->unreg(&st_map->kvalue.data, link); - RCU_INIT_POINTER(st_link->map, NULL); + WRITE_ONCE(st_link->map, NULL); /* Pair with bpf_map_get() in bpf_struct_ops_link_create() or * bpf_map_inc() in bpf_struct_ops_map_link_update(). */ @@ -1422,7 +1456,7 @@ static __poll_t bpf_struct_ops_map_link_poll(struct file *file, poll_wait(file, &st_link->wait_hup, pts); - return rcu_access_pointer(st_link->map) ? 0 : EPOLLHUP; + return READ_ONCE(st_link->map) ? 0 : EPOLLHUP; } static const struct bpf_link_ops bpf_struct_ops_map_lops = { @@ -1440,6 +1474,7 @@ int bpf_struct_ops_link_create(union bpf_attr *attr) struct bpf_link_primer link_primer; struct bpf_struct_ops_map *st_map; struct bpf_map *map; + int cgroup_atype; int err; map = bpf_map_get(attr->link_create.map_fd); @@ -1453,6 +1488,19 @@ int bpf_struct_ops_link_create(union bpf_attr *attr) goto err_out; } + cgroup_atype = st_map->st_ops_desc->st_ops->cgroup_atype; + if (cgroup_atype) { + err = cgroup_bpf_struct_ops_attach(map, attr); + bpf_map_put(map); + return err; + } + + if (memchr_inv(&attr->link_create.cgroup, 0, sizeof(attr->link_create.cgroup)) || + attr->link_create.target_fd) { + err = -EINVAL; + goto err_out; + } + link = kzalloc_obj(*link, GFP_USER); if (!link) { err = -ENOMEM; @@ -1478,7 +1526,7 @@ int bpf_struct_ops_link_create(union bpf_attr *attr) link = NULL; goto err_out; } - RCU_INIT_POINTER(link->map, map); + link->map = map; mutex_unlock(&update_mutex); return bpf_link_settle(&link_primer); diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index d870bc5e50bc..538129ed2418 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -20,15 +20,17 @@ #include #include #include +#include #include #include #include #include -#include #include -#include #include #include +#include +#include +#include #include @@ -345,6 +347,9 @@ static const char * const btf_kind_str[NR_BTF_KINDS] = { [BTF_KIND_DECL_TAG] = "DECL_TAG", [BTF_KIND_TYPE_TAG] = "TYPE_TAG", [BTF_KIND_ENUM64] = "ENUM64", + [BTF_KIND_LOC_PARAM] = "LOC_PARAM", + [BTF_KIND_LOC_PROTO] = "LOC_PROTO", + [BTF_KIND_LOCSEC] = "LOCSEC", }; const char *btf_type_str(const struct btf_type *t) @@ -517,11 +522,27 @@ static bool btf_type_is_decl_tag(const struct btf_type *t) return BTF_INFO_KIND(t->info) == BTF_KIND_DECL_TAG; } +static bool btf_type_is_loc_param(const struct btf_type *t) +{ + return BTF_INFO_KIND(t->info) == BTF_KIND_LOC_PARAM; +} + +static bool btf_type_is_loc_proto(const struct btf_type *t) +{ + return BTF_INFO_KIND(t->info) == BTF_KIND_LOC_PROTO; +} + +static bool btf_type_is_locsec(const struct btf_type *t) +{ + return BTF_INFO_KIND(t->info) == BTF_KIND_LOCSEC; +} + static bool btf_type_nosize(const struct btf_type *t) { return btf_type_is_void(t) || btf_type_is_fwd(t) || btf_type_is_func(t) || btf_type_is_func_proto(t) || - btf_type_is_decl_tag(t); + btf_type_is_decl_tag(t) || btf_type_is_loc_param(t) || + btf_type_is_loc_proto(t) || btf_type_is_locsec(t); } static bool btf_type_nosize_or_null(const struct btf_type *t) @@ -768,7 +789,9 @@ static bool btf_type_is_resolve_source_only(const struct btf_type *t) { return btf_type_is_var(t) || btf_type_is_decl_tag(t) || - btf_type_is_datasec(t); + btf_type_is_datasec(t) || + btf_type_is_loc_proto(t) || + btf_type_is_locsec(t); } /* What types need to be resolved? @@ -796,7 +819,9 @@ static bool btf_type_needs_resolve(const struct btf_type *t) btf_type_is_var(t) || btf_type_is_func(t) || btf_type_is_decl_tag(t) || - btf_type_is_datasec(t); + btf_type_is_datasec(t) || + btf_type_is_loc_proto(t) || + btf_type_is_locsec(t); } /* t->size can be used */ @@ -877,16 +902,6 @@ static bool btf_name_offset_valid(const struct btf *btf, u32 offset) return offset < btf->hdr.str_len; } -static bool __btf_name_char_ok(char c, bool first) -{ - if ((first ? !isalpha(c) : - !isalnum(c)) && - c != '_' && - c != '.') - return false; - return true; -} - const char *btf_str_by_offset(const struct btf *btf, u32 offset) { while (offset < btf->start_str_off) @@ -899,20 +914,21 @@ const char *btf_str_by_offset(const struct btf *btf, u32 offset) return NULL; } +/* Names in BTF of Rust are not C identifiers. Allow any printable character */ static bool btf_name_valid_identifier(const struct btf *btf, u32 offset) { /* offset must be valid */ const char *src = btf_str_by_offset(btf, offset); const char *src_limit; - if (!__btf_name_char_ok(*src, true)) + if (!isprint(*src)) return false; /* set a limit on identifier length */ src_limit = src + KSYM_NAME_LEN; src++; while (*src && src < src_limit) { - if (!__btf_name_char_ok(*src, false)) + if (!isprint(*src)) return false; src++; } @@ -1065,6 +1081,8 @@ static const struct btf_type *btf_type_skip_qualifiers(const struct btf *btf, } #define BTF_SHOW_MAX_ITER 10 +/* nested structs and arrays shown; __btf_show_indent() has as many columns */ +#define BTF_SHOW_MAX_DEPTH 32 #define BTF_KIND_BIT(kind) (1ULL << kind) @@ -1458,8 +1476,18 @@ static void *btf_show_start_aggr_type(struct btf_show *show, const struct btf_type *t, u32 type_id, void *data) { - void *safe_data = btf_show_start_type(show, t, type_id, data); + void *safe_data; + /* + * Structs and arrays are shown by recursion. BTF loading bounds the + * depth of one resolve walk, not how deep resolved types may nest. + */ + if (show->state.depth >= BTF_SHOW_MAX_DEPTH) { + show->state.status = -E2BIG; + return NULL; + } + + safe_data = btf_show_start_type(show, t, type_id, data); if (!safe_data) return safe_data; @@ -1920,7 +1948,7 @@ void btf_set_base_btf(struct btf *btf, const struct btf *base_btf) { btf->base_btf = (struct btf *)base_btf; btf->start_id = btf_nr_types(base_btf); - btf->start_str_off = base_btf->hdr.str_len; + btf->start_str_off = base_btf->start_str_off + base_btf->hdr.str_len; } static int env_resolve_init(struct btf_verifier_env *env) @@ -2064,7 +2092,8 @@ static const struct resolve_vertex *env_stack_peak(struct btf_verifier_env *env) * *elem_id: id of u32 * *total_nelems: (x * y). Hence, individual elem size is * (*type_size / *total_nelems) - * *type_id: id of type if it's changed within the function, 0 if not + * *type_id: ID of the returned type if leading modifiers were stripped; + * otherwise left unchanged. * * type: is not an array (e.g. const struct X) * return type: type "struct X" @@ -2072,7 +2101,8 @@ static const struct resolve_vertex *env_stack_peak(struct btf_verifier_env *env) * *elem_type: same as return type ("struct X") * *elem_id: 0 * *total_nelems: 1 - * *type_id: id of type if it's changed within the function, 0 if not + * *type_id: ID of the returned type if leading modifiers were stripped; + * otherwise left unchanged. */ static const struct btf_type * __btf_resolve_size(const struct btf *btf, const struct btf_type *type, @@ -2105,7 +2135,8 @@ __btf_resolve_size(const struct btf *btf, const struct btf_type *type, case BTF_KIND_CONST: case BTF_KIND_RESTRICT: case BTF_KIND_TYPE_TAG: - id = type->type; + if (!array_type) + id = type->type; type = btf_type_by_id(btf, type->type); break; @@ -3538,6 +3569,28 @@ static int btf_type_tag_walk(const struct btf *btf, return 0; } +/* Whether the pointee of pointer type @t carries the type tag @name */ +static bool btf_ptr_has_type_tag(const struct btf *btf, const struct btf_type *t, const char *name) +{ + if (!btf_type_is_ptr(t)) + return false; + + for (t = btf_type_by_id(btf, t->type); btf_type_is_modifier(t); + t = btf_type_by_id(btf, t->type)) { + if (!btf_type_is_type_tag(t) || btf_type_kflag(t)) + continue; + if (!strcmp(__btf_name_by_offset(btf, t->name_off), name)) + return true; + } + + return false; +} + +bool btf_type_is_arena_ptr(const struct btf *btf, const struct btf_type *t) +{ + return btf_ptr_has_type_tag(btf, t, "arena"); +} + static int btf_find_kptr(const struct btf *btf, const struct btf_type *t, u32 off, int sz, struct btf_field_info *info, u32 field_mask) { @@ -3679,6 +3732,7 @@ static int btf_get_field_type(const struct btf *btf, const struct btf_type *var_ { BPF_TIMER, "bpf_timer", true }, { BPF_WORKQUEUE, "bpf_wq", true }, { BPF_TASK_WORK, "bpf_task_work", true }, + { BPF_RCU_HEAD, "bpf_rcu_head", true }, { BPF_LIST_HEAD, "bpf_list_head", false }, { BPF_LIST_NODE, "bpf_list_node", false }, { BPF_RB_ROOT, "bpf_rb_root", false }, @@ -3864,6 +3918,7 @@ static int btf_find_field_one(const struct btf *btf, case BPF_RB_NODE: case BPF_REFCOUNT: case BPF_TASK_WORK: + case BPF_RCU_HEAD: ret = btf_find_struct(btf, var_type, off, sz, field_type, info_cnt ? &info[0] : &tmp); if (ret < 0) @@ -4159,6 +4214,7 @@ struct btf_record *btf_parse_fields(const struct btf *btf, const struct btf_type rec->wq_off = -EINVAL; rec->refcount_off = -EINVAL; rec->task_work_off = -EINVAL; + rec->rcu_head_off = -EINVAL; for (i = 0; i < cnt; i++) { field_type_size = btf_field_type_size(info_arr[i].type); if (info_arr[i].off + field_type_size > value_size) { @@ -4202,6 +4258,10 @@ struct btf_record *btf_parse_fields(const struct btf *btf, const struct btf_type WARN_ON_ONCE(rec->task_work_off >= 0); rec->task_work_off = rec->fields[i].offset; break; + case BPF_RCU_HEAD: + WARN_ON_ONCE(rec->rcu_head_off >= 0); + rec->rcu_head_off = rec->fields[i].offset; + break; case BPF_REFCOUNT: WARN_ON_ONCE(rec->refcount_off >= 0); /* Cache offset for faster lookup at runtime */ @@ -4747,6 +4807,336 @@ static const struct btf_kind_operations enum64_ops = { .show = btf_enum64_show, }; +static s32 btf_loc_param_check_meta(struct btf_verifier_env *env, + const struct btf_type *t, + u32 meta_left) +{ + const struct btf_loc_param *p = btf_loc_param(t); + u32 size, meta_needed, vlen = btf_vlen(t); + + meta_needed = sizeof(*p) + sizeof(__u32) * vlen; + if (meta_left < meta_needed) { + btf_verifier_log_basic(env, t, + "meta_left:%u meta_needed:%u", + meta_left, meta_needed); + return -EINVAL; + } + + if (t->name_off) { + btf_verifier_log_type(env, t, "Invalid name"); + return -EINVAL; + } + size = t->size; + if (!size || size > 16) { + btf_verifier_log_type(env, t, "Unexpected size"); + return -EINVAL; + } + + if (btf_type_kflag(t)) { + btf_verifier_log_type(env, t, "Invalid btf_info kind_flag"); + return -EINVAL; + } + /* All LOC_PARAMs have vlen of at least 1 */ + if (vlen < 1) + goto invalid_vlen; + + switch (p->flags) { + case BTF_LOC_PARAM_CONST: + case BTF_LOC_PARAM_CONST | BTF_LOC_PARAM_SIGNED: + case BTF_LOC_PARAM_CONST | BTF_LOC_PARAM_ADDR: + case BTF_LOC_PARAM_CONST | BTF_LOC_PARAM_ADDR | BTF_LOC_PARAM_SIGNED: + if (vlen < 1 || vlen > 2) + goto invalid_vlen; + if (!size || size > sizeof(__u64)) + goto invalid_vlen; + /* Ensure size/vlen are consistent */ + if (vlen != (size > sizeof(__u32) ? 2 : 1)) + goto invalid_vlen; + break; + case BTF_LOC_PARAM_REG: + /* Reg or reg/reg pair */ + if (vlen > 2) + goto invalid_vlen; + break; + case BTF_LOC_PARAM_REG | BTF_LOC_PARAM_DEREF: + /* A reg deref without offset can only have vlen 1 */ + if (vlen != 1) + goto invalid_vlen; + break; + case BTF_LOC_PARAM_REG | BTF_LOC_PARAM_OFFSET: + case BTF_LOC_PARAM_REG | BTF_LOC_PARAM_OFFSET | BTF_LOC_PARAM_SIGNED: + case BTF_LOC_PARAM_REG | BTF_LOC_PARAM_DEREF | BTF_LOC_PARAM_OFFSET: + case BTF_LOC_PARAM_REG | BTF_LOC_PARAM_DEREF | BTF_LOC_PARAM_OFFSET | + BTF_LOC_PARAM_SIGNED: + if (vlen < 2 || vlen > 3) + goto invalid_vlen; + break; + default: + btf_verifier_log_type(env, t, "Invalid flags"); + return -EINVAL; + } + + btf_verifier_log_type(env, t, NULL); + + return meta_needed; + +invalid_vlen: + btf_verifier_log_type(env, t, "Invalid vlen"); + return -EINVAL; +} + +static void btf_loc_param_log(struct btf_verifier_env *env, + const struct btf_type *t) +{ + const struct btf_loc_param *p = btf_loc_param(t); + u32 i, vlen = btf_vlen(t); + + btf_verifier_log(env, "size=%u vlen=%u flags=0x%x", t->size, vlen, p->flags); + for (i = 0; i < vlen; i++) + btf_verifier_log(env, ", %u", p->values[i]); +} + +static const struct btf_kind_operations loc_param_ops = { + .check_meta = btf_loc_param_check_meta, + .resolve = btf_df_resolve, + .check_member = btf_df_check_member, + .check_kflag_member = btf_df_check_kflag_member, + .log_details = btf_loc_param_log, + .show = btf_df_show, +}; + +static s32 btf_loc_proto_check_meta(struct btf_verifier_env *env, + const struct btf_type *t, + u32 meta_left) +{ + u32 meta_needed; + + meta_needed = sizeof(__u32) * btf_type_vlen(t); + + if (meta_left < meta_needed) { + btf_verifier_log_basic(env, t, + "meta_left:%u meta_needed:%u", + meta_left, meta_needed); + return -EINVAL; + } + + if (t->name_off) { + btf_verifier_log_type(env, t, "Invalid name"); + return -EINVAL; + } + + if (btf_type_kflag(t)) { + btf_verifier_log_type(env, t, "Invalid btf_info kind_flag"); + return -EINVAL; + } + + btf_verifier_log_type(env, t, NULL); + + return meta_needed; +} + +static void btf_loc_proto_log(struct btf_verifier_env *env, + const struct btf_type *t) +{ + const __u32 *params = btf_loc_proto_params(t); + u32 i, nr_params = btf_type_vlen(t); + + btf_verifier_log(env, "vlen=%u", nr_params); + for (i = 0; i < nr_params; i++, params++) + btf_verifier_log(env, ", %u", *params); +} + +static int btf_loc_proto_resolve(struct btf_verifier_env *env, + const struct resolve_vertex *v) +{ + const struct btf_type *t = v->t; + const __u32 *params = btf_loc_proto_params(t); + u32 i, nr_params = btf_type_vlen(t); + struct btf *btf = env->btf; + + if (t->type) { + btf_verifier_log_type(env, t, "Invalid loc_proto type"); + return -EINVAL; + } + + for (i = 0; i < nr_params; i++) { + const struct btf_type *param_type; + u32 param_type_id = params[i]; + + if (!param_type_id) + continue; + + param_type = btf_type_by_id(btf, param_type_id); + if (!param_type || !btf_type_is_loc_param(param_type)) { + btf_verifier_log_type(env, t, + "Invalid loc_param#%u", i + 1); + return -EINVAL; + } + } + + env_stack_pop_resolved(env, 0, 0); + return 0; +} + +static const struct btf_kind_operations loc_proto_ops = { + .check_meta = btf_loc_proto_check_meta, + .resolve = btf_loc_proto_resolve, + .check_member = btf_df_check_member, + .check_kflag_member = btf_df_check_kflag_member, + .log_details = btf_loc_proto_log, + .show = btf_df_show, +}; + +__printf(4, 5) +static void btf_verifier_log_loc(struct btf_verifier_env *env, + const struct btf_type *locsec_type, + const struct btf_loc *loc, + const char *fmt, ...) +{ + struct bpf_verifier_log *log = &env->log; + va_list args; + + if (!bpf_verifier_log_needed(log)) + return; + if (log->level == BPF_LOG_KERNEL && !fmt) + return; + if (env->phase != CHECK_META) + btf_verifier_log_type(env, locsec_type, NULL); + + __btf_verifier_log(log, "\t func=%u loc_proto=%u offset=%u", + loc->func, loc->loc_proto, loc->offset); + if (fmt && *fmt) { + __btf_verifier_log(log, " "); + va_start(args, fmt); + bpf_verifier_vlog(log, fmt, args); + va_end(args); + } + + __btf_verifier_log(log, "\n"); +} + +static s32 btf_locsec_check_meta(struct btf_verifier_env *env, + const struct btf_type *t, + u32 meta_left) +{ + u32 i, meta_needed, vlen = btf_type_vlen(t); + const struct btf_loc *loc; + + meta_needed = sizeof(struct btf_loc) * vlen; + + if (meta_left < meta_needed) { + btf_verifier_log_basic(env, t, + "meta_left:%u meta_needed:%u", + meta_left, meta_needed); + return -EINVAL; + } + + if (btf_type_kflag(t)) { + btf_verifier_log_type(env, t, "Invalid btf_info kind_flag"); + return -EINVAL; + } + + if (!t->name_off || + !btf_name_valid_section(env->btf, t->name_off)) { + btf_verifier_log_type(env, t, "Invalid name"); + return -EINVAL; + } + + for_each_loc(i, t, loc) { + /* A loc func, loc proto cannot be in type void */ + if (!loc->func || !BTF_TYPE_ID_VALID(loc->func)) { + btf_verifier_log_loc(env, t, loc, "Invalid func"); + return -EINVAL; + } + if (!loc->loc_proto || !BTF_TYPE_ID_VALID(loc->loc_proto)) { + btf_verifier_log_loc(env, t, loc, "Invalid loc_proto"); + return -EINVAL; + } + btf_verifier_log_loc(env, t, loc, NULL); + } + + return meta_needed; +} + +static void btf_locsec_log(struct btf_verifier_env *env, + const struct btf_type *t) +{ + btf_verifier_log(env, "vlen=%u", btf_type_vlen(t)); +} + +static int btf_locsec_resolve(struct btf_verifier_env *env, + const struct resolve_vertex *v) +{ + const struct btf_type *t = v->t; + const struct btf_loc *loc; + struct btf *btf = env->btf; + u32 i; + + if (t->type) { + btf_verifier_log_type(env, t, "Invalid locsec type"); + return -EINVAL; + } + + env->resolve_mode = RESOLVE_TBD; + for (i = v->next_member, loc = btf_type_loc_secinfo(t) + i; + i < btf_type_vlen(t); i++, loc++) { + const struct btf_type *func_type, *func_proto_type; + const struct btf_type *loc_proto_type; + u32 func_type_id = loc->func; + u32 loc_proto_type_id = loc->loc_proto; + u32 proto_vlen; + + func_type = btf_type_by_id(btf, func_type_id); + if (!func_type || !btf_type_is_func(func_type)) { + btf_verifier_log_type(env, t, + "Invalid func#%u", i + 1); + return -EINVAL; + } + func_proto_type = btf_type_by_id(btf, func_type->type); + if (!func_proto_type || !btf_type_is_func_proto(func_proto_type)) { + btf_verifier_log_type(env, t, + "Invalid func#%u", i + 1); + return -EINVAL; + } + proto_vlen = btf_vlen(func_proto_type); + + if (!env_type_is_resolved(env, func_type_id)) { + env_stack_set_next_member(env, i); + return env_stack_push(env, func_type, func_type_id); + } + + loc_proto_type = btf_type_by_id(btf, loc_proto_type_id); + if (!loc_proto_type || !btf_type_is_loc_proto(loc_proto_type)) { + btf_verifier_log_type(env, t, + "Invalid loc_proto#%u", i + 1); + return -EINVAL; + } + if (proto_vlen != btf_vlen(loc_proto_type)) { + btf_verifier_log_type(env, t, + "Mismatched vlen for loc_proto#%u", i + 1); + return -EINVAL; + } + + if (!env_type_is_resolved(env, loc_proto_type_id)) { + env_stack_set_next_member(env, i + 1); + return env_stack_push(env, loc_proto_type, + loc_proto_type_id); + } + } + + env_stack_pop_resolved(env, 0, 0); + return 0; +} + +static const struct btf_kind_operations locsec_ops = { + .check_meta = btf_locsec_check_meta, + .resolve = btf_locsec_resolve, + .check_member = btf_df_check_member, + .check_kflag_member = btf_df_check_kflag_member, + .log_details = btf_locsec_log, + .show = btf_df_show, +}; + static s32 btf_func_proto_check_meta(struct btf_verifier_env *env, const struct btf_type *t, u32 meta_left) @@ -5029,7 +5419,7 @@ static int btf_datasec_resolve(struct btf_verifier_env *env, env->resolve_mode = RESOLVE_TBD; for_each_vsi_from(i, v->next_member, v->t, vsi) { - u32 var_type_id = vsi->type, type_id, type_size = 0; + u32 var_type_id = vsi->type, type_id; const struct btf_type *var_type = btf_type_by_id(env->btf, var_type_id); if (!var_type || !btf_type_is_var(var_type)) { @@ -5044,16 +5434,16 @@ static int btf_datasec_resolve(struct btf_verifier_env *env, return env_stack_push(env, var_type, var_type_id); } + /* + * The variable can be smaller than its type. It's a piece of + * a variable that the compiler split then, with the type of + * the whole variable. + */ type_id = var_type->type; - if (!btf_type_id_size(btf, &type_id, &type_size)) { + if (!btf_type_id_size(btf, &type_id, NULL)) { btf_verifier_log_vsi(env, v->t, vsi, "Invalid type"); return -EINVAL; } - - if (vsi->size < type_size) { - btf_verifier_log_vsi(env, v->t, vsi, "Invalid size"); - return -EINVAL; - } } env_stack_pop_resolved(env, 0, 0); @@ -5066,6 +5456,16 @@ static void btf_datasec_log(struct btf_verifier_env *env, btf_verifier_log(env, "size=%u vlen=%u", t->size, btf_type_vlen(t)); } +/* A piece of a variable is smaller than its type, which is the one of the whole variable */ +static bool btf_var_is_piece(const struct btf *btf, const struct btf_type *var, + const struct btf_var_secinfo *vsi) +{ + u32 size; + + return IS_ERR(btf_resolve_size(btf, btf_type_by_id(btf, var->type), &size)) || + vsi->size < size; +} + static void btf_datasec_show(const struct btf *btf, const struct btf_type *t, u32 type_id, void *data, u8 bits_offset, @@ -5073,6 +5473,7 @@ static void btf_datasec_show(const struct btf *btf, { const struct btf_var_secinfo *vsi; const struct btf_type *var; + bool comma = false; u32 i; if (!btf_show_start_type(show, t, type_id, data)) @@ -5082,8 +5483,12 @@ static void btf_datasec_show(const struct btf *btf, __btf_name_by_offset(btf, t->name_off)); for_each_vsi(i, t, vsi) { var = btf_type_by_id(btf, vsi->type); - if (i) + /* there is less data than the type takes */ + if (btf_var_is_piece(btf, var, vsi)) + continue; + if (comma) btf_show(show, ","); + comma = true; btf_type_ops(var)->show(btf, var, vsi->type, data + vsi->offset, bits_offset, show); } @@ -5384,6 +5789,10 @@ static int btf_func_check(struct btf_verifier_env *env, return -EINVAL; } + /* Rust leaves out names of some arguments of static functions */ + if (btf_func_linkage(t) == BTF_FUNC_STATIC) + return 0; + args = (const struct btf_param *)(proto_type + 1); nr_args = btf_type_vlen(proto_type); for (i = 0; i < nr_args; i++) { @@ -5416,6 +5825,9 @@ static const struct btf_kind_operations * const kind_ops[NR_BTF_KINDS] = { [BTF_KIND_DECL_TAG] = &decl_tag_ops, [BTF_KIND_TYPE_TAG] = &modifier_ops, [BTF_KIND_ENUM64] = &enum64_ops, + [BTF_KIND_LOC_PARAM] = &loc_param_ops, + [BTF_KIND_LOC_PROTO] = &loc_proto_ops, + [BTF_KIND_LOCSEC] = &locsec_ops, }; static s32 btf_check_meta(struct btf_verifier_env *env, @@ -5490,7 +5902,8 @@ static bool btf_resolve_valid(struct btf_verifier_env *env, if (!env_type_is_resolved(env, type_id)) return false; - if (btf_type_is_struct(t) || btf_type_is_datasec(t)) + if (btf_type_is_struct(t) || btf_type_is_datasec(t) || + btf_type_is_loc_proto(t) || btf_type_is_locsec(t)) return !btf_resolved_type_id(btf, type_id) && !btf_resolved_type_size(btf, type_id); @@ -5815,7 +6228,7 @@ static int btf_parse_hdr(struct btf_verifier_env *env) return btf_check_sec_info(env, btf_data_size); } -static const char *alloc_obj_fields[] = { +static const char * const alloc_obj_fields[] = { "bpf_spin_lock", "bpf_list_head", "bpf_list_node", @@ -6530,51 +6943,22 @@ __u32 btf_relocate_id(const struct btf *btf, __u32 id) #ifdef CONFIG_DEBUG_INFO_BTF_MODULES -static struct btf *btf_parse_module(const char *module_name, const void *data, - unsigned int data_size, void *base_data, - unsigned int base_data_size) +static struct btf *btf_parse_split(struct btf_verifier_env *env, + const char *name, const void *data, + unsigned int data_size, struct btf *base_btf) { - struct btf *btf = NULL, *vmlinux_btf, *base_btf = NULL; - struct btf_verifier_env *env = NULL; - struct bpf_verifier_log *log; - int err = 0; - - vmlinux_btf = bpf_get_btf_vmlinux(); - if (IS_ERR(vmlinux_btf)) - return vmlinux_btf; - if (!vmlinux_btf) - return ERR_PTR(-EINVAL); - - env = kzalloc_obj(*env, GFP_KERNEL | __GFP_NOWARN); - if (!env) - return ERR_PTR(-ENOMEM); - - log = &env->log; - log->level = BPF_LOG_KERNEL; - - if (base_data) { - base_btf = btf_parse_base(env, ".BTF.base", base_data, base_data_size); - if (IS_ERR(base_btf)) { - err = PTR_ERR(base_btf); - goto errout; - } - } else { - base_btf = vmlinux_btf; - } + struct btf *btf; + int err; btf = kzalloc_obj(*btf, GFP_KERNEL | __GFP_NOWARN); - if (!btf) { - err = -ENOMEM; - goto errout; - } + if (!btf) + return ERR_PTR(-ENOMEM); env->btf = btf; - btf->base_btf = base_btf; - btf->start_id = base_btf->nr_types; - btf->start_str_off = base_btf->hdr.str_len; + btf_set_base_btf(btf, base_btf); btf->kernel_btf = true; btf->named_start_id = 0; - strscpy(btf->name, module_name); + strscpy(btf->name, name); btf->data = kvmemdup(data, data_size, GFP_KERNEL | __GFP_NOWARN); if (!btf->data) { @@ -6601,28 +6985,180 @@ static struct btf *btf_parse_module(const char *module_name, const void *data, if (err) goto errout; + return btf; + +errout: + btf_free(btf); + return ERR_PTR(err); +} + +static int btf_rebase_inline(struct btf *inline_btf, + const struct btf *module_btf, + const u32 *module_id_map, + const u32 *module_str_map, + u32 old_module_type_cnt) +{ + u32 old_start_id = inline_btf->start_id; + u32 old_start_str_off = inline_btf->start_str_off; + u32 old_module_start_str_off = old_start_str_off - module_btf->hdr.str_len; + u32 new_start_id = btf_nr_types(module_btf); + u32 new_start_str_off = module_btf->start_str_off + module_btf->hdr.str_len; + s64 id_delta = (s64)new_start_id - old_start_id; + s64 str_delta = (s64)new_start_str_off - old_start_str_off; + u32 i; + + /* + * The inline BTF was parsed relative to the original module BTF. Its + * base IDs must therefore use the map generated when that BTF was + * relocated, while IDs for inline-local types only move by the change + * in the module BTF's starting ID. + */ + for (i = 0; i < inline_btf->nr_types; i++) { + struct btf_field_iter it; + struct btf_type *t = inline_btf->types[i]; + u32 *id, *str_off; + int err; + + err = btf_field_iter_init(&it, t, BTF_FIELD_ITER_IDS); + if (err) + return err; + while ((id = btf_field_iter_next(&it))) { + if (!*id) + continue; + if (*id < old_module_type_cnt) { + if (module_id_map) + *id = module_id_map[*id]; + } else if (*id >= old_start_id) { + *id += id_delta; + } else { + return -EINVAL; + } + } + + err = btf_field_iter_init(&it, t, BTF_FIELD_ITER_STRS); + if (err) + return err; + while ((str_off = btf_field_iter_next(&it))) { + if (!*str_off) + continue; + if (*str_off < old_module_start_str_off) { + /* Vmlinux strings retain their offsets for in-tree modules. */ + if (!module_id_map) + continue; + if (!module_str_map || !module_str_map[*str_off]) + return -EINVAL; + *str_off = module_str_map[*str_off]; + continue; + } + *str_off += str_delta; + } + } + + btf_set_base_btf(inline_btf, module_btf); + btf_check_sorted(inline_btf); + return 0; +} + +static struct btf *btf_parse_module(const char *module_name, const void *data, + unsigned int data_size, void *base_data, + unsigned int base_data_size, + const void *inline_data, + unsigned int inline_data_size, + void **relocated_inline_data) +{ + struct btf *btf = NULL, *inline_btf = NULL, *vmlinux_btf, *base_btf = NULL; + struct btf_verifier_env *env = NULL; + struct bpf_verifier_log *log; + u32 *module_str_map = NULL; + u32 old_module_type_cnt; + int err = 0; + + *relocated_inline_data = NULL; + vmlinux_btf = bpf_get_btf_vmlinux(); + if (IS_ERR(vmlinux_btf)) + return vmlinux_btf; + if (!vmlinux_btf) + return ERR_PTR(-EINVAL); + + env = kzalloc_obj(*env, GFP_KERNEL | __GFP_NOWARN); + if (!env) + return ERR_PTR(-ENOMEM); + + log = &env->log; + log->level = BPF_LOG_KERNEL; + + if (base_data) { + base_btf = btf_parse_base(env, ".BTF.base", base_data, base_data_size); + if (IS_ERR(base_btf)) { + err = PTR_ERR(base_btf); + goto errout; + } + } else { + base_btf = vmlinux_btf; + } + + btf = btf_parse_split(env, module_name, data, data_size, base_btf); + if (IS_ERR(btf)) { + err = PTR_ERR(btf); + btf = NULL; + goto errout; + } + old_module_type_cnt = btf_nr_types(btf); + + if (inline_data_size) { + inline_btf = btf_parse_split(env, module_name, inline_data, + inline_data_size, btf); + if (IS_ERR(inline_btf)) { + pr_warn("failed to validate module [%s] inline BTF: %ld\n", + module_name, PTR_ERR(inline_btf)); + inline_btf = NULL; + } + } + if (base_btf != vmlinux_btf) { - err = btf_relocate(btf, vmlinux_btf, &btf->base_id_map); + err = btf_relocate(btf, vmlinux_btf, &btf->base_id_map, + &module_str_map); if (err) goto errout; btf_free(base_btf); base_btf = vmlinux_btf; } - btf_verifier_env_free(env); + if (inline_btf) { + err = btf_rebase_inline(inline_btf, btf, btf->base_id_map, + module_str_map, old_module_type_cnt); + if (err) { + pr_warn("failed to relocate module [%s] inline BTF: %d\n", + module_name, err); + btf_free(inline_btf); + } else { + *relocated_inline_data = inline_btf->data; + inline_btf->data = NULL; + btf_free(inline_btf); + } + } + + /* + * With a distilled base, btf_relocate() replaces the base BTF and + * rewrites string offsets. Check ordering only after that final BTF + * view has been established, so named_start_id describes the BTF used + * by name lookups. + */ btf_check_sorted(btf); + btf_verifier_env_free(env); + kvfree(module_str_map); refcount_set(&btf->refcnt, 1); return btf; errout: + kvfree(module_str_map); btf_verifier_env_free(env); + if (inline_btf) + btf_free(inline_btf); if (!IS_ERR(base_btf) && base_btf != vmlinux_btf) btf_free(base_btf); - if (btf) { - kvfree(btf->data); - kvfree(btf->types); - kfree(btf); - } + if (btf) + btf_free(btf); return ERR_PTR(err); } @@ -7016,8 +7552,9 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type, } /* - * Check for PTR_TO_RDONLY_BUF_OR_NULL, PTR_TO_RDWR_BUF_OR_NULL or - * PTR_TO_ARENA (both nullable and non-nullable cases). + * Check for PTR_TO_RDONLY_BUF_OR_NULL, PTR_TO_RDWR_BUF_OR_NULL, + * PTR_TO_ARENA (both nullable and non-nullable cases) or fixed-size + * PTR_TO_MEM. */ for (i = 0; i < prog->aux->ctx_arg_info_size; i++) { const struct bpf_ctx_arg_aux *ctx_arg_info = &prog->aux->ctx_arg_info[i]; @@ -7027,8 +7564,10 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type, flag = type_flag(ctx_arg_info->reg_type); if (ctx_arg_info->offset == off && (type == PTR_TO_ARENA || + type == PTR_TO_MEM || (type == PTR_TO_BUF && (flag & PTR_MAYBE_NULL)))) { info->reg_type = ctx_arg_info->reg_type; + info->mem_size = ctx_arg_info->mem_size; return true; } } @@ -7155,18 +7694,60 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type, } EXPORT_SYMBOL_GPL(btf_ctx_access); +#define MAX_ARRAYS_WALK 16 + +/* Description of an array crossed while walking a BTF access chain */ +struct array_access { + u32 id; /* BTF_KIND_ARRAY type id */ + u32 off; /* offset of the access within the array */ +}; + enum bpf_struct_walk_result { /* < 0 error */ WALK_SCALAR = 0, WALK_PTR, WALK_PTR_UNTRUSTED, + WALK_PTR_ARENA, WALK_STRUCT, }; +static const struct btf_member *find_flex_member(const struct btf *btf, const struct btf_type *t) +{ + const struct btf_array *array; + const struct btf_type *mtype; + const struct btf_member *member; + u32 vlen = btf_type_vlen(t); + + if (vlen == 0) + return NULL; + + member = btf_type_member(t) + vlen - 1; + mtype = btf_type_skip_modifiers(btf, member->type, NULL); + if (!btf_type_is_array(mtype)) + return NULL; + + array = (const struct btf_array *)(mtype + 1); + if (array->nelems != 0) + return NULL; + + return member; +} + +static int push_array(struct array_access *arrays, u32 *arrays_cnt, struct array_access item) +{ + if (arrays == NULL) + return 0; + if (*arrays_cnt == MAX_ARRAYS_WALK) + return -1; + arrays[(*arrays_cnt)++] = item; + return 0; +} + static int btf_struct_walk(struct bpf_verifier_log *log, const struct btf *btf, const struct btf_type *t, int off, int size, u32 *next_btf_id, enum bpf_type_flag *flag, - const char **field_name, bool walk_flex_arrays) + const char **field_name, bool walk_flex_arrays, + struct array_access *arrays, u32 *arrays_cnt) { u32 i, moff, mtrue_end, msize = 0, total_nelems = 0; const struct btf_type *mtype, *elem_type = NULL; @@ -7201,23 +7782,20 @@ static int btf_struct_walk(struct bpf_verifier_log *log, const struct btf *btf, /* If the last element is a variable size array, we may * need to relax the rule. */ - if (vlen == 0) - goto error; - - member = btf_type_member(t) + vlen - 1; - mtype = btf_type_skip_modifiers(btf, member->type, - NULL); - if (!btf_type_is_array(mtype)) - goto error; - - array_elem = (struct btf_array *)(mtype + 1); - if (array_elem->nelems != 0) + member = find_flex_member(btf, t); + if (!member) goto error; moff = __btf_member_bit_offset(t, member) / 8; if (off < moff) goto error; + mtype = btf_type_skip_modifiers(btf, member->type, &mid); + array_elem = (struct btf_array *)(mtype + 1); + + if (push_array(arrays, arrays_cnt, (struct array_access){ mid, off - moff })) + return -E2BIG; + /* allow structure and integer */ t = btf_type_skip_modifiers(btf, array_elem->type, NULL); @@ -7339,6 +7917,9 @@ static int btf_struct_walk(struct bpf_verifier_log *log, const struct btf *btf, * the array's element as long as it is * within the mtrue_end boundary. */ + if (push_array(arrays, arrays_cnt, + (struct array_access){ mid, off - moff })) + return -E2BIG; /* skip empty array */ if (moff == mtrue_end) @@ -7399,6 +7980,38 @@ static int btf_struct_walk(struct bpf_verifier_log *log, const struct btf *btf, id = ctx.id; stype = ctx.t; + /* + * __nullable combines with any of the above: it only says + * that the pointer may be NULL. The caller decides whether + * that matters for the register type it derives. + */ + if (btf_ptr_has_type_tag(btf, mtype, "nullable")) + tmp_flag |= PTR_MAYBE_NULL; + + /* + * An __arena member of a kernel struct holds the kernel + * address of arena memory, which the verifier turns into + * an arena pointer on load. Program BTF is not affected: + * there the tag is what the compiler emits for the arena + * address space. An offset into a union resolves to its + * first member, so a program reading another member there + * would get an arena pointer it did not ask for; the walk + * marked such a union untrusted. + */ + if (btf_is_kernel(btf) && btf_type_is_arena_ptr(btf, mtype)) { + if (*flag & PTR_UNTRUSTED) { + bpf_log(log, + "arena pointer member %s of %s is inside a union\n", + mname, tname); + return -EACCES; + } + *next_btf_id = id; + *flag |= tmp_flag & PTR_MAYBE_NULL; + if (field_name) + *field_name = mname; + return WALK_PTR_ARENA; + } + if (btf_type_is_struct(stype)) { *next_btf_id = id; *flag |= tmp_flag; @@ -7431,15 +8044,29 @@ static int btf_struct_walk(struct bpf_verifier_log *log, const struct btf *btf, int btf_struct_access(struct bpf_verifier_log *log, const struct bpf_reg_state *reg, - int off, int size, enum bpf_access_type atype __maybe_unused, + int _off, int size, enum bpf_access_type atype __maybe_unused, u32 *next_btf_id, enum bpf_type_flag *flag, const char **field_name) { const struct btf *btf = reg->btf; enum bpf_type_flag tmp_flag = 0; + struct array_access arrays[MAX_ARRAYS_WALK]; + const struct btf_member *member; const struct btf_type *t; + u32 i, arrays_cnt = 0; u32 id = reg->btf_id; - int err; + s64 min_off, max_off, flex_off; + int err, off, ret; + + if (check_add_overflow(reg_smin(reg), (s64)_off, &min_off) || + check_add_overflow(reg_smax(reg), (s64)_off, &max_off) || + min_off != (int)min_off) { + bpf_log(log, + "offset computation overflows: register offset range is [%lld, %lld], instruction offset is %d\n", + reg_smin(reg), reg_smax(reg), _off); + return -EINVAL; + } + off = min_off; while (type_is_alloc(reg->type)) { struct btf_struct_meta *meta; @@ -7466,26 +8093,35 @@ int btf_struct_access(struct bpf_verifier_log *log, t = btf_type_by_id(btf, id); do { err = btf_struct_walk(log, btf, t, off, size, &id, &tmp_flag, - field_name, !type_is_alloc(reg->type)); - + field_name, !type_is_alloc(reg->type), arrays, &arrays_cnt); switch (err) { case WALK_PTR: /* For local types, the destination register cannot * become a pointer again. */ - if (type_is_alloc(reg->type)) - return SCALAR_VALUE; + if (type_is_alloc(reg->type)) { + ret = SCALAR_VALUE; + goto check_variable_offset; + } /* If we found the pointer or scalar on t+off, * we're done. */ *next_btf_id = id; *flag = tmp_flag; - return PTR_TO_BTF_ID; + ret = PTR_TO_BTF_ID; + goto check_variable_offset; case WALK_PTR_UNTRUSTED: *flag = MEM_RDONLY | PTR_UNTRUSTED; - return PTR_TO_MEM; + ret = PTR_TO_MEM; + goto check_variable_offset; + case WALK_PTR_ARENA: + *next_btf_id = id; + *flag = tmp_flag; + ret = PTR_TO_ARENA; + goto check_variable_offset; case WALK_SCALAR: - return SCALAR_VALUE; + ret = SCALAR_VALUE; + goto check_variable_offset; case WALK_STRUCT: /* We found nested struct, so continue the search * by diving in it. At this point the offset is @@ -7505,6 +8141,55 @@ int btf_struct_access(struct bpf_verifier_log *log, } while (t); return -EINVAL; + +check_variable_offset: + if (min_off == max_off) + return ret; + + /* Find an offset at which access would go to a flexible array tail (if any). */ + t = btf_type_skip_modifiers(btf, reg->btf_id, NULL); + member = find_flex_member(btf, t); + flex_off = member ? __btf_member_bit_offset(t, member) / 8 : S64_MAX; + + /* + * If this is a varying offset access, the step recorded within a register + * should correspond to one of the arrays visited while walking. + */ + for (i = 0; i < arrays_cnt; i++) { + s64 array_start, array_end, access_end; + u32 asize, esize, elem_id; + + /* + * 'arrays' records top-level arrays only, use __btf_resolve_size() + * to get flattened representation. + */ + t = btf_type_by_id(btf, arrays[i].id); + if (IS_ERR(__btf_resolve_size(btf, t, &asize, NULL, &elem_id, NULL, NULL)) || + IS_ERR(btf_resolve_size(btf, btf_type_by_id(btf, elem_id), &esize))) + continue; + /* Make sure every accessed offset lands on the same field of some element. */ + if (esize == 0 || reg->step % esize != 0) + continue; + + array_start = min_off - arrays[i].off; + /* An array within the flexible tail has no upper bound to exceed */ + if (array_start >= flex_off && btf_type_is_array(t) && + btf_type_array(t)->nelems == 0) + return ret; + + /* the 'size' bytes read at max_off must stay within the array */ + if (check_add_overflow(array_start, (s64)asize, &array_end) || + check_add_overflow(max_off, (s64)size, &access_end) || + access_end > array_end) + continue; + return ret; + } + + bpf_log(log, + "invalid variable offset access into struct %s: offsets [%lld, %lld] size %d step %d do not match any array\n", + btf_name_by_offset(btf, btf_type_by_id(btf, reg->btf_id)->name_off), + min_off, max_off, size, reg->step); + return -EINVAL; } /* Check that two BTF types, each specified as an BTF object + id, are exactly @@ -7545,7 +8230,7 @@ bool btf_struct_ids_match(struct bpf_verifier_log *log, if (!type) return false; err = btf_struct_walk(log, btf, type, off, 1, &id, &flag, NULL, - walk_flex_arrays); + walk_flex_arrays, NULL, NULL); if (err != WALK_STRUCT) return false; @@ -7601,6 +8286,9 @@ static u8 __get_arg_fmodel_flags(const struct btf *btf, { u8 flags = __get_type_fmodel_flags(t); + if (btf_func_arg_align(btf, t) > sizeof(u64)) + flags |= BTF_FMODEL_ALIGN16_ARG; + if (btf_param_match_suffix(btf, arg, "__arena__nullable")) flags |= BTF_FMODEL_ARENA_ARG | BTF_FMODEL_NULLABLE_ARG; else if (btf_param_match_suffix(btf, arg, "__arena")) @@ -7644,7 +8332,7 @@ int btf_distill_func_proto(struct bpf_verifier_log *log, return -EINVAL; } ret = __get_type_size(btf, func->type, &t); - if (ret < 0 || btf_type_is_struct(t)) { + if (ret < 0 || ret > 16) { bpf_log(log, "The function %s return type %s is unsupported.\n", tname, btf_type_str(t)); @@ -7737,6 +8425,12 @@ static int btf_check_func_type_match(struct bpf_verifier_log *log, btf_type_str(t2), fn2); return -EINVAL; } + if (btf_type_has_size(t1) && (t1->size > 8 || t2->size > 8)) { + bpf_log(log, + "Return type of %s() has size %u and of %s() size %u, and a size above 8 bytes cannot be replaced\n", + fn1, t1->size, fn2, t2->size); + return -EINVAL; + } for (i = 0; i < nargs1; i++) { t1 = btf_type_skip_modifiers(btf1, args1[i].type, NULL); @@ -7981,51 +8675,19 @@ static int btf_scan_decl_tags(struct bpf_verifier_env *env, return 0; } -static int btf_scan_type_tags(struct bpf_verifier_env *env, - const struct btf *btf, u32 type_id, - u32 *tags) +static void btf_scan_type_tags(const struct btf *btf, u32 type_id, u32 *tags) { - static const struct btf_type_tag_match func_type_tags[] = { - { "arena", ARG_TAG_ARENA }, - }; - struct btf_type_tag_walk_ctx ctx; - const struct btf_type *t; - int err; - /* Find the first pointer type in the chain. */ - t = btf_type_skip_modifiers(btf, type_id, NULL); + const struct btf_type *t = btf_type_skip_modifiers(btf, type_id, NULL); - /* - * We currently reject type tags on non-pointer types, - * which neither LLVM nor GCC support anyway. - */ - if (!t || !btf_type_is_ptr(t)) - return 0; - - ctx.t = t; - err = btf_type_tag_walk(btf, &ctx, func_type_tags, - ARRAY_SIZE(func_type_tags)); - if (err) { - bpf_log(&env->log, - "function signature member has multiple type tags\n"); - return err; - } - *tags |= ctx.res; - - return 0; + if (btf_type_is_arena_ptr(btf, t)) + *tags |= ARG_TAG_ARENA; } /* Check whether the type is a valid return type. */ static int btf_validate_return_type(struct bpf_verifier_env *env, struct btf *btf, - const struct btf_type *t, int subprog) + const struct btf_type *t, int subprog, bool is_global) { - u32 tags = 0; - int err; - - err = btf_scan_type_tags(env, btf, t->type, &tags); - if (err) - return err; - t = btf_type_skip_modifiers(btf, t->type, NULL); /* @@ -8033,16 +8695,62 @@ static int btf_validate_return_type(struct bpf_verifier_env *env, struct btf *bt * General arena variables are not allowed, since it makes no sense to return by value * a variable that's on the heap in the first place. */ - if (subprog && (tags & ARG_TAG_ARENA) && btf_type_is_ptr(t)) + if (subprog && btf_type_is_arena_ptr(btf, t)) return 0; /* We always accept void or scalars. */ if (btf_type_is_void(t) || btf_type_is_int(t) || btf_is_any_enum(t)) return 0; + if (btf_type_is_struct(t) && t->size <= 16) { + /* + * A global function's caller models the return as an opaque + * scalar pair, so a pointer member would be laundered into a + * scalar and escape provenance and reference tracking. Only + * scalars and arena pointers are allowed: an arena pointer has + * no provenance to lose, since a program may already derive one + * from any scalar with addr_space_cast(), which confines the + * result to the arena. The main program is the exception: it + * returns to the kernel, which has no arena to cast the address + * back into. A local function is verified inline, so its caller + * receives the real register state and any member is fine. + */ + bool local_func = subprog && !is_global; + u32 member_kinds = BTF_MEMBER_SCALAR; + + if (subprog) + member_kinds |= BTF_MEMBER_ARENA_PTR; + + if (local_func || btf_struct_is_composed_of(env, btf, t, member_kinds)) + return 0; + } + return -EOPNOTSUPP; } +static int btf_check_arg_slots(struct bpf_verifier_log *log, const char *tname, + bool is_global, u32 slot_cnt, + struct bpf_subprog_info *sub) +{ + if (slot_cnt <= MAX_BPF_FUNC_REG_ARGS) + return 0; + + if (is_global) { + bpf_log(log, + "global function %s() needs %d > %d argument slots, " + "stack args not supported\n", + tname, slot_cnt, MAX_BPF_FUNC_REG_ARGS); + return -EINVAL; + } + if (!bpf_jit_supports_stack_args()) { + bpf_log(log, "JIT does not support function %s() with %d argument slots\n", + tname, slot_cnt); + return -EFAULT; + } + sub->stack_arg_cnt = slot_cnt - MAX_BPF_FUNC_REG_ARGS; + return 0; +} + /* Process BTF of a function to produce high-level expectation of function * arguments (like ARG_PTR_TO_CTX, or ARG_PTR_TO_MEM, etc). This information * is cached in subprog info for reuse. @@ -8062,7 +8770,7 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) const struct btf_param *args; const struct btf_type *t, *ref_t, *fn_t; int err; - u32 i, nargs, btf_id; + u32 i, slots_used, nargs, btf_id; const char *tname; if (sub->args_cached) @@ -8106,34 +8814,29 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) } args = (const struct btf_param *)(t + 1); nargs = btf_type_vlen(t); - sub->arg_cnt = nargs; + sub->arg_slot_cnt = nargs; if (nargs > MAX_BPF_FUNC_ARGS) { bpf_log(log, "kernel supports at most %d parameters, function %s has %d\n", MAX_BPF_FUNC_ARGS, tname, nargs); return -EFAULT; } - if (nargs > MAX_BPF_FUNC_REG_ARGS) { - if (!bpf_jit_supports_stack_args()) { - bpf_log(log, "JIT does not support function %s() with %d args\n", - tname, nargs); - return -EFAULT; - } - sub->stack_arg_cnt = nargs - MAX_BPF_FUNC_REG_ARGS; - } + err = btf_check_arg_slots(log, tname, is_global, nargs, sub); + if (err) + return err; - if (is_global && nargs > MAX_BPF_FUNC_REG_ARGS) { - bpf_log(log, "global function %s has %d > %d args, stack args not supported\n", - tname, nargs, MAX_BPF_FUNC_REG_ARGS); - return -EINVAL; - } - - err = btf_validate_return_type(env, btf, t, subprog); + err = btf_validate_return_type(env, btf, t, subprog, is_global); if (err) { if (is_global) { + /* Only a subprogram may return arena pointers. */ + const char *supported = subprog ? + "void, scalar, arena pointer, or a struct/union of " + "scalars and arena pointers" : + "void, scalar, or a scalar-only struct/union"; + bpf_log(log, - "Global function %s() return value not void or scalar. " - "Only those are supported.\n", - tname); + "Global function %s() has unsupported return type. " + "Only %s up to 16 bytes is supported.\n", + tname, supported); } return err; } @@ -8141,15 +8844,17 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) /* Convert BTF function arguments into verifier types. * Only PTR_TO_CTX and SCALAR are supported atm. */ - for (i = 0; i < nargs; i++) { + for (i = 0, slots_used = 0; i < nargs; i++) { u32 tags = 0; + + if (slots_used >= MAX_BPF_FUNC_ARGS) + goto too_many_slots; + err = btf_scan_decl_tags(env, btf, fn_t, i, is_global, &tags); if (err) return err; - err = btf_scan_type_tags(env, btf, args[i].type, &tags); - if (err) - return err; + btf_scan_type_tags(btf, args[i].type, &tags); t = btf_type_by_id(btf, args[i].type); while (btf_type_is_modifier(t)) @@ -8166,7 +8871,7 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) btf_validate_prog_ctx_type(log, btf, t, i, prog_type, prog->expected_attach_type)) return -EINVAL; - sub->args[i].arg_type = ARG_PTR_TO_CTX; + sub->args[slots_used++].arg_type = ARG_PTR_TO_CTX; continue; } if (btf_is_dynptr_ptr(btf, t)) { @@ -8174,7 +8879,7 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) bpf_log(log, "arg#%d has invalid combination of tags\n", i); return -EINVAL; } - sub->args[i].arg_type = ARG_PTR_TO_DYNPTR; + sub->args[slots_used++].arg_type = ARG_PTR_TO_DYNPTR; continue; } if (tags & ARG_TAG_TRUSTED) { @@ -8189,10 +8894,11 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) if (kern_type_id < 0) return kern_type_id; - sub->args[i].arg_type = ARG_PTR_TO_BTF_ID | PTR_TRUSTED; + sub->args[slots_used].arg_type = ARG_PTR_TO_BTF_ID | PTR_TRUSTED; if (tags & ARG_TAG_NULLABLE) - sub->args[i].arg_type |= PTR_MAYBE_NULL; - sub->args[i].btf_id = kern_type_id; + sub->args[slots_used].arg_type |= PTR_MAYBE_NULL; + sub->args[slots_used].btf_id = kern_type_id; + slots_used++; continue; } if (tags & ARG_TAG_UNTRUSTED) { @@ -8206,8 +8912,10 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) ref_t = btf_type_skip_modifiers(btf, t->type, NULL); if (btf_type_is_void(ref_t) || btf_type_is_primitive(ref_t)) { - sub->args[i].arg_type = ARG_PTR_TO_MEM | MEM_RDONLY | PTR_UNTRUSTED; - sub->args[i].mem_size = 0; + sub->args[slots_used].arg_type = ARG_PTR_TO_MEM | MEM_RDONLY | + PTR_UNTRUSTED; + sub->args[slots_used].mem_size = 0; + slots_used++; continue; } @@ -8223,8 +8931,9 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) i, btf_type_str(ref_t), tname); return -EINVAL; } - sub->args[i].arg_type = ARG_PTR_TO_BTF_ID | PTR_UNTRUSTED; - sub->args[i].btf_id = kern_type_id; + sub->args[slots_used].arg_type = ARG_PTR_TO_BTF_ID | PTR_UNTRUSTED; + sub->args[slots_used].btf_id = kern_type_id; + slots_used++; continue; } if (tags & ARG_TAG_ARENA) { @@ -8232,7 +8941,7 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) bpf_log(log, "arg#%d arena cannot be combined with any other tags\n", i); return -EINVAL; } - sub->args[i].arg_type = ARG_PTR_TO_ARENA; + sub->args[slots_used++].arg_type = ARG_PTR_TO_ARENA; continue; } if (is_global) { /* generic user data pointer */ @@ -8252,10 +8961,11 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) return -EINVAL; } - sub->args[i].arg_type = ARG_PTR_TO_MEM | PTR_MAYBE_NULL; + sub->args[slots_used].arg_type = ARG_PTR_TO_MEM | PTR_MAYBE_NULL; if (tags & ARG_TAG_NONNULL) - sub->args[i].arg_type &= ~PTR_MAYBE_NULL; - sub->args[i].mem_size = mem_size; + sub->args[slots_used].arg_type &= ~PTR_MAYBE_NULL; + sub->args[slots_used].mem_size = mem_size; + slots_used++; continue; } @@ -8264,8 +8974,32 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) bpf_log(log, "arg#%d has pointer tag, but is not a pointer type\n", i); return -EINVAL; } - if (btf_type_is_int(t) || btf_is_any_enum(t)) { - sub->args[i].arg_type = ARG_ANYTHING; + if (btf_type_is_int(t) || btf_is_any_enum(t) || btf_type_is_struct(t)) { + u32 nslots; + + if (!t->size || t->size > 2 * BPF_REG_SIZE) { + if (!is_global) + return -EINVAL; + bpf_log(log, + "Arg#%d type %s in %s() has size %u, only 1 to %d bytes " + "can be passed by value\n", + i, btf_type_str(t), tname, t->size, 2 * BPF_REG_SIZE); + return -EINVAL; + } + if (btf_type_is_struct(t) && + !btf_struct_is_composed_of(env, btf, t, BTF_MEMBER_SCALAR)) { + if (!is_global) + return -EINVAL; + bpf_log(log, "Arg#%d type %s in %s() is not composed of scalars\n", + i, btf_type_str(t), tname); + return -EINVAL; + } + + nslots = (t->size + BPF_REG_SIZE - 1) / BPF_REG_SIZE; + if (slots_used + nslots > MAX_BPF_FUNC_ARGS) + goto too_many_slots; + while (nslots--) + sub->args[slots_used++].arg_type = ARG_SCALAR; continue; } if (!is_global) @@ -8275,9 +9009,21 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) return -EINVAL; } + err = btf_check_arg_slots(log, tname, is_global, slots_used, sub); + if (err) + return err; + sub->arg_slot_cnt = slots_used; + sub->args_cached = true; return 0; + +too_many_slots: + if (!is_global) + return -EINVAL; + bpf_log(log, "Arguments of %s() need more than %d argument slots\n", + tname, MAX_BPF_FUNC_ARGS); + return -EINVAL; } static void btf_type_show(const struct btf *btf, u32 type_id, void *obj, @@ -8552,6 +9298,8 @@ struct btf_module { struct module *module; struct btf *btf; struct bin_attribute *sysfs_attr; + struct bin_attribute *sysfs_inline_attr; + void *btf_inline_data; int flags; }; @@ -8560,28 +9308,149 @@ static DEFINE_MUTEX(btf_module_mutex); static void purge_cand_cache(struct btf *btf); +#if IS_MODULE(CONFIG_DEBUG_INFO_BTF_INLINE) +extern char __start_BTF_inline_link[]; +extern char __stop_BTF_inline_link[]; + +static struct bin_attribute *vmlinux_inline_attr; +static const struct btf_link *vmlinux_inline_link; + +static const struct btf_link *btf_parse_link(void *start, unsigned int size) +{ + const struct btf_link *link = start; + unsigned int module_name_len; + + if (!link || size != sizeof(*link) || link->btf_size == 0) + return NULL; + + module_name_len = strnlen(link->module_name, sizeof(link->module_name)); + if (!module_name_len || module_name_len == sizeof(link->module_name) || + memchr_inv(link->module_name + module_name_len + 1, 0, + sizeof(link->module_name) - module_name_len - 1)) + return NULL; + return link; +} + +static bool btf_verify_link(const struct btf_link *link, const struct module *mod, + const void *data, size_t data_size) +{ + u8 actual_digest[BTF_LINK_SHA256_LEN]; + + if (!link || data_size != link->btf_size) { + pr_warn(" BTF link from module [%s] missing or has unexpected size\n", mod->name); + return false; + } + sha256(data, data_size, actual_digest); + if (memcmp(actual_digest, link->sha256, sizeof(actual_digest))) { + pr_warn("BTF link from module [%s] has unexpected digest\n", mod->name); + return false; + } + + return true; +} + +static void btf_vmlinux_inline_lazy_init(void) +{ + vmlinux_inline_link = btf_parse_link(__start_BTF_inline_link, + __stop_BTF_inline_link - + __start_BTF_inline_link); + if (!vmlinux_inline_link) { + pr_warn("invalid vmlinux inline BTF link\n"); + return; + } + vmlinux_inline_attr = sysfs_btf_add("vmlinux.inline", NULL, + vmlinux_inline_link->btf_size, true, + vmlinux_inline_link->module_name); + if (IS_ERR(vmlinux_inline_attr)) { + pr_warn("failed to register vmlinux inline BTF in sysfs: %ld\n", + PTR_ERR(vmlinux_inline_attr)); + vmlinux_inline_attr = NULL; + } +} + +static bool btf_module_is_vmlinux_inline(const struct module *mod) +{ + return vmlinux_inline_link && !strcmp(mod->name, vmlinux_inline_link->module_name); +} + +static int btf_module_handle_vmlinux_inline(const struct module *mod) +{ + void *data; + + if (!mod->btf_inline_data || + mod->btf_inline_data_size != vmlinux_inline_link->btf_size) + return -EINVAL; + + data = vmalloc_user(mod->btf_inline_data_size); + if (!data) + return -ENOMEM; + memcpy(data, mod->btf_inline_data, mod->btf_inline_data_size); + + if (!btf_verify_link(vmlinux_inline_link, mod, data, mod->btf_inline_data_size) || + !sysfs_btf_update(vmlinux_inline_attr, data)) { + kvfree(data); + return -EINVAL; + } + return 0; +} + +#else + +static bool btf_module_is_vmlinux_inline(const struct module *mod) +{ + return false; +} + +static int btf_module_handle_vmlinux_inline(const struct module *mod) +{ + return 0; +} + +static void btf_vmlinux_inline_lazy_init(void) +{ +} + +#endif /* IS_MODULE(CONFIG_DEBUG_INFO_BTF_INLINE) */ + static int btf_module_notify(struct notifier_block *nb, unsigned long op, void *module) { struct btf_module *btf_mod, *tmp; struct module *mod = module; + struct bin_attribute *attr; struct btf *btf; + void *inline_data = NULL, *relocated_inline_data = NULL; + unsigned int inline_data_size = 0; int err = 0; - if (mod->btf_data_size == 0 || - (op != MODULE_STATE_COMING && op != MODULE_STATE_LIVE && - op != MODULE_STATE_GOING)) + if (op != MODULE_STATE_COMING && op != MODULE_STATE_LIVE && + op != MODULE_STATE_GOING) goto out; switch (op) { case MODULE_STATE_COMING: + if (btf_module_is_vmlinux_inline(mod)) { + err = btf_module_handle_vmlinux_inline(mod); + break; + } + +#if IS_ENABLED(CONFIG_DEBUG_INFO_BTF_INLINE) + if (mod->btf_inline_data_size) { + inline_data = mod->btf_inline_data; + inline_data_size = mod->btf_inline_data_size; + } +#endif + if (!mod->btf_data_size) + break; btf_mod = kzalloc_obj(*btf_mod); if (!btf_mod) { err = -ENOMEM; goto out; } btf = btf_parse_module(mod->name, mod->btf_data, mod->btf_data_size, - mod->btf_base_data, mod->btf_base_data_size); + mod->btf_base_data, mod->btf_base_data_size, + inline_data, inline_data_size, + &relocated_inline_data); if (IS_ERR(btf)) { kfree(btf_mod); if (!IS_ENABLED(CONFIG_MODULE_ALLOW_BTF_MISMATCH)) { @@ -8596,6 +9465,7 @@ static int btf_module_notify(struct notifier_block *nb, unsigned long op, err = btf_alloc_id(btf); if (err) { btf_free(btf); + kvfree(relocated_inline_data); kfree(btf_mod); goto out; } @@ -8604,37 +9474,35 @@ static int btf_module_notify(struct notifier_block *nb, unsigned long op, mutex_lock(&btf_module_mutex); btf_mod->module = module; btf_mod->btf = btf; + btf_mod->btf_inline_data = relocated_inline_data; list_add(&btf_mod->list, &btf_modules); mutex_unlock(&btf_module_mutex); - if (IS_ENABLED(CONFIG_SYSFS)) { - struct bin_attribute *attr; - - attr = kzalloc_obj(*attr); - if (!attr) - goto out; - - sysfs_bin_attr_init(attr); - attr->attr.name = btf->name; - attr->attr.mode = 0444; - attr->size = btf->data_size; - attr->private = btf->data; - attr->read = sysfs_bin_attr_simple_read; - - err = sysfs_create_bin_file(btf_kobj, attr); - if (err) { - pr_warn("failed to register module [%s] BTF in sysfs: %d\n", - mod->name, err); - kfree(attr); - err = 0; - goto out; - } - - btf_mod->sysfs_attr = attr; + attr = sysfs_btf_add(btf->name, btf->data, btf->data_size, false, NULL); + if (IS_ERR(attr)) { + err = 0; + goto out; } + btf_mod->sysfs_attr = attr; + if (relocated_inline_data) { + char name[MODULE_NAME_LEN + sizeof(".inline")]; + + snprintf(name, sizeof(name), "%s.inline", mod->name); + attr = sysfs_btf_add(name, relocated_inline_data, + inline_data_size, false, NULL); + if (IS_ERR(attr)) { + kvfree(relocated_inline_data); + btf_mod->btf_inline_data = NULL; + err = 0; + break; + } + btf_mod->sysfs_inline_attr = attr; + } break; case MODULE_STATE_LIVE: + if (!mod->btf_data_size) + break; mutex_lock(&btf_module_mutex); list_for_each_entry_safe(btf_mod, tmp, &btf_modules, list) { if (btf_mod->module != module) @@ -8646,6 +9514,8 @@ static int btf_module_notify(struct notifier_block *nb, unsigned long op, mutex_unlock(&btf_module_mutex); break; case MODULE_STATE_GOING: + if (!mod->btf_data_size) + break; mutex_lock(&btf_module_mutex); list_for_each_entry_safe(btf_mod, tmp, &btf_modules, list) { if (btf_mod->module != module) @@ -8660,10 +9530,12 @@ static int btf_module_notify(struct notifier_block *nb, unsigned long op, btf_free_id(btf_mod->btf); list_del(&btf_mod->list); if (btf_mod->sysfs_attr) - sysfs_remove_bin_file(btf_kobj, btf_mod->sysfs_attr); + sysfs_btf_remove(btf_mod->sysfs_attr); + if (btf_mod->sysfs_inline_attr) + sysfs_btf_remove(btf_mod->sysfs_inline_attr); + kvfree(btf_mod->btf_inline_data); purge_cand_cache(btf_mod->btf); btf_put(btf_mod->btf); - kfree(btf_mod->sysfs_attr); kfree(btf_mod); break; } @@ -8680,6 +9552,7 @@ static struct notifier_block btf_module_nb = { static int __init btf_module_init(void) { + btf_vmlinux_inline_lazy_init(); register_module_notifier(&btf_module_nb); return 0; } @@ -9953,6 +10826,7 @@ btf_add_struct_ops(struct btf *btf, struct bpf_struct_ops *st_ops, struct bpf_verifier_log *log) { struct btf_struct_ops_tab *tab, *new_tab; + int cgroup_atype; int i, err; tab = btf->struct_ops_tab; @@ -9964,8 +10838,10 @@ btf_add_struct_ops(struct btf *btf, struct bpf_struct_ops *st_ops, btf->struct_ops_tab = tab; } + cgroup_atype = st_ops->cgroup_atype; for (i = 0; i < tab->cnt; i++) - if (tab->ops[i].st_ops == st_ops) + if (tab->ops[i].st_ops == st_ops || + (cgroup_atype && cgroup_atype == tab->ops[i].st_ops->cgroup_atype)) return -EEXIST; if (tab->cnt == tab->capacity) { @@ -9985,6 +10861,31 @@ btf_add_struct_ops(struct btf *btf, struct bpf_struct_ops *st_ops, if (err) return err; + if (cgroup_atype) { + /* + * Cgroup struct_ops callers hold the RCU read lock around the + * entire trampoline call, including its trailing instructions. + * A regular RCU grace period therefore protects both the kdata + * and the trampoline image, so a tasks RCU grace period is not + * needed. + */ + if (!cgroup_bpf_is_struct_ops_atype(cgroup_atype) || + st_ops->reg || st_ops->unreg || st_ops->free_after_tasks_rcu_gp) { + bpf_struct_ops_desc_release(&tab->ops[btf->struct_ops_tab->cnt]); + return -EINVAL; + } + + /* + * There is no need to unregister from the cgroup when btf_free() + * runs. No struct_ops map or cgroup link can be created once its + * BTF is gone. + */ + cgroup_bpf_struct_ops_register(cgroup_atype, + tab->ops[btf->struct_ops_tab->cnt].type_id, + st_ops->cfi_stubs, + st_ops->free_after_mult_rcu_gp); + } + btf->struct_ops_tab->cnt++; return 0; diff --git a/kernel/bpf/btf_vmlinux_inline.c b/kernel/bpf/btf_vmlinux_inline.c new file mode 100644 index 000000000000..d5c599d9f729 --- /dev/null +++ b/kernel/bpf/btf_vmlinux_inline.c @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026, Oracle and/or its affiliates. */ +/* + * Provide kernel BTF inline function information for use by BPF tools. + */ +#include +#include +#include +#include + +#if IS_BUILTIN(CONFIG_DEBUG_INFO_BTF_INLINE) +extern char __start_BTF_inline[]; +extern char __stop_BTF_inline[]; +#endif + +static int __init btf_vmlinux_inline_init(void) +{ +#if IS_BUILTIN(CONFIG_DEBUG_INFO_BTF_INLINE) + size_t data_size = __stop_BTF_inline - __start_BTF_inline; + + if (data_size) + sysfs_btf_add("vmlinux.inline", __start_BTF_inline, data_size, true, NULL); +#endif + return 0; +} +subsys_initcall(btf_vmlinux_inline_init); + +MODULE_DESCRIPTION("BTF inline information for vmlinux"); +MODULE_LICENSE("GPL"); diff --git a/kernel/bpf/cfg.c b/kernel/bpf/cfg.c index 842c7d1eabcc..2ee950e9ac82 100644 --- a/kernel/bpf/cfg.c +++ b/kernel/bpf/cfg.c @@ -9,6 +9,8 @@ #define verbose(env, fmt, args...) bpf_verifier_log_write(env, fmt, ##args) +#define BPF_MAX_GOTOX_EDGES BPF_COMPLEXITY_LIMIT_INSNS + /* non-recursive DFS pseudo code * 1 procedure DFS-iterative(G,v): * 2 label v as discovered @@ -284,15 +286,17 @@ static struct bpf_iarray *jt_from_map(struct bpf_map *map) } /* - * Find and collect all maps which fit in the subprog. Return the result as one - * combined jump table in jt->items (allocated with kvcalloc) + * Collect the jump table of every subprogram that has one, as the combined + * table of all maps whose targets land inside that subprogram. All gotox + * instructions of a subprogram share the same table, so this is done in a + * single pass over the maps rather than once per gotox. */ -static struct bpf_iarray *jt_from_subprog(struct bpf_verifier_env *env, - int subprog_start, int subprog_end) +static int __compute_subprog_jts(struct bpf_verifier_env *env) { - struct bpf_iarray *jt = NULL; + struct bpf_subprog_info *subprog; + struct bpf_iarray *jt, *jt_cur; struct bpf_map *map; - struct bpf_iarray *jt_cur; + u32 old_cnt; int i; for (i = 0; i < env->insn_array_map_cnt; i++) { @@ -303,73 +307,100 @@ static struct bpf_iarray *jt_from_subprog(struct bpf_verifier_env *env, map = env->insn_array_maps[i]; jt_cur = jt_from_map(map); - if (IS_ERR(jt_cur)) { - kvfree(jt); - return jt_cur; + if (IS_ERR(jt_cur)) + return PTR_ERR(jt_cur); + + subprog = bpf_find_containing_subprog(env, jt_cur->items[0]); + if (verifier_bug_if(!subprog, env, "no subprog contains insn %u of an insn array map", + jt_cur->items[0])) { + kvfree(jt_cur); + return -EFAULT; + } + if (jt_cur->items[jt_cur->cnt - 1] >= (subprog + 1)->start) { + verbose(env, "jump table of subprog starting at %u spans multiple subprogs, map id=%d\n", + subprog->start, map->id); + bpf_diag_program_structure( + env, subprog->start, "jump table spans subprograms", + "Keep every entry of a jump table inside one subprogram.", + "The jump table in map id %d for the subprogram that starts at instruction %u reaches past its end at instruction %u.", + map->id, subprog->start, (subprog + 1)->start); + kvfree(jt_cur); + return -EINVAL; } - /* - * This is enough to check one element. The full table is - * checked to fit inside the subprog later in create_jt() - */ - if (jt_cur->items[0] >= subprog_start && jt_cur->items[0] < subprog_end) { - u32 old_cnt = jt ? jt->cnt : 0; - jt = bpf_iarray_realloc(jt, old_cnt + jt_cur->cnt); - if (!jt) { - kvfree(jt_cur); - return ERR_PTR(-ENOMEM); - } - memcpy(jt->items + old_cnt, jt_cur->items, jt_cur->cnt << 2); + old_cnt = subprog->jt ? subprog->jt->cnt : 0; + jt = bpf_iarray_realloc(subprog->jt, old_cnt + jt_cur->cnt); + if (!jt) { + subprog->jt = NULL; + kvfree(jt_cur); + return -ENOMEM; } + memcpy(jt->items + old_cnt, jt_cur->items, jt_cur->cnt << 2); + subprog->jt = jt; kvfree(jt_cur); } - if (!jt) { + for (i = 0; i < env->subprog_cnt; i++) { + jt = env->subprog_info[i].jt; + if (jt) + jt->cnt = sort_insn_array_uniq(jt->items, jt->cnt); + } + + env->cfg.subprog_jts_ready = true; + return 0; +} + +/* Leave nothing behind on failure. */ +static int compute_subprog_jts(struct bpf_verifier_env *env) +{ + int err; + + err = __compute_subprog_jts(env); + if (err) + bpf_free_subprog_jts(env); + return err; +} + +void bpf_free_subprog_jts(struct bpf_verifier_env *env) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(env->subprog_info); i++) { + kvfree(env->subprog_info[i].jt); + env->subprog_info[i].jt = NULL; + } + env->cfg.subprog_jts_ready = false; +} + +static struct bpf_iarray * +subprog_jt(int t, struct bpf_verifier_env *env) +{ + struct bpf_subprog_info *subprog; + int subprog_start, err; + + if (!env->cfg.subprog_jts_ready) { + err = compute_subprog_jts(env); + if (err) + return ERR_PTR(err); + } + + subprog = bpf_find_containing_subprog(env, t); + if (verifier_bug_if(!subprog, env, "no subprog contains insn %d", t)) + return ERR_PTR(-EFAULT); + subprog_start = subprog->start; + + if (!subprog->jt) { verbose(env, "no jump tables found for subprog starting at %u\n", subprog_start); bpf_diag_program_structure( - env, subprog_start, "missing jump table", + env, t, "missing jump table", "Make sure subprograms containing gotox instructions are accompanied by jump tables referencing these subprograms.", "No jump table was found for the subprogram that starts at instruction %u.", subprog_start); return ERR_PTR(-EINVAL); } - jt->cnt = sort_insn_array_uniq(jt->items, jt->cnt); - return jt; -} - -static struct bpf_iarray * -create_jt(int t, struct bpf_verifier_env *env) -{ - struct bpf_subprog_info *subprog; - int subprog_start, subprog_end; - struct bpf_iarray *jt; - int i; - - subprog = bpf_find_containing_subprog(env, t); - subprog_start = subprog->start; - subprog_end = (subprog + 1)->start; - jt = jt_from_subprog(env, subprog_start, subprog_end); - if (IS_ERR(jt)) - return jt; - - /* Check that the every element of the jump table fits within the given subprogram */ - for (i = 0; i < jt->cnt; i++) { - if (jt->items[i] < subprog_start || jt->items[i] >= subprog_end) { - verbose(env, "jump table for insn %d points outside of the subprog [%u,%u]\n", - t, subprog_start, subprog_end); - bpf_diag_program_structure( - env, t, "jump table target out of range", - "Keep every jump-table target inside the same subprogram.", - "The jump table for instruction %d points outside subprogram range [%u,%u).", - t, subprog_start, subprog_end); - kvfree(jt); - return ERR_PTR(-EINVAL); - } - } - - return jt; + return subprog->jt; } /* "conditional jump with N edges" */ @@ -381,27 +412,39 @@ static int visit_gotox_insn(int t, struct bpf_verifier_env *env) struct bpf_iarray *jt; int i, w; - jt = env->insn_aux_data[t].jt; - if (!jt) { - jt = create_jt(t, env); - if (IS_ERR(jt)) - return PTR_ERR(jt); + /* + * The first visit pushes every target that still needs exploring, and + * those are all done by the time this insn is revisited, so there is + * nothing left to walk the jump table for. + */ + if (insn_state[t] & BRANCH) + return DONE_EXPLORING; - env->insn_aux_data[t].jt = jt; + jt = subprog_jt(t, env); + if (IS_ERR(jt)) + return PTR_ERR(jt); + + insn_state[t] |= BRANCH; + + if (check_add_overflow(env->cfg.gotox_edges, jt->cnt, + &env->cfg.gotox_edges) || + env->cfg.gotox_edges > BPF_MAX_GOTOX_EDGES) { + verbose(env, "number of indirect jump edges in the program exceeds %u\n", + BPF_MAX_GOTOX_EDGES); + bpf_diag_program_structure( + env, t, "too many indirect jump edges", + "Reduce the number of indirect jumps, or the number of distinct targets they can reach.", + "The program has more than %u indirect jump edges in total, counted over every gotox instruction.", + BPF_MAX_GOTOX_EDGES); + return -E2BIG; } mark_prune_point(env, t); for (i = 0; i < jt->cnt; i++) { w = jt->items[i]; - if (w < 0 || w >= env->prog->len) { - verbose(env, "indirect jump out of range from insn %d to %d\n", t, w); - bpf_diag_program_structure( - env, t, "indirect jump out of range", - "Keep indirect jump targets inside the program.", - "Instruction %d can jump indirectly to instruction %d, but the program only contains instructions 0 through %d.", - t, w, env->prog->len - 1); - return -EINVAL; - } + if (verifier_bug_if(w < 0 || w >= env->prog->len, env, + "indirect jump from insn %d to %d out of range", t, w)) + return -EFAULT; mark_jmp_point(env, w); mark_jump_target(env, w); @@ -422,27 +465,73 @@ static int visit_gotox_insn(int t, struct bpf_verifier_env *env) } /* - * Instructions that can abnormally return from a subprog (tail_call - * upon success, ld_{abs,ind} upon load failure) have a hidden exit - * that the verifier must account for. + * Return pointers to functions in the read-only map that ld_imm64 instruction + * 't' loads the address of, or of its value, if there are any. */ -static int visit_abnormal_return_insn(struct bpf_verifier_env *env, int t) +static struct bpf_func_ptr *insn_func_ptrs(struct bpf_verifier_env *env, int t, u32 *cnt) { - struct bpf_subprog_info *subprog; - struct bpf_iarray *jt; + struct bpf_insn *insn = &env->prog->insnsi[t]; - if (env->insn_aux_data[t].jt) - return 0; + *cnt = 0; + if (!env->func_ptr_cnt || !bpf_is_ldimm64(insn)) + return NULL; + if (insn->src_reg != BPF_PSEUDO_MAP_VALUE && + insn->src_reg != BPF_PSEUDO_MAP_IDX_VALUE && + insn->src_reg != BPF_PSEUDO_MAP_FD && + insn->src_reg != BPF_PSEUDO_MAP_IDX) + return NULL; - jt = bpf_iarray_realloc(NULL, 2); - if (!jt) - return -ENOMEM; + return bpf_map_func_ptrs(env, env->used_maps[env->insn_aux_data[t].map_index], cnt); +} - subprog = bpf_find_containing_subprog(env, t); - jt->items[0] = t + 1; - jt->items[1] = subprog->exit_idx; - env->insn_aux_data[t].jt = jt; - return 0; +/* + * ld_imm64 that loads the address of a map that has pointers to functions + * is similar to ld_imm64 with BPF_PSEUDO_FUNC that loads the address of one + * function: any of them may be read from the map and called via callx later. + * Treat it as a call of all of them. + */ +static int visit_func_ptrs_insn(int t, struct bpf_verifier_env *env, + struct bpf_func_ptr *ptrs, u32 cnt) +{ + int *insn_stack = env->cfg.insn_stack; + int *insn_state = env->cfg.insn_state; + int ret, w; + u32 i; + + ret = push_insn(t, t + 2, FALLTHROUGH, env); + if (ret) + return ret; + + /* + * Explore one function at a time like push_insn() does, so that + * DISCOVERED function is a caller of 't' and not the one that waits in + * insn_stack[], which merge_callee_effects() relies on. + */ + for (i = 0; i < cnt; i++) { + w = ptrs[i].xlated_off; + if (insn_state[w]) + continue; + + if (env->cfg.cur_stack >= env->prog->len) + return -E2BIG; + + insn_stack[env->cfg.cur_stack++] = w; + insn_state[w] = DISCOVERED; + return KEEP_EXPLORING; + } + + /* all functions are explored, their effects are known */ + mark_prune_point(env, t); + for (i = 0; i < cnt; i++) { + w = ptrs[i].xlated_off; + merge_callee_effects(env, t, w); + + /* the same marks as push_insn() leaves on a branch target */ + mark_prune_point(env, w); + mark_jmp_point(env, w); + mark_jump_target(env, w); + } + return DONE_EXPLORING; } /* Visits the instruction at index t and returns one of the following: @@ -453,21 +542,20 @@ static int visit_abnormal_return_insn(struct bpf_verifier_env *env, int t) static int visit_insn(int t, struct bpf_verifier_env *env) { struct bpf_insn *insns = env->prog->insnsi, *insn = &insns[t]; + struct bpf_func_ptr *ptrs; int ret, off, insn_sz; + u32 cnt; if (bpf_pseudo_func(insn)) return visit_func_call_insn(t, insns, env, true); + ptrs = insn_func_ptrs(env, t, &cnt); + if (ptrs) + return visit_func_ptrs_insn(t, env, ptrs, cnt); + /* All non-branch instructions have a single fall-through edge. */ if (BPF_CLASS(insn->code) != BPF_JMP && BPF_CLASS(insn->code) != BPF_JMP32) { - if (BPF_CLASS(insn->code) == BPF_LD && - (BPF_MODE(insn->code) == BPF_ABS || - BPF_MODE(insn->code) == BPF_IND)) { - ret = visit_abnormal_return_insn(env, t); - if (ret) - return ret; - } insn_sz = bpf_is_ldimm64(insn) ? 2 : 1; return push_insn(t, t + insn_sz, FALLTHROUGH, env); } @@ -510,13 +598,10 @@ static int visit_insn(int t, struct bpf_verifier_env *env) */ if (ret == 0 && fp->might_sleep) mark_subprog_might_sleep(env, t); - if (bpf_helper_changes_pkt_data(insn->imm)) + /* nobody knows yet whether the dynptr is that of an skb */ + if (bpf_helper_changes_pkt_data(insn->imm) || + insn->imm == BPF_FUNC_dynptr_write) mark_subprog_changes_pkt_data(env, t); - if (insn->imm == BPF_FUNC_tail_call) { - ret = visit_abnormal_return_insn(env, t); - if (ret) - return ret; - } } else if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) { struct bpf_call_arg_meta meta; @@ -684,6 +769,11 @@ int bpf_check_cfg(struct bpf_verifier_env *env) return ret; } +struct dfs_state { + u32 traversed:1; + u32 next_succ:31; +}; + /* * For each subprogram 'i' fill array env->cfg.insn_subprogram sub-range * [env->subprog_info[i].postorder_start, env->subprog_info[i+1].postorder_start) @@ -691,43 +781,52 @@ int bpf_check_cfg(struct bpf_verifier_env *env) */ int bpf_compute_postorder(struct bpf_verifier_env *env) { - u32 cur_postorder, i, top, stack_sz, s; - int *stack = NULL, *postorder = NULL, *state = NULL; - struct bpf_iarray *succ; + int *stack = NULL, *postorder = NULL, *postorder_nums = NULL; + int subprog_idx, stack_sz, cur, s, cur_postorder, start; + struct dfs_state *state = NULL; + struct bpf_iarray *succ = NULL; + postorder_nums = kvzalloc_objs(int, env->prog->len, GFP_KERNEL_ACCOUNT); postorder = kvzalloc_objs(int, env->prog->len, GFP_KERNEL_ACCOUNT); - state = kvzalloc_objs(int, env->prog->len, GFP_KERNEL_ACCOUNT); stack = kvzalloc_objs(int, env->prog->len, GFP_KERNEL_ACCOUNT); - if (!postorder || !state || !stack) { + state = kvzalloc_objs(struct dfs_state, env->prog->len, GFP_KERNEL_ACCOUNT); + if (!postorder_nums || !postorder || !stack || !state) { + kvfree(postorder_nums); kvfree(postorder); - kvfree(state); kvfree(stack); + kvfree(state); return -ENOMEM; } cur_postorder = 0; - for (i = 0; i < env->subprog_cnt; i++) { - env->subprog_info[i].postorder_start = cur_postorder; - stack[0] = env->subprog_info[i].start; + for (subprog_idx = 0; subprog_idx < env->subprog_cnt; subprog_idx++) { + start = env->subprog_info[subprog_idx].start; + env->subprog_info[subprog_idx].postorder_start = cur_postorder; + stack[0] = start; stack_sz = 1; + state[start].traversed = true; do { - top = stack[stack_sz - 1]; - state[top] |= DISCOVERED; - if (state[top] & EXPLORED) { - postorder[cur_postorder++] = top; + cur = stack[stack_sz - 1]; + succ = bpf_insn_successors(env, cur); + if (state[cur].next_succ == succ->cnt) { + postorder_nums[cur] = cur_postorder; + postorder[cur_postorder] = cur; + cur_postorder++; stack_sz--; continue; } - succ = bpf_insn_successors(env, top); - for (s = 0; s < succ->cnt; ++s) { - if (!state[succ->items[s]]) { - stack[stack_sz++] = succ->items[s]; - state[succ->items[s]] |= DISCOVERED; - } + s = succ->items[state[cur].next_succ]; + if (!state[s].traversed) { + state[s].traversed = true; + state[s].next_succ = 0; + stack[stack_sz] = s; + stack_sz++; + continue; } - state[top] |= EXPLORED; + state[cur].next_succ++; } while (stack_sz); } - env->subprog_info[i].postorder_start = cur_postorder; + env->subprog_info[subprog_idx].postorder_start = cur_postorder; + env->cfg.postorder_nums = postorder_nums; env->cfg.insn_postorder = postorder; env->cfg.cur_postorder = cur_postorder; kvfree(stack); @@ -749,7 +848,7 @@ int bpf_compute_scc(struct bpf_verifier_env *env) struct bpf_insn_aux_data *aux = env->insn_aux_data; const u32 insn_cnt = env->prog->len; int stack_sz, dfs_sz, err = 0; - u32 *stack, *pre, *low, *dfs; + u32 *stack, *pre, *low, *dfs, *dfs_pos; u32 i, j, t, w; u32 next_preorder_num; u32 next_scc_id; @@ -762,13 +861,16 @@ int bpf_compute_scc(struct bpf_verifier_env *env) * - 'stack' accumulates vertices in DFS order, see invariant comment below; * - 'pre[t] == p' => preorder number of vertex 't' is 'p'; * - 'low[t] == n' => smallest preorder number of the vertex reachable from 't' is 'n'; - * - 'dfs' DFS traversal stack, used to emulate explicit recursion. + * - 'dfs' DFS traversal stack, used to emulate explicit recursion; + * - 'dfs_pos[k] == j' => the frame 'dfs[k]' resumes visiting its + * successors at index 'j'. */ stack = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL_ACCOUNT); pre = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL_ACCOUNT); low = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL_ACCOUNT); dfs = kvcalloc(insn_cnt, sizeof(*dfs), GFP_KERNEL_ACCOUNT); - if (!stack || !pre || !low || !dfs) { + dfs_pos = kvcalloc(insn_cnt, sizeof(*dfs_pos), GFP_KERNEL_ACCOUNT); + if (!stack || !pre || !low || !dfs || !dfs_pos) { err = -ENOMEM; goto exit; } @@ -851,6 +953,7 @@ int bpf_compute_scc(struct bpf_verifier_env *env) stack_sz = 0; dfs_sz = 1; dfs[0] = i; + dfs_pos[0] = 0; dfs_continue: while (dfs_sz) { w = dfs[dfs_sz - 1]; @@ -860,13 +963,22 @@ int bpf_compute_scc(struct bpf_verifier_env *env) next_preorder_num++; stack[stack_sz++] = w; } - /* Visit 'w' successors */ + /* Visit remaining 'w' successors */ succ = bpf_insn_successors(env, w); - for (j = 0; j < succ->cnt; ++j) { + for (j = dfs_pos[dfs_sz - 1]; j < succ->cnt; ++j) { if (pre[succ->items[j]]) { low[w] = min(low[w], low[succ->items[j]]); } else { - dfs[dfs_sz++] = succ->items[j]; + /* + * Once DFS for succ->items[j] is complete, + * pre[succ->items[j]] is non-zero, hence + * resuming at 'j' allows to follow the + * low[w] = min(...) update branch above. + */ + dfs_pos[dfs_sz - 1] = j; + dfs_pos[dfs_sz] = 0; + dfs[dfs_sz] = succ->items[j]; + dfs_sz++; goto dfs_continue; } } @@ -916,5 +1028,6 @@ int bpf_compute_scc(struct bpf_verifier_env *env) kvfree(pre); kvfree(low); kvfree(dfs); + kvfree(dfs_pos); return err; } diff --git a/kernel/bpf/cgroup.c b/kernel/bpf/cgroup.c index 149672c76c49..ce82963bcfc7 100644 --- a/kernel/bpf/cgroup.c +++ b/kernel/bpf/cgroup.c @@ -24,6 +24,34 @@ DEFINE_STATIC_KEY_ARRAY_FALSE(cgroup_bpf_enabled_key, MAX_CGROUP_BPF_ATTACH_TYPE); EXPORT_SYMBOL(cgroup_bpf_enabled_key); +struct cgroup_struct_ops { + u32 type_id; + void *cfi_stubs; + bool mult_rcu; +}; + +static struct cgroup_struct_ops cgroup_struct_ops[MAX_CGROUP_BPF_ATTACH_TYPE]; + +void cgroup_bpf_struct_ops_register(int atype, u32 type_id, void *cfi_stubs, bool mult_rcu) +{ + cgroup_struct_ops[atype].type_id = type_id; + cgroup_struct_ops[atype].cfi_stubs = cfi_stubs; + cgroup_struct_ops[atype].mult_rcu = mult_rcu; +} + +static enum cgroup_bpf_attach_type find_atype_by_struct_ops_id(u32 type_id) +{ + enum cgroup_bpf_attach_type atype; + + for (atype = 0; atype < MAX_CGROUP_BPF_ATTACH_TYPE; atype++) { + if (cgroup_bpf_is_struct_ops_atype(atype) && + cgroup_struct_ops[atype].type_id && + cgroup_struct_ops[atype].type_id == type_id) + return atype; + } + return CGROUP_BPF_ATTACH_TYPE_INVALID; +} + /* * cgroup bpf destruction makes heavy use of work items and there can be a lot * of concurrent destructions. Use a separate workqueue so that cgroup bpf @@ -306,6 +334,19 @@ static void bpf_cgroup_storages_link(struct bpf_cgroup_storage *storages[], bpf_cgroup_storage_link(storages[stype], cgrp, attach_type); } +static void cgroup_struct_ops_link_detach_wake(struct bpf_cgroup_link *link, bool wake_poll) +{ + cgroup_put(link->cgroup); + link->cgroup = NULL; + + bpf_map_put(link->map); + /* READ_ONCE in cgroup_struct_ops_link_poll */ + WRITE_ONCE(link->map, NULL); + + if (wake_poll) + wake_up_interruptible_poll(&link->wait_hup, EPOLLHUP); +} + /* Called when bpf_cgroup_link is auto-detached from dying cgroup. * It drops cgroup and bpf_prog refcounts, and marks bpf_link as defunct. It * doesn't free link memory, which will eventually be done by bpf_link's @@ -313,10 +354,96 @@ static void bpf_cgroup_storages_link(struct bpf_cgroup_storage *storages[], */ static void bpf_cgroup_link_auto_detach(struct bpf_cgroup_link *link) { + if (link->map) { + cgroup_struct_ops_link_detach_wake(link, true); + return; + } + + if (link->link.prog->expected_attach_type == BPF_LSM_CGROUP) + bpf_trampoline_unlink_cgroup_shim(link->link.prog); cgroup_put(link->cgroup); link->cgroup = NULL; } +static void bpf_cgroup_array_free_rcu(struct rcu_head *rcu) +{ + kfree(container_of(rcu, struct bpf_prog_array, rcu)); +} + +static void bpf_cgroup_array_free(struct bpf_prog_array *array, + enum cgroup_bpf_attach_type atype) +{ + if (!array || array == &bpf_empty_prog_array) + return; + if (cgroup_struct_ops[atype].mult_rcu) + /* RCU tasks trace grace period implies RCU grace period. */ + call_rcu_tasks_trace(&array->rcu, bpf_cgroup_array_free_rcu); + else + kfree_rcu(array, rcu); +} + +static void *bpf_cgroup_array_dummy(enum cgroup_bpf_attach_type atype) +{ + if (cgroup_bpf_is_struct_ops_atype(atype)) + return cgroup_struct_ops[atype].cfi_stubs; + return bpf_prog_dummy(); +} + +static int bpf_cgroup_array_length(struct bpf_prog_array *array, + enum cgroup_bpf_attach_type atype) +{ + struct bpf_prog_array_item *item; + int cnt = 0; + + for (item = array->items; item->prog; item++) { + if (item->prog != bpf_cgroup_array_dummy(atype)) + cnt++; + } + + return cnt; +} + +static int bpf_cgroup_array_copy_to_user(struct bpf_prog_array *array, + __u32 __user *prog_ids, int cnt, + enum cgroup_bpf_attach_type atype) +{ + struct bpf_prog_array_item *item; + int i = 0; + u32 id; + + for (item = array->items; item->prog && i < cnt; item++) { + if (item->prog == bpf_cgroup_array_dummy(atype)) + continue; + + if (cgroup_bpf_is_struct_ops_atype(atype)) + id = bpf_struct_ops_id(item->kdata); + else + id = item->prog->aux->id; + + if (copy_to_user(prog_ids + i, &id, sizeof(id))) + return -EFAULT; + i++; + } + return item->prog ? -ENOSPC : 0; +} + +static int bpf_cgroup_array_delete_safe_at(struct bpf_prog_array *array, + int index, enum cgroup_bpf_attach_type atype) +{ + struct bpf_prog_array_item *item; + + for (item = array->items; item->prog; item++) { + if (item->prog == bpf_cgroup_array_dummy(atype)) + continue; + if (!index) { + WRITE_ONCE(item->prog, bpf_cgroup_array_dummy(atype)); + return 0; + } + index--; + } + return -ENOENT; +} + /** * cgroup_bpf_release() - put references of all bpf programs and * release all cgroup bpf data @@ -346,18 +473,15 @@ static void cgroup_bpf_release(struct work_struct *work) bpf_trampoline_unlink_cgroup_shim(pl->prog); bpf_prog_put(pl->prog); } - if (pl->link) { - if (pl->link->link.prog->expected_attach_type == BPF_LSM_CGROUP) - bpf_trampoline_unlink_cgroup_shim(pl->link->link.prog); + if (pl->link) bpf_cgroup_link_auto_detach(pl->link); - } kfree(pl); static_branch_dec(&cgroup_bpf_enabled_key[atype]); } old_array = rcu_dereference_protected( cgrp->bpf.effective[atype], lockdep_is_held(&cgroup_mutex)); - bpf_prog_array_free(old_array); + bpf_cgroup_array_free(old_array, atype); } list_for_each_entry_safe(storage, stmp, storages, list_cg) { @@ -399,6 +523,37 @@ static struct bpf_prog *prog_list_prog(struct bpf_prog_list *pl) return NULL; } +static bool prog_list_is_struct_ops(const struct bpf_prog_list *pl) +{ + return pl->link && pl->link->map; +} + +static void prog_list_init_item(struct bpf_prog_list *pl, struct bpf_prog_array_item *item) +{ + if (prog_list_is_struct_ops(pl)) { + item->kdata = bpf_struct_ops_map_kdata(pl->link->map); + return; + } + + item->prog = prog_list_prog(pl); + bpf_cgroup_storages_assign(item->cgroup_storage, pl->storage); +} + +static void prog_list_replace_item(struct bpf_prog_list *pl, struct bpf_prog_array_item *item) +{ + if (prog_list_is_struct_ops(pl)) + WRITE_ONCE(item->kdata, bpf_struct_ops_map_kdata(pl->link->map)); + else + WRITE_ONCE(item->prog, pl->link->link.prog); +} + +static u32 prog_list_id(struct bpf_prog_list *pl) +{ + if (prog_list_is_struct_ops(pl)) + return pl->link->map->id; + return prog_list_prog(pl)->aux->id; +} + /* count number of elements in the list. * it's slow but the list cannot be long */ @@ -408,7 +563,7 @@ static u32 prog_list_length(struct hlist_head *head, int *preorder_cnt) u32 cnt = 0; hlist_for_each_entry(pl, head, node) { - if (!prog_list_prog(pl)) + if (!pl->prog && !pl->link) continue; if (preorder_cnt && (pl->flags & BPF_F_PREORDER)) (*preorder_cnt)++; @@ -482,7 +637,7 @@ static int compute_effective_progs(struct cgroup *cgrp, init_bstart = bstart; hlist_for_each_entry(pl, &p->bpf.progs[atype], node) { - if (!prog_list_prog(pl)) + if (!pl->prog && !pl->link) continue; if (pl->flags & BPF_F_PREORDER) { @@ -492,9 +647,7 @@ static int compute_effective_progs(struct cgroup *cgrp, item = &progs->items[fstart]; fstart++; } - item->prog = prog_list_prog(pl); - bpf_cgroup_storages_assign(item->cgroup_storage, - pl->storage); + prog_list_init_item(pl, item); cnt++; } @@ -517,7 +670,7 @@ static void activate_effective_progs(struct cgroup *cgrp, /* free prog array after grace period, since __cgroup_bpf_run_*() * might be still walking the array */ - bpf_prog_array_free(old_array); + bpf_cgroup_array_free(old_array, atype); } /** @@ -557,7 +710,7 @@ static int cgroup_bpf_inherit(struct cgroup *cgrp) return 0; cleanup: for (i = 0; i < NR; i++) - bpf_prog_array_free(arrays[i]); + bpf_cgroup_array_free(arrays[i], i); for (p = cgroup_parent(cgrp); p; p = cgroup_parent(p)) cgroup_bpf_put(p); @@ -612,7 +765,7 @@ static int update_effective_progs(struct cgroup *cgrp, if (percpu_ref_is_zero(&desc->bpf.refcnt)) { if (unlikely(desc->bpf.inactive)) { - bpf_prog_array_free(desc->bpf.inactive); + bpf_cgroup_array_free(desc->bpf.inactive, atype); desc->bpf.inactive = NULL; } continue; @@ -631,7 +784,7 @@ static int update_effective_progs(struct cgroup *cgrp, css_for_each_descendant_pre(css, &cgrp->self) { struct cgroup *desc = container_of(css, struct cgroup, self); - bpf_prog_array_free(desc->bpf.inactive); + bpf_cgroup_array_free(desc->bpf.inactive, atype); desc->bpf.inactive = NULL; } @@ -870,7 +1023,7 @@ static int __cgroup_bpf_attach(struct cgroup *cgrp, old_pl_flags = pl->flags; bpf_cgroup_storages_assign(old_storage, pl->storage); } else { - pl = kmalloc_obj(*pl); + pl = kzalloc_obj(*pl); if (!pl) { bpf_cgroup_storages_free(new_storage); return -ENOMEM; @@ -973,9 +1126,10 @@ static int effective_prog_pos(struct cgroup *cgrp, init_bstart = bstart; hlist_for_each_entry(pl, &p->bpf.progs[atype], node) { - if (!prog_list_prog(pl)) - continue; - + /* + * No detaching pl (NULL prog and link) is visible to the callers, + * so skip the check compute_effective_progs() needs. + */ if (pl->flags & BPF_F_PREORDER) { if (pl == target_pl) pos = bstart; @@ -1022,7 +1176,7 @@ static void replace_effective_prog(struct cgroup *cgrp, desc->bpf.effective[atype], lockdep_is_held(&cgroup_mutex)); item = &progs->items[pos]; - WRITE_ONCE(item->prog, pl->link->link.prog); + prog_list_replace_item(pl, item); } } @@ -1177,7 +1331,7 @@ static void purge_effective_progs(struct cgroup *cgrp, struct bpf_prog_list *pl, lockdep_is_held(&cgroup_mutex)); /* Remove the program from the array */ - WARN_ONCE(bpf_prog_array_delete_safe_at(progs, pos), + WARN_ONCE(bpf_cgroup_array_delete_safe_at(progs, pos, atype), "Failed to purge a prog from array at index %d", pos); } } @@ -1287,7 +1441,15 @@ static int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr, if (effective_query && prog_attach_flags) return -EINVAL; - if (type == BPF_LSM_CGROUP) { + if (type == BPF_STRUCT_OPS) { + u32 type_id = attr->query.type_id; + + atype = find_atype_by_struct_ops_id(type_id); + if (atype == CGROUP_BPF_ATTACH_TYPE_INVALID) + return -ENOENT; + from_atype = to_atype = atype; + flags = 0; + } else if (type == BPF_LSM_CGROUP) { if (!effective_query && attr->query.prog_cnt && prog_ids && !prog_attach_flags) return -EINVAL; @@ -1307,7 +1469,7 @@ static int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr, if (effective_query) { effective = rcu_dereference_protected(cgrp->bpf.effective[atype], lockdep_is_held(&cgroup_mutex)); - total_cnt += bpf_prog_array_length(effective); + total_cnt += bpf_cgroup_array_length(effective, atype); } else { total_cnt += prog_list_length(&cgrp->bpf.progs[atype], NULL); } @@ -1337,35 +1499,33 @@ static int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr, if (effective_query) { effective = rcu_dereference_protected(cgrp->bpf.effective[atype], lockdep_is_held(&cgroup_mutex)); - cnt = min_t(int, bpf_prog_array_length(effective), total_cnt); - ret = bpf_prog_array_copy_to_user(effective, prog_ids, cnt); + cnt = min_t(int, bpf_cgroup_array_length(effective, atype), total_cnt); + ret = bpf_cgroup_array_copy_to_user(effective, prog_ids, cnt, atype); } else { struct hlist_head *progs; struct bpf_prog_list *pl; - struct bpf_prog *prog; u32 id; progs = &cgrp->bpf.progs[atype]; cnt = min_t(int, prog_list_length(progs, NULL), total_cnt); i = 0; hlist_for_each_entry(pl, progs, node) { - prog = prog_list_prog(pl); - id = prog->aux->id; + id = prog_list_id(pl); if (copy_to_user(prog_ids + i, &id, sizeof(id))) return -EFAULT; + if (prog_attach_flags) { + flags = cgrp->bpf.flags[atype] | + (pl->flags & BPF_F_PREORDER); + if (copy_to_user(prog_attach_flags + i, + &flags, sizeof(flags))) + return -EFAULT; + } if (++i == cnt) break; } - if (prog_attach_flags) { - flags = cgrp->bpf.flags[atype]; - - for (i = 0; i < cnt; i++) - if (copy_to_user(prog_attach_flags + i, - &flags, sizeof(flags))) - return -EFAULT; + if (prog_attach_flags) prog_attach_flags += cnt; - } } prog_ids += cnt; @@ -2329,7 +2489,7 @@ static const struct bpf_func_proto bpf_sysctl_get_name_proto = { .gpl_only = false, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, - .arg2_type = ARG_PTR_TO_MEM | MEM_WRITE, + .arg2_type = ARG_PTR_TO_MEM | MEM_UNINIT | MEM_WRITE, .arg3_type = ARG_MEM_SIZE, .arg4_type = ARG_ANYTHING, }; @@ -2772,6 +2932,274 @@ const struct bpf_verifier_ops cg_sockopt_verifier_ops = { const struct bpf_prog_ops cg_sockopt_prog_ops = { }; +static int __cgroup_struct_ops_link_detach(struct bpf_link *link, bool wake_poll) +{ + struct bpf_cgroup_link *cg_link = container_of(link, struct bpf_cgroup_link, link); + enum cgroup_bpf_attach_type atype; + struct bpf_prog_list *pl; + struct bpf_map *map; + struct cgroup *cgrp; + + cgroup_lock(); + + cgrp = cg_link->cgroup; + if (!cgrp) { + cgroup_unlock(); + return 0; + } + + map = cg_link->map; + atype = bpf_struct_ops_map_cgroup_atype(map); + + hlist_for_each_entry(pl, &cgrp->bpf.progs[atype], node) { + if (pl->link == cg_link) + break; + } + if (WARN_ON_ONCE(!pl)) { + cgroup_unlock(); + return -ENOENT; + } + + /* mark deleted so compute_effective_progs() skips it */ + pl->link = NULL; + if (update_effective_progs(cgrp, atype)) { + pl->link = cg_link; + purge_effective_progs(cgrp, pl, atype); + } + + hlist_del(&pl->node); + cgroup_struct_ops_link_detach_wake(cg_link, wake_poll); + cgrp->bpf.revisions[atype]++; + + kfree(pl); + static_branch_dec(&cgroup_bpf_enabled_key[atype]); + + cgroup_unlock(); + + return 0; +} + +static int cgroup_struct_ops_link_detach(struct bpf_link *link) +{ + return __cgroup_struct_ops_link_detach(link, true); +} + +static void cgroup_struct_ops_link_dealloc(struct bpf_link *link) +{ + struct bpf_cgroup_link *cg_link = container_of(link, struct bpf_cgroup_link, link); + + __cgroup_struct_ops_link_detach(link, false); + kfree(cg_link); +} + +static void cgroup_struct_ops_link_show_fdinfo(const struct bpf_link *link, struct seq_file *seq) +{ + struct bpf_cgroup_link *cg_link = + container_of(link, struct bpf_cgroup_link, link); + + cgroup_lock(); + if (!cg_link->cgroup) { + cgroup_unlock(); + return; + } + + seq_printf(seq, "map_id:\t%u\n", cg_link->map->id); + seq_printf(seq, "cgroup_id:\t%llu\n", cgroup_id(cg_link->cgroup)); + cgroup_unlock(); +} + +static int cgroup_struct_ops_link_fill_link_info(const struct bpf_link *link, + struct bpf_link_info *info) +{ + struct bpf_cgroup_link *cg_link = container_of(link, struct bpf_cgroup_link, link); + + cgroup_lock(); + if (!cg_link->cgroup) { + cgroup_unlock(); + return 0; + } + + info->struct_ops.map_id = cg_link->map->id; + info->struct_ops.cgroup_id = cgroup_id(cg_link->cgroup); + cgroup_unlock(); + return 0; +} + +static int cgroup_struct_ops_link_update(struct bpf_link *link, struct bpf_map *new_map, + struct bpf_map *expected_old_map) +{ + struct bpf_cgroup_link *cg_link = container_of(link, struct bpf_cgroup_link, link); + enum cgroup_bpf_attach_type atype; + struct bpf_prog_list *pl; + struct bpf_map *old_map; + struct cgroup *cgrp; + bool found = false; + int err; + + if (!bpf_struct_ops_valid_to_reg(new_map)) + return -EINVAL; + + cgroup_lock(); + + cgrp = cg_link->cgroup; + if (!cgrp) { + err = -ENOLINK; + goto out; + } + + old_map = cg_link->map; + err = bpf_struct_ops_link_update_check(new_map, old_map, expected_old_map); + if (err) + goto out; + + atype = bpf_struct_ops_map_cgroup_atype(new_map); + + hlist_for_each_entry(pl, &cgrp->bpf.progs[atype], node) { + if (pl->link == cg_link) { + found = true; + break; + } + } + if (!found) { + err = -ENOENT; + goto out; + } + + bpf_map_inc(new_map); + WRITE_ONCE(cg_link->map, new_map); + replace_effective_prog(cgrp, atype, pl); + bpf_map_put(old_map); + cgrp->bpf.revisions[atype]++; + +out: + cgroup_unlock(); + return err; +} + +static __poll_t cgroup_struct_ops_link_poll(struct file *file, struct poll_table_struct *pts) +{ + struct bpf_cgroup_link *link = file->private_data; + + poll_wait(file, &link->wait_hup, pts); + + return READ_ONCE(link->map) ? 0 : EPOLLHUP; +} + +static const struct bpf_link_ops cgroup_struct_ops_link_ops = { + .dealloc = cgroup_struct_ops_link_dealloc, + .detach = cgroup_struct_ops_link_detach, + .show_fdinfo = cgroup_struct_ops_link_show_fdinfo, + .fill_link_info = cgroup_struct_ops_link_fill_link_info, + .update_map = cgroup_struct_ops_link_update, + .poll = cgroup_struct_ops_link_poll, +}; + +int cgroup_bpf_struct_ops_attach(struct bpf_map *map, const union bpf_attr *attr) +{ + u32 flags = attr->link_create.flags; + u32 pl_flags = (flags & BPF_F_PREORDER) | BPF_F_ALLOW_MULTI; + enum cgroup_bpf_attach_type atype; + struct bpf_link_primer link_primer; + struct bpf_cgroup_link *link; + struct bpf_prog_list *pl = NULL; + struct hlist_head *progs; + struct cgroup *cgrp; + int err; + + if (flags & ~BPF_F_LINK_ATTACH_MASK) + return -EINVAL; + + /* + * Attaching struct_ops to cgroup is through link only. All relative + * position must be corresponding to a link id or fd. + */ + if (attr->link_create.cgroup.relative_fd && !(flags & BPF_F_LINK)) + return -EINVAL; + + link = kzalloc_obj(*link, GFP_USER); + if (!link) + return -ENOMEM; + + bpf_link_init(&link->link, BPF_LINK_TYPE_STRUCT_OPS, + &cgroup_struct_ops_link_ops, NULL, + attr->link_create.attach_type); + + err = bpf_link_prime(&link->link, &link_primer); + if (err) { + kfree(link); + return err; + } + + cgrp = cgroup_get_from_fd(attr->link_create.target_fd); + if (IS_ERR(cgrp)) { + err = PTR_ERR(cgrp); + goto cleanup; + } + + bpf_map_inc(map); + link->map = map; + link->cgroup = cgrp; + init_waitqueue_head(&link->wait_hup); + + atype = bpf_struct_ops_map_cgroup_atype(map); + progs = &cgrp->bpf.progs[atype]; + + cgroup_lock(); + + if (attr->link_create.cgroup.expected_revision && + attr->link_create.cgroup.expected_revision != cgrp->bpf.revisions[atype]) { + err = -ESTALE; + goto unlock; + } + + if (prog_list_length(progs, NULL) >= BPF_CGROUP_MAX_PROGS) { + err = -E2BIG; + goto unlock; + } + + pl = kzalloc_obj(*pl); + if (!pl) { + err = -ENOMEM; + goto unlock; + } + + pl->link = link; + pl->flags = pl_flags; + cgrp->bpf.flags[atype] = BPF_F_ALLOW_MULTI; + + err = insert_pl_to_hlist(pl, progs, NULL, link, + flags | BPF_F_ALLOW_MULTI, attr->link_create.cgroup.relative_fd); + if (err) + goto unlock; + + err = update_effective_progs(cgrp, atype); + if (err) { + hlist_del(&pl->node); + goto unlock; + } + + cgrp->bpf.revisions[atype]++; + static_branch_inc(&cgroup_bpf_enabled_key[atype]); + + cgroup_unlock(); + + return bpf_link_settle(&link_primer); + +unlock: + cgroup_unlock(); + +cleanup: + kfree(pl); + if (link->cgroup) { + cgroup_put(link->cgroup); + link->cgroup = NULL; + bpf_map_put(link->map); + link->map = NULL; + } + bpf_link_cleanup(&link_primer); + return err; +} + /* Common helpers for cgroup hooks. */ const struct bpf_func_proto * cgroup_common_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) diff --git a/kernel/bpf/cnum.c b/kernel/bpf/cnum.c index 86142cb2aee5..8ec5187d7c75 100644 --- a/kernel/bpf/cnum.c +++ b/kernel/bpf/cnum.c @@ -2,6 +2,7 @@ /* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ #include +#include /* for imod() */ #define T 32 #include "cnum_defs.h" @@ -118,3 +119,38 @@ struct cnum64 cnum64_cnum32_intersect(struct cnum64 a, struct cnum32 b) } return t; } + +/* Intersect 'in' with the set of integers defined by equation 'base + step * k'. */ +struct cnum64 cnum64_intersect_linear(struct cnum64 in, u16 base, u16 step) +{ + s64 smin = cnum64_smin(in); + s64 smax = cnum64_smax(in); + s64 lo, hi; + u16 d; + + if (step <= 1 || cnum64_is_empty(in) || cnum64_srange_overflow(in)) + return in; + /* + * Round smin up to the next value congruent to 'base' modulo 'step', + * i.e. increase smin by d = (base - smin) mod step: + * + * |<---- d ---->| + * |-----------|=============|... + * base+step*k smin base+step*(k+1) + */ + d = imod(base - imod(smin, step), step); + if ((u64)smax - (u64)smin < d) + return CNUM64_EMPTY; + lo = smin + d; + /* + * Round smax down to the previous value congruent to 'base' modulo 'step', + * i.e. decrease smax by d = (smax - base) mod step: + * + * |<--- d --->| + * ...|===========|-------------| + * base+step*k smax base+step*(k+1) + */ + d = imod(imod(smax, step) - base, step); + hi = smax - d; + return cnum64_from_srange(lo, hi); +} diff --git a/kernel/bpf/cnum_defs.h b/kernel/bpf/cnum_defs.h index a90e317e3578..05b17a8b4814 100644 --- a/kernel/bpf/cnum_defs.h +++ b/kernel/bpf/cnum_defs.h @@ -33,7 +33,12 @@ struct cnum_t FN(from_srange)(st min, st max) return (struct cnum_t){ .base = base, .size = size }; } -/* True if this cnum represents two unsigned ranges. */ +/* + * True if this cnum represents two unsigned ranges. + * + * The caller should ensure !is_empty(cnum) holds when calling + * this function. + */ static inline bool FN(urange_overflow)(struct cnum_t cnum) { /* Same as cnum.base + cnum.size > UT_MAX but avoids overflow */ @@ -44,6 +49,9 @@ static inline bool FN(urange_overflow)(struct cnum_t cnum) * cnum{T}_umin / cnum{T}_umax query an unsigned range represented by this cnum. * If cnum represents a range crossing the UT_MAX/0 boundary, the unbound range * [0..UT_MAX] is returned. + * + * The caller should ensure !is_empty(cnum) holds when calling + * cnum{T}_umin / cnum{T}_umax. */ ut FN(umin)(struct cnum_t cnum) { @@ -57,7 +65,12 @@ ut FN(umax)(struct cnum_t cnum) } EXPORT_SYMBOL_GPL(FN(umax)); -/* True if this cnum represents two signed ranges. */ +/* + * True if this cnum represents two signed ranges. + * + * The caller should ensure !is_empty(cnum) holds when calling + * this function. + */ static inline bool FN(srange_overflow)(struct cnum_t cnum) { return FN(contains)(cnum, (ut)ST_MAX) && FN(contains)(cnum, (ut)ST_MIN); @@ -67,19 +80,18 @@ static inline bool FN(srange_overflow)(struct cnum_t cnum) * cnum{T}_smin / cnum{T}_smax query a signed range represented by this cnum. * If cnum represents a range crossing the ST_MAX/ST_MIN boundary, the unbound range * [ST_MIN..ST_MAX] is returned. + * + * The caller should ensure !is_empty(cnum) holds when calling + * cnum{T}_smin / cnum{T}_smax. */ st FN(smin)(struct cnum_t cnum) { - return FN(srange_overflow)(cnum) - ? ST_MIN - : min((st)cnum.base, (st)(cnum.base + cnum.size)); + return FN(srange_overflow)(cnum) ? ST_MIN : (st)cnum.base; } st FN(smax)(struct cnum_t cnum) { - return FN(srange_overflow)(cnum) - ? ST_MAX - : max((st)cnum.base, (st)(cnum.base + cnum.size)); + return FN(srange_overflow)(cnum) ? ST_MAX : (st)(cnum.base + cnum.size); } /* @@ -181,11 +193,55 @@ void FN(intersect_with_srange)(struct cnum_t *dst, st min, st max) static inline struct cnum_t FN(normalize)(struct cnum_t cnum) { - if (cnum.size == UT_MAX && cnum.base != 0 && cnum.base != (ut)ST_MAX) + if (cnum.size == UT_MAX) cnum.base = 0; return cnum; } +/* + * Return a smallest arc containing both 'a' and 'b'. + * Break equal-size ties by choosing the smaller base. + */ +struct cnum_t FN(union)(struct cnum_t a, struct cnum_t b) +{ + struct cnum_t b1, ab, ba; + ut end; + + if (FN(is_empty)(a)) + return b; + if (FN(is_empty)(b)) + return a; + + /* + * Rotate so that a1.base == 0 and a1.end == a.size. + * Normalize b1 to preserve the full-circle representation. + */ + b1 = FN(normalize)((struct cnum_t){ b.base - a.base, b.size }); + end = max(a.size, (ut)(b1.base + b1.size)); + + if (FN(urange_overflow)(b1)) { + /* a1 reaches b1's main arc: together they cover the circle. */ + if (b1.base <= a.size) + return (struct cnum_t){ 0, UT_MAX }; + + /* Extend b1's tail through a1's end, then rotate back. */ + return FN(normalize)((struct cnum_t){ b.base, end - b1.base }); + } + + /* ab, rotated back, covers both nonwrapping arcs. */ + ab = (struct cnum_t){ a.base, end }; + if (b1.base <= a.size) + return FN(normalize)(ab); + + /* The arcs are disjoint; ba is the other possible covering arc. */ + ba = (struct cnum_t){ b.base, a.size - b1.base }; + if (ba.size < ab.size || + (ba.size == ab.size && ba.base < ab.base)) + ab = ba; + + return FN(normalize)(ab); +} + struct cnum_t FN(add)(struct cnum_t a, struct cnum_t b) { if (FN(is_empty)(a) || FN(is_empty)(b)) @@ -210,12 +266,7 @@ bool FN(is_empty)(struct cnum_t cnum) bool FN(contains)(struct cnum_t cnum, ut v) { - if (FN(is_empty)(cnum)) - return false; - if (FN(urange_overflow)(cnum)) - return v >= cnum.base || v <= (ut)cnum.base + cnum.size; - else - return v >= cnum.base && v <= (ut)cnum.base + cnum.size; + return !FN(is_empty)(cnum) && v - cnum.base <= cnum.size; } bool FN(is_const)(struct cnum_t cnum) diff --git a/kernel/bpf/const_fold.c b/kernel/bpf/const_fold.c index 7f1b30059cc8..f9edfde965dc 100644 --- a/kernel/bpf/const_fold.c +++ b/kernel/bpf/const_fold.c @@ -180,9 +180,12 @@ static void const_reg_xfer(struct bpf_verifier_env *env, struct const_arg_info * bool is_ldsx = mode == BPF_MEMSX; int off = src->val + insn->off; u64 val = 0; + u32 cnt; if (!bpf_map_is_rdonly(map) || !map->ops->map_direct_value_addr || off < 0 || off + size > map->value_size || + /* so are the addresses of functions that the map points to */ + bpf_map_range_func_ptrs(env, map, off, size, &cnt) || bpf_map_direct_read(map, off, size, &val, is_ldsx)) { *dst = unknown; break; @@ -191,7 +194,8 @@ static void const_reg_xfer(struct bpf_verifier_env *env, struct const_arg_info * dst->val = val; break; case BPF_JMP: - if (opcode != BPF_CALL) + /* both 'call imm' and 'callx reg' clobber caller saved registers */ + if (BPF_OP(insn->code) != BPF_CALL) break; process_call: for (r = BPF_REG_0; r <= BPF_REG_5; r++) @@ -353,7 +357,7 @@ int bpf_prune_dead_branches(struct bpf_verifier_env *env) if (!bpf_insn_is_cond_jump(insn->code)) continue; - if (bpf_is_may_goto_insn(insn)) + if (BPF_OP(insn->code) == BPF_JCOND) continue; if (!(aux->const_reg_mask & BIT(insn->dst_reg))) @@ -394,5 +398,7 @@ int bpf_prune_dead_branches(struct bpf_verifier_env *env) /* recompute postorder, since CFG has changed */ kvfree(env->cfg.insn_postorder); env->cfg.insn_postorder = NULL; + kvfree(env->cfg.postorder_nums); + env->cfg.postorder_nums = NULL; return bpf_compute_postorder(env); } diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 7f11555a5070..3030beb531d9 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -18,6 +18,7 @@ */ #include +#include #include #include #include @@ -142,10 +143,6 @@ struct bpf_prog *bpf_prog_alloc_no_stats(unsigned int size, gfp_t gfp_extra_flag mutex_init(&fp->aux->dst_mutex); mutex_init(&fp->aux->st_ops_assoc_mutex); -#ifdef CONFIG_BPF_SYSCALL - bpf_prog_stream_init(fp); -#endif - return fp; } @@ -175,7 +172,6 @@ struct bpf_prog *bpf_prog_alloc(unsigned int size, gfp_t gfp_extra_flags) } return prog; } -EXPORT_SYMBOL_GPL(bpf_prog_alloc); int bpf_prog_alloc_jited_linfo(struct bpf_prog *prog) { @@ -273,6 +269,7 @@ struct bpf_prog *bpf_prog_realloc(struct bpf_prog *fp_old, unsigned int size, memcpy(fp, fp_old, fp_old->pages * PAGE_SIZE); fp->pages = pages; fp->aux->prog = fp; + bpf_prog_dev_bound_moved(fp); /* We keep fp->aux from fp_old around in the new * reallocated structure. @@ -289,6 +286,9 @@ struct bpf_prog *bpf_prog_realloc(struct bpf_prog *fp_old, unsigned int size, void __bpf_prog_free(struct bpf_prog *fp) { if (fp->aux) { +#ifdef CONFIG_BPF_SYSCALL + bpf_prog_stream_free(fp); +#endif mutex_destroy(&fp->aux->used_maps_mutex); mutex_destroy(&fp->aux->dst_mutex); mutex_destroy(&fp->aux->st_ops_assoc_mutex); @@ -590,6 +590,10 @@ bpf_prog_ksym_set_name(struct bpf_prog *prog) prog->aux->func_info[prog->aux->func_idx].type_id); func_name = btf_name_by_offset(prog->aux->btf, type->name_off); snprintf(sym, (size_t)(end - sym), "_%s", func_name); + /* the name of a function of Rust is not an identifier */ + for (; *sym; sym++) + if (!isalnum(*sym) && *sym != '_' && *sym != '.') + *sym = '_'; return; } @@ -1555,8 +1559,7 @@ void bpf_jit_prog_release_other(struct bpf_prog *fp, struct bpf_prog *fp_other) * know whether fp here is the clone or the original. */ fp->aux->prog = fp; - if (fp->aux->offload) - fp->aux->offload->prog = fp; + bpf_prog_dev_bound_moved(fp); bpf_prog_clone_free(fp_other); } @@ -1677,7 +1680,6 @@ noinline u64 __bpf_call_base(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5) { return 0; } -EXPORT_SYMBOL_GPL(__bpf_call_base); /* All UAPI available opcodes. */ #define BPF_INSN_MAP(INSN_2, INSN_3) \ @@ -1840,6 +1842,7 @@ bool bpf_opcode_in_insntable(u8 code) [BPF_LD | BPF_IND | BPF_H] = true, [BPF_LD | BPF_IND | BPF_W] = true, [BPF_JMP | BPF_JA | BPF_X] = true, + [BPF_JMP | BPF_CALL | BPF_X] = true, [BPF_JMP | BPF_JCOND] = true, }; #undef BPF_INSN_3_TBL @@ -2634,11 +2637,68 @@ static bool bpf_prog_select_interpreter(struct bpf_prog *fp) return select_interpreter; } +/* + * Whether a function has an insn that works with the arena base, which a JIT + * sets up in the prologue when bpf_prog_uses_arena_insns() says so. Decided on + * the final insns of the function: an access through an arena pointer, or + * through a number in a program loaded with BPF_F_ARENA_SCALAR, has been + * rewritten to a BPF_PROBE_MEM32 form by then, a cast to the user address + * space keeps BPF_ADDR_SPACE_CAST, and a kfunc call with an __arena argument, + * which the JIT rebases, has a function model that says so. A load of an + * __arena struct member is rewritten with an immediate and needs none; + * dereferencing the result does. + */ +bool bpf_prog_has_arena_insns(const struct bpf_prog *prog) +{ + const struct bpf_insn *insn = prog->insnsi; + const struct btf_func_model *m; + int i, j; + + if (!prog->aux->arena) + return false; + + for (i = 0; i < prog->len; i++, insn++) { + switch (BPF_CLASS(insn->code)) { + case BPF_LDX: + if (BPF_MODE(insn->code) == BPF_PROBE_MEM32 || + BPF_MODE(insn->code) == BPF_PROBE_MEM32SX) + return true; + break; + case BPF_ST: + if (BPF_MODE(insn->code) == BPF_PROBE_MEM32) + return true; + break; + case BPF_STX: + if (BPF_MODE(insn->code) == BPF_PROBE_MEM32 || + BPF_MODE(insn->code) == BPF_PROBE_ATOMIC) + return true; + break; + case BPF_ALU64: + if (BPF_OP(insn->code) == BPF_MOV && BPF_SRC(insn->code) == BPF_X && + insn->off == BPF_ADDR_SPACE_CAST) + return true; + break; + case BPF_JMP: + if (!bpf_pseudo_kfunc_call(insn)) + break; + m = bpf_jit_find_kfunc_model(prog, insn); + if (!m) + break; + for (j = 0; j < m->nr_args; j++) + if (m->arg_flags[j] & BTF_FMODEL_ARENA_ARG) + return true; + break; + } + } + return false; +} + static struct bpf_prog *bpf_prog_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog) { #ifdef CONFIG_BPF_JIT struct bpf_prog *orig_prog; + prog->aux->uses_arena_insns = bpf_prog_has_arena_insns(prog); if (!bpf_prog_need_blind(prog)) return bpf_int_jit_compile(env, prog); @@ -2742,7 +2802,6 @@ struct bpf_prog *bpf_prog_select_runtime(struct bpf_prog *fp, int *err) { return __bpf_prog_select_runtime(NULL, fp, err); } -EXPORT_SYMBOL_GPL(bpf_prog_select_runtime); static unsigned int __bpf_prog_ret1(const void *ctx, const struct bpf_insn *insn) @@ -2803,6 +2862,11 @@ void bpf_prog_array_free_sleepable(struct bpf_prog_array *progs) call_rcu_tasks_trace(&progs->rcu, __bpf_prog_array_free_sleepable_cb); } +struct bpf_prog *bpf_prog_dummy(void) +{ + return &dummy_bpf_prog.prog; +} + int bpf_prog_array_length(struct bpf_prog_array *array) { struct bpf_prog_array_item *item; @@ -3035,13 +3099,18 @@ void __bpf_free_used_maps(struct bpf_prog_aux *aux, bool sleepable; u32 i; - sleepable = aux->prog->sleepable; + sleepable = bpf_prog_runs_sleepable(aux->prog); for (i = 0; i < len; i++) { map = used_maps[i]; if (map->ops->map_poke_untrack) map->ops->map_poke_untrack(map, aux); if (sleepable) atomic64_dec(&map->sleepable_refcnt); + /* + * The program that didn't load is not a user of the map. libbpf + * loads the program again to get the log of the verifier. + */ + cmpxchg(&map->user, (unsigned long)aux, 0); bpf_map_put(map); } } @@ -3073,31 +3142,12 @@ static void bpf_free_used_btfs(struct bpf_prog_aux *aux) kfree(aux->used_btfs); } -static void bpf_prog_free_deferred(struct work_struct *work) +static void bpf_prog_free_images(struct work_struct *work) { struct bpf_prog_aux *aux; int i; - aux = container_of(work, struct bpf_prog_aux, work); -#ifdef CONFIG_BPF_SYSCALL - bpf_free_kfunc_btf_tab(aux->kfunc_btf_tab); - bpf_prog_stream_free(aux->prog); -#endif -#ifdef CONFIG_CGROUP_BPF - if (aux->cgroup_atype != CGROUP_BPF_ATTACH_TYPE_INVALID) - bpf_cgroup_atype_put(aux->cgroup_atype); -#endif - bpf_free_used_maps(aux); - bpf_free_used_btfs(aux); - bpf_prog_disassoc_struct_ops(aux->prog); - if (bpf_prog_is_dev_bound(aux)) - bpf_prog_dev_bound_destroy(aux->prog); -#ifdef CONFIG_PERF_EVENTS - if (aux->prog->has_callchain_buf) - put_callchain_buffers(); -#endif - if (aux->dst_trampoline) - bpf_trampoline_put(aux->dst_trampoline); + aux = container_of(to_rcu_work(work), struct bpf_prog_aux, rwork); for (i = 0; i < aux->real_func_cnt; i++) { /* We can just unlink the subprog poke descriptor table as * it was originally linked to the main program and is also @@ -3114,6 +3164,44 @@ static void bpf_prog_free_deferred(struct work_struct *work) } } +static void bpf_prog_free_deferred(struct work_struct *work) +{ + struct bpf_prog_aux *aux; + bool ksyms; + + aux = container_of(work, struct bpf_prog_aux, work); + /* + * A prog that still runs when its last reference goes needs its symbols, + * by which its exception table is found. It has ended by now. + */ + ksyms = aux->real_func_cnt || !list_empty(&aux->ksym.lnode); + bpf_prog_kallsyms_del_all(aux->prog); +#ifdef CONFIG_BPF_SYSCALL + bpf_free_kfunc_btf_tab(aux->kfunc_btf_tab); +#endif +#ifdef CONFIG_CGROUP_BPF + if (aux->cgroup_atype != CGROUP_BPF_ATTACH_TYPE_INVALID) + bpf_cgroup_atype_put(aux->cgroup_atype); +#endif + bpf_free_used_maps(aux); + bpf_free_used_btfs(aux); + bpf_prog_disassoc_struct_ops(aux->prog); + if (bpf_prog_is_dev_bound(aux)) + bpf_prog_dev_bound_destroy(aux->prog); +#ifdef CONFIG_PERF_EVENTS + if (aux->prog->has_callchain_buf) + put_callchain_buffers(); +#endif + if (aux->dst_trampoline) + bpf_trampoline_put(aux->dst_trampoline); + /* The symbols are looked up under rcu_read_lock() */ + INIT_RCU_WORK(&aux->rwork, bpf_prog_free_images); + if (ksyms) + queue_rcu_work(system_percpu_wq, &aux->rwork); + else + bpf_prog_free_images(&aux->rwork.work); +} + void bpf_prog_free(struct bpf_prog *fp) { struct bpf_prog_aux *aux = fp->aux; @@ -3124,7 +3212,6 @@ void bpf_prog_free(struct bpf_prog *fp) INIT_WORK(&aux->work, bpf_prog_free_deferred); schedule_work(&aux->work); } -EXPORT_SYMBOL_GPL(bpf_prog_free); /* RNG for unprivileged user space with separated state from prandom_u32(). */ static DEFINE_PER_CPU(struct rnd_state, bpf_user_rnd_state); @@ -3301,6 +3388,117 @@ bool __weak bpf_jit_supports_kfunc_call(void) return false; } +/* Return TRUE if the JIT backend supports callx (indirect call) instruction. */ +bool __weak bpf_jit_supports_callx(void) +{ + return false; +} + +bool __weak bpf_jit_supports_kfunc_ret_reg_pair(void) +{ + return false; +} + +/* unwind_goto and unwind_exit */ +bool __weak bpf_jit_supports_unwind(void) +{ + return false; +} + +/* + * How this arch places a by-value kfunc argument, or NULL for one that has + * not opted in and so only takes an argument of a single eightbyte, which + * every convention places in slot order. + */ +const struct bpf_jit_arg_abi * __weak bpf_jit_arg_abi(void) +{ + return NULL; +} + +u32 bpf_jit_place_args(const struct bpf_jit_arg_abi *abi, + const struct btf_func_model *fm, u8 *pos_of_slot) +{ + u32 i, k, nslots, slot = 0, nregs_used = 0, stack_off = 0; + bool on_stack = false; + + for (i = 0; i < fm->nr_args; i++) { + bool align16 = fm->arg_flags[i] & BTF_FMODEL_ALIGN16_ARG; + u32 pos; + + nslots = btf_func_model_arg_slots(fm, i); + + if (align16 && abi->even_reg_align) + nregs_used = round_up(nregs_used, 2); + + if (!on_stack && nregs_used + nslots <= abi->nr_arg_regs) { + /* wholly in registers */ + pos = nregs_used; + nregs_used += nslots; + } else if (!on_stack && abi->split_at_boundary) { + /* the last registers hold what fits, the stack the rest */ + pos = nregs_used; + stack_off = (nregs_used + nslots - abi->nr_arg_regs) * BPF_REG_SIZE; + nregs_used = abi->nr_arg_regs; + on_stack = true; + } else { + /* wholly on the stack */ + if (align16 && abi->even_stack_align) + stack_off = round_up(stack_off, 2 * BPF_REG_SIZE); + pos = abi->nr_arg_regs + stack_off / BPF_REG_SIZE; + stack_off += nslots * BPF_REG_SIZE; + if (!abi->backfill_after_stack) + on_stack = true; + } + + for (k = 0; k < nslots; k++) + pos_of_slot[slot + k] = pos + k; + slot += nslots; + } + + return slot; +} + +u32 bpf_jit_plan_arg_moves(const struct bpf_jit_arg_abi *abi, + const struct btf_func_model *fm, + struct bpf_jit_arg_move *moves) +{ + u8 pos_of_slot[MAX_BPF_FUNC_ARG_SLOTS]; + u32 nslots, n = 0, s, back; + + nslots = bpf_jit_place_args(abi, fm, pos_of_slot); + back = nslots; + + /* + * An argument is two eightbytes at most, so it frees one register at + * most and only one argument ever moves down. Its destination is + * still in use, so carry it in the scratch. Only a lower slot can + * take the one it leaves, so the walk reaches it first. + */ + for (s = nslots; s > 0; s--) { + u8 slot = s - 1, pos = pos_of_slot[slot]; + + if (pos == slot) + continue; + + if (pos < slot) { + moves[n].dst = BPF_JIT_ARG_TMP; + back = slot; + } else { + moves[n].dst = pos; + } + moves[n].src = slot; + n++; + } + + if (back < nslots) { + moves[n].dst = pos_of_slot[back]; + moves[n].src = BPF_JIT_ARG_TMP; + n++; + } + + return n; +} + bool __weak bpf_jit_supports_stack_args(void) { return false; @@ -3321,6 +3519,12 @@ bool __weak bpf_jit_supports_arena(void) return false; } +/* Whether JIT takes BPF_REG_AX as the address of arena access */ +bool __weak bpf_jit_supports_arena_scalar(void) +{ + return false; +} + bool __weak bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena) { return false; @@ -3386,6 +3590,19 @@ bool __weak bpf_jit_supports_private_stack(void) return false; } +/* + * Return TRUE if the JIT lays out frames of up to MAX_BPF_STACK_JIT bytes. + * Its prologue, epilogue and tail call sequences must encode such frame + * sizes and a private stack must be sized from the program's depth. The + * budget is only granted alongside bpf_jit_supports_subprog_tailcalls(), + * whose tail calls land before the target sets up its own frame; see + * bpf_prog_stack_limit(). + */ +bool __weak bpf_jit_supports_large_stack(void) +{ + return false; +} + void __weak arch_bpf_stack_walk(bool (*consume_fn)(void *cookie, u64 ip, u64 sp, u64 bp), void *cookie) { } diff --git a/kernel/bpf/cpumap.c b/kernel/bpf/cpumap.c index 5e59ab896f05..eb87131b1bef 100644 --- a/kernel/bpf/cpumap.c +++ b/kernel/bpf/cpumap.c @@ -45,6 +45,7 @@ */ #define CPU_MAP_BULK_SIZE 8 /* 8 == one cacheline on 64-bit archs */ +#define CPU_MAP_HELD_DEVS 2048 struct bpf_cpu_map_entry; struct bpf_cpu_map; @@ -74,6 +75,13 @@ struct bpf_cpu_map_entry { struct completion kthread_running; struct rcu_work free_work; + + /* + * Every frame and skb in the queue holds a reference to its device. + * The kthread keeps it until nothing is left in GRO. + */ + struct net_device *held[CPU_MAP_HELD_DEVS]; + u32 nr_held; }; struct bpf_cpu_map { @@ -129,9 +137,11 @@ static void __cpu_map_ring_cleanup(struct ptr_ring *ring) WARN_ON_ONCE(1); if (unlikely(__ptr_test_bit(0, &ptr))) { __ptr_clear_bit(0, &ptr); + dev_put(((struct sk_buff *)ptr)->dev); kfree_skb(ptr); continue; } + dev_put(((struct xdp_frame *)ptr)->dev_rx); xdp_return_frame(ptr); } } @@ -311,7 +321,7 @@ static int cpu_map_kthread_run(void *data) void *frames[CPUMAP_BATCH]; void *skbs[CPUMAP_BATCH]; u32 i, n, m; - bool empty; + bool empty, full; /* Release CPU reschedule checks */ if (__ptr_ring_empty(rcpu->queue)) { @@ -344,10 +354,12 @@ static int cpu_map_kthread_run(void *data) struct sk_buff *skb = f; __ptr_clear_bit(0, &skb); + rcpu->held[rcpu->nr_held++] = skb->dev; skbs[ret.skb_n++] = skb; continue; } + rcpu->held[rcpu->nr_held++] = ((struct xdp_frame *)f)->dev_rx; frames[ret.xdp_n++] = f; page = virt_to_page(f); @@ -398,10 +410,13 @@ static int cpu_map_kthread_run(void *data) /* Flush either every 64 packets or in case of empty ring */ packets += n; empty = __ptr_ring_empty(rcpu->queue); - if (packets >= NAPI_POLL_WEIGHT || empty) { - cpu_map_gro_flush(rcpu, empty); + full = empty || rcpu->nr_held > CPU_MAP_HELD_DEVS - CPUMAP_BATCH; + if (packets >= NAPI_POLL_WEIGHT || full) { + cpu_map_gro_flush(rcpu, full); packets = 0; } + while (full && rcpu->nr_held) + dev_put(rcpu->held[--rcpu->nr_held]); local_bh_enable(); /* resched point, may call do_softirq() */ } @@ -738,12 +753,14 @@ static void bq_flush_to_queue(struct xdp_bulk_queue *bq) for (i = 0; i < bq->count; i++) { struct xdp_frame *xdpf = bq->q[i]; - int err; - err = __ptr_ring_produce(q, xdpf); - if (err) { + if (__ptr_ring_full(q)) { drops++; xdp_return_frame_rx_napi(xdpf); + } else { + /* The device cannot go before the flush, from now on it can */ + dev_hold(xdpf->dev_rx); + __ptr_ring_produce(q, xdpf); } processed++; } @@ -807,11 +824,15 @@ int cpu_map_generic_redirect(struct bpf_cpu_map_entry *rcpu, __skb_pull(skb, skb->mac_len); skb_set_redirected(skb, false); + dev_hold(skb->dev); __ptr_set_bit(0, &skb); ret = ptr_ring_produce(rcpu->queue, skb); - if (ret < 0) + if (ret < 0) { + __ptr_clear_bit(0, &skb); + dev_put(skb->dev); goto trace; + } wake_up_process(rcpu->kthread); trace: diff --git a/kernel/bpf/crypto.c b/kernel/bpf/crypto.c index 3f3fe2450fc6..69bc29d0a4d4 100644 --- a/kernel/bpf/crypto.c +++ b/kernel/bpf/crypto.c @@ -1,19 +1,13 @@ // SPDX-License-Identifier: GPL-2.0-only /* Copyright (c) 2024 Meta, Inc */ #include -#include #include #include #include #include -#include #include -#include - -struct bpf_crypto_type_list { - const struct bpf_crypto_type *type; - struct list_head list; -}; +#include +#include /* BPF crypto initialization parameters struct */ /** @@ -36,94 +30,53 @@ struct bpf_crypto_params { u32 authsize; }; -static LIST_HEAD(bpf_crypto_types); -static DECLARE_RWSEM(bpf_crypto_types_sem); +enum bpf_crypto_algo_id { + BPF_ALGO_AES_CBC, + BPF_ALGO_AES_ECB, +}; + +static const struct { + const char *type_name; + const char *algo_name; + enum bpf_crypto_algo_id algo; +} bpf_crypto_algos[] = { + { "skcipher", "cbc(aes)", BPF_ALGO_AES_CBC }, + { "skcipher", "ecb(aes)", BPF_ALGO_AES_ECB }, +}; + +static bool bpf_crypto_find_algo(const struct bpf_crypto_params *params, + enum bpf_crypto_algo_id *id_ret) +{ + for (size_t i = 0; i < ARRAY_SIZE(bpf_crypto_algos); i++) { + if (strncmp(bpf_crypto_algos[i].type_name, params->type, + sizeof(params->type)) == 0 && + strncmp(bpf_crypto_algos[i].algo_name, params->algo, + sizeof(params->algo)) == 0) { + *id_ret = bpf_crypto_algos[i].algo; + return true; + } + } + return false; +} /** * struct bpf_crypto_ctx - refcounted BPF crypto context structure - * @type: The pointer to bpf crypto type - * @tfm: The pointer to instance of crypto API struct. - * @siv_len: Size of IV and state storage for cipher + * @key: The crypto key + * @algo: The crypto algorithm ID * @rcu: The RCU head used to free the crypto context with RCU safety. * @usage: Object reference counter. When the refcount goes to 0, the * memory is released back to the BPF allocator, which provides * RCU safety. */ struct bpf_crypto_ctx { - const struct bpf_crypto_type *type; - void *tfm; - u32 siv_len; + union { + struct aes_key aes; + } key; + enum bpf_crypto_algo_id algo; struct rcu_head rcu; refcount_t usage; }; -int bpf_crypto_register_type(const struct bpf_crypto_type *type) -{ - struct bpf_crypto_type_list *node; - int err = -EBUSY; - - down_write(&bpf_crypto_types_sem); - list_for_each_entry(node, &bpf_crypto_types, list) { - if (!strcmp(node->type->name, type->name)) - goto unlock; - } - - node = kmalloc_obj(*node); - err = -ENOMEM; - if (!node) - goto unlock; - - node->type = type; - list_add(&node->list, &bpf_crypto_types); - err = 0; - -unlock: - up_write(&bpf_crypto_types_sem); - - return err; -} -EXPORT_SYMBOL_GPL(bpf_crypto_register_type); - -int bpf_crypto_unregister_type(const struct bpf_crypto_type *type) -{ - struct bpf_crypto_type_list *node; - int err = -ENOENT; - - down_write(&bpf_crypto_types_sem); - list_for_each_entry(node, &bpf_crypto_types, list) { - if (strcmp(node->type->name, type->name)) - continue; - - list_del(&node->list); - kfree(node); - err = 0; - break; - } - up_write(&bpf_crypto_types_sem); - - return err; -} -EXPORT_SYMBOL_GPL(bpf_crypto_unregister_type); - -static const struct bpf_crypto_type *bpf_crypto_get_type(const char *name) -{ - const struct bpf_crypto_type *type = ERR_PTR(-ENOENT); - struct bpf_crypto_type_list *node; - - down_read(&bpf_crypto_types_sem); - list_for_each_entry(node, &bpf_crypto_types, list) { - if (strcmp(node->type->name, name)) - continue; - - if (try_module_get(node->type->owner)) - type = node->type; - break; - } - up_read(&bpf_crypto_types_sem); - - return type; -} - __bpf_kfunc_start_defs(); /** @@ -132,93 +85,70 @@ __bpf_kfunc_start_defs(); * Allocates a crypto context that can be used, acquired, and released by * a BPF program. The crypto context returned by this function must either * be embedded in a map as a kptr, or freed with bpf_crypto_ctx_release(). - * As crypto API functions use GFP_KERNEL allocations, this function can - * only be used in sleepable BPF programs. + * As this uses a GFP_KERNEL allocation, this function can only be used in + * sleepable BPF programs. * * bpf_crypto_ctx_create() allocates memory for crypto context. * It may return NULL if no memory is available. * @params: pointer to struct bpf_crypto_params which contains all the * details needed to initialise crypto context. - * @params__sz: size of steuct bpf_crypto_params usef by bpf program - * @err: integer to store error code when NULL is returned. + * @params__sz: size of struct bpf_crypto_params used by bpf program + * @err_ret: integer to store error code when NULL is returned. */ __bpf_kfunc struct bpf_crypto_ctx * bpf_crypto_ctx_create(const struct bpf_crypto_params *params, u32 params__sz, - int *err) + int *err_ret) { - const struct bpf_crypto_type *type; struct bpf_crypto_ctx *ctx; + int err; if (!params || params__sz != sizeof(struct bpf_crypto_params) || params->reserved[0] || params->reserved[1]) { - *err = -EINVAL; + *err_ret = -EINVAL; return NULL; } - type = bpf_crypto_get_type(params->type); - if (IS_ERR(type)) { - *err = PTR_ERR(type); - return NULL; - } - - if (!type->has_algo(params->algo)) { - *err = -EOPNOTSUPP; - goto err_module_put; - } - - if (!!params->authsize ^ !!type->setauthsize) { - *err = -EOPNOTSUPP; - goto err_module_put; - } - if (!params->key_len || params->key_len > sizeof(params->key)) { - *err = -EINVAL; - goto err_module_put; + *err_ret = -EINVAL; + return NULL; } ctx = kzalloc_obj(*ctx); if (!ctx) { - *err = -ENOMEM; - goto err_module_put; + *err_ret = -ENOMEM; + return NULL; } - ctx->type = type; - ctx->tfm = type->alloc_tfm(params->algo); - if (IS_ERR(ctx->tfm)) { - *err = PTR_ERR(ctx->tfm); - goto err_free_ctx; + if (!bpf_crypto_find_algo(params, &ctx->algo)) { + err = -EOPNOTSUPP; + goto out; } - if (params->authsize) { - *err = type->setauthsize(ctx->tfm, params->authsize); - if (*err) - goto err_free_tfm; + switch (ctx->algo) { + case BPF_ALGO_AES_CBC: + case BPF_ALGO_AES_ECB: + if (params->authsize) + err = -EOPNOTSUPP; + else + err = aes_preparekey(&ctx->key.aes, params->key, + params->key_len); + break; + default: + WARN_ON_ONCE(1); + err = -EOPNOTSUPP; + break; } - *err = type->setkey(ctx->tfm, params->key, params->key_len); - if (*err) - goto err_free_tfm; - - if (type->get_flags(ctx->tfm) & CRYPTO_TFM_NEED_KEY) { - *err = -EINVAL; - goto err_free_tfm; +out: + if (err) { + kfree_sensitive(ctx); + *err_ret = err; + return NULL; } - - ctx->siv_len = type->ivsize(ctx->tfm) + type->statesize(ctx->tfm); - refcount_set(&ctx->usage, 1); - + *err_ret = 0; return ctx; - -err_free_tfm: - type->free_tfm(ctx->tfm); -err_free_ctx: - kfree(ctx); -err_module_put: - module_put(type->owner); - - return NULL; } static void crypto_free_cb(struct rcu_head *head) @@ -226,9 +156,7 @@ static void crypto_free_cb(struct rcu_head *head) struct bpf_crypto_ctx *ctx; ctx = container_of(head, struct bpf_crypto_ctx, rcu); - ctx->type->free_tfm(ctx->tfm); - module_put(ctx->type->owner); - kfree(ctx); + kfree_sensitive(ctx); } /** @@ -268,27 +196,53 @@ __bpf_kfunc void bpf_crypto_ctx_release_dtor(void *ctx) } CFI_NOSEAL(bpf_crypto_ctx_release_dtor); +static int bpf_aes_cbc_crypt(u8 *dst, u32 dst_len, const u8 *src, u32 src_len, + u8 *iv, u32 iv_len, + const struct bpf_crypto_ctx *ctx, bool decrypt) +{ + if (iv_len != AES_BLOCK_SIZE) + return -EINVAL; + if (src_len % AES_BLOCK_SIZE || dst_len < src_len) + return -EINVAL; + if (decrypt) + aes_cbc_decrypt(dst, src, src_len, iv, &ctx->key.aes); + else + aes_cbc_encrypt(dst, src, src_len, iv, &ctx->key.aes); + return 0; +} + +static int bpf_aes_ecb_crypt(u8 *dst, u32 dst_len, const u8 *src, u32 src_len, + u8 *iv, u32 iv_len, + const struct bpf_crypto_ctx *ctx, bool decrypt) +{ + if (iv_len != 0) + return -EINVAL; + if (src_len % AES_BLOCK_SIZE || dst_len < src_len) + return -EINVAL; + if (decrypt) + aes_ecb_decrypt(dst, src, src_len, &ctx->key.aes); + else + aes_ecb_encrypt(dst, src, src_len, &ctx->key.aes); + return 0; +} + static int bpf_crypto_crypt(const struct bpf_crypto_ctx *ctx, const struct bpf_dynptr_kern *src, const struct bpf_dynptr_kern *dst, - const struct bpf_dynptr_kern *siv, + const struct bpf_dynptr_kern *iv, bool decrypt) { - u32 src_len, dst_len, siv_len; + u32 src_len, dst_len, iv_len; const u8 *psrc; u8 *pdst, *piv; - int err; if (__bpf_dynptr_is_rdonly(dst)) return -EINVAL; - siv_len = siv ? __bpf_dynptr_size(siv) : 0; + iv_len = iv ? __bpf_dynptr_size(iv) : 0; src_len = __bpf_dynptr_size(src); dst_len = __bpf_dynptr_size(dst); - if (!src_len || !dst_len || src_len > dst_len) - return -EINVAL; - - if (siv_len != ctx->siv_len) + if (!src_len || !dst_len) return -EINVAL; psrc = __bpf_dynptr_data(src, src_len); @@ -298,14 +252,20 @@ static int bpf_crypto_crypt(const struct bpf_crypto_ctx *ctx, if (!pdst) return -EINVAL; - piv = siv_len ? __bpf_dynptr_data_rw(siv, siv_len) : NULL; - if (siv_len && !piv) + piv = iv_len ? __bpf_dynptr_data_rw(iv, iv_len) : NULL; + if (iv_len && !piv) return -EINVAL; - err = decrypt ? ctx->type->decrypt(ctx->tfm, psrc, pdst, src_len, piv) - : ctx->type->encrypt(ctx->tfm, psrc, pdst, src_len, piv); - - return err; + switch (ctx->algo) { + case BPF_ALGO_AES_CBC: + return bpf_aes_cbc_crypt(pdst, dst_len, psrc, src_len, piv, + iv_len, ctx, decrypt); + case BPF_ALGO_AES_ECB: + return bpf_aes_ecb_crypt(pdst, dst_len, psrc, src_len, piv, + iv_len, ctx, decrypt); + default: + return -EINVAL; + } } /** @@ -313,20 +273,20 @@ static int bpf_crypto_crypt(const struct bpf_crypto_ctx *ctx, * @ctx: The crypto context being used. The ctx must be a trusted pointer. * @src: bpf_dynptr to the encrypted data. Must be a trusted pointer. * @dst: bpf_dynptr to the buffer where to store the result. Must be a trusted pointer. - * @siv__nullable: bpf_dynptr to IV data and state data to be used by decryptor. May be NULL. + * @iv__nullable: bpf_dynptr to the initialization vector. May be NULL. * * Decrypts provided buffer using IV data and the crypto context. Crypto context must be configured. */ __bpf_kfunc int bpf_crypto_decrypt(struct bpf_crypto_ctx *ctx, const struct bpf_dynptr *src, const struct bpf_dynptr *dst, - const struct bpf_dynptr *siv__nullable) + const struct bpf_dynptr *iv__nullable) { const struct bpf_dynptr_kern *src_kern = (struct bpf_dynptr_kern *)src; const struct bpf_dynptr_kern *dst_kern = (struct bpf_dynptr_kern *)dst; - const struct bpf_dynptr_kern *siv_kern = (struct bpf_dynptr_kern *)siv__nullable; + const struct bpf_dynptr_kern *iv_kern = (struct bpf_dynptr_kern *)iv__nullable; - return bpf_crypto_crypt(ctx, src_kern, dst_kern, siv_kern, true); + return bpf_crypto_crypt(ctx, src_kern, dst_kern, iv_kern, true); } /** @@ -334,20 +294,20 @@ __bpf_kfunc int bpf_crypto_decrypt(struct bpf_crypto_ctx *ctx, * @ctx: The crypto context being used. The ctx must be a trusted pointer. * @src: bpf_dynptr to the plain data. Must be a trusted pointer. * @dst: bpf_dynptr to the buffer where to store the result. Must be a trusted pointer. - * @siv__nullable: bpf_dynptr to IV data and state data to be used by decryptor. May be NULL. + * @iv__nullable: bpf_dynptr to the initialization vector. May be NULL. * * Encrypts provided buffer using IV data and the crypto context. Crypto context must be configured. */ __bpf_kfunc int bpf_crypto_encrypt(struct bpf_crypto_ctx *ctx, const struct bpf_dynptr *src, const struct bpf_dynptr *dst, - const struct bpf_dynptr *siv__nullable) + const struct bpf_dynptr *iv__nullable) { const struct bpf_dynptr_kern *src_kern = (struct bpf_dynptr_kern *)src; const struct bpf_dynptr_kern *dst_kern = (struct bpf_dynptr_kern *)dst; - const struct bpf_dynptr_kern *siv_kern = (struct bpf_dynptr_kern *)siv__nullable; + const struct bpf_dynptr_kern *iv_kern = (struct bpf_dynptr_kern *)iv__nullable; - return bpf_crypto_crypt(ctx, src_kern, dst_kern, siv_kern, false); + return bpf_crypto_crypt(ctx, src_kern, dst_kern, iv_kern, false); } __bpf_kfunc_end_defs(); diff --git a/kernel/bpf/devmap.c b/kernel/bpf/devmap.c index dc7b859e8bbf..1b8f0d7f8455 100644 --- a/kernel/bpf/devmap.c +++ b/kernel/bpf/devmap.c @@ -350,6 +350,7 @@ static int dev_map_bpf_prog_run(struct bpf_prog *xdp_prog, xdp_convert_frame_to_buff(xdpf, &xdp); xdp.txq = &txq; + rxq.mem.type = xdpf->mem_type; xdp.rxq = &rxq; act = bpf_prog_run_xdp(xdp_prog, &xdp); diff --git a/kernel/bpf/diagnostics.c b/kernel/bpf/diagnostics.c index 8f64bfd9afee..b5204798231d 100644 --- a/kernel/bpf/diagnostics.c +++ b/kernel/bpf/diagnostics.c @@ -790,8 +790,9 @@ static void diag_print_insn_context(struct bpf_verifier_env *env, u32 insn_idx, } } -static void bpf_diag_source(struct bpf_verifier_env *env, u32 insn_idx, const char *label, - const char *fmt, ...) +static __printf(4, 5) void bpf_diag_source(struct bpf_verifier_env *env, + u32 insn_idx, const char *label, + const char *fmt, ...) { struct bpf_diag_scratch *scratch; struct bpf_linfo_source *source_lines; @@ -956,6 +957,37 @@ const char *bpf_diag_reg_type_plain(struct bpf_verifier_env *env, enum bpf_reg_t } } +const char *bpf_diag_arg_type_plain(enum bpf_arg_type type) +{ + switch (base_type(type)) { + case ARG_MEM_SIZE: + case ARG_CONST_MEM_SIZE: + return "an integer scalar length for this memory argument"; + case ARG_PTR_TO_CTX: + return "the original program context pointer or preserve it before modifying registers"; + case ARG_SCALAR: + case ARG_CONST_SCALAR: + case ARG_CONST_ALLOC_SIZE_OR_ZERO: + return "an integer scalar value for this argument, not a pointer or resource object"; + case ARG_PTR_TO_CONST_STR: + return "a constant string pointer that the verifier recognizes, such as a string stored in a read-only map value"; + case ARG_PTR_TO_DYNPTR: + return "the address of a stack dynptr object, or use a const dynptr pointer returned by the verifier-supported path"; + case ARG_PTR_TO_ALLOC_BTF_ID: + return "a pointer returned by the matching BPF object allocation path"; + case ARG_PTR_TO_REFCOUNTED_KPTR: + return "an owning or non-owning pointer to a BPF-managed object containing a bpf_refcount field"; + case ARG_PTR_TO_ITER: + return "the address of a stack iterator object for iterator new, next, and destroy calls"; + case ARG_PTR_TO_IRQ_FLAG: + return "the same stack slot used by bpf_local_irq_save() or bpf_res_spin_lock_irqsave()"; + case ARG_PTR_TO_CTX_OUT: + return "the attach hook's own output argument, loaded directly from the program context"; + default: + return "a value with one of the accepted pointer or scalar types for this call"; + } +} + static const char *diag_arg_ordinal(int argno) { switch (argno) { @@ -1083,16 +1115,17 @@ void bpf_diag_ctx_forbidden(struct bpf_verifier_env *env, u32 insn_idx, const char *constraint, *context; u32 depth; - if (env->cur_state->active_rcu_locks) - ctx_kind = BPF_DIAG_CONTEXT_RCU; - else if (env->cur_state->active_preempt_locks) - ctx_kind = BPF_DIAG_CONTEXT_PREEMPT; - else if (env->cur_state->active_irq_id) - ctx_kind = BPF_DIAG_CONTEXT_IRQ; - else if (env->cur_state->active_locks) - ctx_kind = BPF_DIAG_CONTEXT_LOCK; - else - ctx_kind = BPF_DIAG_CONTEXT_NONE; + ctx_kind = BPF_DIAG_CONTEXT_NONE; + if (env->cur_state->in_sleepable) { + if (env->cur_state->active_rcu_locks) + ctx_kind = BPF_DIAG_CONTEXT_RCU; + else if (env->cur_state->active_preempt_locks) + ctx_kind = BPF_DIAG_CONTEXT_PREEMPT; + else if (env->cur_state->active_irq_id) + ctx_kind = BPF_DIAG_CONTEXT_IRQ; + else if (env->cur_state->active_locks) + ctx_kind = BPF_DIAG_CONTEXT_LOCK; + } depth = diag_context_depth(env, ctx_kind); opts = (struct bpf_diag_history_opts) { @@ -1531,8 +1564,7 @@ static void diag_record_mod(struct bpf_verifier_env *env, u32 insn_idx, } else if (diag_mod_insn_origin(env, insn_idx, &target, &event.mod.origin)) { event.mod.origin_valid = true; } - if (old_reg && new_reg && - (reason == BPF_DIAG_MOD_WRITE || reason == BPF_DIAG_MOD_SPILL) && + if (old_reg && new_reg && reason == BPF_DIAG_MOD_WRITE && !memcmp(&event.mod.old, &event.mod.new, sizeof(event.mod.old)) && !event.mod.origin_valid && diag_mod_keeps_lineage(env, &event)) @@ -1564,9 +1596,9 @@ static struct bpf_reg_state *target_to_reg(struct bpf_verifier_env *env, return NULL; return &state->stack_arg_regs[target->stack_arg]; case BPF_DIAG_MOD_TARGET_STACK_SLOT: - if (target->spi >= state->allocated_stack / BPF_REG_SIZE) + if (target->spi >= bpf_stack_nr_slots(state)) return NULL; - return &state->stack[target->spi].spilled_ptr; + return &bpf_stack_slot(state, target->spi)->spilled_ptr; default: return NULL; } @@ -1582,7 +1614,7 @@ static bool reg_to_target(struct bpf_verifier_env *env, const struct bpf_reg_sta for (frame = 0; frame <= vstate->curframe; frame++) { struct bpf_func_state *state = vstate->frame[frame]; unsigned long start, end; - u32 nslots = state->allocated_stack / BPF_REG_SIZE; + u32 nslots = bpf_stack_nr_slots(state); int spi; start = (unsigned long)state->regs; @@ -1601,6 +1633,11 @@ static bool reg_to_target(struct bpf_verifier_env *env, const struct bpf_reg_sta return true; } + /* + * Map the pointer back to its slot by address, which relies on + * the slots forming one contiguous array as bpf_stack_slot() + * indexes it. + */ start = (unsigned long)state->stack; end = (unsigned long)(state->stack + nslots); if (nslots && addr >= start && addr < end) { diff --git a/kernel/bpf/diagnostics.h b/kernel/bpf/diagnostics.h index d1b79945008a..a4102fb049ec 100644 --- a/kernel/bpf/diagnostics.h +++ b/kernel/bpf/diagnostics.h @@ -51,6 +51,7 @@ const char *bpf_diag_vfmt(struct bpf_verifier_env *env, const char *fmt, va_list const char *bpf_diag_fmt(struct bpf_verifier_env *env, const char *fmt, ...) __printf(2, 3); const char *bpf_diag_fmt_btf_type(struct bpf_verifier_env *env, const struct btf *btf, u32 type_id); const char *bpf_diag_reg_type_plain(struct bpf_verifier_env *env, enum bpf_reg_type type); +const char *bpf_diag_arg_type_plain(enum bpf_arg_type type); u64 bpf_diag_event_log_save(struct bpf_verifier_env *env); void bpf_diag_event_log_restore(struct bpf_verifier_env *env, u64 log_pos); u32 bpf_diag_irq_depth(const struct bpf_verifier_state *state); diff --git a/kernel/bpf/disasm.c b/kernel/bpf/disasm.c index 3ce8d74b0e40..889fc074496f 100644 --- a/kernel/bpf/disasm.c +++ b/kernel/bpf/disasm.c @@ -350,7 +350,10 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs, if (opcode == BPF_CALL) { char tmp[64]; - if (insn->src_reg == BPF_PSEUDO_CALL) { + if (BPF_SRC(insn->code) == BPF_X) { + verbose(cbs->private_data, "(%02x) callx r%d", + insn->code, insn->dst_reg); + } else if (insn->src_reg == BPF_PSEUDO_CALL) { verbose(cbs->private_data, "(%02x) call pc%s", insn->code, __func_get_name(cbs, insn, @@ -372,11 +375,16 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs, insn->src_reg == BPF_MAY_GOTO) { verbose(cbs->private_data, "(%02x) may_goto pc%+d", insn->code, insn->off); + } else if (insn->code == (BPF_JMP | BPF_JCOND) && + insn->src_reg == BPF_UNWIND_GOTO) { + verbose(cbs->private_data, "(%02x) unwind_goto pc%+d", + insn->code, insn->off); } else if (insn->code == (BPF_JMP32 | BPF_JA)) { verbose(cbs->private_data, "(%02x) gotol pc%+d", insn->code, insn->imm); } else if (insn->code == (BPF_JMP | BPF_EXIT)) { - verbose(cbs->private_data, "(%02x) exit", insn->code); + verbose(cbs->private_data, "(%02x) %s", insn->code, + insn->src_reg == BPF_UNWIND_EXIT ? "unwind_exit" : "exit"); } else if (BPF_SRC(insn->code) == BPF_X) { verbose(cbs->private_data, "(%02x) if %c%d %s %c%d goto pc%+d", diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c index d6f83521fc78..f062ef7dd943 100644 --- a/kernel/bpf/fixups.c +++ b/kernel/bpf/fixups.c @@ -5,8 +5,6 @@ #include #include #include -#include -#include #include #include #include @@ -46,6 +44,31 @@ static bool is_addr_space_cast32(struct bpf_prog *prog, const struct bpf_insn *i return false; } +/* + * The insn accesses arena through a number. JITs add the base of arena + * to the register as it is, so the access goes through the low 32 bits of + * the number in BPF_REG_AX. Registers of the program are not changed: the + * number may be compared or stored later, and on another path the register + * may be PTR_TO_ARENA. + * + * Constant blinding leaves insns that use BPF_REG_AX alone, so the immediate + * of st through a number is not blinded. + */ +static int arena_scalar_access(const struct bpf_insn *insn, struct bpf_insn *buf) +{ + bool load = BPF_CLASS(insn->code) == BPF_LDX || bpf_atomic_is_load_acq(insn); + struct bpf_insn *patch = buf; + + *patch++ = BPF_MOV32_REG(BPF_REG_AX, load ? insn->src_reg : insn->dst_reg); + *patch = *insn; + if (load) + patch->src_reg = BPF_REG_AX; + else + patch->dst_reg = BPF_REG_AX; + patch++; + return patch - buf; +} + /* Return the regno defined by the insn, or -1. */ static int insn_def_regno(const struct bpf_insn *insn) { @@ -117,73 +140,26 @@ int bpf_insn_def32(struct bpf_prog *prog, struct bpf_insn *insn) return dst_reg; } -static int kfunc_desc_cmp_by_imm_off(const void *a, const void *b) -{ - const struct bpf_kfunc_desc *d0 = a; - const struct bpf_kfunc_desc *d1 = b; - - if (d0->imm != d1->imm) - return d0->imm < d1->imm ? -1 : 1; - if (d0->offset != d1->offset) - return d0->offset < d1->offset ? -1 : 1; - return 0; -} - const struct btf_func_model * bpf_jit_find_kfunc_model(const struct bpf_prog *prog, const struct bpf_insn *insn) { - const struct bpf_kfunc_desc desc = { - .imm = insn->imm, - .offset = insn->off, - }; const struct bpf_kfunc_desc *res; struct bpf_kfunc_desc_tab *tab; tab = prog->aux->kfunc_tab; - res = bsearch(&desc, tab->descs, tab->nr_descs, - sizeof(tab->descs[0]), kfunc_desc_cmp_by_imm_off); - - return res ? &res->func_model : NULL; -} - -static int set_kfunc_desc_imm(struct bpf_verifier_env *env, struct bpf_kfunc_desc *desc) -{ - unsigned long call_imm; + if (insn->off < 0 || insn->off >= tab->nr_descs) + return NULL; + res = &tab->descs[insn->off]; if (bpf_jit_supports_far_kfunc_call()) { - call_imm = desc->func_id; - } else { - call_imm = BPF_CALL_IMM(desc->addr); - /* Check whether the relative offset overflows desc->imm */ - if ((unsigned long)(s32)call_imm != call_imm) { - verbose(env, "address of kernel func_id %u is out of range\n", - desc->func_id); - return -EINVAL; - } - } - desc->imm = call_imm; - return 0; -} - -static int sort_kfunc_descs_by_imm_off(struct bpf_verifier_env *env) -{ - struct bpf_kfunc_desc_tab *tab; - int i, err; - - tab = env->prog->aux->kfunc_tab; - if (!tab) - return 0; - - for (i = 0; i < tab->nr_descs; i++) { - err = set_kfunc_desc_imm(env, &tab->descs[i]); - if (err) - return err; + if (res->func_id != insn->imm) + return NULL; + } else if ((s32)BPF_CALL_IMM(res->addr) != insn->imm) { + return NULL; } - sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]), - kfunc_desc_cmp_by_imm_off, NULL); - return 0; + return &res->func_model; } static int add_kfunc_in_insns(struct bpf_verifier_env *env, @@ -214,12 +190,19 @@ static int get_callee_stack_depth(struct bpf_verifier_env *env, } #endif +enum bpf_patch_mode { + BPF_PATCH_KEEP_TARGET, /* targets of 'off' keep naming 'off', the first new insn */ + BPF_PATCH_MOVE_TARGET, /* targets of 'off' follow the original insn to 'off + len - 1' */ +}; + /* single env->prog->insni[off] instruction was replaced with the range * insni[off, off + cnt). Adjust corresponding insn_aux_data by copying * [0, off) and [off, end) to new locations, so the patched range stays zero */ static void adjust_insn_aux_data(struct bpf_verifier_env *env, - struct bpf_prog *new_prog, u32 off, u32 cnt) + struct bpf_prog *new_prog, u32 off, u32 cnt, + struct bpf_insn *original_insn, + enum bpf_patch_mode mode) { struct bpf_insn_aux_data *data = env->insn_aux_data; struct bpf_insn *insn = new_prog->insnsi; @@ -233,8 +216,15 @@ static void adjust_insn_aux_data(struct bpf_verifier_env *env, */ data[off].zext_dst = bpf_insn_def32(new_prog, insn + off + cnt - 1) >= 0; - if (cnt == 1) + if (cnt == 1) { + /* + * A non-memory accessing insn could have been replaced by a + * memory accessing insn, systematically mark it for non-stack + * access + */ + data[off].non_stack_access = bpf_is_mem_insn(insn + off); return; + } prog_len = new_prog->len; env->insn_aux_data_len = prog_len; @@ -245,16 +235,39 @@ static void adjust_insn_aux_data(struct bpf_verifier_env *env, /* Expand insni[off]'s seen count to the patched range. */ data[i].seen = old_seen; data[i].zext_dst = bpf_insn_def32(new_prog, insn + i) >= 0; + /* + * In BPF_PATCH_MOVE_TARGET mode the original insn is the last + * slot by construction. + */ + if (mode == BPF_PATCH_KEEP_TARGET && + !memcmp(insn + i, original_insn, sizeof(struct bpf_insn))) { + data[i].non_stack_access = + data[off + cnt - 1].non_stack_access; + data[off + cnt - 1].non_stack_access = false; + } else if (bpf_is_mem_insn(insn + i)) { + data[i].non_stack_access = true; + } } + /* + * Last slot instruction could be a newly generated + * BPF_ST/BPF_LDX/BPF_STX, systematically mark it for non-stack access + * if it is not the original instruction, otherwise keep the + * original marking + */ + if (bpf_is_mem_insn(insn + off + cnt - 1) && + memcmp(insn + off + cnt - 1, original_insn, sizeof(struct bpf_insn))) + data[off + cnt - 1].non_stack_access = true; + /* * The indirect_target flag of the original instruction was moved to the last of the * new instructions by the above memmove and memset, but the indirect jump target is * actually the first instruction, so move it back. This also matches with the behavior * of bpf_insn_array_adjust(), which preserves xlated_off to point to the first new - * instruction. + * instruction. For BPF_PATCH_MOVE_TARGET the original instruction is the last one, + * so the flag already sits where needed. */ - if (data[off + cnt - 1].indirect_target) { + if (mode == BPF_PATCH_KEEP_TARGET && data[off + cnt - 1].indirect_target) { data[off].indirect_target = 1; data[off + cnt - 1].indirect_target = 0; } @@ -274,7 +287,7 @@ static void adjust_subprog_starts(struct bpf_verifier_env *env, u32 off, u32 len } } -static void adjust_insn_arrays(struct bpf_verifier_env *env, u32 off, u32 len) +static void adjust_insn_arrays(struct bpf_verifier_env *env, u32 first, u32 len) { int i; @@ -282,7 +295,7 @@ static void adjust_insn_arrays(struct bpf_verifier_env *env, u32 off, u32 len) return; for (i = 0; i < env->insn_array_map_cnt; i++) - bpf_insn_array_adjust(env->insn_array_maps[i], off, len); + bpf_insn_array_adjust(env->insn_array_maps[i], first, len); } static void adjust_insn_arrays_after_remove(struct bpf_verifier_env *env, u32 off, u32 len) @@ -293,7 +306,7 @@ static void adjust_insn_arrays_after_remove(struct bpf_verifier_env *env, u32 of bpf_insn_array_adjust_after_remove(env->insn_array_maps[i], off, len); } -static void adjust_poke_descs(struct bpf_prog *prog, u32 off, u32 len) +static void adjust_poke_descs(struct bpf_prog *prog, u32 first, u32 len) { struct bpf_jit_poke_descriptor *tab = prog->aux->poke_tab; int i, sz = prog->aux->size_poke_tab; @@ -301,7 +314,7 @@ static void adjust_poke_descs(struct bpf_prog *prog, u32 off, u32 len) for (i = 0; i < sz; i++) { desc = &tab[i]; - if (desc->insn_idx <= off) + if (desc->insn_idx < first) continue; desc->insn_idx += len - 1; } @@ -320,11 +333,15 @@ static bool bpf_rewrite_must_abort(void) return false; } -struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, - const struct bpf_insn *patch, u32 len) +static struct bpf_prog *__bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, + const struct bpf_insn *patch, u32 len, + enum bpf_patch_mode mode) { struct bpf_prog *new_prog; struct bpf_insn_aux_data *new_data = NULL; + /* The first slot whose targets move. */ + u32 first = mode == BPF_PATCH_MOVE_TARGET ? off : off + 1; + struct bpf_insn original_insn; if (bpf_rewrite_must_abort()) return NULL; @@ -340,6 +357,7 @@ struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, env->insn_aux_data = new_data; } + memcpy(&original_insn, env->prog->insnsi + off, sizeof(struct bpf_insn)); new_prog = bpf_patch_insn_single(env->prog, off, patch, len); if (IS_ERR(new_prog)) { if (PTR_ERR(new_prog) == -ERANGE) @@ -348,13 +366,58 @@ struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, env->insn_aux_data[off].orig_idx); return NULL; } - adjust_insn_aux_data(env, new_prog, off, len); + adjust_insn_aux_data(env, new_prog, off, len, &original_insn, mode); adjust_subprog_starts(env, off, len); - adjust_insn_arrays(env, off, len); - adjust_poke_descs(new_prog, off, len); + adjust_insn_arrays(env, first, len); + bpf_adjust_func_ptrs(env, off, len); + adjust_poke_descs(new_prog, first, len); return new_prog; } +struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, + const struct bpf_insn *patch, u32 len) +{ + return __bpf_patch_insn_data(env, off, patch, len, BPF_PATCH_KEEP_TARGET); +} + +/* + * insn was moved down by delta insns inside its own patch. Operands relative + * to the pc that point in front of the old position did not move with it. + */ +static int bpf_adj_moved_insn(struct bpf_insn *insn, u32 delta) +{ + u8 class = BPF_CLASS(insn->code), op = BPF_OP(insn->code); + s64 off = insn->imm, off_min = S32_MIN; + bool is_imm = true; + + if (bpf_pseudo_func(insn) || bpf_pseudo_call(insn)) { + /* subprog that started at the old position starts with the patch */ + if (off >= 0) + return 0; + } else if ((class == BPF_JMP || class == BPF_JMP32) && + op != BPF_CALL && op != BPF_EXIT) { + if (insn->code != (BPF_JMP32 | BPF_JA)) { + off = insn->off; + off_min = S16_MIN; + is_imm = false; + } + /* jump to itself stays */ + if (off >= -1) + return 0; + } else { + return 0; + } + + off -= delta; + if (off < off_min) + return -ERANGE; + if (is_imm) + insn->imm = off; + else + insn->off = off; + return 0; +} + /* * For all jmp insns in a given 'prog' that point to 'tgt_idx' insn adjust the * jump offset by 'delta'. @@ -440,6 +503,9 @@ static int adjust_subprog_starts_after_remove(struct bpf_verifier_env *env, * in adjust_btf_func() - no need to adjust */ } + /* The exception callback is live, so it cannot sit inside the removed range. */ + if (env->exception_callback_subprog >= j) + env->exception_callback_subprog -= j - i; } else { /* convert i from "first prog to remove" to "first to adjust" */ if (env->subprog_info[i].start == off) @@ -517,9 +583,7 @@ static int bpf_adj_linfo_after_remove(struct bpf_verifier_env *env, u32 off, return 0; } -/* - * Clean up dynamically allocated fields of aux data for instructions [start, ...] - */ +/* Clean up dynamically allocated fields of aux data for instructions [start, ...] */ void bpf_clear_insn_aux_data(struct bpf_verifier_env *env, int start, int len) { struct bpf_insn_aux_data *aux_data = env->insn_aux_data; @@ -527,9 +591,10 @@ void bpf_clear_insn_aux_data(struct bpf_verifier_env *env, int start, int len) int i; for (i = start; i < end; i++) { - if (aux_data[i].jt) { - kvfree(aux_data[i].jt); - aux_data[i].jt = NULL; + if (aux_data[i].loop) { + kvfree(aux_data[i].loop->exits); + kvfree(aux_data[i].loop); + aux_data[i].loop = NULL; } } } @@ -552,6 +617,9 @@ static int verifier_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt) if (err) return err; + /* before subprogs are adjusted, since it looks at them */ + bpf_adjust_func_ptrs_after_remove(env, off, cnt); + err = adjust_subprog_starts_after_remove(env, off, cnt); if (err) return err; @@ -590,6 +658,7 @@ void bpf_opt_hard_wire_dead_code_branches(struct bpf_verifier_env *env) { struct bpf_insn_aux_data *aux_data = env->insn_aux_data; struct bpf_insn ja = BPF_JMP_IMM(BPF_JA, 0, 0, 0); + struct bpf_insn trap = BPF_JMP_IMM(BPF_JA, 0, 0, -1); struct bpf_insn *insn = env->prog->insnsi; const int insn_cnt = env->prog->len; int i; @@ -598,6 +667,20 @@ void bpf_opt_hard_wire_dead_code_branches(struct bpf_verifier_env *env) if (!bpf_insn_is_cond_jump(insn->code)) continue; + if (bpf_is_unwind_goto(insn) && aux_data[i + 1 + insn->off].seen) { + /* + * The callee never exits. Its unwind_exit needs + * unwind_goto all the same, and the cpu predicts that it + * returns to what follows. Don't let the removal of dead + * code put something else there. + */ + if (!aux_data[i + 1].seen) { + memcpy(insn + 1, &trap, sizeof(trap)); + aux_data[i + 1].seen = env->pass_cnt; + } + continue; + } + if (!aux_data[i + 1].seen) ja.off = insn->off; else if (!aux_data[i + 1 + insn->off].seen) @@ -612,12 +695,41 @@ void bpf_opt_hard_wire_dead_code_branches(struct bpf_verifier_env *env) } } +/* + * The address of a function might be taken by the code that is not dead, + * while the function itself is dead code, since nothing calls it. Keep the first + * instruction of such function, so that the address remains valid and differs + * from the addresses of other functions. Make it a trap like + * sanitize_dead_code() does. + */ +static void keep_funcs_with_addr_taken(struct bpf_verifier_env *env) +{ + struct bpf_insn_aux_data *aux_data = env->insn_aux_data; + struct bpf_insn trap = BPF_JMP_IMM(BPF_JA, 0, 0, -1); + struct bpf_insn *insn = env->prog->insnsi; + int insn_cnt = env->prog->len; + int i, t; + + for (i = 0; i < insn_cnt; i++) { + if (!aux_data[i].seen || !bpf_pseudo_func(insn + i)) + continue; + t = i + insn[i].imm + 1; + if (aux_data[t].seen) + continue; + memcpy(insn + t, &trap, sizeof(trap)); + aux_data[t].zext_dst = false; + aux_data[t].seen = env->pass_cnt; + } +} + int bpf_opt_remove_dead_code(struct bpf_verifier_env *env) { struct bpf_insn_aux_data *aux_data = env->insn_aux_data; int insn_cnt = env->prog->len; int i, err; + keep_funcs_with_addr_taken(env); + for (i = 0; i < insn_cnt; i++) { int j; @@ -638,6 +750,7 @@ int bpf_opt_remove_dead_code(struct bpf_verifier_env *env) int bpf_opt_remove_nops(struct bpf_verifier_env *env) { + struct bpf_insn_aux_data *aux = env->insn_aux_data; struct bpf_insn *insn = env->prog->insnsi; int insn_cnt = env->prog->len; bool is_may_goto_0, is_ja; @@ -649,6 +762,8 @@ int bpf_opt_remove_nops(struct bpf_verifier_env *env) if (!is_may_goto_0 && !is_ja) continue; + if (aux[i].indirect_target) + continue; err = verifier_remove_insns(env, i, 1); if (err) @@ -758,11 +873,61 @@ int bpf_opt_subreg_zext_lo32_rnd_hi32(struct bpf_verifier_env *env, return 0; } +/* + * The prologue takes the context from R1, and it can clobber R2-R5. An + * extension gets the arguments of the function that it replaces. + */ +static bool ext_prog_takes_prologue(const struct bpf_verifier_env *env) +{ + const struct bpf_subprog_info *sub = &env->subprog_info[0]; + int i, cnt = min_t(int, sub->arg_slot_cnt, MAX_BPF_FUNC_REG_ARGS); + + if (!cnt || sub->args[0].arg_type != ARG_PTR_TO_CTX) + return false; + + for (i = 1; i < cnt; i++) + if (sub->args[i].arg_type != ARG_SCALAR) + return false; + + return true; +} + /* convert load instructions that access fields of a context type into a * sequence of instructions that access fields of the underlying structure: * struct __sk_buff -> struct sk_buff * struct bpf_sock_ops -> struct sock */ +/* + * A load through an __arena pointer member of a kernel struct reads the kernel + * address of arena memory. Append the conversion to the arena pointer form, + * (u32)(kaddr - kern_vm_start), that the struct_ops trampoline applies to + * __arena arguments. A __nullable member keeps NULL as NULL, tested on the + * full kernel address, and so does a probed load, which reads as 0 when it + * faults. Returns the number of insns in @buf, 0 when the insn loads no such + * member, or an error when it does so on some paths only. + */ +static int arena_member_fixup(struct bpf_verifier_env *env, int idx, struct bpf_insn *insn, + struct bpf_insn *buf) +{ + struct bpf_insn_aux_data *aux = &env->insn_aux_data[idx]; + int cnt = 0; + u32 base; + + if (!aux->arena_member) + return 0; + if (aux->non_arena_load) { + verbose(env, "an insn loads an arena pointer member on one path and something else on another\n"); + return -EINVAL; + } + + base = bpf_arena_get_kern_vm_start(env->prog->aux->arena); + buf[cnt++] = *insn; + if (aux->arena_member_nullable || BPF_MODE(insn->code) == BPF_PROBE_MEM) + buf[cnt++] = BPF_JMP_IMM(BPF_JEQ, insn->dst_reg, 0, 1); + buf[cnt++] = BPF_ALU32_IMM(BPF_SUB, insn->dst_reg, (s32)base); + return cnt; +} + int bpf_convert_ctx_accesses(struct bpf_verifier_env *env) { struct bpf_subprog_info *subprogs = env->subprog_info; @@ -791,7 +956,8 @@ int bpf_convert_ctx_accesses(struct bpf_verifier_env *env) insn_buf[cnt++] = BPF_STX_MEM(BPF_DW, BPF_REG_FP, BPF_REG_1, -subprogs[0].stack_depth); insn_buf[cnt++] = env->prog->insnsi[0]; - new_prog = bpf_patch_insn_data(env, 0, insn_buf, cnt); + new_prog = __bpf_patch_insn_data(env, 0, insn_buf, cnt, + BPF_PATCH_MOVE_TARGET); if (!new_prog) return -ENOMEM; env->prog = new_prog; @@ -814,7 +980,20 @@ int bpf_convert_ctx_accesses(struct bpf_verifier_env *env) verifier_bug(env, "prologue is too long"); return -EFAULT; } else if (cnt) { - new_prog = bpf_patch_insn_data(env, 0, insn_buf, cnt); + if (env->prog->type == BPF_PROG_TYPE_EXT && + !ext_prog_takes_prologue(env)) { + verbose(env, + "Extension writing packet data must take ctx and scalars only\n"); + return -EINVAL; + } + if (env->prog->type == BPF_PROG_TYPE_EXT && + bpf_check_ext_prologue(&env->log, + env->prog->aux->dst_prog, + env->prog->aux->attach_btf_id)) + return -EINVAL; + env->prog->aux->has_prologue = true; + new_prog = __bpf_patch_insn_data(env, 0, insn_buf, cnt, + BPF_PATCH_MOVE_TARGET); if (!new_prog) return -ENOMEM; @@ -845,7 +1024,13 @@ int bpf_convert_ctx_accesses(struct bpf_verifier_env *env) struct bpf_insn *patch = insn_buf; *patch++ = BPF_ST_NOSPEC(); - *patch++ = *insn; + *patch = *insn; + ret = bpf_adj_moved_insn(patch++, 1); + if (ret) { + verbose(env, "insn %d cannot be patched due to 16-bit range\n", + env->insn_aux_data[i + delta].orig_idx); + return ret; + } cnt = patch - insn_buf; new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); if (!new_prog) @@ -887,6 +1072,21 @@ int bpf_convert_ctx_accesses(struct bpf_verifier_env *env) insn->code = BPF_STX | BPF_PROBE_ATOMIC | BPF_SIZE(insn->code); env->prog->aux->num_exentries++; continue; + } else if (insn->code == (BPF_STX | BPF_ATOMIC | BPF_DW) && + insn->imm == BPF_LOAD_ACQ && + env->insn_aux_data[i + delta].arena_member) { + /* an acquire load of an __arena member is rewritten like a plain load */ + cnt = arena_member_fixup(env, i + delta, insn, insn_buf); + if (cnt < 0) + return cnt; + new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); + if (!new_prog) + return -ENOMEM; + + delta += cnt - 1; + env->prog = new_prog; + insn = new_prog->insnsi + i + delta; + continue; } else if (insn->code == (BPF_JMP | BPF_EXIT) && epilogue_cnt && i + delta < subprogs[1].start) { @@ -978,11 +1178,23 @@ int bpf_convert_ctx_accesses(struct bpf_verifier_env *env) insn->code = BPF_LDX | BPF_PROBE_MEMSX | BPF_SIZE(insn->code); env->prog->aux->num_exentries++; - continue; - } - if (verifier_bug_if(bpf_may_fault_on_deref(ptr_type), env, - "access to a fault prone pointer is not rewritten as a probed one")) + } else if (verifier_bug_if(bpf_may_fault_on_deref(ptr_type), env, + "access to a fault prone pointer is not rewritten as a probed one")) { return -EFAULT; + } + + cnt = arena_member_fixup(env, i + delta, insn, insn_buf); + if (cnt < 0) + return cnt; + if (!cnt) + continue; + new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); + if (!new_prog) + return -ENOMEM; + + delta += cnt - 1; + env->prog = new_prog; + insn = new_prog->insnsi + i + delta; continue; } @@ -1088,6 +1300,59 @@ static void bpf_restore_subprog_starts(struct bpf_verifier_env *env, u32 *orig_s env->subprog_info[env->subprog_cnt].start = env->prog->len; } +/* + * Replace the offsets of functions with their addresses in the maps of + * the program, see resolve_func_ptrs() in verifier.c. The program must be + * the only user of such map. From now on no other program can use it, see + * bpf_map_claim(). + */ +static int resolve_func_ptrs(struct bpf_verifier_env *env, struct bpf_prog *prog, + struct bpf_prog **func) +{ + unsigned long me = (unsigned long)prog->aux; + struct bpf_func_ptr *ptr; + int i, err, subprog; + u64 addr, new, *slot; + + for (i = 0; i < env->func_ptr_cnt; i++) { + ptr = &env->func_ptrs[i]; + + /* pointers are sorted by map */ + if ((!i || ptr->map != ptr[-1].map) && + cmpxchg(&ptr->map->user, me, me | BPF_MAP_USER_PATCHED) != me) { + verbose(env, "map '%s' is used by another program\n", ptr->map->name); + return -EBUSY; + } + + /* it's the address of the value of the map whatever the offset is */ + err = ptr->map->ops->map_direct_value_addr(ptr->map, &addr, 0); + if (verifier_bug_if(err, env, "no value of map '%s'", ptr->map->name)) + return -EFAULT; + slot = (u64 *)(unsigned long)(addr + ptr->map_off); + + /* + * The function that is dead code is removed and the pointer is + * NULL. The program may read it, but callx of it is dead code + * too, see check_func_callx(). + */ + new = 0; + if (ptr->xlated_off != BPF_FUNC_PTR_DELETED) { + subprog = bpf_find_subprog(env, ptr->xlated_off); + if (verifier_bug_if(!func || subprog <= 0, env, "no function at insn %u", + ptr->xlated_off)) + return -EFAULT; + new = (unsigned long)func[subprog]->bpf_func; + } + + /* the map is frozen and nothing else uses it */ + if (verifier_bug_if(*slot != (u64)ptr->orig_off * sizeof(struct bpf_insn), env, + "map '%s' offset %u changed", ptr->map->name, ptr->map_off)) + return -EFAULT; + WRITE_ONCE(*slot, new); + } + return 0; +} + static int jit_subprogs(struct bpf_verifier_env *env) { struct bpf_prog *prog = env->prog, **func, *tmp; @@ -1198,6 +1463,7 @@ static int jit_subprogs(struct bpf_verifier_env *env) func[i]->aux->jited_linfo = prog->aux->jited_linfo; func[i]->aux->linfo_idx = env->subprog_info[i].linfo_idx; func[i]->aux->arena = prog->aux->arena; + func[i]->aux->uses_arena_insns = bpf_prog_has_arena_insns(func[i]); func[i]->aux->used_maps = env->used_maps; func[i]->aux->used_map_cnt = env->used_map_cnt; num_exentries = 0; @@ -1216,12 +1482,15 @@ static int jit_subprogs(struct bpf_verifier_env *env) if (BPF_CLASS(insn->code) == BPF_STX && BPF_MODE(insn->code) == BPF_PROBE_ATOMIC) num_exentries++; + if (bpf_is_unwind_goto(insn)) + num_exentries++; } func[i]->aux->num_exentries = num_exentries; func[i]->aux->tail_call_reachable = env->subprog_info[i].tail_call_reachable; func[i]->aux->exception_cb = env->subprog_info[i].is_exception_cb; func[i]->aux->changes_pkt_data = env->subprog_info[i].changes_pkt_data; func[i]->aux->might_sleep = env->subprog_info[i].might_sleep; + func[i]->aux->can_unwind = env->subprog_info[i].can_unwind; func[i]->aux->token = prog->aux->token; if (!i) func[i]->aux->exception_boundary = env->seen_exception; @@ -1278,6 +1547,11 @@ static int jit_subprogs(struct bpf_verifier_env *env) cond_resched(); } + /* the addresses of all functions are final */ + err = resolve_func_ptrs(env, prog, func); + if (err) + goto out_free; + /* * Cleanup func[i]->aux fields which aren't required * or can become invalid in future @@ -1364,8 +1638,9 @@ int bpf_jit_subprogs(struct bpf_verifier_env *env) struct bpf_prog *prog, *orig_prog; u32 *orig_subprog_starts; + /* all functions that the maps point to are removed as dead code */ if (env->subprog_cnt <= 1) - return 0; + return resolve_func_ptrs(env, env->prog, NULL); prog = orig_prog = env->prog; if (bpf_prog_need_blind(prog)) { @@ -1522,11 +1797,37 @@ static int add_hidden_subprog(struct bpf_verifier_env *env, struct bpf_insn *pat env->prog = prog; info[cnt + 1].start = info[cnt].start; info[cnt].start = prog->len - len + 1; + /* The patch ends with the exit. */ + info[cnt].exit_idx = prog->len - 1; env->subprog_cnt++; env->hidden_subprog_cnt++; return 0; } +/* + * Expand may_goto: load the count from the stack, jump to the target of + * may_goto when it is zero, then the tail that updates the count. + * Use gotol when the target is too far for 16-bit offset of a conditional jump. + */ +static int may_goto_expand(struct bpf_insn *insn_buf, int off, int stack_off, + const struct bpf_insn *tail, int tail_cnt) +{ + int cnt = 0; + + /* Forward jump has to step over the tail */ + off = off >= 0 ? off + tail_cnt : off - 1; + + insn_buf[cnt++] = BPF_LDX_MEM(BPF_DW, BPF_REG_AX, BPF_REG_10, stack_off); + if (off == (s16)off) { + insn_buf[cnt++] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_AX, 0, off); + } else { + insn_buf[cnt++] = BPF_JMP_IMM(BPF_JNE, BPF_REG_AX, 0, 1); + insn_buf[cnt++] = BPF_JMP32_A(off >= 0 ? off : off - 1); + } + memcpy(insn_buf + cnt, tail, tail_cnt * sizeof(*tail)); + return cnt + tail_cnt; +} + /* Do various post-verification rewrites in a single program pass. * These rewrites simplify JIT and interpreter implementations. */ @@ -1579,6 +1880,20 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) /* Convert BPF_CLASS(insn->code) == BPF_ALU64 to 32-bit ALU */ insn->code = BPF_ALU | BPF_OP(insn->code) | BPF_SRC(insn->code); + if (env->insn_aux_data[i + delta].arena_scalar) { + cnt = arena_scalar_access(insn, insn_buf); + + new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); + if (!new_prog) + return -ENOMEM; + + delta += cnt - 1; + prog = new_prog; + env->prog = prog; + insn = prog->insnsi + i + delta; + goto next_insn; + } + /* Make sdiv/smod divide-by-minus-one exceptions impossible. */ if ((insn->code == (BPF_ALU64 | BPF_MOD | BPF_K) || insn->code == (BPF_ALU64 | BPF_DIV | BPF_K) || @@ -1804,6 +2119,18 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) if (bpf_is_may_goto_insn(insn) && bpf_jit_supports_timed_may_goto()) { int stack_off_cnt = -stack_depth - 16; + /* + * AX is used as an argument to pass in stack_off_cnt + * (to add to r10/fp), and also as the return value of + * the call to arch_bpf_timed_may_goto. + */ + struct bpf_insn tail[] = { + BPF_ALU64_IMM(BPF_SUB, BPF_REG_AX, 1), + BPF_JMP_IMM(BPF_JNE, BPF_REG_AX, 0, 2), + BPF_MOV64_IMM(BPF_REG_AX, stack_off_cnt), + BPF_EMIT_CALL(arch_bpf_timed_may_goto), + BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_AX, stack_off_cnt), + }; /* * Two 8 byte slots, depth-16 stores the count, and @@ -1820,22 +2147,8 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) * after subtraction, rinse and repeat. */ stack_depth_extra = 16; - insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_AX, BPF_REG_10, stack_off_cnt); - if (insn->off >= 0) - insn_buf[1] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_AX, 0, insn->off + 5); - else - insn_buf[1] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_AX, 0, insn->off - 1); - insn_buf[2] = BPF_ALU64_IMM(BPF_SUB, BPF_REG_AX, 1); - insn_buf[3] = BPF_JMP_IMM(BPF_JNE, BPF_REG_AX, 0, 2); - /* - * AX is used as an argument to pass in stack_off_cnt - * (to add to r10/fp), and also as the return value of - * the call to arch_bpf_timed_may_goto. - */ - insn_buf[4] = BPF_MOV64_IMM(BPF_REG_AX, stack_off_cnt); - insn_buf[5] = BPF_EMIT_CALL(arch_bpf_timed_may_goto); - insn_buf[6] = BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_AX, stack_off_cnt); - cnt = 7; + cnt = may_goto_expand(insn_buf, insn->off, stack_off_cnt, + tail, ARRAY_SIZE(tail)); new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); if (!new_prog) @@ -1847,16 +2160,14 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) goto next_insn; } else if (bpf_is_may_goto_insn(insn)) { int stack_off = -stack_depth - 8; + struct bpf_insn tail[] = { + BPF_ALU64_IMM(BPF_SUB, BPF_REG_AX, 1), + BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_AX, stack_off), + }; stack_depth_extra = 8; - insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_AX, BPF_REG_10, stack_off); - if (insn->off >= 0) - insn_buf[1] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_AX, 0, insn->off + 2); - else - insn_buf[1] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_AX, 0, insn->off - 1); - insn_buf[2] = BPF_ALU64_IMM(BPF_SUB, BPF_REG_AX, 1); - insn_buf[3] = BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_AX, stack_off); - cnt = 4; + cnt = may_goto_expand(insn_buf, insn->off, stack_off, + tail, ARRAY_SIZE(tail)); new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); if (!new_prog) @@ -1948,6 +2259,10 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) if (!bpf_allow_tail_call_in_subprogs(env)) prog->aux->stack_depth = MAX_BPF_STACK; prog->aux->max_pkt_offset = MAX_PACKET_OFF; + prog->aux->max_rdonly_access = U32_MAX; + prog->aux->max_rdwr_access = U32_MAX; + if (prog->type == BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE) + prog->aux->max_tp_access = U32_MAX; /* mark bpf_tail_call as different opcode to avoid * conditional branch in the interpreter for every normal @@ -2013,7 +2328,8 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) goto next_insn; } - if (insn->imm == BPF_FUNC_timer_set_callback) { + aux = &env->insn_aux_data[i + delta]; + if (aux->arg_prog) { /* The verifier will process callback_fn as many times as necessary * with different maps and the register states prepared by * set_timer_callback_state will be accurate. @@ -2028,7 +2344,7 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) * bpf_timer_set_callback-ed will return -EINVAL. */ struct bpf_insn ld_addrs[2] = { - BPF_LD_IMM64(BPF_REG_3, (long)prog->aux), + BPF_LD_IMM64(aux->arg_prog, (long)prog->aux), }; insn_buf[0] = ld_addrs[0]; @@ -2256,7 +2572,7 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) cnt = 2; } insn_buf[cnt++] = BPF_JMP32_REG(BPF_JGE, BPF_REG_2, BPF_REG_0, 6); - insn_buf[cnt++] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_2, 3); + insn_buf[cnt++] = BPF_ALU32_IMM(BPF_LSH, BPF_REG_2, 3); insn_buf[cnt++] = BPF_ALU64_REG(BPF_ADD, BPF_REG_2, BPF_REG_1); insn_buf[cnt++] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_2, 0); insn_buf[cnt++] = BPF_STX_MEM(BPF_DW, BPF_REG_3, BPF_REG_0, 0); @@ -2477,9 +2793,17 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) BPF_MAX_LOOPS); } /* Copy first actual insn to preserve it */ - insn_buf[cnt++] = env->prog->insnsi[subprog_start]; + insn_buf[cnt] = env->prog->insnsi[subprog_start]; + ret = bpf_adj_moved_insn(&insn_buf[cnt], cnt); + if (ret) { + verbose(env, "insn %d cannot be patched due to 16-bit range\n", + env->insn_aux_data[subprog_start].orig_idx); + return ret; + } + cnt++; - new_prog = bpf_patch_insn_data(env, subprog_start, insn_buf, cnt); + new_prog = __bpf_patch_insn_data(env, subprog_start, insn_buf, cnt, + BPF_PATCH_MOVE_TARGET); if (!new_prog) return -ENOMEM; env->prog = prog = new_prog; @@ -2509,10 +2833,6 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) } } - ret = sort_kfunc_descs_by_imm_off(env); - if (ret) - return ret; - return 0; } @@ -2686,4 +3006,3 @@ int bpf_remove_fastcall_spills_fills(struct bpf_verifier_env *env) return 0; } - diff --git a/kernel/bpf/hashtab.c b/kernel/bpf/hashtab.c index 13a2356c84cf..bf78e09f4d51 100644 --- a/kernel/bpf/hashtab.c +++ b/kernel/bpf/hashtab.c @@ -100,8 +100,10 @@ struct bpf_htab { struct percpu_counter pcount; atomic_t count; bool use_percpu_counter; + bool has_hash; u32 n_buckets; /* number of hash buckets */ u32 elem_size; /* size of each element in bytes */ + u32 elem_offset;/* offset of htab_elem in bytes */ u32 hashrnd; }; @@ -117,19 +119,46 @@ struct htab_elem { }; }; }; - union { - /* pointer to per-cpu pointer */ - void *ptr_to_pptr; - struct bpf_lru_node lru_node; - }; - u32 hash; char key[] __aligned(8); }; +struct htab_elem_hashed { + u32 hash __aligned(8); + struct htab_elem elem; +}; + +struct htab_elem_lru { + struct bpf_lru_node lru_node; + struct htab_elem_hashed; +}; + +/* + * Only for non-preallocated PCPU maps with small keys. Preallocated PCPU maps + * don't need ptr_to_pptr, and use htab_elem. + */ +struct htab_elem_pcpu { + /* pointer to per-cpu pointer */ + void *ptr_to_pptr; + struct htab_elem elem; +}; + +/* + * Only for non-preallocated PCPU maps with large keys. Preallocated PCPU maps + * don't need ptr_to_pptr, and use htab_elem_hashed. + */ +struct htab_elem_pcpu_hashed { + /* pointer to per-cpu pointer */ + void *ptr_to_pptr; + struct htab_elem_hashed; +}; + +static_assert(offsetof(struct htab_elem_pcpu_hashed, ptr_to_pptr) == + offsetof(struct htab_elem_pcpu, ptr_to_pptr)); + struct htab_btf_record { struct btf_record *record; struct btf *btf; - u32 key_size; + u32 value_offset; }; static inline bool htab_is_prealloc(const struct bpf_htab *htab) @@ -184,6 +213,71 @@ static inline bool is_fd_htab(const struct bpf_htab *htab) return htab->map.map_type == BPF_MAP_TYPE_HASH_OF_MAPS; } +/* + * u32 hash reads are always atomic. If the hash is elided, key comparisons + * must also be atomic to avoid false positive matches from torn key + * reads/writes on recycled elements. That is only possible when the + * key fits within a word. + */ +static __always_inline bool htab_key_size_needs_hash(u32 key_size) +{ + return key_size > sizeof(unsigned long); +} + +/* + * When hash is omitted, key reads and writes must be atomic to support lockless + * RCU readers. Zero-extend keys to the word size to support all sub-word key + * sizes, regardless of the size of the caller's key allocation. + */ +static __always_inline unsigned long htab_zero_extend_key(const void *key, + u32 key_size) +{ + unsigned long k = 0; + + memcpy(&k, key, min_t(u32, key_size, sizeof(k))); + return k; +} + +static bool htab_has_hash(const struct bpf_htab *htab) +{ + return htab->has_hash; +} + +static u32 htab_elem_hash(struct htab_elem *l) +{ + return container_of(l, struct htab_elem_hashed, elem)->hash; +} + +static void htab_elem_set_hash(struct htab_elem *l, u32 hash) +{ + container_of(l, struct htab_elem_hashed, elem)->hash = hash; +} + +static void *htab_elem_container(const struct bpf_htab *htab, struct htab_elem *l) +{ + return (void *)l - htab->elem_offset; +} + +static void *htab_elem_get_ptr_to_pptr(const struct bpf_htab *htab, struct htab_elem *l) +{ + struct htab_elem_pcpu *pcpu_elem = htab_elem_container(htab, l); + + return pcpu_elem->ptr_to_pptr; +} + +static void htab_elem_set_ptr_to_pptr(const struct bpf_htab *htab, struct htab_elem *l, + void *ptr) +{ + struct htab_elem_pcpu *pcpu_elem = htab_elem_container(htab, l); + + pcpu_elem->ptr_to_pptr = ptr; +} + +static struct bpf_lru_node *htab_elem_lru_node(struct htab_elem *l) +{ + return &container_of(l, struct htab_elem_lru, elem)->lru_node; +} + static inline void *htab_elem_value(struct htab_elem *l, u32 key_size) { return l->key + round_up(key_size, 8); @@ -207,7 +301,7 @@ static void *fd_htab_map_get_ptr(const struct bpf_map *map, struct htab_elem *l) static struct htab_elem *get_htab_elem(struct bpf_htab *htab, int i) { - return (struct htab_elem *) (htab->elems + i * (u64)htab->elem_size); + return htab->elems + i * (u64)htab->elem_size + htab->elem_offset; } /* Both percpu and fd htab support in-place update, so no need for @@ -301,16 +395,16 @@ static void htab_free_elems(struct bpf_htab *htab) * bucket_lock followed by lru_lock is not allowed. In such cases, * bucket_lock needs to be released first before acquiring lru_lock. */ -static struct htab_elem *prealloc_lru_pop(struct bpf_htab *htab, void *key, - u32 hash) +static struct htab_elem_lru *prealloc_lru_pop(struct bpf_htab *htab, void *key, + u32 hash) { struct bpf_lru_node *node = bpf_lru_pop_free(&htab->lru, hash); - struct htab_elem *l; + struct htab_elem_lru *l; if (node) { bpf_map_inc_elem_count(&htab->map); - l = container_of(node, struct htab_elem, lru_node); - memcpy(l->key, key, htab->map.key_size); + l = container_of(node, struct htab_elem_lru, lru_node); + memcpy(l->elem.key, key, htab->map.key_size); return l; } @@ -350,8 +444,8 @@ static int prealloc_init(struct bpf_htab *htab) if (htab_is_lru(htab)) err = bpf_lru_init(&htab->lru, htab->map.map_flags & BPF_F_NO_COMMON_LRU, - offsetof(struct htab_elem, hash) - - offsetof(struct htab_elem, lru_node), + offsetof(struct htab_elem_lru, hash) - + offsetof(struct htab_elem_lru, lru_node), htab_lru_map_delete_node, htab); else @@ -362,11 +456,12 @@ static int prealloc_init(struct bpf_htab *htab) if (htab_is_lru(htab)) bpf_lru_populate(&htab->lru, htab->elems, - offsetof(struct htab_elem, lru_node), + offsetof(struct htab_elem_lru, lru_node), htab->elem_size, num_entries); else pcpu_freelist_populate(&htab->freelist, - htab->elems + offsetof(struct htab_elem, fnode), + htab->elems + htab->elem_offset + + offsetof(struct htab_elem, fnode), htab->elem_size, num_entries); return 0; @@ -454,7 +549,7 @@ static int htab_map_alloc_check(union bpf_attr *attr) return -EINVAL; if ((u64)attr->key_size + attr->value_size >= KMALLOC_MAX_SIZE - - sizeof(struct htab_elem)) + sizeof(struct htab_elem_lru)) /* if key_size + value_size is bigger, the user space won't be * able to access the elements via bpf syscall. This check * also makes sure that the elem_size doesn't overflow and it's @@ -471,14 +566,11 @@ static int htab_map_alloc_check(union bpf_attr *attr) static void htab_mem_dtor(void *obj, void *ctx) { struct htab_btf_record *hrec = ctx; - struct htab_elem *elem = obj; - void *map_value; if (IS_ERR_OR_NULL(hrec->record)) return; - map_value = htab_elem_value(elem, hrec->key_size); - bpf_obj_free_fields(hrec->record, map_value); + bpf_obj_free_fields(hrec->record, obj + hrec->value_offset); } static void htab_pcpu_mem_dtor(void *obj, void *ctx) @@ -508,7 +600,7 @@ static void htab_dtor_ctx_free(void *ctx) } static int bpf_ma_set_dtor(struct bpf_map *map, struct bpf_mem_alloc *ma, - void (*dtor)(void *, void *)) + void (*dtor)(void *, void *), u32 value_offset) { struct htab_btf_record *hrec; int err; @@ -520,7 +612,7 @@ static int bpf_ma_set_dtor(struct bpf_map *map, struct bpf_mem_alloc *ma, hrec = kzalloc_obj(*hrec); if (!hrec) return -ENOMEM; - hrec->key_size = map->key_size; + hrec->value_offset = value_offset; hrec->record = btf_record_dup(map->record); if (IS_ERR(hrec->record)) { err = PTR_ERR(hrec->record); @@ -555,9 +647,12 @@ static int htab_map_check_btf(struct bpf_map *map, const struct btf *btf, * populated in htab_map_alloc(), so it will always appear as NULL. */ if (htab_is_percpu(htab)) - return bpf_ma_set_dtor(map, &htab->pcpu_ma, htab_pcpu_mem_dtor); + return bpf_ma_set_dtor(map, &htab->pcpu_ma, htab_pcpu_mem_dtor, 0); else - return bpf_ma_set_dtor(map, &htab->ma, htab_mem_dtor); + return bpf_ma_set_dtor(map, &htab->ma, htab_mem_dtor, + htab->elem_offset + + offsetof(struct htab_elem, key) + + round_up(map->key_size, 8)); } static struct bpf_map *htab_map_alloc(union bpf_attr *attr) @@ -580,6 +675,10 @@ static struct bpf_map *htab_map_alloc(union bpf_attr *attr) bpf_map_init_from_attr(&htab->map, attr); + /* Avoid hash memory use and comparisons where unnecessary. */ + htab->has_hash = htab_is_lru(htab) || + htab_key_size_needs_hash(htab->map.key_size); + if (percpu_lru) { /* ensure each CPU's lru list has >=1 elements. * since we are at it, make each lru list has the same @@ -601,7 +700,17 @@ static struct bpf_map *htab_map_alloc(union bpf_attr *attr) htab->n_buckets = roundup_pow_of_two(htab->map.max_entries); - htab->elem_size = sizeof(struct htab_elem) + + if (htab_is_lru(htab)) + htab->elem_offset = offsetof(struct htab_elem_lru, elem); + else if (percpu && !prealloc) + htab->elem_offset = htab_has_hash(htab) ? + offsetof(struct htab_elem_pcpu_hashed, elem) : + offsetof(struct htab_elem_pcpu, elem); + else if (htab_has_hash(htab)) + htab->elem_offset = offsetof(struct htab_elem_hashed, elem); + + htab->elem_size = htab->elem_offset + + sizeof(struct htab_elem) + round_up(htab->map.key_size, 8); if (percpu) htab->elem_size += sizeof(void *); @@ -692,11 +801,18 @@ static struct bpf_map *htab_map_alloc(union bpf_attr *attr) return ERR_PTR(err); } -static inline u32 htab_map_hash(const void *key, u32 key_len, u32 hashrnd) +static __always_inline u32 htab_map_hash(const void *key, u32 key_len, u32 hashrnd) { - if (likely(key_len % 4 == 0)) + const u32 *k = key; + + if (key_len == sizeof(u32)) + return jhash_1word(k[0], hashrnd); + else if (key_len == sizeof(u64)) + return jhash_2words(k[0], k[1], hashrnd); + else if (likely(key_len % 4 == 0)) return jhash2(key, key_len / 4, hashrnd); - return jhash(key, key_len, hashrnd); + else + return jhash(key, key_len, hashrnd); } static inline struct bucket *__select_bucket(struct bpf_htab *htab, u32 hash) @@ -709,35 +825,57 @@ static inline struct hlist_nulls_head *select_bucket(struct bpf_htab *htab, u32 return &__select_bucket(htab, hash)->head; } -/* this lookup function can only be called with bucket lock taken */ -static struct htab_elem *lookup_elem_raw(struct hlist_nulls_head *head, u32 hash, - void *key, u32 key_size) +static __always_inline struct hlist_nulls_node * +__lookup_elem_raw(struct hlist_nulls_head *head, u32 hash, void *key, + u32 key_size, bool elem_has_hash) { struct hlist_nulls_node *n; struct htab_elem *l; - hlist_nulls_for_each_entry_rcu(l, n, head, hash_node) - if (l->hash == hash && !memcmp(&l->key, key, key_size)) - return l; + if (elem_has_hash) { + hlist_nulls_for_each_entry_rcu(l, n, head, hash_node) + if (htab_elem_hash(l) == hash && + !memcmp(&l->key, key, key_size)) + break; + } else { + unsigned long k = htab_zero_extend_key(key, key_size); - return NULL; + hlist_nulls_for_each_entry_rcu(l, n, head, hash_node) + if (READ_ONCE(*(unsigned long *)l->key) == k) + break; + } + + return n; +} + +/* this lookup function can only be called with bucket lock taken */ +static __always_inline struct htab_elem * +lookup_elem_raw(struct hlist_nulls_head *head, u32 hash, void *key, + u32 key_size, bool elem_has_hash) +{ + struct hlist_nulls_node *n; + + n = __lookup_elem_raw(head, hash, key, key_size, elem_has_hash); + if (is_a_nulls(n)) + return NULL; + + return hlist_nulls_entry(n, struct htab_elem, hash_node); } /* can be called without bucket lock. it will repeat the loop in * the unlikely event when elements moved from one bucket into another * while link list is being walked */ -static struct htab_elem *lookup_nulls_elem_raw(struct hlist_nulls_head *head, - u32 hash, void *key, - u32 key_size, u32 n_buckets) +static __always_inline struct htab_elem * +lookup_nulls_elem_raw(struct hlist_nulls_head *head, u32 hash, void *key, + u32 key_size, bool elem_has_hash, u32 n_buckets) { struct hlist_nulls_node *n; - struct htab_elem *l; again: - hlist_nulls_for_each_entry_rcu(l, n, head, hash_node) - if (l->hash == hash && !memcmp(&l->key, key, key_size)) - return l; + n = __lookup_elem_raw(head, hash, key, key_size, elem_has_hash); + if (!is_a_nulls(n)) + return hlist_nulls_entry(n, struct htab_elem, hash_node); if (unlikely(get_nulls_value(n) != (hash & (n_buckets - 1)))) goto again; @@ -745,31 +883,56 @@ static struct htab_elem *lookup_nulls_elem_raw(struct hlist_nulls_head *head, return NULL; } -/* Called from syscall or from eBPF program directly, so - * arguments have to match bpf_map_lookup_elem() exactly. - * The return value is adjusted by BPF instructions - * in htab_map_gen_lookup(). - */ -static void *__htab_map_lookup_elem(struct bpf_map *map, void *key) +static __always_inline void *__htab_lookup(struct bpf_map *map, void *key, + u32 key_size, bool elem_has_hash) { struct bpf_htab *htab = container_of(map, struct bpf_htab, map); struct hlist_nulls_head *head; struct htab_elem *l; - u32 hash, key_size; + u32 hash; WARN_ON_ONCE(!bpf_rcu_lock_held()); - key_size = map->key_size; - hash = htab_map_hash(key, key_size, htab->hashrnd); head = select_bucket(htab, hash); - l = lookup_nulls_elem_raw(head, hash, key, key_size, htab->n_buckets); + l = lookup_nulls_elem_raw(head, hash, key, key_size, elem_has_hash, + htab->n_buckets); return l; } +/* + * Called from syscall or from eBPF program directly, so + * arguments have to match bpf_map_lookup_elem() exactly. + * The return value is adjusted by BPF instructions + * in htab_map_gen_lookup(). u32/u64 specializations + * allow utilizing compile-time key_size for optimization. + */ +static void *__htab_map_lookup_elem(struct bpf_map *map, void *key) +{ + struct bpf_htab *htab = container_of(map, struct bpf_htab, map); + + return __htab_lookup(map, key, map->key_size, htab_has_hash(htab)); +} + +/* + * Specialized lookups for htab_map_gen_lookup(). These are only used for + * BPF_MAP_TYPE_HASH (never LRU), so elem_has_hash depends only on key size. + */ +static void *__htab_map_lookup_elem_u32(struct bpf_map *map, void *key) +{ + return __htab_lookup(map, key, sizeof(u32), + htab_key_size_needs_hash(sizeof(u32))); +} + +static void *__htab_map_lookup_elem_u64(struct bpf_map *map, void *key) +{ + return __htab_lookup(map, key, sizeof(u64), + htab_key_size_needs_hash(sizeof(u64))); +} + static void *htab_map_lookup_elem(struct bpf_map *map, void *key) { struct htab_elem *l = __htab_map_lookup_elem(map, key); @@ -798,7 +961,16 @@ static int htab_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn_buf) BUILD_BUG_ON(!__same_type(&__htab_map_lookup_elem, (void *(*)(struct bpf_map *map, void *key))NULL)); - *insn++ = BPF_EMIT_CALL(__htab_map_lookup_elem); + BUILD_BUG_ON(!__same_type(&__htab_map_lookup_elem_u32, + (void *(*)(struct bpf_map *map, void *key))NULL)); + BUILD_BUG_ON(!__same_type(&__htab_map_lookup_elem_u64, + (void *(*)(struct bpf_map *map, void *key))NULL)); + if (map->key_size == sizeof(u32)) + *insn++ = BPF_EMIT_CALL(__htab_map_lookup_elem_u32); + else if (map->key_size == sizeof(u64)) + *insn++ = BPF_EMIT_CALL(__htab_map_lookup_elem_u64); + else + *insn++ = BPF_EMIT_CALL(__htab_map_lookup_elem); *insn++ = BPF_JMP_IMM(BPF_JEQ, ret, 0, 1); *insn++ = BPF_ALU64_IMM(BPF_ADD, ret, offsetof(struct htab_elem, key) + @@ -813,7 +985,7 @@ static __always_inline void *__htab_lru_map_lookup_elem(struct bpf_map *map, if (l) { if (mark) - bpf_lru_node_set_ref(&l->lru_node); + bpf_lru_node_set_ref(htab_elem_lru_node(l)); return htab_elem_value(l, map->key_size); } @@ -836,19 +1008,17 @@ static int htab_lru_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn = insn_buf; const int ret = BPF_REG_0; const int ref_reg = BPF_REG_1; + const s16 ref_off = (int)offsetof(struct htab_elem_lru, lru_node) + + (int)offsetof(struct bpf_lru_node, ref) - + (int)offsetof(struct htab_elem_lru, elem); BUILD_BUG_ON(!__same_type(&__htab_map_lookup_elem, (void *(*)(struct bpf_map *map, void *key))NULL)); *insn++ = BPF_EMIT_CALL(__htab_map_lookup_elem); *insn++ = BPF_JMP_IMM(BPF_JEQ, ret, 0, 4); - *insn++ = BPF_LDX_MEM(BPF_B, ref_reg, ret, - offsetof(struct htab_elem, lru_node) + - offsetof(struct bpf_lru_node, ref)); + *insn++ = BPF_LDX_MEM(BPF_B, ref_reg, ret, ref_off); *insn++ = BPF_JMP_IMM(BPF_JNE, ref_reg, 0, 1); - *insn++ = BPF_ST_MEM(BPF_B, ret, - offsetof(struct htab_elem, lru_node) + - offsetof(struct bpf_lru_node, ref), - 1); + *insn++ = BPF_ST_MEM(BPF_B, ret, ref_off, 1); *insn++ = BPF_ALU64_IMM(BPF_ADD, ret, offsetof(struct htab_elem, key) + round_up(map->key_size, 8)); @@ -880,14 +1050,15 @@ static void check_and_cancel_fields(struct bpf_htab *htab, static bool htab_lru_map_delete_node(void *arg, struct bpf_lru_node *node) { struct bpf_htab *htab = arg; - struct htab_elem *l = NULL, *tgt_l; + struct htab_elem_lru *tgt_l; + struct htab_elem *l = NULL; struct hlist_nulls_head *head; struct hlist_nulls_node *n; unsigned long flags; struct bucket *b; int ret; - tgt_l = container_of(node, struct htab_elem, lru_node); + tgt_l = container_of(node, struct htab_elem_lru, lru_node); b = __select_bucket(htab, tgt_l->hash); head = &b->head; @@ -896,7 +1067,7 @@ static bool htab_lru_map_delete_node(void *arg, struct bpf_lru_node *node) return false; hlist_nulls_for_each_entry_rcu(l, n, head, hash_node) - if (l == tgt_l) { + if (l == &tgt_l->elem) { hlist_nulls_del_rcu(&l->hash_node); bpf_map_dec_elem_count(&htab->map); break; @@ -904,9 +1075,9 @@ static bool htab_lru_map_delete_node(void *arg, struct bpf_lru_node *node) htab_unlock_bucket(b, flags); - if (l == tgt_l) + if (l == &tgt_l->elem) check_and_cancel_fields(htab, l); - return l == tgt_l; + return l == &tgt_l->elem; } /* Called from syscall */ @@ -930,7 +1101,8 @@ static int htab_map_get_next_key(struct bpf_map *map, void *key, void *next_key) head = select_bucket(htab, hash); /* lookup the key */ - l = lookup_nulls_elem_raw(head, hash, key, key_size, htab->n_buckets); + l = lookup_nulls_elem_raw(head, hash, key, key_size, + htab_has_hash(htab), htab->n_buckets); if (!l) goto find_first_elem; @@ -973,8 +1145,8 @@ static void htab_elem_free(struct bpf_htab *htab, struct htab_elem *l) check_and_cancel_fields(htab, l); if (htab->map.map_type == BPF_MAP_TYPE_PERCPU_HASH) - bpf_mem_cache_free(&htab->pcpu_ma, l->ptr_to_pptr); - bpf_mem_cache_free(&htab->ma, l); + bpf_mem_cache_free(&htab->pcpu_ma, htab_elem_get_ptr_to_pptr(htab, l)); + bpf_mem_cache_free(&htab->ma, htab_elem_container(htab, l)); } static void htab_put_fd_value(struct bpf_htab *htab, struct htab_elem *l) @@ -1107,11 +1279,17 @@ static struct htab_elem *alloc_htab_elem(struct bpf_htab *htab, void *key, if (prealloc) { if (old_elem) { /* if we're updating the existing element, - * use per-cpu extra elems to avoid freelist_pop/push + * use per-cpu extra elems to avoid freelist_pop/push. + * The caller puts old_elem into the slot once it is + * unlinked. Until then the slot is empty for an NMI + * or a kprobe that updates this map on this cpu. */ pl_new = this_cpu_ptr(htab->extra_elems); - l_new = *pl_new; - *pl_new = old_elem; + l_new = READ_ONCE(*pl_new); + do { + if (!l_new) + return ERR_PTR(-EBUSY); + } while (!try_cmpxchg_local(pl_new, &l_new, NULL)); } else { struct pcpu_freelist_node *l; @@ -1122,6 +1300,8 @@ static struct htab_elem *alloc_htab_elem(struct bpf_htab *htab, void *key, bpf_map_inc_elem_count(&htab->map); } } else { + void *container; + if (is_map_full(htab)) if (!old_elem) /* when map is full and update() is replacing @@ -1131,14 +1311,17 @@ static struct htab_elem *alloc_htab_elem(struct bpf_htab *htab, void *key, */ return ERR_PTR(-E2BIG); inc_elem_count(htab); - l_new = bpf_mem_cache_alloc(&htab->ma); - if (!l_new) { + container = bpf_mem_cache_alloc(&htab->ma); + if (!container) { l_new = ERR_PTR(-ENOMEM); goto dec_count; } + l_new = container + htab->elem_offset; } - memcpy(l_new->key, key, key_size); + if (htab_has_hash(htab)) + memcpy(l_new->key, key, key_size); + if (percpu) { if (prealloc) { pptr = htab_elem_get_ptr(l_new, key_size); @@ -1147,11 +1330,11 @@ static struct htab_elem *alloc_htab_elem(struct bpf_htab *htab, void *key, void *ptr = bpf_mem_cache_alloc(&htab->pcpu_ma); if (!ptr) { - bpf_mem_cache_free(&htab->ma, l_new); + bpf_mem_cache_free(&htab->ma, htab_elem_container(htab, l_new)); l_new = ERR_PTR(-ENOMEM); goto dec_count; } - l_new->ptr_to_pptr = ptr; + htab_elem_set_ptr_to_pptr(htab, l_new, ptr); pptr = *(void __percpu **)ptr; } @@ -1170,7 +1353,11 @@ static struct htab_elem *alloc_htab_elem(struct bpf_htab *htab, void *key, copy_map_value(&htab->map, htab_elem_value(l_new, key_size), value); } - l_new->hash = hash; + if (htab_has_hash(htab)) + htab_elem_set_hash(l_new, hash); + else + WRITE_ONCE(*(unsigned long *)l_new->key, + htab_zero_extend_key(key, key_size)); return l_new; dec_count: dec_elem_count(htab); @@ -1180,11 +1367,11 @@ static struct htab_elem *alloc_htab_elem(struct bpf_htab *htab, void *key, static int check_flags(struct bpf_htab *htab, struct htab_elem *l_old, u64 map_flags) { - if (l_old && (map_flags & ~BPF_F_LOCK) == BPF_NOEXIST) + if (l_old && (map_flags & BPF_NOEXIST)) /* elem already exists */ return -EEXIST; - if (!l_old && (map_flags & ~BPF_F_LOCK) == BPF_EXIST) + if (!l_old && (map_flags & BPF_EXIST)) /* elem doesn't exist, cannot update it */ return -ENOENT; @@ -1221,6 +1408,7 @@ static long htab_map_update_elem(struct bpf_map *map, void *key, void *value, return -EINVAL; /* find an element without taking the bucket lock */ l_old = lookup_nulls_elem_raw(head, hash, key, key_size, + htab_has_hash(htab), htab->n_buckets); ret = check_flags(htab, l_old, map_flags); if (ret) @@ -1242,7 +1430,7 @@ static long htab_map_update_elem(struct bpf_map *map, void *key, void *value, if (ret) return ret; - l_old = lookup_elem_raw(head, hash, key, key_size); + l_old = lookup_elem_raw(head, hash, key, key_size, htab_has_hash(htab)); ret = check_flags(htab, l_old, map_flags); if (ret) @@ -1277,11 +1465,15 @@ static long htab_map_update_elem(struct bpf_map *map, void *key, void *value, if (l_old) { hlist_nulls_del_rcu(&l_old->hash_node); - /* l_old has already been stashed in htab->extra_elems, cancel - * its reusable special fields before it is available for reuse. + /* + * l_old replaces the extra element that alloc_htab_elem() took. + * Cancel its reusable special fields first: once it is in the + * slot, the next update on this cpu can reuse it. */ - if (htab_is_prealloc(htab)) + if (htab_is_prealloc(htab)) { check_and_cancel_fields(htab, l_old); + WRITE_ONCE(*this_cpu_ptr(htab->extra_elems), l_old); + } } htab_unlock_bucket(b, flags); if (l_old && !htab_is_prealloc(htab)) @@ -1296,14 +1488,15 @@ static void htab_lru_push_free(struct bpf_htab *htab, struct htab_elem *elem) { check_and_cancel_fields(htab, elem); bpf_map_dec_elem_count(&htab->map); - bpf_lru_push_free(&htab->lru, &elem->lru_node); + bpf_lru_push_free(&htab->lru, htab_elem_lru_node(elem)); } static long htab_lru_map_update_elem(struct bpf_map *map, void *key, void *value, u64 map_flags) { struct bpf_htab *htab = container_of(map, struct bpf_htab, map); - struct htab_elem *l_new, *l_old = NULL; + struct htab_elem *l_old = NULL; + struct htab_elem_lru *l_new; struct hlist_nulls_head *head; unsigned long flags; struct bucket *b; @@ -1331,13 +1524,13 @@ static long htab_lru_map_update_elem(struct bpf_map *map, void *key, void *value l_new = prealloc_lru_pop(htab, key, hash); if (!l_new) return -ENOMEM; - copy_map_value(&htab->map, htab_elem_value(l_new, map->key_size), value); + copy_map_value(&htab->map, htab_elem_value(&l_new->elem, map->key_size), value); ret = htab_lock_bucket(b, &flags); if (ret) goto err_lock_bucket; - l_old = lookup_elem_raw(head, hash, key, key_size); + l_old = lookup_elem_raw(head, hash, key, key_size, true); ret = check_flags(htab, l_old, map_flags); if (ret) @@ -1346,7 +1539,7 @@ static long htab_lru_map_update_elem(struct bpf_map *map, void *key, void *value /* add new element to the head of the list, so that * concurrent search will find it before old elem */ - hlist_nulls_add_head_rcu(&l_new->hash_node, head); + hlist_nulls_add_head_rcu(&l_new->elem.hash_node, head); if (l_old) { bpf_lru_node_set_ref(&l_new->lru_node); hlist_nulls_del_rcu(&l_old->hash_node); @@ -1358,7 +1551,7 @@ static long htab_lru_map_update_elem(struct bpf_map *map, void *key, void *value err_lock_bucket: if (ret) - htab_lru_push_free(htab, l_new); + htab_lru_push_free(htab, &l_new->elem); else if (l_old) htab_lru_push_free(htab, l_old); @@ -1367,9 +1560,11 @@ static long htab_lru_map_update_elem(struct bpf_map *map, void *key, void *value static int htab_map_check_update_flags(bool onallcpus, u64 map_flags) { + if (unlikely((map_flags & BPF_EXIST_FLAGS) == BPF_EXIST_FLAGS)) + return -EINVAL; if (unlikely(!onallcpus && map_flags > BPF_EXIST)) return -EINVAL; - if (unlikely(onallcpus && ((map_flags & BPF_F_LOCK) || (u32)map_flags > BPF_F_ALL_CPUS))) + if (unlikely(onallcpus && ((u32)map_flags & ~(BPF_EXIST_FLAGS | BPF_CPU_FLAGS)))) return -EINVAL; return 0; } @@ -1404,7 +1599,7 @@ static long htab_map_update_elem_in_place(struct bpf_map *map, void *key, if (ret) return ret; - l_old = lookup_elem_raw(head, hash, key, key_size); + l_old = lookup_elem_raw(head, hash, key, key_size, htab_has_hash(htab)); ret = check_flags(htab, l_old, map_flags); if (ret) @@ -1442,7 +1637,8 @@ static long __htab_lru_percpu_map_update_elem(struct bpf_map *map, void *key, bool onallcpus) { struct bpf_htab *htab = container_of(map, struct bpf_htab, map); - struct htab_elem *l_new = NULL, *l_old; + struct htab_elem_lru *l_new = NULL; + struct htab_elem *l_old; struct hlist_nulls_head *head; unsigned long flags; struct bucket *b; @@ -1467,7 +1663,7 @@ static long __htab_lru_percpu_map_update_elem(struct bpf_map *map, void *key, * to remove older elem from htab and this removal * operation will need a bucket lock. */ - if (map_flags != BPF_EXIST) { + if (!(map_flags & BPF_EXIST)) { l_new = prealloc_lru_pop(htab, key, hash); if (!l_new) return -ENOMEM; @@ -1477,22 +1673,22 @@ static long __htab_lru_percpu_map_update_elem(struct bpf_map *map, void *key, if (ret) goto err_lock_bucket; - l_old = lookup_elem_raw(head, hash, key, key_size); + l_old = lookup_elem_raw(head, hash, key, key_size, true); ret = check_flags(htab, l_old, map_flags); if (ret) goto err; if (l_old) { - bpf_lru_node_set_ref(&l_old->lru_node); + bpf_lru_node_set_ref(htab_elem_lru_node(l_old)); /* per-cpu hash map can update value in-place */ pcpu_copy_value(htab, htab_elem_get_ptr(l_old, key_size), value, onallcpus, map_flags); } else { - pcpu_init_value(htab, htab_elem_get_ptr(l_new, key_size), + pcpu_init_value(htab, htab_elem_get_ptr(&l_new->elem, key_size), value, onallcpus, map_flags); - hlist_nulls_add_head_rcu(&l_new->hash_node, head); + hlist_nulls_add_head_rcu(&l_new->elem.hash_node, head); l_new = NULL; } ret = 0; @@ -1542,7 +1738,7 @@ static long htab_map_delete_elem(struct bpf_map *map, void *key) if (ret) return ret; - l = lookup_elem_raw(head, hash, key, key_size); + l = lookup_elem_raw(head, hash, key, key_size, htab_has_hash(htab)); if (l) hlist_nulls_del_rcu(&l->hash_node); else @@ -1577,7 +1773,7 @@ static long htab_lru_map_delete_elem(struct bpf_map *map, void *key) if (ret) return ret; - l = lookup_elem_raw(head, hash, key, key_size); + l = lookup_elem_raw(head, hash, key, key_size, true); if (l) hlist_nulls_del_rcu(&l->hash_node); @@ -1718,7 +1914,7 @@ static int __htab_map_lookup_and_delete_elem(struct bpf_map *map, void *key, if (ret) return ret; - l = lookup_elem_raw(head, hash, key, key_size); + l = lookup_elem_raw(head, hash, key, key_size, htab_has_hash(htab)); if (!l) { ret = -ENOENT; goto out_unlock; @@ -2489,7 +2685,7 @@ static void *htab_lru_percpu_map_lookup_elem(struct bpf_map *map, void *key) struct htab_elem *l = __htab_map_lookup_elem(map, key); if (l) { - bpf_lru_node_set_ref(&l->lru_node); + bpf_lru_node_set_ref(htab_elem_lru_node(l)); return this_cpu_ptr(htab_elem_get_ptr(l, map->key_size)); } @@ -2505,7 +2701,7 @@ static void *htab_lru_percpu_map_lookup_percpu_elem(struct bpf_map *map, void *k l = __htab_map_lookup_elem(map, key); if (l) { - bpf_lru_node_set_ref(&l->lru_node); + bpf_lru_node_set_ref(htab_elem_lru_node(l)); return per_cpu_ptr(htab_elem_get_ptr(l, map->key_size), cpu); } @@ -2904,18 +3100,6 @@ static int rhtab_map_alloc_check(union bpf_attr *attr) return htab_map_alloc_check(attr); } -static void rhtab_mem_dtor(void *obj, void *ctx) -{ - struct htab_btf_record *hrec = ctx; - struct rhtab_elem *elem = obj; - - if (IS_ERR_OR_NULL(hrec->record)) - return; - - bpf_obj_free_fields(hrec->record, - rhtab_elem_value(elem, hrec->key_size)); -} - static void rhtab_free_elem(void *ptr, void *arg) { struct bpf_rhtab *rhtab = arg; @@ -3141,7 +3325,9 @@ static int rhtab_map_check_btf(struct bpf_map *map, const struct btf *btf, if (btf_type_is_void(key_type)) return -EINVAL; - return bpf_ma_set_dtor(map, &rhtab->ma, rhtab_mem_dtor); + return bpf_ma_set_dtor(map, &rhtab->ma, htab_mem_dtor, + offsetof(struct rhtab_elem, data) + + round_up(map->key_size, 8)); } static void rhtab_map_free_internal_structs(struct bpf_map *map) diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c index 712dca5a2c5b..648abe1ba63a 100644 --- a/kernel/bpf/helpers.c +++ b/kernel/bpf/helpers.c @@ -679,9 +679,44 @@ const struct bpf_func_proto bpf_copy_from_user_proto = { .arg3_type = ARG_ANYTHING, }; +static int __bpf_copy_from_user_mm(void *dst, u32 size, + const void __user *user_ptr, + struct mm_struct *mm) +{ + int ret; + + ret = access_remote_vm(mm, (unsigned long)user_ptr, dst, size, 0); + if (ret == size) + return 0; + + memset(dst, 0, size); + /* Return -EFAULT for partial read */ + return ret < 0 ? ret : -EFAULT; +} + +static int __bpf_copy_from_user_mm_str(void *dst, u32 size, + const void __user *user_ptr, + struct mm_struct *mm, u64 flags) +{ + int ret; + + ret = copy_remote_mm_str(mm, (unsigned long)user_ptr, dst, size, 0); + if (ret < 0) { + if (flags & BPF_F_PAD_ZEROS) + memset(dst, 0, size); + return ret; + } + + if (flags & BPF_F_PAD_ZEROS) + memset(dst + ret, 0, size - ret); + + return ret + 1; +} + BPF_CALL_5(bpf_copy_from_user_task, void *, dst, u32, size, const void __user *, user_ptr, struct task_struct *, tsk, u64, flags) { + struct mm_struct *mm; int ret; /* flags is not used yet */ @@ -691,13 +726,16 @@ BPF_CALL_5(bpf_copy_from_user_task, void *, dst, u32, size, if (unlikely(!size)) return 0; - ret = access_process_vm(tsk, (unsigned long)user_ptr, dst, size, 0); - if (ret == size) - return 0; + mm = get_task_mm(tsk); + if (!mm) { + memset(dst, 0, size); + return -EFAULT; + } - memset(dst, 0, size); - /* Return -EFAULT for partial read */ - return ret < 0 ? ret : -EFAULT; + ret = __bpf_copy_from_user_mm(dst, size, user_ptr, mm); + mmput(mm); + + return ret; } const struct bpf_func_proto bpf_copy_from_user_task_proto = { @@ -1092,7 +1130,7 @@ const struct bpf_func_proto bpf_snprintf_proto = { .func = bpf_snprintf, .gpl_only = true, .ret_type = RET_INTEGER, - .arg1_type = ARG_PTR_TO_MEM_OR_NULL | MEM_WRITE, + .arg1_type = ARG_PTR_TO_MEM_OR_NULL | MEM_UNINIT | MEM_WRITE, .arg2_type = ARG_MEM_SIZE_OR_ZERO, .arg3_type = ARG_PTR_TO_CONST_STR, .arg4_type = ARG_PTR_TO_MEM | PTR_MAYBE_NULL | MEM_RDONLY, @@ -1224,18 +1262,21 @@ static void bpf_wq_work(struct work_struct *work) BTF_TYPE_EMIT(struct bpf_wq); + /* keeps the program that callback_fn belongs to */ + rcu_read_lock_trace(); + callback_fn = READ_ONCE(cb->callback_fn); if (!callback_fn) - return; + goto out; key = map_key_from_value(map, value, &idx); - rcu_read_lock_trace(); - migrate_disable(); + migrate_disable(); callback_fn((u64)(long)map, (u64)(long)key, (u64)(long)value, 0, 0); migrate_enable(); +out: rcu_read_unlock_trace(); } @@ -1510,6 +1551,7 @@ static const struct bpf_func_proto bpf_timer_set_callback_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_TIMER, .arg2_type = ARG_PTR_TO_FUNC, + .arg3_type = ARG_PTR_TO_PROG_AUX, }; static bool defer_timer_wq_op(void) @@ -3658,6 +3700,92 @@ __bpf_kfunc int bpf_copy_from_user_str(void *dst, u32 dst__sz, const void __user return ret + 1; } +/** + * bpf_copy_from_user_mm() - Copy data from an address space + * @dst: Destination address, in kernel space + * @dst__sz: Number of bytes to copy + * @unsafe_ptr__ign: Source address in the address space + * @mm: Address space to copy from + * @flags: Reserved for future use; must be zero + * + * Copies data from the user address space associated with @mm. The destination + * is zeroed if an attempted copy cannot be completed in full. Unsupported + * flags return -EINVAL without modifying @dst. + * + * Return: 0 on success, -EINVAL if @flags is non-zero, or -EFAULT if @mm is no + * longer active or the copy fails or is partial. + */ +__bpf_kfunc int bpf_copy_from_user_mm(void *dst, u32 dst__sz, + const void __user *unsafe_ptr__ign, + struct mm_struct *mm, u64 flags) +{ + int ret; + + if (unlikely(flags)) + return -EINVAL; + + if (unlikely(!dst__sz)) + return 0; + + if (unlikely(!mmget_not_zero(mm))) { + memset(dst, 0, dst__sz); + return -EFAULT; + } + + ret = __bpf_copy_from_user_mm(dst, dst__sz, unsafe_ptr__ign, mm); + mmput(mm); + + return ret; +} + +/** + * bpf_copy_from_user_mm_str() - Copy a string from an address space + * @dst: Destination address, in kernel space. This buffer must be + * at least @dst__sz bytes long + * @dst__sz: Maximum number of bytes to copy, including the trailing NUL + * @unsafe_ptr__ign: Source address in the address space + * @mm: Address space to copy from + * @flags: The only supported flag is BPF_F_PAD_ZEROS + * + * Copies a NUL-terminated string from the user address space associated with + * @mm. If the string is too long, @dst is still NUL-terminated unless @dst__sz + * is zero. + * + * If the flags are valid and BPF_F_PAD_ZEROS is set, the unused portion of + * @dst is cleared on success and all of @dst is cleared on a copy failure. + * Unsupported flags return -EINVAL without modifying @dst. + * + * Return: The number of copied bytes including the NUL terminator on success, + * -EINVAL if @flags is invalid, or -EFAULT if @mm is no longer active or the + * copy fails. + */ +__bpf_kfunc int bpf_copy_from_user_mm_str(void *dst, u32 dst__sz, + const void __user *unsafe_ptr__ign, + struct mm_struct *mm, u64 flags) +{ + int ret; + + if (unlikely(flags & ~BPF_F_PAD_ZEROS)) + return -EINVAL; + + if (unlikely(dst__sz == 0)) + return 0; + + if (unlikely(!mmget_not_zero(mm))) { + if (flags & BPF_F_PAD_ZEROS) + memset(dst, 0, dst__sz); + else + *(char *)dst = '\0'; + return -EFAULT; + } + + ret = __bpf_copy_from_user_mm_str(dst, dst__sz, unsafe_ptr__ign, + mm, flags); + mmput(mm); + + return ret; +} + /** * bpf_copy_from_user_task_str() - Copy a string from an task's address space * @dst: Destination address, in kernel space. This buffer must be @@ -3681,6 +3809,7 @@ __bpf_kfunc int bpf_copy_from_user_task_str(void *dst, u32 dst__sz, const void __user *unsafe_ptr__ign, struct task_struct *tsk, u64 flags) { + struct mm_struct *mm; int ret; if (unlikely(flags & ~BPF_F_PAD_ZEROS)) @@ -3689,17 +3818,20 @@ __bpf_kfunc int bpf_copy_from_user_task_str(void *dst, u32 dst__sz, if (unlikely(dst__sz == 0)) return 0; - ret = copy_remote_vm_str(tsk, (unsigned long)unsafe_ptr__ign, dst, dst__sz, 0); - if (ret < 0) { + mm = get_task_mm(tsk); + if (!mm) { if (flags & BPF_F_PAD_ZEROS) memset(dst, 0, dst__sz); - return ret; + else + *(char *)dst = '\0'; + return -EFAULT; } - if (flags & BPF_F_PAD_ZEROS) - memset(dst + ret, 0, dst__sz - ret); + ret = __bpf_copy_from_user_mm_str(dst, dst__sz, unsafe_ptr__ign, + mm, flags); + mmput(mm); - return ret + 1; + return ret; } /* Keep unsigned long in prototype so that kfunc is usable when emitted to @@ -4437,15 +4569,21 @@ static void bpf_task_work_ctx_put(struct bpf_task_work_ctx *ctx) } } -static void bpf_task_work_cancel(struct bpf_task_work_ctx *ctx) +static void bpf_task_work_cancel(struct bpf_task_work_ctx *ctx, + struct task_struct *task) { /* * Scheduled task_work callback holds ctx ref, so if we successfully * cancelled, we put that ref on callback's behalf. If we couldn't * cancel, callback will inevitably run or has already completed * running, and it would have taken care of its ctx ref itself. + * + * The caller passes the task explicitly: the scheduling handler must + * not read ctx->task after the work is published (the callback may + * have reset it), while in the asynchronous cancellation path + * ctx->task is stable, see bpf_task_work_cancel_scheduled(). */ - if (task_work_cancel(ctx->task, &ctx->work)) + if (task_work_cancel(task, &ctx->work)) bpf_task_work_ctx_put(ctx); } @@ -4486,6 +4624,7 @@ static void bpf_task_work_callback(struct callback_head *cb) static void bpf_task_work_irq(struct irq_work *irq_work) { struct bpf_task_work_ctx *ctx = container_of(irq_work, struct bpf_task_work_ctx, irq_work); + struct task_struct *task; enum bpf_task_work_state state; int err; @@ -4496,7 +4635,16 @@ static void bpf_task_work_irq(struct irq_work *irq_work) return; } - err = task_work_add(ctx->task, &ctx->work, ctx->mode); + /* + * Capture the task pointer before the work is published: once the + * callback runs, it may reset ctx->task. The capture happens while + * this round's task reference is held, and every path that can + * release it runs after this handler entered its rcu read-side + * section. bpf_task_release() is put_task_struct_rcu_user(), so the + * task_struct itself cannot be freed before this section ends. + */ + task = ctx->task; + err = task_work_add(task, &ctx->work, ctx->mode); if (err) { bpf_task_work_ctx_reset(ctx); /* @@ -4510,14 +4658,14 @@ static void bpf_task_work_irq(struct irq_work *irq_work) /* * It's technically possible for just scheduled task_work callback to - * complete running by now, going SCHEDULING -> RUNNING and then - * dropping its ctx refcount. Instead of capturing an extra ref just - * to protect below ctx->state access, we rely on rcu_read_lock - * above to prevent kfree_rcu from freeing ctx before we return. + * complete running by now, going SCHEDULING -> RUNNING, resetting + * ctx->task and dropping its ctx refcount. The rcu read-side section + * above keeps both the ctx memory and the captured task pointer + * valid until this handler returns. */ state = cmpxchg(&ctx->state, BPF_TW_SCHEDULING, BPF_TW_SCHEDULED); if (state == BPF_TW_FREED) - bpf_task_work_cancel(ctx); /* clean up if we switched into FREED state */ + bpf_task_work_cancel(ctx, task); /* clean up if we switched into FREED state */ } static struct bpf_task_work_ctx *bpf_task_work_fetch_ctx(struct bpf_task_work *tw, @@ -4537,6 +4685,7 @@ static struct bpf_task_work_ctx *bpf_task_work_fetch_ctx(struct bpf_task_work *t memset(ctx, 0, sizeof(*ctx)); refcount_set(&ctx->refcnt, 1); /* map's own ref */ ctx->state = BPF_TW_STANDBY; + init_irq_work(&ctx->irq_work, bpf_task_work_irq); old_ctx = cmpxchg(&twk->ctx, NULL, ctx); if (old_ctx) { @@ -4579,6 +4728,18 @@ static struct bpf_task_work_ctx *bpf_task_work_acquire_ctx(struct bpf_task_work bpf_task_work_ctx_put(ctx); return ERR_PTR(-EBUSY); } + /* + * STANDBY can be published before the previous scheduling irq_work + * returns. Claim PENDING before checking BUSY to order the check after + * that publication. No irq_work has been queued for this attempt yet, + * so reject reuse and roll back to STANDBY while BUSY is set. Preserve + * FREED if deletion wins the rollback race. + */ + if (unlikely(irq_work_is_busy(&ctx->irq_work))) { + (void)cmpxchg(&ctx->state, BPF_TW_PENDING, BPF_TW_STANDBY); + bpf_task_work_ctx_put(ctx); + return ERR_PTR(-EBUSY); + } /* * If no process or bpffs is holding a reference to the map, no new callbacks should be @@ -4628,7 +4789,6 @@ static int bpf_task_work_schedule(struct task_struct *task, struct bpf_task_work ctx->map = map; ctx->map_val = (void *)tw - map->record->task_work_off; init_task_work(&ctx->work, bpf_task_work_callback); - init_irq_work(&ctx->irq_work, bpf_task_work_irq); irq_work_queue(&ctx->irq_work); return 0; @@ -4676,6 +4836,106 @@ __bpf_kfunc int bpf_task_work_schedule_resume(struct task_struct *task, struct b return bpf_task_work_schedule(task, tw, map__const_map, callback, aux, TWA_RESUME); } +typedef int (*bpf_rcu_callback_t)(struct bpf_map *map, void *key, void *value); + +/* Actual type for struct bpf_rcu_head */ +struct bpf_rcu_head_kern { + struct rcu_head rcu; + bpf_callback_t callback_fn; + struct bpf_map *map; + struct bpf_prog *prog; + u32 armed; +} __aligned(8); + +static void bpf_rcu_run_callback(struct rcu_head *rcu) +{ + struct bpf_rcu_head_kern *rh = container_of(rcu, struct bpf_rcu_head_kern, rcu); + bpf_callback_t callback_fn = rh->callback_fn; + struct bpf_prog *prog = rh->prog; + struct bpf_map *map = rh->map; + void *value, *key; + u32 idx; + + value = (void *)rh - map->record->rcu_head_off; + key = map_key_from_value(map, value, &idx); + + /* Pairs with the arming cmpxchg(): rh may be re-armed as soon as this store lands. */ + smp_store_release(&rh->armed, 0); + + rcu_read_lock_dont_migrate(); + callback_fn((u64)(long)map, (u64)(long)key, (u64)(long)value, 0, 0); + rcu_read_unlock_migrate(); + + bpf_prog_put(prog); +} + +static int __bpf_call_rcu(struct bpf_rcu_head *rh, struct bpf_map *map, void *callback, + struct bpf_prog_aux *aux, bool trace) +{ + struct bpf_rcu_head_kern *rhk = (void *)rh; + struct bpf_prog *prog; + + BUILD_BUG_ON(sizeof(struct bpf_rcu_head_kern) > sizeof(struct bpf_rcu_head)); + BUILD_BUG_ON(__alignof__(struct bpf_rcu_head_kern) != __alignof__(struct bpf_rcu_head)); + BTF_TYPE_EMIT(struct bpf_rcu_head); + + /* A queued callback cannot be cancelled, so a self-rearming one would pin prog and map. */ + if (!atomic64_read(&map->usercnt)) + return -EPERM; + + if (cmpxchg(&rhk->armed, 0, 1)) + return -EBUSY; + + prog = bpf_prog_inc_not_zero(aux->prog); + if (IS_ERR(prog)) { + WRITE_ONCE(rhk->armed, 0); + return -EBADF; + } + + rhk->callback_fn = (bpf_callback_t)callback; + rhk->map = map; + rhk->prog = prog; + if (trace) + call_rcu_tasks_trace(&rhk->rcu, bpf_rcu_run_callback); + else + call_rcu(&rhk->rcu, bpf_rcu_run_callback); + return 0; +} + +/** + * bpf_call_rcu - Invoke a BPF callback after an RCU grace period + * @rh: struct bpf_rcu_head in a BPF map value + * @map__const_map: bpf_map that embeds struct bpf_rcu_head in the values + * @callback: BPF subprogram, invoked as callback(map, key, value) for the value holding @rh + * @aux: bpf_prog_aux of the caller, implicitly set by the verifier + * + * Return: 0, -EBUSY if @rh is already queued, -EPERM if @map is held by neither a process + * nor bpffs, or -EBADF if the calling program is going away. + */ +__bpf_kfunc int bpf_call_rcu(struct bpf_rcu_head *rh, void *map__const_map, + bpf_rcu_callback_t callback, struct bpf_prog_aux *aux) +{ + return __bpf_call_rcu(rh, map__const_map, callback, aux, false); +} + +/** + * bpf_call_rcu_tasks_trace - Invoke a BPF callback after an RCU tasks trace grace period + * @rh: struct bpf_rcu_head in a BPF map value + * @map__const_map: bpf_map that embeds struct bpf_rcu_head in the values + * @callback: BPF subprogram, invoked as callback(map, key, value) for the value holding @rh + * @aux: bpf_prog_aux of the caller, implicitly set by the verifier + * + * Waits for sleepable BPF programs too. The callback itself is not sleepable either way. + * + * Return: 0, -EBUSY if @rh is already queued, -EPERM if @map is held by neither a process + * nor bpffs, or -EBADF if the calling program is going away. + */ +__bpf_kfunc int bpf_call_rcu_tasks_trace(struct bpf_rcu_head *rh, void *map__const_map, + bpf_rcu_callback_t callback, struct bpf_prog_aux *aux) +{ + return __bpf_call_rcu(rh, map__const_map, callback, aux, true); +} + static int make_file_dynptr(struct file *file, u32 flags, bool may_sleep, struct bpf_dynptr_kern *ptr) { @@ -4777,7 +5037,13 @@ static void bpf_task_work_cancel_scheduled(struct irq_work *irq_work) { struct bpf_task_work_ctx *ctx = container_of(irq_work, struct bpf_task_work_ctx, irq_work); - bpf_task_work_cancel(ctx); /* this might put task_work callback's ref */ + /* + * Deletion observed SCHEDULED and won against the callback's + * transition to RUNNING. The callback takes its FREED exit without + * resetting ctx->task, and the transferred map reference prevents + * destruction until cancellation completes. + */ + bpf_task_work_cancel(ctx, ctx->task); /* this might put task_work callback's ref */ bpf_task_work_ctx_put(ctx); /* and here we put map's own ref that was transferred to us */ } @@ -4924,6 +5190,8 @@ BTF_ID_FLAGS(func, bpf_iter_bits_new, KF_ITER_NEW | KF_PERFMON) BTF_ID_FLAGS(func, bpf_iter_bits_next, KF_ITER_NEXT | KF_RET_NULL) BTF_ID_FLAGS(func, bpf_iter_bits_destroy, KF_ITER_DESTROY) BTF_ID_FLAGS(func, bpf_copy_from_user_str, KF_SLEEPABLE | KF_PERFMON) +BTF_ID_FLAGS(func, bpf_copy_from_user_mm, KF_SLEEPABLE | KF_PERFMON) +BTF_ID_FLAGS(func, bpf_copy_from_user_mm_str, KF_SLEEPABLE | KF_PERFMON) BTF_ID_FLAGS(func, bpf_copy_from_user_task_str, KF_SLEEPABLE | KF_PERFMON) BTF_ID_FLAGS(func, bpf_get_kmem_cache, KF_PERFMON) BTF_ID_FLAGS(func, bpf_iter_kmem_cache_new, KF_ITER_NEW | KF_SLEEPABLE) @@ -4969,6 +5237,8 @@ BTF_ID_FLAGS(func, bpf_stream_vprintk, KF_IMPLICIT_ARGS | KF_SPINLOCK_SAFE | KF_ BTF_ID_FLAGS(func, bpf_stream_print_stack, KF_IMPLICIT_ARGS | KF_SPINLOCK_SAFE | KF_PERFMON) BTF_ID_FLAGS(func, bpf_task_work_schedule_signal, KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, bpf_task_work_schedule_resume, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, bpf_call_rcu, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, bpf_call_rcu_tasks_trace, KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, bpf_dynptr_from_file) BTF_ID_FLAGS(func, bpf_dynptr_file_discard, KF_RELEASE) BTF_ID_FLAGS(func, bpf_timer_cancel_async) diff --git a/kernel/bpf/keys.c b/kernel/bpf/keys.c new file mode 100644 index 000000000000..bffc356839ac --- /dev/null +++ b/kernel/bpf/keys.c @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (c) 2026 Isovalent */ + +#include +#include +#include +#include +#include +#include +#include + +#undef MODULE_PARAM_PREFIX +#define MODULE_PARAM_PREFIX "bpf." + +static struct key *bpf_keyring; + +static bool bpf_keyring_unsealed __ro_after_init; +module_param_named(keyring_unsealed, bpf_keyring_unsealed, bool, 0444); +MODULE_PARM_DESC(keyring_unsealed, "Leave the bpf keyring unsealed"); + +bool bpf_keyring_enforced(void) +{ + return bpf_keyring_unsealed; +} + +struct bpf_key *bpf_lookup_keyring(void) +{ + struct bpf_key *bkey; + + if (!bpf_keyring) + return NULL; + if (!READ_ONCE(bpf_keyring->keys.nr_leaves_on_tree) || + !READ_ONCE(bpf_keyring->restrict_link)) + return NULL; + + bkey = kmalloc_obj(*bkey); + if (!bkey) + return NULL; + + bkey->key = bpf_keyring; + bkey->has_ref = false; + return bkey; +} + +static int __init bpf_keyring_init(void) +{ + struct key *keyring; + + keyring = keyring_alloc(".bpf", + GLOBAL_ROOT_UID, GLOBAL_ROOT_GID, + current_cred(), KEY_POS_SEARCH | + KEY_USR_VIEW | KEY_USR_READ | + KEY_USR_WRITE | KEY_USR_SEARCH | + KEY_USR_SETATTR, KEY_ALLOC_NOT_IN_QUOTA, + NULL, NULL); + if (IS_ERR(keyring)) { + pr_err("bpf: cannot allocate bpf keyring: %ld\n", + PTR_ERR(keyring)); + return 0; + } + if (!bpf_keyring_unsealed && + keyring_restrict(make_key_ref(keyring, true), NULL, NULL)) { + pr_err("bpf: cannot seal bpf keyring\n"); + key_revoke(keyring); + key_put(keyring); + return 0; + } + + bpf_keyring = keyring; + key_register_bpf_keyring(keyring); + return 0; +} +late_initcall(bpf_keyring_init); diff --git a/kernel/bpf/liveness.c b/kernel/bpf/liveness.c index 301fc60bddc4..e657e6d95408 100644 --- a/kernel/bpf/liveness.c +++ b/kernel/bpf/liveness.c @@ -10,10 +10,40 @@ #define verbose(env, fmt, args...) bpf_verifier_log_write(env, fmt, ##args) -struct per_frame_masks { - spis_t may_read; /* stack slots that may be read by this instruction */ - spis_t must_write; /* stack slots written by this instruction */ - spis_t live_before; /* stack slots that may be read by this insn and its successors */ +/* + * Stack liveness is tracked with a 4-byte (half register) granularity. + * Half-slot 0 covers [fp-4, fp), half-slot 1 covers [fp-8, fp-4), and so on, + * hence FRAME_HALF_SPIS - 1 is the deepest half-slot a frame can have. + */ +#define FRAME_HALF_SPIS (MAX_BPF_STACK_JIT / BPF_HALF_REG_SIZE) +#define FRAME_MAX_WORDS BITS_TO_LONGS(FRAME_HALF_SPIS) + +/* Masks tracked for each instruction of a frame */ +enum { + FM_MAY_READ, /* stack slots that may be read by this instruction */ + FM_MUST_WRITE, /* stack slots written by this instruction */ + FM_LIVE_BEFORE, /* stack slots that may be read by this insn and its successors */ + FM_MAY_WRITE, /* stack slots that may be written by this instruction */ + FM_MASK_CNT, +}; + +/* + * Per instruction stack masks for one frame of a function instance. + * + * Most frames use only a fraction of the stack budget, so instead of masks + * wide enough for every half-slot of the largest possible frame, all masks + * of one array share the width @words, and the marking functions widen the + * array as deeper half-slots are recorded. Mask @kind of the instruction at + * relative index @i is at &bits[(i * FM_MASK_CNT + kind) * words]. A + * half-slot at or past @words * BITS_PER_LONG is never read by this frame, + * hence never live. An instruction that may read the whole frame, such as a + * call passing a frame pointer to another subprog, widens the array to the + * program's stack budget, the deepest an accepted program can reach, so + * that the read cannot lose half-slots to a later widening. + */ +struct frame_masks { + u32 words; + unsigned long bits[]; }; /* @@ -29,7 +59,7 @@ struct func_instance { u32 subprog_start; /* cached env->subprog_info[subprog].start */ u32 insn_cnt; /* cached number of insns in the function */ /* Per frame, per instruction masks, frames allocated lazily. */ - struct per_frame_masks *frames[MAX_CALL_FRAMES]; + struct frame_masks *frames[MAX_CALL_FRAMES]; bool must_write_initialized; }; @@ -151,53 +181,145 @@ static int relative_idx(struct func_instance *instance, u32 insn_idx) return insn_idx - instance->subprog_start; } -static struct per_frame_masks *get_frame_masks(struct func_instance *instance, - u32 frame, u32 insn_idx) +static u32 frame_mask_bits(struct frame_masks *fm) { - if (!instance->frames[frame]) + return fm->words * BITS_PER_LONG; +} + +static size_t frame_mask_words(struct func_instance *instance, u32 words) +{ + return (size_t)instance->insn_cnt * FM_MASK_CNT * words; +} + +/* Mask @kind of the instruction at relative index @rel */ +static unsigned long *rel_mask(struct frame_masks *fm, u32 rel, u32 kind) +{ + return fm->bits + ((size_t)rel * FM_MASK_CNT + kind) * fm->words; +} + +/* + * Make sure @frame has a mask array at least @words wide, allocating it or + * copying the existing masks into the wider stride as needed. + * @words must be in range [1, FRAME_MAX_WORDS]. + */ +static struct frame_masks *widen_frame_masks(struct func_instance *instance, + u32 frame, u32 words) +{ + struct frame_masks *old = instance->frames[frame], *new; + u32 i, kind; + + if (old && old->words >= words) + return old; + new = kvzalloc_flex(*new, bits, frame_mask_words(instance, words), GFP_KERNEL_ACCOUNT); + if (!new) return NULL; - - return &instance->frames[frame][relative_idx(instance, insn_idx)]; -} - -static struct per_frame_masks *alloc_frame_masks(struct func_instance *instance, - u32 frame, u32 insn_idx) -{ - struct per_frame_masks *arr; - - if (!instance->frames[frame]) { - arr = kvzalloc_objs(*arr, instance->insn_cnt, - GFP_KERNEL_ACCOUNT); - instance->frames[frame] = arr; - if (!arr) - return ERR_PTR(-ENOMEM); + new->words = words; + if (old) { + for (i = 0; i < instance->insn_cnt; i++) + for (kind = 0; kind < FM_MASK_CNT; kind++) + memcpy(rel_mask(new, i, kind), rel_mask(old, i, kind), + old->words * sizeof(*old->bits)); + kvfree(old); } - return get_frame_masks(instance, frame, insn_idx); + instance->frames[frame] = new; + return new; } -/* Accumulate may_read masks for @frame at @insn_idx */ -static int mark_stack_read(struct func_instance *instance, u32 frame, u32 insn_idx, spis_t mask) +/* + * Set the inclusive half-slot range [lo, hi] in mask @kind of @frame at @insn_idx. + * An empty range, including one with a negative @hi as computed for a write + * that does not fully cover any half-slot, marks nothing. + */ +static int mark_stack_range(struct func_instance *instance, u32 frame, u32 insn_idx, + u32 kind, s32 lo, s32 hi) { - struct per_frame_masks *masks; + struct frame_masks *fm; - masks = alloc_frame_masks(instance, frame, insn_idx); - if (IS_ERR(masks)) - return PTR_ERR(masks); - masks->may_read = spis_or(masks->may_read, mask); + /* + * An access past the frame bottom is rejected by the main verifier + * pass later, liveness only has to avoid running off the masks. + */ + hi = min_t(s32, hi, FRAME_HALF_SPIS - 1); + if (lo > hi) + return 0; + fm = widen_frame_masks(instance, frame, BITS_TO_LONGS(hi + 1)); + if (!fm) + return -ENOMEM; + bitmap_set(rel_mask(fm, relative_idx(instance, insn_idx), kind), lo, hi - lo + 1); return 0; } -static int mark_stack_write(struct func_instance *instance, u32 frame, u32 insn_idx, spis_t mask) +/* Accumulate may_read for half-slots [lo, hi] of @frame at @insn_idx */ +static int mark_stack_read(struct func_instance *instance, u32 frame, u32 insn_idx, + s32 lo, s32 hi) { - struct per_frame_masks *masks; + return mark_stack_range(instance, frame, insn_idx, FM_MAY_READ, lo, hi); +} - masks = alloc_frame_masks(instance, frame, insn_idx); - if (IS_ERR(masks)) - return PTR_ERR(masks); - masks->must_write = spis_or(masks->must_write, mask); +/* Accumulate must_write for half-slots [lo, hi] of @frame at @insn_idx */ +static int mark_stack_write(struct func_instance *instance, u32 frame, u32 insn_idx, + s32 lo, s32 hi) +{ + return mark_stack_range(instance, frame, insn_idx, FM_MUST_WRITE, lo, hi); +} + +static int mark_stack_may_write(struct func_instance *instance, u32 frame, u32 insn_idx, + s32 lo, s32 hi) +{ + return mark_stack_range(instance, frame, insn_idx, FM_MAY_WRITE, lo, hi); +} + +/* + * Mark every half-slot of @frame as possibly read by @insn_idx. This widens + * the masks to the program's stack budget: a full read recorded at a narrower + * width would leave the bits added by a later widening clear and lose part of + * it. No verifier state holds a slot past the budget, so no liveness query + * reaches beyond it; an access past the budget in unreachable code may still + * widen a frame's masks further, which is harmless. + */ +static int mark_stack_read_all(struct bpf_verifier_env *env, struct func_instance *instance, + u32 frame, u32 insn_idx) +{ + return mark_stack_read(instance, frame, insn_idx, 0, + env->stack_limit / BPF_HALF_REG_SIZE - 1); +} + +static int mark_stack_may_write_all(struct bpf_verifier_env *env, struct func_instance *instance, + u32 frame, u32 insn_idx) +{ + return mark_stack_may_write(instance, frame, insn_idx, 0, + env->stack_limit / BPF_HALF_REG_SIZE - 1); +} + +/* Accumulate @src, a mask @src_words wide, into @kind mask of @frame at @insn_idx */ +static int mark_stack_mask(struct func_instance *instance, u32 frame, u32 insn_idx, u32 kind, + const unsigned long *src, u32 src_words) +{ + u32 nbits = src_words * BITS_PER_LONG; + struct frame_masks *fm; + unsigned long *dst; + u32 last, w; + + last = find_last_bit(src, nbits); + if (last == nbits) + return 0; + fm = widen_frame_masks(instance, frame, BITS_TO_LONGS(last + 1)); + if (!fm) + return -ENOMEM; + dst = rel_mask(fm, relative_idx(instance, insn_idx), kind); + /* @src has no bits set past @last, hence none past @fm->words either */ + src_words = min(src_words, fm->words); + for (w = 0; w < src_words; w++) + dst[w] |= src[w]; return 0; } +static int mark_stack_read_mask(struct func_instance *instance, u32 frame, u32 insn_idx, + const unsigned long *src, u32 src_words) +{ + return mark_stack_mask(instance, frame, insn_idx, FM_MAY_READ, src, src_words); +} + int bpf_jmp_offset(struct bpf_insn *insn) { u8 code = insn->code; @@ -248,14 +370,18 @@ bpf_insn_successors(struct bpf_verifier_env *env, u32 idx) struct bpf_iarray *succ, *jt; int insn_sz; - jt = env->insn_aux_data[idx].jt; - if (unlikely(jt)) - return jt; - /* pre-allocated array of size up to 2; reset cnt, as it may have been used already */ succ = env->succ; succ->cnt = 0; + /* All gotox of a subprogram share its jump table, see compute_subprog_jts() */ + if (unlikely(insn_is_gotox(insn))) { + jt = bpf_find_containing_subprog(env, idx)->jt; + if (verifier_bug_if(!jt, env, "no jump table for gotox insn %u", idx)) + return succ; + return jt; + } + opcode_info = &opcode_info_tbl[BPF_CLASS(insn->code) | BPF_OP(insn->code)]; insn_sz = bpf_is_ldimm64(insn) ? 2 : 1; if (opcode_info->can_fallthrough) @@ -264,6 +390,20 @@ bpf_insn_successors(struct bpf_verifier_env *env, u32 idx) if (opcode_info->can_jump) succ->items[succ->cnt++] = idx + bpf_jmp_offset(insn) + 1; + /* + * tail_call upon success and ld_{abs,ind} upon load failure return + * from the subprogram, which is a hidden edge to its exit. A subprogram + * without an exit insn has no exit to model. + */ + if (unlikely((bpf_helper_call(insn) && insn->imm == BPF_FUNC_tail_call) || + (BPF_CLASS(insn->code) == BPF_LD && + (BPF_MODE(insn->code) == BPF_ABS || BPF_MODE(insn->code) == BPF_IND)))) { + u32 exit_idx = bpf_find_containing_subprog(env, idx)->exit_idx; + + if (exit_idx != U32_MAX) + succ->items[succ->cnt++] = exit_idx; + } + return succ; } @@ -272,33 +412,42 @@ __diag_pop(); static inline bool update_insn(struct bpf_verifier_env *env, struct func_instance *instance, u32 frame, u32 insn_idx) { - spis_t new_before, new_after; - struct per_frame_masks *insn, *succ_insn; + unsigned long new_after[FRAME_MAX_WORDS] = {}; + unsigned long *may_read, *must_write, *live_before; + struct frame_masks *fm = instance->frames[frame]; + u32 rel = relative_idx(instance, insn_idx); struct bpf_iarray *succ; - u32 s; - bool changed; + bool changed = false; + u32 s, w; succ = bpf_insn_successors(env, insn_idx); if (succ->cnt == 0) return false; - changed = false; - insn = get_frame_masks(instance, frame, insn_idx); - new_before = SPIS_ZERO; - new_after = SPIS_ZERO; + /* All instructions of one frame array share the same mask width */ for (s = 0; s < succ->cnt; ++s) { - succ_insn = get_frame_masks(instance, frame, succ->items[s]); - new_after = spis_or(new_after, succ_insn->live_before); + unsigned long *succ_live; + + succ_live = rel_mask(fm, relative_idx(instance, succ->items[s]), FM_LIVE_BEFORE); + for (w = 0; w < fm->words; w++) + new_after[w] |= succ_live[w]; } + may_read = rel_mask(fm, rel, FM_MAY_READ); + must_write = rel_mask(fm, rel, FM_MUST_WRITE); + live_before = rel_mask(fm, rel, FM_LIVE_BEFORE); /* * New "live_before" is a union of all "live_before" of successors * minus slots written by instruction plus slots read by instruction. * new_before = (new_after & ~insn->must_write) | insn->may_read */ - new_before = spis_or(spis_and(new_after, spis_not(insn->must_write)), - insn->may_read); - changed |= !spis_equal(new_before, insn->live_before); - insn->live_before = new_before; + for (w = 0; w < fm->words; w++) { + unsigned long new_before = (new_after[w] & ~must_write[w]) | may_read[w]; + + if (new_before != live_before[w]) { + live_before[w] = new_before; + changed = true; + } + } return changed; } @@ -329,10 +478,12 @@ static void update_instance(struct bpf_verifier_env *env, struct func_instance * static bool is_live_before(struct func_instance *instance, u32 insn_idx, u32 frameno, u32 half_spi) { - struct per_frame_masks *masks; + struct frame_masks *fm = instance->frames[frameno]; - masks = get_frame_masks(instance, frameno, insn_idx); - return masks && spis_test_bit(masks->live_before, half_spi); + /* No recorded access reaches past the masks, so nothing there is live */ + if (!fm || half_spi >= frame_mask_bits(fm)) + return false; + return test_bit(half_spi, rel_mask(fm, relative_idx(instance, insn_idx), FM_LIVE_BEFORE)); } int bpf_live_stack_query_init(struct bpf_verifier_env *env, struct bpf_verifier_state *st) @@ -356,12 +507,29 @@ int bpf_live_stack_query_init(struct bpf_verifier_env *env, struct bpf_verifier_ return 0; } +/* + * Stack slots of outer frames that the callee may read are accounted as read + * by the @callsite insn itself, i.e. they are live before @callsite and may be + * dead after it, when: + * - @callsite calls a callback, which may be called several times; + * - @callsite is callx. Its callee is not known when stack liveness is + * computed, so there is no func instance for it at @callsite. While + * the verifier is in the callee lookup_instance() finds its standalone + * instance that knows nothing about outer frames. What the callee may read + * there is recorded by record_call_access(). + */ +static bool callee_stack_access_at_callsite(struct bpf_verifier_env *env, u32 callsite) +{ + return bpf_calls_callback(env, callsite) || + bpf_is_callx(&env->prog->insnsi[callsite]); +} + bool bpf_stack_slot_alive(struct bpf_verifier_env *env, u32 frameno, u32 half_spi) { /* * Slot is alive if it is read before q->insn_idx in current func instance, * or if for some outer func instance: - * - alive before callsite if callsite calls callback, otherwise + * - alive before callsite if callsite calls callback or is callx, otherwise * - alive after callsite */ struct live_stack_query *q = &env->liveness->live_stack_query; @@ -394,7 +562,7 @@ bool bpf_stack_slot_alive(struct bpf_verifier_env *env, u32 frameno, u32 half_sp /* Get callsite from verifier state, not from instance callchain */ callsite = q->callsites[i]; - alive = bpf_calls_callback(env, callsite) + alive = callee_stack_access_at_callsite(env, callsite) ? is_live_before(instance, callsite, rel, half_spi) : is_live_before(instance, callsite + 1, rel, half_spi); if (alive) @@ -430,17 +598,19 @@ static int spi_off(int spi) * When only one half is set, print as "-4h","-8h",... * Runs of 3+ consecutive fully-set SPIs are collapsed: "fp0-8..-24" */ -static char *fmt_spis_mask(struct bpf_verifier_env *env, int frame, bool first, spis_t spis) +static char *fmt_spis_mask(struct bpf_verifier_env *env, int frame, bool first, + const unsigned long *spis, u32 words) { int buf_sz = sizeof(env->tmp_str_buf); + int spi_cnt = words * BITS_PER_LONG / 2; char *buf = env->tmp_str_buf; int spi, n, run_start; buf[0] = '\0'; - for (spi = 0; spi < STACK_SLOTS / 2 && buf_sz > 0; spi++) { - bool lo = spis_test_bit(spis, spi * 2); - bool hi = spis_test_bit(spis, spi * 2 + 1); + for (spi = 0; spi < spi_cnt && buf_sz > 0; spi++) { + bool lo = test_bit(spi * 2, spis); + bool hi = test_bit(spi * 2 + 1, spis); const char *space = first ? "" : " "; if (!lo && !hi) @@ -450,16 +620,16 @@ static char *fmt_spis_mask(struct bpf_verifier_env *env, int frame, bool first, /* half-spi */ n = scnprintf(buf, buf_sz, "%sfp%d%d%s", space, frame, spi_off(spi) + (lo ? STACK_SLOT_SZ : 0), "h"); - } else if (spi + 2 < STACK_SLOTS / 2 && - spis_test_bit(spis, spi * 2 + 2) && - spis_test_bit(spis, spi * 2 + 3) && - spis_test_bit(spis, spi * 2 + 4) && - spis_test_bit(spis, spi * 2 + 5)) { + } else if (spi + 2 < spi_cnt && + test_bit(spi * 2 + 2, spis) && + test_bit(spi * 2 + 3, spis) && + test_bit(spi * 2 + 4, spis) && + test_bit(spi * 2 + 5, spis)) { /* 3+ consecutive full spis */ run_start = spi; - while (spi + 1 < STACK_SLOTS / 2 && - spis_test_bit(spis, (spi + 1) * 2) && - spis_test_bit(spis, (spi + 1) * 2 + 1)) + while (spi + 1 < spi_cnt && + test_bit((spi + 1) * 2, spis) && + test_bit((spi + 1) * 2 + 1, spis)) spi++; n = scnprintf(buf, buf_sz, "%sfp%d%d..%d", space, frame, spi_off(run_start), spi_off(spi)); @@ -474,15 +644,41 @@ static char *fmt_spis_mask(struct bpf_verifier_env *env, int frame, bool first, return env->tmp_str_buf; } +/* Print mask @kind of the instruction at relative index @i for every frame, if any bit is set. */ +static bool print_mask(struct bpf_verifier_env *env, struct func_instance *instance, int i, + const char *name, u32 kind) +{ + struct frame_masks *fm; + bool printed = false; + unsigned long *mask; + int frame; + u64 pos; + + pos = env->log.end_pos; + verbose(env, "%s", name); + for (frame = instance->depth; frame >= 0; --frame) { + fm = instance->frames[frame]; + if (!fm) + continue; + mask = rel_mask(fm, i, kind); + if (bitmap_empty(mask, frame_mask_bits(fm))) + continue; + verbose(env, "%s", fmt_spis_mask(env, frame, !printed, mask, fm->words)); + printed = true; + } + if (!printed) + bpf_vlog_reset(&env->log, pos); + return printed; +} + static void print_instance(struct bpf_verifier_env *env, struct func_instance *instance) { int start = env->subprog_info[instance->subprog].start; struct bpf_insn *insns = env->prog->insnsi; - struct per_frame_masks *masks; int len = instance->insn_cnt; - int insn_idx, frame, i; - bool has_use, has_def; u64 pos, insn_pos; + int insn_idx, i; + bool printed; if (!(env->log.level & BPF_LOG_LEVEL2)) return; @@ -491,35 +687,17 @@ static void print_instance(struct bpf_verifier_env *env, struct func_instance *i verbose(env, "%s:\n", fmt_instance(env, instance)); for (i = 0; i < len; i++) { insn_idx = start + i; - has_use = false; - has_def = false; pos = env->log.end_pos; verbose(env, "%3d: ", insn_idx); bpf_verbose_insn(env, &insns[insn_idx]); insn_pos = env->log.end_pos; verbose(env, "%*c;", bpf_vlog_alignment(insn_pos - pos), ' '); - pos = env->log.end_pos; - verbose(env, " use: "); - for (frame = instance->depth; frame >= 0; --frame) { - masks = get_frame_masks(instance, frame, insn_idx); - if (!masks || spis_is_zero(masks->may_read)) - continue; - verbose(env, "%s", fmt_spis_mask(env, frame, !has_use, masks->may_read)); - has_use = true; - } - if (!has_use) - bpf_vlog_reset(&env->log, pos); - pos = env->log.end_pos; - verbose(env, " def: "); - for (frame = instance->depth; frame >= 0; --frame) { - masks = get_frame_masks(instance, frame, insn_idx); - if (!masks || spis_is_zero(masks->must_write)) - continue; - verbose(env, "%s", fmt_spis_mask(env, frame, !has_def, masks->must_write)); - has_def = true; - } - if (!has_def) - bpf_vlog_reset(&env->log, has_use ? pos : insn_pos); + printed = false; + printed |= print_mask(env, instance, i, " use: ", FM_MAY_READ); + printed |= print_mask(env, instance, i, " def: ", FM_MUST_WRITE); + printed |= print_mask(env, instance, i, " may_def: ", FM_MAY_WRITE); + if (!printed) + bpf_vlog_reset(&env->log, insn_pos); verbose(env, "\n"); if (bpf_is_ldimm64(&insns[insn_idx])) i++; @@ -540,6 +718,49 @@ static int cmp_instances(const void *pa, const void *pb) return 0; } +/* OR the 8-byte slots touched by a half-slot (4-byte) mask into @slots. */ +static void half_spis_to_slots(unsigned long *slots, const unsigned long *mask, u32 nbits) +{ + u32 slot; + + for (slot = 0; slot < MAX_BPF_STACK_SLOTS && slot * 2 + 1 < nbits; slot++) + if (test_bit(slot * 2, mask) || test_bit(slot * 2 + 1, mask)) + __set_bit(slot, slots); +} + +/* + * Precompute, for each instruction, the OR of may_write and live_before masks + * over its top frame across all func_instances reaching it, stash them in the + * insn_aux_data. + */ +static void compute_may_write_masks(struct bpf_verifier_env *env) +{ + struct bpf_insn_aux_data *aux = env->insn_aux_data; + struct bpf_liveness *liveness = env->liveness; + struct func_instance *instance; + struct frame_masks *fm; + u32 nbits; + int bkt, i; + + hash_for_each(liveness->func_instances, bkt, instance, hl_node) { + fm = instance->frames[instance->depth]; + if (!fm) + continue; + nbits = frame_mask_bits(fm); + for (i = 0; i < instance->insn_cnt; i++) { + half_spis_to_slots(aux[instance->subprog_start + i].may_write_mask, + rel_mask(fm, i, FM_MAY_WRITE), nbits); + half_spis_to_slots(aux[instance->subprog_start + i].live_stack_before, + rel_mask(fm, i, FM_LIVE_BEFORE), nbits); + } + } +} + +const unsigned long *bpf_may_write_mask(struct bpf_verifier_env *env, u32 insn_idx) +{ + return env->insn_aux_data[insn_idx].may_write_mask; +} + /* print use/def slots for all instances ordered by callsite first, then by depth */ static int print_instances(struct bpf_verifier_env *env) { @@ -584,9 +805,9 @@ static int print_instances(struct bpf_verifier_env *env) * - same frame + different offset -> offset-imprecise * - different frames -> fully-imprecise (bitmask OR) * - * At memory access sites (LDX/STX/ST), offset-imprecise marks only - * the known frame's access mask as SPIS_ALL, while fully-imprecise - * iterates bits in the bitmask and routes each frame to its target. + * At memory access sites (LDX/STX/ST), offset-imprecise marks the known + * frame as fully read, while fully-imprecise iterates bits in the bitmask + * and routes each frame to its target. */ #define MAX_ARG_OFFSETS 4 @@ -605,8 +826,15 @@ enum arg_track_state { ARG_IMPRECISE = -3, /* lost identity; .mask is arg bitmask */ }; -/* Track callee stack slots fp-8 through fp-512 (64 slots of 8 bytes each) */ -#define MAX_ARG_SPILL_SLOTS 64 +/* + * The tracker follows as many spill slots as a subprog addresses directly + * through R10, see subprog_spill_slots(). A frame's slots at a call site are + * kept in a snapshot for the callees that receive pointers into that frame. + */ +struct spill_snapshot { + u32 slots; + struct arg_track at[]; +}; /* * Combined register + stack arg tracking: R0-R10 at indices 0-10, @@ -881,13 +1109,60 @@ static void arg_padd(struct arg_track *at, s64 delta) } /* - * Convert a byte offset from FP to a callee stack slot index. + * Slots the spill tracker may follow for a subprog of @len instructions + * without its per-instruction tables costing more than they could for the + * largest program while every frame stayed within MAX_BPF_STACK: as many + * entries as 64 slots need for BPF_COMPLEXITY_LIMIT_INSNS instructions. + */ +static u32 spill_slots_affordable(int len) +{ + u32 base = MAX_BPF_STACK / BPF_REG_SIZE; + + return max_t(u32, base, base * BPF_COMPLEXITY_LIMIT_INSNS / len); +} + +/* + * Number of 8-byte spill slots to track for the instructions in [@start, @end): + * the 64 slots of a MAX_BPF_STACK frame, which the tracker has always + * followed, or the deepest 8-byte stack access made directly through R10 when + * that reaches further, up to the program's stack budget. Spills and fills are + * compiled as direct R10 accesses, so the count covers them whatever the frame + * size; a fill through a derived pointer that reaches past it is treated as + * imprecise, as one past the frame always was. + */ +static u32 subprog_spill_slots(struct bpf_verifier_env *env, int start, int end) +{ + struct bpf_insn *insns = env->prog->insnsi; + int deepest = 0; + + for (int i = start; i < end; i++) { + struct bpf_insn *insn = &insns[i]; + u8 class = BPF_CLASS(insn->code); + u8 base; + + if (BPF_SIZE(insn->code) != BPF_DW) + continue; + if (class == BPF_LDX) + base = insn->src_reg; + else if (class == BPF_STX || class == BPF_ST) + base = insn->dst_reg; + else + continue; + if (base == BPF_REG_FP && insn->off < 0) + deepest = max(deepest, -insn->off); + } + return clamp_t(u32, DIV_ROUND_UP(deepest, 8), MAX_BPF_STACK / BPF_REG_SIZE, + env->stack_limit / BPF_REG_SIZE); +} + +/* + * Convert a byte offset from FP to one of the @nslots tracked stack slots. * Returns -1 if out of range or not 8-byte aligned. * Slot 0 = fp-8, slot 1 = fp-16, ..., slot 7 = fp-64, .... */ -static int fp_off_to_slot(s16 off) +static int fp_off_to_slot(s16 off, u32 nslots) { - if (off >= 0 || off < -(int)(MAX_ARG_SPILL_SLOTS * 8)) + if (off >= 0 || off < -(int)(nslots * 8)) return -1; if (off % 8) return -1; @@ -896,7 +1171,7 @@ static int fp_off_to_slot(s16 off) static struct arg_track fill_from_stack(struct bpf_insn *insn, struct arg_track *at_out, int reg, - struct arg_track *at_stack_out, + struct arg_track *at_stack_out, u32 nslots, int depth) { struct arg_track imp = { @@ -907,7 +1182,7 @@ static struct arg_track fill_from_stack(struct bpf_insn *insn, int cnt, i; if (reg == BPF_REG_FP) { - int slot = fp_off_to_slot(insn->off); + int slot = fp_off_to_slot(insn->off, nslots); return slot >= 0 ? at_stack_out[slot] : imp; } @@ -920,7 +1195,7 @@ static struct arg_track fill_from_stack(struct bpf_insn *insn, if (arg_add(at_out[reg].off[i], insn->off, &fp_off)) return imp; - slot = fp_off_to_slot(fp_off); + slot = fp_off_to_slot(fp_off, nslots); if (slot < 0) return imp; result = __arg_track_join(result, at_stack_out[slot]); @@ -935,7 +1210,7 @@ static struct arg_track fill_from_stack(struct bpf_insn *insn, * When exact offset is unknown conservatively add reg values to all slots in at_stack_out. */ static void spill_to_stack(struct bpf_insn *insn, struct arg_track *at_out, - int reg, struct arg_track *at_stack_out, + int reg, struct arg_track *at_stack_out, u32 nslots, struct arg_track *val, u32 sz) { struct arg_track none = { .frame = ARG_NONE }; @@ -943,7 +1218,7 @@ static void spill_to_stack(struct bpf_insn *insn, struct arg_track *at_out, int cnt, i; if (reg == BPF_REG_FP) { - int slot = fp_off_to_slot(insn->off); + int slot = fp_off_to_slot(insn->off, nslots); if (slot >= 0) at_stack_out[slot] = new_val; @@ -951,7 +1226,7 @@ static void spill_to_stack(struct bpf_insn *insn, struct arg_track *at_out, } cnt = at_out[reg].off_cnt; if (cnt == 0) { - for (int slot = 0; slot < MAX_ARG_SPILL_SLOTS; slot++) + for (int slot = 0; slot < nslots; slot++) at_stack_out[slot] = __arg_track_join(at_stack_out[slot], new_val); return; } @@ -961,7 +1236,7 @@ static void spill_to_stack(struct bpf_insn *insn, struct arg_track *at_out, if (arg_add(at_out[reg].off[i], insn->off, &fp_off)) continue; - slot = fp_off_to_slot(fp_off); + slot = fp_off_to_slot(fp_off, nslots); if (slot < 0) continue; if (cnt == 1) @@ -975,16 +1250,17 @@ static void spill_to_stack(struct bpf_insn *insn, struct arg_track *at_out, * Clear all tracked callee stack slots overlapping the byte range * [off, off+sz-1] where off is a negative FP-relative offset. */ -static void clear_overlapping_stack_slots(struct arg_track *at_stack, s16 off, u32 sz, int cnt) +static void clear_overlapping_stack_slots(struct arg_track *at_stack, u32 nslots, s16 off, + u32 sz, int cnt) { struct arg_track none = { .frame = ARG_NONE }; if (cnt == 0) { - for (int i = 0; i < MAX_ARG_SPILL_SLOTS; i++) + for (int i = 0; i < nslots; i++) at_stack[i] = __arg_track_join(at_stack[i], none); return; } - for (int i = 0; i < MAX_ARG_SPILL_SLOTS; i++) { + for (int i = 0; i < nslots; i++) { int slot_start = -((i + 1) * 8); int slot_end = slot_start + 8; @@ -1002,33 +1278,33 @@ static void clear_overlapping_stack_slots(struct arg_track *at_stack, s16 off, u */ static void clear_stack_for_all_offs(struct bpf_insn *insn, struct arg_track *at_out, int reg, - struct arg_track *at_stack_out, u32 sz) + struct arg_track *at_stack_out, u32 nslots, u32 sz) { int cnt, i; if (reg == BPF_REG_FP) { - clear_overlapping_stack_slots(at_stack_out, insn->off, sz, 1); + clear_overlapping_stack_slots(at_stack_out, nslots, insn->off, sz, 1); return; } cnt = at_out[reg].off_cnt; if (cnt == 0) { - clear_overlapping_stack_slots(at_stack_out, 0, sz, cnt); + clear_overlapping_stack_slots(at_stack_out, nslots, 0, sz, cnt); return; } for (i = 0; i < cnt; i++) { s16 fp_off; if (arg_add(at_out[reg].off[i], insn->off, &fp_off)) { - clear_overlapping_stack_slots(at_stack_out, 0, sz, 0); + clear_overlapping_stack_slots(at_stack_out, nslots, 0, sz, 0); break; } - clear_overlapping_stack_slots(at_stack_out, fp_off, sz, cnt); + clear_overlapping_stack_slots(at_stack_out, nslots, fp_off, sz, cnt); } } static void arg_track_log(struct bpf_verifier_env *env, struct bpf_insn *insn, int idx, struct arg_track *at_in, struct arg_track *at_stack_in, - struct arg_track *at_out, struct arg_track *at_stack_out) + struct arg_track *at_out, struct arg_track *at_stack_out, u32 nslots) { bool printed = false; int i; @@ -1060,7 +1336,7 @@ static void arg_track_log(struct bpf_verifier_env *env, struct bpf_insn *insn, i verbose(env, "\tsa%d: ", i); verbose_arg_track(env, &at_in[ai]); verbose(env, " -> "); verbose_arg_track(env, &at_out[ai]); } - for (i = 0; i < MAX_ARG_SPILL_SLOTS; i++) { + for (i = 0; i < nslots; i++) { if (arg_track_eq(&at_stack_out[i], &at_stack_in[i])) continue; if (!printed) { @@ -1081,6 +1357,29 @@ static bool can_be_local_fp(int depth, int regno, struct arg_track *at) (at->frame == ARG_IMPRECISE && (at->mask & BIT(depth))); } +/* + * Model 8-byte reads through @reg, including load-acquire + * and the old value fetched by atomic operations. + */ +static struct arg_track arg_track_deref(struct bpf_verifier_env *env, struct bpf_insn *insn, + struct arg_track *at_out, int reg, + struct arg_track *at_stack_out, u32 nslots, + int depth, u32 *callsites) +{ + struct arg_track *ptr = &at_out[reg]; + + if (can_be_local_fp(depth, reg, ptr)) + return fill_from_stack(insn, at_out, reg, at_stack_out, nslots, depth); + if (ptr->frame >= 0 && ptr->frame < depth) { + struct spill_snapshot *parent = env->callsite_at_stack[callsites[ptr->frame]]; + + return fill_from_stack(insn, at_out, reg, parent->at, parent->slots, ptr->frame); + } + if (ptr->frame == ARG_IMPRECISE && !(ptr->mask & BIT(depth)) && ptr->mask) + return *ptr; + return (struct arg_track){ .frame = ARG_NONE }; +} + /* * Pure dataflow transfer function for arg_track state. * Updates at_out[] based on how the instruction modifies registers. @@ -1088,7 +1387,7 @@ static bool can_be_local_fp(int depth, int regno, struct arg_track *at) */ static void arg_track_xfer(struct bpf_verifier_env *env, struct bpf_insn *insn, int insn_idx, - struct arg_track *at_out, struct arg_track *at_stack_out, + struct arg_track *at_out, struct arg_track *at_stack_out, u32 nslots, const struct arg_track *at_stack_arg_entry, struct func_instance *instance, u32 *callsites) @@ -1162,55 +1461,42 @@ static void arg_track_xfer(struct bpf_verifier_env *env, struct bpf_insn *insn, for (r = BPF_REG_0; r <= BPF_REG_5; r++) at_out[r] = none; } else if (class == BPF_LDX) { - u32 sz = bpf_size_to_bytes(BPF_SIZE(insn->code)); - bool src_is_local_fp = can_be_local_fp(depth, insn->src_reg, src); - - /* - * Reload from callee stack: if src is current-frame FP-derived - * and the load is an 8-byte BPF_MEM, try to restore the spill - * identity. For imprecise sources fill_from_stack() returns - * ARG_IMPRECISE (off_cnt == 0). - */ - if (src_is_local_fp && BPF_MODE(insn->code) == BPF_MEM && sz == 8) { - *dst = fill_from_stack(insn, at_out, insn->src_reg, at_stack_out, depth); - } else if (src->frame >= 0 && src->frame < depth && - BPF_MODE(insn->code) == BPF_MEM && sz == 8) { - struct arg_track *parent_stack = - env->callsite_at_stack[callsites[src->frame]]; - - *dst = fill_from_stack(insn, at_out, insn->src_reg, - parent_stack, src->frame); - } else if (src->frame == ARG_IMPRECISE && - !(src->mask & BIT(depth)) && src->mask && - BPF_MODE(insn->code) == BPF_MEM && sz == 8) { - /* - * Imprecise src with only parent-frame bits: - * conservative fallback. - */ - *dst = *src; - } else { + /* Only an 8-byte BPF_MEM load can be a fill, see arg_track_deref(). */ + if (BPF_MODE(insn->code) == BPF_MEM && BPF_SIZE(insn->code) == BPF_DW) + *dst = arg_track_deref(env, insn, at_out, insn->src_reg, + at_stack_out, nslots, depth, callsites); + else *dst = none; - } } else if (class == BPF_LD && BPF_MODE(insn->code) == BPF_IMM) { *dst = none; } else if (class == BPF_STX) { u32 sz = bpf_size_to_bytes(BPF_SIZE(insn->code)); - bool dst_is_local_fp; + bool dst_is_local_fp = can_be_local_fp(depth, insn->dst_reg, dst); + u8 mode = BPF_MODE(insn->code); - /* Track spills to current-frame FP-derived callee stack */ - dst_is_local_fp = can_be_local_fp(depth, insn->dst_reg, dst); - if (dst_is_local_fp && BPF_MODE(insn->code) == BPF_MEM) - spill_to_stack(insn, at_out, insn->dst_reg, - at_stack_out, src, sz); - - if (BPF_MODE(insn->code) == BPF_ATOMIC) { - if (dst_is_local_fp && insn->imm != BPF_LOAD_ACQ) - clear_stack_for_all_offs(insn, at_out, insn->dst_reg, - at_stack_out, sz); + if (mode == BPF_MEM || (mode == BPF_ATOMIC && insn->imm == BPF_STORE_REL)) { + if (dst_is_local_fp) + spill_to_stack(insn, at_out, insn->dst_reg, + at_stack_out, nslots, src, sz); + } else if (mode == BPF_ATOMIC && insn->imm == BPF_LOAD_ACQ) { + if (sz == 8) + *dst = arg_track_deref(env, insn, at_out, insn->src_reg, + at_stack_out, nslots, depth, callsites); + else + *dst = none; + } else if (mode == BPF_ATOMIC) { + /* BPF_XCHG, BPF_CMPXCHG or BPF_{ADD,AND,OR,XOR} | [BPF_FETCH] */ + struct arg_track old = none; r = bpf_atomic_load_reg(insn); + if (r >= 0 && sz == 8) + old = arg_track_deref(env, insn, at_out, insn->dst_reg, + at_stack_out, nslots, depth, callsites); + if (dst_is_local_fp) + clear_stack_for_all_offs(insn, at_out, insn->dst_reg, + at_stack_out, nslots, sz); if (r >= 0) - at_out[r] = none; + at_out[r] = old; } } else if (class == BPF_ST && BPF_MODE(insn->code) == BPF_MEM) { u32 sz = bpf_size_to_bytes(BPF_SIZE(insn->code)); @@ -1219,23 +1505,21 @@ static void arg_track_xfer(struct bpf_verifier_env *env, struct bpf_insn *insn, /* BPF_ST to FP-derived dst: clear overlapping stack slots */ if (dst_is_local_fp) clear_stack_for_all_offs(insn, at_out, insn->dst_reg, - at_stack_out, sz); + at_stack_out, nslots, sz); } } /* - * Record access_bytes from helper/kfunc or load/store insn. - * access_bytes > 0: stack read - * access_bytes < 0: stack write - * access_bytes == S64_MIN: unknown — conservative, mark [0..slot] as read - * access_bytes == 0: no access - * + * Record possible reads and writes for every touched half-slot. A definite + * write requires full coverage, a known size, and a single possible offset. */ -static int record_stack_access_off(struct func_instance *instance, s64 fp_off, - s64 access_bytes, u32 frame, u32 insn_idx) +static int record_stack_access_off(struct func_instance *instance, const struct arg_track *arg, + s64 off_idx, struct arg_access_info info, + u32 frame, u32 insn_idx) { + s64 fp_off = arg->off[off_idx]; s32 slot_hi, slot_lo; - spis_t mask; + int err; if (fp_off >= 0) /* @@ -1244,68 +1528,68 @@ static int record_stack_access_off(struct func_instance *instance, s64 fp_off, * by the main verifier pass later. */ return 0; - if (access_bytes == S64_MIN) { - /* helper/kfunc read unknown amount of bytes from fp_off until fp+0 */ - slot_hi = (-fp_off - 1) / STACK_SLOT_SZ; - mask = SPIS_ZERO; - spis_or_range(&mask, 0, slot_hi); - return mark_stack_read(instance, frame, insn_idx, mask); + + /* + * Read and may_write marks include partially covered slots. + * An unknown access size may reach from fp_off to the frame top. + */ + slot_hi = (-fp_off - 1) / STACK_SLOT_SZ; + slot_lo = info.size == U32_MAX + ? 0 + : max_t(s32, (-fp_off - info.size) / STACK_SLOT_SZ, 0); + if (info.may_read) { + err = mark_stack_read(instance, frame, insn_idx, slot_lo, slot_hi); + if (err) + return err; } - if (access_bytes > 0) { - /* Mark any touched slot as use */ - slot_hi = (-fp_off - 1) / STACK_SLOT_SZ; - slot_lo = max_t(s32, (-fp_off - access_bytes) / STACK_SLOT_SZ, 0); - mask = SPIS_ZERO; - spis_or_range(&mask, slot_lo, slot_hi); - return mark_stack_read(instance, frame, insn_idx, mask); - } else if (access_bytes < 0) { + if (info.may_write) { + err = mark_stack_may_write(instance, frame, insn_idx, slot_lo, slot_hi); + if (err) + return err; + } + if (info.must_write && info.size != U32_MAX && arg->off_cnt == 1) { /* Mark only fully covered slots as def */ - access_bytes = -access_bytes; slot_hi = (-fp_off) / STACK_SLOT_SZ - 1; - slot_lo = max_t(s32, (-fp_off - access_bytes + STACK_SLOT_SZ - 1) / STACK_SLOT_SZ, 0); - if (slot_lo <= slot_hi) { - mask = SPIS_ZERO; - spis_or_range(&mask, slot_lo, slot_hi); - return mark_stack_write(instance, frame, insn_idx, mask); - } + slot_lo = max_t(s32, (-fp_off - info.size + STACK_SLOT_SZ - 1) / STACK_SLOT_SZ, 0); + return mark_stack_write(instance, frame, insn_idx, slot_lo, slot_hi); } return 0; } -/* - * 'arg' is FP-derived argument to helper/kfunc or load/store that - * reads (positive) or writes (negative) 'access_bytes' into 'use' or 'def'. - */ -static int record_stack_access(struct func_instance *instance, +/* Record access through a pointer with a known frame and possibly known offsets. */ +static int record_stack_access(struct bpf_verifier_env *env, + struct func_instance *instance, const struct arg_track *arg, - s64 access_bytes, u32 frame, u32 insn_idx) + struct arg_access_info info, u32 frame, u32 insn_idx) { int i, err; - if (access_bytes == 0) + if (!info.size) return 0; if (arg->off_cnt == 0) { - if (access_bytes > 0 || access_bytes == S64_MIN) - return mark_stack_read(instance, frame, insn_idx, SPIS_ALL); + if (info.may_read) { + err = mark_stack_read_all(env, instance, frame, insn_idx); + if (err) + return err; + } + if (info.may_write) { + err = mark_stack_may_write_all(env, instance, frame, insn_idx); + if (err) + return err; + } return 0; } - if (access_bytes != S64_MIN && access_bytes < 0 && arg->off_cnt != 1) - /* multi-offset write cannot set stack_def */ - return 0; - for (i = 0; i < arg->off_cnt; i++) { - err = record_stack_access_off(instance, arg->off[i], access_bytes, frame, insn_idx); + err = record_stack_access_off(instance, arg, i, info, frame, insn_idx); if (err) return err; } return 0; } -/* - * When a pointer is ARG_IMPRECISE, conservatively mark every frame in - * the bitmask as fully used. - */ -static int record_imprecise(struct func_instance *instance, u32 mask, u32 insn_idx) +/* Record possible effects on every candidate frame of an imprecise pointer. */ +static int record_imprecise(struct bpf_verifier_env *env, struct func_instance *instance, + struct arg_access_info info, u32 mask, u32 insn_idx) { int depth = instance->depth; int f, err; @@ -1314,9 +1598,16 @@ static int record_imprecise(struct func_instance *instance, u32 mask, u32 insn_i if (!(mask & 1)) continue; if (f <= depth) { - err = mark_stack_read(instance, f, insn_idx, SPIS_ALL); - if (err) - return err; + if (info.may_read) { + err = mark_stack_read_all(env, instance, f, insn_idx); + if (err) + return err; + } + if (info.may_write) { + err = mark_stack_may_write_all(env, instance, f, insn_idx); + if (err) + return err; + } } } return 0; @@ -1328,9 +1619,10 @@ static int record_load_store_access(struct bpf_verifier_env *env, struct arg_track *at, int insn_idx) { struct bpf_insn *insn = &env->prog->insnsi[insn_idx]; + struct arg_access_info info = {}; int depth = instance->depth; - s32 sz = bpf_size_to_bytes(BPF_SIZE(insn->code)); u8 class = BPF_CLASS(insn->code); + bool read = false, write = false; struct arg_track resolved, *ptr; int oi; @@ -1347,23 +1639,34 @@ static int record_load_store_access(struct bpf_verifier_env *env, switch (class) { case BPF_LDX: ptr = &at[insn->src_reg]; + read = true; break; case BPF_STX: if (BPF_MODE(insn->code) == BPF_ATOMIC) { - if (insn->imm == BPF_STORE_REL) - sz = -sz; - if (insn->imm == BPF_LOAD_ACQ) + switch (insn->imm) { + case BPF_LOAD_ACQ: ptr = &at[insn->src_reg]; - else + read = true; + break; + case BPF_STORE_REL: ptr = &at[insn->dst_reg]; + write = true; + break; + default: + /* ADD/AND/OR/XOR(+FETCH), XCHG, CMPXCHG */ + ptr = &at[insn->dst_reg]; + read = true; + write = true; + break; + } } else { ptr = &at[insn->dst_reg]; - sz = -sz; + write = true; } break; case BPF_ST: ptr = &at[insn->dst_reg]; - sz = -sz; + write = true; break; default: return 0; @@ -1382,10 +1685,14 @@ static int record_load_store_access(struct bpf_verifier_env *env, ptr = &resolved; } + info.may_read = read; + info.may_write = write; + info.must_write = write; + info.size = bpf_size_to_bytes(BPF_SIZE(insn->code)); if (ptr->frame >= 0 && ptr->frame <= depth) - return record_stack_access(instance, ptr, sz, ptr->frame, insn_idx); + return record_stack_access(env, instance, ptr, info, ptr->frame, insn_idx); if (ptr->frame == ARG_IMPRECISE) - return record_imprecise(instance, ptr->mask, insn_idx); + return record_imprecise(env, instance, info, ptr->mask, insn_idx); /* ARG_NONE: not derived from any frame pointer, skip */ return 0; } @@ -1396,37 +1703,42 @@ static int record_arg_access(struct bpf_verifier_env *env, struct arg_track *at, int arg_idx, int insn_idx) { + struct arg_access_info info; int depth = instance->depth; int frame = at->frame; int err = 0; - s64 bytes; if (!arg_is_fp(at)) return 0; if (bpf_helper_call(insn)) { - bytes = bpf_helper_stack_access_bytes(env, insn, arg_idx, insn_idx); + info = bpf_helper_stack_access_bytes(env, insn, arg_idx, insn_idx); } else if (bpf_pseudo_kfunc_call(insn)) { - bytes = bpf_kfunc_stack_access_bytes(env, insn, arg_idx, insn_idx); + info = bpf_kfunc_stack_access_bytes(env, insn, arg_idx, insn_idx); + } else if (bpf_pseudo_call(insn)) { + info = bpf_global_subprog_stack_access_bytes(env, insn, arg_idx, insn_idx); } else { for (int f = 0; f <= depth; f++) { - err = mark_stack_read(instance, f, insn_idx, SPIS_ALL); + err = mark_stack_read_all(env, instance, f, insn_idx); + if (err) + return err; + err = mark_stack_may_write_all(env, instance, f, insn_idx); if (err) return err; } return 0; } - if (bytes == 0) + if (!info.size) return 0; if (frame >= 0 && frame <= depth) - err = record_stack_access(instance, at, bytes, frame, insn_idx); + err = record_stack_access(env, instance, at, info, frame, insn_idx); else if (frame == ARG_IMPRECISE) - err = record_imprecise(instance, at->mask, insn_idx); + err = record_imprecise(env, instance, info, at->mask, insn_idx); return err; } -/* Record stack access for a given 'at' state of helper/kfunc 'insn' */ +/* Record stack access for a given 'at' state of helper/kfunc/global subprog 'insn' */ static int record_call_access(struct bpf_verifier_env *env, struct func_instance *instance, struct arg_track *at, @@ -1434,21 +1746,36 @@ static int record_call_access(struct bpf_verifier_env *env, { struct bpf_insn *insn = &env->prog->insnsi[insn_idx]; struct bpf_call_summary cs; - int r, err, num_params = 5; + int r, err, callee, arg_slot_cnt = 5; - if (bpf_pseudo_call(insn)) - return 0; + if (bpf_pseudo_call(insn)) { + /* + * analyze_subprog() handles static callees. For global callees, + * derive effects from argument types so may_write marks remain + * conservative under freplace. + */ + callee = bpf_find_subprog(env, insn_idx + insn->imm + 1); + if (callee < 0 || !bpf_subprog_is_global(env, callee)) + return 0; + arg_slot_cnt = MAX_BPF_FUNC_ARGS; + } else if (bpf_is_callx(insn)) + /* + * The callee is not known statically. Assume that all arg + * slots are passed and let record_arg_access() conservatively + * mark the stack of all frames as read if any of them is + * derived from a frame pointer. + */ + arg_slot_cnt = MAX_BPF_FUNC_REG_ARGS + MAX_STACK_ARG_SLOTS; + else if (bpf_get_call_summary(env, insn, &cs)) + arg_slot_cnt = cs.arg_slot_cnt; - if (bpf_get_call_summary(env, insn, &cs)) - num_params = cs.num_params; - - for (r = BPF_REG_1; r < BPF_REG_1 + min(num_params, MAX_BPF_FUNC_REG_ARGS); r++) { + for (r = BPF_REG_1; r < BPF_REG_1 + min(arg_slot_cnt, MAX_BPF_FUNC_REG_ARGS); r++) { err = record_arg_access(env, instance, insn, &at[r], r - 1, insn_idx); if (err) return err; } - for (r = 0; r < MAX_STACK_ARG_SLOTS && r < num_params - MAX_BPF_FUNC_REG_ARGS; r++) { + for (r = 0; r < MAX_STACK_ARG_SLOTS && r < arg_slot_cnt - MAX_BPF_FUNC_REG_ARGS; r++) { err = record_arg_access(env, instance, insn, &at[MAX_BPF_REG + r], r + MAX_BPF_FUNC_REG_ARGS, insn_idx); if (err) @@ -1517,7 +1844,7 @@ struct subprog_at_info { static void print_subprog_arg_access(struct bpf_verifier_env *env, int subprog, struct subprog_at_info *info, - struct arg_track (*at_stack_in)[MAX_ARG_SPILL_SLOTS]) + struct arg_track *at_stack_in, u32 nslots) { struct bpf_insn *insns = env->prog->insnsi; int start = env->subprog_info[subprog].start; @@ -1533,7 +1860,7 @@ static void print_subprog_arg_access(struct bpf_verifier_env *env, bool has_extra = false; u8 cls = BPF_CLASS(insns[idx].code); bool is_ldx_stx_call = cls == BPF_LDX || cls == BPF_STX || - insns[idx].code == (BPF_JMP | BPF_CALL); + (cls == BPF_JMP && BPF_OP(insns[idx].code) == BPF_CALL); verbose(env, "%3d: ", idx); bpf_verbose_insn(env, &insns[idx]); @@ -1550,8 +1877,8 @@ static void print_subprog_arg_access(struct bpf_verifier_env *env, has_extra = true; } if (is_ldx_stx_call) { - for (r = 0; r < MAX_ARG_SPILL_SLOTS; r++) - if (arg_is_fp(&at_stack_in[i][r])) + for (r = 0; r < nslots; r++) + if (arg_is_fp(&at_stack_in[(size_t)i * nslots + r])) has_extra = true; } @@ -1581,11 +1908,11 @@ static void print_subprog_arg_access(struct bpf_verifier_env *env, } if (is_ldx_stx_call) { - for (r = 0; r < MAX_ARG_SPILL_SLOTS; r++) { - if (!arg_is_fp(&at_stack_in[i][r])) + for (r = 0; r < nslots; r++) { + if (!arg_is_fp(&at_stack_in[(size_t)i * nslots + r])) continue; verbose(env, " fp%+d=", -(r + 1) * 8); - verbose_arg_track(env, &at_stack_in[i][r]); + verbose_arg_track(env, &at_stack_in[(size_t)i * nslots + r]); } } @@ -1595,6 +1922,17 @@ static void print_subprog_arg_access(struct bpf_verifier_env *env, } } +static void record_stack_ptrs(struct bpf_verifier_env *env, int idx, struct arg_track *at_in) +{ + u16 r, mask = 0; + + for (r = 0; r < MAX_BPF_REG; r++) + if (arg_is_fp(&at_in[r])) + mask |= BIT(r); + + env->insn_aux_data[idx].stack_ptrs |= mask; +} + /* * Compute arg tracking dataflow for a single subprog. * Runs forward fixed-point with arg_track_xfer(), then records @@ -1618,9 +1956,10 @@ static int compute_subprog_args(struct bpf_verifier_env *env, int end = env->subprog_info[subprog + 1].start; int po_end = env->subprog_info[subprog + 1].postorder_start; int len = end - start; + u32 nslots = min(subprog_spill_slots(env, start, end), spill_slots_affordable(len)); struct arg_track (*at_in)[MAX_AT_TRACK_REGS] = NULL; struct arg_track at_out[MAX_AT_TRACK_REGS]; - struct arg_track (*at_stack_in)[MAX_ARG_SPILL_SLOTS] = NULL; + struct arg_track *at_stack_in = NULL; struct arg_track *at_stack_out = NULL; struct arg_track at_stack_arg_entry[MAX_STACK_ARG_SLOTS]; struct arg_track unvisited = { .frame = ARG_UNVISITED }; @@ -1632,19 +1971,19 @@ static int compute_subprog_args(struct bpf_verifier_env *env, if (!at_in) goto err_free; - at_stack_in = kvmalloc_objs(*at_stack_in, len, GFP_KERNEL_ACCOUNT); + at_stack_in = kvmalloc_objs(*at_stack_in, (size_t)len * nslots, GFP_KERNEL_ACCOUNT); if (!at_stack_in) goto err_free; - at_stack_out = kvmalloc_objs(*at_stack_out, MAX_ARG_SPILL_SLOTS, GFP_KERNEL_ACCOUNT); + at_stack_out = kvmalloc_objs(*at_stack_out, nslots, GFP_KERNEL_ACCOUNT); if (!at_stack_out) goto err_free; for (i = 0; i < len; i++) { for (r = 0; r < MAX_AT_TRACK_REGS; r++) at_in[i][r] = unvisited; - for (r = 0; r < MAX_ARG_SPILL_SLOTS; r++) - at_stack_in[i][r] = unvisited; + for (r = 0; r < nslots; r++) + at_stack_in[(size_t)i * nslots + r] = unvisited; } for (r = 0; r < MAX_AT_TRACK_REGS; r++) @@ -1660,8 +1999,8 @@ static int compute_subprog_args(struct bpf_verifier_env *env, } /* Entry: all stack slots are ARG_NONE */ - for (r = 0; r < MAX_ARG_SPILL_SLOTS; r++) - at_stack_in[0][r] = none; + for (r = 0; r < nslots; r++) + at_stack_in[r] = none; /* Entry: incoming stack args from caller, or ARG_NONE for main */ for (r = 0; r < MAX_STACK_ARG_SLOTS; r++) @@ -1683,11 +2022,12 @@ static int compute_subprog_args(struct bpf_verifier_env *env, continue; memcpy(at_out, at_in[i], sizeof(at_out)); - memcpy(at_stack_out, at_stack_in[i], MAX_ARG_SPILL_SLOTS * sizeof(*at_stack_out)); + memcpy(at_stack_out, &at_stack_in[(size_t)i * nslots], nslots * sizeof(*at_stack_out)); - arg_track_xfer(env, insn, idx, at_out, at_stack_out, + arg_track_xfer(env, insn, idx, at_out, at_stack_out, nslots, at_stack_arg_entry, instance, callsites); - arg_track_log(env, insn, idx, at_in[i], at_stack_in[i], at_out, at_stack_out); + arg_track_log(env, insn, idx, at_in[i], &at_stack_in[(size_t)i * nslots], at_out, + at_stack_out, nslots); /* Propagate to successors within this subprogram */ succ = bpf_insn_successors(env, idx); @@ -1704,9 +2044,10 @@ static int compute_subprog_args(struct bpf_verifier_env *env, changed |= arg_track_join(env, idx, target, r, &at_in[ti][r], at_out[r]); - for (r = 0; r < MAX_ARG_SPILL_SLOTS; r++) + for (r = 0; r < nslots; r++) changed |= arg_track_join(env, idx, target, -r - 1, - &at_stack_in[ti][r], at_stack_out[r]); + &at_stack_in[(size_t)ti * nslots + r], + at_stack_out[r]); } } if (changed) @@ -1722,30 +2063,33 @@ static int compute_subprog_args(struct bpf_verifier_env *env, if (err) goto err_free; - if (insn->code == (BPF_JMP | BPF_CALL)) { + if (BPF_CLASS(insn->code) == BPF_JMP && BPF_OP(insn->code) == BPF_CALL) { err = record_call_access(env, instance, at_in[i], idx); if (err) goto err_free; } if (bpf_pseudo_call(insn) || bpf_calls_callback(env, idx)) { + struct spill_snapshot *snap; + kvfree(env->callsite_at_stack[idx]); - env->callsite_at_stack[idx] = - kvmalloc_objs(*env->callsite_at_stack[idx], - MAX_ARG_SPILL_SLOTS, GFP_KERNEL_ACCOUNT); - if (!env->callsite_at_stack[idx]) { + env->callsite_at_stack[idx] = kvmalloc_flex(*snap, at, nslots, GFP_KERNEL_ACCOUNT); + snap = env->callsite_at_stack[idx]; + if (!snap) { err = -ENOMEM; goto err_free; } - memcpy(env->callsite_at_stack[idx], - at_stack_in[i], sizeof(struct arg_track) * MAX_ARG_SPILL_SLOTS); + snap->slots = nslots; + memcpy(snap->at, &at_stack_in[(size_t)i * nslots], nslots * sizeof(*snap->at)); } + + record_stack_ptrs(env, idx, at_in[i]); } info->at_in = at_in; at_in = NULL; info->len = len; - print_subprog_arg_access(env, subprog, info, at_stack_in); + print_subprog_arg_access(env, subprog, info, at_stack_in, nslots); err = 0; err_free: @@ -1770,38 +2114,87 @@ static bool has_fp_args(struct arg_track *args) /* * Merge a freshly analyzed instance into the original. * may_read: union (any pass might read the slot). + * may_write: union (slots written on ANY pass). * must_write: intersection (only slots written on ALL passes are guaranteed). * live_before is recomputed by a subsequent update_instance() on @dst. + * + * The two instances may have settled on different mask widths for the same + * frame, so @dst is widened to cover @src first. A word only @dst has counts + * as zero on the @src side: it unions into may_read as a no-op and intersects + * must_write to empty. */ -static void merge_instances(struct func_instance *dst, struct func_instance *src) +static int merge_instances(struct func_instance *dst, struct func_instance *src) { - int f, i; + struct frame_masks *d, *s; + u32 f, i, w; for (f = 0; f <= dst->depth; f++) { - if (!src->frames[f]) { + s = src->frames[f]; + d = dst->frames[f]; + if (!s) { /* This pass didn't touch frame f — must_write intersects with empty. */ - if (dst->frames[f]) + if (d) for (i = 0; i < dst->insn_cnt; i++) - dst->frames[f][i].must_write = SPIS_ZERO; + bitmap_zero(rel_mask(d, i, FM_MUST_WRITE), + frame_mask_bits(d)); continue; } - if (!dst->frames[f]) { + if (!d) { /* Previous pass didn't touch frame f — take src, zero must_write. */ - dst->frames[f] = src->frames[f]; + dst->frames[f] = s; src->frames[f] = NULL; for (i = 0; i < dst->insn_cnt; i++) - dst->frames[f][i].must_write = SPIS_ZERO; + bitmap_zero(rel_mask(s, i, FM_MUST_WRITE), frame_mask_bits(s)); continue; } + d = widen_frame_masks(dst, f, s->words); + if (!d) + return -ENOMEM; for (i = 0; i < dst->insn_cnt; i++) { - dst->frames[f][i].may_read = - spis_or(dst->frames[f][i].may_read, - src->frames[f][i].may_read); - dst->frames[f][i].must_write = - spis_and(dst->frames[f][i].must_write, - src->frames[f][i].must_write); + unsigned long *dst_read = rel_mask(d, i, FM_MAY_READ); + unsigned long *dst_write = rel_mask(d, i, FM_MUST_WRITE); + unsigned long *dst_may_write = rel_mask(d, i, FM_MAY_WRITE); + unsigned long *src_read = rel_mask(s, i, FM_MAY_READ); + unsigned long *src_write = rel_mask(s, i, FM_MUST_WRITE); + unsigned long *src_may_write = rel_mask(s, i, FM_MAY_WRITE); + + for (w = 0; w < d->words; w++) { + dst_read[w] |= w < s->words ? src_read[w] : 0; + dst_write[w] &= w < s->words ? src_write[w] : 0; + dst_may_write[w] |= w < s->words ? src_may_write[w] : 0; + } } } + return 0; +} + +/* + * Fold a fully analyzed callee instance writes to upper frames as + * may_write marks at callsite in caller's frames. + */ +static int merge_may_write(struct func_instance *caller, struct func_instance *callee) +{ + DECLARE_BITMAP(acc, FRAME_HALF_SPIS); + u32 call_idx = callee->callsite; + struct frame_masks *fm; + u32 f, i, nbits; + int err; + + for (f = 0; f < callee->depth; f++) { + fm = callee->frames[f]; + if (!fm) + continue; + nbits = frame_mask_bits(fm); + bitmap_zero(acc, nbits); + for (i = 0; i < callee->insn_cnt; i++) + bitmap_or(acc, acc, rel_mask(fm, i, FM_MAY_WRITE), nbits); + if (bitmap_empty(acc, nbits)) + continue; + err = mark_stack_mask(caller, f, call_idx, FM_MAY_WRITE, acc, fm->words); + if (err) + return err; + } + return 0; } static struct func_instance *fresh_instance(struct func_instance *src) @@ -1898,7 +2291,12 @@ static int analyze_subprog(struct bpf_verifier_env *env, if (bpf_pseudo_call(insn)) { target = idx + insn->imm + 1; callee = bpf_find_subprog(env, target); - if (callee < 0) + /* + * Global subprograms can be freplaced, so conservatively derive + * their stack effects from their BTF signatures instead of + * analyzing them in the context of their callers. + */ + if (callee < 0 || bpf_subprog_is_global(env, callee)) continue; /* Build entry args: R1-R5 and stack args from at_in at call site */ @@ -1916,7 +2314,10 @@ static int analyze_subprog(struct bpf_verifier_env *env, if (info[subprog].at_in[j][caller_reg].frame == ARG_NONE) continue; for (int f = 0; f <= depth; f++) { - err = mark_stack_read(instance, f, idx, SPIS_ALL); + err = mark_stack_read_all(env, instance, f, idx); + if (err) + goto out_free; + err = mark_stack_may_write_all(env, instance, f, idx); if (err) goto out_free; } @@ -1955,23 +2356,35 @@ static int analyze_subprog(struct bpf_verifier_env *env, /* Pull callee's entry liveness back to caller's callsite */ { u32 callee_start = callee_instance->subprog_start; - struct per_frame_masks *entry; + struct frame_masks *callee_fm; for (int f = 0; f < callee_instance->depth; f++) { - entry = get_frame_masks(callee_instance, f, callee_start); - if (!entry) + callee_fm = callee_instance->frames[f]; + if (!callee_fm) continue; - err = mark_stack_read(instance, f, idx, entry->live_before); + err = mark_stack_read_mask(instance, f, idx, + rel_mask(callee_fm, + relative_idx(callee_instance, + callee_start), + FM_LIVE_BEFORE), + callee_fm->words); if (err) goto out_free; } } + + /* Summarize callee's writes to ancestor frames onto the callsite */ + err = merge_may_write(instance, callee_instance); + if (err) + goto out_free; } if (prev_instance) { - merge_instances(prev_instance, instance); + err = merge_instances(prev_instance, instance); free_instance(instance); instance = prev_instance; + if (err) + return err; } update_instance(env, instance); return 0; @@ -2026,6 +2439,8 @@ int bpf_compute_subprog_arg_access(struct bpf_verifier_env *env) goto out; } + compute_may_write_masks(env); + if (env->log.level & BPF_LOG_LEVEL2) err = print_instances(env); @@ -2057,10 +2472,17 @@ static inline u32 mask_widen(u32 m) { return m | (m << 16); } static inline u16 mask_lo(u32 m) { return (u16)m; } static inline u16 mask_hi(u32 m) { return (u16)(m >> 16); } +/* Index of the subprogram called by pseudo call @insn */ +static int callee_subprog(struct bpf_verifier_env *env, const struct bpf_insn *insn) +{ + return bpf_find_subprog(env, insn - env->prog->insnsi + insn->imm + 1); +} + /* Compute info->{use,def} fields for the instruction */ static void compute_insn_live_regs(struct bpf_verifier_env *env, struct bpf_insn *insn, - struct insn_live_regs *info) + struct insn_live_regs *info, + bool ret_reg_pair) { struct bpf_call_summary cs; const u8 class = BPF_CLASS(insn->code); @@ -2072,6 +2494,8 @@ static void compute_insn_live_regs(struct bpf_verifier_env *env, const u32 src32 = mask_lo(src); const u32 dst32 = mask_lo(dst); const u32 r0 = reg64_mask(0); + const u32 r2 = reg64_mask(BPF_REG_2); + int subprog, reg_args; u32 def = 0; u32 use = U32_MAX; @@ -2191,15 +2615,28 @@ static void compute_insn_live_regs(struct bpf_verifier_env *env, break; case BPF_EXIT: def = 0; - use = r0; + use = ret_reg_pair ? (r0 | r2) : r0; break; case BPF_CALL: def = ALL_CALLER_SAVED_REGS; use = def & ~BIT(BPF_REG_0); - if (bpf_get_call_summary(env, insn, &cs)) - use = GENMASK(min_t(u8, cs.num_params, MAX_BPF_FUNC_REG_ARGS), 1); + if (bpf_get_call_summary(env, insn, &cs)) { + use = GENMASK(min_t(u8, cs.arg_slot_cnt, MAX_BPF_FUNC_REG_ARGS), 1); + } else if (bpf_pseudo_call(insn)) { + /* a static callee's reads are added in bpf_compute_live_registers() */ + subprog = callee_subprog(env, insn); + if (bpf_subprog_is_global(env, subprog)) { + reg_args = min_t(u8, env->subprog_info[subprog].arg_slot_cnt, MAX_BPF_FUNC_REG_ARGS); + use = GENMASK(reg_args, 1); + } else { + use = 0; + } + } def = mask_widen(def); use = mask_widen(use); + /* callx reads the address of the callee from dst_reg */ + if (bpf_is_callx(insn)) + use |= dst; break; default: def = 0; @@ -2227,9 +2664,8 @@ int bpf_compute_live_registers(struct bpf_verifier_env *env) struct bpf_insn *insns = env->prog->insnsi; struct insn_live_regs *state; int insn_cnt = env->prog->len; - u64 pos, insn_pos; - int err = 0, i, j; - bool changed; + int err = 0, i, subprog, start, end; + bool changed, ret_reg_pair; /* Use the following algorithm: * - define the following: @@ -2256,8 +2692,14 @@ int bpf_compute_live_registers(struct bpf_verifier_env *env) goto out; } - for (i = 0; i < insn_cnt; ++i) - compute_insn_live_regs(env, &insns[i], &state[i]); + for (subprog = 0; subprog < env->subprog_cnt; subprog++) { + start = env->subprog_info[subprog].start; + end = env->subprog_info[subprog + 1].start; + ret_reg_pair = bpf_ret_reg_pair(env, subprog); + + for (i = start; i < end; ++i) + compute_insn_live_regs(env, &insns[i], &state[i], ret_reg_pair); + } /* Forward pass: resolve stack access through FP-derived pointers */ err = bpf_compute_subprog_arg_access(env); @@ -2273,11 +2715,21 @@ int bpf_compute_live_registers(struct bpf_verifier_env *env) struct bpf_iarray *succ; u32 new_out = 0; u32 new_in = 0; + u32 use; succ = bpf_insn_successors(env, insn_idx); for (int s = 0; s < succ->cnt; ++s) new_out |= state[succ->items[s]].in; - new_in = (new_out & ~live->def) | live->use; + use = live->use; + /* + * For a call to a static subprogram include the callee's + * entry 'in' as computed so far. + */ + if (bpf_pseudo_call(&insns[insn_idx]) && + !bpf_subprog_is_global(env, callee_subprog(env, &insns[insn_idx]))) + use |= state[insn_idx + insns[insn_idx].imm + 1].in & + mask_widen(GENMASK(BPF_REG_5, BPF_REG_1)); + new_in = (new_out & ~live->def) | use; if (new_out != live->out || new_in != live->in) { live->in = new_in; live->out = new_out; @@ -2301,31 +2753,6 @@ int bpf_compute_live_registers(struct bpf_verifier_env *env) insn_aux[i].zext_dst = def32 >= 0 && (mask_hi(out) & BIT(def32)); } - if (env->log.level & BPF_LOG_LEVEL2) { - verbose(env, "Live regs before insn:\n"); - for (i = 0; i < insn_cnt; ++i) { - if (env->insn_aux_data[i].scc) - verbose(env, "%3d ", env->insn_aux_data[i].scc); - else - verbose(env, " "); - verbose(env, "%3d: ", i); - for (j = BPF_REG_0; j < BPF_REG_10; ++j) - if (insn_aux[i].live_regs_before & BIT(j)) - verbose(env, "%d", j); - else - verbose(env, "."); - verbose(env, " "); - pos = env->log.end_pos; - bpf_verbose_insn(env, &insns[i]); - insn_pos = env->log.end_pos; - if (insn_aux[i].zext_dst) - verbose(env, "%*c; zext", bpf_vlog_alignment(insn_pos - pos), ' '); - verbose(env, "\n"); - if (bpf_is_ldimm64(&insns[i])) - i++; - } - } - out: kvfree(state); return err; diff --git a/kernel/bpf/log.c b/kernel/bpf/log.c index fb032dfdc0de..5432c0fc442b 100644 --- a/kernel/bpf/log.c +++ b/kernel/bpf/log.c @@ -527,7 +527,7 @@ static void verbose_unum(struct bpf_verifier_env *env, u64 num) verbose(env, "%#llx", num); } -static void verbose_snum(struct bpf_verifier_env *env, s64 num) +void bpf_verbose_snum(struct bpf_verifier_env *env, s64 num) { if (is_snum_decimal(num)) verbose(env, "%lld", num); @@ -609,7 +609,7 @@ static void print_scalar_ranges(struct bpf_verifier_env *env, } if (m1->name[0] == 's') - verbose_snum(env, m1->val); + bpf_verbose_snum(env, m1->val); else verbose_unum(env, m1->val); } @@ -632,7 +632,7 @@ static void print_reg_state(struct bpf_verifier_env *env, if (t == SCALAR_VALUE && reg->precise) verbose(env, "P"); if (t == SCALAR_VALUE && tnum_is_const(reg->var_off)) { - verbose_snum(env, reg->var_off.value); + bpf_verbose_snum(env, reg->var_off.value); return; } @@ -643,7 +643,7 @@ static void print_reg_state(struct bpf_verifier_env *env, if (state->frameno != reg->frameno) verbose(env, "[%d]", reg->frameno); if (tnum_is_const(reg->var_off)) { - verbose_snum(env, reg->var_off.value + reg->delta); + bpf_verbose_snum(env, reg->var_off.value + reg->delta); return; } } @@ -667,7 +667,7 @@ static void print_reg_state(struct bpf_verifier_env *env, } if (t != SCALAR_VALUE && reg->delta) { verbose_a("off="); - verbose_snum(env, reg->delta); + bpf_verbose_snum(env, reg->delta); } if (type_is_pkt_pointer(t)) { verbose_a("r="); @@ -683,7 +683,7 @@ static void print_reg_state(struct bpf_verifier_env *env, /* a pointer register with fixed offset */ if (reg->var_off.value) { verbose_a("imm="); - verbose_snum(env, reg->var_off.value); + bpf_verbose_snum(env, reg->var_off.value); } } else { print_scalar_ranges(env, reg, &sep); @@ -693,6 +693,8 @@ static void print_reg_state(struct bpf_verifier_env *env, tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); verbose_a("var_off=%s", tn_buf); } + if (reg->base != 0 || reg->step != 1) + verbose_a("step=%d+%d", reg->base, reg->step); } verbose(env, ")"); } @@ -716,7 +718,8 @@ void print_verifier_state(struct bpf_verifier_env *env, const struct bpf_verifie verbose(env, "="); print_reg_state(env, state, reg); } - for (i = 0; i < state->allocated_stack / BPF_REG_SIZE; i++) { + for (i = 0; i < bpf_stack_nr_slots(state); i++) { + struct bpf_stack_state *slot = bpf_stack_slot(state, i); char types_buf[BPF_REG_SIZE + 1]; const char *sep = ""; bool valid = false; @@ -727,7 +730,7 @@ void print_verifier_state(struct bpf_verifier_env *env, const struct bpf_verifie continue; for (j = 0; j < BPF_REG_SIZE; j++) { - slot_type = state->stack[i].slot_type[j]; + slot_type = slot->slot_type[j]; if (slot_type != STACK_INVALID && slot_type != STACK_POISON) valid = true; types_buf[j] = slot_type_char[slot_type]; @@ -736,12 +739,12 @@ void print_verifier_state(struct bpf_verifier_env *env, const struct bpf_verifie if (!valid) continue; - reg = &state->stack[i].spilled_ptr; - switch (state->stack[i].slot_type[BPF_REG_SIZE - 1]) { + reg = &slot->spilled_ptr; + switch (slot->slot_type[BPF_REG_SIZE - 1]) { case STACK_SPILL: /* print MISC/ZERO/INVALID slots above subreg spill */ for (j = 0; j < BPF_REG_SIZE; j++) - if (state->stack[i].slot_type[j] == STACK_SPILL) + if (slot->slot_type[j] == STACK_SPILL) break; types_buf[j] = '\0'; @@ -751,7 +754,7 @@ void print_verifier_state(struct bpf_verifier_env *env, const struct bpf_verifie case STACK_DYNPTR: /* skip to main dynptr slot */ i += BPF_DYNPTR_NR_SLOTS - 1; - reg = &state->stack[i].spilled_ptr; + reg = &bpf_stack_slot(state, i)->spilled_ptr; verbose(env, " fp%d", (-i - 1) * BPF_REG_SIZE); verbose(env, "=dynptr_%s(", dynptr_type_str(reg->dynptr.type)); @@ -789,6 +792,11 @@ void print_verifier_state(struct bpf_verifier_env *env, const struct bpf_verifie verbose(env, " cb"); if (state->in_async_callback_fn) verbose(env, " async_cb"); + if (state->loop_stack_cnt) { + verbose(env, " loop_stack="); + for (i = 0; i < state->loop_stack_cnt; i++) + verbose(env, "%s%d", i ? "," : "", state->loop_stack[i].loop_id); + } verbose(env, "\n"); if (!print_all) mark_verifier_state_clean(env); diff --git a/kernel/bpf/loops.c b/kernel/bpf/loops.c new file mode 100644 index 000000000000..2568079c2945 --- /dev/null +++ b/kernel/bpf/loops.c @@ -0,0 +1,616 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include +#include +#include + +static bool is_cfg_jump(struct bpf_insn *insn) +{ + u8 class = BPF_CLASS(insn->code); + u8 opcode = BPF_OP(insn->code); + + switch (class) { + case BPF_JMP: + case BPF_JMP32: + return opcode != BPF_CALL; + default: + return false; + } +} + +static struct bpf_iarray **compute_predecessors(struct bpf_verifier_env *env) +{ + struct bpf_insn_aux_data *aux = env->insn_aux_data; + struct bpf_iarray *succ, *preds, **result; + struct bpf_prog *prog = env->prog; + u32 *num_preds, i, s, len = prog->len; + struct bpf_insn *insn; + size_t sz; + void *tmp; + + num_preds = kvcalloc(prog->len, sizeof(u32), GFP_KERNEL_ACCOUNT); + if (!num_preds) + return NULL; + + /* + * 'result' layout: + * - array of pointers (struct bpf_iarray *)[len] + * - struct bpf_iarray one after another + */ + sz = size_mul(len, sizeof(*result) + sizeof(**result)); + for (i = 0; i < len; i++) { + insn = env->prog->insnsi + i; + succ = bpf_insn_successors(env, i); + sz = size_add(sz, array_size(succ->cnt, sizeof(*succ->items))); + iarray_for_each(s, succ) { + num_preds[s]++; + } + if (bpf_is_ldimm64(insn)) + i++; + } + + result = kvzalloc(sz, GFP_KERNEL_ACCOUNT); + if (!result) { + kvfree(num_preds); + return NULL; + } + + tmp = (void *)&result[len]; + for (i = 0; i < len; i++) { + result[i] = tmp; + tmp += struct_size(result[i], items, num_preds[i]); + } + + for (i = 0; i < len; i++) { + insn = env->prog->insnsi + i; + succ = bpf_insn_successors(env, i); + iarray_for_each(s, succ) { + preds = result[s]; + preds->items[preds->cnt++] = i; + } + if (succ->cnt > 1 || + (succ->cnt == 1 && num_preds[succ->items[0]] > 1) || + is_cfg_jump(insn)) + aux[i].bb_end = true; + if (bpf_is_ldimm64(insn)) + i++; + } + + kvfree(num_preds); + return result; +} + +static int idoms_intersect(struct bpf_verifier_env *env, int a, int b) +{ + int *postorder_nums = env->cfg.postorder_nums; + int *idoms = env->idoms; + + while (a != b) { + while (postorder_nums[a] < postorder_nums[b]) { + a = idoms[a]; + } + while (postorder_nums[b] < postorder_nums[a]) { + b = idoms[b]; + } + } + return a; +} + +/* See "A Simple, Fast Dominance Algorithm" by Cooper et al. for details. */ +static int compute_subprog_idoms(struct bpf_verifier_env *env, struct bpf_iarray **preds, + int subprog_idx) +{ + struct bpf_subprog_info *subprog = &env->subprog_info[subprog_idx]; + int start = subprog->start; + int po_first = subprog->postorder_start; + int po_last = (subprog + 1)->postorder_start - 1; + int *idoms = env->idoms; + int po_num, pred; + u32 work = 0; + bool changed; + + idoms[start] = start; + changed = true; + do { + changed = false; + /* + * Iterate in reverse postorder, skipping the subprog entry: + * insn_postorder[po_last] is the subprogram entry. + */ + for (po_num = po_last - 1; po_num >= po_first; po_num--) { + int idx = env->cfg.insn_postorder[po_num]; + int new_idom = -1; + + iarray_for_each(pred, preds[idx]) { + if (++work % 1024 == 0) { + if (signal_pending(current)) + return -EAGAIN; + cond_resched(); + } + + if (idoms[pred] == -1) + continue; + if (new_idom == -1) + new_idom = pred; + else + new_idom = idoms_intersect(env, pred, new_idom); + } + if (new_idom != -1 && idoms[idx] != new_idom) { + idoms[idx] = new_idom; + changed = true; + } + } + } while (changed); + idoms[start] = -1; + return 0; +} + +int bpf_compute_idoms(struct bpf_verifier_env *env) +{ + struct bpf_iarray **preds; + u32 len = env->prog->len; + int *idoms, i, err = 0; + + preds = compute_predecessors(env); + if (!preds) + return -ENOMEM; + + idoms = kvcalloc(len, sizeof(*idoms), GFP_KERNEL_ACCOUNT); + if (!idoms) { + kvfree(preds); + return -ENOMEM; + } + + env->idoms = idoms; + for (i = 0; i < len; i++) + idoms[i] = -1; + + for (i = 0; i < env->subprog_cnt; i++) { + err = compute_subprog_idoms(env, preds, i); + if (err) + break; + } + + kvfree(preds); + return err; +} + +struct dfs_state { + u32 traversed:1; + u32 next_succ:31; +}; + +struct loops_dfs { + struct dfs_state *state; + int *dfs_pos; + int *stack; +}; + +static void mark_irreducible(struct bpf_verifier_env *env, int h) +{ + env->insn_aux_data[h].loop->irreducible = true; +} + +static void add_backedge(struct bpf_verifier_env *env, int from, int h) +{ + struct bpf_insn_aux_data *aux = env->insn_aux_data; + struct bpf_loop *loop = aux[h].loop; + int cnt = loop->backedges_cnt; + + if (cnt == MAX_BACKEDGES) { + loop->backedges_overflow = true; + return; + } + loop->backedges[cnt].from = from; + loop->backedges[cnt].latches_cnt = 0; + loop->backedges_cnt++; +} + +static int add_exit(struct bpf_loop *loop, int from, int to) +{ + if (loop->exits_overflow) + return 0; + if (loop->exits_cnt == MAX_LOOP_EXITS) { + loop->exits_overflow = true; + return 0; + } + if (!loop->exits) { + loop->exits = kvcalloc(MAX_LOOP_EXITS, sizeof(*loop->exits), GFP_KERNEL_ACCOUNT); + if (!loop->exits) + return -ENOMEM; + } + loop->exits[loop->exits_cnt] = (struct bpf_loop_exit) { + .from = from, + .to = to, + }; + loop->exits_cnt++; + return 0; +} + +static int mark_as_header(struct bpf_verifier_env *env, int h) +{ + struct bpf_insn_aux_data *aux = env->insn_aux_data; + + if (!aux[h].loop) { + aux[h].loop = kvzalloc_obj(struct bpf_loop, GFP_KERNEL_ACCOUNT); + if (!aux[h].loop) + return -ENOMEM; + } + return 0; +} + +static int assign_header(struct bpf_verifier_env *env, struct loops_dfs *dfs, int n, int h) +{ + struct bpf_insn_aux_data *aux = env->insn_aux_data; + int *dfs_pos = dfs->dfs_pos; + int err, nh; + + err = mark_as_header(env, h); + if (err) + return err; + + /* Don't encode self-loops, otherwise can't reflect loops nesting structure. */ + if (n == h) + return 0; + + /* Make sure that loop headers up the chain are sorted by dfs_pos. */ + while (aux[n].loop_header != -1) { + nh = aux[n].loop_header; + if (nh == h) + return 0; + if (dfs_pos[nh] < dfs_pos[h]) { + aux[n].loop_header = h; + n = h; + h = nh; + } else { + n = nh; + } + } + aux[n].loop_header = h; + return 0; +} + +static bool is_cond_jmp_insn(struct bpf_insn *insn) +{ + u8 class = BPF_CLASS(insn->code); + u8 opcode = BPF_OP(insn->code); + + if (class != BPF_JMP && class != BPF_JMP32) + return false; + + switch (opcode) { + case BPF_JEQ: + case BPF_JGE: + case BPF_JGT: + case BPF_JLE: + case BPF_JLT: + case BPF_JNE: + case BPF_JSET: + case BPF_JSGE: + case BPF_JSGT: + case BPF_JSLE: + case BPF_JSLT: + return true; + default: + return false; + } +} + +int bpf_loop_at_index(struct bpf_verifier_env *env, u32 idx) +{ + struct bpf_insn_aux_data *aux = env->insn_aux_data; + + return aux[idx].loop ? idx : aux[idx].loop_header; +} + +/* + * Collect conditional jumps within n's loop that dominate 'n' and have + * a successor outside the loop, nearest first. + */ +static void find_dominating_conditions(struct bpf_verifier_env *env, int n, + struct bpf_backedge *backedge) +{ + struct bpf_insn *insns = env->prog->insnsi; + bool t_exits_loop, f_exits_loop; + int n_loop, t_loop, f_loop; + int *idoms = env->idoms; + + n_loop = bpf_loop_at_index(env, n); + if (idoms_intersect(env, n, n_loop) != n_loop) + return; + while (n >= 0) { + if (is_cond_jmp_insn(&insns[n]) && bpf_loop_at_index(env, n) == n_loop) { + t_loop = bpf_loop_at_index(env, n + insns[n].off + 1); + f_loop = bpf_loop_at_index(env, n + 1); + t_exits_loop = t_loop != n_loop && !bpf_is_nested_loop(env, t_loop, n_loop); + f_exits_loop = f_loop != n_loop && !bpf_is_nested_loop(env, f_loop, n_loop); + if (t_exits_loop || f_exits_loop) { + if (backedge->latches_cnt == MAX_LATCHES) + return; + backedge->latches[backedge->latches_cnt++] = n; + } + } + if (n == n_loop) + break; + n = idoms[n]; + } +} + +/* + * As described in "A New Algorithm for Identifying Loops in Decompilation" by Wei et al, + * adapted to be non-recursive. + */ +static int compute_loops_in_subprog(struct bpf_verifier_env *env, struct loops_dfs *dfs, + int subprog_idx) +{ + struct bpf_insn_aux_data *aux = env->insn_aux_data; + struct dfs_state *state = dfs->state; + int start = env->subprog_info[subprog_idx].start; + int *dfs_pos = dfs->dfs_pos; + int *stack = dfs->stack; + int s, h, err, cur, stack_sz; + struct bpf_iarray *succ; + u32 i; + + stack[0] = start; + state[start].traversed = true; + state[start].next_succ = 0; + dfs_pos[start] = 1; + stack_sz = 1; + i = 0; + do { + /* + * The algorithm should be very fast in practice, + * guard against pathological inputs, just in case. + */ + if ((++i % 1024) == 0) { + if (signal_pending(current)) + return -EAGAIN; + cond_resched(); + } + + cur = stack[stack_sz - 1]; + succ = bpf_insn_successors(env, cur); + if (state[cur].next_succ == succ->cnt) { + dfs_pos[cur] = 0; + stack_sz--; + continue; + } + s = succ->items[state[cur].next_succ]; + if (!state[s].traversed) { + /* Case A: start -> ... -> cur -> s [unexplored] */ + state[s].traversed = true; + state[s].next_succ = 0; + stack[stack_sz] = s; + dfs_pos[s] = stack_sz + 1; + stack_sz++; + continue; + } + /* 's' is fully explored at this point */ + if (dfs_pos[s]) { + /* + * start -> ... -> s -> cur --. + * ^ | + * '----------' + * Case B: 's' is in the current DFS path. + */ + err = assign_header(env, dfs, cur, s); + if (err) + return err; + add_backedge(env, cur, s); + } else if (aux[s].loop_header == -1) { + /* + * start -> ... -> ... -> s -> ... -> end + * | ^ + * '---> cur ---' + * Case C: 's' is explored, not in the current DFS path, + * and not a part of any loop. + */ + } else if (dfs_pos[aux[s].loop_header]) { + /* + * .----------------------. + * v | + * start -> ... -> h -> ... -> ... -> s --' + * | ^ + * '---> cur ---' + * Case D: 's' is explored, not in current DFS path, + * but its innermost loop header is. + */ + err = assign_header(env, dfs, cur, aux[s].loop_header); + if (err) + return err; + } else { + /* + * case E: 's' is explored, not in current DFS path, + * its innermost loop header is not in current DFS path, + * hence 's' is another entry into the same loop. + */ + h = aux[s].loop_header; + mark_irreducible(env, h); + while (aux[h].loop_header != -1) { + h = aux[h].loop_header; + if (dfs_pos[h]) { + err = assign_header(env, dfs, cur, h); + if (err) + return err; + break; + } + mark_irreducible(env, h); + } + } + state[cur].next_succ++; + } while (stack_sz); + + return 0; +} + +bool bpf_is_nested_loop(struct bpf_verifier_env *env, int inner_header, int outer_header) +{ + struct bpf_insn_aux_data *aux = env->insn_aux_data; + int idx; + + for (idx = inner_header; idx >= 0; idx = aux[idx].loop_header) + if (aux[idx].loop_header == outer_header) + return true; + + return false; +} + +int bpf_compute_loops(struct bpf_verifier_env *env) +{ + int i, j, k, s, t, iloop, sloop, err = 0, len = env->prog->len; + struct bpf_insn_aux_data *aux = env->insn_aux_data; + struct bpf_verifier_log *log = &env->log; + struct bpf_backedge *backedge; + struct loops_dfs dfs = {}; + struct bpf_iarray *succ; + struct bpf_loop *loop; + + dfs.dfs_pos = kvcalloc(len, sizeof(int), GFP_KERNEL_ACCOUNT); + dfs.state = kvcalloc(len, sizeof(struct dfs_state), GFP_KERNEL_ACCOUNT); + dfs.stack = kvcalloc(len, sizeof(int), GFP_KERNEL_ACCOUNT); + if (!dfs.dfs_pos || !dfs.state || !dfs.stack) { + err = -ENOMEM; + goto out; + } + for (i = 0; i < len; i++) + aux[i].loop_header = -1; + for (i = 0; i < env->subprog_cnt; i++) { + err = compute_loops_in_subprog(env, &dfs, i); + if (err) + goto out; + } + /* find latches */ + for (i = 0; i < len; i++) { + loop = aux[i].loop; + if (!loop) + continue; + for (j = 0; j < loop->backedges_cnt; j++) { + backedge = &loop->backedges[j]; + /* + * In theory, the backedge->from and it's latch can reside in + * an inner loop of `i`, e.g.: + * + * 1: r7 += 1; + * 2: r6 += 1; + * if r7 == 2 goto 1b; + * if r6 < 2 goto 2b; + * + * For now, let's assume that latches are unknown for such cases. + * (GCC/LLVM handle this by inserting artificial cfg nodes). + */ + if (bpf_loop_at_index(env, backedge->from) != i) + continue; + find_dominating_conditions(env, backedge->from, backedge); + } + } + /* find exits */ + for (i = 0; i < len; i++) { + iloop = aux[i].loop ? i : aux[i].loop_header; + if (iloop < 0) + continue; + succ = bpf_insn_successors(env, i); + iarray_for_each(s, succ) { + /* + * Nothing left to record once the innermost loop of 'i' + * overflowed: the walk below would stop at it right away. + */ + if (aux[iloop].loop->exits_overflow) + break; + sloop = aux[s].loop ? s : aux[s].loop_header; + if (iloop == sloop) + continue; + if (bpf_is_nested_loop(env, sloop, iloop)) + continue; + /* + * At this point 'sloop' is either -1, an outer loop, + * or a loop in another branch of the loop hierarchy. + */ + for (t = iloop; t >= 0 && t != sloop; t = aux[t].loop_header) { + /* + * Account for the following configuration: + * + * tloop { + * iloop { + * ... i: goto s; + * } + * sloop { + * s: + * ... + * } + * } + */ + if (bpf_is_nested_loop(env, sloop, t)) + break; + /* + * Record edges i -> s as exits from tloop when: + * + * sloop { + * tloop { + * iloop { + * ... i: goto s; + * } + * } + * s: ... + * } + */ + /* Enclosing loops inherit the overflow, see below */ + if (aux[t].loop->exits_overflow) + break; + err = add_exit(aux[t].loop, i, s); + if (err) + goto out; + } + } + if (bpf_is_ldimm64(env->prog->insnsi + i)) + i++; + } + /* A nested loop with a truncated exit list can't be abstracted by SCEV */ + for (i = 0; i < len; i++) { + loop = aux[i].loop; + if (!loop || !loop->exits_overflow) + continue; + for (t = aux[i].loop_header; t >= 0; t = aux[t].loop_header) + aux[t].loop->exits_overflow = true; + } + + if (env->log.level & BPF_LOG_LEVEL2) { + for (i = 0; i < len; i++) { + loop = aux[i].loop; + if (!loop) + continue; + bpf_log(log, "loop at %d", i); + if (aux[i].loop_header >= 0) + bpf_log(log, ", nested in %d", aux[i].loop_header); + if (loop->irreducible) + bpf_log(log, ", irreducible"); + if (loop->exits_overflow) + bpf_log(log, ", too many exits"); + bpf_log(log, "\n"); + for (j = 0; j < loop->backedges_cnt; j++) { + backedge = &loop->backedges[j]; + bpf_log(log, " backedge from %d", backedge->from); + for (k = 0; k < backedge->latches_cnt; k++) { + if (k == 0) + bpf_log(log, ", latch at "); + if (k == 1) + bpf_log(log, ", dominating exits at "); + if (k > 1) + bpf_log(log, ", "); + bpf_log(log, "%d", backedge->latches[k]); + } + bpf_log(log, "\n"); + } + for (j = 0; j < loop->exits_cnt; j++) + bpf_log(log, " exit from %d to %d\n", + loop->exits[j].from, loop->exits[j].to); + } + } + +out: + kvfree(dfs.dfs_pos); + kvfree(dfs.stack); + kvfree(dfs.state); + return err; +} diff --git a/kernel/bpf/map_in_map.c b/kernel/bpf/map_in_map.c index d2cbab4bdf64..0fa3f2ac0f99 100644 --- a/kernel/bpf/map_in_map.c +++ b/kernel/bpf/map_in_map.c @@ -25,6 +25,10 @@ struct bpf_map *bpf_map_meta_alloc(int inner_map_ufd) if (!inner_map->ops->map_meta_equal) return ERR_PTR(-ENOTSUPP); + /* An inner map has no used_maps reference to hold it under a queued callback. */ + if (btf_record_has_field(inner_map->record, BPF_RCU_HEAD)) + return ERR_PTR(-EOPNOTSUPP); + inner_map_meta_size = sizeof(*inner_map_meta); /* In some cases verifier needs to access beyond just base map. */ if (inner_map->ops == &array_map_ops || inner_map->ops == &percpu_array_map_ops) @@ -106,10 +110,13 @@ void *bpf_map_fd_get_ptr(struct bpf_map *map, return ERR_PTR(-ENOTSUPP); inner_map_meta = map->inner_map_meta; - if (inner_map_meta->ops->map_meta_equal(inner_map_meta, inner_map)) + if (inner_map_meta->ops->map_meta_equal(inner_map_meta, inner_map)) { bpf_map_inc(inner_map); - else + /* Its users are those of the outer maps from now on */ + WRITE_ONCE(inner_map->was_inner_map, true); + } else { inner_map = ERR_PTR(-EINVAL); + } return inner_map; } diff --git a/kernel/bpf/map_iter.c b/kernel/bpf/map_iter.c index c19b360bad9e..a58c148692e6 100644 --- a/kernel/bpf/map_iter.c +++ b/kernel/bpf/map_iter.c @@ -97,13 +97,57 @@ static struct bpf_iter_reg bpf_map_reg_info = { .seq_info = &bpf_map_seq_info, }; +/* + * For an iterator that hands the live map value to the program as a writable + * buffer: the verifier knows nothing of the map, so the special fields of the + * value have to lie outside of what the program may write. + */ +int bpf_iter_map_check_value_write(const struct bpf_prog *prog, + const struct bpf_map *map) +{ + const struct btf_record *rec = map->record; + const struct btf_field *field; + u32 i; + + if (IS_ERR_OR_NULL(rec) || !prog->aux->max_rdwr_write) + return 0; + + for (i = 0; i < rec->cnt; i++) { + field = &rec->fields[i]; + if (field->offset < prog->aux->max_rdwr_write && + field->offset + field->size > prog->aux->min_rdwr_write) + return -EACCES; + } + + return 0; +} + +static int bpf_iter_map_check_prog(const struct bpf_prog *prog, + const struct bpf_iter_aux_info *aux) +{ + const struct bpf_map *map = aux->map; + u32 value_size = map->value_size; + bool is_percpu; + + is_percpu = map->map_type == BPF_MAP_TYPE_PERCPU_HASH || + map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH || + map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY; + if (is_percpu) + value_size = round_up(value_size, 8) * num_possible_cpus(); + + if (prog->aux->max_rdonly_access > map->key_size || + prog->aux->max_rdwr_access > value_size) + return -EACCES; + + /* the program gets a copy of a per-cpu value */ + return is_percpu ? 0 : bpf_iter_map_check_value_write(prog, map); +} + static int bpf_iter_attach_map(struct bpf_prog *prog, union bpf_iter_link_info *linfo, struct bpf_iter_aux_info *aux) { - u32 key_acc_size, value_acc_size, key_size, value_size; struct bpf_map *map; - bool is_percpu = false; int err = -EINVAL; if (!linfo->map.map_fd) @@ -117,30 +161,26 @@ static int bpf_iter_attach_map(struct bpf_prog *prog, goto put_map; } - if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || - map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH || - map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) - is_percpu = true; - else if (map->map_type != BPF_MAP_TYPE_HASH && - map->map_type != BPF_MAP_TYPE_LRU_HASH && - map->map_type != BPF_MAP_TYPE_ARRAY && - map->map_type != BPF_MAP_TYPE_RHASH) - goto put_map; - - key_acc_size = prog->aux->max_rdonly_access; - value_acc_size = prog->aux->max_rdwr_access; - key_size = map->key_size; - if (!is_percpu) - value_size = map->value_size; - else - value_size = round_up(map->value_size, 8) * num_possible_cpus(); - - if (key_acc_size > key_size || value_acc_size > value_size) { - err = -EACCES; + /* The value ctx arg is writable and aliases the live element. */ + if (btf_record_has_field(map->record, BPF_RCU_HEAD)) { + err = -EOPNOTSUPP; goto put_map; } + if (map->map_type != BPF_MAP_TYPE_PERCPU_HASH && + map->map_type != BPF_MAP_TYPE_LRU_PERCPU_HASH && + map->map_type != BPF_MAP_TYPE_PERCPU_ARRAY && + map->map_type != BPF_MAP_TYPE_HASH && + map->map_type != BPF_MAP_TYPE_LRU_HASH && + map->map_type != BPF_MAP_TYPE_ARRAY && + map->map_type != BPF_MAP_TYPE_RHASH) + goto put_map; + aux->map = map; + err = bpf_iter_map_check_prog(prog, aux); + if (err) + goto put_map; + return 0; put_map: @@ -173,6 +213,7 @@ static const struct bpf_iter_reg bpf_map_elem_reg_info = { .target = "bpf_map_elem", .attach_target = bpf_iter_attach_map, .detach_target = bpf_iter_detach_map, + .check_prog = bpf_iter_map_check_prog, .show_fdinfo = bpf_iter_map_show_fdinfo, .fill_link_info = bpf_iter_map_fill_link_info, .ctx_arg_info_size = 2, diff --git a/kernel/bpf/offload.c b/kernel/bpf/offload.c index d855399812ee..f7bfff1760cc 100644 --- a/kernel/bpf/offload.c +++ b/kernel/bpf/offload.c @@ -96,6 +96,22 @@ static int __bpf_offload_dev_netdev_register(struct bpf_offload_dev *offdev, return err; } +/* + * @prog takes the place of a copy that the caller is about to free. Nothing + * keeps the netdev while a program is loaded: once the lock is dropped, no + * unregistration is left that has got the old copy from offload->prog. + */ +void bpf_prog_dev_bound_moved(struct bpf_prog *prog) +{ + if (!bpf_prog_is_dev_bound(prog->aux)) + return; + + down_read(&bpf_devs_lock); + if (prog->aux->offload) + prog->aux->offload->prog = prog; + up_read(&bpf_devs_lock); +} + static void __bpf_prog_offload_destroy(struct bpf_prog *prog) { struct bpf_prog_offload *offload = prog->aux->offload; @@ -250,7 +266,14 @@ int bpf_prog_dev_bound_init(struct bpf_prog *prog, union bpf_attr *attr) prog->aux->offload_requested = !(attr->prog_flags & BPF_F_XDP_DEV_BOUND_ONLY); down_write(&bpf_devs_lock); - err = __bpf_prog_dev_bound_init(prog, netdev); + /* + * Behind bpf_dev_bound_netdev_unregister() nobody would take the + * program off the device, which goes once the reference is dropped. + */ + if (READ_ONCE(netdev->reg_state) == NETREG_REGISTERED) + err = __bpf_prog_dev_bound_init(prog, netdev); + else + err = -EINVAL; up_write(&bpf_devs_lock); out: diff --git a/kernel/bpf/queue_stack_maps.c b/kernel/bpf/queue_stack_maps.c index 6e8b18c32a10..1cee2ba6e332 100644 --- a/kernel/bpf/queue_stack_maps.c +++ b/kernel/bpf/queue_stack_maps.c @@ -59,6 +59,9 @@ static int queue_stack_map_alloc_check(union bpf_attr *attr) */ return -E2BIG; + if (attr->max_entries >= U32_MAX / attr->value_size) + return -E2BIG; + return 0; } diff --git a/kernel/bpf/range_tree.c b/kernel/bpf/range_tree.c index 2f28886f3ff7..2460e78d6a55 100644 --- a/kernel/bpf/range_tree.c +++ b/kernel/bpf/range_tree.c @@ -143,16 +143,22 @@ int range_tree_clear(struct range_tree *rt, u32 start, u32 len) if (rn->rn_start < start && rn->rn_last > last) { u32 old_last = rn->rn_last; + /* + * Pre-allocate the right-half node before modifying + * the tree. If allocation fails we return -ENOMEM + * without altering the range tree. + */ + new_rn = kmalloc_nolock(sizeof(struct range_node), + __GFP_ACCOUNT, NUMA_NO_NODE); + if (!new_rn) + return -ENOMEM; + /* Overlaps with the entire clearing range */ range_it_remove(rn, rt); rn->rn_last = start - 1; range_it_insert(rn, rt); - /* Add a range */ - new_rn = kmalloc_nolock(sizeof(struct range_node), __GFP_ACCOUNT, - NUMA_NO_NODE); - if (!new_rn) - return -ENOMEM; + /* Add right-half range */ new_rn->rn_start = last + 1; new_rn->rn_last = old_last; range_it_insert(new_rn, rt); @@ -193,6 +199,7 @@ int is_range_tree_set(struct range_tree *rt, u32 start, u32 len) int range_tree_set(struct range_tree *rt, u32 start, u32 len) { u32 last = start + len - 1; + struct range_node *new_rn = NULL; struct range_node *right; struct range_node *left; int err; @@ -202,20 +209,19 @@ int range_tree_set(struct range_tree *rt, u32 start, u32 len) if (left && left->rn_start <= start && left->rn_last >= last) return 0; + left = range_it_iter_first(rt, start - 1, start - 1); + right = range_it_iter_first(rt, last + 1, last + 1); + if (!left && !right) { + new_rn = kmalloc_nolock(sizeof(struct range_node), + __GFP_ACCOUNT, NUMA_NO_NODE); + if (!new_rn) + return -ENOMEM; + } + /* Clear out everything in the range we want to set. */ err = range_tree_clear(rt, start, len); if (err) - return err; - - /* Do we have a left-adjacent range ? */ - left = range_it_iter_first(rt, start - 1, start - 1); - if (left && left->rn_last + 1 != start) - return -EFAULT; - - /* Do we have a right-adjacent range ? */ - right = range_it_iter_first(rt, last + 1, last + 1); - if (right && right->rn_start != last + 1) - return -EFAULT; + goto out_free_new; if (left && right) { /* Combine left and right adjacent ranges */ @@ -235,14 +241,16 @@ int range_tree_set(struct range_tree *rt, u32 start, u32 len) right->rn_start = start; range_it_insert(right, rt); } else { - left = kmalloc_nolock(sizeof(struct range_node), __GFP_ACCOUNT, NUMA_NO_NODE); - if (!left) - return -ENOMEM; - left->rn_start = start; - left->rn_last = last; - range_it_insert(left, rt); + /* No adjacent ranges; use the pre-allocated node */ + new_rn->rn_start = start; + new_rn->rn_last = last; + range_it_insert(new_rn, rt); } return 0; + +out_free_new: + kfree_nolock(new_rn); + return err; } void range_tree_destroy(struct range_tree *rt) diff --git a/kernel/bpf/ringbuf.c b/kernel/bpf/ringbuf.c index 3f1013d80544..447863ece26a 100644 --- a/kernel/bpf/ringbuf.c +++ b/kernel/bpf/ringbuf.c @@ -722,7 +722,7 @@ BPF_CALL_2(bpf_ringbuf_submit_dynptr, struct bpf_dynptr_kern *, ptr, u64, flags) const struct bpf_func_proto bpf_ringbuf_submit_dynptr_proto = { .func = bpf_ringbuf_submit_dynptr, .ret_type = RET_VOID, - .arg1_type = ARG_PTR_TO_DYNPTR | DYNPTR_TYPE_RINGBUF | OBJ_RELEASE, + .arg1_type = ARG_PTR_TO_DYNPTR | DYNPTR_TYPE_RINGBUF | OBJ_RELEASE | MEM_WRITE, .arg2_type = ARG_ANYTHING, }; @@ -741,7 +741,7 @@ BPF_CALL_2(bpf_ringbuf_discard_dynptr, struct bpf_dynptr_kern *, ptr, u64, flags const struct bpf_func_proto bpf_ringbuf_discard_dynptr_proto = { .func = bpf_ringbuf_discard_dynptr, .ret_type = RET_VOID, - .arg1_type = ARG_PTR_TO_DYNPTR | DYNPTR_TYPE_RINGBUF | OBJ_RELEASE, + .arg1_type = ARG_PTR_TO_DYNPTR | DYNPTR_TYPE_RINGBUF | OBJ_RELEASE | MEM_WRITE, .arg2_type = ARG_ANYTHING, }; diff --git a/kernel/bpf/scev.c b/kernel/bpf/scev.c new file mode 100644 index 000000000000..fea77772ca96 --- /dev/null +++ b/kernel/bpf/scev.c @@ -0,0 +1,2916 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */ + +#include "linux/cnum.h" +#include +#include +#include +#include +#include +#include +#include +#include + +#define REGS_NUM BPF_SCEV_REGS_NUM +#define UNKNOWN_EXPR_ID 0 +#define OPAQUE_EXPR_ID 1 + +/* + * BPF instructions use 'code', 'src_reg', 'off' and 'imm' fields for instruction encoding. + * For scalar evolution purpose we want to reuse most of the opcode definitions, + * but also add a few custom operations (REG and IMM). + * Use the enum below to uniformly represent all operations relevant for SCEV. + */ +enum expr_op { + /* leave range 0..255 for standard bpf opcodes */ + UNKNOWN = 256, /* start custom opcodes from the second byte */ + REG, + IMM, + SDIV, SMOD, + SEXT8, SEXT16, SEXT32, + ZEXT8, ZEXT16, ZEXT32, + BSWAP16, BSWAP32, BSWAP64, + /* during the loop body execution the value can be either of param[0] or param[1] */ + ANY, + /* some value that verifier is not going to track precisely */ + OPAQUE, + /* '*(u8/16/32 *)(r10 + X) = Y' writes define slot contents only partially */ + SPILL8, SPILL16, SPILL32, + /* + * SCEV expression corresponding to linear equation 'param[0] + param[1] * k', + * where k is a loop iteration number. Loop here refers to innermost loop + * containing instruction associated with this expression, as returned by + * bpf_loop_at_index(). + */ + LINEAR_SCEV, +}; + +struct expr { + u32 op; + union { + u32 params[2]; + s64 imm; + }; +}; + +struct expr_bucket { + u32 cnt; + u32 cap; + u32 ids[]; +}; + +struct env { + bool empty; + u32 reg2expr[REGS_NUM]; + u32 reg2scev[REGS_NUM]; +}; + +struct insn_envs { + u32 cnt; + struct { + int loop_header; + struct env *env; + } entries[]; +}; + +#define EXPR_STACK_DEPTH 8 + +struct expr_stack_elt { + u32 id:28; + u32 pre:1; + u32 next_param:2; +}; + +DEFINE_MIN_HEAP(int, scev_worklist); + +struct scev { + /* Read-only representatives for stack slots without a saved register. */ + struct bpf_reg_state zero_reg; + struct bpf_reg_state unknown_reg; + /* + * Expressions are identified by id, exprs_ht ensures that + * each expression exists as a unique instance. + * This allows for fast equivalence check: id1 === id2. + */ + struct expr_bucket **exprs_ht; + struct scev_worklist worklist; + struct insn_envs **envs; + struct expr *exprs; + /* + * Loops are analyzed one by one, this array keeps track if a particular + * instruction was visited on a current pass. + */ + u32 *discovered; + u32 exprs_ht_cnt; + int exprs_cnt; + int exprs_cap; + int envs_cnt; + int stack_sz; + struct expr_stack_elt expr_stack[EXPR_STACK_DEPTH]; + u32 ids_buf[EXPR_STACK_DEPTH]; +}; + +static u32 expr_hash(struct expr *e) +{ + return jhash_3words(e->op, e->params[0], e->params[1], 0); +} + +static int expr_eq(struct expr *a, struct expr *b) +{ + return a->op == b->op && a->imm == b->imm; +} + +static int add_expr(struct scev *scev, struct expr e) +{ + struct expr_bucket *bucket; + u32 i, id, hash, new_cap; + void *tmp; + + hash = expr_hash(&e) & (scev->exprs_ht_cnt - 1); + bucket = scev->exprs_ht[hash]; + + if (bucket) { + for (i = 0; i < bucket->cnt; i++) { + id = bucket->ids[i]; + if (expr_eq(&e, &scev->exprs[id])) + return id; + } + } + + if (!bucket || bucket->cap == bucket->cnt) { + new_cap = bucket ? bucket->cap * 2 : 4; + bucket = kvrealloc(bucket, sizeof(*bucket) + sizeof(u32) * new_cap, GFP_KERNEL_ACCOUNT | __GFP_ZERO); + if (!bucket) + return -ENOMEM; + scev->exprs_ht[hash] = bucket; + bucket->cap = new_cap; + } + + if (scev->exprs_cnt == scev->exprs_cap) { + new_cap = scev->exprs_cap ? scev->exprs_cap * 2 : 256; + tmp = kvrealloc(scev->exprs, sizeof(struct expr) * new_cap, GFP_KERNEL_ACCOUNT); + if (!tmp) + return -ENOMEM; + scev->exprs = tmp; + scev->exprs_cap = new_cap; + } + + id = scev->exprs_cnt++; + scev->exprs[id] = e; + bucket->ids[bucket->cnt++] = id; + return id; +} + +static int expr2(struct scev *scev, u32 op, int a, int b) +{ + if (a < 0) + return a; + if (b < 0) + return b; + return add_expr(scev, (struct expr){ .op = op, .params = {a, b} }); +} + +static int expr1(struct scev *scev, u32 op, int a) +{ + if (a < 0) + return a; + return expr2(scev, op, a, 0); +} + +static int expr0(struct scev *scev, u32 op) +{ + return expr2(scev, op, 0, 0); +} + +static int imm_expr(struct scev *scev, s64 value) +{ + return add_expr(scev, (struct expr){ .op = IMM, .imm = value }); +} + +static bool same_exprs(struct scev *scev, int id_a, int id_b) +{ + return id_a == id_b; +} + +static bool is_imm(struct scev *scev, int id, s64 *imm) +{ + if (scev->exprs[id].op != IMM) + return false; + *imm = scev->exprs[id].imm; + return true; +} + +static bool is_op(struct scev *scev, int id, u32 op) +{ + if (scev->exprs[id].op != op) + return false; + return true; +} + +static bool is_unop(struct scev *scev, enum expr_op op, int id, u32 *p0) +{ + if (scev->exprs[id].op != op) + return false; + *p0 = scev->exprs[id].params[0]; + return true; +} + +static bool is_binop(struct scev *scev, enum expr_op op, int id, u32 *p0, u32 *p1) +{ + if (scev->exprs[id].op != op) + return false; + *p0 = scev->exprs[id].params[0]; + *p1 = scev->exprs[id].params[1]; + return true; +} + +static bool is_reg(struct scev *scev, int id, u32 *reg) +{ + return is_unop(scev, REG, id, reg); +} + +static bool is_add(struct scev *scev, int id, u32 *left, u32 *right) +{ + return is_binop(scev, BPF_ADD, id, left, right); +} + +static bool is_zext32(struct scev *scev, int id, u32 *left) +{ + return is_unop(scev, ZEXT32, id, left); +} + +static bool is_any(struct scev *scev, int id, u32 *left, u32 *right) +{ + return is_binop(scev, ANY, id, left, right); +} + +static bool is_linear(struct scev *scev, int id, u32 *base, u32 *slope) +{ + return is_binop(scev, LINEAR_SCEV, id, base, slope); +} + +static bool is_linear_or_linear32(struct scev *scev, int id, u32 *base, u32 *slope, bool *zext32) +{ + u32 t; + + *zext32 = is_zext32(scev, id, &t); + return is_linear(scev, *zext32 ? t : id, base, slope); +} + +static bool is_opaque(struct scev *scev, int id) +{ + return is_op(scev, id, OPAQUE); +} + +static void log_reg(struct bpf_verifier_env *env, u32 reg) +{ + if (reg < MAX_BPF_REG) + bpf_log(&env->log, "r%d", reg); + else + bpf_log(&env->log, "*fp%d", (MAX_BPF_REG - reg - 1) * 8); +} + +static const char *op_str(u32 op) +{ + switch (op) { + case BPF_ADD: return "+"; + case BPF_SUB: return "-"; + case BPF_MUL: return "*"; + case BPF_DIV: return "/"; + case SDIV: return "s/"; + case BPF_OR: return "|"; + case BPF_AND: return "&"; + case BPF_LSH: return "<<"; + case BPF_RSH: return ">>"; + case BPF_NEG: return "-"; + case BPF_MOD: return "%"; + case SMOD: return "s%"; + case BPF_XOR: return "^"; + case BPF_ARSH: return "s>>"; + case SEXT8: return "sext8"; + case SEXT16: return "sext16"; + case SEXT32: return "sext32"; + case ZEXT8: return "zext8"; + case ZEXT16: return "zext16"; + case ZEXT32: return "zext32"; + case BSWAP16: return "bswap16"; + case BSWAP32: return "bswap32"; + case BSWAP64: return "bswap64"; + case SPILL8: return "spill8"; + case SPILL16: return "spill16"; + case SPILL32: return "spill32"; + case ANY: return "any"; + case LINEAR_SCEV: return "linear"; + } + return NULL; +} + +static u32 op_params_num(u32 op) +{ + switch (op) { + case BPF_ADD: + case BPF_SUB: + case BPF_MUL: + case BPF_DIV: + case BPF_MOD: + case BPF_OR: + case BPF_XOR: + case BPF_AND: + case BPF_LSH: + case BPF_RSH: + case BPF_ARSH: + case SDIV: + case SMOD: + case ANY: + case LINEAR_SCEV: + return 2; + case BPF_NEG: + case SEXT8: + case SEXT16: + case SEXT32: + case ZEXT8: + case ZEXT16: + case ZEXT32: + case BSWAP16: + case BSWAP32: + case BSWAP64: + case SPILL8: + case SPILL16: + case SPILL32: + return 1; + case REG: + case IMM: + case OPAQUE: + return 0; + default: + return 0; + } +} + +static bool expr_stack_push(struct scev *scev, u32 id) +{ + if (scev->stack_sz >= EXPR_STACK_DEPTH) + return false; + scev->expr_stack[scev->stack_sz].id = id; + scev->expr_stack[scev->stack_sz].pre = true; + scev->expr_stack[scev->stack_sz].next_param = 0; + scev->stack_sz++; + return true; +} + +enum { + PRE = BIT(1), POST = BIT(2), DEPTH_LIMIT = BIT(3) +}; + +static bool expr_next(struct scev *scev, u32 *id, u32 *order) +{ + struct expr_stack_elt *elt; + struct expr *expr; + u32 num_params; + + if (scev->stack_sz == 0) + return false; + + elt = &scev->expr_stack[scev->stack_sz - 1]; + *id = elt->id; + *order = 0; + expr = &scev->exprs[elt->id]; + num_params = op_params_num(expr->op); + if (elt->pre) { + elt->pre = false; + *order = PRE; + return true; + } + if (elt->next_param == num_params) { + *order = POST; + scev->stack_sz--; + return true; + } + if (scev->stack_sz == EXPR_STACK_DEPTH) { + *order = POST | DEPTH_LIMIT; + scev->stack_sz--; + return true; + } + expr_stack_push(scev, expr->params[elt->next_param]); + elt->next_param++; + return expr_next(scev, id, order); +} + +static void log_expr(struct bpf_verifier_env *env, u32 id) +{ + struct bpf_verifier_log *log = &env->log; + struct scev *scev = env->scev; + struct expr *expr; + const char *str; + u32 order; + + scev->stack_sz = 0; + expr_stack_push(scev, id); + while (expr_next(scev, &id, &order)) { + if ((order & PRE) && scev->stack_sz > 1) + bpf_log(log, " "); + expr = &scev->exprs[id]; + switch (expr->op) { + case UNKNOWN: + if (order & PRE) + bpf_log(log, "?"); + break; + case OPAQUE: + if (order & PRE) + bpf_log(log, "_"); + break; + case REG: + if (order & PRE) + log_reg(env, expr->params[0]); + break; + case IMM: + if (order & PRE) + bpf_log(log, "%lld", expr->imm); + break; + default: + if (order & PRE) { + str = op_str(expr->op); + bpf_log(log, "("); + if (str) + bpf_log(log, "%s", str); + else + bpf_log(log, "bad-expr-op %x", expr->op); + } + if (order & DEPTH_LIMIT) + bpf_log(log, "..."); + if (order & POST) + bpf_log(log, ")"); + } + } +} + +static bool reg_alive_at(struct bpf_verifier_env *env, u32 reg, u32 insn_idx) +{ + struct bpf_insn_aux_data *aux = env->insn_aux_data; + u16 live_regs = aux[insn_idx].live_regs_before; + + return reg < MAX_BPF_REG + ? (BIT(reg) & live_regs) + : test_bit(reg - MAX_BPF_REG, aux[insn_idx].live_stack_before); +} + +static void print_env(struct bpf_verifier_env *env, struct env *e, u32 insn_idx) +{ + struct bpf_verifier_log *log = &env->log; + struct scev *scev = env->scev; + bool printed_some = false; + bool is_self_scev; + bool is_self_reg; + int i, r; + + if (e->empty) { + bpf_log(log, " \n"); + return; + } + + for (i = 0; i < REGS_NUM; i++) { + if (!reg_alive_at(env, i, insn_idx)) + continue; + is_self_reg = is_reg(scev, e->reg2expr[i], &r) && i == r; + is_self_scev = is_reg(scev, e->reg2scev[i], &r) && i == r; + if (is_self_reg && is_self_scev) + continue; + printed_some = true; + bpf_log(log, " "); + log_reg(env, i); + bpf_log(log, "="); + log_expr(env, e->reg2expr[i]); + if (e->reg2expr[i] != UNKNOWN_EXPR_ID) { + bpf_log(log, " / "); + log_expr(env, e->reg2scev[i]); + } + bpf_log(log, "\n"); + } + if (!printed_some) + bpf_log(log, " \n"); +} +static struct env *find_loop_env(struct scev *scev, int loop_header, int insn_idx) +{ + struct insn_envs *envs = scev->envs[insn_idx]; + u32 i; + + if (!envs) + return NULL; + for (i = 0; i < envs->cnt; i++) + if (envs->entries[i].loop_header == loop_header) + return envs->entries[i].env; + return NULL; +} + +static struct env *find_header_env(struct scev *scev, int header) +{ + return find_loop_env(scev, header, header); +} + +static struct env *get_loop_env(struct scev *scev, int loop_header, int insn_idx) +{ + struct insn_envs *envs = scev->envs[insn_idx]; + u32 cnt = envs ? envs->cnt : 0; + struct insn_envs *tmp; + struct env *e; + + e = find_loop_env(scev, loop_header, insn_idx); + if (e) + return e; + + e = kzalloc(sizeof(*e), GFP_KERNEL_ACCOUNT); + if (!e) + return NULL; + + tmp = krealloc(envs, struct_size(envs, entries, cnt + 1), GFP_KERNEL_ACCOUNT); + if (!tmp) { + kfree(e); + return NULL; + } + + e->empty = true; + tmp->entries[cnt].loop_header = loop_header; + tmp->entries[cnt].env = e; + tmp->cnt = cnt + 1; + scev->envs[insn_idx] = tmp; + return e; +} + +static int replace_reg(struct scev *scev, struct env *e, u32 reg, int id) +{ + if (id < 0) + return id; + e->reg2expr[reg] = id; + return 0; +} + +static int setup_initial_loop_env(struct bpf_verifier_env *env, struct env *e, int insn_idx) +{ + struct scev *scev = env->scev; + int i, err; + + /* + * Initial assignments are of form rX -> rX are needed as seeds for subsequent + * work queue pop / transfer() calls. Because e->empty is true, the first backedge + * join() will overwrite the them. compute_header_scevs() accounts for register + * states at loop entry. + */ + for (i = 0; i < REGS_NUM; i++) { + err = replace_reg(scev, e, i, expr1(scev, REG, i)); + if (err) + return err; + } + return 0; +} + +static void forget_call_regs(struct env *e) +{ + int i; + + for (i = BPF_REG_0; i <= BPF_REG_5; i++) + e->reg2expr[i] = UNKNOWN_EXPR_ID; +} + +static u32 spill_spi(struct bpf_insn *insn) +{ + return -insn->off / BPF_REG_SIZE - 1; +} + +static u32 off_to_reg(struct bpf_insn *insn) +{ + return spill_spi(insn) + MAX_BPF_REG; +} + +static bool is_spill_off(int off) +{ + return off % BPF_REG_SIZE == 0 && + off <= -BPF_REG_SIZE && + off >= -MAX_BPF_STACK_JIT; +} + +static void mark_opaque(struct env *e, int off, int size) +{ + int b, spi; + + for (b = off; b < off + size; b++) { + if (b >= 0 || b < -MAX_BPF_STACK_JIT) + continue; + spi = (-b - 1) / BPF_REG_SIZE; + e->reg2expr[MAX_BPF_REG + spi] = OPAQUE_EXPR_ID; + } +} + +static int mk_spill(struct scev *scev, u8 size, int id) +{ + switch (size) { + case BPF_B: return expr1(scev, SPILL8, id); + case BPF_H: return expr1(scev, SPILL16, id); + case BPF_W: return expr1(scev, SPILL32, id); + case BPF_DW: return id; + } + return UNKNOWN_EXPR_ID; +} + +static int mk_fill(struct scev *scev, int id, u8 code) +{ + bool sx = BPF_MODE(code) == BPF_MEMSX; + +#ifdef __BIG_ENDIAN + /* A narrow fill from a DW spill reads high bits on big-endian. */ + if (BPF_SIZE(code) != BPF_DW) + return OPAQUE_EXPR_ID; +#endif + + switch (BPF_SIZE(code)) { + case BPF_B: return expr1(scev, sx ? SEXT8 : ZEXT8, id); + case BPF_H: return expr1(scev, sx ? SEXT16 : ZEXT16, id); + case BPF_W: return expr1(scev, sx ? SEXT32 : ZEXT32, id); + case BPF_DW: return sx ? UNKNOWN_EXPR_ID : id; /* DW sign-extended load is invalid */ + } + return UNKNOWN_EXPR_ID; +} + +static int maybe_store_fp(struct scev *scev, struct env *e, struct bpf_insn *insn, int id) +{ + u8 size = BPF_SIZE(insn->code); + + if (insn->dst_reg != BPF_REG_FP) + return 0; + if (is_spill_off(insn->off)) + return replace_reg(scev, e, off_to_reg(insn), mk_spill(scev, size, id)); + mark_opaque(e, insn->off, bpf_size_to_bytes(size)); + return 0; +} + +static int maybe_load_fp(struct scev *scev, struct env *e, struct bpf_insn *insn) +{ + int id; + + if (insn->src_reg == BPF_REG_FP && is_spill_off(insn->off)) + id = mk_fill(scev, e->reg2expr[off_to_reg(insn)], insn->code); + else + id = OPAQUE_EXPR_ID; + return replace_reg(scev, e, insn->dst_reg, id); +} + +static int transfer(struct bpf_verifier_env *env, struct env *e, int idx) +{ + const bool little_endian = htons(0x3412) == 0x1234; + struct bpf_insn *insn = &env->prog->insnsi[idx]; + struct scev *scev = env->scev; + u32 *reg2expr = e->reg2expr; + u8 class = BPF_CLASS(insn->code); + u8 x_or_k = BPF_SRC(insn->code); + u8 opcode = BPF_OP(insn->code); + u8 mode = BPF_MODE(insn->code); + u32 dst = insn->dst_reg; + u32 src = insn->src_reg; + u32 op, sext; + bool need_bswap; + int i, id; + s64 imm; + + switch (class) { + case BPF_ALU: + case BPF_ALU64: + switch (opcode) { + case BPF_MOV: + switch (insn->off) { + case 0: sext = 0; break; + case 8: sext = SEXT8; break; + case 16: sext = SEXT16; break; + case 32: sext = SEXT32; break; + default: + goto mark_dst_unknown; + } + + if (x_or_k == BPF_X && insn->imm == 0) + id = reg2expr[src]; + else if (x_or_k == BPF_K && src == 0 && insn->off == 0) + id = imm_expr(scev, insn->imm); + else + goto mark_dst_unknown; + + if (sext) + id = expr1(scev, sext, id); + if (class == BPF_ALU) + id = expr1(scev, ZEXT32, id); + + return replace_reg(scev, e, dst, id); + + case BPF_ADD: + case BPF_SUB: + case BPF_MUL: + case BPF_DIV: + case BPF_MOD: + case BPF_OR: + case BPF_XOR: + case BPF_AND: + case BPF_LSH: + case BPF_RSH: + case BPF_ARSH: + if (opcode == BPF_DIV && insn->off == 1) + op = SDIV; + else if (opcode == BPF_MOD && insn->off == 1) + op = SMOD; + else if (insn->off == 0) + op = opcode; + else + goto mark_dst_unknown; + + if (x_or_k == BPF_X && insn->imm == 0) + id = reg2expr[src]; + else if (x_or_k == BPF_K && src == 0) + id = imm_expr(scev, insn->imm); + else + goto mark_dst_unknown; + + id = expr2(scev, op, reg2expr[dst], id); + + if (class == BPF_ALU) + id = expr1(scev, ZEXT32, id); + + return replace_reg(scev, e, dst, id); + + case BPF_NEG: + if (src == 0 && insn->off == 0 && insn->imm == 0) + id = expr1(scev, BPF_NEG, reg2expr[dst]); + else + goto mark_dst_unknown; + + if (class == BPF_ALU) + id = expr1(scev, ZEXT32, id); + + return replace_reg(scev, e, dst, id); + + case BPF_END: + if (src != 0 || insn->off != 0 || (class == BPF_ALU64 && x_or_k != 0)) + goto mark_dst_unknown; + + need_bswap = class == BPF_ALU64 || ((x_or_k == BPF_TO_LE) != little_endian); + switch (insn->imm) { + case 16: op = need_bswap ? BSWAP16 : ZEXT16; break; + case 32: op = need_bswap ? BSWAP32 : ZEXT32; break; + case 64: op = need_bswap ? BSWAP64 : 0; break; + default: + goto mark_dst_unknown; + } + + id = reg2expr[dst]; + if (op) + id = expr1(scev, op, reg2expr[dst]); + + return replace_reg(scev, e, dst, id); + default: + goto mark_dst_unknown; + } + break; + case BPF_LDX: + switch (mode) { + case BPF_MEM: + case BPF_MEMSX: + return maybe_load_fp(scev, e, insn); + default: + goto mark_dst_unknown; + } + case BPF_STX: + switch (mode) { + case BPF_MEM: + return maybe_store_fp(scev, e, insn, reg2expr[src]); + case BPF_ATOMIC: + if (insn->imm == BPF_LOAD_ACQ) + return maybe_load_fp(scev, e, insn); + if (insn->imm == BPF_STORE_REL) { + return maybe_store_fp(scev, e, insn, reg2expr[src]); + } + /* + * verifier does not track other atomic ops precisely, + * hence mark the results as opaque. + */ + if (insn->dst_reg == BPF_REG_FP) + mark_opaque(e, insn->off, bpf_size_to_bytes(BPF_SIZE(insn->code))); + if (insn->imm == BPF_CMPXCHG) + return replace_reg(scev, e, BPF_REG_0, OPAQUE_EXPR_ID); + if (insn->imm & BPF_FETCH) + return replace_reg(scev, e, src, OPAQUE_EXPR_ID); + break; + } + break; + case BPF_ST: + if (insn->dst_reg == BPF_REG_FP) { + id = imm_expr(scev, insn->imm); + if (id < 0) + return id; + return maybe_store_fp(scev, e, insn, id); + } + break; + case BPF_JMP: + case BPF_JMP32: + if (opcode == BPF_CALL) + forget_call_regs(e); + /* for non-CALL there are no changes in register states */ + break; + case BPF_LD: + switch (mode) { + case BPF_IMM: + /* rX = imm ll */ + if (BPF_SIZE(insn->code) == BPF_DW && insn->src_reg == 0) { + imm = ((u64)(insn + 1)->imm << 32) | (u32)insn->imm; + return replace_reg(scev, e, dst, imm_expr(scev, imm)); + } + /* map, map value, BTF id, function */ + if (BPF_SIZE(insn->code) == BPF_DW) + return replace_reg(scev, e, dst, OPAQUE_EXPR_ID); + goto mark_dst_unknown; + case BPF_ABS: + case BPF_IND: + forget_call_regs(e); + break; + default: + goto mark_dst_unknown; + } + break; + default: + /* unknown instruction, nuke state */ + for (i = 0; i < REGS_NUM; i++) + reg2expr[i] = UNKNOWN_EXPR_ID; + break; + } + return 0; + +mark_dst_unknown: + reg2expr[dst] = UNKNOWN_EXPR_ID; + return 0; +} + +/* + * Construct minimal 'ANY' expression by traversing 'a' and 'b' + * and accumulating non-duplicated non-ANY entries. + */ +static int mk_any(struct scev *scev, int a, int b) +{ + struct expr_stack_elt *elt; + struct expr *expr; + u32 i, j, ids_buf_sz; + u32 roots[2] = {a, b}; + int id; + + if (a < 0) + return a; + if (b < 0) + return b; + ids_buf_sz = 0; + for (i = 0; i < ARRAY_SIZE(roots); i++) { + scev->stack_sz = 0; + expr_stack_push(scev, roots[i]); + while (scev->stack_sz) { + elt = &scev->expr_stack[--scev->stack_sz]; + id = elt->id; + expr = &scev->exprs[elt->id]; + if (expr->op == ANY) { + if (!expr_stack_push(scev, expr->params[1]) || + !expr_stack_push(scev, expr->params[0])) + return UNKNOWN_EXPR_ID; + } else { + for (j = 0; j < ids_buf_sz; j++) { + if (same_exprs(scev, scev->ids_buf[j], id)) + goto next; + } + if (ids_buf_sz == ARRAY_SIZE(scev->ids_buf)) + return UNKNOWN_EXPR_ID; + scev->ids_buf[ids_buf_sz++] = id; + } +next:; + } + } + if (WARN_ON(ids_buf_sz == 0)) + return -EFAULT; + id = scev->ids_buf[0]; + for (i = 1; i < ids_buf_sz; i++) { + id = expr2(scev, ANY, id, scev->ids_buf[i]); + if (id < 0) + return id; + } + return id; +} + +static int join(struct scev *scev, struct env *acc, struct env *cur) +{ + int i, id; + + if (acc->empty) { + memcpy(acc, cur, sizeof(*acc)); + acc->empty = false; + return 0; + } + + for (i = 0; i < REGS_NUM; i++) { + if (!same_exprs(scev, acc->reg2expr[i], cur->reg2expr[i])) { + id = mk_any(scev, acc->reg2expr[i], cur->reg2expr[i]); + if (id < 0) + return id; + acc->reg2expr[i] = id; + } + } + return 0; +} + +/* Mark any register modified in 'e' as unknown in 'acc'. */ +static void forget_modified_regs(struct scev *scev, struct env *acc, struct env *e) +{ + u32 i, reg; + + for (i = 0; i < REGS_NUM; i++) { + if (e && is_reg(scev, e->reg2expr[i], ®) && reg == i) + continue; + acc->reg2expr[i] = UNKNOWN_EXPR_ID; + } +} + +/* For a loop H at `header`, mark any register modified as a result of H execution as unknown. */ +static void forget_non_invariants(struct bpf_verifier_env *env, struct env *acc, int header) +{ + struct bpf_insn_aux_data *aux = env->insn_aux_data; + struct bpf_loop *loop = aux[header].loop; + struct scev *scev = env->scev; + int i, idx, h; + + acc->empty = false; + forget_modified_regs(scev, acc, find_header_env(scev, header)); + for (i = 0; i < loop->exits_cnt; i++) { + idx = loop->exits[i].from; + h = bpf_loop_at_index(env, idx); + /* + * In order to account for situations like this: + * + * header: + * do { + * saved = rX; + * rX = -1000; + * do { // H + * if (random()) + * goto out; // E + * } while (inner_again); + * rX = saved; + * } while (outer_again); + * out: + * + * For exits originating in nested loops check that rX is invariant + * both at the header (H) and at exit (E). + */ + while (h != header) { + forget_modified_regs(scev, acc, find_header_env(scev, h)); + forget_modified_regs(scev, acc, find_loop_env(scev, h, idx)); + idx = h; + h = aux[h].loop_header; + } + /* + * For exits originating at the same loop the header is already accounted for, + * check only the exit environment. + */ + forget_modified_regs(scev, acc, find_loop_env(scev, header, idx)); + } +} + +/* Order worklist in reverse post-order. */ +static bool reverse_postorder_less(const void *a, const void *b, void *arg) +{ + const int *rank = arg; + + return rank[*(const int *)a] > rank[*(const int *)b]; +} + +static const struct min_heap_callbacks worklist_cb = { + .less = reverse_postorder_less, +}; + +static int worklist_push(struct bpf_verifier_env *env, int loop_header, int idx) +{ + struct scev *scev = env->scev; + struct scev_worklist *worklist = &scev->worklist; + u32 tag = (u32)loop_header + 1; + + if (scev->discovered[idx] == tag) + return 0; + + if (min_heap_full_inline(worklist)) { + size_t size = worklist->size ? worklist->size * 2 : 16; + int *data; + + data = kvrealloc(worklist->data, array_size(size, sizeof(*data)), + GFP_KERNEL_ACCOUNT); + if (!data) + return -ENOMEM; + worklist->data = data; + worklist->size = size; + } + if (!min_heap_push_inline(worklist, &idx, &worklist_cb, env->cfg.postorder_nums)) + return -EFAULT; + + scev->discovered[idx] = tag; + return 0; +} + +static bool reg_alive_at_succ(struct bpf_verifier_env *env, u32 r, u32 insn_idx) +{ + struct bpf_iarray *succ = bpf_insn_successors(env, insn_idx); + u32 i; + + for (i = 0; i < succ->cnt; i++) + if (reg_alive_at(env, r, succ->items[i])) + return true; + return false; +} + +enum changes_at { + LOG_AT_TRANSFER, + LOG_AT_JOIN, +}; + +static void log_env_changes(struct bpf_verifier_env *env, enum changes_at at, + struct env *old, struct env *new, int insn_idx) +{ + struct bpf_verifier_log *log = &env->log; + struct scev *scev = env->scev; + u64 null_pos = log->end_pos; + u32 old_id, new_id, regno; + bool any_changes = false; + bool is_self_scev; + u64 len; + int r; + + bpf_log(log, "%s %4d: ", at == LOG_AT_TRANSFER ? "t" : "j", insn_idx); + bpf_verbose_insn(env, &env->prog->insnsi[insn_idx]); + len = log->end_pos - null_pos; + bpf_log(log, "%*s", max(37 - (int)len, 1), " "); + bpf_log(log, " ; "); + for (r = 0; r < REGS_NUM; r++) { + old_id = old->reg2expr[r]; + new_id = new->reg2expr[r]; + if (at == LOG_AT_TRANSFER && !reg_alive_at_succ(env, r, insn_idx)) + continue; + if (at == LOG_AT_JOIN && !reg_alive_at(env, r, insn_idx)) + continue; + if (same_exprs(scev, old_id, new_id)) + continue; + is_self_scev = is_reg(scev, new_id, ®no) && r == regno; + if (old->empty && is_self_scev) + continue; + if (any_changes) + bpf_log(log, ", "); + log_reg(env, r); + bpf_log(log, " "); + log_expr(env, old_id); + bpf_log(log, " -> "); + log_expr(env, new_id); + any_changes = true; + } + if (old->empty && !any_changes) + bpf_log(log, ""); + bpf_log(log, "\n"); + if (!old->empty && !any_changes) + bpf_vlog_reset(log, null_pos); +} + +static bool is_probe_read_helper(u32 func_id) +{ + return func_id == BPF_FUNC_probe_read || + func_id == BPF_FUNC_probe_read_kernel || + func_id == BPF_FUNC_probe_read_user || + func_id == BPF_FUNC_probe_read_str || + func_id == BPF_FUNC_probe_read_kernel_str || + func_id == BPF_FUNC_probe_read_user_str; +} + +/* + * If instruction is an indirect write to stack, invalidate SCEVs for spi's + * that this instruction can touch. + */ +static void reset_scevs_at_indirect_writes(struct bpf_verifier_env *env, struct env *cur_env, int idx) +{ + struct bpf_insn *insn = &env->prog->insnsi[idx]; + struct scev *scev = env->scev; + u8 class = BPF_CLASS(insn->code); + const unsigned long *mask; + u32 spi, reg; + bool opaque; + + /* Direct fp stores are fine. */ + if ((class == BPF_ST || class == BPF_STX) && insn->dst_reg == BPF_REG_FP) + return; + + mask = bpf_may_write_mask(env, idx); + opaque = bpf_helper_call(insn) && is_probe_read_helper(insn->imm); + for_each_set_bit(spi, mask, MAX_BPF_STACK_SLOTS) { + reg = MAX_BPF_REG + spi; + if (cur_env->reg2expr[reg] == UNKNOWN_EXPR_ID) + continue; + replace_reg(scev, cur_env, reg, opaque ? OPAQUE_EXPR_ID : UNKNOWN_EXPR_ID); + } +} + +/* + * Collect loop-entry registers referenced by expr 'id', best effort. + * DEPTH_LIMIT may leave dependencies unrecorded. This mask only avoids + * widening that would lose stack-access precision; missing dependencies + * may cause false rejections, but this is not a soundness issue. + */ +static void or_expr_regs(struct bpf_verifier_env *env, u32 id, unsigned long *mask) +{ + struct scev *scev = env->scev; + u32 order; + + scev->stack_sz = 0; + expr_stack_push(scev, id); + while (expr_next(scev, &id, &order)) { + if ((order & PRE) && scev->exprs[id].op == REG) + __set_bit(scev->exprs[id].params[0], mask); + } +} + +/* Mask of argument registers (R1..R5) a call at 'idx' passes by register. */ +static u16 call_params_mask(struct bpf_verifier_env *env, int idx) +{ + struct bpf_insn *insn = &env->prog->insnsi[idx]; + struct bpf_call_summary cs; + int n = bpf_get_call_summary(env, insn, &cs) ? cs.arg_slot_cnt : MAX_BPF_FUNC_REG_ARGS; + + return n ? GENMASK(BPF_REG_1 + n - 1, BPF_REG_1) : 0; +} + +/* + * For instructions like: + * - *(u64 *)(rBase + off) = rX + * - rX = *(u64 *)(rBase + off) + * - calls that construct objects on stack (e.g. dynptr_from_mem(rBase, ...)) + * When 'rBase' can be a stack pointer and is derived from some registers Rs + * defined at loop entry, record Rs into 'mask'. + */ +static void collect_store_base_regs(struct bpf_verifier_env *env, + struct env *cur_env, int idx, unsigned long *mask) +{ + struct bpf_insn_aux_data *aux = env->insn_aux_data; + struct bpf_insn *insn = &env->prog->insnsi[idx]; + u8 class = BPF_CLASS(insn->code); + u8 size = BPF_SIZE(insn->code); + u16 base_regs = 0; + u32 r; + + if (size == BPF_W || size == BPF_DW) { + if ((class == BPF_STX || class == BPF_ST) && insn->dst_reg != BPF_REG_FP) + base_regs |= BIT(insn->dst_reg); + else if (class == BPF_LDX && insn->src_reg != BPF_REG_FP) + base_regs |= BIT(insn->src_reg); + } + + if (class == BPF_JMP && BPF_OP(insn->code) == BPF_CALL && + bpf_needs_fixed_stack_off(env, idx)) + base_regs |= call_params_mask(env, idx); + + base_regs &= aux[idx].stack_ptrs; + for (r = 0; r < MAX_BPF_REG; r++) + if (base_regs & BIT(r)) + or_expr_regs(env, cur_env->reg2expr[r], mask); +} + +/* Find the topmost loop header containing idx inside cur_header, or -1 if none. */ +static int topmost_nested_loop(struct bpf_verifier_env *env, int idx, int cur_header) +{ + struct bpf_insn_aux_data *aux = env->insn_aux_data; + int header; + + for (header = bpf_loop_at_index(env, idx); header >= 0; + header = aux[header].loop_header) + if (aux[header].loop_header == cur_header) + return header; + + return -1; +} + +/* + * Join cur_env into the successor's environment and schedule their traversal. + * Map successors in nested loops to their topmost nested header. + * Ignore successors outside the current loop and its nested loops. + */ +static int join_successor(struct bpf_verifier_env *env, int cur_header, int succ_idx, + struct env *old_env, struct env *cur_env) +{ + bool log_level2 = env->log.level & BPF_LOG_LEVEL2; + struct scev *scev = env->scev; + int err, nested_header; + struct env *succ_env; + + /* + * There are several possibilities for a successor: + * - succ_idx can be a part of a loop outside of the cur_header's loop, + * such edges are ignored. + * - succ_idx can be a part of the same loop as cur_header, + * for such edges succ_idx environment is updated: + * e[succ_idx] = join(e[succ_idx], cur_env) + * - succ_idx can be a part of some loop inner to cur_header, + * in such a case there exists some loop header H, + * such that H.loop_header == cur_header + * and H is the same as succ_idx's loop or contains it. + */ + if (bpf_loop_at_index(env, succ_idx) != cur_header) { + /* + * topmost_nested_loop() either finds H or returns -1, + * in case if succ_idx is a part of a loop outer to cur_header. + */ + nested_header = topmost_nested_loop(env, succ_idx, cur_header); + if (nested_header < 0) + return 0; + succ_idx = nested_header; + } + succ_env = get_loop_env(scev, cur_header, succ_idx); + if (!succ_env) + return -ENOMEM; + if (log_level2) + memcpy(old_env, succ_env, sizeof(*old_env)); + err = join(scev, succ_env, cur_env); + if (err) + return err; + err = worklist_push(env, cur_header, succ_idx); + if (err) + return err; + if (log_level2) + log_env_changes(env, LOG_AT_JOIN, old_env, succ_env, succ_idx); + return 0; +} + +/* Unsupported loops get only an unknown header environment. */ +static bool can_compute_loop_scev(const struct bpf_loop *loop) +{ + return !loop->irreducible && !loop->exits_overflow; +} + +static int compute_scev_for_loop(struct bpf_verifier_env *env, int cur_header) +{ + struct scev_worklist *worklist = &env->scev->worklist; + struct bpf_insn_aux_data *aux = env->insn_aux_data; + struct bpf_loop *cur_loop = aux[cur_header].loop; + struct env *header_env; + struct scev *scev = env->scev; + struct bpf_loop *nested_loop; + struct env *cur_env = NULL; + struct env *old_env = NULL; + struct bpf_iarray *succ; + bool log_level2 = env->log.level & BPF_LOG_LEVEL2; + int s, i, err, idx, succ_idx; + u32 r; + + if (log_level2) + bpf_log(&env->log, "Computing SCEV for loop at %d:\n", cur_header); + + /* + * For irreducible loops, and for loops nesting a loop with a truncated + * exit list, just assume that everything is clobbered for now. + */ + if (!can_compute_loop_scev(cur_loop)) { + header_env = get_loop_env(scev, cur_header, cur_header); + if (!header_env) + return -ENOMEM; + /* The freshly allocated environment has all expressions unknown. */ + bitmap_fill(cur_loop->store_base_regs, REGS_NUM); + header_env->empty = false; + return 0; + } + + cur_env = kzalloc(sizeof(*cur_env), GFP_KERNEL_ACCOUNT); + if (!cur_env) + goto nomem; + if (log_level2) { + old_env = kzalloc(sizeof(*old_env), GFP_KERNEL_ACCOUNT); + if (!old_env) + goto nomem; + } + header_env = get_loop_env(scev, cur_header, cur_header); + if (!header_env) + goto nomem; + err = setup_initial_loop_env(env, header_env, cur_header); + if (err) + goto out; + err = worklist_push(env, cur_header, cur_header); + if (err) + goto out; + + while (worklist->nr) { + idx = *min_heap_peek_inline(worklist); + min_heap_pop_inline(worklist, &worklist_cb, env->cfg.postorder_nums); + + /* join_successor() maps nested-loop entries to their representative header. */ + nested_loop = idx != cur_header ? aux[idx].loop : NULL; + memcpy(cur_env, find_loop_env(scev, cur_header, idx), sizeof(*cur_env)); + if (nested_loop) { + /* + * Process nested loop as a single instruction by + * forgetting anything non-invariant in the nested loop + */ + if (log_level2) + memcpy(old_env, cur_env, sizeof(*old_env)); + /* Pull the nested loop's stack-store base dependencies up. */ + for_each_set_bit(r, nested_loop->store_base_regs, BPF_SCEV_REGS_NUM) + or_expr_regs(env, cur_env->reg2expr[r], cur_loop->store_base_regs); + forget_non_invariants(env, cur_env, idx); + if (log_level2) + log_env_changes(env, LOG_AT_TRANSFER, old_env, cur_env, idx); + /* + * Treat nested loop exits as successors, + * join cur_env into successor's envs. + */ + for (i = 0; i < nested_loop->exits_cnt; i++) { + s = nested_loop->exits[i].to; + err = join_successor(env, cur_header, s, old_env, cur_env); + if (err) + goto out; + } + } else { + /* + * Iterate instructions within a single basic block + * starting at 'idx' mutating 'cur_env'. + */ + for (;;) { + if (log_level2) + memcpy(old_env, cur_env, sizeof(*old_env)); + collect_store_base_regs(env, cur_env, idx, cur_loop->store_base_regs); + err = transfer(env, cur_env, idx); + if (err) + goto out; + reset_scevs_at_indirect_writes(env, cur_env, idx); + if (log_level2) + log_env_changes(env, LOG_AT_TRANSFER, old_env, cur_env, idx); + succ = bpf_insn_successors(env, idx); + if (succ->cnt != 1) + break; + succ_idx = succ->items[0]; + if (aux[idx].bb_end || aux[succ_idx].need_scev || + bpf_loop_at_index(env, succ_idx) != cur_header) + break; + idx = succ_idx; + } + /* Join cur_env into basic block successor's envs. */ + iarray_for_each(s, succ) { + err = join_successor(env, cur_header, s, old_env, cur_env); + if (err) + goto out; + } + } + } + + err = 0; +out: + kfree(cur_env); + kfree(old_env); + return err; +nomem: + err = -ENOMEM; + goto out; +} + +static void mark_latches(struct bpf_verifier_env *env) +{ + struct bpf_insn_aux_data *aux = env->insn_aux_data; + struct bpf_loop *loop; + int len = env->prog->len; + int i, j, k, from; + + for (i = 0; i < len; i++) { + loop = aux[i].loop; + if (!loop) + continue; + aux[i].need_scev = true; + if (!can_compute_loop_scev(loop)) + continue; + /* + * Make sure to only set .need_scev for exits and latches + * belonging to the loop at `i`. If can_compute_loop_scev() + * fails for a loop, it should hide it's instructions from + * compute_insn_scevs(). + */ + for (j = 0; j < loop->backedges_cnt; j++) + for (k = 0; k < loop->backedges[j].latches_cnt; k++) + aux[loop->backedges[j].latches[k]].need_scev = true; + for (j = 0; j < loop->exits_cnt; j++) { + from = loop->exits[j].from; + if (bpf_loop_at_index(env, from) == i) + aux[from].need_scev = true; + } + } +} + +static bool is_any_imm_reg_opaque(struct scev *scev, u32 id) +{ + u32 l, r, reg, order; + s64 imm; + + scev->stack_sz = 0; + expr_stack_push(scev, id); + while (expr_next(scev, &id, &order)) { + if (order & DEPTH_LIMIT) + return false; + if (!(order & PRE) || + is_imm(scev, id, &imm) || + is_reg(scev, id, ®) || + is_opaque(scev, id) || + is_any(scev, id, &l, &r)) + continue; + return false; + } + return true; +} + +/* + * Can implement explicit stack version, but it is harder to read. + * Stick with recursive version for now. + */ +static int transform_expr_once(struct scev *scev, u32 lvl, u32 root, void *priv, + int (*fn)(struct scev *scev, u32 id, void *priv)) +{ + struct expr expr; + int p0, p1, id; + + if (lvl >= EXPR_STACK_DEPTH) + return UNKNOWN_EXPR_ID; + + expr = scev->exprs[root]; /* snapshot the expr before potential realloc */ + switch (op_params_num(expr.op)) { + case 0: + id = root; + break; + case 1: + p0 = transform_expr_once(scev, lvl + 1, expr.params[0], priv, fn); + id = expr1(scev, expr.op, p0); + break; + case 2: + p0 = transform_expr_once(scev, lvl + 1, expr.params[0], priv, fn); + p1 = transform_expr_once(scev, lvl + 1, expr.params[1], priv, fn); + id = expr2(scev, expr.op, p0, p1); + break; + } + return id < 0 ? id : fn(scev, id, priv); +} + +static int transform_expr(struct scev *scev, u32 root, void *priv, + int (*fn)(struct scev *scev, u32 id, void *priv)) +{ + int i = 0, id_old, id_new = root; + + do { + /* + * The depth limit is somewhat arbitrary and is essentially a failsafe. + * oscillation should not really happen. Still, it seems harsh to error out. + */ + if (i++ == 4 * EXPR_STACK_DEPTH) + return UNKNOWN_EXPR_ID; + id_old = id_new; + id_new = transform_expr_once(scev, 0, id_old, priv, fn); + if (id_new < 0) + return id_new; + } while (id_old != id_new); + return id_new; +} + +static int simplify(struct scev *scev, u32 id, void *priv) +{ + int a, b, l, r, t, base, slope; + s64 imm1, imm2; + + /* (+ (linear base slope) imm) -> (linear (+ base imm) slope) */ + if (is_add(scev, id, &l, &r) && + is_linear(scev, l, &base, &slope) && + is_imm(scev, r, &imm1)) + return expr2(scev, LINEAR_SCEV, expr2(scev, BPF_ADD, base, r), slope); + + /* (+ imm imm) -> imm */ + if (is_add(scev, id, &l, &r) && + is_imm(scev, l, &imm1) && + is_imm(scev, r, &imm2)) + return imm_expr(scev, imm1 + imm2); + + /* (zext32 (imm X)) -> (imm (u32)X) */ + if (is_zext32(scev, id, &l) && is_imm(scev, l, &imm1)) + return imm_expr(scev, (u64)(u32)imm1); + + /* + * (zext32 (+ (zext32 $a) $b)) -> (zext32 (+ $a $b)) + * (zext32 (+ $a (zext32 $b))) -> (zext32 (+ $a $b)) + */ + if ((is_zext32(scev, id, &t) && is_add(scev, t, &l, &b) && is_zext32(scev, l, &a)) || + (is_zext32(scev, id, &t) && is_add(scev, t, &a, &r) && is_zext32(scev, r, &b))) { + t = expr2(scev, BPF_ADD, a, b); + t = expr1(scev, ZEXT32, t); + return t; + } + + return id; +} + +static int compute_header_scevs(struct bpf_verifier_env *env, struct env *header_env) +{ + struct scev *scev = env->scev; + int l, r, t, id, ra, rb, ra_expr; + s64 imm; + + /* + * Note: + * reg2expr[ra] describes a change in rA's value over a single loop iteration. + * It does not account for rA's entry value. However, the SCEV constructed for rA + * must include an additional rA entry edge. For example: + * + * r0 = ..., r1 = 7 + * l: if r0 == 10 goto e // reg2expr[r1] == (imm 42) => reg2scev[r1] == (any r1 42) + * r0 += 1 + * r1 = 42 + * goto l + * e: + * ... at exit r1 is in {7,42} ... + */ + for (ra = 0; ra < REGS_NUM; ra++) { + ra_expr = transform_expr(scev, header_env->reg2expr[ra], NULL, simplify); + if (ra_expr < 0) + return ra_expr; + header_env->reg2expr[ra] = ra_expr; + /* rA = (+ rA IMM) -> (linear rA IMM) */ + if (is_add(scev, ra_expr, &l, &r) && + is_reg(scev, l, &rb) && + is_imm(scev, r, &imm) && + ra == rb) { + id = expr2(scev, LINEAR_SCEV, l, r); + if (id < 0) + return id; + header_env->reg2scev[ra] = id; + continue; + } + /* rA = (is_zext32 (+ rA IMM)) -> (is_zext32 (linear rA IMM)) */ + if (is_zext32(scev, ra_expr, &t) && + is_add(scev, t, &l, &r) && + is_reg(scev, l, &rb) && + is_imm(scev, r, &imm) && + ra == rb) { + id = expr2(scev, LINEAR_SCEV, l, r); + id = expr1(scev, ZEXT32, id); + if (id < 0) + return id; + header_env->reg2scev[ra] = id; + continue; + } + /* rA = rA -> rA*/ + if (is_reg(scev, ra_expr, &rb) && ra == rb) { + header_env->reg2scev[ra] = ra_expr; + continue; + } + /* rA = (any 1 2 3 4 ...) -> (any rA 1 2 3 4 ...) */ + if (is_any_imm_reg_opaque(scev, ra_expr)) { + id = mk_any(scev, expr1(scev, REG, ra), ra_expr); + if (id < 0) + return id; + header_env->reg2scev[ra] = id; + continue; + } + + } + return 0; +} +static int instantiate_header_scevs(struct scev *scev, u32 id, void *priv) +{ + u32 reg, *reg2scev = priv; + + /* (reg r) -> ... equation at header ... */ + if (is_reg(scev, id, ®)) + return reg2scev[reg]; + /* keep expression shapes */ + return id; +} + +static int compute_insn_scevs(struct bpf_verifier_env *env, struct env *eheader, struct env *einsn) +{ + struct scev *scev = env->scev; + int id, reg; + + for (reg = 0; reg < REGS_NUM; reg++) { + id = einsn->reg2expr[reg]; + id = transform_expr_once(scev, 0, id, eheader->reg2scev, instantiate_header_scevs); + if (id < 0) + return id; + id = transform_expr(scev, id, NULL, simplify); + if (id < 0) + return id; + einsn->reg2scev[reg] = id; + } + + return 0; +} + +static int log_scevs(struct bpf_verifier_env *env) +{ + struct bpf_insn_aux_data *aux = env->insn_aux_data; + struct bpf_verifier_log *log = &env->log; + struct env *header_env, *latch_env; + struct scev *scev = env->scev; + struct bpf_loop *loop; + int i, j, k, len, latch; + + len = env->prog->len; + for (i = 0; i < len; i++) { + loop = aux[i].loop; + if (!loop) + continue; + header_env = find_header_env(scev, i); + bpf_log(log, "scev at header %d:\n", i); + print_env(env, header_env, i); + if (!can_compute_loop_scev(loop)) + continue; + for (j = 0; j < loop->backedges_cnt; j++) { + for (k = 0; k < loop->backedges[j].latches_cnt; k++) { + latch = loop->backedges[j].latches[k]; + bpf_log(log, " scev at %s %d:\n", k ? "exit" : "latch", latch); + latch_env = find_loop_env(scev, i, latch); + if (verifier_bug_if(!latch_env, env, "i=%d, latch=%d", i, latch)) + return -EFAULT; + print_env(env, latch_env, latch); + } + } + } + return 0; +} + +int bpf_compute_scev(struct bpf_verifier_env *env) +{ + struct bpf_insn_aux_data *aux = env->insn_aux_data; + struct scev *scev = env->scev; + int *postorder = env->cfg.insn_postorder; + int cnt = env->cfg.cur_postorder; + int i, idx, err, header; + + mark_latches(env); + /* + * Visit loop headers in postorder, to guarantee that scevs + * for innermost loops are computed first. + */ + for (i = 0; i < cnt; i++) { + idx = postorder[i]; + if (!aux[idx].loop) + continue; + err = compute_scev_for_loop(env, idx); + if (err) + return err; + } + + /* + * Compute scevs from exprs collected on a previous step. Iterate instructions in + * reverse post-order so that each loop header is processed before instructions + * reachable from it. + */ + for (i = cnt - 1; i >= 0; i--) { + idx = postorder[i]; + if (!aux[idx].need_scev) + continue; + header = bpf_loop_at_index(env, idx); + err = aux[idx].loop + ? compute_header_scevs(env, find_header_env(scev, header)) + : compute_insn_scevs(env, + find_header_env(scev, header), + find_loop_env(scev, header, idx)); + if (err) + return err; + } + + for (i = 0; i < env->prog->len; i++) + if (aux[i].loop) + aux[i].prune_point = true; + + if (env->log.level & BPF_LOG_LEVEL2) { + err = log_scevs(env); + if (err) + return err; + } + + return 0; +} + +void bpf_free_scev(struct bpf_verifier_env *env) +{ + struct scev *scev = env->scev; + struct insn_envs *envs; + int i; + u32 j; + + if (!scev) + return; + for (i = 0; i < scev->envs_cnt; i++) { + envs = scev->envs[i]; + if (!envs) + continue; + for (j = 0; j < envs->cnt; j++) + kfree(envs->entries[j].env); + kfree(envs); + } + if (scev->exprs_ht) + for (i = 0; i < scev->exprs_ht_cnt; i++) + kvfree(scev->exprs_ht[i]); + kvfree(scev->exprs_ht); + kvfree(scev->worklist.data); + kvfree(scev->envs); + kvfree(scev->exprs); + kvfree(scev->discovered); + kfree(scev); + env->scev = NULL; +} + +int bpf_init_scev(struct bpf_verifier_env *env) +{ + struct scev *scev; + + scev = kzalloc(sizeof(struct scev), GFP_KERNEL_ACCOUNT); + if (!scev) + return -ENOMEM; + env->scev = scev; + bpf_mark_reg_known_scalar(&scev->zero_reg, 0); + bpf_mark_reg_unknown_imprecise(&scev->unknown_reg); + /* A power of two for hash & (exprs_ht_cnt - 1)` indexing. */ + scev->exprs_ht_cnt = roundup_pow_of_two(max(256U, DIV_ROUND_UP(env->prog->len, 4))); + scev->exprs_ht = kvcalloc(scev->exprs_ht_cnt, sizeof(*scev->exprs_ht), GFP_KERNEL_ACCOUNT); + if (!scev->exprs_ht) + goto nomem; + if (expr0(scev, UNKNOWN) < 0 || expr0(scev, OPAQUE) < 0) + goto nomem; + scev->envs = kvcalloc(env->prog->len, sizeof(*scev->envs), GFP_KERNEL_ACCOUNT); + scev->discovered = kvcalloc(env->prog->len, sizeof(*scev->discovered), GFP_KERNEL_ACCOUNT); + if (!scev->envs || !scev->discovered) + goto nomem; + /* + * Remember original program length, in case bpf_free_scev() + * is called after bpf program rewrites that increase program + * length. + */ + scev->envs_cnt = env->prog->len; + return 0; +nomem: + bpf_free_scev(env); + return -ENOMEM; +} + +static struct bpf_stack_state *scev_regno_to_stack(const struct bpf_func_state *st, u32 r) +{ + int spi, slots_available; + + if (r < MAX_BPF_REG) + return NULL; + + slots_available = st->allocated_stack / BPF_REG_SIZE; + spi = r - MAX_BPF_REG; + if (spi < slots_available) + return &st->stack[spi]; + + return NULL; +} + +static struct bpf_reg_state *scev_regno_to_reg(struct bpf_func_state *st, u32 r) +{ + return r < MAX_BPF_REG ? &st->regs[r] : NULL; +} + +/* Linear widening requires a physical register or a full spill. */ +static struct bpf_reg_state *scev_reg_for_linear(struct bpf_func_state *st, u32 r) +{ + struct bpf_stack_state *stack; + + if (r < MAX_BPF_REG) + return scev_regno_to_reg(st, r); + stack = scev_regno_to_stack(st, r); + if (!stack || bpf_spill_size(stack) != BPF_REG_SIZE) + return NULL; + return &stack->spilled_ptr; +} + +/* + * Return a borrowed value; never use it as a widening destination. + * NULL denotes no readable value. + */ +static const struct bpf_reg_state * +scev_reg_for_read(struct bpf_verifier_env *env, const struct bpf_func_state *st, u32 r) +{ + const struct bpf_stack_state *stack; + bool zero = true; + int i; + + if (r < MAX_BPF_REG) + return st->regs[r].type == NOT_INIT ? NULL : &st->regs[r]; + if (r >= REGS_NUM || r - MAX_BPF_REG >= env->stack_limit / BPF_REG_SIZE) + return NULL; + + stack = scev_regno_to_stack(st, r); + if (!stack) + return &env->scev->unknown_reg; + if (bpf_spill_size(stack) == BPF_REG_SIZE) + return &stack->spilled_ptr; + + /* Partial spills do not describe the value of the whole slot. */ + for (i = 0; i < BPF_REG_SIZE; i++) { + switch (stack->slot_type[i]) { + case STACK_ZERO: + break; + case STACK_INVALID: + case STACK_POISON: + case STACK_MISC: + zero = false; + break; + case STACK_SPILL: + if (!bpf_is_spilled_scalar_reg(stack)) + return NULL; + zero = false; + break; + default: + return NULL; + } + } + return zero ? &env->scev->zero_reg : &env->scev->unknown_reg; +} + +/* Materialize a writable value in the current state, growing its stack if needed. */ +static int scev_reg_for_write(struct bpf_verifier_env *env, struct bpf_func_state *st, + u32 r, struct bpf_reg_state **out) +{ + const struct bpf_reg_state *value; + struct bpf_stack_state *stack; + u32 spi; + int err; + + *out = scev_regno_to_reg(st, r); + if (*out) + return 0; + if (r >= REGS_NUM || r - MAX_BPF_REG >= env->stack_limit / BPF_REG_SIZE) + return -EINVAL; + + spi = r - MAX_BPF_REG; + err = bpf_grow_stack_state(env, st, (spi + 1) * BPF_REG_SIZE); + if (err) + return err; + + /* Resolve pointers after growth, which may relocate the stack. */ + value = scev_reg_for_read(env, st, r); + if (!value) + return -EINVAL; + stack = scev_regno_to_stack(st, r); + stack->spilled_ptr = *value; + memset(stack->slot_type, STACK_SPILL, sizeof(stack->slot_type)); + *out = &stack->spilled_ptr; + return 0; +} + +static bool scev_reg_alive(struct bpf_verifier_env *env, struct bpf_verifier_state *st, u32 r) +{ + int insn_idx = bpf_frame_insn_idx(st, st->curframe); + u16 live_regs = env->insn_aux_data[insn_idx].live_regs_before; + int spi; + + if (r < MAX_BPF_REG) { + return BIT(r) & live_regs; + } else { + spi = r - MAX_BPF_REG; + return bpf_stack_slot_alive(env, st->curframe, spi * 2) || + bpf_stack_slot_alive(env, st->curframe, spi * 2 + 1); + } +} + +/* + * Latch is a condition deciding if execution remains inside a loop. + * Linear latch represents a condition 'if goto ', + * where equation ' + i * ' describes values taken by register , + * 'i' is the loop iteration number, starting from 0. + * 'zext32' indicates the latch is for (zext32 (linear ...)) SCEV. + * 'jmp32' indicates that the latch is a JMP32 instruction. + */ +struct linear_latch { + u32 insn_idx; + u32 base_expr; + u32 step_expr; + u32 bound_expr; + u8 op; + bool zext32; + bool jmp32; +}; + +static bool loop_invariant(struct scev *scev, u32 id) +{ + s64 imm; + u32 reg; + + return is_reg(scev, id, ®) || is_imm(scev, id, &imm); +} + +static int match_linear_latch(struct bpf_verifier_env *env, + u32 header, + u32 latch_idx, + struct linear_latch *latch) +{ + struct bpf_insn *insn = &env->prog->insnsi[latch_idx]; + struct scev *scev = env->scev; + struct env *latch_env; + u32 true_branch_loop; + u32 true_branch_tgt; + u32 src_reg_scev; + u32 dst_reg_scev; + u32 t, l, r, op; + int id; + + op = BPF_OP(insn->code); + latch->jmp32 = BPF_CLASS(insn->code) == BPF_JMP32; + /* Flip the condition if true branch jumps out of the loop. */ + true_branch_tgt = latch_idx + bpf_jmp_offset(insn) + 1; + true_branch_loop = bpf_loop_at_index(env, true_branch_tgt); + if (true_branch_loop != header && + !bpf_is_nested_loop(env, true_branch_loop, header)) + op = bpf_rev_opcode(op); + switch (op) { + case BPF_JSLT: + case BPF_JSLE: + case BPF_JSGT: + case BPF_JSGE: + case BPF_JLT: + case BPF_JLE: + case BPF_JGT: + case BPF_JGE: + case BPF_JNE: + break; + default: + return false; + } + + latch->insn_idx = latch_idx; + + latch_env = find_loop_env(scev, header, latch_idx); + if (verifier_bug_if(!latch_env, env, "header_idx=%d, latch_idx=%d\n", header, latch_idx)) + return -EFAULT; + dst_reg_scev = latch_env->reg2scev[insn->dst_reg]; + src_reg_scev = latch_env->reg2scev[insn->src_reg]; + /* + * Make sure (linear ...) is in dst_reg_scev: + * 'if ' ⇔ + * 'if '. + */ + if (BPF_SRC(insn->code) == BPF_X && + is_linear_or_linear32(scev, src_reg_scev, &l, &r, &latch->zext32)) { + swap(dst_reg_scev, src_reg_scev); + op = bpf_flip_opcode(op); + } + latch->op = op; + /* + * simplify() produces two shapes for linear latches: + * - (linear register step) + * ^----------------------------+-- base_expr + * base_expr | + * - (linear (+ register imm) step) | + * ^----------------------------' + */ + if (!is_linear_or_linear32(scev, dst_reg_scev, + &latch->base_expr, &latch->step_expr, &latch->zext32)) + return false; + + if (BPF_SRC(insn->code) == BPF_K) { + id = imm_expr(scev, insn->imm); + if (id < 0) + return id; + latch->bound_expr = id; + } else { + latch->bound_expr = src_reg_scev; + } + + if (!loop_invariant(scev, latch->step_expr) || + !loop_invariant(scev, latch->bound_expr)) + return false; + + /* (linear register step) */ + if (is_reg(scev, latch->base_expr, &t)) + return true; + + /* (linear (+ register imm) step) */ + if (is_add(scev, latch->base_expr, &l, &r) && + is_reg(scev, l, &t) && + loop_invariant(scev, r)) + return true; + + return false; +} + +struct scev_value { + u64 value; /* scalar value or a pointer offset */ + int ptr_reg; /* SCEV register identifying the pointer origin, or -1. */ +}; + +static void log_scev_value(struct bpf_verifier_env *env, struct bpf_func_state *st, + const struct scev_value *v) +{ + const struct bpf_reg_state *reg; + + if (v->ptr_reg >= 0) { + reg = scev_reg_for_read(env, st, v->ptr_reg); + log_reg(env, v->ptr_reg); + bpf_log(&env->log, " (%s)", reg ? reg_type_str(env, reg->type) : "unavailable"); + } else { + bpf_log(&env->log, "scalar %llu", v->value); + } +} + +static void log_incompatible_scev_values(struct bpf_verifier_env *env, const char *reason, + struct bpf_func_state *st, + const struct scev_value *a, const struct scev_value *b) +{ + bpf_log(&env->log, "scev: %s ", reason); + log_scev_value(env, st, a); + bpf_log(&env->log, ", "); + log_scev_value(env, st, b); + bpf_log(&env->log, "\n"); +} + +static bool scev_value_compatible(struct bpf_verifier_env *env, struct bpf_func_state *st, + const struct scev_value *a, const struct scev_value *b) +{ + const struct bpf_reg_state *ra, *rb; + + if (a->ptr_reg < 0 || b->ptr_reg < 0) { + if (a->ptr_reg < 0 && b->ptr_reg < 0) + return true; + goto incompatible; + } + ra = scev_reg_for_read(env, st, a->ptr_reg); + rb = scev_reg_for_read(env, st, b->ptr_reg); + if (ra && rb && bpf_same_memory_origin(ra, rb)) + return true; + +incompatible: + if (env->log.level & BPF_LOG_LEVEL2) + log_incompatible_scev_values(env, "incompatible latch operands", st, a, b); + return false; +} + +static bool scev_value_add(struct bpf_verifier_env *env, struct bpf_func_state *st, + const struct scev_value *a, const struct scev_value *b, + struct scev_value *result) +{ + if (a->ptr_reg >= 0 && b->ptr_reg >= 0) { + if (env->log.level & BPF_LOG_LEVEL2) + log_incompatible_scev_values(env, "can't add two pointers", st, a, b); + return false; + } + result->value = a->value + b->value; + result->ptr_reg = a->ptr_reg >= 0 ? a->ptr_reg : b->ptr_reg; + return true; +} + +static bool eval_expr(struct bpf_verifier_env *env, struct scev *scev, + struct bpf_func_state *st, u32 id, struct scev_value *result) +{ + struct scev_value lval, rval; + const struct bpf_reg_state *reg; + u32 l, r, regno; + s64 imm; + + if (is_reg(scev, id, ®no)) { + reg = scev_reg_for_read(env, st, regno); + if (!reg || reg->type == NOT_INIT || !tnum_is_const(reg->var_off)) + return false; + result->value = reg->var_off.value; + result->ptr_reg = reg->type == SCALAR_VALUE ? -1 : (int)regno; + return true; + } else if (is_imm(scev, id, &imm)) { + result->value = imm; + result->ptr_reg = -1; + return true; + } else if (is_add(scev, id, &l, &r) && + eval_expr(env, scev, st, l, &lval) && + eval_expr(env, scev, st, r, &rval)) { + return scev_value_add(env, st, &lval, &rval, result); + } + return false; +} + +static bool is_signed_op(u8 op) +{ + switch (op) { + case BPF_JSLT: + case BPF_JSLE: + case BPF_JSGT: + case BPF_JSGE: + return true; + default: + return false; + } +} + +static bool compute_backedge_taken_count(u8 op, u64 initial, u64 step, u64 bound, u64 *result) +{ + u64 rem, diff, backedge_taken_count; + bool up, down; + + if (step == 0) + return false; + down = (s64)step < 0; + up = (s64)step > 0; + diff = bound - initial; + switch (op) { + case BPF_JLT: + /* + * E.g.: 3 + 2*i < 7 ⇔ 2*i < 4 ⇔ i ∈ [0,1]. + * Here the inequality describes a set of values `i` might take + * while the latch condition remains true (the backedge is taken). + * The backedge_taken_count is the size of this set. + */ + if (/* + * If the condition is initially true, decreasing values won't + * change that without underflow. If the condition is initially false, + * the main pass would do a single iteration anyway. + */ + down || + /* Is the latch condition false on the first iteration? */ + initial >= bound || + /* Count iterations satisfying the strict inequality, including i=0. */ + check_add_overflow(div64_u64(diff - 1, step), 1, &backedge_taken_count) || + /* + * Check if the last iteration overflows the counter: + * initial + backedge_taken_count * step > U64_MAX + */ + backedge_taken_count > div64_u64(U64_MAX - initial, step)) + return false; + break; + case BPF_JLE: + /* E.g.: 3 + 2*i ≤ 7 ⇔ 2*i ≤ 4 ⇔ i ∈ [0,1,2]. */ + if (down || initial > bound || + /* Count iterations satisfying the non-strict inequality. */ + check_add_overflow(div64_u64(diff, step), 1, &backedge_taken_count) || + backedge_taken_count > div64_u64(U64_MAX - initial, step)) + return false; + break; + case BPF_JGT: + /* E.g.: 7 - 2*i > 3 ⇔ 2*i < 4 ⇔ i ∈ [0,1]. */ + if (up || initial <= bound || + check_add_overflow(div64_u64(-diff - 1, -step), 1, &backedge_taken_count) || + /* + * The last iteration underflows the counter if: + * initial + backedge_taken_count * step < 0 ⇔ + * backedge_taken_count * step < -initial ⇔ + * backedge_taken_count > initial / -step + * (direction flips because step < 0) + */ + backedge_taken_count > div64_u64(initial, -step)) + return false; + break; + case BPF_JGE: + /* E.g.: 7 - 2*i ≥ 3 ⇔ 2*i ≤ 4 ⇔ i ∈ [0,1,2]. */ + if (up || initial < bound || + check_add_overflow(div64_u64(-diff, -step), 1, &backedge_taken_count) || + backedge_taken_count > div64_u64(initial, -step)) + return false; + break; + case BPF_JSLT: + if (down || (s64)initial >= (s64)bound || + check_add_overflow(div64_u64(diff - 1, step), 1, &backedge_taken_count) || + /* + * Check if the last iteration overflows the signed counter: + * (s64)initial + backedge_taken_count * step > S64_MAX + */ + backedge_taken_count > div64_u64((u64)S64_MAX - initial, step)) + return false; + break; + case BPF_JSLE: + if (down || (s64)initial > (s64)bound || + check_add_overflow(div64_u64(diff, step), 1, &backedge_taken_count) || + backedge_taken_count > div64_u64((u64)S64_MAX - initial, step)) + return false; + break; + case BPF_JSGT: + if (up || (s64)initial <= (s64)bound || + check_add_overflow(div64_u64(-diff - 1, -step), 1, &backedge_taken_count) || + /* + * The last iteration underflows the signed counter if: + * (s64)initial + backedge_taken_count * step < S64_MIN ⇔ + * backedge_taken_count * step < S64_MIN - (s64)initial ⇔ + * backedge_taken_count > initial - (u64)S64_MIN / -step + * (direction flips because step < 0) + */ + backedge_taken_count > div64_u64(initial - (u64)S64_MIN, -step)) + return false; + break; + case BPF_JSGE: + if (up || (s64)initial < (s64)bound || + check_add_overflow(div64_u64(-diff, -step), 1, &backedge_taken_count) || + backedge_taken_count > div64_u64(initial - (u64)S64_MIN, -step)) + return false; + break; + case BPF_JNE: + /* + * For a decreasing counter flip the diff and step: + * 7 - 2*i ≠ 3 ⇔ 2*i ≠ 7 - 3. + */ + diff = down ? -diff : diff; + step = down ? -step : step; + backedge_taken_count = div64_u64_rem(diff, step, &rem); + if (diff == 0 || rem != 0) + return false; + break; + default: + return false; + } + + *result = backedge_taken_count; + return true; +} + +/* + * Loop with post-condition: + * + * r0 = 0 + * l: ... r0 ∈ [0,1,2] header executed 3 times + * r0 += 1 r0 ∈ [0,1,2] + * ... r0 ∈ [1,2,3] + * if r0 != 3 goto l r0 ∈ [1,2,3] backedge taken 2 times + * ... r0 ∈ [3] + * + * SCEV at header: r0 = k + * SCEV at latch: r0 = 1 + k + * + * Loop with pre-condition: + * + * r0 = 0 + * l: ... r0 ∈ [0,1,2,3] header executed 4 times + * if r0 == 3 goto e r0 ∈ [0,1,2,3] backedge taken 3 times + * r0 += 1 r0 ∈ [0,1,2] + * ... r0 ∈ [1,2,3] + * goto l r0 ∈ [1,2,3] + * e: ... r0 ∈ [3] + * + * SCEV at header: r0 = k + * SCEV at latch: r0 = k + */ +static bool compute_max_iters(struct bpf_verifier_env *env, + struct bpf_func_state *st, + struct linear_latch *latch, + struct bpf_loop_iters *iters) +{ + struct bpf_insn_aux_data *aux = env->insn_aux_data; + struct bpf_loop *loop = aux[bpf_loop_at_index(env, latch->insn_idx)].loop; + struct scev *scev = env->scev; + struct scev_value initial_val, step_val, bound_val; + u64 step, bound, initial, total_change, backedge_taken_count; + u8 op = latch->op; + + if (!eval_expr(env, scev, st, latch->base_expr, &initial_val) || + !eval_expr(env, scev, st, latch->step_expr, &step_val) || + !eval_expr(env, scev, st, latch->bound_expr, &bound_val)) + return false; + + if (step_val.ptr_reg >= 0) { + if (env->log.level & BPF_LOG_LEVEL2) { + bpf_log(&env->log, "scev: latch step is a pointer "); + log_scev_value(env, st, &step_val); + bpf_log(&env->log, "\n"); + } + return false; + } + if (!scev_value_compatible(env, st, &initial_val, &bound_val)) + return false; + + initial = initial_val.value; + step = step_val.value; + bound = bound_val.value; + + /* + * The code below only handles mixed 64-bit/32-bit comparisons/recurrences + * when initial bounds and step fit in 32-bit domain. + */ + if (latch->zext32 || latch->jmp32) { + /* + * With the above constraint doing `l: w0 += 1; if r0 s< 256 goto l;` + * is equivalent to `l: w0 += 1; if r0 < 256 goto l;` + */ + if (latch->jmp32 && is_signed_op(op)) { + initial = (s64)(s32)(u32)initial; + bound = (s64)(s32)(u32)bound; + step = (s64)(s32)(u32)step; + } else { + initial = (u32)initial; + bound = (u32)bound; + step = (s64)(s32)(u32)step; + } + } + if (latch->zext32 ^ latch->jmp32 && (initial != initial_val.value || + bound != bound_val.value || + step != step_val.value)) { + if (env->log.level & BPF_LOG_LEVEL2) + bpf_log(&env->log, + "scev: mixed-width latch has operands outside 32-bit ranges\n"); + return false; + } + + + if (!compute_backedge_taken_count(op, initial, step, bound, &backedge_taken_count)) + return false; + + if (latch->zext32 || latch->jmp32) { + /* + * compute_backedge_taken_count() guarantees that 'total_change' + * does not overflow in 64-bit domain. + */ + total_change = initial + step * backedge_taken_count; + if (latch->jmp32 && is_signed_op(op)) { + if ((s64)total_change < S32_MIN || (s64)total_change > S32_MAX) + return false; + } else { + if (total_change > U32_MAX) + return false; + } + } + + /* + * max_header_count is u32 and U32_MAX means "infinite", + * check to avoid overflow below. + */ + if (backedge_taken_count >= U32_MAX - 1) + return false; + + /* The header executes once on entry and once for each taken backedge, hence +1. */ + iters->max_header_count = backedge_taken_count + 1; + /* + * The latch is a conditional jump with one jump target exiting the loop. + * Linear latch is matched only if the loop has a single backedge. + * The loop still, however can have multiple exits. + * In such case, conservatively assume that non-latch exit can happen + * at any iteration, thus setting minimal number of iterations as 0. + */ + iters->min_header_count = loop->exits_cnt == 1 ? iters->max_header_count : 0; + return true; +} + +static void mark_scev_reg_scratched(struct bpf_verifier_env *env, u32 r) +{ + if (r < MAX_BPF_REG) + mark_reg_scratched(env, r); + else + mark_stack_slot_scratched(env, r - __MAX_BPF_REG); +} + +/* + * An unknown scalar is its own widening. + * Stack bytes without a value are represented by scev->unknown_reg. + */ +static bool is_unbound_scev_reg(struct bpf_verifier_env *env, const struct bpf_func_state *st, + u32 r) +{ + const struct bpf_reg_state *reg = scev_reg_for_read(env, st, r); + + return reg && reg->type == SCALAR_VALUE && !reg->id && reg->step <= 1 && + tnum_is_unknown(reg->var_off) && + reg->r64.size == U64_MAX && reg->r32.size == U32_MAX; +} + +/* Main logic in verifier.c forbids varying offsets for certain register types. */ +static bool is_widenable_reg_type(const struct bpf_reg_state *reg) +{ + if (type_may_be_null(reg->type)) + return false; + + switch (base_type(reg->type)) { + case SCALAR_VALUE: + case PTR_TO_MAP_VALUE: + case PTR_TO_MAP_KEY: + case PTR_TO_STACK: + case PTR_TO_PACKET: + case PTR_TO_PACKET_META: + case PTR_TO_MEM: + case PTR_TO_BUF: + case PTR_TO_BTF_ID: + return true; + default: + return false; + } +} + +static void scratch_widened_reg_id(struct bpf_verifier_env *env, struct bpf_reg_state *reg) +{ + switch (base_type(reg->type)) { + case SCALAR_VALUE: + reg->id = 0; + break; + case PTR_TO_PACKET: + case PTR_TO_PACKET_META: + reg->id = ++env->id_gen; + break; + default: + break; + } +} + +/* Check that ANY leaves can be unioned with r's loop-entry value. */ +static bool is_widenable_any(struct bpf_verifier_env *env, struct bpf_func_state *loop_entry, + struct env *header_env, u32 r, u32 id) +{ + const struct bpf_reg_state *reg, *leaf_reg; + struct scev *scev = env->scev; + u32 l, ra, rr, leaf, order; + s64 imm; + + if (!is_any(scev, id, &l, &rr)) + return false; + reg = scev_reg_for_read(env, loop_entry, r); + if (!reg || !is_widenable_reg_type(reg)) + return false; + + scev->stack_sz = 0; + expr_stack_push(scev, id); + while (expr_next(scev, &id, &order)) { + if (order & DEPTH_LIMIT) + return false; + if (!(order & PRE) || is_any(scev, id, &l, &rr)) + continue; + if (is_imm(scev, id, &imm)) { + if (reg->type != SCALAR_VALUE) + return false; + } else if (is_reg(scev, id, &leaf)) { + /* Other leaves must denote loop-invariant registers. */ + if (leaf != r && + !(is_reg(scev, header_env->reg2scev[leaf], &ra) && ra == leaf)) + return false; + leaf_reg = scev_reg_for_read(env, loop_entry, leaf); + if (!leaf_reg || !bpf_reg_union_compatible(reg, leaf_reg)) + return false; + } else { + return false; + } + } + return true; +} + +struct bounds { + struct cnum64 r64; + struct cnum32 r32; + u16 base; + u16 step; +}; + +static void log_bounds(struct bpf_verifier_env *env, const struct bounds *bounds) +{ + struct bpf_verifier_log *log = &env->log; + + bpf_verbose_snum(env, cnum64_smin(bounds->r64)); + bpf_log(log, ".."); + bpf_verbose_snum(env, cnum64_smax(bounds->r64)); + if (bounds->r32.size < U32_MAX) { + bpf_log(log, " (s32 "); + bpf_verbose_snum(env, cnum32_smin(bounds->r32)); + bpf_log(log, ".."); + bpf_verbose_snum(env, cnum32_smax(bounds->r32)); + bpf_log(log, ")"); + } + bpf_log(log, " step %u", bounds->step); +} + +struct linear_coefs { + u32 base_reg; + s64 slope; + bool zext32; +}; + +static bool is_simple_linear(struct scev *scev, u32 id, struct linear_coefs *out) +{ + u32 base, slope; + + return is_linear_or_linear32(scev, id, &base, &slope, &out->zext32) && + is_reg(scev, base, &out->base_reg) && + is_imm(scev, slope, &out->slope) && + out->slope <= S16_MAX && + out->slope >= S16_MIN && + out->slope != 0; +} + +static struct cnum64 cnum64_zext32(struct cnum32 r32) +{ + return cnum64_from_urange(cnum32_umin(r32), cnum32_umax(r32)); +} + +/* Compute the range an induction variable in `reg` spans over the loop. */ +static void linear_bounds(struct bpf_reg_state *reg, u32 max_header_count, + struct linear_coefs *coefs, struct bounds *out) +{ + s64 slope = coefs->slope; + s64 slope_abs = slope < 0 ? -slope : slope; + s64 min_val = reg_smin(reg); + s64 max_val = reg_smax(reg); + struct cnum64 r64; + struct cnum32 r32; + s64 total_change; + bool overflow; + + overflow = check_mul_overflow(slope, (s64)max_header_count - 1, &total_change) || + (slope > 0 && check_add_overflow(max_val, total_change, &max_val)) || + (slope < 0 && check_add_overflow(min_val, total_change, &min_val)); + r64 = overflow ? CNUM64_UNBOUNDED : cnum64_from_srange(min_val, max_val); + r32 = CNUM32_UNBOUNDED; + if (coefs->zext32) { + r32 = cnum32_from_cnum64(r64); + r64 = cnum64_zext32(r32); + /* + * 32-bit iteration overflow is equivalent to subtracting N * 2**32: + * (base + slope * k) % slope ≠ (base + slope * k - N * 2**32) % slope. + */ + overflow |= cnum32_contains(r32, 0) && cnum32_contains(r32, U32_MAX); + /* + * The entry and zero-extended values must agree on how they define `base`, + * otherwise `base + slope * k` would define different sets of points between + * the two. + */ + overflow |= imod(reg_smin(reg), slope_abs) != + imod(cnum64_smin(r64), slope_abs); + /* + * Full range is a union of a reg's value at loop entry, + * plus a range produced by 32-bit iteration. + */ + r64 = cnum64_union(reg->r64, r64); + r32 = cnum32_union(reg->r32, r32); + } + /* + * If the entry value is a single point the value set is 'v + slope * k', + * so the step is |slope|. Otherwise, only the power-of-two alignment + * shared by the entry value and the slope. + */ + if (cnum64_is_const(reg->r64) && !overflow) { + out->step = slope_abs; + out->base = imod(reg_smin(reg), out->step); + } else { + out->step = 1u << min_t(u32, tnum_alignment(reg->var_off), __ffs(slope_abs)); + out->base = 0; + } + out->r64 = r64; + out->r32 = r32; +} + +static int mark_expr_precise(struct bpf_verifier_env *env, struct bpf_func_state *st, u32 id, + unsigned long *precise_regs) +{ + struct scev *scev = env->scev; + struct bpf_reg_state *reg; + u32 r, order; + + scev->stack_sz = 0; + expr_stack_push(scev, id); + while (expr_next(scev, &id, &order)) { + if (order & DEPTH_LIMIT) { + verifier_bug(env, "scev precision marking exceeds expression depth limit"); + return -EFAULT; + } + if (!(order & PRE) || !is_reg(scev, id, &r)) + continue; + /* Synthetic values have no spilled register to mark precise. */ + reg = scev_reg_for_linear(st, r); + if (reg && reg->type == SCALAR_VALUE) + __set_bit(r, precise_regs); + } + return 0; +} + +int bpf_compute_loop_iters(struct bpf_verifier_env *env, struct bpf_verifier_state *st, + struct bpf_loop_iters *iters) +{ + struct bpf_func_state *cur_func = st->frame[st->curframe]; + struct bpf_insn_aux_data *aux = env->insn_aux_data; + struct bpf_verifier_log *log = &env->log; + struct scev *scev = env->scev; + struct env *header_env; + struct linear_latch latch, best; + struct linear_coefs coefs; + struct bpf_backedge *backedge; + struct bpf_loop_iters tmp; + struct bpf_loop *loop; + int insn_idx = st->insn_idx; + bool found = false; + int linear_latch; + int latch_idx; + int err, j; + u32 r; + + if (!env->allow_uninit_stack) + return 0; + + /* + * If insn_idx is a loop header for a reducible loop with a single backedge. + * loop is NULL for secondary entries to irreducible loops. + */ + loop = aux[insn_idx].loop; + if (!loop || loop->irreducible || loop->backedges_cnt != 1 || loop->backedges_overflow || + loop->exits_overflow) { + if (log->level & BPF_LOG_LEVEL2) + bpf_log(log, "loop header at %d, unsupported loop:%s%s%s\n", insn_idx, + !loop || loop->irreducible ? " irreducible" : "", + loop && loop->backedges_cnt > 1 ? " multiple backedges" : "", + loop && loop->exits_overflow ? " too many exits" : ""); + return 0; + } + + /* If this backedge has a latch */ + backedge = &loop->backedges[0]; + if (!backedge->latches_cnt) { + if (log->level & BPF_LOG_LEVEL2) + bpf_log(log, "loop header at %d, latch not identified\n", insn_idx); + return 0; + } + + /* + * Every condition dominating the backedge bounds the number of + * iterations from above, keep the smallest bound. + */ + for (j = 0; j < backedge->latches_cnt; j++) { + latch_idx = backedge->latches[j]; + linear_latch = match_linear_latch(env, insn_idx, latch_idx, &latch); + if (linear_latch < 0) + return linear_latch; + if (!linear_latch) { + if (log->level & BPF_LOG_LEVEL2) + bpf_log(log, "loop header at %d, non-linear %s at %d\n", + insn_idx, j ? "exit" : "latch", latch_idx); + continue; + } + if (!compute_max_iters(env, cur_func, &latch, &tmp)) { + if (log->level & BPF_LOG_LEVEL2) + bpf_log(log, + "loop header at %d, can't compute iterations count at %d\n", + insn_idx, latch_idx); + continue; + } + if (!found || tmp.max_header_count < iters->max_header_count) { + iters->max_header_count = tmp.max_header_count; + best = latch; + } + iters->min_header_count = found ? min(iters->min_header_count, tmp.min_header_count) + : tmp.min_header_count; + found = true; + } + if (!found) { + if (log->level & BPF_LOG_LEVEL2) + bpf_log(log, "loop header at %d, can't compute iterations count\n", insn_idx); + return 0; + } + latch = best; + + if (iters->max_header_count == 0) { + if (log->level & BPF_LOG_LEVEL2) + bpf_log(log, "loop header at %d, 0 iterations count\n", insn_idx); + return 0; + } + + if (iters->max_header_count == U32_MAX) { + if (log->level & BPF_LOG_LEVEL2) + bpf_log(log, "loop header at %d, inf iterations count\n", insn_idx); + return 0; + } + + if (log->level & BPF_LOG_LEVEL2) { + bpf_log(log, "loop header at %d, header_count is ", insn_idx); + if (iters->min_header_count == iters->max_header_count) + bpf_log(log, "%u ", iters->max_header_count); + else + bpf_log(log, "[%u..%u] ", iters->min_header_count, iters->max_header_count); + bpf_log(log, "\n"); + } + + err = bpf_live_stack_query_init(env, st); + if (err) + return err; + + header_env = find_header_env(scev, insn_idx); + for (r = 0; r < REGS_NUM; r++) { + struct bpf_reg_state *reg; + u32 ra, r_scev, r_expr; + bool spill_base = false; + + if (!scev_reg_alive(env, st, r)) + continue; + /* + * A heuristic: + * Ignore SCEVs computed for unbound values and treat those as widened. + * This is poor man's modelling of e.g. bpf_probe_read_kernel() effect + * on stack, when called from a subprogram. Currently this would produce + * an unknown ('?') SCEV as a summary of a subprogram call effects. + * Ideally, this should be computed as an OPAQUE value. + * However, that would require tracking STACK_MISC writes in the + * liveness analysis. Not implemented yet. + */ + if (is_unbound_scev_reg(env, cur_func, r)) + continue; + + r_scev = header_env->reg2scev[r]; + reg = scev_reg_for_linear(cur_func, r); + /* If SCEV for r is (linear ) */ + if (reg && is_simple_linear(scev, r_scev, &coefs) && is_widenable_reg_type(reg)) { + /* Spills at varying offsets lose precision */ + if (test_bit(r, loop->store_base_regs)) { + spill_base = true; + goto cant_widen; + } + continue; + } + + /* rA = rA, loop does not change this reg */ + if (is_reg(scev, r_scev, &ra) && r == ra) + continue; + + /* (any 1 (any 2 (any 3 4))) */ + if (is_widenable_any(env, cur_func, header_env, r, r_scev)) { + if (test_bit(r, loop->store_base_regs)) { + spill_base = true; + goto cant_widen; + } + continue; + } +cant_widen: + if (log->level & BPF_LOG_LEVEL2) { + r_expr = header_env->reg2expr[r]; + bpf_log(log, "loop header at %d, can't widen ", insn_idx); + log_reg(env, r); + bpf_log(log, ", expr is "); + log_expr(env, r_expr); + if (spill_base) + bpf_log(log, ", requires exact stack-offset tracking"); + bpf_log(log, "\n"); + } + return 0; + } + + bitmap_zero(iters->precise_regs, REGS_NUM); + err = mark_expr_precise(env, cur_func, latch.base_expr, iters->precise_regs); + if (err) + return err; + err = mark_expr_precise(env, cur_func, latch.step_expr, iters->precise_regs); + if (err) + return err; + err = mark_expr_precise(env, cur_func, latch.bound_expr, iters->precise_regs); + if (err) + return err; + + return 1; +} + +/* bpf_compute_loop_iters() checked that every leaf can be unioned into acc. */ +static int union_any_reg(struct bpf_verifier_env *env, struct bpf_func_state *loop_entry, + struct bpf_reg_state *acc, u32 id) +{ + struct bpf_reg_state *tmp = &env->fake_reg[0]; + const struct bpf_reg_state *leaf_reg; + struct scev *scev = env->scev; + u32 l, r, leaf, order; + s64 imm; + int err; + + scev->stack_sz = 0; + expr_stack_push(scev, id); + while (expr_next(scev, &id, &order)) { + if (order & DEPTH_LIMIT) { + verifier_bug(env, "scev ANY union exceeds expression depth limit"); + return -EFAULT; + } + if (!(order & PRE) || is_any(scev, id, &l, &r)) + continue; + if (is_imm(scev, id, &imm)) { + bpf_mark_reg_known_scalar(tmp, imm); + leaf_reg = tmp; + } else if (is_reg(scev, id, &leaf)) { + leaf_reg = scev_reg_for_read(env, loop_entry, leaf); + if (verifier_bug_if(!leaf_reg, env, "unreadable SCEV ANY leaf %u", leaf)) + return -EFAULT; + } else { + verifier_bug(env, "scev ANY union has an unsupported leaf"); + return -EFAULT; + } + err = bpf_reg_union(env, acc, leaf_reg); + if (err) + return err; + } + return 0; +} + +int bpf_widen_scev_regs(struct bpf_verifier_env *env, struct bpf_verifier_state *st, + struct bpf_verifier_state *loop_entry, struct bpf_loop_iters *iters) +{ + struct bpf_func_state *cur_func = st->frame[st->curframe]; + struct bpf_func_state *entry_func = loop_entry->frame[loop_entry->curframe]; + struct bpf_verifier_log *log = &env->log; + struct scev *scev = env->scev; + struct bpf_reg_state *reg; + struct env *header_env; + struct linear_coefs coefs; + struct bounds bounds; + u32 r, r_expr, a, b; + int insn_idx = st->insn_idx; + int err; + + for_each_set_bit(r, iters->precise_regs, REGS_NUM) { + if (r < MAX_BPF_REG) + bpf_bt_set_frame_reg(&env->bt, st->curframe, r); + else + bpf_bt_set_frame_slot(&env->bt, st->curframe, r - MAX_BPF_REG); + } + err = bpf_mark_chain_precision(env, st, -1, NULL); + if (err) + return err; + + header_env = find_header_env(scev, insn_idx); + for (r = 0; r < REGS_NUM; r++) { + if (!scev_reg_alive(env, st, r) || is_unbound_scev_reg(env, cur_func, r)) + continue; + + r_expr = header_env->reg2scev[r]; + /* If SCEV for r is (linear )*/ + if (is_simple_linear(scev, r_expr, &coefs) && coefs.base_reg == r) { + err = scev_reg_for_write(env, cur_func, r, ®); + if (err) + return err; + linear_bounds(reg, iters->max_header_count, &coefs, &bounds); + if (log->level & BPF_LOG_LEVEL2) { + bpf_log(log, "loop header at %d, widening ", insn_idx); + log_reg(env, r); + bpf_log(log, " to "); + log_bounds(env, &bounds); + bpf_log(log, "\n"); + } + scratch_widened_reg_id(env, reg); + err = bpf_set_reg_range(env, reg, bounds.r64, bounds.r32, + bounds.base, bounds.step); + if (err) + return err; + mark_scev_reg_scratched(env, r); + } else if (is_any(scev, r_expr, &a, &b)) { + err = scev_reg_for_write(env, cur_func, r, ®); + if (err) + return err; + err = union_any_reg(env, entry_func, reg, r_expr); + if (err) + return err; + if (log->level & BPF_LOG_LEVEL2) { + bpf_log(log, "loop header at %d, widening ", insn_idx); + log_reg(env, r); + bpf_log(log, " to "); + bpf_verbose_snum(env, reg_smin(reg)); + bpf_log(log, ".."); + bpf_verbose_snum(env, reg_smax(reg)); + bpf_log(log, " step %u\n", reg->step); + } + scratch_widened_reg_id(env, reg); + mark_scev_reg_scratched(env, r); + } + } + return 1; +} + +int bpf_clamp_scev_regs(struct bpf_verifier_env *env, struct bpf_func_state *cur_func_state, u32 insn_idx, + struct bpf_verifier_state *entry_state, u32 max_header_count) +{ + struct bpf_func_state *entry_st = entry_state->frame[entry_state->curframe]; + struct bpf_verifier_log *log = &env->log; + struct bpf_reg_state *reg, *entry_reg; + struct scev *scev = env->scev; + struct linear_coefs coefs; + struct env *header_env; + struct bounds bounds; + u32 r; + int err; + + if (entry_state->curframe != cur_func_state->frameno) { + verifier_bug(env, "clamping registers for a wrong frame: %d vs %d\n", + entry_state->curframe, cur_func_state->frameno); + return -EFAULT; + } + + header_env = find_header_env(scev, insn_idx); + for (r = 0; r < REGS_NUM; r++) { + /* If SCEV for r is (linear )*/ + if (!is_simple_linear(scev, header_env->reg2scev[r], &coefs) || + coefs.base_reg != r) + continue; + + entry_reg = scev_reg_for_linear(entry_st, r); + reg = scev_reg_for_linear(cur_func_state, r); + if (!entry_reg || !reg || entry_reg->type == NOT_INIT || reg->type == NOT_INIT || + is_unbound_scev_reg(env, entry_st, r)) + continue; + + linear_bounds(entry_reg, max_header_count, &coefs, &bounds); + bounds.r64 = cnum64_intersect(reg->r64, bounds.r64); + bounds.r32 = cnum32_intersect(reg->r32, bounds.r32); + if (cnum64_is_empty(bounds.r64)) { + verifier_bug(env, "scev clamp produced empty range for r%d", r); + return -EFAULT; + } + if (log->level & BPF_LOG_LEVEL2) { + bpf_log(log, "loop header at %d, clamping ", insn_idx); + log_reg(env, r); + bpf_log(log, " to "); + log_bounds(env, &bounds); + bpf_log(log, "\n"); + } + scratch_widened_reg_id(env, reg); + err = bpf_set_reg_range(env, reg, bounds.r64, bounds.r32, + bounds.base, bounds.step); + if (err) + return err; + mark_scev_reg_scratched(env, r); + } + return 0; +} diff --git a/kernel/bpf/states.c b/kernel/bpf/states.c index 012b82513a3b..d7df6632a5c4 100644 --- a/kernel/bpf/states.c +++ b/kernel/bpf/states.c @@ -302,8 +302,30 @@ int bpf_update_branch_counts(struct bpf_verifier_env *env, struct bpf_verifier_s static bool range_within(const struct bpf_reg_state *old, const struct bpf_reg_state *cur) { - return cnum64_is_subset(old->r64, cur->r64) && - cnum32_is_subset(old->r32, cur->r32); + if (!cnum64_is_subset(old->r64, cur->r64) || + !cnum32_is_subset(old->r32, cur->r32)) + return false; + + if (old->step <= 1) + return true; + + if (cnum64_is_const(cur->r64)) + return imod((s64)cnum64_smin(cur->r64), old->step) == old->base; + + /* + * Both `old` and `cur` define some sets of points. + * Return true, if points defined by `cur` are a subset of points defined by `old`: + * - bounds for `cur` should be within bounds for `old`; + * - cur->step should be dividable by old->step; + * - cur->base should start at integer number of old->step + * steps from old->base. + * + * E.g. the following ranges are compatible: + * - old [0, 10] step 2 + * - new [2, 10] step 4 + */ + return cur->step % old->step == 0 && + (cur->base - old->base) % old->step == 0; } /* If in the old state two registers had the same id, then they need to have @@ -415,14 +437,14 @@ static void __clean_func_state(struct bpf_verifier_env *env, * half_spi 2*i → lower half: slot_type[0..3] (closer to FP) * half_spi 2*i+1 → upper half: slot_type[4..7] (farther from FP) */ - for (i = 0; i < st->allocated_stack / BPF_REG_SIZE; i++) { + for (i = 0; i < bpf_stack_nr_slots(st); i++) { bool lo_live = bpf_stack_slot_alive(env, frame, i * 2); bool hi_live = bpf_stack_slot_alive(env, frame, i * 2 + 1); if (!hi_live || !lo_live) { int start = !lo_live ? 0 : BPF_REG_SIZE / 2; int end = !hi_live ? BPF_REG_SIZE : BPF_REG_SIZE / 2; - u8 stype = st->stack[i].slot_type[7]; + u8 stype = bpf_stack_slot(st, i)->slot_type[7]; /* * Don't clear special slots. @@ -442,7 +464,7 @@ static void __clean_func_state(struct bpf_verifier_env *env, * rejecting as non-scalar register fills. */ if (!hi_live) { - struct bpf_reg_state *spill = &st->stack[i].spilled_ptr; + struct bpf_reg_state *spill = &bpf_stack_slot(st, i)->spilled_ptr; if (lo_live && stype == STACK_SPILL) { if (spill->type != SCALAR_VALUE) @@ -454,7 +476,7 @@ static void __clean_func_state(struct bpf_verifier_env *env, if (bpf_register_is_null(spill)) continue; for (j = 0; j < 4; j++) { - u8 *t = &st->stack[i].slot_type[j]; + u8 *t = &bpf_stack_slot(st, i)->slot_type[j]; if (*t == STACK_SPILL) *t = STACK_MISC; @@ -463,7 +485,7 @@ static void __clean_func_state(struct bpf_verifier_env *env, bpf_mark_reg_not_init(env, spill); } for (j = start; j < end; j++) - st->stack[i].slot_type[j] = STACK_POISON; + bpf_stack_slot(st, i)->slot_type[j] = STACK_POISON; } } } @@ -610,6 +632,8 @@ static bool regsafe(struct bpf_verifier_env *env, struct bpf_reg_state *rold, case PTR_TO_MEM: case PTR_TO_BUF: case PTR_TO_TP_BUFFER: + case PTR_TO_STACK: + case PTR_TO_BTF_ID: /* If the new min/max/var_off satisfy the old ones and * everything else matches, we are OK. */ @@ -643,8 +667,6 @@ static bool regsafe(struct bpf_verifier_env *env, struct bpf_reg_state *rold, /* new val must satisfy old val knowledge */ return range_within(rold, rcur) && tnum_in(rold->var_off, rcur->var_off); - case PTR_TO_STACK: - return regs_exact(rold, rcur, idmap); case PTR_TO_ARENA: return true; case PTR_TO_INSN: @@ -709,37 +731,38 @@ static bool stacksafe(struct bpf_verifier_env *env, struct bpf_func_state *old, * didn't use them */ for (i = 0; i < old->allocated_stack; i++) { + struct bpf_stack_state *old_slot, *cur_slot; struct bpf_reg_state *old_reg, *cur_reg; int im = i % BPF_REG_SIZE; + u8 old_type; spi = i / BPF_REG_SIZE; + old_slot = bpf_stack_slot(old, spi); + old_type = old_slot->slot_type[im]; + cur_slot = i < cur->allocated_stack ? bpf_stack_slot(cur, spi) : NULL; if (exact == EXACT) { - u8 old_type = old->stack[spi].slot_type[i % BPF_REG_SIZE]; - u8 cur_type = i < cur->allocated_stack ? - cur->stack[spi].slot_type[i % BPF_REG_SIZE] : STACK_INVALID; + u8 cur_type = cur_slot ? cur_slot->slot_type[im] : STACK_INVALID; /* STACK_INVALID and STACK_POISON are equivalent for pruning */ if (old_type == STACK_POISON) old_type = STACK_INVALID; if (cur_type == STACK_POISON) cur_type = STACK_INVALID; - if (i >= cur->allocated_stack || old_type != cur_type) + if (!cur_slot || old_type != cur_type) return false; } - if (old->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_INVALID || - old->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_POISON) + if (old_type == STACK_INVALID || old_type == STACK_POISON) continue; - if (env->allow_uninit_stack && - old->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_MISC) + if (env->allow_uninit_stack && old_type == STACK_MISC) continue; /* explored stack has more populated slots than current stack * and these slots were used */ - if (i >= cur->allocated_stack) + if (!cur_slot) return false; /* @@ -749,8 +772,8 @@ static bool stacksafe(struct bpf_verifier_env *env, struct bpf_func_state *old, * regsafe() to ensure scalar ids are compared. */ if (im == 0 || im == 4) { - old_reg = scalar_reg_for_stack(env, &old->stack[spi], im); - cur_reg = scalar_reg_for_stack(env, &cur->stack[spi], im); + old_reg = scalar_reg_for_stack(env, old_slot, im); + cur_reg = scalar_reg_for_stack(env, cur_slot, im); if (old_reg && cur_reg) { if (!regsafe(env, old_reg, cur_reg, idmap, exact)) return false; @@ -763,21 +786,19 @@ static bool stacksafe(struct bpf_verifier_env *env, struct bpf_func_state *old, * it will be safe with zero-initialized stack. * The opposite is not true */ - if (old->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_MISC && - cur->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_ZERO) + if (old_type == STACK_MISC && cur_slot->slot_type[im] == STACK_ZERO) continue; - if (old->stack[spi].slot_type[i % BPF_REG_SIZE] != - cur->stack[spi].slot_type[i % BPF_REG_SIZE]) + if (old_type != cur_slot->slot_type[im]) /* Ex: old explored (safe) state has STACK_SPILL in * this stack slot, but current has STACK_MISC -> * this verifier states are not equivalent, * return false to continue verification of this path */ return false; - if (i % BPF_REG_SIZE != BPF_REG_SIZE - 1) + if (im != BPF_REG_SIZE - 1) continue; /* Both old and cur are having same slot_type */ - switch (old->stack[spi].slot_type[BPF_REG_SIZE - 1]) { + switch (old_type) { case STACK_SPILL: /* when explored and current stack slot are both storing * spilled registers, check that stored pointers types @@ -789,13 +810,13 @@ static bool stacksafe(struct bpf_verifier_env *env, struct bpf_func_state *old, * such verifier states are not equivalent. * return false to continue verification of this path */ - if (!regsafe(env, &old->stack[spi].spilled_ptr, - &cur->stack[spi].spilled_ptr, idmap, exact)) + if (!regsafe(env, &old_slot->spilled_ptr, &cur_slot->spilled_ptr, + idmap, exact)) return false; break; case STACK_DYNPTR: - old_reg = &old->stack[spi].spilled_ptr; - cur_reg = &cur->stack[spi].spilled_ptr; + old_reg = &old_slot->spilled_ptr; + cur_reg = &cur_slot->spilled_ptr; if (old_reg->dynptr.type != cur_reg->dynptr.type || old_reg->dynptr.first_slot != cur_reg->dynptr.first_slot || !check_ids(old_reg->id, cur_reg->id, idmap) || @@ -803,8 +824,8 @@ static bool stacksafe(struct bpf_verifier_env *env, struct bpf_func_state *old, return false; break; case STACK_ITER: - old_reg = &old->stack[spi].spilled_ptr; - cur_reg = &cur->stack[spi].spilled_ptr; + old_reg = &old_slot->spilled_ptr; + cur_reg = &cur_slot->spilled_ptr; /* iter.depth is not compared between states as it * doesn't matter for correctness and would otherwise * prevent convergence; we maintain it only to prevent @@ -820,8 +841,8 @@ static bool stacksafe(struct bpf_verifier_env *env, struct bpf_func_state *old, return false; break; case STACK_IRQ_FLAG: - old_reg = &old->stack[spi].spilled_ptr; - cur_reg = &cur->stack[spi].spilled_ptr; + old_reg = &old_slot->spilled_ptr; + cur_reg = &cur_slot->spilled_ptr; if (!check_ids(old_reg->id, cur_reg->id, idmap) || old_reg->irq.kfunc_class != cur_reg->irq.kfunc_class) return false; @@ -915,6 +936,27 @@ static bool refsafe(struct bpf_verifier_state *old, struct bpf_verifier_state *c return true; } +static bool loop_stack_safe(struct bpf_verifier_env *env, struct bpf_func_state *old, + struct bpf_func_state *cur) +{ + struct loop_stack_entry *old_stack = old->loop_stack; + struct loop_stack_entry *cur_stack = cur->loop_stack; + u32 i; + + if (old->loop_stack_cnt != cur->loop_stack_cnt) + return false; + + for (i = 0; i < old->loop_stack_cnt; i++) { + if (old_stack[i].loop_id == cur_stack[i].loop_id && + old_stack[i].max_header_count == cur_stack[i].max_header_count && + old_stack[i].terminates == cur_stack[i].terminates) + continue; + return false; + } + + return true; +} + /* compare two verifier states * * all states stored in state_list are known to be valid, since @@ -953,6 +995,9 @@ static bool func_states_equal(struct bpf_verifier_env *env, struct bpf_func_stat if (!old->no_stack_arg_load && cur->no_stack_arg_load) return false; + if (old->in_rcu_callback_fn != cur->in_rcu_callback_fn) + return false; + for (i = 0; i < MAX_BPF_REG; i++) if (((1 << i) & live_regs) && !regsafe(env, &old->regs[i], &cur->regs[i], @@ -965,6 +1010,9 @@ static bool func_states_equal(struct bpf_verifier_env *env, struct bpf_func_stat if (!stack_arg_safe(env, old, cur, &env->idmap_scratch, exact)) return false; + if (!loop_stack_safe(env, old, cur)) + return false; + return true; } @@ -998,6 +1046,9 @@ static bool states_equal(struct bpf_verifier_env *env, if (old->in_sleepable != cur->in_sleepable) return false; + if (old->unwound != cur->unwound) + return false; + if (!refsafe(old, cur, &env->idmap_scratch)) return false; @@ -1011,6 +1062,7 @@ static bool states_equal(struct bpf_verifier_env *env, if (!func_states_equal(env, old->frame[i], cur->frame[i], insn_idx, exact)) return false; } + return true; } @@ -1045,10 +1097,12 @@ static int propagate_precision(struct bpf_verifier_env *env, first = false; } - for (i = 0; i < state->allocated_stack / BPF_REG_SIZE; i++) { - if (!bpf_is_spilled_reg(&state->stack[i])) + for (i = 0; i < bpf_stack_nr_slots(state); i++) { + struct bpf_stack_state *ss = bpf_stack_slot(state, i); + + if (!bpf_is_spilled_reg(ss)) continue; - state_reg = &state->stack[i].spilled_ptr; + state_reg = &ss->spilled_ptr; if (state_reg->type != SCALAR_VALUE || !state_reg->precise) continue; @@ -1123,7 +1177,7 @@ static bool states_maybe_looping(struct bpf_verifier_state *old, fcur = cur->frame[fr]; for (i = 0; i < MAX_BPF_REG; i++) if (memcmp(&fold->regs[i], &fcur->regs[i], - offsetof(struct bpf_reg_state, precise))) + offsetofend(struct bpf_reg_state, step))) return false; return true; } @@ -1194,15 +1248,17 @@ static bool iter_active_depths_differ(struct bpf_verifier_state *old, struct bpf for (fr = old->curframe; fr >= 0; fr--) { state = old->frame[fr]; - for (i = 0; i < state->allocated_stack / BPF_REG_SIZE; i++) { - if (state->stack[i].slot_type[0] != STACK_ITER) + for (i = 0; i < bpf_stack_nr_slots(state); i++) { + struct bpf_stack_state *ss = bpf_stack_slot(state, i); + + if (ss->slot_type[0] != STACK_ITER) continue; - slot = &state->stack[i].spilled_ptr; + slot = &ss->spilled_ptr; if (slot->iter.state != BPF_ITER_STATE_ACTIVE) continue; - cur_slot = &cur->frame[fr]->stack[i].spilled_ptr; + cur_slot = &bpf_stack_slot(cur->frame[fr], i)->spilled_ptr; if (cur_slot->iter.depth != slot->iter.depth) return true; } @@ -1224,10 +1280,12 @@ static void mark_all_scalars_imprecise(struct bpf_verifier_env *env, struct bpf_ continue; reg->precise = false; } - for (j = 0; j < func->allocated_stack / BPF_REG_SIZE; j++) { - if (!bpf_is_spilled_reg(&func->stack[j])) + for (j = 0; j < bpf_stack_nr_slots(func); j++) { + struct bpf_stack_state *ss = bpf_stack_slot(func, j); + + if (!bpf_is_spilled_reg(ss)) continue; - reg = &func->stack[j].spilled_ptr; + reg = &ss->spilled_ptr; if (reg->type != SCALAR_VALUE) continue; reg->precise = false; @@ -1235,16 +1293,92 @@ static void mark_all_scalars_imprecise(struct bpf_verifier_env *env, struct bpf_ } } +int bpf_split_cur_state(struct bpf_verifier_env *env) +{ + struct bpf_verifier_state *cur = env->cur_state, *new; + struct bpf_verifier_state_list *new_sl; + struct list_head *head; + int insn_idx = cur->insn_idx; + int err; + + head = bpf_explored_state(env, insn_idx); + new_sl = kzalloc_obj(struct bpf_verifier_state_list, GFP_KERNEL_ACCOUNT); + if (!new_sl) + return -ENOMEM; + env->total_states++; + env->explored_states_size++; + update_peak_states(env); + env->prev_jmps_processed = env->jmps_processed; + env->prev_insn_processed = env->insn_processed; + + /* forget precise markings we inherited, see __mark_chain_precision */ + if (env->bpf_capable) + mark_all_scalars_imprecise(env, cur); + + bpf_clear_singular_ids(env, cur); + + /* add new state to the head of linked list */ + new = &new_sl->state; + err = bpf_copy_verifier_state(new, cur); + if (err) { + bpf_free_verifier_state(new, false); + kfree(new_sl); + return err; + } + new->insn_idx = insn_idx; + verifier_bug_if(new->branches != 1, env, + "%s:branches_to_explore=%d insn %d", + __func__, new->branches, insn_idx); + err = maybe_enter_scc(env, new); + if (err) { + bpf_free_verifier_state(new, false); + kfree(new_sl); + return err; + } + + cur->parent = new; + cur->last_insn_idx = -1; + cur->first_insn_idx = insn_idx; + cur->dfs_depth = new->dfs_depth + 1; + bpf_clear_jmp_history(cur); + list_add(&new_sl->node, head); + return 0; +} + +/* Force a checkpoint upon reaching a loop header for a terminating loop. */ +static bool need_loop_checkpoint(struct bpf_verifier_env *env, int insn_idx) +{ + struct bpf_verifier_state *cur = env->cur_state; + struct bpf_func_state *frame = cur->frame[cur->curframe]; + struct loop_stack_entry *top; + + if (bpf_loop_at_index(env, insn_idx) != insn_idx || !frame->loop_stack_cnt) + return false; + top = &frame->loop_stack[frame->loop_stack_cnt - 1]; + return top->loop_id == insn_idx && top->terminates; +} + +static struct loop_stack_entry *current_loop(struct bpf_verifier_state *st) +{ + struct bpf_func_state *frame = st->frame[st->curframe]; + + if (frame->loop_stack_cnt == 0) + return NULL; + return &frame->loop_stack[frame->loop_stack_cnt - 1]; +} + int bpf_is_state_visited(struct bpf_verifier_env *env, int insn_idx) { - struct bpf_verifier_state_list *new_sl; + struct loop_stack_entry *old_loop, *cur_loop; struct bpf_verifier_state_list *sl; - struct bpf_verifier_state *cur = env->cur_state, *new; - bool force_new_state, add_new_state, loop; + struct bpf_verifier_state *cur = env->cur_state; + bool force_new_state, add_new_state, loop, loop_checkpoint; int n, err, states_cnt = 0; struct list_head *pos, *tmp, *head; + loop_checkpoint = need_loop_checkpoint(env, insn_idx); force_new_state = env->test_state_freq || bpf_is_force_checkpoint(env, insn_idx) || + loop_checkpoint || /* Avoid accumulating infinitely long jmp history */ cur->jmp_history_cnt > 40; @@ -1275,10 +1409,10 @@ int bpf_is_state_visited(struct bpf_verifier_env *env, int insn_idx) continue; if (sl->state.branches) { - struct bpf_func_state *frame = sl->state.frame[sl->state.curframe]; + struct bpf_func_state *old_top_frame = sl->state.frame[0]; - if (frame->in_async_callback_fn && - frame->async_entry_cnt != cur->frame[cur->curframe]->async_entry_cnt) { + if (old_top_frame->in_async_callback_fn && + old_top_frame->async_entry_cnt != cur->frame[0]->async_entry_cnt) { /* Different async_entry_cnt means that the verifier is * processing another entry into async callback. * Seeing the same state is not an indication of infinite @@ -1330,8 +1464,9 @@ int bpf_is_state_visited(struct bpf_verifier_env *env, int insn_idx) */ if (is_iter_next_insn(env, insn_idx)) { if (states_equal(env, &sl->state, cur, RANGE_WITHIN)) { - struct bpf_func_state *cur_frame; + struct bpf_func_state *cur_frame, *iter_frame, *old_frame; struct bpf_reg_state *iter_state, *iter_reg; + struct bpf_reg_state *old_iter; int spi; cur_frame = cur->frame[cur->curframe]; @@ -1344,8 +1479,18 @@ int bpf_is_state_visited(struct bpf_verifier_env *env, int insn_idx) * no need for extra (re-)validations */ spi = bpf_get_spi(iter_reg->var_off.value); - iter_state = &bpf_func(env, iter_reg)->stack[spi].spilled_ptr; - if (iter_state->iter.state == BPF_ITER_STATE_ACTIVE) { + iter_frame = bpf_func(env, iter_reg); + iter_state = &bpf_stack_slot(iter_frame, spi)->spilled_ptr; + old_frame = sl->state.frame[iter_reg->frameno]; + old_iter = &bpf_stack_slot(old_frame, spi)->spilled_ptr; + /* + * states_equal() matches ids through idmap. + * The loop converged only if it's the same + * iterator. The one that was destroyed and + * created again starts from the beginning. + */ + if (iter_state->iter.state == BPF_ITER_STATE_ACTIVE && + iter_state->id == old_iter->id) { loop = true; goto hit; } @@ -1366,6 +1511,33 @@ int bpf_is_state_visited(struct bpf_verifier_env *env, int insn_idx) } goto skip_inf_loop_check; } + /* + * If old state belongs to a control flow loop that we know terminates, + * it should be safe to prune current state. However, account for the + * following situation: + * + * for (;;) { + * for (i = 0; i < 10; i++) + * ... + * } + * + * Here a backedge to a non-terminating outer loop is dominated by an exit + * from the inner loop. Pruning at a re-entry to the inner loop therefore + * would hide the fact that outer loop is non-terminating. + * Hence, only allow pruning states within the same entry state. + */ + old_loop = current_loop(&sl->state); + cur_loop = current_loop(cur); + if (old_loop && cur_loop && + old_loop->terminates && cur_loop->terminates && + old_loop->entry_state == cur_loop->entry_state) { + if (states_equal(env, &sl->state, cur, RANGE_WITHIN)) { + loop = true; + goto hit; + } + goto skip_inf_loop_check; + } + /* attempt to detect infinite loop to avoid unnecessary doomed work */ if (states_maybe_looping(&sl->state, cur) && states_equal(env, &sl->state, cur, EXACT) && @@ -1412,7 +1584,7 @@ int bpf_is_state_visited(struct bpf_verifier_env *env, int insn_idx) */ err = 0; if (bpf_is_jmp_point(env, env->insn_idx)) - err = bpf_push_jmp_history(env, cur, 0, 0, 0, 0); + err = bpf_push_jmp_history(env, cur, 0, 0, 0, NULL, 0); err = err ? : propagate_precision(env, &sl->state, cur, NULL); if (err) return err; @@ -1524,7 +1696,8 @@ int bpf_is_state_visited(struct bpf_verifier_env *env, int insn_idx) * Use bigger 'n' for checkpoints because evicting checkpoint states * too early would hinder iterator convergence. */ - n = bpf_is_force_checkpoint(env, insn_idx) && sl->state.branches > 0 ? 64 : 3; + n = (bpf_is_force_checkpoint(env, insn_idx) || loop_checkpoint) && + sl->state.branches > 0 ? 64 : 3; if (sl->miss_cnt > sl->hit_cnt * n + n) { /* the state is unlikely to be useful. Remove it to * speed up verification @@ -1556,44 +1729,5 @@ int bpf_is_state_visited(struct bpf_verifier_env *env, int insn_idx) * When looping the sl->state.branches will be > 0 and this state * will not be considered for equivalence until branches == 0. */ - new_sl = kzalloc_obj(struct bpf_verifier_state_list, GFP_KERNEL_ACCOUNT); - if (!new_sl) - return -ENOMEM; - env->total_states++; - env->explored_states_size++; - update_peak_states(env); - env->prev_jmps_processed = env->jmps_processed; - env->prev_insn_processed = env->insn_processed; - - /* forget precise markings we inherited, see __mark_chain_precision */ - if (env->bpf_capable) - mark_all_scalars_imprecise(env, cur); - - bpf_clear_singular_ids(env, cur); - - /* add new state to the head of linked list */ - new = &new_sl->state; - err = bpf_copy_verifier_state(new, cur); - if (err) { - bpf_free_verifier_state(new, false); - kfree(new_sl); - return err; - } - new->insn_idx = insn_idx; - verifier_bug_if(new->branches != 1, env, - "%s:branches_to_explore=%d insn %d", - __func__, new->branches, insn_idx); - err = maybe_enter_scc(env, new); - if (err) { - bpf_free_verifier_state(new, false); - kfree(new_sl); - return err; - } - - cur->parent = new; - cur->first_insn_idx = insn_idx; - cur->dfs_depth = new->dfs_depth + 1; - bpf_clear_jmp_history(cur); - list_add(&new_sl->node, head); - return 0; + return bpf_split_cur_state(env); } diff --git a/kernel/bpf/stream.c b/kernel/bpf/stream.c index be9ce98e9469..83e803ea54ef 100644 --- a/kernel/bpf/stream.c +++ b/kernel/bpf/stream.c @@ -2,11 +2,15 @@ /* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */ #include +#include #include #include #include +#include #include #include +#include +#include static void bpf_stream_elem_init(struct bpf_stream_elem *elem, int len) { @@ -22,11 +26,11 @@ static struct bpf_stream_elem *bpf_stream_elem_alloc(int len) size_t alloc_size; /* - * Length denotes the amount of data to be written as part of stream element, - * thus includes '\0' byte. We're capped by how much bpf_bprintf_buffers can - * accomodate, therefore deny allocations that won't fit into them. + * Length is the payload pushed into the stream, excluding the + * trailing NUL of the bprintf buffer. Reject anything that cannot + * fit without copying that NUL into the stream element. */ - if (len < 0 || len > max_len) + if (len < 0 || len >= max_len) return NULL; alloc_size = offsetof(struct bpf_stream_elem, str[len]); @@ -68,25 +72,82 @@ static int bpf_stream_consume_capacity(struct bpf_stream *stream, int len) return 0; } -static void bpf_stream_release_capacity(struct bpf_stream *stream, struct bpf_stream_elem *elem) +static void bpf_stream_release_capacity(struct bpf_stream *stream, int len) { - int len = elem->total_len; - atomic_sub(len, &stream->capacity); } +static void bpf_stream_notify(struct irq_work *work) +{ + struct bpf_stream *stream = container_of(work, struct bpf_stream, notify_work); + + /* + * Writers run in arbitrary program contexts, including NMI and regions + * that already hold wait queue or epoll locks. Wake waiters from + * irq_work instead, where taking those locks is safe. + */ + wake_up_interruptible_poll(&stream->waitq, EPOLLIN | EPOLLRDNORM); +} + +static void bpf_stream_queue_notify(struct bpf_stream *stream) +{ + /* + * Record that the work has been used so that teardown only pays for + * irq_work_sync(), which may wait for an RCU grace period, when a + * callback could actually be in flight. + */ + if (!READ_ONCE(stream->notify_used)) + WRITE_ONCE(stream->notify_used, true); + irq_work_queue(&stream->notify_work); +} + +static int bpf_stream_readable_bytes(struct bpf_stream *stream) +{ + return atomic_read_acquire(&stream->readable); +} + +static void bpf_stream_publish(struct bpf_stream *stream, int len) +{ + /* + * Pairs with atomic_read_acquire() in bpf_stream_readable_bytes(). + * + * Notify only when the stream turns from empty to readable. Readers + * block and pollers wait only after finding it empty, and a read never + * takes the count below zero, so the publication that makes it positive + * is the one they wait for. Publishing into a stream that already holds + * data would only raise an interrupt for waiters that are being woken + * already, or for nobody at all. + */ + if (atomic_add_return_release(len, &stream->readable) == len) + bpf_stream_queue_notify(stream); +} + static int bpf_stream_push_str(struct bpf_stream *stream, const char *str, int len) { - int ret = bpf_stream_consume_capacity(stream, len); + int ret; - return ret ?: __bpf_stream_push_str(&stream->log, str, len); + /* Nothing to publish; do not allocate an element for it. */ + if (!len) + return 0; + + ret = bpf_stream_consume_capacity(stream, len); + if (ret) + return ret; + + ret = __bpf_stream_push_str(&stream->log, str, len); + if (ret) + bpf_stream_release_capacity(stream, len); + else + bpf_stream_publish(stream, len); + + return ret; } static struct bpf_stream *bpf_stream_get(enum bpf_stream_id stream_id, struct bpf_prog_aux *aux) { if (stream_id != BPF_STDOUT && stream_id != BPF_STDERR) return NULL; - return &aux->stream[stream_id - 1]; + return aux->stream[stream_id - 1]; } static void bpf_stream_free_elem(struct bpf_stream_elem *elem) @@ -154,14 +215,17 @@ static bool bpf_stream_consume_elem(struct bpf_stream_elem *elem, int *len) static int bpf_stream_read(struct bpf_stream *stream, void __user *buf, int len) { - int rem_len = len, cons_len, ret = 0; + int read_len, rem_len, cons_len, ret = 0; struct bpf_stream_elem *elem = NULL; struct llist_node *node; mutex_lock(&stream->lock); + read_len = min(len, bpf_stream_readable_bytes(stream)); + rem_len = read_len; while (rem_len) { - int pos = len - rem_len; + int pos = read_len - rem_len; + int chunk, n; bool cont; node = bpf_stream_backlog_peek(stream); @@ -175,37 +239,146 @@ static int bpf_stream_read(struct bpf_stream *stream, void __user *buf, int len) cons_len = elem->consumed_len; cont = bpf_stream_consume_elem(elem, &rem_len) == false; + chunk = elem->consumed_len - cons_len; - ret = copy_to_user(buf + pos, elem->str + cons_len, - elem->consumed_len - cons_len); - /* Restore in case of error. */ - if (ret) { - ret = -EFAULT; - elem->consumed_len = cons_len; + n = copy_to_user(buf + pos, elem->str + cons_len, chunk); + if (n) { + /* Keep any successfully copied bytes; -EFAULT only if none. */ + elem->consumed_len -= n; + rem_len += n; + ret = (read_len == rem_len) ? -EFAULT : 0; break; } if (cont) continue; bpf_stream_backlog_pop(stream); - bpf_stream_release_capacity(stream, elem); + bpf_stream_release_capacity(stream, elem->total_len); bpf_stream_free_elem(elem); } + atomic_sub(read_len - rem_len, &stream->readable); mutex_unlock(&stream->lock); - return ret ? ret : len - rem_len; + return ret ? ret : read_len - rem_len; } -int bpf_prog_stream_read(struct bpf_prog *prog, enum bpf_stream_id stream_id, void __user *buf, int len) +int bpf_prog_stream_read(struct bpf_prog *prog, enum bpf_stream_id stream_id, void __user *buf, u32 len) { struct bpf_stream *stream; stream = bpf_stream_get(stream_id, prog->aux); if (!stream) return -ENOENT; + if (len > INT_MAX) + return -EINVAL; return bpf_stream_read(stream, buf, len); } +static bool bpf_stream_has_data(struct bpf_stream *stream) +{ + return bpf_stream_readable_bytes(stream) > 0; +} + +static void bpf_stream_put(struct bpf_stream *stream) +{ + if (refcount_dec_and_test(&stream->refcnt)) { + struct llist_node *list; + + /* Only a stream that ever queued its work can have a callback in flight. */ + if (READ_ONCE(stream->notify_used)) + irq_work_sync(&stream->notify_work); + list = llist_del_all(&stream->log); + bpf_stream_free_list(list); + bpf_stream_free_list(stream->backlog_head); + mutex_destroy(&stream->lock); + kfree(stream); + } +} + +static int bpf_stream_release(struct inode *inode, struct file *file) +{ + bpf_stream_put(file->private_data); + return 0; +} + +static ssize_t bpf_stream_file_read(struct file *file, char __user *buf, size_t len, + loff_t *ppos) +{ + struct bpf_stream *stream = file->private_data; + bool dead; + int ret; + + if (!len) + return 0; + + for (;;) { + /* + * Sample teardown state before looking for data. Nothing is + * published once the program is gone, so finding the stream + * empty after observing dead means EOF. The opposite order could + * report EOF while data published just before teardown is still + * buffered. + */ + dead = smp_load_acquire(&stream->dead); + ret = bpf_stream_read(stream, buf, len); + if (ret) + return ret; + if (dead) + return 0; + if (file->f_flags & O_NONBLOCK) + return -EAGAIN; + + ret = wait_event_interruptible(stream->waitq, + bpf_stream_has_data(stream) || + READ_ONCE(stream->dead)); + if (ret) + return ret; + } +} + +static __poll_t bpf_stream_poll(struct file *file, struct poll_table_struct *pts) +{ + struct bpf_stream *stream = file->private_data; + __poll_t events = 0; + + /* + * poll_wait() only registers the wait queue callback. Register before + * checking persistent state so a concurrent publication or teardown is + * observed either by the callback or by the checks below. + */ + poll_wait(file, &stream->waitq, pts); + if (bpf_stream_has_data(stream)) + events |= EPOLLIN | EPOLLRDNORM; + if (READ_ONCE(stream->dead)) + events |= EPOLLHUP; + return events; +} + +static const struct file_operations bpf_stream_fops = { + .release = bpf_stream_release, + .read = bpf_stream_file_read, + .poll = bpf_stream_poll, +}; + +int bpf_prog_stream_new_fd(struct bpf_prog *prog, enum bpf_stream_id stream_id, u32 flags) +{ + struct bpf_stream *stream; + int fd_flags = O_RDONLY | O_CLOEXEC; + int fd; + + stream = bpf_stream_get(stream_id, prog->aux); + if (!stream) + return -ENOENT; + if (flags & BPF_F_STREAM_NONBLOCK) + fd_flags |= O_NONBLOCK; + + refcount_inc(&stream->refcnt); + fd = anon_inode_getfd("bpf-stream", &bpf_stream_fops, stream, fd_flags); + if (fd < 0) + bpf_stream_put(stream); + return fd; +} + __bpf_kfunc_start_defs(); /* @@ -238,6 +411,11 @@ __bpf_kfunc int bpf_stream_vprintk(int stream_id, const char *fmt__str, const vo return ret; ret = bstr_printf(data.buf, MAX_BPRINTF_BUF, fmt__str, data.bin_args); + /* Truncation: reject before capacity charge (not -ENOMEM). */ + if (ret >= MAX_BPRINTF_BUF) { + bpf_bprintf_cleanup(&data); + return -E2BIG; + } /* Exclude NULL byte during push. */ ret = bpf_stream_push_str(stream, data.buf, ret); bpf_bprintf_cleanup(&data); @@ -268,28 +446,47 @@ __bpf_kfunc_end_defs(); /* Added kfunc to common_btf_ids */ -void bpf_prog_stream_init(struct bpf_prog *prog) +int bpf_prog_stream_init(struct bpf_prog *prog, gfp_t gfp_extra_flags) { int i; for (i = 0; i < ARRAY_SIZE(prog->aux->stream); i++) { - atomic_set(&prog->aux->stream[i].capacity, 0); - init_llist_head(&prog->aux->stream[i].log); - mutex_init(&prog->aux->stream[i].lock); - prog->aux->stream[i].backlog_head = NULL; - prog->aux->stream[i].backlog_tail = NULL; + struct bpf_stream *stream; + + /* On failure, bpf_prog_stream_free() releases the streams allocated so far. */ + stream = kzalloc_obj(*stream, + bpf_memcg_flags(GFP_KERNEL | gfp_extra_flags)); + if (!stream) + return -ENOMEM; + + refcount_set(&stream->refcnt, 1); + init_llist_head(&stream->log); + mutex_init(&stream->lock); + init_waitqueue_head(&stream->waitq); + init_irq_work(&stream->notify_work, bpf_stream_notify); + prog->aux->stream[i] = stream; } + return 0; } void bpf_prog_stream_free(struct bpf_prog *prog) { - struct llist_node *list; int i; for (i = 0; i < ARRAY_SIZE(prog->aux->stream); i++) { - list = llist_del_all(&prog->aux->stream[i].log); - bpf_stream_free_list(list); - bpf_stream_free_list(prog->aux->stream[i].backlog_head); + struct bpf_stream *stream = prog->aux->stream[i]; + + if (!stream) + continue; + /* + * Pairs with smp_load_acquire() in bpf_stream_file_read(): every + * publication precedes the dead flag, so a reader that observes + * it also observes all buffered data. + */ + smp_store_release(&stream->dead, true); + wake_up_interruptible_poll(&stream->waitq, EPOLLHUP); + bpf_stream_put(stream); + prog->aux->stream[i] = NULL; } } @@ -311,17 +508,18 @@ int bpf_stream_stage_printk(struct bpf_stream_stage *ss, const char *fmt, ...) { struct bpf_bprintf_buffers *buf; va_list args; - int ret; + int len, ret; if (bpf_try_get_buffers(&buf)) return -EBUSY; va_start(args, fmt); - ret = vsnprintf(buf->buf, ARRAY_SIZE(buf->buf), fmt, args); + len = vscnprintf(buf->buf, ARRAY_SIZE(buf->buf), fmt, args); va_end(args); - ss->len += ret; - /* Exclude NULL byte during push. */ - ret = __bpf_stream_push_str(&ss->log, buf->buf, ret); + /* Exclude NULL byte during push; skip empty output entirely. */ + ret = len ? __bpf_stream_push_str(&ss->log, buf->buf, len) : 0; + if (!ret) + ss->len += len; bpf_put_buffers(); return ret; } @@ -351,6 +549,7 @@ int bpf_stream_stage_commit(struct bpf_stream_stage *ss, struct bpf_prog *prog, list = tail; } llist_add_batch(head, tail, &stream->log); + bpf_stream_publish(stream, ss->len); return 0; } diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 96217b99399d..90bc7230afa8 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -602,38 +602,45 @@ static bool can_alloc_pages(void) !IS_ENABLED(CONFIG_PREEMPT_RT); } -static struct page *__bpf_alloc_page(int nid) +struct page *bpf_alloc_page(int nid, bool sleepable) { - if (!can_alloc_pages()) + if (!sleepable || !can_alloc_pages()) return alloc_pages_nolock(__GFP_ACCOUNT, nid, 0); return alloc_pages_node(nid, GFP_KERNEL | __GFP_ZERO | __GFP_ACCOUNT - | __GFP_NOWARN, + | __GFP_NOWARN | __GFP_RETRY_MAYFAIL, 0); } -int bpf_map_alloc_pages(const struct bpf_map *map, int nid, - unsigned long nr_pages, struct page **pages) +void bpf_free_pages(struct llist_head *pages) { - unsigned long i, j; + struct llist_node *node; + struct page *page, *tmp; + + node = llist_del_all(pages); + llist_for_each_entry_safe(page, tmp, node, pcp_llist) + free_pages_nolock(page, 0); +} + +int bpf_alloc_pages(int nid, unsigned long nr_pages, + struct llist_head *pages, bool sleepable) +{ + unsigned long i; struct page *pg; - int ret = 0; for (i = 0; i < nr_pages; i++) { - pg = __bpf_alloc_page(nid); - - if (pg) { - pages[i] = pg; - continue; - } - for (j = 0; j < i; j++) - free_pages_nolock(pages[j], 0); - ret = -ENOMEM; - break; + pg = bpf_alloc_page(nid, sleepable); + if (!pg) + goto free_pages; + llist_add(&pg->pcp_llist, pages); } - return ret; + return 0; + +free_pages: + bpf_free_pages(pages); + return -ENOMEM; } static int btf_field_cmp(const void *a, const void *b) @@ -687,6 +694,7 @@ void btf_record_free(struct btf_record *rec) case BPF_REFCOUNT: case BPF_WORKQUEUE: case BPF_TASK_WORK: + case BPF_RCU_HEAD: /* Nothing to release */ break; default: @@ -741,6 +749,7 @@ struct btf_record *btf_record_dup(const struct btf_record *rec) case BPF_REFCOUNT: case BPF_WORKQUEUE: case BPF_TASK_WORK: + case BPF_RCU_HEAD: /* Nothing to acquire */ break; default: @@ -874,6 +883,7 @@ void bpf_obj_free_fields(const struct btf_record *rec, void *obj) case BPF_LIST_NODE: case BPF_RB_NODE: case BPF_REFCOUNT: + case BPF_RCU_HEAD: break; default: WARN_ON_ONCE(1); @@ -954,6 +964,9 @@ void bpf_map_put(struct bpf_map *map) /* bpf_map_free_id() must be called first */ bpf_map_free_id(map); + if (map->ops->map_free_pre_rcu) + map->ops->map_free_pre_rcu(map); + WARN_ON_ONCE(atomic64_read(&map->sleepable_refcnt)); /* RCU tasks trace grace period implies RCU grace period. */ if (READ_ONCE(map->free_after_mult_rcu_gp)) @@ -1277,7 +1290,7 @@ static int map_check_btf(struct bpf_map *map, struct bpf_token *token, map->record = btf_parse_fields(btf, value_type, BPF_SPIN_LOCK | BPF_RES_SPIN_LOCK | BPF_TIMER | BPF_KPTR | BPF_LIST_HEAD | BPF_RB_ROOT | BPF_REFCOUNT | BPF_WORKQUEUE | BPF_UPTR | - BPF_TASK_WORK, + BPF_TASK_WORK | BPF_RCU_HEAD, map->value_size); if (!IS_ERR_OR_NULL(map->record)) { int i; @@ -1319,6 +1332,12 @@ static int map_check_btf(struct bpf_map *map, struct bpf_token *token, goto free_map_tab; } break; + case BPF_RCU_HEAD: + if (map->map_type != BPF_MAP_TYPE_ARRAY) { + ret = -EOPNOTSUPP; + goto free_map_tab; + } + break; case BPF_KPTR_UNREF: case BPF_KPTR_REF: case BPF_KPTR_PERCPU: @@ -2441,6 +2460,9 @@ static void __bpf_prog_put_rcu(struct rcu_head *rcu) { struct bpf_prog_aux *aux = container_of(rcu, struct bpf_prog_aux, rcu); + btf_put(aux->btf); + kvfree(aux->jited_linfo); + kvfree(aux->linfo); kvfree(aux->func_info); kfree(aux->func_info_aux); free_uid(aux->user); @@ -2457,7 +2479,7 @@ static void __bpf_prog_put_rcu_tasks(struct rcu_head *rcu) { struct bpf_prog *prog = container_of(rcu, struct bpf_prog_aux, rcu)->prog; - if (prog->sleepable) + if (bpf_prog_runs_sleepable(prog)) call_rcu_tasks_trace(rcu, __bpf_prog_put_rcu); else call_rcu(rcu, __bpf_prog_put_rcu); @@ -2465,11 +2487,7 @@ static void __bpf_prog_put_rcu_tasks(struct rcu_head *rcu) static void __bpf_prog_put_noref(struct bpf_prog *prog, bool deferred) { - bpf_prog_kallsyms_del_all(prog); - btf_put(prog->aux->btf); module_put(prog->aux->mod); - kvfree(prog->aux->jited_linfo); - kvfree(prog->aux->linfo); kfree(prog->aux->kfunc_tab); kfree(prog->aux->ctx_arg_info); if (prog->aux->attach_btf) @@ -2478,7 +2496,7 @@ static void __bpf_prog_put_noref(struct bpf_prog *prog, bool deferred) if (deferred) { if (IS_ENABLED(CONFIG_TASKS_RCU) && prog->aux->tramp_linked) call_rcu_tasks(&prog->aux->rcu, __bpf_prog_put_rcu_tasks); - else if (prog->sleepable) + else if (bpf_prog_runs_sleepable(prog)) call_rcu_tasks_trace(&prog->aux->rcu, __bpf_prog_put_rcu); else call_rcu(&prog->aux->rcu, __bpf_prog_put_rcu); @@ -2953,9 +2971,13 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, struct bpf_log_at BPF_F_XDP_HAS_FRAGS | BPF_F_XDP_DEV_BOUND_ONLY | BPF_F_TEST_REG_INVARIANTS | + BPF_F_ARENA_SCALAR | BPF_F_TOKEN_FD)) return -EINVAL; + if ((attr->prog_flags & BPF_F_ARENA_SCALAR) && !bpf_jit_supports_arena_scalar()) + return -EOPNOTSUPP; + bpf_prog_load_fixup_attach_type(attr); if (attr->prog_flags & BPF_F_TOKEN_FD) { @@ -3085,6 +3107,10 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, struct bpf_log_at prog->aux->user = get_current_user(); prog->len = attr->insn_cnt; + err = bpf_prog_stream_init(prog, GFP_USER); + if (err) + goto free_prog; + err = -EFAULT; if (copy_from_bpfptr(prog->insns, make_bpfptr(attr->insns, uattr.is_kernel), @@ -3824,6 +3850,19 @@ static int bpf_tracing_prog_attach(struct bpf_prog *prog, err = -EINVAL; goto out_unlock; } + /* + * The size of these contexts depends on what the program is attached + * to, or run with. That is compared with max_ctx_offset of the program, + * and nobody looks at the one of its extensions. + */ + if (prog->type == BPF_PROG_TYPE_EXT && + (tgt_prog->type == BPF_PROG_TYPE_RAW_TRACEPOINT || + tgt_prog->type == BPF_PROG_TYPE_TRACEPOINT || + tgt_prog->type == BPF_PROG_TYPE_SYSCALL) && + prog->aux->max_ctx_offset > tgt_prog->aux->max_ctx_offset) { + err = -EINVAL; + goto out_unlock; + } err = bpf_link_prime(&link->link.link, &link_primer); if (err) @@ -3993,19 +4032,19 @@ static int bpf_perf_link_fill_common(const struct perf_event *event, *ulenp = 1; } if (!uname) - return 0; + goto out; if (buf) { err = bpf_copy_to_user(uname, buf, ulen, len); - if (err) - return err; } else { char zero = '\0'; if (put_user(zero, uname)) - return -EFAULT; + err = -EFAULT; } - return 0; +out: + kfree_const(buf); + return err; } #ifdef CONFIG_KPROBE_EVENTS @@ -4062,6 +4101,7 @@ static void bpf_perf_link_fdinfo_kprobe(const struct perf_event *event, name, offset, missed, addr, type == BPF_FD_TYPE_KRETPROBE ? "kretprobe" : "kprobe", event->bpf_cookie); + kfree_const(name); } #endif @@ -4115,6 +4155,7 @@ static void bpf_perf_link_fdinfo_uprobe(const struct perf_event *event, name, offset, ref_ctr_offset, type == BPF_FD_TYPE_URETPROBE ? "uretprobe" : "uprobe", event->bpf_cookie); + kfree_const(name); } #endif @@ -4213,6 +4254,7 @@ static void bpf_tracepoint_link_show_fdinfo(const struct perf_event *event, "event_type:\t%s\n" "cookie:\t%llu\n", name, "tracepoint", event->bpf_cookie); + kfree_const(name); } static void bpf_probe_link_show_fdinfo(const struct perf_event *event, @@ -4772,6 +4814,7 @@ static int bpf_prog_query(const union bpf_attr *attr, case BPF_CGROUP_GETSOCKOPT: case BPF_CGROUP_SETSOCKOPT: case BPF_LSM_CGROUP: + case BPF_STRUCT_OPS: return cgroup_bpf_prog_query(attr, uattr, uattr_size); case BPF_LIRC_MODE2: return lirc_prog_query(attr, uattr); @@ -5044,6 +5087,12 @@ static struct bpf_insn *bpf_insn_prepare_dump(const struct bpf_prog *prog, insns[i].imm = BPF_FUNC_tail_call; /* fall-through */ } + /* + * BPF_JCOND is the same opcode as BPF_CALL_ARGS. The latter is for + * the interpreter, and a prog with unwind_goto is always JITed. + */ + if (prog->jited && bpf_is_unwind_goto(&insns[i])) + continue; if (code == (BPF_JMP | BPF_CALL) || code == (BPF_JMP | BPF_CALL_ARGS)) { /* Restore the legacy xlated dump layout. */ @@ -5420,7 +5469,11 @@ static int bpf_map_get_info_by_fd(struct file *file, info_len = min_t(u32, sizeof(info), info_len); memset(&info, 0, sizeof(info)); - if (copy_from_user(&info, uinfo, info_len)) + if (info_len >= offsetofend(struct bpf_map_info, hash) && + get_user(info.hash, &uinfo->hash)) + return -EFAULT; + if (info_len >= offsetofend(struct bpf_map_info, hash_size) && + get_user(info.hash_size, &uinfo->hash_size)) return -EFAULT; info.type = map->map_type; @@ -5735,11 +5788,13 @@ static int bpf_task_fd_query(const union bpf_attr *attr, err = bpf_get_perf_event_info(event, &prog_id, &fd_type, &buf, &probe_offset, &probe_addr, NULL); - if (!err) + if (!err) { err = bpf_task_fd_query_copy(attr, uattr, prog_id, fd_type, buf, probe_offset, probe_addr); + kfree_const(buf); + } goto put_file; } @@ -6220,7 +6275,7 @@ static int bpf_prog_bind_map(union bpf_attr *attr) /* The bpf program will not access the bpf map, but for the sake of * simplicity, increase sleepable_refcnt for sleepable program as well. */ - if (prog->sleepable) + if (bpf_prog_runs_sleepable(prog)) atomic64_inc(&map->sleepable_refcnt); memcpy(used_maps_new, used_maps_old, sizeof(used_maps_old[0]) * prog->aux->used_map_cnt); @@ -6320,6 +6375,28 @@ static int prog_assoc_struct_ops(union bpf_attr *attr) return ret; } +#define BPF_PROG_STREAM_OPEN_LAST_FIELD prog_stream_open.flags + +static int prog_stream_open(union bpf_attr *attr) +{ + struct bpf_prog *prog; + u32 flags = attr->prog_stream_open.flags; + int ret; + + if (CHECK_ATTR(BPF_PROG_STREAM_OPEN)) + return -EINVAL; + if (flags & ~BPF_F_STREAM_NONBLOCK) + return -EINVAL; + + prog = bpf_prog_get(attr->prog_stream_open.prog_fd); + if (IS_ERR(prog)) + return PTR_ERR(prog); + + ret = bpf_prog_stream_new_fd(prog, attr->prog_stream_open.stream_id, flags); + bpf_prog_put(prog); + return ret; +} + static int __sys_bpf(enum bpf_cmd cmd, bpfptr_t uattr, unsigned int size, bpfptr_t uattr_common, unsigned int size_common) { @@ -6492,6 +6569,9 @@ static int __sys_bpf(enum bpf_cmd cmd, bpfptr_t uattr, unsigned int size, case BPF_PROG_ASSOC_STRUCT_OPS: err = prog_assoc_struct_ops(&attr); break; + case BPF_PROG_STREAM_OPEN: + err = prog_stream_open(&attr); + break; default: err = -EINVAL; break; diff --git a/kernel/bpf/sysfs_btf.c b/kernel/bpf/sysfs_btf.c index 9cbe15ce3540..556fe92c3649 100644 --- a/kernel/bpf/sysfs_btf.c +++ b/kernel/bpf/sysfs_btf.c @@ -6,8 +6,10 @@ #include #include #include +#include #include #include +#include #include #include @@ -15,16 +17,12 @@ extern char __start_BTF[]; extern char __stop_BTF[]; -static int btf_sysfs_vmlinux_mmap(struct file *filp, struct kobject *kobj, - const struct bin_attribute *attr, - struct vm_area_struct *vma) +static int btf_sysfs_mmap_check(void *data, size_t size, struct vm_area_struct *vma) { - unsigned long pages = PAGE_ALIGN(attr->size) >> PAGE_SHIFT; + unsigned long pages = PAGE_ALIGN(size) >> PAGE_SHIFT; size_t vm_size = vma->vm_end - vma->vm_start; - phys_addr_t addr = __pa_symbol(__start_BTF); - unsigned long pfn = addr >> PAGE_SHIFT; - if (attr->private != __start_BTF || !PAGE_ALIGNED(addr)) + if (!data || !PAGE_ALIGNED((unsigned long)data)) return -EINVAL; if (vma->vm_pgoff) @@ -33,24 +31,153 @@ static int btf_sysfs_vmlinux_mmap(struct file *filp, struct kobject *kobj, if (vma->vm_flags & (VM_WRITE | VM_EXEC | VM_MAYSHARE)) return -EACCES; - if (pfn + pages < pfn) - return -EINVAL; - if ((vm_size >> PAGE_SHIFT) > pages) return -EINVAL; vm_flags_mod(vma, VM_DONTDUMP, VM_MAYEXEC | VM_MAYWRITE); + return 0; +} + +static int btf_sysfs_mmap_image(struct file *filp, struct kobject *kobj, + const struct bin_attribute *attr, + struct vm_area_struct *vma) +{ + void *data = READ_ONCE(attr->private); + phys_addr_t addr; + unsigned long pfn; + size_t vm_size = vma->vm_end - vma->vm_start; + int err; + + err = btf_sysfs_mmap_check(data, attr->size, vma); + if (err) + return err; + + /* Kernel image addresses can lie inside the vmalloc range on arm64. */ + addr = __pa_symbol(data); + pfn = addr >> PAGE_SHIFT; + if (pfn + (PAGE_ALIGN(attr->size) >> PAGE_SHIFT) < pfn) + return -EINVAL; + return remap_pfn_range(vma, vma->vm_start, pfn, vm_size, vma->vm_page_prot); } static struct bin_attribute bin_attr_btf_vmlinux __ro_after_init = { .attr = { .name = "vmlinux", .mode = 0444, }, .read = sysfs_bin_attr_simple_read, - .mmap = btf_sysfs_vmlinux_mmap, + .mmap = btf_sysfs_mmap_image, }; struct kobject *btf_kobj; +struct btf_sysfs_entry { + struct bin_attribute attr; + char *module_name; +}; + +static void *btf_sysfs_lazy_data(const struct bin_attribute *attr) +{ + struct btf_sysfs_entry *entry = container_of(attr, struct btf_sysfs_entry, attr); + void *data; + + data = smp_load_acquire(&attr->private); + if (!data) { + request_module("%s", entry->module_name); + data = smp_load_acquire(&attr->private); + } + return data; +} + +static ssize_t btf_sysfs_read_lazy(struct file *filp, struct kobject *kobj, + const struct bin_attribute *attr, char *buf, + loff_t off, size_t count) +{ + void *data = btf_sysfs_lazy_data(attr); + + if (!data) + return -ENODEV; + + return memory_read_from_buffer(buf, count, &off, data, attr->size); +} + +static int btf_sysfs_mmap_lazy(struct file *filp, struct kobject *kobj, + const struct bin_attribute *attr, + struct vm_area_struct *vma) +{ + void *data = btf_sysfs_lazy_data(attr); + int err; + + if (!data) + return -ENODEV; + + err = btf_sysfs_mmap_check(data, attr->size, vma); + if (err) + return err; + + return remap_vmalloc_range(vma, data, 0); +} + +struct bin_attribute *sysfs_btf_add(const char *name, void *data, size_t data_size, + bool mmap, const char *lazy_module_name) +{ + struct btf_sysfs_entry *entry; + struct bin_attribute *attr; + int err; + + entry = kzalloc_obj(*entry); + if (!entry) + return ERR_PTR(-ENOMEM); + + attr = &entry->attr; + sysfs_bin_attr_init(attr); + attr->attr.mode = 0444; + attr->size = data_size; + attr->private = data; + attr->read = lazy_module_name ? btf_sysfs_read_lazy : sysfs_bin_attr_simple_read; + if (mmap) + attr->mmap = lazy_module_name ? btf_sysfs_mmap_lazy : btf_sysfs_mmap_image; + attr->attr.name = kstrdup(name, GFP_KERNEL); + if (!attr->attr.name) { + err = -ENOMEM; + goto err_free; + } + if (lazy_module_name) { + entry->module_name = kstrdup(lazy_module_name, GFP_KERNEL); + if (!entry->module_name) { + err = -ENOMEM; + goto err_free; + } + } + + err = sysfs_create_bin_file(btf_kobj, attr); + if (err) { + pr_warn("failed to register [%s] BTF in sysfs: %d\n", name, err); + goto err_free; + } + + return attr; + +err_free: + kfree(entry->module_name); + kfree(attr->attr.name); + kfree(entry); + return ERR_PTR(err); +} + +bool sysfs_btf_update(struct bin_attribute *attr, void *data) +{ + return attr && !cmpxchg(&attr->private, NULL, data); +} + +void sysfs_btf_remove(struct bin_attribute *attr) +{ + struct btf_sysfs_entry *entry = container_of(attr, struct btf_sysfs_entry, attr); + + sysfs_remove_bin_file(btf_kobj, attr); + kfree(entry->module_name); + kfree(attr->attr.name); + kfree(entry); +} + static int __init btf_vmlinux_init(void) { bin_attr_btf_vmlinux.private = __start_BTF; diff --git a/kernel/bpf/task_iter.c b/kernel/bpf/task_iter.c index c65ba1dcd866..2ee7b3b3c10b 100644 --- a/kernel/bpf/task_iter.c +++ b/kernel/bpf/task_iter.c @@ -737,7 +737,11 @@ static const struct bpf_iter_seq_info task_vma_seq_info = { static struct bpf_iter_reg task_vma_reg_info = { .target = "task_vma", .attach_target = bpf_iter_attach_task, - .feature = BPF_ITER_RESCHED, + /* + * ->show() runs with the mmap_lock of the iterated mm held, which a + * sleepable program takes again when it faults on user memory. + */ + .feature = BPF_ITER_RESCHED | BPF_ITER_NOSLEEP, .ctx_arg_info_size = 2, .ctx_arg_info = { { offsetof(struct bpf_iter__task_vma, task), @@ -806,8 +810,14 @@ BPF_CALL_5(bpf_find_vma, struct task_struct *, task, u64, start, vma = find_vma(mm, start); if (vma && vma->vm_start <= start && vma->vm_end > start) { + /* + * The callback must not sleep while mmap_lock is held, and the + * verifier checks it as an RCU read-side critical section. + */ + rcu_read_lock(); callback_fn((u64)(long)task, (u64)(long)vma, (u64)(long)callback_ctx, 0, 0); + rcu_read_unlock(); ret = 0; } bpf_mmap_unlock_mm(work, mm); diff --git a/kernel/bpf/tnum.c b/kernel/bpf/tnum.c index ec9c310cf5d7..1f620facd1b7 100644 --- a/kernel/bpf/tnum.c +++ b/kernel/bpf/tnum.c @@ -317,3 +317,14 @@ u64 tnum_step(struct tnum t, u64 z) inc = (filled + 1) & t.mask; return t.value | inc; } + +/* + * Return the number of trailing bits known to be zero in a tnum. + * Return 64 for a tnum representing zero. + */ +u32 tnum_alignment(struct tnum a) +{ + u64 v = a.value | a.mask; + + return v ? __ffs64(v) : 64; +} diff --git a/kernel/bpf/token.c b/kernel/bpf/token.c index da915a4f972b..aa42243af16b 100644 --- a/kernel/bpf/token.c +++ b/kernel/bpf/token.c @@ -169,8 +169,11 @@ int bpf_token_create(union bpf_attr *attr) FD_PREPARE(fdf, O_CLOEXEC, alloc_file_pseudo(inode, path.mnt, BPF_TOKEN_INODE_NAME, O_RDWR, &bpf_token_fops)); - if (fdf->fd < 0) + if (fdf->fd < 0) { + /* no file took over the inode reference */ + iput(inode); return fdf->fd; + } token = kzalloc_obj(*token, GFP_USER); if (!token) diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c index bf4cb3dd444d..21d460866dfe 100644 --- a/kernel/bpf/trampoline.c +++ b/kernel/bpf/trampoline.c @@ -475,18 +475,11 @@ static int register_fentry(struct bpf_trampoline *tr, struct bpf_tramp_image *im void *data __maybe_unused) { void *new_addr = im->image; - void *ip = tr->func.addr; - unsigned long faddr; int ret; - faddr = ftrace_location((unsigned long)ip); - if (faddr) { + if (tr->func.ftrace_managed) { if (!tr->fops) return -ENOTSUPP; - tr->func.ftrace_managed = true; - } - - if (tr->func.ftrace_managed) { ret = direct_ops_add(tr, new_addr); } else { ret = bpf_trampoline_update_fentry(tr, 0, NULL, new_addr); @@ -859,7 +852,7 @@ static enum bpf_tramp_prog_type bpf_attach_type_to_tramp(struct bpf_prog *prog) } } -static int bpf_freplace_check_tgt_prog(struct bpf_prog *tgt_prog) +static int bpf_freplace_link_tgt_prog(struct bpf_prog *tgt_prog) { struct bpf_prog_aux *aux = tgt_prog->aux; @@ -873,7 +866,7 @@ static int bpf_freplace_check_tgt_prog(struct bpf_prog *tgt_prog) */ return -EBUSY; - aux->is_extended = true; + aux->freplace_link_cnt++; return 0; } @@ -1023,13 +1016,20 @@ static int __bpf_trampoline_link_prog(struct bpf_tramp_node *node, /* Cannot attach extension if fentry/fexit are in use. */ if (cnt) return -EBUSY; - err = bpf_freplace_check_tgt_prog(tgt_prog); + err = bpf_freplace_link_tgt_prog(tgt_prog); if (err) return err; + err = bpf_arch_text_poke(tr->func.addr, BPF_MOD_NOP, + BPF_MOD_JUMP, NULL, + node->link->prog->bpf_func); + if (err) { + /* Undo the claim from bpf_freplace_link_tgt_prog(). */ + guard(mutex)(&tgt_prog->aux->ext_mutex); + tgt_prog->aux->freplace_link_cnt--; + return err; + } tr->extension_prog = node->link->prog; - return bpf_arch_text_poke(tr->func.addr, BPF_MOD_NOP, - BPF_MOD_JUMP, NULL, - node->link->prog->bpf_func); + return 0; } err = bpf_trampoline_add_prog(tr, node, cnt); if (err) @@ -1069,7 +1069,7 @@ static int __bpf_trampoline_unlink_prog(struct bpf_tramp_node *node, tr->extension_prog->bpf_func, NULL); tr->extension_prog = NULL; guard(mutex)(&tgt_prog->aux->ext_mutex); - tgt_prog->aux->is_extended = false; + tgt_prog->aux->freplace_link_cnt--; return err; } bpf_trampoline_remove_prog(tr, node); @@ -1279,6 +1279,7 @@ struct bpf_trampoline *bpf_trampoline_get(u64 key, memcpy(&tr->func.model, &tgt_info->fmodel, sizeof(tgt_info->fmodel)); tr->func.addr = (void *)tgt_info->tgt_addr; + tr->func.ftrace_managed = tr->ip != 0; out: trampoline_unlock(tr); return tr; @@ -1628,13 +1629,20 @@ static void ftrace_hash_add(struct ftrace_hash *hash, struct ftrace_func_entry * static int register_fentry_multi(struct bpf_trampoline *tr, struct bpf_tramp_image *im, void *ptr) { unsigned long addr = (unsigned long) im->image; - unsigned long ip = ftrace_location(tr->ip); struct bpf_tracing_multi_data *data = ptr; + unsigned long ip = tr->ip; + + /* + * Different BTF ids get different trampolines, but they can still + * resolve to the same address (aliases, functions of the same name). + * There's only one call site to attach to. + */ + if (ftrace_lookup_ip(data->reg, ip)) + return -EINVAL; if (bpf_trampoline_use_jmp(tr->flags)) addr = ftrace_jmp_set(addr); - tr->func.ftrace_managed = true; ftrace_hash_add(data->reg, data->entry, ip, addr); tr->cur_image = im; return 0; @@ -1643,13 +1651,12 @@ static int register_fentry_multi(struct bpf_trampoline *tr, struct bpf_tramp_ima static int unregister_fentry_multi(struct bpf_trampoline *tr, u32 orig_flags, void *ptr) { unsigned long addr = (unsigned long) tr->cur_image->image; - unsigned long ip = ftrace_location(tr->ip); struct bpf_tracing_multi_data *data = ptr; if (bpf_trampoline_use_jmp(tr->flags)) addr = ftrace_jmp_set(addr); - ftrace_hash_add(data->unreg, data->entry, ip, addr); + ftrace_hash_add(data->unreg, data->entry, tr->ip, addr); tr->cur_image = NULL; return 0; } @@ -1658,13 +1665,12 @@ static int modify_fentry_multi(struct bpf_trampoline *tr, u32 orig_flags, struct bool lock_direct_mutex, void *ptr) { unsigned long addr = (unsigned long) im->image; - unsigned long ip = ftrace_location(tr->ip); struct bpf_tracing_multi_data *data = ptr; if (bpf_trampoline_use_jmp(tr->flags)) addr = ftrace_jmp_set(addr); - ftrace_hash_add(data->modify, data->entry, ip, addr); + ftrace_hash_add(data->modify, data->entry, tr->ip, addr); tr->cur_image = im; return 0; } diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 5f874979b8d7..5601d2056218 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -5,6 +5,7 @@ */ #include #include +#include #include #include #include @@ -24,11 +25,13 @@ #include #include #include +#include #include #include #include #include #include +#include #include #include #include @@ -216,6 +219,7 @@ static int ref_set_non_owning(struct bpf_verifier_env *env, struct bpf_reg_state *reg); static bool is_trusted_reg(struct bpf_verifier_env *env, const struct bpf_reg_state *reg); static inline bool in_sleepable_context(struct bpf_verifier_env *env); +static bool in_rcu_callback(struct bpf_verifier_env *env); static const char *non_sleepable_context_description(struct bpf_verifier_env *env); static void scalar32_min_max_add(struct bpf_reg_state *dst_reg, struct bpf_reg_state *src_reg); static void scalar_min_max_add(struct bpf_reg_state *dst_reg, struct bpf_reg_state *src_reg); @@ -301,6 +305,12 @@ static int arg_idx_from_argno(argno_t a) return arg_from_argno(a) - 1; } +/* Normalize helper register numbers and kfunc argument numbers to ABI slots. */ +static u32 arg_slot_from_argno(argno_t a) +{ + return abs(a.argno) - 1; +} + static const char *btf_type_name(const struct btf *btf, u32 id) { return btf_name_by_offset(btf, btf_type_by_id(btf, id)->name_off); @@ -394,27 +404,69 @@ bool bpf_subprog_is_global(const struct bpf_verifier_env *env, int subprog) return aux && aux[subprog].linkage == BTF_FUNC_GLOBAL; } -static bool subprog_returns_void(struct bpf_verifier_env *env, int subprog) +static const struct btf_type *subprog_ret_type(struct bpf_verifier_env *env, int subprog) { - const struct btf_type *type, *func, *func_proto; + const struct btf_type *func, *func_proto; const struct btf *btf = env->prog->aux->btf; u32 btf_id; + if (!btf || !env->prog->aux->func_info) + return NULL; + btf_id = env->prog->aux->func_info[subprog].type_id; + /* Both already validated by prepare_btf_func() at prog load. */ func = btf_type_by_id(btf, btf_id); - if (verifier_bug_if(!func, env, "btf_id %u not found", btf_id)) - return false; - func_proto = btf_type_by_id(btf, func->type); - if (!func_proto) - return false; - type = btf_type_skip_modifiers(btf, func_proto->type, NULL); - if (!type) - return false; + return btf_type_skip_modifiers(btf, func_proto->type, NULL); +} - return btf_type_is_void(type); +static bool subprog_returns_void(struct bpf_verifier_env *env, int subprog) +{ + const struct btf_type *type = subprog_ret_type(env, subprog); + + return type && btf_type_is_void(type); +} + +static u32 ret_regs_cnt(u32 size) +{ + return size > 8 && size <= 16 ? 2 : 1; +} + +/* Registers holding a function return value, in order. See ret_regs_cnt(). */ +static const int ret_regs[] = { BPF_REG_0, BPF_REG_2 }; + +static int bpf_compute_subprog_ret_regs(struct bpf_verifier_env *env) +{ + const struct btf *btf = env->prog->aux->btf; + const struct btf_type *type; + int subprog; + u32 size; + + if (!env->prog->jit_requested || bpf_prog_is_offloaded(env->prog->aux)) + return 0; + + /* + * Skip the main program: its return value is the program's exit code, + * read out of R0, so it never uses the register pair. An extension does + * have a real prototype for subprog 0, but btf_check_func_type_match() + * refuses to replace a function returning more than 8 bytes. + */ + for (subprog = 1; subprog < env->subprog_cnt; subprog++) { + type = subprog_ret_type(env, subprog); + if (!type || btf_type_is_void(type)) + continue; + if (verifier_bug_if(IS_ERR(btf_resolve_size(btf, type, &size)), env, + "cannot size return type of subprog %d", subprog)) + return -EFAULT; + if (ret_regs_cnt(size) > 1) { + subprog_info(env, subprog)->ret_reg_pair = true; + env->prog->jit_required = 1; + } + } + + return 0; } const char *bpf_subprog_name(const struct bpf_verifier_env *env, int subprog) @@ -486,6 +538,7 @@ static bool is_ptr_cast_function(enum bpf_func_id func_id) static bool is_sync_callback_calling_kfunc(u32 btf_id); static bool is_async_callback_calling_kfunc(u32 btf_id); +static bool is_call_rcu_kfunc(u32 btf_id); static bool is_callback_calling_kfunc(u32 btf_id); static bool is_bpf_wq_set_callback_kfunc(u32 btf_id); @@ -528,6 +581,10 @@ static bool is_async_cb_sleepable(struct bpf_verifier_env *env, struct bpf_insn if (bpf_helper_call(insn) && insn->imm == BPF_FUNC_timer_set_callback) return false; + /* bpf_call_rcu and bpf_call_rcu_tasks_trace callbacks are never sleepable. */ + if (bpf_pseudo_kfunc_call(insn) && insn->off == 0 && is_call_rcu_kfunc(insn->imm)) + return false; + /* bpf_wq and bpf_task_work callbacks are always sleepable. */ if (bpf_pseudo_kfunc_call(insn) && insn->off == 0 && (is_bpf_wq_set_callback_kfunc(insn->imm) || is_task_work_add_kfunc(insn->imm))) @@ -537,6 +594,19 @@ static bool is_async_cb_sleepable(struct bpf_verifier_env *env, struct bpf_insn return false; } +/* From here on the maps of the program count it as a sleepable user. */ +static void mark_sleepable_async_cb(struct bpf_verifier_env *env) +{ + int i; + + if (bpf_prog_runs_sleepable(env->prog)) + return; + + env->prog->aux->sleepable_async_cb = true; + for (i = 0; i < env->used_map_cnt; i++) + atomic64_inc(&env->used_maps[i]->sleepable_refcnt); +} + bool bpf_is_may_goto_insn(struct bpf_insn *insn) { return insn->code == (BPF_JMP | BPF_JCOND) && insn->src_reg == BPF_MAY_GOTO; @@ -544,16 +614,17 @@ bool bpf_is_may_goto_insn(struct bpf_insn *insn) static bool is_spi_bounds_valid(struct bpf_func_state *state, int spi, int nr_slots) { - int allocated_slots = state->allocated_stack / BPF_REG_SIZE; + int allocated_slots = bpf_stack_nr_slots(state); - /* We need to check that slots between [spi - nr_slots + 1, spi] are - * within [0, allocated_stack). - * - * Please note that the spi grows downwards. For example, a dynptr - * takes the size of two stack slots; the first slot will be at - * spi and the second slot will be at spi - 1. - */ - return spi - nr_slots + 1 >= 0 && spi < allocated_slots; + /* + * We need to check that slots between [spi - nr_slots + 1, spi] are + * within [0, allocated_stack). + * + * Please note that the spi grows downwards. For example, a dynptr + * takes the size of two stack slots; the first slot will be at + * spi and the second slot will be at spi - 1. + */ + return spi - nr_slots + 1 >= 0 && spi < allocated_slots; } static int stack_slot_obj_get_spi(struct bpf_verifier_env *env, struct bpf_reg_state *reg, @@ -668,6 +739,21 @@ static void mark_dynptr_cb_reg(struct bpf_verifier_env *env, static int destroy_if_dynptr_stack_slot(struct bpf_verifier_env *env, struct bpf_func_state *state, int spi); +static int destroy_dynptrs_in_stack_slots(struct bpf_verifier_env *env, + struct bpf_func_state *state, + int first_spi, int last_spi) +{ + int spi, err; + + for (spi = first_spi; spi <= last_spi; spi++) { + err = destroy_if_dynptr_stack_slot(env, state, spi); + if (err) + return err; + } + + return 0; +} + static int mark_stack_slots_dynptr(struct bpf_verifier_env *env, struct bpf_reg_state *reg, enum bpf_arg_type arg_type, int insn_idx, struct ref_obj_desc *ref_obj, struct bpf_dynptr_desc *dynptr) @@ -681,7 +767,7 @@ static int mark_stack_slots_dynptr(struct bpf_verifier_env *env, struct bpf_reg_ return spi; /* We cannot assume both spi and spi - 1 belong to the same dynptr, - * hence we need to call destroy_if_dynptr_stack_slot twice for both, + * hence we need to destroy dynptrs in both slots, * to ensure that for the following example: * [d1][d1][d2][d2] * spi 3 2 1 0 @@ -689,16 +775,13 @@ static int mark_stack_slots_dynptr(struct bpf_verifier_env *env, struct bpf_reg_ * case they do belong to same dynptr, second call won't see slot_type * as STACK_DYNPTR and will simply skip destruction. */ - err = destroy_if_dynptr_stack_slot(env, state, spi); - if (err) - return err; - err = destroy_if_dynptr_stack_slot(env, state, spi - 1); + err = destroy_dynptrs_in_stack_slots(env, state, spi - 1, spi); if (err) return err; for (i = 0; i < BPF_REG_SIZE; i++) { - state->stack[spi].slot_type[i] = STACK_DYNPTR; - state->stack[spi - 1].slot_type[i] = STACK_DYNPTR; + bpf_stack_slot(state, spi)->slot_type[i] = STACK_DYNPTR; + bpf_stack_slot(state, spi - 1)->slot_type[i] = STACK_DYNPTR; } type = arg_to_dynptr_type(arg_type); @@ -731,8 +814,8 @@ static int mark_stack_slots_dynptr(struct bpf_verifier_env *env, struct bpf_reg_ parent_id = dynptr->parent_id; } - mark_dynptr_stack_regs(env, &state->stack[spi].spilled_ptr, - &state->stack[spi - 1].spilled_ptr, type, parent_id); + mark_dynptr_stack_regs(env, &bpf_stack_slot(state, spi)->spilled_ptr, + &bpf_stack_slot(state, spi - 1)->spilled_ptr, type, parent_id); return 0; } @@ -764,7 +847,7 @@ static int unmark_stack_slots_dynptr(struct bpf_verifier_env *env, struct bpf_re * all clones and derived slices. For non-referenced dynptr, only * the dynptr and slices derived from it will be invalidated. */ - reg = &state->stack[spi].spilled_ptr; + reg = &bpf_stack_slot(state, spi)->spilled_ptr; return release_reference(env, dynptr_type_referenced(reg->dynptr.type) ? reg->parent_id : reg->id); @@ -803,6 +886,7 @@ static int dynptr_ref_cnt(struct bpf_verifier_env *env, int v_parent_id) static int destroy_if_dynptr_stack_slot(struct bpf_verifier_env *env, struct bpf_func_state *state, int spi) { + struct bpf_stack_state *slot = bpf_stack_slot(state, spi); int err = 0; /* We always ensure that STACK_DYNPTR is never set partially, @@ -810,20 +894,22 @@ static int destroy_if_dynptr_stack_slot(struct bpf_verifier_env *env, * different for STACK_SPILL, where it may be only set for * 1 byte, so code has to use is_spilled_reg. */ - if (state->stack[spi].slot_type[0] != STACK_DYNPTR) + if (slot->slot_type[0] != STACK_DYNPTR) return 0; /* Reposition spi to first slot */ - if (!state->stack[spi].spilled_ptr.dynptr.first_slot) + if (!slot->spilled_ptr.dynptr.first_slot) { spi = spi + 1; + slot = bpf_stack_slot(state, spi); + } /* * A referenced dynptr can be overwritten only if there is at * least one other dynptr sharing the same virtual ref parent, * ensuring the reference can still be properly released. */ - if (dynptr_type_referenced(state->stack[spi].spilled_ptr.dynptr.type) && - dynptr_ref_cnt(env, state->stack[spi].spilled_ptr.parent_id) <= 1) { + if (dynptr_type_referenced(slot->spilled_ptr.dynptr.type) && + dynptr_ref_cnt(env, slot->spilled_ptr.parent_id) <= 1) { verbose(env, "cannot overwrite referenced dynptr\n"); bpf_diag_res( env, env->insn_idx, "referenced dynptr overwrite", @@ -833,7 +919,7 @@ static int destroy_if_dynptr_stack_slot(struct bpf_verifier_env *env, } /* Invalidate the dynptr and any derived slices */ - err = release_reference(env, state->stack[spi].spilled_ptr.id); + err = release_reference(env, slot->spilled_ptr.id); if (!err) { mark_stack_slot_scratched(env, spi); mark_stack_slot_scratched(env, spi - 1); @@ -873,6 +959,7 @@ static bool is_dynptr_reg_valid_uninit(struct bpf_verifier_env *env, struct bpf_ static bool is_dynptr_reg_valid_init(struct bpf_verifier_env *env, struct bpf_reg_state *reg) { struct bpf_func_state *state = bpf_func(env, reg); + struct bpf_stack_state *slot; int i, spi; /* This already represents first slot of initialized bpf_dynptr. @@ -887,12 +974,13 @@ static bool is_dynptr_reg_valid_init(struct bpf_verifier_env *env, struct bpf_re spi = dynptr_get_spi(env, reg); if (spi < 0) return false; - if (!state->stack[spi].spilled_ptr.dynptr.first_slot) + slot = bpf_stack_slot(state, spi); + if (!slot->spilled_ptr.dynptr.first_slot) return false; for (i = 0; i < BPF_REG_SIZE; i++) { - if (state->stack[spi].slot_type[i] != STACK_DYNPTR || - state->stack[spi - 1].slot_type[i] != STACK_DYNPTR) + if (slot->slot_type[i] != STACK_DYNPTR || + bpf_stack_slot(state, spi - 1)->slot_type[i] != STACK_DYNPTR) return false; } @@ -911,14 +999,23 @@ static enum bpf_dynptr_type dynptr_reg_type(struct bpf_verifier_env *env, struct if (spi < 0) return BPF_DYNPTR_TYPE_INVALID; state = bpf_func(env, reg); - return state->stack[spi].spilled_ptr.dynptr.type; + return bpf_stack_slot(state, spi)->spilled_ptr.dynptr.type; +} + +static bool dynptr_write_changes_pkt_data(enum bpf_dynptr_type type) +{ + return type == BPF_DYNPTR_TYPE_SKB || type == BPF_DYNPTR_TYPE_SKB_META; } static bool is_dynptr_type_expected(struct bpf_verifier_env *env, struct bpf_reg_state *reg, enum bpf_arg_type arg_type) { - /* ARG_PTR_TO_DYNPTR takes any type of dynptr */ - if (arg_type == ARG_PTR_TO_DYNPTR) + /* + * ARG_PTR_TO_DYNPTR without a type flag takes any type of dynptr. + * Test the flags rather than the whole arg_type, which may carry + * unrelated ones such as PTR_MAYBE_NULL. + */ + if (!(arg_type & DYNPTR_TYPE_FLAG_MASK)) return true; return dynptr_reg_type(env, reg) == arg_to_dynptr_type(arg_type); @@ -947,7 +1044,7 @@ static int mark_stack_slots_iter(struct bpf_verifier_env *env, return id; for (i = 0; i < nr_slots; i++) { - struct bpf_stack_state *slot = &state->stack[spi - i]; + struct bpf_stack_state *slot = bpf_stack_slot(state, spi - i); struct bpf_reg_state *st = &slot->spilled_ptr; __mark_reg_known_zero(st); @@ -984,7 +1081,7 @@ static int unmark_stack_slots_iter(struct bpf_verifier_env *env, return spi; for (i = 0; i < nr_slots; i++) { - struct bpf_stack_state *slot = &state->stack[spi - i]; + struct bpf_stack_state *slot = bpf_stack_slot(state, spi - i); struct bpf_reg_state *st = &slot->spilled_ptr; if (i == 0) @@ -1018,7 +1115,7 @@ static bool is_iter_reg_valid_uninit(struct bpf_verifier_env *env, return false; for (i = 0; i < nr_slots; i++) { - struct bpf_stack_state *slot = &state->stack[spi - i]; + struct bpf_stack_state *slot = bpf_stack_slot(state, spi - i); for (j = 0; j < BPF_REG_SIZE; j++) if (slot->slot_type[j] == STACK_ITER) @@ -1039,7 +1136,7 @@ static int is_iter_reg_valid_init(struct bpf_verifier_env *env, struct bpf_reg_s return -EINVAL; for (i = 0; i < nr_slots; i++) { - struct bpf_stack_state *slot = &state->stack[spi - i]; + struct bpf_stack_state *slot = bpf_stack_slot(state, spi - i); struct bpf_reg_state *st = &slot->spilled_ptr; if (st->type & PTR_UNTRUSTED) @@ -1081,7 +1178,7 @@ static int mark_stack_slot_irq_flag(struct bpf_verifier_env *env, if (id < 0) return id; - slot = &state->stack[spi]; + slot = bpf_stack_slot(state, spi); st = &slot->spilled_ptr; __mark_reg_known_zero(st); @@ -1108,7 +1205,7 @@ static int unmark_stack_slot_irq_flag(struct bpf_verifier_env *env, struct bpf_r if (spi < 0) return spi; - slot = &state->stack[spi]; + slot = bpf_stack_slot(state, spi); st = &slot->spilled_ptr; if (st->irq.kfunc_class != kfunc_class) { @@ -1177,7 +1274,7 @@ static bool is_irq_flag_reg_valid_uninit(struct bpf_verifier_env *env, struct bp if (spi < 0) return false; - slot = &state->stack[spi]; + slot = bpf_stack_slot(state, spi); for (i = 0; i < BPF_REG_SIZE; i++) if (slot->slot_type[i] == STACK_IRQ_FLAG) @@ -1196,7 +1293,7 @@ static int is_irq_flag_reg_valid_init(struct bpf_verifier_env *env, struct bpf_r if (spi < 0) return -EINVAL; - slot = &state->stack[spi]; + slot = bpf_stack_slot(state, spi); st = &slot->spilled_ptr; if (!st->id) @@ -1343,7 +1440,7 @@ static int copy_reference_state(struct bpf_verifier_state *dst, const struct bpf static int copy_stack_state(struct bpf_func_state *dst, const struct bpf_func_state *src) { - size_t n = src->allocated_stack / BPF_REG_SIZE; + size_t n = bpf_stack_nr_slots(src); dst->stack = copy_array(dst->stack, src->stack, n, sizeof(struct bpf_stack_state), GFP_KERNEL_ACCOUNT); @@ -1380,9 +1477,9 @@ static int resize_reference_state(struct bpf_verifier_state *state, size_t n) /* Possibly update state->allocated_stack to be at least size bytes. Also * possibly update the function's high-water mark in its bpf_subprog_info. */ -static int grow_stack_state(struct bpf_verifier_env *env, struct bpf_func_state *state, int size) +int bpf_grow_stack_state(struct bpf_verifier_env *env, struct bpf_func_state *state, int size) { - size_t old_n = state->allocated_stack / BPF_REG_SIZE, n; + size_t old_n = bpf_stack_nr_slots(state), n; /* The stack size is always a multiple of BPF_REG_SIZE. */ size = round_up(size, BPF_REG_SIZE); @@ -1603,6 +1700,7 @@ static void free_func_state(struct bpf_func_state *state) { if (!state) return; + kfree(state->loop_stack); kfree(state->stack_arg_regs); kfree(state->stack); kfree(state); @@ -1639,6 +1737,10 @@ static int copy_func_state(struct bpf_func_state *dst, memcpy(dst, src, offsetof(struct bpf_func_state, stack)); /* Instruction accounting is path-local, not part of verifier state. */ dst->insns_subtotal = 0; + dst->loop_stack = copy_array(dst->loop_stack, src->loop_stack, src->loop_stack_cnt, + sizeof(*src->loop_stack), GFP_KERNEL_ACCOUNT); + if (!dst->loop_stack) + return -ENOMEM; return copy_stack_state(dst, src); } @@ -1667,6 +1769,7 @@ int bpf_copy_verifier_state(struct bpf_verifier_state *dst_state, return err; dst_state->speculative = src->speculative; dst_state->in_sleepable = src->in_sleepable; + dst_state->unwound = src->unwound; dst_state->curframe = src->curframe; dst_state->branches = src->branches; dst_state->parent = src->parent; @@ -1846,12 +1949,124 @@ static void bpf_diag_record_caller_saved(struct bpf_verifier_env *env, } } +static void reg_step_reset(struct bpf_reg_state *reg) +{ + reg->base = 0; + reg->step = 1; +} + +static void scalar_step_add(struct bpf_reg_state *dst_reg, + const struct bpf_reg_state *a, + const struct bpf_reg_state *b) +{ + const struct bpf_reg_state *reg; + s64 amount, tmp; + + if (tnum_is_const(b->var_off)) { + reg = a; + amount = (s64)b->var_off.value; + } else if (tnum_is_const(a->var_off)) { + reg = b; + amount = (s64)a->var_off.value; + } else { + reg_step_reset(dst_reg); + return; + } + + /* + * Let x be a possible signed value of the register, + * d = amount, s = reg->step, and b = reg->base. + * Then x mod s = b, with s > 0 and 0 <= b < s. + * If the addition causes signed underflow, the resulting value + * is x + d + 2**64. + * + * Taking the result modulo s and substituting x mod s = b gives: + * (x + d + 2**64) mod s = (b + d + (2**64 mod s)) mod s (1) + * Keeping reg->step = s and updating reg->base to (b + d) mod s + * requires (1) to equal (b + d) mod s, which holds if and only + * if 2**64 mod s = 0. + * Signed overflow subtracts 2**64 and has the same requirement. + * + * Conservatively reset the stride if either wrap is possible. + */ + if (check_add_overflow(reg_smin(reg), amount, &tmp) || + check_add_overflow(reg_smax(reg), amount, &tmp)) { + reg_step_reset(dst_reg); + return; + } + + dst_reg->base = (reg->base + imod(amount, reg->step)) % reg->step; + dst_reg->step = reg->step; +} + +static void scalar_step_scale(struct bpf_reg_state *dst_reg, u64 amount) +{ + u16 step; + s64 tmp; + + /* + * Using the same notation as scalar_step_add(), with d > 0, + * the scaled value satisfies (x * d) mod (s * d) = b * d. + * If the multiplication wraps, its signed result can be represented + * as x * d - n * 2**64 for some nonzero integer n. + * + * Taking the result modulo s * d gives: + * (x * d - n * 2**64) mod (s * d) = + * (b * d - (n * 2**64 mod (s * d))) mod (s * d) (1) + * Updating dst_reg->base to b * d and dst_reg->step to s * d + * requires (1) to equal b * d, which holds if and only if + * n * 2**64 mod (s * d) = 0. + * + * Conservatively reset the stride if a wrap is possible. + */ + if (amount == 0 || check_mul_overflow(dst_reg->step, amount, &step) || + check_mul_overflow(reg_smin(dst_reg), (s64)amount, &tmp) || + check_mul_overflow(reg_smax(dst_reg), (s64)amount, &tmp)) { + reg_step_reset(dst_reg); + return; + } + /* + * The above established that `step = dst_reg->step * amount` + * does not overflow, therefore the base < step invariant holds + * after assignments below. + */ + dst_reg->base = dst_reg->base * amount; + dst_reg->step = step; +} + +/* + * Let x be a possible signed value of the register, + * s = reg->step, and b = reg->base. + * Then x mod s = b, with s > 0 and 0 <= b < s. + * Truncating to N bits and then zero- or sign-extending x + * is equivalent to computing x + k * 2**N for some integer k. + * For example, sign-extending the low 8 bits of x = 255 can + * be expressed as 255 - 2**8 = -1. + * + * Taking the result modulo s and substituting x mod s = b gives: + * (x + k * 2**N) mod s = (b + (k * 2**N mod s)) mod s + * Keeping reg->base = b and reg->step = s requires this to equal b, + * which holds if and only if k * 2**N mod s = 0. + * Continuing the example with s = 3 and b = 0, we have 255 mod 3 = 0, + * but (-1) mod 3 = 2, so the original base and step no longer hold. + * + * Conservatively reset the stride unless all possible values fit in + * the range where the conversion leaves them unchanged. + * Call before updating the register's bounds. Requires 0 < bits < 64. + */ +static void reg_step_check_unsigned(struct bpf_reg_state *reg, u32 bits) +{ + if (reg_umax(reg) >= (1ULL << bits)) + reg_step_reset(reg); +} + /* This helper doesn't clear reg->id */ static void ___mark_reg_known(struct bpf_reg_state *reg, u64 imm) { reg->var_off = tnum_const(imm); reg->r64 = cnum64_from_urange(imm, imm); reg->r32 = cnum32_from_urange((u32)imm, (u32)imm); + reg_step_reset(reg); } /* Mark the unknown part of a register (variable offset or scalar value) as @@ -1868,6 +2083,12 @@ static void __mark_reg_known(struct bpf_reg_state *reg, u64 imm) ___mark_reg_known(reg, imm); } +void bpf_mark_reg_known_scalar(struct bpf_reg_state *reg, u64 imm) +{ + __mark_reg_known(reg, imm); + reg->type = SCALAR_VALUE; +} + static void __mark_reg32_known(struct bpf_reg_state *reg, u64 imm) { reg->var_off = tnum_const_subreg(reg->var_off, imm); @@ -2109,8 +2330,24 @@ static void __reg_bound_offset(struct bpf_reg_state *reg) struct tnum var32_off = tnum_intersect(tnum_subreg(var64_off), tnum_range(reg_u32_min(reg), reg_u32_max(reg))); + u32 trailing_zero_bits; + u16 base = reg->base; + u16 step = reg->step; reg->var_off = tnum_or(tnum_clear_subreg(var64_off), var32_off); + + if (base == 0) + trailing_zero_bits = count_trailing_zeros(step); + else + trailing_zero_bits = min(count_trailing_zeros(base), + count_trailing_zeros(step)); + reg->var_off = tnum_and(reg->var_off, tnum_const(~0ULL << trailing_zero_bits)); +} + +static void __update_bounds_from_step(struct bpf_reg_state *reg) +{ + reg->r64 = cnum64_intersect_linear(reg->r64, reg->base, reg->step); + deduce_bounds_32_from_64(reg); } static bool range_bounds_violation(struct bpf_reg_state *reg); @@ -2132,6 +2369,15 @@ static void reg_bounds_sync(struct bpf_reg_state *reg) * then new var_off is (0; 0x7f...fc) which improves our umax. */ __update_reg_bounds(reg); + /* + * Both tnum and base+step define a sequence of points, + * potentially w/o intersection, or intersecting after N + * tnum_step()'s, where N can be large. On the other hand, + * step+base *must* be in sync with r64 in order for + * array traversal logic starting at reg_smin() to be sound. + * Hence, place this as a final step in the reg_bounds_sync(). + */ + __update_bounds_from_step(reg); } static bool const_tnum_range_mismatch(struct bpf_reg_state *reg) @@ -2175,17 +2421,24 @@ static int reg_bounds_sanity_check(struct bpf_verifier_env *env, goto out; } + if (reg->step == 0 || reg->base >= reg->step) { + msg = "invalid base/step"; + goto out; + } + return 0; out: verifier_bug(env, "REG INVARIANTS VIOLATION (%s): %s r64={.base=%#llx, .size=%#llx} " - "r32={.base=%#x, .size=%#x} var_off=(%#llx, %#llx)", + "r32={.base=%#x, .size=%#x} var_off=(%#llx, %#llx) step=%u+%u", ctx, msg, reg->r64.base, reg->r64.size, reg->r32.base, reg->r32.size, - reg->var_off.value, reg->var_off.mask); + reg->var_off.value, reg->var_off.mask, + reg->base, reg->step); if (env->test_reg_invariants) return -EFAULT; __mark_reg_unbounded(reg); + reg_step_reset(reg); return 0; } @@ -2196,6 +2449,7 @@ void bpf_mark_reg_unknown_imprecise(struct bpf_reg_state *reg) reg->type = SCALAR_VALUE; reg->var_off = tnum_unknown; __mark_reg_unbounded(reg); + reg_step_reset(reg); } /* Mark a register as having a completely unknown (scalar) value, @@ -2406,6 +2660,7 @@ static int add_subprog(struct bpf_verifier_env *env, int off) return -E2BIG; } /* determine subprog starts. The end is one before the next starts */ + env->subprog_info[env->subprog_cnt].exit_idx = U32_MAX; env->subprog_info[env->subprog_cnt++].start = off; sort(env->subprog_info, env->subprog_cnt, sizeof(env->subprog_info[0]), cmp_subprogs, NULL); @@ -2514,17 +2769,21 @@ find_kfunc_desc(const struct bpf_prog *prog, u32 func_id, u16 offset) struct bpf_kfunc_desc_tab *tab; tab = prog->aux->kfunc_tab; - return bsearch(&desc, tab->descs, tab->nr_descs, + return bsearch(&desc, tab->descs, tab->nr_base_descs, sizeof(tab->descs[0]), kfunc_desc_cmp_by_id_off); } int bpf_get_kfunc_addr(const struct bpf_prog *prog, u32 func_id, - u16 btf_fd_idx, u8 **func_addr) + u16 desc_idx, u8 **func_addr) { + struct bpf_kfunc_desc_tab *tab; const struct bpf_kfunc_desc *desc; - desc = find_kfunc_desc(prog, func_id, btf_fd_idx); - if (!desc) + tab = prog->aux->kfunc_tab; + if (desc_idx >= tab->nr_descs) + return -EFAULT; + desc = &tab->descs[desc_idx]; + if (desc->func_id != func_id) return -EFAULT; *func_addr = (u8 *)desc->addr; @@ -2799,7 +3058,7 @@ static int fetch_kfunc_meta(struct bpf_verifier_env *env, } static int gen_kfunc_arg_proto(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, - struct bpf_func_proto *proto); + const struct btf_func_model *fm, struct bpf_func_proto *proto); int bpf_add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, u16 offset) { @@ -2864,10 +3123,15 @@ int bpf_add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, u16 offset) if (find_kfunc_desc(env->prog, func_id, offset)) return 0; - if (tab->nr_descs == MAX_KFUNC_DESCS) { + if (tab->nr_base_descs == MAX_KFUNC_DESCS) { verbose(env, "too many different kernel function calls\n"); return -E2BIG; } + if (tab->nr_descs != tab->nr_base_descs) { + verifier_bug(env, "unexpected specialized func desc found (%d descs, %d base descs)", + tab->nr_descs, tab->nr_base_descs); + return -EFAULT; + } err = fetch_kfunc_meta(env, func_id, offset, &kfunc); if (err) @@ -2888,6 +3152,18 @@ int bpf_add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, u16 offset) err = btf_distill_func_proto(&env->log, kfunc.btf, kfunc.proto, kfunc.name, &func_model); if (err) return err; + if (func_model.ret_size > 8) { + if (kfunc.flags && (*kfunc.flags & KF_FASTCALL)) { + verbose(env, "kfunc %s with >8-byte return is not supported with KF_FASTCALL\n", + kfunc.name); + return -EOPNOTSUPP; + } + if (!bpf_jit_supports_kfunc_ret_reg_pair()) { + verbose(env, "kfunc %s with >8-byte return is not supported by JIT\n", + kfunc.name); + return -EOPNOTSUPP; + } + } memset(&meta, 0, sizeof(meta)); meta.btf = kfunc.btf; @@ -2904,7 +3180,7 @@ int bpf_add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, u16 offset) desc = &tab->descs[tab->nr_descs]; memset(desc, 0, sizeof(*desc)); - err = gen_kfunc_arg_proto(env, &meta, &desc->proto); + err = gen_kfunc_arg_proto(env, &meta, &func_model, &desc->proto); if (err) return err; @@ -2913,7 +3189,8 @@ int bpf_add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, u16 offset) desc->addr = addr; desc->func_model = func_model; tab->nr_descs++; - sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]), + tab->nr_base_descs++; + sort(tab->descs, tab->nr_base_descs, sizeof(tab->descs[0]), kfunc_desc_cmp_by_id_off, NULL); return 0; } @@ -2955,6 +3232,22 @@ static int add_subprogs(struct bpf_verifier_env *env) return ret; } + /* + * func_info describes all functions of the program. Those that are + * referenced only from data, e.g. from a table of functions to be + * called via callx, are not seen by the loop above. They are possible + * callees that have to be known upfront as well. + */ + if (env->bpf_capable) { + struct bpf_prog_aux *aux = env->prog->aux; + + for (i = 1; i < aux->func_info_cnt; i++) { + ret = add_subprog(env, aux->func_info[i].insn_off); + if (ret < 0) + return ret; + } + } + ret = bpf_find_exception_callback_insn_off(env); if (ret < 0) return ret; @@ -3014,6 +3307,34 @@ static int add_kfuncs(struct bpf_verifier_env *env) return 0; } +static void find_subprog_properties(struct bpf_verifier_env *env) +{ + struct bpf_subprog_info *subprog = env->subprog_info; + struct bpf_insn *insn = env->prog->insnsi; + int cur_subprog; + + for (cur_subprog = 0; cur_subprog < env->subprog_cnt; cur_subprog++) { + int i; + + for (i = subprog[cur_subprog].start; + i < subprog[cur_subprog + 1].start; i++) { + u8 code = insn[i].code; + + if (code == (BPF_JMP | BPF_CALL) && + insn[i].src_reg == 0 && + insn[i].imm == BPF_FUNC_tail_call) { + subprog[cur_subprog].has_tail_call = true; + subprog[cur_subprog].tail_call_reachable = true; + } + if (BPF_CLASS(code) == BPF_LD && + (BPF_MODE(code) == BPF_ABS || BPF_MODE(code) == BPF_IND)) + subprog[cur_subprog].has_ld_abs = true; + if (bpf_is_callx(&insn[i])) + env->has_callx = true; + } + } +} + static int check_subprogs(struct bpf_verifier_env *env) { int i, subprog_start, subprog_end, off, cur_subprog = 0; @@ -3027,15 +3348,6 @@ static int check_subprogs(struct bpf_verifier_env *env) for (i = 0; i < insn_cnt; i++) { u8 code = insn[i].code; - if (code == (BPF_JMP | BPF_CALL) && - insn[i].src_reg == 0 && - insn[i].imm == BPF_FUNC_tail_call) { - subprog[cur_subprog].has_tail_call = true; - subprog[cur_subprog].tail_call_reachable = true; - } - if (BPF_CLASS(code) == BPF_LD && - (BPF_MODE(code) == BPF_ABS || BPF_MODE(code) == BPF_IND)) - subprog[cur_subprog].has_ld_abs = true; if (BPF_CLASS(code) != BPF_JMP && BPF_CLASS(code) != BPF_JMP32) goto next; if (BPF_OP(code) == BPF_CALL) @@ -3059,9 +3371,10 @@ static int check_subprogs(struct bpf_verifier_env *env) } next: if (i == subprog_end - 1) { - /* to avoid fall-through from one subprog into another + /* + * To avoid fall-through from one subprog into another, * the last insn of the subprog should be either exit - * or unconditional jump back or bpf_throw call + * or unconditional jump back or bpf_throw call. */ if (code != (BPF_JMP | BPF_EXIT) && code != (BPF_JMP32 | BPF_JA) && @@ -3084,12 +3397,90 @@ static int check_subprogs(struct bpf_verifier_env *env) return 0; } +/* + * The callee of callx is known to the main verification pass only, which + * records the 'caller' -> 'callee' edge of the call graph for the checks + * that follow it: absence of recursion and the maximum stack depth. + */ +static int record_callx_edge(struct bpf_verifier_env *env, int caller, int callee) +{ + u32 cnt = env->subprog_cnt; + + if (!env->callx_edges) { + env->callx_edges = kvcalloc(BITS_TO_LONGS(cnt * cnt), sizeof(long), + GFP_KERNEL_ACCOUNT); + if (!env->callx_edges) + return -ENOMEM; + } + __set_bit(caller * cnt + callee, env->callx_edges); + return 0; +} + +/* + * Return the first subprog with the number >= 'from' that 'caller' calls + * via callx, or -1 when there is none. + */ +static int next_callx_callee(struct bpf_verifier_env *env, int caller, int from) +{ + u32 cnt = env->subprog_cnt; + unsigned long bit, end = (caller + 1) * cnt; + + if (!env->callx_edges || from >= cnt) + return -1; + bit = find_next_bit(env->callx_edges, end, caller * cnt + from); + return bit < end ? bit - caller * cnt : -1; +} + +/* + * 'call foo; unwind_goto pad' goes to the pad when foo left with unwind_exit. + * JITs don't emit a branch for it. unwind_exit finds the pad by its return + * address. So unwind_goto has to follow the call and nothing may jump to it. + */ +static int check_unwind_insns(struct bpf_verifier_env *env) +{ + struct bpf_insn *insn = env->prog->insnsi; + bool seen = false; + int i; + + for (i = 0; i < env->prog->len; i++, insn++) { + if (bpf_is_unwind_exit(insn)) { + bpf_find_containing_subprog(env, i)->can_unwind = true; + seen = true; + } + if (!bpf_is_unwind_goto(insn)) + continue; + seen = true; + if (!i || !(bpf_pseudo_call(insn - 1) || bpf_is_callx(insn - 1))) { + verbose(env, "unwind_goto at insn %d doesn't follow a call of a subprog\n", + i); + return -EINVAL; + } + if (bpf_is_jump_target(env, i)) { + verbose(env, "jump to unwind_goto at insn %d\n", i); + return -EINVAL; + } + } + if (!seen) + return 0; + if (!bpf_jit_supports_unwind() || bpf_prog_is_offloaded(env->prog->aux)) { + verbose(env, "JIT does not support unwinding\n"); + return -ENOTSUPP; + } + env->prog->jit_required = 1; + env->prog->aux->can_unwind = env->subprog_info[0].can_unwind; + return 0; +} + /* * Sort subprogs in topological order so that leaf subprogs come first and * their callers come later. This is a DFS post-order traversal of the call * graph. Scan only reachable instructions (those in the computed postorder) of * the current subprog to discover callees (direct subprogs and sync * callbacks). + * + * The callees of callx are not known before the main verification pass. + * When callx is used the sort is repeated after it with the recorded callx + * edges added to the call graph to reject recursion through indirect calls. */ static int sort_subprogs_topo(struct bpf_verifier_env *env) { @@ -3130,12 +3521,22 @@ static int sort_subprogs_topo(struct bpf_verifier_env *env) int idx = insn_postorder[j]; int callee; - if (!bpf_pseudo_call(&insn[idx]) && !bpf_pseudo_func(&insn[idx])) + if (bpf_is_callx(&insn[idx])) { + /* find a callee that is not explored yet */ + callee = -1; + do { + callee = next_callx_callee(env, cur, callee + 1); + } while (callee >= 0 && color[callee] == 2); + if (callee < 0) + continue; + } else if (bpf_pseudo_call(&insn[idx]) || bpf_pseudo_func(&insn[idx])) { + callee = bpf_find_subprog(env, idx + insn[idx].imm + 1); + if (callee < 0) { + ret = -EFAULT; + goto out; + } + } else { continue; - callee = bpf_find_subprog(env, idx + insn[idx].imm + 1); - if (callee < 0) { - ret = -EFAULT; - goto out; } if (color[callee] == 2) continue; @@ -3232,26 +3633,30 @@ static void mark_indirect_target(struct bpf_verifier_env *env, int idx) env->insn_aux_data[idx].indirect_target = true; } +static void mark_non_stack_access(struct bpf_verifier_env *env, int idx) +{ + env->insn_aux_data[idx].non_stack_access = true; +} + +/* Layout of one packed linked register in the jump history, see linked_regs_pack() */ #define LR_FRAMENO_BITS 4 -#define LR_SPI_BITS 6 -#define LR_ENTRY_BITS (LR_SPI_BITS + LR_FRAMENO_BITS + 1) -#define LR_SIZE_BITS 4 -#define LR_FRAMENO_MASK ((1ull << LR_FRAMENO_BITS) - 1) -#define LR_SPI_MASK ((1ull << LR_SPI_BITS) - 1) -#define LR_SIZE_MASK ((1ull << LR_SIZE_BITS) - 1) -#define LR_SPI_OFF LR_FRAMENO_BITS -#define LR_IS_REG_OFF (LR_SPI_BITS + LR_FRAMENO_BITS) -#define LINKED_REGS_MAX 5 +#define LR_INDEX_BITS 11 +#define LR_FRAMENO_MASK ((1u << LR_FRAMENO_BITS) - 1) +#define LR_IS_REG BIT(LR_FRAMENO_BITS) +#define LR_INDEX_OFF (LR_FRAMENO_BITS + 1) +#define LR_INDEX_MASK ((1u << LR_INDEX_BITS) - 1) +#define LINKED_REGS_MAX BPF_LINKED_REGS_MAX static_assert(MAX_CALL_FRAMES <= (1 << LR_FRAMENO_BITS)); -static_assert(LINKED_REGS_MAX < (1 << LR_SIZE_BITS)); -static_assert(LINKED_REGS_MAX * LR_ENTRY_BITS + LR_SIZE_BITS <= 64); +static_assert(MAX_BPF_REG <= (1 << LR_INDEX_BITS)); +static_assert(MAX_BPF_STACK_SLOTS <= (1 << LR_INDEX_BITS)); +static_assert(LR_INDEX_OFF + LR_INDEX_BITS <= 16); struct linked_reg { u8 frameno; union { - u8 spi; - u8 regno; + u16 spi; + u16 regno; }; bool is_reg; }; @@ -3270,48 +3675,34 @@ static struct linked_reg *linked_regs_push(struct linked_regs *s) } /* - * Use u64 as a vector of 5 11-bit values, use first 4-bits to track - * number of elements currently in stack. - * Pack one history entry for linked registers as 11 bits in the following format: - * - 4-bits frameno - * - 6-bits spi_or_reg - * - 1-bit is_reg + * Pack linked registers for a jump history entry, one u16 each: + * - 4 bits frameno + * - 1 bit is_reg + * - 11 bits register or stack slot index */ -static u64 linked_regs_pack(struct linked_regs *s) +static void linked_regs_pack(const struct linked_regs *s, u16 *packed) { - u64 val = 0; int i; for (i = 0; i < s->cnt; ++i) { - struct linked_reg *e = &s->entries[i]; - u64 tmp = 0; + const struct linked_reg *e = &s->entries[i]; - tmp |= e->frameno; - tmp |= e->spi << LR_SPI_OFF; - tmp |= (e->is_reg ? 1 : 0) << LR_IS_REG_OFF; - - val <<= LR_ENTRY_BITS; - val |= tmp; + packed[i] = e->frameno | (e->is_reg ? LR_IS_REG : 0) | (e->spi << LR_INDEX_OFF); } - val <<= LR_SIZE_BITS; - val |= s->cnt; - return val; } -static void linked_regs_unpack(u64 val, struct linked_regs *s) +static void linked_regs_unpack(const struct bpf_jmp_history_entry *hist, struct linked_regs *s) { int i; - s->cnt = val & LR_SIZE_MASK; - val >>= LR_SIZE_BITS; - + s->cnt = hist->linked_regs_cnt; for (i = 0; i < s->cnt; ++i) { struct linked_reg *e = &s->entries[i]; + u16 packed = hist->linked_regs[i]; - e->frameno = val & LR_FRAMENO_MASK; - e->spi = (val >> LR_SPI_OFF) & LR_SPI_MASK; - e->is_reg = (val >> LR_IS_REG_OFF) & 0x1; - val >>= LR_ENTRY_BITS; + e->frameno = packed & LR_FRAMENO_MASK; + e->is_reg = packed & LR_IS_REG; + e->spi = (packed >> LR_INDEX_OFF) & LR_INDEX_MASK; } } @@ -3353,10 +3744,10 @@ void bpf_bt_sync_linked_regs(struct backtrack_state *bt, struct bpf_jmp_history_ bool some_precise = false; int i; - if (!hist || hist->linked_regs == 0) + if (!hist || !hist->linked_regs_cnt) return; - linked_regs_unpack(hist->linked_regs, &linked_regs); + linked_regs_unpack(hist, &linked_regs); for (i = 0; i < linked_regs.cnt; ++i) { struct linked_reg *e = &linked_regs.entries[i]; @@ -3453,17 +3844,18 @@ static void save_register_state(struct bpf_verifier_env *env, int spi, struct bpf_reg_state *reg, int size) { + struct bpf_stack_state *slot = bpf_stack_slot(state, spi); int i; - bpf_diag_mod_begin(env, &state->stack[spi].spilled_ptr, reg, BPF_DIAG_MOD_SPILL); - state->stack[spi].spilled_ptr = *reg; + bpf_diag_mod_begin(env, &slot->spilled_ptr, reg, BPF_DIAG_MOD_SPILL); + slot->spilled_ptr = *reg; for (i = BPF_REG_SIZE; i > BPF_REG_SIZE - size; i--) - state->stack[spi].slot_type[i - 1] = STACK_SPILL; + slot->slot_type[i - 1] = STACK_SPILL; /* size < 8 bytes spill */ for (; i; i--) - mark_stack_slot_misc(env, &state->stack[spi].slot_type[i - 1]); + mark_stack_slot_misc(env, &slot->slot_type[i - 1]); bpf_diag_mod_end(env); } @@ -3505,14 +3897,15 @@ static void check_fastcall_stack_contract(struct bpf_verifier_env *env, static void scrub_special_slot(struct bpf_func_state *state, int spi) { + struct bpf_stack_state *slot = bpf_stack_slot(state, spi); int i; /* regular write of data into stack destroys any spilled ptr */ - state->stack[spi].spilled_ptr.type = NOT_INIT; + slot->spilled_ptr.type = NOT_INIT; /* Mark slots as STACK_MISC if they belonged to spilled ptr/dynptr/iter. */ - if (is_stack_slot_special(&state->stack[spi])) + if (is_stack_slot_special(slot)) for (i = 0; i < BPF_REG_SIZE; i++) - scrub_spilled_slot(&state->stack[spi].slot_type[i]); + scrub_spilled_slot(&slot->slot_type[i]); } /* check_stack_{read,write}_fixed_off functions track spill/fill of registers, @@ -3530,13 +3923,15 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env, struct bpf_reg_state *reg = NULL; int insn_flags = INSN_F_STACK_ACCESS; int hist_spi = spi, hist_frame = state->frameno; + struct bpf_stack_state *ss = bpf_stack_slot(state, spi); - /* caller checked that off % size == 0 and -MAX_BPF_STACK <= off < 0, + /* + * caller checked that off % size == 0 and -env->stack_limit <= off < 0, * so it's aligned access and [off, off + size) are within stack limits */ if (!env->allow_ptr_leaks && - bpf_is_spilled_reg(&state->stack[spi]) && - !bpf_is_spilled_scalar_reg(&state->stack[spi]) && + bpf_is_spilled_reg(ss) && + !bpf_is_spilled_scalar_reg(ss) && size != BPF_REG_SIZE) { const char *reason; @@ -3558,7 +3953,7 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env, bool sanitize = reg && is_pointer_regtype(reg->type); for (i = 0; i < size; i++) { - u8 type = state->stack[spi].slot_type[(slot - i) % + u8 type = ss->slot_type[(slot - i) % BPF_REG_SIZE]; if (type != STACK_MISC && type != STACK_ZERO) { @@ -3575,7 +3970,6 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env, if (err) return err; - check_fastcall_stack_contract(env, state, insn_idx, off); mark_stack_slot_scratched(env, spi); if (reg && !(off % BPF_REG_SIZE) && reg->type == SCALAR_VALUE && env->bpf_capable) { bool reg_value_fits; @@ -3587,7 +3981,7 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env, save_register_state(env, state, spi, reg, size); /* Break the relation on a narrowing spill. */ if (!reg_value_fits) - clear_scalar_id(&state->stack[spi].spilled_ptr); + clear_scalar_id(&ss->spilled_ptr); } else if (!reg && !(off % BPF_REG_SIZE) && is_bpf_st_mem(insn) && env->bpf_capable) { struct bpf_reg_state *tmp_reg = &env->fake_reg[0]; @@ -3611,8 +4005,8 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env, } else { u8 type = STACK_MISC; - if (bpf_is_spilled_reg(&state->stack[spi])) - bpf_diag_record_scrub(env, &state->stack[spi].spilled_ptr, + if (bpf_is_spilled_reg(ss)) + bpf_diag_record_scrub(env, &ss->spilled_ptr, BPF_DIAG_MOD_WRITE); scrub_special_slot(state, spi); @@ -3633,13 +4027,13 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env, /* Mark slots affected by this stack write. */ for (i = 0; i < size; i++) - state->stack[spi].slot_type[(slot - i) % BPF_REG_SIZE] = type; + ss->slot_type[(slot - i) % BPF_REG_SIZE] = type; insn_flags = 0; /* not a register spill */ } if (insn_flags) return bpf_push_jmp_history(env, env->cur_state, insn_flags, - hist_spi, hist_frame, 0); + hist_spi, hist_frame, NULL, 0); return 0; } @@ -3687,24 +4081,21 @@ static int check_stack_write_var_off(struct bpf_verifier_env *env, (!value_reg && is_bpf_st_mem(insn) && insn->imm == 0)) writing_zero = true; - for (i = min_off; i < max_off; i++) { - int spi; + err = destroy_dynptrs_in_stack_slots(env, state, bpf_get_spi(max_off - 1), + bpf_get_spi(min_off)); + if (err) + return err; - spi = bpf_get_spi(i); - err = destroy_if_dynptr_stack_slot(env, state, spi); - if (err) - return err; - } - - check_fastcall_stack_contract(env, state, insn_idx, min_off); /* Variable offset writes destroy any spilled pointers in range. */ for (i = min_off; i < max_off; i++) { + struct bpf_stack_state *ss; u8 new_type, *stype; int slot, spi; slot = -i - 1; spi = slot / BPF_REG_SIZE; - stype = &state->stack[spi].slot_type[slot % BPF_REG_SIZE]; + ss = bpf_stack_slot(state, spi); + stype = &ss->slot_type[slot % BPF_REG_SIZE]; mark_stack_slot_scratched(env, spi); if (!env->allow_ptr_leaks && *stype != STACK_MISC && *stype != STACK_ZERO) { @@ -3727,9 +4118,8 @@ static int check_stack_write_var_off(struct bpf_verifier_env *env, /* If writing_zero and the spi slot contains a spill of value 0, * maintain the spill type. */ - if (writing_zero && *stype == STACK_SPILL && - bpf_is_spilled_scalar_reg(&state->stack[spi])) { - struct bpf_reg_state *spill_reg = &state->stack[spi].spilled_ptr; + if (writing_zero && *stype == STACK_SPILL && bpf_is_spilled_scalar_reg(ss)) { + struct bpf_reg_state *spill_reg = &ss->spilled_ptr; if (tnum_is_const(spill_reg->var_off) && spill_reg->var_off.value == 0) { zero_used = true; @@ -3800,23 +4190,22 @@ static int mark_reg_stack_read(struct bpf_verifier_env *env, { struct bpf_verifier_state *vstate = env->cur_state; struct bpf_func_state *state = vstate->frame[vstate->curframe]; - u64 zero_spill_mask = 0; int i, slot, spi; u8 *stype; - int zeros = 0; + int zeros = 0, zero_spills = 0; for (i = min_off; i < max_off; i++) { slot = -i - 1; spi = slot / BPF_REG_SIZE; mark_stack_slot_scratched(env, spi); - stype = ptr_state->stack[spi].slot_type; + stype = bpf_stack_slot(ptr_state, spi)->slot_type; if (stype[slot % BPF_REG_SIZE] == STACK_ZERO) { zeros++; continue; } if (stype[slot % BPF_REG_SIZE] == STACK_SPILL && - bpf_register_is_null(&ptr_state->stack[spi].spilled_ptr)) { - zero_spill_mask |= 1ull << spi; + bpf_register_is_null(&bpf_stack_slot(ptr_state, spi)->spilled_ptr)) { + zero_spills++; zeros++; continue; } @@ -3827,8 +4216,14 @@ static int mark_reg_stack_read(struct bpf_verifier_env *env, * so the whole register == const_zero. */ __mark_reg_const_zero(env, &state->regs[dst_regno]); - if (zero_spill_mask) { - bpf_bt_set_frame_slot_mask(&env->bt, ptr_state->frameno, zero_spill_mask); + if (zero_spills) { + for (i = min_off; i < max_off; i++) { + slot = -i - 1; + spi = slot / BPF_REG_SIZE; + stype = bpf_stack_slot(ptr_state, spi)->slot_type; + if (stype[slot % BPF_REG_SIZE] == STACK_SPILL) + bpf_bt_set_frame_slot(&env->bt, ptr_state->frameno, spi); + } return mark_chain_precision_batch(env, env->cur_state); } } else { @@ -3839,19 +4234,46 @@ static int mark_reg_stack_read(struct bpf_verifier_env *env, return 0; } -static void bpf_diag_stack_read_uninit(struct bpf_verifier_env *env, int off, int i, - int size) +static void bpf_diag_stack_read_invalid(struct bpf_verifier_env *env, int off, int i, int size, + enum bpf_stack_slot_type type) { - const char *reason; + const char *problem, *reason, *suggestion, *kind; - reason = bpf_diag_fmt(env, - "This rejected read uses %d bytes at stack offset %d, but byte %d in that range is uninitialized on this path. " - "Programs loaded with CAP_PERFMON can be allowed to read uninitialized stack bytes, but this program is being rejected without that allowance.", - size, off, i); - bpf_diag_memory( - env, env->insn_idx, "uninitialized stack read", reason, - "Initialize every byte in the stack range before reading it, adjust the offset and size so the read covers only initialized bytes, " - "or load with CAP_PERFMON if uninitialized stack reads are intended."); + if (type == STACK_INVALID) { + reason = bpf_diag_fmt( + env, "This rejected read uses %d bytes at stack offset %d, but byte %d in that range is uninitialized on this path. " + "Programs loaded with CAP_PERFMON can be allowed to read uninitialized stack bytes, but this program is being rejected without that allowance.", + size, off, i); + bpf_diag_memory( + env, env->insn_idx, "uninitialized stack read", reason, + "Initialize every byte in the stack range before reading it, adjust the offset and size so the read covers only initialized bytes, " + "or load with CAP_PERFMON if uninitialized stack reads are intended."); + return; + } + + switch (type) { + case STACK_DYNPTR: + kind = "dynptr"; + suggestion = "Use dynptr helpers or kfuncs to access the object represented by the dynptr instead of reading the dynptr state directly."; + break; + case STACK_ITER: + kind = "iterator"; + suggestion = "Use iterator kfuncs to advance or destroy the iterator instead of reading its state directly."; + break; + case STACK_IRQ_FLAG: + kind = "IRQ flag"; + suggestion = "Pass the saved IRQ flag to the matching restore kfunc instead of reading its state directly."; + break; + default: + return; + } + + problem = bpf_diag_fmt(env, "direct read of %s stack state", kind); + reason = bpf_diag_fmt( + env, "This rejected read uses %d bytes at stack offset %d, but byte %d in that range belongs to verifier-managed %s state. " + "This state has an opaque representation that BPF programs cannot read directly.", + size, off, i, kind); + bpf_diag_memory(env, env->insn_idx, problem, reason, suggestion); } /* Read the stack at 'off' and put the results into the register indicated by @@ -3876,12 +4298,12 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env, int err; int insn_flags = INSN_F_STACK_ACCESS; int hist_spi = spi, hist_frame = reg_state->frameno; + struct bpf_stack_state *ss = bpf_stack_slot(reg_state, spi); - stype = reg_state->stack[spi].slot_type; - reg = ®_state->stack[spi].spilled_ptr; + stype = ss->slot_type; + reg = &ss->spilled_ptr; mark_stack_slot_scratched(env, spi); - check_fastcall_stack_contract(env, state, env->insn_idx, off); /* * Refine the in-progress load record's origin to the source stack slot. @@ -3889,11 +4311,8 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env, if (dst_regno >= 0) bpf_diag_mod_begin(env, &state->regs[dst_regno], reg, BPF_DIAG_MOD_WRITE); - if (bpf_is_spilled_reg(®_state->stack[spi])) { - u8 spill_size = 1; - - for (i = BPF_REG_SIZE - 1; i > 0 && stype[i - 1] == STACK_SPILL; i--) - spill_size++; + if (bpf_is_spilled_reg(ss)) { + u8 spill_size = bpf_spill_size(ss); if (size != BPF_REG_SIZE || spill_size != BPF_REG_SIZE) { if (reg->type != SCALAR_VALUE) { @@ -3902,11 +4321,11 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env, return -EACCES; } - if (dst_regno < 0) - return 0; - if (size <= spill_size && bpf_stack_narrow_access_ok(off, size, spill_size)) { + if (dst_regno < 0) + return 0; + if (env->bpf_capable && size == 4 && spill_size == 4 && get_reg_width(reg) <= 32) /* Ensure stack slot has an ID to build a relation @@ -3943,11 +4362,14 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env, } else { verbose(env, "invalid read from stack off %d+%d size %d\n", off, i, size); - bpf_diag_stack_read_uninit(env, off, i, size); + bpf_diag_stack_read_invalid(env, off, i, size, type); } return -EACCES; } + if (dst_regno < 0) + return 0; + if (spill_cnt == size && tnum_is_const(reg->var_off) && reg->var_off.value == 0) { __mark_reg_const_zero(env, &state->regs[dst_regno]); @@ -4002,7 +4424,7 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env, } else { verbose(env, "invalid read from stack off %d+%d size %d\n", off, i, size); - bpf_diag_stack_read_uninit(env, off, i, size); + bpf_diag_stack_read_invalid(env, off, i, size, type); } return -EACCES; } @@ -4015,7 +4437,7 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env, } if (insn_flags) return bpf_push_jmp_history(env, env->cur_state, insn_flags, - hist_spi, hist_frame, 0); + hist_spi, hist_frame, NULL, 0); return 0; } @@ -4068,7 +4490,6 @@ static int check_stack_read_var_off(struct bpf_verifier_env *env, struct bpf_reg dst_regno); if (err) return err; - check_fastcall_stack_contract(env, ptr_state, env->insn_idx, min_off); return 0; } @@ -4101,13 +4522,13 @@ static int check_stack_read(struct bpf_verifier_env *env, tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); verbose(env, "variable offset stack pointer cannot be passed into helper function; var_off=%s off=%d size=%d\n", tn_buf, off, size); - reason = bpf_diag_fmt(env, - "The helper would access the stack through variable offset %s plus fixed offset %d and size %d. " - "Helper stack memory arguments require a constant stack offset and a precise initialized range.", + reason = bpf_diag_fmt( + env, "The instruction would access the stack through variable offset %s plus fixed offset %d and size %d. " + "This stack access requires a constant stack offset and a precise initialized range.", tn_buf, off, size); bpf_diag_memory( - env, env->insn_idx, "variable stack access", reason, - "Use a fixed stack offset for helper memory arguments, or copy the needed bytes into a fixed stack slot first."); + env, env->insn_idx, "variable-offset stack access", reason, + "Use a fixed stack offset for the instruction, selecting the target stack slot on separate control-flow paths if necessary."); return -EACCES; } /* Variable offset is prohibited for unprivileged mode for simplicity @@ -4205,7 +4626,7 @@ static int check_stack_arg_write(struct bpf_verifier_env *env, struct bpf_func_s bpf_diag_mod_end(env); state->no_stack_arg_load = true; return bpf_push_jmp_history(env, env->cur_state, - INSN_F_STACK_ARG_ACCESS, spi, 0, 0); + INSN_F_STACK_ARG_ACCESS, spi, 0, NULL, 0); } /* @@ -4239,7 +4660,7 @@ static int check_stack_arg_read(struct bpf_verifier_env *env, struct bpf_func_st cur->regs[dst_regno] = *arg; bpf_diag_mod_end(env); return bpf_push_jmp_history(env, env->cur_state, - INSN_F_STACK_ARG_ACCESS, spi, 0, 0); + INSN_F_STACK_ARG_ACCESS, spi, 0, NULL, 0); } static int mark_stack_arg_precision(struct bpf_verifier_env *env, int arg_idx) @@ -4599,7 +5020,7 @@ BTF_ID(struct, cgroup) BTF_ID(struct, bpf_cpumask) #endif BTF_ID(struct, task_struct) -#ifdef CONFIG_CRYPTO +#ifdef CONFIG_BPF_CRYPTO BTF_ID(struct, bpf_crypto_ctx) #endif #ifdef CONFIG_INET @@ -4822,7 +5243,7 @@ static int check_map_access(struct bpf_verifier_env *env, struct bpf_reg_state * } static bool may_access_direct_pkt_data(struct bpf_verifier_env *env, - const struct bpf_func_proto *fn, + const struct bpf_call_arg_meta *meta, enum bpf_access_type t) { enum bpf_prog_type prog_type = resolve_prog_type(env->prog); @@ -4846,10 +5267,11 @@ static bool may_access_direct_pkt_data(struct bpf_verifier_env *env, case BPF_PROG_TYPE_LWT_XMIT: case BPF_PROG_TYPE_SK_SKB: case BPF_PROG_TYPE_SK_MSG: - if (fn) - return fn->pkt_access; + if (meta && !meta->btf && meta->func_id) + return meta->fn->pkt_access; - env->seen_direct_write = true; + if (t == BPF_WRITE) + env->seen_direct_write = true; return true; case BPF_PROG_TYPE_CGROUP_SOCKOPT: @@ -5061,6 +5483,19 @@ static bool is_arena_reg(struct bpf_verifier_env *env, int regno) return reg->type == PTR_TO_ARENA; } +/* + * There are no address spaces in Rust, addresses of arena are plain numbers. + * When the program is loaded with BPF_F_ARENA_SCALAR and has an arena a load or + * a store through a number is an access to arena at the low 32 bits of it, + * like the access through PTR_TO_ARENA is. The register stays a number. + */ +static bool is_arena_scalar(struct bpf_verifier_env *env, int regno) +{ + const struct bpf_reg_state *reg = reg_state(env, regno); + + return reg->type == SCALAR_VALUE && env->arena_scalar && env->prog->aux->arena; +} + static bool is_load_acq_unsafe(struct bpf_verifier_env *env, int regno, struct bpf_insn *insn) { @@ -5091,7 +5526,7 @@ static bool atomic_ptr_type_ok(struct bpf_verifier_env *env, int regno, return false; if (is_sk_reg(env, regno)) return false; - if (is_arena_reg(env, regno)) + if (is_arena_reg(env, regno) || is_arena_scalar(env, regno)) return bpf_jit_supports_insn(insn, true); if (is_load_acq_unsafe(env, regno, insn)) return false; @@ -5107,27 +5542,17 @@ static u32 *reg2btf_ids[__BPF_REG_TYPE_MAX] = { [CONST_PTR_TO_MAP] = btf_bpf_map_id, }; -static enum bpf_reg_type lookup_reg2btf_ids(u32 ref_id) -{ - enum bpf_reg_type type; - - for (type = 0; type < __BPF_REG_TYPE_MAX; type++) { - if (reg2btf_ids[type] && *reg2btf_ids[type] == ref_id) - return type; - } - - return NOT_INIT; -} - -static bool is_trusted_reg(struct bpf_verifier_env *env, const struct bpf_reg_state *reg) +static bool is_trusted_reg_type(struct bpf_verifier_env *env, + const struct bpf_reg_state *reg, + enum bpf_reg_type type) { /* A referenced register is always trusted. */ if (reg_is_referenced(env, reg)) return true; /* Types listed in the reg2btf_ids are always trusted */ - if (reg2btf_ids[base_type(reg->type)] && - !bpf_type_has_unsafe_modifiers(reg->type)) + if (reg2btf_ids[base_type(type)] && + !bpf_type_has_unsafe_modifiers(type)) return true; /* If a register is not referenced, it is trusted if it has the @@ -5139,8 +5564,14 @@ static bool is_trusted_reg(struct bpf_verifier_env *env, const struct bpf_reg_st * Eventually, we should make PTR_TRUSTED the single source of truth * for whether a register is trusted. */ - return type_flag(reg->type) & BPF_REG_TRUSTED_MODIFIERS && - !bpf_type_has_unsafe_modifiers(reg->type); + return type_flag(type) & BPF_REG_TRUSTED_MODIFIERS && + !bpf_type_has_unsafe_modifiers(type); +} + +static bool is_trusted_reg(struct bpf_verifier_env *env, + const struct bpf_reg_state *reg) +{ + return is_trusted_reg_type(env, reg, reg->type); } static bool is_rcu_reg(const struct bpf_reg_state *reg) @@ -5313,6 +5744,9 @@ struct bpf_subprog_call_depth_info { int ret_insn; /* caller instruction where we return to. */ int caller; /* caller subprogram idx */ int frame; /* # of consecutive static call stack frames on top of stack */ + int callx_insn; /* callx instruction whose callees are being walked */ + int callx_next; /* next callee of callx_insn to walk */ + bool via_callx; /* the subprogram is entered via callx */ }; /* starting from main bpf function walk all instructions of the function @@ -5332,11 +5766,13 @@ static int check_max_stack_depth_subprog(struct bpf_verifier_env *env, int idx, /* no caller idx */ dinfo[idx].caller = -1; + dinfo[idx].via_callx = false; i = subprog[idx].start; if (!priv_stack_supported) subprog[idx].priv_stack_mode = NO_PRIV_STACK; process_func: + dinfo[idx].callx_insn = -1; if (subprog[idx].has_ld_abs) { for (tmp = idx; tmp >= 0; tmp = dinfo[tmp].caller) { if (subprog[tmp].is_cb) { @@ -5389,7 +5825,7 @@ static int check_max_stack_depth_subprog(struct bpf_verifier_env *env, int idx, if (subprog[idx].priv_stack_mode == PRIV_STACK_ADAPTIVE) { if (subprog_depth > env->max_stack_depth) env->max_stack_depth = subprog_depth; - if (subprog_depth > MAX_BPF_STACK) { + if (subprog_depth > env->stack_limit) { verbose(env, "stack size of subprog %d is %d. Too large\n", idx, subprog_depth); return -EACCES; @@ -5398,7 +5834,7 @@ static int check_max_stack_depth_subprog(struct bpf_verifier_env *env, int idx, depth += subprog_depth; if (depth > env->max_stack_depth) env->max_stack_depth = depth; - if (depth > MAX_BPF_STACK) { + if (depth > env->stack_limit) { total = 0; for (tmp = idx; tmp >= 0; tmp = dinfo[tmp].caller) total++; @@ -5413,11 +5849,10 @@ static int check_max_stack_depth_subprog(struct bpf_verifier_env *env, int idx, for (; i < subprog_end; i++) { int next_insn, sidx; - if (bpf_pseudo_kfunc_call(insn + i) && !insn[i].off) { + if (bpf_pseudo_kfunc_call(insn + i) && !insn[i].off && + bpf_is_throw_kfunc(insn + i)) { bool err = false; - if (!bpf_is_throw_kfunc(insn + i)) - continue; for (tmp = idx; tmp >= 0 && !err; tmp = dinfo[tmp].caller) { if (subprog[tmp].is_cb) { err = true; @@ -5432,31 +5867,70 @@ static int check_max_stack_depth_subprog(struct bpf_verifier_env *env, int idx, return -EINVAL; } - if (!bpf_pseudo_call(insn + i) && !bpf_pseudo_func(insn + i)) - continue; - /* remember insn and function to return to */ - - /* find the callee */ - next_insn = i + insn[i].imm + 1; - sidx = bpf_find_subprog(env, next_insn); - if (verifier_bug_if(sidx < 0, env, "callee not found at insn %d", next_insn)) - return -EFAULT; - if (subprog[sidx].is_async_cb) { - /* async callbacks don't increase bpf prog stack size unless called directly */ - if (!bpf_pseudo_call(insn + i)) - continue; - if (subprog[sidx].is_exception_cb) { - verbose(env, "insn %d cannot call exception cb directly", i); - return -EINVAL; + if (bpf_is_callx(insn + i) || bpf_calls_callback(env, i)) { + /* + * Walk the callees and the callbacks recorded by + * the main verification pass one by one, returning to + * this insn after each. + */ + if (dinfo[idx].callx_insn != i) { + dinfo[idx].callx_insn = i; + dinfo[idx].callx_next = 0; } + sidx = next_callx_callee(env, idx, dinfo[idx].callx_next); + if (sidx < 0) + continue; + dinfo[idx].callx_next = sidx + 1; + dinfo[idx].ret_insn = i; + next_insn = subprog[sidx].start; + } else if (bpf_pseudo_call(insn + i) || bpf_pseudo_func(insn + i)) { + /* find the callee */ + next_insn = i + insn[i].imm + 1; + sidx = bpf_find_subprog(env, next_insn); + if (verifier_bug_if(sidx < 0, env, "callee not found at insn %d", next_insn)) + return -EFAULT; + if (subprog[sidx].is_async_cb) { + /* async callbacks don't increase bpf prog stack size unless called directly */ + if (!bpf_pseudo_call(insn + i)) + continue; + if (subprog[sidx].is_exception_cb) { + verbose(env, "insn %d cannot call exception cb directly", i); + return -EINVAL; + } + } + /* remember insn to return to */ + dinfo[idx].ret_insn = i + 1; + } else { + continue; + } + + /* + * sort_subprogs_topo() tolerates cycles in the call graph that + * go through the address of a function being taken, since it + * doesn't know what it is taken for. Such cycle is a recursion + * unless it's an async callback, which are skipped above. + * The main verification pass limits the depth of the recursion, + * but it doesn't follow calls of global functions. + */ + for (tmp = idx; tmp >= 0; tmp = dinfo[tmp].caller) { + if (tmp != sidx) + continue; + verbose(env, "recursive call from %s() to %s()\n", + bpf_subprog_name(env, idx), bpf_subprog_name(env, sidx)); + bpf_diag_program_structure( + env, i, "recursive subprogram call", + "Rewrite the recursion as an explicit bounded loop, or split the logic so subprogram calls do not form a cycle.", + "This call or callback would make the subprogram call graph recursive. " + "The verifier requires a finite, acyclic call graph so it can bound stack depth and analysis."); + return -EINVAL; } /* store caller info for after we return from callee */ dinfo[idx].frame = frame; - dinfo[idx].ret_insn = i + 1; /* push caller idx into callee's dinfo */ dinfo[sidx].caller = idx; + dinfo[sidx].via_callx = bpf_is_callx(insn + i); i = next_insn; @@ -5490,6 +5964,14 @@ static int check_max_stack_depth_subprog(struct bpf_verifier_env *env, int idx, verbose(env, "tail_calls are not allowed in programs with stack args\n"); return -EINVAL; } + /* + * JITs pass tail call counter in a register that is + * not available when the callee is called via callx. + */ + if (dinfo[tmp].via_callx) { + verbose(env, "tail_calls are not allowed in functions called via callx\n"); + return -EINVAL; + } subprog[tmp].tail_call_reachable = true; } } else if (!idx && subprog[0].has_tail_call && subprog[0].stack_arg_cnt) { @@ -5625,9 +6107,10 @@ static int check_buffer_access(struct bpf_verifier_env *env, const struct bpf_reg_state *reg, argno_t argno, int off, int size, bool zero_size_allowed, - u32 *max_access) + u32 *max_access, bool write) { const char *buf_info = type_is_rdonly_mem(reg->type) ? "rdonly" : "rdwr"; + struct bpf_prog_aux *aux = env->prog->aux; u32 access_end; int err; @@ -5636,6 +6119,11 @@ static int check_buffer_access(struct bpf_verifier_env *env, return err; *max_access = max(access_end, *max_access); + if (write && size) { + if (!aux->max_rdwr_write || access_end - size < aux->min_rdwr_write) + aux->min_rdwr_write = access_end - size; + aux->max_rdwr_write = max(access_end, aux->max_rdwr_write); + } return 0; } @@ -5643,6 +6131,7 @@ static int check_buffer_access(struct bpf_verifier_env *env, /* BPF architecture zero extends alu32 ops into 64-bit registesr */ static void zext_32_to_64(struct bpf_reg_state *reg) { + reg_step_check_unsigned(reg, 32); reg->var_off = tnum_subreg(reg->var_off); reg_set_urange64(reg, reg_u32_min(reg), reg_u32_max(reg)); } @@ -5652,13 +6141,14 @@ static void zext_32_to_64(struct bpf_reg_state *reg) */ static void coerce_reg_to_size(struct bpf_reg_state *reg, int size) { - u64 mask; + u64 mask = (1ULL << (size * 8)) - 1; + + reg_step_check_unsigned(reg, size * 8); /* clear high bits in bit representation */ reg->var_off = tnum_cast(reg->var_off, size); /* fix arithmetic bounds */ - mask = ((u64)1 << (size * 8)) - 1; if ((reg_umin(reg) & ~mask) == (reg_umax(reg) & ~mask)) reg_set_urange64(reg, reg_umin(reg) & mask, reg_umax(reg) & mask); else @@ -5695,6 +6185,14 @@ static void coerce_reg_to_size_sx(struct bpf_reg_state *reg, int size) s64 init_s64_max, init_s64_min, s64_max, s64_min, u64_cval; u64 top_smax_value, top_smin_value; u64 num_bits = size * 8; + s64 limit = 1LL << (num_bits - 1); + + /* If every value fits the signed N-bit range sign extension is a noop. */ + if (reg_smin(reg) >= -limit && reg_smax(reg) < limit) + return; + + /* See commen at reg_step_check_unsigned(). */ + reg_step_reset(reg); if (tnum_is_const(reg->var_off)) { u64_cval = reg->var_off.value; @@ -5761,6 +6259,16 @@ static void coerce_subreg_to_size_sx(struct bpf_reg_state *reg, int size) u32 top_smax_value, top_smin_value; u32 num_bits = size * 8; + /* + * If value fits the non-negative half of the signed N-bit range, + * sign extension is a noop. + */ + if (reg_umax(reg) < (1ULL << (num_bits - 1))) + return; + + /* See comment at reg_step_check_unsigned(). */ + reg_step_reset(reg); + if (tnum_is_const(reg->var_off)) { u32_val = reg->var_off.value; if (size == 1) @@ -5854,6 +6362,110 @@ int bpf_map_direct_read(struct bpf_map *map, int off, int size, u64 *val, return 0; } +static int cmp_func_ptrs(const void *_a, const void *_b) +{ + const struct bpf_func_ptr *a = _a, *b = _b; + + if (a->map != b->map) + return a->map < b->map ? -1 : 1; + if (a->map_off != b->map_off) + return a->map_off < b->map_off ? -1 : 1; + return 0; +} + +/* Find the first pointer to a function at or after 'off' in the value of 'map' */ +static u32 func_ptr_lower_bound(struct bpf_verifier_env *env, const struct bpf_map *map, u64 off) +{ + u32 l = 0, r = env->func_ptr_cnt, m; + struct bpf_func_ptr *p; + + while (l < r) { + m = l + (r - l) / 2; + p = &env->func_ptrs[m]; + if (p->map < map || (p->map == map && p->map_off < off)) + l = m + 1; + else + r = m; + } + return l; +} + +/* + * Return pointers to functions that overlap with 'size' bytes at offset 'off' + * of the value of 'map' and their number in 'cnt'. + */ +struct bpf_func_ptr *bpf_map_range_func_ptrs(struct bpf_verifier_env *env, + const struct bpf_map *map, + u64 off, u64 size, u32 *cnt) +{ + u32 first, last; + + *cnt = 0; + if (!env->func_ptr_cnt || !size) + return NULL; + + /* a pointer that starts up to 7 bytes before 'off' overlaps too */ + first = func_ptr_lower_bound(env, map, off >= sizeof(u64) ? off - sizeof(u64) + 1 : 0); + last = func_ptr_lower_bound(env, map, off + size); + if (first >= last) + return NULL; + + *cnt = last - first; + return &env->func_ptrs[first]; +} + +/* Return all pointers to functions in the value of 'map' */ +struct bpf_func_ptr *bpf_map_func_ptrs(struct bpf_verifier_env *env, + const struct bpf_map *map, u32 *cnt) +{ + return bpf_map_range_func_ptrs(env, map, 0, (u64)map->value_size, cnt); +} + +/* instructions [off, off + len) replaced the instruction at 'off' */ +void bpf_adjust_func_ptrs(struct bpf_verifier_env *env, u32 off, u32 len) +{ + struct bpf_func_ptr *p; + u32 i; + + if (len <= 1) + return; + + for (i = 0; i < env->func_ptr_cnt; i++) { + p = &env->func_ptrs[i]; + if (p->xlated_off <= off || p->xlated_off == BPF_FUNC_PTR_DELETED) + continue; + p->xlated_off += len - 1; + } +} + +/* + * Instructions [off, off + len) are about to be removed. It's called before + * the starts of subprogs are adjusted. A subprog is gone when all of its + * instructions are. Otherwise, e.g. when its first instruction is a nop, + * it starts where the removed instructions did. + */ +void bpf_adjust_func_ptrs_after_remove(struct bpf_verifier_env *env, u32 off, u32 len) +{ + struct bpf_func_ptr *p; + int subprog; + u32 i; + + for (i = 0; i < env->func_ptr_cnt; i++) { + p = &env->func_ptrs[i]; + if (p->xlated_off < off || p->xlated_off == BPF_FUNC_PTR_DELETED) + continue; + if (p->xlated_off >= off + len) { + p->xlated_off -= len; + continue; + } + subprog = bpf_find_subprog(env, p->xlated_off); + if (subprog > 0 && env->subprog_info[subprog + 1].start > off + len) + p->xlated_off = off; + else + p->xlated_off = BPF_FUNC_PTR_DELETED; + } +} + #define BTF_TYPE_SAFE_RCU(__type) __PASTE(__type, __safe_rcu) #define BTF_TYPE_SAFE_RCU_OR_NULL(__type) __PASTE(__type, __safe_rcu_or_null) #define BTF_TYPE_SAFE_TRUSTED(__type) __PASTE(__type, __safe_trusted) @@ -5923,10 +6535,36 @@ BTF_TYPE_SAFE_TRUSTED(struct file) { struct inode *f_inode; }; +/* + * The pointer fields in the sched_ext ops argument containers are pinned by the + * callers for the duration of the ops calls and are never NULL. + */ +BTF_TYPE_SAFE_TRUSTED(struct scx_init_task_args) { +#ifdef CONFIG_EXT_GROUP_SCHED + struct cgroup *cgroup; +#endif +}; + +BTF_TYPE_SAFE_TRUSTED(struct scx_cpu_release_args) { + struct task_struct *task; +}; + +BTF_TYPE_SAFE_TRUSTED(struct scx_sub_attach_args) { + struct sched_ext_ops *ops; +}; + +BTF_TYPE_SAFE_TRUSTED(struct scx_sub_detach_args) { + struct sched_ext_ops *ops; +}; + BTF_TYPE_SAFE_TRUSTED_OR_NULL(struct dentry) { struct inode *d_inode; }; +BTF_TYPE_SAFE_TRUSTED_OR_NULL(struct linux_binprm) { + struct mm_struct *mm; +}; + BTF_TYPE_SAFE_TRUSTED_OR_NULL(struct socket) { struct sock *sk; }; @@ -5967,6 +6605,10 @@ static bool type_is_trusted(struct bpf_verifier_env *env, BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct bpf_iter__task)); BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct linux_binprm)); BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct file)); + BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct scx_init_task_args)); + BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct scx_cpu_release_args)); + BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct scx_sub_attach_args)); + BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct scx_sub_detach_args)); return btf_nested_type_is_trusted(&env->log, reg, field_name, btf_id, "__safe_trusted"); } @@ -5977,12 +6619,52 @@ static bool type_is_trusted_or_null(struct bpf_verifier_env *env, { BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED_OR_NULL(struct socket)); BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED_OR_NULL(struct dentry)); + BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED_OR_NULL(struct linux_binprm)); BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED_OR_NULL(struct vm_area_struct)); return btf_nested_type_is_trusted(&env->log, reg, field_name, btf_id, "__safe_trusted_or_null"); } +/* + * The member holds the kernel address of arena memory, which the rewritten + * load turns into an arena pointer, see arena_member_fixup(). NULL needs no + * tracking, PTR_TO_ARENA covers it. + */ +static int check_arena_member_load(struct bpf_verifier_env *env, struct bpf_reg_state *regs, + int value_regno, enum bpf_type_flag flag, const char *tname, + const char *field_name) +{ + struct bpf_insn_aux_data *aux = &env->insn_aux_data[env->insn_idx]; + struct bpf_insn *insn; + + /* a helper reading the member as bytes needs no rewrite */ + if (value_regno < 0) + return 0; + if (!env->prog->aux->arena) { + verbose(env, + "arena pointer member %s of struct %s requires a program with an associated arena\n", + field_name, tname); + return -EINVAL; + } + /* only a plain load or an acquire load is rewritten */ + insn = &env->prog->insnsi[env->insn_idx]; + if (BPF_CLASS(insn->code) != BPF_LDX && + !(insn->code == (BPF_STX | BPF_ATOMIC | BPF_DW) && insn->imm == BPF_LOAD_ACQ)) { + verbose(env, + "arena pointer member %s of struct %s can only be read with a plain or an acquire load\n", + field_name, tname); + return -EACCES; + } + + aux->arena_member = true; + if (flag & PTR_MAYBE_NULL) + aux->arena_member_nullable = true; + mark_reg_unknown(env, regs, value_regno); + regs[value_regno].type = PTR_TO_ARENA; + return 0; +} + static int check_ptr_to_btf_access(struct bpf_verifier_env *env, struct bpf_reg_state *regs, struct bpf_reg_state *reg, argno_t argno, int off, int size, @@ -5994,6 +6676,7 @@ static int check_ptr_to_btf_access(struct bpf_verifier_env *env, const char *field_name = NULL; enum bpf_type_flag flag = 0; u32 btf_id = 0; + s64 min_off; int ret; if (!env->allow_ptr_leaks) { @@ -6009,36 +6692,31 @@ static int check_ptr_to_btf_access(struct bpf_verifier_env *env, return -EINVAL; } - if (!tnum_is_const(reg->var_off)) { - char tn_buf[48]; - - tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); + if (check_add_overflow(reg_smin(reg), off, &min_off)) { verbose(env, - "%s is ptr_%s invalid variable offset: off=%d, var_off=%s\n", - reg_arg_name(env, argno), tname, off, tn_buf); + "%s is ptr_%s access, offset computation overflows: register's minimal offset is %lld, instruction offset is %d\n", + reg_arg_name(env, argno), tname, reg_smin(reg), off); return -EACCES; } - off += reg->var_off.value; - - if (off < 0) { + if (min_off < 0) { verbose(env, - "%s is ptr_%s invalid negative access: off=%d\n", - reg_arg_name(env, argno), tname, off); + "%s is ptr_%s invalid negative access: off=%lld\n", + reg_arg_name(env, argno), tname, min_off); return -EACCES; } if (reg->type & MEM_USER) { verbose(env, - "%s is ptr_%s access user memory: off=%d\n", - reg_arg_name(env, argno), tname, off); + "%s is ptr_%s access user memory\n", + reg_arg_name(env, argno), tname); return -EACCES; } if (reg->type & MEM_PERCPU) { verbose(env, - "%s is ptr_%s access percpu memory: off=%d\n", - reg_arg_name(env, argno), tname, off); + "%s is ptr_%s access percpu memory\n", + reg_arg_name(env, argno), tname); return -EACCES; } @@ -6048,6 +6726,18 @@ static int check_ptr_to_btf_access(struct bpf_verifier_env *env, } if (env->ops->btf_struct_access && !type_is_alloc(reg->type) && atype == BPF_WRITE) { + if (!tnum_is_const(reg->var_off)) { + char tn_buf[48]; + + tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); + verbose(env, + "%s is ptr_%s invalid variable offset: off=%d, var_off=%s\n", + reg_arg_name(env, argno), tname, off, tn_buf); + return -EACCES; + } + + off += reg->var_off.value; + if (!btf_is_kernel(reg->btf)) { verifier_bug(env, "reg->btf must be kernel btf"); return -EFAULT; @@ -6084,6 +6774,9 @@ static int check_ptr_to_btf_access(struct bpf_verifier_env *env, if (ret < 0) return ret; + if (ret == PTR_TO_ARENA) + return check_arena_member_load(env, regs, value_regno, flag, tname, field_name); + if (ret != PTR_TO_BTF_ID) { /* just mark; */ @@ -6144,6 +6837,16 @@ static int check_ptr_to_btf_access(struct bpf_verifier_env *env, clear_trusted_flags(&flag); } + /* + * A __nullable member only matters where NULL would otherwise be + * assumed away, which is anywhere but on a plain or an untrusted + * pointer: those are probed on a dereference, which tolerates NULL, and + * the BTF_TYPE_SAFE_*_OR_NULL lists never applied to them either. A + * per-CPU pointer counts, per_cpu_ptr() does not check for NULL. + */ + if (bpf_may_fault_on_deref(PTR_TO_BTF_ID | (flag & ~PTR_MAYBE_NULL))) + flag &= ~PTR_MAYBE_NULL; + if (atype == BPF_READ && value_regno >= 0) { ret = mark_btf_ld_reg(env, regs, value_regno, ret, reg->btf, btf_id, flag); if (ret < 0) @@ -6218,10 +6921,11 @@ static int check_ptr_to_map_access(struct bpf_verifier_env *env, return 0; } -/* Check that the stack access at the given offset is within bounds. The +/* + * Check that the stack access at the given offset is within bounds. The * maximum valid offset is -1. * - * The minimum valid offset is -MAX_BPF_STACK for writes, and + * The minimum valid offset is -env->stack_limit for writes, and * -state->allocated_stack for reads. */ static int check_stack_slot_within_bounds(struct bpf_verifier_env *env, @@ -6232,7 +6936,7 @@ static int check_stack_slot_within_bounds(struct bpf_verifier_env *env, int min_valid_off; if (t == BPF_WRITE || env->allow_uninit_stack) - min_valid_off = -MAX_BPF_STACK; + min_valid_off = -(int)env->stack_limit; else min_valid_off = -state->allocated_stack; @@ -6298,10 +7002,19 @@ static int check_stack_access_within_bounds( return err; } + /* + * Every stack access passes through here, including buffers passed to + * helpers and kfuncs and accesses into a caller's frame, so check the + * fastcall contract of the frame that owns the slots once for all of + * them. Zero-sized accesses touch no stack and keep the rewrite enabled. + */ + if (access_size) + check_fastcall_stack_contract(env, state, env->insn_idx, min_off); + /* Note that there is no stack access with offset zero, so the needed stack * size is -min_off, not -min_off+1. */ - return grow_stack_state(env, state, -min_off /* size */); + return bpf_grow_stack_state(env, state, -min_off /* size */); } static bool get_func_retval_range(struct bpf_prog *prog, @@ -6325,6 +7038,7 @@ static void add_scalar_to_reg(struct bpf_reg_state *dst_reg, s64 val) fake_reg.type = SCALAR_VALUE; __mark_reg_known(&fake_reg, val); + scalar_step_add(dst_reg, dst_reg, &fake_reg); scalar32_min_max_add(dst_reg, &fake_reg); scalar_min_max_add(dst_reg, &fake_reg); dst_reg->var_off = tnum_add(dst_reg->var_off, fake_reg.var_off); @@ -6332,12 +7046,147 @@ static void add_scalar_to_reg(struct bpf_reg_state *dst_reg, s64 val) reg_bounds_sync(dst_reg); } +/* there is no support for callx in the interpreter */ +static int require_callx_jit(struct bpf_verifier_env *env) +{ + if (!env->prog->jit_requested) { + verbose(env, "JIT is required to use callx\n"); + return -EOPNOTSUPP; + } + if (!bpf_jit_supports_callx()) { + verbose(env, "JIT doesn't support callx\n"); + return -EOPNOTSUPP; + } + env->prog->jit_required = true; + return 0; +} + +static void mark_reg_func_ptr(struct bpf_verifier_env *env, struct bpf_reg_state *regs, + int regno, int subprog) +{ + mark_reg_known_zero(env, regs, regno); + regs[regno].type = PTR_TO_FUNC; + regs[regno].subprogno = subprog; +} + +/* a read from a table of functions branches into that many states at most */ +#define BPF_MAX_FUNC_PTR_TARGETS 64 +/* and that many pointers at most are next to what is read at variable offset */ +#define BPF_MAX_FUNC_PTR_RANGE 4096 + +/* Count up to 'limit' offsets in [min_off, max_off] that the read is possible at */ +static u32 count_read_offsets(struct tnum offs, u64 min_off, u64 max_off, u32 limit) +{ + u64 o = min_off, next; + u32 cnt = 0; + + if (!tnum_in(offs, tnum_const(o))) { + next = tnum_step(offs, o); + if (next <= o) + return 0; + o = next; + } + while (o <= max_off && cnt < limit) { + cnt++; + next = tnum_step(offs, o); + if (next <= o) + break; + o = next; + } + return cnt; +} + +/* + * A read from a frozen read-only map that has pointers to functions, see + * resolve_func_ptrs(). The bounds and var_off of the offset tell where the read + * is possible at, e.g. var_off is the stride of the elements of an array. + * A read of exactly one pointer yields PTR_TO_FUNC. When the offset is variable + * and only pointers can be read, which is how an element of a table of + * functions is loaded, the verification continues with each of them. Other + * reads that overlap with a pointer are rejected, because their result is not + * known until the program is jitted. + * + * Return -ENOENT if there are no pointers to functions in the bytes that are read. + */ +static int check_func_ptr_read(struct bpf_verifier_env *env, struct bpf_reg_state *reg, int off, + int size, int value_regno) +{ + u64 min_off = reg_umin(reg) + off, max_off = reg_umax(reg) + off; + struct tnum offs = tnum_add(reg->var_off, tnum_const(off)); + struct bpf_reg_state *regs = cur_regs(env); + struct bpf_map *map = reg->map_ptr; + struct bpf_verifier_state *branch; + int subprog, targets[BPF_MAX_FUNC_PTR_TARGETS]; + struct bpf_func_ptr *ptrs; + u32 i, cnt, n = 0; + u64 o, lo, hi; + int err; + + ptrs = bpf_map_range_func_ptrs(env, map, min_off, max_off - min_off + size, &cnt); + if (!ptrs) + return -ENOENT; + if (cnt > BPF_MAX_FUNC_PTR_RANGE) + goto overlap; + + for (i = 0; i < cnt; i++) { + /* the offsets at which the read overlaps with the pointer */ + lo = ptrs[i].map_off >= size ? ptrs[i].map_off - size + 1 : 0; + lo = max(lo, min_off); + hi = min(ptrs[i].map_off + sizeof(u64) - 1, max_off); + for (o = lo; o <= hi; o++) { + if (!tnum_in(offs, tnum_const(o))) + continue; + if (o != ptrs[i].map_off || size != sizeof(u64) || value_regno < 0) + goto overlap; + if (n == BPF_MAX_FUNC_PTR_TARGETS) { + verbose(env, "read from map '%s' may yield more than %d pointers to functions\n", + map->name, BPF_MAX_FUNC_PTR_TARGETS); + return -E2BIG; + } + subprog = bpf_find_subprog(env, ptrs[i].xlated_off); + if (verifier_bug_if(subprog <= 0, env, + "no function at insn %u for map '%s' offset %u", + ptrs[i].xlated_off, map->name, ptrs[i].map_off)) + return -EFAULT; + targets[n++] = subprog; + } + } + /* pointers are next to what is read, e.g. the data of an array of vtables */ + if (!n) + return -ENOENT; + /* either every offset that the read is possible at is a pointer or none */ + if (count_read_offsets(offs, min_off, max_off, n + 1) != n) + goto overlap; + + /* the map has the offset of the function until the program is jitted */ + err = require_callx_jit(env); + if (err) + return err; + + for (i = 0; i < n - 1; i++) { + branch = push_stack(env, env->insn_idx + 1, env->insn_idx, + env->cur_state->speculative); + if (IS_ERR(branch)) + return PTR_ERR(branch); + mark_reg_func_ptr(env, branch->frame[branch->curframe]->regs, value_regno, + targets[i]); + } + mark_reg_func_ptr(env, regs, value_regno, targets[n - 1]); + return 0; + +overlap: + verbose(env, "read of %d bytes at offset [%llu,%llu] of map '%s' overlaps with a pointer to a function\n", + size, min_off, max_off, map->name); + return -EACCES; +} + static int check_map_mem_read(struct bpf_verifier_env *env, struct bpf_reg_state *reg, int off, int bpf_size, int value_regno, bool is_ldsx) { struct bpf_reg_state *regs = cur_regs(env); int size = bpf_size_to_bytes(bpf_size); struct bpf_map *map = reg->map_ptr; + int err; switch (map->map_type) { case BPF_MAP_TYPE_INSN_ARRAY: @@ -6355,13 +7204,18 @@ static int check_map_mem_read(struct bpf_verifier_env *env, struct bpf_reg_state break; } + if (env->func_ptr_cnt) { + err = check_func_ptr_read(env, reg, off, size, value_regno); + if (err != -ENOENT) + return err; + } + /* If map is read-only, track its contents as scalars. */ if (tnum_is_const(reg->var_off) && bpf_map_is_rdonly(map) && map->ops->map_direct_value_addr) { int map_off = off + reg->var_off.value; u64 val = 0; - int err; err = bpf_map_direct_read(map, map_off, size, &val, is_ldsx); if (err) @@ -6388,6 +7242,7 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct b int value_regno, bool strict_alignment_once, bool is_ldsx) { struct bpf_reg_state *regs = cur_regs(env); + enum bpf_reg_type ptr_type = reg->type; int size, err = 0; size = bpf_size_to_bytes(bpf_size); @@ -6513,6 +7368,8 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct b regs[value_regno].btf = info.btf; regs[value_regno].btf_id = info.btf_id; regs[value_regno].id = info.ref_id; + } else if (base_type(info.reg_type) == PTR_TO_MEM) { + regs[value_regno].mem_size = info.mem_size; } if (type_may_be_null(info.reg_type) && !regs[value_regno].id) regs[value_regno].id = ++env->id_gen; @@ -6594,7 +7451,7 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct b } err = check_buffer_access(env, reg, argno, off, size, false, - max_access); + max_access, t == BPF_WRITE); if (!err && value_regno >= 0 && (rdonly_mem || t == BPF_READ)) mark_reg_unknown(env, regs, value_regno); @@ -6635,12 +7492,32 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct b clear_scalar_id(®s[value_regno]); } } + + if (!err && bpf_is_mem_insn(&env->prog->insnsi[insn_idx]) && ptr_type != PTR_TO_STACK) + mark_non_stack_access(env, insn_idx); + return err; } static int save_aux_ptr_type(struct bpf_verifier_env *env, enum bpf_reg_type type, bool allow_trust_mismatch); +/* + * Returns the register to check the access of the current insn with. + * For a number in a program with an arena that is 'arena'. + */ +static struct bpf_reg_state *mem_access_reg(struct bpf_verifier_env *env, int regno, + struct bpf_reg_state *arena) +{ + if (!is_arena_scalar(env, regno)) + return cur_regs(env) + regno; + + memset(arena, 0, sizeof(*arena)); + arena->type = PTR_TO_ARENA; + env->insn_aux_data[env->insn_idx].arena_scalar = true; + return arena; +} + static int check_load_mem(struct bpf_verifier_env *env, struct bpf_insn *insn, bool strict_alignment_once, bool is_ldsx, bool allow_trust_mismatch, const char *ctx) @@ -6648,6 +7525,7 @@ static int check_load_mem(struct bpf_verifier_env *env, struct bpf_insn *insn, struct bpf_verifier_state *vstate = env->cur_state; struct bpf_func_state *state = vstate->frame[vstate->curframe]; struct bpf_reg_state *regs = cur_regs(env); + struct bpf_reg_state arena, *src_reg; enum bpf_reg_type src_reg_type; int err; @@ -6669,16 +7547,28 @@ static int check_load_mem(struct bpf_verifier_env *env, struct bpf_insn *insn, if (err) return err; - src_reg_type = regs[insn->src_reg].type; + src_reg = mem_access_reg(env, insn->src_reg, &arena); + src_reg_type = src_reg->type; /* * check_stack_read_fixed_off() may refine the modification's origin to * the source stack slot. */ bpf_diag_mod_begin(env, ®s[insn->dst_reg], NULL, BPF_DIAG_MOD_WRITE); - err = check_mem_access(env, env->insn_idx, regs + insn->src_reg, argno_from_reg(insn->src_reg), insn->off, - BPF_SIZE(insn->code), BPF_READ, insn->dst_reg, + err = check_mem_access(env, env->insn_idx, src_reg, argno_from_reg(insn->src_reg), + insn->off, BPF_SIZE(insn->code), BPF_READ, insn->dst_reg, strict_alignment_once, is_ldsx); + /* + * Paths through a kernel struct pointer and through other pointers + * may share the insn, see save_aux_ptr_type(). Record a load that did + * not produce an arena pointer from an __arena member, the only load + * through a kernel struct pointer that does, next to the one + * check_arena_member_load() records, so that arena_member_fixup() can + * tell when the paths disagree. + */ + if (!err && !(base_type(src_reg_type) == PTR_TO_BTF_ID && + regs[insn->dst_reg].type == PTR_TO_ARENA)) + env->insn_aux_data[env->insn_idx].non_arena_load = true; err = err ?: save_aux_ptr_type(env, src_reg_type, allow_trust_mismatch); err = err ?: reg_bounds_sanity_check(env, ®s[insn->dst_reg], ctx); @@ -6694,6 +7584,7 @@ static int check_store_reg(struct bpf_verifier_env *env, struct bpf_insn *insn, struct bpf_verifier_state *vstate = env->cur_state; struct bpf_func_state *state = vstate->frame[vstate->curframe]; struct bpf_reg_state *regs = cur_regs(env); + struct bpf_reg_state arena, *dst_reg; enum bpf_reg_type dst_reg_type; int err; @@ -6715,11 +7606,12 @@ static int check_store_reg(struct bpf_verifier_env *env, struct bpf_insn *insn, if (err) return err; - dst_reg_type = regs[insn->dst_reg].type; + dst_reg = mem_access_reg(env, insn->dst_reg, &arena); + dst_reg_type = dst_reg->type; /* Check if (dst_reg + off) is writeable. */ - err = check_mem_access(env, env->insn_idx, regs + insn->dst_reg, argno_from_reg(insn->dst_reg), insn->off, - BPF_SIZE(insn->code), BPF_WRITE, insn->src_reg, + err = check_mem_access(env, env->insn_idx, dst_reg, argno_from_reg(insn->dst_reg), + insn->off, BPF_SIZE(insn->code), BPF_WRITE, insn->src_reg, strict_alignment_once, false); err = err ?: save_aux_ptr_type(env, dst_reg_type, false); @@ -6729,7 +7621,7 @@ static int check_store_reg(struct bpf_verifier_env *env, struct bpf_insn *insn, static int check_atomic_rmw(struct bpf_verifier_env *env, struct bpf_insn *insn) { - struct bpf_reg_state *dst_reg; + struct bpf_reg_state arena, *dst_reg; int load_reg; int err; @@ -6774,6 +7666,9 @@ static int check_atomic_rmw(struct bpf_verifier_env *env, return -EACCES; } + /* load_reg may be dst_reg. Look at dst_reg before it's marked as unknown. */ + dst_reg = mem_access_reg(env, insn->dst_reg, &arena); + load_reg = bpf_atomic_load_reg(insn); if (load_reg >= 0) { /* check and record load of old value */ @@ -6782,8 +7677,6 @@ static int check_atomic_rmw(struct bpf_verifier_env *env, return err; } - dst_reg = cur_regs(env) + insn->dst_reg; - /* Check whether we can read the memory, with second call for fetch * case to simulate the register fill. */ @@ -6910,8 +7803,11 @@ static int check_stack_range_initialized( * but BTF based global subprog validation isn't accurate enough. */ bool allow_poison = access_size < 0 || clobber; - /* The call will initialize the memory; uninitialized stack allowed */ - bool raw_mode = meta && meta->arg_raw_mem.regno == reg_from_argno(argno); + u32 arg_slot = arg_slot_from_argno(argno); + bool uninit = clobber && meta && arg_slot < MAX_BPF_FUNC_ARGS && + (meta->arg_raw_mem.mask & BIT(arg_slot)); + bool raw_mode = uninit && env->allow_uninit_stack && + !(meta->arg_raw_mem.var_size_mask & BIT(arg_slot)); access_size = abs(access_size); @@ -6940,11 +7836,9 @@ static int check_stack_range_initialized( reg_arg_name(env, argno), tn_buf); return -EACCES; } - /* Only initialized buffer on stack is allowed to be accessed - * with variable offset. With uninitialized buffer it's hard to - * guarantee that whole memory is marked as initialized on - * helper return since specific bounds are unknown what may - * cause uninitialized stack leaking. + /* + * The call may touch any byte in the possible range, but does not + * definitely initialize all of it. Fall back to ordinary stack checks. */ raw_mode = false; @@ -6952,12 +7846,14 @@ static int check_stack_range_initialized( max_off = reg_smax(reg) + off; } + /* Unprivileged outputs retain each byte's initialization state. */ if (raw_mode) { - meta->arg_raw_mem.size = access_size; + meta->arg_raw_mem.size[arg_slot] = access_size; return 0; } for (i = min_off; i < max_off + access_size; i++) { + struct bpf_stack_state *ss; u8 *stype; slot = -i - 1; @@ -6967,32 +7863,41 @@ static int check_stack_range_initialized( return -EFAULT; } - stype = &state->stack[spi].slot_type[slot % BPF_REG_SIZE]; + ss = bpf_stack_slot(state, spi); + stype = &ss->slot_type[slot % BPF_REG_SIZE]; if (*stype == STACK_MISC) goto mark; if ((*stype == STACK_ZERO) || - (*stype == STACK_INVALID && env->allow_uninit_stack)) { - if (clobber) { + (*stype == STACK_INVALID && (uninit || env->allow_uninit_stack))) { + if (clobber && (*stype != STACK_INVALID || env->allow_uninit_stack)) { /* helper can write anything into the stack */ *stype = STACK_MISC; } goto mark; } - if (bpf_is_spilled_reg(&state->stack[spi]) && - (state->stack[spi].spilled_ptr.type == SCALAR_VALUE || + /* + * Only the bytes marked STACK_SPILL hold the spilled register. + * The rest of a narrowly spilled slot keeps its previous type + * and must be initialized on its own. + */ + if (*stype == STACK_SPILL && + (ss->spilled_ptr.type == SCALAR_VALUE || env->allow_ptr_leaks)) { if (clobber) { - __mark_reg_unknown(env, &state->stack[spi].spilled_ptr); + __mark_reg_unknown(env, &ss->spilled_ptr); for (j = 0; j < BPF_REG_SIZE; j++) - scrub_spilled_slot(&state->stack[spi].slot_type[j]); + scrub_spilled_slot(&ss->slot_type[j]); } goto mark; } if (*stype == STACK_POISON) { - if (allow_poison) + if (allow_poison) { + if (uninit && env->allow_uninit_stack) + *stype = STACK_MISC; goto mark; + } verbose(env, "reading from stack %s off %d+%d size %d, slot poisoned by dead code elimination\n", reg_arg_name(env, argno), min_off, i - min_off, access_size); } else if (tnum_is_const(reg->var_off)) { @@ -7026,6 +7931,10 @@ static int check_helper_mem_access(struct bpf_verifier_env *env, struct bpf_reg_ switch (base_type(reg->type)) { case PTR_TO_PACKET: case PTR_TO_PACKET_META: + if (!may_access_direct_pkt_data(env, meta, access_type)) { + verbose(env, "function access to the packet is not allowed\n"); + return -EACCES; + } return check_packet_access(env, reg, argno, 0, access_size, zero_size_allowed); case PTR_TO_MAP_KEY: @@ -7066,7 +7975,7 @@ static int check_helper_mem_access(struct bpf_verifier_env *env, struct bpf_reg_ } return check_buffer_access(env, reg, argno, 0, access_size, zero_size_allowed, - max_access); + max_access, access_type == BPF_WRITE); case PTR_TO_STACK: return check_stack_range_initialized( env, reg, @@ -7136,13 +8045,13 @@ static int check_mem_size_reg(struct bpf_verifier_env *env, */ meta->msize_max_value = reg_umax(size_reg); - /* The register is SCALAR_VALUE; the access check happens using - * its boundaries. For unprivileged variable accesses, disable - * raw mode so that the program is required to initialize all - * the memory that the helper could just partially fill up. + /* + * Check variable ranges byte by byte instead of using raw mode. Keep the + * MEM_UNINIT annotation so invalid bytes are accepted without marking them + * initialized when the caller cannot read uninitialized stack memory. */ if (!tnum_is_const(size_reg->var_off)) - meta = NULL; + meta->arg_raw_mem.var_size_mask |= BIT(arg_slot_from_argno(mem_argno)); if (reg_smin(size_reg) < 0) { verbose(env, "%s min value is negative, either use unsigned or 'var &= const'\n", @@ -7185,6 +8094,11 @@ static int check_mem_size_reg(struct bpf_verifier_env *env, return err; } +static bool is_subprog(const struct bpf_call_arg_meta *meta) +{ + return meta->btf && !meta->func_id; +} + static int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, u32 mem_size, enum bpf_access_type access_type, struct bpf_call_arg_meta *meta, bool *known_memory) @@ -7203,10 +8117,11 @@ static int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg } /* - * Only a global subprog (meta == NULL) may read poisoned stack slots: + * Only a global subprog may read poisoned stack slots: * its static stack liveness proved the callee body skips them. */ - size = (!meta && base_type(reg->type) == PTR_TO_STACK) ? -(int)mem_size : mem_size; + size = (is_subprog(meta) && + base_type(reg->type) == PTR_TO_STACK) ? -(int)mem_size : mem_size; if (access_type & BPF_READ) err = check_helper_mem_access(env, reg, argno, size, BPF_READ, true, meta, @@ -7498,6 +8413,9 @@ static int check_map_field_pointer(struct bpf_verifier_env *env, struct bpf_reg_ case BPF_TASK_WORK: field_off = map->record->task_work_off; break; + case BPF_RCU_HEAD: + field_off = map->record->rcu_head_off; + break; case BPF_WORKQUEUE: field_off = map->record->wq_off; break; @@ -7586,11 +8504,11 @@ __printf(6, 7) static void bpf_diag_call_arg_fmt(struct bpf_verifier_env *env, u /* * Validate dynptr arguments for helper, kfunc and subprog. * - * @dynptr is both input and output. It is populated when the argument is - * tagged with MEM_UNINIT (i.e., the dynptr argument that will be constructed) - * and consumed when the argument is expecting to be an initialized dynptr. - * @parent_id is used to track the referenced parent object (e.g., file or skb in - * qdisc program) when constructing a dynptr. + * @meta carries the dynptr and referenced-object state. The dynptr is populated + * when the argument is tagged with MEM_UNINIT (i.e., the dynptr argument that + * will be constructed) and consumed when the argument is expected to be an + * initialized dynptr. The reference tracks the parent object (e.g., file or skb + * in qdisc program) when constructing a dynptr. * * There are two register types representing a bpf_dynptr, one is PTR_TO_STACK * which points to a stack slot, and the other is CONST_PTR_TO_DYNPTR. @@ -7607,24 +8525,11 @@ __printf(6, 7) static void bpf_diag_call_arg_fmt(struct bpf_verifier_env *env, u * and checked dynamically during runtime. */ static int process_dynptr_func(struct bpf_verifier_env *env, struct bpf_reg_state *reg, - argno_t argno, int insn_idx, const char *call_name, - enum bpf_arg_type arg_type, - struct ref_obj_desc *ref_obj, struct bpf_dynptr_desc *dynptr) + argno_t argno, int insn_idx, enum bpf_arg_type arg_type, + struct bpf_call_arg_meta *meta) { int spi, err = 0; - if (reg->type != PTR_TO_STACK && reg->type != CONST_PTR_TO_DYNPTR) { - verbose(env, - "%s expected pointer to stack or const struct bpf_dynptr\n", - reg_arg_name(env, argno)); - bpf_diag_call_arg_fmt( - env, insn_idx, argno, call_name, - "Pass the address of a stack dynptr object, or use a const dynptr pointer returned by the verifier-supported path.", - "a dynptr argument must be a pointer to a dynptr stack slot or a verifier-provided const struct bpf_dynptr, but %s is %s", - reg_arg_name(env, argno), bpf_diag_reg_type_plain(env, reg->type)); - return -EINVAL; - } - /* MEM_UNINIT - Points to memory that is an appropriate candidate for * constructing a mutable bpf_dynptr object. * @@ -7646,7 +8551,7 @@ static int process_dynptr_func(struct bpf_verifier_env *env, struct bpf_reg_stat verbose(env, "Dynptr has to be an uninitialized dynptr\n"); bpf_diag_res( env, insn_idx, "dynptr is already initialized", - "This kfunc constructs a dynptr and requires an uninitialized dynptr stack slot, but the selected slot already holds dynptr state.", + "This function constructs a dynptr and requires an uninitialized dynptr stack slot, but the selected slot already holds dynptr state.", "Use a fresh stack dynptr slot, or release/destroy the existing dynptr before reusing the slot."); return -EINVAL; } @@ -7659,7 +8564,8 @@ static int process_dynptr_func(struct bpf_verifier_env *env, struct bpf_reg_stat return err; } - err = mark_stack_slots_dynptr(env, reg, arg_type, insn_idx, ref_obj, dynptr); + err = mark_stack_slots_dynptr(env, reg, arg_type, insn_idx, + &meta->ref_obj, &meta->dynptr); } else /* OBJ_RELEASE and None case from above */ { /* For the reg->type == PTR_TO_STACK case, bpf_dynptr is never const */ if (reg->type == CONST_PTR_TO_DYNPTR && (arg_type & OBJ_RELEASE)) { @@ -7689,7 +8595,7 @@ static int process_dynptr_func(struct bpf_verifier_env *env, struct bpf_reg_stat verbose(env, "Expected a dynptr of type %s as %s\n", dynptr_type_str(expected_type), reg_arg_name(env, argno)); bpf_diag_call_arg_fmt( - env, insn_idx, argno, call_name, + env, insn_idx, argno, meta->func_name, "Use a dynptr constructor that matches this operation, or call an operation that accepts the dynptr's current type.", "the dynptr is initialized with backing object type %s, but this operation expects dynptr type %s", dynptr_type_str(actual_type), dynptr_type_str(expected_type)); @@ -7705,14 +8611,12 @@ static int process_dynptr_func(struct bpf_verifier_env *env, struct bpf_reg_stat mark_stack_slots_scratched(env, spi, BPF_DYNPTR_NR_SLOTS); - reg = &state->stack[spi].spilled_ptr; + reg = &bpf_stack_slot(state, spi)->spilled_ptr; } - if (dynptr) { - dynptr->type = reg->dynptr.type; - dynptr->id = reg->id; - dynptr->parent_id = reg->parent_id; - } + meta->dynptr.type = reg->dynptr.type; + meta->dynptr.id = reg->id; + meta->dynptr.parent_id = reg->parent_id; } return err; } @@ -7745,12 +8649,12 @@ static bool is_kfunc_arg_iter(struct bpf_call_arg_meta *meta, int arg_idx, return btf_param_match_suffix(meta->btf, arg, "__iter"); } -static int process_iter_arg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, int insn_idx, +static int process_iter_arg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, + u32 arg, argno_t argno, int insn_idx, struct bpf_call_arg_meta *meta) { struct bpf_func_state *state = bpf_func(env, reg); const struct btf_type *t; - u32 arg_idx = arg_idx_from_argno(argno); int spi, err, i, nr_slots, btf_id; if (reg->type != PTR_TO_STACK) { @@ -7770,14 +8674,14 @@ static int process_iter_arg(struct bpf_verifier_env *env, struct bpf_reg_state * * to any kfunc, if arg has "__iter" suffix, we need to be a bit more * conservative here. */ - btf_id = btf_check_iter_arg(meta->btf, meta->func_proto, arg_idx); + btf_id = btf_check_iter_arg(meta->btf, meta->func_proto, arg); if (btf_id < 0) { verbose(env, "expected valid iter pointer as %s\n", reg_arg_name(env, argno)); bpf_diag_call_arg( env, insn_idx, argno, meta->func_name, - "the kfunc expects a recognized iterator state pointer, but this argument does not match a valid iterator type", - "Pass the exact iterator state type expected by this kfunc."); + "the function expects a recognized iterator state pointer, but this argument does not match a valid iterator type", + "Pass the exact iterator state type expected by this function."); return -EINVAL; } t = btf_type_by_id(meta->btf, btf_id); @@ -7840,7 +8744,7 @@ static int process_iter_arg(struct bpf_verifier_env *env, struct bpf_reg_state * /* remember meta->iter info for process_iter_next_call() */ meta->iter.spi = spi; meta->iter.frameno = reg->frameno; - update_ref_obj(&meta->ref_obj, &state->stack[spi].spilled_ptr); + update_ref_obj(&meta->ref_obj, &bpf_stack_slot(state, spi)->spilled_ptr); if (is_iter_destroy_kfunc(meta)) { err = unmark_stack_slots_iter(env, reg, nr_slots); @@ -7917,16 +8821,15 @@ static int widen_imprecise_scalars(struct bpf_verifier_env *env, &fold->regs[i], &fcur->regs[i]); - num_slots = min(fold->allocated_stack / BPF_REG_SIZE, - fcur->allocated_stack / BPF_REG_SIZE); + num_slots = min(bpf_stack_nr_slots(fold), bpf_stack_nr_slots(fcur)); for (i = 0; i < num_slots; i++) { - if (!bpf_is_spilled_reg(&fold->stack[i]) || - !bpf_is_spilled_reg(&fcur->stack[i])) + struct bpf_stack_state *old_ss = bpf_stack_slot(fold, i); + struct bpf_stack_state *cur_ss = bpf_stack_slot(fcur, i); + + if (!bpf_is_spilled_reg(old_ss) || !bpf_is_spilled_reg(cur_ss)) continue; - maybe_widen_reg(env, - &fold->stack[i].spilled_ptr, - &fcur->stack[i].spilled_ptr); + maybe_widen_reg(env, &old_ss->spilled_ptr, &cur_ss->spilled_ptr); } } return 0; @@ -7938,7 +8841,7 @@ static struct bpf_reg_state *get_iter_from_state(struct bpf_verifier_state *cur_ int iter_frameno = meta->iter.frameno; int iter_spi = meta->iter.spi; - return &cur_st->frame[iter_frameno]->stack[iter_spi].spilled_ptr; + return &bpf_stack_slot(cur_st->frame[iter_frameno], iter_spi)->spilled_ptr; } /* process_iter_next_call() is called when verifier gets to iterator's next @@ -8101,9 +9004,52 @@ static bool arg_type_is_dynptr(enum bpf_arg_type type) return base_type(type) == ARG_PTR_TO_DYNPTR; } +/* + * An argument that only ever takes a scalar, so a zero register passed to it + * is a value rather than a NULL pointer. + */ +static bool arg_type_is_scalar(enum bpf_arg_type type) +{ + switch (base_type(type)) { + case ARG_SCALAR: + case ARG_CONST_SCALAR: + case ARG_MEM_SIZE: + case ARG_MEM_SIZE_OR_ZERO: + case ARG_CONST_MEM_SIZE: + case ARG_CONST_ALLOC_SIZE_OR_ZERO: + return true; + default: + return false; + } +} + +/* + * A helper has no BTF and a nonzero function ID. A kfunc has both, while a + * BPF subprogram has BTF and a zero function ID. + */ +static bool is_helper(const struct bpf_call_arg_meta *meta) +{ + return !meta->btf && meta->func_id; +} + +static bool is_helper_call(const struct bpf_call_arg_meta *meta, enum bpf_func_id func_id) +{ + return is_helper(meta) && meta->func_id == func_id; +} + +static bool is_kfunc(const struct bpf_call_arg_meta *meta) +{ + return meta->btf && meta->func_id; +} + +static bool is_kfunc_call(const struct bpf_call_arg_meta *meta, u32 btf_id) +{ + return is_kfunc(meta) && meta->func_id == btf_id; +} + static int resolve_map_arg_type(struct bpf_verifier_env *env, - const struct bpf_call_arg_meta *meta, - enum bpf_arg_type *arg_type) + const struct bpf_call_arg_meta *meta, + enum bpf_arg_type *arg_type) { if (!meta->map.ptr) { /* kernel subsystem misconfigured verifier */ @@ -8122,7 +9068,7 @@ static int resolve_map_arg_type(struct bpf_verifier_env *env, } break; case BPF_MAP_TYPE_BLOOM_FILTER: - if (meta->func_id == BPF_FUNC_map_peek_elem) + if (is_helper_call(meta, BPF_FUNC_map_peek_elem)) *arg_type = ARG_PTR_TO_MAP_VALUE; break; default: @@ -8131,6 +9077,49 @@ static int resolve_map_arg_type(struct bpf_verifier_env *env, return 0; } +static int resolve_func_arg_type(struct bpf_verifier_env *env, + struct bpf_reg_state *reg, u32 arg, argno_t argno, + struct bpf_call_arg_meta *meta, + enum bpf_arg_type *arg_type, u32 *arg_size); +static int process_arg_ptr_to_btf_id(struct bpf_verifier_env *env, struct bpf_reg_state *reg, + argno_t argno, enum bpf_arg_type arg_type, + const struct btf *arg_btf, u32 arg_btf_id, + struct bpf_call_arg_meta *meta, int insn_idx); +static bool is_kfunc_arg_nonown_allowed(const struct btf *btf, + const struct btf_param *arg); +static bool is_kfunc_arg_scalar_with_name(const struct btf *btf, + const struct btf_param *arg, + const char *name); +static bool is_bpf_cast_to_kern_ctx_kfunc(const struct bpf_call_arg_meta *meta); +static bool is_bpf_dynptr_clone_kfunc(const struct bpf_call_arg_meta *meta); +static bool is_bpf_iter_css_task_new_kfunc(const struct bpf_call_arg_meta *meta); +static bool is_kfunc_dynptr_write(struct bpf_call_arg_meta *meta); +static bool is_bpf_obj_drop_kfunc(u32 func_id); +static bool is_bpf_percpu_obj_drop_kfunc(u32 func_id); +static bool is_bpf_rbtree_add_kfunc(u32 func_id); +static int get_bpf_res_spin_lock_kfunc_flags(const struct bpf_call_arg_meta *meta); +static struct bpf_insn_aux_data *cur_aux(const struct bpf_verifier_env *env); +static int process_irq_flag(struct bpf_verifier_env *env, + struct bpf_reg_state *reg, argno_t argno, + struct bpf_call_arg_meta *meta); +static int process_kf_arg_ptr_to_list_head(struct bpf_verifier_env *env, + struct bpf_reg_state *reg, + argno_t argno, + struct bpf_call_arg_meta *meta); +static int process_kf_arg_ptr_to_rbtree_root(struct bpf_verifier_env *env, + struct bpf_reg_state *reg, + argno_t argno, + struct bpf_call_arg_meta *meta); +static int process_kf_arg_ptr_to_list_node(struct bpf_verifier_env *env, + struct bpf_reg_state *reg, + argno_t argno, + struct bpf_call_arg_meta *meta); +static int process_kf_arg_ptr_to_rbtree_node(struct bpf_verifier_env *env, + struct bpf_reg_state *reg, + argno_t argno, + struct bpf_call_arg_meta *meta); +static bool check_css_task_iter_allowlist(struct bpf_verifier_env *env); + struct bpf_reg_types { const enum bpf_reg_type types[10]; u32 *btf_id; @@ -8174,7 +9163,7 @@ static const struct bpf_reg_types mem_types = { }, }; -static const struct bpf_reg_types spin_lock_types = { +static const struct bpf_reg_types map_value_or_alloc_obj_types = { .types = { PTR_TO_MAP_VALUE, PTR_TO_BTF_ID | MEM_ALLOC, @@ -8203,7 +9192,33 @@ static const struct bpf_reg_types percpu_btf_ptr_types = { static const struct bpf_reg_types func_ptr_types = { .types = { PTR_TO_FUNC } }; static const struct bpf_reg_types stack_ptr_types = { .types = { PTR_TO_STACK } }; static const struct bpf_reg_types const_str_ptr_types = { .types = { PTR_TO_MAP_VALUE } }; -static const struct bpf_reg_types timer_types = { .types = { PTR_TO_MAP_VALUE } }; +static const struct bpf_reg_types map_value_types = { .types = { PTR_TO_MAP_VALUE } }; +static const struct bpf_reg_types arena_types = { + .types = { + PTR_TO_ARENA, + SCALAR_VALUE, + } +}; +static const struct bpf_reg_types ctx_out_types = { + .types = { PTR_TO_MEM | MEM_RDONLY | PTR_TRUSTED }, +}; + +static const struct bpf_reg_types alloc_obj_drop_types = { + .types = { + PTR_TO_BTF_ID | MEM_ALLOC, + PTR_TO_BTF_ID | MEM_ALLOC | MEM_PERCPU, + } +}; + +static const struct bpf_reg_types alloc_obj_types = { + .types = { + PTR_TO_BTF_ID | MEM_ALLOC, + PTR_TO_BTF_ID | MEM_ALLOC | MEM_RCU, + PTR_TO_BTF_ID | MEM_ALLOC | NON_OWN_REF, + PTR_TO_BTF_ID | MEM_ALLOC | NON_OWN_REF | MEM_RCU, + } +}; + static const struct bpf_reg_types kptr_xchg_dest_types = { .types = { PTR_TO_MAP_VALUE, @@ -8234,16 +9249,32 @@ static const struct bpf_reg_types *compatible_reg_types[__BPF_ARG_TYPE_MAX] = { #endif [ARG_PTR_TO_SOCKET] = &fullsock_types, [ARG_PTR_TO_BTF_ID] = &btf_ptr_types, - [ARG_PTR_TO_SPIN_LOCK] = &spin_lock_types, + [ARG_PTR_TO_SPIN_LOCK] = &map_value_or_alloc_obj_types, [ARG_PTR_TO_MEM] = &mem_types, [ARG_PTR_TO_RINGBUF_MEM] = &ringbuf_mem_types, [ARG_PTR_TO_PERCPU_BTF_ID] = &percpu_btf_ptr_types, [ARG_PTR_TO_FUNC] = &func_ptr_types, [ARG_PTR_TO_STACK] = &stack_ptr_types, [ARG_PTR_TO_CONST_STR] = &const_str_ptr_types, - [ARG_PTR_TO_TIMER] = &timer_types, + [ARG_PTR_TO_TIMER] = &map_value_types, [ARG_KPTR_XCHG_DEST] = &kptr_xchg_dest_types, [ARG_PTR_TO_DYNPTR] = &dynptr_types, + [ARG_CONST_SCALAR] = &scalar_types, + [ARG_CONST_MEM_SIZE] = &scalar_types, + [ARG_PTR_TO_ALLOC_BTF_ID] = &alloc_obj_drop_types, + [ARG_PTR_TO_REFCOUNTED_KPTR] = &alloc_obj_types, + [ARG_PTR_TO_ITER] = &stack_ptr_types, + [ARG_PTR_TO_LIST_HEAD] = &map_value_or_alloc_obj_types, + [ARG_PTR_TO_LIST_NODE] = &alloc_obj_types, + [ARG_PTR_TO_RB_ROOT] = &map_value_or_alloc_obj_types, + [ARG_PTR_TO_RB_NODE] = &alloc_obj_types, + [ARG_PTR_TO_RES_SPIN_LOCK] = &map_value_or_alloc_obj_types, + [ARG_PTR_TO_WORKQUEUE] = &map_value_types, + [ARG_PTR_TO_TASK_WORK] = &map_value_types, + [ARG_PTR_TO_RCU_HEAD] = &map_value_types, + [ARG_PTR_TO_IRQ_FLAG] = &stack_ptr_types, + [ARG_PTR_TO_ARENA] = &arena_types, + [ARG_PTR_TO_CTX_OUT] = &ctx_out_types, }; static void bpf_diag_call_arg(struct bpf_verifier_env *env, u32 insn_idx, argno_t argno, @@ -8283,6 +9314,71 @@ __printf(6, 7) static void bpf_diag_call_arg_fmt(struct bpf_verifier_env *env, u bpf_diag_call_arg(env, insn_idx, argno, call_name, reason, suggestion); } +static int check_func_arg_nullability(struct bpf_verifier_env *env, + struct bpf_reg_state *reg, u32 arg, argno_t argno, + enum bpf_arg_type arg_type, + struct bpf_call_arg_meta *meta, int insn_idx) +{ + const char *expected_type = "pointer"; + + if (arg_type_is_scalar(arg_type) || type_may_be_null(arg_type) || + (!bpf_register_is_null(reg) && !type_may_be_null(reg->type))) + return 0; + + if (meta->btf) { + u32 arg_btf_id; + + arg_btf_id = btf_params(meta->func_proto)[arg].type; + expected_type = bpf_diag_fmt(env, "value of type %s", + bpf_diag_fmt_btf_type(env, meta->btf, arg_btf_id)); + } + + verbose(env, "Possibly NULL pointer passed to trusted %s\n", + reg_arg_name(env, argno)); + bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name, + "Add a NULL check and make the call only on the non-NULL path.", + "the pointer may be NULL, but this call requires a non-NULL %s", + expected_type); + return -EACCES; +} + +static int check_func_arg_release(struct bpf_verifier_env *env, struct bpf_reg_state *reg, + u32 arg, argno_t argno, enum bpf_arg_type arg_type, + struct bpf_call_arg_meta *meta, int insn_idx) +{ + const char *expected_type = "pointer"; + + if (!arg_type_is_release(arg_type)) + return 0; + + if (arg_type_is_dynptr(arg_type) || reg_is_referenced(env, reg) || + bpf_register_is_null(reg)) + return 0; + + verbose(env, "release function %s expects referenced PTR_TO_BTF_ID passed to %s\n", + meta->func_name, reg_arg_name(env, argno)); + + if (is_kfunc(meta)) { + const struct btf_param *btf_arg; + const struct btf_type *t; + u32 ref_id; + + btf_arg = &btf_params(meta->func_proto)[arg]; + ref_id = btf_arg->type; + t = btf_type_skip_modifiers(meta->btf, btf_arg->type, NULL); + if (btf_type_is_ptr(t)) + btf_type_skip_modifiers(meta->btf, t->type, &ref_id); + expected_type = bpf_diag_fmt(env, "value of type %s", + bpf_diag_fmt_btf_type(env, meta->btf, ref_id)); + } + + bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name, + bpf_diag_fmt(env, "Pass the resource-owning %s returned by the matching acquire call, or avoid the release function after ownership has already been transferred or released.", + expected_type), + "release functions require a value that owns a live resource returned by a matching acquire function"); + return -EINVAL; +} + static const char *bpf_diag_expected_reg_types(struct bpf_verifier_env *env, const enum bpf_reg_type *types, int count) { @@ -8304,19 +9400,21 @@ static const char *bpf_diag_expected_reg_types(struct bpf_verifier_env *env, } static int check_reg_type(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, - enum bpf_arg_type arg_type, const u32 *arg_btf_id, - struct bpf_call_arg_meta *meta, const char *call_name) + enum bpf_arg_type arg_type, struct bpf_call_arg_meta *meta) { enum bpf_reg_type expected, type = reg->type; const struct bpf_reg_types *compatible; const char *actual, *accepted; - int i, j, err; + int i, j; compatible = compatible_reg_types[base_type(arg_type)]; if (!compatible) { verifier_bug(env, "unsupported arg type %d", arg_type); return -EFAULT; } + if (is_kfunc(meta) && base_type(arg_type) == ARG_PTR_TO_BTF_ID && + (base_type(type) == PTR_TO_BTF_ID || reg2btf_ids[base_type(type)])) + goto found; /* ARG_PTR_TO_MEM + RDONLY is compatible with PTR_TO_MEM and PTR_TO_MEM + RDONLY, * but ARG_PTR_TO_MEM is compatible only with PTR_TO_MEM and NOT with PTR_TO_MEM + RDONLY @@ -8336,9 +9434,14 @@ static int check_reg_type(struct bpf_verifier_env *env, struct bpf_reg_state *re type &= ~PTR_MAYBE_NULL; if (base_type(arg_type) == ARG_PTR_TO_MEM) type &= ~DYNPTR_TYPE_FLAG_MASK; + /* Allow allocated memory for BTF-defined ARG_PTR_TO_MEM but not helpers. */ + if (!is_helper(meta) && base_type(arg_type) == ARG_PTR_TO_MEM && + type_is_ptr_alloc_obj(type)) + type = PTR_TO_MEM; /* Local kptr types are allowed as the source argument of bpf_kptr_xchg */ - if (meta->func_id == BPF_FUNC_kptr_xchg && type_is_alloc(type) && reg_from_argno(argno) == BPF_REG_2) { + if (is_helper_call(meta, BPF_FUNC_kptr_xchg) && type_is_alloc(type) && + reg_from_argno(argno) == BPF_REG_2) { type &= ~MEM_ALLOC; type &= ~MEM_PERCPU; } @@ -8358,115 +9461,13 @@ static int check_reg_type(struct bpf_verifier_env *env, struct bpf_reg_state *re verbose(env, "%s\n", reg_type_str(env, compatible->types[j])); actual = bpf_diag_fmt(env, "%s", reg_type_str(env, reg->type)); accepted = bpf_diag_expected_reg_types(env, compatible->types, i); - bpf_diag_call_arg_fmt(env, env->insn_idx, argno, call_name, - "Pass a value with one of the accepted pointer or scalar types for this call.", + bpf_diag_call_arg_fmt(env, env->insn_idx, argno, meta->func_name, + bpf_diag_fmt(env, "Pass %s.", bpf_diag_arg_type_plain(arg_type)), "it has type %s, but this argument accepts %s", actual, accepted); return -EACCES; found: - if (base_type(reg->type) != PTR_TO_BTF_ID) - return 0; - - if (compatible == &mem_types) { - if (!(arg_type & MEM_RDONLY)) { - verbose(env, - "%s() may write into memory pointed by %s type=%s\n", - func_id_name(meta->func_id), - reg_arg_name(env, argno), reg_type_str(env, reg->type)); - return -EACCES; - } - return 0; - } - - switch ((int)reg->type) { - case PTR_TO_BTF_ID: - case PTR_TO_BTF_ID | PTR_TRUSTED: - case PTR_TO_BTF_ID | PTR_TRUSTED | PTR_MAYBE_NULL: - case PTR_TO_BTF_ID | MEM_RCU: - case PTR_TO_BTF_ID | PTR_MAYBE_NULL: - case PTR_TO_BTF_ID | PTR_MAYBE_NULL | MEM_RCU: - { - /* For bpf_sk_release, it needs to match against first member - * 'struct sock_common', hence make an exception for it. This - * allows bpf_sk_release to work for multiple socket types. - */ - bool strict_type_match = arg_type_is_release(arg_type) && - meta->func_id != BPF_FUNC_sk_release; - - if (type_may_be_null(reg->type) && - (!type_may_be_null(arg_type) || arg_type_is_release(arg_type))) { - verbose(env, "Possibly NULL pointer passed to helper %s\n", - reg_arg_name(env, argno)); - bpf_diag_call_arg( - env, env->insn_idx, argno, call_name, - "the pointer may be NULL, but this call requires a non-NULL pointer", - "Add a NULL check and make the call only on the non-NULL path."); - return -EACCES; - } - - if (!arg_btf_id) { - if (!compatible->btf_id) { - verifier_bug(env, "missing arg compatible BTF ID"); - return -EFAULT; - } - arg_btf_id = compatible->btf_id; - } - - if (meta->func_id == BPF_FUNC_kptr_xchg) { - if (map_kptr_match_type(env, meta->kptr_field, reg, reg_from_argno(argno))) - return -EACCES; - } else { - if (arg_btf_id == BPF_PTR_POISON) { - verbose(env, "verifier internal error:"); - verbose(env, "%s has non-overwritten BPF_PTR_POISON type\n", - reg_arg_name(env, argno)); - return -EACCES; - } - - err = __check_ptr_off_reg(env, reg, argno, true); - if (err) - return err; - - if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, - reg->var_off.value, btf_vmlinux, *arg_btf_id, - strict_type_match, !type_is_alloc(reg->type))) { - verbose(env, "%s is of type %s but %s is expected\n", - reg_arg_name(env, argno), - btf_type_name(reg->btf, reg->btf_id), - btf_type_name(btf_vmlinux, *arg_btf_id)); - return -EACCES; - } - } - break; - } - case PTR_TO_BTF_ID | MEM_ALLOC: - case PTR_TO_BTF_ID | MEM_PERCPU | MEM_ALLOC: - case PTR_TO_BTF_ID | MEM_ALLOC | NON_OWN_REF: - case PTR_TO_BTF_ID | MEM_ALLOC | NON_OWN_REF | MEM_RCU: - if (meta->func_id != BPF_FUNC_spin_lock && meta->func_id != BPF_FUNC_spin_unlock && - meta->func_id != BPF_FUNC_kptr_xchg) { - verifier_bug(env, "unimplemented handling of MEM_ALLOC"); - return -EFAULT; - } - /* Check if local kptr in src arg matches kptr in dst arg */ - if (meta->func_id == BPF_FUNC_kptr_xchg) { - int regno = reg_from_argno(argno); - - if (regno == BPF_REG_2 && - map_kptr_match_type(env, meta->kptr_field, reg, regno)) - return -EACCES; - } - break; - case PTR_TO_BTF_ID | MEM_PERCPU: - case PTR_TO_BTF_ID | MEM_PERCPU | MEM_RCU: - case PTR_TO_BTF_ID | MEM_PERCPU | PTR_TRUSTED: - /* Handled by helper specific checks */ - break; - default: - verifier_bug(env, "invalid PTR_TO_BTF_ID register for type match"); - return -EFAULT; - } return 0; } @@ -8487,10 +9488,9 @@ reg_find_field_offset(const struct bpf_reg_state *reg, s32 off, u32 fields) return field; } -static int __check_func_arg_reg_off(struct bpf_verifier_env *env, - const struct bpf_reg_state *reg, argno_t argno, - enum bpf_arg_type arg_type, - bool btf_id_fixed_off_ok) +static int check_func_arg_reg_off(struct bpf_verifier_env *env, + const struct bpf_reg_state *reg, argno_t argno, + enum bpf_arg_type arg_type) { u32 type = reg->type; @@ -8546,12 +9546,15 @@ static int __check_func_arg_reg_off(struct bpf_verifier_env *env, case PTR_TO_BTF_ID | MEM_ALLOC | NON_OWN_REF: case PTR_TO_BTF_ID | MEM_ALLOC | NON_OWN_REF | MEM_RCU: /* When referenced PTR_TO_BTF_ID is passed to release function, - * its fixed offset must be 0. In the other cases, fixed offset - * can be non-zero unless the caller requires otherwise. - * var_off always must be 0 for PTR_TO_BTF_ID, hence we still - * need to do checks instead of returning. + * its fixed offset must be 0. bpf_refcount_acquire() returns the + * pointer it was given while incrementing the refcount at the + * refcount field offset, so it needs a zero offset too. In the + * other cases, fixed offset can be non-zero. var_off always must + * be 0 for PTR_TO_BTF_ID, hence we still need to do checks + * instead of returning. */ - return __check_ptr_off_reg(env, reg, argno, btf_id_fixed_off_ok); + return __check_ptr_off_reg(env, reg, argno, + base_type(arg_type) != ARG_PTR_TO_REFCOUNTED_KPTR); case PTR_TO_CTX: /* * Allow fixed and variable offsets for syscall context, but @@ -8559,7 +9562,7 @@ static int __check_func_arg_reg_off(struct bpf_verifier_env *env, * otherwise we may get modified ctx in tail called programs and * global subprogs (that may act as extension prog hooks). */ - if (arg_type != ARG_PTR_TO_CTX && is_var_ctx_off_allowed(env->prog)) + if (base_type(arg_type) != ARG_PTR_TO_CTX && is_var_ctx_off_allowed(env->prog)) return 0; fallthrough; default: @@ -8567,13 +9570,6 @@ static int __check_func_arg_reg_off(struct bpf_verifier_env *env, } } -static int check_func_arg_reg_off(struct bpf_verifier_env *env, - const struct bpf_reg_state *reg, argno_t argno, - enum bpf_arg_type arg_type) -{ - return __check_func_arg_reg_off(env, reg, argno, arg_type, true); -} - static int check_arg_const_str(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno) { @@ -8582,6 +9578,7 @@ static int check_arg_const_str(struct bpf_verifier_env *env, int map_off; u64 map_addr; char *str_ptr; + u32 cnt; if (reg->type != PTR_TO_MAP_VALUE) return -EINVAL; @@ -8631,6 +9628,11 @@ static int check_arg_const_str(struct bpf_verifier_env *env, verbose(env, "string is not zero-terminated\n"); return -EINVAL; } + /* the bytes of a pointer to a function are not known until the program is jitted */ + if (bpf_map_range_func_ptrs(env, map, map_off, strlen(str_ptr + map_off) + 1, &cnt)) { + verbose(env, "string overlaps with a pointer to a function\n"); + return -EACCES; + } return 0; } @@ -8641,6 +9643,7 @@ static int get_constant_map_key(struct bpf_verifier_env *env, s64 *value) { struct bpf_func_state *state = bpf_func(env, key); + struct bpf_stack_state *ss; struct bpf_reg_state *reg; int slot, spi, off; int spill_size = 0; @@ -8660,7 +9663,8 @@ static int get_constant_map_key(struct bpf_verifier_env *env, slot = -stack_off - 1; spi = slot / BPF_REG_SIZE; off = slot % BPF_REG_SIZE; - stype = state->stack[spi].slot_type; + ss = bpf_stack_slot(state, spi); + stype = ss->slot_type; /* First handle precisely tracked STACK_ZERO */ for (i = off; i >= 0 && stype[i] == STACK_ZERO; i--) @@ -8671,14 +9675,14 @@ static int get_constant_map_key(struct bpf_verifier_env *env, } /* Check that stack contains a scalar spill of expected size */ - if (!bpf_is_spilled_scalar_reg(&state->stack[spi])) + if (!bpf_is_spilled_scalar_reg(ss)) return -EOPNOTSUPP; for (i = off; i >= 0 && stype[i] == STACK_SPILL; i--) spill_size++; if (spill_size != key_size) return -EOPNOTSUPP; - reg = &state->stack[spi].spilled_ptr; + reg = &ss->spilled_ptr; if (!tnum_is_const(reg->var_off)) /* Stack value not statically known */ return -EOPNOTSUPP; @@ -8722,6 +9726,8 @@ static int process_map_ptr_arg(struct bpf_verifier_env *env, struct bpf_reg_stat obj_name = "timer"; else if (rec->task_work_off >= 0) obj_name = "bpf_task_work"; + else if (rec->rcu_head_off >= 0) + obj_name = "bpf_rcu_head"; verbose(env, "%s pointer in %s map_uid=%d ", obj_name, reg_arg_name(env, obj_argno), meta->map.uid); @@ -8735,65 +9741,89 @@ static int process_map_ptr_arg(struct bpf_verifier_env *env, struct bpf_reg_stat return 0; } -static int check_func_arg(struct bpf_verifier_env *env, u32 arg, +/* + * MEM_WRITE alone denotes an input/output buffer. MEM_UNINIT marks an output + * whose incoming contents are not read. + */ +static enum bpf_access_type func_arg_access_type(enum bpf_arg_type arg_type) +{ + if (!(arg_type & MEM_WRITE)) + return BPF_READ; + if (arg_type & MEM_UNINIT) + return BPF_WRITE; + return BPF_READ | BPF_WRITE; +} + +static int check_func_arg(struct bpf_verifier_env *env, u32 arg, u32 slot, u32 prev_slot, struct bpf_call_arg_meta *meta, int insn_idx) { + const struct btf_param *btf_arg = is_kfunc(meta) ? + &btf_params(meta->func_proto)[arg] : NULL; const struct bpf_func_proto *fn = meta->fn; - u32 regno = BPF_REG_1 + arg; - struct bpf_reg_state *reg = reg_state(env, regno); + struct bpf_func_state *caller = cur_func(env); + struct bpf_reg_state *regs = cur_regs(env); + argno_t argno = argno_from_arg(slot + 1); + struct bpf_reg_state *reg = get_func_arg_reg(caller, regs, slot); enum bpf_arg_type arg_type = fn->arg_type[arg]; - argno_t argno = argno_from_reg(regno); - enum bpf_reg_type type = reg->type; - u32 *arg_btf_id = NULL; + int regno = reg_from_argno(argno); + u32 arg_size = arg_type & MEM_FIXED_SIZE ? fn->arg_size[arg] : 0; u32 key_size; int err = 0; - if (arg_type == ARG_DONTCARE) + if (arg_type == ARG_PTR_TO_PROG_AUX) { + cur_aux(env)->arg_prog = regno; + return 0; + } + + if (arg_type == ARG_IGNORE) return 0; - err = check_reg_arg(env, regno, SRC_OP); - if (err) - return err; + if (regno >= 0) { + err = check_reg_arg(env, regno, SRC_OP); + if (err) + return err; + } + /* Preserve the legacy helper behavior for privileged pointer leaks. */ if (arg_type == ARG_ANYTHING) { - if (is_pointer_value(env, regno)) { - verbose(env, "R%d leaks addr into helper function\n", - regno); + if (__is_pointer_value(env->allow_ptr_leaks, reg)) { + verbose(env, "%s leaks addr into helper function\n", + reg_arg_name(env, argno)); return -EACCES; } return 0; } - if (type_is_pkt_pointer(type) && - !may_access_direct_pkt_data(env, fn, BPF_READ)) { - verbose(env, "helper access to the packet is not allowed\n"); + err = resolve_func_arg_type(env, reg, arg, argno, meta, &arg_type, &arg_size); + if (err) + return err; + + /* + * A slice of a packet is packet data under another name. A helper which + * may reallocate the packet must not go on reading from the old one. + */ + if (is_helper(meta) && reg_is_dynptr_slice_pkt(reg) && + bpf_helper_changes_pkt_data(meta->func_id)) { + verbose(env, "function access to the packet is not allowed\n"); return -EACCES; } - if (base_type(arg_type) == ARG_PTR_TO_MAP_VALUE) { - err = resolve_map_arg_type(env, meta, &arg_type); - if (err) - return err; - } + if (arg_type_is_raw_mem(arg_type)) + meta->arg_raw_mem.mask |= BIT(slot); if (bpf_register_is_null(reg) && type_may_be_null(arg_type)) { - /* A NULL register has a SCALAR_VALUE type, so skip - * type checking. - */ - err = mark_chain_precision(env, regno); + err = mark_arg_precision(env, argno); if (err) return err; - goto skip_type_check; + return 0; } - /* arg_btf_id and arg_size are in a union. */ - if (base_type(arg_type) == ARG_PTR_TO_BTF_ID || - base_type(arg_type) == ARG_PTR_TO_SPIN_LOCK) - arg_btf_id = fn->arg_btf_id[arg]; + err = check_func_arg_nullability(env, reg, arg, argno, arg_type, meta, insn_idx); + if (err) + return err; - err = check_reg_type(env, reg, argno, arg_type, arg_btf_id, meta, - func_id_name(meta->func_id)); + err = check_reg_type(env, reg, argno, arg_type, meta); if (err) return err; @@ -8801,22 +9831,27 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, if (err) return err; -skip_type_check: - if (arg_type_is_release(arg_type) && !arg_type_is_dynptr(arg_type) && - !reg_is_referenced(env, reg) && !bpf_register_is_null(reg)) { - verbose(env, "release helper %s expects referenced PTR_TO_BTF_ID passed to %s\n", - func_id_name(meta->func_id), reg_arg_name(env, argno)); - bpf_diag_call_arg( - env, insn_idx, argno, func_id_name(meta->func_id), - "release helpers require a value that owns a live resource returned by a matching acquire helper", - "Pass the resource-owning pointer returned by the matching acquire helper, and avoid calling the release helper after ownership has already been transferred or released."); - return -EINVAL; - } + err = check_func_arg_release(env, reg, arg, argno, arg_type, meta, insn_idx); + if (err) + return err; if (reg_is_referenced(env, reg)) update_ref_obj(&meta->ref_obj, reg); switch (base_type(arg_type)) { + case ARG_CONST_SCALAR: + err = process_const_arg(env, reg, argno, meta); + if (err < 0) { + if (err == -EINVAL) + bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name, + "Pass a compile-time constant or a value the verifier can prove is constant at this call.", + "the function requires this scalar argument to be a verifier-known constant, but %s is variable on this path", + reg_arg_name(env, argno)); + return err; + } + break; + case ARG_SCALAR: + break; case ARG_CONST_MAP_PTR: /* bpf_map_xxx(map_ptr) call: remember that map_ptr */ err = process_map_ptr_arg(env, reg, argno, meta); @@ -8838,7 +9873,7 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, return -EFAULT; } key_size = meta->map.ptr->key_size; - err = check_helper_mem_access(env, reg, argno, key_size, BPF_READ, false, NULL, + err = check_helper_mem_access(env, reg, argno, key_size, BPF_READ, false, meta, NULL); if (err) return err; @@ -8866,21 +9901,86 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, return -EFAULT; } - /* - * Disable raw mode for bpf_map_peek_elem() on a bloom filter. The helper reads - * the value buffer as an input rather than filling it. - */ - if (meta->func_id == BPF_FUNC_map_peek_elem && - meta->map.ptr->map_type == BPF_MAP_TYPE_BLOOM_FILTER) - meta->arg_raw_mem.regno = 0; - err = check_helper_mem_access(env, reg, argno, meta->map.ptr->value_size, arg_type & MEM_WRITE ? BPF_WRITE : BPF_READ, false, meta, NULL); break; + case ARG_PTR_TO_BTF_ID: + case ARG_PTR_TO_BTF_ID_SOCK_COMMON: + { + const u32 *arg_btf_id = fn->arg_btf_id[arg]; + const struct btf *arg_btf = is_kfunc(meta) ? meta->btf : btf_vmlinux; + enum bpf_reg_type reg_type; + + if (!meta->btf) { + const struct bpf_reg_types *compatible; + + if (base_type(reg->type) != PTR_TO_BTF_ID) + break; + + if (is_helper_call(meta, BPF_FUNC_kptr_xchg)) + return map_kptr_match_type(env, meta->kptr_field, reg, regno) ? + -EACCES : 0; + + if (!arg_btf_id) { + compatible = compatible_reg_types[base_type(arg_type)]; + if (!compatible->btf_id) { + verifier_bug(env, "missing arg compatible BTF ID"); + return -EFAULT; + } + arg_btf_id = compatible->btf_id; + } + if (arg_btf_id == BPF_PTR_POISON) { + verbose(env, "verifier internal error:"); + verbose(env, "%s has non-overwritten BPF_PTR_POISON type\n", + reg_arg_name(env, argno)); + return -EACCES; + } + } + + reg_type = reg->type & ~(arg_type & PTR_MAYBE_NULL); + if ((arg_type & PTR_TRUSTED) && + (!is_trusted_reg_type(env, reg, reg_type) || + bpf_type_has_unsafe_modifiers(reg_type))) { + if (!(arg_type & MEM_RCU)) { + const char *actual_type, *arg_name, *expected_type; + + expected_type = bpf_diag_fmt_btf_type(env, arg_btf, *arg_btf_id); + verbose(env, "%s must be referenced or trusted\n", + reg_arg_name(env, argno)); + arg_name = reg_arg_name(env, argno); + actual_type = bpf_diag_reg_type_plain(env, reg->type); + bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name, + "Pass a pointer acquired from a verifier-tracked source, or call this function only inside the required protection if it accepts RCU pointers.", + "the function requires a trusted or resource-owning pointer to %s, but %s is %s", + expected_type, arg_name, actual_type); + return -EINVAL; + } + if (!is_rcu_reg(reg)) { + const char *actual_type, *arg_name, *expected_type; + + expected_type = bpf_diag_fmt_btf_type(env, arg_btf, *arg_btf_id); + verbose(env, "%s must be a rcu pointer\n", + reg_arg_name(env, argno)); + arg_name = reg_arg_name(env, argno); + actual_type = bpf_diag_reg_type_plain(env, reg->type); + bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name, + "Use this function with a pointer that is valid in an RCU read lock region.", + "the function requires an RCU-protected pointer to %s, but %s is %s", + expected_type, arg_name, actual_type); + return -EINVAL; + } + } + + err = process_arg_ptr_to_btf_id(env, reg, argno, arg_type, arg_btf, + *arg_btf_id, meta, insn_idx); + if (err < 0) + return err; + break; + } case ARG_PTR_TO_PERCPU_BTF_ID: if (!reg->btf_id) { - verbose(env, "Helper has invalid btf_id in R%d\n", regno); + verbose(env, "Helper has invalid btf_id in %s\n", reg_arg_name(env, argno)); return -EACCES; } meta->ret_btf = reg->btf; @@ -8891,11 +9991,11 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, verbose(env, "can't spin_{lock,unlock} in rbtree cb\n"); return -EACCES; } - if (meta->func_id == BPF_FUNC_spin_lock) { + if (is_helper_call(meta, BPF_FUNC_spin_lock)) { err = process_spin_lock(env, reg, argno, PROCESS_SPIN_LOCK); if (err) return err; - } else if (meta->func_id == BPF_FUNC_spin_unlock) { + } else if (is_helper_call(meta, BPF_FUNC_spin_unlock)) { err = process_spin_lock(env, reg, argno, 0); if (err) return err; @@ -8909,45 +10009,283 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, if (err) return err; break; + case ARG_PTR_TO_CTX: + if (is_bpf_cast_to_kern_ctx_kfunc(meta)) { + err = get_kern_ctx_btf_id(&env->log, resolve_prog_type(env->prog)); + if (err < 0) + return -EINVAL; + meta->ret_btf_id = err; + } + break; + case ARG_PTR_TO_ARENA: + break; + case ARG_PTR_TO_ALLOC_BTF_ID: + if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC)) { + if (!is_bpf_obj_drop_kfunc(meta->func_id)) { + verbose(env, "%s expected for bpf_obj_drop()\n", + reg_arg_name(env, argno)); + return -EINVAL; + } + } else if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC | MEM_PERCPU)) { + if (!is_bpf_percpu_obj_drop_kfunc(meta->func_id)) { + verbose(env, "%s expected for bpf_percpu_obj_drop()\n", + reg_arg_name(env, argno)); + return -EINVAL; + } + } + if (!reg_is_referenced(env, reg)) { + verbose(env, "allocated object must be referenced\n"); + bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name, + "Pass the owned object pointer before it is released or transferred.", + "the allocated object pointer in %s must still carry verifier-tracked ownership, but this pointer no longer owns a live resource", + reg_arg_name(env, argno)); + return -EINVAL; + } + if (meta->btf == btf_vmlinux) { + meta->arg_btf = reg->btf; + meta->arg_btf_id = reg->btf_id; + } + break; case ARG_PTR_TO_FUNC: meta->subprogno = reg->subprogno; break; case ARG_PTR_TO_MEM: + { + enum bpf_access_type access_type; + bool known_memory; + /* The access to this pointer is only checked when we hit the * next is_mem_size argument below. */ - if (arg_type & MEM_FIXED_SIZE) { - err = check_mem_reg(env, reg, argno_from_reg(regno), fn->arg_size[arg], - arg_type & MEM_WRITE ? BPF_WRITE : BPF_READ, meta, NULL); - if (err) - return err; - if (arg_type & MEM_ALIGNED) - err = check_ptr_alignment(env, reg, 0, fn->arg_size[arg], true); + if (!(arg_type & MEM_FIXED_SIZE)) + break; + + access_type = func_arg_access_type(arg_type); + + err = check_mem_reg(env, reg, argno, arg_size, access_type, meta, &known_memory); + if (err < 0) { + if (known_memory) + bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name, + "Pass memory with at least the required number of accessible bytes and suitable read or write access.", + "the function expects %u bytes of memory, but the verifier cannot prove that %s provides a range of that size with the required read or write access", + arg_size, + bpf_diag_reg_type_plain(env, reg->type)); + else + bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name, + "Pass stack, map, context, or other verifier-known memory of the expected type and size, not an integer cast to a pointer.", + "the function expects %u bytes of memory, but it is %s and not verifier-known memory", + arg_size, + bpf_diag_reg_type_plain(env, reg->type)); + return err; + } + /* + * PTR_TO_PACKET gets passed as PTR_TO_MEM, preventing us from adjusting + * bounds tracking information. + */ + if (is_subprog(meta) && meta->subprog_may_change_pkt && + (reg_is_pkt_pointer_any(reg) || reg_is_dynptr_slice_pkt(reg))) { + verbose(env, + "cannot pass packet pointer %s to %s(): function may change packet data\n", + reg_arg_name(env, argno), meta->func_name); + return -EINVAL; + } + if (arg_type & MEM_ALIGNED) + err = check_ptr_alignment(env, reg, 0, arg_size, true); + break; + } + case ARG_CONST_MEM_SIZE: + err = process_const_arg(env, reg, argno, meta); + if (err < 0) { + if (err == -EINVAL) + bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name, + "Pass a compile-time constant or a value the verifier can prove is constant at this call.", + "the function requires this memory size to be a verifier-known constant, but %s is variable on this path", + reg_arg_name(env, argno)); + return err; + } + fallthrough; + case ARG_MEM_SIZE: + case ARG_MEM_SIZE_OR_ZERO: + { + struct bpf_reg_state *buff_reg = get_func_arg_reg(caller, regs, prev_slot); + argno_t buff_argno = argno_from_arg(prev_slot + 1); + enum bpf_mem_size_failure failure; + const char *buff_arg, *size_arg; + bool zero_size_allowed; + u32 access_type; + + if (meta->btf && bpf_register_is_null(buff_reg)) + break; + + access_type = func_arg_access_type(fn->arg_type[arg - 1]); + + zero_size_allowed = meta->btf || base_type(arg_type) == ARG_MEM_SIZE_OR_ZERO; + + err = check_mem_size_reg(env, buff_reg, reg, buff_argno, argno, + access_type, zero_size_allowed, meta, &failure); + if (!err) + break; + + buff_arg = bpf_diag_arg_name(env, buff_argno); + size_arg = bpf_diag_arg_name(env, argno); + verbose(env, "%s and ", reg_arg_name(env, buff_argno)); + verbose(env, "%s memory, len pair leads to invalid memory access\n", + reg_arg_name(env, argno)); + if (failure == BPF_MEM_SIZE_FAIL_MEMORY) { + bpf_diag_call_arg_fmt(env, insn_idx, buff_argno, meta->func_name, + "Pass a stack, map, context, or other verifier-known memory pointer, and keep the paired length within that object.", + "it is the memory pointer in a memory/length pair with %s, but %s does not provide a verifier-accessible range of the requested length", + size_arg, buff_arg); + } else if (failure == BPF_MEM_SIZE_FAIL_SIZE) { + if (reg_smin(reg) < 0) + bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name, + "Constrain the memory size to a non-negative value smaller than BPF_MAX_VAR_SIZ before this call.", + "the memory size in %s may be negative because its signed minimum is %lld", + size_arg, reg_smin(reg)); + else if (!zero_size_allowed && reg_umin(reg) == 0) + bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name, + "Ensure the memory size is non-zero before this call.", + "the memory size in %s may be zero, but the function requires a non-zero size", + size_arg); + else + bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name, + "Constrain the memory size to a non-negative value smaller than BPF_MAX_VAR_SIZ before this call.", + "the memory size in %s may reach %llu bytes, but variable memory accesses must stay below %u bytes", + size_arg, reg_umax(reg), BPF_MAX_VAR_SIZ); } break; - case ARG_MEM_SIZE: - err = check_mem_size_reg(env, reg_state(env, regno - 1), reg, - argno_from_reg(regno - 1), argno, - fn->arg_type[arg - 1] & MEM_WRITE ? BPF_WRITE : BPF_READ, - false, meta, NULL); - break; - case ARG_MEM_SIZE_OR_ZERO: - err = check_mem_size_reg(env, reg_state(env, regno - 1), reg, - argno_from_reg(regno - 1), argno, - fn->arg_type[arg - 1] & MEM_WRITE ? BPF_WRITE : BPF_READ, - true, meta, NULL); - break; - case ARG_PTR_TO_DYNPTR: - err = process_dynptr_func(env, reg, argno, insn_idx, func_id_name(meta->func_id), - arg_type, &meta->ref_obj, &meta->dynptr); + } + case ARG_PTR_TO_DYNPTR: { + if (is_bpf_dynptr_clone_kfunc(meta) && + (arg_type & MEM_UNINIT)) { + enum bpf_dynptr_type parent_type = meta->dynptr.type; + + if (parent_type == BPF_DYNPTR_TYPE_INVALID) { + verifier_bug(env, "no dynptr type for parent of clone"); + return -EFAULT; + } + + arg_type |= (unsigned int)get_dynptr_type_flag(parent_type); + } + + err = process_dynptr_func(env, reg, argno, insn_idx, arg_type, meta); if (err) return err; + if (!arg && is_kfunc_dynptr_write(meta)) + meta->dynptr_write_type = meta->dynptr.type; + break; + } + case ARG_PTR_TO_ITER: + if (is_bpf_iter_css_task_new_kfunc(meta) && + !check_css_task_iter_allowlist(env)) { + verbose(env, "css_task_iter is only allowed in bpf_lsm, bpf_iter and sleepable progs\n"); + return -EINVAL; + } + err = process_iter_arg(env, reg, arg, argno, insn_idx, meta); + if (err < 0) + return err; + break; + case ARG_PTR_TO_LIST_HEAD: + if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC) && + !reg_is_referenced(env, reg)) { + verbose(env, "allocated object must be referenced\n"); + return -EINVAL; + } + err = process_kf_arg_ptr_to_list_head(env, reg, argno, meta); + if (err < 0) + return err; + break; + case ARG_PTR_TO_RB_ROOT: + if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC) && + !reg_is_referenced(env, reg)) { + verbose(env, "allocated object must be referenced\n"); + return -EINVAL; + } + err = process_kf_arg_ptr_to_rbtree_root(env, reg, argno, meta); + if (err < 0) + return err; + break; + case ARG_PTR_TO_LIST_NODE: + if (!(is_kfunc_arg_nonown_allowed(meta->btf, btf_arg) && + type_is_non_owning_ref(reg->type) && !reg_is_referenced(env, reg))) { + if (reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) { + verbose(env, "%s expected pointer to allocated object\n", + reg_arg_name(env, argno)); + return -EINVAL; + } + if (!reg_is_referenced(env, reg)) { + verbose(env, "allocated object must be referenced\n"); + return -EINVAL; + } + } + err = process_kf_arg_ptr_to_list_node(env, reg, argno, meta); + if (err < 0) + return err; + break; + case ARG_PTR_TO_RB_NODE: + if (is_bpf_rbtree_add_kfunc(meta->func_id)) { + if (reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) { + verbose(env, "%s expected pointer to allocated object\n", + reg_arg_name(env, argno)); + return -EINVAL; + } + if (!reg_is_referenced(env, reg)) { + verbose(env, "allocated object must be referenced\n"); + return -EINVAL; + } + } else { + if (!type_is_non_owning_ref(reg->type) && + !reg_is_referenced(env, reg)) { + verbose(env, "%s can only take non-owning or refcounted bpf_rb_node pointer\n", + meta->func_name); + return -EINVAL; + } + if (in_rbtree_lock_required_cb(env)) { + verbose(env, "%s not allowed in rbtree cb\n", meta->func_name); + return -EINVAL; + } + } + err = process_kf_arg_ptr_to_rbtree_node(env, reg, argno, meta); + if (err < 0) + return err; break; case ARG_CONST_ALLOC_SIZE_OR_ZERO: + if (meta->btf && is_kfunc_arg_scalar_with_name(meta->btf, btf_arg, + "rdonly_buf_size")) + meta->r0_rdonly = true; err = process_const_alloc_mem_size(env, reg, argno, &meta->ret_mem); - if (err) + if (err < 0) { + if (meta->btf && err == -EINVAL) + bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name, + "Pass a verifier-known constant size for this function's buffer argument.", + "the function uses this argument as a return-buffer size, but %s is invalid or variable on this path", + reg_arg_name(env, argno)); return err; + } break; + case ARG_PTR_TO_REFCOUNTED_KPTR: + { + struct btf_record *rec; + + if (!type_is_non_owning_ref(reg->type) && reg_is_referenced(env, reg)) + meta->arg_owning_ref = true; + + rec = reg_btf_record(reg); + if (!rec) { + verifier_bug(env, "Couldn't find btf_record"); + return -EFAULT; + } + + if (rec->refcount_off < 0) { + verbose(env, "%s doesn't point to a type with bpf_refcount field\n", + reg_arg_name(env, argno)); + return -EINVAL; + } + + meta->arg_btf = reg->btf; + meta->arg_btf_id = reg->btf_id; + break; + } case ARG_PTR_TO_CONST_STR: { err = check_arg_const_str(env, reg, argno); @@ -8955,6 +10293,50 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, return err; break; } + case ARG_PTR_TO_WORKQUEUE: + err = check_map_field_pointer(env, reg, argno, BPF_WORKQUEUE, &meta->map); + if (err < 0) + return err; + break; + case ARG_PTR_TO_TASK_WORK: + err = check_map_field_pointer(env, reg, argno, BPF_TASK_WORK, &meta->map); + if (err < 0) + return err; + break; + case ARG_PTR_TO_RCU_HEAD: + err = check_map_field_pointer(env, reg, argno, BPF_RCU_HEAD, &meta->map); + if (err < 0) + return err; + break; + case ARG_PTR_TO_IRQ_FLAG: + err = process_irq_flag(env, reg, argno, meta); + if (err < 0) + return err; + break; + case ARG_PTR_TO_CTX_OUT: + if (reg->mem_size != arg_size) { + verbose(env, "%s expected %u bytes of ctx-provided memory, got %u\n", + reg_arg_name(env, argno), arg_size, reg->mem_size); + return -EINVAL; + } + break; + case ARG_PTR_TO_RES_SPIN_LOCK: + { + int flags; + + if (in_rbtree_lock_required_cb(env)) { + verbose(env, "can't res_spin_{lock,unlock} in rbtree cb\n"); + return -EACCES; + } + + flags = get_bpf_res_spin_lock_kfunc_flags(meta); + if (!flags) + return -EFAULT; + err = process_spin_lock(env, reg, argno, flags); + if (err < 0) + return err; + break; + } case ARG_KPTR_XCHG_DEST: err = process_kptr_func(env, regno, meta); if (err) @@ -8965,6 +10347,139 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, return err; } +/* + * The slots a parameter takes, from its BTF type. This has to agree with + * btf_func_model_arg_slots(), which answers the same from the size the + * func model recorded, or the verifier would check an argument at a slot + * the JIT does not place it at. + */ +static u32 btf_arg_slots(const struct btf_type *t) +{ + if (btf_type_is_int(t) || btf_type_is_struct(t)) + return (t->size + BPF_REG_SIZE - 1) / BPF_REG_SIZE; + return 1; +} + +static u32 btf_proto_slots(const struct btf *btf, const struct btf_type *func_proto) +{ + const struct btf_param *args = btf_params(func_proto); + u32 i, nargs = btf_type_vlen(func_proto), slots_used = 0; + + for (i = 0; i < nargs; i++) + slots_used += btf_arg_slots(btf_type_skip_modifiers(btf, args[i].type, NULL)); + + return slots_used; +} + +/* The argument slot @slot holds an eightbyte of a by-value argument. */ +static int check_arg_extra_slot(struct bpf_verifier_env *env, struct bpf_func_state *caller, + u32 slot, struct bpf_call_arg_meta *meta) +{ + struct bpf_reg_state *reg = get_func_arg_reg(caller, cur_regs(env), slot); + argno_t argno = argno_from_arg(slot + 1); + int regno = reg_from_argno(argno); + int err; + + if (regno >= 0) { + err = check_reg_arg(env, regno, SRC_OP); + if (err) + return err; + } + + return check_reg_type(env, reg, argno, ARG_SCALAR, meta); +} + +static int check_func_args(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, + int insn_idx) +{ + u32 slot = 0, prev_slot = 0, proto_slots, nslots; + struct bpf_func_state *caller = cur_func(env); + const struct btf_param *args = NULL; + u32 arg, nargs = MAX_BPF_FUNC_REG_ARGS; + int err; + + if (meta->btf) { + args = btf_params(meta->func_proto); + nargs = btf_type_vlen(meta->func_proto); + } + + /* + * A by-value argument of more than one eightbyte takes a slot per + * eightbyte, so the slots a call occupies are no longer its parameter + * count. Only a proto whose parameters take a slot each can name the + * argument a stack slot belongs to. + */ + proto_slots = meta->btf ? btf_proto_slots(meta->btf, meta->func_proto) : nargs; + + if (proto_slots > MAX_BPF_FUNC_REG_ARGS) { + err = check_outgoing_stack_args(env, caller, proto_slots, meta->func_name, + meta->btf, proto_slots == nargs ? args : NULL); + if (err) + return err; + } + + for (arg = 0; arg < nargs; arg++, prev_slot = slot, slot += nslots) { + const struct btf_type *t; + u32 k; + + nslots = 1; + if (args) { + t = btf_type_skip_modifiers(meta->btf, args[arg].type, NULL); + nslots = btf_arg_slots(t); + } + + if (meta->fn->arg_type[arg] == ARG_UNUSED) + break; + err = check_func_arg(env, arg, slot, prev_slot, meta, insn_idx); + if (err) + return err; + + /* + * check_func_arg() took the first slot. A parameter of more + * than one eightbyte is always a scalar, so every slot it + * takes beyond the first holds one too. + */ + for (k = 1; k < nslots; k++) { + err = check_arg_extra_slot(env, caller, slot + k, meta); + if (err) + return err; + } + } + + return 0; +} + +static int mark_raw_stack(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, + int insn_idx) +{ + struct bpf_func_state *caller = cur_func(env); + u32 slot; + int i, err; + + /* + * Validate every argument before initializing outputs: an input argument + * may alias an output buffer. Use the normal stack-write checks to discard + * stale spills and preserve the rules for special stack objects. + */ + for (slot = 0; slot < MAX_BPF_FUNC_ARGS; slot++) { + struct bpf_reg_state *reg; + argno_t argno = argno_from_arg(slot + 1); + + if (!meta->arg_raw_mem.size[slot]) + continue; + reg = get_func_arg_reg(caller, cur_regs(env), slot); + + for (i = 0; i < meta->arg_raw_mem.size[slot]; i++) { + err = check_mem_access(env, insn_idx, reg, argno, i, BPF_B, + BPF_WRITE, -1, false, false); + if (err) + return err; + } + } + + return 0; +} + static bool may_update_sockmap(struct bpf_verifier_env *env, int func_id) { enum bpf_attach_type eatype = env->prog->expected_attach_type; @@ -9004,12 +10519,71 @@ bool bpf_allow_tail_call_in_subprogs(struct bpf_verifier_env *env) bpf_jit_supports_subprog_tailcalls(); } +/* + * Is rcu_read_lock_held() true at run time? Unlike for in_rcu_cs(), disabled + * preemption or interrupts will not do. + */ +static bool has_rcu_read_lock(struct bpf_verifier_env *env) +{ + struct bpf_verifier_state *state = env->cur_state; + + if (state->active_rcu_locks || in_rcu_callback(env)) + return true; + if (in_sleepable(env)) + return false; + return !env->no_rcu_read_lock || state->frame[0]->in_async_callback_fn; +} + +/* May the map be used outside of an RCU read side critical section? */ +static bool map_needs_no_rcu(const struct bpf_map *map) +{ + switch (map->map_type) { + case BPF_MAP_TYPE_HASH: + case BPF_MAP_TYPE_RHASH: + case BPF_MAP_TYPE_LRU_HASH: + case BPF_MAP_TYPE_ARRAY: + case BPF_MAP_TYPE_PERCPU_HASH: + case BPF_MAP_TYPE_PERCPU_ARRAY: + case BPF_MAP_TYPE_LRU_PERCPU_HASH: + case BPF_MAP_TYPE_LPM_TRIE: + case BPF_MAP_TYPE_ARRAY_OF_MAPS: + case BPF_MAP_TYPE_HASH_OF_MAPS: + case BPF_MAP_TYPE_RINGBUF: + case BPF_MAP_TYPE_USER_RINGBUF: + case BPF_MAP_TYPE_INODE_STORAGE: + case BPF_MAP_TYPE_SK_STORAGE: + case BPF_MAP_TYPE_TASK_STORAGE: + case BPF_MAP_TYPE_CGRP_STORAGE: + case BPF_MAP_TYPE_QUEUE: + case BPF_MAP_TYPE_STACK: + case BPF_MAP_TYPE_ARENA: + case BPF_MAP_TYPE_INSN_ARRAY: + case BPF_MAP_TYPE_PROG_ARRAY: + case BPF_MAP_TYPE_BLOOM_FILTER: + return true; + default: + return false; + } +} + static int check_map_func_compatibility(struct bpf_verifier_env *env, struct bpf_map *map, int func_id) { if (!map) return 0; + /* + * check_map_prog_compatibility() asks the program. That misses the + * sleepable callback of another program, and an inner map. They may + * use any map under the RCU read lock. Disabled preemption or + * interrupts will not do: the maps ask for rcu_read_lock_held(). + */ + if (!has_rcu_read_lock(env) && !map_needs_no_rcu(map)) { + verbose(env, "map '%s' needs bpf_rcu_read_lock() in sleepable context\n", + map->name); + return -EINVAL; + } + /* We need a two way check, first is from map perspective ... */ switch (map->map_type) { case BPF_MAP_TYPE_PROG_ARRAY: @@ -9257,23 +10831,6 @@ static int check_map_func_compatibility(struct bpf_verifier_env *env, return -EINVAL; } -static bool check_raw_mode_ok(const struct bpf_func_proto *fn, struct bpf_call_arg_meta *meta) -{ - int i; - - for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++) { - if (fn->arg_type[i] == ARG_DONTCARE) - break; - if (!arg_type_is_raw_mem(fn->arg_type[i])) - continue; - if (meta->arg_raw_mem.regno) - return false; - meta->arg_raw_mem.regno = i + 1; - } - - return true; -} - static bool check_args_pair_invalid(const struct bpf_func_proto *fn, int arg) { bool is_fixed = fn->arg_type[arg] & MEM_FIXED_SIZE; @@ -9312,7 +10869,7 @@ static bool check_btf_id_ok(const struct bpf_func_proto *fn) int i; for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++) { - if (fn->arg_type[i] == ARG_DONTCARE) + if (fn->arg_type[i] == ARG_UNUSED) break; if (base_type(fn->arg_type[i]) == ARG_PTR_TO_BTF_ID) return !!fn->arg_btf_id[i]; @@ -9335,7 +10892,7 @@ static bool check_mem_arg_rw_flag_ok(const struct bpf_func_proto *fn) for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++) { enum bpf_arg_type arg_type = fn->arg_type[i]; - if (arg_type == ARG_DONTCARE) + if (arg_type == ARG_UNUSED) break; if (base_type(arg_type) != ARG_PTR_TO_MEM) continue; @@ -9353,7 +10910,7 @@ static bool check_proto_release_reg(const struct bpf_func_proto *fn, struct bpf_ for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++) { enum bpf_arg_type arg_type = fn->arg_type[i]; - if (arg_type == ARG_DONTCARE) + if (arg_type == ARG_UNUSED) break; if (arg_type_is_release(arg_type)) { if (meta->release_regno) @@ -9365,9 +10922,41 @@ static bool check_proto_release_reg(const struct bpf_func_proto *fn, struct bpf_ return true; } -static int check_func_proto(const struct bpf_func_proto *fn, struct bpf_call_arg_meta *meta) +static bool check_arg_prog_aux(struct bpf_verifier_env *env, + const struct bpf_func_proto *proto) { - return check_raw_mode_ok(fn, meta) && + bool seen = false; + argno_t argno; + u32 i; + + for (i = 0; i < ARRAY_SIZE(proto->arg_type); i++) { + if (proto->arg_type[i] == ARG_UNUSED) + break; + if (proto->arg_type[i] != ARG_PTR_TO_PROG_AUX) + continue; + + if (seen) { + verifier_bug(env, "Only 1 prog->aux argument supported"); + return false; + } + + argno = argno_from_arg(i + 1); + if (reg_from_argno(argno) < 0) { + verbose(env, "%s prog->aux cannot be a stack argument\n", + reg_arg_name(env, argno)); + return false; + } + + seen = true; + } + + return true; +} + +static int check_func_proto(struct bpf_verifier_env *env, const struct bpf_func_proto *fn, + struct bpf_call_arg_meta *meta) +{ + return check_arg_prog_aux(env, fn) && check_arg_pair_ok(fn) && check_mem_arg_rw_flag_ok(fn) && check_proto_release_reg(fn, meta) && @@ -9393,6 +10982,20 @@ static void clear_all_pkt_pointers(struct bpf_verifier_env *env) })); } +static bool holds_pkt_pointers(struct bpf_verifier_env *env) +{ + struct bpf_func_state *state; + struct bpf_reg_state *reg; + bool ret = false; + + bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({ + if (reg_is_pkt_pointer_any(reg) || reg_is_dynptr_slice_pkt(reg)) + ret = true; + })); + + return ret; +} + enum { AT_PKT_END = -1, BEYOND_PKT_END = -2, @@ -9587,7 +11190,8 @@ static int ref_convert_alloc_rcu_protected(struct bpf_verifier_env *env, u32 id) continue; if ((reg->type & MEM_ALLOC) && (reg->type & MEM_PERCPU)) { bpf_diag_mod_begin(env, reg, NULL, BPF_DIAG_MOD_WRITE); - reg->id = 0; + if (!type_may_be_null(reg->type)) + reg->id = 0; reg->type &= ~MEM_ALLOC; reg->type |= MEM_RCU; bpf_diag_mod_end(env); @@ -9679,23 +11283,57 @@ static int setup_func_entry(struct bpf_verifier_env *env, int subprog, int calls return err; } -static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, - const struct btf *btf, - struct bpf_reg_state *regs) +static void gen_subprog_arg_proto(const struct bpf_subprog_info *sub, const struct btf *btf, + const struct btf_type *func_proto, struct bpf_func_proto *proto) +{ + const struct btf_param *args = btf_params(func_proto); + u32 arg, slot = 0; + + memset(proto, 0, sizeof(*proto)); + for (arg = 0; arg < btf_type_vlen(func_proto); arg++) { + enum bpf_arg_type arg_type = sub->args[slot].arg_type; + const struct btf_type *t; + + if (arg_type & PTR_UNTRUSTED) { + /* + * An __arg_untrusted argument accepts any caller value. The + * callee treats it as read-only and uses probe-read instructions + * to protect against invalid memory access. + */ + arg_type = ARG_IGNORE; + } else if (base_type(arg_type) == ARG_PTR_TO_ARENA) { + arg_type |= PTR_MAYBE_NULL; + } else if (base_type(arg_type) == ARG_PTR_TO_MEM) { + proto->arg_size[arg] = sub->args[slot].mem_size; + arg_type |= MEM_FIXED_SIZE | MEM_WRITE; + } else if (base_type(arg_type) == ARG_PTR_TO_BTF_ID) { + proto->arg_btf_id[arg] = &sub->args[slot].btf_id; + } + proto->arg_type[arg] = arg_type; + t = btf_type_skip_modifiers(btf, args[arg].type, NULL); + slot += btf_arg_slots(t); + } +} + +static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, struct btf *btf) { struct bpf_subprog_info *sub = subprog_info(env, subprog); struct bpf_func_state *caller = cur_func(env); - struct bpf_verifier_log *log = &env->log; - struct ref_obj_desc ref_obj = {}; - const struct btf_param *args; const struct btf_type *func, *func_proto; - u32 i; + struct bpf_call_arg_meta meta; + struct bpf_func_proto *fn; int ret, err; + u32 i; + + memset(&meta, 0, sizeof(meta)); + meta.btf = btf; + meta.subprog_may_change_pkt = sub->changes_pkt_data; + meta.func_name = bpf_subprog_name(env, subprog); ret = btf_prepare_func_args(env, subprog); if (ret) { if (bpf_in_stack_arg_cnt(sub) > 0) { - err = check_outgoing_stack_args(env, caller, sub->arg_cnt, + err = check_outgoing_stack_args(env, caller, sub->arg_slot_cnt, bpf_subprog_name(env, subprog), NULL, NULL); if (err) @@ -9706,125 +11344,33 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, func = btf_type_by_id(btf, env->prog->aux->func_info[subprog].type_id); func_proto = btf_type_by_id(btf, func->type); - args = btf_params(func_proto); - ret = check_outgoing_stack_args(env, caller, sub->arg_cnt, - bpf_subprog_name(env, subprog), btf, args); - if (ret) - return ret; - /* check that BTF function arguments match actual types that the - * verifier sees. + fn = &env->bpf_subprog_scratch; + gen_subprog_arg_proto(sub, btf, func_proto, fn); + meta.fn = fn; + meta.func_proto = func_proto; + + /* + * Check if all register arguments declared in BTF are initialized. + * If they aren't the BTF is unreliable. */ - for (i = 0; i < sub->arg_cnt; i++) { - argno_t argno = argno_from_arg(i + 1); - struct bpf_reg_state *reg = get_func_arg_reg(caller, regs, i); - struct bpf_subprog_arg_info *arg = &sub->args[i]; - - if (arg->arg_type == ARG_ANYTHING) { - if (reg->type != SCALAR_VALUE) { - bpf_log(log, "%s is not a scalar\n", reg_arg_name(env, argno)); + if (!bpf_subprog_is_global(env, subprog)) { + for (i = 0; i < min_t(u32, sub->arg_slot_cnt, MAX_BPF_FUNC_REG_ARGS); i++) + if (caller->regs[BPF_REG_1 + i].type == NOT_INIT) return -EINVAL; - } - } else if (arg->arg_type & PTR_UNTRUSTED) { - /* - * Anything is allowed for untrusted arguments, as these are - * read-only and probe read instructions would protect against - * invalid memory access. - */ - } else if (arg->arg_type == ARG_PTR_TO_CTX) { - ret = check_func_arg_reg_off(env, reg, argno, ARG_PTR_TO_CTX); - if (ret < 0) - return ret; - /* If function expects ctx type in BTF check that caller - * is passing PTR_TO_CTX. - */ - if (reg->type != PTR_TO_CTX) { - bpf_log(log, "%s expects pointer to ctx\n", - reg_arg_name(env, argno)); - return -EINVAL; - } - } else if (base_type(arg->arg_type) == ARG_PTR_TO_MEM) { - ret = check_func_arg_reg_off(env, reg, argno, ARG_DONTCARE); - if (ret < 0) - return ret; - if (check_mem_reg(env, reg, argno, arg->mem_size, BPF_READ | BPF_WRITE, NULL, - NULL)) - return -EINVAL; - /* - * PTR_TO_PACKET get passed as PTR_TO_MEM, preventing - * us from adjusting bounds tracking info. - */ - if ((reg_is_pkt_pointer_any(reg) || reg_is_dynptr_slice_pkt(reg)) && - sub->changes_pkt_data) { - bpf_log(log, "%s is a packet pointer, but func#%d may change packet data\n", - reg_arg_name(env, argno), subprog); - return -EINVAL; - } - if (!(arg->arg_type & PTR_MAYBE_NULL) && - (type_may_be_null(reg->type) || bpf_register_is_null(reg))) { - bpf_log(log, "%s is expected to be non-NULL\n", - reg_arg_name(env, argno)); - return -EINVAL; - } - } else if (base_type(arg->arg_type) == ARG_PTR_TO_ARENA) { - /* - * Can pass any value and the kernel won't crash, but - * only PTR_TO_ARENA or SCALAR make sense. Everything - * else is a bug in the bpf program. Point it out to - * the user at the verification time instead of - * run-time debug nightmare. - */ - if (reg->type != PTR_TO_ARENA && reg->type != SCALAR_VALUE) { - bpf_log(log, "%s is not a pointer to arena or scalar.\n", - reg_arg_name(env, argno)); - return -EINVAL; - } - } else if (arg->arg_type == ARG_PTR_TO_DYNPTR) { - ret = check_func_arg_reg_off(env, reg, argno, ARG_PTR_TO_DYNPTR); - if (ret) - return ret; - - ret = process_dynptr_func(env, reg, argno, env->insn_idx, - bpf_subprog_name(env, subprog), arg->arg_type, - &ref_obj, NULL); - if (ret) - return ret; - } else if (base_type(arg->arg_type) == ARG_PTR_TO_BTF_ID) { - struct bpf_call_arg_meta meta; - int err; - - if (bpf_register_is_null(reg) && type_may_be_null(arg->arg_type)) { - err = mark_arg_precision(env, argno); - if (err) - return err; - continue; - } - - memset(&meta, 0, sizeof(meta)); /* leave func_id as zero */ - err = check_reg_type(env, reg, argno, arg->arg_type, &arg->btf_id, &meta, - bpf_subprog_name(env, subprog)); - err = err ?: check_func_arg_reg_off(env, reg, argno, arg->arg_type); - if (err) - return err; - } else { - verifier_bug(env, "unrecognized %s type %d", - reg_arg_name(env, argno), arg->arg_type); - return -EFAULT; - } } - - return 0; + return check_func_args(env, &meta, env->insn_idx); } -/* Compare BTF of a function call with given bpf_reg_state. +/* + * Check that call-site argument states match a subprog's BTF signature. + * * Returns: * EFAULT - there is a verifier bug. Abort verification. - * EINVAL - there is a type mismatch or BTF is not available. + * Other errors - there is a type mismatch or BTF is not available. * 0 - BTF matches with what bpf_reg_state expects. - * Only PTR_TO_CTX and SCALAR_VALUE states are recognized. */ -static int btf_check_subprog_call(struct bpf_verifier_env *env, int subprog, - struct bpf_reg_state *regs) +static int btf_check_subprog_call(struct bpf_verifier_env *env, int subprog) { struct bpf_prog *prog = env->prog; struct btf *btf = prog->aux->btf; @@ -9841,7 +11387,7 @@ static int btf_check_subprog_call(struct bpf_verifier_env *env, int subprog, if (prog->aux->func_info_aux[subprog].unreliable) return -EINVAL; - err = btf_check_func_arg_match(env, subprog, btf, regs); + err = btf_check_func_arg_match(env, subprog, btf); /* Compiler optimizations can remove arguments from static functions * or mismatched type can be passed into a global function. * In such cases mark the function as unreliable from BTF point of view. @@ -9860,7 +11406,7 @@ static int push_callback_call(struct bpf_verifier_env *env, struct bpf_insn *ins int err; caller = state->frame[state->curframe]; - err = btf_check_subprog_call(env, subprog, caller->regs); + err = btf_check_subprog_call(env, subprog); if (err == -EFAULT) return err; @@ -9891,8 +11437,10 @@ static int push_callback_call(struct bpf_verifier_env *env, struct bpf_insn *ins is_async_cb_sleepable(env, insn)); if (IS_ERR(async_cb)) return PTR_ERR(async_cb); + if (is_async_cb_sleepable(env, insn)) + mark_sleepable_async_cb(env); callee = async_cb->frame[0]; - callee->async_entry_cnt = caller->async_entry_cnt + 1; + callee->async_entry_cnt = state->frame[0]->async_entry_cnt + 1; /* Convert bpf_timer_set_callback() args into timer callback args */ err = set_callee_state_cb(env, caller, callee, insn_idx); @@ -9902,6 +11450,15 @@ static int push_callback_call(struct bpf_verifier_env *env, struct bpf_insn *ins return 0; } + /* + * The address of the callback might be taken in another function or + * read from a map, see check_func_ptr_read(). Like with callx it's + * known here what is called. + */ + err = record_callx_edge(env, caller->subprogno, subprog); + if (err) + return err; + /* for callback functions enqueue entry to callback and * proceed with next instruction within current frame. */ @@ -9923,89 +11480,21 @@ static int push_callback_call(struct bpf_verifier_env *env, struct bpf_insn *ins static int process_bpf_exit_full(struct bpf_verifier_env *env, bool *do_print_state, bool exception_exit); -static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, - int *insn_idx) +/* + * Call of a static subprog. The callee is verified in the context of + * the caller, hence set up a new frame and continue from the first + * instruction of the callee. + */ +static int check_static_func_call(struct bpf_verifier_env *env, int subprog, + int *insn_idx) { struct bpf_verifier_state *state = env->cur_state; struct bpf_subprog_info *caller_info; u16 callee_incoming, stack_arg_cnt; struct bpf_func_state *caller; - int err, subprog, target_insn; - - target_insn = *insn_idx + insn->imm + 1; - subprog = bpf_find_subprog(env, target_insn); - if (verifier_bug_if(subprog < 0, env, "target of func call at insn %d is not a program", - target_insn)) - return -EFAULT; + int err; caller = state->frame[state->curframe]; - err = btf_check_subprog_call(env, subprog, caller->regs); - if (err == -EFAULT) - return err; - if (bpf_subprog_is_global(env, subprog)) { - struct bpf_func_info_aux *sub_aux = subprog_aux(env, subprog); - const char *sub_name = bpf_subprog_name(env, subprog); - const char *operation; - bool returns_void; - - if (env->cur_state->active_locks) { - verbose(env, "global function calls are not allowed while holding a lock,\n" - "use static function instead\n"); - operation = bpf_diag_fmt(env, "global function %s()", sub_name); - bpf_diag_ctx_active(env, *insn_idx, operation, BPF_DIAG_CONTEXT_LOCK, - "Release the lock before calling the global function, or use a static function instead."); - return -EINVAL; - } - - if (env->subprog_info[subprog].might_sleep && !in_sleepable_context(env)) { - verbose(env, "sleepable global function %s() called in %s\n", - sub_name, non_sleepable_context_description(env)); - operation = bpf_diag_fmt(env, "sleepable global function %s()", sub_name); - bpf_diag_ctx_forbidden(env, *insn_idx, operation, - "Move the call outside the critical section, or use a non-sleepable function."); - return -EINVAL; - } - - if (err) { - verbose(env, "Caller passes invalid args into func#%d ('%s')\n", - subprog, sub_name); - return err; - } - - if (env->log.level & BPF_LOG_LEVEL) - verbose(env, "Func#%d ('%s') is global and assumed valid.\n", - subprog, sub_name); - sub_aux->called[in_sleepable_context(env)] = true; - returns_void = subprog_returns_void(env, subprog); - if (env->subprog_info[subprog].changes_pkt_data) - clear_all_pkt_pointers(env); - if (returns_void) - bpf_diag_record_scrub(env, &caller->regs[BPF_REG_0], BPF_DIAG_MOD_CALLER_SAVED); - else - bpf_diag_mod_begin(env, &caller->regs[BPF_REG_0], NULL, BPF_DIAG_MOD_WRITE); - clear_caller_saved_regs(env, caller->regs); - invalidate_outgoing_stack_args(env, cur_func(env)); - - /* All non-void global functions return a 64-bit SCALAR_VALUE. */ - if (!returns_void) { - mark_reg_unknown(env, caller->regs, BPF_REG_0); - bpf_diag_mod_end(env); - } - - if (env->subprog_info[subprog].might_throw) { - struct bpf_verifier_state *branch; - - branch = push_stack(env, *insn_idx + 1, *insn_idx, false); - if (IS_ERR(branch)) { - verbose(env, "failed to push state for global subprog exception path\n"); - return PTR_ERR(branch); - } - return process_bpf_exit_full(env, NULL, true); - } - - /* continue with next insn after call */ - return 0; - } /* * Track caller's total stack arg count (incoming + max outgoing). @@ -10017,7 +11506,8 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, if (stack_arg_cnt > caller_info->stack_arg_cnt) caller_info->stack_arg_cnt = stack_arg_cnt; - /* for regular function entry setup new frame and continue + /* + * For regular function entry setup new frame and continue * from that frame. */ err = setup_func_entry(env, subprog, *insn_idx, set_callee_state, state); @@ -10040,6 +11530,194 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, return 0; } +static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, + int *insn_idx) +{ + struct bpf_verifier_state *state = env->cur_state; + struct bpf_func_state *caller; + int err, subprog, target_insn; + u32 i, nregs; + + target_insn = *insn_idx + insn->imm + 1; + subprog = bpf_find_subprog(env, target_insn); + if (verifier_bug_if(subprog < 0, env, "target of func call at insn %d is not a program", + target_insn)) + return -EFAULT; + + caller = state->frame[state->curframe]; + err = btf_check_subprog_call(env, subprog); + if (err == -EFAULT) + return err; + if (bpf_subprog_is_global(env, subprog)) { + struct bpf_func_info_aux *sub_aux = subprog_aux(env, subprog); + const char *sub_name = bpf_subprog_name(env, subprog); + const char *operation; + bool returns_void; + + if (env->cur_state->active_locks) { + verbose(env, "global function calls are not allowed while holding a lock,\n" + "use static function instead\n"); + operation = bpf_diag_fmt(env, "global function %s()", sub_name); + bpf_diag_ctx_active(env, *insn_idx, operation, BPF_DIAG_CONTEXT_LOCK, + "Release the lock before calling the global function, or use a static function instead."); + return -EINVAL; + } + + if (env->subprog_info[subprog].might_sleep && !in_sleepable_context(env)) { + const char *suggestion; + + verbose(env, "sleepable global function %s() called in %s\n", + sub_name, non_sleepable_context_description(env)); + if (in_sleepable(env)) + suggestion = "Move the call outside the critical section, or use a non-sleepable function."; + else + suggestion = "Mark the program sleepable if the program type allows it, or use a non-sleepable function."; + operation = bpf_diag_fmt(env, "sleepable global function %s()", sub_name); + bpf_diag_ctx_forbidden(env, *insn_idx, operation, suggestion); + return -EINVAL; + } + + if (err) { + verbose(env, "Caller passes invalid args into func#%d ('%s')\n", + subprog, sub_name); + return err; + } + + if (env->log.level & BPF_LOG_LEVEL) + verbose(env, "Func#%d ('%s') is global and assumed valid.\n", + subprog, sub_name); + if (in_sleepable_context(env)) + sub_aux->called[BPF_FUNC_CTX_SLEEPABLE] = true; + else if (has_rcu_read_lock(env)) + sub_aux->called[BPF_FUNC_CTX_NON_SLEEPABLE] = true; + else + sub_aux->called[BPF_FUNC_CTX_NO_RCU_LOCK] = true; + returns_void = subprog_returns_void(env, subprog); + if (env->subprog_info[subprog].changes_pkt_data) + clear_all_pkt_pointers(env); + if (returns_void) + bpf_diag_record_scrub(env, &caller->regs[BPF_REG_0], BPF_DIAG_MOD_CALLER_SAVED); + else + bpf_diag_mod_begin(env, &caller->regs[BPF_REG_0], NULL, BPF_DIAG_MOD_WRITE); + clear_caller_saved_regs(env, caller->regs); + invalidate_outgoing_stack_args(env, cur_func(env)); + /* + * What the callers of a global function hold is not known + * while it is verified. + */ + if (!env->subprog_info[subprog].changes_pkt_data && + (state->frame[0]->subprogno || holds_pkt_pointers(env))) + sub_aux->called_with_pkt_ptrs = true; + + /* + * A non-void global function returns a 64-bit SCALAR_VALUE in + * R0, or a >8 byte SCALAR_VALUE in the R0:R2 register pair. + */ + if (!returns_void) { + nregs = bpf_ret_reg_pair(env, subprog) ? 2 : 1; + mark_reg_unknown(env, caller->regs, ret_regs[0]); + bpf_diag_mod_end(env); + for (i = 1; i < nregs; i++) { + bpf_diag_mod_begin(env, &caller->regs[ret_regs[i]], NULL, + BPF_DIAG_MOD_WRITE); + mark_reg_unknown(env, caller->regs, ret_regs[i]); + bpf_diag_mod_end(env); + } + } + + if (env->subprog_info[subprog].can_unwind) { + struct bpf_verifier_state *branch; + + if (!bpf_is_unwind_goto(insn + 1)) { + verbose(env, "no unwind_goto after the call of %s()\n", sub_name); + return -EINVAL; + } + branch = push_stack(env, *insn_idx + 1, *insn_idx, false); + if (IS_ERR(branch)) + return PTR_ERR(branch); + branch->unwound = true; + } + + if (env->subprog_info[subprog].might_throw) { + struct bpf_verifier_state *branch; + + branch = push_stack(env, *insn_idx + 1, *insn_idx, false); + if (IS_ERR(branch)) { + verbose(env, "failed to push state for global subprog exception path\n"); + return PTR_ERR(branch); + } + return process_bpf_exit_full(env, NULL, true); + } + + /* continue with next insn after call */ + return 0; + } + + return check_static_func_call(env, subprog, insn_idx); +} + +/* + * callx dst_reg: call a bpf subprog whose address is in dst_reg. + * + * The address of a subprog is either loaded into a register by ld_imm64 with + * src_reg == BPF_PSEUDO_FUNC, or it is read from a frozen read-only map, see + * resolve_func_ptrs(). Both are possible for static subprogs only. Hence all + * possible callees of callx are discovered by add_subprogs() and are reachable + * in the control flow graph before the main verification pass begins. + * PTR_TO_FUNC register identifies the callee, so from here on callx is verified + * as a direct call of that static subprog. + */ +static int check_func_callx(struct bpf_verifier_env *env, struct bpf_insn *insn, + int *insn_idx) +{ + struct bpf_func_state *caller = cur_func(env); + struct bpf_reg_state *reg; + const char *reason; + int err, subprog; + + err = check_reg_arg(env, insn->dst_reg, SRC_OP); + if (err) + return err; + + reg = reg_state(env, insn->dst_reg); + if (reg->type != PTR_TO_FUNC) { + verbose(env, "R%d has type %s, expected func\n", insn->dst_reg, + reg_type_str(env, reg->type)); + reason = bpf_diag_fmt( + env, "R%d holds %s, but callx can only call through the address of a static BPF function.", + insn->dst_reg, bpf_diag_reg_type_plain(env, reg->type)); + bpf_diag_register_type( + env, *insn_idx, insn->dst_reg, "indirect call through a non-function pointer", reason, + "Load the address of a static BPF function into the register before callx."); + return -EACCES; + } + + /* + * Arithmetic on PTR_TO_FUNC is allowed, but only unmodified address + * of a subprog can be called. + */ + err = check_ptr_off_reg(env, reg, insn->dst_reg); + if (err) + return err; + + /* the callx is valid, but only a JIT can execute it */ + err = require_callx_jit(env); + if (err) + return err; + + /* PTR_TO_FUNC is a pointer to a static subprog */ + subprog = reg->subprogno; + err = btf_check_subprog_call(env, subprog); + if (err == -EFAULT) + return err; + + err = record_callx_edge(env, caller->subprogno, subprog); + if (err) + return err; + + return check_static_func_call(env, subprog, insn_idx); +} + int map_set_for_each_callback_args(struct bpf_verifier_env *env, struct bpf_func_state *caller, struct bpf_func_state *callee) @@ -10105,6 +11783,19 @@ static int set_map_elem_callback_state(struct bpf_verifier_env *env, if (err) return err; + switch (map->map_type) { + case BPF_MAP_TYPE_HASH: + case BPF_MAP_TYPE_PERCPU_HASH: + case BPF_MAP_TYPE_LRU_HASH: + case BPF_MAP_TYPE_LRU_PERCPU_HASH: + case BPF_MAP_TYPE_RHASH: + /* Hash tables run the callback under rcu_read_lock(). */ + callee->in_rcu_callback_fn = true; + break; + default: + break; + } + callee->in_callback_fn = true; callee->callback_ret_range = retval_range(0, 1); return 0; @@ -10191,6 +11882,8 @@ static int set_find_vma_callback_state(struct bpf_verifier_env *env, bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_4]); bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_5]); callee->in_callback_fn = true; + /* bpf_find_vma() runs the callback under mmap_lock and RCU. */ + callee->in_rcu_callback_fn = true; callee->callback_ret_range = retval_range(0, 1); return 0; } @@ -10286,6 +11979,40 @@ static int set_task_work_schedule_callback_state(struct bpf_verifier_env *env, return 0; } +static int set_rcu_callback_state(struct bpf_verifier_env *env, + struct bpf_func_state *caller, + struct bpf_func_state *callee, + int insn_idx) +{ + struct bpf_map *map_ptr = caller->regs[BPF_REG_2].map_ptr; + u32 map_uid = caller->regs[BPF_REG_2].map_uid; + + /* + * callback_fn(struct bpf_map *map, void *key, void *value); + */ + callee->regs[BPF_REG_1].type = CONST_PTR_TO_MAP; + __mark_reg_known_zero(&callee->regs[BPF_REG_1]); + callee->regs[BPF_REG_1].map_ptr = map_ptr; + callee->regs[BPF_REG_1].map_uid = map_uid; + + callee->regs[BPF_REG_2].type = PTR_TO_MAP_KEY; + __mark_reg_known_zero(&callee->regs[BPF_REG_2]); + callee->regs[BPF_REG_2].map_ptr = map_ptr; + callee->regs[BPF_REG_2].map_uid = map_uid; + + callee->regs[BPF_REG_3].type = PTR_TO_MAP_VALUE; + __mark_reg_known_zero(&callee->regs[BPF_REG_3]); + callee->regs[BPF_REG_3].map_ptr = map_ptr; + callee->regs[BPF_REG_3].map_uid = map_uid; + + /* unused */ + bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_4]); + bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_5]); + callee->in_async_callback_fn = true; + callee->callback_ret_range = retval_range(S32_MIN, S32_MAX); + return 0; +} + static bool is_rbtree_lock_required_kfunc(u32 btf_id); static void account_processed_insn(struct bpf_verifier_env *env) @@ -10364,11 +12091,15 @@ static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx) struct bpf_func_state *caller, *callee; struct bpf_reg_state *r0; bool in_callback_fn; + u32 i, nregs; int err; callee = state->frame[state->curframe]; r0 = &callee->regs[BPF_REG_0]; - if (r0->type == PTR_TO_STACK) { + nregs = bpf_ret_reg_pair(env, callee->subprogno) ? 2 : 1; + for (i = 0; i < nregs; i++) { + if (callee->regs[ret_regs[i]].type != PTR_TO_STACK) + continue; /* technically it's ok to return caller's stack pointer * (or caller's caller's pointer) back to the caller, * since these pointers are valid. Only current stack @@ -10380,6 +12111,19 @@ static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx) } caller = state->frame[state->curframe - 1]; + if (bpf_is_unwind_exit(&env->prog->insnsi[*insn_idx])) { + /* the helper or the kfunc that called it doesn't expect that */ + if (callee->in_callback_fn) { + verbose(env, "callback cannot unwind\n"); + return -EINVAL; + } + if (!bpf_is_unwind_goto(&env->prog->insnsi[callee->callsite + 1])) { + verbose(env, "unwind_exit, but no unwind_goto after the call at insn %d\n", + callee->callsite); + return -EINVAL; + } + state->unwound = true; + } if (callee->in_callback_fn) { if (r0->type != SCALAR_VALUE) { verbose(env, "R0 not a scalar value\n"); @@ -10403,12 +12147,26 @@ static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx) return -EFAULT; } } else { - /* return to the caller whatever r0 had in the callee */ - bpf_diag_mod_begin(env, &caller->regs[BPF_REG_0], r0, BPF_DIAG_MOD_WRITE); - caller->regs[BPF_REG_0] = *r0; - bpf_diag_mod_end(env); + /* + * return to the caller whatever the callee had in the + * return register(s) + */ + for (i = 0; i < nregs; i++) { + u32 regno = ret_regs[i]; + + bpf_diag_mod_begin(env, &caller->regs[regno], &callee->regs[regno], + BPF_DIAG_MOD_WRITE); + caller->regs[regno] = callee->regs[regno]; + bpf_diag_mod_end(env); + } } + /* Invalidate callee-local dynptrs and their slices before the frame goes away. */ + err = destroy_dynptrs_in_stack_slots(env, callee, 0, + callee->allocated_stack / BPF_REG_SIZE - 1); + if (err) + return err; + /* for callbacks like bpf_loop or bpf_for_each_map_elem go back to callsite, * there function call logic would reschedule callback visit. If iteration * converges is_state_visited() would prune that visit eventually. @@ -10796,22 +12554,37 @@ int bpf_get_helper_proto(struct bpf_verifier_env *env, int func_id, return *ptr && (*ptr)->func ? 0 : -EINVAL; } +/* Is any frame on the current path a callback that runs under RCU? */ +static bool in_rcu_callback(struct bpf_verifier_env *env) +{ + struct bpf_verifier_state *state = env->cur_state; + int i; + + for (i = 0; i <= state->curframe; i++) + if (state->frame[i]->in_rcu_callback_fn) + return true; + return false; +} + /* Check if we're in a sleepable context. */ static inline bool in_sleepable_context(struct bpf_verifier_env *env) { - return !in_rcu_cs(env); + return !in_rcu_cs(env) && !in_rcu_callback(env); } static const char *non_sleepable_context_description(struct bpf_verifier_env *env) { - if (env->cur_state->active_rcu_locks) - return "rcu_read_lock region"; - if (env->cur_state->active_preempt_locks) - return "non-preemptible region"; - if (env->cur_state->active_irq_id) - return "IRQ-disabled region"; - if (env->cur_state->active_locks) - return "lock region"; + if (in_sleepable(env)) { + if (env->cur_state->active_rcu_locks) + return "rcu_read_lock region"; + if (env->cur_state->active_preempt_locks) + return "non-preemptible region"; + if (env->cur_state->active_irq_id) + return "IRQ-disabled region"; + if (env->cur_state->active_locks) + return "lock region"; + return "callback running under rcu_read_lock"; + } return "non-sleepable prog"; } @@ -10896,19 +12669,24 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn memset(&meta, 0, sizeof(meta)); - err = check_func_proto(fn, &meta); + err = check_func_proto(env, fn, &meta); if (err) { verifier_bug(env, "incorrect func proto %s#%d", func_id_name(func_id), func_id); return err; } if (fn->might_sleep && !in_sleepable_context(env)) { + const char *suggestion; + verbose(env, "sleepable helper %s#%d in %s\n", func_id_name(func_id), func_id, non_sleepable_context_description(env)); + if (in_sleepable(env)) + suggestion = "Move the helper call outside the critical section, or use a non-sleepable helper."; + else + suggestion = "Mark the program sleepable if the program type allows it, or use a non-sleepable helper."; operation = bpf_diag_fmt(env, "sleepable helper %s#%d", func_id_name(func_id), func_id); - bpf_diag_ctx_forbidden(env, insn_idx, operation, - "Move the helper call outside the critical section, or use a non-sleepable helper."); + bpf_diag_ctx_forbidden(env, insn_idx, operation, suggestion); return -EINVAL; } @@ -10917,13 +12695,11 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn env->insn_aux_data[insn_idx].non_sleepable = true; meta.func_id = func_id; + meta.func_name = func_id_name(func_id); meta.fn = fn; - /* check args */ - for (i = 0; i < MAX_BPF_FUNC_REG_ARGS; i++) { - err = check_func_arg(env, i, &meta, insn_idx); - if (err) - return err; - } + err = check_func_args(env, &meta, insn_idx); + if (err) + return err; err = record_func_map(env, &meta, func_id, insn_idx); if (err) @@ -10935,16 +12711,9 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn regs = cur_regs(env); - /* Mark slots with STACK_MISC in case of raw mode, stack offset - * is inferred from register state. - */ - for (i = 0; i < meta.arg_raw_mem.size; i++) { - err = check_mem_access(env, insn_idx, regs + meta.arg_raw_mem.regno, - argno_from_reg(meta.arg_raw_mem.regno), i, BPF_B, - BPF_WRITE, -1, false, false); - if (err) - return err; - } + err = mark_raw_stack(env, &meta, insn_idx); + if (err) + return err; if (meta.release_regno) { struct bpf_reg_state *reg = ®s[meta.release_regno]; @@ -11065,8 +12834,7 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn if (dynptr_type == BPF_DYNPTR_TYPE_INVALID) return -EFAULT; - if (dynptr_type == BPF_DYNPTR_TYPE_SKB || - dynptr_type == BPF_DYNPTR_TYPE_SKB_META) + if (dynptr_write_changes_pkt_data(dynptr_type)) /* this will trigger clear_all_pkt_pointers(), which will * invalidate all dynptr slices associated with the skb */ @@ -11364,6 +13132,37 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn return 0; } +/* + * Mark the register(s) holding a @size byte kfunc return value as unknown + * scalars and return how many registers that took. Both halves of a register + * pair are treated the same way. + * + * R0 is written in the diagnostic modification scope the caller opened before + * the caller-saved scrub, since R0 gets no scrub record of its own and that + * scope is what carries its pre-call value into the history. A scope tracks a + * single register and does not nest, so close it before recording the rest of + * the return value. A value that fits in R0 leaves the caller's scope open for + * the caller to close, so a later write to R0 still lands in it. + */ +static u32 mark_kfunc_ret_regs(struct bpf_verifier_env *env, + struct bpf_reg_state *regs, u32 size) +{ + u32 i, nregs = ret_regs_cnt(size); + + mark_reg_unknown(env, regs, ret_regs[0]); + if (nregs == 1) + return nregs; + + bpf_diag_mod_end(env); + for (i = 1; i < nregs; i++) { + bpf_diag_mod_begin(env, ®s[ret_regs[i]], NULL, BPF_DIAG_MOD_WRITE); + mark_reg_unknown(env, regs, ret_regs[i]); + bpf_diag_mod_end(env); + } + + return nregs; +} + static bool is_kfunc_acquire(struct bpf_call_arg_meta *meta) { return meta->kfunc_flags & KF_ACQUIRE; @@ -11474,6 +13273,11 @@ static bool is_kfunc_arg_irq_flag(const struct btf *btf, const struct btf_param return btf_param_match_suffix(btf, arg, "__irq_flag"); } +static bool is_kfunc_arg_ctx_out(const struct btf *btf, const struct btf_param *arg) +{ + return btf_param_match_suffix(btf, arg, "__ctx_out"); +} + static bool is_kfunc_arg_arena(const struct btf *btf, const struct btf_param *arg) { return btf_param_match_suffix(btf, arg, "__arena__nullable") || @@ -11509,7 +13313,8 @@ enum { KF_ARG_RES_SPIN_LOCK_ID, KF_ARG_TASK_WORK_ID, KF_ARG_PROG_AUX_ID, - KF_ARG_TIMER_ID + KF_ARG_TIMER_ID, + KF_ARG_RCU_HEAD_ID }; BTF_ID_LIST(kf_arg_btf_ids) @@ -11523,6 +13328,7 @@ BTF_ID(struct, bpf_res_spin_lock) BTF_ID(struct, bpf_task_work) BTF_ID(struct, bpf_prog_aux) BTF_ID(struct, bpf_timer) +BTF_ID(struct, bpf_rcu_head) static bool __is_kfunc_ptr_arg_type(const struct btf *btf, const struct btf_param *arg, int type) @@ -11581,6 +13387,11 @@ static bool is_kfunc_arg_task_work(const struct btf *btf, const struct btf_param return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_TASK_WORK_ID); } +static bool is_kfunc_arg_rcu_head(const struct btf *btf, const struct btf_param *arg) +{ + return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_RCU_HEAD_ID); +} + static bool is_kfunc_arg_res_spin_lock(const struct btf *btf, const struct btf_param *arg) { return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_RES_SPIN_LOCK_ID); @@ -11635,13 +13446,39 @@ static bool is_kfunc_arg_implicit(const struct bpf_call_arg_meta *meta, u32 arg_ return argn <= arg_idx; } -/* Returns true if struct is composed of scalars, 4 levels of nesting allowed */ -static bool __btf_type_is_scalar_struct(struct bpf_verifier_env *env, - const struct btf *btf, - const struct btf_type *t, int rec) +#define BTF_MEMBER_MAX_DEPTH 4 +#define BTF_MEMBER_PATH_LEN 64 + +struct btf_member_path { + const struct btf_member *member[BTF_MEMBER_MAX_DEPTH]; + bool in_array[BTF_MEMBER_MAX_DEPTH]; + const struct btf_type *bad_type; + int depth; + bool too_deep; +}; + +static bool btf_member_kind_allowed(const struct btf *btf, const struct btf_type *t, + u32 member_kinds) +{ + if ((member_kinds & BTF_MEMBER_SCALAR) && btf_type_is_scalar(t)) + return true; + if ((member_kinds & BTF_MEMBER_ARENA_PTR) && btf_type_is_arena_ptr(btf, t)) + return true; + return false; +} + +/* + * Returns true if every member of struct @t is of a kind listed in + * @member_kinds, 4 levels of nesting allowed. An array member counts as its + * element type. + */ +static bool btf_struct_member_walk(struct bpf_verifier_env *env, const struct btf *btf, + const struct btf_type *t, u32 member_kinds, int rec, + struct btf_member_path *path) { const struct btf_type *member_type; const struct btf_member *member; + bool in_array; u32 i; if (!btf_type_is_struct(t)) @@ -11650,58 +13487,128 @@ static bool __btf_type_is_scalar_struct(struct bpf_verifier_env *env, for_each_member(i, t, member) { const struct btf_array *array; + in_array = false; member_type = btf_type_skip_modifiers(btf, member->type, NULL); + /* + * Every element of an array is laid out in the value being + * returned, so an array counts as its element type however many + * dimensions deep that is. + */ + while (btf_type_is_array(member_type)) { + array = btf_array(member_type); + if (!array->nelems) { + member_type = NULL; + goto bad_member; + } + member_type = btf_type_skip_modifiers(btf, array->type, NULL); + in_array = true; + } if (btf_type_is_struct(member_type)) { - if (rec >= 3) { + if (rec >= BTF_MEMBER_MAX_DEPTH - 1) { verbose(env, "max struct nesting depth exceeded\n"); + if (path) + path->too_deep = true; return false; } - if (!__btf_type_is_scalar_struct(env, btf, member_type, rec + 1)) - return false; + if (!btf_struct_member_walk(env, btf, member_type, member_kinds, + rec + 1, path)) + goto bad_path; continue; } - if (btf_type_is_array(member_type)) { - array = btf_array(member_type); - if (!array->nelems) - return false; - member_type = btf_type_skip_modifiers(btf, array->type, NULL); - if (!btf_type_is_scalar(member_type)) - return false; - continue; - } - if (!btf_type_is_scalar(member_type)) - return false; + if (!btf_member_kind_allowed(btf, member_type, member_kinds)) + goto bad_member; } return true; + +bad_member: + if (path) { + path->depth = rec + 1; + path->bad_type = member_type; + } +bad_path: + if (path && path->depth) { + path->member[rec] = member; + path->in_array[rec] = in_array; + } + return false; } -enum kfunc_ptr_arg_type { - KF_ARG_CONST_MEM_SIZE, - KF_ARG_MEM_SIZE, - KF_ARG_CONST, - KF_ARG_CONST_ALLOC_SIZE_OR_ZERO, - KF_ARG_ANYTHING, - KF_ARG_PTR_TO_CTX, - KF_ARG_PTR_TO_ALLOC_BTF_ID, /* Allocated object */ - KF_ARG_PTR_TO_REFCOUNTED_KPTR, /* Refcounted local kptr */ - KF_ARG_PTR_TO_DYNPTR, - KF_ARG_PTR_TO_ITER, - KF_ARG_PTR_TO_LIST_HEAD, - KF_ARG_PTR_TO_LIST_NODE, - KF_ARG_PTR_TO_BTF_ID, /* Also covers reg2btf_ids conversions */ - KF_ARG_PTR_TO_MEM, - KF_ARG_PTR_TO_CALLBACK, - KF_ARG_PTR_TO_RB_ROOT, - KF_ARG_PTR_TO_RB_NODE, - KF_ARG_PTR_TO_CONST_STR, - KF_ARG_CONST_MAP_PTR, - KF_ARG_PTR_TO_TIMER, - KF_ARG_PTR_TO_WORKQUEUE, - KF_ARG_PTR_TO_IRQ_FLAG, - KF_ARG_PTR_TO_RES_SPIN_LOCK, - KF_ARG_PTR_TO_TASK_WORK, - KF_ARG_PTR_TO_ARENA, -}; +bool btf_struct_is_composed_of(struct bpf_verifier_env *env, + const struct btf *btf, + const struct btf_type *t, u32 member_kinds) +{ + return btf_struct_member_walk(env, btf, t, member_kinds, 0, NULL); +} + +static bool btf_type_is_scalar_struct(struct bpf_verifier_env *env, + const struct btf *btf, + const struct btf_type *t) +{ + return btf_struct_is_composed_of(env, btf, t, BTF_MEMBER_SCALAR); +} + +static int resolve_func_arg_type(struct bpf_verifier_env *env, + struct bpf_reg_state *reg, u32 arg, argno_t argno, + struct bpf_call_arg_meta *meta, + enum bpf_arg_type *arg_type, u32 *arg_size) +{ + const struct btf_param *args; + const struct btf_type *ref_t, *resolve_ret; + const struct btf *btf; + const char *ref_tname; + u32 ref_id; + + if (base_type(*arg_type) == ARG_PTR_TO_MAP_VALUE) + return resolve_map_arg_type(env, meta, arg_type); + + if (base_type(*arg_type) != ARG_PTR_TO_BTF_ID) + return 0; + + if (!is_kfunc(meta) || arg_type_is_release(*arg_type) || + base_type(reg->type) == PTR_TO_BTF_ID || + reg2btf_ids[base_type(reg->type)]) + return 0; + + args = btf_params(meta->func_proto); + ref_id = *meta->fn->arg_btf_id[arg]; + btf = is_kfunc_arg_map(meta->btf, &args[arg]) ? btf_vmlinux : meta->btf; + ref_t = btf_type_skip_modifiers(btf, ref_id, &ref_id); + ref_tname = btf_name_by_offset(btf, ref_t->name_off); + + if (!btf_type_is_scalar_struct(env, btf, ref_t)) + return 0; + + resolve_ret = btf_resolve_size(btf, ref_t, arg_size); + if (IS_ERR(resolve_ret)) { + verbose(env, "%s reference type('%s %s') size cannot be determined: %ld\n", + reg_arg_name(env, argno), btf_type_str(ref_t), ref_tname, + PTR_ERR(resolve_ret)); + return -EINVAL; + } + *arg_type = ARG_PTR_TO_MEM | MEM_FIXED_SIZE | MEM_WRITE | + (*arg_type & (PTR_MAYBE_NULL | MEM_UNINIT)); + + return 0; +} + +static void btf_member_path_str(const struct btf *btf, const struct btf_member_path *path, + char *buf, size_t buf_sz) +{ + size_t len = 0; + int i; + + buf[0] = '\0'; + for (i = 0; i < path->depth; i++) { + const char *name = btf_name_by_offset(btf, path->member[i]->name_off); + + /* An anonymous struct or union has no name to spell. */ + if (!name || !name[0]) + continue; + len += scnprintf(buf + len, buf_sz - len, "%s%s", len ? "." : "", name); + if (path->in_array[i]) + len += scnprintf(buf + len, buf_sz - len, "[]"); + } +} enum special_kfunc_type { KF_bpf_obj_new_impl, @@ -11768,6 +13675,8 @@ enum special_kfunc_type { KF___bpf_trap, KF_bpf_task_work_schedule_signal, KF_bpf_task_work_schedule_resume, + KF_bpf_call_rcu, + KF_bpf_call_rcu_tasks_trace, KF_bpf_arena_alloc_pages, KF_bpf_arena_free_pages, KF_bpf_session_is_return, @@ -11858,14 +13767,36 @@ BTF_ID(func, bpf_dynptr_file_discard) BTF_ID(func, __bpf_trap) BTF_ID(func, bpf_task_work_schedule_signal) BTF_ID(func, bpf_task_work_schedule_resume) +BTF_ID(func, bpf_call_rcu) +BTF_ID(func, bpf_call_rcu_tasks_trace) +#if defined(CONFIG_MMU) && defined(CONFIG_64BIT) BTF_ID(func, bpf_arena_alloc_pages) BTF_ID(func, bpf_arena_free_pages) +#else +BTF_ID_UNUSED +BTF_ID_UNUSED +#endif #ifdef CONFIG_BPF_EVENTS BTF_ID(func, bpf_session_is_return) #else BTF_ID_UNUSED #endif +static bool is_bpf_cast_to_kern_ctx_kfunc(const struct bpf_call_arg_meta *meta) +{ + return is_kfunc_call(meta, special_kfunc_list[KF_bpf_cast_to_kern_ctx]); +} + +static bool is_bpf_dynptr_clone_kfunc(const struct bpf_call_arg_meta *meta) +{ + return is_kfunc_call(meta, special_kfunc_list[KF_bpf_dynptr_clone]); +} + +static bool is_bpf_iter_css_task_new_kfunc(const struct bpf_call_arg_meta *meta) +{ + return is_kfunc_call(meta, special_kfunc_list[KF_bpf_iter_css_task_new]); +} + static bool is_bpf_obj_new_kfunc(u32 func_id) { return func_id == special_kfunc_list[KF_bpf_obj_new] || @@ -11911,6 +13842,12 @@ static bool is_bpf_rbtree_add_kfunc(u32 func_id) func_id == special_kfunc_list[KF_bpf_rbtree_add_impl]; } +static bool is_call_rcu_kfunc(u32 func_id) +{ + return func_id == special_kfunc_list[KF_bpf_call_rcu] || + func_id == special_kfunc_list[KF_bpf_call_rcu_tasks_trace]; +} + static bool is_task_work_add_kfunc(u32 func_id) { return func_id == special_kfunc_list[KF_bpf_task_work_schedule_signal] || @@ -11927,52 +13864,159 @@ static bool is_kfunc_ret_null(struct bpf_call_arg_meta *meta) static bool is_kfunc_bpf_rcu_read_lock(struct bpf_call_arg_meta *meta) { - return meta->func_id == special_kfunc_list[KF_bpf_rcu_read_lock]; + return is_kfunc_call(meta, special_kfunc_list[KF_bpf_rcu_read_lock]); } static bool is_kfunc_bpf_rcu_read_unlock(struct bpf_call_arg_meta *meta) { - return meta->func_id == special_kfunc_list[KF_bpf_rcu_read_unlock]; + return is_kfunc_call(meta, special_kfunc_list[KF_bpf_rcu_read_unlock]); } static bool is_kfunc_bpf_preempt_disable(struct bpf_call_arg_meta *meta) { - return meta->func_id == special_kfunc_list[KF_bpf_preempt_disable]; + return is_kfunc_call(meta, special_kfunc_list[KF_bpf_preempt_disable]); } static bool is_kfunc_bpf_preempt_enable(struct bpf_call_arg_meta *meta) { - return meta->func_id == special_kfunc_list[KF_bpf_preempt_enable]; + return is_kfunc_call(meta, special_kfunc_list[KF_bpf_preempt_enable]); } +/* These write to what their first argument, a dynptr, points to */ +BTF_SET_START(dynptr_write_kfuncs) +BTF_ID(func, bpf_dynptr_copy) +BTF_ID(func, bpf_dynptr_memset) +#ifdef CONFIG_BPF_EVENTS +BTF_ID(func, bpf_probe_read_user_dynptr) +BTF_ID(func, bpf_probe_read_kernel_dynptr) +BTF_ID(func, bpf_probe_read_user_str_dynptr) +BTF_ID(func, bpf_probe_read_kernel_str_dynptr) +BTF_ID(func, bpf_copy_from_user_dynptr) +BTF_ID(func, bpf_copy_from_user_str_dynptr) +BTF_ID(func, bpf_copy_from_user_task_dynptr) +BTF_ID(func, bpf_copy_from_user_task_str_dynptr) +#endif +BTF_SET_END(dynptr_write_kfuncs) + +static bool is_kfunc_dynptr_write(struct bpf_call_arg_meta *meta) +{ + return meta->btf == btf_vmlinux && + btf_id_set_contains(&dynptr_write_kfuncs, meta->func_id); +} + +/* + * A write through the dynptr of an skb may reallocate its head. Before the + * arguments are checked nobody knows what kind of dynptr it is. + */ bool bpf_is_kfunc_pkt_changing(struct bpf_call_arg_meta *meta) { - return meta->func_id == special_kfunc_list[KF_bpf_xdp_pull_data]; + return is_kfunc_call(meta, special_kfunc_list[KF_bpf_xdp_pull_data]) || + is_kfunc_dynptr_write(meta); +} + +static u32 kfunc_abi_slots(const struct btf_func_model *fm) +{ + const struct bpf_jit_arg_abi *abi = bpf_jit_arg_abi(); + u8 pos_of_slot[MAX_BPF_FUNC_ARG_SLOTS]; + u32 i, nslots, slots = 0; + + for (i = 0; i < fm->nr_args; i++) + slots += btf_func_model_arg_slots(fm, i); + + if (!abi) + return slots; + + nslots = bpf_jit_place_args(abi, fm, pos_of_slot); + for (i = 0; i < nslots; i++) + if (pos_of_slot[i] + 1 > slots) + slots = pos_of_slot[i] + 1; + + return slots; +} + +static u32 __btf_func_arg_align(const struct btf *btf, const struct btf_type *t, int rec) +{ + const struct btf_member *member; + const struct btf_type *mt; + u32 align, i; + + while (btf_type_is_array(t)) + t = btf_type_skip_modifiers(btf, btf_array(t)->type, NULL); + + if (btf_type_is_int(t)) + return t->size > BPF_REG_SIZE ? t->size : BPF_REG_SIZE; + if (!btf_type_is_struct(t)) + return BPF_REG_SIZE; + if (rec >= BTF_MEMBER_MAX_DEPTH) + return 0; + + for_each_member(i, t, member) { + mt = btf_type_skip_modifiers(btf, member->type, NULL); + align = __btf_func_arg_align(btf, mt, rec + 1); + if (!align) + return 0; + if (align > BPF_REG_SIZE) + return 2 * BPF_REG_SIZE; + } + return BPF_REG_SIZE; +} + +u32 btf_func_arg_align(const struct btf *btf, const struct btf_type *t) +{ + return __btf_func_arg_align(btf, t, 0); } static int get_kfunc_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, - const struct btf_param *args, int arg, int nargs) + const struct btf_param *args, int arg, int nargs, u32 slot, + struct bpf_func_proto *proto) { - const struct btf_type *t, *ref_t = NULL; - argno_t argno = argno_from_arg(arg + 1); + const struct btf_type *t, *ref_t = NULL, *resolve_ret; + const u32 *ref_id_ptr = NULL; + argno_t argno = argno_from_arg(slot + 1); const char *ref_tname = NULL; + u32 ref_id, type_size; int arg_type; + proto->arg_btf_id[arg] = NULL; + + if (is_kfunc_arg_prog_aux(meta->btf, &args[arg])) + return ARG_PTR_TO_PROG_AUX; + + if (is_kfunc_arg_ignore(meta->btf, &args[arg]) || is_kfunc_arg_implicit(meta, arg)) + return ARG_IGNORE; + t = btf_type_skip_modifiers(meta->btf, args[arg].type, NULL); /* Scalar arguments are classified from their BTF suffix/name alone. */ if (btf_type_is_scalar(t)) { if (is_kfunc_arg_constant(meta->btf, &args[arg])) - return KF_ARG_CONST; + return ARG_CONST_SCALAR; if (is_kfunc_arg_const_mem_size(meta->btf, &args[arg])) - return KF_ARG_CONST_MEM_SIZE; + return ARG_CONST_MEM_SIZE; if (is_kfunc_arg_mem_size(meta->btf, &args[arg])) - return KF_ARG_MEM_SIZE; + return ARG_MEM_SIZE; if (is_kfunc_arg_scalar_with_name(meta->btf, &args[arg], "rdonly_buf_size") || is_kfunc_arg_scalar_with_name(meta->btf, &args[arg], "rdwr_buf_size")) - return KF_ARG_CONST_ALLOC_SIZE_OR_ZERO; - return KF_ARG_ANYTHING; + return ARG_CONST_ALLOC_SIZE_OR_ZERO; + return ARG_SCALAR; + } + + if (btf_type_is_struct(t)) { + if (!t->size || t->size > 2 * BPF_REG_SIZE) { + verbose(env, + "%s type %s has size %u, only 1 to %d bytes " + "can be passed by value\n", + reg_arg_name(env, argno), btf_type_str(t), t->size, + 2 * BPF_REG_SIZE); + return -EINVAL; + } + if (!btf_type_is_scalar_struct(env, meta->btf, t)) { + verbose(env, "%s type %s is not composed of scalars\n", + reg_arg_name(env, argno), btf_type_str(t)); + return -EINVAL; + } + return ARG_SCALAR; } if (!btf_type_is_ptr(t)) { @@ -11980,54 +14024,87 @@ get_kfunc_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, reg_arg_name(env, argno), btf_type_str(t)); return -EINVAL; } - ref_t = btf_type_skip_modifiers(meta->btf, t->type, NULL); + /* Keep a pointer to the BTF field containing the resolved referent ID. */ + ref_id_ptr = &t->type; + ref_t = btf_type_skip_modifiers(meta->btf, *ref_id_ptr, &ref_id); + while (*ref_id_ptr != ref_id) + ref_id_ptr = &btf_type_by_id(meta->btf, *ref_id_ptr)->type; ref_tname = btf_name_by_offset(meta->btf, ref_t->name_off); /* In this function, we verify the kfunc's BTF as per the argument type, * leaving the rest of the verification with respect to the register * type to our caller. When a set of conditions hold in the BTF type of - * arguments, we resolve it to a known kfunc_ptr_arg_type. + * arguments, we resolve it to a known bpf_arg_type. */ - if (meta->func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx] || - meta->func_id == special_kfunc_list[KF_bpf_session_is_return] || - meta->func_id == special_kfunc_list[KF_bpf_session_cookie]) - arg_type = KF_ARG_PTR_TO_CTX; + if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_cast_to_kern_ctx]) || + is_kfunc_call(meta, special_kfunc_list[KF_bpf_session_is_return]) || + is_kfunc_call(meta, special_kfunc_list[KF_bpf_session_cookie])) + arg_type = ARG_PTR_TO_CTX; else if (btf_is_prog_ctx_type(&env->log, meta->btf, t, resolve_prog_type(env->prog), arg)) - arg_type = KF_ARG_PTR_TO_CTX; + arg_type = ARG_PTR_TO_CTX; else if (is_kfunc_arg_alloc_obj(meta->btf, &args[arg])) - arg_type = KF_ARG_PTR_TO_ALLOC_BTF_ID; + arg_type = ARG_PTR_TO_ALLOC_BTF_ID; else if (is_kfunc_arg_refcounted_kptr(meta->btf, &args[arg])) - arg_type = KF_ARG_PTR_TO_REFCOUNTED_KPTR; - else if (is_kfunc_arg_dynptr(meta->btf, &args[arg])) - arg_type = KF_ARG_PTR_TO_DYNPTR; - else if (is_kfunc_arg_iter(meta, arg, &args[arg])) - arg_type = KF_ARG_PTR_TO_ITER; + arg_type = ARG_PTR_TO_REFCOUNTED_KPTR; + else if (is_kfunc_arg_dynptr(meta->btf, &args[arg])) { + arg_type = ARG_PTR_TO_DYNPTR; + + if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_dynptr_from_skb])) + arg_type |= DYNPTR_TYPE_SKB; + else if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_dynptr_from_xdp])) + arg_type |= DYNPTR_TYPE_XDP; + else if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_dynptr_from_skb_meta])) + arg_type |= DYNPTR_TYPE_SKB_META; + else if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_dynptr_from_file]) || + is_kfunc_call(meta, special_kfunc_list[KF_bpf_dynptr_file_discard])) + /* OBJ_RELEASE for the latter comes from KF_RELEASE below */ + arg_type |= DYNPTR_TYPE_FILE; + } else if (is_kfunc_arg_iter(meta, arg, &args[arg])) + arg_type = ARG_PTR_TO_ITER; else if (is_kfunc_arg_list_head(meta->btf, &args[arg])) - arg_type = KF_ARG_PTR_TO_LIST_HEAD; + arg_type = ARG_PTR_TO_LIST_HEAD; else if (is_kfunc_arg_list_node(meta->btf, &args[arg])) - arg_type = KF_ARG_PTR_TO_LIST_NODE; + arg_type = ARG_PTR_TO_LIST_NODE; else if (is_kfunc_arg_rbtree_root(meta->btf, &args[arg])) - arg_type = KF_ARG_PTR_TO_RB_ROOT; + arg_type = ARG_PTR_TO_RB_ROOT; else if (is_kfunc_arg_rbtree_node(meta->btf, &args[arg])) - arg_type = KF_ARG_PTR_TO_RB_NODE; + arg_type = ARG_PTR_TO_RB_NODE; else if (is_kfunc_arg_const_str(meta->btf, &args[arg])) - arg_type = KF_ARG_PTR_TO_CONST_STR; + arg_type = ARG_PTR_TO_CONST_STR; else if (is_kfunc_arg_const_map(meta->btf, &args[arg])) - arg_type = KF_ARG_CONST_MAP_PTR; + arg_type = ARG_CONST_MAP_PTR; else if (is_kfunc_arg_map(meta->btf, &args[arg])) - arg_type = KF_ARG_PTR_TO_BTF_ID; + arg_type = ARG_PTR_TO_BTF_ID; else if (is_kfunc_arg_wq(meta->btf, &args[arg])) - arg_type = KF_ARG_PTR_TO_WORKQUEUE; + arg_type = ARG_PTR_TO_WORKQUEUE; else if (is_kfunc_arg_timer(meta->btf, &args[arg])) - arg_type = KF_ARG_PTR_TO_TIMER; + arg_type = ARG_PTR_TO_TIMER; else if (is_kfunc_arg_task_work(meta->btf, &args[arg])) - arg_type = KF_ARG_PTR_TO_TASK_WORK; + arg_type = ARG_PTR_TO_TASK_WORK; + else if (is_kfunc_arg_rcu_head(meta->btf, &args[arg])) + arg_type = ARG_PTR_TO_RCU_HEAD; else if (is_kfunc_arg_irq_flag(meta->btf, &args[arg])) - arg_type = KF_ARG_PTR_TO_IRQ_FLAG; + arg_type = ARG_PTR_TO_IRQ_FLAG; else if (is_kfunc_arg_res_spin_lock(meta->btf, &args[arg])) - arg_type = KF_ARG_PTR_TO_RES_SPIN_LOCK; - else if (is_kfunc_arg_callback(env, meta->btf, &args[arg])) - arg_type = KF_ARG_PTR_TO_CALLBACK; + arg_type = ARG_PTR_TO_RES_SPIN_LOCK; + else if (is_kfunc_arg_ctx_out(meta->btf, &args[arg])) { + if (!btf_type_is_scalar(ref_t)) { + verbose(env, "%s __ctx_out argument must point to a scalar\n", + reg_arg_name(env, argno)); + return -EINVAL; + } + resolve_ret = btf_resolve_size(meta->btf, ref_t, &type_size); + if (IS_ERR(resolve_ret)) { + verbose(env, + "%s reference type('%s %s') size cannot be determined: %ld\n", + reg_arg_name(env, argno), btf_type_str(ref_t), + ref_tname, PTR_ERR(resolve_ret)); + return -EINVAL; + } + proto->arg_size[arg] = type_size; + arg_type = ARG_PTR_TO_CTX_OUT | MEM_FIXED_SIZE; + } else if (is_kfunc_arg_callback(env, meta->btf, &args[arg])) + arg_type = ARG_PTR_TO_FUNC; else if (is_kfunc_arg_arena(meta->btf, &args[arg])) { if (!bpf_jit_supports_arena_args()) { verbose(env, "JIT does not support kfunc %s() with arena pointer arguments\n", @@ -12050,20 +14127,20 @@ get_kfunc_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, * whether the JIT rebases it to the arena base or preserves NULL. * The common nullable path below records that verifier property. */ - arg_type = KF_ARG_PTR_TO_ARENA; + arg_type = ARG_PTR_TO_ARENA; } else if (arg + 1 < nargs && (is_kfunc_arg_mem_size(meta->btf, &args[arg + 1]) || is_kfunc_arg_const_mem_size(meta->btf, &args[arg + 1]))) { if (!btf_type_is_void(ref_t) && !btf_type_is_scalar(ref_t) && - !__btf_type_is_scalar_struct(env, meta->btf, ref_t, 0)) { + !btf_type_is_scalar_struct(env, meta->btf, ref_t)) { verbose(env, "%s pointer type %s %s must point to void, scalar, or struct with scalar\n", reg_arg_name(env, argno), btf_type_str(ref_t), ref_tname); return -EINVAL; } - arg_type = KF_ARG_PTR_TO_MEM; + arg_type = ARG_PTR_TO_MEM | MEM_WRITE; } else if (btf_type_is_struct(ref_t)) - /* A pointer to a struct without a size argument is classified as KF_ARG_PTR_TO_BTF_ID */ - arg_type = KF_ARG_PTR_TO_BTF_ID; + /* A pointer to a struct without a size argument is classified as ARG_PTR_TO_BTF_ID */ + arg_type = ARG_PTR_TO_BTF_ID; else { /* * Otherwise this is a fixed-size memory buffer supported by @@ -12071,26 +14148,64 @@ get_kfunc_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, * scalars. The access size is derived from the pointed-to BTF type. */ if (!btf_type_is_scalar(ref_t) && - !__btf_type_is_scalar_struct(env, meta->btf, ref_t, 0)) { + !btf_type_is_scalar_struct(env, meta->btf, ref_t)) { verbose(env, "%s pointer type %s %s must point to scalar, or struct with scalar\n", reg_arg_name(env, argno), btf_type_str(ref_t), ref_tname); return -EINVAL; } - arg_type = KF_ARG_PTR_TO_MEM | MEM_FIXED_SIZE; + resolve_ret = btf_resolve_size(meta->btf, ref_t, &type_size); + if (IS_ERR(resolve_ret)) { + verbose(env, + "%s reference type('%s %s') size cannot be determined: %ld\n", + reg_arg_name(env, argno), btf_type_str(ref_t), + ref_tname, PTR_ERR(resolve_ret)); + return -EINVAL; + } + proto->arg_size[arg] = type_size; + arg_type = ARG_PTR_TO_MEM | MEM_FIXED_SIZE | MEM_WRITE; } + if (is_kfunc_arg_uninit(meta->btf, &args[arg])) + arg_type |= MEM_UNINIT; + if (is_kfunc_arg_nullable(meta->btf, &args[arg])) arg_type |= PTR_MAYBE_NULL; + /* + * Only the first argument of a KF_RELEASE kfunc releases anything, and + * bpf_fetch_kfunc_arg_meta() only ever records BPF_REG_1 for it. + */ + if (is_kfunc_release(meta) && arg == 0) + arg_type |= OBJ_RELEASE; + + if (base_type(arg_type) == ARG_PTR_TO_BTF_ID) { + if (is_kfunc_arg_map(meta->btf, &args[arg])) + proto->arg_btf_id[arg] = reg2btf_ids[CONST_PTR_TO_MAP]; + else + proto->arg_btf_id[arg] = ref_id_ptr; + + arg_type |= PTR_TRUSTED; + + /* + * A KF_RCU kfunc accepts an RCU-protected pointer where it would + * otherwise demand a referenced or trusted one. Other argument kinds + * have their own provenance requirements and must not inherit MEM_RCU. + */ + if (is_kfunc_rcu(meta)) + arg_type |= MEM_RCU; + } + return arg_type; } static int gen_kfunc_arg_proto(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, - struct bpf_func_proto *proto) + const struct btf_func_model *fm, struct bpf_func_proto *proto) { + const struct bpf_jit_arg_abi *abi; const struct btf *btf = meta->btf; const struct btf_param *args; - u32 i, nargs; + const struct btf_type *t; + u32 i, nargs, slots_used; int arg_type; args = (const struct btf_param *)(meta->func_proto + 1); @@ -12100,54 +14215,84 @@ static int gen_kfunc_arg_proto(struct bpf_verifier_env *env, struct bpf_call_arg nargs, MAX_BPF_FUNC_ARGS); return -EINVAL; } - if (nargs > MAX_BPF_FUNC_REG_ARGS && !bpf_jit_supports_stack_args()) { - verbose(env, "JIT does not support kfunc %s() with %d args\n", - meta->func_name, nargs); - return -ENOTSUPP; + + for (i = 0, slots_used = 0; i < nargs; i++) { + u32 nslots = btf_func_model_arg_slots(fm, i); + + if (nslots > 1) { + t = btf_type_skip_modifiers(btf, args[i].type, NULL); + if (!btf_func_arg_align(btf, t)) { + verbose(env, + "Function %s arg#%d type %s nests structs more than " + "%d levels deep\n", + meta->func_name, i, btf_type_str(t), + BTF_MEMBER_MAX_DEPTH); + return -EINVAL; + } + } + slots_used += nslots; } - for (i = 0; i < nargs; i++) { - if (is_kfunc_arg_prog_aux(btf, &args[i]) || - is_kfunc_arg_ignore(btf, &args[i]) || - is_kfunc_arg_implicit(meta, i)) - continue; + if (slots_used > MAX_BPF_FUNC_ARGS) { + verbose(env, "Function %s needs %d > %d argument slots\n", meta->func_name, + slots_used, MAX_BPF_FUNC_ARGS); + return -EINVAL; + } - arg_type = get_kfunc_arg_type(env, meta, args, i, nargs); + abi = bpf_jit_arg_abi(); + + for (i = 0, slots_used = 0; i < nargs; i++) { + u32 nslots = btf_func_model_arg_slots(fm, i); + + if (!abi && nslots > 1) { + t = btf_type_skip_modifiers(btf, args[i].type, NULL); + verbose(env, + "Function %s arg#%d type %s cannot be passed at " + "argument slot %d on this architecture\n", + meta->func_name, i, btf_type_str(t), slots_used); + return -EINVAL; + } + slots_used += nslots; + + arg_type = get_kfunc_arg_type(env, meta, args, i, nargs, + slots_used - nslots, proto); if (arg_type < 0) return arg_type; proto->arg_type[i] = arg_type; } - return 0; + if (slots_used > MAX_BPF_FUNC_REG_ARGS && !bpf_jit_supports_stack_args()) { + verbose(env, "JIT does not support kfunc %s() with %d argument slots\n", + meta->func_name, slots_used); + return -ENOTSUPP; + } + + return check_arg_prog_aux(env, proto) ? 0 : -EINVAL; } -static int process_kf_arg_ptr_to_btf_id(struct bpf_verifier_env *env, - struct bpf_reg_state *reg, - const struct btf_type *ref_t, - const char *ref_tname, u32 ref_id, - struct bpf_call_arg_meta *meta, - int arg, argno_t argno) +static int process_arg_ptr_to_btf_id(struct bpf_verifier_env *env, struct bpf_reg_state *reg, + argno_t argno, enum bpf_arg_type arg_type, + const struct btf *arg_btf, u32 arg_btf_id, + struct bpf_call_arg_meta *meta, int insn_idx) { - const struct btf_type *reg_ref_t; - bool strict_type_match = false; + bool taking_projection, struct_same, strict_type_match = false; + const struct btf_type *arg_t, *reg_t; + const char *arg_tname, *reg_tname; const struct btf *reg_btf; - const char *reg_ref_tname; - bool taking_projection; - bool struct_same; - u32 reg_ref_id; + u32 reg_btf_id; if (base_type(reg->type) == PTR_TO_BTF_ID) { reg_btf = reg->btf; - reg_ref_id = reg->btf_id; + reg_btf_id = reg->btf_id; } else { reg_btf = btf_vmlinux; - reg_ref_id = *reg2btf_ids[base_type(reg->type)]; + reg_btf_id = *reg2btf_ids[base_type(reg->type)]; } - /* Enforce strict type matching for calls to kfuncs that are acquiring - * or releasing a reference, or are no-cast aliases. We do _not_ - * enforce strict matching for kfuncs by default, + /* + * Enforce strict type matching for arguments that release a reference, + * or are no-cast aliases. We do _not_ enforce strict matching by default, * as we want to enable BPF programs to pass types that are bitwise * equivalent without forcing them to explicitly cast with something * like bpf_cast_to_kern_ctx(). @@ -12169,27 +14314,30 @@ static int process_kf_arg_ptr_to_btf_id(struct bpf_verifier_env *env, * btf_struct_ids_match() to walk the struct at the 0th offset, and * resolve types. */ - if ((is_kfunc_release(meta) && reg_is_referenced(env, reg)) || - btf_type_ids_nocast_alias(&env->log, reg_btf, reg_ref_id, meta->btf, ref_id)) + if ((arg_type_is_release(arg_type) && !is_helper_call(meta, BPF_FUNC_sk_release)) || + (is_kfunc(meta) && btf_type_ids_nocast_alias(&env->log, reg_btf, reg_btf_id, + arg_btf, arg_btf_id))) strict_type_match = true; - WARN_ON_ONCE(is_kfunc_release(meta) && !tnum_is_const(reg->var_off)); + arg_t = btf_type_skip_modifiers(arg_btf, arg_btf_id, &arg_btf_id); + arg_tname = btf_name_by_offset(arg_btf, arg_t->name_off); + reg_t = btf_type_skip_modifiers(reg_btf, reg_btf_id, ®_btf_id); + reg_tname = btf_name_by_offset(reg_btf, reg_t->name_off); + + struct_same = btf_struct_ids_match(&env->log, reg_btf, reg_btf_id, + reg->var_off.value, arg_btf, arg_btf_id, + strict_type_match, !type_is_alloc(reg->type)); - reg_ref_t = btf_type_skip_modifiers(reg_btf, reg_ref_id, ®_ref_id); - reg_ref_tname = btf_name_by_offset(reg_btf, reg_ref_t->name_off); - struct_same = btf_struct_ids_match(&env->log, reg_btf, reg_ref_id, reg->var_off.value, - meta->btf, ref_id, strict_type_match, - !type_is_alloc(reg->type)); /* If kfunc is accepting a projection type (ie. __sk_buff), it cannot * actually use it -- it must cast to the underlying type. So we allow * caller to pass in the underlying type. */ - taking_projection = btf_is_projection_of(ref_tname, reg_ref_tname); + taking_projection = is_kfunc(meta) && btf_is_projection_of(arg_tname, reg_tname); if (!taking_projection && !struct_same) { - verbose(env, "kernel function %s %s expected pointer to %s %s but %s has a pointer to %s %s\n", + verbose(env, "%s %s expected pointer to %s %s but %s has a pointer to %s %s\n", meta->func_name, reg_arg_name(env, argno), - btf_type_str(ref_t), ref_tname, reg_arg_name(env, argno), - btf_type_str(reg_ref_t), reg_ref_tname); + btf_type_str(arg_t), arg_tname, + reg_arg_name(env, argno), btf_type_str(reg_t), reg_tname); return -EINVAL; } return 0; @@ -12201,15 +14349,15 @@ static int process_irq_flag(struct bpf_verifier_env *env, struct bpf_reg_state * int err, spi, kfunc_class = IRQ_NATIVE_KFUNC; bool irq_save; - if (meta->func_id == special_kfunc_list[KF_bpf_local_irq_save] || - meta->func_id == special_kfunc_list[KF_bpf_res_spin_lock_irqsave]) { + if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_local_irq_save]) || + is_kfunc_call(meta, special_kfunc_list[KF_bpf_res_spin_lock_irqsave])) { irq_save = true; - if (meta->func_id == special_kfunc_list[KF_bpf_res_spin_lock_irqsave]) + if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_res_spin_lock_irqsave])) kfunc_class = IRQ_LOCK_KFUNC; - } else if (meta->func_id == special_kfunc_list[KF_bpf_local_irq_restore] || - meta->func_id == special_kfunc_list[KF_bpf_res_spin_unlock_irqrestore]) { + } else if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_local_irq_restore]) || + is_kfunc_call(meta, special_kfunc_list[KF_bpf_res_spin_unlock_irqrestore])) { irq_save = false; - if (meta->func_id == special_kfunc_list[KF_bpf_res_spin_unlock_irqrestore]) + if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_res_spin_unlock_irqrestore])) kfunc_class = IRQ_LOCK_KFUNC; } else { verifier_bug(env, "unknown irq flags kfunc"); @@ -12402,12 +14550,20 @@ static bool is_bpf_rbtree_api_kfunc(u32 btf_id) btf_id == special_kfunc_list[KF_bpf_rbtree_right]; } -static bool is_bpf_res_spin_lock_kfunc(u32 btf_id) +static int get_bpf_res_spin_lock_kfunc_flags(const struct bpf_call_arg_meta *meta) { - return btf_id == special_kfunc_list[KF_bpf_res_spin_lock] || - btf_id == special_kfunc_list[KF_bpf_res_spin_unlock] || - btf_id == special_kfunc_list[KF_bpf_res_spin_lock_irqsave] || - btf_id == special_kfunc_list[KF_bpf_res_spin_unlock_irqrestore]; + int flags = PROCESS_RES_LOCK; + + if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_res_spin_lock]) || + is_kfunc_call(meta, special_kfunc_list[KF_bpf_res_spin_lock_irqsave])) + flags |= PROCESS_SPIN_LOCK; + else if (!is_kfunc_call(meta, special_kfunc_list[KF_bpf_res_spin_unlock]) && + !is_kfunc_call(meta, special_kfunc_list[KF_bpf_res_spin_unlock_irqrestore])) + return 0; + if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_res_spin_lock_irqsave]) || + is_kfunc_call(meta, special_kfunc_list[KF_bpf_res_spin_unlock_irqrestore])) + flags |= PROCESS_LOCK_IRQ; + return flags; } static bool kfunc_spin_allowed(struct bpf_verifier_env *env, s32 func_id, s16 offset) @@ -12422,6 +14578,55 @@ static bool kfunc_spin_allowed(struct bpf_verifier_env *env, s32 func_id, s16 of return *kfunc.flags & KF_SPINLOCK_SAFE; } +/* + * True if insn calls a helper/kfunc that requires one of its arguments to + * be a stack pointer with a constant offset. + */ +bool bpf_needs_fixed_stack_off(struct bpf_verifier_env *env, int insn_idx) +{ + const struct bpf_insn *insn = &env->prog->insnsi[insn_idx]; + const struct bpf_func_proto *fn; + struct bpf_kfunc_desc *desc; + u32 *flags, btf_id; + int i; + + if (bpf_helper_call(insn)) { + if (bpf_get_helper_proto(env, insn->imm, &fn) < 0) + return false; + } else if (bpf_pseudo_kfunc_call(insn)) { + desc = find_kfunc_desc(env->prog, insn->imm, insn->off); + if (!desc) + return false; + fn = &desc->proto; + } else { + return false; + } + + /* Both initialized and uninitialized stack dynptrs need a fixed offset. */ + for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++) + if (arg_type_is_dynptr(fn->arg_type[i])) + return true; + + /* vmlinux kfuncs only */ + if (!bpf_pseudo_kfunc_call(insn) || insn->off != 0) + return false; + btf_id = insn->imm; + + flags = btf_kfunc_flags(btf_vmlinux, btf_id, env->prog); + if (flags && (*flags & (KF_ITER_NEW | KF_ITER_NEXT | KF_ITER_DESTROY))) + return true; + + if (btf_id == special_kfunc_list[KF_bpf_res_spin_lock] || + btf_id == special_kfunc_list[KF_bpf_res_spin_unlock] || + btf_id == special_kfunc_list[KF_bpf_res_spin_lock_irqsave] || + btf_id == special_kfunc_list[KF_bpf_res_spin_unlock_irqrestore] || + btf_id == special_kfunc_list[KF_bpf_local_irq_save] || + btf_id == special_kfunc_list[KF_bpf_local_irq_restore]) + return true; + + return false; +} + static bool is_sync_callback_calling_kfunc(u32 btf_id) { return is_bpf_rbtree_add_kfunc(btf_id); @@ -12430,7 +14635,8 @@ static bool is_sync_callback_calling_kfunc(u32 btf_id) static bool is_async_callback_calling_kfunc(u32 btf_id) { return is_bpf_wq_set_callback_kfunc(btf_id) || - is_task_work_add_kfunc(btf_id); + is_task_work_add_kfunc(btf_id) || + is_call_rcu_kfunc(btf_id); } bool bpf_is_throw_kfunc(struct bpf_insn *insn) @@ -12684,707 +14890,6 @@ static bool check_css_task_iter_allowlist(struct bpf_verifier_env *env) } } -static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, - int insn_idx) -{ - const char *func_name = meta->func_name, *ref_tname; - struct bpf_func_state *caller = cur_func(env); - struct bpf_reg_state *regs = cur_regs(env); - const struct btf *btf = meta->btf; - const struct btf_param *args; - struct btf_record *rec; - u32 i, nargs; - int ret; - - args = (const struct btf_param *)(meta->func_proto + 1); - nargs = btf_type_vlen(meta->func_proto); - - ret = check_outgoing_stack_args(env, caller, nargs, func_name, btf, args); - if (ret) - return ret; - - /* Check that BTF function arguments match actual types that the - * verifier sees. - */ - for (i = 0; i < nargs; i++) { - struct bpf_reg_state *reg = get_func_arg_reg(caller, regs, i); - const struct btf_type *t, *ref_t, *resolve_ret; - enum bpf_arg_type arg_type = ARG_DONTCARE; - argno_t argno = argno_from_arg(i + 1); - int regno = reg_from_argno(argno); - bool btf_id_fixed_off_ok = true; - u32 ref_id = args[i].type, type_size; - int kf_arg_type = meta->fn->arg_type[i]; - - if (is_kfunc_arg_prog_aux(btf, &args[i])) { - /* Reject repeated use bpf_prog_aux */ - if (meta->arg_prog) { - verifier_bug(env, "Only 1 prog->aux argument supported per-kfunc"); - return -EFAULT; - } - if (regno < 0) { - verbose(env, "%s prog->aux cannot be a stack argument\n", - reg_arg_name(env, argno)); - return -EINVAL; - } - meta->arg_prog = true; - cur_aux(env)->arg_prog = regno; - continue; - } - - if (is_kfunc_arg_ignore(btf, &args[i]) || is_kfunc_arg_implicit(meta, i)) - continue; - - t = btf_type_skip_modifiers(btf, args[i].type, NULL); - - if (btf_type_is_ptr(t)) { - ref_t = btf_type_skip_modifiers(btf, t->type, &ref_id); - ref_tname = btf_name_by_offset(btf, ref_t->name_off); - } - - if (btf_type_is_ptr(t) && - (bpf_register_is_null(reg) || type_may_be_null(reg->type)) && - !type_may_be_null(kf_arg_type)) { - const char *expected_type; - - expected_type = bpf_diag_fmt_btf_type(env, btf, args[i].type); - verbose(env, "Possibly NULL pointer passed to trusted %s\n", - reg_arg_name(env, argno)); - bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, - "Add a NULL check and call the kfunc only on the non-NULL path.", - "the pointer may be NULL, but this kfunc requires a non-NULL value of type %s", - expected_type); - return -EACCES; - } - - if (regno == meta->release_regno && !is_kfunc_arg_dynptr(meta->btf, &args[i]) && - !reg_is_referenced(env, reg) && !bpf_register_is_null(reg)) { - const char *expected_type; - - expected_type = bpf_diag_fmt_btf_type(env, btf, ref_id); - verbose(env, "release kfunc %s expects referenced PTR_TO_BTF_ID passed to %s\n", - func_name, reg_arg_name(env, argno)); - bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, - "Pass the resource-owning pointer returned by the matching acquire kfunc, and avoid calling the release kfunc after ownership has already been transferred or released.", - "release kfuncs require a resource-owning value of type %s returned by a matching acquire kfunc", - expected_type); - return -EINVAL; - } - - if (reg_is_referenced(env, reg)) - update_ref_obj(&meta->ref_obj, reg); - - if (bpf_register_is_null(reg) && type_may_be_null(kf_arg_type)) { - ret = mark_arg_precision(env, argno); - if (ret) - return ret; - continue; - } - - if (is_kfunc_arg_map(btf, &args[i])) { - ref_id = *reg2btf_ids[CONST_PTR_TO_MAP]; - ref_t = btf_type_by_id(btf_vmlinux, ref_id); - ref_tname = btf_name_by_offset(btf, ref_t->name_off); - } - - switch (base_type(kf_arg_type)) { - case KF_ARG_CONST: - case KF_ARG_CONST_MEM_SIZE: - case KF_ARG_MEM_SIZE: - case KF_ARG_ANYTHING: - case KF_ARG_CONST_ALLOC_SIZE_OR_ZERO: - case KF_ARG_PTR_TO_ALLOC_BTF_ID: - case KF_ARG_PTR_TO_BTF_ID: - case KF_ARG_CONST_MAP_PTR: - case KF_ARG_PTR_TO_ITER: - case KF_ARG_PTR_TO_LIST_HEAD: - case KF_ARG_PTR_TO_LIST_NODE: - case KF_ARG_PTR_TO_RB_ROOT: - case KF_ARG_PTR_TO_RB_NODE: - case KF_ARG_PTR_TO_MEM: - case KF_ARG_PTR_TO_CALLBACK: - case KF_ARG_PTR_TO_CONST_STR: - case KF_ARG_PTR_TO_WORKQUEUE: - case KF_ARG_PTR_TO_TIMER: - case KF_ARG_PTR_TO_TASK_WORK: - case KF_ARG_PTR_TO_IRQ_FLAG: - case KF_ARG_PTR_TO_RES_SPIN_LOCK: - case KF_ARG_PTR_TO_ARENA: - break; - case KF_ARG_PTR_TO_DYNPTR: - arg_type = ARG_PTR_TO_DYNPTR; - break; - case KF_ARG_PTR_TO_CTX: - arg_type = ARG_PTR_TO_CTX; - break; - case KF_ARG_PTR_TO_REFCOUNTED_KPTR: - arg_type = ARG_PTR_TO_BTF_ID; - btf_id_fixed_off_ok = false; - break; - default: - verifier_bug(env, "unknown kfunc arg type %d", kf_arg_type); - return -EFAULT; - } - - if (regno == meta->release_regno) - arg_type |= OBJ_RELEASE; - ret = __check_func_arg_reg_off(env, reg, argno, arg_type, - btf_id_fixed_off_ok); - if (ret < 0) - return ret; - - switch (base_type(kf_arg_type)) { - case KF_ARG_CONST: - if (reg->type != SCALAR_VALUE) { - verbose(env, "%s is not a scalar\n", reg_arg_name(env, argno)); - bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, - "Pass an integer scalar value for this argument, not a pointer or resource object.", - "the kfunc expects an integer scalar, but %s is %s", - reg_arg_name(env, argno), - bpf_diag_reg_type_plain(env, reg->type)); - return -EINVAL; - } - - ret = process_const_arg(env, reg, argno, meta); - if (ret < 0) { - if (ret == -EINVAL) - bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, - "Pass a compile-time constant or a value the verifier can prove is constant at this call.", - "the kfunc requires this scalar argument to be a verifier-known constant, but %s is variable on this path", - reg_arg_name(env, argno)); - return ret; - } - break; - case KF_ARG_ANYTHING: - if (reg->type != SCALAR_VALUE) { - verbose(env, "%s is not a scalar\n", reg_arg_name(env, argno)); - bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, - "Pass an integer scalar value for this argument, not a pointer or resource object.", - "the kfunc expects an integer scalar, but %s is %s", - reg_arg_name(env, argno), - bpf_diag_reg_type_plain(env, reg->type)); - return -EINVAL; - } - break; - case KF_ARG_CONST_ALLOC_SIZE_OR_ZERO: - if (reg->type != SCALAR_VALUE) { - verbose(env, "%s is not a scalar\n", reg_arg_name(env, argno)); - bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, - "Pass an integer scalar value for this argument, not a pointer or resource object.", - "the kfunc expects an integer scalar, but %s is %s", - reg_arg_name(env, argno), - bpf_diag_reg_type_plain(env, reg->type)); - return -EINVAL; - } - - if (is_kfunc_arg_scalar_with_name(btf, &args[i], "rdonly_buf_size")) - meta->r0_rdonly = true; - ret = process_const_alloc_mem_size(env, reg, argno, &meta->ret_mem); - if (ret < 0) { - if (ret == -EINVAL) - bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, - "Pass a verifier-known constant size for this kfunc buffer argument.", - "the kfunc uses this argument as a return-buffer size, but %s is invalid or variable on this path", - reg_arg_name(env, argno)); - return ret; - } - break; - case KF_ARG_PTR_TO_CTX: - if (reg->type != PTR_TO_CTX) { - verbose(env, "%s expected pointer to ctx, but got %s\n", - reg_arg_name(env, argno), reg_type_str(env, reg->type)); - bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, - "Pass the original program context pointer or preserve it before modifying registers.", - "the kfunc expects a context pointer, but %s is %s", - reg_arg_name(env, argno), - bpf_diag_reg_type_plain(env, reg->type)); - return -EINVAL; - } - - if (meta->func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx]) { - ret = get_kern_ctx_btf_id(&env->log, resolve_prog_type(env->prog)); - if (ret < 0) - return -EINVAL; - meta->ret_btf_id = ret; - } - break; - case KF_ARG_PTR_TO_ARENA: - if (reg->type != PTR_TO_ARENA && reg->type != SCALAR_VALUE) { - verbose(env, "%s is not a pointer to arena or scalar\n", - reg_arg_name(env, argno)); - return -EINVAL; - } - break; - case KF_ARG_PTR_TO_ALLOC_BTF_ID: - if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC)) { - if (!is_bpf_obj_drop_kfunc(meta->func_id)) { - verbose(env, "%s expected for bpf_obj_drop()\n", - reg_arg_name(env, argno)); - return -EINVAL; - } - } else if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC | MEM_PERCPU)) { - if (!is_bpf_percpu_obj_drop_kfunc(meta->func_id)) { - verbose(env, "%s expected for bpf_percpu_obj_drop()\n", - reg_arg_name(env, argno)); - return -EINVAL; - } - } else { - verbose(env, "%s expected pointer to allocated object\n", - reg_arg_name(env, argno)); - bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, - "Pass a pointer returned by the matching BPF object allocation path.", - "the kfunc expects an allocated object pointer, but %s is %s", - reg_arg_name(env, argno), - bpf_diag_reg_type_plain(env, reg->type)); - return -EINVAL; - } - if (!reg_is_referenced(env, reg)) { - verbose(env, "allocated object must be referenced\n"); - bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, - "Pass the owned object pointer before it is released or transferred.", - "the allocated object pointer in %s must still carry verifier-tracked ownership, but this pointer no longer owns a live resource", - reg_arg_name(env, argno)); - return -EINVAL; - } - if (meta->btf == btf_vmlinux) { - meta->arg_btf = reg->btf; - meta->arg_btf_id = reg->btf_id; - } - break; - case KF_ARG_PTR_TO_DYNPTR: - { - enum bpf_arg_type dynptr_arg_type = ARG_PTR_TO_DYNPTR; - - if (is_kfunc_arg_uninit(btf, &args[i])) - dynptr_arg_type |= MEM_UNINIT; - - if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_from_skb]) { - dynptr_arg_type |= DYNPTR_TYPE_SKB; - } else if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_from_xdp]) { - dynptr_arg_type |= DYNPTR_TYPE_XDP; - } else if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_from_skb_meta]) { - dynptr_arg_type |= DYNPTR_TYPE_SKB_META; - } else if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_from_file]) { - dynptr_arg_type |= DYNPTR_TYPE_FILE; - } else if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_file_discard]) { - dynptr_arg_type |= DYNPTR_TYPE_FILE | OBJ_RELEASE; - } else if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_clone] && - (dynptr_arg_type & MEM_UNINIT)) { - enum bpf_dynptr_type parent_type = meta->dynptr.type; - - if (parent_type == BPF_DYNPTR_TYPE_INVALID) { - verifier_bug(env, "no dynptr type for parent of clone"); - return -EFAULT; - } - - dynptr_arg_type |= (unsigned int)get_dynptr_type_flag(parent_type); - } - - ret = process_dynptr_func(env, reg, argno, insn_idx, func_name, - dynptr_arg_type, &meta->ref_obj, &meta->dynptr); - if (ret < 0) - return ret; - break; - } - case KF_ARG_PTR_TO_ITER: - if (meta->func_id == special_kfunc_list[KF_bpf_iter_css_task_new]) { - if (!check_css_task_iter_allowlist(env)) { - verbose(env, "css_task_iter is only allowed in bpf_lsm, bpf_iter and sleepable progs\n"); - return -EINVAL; - } - } - ret = process_iter_arg(env, reg, argno, insn_idx, meta); - if (ret < 0) - return ret; - break; - case KF_ARG_PTR_TO_LIST_HEAD: - if (reg->type != PTR_TO_MAP_VALUE && - reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) { - verbose(env, "%s expected pointer to map value or allocated object\n", - reg_arg_name(env, argno)); - return -EINVAL; - } - if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC) && - !reg_is_referenced(env, reg)) { - verbose(env, "allocated object must be referenced\n"); - return -EINVAL; - } - ret = process_kf_arg_ptr_to_list_head(env, reg, argno, meta); - if (ret < 0) - return ret; - break; - case KF_ARG_PTR_TO_RB_ROOT: - if (reg->type != PTR_TO_MAP_VALUE && - reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) { - verbose(env, "%s expected pointer to map value or allocated object\n", - reg_arg_name(env, argno)); - return -EINVAL; - } - if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC) && - !reg_is_referenced(env, reg)) { - verbose(env, "allocated object must be referenced\n"); - return -EINVAL; - } - ret = process_kf_arg_ptr_to_rbtree_root(env, reg, argno, meta); - if (ret < 0) - return ret; - break; - case KF_ARG_PTR_TO_LIST_NODE: - if (is_kfunc_arg_nonown_allowed(btf, &args[i]) && - type_is_non_owning_ref(reg->type) && !reg_is_referenced(env, reg)) { - /* Allow bpf_list_front/back return value for - * __nonown_allowed list-node arguments. - */ - goto check_ok; - } - if (reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) { - verbose(env, "%s expected pointer to allocated object\n", - reg_arg_name(env, argno)); - return -EINVAL; - } - if (!reg_is_referenced(env, reg)) { - verbose(env, "allocated object must be referenced\n"); - return -EINVAL; - } -check_ok: - ret = process_kf_arg_ptr_to_list_node(env, reg, argno, meta); - if (ret < 0) - return ret; - break; - case KF_ARG_PTR_TO_RB_NODE: - if (is_bpf_rbtree_add_kfunc(meta->func_id)) { - if (reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) { - verbose(env, "%s expected pointer to allocated object\n", - reg_arg_name(env, argno)); - return -EINVAL; - } - if (!reg_is_referenced(env, reg)) { - verbose(env, "allocated object must be referenced\n"); - return -EINVAL; - } - } else { - if (!type_is_non_owning_ref(reg->type) && - !reg_is_referenced(env, reg)) { - verbose(env, "%s can only take non-owning or refcounted bpf_rb_node pointer\n", func_name); - return -EINVAL; - } - if (in_rbtree_lock_required_cb(env)) { - verbose(env, "%s not allowed in rbtree cb\n", func_name); - return -EINVAL; - } - } - - ret = process_kf_arg_ptr_to_rbtree_node(env, reg, argno, meta); - if (ret < 0) - return ret; - break; - case KF_ARG_CONST_MAP_PTR: - if (base_type(reg->type) != CONST_PTR_TO_MAP || - type_may_be_null(reg->type)) { - verbose(env, "pointer in %s isn't map pointer\n", - reg_arg_name(env, argno)); - return -EINVAL; - } - ret = process_map_ptr_arg(env, reg, argno, meta); - if (ret < 0) - return ret; - break; - case KF_ARG_PTR_TO_BTF_ID: - /* Only base_type is checked, further checks are done here */ - if (base_type(reg->type) == PTR_TO_BTF_ID || - reg2btf_ids[base_type(reg->type)]) { - if (!is_trusted_reg(env, reg) || - bpf_type_has_unsafe_modifiers(reg->type)) { - if (!is_kfunc_rcu(meta)) { - const char *expected_type; - - expected_type = bpf_diag_fmt_btf_type(env, btf, ref_id); - verbose(env, "%s must be referenced or trusted\n", - reg_arg_name(env, argno)); - bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, - "Pass a pointer acquired from a verifier-tracked source, or call this kfunc only inside the required protection if it accepts RCU pointers.", - "the kfunc requires a trusted or resource-owning pointer to %s, but %s is %s", - expected_type, - reg_arg_name(env, argno), - bpf_diag_reg_type_plain(env, reg->type)); - return -EINVAL; - } - if (!is_rcu_reg(reg)) { - const char *expected_type; - - expected_type = bpf_diag_fmt_btf_type(env, btf, ref_id); - verbose(env, "%s must be a rcu pointer\n", - reg_arg_name(env, argno)); - bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, - "Use this kfunc with a pointer that is valid in an RCU read lock region.", - "the kfunc requires an RCU-protected pointer to %s, but %s is %s", - expected_type, - reg_arg_name(env, argno), - bpf_diag_reg_type_plain(env, reg->type)); - return -EINVAL; - } - } - - ret = process_kf_arg_ptr_to_btf_id(env, reg, ref_t, ref_tname, ref_id, meta, i, argno); - if (ret < 0) - return ret; - break; - } - - if (!__btf_type_is_scalar_struct(env, meta->btf, ref_t, 0)) { - enum bpf_reg_type reg2btf_type = lookup_reg2btf_ids(ref_id); - const char *expected_type; - - verbose(env, "%s is %s expected %s %s", - reg_arg_name(env, argno), reg_type_str(env, reg->type), - btf_type_str(ref_t), ref_tname); - if (reg2btf_type != NOT_INIT) - verbose(env, " or %s", reg_type_str(env, reg2btf_type)); - verbose(env, "\n"); - expected_type = bpf_diag_fmt_btf_type(env, btf, ref_id); - bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, - "Pass a verifier-tracked pointer to the expected kernel object type, not a pointer to stack storage or another memory buffer.", - "the kfunc expects a pointer to %s, but this argument is %s and cannot be used as that kernel object pointer", - expected_type, - bpf_diag_reg_type_plain(env, reg->type)); - return -EINVAL; - } - - /* - * If the register does not contain btf id but the argument type is a pointer to - * scalar-only struct, allow verifying it as a fixed size memory. - */ - kf_arg_type = KF_ARG_PTR_TO_MEM | MEM_FIXED_SIZE; - fallthrough; - case KF_ARG_PTR_TO_MEM: - if (kf_arg_type & MEM_FIXED_SIZE) { - bool known_memory; - - resolve_ret = btf_resolve_size(btf, ref_t, &type_size); - if (IS_ERR(resolve_ret)) { - verbose(env, "%s reference type('%s %s') size cannot be determined: %ld\n", - reg_arg_name(env, argno), btf_type_str(ref_t), - ref_tname, PTR_ERR(resolve_ret)); - return -EINVAL; - } - ret = check_mem_reg(env, reg, argno, type_size, BPF_READ | BPF_WRITE, - meta, &known_memory); - if (ret < 0) { - const char *expected_type; - - expected_type = bpf_diag_fmt_btf_type(env, btf, ref_id); - if (known_memory) - bpf_diag_call_arg_fmt( - env, insn_idx, argno, func_name, - "Pass memory with at least the required number of accessible bytes and suitable read and write access.", - "the kfunc expects %u bytes of memory for %s, but the verifier cannot prove that %s provides a readable and writable range of that size", - type_size, expected_type, - bpf_diag_reg_type_plain(env, reg->type)); - else - bpf_diag_call_arg_fmt( - env, insn_idx, argno, func_name, - "Pass stack, map, context, or other verifier-known memory of the expected type and size, not an integer cast to a pointer.", - "the kfunc expects %u bytes of memory for %s, but it is %s and not verifier-known memory", - type_size, expected_type, - bpf_diag_reg_type_plain(env, reg->type)); - return ret; - } - } - break; - case KF_ARG_CONST_MEM_SIZE: - ret = process_const_arg(env, reg, argno, meta); - if (ret < 0) { - if (ret == -EINVAL) - bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, - "Pass a compile-time constant or a value the verifier can prove is constant at this call.", - "the kfunc requires this memory size to be a verifier-known constant, but %s is variable on this path", - reg_arg_name(env, argno)); - return ret; - } - fallthrough; - case KF_ARG_MEM_SIZE: - { - struct bpf_reg_state *buff_reg = get_func_arg_reg(caller, regs, i - 1); - struct bpf_reg_state *size_reg = reg; - argno_t buff_argno = argno_from_arg(i); - enum bpf_mem_size_failure failure; - - if (reg->type != SCALAR_VALUE) { - verbose(env, "%s is not a scalar\n", reg_arg_name(env, argno)); - bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, - "Pass an integer scalar length for this memory argument.", - "the kfunc expects a scalar memory size, but %s is %s", - reg_arg_name(env, argno), - bpf_diag_reg_type_plain(env, reg->type)); - return -EINVAL; - } - - if (bpf_register_is_null(buff_reg)) - break; - - ret = check_mem_size_reg(env, buff_reg, size_reg, buff_argno, argno, - BPF_READ | BPF_WRITE, true, meta, &failure); - if (ret < 0) { - const char *buff_arg, *size_arg; - - buff_arg = bpf_diag_arg_name(env, buff_argno); - size_arg = bpf_diag_arg_name(env, argno); - verbose(env, "%s and ", reg_arg_name(env, buff_argno)); - verbose(env, "%s memory, len pair leads to invalid memory access\n", - reg_arg_name(env, argno)); - if (failure == BPF_MEM_SIZE_FAIL_MEMORY) { - bpf_diag_call_arg_fmt(env, insn_idx, buff_argno, func_name, - "Pass a stack, map, context, or other verifier-known memory pointer, and keep the paired length within that object.", - "it is the memory pointer in a memory/length pair with %s, but %s does not describe verifier-readable memory for the requested length", - size_arg, buff_arg); - } else if (failure == BPF_MEM_SIZE_FAIL_SIZE) { - if (reg_smin(size_reg) < 0) - bpf_diag_call_arg_fmt( - env, insn_idx, argno, func_name, - "Constrain the memory size to a non-negative value smaller than BPF_MAX_VAR_SIZ before this call.", - "the memory size in %s may be negative because its signed minimum is %lld", - size_arg, reg_smin(size_reg)); - else - bpf_diag_call_arg_fmt( - env, insn_idx, argno, func_name, - "Constrain the memory size to a non-negative value smaller than BPF_MAX_VAR_SIZ before this call.", - "the memory size in %s may reach %llu bytes, but variable memory accesses must stay below %u bytes", - size_arg, reg_umax(size_reg), BPF_MAX_VAR_SIZ); - } - return ret; - } - break; - } - case KF_ARG_PTR_TO_CALLBACK: - if (reg->type != PTR_TO_FUNC) { - verbose(env, "%s expected pointer to func\n", reg_arg_name(env, argno)); - return -EINVAL; - } - meta->subprogno = reg->subprogno; - break; - case KF_ARG_PTR_TO_REFCOUNTED_KPTR: - if (!type_is_ptr_alloc_obj(reg->type)) { - verbose(env, "%s is neither owning or non-owning ref\n", - reg_arg_name(env, argno)); - bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, - "Pass an owning or non-owning pointer to a BPF-managed object containing a bpf_refcount field.", - "the kfunc expects a pointer to a BPF-managed refcounted object, but %s is %s", - reg_arg_name(env, argno), - bpf_diag_reg_type_plain(env, reg->type)); - return -EINVAL; - } - if (!type_is_non_owning_ref(reg->type) && reg_is_referenced(env, reg)) - meta->arg_owning_ref = true; - - rec = reg_btf_record(reg); - if (!rec) { - verifier_bug(env, "Couldn't find btf_record"); - return -EFAULT; - } - - if (rec->refcount_off < 0) { - verbose(env, "%s doesn't point to a type with bpf_refcount field\n", - reg_arg_name(env, argno)); - return -EINVAL; - } - - meta->arg_btf = reg->btf; - meta->arg_btf_id = reg->btf_id; - break; - case KF_ARG_PTR_TO_CONST_STR: - if (reg->type != PTR_TO_MAP_VALUE) { - verbose(env, "%s doesn't point to a const string\n", - reg_arg_name(env, argno)); - bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, - "Pass a constant string pointer that the verifier recognizes, such as a string stored in a read-only map value.", - "the kfunc expects a pointer to a constant string stored in verifier-known memory, but %s is %s", - reg_arg_name(env, argno), - bpf_diag_reg_type_plain(env, reg->type)); - return -EINVAL; - } - ret = check_arg_const_str(env, reg, argno); - if (ret) - return ret; - break; - case KF_ARG_PTR_TO_WORKQUEUE: - if (reg->type != PTR_TO_MAP_VALUE) { - verbose(env, "%s doesn't point to a map value\n", - reg_arg_name(env, argno)); - return -EINVAL; - } - ret = check_map_field_pointer(env, reg, argno, BPF_WORKQUEUE, &meta->map); - if (ret < 0) - return ret; - break; - case KF_ARG_PTR_TO_TIMER: - if (reg->type != PTR_TO_MAP_VALUE) { - verbose(env, "%s doesn't point to a map value\n", - reg_arg_name(env, argno)); - return -EINVAL; - } - ret = process_timer_func(env, reg, argno, &meta->map); - if (ret < 0) - return ret; - break; - case KF_ARG_PTR_TO_TASK_WORK: - if (reg->type != PTR_TO_MAP_VALUE) { - verbose(env, "%s doesn't point to a map value\n", - reg_arg_name(env, argno)); - return -EINVAL; - } - ret = check_map_field_pointer(env, reg, argno, BPF_TASK_WORK, &meta->map); - if (ret < 0) - return ret; - break; - case KF_ARG_PTR_TO_IRQ_FLAG: - if (reg->type != PTR_TO_STACK) { - verbose(env, "%s doesn't point to an irq flag on stack\n", - reg_arg_name(env, argno)); - bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, - "Pass the same stack slot used by bpf_local_irq_save() or bpf_res_spin_lock_irqsave().", - "the kfunc expects a stack pointer to an IRQ flag slot, but %s is %s", - reg_arg_name(env, argno), - bpf_diag_reg_type_plain(env, reg->type)); - return -EINVAL; - } - ret = process_irq_flag(env, reg, argno, meta); - if (ret < 0) - return ret; - break; - case KF_ARG_PTR_TO_RES_SPIN_LOCK: - { - int flags = PROCESS_RES_LOCK; - - if (in_rbtree_lock_required_cb(env)) { - verbose(env, "can't res_spin_{lock,unlock} in rbtree cb\n"); - return -EACCES; - } - - if (reg->type != PTR_TO_MAP_VALUE && reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) { - verbose(env, "%s doesn't point to map value or allocated object\n", - reg_arg_name(env, argno)); - return -EINVAL; - } - - if (!is_bpf_res_spin_lock_kfunc(meta->func_id)) - return -EFAULT; - if (meta->func_id == special_kfunc_list[KF_bpf_res_spin_lock] || - meta->func_id == special_kfunc_list[KF_bpf_res_spin_lock_irqsave]) - flags |= PROCESS_SPIN_LOCK; - if (meta->func_id == special_kfunc_list[KF_bpf_res_spin_lock_irqsave] || - meta->func_id == special_kfunc_list[KF_bpf_res_spin_unlock_irqrestore]) - flags |= PROCESS_LOCK_IRQ; - ret = process_spin_lock(env, reg, argno, flags); - if (ret < 0) - return ret; - break; - } - } - } - - return 0; -} - int bpf_fetch_kfunc_arg_meta(struct bpf_verifier_env *env, s32 func_id, s16 offset, @@ -13417,27 +14922,32 @@ int bpf_fetch_kfunc_arg_meta(struct bpf_verifier_env *env, } /* - * Determine how many bytes a helper accesses through a stack pointer at - * argument position @arg (0-based, corresponding to R1-R5). - * - * Returns: - * > 0 known read access size in bytes - * 0 doesn't read anything directly - * S64_MIN unknown - * < 0 known write access of (-return) bytes + * Describe a helper's stack access through argument @arg (0-based, R1-R5). + * must_write means the verifier destroys prior state throughout size bytes. + * In other words, must_write follows the verifier's model of the + * function's behaviour, which is safe to use for is_state_visited() pruning. */ -s64 bpf_helper_stack_access_bytes(struct bpf_verifier_env *env, struct bpf_insn *insn, - int arg, int insn_idx) +struct arg_access_info +bpf_helper_stack_access_bytes(struct bpf_verifier_env *env, struct bpf_insn *insn, + int arg, int insn_idx) { struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx]; + struct arg_access_info info = { + .size = U32_MAX, + .may_read = true, + .may_write = true, + }; const struct bpf_func_proto *fn; + enum bpf_access_type access_type; enum bpf_arg_type at; - s64 size; + bool exact_size = true; + u64 size = U32_MAX; if (bpf_get_helper_proto(env, insn->imm, &fn) < 0) - return S64_MIN; + return info; at = fn->arg_type[arg]; + access_type = func_arg_access_type(at); switch (base_type(at)) { case ARG_PTR_TO_MAP_KEY: @@ -13460,6 +14970,10 @@ s64 bpf_helper_stack_access_bytes(struct bpf_verifier_env *env, struct bpf_insn i = aux->const_reg_vals[map_reg]; if (i < env->used_map_cnt) { + /* Bloom-filter peek reads the value buffer. */ + if (!is_key && insn->imm == BPF_FUNC_map_peek_elem && + env->used_maps[i]->map_type == BPF_MAP_TYPE_BLOOM_FILTER) + access_type = BPF_READ; size = is_key ? env->used_maps[i]->key_size : env->used_maps[i]->value_size; goto out; @@ -13469,7 +14983,12 @@ s64 bpf_helper_stack_access_bytes(struct bpf_verifier_env *env, struct bpf_insn * Map pointer is not known at this call site (e.g. different * maps on merged paths). Conservatively return the largest * key_size or value_size across all maps used by the program. + * This is only an upper bound, so it cannot establish a definite + * write. Map-dependent argument types can also turn an output + * into an input, so conservatively retain a read dependency. */ + exact_size = false; + access_type |= BPF_READ; val = 0; for (i = 0; i < env->used_map_cnt; i++) { struct bpf_map *map = env->used_maps[i]; @@ -13485,7 +15004,7 @@ s64 bpf_helper_stack_access_bytes(struct bpf_verifier_env *env, struct bpf_insn } } if (!val) - return S64_MIN; + return info; size = val; goto out; } @@ -13499,18 +15018,12 @@ s64 bpf_helper_stack_access_bytes(struct bpf_verifier_env *env, struct bpf_insn int size_reg = BPF_REG_1 + arg + 1; if (aux->const_reg_mask & BIT(size_reg)) { - size = (s64)aux->const_reg_vals[size_reg]; + size = aux->const_reg_vals[size_reg]; goto out; } - /* - * Size arg is const on each path but differs across merged - * paths. MAX_BPF_STACK is a safe upper bound for reads. - */ - if (at & MEM_UNINIT) - return 0; - return MAX_BPF_STACK; } - return S64_MIN; + /* Preserve access directions even when the extent is unknown. */ + goto out; case ARG_PTR_TO_DYNPTR: size = BPF_DYNPTR_SIZE; break; @@ -13520,71 +15033,86 @@ s64 bpf_helper_stack_access_bytes(struct bpf_verifier_env *env, struct bpf_insn * doesn't access stack. The callback subprog does and it's * analyzed separately. */ - return 0; + return (struct arg_access_info) {}; default: - return S64_MIN; + return info; } out: - /* - * MEM_UNINIT args are write-only: the helper initializes the - * buffer without reading it. - */ - if (at & MEM_UNINIT) - return -size; - return size; + info.size = min_t(u64, size, U32_MAX); + info.may_read = !!(access_type & BPF_READ); + info.may_write = !!(access_type & BPF_WRITE); + info.must_write = info.may_write && exact_size && info.size != U32_MAX; + /* Generic unprivileged outputs retain prior initialization state. */ + if (!env->allow_uninit_stack && arg_type_is_raw_mem(at)) { + info.may_read = true; + info.must_write = false; + } + return info; } /* - * Determine how many bytes a kfunc accesses through a stack pointer at - * argument position @arg (0-based, corresponding to R1-R5). - * - * Returns: - * > 0 known read access size in bytes - * 0 doesn't access memory through that argument (ex: not a pointer) - * S64_MIN unknown - * < 0 known write access of (-return) bytes + * Describe a kfunc's stack access through argument slot @arg (0-based). + * As for helpers, must_write describes destruction of prior verifier state, + * rather than a guarantee that the callee writes every byte at runtime. */ -s64 bpf_kfunc_stack_access_bytes(struct bpf_verifier_env *env, struct bpf_insn *insn, - int arg, int insn_idx) +struct arg_access_info +bpf_kfunc_stack_access_bytes(struct bpf_verifier_env *env, struct bpf_insn *insn, + int arg, int insn_idx) { struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx]; + struct arg_access_info info = { + .size = U32_MAX, + .may_read = true, + .may_write = true, + }; struct bpf_call_arg_meta meta; const struct btf_param *args; const struct btf_type *t, *ref_t; const struct btf *btf; - u32 nargs, type_size; - s64 size; + u32 i, slot, nargs, type_size; + u64 size; if (bpf_fetch_kfunc_arg_meta(env, insn->imm, insn->off, &meta) < 0) - return S64_MIN; + return info; btf = meta.btf; args = btf_params(meta.func_proto); nargs = btf_type_vlen(meta.func_proto); - if (arg >= nargs) - return 0; - t = btf_type_skip_modifiers(btf, args[arg].type, NULL); + /* + * @arg is an argument slot and a 16-byte parameter takes two of them, + * so walk the parameters to find the one that starts at this slot. A + * slot holding the upper eightbyte of such a parameter belongs to no + * pointer, and neither does a slot past the last parameter. + */ + for (i = 0, slot = 0; i < nargs && slot < arg; i++) + slot += btf_arg_slots(btf_type_skip_modifiers(btf, args[i].type, NULL)); + if (i >= nargs || slot != arg) + return (struct arg_access_info) {}; + + t = btf_type_skip_modifiers(btf, args[i].type, NULL); if (!btf_type_is_ptr(t)) - return 0; + return (struct arg_access_info) {}; /* dynptr: fixed 16-byte on-stack representation */ - if (is_kfunc_arg_dynptr(btf, &args[arg])) { + if (is_kfunc_arg_dynptr(btf, &args[i])) { size = BPF_DYNPTR_SIZE; goto out; } - /* ptr + __sz/__szk pair: size is in the next register */ - if (arg + 1 < nargs && - (btf_param_match_suffix(btf, &args[arg + 1], "__sz") || - btf_param_match_suffix(btf, &args[arg + 1], "__szk"))) { + /* ptr + __sz/__szk pair: the size follows the pointer */ + if (i + 1 < nargs && + (btf_param_match_suffix(btf, &args[i + 1], "__sz") || + btf_param_match_suffix(btf, &args[i + 1], "__szk"))) { int size_reg = BPF_REG_1 + arg + 1; - if (aux->const_reg_mask & BIT(size_reg)) { - size = (s64)aux->const_reg_vals[size_reg]; + if (size_reg <= MAX_BPF_FUNC_REG_ARGS && + (aux->const_reg_mask & BIT(size_reg))) { + size = aux->const_reg_vals[size_reg]; goto out; } - return MAX_BPF_STACK; + /* Unknown size: the read may extend anywhere up to the frame top. */ + return info; } /* fixed-size pointed-to type: resolve via BTF */ @@ -13594,14 +15122,99 @@ s64 bpf_kfunc_stack_access_bytes(struct bpf_verifier_env *env, struct bpf_insn * goto out; } - return S64_MIN; + return info; out: + info.size = min_t(u64, size, U32_MAX); /* KF_ITER_NEW kfuncs initialize the iterator state at arg 0 */ if (arg == 0 && meta.kfunc_flags & KF_ITER_NEW) - return -size; - if (is_kfunc_arg_uninit(btf, &args[arg])) - return -size; - return size; + info.may_read = false; + if (is_kfunc_arg_uninit(btf, &args[i]) && + (is_kfunc_arg_dynptr(btf, &args[i]) || env->allow_uninit_stack)) + info.may_read = false; + info.must_write = !info.may_read && info.size != U32_MAX; + return info; +} + +/* + * Called before dead code elimination, some subprograms may be genuinely unreachable. + * Suppress diagnostics so callers can use a conservative result on error. + * The main pass would report errors for reachable subprograms. + */ +static bool try_btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) +{ + struct bpf_prog *prog = env->prog; + u32 log_level = env->log.level; + int err; + + if (!prog->aux->func_info || !prog->aux->func_info[subprog].type_id) + return false; + + env->log.level = 0; + err = btf_prepare_func_args(env, subprog); + env->log.level = log_level; + return !err; +} + +/* Prepare argument metadata for liveness; the main program is handled separately. */ +static void prepare_subprog_args(struct bpf_verifier_env *env) +{ + int i; + + for (i = 1; i < env->subprog_cnt; i++) { + if (!try_btf_prepare_func_args(env, i) && bpf_subprog_is_global(env, i)) + /* Keep all argument registers live if the prototype is unavailable. */ + env->subprog_info[i].arg_slot_cnt = MAX_BPF_FUNC_REG_ARGS; + } +} + +/* + * Derive stack effects for a global subprog's argument slot @arg (0-based) + * from its declared type, matching the main pass's argument checks. + */ +struct arg_access_info +bpf_global_subprog_stack_access_bytes(struct bpf_verifier_env *env, struct bpf_insn *insn, + int arg, int insn_idx) +{ + struct arg_access_info info = { + .size = U32_MAX, + .may_read = true, + .may_write = true, + }; + struct bpf_subprog_info *sub; + int callee; + + callee = bpf_find_subprog(env, insn_idx + insn->imm + 1); + if (callee < 0) + return info; + sub = subprog_info(env, callee); + if (!try_btf_prepare_func_args(env, callee)) + return info; + + switch (base_type(sub->args[arg].arg_type)) { + case ARG_UNUSED: /* Not an argument slot. */ + case ARG_SCALAR: /* A scalar, the callee can't dereference it. */ + return (struct arg_access_info) {}; + case ARG_PTR_TO_BTF_ID: + /* + * Trusted arguments cannot be stack pointers. + * Untrusted arguments use ARG_IGNORE: + * probe reads need no caller stack state, writes are prohibited. + */ + return (struct arg_access_info) {}; + case ARG_PTR_TO_MEM: + /* mem_size is zero for untrusted pointers of unknown size. */ + if (sub->args[arg].mem_size) { + info.size = sub->args[arg].mem_size; + info.must_write = true; + } + return info; + case ARG_PTR_TO_DYNPTR: + /* The callee may adjust the view, but cannot reinitialize the dynptr. */ + info.size = BPF_DYNPTR_SIZE; + return info; + default: + return info; + } } /* check special kfuncs and return: @@ -13678,7 +15291,7 @@ static int check_special_kfunc(struct bpf_verifier_env *env, struct bpf_call_arg struct_meta = btf_find_struct_meta(ret_btf, ret_btf_id); if (is_bpf_percpu_obj_new_kfunc(meta->func_id)) { - if (!__btf_type_is_scalar_struct(env, ret_btf, ret_t, 0)) { + if (!btf_type_is_scalar_struct(env, ret_btf, ret_t)) { verbose(env, "bpf_percpu_obj_new type ID argument must be of a struct of scalars\n"); return -EINVAL; } @@ -13715,12 +15328,12 @@ static int check_special_kfunc(struct bpf_verifier_env *env, struct bpf_call_arg struct btf_field *field = meta->arg_rbtree_root.field; mark_reg_graph_node(regs, BPF_REG_0, &field->graph_root); - } else if (meta->func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx]) { + } else if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_cast_to_kern_ctx])) { mark_reg_known_zero(env, regs, BPF_REG_0); regs[BPF_REG_0].type = PTR_TO_BTF_ID | PTR_TRUSTED; regs[BPF_REG_0].btf = desc_btf; regs[BPF_REG_0].btf_id = meta->ret_btf_id; - } else if (meta->func_id == special_kfunc_list[KF_bpf_rdonly_cast]) { + } else if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_rdonly_cast])) { ret_t = btf_type_by_id(desc_btf, meta->arg_constant.value); if (!ret_t) { verbose(env, "Unknown type ID %lld passed to kfunc bpf_rdonly_cast\n", @@ -13740,8 +15353,8 @@ static int check_special_kfunc(struct bpf_verifier_env *env, struct bpf_call_arg "kfunc bpf_rdonly_cast type ID argument must be of a struct or void\n"); return -EINVAL; } - } else if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_slice] || - meta->func_id == special_kfunc_list[KF_bpf_dynptr_slice_rdwr]) { + } else if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_dynptr_slice]) || + is_kfunc_call(meta, special_kfunc_list[KF_bpf_dynptr_slice_rdwr])) { enum bpf_type_flag type_flag = get_dynptr_type_flag(meta->dynptr.type); mark_reg_known_zero(env, regs, BPF_REG_0); @@ -13756,7 +15369,7 @@ static int check_special_kfunc(struct bpf_verifier_env *env, struct bpf_call_arg /* PTR_MAYBE_NULL will be added when is_kfunc_ret_null is checked */ regs[BPF_REG_0].type = PTR_TO_MEM | type_flag; - if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_slice]) { + if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_dynptr_slice])) { regs[BPF_REG_0].type |= MEM_RDONLY; } else { /* this will set env->seen_direct_write to true */ @@ -13792,7 +15405,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, struct bpf_insn_aux_data *insn_aux; const char *operation; int err, insn_idx = *insn_idx_p; - u32 i, nargs, ptr_type_id; + u32 i, proto_slots, ptr_type_id, ret_nregs = 1; struct bpf_kfunc_desc *desc; struct btf *desc_btf; int id; @@ -13872,11 +15485,20 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, } sleepable = bpf_is_kfunc_sleepable(&meta); - if (sleepable && !in_sleepable(env)) { - verbose(env, "program must be sleepable to call sleepable kfunc %s\n", func_name); + if (sleepable && !in_sleepable_context(env)) { + const char *suggestion; + + if (in_sleepable(env)) { + verbose(env, "kernel func %s is sleepable within %s\n", + func_name, non_sleepable_context_description(env)); + suggestion = "Move the kfunc call outside the critical section, or use a non-sleepable kfunc."; + } else { + verbose(env, "program must be sleepable to call sleepable kfunc %s\n", + func_name); + suggestion = "Mark the program sleepable if the program type allows it, or use a non-sleepable kfunc."; + } operation = bpf_diag_fmt(env, "sleepable kfunc %s", func_name); - bpf_diag_ctx_forbidden(env, insn_idx, operation, - "Mark the program sleepable if the program type allows it, or use a non-sleepable kfunc."); + bpf_diag_ctx_forbidden(env, insn_idx, operation, suggestion); return -EACCES; } @@ -13885,10 +15507,14 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, insn_aux->non_sleepable = true; /* Check the arguments */ - err = check_kfunc_args(env, &meta, insn_idx); + err = check_func_args(env, &meta, insn_idx); if (err < 0) return err; + err = mark_raw_stack(env, &meta, insn_idx); + if (err) + return err; + if ((is_bpf_obj_drop_kfunc(meta.func_id) || is_bpf_percpu_obj_drop_kfunc(meta.func_id)) && (is_tracing_prog_type(prog_type) || /* is_tracing_prog_type() for now doesn't cover non-iterator tracing progs. */ @@ -13934,6 +15560,16 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, } } + if (is_call_rcu_kfunc(meta.func_id)) { + err = push_callback_call(env, insn, insn_idx, meta.subprogno, + set_rcu_callback_state); + if (err) { + verbose(env, "kfunc %s#%d failed callback verification\n", + func_name, meta.func_id); + return err; + } + } + rcu_lock = is_kfunc_bpf_rcu_read_lock(&meta); rcu_unlock = is_kfunc_bpf_rcu_read_unlock(&meta); @@ -13976,15 +15612,6 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, invalidate_rcu_protected_refs(env); } - if (sleepable && !in_sleepable_context(env)) { - verbose(env, "kernel func %s is sleepable within %s\n", - func_name, non_sleepable_context_description(env)); - operation = bpf_diag_fmt(env, "sleepable kfunc %s", func_name); - bpf_diag_ctx_forbidden(env, insn_idx, operation, - "Move the kfunc call outside the critical section, or use a non-sleepable kfunc."); - return -EACCES; - } - if (in_rbtree_lock_required_cb(env) && (rcu_lock || rcu_unlock)) { verbose(env, "Calling bpf_rcu_read_{lock,unlock} in unnecessary rbtree callback\n"); return -EACCES; @@ -14055,10 +15682,63 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, } if (btf_type_is_scalar(t)) { - mark_reg_unknown(env, regs, BPF_REG_0); + ret_nregs = mark_kfunc_ret_regs(env, regs, t->size); if (meta.btf == btf_vmlinux && (meta.func_id == special_kfunc_list[KF_bpf_res_spin_lock] || meta.func_id == special_kfunc_list[KF_bpf_res_spin_lock_irqsave])) __mark_reg_const_zero(env, ®s[BPF_REG_0]); + } else if (btf_type_is_struct(t)) { + struct btf_member_path path = {}; + const char *member_note = ""; + + /* + * The returned struct comes back as raw register bits modeled + * as an unknown scalar, so a pointer member would be laundered + * into a scalar and escape provenance and reference tracking. + * Only scalars and arena pointers are allowed; see + * btf_validate_return_type() in btf.c for why an arena pointer + * is safe here. + */ + if (!btf_struct_member_walk(env, desc_btf, t, + BTF_MEMBER_SCALAR | BTF_MEMBER_ARENA_PTR, 0, &path)) { + verbose(env, + "kernel function %s returns %s %s that is not composed of scalars or arena pointers\n", + func_name, btf_type_str(t), + btf_name_by_offset(desc_btf, t->name_off)); + if (path.too_deep) { + member_note = bpf_diag_fmt( + env, " It nests structs more than %d levels deep.", + BTF_MEMBER_MAX_DEPTH); + } else if (path.depth) { + char bad_name[BTF_MEMBER_PATH_LEN]; + + btf_member_path_str(desc_btf, &path, bad_name, sizeof(bad_name)); + if (!path.bad_type) { + verbose(env, "member '%s' is a zero-length array\n", + bad_name); + member_note = bpf_diag_fmt( + env, " Its member '%s' is a zero-length array.", + bad_name); + } else { + verbose(env, "member '%s' has type %s\n", bad_name, + btf_type_str(path.bad_type)); + member_note = bpf_diag_fmt( + env, + " Its member '%s' is %s, not a scalar or an arena pointer.", + bad_name, btf_type_str(path.bad_type)); + } + } + bpf_diag_program_structure( + env, insn_idx, "unsupported kernel function return type", + "Call a kernel function that returns only scalars or arena pointers by value.", + "%s() returns %s %s by value.%s " + "Only kfuncs returning scalar values or arena pointers, or " + "structures composed of scalar values or arena pointers are " + "supported.", + func_name, btf_type_str(t), + btf_name_by_offset(desc_btf, t->name_off), member_note); + return -EINVAL; + } + ret_nregs = mark_kfunc_ret_regs(env, regs, t->size); } else if (btf_type_is_ptr(t)) { ptr_type = btf_type_skip_modifiers(desc_btf, t->type, &ptr_type_id); err = check_special_kfunc(env, &meta, regs, insn_aux, ptr_type, desc_btf); @@ -14171,14 +15851,24 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, } } - if (bpf_is_kfunc_pkt_changing(&meta)) + if (is_kfunc_dynptr_write(&meta) ? + dynptr_write_changes_pkt_data(meta.dynptr_write_type) : + bpf_is_kfunc_pkt_changing(&meta)) clear_all_pkt_pointers(env); - nargs = btf_type_vlen(meta.func_proto); - if (nargs > MAX_BPF_FUNC_REG_ARGS) { + /* + * These kfuncs write to the head of the skb in place if what they write + * to is linear. Like a direct write that takes bpf_unclone_prologue(). + */ + if (is_kfunc_dynptr_write(&meta) && + dynptr_write_changes_pkt_data(meta.dynptr_write_type)) + may_access_direct_pkt_data(env, NULL, BPF_WRITE); + + proto_slots = kfunc_abi_slots(&desc->func_model); + if (proto_slots > MAX_BPF_FUNC_REG_ARGS) { struct bpf_func_state *caller = cur_func(env); struct bpf_subprog_info *caller_info = &env->subprog_info[caller->subprogno]; - u16 out_stack_arg_cnt = nargs - MAX_BPF_FUNC_REG_ARGS; + u16 out_stack_arg_cnt = proto_slots - MAX_BPF_FUNC_REG_ARGS; u16 stack_arg_cnt = bpf_in_stack_arg_cnt(caller_info) + out_stack_arg_cnt; if (stack_arg_cnt > caller_info->stack_arg_cnt) @@ -14189,7 +15879,8 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, * Record R0 before process_iter_next_call() snapshots the alternate * iterator path's diagnostic position. */ - bpf_diag_mod_end(env); + if (ret_nregs == 1) + bpf_diag_mod_end(env); if (bpf_is_iter_next_kfunc(&meta)) { err = process_iter_next_call(env, insn_idx, &meta); @@ -14266,7 +15957,8 @@ enum { REASON_STACK = -5, }; -static int retrieve_ptr_limit(const struct bpf_reg_state *ptr_reg, +static int retrieve_ptr_limit(const struct bpf_verifier_env *env, + const struct bpf_reg_state *ptr_reg, u32 *alu_limit, bool mask_to_left) { u32 max = 0, ptr_limit = 0; @@ -14278,7 +15970,7 @@ static int retrieve_ptr_limit(const struct bpf_reg_state *ptr_reg, * offset where we would need to deal with min/max bounds is * currently prohibited for unprivileged. */ - max = MAX_BPF_STACK + mask_to_left; + max = env->stack_limit + mask_to_left; ptr_limit = -ptr_reg->var_off.value; break; case PTR_TO_MAP_VALUE: @@ -14398,7 +16090,7 @@ static int sanitize_ptr_alu(struct bpf_verifier_env *env, (opcode == BPF_SUB && !off_is_neg); } - err = retrieve_ptr_limit(ptr_reg, &alu_limit, info->mask_to_left); + err = retrieve_ptr_limit(env, ptr_reg, &alu_limit, info->mask_to_left); if (err < 0) return err; @@ -14528,7 +16220,7 @@ static int check_stack_access_for_ptr_arithmetic( return -EACCES; } - if (off >= 0 || off < -MAX_BPF_STACK) { + if (off >= 0 || off < -(int)env->stack_limit) { verbose(env, "R%d stack pointer arithmetic goes out of range, " "prohibited for !root; off=%d\n", regno, off); return -EACCES; @@ -14569,6 +16261,24 @@ static int sanitize_check_bounds(struct bpf_verifier_env *env, return 0; } +static void scalar_step_mul(struct bpf_reg_state *dst_reg, struct bpf_reg_state *src_reg) +{ + if (tnum_is_const(src_reg->var_off)) + scalar_step_scale(dst_reg, src_reg->var_off.value); + else + reg_step_reset(dst_reg); +} + +static void scalar_step_lsh(struct bpf_reg_state *dst_reg, struct bpf_reg_state *src_reg) +{ + u64 amount = src_reg->var_off.value; + + if (tnum_is_const(src_reg->var_off) && amount < 64) + scalar_step_scale(dst_reg, 1ULL << amount); + else + reg_step_reset(dst_reg); +} + /* Handles arithmetic on a pointer and a scalar: computes new min/max and var_off. * Caller should also handle BPF_MOV case separately. * If we return -EACCES, caller may want to try again treating pointer as a @@ -14589,8 +16299,14 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, struct bpf_insn u32 dst = insn->dst_reg; const char *reason; int ret, bounds_ret; + bool to_scalar; dst_reg = ®s[dst]; + /* + * Only += and -= keep a pointer. Any other op makes a number of it, + * which is fine when the program may see values of pointers anyway. + */ + to_scalar = opcode != BPF_ADD && opcode != BPF_SUB && env->allow_ptr_leaks; if ((known && (smin_val != smax_val || umin_val != umax_val)) || smin_val > smax_val || umin_val > umax_val) { @@ -14601,7 +16317,7 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, struct bpf_insn return 0; } - if (BPF_CLASS(insn->code) != BPF_ALU64) { + if (BPF_CLASS(insn->code) != BPF_ALU64 && !to_scalar) { /* 32-bit ALU ops on pointers produce (meaningless) scalars */ if (opcode == BPF_SUB && env->allow_ptr_leaks) { __mark_reg_unknown(env, dst_reg); @@ -14667,6 +16383,11 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, struct bpf_insn return -EACCES; } + if (to_scalar) { + __mark_reg_unknown(env, dst_reg); + return 0; + } + /* For 'scalar += pointer', dst_reg inherits the complete pointer * register state. Individual fields may be adjusted later by pointer * arithmetic. Callers guarantee that below does not overwrite off_reg. @@ -14726,6 +16447,7 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, struct bpf_insn * added into the variable offset, and we copy the fixed offset * from ptr_reg. */ + scalar_step_add(dst_reg, ptr_reg, off_reg); dst_reg->r64 = cnum64_add(ptr_reg->r64, off_reg->r64); dst_reg->var_off = tnum_add(ptr_reg->var_off, off_reg->var_off); dst_reg->raw = ptr_reg->raw; @@ -14787,6 +16509,7 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, struct bpf_insn if ((!known && smin_val < 0) || dst_reg->range < 0) memset(&dst_reg->raw, 0, sizeof(dst_reg->raw)); } + reg_step_reset(dst_reg); break; case BPF_AND: case BPF_OR: @@ -15494,6 +17217,7 @@ static void scalar_byte_swap(struct bpf_reg_state *dst_reg, struct bpf_insn *ins * Bounds will be re-derived from the new tnum later. */ __mark_reg_unbounded(dst_reg); + reg_step_reset(dst_reg); } /* For bswap16/32, truncate dst register to match the swapped size */ if (insn->imm == 16 || insn->imm == 32) @@ -15620,6 +17344,7 @@ static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env, */ switch (opcode) { case BPF_ADD: + scalar_step_add(dst_reg, dst_reg, &src_reg); scalar32_min_max_add(dst_reg, &src_reg); scalar_min_max_add(dst_reg, &src_reg); dst_reg->var_off = tnum_add(dst_reg->var_off, src_reg.var_off); @@ -15628,6 +17353,7 @@ static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env, scalar32_min_max_sub(dst_reg, &src_reg); scalar_min_max_sub(dst_reg, &src_reg); dst_reg->var_off = tnum_sub(dst_reg->var_off, src_reg.var_off); + reg_step_reset(dst_reg); break; case BPF_NEG: env->fake_reg[0] = *dst_reg; @@ -15635,8 +17361,10 @@ static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env, scalar32_min_max_sub(dst_reg, &env->fake_reg[0]); scalar_min_max_sub(dst_reg, &env->fake_reg[0]); dst_reg->var_off = tnum_neg(env->fake_reg[0].var_off); + reg_step_reset(dst_reg); break; case BPF_MUL: + scalar_step_mul(dst_reg, &src_reg); dst_reg->var_off = tnum_mul(dst_reg->var_off, src_reg.var_off); scalar32_min_max_mul(dst_reg, &src_reg); scalar_min_max_mul(dst_reg, &src_reg); @@ -15657,6 +17385,7 @@ static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env, scalar_min_max_sdiv(dst_reg, &src_reg); else scalar_min_max_udiv(dst_reg, &src_reg); + reg_step_reset(dst_reg); break; case BPF_MOD: /* BPF mod specification: x % 0 = x */ @@ -15672,6 +17401,7 @@ static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env, scalar_min_max_smod(dst_reg, &src_reg); else scalar_min_max_umod(dst_reg, &src_reg); + reg_step_reset(dst_reg); break; case BPF_AND: if (tnum_is_const(src_reg.var_off)) { @@ -15682,6 +17412,7 @@ static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env, dst_reg->var_off = tnum_and(dst_reg->var_off, src_reg.var_off); scalar32_min_max_and(dst_reg, &src_reg); scalar_min_max_and(dst_reg, &src_reg); + reg_step_reset(dst_reg); break; case BPF_OR: if (tnum_is_const(src_reg.var_off)) { @@ -15692,13 +17423,16 @@ static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env, dst_reg->var_off = tnum_or(dst_reg->var_off, src_reg.var_off); scalar32_min_max_or(dst_reg, &src_reg); scalar_min_max_or(dst_reg, &src_reg); + reg_step_reset(dst_reg); break; case BPF_XOR: dst_reg->var_off = tnum_xor(dst_reg->var_off, src_reg.var_off); scalar32_min_max_xor(dst_reg, &src_reg); scalar_min_max_xor(dst_reg, &src_reg); + reg_step_reset(dst_reg); break; case BPF_LSH: + scalar_step_lsh(dst_reg, &src_reg); if (alu32) scalar32_min_max_lsh(dst_reg, &src_reg); else @@ -15709,15 +17443,18 @@ static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env, scalar32_min_max_rsh(dst_reg, &src_reg); else scalar_min_max_rsh(dst_reg, &src_reg); + reg_step_reset(dst_reg); break; case BPF_ARSH: if (alu32) scalar32_min_max_arsh(dst_reg, &src_reg); else scalar_min_max_arsh(dst_reg, &src_reg); + reg_step_reset(dst_reg); break; case BPF_END: scalar_byte_swap(dst_reg, insn); + reg_step_reset(dst_reg); break; default: break; @@ -15929,6 +17666,112 @@ static int adjust_reg_min_max_vals(struct bpf_verifier_env *env, return 0; } +/* + * Many checks done by this function are quite conservative. + * This is because main verification pass does not maintain + * enough information to track object identities for some + * of the interesting types, e.g. PTR_TO_MEM. + */ +bool bpf_same_memory_origin(const struct bpf_reg_state *reg_a, + const struct bpf_reg_state *reg_b) +{ + if (reg_a == reg_b) + return true; + /* Require matching flags and base types. */ + if (reg_a->type != reg_b->type) + return false; + /* NULL can't be compared to some base+offset pointer. */ + if (type_may_be_null(reg_a->type)) + return false; + + switch (base_type(reg_a->type)) { + case PTR_TO_STACK: + return reg_a->frameno == reg_b->frameno; + case PTR_TO_MAP_VALUE: + if (reg_a->map_ptr != reg_b->map_ptr || reg_a->map_uid != reg_b->map_uid) + return false; + if (reg_a->id && reg_a->id == reg_b->id) + return true; + /* + * A plain single-element array has one stable value address, + * but not when BPF_F_INNER_MAP is set, see arraymap.c:array_map_meta_equal(). + */ + if (reg_a->map_ptr->map_type == BPF_MAP_TYPE_ARRAY && + reg_a->map_ptr->max_entries == 1 && + !(reg_a->map_ptr->map_flags & BPF_F_INNER_MAP)) + return true; + /* + * The rules above can be simplified / relaxed if: + * - fresh IDs would always be assigned for map-value lookups; + * - direct map value loads would always have and ID of zero. + */ + return false; + case PTR_TO_MEM: + case PTR_TO_BUF: + return reg_a->id && reg_a->id == reg_b->id; + + default: + return false; + } +} + +int bpf_set_reg_range(struct bpf_verifier_env *env, struct bpf_reg_state *reg, + struct cnum64 r64, struct cnum32 r32, u16 base, u16 step) +{ + reg->r64 = r64; + reg->r32 = r32; + reg->step = step; + reg->base = base; + reg->var_off = tnum_unknown; + reg_bounds_sync(reg); /* this should infer the tnum alignment */ + return reg_bounds_sanity_check(env, reg, "bpf_set_reg_range"); +} + +/* + * Checks if pointer and packet metadata matches between two registers. + * For use with bpf_reg_union(). + */ +bool bpf_reg_union_compatible(const struct bpf_reg_state *a, const struct bpf_reg_state *b) +{ + if (a->type != b->type) + return false; + if (a->type == SCALAR_VALUE) + return true; + return !memcmp(a, b, offsetof(struct bpf_reg_state, var_off)) && + a->id == b->id && a->parent_id == b->parent_id && a->map_uid == b->map_uid; +} + +/* + * Extends acc's scalar range to include src's scalar range. + * Caller must ensure that other kinds of metadata is compatible. + * Caller must clear acc's scalar ID. + */ +int bpf_reg_union(struct bpf_verifier_env *env, struct bpf_reg_state *acc, + const struct bpf_reg_state *src) +{ + u16 base, step; + + if (!bpf_reg_union_compatible(acc, src)) { + verifier_bug(env, "union of incompatible registers"); + return -EFAULT; + } + acc->r64 = cnum64_union(acc->r64, src->r64); + acc->r32 = cnum32_union(acc->r32, src->r32); + acc->var_off = tnum_union(acc->var_off, src->var_off); + + /* Retain a common congruence if the bases agree modulo the gcd. */ + step = gcd(acc->step, src->step); + base = acc->base % step; + if (base != src->base % step) { + reg_step_reset(acc); + } else { + acc->base = base; + acc->step = step; + } + reg_bounds_sync(acc); + return reg_bounds_sanity_check(env, acc, "bpf_reg_union"); +} + /* check validity of 32-bit and 64-bit arithmetic operations */ static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn) { @@ -16198,8 +18041,6 @@ static void find_good_pkt_pointers(struct bpf_verifier_state *vstate, static void regs_refine_cond_op(struct bpf_reg_state *reg1, struct bpf_reg_state *reg2, u8 opcode, bool is_jmp32); -static u8 rev_opcode(u8 opcode); - /* * Learn more information about live branches by simulating refinement on both branches. * regs_refine_cond_op() is sound, so producing ill-formed register bounds for the branch means @@ -16208,7 +18049,7 @@ static u8 rev_opcode(u8 opcode); static int simulate_both_branches_taken(struct bpf_verifier_env *env, u8 opcode, bool is_jmp32) { /* Fallthrough (FALSE) branch */ - regs_refine_cond_op(&env->false_reg1, &env->false_reg2, rev_opcode(opcode), is_jmp32); + regs_refine_cond_op(&env->false_reg1, &env->false_reg2, bpf_rev_opcode(opcode), is_jmp32); reg_bounds_sync(&env->false_reg1); reg_bounds_sync(&env->false_reg2); /* @@ -16394,7 +18235,7 @@ static int is_scalar_branch_taken(struct bpf_verifier_env *env, struct bpf_reg_s return simulate_both_branches_taken(env, opcode, is_jmp32); } -static int flip_opcode(u32 opcode) +int bpf_flip_opcode(u32 opcode) { /* How can we transform "a b" into "b a"? */ static const u8 opcode_flip[16] = { @@ -16425,7 +18266,7 @@ static int is_pkt_ptr_branch_taken(struct bpf_reg_state *dst_reg, pkt = dst_reg; } else if (dst_reg->type == PTR_TO_PACKET_END) { pkt = src_reg; - opcode = flip_opcode(opcode); + opcode = bpf_flip_opcode(opcode); } else { return -1; } @@ -16480,7 +18321,7 @@ static int is_branch_taken(struct bpf_verifier_env *env, struct bpf_reg_state *r /* arrange that reg2 is a scalar, and reg1 is a pointer */ if (!is_reg_const(reg2, is_jmp32)) { - opcode = flip_opcode(opcode); + opcode = bpf_flip_opcode(opcode); swap(reg1, reg2); } /* and ensure that reg2 is a constant */ @@ -16514,7 +18355,7 @@ static int is_branch_taken(struct bpf_verifier_env *env, struct bpf_reg_state *r /* Opcode that corresponds to a *false* branch condition. * E.g., if r1 < r2, then reverse (false) condition is r1 >= r2 */ -static u8 rev_opcode(u8 opcode) +u8 bpf_rev_opcode(u8 opcode) { switch (opcode) { case BPF_JEQ: return BPF_JNE; @@ -16549,7 +18390,7 @@ static void regs_refine_cond_op(struct bpf_reg_state *reg1, struct bpf_reg_state case BPF_JGT: case BPF_JSGE: case BPF_JSGT: - opcode = flip_opcode(opcode); + opcode = bpf_flip_opcode(opcode); swap(reg1, reg2); break; default: @@ -16607,7 +18448,7 @@ static void regs_refine_cond_op(struct bpf_reg_state *reg1, struct bpf_reg_state * this restriction. I.e., !(r1 & 0x3) means neither bit 0 nor * bit 1 is set, which we can readily use in adjustments. */ - if (!is_power_of_2(val)) + if (hweight64(val) != 1) break; if (is_jmp32) { t = tnum_or(tnum_subreg(reg1->var_off), tnum_const(val)); @@ -16616,7 +18457,7 @@ static void regs_refine_cond_op(struct bpf_reg_state *reg1, struct bpf_reg_state reg1->var_off = tnum_or(reg1->var_off, tnum_const(val)); } break; - case BPF_JSET | BPF_X: /* reverse of BPF_JSET, see rev_opcode() */ + case BPF_JSET | BPF_X: /* reverse of BPF_JSET, see bpf_rev_opcode() */ if (!is_reg_const(reg2, is_jmp32)) swap(reg1, reg2); if (!is_reg_const(reg2, is_jmp32)) @@ -16626,6 +18467,7 @@ static void regs_refine_cond_op(struct bpf_reg_state *reg1, struct bpf_reg_state * violations if we're on a dead branch. */ __mark_reg_unbounded(reg1); + reg_step_reset(reg1); if (is_jmp32) { t = tnum_and(tnum_subreg(reg1->var_off), tnum_const(~val)); reg1->var_off = tnum_with_subreg(reg1->var_off, t); @@ -16893,10 +18735,12 @@ static void collect_linked_regs(struct bpf_verifier_env *env, reg = &func->regs[j]; __collect_linked_regs(linked_regs, reg, id, i, j, true); } - for (j = 0; j < func->allocated_stack / BPF_REG_SIZE; j++) { - if (!bpf_is_spilled_reg(&func->stack[j])) + for (j = 0; j < bpf_stack_nr_slots(func); j++) { + struct bpf_stack_state *ss = bpf_stack_slot(func, j); + + if (!bpf_is_spilled_reg(ss)) continue; - reg = &func->stack[j].spilled_ptr; + reg = &ss->spilled_ptr; __collect_linked_regs(linked_regs, reg, id, i, j, false); } } @@ -16916,7 +18760,7 @@ static void sync_linked_regs(struct bpf_verifier_env *env, struct bpf_verifier_s for (i = 0; i < linked_regs->cnt; ++i) { e = &linked_regs->entries[i]; reg = e->is_reg ? &vstate->frame[e->frameno]->regs[e->regno] - : &vstate->frame[e->frameno]->stack[e->spi].spilled_ptr; + : &bpf_stack_slot(vstate->frame[e->frameno], e->spi)->spilled_ptr; if (reg->type != SCALAR_VALUE || reg == known_reg) continue; if ((reg->id & ~BPF_ADD_CONST) != (known_reg->id & ~BPF_ADD_CONST)) @@ -16946,6 +18790,7 @@ static void sync_linked_regs(struct bpf_verifier_env *env, struct bpf_verifier_s reg->delta = saved_off; reg->id = saved_id; + scalar_step_add(reg, reg, &fake_reg); scalar32_min_max_add(reg, &fake_reg); scalar_min_max_add(reg, &fake_reg); reg->var_off = tnum_add(reg->var_off, fake_reg.var_off); @@ -16981,6 +18826,23 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env, return -EINVAL; } + if (bpf_is_unwind_goto(insn)) { + if (!env->cur_state->unwound) + return 0; + env->cur_state->unwound = false; + /* + * unwind_exit changed its return address, so the return is + * mispredicted and what follows unwind_goto runs speculatively. + */ + if (!env->bypass_spec_v1) { + err = sanitize_speculative_path(env, NULL, *insn_idx + 1, *insn_idx); + if (err) + return err; + } + *insn_idx += insn->off; + return 0; + } + if (opcode == BPF_JCOND) { struct bpf_verifier_state *cur_st = env->cur_state, *queued_st, *prev_st; int idx = *insn_idx; @@ -17034,7 +18896,7 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env, } if (insn_flags) { - err = bpf_push_jmp_history(env, this_branch, insn_flags, 0, 0, 0); + err = bpf_push_jmp_history(env, this_branch, insn_flags, 0, 0, NULL, 0); if (err) return err; } @@ -17104,7 +18966,10 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env, * if parent state is created. */ if (linked_regs.cnt > 1) { - err = bpf_push_jmp_history(env, this_branch, 0, 0, 0, linked_regs_pack(&linked_regs)); + u16 packed[LINKED_REGS_MAX]; + + linked_regs_pack(&linked_regs, packed); + err = bpf_push_jmp_history(env, this_branch, 0, 0, 0, packed, linked_regs.cnt); if (err) return err; } @@ -17669,37 +19534,54 @@ static int check_return_code(struct bpf_verifier_env *env, int regno, const char return 0; } -static int check_global_subprog_return_code(struct bpf_verifier_env *env) +static int check_global_ret_scalar_reg(struct bpf_verifier_env *env, u32 regno) { - struct bpf_reg_state *reg = reg_state(env, BPF_REG_0); - struct bpf_func_state *cur_frame = cur_func(env); + struct bpf_reg_state *reg; int err; - if (subprog_returns_void(env, cur_frame->subprogno)) - return 0; - - err = check_reg_arg(env, BPF_REG_0, SRC_OP); + err = check_reg_arg(env, regno, SRC_OP); if (err) return err; /* Pointers to arena are safe to pass between subprograms. */ - if (is_arena_reg(env, BPF_REG_0)) + if (is_arena_reg(env, regno)) return 0; - if (is_pointer_value(env, BPF_REG_0)) { - verbose(env, "R%d leaks addr as return value\n", BPF_REG_0); + if (is_pointer_value(env, regno)) { + verbose(env, "R%d leaks addr as return value\n", regno); return -EACCES; } + reg = reg_state(env, regno); if (reg->type != SCALAR_VALUE) { - verbose(env, "At subprogram exit the register R0 is not a scalar value (%s)\n", - reg_type_str(env, reg->type)); + verbose(env, "At subprogram exit the register R%d is not a scalar value (%s)\n", + regno, reg_type_str(env, reg->type)); return -EINVAL; } return 0; } +static int check_global_subprog_return_code(struct bpf_verifier_env *env) +{ + struct bpf_func_state *cur_frame = cur_func(env); + u32 subprog = cur_frame->subprogno; + u32 i, nregs; + int err; + + if (subprog_returns_void(env, subprog)) + return 0; + + nregs = bpf_ret_reg_pair(env, subprog) ? 2 : 1; + for (i = 0; i < nregs; i++) { + err = check_global_ret_scalar_reg(env, ret_regs[i]); + if (err) + return err; + } + + return 0; +} + /* Bitmask with 1s for all caller saved registers */ #define ALL_CALLER_SAVED_REGS ((1u << CALLER_SAVED_REGS) - 1) @@ -17741,11 +19623,11 @@ bool bpf_get_call_summary(struct bpf_verifier_env *env, struct bpf_insn *call, (bpf_verifier_inlines_helper_call(env, call->imm) || bpf_jit_inlines_helper_call(call->imm)); cs->is_void = fn->ret_type == RET_VOID; - cs->num_params = 0; + cs->arg_slot_cnt = 0; for (i = 0; i < ARRAY_SIZE(fn->arg_type); ++i) { - if (fn->arg_type[i] == ARG_DONTCARE) + if (fn->arg_type[i] == ARG_UNUSED) break; - cs->num_params++; + cs->arg_slot_cnt++; } return true; } @@ -17757,7 +19639,7 @@ bool bpf_get_call_summary(struct bpf_verifier_env *env, struct bpf_insn *call, if (err < 0) /* error would be reported later */ return false; - cs->num_params = btf_type_vlen(meta.func_proto); + cs->arg_slot_cnt = btf_proto_slots(meta.btf, meta.func_proto); cs->fastcall = meta.kfunc_flags & KF_FASTCALL; cs->is_void = btf_type_is_void(btf_type_by_id(meta.btf, meta.func_proto->type)); return true; @@ -17866,7 +19748,7 @@ static void mark_fastcall_pattern_for_call(struct bpf_verifier_env *env, * - includes R1-R5 if corresponding parameter has is described * in the function prototype. */ - clobbered_regs_mask = GENMASK(cs.num_params, cs.is_void ? 1 : 0); + clobbered_regs_mask = GENMASK(cs.arg_slot_cnt, cs.is_void ? 1 : 0); /* e.g. if helper call clobbers r{0,1}, expect r{2,3,4,5} in the pattern */ expected_regs_mask = ~clobbered_regs_mask & ALL_CALLER_SAVED_REGS; @@ -18232,10 +20114,25 @@ static int indirect_jump_min_max_index(struct bpf_verifier_env *env, return 0; } +static int reject_gotox_out_of_subprog(struct bpf_verifier_env *env, u32 target, + u32 subprog_start, u32 subprog_end) +{ + verbose(env, "indirect jump from insn %d to %u leaves the subprog [%u,%u)\n", + env->insn_idx, target, subprog_start, subprog_end); + bpf_diag_program_structure( + env, env->insn_idx, "indirect jump leaves subprogram", + "Keep every reachable jump-table target inside the subprogram of the indirect jump.", + "Instruction %d can jump indirectly to instruction %u, which is outside its own subprogram [%u,%u).", + env->insn_idx, target, subprog_start, subprog_end); + return -EINVAL; +} + /* gotox *dst_reg */ static int check_indirect_jump(struct bpf_verifier_env *env, struct bpf_insn *insn) { struct bpf_verifier_state *other_branch; + struct bpf_subprog_info *subprog; + u32 subprog_start, subprog_end; struct bpf_reg_state *dst_reg; struct bpf_map *map; u32 min_index, max_index; @@ -18279,6 +20176,24 @@ static int check_indirect_jump(struct bpf_verifier_env *env, struct bpf_insn *in return -EINVAL; } + subprog = bpf_find_containing_subprog(env, env->insn_idx); + if (verifier_bug_if(!subprog, env, "no subprog contains insn %d", env->insn_idx)) + return -EFAULT; + subprog_start = subprog->start; + subprog_end = (subprog + 1)->start; + + /* + * Maps are confined to a subprog, see compute_subprog_jts(), and the + * targets are sorted and free of duplicates, see sort_insn_array_uniq(), + * so either all of them belong to this subprog or none of them does. + */ + if (env->gotox_tmp_buf->items[0] < subprog_start) + return reject_gotox_out_of_subprog(env, env->gotox_tmp_buf->items[0], + subprog_start, subprog_end); + if (env->gotox_tmp_buf->items[n - 1] >= subprog_end) + return reject_gotox_out_of_subprog(env, env->gotox_tmp_buf->items[n - 1], + subprog_start, subprog_end); + for (i = 0; i < n - 1; i++) { mark_indirect_target(env, env->gotox_tmp_buf->items[i]); other_branch = push_stack(env, env->gotox_tmp_buf->items[i], @@ -18321,15 +20236,17 @@ static int do_check_insn(struct bpf_verifier_env *env, bool *do_print_state) return check_stack_arg_write(env, state, insn->off, NULL); } + struct bpf_reg_state arena, *dst_reg; enum bpf_reg_type dst_reg_type; err = check_reg_arg(env, insn->dst_reg, SRC_OP); if (err) return err; - dst_reg_type = cur_regs(env)[insn->dst_reg].type; + dst_reg = mem_access_reg(env, insn->dst_reg, &arena); + dst_reg_type = dst_reg->type; - err = check_mem_access(env, env->insn_idx, cur_regs(env) + insn->dst_reg, argno_from_reg(insn->dst_reg), + err = check_mem_access(env, env->insn_idx, dst_reg, argno_from_reg(insn->dst_reg), insn->off, BPF_SIZE(insn->code), BPF_WRITE, -1, false, false); if (err) @@ -18344,11 +20261,13 @@ static int do_check_insn(struct bpf_verifier_env *env, bool *do_print_state) env->jmps_processed++; if (opcode == BPF_CALL) { if (env->cur_state->active_locks) { - if ((insn->src_reg == BPF_REG_0 && - insn->imm != BPF_FUNC_spin_unlock && - insn->imm != BPF_FUNC_kptr_xchg) || - (insn->src_reg == BPF_PSEUDO_KFUNC_CALL && - !kfunc_spin_allowed(env, insn->imm, insn->off))) { + /* similar to static subprog calls callx is allowed under a lock */ + if (!bpf_is_callx(insn) && + ((insn->src_reg == BPF_REG_0 && + insn->imm != BPF_FUNC_spin_unlock && + insn->imm != BPF_FUNC_kptr_xchg) || + (insn->src_reg == BPF_PSEUDO_KFUNC_CALL && + !kfunc_spin_allowed(env, insn->imm, insn->off)))) { verbose(env, "function calls are not allowed while holding a lock\n"); bpf_diag_ctx_active( @@ -18361,6 +20280,8 @@ static int do_check_insn(struct bpf_verifier_env *env, bool *do_print_state) mark_reg_scratched(env, BPF_REG_0); if (bpf_in_stack_arg_cnt(&env->subprog_info[cur_func(env)->subprogno])) cur_func(env)->no_stack_arg_load = true; + if (bpf_is_callx(insn)) + return check_func_callx(env, insn, &env->insn_idx); if (insn->src_reg == BPF_PSEUDO_CALL) return check_func_call(env, insn, &env->insn_idx); if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) @@ -18401,6 +20322,178 @@ static int do_check_insn(struct bpf_verifier_env *env, bool *do_print_state) return -EFAULT; } +/* + * Push the loop entered at insn_idx onto the stack. Usually this adds a single + * entry on top of its enclosing loop. However, an edge may enter an inner loop + * directly, bypassing the header(s) of its enclosing loop(s), e.g.: + * + * 1: for (...): // enclosing loop, header at 1 + * 2: for (...): // inner loop, header at 2 + * ... + * 3: if ...: + * goto 2b; // enters loop 2 without going through header 1 + * + * Such a bypass makes the enclosing loop irreducible, so its header is missing + * from the stack and both headers (1) and (2) need to be pushed onto stack. + * Only the innermost loop (the one actually entered at insn_idx) carries SCEV bounds; + * the bypassed ancestors are irreducible and pushed as non-terminating. + * + * Assumes loop_stack_pop() has already truncated the stack to the common ancestor + * of the bpf_loop_at_index(env->insn_idx) and whatever was at the top of the loop stack. + */ +static int loop_stack_push(struct bpf_verifier_env *env, bool *pushed) +{ + struct bpf_insn_aux_data *aux = env->insn_aux_data; + struct bpf_func_state *frame = cur_func(env); + struct loop_stack_entry *loop_stack = frame->loop_stack; + u32 missing_headers[LOOP_STACK_SIZE]; + u32 cnt = frame->loop_stack_cnt; + u32 num_missing = 0; + int h; + + *pushed = false; + + for (h = bpf_loop_at_index(env, env->insn_idx); h >= 0; h = aux[h].loop_header, num_missing++) { + if (cnt && loop_stack[cnt - 1].loop_id == h) + break; + if (num_missing == LOOP_STACK_SIZE) + goto e2big; + missing_headers[num_missing] = h; + } + + if (num_missing == 0) + return 0; + if (cnt + num_missing > LOOP_STACK_SIZE) + goto e2big; + frame->loop_stack = realloc_array(frame->loop_stack, cnt, cnt + num_missing, + sizeof(*frame->loop_stack)); + if (!frame->loop_stack) + return -ENOMEM; + loop_stack = frame->loop_stack; + + for (; num_missing; num_missing--, cnt++) { + h = missing_headers[num_missing - 1]; + loop_stack[cnt] = (struct loop_stack_entry){ .loop_id = h }; + if (env->log.level & BPF_LOG_LEVEL2) + verbose(env, "entering loop %d\n", h); + *pushed = true; + } + frame->loop_stack_cnt = cnt; + return 0; + +e2big: + verbose(env, "Too many nested loops (%d/%d) at %d\n", cnt, num_missing, env->insn_idx); + return -E2BIG; +} + +/* + * An exit from a loop can cross several nested loops, e.g.: + * + * 1: for (...): + * 2: for (...): + * 3: for (...): + * if ...: // before goto the loop stack is [1, 2, 3 ] + * goto 1b; // after goto it should become [1] + */ +static int loop_stack_pop(struct bpf_verifier_env *env) +{ + struct bpf_insn_aux_data *aux = env->insn_aux_data; + struct bpf_func_state *frame = cur_func(env); + int new_cnt = 0; + int h, i; + + /* + * Walk the loop nest of insn_idx outwards (innermost first) and stop at + * the first loop that is present on the stack - that loop is the common + * ancestor and becomes the new top. If no enclosing loop is on the + * stack, drop all loops. + */ + for (h = bpf_loop_at_index(env, env->insn_idx); h >= 0; h = aux[h].loop_header) { + for (i = frame->loop_stack_cnt - 1; i >= 0; i--) { + if (frame->loop_stack[i].loop_id == h) { + new_cnt = i + 1; + goto pop; + } + } + } + +pop: + if (env->log.level & BPF_LOG_LEVEL2) + for (i = frame->loop_stack_cnt - 1; i >= new_cnt; i--) + verbose(env, "exiting loop %d\n", frame->loop_stack[i].loop_id); + frame->loop_stack_cnt = new_cnt; + return 0; +} + +static int maybe_clamp_scev_regs(struct bpf_verifier_env *env) +{ + struct bpf_func_state *frame = cur_func(env); + struct loop_stack_entry *entry; + + if (frame->loop_stack_cnt == 0) { + verifier_bug(env, "%s: loop stack empty at %d", __FUNCTION__, env->insn_idx); + return -EFAULT; + } + + entry = &frame->loop_stack[frame->loop_stack_cnt - 1]; + if (!entry->terminates) + return 0; + + return bpf_clamp_scev_regs(env, frame, env->insn_idx, entry->entry_state, + entry->max_header_count); +} + +static int handle_loop_entry_exit(struct bpf_verifier_env *env) +{ + struct bpf_verifier_state *cur = env->cur_state; + struct bpf_func_state *frame = cur_func(env); + struct loop_stack_entry *entry; + struct bpf_loop_iters iters; + bool pushed, terminates; + int err; + + err = loop_stack_pop(env); + if (err) + return err; + + err = loop_stack_push(env, &pushed); + if (err) + return err; + + if (frame->loop_stack_cnt == 0) + return 0; + + entry = &frame->loop_stack[frame->loop_stack_cnt - 1]; + if (env->insn_idx != entry->loop_id) + return 0; + + /* If nothing was pushed and env->insn_idx is a loop header, we've taken a backedge. */ + if (!pushed) + return maybe_clamp_scev_regs(env); + + err = bpf_compute_loop_iters(env, env->cur_state, &iters); + if (err < 0) + return err; + + terminates = err == 1; + if (!terminates) + return 0; + + /* Create a loop_entry checkpoint before widening */ + err = bpf_split_cur_state(env); + if (err) + return err; + + entry->min_header_count = iters.min_header_count; + entry->max_header_count = iters.max_header_count; + entry->entry_state = cur->parent; + entry->terminates = true; + err = bpf_widen_scev_regs(env, cur, cur->parent, &iters); + if (err < 0) + return err; + return 0; +} + static int do_check(struct bpf_verifier_env *env) { bool pop_log = !(env->log.level & BPF_LOG_LEVEL2); @@ -18415,6 +20508,12 @@ static int do_check(struct bpf_verifier_env *env) struct bpf_insn_aux_data *insn_aux; int err; + if (signal_pending(current)) + return -EAGAIN; + + if (need_resched()) + cond_resched(); + /* reset current history entry on each new instruction */ env->cur_hist_ent = NULL; @@ -18461,6 +20560,15 @@ static int do_check(struct bpf_verifier_env *env) } } + /* + * Possibly widen the registers before creating a checkpoint + * in bpf_is_state_visited(). The next loop iteration will + * have a chance to hit this checkpoint and converge. + */ + err = handle_loop_entry_exit(env); + if (err) + return err; + if (bpf_is_prune_point(env, env->insn_idx)) { err = bpf_is_state_visited(env, env->insn_idx); if (err < 0) @@ -18481,17 +20589,11 @@ static int do_check(struct bpf_verifier_env *env) } if (bpf_is_jmp_point(env, env->insn_idx)) { - err = bpf_push_jmp_history(env, state, 0, 0, 0, 0); + err = bpf_push_jmp_history(env, state, 0, 0, 0, NULL, 0); if (err) return err; } - if (signal_pending(current)) - return -EAGAIN; - - if (need_resched()) - cond_resched(); - if (env->log.level & BPF_LOG_LEVEL2 && do_print_state) { verbose(env, "\nfrom %d to %d%s:", env->prev_insn_idx, env->insn_idx, @@ -18501,6 +20603,13 @@ static int do_check(struct bpf_verifier_env *env) do_print_state = false; } + if (bpf_loop_at_index(env, env->insn_idx) >= 0 && + cur_func(env)->loop_stack_cnt == 0) { + verifier_bug(env, "loop stack empty at %d, while inside the loop %d\n", + env->insn_idx, bpf_loop_at_index(env, env->insn_idx)); + return -EFAULT; + } + if (env->log.level & BPF_LOG_LEVEL) { if (verifier_state_scratched(env)) print_insn_state(env, state, state->curframe); @@ -18833,12 +20942,6 @@ static int check_map_prog_compatibility(struct bpf_verifier_env *env, { enum bpf_prog_type prog_type = resolve_prog_type(prog); - if (map->excl_prog_sha && - memcmp(map->excl_prog_sha, prog->digest, SHA256_DIGEST_SIZE)) { - verbose(env, "program's hash doesn't match map's excl_prog_hash\n"); - return -EACCES; - } - if (btf_record_has_field(map->record, BPF_LIST_HEAD) || btf_record_has_field(map->record, BPF_RB_ROOT)) { if (is_tracing_prog_type(prog_type)) { @@ -18872,35 +20975,11 @@ static int check_map_prog_compatibility(struct bpf_verifier_env *env, return -EINVAL; } - if (prog->sleepable) - switch (map->map_type) { - case BPF_MAP_TYPE_HASH: - case BPF_MAP_TYPE_RHASH: - case BPF_MAP_TYPE_LRU_HASH: - case BPF_MAP_TYPE_ARRAY: - case BPF_MAP_TYPE_PERCPU_HASH: - case BPF_MAP_TYPE_PERCPU_ARRAY: - case BPF_MAP_TYPE_LRU_PERCPU_HASH: - case BPF_MAP_TYPE_LPM_TRIE: - case BPF_MAP_TYPE_ARRAY_OF_MAPS: - case BPF_MAP_TYPE_HASH_OF_MAPS: - case BPF_MAP_TYPE_RINGBUF: - case BPF_MAP_TYPE_USER_RINGBUF: - case BPF_MAP_TYPE_INODE_STORAGE: - case BPF_MAP_TYPE_SK_STORAGE: - case BPF_MAP_TYPE_TASK_STORAGE: - case BPF_MAP_TYPE_CGRP_STORAGE: - case BPF_MAP_TYPE_QUEUE: - case BPF_MAP_TYPE_STACK: - case BPF_MAP_TYPE_ARENA: - case BPF_MAP_TYPE_INSN_ARRAY: - case BPF_MAP_TYPE_PROG_ARRAY: - break; - default: - verbose(env, - "Sleepable programs can only use array, hash, ringbuf and local storage maps\n"); - return -EINVAL; - } + if (prog->sleepable && !map_needs_no_rcu(map)) { + verbose(env, + "Sleepable programs can only use array, hash, ringbuf and local storage maps\n"); + return -EINVAL; + } if (bpf_map_is_cgroup_storage(map) && bpf_cgroup_storage_assign(env->prog->aux, map)) { @@ -18936,6 +21015,85 @@ static int check_map_prog_compatibility(struct bpf_verifier_env *env, return 0; } +/* + * Keep track of whether the map is used by one program only. Such program may + * store the addresses of its functions into the map when it's frozen, see + * resolve_func_ptrs(), since nothing else relies on what the map has. After + * that the map is not available to other programs. + */ +static int bpf_map_claim(struct bpf_verifier_env *env, struct bpf_map *map) +{ + unsigned long me = (unsigned long)env->prog->aux, old; + + for (;;) { + old = READ_ONCE(map->user); + if (old == me || old == BPF_MAP_USER_MANY) + return 0; + if (old & BPF_MAP_USER_PATCHED) { + verbose(env, "map '%s' has addresses of functions of another program\n", + map->name); + return -EBUSY; + } + if (cmpxchg(&map->user, old, old ? BPF_MAP_USER_MANY : me) == old) + return 0; + } +} + +/* + * An exclusive map is bound to the digest of its program. The digest covers + * the insns as they are verified, after the CO-RE relocations the kernel + * applies, so it exists only once check_and_resolve_insns() has computed it, + * and the maps are checked against it there, whether an insn referred to + * them or a continuous fd_array bound them before. + */ +static int check_excl_maps(struct bpf_verifier_env *env) +{ + int i; + + for (i = 0; i < env->used_map_cnt; i++) { + struct bpf_map *map = env->used_maps[i]; + + if (map->excl_prog_sha && + memcmp(map->excl_prog_sha, env->prog->digest, SHA256_DIGEST_SIZE)) { + verbose(env, "program's hash doesn't match map's excl_prog_hash\n"); + return -EACCES; + } + } + return 0; +} + +/* + * The program may use every map it referred to, see check_excl_maps(). Only now + * does it leave a trace on them: it claims each as its user, see + * bpf_map_claim(), and takes over its insn arrays. A program rejected before + * this point, one that is not the owner of an exclusive map in particular, + * leaves the maps as it found them. Nothing consults a claim before + * resolve_func_ptrs() and the fixup passes, which run later. + */ +static int claim_used_maps(struct bpf_verifier_env *env) +{ + int i, err; + + for (i = 0; i < env->used_map_cnt; i++) { + struct bpf_map *map = env->used_maps[i]; + + err = bpf_map_claim(env, map); + if (err) + return err; + + if (map->map_type != BPF_MAP_TYPE_INSN_ARRAY) + continue; + err = bpf_insn_array_init(map, env->prog); + if (err) { + verbose(env, "Failed to properly initialize insn array\n"); + return err; + } + env->insn_array_maps[env->insn_array_map_cnt++] = map; + env->prog->jit_required = true; + } + return 0; +} + static int __add_used_map(struct bpf_verifier_env *env, struct bpf_map *map) { int i, err; @@ -18961,7 +21119,7 @@ static int __add_used_map(struct bpf_verifier_env *env, struct bpf_map *map) if (err) return err; - if (env->prog->sleepable) + if (bpf_prog_runs_sleepable(env->prog)) atomic64_inc(&map->sleepable_refcnt); /* hold the map. If the program is rejected by verifier, @@ -18973,16 +21131,6 @@ static int __add_used_map(struct bpf_verifier_env *env, struct bpf_map *map) env->used_maps[env->used_map_cnt++] = map; - if (map->map_type == BPF_MAP_TYPE_INSN_ARRAY) { - err = bpf_insn_array_init(map, env->prog); - if (err) { - verbose(env, "Failed to properly initialize insn array\n"); - return err; - } - env->insn_array_maps[env->insn_array_map_cnt++] = map; - env->prog->jit_required = true; - } - return env->used_map_cnt - 1; } @@ -19127,6 +21275,14 @@ static int check_jmp_fields(struct bpf_verifier_env *env, struct bpf_insn *insn) switch (opcode) { case BPF_CALL: + if (bpf_is_callx(insn)) { + /* callx dst_reg */ + if (insn->src_reg != BPF_REG_0 || insn->imm != 0 || insn->off != 0) { + verbose(env, "BPF_CALL|BPF_X uses reserved fields\n"); + return -EINVAL; + } + return 0; + } if (BPF_SRC(insn->code) != BPF_K || (insn->src_reg != BPF_PSEUDO_KFUNC_CALL && insn->off != 0) || (insn->src_reg != BPF_REG_0 && insn->src_reg != BPF_PSEUDO_CALL && @@ -19151,14 +21307,14 @@ static int check_jmp_fields(struct bpf_verifier_env *env, struct bpf_insn *insn) return 0; case BPF_EXIT: if (BPF_SRC(insn->code) != BPF_K || insn->imm != 0 || - insn->src_reg != BPF_REG_0 || insn->dst_reg != BPF_REG_0 || + insn->src_reg > BPF_UNWIND_EXIT || insn->dst_reg != BPF_REG_0 || class == BPF_JMP32) { verbose(env, "BPF_EXIT uses reserved fields\n"); return -EINVAL; } return 0; case BPF_JCOND: - if (insn->code != (BPF_JMP | BPF_JCOND) || insn->src_reg != BPF_MAY_GOTO || + if (insn->code != (BPF_JMP | BPF_JCOND) || insn->src_reg > BPF_UNWIND_GOTO || insn->dst_reg || insn->imm) { verbose(env, "invalid may_goto imm %d\n", insn->imm); return -EINVAL; @@ -19378,6 +21534,107 @@ static int check_and_resolve_insns(struct bpf_verifier_env *env) * 'struct bpf_map *' into a register instead of user map_fd. * These pointers will be used later by verifier to validate map access. */ + err = check_excl_maps(env); + if (err) + return err; + return claim_used_maps(env); +} + +/* that many pointers to functions are recognized in the maps of a program at most */ +#define BPF_MAX_FUNC_PTRS (1 << 20) + +static int add_func_ptr(struct bpf_verifier_env *env, struct bpf_map *map, u32 map_off, + u32 xlated_off) +{ + struct bpf_func_ptr *ptrs; + + if (env->func_ptr_cnt == BPF_MAX_FUNC_PTRS) { + verbose(env, "too many pointers to functions in maps, the limit is %d\n", + BPF_MAX_FUNC_PTRS); + return -E2BIG; + } + + /* grow by doubling, the array is sorted and searched later */ + if (!(env->func_ptr_cnt & (env->func_ptr_cnt - 1))) { + ptrs = kvrealloc(env->func_ptrs, + array_size(max(2 * env->func_ptr_cnt, 16U), sizeof(*ptrs)), + GFP_KERNEL_ACCOUNT); + if (!ptrs) + return -ENOMEM; + env->func_ptrs = ptrs; + } + env->func_ptrs[env->func_ptr_cnt++] = (struct bpf_func_ptr){ + .map = map, + .map_off = map_off, + .orig_off = xlated_off, + .xlated_off = xlated_off, + }; + return 0; +} + +/* + * Compilers put pointers to functions into read-only data: tables of functions, + * structures of operations, vtables, where they are mixed with other data. + * The loader stores such data in a frozen read-only array map and resolves + * a pointer to a static function to the offset in bytes of its first + * instruction in the program: the address of the function in the program. + * + * Find 64-bit values that look like that in the maps of a program that uses + * callx. It's a guess. When the value is not a pointer, the program either + * fails to load, because it does with a pointer what can be done with + * a number only, or it sees the address of a function instead of the number. + * It's known before the control flow graph of the program is built and the main + * verification pass begins which functions may be called via callx. + * + * When the program is jitted the offsets are replaced with the addresses of + * the functions in the map itself, see jit_subprogs(). Hence the program has to + * be the only user of the map, see bpf_map_claim(): nothing else may rely on + * what the map had. + * + * The program reads the addresses of its functions from there like any other + * data, so it has to be allowed to leak pointers. + */ +static int resolve_func_ptrs(struct bpf_verifier_env *env) +{ + int insn_cnt = env->prog->len; + int i, err, subprog; + struct bpf_map *map; + u64 addr, val; + u32 off; + + if (!env->has_callx || !env->allow_ptr_leaks) + return 0; + + for (i = 0; i < env->used_map_cnt; i++) { + map = env->used_maps[i]; + /* coincidences in maps that are shared with other programs don't matter */ + if (READ_ONCE(map->user) != (unsigned long)env->prog->aux) + continue; + if (map->map_type != BPF_MAP_TYPE_ARRAY || map->max_entries != 1 || + !bpf_map_is_rdonly(map) || !map->ops->map_direct_value_addr || + !IS_ERR_OR_NULL(map->record)) + continue; + if (map->ops->map_direct_value_addr(map, &addr, 0)) + continue; + + for (off = 0; off + sizeof(u64) <= map->value_size; off += sizeof(u64)) { + if (!(off % SZ_64K)) + cond_resched(); + val = *(u64 *)(unsigned long)(addr + off); + if (!val || val % sizeof(struct bpf_insn) || + val / sizeof(struct bpf_insn) >= insn_cnt) + continue; + subprog = bpf_find_subprog(env, val / sizeof(struct bpf_insn)); + if (subprog <= 0 || bpf_subprog_is_global(env, subprog)) + continue; + err = add_func_ptr(env, map, off, val / sizeof(struct bpf_insn)); + if (err) + return err; + } + } + if (env->func_ptr_cnt) + sort(env->func_ptrs, env->func_ptr_cnt, sizeof(*env->func_ptrs), + cmp_func_ptrs, NULL); return 0; } @@ -19488,7 +21745,8 @@ static void free_states(struct bpf_verifier_env *env) } } -static int do_check_common(struct bpf_verifier_env *env, int subprog, bool is_sleepable) +static int do_check_common(struct bpf_verifier_env *env, int subprog, + enum bpf_func_context context) { bool pop_log = !(env->log.level & BPF_LOG_LEVEL2); struct bpf_subprog_info *sub = subprog_info(env, subprog); @@ -19508,7 +21766,8 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog, bool is_sl state->curframe = 0; state->speculative = false; state->branches = 1; - state->in_sleepable = is_sleepable; + state->in_sleepable = context == BPF_FUNC_CTX_SLEEPABLE; + env->no_rcu_read_lock = context == BPF_FUNC_CTX_NO_RCU_LOCK; state->frame[0] = kzalloc_obj(struct bpf_func_state, GFP_KERNEL_ACCOUNT); if (!state->frame[0]) { kfree(state); @@ -19548,25 +21807,30 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog, bool is_sl } /* Also ensure the callback only has a single scalar argument. */ - if (sub->arg_cnt != 1 || sub->args[0].arg_type != ARG_ANYTHING) { + if (sub->arg_slot_cnt != 1 || sub->args[0].arg_type != ARG_SCALAR) { verbose(env, "exception cb only supports single integer argument\n"); ret = -EINVAL; goto out; } } - for (i = BPF_REG_1; i <= min_t(u32, sub->arg_cnt, MAX_BPF_FUNC_REG_ARGS); i++) { + for (i = BPF_REG_1; + i <= min_t(u32, sub->arg_slot_cnt, MAX_BPF_FUNC_REG_ARGS); i++) { arg = &sub->args[i - BPF_REG_1]; reg = ®s[i]; if (arg->arg_type == ARG_PTR_TO_CTX) { reg->type = PTR_TO_CTX; mark_reg_known_zero(env, regs, i); - } else if (arg->arg_type == ARG_ANYTHING) { + } else if (arg->arg_type == ARG_SCALAR) { reg->type = SCALAR_VALUE; mark_reg_unknown(env, regs, i); } else if (arg->arg_type == ARG_PTR_TO_DYNPTR) { - /* assume unspecial LOCAL dynptr type */ - __mark_dynptr_reg(reg, BPF_DYNPTR_TYPE_LOCAL, true, ++env->id_gen, 0); + /* + * The callers may hand in a dynptr of any type. + * Take it for that of an skb, so that a write + * through it and its slices are treated with care. + */ + __mark_dynptr_reg(reg, BPF_DYNPTR_TYPE_SKB, true, ++env->id_gen, 0); } else if (base_type(arg->arg_type) == ARG_PTR_TO_MEM) { reg->type = PTR_TO_MEM; reg->type |= arg->arg_type & @@ -19597,7 +21861,8 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog, bool is_sl goto out; } } - if (env->prog->type == BPF_PROG_TYPE_EXT && sub->arg_cnt > MAX_BPF_FUNC_REG_ARGS) { + if (env->prog->type == BPF_PROG_TYPE_EXT && + sub->arg_slot_cnt > MAX_BPF_FUNC_REG_ARGS) { verbose(env, "freplace programs with >%d args not supported yet\n", MAX_BPF_FUNC_REG_ARGS); ret = -EINVAL; @@ -19610,9 +21875,10 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog, bool is_sl */ if (env->prog->aux->func_info_aux) { ret = btf_prepare_func_args(env, 0); - if (ret || sub->arg_cnt != 1 || sub->args[0].arg_type != ARG_PTR_TO_CTX) { + if (ret || sub->arg_slot_cnt != 1 || + sub->args[0].arg_type != ARG_PTR_TO_CTX) { env->prog->aux->func_info_aux[0].unreliable = true; - sub->arg_cnt = 1; + sub->arg_slot_cnt = 1; sub->stack_arg_cnt = 0; } } @@ -19722,7 +21988,7 @@ static int do_check_subprogs(struct bpf_verifier_env *env) /* * We can't loop forever as each pass verifies at least one new context, - * and there are only two contexts per global subprog. + * and there are only three contexts per global subprog. */ if (new_cnt) goto again; @@ -20108,6 +22374,109 @@ static bool attach_uses_trampoline_retval(enum bpf_attach_type type) } } +/* + * BPF code passes argument slot N where the kernel convention passes its Nth + * u64 argument, so describe a BPF function to its trampoline as one u64 per + * slot, and every arch places its arguments right. + */ +static int flatten_bpf_func_model(struct bpf_verifier_log *log, const char *tname, + struct btf_func_model *m) +{ + u32 i, nslots = 0; + + for (i = 0; i < m->nr_args; i++) + nslots += btf_func_model_arg_slots(m, i); + if (nslots > MAX_BPF_FUNC_ARGS) { + bpf_log(log, "The function %s has %u argument slots. Too many.\n", + tname, nslots); + return -EINVAL; + } + + for (i = 0; i < nslots; i++) { + m->arg_size[i] = sizeof(u64); + m->arg_flags[i] = 0; + } + m->nr_args = nslots; + return 0; +} + +/* + * The verifier lets a static function use stack arguments counted from its + * BTF, even if that BTF turns out unreliable. A trampoline on the function is + * then given only MAX_BPF_FUNC_REG_ARGS u64 arguments and would lose the + * stack arguments, so such a function cannot be attached to. + */ +static int check_unreliable_func_args(struct bpf_verifier_log *log, struct btf *btf, + const struct btf_type *func_proto, const char *tname) +{ + struct btf_func_model m; + u32 i, nslots = 0; + + if (btf_distill_func_proto(NULL, btf, func_proto, tname, &m)) { + /* at least a slot for each argument */ + nslots = btf_type_vlen(func_proto); + } else { + for (i = 0; i < m.nr_args; i++) + nslots += btf_func_model_arg_slots(&m, i); + } + if (nslots <= MAX_BPF_FUNC_REG_ARGS) + return 0; + + bpf_log(log, "Cannot attach to static function %s: unreliable BTF and stack arguments\n", + tname); + return -EINVAL; +} + +/* Functions may share a type id. The first of them is attached to. */ +static int find_tgt_subprog(const struct bpf_prog *tgt_prog, u32 btf_id) +{ + const struct bpf_prog_aux *aux = tgt_prog->aux; + int i; + + for (i = 0; i < aux->func_info_cnt; i++) + if (aux->func_info[i].type_id == btf_id) + return i; + + return -1; +} + +/* + * The prologue of an extension gives a cloned skb a new head, unless the + * prologue of the target program has done that. The callers of the function + * that it replaces must not hold on to the old head. + */ +int bpf_check_ext_prologue(struct bpf_verifier_log *log, + const struct bpf_prog *tgt_prog, u32 btf_id) +{ + const struct bpf_prog_aux *aux = tgt_prog->aux; + int subprog = find_tgt_subprog(tgt_prog, btf_id); + + if (aux->has_prologue) + return 0; + + if (subprog >= 0 && !aux->func_info_aux[subprog].called_with_pkt_ptrs) + return 0; + + bpf_log(log, + "Extension writing packet data cannot replace a function whose callers keep packet pointers\n"); + return -EINVAL; +} + +/* + * An extension is verified as a program that runs under the RCU read lock. Does + * the function that it is to replace? + */ +static bool func_has_rcu_read_lock(const struct bpf_prog *prog, int subprog) +{ + const struct bpf_func_info_aux *sub_aux = &prog->aux->func_info_aux[subprog]; + + if (!subprog) + return !prog->sleepable; + + return !sub_aux->called[BPF_FUNC_CTX_SLEEPABLE] && + !sub_aux->called[BPF_FUNC_CTX_NO_RCU_LOCK]; +} + int bpf_check_attach_target(struct bpf_verifier_log *log, const struct bpf_prog *prog, const struct bpf_prog *tgt_prog, @@ -20119,7 +22488,7 @@ int bpf_check_attach_target(struct bpf_verifier_log *log, char trace_symbol[KSYM_SYMBOL_LEN]; const char prefix[] = "btf_trace_"; struct bpf_raw_event_map *btp; - int ret = 0, subprog = -1, i; + int ret = 0, subprog = -1; const struct btf_type *t; bool conservative = true; const char *tname, *fname; @@ -20151,6 +22520,7 @@ int bpf_check_attach_target(struct bpf_verifier_log *log, struct bpf_prog_aux *aux = tgt_prog->aux; bool tgt_changes_pkt_data; bool tgt_might_sleep; + bool tgt_can_unwind; if (bpf_prog_is_dev_bound(prog->aux) && !bpf_prog_dev_bound_match(prog, tgt_prog)) { @@ -20158,11 +22528,7 @@ int bpf_check_attach_target(struct bpf_verifier_log *log, return -EINVAL; } - for (i = 0; i < aux->func_info_cnt; i++) - if (aux->func_info[i].type_id == btf_id) { - subprog = i; - break; - } + subprog = find_tgt_subprog(tgt_prog, btf_id); if (subprog == -1) { bpf_log(log, "Subprog %s doesn't exist\n", tname); return -EINVAL; @@ -20183,9 +22549,20 @@ int bpf_check_attach_target(struct bpf_verifier_log *log, prog_extension ? "Extension" : "Tracing"); return -EINVAL; } + /* unwind_exit of the main prog is exit, unless it's an extension */ + tgt_can_unwind = (subprog || tgt_prog->type == BPF_PROG_TYPE_EXT) && + (aux->func ? aux->func[subprog]->aux->can_unwind + : aux->can_unwind); + /* unwind_exit would return to the trampoline that called the target */ + if (tgt_can_unwind && !prog_extension && + prog->expected_attach_type != BPF_TRACE_FENTRY) { + bpf_log(log, "Only fentry can attach to %s, which can unwind\n", tname); + return -EINVAL; + } conservative = aux->func_info_aux[subprog].unreliable; if (prog_extension) { - if (conservative) { + if (aux->func_info_aux[subprog].linkage != BTF_FUNC_GLOBAL || + conservative) { bpf_log(log, "Cannot replace static functions\n"); return -EINVAL; @@ -20195,6 +22572,20 @@ int bpf_check_attach_target(struct bpf_verifier_log *log, "Extension programs should be JITed\n"); return -EINVAL; } + /* + * How much of the buffer of the tracepoint is used + * is checked for the program that is attached to it. + */ + if (tgt_prog->type == BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE) { + bpf_log(log, + "Cannot extend a writable raw tracepoint program\n"); + return -EINVAL; + } + if (!func_has_rcu_read_lock(tgt_prog, subprog)) { + bpf_log(log, + "Cannot extend a function that runs without RCU read lock\n"); + return -EINVAL; + } tgt_changes_pkt_data = aux->func ? aux->func[subprog]->aux->changes_pkt_data : aux->changes_pkt_data; @@ -20203,6 +22594,9 @@ int bpf_check_attach_target(struct bpf_verifier_log *log, "Extension program changes packet data, while original does not\n"); return -EINVAL; } + if (prog->aux->has_prologue && + bpf_check_ext_prologue(log, tgt_prog, btf_id)) + return -EINVAL; tgt_might_sleep = aux->func ? aux->func[subprog]->aux->might_sleep @@ -20212,6 +22606,12 @@ int bpf_check_attach_target(struct bpf_verifier_log *log, "Extension program may sleep, while original does not\n"); return -EINVAL; } + /* There is no unwind_goto after the calls of the original */ + if (prog->aux->can_unwind && !tgt_can_unwind) { + bpf_log(log, + "Extension program can unwind, while original does not\n"); + return -EINVAL; + } } if (!tgt_prog->jited) { bpf_log(log, "Can attach to only JITed progs\n"); @@ -20375,12 +22775,23 @@ int bpf_check_attach_target(struct bpf_verifier_log *log, prog->aux->saved_dst_attach_type != tgt_prog->expected_attach_type)) return -EINVAL; - if (tgt_prog && conservative) + if (tgt_prog && conservative) { + if (subprog > 0) { + ret = check_unreliable_func_args(log, btf, t, tname); + if (ret) + return ret; + } t = NULL; + } ret = btf_distill_func_proto(log, btf, t, tname, &tgt_info->fmodel); if (ret < 0) return ret; + if (tgt_prog) { + ret = flatten_bpf_func_model(log, tname, &tgt_info->fmodel); + if (ret < 0) + return ret; + } if (tgt_info->fmodel.ret_size > 8 && attach_uses_trampoline_retval(prog->expected_attach_type)) { @@ -20857,6 +23268,66 @@ static int specialize_kfunc(struct bpf_verifier_env *env, struct bpf_kfunc_desc return 0; } +static int add_kfunc_desc_target(struct bpf_verifier_env *env, + const struct bpf_kfunc_desc *target_desc, + u16 base_desc_idx, u16 *desc_idx) +{ + struct bpf_kfunc_desc desc = *target_desc; + struct bpf_kfunc_desc_tab *new_tab; + struct bpf_kfunc_desc_tab *tab; + struct bpf_prog_aux *prog_aux; + u32 i; + + prog_aux = env->prog->aux; + tab = prog_aux->kfunc_tab; + + if (base_desc_idx >= tab->nr_base_descs) { + verifier_bug(env, "kfunc desc_idx %d out of bounds (%d descriptors)", + base_desc_idx, tab->nr_base_descs); + return -EFAULT; + } + if (tab->descs[base_desc_idx].func_id != desc.func_id) { + verifier_bug(env, "kfunc desc %d has invalid id (expected %d, got %d)", + base_desc_idx, tab->descs[base_desc_idx].func_id, desc.func_id); + return -EFAULT; + } + if (tab->descs[base_desc_idx].offset != desc.offset) { + verifier_bug(env, "kfunc desc %d has invalid offset (expected %d, got %d)", + base_desc_idx, tab->descs[base_desc_idx].offset, desc.offset); + return -EFAULT; + } + + if (tab->descs[base_desc_idx].addr == desc.addr) { + *desc_idx = base_desc_idx; + return 0; + } + + for (i = tab->nr_base_descs; i < tab->nr_descs; i++) { + if (tab->descs[i].func_id == desc.func_id && + tab->descs[i].offset == desc.offset && + tab->descs[i].addr == desc.addr) { + *desc_idx = i; + return 0; + } + } + + if (tab->nr_descs == MAX_KFUNC_CALL_DESCS) { + verbose(env, "too many different kernel function call targets\n"); + return -E2BIG; + } + + new_tab = krealloc(tab, struct_size(tab, descs, tab->nr_descs + 1), + GFP_KERNEL_ACCOUNT); + if (!new_tab) + return -ENOMEM; + tab = new_tab; + prog_aux->kfunc_tab = tab; + + *desc_idx = tab->nr_descs; + tab->descs[tab->nr_descs++] = desc; + return 0; +} + static void __fixup_collection_insert_kfunc(struct bpf_insn_aux_data *insn_aux, u16 struct_meta_reg, u16 node_offset_reg, @@ -20877,7 +23348,11 @@ static void __fixup_collection_insert_kfunc(struct bpf_insn_aux_data *insn_aux, int bpf_fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, struct bpf_insn *insn_buf, int insn_idx, int *cnt) { + struct bpf_kfunc_desc desc_copy; struct bpf_kfunc_desc *desc; + unsigned long call_imm; + u16 base_desc_idx, desc_idx; + bool near_call; int err; if (!insn->imm) { @@ -20897,13 +23372,30 @@ int bpf_fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, insn->imm); return -EFAULT; } + base_desc_idx = desc - env->prog->aux->kfunc_tab->descs; + + near_call = !bpf_jit_supports_far_kfunc_call(); + desc_copy = *desc; + desc = &desc_copy; err = specialize_kfunc(env, desc, insn_idx); if (err) return err; - if (!bpf_jit_supports_far_kfunc_call()) - insn->imm = BPF_CALL_IMM(desc->addr); + if (near_call) { + call_imm = BPF_CALL_IMM(desc->addr); + if ((unsigned long)(s32)call_imm != call_imm) { + verbose(env, "address of kernel func_id %u is out of range\n", + desc->func_id); + return -EINVAL; + } + insn->imm = call_imm; + } + + err = add_kfunc_desc_target(env, desc, base_desc_idx, &desc_idx); + if (err) + return err; + insn->off = desc_idx; if (is_bpf_obj_new_kfunc(desc->func_id) || is_bpf_percpu_obj_new_kfunc(desc->func_id)) { struct btf_struct_meta *kptr_struct_meta = env->insn_aux_data[insn_idx].kptr_struct_meta; @@ -21063,6 +23555,14 @@ int bpf_fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, return 0; } +/* + * Upper bound on the PKCS#7 signature blob passed with a program. Comfortably + * above the largest signature the kernel can verify, and far below anything + * that would make rejecting a load expensive. Deliberately a fixed number so + * that what the syscall accepts does not depend on PAGE_SIZE. + */ +#define BPF_PROG_MAX_SIGNATURE_SIZE (64 * 1024) + static enum bpf_sig_keyring bpf_classify_keyring(s32 keyring_id) { switch (keyring_id) { @@ -21072,6 +23572,8 @@ static enum bpf_sig_keyring bpf_classify_keyring(s32 keyring_id) return BPF_SIG_KEYRING_SECONDARY; case (s32)(unsigned long)VERIFY_USE_PLATFORM_KEYRING: return BPF_SIG_KEYRING_PLATFORM; + case KEY_SPEC_BPF_KEYRING: + return BPF_SIG_KEYRING_BPF; default: return BPF_SIG_KEYRING_USER; } @@ -21101,17 +23603,33 @@ static int bpf_prog_verify_signature(struct bpf_verifier_env *env, int err = 0; /* - * Don't attempt to use kmalloc_large or vmalloc for signatures. - * Practical signature for BPF program should be below this limit. + * The signature covers the insns as submitted. CO-RE relocations would + * have the kernel rewrite them afterwards, with values the loader picks + * through the BTF it supplies, see bpf_check_core_relo(). */ + if (attr->core_relo_cnt) { + verbose(env, "signed program cannot carry CO-RE relocations\n"); + return -EACCES; + } if (!attr->signature_size || - attr->signature_size > KMALLOC_MAX_CACHE_SIZE) + attr->signature_size > BPF_PROG_MAX_SIGNATURE_SIZE) return -EINVAL; - if (system_keyring_id_check(attr->keyring_id) == 0) + + if (system_keyring_id_check(attr->keyring_id) == 0) { key = bpf_lookup_system_key(attr->keyring_id); - else + } else if (attr->keyring_id == KEY_SPEC_BPF_KEYRING) { + key = bpf_lookup_keyring(); + } else if (bpf_keyring_enforced()) { + verbose(env, "caller-supplied keyring refused, use bpf keyring\n"); + return -EPERM; + } else { key = bpf_lookup_user_key(attr->keyring_id, 0); + } if (!key) { + if (attr->keyring_id == KEY_SPEC_BPF_KEYRING) { + verbose(env, "the bpf keyring is empty or has not been restricted\n"); + return -ENOKEY; + } verbose(env, "cannot resolve signing keyring with keyring_id %d\n", attr->keyring_id); return -EINVAL; @@ -21203,6 +23721,65 @@ static int bpf_prog_verify_signature(struct bpf_verifier_env *env, return err; } +/* + * bpf_clone_redirect() transmits from inside the program, so a tc egress or + * lwt_xmit program that redirects to its own device runs again on top of its + * own frame, and again from there, until the datapath's recursion limit drops + * the packet: ten frames deep. Ten MAX_BPF_STACK frames fit the kernel stack + * as they always did; ten MAX_BPF_STACK_JIT frames would not, so a program + * calling it keeps the smaller budget. The redirect helpers that transmit + * after the program has returned leave no frame behind. + */ +static bool bpf_prog_reenters_datapath(const struct bpf_prog *prog) +{ + const struct bpf_insn *insn = prog->insnsi; + int i; + + for (i = 0; i < prog->len; i++, insn++) + if (bpf_helper_call(insn) && insn->imm == BPF_FUNC_clone_redirect) + return true; + return false; +} + +/* Various log level 2 information about the program */ +static void log_program(struct bpf_verifier_env *env) +{ + struct bpf_insn_aux_data *insn_aux = env->insn_aux_data; + struct bpf_insn *insns = env->prog->insnsi; + u32 insn_cnt = env->prog->len; + u64 pos, insn_pos; + u32 i, j; + + verbose(env, "Program dump (scc? loop_header? idom insn#: live_regs_before):\n"); + for (i = 0; i < insn_cnt; ++i) { + verbose_linfo(env, i, " ; "); + if (env->insn_aux_data[i].scc) + verbose(env, "%3d ", env->insn_aux_data[i].scc); + else + verbose(env, " "); + if (env->insn_aux_data[i].loop_header >= 0) + verbose(env, "%3d ", env->insn_aux_data[i].loop_header); + else + verbose(env, " "); + verbose(env, "%3d ", env->idoms[i]); + verbose(env, "%3d: ", i); + for (j = BPF_REG_0; j < BPF_REG_10; ++j) + if (insn_aux[i].live_regs_before & BIT(j)) + verbose(env, "%d", j); + else + verbose(env, "."); + verbose(env, " "); + pos = env->log.end_pos; + bpf_verbose_insn(env, &insns[i]); + insn_pos = env->log.end_pos; + if (insn_aux[i].zext_dst) + verbose(env, "%*c; zext", bpf_vlog_alignment(insn_pos - pos), ' '); + verbose(env, "\n"); + if (bpf_is_ldimm64(&insns[i])) + i++; + } +} + int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, struct bpf_log_attr *attr_log) { @@ -21227,6 +23804,9 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, env->bt.env = env; env->prog = *prog; env->ops = bpf_verifier_ops[env->prog->type]; + env->stack_limit = bpf_prog_stack_limit(env->prog); + if (bpf_prog_reenters_datapath(env->prog)) + env->stack_limit = MAX_BPF_STACK; env->allow_ptr_leaks = bpf_allow_ptr_leaks(env->prog->aux->token); env->allow_uninit_stack = bpf_allow_uninit_stack(env->prog->aux->token); @@ -21249,11 +23829,6 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, ret = -EINVAL; goto err_prep; } - if (env->signature) { - ret = bpf_prog_calc_tag(env->prog); - if (ret < 0) - goto err_prep; - } ret = process_fd_array(env, attr, uattr); if (ret) @@ -21307,6 +23882,7 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, if (is_priv) env->test_state_freq = attr->prog_flags & BPF_F_TEST_STATE_FREQ; env->test_reg_invariants = attr->prog_flags & BPF_F_TEST_REG_INVARIANTS; + env->arena_scalar = attr->prog_flags & BPF_F_ARENA_SCALAR; env->explored_states = kvzalloc_objs(struct list_head, state_htab_size(env), @@ -21329,22 +23905,29 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, if (ret < 0) goto skip_full_check; - /* Discover all subprograms before validating their layout and BTF. */ + /* Discover all subprograms and collect the properties needed by BTF validation. */ ret = add_subprogs(env); if (ret < 0) goto skip_full_check; + find_subprog_properties(env); + + /* Validate BTF before reporting subprogram layout errors. */ + ret = bpf_check_btf_info(env, attr, uattr); + if (ret < 0) + goto skip_full_check; + ret = check_subprogs(env); if (ret < 0) goto skip_full_check; - /* Validate BTF against the complete subprogram layout. */ - ret = bpf_check_btf_info(env, attr, uattr); + /* Validate instructions and resolve the program's referenced resources. */ + ret = check_and_resolve_insns(env); if (ret < 0) goto skip_full_check; - /* Validate instructions and resolve the program's referenced resources. */ - ret = check_and_resolve_insns(env); + /* Find pointers to functions in the read-only maps of the program. */ + ret = resolve_func_ptrs(env); if (ret < 0) goto skip_full_check; @@ -21363,6 +23946,10 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, if (ret < 0) goto skip_full_check; + ret = check_unwind_insns(env); + if (ret < 0) + goto skip_full_check; + ret = bpf_compute_postorder(env); if (ret < 0) goto skip_full_check; @@ -21391,10 +23978,38 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, if (ret < 0) goto skip_full_check; + /* must precede the first bpf_ret_reg_pair() user below */ + ret = bpf_compute_subprog_ret_regs(env); + if (ret < 0) + goto skip_full_check; + + prepare_subprog_args(env); + + ret = bpf_compute_idoms(env); + if (ret < 0) + goto skip_full_check; + + ret = bpf_compute_loops(env); + if (ret < 0) + goto skip_full_check; + ret = bpf_compute_live_registers(env); if (ret < 0) goto skip_full_check; + if (env->log.level & BPF_LOG_LEVEL2) + log_program(env); + + ret = bpf_init_scev(env); + if (ret < 0) + goto skip_full_check; + + ret = bpf_compute_scev(env); + if (ret < 0) + goto skip_full_check; + + bpf_free_subprog_jts(env); + ret = mark_fastcall_patterns(env); if (ret < 0) goto skip_full_check; @@ -21402,6 +24017,10 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, ret = do_check_main(env); ret = ret ?: do_check_subprogs(env); + /* reject recursion through the callx edges found by the main pass */ + if (ret == 0 && env->callx_edges) + ret = sort_subprogs_topo(env); + if (ret == 0 && bpf_prog_is_offloaded(env->prog->aux)) ret = bpf_prog_offload_finalize(env); @@ -21535,16 +24154,22 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, release_maps(env); release_btfs(env); err_free_env: + bpf_free_subprog_jts(env); if (env->insn_aux_data) bpf_clear_insn_aux_data(env, 0, env->insn_aux_data_len); vfree(env->insn_aux_data); kvfree(env->fd_array); + bpf_free_scev(env); bpf_stack_liveness_free(env); + kvfree(env->cfg.postorder_nums); kvfree(env->cfg.insn_postorder); kvfree(env->scc_info); kvfree(env->succ); kvfree(env->gotox_tmp_buf); + kvfree(env->callx_edges); + kvfree(env->func_ptrs); bpf_diag_free(env); + kvfree(env->idoms); kvfree(env); return ret; } diff --git a/kernel/module/main.c b/kernel/module/main.c index 4752777e5bef..34e0204ea37c 100644 --- a/kernel/module/main.c +++ b/kernel/module/main.c @@ -2735,6 +2735,10 @@ static int find_module_sections(struct module *mod, struct load_info *info) mod->btf_base_data = any_section_objs(info, ".BTF.base", 1, &mod->btf_base_data_size); #endif +#if IS_ENABLED(CONFIG_DEBUG_INFO_BTF_INLINE) + mod->btf_inline_data = any_section_objs(info, ".BTF.inline", 1, + &mod->btf_inline_data_size); +#endif #ifdef CONFIG_JUMP_LABEL mod->jump_entries = section_objs(info, "__jump_table", sizeof(*mod->jump_entries), @@ -3188,6 +3192,9 @@ static noinline int do_init_module(struct module *mod) /* .BTF is not SHF_ALLOC and will get removed, so sanitize pointers */ mod->btf_data = NULL; mod->btf_base_data = NULL; +#endif +#if IS_ENABLED(CONFIG_DEBUG_INFO_BTF_INLINE) + mod->btf_inline_data = NULL; #endif /* * We want to free module_init, but be aware that kallsyms may be diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c index 88c3b5b513ab..dfac5e1b8f40 100644 --- a/kernel/trace/bpf_trace.c +++ b/kernel/trace/bpf_trace.c @@ -823,6 +823,7 @@ const struct bpf_func_proto bpf_task_pt_regs_proto = { struct send_signal_irq_work { struct irq_work irq_work; + /* The claim: NULL marks the slot free for the next caller. */ struct task_struct *task; u32 sig; enum pid_type type; @@ -842,6 +843,8 @@ static void do_bpf_send_signal(struct irq_work *entry) group_send_sig_info(work->sig, siginfo, work->task, work->type); put_task_struct(work->task); + /* Release once the fields are consumed. */ + smp_store_release(&work->task, NULL); } static int bpf_send_signal_common(u32 sig, enum pid_type type, struct task_struct *task, u64 value) @@ -885,14 +888,14 @@ static int bpf_send_signal_common(u32 sig, enum pid_type type, struct task_struc return -EINVAL; work = this_cpu_ptr(&send_signal_work); - if (irq_work_is_busy(&work->irq_work)) + if (cmpxchg(&work->task, NULL, task)) return -EBUSY; /* Add the current task, which is the target of sending signal, * to the irq_work. The current task may change when queued * irq works get executed. */ - work->task = get_task_struct(task); + get_task_struct(task); work->has_siginfo = siginfo == &info; if (work->has_siginfo) copy_siginfo(&work->info, &info); @@ -995,7 +998,7 @@ static const struct bpf_func_proto bpf_d_path_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_BTF_ID, .arg1_btf_id = &bpf_d_path_btf_ids[0], - .arg2_type = ARG_PTR_TO_MEM | MEM_WRITE, + .arg2_type = ARG_PTR_TO_MEM | MEM_UNINIT | MEM_WRITE, .arg3_type = ARG_MEM_SIZE_OR_ZERO, .allowed = bpf_d_path_allowed, }; @@ -1052,7 +1055,7 @@ const struct bpf_func_proto bpf_snprintf_btf_proto = { .func = bpf_snprintf_btf, .gpl_only = false, .ret_type = RET_INTEGER, - .arg1_type = ARG_PTR_TO_MEM | MEM_WRITE, + .arg1_type = ARG_PTR_TO_MEM | MEM_UNINIT | MEM_WRITE, .arg2_type = ARG_MEM_SIZE, .arg3_type = ARG_PTR_TO_MEM | MEM_RDONLY, .arg4_type = ARG_MEM_SIZE, @@ -1565,7 +1568,7 @@ static const struct bpf_func_proto bpf_read_branch_records_proto = { .gpl_only = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, - .arg2_type = ARG_PTR_TO_MEM_OR_NULL | MEM_WRITE, + .arg2_type = ARG_PTR_TO_MEM_OR_NULL | MEM_UNINIT | MEM_WRITE, .arg3_type = ARG_MEM_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, }; @@ -2206,6 +2209,7 @@ int bpf_probe_unregister(struct bpf_raw_event_map *btp, struct bpf_raw_tp_link * return tracepoint_probe_unregister(btp->tp, (void *)btp->bpf_func, link); } +/* On success the caller releases *buf with kfree_const() */ int bpf_get_perf_event_info(const struct perf_event *event, u32 *prog_id, u32 *fd_type, const char **buf, u64 *probe_offset, u64 *probe_addr, @@ -2215,6 +2219,12 @@ int bpf_get_perf_event_info(const struct perf_event *event, u32 *prog_id, struct bpf_prog *prog; int flags, err = 0; + /* + * The list of dynamic events is walked, and a probe which has siblings + * can be deleted, with its strings, while its event is in use. + */ + guard(mutex)(&event_mutex); + prog = event->prog; if (!prog) return -ENOENT; @@ -2255,6 +2265,12 @@ int bpf_get_perf_event_info(const struct perf_event *event, u32 *prog_id, #endif } + if (!err && *buf) { + *buf = kstrdup_const(*buf, GFP_KERNEL); + if (!*buf) + err = -ENOMEM; + } + return err; } diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h index 3749485a7d85..9044f03fe2d9 100644 --- a/kernel/trace/trace.h +++ b/kernel/trace/trace.h @@ -1006,9 +1006,6 @@ struct ftrace_hash { struct rcu_head rcu; }; -struct ftrace_func_entry * -ftrace_lookup_ip(struct ftrace_hash *hash, unsigned long ip); - static __always_inline bool ftrace_hash_empty(struct ftrace_hash *hash) { return !hash || !(hash->count || (hash->flags & FTRACE_HASH_FL_MOD)); diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index 004188ac0d52..6759087154ed 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -425,6 +425,12 @@ config PAHOLE_HAS_LANG_EXCLUDE otherwise it would emit malformed kernel and module binaries when using DEBUG_INFO_BTF_MODULES. +config PAHOLE_HAS_INLINE + def_bool PAHOLE_VERSION >= 132 + help + Support for the "inline" BTF feature is available. It encodes + information about inline sites and how to retrieve their parameters. + config DEBUG_INFO_BTF_MODULES bool "Generate BTF type information for kernel modules" default y @@ -432,6 +438,18 @@ config DEBUG_INFO_BTF_MODULES help Generate compact split BTF type information for kernel modules. +config DEBUG_INFO_BTF_INLINE + tristate "Provide information about inline sites in BTF" + default n + depends on DEBUG_INFO_BTF && PAHOLE_HAS_INLINE && SYSFS + select DEBUG_INFO_BTF_MODULES if MODULES + help + Generate information about inline sites in .BTF.inline sections. + These sections contain split BTF relative to the kernel or module BTF + and are made available in /sys/kernel/btf with a ".inline" suffix. + The information describes inline locations and how to retrieve their + associated parameters. + config MODULE_ALLOW_BTF_MISMATCH bool "Allow loading modules with non-matching BTF type info" depends on DEBUG_INFO_BTF_MODULES @@ -2649,19 +2667,6 @@ config TEST_WORKQUEUE If unsure, say N. -config TEST_BPF - tristate "Test BPF filter functionality" - depends on m && NET - help - This builds the "test_bpf" module that runs various test vectors - against the BPF interpreter or BPF JIT compiler depending on the - current setting. This is in particular useful for BPF JIT compiler - development, but also to run regression tests against changes in - the interpreter code. It also enables test stubs for eBPF maps and - verifier used by user space verifier testsuite. - - If unsure, say N. - config FIND_BIT_BENCHMARK tristate "Test find_bit functions" help diff --git a/lib/Makefile b/lib/Makefile index 3104710e1e68..a2afe954805c 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -66,7 +66,6 @@ obj-y += kstrtox.o obj-$(CONFIG_FIND_BIT_BENCHMARK) += find_bit_benchmark.o obj-$(CONFIG_REGION_ALLOC_BENCHMARK) += region_alloc_benchmark.o obj-$(CONFIG_FIND_BIT_BENCHMARK_RUST) += find_bit_benchmark_rust.o -obj-$(CONFIG_TEST_BPF) += test_bpf.o test_dhry-objs := dhry_1.o dhry_2.o dhry_run.o obj-$(CONFIG_TEST_DHRY) += test_dhry.o obj-$(CONFIG_TEST_FIRMWARE) += test_firmware.o diff --git a/lib/buildid.c b/lib/buildid.c index c4b737640621..995a327f46ae 100644 --- a/lib/buildid.c +++ b/lib/buildid.c @@ -47,6 +47,10 @@ static int freader_get_folio(struct freader *r, loff_t file_off) freader_put_folio(r); + /* the page index must fit in pgoff_t */ + if (file_off < 0 || file_off > MAX_LFS_FILESIZE) + return -EFAULT; + /* only use page cache lookup - fail if not already cached */ r->folio = filemap_get_folio(r->file->f_mapping, file_off >> PAGE_SHIFT); diff --git a/lib/test_bpf.c b/lib/test_bpf.c deleted file mode 100644 index af6f3340c034..000000000000 --- a/lib/test_bpf.c +++ /dev/null @@ -1,15934 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Testsuite for BPF interpreter and BPF JIT compiler - * - * Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com - */ - -#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -/* General test specific settings */ -#define MAX_SUBTESTS 3 -#define MAX_TESTRUNS 1000 -#define MAX_DATA 128 -#define MAX_INSNS 512 -#define MAX_K 0xffffFFFF - -/* Few constants used to init test 'skb' */ -#define SKB_TYPE 3 -#define SKB_MARK 0x1234aaaa -#define SKB_HASH 0x1234aaab -#define SKB_QUEUE_MAP 123 -#define SKB_VLAN_TCI 0xffff -#define SKB_VLAN_PRESENT 1 -#define SKB_DEV_IFINDEX 577 -#define SKB_DEV_TYPE 588 - -/* Redefine REGs to make tests less verbose */ -#define R0 BPF_REG_0 -#define R1 BPF_REG_1 -#define R2 BPF_REG_2 -#define R3 BPF_REG_3 -#define R4 BPF_REG_4 -#define R5 BPF_REG_5 -#define R6 BPF_REG_6 -#define R7 BPF_REG_7 -#define R8 BPF_REG_8 -#define R9 BPF_REG_9 -#define R10 BPF_REG_10 - -/* Flags that can be passed to test cases */ -#define FLAG_NO_DATA BIT(0) -#define FLAG_EXPECTED_FAIL BIT(1) -#define FLAG_SKB_FRAG BIT(2) -#define FLAG_VERIFIER_ZEXT BIT(3) -#define FLAG_LARGE_MEM BIT(4) - -enum { - CLASSIC = BIT(6), /* Old BPF instructions only. */ - INTERNAL = BIT(7), /* Extended instruction set. */ -}; - -#define TEST_TYPE_MASK (CLASSIC | INTERNAL) - -struct bpf_test { - const char *descr; - union { - struct sock_filter insns[MAX_INSNS]; - struct bpf_insn insns_int[MAX_INSNS]; - struct { - void *insns; - unsigned int len; - } ptr; - } u; - __u8 aux; - __u8 data[MAX_DATA]; - struct { - int data_size; - __u32 result; - } test[MAX_SUBTESTS]; - int (*fill_helper)(struct bpf_test *self); - int expected_errcode; /* used when FLAG_EXPECTED_FAIL is set in the aux */ - __u8 frag_data[MAX_DATA]; - int stack_depth; /* for eBPF only, since tests don't call verifier */ - int nr_testruns; /* Custom run count, defaults to MAX_TESTRUNS if 0 */ -}; - -/* Large test cases need separate allocation and fill handler. */ - -static int bpf_fill_maxinsns1(struct bpf_test *self) -{ - unsigned int len = BPF_MAXINSNS; - struct sock_filter *insn; - __u32 k = ~0; - int i; - - insn = kmalloc_objs(*insn, len); - if (!insn) - return -ENOMEM; - - for (i = 0; i < len; i++, k--) - insn[i] = __BPF_STMT(BPF_RET | BPF_K, k); - - self->u.ptr.insns = insn; - self->u.ptr.len = len; - - return 0; -} - -static int bpf_fill_maxinsns2(struct bpf_test *self) -{ - unsigned int len = BPF_MAXINSNS; - struct sock_filter *insn; - int i; - - insn = kmalloc_objs(*insn, len); - if (!insn) - return -ENOMEM; - - for (i = 0; i < len; i++) - insn[i] = __BPF_STMT(BPF_RET | BPF_K, 0xfefefefe); - - self->u.ptr.insns = insn; - self->u.ptr.len = len; - - return 0; -} - -static int bpf_fill_maxinsns3(struct bpf_test *self) -{ - unsigned int len = BPF_MAXINSNS; - struct sock_filter *insn; - struct rnd_state rnd; - int i; - - insn = kmalloc_objs(*insn, len); - if (!insn) - return -ENOMEM; - - prandom_seed_state(&rnd, 3141592653589793238ULL); - - for (i = 0; i < len - 1; i++) { - __u32 k = prandom_u32_state(&rnd); - - insn[i] = __BPF_STMT(BPF_ALU | BPF_ADD | BPF_K, k); - } - - insn[len - 1] = __BPF_STMT(BPF_RET | BPF_A, 0); - - self->u.ptr.insns = insn; - self->u.ptr.len = len; - - return 0; -} - -static int bpf_fill_maxinsns4(struct bpf_test *self) -{ - unsigned int len = BPF_MAXINSNS + 1; - struct sock_filter *insn; - int i; - - insn = kmalloc_objs(*insn, len); - if (!insn) - return -ENOMEM; - - for (i = 0; i < len; i++) - insn[i] = __BPF_STMT(BPF_RET | BPF_K, 0xfefefefe); - - self->u.ptr.insns = insn; - self->u.ptr.len = len; - - return 0; -} - -static int bpf_fill_maxinsns5(struct bpf_test *self) -{ - unsigned int len = BPF_MAXINSNS; - struct sock_filter *insn; - int i; - - insn = kmalloc_objs(*insn, len); - if (!insn) - return -ENOMEM; - - insn[0] = __BPF_JUMP(BPF_JMP | BPF_JA, len - 2, 0, 0); - - for (i = 1; i < len - 1; i++) - insn[i] = __BPF_STMT(BPF_RET | BPF_K, 0xfefefefe); - - insn[len - 1] = __BPF_STMT(BPF_RET | BPF_K, 0xabababab); - - self->u.ptr.insns = insn; - self->u.ptr.len = len; - - return 0; -} - -static int bpf_fill_maxinsns6(struct bpf_test *self) -{ - unsigned int len = BPF_MAXINSNS; - struct sock_filter *insn; - int i; - - insn = kmalloc_objs(*insn, len); - if (!insn) - return -ENOMEM; - - for (i = 0; i < len - 1; i++) - insn[i] = __BPF_STMT(BPF_LD | BPF_W | BPF_ABS, SKF_AD_OFF + - SKF_AD_VLAN_TAG_PRESENT); - - insn[len - 1] = __BPF_STMT(BPF_RET | BPF_A, 0); - - self->u.ptr.insns = insn; - self->u.ptr.len = len; - - return 0; -} - -static int bpf_fill_maxinsns7(struct bpf_test *self) -{ - unsigned int len = BPF_MAXINSNS; - struct sock_filter *insn; - int i; - - insn = kmalloc_objs(*insn, len); - if (!insn) - return -ENOMEM; - - for (i = 0; i < len - 4; i++) - insn[i] = __BPF_STMT(BPF_LD | BPF_W | BPF_ABS, SKF_AD_OFF + - SKF_AD_CPU); - - insn[len - 4] = __BPF_STMT(BPF_MISC | BPF_TAX, 0); - insn[len - 3] = __BPF_STMT(BPF_LD | BPF_W | BPF_ABS, SKF_AD_OFF + - SKF_AD_CPU); - insn[len - 2] = __BPF_STMT(BPF_ALU | BPF_SUB | BPF_X, 0); - insn[len - 1] = __BPF_STMT(BPF_RET | BPF_A, 0); - - self->u.ptr.insns = insn; - self->u.ptr.len = len; - - return 0; -} - -static int bpf_fill_maxinsns8(struct bpf_test *self) -{ - unsigned int len = BPF_MAXINSNS; - struct sock_filter *insn; - int i, jmp_off = len - 3; - - insn = kmalloc_objs(*insn, len); - if (!insn) - return -ENOMEM; - - insn[0] = __BPF_STMT(BPF_LD | BPF_IMM, 0xffffffff); - - for (i = 1; i < len - 1; i++) - insn[i] = __BPF_JUMP(BPF_JMP | BPF_JGT, 0xffffffff, jmp_off--, 0); - - insn[len - 1] = __BPF_STMT(BPF_RET | BPF_A, 0); - - self->u.ptr.insns = insn; - self->u.ptr.len = len; - - return 0; -} - -static int bpf_fill_maxinsns9(struct bpf_test *self) -{ - unsigned int len = BPF_MAXINSNS; - struct bpf_insn *insn; - int i; - - insn = kmalloc_objs(*insn, len); - if (!insn) - return -ENOMEM; - - insn[0] = BPF_JMP_IMM(BPF_JA, 0, 0, len - 2); - insn[1] = BPF_ALU32_IMM(BPF_MOV, R0, 0xcbababab); - insn[2] = BPF_EXIT_INSN(); - - for (i = 3; i < len - 2; i++) - insn[i] = BPF_ALU32_IMM(BPF_MOV, R0, 0xfefefefe); - - insn[len - 2] = BPF_EXIT_INSN(); - insn[len - 1] = BPF_JMP_IMM(BPF_JA, 0, 0, -(len - 1)); - - self->u.ptr.insns = insn; - self->u.ptr.len = len; - - return 0; -} - -static int bpf_fill_maxinsns10(struct bpf_test *self) -{ - unsigned int len = BPF_MAXINSNS, hlen = len - 2; - struct bpf_insn *insn; - int i; - - insn = kmalloc_objs(*insn, len); - if (!insn) - return -ENOMEM; - - for (i = 0; i < hlen / 2; i++) - insn[i] = BPF_JMP_IMM(BPF_JA, 0, 0, hlen - 2 - 2 * i); - for (i = hlen - 1; i > hlen / 2; i--) - insn[i] = BPF_JMP_IMM(BPF_JA, 0, 0, hlen - 1 - 2 * i); - - insn[hlen / 2] = BPF_JMP_IMM(BPF_JA, 0, 0, hlen / 2 - 1); - insn[hlen] = BPF_ALU32_IMM(BPF_MOV, R0, 0xabababac); - insn[hlen + 1] = BPF_EXIT_INSN(); - - self->u.ptr.insns = insn; - self->u.ptr.len = len; - - return 0; -} - -static int __bpf_fill_ja(struct bpf_test *self, unsigned int len, - unsigned int plen) -{ - struct sock_filter *insn; - unsigned int rlen; - int i, j; - - insn = kmalloc_objs(*insn, len); - if (!insn) - return -ENOMEM; - - rlen = (len % plen) - 1; - - for (i = 0; i + plen < len; i += plen) - for (j = 0; j < plen; j++) - insn[i + j] = __BPF_JUMP(BPF_JMP | BPF_JA, - plen - 1 - j, 0, 0); - for (j = 0; j < rlen; j++) - insn[i + j] = __BPF_JUMP(BPF_JMP | BPF_JA, rlen - 1 - j, - 0, 0); - - insn[len - 1] = __BPF_STMT(BPF_RET | BPF_K, 0xababcbac); - - self->u.ptr.insns = insn; - self->u.ptr.len = len; - - return 0; -} - -static int bpf_fill_maxinsns11(struct bpf_test *self) -{ - /* Hits 70 passes on x86_64 and triggers NOPs padding. */ - return __bpf_fill_ja(self, BPF_MAXINSNS, 68); -} - -static int bpf_fill_maxinsns12(struct bpf_test *self) -{ - unsigned int len = BPF_MAXINSNS; - struct sock_filter *insn; - int i = 0; - - insn = kmalloc_objs(*insn, len); - if (!insn) - return -ENOMEM; - - insn[0] = __BPF_JUMP(BPF_JMP | BPF_JA, len - 2, 0, 0); - - for (i = 1; i < len - 1; i++) - insn[i] = __BPF_STMT(BPF_LDX | BPF_B | BPF_MSH, 0); - - insn[len - 1] = __BPF_STMT(BPF_RET | BPF_K, 0xabababab); - - self->u.ptr.insns = insn; - self->u.ptr.len = len; - - return 0; -} - -static int bpf_fill_maxinsns13(struct bpf_test *self) -{ - unsigned int len = BPF_MAXINSNS; - struct sock_filter *insn; - int i = 0; - - insn = kmalloc_objs(*insn, len); - if (!insn) - return -ENOMEM; - - for (i = 0; i < len - 3; i++) - insn[i] = __BPF_STMT(BPF_LDX | BPF_B | BPF_MSH, 0); - - insn[len - 3] = __BPF_STMT(BPF_LD | BPF_IMM, 0xabababab); - insn[len - 2] = __BPF_STMT(BPF_ALU | BPF_XOR | BPF_X, 0); - insn[len - 1] = __BPF_STMT(BPF_RET | BPF_A, 0); - - self->u.ptr.insns = insn; - self->u.ptr.len = len; - - return 0; -} - -static int bpf_fill_ja(struct bpf_test *self) -{ - /* Hits exactly 11 passes on x86_64 JIT. */ - return __bpf_fill_ja(self, 12, 9); -} - -static int bpf_fill_ld_abs_get_processor_id(struct bpf_test *self) -{ - unsigned int len = BPF_MAXINSNS; - struct sock_filter *insn; - int i; - - insn = kmalloc_objs(*insn, len); - if (!insn) - return -ENOMEM; - - for (i = 0; i < len - 1; i += 2) { - insn[i] = __BPF_STMT(BPF_LD | BPF_B | BPF_ABS, 0); - insn[i + 1] = __BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_CPU); - } - - insn[len - 1] = __BPF_STMT(BPF_RET | BPF_K, 0xbee); - - self->u.ptr.insns = insn; - self->u.ptr.len = len; - - return 0; -} - -static int __bpf_fill_stxdw(struct bpf_test *self, int size) -{ - unsigned int len = BPF_MAXINSNS; - struct bpf_insn *insn; - int i; - - insn = kmalloc_objs(*insn, len); - if (!insn) - return -ENOMEM; - - insn[0] = BPF_ALU32_IMM(BPF_MOV, R0, 1); - insn[1] = BPF_ST_MEM(size, R10, -40, 42); - - for (i = 2; i < len - 2; i++) - insn[i] = BPF_STX_XADD(size, R10, R0, -40); - - insn[len - 2] = BPF_LDX_MEM(size, R0, R10, -40); - insn[len - 1] = BPF_EXIT_INSN(); - - self->u.ptr.insns = insn; - self->u.ptr.len = len; - self->stack_depth = 40; - - return 0; -} - -static int bpf_fill_stxw(struct bpf_test *self) -{ - return __bpf_fill_stxdw(self, BPF_W); -} - -static int bpf_fill_stxdw(struct bpf_test *self) -{ - return __bpf_fill_stxdw(self, BPF_DW); -} - -static int __bpf_ld_imm64(struct bpf_insn insns[2], u8 reg, s64 imm64) -{ - struct bpf_insn tmp[] = {BPF_LD_IMM64(reg, imm64)}; - - memcpy(insns, tmp, sizeof(tmp)); - return 2; -} - -/* - * Branch conversion tests. Complex operations can expand to a lot - * of instructions when JITed. This in turn may cause jump offsets - * to overflow the field size of the native instruction, triggering - * a branch conversion mechanism in some JITs. - */ -static int __bpf_fill_max_jmp(struct bpf_test *self, int jmp, int imm, bool alu32) -{ - struct bpf_insn *insns; - int len = S16_MAX + 5; - int i; - - insns = kmalloc_objs(*insns, len); - if (!insns) - return -ENOMEM; - - i = __bpf_ld_imm64(insns, R1, 0x0123456789abcdefULL); - insns[i++] = BPF_ALU64_IMM(BPF_MOV, R0, 1); - insns[i++] = BPF_JMP_IMM(jmp, R0, imm, S16_MAX); - insns[i++] = BPF_ALU64_IMM(BPF_MOV, R0, 2); - insns[i++] = BPF_EXIT_INSN(); - - while (i < len - 1) { - static const int ops[] = { - BPF_LSH, BPF_RSH, BPF_ARSH, BPF_ADD, - BPF_SUB, BPF_MUL, BPF_DIV, BPF_MOD, - }; - int op = ops[(i >> 1) % ARRAY_SIZE(ops)]; - - if ((i & 1) || alu32) - insns[i++] = BPF_ALU32_REG(op, R0, R1); - else - insns[i++] = BPF_ALU64_REG(op, R0, R1); - } - - insns[i++] = BPF_EXIT_INSN(); - self->u.ptr.insns = insns; - self->u.ptr.len = len; - BUG_ON(i != len); - - return 0; -} - -/* Branch taken by runtime decision */ -static int bpf_fill_max_jmp_taken_32(struct bpf_test *self) -{ - return __bpf_fill_max_jmp(self, BPF_JEQ, 1, true); -} - -static int bpf_fill_max_jmp_taken(struct bpf_test *self) -{ - return __bpf_fill_max_jmp(self, BPF_JEQ, 1, false); -} - -/* Branch not taken by runtime decision */ -static int bpf_fill_max_jmp_not_taken_32(struct bpf_test *self) -{ - return __bpf_fill_max_jmp(self, BPF_JEQ, 0, true); -} - -static int bpf_fill_max_jmp_not_taken(struct bpf_test *self) -{ - return __bpf_fill_max_jmp(self, BPF_JEQ, 0, false); -} - -/* Branch always taken, known at JIT time */ -static int bpf_fill_max_jmp_always_taken_32(struct bpf_test *self) -{ - return __bpf_fill_max_jmp(self, BPF_JGE, 0, true); -} - -static int bpf_fill_max_jmp_always_taken(struct bpf_test *self) -{ - return __bpf_fill_max_jmp(self, BPF_JGE, 0, false); -} - -/* Branch never taken, known at JIT time */ -static int bpf_fill_max_jmp_never_taken_32(struct bpf_test *self) -{ - return __bpf_fill_max_jmp(self, BPF_JLT, 0, true); -} - -static int bpf_fill_max_jmp_never_taken(struct bpf_test *self) -{ - return __bpf_fill_max_jmp(self, BPF_JLT, 0, false); -} - -/* ALU result computation used in tests */ -enum { F_ALU32 = 1, F_SIGNED = 2 }; - -static bool __bpf_alu_result(u64 *res, u64 v1, u64 v2, u8 op, u32 flags) -{ - bool is_signed = flags & F_SIGNED; - - /* Narrow operands for ALU32 */ - if (flags & F_ALU32) { - if (is_signed) { - v1 = (u64)(s32)v1; - v2 = (u64)(s32)v2; - } else { - v1 = (u32)v1; - v2 = (u32)v2; - } - } - - *res = 0; - switch (op) { - case BPF_MOV: - *res = v2; - break; - case BPF_AND: - *res = v1 & v2; - break; - case BPF_OR: - *res = v1 | v2; - break; - case BPF_XOR: - *res = v1 ^ v2; - break; - case BPF_LSH: - *res = v1 << v2; - break; - case BPF_RSH: - *res = v1 >> v2; - break; - case BPF_ARSH: - *res = v1 >> v2; - if (v2 > 0 && v1 > S64_MAX) - *res |= ~0ULL << (64 - v2); - break; - case BPF_ADD: - *res = v1 + v2; - break; - case BPF_SUB: - *res = v1 - v2; - break; - case BPF_MUL: - *res = v1 * v2; - break; - case BPF_DIV: - if (v2 == 0) - return false; - if (!is_signed) { - *res = div64_u64(v1, v2); - } else { - if ((s64)v2 == -1) /* Handled by verifier */ - return false; - *res = (u64)div64_s64(v1, v2); - } - break; - case BPF_MOD: - if (v2 == 0) - return false; - if (!is_signed) { - div64_u64_rem(v1, v2, res); - } else { - if ((s64)v2 == -1) - return false; - /* - * Avoid s64 % s64 which generates __moddi3 on - * 32-bit architectures. Use div64_s64 instead. - */ - *res = (u64)((s64)v1 - div64_s64(v1, v2) * (s64)v2); - } - break; - } - return true; -} - -/* Test an ALU shift operation for all valid shift values */ -static int __bpf_fill_alu_shift(struct bpf_test *self, u8 op, - u8 mode, u32 flags) -{ - static const s64 regs[] = { - 0x0123456789abcdefLL, /* dword > 0, word < 0 */ - 0xfedcba9876543210LL, /* dword < 0, word > 0 */ - 0xfedcba0198765432LL, /* dword < 0, word < 0 */ - 0x0123458967abcdefLL, /* dword > 0, word > 0 */ - }; - int bits = (flags & F_ALU32) ? 32 : 64; - int len = (2 + 7 * bits) * ARRAY_SIZE(regs) + 3; - struct bpf_insn *insn; - int imm, k; - int i = 0; - - insn = kmalloc_objs(*insn, len); - if (!insn) - return -ENOMEM; - - insn[i++] = BPF_ALU64_IMM(BPF_MOV, R0, 0); - - for (k = 0; k < ARRAY_SIZE(regs); k++) { - s64 reg = regs[k]; - - i += __bpf_ld_imm64(&insn[i], R3, reg); - - for (imm = 0; imm < bits; imm++) { - u64 val; - - /* Perform operation */ - insn[i++] = BPF_ALU64_REG(BPF_MOV, R1, R3); - insn[i++] = BPF_ALU64_IMM(BPF_MOV, R2, imm); - if (flags & F_ALU32) { - if (mode == BPF_K) - insn[i++] = BPF_ALU32_IMM(op, R1, imm); - else - insn[i++] = BPF_ALU32_REG(op, R1, R2); - - if (op == BPF_ARSH) - reg = (s32)reg; - else - reg = (u32)reg; - __bpf_alu_result(&val, reg, imm, op, 0); - val = (u32)val; - } else { - if (mode == BPF_K) - insn[i++] = BPF_ALU64_IMM(op, R1, imm); - else - insn[i++] = BPF_ALU64_REG(op, R1, R2); - __bpf_alu_result(&val, reg, imm, op, 0); - } - - /* - * When debugging a JIT that fails this test, one - * can write the immediate value to R0 here to find - * out which operand values that fail. - */ - - /* Load reference and check the result */ - i += __bpf_ld_imm64(&insn[i], R4, val); - insn[i++] = BPF_JMP_REG(BPF_JEQ, R1, R4, 1); - insn[i++] = BPF_EXIT_INSN(); - } - } - - insn[i++] = BPF_ALU64_IMM(BPF_MOV, R0, 1); - insn[i++] = BPF_EXIT_INSN(); - - self->u.ptr.insns = insn; - self->u.ptr.len = len; - BUG_ON(i != len); - - return 0; -} - -static int bpf_fill_alu64_lsh_imm(struct bpf_test *self) -{ - return __bpf_fill_alu_shift(self, BPF_LSH, BPF_K, 0); -} - -static int bpf_fill_alu64_rsh_imm(struct bpf_test *self) -{ - return __bpf_fill_alu_shift(self, BPF_RSH, BPF_K, 0); -} - -static int bpf_fill_alu64_arsh_imm(struct bpf_test *self) -{ - return __bpf_fill_alu_shift(self, BPF_ARSH, BPF_K, 0); -} - -static int bpf_fill_alu64_lsh_reg(struct bpf_test *self) -{ - return __bpf_fill_alu_shift(self, BPF_LSH, BPF_X, 0); -} - -static int bpf_fill_alu64_rsh_reg(struct bpf_test *self) -{ - return __bpf_fill_alu_shift(self, BPF_RSH, BPF_X, 0); -} - -static int bpf_fill_alu64_arsh_reg(struct bpf_test *self) -{ - return __bpf_fill_alu_shift(self, BPF_ARSH, BPF_X, 0); -} - -static int bpf_fill_alu32_lsh_imm(struct bpf_test *self) -{ - return __bpf_fill_alu_shift(self, BPF_LSH, BPF_K, F_ALU32); -} - -static int bpf_fill_alu32_rsh_imm(struct bpf_test *self) -{ - return __bpf_fill_alu_shift(self, BPF_RSH, BPF_K, F_ALU32); -} - -static int bpf_fill_alu32_arsh_imm(struct bpf_test *self) -{ - return __bpf_fill_alu_shift(self, BPF_ARSH, BPF_K, F_ALU32); -} - -static int bpf_fill_alu32_lsh_reg(struct bpf_test *self) -{ - return __bpf_fill_alu_shift(self, BPF_LSH, BPF_X, F_ALU32); -} - -static int bpf_fill_alu32_rsh_reg(struct bpf_test *self) -{ - return __bpf_fill_alu_shift(self, BPF_RSH, BPF_X, F_ALU32); -} - -static int bpf_fill_alu32_arsh_reg(struct bpf_test *self) -{ - return __bpf_fill_alu_shift(self, BPF_ARSH, BPF_X, F_ALU32); -} - -/* - * Test an ALU register shift operation for all valid shift values - * for the case when the source and destination are the same. - */ -static int __bpf_fill_alu_shift_same_reg(struct bpf_test *self, u8 op, - u32 flags) -{ - int bits = (flags & F_ALU32) ? 32 : 64; - int len = 3 + 6 * bits; - struct bpf_insn *insn; - int i = 0; - u64 val; - - insn = kmalloc_objs(*insn, len); - if (!insn) - return -ENOMEM; - - insn[i++] = BPF_ALU64_IMM(BPF_MOV, R0, 0); - - for (val = 0; val < bits; val++) { - u64 res; - - /* Perform operation */ - insn[i++] = BPF_ALU64_IMM(BPF_MOV, R1, val); - if (flags & F_ALU32) - insn[i++] = BPF_ALU32_REG(op, R1, R1); - else - insn[i++] = BPF_ALU64_REG(op, R1, R1); - - /* Compute the reference result */ - __bpf_alu_result(&res, val, val, op, 0); - if (flags & F_ALU32) - res = (u32)res; - i += __bpf_ld_imm64(&insn[i], R2, res); - - /* Check the actual result */ - insn[i++] = BPF_JMP_REG(BPF_JEQ, R1, R2, 1); - insn[i++] = BPF_EXIT_INSN(); - } - - insn[i++] = BPF_ALU64_IMM(BPF_MOV, R0, 1); - insn[i++] = BPF_EXIT_INSN(); - - self->u.ptr.insns = insn; - self->u.ptr.len = len; - BUG_ON(i != len); - - return 0; -} - -static int bpf_fill_alu64_lsh_same_reg(struct bpf_test *self) -{ - return __bpf_fill_alu_shift_same_reg(self, BPF_LSH, 0); -} - -static int bpf_fill_alu64_rsh_same_reg(struct bpf_test *self) -{ - return __bpf_fill_alu_shift_same_reg(self, BPF_RSH, 0); -} - -static int bpf_fill_alu64_arsh_same_reg(struct bpf_test *self) -{ - return __bpf_fill_alu_shift_same_reg(self, BPF_ARSH, 0); -} - -static int bpf_fill_alu32_lsh_same_reg(struct bpf_test *self) -{ - return __bpf_fill_alu_shift_same_reg(self, BPF_LSH, F_ALU32); -} - -static int bpf_fill_alu32_rsh_same_reg(struct bpf_test *self) -{ - return __bpf_fill_alu_shift_same_reg(self, BPF_RSH, F_ALU32); -} - -static int bpf_fill_alu32_arsh_same_reg(struct bpf_test *self) -{ - return __bpf_fill_alu_shift_same_reg(self, BPF_ARSH, F_ALU32); -} - -/* - * Common operand pattern generator for exhaustive power-of-two magnitudes - * tests. The block size parameters can be adjusted to increase/reduce the - * number of combinatons tested and thereby execution speed and memory - * footprint. - */ - -static inline s64 value(int msb, int delta, int sign) -{ - return sign * (1LL << msb) + delta; -} - -static int __bpf_fill_pattern(struct bpf_test *self, void *arg, - int dbits, int sbits, int block1, int block2, - int (*emit)(struct bpf_test*, void*, - struct bpf_insn*, s64, s64)) -{ - static const int sgn[][2] = {{1, 1}, {1, -1}, {-1, 1}, {-1, -1}}; - struct bpf_insn *insns; - int di, si, bt, db, sb; - int count, len, k; - int extra = 1 + 2; - int i = 0; - - /* Total number of iterations for the two pattern */ - count = (dbits - 1) * (sbits - 1) * block1 * block1 * ARRAY_SIZE(sgn); - count += (max(dbits, sbits) - 1) * block2 * block2 * ARRAY_SIZE(sgn); - - /* Compute the maximum number of insns and allocate the buffer */ - len = extra + count * (*emit)(self, arg, NULL, 0, 0); - insns = kmalloc_array(len, sizeof(*insns), GFP_KERNEL); - if (!insns) - return -ENOMEM; - - /* Add head instruction(s) */ - insns[i++] = BPF_ALU64_IMM(BPF_MOV, R0, 0); - - /* - * Pattern 1: all combinations of power-of-two magnitudes and sign, - * and with a block of contiguous values around each magnitude. - */ - for (di = 0; di < dbits - 1; di++) /* Dst magnitudes */ - for (si = 0; si < sbits - 1; si++) /* Src magnitudes */ - for (k = 0; k < ARRAY_SIZE(sgn); k++) /* Sign combos */ - for (db = -(block1 / 2); - db < (block1 + 1) / 2; db++) - for (sb = -(block1 / 2); - sb < (block1 + 1) / 2; sb++) { - s64 dst, src; - - dst = value(di, db, sgn[k][0]); - src = value(si, sb, sgn[k][1]); - i += (*emit)(self, arg, - &insns[i], - dst, src); - } - /* - * Pattern 2: all combinations for a larger block of values - * for each power-of-two magnitude and sign, where the magnitude is - * the same for both operands. - */ - for (bt = 0; bt < max(dbits, sbits) - 1; bt++) /* Magnitude */ - for (k = 0; k < ARRAY_SIZE(sgn); k++) /* Sign combos */ - for (db = -(block2 / 2); db < (block2 + 1) / 2; db++) - for (sb = -(block2 / 2); - sb < (block2 + 1) / 2; sb++) { - s64 dst, src; - - dst = value(bt % dbits, db, sgn[k][0]); - src = value(bt % sbits, sb, sgn[k][1]); - i += (*emit)(self, arg, &insns[i], - dst, src); - } - - /* Append tail instructions */ - insns[i++] = BPF_ALU64_IMM(BPF_MOV, R0, 1); - insns[i++] = BPF_EXIT_INSN(); - BUG_ON(i > len); - - self->u.ptr.insns = insns; - self->u.ptr.len = i; - - return 0; -} - -/* - * Block size parameters used in pattern tests below. une as needed to - * increase/reduce the number combinations tested, see following examples. - * block values per operand MSB - * ---------------------------------------- - * 0 none - * 1 (1 << MSB) - * 2 (1 << MSB) + [-1, 0] - * 3 (1 << MSB) + [-1, 0, 1] - */ -#define PATTERN_BLOCK1 1 -#define PATTERN_BLOCK2 5 - -/* Number of test runs for a pattern test */ -#define NR_PATTERN_RUNS 1 - -/* - * Exhaustive tests of ALU operations for all combinations of power-of-two - * magnitudes of the operands, both for positive and negative values. The - * test is designed to verify e.g. the ALU and ALU64 operations for JITs that - * emit different code depending on the magnitude of the immediate value. - */ -static int __bpf_emit_alu64_imm(struct bpf_test *self, void *arg, - struct bpf_insn *insns, s64 dst, s64 imm) -{ - int *a = arg; - int op = a[0]; - u32 flags = a[1]; - s16 off = (flags & F_SIGNED) ? 1 : 0; - int i = 0; - u64 res; - - if (!insns) - return 7; - - if (__bpf_alu_result(&res, dst, (s32)imm, op, flags)) { - i += __bpf_ld_imm64(&insns[i], R1, dst); - i += __bpf_ld_imm64(&insns[i], R3, res); - insns[i++] = BPF_ALU64_IMM_OFF(op, R1, imm, off); - insns[i++] = BPF_JMP_REG(BPF_JEQ, R1, R3, 1); - insns[i++] = BPF_EXIT_INSN(); - } - - return i; -} - -static int __bpf_emit_alu32_imm(struct bpf_test *self, void *arg, - struct bpf_insn *insns, s64 dst, s64 imm) -{ - int *a = arg; - int op = a[0]; - u32 flags = a[1]; - s16 off = (flags & F_SIGNED) ? 1 : 0; - int i = 0; - u64 res; - - if (!insns) - return 7; - - if (__bpf_alu_result(&res, dst, (s32)imm, op, flags | F_ALU32)) { - i += __bpf_ld_imm64(&insns[i], R1, dst); - i += __bpf_ld_imm64(&insns[i], R3, (u32)res); - insns[i++] = BPF_ALU32_IMM_OFF(op, R1, imm, off); - insns[i++] = BPF_JMP_REG(BPF_JEQ, R1, R3, 1); - insns[i++] = BPF_EXIT_INSN(); - } - - return i; -} - -static int __bpf_emit_alu64_reg(struct bpf_test *self, void *arg, - struct bpf_insn *insns, s64 dst, s64 src) -{ - int op = *(int *)arg; - int i = 0; - u64 res; - - if (!insns) - return 9; - - if (__bpf_alu_result(&res, dst, src, op, 0)) { - i += __bpf_ld_imm64(&insns[i], R1, dst); - i += __bpf_ld_imm64(&insns[i], R2, src); - i += __bpf_ld_imm64(&insns[i], R3, res); - insns[i++] = BPF_ALU64_REG(op, R1, R2); - insns[i++] = BPF_JMP_REG(BPF_JEQ, R1, R3, 1); - insns[i++] = BPF_EXIT_INSN(); - } - - return i; -} - -static int __bpf_emit_alu32_reg(struct bpf_test *self, void *arg, - struct bpf_insn *insns, s64 dst, s64 src) -{ - int op = *(int *)arg; - int i = 0; - u64 res; - - if (!insns) - return 9; - - if (__bpf_alu_result(&res, (u32)dst, (u32)src, op, 0)) { - i += __bpf_ld_imm64(&insns[i], R1, dst); - i += __bpf_ld_imm64(&insns[i], R2, src); - i += __bpf_ld_imm64(&insns[i], R3, (u32)res); - insns[i++] = BPF_ALU32_REG(op, R1, R2); - insns[i++] = BPF_JMP_REG(BPF_JEQ, R1, R3, 1); - insns[i++] = BPF_EXIT_INSN(); - } - - return i; -} - -static int __bpf_fill_alu64_imm(struct bpf_test *self, int op, u32 flags) -{ - int arg[2] = {op, flags}; - - return __bpf_fill_pattern(self, &arg, 64, 32, - PATTERN_BLOCK1, PATTERN_BLOCK2, - &__bpf_emit_alu64_imm); -} - -static int __bpf_fill_alu32_imm(struct bpf_test *self, int op, u32 flags) -{ - int arg[2] = {op, flags}; - - return __bpf_fill_pattern(self, &arg, 64, 32, - PATTERN_BLOCK1, PATTERN_BLOCK2, - &__bpf_emit_alu32_imm); -} - -static int __bpf_fill_alu64_reg(struct bpf_test *self, int op) -{ - return __bpf_fill_pattern(self, &op, 64, 64, - PATTERN_BLOCK1, PATTERN_BLOCK2, - &__bpf_emit_alu64_reg); -} - -static int __bpf_fill_alu32_reg(struct bpf_test *self, int op) -{ - return __bpf_fill_pattern(self, &op, 64, 64, - PATTERN_BLOCK1, PATTERN_BLOCK2, - &__bpf_emit_alu32_reg); -} - -/* ALU64 immediate operations */ -static int bpf_fill_alu64_mov_imm(struct bpf_test *self) -{ - return __bpf_fill_alu64_imm(self, BPF_MOV, 0); -} - -static int bpf_fill_alu64_and_imm(struct bpf_test *self) -{ - return __bpf_fill_alu64_imm(self, BPF_AND, 0); -} - -static int bpf_fill_alu64_or_imm(struct bpf_test *self) -{ - return __bpf_fill_alu64_imm(self, BPF_OR, 0); -} - -static int bpf_fill_alu64_xor_imm(struct bpf_test *self) -{ - return __bpf_fill_alu64_imm(self, BPF_XOR, 0); -} - -static int bpf_fill_alu64_add_imm(struct bpf_test *self) -{ - return __bpf_fill_alu64_imm(self, BPF_ADD, 0); -} - -static int bpf_fill_alu64_sub_imm(struct bpf_test *self) -{ - return __bpf_fill_alu64_imm(self, BPF_SUB, 0); -} - -static int bpf_fill_alu64_mul_imm(struct bpf_test *self) -{ - return __bpf_fill_alu64_imm(self, BPF_MUL, 0); -} - -static int bpf_fill_alu64_div_imm(struct bpf_test *self) -{ - return __bpf_fill_alu64_imm(self, BPF_DIV, 0); -} - -static int bpf_fill_alu64_mod_imm(struct bpf_test *self) -{ - return __bpf_fill_alu64_imm(self, BPF_MOD, 0); -} - -/* Signed ALU64 immediate operations */ -static int bpf_fill_alu64_sdiv_imm(struct bpf_test *self) -{ - return __bpf_fill_alu64_imm(self, BPF_DIV, F_SIGNED); -} - -static int bpf_fill_alu64_smod_imm(struct bpf_test *self) -{ - return __bpf_fill_alu64_imm(self, BPF_MOD, F_SIGNED); -} - -/* Signed ALU32 immediate operations */ -static int bpf_fill_alu32_sdiv_imm(struct bpf_test *self) -{ - return __bpf_fill_alu32_imm(self, BPF_DIV, F_SIGNED); -} - -static int bpf_fill_alu32_smod_imm(struct bpf_test *self) -{ - return __bpf_fill_alu32_imm(self, BPF_MOD, F_SIGNED); -} - -/* ALU32 immediate operations */ -static int bpf_fill_alu32_mov_imm(struct bpf_test *self) -{ - return __bpf_fill_alu32_imm(self, BPF_MOV, 0); -} - -static int bpf_fill_alu32_and_imm(struct bpf_test *self) -{ - return __bpf_fill_alu32_imm(self, BPF_AND, 0); -} - -static int bpf_fill_alu32_or_imm(struct bpf_test *self) -{ - return __bpf_fill_alu32_imm(self, BPF_OR, 0); -} - -static int bpf_fill_alu32_xor_imm(struct bpf_test *self) -{ - return __bpf_fill_alu32_imm(self, BPF_XOR, 0); -} - -static int bpf_fill_alu32_add_imm(struct bpf_test *self) -{ - return __bpf_fill_alu32_imm(self, BPF_ADD, 0); -} - -static int bpf_fill_alu32_sub_imm(struct bpf_test *self) -{ - return __bpf_fill_alu32_imm(self, BPF_SUB, 0); -} - -static int bpf_fill_alu32_mul_imm(struct bpf_test *self) -{ - return __bpf_fill_alu32_imm(self, BPF_MUL, 0); -} - -static int bpf_fill_alu32_div_imm(struct bpf_test *self) -{ - return __bpf_fill_alu32_imm(self, BPF_DIV, 0); -} - -static int bpf_fill_alu32_mod_imm(struct bpf_test *self) -{ - return __bpf_fill_alu32_imm(self, BPF_MOD, 0); -} - -/* ALU64 register operations */ -static int bpf_fill_alu64_mov_reg(struct bpf_test *self) -{ - return __bpf_fill_alu64_reg(self, BPF_MOV); -} - -static int bpf_fill_alu64_and_reg(struct bpf_test *self) -{ - return __bpf_fill_alu64_reg(self, BPF_AND); -} - -static int bpf_fill_alu64_or_reg(struct bpf_test *self) -{ - return __bpf_fill_alu64_reg(self, BPF_OR); -} - -static int bpf_fill_alu64_xor_reg(struct bpf_test *self) -{ - return __bpf_fill_alu64_reg(self, BPF_XOR); -} - -static int bpf_fill_alu64_add_reg(struct bpf_test *self) -{ - return __bpf_fill_alu64_reg(self, BPF_ADD); -} - -static int bpf_fill_alu64_sub_reg(struct bpf_test *self) -{ - return __bpf_fill_alu64_reg(self, BPF_SUB); -} - -static int bpf_fill_alu64_mul_reg(struct bpf_test *self) -{ - return __bpf_fill_alu64_reg(self, BPF_MUL); -} - -static int bpf_fill_alu64_div_reg(struct bpf_test *self) -{ - return __bpf_fill_alu64_reg(self, BPF_DIV); -} - -static int bpf_fill_alu64_mod_reg(struct bpf_test *self) -{ - return __bpf_fill_alu64_reg(self, BPF_MOD); -} - -/* ALU32 register operations */ -static int bpf_fill_alu32_mov_reg(struct bpf_test *self) -{ - return __bpf_fill_alu32_reg(self, BPF_MOV); -} - -static int bpf_fill_alu32_and_reg(struct bpf_test *self) -{ - return __bpf_fill_alu32_reg(self, BPF_AND); -} - -static int bpf_fill_alu32_or_reg(struct bpf_test *self) -{ - return __bpf_fill_alu32_reg(self, BPF_OR); -} - -static int bpf_fill_alu32_xor_reg(struct bpf_test *self) -{ - return __bpf_fill_alu32_reg(self, BPF_XOR); -} - -static int bpf_fill_alu32_add_reg(struct bpf_test *self) -{ - return __bpf_fill_alu32_reg(self, BPF_ADD); -} - -static int bpf_fill_alu32_sub_reg(struct bpf_test *self) -{ - return __bpf_fill_alu32_reg(self, BPF_SUB); -} - -static int bpf_fill_alu32_mul_reg(struct bpf_test *self) -{ - return __bpf_fill_alu32_reg(self, BPF_MUL); -} - -static int bpf_fill_alu32_div_reg(struct bpf_test *self) -{ - return __bpf_fill_alu32_reg(self, BPF_DIV); -} - -static int bpf_fill_alu32_mod_reg(struct bpf_test *self) -{ - return __bpf_fill_alu32_reg(self, BPF_MOD); -} - -/* - * Test JITs that implement complex ALU operations as function - * calls, and must re-arrange operands for argument passing. - */ -static int __bpf_fill_alu_imm_regs(struct bpf_test *self, u8 op, - u32 flags) -{ - int len = 2 + 10 * 10; - struct bpf_insn *insns; - u64 dst, res; - int i = 0; - u32 imm; - int rd; - - insns = kmalloc_objs(*insns, len); - if (!insns) - return -ENOMEM; - - /* Operand and result values according to operation */ - if (flags & F_SIGNED) { - if (flags & F_ALU32) - dst = -76543210; - else - dst = -7654321076543210LL; - } else { - if (flags & F_ALU32) - dst = 0x76543210U; - else - dst = 0x7edcba9876543210ULL; - } - imm = 0x01234567U; - - if (op == BPF_LSH || op == BPF_RSH || op == BPF_ARSH) - imm &= 31; - - __bpf_alu_result(&res, dst, imm, op, flags); - - if (flags & F_ALU32) - res = (u32)res; - - /* Check all operand registers */ - for (rd = R0; rd <= R9; rd++) { - i += __bpf_ld_imm64(&insns[i], rd, dst); - - s16 off = (flags & F_SIGNED) ? 1 : 0; - - if (flags & F_ALU32) - insns[i++] = BPF_ALU32_IMM_OFF(op, rd, imm, off); - else - insns[i++] = BPF_ALU64_IMM_OFF(op, rd, imm, off); - - insns[i++] = BPF_JMP32_IMM(BPF_JEQ, rd, res, 2); - insns[i++] = BPF_MOV64_IMM(R0, __LINE__); - insns[i++] = BPF_EXIT_INSN(); - - insns[i++] = BPF_ALU64_IMM(BPF_RSH, rd, 32); - insns[i++] = BPF_JMP32_IMM(BPF_JEQ, rd, res >> 32, 2); - insns[i++] = BPF_MOV64_IMM(R0, __LINE__); - insns[i++] = BPF_EXIT_INSN(); - } - - insns[i++] = BPF_MOV64_IMM(R0, 1); - insns[i++] = BPF_EXIT_INSN(); - - self->u.ptr.insns = insns; - self->u.ptr.len = len; - BUG_ON(i != len); - - return 0; -} - -/* ALU64 K registers */ -static int bpf_fill_alu64_mov_imm_regs(struct bpf_test *self) -{ - return __bpf_fill_alu_imm_regs(self, BPF_MOV, 0); -} - -static int bpf_fill_alu64_and_imm_regs(struct bpf_test *self) -{ - return __bpf_fill_alu_imm_regs(self, BPF_AND, 0); -} - -static int bpf_fill_alu64_or_imm_regs(struct bpf_test *self) -{ - return __bpf_fill_alu_imm_regs(self, BPF_OR, 0); -} - -static int bpf_fill_alu64_xor_imm_regs(struct bpf_test *self) -{ - return __bpf_fill_alu_imm_regs(self, BPF_XOR, 0); -} - -static int bpf_fill_alu64_lsh_imm_regs(struct bpf_test *self) -{ - return __bpf_fill_alu_imm_regs(self, BPF_LSH, 0); -} - -static int bpf_fill_alu64_rsh_imm_regs(struct bpf_test *self) -{ - return __bpf_fill_alu_imm_regs(self, BPF_RSH, 0); -} - -static int bpf_fill_alu64_arsh_imm_regs(struct bpf_test *self) -{ - return __bpf_fill_alu_imm_regs(self, BPF_ARSH, 0); -} - -static int bpf_fill_alu64_add_imm_regs(struct bpf_test *self) -{ - return __bpf_fill_alu_imm_regs(self, BPF_ADD, 0); -} - -static int bpf_fill_alu64_sub_imm_regs(struct bpf_test *self) -{ - return __bpf_fill_alu_imm_regs(self, BPF_SUB, 0); -} - -static int bpf_fill_alu64_mul_imm_regs(struct bpf_test *self) -{ - return __bpf_fill_alu_imm_regs(self, BPF_MUL, 0); -} - -static int bpf_fill_alu64_div_imm_regs(struct bpf_test *self) -{ - return __bpf_fill_alu_imm_regs(self, BPF_DIV, 0); -} - -static int bpf_fill_alu64_mod_imm_regs(struct bpf_test *self) -{ - return __bpf_fill_alu_imm_regs(self, BPF_MOD, 0); -} - -/* Signed ALU64 K registers */ -static int bpf_fill_alu64_sdiv_imm_regs(struct bpf_test *self) -{ - return __bpf_fill_alu_imm_regs(self, BPF_DIV, F_SIGNED); -} - -static int bpf_fill_alu64_smod_imm_regs(struct bpf_test *self) -{ - return __bpf_fill_alu_imm_regs(self, BPF_MOD, F_SIGNED); -} - -/* ALU32 K registers */ -static int bpf_fill_alu32_mov_imm_regs(struct bpf_test *self) -{ - return __bpf_fill_alu_imm_regs(self, BPF_MOV, F_ALU32); -} - -static int bpf_fill_alu32_and_imm_regs(struct bpf_test *self) -{ - return __bpf_fill_alu_imm_regs(self, BPF_AND, F_ALU32); -} - -static int bpf_fill_alu32_or_imm_regs(struct bpf_test *self) -{ - return __bpf_fill_alu_imm_regs(self, BPF_OR, F_ALU32); -} - -static int bpf_fill_alu32_xor_imm_regs(struct bpf_test *self) -{ - return __bpf_fill_alu_imm_regs(self, BPF_XOR, F_ALU32); -} - -static int bpf_fill_alu32_lsh_imm_regs(struct bpf_test *self) -{ - return __bpf_fill_alu_imm_regs(self, BPF_LSH, F_ALU32); -} - -static int bpf_fill_alu32_rsh_imm_regs(struct bpf_test *self) -{ - return __bpf_fill_alu_imm_regs(self, BPF_RSH, F_ALU32); -} - -static int bpf_fill_alu32_arsh_imm_regs(struct bpf_test *self) -{ - return __bpf_fill_alu_imm_regs(self, BPF_ARSH, F_ALU32); -} - -static int bpf_fill_alu32_add_imm_regs(struct bpf_test *self) -{ - return __bpf_fill_alu_imm_regs(self, BPF_ADD, F_ALU32); -} - -static int bpf_fill_alu32_sub_imm_regs(struct bpf_test *self) -{ - return __bpf_fill_alu_imm_regs(self, BPF_SUB, F_ALU32); -} - -static int bpf_fill_alu32_mul_imm_regs(struct bpf_test *self) -{ - return __bpf_fill_alu_imm_regs(self, BPF_MUL, F_ALU32); -} - -static int bpf_fill_alu32_div_imm_regs(struct bpf_test *self) -{ - return __bpf_fill_alu_imm_regs(self, BPF_DIV, F_ALU32); -} - -static int bpf_fill_alu32_mod_imm_regs(struct bpf_test *self) -{ - return __bpf_fill_alu_imm_regs(self, BPF_MOD, F_ALU32); -} - -/* Signed ALU32 K registers */ -static int bpf_fill_alu32_sdiv_imm_regs(struct bpf_test *self) -{ - return __bpf_fill_alu_imm_regs(self, BPF_DIV, F_ALU32 | F_SIGNED); -} - -static int bpf_fill_alu32_smod_imm_regs(struct bpf_test *self) -{ - return __bpf_fill_alu_imm_regs(self, BPF_MOD, F_ALU32 | F_SIGNED); -} - -/* - * Test JITs that implement complex ALU operations as function - * calls, and must re-arrange operands for argument passing. - */ -static int __bpf_fill_alu_reg_pairs(struct bpf_test *self, u8 op, bool alu32) -{ - int len = 2 + 10 * 10 * 12; - u64 dst, src, res, same; - struct bpf_insn *insns; - int rd, rs; - int i = 0; - - insns = kmalloc_objs(*insns, len); - if (!insns) - return -ENOMEM; - - /* Operand and result values according to operation */ - if (alu32) { - dst = 0x76543210U; - src = 0x01234567U; - } else { - dst = 0x7edcba9876543210ULL; - src = 0x0123456789abcdefULL; - } - - if (op == BPF_LSH || op == BPF_RSH || op == BPF_ARSH) - src &= 31; - - __bpf_alu_result(&res, dst, src, op, 0); - __bpf_alu_result(&same, src, src, op, 0); - - if (alu32) { - res = (u32)res; - same = (u32)same; - } - - /* Check all combinations of operand registers */ - for (rd = R0; rd <= R9; rd++) { - for (rs = R0; rs <= R9; rs++) { - u64 val = rd == rs ? same : res; - - i += __bpf_ld_imm64(&insns[i], rd, dst); - i += __bpf_ld_imm64(&insns[i], rs, src); - - if (alu32) - insns[i++] = BPF_ALU32_REG(op, rd, rs); - else - insns[i++] = BPF_ALU64_REG(op, rd, rs); - - insns[i++] = BPF_JMP32_IMM(BPF_JEQ, rd, val, 2); - insns[i++] = BPF_MOV64_IMM(R0, __LINE__); - insns[i++] = BPF_EXIT_INSN(); - - insns[i++] = BPF_ALU64_IMM(BPF_RSH, rd, 32); - insns[i++] = BPF_JMP32_IMM(BPF_JEQ, rd, val >> 32, 2); - insns[i++] = BPF_MOV64_IMM(R0, __LINE__); - insns[i++] = BPF_EXIT_INSN(); - } - } - - insns[i++] = BPF_MOV64_IMM(R0, 1); - insns[i++] = BPF_EXIT_INSN(); - - self->u.ptr.insns = insns; - self->u.ptr.len = len; - BUG_ON(i != len); - - return 0; -} - -/* ALU64 X register combinations */ -static int bpf_fill_alu64_mov_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_alu_reg_pairs(self, BPF_MOV, false); -} - -static int bpf_fill_alu64_and_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_alu_reg_pairs(self, BPF_AND, false); -} - -static int bpf_fill_alu64_or_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_alu_reg_pairs(self, BPF_OR, false); -} - -static int bpf_fill_alu64_xor_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_alu_reg_pairs(self, BPF_XOR, false); -} - -static int bpf_fill_alu64_lsh_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_alu_reg_pairs(self, BPF_LSH, false); -} - -static int bpf_fill_alu64_rsh_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_alu_reg_pairs(self, BPF_RSH, false); -} - -static int bpf_fill_alu64_arsh_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_alu_reg_pairs(self, BPF_ARSH, false); -} - -static int bpf_fill_alu64_add_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_alu_reg_pairs(self, BPF_ADD, false); -} - -static int bpf_fill_alu64_sub_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_alu_reg_pairs(self, BPF_SUB, false); -} - -static int bpf_fill_alu64_mul_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_alu_reg_pairs(self, BPF_MUL, false); -} - -static int bpf_fill_alu64_div_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_alu_reg_pairs(self, BPF_DIV, false); -} - -static int bpf_fill_alu64_mod_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_alu_reg_pairs(self, BPF_MOD, false); -} - -/* ALU32 X register combinations */ -static int bpf_fill_alu32_mov_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_alu_reg_pairs(self, BPF_MOV, true); -} - -static int bpf_fill_alu32_and_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_alu_reg_pairs(self, BPF_AND, true); -} - -static int bpf_fill_alu32_or_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_alu_reg_pairs(self, BPF_OR, true); -} - -static int bpf_fill_alu32_xor_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_alu_reg_pairs(self, BPF_XOR, true); -} - -static int bpf_fill_alu32_lsh_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_alu_reg_pairs(self, BPF_LSH, true); -} - -static int bpf_fill_alu32_rsh_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_alu_reg_pairs(self, BPF_RSH, true); -} - -static int bpf_fill_alu32_arsh_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_alu_reg_pairs(self, BPF_ARSH, true); -} - -static int bpf_fill_alu32_add_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_alu_reg_pairs(self, BPF_ADD, true); -} - -static int bpf_fill_alu32_sub_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_alu_reg_pairs(self, BPF_SUB, true); -} - -static int bpf_fill_alu32_mul_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_alu_reg_pairs(self, BPF_MUL, true); -} - -static int bpf_fill_alu32_div_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_alu_reg_pairs(self, BPF_DIV, true); -} - -static int bpf_fill_alu32_mod_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_alu_reg_pairs(self, BPF_MOD, true); -} - -/* - * Exhaustive tests of atomic operations for all power-of-two operand - * magnitudes, both for positive and negative values. - */ - -static int __bpf_emit_atomic64(struct bpf_test *self, void *arg, - struct bpf_insn *insns, s64 dst, s64 src) -{ - int op = *(int *)arg; - u64 keep, fetch, res; - int i = 0; - - if (!insns) - return 21; - - switch (op) { - case BPF_XCHG: - res = src; - break; - default: - __bpf_alu_result(&res, dst, src, BPF_OP(op), 0); - } - - keep = 0x0123456789abcdefULL; - if (op & BPF_FETCH) - fetch = dst; - else - fetch = src; - - i += __bpf_ld_imm64(&insns[i], R0, keep); - i += __bpf_ld_imm64(&insns[i], R1, dst); - i += __bpf_ld_imm64(&insns[i], R2, src); - i += __bpf_ld_imm64(&insns[i], R3, res); - i += __bpf_ld_imm64(&insns[i], R4, fetch); - i += __bpf_ld_imm64(&insns[i], R5, keep); - - insns[i++] = BPF_STX_MEM(BPF_DW, R10, R1, -8); - insns[i++] = BPF_ATOMIC_OP(BPF_DW, op, R10, R2, -8); - insns[i++] = BPF_LDX_MEM(BPF_DW, R1, R10, -8); - - insns[i++] = BPF_JMP_REG(BPF_JEQ, R1, R3, 1); - insns[i++] = BPF_EXIT_INSN(); - - insns[i++] = BPF_JMP_REG(BPF_JEQ, R2, R4, 1); - insns[i++] = BPF_EXIT_INSN(); - - insns[i++] = BPF_JMP_REG(BPF_JEQ, R0, R5, 1); - insns[i++] = BPF_EXIT_INSN(); - - return i; -} - -static int __bpf_emit_atomic32(struct bpf_test *self, void *arg, - struct bpf_insn *insns, s64 dst, s64 src) -{ - int op = *(int *)arg; - u64 keep, fetch, res; - int i = 0; - - if (!insns) - return 21; - - switch (op) { - case BPF_XCHG: - res = src; - break; - default: - __bpf_alu_result(&res, (u32)dst, (u32)src, BPF_OP(op), 0); - } - - keep = 0x0123456789abcdefULL; - if (op & BPF_FETCH) - fetch = (u32)dst; - else - fetch = src; - - i += __bpf_ld_imm64(&insns[i], R0, keep); - i += __bpf_ld_imm64(&insns[i], R1, (u32)dst); - i += __bpf_ld_imm64(&insns[i], R2, src); - i += __bpf_ld_imm64(&insns[i], R3, (u32)res); - i += __bpf_ld_imm64(&insns[i], R4, fetch); - i += __bpf_ld_imm64(&insns[i], R5, keep); - - insns[i++] = BPF_STX_MEM(BPF_W, R10, R1, -4); - insns[i++] = BPF_ATOMIC_OP(BPF_W, op, R10, R2, -4); - insns[i++] = BPF_LDX_MEM(BPF_W, R1, R10, -4); - - insns[i++] = BPF_JMP_REG(BPF_JEQ, R1, R3, 1); - insns[i++] = BPF_EXIT_INSN(); - - insns[i++] = BPF_JMP_REG(BPF_JEQ, R2, R4, 1); - insns[i++] = BPF_EXIT_INSN(); - - insns[i++] = BPF_JMP_REG(BPF_JEQ, R0, R5, 1); - insns[i++] = BPF_EXIT_INSN(); - - return i; -} - -static int __bpf_emit_cmpxchg64(struct bpf_test *self, void *arg, - struct bpf_insn *insns, s64 dst, s64 src) -{ - int i = 0; - - if (!insns) - return 23; - - i += __bpf_ld_imm64(&insns[i], R0, ~dst); - i += __bpf_ld_imm64(&insns[i], R1, dst); - i += __bpf_ld_imm64(&insns[i], R2, src); - - /* Result unsuccessful */ - insns[i++] = BPF_STX_MEM(BPF_DW, R10, R1, -8); - insns[i++] = BPF_ATOMIC_OP(BPF_DW, BPF_CMPXCHG, R10, R2, -8); - insns[i++] = BPF_LDX_MEM(BPF_DW, R3, R10, -8); - - insns[i++] = BPF_JMP_REG(BPF_JEQ, R1, R3, 2); - insns[i++] = BPF_MOV64_IMM(R0, __LINE__); - insns[i++] = BPF_EXIT_INSN(); - - insns[i++] = BPF_JMP_REG(BPF_JEQ, R0, R3, 2); - insns[i++] = BPF_MOV64_IMM(R0, __LINE__); - insns[i++] = BPF_EXIT_INSN(); - - /* Result successful */ - insns[i++] = BPF_ATOMIC_OP(BPF_DW, BPF_CMPXCHG, R10, R2, -8); - insns[i++] = BPF_LDX_MEM(BPF_DW, R3, R10, -8); - - insns[i++] = BPF_JMP_REG(BPF_JEQ, R2, R3, 2); - insns[i++] = BPF_MOV64_IMM(R0, __LINE__); - insns[i++] = BPF_EXIT_INSN(); - - insns[i++] = BPF_JMP_REG(BPF_JEQ, R0, R1, 2); - insns[i++] = BPF_MOV64_IMM(R0, __LINE__); - insns[i++] = BPF_EXIT_INSN(); - - return i; -} - -static int __bpf_emit_cmpxchg32(struct bpf_test *self, void *arg, - struct bpf_insn *insns, s64 dst, s64 src) -{ - int i = 0; - - if (!insns) - return 27; - - i += __bpf_ld_imm64(&insns[i], R0, ~dst); - i += __bpf_ld_imm64(&insns[i], R1, (u32)dst); - i += __bpf_ld_imm64(&insns[i], R2, src); - - /* Result unsuccessful */ - insns[i++] = BPF_STX_MEM(BPF_W, R10, R1, -4); - insns[i++] = BPF_ATOMIC_OP(BPF_W, BPF_CMPXCHG, R10, R2, -4); - insns[i++] = BPF_ZEXT_REG(R0); /* Zext always inserted by verifier */ - insns[i++] = BPF_LDX_MEM(BPF_W, R3, R10, -4); - - insns[i++] = BPF_JMP32_REG(BPF_JEQ, R1, R3, 2); - insns[i++] = BPF_MOV32_IMM(R0, __LINE__); - insns[i++] = BPF_EXIT_INSN(); - - insns[i++] = BPF_JMP_REG(BPF_JEQ, R0, R3, 2); - insns[i++] = BPF_MOV32_IMM(R0, __LINE__); - insns[i++] = BPF_EXIT_INSN(); - - /* Result successful */ - i += __bpf_ld_imm64(&insns[i], R0, dst); - insns[i++] = BPF_ATOMIC_OP(BPF_W, BPF_CMPXCHG, R10, R2, -4); - insns[i++] = BPF_ZEXT_REG(R0); /* Zext always inserted by verifier */ - insns[i++] = BPF_LDX_MEM(BPF_W, R3, R10, -4); - - insns[i++] = BPF_JMP32_REG(BPF_JEQ, R2, R3, 2); - insns[i++] = BPF_MOV32_IMM(R0, __LINE__); - insns[i++] = BPF_EXIT_INSN(); - - insns[i++] = BPF_JMP_REG(BPF_JEQ, R0, R1, 2); - insns[i++] = BPF_MOV32_IMM(R0, __LINE__); - insns[i++] = BPF_EXIT_INSN(); - - return i; -} - -static int __bpf_fill_atomic64(struct bpf_test *self, int op) -{ - return __bpf_fill_pattern(self, &op, 64, 64, - 0, PATTERN_BLOCK2, - &__bpf_emit_atomic64); -} - -static int __bpf_fill_atomic32(struct bpf_test *self, int op) -{ - return __bpf_fill_pattern(self, &op, 64, 64, - 0, PATTERN_BLOCK2, - &__bpf_emit_atomic32); -} - -/* 64-bit atomic operations */ -static int bpf_fill_atomic64_add(struct bpf_test *self) -{ - return __bpf_fill_atomic64(self, BPF_ADD); -} - -static int bpf_fill_atomic64_and(struct bpf_test *self) -{ - return __bpf_fill_atomic64(self, BPF_AND); -} - -static int bpf_fill_atomic64_or(struct bpf_test *self) -{ - return __bpf_fill_atomic64(self, BPF_OR); -} - -static int bpf_fill_atomic64_xor(struct bpf_test *self) -{ - return __bpf_fill_atomic64(self, BPF_XOR); -} - -static int bpf_fill_atomic64_add_fetch(struct bpf_test *self) -{ - return __bpf_fill_atomic64(self, BPF_ADD | BPF_FETCH); -} - -static int bpf_fill_atomic64_and_fetch(struct bpf_test *self) -{ - return __bpf_fill_atomic64(self, BPF_AND | BPF_FETCH); -} - -static int bpf_fill_atomic64_or_fetch(struct bpf_test *self) -{ - return __bpf_fill_atomic64(self, BPF_OR | BPF_FETCH); -} - -static int bpf_fill_atomic64_xor_fetch(struct bpf_test *self) -{ - return __bpf_fill_atomic64(self, BPF_XOR | BPF_FETCH); -} - -static int bpf_fill_atomic64_xchg(struct bpf_test *self) -{ - return __bpf_fill_atomic64(self, BPF_XCHG); -} - -static int bpf_fill_cmpxchg64(struct bpf_test *self) -{ - return __bpf_fill_pattern(self, NULL, 64, 64, 0, PATTERN_BLOCK2, - &__bpf_emit_cmpxchg64); -} - -/* 32-bit atomic operations */ -static int bpf_fill_atomic32_add(struct bpf_test *self) -{ - return __bpf_fill_atomic32(self, BPF_ADD); -} - -static int bpf_fill_atomic32_and(struct bpf_test *self) -{ - return __bpf_fill_atomic32(self, BPF_AND); -} - -static int bpf_fill_atomic32_or(struct bpf_test *self) -{ - return __bpf_fill_atomic32(self, BPF_OR); -} - -static int bpf_fill_atomic32_xor(struct bpf_test *self) -{ - return __bpf_fill_atomic32(self, BPF_XOR); -} - -static int bpf_fill_atomic32_add_fetch(struct bpf_test *self) -{ - return __bpf_fill_atomic32(self, BPF_ADD | BPF_FETCH); -} - -static int bpf_fill_atomic32_and_fetch(struct bpf_test *self) -{ - return __bpf_fill_atomic32(self, BPF_AND | BPF_FETCH); -} - -static int bpf_fill_atomic32_or_fetch(struct bpf_test *self) -{ - return __bpf_fill_atomic32(self, BPF_OR | BPF_FETCH); -} - -static int bpf_fill_atomic32_xor_fetch(struct bpf_test *self) -{ - return __bpf_fill_atomic32(self, BPF_XOR | BPF_FETCH); -} - -static int bpf_fill_atomic32_xchg(struct bpf_test *self) -{ - return __bpf_fill_atomic32(self, BPF_XCHG); -} - -static int bpf_fill_cmpxchg32(struct bpf_test *self) -{ - return __bpf_fill_pattern(self, NULL, 64, 64, 0, PATTERN_BLOCK2, - &__bpf_emit_cmpxchg32); -} - -/* - * Test JITs that implement ATOMIC operations as function calls or - * other primitives, and must re-arrange operands for argument passing. - */ -static int __bpf_fill_atomic_reg_pairs(struct bpf_test *self, u8 width, u8 op) -{ - struct bpf_insn *insn; - int len = 2 + 34 * 10 * 10; - u64 mem, upd, res; - int rd, rs, i = 0; - - insn = kmalloc_objs(*insn, len); - if (!insn) - return -ENOMEM; - - /* Operand and memory values */ - if (width == BPF_DW) { - mem = 0x0123456789abcdefULL; - upd = 0xfedcba9876543210ULL; - } else { /* BPF_W */ - mem = 0x01234567U; - upd = 0x76543210U; - } - - /* Memory updated according to operation */ - switch (op) { - case BPF_XCHG: - res = upd; - break; - case BPF_CMPXCHG: - res = mem; - break; - default: - __bpf_alu_result(&res, mem, upd, BPF_OP(op), 0); - } - - /* Test all operand registers */ - for (rd = R0; rd <= R9; rd++) { - for (rs = R0; rs <= R9; rs++) { - u64 cmp, src; - - /* Initialize value in memory */ - i += __bpf_ld_imm64(&insn[i], R0, mem); - insn[i++] = BPF_STX_MEM(width, R10, R0, -8); - - /* Initialize registers in order */ - i += __bpf_ld_imm64(&insn[i], R0, ~mem); - i += __bpf_ld_imm64(&insn[i], rs, upd); - insn[i++] = BPF_MOV64_REG(rd, R10); - - /* Perform atomic operation */ - insn[i++] = BPF_ATOMIC_OP(width, op, rd, rs, -8); - if (op == BPF_CMPXCHG && width == BPF_W) - insn[i++] = BPF_ZEXT_REG(R0); - - /* Check R0 register value */ - if (op == BPF_CMPXCHG) - cmp = mem; /* Expect value from memory */ - else if (R0 == rd || R0 == rs) - cmp = 0; /* Aliased, checked below */ - else - cmp = ~mem; /* Expect value to be preserved */ - if (cmp) { - insn[i++] = BPF_JMP32_IMM(BPF_JEQ, R0, - (u32)cmp, 2); - insn[i++] = BPF_MOV32_IMM(R0, __LINE__); - insn[i++] = BPF_EXIT_INSN(); - insn[i++] = BPF_ALU64_IMM(BPF_RSH, R0, 32); - insn[i++] = BPF_JMP32_IMM(BPF_JEQ, R0, - cmp >> 32, 2); - insn[i++] = BPF_MOV32_IMM(R0, __LINE__); - insn[i++] = BPF_EXIT_INSN(); - } - - /* Check source register value */ - if (rs == R0 && op == BPF_CMPXCHG) - src = 0; /* Aliased with R0, checked above */ - else if (rs == rd && (op == BPF_CMPXCHG || - !(op & BPF_FETCH))) - src = 0; /* Aliased with rd, checked below */ - else if (op == BPF_CMPXCHG) - src = upd; /* Expect value to be preserved */ - else if (op & BPF_FETCH) - src = mem; /* Expect fetched value from mem */ - else /* no fetch */ - src = upd; /* Expect value to be preserved */ - if (src) { - insn[i++] = BPF_JMP32_IMM(BPF_JEQ, rs, - (u32)src, 2); - insn[i++] = BPF_MOV32_IMM(R0, __LINE__); - insn[i++] = BPF_EXIT_INSN(); - insn[i++] = BPF_ALU64_IMM(BPF_RSH, rs, 32); - insn[i++] = BPF_JMP32_IMM(BPF_JEQ, rs, - src >> 32, 2); - insn[i++] = BPF_MOV32_IMM(R0, __LINE__); - insn[i++] = BPF_EXIT_INSN(); - } - - /* Check destination register value */ - if (!(rd == R0 && op == BPF_CMPXCHG) && - !(rd == rs && (op & BPF_FETCH))) { - insn[i++] = BPF_JMP_REG(BPF_JEQ, rd, R10, 2); - insn[i++] = BPF_MOV32_IMM(R0, __LINE__); - insn[i++] = BPF_EXIT_INSN(); - } - - /* Check value in memory */ - if (rs != rd) { /* No aliasing */ - i += __bpf_ld_imm64(&insn[i], R1, res); - } else if (op == BPF_XCHG) { /* Aliased, XCHG */ - insn[i++] = BPF_MOV64_REG(R1, R10); - } else if (op == BPF_CMPXCHG) { /* Aliased, CMPXCHG */ - i += __bpf_ld_imm64(&insn[i], R1, mem); - } else { /* Aliased, ALU oper */ - i += __bpf_ld_imm64(&insn[i], R1, mem); - insn[i++] = BPF_ALU64_REG(BPF_OP(op), R1, R10); - } - - insn[i++] = BPF_LDX_MEM(width, R0, R10, -8); - if (width == BPF_DW) - insn[i++] = BPF_JMP_REG(BPF_JEQ, R0, R1, 2); - else /* width == BPF_W */ - insn[i++] = BPF_JMP32_REG(BPF_JEQ, R0, R1, 2); - insn[i++] = BPF_MOV32_IMM(R0, __LINE__); - insn[i++] = BPF_EXIT_INSN(); - } - } - - insn[i++] = BPF_MOV64_IMM(R0, 1); - insn[i++] = BPF_EXIT_INSN(); - - self->u.ptr.insns = insn; - self->u.ptr.len = i; - BUG_ON(i > len); - - return 0; -} - -/* 64-bit atomic register tests */ -static int bpf_fill_atomic64_add_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_atomic_reg_pairs(self, BPF_DW, BPF_ADD); -} - -static int bpf_fill_atomic64_and_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_atomic_reg_pairs(self, BPF_DW, BPF_AND); -} - -static int bpf_fill_atomic64_or_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_atomic_reg_pairs(self, BPF_DW, BPF_OR); -} - -static int bpf_fill_atomic64_xor_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_atomic_reg_pairs(self, BPF_DW, BPF_XOR); -} - -static int bpf_fill_atomic64_add_fetch_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_atomic_reg_pairs(self, BPF_DW, BPF_ADD | BPF_FETCH); -} - -static int bpf_fill_atomic64_and_fetch_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_atomic_reg_pairs(self, BPF_DW, BPF_AND | BPF_FETCH); -} - -static int bpf_fill_atomic64_or_fetch_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_atomic_reg_pairs(self, BPF_DW, BPF_OR | BPF_FETCH); -} - -static int bpf_fill_atomic64_xor_fetch_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_atomic_reg_pairs(self, BPF_DW, BPF_XOR | BPF_FETCH); -} - -static int bpf_fill_atomic64_xchg_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_atomic_reg_pairs(self, BPF_DW, BPF_XCHG); -} - -static int bpf_fill_atomic64_cmpxchg_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_atomic_reg_pairs(self, BPF_DW, BPF_CMPXCHG); -} - -/* 32-bit atomic register tests */ -static int bpf_fill_atomic32_add_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_atomic_reg_pairs(self, BPF_W, BPF_ADD); -} - -static int bpf_fill_atomic32_and_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_atomic_reg_pairs(self, BPF_W, BPF_AND); -} - -static int bpf_fill_atomic32_or_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_atomic_reg_pairs(self, BPF_W, BPF_OR); -} - -static int bpf_fill_atomic32_xor_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_atomic_reg_pairs(self, BPF_W, BPF_XOR); -} - -static int bpf_fill_atomic32_add_fetch_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_atomic_reg_pairs(self, BPF_W, BPF_ADD | BPF_FETCH); -} - -static int bpf_fill_atomic32_and_fetch_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_atomic_reg_pairs(self, BPF_W, BPF_AND | BPF_FETCH); -} - -static int bpf_fill_atomic32_or_fetch_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_atomic_reg_pairs(self, BPF_W, BPF_OR | BPF_FETCH); -} - -static int bpf_fill_atomic32_xor_fetch_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_atomic_reg_pairs(self, BPF_W, BPF_XOR | BPF_FETCH); -} - -static int bpf_fill_atomic32_xchg_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_atomic_reg_pairs(self, BPF_W, BPF_XCHG); -} - -static int bpf_fill_atomic32_cmpxchg_reg_pairs(struct bpf_test *self) -{ - return __bpf_fill_atomic_reg_pairs(self, BPF_W, BPF_CMPXCHG); -} - -/* - * Test the two-instruction 64-bit immediate load operation for all - * power-of-two magnitudes of the immediate operand. For each MSB, a block - * of immediate values centered around the power-of-two MSB are tested, - * both for positive and negative values. The test is designed to verify - * the operation for JITs that emit different code depending on the magnitude - * of the immediate value. This is often the case if the native instruction - * immediate field width is narrower than 32 bits. - */ -static int bpf_fill_ld_imm64_magn(struct bpf_test *self) -{ - int block = 64; /* Increase for more tests per MSB position */ - int len = 3 + 8 * 63 * block * 2; - struct bpf_insn *insn; - int bit, adj, sign; - int i = 0; - - insn = kmalloc_objs(*insn, len); - if (!insn) - return -ENOMEM; - - insn[i++] = BPF_ALU64_IMM(BPF_MOV, R0, 0); - - for (bit = 0; bit <= 62; bit++) { - for (adj = -block / 2; adj < block / 2; adj++) { - for (sign = -1; sign <= 1; sign += 2) { - s64 imm = sign * ((1LL << bit) + adj); - - /* Perform operation */ - i += __bpf_ld_imm64(&insn[i], R1, imm); - - /* Load reference */ - insn[i++] = BPF_ALU32_IMM(BPF_MOV, R2, imm); - insn[i++] = BPF_ALU32_IMM(BPF_MOV, R3, - (u32)(imm >> 32)); - insn[i++] = BPF_ALU64_IMM(BPF_LSH, R3, 32); - insn[i++] = BPF_ALU64_REG(BPF_OR, R2, R3); - - /* Check result */ - insn[i++] = BPF_JMP_REG(BPF_JEQ, R1, R2, 1); - insn[i++] = BPF_EXIT_INSN(); - } - } - } - - insn[i++] = BPF_ALU64_IMM(BPF_MOV, R0, 1); - insn[i++] = BPF_EXIT_INSN(); - - self->u.ptr.insns = insn; - self->u.ptr.len = len; - BUG_ON(i != len); - - return 0; -} - -/* - * Test the two-instruction 64-bit immediate load operation for different - * combinations of bytes. Each byte in the 64-bit word is constructed as - * (base & mask) | (rand() & ~mask), where rand() is a deterministic LCG. - * All patterns (base1, mask1) and (base2, mask2) bytes are tested. - */ -static int __bpf_fill_ld_imm64_bytes(struct bpf_test *self, - u8 base1, u8 mask1, - u8 base2, u8 mask2) -{ - struct bpf_insn *insn; - int len = 3 + 8 * BIT(8); - int pattern, index; - u32 rand = 1; - int i = 0; - - insn = kmalloc_objs(*insn, len); - if (!insn) - return -ENOMEM; - - insn[i++] = BPF_ALU64_IMM(BPF_MOV, R0, 0); - - for (pattern = 0; pattern < BIT(8); pattern++) { - u64 imm = 0; - - for (index = 0; index < 8; index++) { - int byte; - - if (pattern & BIT(index)) - byte = (base1 & mask1) | (rand & ~mask1); - else - byte = (base2 & mask2) | (rand & ~mask2); - imm = (imm << 8) | byte; - } - - /* Update our LCG */ - rand = rand * 1664525 + 1013904223; - - /* Perform operation */ - i += __bpf_ld_imm64(&insn[i], R1, imm); - - /* Load reference */ - insn[i++] = BPF_ALU32_IMM(BPF_MOV, R2, imm); - insn[i++] = BPF_ALU32_IMM(BPF_MOV, R3, (u32)(imm >> 32)); - insn[i++] = BPF_ALU64_IMM(BPF_LSH, R3, 32); - insn[i++] = BPF_ALU64_REG(BPF_OR, R2, R3); - - /* Check result */ - insn[i++] = BPF_JMP_REG(BPF_JEQ, R1, R2, 1); - insn[i++] = BPF_EXIT_INSN(); - } - - insn[i++] = BPF_ALU64_IMM(BPF_MOV, R0, 1); - insn[i++] = BPF_EXIT_INSN(); - - self->u.ptr.insns = insn; - self->u.ptr.len = len; - BUG_ON(i != len); - - return 0; -} - -static int bpf_fill_ld_imm64_checker(struct bpf_test *self) -{ - return __bpf_fill_ld_imm64_bytes(self, 0, 0xff, 0xff, 0xff); -} - -static int bpf_fill_ld_imm64_pos_neg(struct bpf_test *self) -{ - return __bpf_fill_ld_imm64_bytes(self, 1, 0x81, 0x80, 0x80); -} - -static int bpf_fill_ld_imm64_pos_zero(struct bpf_test *self) -{ - return __bpf_fill_ld_imm64_bytes(self, 1, 0x81, 0, 0xff); -} - -static int bpf_fill_ld_imm64_neg_zero(struct bpf_test *self) -{ - return __bpf_fill_ld_imm64_bytes(self, 0x80, 0x80, 0, 0xff); -} - -/* - * Exhaustive tests of JMP operations for all combinations of power-of-two - * magnitudes of the operands, both for positive and negative values. The - * test is designed to verify e.g. the JMP and JMP32 operations for JITs that - * emit different code depending on the magnitude of the immediate value. - */ - -static bool __bpf_match_jmp_cond(s64 v1, s64 v2, u8 op) -{ - switch (op) { - case BPF_JSET: - return !!(v1 & v2); - case BPF_JEQ: - return v1 == v2; - case BPF_JNE: - return v1 != v2; - case BPF_JGT: - return (u64)v1 > (u64)v2; - case BPF_JGE: - return (u64)v1 >= (u64)v2; - case BPF_JLT: - return (u64)v1 < (u64)v2; - case BPF_JLE: - return (u64)v1 <= (u64)v2; - case BPF_JSGT: - return v1 > v2; - case BPF_JSGE: - return v1 >= v2; - case BPF_JSLT: - return v1 < v2; - case BPF_JSLE: - return v1 <= v2; - } - return false; -} - -static int __bpf_emit_jmp_imm(struct bpf_test *self, void *arg, - struct bpf_insn *insns, s64 dst, s64 imm) -{ - int op = *(int *)arg; - - if (insns) { - bool match = __bpf_match_jmp_cond(dst, (s32)imm, op); - int i = 0; - - insns[i++] = BPF_ALU32_IMM(BPF_MOV, R0, match); - - i += __bpf_ld_imm64(&insns[i], R1, dst); - insns[i++] = BPF_JMP_IMM(op, R1, imm, 1); - if (!match) - insns[i++] = BPF_JMP_IMM(BPF_JA, 0, 0, 1); - insns[i++] = BPF_EXIT_INSN(); - - return i; - } - - return 5 + 1; -} - -static int __bpf_emit_jmp32_imm(struct bpf_test *self, void *arg, - struct bpf_insn *insns, s64 dst, s64 imm) -{ - int op = *(int *)arg; - - if (insns) { - bool match = __bpf_match_jmp_cond((s32)dst, (s32)imm, op); - int i = 0; - - i += __bpf_ld_imm64(&insns[i], R1, dst); - insns[i++] = BPF_JMP32_IMM(op, R1, imm, 1); - if (!match) - insns[i++] = BPF_JMP_IMM(BPF_JA, 0, 0, 1); - insns[i++] = BPF_EXIT_INSN(); - - return i; - } - - return 5; -} - -static int __bpf_emit_jmp_reg(struct bpf_test *self, void *arg, - struct bpf_insn *insns, s64 dst, s64 src) -{ - int op = *(int *)arg; - - if (insns) { - bool match = __bpf_match_jmp_cond(dst, src, op); - int i = 0; - - i += __bpf_ld_imm64(&insns[i], R1, dst); - i += __bpf_ld_imm64(&insns[i], R2, src); - insns[i++] = BPF_JMP_REG(op, R1, R2, 1); - if (!match) - insns[i++] = BPF_JMP_IMM(BPF_JA, 0, 0, 1); - insns[i++] = BPF_EXIT_INSN(); - - return i; - } - - return 7; -} - -static int __bpf_emit_jmp32_reg(struct bpf_test *self, void *arg, - struct bpf_insn *insns, s64 dst, s64 src) -{ - int op = *(int *)arg; - - if (insns) { - bool match = __bpf_match_jmp_cond((s32)dst, (s32)src, op); - int i = 0; - - i += __bpf_ld_imm64(&insns[i], R1, dst); - i += __bpf_ld_imm64(&insns[i], R2, src); - insns[i++] = BPF_JMP32_REG(op, R1, R2, 1); - if (!match) - insns[i++] = BPF_JMP_IMM(BPF_JA, 0, 0, 1); - insns[i++] = BPF_EXIT_INSN(); - - return i; - } - - return 7; -} - -static int __bpf_fill_jmp_imm(struct bpf_test *self, int op) -{ - return __bpf_fill_pattern(self, &op, 64, 32, - PATTERN_BLOCK1, PATTERN_BLOCK2, - &__bpf_emit_jmp_imm); -} - -static int __bpf_fill_jmp32_imm(struct bpf_test *self, int op) -{ - return __bpf_fill_pattern(self, &op, 64, 32, - PATTERN_BLOCK1, PATTERN_BLOCK2, - &__bpf_emit_jmp32_imm); -} - -static int __bpf_fill_jmp_reg(struct bpf_test *self, int op) -{ - return __bpf_fill_pattern(self, &op, 64, 64, - PATTERN_BLOCK1, PATTERN_BLOCK2, - &__bpf_emit_jmp_reg); -} - -static int __bpf_fill_jmp32_reg(struct bpf_test *self, int op) -{ - return __bpf_fill_pattern(self, &op, 64, 64, - PATTERN_BLOCK1, PATTERN_BLOCK2, - &__bpf_emit_jmp32_reg); -} - -/* JMP immediate tests */ -static int bpf_fill_jmp_jset_imm(struct bpf_test *self) -{ - return __bpf_fill_jmp_imm(self, BPF_JSET); -} - -static int bpf_fill_jmp_jeq_imm(struct bpf_test *self) -{ - return __bpf_fill_jmp_imm(self, BPF_JEQ); -} - -static int bpf_fill_jmp_jne_imm(struct bpf_test *self) -{ - return __bpf_fill_jmp_imm(self, BPF_JNE); -} - -static int bpf_fill_jmp_jgt_imm(struct bpf_test *self) -{ - return __bpf_fill_jmp_imm(self, BPF_JGT); -} - -static int bpf_fill_jmp_jge_imm(struct bpf_test *self) -{ - return __bpf_fill_jmp_imm(self, BPF_JGE); -} - -static int bpf_fill_jmp_jlt_imm(struct bpf_test *self) -{ - return __bpf_fill_jmp_imm(self, BPF_JLT); -} - -static int bpf_fill_jmp_jle_imm(struct bpf_test *self) -{ - return __bpf_fill_jmp_imm(self, BPF_JLE); -} - -static int bpf_fill_jmp_jsgt_imm(struct bpf_test *self) -{ - return __bpf_fill_jmp_imm(self, BPF_JSGT); -} - -static int bpf_fill_jmp_jsge_imm(struct bpf_test *self) -{ - return __bpf_fill_jmp_imm(self, BPF_JSGE); -} - -static int bpf_fill_jmp_jslt_imm(struct bpf_test *self) -{ - return __bpf_fill_jmp_imm(self, BPF_JSLT); -} - -static int bpf_fill_jmp_jsle_imm(struct bpf_test *self) -{ - return __bpf_fill_jmp_imm(self, BPF_JSLE); -} - -/* JMP32 immediate tests */ -static int bpf_fill_jmp32_jset_imm(struct bpf_test *self) -{ - return __bpf_fill_jmp32_imm(self, BPF_JSET); -} - -static int bpf_fill_jmp32_jeq_imm(struct bpf_test *self) -{ - return __bpf_fill_jmp32_imm(self, BPF_JEQ); -} - -static int bpf_fill_jmp32_jne_imm(struct bpf_test *self) -{ - return __bpf_fill_jmp32_imm(self, BPF_JNE); -} - -static int bpf_fill_jmp32_jgt_imm(struct bpf_test *self) -{ - return __bpf_fill_jmp32_imm(self, BPF_JGT); -} - -static int bpf_fill_jmp32_jge_imm(struct bpf_test *self) -{ - return __bpf_fill_jmp32_imm(self, BPF_JGE); -} - -static int bpf_fill_jmp32_jlt_imm(struct bpf_test *self) -{ - return __bpf_fill_jmp32_imm(self, BPF_JLT); -} - -static int bpf_fill_jmp32_jle_imm(struct bpf_test *self) -{ - return __bpf_fill_jmp32_imm(self, BPF_JLE); -} - -static int bpf_fill_jmp32_jsgt_imm(struct bpf_test *self) -{ - return __bpf_fill_jmp32_imm(self, BPF_JSGT); -} - -static int bpf_fill_jmp32_jsge_imm(struct bpf_test *self) -{ - return __bpf_fill_jmp32_imm(self, BPF_JSGE); -} - -static int bpf_fill_jmp32_jslt_imm(struct bpf_test *self) -{ - return __bpf_fill_jmp32_imm(self, BPF_JSLT); -} - -static int bpf_fill_jmp32_jsle_imm(struct bpf_test *self) -{ - return __bpf_fill_jmp32_imm(self, BPF_JSLE); -} - -/* JMP register tests */ -static int bpf_fill_jmp_jset_reg(struct bpf_test *self) -{ - return __bpf_fill_jmp_reg(self, BPF_JSET); -} - -static int bpf_fill_jmp_jeq_reg(struct bpf_test *self) -{ - return __bpf_fill_jmp_reg(self, BPF_JEQ); -} - -static int bpf_fill_jmp_jne_reg(struct bpf_test *self) -{ - return __bpf_fill_jmp_reg(self, BPF_JNE); -} - -static int bpf_fill_jmp_jgt_reg(struct bpf_test *self) -{ - return __bpf_fill_jmp_reg(self, BPF_JGT); -} - -static int bpf_fill_jmp_jge_reg(struct bpf_test *self) -{ - return __bpf_fill_jmp_reg(self, BPF_JGE); -} - -static int bpf_fill_jmp_jlt_reg(struct bpf_test *self) -{ - return __bpf_fill_jmp_reg(self, BPF_JLT); -} - -static int bpf_fill_jmp_jle_reg(struct bpf_test *self) -{ - return __bpf_fill_jmp_reg(self, BPF_JLE); -} - -static int bpf_fill_jmp_jsgt_reg(struct bpf_test *self) -{ - return __bpf_fill_jmp_reg(self, BPF_JSGT); -} - -static int bpf_fill_jmp_jsge_reg(struct bpf_test *self) -{ - return __bpf_fill_jmp_reg(self, BPF_JSGE); -} - -static int bpf_fill_jmp_jslt_reg(struct bpf_test *self) -{ - return __bpf_fill_jmp_reg(self, BPF_JSLT); -} - -static int bpf_fill_jmp_jsle_reg(struct bpf_test *self) -{ - return __bpf_fill_jmp_reg(self, BPF_JSLE); -} - -/* JMP32 register tests */ -static int bpf_fill_jmp32_jset_reg(struct bpf_test *self) -{ - return __bpf_fill_jmp32_reg(self, BPF_JSET); -} - -static int bpf_fill_jmp32_jeq_reg(struct bpf_test *self) -{ - return __bpf_fill_jmp32_reg(self, BPF_JEQ); -} - -static int bpf_fill_jmp32_jne_reg(struct bpf_test *self) -{ - return __bpf_fill_jmp32_reg(self, BPF_JNE); -} - -static int bpf_fill_jmp32_jgt_reg(struct bpf_test *self) -{ - return __bpf_fill_jmp32_reg(self, BPF_JGT); -} - -static int bpf_fill_jmp32_jge_reg(struct bpf_test *self) -{ - return __bpf_fill_jmp32_reg(self, BPF_JGE); -} - -static int bpf_fill_jmp32_jlt_reg(struct bpf_test *self) -{ - return __bpf_fill_jmp32_reg(self, BPF_JLT); -} - -static int bpf_fill_jmp32_jle_reg(struct bpf_test *self) -{ - return __bpf_fill_jmp32_reg(self, BPF_JLE); -} - -static int bpf_fill_jmp32_jsgt_reg(struct bpf_test *self) -{ - return __bpf_fill_jmp32_reg(self, BPF_JSGT); -} - -static int bpf_fill_jmp32_jsge_reg(struct bpf_test *self) -{ - return __bpf_fill_jmp32_reg(self, BPF_JSGE); -} - -static int bpf_fill_jmp32_jslt_reg(struct bpf_test *self) -{ - return __bpf_fill_jmp32_reg(self, BPF_JSLT); -} - -static int bpf_fill_jmp32_jsle_reg(struct bpf_test *self) -{ - return __bpf_fill_jmp32_reg(self, BPF_JSLE); -} - -/* - * Set up a sequence of staggered jumps, forwards and backwards with - * increasing offset. This tests the conversion of relative jumps to - * JITed native jumps. On some architectures, for example MIPS, a large - * PC-relative jump offset may overflow the immediate field of the native - * conditional branch instruction, triggering a conversion to use an - * absolute jump instead. Since this changes the jump offsets, another - * offset computation pass is necessary, and that may in turn trigger - * another branch conversion. This jump sequence is particularly nasty - * in that regard. - * - * The sequence generation is parameterized by size and jump type. - * The size must be even, and the expected result is always size + 1. - * Below is an example with size=8 and result=9. - * - * ________________________Start - * R0 = 0 - * R1 = r1 - * R2 = r2 - * ,------- JMP +4 * 3______________Preamble: 4 insns - * ,----------|-ind 0- if R0 != 7 JMP 8 * 3 + 1 <--------------------. - * | | R0 = 8 | - * | | JMP +7 * 3 ------------------------. - * | ,--------|-----1- if R0 != 5 JMP 7 * 3 + 1 <--------------. | | - * | | | R0 = 6 | | | - * | | | JMP +5 * 3 ------------------. | | - * | | ,------|-----2- if R0 != 3 JMP 6 * 3 + 1 <--------. | | | | - * | | | | R0 = 4 | | | | | - * | | | | JMP +3 * 3 ------------. | | | | - * | | | ,----|-----3- if R0 != 1 JMP 5 * 3 + 1 <--. | | | | | | - * | | | | | R0 = 2 | | | | | | | - * | | | | | JMP +1 * 3 ------. | | | | | | - * | | | | ,--t=====4> if R0 != 0 JMP 4 * 3 + 1 1 2 3 4 5 6 7 8 loc - * | | | | | R0 = 1 -1 +2 -3 +4 -5 +6 -7 +8 off - * | | | | | JMP -2 * 3 ---' | | | | | | | - * | | | | | ,------5- if R0 != 2 JMP 3 * 3 + 1 <-----' | | | | | | - * | | | | | | R0 = 3 | | | | | | - * | | | | | | JMP -4 * 3 ---------' | | | | | - * | | | | | | ,----6- if R0 != 4 JMP 2 * 3 + 1 <-----------' | | | | - * | | | | | | | R0 = 5 | | | | - * | | | | | | | JMP -6 * 3 ---------------' | | | - * | | | | | | | ,--7- if R0 != 6 JMP 1 * 3 + 1 <-----------------' | | - * | | | | | | | | R0 = 7 | | - * | | Error | | | JMP -8 * 3 ---------------------' | - * | | paths | | | ,8- if R0 != 8 JMP 0 * 3 + 1 <-----------------------' - * | | | | | | | | | R0 = 9__________________Sequence: 3 * size - 1 insns - * `-+-+-+-+-+-+-+-+-> EXIT____________________Return: 1 insn - * - */ - -/* The maximum size parameter */ -#define MAX_STAGGERED_JMP_SIZE ((0x7fff / 3) & ~1) - -/* We use a reduced number of iterations to get a reasonable execution time */ -#define NR_STAGGERED_JMP_RUNS 10 - -static int __bpf_fill_staggered_jumps(struct bpf_test *self, - const struct bpf_insn *jmp, - u64 r1, u64 r2) -{ - int size = self->test[0].result - 1; - int len = 4 + 3 * (size + 1); - struct bpf_insn *insns; - int off, ind; - - insns = kmalloc_objs(*insns, len); - if (!insns) - return -ENOMEM; - - /* Preamble */ - insns[0] = BPF_ALU64_IMM(BPF_MOV, R0, 0); - insns[1] = BPF_ALU64_IMM(BPF_MOV, R1, r1); - insns[2] = BPF_ALU64_IMM(BPF_MOV, R2, r2); - insns[3] = BPF_JMP_IMM(BPF_JA, 0, 0, 3 * size / 2); - - /* Sequence */ - for (ind = 0, off = size; ind <= size; ind++, off -= 2) { - struct bpf_insn *ins = &insns[4 + 3 * ind]; - int loc; - - if (off == 0) - off--; - - loc = abs(off); - ins[0] = BPF_JMP_IMM(BPF_JNE, R0, loc - 1, - 3 * (size - ind) + 1); - ins[1] = BPF_ALU64_IMM(BPF_MOV, R0, loc); - ins[2] = *jmp; - ins[2].off = 3 * (off - 1); - } - - /* Return */ - insns[len - 1] = BPF_EXIT_INSN(); - - self->u.ptr.insns = insns; - self->u.ptr.len = len; - - return 0; -} - -/* 64-bit unconditional jump */ -static int bpf_fill_staggered_ja(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP_IMM(BPF_JA, 0, 0, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, 0, 0); -} - -/* 64-bit immediate jumps */ -static int bpf_fill_staggered_jeq_imm(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP_IMM(BPF_JEQ, R1, 1234, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, 1234, 0); -} - -static int bpf_fill_staggered_jne_imm(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP_IMM(BPF_JNE, R1, 1234, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, 4321, 0); -} - -static int bpf_fill_staggered_jset_imm(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP_IMM(BPF_JSET, R1, 0x82, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, 0x86, 0); -} - -static int bpf_fill_staggered_jgt_imm(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP_IMM(BPF_JGT, R1, 1234, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, 0x80000000, 0); -} - -static int bpf_fill_staggered_jge_imm(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP_IMM(BPF_JGE, R1, 1234, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, 1234, 0); -} - -static int bpf_fill_staggered_jlt_imm(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP_IMM(BPF_JLT, R1, 0x80000000, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, 1234, 0); -} - -static int bpf_fill_staggered_jle_imm(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP_IMM(BPF_JLE, R1, 1234, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, 1234, 0); -} - -static int bpf_fill_staggered_jsgt_imm(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP_IMM(BPF_JSGT, R1, -2, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, -1, 0); -} - -static int bpf_fill_staggered_jsge_imm(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP_IMM(BPF_JSGE, R1, -2, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, -2, 0); -} - -static int bpf_fill_staggered_jslt_imm(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP_IMM(BPF_JSLT, R1, -1, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, -2, 0); -} - -static int bpf_fill_staggered_jsle_imm(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP_IMM(BPF_JSLE, R1, -1, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, -1, 0); -} - -/* 64-bit register jumps */ -static int bpf_fill_staggered_jeq_reg(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP_REG(BPF_JEQ, R1, R2, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, 1234, 1234); -} - -static int bpf_fill_staggered_jne_reg(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP_REG(BPF_JNE, R1, R2, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, 4321, 1234); -} - -static int bpf_fill_staggered_jset_reg(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP_REG(BPF_JSET, R1, R2, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, 0x86, 0x82); -} - -static int bpf_fill_staggered_jgt_reg(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP_REG(BPF_JGT, R1, R2, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, 0x80000000, 1234); -} - -static int bpf_fill_staggered_jge_reg(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP_REG(BPF_JGE, R1, R2, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, 1234, 1234); -} - -static int bpf_fill_staggered_jlt_reg(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP_REG(BPF_JLT, R1, R2, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, 1234, 0x80000000); -} - -static int bpf_fill_staggered_jle_reg(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP_REG(BPF_JLE, R1, R2, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, 1234, 1234); -} - -static int bpf_fill_staggered_jsgt_reg(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP_REG(BPF_JSGT, R1, R2, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, -1, -2); -} - -static int bpf_fill_staggered_jsge_reg(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP_REG(BPF_JSGE, R1, R2, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, -2, -2); -} - -static int bpf_fill_staggered_jslt_reg(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP_REG(BPF_JSLT, R1, R2, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, -2, -1); -} - -static int bpf_fill_staggered_jsle_reg(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP_REG(BPF_JSLE, R1, R2, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, -1, -1); -} - -/* 32-bit immediate jumps */ -static int bpf_fill_staggered_jeq32_imm(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP32_IMM(BPF_JEQ, R1, 1234, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, 1234, 0); -} - -static int bpf_fill_staggered_jne32_imm(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP32_IMM(BPF_JNE, R1, 1234, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, 4321, 0); -} - -static int bpf_fill_staggered_jset32_imm(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP32_IMM(BPF_JSET, R1, 0x82, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, 0x86, 0); -} - -static int bpf_fill_staggered_jgt32_imm(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP32_IMM(BPF_JGT, R1, 1234, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, 0x80000000, 0); -} - -static int bpf_fill_staggered_jge32_imm(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP32_IMM(BPF_JGE, R1, 1234, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, 1234, 0); -} - -static int bpf_fill_staggered_jlt32_imm(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP32_IMM(BPF_JLT, R1, 0x80000000, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, 1234, 0); -} - -static int bpf_fill_staggered_jle32_imm(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP32_IMM(BPF_JLE, R1, 1234, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, 1234, 0); -} - -static int bpf_fill_staggered_jsgt32_imm(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP32_IMM(BPF_JSGT, R1, -2, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, -1, 0); -} - -static int bpf_fill_staggered_jsge32_imm(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP32_IMM(BPF_JSGE, R1, -2, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, -2, 0); -} - -static int bpf_fill_staggered_jslt32_imm(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP32_IMM(BPF_JSLT, R1, -1, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, -2, 0); -} - -static int bpf_fill_staggered_jsle32_imm(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP32_IMM(BPF_JSLE, R1, -1, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, -1, 0); -} - -/* 32-bit register jumps */ -static int bpf_fill_staggered_jeq32_reg(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP32_REG(BPF_JEQ, R1, R2, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, 1234, 1234); -} - -static int bpf_fill_staggered_jne32_reg(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP32_REG(BPF_JNE, R1, R2, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, 4321, 1234); -} - -static int bpf_fill_staggered_jset32_reg(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP32_REG(BPF_JSET, R1, R2, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, 0x86, 0x82); -} - -static int bpf_fill_staggered_jgt32_reg(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP32_REG(BPF_JGT, R1, R2, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, 0x80000000, 1234); -} - -static int bpf_fill_staggered_jge32_reg(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP32_REG(BPF_JGE, R1, R2, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, 1234, 1234); -} - -static int bpf_fill_staggered_jlt32_reg(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP32_REG(BPF_JLT, R1, R2, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, 1234, 0x80000000); -} - -static int bpf_fill_staggered_jle32_reg(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP32_REG(BPF_JLE, R1, R2, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, 1234, 1234); -} - -static int bpf_fill_staggered_jsgt32_reg(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP32_REG(BPF_JSGT, R1, R2, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, -1, -2); -} - -static int bpf_fill_staggered_jsge32_reg(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP32_REG(BPF_JSGE, R1, R2, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, -2, -2); -} - -static int bpf_fill_staggered_jslt32_reg(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP32_REG(BPF_JSLT, R1, R2, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, -2, -1); -} - -static int bpf_fill_staggered_jsle32_reg(struct bpf_test *self) -{ - struct bpf_insn jmp = BPF_JMP32_REG(BPF_JSLE, R1, R2, 0); - - return __bpf_fill_staggered_jumps(self, &jmp, -1, -1); -} - - -static struct bpf_test tests[] = { - { - "TAX", - .u.insns = { - BPF_STMT(BPF_LD | BPF_IMM, 1), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_LD | BPF_IMM, 2), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_X, 0), - BPF_STMT(BPF_ALU | BPF_NEG, 0), /* A == -3 */ - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_LD | BPF_LEN, 0), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_X, 0), - BPF_STMT(BPF_MISC | BPF_TAX, 0), /* X == len - 3 */ - BPF_STMT(BPF_LD | BPF_B | BPF_IND, 1), - BPF_STMT(BPF_RET | BPF_A, 0) - }, - CLASSIC, - { 10, 20, 30, 40, 50 }, - { { 2, 10 }, { 3, 20 }, { 4, 30 } }, - }, - { - "TXA", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_LEN, 0), - BPF_STMT(BPF_MISC | BPF_TXA, 0), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_X, 0), - BPF_STMT(BPF_RET | BPF_A, 0) /* A == len * 2 */ - }, - CLASSIC, - { 10, 20, 30, 40, 50 }, - { { 1, 2 }, { 3, 6 }, { 4, 8 } }, - }, - { - "ADD_SUB_MUL_K", - .u.insns = { - BPF_STMT(BPF_LD | BPF_IMM, 1), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_K, 2), - BPF_STMT(BPF_LDX | BPF_IMM, 3), - BPF_STMT(BPF_ALU | BPF_SUB | BPF_X, 0), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_K, 0xffffffff), - BPF_STMT(BPF_ALU | BPF_MUL | BPF_K, 3), - BPF_STMT(BPF_RET | BPF_A, 0) - }, - CLASSIC | FLAG_NO_DATA, - { }, - { { 0, 0xfffffffd } } - }, - { - "DIV_MOD_KX", - .u.insns = { - BPF_STMT(BPF_LD | BPF_IMM, 8), - BPF_STMT(BPF_ALU | BPF_DIV | BPF_K, 2), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_LD | BPF_IMM, 0xffffffff), - BPF_STMT(BPF_ALU | BPF_DIV | BPF_X, 0), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_LD | BPF_IMM, 0xffffffff), - BPF_STMT(BPF_ALU | BPF_DIV | BPF_K, 0x70000000), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_LD | BPF_IMM, 0xffffffff), - BPF_STMT(BPF_ALU | BPF_MOD | BPF_X, 0), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_LD | BPF_IMM, 0xffffffff), - BPF_STMT(BPF_ALU | BPF_MOD | BPF_K, 0x70000000), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_X, 0), - BPF_STMT(BPF_RET | BPF_A, 0) - }, - CLASSIC | FLAG_NO_DATA, - { }, - { { 0, 0x20000000 } } - }, - { - "AND_OR_LSH_K", - .u.insns = { - BPF_STMT(BPF_LD | BPF_IMM, 0xff), - BPF_STMT(BPF_ALU | BPF_AND | BPF_K, 0xf0), - BPF_STMT(BPF_ALU | BPF_LSH | BPF_K, 27), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_LD | BPF_IMM, 0xf), - BPF_STMT(BPF_ALU | BPF_OR | BPF_K, 0xf0), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_X, 0), - BPF_STMT(BPF_RET | BPF_A, 0) - }, - CLASSIC | FLAG_NO_DATA, - { }, - { { 0, 0x800000ff }, { 1, 0x800000ff } }, - }, - { - "LD_IMM_0", - .u.insns = { - BPF_STMT(BPF_LD | BPF_IMM, 0), /* ld #0 */ - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, 0, 1, 0), - BPF_STMT(BPF_RET | BPF_K, 0), - BPF_STMT(BPF_RET | BPF_K, 1), - }, - CLASSIC, - { }, - { { 1, 1 } }, - }, - { - "LD_IND", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_LEN, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_IND, MAX_K), - BPF_STMT(BPF_RET | BPF_K, 1) - }, - CLASSIC, - { }, - { { 1, 0 }, { 10, 0 }, { 60, 0 } }, - }, - { - "LD_ABS", - .u.insns = { - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, 1000), - BPF_STMT(BPF_RET | BPF_K, 1) - }, - CLASSIC, - { }, - { { 1, 0 }, { 10, 0 }, { 60, 0 } }, - }, - { - "LD_ABS_LL", - .u.insns = { - BPF_STMT(BPF_LD | BPF_B | BPF_ABS, SKF_LL_OFF), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_LD | BPF_B | BPF_ABS, SKF_LL_OFF + 1), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_X, 0), - BPF_STMT(BPF_RET | BPF_A, 0) - }, - CLASSIC, - { 1, 2, 3 }, - { { 1, 0 }, { 2, 3 } }, - }, - { - "LD_IND_LL", - .u.insns = { - BPF_STMT(BPF_LD | BPF_IMM, SKF_LL_OFF - 1), - BPF_STMT(BPF_LDX | BPF_LEN, 0), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_X, 0), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_LD | BPF_B | BPF_IND, 0), - BPF_STMT(BPF_RET | BPF_A, 0) - }, - CLASSIC, - { 1, 2, 3, 0xff }, - { { 1, 1 }, { 3, 3 }, { 4, 0xff } }, - }, - { - "LD_ABS_NET", - .u.insns = { - BPF_STMT(BPF_LD | BPF_B | BPF_ABS, SKF_NET_OFF), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_LD | BPF_B | BPF_ABS, SKF_NET_OFF + 1), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_X, 0), - BPF_STMT(BPF_RET | BPF_A, 0) - }, - CLASSIC, - { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3 }, - { { 15, 0 }, { 16, 3 } }, - }, - { - "LD_IND_NET", - .u.insns = { - BPF_STMT(BPF_LD | BPF_IMM, SKF_NET_OFF - 15), - BPF_STMT(BPF_LDX | BPF_LEN, 0), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_X, 0), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_LD | BPF_B | BPF_IND, 0), - BPF_STMT(BPF_RET | BPF_A, 0) - }, - CLASSIC, - { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3 }, - { { 14, 0 }, { 15, 1 }, { 17, 3 } }, - }, - { - "LD_PKTTYPE", - .u.insns = { - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_PKTTYPE), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, SKB_TYPE, 1, 0), - BPF_STMT(BPF_RET | BPF_K, 1), - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_PKTTYPE), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, SKB_TYPE, 1, 0), - BPF_STMT(BPF_RET | BPF_K, 1), - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_PKTTYPE), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, SKB_TYPE, 1, 0), - BPF_STMT(BPF_RET | BPF_K, 1), - BPF_STMT(BPF_RET | BPF_A, 0) - }, - CLASSIC, - { }, - { { 1, 3 }, { 10, 3 } }, - }, - { - "LD_MARK", - .u.insns = { - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_MARK), - BPF_STMT(BPF_RET | BPF_A, 0) - }, - CLASSIC, - { }, - { { 1, SKB_MARK}, { 10, SKB_MARK} }, - }, - { - "LD_RXHASH", - .u.insns = { - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_RXHASH), - BPF_STMT(BPF_RET | BPF_A, 0) - }, - CLASSIC, - { }, - { { 1, SKB_HASH}, { 10, SKB_HASH} }, - }, - { - "LD_QUEUE", - .u.insns = { - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_QUEUE), - BPF_STMT(BPF_RET | BPF_A, 0) - }, - CLASSIC, - { }, - { { 1, SKB_QUEUE_MAP }, { 10, SKB_QUEUE_MAP } }, - }, - { - "LD_PROTOCOL", - .u.insns = { - BPF_STMT(BPF_LD | BPF_B | BPF_ABS, 1), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, 20, 1, 0), - BPF_STMT(BPF_RET | BPF_K, 0), - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_PROTOCOL), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_LD | BPF_B | BPF_ABS, 2), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, 30, 1, 0), - BPF_STMT(BPF_RET | BPF_K, 0), - BPF_STMT(BPF_MISC | BPF_TXA, 0), - BPF_STMT(BPF_RET | BPF_A, 0) - }, - CLASSIC, - { 10, 20, 30 }, - { { 10, ETH_P_IP }, { 100, ETH_P_IP } }, - }, - { - "LD_VLAN_TAG", - .u.insns = { - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_VLAN_TAG), - BPF_STMT(BPF_RET | BPF_A, 0) - }, - CLASSIC, - { }, - { - { 1, SKB_VLAN_TCI }, - { 10, SKB_VLAN_TCI } - }, - }, - { - "LD_VLAN_TAG_PRESENT", - .u.insns = { - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_VLAN_TAG_PRESENT), - BPF_STMT(BPF_RET | BPF_A, 0) - }, - CLASSIC, - { }, - { - { 1, SKB_VLAN_PRESENT }, - { 10, SKB_VLAN_PRESENT } - }, - }, - { - "LD_IFINDEX", - .u.insns = { - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_IFINDEX), - BPF_STMT(BPF_RET | BPF_A, 0) - }, - CLASSIC, - { }, - { { 1, SKB_DEV_IFINDEX }, { 10, SKB_DEV_IFINDEX } }, - }, - { - "LD_HATYPE", - .u.insns = { - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_HATYPE), - BPF_STMT(BPF_RET | BPF_A, 0) - }, - CLASSIC, - { }, - { { 1, SKB_DEV_TYPE }, { 10, SKB_DEV_TYPE } }, - }, - { - "LD_CPU", - .u.insns = { - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_CPU), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_CPU), - BPF_STMT(BPF_ALU | BPF_SUB | BPF_X, 0), - BPF_STMT(BPF_RET | BPF_A, 0) - }, - CLASSIC, - { }, - { { 1, 0 }, { 10, 0 } }, - }, - { - "LD_NLATTR", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_IMM, 2), - BPF_STMT(BPF_MISC | BPF_TXA, 0), - BPF_STMT(BPF_LDX | BPF_IMM, 3), - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_NLATTR), - BPF_STMT(BPF_RET | BPF_A, 0) - }, - CLASSIC, -#ifdef __BIG_ENDIAN - { 0xff, 0xff, 0, 4, 0, 2, 0, 4, 0, 3 }, -#else - { 0xff, 0xff, 4, 0, 2, 0, 4, 0, 3, 0 }, -#endif - { { 4, 0 }, { 20, 6 } }, - }, - { - "LD_NLATTR_NEST", - .u.insns = { - BPF_STMT(BPF_LD | BPF_IMM, 2), - BPF_STMT(BPF_LDX | BPF_IMM, 3), - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_NLATTR_NEST), - BPF_STMT(BPF_LD | BPF_IMM, 2), - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_NLATTR_NEST), - BPF_STMT(BPF_LD | BPF_IMM, 2), - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_NLATTR_NEST), - BPF_STMT(BPF_LD | BPF_IMM, 2), - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_NLATTR_NEST), - BPF_STMT(BPF_LD | BPF_IMM, 2), - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_NLATTR_NEST), - BPF_STMT(BPF_LD | BPF_IMM, 2), - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_NLATTR_NEST), - BPF_STMT(BPF_LD | BPF_IMM, 2), - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_NLATTR_NEST), - BPF_STMT(BPF_LD | BPF_IMM, 2), - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_NLATTR_NEST), - BPF_STMT(BPF_RET | BPF_A, 0) - }, - CLASSIC, -#ifdef __BIG_ENDIAN - { 0xff, 0xff, 0, 12, 0, 1, 0, 4, 0, 2, 0, 4, 0, 3 }, -#else - { 0xff, 0xff, 12, 0, 1, 0, 4, 0, 2, 0, 4, 0, 3, 0 }, -#endif - { { 4, 0 }, { 20, 10 } }, - }, - { - "LD_PAYLOAD_OFF", - .u.insns = { - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_PAY_OFFSET), - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_PAY_OFFSET), - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_PAY_OFFSET), - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_PAY_OFFSET), - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_PAY_OFFSET), - BPF_STMT(BPF_RET | BPF_A, 0) - }, - CLASSIC, - /* 00:00:00:00:00:00 > 00:00:00:00:00:00, ethtype IPv4 (0x0800), - * length 98: 127.0.0.1 > 127.0.0.1: ICMP echo request, - * id 9737, seq 1, length 64 - */ - { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x08, 0x00, - 0x45, 0x00, 0x00, 0x54, 0xac, 0x8b, 0x40, 0x00, 0x40, - 0x01, 0x90, 0x1b, 0x7f, 0x00, 0x00, 0x01 }, - { { 30, 0 }, { 100, 42 } }, - }, - { - "LD_ANC_XOR", - .u.insns = { - BPF_STMT(BPF_LD | BPF_IMM, 10), - BPF_STMT(BPF_LDX | BPF_IMM, 300), - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_ALU_XOR_X), - BPF_STMT(BPF_RET | BPF_A, 0) - }, - CLASSIC, - { }, - { { 4, 0xA ^ 300 }, { 20, 0xA ^ 300 } }, - }, - { - "SPILL_FILL", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_LEN, 0), - BPF_STMT(BPF_LD | BPF_IMM, 2), - BPF_STMT(BPF_ALU | BPF_RSH, 1), - BPF_STMT(BPF_ALU | BPF_XOR | BPF_X, 0), - BPF_STMT(BPF_ST, 1), /* M1 = 1 ^ len */ - BPF_STMT(BPF_ALU | BPF_XOR | BPF_K, 0x80000000), - BPF_STMT(BPF_ST, 2), /* M2 = 1 ^ len ^ 0x80000000 */ - BPF_STMT(BPF_STX, 15), /* M3 = len */ - BPF_STMT(BPF_LDX | BPF_MEM, 1), - BPF_STMT(BPF_LD | BPF_MEM, 2), - BPF_STMT(BPF_ALU | BPF_XOR | BPF_X, 0), - BPF_STMT(BPF_LDX | BPF_MEM, 15), - BPF_STMT(BPF_ALU | BPF_XOR | BPF_X, 0), - BPF_STMT(BPF_RET | BPF_A, 0) - }, - CLASSIC, - { }, - { { 1, 0x80000001 }, { 2, 0x80000002 }, { 60, 0x80000000 ^ 60 } } - }, - { - "JEQ", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_LEN, 0), - BPF_STMT(BPF_LD | BPF_B | BPF_ABS, 2), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_X, 0, 0, 1), - BPF_STMT(BPF_RET | BPF_K, 1), - BPF_STMT(BPF_RET | BPF_K, MAX_K) - }, - CLASSIC, - { 3, 3, 3, 3, 3 }, - { { 1, 0 }, { 3, 1 }, { 4, MAX_K } }, - }, - { - "JGT", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_LEN, 0), - BPF_STMT(BPF_LD | BPF_B | BPF_ABS, 2), - BPF_JUMP(BPF_JMP | BPF_JGT | BPF_X, 0, 0, 1), - BPF_STMT(BPF_RET | BPF_K, 1), - BPF_STMT(BPF_RET | BPF_K, MAX_K) - }, - CLASSIC, - { 4, 4, 4, 3, 3 }, - { { 2, 0 }, { 3, 1 }, { 4, MAX_K } }, - }, - { - "JGE (jt 0), test 1", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_LEN, 0), - BPF_STMT(BPF_LD | BPF_B | BPF_ABS, 2), - BPF_JUMP(BPF_JMP | BPF_JGE | BPF_X, 0, 0, 1), - BPF_STMT(BPF_RET | BPF_K, 1), - BPF_STMT(BPF_RET | BPF_K, MAX_K) - }, - CLASSIC, - { 4, 4, 4, 3, 3 }, - { { 2, 0 }, { 3, 1 }, { 4, 1 } }, - }, - { - "JGE (jt 0), test 2", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_LEN, 0), - BPF_STMT(BPF_LD | BPF_B | BPF_ABS, 2), - BPF_JUMP(BPF_JMP | BPF_JGE | BPF_X, 0, 0, 1), - BPF_STMT(BPF_RET | BPF_K, 1), - BPF_STMT(BPF_RET | BPF_K, MAX_K) - }, - CLASSIC, - { 4, 4, 5, 3, 3 }, - { { 4, 1 }, { 5, 1 }, { 6, MAX_K } }, - }, - { - "JGE", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_LEN, 0), - BPF_STMT(BPF_LD | BPF_B | BPF_IND, MAX_K), - BPF_JUMP(BPF_JMP | BPF_JGE | BPF_K, 1, 1, 0), - BPF_STMT(BPF_RET | BPF_K, 10), - BPF_JUMP(BPF_JMP | BPF_JGE | BPF_K, 2, 1, 0), - BPF_STMT(BPF_RET | BPF_K, 20), - BPF_JUMP(BPF_JMP | BPF_JGE | BPF_K, 3, 1, 0), - BPF_STMT(BPF_RET | BPF_K, 30), - BPF_JUMP(BPF_JMP | BPF_JGE | BPF_K, 4, 1, 0), - BPF_STMT(BPF_RET | BPF_K, 40), - BPF_STMT(BPF_RET | BPF_K, MAX_K) - }, - CLASSIC, - { 1, 2, 3, 4, 5 }, - { { 1, 20 }, { 3, 40 }, { 5, MAX_K } }, - }, - { - "JSET", - .u.insns = { - BPF_JUMP(BPF_JMP | BPF_JA, 0, 0, 0), - BPF_JUMP(BPF_JMP | BPF_JA, 1, 1, 1), - BPF_JUMP(BPF_JMP | BPF_JA, 0, 0, 0), - BPF_JUMP(BPF_JMP | BPF_JA, 0, 0, 0), - BPF_STMT(BPF_LDX | BPF_LEN, 0), - BPF_STMT(BPF_MISC | BPF_TXA, 0), - BPF_STMT(BPF_ALU | BPF_SUB | BPF_K, 4), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_LD | BPF_W | BPF_IND, 0), - BPF_JUMP(BPF_JMP | BPF_JSET | BPF_K, 1, 0, 1), - BPF_STMT(BPF_RET | BPF_K, 10), - BPF_JUMP(BPF_JMP | BPF_JSET | BPF_K, 0x80000000, 0, 1), - BPF_STMT(BPF_RET | BPF_K, 20), - BPF_JUMP(BPF_JMP | BPF_JSET | BPF_K, 0xffffff, 1, 0), - BPF_STMT(BPF_RET | BPF_K, 30), - BPF_JUMP(BPF_JMP | BPF_JSET | BPF_K, 0xffffff, 1, 0), - BPF_STMT(BPF_RET | BPF_K, 30), - BPF_JUMP(BPF_JMP | BPF_JSET | BPF_K, 0xffffff, 1, 0), - BPF_STMT(BPF_RET | BPF_K, 30), - BPF_JUMP(BPF_JMP | BPF_JSET | BPF_K, 0xffffff, 1, 0), - BPF_STMT(BPF_RET | BPF_K, 30), - BPF_JUMP(BPF_JMP | BPF_JSET | BPF_K, 0xffffff, 1, 0), - BPF_STMT(BPF_RET | BPF_K, 30), - BPF_STMT(BPF_RET | BPF_K, MAX_K) - }, - CLASSIC, - { 0, 0xAA, 0x55, 1 }, - { { 4, 10 }, { 5, 20 }, { 6, MAX_K } }, - }, - { - "tcpdump port 22", - .u.insns = { - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 12), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, 0x86dd, 0, 8), /* IPv6 */ - BPF_STMT(BPF_LD | BPF_B | BPF_ABS, 20), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, 0x84, 2, 0), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, 0x6, 1, 0), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, 0x11, 0, 17), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 54), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, 22, 14, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 56), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, 22, 12, 13), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, 0x0800, 0, 12), /* IPv4 */ - BPF_STMT(BPF_LD | BPF_B | BPF_ABS, 23), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, 0x84, 2, 0), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, 0x6, 1, 0), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, 0x11, 0, 8), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 20), - BPF_JUMP(BPF_JMP | BPF_JSET | BPF_K, 0x1fff, 6, 0), - BPF_STMT(BPF_LDX | BPF_B | BPF_MSH, 14), - BPF_STMT(BPF_LD | BPF_H | BPF_IND, 14), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, 22, 2, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_IND, 16), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, 22, 0, 1), - BPF_STMT(BPF_RET | BPF_K, 0xffff), - BPF_STMT(BPF_RET | BPF_K, 0), - }, - CLASSIC, - /* 3c:07:54:43:e5:76 > 10:bf:48:d6:43:d6, ethertype IPv4(0x0800) - * length 114: 10.1.1.149.49700 > 10.1.2.10.22: Flags [P.], - * seq 1305692979:1305693027, ack 3650467037, win 65535, - * options [nop,nop,TS val 2502645400 ecr 3971138], length 48 - */ - { 0x10, 0xbf, 0x48, 0xd6, 0x43, 0xd6, - 0x3c, 0x07, 0x54, 0x43, 0xe5, 0x76, - 0x08, 0x00, - 0x45, 0x10, 0x00, 0x64, 0x75, 0xb5, - 0x40, 0x00, 0x40, 0x06, 0xad, 0x2e, /* IP header */ - 0x0a, 0x01, 0x01, 0x95, /* ip src */ - 0x0a, 0x01, 0x02, 0x0a, /* ip dst */ - 0xc2, 0x24, - 0x00, 0x16 /* dst port */ }, - { { 10, 0 }, { 30, 0 }, { 100, 65535 } }, - }, - { - "tcpdump complex", - .u.insns = { - /* tcpdump -nei eth0 'tcp port 22 and (((ip[2:2] - - * ((ip[0]&0xf)<<2)) - ((tcp[12]&0xf0)>>2)) != 0) and - * (len > 115 or len < 30000000000)' -d - */ - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 12), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, 0x86dd, 30, 0), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, 0x800, 0, 29), - BPF_STMT(BPF_LD | BPF_B | BPF_ABS, 23), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, 0x6, 0, 27), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 20), - BPF_JUMP(BPF_JMP | BPF_JSET | BPF_K, 0x1fff, 25, 0), - BPF_STMT(BPF_LDX | BPF_B | BPF_MSH, 14), - BPF_STMT(BPF_LD | BPF_H | BPF_IND, 14), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, 22, 2, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_IND, 16), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, 22, 0, 20), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 16), - BPF_STMT(BPF_ST, 1), - BPF_STMT(BPF_LD | BPF_B | BPF_ABS, 14), - BPF_STMT(BPF_ALU | BPF_AND | BPF_K, 0xf), - BPF_STMT(BPF_ALU | BPF_LSH | BPF_K, 2), - BPF_STMT(BPF_MISC | BPF_TAX, 0x5), /* libpcap emits K on TAX */ - BPF_STMT(BPF_LD | BPF_MEM, 1), - BPF_STMT(BPF_ALU | BPF_SUB | BPF_X, 0), - BPF_STMT(BPF_ST, 5), - BPF_STMT(BPF_LDX | BPF_B | BPF_MSH, 14), - BPF_STMT(BPF_LD | BPF_B | BPF_IND, 26), - BPF_STMT(BPF_ALU | BPF_AND | BPF_K, 0xf0), - BPF_STMT(BPF_ALU | BPF_RSH | BPF_K, 2), - BPF_STMT(BPF_MISC | BPF_TAX, 0x9), /* libpcap emits K on TAX */ - BPF_STMT(BPF_LD | BPF_MEM, 5), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_X, 0, 4, 0), - BPF_STMT(BPF_LD | BPF_LEN, 0), - BPF_JUMP(BPF_JMP | BPF_JGT | BPF_K, 0x73, 1, 0), - BPF_JUMP(BPF_JMP | BPF_JGE | BPF_K, 0xfc23ac00, 1, 0), - BPF_STMT(BPF_RET | BPF_K, 0xffff), - BPF_STMT(BPF_RET | BPF_K, 0), - }, - CLASSIC, - { 0x10, 0xbf, 0x48, 0xd6, 0x43, 0xd6, - 0x3c, 0x07, 0x54, 0x43, 0xe5, 0x76, - 0x08, 0x00, - 0x45, 0x10, 0x00, 0x64, 0x75, 0xb5, - 0x40, 0x00, 0x40, 0x06, 0xad, 0x2e, /* IP header */ - 0x0a, 0x01, 0x01, 0x95, /* ip src */ - 0x0a, 0x01, 0x02, 0x0a, /* ip dst */ - 0xc2, 0x24, - 0x00, 0x16 /* dst port */ }, - { { 10, 0 }, { 30, 0 }, { 100, 65535 } }, - }, - { - "RET_A", - .u.insns = { - /* check that uninitialized X and A contain zeros */ - BPF_STMT(BPF_MISC | BPF_TXA, 0), - BPF_STMT(BPF_RET | BPF_A, 0) - }, - CLASSIC, - { }, - { {1, 0}, {2, 0} }, - }, - { - "INT: ADD trivial", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R1, 1), - BPF_ALU64_IMM(BPF_ADD, R1, 2), - BPF_ALU64_IMM(BPF_MOV, R2, 3), - BPF_ALU64_REG(BPF_SUB, R1, R2), - BPF_ALU64_IMM(BPF_ADD, R1, -1), - BPF_ALU64_IMM(BPF_MUL, R1, 3), - BPF_ALU64_REG(BPF_MOV, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xfffffffd } } - }, - { - "INT: MUL_X", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, -1), - BPF_ALU64_IMM(BPF_MOV, R1, -1), - BPF_ALU64_IMM(BPF_MOV, R2, 3), - BPF_ALU64_REG(BPF_MUL, R1, R2), - BPF_JMP_IMM(BPF_JEQ, R1, 0xfffffffd, 1), - BPF_EXIT_INSN(), - BPF_ALU64_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } } - }, - { - "INT: MUL_X2", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, -1), - BPF_ALU32_IMM(BPF_MOV, R1, -1), - BPF_ALU32_IMM(BPF_MOV, R2, 3), - BPF_ALU64_REG(BPF_MUL, R1, R2), - BPF_ALU64_IMM(BPF_RSH, R1, 8), - BPF_JMP_IMM(BPF_JEQ, R1, 0x2ffffff, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } } - }, - { - "INT: MUL32_X", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, -1), - BPF_ALU64_IMM(BPF_MOV, R1, -1), - BPF_ALU32_IMM(BPF_MOV, R2, 3), - BPF_ALU32_REG(BPF_MUL, R1, R2), - BPF_ALU64_IMM(BPF_RSH, R1, 8), - BPF_JMP_IMM(BPF_JEQ, R1, 0xffffff, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } } - }, - { - /* Have to test all register combinations, since - * JITing of different registers will produce - * different asm code. - */ - "INT: ADD 64-bit", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_ALU64_IMM(BPF_MOV, R1, 1), - BPF_ALU64_IMM(BPF_MOV, R2, 2), - BPF_ALU64_IMM(BPF_MOV, R3, 3), - BPF_ALU64_IMM(BPF_MOV, R4, 4), - BPF_ALU64_IMM(BPF_MOV, R5, 5), - BPF_ALU64_IMM(BPF_MOV, R6, 6), - BPF_ALU64_IMM(BPF_MOV, R7, 7), - BPF_ALU64_IMM(BPF_MOV, R8, 8), - BPF_ALU64_IMM(BPF_MOV, R9, 9), - BPF_ALU64_IMM(BPF_ADD, R0, 20), - BPF_ALU64_IMM(BPF_ADD, R1, 20), - BPF_ALU64_IMM(BPF_ADD, R2, 20), - BPF_ALU64_IMM(BPF_ADD, R3, 20), - BPF_ALU64_IMM(BPF_ADD, R4, 20), - BPF_ALU64_IMM(BPF_ADD, R5, 20), - BPF_ALU64_IMM(BPF_ADD, R6, 20), - BPF_ALU64_IMM(BPF_ADD, R7, 20), - BPF_ALU64_IMM(BPF_ADD, R8, 20), - BPF_ALU64_IMM(BPF_ADD, R9, 20), - BPF_ALU64_IMM(BPF_SUB, R0, 10), - BPF_ALU64_IMM(BPF_SUB, R1, 10), - BPF_ALU64_IMM(BPF_SUB, R2, 10), - BPF_ALU64_IMM(BPF_SUB, R3, 10), - BPF_ALU64_IMM(BPF_SUB, R4, 10), - BPF_ALU64_IMM(BPF_SUB, R5, 10), - BPF_ALU64_IMM(BPF_SUB, R6, 10), - BPF_ALU64_IMM(BPF_SUB, R7, 10), - BPF_ALU64_IMM(BPF_SUB, R8, 10), - BPF_ALU64_IMM(BPF_SUB, R9, 10), - BPF_ALU64_REG(BPF_ADD, R0, R0), - BPF_ALU64_REG(BPF_ADD, R0, R1), - BPF_ALU64_REG(BPF_ADD, R0, R2), - BPF_ALU64_REG(BPF_ADD, R0, R3), - BPF_ALU64_REG(BPF_ADD, R0, R4), - BPF_ALU64_REG(BPF_ADD, R0, R5), - BPF_ALU64_REG(BPF_ADD, R0, R6), - BPF_ALU64_REG(BPF_ADD, R0, R7), - BPF_ALU64_REG(BPF_ADD, R0, R8), - BPF_ALU64_REG(BPF_ADD, R0, R9), /* R0 == 155 */ - BPF_JMP_IMM(BPF_JEQ, R0, 155, 1), - BPF_EXIT_INSN(), - BPF_ALU64_REG(BPF_ADD, R1, R0), - BPF_ALU64_REG(BPF_ADD, R1, R1), - BPF_ALU64_REG(BPF_ADD, R1, R2), - BPF_ALU64_REG(BPF_ADD, R1, R3), - BPF_ALU64_REG(BPF_ADD, R1, R4), - BPF_ALU64_REG(BPF_ADD, R1, R5), - BPF_ALU64_REG(BPF_ADD, R1, R6), - BPF_ALU64_REG(BPF_ADD, R1, R7), - BPF_ALU64_REG(BPF_ADD, R1, R8), - BPF_ALU64_REG(BPF_ADD, R1, R9), /* R1 == 456 */ - BPF_JMP_IMM(BPF_JEQ, R1, 456, 1), - BPF_EXIT_INSN(), - BPF_ALU64_REG(BPF_ADD, R2, R0), - BPF_ALU64_REG(BPF_ADD, R2, R1), - BPF_ALU64_REG(BPF_ADD, R2, R2), - BPF_ALU64_REG(BPF_ADD, R2, R3), - BPF_ALU64_REG(BPF_ADD, R2, R4), - BPF_ALU64_REG(BPF_ADD, R2, R5), - BPF_ALU64_REG(BPF_ADD, R2, R6), - BPF_ALU64_REG(BPF_ADD, R2, R7), - BPF_ALU64_REG(BPF_ADD, R2, R8), - BPF_ALU64_REG(BPF_ADD, R2, R9), /* R2 == 1358 */ - BPF_JMP_IMM(BPF_JEQ, R2, 1358, 1), - BPF_EXIT_INSN(), - BPF_ALU64_REG(BPF_ADD, R3, R0), - BPF_ALU64_REG(BPF_ADD, R3, R1), - BPF_ALU64_REG(BPF_ADD, R3, R2), - BPF_ALU64_REG(BPF_ADD, R3, R3), - BPF_ALU64_REG(BPF_ADD, R3, R4), - BPF_ALU64_REG(BPF_ADD, R3, R5), - BPF_ALU64_REG(BPF_ADD, R3, R6), - BPF_ALU64_REG(BPF_ADD, R3, R7), - BPF_ALU64_REG(BPF_ADD, R3, R8), - BPF_ALU64_REG(BPF_ADD, R3, R9), /* R3 == 4063 */ - BPF_JMP_IMM(BPF_JEQ, R3, 4063, 1), - BPF_EXIT_INSN(), - BPF_ALU64_REG(BPF_ADD, R4, R0), - BPF_ALU64_REG(BPF_ADD, R4, R1), - BPF_ALU64_REG(BPF_ADD, R4, R2), - BPF_ALU64_REG(BPF_ADD, R4, R3), - BPF_ALU64_REG(BPF_ADD, R4, R4), - BPF_ALU64_REG(BPF_ADD, R4, R5), - BPF_ALU64_REG(BPF_ADD, R4, R6), - BPF_ALU64_REG(BPF_ADD, R4, R7), - BPF_ALU64_REG(BPF_ADD, R4, R8), - BPF_ALU64_REG(BPF_ADD, R4, R9), /* R4 == 12177 */ - BPF_JMP_IMM(BPF_JEQ, R4, 12177, 1), - BPF_EXIT_INSN(), - BPF_ALU64_REG(BPF_ADD, R5, R0), - BPF_ALU64_REG(BPF_ADD, R5, R1), - BPF_ALU64_REG(BPF_ADD, R5, R2), - BPF_ALU64_REG(BPF_ADD, R5, R3), - BPF_ALU64_REG(BPF_ADD, R5, R4), - BPF_ALU64_REG(BPF_ADD, R5, R5), - BPF_ALU64_REG(BPF_ADD, R5, R6), - BPF_ALU64_REG(BPF_ADD, R5, R7), - BPF_ALU64_REG(BPF_ADD, R5, R8), - BPF_ALU64_REG(BPF_ADD, R5, R9), /* R5 == 36518 */ - BPF_JMP_IMM(BPF_JEQ, R5, 36518, 1), - BPF_EXIT_INSN(), - BPF_ALU64_REG(BPF_ADD, R6, R0), - BPF_ALU64_REG(BPF_ADD, R6, R1), - BPF_ALU64_REG(BPF_ADD, R6, R2), - BPF_ALU64_REG(BPF_ADD, R6, R3), - BPF_ALU64_REG(BPF_ADD, R6, R4), - BPF_ALU64_REG(BPF_ADD, R6, R5), - BPF_ALU64_REG(BPF_ADD, R6, R6), - BPF_ALU64_REG(BPF_ADD, R6, R7), - BPF_ALU64_REG(BPF_ADD, R6, R8), - BPF_ALU64_REG(BPF_ADD, R6, R9), /* R6 == 109540 */ - BPF_JMP_IMM(BPF_JEQ, R6, 109540, 1), - BPF_EXIT_INSN(), - BPF_ALU64_REG(BPF_ADD, R7, R0), - BPF_ALU64_REG(BPF_ADD, R7, R1), - BPF_ALU64_REG(BPF_ADD, R7, R2), - BPF_ALU64_REG(BPF_ADD, R7, R3), - BPF_ALU64_REG(BPF_ADD, R7, R4), - BPF_ALU64_REG(BPF_ADD, R7, R5), - BPF_ALU64_REG(BPF_ADD, R7, R6), - BPF_ALU64_REG(BPF_ADD, R7, R7), - BPF_ALU64_REG(BPF_ADD, R7, R8), - BPF_ALU64_REG(BPF_ADD, R7, R9), /* R7 == 328605 */ - BPF_JMP_IMM(BPF_JEQ, R7, 328605, 1), - BPF_EXIT_INSN(), - BPF_ALU64_REG(BPF_ADD, R8, R0), - BPF_ALU64_REG(BPF_ADD, R8, R1), - BPF_ALU64_REG(BPF_ADD, R8, R2), - BPF_ALU64_REG(BPF_ADD, R8, R3), - BPF_ALU64_REG(BPF_ADD, R8, R4), - BPF_ALU64_REG(BPF_ADD, R8, R5), - BPF_ALU64_REG(BPF_ADD, R8, R6), - BPF_ALU64_REG(BPF_ADD, R8, R7), - BPF_ALU64_REG(BPF_ADD, R8, R8), - BPF_ALU64_REG(BPF_ADD, R8, R9), /* R8 == 985799 */ - BPF_JMP_IMM(BPF_JEQ, R8, 985799, 1), - BPF_EXIT_INSN(), - BPF_ALU64_REG(BPF_ADD, R9, R0), - BPF_ALU64_REG(BPF_ADD, R9, R1), - BPF_ALU64_REG(BPF_ADD, R9, R2), - BPF_ALU64_REG(BPF_ADD, R9, R3), - BPF_ALU64_REG(BPF_ADD, R9, R4), - BPF_ALU64_REG(BPF_ADD, R9, R5), - BPF_ALU64_REG(BPF_ADD, R9, R6), - BPF_ALU64_REG(BPF_ADD, R9, R7), - BPF_ALU64_REG(BPF_ADD, R9, R8), - BPF_ALU64_REG(BPF_ADD, R9, R9), /* R9 == 2957380 */ - BPF_ALU64_REG(BPF_MOV, R0, R9), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 2957380 } } - }, - { - "INT: ADD 32-bit", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 20), - BPF_ALU32_IMM(BPF_MOV, R1, 1), - BPF_ALU32_IMM(BPF_MOV, R2, 2), - BPF_ALU32_IMM(BPF_MOV, R3, 3), - BPF_ALU32_IMM(BPF_MOV, R4, 4), - BPF_ALU32_IMM(BPF_MOV, R5, 5), - BPF_ALU32_IMM(BPF_MOV, R6, 6), - BPF_ALU32_IMM(BPF_MOV, R7, 7), - BPF_ALU32_IMM(BPF_MOV, R8, 8), - BPF_ALU32_IMM(BPF_MOV, R9, 9), - BPF_ALU64_IMM(BPF_ADD, R1, 10), - BPF_ALU64_IMM(BPF_ADD, R2, 10), - BPF_ALU64_IMM(BPF_ADD, R3, 10), - BPF_ALU64_IMM(BPF_ADD, R4, 10), - BPF_ALU64_IMM(BPF_ADD, R5, 10), - BPF_ALU64_IMM(BPF_ADD, R6, 10), - BPF_ALU64_IMM(BPF_ADD, R7, 10), - BPF_ALU64_IMM(BPF_ADD, R8, 10), - BPF_ALU64_IMM(BPF_ADD, R9, 10), - BPF_ALU32_REG(BPF_ADD, R0, R1), - BPF_ALU32_REG(BPF_ADD, R0, R2), - BPF_ALU32_REG(BPF_ADD, R0, R3), - BPF_ALU32_REG(BPF_ADD, R0, R4), - BPF_ALU32_REG(BPF_ADD, R0, R5), - BPF_ALU32_REG(BPF_ADD, R0, R6), - BPF_ALU32_REG(BPF_ADD, R0, R7), - BPF_ALU32_REG(BPF_ADD, R0, R8), - BPF_ALU32_REG(BPF_ADD, R0, R9), /* R0 == 155 */ - BPF_JMP_IMM(BPF_JEQ, R0, 155, 1), - BPF_EXIT_INSN(), - BPF_ALU32_REG(BPF_ADD, R1, R0), - BPF_ALU32_REG(BPF_ADD, R1, R1), - BPF_ALU32_REG(BPF_ADD, R1, R2), - BPF_ALU32_REG(BPF_ADD, R1, R3), - BPF_ALU32_REG(BPF_ADD, R1, R4), - BPF_ALU32_REG(BPF_ADD, R1, R5), - BPF_ALU32_REG(BPF_ADD, R1, R6), - BPF_ALU32_REG(BPF_ADD, R1, R7), - BPF_ALU32_REG(BPF_ADD, R1, R8), - BPF_ALU32_REG(BPF_ADD, R1, R9), /* R1 == 456 */ - BPF_JMP_IMM(BPF_JEQ, R1, 456, 1), - BPF_EXIT_INSN(), - BPF_ALU32_REG(BPF_ADD, R2, R0), - BPF_ALU32_REG(BPF_ADD, R2, R1), - BPF_ALU32_REG(BPF_ADD, R2, R2), - BPF_ALU32_REG(BPF_ADD, R2, R3), - BPF_ALU32_REG(BPF_ADD, R2, R4), - BPF_ALU32_REG(BPF_ADD, R2, R5), - BPF_ALU32_REG(BPF_ADD, R2, R6), - BPF_ALU32_REG(BPF_ADD, R2, R7), - BPF_ALU32_REG(BPF_ADD, R2, R8), - BPF_ALU32_REG(BPF_ADD, R2, R9), /* R2 == 1358 */ - BPF_JMP_IMM(BPF_JEQ, R2, 1358, 1), - BPF_EXIT_INSN(), - BPF_ALU32_REG(BPF_ADD, R3, R0), - BPF_ALU32_REG(BPF_ADD, R3, R1), - BPF_ALU32_REG(BPF_ADD, R3, R2), - BPF_ALU32_REG(BPF_ADD, R3, R3), - BPF_ALU32_REG(BPF_ADD, R3, R4), - BPF_ALU32_REG(BPF_ADD, R3, R5), - BPF_ALU32_REG(BPF_ADD, R3, R6), - BPF_ALU32_REG(BPF_ADD, R3, R7), - BPF_ALU32_REG(BPF_ADD, R3, R8), - BPF_ALU32_REG(BPF_ADD, R3, R9), /* R3 == 4063 */ - BPF_JMP_IMM(BPF_JEQ, R3, 4063, 1), - BPF_EXIT_INSN(), - BPF_ALU32_REG(BPF_ADD, R4, R0), - BPF_ALU32_REG(BPF_ADD, R4, R1), - BPF_ALU32_REG(BPF_ADD, R4, R2), - BPF_ALU32_REG(BPF_ADD, R4, R3), - BPF_ALU32_REG(BPF_ADD, R4, R4), - BPF_ALU32_REG(BPF_ADD, R4, R5), - BPF_ALU32_REG(BPF_ADD, R4, R6), - BPF_ALU32_REG(BPF_ADD, R4, R7), - BPF_ALU32_REG(BPF_ADD, R4, R8), - BPF_ALU32_REG(BPF_ADD, R4, R9), /* R4 == 12177 */ - BPF_JMP_IMM(BPF_JEQ, R4, 12177, 1), - BPF_EXIT_INSN(), - BPF_ALU32_REG(BPF_ADD, R5, R0), - BPF_ALU32_REG(BPF_ADD, R5, R1), - BPF_ALU32_REG(BPF_ADD, R5, R2), - BPF_ALU32_REG(BPF_ADD, R5, R3), - BPF_ALU32_REG(BPF_ADD, R5, R4), - BPF_ALU32_REG(BPF_ADD, R5, R5), - BPF_ALU32_REG(BPF_ADD, R5, R6), - BPF_ALU32_REG(BPF_ADD, R5, R7), - BPF_ALU32_REG(BPF_ADD, R5, R8), - BPF_ALU32_REG(BPF_ADD, R5, R9), /* R5 == 36518 */ - BPF_JMP_IMM(BPF_JEQ, R5, 36518, 1), - BPF_EXIT_INSN(), - BPF_ALU32_REG(BPF_ADD, R6, R0), - BPF_ALU32_REG(BPF_ADD, R6, R1), - BPF_ALU32_REG(BPF_ADD, R6, R2), - BPF_ALU32_REG(BPF_ADD, R6, R3), - BPF_ALU32_REG(BPF_ADD, R6, R4), - BPF_ALU32_REG(BPF_ADD, R6, R5), - BPF_ALU32_REG(BPF_ADD, R6, R6), - BPF_ALU32_REG(BPF_ADD, R6, R7), - BPF_ALU32_REG(BPF_ADD, R6, R8), - BPF_ALU32_REG(BPF_ADD, R6, R9), /* R6 == 109540 */ - BPF_JMP_IMM(BPF_JEQ, R6, 109540, 1), - BPF_EXIT_INSN(), - BPF_ALU32_REG(BPF_ADD, R7, R0), - BPF_ALU32_REG(BPF_ADD, R7, R1), - BPF_ALU32_REG(BPF_ADD, R7, R2), - BPF_ALU32_REG(BPF_ADD, R7, R3), - BPF_ALU32_REG(BPF_ADD, R7, R4), - BPF_ALU32_REG(BPF_ADD, R7, R5), - BPF_ALU32_REG(BPF_ADD, R7, R6), - BPF_ALU32_REG(BPF_ADD, R7, R7), - BPF_ALU32_REG(BPF_ADD, R7, R8), - BPF_ALU32_REG(BPF_ADD, R7, R9), /* R7 == 328605 */ - BPF_JMP_IMM(BPF_JEQ, R7, 328605, 1), - BPF_EXIT_INSN(), - BPF_ALU32_REG(BPF_ADD, R8, R0), - BPF_ALU32_REG(BPF_ADD, R8, R1), - BPF_ALU32_REG(BPF_ADD, R8, R2), - BPF_ALU32_REG(BPF_ADD, R8, R3), - BPF_ALU32_REG(BPF_ADD, R8, R4), - BPF_ALU32_REG(BPF_ADD, R8, R5), - BPF_ALU32_REG(BPF_ADD, R8, R6), - BPF_ALU32_REG(BPF_ADD, R8, R7), - BPF_ALU32_REG(BPF_ADD, R8, R8), - BPF_ALU32_REG(BPF_ADD, R8, R9), /* R8 == 985799 */ - BPF_JMP_IMM(BPF_JEQ, R8, 985799, 1), - BPF_EXIT_INSN(), - BPF_ALU32_REG(BPF_ADD, R9, R0), - BPF_ALU32_REG(BPF_ADD, R9, R1), - BPF_ALU32_REG(BPF_ADD, R9, R2), - BPF_ALU32_REG(BPF_ADD, R9, R3), - BPF_ALU32_REG(BPF_ADD, R9, R4), - BPF_ALU32_REG(BPF_ADD, R9, R5), - BPF_ALU32_REG(BPF_ADD, R9, R6), - BPF_ALU32_REG(BPF_ADD, R9, R7), - BPF_ALU32_REG(BPF_ADD, R9, R8), - BPF_ALU32_REG(BPF_ADD, R9, R9), /* R9 == 2957380 */ - BPF_ALU32_REG(BPF_MOV, R0, R9), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 2957380 } } - }, - { /* Mainly checking JIT here. */ - "INT: SUB", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_ALU64_IMM(BPF_MOV, R1, 1), - BPF_ALU64_IMM(BPF_MOV, R2, 2), - BPF_ALU64_IMM(BPF_MOV, R3, 3), - BPF_ALU64_IMM(BPF_MOV, R4, 4), - BPF_ALU64_IMM(BPF_MOV, R5, 5), - BPF_ALU64_IMM(BPF_MOV, R6, 6), - BPF_ALU64_IMM(BPF_MOV, R7, 7), - BPF_ALU64_IMM(BPF_MOV, R8, 8), - BPF_ALU64_IMM(BPF_MOV, R9, 9), - BPF_ALU64_REG(BPF_SUB, R0, R0), - BPF_ALU64_REG(BPF_SUB, R0, R1), - BPF_ALU64_REG(BPF_SUB, R0, R2), - BPF_ALU64_REG(BPF_SUB, R0, R3), - BPF_ALU64_REG(BPF_SUB, R0, R4), - BPF_ALU64_REG(BPF_SUB, R0, R5), - BPF_ALU64_REG(BPF_SUB, R0, R6), - BPF_ALU64_REG(BPF_SUB, R0, R7), - BPF_ALU64_REG(BPF_SUB, R0, R8), - BPF_ALU64_REG(BPF_SUB, R0, R9), - BPF_ALU64_IMM(BPF_SUB, R0, 10), - BPF_JMP_IMM(BPF_JEQ, R0, -55, 1), - BPF_EXIT_INSN(), - BPF_ALU64_REG(BPF_SUB, R1, R0), - BPF_ALU64_REG(BPF_SUB, R1, R2), - BPF_ALU64_REG(BPF_SUB, R1, R3), - BPF_ALU64_REG(BPF_SUB, R1, R4), - BPF_ALU64_REG(BPF_SUB, R1, R5), - BPF_ALU64_REG(BPF_SUB, R1, R6), - BPF_ALU64_REG(BPF_SUB, R1, R7), - BPF_ALU64_REG(BPF_SUB, R1, R8), - BPF_ALU64_REG(BPF_SUB, R1, R9), - BPF_ALU64_IMM(BPF_SUB, R1, 10), - BPF_ALU64_REG(BPF_SUB, R2, R0), - BPF_ALU64_REG(BPF_SUB, R2, R1), - BPF_ALU64_REG(BPF_SUB, R2, R3), - BPF_ALU64_REG(BPF_SUB, R2, R4), - BPF_ALU64_REG(BPF_SUB, R2, R5), - BPF_ALU64_REG(BPF_SUB, R2, R6), - BPF_ALU64_REG(BPF_SUB, R2, R7), - BPF_ALU64_REG(BPF_SUB, R2, R8), - BPF_ALU64_REG(BPF_SUB, R2, R9), - BPF_ALU64_IMM(BPF_SUB, R2, 10), - BPF_ALU64_REG(BPF_SUB, R3, R0), - BPF_ALU64_REG(BPF_SUB, R3, R1), - BPF_ALU64_REG(BPF_SUB, R3, R2), - BPF_ALU64_REG(BPF_SUB, R3, R4), - BPF_ALU64_REG(BPF_SUB, R3, R5), - BPF_ALU64_REG(BPF_SUB, R3, R6), - BPF_ALU64_REG(BPF_SUB, R3, R7), - BPF_ALU64_REG(BPF_SUB, R3, R8), - BPF_ALU64_REG(BPF_SUB, R3, R9), - BPF_ALU64_IMM(BPF_SUB, R3, 10), - BPF_ALU64_REG(BPF_SUB, R4, R0), - BPF_ALU64_REG(BPF_SUB, R4, R1), - BPF_ALU64_REG(BPF_SUB, R4, R2), - BPF_ALU64_REG(BPF_SUB, R4, R3), - BPF_ALU64_REG(BPF_SUB, R4, R5), - BPF_ALU64_REG(BPF_SUB, R4, R6), - BPF_ALU64_REG(BPF_SUB, R4, R7), - BPF_ALU64_REG(BPF_SUB, R4, R8), - BPF_ALU64_REG(BPF_SUB, R4, R9), - BPF_ALU64_IMM(BPF_SUB, R4, 10), - BPF_ALU64_REG(BPF_SUB, R5, R0), - BPF_ALU64_REG(BPF_SUB, R5, R1), - BPF_ALU64_REG(BPF_SUB, R5, R2), - BPF_ALU64_REG(BPF_SUB, R5, R3), - BPF_ALU64_REG(BPF_SUB, R5, R4), - BPF_ALU64_REG(BPF_SUB, R5, R6), - BPF_ALU64_REG(BPF_SUB, R5, R7), - BPF_ALU64_REG(BPF_SUB, R5, R8), - BPF_ALU64_REG(BPF_SUB, R5, R9), - BPF_ALU64_IMM(BPF_SUB, R5, 10), - BPF_ALU64_REG(BPF_SUB, R6, R0), - BPF_ALU64_REG(BPF_SUB, R6, R1), - BPF_ALU64_REG(BPF_SUB, R6, R2), - BPF_ALU64_REG(BPF_SUB, R6, R3), - BPF_ALU64_REG(BPF_SUB, R6, R4), - BPF_ALU64_REG(BPF_SUB, R6, R5), - BPF_ALU64_REG(BPF_SUB, R6, R7), - BPF_ALU64_REG(BPF_SUB, R6, R8), - BPF_ALU64_REG(BPF_SUB, R6, R9), - BPF_ALU64_IMM(BPF_SUB, R6, 10), - BPF_ALU64_REG(BPF_SUB, R7, R0), - BPF_ALU64_REG(BPF_SUB, R7, R1), - BPF_ALU64_REG(BPF_SUB, R7, R2), - BPF_ALU64_REG(BPF_SUB, R7, R3), - BPF_ALU64_REG(BPF_SUB, R7, R4), - BPF_ALU64_REG(BPF_SUB, R7, R5), - BPF_ALU64_REG(BPF_SUB, R7, R6), - BPF_ALU64_REG(BPF_SUB, R7, R8), - BPF_ALU64_REG(BPF_SUB, R7, R9), - BPF_ALU64_IMM(BPF_SUB, R7, 10), - BPF_ALU64_REG(BPF_SUB, R8, R0), - BPF_ALU64_REG(BPF_SUB, R8, R1), - BPF_ALU64_REG(BPF_SUB, R8, R2), - BPF_ALU64_REG(BPF_SUB, R8, R3), - BPF_ALU64_REG(BPF_SUB, R8, R4), - BPF_ALU64_REG(BPF_SUB, R8, R5), - BPF_ALU64_REG(BPF_SUB, R8, R6), - BPF_ALU64_REG(BPF_SUB, R8, R7), - BPF_ALU64_REG(BPF_SUB, R8, R9), - BPF_ALU64_IMM(BPF_SUB, R8, 10), - BPF_ALU64_REG(BPF_SUB, R9, R0), - BPF_ALU64_REG(BPF_SUB, R9, R1), - BPF_ALU64_REG(BPF_SUB, R9, R2), - BPF_ALU64_REG(BPF_SUB, R9, R3), - BPF_ALU64_REG(BPF_SUB, R9, R4), - BPF_ALU64_REG(BPF_SUB, R9, R5), - BPF_ALU64_REG(BPF_SUB, R9, R6), - BPF_ALU64_REG(BPF_SUB, R9, R7), - BPF_ALU64_REG(BPF_SUB, R9, R8), - BPF_ALU64_IMM(BPF_SUB, R9, 10), - BPF_ALU64_IMM(BPF_SUB, R0, 10), - BPF_ALU64_IMM(BPF_NEG, R0, 0), - BPF_ALU64_REG(BPF_SUB, R0, R1), - BPF_ALU64_REG(BPF_SUB, R0, R2), - BPF_ALU64_REG(BPF_SUB, R0, R3), - BPF_ALU64_REG(BPF_SUB, R0, R4), - BPF_ALU64_REG(BPF_SUB, R0, R5), - BPF_ALU64_REG(BPF_SUB, R0, R6), - BPF_ALU64_REG(BPF_SUB, R0, R7), - BPF_ALU64_REG(BPF_SUB, R0, R8), - BPF_ALU64_REG(BPF_SUB, R0, R9), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 11 } } - }, - { /* Mainly checking JIT here. */ - "INT: XOR", - .u.insns_int = { - BPF_ALU64_REG(BPF_SUB, R0, R0), - BPF_ALU64_REG(BPF_XOR, R1, R1), - BPF_JMP_REG(BPF_JEQ, R0, R1, 1), - BPF_EXIT_INSN(), - BPF_ALU64_IMM(BPF_MOV, R0, 10), - BPF_ALU64_IMM(BPF_MOV, R1, -1), - BPF_ALU64_REG(BPF_SUB, R1, R1), - BPF_ALU64_REG(BPF_XOR, R2, R2), - BPF_JMP_REG(BPF_JEQ, R1, R2, 1), - BPF_EXIT_INSN(), - BPF_ALU64_REG(BPF_SUB, R2, R2), - BPF_ALU64_REG(BPF_XOR, R3, R3), - BPF_ALU64_IMM(BPF_MOV, R0, 10), - BPF_ALU64_IMM(BPF_MOV, R1, -1), - BPF_JMP_REG(BPF_JEQ, R2, R3, 1), - BPF_EXIT_INSN(), - BPF_ALU64_REG(BPF_SUB, R3, R3), - BPF_ALU64_REG(BPF_XOR, R4, R4), - BPF_ALU64_IMM(BPF_MOV, R2, 1), - BPF_ALU64_IMM(BPF_MOV, R5, -1), - BPF_JMP_REG(BPF_JEQ, R3, R4, 1), - BPF_EXIT_INSN(), - BPF_ALU64_REG(BPF_SUB, R4, R4), - BPF_ALU64_REG(BPF_XOR, R5, R5), - BPF_ALU64_IMM(BPF_MOV, R3, 1), - BPF_ALU64_IMM(BPF_MOV, R7, -1), - BPF_JMP_REG(BPF_JEQ, R5, R4, 1), - BPF_EXIT_INSN(), - BPF_ALU64_IMM(BPF_MOV, R5, 1), - BPF_ALU64_REG(BPF_SUB, R5, R5), - BPF_ALU64_REG(BPF_XOR, R6, R6), - BPF_ALU64_IMM(BPF_MOV, R1, 1), - BPF_ALU64_IMM(BPF_MOV, R8, -1), - BPF_JMP_REG(BPF_JEQ, R5, R6, 1), - BPF_EXIT_INSN(), - BPF_ALU64_REG(BPF_SUB, R6, R6), - BPF_ALU64_REG(BPF_XOR, R7, R7), - BPF_JMP_REG(BPF_JEQ, R7, R6, 1), - BPF_EXIT_INSN(), - BPF_ALU64_REG(BPF_SUB, R7, R7), - BPF_ALU64_REG(BPF_XOR, R8, R8), - BPF_JMP_REG(BPF_JEQ, R7, R8, 1), - BPF_EXIT_INSN(), - BPF_ALU64_REG(BPF_SUB, R8, R8), - BPF_ALU64_REG(BPF_XOR, R9, R9), - BPF_JMP_REG(BPF_JEQ, R9, R8, 1), - BPF_EXIT_INSN(), - BPF_ALU64_REG(BPF_SUB, R9, R9), - BPF_ALU64_REG(BPF_XOR, R0, R0), - BPF_JMP_REG(BPF_JEQ, R9, R0, 1), - BPF_EXIT_INSN(), - BPF_ALU64_REG(BPF_SUB, R1, R1), - BPF_ALU64_REG(BPF_XOR, R0, R0), - BPF_JMP_REG(BPF_JEQ, R9, R0, 2), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - BPF_ALU64_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } } - }, - { /* Mainly checking JIT here. */ - "INT: MUL", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 11), - BPF_ALU64_IMM(BPF_MOV, R1, 1), - BPF_ALU64_IMM(BPF_MOV, R2, 2), - BPF_ALU64_IMM(BPF_MOV, R3, 3), - BPF_ALU64_IMM(BPF_MOV, R4, 4), - BPF_ALU64_IMM(BPF_MOV, R5, 5), - BPF_ALU64_IMM(BPF_MOV, R6, 6), - BPF_ALU64_IMM(BPF_MOV, R7, 7), - BPF_ALU64_IMM(BPF_MOV, R8, 8), - BPF_ALU64_IMM(BPF_MOV, R9, 9), - BPF_ALU64_REG(BPF_MUL, R0, R0), - BPF_ALU64_REG(BPF_MUL, R0, R1), - BPF_ALU64_REG(BPF_MUL, R0, R2), - BPF_ALU64_REG(BPF_MUL, R0, R3), - BPF_ALU64_REG(BPF_MUL, R0, R4), - BPF_ALU64_REG(BPF_MUL, R0, R5), - BPF_ALU64_REG(BPF_MUL, R0, R6), - BPF_ALU64_REG(BPF_MUL, R0, R7), - BPF_ALU64_REG(BPF_MUL, R0, R8), - BPF_ALU64_REG(BPF_MUL, R0, R9), - BPF_ALU64_IMM(BPF_MUL, R0, 10), - BPF_JMP_IMM(BPF_JEQ, R0, 439084800, 1), - BPF_EXIT_INSN(), - BPF_ALU64_REG(BPF_MUL, R1, R0), - BPF_ALU64_REG(BPF_MUL, R1, R2), - BPF_ALU64_REG(BPF_MUL, R1, R3), - BPF_ALU64_REG(BPF_MUL, R1, R4), - BPF_ALU64_REG(BPF_MUL, R1, R5), - BPF_ALU64_REG(BPF_MUL, R1, R6), - BPF_ALU64_REG(BPF_MUL, R1, R7), - BPF_ALU64_REG(BPF_MUL, R1, R8), - BPF_ALU64_REG(BPF_MUL, R1, R9), - BPF_ALU64_IMM(BPF_MUL, R1, 10), - BPF_ALU64_REG(BPF_MOV, R2, R1), - BPF_ALU64_IMM(BPF_RSH, R2, 32), - BPF_JMP_IMM(BPF_JEQ, R2, 0x5a924, 1), - BPF_EXIT_INSN(), - BPF_ALU64_IMM(BPF_LSH, R1, 32), - BPF_ALU64_IMM(BPF_ARSH, R1, 32), - BPF_JMP_IMM(BPF_JEQ, R1, 0xebb90000, 1), - BPF_EXIT_INSN(), - BPF_ALU64_REG(BPF_MUL, R2, R0), - BPF_ALU64_REG(BPF_MUL, R2, R1), - BPF_ALU64_REG(BPF_MUL, R2, R3), - BPF_ALU64_REG(BPF_MUL, R2, R4), - BPF_ALU64_REG(BPF_MUL, R2, R5), - BPF_ALU64_REG(BPF_MUL, R2, R6), - BPF_ALU64_REG(BPF_MUL, R2, R7), - BPF_ALU64_REG(BPF_MUL, R2, R8), - BPF_ALU64_REG(BPF_MUL, R2, R9), - BPF_ALU64_IMM(BPF_MUL, R2, 10), - BPF_ALU64_IMM(BPF_RSH, R2, 32), - BPF_ALU64_REG(BPF_MOV, R0, R2), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x35d97ef2 } } - }, - { /* Mainly checking JIT here. */ - "MOV REG64", - .u.insns_int = { - BPF_LD_IMM64(R0, 0xffffffffffffffffLL), - BPF_MOV64_REG(R1, R0), - BPF_MOV64_REG(R2, R1), - BPF_MOV64_REG(R3, R2), - BPF_MOV64_REG(R4, R3), - BPF_MOV64_REG(R5, R4), - BPF_MOV64_REG(R6, R5), - BPF_MOV64_REG(R7, R6), - BPF_MOV64_REG(R8, R7), - BPF_MOV64_REG(R9, R8), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_ALU64_IMM(BPF_MOV, R1, 0), - BPF_ALU64_IMM(BPF_MOV, R2, 0), - BPF_ALU64_IMM(BPF_MOV, R3, 0), - BPF_ALU64_IMM(BPF_MOV, R4, 0), - BPF_ALU64_IMM(BPF_MOV, R5, 0), - BPF_ALU64_IMM(BPF_MOV, R6, 0), - BPF_ALU64_IMM(BPF_MOV, R7, 0), - BPF_ALU64_IMM(BPF_MOV, R8, 0), - BPF_ALU64_IMM(BPF_MOV, R9, 0), - BPF_ALU64_REG(BPF_ADD, R0, R0), - BPF_ALU64_REG(BPF_ADD, R0, R1), - BPF_ALU64_REG(BPF_ADD, R0, R2), - BPF_ALU64_REG(BPF_ADD, R0, R3), - BPF_ALU64_REG(BPF_ADD, R0, R4), - BPF_ALU64_REG(BPF_ADD, R0, R5), - BPF_ALU64_REG(BPF_ADD, R0, R6), - BPF_ALU64_REG(BPF_ADD, R0, R7), - BPF_ALU64_REG(BPF_ADD, R0, R8), - BPF_ALU64_REG(BPF_ADD, R0, R9), - BPF_ALU64_IMM(BPF_ADD, R0, 0xfefe), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xfefe } } - }, - { /* Mainly checking JIT here. */ - "MOV REG32", - .u.insns_int = { - BPF_LD_IMM64(R0, 0xffffffffffffffffLL), - BPF_MOV64_REG(R1, R0), - BPF_MOV64_REG(R2, R1), - BPF_MOV64_REG(R3, R2), - BPF_MOV64_REG(R4, R3), - BPF_MOV64_REG(R5, R4), - BPF_MOV64_REG(R6, R5), - BPF_MOV64_REG(R7, R6), - BPF_MOV64_REG(R8, R7), - BPF_MOV64_REG(R9, R8), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_ALU32_IMM(BPF_MOV, R1, 0), - BPF_ALU32_IMM(BPF_MOV, R2, 0), - BPF_ALU32_IMM(BPF_MOV, R3, 0), - BPF_ALU32_IMM(BPF_MOV, R4, 0), - BPF_ALU32_IMM(BPF_MOV, R5, 0), - BPF_ALU32_IMM(BPF_MOV, R6, 0), - BPF_ALU32_IMM(BPF_MOV, R7, 0), - BPF_ALU32_IMM(BPF_MOV, R8, 0), - BPF_ALU32_IMM(BPF_MOV, R9, 0), - BPF_ALU64_REG(BPF_ADD, R0, R0), - BPF_ALU64_REG(BPF_ADD, R0, R1), - BPF_ALU64_REG(BPF_ADD, R0, R2), - BPF_ALU64_REG(BPF_ADD, R0, R3), - BPF_ALU64_REG(BPF_ADD, R0, R4), - BPF_ALU64_REG(BPF_ADD, R0, R5), - BPF_ALU64_REG(BPF_ADD, R0, R6), - BPF_ALU64_REG(BPF_ADD, R0, R7), - BPF_ALU64_REG(BPF_ADD, R0, R8), - BPF_ALU64_REG(BPF_ADD, R0, R9), - BPF_ALU64_IMM(BPF_ADD, R0, 0xfefe), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xfefe } } - }, - { /* Mainly checking JIT here. */ - "LD IMM64", - .u.insns_int = { - BPF_LD_IMM64(R0, 0xffffffffffffffffLL), - BPF_MOV64_REG(R1, R0), - BPF_MOV64_REG(R2, R1), - BPF_MOV64_REG(R3, R2), - BPF_MOV64_REG(R4, R3), - BPF_MOV64_REG(R5, R4), - BPF_MOV64_REG(R6, R5), - BPF_MOV64_REG(R7, R6), - BPF_MOV64_REG(R8, R7), - BPF_MOV64_REG(R9, R8), - BPF_LD_IMM64(R0, 0x0LL), - BPF_LD_IMM64(R1, 0x0LL), - BPF_LD_IMM64(R2, 0x0LL), - BPF_LD_IMM64(R3, 0x0LL), - BPF_LD_IMM64(R4, 0x0LL), - BPF_LD_IMM64(R5, 0x0LL), - BPF_LD_IMM64(R6, 0x0LL), - BPF_LD_IMM64(R7, 0x0LL), - BPF_LD_IMM64(R8, 0x0LL), - BPF_LD_IMM64(R9, 0x0LL), - BPF_ALU64_REG(BPF_ADD, R0, R0), - BPF_ALU64_REG(BPF_ADD, R0, R1), - BPF_ALU64_REG(BPF_ADD, R0, R2), - BPF_ALU64_REG(BPF_ADD, R0, R3), - BPF_ALU64_REG(BPF_ADD, R0, R4), - BPF_ALU64_REG(BPF_ADD, R0, R5), - BPF_ALU64_REG(BPF_ADD, R0, R6), - BPF_ALU64_REG(BPF_ADD, R0, R7), - BPF_ALU64_REG(BPF_ADD, R0, R8), - BPF_ALU64_REG(BPF_ADD, R0, R9), - BPF_ALU64_IMM(BPF_ADD, R0, 0xfefe), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xfefe } } - }, - { - "INT: ALU MIX", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 11), - BPF_ALU64_IMM(BPF_ADD, R0, -1), - BPF_ALU64_IMM(BPF_MOV, R2, 2), - BPF_ALU64_IMM(BPF_XOR, R2, 3), - BPF_ALU64_REG(BPF_DIV, R0, R2), - BPF_JMP_IMM(BPF_JEQ, R0, 10, 1), - BPF_EXIT_INSN(), - BPF_ALU64_IMM(BPF_MOD, R0, 3), - BPF_JMP_IMM(BPF_JEQ, R0, 1, 1), - BPF_EXIT_INSN(), - BPF_ALU64_IMM(BPF_MOV, R0, -1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -1 } } - }, - { - "INT: shifts by register", - .u.insns_int = { - BPF_MOV64_IMM(R0, -1234), - BPF_MOV64_IMM(R1, 1), - BPF_ALU32_REG(BPF_RSH, R0, R1), - BPF_JMP_IMM(BPF_JEQ, R0, 0x7ffffd97, 1), - BPF_EXIT_INSN(), - BPF_MOV64_IMM(R2, 1), - BPF_ALU64_REG(BPF_LSH, R0, R2), - BPF_MOV32_IMM(R4, -1234), - BPF_JMP_REG(BPF_JEQ, R0, R4, 1), - BPF_EXIT_INSN(), - BPF_ALU64_IMM(BPF_AND, R4, 63), - BPF_ALU64_REG(BPF_LSH, R0, R4), /* R0 <= 46 */ - BPF_MOV64_IMM(R3, 47), - BPF_ALU64_REG(BPF_ARSH, R0, R3), - BPF_JMP_IMM(BPF_JEQ, R0, -617, 1), - BPF_EXIT_INSN(), - BPF_MOV64_IMM(R2, 1), - BPF_ALU64_REG(BPF_LSH, R4, R2), /* R4 = 46 << 1 */ - BPF_JMP_IMM(BPF_JEQ, R4, 92, 1), - BPF_EXIT_INSN(), - BPF_MOV64_IMM(R4, 4), - BPF_ALU64_REG(BPF_LSH, R4, R4), /* R4 = 4 << 4 */ - BPF_JMP_IMM(BPF_JEQ, R4, 64, 1), - BPF_EXIT_INSN(), - BPF_MOV64_IMM(R4, 5), - BPF_ALU32_REG(BPF_LSH, R4, R4), /* R4 = 5 << 5 */ - BPF_JMP_IMM(BPF_JEQ, R4, 160, 1), - BPF_EXIT_INSN(), - BPF_MOV64_IMM(R0, -1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -1 } } - }, -#ifdef CONFIG_32BIT - { - "INT: 32-bit context pointer word order and zero-extension", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_JMP32_IMM(BPF_JEQ, R1, 0, 3), - BPF_ALU64_IMM(BPF_RSH, R1, 32), - BPF_JMP32_IMM(BPF_JNE, R1, 0, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } } - }, -#endif - { - "check: missing ret", - .u.insns = { - BPF_STMT(BPF_LD | BPF_IMM, 1), - }, - CLASSIC | FLAG_NO_DATA | FLAG_EXPECTED_FAIL, - { }, - { }, - .fill_helper = NULL, - .expected_errcode = -EINVAL, - }, - { - "check: div_k_0", - .u.insns = { - BPF_STMT(BPF_ALU | BPF_DIV | BPF_K, 0), - BPF_STMT(BPF_RET | BPF_K, 0) - }, - CLASSIC | FLAG_NO_DATA | FLAG_EXPECTED_FAIL, - { }, - { }, - .fill_helper = NULL, - .expected_errcode = -EINVAL, - }, - { - "check: unknown insn", - .u.insns = { - /* seccomp insn, rejected in socket filter */ - BPF_STMT(BPF_LDX | BPF_W | BPF_ABS, 0), - BPF_STMT(BPF_RET | BPF_K, 0) - }, - CLASSIC | FLAG_EXPECTED_FAIL, - { }, - { }, - .fill_helper = NULL, - .expected_errcode = -EINVAL, - }, - { - "check: out of range spill/fill", - .u.insns = { - BPF_STMT(BPF_STX, 16), - BPF_STMT(BPF_RET | BPF_K, 0) - }, - CLASSIC | FLAG_NO_DATA | FLAG_EXPECTED_FAIL, - { }, - { }, - .fill_helper = NULL, - .expected_errcode = -EINVAL, - }, - { - "JUMPS + HOLES", - .u.insns = { - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_JUMP(BPF_JMP | BPF_JGE, 0, 13, 15), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_JUMP(BPF_JMP | BPF_JEQ, 0x90c2894d, 3, 4), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_JUMP(BPF_JMP | BPF_JEQ, 0x90c2894d, 1, 2), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_JUMP(BPF_JMP | BPF_JGE, 0, 14, 15), - BPF_JUMP(BPF_JMP | BPF_JGE, 0, 13, 14), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_JUMP(BPF_JMP | BPF_JEQ, 0x2ac28349, 2, 3), - BPF_JUMP(BPF_JMP | BPF_JEQ, 0x2ac28349, 1, 2), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_JUMP(BPF_JMP | BPF_JGE, 0, 14, 15), - BPF_JUMP(BPF_JMP | BPF_JGE, 0, 13, 14), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_JUMP(BPF_JMP | BPF_JEQ, 0x90d2ff41, 2, 3), - BPF_JUMP(BPF_JMP | BPF_JEQ, 0x90d2ff41, 1, 2), - BPF_STMT(BPF_LD | BPF_H | BPF_ABS, 0), - BPF_STMT(BPF_RET | BPF_A, 0), - BPF_STMT(BPF_RET | BPF_A, 0), - }, - CLASSIC, - { 0x00, 0x1b, 0x21, 0x3c, 0x9d, 0xf8, - 0x90, 0xe2, 0xba, 0x0a, 0x56, 0xb4, - 0x08, 0x00, - 0x45, 0x00, 0x00, 0x28, 0x00, 0x00, - 0x20, 0x00, 0x40, 0x11, 0x00, 0x00, /* IP header */ - 0xc0, 0xa8, 0x33, 0x01, - 0xc0, 0xa8, 0x33, 0x02, - 0xbb, 0xb6, - 0xa9, 0xfa, - 0x00, 0x14, 0x00, 0x00, - 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, - 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, - 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, - 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, - 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, - 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, - 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, - 0xcc, 0xcc, 0xcc, 0xcc }, - { { 88, 0x001b } } - }, - { - "check: RET X", - .u.insns = { - BPF_STMT(BPF_RET | BPF_X, 0), - }, - CLASSIC | FLAG_NO_DATA | FLAG_EXPECTED_FAIL, - { }, - { }, - .fill_helper = NULL, - .expected_errcode = -EINVAL, - }, - { - "check: LDX + RET X", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_IMM, 42), - BPF_STMT(BPF_RET | BPF_X, 0), - }, - CLASSIC | FLAG_NO_DATA | FLAG_EXPECTED_FAIL, - { }, - { }, - .fill_helper = NULL, - .expected_errcode = -EINVAL, - }, - { /* Mainly checking JIT here. */ - "M[]: alt STX + LDX", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_IMM, 100), - BPF_STMT(BPF_STX, 0), - BPF_STMT(BPF_LDX | BPF_MEM, 0), - BPF_STMT(BPF_MISC | BPF_TXA, 0), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_K, 1), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_STX, 1), - BPF_STMT(BPF_LDX | BPF_MEM, 1), - BPF_STMT(BPF_MISC | BPF_TXA, 0), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_K, 1), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_STX, 2), - BPF_STMT(BPF_LDX | BPF_MEM, 2), - BPF_STMT(BPF_MISC | BPF_TXA, 0), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_K, 1), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_STX, 3), - BPF_STMT(BPF_LDX | BPF_MEM, 3), - BPF_STMT(BPF_MISC | BPF_TXA, 0), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_K, 1), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_STX, 4), - BPF_STMT(BPF_LDX | BPF_MEM, 4), - BPF_STMT(BPF_MISC | BPF_TXA, 0), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_K, 1), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_STX, 5), - BPF_STMT(BPF_LDX | BPF_MEM, 5), - BPF_STMT(BPF_MISC | BPF_TXA, 0), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_K, 1), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_STX, 6), - BPF_STMT(BPF_LDX | BPF_MEM, 6), - BPF_STMT(BPF_MISC | BPF_TXA, 0), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_K, 1), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_STX, 7), - BPF_STMT(BPF_LDX | BPF_MEM, 7), - BPF_STMT(BPF_MISC | BPF_TXA, 0), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_K, 1), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_STX, 8), - BPF_STMT(BPF_LDX | BPF_MEM, 8), - BPF_STMT(BPF_MISC | BPF_TXA, 0), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_K, 1), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_STX, 9), - BPF_STMT(BPF_LDX | BPF_MEM, 9), - BPF_STMT(BPF_MISC | BPF_TXA, 0), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_K, 1), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_STX, 10), - BPF_STMT(BPF_LDX | BPF_MEM, 10), - BPF_STMT(BPF_MISC | BPF_TXA, 0), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_K, 1), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_STX, 11), - BPF_STMT(BPF_LDX | BPF_MEM, 11), - BPF_STMT(BPF_MISC | BPF_TXA, 0), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_K, 1), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_STX, 12), - BPF_STMT(BPF_LDX | BPF_MEM, 12), - BPF_STMT(BPF_MISC | BPF_TXA, 0), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_K, 1), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_STX, 13), - BPF_STMT(BPF_LDX | BPF_MEM, 13), - BPF_STMT(BPF_MISC | BPF_TXA, 0), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_K, 1), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_STX, 14), - BPF_STMT(BPF_LDX | BPF_MEM, 14), - BPF_STMT(BPF_MISC | BPF_TXA, 0), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_K, 1), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_STX, 15), - BPF_STMT(BPF_LDX | BPF_MEM, 15), - BPF_STMT(BPF_MISC | BPF_TXA, 0), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_K, 1), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_RET | BPF_A, 0), - }, - CLASSIC | FLAG_NO_DATA, - { }, - { { 0, 116 } }, - }, - { /* Mainly checking JIT here. */ - "M[]: full STX + full LDX", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_IMM, 0xbadfeedb), - BPF_STMT(BPF_STX, 0), - BPF_STMT(BPF_LDX | BPF_IMM, 0xecabedae), - BPF_STMT(BPF_STX, 1), - BPF_STMT(BPF_LDX | BPF_IMM, 0xafccfeaf), - BPF_STMT(BPF_STX, 2), - BPF_STMT(BPF_LDX | BPF_IMM, 0xbffdcedc), - BPF_STMT(BPF_STX, 3), - BPF_STMT(BPF_LDX | BPF_IMM, 0xfbbbdccb), - BPF_STMT(BPF_STX, 4), - BPF_STMT(BPF_LDX | BPF_IMM, 0xfbabcbda), - BPF_STMT(BPF_STX, 5), - BPF_STMT(BPF_LDX | BPF_IMM, 0xaedecbdb), - BPF_STMT(BPF_STX, 6), - BPF_STMT(BPF_LDX | BPF_IMM, 0xadebbade), - BPF_STMT(BPF_STX, 7), - BPF_STMT(BPF_LDX | BPF_IMM, 0xfcfcfaec), - BPF_STMT(BPF_STX, 8), - BPF_STMT(BPF_LDX | BPF_IMM, 0xbcdddbdc), - BPF_STMT(BPF_STX, 9), - BPF_STMT(BPF_LDX | BPF_IMM, 0xfeefdfac), - BPF_STMT(BPF_STX, 10), - BPF_STMT(BPF_LDX | BPF_IMM, 0xcddcdeea), - BPF_STMT(BPF_STX, 11), - BPF_STMT(BPF_LDX | BPF_IMM, 0xaccfaebb), - BPF_STMT(BPF_STX, 12), - BPF_STMT(BPF_LDX | BPF_IMM, 0xbdcccdcf), - BPF_STMT(BPF_STX, 13), - BPF_STMT(BPF_LDX | BPF_IMM, 0xaaedecde), - BPF_STMT(BPF_STX, 14), - BPF_STMT(BPF_LDX | BPF_IMM, 0xfaeacdad), - BPF_STMT(BPF_STX, 15), - BPF_STMT(BPF_LDX | BPF_MEM, 0), - BPF_STMT(BPF_MISC | BPF_TXA, 0), - BPF_STMT(BPF_LDX | BPF_MEM, 1), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_X, 0), - BPF_STMT(BPF_LDX | BPF_MEM, 2), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_X, 0), - BPF_STMT(BPF_LDX | BPF_MEM, 3), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_X, 0), - BPF_STMT(BPF_LDX | BPF_MEM, 4), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_X, 0), - BPF_STMT(BPF_LDX | BPF_MEM, 5), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_X, 0), - BPF_STMT(BPF_LDX | BPF_MEM, 6), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_X, 0), - BPF_STMT(BPF_LDX | BPF_MEM, 7), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_X, 0), - BPF_STMT(BPF_LDX | BPF_MEM, 8), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_X, 0), - BPF_STMT(BPF_LDX | BPF_MEM, 9), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_X, 0), - BPF_STMT(BPF_LDX | BPF_MEM, 10), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_X, 0), - BPF_STMT(BPF_LDX | BPF_MEM, 11), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_X, 0), - BPF_STMT(BPF_LDX | BPF_MEM, 12), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_X, 0), - BPF_STMT(BPF_LDX | BPF_MEM, 13), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_X, 0), - BPF_STMT(BPF_LDX | BPF_MEM, 14), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_X, 0), - BPF_STMT(BPF_LDX | BPF_MEM, 15), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_X, 0), - BPF_STMT(BPF_RET | BPF_A, 0), - }, - CLASSIC | FLAG_NO_DATA, - { }, - { { 0, 0x2a5a5e5 } }, - }, - { - "check: SKF_AD_MAX", - .u.insns = { - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF + SKF_AD_MAX), - BPF_STMT(BPF_RET | BPF_A, 0), - }, - CLASSIC | FLAG_NO_DATA | FLAG_EXPECTED_FAIL, - { }, - { }, - .fill_helper = NULL, - .expected_errcode = -EINVAL, - }, - { /* Passes checker but fails during runtime. */ - "LD [SKF_AD_OFF-1]", - .u.insns = { - BPF_STMT(BPF_LD | BPF_W | BPF_ABS, - SKF_AD_OFF - 1), - BPF_STMT(BPF_RET | BPF_K, 1), - }, - CLASSIC, - { }, - { { 1, 0 } }, - }, - { - "load 64-bit immediate", - .u.insns_int = { - BPF_LD_IMM64(R1, 0x567800001234LL), - BPF_MOV64_REG(R2, R1), - BPF_MOV64_REG(R3, R2), - BPF_ALU64_IMM(BPF_RSH, R2, 32), - BPF_ALU64_IMM(BPF_LSH, R3, 32), - BPF_ALU64_IMM(BPF_RSH, R3, 32), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_JMP_IMM(BPF_JEQ, R2, 0x5678, 1), - BPF_EXIT_INSN(), - BPF_JMP_IMM(BPF_JEQ, R3, 0x1234, 1), - BPF_EXIT_INSN(), - BPF_LD_IMM64(R0, 0x1ffffffffLL), - BPF_ALU64_IMM(BPF_RSH, R0, 32), /* R0 = 1 */ - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } } - }, - /* BPF_ALU | BPF_MOV | BPF_X */ - { - "ALU_MOV_X: dst = 2", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R1, 2), - BPF_ALU32_REG(BPF_MOV, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 2 } }, - }, - { - "ALU_MOV_X: dst = 4294967295", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R1, 4294967295U), - BPF_ALU32_REG(BPF_MOV, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 4294967295U } }, - }, - { - "ALU64_MOV_X: dst = 2", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R1, 2), - BPF_ALU64_REG(BPF_MOV, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 2 } }, - }, - { - "ALU64_MOV_X: dst = 4294967295", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R1, 4294967295U), - BPF_ALU64_REG(BPF_MOV, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 4294967295U } }, - }, - /* BPF_ALU | BPF_MOV | BPF_K */ - { - "ALU_MOV_K: dst = 2", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 2), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 2 } }, - }, - { - "ALU_MOV_K: dst = 4294967295", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 4294967295U), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 4294967295U } }, - }, - { - "ALU_MOV_K: 0x0000ffffffff0000 = 0x00000000ffffffff", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x0000ffffffff0000LL), - BPF_LD_IMM64(R3, 0x00000000ffffffffLL), - BPF_ALU32_IMM(BPF_MOV, R2, 0xffffffff), - BPF_JMP_REG(BPF_JEQ, R2, R3, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - { - "ALU_MOV_K: small negative", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, -123), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -123 } } - }, - { - "ALU_MOV_K: small negative zero extension", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, -123), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } } - }, - { - "ALU_MOV_K: large negative", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, -123456789), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -123456789 } } - }, - { - "ALU_MOV_K: large negative zero extension", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, -123456789), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } } - }, - { - "ALU64_MOV_K: dst = 2", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 2), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 2 } }, - }, - { - "ALU64_MOV_K: dst = 2147483647", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 2147483647), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 2147483647 } }, - }, - { - "ALU64_OR_K: dst = 0x0", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x0000ffffffff0000LL), - BPF_LD_IMM64(R3, 0x0), - BPF_ALU64_IMM(BPF_MOV, R2, 0x0), - BPF_JMP_REG(BPF_JEQ, R2, R3, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - { - "ALU64_MOV_K: dst = -1", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x0000ffffffff0000LL), - BPF_LD_IMM64(R3, 0xffffffffffffffffLL), - BPF_ALU64_IMM(BPF_MOV, R2, 0xffffffff), - BPF_JMP_REG(BPF_JEQ, R2, R3, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - { - "ALU64_MOV_K: small negative", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, -123), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -123 } } - }, - { - "ALU64_MOV_K: small negative sign extension", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, -123), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xffffffff } } - }, - { - "ALU64_MOV_K: large negative", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, -123456789), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -123456789 } } - }, - { - "ALU64_MOV_K: large negative sign extension", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, -123456789), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xffffffff } } - }, - /* MOVSX32 */ - { - "ALU_MOVSX | BPF_B", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x00000000ffffffefLL), - BPF_LD_IMM64(R3, 0xdeadbeefdeadbeefLL), - BPF_MOVSX32_REG(R1, R3, 8), - BPF_JMP_REG(BPF_JEQ, R2, R1, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - { - "ALU_MOVSX | BPF_H", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x00000000ffffbeefLL), - BPF_LD_IMM64(R3, 0xdeadbeefdeadbeefLL), - BPF_MOVSX32_REG(R1, R3, 16), - BPF_JMP_REG(BPF_JEQ, R2, R1, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - /* MOVSX64 REG */ - { - "ALU64_MOVSX | BPF_B", - .u.insns_int = { - BPF_LD_IMM64(R2, 0xffffffffffffffefLL), - BPF_LD_IMM64(R3, 0xdeadbeefdeadbeefLL), - BPF_MOVSX64_REG(R1, R3, 8), - BPF_JMP_REG(BPF_JEQ, R2, R1, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - { - "ALU64_MOVSX | BPF_H", - .u.insns_int = { - BPF_LD_IMM64(R2, 0xffffffffffffbeefLL), - BPF_LD_IMM64(R3, 0xdeadbeefdeadbeefLL), - BPF_MOVSX64_REG(R1, R3, 16), - BPF_JMP_REG(BPF_JEQ, R2, R1, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - { - "ALU64_MOVSX | BPF_W", - .u.insns_int = { - BPF_LD_IMM64(R2, 0xffffffffdeadbeefLL), - BPF_LD_IMM64(R3, 0xdeadbeefdeadbeefLL), - BPF_MOVSX64_REG(R1, R3, 32), - BPF_JMP_REG(BPF_JEQ, R2, R1, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - /* BPF_ALU | BPF_ADD | BPF_X */ - { - "ALU_ADD_X: 1 + 2 = 3", - .u.insns_int = { - BPF_LD_IMM64(R0, 1), - BPF_ALU32_IMM(BPF_MOV, R1, 2), - BPF_ALU32_REG(BPF_ADD, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 3 } }, - }, - { - "ALU_ADD_X: 1 + 4294967294 = 4294967295", - .u.insns_int = { - BPF_LD_IMM64(R0, 1), - BPF_ALU32_IMM(BPF_MOV, R1, 4294967294U), - BPF_ALU32_REG(BPF_ADD, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 4294967295U } }, - }, - { - "ALU_ADD_X: 2 + 4294967294 = 0", - .u.insns_int = { - BPF_LD_IMM64(R0, 2), - BPF_LD_IMM64(R1, 4294967294U), - BPF_ALU32_REG(BPF_ADD, R0, R1), - BPF_JMP_IMM(BPF_JEQ, R0, 0, 2), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "ALU64_ADD_X: 1 + 2 = 3", - .u.insns_int = { - BPF_LD_IMM64(R0, 1), - BPF_ALU32_IMM(BPF_MOV, R1, 2), - BPF_ALU64_REG(BPF_ADD, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 3 } }, - }, - { - "ALU64_ADD_X: 1 + 4294967294 = 4294967295", - .u.insns_int = { - BPF_LD_IMM64(R0, 1), - BPF_ALU32_IMM(BPF_MOV, R1, 4294967294U), - BPF_ALU64_REG(BPF_ADD, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 4294967295U } }, - }, - { - "ALU64_ADD_X: 2 + 4294967294 = 4294967296", - .u.insns_int = { - BPF_LD_IMM64(R0, 2), - BPF_LD_IMM64(R1, 4294967294U), - BPF_LD_IMM64(R2, 4294967296ULL), - BPF_ALU64_REG(BPF_ADD, R0, R1), - BPF_JMP_REG(BPF_JEQ, R0, R2, 2), - BPF_MOV32_IMM(R0, 0), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - /* BPF_ALU | BPF_ADD | BPF_K */ - { - "ALU_ADD_K: 1 + 2 = 3", - .u.insns_int = { - BPF_LD_IMM64(R0, 1), - BPF_ALU32_IMM(BPF_ADD, R0, 2), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 3 } }, - }, - { - "ALU_ADD_K: 3 + 0 = 3", - .u.insns_int = { - BPF_LD_IMM64(R0, 3), - BPF_ALU32_IMM(BPF_ADD, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 3 } }, - }, - { - "ALU_ADD_K: 1 + 4294967294 = 4294967295", - .u.insns_int = { - BPF_LD_IMM64(R0, 1), - BPF_ALU32_IMM(BPF_ADD, R0, 4294967294U), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 4294967295U } }, - }, - { - "ALU_ADD_K: 4294967294 + 2 = 0", - .u.insns_int = { - BPF_LD_IMM64(R0, 4294967294U), - BPF_ALU32_IMM(BPF_ADD, R0, 2), - BPF_JMP_IMM(BPF_JEQ, R0, 0, 2), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "ALU_ADD_K: 0 + (-1) = 0x00000000ffffffff", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x0), - BPF_LD_IMM64(R3, 0x00000000ffffffff), - BPF_ALU32_IMM(BPF_ADD, R2, 0xffffffff), - BPF_JMP_REG(BPF_JEQ, R2, R3, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - { - "ALU_ADD_K: 0 + 0xffff = 0xffff", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x0), - BPF_LD_IMM64(R3, 0xffff), - BPF_ALU32_IMM(BPF_ADD, R2, 0xffff), - BPF_JMP_REG(BPF_JEQ, R2, R3, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - { - "ALU_ADD_K: 0 + 0x7fffffff = 0x7fffffff", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x0), - BPF_LD_IMM64(R3, 0x7fffffff), - BPF_ALU32_IMM(BPF_ADD, R2, 0x7fffffff), - BPF_JMP_REG(BPF_JEQ, R2, R3, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - { - "ALU_ADD_K: 0 + 0x80000000 = 0x80000000", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x0), - BPF_LD_IMM64(R3, 0x80000000), - BPF_ALU32_IMM(BPF_ADD, R2, 0x80000000), - BPF_JMP_REG(BPF_JEQ, R2, R3, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - { - "ALU_ADD_K: 0 + 0x80008000 = 0x80008000", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x0), - BPF_LD_IMM64(R3, 0x80008000), - BPF_ALU32_IMM(BPF_ADD, R2, 0x80008000), - BPF_JMP_REG(BPF_JEQ, R2, R3, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - { - "ALU64_ADD_K: 1 + 2 = 3", - .u.insns_int = { - BPF_LD_IMM64(R0, 1), - BPF_ALU64_IMM(BPF_ADD, R0, 2), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 3 } }, - }, - { - "ALU64_ADD_K: 3 + 0 = 3", - .u.insns_int = { - BPF_LD_IMM64(R0, 3), - BPF_ALU64_IMM(BPF_ADD, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 3 } }, - }, - { - "ALU64_ADD_K: 1 + 2147483646 = 2147483647", - .u.insns_int = { - BPF_LD_IMM64(R0, 1), - BPF_ALU64_IMM(BPF_ADD, R0, 2147483646), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 2147483647 } }, - }, - { - "ALU64_ADD_K: 4294967294 + 2 = 4294967296", - .u.insns_int = { - BPF_LD_IMM64(R0, 4294967294U), - BPF_LD_IMM64(R1, 4294967296ULL), - BPF_ALU64_IMM(BPF_ADD, R0, 2), - BPF_JMP_REG(BPF_JEQ, R0, R1, 2), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "ALU64_ADD_K: 2147483646 + -2147483647 = -1", - .u.insns_int = { - BPF_LD_IMM64(R0, 2147483646), - BPF_ALU64_IMM(BPF_ADD, R0, -2147483647), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -1 } }, - }, - { - "ALU64_ADD_K: 1 + 0 = 1", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x1), - BPF_LD_IMM64(R3, 0x1), - BPF_ALU64_IMM(BPF_ADD, R2, 0x0), - BPF_JMP_REG(BPF_JEQ, R2, R3, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - { - "ALU64_ADD_K: 0 + (-1) = 0xffffffffffffffff", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x0), - BPF_LD_IMM64(R3, 0xffffffffffffffffLL), - BPF_ALU64_IMM(BPF_ADD, R2, 0xffffffff), - BPF_JMP_REG(BPF_JEQ, R2, R3, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - { - "ALU64_ADD_K: 0 + 0xffff = 0xffff", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x0), - BPF_LD_IMM64(R3, 0xffff), - BPF_ALU64_IMM(BPF_ADD, R2, 0xffff), - BPF_JMP_REG(BPF_JEQ, R2, R3, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - { - "ALU64_ADD_K: 0 + 0x7fffffff = 0x7fffffff", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x0), - BPF_LD_IMM64(R3, 0x7fffffff), - BPF_ALU64_IMM(BPF_ADD, R2, 0x7fffffff), - BPF_JMP_REG(BPF_JEQ, R2, R3, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - { - "ALU64_ADD_K: 0 + 0x80000000 = 0xffffffff80000000", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x0), - BPF_LD_IMM64(R3, 0xffffffff80000000LL), - BPF_ALU64_IMM(BPF_ADD, R2, 0x80000000), - BPF_JMP_REG(BPF_JEQ, R2, R3, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - { - "ALU_ADD_K: 0 + 0x80008000 = 0xffffffff80008000", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x0), - BPF_LD_IMM64(R3, 0xffffffff80008000LL), - BPF_ALU64_IMM(BPF_ADD, R2, 0x80008000), - BPF_JMP_REG(BPF_JEQ, R2, R3, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - /* BPF_ALU | BPF_SUB | BPF_X */ - { - "ALU_SUB_X: 3 - 1 = 2", - .u.insns_int = { - BPF_LD_IMM64(R0, 3), - BPF_ALU32_IMM(BPF_MOV, R1, 1), - BPF_ALU32_REG(BPF_SUB, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 2 } }, - }, - { - "ALU_SUB_X: 4294967295 - 4294967294 = 1", - .u.insns_int = { - BPF_LD_IMM64(R0, 4294967295U), - BPF_ALU32_IMM(BPF_MOV, R1, 4294967294U), - BPF_ALU32_REG(BPF_SUB, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "ALU64_SUB_X: 3 - 1 = 2", - .u.insns_int = { - BPF_LD_IMM64(R0, 3), - BPF_ALU32_IMM(BPF_MOV, R1, 1), - BPF_ALU64_REG(BPF_SUB, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 2 } }, - }, - { - "ALU64_SUB_X: 4294967295 - 4294967294 = 1", - .u.insns_int = { - BPF_LD_IMM64(R0, 4294967295U), - BPF_ALU32_IMM(BPF_MOV, R1, 4294967294U), - BPF_ALU64_REG(BPF_SUB, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - /* BPF_ALU | BPF_SUB | BPF_K */ - { - "ALU_SUB_K: 3 - 1 = 2", - .u.insns_int = { - BPF_LD_IMM64(R0, 3), - BPF_ALU32_IMM(BPF_SUB, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 2 } }, - }, - { - "ALU_SUB_K: 3 - 0 = 3", - .u.insns_int = { - BPF_LD_IMM64(R0, 3), - BPF_ALU32_IMM(BPF_SUB, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 3 } }, - }, - { - "ALU_SUB_K: 4294967295 - 4294967294 = 1", - .u.insns_int = { - BPF_LD_IMM64(R0, 4294967295U), - BPF_ALU32_IMM(BPF_SUB, R0, 4294967294U), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "ALU64_SUB_K: 3 - 1 = 2", - .u.insns_int = { - BPF_LD_IMM64(R0, 3), - BPF_ALU64_IMM(BPF_SUB, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 2 } }, - }, - { - "ALU64_SUB_K: 3 - 0 = 3", - .u.insns_int = { - BPF_LD_IMM64(R0, 3), - BPF_ALU64_IMM(BPF_SUB, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 3 } }, - }, - { - "ALU64_SUB_K: 4294967294 - 4294967295 = -1", - .u.insns_int = { - BPF_LD_IMM64(R0, 4294967294U), - BPF_ALU64_IMM(BPF_SUB, R0, 4294967295U), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -1 } }, - }, - { - "ALU64_ADD_K: 2147483646 - 2147483647 = -1", - .u.insns_int = { - BPF_LD_IMM64(R0, 2147483646), - BPF_ALU64_IMM(BPF_SUB, R0, 2147483647), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -1 } }, - }, - /* BPF_ALU | BPF_MUL | BPF_X */ - { - "ALU_MUL_X: 2 * 3 = 6", - .u.insns_int = { - BPF_LD_IMM64(R0, 2), - BPF_ALU32_IMM(BPF_MOV, R1, 3), - BPF_ALU32_REG(BPF_MUL, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 6 } }, - }, - { - "ALU_MUL_X: 2 * 0x7FFFFFF8 = 0xFFFFFFF0", - .u.insns_int = { - BPF_LD_IMM64(R0, 2), - BPF_ALU32_IMM(BPF_MOV, R1, 0x7FFFFFF8), - BPF_ALU32_REG(BPF_MUL, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xFFFFFFF0 } }, - }, - { - "ALU_MUL_X: -1 * -1 = 1", - .u.insns_int = { - BPF_LD_IMM64(R0, -1), - BPF_ALU32_IMM(BPF_MOV, R1, -1), - BPF_ALU32_REG(BPF_MUL, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "ALU64_MUL_X: 2 * 3 = 6", - .u.insns_int = { - BPF_LD_IMM64(R0, 2), - BPF_ALU32_IMM(BPF_MOV, R1, 3), - BPF_ALU64_REG(BPF_MUL, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 6 } }, - }, - { - "ALU64_MUL_X: 1 * 2147483647 = 2147483647", - .u.insns_int = { - BPF_LD_IMM64(R0, 1), - BPF_ALU32_IMM(BPF_MOV, R1, 2147483647), - BPF_ALU64_REG(BPF_MUL, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 2147483647 } }, - }, - { - "ALU64_MUL_X: 64x64 multiply, low word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0fedcba987654321LL), - BPF_LD_IMM64(R1, 0x123456789abcdef0LL), - BPF_ALU64_REG(BPF_MUL, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xe5618cf0 } } - }, - { - "ALU64_MUL_X: 64x64 multiply, high word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0fedcba987654321LL), - BPF_LD_IMM64(R1, 0x123456789abcdef0LL), - BPF_ALU64_REG(BPF_MUL, R0, R1), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x2236d88f } } - }, - /* BPF_ALU | BPF_MUL | BPF_K */ - { - "ALU_MUL_K: 2 * 3 = 6", - .u.insns_int = { - BPF_LD_IMM64(R0, 2), - BPF_ALU32_IMM(BPF_MUL, R0, 3), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 6 } }, - }, - { - "ALU_MUL_K: 3 * 1 = 3", - .u.insns_int = { - BPF_LD_IMM64(R0, 3), - BPF_ALU32_IMM(BPF_MUL, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 3 } }, - }, - { - "ALU_MUL_K: 2 * 0x7FFFFFF8 = 0xFFFFFFF0", - .u.insns_int = { - BPF_LD_IMM64(R0, 2), - BPF_ALU32_IMM(BPF_MUL, R0, 0x7FFFFFF8), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xFFFFFFF0 } }, - }, - { - "ALU_MUL_K: 1 * (-1) = 0x00000000ffffffff", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x1), - BPF_LD_IMM64(R3, 0x00000000ffffffff), - BPF_ALU32_IMM(BPF_MUL, R2, 0xffffffff), - BPF_JMP_REG(BPF_JEQ, R2, R3, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - { - "ALU64_MUL_K: 2 * 3 = 6", - .u.insns_int = { - BPF_LD_IMM64(R0, 2), - BPF_ALU64_IMM(BPF_MUL, R0, 3), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 6 } }, - }, - { - "ALU64_MUL_K: 3 * 1 = 3", - .u.insns_int = { - BPF_LD_IMM64(R0, 3), - BPF_ALU64_IMM(BPF_MUL, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 3 } }, - }, - { - "ALU64_MUL_K: 1 * 2147483647 = 2147483647", - .u.insns_int = { - BPF_LD_IMM64(R0, 1), - BPF_ALU64_IMM(BPF_MUL, R0, 2147483647), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 2147483647 } }, - }, - { - "ALU64_MUL_K: 1 * -2147483647 = -2147483647", - .u.insns_int = { - BPF_LD_IMM64(R0, 1), - BPF_ALU64_IMM(BPF_MUL, R0, -2147483647), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -2147483647 } }, - }, - { - "ALU64_MUL_K: 1 * (-1) = 0xffffffffffffffff", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x1), - BPF_LD_IMM64(R3, 0xffffffffffffffffLL), - BPF_ALU64_IMM(BPF_MUL, R2, 0xffffffff), - BPF_JMP_REG(BPF_JEQ, R2, R3, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - { - "ALU64_MUL_K: 64x32 multiply, low word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_ALU64_IMM(BPF_MUL, R0, 0x12345678), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xe242d208 } } - }, - { - "ALU64_MUL_K: 64x32 multiply, high word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_ALU64_IMM(BPF_MUL, R0, 0x12345678), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xc28f5c28 } } - }, - /* BPF_ALU | BPF_DIV | BPF_X */ - { - "ALU_DIV_X: 6 / 2 = 3", - .u.insns_int = { - BPF_LD_IMM64(R0, 6), - BPF_ALU32_IMM(BPF_MOV, R1, 2), - BPF_ALU32_REG(BPF_DIV, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 3 } }, - }, - { - "ALU_DIV_X: 4294967295 / 4294967295 = 1", - .u.insns_int = { - BPF_LD_IMM64(R0, 4294967295U), - BPF_ALU32_IMM(BPF_MOV, R1, 4294967295U), - BPF_ALU32_REG(BPF_DIV, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "ALU64_DIV_X: 6 / 2 = 3", - .u.insns_int = { - BPF_LD_IMM64(R0, 6), - BPF_ALU32_IMM(BPF_MOV, R1, 2), - BPF_ALU64_REG(BPF_DIV, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 3 } }, - }, - { - "ALU64_DIV_X: 2147483647 / 2147483647 = 1", - .u.insns_int = { - BPF_LD_IMM64(R0, 2147483647), - BPF_ALU32_IMM(BPF_MOV, R1, 2147483647), - BPF_ALU64_REG(BPF_DIV, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "ALU64_DIV_X: 0xffffffffffffffff / (-1) = 0x0000000000000001", - .u.insns_int = { - BPF_LD_IMM64(R2, 0xffffffffffffffffLL), - BPF_LD_IMM64(R4, 0xffffffffffffffffLL), - BPF_LD_IMM64(R3, 0x0000000000000001LL), - BPF_ALU64_REG(BPF_DIV, R2, R4), - BPF_JMP_REG(BPF_JEQ, R2, R3, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - /* BPF_ALU | BPF_DIV | BPF_K */ - { - "ALU_DIV_K: 6 / 2 = 3", - .u.insns_int = { - BPF_LD_IMM64(R0, 6), - BPF_ALU32_IMM(BPF_DIV, R0, 2), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 3 } }, - }, - { - "ALU_DIV_K: 3 / 1 = 3", - .u.insns_int = { - BPF_LD_IMM64(R0, 3), - BPF_ALU32_IMM(BPF_DIV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 3 } }, - }, - { - "ALU_DIV_K: 4294967295 / 4294967295 = 1", - .u.insns_int = { - BPF_LD_IMM64(R0, 4294967295U), - BPF_ALU32_IMM(BPF_DIV, R0, 4294967295U), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "ALU_DIV_K: 0xffffffffffffffff / (-1) = 0x1", - .u.insns_int = { - BPF_LD_IMM64(R2, 0xffffffffffffffffLL), - BPF_LD_IMM64(R3, 0x1UL), - BPF_ALU32_IMM(BPF_DIV, R2, 0xffffffff), - BPF_JMP_REG(BPF_JEQ, R2, R3, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - { - "ALU64_DIV_K: 6 / 2 = 3", - .u.insns_int = { - BPF_LD_IMM64(R0, 6), - BPF_ALU64_IMM(BPF_DIV, R0, 2), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 3 } }, - }, - { - "ALU64_DIV_K: 3 / 1 = 3", - .u.insns_int = { - BPF_LD_IMM64(R0, 3), - BPF_ALU64_IMM(BPF_DIV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 3 } }, - }, - { - "ALU64_DIV_K: 2147483647 / 2147483647 = 1", - .u.insns_int = { - BPF_LD_IMM64(R0, 2147483647), - BPF_ALU64_IMM(BPF_DIV, R0, 2147483647), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "ALU64_DIV_K: 0xffffffffffffffff / (-1) = 0x0000000000000001", - .u.insns_int = { - BPF_LD_IMM64(R2, 0xffffffffffffffffLL), - BPF_LD_IMM64(R3, 0x0000000000000001LL), - BPF_ALU64_IMM(BPF_DIV, R2, 0xffffffff), - BPF_JMP_REG(BPF_JEQ, R2, R3, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - /* BPF_ALU | BPF_MOD | BPF_X */ - { - "ALU_MOD_X: 3 % 2 = 1", - .u.insns_int = { - BPF_LD_IMM64(R0, 3), - BPF_ALU32_IMM(BPF_MOV, R1, 2), - BPF_ALU32_REG(BPF_MOD, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "ALU_MOD_X: 4294967295 % 4294967293 = 2", - .u.insns_int = { - BPF_LD_IMM64(R0, 4294967295U), - BPF_ALU32_IMM(BPF_MOV, R1, 4294967293U), - BPF_ALU32_REG(BPF_MOD, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 2 } }, - }, - { - "ALU64_MOD_X: 3 % 2 = 1", - .u.insns_int = { - BPF_LD_IMM64(R0, 3), - BPF_ALU32_IMM(BPF_MOV, R1, 2), - BPF_ALU64_REG(BPF_MOD, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "ALU64_MOD_X: 2147483647 % 2147483645 = 2", - .u.insns_int = { - BPF_LD_IMM64(R0, 2147483647), - BPF_ALU32_IMM(BPF_MOV, R1, 2147483645), - BPF_ALU64_REG(BPF_MOD, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 2 } }, - }, - /* BPF_ALU | BPF_MOD | BPF_K */ - { - "ALU_MOD_K: 3 % 2 = 1", - .u.insns_int = { - BPF_LD_IMM64(R0, 3), - BPF_ALU32_IMM(BPF_MOD, R0, 2), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "ALU_MOD_K: 3 % 1 = 0", - .u.insns_int = { - BPF_LD_IMM64(R0, 3), - BPF_ALU32_IMM(BPF_MOD, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } }, - }, - { - "ALU_MOD_K: 4294967295 % 4294967293 = 2", - .u.insns_int = { - BPF_LD_IMM64(R0, 4294967295U), - BPF_ALU32_IMM(BPF_MOD, R0, 4294967293U), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 2 } }, - }, - { - "ALU64_MOD_K: 3 % 2 = 1", - .u.insns_int = { - BPF_LD_IMM64(R0, 3), - BPF_ALU64_IMM(BPF_MOD, R0, 2), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "ALU64_MOD_K: 3 % 1 = 0", - .u.insns_int = { - BPF_LD_IMM64(R0, 3), - BPF_ALU64_IMM(BPF_MOD, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } }, - }, - { - "ALU64_MOD_K: 2147483647 % 2147483645 = 2", - .u.insns_int = { - BPF_LD_IMM64(R0, 2147483647), - BPF_ALU64_IMM(BPF_MOD, R0, 2147483645), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 2 } }, - }, - /* BPF_ALU | BPF_DIV | BPF_X off=1 (SDIV) */ - { - "ALU_SDIV_X: -6 / 2 = -3", - .u.insns_int = { - BPF_LD_IMM64(R0, -6), - BPF_ALU32_IMM(BPF_MOV, R1, 2), - BPF_ALU32_REG_OFF(BPF_DIV, R0, R1, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -3 } }, - }, - /* BPF_ALU | BPF_DIV | BPF_K off=1 (SDIV) */ - { - "ALU_SDIV_K: -6 / 2 = -3", - .u.insns_int = { - BPF_LD_IMM64(R0, -6), - BPF_ALU32_IMM_OFF(BPF_DIV, R0, 2, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -3 } }, - }, - /* BPF_ALU64 | BPF_DIV | BPF_X off=1 (SDIV64) */ - { - "ALU64_SDIV_X: -6 / 2 = -3", - .u.insns_int = { - BPF_LD_IMM64(R0, -6), - BPF_ALU32_IMM(BPF_MOV, R1, 2), - BPF_ALU64_REG_OFF(BPF_DIV, R0, R1, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -3 } }, - }, - /* BPF_ALU64 | BPF_DIV | BPF_K off=1 (SDIV64) */ - { - "ALU64_SDIV_K: -6 / 2 = -3", - .u.insns_int = { - BPF_LD_IMM64(R0, -6), - BPF_ALU64_IMM_OFF(BPF_DIV, R0, 2, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -3 } }, - }, - /* BPF_ALU | BPF_MOD | BPF_X off=1 (SMOD) */ - { - "ALU_SMOD_X: -7 % 2 = -1", - .u.insns_int = { - BPF_LD_IMM64(R0, -7), - BPF_ALU32_IMM(BPF_MOV, R1, 2), - BPF_ALU32_REG_OFF(BPF_MOD, R0, R1, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -1 } }, - }, - /* BPF_ALU | BPF_MOD | BPF_K off=1 (SMOD) */ - { - "ALU_SMOD_K: -7 % 2 = -1", - .u.insns_int = { - BPF_LD_IMM64(R0, -7), - BPF_ALU32_IMM_OFF(BPF_MOD, R0, 2, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -1 } }, - }, - /* BPF_ALU64 | BPF_MOD | BPF_X off=1 (SMOD64) */ - { - "ALU64_SMOD_X: -7 % 2 = -1", - .u.insns_int = { - BPF_LD_IMM64(R0, -7), - BPF_ALU32_IMM(BPF_MOV, R1, 2), - BPF_ALU64_REG_OFF(BPF_MOD, R0, R1, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -1 } }, - }, - /* BPF_ALU64 | BPF_MOD | BPF_K off=1 (SMOD64) */ - { - "ALU64_SMOD_K: -7 % 2 = -1", - .u.insns_int = { - BPF_LD_IMM64(R0, -7), - BPF_ALU64_IMM_OFF(BPF_MOD, R0, 2, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -1 } }, - }, - /* BPF_ALU | BPF_AND | BPF_X */ - { - "ALU_AND_X: 3 & 2 = 2", - .u.insns_int = { - BPF_LD_IMM64(R0, 3), - BPF_ALU32_IMM(BPF_MOV, R1, 2), - BPF_ALU32_REG(BPF_AND, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 2 } }, - }, - { - "ALU_AND_X: 0xffffffff & 0xffffffff = 0xffffffff", - .u.insns_int = { - BPF_LD_IMM64(R0, 0xffffffff), - BPF_ALU32_IMM(BPF_MOV, R1, 0xffffffff), - BPF_ALU32_REG(BPF_AND, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xffffffff } }, - }, - { - "ALU64_AND_X: 3 & 2 = 2", - .u.insns_int = { - BPF_LD_IMM64(R0, 3), - BPF_ALU32_IMM(BPF_MOV, R1, 2), - BPF_ALU64_REG(BPF_AND, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 2 } }, - }, - { - "ALU64_AND_X: 0xffffffff & 0xffffffff = 0xffffffff", - .u.insns_int = { - BPF_LD_IMM64(R0, 0xffffffff), - BPF_ALU32_IMM(BPF_MOV, R1, 0xffffffff), - BPF_ALU64_REG(BPF_AND, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xffffffff } }, - }, - /* BPF_ALU | BPF_AND | BPF_K */ - { - "ALU_AND_K: 3 & 2 = 2", - .u.insns_int = { - BPF_LD_IMM64(R0, 3), - BPF_ALU32_IMM(BPF_AND, R0, 2), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 2 } }, - }, - { - "ALU_AND_K: 0xffffffff & 0xffffffff = 0xffffffff", - .u.insns_int = { - BPF_LD_IMM64(R0, 0xffffffff), - BPF_ALU32_IMM(BPF_AND, R0, 0xffffffff), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xffffffff } }, - }, - { - "ALU_AND_K: Small immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0x01020304), - BPF_ALU32_IMM(BPF_AND, R0, 15), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 4 } } - }, - { - "ALU_AND_K: Large immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0xf1f2f3f4), - BPF_ALU32_IMM(BPF_AND, R0, 0xafbfcfdf), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xa1b2c3d4 } } - }, - { - "ALU_AND_K: Zero extension", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_LD_IMM64(R1, 0x0000000080a0c0e0LL), - BPF_ALU32_IMM(BPF_AND, R0, 0xf0f0f0f0), - BPF_JMP_REG(BPF_JEQ, R0, R1, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } } - }, - { - "ALU64_AND_K: 3 & 2 = 2", - .u.insns_int = { - BPF_LD_IMM64(R0, 3), - BPF_ALU64_IMM(BPF_AND, R0, 2), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 2 } }, - }, - { - "ALU64_AND_K: 0xffffffff & 0xffffffff = 0xffffffff", - .u.insns_int = { - BPF_LD_IMM64(R0, 0xffffffff), - BPF_ALU64_IMM(BPF_AND, R0, 0xffffffff), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xffffffff } }, - }, - { - "ALU64_AND_K: 0x0000ffffffff0000 & 0x0 = 0x0000000000000000", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x0000ffffffff0000LL), - BPF_LD_IMM64(R3, 0x0000000000000000LL), - BPF_ALU64_IMM(BPF_AND, R2, 0x0), - BPF_JMP_REG(BPF_JEQ, R2, R3, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - { - "ALU64_AND_K: 0x0000ffffffff0000 & -1 = 0x0000ffffffff0000", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x0000ffffffff0000LL), - BPF_LD_IMM64(R3, 0x0000ffffffff0000LL), - BPF_ALU64_IMM(BPF_AND, R2, 0xffffffff), - BPF_JMP_REG(BPF_JEQ, R2, R3, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - { - "ALU64_AND_K: 0xffffffffffffffff & -1 = 0xffffffffffffffff", - .u.insns_int = { - BPF_LD_IMM64(R2, 0xffffffffffffffffLL), - BPF_LD_IMM64(R3, 0xffffffffffffffffLL), - BPF_ALU64_IMM(BPF_AND, R2, 0xffffffff), - BPF_JMP_REG(BPF_JEQ, R2, R3, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - { - "ALU64_AND_K: Sign extension 1", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_LD_IMM64(R1, 0x00000000090b0d0fLL), - BPF_ALU64_IMM(BPF_AND, R0, 0x0f0f0f0f), - BPF_JMP_REG(BPF_JEQ, R0, R1, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } } - }, - { - "ALU64_AND_K: Sign extension 2", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_LD_IMM64(R1, 0x0123456780a0c0e0LL), - BPF_ALU64_IMM(BPF_AND, R0, 0xf0f0f0f0), - BPF_JMP_REG(BPF_JEQ, R0, R1, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } } - }, - /* BPF_ALU | BPF_OR | BPF_X */ - { - "ALU_OR_X: 1 | 2 = 3", - .u.insns_int = { - BPF_LD_IMM64(R0, 1), - BPF_ALU32_IMM(BPF_MOV, R1, 2), - BPF_ALU32_REG(BPF_OR, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 3 } }, - }, - { - "ALU_OR_X: 0x0 | 0xffffffff = 0xffffffff", - .u.insns_int = { - BPF_LD_IMM64(R0, 0), - BPF_ALU32_IMM(BPF_MOV, R1, 0xffffffff), - BPF_ALU32_REG(BPF_OR, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xffffffff } }, - }, - { - "ALU64_OR_X: 1 | 2 = 3", - .u.insns_int = { - BPF_LD_IMM64(R0, 1), - BPF_ALU32_IMM(BPF_MOV, R1, 2), - BPF_ALU64_REG(BPF_OR, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 3 } }, - }, - { - "ALU64_OR_X: 0 | 0xffffffff = 0xffffffff", - .u.insns_int = { - BPF_LD_IMM64(R0, 0), - BPF_ALU32_IMM(BPF_MOV, R1, 0xffffffff), - BPF_ALU64_REG(BPF_OR, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xffffffff } }, - }, - /* BPF_ALU | BPF_OR | BPF_K */ - { - "ALU_OR_K: 1 | 2 = 3", - .u.insns_int = { - BPF_LD_IMM64(R0, 1), - BPF_ALU32_IMM(BPF_OR, R0, 2), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 3 } }, - }, - { - "ALU_OR_K: 0 & 0xffffffff = 0xffffffff", - .u.insns_int = { - BPF_LD_IMM64(R0, 0), - BPF_ALU32_IMM(BPF_OR, R0, 0xffffffff), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xffffffff } }, - }, - { - "ALU_OR_K: Small immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0x01020304), - BPF_ALU32_IMM(BPF_OR, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x01020305 } } - }, - { - "ALU_OR_K: Large immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0x01020304), - BPF_ALU32_IMM(BPF_OR, R0, 0xa0b0c0d0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xa1b2c3d4 } } - }, - { - "ALU_OR_K: Zero extension", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_LD_IMM64(R1, 0x00000000f9fbfdffLL), - BPF_ALU32_IMM(BPF_OR, R0, 0xf0f0f0f0), - BPF_JMP_REG(BPF_JEQ, R0, R1, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } } - }, - { - "ALU64_OR_K: 1 | 2 = 3", - .u.insns_int = { - BPF_LD_IMM64(R0, 1), - BPF_ALU64_IMM(BPF_OR, R0, 2), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 3 } }, - }, - { - "ALU64_OR_K: 0 & 0xffffffff = 0xffffffff", - .u.insns_int = { - BPF_LD_IMM64(R0, 0), - BPF_ALU64_IMM(BPF_OR, R0, 0xffffffff), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xffffffff } }, - }, - { - "ALU64_OR_K: 0x0000ffffffff0000 | 0x0 = 0x0000ffffffff0000", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x0000ffffffff0000LL), - BPF_LD_IMM64(R3, 0x0000ffffffff0000LL), - BPF_ALU64_IMM(BPF_OR, R2, 0x0), - BPF_JMP_REG(BPF_JEQ, R2, R3, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - { - "ALU64_OR_K: 0x0000ffffffff0000 | -1 = 0xffffffffffffffff", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x0000ffffffff0000LL), - BPF_LD_IMM64(R3, 0xffffffffffffffffLL), - BPF_ALU64_IMM(BPF_OR, R2, 0xffffffff), - BPF_JMP_REG(BPF_JEQ, R2, R3, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - { - "ALU64_OR_K: 0x000000000000000 | -1 = 0xffffffffffffffff", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x0000000000000000LL), - BPF_LD_IMM64(R3, 0xffffffffffffffffLL), - BPF_ALU64_IMM(BPF_OR, R2, 0xffffffff), - BPF_JMP_REG(BPF_JEQ, R2, R3, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - { - "ALU64_OR_K: Sign extension 1", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_LD_IMM64(R1, 0x012345678fafcfefLL), - BPF_ALU64_IMM(BPF_OR, R0, 0x0f0f0f0f), - BPF_JMP_REG(BPF_JEQ, R0, R1, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } } - }, - { - "ALU64_OR_K: Sign extension 2", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_LD_IMM64(R1, 0xfffffffff9fbfdffLL), - BPF_ALU64_IMM(BPF_OR, R0, 0xf0f0f0f0), - BPF_JMP_REG(BPF_JEQ, R0, R1, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } } - }, - /* BPF_ALU | BPF_XOR | BPF_X */ - { - "ALU_XOR_X: 5 ^ 6 = 3", - .u.insns_int = { - BPF_LD_IMM64(R0, 5), - BPF_ALU32_IMM(BPF_MOV, R1, 6), - BPF_ALU32_REG(BPF_XOR, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 3 } }, - }, - { - "ALU_XOR_X: 0x1 ^ 0xffffffff = 0xfffffffe", - .u.insns_int = { - BPF_LD_IMM64(R0, 1), - BPF_ALU32_IMM(BPF_MOV, R1, 0xffffffff), - BPF_ALU32_REG(BPF_XOR, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xfffffffe } }, - }, - { - "ALU64_XOR_X: 5 ^ 6 = 3", - .u.insns_int = { - BPF_LD_IMM64(R0, 5), - BPF_ALU32_IMM(BPF_MOV, R1, 6), - BPF_ALU64_REG(BPF_XOR, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 3 } }, - }, - { - "ALU64_XOR_X: 1 ^ 0xffffffff = 0xfffffffe", - .u.insns_int = { - BPF_LD_IMM64(R0, 1), - BPF_ALU32_IMM(BPF_MOV, R1, 0xffffffff), - BPF_ALU64_REG(BPF_XOR, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xfffffffe } }, - }, - /* BPF_ALU | BPF_XOR | BPF_K */ - { - "ALU_XOR_K: 5 ^ 6 = 3", - .u.insns_int = { - BPF_LD_IMM64(R0, 5), - BPF_ALU32_IMM(BPF_XOR, R0, 6), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 3 } }, - }, - { - "ALU_XOR_K: 1 ^ 0xffffffff = 0xfffffffe", - .u.insns_int = { - BPF_LD_IMM64(R0, 1), - BPF_ALU32_IMM(BPF_XOR, R0, 0xffffffff), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xfffffffe } }, - }, - { - "ALU_XOR_K: Small immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0x01020304), - BPF_ALU32_IMM(BPF_XOR, R0, 15), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x0102030b } } - }, - { - "ALU_XOR_K: Large immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0xf1f2f3f4), - BPF_ALU32_IMM(BPF_XOR, R0, 0xafbfcfdf), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x5e4d3c2b } } - }, - { - "ALU_XOR_K: Zero extension", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_LD_IMM64(R1, 0x00000000795b3d1fLL), - BPF_ALU32_IMM(BPF_XOR, R0, 0xf0f0f0f0), - BPF_JMP_REG(BPF_JEQ, R0, R1, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } } - }, - { - "ALU64_XOR_K: 5 ^ 6 = 3", - .u.insns_int = { - BPF_LD_IMM64(R0, 5), - BPF_ALU64_IMM(BPF_XOR, R0, 6), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 3 } }, - }, - { - "ALU64_XOR_K: 1 ^ 0xffffffff = 0xfffffffe", - .u.insns_int = { - BPF_LD_IMM64(R0, 1), - BPF_ALU64_IMM(BPF_XOR, R0, 0xffffffff), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xfffffffe } }, - }, - { - "ALU64_XOR_K: 0x0000ffffffff0000 ^ 0x0 = 0x0000ffffffff0000", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x0000ffffffff0000LL), - BPF_LD_IMM64(R3, 0x0000ffffffff0000LL), - BPF_ALU64_IMM(BPF_XOR, R2, 0x0), - BPF_JMP_REG(BPF_JEQ, R2, R3, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - { - "ALU64_XOR_K: 0x0000ffffffff0000 ^ -1 = 0xffff00000000ffff", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x0000ffffffff0000LL), - BPF_LD_IMM64(R3, 0xffff00000000ffffLL), - BPF_ALU64_IMM(BPF_XOR, R2, 0xffffffff), - BPF_JMP_REG(BPF_JEQ, R2, R3, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - { - "ALU64_XOR_K: 0x000000000000000 ^ -1 = 0xffffffffffffffff", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x0000000000000000LL), - BPF_LD_IMM64(R3, 0xffffffffffffffffLL), - BPF_ALU64_IMM(BPF_XOR, R2, 0xffffffff), - BPF_JMP_REG(BPF_JEQ, R2, R3, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - }, - { - "ALU64_XOR_K: Sign extension 1", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_LD_IMM64(R1, 0x0123456786a4c2e0LL), - BPF_ALU64_IMM(BPF_XOR, R0, 0x0f0f0f0f), - BPF_JMP_REG(BPF_JEQ, R0, R1, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } } - }, - { - "ALU64_XOR_K: Sign extension 2", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_LD_IMM64(R1, 0xfedcba98795b3d1fLL), - BPF_ALU64_IMM(BPF_XOR, R0, 0xf0f0f0f0), - BPF_JMP_REG(BPF_JEQ, R0, R1, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } } - }, - /* BPF_ALU | BPF_LSH | BPF_X */ - { - "ALU_LSH_X: 1 << 1 = 2", - .u.insns_int = { - BPF_LD_IMM64(R0, 1), - BPF_ALU32_IMM(BPF_MOV, R1, 1), - BPF_ALU32_REG(BPF_LSH, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 2 } }, - }, - { - "ALU_LSH_X: 1 << 31 = 0x80000000", - .u.insns_int = { - BPF_LD_IMM64(R0, 1), - BPF_ALU32_IMM(BPF_MOV, R1, 31), - BPF_ALU32_REG(BPF_LSH, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x80000000 } }, - }, - { - "ALU_LSH_X: 0x12345678 << 12 = 0x45678000", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0x12345678), - BPF_ALU32_IMM(BPF_MOV, R1, 12), - BPF_ALU32_REG(BPF_LSH, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x45678000 } } - }, - { - "ALU64_LSH_X: 1 << 1 = 2", - .u.insns_int = { - BPF_LD_IMM64(R0, 1), - BPF_ALU32_IMM(BPF_MOV, R1, 1), - BPF_ALU64_REG(BPF_LSH, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 2 } }, - }, - { - "ALU64_LSH_X: 1 << 31 = 0x80000000", - .u.insns_int = { - BPF_LD_IMM64(R0, 1), - BPF_ALU32_IMM(BPF_MOV, R1, 31), - BPF_ALU64_REG(BPF_LSH, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x80000000 } }, - }, - { - "ALU64_LSH_X: Shift < 32, low word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_ALU32_IMM(BPF_MOV, R1, 12), - BPF_ALU64_REG(BPF_LSH, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xbcdef000 } } - }, - { - "ALU64_LSH_X: Shift < 32, high word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_ALU32_IMM(BPF_MOV, R1, 12), - BPF_ALU64_REG(BPF_LSH, R0, R1), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x3456789a } } - }, - { - "ALU64_LSH_X: Shift > 32, low word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_ALU32_IMM(BPF_MOV, R1, 36), - BPF_ALU64_REG(BPF_LSH, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } } - }, - { - "ALU64_LSH_X: Shift > 32, high word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_ALU32_IMM(BPF_MOV, R1, 36), - BPF_ALU64_REG(BPF_LSH, R0, R1), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x9abcdef0 } } - }, - { - "ALU64_LSH_X: Shift == 32, low word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_ALU32_IMM(BPF_MOV, R1, 32), - BPF_ALU64_REG(BPF_LSH, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } } - }, - { - "ALU64_LSH_X: Shift == 32, high word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_ALU32_IMM(BPF_MOV, R1, 32), - BPF_ALU64_REG(BPF_LSH, R0, R1), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x89abcdef } } - }, - { - "ALU64_LSH_X: Zero shift, low word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_ALU32_IMM(BPF_MOV, R1, 0), - BPF_ALU64_REG(BPF_LSH, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x89abcdef } } - }, - { - "ALU64_LSH_X: Zero shift, high word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_ALU32_IMM(BPF_MOV, R1, 0), - BPF_ALU64_REG(BPF_LSH, R0, R1), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x01234567 } } - }, - /* BPF_ALU | BPF_LSH | BPF_K */ - { - "ALU_LSH_K: 1 << 1 = 2", - .u.insns_int = { - BPF_LD_IMM64(R0, 1), - BPF_ALU32_IMM(BPF_LSH, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 2 } }, - }, - { - "ALU_LSH_K: 1 << 31 = 0x80000000", - .u.insns_int = { - BPF_LD_IMM64(R0, 1), - BPF_ALU32_IMM(BPF_LSH, R0, 31), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x80000000 } }, - }, - { - "ALU_LSH_K: 0x12345678 << 12 = 0x45678000", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0x12345678), - BPF_ALU32_IMM(BPF_LSH, R0, 12), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x45678000 } } - }, - { - "ALU_LSH_K: 0x12345678 << 0 = 0x12345678", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0x12345678), - BPF_ALU32_IMM(BPF_LSH, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x12345678 } } - }, - { - "ALU64_LSH_K: 1 << 1 = 2", - .u.insns_int = { - BPF_LD_IMM64(R0, 1), - BPF_ALU64_IMM(BPF_LSH, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 2 } }, - }, - { - "ALU64_LSH_K: 1 << 31 = 0x80000000", - .u.insns_int = { - BPF_LD_IMM64(R0, 1), - BPF_ALU64_IMM(BPF_LSH, R0, 31), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x80000000 } }, - }, - { - "ALU64_LSH_K: Shift < 32, low word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_ALU64_IMM(BPF_LSH, R0, 12), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xbcdef000 } } - }, - { - "ALU64_LSH_K: Shift < 32, high word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_ALU64_IMM(BPF_LSH, R0, 12), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x3456789a } } - }, - { - "ALU64_LSH_K: Shift > 32, low word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_ALU64_IMM(BPF_LSH, R0, 36), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } } - }, - { - "ALU64_LSH_K: Shift > 32, high word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_ALU64_IMM(BPF_LSH, R0, 36), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x9abcdef0 } } - }, - { - "ALU64_LSH_K: Shift == 32, low word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_ALU64_IMM(BPF_LSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } } - }, - { - "ALU64_LSH_K: Shift == 32, high word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_ALU64_IMM(BPF_LSH, R0, 32), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x89abcdef } } - }, - { - "ALU64_LSH_K: Zero shift", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_ALU64_IMM(BPF_LSH, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x89abcdef } } - }, - /* BPF_ALU | BPF_RSH | BPF_X */ - { - "ALU_RSH_X: 2 >> 1 = 1", - .u.insns_int = { - BPF_LD_IMM64(R0, 2), - BPF_ALU32_IMM(BPF_MOV, R1, 1), - BPF_ALU32_REG(BPF_RSH, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "ALU_RSH_X: 0x80000000 >> 31 = 1", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x80000000), - BPF_ALU32_IMM(BPF_MOV, R1, 31), - BPF_ALU32_REG(BPF_RSH, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "ALU_RSH_X: 0x12345678 >> 20 = 0x123", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0x12345678), - BPF_ALU32_IMM(BPF_MOV, R1, 20), - BPF_ALU32_REG(BPF_RSH, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x123 } } - }, - { - "ALU64_RSH_X: 2 >> 1 = 1", - .u.insns_int = { - BPF_LD_IMM64(R0, 2), - BPF_ALU32_IMM(BPF_MOV, R1, 1), - BPF_ALU64_REG(BPF_RSH, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "ALU64_RSH_X: 0x80000000 >> 31 = 1", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x80000000), - BPF_ALU32_IMM(BPF_MOV, R1, 31), - BPF_ALU64_REG(BPF_RSH, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "ALU64_RSH_X: Shift < 32, low word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x8123456789abcdefLL), - BPF_ALU32_IMM(BPF_MOV, R1, 12), - BPF_ALU64_REG(BPF_RSH, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x56789abc } } - }, - { - "ALU64_RSH_X: Shift < 32, high word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x8123456789abcdefLL), - BPF_ALU32_IMM(BPF_MOV, R1, 12), - BPF_ALU64_REG(BPF_RSH, R0, R1), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x00081234 } } - }, - { - "ALU64_RSH_X: Shift > 32, low word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x8123456789abcdefLL), - BPF_ALU32_IMM(BPF_MOV, R1, 36), - BPF_ALU64_REG(BPF_RSH, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x08123456 } } - }, - { - "ALU64_RSH_X: Shift > 32, high word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x8123456789abcdefLL), - BPF_ALU32_IMM(BPF_MOV, R1, 36), - BPF_ALU64_REG(BPF_RSH, R0, R1), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } } - }, - { - "ALU64_RSH_X: Shift == 32, low word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x8123456789abcdefLL), - BPF_ALU32_IMM(BPF_MOV, R1, 32), - BPF_ALU64_REG(BPF_RSH, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x81234567 } } - }, - { - "ALU64_RSH_X: Shift == 32, high word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x8123456789abcdefLL), - BPF_ALU32_IMM(BPF_MOV, R1, 32), - BPF_ALU64_REG(BPF_RSH, R0, R1), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } } - }, - { - "ALU64_RSH_X: Zero shift, low word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x8123456789abcdefLL), - BPF_ALU32_IMM(BPF_MOV, R1, 0), - BPF_ALU64_REG(BPF_RSH, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x89abcdef } } - }, - { - "ALU64_RSH_X: Zero shift, high word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x8123456789abcdefLL), - BPF_ALU32_IMM(BPF_MOV, R1, 0), - BPF_ALU64_REG(BPF_RSH, R0, R1), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x81234567 } } - }, - /* BPF_ALU | BPF_RSH | BPF_K */ - { - "ALU_RSH_K: 2 >> 1 = 1", - .u.insns_int = { - BPF_LD_IMM64(R0, 2), - BPF_ALU32_IMM(BPF_RSH, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "ALU_RSH_K: 0x80000000 >> 31 = 1", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x80000000), - BPF_ALU32_IMM(BPF_RSH, R0, 31), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "ALU_RSH_K: 0x12345678 >> 20 = 0x123", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0x12345678), - BPF_ALU32_IMM(BPF_RSH, R0, 20), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x123 } } - }, - { - "ALU_RSH_K: 0x12345678 >> 0 = 0x12345678", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0x12345678), - BPF_ALU32_IMM(BPF_RSH, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x12345678 } } - }, - { - "ALU64_RSH_K: 2 >> 1 = 1", - .u.insns_int = { - BPF_LD_IMM64(R0, 2), - BPF_ALU64_IMM(BPF_RSH, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "ALU64_RSH_K: 0x80000000 >> 31 = 1", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x80000000), - BPF_ALU64_IMM(BPF_RSH, R0, 31), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "ALU64_RSH_K: Shift < 32, low word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x8123456789abcdefLL), - BPF_ALU64_IMM(BPF_RSH, R0, 12), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x56789abc } } - }, - { - "ALU64_RSH_K: Shift < 32, high word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x8123456789abcdefLL), - BPF_ALU64_IMM(BPF_RSH, R0, 12), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x00081234 } } - }, - { - "ALU64_RSH_K: Shift > 32, low word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x8123456789abcdefLL), - BPF_ALU64_IMM(BPF_RSH, R0, 36), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x08123456 } } - }, - { - "ALU64_RSH_K: Shift > 32, high word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x8123456789abcdefLL), - BPF_ALU64_IMM(BPF_RSH, R0, 36), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } } - }, - { - "ALU64_RSH_K: Shift == 32, low word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x8123456789abcdefLL), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x81234567 } } - }, - { - "ALU64_RSH_K: Shift == 32, high word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x8123456789abcdefLL), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } } - }, - { - "ALU64_RSH_K: Zero shift", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_ALU64_IMM(BPF_RSH, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x89abcdef } } - }, - /* BPF_ALU | BPF_ARSH | BPF_X */ - { - "ALU32_ARSH_X: -1234 >> 7 = -10", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, -1234), - BPF_ALU32_IMM(BPF_MOV, R1, 7), - BPF_ALU32_REG(BPF_ARSH, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -10 } } - }, - { - "ALU64_ARSH_X: 0xff00ff0000000000 >> 40 = 0xffffffffffff00ff", - .u.insns_int = { - BPF_LD_IMM64(R0, 0xff00ff0000000000LL), - BPF_ALU32_IMM(BPF_MOV, R1, 40), - BPF_ALU64_REG(BPF_ARSH, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xffff00ff } }, - }, - { - "ALU64_ARSH_X: Shift < 32, low word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x8123456789abcdefLL), - BPF_ALU32_IMM(BPF_MOV, R1, 12), - BPF_ALU64_REG(BPF_ARSH, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x56789abc } } - }, - { - "ALU64_ARSH_X: Shift < 32, high word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x8123456789abcdefLL), - BPF_ALU32_IMM(BPF_MOV, R1, 12), - BPF_ALU64_REG(BPF_ARSH, R0, R1), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xfff81234 } } - }, - { - "ALU64_ARSH_X: Shift > 32, low word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x8123456789abcdefLL), - BPF_ALU32_IMM(BPF_MOV, R1, 36), - BPF_ALU64_REG(BPF_ARSH, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xf8123456 } } - }, - { - "ALU64_ARSH_X: Shift > 32, high word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x8123456789abcdefLL), - BPF_ALU32_IMM(BPF_MOV, R1, 36), - BPF_ALU64_REG(BPF_ARSH, R0, R1), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -1 } } - }, - { - "ALU64_ARSH_X: Shift == 32, low word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x8123456789abcdefLL), - BPF_ALU32_IMM(BPF_MOV, R1, 32), - BPF_ALU64_REG(BPF_ARSH, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x81234567 } } - }, - { - "ALU64_ARSH_X: Shift == 32, high word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x8123456789abcdefLL), - BPF_ALU32_IMM(BPF_MOV, R1, 32), - BPF_ALU64_REG(BPF_ARSH, R0, R1), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -1 } } - }, - { - "ALU64_ARSH_X: Zero shift, low word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x8123456789abcdefLL), - BPF_ALU32_IMM(BPF_MOV, R1, 0), - BPF_ALU64_REG(BPF_ARSH, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x89abcdef } } - }, - { - "ALU64_ARSH_X: Zero shift, high word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x8123456789abcdefLL), - BPF_ALU32_IMM(BPF_MOV, R1, 0), - BPF_ALU64_REG(BPF_ARSH, R0, R1), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x81234567 } } - }, - /* BPF_ALU | BPF_ARSH | BPF_K */ - { - "ALU32_ARSH_K: -1234 >> 7 = -10", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, -1234), - BPF_ALU32_IMM(BPF_ARSH, R0, 7), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -10 } } - }, - { - "ALU32_ARSH_K: -1234 >> 0 = -1234", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, -1234), - BPF_ALU32_IMM(BPF_ARSH, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -1234 } } - }, - { - "ALU64_ARSH_K: 0xff00ff0000000000 >> 40 = 0xffffffffffff00ff", - .u.insns_int = { - BPF_LD_IMM64(R0, 0xff00ff0000000000LL), - BPF_ALU64_IMM(BPF_ARSH, R0, 40), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xffff00ff } }, - }, - { - "ALU64_ARSH_K: Shift < 32, low word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x8123456789abcdefLL), - BPF_ALU64_IMM(BPF_RSH, R0, 12), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x56789abc } } - }, - { - "ALU64_ARSH_K: Shift < 32, high word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x8123456789abcdefLL), - BPF_ALU64_IMM(BPF_ARSH, R0, 12), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xfff81234 } } - }, - { - "ALU64_ARSH_K: Shift > 32, low word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x8123456789abcdefLL), - BPF_ALU64_IMM(BPF_ARSH, R0, 36), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xf8123456 } } - }, - { - "ALU64_ARSH_K: Shift > 32, high word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0xf123456789abcdefLL), - BPF_ALU64_IMM(BPF_ARSH, R0, 36), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -1 } } - }, - { - "ALU64_ARSH_K: Shift == 32, low word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x8123456789abcdefLL), - BPF_ALU64_IMM(BPF_ARSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x81234567 } } - }, - { - "ALU64_ARSH_K: Shift == 32, high word", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x8123456789abcdefLL), - BPF_ALU64_IMM(BPF_ARSH, R0, 32), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -1 } } - }, - { - "ALU64_ARSH_K: Zero shift", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x8123456789abcdefLL), - BPF_ALU64_IMM(BPF_ARSH, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x89abcdef } } - }, - /* BPF_ALU | BPF_NEG */ - { - "ALU_NEG: -(3) = -3", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 3), - BPF_ALU32_IMM(BPF_NEG, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -3 } }, - }, - { - "ALU_NEG: -(-3) = 3", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, -3), - BPF_ALU32_IMM(BPF_NEG, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 3 } }, - }, - { - "ALU64_NEG: -(3) = -3", - .u.insns_int = { - BPF_LD_IMM64(R0, 3), - BPF_ALU64_IMM(BPF_NEG, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -3 } }, - }, - { - "ALU64_NEG: -(-3) = 3", - .u.insns_int = { - BPF_LD_IMM64(R0, -3), - BPF_ALU64_IMM(BPF_NEG, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 3 } }, - }, - /* BPF_ALU | BPF_END | BPF_FROM_BE */ - { - "ALU_END_FROM_BE 16: 0x0123456789abcdef -> 0xcdef", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_ENDIAN(BPF_FROM_BE, R0, 16), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, cpu_to_be16(0xcdef) } }, - }, - { - "ALU_END_FROM_BE 32: 0x0123456789abcdef -> 0x89abcdef", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_ENDIAN(BPF_FROM_BE, R0, 32), - BPF_ALU64_REG(BPF_MOV, R1, R0), - BPF_ALU64_IMM(BPF_RSH, R1, 32), - BPF_ALU32_REG(BPF_ADD, R0, R1), /* R1 = 0 */ - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, cpu_to_be32(0x89abcdef) } }, - }, - { - "ALU_END_FROM_BE 64: 0x0123456789abcdef -> 0x89abcdef", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_ENDIAN(BPF_FROM_BE, R0, 64), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, (u32) cpu_to_be64(0x0123456789abcdefLL) } }, - }, - { - "ALU_END_FROM_BE 64: 0x0123456789abcdef >> 32 -> 0x01234567", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_ENDIAN(BPF_FROM_BE, R0, 64), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, (u32) (cpu_to_be64(0x0123456789abcdefLL) >> 32) } }, - }, - /* BPF_ALU | BPF_END | BPF_FROM_BE, reversed */ - { - "ALU_END_FROM_BE 16: 0xfedcba9876543210 -> 0x3210", - .u.insns_int = { - BPF_LD_IMM64(R0, 0xfedcba9876543210ULL), - BPF_ENDIAN(BPF_FROM_BE, R0, 16), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, cpu_to_be16(0x3210) } }, - }, - { - "ALU_END_FROM_BE 32: 0xfedcba9876543210 -> 0x76543210", - .u.insns_int = { - BPF_LD_IMM64(R0, 0xfedcba9876543210ULL), - BPF_ENDIAN(BPF_FROM_BE, R0, 32), - BPF_ALU64_REG(BPF_MOV, R1, R0), - BPF_ALU64_IMM(BPF_RSH, R1, 32), - BPF_ALU32_REG(BPF_ADD, R0, R1), /* R1 = 0 */ - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, cpu_to_be32(0x76543210) } }, - }, - { - "ALU_END_FROM_BE 64: 0xfedcba9876543210 -> 0x76543210", - .u.insns_int = { - BPF_LD_IMM64(R0, 0xfedcba9876543210ULL), - BPF_ENDIAN(BPF_FROM_BE, R0, 64), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, (u32) cpu_to_be64(0xfedcba9876543210ULL) } }, - }, - { - "ALU_END_FROM_BE 64: 0xfedcba9876543210 >> 32 -> 0xfedcba98", - .u.insns_int = { - BPF_LD_IMM64(R0, 0xfedcba9876543210ULL), - BPF_ENDIAN(BPF_FROM_BE, R0, 64), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, (u32) (cpu_to_be64(0xfedcba9876543210ULL) >> 32) } }, - }, - /* BPF_ALU | BPF_END | BPF_FROM_LE */ - { - "ALU_END_FROM_LE 16: 0x0123456789abcdef -> 0xefcd", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_ENDIAN(BPF_FROM_LE, R0, 16), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, cpu_to_le16(0xcdef) } }, - }, - { - "ALU_END_FROM_LE 32: 0x0123456789abcdef -> 0xefcdab89", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_ENDIAN(BPF_FROM_LE, R0, 32), - BPF_ALU64_REG(BPF_MOV, R1, R0), - BPF_ALU64_IMM(BPF_RSH, R1, 32), - BPF_ALU32_REG(BPF_ADD, R0, R1), /* R1 = 0 */ - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, cpu_to_le32(0x89abcdef) } }, - }, - { - "ALU_END_FROM_LE 64: 0x0123456789abcdef -> 0x67452301", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_ENDIAN(BPF_FROM_LE, R0, 64), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, (u32) cpu_to_le64(0x0123456789abcdefLL) } }, - }, - { - "ALU_END_FROM_LE 64: 0x0123456789abcdef >> 32 -> 0xefcdab89", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_ENDIAN(BPF_FROM_LE, R0, 64), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, (u32) (cpu_to_le64(0x0123456789abcdefLL) >> 32) } }, - }, - /* BPF_ALU | BPF_END | BPF_FROM_LE, reversed */ - { - "ALU_END_FROM_LE 16: 0xfedcba9876543210 -> 0x1032", - .u.insns_int = { - BPF_LD_IMM64(R0, 0xfedcba9876543210ULL), - BPF_ENDIAN(BPF_FROM_LE, R0, 16), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, cpu_to_le16(0x3210) } }, - }, - { - "ALU_END_FROM_LE 32: 0xfedcba9876543210 -> 0x10325476", - .u.insns_int = { - BPF_LD_IMM64(R0, 0xfedcba9876543210ULL), - BPF_ENDIAN(BPF_FROM_LE, R0, 32), - BPF_ALU64_REG(BPF_MOV, R1, R0), - BPF_ALU64_IMM(BPF_RSH, R1, 32), - BPF_ALU32_REG(BPF_ADD, R0, R1), /* R1 = 0 */ - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, cpu_to_le32(0x76543210) } }, - }, - { - "ALU_END_FROM_LE 64: 0xfedcba9876543210 -> 0x10325476", - .u.insns_int = { - BPF_LD_IMM64(R0, 0xfedcba9876543210ULL), - BPF_ENDIAN(BPF_FROM_LE, R0, 64), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, (u32) cpu_to_le64(0xfedcba9876543210ULL) } }, - }, - { - "ALU_END_FROM_LE 64: 0xfedcba9876543210 >> 32 -> 0x98badcfe", - .u.insns_int = { - BPF_LD_IMM64(R0, 0xfedcba9876543210ULL), - BPF_ENDIAN(BPF_FROM_LE, R0, 64), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, (u32) (cpu_to_le64(0xfedcba9876543210ULL) >> 32) } }, - }, - /* BSWAP */ - { - "BSWAP 16: 0x0123456789abcdef -> 0xefcd", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_BSWAP(R0, 16), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xefcd } }, - }, - { - "BSWAP 32: 0x0123456789abcdef -> 0xefcdab89", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_BSWAP(R0, 32), - BPF_ALU64_REG(BPF_MOV, R1, R0), - BPF_ALU64_IMM(BPF_RSH, R1, 32), - BPF_ALU32_REG(BPF_ADD, R0, R1), /* R1 = 0 */ - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xefcdab89 } }, - }, - { - "BSWAP 64: 0x0123456789abcdef -> 0x67452301", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_BSWAP(R0, 64), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x67452301 } }, - }, - { - "BSWAP 64: 0x0123456789abcdef >> 32 -> 0xefcdab89", - .u.insns_int = { - BPF_LD_IMM64(R0, 0x0123456789abcdefLL), - BPF_BSWAP(R0, 64), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xefcdab89 } }, - }, - /* BSWAP, reversed */ - { - "BSWAP 16: 0xfedcba9876543210 -> 0x1032", - .u.insns_int = { - BPF_LD_IMM64(R0, 0xfedcba9876543210ULL), - BPF_BSWAP(R0, 16), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1032 } }, - }, - { - "BSWAP 32: 0xfedcba9876543210 -> 0x10325476", - .u.insns_int = { - BPF_LD_IMM64(R0, 0xfedcba9876543210ULL), - BPF_BSWAP(R0, 32), - BPF_ALU64_REG(BPF_MOV, R1, R0), - BPF_ALU64_IMM(BPF_RSH, R1, 32), - BPF_ALU32_REG(BPF_ADD, R0, R1), /* R1 = 0 */ - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x10325476 } }, - }, - { - "BSWAP 64: 0xfedcba9876543210 -> 0x98badcfe", - .u.insns_int = { - BPF_LD_IMM64(R0, 0xfedcba9876543210ULL), - BPF_BSWAP(R0, 64), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x98badcfe } }, - }, - { - "BSWAP 64: 0xfedcba9876543210 >> 32 -> 0x10325476", - .u.insns_int = { - BPF_LD_IMM64(R0, 0xfedcba9876543210ULL), - BPF_BSWAP(R0, 64), - BPF_ALU64_IMM(BPF_RSH, R0, 32), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x10325476 } }, - }, - /* BPF_LDX_MEM B/H/W/DW */ - { - "BPF_LDX_MEM | BPF_B, base", - .u.insns_int = { - BPF_LD_IMM64(R1, 0x0102030405060708ULL), - BPF_LD_IMM64(R2, 0x0000000000000008ULL), - BPF_STX_MEM(BPF_DW, R10, R1, -8), -#ifdef __BIG_ENDIAN - BPF_LDX_MEM(BPF_B, R0, R10, -1), -#else - BPF_LDX_MEM(BPF_B, R0, R10, -8), -#endif - BPF_JMP_REG(BPF_JNE, R0, R2, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } }, - .stack_depth = 8, - }, - { - "BPF_LDX_MEM | BPF_B, MSB set", - .u.insns_int = { - BPF_LD_IMM64(R1, 0x8182838485868788ULL), - BPF_LD_IMM64(R2, 0x0000000000000088ULL), - BPF_STX_MEM(BPF_DW, R10, R1, -8), -#ifdef __BIG_ENDIAN - BPF_LDX_MEM(BPF_B, R0, R10, -1), -#else - BPF_LDX_MEM(BPF_B, R0, R10, -8), -#endif - BPF_JMP_REG(BPF_JNE, R0, R2, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } }, - .stack_depth = 8, - }, - { - "BPF_LDX_MEM | BPF_B, negative offset", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x8182838485868788ULL), - BPF_LD_IMM64(R3, 0x0000000000000088ULL), - BPF_ALU64_IMM(BPF_ADD, R1, 512), - BPF_STX_MEM(BPF_B, R1, R2, -256), - BPF_LDX_MEM(BPF_B, R0, R1, -256), - BPF_JMP_REG(BPF_JNE, R0, R3, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_LARGE_MEM, - { }, - { { 512, 0 } }, - .stack_depth = 0, - }, - { - "BPF_LDX_MEM | BPF_B, small positive offset", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x8182838485868788ULL), - BPF_LD_IMM64(R3, 0x0000000000000088ULL), - BPF_STX_MEM(BPF_B, R1, R2, 256), - BPF_LDX_MEM(BPF_B, R0, R1, 256), - BPF_JMP_REG(BPF_JNE, R0, R3, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_LARGE_MEM, - { }, - { { 512, 0 } }, - .stack_depth = 0, - }, - { - "BPF_LDX_MEM | BPF_B, large positive offset", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x8182838485868788ULL), - BPF_LD_IMM64(R3, 0x0000000000000088ULL), - BPF_STX_MEM(BPF_B, R1, R2, 4096), - BPF_LDX_MEM(BPF_B, R0, R1, 4096), - BPF_JMP_REG(BPF_JNE, R0, R3, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_LARGE_MEM, - { }, - { { 4096 + 16, 0 } }, - .stack_depth = 0, - }, - { - "BPF_LDX_MEM | BPF_H, base", - .u.insns_int = { - BPF_LD_IMM64(R1, 0x0102030405060708ULL), - BPF_LD_IMM64(R2, 0x0000000000000708ULL), - BPF_STX_MEM(BPF_DW, R10, R1, -8), -#ifdef __BIG_ENDIAN - BPF_LDX_MEM(BPF_H, R0, R10, -2), -#else - BPF_LDX_MEM(BPF_H, R0, R10, -8), -#endif - BPF_JMP_REG(BPF_JNE, R0, R2, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } }, - .stack_depth = 8, - }, - { - "BPF_LDX_MEM | BPF_H, MSB set", - .u.insns_int = { - BPF_LD_IMM64(R1, 0x8182838485868788ULL), - BPF_LD_IMM64(R2, 0x0000000000008788ULL), - BPF_STX_MEM(BPF_DW, R10, R1, -8), -#ifdef __BIG_ENDIAN - BPF_LDX_MEM(BPF_H, R0, R10, -2), -#else - BPF_LDX_MEM(BPF_H, R0, R10, -8), -#endif - BPF_JMP_REG(BPF_JNE, R0, R2, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } }, - .stack_depth = 8, - }, - { - "BPF_LDX_MEM | BPF_H, negative offset", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x8182838485868788ULL), - BPF_LD_IMM64(R3, 0x0000000000008788ULL), - BPF_ALU64_IMM(BPF_ADD, R1, 512), - BPF_STX_MEM(BPF_H, R1, R2, -256), - BPF_LDX_MEM(BPF_H, R0, R1, -256), - BPF_JMP_REG(BPF_JNE, R0, R3, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_LARGE_MEM, - { }, - { { 512, 0 } }, - .stack_depth = 0, - }, - { - "BPF_LDX_MEM | BPF_H, small positive offset", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x8182838485868788ULL), - BPF_LD_IMM64(R3, 0x0000000000008788ULL), - BPF_STX_MEM(BPF_H, R1, R2, 256), - BPF_LDX_MEM(BPF_H, R0, R1, 256), - BPF_JMP_REG(BPF_JNE, R0, R3, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_LARGE_MEM, - { }, - { { 512, 0 } }, - .stack_depth = 0, - }, - { - "BPF_LDX_MEM | BPF_H, large positive offset", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x8182838485868788ULL), - BPF_LD_IMM64(R3, 0x0000000000008788ULL), - BPF_STX_MEM(BPF_H, R1, R2, 8192), - BPF_LDX_MEM(BPF_H, R0, R1, 8192), - BPF_JMP_REG(BPF_JNE, R0, R3, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_LARGE_MEM, - { }, - { { 8192 + 16, 0 } }, - .stack_depth = 0, - }, - { - "BPF_LDX_MEM | BPF_H, unaligned positive offset", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x8182838485868788ULL), - BPF_LD_IMM64(R3, 0x0000000000008788ULL), - BPF_STX_MEM(BPF_H, R1, R2, 13), - BPF_LDX_MEM(BPF_H, R0, R1, 13), - BPF_JMP_REG(BPF_JNE, R0, R3, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_LARGE_MEM, - { }, - { { 32, 0 } }, - .stack_depth = 0, - }, - { - "BPF_LDX_MEM | BPF_W, base", - .u.insns_int = { - BPF_LD_IMM64(R1, 0x0102030405060708ULL), - BPF_LD_IMM64(R2, 0x0000000005060708ULL), - BPF_STX_MEM(BPF_DW, R10, R1, -8), -#ifdef __BIG_ENDIAN - BPF_LDX_MEM(BPF_W, R0, R10, -4), -#else - BPF_LDX_MEM(BPF_W, R0, R10, -8), -#endif - BPF_JMP_REG(BPF_JNE, R0, R2, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } }, - .stack_depth = 8, - }, - { - "BPF_LDX_MEM | BPF_W, MSB set", - .u.insns_int = { - BPF_LD_IMM64(R1, 0x8182838485868788ULL), - BPF_LD_IMM64(R2, 0x0000000085868788ULL), - BPF_STX_MEM(BPF_DW, R10, R1, -8), -#ifdef __BIG_ENDIAN - BPF_LDX_MEM(BPF_W, R0, R10, -4), -#else - BPF_LDX_MEM(BPF_W, R0, R10, -8), -#endif - BPF_JMP_REG(BPF_JNE, R0, R2, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } }, - .stack_depth = 8, - }, - { - "BPF_LDX_MEM | BPF_W, negative offset", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x8182838485868788ULL), - BPF_LD_IMM64(R3, 0x0000000085868788ULL), - BPF_ALU64_IMM(BPF_ADD, R1, 512), - BPF_STX_MEM(BPF_W, R1, R2, -256), - BPF_LDX_MEM(BPF_W, R0, R1, -256), - BPF_JMP_REG(BPF_JNE, R0, R3, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_LARGE_MEM, - { }, - { { 512, 0 } }, - .stack_depth = 0, - }, - { - "BPF_LDX_MEM | BPF_W, small positive offset", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x8182838485868788ULL), - BPF_LD_IMM64(R3, 0x0000000085868788ULL), - BPF_STX_MEM(BPF_W, R1, R2, 256), - BPF_LDX_MEM(BPF_W, R0, R1, 256), - BPF_JMP_REG(BPF_JNE, R0, R3, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_LARGE_MEM, - { }, - { { 512, 0 } }, - .stack_depth = 0, - }, - { - "BPF_LDX_MEM | BPF_W, large positive offset", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x8182838485868788ULL), - BPF_LD_IMM64(R3, 0x0000000085868788ULL), - BPF_STX_MEM(BPF_W, R1, R2, 16384), - BPF_LDX_MEM(BPF_W, R0, R1, 16384), - BPF_JMP_REG(BPF_JNE, R0, R3, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_LARGE_MEM, - { }, - { { 16384 + 16, 0 } }, - .stack_depth = 0, - }, - { - "BPF_LDX_MEM | BPF_W, unaligned positive offset", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x8182838485868788ULL), - BPF_LD_IMM64(R3, 0x0000000085868788ULL), - BPF_STX_MEM(BPF_W, R1, R2, 13), - BPF_LDX_MEM(BPF_W, R0, R1, 13), - BPF_JMP_REG(BPF_JNE, R0, R3, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_LARGE_MEM, - { }, - { { 32, 0 } }, - .stack_depth = 0, - }, - { - "BPF_LDX_MEM | BPF_DW, base", - .u.insns_int = { - BPF_LD_IMM64(R1, 0x0102030405060708ULL), - BPF_STX_MEM(BPF_DW, R10, R1, -8), - BPF_LDX_MEM(BPF_DW, R0, R10, -8), - BPF_JMP_REG(BPF_JNE, R0, R1, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } }, - .stack_depth = 8, - }, - { - "BPF_LDX_MEM | BPF_DW, MSB set", - .u.insns_int = { - BPF_LD_IMM64(R1, 0x8182838485868788ULL), - BPF_STX_MEM(BPF_DW, R10, R1, -8), - BPF_LDX_MEM(BPF_DW, R0, R10, -8), - BPF_JMP_REG(BPF_JNE, R0, R1, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } }, - .stack_depth = 8, - }, - { - "BPF_LDX_MEM | BPF_DW, negative offset", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x8182838485868788ULL), - BPF_ALU64_IMM(BPF_ADD, R1, 512), - BPF_STX_MEM(BPF_DW, R1, R2, -256), - BPF_LDX_MEM(BPF_DW, R0, R1, -256), - BPF_JMP_REG(BPF_JNE, R0, R2, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_LARGE_MEM, - { }, - { { 512, 0 } }, - .stack_depth = 0, - }, - { - "BPF_LDX_MEM | BPF_DW, small positive offset", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x8182838485868788ULL), - BPF_STX_MEM(BPF_DW, R1, R2, 256), - BPF_LDX_MEM(BPF_DW, R0, R1, 256), - BPF_JMP_REG(BPF_JNE, R0, R2, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_LARGE_MEM, - { }, - { { 512, 0 } }, - .stack_depth = 8, - }, - { - "BPF_LDX_MEM | BPF_DW, large positive offset", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x8182838485868788ULL), - BPF_STX_MEM(BPF_DW, R1, R2, 32760), - BPF_LDX_MEM(BPF_DW, R0, R1, 32760), - BPF_JMP_REG(BPF_JNE, R0, R2, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_LARGE_MEM, - { }, - { { 32768, 0 } }, - .stack_depth = 0, - }, - { - "BPF_LDX_MEM | BPF_DW, unaligned positive offset", - .u.insns_int = { - BPF_LD_IMM64(R2, 0x8182838485868788ULL), - BPF_STX_MEM(BPF_DW, R1, R2, 13), - BPF_LDX_MEM(BPF_DW, R0, R1, 13), - BPF_JMP_REG(BPF_JNE, R0, R2, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_LARGE_MEM, - { }, - { { 32, 0 } }, - .stack_depth = 0, - }, - /* BPF_LDX_MEMSX B/H/W */ - { - "BPF_LDX_MEMSX | BPF_B", - .u.insns_int = { - BPF_LD_IMM64(R1, 0xdead0000000000f0ULL), - BPF_LD_IMM64(R2, 0xfffffffffffffff0ULL), - BPF_STX_MEM(BPF_DW, R10, R1, -8), -#ifdef __BIG_ENDIAN - BPF_LDX_MEMSX(BPF_B, R0, R10, -1), -#else - BPF_LDX_MEMSX(BPF_B, R0, R10, -8), -#endif - BPF_JMP_REG(BPF_JNE, R0, R2, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } }, - .stack_depth = 8, - }, - { - "BPF_LDX_MEMSX | BPF_H", - .u.insns_int = { - BPF_LD_IMM64(R1, 0xdead00000000f123ULL), - BPF_LD_IMM64(R2, 0xfffffffffffff123ULL), - BPF_STX_MEM(BPF_DW, R10, R1, -8), -#ifdef __BIG_ENDIAN - BPF_LDX_MEMSX(BPF_H, R0, R10, -2), -#else - BPF_LDX_MEMSX(BPF_H, R0, R10, -8), -#endif - BPF_JMP_REG(BPF_JNE, R0, R2, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } }, - .stack_depth = 8, - }, - { - "BPF_LDX_MEMSX | BPF_W", - .u.insns_int = { - BPF_LD_IMM64(R1, 0x00000000deadbeefULL), - BPF_LD_IMM64(R2, 0xffffffffdeadbeefULL), - BPF_STX_MEM(BPF_DW, R10, R1, -8), -#ifdef __BIG_ENDIAN - BPF_LDX_MEMSX(BPF_W, R0, R10, -4), -#else - BPF_LDX_MEMSX(BPF_W, R0, R10, -8), -#endif - BPF_JMP_REG(BPF_JNE, R0, R2, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } }, - .stack_depth = 8, - }, - /* BPF_STX_MEM B/H/W/DW */ - { - "BPF_STX_MEM | BPF_B", - .u.insns_int = { - BPF_LD_IMM64(R1, 0x8090a0b0c0d0e0f0ULL), - BPF_LD_IMM64(R2, 0x0102030405060708ULL), - BPF_LD_IMM64(R3, 0x8090a0b0c0d0e008ULL), - BPF_STX_MEM(BPF_DW, R10, R1, -8), -#ifdef __BIG_ENDIAN - BPF_STX_MEM(BPF_B, R10, R2, -1), -#else - BPF_STX_MEM(BPF_B, R10, R2, -8), -#endif - BPF_LDX_MEM(BPF_DW, R0, R10, -8), - BPF_JMP_REG(BPF_JNE, R0, R3, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } }, - .stack_depth = 8, - }, - { - "BPF_STX_MEM | BPF_B, MSB set", - .u.insns_int = { - BPF_LD_IMM64(R1, 0x8090a0b0c0d0e0f0ULL), - BPF_LD_IMM64(R2, 0x8182838485868788ULL), - BPF_LD_IMM64(R3, 0x8090a0b0c0d0e088ULL), - BPF_STX_MEM(BPF_DW, R10, R1, -8), -#ifdef __BIG_ENDIAN - BPF_STX_MEM(BPF_B, R10, R2, -1), -#else - BPF_STX_MEM(BPF_B, R10, R2, -8), -#endif - BPF_LDX_MEM(BPF_DW, R0, R10, -8), - BPF_JMP_REG(BPF_JNE, R0, R3, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } }, - .stack_depth = 8, - }, - { - "BPF_STX_MEM | BPF_H", - .u.insns_int = { - BPF_LD_IMM64(R1, 0x8090a0b0c0d0e0f0ULL), - BPF_LD_IMM64(R2, 0x0102030405060708ULL), - BPF_LD_IMM64(R3, 0x8090a0b0c0d00708ULL), - BPF_STX_MEM(BPF_DW, R10, R1, -8), -#ifdef __BIG_ENDIAN - BPF_STX_MEM(BPF_H, R10, R2, -2), -#else - BPF_STX_MEM(BPF_H, R10, R2, -8), -#endif - BPF_LDX_MEM(BPF_DW, R0, R10, -8), - BPF_JMP_REG(BPF_JNE, R0, R3, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } }, - .stack_depth = 8, - }, - { - "BPF_STX_MEM | BPF_H, MSB set", - .u.insns_int = { - BPF_LD_IMM64(R1, 0x8090a0b0c0d0e0f0ULL), - BPF_LD_IMM64(R2, 0x8182838485868788ULL), - BPF_LD_IMM64(R3, 0x8090a0b0c0d08788ULL), - BPF_STX_MEM(BPF_DW, R10, R1, -8), -#ifdef __BIG_ENDIAN - BPF_STX_MEM(BPF_H, R10, R2, -2), -#else - BPF_STX_MEM(BPF_H, R10, R2, -8), -#endif - BPF_LDX_MEM(BPF_DW, R0, R10, -8), - BPF_JMP_REG(BPF_JNE, R0, R3, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } }, - .stack_depth = 8, - }, - { - "BPF_STX_MEM | BPF_W", - .u.insns_int = { - BPF_LD_IMM64(R1, 0x8090a0b0c0d0e0f0ULL), - BPF_LD_IMM64(R2, 0x0102030405060708ULL), - BPF_LD_IMM64(R3, 0x8090a0b005060708ULL), - BPF_STX_MEM(BPF_DW, R10, R1, -8), -#ifdef __BIG_ENDIAN - BPF_STX_MEM(BPF_W, R10, R2, -4), -#else - BPF_STX_MEM(BPF_W, R10, R2, -8), -#endif - BPF_LDX_MEM(BPF_DW, R0, R10, -8), - BPF_JMP_REG(BPF_JNE, R0, R3, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } }, - .stack_depth = 8, - }, - { - "BPF_STX_MEM | BPF_W, MSB set", - .u.insns_int = { - BPF_LD_IMM64(R1, 0x8090a0b0c0d0e0f0ULL), - BPF_LD_IMM64(R2, 0x8182838485868788ULL), - BPF_LD_IMM64(R3, 0x8090a0b085868788ULL), - BPF_STX_MEM(BPF_DW, R10, R1, -8), -#ifdef __BIG_ENDIAN - BPF_STX_MEM(BPF_W, R10, R2, -4), -#else - BPF_STX_MEM(BPF_W, R10, R2, -8), -#endif - BPF_LDX_MEM(BPF_DW, R0, R10, -8), - BPF_JMP_REG(BPF_JNE, R0, R3, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } }, - .stack_depth = 8, - }, - /* BPF_ST(X) | BPF_MEM | BPF_B/H/W/DW */ - { - "ST_MEM_B: Store/Load byte: max negative", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_ST_MEM(BPF_B, R10, -40, 0xff), - BPF_LDX_MEM(BPF_B, R0, R10, -40), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xff } }, - .stack_depth = 40, - }, - { - "ST_MEM_B: Store/Load byte: max positive", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_ST_MEM(BPF_H, R10, -40, 0x7f), - BPF_LDX_MEM(BPF_H, R0, R10, -40), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x7f } }, - .stack_depth = 40, - }, - { - "STX_MEM_B: Store/Load byte: max negative", - .u.insns_int = { - BPF_LD_IMM64(R0, 0), - BPF_LD_IMM64(R1, 0xffLL), - BPF_STX_MEM(BPF_B, R10, R1, -40), - BPF_LDX_MEM(BPF_B, R0, R10, -40), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xff } }, - .stack_depth = 40, - }, - { - "ST_MEM_H: Store/Load half word: max negative", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_ST_MEM(BPF_H, R10, -40, 0xffff), - BPF_LDX_MEM(BPF_H, R0, R10, -40), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xffff } }, - .stack_depth = 40, - }, - { - "ST_MEM_H: Store/Load half word: max positive", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_ST_MEM(BPF_H, R10, -40, 0x7fff), - BPF_LDX_MEM(BPF_H, R0, R10, -40), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x7fff } }, - .stack_depth = 40, - }, - { - "STX_MEM_H: Store/Load half word: max negative", - .u.insns_int = { - BPF_LD_IMM64(R0, 0), - BPF_LD_IMM64(R1, 0xffffLL), - BPF_STX_MEM(BPF_H, R10, R1, -40), - BPF_LDX_MEM(BPF_H, R0, R10, -40), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xffff } }, - .stack_depth = 40, - }, - { - "ST_MEM_W: Store/Load word: max negative", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_ST_MEM(BPF_W, R10, -40, 0xffffffff), - BPF_LDX_MEM(BPF_W, R0, R10, -40), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xffffffff } }, - .stack_depth = 40, - }, - { - "ST_MEM_W: Store/Load word: max positive", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_ST_MEM(BPF_W, R10, -40, 0x7fffffff), - BPF_LDX_MEM(BPF_W, R0, R10, -40), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x7fffffff } }, - .stack_depth = 40, - }, - { - "STX_MEM_W: Store/Load word: max negative", - .u.insns_int = { - BPF_LD_IMM64(R0, 0), - BPF_LD_IMM64(R1, 0xffffffffLL), - BPF_STX_MEM(BPF_W, R10, R1, -40), - BPF_LDX_MEM(BPF_W, R0, R10, -40), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xffffffff } }, - .stack_depth = 40, - }, - { - "ST_MEM_DW: Store/Load double word: max negative", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_ST_MEM(BPF_DW, R10, -40, 0xffffffff), - BPF_LDX_MEM(BPF_DW, R0, R10, -40), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xffffffff } }, - .stack_depth = 40, - }, - { - "ST_MEM_DW: Store/Load double word: max negative 2", - .u.insns_int = { - BPF_LD_IMM64(R2, 0xffff00000000ffffLL), - BPF_LD_IMM64(R3, 0xffffffffffffffffLL), - BPF_ST_MEM(BPF_DW, R10, -40, 0xffffffff), - BPF_LDX_MEM(BPF_DW, R2, R10, -40), - BPF_JMP_REG(BPF_JEQ, R2, R3, 2), - BPF_MOV32_IMM(R0, 2), - BPF_EXIT_INSN(), - BPF_MOV32_IMM(R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x1 } }, - .stack_depth = 40, - }, - { - "ST_MEM_DW: Store/Load double word: max positive", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_ST_MEM(BPF_DW, R10, -40, 0x7fffffff), - BPF_LDX_MEM(BPF_DW, R0, R10, -40), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x7fffffff } }, - .stack_depth = 40, - }, - { - "STX_MEM_DW: Store/Load double word: max negative", - .u.insns_int = { - BPF_LD_IMM64(R0, 0), - BPF_LD_IMM64(R1, 0xffffffffffffffffLL), - BPF_STX_MEM(BPF_DW, R10, R1, -40), - BPF_LDX_MEM(BPF_DW, R0, R10, -40), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xffffffff } }, - .stack_depth = 40, - }, - { - "STX_MEM_DW: Store double word: first word in memory", - .u.insns_int = { - BPF_LD_IMM64(R0, 0), - BPF_LD_IMM64(R1, 0x0123456789abcdefLL), - BPF_STX_MEM(BPF_DW, R10, R1, -40), - BPF_LDX_MEM(BPF_W, R0, R10, -40), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, -#ifdef __BIG_ENDIAN - { { 0, 0x01234567 } }, -#else - { { 0, 0x89abcdef } }, -#endif - .stack_depth = 40, - }, - { - "STX_MEM_DW: Store double word: second word in memory", - .u.insns_int = { - BPF_LD_IMM64(R0, 0), - BPF_LD_IMM64(R1, 0x0123456789abcdefLL), - BPF_STX_MEM(BPF_DW, R10, R1, -40), - BPF_LDX_MEM(BPF_W, R0, R10, -36), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, -#ifdef __BIG_ENDIAN - { { 0, 0x89abcdef } }, -#else - { { 0, 0x01234567 } }, -#endif - .stack_depth = 40, - }, - /* BPF_STX | BPF_ATOMIC | BPF_W/DW */ - { - "STX_XADD_W: X + 1 + 1 + 1 + ...", - { }, - INTERNAL, - { }, - { { 0, 4134 } }, - .fill_helper = bpf_fill_stxw, - }, - { - "STX_XADD_DW: X + 1 + 1 + 1 + ...", - { }, - INTERNAL, - { }, - { { 0, 4134 } }, - .fill_helper = bpf_fill_stxdw, - }, - /* - * Exhaustive tests of atomic operation variants. - * Individual tests are expanded from template macros for all - * combinations of ALU operation, word size and fetching. - */ -#define BPF_ATOMIC_POISON(width) ((width) == BPF_W ? (0xbaadf00dULL << 32) : 0) - -#define BPF_ATOMIC_OP_TEST1(width, op, logic, old, update, result) \ -{ \ - "BPF_ATOMIC | " #width ", " #op ": Test: " \ - #old " " #logic " " #update " = " #result, \ - .u.insns_int = { \ - BPF_LD_IMM64(R5, (update) | BPF_ATOMIC_POISON(width)), \ - BPF_ST_MEM(width, R10, -40, old), \ - BPF_ATOMIC_OP(width, op, R10, R5, -40), \ - BPF_LDX_MEM(width, R0, R10, -40), \ - BPF_ALU64_REG(BPF_MOV, R1, R0), \ - BPF_ALU64_IMM(BPF_RSH, R1, 32), \ - BPF_ALU64_REG(BPF_OR, R0, R1), \ - BPF_EXIT_INSN(), \ - }, \ - INTERNAL, \ - { }, \ - { { 0, result } }, \ - .stack_depth = 40, \ -} -#define BPF_ATOMIC_OP_TEST2(width, op, logic, old, update, result) \ -{ \ - "BPF_ATOMIC | " #width ", " #op ": Test side effects, r10: " \ - #old " " #logic " " #update " = " #result, \ - .u.insns_int = { \ - BPF_ALU64_REG(BPF_MOV, R1, R10), \ - BPF_LD_IMM64(R0, (update) | BPF_ATOMIC_POISON(width)), \ - BPF_ST_MEM(BPF_W, R10, -40, old), \ - BPF_ATOMIC_OP(width, op, R10, R0, -40), \ - BPF_ALU64_REG(BPF_MOV, R0, R10), \ - BPF_ALU64_REG(BPF_SUB, R0, R1), \ - BPF_ALU64_REG(BPF_MOV, R1, R0), \ - BPF_ALU64_IMM(BPF_RSH, R1, 32), \ - BPF_ALU64_REG(BPF_OR, R0, R1), \ - BPF_EXIT_INSN(), \ - }, \ - INTERNAL, \ - { }, \ - { { 0, 0 } }, \ - .stack_depth = 40, \ -} -#define BPF_ATOMIC_OP_TEST3(width, op, logic, old, update, result) \ -{ \ - "BPF_ATOMIC | " #width ", " #op ": Test side effects, r0: " \ - #old " " #logic " " #update " = " #result, \ - .u.insns_int = { \ - BPF_ALU64_REG(BPF_MOV, R0, R10), \ - BPF_LD_IMM64(R1, (update) | BPF_ATOMIC_POISON(width)), \ - BPF_ST_MEM(width, R10, -40, old), \ - BPF_ATOMIC_OP(width, op, R10, R1, -40), \ - BPF_ALU64_REG(BPF_SUB, R0, R10), \ - BPF_ALU64_REG(BPF_MOV, R1, R0), \ - BPF_ALU64_IMM(BPF_RSH, R1, 32), \ - BPF_ALU64_REG(BPF_OR, R0, R1), \ - BPF_EXIT_INSN(), \ - }, \ - INTERNAL, \ - { }, \ - { { 0, 0 } }, \ - .stack_depth = 40, \ -} -#define BPF_ATOMIC_OP_TEST4(width, op, logic, old, update, result) \ -{ \ - "BPF_ATOMIC | " #width ", " #op ": Test fetch: " \ - #old " " #logic " " #update " = " #result, \ - .u.insns_int = { \ - BPF_LD_IMM64(R3, (update) | BPF_ATOMIC_POISON(width)), \ - BPF_ST_MEM(width, R10, -40, old), \ - BPF_ATOMIC_OP(width, op, R10, R3, -40), \ - BPF_ALU32_REG(BPF_MOV, R0, R3), \ - BPF_EXIT_INSN(), \ - }, \ - INTERNAL, \ - { }, \ - { { 0, (op) & BPF_FETCH ? old : update } }, \ - .stack_depth = 40, \ -} - /* BPF_ATOMIC | BPF_W: BPF_ADD */ - BPF_ATOMIC_OP_TEST1(BPF_W, BPF_ADD, +, 0x12, 0xab, 0xbd), - BPF_ATOMIC_OP_TEST2(BPF_W, BPF_ADD, +, 0x12, 0xab, 0xbd), - BPF_ATOMIC_OP_TEST3(BPF_W, BPF_ADD, +, 0x12, 0xab, 0xbd), - BPF_ATOMIC_OP_TEST4(BPF_W, BPF_ADD, +, 0x12, 0xab, 0xbd), - /* BPF_ATOMIC | BPF_W: BPF_ADD | BPF_FETCH */ - BPF_ATOMIC_OP_TEST1(BPF_W, BPF_ADD | BPF_FETCH, +, 0x12, 0xab, 0xbd), - BPF_ATOMIC_OP_TEST2(BPF_W, BPF_ADD | BPF_FETCH, +, 0x12, 0xab, 0xbd), - BPF_ATOMIC_OP_TEST3(BPF_W, BPF_ADD | BPF_FETCH, +, 0x12, 0xab, 0xbd), - BPF_ATOMIC_OP_TEST4(BPF_W, BPF_ADD | BPF_FETCH, +, 0x12, 0xab, 0xbd), - /* BPF_ATOMIC | BPF_DW: BPF_ADD */ - BPF_ATOMIC_OP_TEST1(BPF_DW, BPF_ADD, +, 0x12, 0xab, 0xbd), - BPF_ATOMIC_OP_TEST2(BPF_DW, BPF_ADD, +, 0x12, 0xab, 0xbd), - BPF_ATOMIC_OP_TEST3(BPF_DW, BPF_ADD, +, 0x12, 0xab, 0xbd), - BPF_ATOMIC_OP_TEST4(BPF_DW, BPF_ADD, +, 0x12, 0xab, 0xbd), - /* BPF_ATOMIC | BPF_DW: BPF_ADD | BPF_FETCH */ - BPF_ATOMIC_OP_TEST1(BPF_DW, BPF_ADD | BPF_FETCH, +, 0x12, 0xab, 0xbd), - BPF_ATOMIC_OP_TEST2(BPF_DW, BPF_ADD | BPF_FETCH, +, 0x12, 0xab, 0xbd), - BPF_ATOMIC_OP_TEST3(BPF_DW, BPF_ADD | BPF_FETCH, +, 0x12, 0xab, 0xbd), - BPF_ATOMIC_OP_TEST4(BPF_DW, BPF_ADD | BPF_FETCH, +, 0x12, 0xab, 0xbd), - /* BPF_ATOMIC | BPF_W: BPF_AND */ - BPF_ATOMIC_OP_TEST1(BPF_W, BPF_AND, &, 0x12, 0xab, 0x02), - BPF_ATOMIC_OP_TEST2(BPF_W, BPF_AND, &, 0x12, 0xab, 0x02), - BPF_ATOMIC_OP_TEST3(BPF_W, BPF_AND, &, 0x12, 0xab, 0x02), - BPF_ATOMIC_OP_TEST4(BPF_W, BPF_AND, &, 0x12, 0xab, 0x02), - /* BPF_ATOMIC | BPF_W: BPF_AND | BPF_FETCH */ - BPF_ATOMIC_OP_TEST1(BPF_W, BPF_AND | BPF_FETCH, &, 0x12, 0xab, 0x02), - BPF_ATOMIC_OP_TEST2(BPF_W, BPF_AND | BPF_FETCH, &, 0x12, 0xab, 0x02), - BPF_ATOMIC_OP_TEST3(BPF_W, BPF_AND | BPF_FETCH, &, 0x12, 0xab, 0x02), - BPF_ATOMIC_OP_TEST4(BPF_W, BPF_AND | BPF_FETCH, &, 0x12, 0xab, 0x02), - /* BPF_ATOMIC | BPF_DW: BPF_AND */ - BPF_ATOMIC_OP_TEST1(BPF_DW, BPF_AND, &, 0x12, 0xab, 0x02), - BPF_ATOMIC_OP_TEST2(BPF_DW, BPF_AND, &, 0x12, 0xab, 0x02), - BPF_ATOMIC_OP_TEST3(BPF_DW, BPF_AND, &, 0x12, 0xab, 0x02), - BPF_ATOMIC_OP_TEST4(BPF_DW, BPF_AND, &, 0x12, 0xab, 0x02), - /* BPF_ATOMIC | BPF_DW: BPF_AND | BPF_FETCH */ - BPF_ATOMIC_OP_TEST1(BPF_DW, BPF_AND | BPF_FETCH, &, 0x12, 0xab, 0x02), - BPF_ATOMIC_OP_TEST2(BPF_DW, BPF_AND | BPF_FETCH, &, 0x12, 0xab, 0x02), - BPF_ATOMIC_OP_TEST3(BPF_DW, BPF_AND | BPF_FETCH, &, 0x12, 0xab, 0x02), - BPF_ATOMIC_OP_TEST4(BPF_DW, BPF_AND | BPF_FETCH, &, 0x12, 0xab, 0x02), - /* BPF_ATOMIC | BPF_W: BPF_OR */ - BPF_ATOMIC_OP_TEST1(BPF_W, BPF_OR, |, 0x12, 0xab, 0xbb), - BPF_ATOMIC_OP_TEST2(BPF_W, BPF_OR, |, 0x12, 0xab, 0xbb), - BPF_ATOMIC_OP_TEST3(BPF_W, BPF_OR, |, 0x12, 0xab, 0xbb), - BPF_ATOMIC_OP_TEST4(BPF_W, BPF_OR, |, 0x12, 0xab, 0xbb), - /* BPF_ATOMIC | BPF_W: BPF_OR | BPF_FETCH */ - BPF_ATOMIC_OP_TEST1(BPF_W, BPF_OR | BPF_FETCH, |, 0x12, 0xab, 0xbb), - BPF_ATOMIC_OP_TEST2(BPF_W, BPF_OR | BPF_FETCH, |, 0x12, 0xab, 0xbb), - BPF_ATOMIC_OP_TEST3(BPF_W, BPF_OR | BPF_FETCH, |, 0x12, 0xab, 0xbb), - BPF_ATOMIC_OP_TEST4(BPF_W, BPF_OR | BPF_FETCH, |, 0x12, 0xab, 0xbb), - /* BPF_ATOMIC | BPF_DW: BPF_OR */ - BPF_ATOMIC_OP_TEST1(BPF_DW, BPF_OR, |, 0x12, 0xab, 0xbb), - BPF_ATOMIC_OP_TEST2(BPF_DW, BPF_OR, |, 0x12, 0xab, 0xbb), - BPF_ATOMIC_OP_TEST3(BPF_DW, BPF_OR, |, 0x12, 0xab, 0xbb), - BPF_ATOMIC_OP_TEST4(BPF_DW, BPF_OR, |, 0x12, 0xab, 0xbb), - /* BPF_ATOMIC | BPF_DW: BPF_OR | BPF_FETCH */ - BPF_ATOMIC_OP_TEST1(BPF_DW, BPF_OR | BPF_FETCH, |, 0x12, 0xab, 0xbb), - BPF_ATOMIC_OP_TEST2(BPF_DW, BPF_OR | BPF_FETCH, |, 0x12, 0xab, 0xbb), - BPF_ATOMIC_OP_TEST3(BPF_DW, BPF_OR | BPF_FETCH, |, 0x12, 0xab, 0xbb), - BPF_ATOMIC_OP_TEST4(BPF_DW, BPF_OR | BPF_FETCH, |, 0x12, 0xab, 0xbb), - /* BPF_ATOMIC | BPF_W: BPF_XOR */ - BPF_ATOMIC_OP_TEST1(BPF_W, BPF_XOR, ^, 0x12, 0xab, 0xb9), - BPF_ATOMIC_OP_TEST2(BPF_W, BPF_XOR, ^, 0x12, 0xab, 0xb9), - BPF_ATOMIC_OP_TEST3(BPF_W, BPF_XOR, ^, 0x12, 0xab, 0xb9), - BPF_ATOMIC_OP_TEST4(BPF_W, BPF_XOR, ^, 0x12, 0xab, 0xb9), - /* BPF_ATOMIC | BPF_W: BPF_XOR | BPF_FETCH */ - BPF_ATOMIC_OP_TEST1(BPF_W, BPF_XOR | BPF_FETCH, ^, 0x12, 0xab, 0xb9), - BPF_ATOMIC_OP_TEST2(BPF_W, BPF_XOR | BPF_FETCH, ^, 0x12, 0xab, 0xb9), - BPF_ATOMIC_OP_TEST3(BPF_W, BPF_XOR | BPF_FETCH, ^, 0x12, 0xab, 0xb9), - BPF_ATOMIC_OP_TEST4(BPF_W, BPF_XOR | BPF_FETCH, ^, 0x12, 0xab, 0xb9), - /* BPF_ATOMIC | BPF_DW: BPF_XOR */ - BPF_ATOMIC_OP_TEST1(BPF_DW, BPF_XOR, ^, 0x12, 0xab, 0xb9), - BPF_ATOMIC_OP_TEST2(BPF_DW, BPF_XOR, ^, 0x12, 0xab, 0xb9), - BPF_ATOMIC_OP_TEST3(BPF_DW, BPF_XOR, ^, 0x12, 0xab, 0xb9), - BPF_ATOMIC_OP_TEST4(BPF_DW, BPF_XOR, ^, 0x12, 0xab, 0xb9), - /* BPF_ATOMIC | BPF_DW: BPF_XOR | BPF_FETCH */ - BPF_ATOMIC_OP_TEST1(BPF_DW, BPF_XOR | BPF_FETCH, ^, 0x12, 0xab, 0xb9), - BPF_ATOMIC_OP_TEST2(BPF_DW, BPF_XOR | BPF_FETCH, ^, 0x12, 0xab, 0xb9), - BPF_ATOMIC_OP_TEST3(BPF_DW, BPF_XOR | BPF_FETCH, ^, 0x12, 0xab, 0xb9), - BPF_ATOMIC_OP_TEST4(BPF_DW, BPF_XOR | BPF_FETCH, ^, 0x12, 0xab, 0xb9), - /* BPF_ATOMIC | BPF_W: BPF_XCHG */ - BPF_ATOMIC_OP_TEST1(BPF_W, BPF_XCHG, xchg, 0x12, 0xab, 0xab), - BPF_ATOMIC_OP_TEST2(BPF_W, BPF_XCHG, xchg, 0x12, 0xab, 0xab), - BPF_ATOMIC_OP_TEST3(BPF_W, BPF_XCHG, xchg, 0x12, 0xab, 0xab), - BPF_ATOMIC_OP_TEST4(BPF_W, BPF_XCHG, xchg, 0x12, 0xab, 0xab), - /* BPF_ATOMIC | BPF_DW: BPF_XCHG */ - BPF_ATOMIC_OP_TEST1(BPF_DW, BPF_XCHG, xchg, 0x12, 0xab, 0xab), - BPF_ATOMIC_OP_TEST2(BPF_DW, BPF_XCHG, xchg, 0x12, 0xab, 0xab), - BPF_ATOMIC_OP_TEST3(BPF_DW, BPF_XCHG, xchg, 0x12, 0xab, 0xab), - BPF_ATOMIC_OP_TEST4(BPF_DW, BPF_XCHG, xchg, 0x12, 0xab, 0xab), -#undef BPF_ATOMIC_POISON -#undef BPF_ATOMIC_OP_TEST1 -#undef BPF_ATOMIC_OP_TEST2 -#undef BPF_ATOMIC_OP_TEST3 -#undef BPF_ATOMIC_OP_TEST4 - /* BPF_ATOMIC | BPF_W, BPF_CMPXCHG */ - { - "BPF_ATOMIC | BPF_W, BPF_CMPXCHG: Test successful return", - .u.insns_int = { - BPF_ST_MEM(BPF_W, R10, -40, 0x01234567), - BPF_ALU32_IMM(BPF_MOV, R0, 0x01234567), - BPF_ALU32_IMM(BPF_MOV, R3, 0x89abcdef), - BPF_ATOMIC_OP(BPF_W, BPF_CMPXCHG, R10, R3, -40), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x01234567 } }, - .stack_depth = 40, - }, - { - "BPF_ATOMIC | BPF_W, BPF_CMPXCHG: Test successful store", - .u.insns_int = { - BPF_ST_MEM(BPF_W, R10, -40, 0x01234567), - BPF_ALU32_IMM(BPF_MOV, R0, 0x01234567), - BPF_ALU32_IMM(BPF_MOV, R3, 0x89abcdef), - BPF_ATOMIC_OP(BPF_W, BPF_CMPXCHG, R10, R3, -40), - BPF_LDX_MEM(BPF_W, R0, R10, -40), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x89abcdef } }, - .stack_depth = 40, - }, - { - "BPF_ATOMIC | BPF_W, BPF_CMPXCHG: Test failure return", - .u.insns_int = { - BPF_ST_MEM(BPF_W, R10, -40, 0x01234567), - BPF_ALU32_IMM(BPF_MOV, R0, 0x76543210), - BPF_ALU32_IMM(BPF_MOV, R3, 0x89abcdef), - BPF_ATOMIC_OP(BPF_W, BPF_CMPXCHG, R10, R3, -40), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x01234567 } }, - .stack_depth = 40, - }, - { - "BPF_ATOMIC | BPF_W, BPF_CMPXCHG: Test failure store", - .u.insns_int = { - BPF_ST_MEM(BPF_W, R10, -40, 0x01234567), - BPF_ALU32_IMM(BPF_MOV, R0, 0x76543210), - BPF_ALU32_IMM(BPF_MOV, R3, 0x89abcdef), - BPF_ATOMIC_OP(BPF_W, BPF_CMPXCHG, R10, R3, -40), - BPF_LDX_MEM(BPF_W, R0, R10, -40), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x01234567 } }, - .stack_depth = 40, - }, - { - "BPF_ATOMIC | BPF_W, BPF_CMPXCHG: Test side effects", - .u.insns_int = { - BPF_ST_MEM(BPF_W, R10, -40, 0x01234567), - BPF_ALU32_IMM(BPF_MOV, R0, 0x01234567), - BPF_ALU32_IMM(BPF_MOV, R3, 0x89abcdef), - BPF_ATOMIC_OP(BPF_W, BPF_CMPXCHG, R10, R3, -40), - BPF_ATOMIC_OP(BPF_W, BPF_CMPXCHG, R10, R3, -40), - BPF_ALU32_REG(BPF_MOV, R0, R3), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x89abcdef } }, - .stack_depth = 40, - }, - /* BPF_ATOMIC | BPF_DW, BPF_CMPXCHG */ - { - "BPF_ATOMIC | BPF_DW, BPF_CMPXCHG: Test successful return", - .u.insns_int = { - BPF_LD_IMM64(R1, 0x0123456789abcdefULL), - BPF_LD_IMM64(R2, 0xfedcba9876543210ULL), - BPF_ALU64_REG(BPF_MOV, R0, R1), - BPF_STX_MEM(BPF_DW, R10, R1, -40), - BPF_ATOMIC_OP(BPF_DW, BPF_CMPXCHG, R10, R2, -40), - BPF_JMP_REG(BPF_JNE, R0, R1, 1), - BPF_ALU64_REG(BPF_SUB, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } }, - .stack_depth = 40, - }, - { - "BPF_ATOMIC | BPF_DW, BPF_CMPXCHG: Test successful store", - .u.insns_int = { - BPF_LD_IMM64(R1, 0x0123456789abcdefULL), - BPF_LD_IMM64(R2, 0xfedcba9876543210ULL), - BPF_ALU64_REG(BPF_MOV, R0, R1), - BPF_STX_MEM(BPF_DW, R10, R0, -40), - BPF_ATOMIC_OP(BPF_DW, BPF_CMPXCHG, R10, R2, -40), - BPF_LDX_MEM(BPF_DW, R0, R10, -40), - BPF_JMP_REG(BPF_JNE, R0, R2, 1), - BPF_ALU64_REG(BPF_SUB, R0, R2), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } }, - .stack_depth = 40, - }, - { - "BPF_ATOMIC | BPF_DW, BPF_CMPXCHG: Test failure return", - .u.insns_int = { - BPF_LD_IMM64(R1, 0x0123456789abcdefULL), - BPF_LD_IMM64(R2, 0xfedcba9876543210ULL), - BPF_ALU64_REG(BPF_MOV, R0, R1), - BPF_ALU64_IMM(BPF_ADD, R0, 1), - BPF_STX_MEM(BPF_DW, R10, R1, -40), - BPF_ATOMIC_OP(BPF_DW, BPF_CMPXCHG, R10, R2, -40), - BPF_JMP_REG(BPF_JNE, R0, R1, 1), - BPF_ALU64_REG(BPF_SUB, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } }, - .stack_depth = 40, - }, - { - "BPF_ATOMIC | BPF_DW, BPF_CMPXCHG: Test failure store", - .u.insns_int = { - BPF_LD_IMM64(R1, 0x0123456789abcdefULL), - BPF_LD_IMM64(R2, 0xfedcba9876543210ULL), - BPF_ALU64_REG(BPF_MOV, R0, R1), - BPF_ALU64_IMM(BPF_ADD, R0, 1), - BPF_STX_MEM(BPF_DW, R10, R1, -40), - BPF_ATOMIC_OP(BPF_DW, BPF_CMPXCHG, R10, R2, -40), - BPF_LDX_MEM(BPF_DW, R0, R10, -40), - BPF_JMP_REG(BPF_JNE, R0, R1, 1), - BPF_ALU64_REG(BPF_SUB, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } }, - .stack_depth = 40, - }, - { - "BPF_ATOMIC | BPF_DW, BPF_CMPXCHG: Test side effects", - .u.insns_int = { - BPF_LD_IMM64(R1, 0x0123456789abcdefULL), - BPF_LD_IMM64(R2, 0xfedcba9876543210ULL), - BPF_ALU64_REG(BPF_MOV, R0, R1), - BPF_STX_MEM(BPF_DW, R10, R1, -40), - BPF_ATOMIC_OP(BPF_DW, BPF_CMPXCHG, R10, R2, -40), - BPF_LD_IMM64(R0, 0xfedcba9876543210ULL), - BPF_JMP_REG(BPF_JNE, R0, R2, 1), - BPF_ALU64_REG(BPF_SUB, R0, R2), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } }, - .stack_depth = 40, - }, - /* BPF_JMP32 | BPF_JEQ | BPF_K */ - { - "JMP32_JEQ_K: Small immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 123), - BPF_JMP32_IMM(BPF_JEQ, R0, 321, 1), - BPF_JMP32_IMM(BPF_JEQ, R0, 123, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 123 } } - }, - { - "JMP32_JEQ_K: Large immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 12345678), - BPF_JMP32_IMM(BPF_JEQ, R0, 12345678 & 0xffff, 1), - BPF_JMP32_IMM(BPF_JEQ, R0, 12345678, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 12345678 } } - }, - { - "JMP32_JEQ_K: negative immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, -123), - BPF_JMP32_IMM(BPF_JEQ, R0, 123, 1), - BPF_JMP32_IMM(BPF_JEQ, R0, -123, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -123 } } - }, - /* BPF_JMP32 | BPF_JEQ | BPF_X */ - { - "JMP32_JEQ_X", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 1234), - BPF_ALU32_IMM(BPF_MOV, R1, 4321), - BPF_JMP32_REG(BPF_JEQ, R0, R1, 2), - BPF_ALU32_IMM(BPF_MOV, R1, 1234), - BPF_JMP32_REG(BPF_JEQ, R0, R1, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1234 } } - }, - /* BPF_JMP32 | BPF_JNE | BPF_K */ - { - "JMP32_JNE_K: Small immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 123), - BPF_JMP32_IMM(BPF_JNE, R0, 123, 1), - BPF_JMP32_IMM(BPF_JNE, R0, 321, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 123 } } - }, - { - "JMP32_JNE_K: Large immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 12345678), - BPF_JMP32_IMM(BPF_JNE, R0, 12345678, 1), - BPF_JMP32_IMM(BPF_JNE, R0, 12345678 & 0xffff, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 12345678 } } - }, - { - "JMP32_JNE_K: negative immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, -123), - BPF_JMP32_IMM(BPF_JNE, R0, -123, 1), - BPF_JMP32_IMM(BPF_JNE, R0, 123, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -123 } } - }, - /* BPF_JMP32 | BPF_JNE | BPF_X */ - { - "JMP32_JNE_X", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 1234), - BPF_ALU32_IMM(BPF_MOV, R1, 1234), - BPF_JMP32_REG(BPF_JNE, R0, R1, 2), - BPF_ALU32_IMM(BPF_MOV, R1, 4321), - BPF_JMP32_REG(BPF_JNE, R0, R1, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1234 } } - }, - /* BPF_JMP32 | BPF_JSET | BPF_K */ - { - "JMP32_JSET_K: Small immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_JMP32_IMM(BPF_JSET, R0, 2, 1), - BPF_JMP32_IMM(BPF_JSET, R0, 3, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } } - }, - { - "JMP32_JSET_K: Large immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0x40000000), - BPF_JMP32_IMM(BPF_JSET, R0, 0x3fffffff, 1), - BPF_JMP32_IMM(BPF_JSET, R0, 0x60000000, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0x40000000 } } - }, - { - "JMP32_JSET_K: negative immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, -123), - BPF_JMP32_IMM(BPF_JSET, R0, -1, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -123 } } - }, - /* BPF_JMP32 | BPF_JSET | BPF_X */ - { - "JMP32_JSET_X", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 8), - BPF_ALU32_IMM(BPF_MOV, R1, 7), - BPF_JMP32_REG(BPF_JSET, R0, R1, 2), - BPF_ALU32_IMM(BPF_MOV, R1, 8 | 2), - BPF_JMP32_REG(BPF_JNE, R0, R1, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 8 } } - }, - /* BPF_JMP32 | BPF_JGT | BPF_K */ - { - "JMP32_JGT_K: Small immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 123), - BPF_JMP32_IMM(BPF_JGT, R0, 123, 1), - BPF_JMP32_IMM(BPF_JGT, R0, 122, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 123 } } - }, - { - "JMP32_JGT_K: Large immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0xfffffffe), - BPF_JMP32_IMM(BPF_JGT, R0, 0xffffffff, 1), - BPF_JMP32_IMM(BPF_JGT, R0, 0xfffffffd, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xfffffffe } } - }, - /* BPF_JMP32 | BPF_JGT | BPF_X */ - { - "JMP32_JGT_X", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0xfffffffe), - BPF_ALU32_IMM(BPF_MOV, R1, 0xffffffff), - BPF_JMP32_REG(BPF_JGT, R0, R1, 2), - BPF_ALU32_IMM(BPF_MOV, R1, 0xfffffffd), - BPF_JMP32_REG(BPF_JGT, R0, R1, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xfffffffe } } - }, - /* BPF_JMP32 | BPF_JGE | BPF_K */ - { - "JMP32_JGE_K: Small immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 123), - BPF_JMP32_IMM(BPF_JGE, R0, 124, 1), - BPF_JMP32_IMM(BPF_JGE, R0, 123, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 123 } } - }, - { - "JMP32_JGE_K: Large immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0xfffffffe), - BPF_JMP32_IMM(BPF_JGE, R0, 0xffffffff, 1), - BPF_JMP32_IMM(BPF_JGE, R0, 0xfffffffe, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xfffffffe } } - }, - /* BPF_JMP32 | BPF_JGE | BPF_X */ - { - "JMP32_JGE_X", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0xfffffffe), - BPF_ALU32_IMM(BPF_MOV, R1, 0xffffffff), - BPF_JMP32_REG(BPF_JGE, R0, R1, 2), - BPF_ALU32_IMM(BPF_MOV, R1, 0xfffffffe), - BPF_JMP32_REG(BPF_JGE, R0, R1, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xfffffffe } } - }, - /* BPF_JMP32 | BPF_JLT | BPF_K */ - { - "JMP32_JLT_K: Small immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 123), - BPF_JMP32_IMM(BPF_JLT, R0, 123, 1), - BPF_JMP32_IMM(BPF_JLT, R0, 124, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 123 } } - }, - { - "JMP32_JLT_K: Large immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0xfffffffe), - BPF_JMP32_IMM(BPF_JLT, R0, 0xfffffffd, 1), - BPF_JMP32_IMM(BPF_JLT, R0, 0xffffffff, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xfffffffe } } - }, - /* BPF_JMP32 | BPF_JLT | BPF_X */ - { - "JMP32_JLT_X", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0xfffffffe), - BPF_ALU32_IMM(BPF_MOV, R1, 0xfffffffd), - BPF_JMP32_REG(BPF_JLT, R0, R1, 2), - BPF_ALU32_IMM(BPF_MOV, R1, 0xffffffff), - BPF_JMP32_REG(BPF_JLT, R0, R1, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xfffffffe } } - }, - /* BPF_JMP32 | BPF_JLE | BPF_K */ - { - "JMP32_JLE_K: Small immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 123), - BPF_JMP32_IMM(BPF_JLE, R0, 122, 1), - BPF_JMP32_IMM(BPF_JLE, R0, 123, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 123 } } - }, - { - "JMP32_JLE_K: Large immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0xfffffffe), - BPF_JMP32_IMM(BPF_JLE, R0, 0xfffffffd, 1), - BPF_JMP32_IMM(BPF_JLE, R0, 0xfffffffe, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xfffffffe } } - }, - /* BPF_JMP32 | BPF_JLE | BPF_X */ - { - "JMP32_JLE_X", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0xfffffffe), - BPF_ALU32_IMM(BPF_MOV, R1, 0xfffffffd), - BPF_JMP32_REG(BPF_JLE, R0, R1, 2), - BPF_ALU32_IMM(BPF_MOV, R1, 0xfffffffe), - BPF_JMP32_REG(BPF_JLE, R0, R1, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xfffffffe } } - }, - /* BPF_JMP32 | BPF_JSGT | BPF_K */ - { - "JMP32_JSGT_K: Small immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, -123), - BPF_JMP32_IMM(BPF_JSGT, R0, -123, 1), - BPF_JMP32_IMM(BPF_JSGT, R0, -124, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -123 } } - }, - { - "JMP32_JSGT_K: Large immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, -12345678), - BPF_JMP32_IMM(BPF_JSGT, R0, -12345678, 1), - BPF_JMP32_IMM(BPF_JSGT, R0, -12345679, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -12345678 } } - }, - /* BPF_JMP32 | BPF_JSGT | BPF_X */ - { - "JMP32_JSGT_X", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, -12345678), - BPF_ALU32_IMM(BPF_MOV, R1, -12345678), - BPF_JMP32_REG(BPF_JSGT, R0, R1, 2), - BPF_ALU32_IMM(BPF_MOV, R1, -12345679), - BPF_JMP32_REG(BPF_JSGT, R0, R1, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -12345678 } } - }, - /* BPF_JMP32 | BPF_JSGE | BPF_K */ - { - "JMP32_JSGE_K: Small immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, -123), - BPF_JMP32_IMM(BPF_JSGE, R0, -122, 1), - BPF_JMP32_IMM(BPF_JSGE, R0, -123, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -123 } } - }, - { - "JMP32_JSGE_K: Large immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, -12345678), - BPF_JMP32_IMM(BPF_JSGE, R0, -12345677, 1), - BPF_JMP32_IMM(BPF_JSGE, R0, -12345678, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -12345678 } } - }, - /* BPF_JMP32 | BPF_JSGE | BPF_X */ - { - "JMP32_JSGE_X", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, -12345678), - BPF_ALU32_IMM(BPF_MOV, R1, -12345677), - BPF_JMP32_REG(BPF_JSGE, R0, R1, 2), - BPF_ALU32_IMM(BPF_MOV, R1, -12345678), - BPF_JMP32_REG(BPF_JSGE, R0, R1, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -12345678 } } - }, - /* BPF_JMP32 | BPF_JSLT | BPF_K */ - { - "JMP32_JSLT_K: Small immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, -123), - BPF_JMP32_IMM(BPF_JSLT, R0, -123, 1), - BPF_JMP32_IMM(BPF_JSLT, R0, -122, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -123 } } - }, - { - "JMP32_JSLT_K: Large immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, -12345678), - BPF_JMP32_IMM(BPF_JSLT, R0, -12345678, 1), - BPF_JMP32_IMM(BPF_JSLT, R0, -12345677, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -12345678 } } - }, - /* BPF_JMP32 | BPF_JSLT | BPF_X */ - { - "JMP32_JSLT_X", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, -12345678), - BPF_ALU32_IMM(BPF_MOV, R1, -12345678), - BPF_JMP32_REG(BPF_JSLT, R0, R1, 2), - BPF_ALU32_IMM(BPF_MOV, R1, -12345677), - BPF_JMP32_REG(BPF_JSLT, R0, R1, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -12345678 } } - }, - /* BPF_JMP32 | BPF_JSLE | BPF_K */ - { - "JMP32_JSLE_K: Small immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, -123), - BPF_JMP32_IMM(BPF_JSLE, R0, -124, 1), - BPF_JMP32_IMM(BPF_JSLE, R0, -123, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -123 } } - }, - { - "JMP32_JSLE_K: Large immediate", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, -12345678), - BPF_JMP32_IMM(BPF_JSLE, R0, -12345679, 1), - BPF_JMP32_IMM(BPF_JSLE, R0, -12345678, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -12345678 } } - }, - /* BPF_JMP32 | BPF_JSLE | BPF_K */ - { - "JMP32_JSLE_X", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, -12345678), - BPF_ALU32_IMM(BPF_MOV, R1, -12345679), - BPF_JMP32_REG(BPF_JSLE, R0, R1, 2), - BPF_ALU32_IMM(BPF_MOV, R1, -12345678), - BPF_JMP32_REG(BPF_JSLE, R0, R1, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, -12345678 } } - }, - /* BPF_JMP | BPF_EXIT */ - { - "JMP_EXIT", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0x4711), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 0x4712), - }, - INTERNAL, - { }, - { { 0, 0x4711 } }, - }, - /* BPF_JMP | BPF_JA */ - { - "JMP_JA: Unconditional jump: if (true) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_JMP_IMM(BPF_JA, 0, 0, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - /* BPF_JMP32 | BPF_JA */ - { - "JMP32_JA: Unconditional jump: if (true) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_JMP32_IMM(BPF_JA, 0, 1, 0), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - /* BPF_JMP | BPF_JSLT | BPF_K */ - { - "JMP_JSLT_K: Signed jump: if (-2 < -1) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 0xfffffffffffffffeLL), - BPF_JMP_IMM(BPF_JSLT, R1, -1, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "JMP_JSLT_K: Signed jump: if (-1 < -1) return 0", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_LD_IMM64(R1, 0xffffffffffffffffLL), - BPF_JMP_IMM(BPF_JSLT, R1, -1, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - /* BPF_JMP | BPF_JSGT | BPF_K */ - { - "JMP_JSGT_K: Signed jump: if (-1 > -2) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 0xffffffffffffffffLL), - BPF_JMP_IMM(BPF_JSGT, R1, -2, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "JMP_JSGT_K: Signed jump: if (-1 > -1) return 0", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_LD_IMM64(R1, 0xffffffffffffffffLL), - BPF_JMP_IMM(BPF_JSGT, R1, -1, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - /* BPF_JMP | BPF_JSLE | BPF_K */ - { - "JMP_JSLE_K: Signed jump: if (-2 <= -1) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 0xfffffffffffffffeLL), - BPF_JMP_IMM(BPF_JSLE, R1, -1, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "JMP_JSLE_K: Signed jump: if (-1 <= -1) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 0xffffffffffffffffLL), - BPF_JMP_IMM(BPF_JSLE, R1, -1, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "JMP_JSLE_K: Signed jump: value walk 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 3), - BPF_JMP_IMM(BPF_JSLE, R1, 0, 6), - BPF_ALU64_IMM(BPF_SUB, R1, 1), - BPF_JMP_IMM(BPF_JSLE, R1, 0, 4), - BPF_ALU64_IMM(BPF_SUB, R1, 1), - BPF_JMP_IMM(BPF_JSLE, R1, 0, 2), - BPF_ALU64_IMM(BPF_SUB, R1, 1), - BPF_JMP_IMM(BPF_JSLE, R1, 0, 1), - BPF_EXIT_INSN(), /* bad exit */ - BPF_ALU32_IMM(BPF_MOV, R0, 1), /* good exit */ - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "JMP_JSLE_K: Signed jump: value walk 2", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 3), - BPF_JMP_IMM(BPF_JSLE, R1, 0, 4), - BPF_ALU64_IMM(BPF_SUB, R1, 2), - BPF_JMP_IMM(BPF_JSLE, R1, 0, 2), - BPF_ALU64_IMM(BPF_SUB, R1, 2), - BPF_JMP_IMM(BPF_JSLE, R1, 0, 1), - BPF_EXIT_INSN(), /* bad exit */ - BPF_ALU32_IMM(BPF_MOV, R0, 1), /* good exit */ - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - /* BPF_JMP | BPF_JSGE | BPF_K */ - { - "JMP_JSGE_K: Signed jump: if (-1 >= -2) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 0xffffffffffffffffLL), - BPF_JMP_IMM(BPF_JSGE, R1, -2, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "JMP_JSGE_K: Signed jump: if (-1 >= -1) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 0xffffffffffffffffLL), - BPF_JMP_IMM(BPF_JSGE, R1, -1, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "JMP_JSGE_K: Signed jump: value walk 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, -3), - BPF_JMP_IMM(BPF_JSGE, R1, 0, 6), - BPF_ALU64_IMM(BPF_ADD, R1, 1), - BPF_JMP_IMM(BPF_JSGE, R1, 0, 4), - BPF_ALU64_IMM(BPF_ADD, R1, 1), - BPF_JMP_IMM(BPF_JSGE, R1, 0, 2), - BPF_ALU64_IMM(BPF_ADD, R1, 1), - BPF_JMP_IMM(BPF_JSGE, R1, 0, 1), - BPF_EXIT_INSN(), /* bad exit */ - BPF_ALU32_IMM(BPF_MOV, R0, 1), /* good exit */ - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "JMP_JSGE_K: Signed jump: value walk 2", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, -3), - BPF_JMP_IMM(BPF_JSGE, R1, 0, 4), - BPF_ALU64_IMM(BPF_ADD, R1, 2), - BPF_JMP_IMM(BPF_JSGE, R1, 0, 2), - BPF_ALU64_IMM(BPF_ADD, R1, 2), - BPF_JMP_IMM(BPF_JSGE, R1, 0, 1), - BPF_EXIT_INSN(), /* bad exit */ - BPF_ALU32_IMM(BPF_MOV, R0, 1), /* good exit */ - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - /* BPF_JMP | BPF_JGT | BPF_K */ - { - "JMP_JGT_K: if (3 > 2) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 3), - BPF_JMP_IMM(BPF_JGT, R1, 2, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "JMP_JGT_K: Unsigned jump: if (-1 > 1) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, -1), - BPF_JMP_IMM(BPF_JGT, R1, 1, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - /* BPF_JMP | BPF_JLT | BPF_K */ - { - "JMP_JLT_K: if (2 < 3) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 2), - BPF_JMP_IMM(BPF_JLT, R1, 3, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "JMP_JGT_K: Unsigned jump: if (1 < -1) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 1), - BPF_JMP_IMM(BPF_JLT, R1, -1, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - /* BPF_JMP | BPF_JGE | BPF_K */ - { - "JMP_JGE_K: if (3 >= 2) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 3), - BPF_JMP_IMM(BPF_JGE, R1, 2, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - /* BPF_JMP | BPF_JLE | BPF_K */ - { - "JMP_JLE_K: if (2 <= 3) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 2), - BPF_JMP_IMM(BPF_JLE, R1, 3, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - /* BPF_JMP | BPF_JGT | BPF_K jump backwards */ - { - "JMP_JGT_K: if (3 > 2) return 1 (jump backwards)", - .u.insns_int = { - BPF_JMP_IMM(BPF_JA, 0, 0, 2), /* goto start */ - BPF_ALU32_IMM(BPF_MOV, R0, 1), /* out: */ - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 0), /* start: */ - BPF_LD_IMM64(R1, 3), /* note: this takes 2 insns */ - BPF_JMP_IMM(BPF_JGT, R1, 2, -6), /* goto out */ - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "JMP_JGE_K: if (3 >= 3) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 3), - BPF_JMP_IMM(BPF_JGE, R1, 3, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - /* BPF_JMP | BPF_JLT | BPF_K jump backwards */ - { - "JMP_JGT_K: if (2 < 3) return 1 (jump backwards)", - .u.insns_int = { - BPF_JMP_IMM(BPF_JA, 0, 0, 2), /* goto start */ - BPF_ALU32_IMM(BPF_MOV, R0, 1), /* out: */ - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 0), /* start: */ - BPF_LD_IMM64(R1, 2), /* note: this takes 2 insns */ - BPF_JMP_IMM(BPF_JLT, R1, 3, -6), /* goto out */ - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "JMP_JLE_K: if (3 <= 3) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 3), - BPF_JMP_IMM(BPF_JLE, R1, 3, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - /* BPF_JMP | BPF_JNE | BPF_K */ - { - "JMP_JNE_K: if (3 != 2) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 3), - BPF_JMP_IMM(BPF_JNE, R1, 2, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - /* BPF_JMP | BPF_JEQ | BPF_K */ - { - "JMP_JEQ_K: if (3 == 3) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 3), - BPF_JMP_IMM(BPF_JEQ, R1, 3, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - /* BPF_JMP | BPF_JSET | BPF_K */ - { - "JMP_JSET_K: if (0x3 & 0x2) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 3), - BPF_JMP_IMM(BPF_JSET, R1, 2, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "JMP_JSET_K: if (0x3 & 0xffffffff) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 3), - BPF_JMP_IMM(BPF_JSET, R1, 0xffffffff, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - /* BPF_JMP | BPF_JSGT | BPF_X */ - { - "JMP_JSGT_X: Signed jump: if (-1 > -2) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, -1), - BPF_LD_IMM64(R2, -2), - BPF_JMP_REG(BPF_JSGT, R1, R2, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "JMP_JSGT_X: Signed jump: if (-1 > -1) return 0", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_LD_IMM64(R1, -1), - BPF_LD_IMM64(R2, -1), - BPF_JMP_REG(BPF_JSGT, R1, R2, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - /* BPF_JMP | BPF_JSLT | BPF_X */ - { - "JMP_JSLT_X: Signed jump: if (-2 < -1) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, -1), - BPF_LD_IMM64(R2, -2), - BPF_JMP_REG(BPF_JSLT, R2, R1, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "JMP_JSLT_X: Signed jump: if (-1 < -1) return 0", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_LD_IMM64(R1, -1), - BPF_LD_IMM64(R2, -1), - BPF_JMP_REG(BPF_JSLT, R1, R2, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - /* BPF_JMP | BPF_JSGE | BPF_X */ - { - "JMP_JSGE_X: Signed jump: if (-1 >= -2) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, -1), - BPF_LD_IMM64(R2, -2), - BPF_JMP_REG(BPF_JSGE, R1, R2, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "JMP_JSGE_X: Signed jump: if (-1 >= -1) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, -1), - BPF_LD_IMM64(R2, -1), - BPF_JMP_REG(BPF_JSGE, R1, R2, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - /* BPF_JMP | BPF_JSLE | BPF_X */ - { - "JMP_JSLE_X: Signed jump: if (-2 <= -1) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, -1), - BPF_LD_IMM64(R2, -2), - BPF_JMP_REG(BPF_JSLE, R2, R1, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "JMP_JSLE_X: Signed jump: if (-1 <= -1) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, -1), - BPF_LD_IMM64(R2, -1), - BPF_JMP_REG(BPF_JSLE, R1, R2, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - /* BPF_JMP | BPF_JGT | BPF_X */ - { - "JMP_JGT_X: if (3 > 2) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 3), - BPF_LD_IMM64(R2, 2), - BPF_JMP_REG(BPF_JGT, R1, R2, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "JMP_JGT_X: Unsigned jump: if (-1 > 1) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, -1), - BPF_LD_IMM64(R2, 1), - BPF_JMP_REG(BPF_JGT, R1, R2, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - /* BPF_JMP | BPF_JLT | BPF_X */ - { - "JMP_JLT_X: if (2 < 3) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 3), - BPF_LD_IMM64(R2, 2), - BPF_JMP_REG(BPF_JLT, R2, R1, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "JMP_JLT_X: Unsigned jump: if (1 < -1) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, -1), - BPF_LD_IMM64(R2, 1), - BPF_JMP_REG(BPF_JLT, R2, R1, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - /* BPF_JMP | BPF_JGE | BPF_X */ - { - "JMP_JGE_X: if (3 >= 2) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 3), - BPF_LD_IMM64(R2, 2), - BPF_JMP_REG(BPF_JGE, R1, R2, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "JMP_JGE_X: if (3 >= 3) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 3), - BPF_LD_IMM64(R2, 3), - BPF_JMP_REG(BPF_JGE, R1, R2, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - /* BPF_JMP | BPF_JLE | BPF_X */ - { - "JMP_JLE_X: if (2 <= 3) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 3), - BPF_LD_IMM64(R2, 2), - BPF_JMP_REG(BPF_JLE, R2, R1, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "JMP_JLE_X: if (3 <= 3) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 3), - BPF_LD_IMM64(R2, 3), - BPF_JMP_REG(BPF_JLE, R1, R2, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - /* Mainly testing JIT + imm64 here. */ - "JMP_JGE_X: ldimm64 test 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 3), - BPF_LD_IMM64(R2, 2), - BPF_JMP_REG(BPF_JGE, R1, R2, 2), - BPF_LD_IMM64(R0, 0xffffffffffffffffULL), - BPF_LD_IMM64(R0, 0xeeeeeeeeeeeeeeeeULL), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xeeeeeeeeU } }, - }, - { - "JMP_JGE_X: ldimm64 test 2", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 3), - BPF_LD_IMM64(R2, 2), - BPF_JMP_REG(BPF_JGE, R1, R2, 0), - BPF_LD_IMM64(R0, 0xffffffffffffffffULL), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xffffffffU } }, - }, - { - "JMP_JGE_X: ldimm64 test 3", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_LD_IMM64(R1, 3), - BPF_LD_IMM64(R2, 2), - BPF_JMP_REG(BPF_JGE, R1, R2, 4), - BPF_LD_IMM64(R0, 0xffffffffffffffffULL), - BPF_LD_IMM64(R0, 0xeeeeeeeeeeeeeeeeULL), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "JMP_JLE_X: ldimm64 test 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 3), - BPF_LD_IMM64(R2, 2), - BPF_JMP_REG(BPF_JLE, R2, R1, 2), - BPF_LD_IMM64(R0, 0xffffffffffffffffULL), - BPF_LD_IMM64(R0, 0xeeeeeeeeeeeeeeeeULL), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xeeeeeeeeU } }, - }, - { - "JMP_JLE_X: ldimm64 test 2", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 3), - BPF_LD_IMM64(R2, 2), - BPF_JMP_REG(BPF_JLE, R2, R1, 0), - BPF_LD_IMM64(R0, 0xffffffffffffffffULL), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0xffffffffU } }, - }, - { - "JMP_JLE_X: ldimm64 test 3", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_LD_IMM64(R1, 3), - BPF_LD_IMM64(R2, 2), - BPF_JMP_REG(BPF_JLE, R2, R1, 4), - BPF_LD_IMM64(R0, 0xffffffffffffffffULL), - BPF_LD_IMM64(R0, 0xeeeeeeeeeeeeeeeeULL), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - /* BPF_JMP | BPF_JNE | BPF_X */ - { - "JMP_JNE_X: if (3 != 2) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 3), - BPF_LD_IMM64(R2, 2), - BPF_JMP_REG(BPF_JNE, R1, R2, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - /* BPF_JMP | BPF_JEQ | BPF_X */ - { - "JMP_JEQ_X: if (3 == 3) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 3), - BPF_LD_IMM64(R2, 3), - BPF_JMP_REG(BPF_JEQ, R1, R2, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - /* BPF_JMP | BPF_JSET | BPF_X */ - { - "JMP_JSET_X: if (0x3 & 0x2) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 3), - BPF_LD_IMM64(R2, 2), - BPF_JMP_REG(BPF_JSET, R1, R2, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "JMP_JSET_X: if (0x3 & 0xffffffff) return 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0), - BPF_LD_IMM64(R1, 3), - BPF_LD_IMM64(R2, 0xffffffff), - BPF_JMP_REG(BPF_JSET, R1, R2, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "JMP_JA: Jump, gap, jump, ...", - { }, - CLASSIC | FLAG_NO_DATA, - { }, - { { 0, 0xababcbac } }, - .fill_helper = bpf_fill_ja, - }, - { /* Mainly checking JIT here. */ - "BPF_MAXINSNS: Maximum possible literals", - { }, - CLASSIC | FLAG_NO_DATA, - { }, - { { 0, 0xffffffff } }, - .fill_helper = bpf_fill_maxinsns1, - }, - { /* Mainly checking JIT here. */ - "BPF_MAXINSNS: Single literal", - { }, - CLASSIC | FLAG_NO_DATA, - { }, - { { 0, 0xfefefefe } }, - .fill_helper = bpf_fill_maxinsns2, - }, - { /* Mainly checking JIT here. */ - "BPF_MAXINSNS: Run/add until end", - { }, - CLASSIC | FLAG_NO_DATA, - { }, - { { 0, 0x947bf368 } }, - .fill_helper = bpf_fill_maxinsns3, - }, - { - "BPF_MAXINSNS: Too many instructions", - { }, - CLASSIC | FLAG_NO_DATA | FLAG_EXPECTED_FAIL, - { }, - { }, - .fill_helper = bpf_fill_maxinsns4, - .expected_errcode = -EINVAL, - }, - { /* Mainly checking JIT here. */ - "BPF_MAXINSNS: Very long jump", - { }, - CLASSIC | FLAG_NO_DATA, - { }, - { { 0, 0xabababab } }, - .fill_helper = bpf_fill_maxinsns5, - }, - { /* Mainly checking JIT here. */ - "BPF_MAXINSNS: Ctx heavy transformations", - { }, - CLASSIC, - { }, - { - { 1, SKB_VLAN_PRESENT }, - { 10, SKB_VLAN_PRESENT } - }, - .fill_helper = bpf_fill_maxinsns6, - }, - { /* Mainly checking JIT here. */ - "BPF_MAXINSNS: Call heavy transformations", - { }, - CLASSIC | FLAG_NO_DATA, - { }, - { { 1, 0 }, { 10, 0 } }, - .fill_helper = bpf_fill_maxinsns7, - }, - { /* Mainly checking JIT here. */ - "BPF_MAXINSNS: Jump heavy test", - { }, - CLASSIC | FLAG_NO_DATA, - { }, - { { 0, 0xffffffff } }, - .fill_helper = bpf_fill_maxinsns8, - }, - { /* Mainly checking JIT here. */ - "BPF_MAXINSNS: Very long jump backwards", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 0xcbababab } }, - .fill_helper = bpf_fill_maxinsns9, - }, - { /* Mainly checking JIT here. */ - "BPF_MAXINSNS: Edge hopping nuthouse", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 0xabababac } }, - .fill_helper = bpf_fill_maxinsns10, - }, - { - "BPF_MAXINSNS: Jump, gap, jump, ...", - { }, - CLASSIC | FLAG_NO_DATA, - { }, - { { 0, 0xababcbac } }, - .fill_helper = bpf_fill_maxinsns11, - }, - { - "BPF_MAXINSNS: jump over MSH", - { }, - CLASSIC | FLAG_EXPECTED_FAIL, - { 0xfa, 0xfb, 0xfc, 0xfd, }, - { { 4, 0xabababab } }, - .fill_helper = bpf_fill_maxinsns12, - .expected_errcode = -EINVAL, - }, - { - "BPF_MAXINSNS: exec all MSH", - { }, - CLASSIC, - { 0xfa, 0xfb, 0xfc, 0xfd, }, - { { 4, 0xababab83 } }, - .fill_helper = bpf_fill_maxinsns13, - }, - { - "BPF_MAXINSNS: ld_abs+get_processor_id", - { }, - CLASSIC, - { }, - { { 1, 0xbee } }, - .fill_helper = bpf_fill_ld_abs_get_processor_id, - }, - /* - * LD_IND / LD_ABS on fragmented SKBs - */ - { - "LD_IND byte frag", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_IMM, 0x40), - BPF_STMT(BPF_LD | BPF_IND | BPF_B, 0x0), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC | FLAG_SKB_FRAG, - { }, - { {0x40, 0x42} }, - .frag_data = { - 0x42, 0x00, 0x00, 0x00, - 0x43, 0x44, 0x00, 0x00, - 0x21, 0x07, 0x19, 0x83, - }, - }, - { - "LD_IND halfword frag", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_IMM, 0x40), - BPF_STMT(BPF_LD | BPF_IND | BPF_H, 0x4), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC | FLAG_SKB_FRAG, - { }, - { {0x40, 0x4344} }, - .frag_data = { - 0x42, 0x00, 0x00, 0x00, - 0x43, 0x44, 0x00, 0x00, - 0x21, 0x07, 0x19, 0x83, - }, - }, - { - "LD_IND word frag", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_IMM, 0x40), - BPF_STMT(BPF_LD | BPF_IND | BPF_W, 0x8), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC | FLAG_SKB_FRAG, - { }, - { {0x40, 0x21071983} }, - .frag_data = { - 0x42, 0x00, 0x00, 0x00, - 0x43, 0x44, 0x00, 0x00, - 0x21, 0x07, 0x19, 0x83, - }, - }, - { - "LD_IND halfword mixed head/frag", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_IMM, 0x40), - BPF_STMT(BPF_LD | BPF_IND | BPF_H, -0x1), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC | FLAG_SKB_FRAG, - { [0x3e] = 0x25, [0x3f] = 0x05, }, - { {0x40, 0x0519} }, - .frag_data = { 0x19, 0x82 }, - }, - { - "LD_IND word mixed head/frag", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_IMM, 0x40), - BPF_STMT(BPF_LD | BPF_IND | BPF_W, -0x2), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC | FLAG_SKB_FRAG, - { [0x3e] = 0x25, [0x3f] = 0x05, }, - { {0x40, 0x25051982} }, - .frag_data = { 0x19, 0x82 }, - }, - { - "LD_ABS byte frag", - .u.insns = { - BPF_STMT(BPF_LD | BPF_ABS | BPF_B, 0x40), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC | FLAG_SKB_FRAG, - { }, - { {0x40, 0x42} }, - .frag_data = { - 0x42, 0x00, 0x00, 0x00, - 0x43, 0x44, 0x00, 0x00, - 0x21, 0x07, 0x19, 0x83, - }, - }, - { - "LD_ABS halfword frag", - .u.insns = { - BPF_STMT(BPF_LD | BPF_ABS | BPF_H, 0x44), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC | FLAG_SKB_FRAG, - { }, - { {0x40, 0x4344} }, - .frag_data = { - 0x42, 0x00, 0x00, 0x00, - 0x43, 0x44, 0x00, 0x00, - 0x21, 0x07, 0x19, 0x83, - }, - }, - { - "LD_ABS word frag", - .u.insns = { - BPF_STMT(BPF_LD | BPF_ABS | BPF_W, 0x48), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC | FLAG_SKB_FRAG, - { }, - { {0x40, 0x21071983} }, - .frag_data = { - 0x42, 0x00, 0x00, 0x00, - 0x43, 0x44, 0x00, 0x00, - 0x21, 0x07, 0x19, 0x83, - }, - }, - { - "LD_ABS halfword mixed head/frag", - .u.insns = { - BPF_STMT(BPF_LD | BPF_ABS | BPF_H, 0x3f), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC | FLAG_SKB_FRAG, - { [0x3e] = 0x25, [0x3f] = 0x05, }, - { {0x40, 0x0519} }, - .frag_data = { 0x19, 0x82 }, - }, - { - "LD_ABS word mixed head/frag", - .u.insns = { - BPF_STMT(BPF_LD | BPF_ABS | BPF_W, 0x3e), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC | FLAG_SKB_FRAG, - { [0x3e] = 0x25, [0x3f] = 0x05, }, - { {0x40, 0x25051982} }, - .frag_data = { 0x19, 0x82 }, - }, - /* - * LD_IND / LD_ABS on non fragmented SKBs - */ - { - /* - * this tests that the JIT/interpreter correctly resets X - * before using it in an LD_IND instruction. - */ - "LD_IND byte default X", - .u.insns = { - BPF_STMT(BPF_LD | BPF_IND | BPF_B, 0x1), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x1] = 0x42 }, - { {0x40, 0x42 } }, - }, - { - "LD_IND byte positive offset", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_IMM, 0x3e), - BPF_STMT(BPF_LD | BPF_IND | BPF_B, 0x1), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 }, - { {0x40, 0x82 } }, - }, - { - "LD_IND byte negative offset", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_IMM, 0x3e), - BPF_STMT(BPF_LD | BPF_IND | BPF_B, -0x1), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 }, - { {0x40, 0x05 } }, - }, - { - "LD_IND byte positive offset, all ff", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_IMM, 0x3e), - BPF_STMT(BPF_LD | BPF_IND | BPF_B, 0x1), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0xff, [0x3d] = 0xff, [0x3e] = 0xff, [0x3f] = 0xff }, - { {0x40, 0xff } }, - }, - { - "LD_IND byte positive offset, out of bounds", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_IMM, 0x3e), - BPF_STMT(BPF_LD | BPF_IND | BPF_B, 0x1), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 }, - { {0x3f, 0 }, }, - }, - { - "LD_IND byte negative offset, out of bounds", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_IMM, 0x3e), - BPF_STMT(BPF_LD | BPF_IND | BPF_B, -0x3f), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 }, - { {0x3f, 0 } }, - }, - { - "LD_IND byte negative offset, multiple calls", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_IMM, 0x3b), - BPF_STMT(BPF_LD | BPF_IND | BPF_B, SKF_LL_OFF + 1), - BPF_STMT(BPF_LD | BPF_IND | BPF_B, SKF_LL_OFF + 2), - BPF_STMT(BPF_LD | BPF_IND | BPF_B, SKF_LL_OFF + 3), - BPF_STMT(BPF_LD | BPF_IND | BPF_B, SKF_LL_OFF + 4), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 }, - { {0x40, 0x82 }, }, - }, - { - "LD_IND halfword positive offset", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_IMM, 0x20), - BPF_STMT(BPF_LD | BPF_IND | BPF_H, 0x2), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { - [0x1c] = 0xaa, [0x1d] = 0x55, - [0x1e] = 0xbb, [0x1f] = 0x66, - [0x20] = 0xcc, [0x21] = 0x77, - [0x22] = 0xdd, [0x23] = 0x88, - }, - { {0x40, 0xdd88 } }, - }, - { - "LD_IND halfword negative offset", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_IMM, 0x20), - BPF_STMT(BPF_LD | BPF_IND | BPF_H, -0x2), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { - [0x1c] = 0xaa, [0x1d] = 0x55, - [0x1e] = 0xbb, [0x1f] = 0x66, - [0x20] = 0xcc, [0x21] = 0x77, - [0x22] = 0xdd, [0x23] = 0x88, - }, - { {0x40, 0xbb66 } }, - }, - { - "LD_IND halfword unaligned", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_IMM, 0x20), - BPF_STMT(BPF_LD | BPF_IND | BPF_H, -0x1), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { - [0x1c] = 0xaa, [0x1d] = 0x55, - [0x1e] = 0xbb, [0x1f] = 0x66, - [0x20] = 0xcc, [0x21] = 0x77, - [0x22] = 0xdd, [0x23] = 0x88, - }, - { {0x40, 0x66cc } }, - }, - { - "LD_IND halfword positive offset, all ff", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_IMM, 0x3d), - BPF_STMT(BPF_LD | BPF_IND | BPF_H, 0x1), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0xff, [0x3d] = 0xff, [0x3e] = 0xff, [0x3f] = 0xff }, - { {0x40, 0xffff } }, - }, - { - "LD_IND halfword positive offset, out of bounds", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_IMM, 0x3e), - BPF_STMT(BPF_LD | BPF_IND | BPF_H, 0x1), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 }, - { {0x3f, 0 }, }, - }, - { - "LD_IND halfword negative offset, out of bounds", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_IMM, 0x3e), - BPF_STMT(BPF_LD | BPF_IND | BPF_H, -0x3f), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 }, - { {0x3f, 0 } }, - }, - { - "LD_IND word positive offset", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_IMM, 0x20), - BPF_STMT(BPF_LD | BPF_IND | BPF_W, 0x4), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { - [0x1c] = 0xaa, [0x1d] = 0x55, - [0x1e] = 0xbb, [0x1f] = 0x66, - [0x20] = 0xcc, [0x21] = 0x77, - [0x22] = 0xdd, [0x23] = 0x88, - [0x24] = 0xee, [0x25] = 0x99, - [0x26] = 0xff, [0x27] = 0xaa, - }, - { {0x40, 0xee99ffaa } }, - }, - { - "LD_IND word negative offset", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_IMM, 0x20), - BPF_STMT(BPF_LD | BPF_IND | BPF_W, -0x4), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { - [0x1c] = 0xaa, [0x1d] = 0x55, - [0x1e] = 0xbb, [0x1f] = 0x66, - [0x20] = 0xcc, [0x21] = 0x77, - [0x22] = 0xdd, [0x23] = 0x88, - [0x24] = 0xee, [0x25] = 0x99, - [0x26] = 0xff, [0x27] = 0xaa, - }, - { {0x40, 0xaa55bb66 } }, - }, - { - "LD_IND word unaligned (addr & 3 == 2)", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_IMM, 0x20), - BPF_STMT(BPF_LD | BPF_IND | BPF_W, -0x2), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { - [0x1c] = 0xaa, [0x1d] = 0x55, - [0x1e] = 0xbb, [0x1f] = 0x66, - [0x20] = 0xcc, [0x21] = 0x77, - [0x22] = 0xdd, [0x23] = 0x88, - [0x24] = 0xee, [0x25] = 0x99, - [0x26] = 0xff, [0x27] = 0xaa, - }, - { {0x40, 0xbb66cc77 } }, - }, - { - "LD_IND word unaligned (addr & 3 == 1)", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_IMM, 0x20), - BPF_STMT(BPF_LD | BPF_IND | BPF_W, -0x3), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { - [0x1c] = 0xaa, [0x1d] = 0x55, - [0x1e] = 0xbb, [0x1f] = 0x66, - [0x20] = 0xcc, [0x21] = 0x77, - [0x22] = 0xdd, [0x23] = 0x88, - [0x24] = 0xee, [0x25] = 0x99, - [0x26] = 0xff, [0x27] = 0xaa, - }, - { {0x40, 0x55bb66cc } }, - }, - { - "LD_IND word unaligned (addr & 3 == 3)", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_IMM, 0x20), - BPF_STMT(BPF_LD | BPF_IND | BPF_W, -0x1), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { - [0x1c] = 0xaa, [0x1d] = 0x55, - [0x1e] = 0xbb, [0x1f] = 0x66, - [0x20] = 0xcc, [0x21] = 0x77, - [0x22] = 0xdd, [0x23] = 0x88, - [0x24] = 0xee, [0x25] = 0x99, - [0x26] = 0xff, [0x27] = 0xaa, - }, - { {0x40, 0x66cc77dd } }, - }, - { - "LD_IND word positive offset, all ff", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_IMM, 0x3b), - BPF_STMT(BPF_LD | BPF_IND | BPF_W, 0x1), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0xff, [0x3d] = 0xff, [0x3e] = 0xff, [0x3f] = 0xff }, - { {0x40, 0xffffffff } }, - }, - { - "LD_IND word positive offset, out of bounds", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_IMM, 0x3e), - BPF_STMT(BPF_LD | BPF_IND | BPF_W, 0x1), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 }, - { {0x3f, 0 }, }, - }, - { - "LD_IND word negative offset, out of bounds", - .u.insns = { - BPF_STMT(BPF_LDX | BPF_IMM, 0x3e), - BPF_STMT(BPF_LD | BPF_IND | BPF_W, -0x3f), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 }, - { {0x3f, 0 } }, - }, - { - "LD_ABS byte", - .u.insns = { - BPF_STMT(BPF_LD | BPF_ABS | BPF_B, 0x20), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { - [0x1c] = 0xaa, [0x1d] = 0x55, - [0x1e] = 0xbb, [0x1f] = 0x66, - [0x20] = 0xcc, [0x21] = 0x77, - [0x22] = 0xdd, [0x23] = 0x88, - [0x24] = 0xee, [0x25] = 0x99, - [0x26] = 0xff, [0x27] = 0xaa, - }, - { {0x40, 0xcc } }, - }, - { - "LD_ABS byte positive offset, all ff", - .u.insns = { - BPF_STMT(BPF_LD | BPF_ABS | BPF_B, 0x3f), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0xff, [0x3d] = 0xff, [0x3e] = 0xff, [0x3f] = 0xff }, - { {0x40, 0xff } }, - }, - { - "LD_ABS byte positive offset, out of bounds", - .u.insns = { - BPF_STMT(BPF_LD | BPF_ABS | BPF_B, 0x3f), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 }, - { {0x3f, 0 }, }, - }, - { - "LD_ABS byte negative offset, out of bounds load", - .u.insns = { - BPF_STMT(BPF_LD | BPF_ABS | BPF_B, -1), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC | FLAG_EXPECTED_FAIL, - .expected_errcode = -EINVAL, - }, - { - "LD_ABS byte negative offset, in bounds", - .u.insns = { - BPF_STMT(BPF_LD | BPF_ABS | BPF_B, SKF_LL_OFF + 0x3f), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 }, - { {0x40, 0x82 }, }, - }, - { - "LD_ABS byte negative offset, out of bounds", - .u.insns = { - BPF_STMT(BPF_LD | BPF_ABS | BPF_B, SKF_LL_OFF + 0x3f), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 }, - { {0x3f, 0 }, }, - }, - { - "LD_ABS byte negative offset, multiple calls", - .u.insns = { - BPF_STMT(BPF_LD | BPF_ABS | BPF_B, SKF_LL_OFF + 0x3c), - BPF_STMT(BPF_LD | BPF_ABS | BPF_B, SKF_LL_OFF + 0x3d), - BPF_STMT(BPF_LD | BPF_ABS | BPF_B, SKF_LL_OFF + 0x3e), - BPF_STMT(BPF_LD | BPF_ABS | BPF_B, SKF_LL_OFF + 0x3f), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 }, - { {0x40, 0x82 }, }, - }, - { - "LD_ABS halfword", - .u.insns = { - BPF_STMT(BPF_LD | BPF_ABS | BPF_H, 0x22), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { - [0x1c] = 0xaa, [0x1d] = 0x55, - [0x1e] = 0xbb, [0x1f] = 0x66, - [0x20] = 0xcc, [0x21] = 0x77, - [0x22] = 0xdd, [0x23] = 0x88, - [0x24] = 0xee, [0x25] = 0x99, - [0x26] = 0xff, [0x27] = 0xaa, - }, - { {0x40, 0xdd88 } }, - }, - { - "LD_ABS halfword unaligned", - .u.insns = { - BPF_STMT(BPF_LD | BPF_ABS | BPF_H, 0x25), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { - [0x1c] = 0xaa, [0x1d] = 0x55, - [0x1e] = 0xbb, [0x1f] = 0x66, - [0x20] = 0xcc, [0x21] = 0x77, - [0x22] = 0xdd, [0x23] = 0x88, - [0x24] = 0xee, [0x25] = 0x99, - [0x26] = 0xff, [0x27] = 0xaa, - }, - { {0x40, 0x99ff } }, - }, - { - "LD_ABS halfword positive offset, all ff", - .u.insns = { - BPF_STMT(BPF_LD | BPF_ABS | BPF_H, 0x3e), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0xff, [0x3d] = 0xff, [0x3e] = 0xff, [0x3f] = 0xff }, - { {0x40, 0xffff } }, - }, - { - "LD_ABS halfword positive offset, out of bounds", - .u.insns = { - BPF_STMT(BPF_LD | BPF_ABS | BPF_H, 0x3f), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 }, - { {0x3f, 0 }, }, - }, - { - "LD_ABS halfword negative offset, out of bounds load", - .u.insns = { - BPF_STMT(BPF_LD | BPF_ABS | BPF_H, -1), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC | FLAG_EXPECTED_FAIL, - .expected_errcode = -EINVAL, - }, - { - "LD_ABS halfword negative offset, in bounds", - .u.insns = { - BPF_STMT(BPF_LD | BPF_ABS | BPF_H, SKF_LL_OFF + 0x3e), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 }, - { {0x40, 0x1982 }, }, - }, - { - "LD_ABS halfword negative offset, out of bounds", - .u.insns = { - BPF_STMT(BPF_LD | BPF_ABS | BPF_H, SKF_LL_OFF + 0x3e), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 }, - { {0x3f, 0 }, }, - }, - { - "LD_ABS word", - .u.insns = { - BPF_STMT(BPF_LD | BPF_ABS | BPF_W, 0x1c), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { - [0x1c] = 0xaa, [0x1d] = 0x55, - [0x1e] = 0xbb, [0x1f] = 0x66, - [0x20] = 0xcc, [0x21] = 0x77, - [0x22] = 0xdd, [0x23] = 0x88, - [0x24] = 0xee, [0x25] = 0x99, - [0x26] = 0xff, [0x27] = 0xaa, - }, - { {0x40, 0xaa55bb66 } }, - }, - { - "LD_ABS word unaligned (addr & 3 == 2)", - .u.insns = { - BPF_STMT(BPF_LD | BPF_ABS | BPF_W, 0x22), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { - [0x1c] = 0xaa, [0x1d] = 0x55, - [0x1e] = 0xbb, [0x1f] = 0x66, - [0x20] = 0xcc, [0x21] = 0x77, - [0x22] = 0xdd, [0x23] = 0x88, - [0x24] = 0xee, [0x25] = 0x99, - [0x26] = 0xff, [0x27] = 0xaa, - }, - { {0x40, 0xdd88ee99 } }, - }, - { - "LD_ABS word unaligned (addr & 3 == 1)", - .u.insns = { - BPF_STMT(BPF_LD | BPF_ABS | BPF_W, 0x21), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { - [0x1c] = 0xaa, [0x1d] = 0x55, - [0x1e] = 0xbb, [0x1f] = 0x66, - [0x20] = 0xcc, [0x21] = 0x77, - [0x22] = 0xdd, [0x23] = 0x88, - [0x24] = 0xee, [0x25] = 0x99, - [0x26] = 0xff, [0x27] = 0xaa, - }, - { {0x40, 0x77dd88ee } }, - }, - { - "LD_ABS word unaligned (addr & 3 == 3)", - .u.insns = { - BPF_STMT(BPF_LD | BPF_ABS | BPF_W, 0x23), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { - [0x1c] = 0xaa, [0x1d] = 0x55, - [0x1e] = 0xbb, [0x1f] = 0x66, - [0x20] = 0xcc, [0x21] = 0x77, - [0x22] = 0xdd, [0x23] = 0x88, - [0x24] = 0xee, [0x25] = 0x99, - [0x26] = 0xff, [0x27] = 0xaa, - }, - { {0x40, 0x88ee99ff } }, - }, - { - "LD_ABS word positive offset, all ff", - .u.insns = { - BPF_STMT(BPF_LD | BPF_ABS | BPF_W, 0x3c), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0xff, [0x3d] = 0xff, [0x3e] = 0xff, [0x3f] = 0xff }, - { {0x40, 0xffffffff } }, - }, - { - "LD_ABS word positive offset, out of bounds", - .u.insns = { - BPF_STMT(BPF_LD | BPF_ABS | BPF_W, 0x3f), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 }, - { {0x3f, 0 }, }, - }, - { - "LD_ABS word negative offset, out of bounds load", - .u.insns = { - BPF_STMT(BPF_LD | BPF_ABS | BPF_W, -1), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC | FLAG_EXPECTED_FAIL, - .expected_errcode = -EINVAL, - }, - { - "LD_ABS word negative offset, in bounds", - .u.insns = { - BPF_STMT(BPF_LD | BPF_ABS | BPF_W, SKF_LL_OFF + 0x3c), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 }, - { {0x40, 0x25051982 }, }, - }, - { - "LD_ABS word negative offset, out of bounds", - .u.insns = { - BPF_STMT(BPF_LD | BPF_ABS | BPF_W, SKF_LL_OFF + 0x3c), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 }, - { {0x3f, 0 }, }, - }, - { - "LDX_MSH standalone, preserved A", - .u.insns = { - BPF_STMT(BPF_LD | BPF_IMM, 0xffeebbaa), - BPF_STMT(BPF_LDX | BPF_B | BPF_MSH, 0x3c), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 }, - { {0x40, 0xffeebbaa }, }, - }, - { - "LDX_MSH standalone, preserved A 2", - .u.insns = { - BPF_STMT(BPF_LD | BPF_IMM, 0x175e9d63), - BPF_STMT(BPF_LDX | BPF_B | BPF_MSH, 0x3c), - BPF_STMT(BPF_LDX | BPF_B | BPF_MSH, 0x3d), - BPF_STMT(BPF_LDX | BPF_B | BPF_MSH, 0x3e), - BPF_STMT(BPF_LDX | BPF_B | BPF_MSH, 0x3f), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 }, - { {0x40, 0x175e9d63 }, }, - }, - { - "LDX_MSH standalone, test result 1", - .u.insns = { - BPF_STMT(BPF_LD | BPF_IMM, 0xffeebbaa), - BPF_STMT(BPF_LDX | BPF_B | BPF_MSH, 0x3c), - BPF_STMT(BPF_MISC | BPF_TXA, 0), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 }, - { {0x40, 0x14 }, }, - }, - { - "LDX_MSH standalone, test result 2", - .u.insns = { - BPF_STMT(BPF_LD | BPF_IMM, 0xffeebbaa), - BPF_STMT(BPF_LDX | BPF_B | BPF_MSH, 0x3e), - BPF_STMT(BPF_MISC | BPF_TXA, 0), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 }, - { {0x40, 0x24 }, }, - }, - { - "LDX_MSH standalone, negative offset", - .u.insns = { - BPF_STMT(BPF_LD | BPF_IMM, 0xffeebbaa), - BPF_STMT(BPF_LDX | BPF_B | BPF_MSH, -1), - BPF_STMT(BPF_MISC | BPF_TXA, 0), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 }, - { {0x40, 0 }, }, - }, - { - "LDX_MSH standalone, negative offset 2", - .u.insns = { - BPF_STMT(BPF_LD | BPF_IMM, 0xffeebbaa), - BPF_STMT(BPF_LDX | BPF_B | BPF_MSH, SKF_LL_OFF + 0x3e), - BPF_STMT(BPF_MISC | BPF_TXA, 0), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 }, - { {0x40, 0x24 }, }, - }, - { - "LDX_MSH standalone, out of bounds", - .u.insns = { - BPF_STMT(BPF_LD | BPF_IMM, 0xffeebbaa), - BPF_STMT(BPF_LDX | BPF_B | BPF_MSH, 0x40), - BPF_STMT(BPF_MISC | BPF_TXA, 0), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC, - { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 }, - { {0x40, 0 }, }, - }, - /* - * verify that the interpreter or JIT correctly sets A and X - * to 0. - */ - { - "ADD default X", - .u.insns = { - /* - * A = 0x42 - * A = A + X - * ret A - */ - BPF_STMT(BPF_LD | BPF_IMM, 0x42), - BPF_STMT(BPF_ALU | BPF_ADD | BPF_X, 0), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC | FLAG_NO_DATA, - {}, - { {0x1, 0x42 } }, - }, - { - "ADD default A", - .u.insns = { - /* - * A = A + 0x42 - * ret A - */ - BPF_STMT(BPF_ALU | BPF_ADD | BPF_K, 0x42), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC | FLAG_NO_DATA, - {}, - { {0x1, 0x42 } }, - }, - { - "SUB default X", - .u.insns = { - /* - * A = 0x66 - * A = A - X - * ret A - */ - BPF_STMT(BPF_LD | BPF_IMM, 0x66), - BPF_STMT(BPF_ALU | BPF_SUB | BPF_X, 0), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC | FLAG_NO_DATA, - {}, - { {0x1, 0x66 } }, - }, - { - "SUB default A", - .u.insns = { - /* - * A = A - -0x66 - * ret A - */ - BPF_STMT(BPF_ALU | BPF_SUB | BPF_K, -0x66), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC | FLAG_NO_DATA, - {}, - { {0x1, 0x66 } }, - }, - { - "MUL default X", - .u.insns = { - /* - * A = 0x42 - * A = A * X - * ret A - */ - BPF_STMT(BPF_LD | BPF_IMM, 0x42), - BPF_STMT(BPF_ALU | BPF_MUL | BPF_X, 0), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC | FLAG_NO_DATA, - {}, - { {0x1, 0x0 } }, - }, - { - "MUL default A", - .u.insns = { - /* - * A = A * 0x66 - * ret A - */ - BPF_STMT(BPF_ALU | BPF_MUL | BPF_K, 0x66), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC | FLAG_NO_DATA, - {}, - { {0x1, 0x0 } }, - }, - { - "DIV default X", - .u.insns = { - /* - * A = 0x42 - * A = A / X ; this halt the filter execution if X is 0 - * ret 0x42 - */ - BPF_STMT(BPF_LD | BPF_IMM, 0x42), - BPF_STMT(BPF_ALU | BPF_DIV | BPF_X, 0), - BPF_STMT(BPF_RET | BPF_K, 0x42), - }, - CLASSIC | FLAG_NO_DATA, - {}, - { {0x1, 0x0 } }, - }, - { - "DIV default A", - .u.insns = { - /* - * A = A / 1 - * ret A - */ - BPF_STMT(BPF_ALU | BPF_DIV | BPF_K, 0x1), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC | FLAG_NO_DATA, - {}, - { {0x1, 0x0 } }, - }, - { - "MOD default X", - .u.insns = { - /* - * A = 0x42 - * A = A mod X ; this halt the filter execution if X is 0 - * ret 0x42 - */ - BPF_STMT(BPF_LD | BPF_IMM, 0x42), - BPF_STMT(BPF_ALU | BPF_MOD | BPF_X, 0), - BPF_STMT(BPF_RET | BPF_K, 0x42), - }, - CLASSIC | FLAG_NO_DATA, - {}, - { {0x1, 0x0 } }, - }, - { - "MOD default A", - .u.insns = { - /* - * A = A mod 1 - * ret A - */ - BPF_STMT(BPF_ALU | BPF_MOD | BPF_K, 0x1), - BPF_STMT(BPF_RET | BPF_A, 0x0), - }, - CLASSIC | FLAG_NO_DATA, - {}, - { {0x1, 0x0 } }, - }, - { - "JMP EQ default A", - .u.insns = { - /* - * cmp A, 0x0, 0, 1 - * ret 0x42 - * ret 0x66 - */ - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, 0x0, 0, 1), - BPF_STMT(BPF_RET | BPF_K, 0x42), - BPF_STMT(BPF_RET | BPF_K, 0x66), - }, - CLASSIC | FLAG_NO_DATA, - {}, - { {0x1, 0x42 } }, - }, - { - "JMP EQ default X", - .u.insns = { - /* - * A = 0x0 - * cmp A, X, 0, 1 - * ret 0x42 - * ret 0x66 - */ - BPF_STMT(BPF_LD | BPF_IMM, 0x0), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_X, 0x0, 0, 1), - BPF_STMT(BPF_RET | BPF_K, 0x42), - BPF_STMT(BPF_RET | BPF_K, 0x66), - }, - CLASSIC | FLAG_NO_DATA, - {}, - { {0x1, 0x42 } }, - }, - /* Checking interpreter vs JIT wrt signed extended imms. */ - { - "JNE signed compare, test 1", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R1, 0xfefbbc12), - BPF_ALU32_IMM(BPF_MOV, R3, 0xffff0000), - BPF_MOV64_REG(R2, R1), - BPF_ALU64_REG(BPF_AND, R2, R3), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_JMP_IMM(BPF_JNE, R2, -17104896, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 2), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "JNE signed compare, test 2", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R1, 0xfefbbc12), - BPF_ALU32_IMM(BPF_MOV, R3, 0xffff0000), - BPF_MOV64_REG(R2, R1), - BPF_ALU64_REG(BPF_AND, R2, R3), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_JMP_IMM(BPF_JNE, R2, 0xfefb0000, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 2), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "JNE signed compare, test 3", - .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R1, 0xfefbbc12), - BPF_ALU32_IMM(BPF_MOV, R3, 0xffff0000), - BPF_ALU32_IMM(BPF_MOV, R4, 0xfefb0000), - BPF_MOV64_REG(R2, R1), - BPF_ALU64_REG(BPF_AND, R2, R3), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_JMP_REG(BPF_JNE, R2, R4, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 2), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 2 } }, - }, - { - "JNE signed compare, test 4", - .u.insns_int = { - BPF_LD_IMM64(R1, -17104896), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_JMP_IMM(BPF_JNE, R1, -17104896, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 2), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 2 } }, - }, - { - "JNE signed compare, test 5", - .u.insns_int = { - BPF_LD_IMM64(R1, 0xfefb0000), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_JMP_IMM(BPF_JNE, R1, 0xfefb0000, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 2), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 1 } }, - }, - { - "JNE signed compare, test 6", - .u.insns_int = { - BPF_LD_IMM64(R1, 0x7efb0000), - BPF_ALU32_IMM(BPF_MOV, R0, 1), - BPF_JMP_IMM(BPF_JNE, R1, 0x7efb0000, 1), - BPF_ALU32_IMM(BPF_MOV, R0, 2), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 2 } }, - }, - { - "JNE signed compare, test 7", - .u.insns = { - BPF_STMT(BPF_LD | BPF_IMM, 0xffff0000), - BPF_STMT(BPF_MISC | BPF_TAX, 0), - BPF_STMT(BPF_LD | BPF_IMM, 0xfefbbc12), - BPF_STMT(BPF_ALU | BPF_AND | BPF_X, 0), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, 0xfefb0000, 1, 0), - BPF_STMT(BPF_RET | BPF_K, 1), - BPF_STMT(BPF_RET | BPF_K, 2), - }, - CLASSIC | FLAG_NO_DATA, - {}, - { { 0, 2 } }, - }, - /* BPF_LDX_MEM with operand aliasing */ - { - "LDX_MEM_B: operand register aliasing", - .u.insns_int = { - BPF_ST_MEM(BPF_B, R10, -8, 123), - BPF_MOV64_REG(R0, R10), - BPF_LDX_MEM(BPF_B, R0, R0, -8), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 123 } }, - .stack_depth = 8, - }, - { - "LDX_MEM_H: operand register aliasing", - .u.insns_int = { - BPF_ST_MEM(BPF_H, R10, -8, 12345), - BPF_MOV64_REG(R0, R10), - BPF_LDX_MEM(BPF_H, R0, R0, -8), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 12345 } }, - .stack_depth = 8, - }, - { - "LDX_MEM_W: operand register aliasing", - .u.insns_int = { - BPF_ST_MEM(BPF_W, R10, -8, 123456789), - BPF_MOV64_REG(R0, R10), - BPF_LDX_MEM(BPF_W, R0, R0, -8), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 123456789 } }, - .stack_depth = 8, - }, - { - "LDX_MEM_DW: operand register aliasing", - .u.insns_int = { - BPF_LD_IMM64(R1, 0x123456789abcdefULL), - BPF_STX_MEM(BPF_DW, R10, R1, -8), - BPF_MOV64_REG(R0, R10), - BPF_LDX_MEM(BPF_DW, R0, R0, -8), - BPF_ALU64_REG(BPF_SUB, R0, R1), - BPF_MOV64_REG(R1, R0), - BPF_ALU64_IMM(BPF_RSH, R1, 32), - BPF_ALU64_REG(BPF_OR, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } }, - .stack_depth = 8, - }, - /* - * Register (non-)clobbering tests for the case where a JIT implements - * complex ALU or ATOMIC operations via function calls. If so, the - * function call must be transparent to the eBPF registers. The JIT - * must therefore save and restore relevant registers across the call. - * The following tests check that the eBPF registers retain their - * values after such an operation. Mainly intended for complex ALU - * and atomic operation, but we run it for all. You never know... - * - * Note that each operations should be tested twice with different - * destinations, to check preservation for all registers. - */ -#define BPF_TEST_CLOBBER_ALU(alu, op, dst, src) \ - { \ - #alu "_" #op " to " #dst ": no clobbering", \ - .u.insns_int = { \ - BPF_ALU64_IMM(BPF_MOV, R0, R0), \ - BPF_ALU64_IMM(BPF_MOV, R1, R1), \ - BPF_ALU64_IMM(BPF_MOV, R2, R2), \ - BPF_ALU64_IMM(BPF_MOV, R3, R3), \ - BPF_ALU64_IMM(BPF_MOV, R4, R4), \ - BPF_ALU64_IMM(BPF_MOV, R5, R5), \ - BPF_ALU64_IMM(BPF_MOV, R6, R6), \ - BPF_ALU64_IMM(BPF_MOV, R7, R7), \ - BPF_ALU64_IMM(BPF_MOV, R8, R8), \ - BPF_ALU64_IMM(BPF_MOV, R9, R9), \ - BPF_##alu(BPF_ ##op, dst, src), \ - BPF_ALU32_IMM(BPF_MOV, dst, dst), \ - BPF_JMP_IMM(BPF_JNE, R0, R0, 10), \ - BPF_JMP_IMM(BPF_JNE, R1, R1, 9), \ - BPF_JMP_IMM(BPF_JNE, R2, R2, 8), \ - BPF_JMP_IMM(BPF_JNE, R3, R3, 7), \ - BPF_JMP_IMM(BPF_JNE, R4, R4, 6), \ - BPF_JMP_IMM(BPF_JNE, R5, R5, 5), \ - BPF_JMP_IMM(BPF_JNE, R6, R6, 4), \ - BPF_JMP_IMM(BPF_JNE, R7, R7, 3), \ - BPF_JMP_IMM(BPF_JNE, R8, R8, 2), \ - BPF_JMP_IMM(BPF_JNE, R9, R9, 1), \ - BPF_ALU64_IMM(BPF_MOV, R0, 1), \ - BPF_EXIT_INSN(), \ - }, \ - INTERNAL, \ - { }, \ - { { 0, 1 } } \ - } - /* ALU64 operations, register clobbering */ - BPF_TEST_CLOBBER_ALU(ALU64_IMM, AND, R8, 123456789), - BPF_TEST_CLOBBER_ALU(ALU64_IMM, AND, R9, 123456789), - BPF_TEST_CLOBBER_ALU(ALU64_IMM, OR, R8, 123456789), - BPF_TEST_CLOBBER_ALU(ALU64_IMM, OR, R9, 123456789), - BPF_TEST_CLOBBER_ALU(ALU64_IMM, XOR, R8, 123456789), - BPF_TEST_CLOBBER_ALU(ALU64_IMM, XOR, R9, 123456789), - BPF_TEST_CLOBBER_ALU(ALU64_IMM, LSH, R8, 12), - BPF_TEST_CLOBBER_ALU(ALU64_IMM, LSH, R9, 12), - BPF_TEST_CLOBBER_ALU(ALU64_IMM, RSH, R8, 12), - BPF_TEST_CLOBBER_ALU(ALU64_IMM, RSH, R9, 12), - BPF_TEST_CLOBBER_ALU(ALU64_IMM, ARSH, R8, 12), - BPF_TEST_CLOBBER_ALU(ALU64_IMM, ARSH, R9, 12), - BPF_TEST_CLOBBER_ALU(ALU64_IMM, ADD, R8, 123456789), - BPF_TEST_CLOBBER_ALU(ALU64_IMM, ADD, R9, 123456789), - BPF_TEST_CLOBBER_ALU(ALU64_IMM, SUB, R8, 123456789), - BPF_TEST_CLOBBER_ALU(ALU64_IMM, SUB, R9, 123456789), - BPF_TEST_CLOBBER_ALU(ALU64_IMM, MUL, R8, 123456789), - BPF_TEST_CLOBBER_ALU(ALU64_IMM, MUL, R9, 123456789), - BPF_TEST_CLOBBER_ALU(ALU64_IMM, DIV, R8, 123456789), - BPF_TEST_CLOBBER_ALU(ALU64_IMM, DIV, R9, 123456789), - BPF_TEST_CLOBBER_ALU(ALU64_IMM, MOD, R8, 123456789), - BPF_TEST_CLOBBER_ALU(ALU64_IMM, MOD, R9, 123456789), - /* ALU32 immediate operations, register clobbering */ - BPF_TEST_CLOBBER_ALU(ALU32_IMM, AND, R8, 123456789), - BPF_TEST_CLOBBER_ALU(ALU32_IMM, AND, R9, 123456789), - BPF_TEST_CLOBBER_ALU(ALU32_IMM, OR, R8, 123456789), - BPF_TEST_CLOBBER_ALU(ALU32_IMM, OR, R9, 123456789), - BPF_TEST_CLOBBER_ALU(ALU32_IMM, XOR, R8, 123456789), - BPF_TEST_CLOBBER_ALU(ALU32_IMM, XOR, R9, 123456789), - BPF_TEST_CLOBBER_ALU(ALU32_IMM, LSH, R8, 12), - BPF_TEST_CLOBBER_ALU(ALU32_IMM, LSH, R9, 12), - BPF_TEST_CLOBBER_ALU(ALU32_IMM, RSH, R8, 12), - BPF_TEST_CLOBBER_ALU(ALU32_IMM, RSH, R9, 12), - BPF_TEST_CLOBBER_ALU(ALU32_IMM, ARSH, R8, 12), - BPF_TEST_CLOBBER_ALU(ALU32_IMM, ARSH, R9, 12), - BPF_TEST_CLOBBER_ALU(ALU32_IMM, ADD, R8, 123456789), - BPF_TEST_CLOBBER_ALU(ALU32_IMM, ADD, R9, 123456789), - BPF_TEST_CLOBBER_ALU(ALU32_IMM, SUB, R8, 123456789), - BPF_TEST_CLOBBER_ALU(ALU32_IMM, SUB, R9, 123456789), - BPF_TEST_CLOBBER_ALU(ALU32_IMM, MUL, R8, 123456789), - BPF_TEST_CLOBBER_ALU(ALU32_IMM, MUL, R9, 123456789), - BPF_TEST_CLOBBER_ALU(ALU32_IMM, DIV, R8, 123456789), - BPF_TEST_CLOBBER_ALU(ALU32_IMM, DIV, R9, 123456789), - BPF_TEST_CLOBBER_ALU(ALU32_IMM, MOD, R8, 123456789), - BPF_TEST_CLOBBER_ALU(ALU32_IMM, MOD, R9, 123456789), - /* ALU64 register operations, register clobbering */ - BPF_TEST_CLOBBER_ALU(ALU64_REG, AND, R8, R1), - BPF_TEST_CLOBBER_ALU(ALU64_REG, AND, R9, R1), - BPF_TEST_CLOBBER_ALU(ALU64_REG, OR, R8, R1), - BPF_TEST_CLOBBER_ALU(ALU64_REG, OR, R9, R1), - BPF_TEST_CLOBBER_ALU(ALU64_REG, XOR, R8, R1), - BPF_TEST_CLOBBER_ALU(ALU64_REG, XOR, R9, R1), - BPF_TEST_CLOBBER_ALU(ALU64_REG, LSH, R8, R1), - BPF_TEST_CLOBBER_ALU(ALU64_REG, LSH, R9, R1), - BPF_TEST_CLOBBER_ALU(ALU64_REG, RSH, R8, R1), - BPF_TEST_CLOBBER_ALU(ALU64_REG, RSH, R9, R1), - BPF_TEST_CLOBBER_ALU(ALU64_REG, ARSH, R8, R1), - BPF_TEST_CLOBBER_ALU(ALU64_REG, ARSH, R9, R1), - BPF_TEST_CLOBBER_ALU(ALU64_REG, ADD, R8, R1), - BPF_TEST_CLOBBER_ALU(ALU64_REG, ADD, R9, R1), - BPF_TEST_CLOBBER_ALU(ALU64_REG, SUB, R8, R1), - BPF_TEST_CLOBBER_ALU(ALU64_REG, SUB, R9, R1), - BPF_TEST_CLOBBER_ALU(ALU64_REG, MUL, R8, R1), - BPF_TEST_CLOBBER_ALU(ALU64_REG, MUL, R9, R1), - BPF_TEST_CLOBBER_ALU(ALU64_REG, DIV, R8, R1), - BPF_TEST_CLOBBER_ALU(ALU64_REG, DIV, R9, R1), - BPF_TEST_CLOBBER_ALU(ALU64_REG, MOD, R8, R1), - BPF_TEST_CLOBBER_ALU(ALU64_REG, MOD, R9, R1), - /* ALU32 register operations, register clobbering */ - BPF_TEST_CLOBBER_ALU(ALU32_REG, AND, R8, R1), - BPF_TEST_CLOBBER_ALU(ALU32_REG, AND, R9, R1), - BPF_TEST_CLOBBER_ALU(ALU32_REG, OR, R8, R1), - BPF_TEST_CLOBBER_ALU(ALU32_REG, OR, R9, R1), - BPF_TEST_CLOBBER_ALU(ALU32_REG, XOR, R8, R1), - BPF_TEST_CLOBBER_ALU(ALU32_REG, XOR, R9, R1), - BPF_TEST_CLOBBER_ALU(ALU32_REG, LSH, R8, R1), - BPF_TEST_CLOBBER_ALU(ALU32_REG, LSH, R9, R1), - BPF_TEST_CLOBBER_ALU(ALU32_REG, RSH, R8, R1), - BPF_TEST_CLOBBER_ALU(ALU32_REG, RSH, R9, R1), - BPF_TEST_CLOBBER_ALU(ALU32_REG, ARSH, R8, R1), - BPF_TEST_CLOBBER_ALU(ALU32_REG, ARSH, R9, R1), - BPF_TEST_CLOBBER_ALU(ALU32_REG, ADD, R8, R1), - BPF_TEST_CLOBBER_ALU(ALU32_REG, ADD, R9, R1), - BPF_TEST_CLOBBER_ALU(ALU32_REG, SUB, R8, R1), - BPF_TEST_CLOBBER_ALU(ALU32_REG, SUB, R9, R1), - BPF_TEST_CLOBBER_ALU(ALU32_REG, MUL, R8, R1), - BPF_TEST_CLOBBER_ALU(ALU32_REG, MUL, R9, R1), - BPF_TEST_CLOBBER_ALU(ALU32_REG, DIV, R8, R1), - BPF_TEST_CLOBBER_ALU(ALU32_REG, DIV, R9, R1), - BPF_TEST_CLOBBER_ALU(ALU32_REG, MOD, R8, R1), - BPF_TEST_CLOBBER_ALU(ALU32_REG, MOD, R9, R1), -#undef BPF_TEST_CLOBBER_ALU -#define BPF_TEST_CLOBBER_ATOMIC(width, op) \ - { \ - "Atomic_" #width " " #op ": no clobbering", \ - .u.insns_int = { \ - BPF_ALU64_IMM(BPF_MOV, R0, 0), \ - BPF_ALU64_IMM(BPF_MOV, R1, 1), \ - BPF_ALU64_IMM(BPF_MOV, R2, 2), \ - BPF_ALU64_IMM(BPF_MOV, R3, 3), \ - BPF_ALU64_IMM(BPF_MOV, R4, 4), \ - BPF_ALU64_IMM(BPF_MOV, R5, 5), \ - BPF_ALU64_IMM(BPF_MOV, R6, 6), \ - BPF_ALU64_IMM(BPF_MOV, R7, 7), \ - BPF_ALU64_IMM(BPF_MOV, R8, 8), \ - BPF_ALU64_IMM(BPF_MOV, R9, 9), \ - BPF_ST_MEM(width, R10, -8, \ - (op) == BPF_CMPXCHG ? 0 : \ - (op) & BPF_FETCH ? 1 : 0), \ - BPF_ATOMIC_OP(width, op, R10, R1, -8), \ - BPF_JMP_IMM(BPF_JNE, R0, 0, 10), \ - BPF_JMP_IMM(BPF_JNE, R1, 1, 9), \ - BPF_JMP_IMM(BPF_JNE, R2, 2, 8), \ - BPF_JMP_IMM(BPF_JNE, R3, 3, 7), \ - BPF_JMP_IMM(BPF_JNE, R4, 4, 6), \ - BPF_JMP_IMM(BPF_JNE, R5, 5, 5), \ - BPF_JMP_IMM(BPF_JNE, R6, 6, 4), \ - BPF_JMP_IMM(BPF_JNE, R7, 7, 3), \ - BPF_JMP_IMM(BPF_JNE, R8, 8, 2), \ - BPF_JMP_IMM(BPF_JNE, R9, 9, 1), \ - BPF_ALU64_IMM(BPF_MOV, R0, 1), \ - BPF_EXIT_INSN(), \ - }, \ - INTERNAL, \ - { }, \ - { { 0, 1 } }, \ - .stack_depth = 8, \ - } - /* 64-bit atomic operations, register clobbering */ - BPF_TEST_CLOBBER_ATOMIC(BPF_DW, BPF_ADD), - BPF_TEST_CLOBBER_ATOMIC(BPF_DW, BPF_AND), - BPF_TEST_CLOBBER_ATOMIC(BPF_DW, BPF_OR), - BPF_TEST_CLOBBER_ATOMIC(BPF_DW, BPF_XOR), - BPF_TEST_CLOBBER_ATOMIC(BPF_DW, BPF_ADD | BPF_FETCH), - BPF_TEST_CLOBBER_ATOMIC(BPF_DW, BPF_AND | BPF_FETCH), - BPF_TEST_CLOBBER_ATOMIC(BPF_DW, BPF_OR | BPF_FETCH), - BPF_TEST_CLOBBER_ATOMIC(BPF_DW, BPF_XOR | BPF_FETCH), - BPF_TEST_CLOBBER_ATOMIC(BPF_DW, BPF_XCHG), - BPF_TEST_CLOBBER_ATOMIC(BPF_DW, BPF_CMPXCHG), - /* 32-bit atomic operations, register clobbering */ - BPF_TEST_CLOBBER_ATOMIC(BPF_W, BPF_ADD), - BPF_TEST_CLOBBER_ATOMIC(BPF_W, BPF_AND), - BPF_TEST_CLOBBER_ATOMIC(BPF_W, BPF_OR), - BPF_TEST_CLOBBER_ATOMIC(BPF_W, BPF_XOR), - BPF_TEST_CLOBBER_ATOMIC(BPF_W, BPF_ADD | BPF_FETCH), - BPF_TEST_CLOBBER_ATOMIC(BPF_W, BPF_AND | BPF_FETCH), - BPF_TEST_CLOBBER_ATOMIC(BPF_W, BPF_OR | BPF_FETCH), - BPF_TEST_CLOBBER_ATOMIC(BPF_W, BPF_XOR | BPF_FETCH), - BPF_TEST_CLOBBER_ATOMIC(BPF_W, BPF_XCHG), - BPF_TEST_CLOBBER_ATOMIC(BPF_W, BPF_CMPXCHG), -#undef BPF_TEST_CLOBBER_ATOMIC - /* Checking that ALU32 src is not zero extended in place */ -#define BPF_ALU32_SRC_ZEXT(op) \ - { \ - "ALU32_" #op "_X: src preserved in zext", \ - .u.insns_int = { \ - BPF_LD_IMM64(R1, 0x0123456789acbdefULL),\ - BPF_LD_IMM64(R2, 0xfedcba9876543210ULL),\ - BPF_ALU64_REG(BPF_MOV, R0, R1), \ - BPF_ALU32_REG(BPF_##op, R2, R1), \ - BPF_ALU64_REG(BPF_SUB, R0, R1), \ - BPF_ALU64_REG(BPF_MOV, R1, R0), \ - BPF_ALU64_IMM(BPF_RSH, R1, 32), \ - BPF_ALU64_REG(BPF_OR, R0, R1), \ - BPF_EXIT_INSN(), \ - }, \ - INTERNAL, \ - { }, \ - { { 0, 0 } }, \ - } - BPF_ALU32_SRC_ZEXT(MOV), - BPF_ALU32_SRC_ZEXT(AND), - BPF_ALU32_SRC_ZEXT(OR), - BPF_ALU32_SRC_ZEXT(XOR), - BPF_ALU32_SRC_ZEXT(ADD), - BPF_ALU32_SRC_ZEXT(SUB), - BPF_ALU32_SRC_ZEXT(MUL), - BPF_ALU32_SRC_ZEXT(DIV), - BPF_ALU32_SRC_ZEXT(MOD), -#undef BPF_ALU32_SRC_ZEXT - /* Checking that ATOMIC32 src is not zero extended in place */ -#define BPF_ATOMIC32_SRC_ZEXT(op) \ - { \ - "ATOMIC_W_" #op ": src preserved in zext", \ - .u.insns_int = { \ - BPF_LD_IMM64(R0, 0x0123456789acbdefULL), \ - BPF_ALU64_REG(BPF_MOV, R1, R0), \ - BPF_ST_MEM(BPF_W, R10, -4, 0), \ - BPF_ATOMIC_OP(BPF_W, BPF_##op, R10, R1, -4), \ - BPF_ALU64_REG(BPF_SUB, R0, R1), \ - BPF_ALU64_REG(BPF_MOV, R1, R0), \ - BPF_ALU64_IMM(BPF_RSH, R1, 32), \ - BPF_ALU64_REG(BPF_OR, R0, R1), \ - BPF_EXIT_INSN(), \ - }, \ - INTERNAL, \ - { }, \ - { { 0, 0 } }, \ - .stack_depth = 8, \ - } - BPF_ATOMIC32_SRC_ZEXT(ADD), - BPF_ATOMIC32_SRC_ZEXT(AND), - BPF_ATOMIC32_SRC_ZEXT(OR), - BPF_ATOMIC32_SRC_ZEXT(XOR), -#undef BPF_ATOMIC32_SRC_ZEXT - /* Checking that CMPXCHG32 src is not zero extended in place */ - { - "ATOMIC_W_CMPXCHG: src preserved in zext", - .u.insns_int = { - BPF_LD_IMM64(R1, 0x0123456789acbdefULL), - BPF_ALU64_REG(BPF_MOV, R2, R1), - BPF_ALU64_REG(BPF_MOV, R0, 0), - BPF_ST_MEM(BPF_W, R10, -4, 0), - BPF_ATOMIC_OP(BPF_W, BPF_CMPXCHG, R10, R1, -4), - BPF_ALU64_REG(BPF_SUB, R1, R2), - BPF_ALU64_REG(BPF_MOV, R2, R1), - BPF_ALU64_IMM(BPF_RSH, R2, 32), - BPF_ALU64_REG(BPF_OR, R1, R2), - BPF_ALU64_REG(BPF_MOV, R0, R1), - BPF_EXIT_INSN(), - }, - INTERNAL, - { }, - { { 0, 0 } }, - .stack_depth = 8, - }, - /* Checking that JMP32 immediate src is not zero extended in place */ -#define BPF_JMP32_IMM_ZEXT(op) \ - { \ - "JMP32_" #op "_K: operand preserved in zext", \ - .u.insns_int = { \ - BPF_LD_IMM64(R0, 0x0123456789acbdefULL),\ - BPF_ALU64_REG(BPF_MOV, R1, R0), \ - BPF_JMP32_IMM(BPF_##op, R0, 1234, 1), \ - BPF_JMP_A(0), /* Nop */ \ - BPF_ALU64_REG(BPF_SUB, R0, R1), \ - BPF_ALU64_REG(BPF_MOV, R1, R0), \ - BPF_ALU64_IMM(BPF_RSH, R1, 32), \ - BPF_ALU64_REG(BPF_OR, R0, R1), \ - BPF_EXIT_INSN(), \ - }, \ - INTERNAL, \ - { }, \ - { { 0, 0 } }, \ - } - BPF_JMP32_IMM_ZEXT(JEQ), - BPF_JMP32_IMM_ZEXT(JNE), - BPF_JMP32_IMM_ZEXT(JSET), - BPF_JMP32_IMM_ZEXT(JGT), - BPF_JMP32_IMM_ZEXT(JGE), - BPF_JMP32_IMM_ZEXT(JLT), - BPF_JMP32_IMM_ZEXT(JLE), - BPF_JMP32_IMM_ZEXT(JSGT), - BPF_JMP32_IMM_ZEXT(JSGE), - BPF_JMP32_IMM_ZEXT(JSLT), - BPF_JMP32_IMM_ZEXT(JSLE), -#undef BPF_JMP2_IMM_ZEXT - /* Checking that JMP32 dst & src are not zero extended in place */ -#define BPF_JMP32_REG_ZEXT(op) \ - { \ - "JMP32_" #op "_X: operands preserved in zext", \ - .u.insns_int = { \ - BPF_LD_IMM64(R0, 0x0123456789acbdefULL),\ - BPF_LD_IMM64(R1, 0xfedcba9876543210ULL),\ - BPF_ALU64_REG(BPF_MOV, R2, R0), \ - BPF_ALU64_REG(BPF_MOV, R3, R1), \ - BPF_JMP32_IMM(BPF_##op, R0, R1, 1), \ - BPF_JMP_A(0), /* Nop */ \ - BPF_ALU64_REG(BPF_SUB, R0, R2), \ - BPF_ALU64_REG(BPF_SUB, R1, R3), \ - BPF_ALU64_REG(BPF_OR, R0, R1), \ - BPF_ALU64_REG(BPF_MOV, R1, R0), \ - BPF_ALU64_IMM(BPF_RSH, R1, 32), \ - BPF_ALU64_REG(BPF_OR, R0, R1), \ - BPF_EXIT_INSN(), \ - }, \ - INTERNAL, \ - { }, \ - { { 0, 0 } }, \ - } - BPF_JMP32_REG_ZEXT(JEQ), - BPF_JMP32_REG_ZEXT(JNE), - BPF_JMP32_REG_ZEXT(JSET), - BPF_JMP32_REG_ZEXT(JGT), - BPF_JMP32_REG_ZEXT(JGE), - BPF_JMP32_REG_ZEXT(JLT), - BPF_JMP32_REG_ZEXT(JLE), - BPF_JMP32_REG_ZEXT(JSGT), - BPF_JMP32_REG_ZEXT(JSGE), - BPF_JMP32_REG_ZEXT(JSLT), - BPF_JMP32_REG_ZEXT(JSLE), -#undef BPF_JMP2_REG_ZEXT - /* ALU64 K register combinations */ - { - "ALU64_MOV_K: registers", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_mov_imm_regs, - }, - { - "ALU64_AND_K: registers", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_and_imm_regs, - }, - { - "ALU64_OR_K: registers", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_or_imm_regs, - }, - { - "ALU64_XOR_K: registers", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_xor_imm_regs, - }, - { - "ALU64_LSH_K: registers", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_lsh_imm_regs, - }, - { - "ALU64_RSH_K: registers", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_rsh_imm_regs, - }, - { - "ALU64_ARSH_K: registers", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_arsh_imm_regs, - }, - { - "ALU64_ADD_K: registers", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_add_imm_regs, - }, - { - "ALU64_SUB_K: registers", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_sub_imm_regs, - }, - { - "ALU64_MUL_K: registers", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_mul_imm_regs, - }, - { - "ALU64_DIV_K: registers", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_div_imm_regs, - }, - { - "ALU64_MOD_K: registers", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_mod_imm_regs, - }, - { - "ALU64_SDIV_K: registers", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_sdiv_imm_regs, - }, - { - "ALU64_SMOD_K: registers", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_smod_imm_regs, - }, - /* ALU32 K registers */ - { - "ALU32_MOV_K: registers", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_mov_imm_regs, - }, - { - "ALU32_AND_K: registers", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_and_imm_regs, - }, - { - "ALU32_OR_K: registers", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_or_imm_regs, - }, - { - "ALU32_XOR_K: registers", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_xor_imm_regs, - }, - { - "ALU32_LSH_K: registers", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_lsh_imm_regs, - }, - { - "ALU32_RSH_K: registers", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_rsh_imm_regs, - }, - { - "ALU32_ARSH_K: registers", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_arsh_imm_regs, - }, - { - "ALU32_ADD_K: registers", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_add_imm_regs, - }, - { - "ALU32_SUB_K: registers", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_sub_imm_regs, - }, - { - "ALU32_MUL_K: registers", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_mul_imm_regs, - }, - { - "ALU32_DIV_K: registers", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_div_imm_regs, - }, - { - "ALU32_MOD_K: registers", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_mod_imm_regs, - }, - { - "ALU32_SDIV_K: registers", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_sdiv_imm_regs, - }, - { - "ALU32_SMOD_K: registers", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_smod_imm_regs, - }, - /* ALU64 X register combinations */ - { - "ALU64_MOV_X: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_mov_reg_pairs, - }, - { - "ALU64_AND_X: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_and_reg_pairs, - }, - { - "ALU64_OR_X: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_or_reg_pairs, - }, - { - "ALU64_XOR_X: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_xor_reg_pairs, - }, - { - "ALU64_LSH_X: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_lsh_reg_pairs, - }, - { - "ALU64_RSH_X: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_rsh_reg_pairs, - }, - { - "ALU64_ARSH_X: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_arsh_reg_pairs, - }, - { - "ALU64_ADD_X: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_add_reg_pairs, - }, - { - "ALU64_SUB_X: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_sub_reg_pairs, - }, - { - "ALU64_MUL_X: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_mul_reg_pairs, - }, - { - "ALU64_DIV_X: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_div_reg_pairs, - }, - { - "ALU64_MOD_X: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_mod_reg_pairs, - }, - /* ALU32 X register combinations */ - { - "ALU32_MOV_X: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_mov_reg_pairs, - }, - { - "ALU32_AND_X: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_and_reg_pairs, - }, - { - "ALU32_OR_X: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_or_reg_pairs, - }, - { - "ALU32_XOR_X: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_xor_reg_pairs, - }, - { - "ALU32_LSH_X: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_lsh_reg_pairs, - }, - { - "ALU32_RSH_X: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_rsh_reg_pairs, - }, - { - "ALU32_ARSH_X: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_arsh_reg_pairs, - }, - { - "ALU32_ADD_X: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_add_reg_pairs, - }, - { - "ALU32_SUB_X: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_sub_reg_pairs, - }, - { - "ALU32_MUL_X: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_mul_reg_pairs, - }, - { - "ALU32_DIV_X: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_div_reg_pairs, - }, - { - "ALU32_MOD_X register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_mod_reg_pairs, - }, - /* Exhaustive test of ALU64 shift operations */ - { - "ALU64_LSH_K: all shift values", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_lsh_imm, - }, - { - "ALU64_RSH_K: all shift values", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_rsh_imm, - }, - { - "ALU64_ARSH_K: all shift values", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_arsh_imm, - }, - { - "ALU64_LSH_X: all shift values", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_lsh_reg, - }, - { - "ALU64_RSH_X: all shift values", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_rsh_reg, - }, - { - "ALU64_ARSH_X: all shift values", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_arsh_reg, - }, - /* Exhaustive test of ALU32 shift operations */ - { - "ALU32_LSH_K: all shift values", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_lsh_imm, - }, - { - "ALU32_RSH_K: all shift values", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_rsh_imm, - }, - { - "ALU32_ARSH_K: all shift values", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_arsh_imm, - }, - { - "ALU32_LSH_X: all shift values", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_lsh_reg, - }, - { - "ALU32_RSH_X: all shift values", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_rsh_reg, - }, - { - "ALU32_ARSH_X: all shift values", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_arsh_reg, - }, - /* - * Exhaustive test of ALU64 shift operations when - * source and destination register are the same. - */ - { - "ALU64_LSH_X: all shift values with the same register", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_lsh_same_reg, - }, - { - "ALU64_RSH_X: all shift values with the same register", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_rsh_same_reg, - }, - { - "ALU64_ARSH_X: all shift values with the same register", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_arsh_same_reg, - }, - /* - * Exhaustive test of ALU32 shift operations when - * source and destination register are the same. - */ - { - "ALU32_LSH_X: all shift values with the same register", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_lsh_same_reg, - }, - { - "ALU32_RSH_X: all shift values with the same register", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_rsh_same_reg, - }, - { - "ALU32_ARSH_X: all shift values with the same register", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_arsh_same_reg, - }, - /* ALU64 immediate magnitudes */ - { - "ALU64_MOV_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_mov_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU64_AND_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_and_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU64_OR_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_or_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU64_XOR_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_xor_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU64_ADD_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_add_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU64_SUB_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_sub_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU64_MUL_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_mul_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU64_DIV_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_div_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU64_MOD_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_mod_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU64_SDIV_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_sdiv_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU64_SMOD_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_smod_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - /* ALU32 immediate magnitudes */ - { - "ALU32_MOV_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_mov_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU32_AND_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_and_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU32_OR_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_or_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU32_XOR_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_xor_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU32_ADD_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_add_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU32_SUB_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_sub_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU32_MUL_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_mul_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU32_DIV_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_div_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU32_MOD_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_mod_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU32_SDIV_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_sdiv_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU32_SMOD_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_smod_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - /* ALU64 register magnitudes */ - { - "ALU64_MOV_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_mov_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU64_AND_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_and_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU64_OR_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_or_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU64_XOR_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_xor_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU64_ADD_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_add_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU64_SUB_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_sub_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU64_MUL_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_mul_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU64_DIV_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_div_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU64_MOD_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu64_mod_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - /* ALU32 register magnitudes */ - { - "ALU32_MOV_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_mov_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU32_AND_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_and_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU32_OR_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_or_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU32_XOR_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_xor_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU32_ADD_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_add_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU32_SUB_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_sub_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU32_MUL_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_mul_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU32_DIV_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_div_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ALU32_MOD_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_alu32_mod_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - /* LD_IMM64 immediate magnitudes and byte patterns */ - { - "LD_IMM64: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_ld_imm64_magn, - }, - { - "LD_IMM64: checker byte patterns", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_ld_imm64_checker, - }, - { - "LD_IMM64: random positive and zero byte patterns", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_ld_imm64_pos_zero, - }, - { - "LD_IMM64: random negative and zero byte patterns", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_ld_imm64_neg_zero, - }, - { - "LD_IMM64: random positive and negative byte patterns", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_ld_imm64_pos_neg, - }, - /* 64-bit ATOMIC register combinations */ - { - "ATOMIC_DW_ADD: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic64_add_reg_pairs, - .stack_depth = 8, - }, - { - "ATOMIC_DW_AND: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic64_and_reg_pairs, - .stack_depth = 8, - }, - { - "ATOMIC_DW_OR: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic64_or_reg_pairs, - .stack_depth = 8, - }, - { - "ATOMIC_DW_XOR: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic64_xor_reg_pairs, - .stack_depth = 8, - }, - { - "ATOMIC_DW_ADD_FETCH: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic64_add_fetch_reg_pairs, - .stack_depth = 8, - }, - { - "ATOMIC_DW_AND_FETCH: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic64_and_fetch_reg_pairs, - .stack_depth = 8, - }, - { - "ATOMIC_DW_OR_FETCH: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic64_or_fetch_reg_pairs, - .stack_depth = 8, - }, - { - "ATOMIC_DW_XOR_FETCH: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic64_xor_fetch_reg_pairs, - .stack_depth = 8, - }, - { - "ATOMIC_DW_XCHG: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic64_xchg_reg_pairs, - .stack_depth = 8, - }, - { - "ATOMIC_DW_CMPXCHG: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic64_cmpxchg_reg_pairs, - .stack_depth = 8, - }, - /* 32-bit ATOMIC register combinations */ - { - "ATOMIC_W_ADD: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic32_add_reg_pairs, - .stack_depth = 8, - }, - { - "ATOMIC_W_AND: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic32_and_reg_pairs, - .stack_depth = 8, - }, - { - "ATOMIC_W_OR: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic32_or_reg_pairs, - .stack_depth = 8, - }, - { - "ATOMIC_W_XOR: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic32_xor_reg_pairs, - .stack_depth = 8, - }, - { - "ATOMIC_W_ADD_FETCH: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic32_add_fetch_reg_pairs, - .stack_depth = 8, - }, - { - "ATOMIC_W_AND_FETCH: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic32_and_fetch_reg_pairs, - .stack_depth = 8, - }, - { - "ATOMIC_W_OR_FETCH: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic32_or_fetch_reg_pairs, - .stack_depth = 8, - }, - { - "ATOMIC_W_XOR_FETCH: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic32_xor_fetch_reg_pairs, - .stack_depth = 8, - }, - { - "ATOMIC_W_XCHG: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic32_xchg_reg_pairs, - .stack_depth = 8, - }, - { - "ATOMIC_W_CMPXCHG: register combinations", - { }, - INTERNAL, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic32_cmpxchg_reg_pairs, - .stack_depth = 8, - }, - /* 64-bit ATOMIC magnitudes */ - { - "ATOMIC_DW_ADD: all operand magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic64_add, - .stack_depth = 8, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ATOMIC_DW_AND: all operand magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic64_and, - .stack_depth = 8, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ATOMIC_DW_OR: all operand magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic64_or, - .stack_depth = 8, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ATOMIC_DW_XOR: all operand magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic64_xor, - .stack_depth = 8, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ATOMIC_DW_ADD_FETCH: all operand magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic64_add_fetch, - .stack_depth = 8, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ATOMIC_DW_AND_FETCH: all operand magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic64_and_fetch, - .stack_depth = 8, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ATOMIC_DW_OR_FETCH: all operand magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic64_or_fetch, - .stack_depth = 8, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ATOMIC_DW_XOR_FETCH: all operand magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic64_xor_fetch, - .stack_depth = 8, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ATOMIC_DW_XCHG: all operand magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic64_xchg, - .stack_depth = 8, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ATOMIC_DW_CMPXCHG: all operand magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_cmpxchg64, - .stack_depth = 8, - .nr_testruns = NR_PATTERN_RUNS, - }, - /* 32-bit atomic magnitudes */ - { - "ATOMIC_W_ADD: all operand magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic32_add, - .stack_depth = 8, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ATOMIC_W_AND: all operand magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic32_and, - .stack_depth = 8, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ATOMIC_W_OR: all operand magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic32_or, - .stack_depth = 8, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ATOMIC_W_XOR: all operand magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic32_xor, - .stack_depth = 8, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ATOMIC_W_ADD_FETCH: all operand magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic32_add_fetch, - .stack_depth = 8, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ATOMIC_W_AND_FETCH: all operand magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic32_and_fetch, - .stack_depth = 8, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ATOMIC_W_OR_FETCH: all operand magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic32_or_fetch, - .stack_depth = 8, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ATOMIC_W_XOR_FETCH: all operand magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic32_xor_fetch, - .stack_depth = 8, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ATOMIC_W_XCHG: all operand magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_atomic32_xchg, - .stack_depth = 8, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "ATOMIC_W_CMPXCHG: all operand magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_cmpxchg32, - .stack_depth = 8, - .nr_testruns = NR_PATTERN_RUNS, - }, - /* JMP immediate magnitudes */ - { - "JMP_JSET_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp_jset_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP_JEQ_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp_jeq_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP_JNE_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp_jne_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP_JGT_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp_jgt_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP_JGE_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp_jge_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP_JLT_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp_jlt_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP_JLE_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp_jle_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP_JSGT_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp_jsgt_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP_JSGE_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp_jsge_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP_JSLT_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp_jslt_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP_JSLE_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp_jsle_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - /* JMP register magnitudes */ - { - "JMP_JSET_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp_jset_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP_JEQ_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp_jeq_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP_JNE_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp_jne_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP_JGT_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp_jgt_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP_JGE_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp_jge_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP_JLT_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp_jlt_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP_JLE_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp_jle_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP_JSGT_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp_jsgt_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP_JSGE_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp_jsge_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP_JSLT_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp_jslt_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP_JSLE_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp_jsle_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - /* JMP32 immediate magnitudes */ - { - "JMP32_JSET_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp32_jset_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP32_JEQ_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp32_jeq_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP32_JNE_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp32_jne_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP32_JGT_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp32_jgt_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP32_JGE_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp32_jge_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP32_JLT_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp32_jlt_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP32_JLE_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp32_jle_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP32_JSGT_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp32_jsgt_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP32_JSGE_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp32_jsge_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP32_JSLT_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp32_jslt_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP32_JSLE_K: all immediate value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp32_jsle_imm, - .nr_testruns = NR_PATTERN_RUNS, - }, - /* JMP32 register magnitudes */ - { - "JMP32_JSET_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp32_jset_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP32_JEQ_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp32_jeq_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP32_JNE_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp32_jne_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP32_JGT_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp32_jgt_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP32_JGE_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp32_jge_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP32_JLT_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp32_jlt_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP32_JLE_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp32_jle_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP32_JSGT_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp32_jsgt_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP32_JSGE_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp32_jsge_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP32_JSLT_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp32_jslt_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - { - "JMP32_JSLE_X: all register value magnitudes", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_jmp32_jsle_reg, - .nr_testruns = NR_PATTERN_RUNS, - }, - /* Conditional jumps with constant decision */ - { - "JMP_JSET_K: imm = 0 -> never taken", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 1), - BPF_JMP_IMM(BPF_JSET, R1, 0, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 0 } }, - }, - { - "JMP_JLT_K: imm = 0 -> never taken", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 1), - BPF_JMP_IMM(BPF_JLT, R1, 0, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 0 } }, - }, - { - "JMP_JGE_K: imm = 0 -> always taken", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 1), - BPF_JMP_IMM(BPF_JGE, R1, 0, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - }, - { - "JMP_JGT_K: imm = 0xffffffff -> never taken", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 1), - BPF_JMP_IMM(BPF_JGT, R1, U32_MAX, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 0 } }, - }, - { - "JMP_JLE_K: imm = 0xffffffff -> always taken", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 1), - BPF_JMP_IMM(BPF_JLE, R1, U32_MAX, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - }, - { - "JMP32_JSGT_K: imm = 0x7fffffff -> never taken", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 1), - BPF_JMP32_IMM(BPF_JSGT, R1, S32_MAX, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 0 } }, - }, - { - "JMP32_JSGE_K: imm = -0x80000000 -> always taken", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 1), - BPF_JMP32_IMM(BPF_JSGE, R1, S32_MIN, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - }, - { - "JMP32_JSLT_K: imm = -0x80000000 -> never taken", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 1), - BPF_JMP32_IMM(BPF_JSLT, R1, S32_MIN, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 0 } }, - }, - { - "JMP32_JSLE_K: imm = 0x7fffffff -> always taken", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 1), - BPF_JMP32_IMM(BPF_JSLE, R1, S32_MAX, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - }, - { - "JMP_JEQ_X: dst = src -> always taken", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 1), - BPF_JMP_REG(BPF_JEQ, R1, R1, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - }, - { - "JMP_JGE_X: dst = src -> always taken", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 1), - BPF_JMP_REG(BPF_JGE, R1, R1, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - }, - { - "JMP_JLE_X: dst = src -> always taken", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 1), - BPF_JMP_REG(BPF_JLE, R1, R1, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - }, - { - "JMP_JSGE_X: dst = src -> always taken", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 1), - BPF_JMP_REG(BPF_JSGE, R1, R1, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - }, - { - "JMP_JSLE_X: dst = src -> always taken", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 1), - BPF_JMP_REG(BPF_JSLE, R1, R1, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - }, - { - "JMP_JNE_X: dst = src -> never taken", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 1), - BPF_JMP_REG(BPF_JNE, R1, R1, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 0 } }, - }, - { - "JMP_JGT_X: dst = src -> never taken", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 1), - BPF_JMP_REG(BPF_JGT, R1, R1, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 0 } }, - }, - { - "JMP_JLT_X: dst = src -> never taken", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 1), - BPF_JMP_REG(BPF_JLT, R1, R1, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 0 } }, - }, - { - "JMP_JSGT_X: dst = src -> never taken", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 1), - BPF_JMP_REG(BPF_JSGT, R1, R1, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 0 } }, - }, - { - "JMP_JSLT_X: dst = src -> never taken", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 1), - BPF_JMP_REG(BPF_JSLT, R1, R1, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 0 } }, - }, - /* Short relative jumps */ - { - "Short relative jump: offset=0", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_JMP_IMM(BPF_JEQ, R0, 0, 0), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, -1), - }, - INTERNAL | FLAG_NO_DATA | FLAG_VERIFIER_ZEXT, - { }, - { { 0, 0 } }, - }, - { - "Short relative jump: offset=1", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_JMP_IMM(BPF_JEQ, R0, 0, 1), - BPF_ALU32_IMM(BPF_ADD, R0, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, -1), - }, - INTERNAL | FLAG_NO_DATA | FLAG_VERIFIER_ZEXT, - { }, - { { 0, 0 } }, - }, - { - "Short relative jump: offset=2", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_JMP_IMM(BPF_JEQ, R0, 0, 2), - BPF_ALU32_IMM(BPF_ADD, R0, 1), - BPF_ALU32_IMM(BPF_ADD, R0, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, -1), - }, - INTERNAL | FLAG_NO_DATA | FLAG_VERIFIER_ZEXT, - { }, - { { 0, 0 } }, - }, - { - "Short relative jump: offset=3", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_JMP_IMM(BPF_JEQ, R0, 0, 3), - BPF_ALU32_IMM(BPF_ADD, R0, 1), - BPF_ALU32_IMM(BPF_ADD, R0, 1), - BPF_ALU32_IMM(BPF_ADD, R0, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, -1), - }, - INTERNAL | FLAG_NO_DATA | FLAG_VERIFIER_ZEXT, - { }, - { { 0, 0 } }, - }, - { - "Short relative jump: offset=4", - .u.insns_int = { - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_JMP_IMM(BPF_JEQ, R0, 0, 4), - BPF_ALU32_IMM(BPF_ADD, R0, 1), - BPF_ALU32_IMM(BPF_ADD, R0, 1), - BPF_ALU32_IMM(BPF_ADD, R0, 1), - BPF_ALU32_IMM(BPF_ADD, R0, 1), - BPF_EXIT_INSN(), - BPF_ALU32_IMM(BPF_MOV, R0, -1), - }, - INTERNAL | FLAG_NO_DATA | FLAG_VERIFIER_ZEXT, - { }, - { { 0, 0 } }, - }, - /* Conditional branch conversions */ - { - "Long conditional jump: taken at runtime (32 bits)", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_max_jmp_taken_32, - }, - { - "Long conditional jump: not taken at runtime (32 bits)", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 2 } }, - .fill_helper = bpf_fill_max_jmp_not_taken_32, - }, - { - "Long conditional jump: always taken, known at JIT time (32 bits)", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_max_jmp_always_taken_32, - }, - { - "Long conditional jump: never taken, known at JIT time (32 bits)", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 2 } }, - .fill_helper = bpf_fill_max_jmp_never_taken_32, - }, - { - "Long conditional jump: taken at runtime", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_max_jmp_taken, - }, - { - "Long conditional jump: not taken at runtime", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 2 } }, - .fill_helper = bpf_fill_max_jmp_not_taken, - }, - { - "Long conditional jump: always taken, known at JIT time", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 1 } }, - .fill_helper = bpf_fill_max_jmp_always_taken, - }, - { - "Long conditional jump: never taken, known at JIT time", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, 2 } }, - .fill_helper = bpf_fill_max_jmp_never_taken, - }, - /* Staggered jump sequences, immediate */ - { - "Staggered jumps: JMP_JA", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_ja, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP_JEQ_K", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jeq_imm, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP_JNE_K", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jne_imm, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP_JSET_K", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jset_imm, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP_JGT_K", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jgt_imm, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP_JGE_K", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jge_imm, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP_JLT_K", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jlt_imm, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP_JLE_K", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jle_imm, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP_JSGT_K", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jsgt_imm, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP_JSGE_K", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jsge_imm, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP_JSLT_K", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jslt_imm, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP_JSLE_K", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jsle_imm, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - /* Staggered jump sequences, register */ - { - "Staggered jumps: JMP_JEQ_X", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jeq_reg, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP_JNE_X", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jne_reg, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP_JSET_X", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jset_reg, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP_JGT_X", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jgt_reg, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP_JGE_X", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jge_reg, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP_JLT_X", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jlt_reg, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP_JLE_X", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jle_reg, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP_JSGT_X", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jsgt_reg, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP_JSGE_X", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jsge_reg, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP_JSLT_X", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jslt_reg, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP_JSLE_X", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jsle_reg, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - /* Staggered jump sequences, JMP32 immediate */ - { - "Staggered jumps: JMP32_JEQ_K", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jeq32_imm, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP32_JNE_K", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jne32_imm, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP32_JSET_K", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jset32_imm, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP32_JGT_K", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jgt32_imm, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP32_JGE_K", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jge32_imm, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP32_JLT_K", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jlt32_imm, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP32_JLE_K", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jle32_imm, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP32_JSGT_K", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jsgt32_imm, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP32_JSGE_K", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jsge32_imm, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP32_JSLT_K", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jslt32_imm, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP32_JSLE_K", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jsle32_imm, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - /* Staggered jump sequences, JMP32 register */ - { - "Staggered jumps: JMP32_JEQ_X", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jeq32_reg, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP32_JNE_X", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jne32_reg, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP32_JSET_X", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jset32_reg, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP32_JGT_X", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jgt32_reg, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP32_JGE_X", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jge32_reg, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP32_JLT_X", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jlt32_reg, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP32_JLE_X", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jle32_reg, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP32_JSGT_X", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jsgt32_reg, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP32_JSGE_X", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jsge32_reg, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP32_JSLT_X", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jslt32_reg, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, - { - "Staggered jumps: JMP32_JSLE_X", - { }, - INTERNAL | FLAG_NO_DATA, - { }, - { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, - .fill_helper = bpf_fill_staggered_jsle32_reg, - .nr_testruns = NR_STAGGERED_JMP_RUNS, - }, -}; - -static struct net_device dev; - -static struct sk_buff *populate_skb(char *buf, int size) -{ - struct sk_buff *skb; - - if (size >= MAX_DATA) - return NULL; - - skb = alloc_skb(MAX_DATA, GFP_KERNEL); - if (!skb) - return NULL; - - __skb_put_data(skb, buf, size); - - /* Initialize a fake skb with test pattern. */ - skb_reset_mac_header(skb); - skb->protocol = htons(ETH_P_IP); - skb->pkt_type = SKB_TYPE; - skb->mark = SKB_MARK; - skb->hash = SKB_HASH; - skb->queue_mapping = SKB_QUEUE_MAP; - skb->vlan_tci = SKB_VLAN_TCI; - skb->vlan_proto = htons(ETH_P_IP); - dev_net_set(&dev, &init_net); - skb->dev = &dev; - skb->dev->ifindex = SKB_DEV_IFINDEX; - skb->dev->type = SKB_DEV_TYPE; - skb_set_network_header(skb, min(size, ETH_HLEN)); - - return skb; -} - -static void *generate_test_data(struct bpf_test *test, int sub) -{ - struct sk_buff *skb; - struct page *page; - - if (test->aux & FLAG_NO_DATA) - return NULL; - - if (test->aux & FLAG_LARGE_MEM) - return kmalloc(test->test[sub].data_size, GFP_KERNEL); - - /* Test case expects an skb, so populate one. Various - * subtests generate skbs of different sizes based on - * the same data. - */ - skb = populate_skb(test->data, test->test[sub].data_size); - if (!skb) - return NULL; - - if (test->aux & FLAG_SKB_FRAG) { - /* - * when the test requires a fragmented skb, add a - * single fragment to the skb, filled with - * test->frag_data. - */ - page = alloc_page(GFP_KERNEL); - if (!page) - goto err_kfree_skb; - - memcpy(page_address(page), test->frag_data, MAX_DATA); - skb_add_rx_frag(skb, 0, page, 0, MAX_DATA, MAX_DATA); - } - - return skb; -err_kfree_skb: - kfree_skb(skb); - return NULL; -} - -static void release_test_data(const struct bpf_test *test, void *data) -{ - if (test->aux & FLAG_NO_DATA) - return; - - if (test->aux & FLAG_LARGE_MEM) - kfree(data); - else - kfree_skb(data); -} - -static int filter_length(int which) -{ - struct sock_filter *fp; - int len; - - if (tests[which].fill_helper) - return tests[which].u.ptr.len; - - fp = tests[which].u.insns; - for (len = MAX_INSNS - 1; len > 0; --len) - if (fp[len].code != 0 || fp[len].k != 0) - break; - - return len + 1; -} - -static void *filter_pointer(int which) -{ - if (tests[which].fill_helper) - return tests[which].u.ptr.insns; - else - return tests[which].u.insns; -} - -static struct bpf_prog *generate_filter(int which, int *err) -{ - __u8 test_type = tests[which].aux & TEST_TYPE_MASK; - unsigned int flen = filter_length(which); - void *fptr = filter_pointer(which); - struct sock_fprog_kern fprog; - struct bpf_prog *fp; - - switch (test_type) { - case CLASSIC: - fprog.filter = fptr; - fprog.len = flen; - - *err = bpf_prog_create(&fp, &fprog); - if (tests[which].aux & FLAG_EXPECTED_FAIL) { - if (*err == tests[which].expected_errcode) { - pr_cont("PASS\n"); - /* Verifier rejected filter as expected. */ - *err = 0; - return NULL; - } else { - pr_cont("UNEXPECTED_PASS\n"); - /* Verifier didn't reject the test that's - * bad enough, just return! - */ - *err = -EINVAL; - return NULL; - } - } - if (*err) { - pr_cont("FAIL to prog_create err=%d len=%d\n", - *err, fprog.len); - return NULL; - } - break; - - case INTERNAL: - fp = bpf_prog_alloc(bpf_prog_size(flen), 0); - if (fp == NULL) { - pr_cont("UNEXPECTED_FAIL no memory left\n"); - *err = -ENOMEM; - return NULL; - } - - fp->len = flen; - /* Type doesn't really matter here as long as it's not unspec. */ - fp->type = BPF_PROG_TYPE_SOCKET_FILTER; - memcpy(fp->insnsi, fptr, fp->len * sizeof(struct bpf_insn)); - fp->aux->stack_depth = tests[which].stack_depth; - fp->aux->verifier_zext = !!(tests[which].aux & - FLAG_VERIFIER_ZEXT); - - /* We cannot error here as we don't need type compatibility - * checks. - */ - fp = bpf_prog_select_runtime(fp, err); - if (*err) { - pr_cont("FAIL to select_runtime err=%d\n", *err); - return NULL; - } - break; - } - - *err = 0; - return fp; -} - -static void release_filter(struct bpf_prog *fp, int which) -{ - __u8 test_type = tests[which].aux & TEST_TYPE_MASK; - - switch (test_type) { - case CLASSIC: - bpf_prog_destroy(fp); - break; - case INTERNAL: - bpf_prog_free(fp); - break; - } -} - -static int __run_one(const struct bpf_prog *fp, const void *data, - int runs, u64 *duration) -{ - u64 start, finish; - int ret = 0, i; - - migrate_disable(); - start = ktime_get_ns(); - - for (i = 0; i < runs; i++) - ret = bpf_prog_run(fp, data); - - finish = ktime_get_ns(); - migrate_enable(); - - *duration = finish - start; - do_div(*duration, runs); - - return ret; -} - -static int run_one(const struct bpf_prog *fp, struct bpf_test *test) -{ - int err_cnt = 0, i, runs = MAX_TESTRUNS; - - if (test->nr_testruns) - runs = min(test->nr_testruns, MAX_TESTRUNS); - - for (i = 0; i < MAX_SUBTESTS; i++) { - void *data; - u64 duration; - u32 ret; - - /* - * NOTE: Several sub-tests may be present, in which case - * a zero {data_size, result} tuple indicates the end of - * the sub-test array. The first test is always run, - * even if both data_size and result happen to be zero. - */ - if (i > 0 && - test->test[i].data_size == 0 && - test->test[i].result == 0) - break; - - data = generate_test_data(test, i); - if (!data && !(test->aux & FLAG_NO_DATA)) { - pr_cont("data generation failed "); - err_cnt++; - break; - } - ret = __run_one(fp, data, runs, &duration); - release_test_data(test, data); - - if (ret == test->test[i].result) { - pr_cont("%lld ", duration); - } else { - s32 res = test->test[i].result; - - pr_cont("ret %d != %d (%#x != %#x)", - ret, res, ret, res); - err_cnt++; - } - } - - return err_cnt; -} - -static char test_name[64]; -module_param_string(test_name, test_name, sizeof(test_name), 0); - -static int test_id = -1; -module_param(test_id, int, 0); - -static int test_range[2] = { 0, INT_MAX }; -module_param_array(test_range, int, NULL, 0); - -static bool exclude_test(int test_id) -{ - return test_id < test_range[0] || test_id > test_range[1]; -} - -static __init struct sk_buff *build_test_skb(void) -{ - u32 headroom = NET_SKB_PAD + NET_IP_ALIGN + ETH_HLEN; - struct sk_buff *skb[2]; - struct page *page[2]; - int i, data_size = 8; - - for (i = 0; i < 2; i++) { - page[i] = alloc_page(GFP_KERNEL); - if (!page[i]) { - if (i == 0) - goto err_page0; - else - goto err_page1; - } - - /* this will set skb[i]->head_frag */ - skb[i] = dev_alloc_skb(headroom + data_size); - if (!skb[i]) { - if (i == 0) - goto err_skb0; - else - goto err_skb1; - } - - skb_reserve(skb[i], headroom); - skb_put(skb[i], data_size); - skb[i]->protocol = htons(ETH_P_IP); - skb_reset_network_header(skb[i]); - skb_set_mac_header(skb[i], -ETH_HLEN); - - skb_add_rx_frag(skb[i], 0, page[i], 0, 64, 64); - // skb_headlen(skb[i]): 8, skb[i]->head_frag = 1 - } - - /* setup shinfo */ - skb_shinfo(skb[0])->gso_size = 1448; - skb_shinfo(skb[0])->gso_type = SKB_GSO_TCPV4; - skb_shinfo(skb[0])->gso_type |= SKB_GSO_DODGY; - skb_shinfo(skb[0])->gso_segs = 0; - skb_shinfo(skb[0])->frag_list = skb[1]; - skb_shinfo(skb[0])->hwtstamps.hwtstamp = 1000; - - /* adjust skb[0]'s len */ - skb[0]->len += skb[1]->len; - skb[0]->data_len += skb[1]->data_len; - skb[0]->truesize += skb[1]->truesize; - - return skb[0]; - -err_skb1: - __free_page(page[1]); -err_page1: - kfree_skb(skb[0]); -err_skb0: - __free_page(page[0]); -err_page0: - return NULL; -} - -static __init struct sk_buff *build_test_skb_linear_no_head_frag(void) -{ - unsigned int alloc_size = 2000; - unsigned int headroom = 102, doffset = 72, data_size = 1308; - struct sk_buff *skb[2]; - int i; - - /* skbs linked in a frag_list, both with linear data, with head_frag=0 - * (data allocated by kmalloc), both have tcp data of 1308 bytes - * (total payload is 2616 bytes). - * Data offset is 72 bytes (40 ipv6 hdr, 32 tcp hdr). Some headroom. - */ - for (i = 0; i < 2; i++) { - skb[i] = alloc_skb(alloc_size, GFP_KERNEL); - if (!skb[i]) { - if (i == 0) - goto err_skb0; - else - goto err_skb1; - } - - skb[i]->protocol = htons(ETH_P_IPV6); - skb_reserve(skb[i], headroom); - skb_put(skb[i], doffset + data_size); - skb_reset_network_header(skb[i]); - if (i == 0) - skb_reset_mac_header(skb[i]); - else - skb_set_mac_header(skb[i], -ETH_HLEN); - __skb_pull(skb[i], doffset); - } - - /* setup shinfo. - * mimic bpf_skb_proto_4_to_6, which resets gso_segs and assigns a - * reduced gso_size. - */ - skb_shinfo(skb[0])->gso_size = 1288; - skb_shinfo(skb[0])->gso_type = SKB_GSO_TCPV6 | SKB_GSO_DODGY; - skb_shinfo(skb[0])->gso_segs = 0; - skb_shinfo(skb[0])->frag_list = skb[1]; - - /* adjust skb[0]'s len */ - skb[0]->len += skb[1]->len; - skb[0]->data_len += skb[1]->len; - skb[0]->truesize += skb[1]->truesize; - - return skb[0]; - -err_skb1: - kfree_skb(skb[0]); -err_skb0: - return NULL; -} - -struct skb_segment_test { - const char *descr; - struct sk_buff *(*build_skb)(void); - netdev_features_t features; -}; - -static struct skb_segment_test skb_segment_tests[] __initconst = { - { - .descr = "gso_with_rx_frags", - .build_skb = build_test_skb, - .features = NETIF_F_SG | NETIF_F_GSO_PARTIAL | NETIF_F_IP_CSUM | - NETIF_F_IPV6_CSUM | NETIF_F_RXCSUM - }, - { - .descr = "gso_linear_no_head_frag", - .build_skb = build_test_skb_linear_no_head_frag, - .features = NETIF_F_SG | NETIF_F_FRAGLIST | - NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_GSO | - NETIF_F_GRO | NETIF_F_IPV6_CSUM | NETIF_F_RXCSUM | - NETIF_F_HW_VLAN_STAG_TX - } -}; - -static __init int test_skb_segment_single(const struct skb_segment_test *test) -{ - struct sk_buff *skb, *segs; - int ret = -1; - - skb = test->build_skb(); - if (!skb) { - pr_info("%s: failed to build_test_skb", __func__); - goto done; - } - - segs = skb_segment(skb, test->features); - if (!IS_ERR(segs)) { - kfree_skb_list(segs); - ret = 0; - } - kfree_skb(skb); -done: - return ret; -} - -static __init int test_skb_segment(void) -{ - int i, err_cnt = 0, pass_cnt = 0; - - for (i = 0; i < ARRAY_SIZE(skb_segment_tests); i++) { - const struct skb_segment_test *test = &skb_segment_tests[i]; - - cond_resched(); - if (exclude_test(i)) - continue; - - pr_info("#%d %s ", i, test->descr); - - if (test_skb_segment_single(test)) { - pr_cont("FAIL\n"); - err_cnt++; - } else { - pr_cont("PASS\n"); - pass_cnt++; - } - } - - pr_info("%s: Summary: %d PASSED, %d FAILED\n", __func__, - pass_cnt, err_cnt); - return err_cnt ? -EINVAL : 0; -} - -static __init int test_bpf(void) -{ - int i, err_cnt = 0, pass_cnt = 0; - int jit_cnt = 0, run_cnt = 0; - - for (i = 0; i < ARRAY_SIZE(tests); i++) { - struct bpf_prog *fp; - int err; - - cond_resched(); - if (exclude_test(i)) - continue; - - pr_info("#%d %s ", i, tests[i].descr); - - if (tests[i].fill_helper && - tests[i].fill_helper(&tests[i]) < 0) { - pr_cont("FAIL to prog_fill\n"); - continue; - } - - fp = generate_filter(i, &err); - - if (tests[i].fill_helper) { - kfree(tests[i].u.ptr.insns); - tests[i].u.ptr.insns = NULL; - } - - if (fp == NULL) { - if (err == 0) { - pass_cnt++; - continue; - } - err_cnt++; - continue; - } - - pr_cont("jited:%u ", fp->jited); - - run_cnt++; - if (fp->jited) - jit_cnt++; - - err = run_one(fp, &tests[i]); - release_filter(fp, i); - - if (err) { - pr_cont("FAIL (%d times)\n", err); - err_cnt++; - } else { - pr_cont("PASS\n"); - pass_cnt++; - } - } - - pr_info("Summary: %d PASSED, %d FAILED, [%d/%d JIT'ed]\n", - pass_cnt, err_cnt, jit_cnt, run_cnt); - - return err_cnt ? -EINVAL : 0; -} - -struct tail_call_test { - const char *descr; - struct bpf_insn insns[MAX_INSNS]; - int flags; - int result; - int stack_depth; - bool has_tail_call; -}; - -/* Flags that can be passed to tail call test cases */ -#define FLAG_NEED_STATE BIT(0) -#define FLAG_RESULT_IN_STATE BIT(1) - -/* - * Magic marker used in test snippets for tail calls below. - * BPF_LD/MOV to R2 and R2 with this immediate value is replaced - * with the proper values by the test runner. - */ -#define TAIL_CALL_MARKER 0x7a11ca11 - -/* Special offset to indicate a NULL call target */ -#define TAIL_CALL_NULL 0x7fff - -/* Special offset to indicate an out-of-range index */ -#define TAIL_CALL_INVALID 0x7ffe - -#define TAIL_CALL(offset) \ - BPF_LD_IMM64(R2, TAIL_CALL_MARKER), \ - BPF_RAW_INSN(BPF_ALU | BPF_MOV | BPF_K, R3, 0, \ - offset, TAIL_CALL_MARKER), \ - BPF_JMP_IMM(BPF_TAIL_CALL, 0, 0, 0) - -/* - * A test function to be called from a BPF program, clobbering a lot of - * CPU registers in the process. A JITed BPF program calling this function - * must save and restore any caller-saved registers it uses for internal - * state, for example the current tail call count. - */ -BPF_CALL_1(bpf_test_func, u64, arg) -{ - char buf[64]; - long a = 0; - long b = 1; - long c = 2; - long d = 3; - long e = 4; - long f = 5; - long g = 6; - long h = 7; - - return snprintf(buf, sizeof(buf), - "%ld %lu %lx %ld %lu %lx %ld %lu %x", - a, b, c, d, e, f, g, h, (int)arg); -} -#define BPF_FUNC_test_func __BPF_FUNC_MAX_ID - -/* - * Tail call tests. Each test case may call any other test in the table, - * including itself, specified as a relative index offset from the calling - * test. The index TAIL_CALL_NULL can be used to specify a NULL target - * function to test the JIT error path. Similarly, the index TAIL_CALL_INVALID - * results in a target index that is out of range. - */ -static struct tail_call_test tail_call_tests[] = { - { - "Tail call leaf", - .insns = { - BPF_ALU64_REG(BPF_MOV, R0, R1), - BPF_ALU64_IMM(BPF_ADD, R0, 1), - BPF_EXIT_INSN(), - }, - .result = 1, - }, - { - "Tail call 2", - .insns = { - BPF_ALU64_IMM(BPF_ADD, R1, 2), - TAIL_CALL(-1), - BPF_ALU64_IMM(BPF_MOV, R0, -1), - BPF_EXIT_INSN(), - }, - .result = 3, - .has_tail_call = true, - }, - { - "Tail call 3", - .insns = { - BPF_ALU64_IMM(BPF_ADD, R1, 3), - TAIL_CALL(-1), - BPF_ALU64_IMM(BPF_MOV, R0, -1), - BPF_EXIT_INSN(), - }, - .result = 6, - .has_tail_call = true, - }, - { - "Tail call 4", - .insns = { - BPF_ALU64_IMM(BPF_ADD, R1, 4), - TAIL_CALL(-1), - BPF_ALU64_IMM(BPF_MOV, R0, -1), - BPF_EXIT_INSN(), - }, - .result = 10, - .has_tail_call = true, - }, - { - "Tail call load/store leaf", - .insns = { - BPF_ALU64_IMM(BPF_MOV, R1, 1), - BPF_ALU64_IMM(BPF_MOV, R2, 2), - BPF_ALU64_REG(BPF_MOV, R3, BPF_REG_FP), - BPF_STX_MEM(BPF_DW, R3, R1, -8), - BPF_STX_MEM(BPF_DW, R3, R2, -16), - BPF_LDX_MEM(BPF_DW, R0, BPF_REG_FP, -8), - BPF_JMP_REG(BPF_JNE, R0, R1, 3), - BPF_LDX_MEM(BPF_DW, R0, BPF_REG_FP, -16), - BPF_JMP_REG(BPF_JNE, R0, R2, 1), - BPF_ALU64_IMM(BPF_MOV, R0, 0), - BPF_EXIT_INSN(), - }, - .result = 0, - .stack_depth = 32, - }, - { - "Tail call load/store", - .insns = { - BPF_ALU64_IMM(BPF_MOV, R0, 3), - BPF_STX_MEM(BPF_DW, BPF_REG_FP, R0, -8), - TAIL_CALL(-1), - BPF_ALU64_IMM(BPF_MOV, R0, -1), - BPF_EXIT_INSN(), - }, - .result = 0, - .stack_depth = 16, - .has_tail_call = true, - }, - { - "Tail call error path, max count reached", - .insns = { - BPF_LDX_MEM(BPF_W, R2, R1, 0), - BPF_ALU64_IMM(BPF_ADD, R2, 1), - BPF_STX_MEM(BPF_W, R1, R2, 0), - TAIL_CALL(0), - BPF_EXIT_INSN(), - }, - .flags = FLAG_NEED_STATE | FLAG_RESULT_IN_STATE, - .result = (MAX_TAIL_CALL_CNT + 1) * MAX_TESTRUNS, - .has_tail_call = true, - }, - { - "Tail call count preserved across function calls", - .insns = { - BPF_LDX_MEM(BPF_W, R2, R1, 0), - BPF_ALU64_IMM(BPF_ADD, R2, 1), - BPF_STX_MEM(BPF_W, R1, R2, 0), - BPF_STX_MEM(BPF_DW, R10, R1, -8), - BPF_CALL_REL(BPF_FUNC_get_numa_node_id), - BPF_CALL_REL(BPF_FUNC_ktime_get_ns), - BPF_CALL_REL(BPF_FUNC_ktime_get_boot_ns), - BPF_CALL_REL(BPF_FUNC_ktime_get_coarse_ns), - BPF_CALL_REL(BPF_FUNC_jiffies64), - BPF_CALL_REL(BPF_FUNC_test_func), - BPF_LDX_MEM(BPF_DW, R1, R10, -8), - BPF_ALU32_REG(BPF_MOV, R0, R1), - TAIL_CALL(0), - BPF_EXIT_INSN(), - }, - .stack_depth = 8, - .flags = FLAG_NEED_STATE | FLAG_RESULT_IN_STATE, - .result = (MAX_TAIL_CALL_CNT + 1) * MAX_TESTRUNS, - .has_tail_call = true, - }, - { - "Tail call error path, NULL target", - .insns = { - BPF_LDX_MEM(BPF_W, R2, R1, 0), - BPF_ALU64_IMM(BPF_ADD, R2, 1), - BPF_STX_MEM(BPF_W, R1, R2, 0), - TAIL_CALL(TAIL_CALL_NULL), - BPF_EXIT_INSN(), - }, - .flags = FLAG_NEED_STATE | FLAG_RESULT_IN_STATE, - .result = MAX_TESTRUNS, - .has_tail_call = true, - }, - { - "Tail call error path, index out of range", - .insns = { - BPF_LDX_MEM(BPF_W, R2, R1, 0), - BPF_ALU64_IMM(BPF_ADD, R2, 1), - BPF_STX_MEM(BPF_W, R1, R2, 0), - TAIL_CALL(TAIL_CALL_INVALID), - BPF_EXIT_INSN(), - }, - .flags = FLAG_NEED_STATE | FLAG_RESULT_IN_STATE, - .result = MAX_TESTRUNS, - .has_tail_call = true, - }, -}; - -static void __init destroy_tail_call_tests(struct bpf_array *progs) -{ - int i; - - for (i = 0; i < ARRAY_SIZE(tail_call_tests); i++) - if (progs->ptrs[i]) - bpf_prog_free(progs->ptrs[i]); - kfree(progs); -} - -static __init int prepare_tail_call_tests(struct bpf_array **pprogs) -{ - int ntests = ARRAY_SIZE(tail_call_tests); - struct bpf_array *progs; - int which, err; - - /* Allocate the table of programs to be used for tail calls */ - progs = kzalloc_flex(*progs, ptrs, ntests + 1); - if (!progs) - goto out_nomem; - - /* Create all eBPF programs and populate the table */ - for (which = 0; which < ntests; which++) { - struct tail_call_test *test = &tail_call_tests[which]; - struct bpf_prog *fp; - int len, i; - - /* Compute the number of program instructions */ - for (len = 0; len < MAX_INSNS; len++) { - struct bpf_insn *insn = &test->insns[len]; - - if (len < MAX_INSNS - 1 && - insn->code == (BPF_LD | BPF_DW | BPF_IMM)) - len++; - if (insn->code == 0) - break; - } - - /* Allocate and initialize the program */ - fp = bpf_prog_alloc(bpf_prog_size(len), 0); - if (!fp) - goto out_nomem; - - fp->len = len; - fp->type = BPF_PROG_TYPE_SOCKET_FILTER; - fp->aux->stack_depth = test->stack_depth; - fp->aux->tail_call_reachable = test->has_tail_call; - memcpy(fp->insnsi, test->insns, len * sizeof(struct bpf_insn)); - - /* Relocate runtime tail call offsets and addresses */ - for (i = 0; i < len; i++) { - struct bpf_insn *insn = &fp->insnsi[i]; - long addr = 0; - - switch (insn->code) { - case BPF_LD | BPF_DW | BPF_IMM: - if (insn->imm != TAIL_CALL_MARKER) - break; - insn[0].imm = (u32)(long)progs; - insn[1].imm = ((u64)(long)progs) >> 32; - break; - - case BPF_ALU | BPF_MOV | BPF_K: - if (insn->imm != TAIL_CALL_MARKER) - break; - if (insn->off == TAIL_CALL_NULL) - insn->imm = ntests; - else if (insn->off == TAIL_CALL_INVALID) - insn->imm = ntests + 1; - else - insn->imm = which + insn->off; - insn->off = 0; - break; - - case BPF_JMP | BPF_CALL: - if (insn->src_reg != BPF_PSEUDO_CALL) - break; - switch (insn->imm) { - case BPF_FUNC_get_numa_node_id: - addr = (long)&numa_node_id; - break; - case BPF_FUNC_ktime_get_ns: - addr = (long)&ktime_get_ns; - break; - case BPF_FUNC_ktime_get_boot_ns: - addr = (long)&ktime_get_boot_fast_ns; - break; - case BPF_FUNC_ktime_get_coarse_ns: - addr = (long)&ktime_get_coarse_ns; - break; - case BPF_FUNC_jiffies64: - addr = (long)&get_jiffies_64; - break; - case BPF_FUNC_test_func: - addr = (long)&bpf_test_func; - break; - default: - err = -EFAULT; - goto out_err; - } - *insn = BPF_EMIT_CALL(addr); - if ((long)__bpf_call_base + insn->imm != addr) - *insn = BPF_JMP_A(0); /* Skip: NOP */ - break; - } - } - - fp = bpf_prog_select_runtime(fp, &err); - if (err) - goto out_err; - - progs->ptrs[which] = fp; - } - - /* The last entry contains a NULL program pointer */ - progs->map.max_entries = ntests + 1; - *pprogs = progs; - return 0; - -out_nomem: - err = -ENOMEM; - -out_err: - if (progs) - destroy_tail_call_tests(progs); - return err; -} - -static __init int test_tail_calls(struct bpf_array *progs) -{ - int i, err_cnt = 0, pass_cnt = 0; - int jit_cnt = 0, run_cnt = 0; - - for (i = 0; i < ARRAY_SIZE(tail_call_tests); i++) { - struct tail_call_test *test = &tail_call_tests[i]; - struct bpf_prog *fp = progs->ptrs[i]; - int *data = NULL; - int state = 0; - u64 duration; - int ret; - - cond_resched(); - if (exclude_test(i)) - continue; - - pr_info("#%d %s ", i, test->descr); - if (!fp) { - err_cnt++; - continue; - } - pr_cont("jited:%u ", fp->jited); - - run_cnt++; - if (fp->jited) - jit_cnt++; - - if (test->flags & FLAG_NEED_STATE) - data = &state; - ret = __run_one(fp, data, MAX_TESTRUNS, &duration); - if (test->flags & FLAG_RESULT_IN_STATE) - ret = state; - if (ret == test->result) { - pr_cont("%lld PASS", duration); - pass_cnt++; - } else { - pr_cont("ret %d != %d FAIL", ret, test->result); - err_cnt++; - } - } - - pr_info("%s: Summary: %d PASSED, %d FAILED, [%d/%d JIT'ed]\n", - __func__, pass_cnt, err_cnt, jit_cnt, run_cnt); - - return err_cnt ? -EINVAL : 0; -} - -static char test_suite[32]; -module_param_string(test_suite, test_suite, sizeof(test_suite), 0); - -static __init int find_test_index(const char *test_name) -{ - int i; - - if (!strcmp(test_suite, "test_bpf")) { - for (i = 0; i < ARRAY_SIZE(tests); i++) { - if (!strcmp(tests[i].descr, test_name)) - return i; - } - } - - if (!strcmp(test_suite, "test_tail_calls")) { - for (i = 0; i < ARRAY_SIZE(tail_call_tests); i++) { - if (!strcmp(tail_call_tests[i].descr, test_name)) - return i; - } - } - - if (!strcmp(test_suite, "test_skb_segment")) { - for (i = 0; i < ARRAY_SIZE(skb_segment_tests); i++) { - if (!strcmp(skb_segment_tests[i].descr, test_name)) - return i; - } - } - - return -1; -} - -static __init int prepare_test_range(void) -{ - int valid_range; - - if (!strcmp(test_suite, "test_bpf")) - valid_range = ARRAY_SIZE(tests); - else if (!strcmp(test_suite, "test_tail_calls")) - valid_range = ARRAY_SIZE(tail_call_tests); - else if (!strcmp(test_suite, "test_skb_segment")) - valid_range = ARRAY_SIZE(skb_segment_tests); - else - return 0; - - if (test_id >= 0) { - /* - * if a test_id was specified, use test_range to - * cover only that test. - */ - if (test_id >= valid_range) { - pr_err("test_bpf: invalid test_id specified for '%s' suite.\n", - test_suite); - return -EINVAL; - } - - test_range[0] = test_id; - test_range[1] = test_id; - } else if (*test_name) { - /* - * if a test_name was specified, find it and setup - * test_range to cover only that test. - */ - int idx = find_test_index(test_name); - - if (idx < 0) { - pr_err("test_bpf: no test named '%s' found for '%s' suite.\n", - test_name, test_suite); - return -EINVAL; - } - test_range[0] = idx; - test_range[1] = idx; - } else if (test_range[0] != 0 || test_range[1] != INT_MAX) { - /* - * check that the supplied test_range is valid. - */ - if (test_range[0] < 0 || test_range[1] >= valid_range) { - pr_err("test_bpf: test_range is out of bound for '%s' suite.\n", - test_suite); - return -EINVAL; - } - - if (test_range[1] < test_range[0]) { - pr_err("test_bpf: test_range is ending before it starts.\n"); - return -EINVAL; - } - } - - return 0; -} - -static int __init test_bpf_init(void) -{ - struct bpf_array *progs = NULL; - int ret; - - if (strlen(test_suite) && - strcmp(test_suite, "test_bpf") && - strcmp(test_suite, "test_tail_calls") && - strcmp(test_suite, "test_skb_segment")) { - pr_err("test_bpf: invalid test_suite '%s' specified.\n", test_suite); - return -EINVAL; - } - - /* - * if test_suite is not specified, but test_id, test_name or test_range - * is specified, set 'test_bpf' as the default test suite. - */ - if (!strlen(test_suite) && - (test_id != -1 || strlen(test_name) || - (test_range[0] != 0 || test_range[1] != INT_MAX))) { - pr_info("test_bpf: set 'test_bpf' as the default test_suite.\n"); - strscpy(test_suite, "test_bpf", sizeof(test_suite)); - } - - ret = prepare_test_range(); - if (ret < 0) - return ret; - - if (!strlen(test_suite) || !strcmp(test_suite, "test_bpf")) { - ret = test_bpf(); - if (ret) - return ret; - } - - if (!strlen(test_suite) || !strcmp(test_suite, "test_tail_calls")) { - ret = prepare_tail_call_tests(&progs); - if (ret) - return ret; - ret = test_tail_calls(progs); - destroy_tail_call_tests(progs); - if (ret) - return ret; - } - - if (!strlen(test_suite) || !strcmp(test_suite, "test_skb_segment")) - return test_skb_segment(); - - return 0; -} - -static void __exit test_bpf_exit(void) -{ -} - -module_init(test_bpf_init); -module_exit(test_bpf_exit); - -MODULE_DESCRIPTION("Testsuite for BPF interpreter and BPF JIT compiler"); -MODULE_LICENSE("GPL"); diff --git a/mm/bpf_memcontrol.c b/mm/bpf_memcontrol.c index 716df49d7647..d8f579c28560 100644 --- a/mm/bpf_memcontrol.c +++ b/mm/bpf_memcontrol.c @@ -8,6 +8,8 @@ #include #include +#include "internal.h" + __bpf_kfunc_start_defs(); /** @@ -159,6 +161,51 @@ __bpf_kfunc void bpf_mem_cgroup_flush_stats(struct mem_cgroup *memcg) mem_cgroup_flush_stats(memcg); } +/** + * bpf_proactive_reclaim - proactively reclaim memory from a memory cgroup + * @memcg: the target memory cgroup to reclaim from. + * @size: the amount of memory to reclaim, in bytes, clamped to + * MEMCG_CHARGE_BATCH. + * @swappiness: the reclaim swappiness, in the range [MIN_SWAPPINESS, + * MAX_SWAPPINESS], or one of the special modes: -1 to use + * the memcg's own swappiness, or SWAPPINESS_ANON_ONLY to + * reclaim only anon folios. + * + * Performs one proactive reclaim pass on @memcg, like a write to + * memory.reclaim but without retrying until @size is reached. Call it + * repeatedly to reclaim more than one batch. + * + * Only available to BPF_PROG_TYPE_SYSCALL, because other sleepable programs + * may run with filesystem locks held, which the reclaim path can deadlock + * on via filesystem shrinkers. + * + * Return: The amount of memory reclaimed, in bytes, or a negative error. + */ +__bpf_kfunc long bpf_proactive_reclaim(struct mem_cgroup *memcg, + unsigned long size, + int swappiness) +{ + unsigned long nr_reclaimed; + unsigned long nr_pages; + + if (swappiness != -1 && swappiness != SWAPPINESS_ANON_ONLY) { + if (swappiness < MIN_SWAPPINESS || swappiness > MAX_SWAPPINESS) + return -EINVAL; + } + + if (size < PAGE_SIZE) + return -EINVAL; + + nr_pages = min(size / PAGE_SIZE, (unsigned long)MEMCG_CHARGE_BATCH); + + nr_reclaimed = try_to_free_mem_cgroup_pages(memcg, nr_pages, GFP_KERNEL, + MEMCG_RECLAIM_MAY_SWAP | + MEMCG_RECLAIM_PROACTIVE, + swappiness == -1 ? NULL : &swappiness); + + return nr_reclaimed * PAGE_SIZE; +} + __bpf_kfunc_end_defs(); BTF_KFUNCS_START(bpf_memcontrol_kfuncs) @@ -174,19 +221,35 @@ BTF_ID_FLAGS(func, bpf_mem_cgroup_flush_stats, KF_SLEEPABLE) BTF_KFUNCS_END(bpf_memcontrol_kfuncs) +BTF_KFUNCS_START(bpf_memcontrol_reclaim_kfuncs) +BTF_ID_FLAGS(func, bpf_proactive_reclaim, KF_SLEEPABLE) +BTF_KFUNCS_END(bpf_memcontrol_reclaim_kfuncs) + static const struct btf_kfunc_id_set bpf_memcontrol_kfunc_set = { .owner = THIS_MODULE, .set = &bpf_memcontrol_kfuncs, }; +static const struct btf_kfunc_id_set bpf_memcontrol_reclaim_kfunc_set = { + .owner = THIS_MODULE, + .set = &bpf_memcontrol_reclaim_kfuncs, +}; + static int __init bpf_memcontrol_init(void) { int err; err = register_btf_kfunc_id_set(BPF_PROG_TYPE_UNSPEC, &bpf_memcontrol_kfunc_set); - if (err) + if (err) { pr_warn("error while registering bpf memcontrol kfuncs: %d", err); + return err; + } + + err = register_btf_kfunc_id_set(BPF_PROG_TYPE_SYSCALL, + &bpf_memcontrol_reclaim_kfunc_set); + if (err) + pr_warn("error registering bpf reclaim kfuncs: %d\n", err); return err; } diff --git a/mm/internal.h b/mm/internal.h index 05179c4b2090..8c6c9db56aa8 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -24,6 +24,9 @@ struct folio_batch; +int __copy_remote_mm_str(struct mm_struct *mm, unsigned long addr, + void *buf, int len, unsigned int gup_flags); + /* mm/workingset.c */ bool workingset_test_recent(void *shadow, bool file, bool *workingset, bool flush); @@ -67,11 +70,13 @@ unsigned long lruvec_lru_size(struct lruvec *lruvec, enum lru_list lru, #define MEMCG_RECLAIM_MAY_SWAP (1 << 1) #define MEMCG_RECLAIM_PROACTIVE (1 << 2) -#define MIN_SWAPPINESS 0 -#define MAX_SWAPPINESS 200 +enum { + MIN_SWAPPINESS = 0, + MAX_SWAPPINESS = 200, -/* Just reclaim from anon folios in proactive memory reclaim */ -#define SWAPPINESS_ANON_ONLY (MAX_SWAPPINESS + 1) + /* Just reclaim from anon folios in proactive memory reclaim */ + SWAPPINESS_ANON_ONLY = MAX_SWAPPINESS + 1, +}; unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg, unsigned long nr_pages, diff --git a/mm/memory.c b/mm/memory.c index 7d821b110df0..797d39572a09 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -7302,13 +7302,12 @@ int access_process_vm(struct task_struct *tsk, unsigned long addr, } EXPORT_SYMBOL_GPL(access_process_vm); -#ifdef CONFIG_BPF_SYSCALL /* * Copy a string from another process's address space as given in mm. * If there is any error return -EFAULT. */ -static int __copy_remote_vm_str(struct mm_struct *mm, unsigned long addr, - void *buf, int len, unsigned int gup_flags) +int __copy_remote_mm_str(struct mm_struct *mm, unsigned long addr, + void *buf, int len, unsigned int gup_flags) { void *old_buf = buf; int err = 0; @@ -7384,44 +7383,6 @@ static int __copy_remote_vm_str(struct mm_struct *mm, unsigned long addr, return buf - old_buf; } -/** - * copy_remote_vm_str - copy a string from another process's address space. - * @tsk: the task of the target address space - * @addr: start address to read from - * @buf: destination buffer - * @len: number of bytes to copy - * @gup_flags: flags modifying lookup behaviour - * - * The caller must hold a reference on @mm. - * - * Return: number of bytes copied from @addr (source) to @buf (destination); - * not including the trailing NUL. Always guaranteed to leave NUL-terminated - * buffer. On any error, return -EFAULT. - */ -int copy_remote_vm_str(struct task_struct *tsk, unsigned long addr, - void *buf, int len, unsigned int gup_flags) -{ - struct mm_struct *mm; - int ret; - - if (unlikely(len == 0)) - return 0; - - mm = get_task_mm(tsk); - if (!mm) { - *(char *)buf = '\0'; - return -EFAULT; - } - - ret = __copy_remote_vm_str(mm, addr, buf, len, gup_flags); - - mmput(mm); - - return ret; -} -EXPORT_SYMBOL_GPL(copy_remote_vm_str); -#endif /* CONFIG_BPF_SYSCALL */ - /* * Print the name of a VMA. */ diff --git a/mm/nommu.c b/mm/nommu.c index ed44510e3770..91ab65afda70 100644 --- a/mm/nommu.c +++ b/mm/nommu.c @@ -1741,13 +1741,12 @@ int access_process_vm(struct task_struct *tsk, unsigned long addr, void *buf, in } EXPORT_SYMBOL_GPL(access_process_vm); -#ifdef CONFIG_BPF_SYSCALL /* * Copy a string from another process's address space as given in mm. * If there is any error return -EFAULT. */ -static int __copy_remote_vm_str(struct mm_struct *mm, unsigned long addr, - void *buf, int len) +int __copy_remote_mm_str(struct mm_struct *mm, unsigned long addr, + void *buf, int len, unsigned int gup_flags) { unsigned long addr_end; struct vm_area_struct *vma; @@ -1782,44 +1781,6 @@ static int __copy_remote_vm_str(struct mm_struct *mm, unsigned long addr, return ret; } -/** - * copy_remote_vm_str - copy a string from another process's address space. - * @tsk: the task of the target address space - * @addr: start address to read from - * @buf: destination buffer - * @len: number of bytes to copy - * @gup_flags: flags modifying lookup behaviour (unused) - * - * The caller must hold a reference on @mm. - * - * Return: number of bytes copied from @addr (source) to @buf (destination); - * not including the trailing NUL. Always guaranteed to leave NUL-terminated - * buffer. On any error, return -EFAULT. - */ -int copy_remote_vm_str(struct task_struct *tsk, unsigned long addr, - void *buf, int len, unsigned int gup_flags) -{ - struct mm_struct *mm; - int ret; - - if (unlikely(len == 0)) - return 0; - - mm = get_task_mm(tsk); - if (!mm) { - *(char *)buf = '\0'; - return -EFAULT; - } - - ret = __copy_remote_vm_str(mm, addr, buf, len); - - mmput(mm); - - return ret; -} -EXPORT_SYMBOL_GPL(copy_remote_vm_str); -#endif /* CONFIG_BPF_SYSCALL */ - /** * nommu_shrink_inode_mappings - Shrink the shared mappings on an inode * @inode: The inode to check diff --git a/mm/util.c b/mm/util.c index bf0513d1d3d0..75c0fc2061aa 100644 --- a/mm/util.c +++ b/mm/util.c @@ -1061,6 +1061,66 @@ int get_cmdline(struct task_struct *task, char *buffer, int buflen) return res; } +/** + * copy_remote_mm_str - copy a string from a remote address space. + * @mm: the remote address space + * @addr: start address to read from + * @buf: destination buffer + * @len: number of bytes to copy + * @gup_flags: flags modifying lookup behaviour + * + * The caller must hold a reference on @mm. + * + * Return: number of bytes copied from @addr (source) to @buf (destination), + * not including the trailing NUL. If @len is zero, return 0 without accessing + * @buf. Otherwise, @buf is always NUL-terminated. On any error, return + * -EFAULT. + */ +int copy_remote_mm_str(struct mm_struct *mm, unsigned long addr, + void *buf, int len, unsigned int gup_flags) +{ + if (unlikely(len == 0)) + return 0; + + return __copy_remote_mm_str(mm, addr, buf, len, gup_flags); +} + +/** + * copy_remote_vm_str - copy a string from another process's address space. + * @tsk: the task of the target address space + * @addr: start address to read from + * @buf: destination buffer + * @len: number of bytes to copy + * @gup_flags: flags modifying lookup behaviour + * + * Return: number of bytes copied from @addr (source) to @buf (destination), + * not including the trailing NUL. If @len is zero, return 0 without accessing + * @buf. Otherwise, @buf is always NUL-terminated. On any error, return + * -EFAULT. + */ +int copy_remote_vm_str(struct task_struct *tsk, unsigned long addr, + void *buf, int len, unsigned int gup_flags) +{ + struct mm_struct *mm; + int ret; + + if (unlikely(len == 0)) + return 0; + + mm = get_task_mm(tsk); + if (!mm) { + *(char *)buf = '\0'; + return -EFAULT; + } + + ret = __copy_remote_mm_str(mm, addr, buf, len, gup_flags); + + mmput(mm); + + return ret; +} +EXPORT_SYMBOL_GPL(copy_remote_vm_str); + int __weak memcmp_pages(struct page *page1, struct page *page2) { char *addr1, *addr2; diff --git a/net/bpf/test_run.c b/net/bpf/test_run.c index 513354e928cb..3043a7393c29 100644 --- a/net/bpf/test_run.c +++ b/net/bpf/test_run.c @@ -1237,7 +1237,13 @@ int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr, memset(__skb_push(skb, hh_len), 0, hh_len); } - if (kattr->test.flags & BPF_F_TEST_SKB_CHECKSUM_COMPLETE) { + /* + * A helper such as bpf_skb_change_tail() may have downgraded the skb + * from CHECKSUM_COMPLETE. skb->csum is then unused by the stack and + * there is nothing to validate. + */ + if ((kattr->test.flags & BPF_F_TEST_SKB_CHECKSUM_COMPLETE) && + skb->ip_summed == CHECKSUM_COMPLETE) { const int off = skb_network_offset(skb); int len = skb->len - off; __wsum csum; @@ -1650,21 +1656,20 @@ int bpf_prog_test_run_sk_lookup(struct bpf_prog *prog, const union bpf_attr *kat ctx.selected_sk = NULL; retval = BPF_PROG_SK_LOOKUP_RUN_ARRAY(progs, ctx, bpf_prog_run); } while (bpf_test_timer_continue(&t, 1, repeat, &ret, &duration)); + + /* Nothing but RCU keeps the selected socket */ + user_ctx->cookie = 0; + if (!ret && ctx.selected_sk) { + if (ctx.selected_sk->sk_reuseport && !ctx.no_reuseport) + ret = -EOPNOTSUPP; + else + user_ctx->cookie = sock_gen_cookie(ctx.selected_sk); + } bpf_test_timer_leave(&t); if (ret < 0) goto out; - user_ctx->cookie = 0; - if (ctx.selected_sk) { - if (ctx.selected_sk->sk_reuseport && !ctx.no_reuseport) { - ret = -EOPNOTSUPP; - goto out; - } - - user_ctx->cookie = sock_gen_cookie(ctx.selected_sk); - } - ret = bpf_test_finish(kattr, uattr, NULL, NULL, 0, 0, retval, duration); if (!ret) ret = bpf_ctx_finish(kattr, uattr, user_ctx, sizeof(*user_ctx)); diff --git a/net/core/bpf_ksock.c b/net/core/bpf_ksock.c index e9943aeeadd3..37168f43c16c 100644 --- a/net/core/bpf_ksock.c +++ b/net/core/bpf_ksock.c @@ -37,13 +37,47 @@ static void ksock_release_work_fn(struct work_struct *work) kfree(ks); } -static bool bpf_ksock_has_user_task_context(void) +static int bpf_ksock_addr_to_sockaddr(const union bpf_ksock_addr *addr, + u32 addr_sz, struct sockaddr_storage *sa) { - /* - * Task work can run from do_exit() after exit_nsproxy_namespaces() - * cleared current->nsproxy, while current is still not a kthread. - */ - return !(current->flags & PF_KTHREAD) && current->nsproxy; + if (!addr || addr_sz != sizeof(*addr)) + return -EINVAL; + + /* Kfunc memory arguments may be unaligned. */ + memcpy(sa, addr, sizeof(*addr)); + + switch (sa->ss_family) { + case AF_INET: + return sizeof(struct sockaddr_in); + case AF_INET6: + return sizeof(struct sockaddr_in6); + default: + return -EAFNOSUPPORT; + } +} + +static int ksock_sendto(struct bpf_ksock *ks, const void *data, u32 data_sz, + const union bpf_ksock_addr *addr, u32 addr_sz) +{ + struct sockaddr_storage sa; + struct msghdr msg = { + .msg_flags = MSG_DONTWAIT, + }; + struct kvec iov = { + .iov_base = (void *)data, + .iov_len = data_sz, + }; + int addrlen; + + if (addr) { + addrlen = bpf_ksock_addr_to_sockaddr(addr, addr_sz, &sa); + if (addrlen < 0) + return addrlen; + msg.msg_name = &sa; + msg.msg_namelen = addrlen; + } + + return kernel_sendmsg(ks->sock, &msg, &iov, 1, data_sz); } __bpf_kfunc_start_defs(); @@ -59,7 +93,7 @@ __bpf_kfunc_start_defs(); * This function may sleep (sock_create), so it can only be used * in sleepable BPF programs (SYSCALL). * It cannot be called from a BPF workqueue callback because that callback - * does not retain the invoking task's namespace or security context. + * does not retain the invoking task's namespace. * * @opts: Pointer to struct bpf_ksock_create_opts with socket parameters. * @opts__sz: Size of the opts struct. @@ -74,11 +108,10 @@ bpf_ksock_create(const struct bpf_ksock_create_opts *opts, u32 opts__sz, int err; /* - * sock_create() derives the network namespace, credentials, and cgroup - * from current. Kernel threads, including BPF workqueue callbacks, do - * not carry the context of the task that invoked the BPF program. + * sock_create() derives the network namespace from current and task + * work can run from do_exit() after exit_nsproxy_namespaces() */ - if (!bpf_ksock_has_user_task_context()) { + if ((current->flags & PF_KTHREAD) || !current->nsproxy) { err = -EOPNOTSUPP; goto err_out; } @@ -165,25 +198,9 @@ __bpf_kfunc int bpf_ksock_connect(struct bpf_ksock *ks, struct sockaddr_storage sa; int addrlen; - if (!bpf_ksock_has_user_task_context()) - return -EOPNOTSUPP; - - if (!addr || addr__sz != sizeof(*addr)) - return -EINVAL; - - /* Kfunc memory arguments may be unaligned. */ - memcpy(&sa, addr, sizeof(*addr)); - - switch (sa.ss_family) { - case AF_INET: - addrlen = sizeof(struct sockaddr_in); - break; - case AF_INET6: - addrlen = sizeof(struct sockaddr_in6); - break; - default: - return -EAFNOSUPPORT; - } + addrlen = bpf_ksock_addr_to_sockaddr(addr, addr__sz, &sa); + if (addrlen < 0) + return addrlen; return connect_socket(ks->sock, &sa, addrlen, 0); } @@ -238,21 +255,29 @@ CFI_NOSEAL(bpf_ksock_release_dtor); __bpf_kfunc int bpf_ksock_send(struct bpf_ksock *ks, const void *data, u32 data__sz) { - struct msghdr msg = { - .msg_flags = MSG_DONTWAIT, - }; - struct kvec iov = { - .iov_base = (void *)data, - .iov_len = data__sz, - }; - int ret; + return ksock_sendto(ks, data, data__sz, NULL, 0); +} - if (!bpf_ksock_has_user_task_context()) - return -EOPNOTSUPP; - - ret = kernel_sendmsg(ks->sock, &msg, &iov, 1, data__sz); - - return ret; +/** + * bpf_ksock_sendto() - Send data through a BPF kernel socket to a remote address. + * @ks: The BPF kernel socket context. Must be an acquired reference. + * @data: Pointer to the data to send. + * @data__sz: Size of the data to send. + * @addr__nullable: Pointer to an IPv4 or IPv6 socket address, or NULL to use + * the connected destination. + * @addr__sz: Size of the address union, ignored when the address is NULL. + * + * Sends data to the specified address, or to the connected destination when + * the address is NULL, best-effort and nonblocking. This may sleep + * (kernel_sendmsg), so it can only be called from sleepable BPF programs. + * + * Return: Number of bytes sent on success, negative errno on error. + */ +__bpf_kfunc int bpf_ksock_sendto(struct bpf_ksock *ks, const void *data, u32 data__sz, + const union bpf_ksock_addr *addr__nullable, + u32 addr__sz) +{ + return ksock_sendto(ks, data, data__sz, addr__nullable, addr__sz); } __bpf_kfunc_end_defs(); @@ -271,10 +296,16 @@ BTF_KFUNCS_START(ksock_kfunc_btf_ids) BTF_ID_FLAGS(func, bpf_ksock_release, KF_RELEASE) BTF_ID_FLAGS(func, bpf_ksock_acquire, KF_ACQUIRE | KF_RCU | KF_RET_NULL) BTF_ID_FLAGS(func, bpf_ksock_send, KF_SLEEPABLE) +BTF_ID_FLAGS(func, bpf_ksock_sendto, KF_SLEEPABLE) BTF_KFUNCS_END(ksock_kfunc_btf_ids) #ifdef CONFIG_BPF_LSM -BTF_ID_LIST_SINGLE(bpf_lsm_socket_sendmsg_id, func, bpf_lsm_socket_sendmsg) +BTF_ID_LIST(bpf_lsm_socket_sendmsg_id) +#ifdef CONFIG_SECURITY_NETWORK +BTF_ID(func, bpf_lsm_socket_sendmsg) +#else +BTF_ID_UNUSED +#endif #endif static int bpf_ksock_kfunc_filter(const struct bpf_prog *prog, u32 kfunc_id) diff --git a/net/core/bpf_sk_storage.c b/net/core/bpf_sk_storage.c index 1d295a8769fa..1c59300af05c 100644 --- a/net/core/bpf_sk_storage.c +++ b/net/core/bpf_sk_storage.c @@ -164,6 +164,13 @@ int bpf_sk_storage_clone(const struct sock *sk, struct sock *newsk) if (!sk_storage || hlist_empty(&sk_storage->list)) goto out; + /* + * The maps lead to newsk as soon as an element is linked, but + * sk_clone() sets sk_refcnt only later. Until then it holds whatever + * the slab object held before: no reference can be had yet. + */ + refcount_set(&newsk->sk_refcnt, 0); + hlist_for_each_entry_rcu(selem, &sk_storage->list, snode) { struct bpf_local_storage_elem *copy_selem; struct bpf_local_storage_map *smap; @@ -692,8 +699,37 @@ struct bpf_iter_seq_sk_storage_map_info { struct bpf_map *map; unsigned int bucket_id; unsigned skip_elems; + struct rcu_head *dead_sks; }; +/* + * sock_put() in two halves. The end of a socket may want process context: + * rfcomm_sock_destruct() takes a mutex. If the iterator drops the last + * reference, it keeps the socket until it has left the RCU read side critical + * section. Nobody uses sk_rcu before sk_destruct(). + */ +static void +bpf_sk_storage_map_seq_put_sk(struct bpf_iter_seq_sk_storage_map_info *info, + struct sock *sk) +{ + if (refcount_dec_and_test(&sk->sk_refcnt)) { + sk->sk_rcu.next = info->dead_sks; + info->dead_sks = &sk->sk_rcu; + } +} + +static void +bpf_sk_storage_map_seq_free_sks(struct bpf_iter_seq_sk_storage_map_info *info) +{ + struct sock *sk; + + while (info->dead_sks) { + sk = container_of(info->dead_sks, struct sock, sk_rcu); + info->dead_sks = sk->sk_rcu.next; + sk_free(sk); + } +} + static struct bpf_local_storage_elem * bpf_sk_storage_map_seq_find_next(struct bpf_iter_seq_sk_storage_map_info *info, struct bpf_local_storage_elem *prev_selem) @@ -722,6 +758,7 @@ bpf_sk_storage_map_seq_find_next(struct bpf_iter_seq_sk_storage_map_info *info, /* not found, unlock and go to the next bucket */ b = &smap->buckets[bucket_id++]; rcu_read_unlock(); + bpf_sk_storage_map_seq_free_sks(info); skip_elems = 0; break; } @@ -789,12 +826,44 @@ DEFINE_BPF_ITER_FUNC(bpf_sk_storage_map, struct bpf_iter_meta *meta, struct bpf_map *map, struct sock *sk, void *value) +/* + * Take a reference on the socket which owns @selem. RCU keeps the element, not + * its owner. As long as the element is linked under the lock of the storage, + * the owner has not got past bpf_sk_storage_free(), unless that has given up + * on the lock. Then it still waits for owner_refcnt. + */ +static struct sock * +bpf_sk_storage_selem_get_sk(struct bpf_local_storage_elem *selem) +{ + struct bpf_local_storage *sk_storage; + struct sock *sk = NULL; + unsigned long flags; + + sk_storage = rcu_dereference(selem->local_storage); + /* reset since ->next() looked: the socket goes away */ + if (!sk_storage) + return NULL; + + if (raw_res_spin_lock_irqsave(&sk_storage->lock, flags)) + return NULL; + + if (!hlist_unhashed(&selem->snode) && + refcount_inc_not_zero(&sk_storage->owner_refcnt)) { + sk = sk_storage->owner; + if (!refcount_inc_not_zero(&sk->sk_refcnt)) + sk = NULL; + bpf_local_storage_unpin_owner(sk_storage); + } + raw_res_spin_unlock_irqrestore(&sk_storage->lock, flags); + + return sk; +} + static int __bpf_sk_storage_map_seq_show(struct seq_file *seq, struct bpf_local_storage_elem *selem) { struct bpf_iter_seq_sk_storage_map_info *info = seq->private; struct bpf_iter__bpf_sk_storage_map ctx = {}; - struct bpf_local_storage *sk_storage; struct bpf_iter_meta meta; struct bpf_prog *prog; int ret = 0; @@ -805,11 +874,14 @@ static int __bpf_sk_storage_map_seq_show(struct seq_file *seq, ctx.meta = &meta; ctx.map = info->map; if (selem) { - sk_storage = rcu_dereference(selem->local_storage); - ctx.sk = sk_storage->owner; + ctx.sk = bpf_sk_storage_selem_get_sk(selem); + if (!ctx.sk) + return SEQ_SKIP; ctx.value = SDATA(selem)->data; } ret = bpf_iter_run_prog(prog, &ctx); + if (ctx.sk) + bpf_sk_storage_map_seq_put_sk(info, ctx.sk); } return ret; @@ -827,6 +899,7 @@ static void bpf_sk_storage_map_seq_stop(struct seq_file *seq, void *v) (void)__bpf_sk_storage_map_seq_show(seq, v); else rcu_read_unlock(); + bpf_sk_storage_map_seq_free_sks(seq->private); } static int bpf_iter_init_sk_storage_map(void *priv_data, @@ -846,6 +919,15 @@ static void bpf_iter_fini_sk_storage_map(void *priv_data) bpf_map_put_with_uref(seq_info->map); } +static int bpf_iter_map_check_prog(const struct bpf_prog *prog, + const struct bpf_iter_aux_info *aux) +{ + if (prog->aux->max_rdwr_access > aux->map->value_size) + return -EACCES; + + return bpf_iter_map_check_value_write(prog, aux->map); +} + static int bpf_iter_attach_map(struct bpf_prog *prog, union bpf_iter_link_info *linfo, struct bpf_iter_aux_info *aux) @@ -863,12 +945,11 @@ static int bpf_iter_attach_map(struct bpf_prog *prog, if (map->map_type != BPF_MAP_TYPE_SK_STORAGE) goto put_map; - if (prog->aux->max_rdwr_access > map->value_size) { - err = -EACCES; - goto put_map; - } - aux->map = map; + err = bpf_iter_map_check_prog(prog, aux); + if (err) + goto put_map; + return 0; put_map: @@ -899,6 +980,7 @@ static struct bpf_iter_reg bpf_sk_storage_map_reg_info = { .target = "bpf_sk_storage_map", .attach_target = bpf_iter_attach_map, .detach_target = bpf_iter_detach_map, + .check_prog = bpf_iter_map_check_prog, .show_fdinfo = bpf_iter_map_show_fdinfo, .fill_link_info = bpf_iter_map_fill_link_info, .ctx_arg_info_size = 2, diff --git a/net/core/filter.c b/net/core/filter.c index 70dc621672f2..b7ac145b68b1 100644 --- a/net/core/filter.c +++ b/net/core/filter.c @@ -1715,6 +1715,24 @@ static inline void bpf_pull_mac_rcsum(struct sk_buff *skb) skb_postpull_rcsum(skb, skb_mac_header(skb), skb->mac_len); } +/* + * The source of a write through a dynptr may be a slice of the same skb. + * Making the skb writable can move its head: follow the move. + */ +static int bpf_try_make_writable_from(struct sk_buff *skb, u32 write_len, + const void **from) +{ + const void *head = skb->head; + bool in_head = *from >= head && + *from < (const void *)skb_end_pointer(skb); + int err; + + err = bpf_try_make_writable(skb, write_len); + if (!err && in_head) + *from = skb->head + (*from - head); + return err; +} + BPF_CALL_5(bpf_skb_store_bytes, struct sk_buff *, skb, u32, offset, const void *, from, u32, len, u64, flags) { @@ -1724,14 +1742,15 @@ BPF_CALL_5(bpf_skb_store_bytes, struct sk_buff *, skb, u32, offset, return -EINVAL; if (unlikely(offset > INT_MAX)) return -EFAULT; - if (unlikely(bpf_try_make_writable(skb, offset + len))) + if (unlikely(bpf_try_make_writable_from(skb, offset + len, &from))) return -EFAULT; ptr = skb->data + offset; if (flags & BPF_F_RECOMPUTE_CSUM) __skb_postpull_rcsum(skb, ptr, len, offset); - memcpy(ptr, from, len); + /* through a dynptr the source may be an overlapping slice of the skb */ + memmove(ptr, from, len); if (flags & BPF_F_RECOMPUTE_CSUM) __skb_postpush_rcsum(skb, ptr, len, offset); @@ -3612,6 +3631,14 @@ static int bpf_skb_net_grow(struct sk_buff *skb, u32 off, u32 len_diff, return -ENOTSUPP; } + /* + * The MSS is lowered by len_diff. Leave the least that the stack lets a + * TCP GSO packet have, or gso_segs can overflow. + */ + if (skb_is_gso(skb) && !(flags & BPF_F_ADJ_ROOM_FIXED_GSO) && + skb_shinfo(skb)->gso_size < len_diff + TCP_MIN_GSO_SIZE) + return -ENOTSUPP; + ret = skb_cow_head(skb, meta_len + len_diff); if (unlikely(ret < 0)) return ret; @@ -4303,7 +4330,8 @@ BPF_CALL_4(bpf_xdp_store_bytes, struct xdp_buff *, xdp, u32, offset, if (!ptr) bpf_xdp_copy_buf(xdp, offset, buf, len, true); else - memcpy(ptr, buf, len); + /* through a dynptr the source may be an overlapping slice */ + memmove(ptr, buf, len); return 0; } @@ -7017,15 +7045,15 @@ static void bpf_update_srh_state(struct sk_buff *skb) { struct seg6_bpf_srh_state *srh_state = this_cpu_ptr(&seg6_bpf_srh_states); - int srhoff = 0; - if (ipv6_find_hdr(skb, &srhoff, IPPROTO_ROUTING, NULL, NULL) < 0) { - srh_state->srh = NULL; - } else { - srh_state->srh = (struct ipv6_sr_hdr *)(skb->data + srhoff); + /* Behind a decapsulation this is a header that nobody has looked at */ + srh_state->srh = seg6_get_srh(skb, 0); + if (srh_state->srh) { srh_state->hdrlen = srh_state->srh->hdrlen << 3; srh_state->valid = true; } + /* it may have been pulled into the head */ + bpf_compute_data_pointers(skb); } BPF_CALL_4(bpf_lwt_seg6_action, struct sk_buff *, skb, @@ -7058,9 +7086,14 @@ BPF_CALL_4(bpf_lwt_seg6_action, struct sk_buff *, skb, if (ipv6_find_hdr(skb, &hdroff, IPPROTO_IPV6, NULL, NULL) < 0) return -EBADMSG; - if (!pskb_pull(skb, hdroff)) + /* As long as nothing has been changed, the program goes on */ + if (!pskb_may_pull(skb, hdroff + sizeof(struct ipv6hdr))) { + /* but the head may have moved */ + bpf_update_srh_state(skb); return -EBADMSG; + } + __skb_pull(skb, hdroff); skb_postpull_rcsum(skb, skb_network_header(skb), hdroff); skb_reset_network_header(skb); skb_reset_transport_header(skb); @@ -7074,8 +7107,8 @@ BPF_CALL_4(bpf_lwt_seg6_action, struct sk_buff *, skb, return -EBADMSG; err = bpf_push_seg6_encap(skb, BPF_LWT_ENCAP_SEG6_INLINE, param, param_len); - if (!err) - bpf_update_srh_state(skb); + /* The header can be in place although err says otherwise */ + bpf_update_srh_state(skb); return err; case SEG6_LOCAL_ACTION_END_B6_ENCAP: @@ -7083,8 +7116,8 @@ BPF_CALL_4(bpf_lwt_seg6_action, struct sk_buff *, skb, return -EBADMSG; err = bpf_push_seg6_encap(skb, BPF_LWT_ENCAP_SEG6, param, param_len); - if (!err) - bpf_update_srh_state(skb); + /* The header can be in place although err says otherwise */ + bpf_update_srh_state(skb); return err; default: @@ -7600,8 +7633,8 @@ bool bpf_tcp_sock_is_valid_access(int off, int size, enum bpf_access_type type, return false; switch (off) { - case offsetof(struct bpf_tcp_sock, bytes_received): - case offsetof(struct bpf_tcp_sock, bytes_acked): + case bpf_ctx_range(struct bpf_tcp_sock, bytes_received): + case bpf_ctx_range(struct bpf_tcp_sock, bytes_acked): return size == sizeof(__u64); default: return size == sizeof(__u32); @@ -8052,17 +8085,14 @@ static const u8 *bpf_search_tcp_opt(const u8 *op, const u8 *opend, return ERR_PTR(-ENOMSG); } -BPF_CALL_4(bpf_sock_ops_load_hdr_opt, struct bpf_sock_ops_kern *, bpf_sock, - void *, search_res, u32, len, u64, flags) +int __bpf_sock_ops_load_hdr_opt(struct bpf_sock_ops_kern *bpf_sock, + void *search_res, u32 len, u64 flags) { bool eol, load_syn = flags & BPF_LOAD_HDR_OPT_TCP_SYN; const u8 *op, *opend, *magic, *search = search_res; u8 search_kind, search_len, copy_len, magic_len; int ret; - if (!is_locked_tcp_sock_ops(bpf_sock)) - return -EOPNOTSUPP; - /* 2 byte is the minimal option len except TCPOPT_NOP and * TCPOPT_EOL which are useless for the bpf prog to learn * and this helper disallow loading them also. @@ -8123,6 +8153,15 @@ BPF_CALL_4(bpf_sock_ops_load_hdr_opt, struct bpf_sock_ops_kern *, bpf_sock, return ret; } +BPF_CALL_4(bpf_sock_ops_load_hdr_opt, struct bpf_sock_ops_kern *, bpf_sock, + void *, search_res, u32, len, u64, flags) +{ + if (!is_locked_tcp_sock_ops(bpf_sock)) + return -EOPNOTSUPP; + + return __bpf_sock_ops_load_hdr_opt(bpf_sock, search_res, len, flags); +} + static const struct bpf_func_proto bpf_sock_ops_load_hdr_opt_proto = { .func = bpf_sock_ops_load_hdr_opt, .gpl_only = false, @@ -8133,17 +8172,14 @@ static const struct bpf_func_proto bpf_sock_ops_load_hdr_opt_proto = { .arg4_type = ARG_ANYTHING, }; -BPF_CALL_4(bpf_sock_ops_store_hdr_opt, struct bpf_sock_ops_kern *, bpf_sock, - const void *, from, u32, len, u64, flags) +int __bpf_sock_ops_store_hdr_opt(struct bpf_sock_ops_kern *bpf_sock, + const void *from, u32 len, u64 flags) { u8 new_kind, new_kind_len, magic_len = 0, *opend; const u8 *op, *new_op, *magic = NULL; struct sk_buff *skb; bool eol; - if (bpf_sock->op != BPF_SOCK_OPS_WRITE_HDR_OPT_CB) - return -EPERM; - if (len < 2 || flags) return -EINVAL; @@ -8201,6 +8237,15 @@ BPF_CALL_4(bpf_sock_ops_store_hdr_opt, struct bpf_sock_ops_kern *, bpf_sock, return 0; } +BPF_CALL_4(bpf_sock_ops_store_hdr_opt, struct bpf_sock_ops_kern *, bpf_sock, + const void *, from, u32, len, u64, flags) +{ + if (bpf_sock->op != BPF_SOCK_OPS_WRITE_HDR_OPT_CB) + return -EPERM; + + return __bpf_sock_ops_store_hdr_opt(bpf_sock, from, len, flags); +} + static const struct bpf_func_proto bpf_sock_ops_store_hdr_opt_proto = { .func = bpf_sock_ops_store_hdr_opt, .gpl_only = false, @@ -12402,7 +12447,7 @@ int __bpf_skb_meta_store_bytes(struct sk_buff *skb, u32 offset, { if (unlikely(flags)) return -EINVAL; - if (unlikely(bpf_try_make_writable(skb, 0))) + if (unlikely(bpf_try_make_writable_from(skb, 0, &from))) return -EFAULT; memmove(bpf_skb_meta_pointer(skb, offset), from, len); @@ -12878,6 +12923,7 @@ static int __init bpf_kfunc_init(void) ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_LWT_SEG6LOCAL, &bpf_kfunc_set_skb); ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_NETFILTER, &bpf_kfunc_set_skb); ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_TRACING, &bpf_kfunc_set_skb); + ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, &bpf_kfunc_set_skb); ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_SCHED_CLS, &bpf_kfunc_set_skb_meta); ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_SCHED_ACT, &bpf_kfunc_set_skb_meta); ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_XDP, &bpf_kfunc_set_xdp); diff --git a/net/core/ptp_classifier.c b/net/core/ptp_classifier.c index 598041b0499e..939104e25c75 100644 --- a/net/core/ptp_classifier.c +++ b/net/core/ptp_classifier.c @@ -2,12 +2,8 @@ /* PTP classifier */ -/* The below program is the bpf_asm (tools/net/) representation of - * the opcode array in the ptp_filter structure. - * - * For convenience, this can easily be altered and reviewed with - * bpf_asm and bpf_dbg, e.g. `./bpf_asm -c prog` where prog is a - * simple file containing the below program: +/* The below program is the classic BPF assembly representation of + * the opcode array in the ptp_filter structure: * * ldh [12] ; load ethertype * diff --git a/net/core/sock_map.c b/net/core/sock_map.c index 38df84284328..057e2f3c0265 100644 --- a/net/core/sock_map.c +++ b/net/core/sock_map.c @@ -1933,6 +1933,15 @@ int sock_map_link_create(const union bpf_attr *attr, struct bpf_prog *prog) return ret; } +static int sock_map_iter_check_prog(const struct bpf_prog *prog, + const struct bpf_iter_aux_info *aux) +{ + if (prog->aux->max_rdonly_access > aux->map->key_size) + return -EACCES; + + return 0; +} + static int sock_map_iter_attach_target(struct bpf_prog *prog, union bpf_iter_link_info *linfo, struct bpf_iter_aux_info *aux) @@ -1951,12 +1960,11 @@ static int sock_map_iter_attach_target(struct bpf_prog *prog, map->map_type != BPF_MAP_TYPE_SOCKHASH) goto put_map; - if (prog->aux->max_rdonly_access > map->key_size) { - err = -EACCES; - goto put_map; - } - aux->map = map; + err = sock_map_iter_check_prog(prog, aux); + if (err) + goto put_map; + return 0; put_map: @@ -1973,6 +1981,7 @@ static struct bpf_iter_reg sock_map_iter_reg = { .target = "sockmap", .attach_target = sock_map_iter_attach_target, .detach_target = sock_map_iter_detach_target, + .check_prog = sock_map_iter_check_prog, .show_fdinfo = bpf_iter_map_show_fdinfo, .fill_link_info = bpf_iter_map_fill_link_info, .ctx_arg_info_size = 2, diff --git a/net/ipv4/Makefile b/net/ipv4/Makefile index 69e4baaed122..e8a0e9605b90 100644 --- a/net/ipv4/Makefile +++ b/net/ipv4/Makefile @@ -71,6 +71,7 @@ obj-$(CONFIG_TCP_AO) += tcp_ao.o ifeq ($(CONFIG_BPF_JIT),y) obj-$(CONFIG_BPF_SYSCALL) += bpf_tcp_ca.o +obj-$(CONFIG_CGROUP_BPF) += bpf_tcp_ops.o endif ifdef CONFIG_GCOV_PROFILE_NETFILTER diff --git a/net/ipv4/af_inet.c b/net/ipv4/af_inet.c index c012e2afc6dc..7677b0343abd 100644 --- a/net/ipv4/af_inet.c +++ b/net/ipv4/af_inet.c @@ -226,7 +226,8 @@ int __inet_listen_sk(struct sock *sk, int backlog) if (err) return err; - tcp_call_bpf(sk, BPF_SOCK_OPS_TCP_LISTEN_CB, 0, NULL); + tcp_call_bpf(sk, BPF_SOCK_OPS_TCP_LISTEN_CB); + bpf_tcp_ops_call(listen, sk); } return 0; } diff --git a/net/ipv4/bpf_tcp_ca.c b/net/ipv4/bpf_tcp_ca.c index 9deed2244c2d..2a71dc81562f 100644 --- a/net/ipv4/bpf_tcp_ca.c +++ b/net/ipv4/bpf_tcp_ca.c @@ -340,6 +340,22 @@ static struct bpf_struct_ops bpf_tcp_congestion_ops = { .validate = bpf_tcp_ca_validate, .name = "tcp_congestion_ops", .cfi_stubs = &__bpf_ops_tcp_congestion_ops, + /* The struct_ops's function may switch to another struct_ops. + * + * For example, bpf_tcp_cc_x->init() may switch to + * another tcp_cc_y by calling + * setsockopt(TCP_CONGESTION, "tcp_cc_y"). + * During the switch, bpf_struct_ops_put(tcp_cc_x) is called + * and its refcount may reach 0 which then free its + * trampoline image while tcp_cc_x is still running. + * + * A vanilla rcu gp is to wait for all bpf-tcp-cc prog + * to finish. bpf-tcp-cc prog is non sleepable. + * A rcu_tasks gp is to wait for the last few insn + * in the tramopline image to finish before releasing + * the trampoline image. + */ + .free_after_tasks_rcu_gp = true, .owner = THIS_MODULE, }; diff --git a/net/ipv4/bpf_tcp_ops.c b/net/ipv4/bpf_tcp_ops.c new file mode 100644 index 000000000000..2ba73dd6c52c --- /dev/null +++ b/net/ipv4/bpf_tcp_ops.c @@ -0,0 +1,453 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include +#include +#include +#include +#include +#include + +static int timeout_init_stub(struct sock *sk, struct request_sock *req__nullable) +{ + struct bpf_tramp_run_ctx *ctx = + container_of(current->bpf_ctx, struct bpf_tramp_run_ctx, run_ctx); + + return ctx->retval; +} + +static int rwnd_init_stub(struct sock *sk, struct request_sock *req__nullable) +{ + struct bpf_tramp_run_ctx *ctx = + container_of(current->bpf_ctx, struct bpf_tramp_run_ctx, run_ctx); + + return ctx->retval; +} + +static void active_established_stub(struct sock *sk, struct sk_buff *skb__nullable) +{ +} + +static void passive_established_stub(struct sock *sk, struct sk_buff *skb) +{ +} + +static void rto_stub(struct sock *sk) +{ +} + +static void rtt_stub(struct sock *sk, long mrtt, u32 srtt) +{ +} + +static void set_state_stub(struct sock *sk, int state) +{ +} + +static void retrans_stub(struct sock *sk, struct sk_buff *skb, int err) +{ +} + +static void connect_stub(struct sock *sk) +{ +} + +static void listen_stub(struct sock *sk) +{ +} + +static void parse_hdr_stub(struct sock *sk, struct sk_buff *skb) +{ +} + +static void hdr_opt_len_stub(struct sock *sk, struct sk_buff *skb__nullable, + struct request_sock *req__nullable, + struct sk_buff *syn_skb__nullable, + enum tcp_synack_type synack_type, + unsigned int *remaining) +{ +} + +static void write_hdr_opt_stub(struct sock *sk, struct sk_buff *skb, + struct request_sock *req__nullable, + struct sk_buff *syn_skb__nullable, + enum tcp_synack_type synack_type, + u32 opt_off) +{ +} + +static void enqueue_rcvq_stub(struct sock *sk, struct sk_buff *skb) +{ +} + +static void dequeue_rcvq_stub(struct sock *sk) +{ +} + +static struct bpf_tcp_ops __bpf_tcp_ops = { + .timeout_init = timeout_init_stub, + .rwnd_init = rwnd_init_stub, + .active_established = active_established_stub, + .passive_established = passive_established_stub, + .rto = rto_stub, + .rtt = rtt_stub, + .set_state = set_state_stub, + .retrans = retrans_stub, + .connect = connect_stub, + .listen = listen_stub, + .parse_hdr = parse_hdr_stub, + .hdr_opt_len = hdr_opt_len_stub, + .write_hdr_opt = write_hdr_opt_stub, + .enqueue_rcvq = enqueue_rcvq_stub, + .dequeue_rcvq = dequeue_rcvq_stub, +}; + +BPF_CALL_4(bpf_tcp_ops_store_hdr_opt, void *, ctx, const void *, from, + u32, len, u64, flags) +{ + u64 *args = ctx; + struct sk_buff *skb = (void *)(unsigned long)args[1]; + struct bpf_sock_ops_kern sock_ops = {}; + u32 opt_off = args[5]; + u8 *op, *opend; + + /* + * bpf_tcp_ops does not keep track of the end of the written TCP header + * options, so search for it every time the helper is called. The free + * space is NOP-filled, so a TCPOPT_NOP ends the search rather than being + * skipped as in a normal option walk in sockops. + */ + op = skb->data + opt_off; + opend = skb->data + tcp_hdrlen(skb); + while (op < opend && *op != TCPOPT_NOP) { + if (*op == TCPOPT_EOL || op + 1 >= opend || op[1] < 2) + break; + op += op[1]; + } + + sock_ops.skb = skb; + sock_ops.skb_data_end = op; + sock_ops.remaining_opt_len = opend - op; + + return __bpf_sock_ops_store_hdr_opt(&sock_ops, from, len, flags); +} + +static const struct bpf_func_proto bpf_tcp_ops_store_hdr_opt_proto = { + .func = bpf_tcp_ops_store_hdr_opt, + .gpl_only = false, + .ret_type = RET_INTEGER, + .arg1_type = ARG_PTR_TO_CTX, + .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, + .arg3_type = ARG_MEM_SIZE, + .arg4_type = ARG_ANYTHING, +}; + +BPF_CALL_4(bpf_tcp_ops_load_hdr_opt, void *, ctx, void *, search_res, + u32, len, u64, flags) +{ + u64 *args = ctx; + struct sk_buff *skb = (void *)(unsigned long)args[1]; + struct bpf_sock_ops_kern sock_ops = {}; + + /* + * No flags supported. In particular BPF_LOAD_HDR_OPT_TCP_SYN, which + * loads from the saved SYN, is not available because bpf_tcp_ops has no + * carrier to track the SYN source across the hooks. + */ + if (flags) + return -EINVAL; + + sock_ops.skb = skb; + sock_ops.skb_data_end = skb->data + tcp_hdrlen(skb); + + return __bpf_sock_ops_load_hdr_opt(&sock_ops, search_res, len, flags); +} + +static const struct bpf_func_proto bpf_tcp_ops_load_hdr_opt_proto = { + .func = bpf_tcp_ops_load_hdr_opt, + .gpl_only = false, + .ret_type = RET_INTEGER, + .arg1_type = ARG_PTR_TO_CTX, + .arg2_type = ARG_PTR_TO_MEM | MEM_WRITE, + .arg3_type = ARG_MEM_SIZE, + .arg4_type = ARG_ANYTHING, +}; + +BPF_CALL_3(bpf_tcp_ops_reserve_hdr_opt, void *, ctx, u32, len, u64, flags) +{ + u64 *args = ctx; + unsigned int *remaining = (void *)(unsigned long)args[5]; + + if (flags || len < 2) + return -EINVAL; + + if (len > *remaining) + return -ENOSPC; + + *remaining -= len; + return 0; +} + +static const struct bpf_func_proto bpf_tcp_ops_reserve_hdr_opt_proto = { + .func = bpf_tcp_ops_reserve_hdr_opt, + .gpl_only = false, + .ret_type = RET_INTEGER, + .arg1_type = ARG_PTR_TO_CTX, + .arg2_type = ARG_ANYTHING, + .arg3_type = ARG_ANYTHING, +}; + +BPF_CALL_0(bpf_tcp_ops_get_retval) +{ + struct bpf_tramp_run_ctx *ctx = + container_of(current->bpf_ctx, struct bpf_tramp_run_ctx, run_ctx); + + /* bpf_get_retval() is only exposed to timeout_init/rwnd_init, which + * always run via bpf_tcp_ops_call_int(). Its run_ctx carries the int + * return value chained across the bpf_tcp_ops attached to the cgroup + * and is this program's saved_run_ctx. + */ + if (WARN_ON_ONCE(!ctx->saved_run_ctx)) + return 0; + + return container_of(ctx->saved_run_ctx, struct bpf_tramp_run_ctx, + run_ctx)->retval; +} + +const struct bpf_func_proto bpf_tcp_ops_get_retval_proto = { + .func = bpf_tcp_ops_get_retval, + .gpl_only = false, + .ret_type = RET_INTEGER, +}; + +static bool is_sockopt_supported(u32 moff) +{ + switch (moff) { + case offsetof(struct bpf_tcp_ops, active_established): + case offsetof(struct bpf_tcp_ops, passive_established): + case offsetof(struct bpf_tcp_ops, rto): + case offsetof(struct bpf_tcp_ops, rtt): + case offsetof(struct bpf_tcp_ops, set_state): + case offsetof(struct bpf_tcp_ops, retrans): + case offsetof(struct bpf_tcp_ops, connect): + case offsetof(struct bpf_tcp_ops, listen): + case offsetof(struct bpf_tcp_ops, parse_hdr): + return true; + } + + return false; +} + +static const struct bpf_func_proto * +get_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) +{ + u32 moff = prog->aux->attach_st_ops_member_off; + + switch (func_id) { + case BPF_FUNC_sk_storage_get: + return &bpf_sk_storage_get_proto; + case BPF_FUNC_sk_storage_delete: + return &bpf_sk_storage_delete_proto; + case BPF_FUNC_setsockopt: + if (is_sockopt_supported(moff)) + return &bpf_sk_setsockopt_proto; + return NULL; + case BPF_FUNC_getsockopt: + if (is_sockopt_supported(moff)) + return &bpf_sk_getsockopt_proto; + return NULL; + case BPF_FUNC_get_retval: + if (moff == offsetof(struct bpf_tcp_ops, timeout_init) || + moff == offsetof(struct bpf_tcp_ops, rwnd_init)) + return &bpf_tcp_ops_get_retval_proto; + return NULL; + case BPF_FUNC_reserve_hdr_opt: + if (moff == offsetof(struct bpf_tcp_ops, hdr_opt_len)) + return &bpf_tcp_ops_reserve_hdr_opt_proto; + return NULL; + case BPF_FUNC_load_hdr_opt: + if (moff == offsetof(struct bpf_tcp_ops, parse_hdr) || + moff == offsetof(struct bpf_tcp_ops, write_hdr_opt)) + return &bpf_tcp_ops_load_hdr_opt_proto; + return NULL; + case BPF_FUNC_store_hdr_opt: + if (moff == offsetof(struct bpf_tcp_ops, write_hdr_opt)) + return &bpf_tcp_ops_store_hdr_opt_proto; + return NULL; + default: + return bpf_base_func_proto(func_id, prog); + } +} + +static bool is_valid_access(int off, int size, enum bpf_access_type type, + const struct bpf_prog *prog, struct bpf_insn_access_aux *info) +{ + if (!bpf_tracing_btf_ctx_access(off, size, type, prog, info)) + return false; + + if (base_type(info->reg_type) == PTR_TO_BTF_ID && + !bpf_type_has_unsafe_modifiers(info->reg_type) && + info->btf_id == btf_sock_ids[BTF_SOCK_TYPE_SOCK]) + /* promote it to tcp_sock */ + info->btf_id = btf_sock_ids[BTF_SOCK_TYPE_TCP]; + + return true; +} + +static int bpf_tcp_ops_init_member(const struct btf_type *t, + const struct btf_member *member, + void *kdata, const void *udata) +{ + return 0; +} + +static int bpf_tcp_ops_check_member(const struct btf_type *t, + const struct btf_member *member, + const struct bpf_prog *prog) +{ + if (prog->sleepable) + return -EINVAL; + + return 0; +} + +static int bpf_tcp_ops_init(struct btf *btf) +{ + return 0; +} + +static int bpf_tcp_ops_validate(void *kdata) +{ + return 0; +} + +static const struct bpf_verifier_ops bpf_tcp_ops_verifier = { + .get_func_proto = get_func_proto, + .is_valid_access = is_valid_access, +}; + +static struct bpf_struct_ops bpf_tcp_ops = { + .verifier_ops = &bpf_tcp_ops_verifier, + .init_member = bpf_tcp_ops_init_member, + .check_member = bpf_tcp_ops_check_member, + .init = bpf_tcp_ops_init, + .validate = bpf_tcp_ops_validate, + .name = "bpf_tcp_ops", + .cgroup_atype = CGROUP_TCP_SOCK_OPS, + .cfi_stubs = &__bpf_tcp_ops, + .owner = THIS_MODULE, +}; + +__bpf_kfunc_start_defs(); + +__bpf_kfunc int bpf_tcp_ops_set_flags(struct tcp_sock *tp, u32 enable, u32 disable) +{ + u32 old, new; + + if ((enable & disable) || (enable | disable) & ~BPF_TCP_OPS_FLAG_ALL) + return -EINVAL; + + if (sk_is_mptcp((struct sock *)tp) && (enable & BPF_TCP_OPS_FLAG_RCVQ)) + return -EOPNOTSUPP; + + old = READ_ONCE(tp->bpf_tcp_ops_flags); + + do { + new = (old | enable) & ~disable; + if (new == old) + break; + } while (!try_cmpxchg(&tp->bpf_tcp_ops_flags, &old, new)); + + return 0; +} + +__bpf_kfunc int bpf_tcp_ops_set_rcvlowat(struct sock *sk, int rcvlowat, + const struct bpf_prog_aux *aux) +{ + u32 moff = aux->attach_st_ops_member_off; + bool wakeup = false; + + if (moff == offsetof(struct bpf_tcp_ops, dequeue_rcvq)) + wakeup = true; + + if (rcvlowat < 0) + rcvlowat = INT_MAX; + + return __tcp_set_rcvlowat(sk, rcvlowat, wakeup); +} + +__bpf_kfunc_end_defs(); + +BTF_KFUNCS_START(bpf_tcp_ops_set_flags_kfunc_set) +BTF_ID_FLAGS(func, bpf_tcp_ops_set_flags) +BTF_KFUNCS_END(bpf_tcp_ops_set_flags_kfunc_set) + +BTF_KFUNCS_START(bpf_tcp_ops_set_rcvlowat_kfunc_set) +BTF_ID_FLAGS(func, bpf_tcp_ops_set_rcvlowat, KF_IMPLICIT_ARGS) +BTF_KFUNCS_END(bpf_tcp_ops_set_rcvlowat_kfunc_set) + +static int bpf_tcp_ops_set_flags_kfunc_filter(const struct bpf_prog *prog, + u32 kfunc_id) +{ + if (!btf_id_set8_contains(&bpf_tcp_ops_set_flags_kfunc_set, kfunc_id)) + return 0; + + if (prog->type == BPF_PROG_TYPE_STRUCT_OPS && + prog->aux->st_ops == &bpf_tcp_ops) + return 0; + + if (prog->type == BPF_PROG_TYPE_CGROUP_SOCKOPT || + prog->type == BPF_PROG_TYPE_CGROUP_SKB) + return 0; + + return -EACCES; +} + +static const struct btf_kfunc_id_set bpf_tcp_ops_set_flags_kfunc_id_set = { + .owner = THIS_MODULE, + .set = &bpf_tcp_ops_set_flags_kfunc_set, + .filter = bpf_tcp_ops_set_flags_kfunc_filter, +}; + +static int bpf_tcp_ops_set_rcvlowat_kfunc_filter(const struct bpf_prog *prog, + u32 kfunc_id) +{ + u32 moff; + + if (!btf_id_set8_contains(&bpf_tcp_ops_set_rcvlowat_kfunc_set, kfunc_id)) + return 0; + + if (prog->aux->st_ops != &bpf_tcp_ops) + return -EACCES; + + moff = prog->aux->attach_st_ops_member_off; + if (moff != offsetof(struct bpf_tcp_ops, enqueue_rcvq) && + moff != offsetof(struct bpf_tcp_ops, dequeue_rcvq)) + return -EACCES; + + return 0; +} + +static const struct btf_kfunc_id_set bpf_tcp_ops_set_rcvlowat_kfunc_id_set = { + .owner = THIS_MODULE, + .set = &bpf_tcp_ops_set_rcvlowat_kfunc_set, + .filter = bpf_tcp_ops_set_rcvlowat_kfunc_filter, +}; + +static int __init __bpf_tcp_ops_init(void) +{ + int ret; + + ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, + &bpf_tcp_ops_set_flags_kfunc_id_set); + /* BPF_PROG_TYPE_CGROUP_{SOCKOPT,SKB} share BTF_KFUNC_HOOK_CGROUP. */ + ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_CGROUP_SOCKOPT, + &bpf_tcp_ops_set_flags_kfunc_id_set); + ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, + &bpf_tcp_ops_set_rcvlowat_kfunc_id_set); + ret = ret ?: register_bpf_struct_ops(&bpf_tcp_ops, bpf_tcp_ops); + + return ret; +} + +late_initcall(__bpf_tcp_ops_init); diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c index 3fd477605704..59e52161c577 100644 --- a/net/ipv4/tcp.c +++ b/net/ipv4/tcp.c @@ -1612,6 +1612,8 @@ void tcp_cleanup_rbuf(struct sock *sk, int copied) "cleanup rbuf bug: copied %X seq %X rcvnxt %X\n", tp->copied_seq, TCP_SKB_CB(skb)->end_seq, tp->rcv_nxt); __tcp_cleanup_rbuf(sk, copied); + + bpf_tcp_ops_call_flag(dequeue_rcvq, RCVQ, sk); } static void tcp_eat_recv_skb(struct sock *sk, struct sk_buff *skb) @@ -1828,8 +1830,7 @@ int tcp_peek_len(struct socket *sock) return tcp_inq(sock->sk); } -/* Make sure sk_rcvbuf is big enough to satisfy SO_RCVLOWAT hint */ -int tcp_set_rcvlowat(struct sock *sk, int val) +int __tcp_set_rcvlowat(struct sock *sk, int val, bool wakeup) { struct tcp_sock *tp = tcp_sk(sk); int space, cap; @@ -1842,7 +1843,8 @@ int tcp_set_rcvlowat(struct sock *sk, int val) WRITE_ONCE(sk->sk_rcvlowat, val ? : 1); /* Check if we need to signal EPOLLIN right now */ - tcp_data_ready(sk); + if (wakeup) + tcp_data_ready(sk); if (sk->sk_userlocks & SOCK_RCVBUF_LOCK) return 0; @@ -1857,6 +1859,12 @@ int tcp_set_rcvlowat(struct sock *sk, int val) return 0; } +/* Make sure sk_rcvbuf is big enough to satisfy SO_RCVLOWAT hint */ +int tcp_set_rcvlowat(struct sock *sk, int val) +{ + return __tcp_set_rcvlowat(sk, val, true); +} + void tcp_set_rcvbuf(struct sock *sk, int val) { tcp_set_window_clamp(sk, tcp_win_from_space(sk, val)); @@ -2986,8 +2994,8 @@ void tcp_set_state(struct sock *sk, int state) */ BTF_TYPE_EMIT_ENUM(BPF_TCP_ESTABLISHED); - if (BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk), BPF_SOCK_OPS_STATE_CB_FLAG)) - tcp_call_bpf_2arg(sk, BPF_SOCK_OPS_STATE_CB, oldstate, state); + tcp_call_bpf_flag(sk, BPF_SOCK_OPS_STATE_CB, oldstate, state); + bpf_tcp_ops_call(set_state, sk, state); switch (state) { case TCP_ESTABLISHED: @@ -5208,6 +5216,9 @@ static void __init tcp_struct_check(void) CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_txrx, lost_out); CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_txrx, sacked_out); CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_txrx, scaling_ratio); +#ifdef CONFIG_BPF + CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_txrx, bpf_tcp_ops_flags); +#endif /* RX read-mostly hotpath cache lines */ CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, copied_seq); diff --git a/net/ipv4/tcp_bpf.c b/net/ipv4/tcp_bpf.c index c2b024a90bbd..8cf63a9e7fed 100644 --- a/net/ipv4/tcp_bpf.c +++ b/net/ipv4/tcp_bpf.c @@ -108,7 +108,6 @@ static int tcp_bpf_push(struct sock *sk, struct sk_msg *msg, u32 apply_bytes, off = sge->offset; page = sg_page(sge); - tcp_rate_check_app_limited(sk); retry: msghdr.msg_flags = flags | MSG_SPLICE_PAGES; has_tx_ulp = tls_sw_has_ctx_tx(sk); diff --git a/net/ipv4/tcp_fastopen.c b/net/ipv4/tcp_fastopen.c index 471c78be5513..6dfe40322fb5 100644 --- a/net/ipv4/tcp_fastopen.c +++ b/net/ipv4/tcp_fastopen.c @@ -281,6 +281,8 @@ void tcp_fastopen_add_skb(struct sock *sk, struct sk_buff *skb) TCP_SKB_CB(skb)->seq++; TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_SYN; + bpf_tcp_ops_call_flag(enqueue_rcvq, RCVQ, sk, skb); + tp->rcv_nxt = TCP_SKB_CB(skb)->end_seq; tcp_add_receive_queue(sk, skb); tp->syn_data_acked = 1; diff --git a/net/ipv4/tcp_input.c b/net/ipv4/tcp_input.c index 1ddce9c4b7e8..e1f2d9765f81 100644 --- a/net/ipv4/tcp_input.c +++ b/net/ipv4/tcp_input.c @@ -146,14 +146,16 @@ EXPORT_SYMBOL_GPL(clean_acked_data_flush); #ifdef CONFIG_CGROUP_BPF static void bpf_skops_parse_hdr(struct sock *sk, struct sk_buff *skb) { - bool unknown_opt = tcp_sk(sk)->rx_opt.saw_unknown && - BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk), - BPF_SOCK_OPS_PARSE_UNKNOWN_HDR_OPT_CB_FLAG); - bool parse_all_opt = BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk), - BPF_SOCK_OPS_PARSE_ALL_HDR_OPT_CB_FLAG); struct bpf_sock_ops_kern sock_ops; + const struct tcp_sock *tp; - if (likely(!unknown_opt && !parse_all_opt)) + if (!cgroup_bpf_enabled(CGROUP_SOCK_OPS)) + return; + + tp = tcp_sk(sk); + if (!(tp->rx_opt.saw_unknown && + BPF_SOCK_OPS_TEST_FLAG(tp, BPF_SOCK_OPS_PARSE_UNKNOWN_HDR_OPT_CB_FLAG)) && + !BPF_SOCK_OPS_TEST_FLAG(tp, BPF_SOCK_OPS_PARSE_ALL_HDR_OPT_CB_FLAG)) return; /* The skb will be handled in the @@ -176,7 +178,7 @@ static void bpf_skops_parse_hdr(struct sock *sk, struct sk_buff *skb) sock_ops.sk = sk; bpf_skops_init_skb(&sock_ops, skb, tcp_hdrlen(skb)); - BPF_CGROUP_RUN_PROG_SOCK_OPS(&sock_ops); + __BPF_CGROUP_RUN_PROG_SOCK_OPS(&sock_ops); } static void bpf_skops_established(struct sock *sk, int bpf_op, @@ -184,6 +186,9 @@ static void bpf_skops_established(struct sock *sk, int bpf_op, { struct bpf_sock_ops_kern sock_ops; + if (!cgroup_bpf_enabled(CGROUP_SOCK_OPS)) + return; + sock_owned_by_me(sk); memset(&sock_ops, 0, offsetof(struct bpf_sock_ops_kern, temp)); @@ -195,7 +200,7 @@ static void bpf_skops_established(struct sock *sk, int bpf_op, if (skb) bpf_skops_init_skb(&sock_ops, skb, tcp_hdrlen(skb)); - BPF_CGROUP_RUN_PROG_SOCK_OPS(&sock_ops); + __BPF_CGROUP_RUN_PROG_SOCK_OPS(&sock_ops); } #else static void bpf_skops_parse_hdr(struct sock *sk, struct sk_buff *skb) @@ -208,6 +213,28 @@ static void bpf_skops_established(struct sock *sk, int bpf_op, } #endif +static void bpf_tcp_ops_parse_hdr(struct sock *sk, struct sk_buff *skb) +{ + const struct tcp_sock *tp; + + if (!cgroup_bpf_enabled(CGROUP_TCP_SOCK_OPS)) + return; + + switch (sk->sk_state) { + case TCP_SYN_RECV: + case TCP_SYN_SENT: + case TCP_LISTEN: + return; + } + + tp = tcp_sk(sk); + + if ((tp->rx_opt.saw_unknown && + BPF_TCP_OPS_TEST_FLAG(tp, PARSE_HDR_OPT_UNKNOWN)) || + BPF_TCP_OPS_TEST_FLAG(tp, PARSE_HDR_OPT_ALL)) + __bpf_tcp_ops_call(parse_hdr, sk, skb); +} + static __cold void tcp_gro_dev_warn(const struct sock *sk, const struct sk_buff *skb, unsigned int len) { @@ -5338,6 +5365,8 @@ static void tcp_ofo_queue(struct sock *sk) continue; } + bpf_tcp_ops_call_flag(enqueue_rcvq, RCVQ, sk, skb); + tail = skb_peek_tail(&sk->sk_receive_queue); eaten = tail && tcp_try_coalesce(sk, tail, skb, &fragstolen); tcp_rcv_nxt_update(tp, TCP_SKB_CB(skb)->end_seq); @@ -5541,6 +5570,8 @@ static int __must_check tcp_queue_rcv(struct sock *sk, struct sk_buff *skb, int eaten; struct sk_buff *tail = skb_peek_tail(&sk->sk_receive_queue); + bpf_tcp_ops_call_flag(enqueue_rcvq, RCVQ, sk, skb); + eaten = (tail && tcp_try_coalesce(sk, tail, skb, fragstolen)) ? 1 : 0; @@ -6480,6 +6511,7 @@ static bool tcp_validate_incoming(struct sock *sk, struct sk_buff *skb, pass: bpf_skops_parse_hdr(sk, skb); + bpf_tcp_ops_parse_hdr(sk, skb); return true; @@ -6744,6 +6776,10 @@ void tcp_init_transfer(struct sock *sk, int bpf_op, struct sk_buff *skb) tp->snd_cwnd_stamp = tcp_jiffies32; bpf_skops_established(sk, bpf_op, skb); + if (bpf_op == BPF_SOCK_OPS_ACTIVE_ESTABLISHED_CB) + bpf_tcp_ops_call(active_established, sk, skb); + else + bpf_tcp_ops_call(passive_established, sk, skb); /* Initialize congestion control unless BPF initialized it already: */ if (!icsk->icsk_ca_initialized) tcp_init_congestion_control(sk); diff --git a/net/ipv4/tcp_nv.c b/net/ipv4/tcp_nv.c index f345897a68df..7b0dae23d9aa 100644 --- a/net/ipv4/tcp_nv.c +++ b/net/ipv4/tcp_nv.c @@ -146,7 +146,7 @@ static void tcpnv_init(struct sock *sk) * within a datacenter, where we have reasonable estimates of * RTTs */ - base_rtt = tcp_call_bpf(sk, BPF_SOCK_OPS_BASE_RTT, 0, NULL); + base_rtt = tcp_call_bpf(sk, BPF_SOCK_OPS_BASE_RTT); if (base_rtt > 0) { ca->nv_base_rtt = base_rtt; ca->nv_lower_bound_rtt = (base_rtt * 205) >> 8; /* 80% */ diff --git a/net/ipv4/tcp_output.c b/net/ipv4/tcp_output.c index c8865205cdc8..df3aaba8574f 100644 --- a/net/ipv4/tcp_output.c +++ b/net/ipv4/tcp_output.c @@ -476,8 +476,8 @@ static u32 bpf_skops_hdr_opt_len(struct sock *sk, struct sk_buff *skb, struct bpf_sock_ops_kern sock_ops; int err; - if (likely(!BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk), - BPF_SOCK_OPS_WRITE_HDR_OPT_CB_FLAG)) || + if (!BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk), + BPF_SOCK_OPS_WRITE_HDR_OPT_CB_FLAG) || !remaining) return remaining; @@ -518,7 +518,7 @@ static u32 bpf_skops_hdr_opt_len(struct sock *sk, struct sk_buff *skb, if (skb) bpf_skops_init_skb(&sock_ops, skb, 0); - err = BPF_CGROUP_RUN_PROG_SOCK_OPS_SK(&sock_ops, sk); + err = __BPF_CGROUP_RUN_PROG_SOCK_OPS_SK(&sock_ops, sk); if (err || sock_ops.remaining_opt_len == remaining) return remaining; @@ -536,43 +536,54 @@ static void bpf_skops_write_hdr_opt(struct sock *sk, struct sk_buff *skb, enum tcp_synack_type synack_type, struct tcp_out_options *opts) { - u8 first_opt_off, nr_written, max_opt_len = opts->bpf_opt_len; - struct bpf_sock_ops_kern sock_ops; - int err; + u8 first_opt_off, nr_written = 0, max_opt_len = opts->bpf_opt_len; if (likely(!max_opt_len)) return; - memset(&sock_ops, 0, offsetof(struct bpf_sock_ops_kern, temp)); - - sock_ops.op = BPF_SOCK_OPS_WRITE_HDR_OPT_CB; - - if (req) { - sock_ops.sk = (struct sock *)req; - sock_ops.syn_skb = syn_skb; - } else { - sock_owned_by_me(sk); - - sock_ops.is_fullsock = 1; - sock_ops.is_locked_tcp_sock = 1; - sock_ops.sk = sk; - } - - sock_ops.args[0] = bpf_skops_write_hdr_opt_arg0(skb, synack_type); - sock_ops.remaining_opt_len = max_opt_len; first_opt_off = tcp_hdrlen(skb) - max_opt_len; - bpf_skops_init_skb(&sock_ops, skb, first_opt_off); - err = BPF_CGROUP_RUN_PROG_SOCK_OPS_SK(&sock_ops, sk); + if (cgroup_bpf_enabled(CGROUP_SOCK_OPS) && + BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk), + BPF_SOCK_OPS_WRITE_HDR_OPT_CB_FLAG)) { + struct bpf_sock_ops_kern sock_ops; + int err; - if (err) - nr_written = 0; - else - nr_written = max_opt_len - sock_ops.remaining_opt_len; + memset(&sock_ops, 0, offsetof(struct bpf_sock_ops_kern, temp)); + + sock_ops.op = BPF_SOCK_OPS_WRITE_HDR_OPT_CB; + + if (req) { + sock_ops.sk = (struct sock *)req; + sock_ops.syn_skb = syn_skb; + } else { + sock_owned_by_me(sk); + + sock_ops.is_fullsock = 1; + sock_ops.is_locked_tcp_sock = 1; + sock_ops.sk = sk; + } + + sock_ops.args[0] = bpf_skops_write_hdr_opt_arg0(skb, synack_type); + sock_ops.remaining_opt_len = max_opt_len; + bpf_skops_init_skb(&sock_ops, skb, first_opt_off); + + err = __BPF_CGROUP_RUN_PROG_SOCK_OPS_SK(&sock_ops, sk); + if (!err) + nr_written = max_opt_len - sock_ops.remaining_opt_len; + } if (nr_written < max_opt_len) memset(skb->data + first_opt_off + nr_written, TCPOPT_NOP, max_opt_len - nr_written); + + /* + * bpf_tcp_ops portion is NOP-filled (everything past the sockops + * writer's bytes). The writer finds the append point by scanning from + * first_opt_off + nr_written to the first NOP. + */ + bpf_tcp_ops_call_flag(write_hdr_opt, WRITE_HDR_OPT, sk, skb, req, + syn_skb, synack_type, first_opt_off + nr_written); } #else static u32 bpf_skops_hdr_opt_len(struct sock *sk, struct sk_buff *skb, @@ -594,6 +605,53 @@ static void bpf_skops_write_hdr_opt(struct sock *sk, struct sk_buff *skb, } #endif +static u32 bpf_tcp_ops_hdr_opt_len(struct sock *sk, struct sk_buff *skb, + struct request_sock *req, + struct sk_buff *syn_skb, + enum tcp_synack_type synack_type, + struct tcp_out_options *opts, + u32 remaining) +{ + unsigned int remaining_out = remaining, reserved; + + if (!BPF_TCP_OPS_TEST_FLAG(tcp_sk(sk), WRITE_HDR_OPT) || + !remaining) + return remaining; + + /* bpf_tcp_ops_reserve_hdr_opt() reserves space via remaining_out */ + __bpf_tcp_ops_call(hdr_opt_len, sk, skb, req, syn_skb, synack_type, &remaining_out); + + reserved = remaining - remaining_out; + if (!reserved) + return remaining; + + /* round up to 4 bytes */ + reserved = (reserved + 3) & ~3; + + opts->bpf_opt_len += reserved; + return remaining - reserved; +} + +static __always_inline u32 tcp_bpf_hdr_opt_len(struct sock *sk, struct sk_buff *skb, + struct request_sock *req, + struct sk_buff *syn_skb, + enum tcp_synack_type synack_type, + struct tcp_out_options *opts, + u32 remaining) +{ + if (cgroup_bpf_enabled(CGROUP_SOCK_OPS)) + remaining = bpf_skops_hdr_opt_len(sk, skb, req, syn_skb, + synack_type, opts, + remaining); + + if (cgroup_bpf_enabled(CGROUP_TCP_SOCK_OPS)) + remaining = bpf_tcp_ops_hdr_opt_len(sk, skb, req, syn_skb, + synack_type, opts, + remaining); + + return remaining; +} + static __be32 *process_tcp_ao_options(struct tcp_sock *tp, const struct tcp_request_sock *tcprsk, struct tcp_out_options *opts, @@ -1051,8 +1109,8 @@ static unsigned int tcp_syn_options(struct sock *sk, struct sk_buff *skb, remaining -= tcp_options_fit_accecn(opts, 0, remaining); } - remaining = bpf_skops_hdr_opt_len(sk, skb, NULL, NULL, 0, opts, - remaining); + remaining = tcp_bpf_hdr_opt_len(sk, skb, NULL, NULL, 0, opts, + remaining); return MAX_TCP_OPTION_SPACE - remaining; } @@ -1139,8 +1197,8 @@ static unsigned int tcp_synack_options(const struct sock *sk, remaining -= tcp_options_fit_accecn(opts, 0, remaining); } - remaining = bpf_skops_hdr_opt_len((struct sock *)sk, skb, req, syn_skb, - synack_type, opts, remaining); + remaining = tcp_bpf_hdr_opt_len((struct sock *)sk, skb, req, syn_skb, + synack_type, opts, remaining); return MAX_TCP_OPTION_SPACE - remaining; } @@ -1153,10 +1211,12 @@ static unsigned int tcp_established_options(struct sock *sk, struct sk_buff *skb struct tcp_key *key) { struct tcp_sock *tp = tcp_sk(sk); - unsigned int size = 0; unsigned int eff_sacks; + unsigned int remaining; + unsigned int size = 0; opts->options = 0; + opts->bpf_opt_len = 0; /* Better than switch (key.type) as it has static branches */ if (tcp_key_is_md5(key)) { @@ -1183,10 +1243,10 @@ static unsigned int tcp_established_options(struct sock *sk, struct sk_buff *skb * left. */ if (sk_is_mptcp(sk)) { - unsigned int remaining = MAX_TCP_OPTION_SPACE - size; bool has_ts = opts->options & OPTION_TS; int opt_size; + remaining = MAX_TCP_OPTION_SPACE - size; opts->mptcp.drop_ts = 0; opt_size = mptcp_established_options(sk, skb, remaining, has_ts, @@ -1203,7 +1263,8 @@ static unsigned int tcp_established_options(struct sock *sk, struct sk_buff *skb eff_sacks = tp->rx_opt.num_sacks + tp->rx_opt.dsack; if (unlikely(eff_sacks)) { - const unsigned int remaining = MAX_TCP_OPTION_SPACE - size; + remaining = MAX_TCP_OPTION_SPACE - size; + if (likely(remaining >= TCPOLEN_SACK_BASE_ALIGNED + TCPOLEN_SACK_PERBLOCK)) { opts->num_sack_blocks = @@ -1234,15 +1295,10 @@ static unsigned int tcp_established_options(struct sock *sk, struct sk_buff *skb } } - if (unlikely(BPF_SOCK_OPS_TEST_FLAG(tp, - BPF_SOCK_OPS_WRITE_HDR_OPT_CB_FLAG))) { - unsigned int remaining = MAX_TCP_OPTION_SPACE - size; + remaining = tcp_bpf_hdr_opt_len(sk, skb, NULL, NULL, 0, opts, + MAX_TCP_OPTION_SPACE - size); - remaining = bpf_skops_hdr_opt_len(sk, skb, NULL, NULL, 0, opts, - remaining); - - size = MAX_TCP_OPTION_SPACE - remaining; - } + size = MAX_TCP_OPTION_SPACE - remaining; return size; } @@ -3683,9 +3739,9 @@ int __tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb, int segs) err = tcp_transmit_skb(sk, skb, 1, GFP_ATOMIC); } - if (BPF_SOCK_OPS_TEST_FLAG(tp, BPF_SOCK_OPS_RETRANS_CB_FLAG)) - tcp_call_bpf_3arg(sk, BPF_SOCK_OPS_RETRANS_CB, - TCP_SKB_CB(skb)->seq, segs, err); + tcp_call_bpf_flag(sk, BPF_SOCK_OPS_RETRANS_CB, + TCP_SKB_CB(skb)->seq, segs, err); + bpf_tcp_ops_call(retrans, sk, skb, err); if (unlikely(err) && err != -EBUSY) NET_ADD_STATS(sock_net(sk), LINUX_MIB_TCPRETRANSFAIL, segs); @@ -4312,7 +4368,8 @@ int tcp_connect(struct sock *sk) struct sk_buff *buff; int err; - tcp_call_bpf(sk, BPF_SOCK_OPS_TCP_CONNECT_CB, 0, NULL); + tcp_call_bpf(sk, BPF_SOCK_OPS_TCP_CONNECT_CB); + bpf_tcp_ops_call(connect, sk); #if defined(CONFIG_TCP_MD5SIG) && defined(CONFIG_TCP_AO) /* Has to be checked late, after setting daddr/saddr/ops. diff --git a/net/ipv4/tcp_timer.c b/net/ipv4/tcp_timer.c index e56eae4bc341..00debcba122b 100644 --- a/net/ipv4/tcp_timer.c +++ b/net/ipv4/tcp_timer.c @@ -286,10 +286,9 @@ static int tcp_write_timeout(struct sock *sk) tcp_fastopen_active_detect_blackhole(sk, expired); mptcp_active_detect_blackhole(sk, expired); - if (BPF_SOCK_OPS_TEST_FLAG(tp, BPF_SOCK_OPS_RTO_CB_FLAG)) - tcp_call_bpf_3arg(sk, BPF_SOCK_OPS_RTO_CB, - icsk->icsk_retransmits, - icsk->icsk_rto, (int)expired); + tcp_call_bpf_flag(sk, BPF_SOCK_OPS_RTO_CB, + icsk->icsk_retransmits, icsk->icsk_rto, (int)expired); + bpf_tcp_ops_call(rto, sk); if (expired) { /* Has it gone just too far? */ diff --git a/net/sched/bpf_qdisc.c b/net/sched/bpf_qdisc.c index 098ca02aed89..5691c13781a8 100644 --- a/net/sched/bpf_qdisc.c +++ b/net/sched/bpf_qdisc.c @@ -280,7 +280,6 @@ BTF_KFUNCS_START(qdisc_kfunc_ids) BTF_ID_FLAGS(func, bpf_skb_get_hash) BTF_ID_FLAGS(func, bpf_kfree_skb, KF_RELEASE) BTF_ID_FLAGS(func, bpf_qdisc_skb_drop, KF_RELEASE) -BTF_ID_FLAGS(func, bpf_dynptr_from_skb) BTF_ID_FLAGS(func, bpf_qdisc_watchdog_schedule) BTF_ID_FLAGS(func, bpf_qdisc_init_prologue) BTF_ID_FLAGS(func, bpf_qdisc_reset_destroy_epilogue) @@ -290,7 +289,6 @@ BTF_KFUNCS_END(qdisc_kfunc_ids) BTF_SET_START(qdisc_common_kfunc_set) BTF_ID(func, bpf_skb_get_hash) BTF_ID(func, bpf_kfree_skb) -BTF_ID(func, bpf_dynptr_from_skb) BTF_SET_END(qdisc_common_kfunc_set) BTF_SET_START(qdisc_enqueue_kfunc_set) diff --git a/samples/bpf/Makefile b/samples/bpf/Makefile index 95a4fa1f1e44..c28e65046986 100644 --- a/samples/bpf/Makefile +++ b/samples/bpf/Makefile @@ -342,6 +342,7 @@ $(obj)/xdp_router_ipv4.bpf.o: $(obj)/xdp_sample.bpf.o $(obj)/%.bpf.o: $(src)/%.bpf.c $(obj)/vmlinux.h $(src)/xdp_sample.bpf.h $(src)/xdp_sample_shared.h @echo " CLANG-BPF " $@ $(Q)$(CLANG) -g -O2 --target=bpf -D__TARGET_ARCH_$(SRCARCH) \ + -fms-extensions -Wno-microsoft-anon-tag \ -Wno-compare-distinct-pointer-types -I$(srctree)/include \ -I$(srctree)/samples/bpf -I$(srctree)/tools/include \ -I$(LIBBPF_INCLUDE) $(CLANG_SYS_INCLUDES) \ diff --git a/samples/bpf/hash_func01.h b/samples/bpf/hash_func01.h deleted file mode 100644 index 38255812e376..000000000000 --- a/samples/bpf/hash_func01.h +++ /dev/null @@ -1,55 +0,0 @@ -/* SPDX-License-Identifier: LGPL-2.1 - * - * Based on Paul Hsieh's (LGPG 2.1) hash function - * From: http://www.azillionmonkeys.com/qed/hash.html - */ - -#define get16bits(d) (*((const __u16 *) (d))) - -static __always_inline -__u32 SuperFastHash (const char *data, int len, __u32 initval) { - __u32 hash = initval; - __u32 tmp; - int rem; - - if (len <= 0 || data == NULL) return 0; - - rem = len & 3; - len >>= 2; - - /* Main loop */ -#pragma clang loop unroll(full) - for (;len > 0; len--) { - hash += get16bits (data); - tmp = (get16bits (data+2) << 11) ^ hash; - hash = (hash << 16) ^ tmp; - data += 2*sizeof (__u16); - hash += hash >> 11; - } - - /* Handle end cases */ - switch (rem) { - case 3: hash += get16bits (data); - hash ^= hash << 16; - hash ^= ((signed char)data[sizeof (__u16)]) << 18; - hash += hash >> 11; - break; - case 2: hash += get16bits (data); - hash ^= hash << 11; - hash += hash >> 17; - break; - case 1: hash += (signed char)*data; - hash ^= hash << 10; - hash += hash >> 1; - } - - /* Force "avalanching" of final 127 bits */ - hash ^= hash << 3; - hash += hash >> 5; - hash ^= hash << 4; - hash += hash >> 17; - hash ^= hash << 25; - hash += hash >> 6; - - return hash; -} diff --git a/scripts/Makefile.btf b/scripts/Makefile.btf index a1812985a61a..98012344a81e 100644 --- a/scripts/Makefile.btf +++ b/scripts/Makefile.btf @@ -22,7 +22,14 @@ endif pahole-flags-$(CONFIG_PAHOLE_HAS_LANG_EXCLUDE) += --lang_exclude=rust +btf-inline := $(CONFIG_DEBUG_INFO_BTF_INLINE) +ifneq ($(btf-inline),) +btf-inline-feat := inline +pahole-flags-$(call test-ge, $(pahole-ver), 132) += --btf_features=$(btf-inline-feat) +endif + export PAHOLE_FLAGS := $(pahole-flags-y) +export BTF_INLINE := $(btf-inline) resolve-btfids-flags-y := resolve-btfids-flags-$(CONFIG_WERROR) += --fatal_warnings diff --git a/scripts/gen-btf.sh b/scripts/gen-btf.sh index 8ca96eb10a69..b17f1189298a 100755 --- a/scripts/gen-btf.sh +++ b/scripts/gen-btf.sh @@ -68,6 +68,7 @@ esac gen_btf_data() { + inline=${BTF_INLINE:+--inline} btf1="${ELF_FILE}.BTF.1" ${PAHOLE} -J ${PAHOLE_FLAGS} \ ${BTF_BASE:+--btf_base ${BTF_BASE}} \ @@ -76,6 +77,7 @@ gen_btf_data() ${RESOLVE_BTFIDS} ${RESOLVE_BTFIDS_FLAGS} \ ${BTF_BASE:+--btf_base ${BTF_BASE}} \ + ${inline} \ --btf ${btf1} "${ELF_FILE}" } @@ -83,14 +85,28 @@ gen_btf_o() { btf_data=${ELF_FILE}.btf.o - # Create ${btf_data} which contains just .BTF section but no symbols. Add + # Create ${btf_data} which contains just BTF sections but no symbols. Add # SHF_ALLOC because .BTF will be part of the vmlinux image. --strip-all # deletes all symbols including __start_BTF and __stop_BTF, which will # be redefined in the linker script. echo "" | ${CC} ${CLANG_FLAGS} ${KBUILD_CPPFLAGS} ${KBUILD_CFLAGS} -fno-lto -c -x c -o ${btf_data} - ${OBJCOPY} --add-section .BTF=${ELF_FILE}.BTF \ --set-section-flags .BTF=alloc,readonly ${btf_data} - ${OBJCOPY} --only-section=.BTF --strip-all ${btf_data} + ONLY_SEC="--only-section=.BTF" + btf_inline=${ELF_FILE}.BTF.inline + if [ "${BTF_INLINE}" = "m" ]; then + if [ -f "${btf_inline}" ]; then + cp "${btf_inline}" "${objtree}/vmlinux.BTF.inline" + else + rm -f "${objtree}/vmlinux.BTF.inline" + fi + fi + if [ "${BTF_INLINE}" = "y" ] && [ -f "${btf_inline}" ]; then + ${OBJCOPY} --add-section .BTF.inline=${btf_inline} \ + --set-section-flags .BTF.inline=alloc,readonly ${btf_data} + ONLY_SEC="${ONLY_SEC} --only-section=.BTF.inline" + fi + ${OBJCOPY} ${ONLY_SEC} --strip-all ${btf_data} # Change e_type to ET_REL so that it can be used to link final vmlinux. # GNU ld 2.35+ and lld do not allow an ET_EXEC input. @@ -111,6 +127,15 @@ embed_btf_data() if [ -f "${btf_base}" ]; then ${OBJCOPY} --add-section .BTF.base=${btf_base} ${ELF_FILE} fi + btf_inline=${ELF_FILE}.BTF.inline + case "${ELF_FILE}" in + */btf_vmlinux_inline.ko) + btf_inline=${BTF_BASE}.BTF.inline + ;; + esac + if [ -n "${BTF_INLINE}" ] && [ -f "${btf_inline}" ]; then + ${OBJCOPY} --add-section .BTF.inline=${btf_inline} ${ELF_FILE} + fi btf_ids="${ELF_FILE}.BTF_ids" if [ -f "${btf_ids}" ]; then ${RESOLVE_BTFIDS} --patch_btfids ${btf_ids} ${ELF_FILE} @@ -121,6 +146,8 @@ cleanup() { rm -f "${ELF_FILE}.BTF.1" rm -f "${ELF_FILE}.BTF" + rm -f "${ELF_FILE}.BTF.inline" + rm -f "${ELF_FILE}.BTF.inline.link" if [ "${BTFGEN_MODE}" = "module" ]; then rm -f "${ELF_FILE}.BTF.base" rm -f "${ELF_FILE}.BTF_ids" diff --git a/scripts/link-vmlinux.sh b/scripts/link-vmlinux.sh index c520741f85c2..d5710e900bd4 100755 --- a/scripts/link-vmlinux.sh +++ b/scripts/link-vmlinux.sh @@ -295,7 +295,13 @@ vmlinux_link "${VMLINUX}" if is_enabled CONFIG_DEBUG_INFO_BTF; then info BTFIDS ${VMLINUX} - ${RESOLVE_BTFIDS} --patch_btfids ${btfids_vmlinux} ${VMLINUX} + if [ "${BTF_INLINE}" = "m" ]; then + ${RESOLVE_BTFIDS} --patch_btfids ${btfids_vmlinux} \ + --btf_link .BTF.inline:btf_vmlinux_inline:vmlinux.BTF.inline \ + ${VMLINUX} + else + ${RESOLVE_BTFIDS} --patch_btfids ${btfids_vmlinux} ${VMLINUX} + fi fi mksysmap "${VMLINUX}" System.map diff --git a/scripts/package/kernel.spec b/scripts/package/kernel.spec index 46e80970f723..8f0933cbbe45 100644 --- a/scripts/package/kernel.spec +++ b/scripts/package/kernel.spec @@ -68,13 +68,13 @@ This package provides debug information for the kernel image and modules from th %undefine _debugsource_packages %undefine _debuginfo_subpackages -# Preserve .BTF and .BTF.base sections in kernel modules during debuginfo -# stripping. find-debuginfo.sh uses eu-strip which removes non-allocated ELF -# sections like .BTF by default. .BTF.base is required for BTF distillation -# support; without it, module BTF validation fails. +# Preserve .BTF, .BTF.base and .BTF.inline sections in kernel modules during +# debuginfo stripping. find-debuginfo.sh uses eu-strip which removes +# non-allocated ELF sections like .BTF by default. .BTF.base is required for +# BTF distillation support; without it, module BTF validation fails. %global with_keep_section %(%{__find_debuginfo} --help 2>&1 | grep -c keep-section) %if %{with_keep_section} -%global _find_debuginfo_opts -r --keep-section .BTF --keep-section .BTF.base +%global _find_debuginfo_opts -r --keep-section .BTF --keep-section .BTF.base --keep-section .BTF.inline %else %global _find_debuginfo_opts -r %endif diff --git a/security/bpf/hooks.c b/security/bpf/hooks.c index 7b98f5d1e2be..8f8c3de3035f 100644 --- a/security/bpf/hooks.c +++ b/security/bpf/hooks.c @@ -33,6 +33,7 @@ static int __init bpf_lsm_init(void) struct lsm_blob_sizes bpf_lsm_blob_sizes __ro_after_init = { .lbs_inode = sizeof(struct bpf_storage_blob), + .lbs_xattr_count = BPF_LSM_INODE_INIT_XATTRS, }; DEFINE_LSM(bpf) = { diff --git a/security/keys/process_keys.c b/security/keys/process_keys.c index 1a51a36ce32a..cf638bf1f7b9 100644 --- a/security/keys/process_keys.c +++ b/security/keys/process_keys.c @@ -22,6 +22,9 @@ /* Session keyring create vs join semaphore */ static DEFINE_MUTEX(key_session_mutex); +/* BPF keyring reachable through KEY_SPEC_BPF_KEYRING */ +static struct key *bpf_keyring __ro_after_init; + /* The root user's tracking struct */ struct key_user root_key_user = { .usage = REFCOUNT_INIT(3), @@ -589,6 +592,20 @@ bool lookup_user_key_possessed(const struct key *key, return key == match_data->raw_data; } +/** + * key_register_bpf_keyring - Publish the BPF keyring for KEY_SPEC_BPF_KEYRING + * @keyring: The keyring to publish + * + * Make @keyring reachable by userspace through the KEY_SPEC_BPF_KEYRING + * special key ID, so that provisioning it does not require scraping its + * serial out of /proc/keys first. Called once, from an initcall, and never + * undone. + */ +void key_register_bpf_keyring(struct key *keyring) +{ + bpf_keyring = keyring; +} + /* * Look up a key ID given us by userspace with a given permissions mask to get * the key it refers to. @@ -740,6 +757,14 @@ key_ref_t lookup_user_key(key_serial_t id, unsigned long lflags, key_ref = make_key_ref(key, 1); break; + case KEY_SPEC_BPF_KEYRING: + key = bpf_keyring; + if (!key) + goto error; + __key_get(key); + key_ref = make_key_ref(key, 0); + break; + default: key_ref = ERR_PTR(-EINVAL); if (id < 1) diff --git a/tools/bpf/.gitignore b/tools/bpf/.gitignore deleted file mode 100644 index cf53342175e7..000000000000 --- a/tools/bpf/.gitignore +++ /dev/null @@ -1,7 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -FEATURE-DUMP.bpf -feature -bpf_asm -bpf_dbg -bpf_exp.yacc.* -bpf_jit_disasm diff --git a/tools/bpf/Makefile b/tools/bpf/Makefile index 9c19e81f3c27..a41360ecb2db 100644 --- a/tools/bpf/Makefile +++ b/tools/bpf/Makefile @@ -1,108 +1,11 @@ # SPDX-License-Identifier: GPL-2.0 include ../scripts/Makefile.include -prefix ?= /usr/local - -LEX = flex -YACC = bison -MAKE = make -INSTALL ?= install - -CFLAGS += -Wall -O2 -CFLAGS += -D__EXPORTED_HEADERS__ -I$(srctree)/tools/include/uapi \ - -I$(srctree)/tools/include -CFLAGS += $(EXTRA_CFLAGS) - -# This will work when bpf is built in tools env. where srctree -# isn't set and when invoked from selftests build, where srctree -# is set to ".". building_out_of_srctree is undefined for in srctree -# builds -ifeq ($(srctree),) -update_srctree := 1 -endif -ifndef building_out_of_srctree -update_srctree := 1 -endif -ifeq ($(update_srctree),1) -srctree := $(patsubst %/,%,$(dir $(CURDIR))) -srctree := $(patsubst %/,%,$(dir $(srctree))) -endif - -FEATURE_USER = .bpf -FEATURE_TESTS = libbfd disassembler-four-args disassembler-init-styled -FEATURE_DISPLAY = libbfd - -check_feat := 1 -NON_CHECK_FEAT_TARGETS := clean bpftool_clean resolve_btfids_clean -ifdef MAKECMDGOALS -ifeq ($(filter-out $(NON_CHECK_FEAT_TARGETS),$(MAKECMDGOALS)),) - check_feat := 0 -endif -endif - -ifeq ($(check_feat),1) -ifeq ($(FEATURES_DUMP),) -include $(srctree)/tools/build/Makefile.feature -else -include $(FEATURES_DUMP) -endif -endif - -ifeq ($(feature-disassembler-four-args), 1) -CFLAGS += -DDISASM_FOUR_ARGS_SIGNATURE -endif -ifeq ($(feature-disassembler-init-styled), 1) -CFLAGS += -DDISASM_INIT_STYLED -endif - -$(OUTPUT)%.yacc.c: $(srctree)/tools/bpf/%.y - $(QUIET_BISON)$(YACC) -o $@ -d $< - -$(OUTPUT)%.lex.c: $(srctree)/tools/bpf/%.l - $(QUIET_FLEX)$(LEX) -o $@ $< - -$(OUTPUT)%.o: $(srctree)/tools/bpf/%.c - $(QUIET_CC)$(CC) $(CFLAGS) -c -o $@ $< - -$(OUTPUT)%.yacc.o: $(OUTPUT)%.yacc.c - $(QUIET_CC)$(CC) $(CFLAGS) -c -o $@ $< -$(OUTPUT)%.lex.o: $(OUTPUT)%.lex.c - $(QUIET_CC)$(CC) $(CFLAGS) -c -o $@ $< - -PROGS = $(OUTPUT)bpf_jit_disasm $(OUTPUT)bpf_dbg $(OUTPUT)bpf_asm - -all: $(PROGS) bpftool - -$(OUTPUT)bpf_jit_disasm: CFLAGS += -DPACKAGE='bpf_jit_disasm' -$(OUTPUT)bpf_jit_disasm: $(OUTPUT)bpf_jit_disasm.o - $(QUIET_LINK)$(CC) $(CFLAGS) -o $@ $^ -lopcodes -lbfd -ldl - -$(OUTPUT)bpf_dbg: $(OUTPUT)bpf_dbg.o - $(QUIET_LINK)$(CC) $(CFLAGS) -o $@ $^ -lreadline - -$(OUTPUT)bpf_asm: $(OUTPUT)bpf_asm.o $(OUTPUT)bpf_exp.yacc.o $(OUTPUT)bpf_exp.lex.o - $(QUIET_LINK)$(CC) $(CFLAGS) -o $@ $^ - -$(OUTPUT)bpf_exp.lex.c: $(OUTPUT)bpf_exp.yacc.c -$(OUTPUT)bpf_exp.yacc.o: $(OUTPUT)bpf_exp.yacc.c -$(OUTPUT)bpf_exp.lex.o: $(OUTPUT)bpf_exp.lex.c +all: bpftool clean: bpftool_clean resolve_btfids_clean - $(call QUIET_CLEAN, bpf-progs) - $(Q)$(RM) -r -- $(OUTPUT)*.o $(OUTPUT)bpf_jit_disasm $(OUTPUT)bpf_dbg \ - $(OUTPUT)bpf_asm $(OUTPUT)bpf_exp.yacc.* $(OUTPUT)bpf_exp.lex.* - $(call QUIET_CLEAN, core-gen) - $(Q)$(RM) -- $(OUTPUT)FEATURE-DUMP.bpf - $(Q)$(RM) -r -- $(OUTPUT)feature -install: $(PROGS) bpftool_install - $(call QUIET_INSTALL, bpf_jit_disasm) - $(Q)$(INSTALL) -m 0755 -d $(DESTDIR)$(prefix)/bin - $(Q)$(INSTALL) $(OUTPUT)bpf_jit_disasm $(DESTDIR)$(prefix)/bin/bpf_jit_disasm - $(call QUIET_INSTALL, bpf_dbg) - $(Q)$(INSTALL) $(OUTPUT)bpf_dbg $(DESTDIR)$(prefix)/bin/bpf_dbg - $(call QUIET_INSTALL, bpf_asm) - $(Q)$(INSTALL) $(OUTPUT)bpf_asm $(DESTDIR)$(prefix)/bin/bpf_asm +install: bpftool_install bpftool: $(call descend,bpftool) diff --git a/tools/bpf/bpf_asm.c b/tools/bpf/bpf_asm.c deleted file mode 100644 index 0063c3c029e7..000000000000 --- a/tools/bpf/bpf_asm.c +++ /dev/null @@ -1,52 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Minimal BPF assembler - * - * Instead of libpcap high-level filter expressions, it can be quite - * useful to define filters in low-level BPF assembler (that is kept - * close to Steven McCanne and Van Jacobson's original BPF paper). - * In particular for BPF JIT implementors, JIT security auditors, or - * just for defining BPF expressions that contain extensions which are - * not supported by compilers. - * - * How to get into it: - * - * 1) read Documentation/networking/filter.rst - * 2) Run `bpf_asm [-c] ` to translate into binary - * blob that is loadable with xt_bpf, cls_bpf et al. Note: -c will - * pretty print a C-like construct. - * - * Copyright 2013 Daniel Borkmann - */ - -#include -#include -#include - -extern void bpf_asm_compile(FILE *fp, bool cstyle); - -int main(int argc, char **argv) -{ - FILE *fp = stdin; - bool cstyle = false; - int i; - - for (i = 1; i < argc; i++) { - if (!strncmp("-c", argv[i], 2)) { - cstyle = true; - continue; - } - - fp = fopen(argv[i], "r"); - if (!fp) { - fp = stdin; - continue; - } - - break; - } - - bpf_asm_compile(fp, cstyle); - - return 0; -} diff --git a/tools/bpf/bpf_dbg.c b/tools/bpf/bpf_dbg.c deleted file mode 100644 index 00e560a17baf..000000000000 --- a/tools/bpf/bpf_dbg.c +++ /dev/null @@ -1,1398 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Minimal BPF debugger - * - * Minimal BPF debugger that mimics the kernel's engine (w/o extensions) - * and allows for single stepping through selected packets from a pcap - * with a provided user filter in order to facilitate verification of a - * BPF program. Besides others, this is useful to verify BPF programs - * before attaching to a live system, and can be used in socket filters, - * cls_bpf, xt_bpf, team driver and e.g. PTP code; in particular when a - * single more complex BPF program is being used. Reasons for a more - * complex BPF program are likely primarily to optimize execution time - * for making a verdict when multiple simple BPF programs are combined - * into one in order to prevent parsing same headers multiple times. - * - * More on how to debug BPF opcodes see Documentation/networking/filter.rst - * which is the main document on BPF. Mini howto for getting started: - * - * 1) `./bpf_dbg` to enter the shell (shell cmds denoted with '>'): - * 2) > load bpf 6,40 0 0 12,21 0 3 20... (output from `bpf_asm` or - * `tcpdump -iem1 -ddd port 22 | tr '\n' ','` to load as filter) - * 3) > load pcap foo.pcap - * 4) > run /disassemble/dump/quit (self-explanatory) - * 5) > breakpoint 2 (sets bp at loaded BPF insns 2, do `run` then; - * multiple bps can be set, of course, a call to `breakpoint` - * w/o args shows currently loaded bps, `breakpoint reset` for - * resetting all breakpoints) - * 6) > select 3 (`run` etc will start from the 3rd packet in the pcap) - * 7) > step [-, +] (performs single stepping through the BPF) - * - * Copyright 2013 Daniel Borkmann - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#define TCPDUMP_MAGIC 0xa1b2c3d4 - -#define BPF_LDX_B (BPF_LDX | BPF_B) -#define BPF_LDX_W (BPF_LDX | BPF_W) -#define BPF_JMP_JA (BPF_JMP | BPF_JA) -#define BPF_JMP_JEQ (BPF_JMP | BPF_JEQ) -#define BPF_JMP_JGT (BPF_JMP | BPF_JGT) -#define BPF_JMP_JGE (BPF_JMP | BPF_JGE) -#define BPF_JMP_JSET (BPF_JMP | BPF_JSET) -#define BPF_ALU_ADD (BPF_ALU | BPF_ADD) -#define BPF_ALU_SUB (BPF_ALU | BPF_SUB) -#define BPF_ALU_MUL (BPF_ALU | BPF_MUL) -#define BPF_ALU_DIV (BPF_ALU | BPF_DIV) -#define BPF_ALU_MOD (BPF_ALU | BPF_MOD) -#define BPF_ALU_NEG (BPF_ALU | BPF_NEG) -#define BPF_ALU_AND (BPF_ALU | BPF_AND) -#define BPF_ALU_OR (BPF_ALU | BPF_OR) -#define BPF_ALU_XOR (BPF_ALU | BPF_XOR) -#define BPF_ALU_LSH (BPF_ALU | BPF_LSH) -#define BPF_ALU_RSH (BPF_ALU | BPF_RSH) -#define BPF_MISC_TAX (BPF_MISC | BPF_TAX) -#define BPF_MISC_TXA (BPF_MISC | BPF_TXA) -#define BPF_LD_B (BPF_LD | BPF_B) -#define BPF_LD_H (BPF_LD | BPF_H) -#define BPF_LD_W (BPF_LD | BPF_W) - -#ifndef array_size -# define array_size(x) (sizeof(x) / sizeof((x)[0])) -#endif - -#ifndef __check_format_printf -# define __check_format_printf(pos_fmtstr, pos_fmtargs) \ - __attribute__ ((format (printf, (pos_fmtstr), (pos_fmtargs)))) -#endif - -enum { - CMD_OK, - CMD_ERR, - CMD_EX, -}; - -struct shell_cmd { - const char *name; - int (*func)(char *args); -}; - -struct pcap_filehdr { - uint32_t magic; - uint16_t version_major; - uint16_t version_minor; - int32_t thiszone; - uint32_t sigfigs; - uint32_t snaplen; - uint32_t linktype; -}; - -struct pcap_timeval { - int32_t tv_sec; - int32_t tv_usec; -}; - -struct pcap_pkthdr { - struct pcap_timeval ts; - uint32_t caplen; - uint32_t len; -}; - -struct bpf_regs { - uint32_t A; - uint32_t X; - uint32_t M[BPF_MEMWORDS]; - uint32_t R; - bool Rs; - uint16_t Pc; -}; - -static struct sock_filter bpf_image[BPF_MAXINSNS + 1]; -static unsigned int bpf_prog_len; - -static int bpf_breakpoints[64]; -static struct bpf_regs bpf_regs[BPF_MAXINSNS + 1]; -static struct bpf_regs bpf_curr; -static unsigned int bpf_regs_len; - -static int pcap_fd = -1; -static unsigned int pcap_packet; -static size_t pcap_map_size; -static char *pcap_ptr_va_start, *pcap_ptr_va_curr; - -static const char * const op_table[] = { - [BPF_ST] = "st", - [BPF_STX] = "stx", - [BPF_LD_B] = "ldb", - [BPF_LD_H] = "ldh", - [BPF_LD_W] = "ld", - [BPF_LDX] = "ldx", - [BPF_LDX_B] = "ldxb", - [BPF_JMP_JA] = "ja", - [BPF_JMP_JEQ] = "jeq", - [BPF_JMP_JGT] = "jgt", - [BPF_JMP_JGE] = "jge", - [BPF_JMP_JSET] = "jset", - [BPF_ALU_ADD] = "add", - [BPF_ALU_SUB] = "sub", - [BPF_ALU_MUL] = "mul", - [BPF_ALU_DIV] = "div", - [BPF_ALU_MOD] = "mod", - [BPF_ALU_NEG] = "neg", - [BPF_ALU_AND] = "and", - [BPF_ALU_OR] = "or", - [BPF_ALU_XOR] = "xor", - [BPF_ALU_LSH] = "lsh", - [BPF_ALU_RSH] = "rsh", - [BPF_MISC_TAX] = "tax", - [BPF_MISC_TXA] = "txa", - [BPF_RET] = "ret", -}; - -static __check_format_printf(1, 2) int rl_printf(const char *fmt, ...) -{ - int ret; - va_list vl; - - va_start(vl, fmt); - ret = vfprintf(rl_outstream, fmt, vl); - va_end(vl); - - return ret; -} - -static int matches(const char *cmd, const char *pattern) -{ - int len = strlen(cmd); - - if (len > strlen(pattern)) - return -1; - - return memcmp(pattern, cmd, len); -} - -static void hex_dump(const uint8_t *buf, size_t len) -{ - int i; - - rl_printf("%3u: ", 0); - for (i = 0; i < len; i++) { - if (i && !(i % 16)) - rl_printf("\n%3u: ", i); - rl_printf("%02x ", buf[i]); - } - rl_printf("\n"); -} - -static bool bpf_prog_loaded(void) -{ - if (bpf_prog_len == 0) - rl_printf("no bpf program loaded!\n"); - - return bpf_prog_len > 0; -} - -static void bpf_disasm(const struct sock_filter f, unsigned int i) -{ - const char *op, *fmt; - int val = f.k; - char buf[256]; - - switch (f.code) { - case BPF_RET | BPF_K: - op = op_table[BPF_RET]; - fmt = "#%#x"; - break; - case BPF_RET | BPF_A: - op = op_table[BPF_RET]; - fmt = "a"; - break; - case BPF_RET | BPF_X: - op = op_table[BPF_RET]; - fmt = "x"; - break; - case BPF_MISC_TAX: - op = op_table[BPF_MISC_TAX]; - fmt = ""; - break; - case BPF_MISC_TXA: - op = op_table[BPF_MISC_TXA]; - fmt = ""; - break; - case BPF_ST: - op = op_table[BPF_ST]; - fmt = "M[%d]"; - break; - case BPF_STX: - op = op_table[BPF_STX]; - fmt = "M[%d]"; - break; - case BPF_LD_W | BPF_ABS: - op = op_table[BPF_LD_W]; - fmt = "[%d]"; - break; - case BPF_LD_H | BPF_ABS: - op = op_table[BPF_LD_H]; - fmt = "[%d]"; - break; - case BPF_LD_B | BPF_ABS: - op = op_table[BPF_LD_B]; - fmt = "[%d]"; - break; - case BPF_LD_W | BPF_LEN: - op = op_table[BPF_LD_W]; - fmt = "#len"; - break; - case BPF_LD_W | BPF_IND: - op = op_table[BPF_LD_W]; - fmt = "[x+%d]"; - break; - case BPF_LD_H | BPF_IND: - op = op_table[BPF_LD_H]; - fmt = "[x+%d]"; - break; - case BPF_LD_B | BPF_IND: - op = op_table[BPF_LD_B]; - fmt = "[x+%d]"; - break; - case BPF_LD | BPF_IMM: - op = op_table[BPF_LD_W]; - fmt = "#%#x"; - break; - case BPF_LDX | BPF_IMM: - op = op_table[BPF_LDX]; - fmt = "#%#x"; - break; - case BPF_LDX_B | BPF_MSH: - op = op_table[BPF_LDX_B]; - fmt = "4*([%d]&0xf)"; - break; - case BPF_LD | BPF_MEM: - op = op_table[BPF_LD_W]; - fmt = "M[%d]"; - break; - case BPF_LDX | BPF_MEM: - op = op_table[BPF_LDX]; - fmt = "M[%d]"; - break; - case BPF_JMP_JA: - op = op_table[BPF_JMP_JA]; - fmt = "%d"; - val = i + 1 + f.k; - break; - case BPF_JMP_JGT | BPF_X: - op = op_table[BPF_JMP_JGT]; - fmt = "x"; - break; - case BPF_JMP_JGT | BPF_K: - op = op_table[BPF_JMP_JGT]; - fmt = "#%#x"; - break; - case BPF_JMP_JGE | BPF_X: - op = op_table[BPF_JMP_JGE]; - fmt = "x"; - break; - case BPF_JMP_JGE | BPF_K: - op = op_table[BPF_JMP_JGE]; - fmt = "#%#x"; - break; - case BPF_JMP_JEQ | BPF_X: - op = op_table[BPF_JMP_JEQ]; - fmt = "x"; - break; - case BPF_JMP_JEQ | BPF_K: - op = op_table[BPF_JMP_JEQ]; - fmt = "#%#x"; - break; - case BPF_JMP_JSET | BPF_X: - op = op_table[BPF_JMP_JSET]; - fmt = "x"; - break; - case BPF_JMP_JSET | BPF_K: - op = op_table[BPF_JMP_JSET]; - fmt = "#%#x"; - break; - case BPF_ALU_NEG: - op = op_table[BPF_ALU_NEG]; - fmt = ""; - break; - case BPF_ALU_LSH | BPF_X: - op = op_table[BPF_ALU_LSH]; - fmt = "x"; - break; - case BPF_ALU_LSH | BPF_K: - op = op_table[BPF_ALU_LSH]; - fmt = "#%d"; - break; - case BPF_ALU_RSH | BPF_X: - op = op_table[BPF_ALU_RSH]; - fmt = "x"; - break; - case BPF_ALU_RSH | BPF_K: - op = op_table[BPF_ALU_RSH]; - fmt = "#%d"; - break; - case BPF_ALU_ADD | BPF_X: - op = op_table[BPF_ALU_ADD]; - fmt = "x"; - break; - case BPF_ALU_ADD | BPF_K: - op = op_table[BPF_ALU_ADD]; - fmt = "#%d"; - break; - case BPF_ALU_SUB | BPF_X: - op = op_table[BPF_ALU_SUB]; - fmt = "x"; - break; - case BPF_ALU_SUB | BPF_K: - op = op_table[BPF_ALU_SUB]; - fmt = "#%d"; - break; - case BPF_ALU_MUL | BPF_X: - op = op_table[BPF_ALU_MUL]; - fmt = "x"; - break; - case BPF_ALU_MUL | BPF_K: - op = op_table[BPF_ALU_MUL]; - fmt = "#%d"; - break; - case BPF_ALU_DIV | BPF_X: - op = op_table[BPF_ALU_DIV]; - fmt = "x"; - break; - case BPF_ALU_DIV | BPF_K: - op = op_table[BPF_ALU_DIV]; - fmt = "#%d"; - break; - case BPF_ALU_MOD | BPF_X: - op = op_table[BPF_ALU_MOD]; - fmt = "x"; - break; - case BPF_ALU_MOD | BPF_K: - op = op_table[BPF_ALU_MOD]; - fmt = "#%d"; - break; - case BPF_ALU_AND | BPF_X: - op = op_table[BPF_ALU_AND]; - fmt = "x"; - break; - case BPF_ALU_AND | BPF_K: - op = op_table[BPF_ALU_AND]; - fmt = "#%#x"; - break; - case BPF_ALU_OR | BPF_X: - op = op_table[BPF_ALU_OR]; - fmt = "x"; - break; - case BPF_ALU_OR | BPF_K: - op = op_table[BPF_ALU_OR]; - fmt = "#%#x"; - break; - case BPF_ALU_XOR | BPF_X: - op = op_table[BPF_ALU_XOR]; - fmt = "x"; - break; - case BPF_ALU_XOR | BPF_K: - op = op_table[BPF_ALU_XOR]; - fmt = "#%#x"; - break; - default: - op = "nosup"; - fmt = "%#x"; - val = f.code; - break; - } - - memset(buf, 0, sizeof(buf)); - snprintf(buf, sizeof(buf), fmt, val); - buf[sizeof(buf) - 1] = 0; - - if ((BPF_CLASS(f.code) == BPF_JMP && BPF_OP(f.code) != BPF_JA)) - rl_printf("l%d:\t%s %s, l%d, l%d\n", i, op, buf, - i + 1 + f.jt, i + 1 + f.jf); - else - rl_printf("l%d:\t%s %s\n", i, op, buf); -} - -static void bpf_dump_curr(struct bpf_regs *r, struct sock_filter *f) -{ - int i, m = 0; - - rl_printf("pc: [%u]\n", r->Pc); - rl_printf("code: [%u] jt[%u] jf[%u] k[%u]\n", - f->code, f->jt, f->jf, f->k); - rl_printf("curr: "); - bpf_disasm(*f, r->Pc); - - if (f->jt || f->jf) { - rl_printf("jt: "); - bpf_disasm(*(f + f->jt + 1), r->Pc + f->jt + 1); - rl_printf("jf: "); - bpf_disasm(*(f + f->jf + 1), r->Pc + f->jf + 1); - } - - rl_printf("A: [%#08x][%u]\n", r->A, r->A); - rl_printf("X: [%#08x][%u]\n", r->X, r->X); - if (r->Rs) - rl_printf("ret: [%#08x][%u]!\n", r->R, r->R); - - for (i = 0; i < BPF_MEMWORDS; i++) { - if (r->M[i]) { - m++; - rl_printf("M[%d]: [%#08x][%u]\n", i, r->M[i], r->M[i]); - } - } - if (m == 0) - rl_printf("M[0,%d]: [%#08x][%u]\n", BPF_MEMWORDS - 1, 0, 0); -} - -static void bpf_dump_pkt(uint8_t *pkt, uint32_t pkt_caplen, uint32_t pkt_len) -{ - if (pkt_caplen != pkt_len) - rl_printf("cap: %u, len: %u\n", pkt_caplen, pkt_len); - else - rl_printf("len: %u\n", pkt_len); - - hex_dump(pkt, pkt_caplen); -} - -static void bpf_disasm_all(const struct sock_filter *f, unsigned int len) -{ - unsigned int i; - - for (i = 0; i < len; i++) - bpf_disasm(f[i], i); -} - -static void bpf_dump_all(const struct sock_filter *f, unsigned int len) -{ - unsigned int i; - - rl_printf("/* { op, jt, jf, k }, */\n"); - for (i = 0; i < len; i++) - rl_printf("{ %#04x, %2u, %2u, %#010x },\n", - f[i].code, f[i].jt, f[i].jf, f[i].k); -} - -static bool bpf_runnable(struct sock_filter *f, unsigned int len) -{ - int sock, ret, i; - struct sock_fprog bpf = { - .filter = f, - .len = len, - }; - - sock = socket(AF_INET, SOCK_DGRAM, 0); - if (sock < 0) { - rl_printf("cannot open socket!\n"); - return false; - } - ret = setsockopt(sock, SOL_SOCKET, SO_ATTACH_FILTER, &bpf, sizeof(bpf)); - close(sock); - if (ret < 0) { - rl_printf("program not allowed to run by kernel!\n"); - return false; - } - for (i = 0; i < len; i++) { - if (BPF_CLASS(f[i].code) == BPF_LD && - f[i].k > SKF_AD_OFF) { - rl_printf("extensions currently not supported!\n"); - return false; - } - } - - return true; -} - -static void bpf_reset_breakpoints(void) -{ - int i; - - for (i = 0; i < array_size(bpf_breakpoints); i++) - bpf_breakpoints[i] = -1; -} - -static void bpf_set_breakpoints(unsigned int where) -{ - int i; - bool set = false; - - for (i = 0; i < array_size(bpf_breakpoints); i++) { - if (bpf_breakpoints[i] == (int) where) { - rl_printf("breakpoint already set!\n"); - set = true; - break; - } - - if (bpf_breakpoints[i] == -1 && set == false) { - bpf_breakpoints[i] = where; - set = true; - } - } - - if (!set) - rl_printf("too many breakpoints set, reset first!\n"); -} - -static void bpf_dump_breakpoints(void) -{ - int i; - - rl_printf("breakpoints: "); - - for (i = 0; i < array_size(bpf_breakpoints); i++) { - if (bpf_breakpoints[i] < 0) - continue; - rl_printf("%d ", bpf_breakpoints[i]); - } - - rl_printf("\n"); -} - -static void bpf_reset(void) -{ - bpf_regs_len = 0; - - memset(bpf_regs, 0, sizeof(bpf_regs)); - memset(&bpf_curr, 0, sizeof(bpf_curr)); -} - -static void bpf_safe_regs(void) -{ - memcpy(&bpf_regs[bpf_regs_len++], &bpf_curr, sizeof(bpf_curr)); -} - -static bool bpf_restore_regs(int off) -{ - unsigned int index = bpf_regs_len - 1 + off; - - if (index == 0) { - bpf_reset(); - return true; - } else if (index < bpf_regs_len) { - memcpy(&bpf_curr, &bpf_regs[index], sizeof(bpf_curr)); - bpf_regs_len = index; - return true; - } else { - rl_printf("reached bottom of register history stack!\n"); - return false; - } -} - -static uint32_t extract_u32(uint8_t *pkt, uint32_t off) -{ - uint32_t r; - - memcpy(&r, &pkt[off], sizeof(r)); - - return ntohl(r); -} - -static uint16_t extract_u16(uint8_t *pkt, uint32_t off) -{ - uint16_t r; - - memcpy(&r, &pkt[off], sizeof(r)); - - return ntohs(r); -} - -static uint8_t extract_u8(uint8_t *pkt, uint32_t off) -{ - return pkt[off]; -} - -static void set_return(struct bpf_regs *r) -{ - r->R = 0; - r->Rs = true; -} - -static void bpf_single_step(struct bpf_regs *r, struct sock_filter *f, - uint8_t *pkt, uint32_t pkt_caplen, - uint32_t pkt_len) -{ - uint32_t K = f->k; - int d; - - switch (f->code) { - case BPF_RET | BPF_K: - r->R = K; - r->Rs = true; - break; - case BPF_RET | BPF_A: - r->R = r->A; - r->Rs = true; - break; - case BPF_RET | BPF_X: - r->R = r->X; - r->Rs = true; - break; - case BPF_MISC_TAX: - r->X = r->A; - break; - case BPF_MISC_TXA: - r->A = r->X; - break; - case BPF_ST: - r->M[K] = r->A; - break; - case BPF_STX: - r->M[K] = r->X; - break; - case BPF_LD_W | BPF_ABS: - d = pkt_caplen - K; - if (d >= sizeof(uint32_t)) - r->A = extract_u32(pkt, K); - else - set_return(r); - break; - case BPF_LD_H | BPF_ABS: - d = pkt_caplen - K; - if (d >= sizeof(uint16_t)) - r->A = extract_u16(pkt, K); - else - set_return(r); - break; - case BPF_LD_B | BPF_ABS: - d = pkt_caplen - K; - if (d >= sizeof(uint8_t)) - r->A = extract_u8(pkt, K); - else - set_return(r); - break; - case BPF_LD_W | BPF_IND: - d = pkt_caplen - (r->X + K); - if (d >= sizeof(uint32_t)) - r->A = extract_u32(pkt, r->X + K); - break; - case BPF_LD_H | BPF_IND: - d = pkt_caplen - (r->X + K); - if (d >= sizeof(uint16_t)) - r->A = extract_u16(pkt, r->X + K); - else - set_return(r); - break; - case BPF_LD_B | BPF_IND: - d = pkt_caplen - (r->X + K); - if (d >= sizeof(uint8_t)) - r->A = extract_u8(pkt, r->X + K); - else - set_return(r); - break; - case BPF_LDX_B | BPF_MSH: - d = pkt_caplen - K; - if (d >= sizeof(uint8_t)) { - r->X = extract_u8(pkt, K); - r->X = (r->X & 0xf) << 2; - } else - set_return(r); - break; - case BPF_LD_W | BPF_LEN: - r->A = pkt_len; - break; - case BPF_LDX_W | BPF_LEN: - r->A = pkt_len; - break; - case BPF_LD | BPF_IMM: - r->A = K; - break; - case BPF_LDX | BPF_IMM: - r->X = K; - break; - case BPF_LD | BPF_MEM: - r->A = r->M[K]; - break; - case BPF_LDX | BPF_MEM: - r->X = r->M[K]; - break; - case BPF_JMP_JA: - r->Pc += K; - break; - case BPF_JMP_JGT | BPF_X: - r->Pc += r->A > r->X ? f->jt : f->jf; - break; - case BPF_JMP_JGT | BPF_K: - r->Pc += r->A > K ? f->jt : f->jf; - break; - case BPF_JMP_JGE | BPF_X: - r->Pc += r->A >= r->X ? f->jt : f->jf; - break; - case BPF_JMP_JGE | BPF_K: - r->Pc += r->A >= K ? f->jt : f->jf; - break; - case BPF_JMP_JEQ | BPF_X: - r->Pc += r->A == r->X ? f->jt : f->jf; - break; - case BPF_JMP_JEQ | BPF_K: - r->Pc += r->A == K ? f->jt : f->jf; - break; - case BPF_JMP_JSET | BPF_X: - r->Pc += r->A & r->X ? f->jt : f->jf; - break; - case BPF_JMP_JSET | BPF_K: - r->Pc += r->A & K ? f->jt : f->jf; - break; - case BPF_ALU_NEG: - r->A = -r->A; - break; - case BPF_ALU_LSH | BPF_X: - r->A <<= r->X; - break; - case BPF_ALU_LSH | BPF_K: - r->A <<= K; - break; - case BPF_ALU_RSH | BPF_X: - r->A >>= r->X; - break; - case BPF_ALU_RSH | BPF_K: - r->A >>= K; - break; - case BPF_ALU_ADD | BPF_X: - r->A += r->X; - break; - case BPF_ALU_ADD | BPF_K: - r->A += K; - break; - case BPF_ALU_SUB | BPF_X: - r->A -= r->X; - break; - case BPF_ALU_SUB | BPF_K: - r->A -= K; - break; - case BPF_ALU_MUL | BPF_X: - r->A *= r->X; - break; - case BPF_ALU_MUL | BPF_K: - r->A *= K; - break; - case BPF_ALU_DIV | BPF_X: - case BPF_ALU_MOD | BPF_X: - if (r->X == 0) { - set_return(r); - break; - } - goto do_div; - case BPF_ALU_DIV | BPF_K: - case BPF_ALU_MOD | BPF_K: - if (K == 0) { - set_return(r); - break; - } -do_div: - switch (f->code) { - case BPF_ALU_DIV | BPF_X: - r->A /= r->X; - break; - case BPF_ALU_DIV | BPF_K: - r->A /= K; - break; - case BPF_ALU_MOD | BPF_X: - r->A %= r->X; - break; - case BPF_ALU_MOD | BPF_K: - r->A %= K; - break; - } - break; - case BPF_ALU_AND | BPF_X: - r->A &= r->X; - break; - case BPF_ALU_AND | BPF_K: - r->A &= K; - break; - case BPF_ALU_OR | BPF_X: - r->A |= r->X; - break; - case BPF_ALU_OR | BPF_K: - r->A |= K; - break; - case BPF_ALU_XOR | BPF_X: - r->A ^= r->X; - break; - case BPF_ALU_XOR | BPF_K: - r->A ^= K; - break; - } -} - -static bool bpf_pc_has_breakpoint(uint16_t pc) -{ - int i; - - for (i = 0; i < array_size(bpf_breakpoints); i++) { - if (bpf_breakpoints[i] < 0) - continue; - if (bpf_breakpoints[i] == pc) - return true; - } - - return false; -} - -static bool bpf_handle_breakpoint(struct bpf_regs *r, struct sock_filter *f, - uint8_t *pkt, uint32_t pkt_caplen, - uint32_t pkt_len) -{ - rl_printf("-- register dump --\n"); - bpf_dump_curr(r, &f[r->Pc]); - rl_printf("-- packet dump --\n"); - bpf_dump_pkt(pkt, pkt_caplen, pkt_len); - rl_printf("(breakpoint)\n"); - return true; -} - -static int bpf_run_all(struct sock_filter *f, uint16_t bpf_len, uint8_t *pkt, - uint32_t pkt_caplen, uint32_t pkt_len) -{ - bool stop = false; - - while (bpf_curr.Rs == false && stop == false) { - bpf_safe_regs(); - - if (bpf_pc_has_breakpoint(bpf_curr.Pc)) - stop = bpf_handle_breakpoint(&bpf_curr, f, pkt, - pkt_caplen, pkt_len); - - bpf_single_step(&bpf_curr, &f[bpf_curr.Pc], pkt, pkt_caplen, - pkt_len); - bpf_curr.Pc++; - } - - return stop ? -1 : bpf_curr.R; -} - -static int bpf_run_stepping(struct sock_filter *f, uint16_t bpf_len, - uint8_t *pkt, uint32_t pkt_caplen, - uint32_t pkt_len, int next) -{ - bool stop = false; - int i = 1; - - while (!bpf_curr.Rs && !stop) { - bpf_safe_regs(); - - if (i++ == next) - stop = bpf_handle_breakpoint(&bpf_curr, f, pkt, - pkt_caplen, pkt_len); - - bpf_single_step(&bpf_curr, &f[bpf_curr.Pc], pkt, pkt_caplen, - pkt_len); - bpf_curr.Pc++; - } - - return stop ? -1 : bpf_curr.R; -} - -static bool pcap_loaded(void) -{ - if (pcap_fd < 0) - rl_printf("no pcap file loaded!\n"); - - return pcap_fd >= 0; -} - -static struct pcap_pkthdr *pcap_curr_pkt(void) -{ - return (void *) pcap_ptr_va_curr; -} - -static bool pcap_next_pkt(void) -{ - struct pcap_pkthdr *hdr = pcap_curr_pkt(); - - if (pcap_ptr_va_curr + sizeof(*hdr) - - pcap_ptr_va_start >= pcap_map_size) - return false; - if (hdr->caplen == 0 || hdr->len == 0 || hdr->caplen > hdr->len) - return false; - if (pcap_ptr_va_curr + sizeof(*hdr) + hdr->caplen - - pcap_ptr_va_start >= pcap_map_size) - return false; - - pcap_ptr_va_curr += (sizeof(*hdr) + hdr->caplen); - return true; -} - -static void pcap_reset_pkt(void) -{ - pcap_ptr_va_curr = pcap_ptr_va_start + sizeof(struct pcap_filehdr); -} - -static int try_load_pcap(const char *file) -{ - struct pcap_filehdr *hdr; - struct stat sb; - int ret; - - pcap_fd = open(file, O_RDONLY); - if (pcap_fd < 0) { - rl_printf("cannot open pcap [%s]!\n", strerror(errno)); - return CMD_ERR; - } - - ret = fstat(pcap_fd, &sb); - if (ret < 0) { - rl_printf("cannot fstat pcap file!\n"); - return CMD_ERR; - } - - if (!S_ISREG(sb.st_mode)) { - rl_printf("not a regular pcap file, duh!\n"); - return CMD_ERR; - } - - pcap_map_size = sb.st_size; - if (pcap_map_size <= sizeof(struct pcap_filehdr)) { - rl_printf("pcap file too small!\n"); - return CMD_ERR; - } - - pcap_ptr_va_start = mmap(NULL, pcap_map_size, PROT_READ, - MAP_SHARED | MAP_LOCKED, pcap_fd, 0); - if (pcap_ptr_va_start == MAP_FAILED) { - rl_printf("mmap of file failed!"); - return CMD_ERR; - } - - hdr = (void *) pcap_ptr_va_start; - if (hdr->magic != TCPDUMP_MAGIC) { - rl_printf("wrong pcap magic!\n"); - return CMD_ERR; - } - - pcap_reset_pkt(); - - return CMD_OK; - -} - -static void try_close_pcap(void) -{ - if (pcap_fd >= 0) { - munmap(pcap_ptr_va_start, pcap_map_size); - close(pcap_fd); - - pcap_ptr_va_start = pcap_ptr_va_curr = NULL; - pcap_map_size = 0; - pcap_packet = 0; - pcap_fd = -1; - } -} - -static int cmd_load_bpf(char *bpf_string) -{ - char sp, *token, separator = ','; - unsigned short bpf_len, i = 0; - struct sock_filter tmp; - - bpf_prog_len = 0; - memset(bpf_image, 0, sizeof(bpf_image)); - - if (sscanf(bpf_string, "%hu%c", &bpf_len, &sp) != 2 || - sp != separator || bpf_len > BPF_MAXINSNS || bpf_len == 0) { - rl_printf("syntax error in head length encoding!\n"); - return CMD_ERR; - } - - token = bpf_string; - while ((token = strchr(token, separator)) && (++token)[0]) { - if (i >= bpf_len) { - rl_printf("program exceeds encoded length!\n"); - return CMD_ERR; - } - - if (sscanf(token, "%hu %hhu %hhu %u,", - &tmp.code, &tmp.jt, &tmp.jf, &tmp.k) != 4) { - rl_printf("syntax error at instruction %d!\n", i); - return CMD_ERR; - } - - bpf_image[i].code = tmp.code; - bpf_image[i].jt = tmp.jt; - bpf_image[i].jf = tmp.jf; - bpf_image[i].k = tmp.k; - - i++; - } - - if (i != bpf_len) { - rl_printf("syntax error exceeding encoded length!\n"); - return CMD_ERR; - } else - bpf_prog_len = bpf_len; - if (!bpf_runnable(bpf_image, bpf_prog_len)) - bpf_prog_len = 0; - - return CMD_OK; -} - -static int cmd_load_pcap(char *file) -{ - char *file_trim, *tmp; - - file_trim = strtok_r(file, " ", &tmp); - if (file_trim == NULL) - return CMD_ERR; - - try_close_pcap(); - - return try_load_pcap(file_trim); -} - -static int cmd_load(char *arg) -{ - char *subcmd, *cont = NULL, *tmp = strdup(arg); - int ret = CMD_OK; - - subcmd = strtok_r(tmp, " ", &cont); - if (subcmd == NULL) - goto out; - if (matches(subcmd, "bpf") == 0) { - bpf_reset(); - bpf_reset_breakpoints(); - - if (!cont) - ret = CMD_ERR; - else - ret = cmd_load_bpf(cont); - } else if (matches(subcmd, "pcap") == 0) { - ret = cmd_load_pcap(cont); - } else { -out: - rl_printf("bpf : load bpf code\n"); - rl_printf("pcap : load pcap file\n"); - ret = CMD_ERR; - } - - free(tmp); - return ret; -} - -static int cmd_step(char *num) -{ - struct pcap_pkthdr *hdr; - int steps, ret; - - if (!bpf_prog_loaded() || !pcap_loaded()) - return CMD_ERR; - - steps = strtol(num, NULL, 10); - if (steps == 0 || strlen(num) == 0) - steps = 1; - if (steps < 0) { - if (!bpf_restore_regs(steps)) - return CMD_ERR; - steps = 1; - } - - hdr = pcap_curr_pkt(); - ret = bpf_run_stepping(bpf_image, bpf_prog_len, - (uint8_t *) hdr + sizeof(*hdr), - hdr->caplen, hdr->len, steps); - if (ret >= 0 || bpf_curr.Rs) { - bpf_reset(); - if (!pcap_next_pkt()) { - rl_printf("(going back to first packet)\n"); - pcap_reset_pkt(); - } else { - rl_printf("(next packet)\n"); - } - } - - return CMD_OK; -} - -static int cmd_select(char *num) -{ - unsigned int which, i; - bool have_next = true; - - if (!pcap_loaded() || strlen(num) == 0) - return CMD_ERR; - - which = strtoul(num, NULL, 10); - if (which == 0) { - rl_printf("packet count starts with 1, clamping!\n"); - which = 1; - } - - pcap_reset_pkt(); - bpf_reset(); - - for (i = 0; i < which && (have_next = pcap_next_pkt()); i++) - /* noop */; - if (!have_next || pcap_curr_pkt() == NULL) { - rl_printf("no packet #%u available!\n", which); - pcap_reset_pkt(); - return CMD_ERR; - } - - return CMD_OK; -} - -static int cmd_breakpoint(char *subcmd) -{ - if (!bpf_prog_loaded()) - return CMD_ERR; - if (strlen(subcmd) == 0) - bpf_dump_breakpoints(); - else if (matches(subcmd, "reset") == 0) - bpf_reset_breakpoints(); - else { - unsigned int where = strtoul(subcmd, NULL, 10); - - if (where < bpf_prog_len) { - bpf_set_breakpoints(where); - rl_printf("breakpoint at: "); - bpf_disasm(bpf_image[where], where); - } - } - - return CMD_OK; -} - -static int cmd_run(char *num) -{ - static uint32_t pass, fail; - bool has_limit = true; - int pkts = 0, i = 0; - - if (!bpf_prog_loaded() || !pcap_loaded()) - return CMD_ERR; - - pkts = strtol(num, NULL, 10); - if (pkts == 0 || strlen(num) == 0) - has_limit = false; - - do { - struct pcap_pkthdr *hdr = pcap_curr_pkt(); - int ret = bpf_run_all(bpf_image, bpf_prog_len, - (uint8_t *) hdr + sizeof(*hdr), - hdr->caplen, hdr->len); - if (ret > 0) - pass++; - else if (ret == 0) - fail++; - else - return CMD_OK; - bpf_reset(); - } while (pcap_next_pkt() && (!has_limit || (++i < pkts))); - - rl_printf("bpf passes:%u fails:%u\n", pass, fail); - - pcap_reset_pkt(); - bpf_reset(); - - pass = fail = 0; - return CMD_OK; -} - -static int cmd_disassemble(char *line_string) -{ - bool single_line = false; - unsigned long line; - - if (!bpf_prog_loaded()) - return CMD_ERR; - if (strlen(line_string) > 0 && - (line = strtoul(line_string, NULL, 10)) < bpf_prog_len) - single_line = true; - if (single_line) - bpf_disasm(bpf_image[line], line); - else - bpf_disasm_all(bpf_image, bpf_prog_len); - - return CMD_OK; -} - -static int cmd_dump(char *dontcare) -{ - if (!bpf_prog_loaded()) - return CMD_ERR; - - bpf_dump_all(bpf_image, bpf_prog_len); - - return CMD_OK; -} - -static int cmd_quit(char *dontcare) -{ - return CMD_EX; -} - -static const struct shell_cmd cmds[] = { - { .name = "load", .func = cmd_load }, - { .name = "select", .func = cmd_select }, - { .name = "step", .func = cmd_step }, - { .name = "run", .func = cmd_run }, - { .name = "breakpoint", .func = cmd_breakpoint }, - { .name = "disassemble", .func = cmd_disassemble }, - { .name = "dump", .func = cmd_dump }, - { .name = "quit", .func = cmd_quit }, -}; - -static int execf(char *arg) -{ - char *cmd, *cont, *tmp = strdup(arg); - int i, ret = 0, len; - - cmd = strtok_r(tmp, " ", &cont); - if (cmd == NULL) - goto out; - len = strlen(cmd); - for (i = 0; i < array_size(cmds); i++) { - if (len != strlen(cmds[i].name)) - continue; - if (strncmp(cmds[i].name, cmd, len) == 0) { - ret = cmds[i].func(cont); - break; - } - } -out: - free(tmp); - return ret; -} - -static char *shell_comp_gen(const char *buf, int state) -{ - static int list_index, len; - - if (!state) { - list_index = 0; - len = strlen(buf); - } - - for (; list_index < array_size(cmds); ) { - const char *name = cmds[list_index].name; - - list_index++; - if (strncmp(name, buf, len) == 0) - return strdup(name); - } - - return NULL; -} - -static char **shell_completion(const char *buf, int start, int end) -{ - char **matches = NULL; - - if (start == 0) - matches = rl_completion_matches(buf, shell_comp_gen); - - return matches; -} - -static void intr_shell(int sig) -{ - if (rl_end) - rl_kill_line(-1, 0); - - rl_crlf(); - rl_refresh_line(0, 0); - rl_free_line_state(); -} - -static void init_shell(FILE *fin, FILE *fout) -{ - char file[128]; - - snprintf(file, sizeof(file), "%s/.bpf_dbg_history", getenv("HOME")); - read_history(file); - - rl_instream = fin; - rl_outstream = fout; - - rl_readline_name = "bpf_dbg"; - rl_terminal_name = getenv("TERM"); - - rl_catch_signals = 0; - rl_catch_sigwinch = 1; - - rl_attempted_completion_function = shell_completion; - - rl_bind_key('\t', rl_complete); - - rl_bind_key_in_map('\t', rl_complete, emacs_meta_keymap); - rl_bind_key_in_map('\033', rl_complete, emacs_meta_keymap); - - snprintf(file, sizeof(file), "%s/.bpf_dbg_init", getenv("HOME")); - rl_read_init_file(file); - - rl_prep_terminal(0); - rl_set_signals(); - - signal(SIGINT, intr_shell); -} - -static void exit_shell(FILE *fin, FILE *fout) -{ - char file[128]; - - snprintf(file, sizeof(file), "%s/.bpf_dbg_history", getenv("HOME")); - write_history(file); - - clear_history(); - rl_deprep_terminal(); - - try_close_pcap(); - - if (fin != stdin) - fclose(fin); - if (fout != stdout) - fclose(fout); -} - -static int run_shell_loop(FILE *fin, FILE *fout) -{ - char *buf; - - init_shell(fin, fout); - - while ((buf = readline("> ")) != NULL) { - int ret = execf(buf); - if (ret == CMD_EX) - break; - if (ret == CMD_OK && strlen(buf) > 0) - add_history(buf); - - free(buf); - } - - exit_shell(fin, fout); - return 0; -} - -int main(int argc, char **argv) -{ - FILE *fin = NULL, *fout = NULL; - - if (argc >= 2) - fin = fopen(argv[1], "r"); - if (argc >= 3) - fout = fopen(argv[2], "w"); - - return run_shell_loop(fin ? : stdin, fout ? : stdout); -} diff --git a/tools/bpf/bpf_exp.l b/tools/bpf/bpf_exp.l deleted file mode 100644 index 4da8d053d68f..000000000000 --- a/tools/bpf/bpf_exp.l +++ /dev/null @@ -1,198 +0,0 @@ -/* - * BPF asm code lexer - * - * This program is free software; you can distribute it and/or modify - * it under the terms of the GNU General Public License as published - * by the Free Software Foundation; either version 2 of the License, - * or (at your option) any later version. - * - * Syntax kept close to: - * - * Steven McCanne and Van Jacobson. 1993. The BSD packet filter: a new - * architecture for user-level packet capture. In Proceedings of the - * USENIX Winter 1993 Conference Proceedings on USENIX Winter 1993 - * Conference Proceedings (USENIX'93). USENIX Association, Berkeley, - * CA, USA, 2-2. - * - * Copyright 2013 Daniel Borkmann - * Licensed under the GNU General Public License, version 2.0 (GPLv2) - */ - -%{ - -#include -#include -#include -#include - -#include - -#include "bpf_exp.yacc.h" - -extern void yyerror(const char *str); - -%} - -%option align -%option ecs - -%option nounput -%option noreject -%option noinput -%option noyywrap - -%option 8bit -%option caseless -%option yylineno - -%% - -"ldb" { return OP_LDB; } -"ldh" { return OP_LDH; } -"ld" { return OP_LD; } -"ldi" { return OP_LDI; } -"ldx" { return OP_LDX; } -"ldxi" { return OP_LDXI; } -"ldxb" { return OP_LDXB; } -"st" { return OP_ST; } -"stx" { return OP_STX; } -"jmp" { return OP_JMP; } -"ja" { return OP_JMP; } -"jeq" { return OP_JEQ; } -"jneq" { return OP_JNEQ; } -"jne" { return OP_JNEQ; } -"jlt" { return OP_JLT; } -"jle" { return OP_JLE; } -"jgt" { return OP_JGT; } -"jge" { return OP_JGE; } -"jset" { return OP_JSET; } -"add" { return OP_ADD; } -"sub" { return OP_SUB; } -"mul" { return OP_MUL; } -"div" { return OP_DIV; } -"mod" { return OP_MOD; } -"neg" { return OP_NEG; } -"and" { return OP_AND; } -"xor" { return OP_XOR; } -"or" { return OP_OR; } -"lsh" { return OP_LSH; } -"rsh" { return OP_RSH; } -"ret" { return OP_RET; } -"tax" { return OP_TAX; } -"txa" { return OP_TXA; } - -"#"?("len") { return K_PKT_LEN; } - -"#"?("proto") { - yylval.number = SKF_AD_PROTOCOL; - return extension; - } -"#"?("type") { - yylval.number = SKF_AD_PKTTYPE; - return extension; - } -"#"?("poff") { - yylval.number = SKF_AD_PAY_OFFSET; - return extension; - } -"#"?("ifidx") { - yylval.number = SKF_AD_IFINDEX; - return extension; - } -"#"?("nla") { - yylval.number = SKF_AD_NLATTR; - return extension; - } -"#"?("nlan") { - yylval.number = SKF_AD_NLATTR_NEST; - return extension; - } -"#"?("mark") { - yylval.number = SKF_AD_MARK; - return extension; - } -"#"?("queue") { - yylval.number = SKF_AD_QUEUE; - return extension; - } -"#"?("hatype") { - yylval.number = SKF_AD_HATYPE; - return extension; - } -"#"?("rxhash") { - yylval.number = SKF_AD_RXHASH; - return extension; - } -"#"?("cpu") { - yylval.number = SKF_AD_CPU; - return extension; - } -"#"?("vlan_tci") { - yylval.number = SKF_AD_VLAN_TAG; - return extension; - } -"#"?("vlan_pr") { - yylval.number = SKF_AD_VLAN_TAG_PRESENT; - return extension; - } -"#"?("vlan_avail") { - yylval.number = SKF_AD_VLAN_TAG_PRESENT; - return extension; - } -"#"?("vlan_tpid") { - yylval.number = SKF_AD_VLAN_TPID; - return extension; - } -"#"?("rand") { - yylval.number = SKF_AD_RANDOM; - return extension; - } - -":" { return ':'; } -"," { return ','; } -"#" { return '#'; } -"%" { return '%'; } -"[" { return '['; } -"]" { return ']'; } -"(" { return '('; } -")" { return ')'; } -"x" { return 'x'; } -"a" { return 'a'; } -"+" { return '+'; } -"M" { return 'M'; } -"*" { return '*'; } -"&" { return '&'; } - -([0][x][a-fA-F0-9]+) { - yylval.number = strtoul(yytext, NULL, 16); - return number; - } -([0][b][0-1]+) { - yylval.number = strtol(yytext + 2, NULL, 2); - return number; - } -(([0])|([-+]?[1-9][0-9]*)) { - yylval.number = strtol(yytext, NULL, 10); - return number; - } -([0][0-7]+) { - yylval.number = strtol(yytext + 1, NULL, 8); - return number; - } -[a-zA-Z_][a-zA-Z0-9_]+ { - yylval.label = strdup(yytext); - return label; - } - -"/*"([^\*]|\*[^/])*"*/" { /* NOP */ } -";"[^\n]* { /* NOP */ } -^#.* { /* NOP */ } -[ \t]+ { /* NOP */ } -[ \n]+ { /* NOP */ } - -. { - printf("unknown character \'%s\'", yytext); - yyerror("lex unknown character"); - } - -%% diff --git a/tools/bpf/bpf_exp.y b/tools/bpf/bpf_exp.y deleted file mode 100644 index dfb7254a24e8..000000000000 --- a/tools/bpf/bpf_exp.y +++ /dev/null @@ -1,668 +0,0 @@ -/* - * BPF asm code parser - * - * This program is free software; you can distribute it and/or modify - * it under the terms of the GNU General Public License as published - * by the Free Software Foundation; either version 2 of the License, - * or (at your option) any later version. - * - * Syntax kept close to: - * - * Steven McCanne and Van Jacobson. 1993. The BSD packet filter: a new - * architecture for user-level packet capture. In Proceedings of the - * USENIX Winter 1993 Conference Proceedings on USENIX Winter 1993 - * Conference Proceedings (USENIX'93). USENIX Association, Berkeley, - * CA, USA, 2-2. - * - * Copyright 2013 Daniel Borkmann - * Licensed under the GNU General Public License, version 2.0 (GPLv2) - */ - -%{ - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "bpf_exp.yacc.h" - -enum jmp_type { JTL, JFL, JKL }; - -extern FILE *yyin; -extern int yylineno; -extern int yylex(void); -extern void yyerror(const char *str); - -extern void bpf_asm_compile(FILE *fp, bool cstyle); -static void bpf_set_curr_instr(uint16_t op, uint8_t jt, uint8_t jf, uint32_t k); -static void bpf_set_curr_label(char *label); -static void bpf_set_jmp_label(char *label, enum jmp_type type); - -%} - -%union { - char *label; - uint32_t number; -} - -%token OP_LDB OP_LDH OP_LD OP_LDX OP_ST OP_STX OP_JMP OP_JEQ OP_JGT OP_JGE -%token OP_JSET OP_ADD OP_SUB OP_MUL OP_DIV OP_AND OP_OR OP_XOR OP_LSH OP_RSH -%token OP_RET OP_TAX OP_TXA OP_LDXB OP_MOD OP_NEG OP_JNEQ OP_JLT OP_JLE OP_LDI -%token OP_LDXI - -%token K_PKT_LEN - -%token ':' ',' '[' ']' '(' ')' 'x' 'a' '+' 'M' '*' '&' '#' '%' - -%token extension number label - -%type