From ba3b86b9cef0c72ae78173f2c4db8a08bf4d3770 Mon Sep 17 00:00:00 2001 From: Ilya Leoshkevich Date: Fri, 5 Mar 2021 00:30:02 +0100 Subject: s390/bpf: Implement new atomic ops Implement BPF_AND, BPF_OR and BPF_XOR as the existing BPF_ADD. Since the corresponding machine instructions return the old value, BPF_FETCH happens by itself, the only additional thing that is required is zero-extension. There is no single instruction that implements BPF_XCHG on s390, so use a COMPARE AND SWAP loop. BPF_CMPXCHG, on the other hand, can be implemented by a single COMPARE AND SWAP. Zero-extension is automatically inserted by the verifier. Signed-off-by: Ilya Leoshkevich Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20210304233002.149096-1-iii@linux.ibm.com --- arch/s390/net/bpf_jit_comp.c | 64 +++++++++++++++++++++++++++++++++++++------- 1 file changed, 55 insertions(+), 9 deletions(-) (limited to 'arch/s390/net') diff --git a/arch/s390/net/bpf_jit_comp.c b/arch/s390/net/bpf_jit_comp.c index f973e2ead197..63cae0476bb4 100644 --- a/arch/s390/net/bpf_jit_comp.c +++ b/arch/s390/net/bpf_jit_comp.c @@ -1209,21 +1209,67 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, */ case BPF_STX | BPF_ATOMIC | BPF_DW: case BPF_STX | BPF_ATOMIC | BPF_W: - if (insn->imm != BPF_ADD) { + { + bool is32 = BPF_SIZE(insn->code) == BPF_W; + + switch (insn->imm) { +/* {op32|op64} {%w0|%src},%src,off(%dst) */ +#define EMIT_ATOMIC(op32, op64) do { \ + EMIT6_DISP_LH(0xeb000000, is32 ? (op32) : (op64), \ + (insn->imm & BPF_FETCH) ? src_reg : REG_W0, \ + src_reg, dst_reg, off); \ + if (is32 && (insn->imm & BPF_FETCH)) \ + EMIT_ZERO(src_reg); \ +} while (0) + case BPF_ADD: + case BPF_ADD | BPF_FETCH: + /* {laal|laalg} */ + EMIT_ATOMIC(0x00fa, 0x00ea); + break; + case BPF_AND: + case BPF_AND | BPF_FETCH: + /* {lan|lang} */ + EMIT_ATOMIC(0x00f4, 0x00e4); + break; + case BPF_OR: + case BPF_OR | BPF_FETCH: + /* {lao|laog} */ + EMIT_ATOMIC(0x00f6, 0x00e6); + break; + case BPF_XOR: + case BPF_XOR | BPF_FETCH: + /* {lax|laxg} */ + EMIT_ATOMIC(0x00f7, 0x00e7); + break; +#undef EMIT_ATOMIC + case BPF_XCHG: + /* {ly|lg} %w0,off(%dst) */ + EMIT6_DISP_LH(0xe3000000, + is32 ? 0x0058 : 0x0004, REG_W0, REG_0, + dst_reg, off); + /* 0: {csy|csg} %w0,%src,off(%dst) */ + EMIT6_DISP_LH(0xeb000000, is32 ? 0x0014 : 0x0030, + REG_W0, src_reg, dst_reg, off); + /* brc 4,0b */ + EMIT4_PCREL_RIC(0xa7040000, 4, jit->prg - 6); + /* {llgfr|lgr} %src,%w0 */ + EMIT4(is32 ? 0xb9160000 : 0xb9040000, src_reg, REG_W0); + if (is32 && insn_is_zext(&insn[1])) + insn_count = 2; + break; + case BPF_CMPXCHG: + /* 0: {csy|csg} %b0,%src,off(%dst) */ + EMIT6_DISP_LH(0xeb000000, is32 ? 0x0014 : 0x0030, + BPF_REG_0, src_reg, dst_reg, off); + break; + default: pr_err("Unknown atomic operation %02x\n", insn->imm); return -1; } - /* *(u32/u64 *)(dst + off) += src - * - * BFW_W: laal %w0,%src,off(%dst) - * BPF_DW: laalg %w0,%src,off(%dst) - */ - EMIT6_DISP_LH(0xeb000000, - BPF_SIZE(insn->code) == BPF_W ? 0x00fa : 0x00ea, - REG_W0, src_reg, dst_reg, off); jit->seen |= SEEN_MEM; break; + } /* * BPF_LDX */ -- cgit v1.2.3