From 7b128ae8c56b1055a93573004148a98465d79857 Mon Sep 17 00:00:00 2001 From: Mizuki Asakura Date: Sun, 17 Apr 2016 20:16:12 +0900 Subject: [PATCH] [mod] added aarch64 bilinear implementations (ver.4.1) Since aarch64 has different neon syntax from aarch32 and has no support for (older) arm-simd, there are no SIMD accelerations for pixman on aarch64. We need new implementations. This patch also contains Ben Avions's series of patches for aarch32 and now the benchmark results are fine to aarch64. Please find the result at the below ticket. Added: https://bugs.freedesktop.org/show_bug.cgi?id=94758 Signed-off-by: Mizuki Asakura --- configure.ac | 36 +- pixman/Makefile.am | 17 +- pixman/pixman-arm-neon.c | 23 +- pixman/pixman-arm.c | 6 + pixman/pixman-arma64-neon-asm-bilinear.S | 1275 ++++++++++ pixman/pixman-arma64-neon-asm.S | 3704 ++++++++++++++++++++++++++++++ pixman/pixman-arma64-neon-asm.h | 1310 +++++++++++ pixman/pixman-private.h | 7 +- 8 files changed, 6374 insertions(+), 4 deletions(-) create mode 100644 pixman/pixman-arma64-neon-asm-bilinear.S create mode 100644 pixman/pixman-arma64-neon-asm.S create mode 100644 pixman/pixman-arma64-neon-asm.h diff --git a/configure.ac b/configure.ac index 6b2134e..26203a8 --- a/configure.ac +++ b/configure.ac @@ -667,6 +667,40 @@ if test $enable_arm_neon = yes && test $have_arm_neon = no ; then AC_MSG_ERROR([ARM NEON intrinsics not detected]) fi +dnl ========================================================================== +dnl Check if assembler is gas compatible and supports ARM-a64 NEON instructions +have_arm_a64_neon=no +AC_MSG_CHECKING(whether to use ARM A64 NEON assembler) +xserver_save_CFLAGS=$CFLAGS +CFLAGS="-x assembler-with-cpp $CFLAGS" +AC_COMPILE_IFELSE([AC_LANG_SOURCE([[ +.text +.arch armv8-a +.altmacro +prfm pldl2strm, [x0] +xtn v0.8b, v0.8h]])], have_arm_a64_neon=yes) +CFLAGS=$xserver_save_CFLAGS + +AC_ARG_ENABLE(arm-a64-neon, + [AC_HELP_STRING([--disable-arm-a64-neon], + [disable ARM A64 NEON fast paths])], + [enable_arm_a64_neon=$enableval], [enable_arm_a64_neon=auto]) + +if test $enable_arm_a64_neon = no ; then + have_arm_a64_neon=disabled +fi + +if test $have_arm_a64_neon = yes ; then + AC_DEFINE(USE_ARM_A64_NEON, 1, [use ARM A64_NEON assembly optimizations]) +fi + +AM_CONDITIONAL(USE_ARM_A64_NEON, test $have_arm_a64_neon = yes) + +AC_MSG_RESULT($have_arm_a64_neon) +if test $enable_arm_a64_neon = yes && test $have_arm_a64_neon4 = no ; then + AC_MSG_ERROR([ARM A64 NEON intrinsics not detected]) +fi + dnl =========================================================================== dnl Check for IWMMXT diff --git a/pixman/Makefile.am b/pixman/Makefile.am index 581b6f6..f1afa27 --- a/pixman/Makefile.am +++ b/pixman/Makefile.am @@ -94,6 +94,21 @@ libpixman_1_la_LIBADD += libpixman-arm-neon.la ASM_CFLAGS_arm_neon= endif +# arm a64 neon code +if USE_ARM_A64_NEON +noinst_LTLIBRARIES += libpixman-arma64-neon.la +libpixman_arma64_neon_la_SOURCES = \ + pixman-arm-neon.c \ + pixman-arm-common.h \ + pixman-arma64-neon-asm.S \ + pixman-arma64-neon-asm-bilinear.S \ + pixman-arm-asm.h \ + pixman-arma64-neon-asm.h +libpixman_1_la_LIBADD += libpixman-arma64-neon.la + +ASM_CFLAGS_arm_neon= +endif + # iwmmxt code if USE_ARM_IWMMXT libpixman_iwmmxt_la_SOURCES = pixman-mmx.c diff --git a/pixman/pixman-arm-neon.c b/pixman/pixman-arm-neon.c index be761c9..62c9442 100644 --- a/pixman/pixman-arm-neon.c +++ b/pixman/pixman-arm-neon.c @@ -194,7 +194,7 @@ arm_neon_fill (pixman_implementation_t *imp, uint32_t _xor) { /* stride is always multiple of 32bit units in pixman */ - uint32_t byte_stride = stride * sizeof(uint32_t); + int32_t byte_stride = stride * sizeof(uint32_t); switch (bpp) { @@ -331,6 +331,7 @@ static const pixman_fast_path_t arm_neon_fast_paths[] = PIXMAN_STD_FAST_PATH (OVER, a8b8g8r8, a8, b5g6r5, neon_composite_over_8888_8_0565), PIXMAN_STD_FAST_PATH (OVER, r5g6b5, a8, r5g6b5, neon_composite_over_0565_8_0565), PIXMAN_STD_FAST_PATH (OVER, b5g6r5, a8, b5g6r5, neon_composite_over_0565_8_0565), + PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, a8r8g8b8, x8r8g8b8, neon_composite_over_8888_8888_8888), PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, a8r8g8b8, a8r8g8b8, neon_composite_over_8888_8888_8888), PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, null, r5g6b5, neon_composite_over_8888_0565), PIXMAN_STD_FAST_PATH (OVER, a8b8g8r8, null, b5g6r5, neon_composite_over_8888_0565), @@ -341,17 +342,33 @@ static const pixman_fast_path_t arm_neon_fast_paths[] = PIXMAN_STD_FAST_PATH (OVER, x8r8g8b8, null, a8r8g8b8, neon_composite_src_x888_8888), PIXMAN_STD_FAST_PATH (OVER, x8b8g8r8, null, a8b8g8r8, neon_composite_src_x888_8888), PIXMAN_STD_FAST_PATH (ADD, solid, a8, a8, neon_composite_add_n_8_8), + PIXMAN_STD_FAST_PATH (ADD, solid, a8, x8r8g8b8, neon_composite_add_n_8_8888), PIXMAN_STD_FAST_PATH (ADD, solid, a8, a8r8g8b8, neon_composite_add_n_8_8888), + PIXMAN_STD_FAST_PATH (ADD, solid, a8, x8b8g8r8, neon_composite_add_n_8_8888), PIXMAN_STD_FAST_PATH (ADD, solid, a8, a8b8g8r8, neon_composite_add_n_8_8888), PIXMAN_STD_FAST_PATH (ADD, a8, a8, a8, neon_composite_add_8_8_8), PIXMAN_STD_FAST_PATH (ADD, r5g6b5, a8, r5g6b5, neon_composite_add_0565_8_0565), PIXMAN_STD_FAST_PATH (ADD, b5g6r5, a8, b5g6r5, neon_composite_add_0565_8_0565), + PIXMAN_STD_FAST_PATH (ADD, x8r8g8b8, a8, x8r8g8b8, neon_composite_add_8888_8_8888), + PIXMAN_STD_FAST_PATH (ADD, a8r8g8b8, a8, x8r8g8b8, neon_composite_add_8888_8_8888), + PIXMAN_STD_FAST_PATH (ADD, x8b8g8r8, a8, x8b8g8r8, neon_composite_add_8888_8_8888), + PIXMAN_STD_FAST_PATH (ADD, a8b8g8r8, a8, x8b8g8r8, neon_composite_add_8888_8_8888), PIXMAN_STD_FAST_PATH (ADD, a8r8g8b8, a8, a8r8g8b8, neon_composite_add_8888_8_8888), PIXMAN_STD_FAST_PATH (ADD, a8b8g8r8, a8, a8b8g8r8, neon_composite_add_8888_8_8888), + PIXMAN_STD_FAST_PATH (ADD, x8r8g8b8, a8r8g8b8, x8r8g8b8, neon_composite_add_8888_8888_8888), + PIXMAN_STD_FAST_PATH (ADD, a8r8g8b8, a8r8g8b8, x8r8g8b8, neon_composite_add_8888_8888_8888), PIXMAN_STD_FAST_PATH (ADD, a8r8g8b8, a8r8g8b8, a8r8g8b8, neon_composite_add_8888_8888_8888), + PIXMAN_STD_FAST_PATH (ADD, x8r8g8b8, solid, x8r8g8b8, neon_composite_add_8888_n_8888), + PIXMAN_STD_FAST_PATH (ADD, a8r8g8b8, solid, x8r8g8b8, neon_composite_add_8888_n_8888), + PIXMAN_STD_FAST_PATH (ADD, x8b8g8r8, solid, x8b8g8r8, neon_composite_add_8888_n_8888), + PIXMAN_STD_FAST_PATH (ADD, a8b8g8r8, solid, x8b8g8r8, neon_composite_add_8888_n_8888), PIXMAN_STD_FAST_PATH (ADD, a8r8g8b8, solid, a8r8g8b8, neon_composite_add_8888_n_8888), PIXMAN_STD_FAST_PATH (ADD, a8b8g8r8, solid, a8b8g8r8, neon_composite_add_8888_n_8888), PIXMAN_STD_FAST_PATH (ADD, a8, null, a8, neon_composite_add_8_8), + PIXMAN_STD_FAST_PATH (ADD, x8r8g8b8, null, x8r8g8b8, neon_composite_add_8888_8888), + PIXMAN_STD_FAST_PATH (ADD, a8r8g8b8, null, x8r8g8b8, neon_composite_add_8888_8888), + PIXMAN_STD_FAST_PATH (ADD, x8b8g8r8, null, x8b8g8r8, neon_composite_add_8888_8888), + PIXMAN_STD_FAST_PATH (ADD, a8b8g8r8, null, x8b8g8r8, neon_composite_add_8888_8888), PIXMAN_STD_FAST_PATH (ADD, a8r8g8b8, null, a8r8g8b8, neon_composite_add_8888_8888), PIXMAN_STD_FAST_PATH (ADD, a8b8g8r8, null, a8b8g8r8, neon_composite_add_8888_8888), PIXMAN_STD_FAST_PATH (IN, solid, null, a8, neon_composite_in_n_8), @@ -359,7 +376,9 @@ static const pixman_fast_path_t arm_neon_fast_paths[] = PIXMAN_STD_FAST_PATH (OVER_REVERSE, solid, null, a8b8g8r8, neon_composite_over_reverse_n_8888), PIXMAN_STD_FAST_PATH (OUT_REVERSE, a8, null, r5g6b5, neon_composite_out_reverse_8_0565), PIXMAN_STD_FAST_PATH (OUT_REVERSE, a8, null, b5g6r5, neon_composite_out_reverse_8_0565), + PIXMAN_STD_FAST_PATH (OUT_REVERSE, a8, null, x8r8g8b8, neon_composite_out_reverse_8_8888), PIXMAN_STD_FAST_PATH (OUT_REVERSE, a8, null, a8r8g8b8, neon_composite_out_reverse_8_8888), + PIXMAN_STD_FAST_PATH (OUT_REVERSE, a8, null, x8b8g8r8, neon_composite_out_reverse_8_8888), PIXMAN_STD_FAST_PATH (OUT_REVERSE, a8, null, a8b8g8r8, neon_composite_out_reverse_8_8888), SIMPLE_NEAREST_FAST_PATH (OVER, a8r8g8b8, a8r8g8b8, neon_8888_8888), @@ -404,6 +423,7 @@ static const pixman_fast_path_t arm_neon_fast_paths[] = SIMPLE_BILINEAR_FAST_PATH (ADD, a8r8g8b8, a8r8g8b8, neon_8888_8888), SIMPLE_BILINEAR_FAST_PATH (ADD, a8r8g8b8, x8r8g8b8, neon_8888_8888), + SIMPLE_BILINEAR_FAST_PATH (ADD, x8r8g8b8, x8r8g8b8, neon_8888_8888), SIMPLE_BILINEAR_A8_MASK_FAST_PATH (SRC, a8r8g8b8, a8r8g8b8, neon_8888_8_8888), SIMPLE_BILINEAR_A8_MASK_FAST_PATH (SRC, a8r8g8b8, x8r8g8b8, neon_8888_8_8888), @@ -420,6 +440,7 @@ static const pixman_fast_path_t arm_neon_fast_paths[] = SIMPLE_BILINEAR_A8_MASK_FAST_PATH (ADD, a8r8g8b8, a8r8g8b8, neon_8888_8_8888), SIMPLE_BILINEAR_A8_MASK_FAST_PATH (ADD, a8r8g8b8, x8r8g8b8, neon_8888_8_8888), + SIMPLE_BILINEAR_A8_MASK_FAST_PATH (ADD, x8r8g8b8, x8r8g8b8, neon_8888_8_8888), { PIXMAN_OP_NONE }, }; diff --git a/pixman/pixman-arm.c b/pixman/pixman-arm.c index 23374e4..734cbea 100644 --- a/pixman/pixman-arm.c +++ b/pixman/pixman-arm.c @@ -221,5 +221,11 @@ _pixman_arm_get_implementations (pixman_implementation_t *imp) imp = _pixman_implementation_create_arm_neon (imp); #endif +#ifdef USE_ARM_A64_NEON + /* neon is a part of aarch64 */ + if (!_pixman_disabled ("arm-neon")) + imp = _pixman_implementation_create_arm_neon (imp); +#endif + return imp; } diff --git a/pixman/pixman-arma64-neon-asm-bilinear.S b/pixman/pixman-arma64-neon-asm-bilinear.S new file mode 100644 index 0000000..aaa4a83 --- /dev/null +++ b/pixman/pixman-arma64-neon-asm-bilinear.S @@ -0,0 +1,1275 @@ +/* + * Copyright © 2011 SCore Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + * + * Author: Siarhei Siamashka (siarhei.siamashka@nokia.com) + * Author: Taekyun Kim (tkq.kim@samsung.com) + */ + +/* + * This file contains scaled bilinear scanline functions implemented + * using older siarhei's bilinear macro template. + * + * << General scanline function procedures >> + * 1. bilinear interpolate source pixels + * 2. load mask pixels + * 3. load destination pixels + * 4. duplicate mask to fill whole register + * 5. interleave source & destination pixels + * 6. apply mask to source pixels + * 7. combine source & destination pixels + * 8, Deinterleave final result + * 9. store destination pixels + * + * All registers with single number (i.e. src0, tmp0) are 64-bits registers. + * Registers with double numbers(src01, dst01) are 128-bits registers. + * All temp registers can be used freely outside the code block. + * Assume that symbol(register .req) OUT and MASK are defined at caller of these macro blocks. + * + * Remarks + * There can be lots of pipeline stalls inside code block and between code blocks. + * Further optimizations will be done by new macro templates using head/tail_head/tail scheme. + */ + +/* Prevent the stack from becoming executable for no reason... */ +#if defined(__linux__) && defined (__ELF__) +.section .note.GNU-stack,"",%progbits +#endif + +.text +.arch armv8-a +.altmacro +.p2align 2 + +#include "pixman-private.h" +#include "pixman-arm-asm.h" +#include "pixman-arma64-neon-asm.h" + +/* + * Bilinear macros from pixman-arm-neon-asm.S + */ + +/* + * Bilinear scaling support code which tries to provide pixel fetching, color + * format conversion, and interpolation as separate macros which can be used + * as the basic building blocks for constructing bilinear scanline functions. + */ + +.macro bilinear_load_8888 reg1, reg2, tmp + asr WTMP1, X, #16 + add X, X, UX + add TMP1, TOP, TMP1, lsl #2 + ld1 {®1&.2s}, [TMP1], STRIDE + ld1 {®2&.2s}, [TMP1] +.endm + +.macro bilinear_load_0565 reg1, reg2, tmp + asr WTMP1, X, #16 + add X, X, UX + add TMP1, TOP, TMP1, lsl #1 + ld1 {®2&.s}[0], [TMP1], STRIDE + ld1 {®2&.s}[1], [TMP1] + convert_four_0565_to_x888_packed reg2, reg1, reg2, tmp +.endm + +.macro bilinear_load_and_vertical_interpolate_two_8888 \ + acc1, acc2, reg1, reg2, reg3, reg4, tmp1, tmp2 + + bilinear_load_8888 reg1, reg2, tmp1 + umull &acc1&.8h, ®1&.8b, v28.8b + umlal &acc1&.8h, ®2&.8b, v29.8b + bilinear_load_8888 reg3, reg4, tmp2 + umull &acc2&.8h, ®3&.8b, v28.8b + umlal &acc2&.8h, ®4&.8b, v29.8b +.endm + +.macro bilinear_load_and_vertical_interpolate_four_8888 \ + xacc1, xacc2, xreg1, xreg2, xreg3, xreg4, xacc2lo, xacc2hi \ + yacc1, yacc2, yreg1, yreg2, yreg3, yreg4, yacc2lo, yacc2hi + + bilinear_load_and_vertical_interpolate_two_8888 \ + xacc1, xacc2, xreg1, xreg2, xreg3, xreg4, xacc2lo, xacc2hi + bilinear_load_and_vertical_interpolate_two_8888 \ + yacc1, yacc2, yreg1, yreg2, yreg3, yreg4, yacc2lo, yacc2hi +.endm + +.macro vzip reg1, reg2 + zip1 v24.8b, reg1, reg2 + zip2 reg2, reg1, reg2 + mov reg1, v24.8b +.endm + +.macro vuzp reg1, reg2 + uzp1 v24.8b, reg1, reg2 + uzp2 reg2, reg1, reg2 + mov reg1, v24.8b +.endm + +.macro bilinear_load_and_vertical_interpolate_two_0565 \ + acc1, acc2, reg1, reg2, reg3, reg4, acc2lo, acc2hi + asr WTMP1, X, #16 + add X, X, UX + add TMP1, TOP, TMP1, lsl #1 + asr WTMP2, X, #16 + add X, X, UX + add TMP2, TOP, TMP2, lsl #1 + ld1 {&acc2&.s}[0], [TMP1], STRIDE + ld1 {&acc2&.s}[2], [TMP2], STRIDE + ld1 {&acc2&.s}[1], [TMP1] + ld1 {&acc2&.s}[3], [TMP2] + convert_0565_to_x888 acc2, reg3, reg2, reg1 + vzip ®1&.8b, ®3&.8b + vzip ®2&.8b, ®4&.8b + vzip ®3&.8b, ®4&.8b + vzip ®1&.8b, ®2&.8b + umull &acc1&.8h, ®1&.8b, v28.8b + umlal &acc1&.8h, ®2&.8b, v29.8b + umull &acc2&.8h, ®3&.8b, v28.8b + umlal &acc2&.8h, ®4&.8b, v29.8b +.endm + +.macro bilinear_load_and_vertical_interpolate_four_0565 \ + xacc1, xacc2, xreg1, xreg2, xreg3, xreg4, xacc2lo, xacc2hi \ + yacc1, yacc2, yreg1, yreg2, yreg3, yreg4, yacc2lo, yacc2hi + + asr WTMP1, X, #16 + add X, X, UX + add TMP1, TOP, TMP1, lsl #1 + asr WTMP2, X, #16 + add X, X, UX + add TMP2, TOP, TMP2, lsl #1 + ld1 {&xacc2&.s}[0], [TMP1], STRIDE + ld1 {&xacc2&.s}[2], [TMP2], STRIDE + ld1 {&xacc2&.s}[1], [TMP1] + ld1 {&xacc2&.s}[3], [TMP2] + convert_0565_to_x888 xacc2, xreg3, xreg2, xreg1 + asr WTMP1, X, #16 + add X, X, UX + add TMP1, TOP, TMP1, lsl #1 + asr WTMP2, X, #16 + add X, X, UX + add TMP2, TOP, TMP2, lsl #1 + ld1 {&yacc2&.s}[0], [TMP1], STRIDE + vzip &xreg1&.8b, &xreg3&.8b + ld1 {&yacc2&.s}[2], [TMP2], STRIDE + vzip &xreg2&.8b, &xreg4&.8b + ld1 {&yacc2&.s}[1], [TMP1] + vzip &xreg3&.8b, &xreg4&.8b + ld1 {&yacc2&.s}[3], [TMP2] + vzip &xreg1&.8b, &xreg2&.8b + convert_0565_to_x888 yacc2, yreg3, yreg2, yreg1 + umull &xacc1&.8h, &xreg1&.8b, v28.8b + vzip &yreg1&.8b, &yreg3&.8b + umlal &xacc1&.8h, &xreg2&.8b, v29.8b + vzip &yreg2&.8b, &yreg4&.8b + umull &xacc2&.8h, &xreg3&.8b, v28.8b + vzip &yreg3&.8b, &yreg4&.8b + umlal &xacc2&.8h, &xreg4&.8b, v29.8b + vzip &yreg1&.8b, &yreg2&.8b + umull &yacc1&.8h, &yreg1&.8b, v28.8b + umlal &yacc1&.8h, &yreg2&.8b, v29.8b + umull &yacc2&.8h, &yreg3&.8b, v28.8b + umlal &yacc2&.8h, &yreg4&.8b, v29.8b +.endm + +.macro bilinear_store_8888 numpix, tmp1, tmp2 +.if numpix == 4 + st1 {v0.2s, v1.2s}, [OUT], #16 +.elseif numpix == 2 + st1 {v0.2s}, [OUT], #8 +.elseif numpix == 1 + st1 {v0.s}[0], [OUT], #4 +.else + .error bilinear_store_8888 numpix is unsupported +.endif +.endm + +.macro bilinear_store_0565 numpix, tmp1, tmp2 + vuzp v0.8b, v1.8b + vuzp v2.8b, v3.8b + vuzp v1.8b, v3.8b + vuzp v0.8b, v2.8b + convert_8888_to_0565 v2, v1, v0, v1, tmp1, tmp2 +.if numpix == 4 + st1 {v1.4h}, [OUT], #8 +.elseif numpix == 2 + st1 {v1.s}[0], [OUT], #4 +.elseif numpix == 1 + st1 {v1.h}[0], [OUT], #2 +.else + .error bilinear_store_0565 numpix is unsupported +.endif +.endm + + +/* + * Macros for loading mask pixels into register 'mask'. + * dup must be done in somewhere else. + */ +.macro bilinear_load_mask_x numpix, mask +.endm + +.macro bilinear_load_mask_8 numpix, mask +.if numpix == 4 + ld1 {&mask&.s}[0], [MASK], #4 +.elseif numpix == 2 + ld1 {&mask&.h}[0], [MASK], #2 +.elseif numpix == 1 + ld1 {&mask&.b}[0], [MASK], #1 +.else + .error bilinear_load_mask_8 numpix is unsupported +.endif + prfm PREFETCH_MODE, [MASK, #prefetch_offset] +.endm + +.macro bilinear_load_mask mask_fmt, numpix, mask + bilinear_load_mask_&mask_fmt numpix, mask +.endm + + +/* + * Macros for loading destination pixels into register 'dst0' and 'dst1'. + * Interleave should be done somewhere else. + */ +.macro bilinear_load_dst_0565_src numpix, dst0, dst1, dst01 +.endm + +.macro bilinear_load_dst_8888_src numpix, dst0, dst1, dst01 +.endm + +.macro bilinear_load_dst_8888 numpix, dst0, dst1, dst01 +.if numpix == 4 + ld1 {&dst0&.2s, &dst1&.2s}, [OUT] +.elseif numpix == 2 + ld1 {&dst0&.2s}, [OUT] +.elseif numpix == 1 + ld1 {&dst0&.s}[0], [OUT] +.else + .error bilinear_load_dst_8888 numpix is unsupported +.endif + mov &dst01&.d[0], &dst0&.d[0] + mov &dst01&.d[1], &dst1&.d[0] + prfm PREFETCH_MODE, [OUT, #(prefetch_offset * 4)] +.endm + +.macro bilinear_load_dst_8888_over numpix, dst0, dst1, dst01 + bilinear_load_dst_8888 numpix, dst0, dst1, dst01 +.endm + +.macro bilinear_load_dst_8888_add numpix, dst0, dst1, dst01 + bilinear_load_dst_8888 numpix, dst0, dst1, dst01 +.endm + +.macro bilinear_load_dst dst_fmt, op, numpix, dst0, dst1, dst01 + bilinear_load_dst_&dst_fmt&_&op numpix, dst0, dst1, dst01 +.endm + +/* + * Macros for duplicating partially loaded mask to fill entire register. + * We will apply mask to interleaved source pixels, that is + * (r0, r1, r2, r3, g0, g1, g2, g3) x (m0, m1, m2, m3, m0, m1, m2, m3) + * (b0, b1, b2, b3, a0, a1, a2, a3) x (m0, m1, m2, m3, m0, m1, m2, m3) + * So, we need to duplicate loaded mask into whole register. + * + * For two pixel case + * (r0, r1, x, x, g0, g1, x, x) x (m0, m1, m0, m1, m0, m1, m0, m1) + * (b0, b1, x, x, a0, a1, x, x) x (m0, m1, m0, m1, m0, m1, m0, m1) + * We can do some optimizations for this including last pixel cases. + */ +.macro bilinear_duplicate_mask_x numpix, mask +.endm + +.macro bilinear_duplicate_mask_8 numpix, mask +.if numpix == 4 + dup &mask&.2s, &mask&.s[0] +.elseif numpix == 2 + dup &mask&.4h, &mask&.h[0] +.elseif numpix == 1 + dup &mask&.8b, &mask&.b[0] +.else + .error bilinear_duplicate_mask_8 is unsupported +.endif +.endm + +.macro bilinear_duplicate_mask mask_fmt, numpix, mask + bilinear_duplicate_mask_&mask_fmt numpix, mask +.endm + +/* + * Macros for interleaving src and dst pixels to rrrr gggg bbbb aaaa form. + * Interleave should be done when maks is enabled or operator is 'over'. + */ +.macro bilinear_interleave src0, src1, src01, dst0, dst1, dst01 + vuzp &src0&.8b, &src1&.8b + vuzp &dst0&.8b, &dst1&.8b + vuzp &src0&.8b, &src1&.8b + vuzp &dst0&.8b, &dst1&.8b + mov &src01&.d[1], &src1&.d[0] + mov &src01&.d[0], &src0&.d[0] + mov &dst01&.d[1], &dst1&.d[0] + mov &dst01&.d[0], &dst0&.d[0] +.endm + +.macro bilinear_interleave_src_dst_x_src \ + numpix, src0, src1, src01, dst0, dst1, dst01 +.endm + +.macro bilinear_interleave_src_dst_x_over \ + numpix, src0, src1, src01, dst0, dst1, dst01 + + bilinear_interleave src0, src1, src01, dst0, dst1, dst01 +.endm + +.macro bilinear_interleave_src_dst_x_add \ + numpix, src0, src1, src01, dst0, dst1, dst01 + bilinear_interleave src0, src1, src01, dst0, dst1, dst01 +.endm + +.macro bilinear_interleave_src_dst_8_src \ + numpix, src0, src1, src01, dst0, dst1, dst01 + + bilinear_interleave src0, src1, src01, dst0, dst1, dst01 +.endm + +.macro bilinear_interleave_src_dst_8_over \ + numpix, src0, src1, src01, dst0, dst1, dst01 + + bilinear_interleave src0, src1, src01, dst0, dst1, dst01 +.endm + +.macro bilinear_interleave_src_dst_8_add \ + numpix, src0, src1, src01, dst0, dst1, dst01 + + bilinear_interleave src0, src1, src01, dst0, dst1, dst01 +.endm + +.macro bilinear_interleave_src_dst \ + mask_fmt, op, numpix, src0, src1, src01, dst0, dst1, dst01 + + bilinear_interleave_src_dst_&mask_fmt&_&op \ + numpix, src0, src1, src01, dst0, dst1, dst01 +.endm + + +/* + * Macros for applying masks to src pixels. (see combine_mask_u() function) + * src, dst should be in interleaved form. + * mask register should be in form (m0, m1, m2, m3). + */ +.macro bilinear_apply_mask_to_src_x \ + numpix, src0, src1, src01, mask, \ + tmp01, tmp23, tmp45, tmp67 +.endm + +.macro bilinear_apply_mask_to_src_8 \ + numpix, src0, src1, src01, mask, \ + tmp01, tmp23, tmp45, tmp67 + + umull &tmp01&.8h, &src0&.8b, &mask&.8b + umull &tmp23&.8h, &src1&.8b, &mask&.8b + /* bubbles */ + urshr &tmp45&.8h, &tmp01&.8h, #8 + urshr &tmp67&.8h, &tmp23&.8h, #8 + /* bubbles */ + raddhn &src0&.8b, &tmp45&.8h, &tmp01&.8h + raddhn &src1&.8b, &tmp67&.8h, &tmp23&.8h + mov &src01&.d[0], &src0&.d[0] + mov &src01&.d[1], &src1&.d[0] +.endm + +.macro bilinear_apply_mask_to_src \ + mask_fmt, numpix, src0, src1, src01, mask, \ + tmp01, tmp23, tmp45, tmp67 + + bilinear_apply_mask_to_src_&mask_fmt \ + numpix, src0, src1, src01, mask, \ + tmp01, tmp23, tmp45, tmp67 +.endm + + +/* + * Macros for combining src and destination pixels. + * Interleave or not is depending on operator 'op'. + */ +.macro bilinear_combine_src \ + numpix, src0, src1, src01, dst0, dst1, dst01, \ + tmp01, tmp23, tmp45, tmp67, tmp8 +.endm + +.macro bilinear_combine_over \ + numpix, src0, src1, src01, dst0, dst1, dst01, \ + tmp01, tmp23, tmp45, tmp67, tmp8 + + dup &tmp8&.2s, &src1&.s[1] + /* bubbles */ + mvn &tmp8&.8b, &tmp8&.8b + /* bubbles */ + umull &tmp01&.8h, &dst0&.8b, &tmp8&.8b + /* bubbles */ + umull &tmp23&.8h, &dst1&.8b, &tmp8&.8b + /* bubbles */ + urshr &tmp45&.8h, &tmp01&.8h, #8 + urshr &tmp67&.8h, &tmp23&.8h, #8 + /* bubbles */ + raddhn &dst0&.8b, &tmp45&.8h, &tmp01&.8h + raddhn &dst1&.8b, &tmp67&.8h, &tmp23&.8h + mov &dst01&.d[0], &dst0&.d[0] + mov &dst01&.d[1], &dst1&.d[0] + /* bubbles */ + uqadd &src0&.8b, &dst0&.8b, &src0&.8b + uqadd &src1&.8b, &dst1&.8b, &src1&.8b + mov &src01&.d[0], &src0&.d[0] + mov &src01&.d[1], &src1&.d[0] +.endm + +.macro bilinear_combine_add \ + numpix, src0, src1, src01, dst0, dst1, dst01, \ + tmp01, tmp23, tmp45, tmp67, tmp8 + + uqadd &src0&.8b, &dst0&.8b, &src0&.8b + uqadd &src1&.8b, &dst1&.8b, &src1&.8b + mov &src01&.d[0], &src0&.d[0] + mov &src01&.d[1], &src1&.d[0] +.endm + +.macro bilinear_combine \ + op, numpix, src0, src1, src01, dst0, dst1, dst01, \ + tmp01, tmp23, tmp45, tmp67, tmp8 + + bilinear_combine_&op \ + numpix, src0, src1, src01, dst0, dst1, dst01, \ + tmp01, tmp23, tmp45, tmp67, tmp8 +.endm + +/* + * Macros for final deinterleaving of destination pixels if needed. + */ +.macro bilinear_deinterleave numpix, dst0, dst1, dst01 + vuzp &dst0&.8b, &dst1&.8b + /* bubbles */ + vuzp &dst0&.8b, &dst1&.8b + mov &dst01&.d[0], &dst0&.d[0] + mov &dst01&.d[1], &dst1&.d[0] +.endm + +.macro bilinear_deinterleave_dst_x_src numpix, dst0, dst1, dst01 +.endm + +.macro bilinear_deinterleave_dst_x_over numpix, dst0, dst1, dst01 + bilinear_deinterleave numpix, dst0, dst1, dst01 +.endm + +.macro bilinear_deinterleave_dst_x_add numpix, dst0, dst1, dst01 + bilinear_deinterleave numpix, dst0, dst1, dst01 +.endm + +.macro bilinear_deinterleave_dst_8_src numpix, dst0, dst1, dst01 + bilinear_deinterleave numpix, dst0, dst1, dst01 +.endm + +.macro bilinear_deinterleave_dst_8_over numpix, dst0, dst1, dst01 + bilinear_deinterleave numpix, dst0, dst1, dst01 +.endm + +.macro bilinear_deinterleave_dst_8_add numpix, dst0, dst1, dst01 + bilinear_deinterleave numpix, dst0, dst1, dst01 +.endm + +.macro bilinear_deinterleave_dst mask_fmt, op, numpix, dst0, dst1, dst01 + bilinear_deinterleave_dst_&mask_fmt&_&op numpix, dst0, dst1, dst01 +.endm + + +.macro bilinear_interpolate_last_pixel src_fmt, mask_fmt, dst_fmt, op + bilinear_load_&src_fmt v0, v1, v2 + bilinear_load_mask mask_fmt, 1, v4 + bilinear_load_dst dst_fmt, op, 1, v18, v19, v9 + umull v2.8h, v0.8b, v28.8b + umlal v2.8h, v1.8b, v29.8b + /* 5 cycles bubble */ + ushll v0.4s, v2.4h, #BILINEAR_INTERPOLATION_BITS + umlsl v0.4s, v2.4h, v15.h[0] + umlal2 v0.4s, v2.8h, v15.h[0] + /* 5 cycles bubble */ + bilinear_duplicate_mask mask_fmt, 1, v4 + shrn v0.4h, v0.4s, #(2 * BILINEAR_INTERPOLATION_BITS) + /* 3 cycles bubble */ + xtn v0.8b, v0.8h + /* 1 cycle bubble */ + bilinear_interleave_src_dst \ + mask_fmt, op, 1, v0, v1, v0, v18, v19, v9 + bilinear_apply_mask_to_src \ + mask_fmt, 1, v0, v1, v0, v4, \ + v3, v8, v10, v11 + bilinear_combine \ + op, 1, v0, v1, v0, v18, v19, v9, \ + v3, v8, v10, v11, v5 + bilinear_deinterleave_dst mask_fmt, op, 1, v0, v1, v0 + bilinear_store_&dst_fmt 1, v17, v18 +.endm + +.macro bilinear_interpolate_two_pixels src_fmt, mask_fmt, dst_fmt, op + bilinear_load_and_vertical_interpolate_two_&src_fmt \ + v1, v11, v18, v19, v20, v21, v22, v23 + bilinear_load_mask mask_fmt, 2, v4 + bilinear_load_dst dst_fmt, op, 2, v18, v19, v9 + ushll v0.4s, v1.4h, #BILINEAR_INTERPOLATION_BITS + umlsl v0.4s, v1.4h, v15.h[0] + umlal2 v0.4s, v1.8h, v15.h[0] + ushll v10.4s, v11.4h, #BILINEAR_INTERPOLATION_BITS + umlsl v10.4s, v11.4h, v15.h[4] + umlal2 v10.4s, v11.8h, v15.h[4] + shrn v0.4h, v0.4s, #(2 * BILINEAR_INTERPOLATION_BITS) + shrn2 v0.8h, v10.4s, #(2 * BILINEAR_INTERPOLATION_BITS) + bilinear_duplicate_mask mask_fmt, 2, v4 + ushr v15.8h, v12.8h, #(16 - BILINEAR_INTERPOLATION_BITS) + add v12.8h, v12.8h, v13.8h + xtn v0.8b, v0.8h + bilinear_interleave_src_dst \ + mask_fmt, op, 2, v0, v1, v0, v18, v19, v9 + bilinear_apply_mask_to_src \ + mask_fmt, 2, v0, v1, v0, v4, \ + v3, v8, v10, v11 + bilinear_combine \ + op, 2, v0, v1, v0, v18, v19, v9, \ + v3, v8, v10, v11, v5 + bilinear_deinterleave_dst mask_fmt, op, 2, v0, v1, v0 + bilinear_store_&dst_fmt 2, v16, v17 +.endm + +.macro bilinear_interpolate_four_pixels src_fmt, mask_fmt, dst_fmt, op + bilinear_load_and_vertical_interpolate_four_&src_fmt \ + v1, v11, v4, v5, v6, v7, v22, v23 \ + v3, v9, v16, v17, v20, v21, v18, v19 + prfm PREFETCH_MODE, [TMP1, PF_OFFS] + sub TMP1, TMP1, STRIDE + prfm PREFETCH_MODE, [TMP1, PF_OFFS] + ushll v0.4s, v1.4h, #BILINEAR_INTERPOLATION_BITS + umlsl v0.4s, v1.4h, v15.h[0] + umlal2 v0.4s, v1.8h, v15.h[0] + ushll v10.4s, v11.4h, #BILINEAR_INTERPOLATION_BITS + umlsl v10.4s, v11.4h, v15.h[4] + umlal2 v10.4s, v11.8h, v15.h[4] + ushr v15.8h, v12.8h, #(16 - BILINEAR_INTERPOLATION_BITS) + ushll v2.4s, v3.4h, #BILINEAR_INTERPOLATION_BITS + umlsl v2.4s, v3.4h, v15.h[0] + umlal2 v2.4s, v3.8h, v15.h[0] + ushll v8.4s, v9.4h, #BILINEAR_INTERPOLATION_BITS + umlsl v8.4s, v9.4h, v15.h[4] + umlal2 v8.4s, v9.8h, v15.h[4] + add v12.8h, v12.8h, v13.8h + shrn v0.4h, v0.4s, #(2 * BILINEAR_INTERPOLATION_BITS) + shrn2 v0.8h, v10.4s, #(2 * BILINEAR_INTERPOLATION_BITS) + shrn v2.4h, v2.4s, #(2 * BILINEAR_INTERPOLATION_BITS) + shrn2 v2.8h, v8.4s, #(2 * BILINEAR_INTERPOLATION_BITS) + bilinear_load_mask mask_fmt, 4, v4 + bilinear_duplicate_mask mask_fmt, 4, v4 + ushr v15.8h, v12.8h, #(16 - BILINEAR_INTERPOLATION_BITS) + xtn v0.8b, v0.8h + xtn v1.8b, v2.8h + add v12.8h, v12.8h, v13.8h + bilinear_load_dst dst_fmt, op, 4, v2, v3, v21 + bilinear_interleave_src_dst \ + mask_fmt, op, 4, v0, v1, v0, v2, v3, v11 + bilinear_apply_mask_to_src \ + mask_fmt, 4, v0, v1, v0, v4, \ + v6, v8, v9, v10 + bilinear_combine \ + op, 4, v0, v1, v0, v2, v3, v1, \ + v6, v8, v9, v10, v23 + bilinear_deinterleave_dst mask_fmt, op, 4, v0, v1, v0 + bilinear_store_&dst_fmt 4, v6, v7 +.endm + +.set BILINEAR_FLAG_USE_MASK, 1 +.set BILINEAR_FLAG_USE_ALL_NEON_REGS, 2 + +/* + * Main template macro for generating NEON optimized bilinear scanline functions. + * + * Bilinear scanline generator macro take folling arguments: + * fname - name of the function to generate + * src_fmt - source color format (8888 or 0565) + * dst_fmt - destination color format (8888 or 0565) + * src/dst_bpp_shift - (1 << bpp_shift) is the size of src/dst pixel in bytes + * process_last_pixel - code block that interpolate one pixel and does not + * update horizontal weight + * process_two_pixels - code block that interpolate two pixels and update + * horizontal weight + * process_four_pixels - code block that interpolate four pixels and update + * horizontal weight + * process_pixblock_head - head part of middle loop + * process_pixblock_tail - tail part of middle loop + * process_pixblock_tail_head - tail_head of middle loop + * pixblock_size - number of pixels processed in a single middle loop + * prefetch_distance - prefetch in the source image by that many pixels ahead + */ + +.macro generate_bilinear_scanline_func \ + fname, \ + src_fmt, dst_fmt, src_bpp_shift, dst_bpp_shift, \ + bilinear_process_last_pixel, \ + bilinear_process_two_pixels, \ + bilinear_process_four_pixels, \ + bilinear_process_pixblock_head, \ + bilinear_process_pixblock_tail, \ + bilinear_process_pixblock_tail_head, \ + pixblock_size, \ + prefetch_distance, \ + flags + +pixman_asm_function fname +.if pixblock_size == 8 +.elseif pixblock_size == 4 +.else + .error unsupported pixblock size +.endif + +.if ((flags) & BILINEAR_FLAG_USE_MASK) == 0 + OUT .req x0 + TOP .req x1 + BOTTOM .req x2 + WT .req x3 + WWT .req w3 + WB .req x4 + WWB .req w4 + X .req w5 + UX .req w6 + WIDTH .req x7 + TMP1 .req x10 + WTMP1 .req w10 + TMP2 .req x11 + WTMP2 .req w11 + PF_OFFS .req x12 + TMP3 .req x13 + WTMP3 .req w13 + TMP4 .req x14 + WTMP4 .req w14 + STRIDE .req x15 + DUMMY .req x30 + + stp x29, x30, [sp, -16]! + mov x29, sp + sub sp, sp, 112 + sub x29, x29, 64 + st1 {v8.8b, v9.8b, v10.8b, v11.8b}, [x29], 32 + st1 {v12.8b, v13.8b, v14.8b, v15.8b}, [x29], 32 + stp x10, x11, [x29, -80] + stp x12, x13, [x29, -96] + stp x14, x15, [x29, -112] +.else + OUT .req x0 + MASK .req x1 + TOP .req x2 + BOTTOM .req x3 + WT .req x4 + WWT .req w4 + WB .req x5 + WWB .req w5 + X .req w6 + UX .req w7 + WIDTH .req x8 + TMP1 .req x10 + WTMP1 .req w10 + TMP2 .req x11 + WTMP2 .req w11 + PF_OFFS .req x12 + TMP3 .req x13 + WTMP3 .req w13 + TMP4 .req x14 + WTMP4 .req w14 + STRIDE .req x15 + DUMMY .req x30 + + .set prefetch_offset, prefetch_distance + + stp x29, x30, [sp, -16]! + mov x29, sp + sub x29, x29, 64 + st1 {v8.8b, v9.8b, v10.8b, v11.8b}, [x29], 32 + st1 {v12.8b, v13.8b, v14.8b, v15.8b}, [x29], 32 + stp x10, x11, [x29, -80] + stp x12, x13, [x29, -96] + stp x14, x15, [x29, -112] + str x8, [x29, -120] + ldr w8, [x29, 16] + sub sp, sp, 120 +.endif + + mov WTMP1, #prefetch_distance + umull PF_OFFS, WTMP1, UX + + sub STRIDE, BOTTOM, TOP + .unreq BOTTOM + + cmp WIDTH, #0 + ble 300f + + dup v12.8h, X + dup v13.8h, UX + dup v28.8b, WWT + dup v29.8b, WWB + mov v25.d[0], v12.d[1] + mov v26.d[0], v13.d[0] + add v25.4h, v25.4h, v26.4h + mov v12.d[1], v25.d[0] + + /* ensure good destination alignment */ + cmp WIDTH, #1 + blt 100f + tst OUT, #(1 << dst_bpp_shift) + beq 100f + ushr v15.8h, v12.8h, #(16 - BILINEAR_INTERPOLATION_BITS) + add v12.8h, v12.8h, v13.8h + bilinear_process_last_pixel + sub WIDTH, WIDTH, #1 +100: + add v13.8h, v13.8h, v13.8h + ushr v15.8h, v12.8h, #(16 - BILINEAR_INTERPOLATION_BITS) + add v12.8h, v12.8h, v13.8h + + cmp WIDTH, #2 + blt 100f + tst OUT, #(1 << (dst_bpp_shift + 1)) + beq 100f + bilinear_process_two_pixels + sub WIDTH, WIDTH, #2 +100: +.if pixblock_size == 8 + cmp WIDTH, #4 + blt 100f + tst OUT, #(1 << (dst_bpp_shift + 2)) + beq 100f + bilinear_process_four_pixels + sub WIDTH, WIDTH, #4 +100: +.endif + subs WIDTH, WIDTH, #pixblock_size + blt 100f + asr PF_OFFS, PF_OFFS, #(16 - src_bpp_shift) + bilinear_process_pixblock_head + subs WIDTH, WIDTH, #pixblock_size + blt 500f +0: + bilinear_process_pixblock_tail_head + subs WIDTH, WIDTH, #pixblock_size + bge 0b +500: + bilinear_process_pixblock_tail +100: +.if pixblock_size == 8 + tst WIDTH, #4 + beq 200f + bilinear_process_four_pixels +200: +.endif + /* handle the remaining trailing pixels */ + tst WIDTH, #2 + beq 200f + bilinear_process_two_pixels +200: + tst WIDTH, #1 + beq 300f + bilinear_process_last_pixel +300: + +.if ((flags) & BILINEAR_FLAG_USE_MASK) == 0 + sub x29, x29, 64 + ld1 {v8.8b, v9.8b, v10.8b, v11.8b}, [x29], 32 + ld1 {v12.8b, v13.8b, v14.8b, v15.8b}, [x29], 32 + ldp x10, x11, [x29, -80] + ldp x12, x13, [x29, -96] + ldp x14, x15, [x29, -112] + mov sp, x29 + ldp x29, x30, [sp], 16 +.else + sub x29, x29, 64 + ld1 {v8.8b, v9.8b, v10.8b, v11.8b}, [x29], 32 + ld1 {v12.8b, v13.8b, v14.8b, v15.8b}, [x29], 32 + ldp x10, x11, [x29, -80] + ldp x12, x13, [x29, -96] + ldp x14, x15, [x29, -112] + ldr x8, [x29, -120] + mov sp, x29 + ldp x29, x30, [sp], 16 +.endif + ret + + .unreq OUT + .unreq TOP + .unreq WT + .unreq WWT + .unreq WB + .unreq WWB + .unreq X + .unreq UX + .unreq WIDTH + .unreq TMP1 + .unreq WTMP1 + .unreq TMP2 + .unreq PF_OFFS + .unreq TMP3 + .unreq TMP4 + .unreq STRIDE +.if ((flags) & BILINEAR_FLAG_USE_MASK) != 0 + .unreq MASK +.endif + +.endfunc + +.endm + +/* src_8888_8_8888 */ +.macro bilinear_src_8888_8_8888_process_last_pixel + bilinear_interpolate_last_pixel 8888, 8, 8888, src +.endm + +.macro bilinear_src_8888_8_8888_process_two_pixels + bilinear_interpolate_two_pixels 8888, 8, 8888, src +.endm + +.macro bilinear_src_8888_8_8888_process_four_pixels + bilinear_interpolate_four_pixels 8888, 8, 8888, src +.endm + +.macro bilinear_src_8888_8_8888_process_pixblock_head + bilinear_src_8888_8_8888_process_four_pixels +.endm + +.macro bilinear_src_8888_8_8888_process_pixblock_tail +.endm + +.macro bilinear_src_8888_8_8888_process_pixblock_tail_head + bilinear_src_8888_8_8888_process_pixblock_tail + bilinear_src_8888_8_8888_process_pixblock_head +.endm + +/* src_8888_8_0565 */ +.macro bilinear_src_8888_8_0565_process_last_pixel + bilinear_interpolate_last_pixel 8888, 8, 0565, src +.endm + +.macro bilinear_src_8888_8_0565_process_two_pixels + bilinear_interpolate_two_pixels 8888, 8, 0565, src +.endm + +.macro bilinear_src_8888_8_0565_process_four_pixels + bilinear_interpolate_four_pixels 8888, 8, 0565, src +.endm + +.macro bilinear_src_8888_8_0565_process_pixblock_head + bilinear_src_8888_8_0565_process_four_pixels +.endm + +.macro bilinear_src_8888_8_0565_process_pixblock_tail +.endm + +.macro bilinear_src_8888_8_0565_process_pixblock_tail_head + bilinear_src_8888_8_0565_process_pixblock_tail + bilinear_src_8888_8_0565_process_pixblock_head +.endm + +/* src_0565_8_x888 */ +.macro bilinear_src_0565_8_x888_process_last_pixel + bilinear_interpolate_last_pixel 0565, 8, 8888, src +.endm + +.macro bilinear_src_0565_8_x888_process_two_pixels + bilinear_interpolate_two_pixels 0565, 8, 8888, src +.endm + +.macro bilinear_src_0565_8_x888_process_four_pixels + bilinear_interpolate_four_pixels 0565, 8, 8888, src +.endm + +.macro bilinear_src_0565_8_x888_process_pixblock_head + bilinear_src_0565_8_x888_process_four_pixels +.endm + +.macro bilinear_src_0565_8_x888_process_pixblock_tail +.endm + +.macro bilinear_src_0565_8_x888_process_pixblock_tail_head + bilinear_src_0565_8_x888_process_pixblock_tail + bilinear_src_0565_8_x888_process_pixblock_head +.endm + +/* src_0565_8_0565 */ +.macro bilinear_src_0565_8_0565_process_last_pixel + bilinear_interpolate_last_pixel 0565, 8, 0565, src +.endm + +.macro bilinear_src_0565_8_0565_process_two_pixels + bilinear_interpolate_two_pixels 0565, 8, 0565, src +.endm + +.macro bilinear_src_0565_8_0565_process_four_pixels + bilinear_interpolate_four_pixels 0565, 8, 0565, src +.endm + +.macro bilinear_src_0565_8_0565_process_pixblock_head + bilinear_src_0565_8_0565_process_four_pixels +.endm + +.macro bilinear_src_0565_8_0565_process_pixblock_tail +.endm + +.macro bilinear_src_0565_8_0565_process_pixblock_tail_head + bilinear_src_0565_8_0565_process_pixblock_tail + bilinear_src_0565_8_0565_process_pixblock_head +.endm + +/* over_8888_8888 */ +.macro bilinear_over_8888_8888_process_last_pixel + bilinear_interpolate_last_pixel 8888, x, 8888, over +.endm + +.macro bilinear_over_8888_8888_process_two_pixels + bilinear_interpolate_two_pixels 8888, x, 8888, over +.endm + +.macro bilinear_over_8888_8888_process_four_pixels + bilinear_interpolate_four_pixels 8888, x, 8888, over +.endm + +.macro bilinear_over_8888_8888_process_pixblock_head + asr WTMP1, X, #16 + add X, X, UX + add TMP1, TOP, TMP1, lsl #2 + asr WTMP2, X, #16 + add X, X, UX + add TMP2, TOP, TMP2, lsl #2 + + ld1 {v22.2s}, [TMP1], STRIDE + ld1 {v23.2s}, [TMP1] + asr WTMP3, X, #16 + add X, X, UX + add TMP3, TOP, TMP3, lsl #2 + umull v8.8h, v22.8b, v28.8b + umlal v8.8h, v23.8b, v29.8b + + ld1 {v22.2s}, [TMP2], STRIDE + ld1 {v23.2s}, [TMP2] + asr WTMP4, X, #16 + add X, X, UX + add TMP4, TOP, TMP4, lsl #2 + umull v9.8h, v22.8b, v28.8b + umlal v9.8h, v23.8b, v29.8b + + ld1 {v22.2s}, [TMP3], STRIDE + ld1 {v23.2s}, [TMP3] + umull v10.8h, v22.8b, v28.8b + umlal v10.8h, v23.8b, v29.8b + + ushll v0.4s, v8.4h, #BILINEAR_INTERPOLATION_BITS + umlsl v0.4s, v8.4h, v15.h[0] + umlal2 v0.4s, v8.8h, v15.h[0] + + prfm PREFETCH_MODE, [TMP4, PF_OFFS] + ld1 {v16.2s}, [TMP4], STRIDE + ld1 {v17.2s}, [TMP4] + prfm PREFETCH_MODE, [TMP4, PF_OFFS] + umull v11.8h, v16.8b, v28.8b + umlal v11.8h, v17.8b, v29.8b + + ushll v1.4s, v9.4h, #BILINEAR_INTERPOLATION_BITS + umlsl v1.4s, v9.4h, v15.h[4] + umlal2 v1.4s, v9.8h, v15.h[4] + ushr v15.8h, v12.8h, #(16 - BILINEAR_INTERPOLATION_BITS) + add v12.8h, v12.8h, v13.8h +.endm + +.macro bilinear_over_8888_8888_process_pixblock_tail + ushll v2.4s, v10.4h, #BILINEAR_INTERPOLATION_BITS + umlsl v2.4s, v10.4h, v15.h[0] + umlal2 v2.4s, v10.8h, v15.h[0] + ushll v3.4s, v11.4h, #BILINEAR_INTERPOLATION_BITS + umlsl v3.4s, v11.4h, v15.h[4] + umlal2 v3.4s, v11.8h, v15.h[4] + shrn v0.4h, v0.4s, #(2 * BILINEAR_INTERPOLATION_BITS) + shrn2 v0.8h, v1.4s, #(2 * BILINEAR_INTERPOLATION_BITS) + shrn v2.4h, v2.4s, #(2 * BILINEAR_INTERPOLATION_BITS) + ushr v15.8h, v12.8h, #(16 - BILINEAR_INTERPOLATION_BITS) + shrn2 v2.8h, v3.4s, #(2 * BILINEAR_INTERPOLATION_BITS) + xtn v6.8b, v0.8h + xtn v7.8b, v2.8h + ld1 {v2.2s, v3.2s}, [OUT] + prfm PREFETCH_MODE, [OUT, #(prefetch_offset * 4)] + vuzp v6.8b, v7.8b + vuzp v2.8b, v3.8b + vuzp v6.8b, v7.8b + vuzp v2.8b, v3.8b + dup v4.2s, v7.s[1] + mvn v4.8b, v4.8b + umull v11.8h, v2.8b, v4.8b + umull v2.8h, v3.8b, v4.8b + urshr v1.8h, v11.8h, #8 + urshr v10.8h, v2.8h, #8 + raddhn v3.8b, v10.8h, v2.8h + raddhn v2.8b, v1.8h, v11.8h + uqadd v6.8b, v2.8b, v6.8b + uqadd v7.8b, v3.8b, v7.8b + vuzp v6.8b, v7.8b + vuzp v6.8b, v7.8b + add v12.8h, v12.8h, v13.8h + st1 {v6.2s, v7.2s}, [OUT], #16 +.endm + +.macro bilinear_over_8888_8888_process_pixblock_tail_head + ushll v2.4s, v10.4h, #BILINEAR_INTERPOLATION_BITS + asr WTMP1, X, #16 + add X, X, UX + add TMP1, TOP, TMP1, lsl #2 + umlsl v2.4s, v10.4h, v15.h[0] + asr WTMP2, X, #16 + add X, X, UX + add TMP2, TOP, TMP2, lsl #2 + umlal2 v2.4s, v10.8h, v15.h[0] + ushll v3.4s, v11.4h, #BILINEAR_INTERPOLATION_BITS + ld1 {v20.2s}, [TMP1], STRIDE + umlsl v3.4s, v11.4h, v15.h[4] + umlal2 v3.4s, v11.8h, v15.h[4] + ld1 {v21.2s}, [TMP1] + umull v8.8h, v20.8b, v28.8b + umlal v8.8h, v21.8b, v29.8b + shrn v0.4h, v0.4s, #(2 * BILINEAR_INTERPOLATION_BITS) + shrn2 v0.8h, v1.4s, #(2 * BILINEAR_INTERPOLATION_BITS) + shrn v2.4h, v2.4s, #(2 * BILINEAR_INTERPOLATION_BITS) + ushr v15.8h, v12.8h, #(16 - BILINEAR_INTERPOLATION_BITS) + ld1 {v22.2s}, [TMP2], STRIDE + shrn2 v2.8h, v3.4s, #(2 * BILINEAR_INTERPOLATION_BITS) + xtn v6.8b, v0.8h + ld1 {v23.2s}, [TMP2] + umull v9.8h, v22.8b, v28.8b + asr WTMP3, X, #16 + add X, X, UX + add TMP3, TOP, TMP3, lsl #2 + asr WTMP4, X, #16 + add X, X, UX + add TMP4, TOP, TMP4, lsl #2 + umlal v9.8h, v23.8b, v29.8b + xtn v7.8b, v2.8h + ld1 {v2.2s, v3.2s}, [OUT] + prfm PREFETCH_MODE, [OUT, PF_OFFS] + ld1 {v22.2s}, [TMP3], STRIDE + vuzp v6.8b, v7.8b + vuzp v2.8b, v3.8b + vuzp v6.8b, v7.8b + vuzp v2.8b, v3.8b + dup v4.2s, v7.s[1] + ld1 {v23.2s}, [TMP3] + mvn v4.8b, v4.8b + umull v10.8h, v22.8b, v28.8b + umlal v10.8h, v23.8b, v29.8b + umull v11.8h, v2.8b, v4.8b + umull v2.8h, v3.8b, v4.8b + ushll v0.4s, v8.4h, #BILINEAR_INTERPOLATION_BITS + umlsl v0.4s, v8.4h, v15.h[0] + urshr v1.8h, v11.8h, #8 + umlal2 v0.4s, v8.8h, v15.h[0] + urshr v8.8h, v2.8h, #8 + raddhn v3.8b, v8.8h, v2.8h + raddhn v2.8b, v1.8h, v11.8h + prfm PREFETCH_MODE, [TMP4, PF_OFFS] + ld1 {v16.2s}, [TMP4], STRIDE + uqadd v6.8b, v2.8b, v6.8b + uqadd v7.8b, v3.8b, v7.8b + ld1 {v17.2s}, [TMP4] + prfm PREFETCH_MODE, [TMP4, PF_OFFS] + umull v11.8h, v16.8b, v28.8b + umlal v11.8h, v17.8b, v29.8b + vuzp v6.8b, v7.8b + ushll v1.4s, v9.4h, #BILINEAR_INTERPOLATION_BITS + vuzp v6.8b, v7.8b + umlsl v1.4s, v9.4h, v15.h[4] + add v12.8h, v12.8h, v13.8h + umlal2 v1.4s, v9.8h, v15.h[4] + ushr v15.8h, v12.8h, #(16 - BILINEAR_INTERPOLATION_BITS) + add v12.8h, v12.8h, v13.8h + st1 {v6.2s, v7.2s}, [OUT], #16 +.endm + +/* over_8888_8_8888 */ +.macro bilinear_over_8888_8_8888_process_last_pixel + bilinear_interpolate_last_pixel 8888, 8, 8888, over +.endm + +.macro bilinear_over_8888_8_8888_process_two_pixels + bilinear_interpolate_two_pixels 8888, 8, 8888, over +.endm + +.macro bilinear_over_8888_8_8888_process_four_pixels + bilinear_interpolate_two_pixels 8888, 8, 8888, over + bilinear_interpolate_two_pixels 8888, 8, 8888, over +.endm + +.macro bilinear_over_8888_8_8888_process_pixblock_head + bilinear_over_8888_8_8888_process_four_pixels +.endm + +.macro bilinear_over_8888_8_8888_process_pixblock_tail +.endm + +.macro bilinear_over_8888_8_8888_process_pixblock_tail_head + bilinear_over_8888_8_8888_process_pixblock_tail + bilinear_over_8888_8_8888_process_pixblock_head +.endm + +/* add_8888_8888 */ +.macro bilinear_add_8888_8888_process_last_pixel + bilinear_interpolate_last_pixel 8888, x, 8888, add +.endm + +.macro bilinear_add_8888_8888_process_two_pixels + bilinear_interpolate_two_pixels 8888, x, 8888, add +.endm + +.macro bilinear_add_8888_8888_process_four_pixels + bilinear_interpolate_two_pixels 8888, x, 8888, add + bilinear_interpolate_two_pixels 8888, x, 8888, add +.endm + +.macro bilinear_add_8888_8888_process_pixblock_head + bilinear_add_8888_8888_process_four_pixels +.endm + +.macro bilinear_add_8888_8888_process_pixblock_tail +.endm + +.macro bilinear_add_8888_8888_process_pixblock_tail_head + bilinear_add_8888_8888_process_pixblock_tail + bilinear_add_8888_8888_process_pixblock_head +.endm + +/* add_8888_8_8888 */ +.macro bilinear_add_8888_8_8888_process_last_pixel + bilinear_interpolate_last_pixel 8888, 8, 8888, add +.endm + +.macro bilinear_add_8888_8_8888_process_two_pixels + bilinear_interpolate_two_pixels 8888, 8, 8888, add +.endm + +.macro bilinear_add_8888_8_8888_process_four_pixels + bilinear_interpolate_four_pixels 8888, 8, 8888, add +.endm + +.macro bilinear_add_8888_8_8888_process_pixblock_head + bilinear_add_8888_8_8888_process_four_pixels +.endm + +.macro bilinear_add_8888_8_8888_process_pixblock_tail +.endm + +.macro bilinear_add_8888_8_8888_process_pixblock_tail_head + bilinear_add_8888_8_8888_process_pixblock_tail + bilinear_add_8888_8_8888_process_pixblock_head +.endm + + +/* Bilinear scanline functions */ +generate_bilinear_scanline_func \ + pixman_scaled_bilinear_scanline_8888_8_8888_SRC_asm_neon, \ + 8888, 8888, 2, 2, \ + bilinear_src_8888_8_8888_process_last_pixel, \ + bilinear_src_8888_8_8888_process_two_pixels, \ + bilinear_src_8888_8_8888_process_four_pixels, \ + bilinear_src_8888_8_8888_process_pixblock_head, \ + bilinear_src_8888_8_8888_process_pixblock_tail, \ + bilinear_src_8888_8_8888_process_pixblock_tail_head, \ + 4, 28, BILINEAR_FLAG_USE_MASK + +generate_bilinear_scanline_func \ + pixman_scaled_bilinear_scanline_8888_8_0565_SRC_asm_neon, \ + 8888, 0565, 2, 1, \ + bilinear_src_8888_8_0565_process_last_pixel, \ + bilinear_src_8888_8_0565_process_two_pixels, \ + bilinear_src_8888_8_0565_process_four_pixels, \ + bilinear_src_8888_8_0565_process_pixblock_head, \ + bilinear_src_8888_8_0565_process_pixblock_tail, \ + bilinear_src_8888_8_0565_process_pixblock_tail_head, \ + 4, 28, BILINEAR_FLAG_USE_MASK + +generate_bilinear_scanline_func \ + pixman_scaled_bilinear_scanline_0565_8_x888_SRC_asm_neon, \ + 0565, 8888, 1, 2, \ + bilinear_src_0565_8_x888_process_last_pixel, \ + bilinear_src_0565_8_x888_process_two_pixels, \ + bilinear_src_0565_8_x888_process_four_pixels, \ + bilinear_src_0565_8_x888_process_pixblock_head, \ + bilinear_src_0565_8_x888_process_pixblock_tail, \ + bilinear_src_0565_8_x888_process_pixblock_tail_head, \ + 4, 28, BILINEAR_FLAG_USE_MASK + +generate_bilinear_scanline_func \ + pixman_scaled_bilinear_scanline_0565_8_0565_SRC_asm_neon, \ + 0565, 0565, 1, 1, \ + bilinear_src_0565_8_0565_process_last_pixel, \ + bilinear_src_0565_8_0565_process_two_pixels, \ + bilinear_src_0565_8_0565_process_four_pixels, \ + bilinear_src_0565_8_0565_process_pixblock_head, \ + bilinear_src_0565_8_0565_process_pixblock_tail, \ + bilinear_src_0565_8_0565_process_pixblock_tail_head, \ + 4, 28, BILINEAR_FLAG_USE_MASK + +generate_bilinear_scanline_func \ + pixman_scaled_bilinear_scanline_8888_8888_OVER_asm_neon, \ + 8888, 8888, 2, 2, \ + bilinear_over_8888_8888_process_last_pixel, \ + bilinear_over_8888_8888_process_two_pixels, \ + bilinear_over_8888_8888_process_four_pixels, \ + bilinear_over_8888_8888_process_pixblock_head, \ + bilinear_over_8888_8888_process_pixblock_tail, \ + bilinear_over_8888_8888_process_pixblock_tail_head, \ + 4, 28, 0 + +generate_bilinear_scanline_func \ + pixman_scaled_bilinear_scanline_8888_8_8888_OVER_asm_neon, \ + 8888, 8888, 2, 2, \ + bilinear_over_8888_8_8888_process_last_pixel, \ + bilinear_over_8888_8_8888_process_two_pixels, \ + bilinear_over_8888_8_8888_process_four_pixels, \ + bilinear_over_8888_8_8888_process_pixblock_head, \ + bilinear_over_8888_8_8888_process_pixblock_tail, \ + bilinear_over_8888_8_8888_process_pixblock_tail_head, \ + 4, 28, BILINEAR_FLAG_USE_MASK + +generate_bilinear_scanline_func \ + pixman_scaled_bilinear_scanline_8888_8888_ADD_asm_neon, \ + 8888, 8888, 2, 2, \ + bilinear_add_8888_8888_process_last_pixel, \ + bilinear_add_8888_8888_process_two_pixels, \ + bilinear_add_8888_8888_process_four_pixels, \ + bilinear_add_8888_8888_process_pixblock_head, \ + bilinear_add_8888_8888_process_pixblock_tail, \ + bilinear_add_8888_8888_process_pixblock_tail_head, \ + 4, 28, 0 + +generate_bilinear_scanline_func \ + pixman_scaled_bilinear_scanline_8888_8_8888_ADD_asm_neon, \ + 8888, 8888, 2, 2, \ + bilinear_add_8888_8_8888_process_last_pixel, \ + bilinear_add_8888_8_8888_process_two_pixels, \ + bilinear_add_8888_8_8888_process_four_pixels, \ + bilinear_add_8888_8_8888_process_pixblock_head, \ + bilinear_add_8888_8_8888_process_pixblock_tail, \ + bilinear_add_8888_8_8888_process_pixblock_tail_head, \ + 4, 28, BILINEAR_FLAG_USE_MASK diff --git a/pixman/pixman-arma64-neon-asm.S b/pixman/pixman-arma64-neon-asm.S new file mode 100644 index 0000000..18ace0e --- /dev/null +++ b/pixman/pixman-arma64-neon-asm.S @@ -0,0 +1,3704 @@ +/* + * Copyright © 2009 Nokia Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + * + * Author: Siarhei Siamashka (siarhei.siamashka@nokia.com) + */ + +/* + * This file contains implementations of NEON optimized pixel processing + * functions. There is no full and detailed tutorial, but some functions + * (those which are exposing some new or interesting features) are + * extensively commented and can be used as examples. + * + * You may want to have a look at the comments for following functions: + * - pixman_composite_over_8888_0565_asm_neon + * - pixman_composite_over_n_8_0565_asm_neon + */ + +/* Prevent the stack from becoming executable for no reason... */ +#if defined(__linux__) && defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif + +.text +.arch armv8-a + +.altmacro +.p2align 2 + +#include "pixman-private.h" +#include "pixman-arm-asm.h" +#include "pixman-arma64-neon-asm.h" + +/* Global configuration options and preferences */ + +/* + * The code can optionally make use of unaligned memory accesses to improve + * performance of handling leading/trailing pixels for each scanline. + * Configuration variable RESPECT_STRICT_ALIGNMENT can be set to 0 for + * example in linux if unaligned memory accesses are not configured to + * generate.exceptions. + */ +.set RESPECT_STRICT_ALIGNMENT, 1 + +/* + * Set default prefetch type. There is a choice between the following options: + * + * PREFETCH_TYPE_NONE (may be useful for the ARM cores where PLD is set to work + * as NOP to workaround some HW bugs or for whatever other reason) + * + * PREFETCH_TYPE_SIMPLE (may be useful for simple single-issue ARM cores where + * advanced prefetch intruduces heavy overhead) + * + * PREFETCH_TYPE_ADVANCED (useful for superscalar cores such as ARM Cortex-A8 + * which can run ARM and NEON instructions simultaneously so that extra ARM + * instructions do not add (many) extra cycles, but improve prefetch efficiency) + * + * Note: some types of function can't support advanced prefetch and fallback + * to simple one (those which handle 24bpp pixels) + */ +.set PREFETCH_TYPE_DEFAULT, PREFETCH_TYPE_ADVANCED + +/* Prefetch distance in pixels for simple prefetch */ +.set PREFETCH_DISTANCE_SIMPLE, 64 + +/* + * Implementation of pixman_composite_over_8888_0565_asm_neon + * + * This function takes a8r8g8b8 source buffer, r5g6b5 destination buffer and + * performs OVER compositing operation. Function fast_composite_over_8888_0565 + * from pixman-fast-path.c does the same in C and can be used as a reference. + * + * First we need to have some NEON assembly code which can do the actual + * operation on the pixels and provide it to the template macro. + * + * Template macro quite conveniently takes care of emitting all the necessary + * code for memory reading and writing (including quite tricky cases of + * handling unaligned leading/trailing pixels), so we only need to deal with + * the data in NEON registers. + * + * NEON registers allocation in general is recommented to be the following: + * v0, v1, v2, v3 - contain loaded source pixel data + * v4, v5, v6, v7 - contain loaded destination pixels (if they are needed) + * v24, v25, v26, v27 - contain loading mask pixel data (if mask is used) + * v28, v29, v30, v31 - place for storing the result (destination pixels) + * + * As can be seen above, four 64-bit NEON registers are used for keeping + * intermediate pixel data and up to 8 pixels can be processed in one step + * for 32bpp formats (16 pixels for 16bpp, 32 pixels for 8bpp). + * + * This particular function uses the following registers allocation: + * v0, v1, v2, v3 - contain loaded source pixel data + * v4, v5 - contain loaded destination pixels (they are needed) + * v28, v29 - place for storing the result (destination pixels) + */ + +/* + * Step one. We need to have some code to do some arithmetics on pixel data. + * This is implemented as a pair of macros: '*_head' and '*_tail'. When used + * back-to-back, they take pixel data from {v0, v1, v2, v3} and {v4, v5}, + * perform all the needed calculations and write the result to {v28, v29}. + * The rationale for having two macros and not just one will be explained + * later. In practice, any single monolitic function which does the work can + * be split into two parts in any arbitrary way without affecting correctness. + * + * There is one special trick here too. Common template macro can optionally + * make our life a bit easier by doing R, G, B, A color components + * deinterleaving for 32bpp pixel formats (and this feature is used in + * 'pixman_composite_over_8888_0565_asm_neon' function). So it means that + * instead of having 8 packed pixels in {v0, v1, v2, v3} registers, we + * actually use v0 register for blue channel (a vector of eight 8-bit + * values), v1 register for green, v2 for red and v3 for alpha. This + * simple conversion can be also done with a few NEON instructions: + * + * Packed to planar conversion: // vuzp8 is a wrapper macro + * vuzp8 v0, v1 + * vuzp8 v2, v3 + * vuzp8 v1, v3 + * vuzp8 v0, v2 + * + * Planar to packed conversion: // vzip8 is a wrapper macro + * vzip8 v0, v2 + * vzip8 v1, v3 + * vzip8 v2, v3 + * vzip8 v0, v1 + * + * But pixel can be loaded directly in planar format using LD4 / b NEON + * instruction. It is 1 cycle slower than LD1 / s, so this is not always + * desirable, that's why deinterleaving is optional. + * + * But anyway, here is the code: + */ + +.macro pixman_composite_over_8888_0565_process_pixblock_head + /* convert 8 r5g6b5 pixel data from {v4} to planar 8-bit format + and put data into v6 - red, v7 - green, v30 - blue */ + mov v4.d[1], v5.d[0] + shrn v6.8b, v4.8h, #8 + shrn v7.8b, v4.8h, #3 + sli v4.8h, v4.8h, #5 + sri v6.8b, v6.8b, #5 + mvn v3.8b, v3.8b /* invert source alpha */ + sri v7.8b, v7.8b, #6 + shrn v30.8b, v4.8h, #2 + /* now do alpha blending, storing results in 8-bit planar format + into v20 - red, v23 - green, v22 - blue */ + umull v10.8h, v3.8b, v6.8b + umull v11.8h, v3.8b, v7.8b + umull v12.8h, v3.8b, v30.8b + urshr v17.8h, v10.8h, #8 + urshr v18.8h, v11.8h, #8 + urshr v19.8h, v12.8h, #8 + raddhn v20.8b, v10.8h, v17.8h + raddhn v23.8b, v11.8h, v18.8h + raddhn v22.8b, v12.8h, v19.8h +.endm + +.macro pixman_composite_over_8888_0565_process_pixblock_tail + /* ... continue alpha blending */ + uqadd v17.8b, v2.8b, v20.8b + uqadd v18.8b, v0.8b, v22.8b + uqadd v19.8b, v1.8b, v23.8b + /* convert the result to r5g6b5 and store it into {v14} */ + ushll v14.8h, v17.8b, #7 + sli v14.8h, v14.8h, #1 + ushll v8.8h, v19.8b, #7 + sli v8.8h, v8.8h, #1 + ushll v9.8h, v18.8b, #7 + sli v9.8h, v9.8h, #1 + sri v14.8h, v8.8h, #5 + sri v14.8h, v9.8h, #11 + mov v28.d[0], v14.d[0] + mov v29.d[0], v14.d[1] +.endm + +/* + * OK, now we got almost everything that we need. Using the above two + * macros, the work can be done right. But now we want to optimize + * it a bit. ARM Cortex-A8 is an in-order core, and benefits really + * a lot from good code scheduling and software pipelining. + * + * Let's construct some code, which will run in the core main loop. + * Some pseudo-code of the main loop will look like this: + * head + * while (...) { + * tail + * head + * } + * tail + * + * It may look a bit weird, but this setup allows to hide instruction + * latencies better and also utilize dual-issue capability more + * efficiently (make pairs of load-store and ALU instructions). + * + * So what we need now is a '*_tail_head' macro, which will be used + * in the core main loop. A trivial straightforward implementation + * of this macro would look like this: + * + * pixman_composite_over_8888_0565_process_pixblock_tail + * st1 {v28.4h, v29.4h}, [DST_W], #32 + * ld1 {v4.4h, v5.4h}, [DST_R], #16 + * ld4 {v0.2s, v1.2s, v2.2s, v3.2s}, [SRC], #32 + * pixman_composite_over_8888_0565_process_pixblock_head + * cache_preload 8, 8 + * + * Now it also got some VLD/VST instructions. We simply can't move from + * processing one block of pixels to the other one with just arithmetics. + * The previously processed data needs to be written to memory and new + * data needs to be fetched. Fortunately, this main loop does not deal + * with partial leading/trailing pixels and can load/store a full block + * of pixels in a bulk. Additionally, destination buffer is already + * 16 bytes aligned here (which is good for performance). + * + * New things here are DST_R, DST_W, SRC and MASK identifiers. These + * are the aliases for ARM registers which are used as pointers for + * accessing data. We maintain separate pointers for reading and writing + * destination buffer (DST_R and DST_W). + * + * Another new thing is 'cache_preload' macro. It is used for prefetching + * data into CPU L2 cache and improve performance when dealing with large + * images which are far larger than cache size. It uses one argument + * (actually two, but they need to be the same here) - number of pixels + * in a block. Looking into 'pixman-arm-neon-asm.h' can provide some + * details about this macro. Moreover, if good performance is needed + * the code from this macro needs to be copied into '*_tail_head' macro + * and mixed with the rest of code for optimal instructions scheduling. + * We are actually doing it below. + * + * Now after all the explanations, here is the optimized code. + * Different instruction streams (originaling from '*_head', '*_tail' + * and 'cache_preload' macro) use different indentation levels for + * better readability. Actually taking the code from one of these + * indentation levels and ignoring a few LD/ST instructions would + * result in exactly the code from '*_head', '*_tail' or 'cache_preload' + * macro! + */ + +#if 1 + +.macro pixman_composite_over_8888_0565_process_pixblock_tail_head + uqadd v17.8b, v2.8b, v20.8b + ld1 {v4.4h, v5.4h}, [DST_R], #16 + mov v4.d[1], v5.d[0] + uqadd v18.8b, v0.8b, v22.8b + uqadd v19.8b, v1.8b, v23.8b + shrn v6.8b, v4.8h, #8 + fetch_src_pixblock + shrn v7.8b, v4.8h, #3 + sli v4.8h, v4.8h, #5 + ushll v14.8h, v17.8b, #7 + sli v14.8h, v14.8h, #1 + PF add PF_X, PF_X, #8 + ushll v8.8h, v19.8b, #7 + sli v8.8h, v8.8h, #1 + PF tst PF_CTL, #0xF + sri v6.8b, v6.8b, #5 + PF beq 10f + PF add PF_X, PF_X, #8 +10: + mvn v3.8b, v3.8b + PF beq 10f + PF sub PF_CTL, PF_CTL, #1 +10: + sri v7.8b, v7.8b, #6 + shrn v30.8b, v4.8h, #2 + umull v10.8h, v3.8b, v6.8b + PF lsl DUMMY, PF_X, #src_bpp_shift + PF prfm PREFETCH_MODE, [PF_SRC, DUMMY] + umull v11.8h, v3.8b, v7.8b + umull v12.8h, v3.8b, v30.8b + PF lsl DUMMY, PF_X, #dst_bpp_shift + PF prfm PREFETCH_MODE, [PF_DST, DUMMY] + sri v14.8h, v8.8h, #5 + PF cmp PF_X, ORIG_W + ushll v9.8h, v18.8b, #7 + sli v9.8h, v9.8h, #1 + urshr v17.8h, v10.8h, #8 + PF ble 10f + PF sub PF_X, PF_X, ORIG_W +10: + urshr v19.8h, v11.8h, #8 + urshr v18.8h, v12.8h, #8 + PF ble 10f + PF subs PF_CTL, PF_CTL, #0x10 +10: + sri v14.8h, v9.8h, #11 + mov v28.d[0], v14.d[0] + mov v29.d[0], v14.d[1] + PF ble 10f + PF lsl DUMMY, SRC_STRIDE, #src_bpp_shift + PF ldrsb DUMMY, [PF_SRC, DUMMY] + PF add PF_SRC, PF_SRC, #1 +10: + raddhn v20.8b, v10.8h, v17.8h + raddhn v23.8b, v11.8h, v19.8h + PF ble 10f + PF lsl DUMMY, DST_STRIDE, #dst_bpp_shift + PF ldrsb DUMMY, [PF_DST, DUMMY] + PF add PF_DST, PF_SRC, #1 +10: + raddhn v22.8b, v12.8h, v18.8h + st1 {v14.8h}, [DST_W], #16 +.endm + +#else + +/* If we did not care much about the performance, we would just use this... */ +.macro pixman_composite_over_8888_0565_process_pixblock_tail_head + pixman_composite_over_8888_0565_process_pixblock_tail + st1 {v14.8h}, [DST_W], #16 + ld1 {v4.4h, v4.5h}, [DST_R], #16 + fetch_src_pixblock + pixman_composite_over_8888_0565_process_pixblock_head + cache_preload 8, 8 +.endm + +#endif + +/* + * And now the final part. We are using 'generate_composite_function' macro + * to put all the stuff together. We are specifying the name of the function + * which we want to get, number of bits per pixel for the source, mask and + * destination (0 if unused, like mask in this case). Next come some bit + * flags: + * FLAG_DST_READWRITE - tells that the destination buffer is both read + * and written, for write-only buffer we would use + * FLAG_DST_WRITEONLY flag instead + * FLAG_DEINTERLEAVE_32BPP - tells that we prefer to work with planar data + * and separate color channels for 32bpp format. + * The next things are: + * - the number of pixels processed per iteration (8 in this case, because + * that's the maximum what can fit into four 64-bit NEON registers). + * - prefetch distance, measured in pixel blocks. In this case it is 5 times + * by 8 pixels. That would be 40 pixels, or up to 160 bytes. Optimal + * prefetch distance can be selected by running some benchmarks. + * + * After that we specify some macros, these are 'default_init', + * 'default_cleanup' here which are empty (but it is possible to have custom + * init/cleanup macros to be able to save/restore some extra NEON registers + * like d8-d15 or do anything else) followed by + * 'pixman_composite_over_8888_0565_process_pixblock_head', + * 'pixman_composite_over_8888_0565_process_pixblock_tail' and + * 'pixman_composite_over_8888_0565_process_pixblock_tail_head' + * which we got implemented above. + * + * The last part is the NEON registers allocation scheme. + */ +generate_composite_function \ + pixman_composite_over_8888_0565_asm_neon, 32, 0, 16, \ + FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \ + 8, /* number of pixels, processed in a single block */ \ + 5, /* prefetch distance */ \ + default_init, \ + default_cleanup, \ + pixman_composite_over_8888_0565_process_pixblock_head, \ + pixman_composite_over_8888_0565_process_pixblock_tail, \ + pixman_composite_over_8888_0565_process_pixblock_tail_head, \ + 28, /* dst_w_basereg */ \ + 4, /* dst_r_basereg */ \ + 0, /* src_basereg */ \ + 24 /* mask_basereg */ + +/******************************************************************************/ + +.macro pixman_composite_over_n_0565_process_pixblock_head + /* convert 8 r5g6b5 pixel data from {v4} to planar 8-bit format + and put data into v6 - red, v7 - green, v30 - blue */ + mov v4.d[1], v5.d[0] + shrn v6.8b, v4.8h, #8 + shrn v7.8b, v4.8h, #3 + sli v4.8h, v4.8h, #5 + sri v6.8b, v6.8b, #5 + sri v7.8b, v7.8b, #6 + shrn v30.8b, v4.8h, #2 + /* now do alpha blending, storing results in 8-bit planar format + into v20 - red, v23 - green, v22 - blue */ + umull v10.8h, v3.8b, v6.8b + umull v11.8h, v3.8b, v7.8b + umull v12.8h, v3.8b, v30.8b + urshr v13.8h, v10.8h, #8 + urshr v14.8h, v11.8h, #8 + urshr v15.8h, v12.8h, #8 + raddhn v20.8b, v10.8h, v13.8h + raddhn v23.8b, v11.8h, v14.8h + raddhn v22.8b, v12.8h, v15.8h +.endm + +.macro pixman_composite_over_n_0565_process_pixblock_tail + /* ... continue alpha blending */ + uqadd v17.8b, v2.8b, v20.8b + uqadd v18.8b, v0.8b, v22.8b + uqadd v19.8b, v1.8b, v23.8b + /* convert the result to r5g6b5 and store it into {v14} */ + ushll v14.8h, v17.8b, #7 + sli v14.8h, v14.8h, #1 + ushll v8.8h, v19.8b, #7 + sli v8.8h, v8.8h, #1 + ushll v9.8h, v18.8b, #7 + sli v9.8h, v9.8h, #1 + sri v14.8h, v8.8h, #5 + sri v14.8h, v9.8h, #11 + mov v28.d[0], v14.d[0] + mov v29.d[0], v14.d[1] +.endm + +/* TODO: expand macros and do better instructions scheduling */ +.macro pixman_composite_over_n_0565_process_pixblock_tail_head + pixman_composite_over_n_0565_process_pixblock_tail + ld1 {v4.4h, v5.4h}, [DST_R], #16 + st1 {v14.8h}, [DST_W], #16 + pixman_composite_over_n_0565_process_pixblock_head + cache_preload 8, 8 +.endm + +.macro pixman_composite_over_n_0565_init + mov v3.s[0], w4 + dup v0.8b, v3.b[0] + dup v1.8b, v3.b[1] + dup v2.8b, v3.b[2] + dup v3.8b, v3.b[3] + mvn v3.8b, v3.8b /* invert source alpha */ +.endm + +generate_composite_function \ + pixman_composite_over_n_0565_asm_neon, 0, 0, 16, \ + FLAG_DST_READWRITE, \ + 8, /* number of pixels, processed in a single block */ \ + 5, /* prefetch distance */ \ + pixman_composite_over_n_0565_init, \ + default_cleanup, \ + pixman_composite_over_n_0565_process_pixblock_head, \ + pixman_composite_over_n_0565_process_pixblock_tail, \ + pixman_composite_over_n_0565_process_pixblock_tail_head, \ + 28, /* dst_w_basereg */ \ + 4, /* dst_r_basereg */ \ + 0, /* src_basereg */ \ + 24 /* mask_basereg */ + +/******************************************************************************/ + +.macro pixman_composite_src_8888_0565_process_pixblock_head + ushll v8.8h, v1.8b, #7 + sli v8.8h, v8.8h, #1 + ushll v14.8h, v2.8b, #7 + sli v14.8h, v14.8h, #1 + ushll v9.8h, v0.8b, #7 + sli v9.8h, v9.8h, #1 +.endm + +.macro pixman_composite_src_8888_0565_process_pixblock_tail + sri v14.8h, v8.8h, #5 + sri v14.8h, v9.8h, #11 + mov v28.d[0], v14.d[0] + mov v29.d[0], v14.d[1] +.endm + +.macro pixman_composite_src_8888_0565_process_pixblock_tail_head + sri v14.8h, v8.8h, #5 + PF add PF_X, PF_X, #8 + PF tst PF_CTL, #0xF + fetch_src_pixblock + PF beq 10f + PF add PF_X, PF_X, #8 + PF sub PF_CTL, PF_CTL, #1 +10: + sri v14.8h, v9.8h, #11 + mov v28.d[0], v14.d[0] + mov v29.d[0], v14.d[1] + PF cmp PF_X, ORIG_W + PF lsl DUMMY, PF_X, #src_bpp_shift + PF prfm PREFETCH_MODE, [PF_SRC, DUMMY] + ushll v8.8h, v1.8b, #7 + sli v8.8h, v8.8h, #1 + st1 {v14.8h}, [DST_W], #16 + PF ble 10f + PF sub PF_X, PF_X, ORIG_W + PF subs PF_CTL, PF_CTL, #0x10 +10: + ushll v14.8h, v2.8b, #7 + sli v14.8h, v14.8h, #1 + PF ble 10f + PF lsl DUMMY, SRC_STRIDE, #src_bpp_shift + PF ldrsb DUMMY, [PF_SRC, DUMMY] + PF add PF_SRC, PF_SRC, #1 +10: + ushll v9.8h, v0.8b, #7 + sli v9.8h, v9.8h, #1 +.endm + +generate_composite_function \ + pixman_composite_src_8888_0565_asm_neon, 32, 0, 16, \ + FLAG_DST_WRITEONLY | FLAG_DEINTERLEAVE_32BPP, \ + 8, /* number of pixels, processed in a single block */ \ + 10, /* prefetch distance */ \ + default_init, \ + default_cleanup, \ + pixman_composite_src_8888_0565_process_pixblock_head, \ + pixman_composite_src_8888_0565_process_pixblock_tail, \ + pixman_composite_src_8888_0565_process_pixblock_tail_head + +/******************************************************************************/ + +.macro pixman_composite_src_0565_8888_process_pixblock_head + mov v0.d[1], v1.d[0] + shrn v30.8b, v0.8h, #8 + shrn v29.8b, v0.8h, #3 + sli v0.8h, v0.8h, #5 + movi v31.8b, #255 + sri v30.8b, v30.8b, #5 + sri v29.8b, v29.8b, #6 + shrn v28.8b, v0.8h, #2 +.endm + +.macro pixman_composite_src_0565_8888_process_pixblock_tail +.endm + +/* TODO: expand macros and do better instructions scheduling */ +.macro pixman_composite_src_0565_8888_process_pixblock_tail_head + pixman_composite_src_0565_8888_process_pixblock_tail + st4 {v28.8b, v29.8b, v30.8b, v31.8b}, [DST_W], #32 + fetch_src_pixblock + pixman_composite_src_0565_8888_process_pixblock_head + cache_preload 8, 8 +.endm + +generate_composite_function \ + pixman_composite_src_0565_8888_asm_neon, 16, 0, 32, \ + FLAG_DST_WRITEONLY | FLAG_DEINTERLEAVE_32BPP, \ + 8, /* number of pixels, processed in a single block */ \ + 10, /* prefetch distance */ \ + default_init, \ + default_cleanup, \ + pixman_composite_src_0565_8888_process_pixblock_head, \ + pixman_composite_src_0565_8888_process_pixblock_tail, \ + pixman_composite_src_0565_8888_process_pixblock_tail_head + +/******************************************************************************/ + +.macro pixman_composite_add_8_8_process_pixblock_head + uqadd v28.8b, v0.8b, v4.8b + uqadd v29.8b, v1.8b, v5.8b + uqadd v30.8b, v2.8b, v6.8b + uqadd v31.8b, v3.8b, v7.8b +.endm + +.macro pixman_composite_add_8_8_process_pixblock_tail +.endm + +.macro pixman_composite_add_8_8_process_pixblock_tail_head + fetch_src_pixblock + PF add PF_X, PF_X, #32 + PF tst PF_CTL, #0xF + ld1 {v4.8b, v5.8b, v6.8b, v7.8b}, [DST_R], #32 + PF beq 10f + PF add PF_X, PF_X, #32 + PF sub PF_CTL, PF_CTL, #1 +10: + st1 {v28.8b, v29.8b, v30.8b, v31.8b}, [DST_W], #32 + PF cmp PF_X, ORIG_W + PF lsl DUMMY, PF_X, #src_bpp_shift + PF prfm PREFETCH_MODE, [PF_SRC, DUMMY] + PF lsl DUMMY, PF_X, #dst_bpp_shift + PF prfm PREFETCH_MODE, [PF_DST, DUMMY] + PF ble 10f + PF sub PF_X, PF_X, ORIG_W + PF subs PF_CTL, PF_CTL, #0x10 +10: + uqadd v28.8b, v0.8b, v4.8b + PF ble 10f + PF lsl DUMMY, SRC_STRIDE, #src_bpp_shift + PF ldrsb DUMMY, [PF_SRC, DUMMY] + PF add PF_SRC, PF_SRC, #1 + PF lsl DUMMY, DST_STRIDE, #dst_bpp_shift + PF ldrsb DUMMY, [PF_DST, DUMMY] + PF add PF_DST, PF_DST, #1 +10: + uqadd v29.8b, v1.8b, v5.8b + uqadd v30.8b, v2.8b, v6.8b + uqadd v31.8b, v3.8b, v7.8b +.endm + +generate_composite_function \ + pixman_composite_add_8_8_asm_neon, 8, 0, 8, \ + FLAG_DST_READWRITE, \ + 32, /* number of pixels, processed in a single block */ \ + 10, /* prefetch distance */ \ + default_init, \ + default_cleanup, \ + pixman_composite_add_8_8_process_pixblock_head, \ + pixman_composite_add_8_8_process_pixblock_tail, \ + pixman_composite_add_8_8_process_pixblock_tail_head + +/******************************************************************************/ + +.macro pixman_composite_add_8888_8888_process_pixblock_tail_head + fetch_src_pixblock + PF add PF_X, PF_X, #8 + PF tst PF_CTL, #0xF + ld1 {v4.8b, v5.8b, v6.8b, v7.8b}, [DST_R], #32 + PF beq 10f + PF add PF_X, PF_X, #8 + PF sub PF_CTL, PF_CTL, #1 +10: + st1 {v28.8b, v29.8b, v30.8b, v31.8b}, [DST_W], #32 + PF cmp PF_X, ORIG_W + PF lsl DUMMY, PF_X, #src_bpp_shift + PF prfm PREFETCH_MODE, [PF_SRC, DUMMY] + PF lsl DUMMY, PF_X, #dst_bpp_shift + PF prfm PREFETCH_MODE, [PF_DST, DUMMY] + PF ble 10f + PF sub PF_X, PF_X, ORIG_W + PF subs PF_CTL, PF_CTL, #0x10 +10: + uqadd v28.8b, v0.8b, v4.8b + PF ble 10f + PF lsl DUMMY, SRC_STRIDE, #src_bpp_shift + PF ldrsb DUMMY, [PF_SRC, DUMMY] + PF add PF_SRC, PF_SRC, #1 + PF lsl DUMMY, DST_STRIDE, #dst_bpp_shift + PF ldrsb DUMMY, [PF_DST, DUMMY] + PF add PF_DST, PF_DST, #1 +10: + uqadd v29.8b, v1.8b, v5.8b + uqadd v30.8b, v2.8b, v6.8b + uqadd v31.8b, v3.8b, v7.8b +.endm + +generate_composite_function \ + pixman_composite_add_8888_8888_asm_neon, 32, 0, 32, \ + FLAG_DST_READWRITE, \ + 8, /* number of pixels, processed in a single block */ \ + 10, /* prefetch distance */ \ + default_init, \ + default_cleanup, \ + pixman_composite_add_8_8_process_pixblock_head, \ + pixman_composite_add_8_8_process_pixblock_tail, \ + pixman_composite_add_8888_8888_process_pixblock_tail_head + +generate_composite_function_single_scanline \ + pixman_composite_scanline_add_asm_neon, 32, 0, 32, \ + FLAG_DST_READWRITE, \ + 8, /* number of pixels, processed in a single block */ \ + default_init, \ + default_cleanup, \ + pixman_composite_add_8_8_process_pixblock_head, \ + pixman_composite_add_8_8_process_pixblock_tail, \ + pixman_composite_add_8888_8888_process_pixblock_tail_head + +/******************************************************************************/ + +.macro pixman_composite_out_reverse_8888_8888_process_pixblock_head + mvn v24.8b, v3.8b /* get inverted alpha */ + /* do alpha blending */ + umull v8.8h, v24.8b, v4.8b + umull v9.8h, v24.8b, v5.8b + umull v10.8h, v24.8b, v6.8b + umull v11.8h, v24.8b, v7.8b +.endm + +.macro pixman_composite_out_reverse_8888_8888_process_pixblock_tail + urshr v14.8h, v8.8h, #8 + urshr v15.8h, v9.8h, #8 + urshr v16.8h, v10.8h, #8 + urshr v17.8h, v11.8h, #8 + raddhn v28.8b, v14.8h, v8.8h + raddhn v29.8b, v15.8h, v9.8h + raddhn v30.8b, v16.8h, v10.8h + raddhn v31.8b, v17.8h, v11.8h +.endm + +.macro pixman_composite_out_reverse_8888_8888_process_pixblock_tail_head + ld4 {v4.8b, v5.8b, v6.8b, v7.8b}, [DST_R], #32 + urshr v14.8h, v8.8h, #8 + PF add PF_X, PF_X, #8 + PF tst PF_CTL, #0xF + urshr v15.8h, v9.8h, #8 + urshr v16.8h, v10.8h, #8 + urshr v17.8h, v11.8h, #8 + PF beq 10f + PF add PF_X, PF_X, #8 + PF sub PF_CTL, PF_CTL, #1 +10: + raddhn v28.8b, v14.8h, v8.8h + raddhn v29.8b, v15.8h, v9.8h + PF cmp PF_X, ORIG_W + raddhn v30.8b, v16.8h, v10.8h + raddhn v31.8b, v17.8h, v11.8h + fetch_src_pixblock + PF lsl DUMMY, PF_X, #src_bpp_shift + PF prfm PREFETCH_MODE, [PF_SRC, DUMMY] + mvn v22.8b, v3.8b + PF lsl DUMMY, PF_X, #dst_bpp_shift + PF prfm PREFETCH_MODE, [PF_DST, DUMMY] + st4 {v28.8b, v29.8b, v30.8b, v31.8b}, [DST_W], #32 + PF ble 10f + PF sub PF_X, PF_X, ORIG_W +10: + umull v8.8h, v22.8b, v4.8b + PF ble 10f + PF subs PF_CTL, PF_CTL, #0x10 +10: + umull v9.8h, v22.8b, v5.8b + PF ble 10f + PF lsl DUMMY, SRC_STRIDE, #src_bpp_shift + PF ldrsb DUMMY, [PF_SRC, DUMMY] + PF add PF_SRC, PF_SRC, #1 +10: + umull v10.8h, v22.8b, v6.8b + PF ble 10f + PF lsl DUMMY, DST_STRIDE, #dst_bpp_shift + PF ldrsb DUMMY, [PF_DST, DUMMY] + PF add PF_DST, PF_DST, #1 +10: + umull v11.8h, v22.8b, v7.8b +.endm + +generate_composite_function_single_scanline \ + pixman_composite_scanline_out_reverse_asm_neon, 32, 0, 32, \ + FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \ + 8, /* number of pixels, processed in a single block */ \ + default_init, \ + default_cleanup, \ + pixman_composite_out_reverse_8888_8888_process_pixblock_head, \ + pixman_composite_out_reverse_8888_8888_process_pixblock_tail, \ + pixman_composite_out_reverse_8888_8888_process_pixblock_tail_head + +/******************************************************************************/ + +.macro pixman_composite_over_8888_8888_process_pixblock_head + pixman_composite_out_reverse_8888_8888_process_pixblock_head +.endm + +.macro pixman_composite_over_8888_8888_process_pixblock_tail + pixman_composite_out_reverse_8888_8888_process_pixblock_tail + uqadd v28.8b, v0.8b, v28.8b + uqadd v29.8b, v1.8b, v29.8b + uqadd v30.8b, v2.8b, v30.8b + uqadd v31.8b, v3.8b, v31.8b +.endm + +.macro pixman_composite_over_8888_8888_process_pixblock_tail_head + ld4 {v4.8b, v5.8b, v6.8b, v7.8b}, [DST_R], #32 + urshr v14.8h, v8.8h, #8 + PF add PF_X, PF_X, #8 + PF tst PF_CTL, #0xF + urshr v15.8h, v9.8h, #8 + urshr v16.8h, v10.8h, #8 + urshr v17.8h, v11.8h, #8 + PF beq 10f + PF add PF_X, PF_X, #8 + PF sub PF_CTL, PF_CTL, #1 +10: + raddhn v28.8b, v14.8h, v8.8h + raddhn v29.8b, v15.8h, v9.8h + PF cmp PF_X, ORIG_W + raddhn v30.8b, v16.8h, v10.8h + raddhn v31.8b, v17.8h, v11.8h + uqadd v28.8b, v0.8b, v28.8b + uqadd v29.8b, v1.8b, v29.8b + uqadd v30.8b, v2.8b, v30.8b + uqadd v31.8b, v3.8b, v31.8b + fetch_src_pixblock + PF lsl DUMMY, PF_X, #src_bpp_shift + PF prfm PREFETCH_MODE, [PF_SRC, DUMMY] + mvn v22.8b, v3.8b + PF lsl DUMMY, PF_X, #dst_bpp_shift + PF prfm PREFETCH_MODE, [PF_DST, DUMMY] + st4 {v28.8b, v29.8b, v30.8b, v31.8b}, [DST_W], #32 + PF ble 10f + PF sub PF_X, PF_X, ORIG_W +10: + umull v8.8h, v22.8b, v4.8b + PF ble 10f + PF subs PF_CTL, PF_CTL, #0x10 +10: + umull v9.8h, v22.8b, v5.8b + PF ble 10f + PF lsl DUMMY, SRC_STRIDE, #src_bpp_shift + PF ldrsb DUMMY, [PF_SRC, DUMMY] + PF add PF_SRC, PF_SRC, #1 +10: + umull v10.8h, v22.8b, v6.8b + PF ble 10f + PF lsl DUMMY, DST_STRIDE, #dst_bpp_shift + PF ldrsb DUMMY, [PF_DST, DUMMY] + PF add PF_DST, PF_DST, #1 +10: + umull v11.8h, v22.8b, v7.8b +.endm + +generate_composite_function \ + pixman_composite_over_8888_8888_asm_neon, 32, 0, 32, \ + FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \ + 8, /* number of pixels, processed in a single block */ \ + 5, /* prefetch distance */ \ + default_init, \ + default_cleanup, \ + pixman_composite_over_8888_8888_process_pixblock_head, \ + pixman_composite_over_8888_8888_process_pixblock_tail, \ + pixman_composite_over_8888_8888_process_pixblock_tail_head + +generate_composite_function_single_scanline \ + pixman_composite_scanline_over_asm_neon, 32, 0, 32, \ + FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \ + 8, /* number of pixels, processed in a single block */ \ + default_init, \ + default_cleanup, \ + pixman_composite_over_8888_8888_process_pixblock_head, \ + pixman_composite_over_8888_8888_process_pixblock_tail, \ + pixman_composite_over_8888_8888_process_pixblock_tail_head + +/******************************************************************************/ + +.macro pixman_composite_over_n_8888_process_pixblock_head + /* deinterleaved source pixels in {v0, v1, v2, v3} */ + /* inverted alpha in {v24} */ + /* destination pixels in {v4, v5, v6, v7} */ + umull v8.8h, v24.8b, v4.8b + umull v9.8h, v24.8b, v5.8b + umull v10.8h, v24.8b, v6.8b + umull v11.8h, v24.8b, v7.8b +.endm + +.macro pixman_composite_over_n_8888_process_pixblock_tail + urshr v14.8h, v8.8h, #8 + urshr v15.8h, v9.8h, #8 + urshr v16.8h, v10.8h, #8 + urshr v17.8h, v11.8h, #8 + raddhn v28.8b, v14.8h, v8.8h + raddhn v29.8b, v15.8h, v9.8h + raddhn v30.8b, v16.8h, v10.8h + raddhn v31.8b, v17.8h, v11.8h + uqadd v28.8b, v0.8b, v28.8b + uqadd v29.8b, v1.8b, v29.8b + uqadd v30.8b, v2.8b, v30.8b + uqadd v31.8b, v3.8b, v31.8b +.endm + +.macro pixman_composite_over_n_8888_process_pixblock_tail_head + urshr v14.8h, v8.8h, #8 + urshr v15.8h, v9.8h, #8 + urshr v16.8h, v10.8h, #8 + urshr v17.8h, v11.8h, #8 + raddhn v28.8b, v14.8h, v8.8h + raddhn v29.8b, v15.8h, v9.8h + raddhn v30.8b, v16.8h, v10.8h + raddhn v31.8b, v17.8h, v11.8h + ld4 {v4.8b, v5.8b, v6.8b, v7.8b}, [DST_R], #32 + uqadd v28.8b, v0.8b, v28.8b + PF add PF_X, PF_X, #8 + PF tst PF_CTL, #0x0F + PF beq 10f + PF add PF_X, PF_X, #8 + PF sub PF_CTL, PF_CTL, #1 +10: + uqadd v29.8b, v1.8b, v29.8b + uqadd v30.8b, v2.8b, v30.8b + uqadd v31.8b, v3.8b, v31.8b + PF cmp PF_X, ORIG_W + umull v8.8h, v24.8b, v4.8b + PF lsl DUMMY, PF_X, #dst_bpp_shift + PF prfm PREFETCH_MODE, [PF_DST, DUMMY] + umull v9.8h, v24.8b, v5.8b + PF ble 10f + PF sub PF_X, PF_X, ORIG_W +10: + umull v10.8h, v24.8b, v6.8b + PF subs PF_CTL, PF_CTL, #0x10 + umull v11.8h, v24.8b, v7.8b + PF ble 10f + PF lsl DUMMY, DST_STRIDE, #dst_bpp_shift + PF ldrsb DUMMY, [PF_DST, DUMMY] + PF add PF_DST, PF_DST, #1 +10: + st4 {v28.8b, v29.8b, v30.8b, v31.8b}, [DST_W], #32 +.endm + +.macro pixman_composite_over_n_8888_init + mov v3.s[0], w4 + dup v0.8b, v3.b[0] + dup v1.8b, v3.b[1] + dup v2.8b, v3.b[2] + dup v3.8b, v3.b[3] + mvn v24.8b, v3.8b /* get inverted alpha */ +.endm + +generate_composite_function \ + pixman_composite_over_n_8888_asm_neon, 0, 0, 32, \ + FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \ + 8, /* number of pixels, processed in a single block */ \ + 5, /* prefetch distance */ \ + pixman_composite_over_n_8888_init, \ + default_cleanup, \ + pixman_composite_over_8888_8888_process_pixblock_head, \ + pixman_composite_over_8888_8888_process_pixblock_tail, \ + pixman_composite_over_n_8888_process_pixblock_tail_head + +/******************************************************************************/ + +.macro pixman_composite_over_reverse_n_8888_process_pixblock_tail_head + urshr v14.8h, v8.8h, #8 + PF add PF_X, PF_X, #8 + PF tst PF_CTL, #0xF + urshr v15.8h, v9.8h, #8 + urshr v12.8h, v10.8h, #8 + urshr v13.8h, v11.8h, #8 + PF beq 10f + PF add PF_X, PF_X, #8 + PF sub PF_CTL, PF_CTL, #1 +10: + raddhn v28.8b, v14.8h, v8.8h + raddhn v29.8b, v15.8h, v9.8h + PF cmp PF_X, ORIG_W + raddhn v30.8b, v12.8h, v10.8h + raddhn v31.8b, v13.8h, v11.8h + uqadd v28.8b, v0.8b, v28.8b + uqadd v29.8b, v1.8b, v29.8b + uqadd v30.8b, v2.8b, v30.8b + uqadd v31.8b, v3.8b, v31.8b + ld4 {v0.8b, v1.8b, v2.8b, v3.8b}, [DST_R], #32 + mvn v22.8b, v3.8b + PF lsl DUMMY, PF_X, #dst_bpp_shift + PF prfm PREFETCH_MODE, [PF_DST, DUMMY] + st4 {v28.8b, v29.8b, v30.8b, v31.8b}, [DST_W], #32 + PF blt 10f + PF sub PF_X, PF_X, ORIG_W +10: + umull v8.8h, v22.8b, v4.8b + PF blt 10f + PF subs PF_CTL, PF_CTL, #0x10 +10: + umull v9.8h, v22.8b, v5.8b + umull v10.8h, v22.8b, v6.8b + PF blt 10f + PF lsl DUMMY, DST_STRIDE, #dst_bpp_shift + PF ldrsb DUMMY, [PF_DST, DUMMY] + PF add PF_DST, PF_DST, #1 +10: + umull v11.8h, v22.8b, v7.8b +.endm + +.macro pixman_composite_over_reverse_n_8888_init + mov v7.s[0], w4 + dup v4.8b, v7.b[0] + dup v5.8b, v7.b[1] + dup v6.8b, v7.b[2] + dup v7.8b, v7.b[3] +.endm + +generate_composite_function \ + pixman_composite_over_reverse_n_8888_asm_neon, 0, 0, 32, \ + FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \ + 8, /* number of pixels, processed in a single block */ \ + 5, /* prefetch distance */ \ + pixman_composite_over_reverse_n_8888_init, \ + default_cleanup, \ + pixman_composite_over_8888_8888_process_pixblock_head, \ + pixman_composite_over_8888_8888_process_pixblock_tail, \ + pixman_composite_over_reverse_n_8888_process_pixblock_tail_head, \ + 28, /* dst_w_basereg */ \ + 0, /* dst_r_basereg */ \ + 4, /* src_basereg */ \ + 24 /* mask_basereg */ + +/******************************************************************************/ + +.macro pixman_composite_over_8888_8_0565_process_pixblock_head + umull v0.8h, v24.8b, v8.8b /* IN for SRC pixels (part1) */ + umull v1.8h, v24.8b, v9.8b + umull v2.8h, v24.8b, v10.8b + umull v3.8h, v24.8b, v11.8b + mov v4.d[1], v5.d[0] + shrn v25.8b, v4.8h, #8 /* convert DST_R data to 32-bpp (part1) */ + shrn v26.8b, v4.8h, #3 + sli v4.8h, v4.8h, #5 + urshr v17.8h, v0.8h, #8 /* IN for SRC pixels (part2) */ + urshr v18.8h, v1.8h, #8 + urshr v19.8h, v2.8h, #8 + urshr v20.8h, v3.8h, #8 + raddhn v0.8b, v0.8h, v17.8h + raddhn v1.8b, v1.8h, v18.8h + raddhn v2.8b, v2.8h, v19.8h + raddhn v3.8b, v3.8h, v20.8h + sri v25.8b, v25.8b, #5 /* convert DST_R data to 32-bpp (part2) */ + sri v26.8b, v26.8b, #6 + mvn v3.8b, v3.8b + shrn v30.8b, v4.8h, #2 + umull v18.8h, v3.8b, v25.8b /* now do alpha blending */ + umull v19.8h, v3.8b, v26.8b + umull v20.8h, v3.8b, v30.8b +.endm + +.macro pixman_composite_over_8888_8_0565_process_pixblock_tail + /* 3 cycle bubble (after vmull.u8) */ + urshr v5.8h, v18.8h, #8 + urshr v6.8h, v19.8h, #8 + urshr v7.8h, v20.8h, #8 + raddhn v17.8b, v18.8h, v5.8h + raddhn v19.8b, v19.8h, v6.8h + raddhn v18.8b, v20.8h, v7.8h + uqadd v5.8b, v2.8b, v17.8b + /* 1 cycle bubble */ + uqadd v6.8b, v0.8b, v18.8b + uqadd v7.8b, v1.8b, v19.8b + ushll v14.8h, v5.8b, #7 /* convert to 16bpp */ + sli v14.8h, v14.8h, #1 + ushll v18.8h, v7.8b, #7 + sli v18.8h, v18.8h, #1 + ushll v19.8h, v6.8b, #7 + sli v19.8h, v19.8h, #1 + sri v14.8h, v18.8h, #5 + /* 1 cycle bubble */ + sri v14.8h, v19.8h, #11 + mov v28.d[0], v14.d[0] + mov v29.d[0], v14.d[1] +.endm + +.macro pixman_composite_over_8888_8_0565_process_pixblock_tail_head +#if 0 + ld1 {v4.8h}, [DST_R], #16 + shrn v25.8b, v4.8h, #8 + fetch_mask_pixblock + shrn v26.8b, v4.8h, #3 + fetch_src_pixblock + umull v22.8h, v24.8b, v10.8b + urshr v13.8h, v18.8h, #8 + urshr v11.8h, v19.8h, #8 + urshr v15.8h, v20.8h, #8 + raddhn v17.8b, v18.8h, v13.8h + raddhn v19.8b, v19.8h, v11.8h + raddhn v18.8b, v20.8h, v15.8h + uqadd v17.8b, v2.8b, v17.8b + umull v21.8h, v24.8b, v9.8b + uqadd v18.8b, v0.8b, v18.8b + uqadd v19.8b, v1.8b, v19.8b + ushll v14.8h, v17.8b, #7 + sli v14.8h, v14.8h, #1 + umull v20.8h, v24.8b, v8.8b + ushll v18.8h, v18.8b, #7 + sli v18.8h, v18.8h, #1 + ushll v19.8h, v19.8b, #7 + sli v19.8h, v19.8h, #1 + sri v14.8h, v18.8h, #5 + umull v23.8h, v24.8b, v11.8b + sri v14.8h, v19.8h, #11 + mov v28.d[0], v14.d[0] + mov v29.d[0], v14.d[1] + + cache_preload 8, 8 + + sli v4.8h, v4.8h, #5 + urshr v16.8h, v20.8h, #8 + urshr v17.8h, v21.8h, #8 + urshr v18.8h, v22.8h, #8 + urshr v19.8h, v23.8h, #8 + raddhn v0.8b, v20.8h, v16.8h + raddhn v1.8b, v21.8h, v17.8h + raddhn v2.8b, v22.8h, v18.8h + raddhn v3.8b, v23.8h, v19.8h + sri v25.8b, v25.8b, #5 + sri v26.8b, v26.8b, #6 + mvn v3.8b, v3.8b + shrn v30.8b, v4.8h, #2 + st1 {v14.8h}, [DST_W], #16 + umull v18.8h, v3.8b, v25.8b + umull v19.8h, v3.8b, v26.8b + umull v20.8h, v3.8b, v30.8b +#else + pixman_composite_over_8888_8_0565_process_pixblock_tail + st1 {v28.4h, v29.4h}, [DST_W], #16 + ld1 {v4.4h, v5.4h}, [DST_R], #16 + fetch_mask_pixblock + fetch_src_pixblock + pixman_composite_over_8888_8_0565_process_pixblock_head +#endif +.endm + +generate_composite_function \ + pixman_composite_over_8888_8_0565_asm_neon, 32, 8, 16, \ + FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \ + 8, /* number of pixels, processed in a single block */ \ + 5, /* prefetch distance */ \ + default_init_need_all_regs, \ + default_cleanup_need_all_regs, \ + pixman_composite_over_8888_8_0565_process_pixblock_head, \ + pixman_composite_over_8888_8_0565_process_pixblock_tail, \ + pixman_composite_over_8888_8_0565_process_pixblock_tail_head, \ + 28, /* dst_w_basereg */ \ + 4, /* dst_r_basereg */ \ + 8, /* src_basereg */ \ + 24 /* mask_basereg */ + +/******************************************************************************/ + +/* + * This function needs a special initialization of solid mask. + * Solid source pixel data is fetched from stack at ARGS_STACK_OFFSET + * offset, split into color components and replicated in d8-d11 + * registers. Additionally, this function needs all the NEON registers, + * so it has to save d8-d15 registers which are callee saved according + * to ABI. These registers are restored from 'cleanup' macro. All the + * other NEON registers are caller saved, so can be clobbered freely + * without introducing any problems. + */ +.macro pixman_composite_over_n_8_0565_init + mov v11.s[0], w4 + dup v8.8b, v11.b[0] + dup v9.8b, v11.b[1] + dup v10.8b, v11.b[2] + dup v11.8b, v11.b[3] +.endm + +.macro pixman_composite_over_n_8_0565_cleanup +.endm + +generate_composite_function \ + pixman_composite_over_n_8_0565_asm_neon, 0, 8, 16, \ + FLAG_DST_READWRITE, \ + 8, /* number of pixels, processed in a single block */ \ + 5, /* prefetch distance */ \ + pixman_composite_over_n_8_0565_init, \ + pixman_composite_over_n_8_0565_cleanup, \ + pixman_composite_over_8888_8_0565_process_pixblock_head, \ + pixman_composite_over_8888_8_0565_process_pixblock_tail, \ + pixman_composite_over_8888_8_0565_process_pixblock_tail_head, \ + 28, /* dst_w_basereg */ \ + 4, /* dst_r_basereg */ \ + 8, /* src_basereg */ \ + 24 /* mask_basereg */ + +/******************************************************************************/ + +.macro pixman_composite_over_8888_n_0565_init + mov v24.s[0], w6 + dup v24.8b, v24.b[3] +.endm + +.macro pixman_composite_over_8888_n_0565_cleanup +.endm + +generate_composite_function \ + pixman_composite_over_8888_n_0565_asm_neon, 32, 0, 16, \ + FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \ + 8, /* number of pixels, processed in a single block */ \ + 5, /* prefetch distance */ \ + pixman_composite_over_8888_n_0565_init, \ + pixman_composite_over_8888_n_0565_cleanup, \ + pixman_composite_over_8888_8_0565_process_pixblock_head, \ + pixman_composite_over_8888_8_0565_process_pixblock_tail, \ + pixman_composite_over_8888_8_0565_process_pixblock_tail_head, \ + 28, /* dst_w_basereg */ \ + 4, /* dst_r_basereg */ \ + 8, /* src_basereg */ \ + 24 /* mask_basereg */ + +/******************************************************************************/ + +.macro pixman_composite_src_0565_0565_process_pixblock_head +.endm + +.macro pixman_composite_src_0565_0565_process_pixblock_tail +.endm + +.macro pixman_composite_src_0565_0565_process_pixblock_tail_head + st1 {v0.4h, v1.4h, v2.4h, v3.4h}, [DST_W], #32 + fetch_src_pixblock + cache_preload 16, 16 +.endm + +generate_composite_function \ + pixman_composite_src_0565_0565_asm_neon, 16, 0, 16, \ + FLAG_DST_WRITEONLY, \ + 16, /* number of pixels, processed in a single block */ \ + 10, /* prefetch distance */ \ + default_init, \ + default_cleanup, \ + pixman_composite_src_0565_0565_process_pixblock_head, \ + pixman_composite_src_0565_0565_process_pixblock_tail, \ + pixman_composite_src_0565_0565_process_pixblock_tail_head, \ + 0, /* dst_w_basereg */ \ + 0, /* dst_r_basereg */ \ + 0, /* src_basereg */ \ + 0 /* mask_basereg */ + +/******************************************************************************/ + +.macro pixman_composite_src_n_8_process_pixblock_head +.endm + +.macro pixman_composite_src_n_8_process_pixblock_tail +.endm + +.macro pixman_composite_src_n_8_process_pixblock_tail_head + st1 {v0.8b, v1.8b, v2.8b, v3.8b}, [DST_W], 32 +.endm + +.macro pixman_composite_src_n_8_init + mov v0.s[0], w4 + dup v3.8b, v0.b[0] + dup v2.8b, v0.b[0] + dup v1.8b, v0.b[0] + dup v0.8b, v0.b[0] +.endm + +.macro pixman_composite_src_n_8_cleanup +.endm + +generate_composite_function \ + pixman_composite_src_n_8_asm_neon, 0, 0, 8, \ + FLAG_DST_WRITEONLY, \ + 32, /* number of pixels, processed in a single block */ \ + 0, /* prefetch distance */ \ + pixman_composite_src_n_8_init, \ + pixman_composite_src_n_8_cleanup, \ + pixman_composite_src_n_8_process_pixblock_head, \ + pixman_composite_src_n_8_process_pixblock_tail, \ + pixman_composite_src_n_8_process_pixblock_tail_head, \ + 0, /* dst_w_basereg */ \ + 0, /* dst_r_basereg */ \ + 0, /* src_basereg */ \ + 0 /* mask_basereg */ + +/******************************************************************************/ + +.macro pixman_composite_src_n_0565_process_pixblock_head +.endm + +.macro pixman_composite_src_n_0565_process_pixblock_tail +.endm + +.macro pixman_composite_src_n_0565_process_pixblock_tail_head + st1 {v0.4h, v1.4h, v2.4h, v3.4h}, [DST_W], #32 +.endm + +.macro pixman_composite_src_n_0565_init + mov v0.s[0], w4 + dup v3.4h, v0.h[0] + dup v2.4h, v0.h[0] + dup v1.4h, v0.h[0] + dup v0.4h, v0.h[0] +.endm + +.macro pixman_composite_src_n_0565_cleanup +.endm + +generate_composite_function \ + pixman_composite_src_n_0565_asm_neon, 0, 0, 16, \ + FLAG_DST_WRITEONLY, \ + 16, /* number of pixels, processed in a single block */ \ + 0, /* prefetch distance */ \ + pixman_composite_src_n_0565_init, \ + pixman_composite_src_n_0565_cleanup, \ + pixman_composite_src_n_0565_process_pixblock_head, \ + pixman_composite_src_n_0565_process_pixblock_tail, \ + pixman_composite_src_n_0565_process_pixblock_tail_head, \ + 0, /* dst_w_basereg */ \ + 0, /* dst_r_basereg */ \ + 0, /* src_basereg */ \ + 0 /* mask_basereg */ + +/******************************************************************************/ + +.macro pixman_composite_src_n_8888_process_pixblock_head +.endm + +.macro pixman_composite_src_n_8888_process_pixblock_tail +.endm + +.macro pixman_composite_src_n_8888_process_pixblock_tail_head + st1 {v0.2s, v1.2s, v2.2s, v3.2s}, [DST_W], #32 +.endm + +.macro pixman_composite_src_n_8888_init + mov v0.s[0], w4 + dup v3.2s, v0.s[0] + dup v2.2s, v0.s[0] + dup v1.2s, v0.s[0] + dup v0.2s, v0.s[0] +.endm + +.macro pixman_composite_src_n_8888_cleanup +.endm + +generate_composite_function \ + pixman_composite_src_n_8888_asm_neon, 0, 0, 32, \ + FLAG_DST_WRITEONLY, \ + 8, /* number of pixels, processed in a single block */ \ + 0, /* prefetch distance */ \ + pixman_composite_src_n_8888_init, \ + pixman_composite_src_n_8888_cleanup, \ + pixman_composite_src_n_8888_process_pixblock_head, \ + pixman_composite_src_n_8888_process_pixblock_tail, \ + pixman_composite_src_n_8888_process_pixblock_tail_head, \ + 0, /* dst_w_basereg */ \ + 0, /* dst_r_basereg */ \ + 0, /* src_basereg */ \ + 0 /* mask_basereg */ + +/******************************************************************************/ + +.macro pixman_composite_src_8888_8888_process_pixblock_head +.endm + +.macro pixman_composite_src_8888_8888_process_pixblock_tail +.endm + +.macro pixman_composite_src_8888_8888_process_pixblock_tail_head + st1 {v0.2s, v1.2s, v2.2s, v3.2s}, [DST_W], #32 + fetch_src_pixblock + cache_preload 8, 8 +.endm + +generate_composite_function \ + pixman_composite_src_8888_8888_asm_neon, 32, 0, 32, \ + FLAG_DST_WRITEONLY, \ + 8, /* number of pixels, processed in a single block */ \ + 10, /* prefetch distance */ \ + default_init, \ + default_cleanup, \ + pixman_composite_src_8888_8888_process_pixblock_head, \ + pixman_composite_src_8888_8888_process_pixblock_tail, \ + pixman_composite_src_8888_8888_process_pixblock_tail_head, \ + 0, /* dst_w_basereg */ \ + 0, /* dst_r_basereg */ \ + 0, /* src_basereg */ \ + 0 /* mask_basereg */ + +/******************************************************************************/ + +.macro pixman_composite_src_x888_8888_process_pixblock_head + orr v0.8b, v0.8b, v4.8b + orr v1.8b, v1.8b, v4.8b + orr v2.8b, v2.8b, v4.8b + orr v3.8b, v3.8b, v4.8b +.endm + +.macro pixman_composite_src_x888_8888_process_pixblock_tail +.endm + +.macro pixman_composite_src_x888_8888_process_pixblock_tail_head + st1 {v0.2s, v1.2s, v2.2s, v3.2s}, [DST_W], #32 + fetch_src_pixblock + orr v0.8b, v0.8b, v4.8b + orr v1.8b, v1.8b, v4.8b + orr v2.8b, v2.8b, v4.8b + orr v3.8b, v3.8b, v4.8b + cache_preload 8, 8 +.endm + +.macro pixman_composite_src_x888_8888_init + movi v4.2s, #0xff, lsl 24 +.endm + +generate_composite_function \ + pixman_composite_src_x888_8888_asm_neon, 32, 0, 32, \ + FLAG_DST_WRITEONLY, \ + 8, /* number of pixels, processed in a single block */ \ + 10, /* prefetch distance */ \ + pixman_composite_src_x888_8888_init, \ + default_cleanup, \ + pixman_composite_src_x888_8888_process_pixblock_head, \ + pixman_composite_src_x888_8888_process_pixblock_tail, \ + pixman_composite_src_x888_8888_process_pixblock_tail_head, \ + 0, /* dst_w_basereg */ \ + 0, /* dst_r_basereg */ \ + 0, /* src_basereg */ \ + 0 /* mask_basereg */ + +/******************************************************************************/ + +.macro pixman_composite_src_n_8_8888_process_pixblock_head + /* expecting solid source in {v0, v1, v2, v3} */ + /* mask is in v24 (v25, v26, v27 are unused) */ + + /* in */ + umull v8.8h, v24.8b, v0.8b + umull v9.8h, v24.8b, v1.8b + umull v10.8h, v24.8b, v2.8b + umull v11.8h, v24.8b, v3.8b + ursra v8.8h, v8.8h, #8 + ursra v9.8h, v9.8h, #8 + ursra v10.8h, v10.8h, #8 + ursra v11.8h, v11.8h, #8 +.endm + +.macro pixman_composite_src_n_8_8888_process_pixblock_tail + rshrn v28.8b, v8.8h, #8 + rshrn v29.8b, v9.8h, #8 + rshrn v30.8b, v10.8h, #8 + rshrn v31.8b, v11.8h, #8 +.endm + +.macro pixman_composite_src_n_8_8888_process_pixblock_tail_head + fetch_mask_pixblock + PF add PF_X, PF_X, #8 + rshrn v28.8b, v8.8h, #8 + PF tst PF_CTL, #0x0F + rshrn v29.8b, v9.8h, #8 + PF beq 10f + PF add PF_X, PF_X, #8 +10: + rshrn v30.8b, v10.8h, #8 + PF beq 10f + PF sub PF_CTL, PF_CTL, #1 +10: + rshrn v31.8b, v11.8h, #8 + PF cmp PF_X, ORIG_W + umull v8.8h, v24.8b, v0.8b + PF lsl DUMMY, PF_X, #mask_bpp_shift + PF prfm PREFETCH_MODE, [PF_MASK, DUMMY] + umull v9.8h, v24.8b, v1.8b + PF ble 10f + PF sub PF_X, PF_X, ORIG_W +10: + umull v10.8h, v24.8b, v2.8b + PF ble 10f + PF subs PF_CTL, PF_CTL, #0x10 +10: + umull v11.8h, v24.8b, v3.8b + PF ble 10f + PF lsl DUMMY, MASK_STRIDE, #mask_bpp_shift + PF ldrsb DUMMY, [PF_MASK, DUMMY] + PF add PF_MASK, PF_MASK, #1 +10: + st4 {v28.8b, v29.8b, v30.8b, v31.8b}, [DST_W], #32 + ursra v8.8h, v8.8h, #8 + ursra v9.8h, v9.8h, #8 + ursra v10.8h, v10.8h, #8 + ursra v11.8h, v11.8h, #8 +.endm + +.macro pixman_composite_src_n_8_8888_init + mov v3.s[0], w4 + dup v0.8b, v3.b[0] + dup v1.8b, v3.b[1] + dup v2.8b, v3.b[2] + dup v3.8b, v3.b[3] +.endm + +.macro pixman_composite_src_n_8_8888_cleanup +.endm + +generate_composite_function \ + pixman_composite_src_n_8_8888_asm_neon, 0, 8, 32, \ + FLAG_DST_WRITEONLY | FLAG_DEINTERLEAVE_32BPP, \ + 8, /* number of pixels, processed in a single block */ \ + 5, /* prefetch distance */ \ + pixman_composite_src_n_8_8888_init, \ + pixman_composite_src_n_8_8888_cleanup, \ + pixman_composite_src_n_8_8888_process_pixblock_head, \ + pixman_composite_src_n_8_8888_process_pixblock_tail, \ + pixman_composite_src_n_8_8888_process_pixblock_tail_head, \ + +/******************************************************************************/ + +.macro pixman_composite_src_n_8_8_process_pixblock_head + umull v0.8h, v24.8b, v16.8b + umull v1.8h, v25.8b, v16.8b + umull v2.8h, v26.8b, v16.8b + umull v3.8h, v27.8b, v16.8b + ursra v0.8h, v0.8h, #8 + ursra v1.8h, v1.8h, #8 + ursra v2.8h, v2.8h, #8 + ursra v3.8h, v3.8h, #8 +.endm + +.macro pixman_composite_src_n_8_8_process_pixblock_tail + rshrn v28.8b, v0.8h, #8 + rshrn v29.8b, v1.8h, #8 + rshrn v30.8b, v2.8h, #8 + rshrn v31.8b, v3.8h, #8 +.endm + +.macro pixman_composite_src_n_8_8_process_pixblock_tail_head + fetch_mask_pixblock + PF add PF_X, PF_X, #8 + rshrn v28.8b, v0.8h, #8 + PF tst PF_CTL, #0x0F + rshrn v29.8b, v1.8h, #8 + PF beq 10f + PF add PF_X, PF_X, #8 +10: + rshrn v30.8b, v2.8h, #8 + PF beq 10f + PF sub PF_CTL, PF_CTL, #1 +10: + rshrn v31.8b, v3.8h, #8 + PF cmp PF_X, ORIG_W + umull v0.8h, v24.8b, v16.8b + PF lsl DUMMY, PF_X, mask_bpp_shift + PF prfm PREFETCH_MODE, [PF_MASK, DUMMY] + umull v1.8h, v25.8b, v16.8b + PF ble 10f + PF sub PF_X, PF_X, ORIG_W +10: + umull v2.8h, v26.8b, v16.8b + PF ble 10f + PF subs PF_CTL, PF_CTL, #0x10 +10: + umull v3.8h, v27.8b, v16.8b + PF ble 10f + PF lsl DUMMY, MASK_STRIDE, #mask_bpp_shift + PF ldrsb DUMMY, [PF_MASK, DUMMY] + PF add PF_MASK, PF_MASK, #1 +10: + st1 {v28.8b, v29.8b, v30.8b, v31.8b}, [DST_W], #32 + ursra v0.8h, v0.8h, #8 + ursra v1.8h, v1.8h, #8 + ursra v2.8h, v2.8h, #8 + ursra v3.8h, v3.8h, #8 +.endm + +.macro pixman_composite_src_n_8_8_init + mov v16.s[0], w4 + dup v16.8b, v16.b[3] +.endm + +.macro pixman_composite_src_n_8_8_cleanup +.endm + +generate_composite_function \ + pixman_composite_src_n_8_8_asm_neon, 0, 8, 8, \ + FLAG_DST_WRITEONLY, \ + 32, /* number of pixels, processed in a single block */ \ + 5, /* prefetch distance */ \ + pixman_composite_src_n_8_8_init, \ + pixman_composite_src_n_8_8_cleanup, \ + pixman_composite_src_n_8_8_process_pixblock_head, \ + pixman_composite_src_n_8_8_process_pixblock_tail, \ + pixman_composite_src_n_8_8_process_pixblock_tail_head + +/******************************************************************************/ + +.macro pixman_composite_over_n_8_8888_process_pixblock_head + /* expecting deinterleaved source data in {v8, v9, v10, v11} */ + /* v8 - blue, v9 - green, v10 - red, v11 - alpha */ + /* and destination data in {v4, v5, v6, v7} */ + /* mask is in v24 (v25, v26, v27 are unused) */ + + /* in */ + umull v12.8h, v24.8b, v8.8b + umull v13.8h, v24.8b, v9.8b + umull v14.8h, v24.8b, v10.8b + umull v15.8h, v24.8b, v11.8b + urshr v16.8h, v12.8h, #8 + urshr v17.8h, v13.8h, #8 + urshr v18.8h, v14.8h, #8 + urshr v19.8h, v15.8h, #8 + raddhn v0.8b, v12.8h, v16.8h + raddhn v1.8b, v13.8h, v17.8h + raddhn v2.8b, v14.8h, v18.8h + raddhn v3.8b, v15.8h, v19.8h + mvn v25.8b, v3.8b /* get inverted alpha */ + /* source: v0 - blue, v1 - green, v2 - red, v3 - alpha */ + /* destination: v4 - blue, v5 - green, v6 - red, v7 - alpha */ + /* now do alpha blending */ + umull v12.8h, v25.8b, v4.8b + umull v13.8h, v25.8b, v5.8b + umull v14.8h, v25.8b, v6.8b + umull v15.8h, v25.8b, v7.8b +.endm + +.macro pixman_composite_over_n_8_8888_process_pixblock_tail + urshr v16.8h, v12.8h, #8 + urshr v17.8h, v13.8h, #8 + urshr v18.8h, v14.8h, #8 + urshr v19.8h, v15.8h, #8 + raddhn v28.8b, v16.8h, v12.8h + raddhn v29.8b, v17.8h, v13.8h + raddhn v30.8b, v18.8h, v14.8h + raddhn v31.8b, v19.8h, v15.8h + uqadd v28.8b, v0.8b, v28.8b + uqadd v29.8b, v1.8b, v29.8b + uqadd v30.8b, v2.8b, v30.8b + uqadd v31.8b, v3.8b, v31.8b +.endm + +.macro pixman_composite_over_n_8_8888_process_pixblock_tail_head + urshr v16.8h, v12.8h, #8 + ld4 {v4.8b, v5.8b, v6.8b, v7.8b}, [DST_R], #32 + urshr v17.8h, v13.8h, #8 + fetch_mask_pixblock + urshr v18.8h, v14.8h, #8 + PF add PF_X, PF_X, #8 + urshr v19.8h, v15.8h, #8 + PF tst PF_CTL, #0x0F + raddhn v28.8b, v16.8h, v12.8h + PF beq 10f + PF add PF_X, PF_X, #8 +10: + raddhn v29.8b, v17.8h, v13.8h + PF beq 10f + PF sub PF_CTL, PF_CTL, #1 +10: + raddhn v30.8b, v18.8h, v14.8h + PF cmp PF_X, ORIG_W + raddhn v31.8b, v19.8h, v15.8h + PF lsl DUMMY, PF_X, #dst_bpp_shift + PF prfm PREFETCH_MODE, [PF_DST, DUMMY] + umull v16.8h, v24.8b, v8.8b + PF lsl DUMMY, PF_X, #mask_bpp_shift + PF prfm PREFETCH_MODE, [PF_MASK, DUMMY] + umull v17.8h, v24.8b, v9.8b + PF ble 10f + PF sub PF_X, PF_X, ORIG_W +10: + umull v18.8h, v24.8b, v10.8b + PF ble 10f + PF subs PF_CTL, PF_CTL, #0x10 +10: + umull v19.8h, v24.8b, v11.8b + PF ble 10f + PF lsl DUMMY, DST_STRIDE, #dst_bpp_shift + PF ldrsb DUMMY, [PF_DST, DUMMY] + PF add PF_DST, PF_DST, #1 +10: + uqadd v28.8b, v0.8b, v28.8b + PF ble 10f + PF lsl DUMMY, MASK_STRIDE, #mask_bpp_shift + PF ldrsb DUMMY, [PF_MASK, DUMMY] + PF add PF_MASK, PF_MASK, #1 +10: + uqadd v29.8b, v1.8b, v29.8b + uqadd v30.8b, v2.8b, v30.8b + uqadd v31.8b, v3.8b, v31.8b + urshr v12.8h, v16.8h, #8 + urshr v13.8h, v17.8h, #8 + urshr v14.8h, v18.8h, #8 + urshr v15.8h, v19.8h, #8 + raddhn v0.8b, v16.8h, v12.8h + raddhn v1.8b, v17.8h, v13.8h + raddhn v2.8b, v18.8h, v14.8h + raddhn v3.8b, v19.8h, v15.8h + st4 {v28.8b, v29.8b, v30.8b, v31.8b}, [DST_W], #32 + mvn v25.8b, v3.8b + umull v12.8h, v25.8b, v4.8b + umull v13.8h, v25.8b, v5.8b + umull v14.8h, v25.8b, v6.8b + umull v15.8h, v25.8b, v7.8b +.endm + +.macro pixman_composite_over_n_8_8888_init + mov v11.s[0], w4 + dup v8.8b, v11.b[0] + dup v9.8b, v11.b[1] + dup v10.8b, v11.b[2] + dup v11.8b, v11.b[3] +.endm + +.macro pixman_composite_over_n_8_8888_cleanup +.endm + +generate_composite_function \ + pixman_composite_over_n_8_8888_asm_neon, 0, 8, 32, \ + FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \ + 8, /* number of pixels, processed in a single block */ \ + 5, /* prefetch distance */ \ + pixman_composite_over_n_8_8888_init, \ + pixman_composite_over_n_8_8888_cleanup, \ + pixman_composite_over_n_8_8888_process_pixblock_head, \ + pixman_composite_over_n_8_8888_process_pixblock_tail, \ + pixman_composite_over_n_8_8888_process_pixblock_tail_head + +/******************************************************************************/ + +.macro pixman_composite_over_n_8_8_process_pixblock_head + umull v0.8h, v24.8b, v8.8b + umull v1.8h, v25.8b, v8.8b + umull v2.8h, v26.8b, v8.8b + umull v3.8h, v27.8b, v8.8b + urshr v10.8h, v0.8h, #8 + urshr v11.8h, v1.8h, #8 + urshr v12.8h, v2.8h, #8 + urshr v13.8h, v3.8h, #8 + raddhn v0.8b, v0.8h, v10.8h + raddhn v1.8b, v1.8h, v11.8h + raddhn v2.8b, v2.8h, v12.8h + raddhn v3.8b, v3.8h, v13.8h + mvn v24.8b, v0.8b + mvn v25.8b, v1.8b + mvn v26.8b, v2.8b + mvn v27.8b, v3.8b + umull v10.8h, v24.8b, v4.8b + umull v11.8h, v25.8b, v5.8b + umull v12.8h, v26.8b, v6.8b + umull v13.8h, v27.8b, v7.8b +.endm + +.macro pixman_composite_over_n_8_8_process_pixblock_tail + urshr v14.8h, v10.8h, #8 + urshr v15.8h, v11.8h, #8 + urshr v16.8h, v12.8h, #8 + urshr v17.8h, v13.8h, #8 + raddhn v28.8b, v14.8h, v10.8h + raddhn v29.8b, v15.8h, v11.8h + raddhn v30.8b, v16.8h, v12.8h + raddhn v31.8b, v17.8h, v13.8h + uqadd v28.8b, v0.8b, v28.8b + uqadd v29.8b, v1.8b, v29.8b + uqadd v30.8b, v2.8b, v30.8b + uqadd v31.8b, v3.8b, v31.8b +.endm + +/* TODO: expand macros and do better instructions scheduling */ +.macro pixman_composite_over_n_8_8_process_pixblock_tail_head + ld1 {v4.8b, v5.8b, v6.8b, v7.8b}, [DST_R], #32 + pixman_composite_over_n_8_8_process_pixblock_tail + fetch_mask_pixblock + cache_preload 32, 32 + st1 {v28.8b, v29.8b, v30.8b, v31.8b}, [DST_W], #32 + pixman_composite_over_n_8_8_process_pixblock_head +.endm + +.macro pixman_composite_over_n_8_8_init + mov v8.s[0], w4 + dup v8.8b, v8.b[3] +.endm + +.macro pixman_composite_over_n_8_8_cleanup +.endm + +generate_composite_function \ + pixman_composite_over_n_8_8_asm_neon, 0, 8, 8, \ + FLAG_DST_READWRITE, \ + 32, /* number of pixels, processed in a single block */ \ + 5, /* prefetch distance */ \ + pixman_composite_over_n_8_8_init, \ + pixman_composite_over_n_8_8_cleanup, \ + pixman_composite_over_n_8_8_process_pixblock_head, \ + pixman_composite_over_n_8_8_process_pixblock_tail, \ + pixman_composite_over_n_8_8_process_pixblock_tail_head + +/******************************************************************************/ + +.macro pixman_composite_over_n_8888_8888_ca_process_pixblock_head + /* + * 'combine_mask_ca' replacement + * + * input: solid src (n) in {v8, v9, v10, v11} + * dest in {v4, v5, v6, v7 } + * mask in {v24, v25, v26, v27} + * output: updated src in {v0, v1, v2, v3 } + * updated mask in {v24, v25, v26, v3 } + */ + umull v0.8h, v24.8b, v8.8b + umull v1.8h, v25.8b, v9.8b + umull v2.8h, v26.8b, v10.8b + umull v3.8h, v27.8b, v11.8b + umull v12.8h, v11.8b, v25.8b + umull v13.8h, v11.8b, v24.8b + umull v14.8h, v11.8b, v26.8b + urshr v15.8h, v0.8h, #8 + urshr v16.8h, v1.8h, #8 + urshr v17.8h, v2.8h, #8 + raddhn v0.8b, v0.8h, v15.8h + raddhn v1.8b, v1.8h, v16.8h + raddhn v2.8b, v2.8h, v17.8h + urshr v15.8h, v13.8h, #8 + urshr v16.8h, v12.8h, #8 + urshr v17.8h, v14.8h, #8 + urshr v18.8h, v3.8h, #8 + raddhn v24.8b, v13.8h, v15.8h + raddhn v25.8b, v12.8h, v16.8h + raddhn v26.8b, v14.8h, v17.8h + raddhn v3.8b, v3.8h, v18.8h + /* + * 'combine_over_ca' replacement + * + * output: updated dest in {v28, v29, v30, v31} + */ + mvn v24.8b, v24.8b + mvn v25.8b, v25.8b + mvn v26.8b, v26.8b + mvn v27.8b, v3.8b + umull v12.8h, v24.8b, v4.8b + umull v13.8h, v25.8b, v5.8b + umull v14.8h, v26.8b, v6.8b + umull v15.8h, v27.8b, v7.8b +.endm + +.macro pixman_composite_over_n_8888_8888_ca_process_pixblock_tail + /* ... continue 'combine_over_ca' replacement */ + urshr v16.8h, v12.8h, #8 + urshr v17.8h, v13.8h, #8 + urshr v18.8h, v14.8h, #8 + urshr v19.8h, v15.8h, #8 + raddhn v28.8b, v16.8h, v12.8h + raddhn v29.8b, v17.8h, v13.8h + raddhn v30.8b, v18.8h, v14.8h + raddhn v31.8b, v19.8h, v15.8h + uqadd v28.8b, v0.8b, v28.8b + uqadd v29.8b, v1.8b, v29.8b + uqadd v30.8b, v2.8b, v30.8b + uqadd v31.8b, v3.8b, v31.8b +.endm + +.macro pixman_composite_over_n_8888_8888_ca_process_pixblock_tail_head + urshr v16.8h, v12.8h, #8 + urshr v17.8h, v13.8h, #8 + ld4 {v4.8b, v5.8b, v6.8b, v7.8b}, [DST_R], #32 + urshr v18.8h, v14.8h, #8 + urshr v19.8h, v15.8h, #8 + raddhn v28.8b, v16.8h, v12.8h + raddhn v29.8b, v17.8h, v13.8h + raddhn v30.8b, v18.8h, v14.8h + raddhn v31.8b, v19.8h, v15.8h + fetch_mask_pixblock + uqadd v28.8b, v0.8b, v28.8b + uqadd v29.8b, v1.8b, v29.8b + uqadd v30.8b, v2.8b, v30.8b + uqadd v31.8b, v3.8b, v31.8b + cache_preload 8, 8 + pixman_composite_over_n_8888_8888_ca_process_pixblock_head + st4 {v28.8b, v29.8b, v30.8b, v31.8b}, [DST_W], #32 +.endm + +.macro pixman_composite_over_n_8888_8888_ca_init + mov v13.s[0], w4 + dup v8.8b, v13.b[0] + dup v9.8b, v13.b[1] + dup v10.8b, v13.b[2] + dup v11.8b, v13.b[3] +.endm + +.macro pixman_composite_over_n_8888_8888_ca_cleanup +.endm + +generate_composite_function \ + pixman_composite_over_n_8888_8888_ca_asm_neon, 0, 32, 32, \ + FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \ + 8, /* number of pixels, processed in a single block */ \ + 5, /* prefetch distance */ \ + pixman_composite_over_n_8888_8888_ca_init, \ + pixman_composite_over_n_8888_8888_ca_cleanup, \ + pixman_composite_over_n_8888_8888_ca_process_pixblock_head, \ + pixman_composite_over_n_8888_8888_ca_process_pixblock_tail, \ + pixman_composite_over_n_8888_8888_ca_process_pixblock_tail_head + +/******************************************************************************/ + +.macro pixman_composite_over_n_8888_0565_ca_process_pixblock_head + /* + * 'combine_mask_ca' replacement + * + * input: solid src (n) in {v8, v9, v10, v11} [B, G, R, A] + * mask in {v24, v25, v26} [B, G, R] + * output: updated src in {v0, v1, v2 } [B, G, R] + * updated mask in {v24, v25, v26} [B, G, R] + */ + umull v0.8h, v24.8b, v8.8b + umull v1.8h, v25.8b, v9.8b + umull v2.8h, v26.8b, v10.8b + umull v12.8h, v11.8b, v24.8b + umull v13.8h, v11.8b, v25.8b + umull v14.8h, v11.8b, v26.8b + urshr v15.8h, v0.8h, #8 + urshr v16.8h, v1.8h, #8 + urshr v17.8h, v2.8h, #8 + raddhn v0.8b, v0.8h, v15.8h + raddhn v1.8b, v1.8h, v16.8h + raddhn v2.8b, v2.8h, v17.8h + urshr v19.8h, v12.8h, #8 + urshr v20.8h, v13.8h, #8 + urshr v21.8h, v14.8h, #8 + raddhn v24.8b, v12.8h, v19.8h + raddhn v25.8b, v13.8h, v20.8h + /* + * convert 8 r5g6b5 pixel data from {v4} to planar 8-bit format + * and put data into v16 - blue, v17 - green, v18 - red + */ + mov v4.d[1], v5.d[0] + shrn v17.8b, v4.8h, #3 + shrn v18.8b, v4.8h, #8 + raddhn v26.8b, v14.8h, v21.8h + sli v4.8h, v4.8h, #5 + sri v18.8b, v18.8b, #5 + sri v17.8b, v17.8b, #6 + /* + * 'combine_over_ca' replacement + * + * output: updated dest in v16 - blue, v17 - green, v18 - red + */ + mvn v24.8b, v24.8b + mvn v25.8b, v25.8b + shrn v16.8b, v4.8h, #2 + mvn v26.8b, v26.8b + umull v5.8h, v16.8b, v24.8b + umull v6.8h, v17.8b, v25.8b + umull v7.8h, v18.8b, v26.8b +.endm + +.macro pixman_composite_over_n_8888_0565_ca_process_pixblock_tail + /* ... continue 'combine_over_ca' replacement */ + urshr v13.8h, v5.8h, #8 + urshr v14.8h, v6.8h, #8 + urshr v15.8h, v7.8h, #8 + raddhn v16.8b, v13.8h, v5.8h + raddhn v17.8b, v14.8h, v6.8h + raddhn v18.8b, v15.8h, v7.8h + uqadd v16.8b, v0.8b, v16.8b + uqadd v17.8b, v1.8b, v17.8b + uqadd v18.8b, v2.8b, v18.8b + /* + * convert the results in v16, v17, v18 to r5g6b5 and store + * them into {v14} + */ + ushll v14.8h, v18.8b, #7 + sli v14.8h, v14.8h, #1 + ushll v12.8h, v17.8b, #7 + sli v12.8h, v12.8h, #1 + ushll v13.8h, v16.8b, #7 + sli v13.8h, v13.8h, #1 + sri v14.8h, v12.8h, #5 + sri v14.8h, v13.8h, #11 + mov v28.d[0], v14.d[0] + mov v29.d[0], v14.d[1] +.endm + +.macro pixman_composite_over_n_8888_0565_ca_process_pixblock_tail_head + fetch_mask_pixblock + urshr v13.8h, v5.8h, #8 + urshr v14.8h, v6.8h, #8 + ld1 {v4.8h}, [DST_R], #16 + urshr v15.8h, v7.8h, #8 + raddhn v16.8b, v13.8h, v5.8h + raddhn v17.8b, v14.8h, v6.8h + raddhn v18.8b, v15.8h, v7.8h + mov v5.d[0], v4.d[1] + /* process_pixblock_head */ + /* + * 'combine_mask_ca' replacement + * + * input: solid src (n) in {v8, v9, v10, v11} [B, G, R, A] + * mask in {v24, v25, v26} [B, G, R] + * output: updated src in {v0, v1, v2 } [B, G, R] + * updated mask in {v24, v25, v26} [B, G, R] + */ + uqadd v16.8b, v0.8b, v16.8b + uqadd v17.8b, v1.8b, v17.8b + uqadd v18.8b, v2.8b, v18.8b + umull v0.8h, v24.8b, v8.8b + umull v1.8h, v25.8b, v9.8b + umull v2.8h, v26.8b, v10.8b + /* + * convert the result in v16, v17, v18 to r5g6b5 and store + * it into {v14} + */ + ushll v14.8h, v18.8b, #7 + sli v14.8h, v14.8h, #1 + ushll v18.8h, v16.8b, #7 + sli v18.8h, v18.8h, #1 + ushll v19.8h, v17.8b, #7 + sli v19.8h, v19.8h, #1 + umull v12.8h, v11.8b, v24.8b + sri v14.8h, v19.8h, #5 + umull v13.8h, v11.8b, v25.8b + umull v15.8h, v11.8b, v26.8b + sri v14.8h, v18.8h, #11 + mov v28.d[0], v14.d[0] + mov v29.d[0], v14.d[1] + cache_preload 8, 8 + urshr v16.8h, v0.8h, #8 + urshr v17.8h, v1.8h, #8 + urshr v18.8h, v2.8h, #8 + raddhn v0.8b, v0.8h, v16.8h + raddhn v1.8b, v1.8h, v17.8h + raddhn v2.8b, v2.8h, v18.8h + urshr v19.8h, v12.8h, #8 + urshr v20.8h, v13.8h, #8 + urshr v21.8h, v15.8h, #8 + raddhn v24.8b, v12.8h, v19.8h + raddhn v25.8b, v13.8h, v20.8h + /* + * convert 8 r5g6b5 pixel data from {v4, v5} to planar + * 8-bit format and put data into v16 - blue, v17 - green, + * v18 - red + */ + mov v4.d[1], v5.d[0] + shrn v17.8b, v4.8h, #3 + shrn v18.8b, v4.8h, #8 + raddhn v26.8b, v15.8h, v21.8h + sli v4.8h, v4.8h, #5 + sri v17.8b, v17.8b, #6 + sri v18.8b, v18.8b, #5 + /* + * 'combine_over_ca' replacement + * + * output: updated dest in v16 - blue, v17 - green, v18 - red + */ + mvn v24.8b, v24.8b + mvn v25.8b, v25.8b + shrn v16.8b, v4.8h, #2 + mvn v26.8b, v26.8b + umull v5.8h, v16.8b, v24.8b + umull v6.8h, v17.8b, v25.8b + umull v7.8h, v18.8b, v26.8b + st1 {v14.8h}, [DST_W], #16 +.endm + +.macro pixman_composite_over_n_8888_0565_ca_init + mov v13.s[0], w4 + dup v8.8b, v13.b[0] + dup v9.8b, v13.b[1] + dup v10.8b, v13.b[2] + dup v11.8b, v13.b[3] +.endm + +.macro pixman_composite_over_n_8888_0565_ca_cleanup +.endm + +generate_composite_function \ + pixman_composite_over_n_8888_0565_ca_asm_neon, 0, 32, 16, \ + FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \ + 8, /* number of pixels, processed in a single block */ \ + 5, /* prefetch distance */ \ + pixman_composite_over_n_8888_0565_ca_init, \ + pixman_composite_over_n_8888_0565_ca_cleanup, \ + pixman_composite_over_n_8888_0565_ca_process_pixblock_head, \ + pixman_composite_over_n_8888_0565_ca_process_pixblock_tail, \ + pixman_composite_over_n_8888_0565_ca_process_pixblock_tail_head + +/******************************************************************************/ + +.macro pixman_composite_in_n_8_process_pixblock_head + /* expecting source data in {v0, v1, v2, v3} */ + /* and destination data in {v4, v5, v6, v7} */ + umull v8.8h, v4.8b, v3.8b + umull v9.8h, v5.8b, v3.8b + umull v10.8h, v6.8b, v3.8b + umull v11.8h, v7.8b, v3.8b +.endm + +.macro pixman_composite_in_n_8_process_pixblock_tail + urshr v14.8h, v8.8h, #8 + urshr v15.8h, v9.8h, #8 + urshr v12.8h, v10.8h, #8 + urshr v13.8h, v11.8h, #8 + raddhn v28.8b, v8.8h, v14.8h + raddhn v29.8b, v9.8h, v15.8h + raddhn v30.8b, v10.8h, v12.8h + raddhn v31.8b, v11.8h, v13.8h +.endm + +.macro pixman_composite_in_n_8_process_pixblock_tail_head + pixman_composite_in_n_8_process_pixblock_tail + ld1 {v4.8b, v5.8b, v6.8b, v7.8b}, [DST_R], #32 + cache_preload 32, 32 + pixman_composite_in_n_8_process_pixblock_head + st1 {v28.8b, v29.8b, v30.8b, v31.8b}, [DST_W], #32 +.endm + +.macro pixman_composite_in_n_8_init + mov v3.s[0], w4 + dup v3.8b, v3.b[3] +.endm + +.macro pixman_composite_in_n_8_cleanup +.endm + +generate_composite_function \ + pixman_composite_in_n_8_asm_neon, 0, 0, 8, \ + FLAG_DST_READWRITE, \ + 32, /* number of pixels, processed in a single block */ \ + 5, /* prefetch distance */ \ + pixman_composite_in_n_8_init, \ + pixman_composite_in_n_8_cleanup, \ + pixman_composite_in_n_8_process_pixblock_head, \ + pixman_composite_in_n_8_process_pixblock_tail, \ + pixman_composite_in_n_8_process_pixblock_tail_head, \ + 28, /* dst_w_basereg */ \ + 4, /* dst_r_basereg */ \ + 0, /* src_basereg */ \ + 24 /* mask_basereg */ + +.macro pixman_composite_add_n_8_8_process_pixblock_head + /* expecting source data in {v8, v9, v10, v11} */ + /* v8 - blue, v9 - green, v10 - red, v11 - alpha */ + /* and destination data in {v4, v5, v6, v7} */ + /* mask is in v24, v25, v26, v27 */ + umull v0.8h, v24.8b, v11.8b + umull v1.8h, v25.8b, v11.8b + umull v2.8h, v26.8b, v11.8b + umull v3.8h, v27.8b, v11.8b + urshr v12.8h, v0.8h, #8 + urshr v13.8h, v1.8h, #8 + urshr v14.8h, v2.8h, #8 + urshr v15.8h, v3.8h, #8 + raddhn v0.8b, v0.8h, v12.8h + raddhn v1.8b, v1.8h, v13.8h + raddhn v2.8b, v2.8h, v14.8h + raddhn v3.8b, v3.8h, v15.8h + uqadd v28.8b, v0.8b, v4.8b + uqadd v29.8b, v1.8b, v5.8b + uqadd v30.8b, v2.8b, v6.8b + uqadd v31.8b, v3.8b, v7.8b +.endm + +.macro pixman_composite_add_n_8_8_process_pixblock_tail +.endm + +/* TODO: expand macros and do better instructions scheduling */ +.macro pixman_composite_add_n_8_8_process_pixblock_tail_head + pixman_composite_add_n_8_8_process_pixblock_tail + st1 {v28.8b, v29.8b, v30.8b, v31.8b}, [DST_W], #32 + ld1 {v4.8b, v5.8b, v6.8b, v7.8b}, [DST_R], #32 + fetch_mask_pixblock + cache_preload 32, 32 + pixman_composite_add_n_8_8_process_pixblock_head +.endm + +.macro pixman_composite_add_n_8_8_init + mov v11.s[0], w4 + dup v11.8b, v11.b[3] +.endm + +.macro pixman_composite_add_n_8_8_cleanup +.endm + +generate_composite_function \ + pixman_composite_add_n_8_8_asm_neon, 0, 8, 8, \ + FLAG_DST_READWRITE, \ + 32, /* number of pixels, processed in a single block */ \ + 5, /* prefetch distance */ \ + pixman_composite_add_n_8_8_init, \ + pixman_composite_add_n_8_8_cleanup, \ + pixman_composite_add_n_8_8_process_pixblock_head, \ + pixman_composite_add_n_8_8_process_pixblock_tail, \ + pixman_composite_add_n_8_8_process_pixblock_tail_head + +/******************************************************************************/ + +.macro pixman_composite_add_8_8_8_process_pixblock_head + /* expecting source data in {v0, v1, v2, v3} */ + /* destination data in {v4, v5, v6, v7} */ + /* mask in {v24, v25, v26, v27} */ + umull v8.8h, v24.8b, v0.8b + umull v9.8h, v25.8b, v1.8b + umull v10.8h, v26.8b, v2.8b + umull v11.8h, v27.8b, v3.8b + urshr v0.8h, v8.8h, #8 + urshr v1.8h, v9.8h, #8 + urshr v12.8h, v10.8h, #8 + urshr v13.8h, v11.8h, #8 + raddhn v0.8b, v0.8h, v8.8h + raddhn v1.8b, v1.8h, v9.8h + raddhn v2.8b, v12.8h, v10.8h + raddhn v3.8b, v13.8h, v11.8h + uqadd v28.8b, v0.8b, v4.8b + uqadd v29.8b, v1.8b, v5.8b + uqadd v30.8b, v2.8b, v6.8b + uqadd v31.8b, v3.8b, v7.8b +.endm + +.macro pixman_composite_add_8_8_8_process_pixblock_tail +.endm + +/* TODO: expand macros and do better instructions scheduling */ +.macro pixman_composite_add_8_8_8_process_pixblock_tail_head + pixman_composite_add_8_8_8_process_pixblock_tail + st1 {v28.8b, v29.8b, v30.8b, v31.8b}, [DST_W], #32 + ld1 {v4.8b, v5.8b, v6.8b, v7.8b}, [DST_R], #32 + fetch_mask_pixblock + fetch_src_pixblock + cache_preload 32, 32 + pixman_composite_add_8_8_8_process_pixblock_head +.endm + +.macro pixman_composite_add_8_8_8_init +.endm + +.macro pixman_composite_add_8_8_8_cleanup +.endm + +generate_composite_function \ + pixman_composite_add_8_8_8_asm_neon, 8, 8, 8, \ + FLAG_DST_READWRITE, \ + 32, /* number of pixels, processed in a single block */ \ + 5, /* prefetch distance */ \ + pixman_composite_add_8_8_8_init, \ + pixman_composite_add_8_8_8_cleanup, \ + pixman_composite_add_8_8_8_process_pixblock_head, \ + pixman_composite_add_8_8_8_process_pixblock_tail, \ + pixman_composite_add_8_8_8_process_pixblock_tail_head + +/******************************************************************************/ + +.macro pixman_composite_add_8888_8888_8888_process_pixblock_head + /* expecting source data in {v0, v1, v2, v3} */ + /* destination data in {v4, v5, v6, v7} */ + /* mask in {v24, v25, v26, v27} */ + umull v8.8h, v27.8b, v0.8b + umull v9.8h, v27.8b, v1.8b + umull v10.8h, v27.8b, v2.8b + umull v11.8h, v27.8b, v3.8b + /* 1 cycle bubble */ + ursra v8.8h, v8.8h, #8 + ursra v9.8h, v9.8h, #8 + ursra v10.8h, v10.8h, #8 + ursra v11.8h, v11.8h, #8 +.endm + +.macro pixman_composite_add_8888_8888_8888_process_pixblock_tail + /* 2 cycle bubble */ + rshrn v28.8b, v8.8h, #8 + rshrn v29.8b, v9.8h, #8 + rshrn v30.8b, v10.8h, #8 + rshrn v31.8b, v11.8h, #8 + uqadd v28.8b, v4.8b, v28.8b + uqadd v29.8b, v5.8b, v29.8b + uqadd v30.8b, v6.8b, v30.8b + uqadd v31.8b, v7.8b, v31.8b +.endm + +.macro pixman_composite_add_8888_8888_8888_process_pixblock_tail_head + fetch_src_pixblock + rshrn v28.8b, v8.8h, #8 + fetch_mask_pixblock + rshrn v29.8b, v9.8h, #8 + umull v8.8h, v27.8b, v0.8b + rshrn v30.8b, v10.8h, #8 + umull v9.8h, v27.8b, v1.8b + rshrn v31.8b, v11.8h, #8 + umull v10.8h, v27.8b, v2.8b + umull v11.8h, v27.8b, v3.8b + uqadd v28.8b, v4.8b, v28.8b + uqadd v29.8b, v5.8b, v29.8b + uqadd v30.8b, v6.8b, v30.8b + uqadd v31.8b, v7.8b, v31.8b + ursra v8.8h, v8.8h, #8 + ld4 {v4.8b, v5.8b, v6.8b, v7.8b}, [DST_R], #32 + ursra v9.8h, v9.8h, #8 + st4 {v28.8b, v29.8b, v30.8b, v31.8b}, [DST_W], #32 + ursra v10.8h, v10.8h, #8 + + cache_preload 8, 8 + + ursra v11.8h, v11.8h, #8 +.endm + +generate_composite_function \ + pixman_composite_add_8888_8888_8888_asm_neon, 32, 32, 32, \ + FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \ + 8, /* number of pixels, processed in a single block */ \ + 10, /* prefetch distance */ \ + default_init, \ + default_cleanup, \ + pixman_composite_add_8888_8888_8888_process_pixblock_head, \ + pixman_composite_add_8888_8888_8888_process_pixblock_tail, \ + pixman_composite_add_8888_8888_8888_process_pixblock_tail_head, \ + 28, /* dst_w_basereg */ \ + 4, /* dst_r_basereg */ \ + 0, /* src_basereg */ \ + 24 /* mask_basereg */ + +generate_composite_function_single_scanline \ + pixman_composite_scanline_add_mask_asm_neon, 32, 32, 32, \ + FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \ + 8, /* number of pixels, processed in a single block */ \ + default_init, \ + default_cleanup, \ + pixman_composite_add_8888_8888_8888_process_pixblock_head, \ + pixman_composite_add_8888_8888_8888_process_pixblock_tail, \ + pixman_composite_add_8888_8888_8888_process_pixblock_tail_head, \ + 28, /* dst_w_basereg */ \ + 4, /* dst_r_basereg */ \ + 0, /* src_basereg */ \ + 24 /* mask_basereg */ + +/******************************************************************************/ + +generate_composite_function \ + pixman_composite_add_8888_8_8888_asm_neon, 32, 8, 32, \ + FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \ + 8, /* number of pixels, processed in a single block */ \ + 5, /* prefetch distance */ \ + default_init, \ + default_cleanup, \ + pixman_composite_add_8888_8888_8888_process_pixblock_head, \ + pixman_composite_add_8888_8888_8888_process_pixblock_tail, \ + pixman_composite_add_8888_8888_8888_process_pixblock_tail_head, \ + 28, /* dst_w_basereg */ \ + 4, /* dst_r_basereg */ \ + 0, /* src_basereg */ \ + 27 /* mask_basereg */ + +/******************************************************************************/ + +.macro pixman_composite_add_n_8_8888_init + mov v3.s[0], w4 + dup v0.8b, v3.b[0] + dup v1.8b, v3.b[1] + dup v2.8b, v3.b[2] + dup v3.8b, v3.b[3] +.endm + +.macro pixman_composite_add_n_8_8888_cleanup +.endm + +generate_composite_function \ + pixman_composite_add_n_8_8888_asm_neon, 0, 8, 32, \ + FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \ + 8, /* number of pixels, processed in a single block */ \ + 5, /* prefetch distance */ \ + pixman_composite_add_n_8_8888_init, \ + pixman_composite_add_n_8_8888_cleanup, \ + pixman_composite_add_8888_8888_8888_process_pixblock_head, \ + pixman_composite_add_8888_8888_8888_process_pixblock_tail, \ + pixman_composite_add_8888_8888_8888_process_pixblock_tail_head, \ + 28, /* dst_w_basereg */ \ + 4, /* dst_r_basereg */ \ + 0, /* src_basereg */ \ + 27 /* mask_basereg */ + +/******************************************************************************/ + +.macro pixman_composite_add_8888_n_8888_init + mov v27.s[0], w6 + dup v27.8b, v27.b[3] +.endm + +.macro pixman_composite_add_8888_n_8888_cleanup +.endm + +generate_composite_function \ + pixman_composite_add_8888_n_8888_asm_neon, 32, 0, 32, \ + FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \ + 8, /* number of pixels, processed in a single block */ \ + 5, /* prefetch distance */ \ + pixman_composite_add_8888_n_8888_init, \ + pixman_composite_add_8888_n_8888_cleanup, \ + pixman_composite_add_8888_8888_8888_process_pixblock_head, \ + pixman_composite_add_8888_8888_8888_process_pixblock_tail, \ + pixman_composite_add_8888_8888_8888_process_pixblock_tail_head, \ + 28, /* dst_w_basereg */ \ + 4, /* dst_r_basereg */ \ + 0, /* src_basereg */ \ + 27 /* mask_basereg */ + +/******************************************************************************/ + +.macro pixman_composite_out_reverse_8888_n_8888_process_pixblock_head + /* expecting source data in {v0, v1, v2, v3} */ + /* destination data in {v4, v5, v6, v7} */ + /* solid mask is in v15 */ + + /* 'in' */ + umull v11.8h, v15.8b, v3.8b + umull v10.8h, v15.8b, v2.8b + umull v9.8h, v15.8b, v1.8b + umull v8.8h, v15.8b, v0.8b + urshr v16.8h, v11.8h, #8 + urshr v14.8h, v10.8h, #8 + urshr v13.8h, v9.8h, #8 + urshr v12.8h, v8.8h, #8 + raddhn v3.8b, v11.8h, v16.8h + raddhn v2.8b, v10.8h, v14.8h + raddhn v1.8b, v9.8h, v13.8h + raddhn v0.8b, v8.8h, v12.8h + mvn v24.8b, v3.8b /* get inverted alpha */ + /* now do alpha blending */ + umull v8.8h, v24.8b, v4.8b + umull v9.8h, v24.8b, v5.8b + umull v10.8h, v24.8b, v6.8b + umull v11.8h, v24.8b, v7.8b +.endm + +.macro pixman_composite_out_reverse_8888_n_8888_process_pixblock_tail + urshr v16.8h, v8.8h, #8 + urshr v17.8h, v9.8h, #8 + urshr v18.8h, v10.8h, #8 + urshr v19.8h, v11.8h, #8 + raddhn v28.8b, v16.8h, v8.8h + raddhn v29.8b, v17.8h, v9.8h + raddhn v30.8b, v18.8h, v10.8h + raddhn v31.8b, v19.8h, v11.8h +.endm + +/* TODO: expand macros and do better instructions scheduling */ +.macro pixman_composite_out_reverse_8888_8888_8888_process_pixblock_tail_head + ld4 {v4.8b, v5.8b, v6.8b, v7.8b}, [DST_R], #32 + pixman_composite_out_reverse_8888_n_8888_process_pixblock_tail + fetch_src_pixblock + cache_preload 8, 8 + fetch_mask_pixblock + pixman_composite_out_reverse_8888_n_8888_process_pixblock_head + st4 {v28.8b, v29.8b, v30.8b, v31.8b}, [DST_W], #32 +.endm + +generate_composite_function_single_scanline \ + pixman_composite_scanline_out_reverse_mask_asm_neon, 32, 32, 32, \ + FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \ + 8, /* number of pixels, processed in a single block */ \ + default_init_need_all_regs, \ + default_cleanup_need_all_regs, \ + pixman_composite_out_reverse_8888_n_8888_process_pixblock_head, \ + pixman_composite_out_reverse_8888_n_8888_process_pixblock_tail, \ + pixman_composite_out_reverse_8888_8888_8888_process_pixblock_tail_head \ + 28, /* dst_w_basereg */ \ + 4, /* dst_r_basereg */ \ + 0, /* src_basereg */ \ + 12 /* mask_basereg */ + +/******************************************************************************/ + +.macro pixman_composite_over_8888_n_8888_process_pixblock_head + pixman_composite_out_reverse_8888_n_8888_process_pixblock_head +.endm + +.macro pixman_composite_over_8888_n_8888_process_pixblock_tail + pixman_composite_out_reverse_8888_n_8888_process_pixblock_tail + uqadd v28.8b, v0.8b, v28.8b + uqadd v29.8b, v1.8b, v29.8b + uqadd v30.8b, v2.8b, v30.8b + uqadd v31.8b, v3.8b, v31.8b +.endm + +/* TODO: expand macros and do better instructions scheduling */ +.macro pixman_composite_over_8888_n_8888_process_pixblock_tail_head + ld4 {v4.8b, v5.8b, v6.8b, v7.8b}, [DST_R], #32 + pixman_composite_over_8888_n_8888_process_pixblock_tail + fetch_src_pixblock + cache_preload 8, 8 + pixman_composite_over_8888_n_8888_process_pixblock_head + st4 {v28.8b, v29.8b, v30.8b, v31.8b}, [DST_W], #32 +.endm + +.macro pixman_composite_over_8888_n_8888_init + mov v15.s[0], w6 + dup v15.8b, v15.b[3] +.endm + +.macro pixman_composite_over_8888_n_8888_cleanup +.endm + +generate_composite_function \ + pixman_composite_over_8888_n_8888_asm_neon, 32, 0, 32, \ + FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \ + 8, /* number of pixels, processed in a single block */ \ + 5, /* prefetch distance */ \ + pixman_composite_over_8888_n_8888_init, \ + pixman_composite_over_8888_n_8888_cleanup, \ + pixman_composite_over_8888_n_8888_process_pixblock_head, \ + pixman_composite_over_8888_n_8888_process_pixblock_tail, \ + pixman_composite_over_8888_n_8888_process_pixblock_tail_head, \ + 28, /* dst_w_basereg */ \ + 4, /* dst_r_basereg */ \ + 0, /* src_basereg */ \ + 12 /* mask_basereg */ + +/******************************************************************************/ + +/* TODO: expand macros and do better instructions scheduling */ +.macro pixman_composite_over_8888_8888_8888_process_pixblock_tail_head + ld4 {v4.8b, v5.8b, v6.8b, v7.8b}, [DST_R], #32 + pixman_composite_over_8888_n_8888_process_pixblock_tail + fetch_src_pixblock + cache_preload 8, 8 + fetch_mask_pixblock + pixman_composite_over_8888_n_8888_process_pixblock_head + st4 {v28.8b, v29.8b, v30.8b, v31.8b}, [DST_W], #32 +.endm + +generate_composite_function \ + pixman_composite_over_8888_8888_8888_asm_neon, 32, 32, 32, \ + FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \ + 8, /* number of pixels, processed in a single block */ \ + 5, /* prefetch distance */ \ + default_init_need_all_regs, \ + default_cleanup_need_all_regs, \ + pixman_composite_over_8888_n_8888_process_pixblock_head, \ + pixman_composite_over_8888_n_8888_process_pixblock_tail, \ + pixman_composite_over_8888_8888_8888_process_pixblock_tail_head \ + 28, /* dst_w_basereg */ \ + 4, /* dst_r_basereg */ \ + 0, /* src_basereg */ \ + 12 /* mask_basereg */ + +generate_composite_function_single_scanline \ + pixman_composite_scanline_over_mask_asm_neon, 32, 32, 32, \ + FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \ + 8, /* number of pixels, processed in a single block */ \ + default_init_need_all_regs, \ + default_cleanup_need_all_regs, \ + pixman_composite_over_8888_n_8888_process_pixblock_head, \ + pixman_composite_over_8888_n_8888_process_pixblock_tail, \ + pixman_composite_over_8888_8888_8888_process_pixblock_tail_head \ + 28, /* dst_w_basereg */ \ + 4, /* dst_r_basereg */ \ + 0, /* src_basereg */ \ + 12 /* mask_basereg */ + +/******************************************************************************/ + +/* TODO: expand macros and do better instructions scheduling */ +.macro pixman_composite_over_8888_8_8888_process_pixblock_tail_head + ld4 {v4.8b, v5.8b, v6.8b, v7.8b}, [DST_R], #32 + pixman_composite_over_8888_n_8888_process_pixblock_tail + fetch_src_pixblock + cache_preload 8, 8 + fetch_mask_pixblock + pixman_composite_over_8888_n_8888_process_pixblock_head + st4 {v28.8b, v29.8b, v30.8b, v31.8b}, [DST_W], #32 +.endm + +generate_composite_function \ + pixman_composite_over_8888_8_8888_asm_neon, 32, 8, 32, \ + FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \ + 8, /* number of pixels, processed in a single block */ \ + 5, /* prefetch distance */ \ + default_init_need_all_regs, \ + default_cleanup_need_all_regs, \ + pixman_composite_over_8888_n_8888_process_pixblock_head, \ + pixman_composite_over_8888_n_8888_process_pixblock_tail, \ + pixman_composite_over_8888_8_8888_process_pixblock_tail_head \ + 28, /* dst_w_basereg */ \ + 4, /* dst_r_basereg */ \ + 0, /* src_basereg */ \ + 15 /* mask_basereg */ + +/******************************************************************************/ + +.macro pixman_composite_src_0888_0888_process_pixblock_head +.endm + +.macro pixman_composite_src_0888_0888_process_pixblock_tail +.endm + +.macro pixman_composite_src_0888_0888_process_pixblock_tail_head + st3 {v0.8b, v1.8b, v2.8b}, [DST_W], #24 + fetch_src_pixblock + cache_preload 8, 8 +.endm + +generate_composite_function \ + pixman_composite_src_0888_0888_asm_neon, 24, 0, 24, \ + FLAG_DST_WRITEONLY, \ + 8, /* number of pixels, processed in a single block */ \ + 10, /* prefetch distance */ \ + default_init, \ + default_cleanup, \ + pixman_composite_src_0888_0888_process_pixblock_head, \ + pixman_composite_src_0888_0888_process_pixblock_tail, \ + pixman_composite_src_0888_0888_process_pixblock_tail_head, \ + 0, /* dst_w_basereg */ \ + 0, /* dst_r_basereg */ \ + 0, /* src_basereg */ \ + 0 /* mask_basereg */ + +/******************************************************************************/ + +.macro pixman_composite_src_0888_8888_rev_process_pixblock_head + mov v31.8b, v2.8b + mov v2.8b, v0.8b + mov v0.8b, v31.8b +.endm + +.macro pixman_composite_src_0888_8888_rev_process_pixblock_tail +.endm + +.macro pixman_composite_src_0888_8888_rev_process_pixblock_tail_head + st4 {v0.8b, v1.8b, v2.8b, v3.8b}, [DST_W], #32 + fetch_src_pixblock + mov v31.8b, v2.8b + mov v2.8b, v0.8b + mov v0.8b, v31.8b + cache_preload 8, 8 +.endm + +.macro pixman_composite_src_0888_8888_rev_init + eor v3.8b, v3.8b, v3.8b +.endm + +generate_composite_function \ + pixman_composite_src_0888_8888_rev_asm_neon, 24, 0, 32, \ + FLAG_DST_WRITEONLY | FLAG_DEINTERLEAVE_32BPP, \ + 8, /* number of pixels, processed in a single block */ \ + 10, /* prefetch distance */ \ + pixman_composite_src_0888_8888_rev_init, \ + default_cleanup, \ + pixman_composite_src_0888_8888_rev_process_pixblock_head, \ + pixman_composite_src_0888_8888_rev_process_pixblock_tail, \ + pixman_composite_src_0888_8888_rev_process_pixblock_tail_head, \ + 0, /* dst_w_basereg */ \ + 0, /* dst_r_basereg */ \ + 0, /* src_basereg */ \ + 0 /* mask_basereg */ + +/******************************************************************************/ + +.macro pixman_composite_src_0888_0565_rev_process_pixblock_head + ushll v8.8h, v1.8b, #7 + sli v8.8h, v8.8h, #1 + ushll v9.8h, v2.8b, #7 + sli v9.8h, v9.8h, #1 +.endm + +.macro pixman_composite_src_0888_0565_rev_process_pixblock_tail + ushll v14.8h, v0.8b, #7 + sli v14.8h, v14.8h, #1 + sri v14.8h, v8.8h, #5 + sri v14.8h, v9.8h, #11 + mov v28.d[0], v14.d[0] + mov v29.d[0], v14.d[1] +.endm + +.macro pixman_composite_src_0888_0565_rev_process_pixblock_tail_head + ushll v14.8h, v0.8b, #7 + sli v14.8h, v14.8h, #1 + fetch_src_pixblock + sri v14.8h, v8.8h, #5 + sri v14.8h, v9.8h, #11 + mov v28.d[0], v14.d[0] + mov v29.d[0], v14.d[1] + ushll v8.8h, v1.8b, #7 + sli v8.8h, v8.8h, #1 + st1 {v14.8h}, [DST_W], #16 + ushll v9.8h, v2.8b, #7 + sli v9.8h, v9.8h, #1 +.endm + +generate_composite_function \ + pixman_composite_src_0888_0565_rev_asm_neon, 24, 0, 16, \ + FLAG_DST_WRITEONLY, \ + 8, /* number of pixels, processed in a single block */ \ + 10, /* prefetch distance */ \ + default_init, \ + default_cleanup, \ + pixman_composite_src_0888_0565_rev_process_pixblock_head, \ + pixman_composite_src_0888_0565_rev_process_pixblock_tail, \ + pixman_composite_src_0888_0565_rev_process_pixblock_tail_head, \ + 28, /* dst_w_basereg */ \ + 0, /* dst_r_basereg */ \ + 0, /* src_basereg */ \ + 0 /* mask_basereg */ + +/******************************************************************************/ + +.macro pixman_composite_src_pixbuf_8888_process_pixblock_head + umull v8.8h, v3.8b, v0.8b + umull v9.8h, v3.8b, v1.8b + umull v10.8h, v3.8b, v2.8b +.endm + +.macro pixman_composite_src_pixbuf_8888_process_pixblock_tail + urshr v11.8h, v8.8h, #8 + mov v30.8b, v31.8b + mov v31.8b, v3.8b + mov v3.8b, v30.8b + urshr v12.8h, v9.8h, #8 + urshr v13.8h, v10.8h, #8 + raddhn v30.8b, v11.8h, v8.8h + raddhn v29.8b, v12.8h, v9.8h + raddhn v28.8b, v13.8h, v10.8h +.endm + +.macro pixman_composite_src_pixbuf_8888_process_pixblock_tail_head + urshr v11.8h, v8.8h, #8 + mov v30.8b, v31.8b + mov v31.8b, v3.8b + mov v3.8b, v31.8b + urshr v12.8h, v9.8h, #8 + urshr v13.8h, v10.8h, #8 + fetch_src_pixblock + raddhn v30.8b, v11.8h, v8.8h + PF add PF_X, PF_X, #8 + PF tst PF_CTL, #0xF + PF beq 10f + PF add PF_X, PF_X, #8 + PF sub PF_CTL, PF_CTL, #1 +10: + raddhn v29.8b, v12.8h, v9.8h + raddhn v28.8b, v13.8h, v10.8h + umull v8.8h, v3.8b, v0.8b + umull v9.8h, v3.8b, v1.8b + umull v10.8h, v3.8b, v2.8b + st4 {v28.8b, v29.8b, v30.8b, v31.8b}, [DST_W], #32 + PF cmp PF_X, ORIG_W + PF lsl DUMMY, PF_X, src_bpp_shift + PF prfm PREFETCH_MODE, [PF_SRC, DUMMY] + PF ble 10f + PF sub PF_X, PF_X, ORIG_W + PF subs PF_CTL, PF_CTL, #0x10 + PF ble 10f + PF lsl DUMMY, SRC_STRIDE, #src_bpp_shift + PF ldrsb DUMMY, [PF_SRC, DUMMY] + PF add PF_SRC, PF_SRC, #1 +10: +.endm + +generate_composite_function \ + pixman_composite_src_pixbuf_8888_asm_neon, 32, 0, 32, \ + FLAG_DST_WRITEONLY | FLAG_DEINTERLEAVE_32BPP, \ + 8, /* number of pixels, processed in a single block */ \ + 10, /* prefetch distance */ \ + default_init, \ + default_cleanup, \ + pixman_composite_src_pixbuf_8888_process_pixblock_head, \ + pixman_composite_src_pixbuf_8888_process_pixblock_tail, \ + pixman_composite_src_pixbuf_8888_process_pixblock_tail_head, \ + 28, /* dst_w_basereg */ \ + 0, /* dst_r_basereg */ \ + 0, /* src_basereg */ \ + 0 /* mask_basereg */ + +/******************************************************************************/ + +.macro pixman_composite_src_rpixbuf_8888_process_pixblock_head + umull v8.8h, v3.8b, v0.8b + umull v9.8h, v3.8b, v1.8b + umull v10.8h, v3.8b, v2.8b +.endm + +.macro pixman_composite_src_rpixbuf_8888_process_pixblock_tail + urshr v11.8h, v8.8h, #8 + mov v30.8b, v31.8b + mov v31.8b, v3.8b + mov v3.8b, v30.8b + urshr v12.8h, v9.8h, #8 + urshr v13.8h, v10.8h, #8 + raddhn v28.8b, v11.8h, v8.8h + raddhn v29.8b, v12.8h, v9.8h + raddhn v30.8b, v13.8h, v10.8h +.endm + +.macro pixman_composite_src_rpixbuf_8888_process_pixblock_tail_head + urshr v11.8h, v8.8h, #8 + mov v30.8b, v31.8b + mov v31.8b, v3.8b + mov v3.8b, v30.8b + urshr v12.8h, v9.8h, #8 + urshr v13.8h, v10.8h, #8 + fetch_src_pixblock + raddhn v28.8b, v11.8h, v8.8h + PF add PF_X, PF_X, #8 + PF tst PF_CTL, #0xF + PF beq 10f + PF add PF_X, PF_X, #8 + PF sub PF_CTL, PF_CTL, #1 +10: + raddhn v29.8b, v12.8h, v9.8h + raddhn v30.8b, v13.8h, v10.8h + umull v8.8h, v3.8b, v0.8b + umull v9.8h, v3.8b, v1.8b + umull v10.8h, v3.8b, v2.8b + st4 {v28.8b, v29.8b, v30.8b, v31.8b}, [DST_W], #32 + PF cmp PF_X, ORIG_W + PF lsl DUMMY, PF_X, src_bpp_shift + PF prfm PREFETCH_MODE, [PF_SRC, DUMMY] + PF ble 10f + PF sub PF_X, PF_X, ORIG_W + PF subs PF_CTL, PF_CTL, #0x10 + PF ble 10f + PF lsl DUMMY, SRC_STRIDE, #src_bpp_shift + PF ldrsb DUMMY, [PF_SRC, DUMMY] + PF add PF_SRC, PF_SRC, #1 +10: +.endm + +generate_composite_function \ + pixman_composite_src_rpixbuf_8888_asm_neon, 32, 0, 32, \ + FLAG_DST_WRITEONLY | FLAG_DEINTERLEAVE_32BPP, \ + 8, /* number of pixels, processed in a single block */ \ + 10, /* prefetch distance */ \ + default_init, \ + default_cleanup, \ + pixman_composite_src_rpixbuf_8888_process_pixblock_head, \ + pixman_composite_src_rpixbuf_8888_process_pixblock_tail, \ + pixman_composite_src_rpixbuf_8888_process_pixblock_tail_head, \ + 28, /* dst_w_basereg */ \ + 0, /* dst_r_basereg */ \ + 0, /* src_basereg */ \ + 0 /* mask_basereg */ + +/******************************************************************************/ + +.macro pixman_composite_over_0565_8_0565_process_pixblock_head + /* mask is in v15 */ + mov v4.d[0], v8.d[0] + mov v4.d[1], v9.d[0] + mov v13.d[0], v10.d[0] + mov v13.d[1], v11.d[0] + convert_0565_to_x888 v4, v2, v1, v0 + convert_0565_to_x888 v13, v6, v5, v4 + /* source pixel data is in {v0, v1, v2, XX} */ + /* destination pixel data is in {v4, v5, v6, XX} */ + mvn v7.8b, v15.8b + umull v10.8h, v15.8b, v2.8b + umull v9.8h, v15.8b, v1.8b + umull v8.8h, v15.8b, v0.8b + umull v11.8h, v7.8b, v4.8b + umull v12.8h, v7.8b, v5.8b + umull v13.8h, v7.8b, v6.8b + urshr v19.8h, v10.8h, #8 + urshr v18.8h, v9.8h, #8 + urshr v17.8h, v8.8h, #8 + raddhn v2.8b, v10.8h, v19.8h + raddhn v1.8b, v9.8h, v18.8h + raddhn v0.8b, v8.8h, v17.8h +.endm + +.macro pixman_composite_over_0565_8_0565_process_pixblock_tail + urshr v17.8h, v11.8h, #8 + urshr v18.8h, v12.8h, #8 + urshr v19.8h, v13.8h, #8 + raddhn v28.8b, v17.8h, v11.8h + raddhn v29.8b, v18.8h, v12.8h + raddhn v30.8b, v19.8h, v13.8h + uqadd v0.8b, v0.8b, v28.8b + uqadd v1.8b, v1.8b, v29.8b + uqadd v2.8b, v2.8b, v30.8b + /* 32bpp result is in {v0, v1, v2, XX} */ + convert_8888_to_0565 v2, v1, v0, v14, v30, v13 + mov v28.d[0], v14.d[0] + mov v29.d[0], v14.d[1] +.endm + +/* TODO: expand macros and do better instructions scheduling */ +.macro pixman_composite_over_0565_8_0565_process_pixblock_tail_head + fetch_mask_pixblock + pixman_composite_over_0565_8_0565_process_pixblock_tail + fetch_src_pixblock + ld1 {v10.4h, v11.4h}, [DST_R], #16 + cache_preload 8, 8 + pixman_composite_over_0565_8_0565_process_pixblock_head + st1 {v14.8h}, [DST_W], #16 +.endm + +generate_composite_function \ + pixman_composite_over_0565_8_0565_asm_neon, 16, 8, 16, \ + FLAG_DST_READWRITE, \ + 8, /* number of pixels, processed in a single block */ \ + 5, /* prefetch distance */ \ + default_init_need_all_regs, \ + default_cleanup_need_all_regs, \ + pixman_composite_over_0565_8_0565_process_pixblock_head, \ + pixman_composite_over_0565_8_0565_process_pixblock_tail, \ + pixman_composite_over_0565_8_0565_process_pixblock_tail_head, \ + 28, /* dst_w_basereg */ \ + 10, /* dst_r_basereg */ \ + 8, /* src_basereg */ \ + 15 /* mask_basereg */ + +/******************************************************************************/ + +.macro pixman_composite_over_0565_n_0565_init + mov v15.s[0], w6 + dup v15.8b, v15.b[3] +.endm + +.macro pixman_composite_over_0565_n_0565_cleanup +.endm + +generate_composite_function \ + pixman_composite_over_0565_n_0565_asm_neon, 16, 0, 16, \ + FLAG_DST_READWRITE, \ + 8, /* number of pixels, processed in a single block */ \ + 5, /* prefetch distance */ \ + pixman_composite_over_0565_n_0565_init, \ + pixman_composite_over_0565_n_0565_cleanup, \ + pixman_composite_over_0565_8_0565_process_pixblock_head, \ + pixman_composite_over_0565_8_0565_process_pixblock_tail, \ + pixman_composite_over_0565_8_0565_process_pixblock_tail_head, \ + 28, /* dst_w_basereg */ \ + 10, /* dst_r_basereg */ \ + 8, /* src_basereg */ \ + 15 /* mask_basereg */ + +/******************************************************************************/ + +.macro pixman_composite_add_0565_8_0565_process_pixblock_head + /* mask is in v15 */ + mov v4.d[0], v8.d[0] + mov v4.d[1], v9.d[0] + mov v13.d[0], v10.d[0] + mov v13.d[1], v11.d[0] + convert_0565_to_x888 v4, v2, v1, v0 + convert_0565_to_x888 v13, v6, v5, v4 + /* source pixel data is in {v0, v1, v2, XX} */ + /* destination pixel data is in {v4, v5, v6, XX} */ + umull v9.8h, v15.8b, v2.8b + umull v8.8h, v15.8b, v1.8b + umull v7.8h, v15.8b, v0.8b + urshr v12.8h, v9.8h, #8 + urshr v11.8h, v8.8h, #8 + urshr v10.8h, v7.8h, #8 + raddhn v2.8b, v9.8h, v12.8h + raddhn v1.8b, v8.8h, v11.8h + raddhn v0.8b, v7.8h, v10.8h +.endm + +.macro pixman_composite_add_0565_8_0565_process_pixblock_tail + uqadd v0.8b, v0.8b, v4.8b + uqadd v1.8b, v1.8b, v5.8b + uqadd v2.8b, v2.8b, v6.8b + /* 32bpp result is in {v0, v1, v2, XX} */ + convert_8888_to_0565 v2, v1, v0, v14, v30, v13 + mov v28.d[0], v14.d[0] + mov v29.d[0], v14.d[1] +.endm + +/* TODO: expand macros and do better instructions scheduling */ +.macro pixman_composite_add_0565_8_0565_process_pixblock_tail_head + fetch_mask_pixblock + pixman_composite_add_0565_8_0565_process_pixblock_tail + fetch_src_pixblock + ld1 {v10.4h, v11.4h}, [DST_R], #16 + cache_preload 8, 8 + pixman_composite_add_0565_8_0565_process_pixblock_head + st1 {v14.8h}, [DST_W], #16 +.endm + +generate_composite_function \ + pixman_composite_add_0565_8_0565_asm_neon, 16, 8, 16, \ + FLAG_DST_READWRITE, \ + 8, /* number of pixels, processed in a single block */ \ + 5, /* prefetch distance */ \ + default_init_need_all_regs, \ + default_cleanup_need_all_regs, \ + pixman_composite_add_0565_8_0565_process_pixblock_head, \ + pixman_composite_add_0565_8_0565_process_pixblock_tail, \ + pixman_composite_add_0565_8_0565_process_pixblock_tail_head, \ + 28, /* dst_w_basereg */ \ + 10, /* dst_r_basereg */ \ + 8, /* src_basereg */ \ + 15 /* mask_basereg */ + +/******************************************************************************/ + +.macro pixman_composite_out_reverse_8_0565_process_pixblock_head + /* mask is in v15 */ + mov v12.d[0], v10.d[0] + mov v12.d[1], v11.d[0] + convert_0565_to_x888 v12, v6, v5, v4 + /* destination pixel data is in {v4, v5, v6, xx} */ + mvn v24.8b, v15.8b /* get inverted alpha */ + /* now do alpha blending */ + umull v8.8h, v24.8b, v4.8b + umull v9.8h, v24.8b, v5.8b + umull v10.8h, v24.8b, v6.8b +.endm + +.macro pixman_composite_out_reverse_8_0565_process_pixblock_tail + urshr v11.8h, v8.8h, #8 + urshr v12.8h, v9.8h, #8 + urshr v13.8h, v10.8h, #8 + raddhn v0.8b, v11.8h, v8.8h + raddhn v1.8b, v12.8h, v9.8h + raddhn v2.8b, v13.8h, v10.8h + /* 32bpp result is in {v0, v1, v2, XX} */ + convert_8888_to_0565 v2, v1, v0, v14, v12, v3 + mov v28.d[0], v14.d[0] + mov v29.d[0], v14.d[1] +.endm + +/* TODO: expand macros and do better instructions scheduling */ +.macro pixman_composite_out_reverse_8_0565_process_pixblock_tail_head + fetch_src_pixblock + pixman_composite_out_reverse_8_0565_process_pixblock_tail + ld1 {v10.4h, v11.4h}, [DST_R], #16 + cache_preload 8, 8 + pixman_composite_out_reverse_8_0565_process_pixblock_head + st1 {v14.8h}, [DST_W], #16 +.endm + +generate_composite_function \ + pixman_composite_out_reverse_8_0565_asm_neon, 8, 0, 16, \ + FLAG_DST_READWRITE, \ + 8, /* number of pixels, processed in a single block */ \ + 5, /* prefetch distance */ \ + default_init_need_all_regs, \ + default_cleanup_need_all_regs, \ + pixman_composite_out_reverse_8_0565_process_pixblock_head, \ + pixman_composite_out_reverse_8_0565_process_pixblock_tail, \ + pixman_composite_out_reverse_8_0565_process_pixblock_tail_head, \ + 28, /* dst_w_basereg */ \ + 10, /* dst_r_basereg */ \ + 15, /* src_basereg */ \ + 0 /* mask_basereg */ + +/******************************************************************************/ + +.macro pixman_composite_out_reverse_8_8888_process_pixblock_head + /* src is in v0 */ + /* destination pixel data is in {v4, v5, v6, v7} */ + mvn v1.8b, v0.8b /* get inverted alpha */ + /* now do alpha blending */ + umull v8.8h, v1.8b, v4.8b + umull v9.8h, v1.8b, v5.8b + umull v10.8h, v1.8b, v6.8b + umull v11.8h, v1.8b, v7.8b +.endm + +.macro pixman_composite_out_reverse_8_8888_process_pixblock_tail + urshr v14.8h, v8.8h, #8 + urshr v15.8h, v9.8h, #8 + urshr v12.8h, v10.8h, #8 + urshr v13.8h, v11.8h, #8 + raddhn v28.8b, v14.8h, v8.8h + raddhn v29.8b, v15.8h, v9.8h + raddhn v30.8b, v12.8h, v10.8h + raddhn v31.8b, v13.8h, v11.8h + /* 32bpp result is in {v28, v29, v30, v31} */ +.endm + +/* TODO: expand macros and do better instructions scheduling */ +.macro pixman_composite_out_reverse_8_8888_process_pixblock_tail_head + fetch_src_pixblock + pixman_composite_out_reverse_8_8888_process_pixblock_tail + ld4 {v4.8b, v5.8b, v6.8b, v7.8b}, [DST_R], #32 + cache_preload 8, 8 + pixman_composite_out_reverse_8_8888_process_pixblock_head + st4 {v28.8b, v29.8b, v30.8b, v31.8b}, [DST_W], #32 +.endm + +generate_composite_function \ + pixman_composite_out_reverse_8_8888_asm_neon, 8, 0, 32, \ + FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \ + 8, /* number of pixels, processed in a single block */ \ + 5, /* prefetch distance */ \ + default_init, \ + default_cleanup, \ + pixman_composite_out_reverse_8_8888_process_pixblock_head, \ + pixman_composite_out_reverse_8_8888_process_pixblock_tail, \ + pixman_composite_out_reverse_8_8888_process_pixblock_tail_head, \ + 28, /* dst_w_basereg */ \ + 4, /* dst_r_basereg */ \ + 0, /* src_basereg */ \ + 0 /* mask_basereg */ + +/******************************************************************************/ + +generate_composite_function_nearest_scanline \ + pixman_scaled_nearest_scanline_8888_8888_OVER_asm_neon, 32, 0, 32, \ + FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \ + 8, /* number of pixels, processed in a single block */ \ + default_init, \ + default_cleanup, \ + pixman_composite_over_8888_8888_process_pixblock_head, \ + pixman_composite_over_8888_8888_process_pixblock_tail, \ + pixman_composite_over_8888_8888_process_pixblock_tail_head + +generate_composite_function_nearest_scanline \ + pixman_scaled_nearest_scanline_8888_0565_OVER_asm_neon, 32, 0, 16, \ + FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \ + 8, /* number of pixels, processed in a single block */ \ + default_init, \ + default_cleanup, \ + pixman_composite_over_8888_0565_process_pixblock_head, \ + pixman_composite_over_8888_0565_process_pixblock_tail, \ + pixman_composite_over_8888_0565_process_pixblock_tail_head, \ + 28, /* dst_w_basereg */ \ + 4, /* dst_r_basereg */ \ + 0, /* src_basereg */ \ + 24 /* mask_basereg */ + +generate_composite_function_nearest_scanline \ + pixman_scaled_nearest_scanline_8888_0565_SRC_asm_neon, 32, 0, 16, \ + FLAG_DST_WRITEONLY | FLAG_DEINTERLEAVE_32BPP, \ + 8, /* number of pixels, processed in a single block */ \ + default_init, \ + default_cleanup, \ + pixman_composite_src_8888_0565_process_pixblock_head, \ + pixman_composite_src_8888_0565_process_pixblock_tail, \ + pixman_composite_src_8888_0565_process_pixblock_tail_head, \ + +generate_composite_function_nearest_scanline \ + pixman_scaled_nearest_scanline_0565_8888_SRC_asm_neon, 16, 0, 32, \ + FLAG_DST_WRITEONLY | FLAG_DEINTERLEAVE_32BPP, \ + 8, /* number of pixels, processed in a single block */ \ + default_init, \ + default_cleanup, \ + pixman_composite_src_0565_8888_process_pixblock_head, \ + pixman_composite_src_0565_8888_process_pixblock_tail, \ + pixman_composite_src_0565_8888_process_pixblock_tail_head + +generate_composite_function_nearest_scanline \ + pixman_scaled_nearest_scanline_8888_8_0565_OVER_asm_neon, 32, 8, 16, \ + FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \ + 8, /* number of pixels, processed in a single block */ \ + default_init_need_all_regs, \ + default_cleanup_need_all_regs, \ + pixman_composite_over_8888_8_0565_process_pixblock_head, \ + pixman_composite_over_8888_8_0565_process_pixblock_tail, \ + pixman_composite_over_8888_8_0565_process_pixblock_tail_head, \ + 28, /* dst_w_basereg */ \ + 4, /* dst_r_basereg */ \ + 8, /* src_basereg */ \ + 24 /* mask_basereg */ + +generate_composite_function_nearest_scanline \ + pixman_scaled_nearest_scanline_0565_8_0565_OVER_asm_neon, 16, 8, 16, \ + FLAG_DST_READWRITE, \ + 8, /* number of pixels, processed in a single block */ \ + default_init_need_all_regs, \ + default_cleanup_need_all_regs, \ + pixman_composite_over_0565_8_0565_process_pixblock_head, \ + pixman_composite_over_0565_8_0565_process_pixblock_tail, \ + pixman_composite_over_0565_8_0565_process_pixblock_tail_head, \ + 28, /* dst_w_basereg */ \ + 10, /* dst_r_basereg */ \ + 8, /* src_basereg */ \ + 15 /* mask_basereg */ + +/******************************************************************************/ + +/* + * Bilinear scaling support code which tries to provide pixel fetching, color + * format conversion, and interpolation as separate macros which can be used + * as the basic building blocks for constructing bilinear scanline functions. + */ + +.macro bilinear_load_8888 reg1, reg2, tmp + asr TMP1, X, #16 + add X, X, UX + add TMP1, TOP, TMP1, lsl #2 + ld1 {®1&.2s}, [TMP1], STRIDE + ld1 {®2&.2s}, [TMP1] +.endm + +.macro bilinear_load_0565 reg1, reg2, tmp + asr TMP1, X, #16 + add X, X, UX + add TMP1, TOP, TMP1, lsl #1 + ld1 {®2&.s}[0], [TMP1], STRIDE + ld1 {®2&.s}[1], [TMP1] + convert_four_0565_to_x888_packed reg2, reg1, reg2, tmp +.endm + +.macro bilinear_load_and_vertical_interpolate_two_8888 \ + acc1, acc2, reg1, reg2, reg3, reg4, tmp1, tmp2 + + bilinear_load_8888 reg1, reg2, tmp1 + umull &acc1&.8h, ®1&.8b, v28.8b + umlal &acc1&.8h, ®2&.8b, v29.8b + bilinear_load_8888 reg3, reg4, tmp2 + umull &acc2&.8h, ®3&.8b, v28.8b + umlal &acc2&.8h, ®4&.8b, v29.8b +.endm + +.macro bilinear_load_and_vertical_interpolate_four_8888 \ + xacc1, xacc2, xreg1, xreg2, xreg3, xreg4, xacc2lo, xacc2hi \ + yacc1, yacc2, yreg1, yreg2, yreg3, yreg4, yacc2lo, yacc2hi + + bilinear_load_and_vertical_interpolate_two_8888 \ + xacc1, xacc2, xreg1, xreg2, xreg3, xreg4, xacc2lo, xacc2hi + bilinear_load_and_vertical_interpolate_two_8888 \ + yacc1, yacc2, yreg1, yreg2, yreg3, yreg4, yacc2lo, yacc2hi +.endm + +.macro vzip reg1, reg2 + umov TMP4, v31.d[0] + zip1 v31.8b, reg1, reg2 + zip2 reg2, reg1, reg2 + mov reg1, v31.8b + mov v31.d[0], TMP4 +.endm + +.macro vuzp reg1, reg2 + umov TMP4, v31.d[0] + uzp1 v31.8b, reg1, reg2 + uzp2 reg2, reg1, reg2 + mov reg1, v31.8b + mov v31.d[0], TMP4 +.endm + +.macro bilinear_load_and_vertical_interpolate_two_0565 \ + acc1, acc2, reg1, reg2, reg3, reg4, acc2lo, acc2hi + asr TMP1, X, #16 + add X, X, UX + add TMP1, TOP, TMP1, lsl #1 + asr TMP2, X, #16 + add X, X, UX + add TMP2, TOP, TMP2, lsl #1 + ld1 {&acc2&.s}[0], [TMP1], STRIDE + ld1 {&acc2&.s}[2], [TMP2], STRIDE + ld1 {&acc2&.s}[1], [TMP1] + ld1 {&acc2&.s}[3], [TMP2] + convert_0565_to_x888 acc2, reg3, reg2, reg1 + vzip ®1&.8b, ®3&.8b + vzip ®2&.8b, ®4&.8b + vzip ®3&.8b, ®4&.8b + vzip ®1&.8b, ®2&.8b + umull &acc1&.8h, ®1&.8b, v28.8b + umlal &acc1&.8h, ®2&.8b, v29.8b + umull &acc2&.8h, ®3&.8b, v28.8b + umlal &acc2&.8h, ®4&.8b, v29.8b +.endm + +.macro bilinear_load_and_vertical_interpolate_four_0565 \ + xacc1, xacc2, xreg1, xreg2, xreg3, xreg4, xacc2lo, xacc2hi \ + yacc1, yacc2, yreg1, yreg2, yreg3, yreg4, yacc2lo, yacc2hi + asr TMP1, X, #16 + add X, X, UX + add TMP1, TOP, TMP1, lsl #1 + asr TMP2, X, #16 + add X, X, UX + add TMP2, TOP, TMP2, lsl #1 + ld1 {&xacc2&.s}[0], [TMP1], STRIDE + ld1 {&xacc2&.s}[2], [TMP2], STRIDE + ld1 {&xacc2&.s}[1], [TMP1] + ld1 {&xacc2&.s}[3], [TMP2] + convert_0565_to_x888 xacc2, xreg3, xreg2, xreg1 + asr TMP1, X, #16 + add X, X, UX + add TMP1, TOP, TMP1, lsl #1 + asr TMP2, X, #16 + add X, X, UX + add TMP2, TOP, TMP2, lsl #1 + ld1 {&yacc2&.s}[0], [TMP1], STRIDE + vzip &xreg1&.8b, &xreg3&.8b + ld1 {&yacc2&.s}[2], [TMP2], STRIDE + vzip &xreg2&.8b, &xreg4&.8b + ld1 {&yacc2&.s}[1], [TMP1] + vzip &xreg3&.8b, &xreg4&.8b + ld1 {&yacc2&.s}[3], [TMP2] + vzip &xreg1&.8b, &xreg2&.8b + convert_0565_to_x888 yacc2, yreg3, yreg2, yreg1 + umull &xacc1&.8h, &xreg1&.8b, v28.8b + vzip &yreg1&.8b, &yreg3&.8b + umlal &xacc1&.8h, &xreg2&.8b, v29.8b + vzip &yreg2&.8b, &yreg4&.8b + umull &xacc2&.8h, &xreg3&.8b, v28.8b + vzip &yreg3&.8b, &yreg4&.8b + umlal &xacc2&.8h, &xreg4&.8b, v29.8b + vzip &yreg1&.8b, &yreg2&.8b + umull &yacc1&.8h, &yreg1&.8b, v28.8b + umlal &yacc1&.8h, &yreg2&.8b, v29.8b + umull &yacc2&.8h, &yreg3&.8b, v28.8b + umlal &yacc2&.8h, &yreg4&.8b, v29.8b +.endm + +.macro bilinear_store_8888 numpix, tmp1, tmp2 +.if numpix == 4 + st1 {v0.2s, v1.2s}, [OUT], #16 +.elseif numpix == 2 + st1 {v0.2s}, [OUT], #8 +.elseif numpix == 1 + st1 {v0.s}[0], [OUT], #4 +.else + .error bilinear_store_8888 numpix is unsupported +.endif +.endm + +.macro bilinear_store_0565 numpix, tmp1, tmp2 + vuzp v0.8b, v1.8b + vuzp v2.8b, v3.8b + vuzp v1.8b, v3.8b + vuzp v0.8b, v2.8b + convert_8888_to_0565 v2, v1, v0, v1, tmp1, tmp2 +.if numpix == 4 + st1 {v1.4h}, [OUT], #8 +.elseif numpix == 2 + st1 {v1.s}[0], [OUT], #4 +.elseif numpix == 1 + st1 {v1.h}[0], [OUT], #2 +.else + .error bilinear_store_0565 numpix is unsupported +.endif +.endm + +.macro bilinear_interpolate_last_pixel src_fmt, dst_fmt + bilinear_load_&src_fmt v0, v1, v2 + umull v2.8h, v0.8b, v28.8b + umlal v2.8h, v1.8b, v29.8b + /* 5 cycles bubble */ + ushll v0.4s, v2.4h, #BILINEAR_INTERPOLATION_BITS + umlsl v0.4s, v2.4h, v15.h[0] + umlal2 v0.4s, v2.8h, v15.h[0] + /* 5 cycles bubble */ + shrn v0.4h, v0.4s, #(2 * BILINEAR_INTERPOLATION_BITS) + /* 3 cycles bubble */ + xtn v0.8b, v0.8h + /* 1 cycle bubble */ + bilinear_store_&dst_fmt 1, v3, v4 +.endm + +.macro bilinear_interpolate_two_pixels src_fmt, dst_fmt + bilinear_load_and_vertical_interpolate_two_&src_fmt \ + v1, v11, v2, v3, v20, v21, v22, v23 + ushll v0.4s, v1.4h, #BILINEAR_INTERPOLATION_BITS + umlsl v0.4s, v1.4h, v15.h[0] + umlal2 v0.4s, v1.8h, v15.h[0] + ushll v10.4s, v11.4h, #BILINEAR_INTERPOLATION_BITS + umlsl v10.4s, v11.4h, v15.h[4] + umlal2 v10.4s, v11.8h, v15.h[4] + shrn v0.4h, v0.4s, #(2 * BILINEAR_INTERPOLATION_BITS) + shrn2 v0.8h, v10.4s, #(2 * BILINEAR_INTERPOLATION_BITS) + ushr v15.8h, v12.8h, #(16 - BILINEAR_INTERPOLATION_BITS) + add v12.8h, v12.8h, v13.8h + xtn v0.8b, v0.8h + bilinear_store_&dst_fmt 2, v3, v4 +.endm + +.macro bilinear_interpolate_four_pixels src_fmt, dst_fmt + bilinear_load_and_vertical_interpolate_four_&src_fmt \ + v1, v11, v14, v20, v16, v17, v22, v23 \ + v3, v9, v24, v25, v26, v27, v18, v19 + prfm PREFETCH_MODE, [TMP1, PF_OFFS] + sub TMP1, TMP1, STRIDE + ushll v0.4s, v1.4h, #BILINEAR_INTERPOLATION_BITS + umlsl v0.4s, v1.4h, v15.h[0] + umlal2 v0.4s, v1.8h, v15.h[0] + ushll v10.4s, v11.4h, #BILINEAR_INTERPOLATION_BITS + umlsl v10.4s, v11.4h, v15.h[4] + umlal2 v10.4s, v11.8h, v15.h[4] + ushr v15.8h, v12.8h, #(16 - BILINEAR_INTERPOLATION_BITS) + ushll v2.4s, v3.4h, #BILINEAR_INTERPOLATION_BITS + umlsl v2.4s, v3.4h, v15.h[0] + umlal2 v2.4s, v3.8h, v15.h[0] + ushll v8.4s, v9.4h, #BILINEAR_INTERPOLATION_BITS + prfm PREFETCH_MODE, [TMP2, PF_OFFS] + umlsl v8.4s, v9.4h, v15.h[4] + umlal2 v8.4s, v9.8h, v15.h[4] + add v12.8h, v12.8h, v13.8h + shrn v0.4h, v0.4s, #(2 * BILINEAR_INTERPOLATION_BITS) + shrn2 v0.8h, v10.4s, #(2 * BILINEAR_INTERPOLATION_BITS) + shrn v2.4h, v2.4s, #(2 * BILINEAR_INTERPOLATION_BITS) + shrn2 v2.8h, v8.4s, #(2 * BILINEAR_INTERPOLATION_BITS) + ushr v15.8h, v12.8h, #(16 - BILINEAR_INTERPOLATION_BITS) + xtn v0.8b, v0.8h + xtn v1.8b, v2.8h + add v12.8h, v12.8h, v13.8h + bilinear_store_&dst_fmt 4, v3, v4 +.endm + +.macro bilinear_interpolate_four_pixels_head src_fmt, dst_fmt +.ifdef have_bilinear_interpolate_four_pixels_&src_fmt&_&dst_fmt + bilinear_interpolate_four_pixels_&src_fmt&_&dst_fmt&_head +.else + bilinear_interpolate_four_pixels src_fmt, dst_fmt +.endif +.endm + +.macro bilinear_interpolate_four_pixels_tail src_fmt, dst_fmt +.ifdef have_bilinear_interpolate_four_pixels_&src_fmt&_&dst_fmt + bilinear_interpolate_four_pixels_&src_fmt&_&dst_fmt&_tail +.endif +.endm + +.macro bilinear_interpolate_four_pixels_tail_head src_fmt, dst_fmt +.ifdef have_bilinear_interpolate_four_pixels_&src_fmt&_&dst_fmt + bilinear_interpolate_four_pixels_&src_fmt&_&dst_fmt&_tail_head +.else + bilinear_interpolate_four_pixels src_fmt, dst_fmt +.endif +.endm + +.macro bilinear_interpolate_eight_pixels_head src_fmt, dst_fmt +.ifdef have_bilinear_interpolate_eight_pixels_&src_fmt&_&dst_fmt + bilinear_interpolate_eight_pixels_&src_fmt&_&dst_fmt&_head +.else + bilinear_interpolate_four_pixels_head src_fmt, dst_fmt + bilinear_interpolate_four_pixels_tail_head src_fmt, dst_fmt +.endif +.endm + +.macro bilinear_interpolate_eight_pixels_tail src_fmt, dst_fmt +.ifdef have_bilinear_interpolate_eight_pixels_&src_fmt&_&dst_fmt + bilinear_interpolate_eight_pixels_&src_fmt&_&dst_fmt&_tail +.else + bilinear_interpolate_four_pixels_tail src_fmt, dst_fmt +.endif +.endm + +.macro bilinear_interpolate_eight_pixels_tail_head src_fmt, dst_fmt +.ifdef have_bilinear_interpolate_eight_pixels_&src_fmt&_&dst_fmt + bilinear_interpolate_eight_pixels_&src_fmt&_&dst_fmt&_tail_head +.else + bilinear_interpolate_four_pixels_tail_head src_fmt, dst_fmt + bilinear_interpolate_four_pixels_tail_head src_fmt, dst_fmt +.endif +.endm + +.set BILINEAR_FLAG_UNROLL_4, 0 +.set BILINEAR_FLAG_UNROLL_8, 1 +.set BILINEAR_FLAG_USE_ALL_NEON_REGS, 2 + +/* + * Main template macro for generating NEON optimized bilinear scanline + * functions. + * + * Bilinear scanline scaler macro template uses the following arguments: + * fname - name of the function to generate + * src_fmt - source color format (8888 or 0565) + * dst_fmt - destination color format (8888 or 0565) + * bpp_shift - (1 << bpp_shift) is the size of source pixel in bytes + * prefetch_distance - prefetch in the source image by that many + * pixels ahead + */ + +.macro generate_bilinear_scanline_func fname, src_fmt, dst_fmt, \ + src_bpp_shift, dst_bpp_shift, \ + prefetch_distance, flags + +pixman_asm_function fname + OUT .req x0 + TOP .req x1 + BOTTOM .req x2 + WT .req x3 + WB .req x4 + X .req x5 + UX .req x6 + WIDTH .req x7 + TMP1 .req x8 + TMP2 .req x9 + PF_OFFS .req x10 + TMP3 .req x11 + TMP4 .req x12 + STRIDE .req x13 + + sxtw x3, w3 + sxtw x4, w4 + sxtw x5, w5 + sxtw x6, w6 + sxtw x7, w7 + + stp x29, x30, [sp, -16]! + mov x29, sp + sub sp, sp, 112 /* push all registers */ + sub x29, x29, 64 + st1 {v8.8b, v9.8b, v10.8b, v11.8b}, [x29], #32 + st1 {v12.8b, v13.8b, v14.8b, v15.8b}, [x29], #32 + stp x8, x9, [x29, -80] + stp x10, x11, [x29, -96] + stp x12, x13, [x29, -112] + + mov PF_OFFS, #prefetch_distance + mul PF_OFFS, PF_OFFS, UX + + subs STRIDE, BOTTOM, TOP + .unreq BOTTOM + + cmp WIDTH, #0 + ble 300f + + dup v12.8h, w5 + dup v13.8h, w6 + dup v28.8b, w3 + dup v29.8b, w4 + mov v25.d[0], v12.d[1] + mov v26.d[0], v13.d[0] + add v25.4h, v25.4h, v26.4h + mov v12.d[1], v25.d[0] + + /* ensure good destination alignment */ + cmp WIDTH, #1 + blt 100f + tst OUT, #(1 << dst_bpp_shift) + beq 100f + ushr v15.8h, v12.8h, #(16 - BILINEAR_INTERPOLATION_BITS) + add v12.8h, v12.8h, v13.8h + bilinear_interpolate_last_pixel src_fmt, dst_fmt + sub WIDTH, WIDTH, #1 +100: + add v13.8h, v13.8h, v13.8h + ushr v15.8h, v12.8h, #(16 - BILINEAR_INTERPOLATION_BITS) + add v12.8h, v12.8h, v13.8h + + cmp WIDTH, #2 + blt 100f + tst OUT, #(1 << (dst_bpp_shift + 1)) + beq 100f + bilinear_interpolate_two_pixels src_fmt, dst_fmt + sub WIDTH, WIDTH, #2 +100: +.if ((flags) & BILINEAR_FLAG_UNROLL_8) != 0 +/*********** 8 pixels per iteration *****************/ + cmp WIDTH, #4 + blt 100f + tst OUT, #(1 << (dst_bpp_shift + 2)) + beq 100f + bilinear_interpolate_four_pixels src_fmt, dst_fmt + sub WIDTH, WIDTH, #4 +100: + subs WIDTH, WIDTH, #8 + blt 100f + asr PF_OFFS, PF_OFFS, #(16 - src_bpp_shift) + bilinear_interpolate_eight_pixels_head src_fmt, dst_fmt + subs WIDTH, WIDTH, #8 + blt 500f +1000: + bilinear_interpolate_eight_pixels_tail_head src_fmt, dst_fmt + subs WIDTH, WIDTH, #8 + bge 1000b +500: + bilinear_interpolate_eight_pixels_tail src_fmt, dst_fmt +100: + tst WIDTH, #4 + beq 200f + bilinear_interpolate_four_pixels src_fmt, dst_fmt +200: +.else +/*********** 4 pixels per iteration *****************/ + subs WIDTH, WIDTH, #4 + blt 100f + asr PF_OFFS, PF_OFFS, #(16 - src_bpp_shift) + bilinear_interpolate_four_pixels_head src_fmt, dst_fmt + subs WIDTH, WIDTH, #4 + blt 500f +1000: + bilinear_interpolate_four_pixels_tail_head src_fmt, dst_fmt + subs WIDTH, WIDTH, #4 + bge 1000b +500: + bilinear_interpolate_four_pixels_tail src_fmt, dst_fmt +100: +/****************************************************/ +.endif + /* handle the remaining trailing pixels */ + tst WIDTH, #2 + beq 200f + bilinear_interpolate_two_pixels src_fmt, dst_fmt +200: + tst WIDTH, #1 + beq 300f + bilinear_interpolate_last_pixel src_fmt, dst_fmt +300: + sub x29, x29, 64 + ld1 {v8.8b, v9.8b, v10.8b, v11.8b}, [x29], #32 + ld1 {v12.8b, v13.8b, v14.8b, v15.8b}, [x29], #32 + ldp x8, x9, [x29, -80] + ldp x10, x11, [x29, -96] + ldp x12, x13, [x29, -104] + mov sp, x29 + ldp x29, x30, [sp], 16 + ret + + .unreq OUT + .unreq TOP + .unreq WT + .unreq WB + .unreq X + .unreq UX + .unreq WIDTH + .unreq TMP1 + .unreq TMP2 + .unreq PF_OFFS + .unreq TMP3 + .unreq TMP4 + .unreq STRIDE +.endfunc + +.endm + +/*****************************************************************************/ + +.set have_bilinear_interpolate_four_pixels_8888_8888, 1 + +.macro bilinear_interpolate_four_pixels_8888_8888_head + asr TMP1, X, #16 + add X, X, UX + add TMP1, TOP, TMP1, lsl #2 + asr TMP2, X, #16 + add X, X, UX + add TMP2, TOP, TMP2, lsl #2 + + ld1 {v22.2s}, [TMP1], STRIDE + ld1 {v23.2s}, [TMP1] + asr TMP3, X, #16 + add X, X, UX + add TMP3, TOP, TMP3, lsl #2 + umull v8.8h, v22.8b, v28.8b + umlal v8.8h, v23.8b, v29.8b + + ld1 {v22.2s}, [TMP2], STRIDE + ld1 {v23.2s}, [TMP2] + asr TMP4, X, #16 + add X, X, UX + add TMP4, TOP, TMP4, lsl #2 + umull v9.8h, v22.8b, v28.8b + umlal v9.8h, v23.8b, v29.8b + + ld1 {v22.2s}, [TMP3], STRIDE + ld1 {v23.2s}, [TMP3] + umull v10.8h, v22.8b, v28.8b + umlal v10.8h, v23.8b, v29.8b + + ushll v0.4s, v8.4h, #BILINEAR_INTERPOLATION_BITS + umlsl v0.4s, v8.4h, v15.h[0] + umlal2 v0.4s, v8.8h, v15.h[0] + + prfm PREFETCH_MODE, [TMP4, PF_OFFS] + ld1 {v16.2s}, [TMP4], STRIDE + ld1 {v17.2s}, [TMP4] + prfm PREFETCH_MODE, [TMP4, PF_OFFS] + umull v11.8h, v16.8b, v28.8b + umlal v11.8h, v17.8b, v29.8b + + ushll v1.4s, v9.4h, #BILINEAR_INTERPOLATION_BITS + umlsl v1.4s, v9.4h, v15.h[4] +.endm + +.macro bilinear_interpolate_four_pixels_8888_8888_tail + umlal2 v1.4s, v9.8h, v15.h[4] + ushr v15.8h, v12.8h, #(16 - BILINEAR_INTERPOLATION_BITS) + ushll v2.4s, v10.4h, #BILINEAR_INTERPOLATION_BITS + umlsl v2.4s, v10.4h, v15.h[0] + umlal2 v2.4s, v10.8h, v15.h[0] + ushll v3.4s, v11.4h, #BILINEAR_INTERPOLATION_BITS + umlsl v3.4s, v11.4h, v15.h[4] + umlal2 v3.4s, v11.8h, v15.h[4] + add v12.8h, v12.8h, v13.8h + shrn v0.4h, v0.4s, #(2 * BILINEAR_INTERPOLATION_BITS) + shrn2 v0.8h, v1.4s, #(2 * BILINEAR_INTERPOLATION_BITS) + shrn v2.4h, v2.4s, #(2 * BILINEAR_INTERPOLATION_BITS) + ushr v15.8h, v12.8h, #(16 - BILINEAR_INTERPOLATION_BITS) + shrn2 v2.8h, v3.4s, #(2 * BILINEAR_INTERPOLATION_BITS) + xtn v6.8b, v0.8h + xtn v7.8b, v2.8h + add v12.8h, v12.8h, v13.8h + st1 {v6.2s, v7.2s}, [OUT], #16 +.endm + +.macro bilinear_interpolate_four_pixels_8888_8888_tail_head + asr TMP1, X, #16 + add X, X, UX + add TMP1, TOP, TMP1, lsl #2 + asr TMP2, X, #16 + add X, X, UX + add TMP2, TOP, TMP2, lsl #2 + umlal2 v1.4s, v9.8h, v15.h[4] + ushr v15.8h, v12.8h, #(16 - BILINEAR_INTERPOLATION_BITS) + ushll v2.4s, v10.4h, #BILINEAR_INTERPOLATION_BITS + umlsl v2.4s, v10.4h, v15.h[0] + umlal2 v2.4s, v10.8h, v15.h[0] + ushll v3.4s, v11.4h, #BILINEAR_INTERPOLATION_BITS + ld1 {v20.2s}, [TMP1], STRIDE + umlsl v3.4s, v11.4h, v15.h[4] + umlal2 v3.4s, v11.8h, v15.h[4] + ld1 {v21.2s}, [TMP1] + umull v8.8h, v20.8b, v28.8b + umlal v8.8h, v21.8b, v29.8b + shrn v0.4h, v0.4s, #(2 * BILINEAR_INTERPOLATION_BITS) + shrn2 v0.8h, v1.4s, #(2 * BILINEAR_INTERPOLATION_BITS) + shrn v4.4h, v2.4s, #(2 * BILINEAR_INTERPOLATION_BITS) + ld1 {v22.2s}, [TMP2], STRIDE + shrn2 v4.8h, v3.4s, #(2 * BILINEAR_INTERPOLATION_BITS) + add v12.8h, v12.8h, v13.8h + ld1 {v23.2s}, [TMP2] + umull v9.8h, v22.8b, v28.8b + asr TMP3, X, #16 + add X, X, UX + add TMP3, TOP, TMP3, lsl #2 + asr TMP4, X, #16 + add X, X, UX + add TMP4, TOP, TMP4, lsl #2 + umlal v9.8h, v23.8b, v29.8b + ld1 {v22.2s}, [TMP3], STRIDE + ushr v15.8h, v12.8h, #(16 - BILINEAR_INTERPOLATION_BITS) + ld1 {v23.2s}, [TMP3] + umull v10.8h, v22.8b, v28.8b + umlal v10.8h, v23.8b, v29.8b + xtn v6.8b, v0.8h + ushll v0.4s, v8.4h, #BILINEAR_INTERPOLATION_BITS + xtn v7.8b, v4.8h + umlsl v0.4s, v8.4h, v15.h[0] + umlal2 v0.4s, v8.8h, v15.h[0] + prfm PREFETCH_MODE, [TMP4, PF_OFFS] + ld1 {v16.2s}, [TMP4], STRIDE + add v12.8h, v12.8h, v13.8h + ld1 {v17.2s}, [TMP4] + prfm PREFETCH_MODE, [TMP4, PF_OFFS] + umull v11.8h, v16.8b, v28.8b + umlal v11.8h, v17.8b, v29.8b + st1 {v6.2s, v7.2s}, [OUT], #16 + ushll v1.4s, v9.4h, #BILINEAR_INTERPOLATION_BITS + umlsl v1.4s, v9.4h, v15.h[4] +.endm + +/*****************************************************************************/ + +generate_bilinear_scanline_func \ + pixman_scaled_bilinear_scanline_8888_8888_SRC_asm_neon, 8888, 8888, \ + 2, 2, 28, BILINEAR_FLAG_UNROLL_4 + +generate_bilinear_scanline_func \ + pixman_scaled_bilinear_scanline_8888_0565_SRC_asm_neon, 8888, 0565, \ + 2, 1, 28, BILINEAR_FLAG_UNROLL_8 | BILINEAR_FLAG_USE_ALL_NEON_REGS + +generate_bilinear_scanline_func \ + pixman_scaled_bilinear_scanline_0565_x888_SRC_asm_neon, 0565, 8888, \ + 1, 2, 28, BILINEAR_FLAG_UNROLL_4 + +generate_bilinear_scanline_func \ + pixman_scaled_bilinear_scanline_0565_0565_SRC_asm_neon, 0565, 0565, \ + 1, 1, 28, BILINEAR_FLAG_UNROLL_4 diff --git a/pixman/pixman-arma64-neon-asm.h b/pixman/pixman-arma64-neon-asm.h new file mode 100644 index 0000000..0389d12 --- /dev/null +++ b/pixman/pixman-arma64-neon-asm.h @@ -0,0 +1,1310 @@ +/* + * Copyright © 2009 Nokia Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + * + * Author: Siarhei Siamashka (siarhei.siamashka@nokia.com) + */ + +/* + * This file contains a macro ('generate_composite_function') which can + * construct 2D image processing functions, based on a common template. + * Any combinations of source, destination and mask images with 8bpp, + * 16bpp, 24bpp, 32bpp color formats are supported. + * + * This macro takes care of: + * - handling of leading and trailing unaligned pixels + * - doing most of the work related to L2 cache preload + * - encourages the use of software pipelining for better instructions + * scheduling + * + * The user of this macro has to provide some configuration parameters + * (bit depths for the images, prefetch distance, etc.) and a set of + * macros, which should implement basic code chunks responsible for + * pixels processing. See 'pixman-armv8-neon-asm.S' file for the usage + * examples. + * + * TODO: + * - try overlapped pixel method (from Ian Rickards) when processing + * exactly two blocks of pixels + * - maybe add an option to do reverse scanline processing + */ + +/* + * Bit flags for 'generate_composite_function' macro which are used + * to tune generated functions behavior. + */ +.set FLAG_DST_WRITEONLY, 0 +.set FLAG_DST_READWRITE, 1 +.set FLAG_DEINTERLEAVE_32BPP, 2 + +/* + * Constants for selecting preferable prefetch type. + */ +.set PREFETCH_TYPE_NONE, 0 /* No prefetch at all */ +.set PREFETCH_TYPE_SIMPLE, 1 /* A simple, fixed-distance-ahead prefetch */ +.set PREFETCH_TYPE_ADVANCED, 2 /* Advanced fine-grained prefetch */ + +/* + * prefetch mode + * available modes are: + * pldl1keep + * pldl1strm + * pldl2keep + * pldl2strm + * pldl3keep + * pldl3strm + */ +#define PREFETCH_MODE pldl1keep + +/* + * Definitions of supplementary pixld/pixst macros (for partial load/store of + * pixel data). + */ + +.macro pixldst1 op, elem_size, reg1, mem_operand, abits + op {v®1&.&elem_size}, [&mem_operand&], #8 +.endm + +.macro pixldst2 op, elem_size, reg1, reg2, mem_operand, abits + op {v®1&.&elem_size, v®2&.&elem_size}, [&mem_operand&], #16 +.endm + +.macro pixldst4 op, elem_size, reg1, reg2, reg3, reg4, mem_operand, abits + op {v®1&.&elem_size, v®2&.&elem_size, v®3&.&elem_size, v®4&.&elem_size}, [&mem_operand&], #32 +.endm + +.macro pixldst0 op, elem_size, reg1, idx, mem_operand, abits, bytes + op {v®1&.&elem_size}[idx], [&mem_operand&], #&bytes& +.endm + +.macro pixldst3 op, elem_size, reg1, reg2, reg3, mem_operand + op {v®1&.&elem_size, v®2&.&elem_size, v®3&.&elem_size}, [&mem_operand&], #24 +.endm + +.macro pixldst30 op, elem_size, reg1, reg2, reg3, idx, mem_operand + op {v®1&.&elem_size, v®2&.&elem_size, v®3&.&elem_size}[idx], [&mem_operand&], #3 +.endm + +.macro pixldst numbytes, op, elem_size, basereg, mem_operand, abits +.if numbytes == 32 + .if elem_size==32 + pixldst4 op, 2s, %(basereg+4), %(basereg+5), \ + %(basereg+6), %(basereg+7), mem_operand, abits + .elseif elem_size==16 + pixldst4 op, 4h, %(basereg+4), %(basereg+5), \ + %(basereg+6), %(basereg+7), mem_operand, abits + .else + pixldst4 op, 8b, %(basereg+4), %(basereg+5), \ + %(basereg+6), %(basereg+7), mem_operand, abits + .endif +.elseif numbytes == 16 + .if elem_size==32 + pixldst2 op, 2s, %(basereg+2), %(basereg+3), mem_operand, abits + .elseif elem_size==16 + pixldst2 op, 4h, %(basereg+2), %(basereg+3), mem_operand, abits + .else + pixldst2 op, 8b, %(basereg+2), %(basereg+3), mem_operand, abits + .endif +.elseif numbytes == 8 + .if elem_size==32 + pixldst1 op, 2s, %(basereg+1), mem_operand, abits + .elseif elem_size==16 + pixldst1 op, 4h, %(basereg+1), mem_operand, abits + .else + pixldst1 op, 8b, %(basereg+1), mem_operand, abits + .endif +.elseif numbytes == 4 + .if !RESPECT_STRICT_ALIGNMENT || (elem_size == 32) + pixldst0 op, s, %(basereg+0), 1, mem_operand, abits, 4 + .elseif elem_size == 16 + pixldst0 op, h, %(basereg+0), 2, mem_operand, abits, 2 + pixldst0 op, h, %(basereg+0), 3, mem_operand, abits, 2 + .else + pixldst0 op, b, %(basereg+0), 4, mem_operand, abits, 1 + pixldst0 op, b, %(basereg+0), 5, mem_operand, abits, 1 + pixldst0 op, b, %(basereg+0), 6, mem_operand, abits, 1 + pixldst0 op, b, %(basereg+0), 7, mem_operand, abits, 1 + .endif +.elseif numbytes == 2 + .if !RESPECT_STRICT_ALIGNMENT || (elem_size == 16) + pixldst0 op, h, %(basereg+0), 1, mem_operand, abits, 2 + .else + pixldst0 op, b, %(basereg+0), 2, mem_operand, abits, 1 + pixldst0 op, b, %(basereg+0), 3, mem_operand, abits, 1 + .endif +.elseif numbytes == 1 + pixldst0 op, b, %(basereg+0), 1, mem_operand, abits, 1 +.else + .error "unsupported size: numbytes" +.endif +.endm + +.macro pixld numpix, bpp, basereg, mem_operand, abits=0 +.if bpp > 0 +.if (bpp == 32) && (numpix == 8) && (DEINTERLEAVE_32BPP_ENABLED != 0) + pixldst4 ld4, 8b, %(basereg+4), %(basereg+5), \ + %(basereg+6), %(basereg+7), mem_operand, abits +.elseif (bpp == 24) && (numpix == 8) + pixldst3 ld3, 8b, %(basereg+3), %(basereg+4), %(basereg+5), mem_operand +.elseif (bpp == 24) && (numpix == 4) + pixldst30 ld3, b, %(basereg+0), %(basereg+1), %(basereg+2), 4, mem_operand + pixldst30 ld3, b, %(basereg+0), %(basereg+1), %(basereg+2), 5, mem_operand + pixldst30 ld3, b, %(basereg+0), %(basereg+1), %(basereg+2), 6, mem_operand + pixldst30 ld3, b, %(basereg+0), %(basereg+1), %(basereg+2), 7, mem_operand +.elseif (bpp == 24) && (numpix == 2) + pixldst30 ld3, b, %(basereg+0), %(basereg+1), %(basereg+2), 2, mem_operand + pixldst30 ld3, b, %(basereg+0), %(basereg+1), %(basereg+2), 3, mem_operand +.elseif (bpp == 24) && (numpix == 1) + pixldst30 ld3, b, %(basereg+0), %(basereg+1), %(basereg+2), 1, mem_operand +.else + pixldst %(numpix * bpp / 8), ld1, %(bpp), basereg, mem_operand, abits +.endif +.endif +.endm + +.macro pixst numpix, bpp, basereg, mem_operand, abits=0 +.if bpp > 0 +.if (bpp == 32) && (numpix == 8) && (DEINTERLEAVE_32BPP_ENABLED != 0) + pixldst4 st4, 8b, %(basereg+4), %(basereg+5), \ + %(basereg+6), %(basereg+7), mem_operand, abits +.elseif (bpp == 24) && (numpix == 8) + pixldst3 st3, 8b, %(basereg+3), %(basereg+4), %(basereg+5), mem_operand +.elseif (bpp == 24) && (numpix == 4) + pixldst30 st3, b, %(basereg+0), %(basereg+1), %(basereg+2), 4, mem_operand + pixldst30 st3, b, %(basereg+0), %(basereg+1), %(basereg+2), 5, mem_operand + pixldst30 st3, b, %(basereg+0), %(basereg+1), %(basereg+2), 6, mem_operand + pixldst30 st3, b, %(basereg+0), %(basereg+1), %(basereg+2), 7, mem_operand +.elseif (bpp == 24) && (numpix == 2) + pixldst30 st3, b, %(basereg+0), %(basereg+1), %(basereg+2), 2, mem_operand + pixldst30 st3, b, %(basereg+0), %(basereg+1), %(basereg+2), 3, mem_operand +.elseif (bpp == 24) && (numpix == 1) + pixldst30 st3, b, %(basereg+0), %(basereg+1), %(basereg+2), 1, mem_operand +.elseif numpix * bpp == 32 && abits == 32 + pixldst 4, st1, 32, basereg, mem_operand, abits +.elseif numpix * bpp == 16 && abits == 16 + pixldst 2, st1, 16, basereg, mem_operand, abits +.else + pixldst %(numpix * bpp / 8), st1, %(bpp), basereg, mem_operand, abits +.endif +.endif +.endm + +.macro pixld_a numpix, bpp, basereg, mem_operand +.if (bpp * numpix) <= 128 + pixld numpix, bpp, basereg, mem_operand, %(bpp * numpix) +.else + pixld numpix, bpp, basereg, mem_operand, 128 +.endif +.endm + +.macro pixst_a numpix, bpp, basereg, mem_operand +.if (bpp * numpix) <= 128 + pixst numpix, bpp, basereg, mem_operand, %(bpp * numpix) +.else + pixst numpix, bpp, basereg, mem_operand, 128 +.endif +.endm + +/* + * Pixel fetcher for nearest scaling (needs TMP1, TMP2, VX, UNIT_X register + * aliases to be defined) + */ +.macro pixld1_s elem_size, reg1, mem_operand +.if elem_size == 16 + asr TMP1, VX, #16 + adds VX, VX, UNIT_X + bmi 55f +5: subs VX, VX, SRC_WIDTH_FIXED + bpl 5b +55: + add TMP1, mem_operand, TMP1, lsl #1 + asr TMP2, VX, #16 + adds VX, VX, UNIT_X + bmi 55f +5: subs VX, VX, SRC_WIDTH_FIXED + bpl 5b +55: + add TMP2, mem_operand, TMP2, lsl #1 + ld1 {v®1&.h}[0], [TMP1] + asr TMP1, VX, #16 + adds VX, VX, UNIT_X + bmi 55f +5: subs VX, VX, SRC_WIDTH_FIXED + bpl 5b +55: + add TMP1, mem_operand, TMP1, lsl #1 + ld1 {v®1&.h}[1], [TMP2] + asr TMP2, VX, #16 + adds VX, VX, UNIT_X + bmi 55f +5: subs VX, VX, SRC_WIDTH_FIXED + bpl 5b +55: + add TMP2, mem_operand, TMP2, lsl #1 + ld1 {v®1&.h}[2], [TMP1] + ld1 {v®1&.h}[3], [TMP2] +.elseif elem_size == 32 + asr TMP1, VX, #16 + adds VX, VX, UNIT_X + bmi 55f +5: subs VX, VX, SRC_WIDTH_FIXED + bpl 5b +55: + add TMP1, mem_operand, TMP1, lsl #2 + asr TMP2, VX, #16 + adds VX, VX, UNIT_X + bmi 55f +5: subs VX, VX, SRC_WIDTH_FIXED + bpl 5b +55: + add TMP2, mem_operand, TMP2, lsl #2 + ld1 {v®1&.s}[0], [TMP1] + ld1 {v®1&.s}[1], [TMP2] +.else + .error "unsupported" +.endif +.endm + +.macro pixld2_s elem_size, reg1, reg2, mem_operand +.if 0 /* elem_size == 32 */ + mov TMP1, VX, asr #16 + add VX, VX, UNIT_X, asl #1 + add TMP1, mem_operand, TMP1, asl #2 + mov TMP2, VX, asr #16 + sub VX, VX, UNIT_X + add TMP2, mem_operand, TMP2, asl #2 + ld1 {v®1&.s}[0], [TMP1] + mov TMP1, VX, asr #16 + add VX, VX, UNIT_X, asl #1 + add TMP1, mem_operand, TMP1, asl #2 + ld1 {v®2&.s}[0], [TMP2, :32] + mov TMP2, VX, asr #16 + add VX, VX, UNIT_X + add TMP2, mem_operand, TMP2, asl #2 + ld1 {v®1&.s}[1], [TMP1] + ld1 {v®2&.s}[1], [TMP2] +.else + pixld1_s elem_size, reg1, mem_operand + pixld1_s elem_size, reg2, mem_operand +.endif +.endm + +.macro pixld0_s elem_size, reg1, idx, mem_operand +.if elem_size == 16 + asr TMP1, VX, #16 + adds VX, VX, UNIT_X + bmi 55f +5: subs VX, VX, SRC_WIDTH_FIXED + bpl 5b +55: + add TMP1, mem_operand, TMP1, lsl #1 + ld1 {v®1&.h}[idx], [TMP1] +.elseif elem_size == 32 + asr DUMMY, VX, #16 + mov TMP1, DUMMY + adds VX, VX, UNIT_X + bmi 55f +5: subs VX, VX, SRC_WIDTH_FIXED + bpl 5b +55: + add TMP1, mem_operand, TMP1, lsl #2 + ld1 {v®1&.s}[idx], [TMP1] +.endif +.endm + +.macro pixld_s_internal numbytes, elem_size, basereg, mem_operand +.if numbytes == 32 + pixld2_s elem_size, %(basereg+4), %(basereg+5), mem_operand + pixld2_s elem_size, %(basereg+6), %(basereg+7), mem_operand + pixdeinterleave elem_size, %(basereg+4) +.elseif numbytes == 16 + pixld2_s elem_size, %(basereg+2), %(basereg+3), mem_operand +.elseif numbytes == 8 + pixld1_s elem_size, %(basereg+1), mem_operand +.elseif numbytes == 4 + .if elem_size == 32 + pixld0_s elem_size, %(basereg+0), 1, mem_operand + .elseif elem_size == 16 + pixld0_s elem_size, %(basereg+0), 2, mem_operand + pixld0_s elem_size, %(basereg+0), 3, mem_operand + .else + pixld0_s elem_size, %(basereg+0), 4, mem_operand + pixld0_s elem_size, %(basereg+0), 5, mem_operand + pixld0_s elem_size, %(basereg+0), 6, mem_operand + pixld0_s elem_size, %(basereg+0), 7, mem_operand + .endif +.elseif numbytes == 2 + .if elem_size == 16 + pixld0_s elem_size, %(basereg+0), 1, mem_operand + .else + pixld0_s elem_size, %(basereg+0), 2, mem_operand + pixld0_s elem_size, %(basereg+0), 3, mem_operand + .endif +.elseif numbytes == 1 + pixld0_s elem_size, %(basereg+0), 1, mem_operand +.else + .error "unsupported size: numbytes" +.endif +.endm + +.macro pixld_s numpix, bpp, basereg, mem_operand +.if bpp > 0 + pixld_s_internal %(numpix * bpp / 8), %(bpp), basereg, mem_operand +.endif +.endm + +.macro vuzp8 reg1, reg2 + umov DUMMY, v16.d[0] + uzp1 v16.8b, v®1&.8b, v®2&.8b + uzp2 v®2&.8b, v®1&.8b, v®2&.8b + mov v®1&.8b, v16.8b + mov v16.d[0], DUMMY +.endm + +.macro vzip8 reg1, reg2 + umov DUMMY, v16.d[0] + zip1 v16.8b, v®1&.8b, v®2&.8b + zip2 v®2&.8b, v®1&.8b, v®2&.8b + mov v®1&.8b, v16.8b + mov v16.d[0], DUMMY +.endm + +/* deinterleave B, G, R, A channels for eight 32bpp pixels in 4 registers */ +.macro pixdeinterleave bpp, basereg +.if (bpp == 32) && (DEINTERLEAVE_32BPP_ENABLED != 0) + vuzp8 %(basereg+0), %(basereg+1) + vuzp8 %(basereg+2), %(basereg+3) + vuzp8 %(basereg+1), %(basereg+3) + vuzp8 %(basereg+0), %(basereg+2) +.endif +.endm + +/* interleave B, G, R, A channels for eight 32bpp pixels in 4 registers */ +.macro pixinterleave bpp, basereg +.if (bpp == 32) && (DEINTERLEAVE_32BPP_ENABLED != 0) + vzip8 %(basereg+0), %(basereg+2) + vzip8 %(basereg+1), %(basereg+3) + vzip8 %(basereg+2), %(basereg+3) + vzip8 %(basereg+0), %(basereg+1) +.endif +.endm + +/* + * This is a macro for implementing cache preload. The main idea is that + * cache preload logic is mostly independent from the rest of pixels + * processing code. It starts at the top left pixel and moves forward + * across pixels and can jump across scanlines. Prefetch distance is + * handled in an 'incremental' way: it starts from 0 and advances to the + * optimal distance over time. After reaching optimal prefetch distance, + * it is kept constant. There are some checks which prevent prefetching + * unneeded pixel lines below the image (but it still can prefetch a bit + * more data on the right side of the image - not a big issue and may + * be actually helpful when rendering text glyphs). Additional trick is + * the use of LDR instruction for prefetch instead of PLD when moving to + * the next line, the point is that we have a high chance of getting TLB + * miss in this case, and PLD would be useless. + * + * This sounds like it may introduce a noticeable overhead (when working with + * fully cached data). But in reality, due to having a separate pipeline and + * instruction queue for NEON unit in ARM Cortex-A8, normal ARM code can + * execute simultaneously with NEON and be completely shadowed by it. Thus + * we get no performance overhead at all (*). This looks like a very nice + * feature of Cortex-A8, if used wisely. We don't have a hardware prefetcher, + * but still can implement some rather advanced prefetch logic in software + * for almost zero cost! + * + * (*) The overhead of the prefetcher is visible when running some trivial + * pixels processing like simple copy. Anyway, having prefetch is a must + * when working with the graphics data. + */ +.macro PF a, x:vararg +.if (PREFETCH_TYPE_CURRENT == PREFETCH_TYPE_ADVANCED) + a x +.endif +.endm + +.macro cache_preload std_increment, boost_increment +.if (src_bpp_shift >= 0) || (dst_r_bpp != 0) || (mask_bpp_shift >= 0) +.if std_increment != 0 + PF add PF_X, PF_X, #std_increment +.endif + PF tst PF_CTL, #0xF + PF beq 71f + PF add PF_X, PF_X, #boost_increment + PF sub PF_CTL, PF_CTL, #1 +71: + PF cmp PF_X, ORIG_W +.if src_bpp_shift >= 0 + PF lsl DUMMY, PF_X, #src_bpp_shift + PF prfm PREFETCH_MODE, [PF_SRC, DUMMY] +.endif +.if dst_r_bpp != 0 + PF lsl DUMMY, PF_X, #dst_bpp_shift + PF prfm PREFETCH_MODE, [PF_DST, DUMMY] +.endif +.if mask_bpp_shift >= 0 + PF lsl DUMMY, PF_X, #mask_bpp_shift + PF prfm PREFETCH_MODE, [PF_MASK, DUMMY] +.endif + PF ble 71f + PF sub PF_X, PF_X, ORIG_W + PF subs PF_CTL, PF_CTL, #0x10 +71: + PF ble 72f +.if src_bpp_shift >= 0 + PF lsl DUMMY, SRC_STRIDE, #src_bpp_shift + PF ldrsb DUMMY, [PF_SRC, DUMMY] + PF add PF_SRC, PF_SRC, #1 +.endif +.if dst_r_bpp != 0 + PF lsl DUMMY, DST_STRIDE, #dst_bpp_shift + PF ldrsb DUMMY, [PF_DST, DUMMY] + PF add PF_DST, PF_DST, #1 +.endif +.if mask_bpp_shift >= 0 + PF lsl DUMMY, MASK_STRIDE, #mask_bpp_shift + PF ldrsb DUMMY, [PF_MASK, DUMMY] + PF add PF_MASK, PF_MASK, #1 +.endif +72: +.endif +.endm + +.macro cache_preload_simple +.if (PREFETCH_TYPE_CURRENT == PREFETCH_TYPE_SIMPLE) +.if src_bpp > 0 + prfm PREFETCH_MODE, [SRC, #(PREFETCH_DISTANCE_SIMPLE * src_bpp / 8)] +.endif +.if dst_r_bpp > 0 + prfm PREFETCH_MODE, [DST_R, #(PREFETCH_DISTANCE_SIMPLE * dst_r_bpp / 8)] +.endif +.if mask_bpp > 0 + prfm PREFETCH_MODE, [MASK, #(PREFETCH_DISTANCE_SIMPLE * mask_bpp / 8)] +.endif +.endif +.endm + +.macro fetch_mask_pixblock + pixld pixblock_size, mask_bpp, \ + (mask_basereg - pixblock_size * mask_bpp / 64), MASK +.endm + +/* + * Macro which is used to process leading pixels until destination + * pointer is properly aligned (at 16 bytes boundary). When destination + * buffer uses 16bpp format, this is unnecessary, or even pointless. + */ +.macro ensure_destination_ptr_alignment process_pixblock_head, \ + process_pixblock_tail, \ + process_pixblock_tail_head +.if dst_w_bpp != 24 + tst DST_R, #0xF + beq 52f + +.if src_bpp > 0 || mask_bpp > 0 || dst_r_bpp > 0 +.irp lowbit, 1, 2, 4, 8, 16 +local skip1 +.if (dst_w_bpp <= (lowbit * 8)) && ((lowbit * 8) < (pixblock_size * dst_w_bpp)) +.if lowbit < 16 /* we don't need more than 16-byte alignment */ + tst DST_R, #lowbit + beq 51f +.endif + pixld_src (lowbit * 8 / dst_w_bpp), src_bpp, src_basereg, SRC + pixld (lowbit * 8 / dst_w_bpp), mask_bpp, mask_basereg, MASK +.if dst_r_bpp > 0 + pixld_a (lowbit * 8 / dst_r_bpp), dst_r_bpp, dst_r_basereg, DST_R +.else + add DST_R, DST_R, #lowbit +.endif + PF add PF_X, PF_X, #(lowbit * 8 / dst_w_bpp) + sub W, W, #(lowbit * 8 / dst_w_bpp) +51: +.endif +.endr +.endif + pixdeinterleave src_bpp, src_basereg + pixdeinterleave mask_bpp, mask_basereg + pixdeinterleave dst_r_bpp, dst_r_basereg + + process_pixblock_head + cache_preload 0, pixblock_size + cache_preload_simple + process_pixblock_tail + + pixinterleave dst_w_bpp, dst_w_basereg + +.irp lowbit, 1, 2, 4, 8, 16 +.if (dst_w_bpp <= (lowbit * 8)) && ((lowbit * 8) < (pixblock_size * dst_w_bpp)) +.if lowbit < 16 /* we don't need more than 16-byte alignment */ + tst DST_W, #lowbit + beq 51f +.endif +.if src_bpp == 0 && mask_bpp == 0 && dst_r_bpp == 0 + sub W, W, #(lowbit * 8 / dst_w_bpp) +.endif + pixst_a (lowbit * 8 / dst_w_bpp), dst_w_bpp, dst_w_basereg, DST_W +51: +.endif +.endr +.endif +52: +.endm + +/* + * Special code for processing up to (pixblock_size - 1) remaining + * trailing pixels. As SIMD processing performs operation on + * pixblock_size pixels, anything smaller than this has to be loaded + * and stored in a special way. Loading and storing of pixel data is + * performed in such a way that we fill some 'slots' in the NEON + * registers (some slots naturally are unused), then perform compositing + * operation as usual. In the end, the data is taken from these 'slots' + * and saved to memory. + * + * cache_preload_flag - allows to suppress prefetch if + * set to 0 + * dst_aligned_flag - selects whether destination buffer + * is aligned + */ +.macro process_trailing_pixels cache_preload_flag, \ + dst_aligned_flag, \ + process_pixblock_head, \ + process_pixblock_tail, \ + process_pixblock_tail_head + tst W, #(pixblock_size - 1) + beq 52f +.if src_bpp > 0 || mask_bpp > 0 || dst_r_bpp > 0 +.irp chunk_size, 16, 8, 4, 2, 1 +.if pixblock_size > chunk_size + tst W, #chunk_size + beq 51f + pixld_src chunk_size, src_bpp, src_basereg, SRC + pixld chunk_size, mask_bpp, mask_basereg, MASK +.if dst_aligned_flag != 0 + pixld_a chunk_size, dst_r_bpp, dst_r_basereg, DST_R +.else + pixld chunk_size, dst_r_bpp, dst_r_basereg, DST_R +.endif +.if cache_preload_flag != 0 + PF add PF_X, PF_X, #chunk_size +.endif +51: +.endif +.endr +.endif + pixdeinterleave src_bpp, src_basereg + pixdeinterleave mask_bpp, mask_basereg + pixdeinterleave dst_r_bpp, dst_r_basereg + + process_pixblock_head +.if cache_preload_flag != 0 + cache_preload 0, pixblock_size + cache_preload_simple +.endif + process_pixblock_tail + pixinterleave dst_w_bpp, dst_w_basereg +.irp chunk_size, 16, 8, 4, 2, 1 +.if pixblock_size > chunk_size + tst W, #chunk_size + beq 51f +.if dst_aligned_flag != 0 + pixst_a chunk_size, dst_w_bpp, dst_w_basereg, DST_W +.else + pixst chunk_size, dst_w_bpp, dst_w_basereg, DST_W +.endif +51: +.endif +.endr +52: +.endm + +/* + * Macro, which performs all the needed operations to switch to the next + * scanline and start the next loop iteration unless all the scanlines + * are already processed. + */ +.macro advance_to_next_scanline start_of_loop_label + mov W, ORIG_W + add DST_W, DST_W, DST_STRIDE, lsl #dst_bpp_shift +.if src_bpp != 0 + add SRC, SRC, SRC_STRIDE, lsl #src_bpp_shift +.endif +.if mask_bpp != 0 + add MASK, MASK, MASK_STRIDE, lsl #mask_bpp_shift +.endif +.if (dst_w_bpp != 24) + sub DST_W, DST_W, W, lsl #dst_bpp_shift +.endif +.if (src_bpp != 24) && (src_bpp != 0) + sub SRC, SRC, W, lsl #src_bpp_shift +.endif +.if (mask_bpp != 24) && (mask_bpp != 0) + sub MASK, MASK, W, lsl #mask_bpp_shift +.endif + subs H, H, #1 + mov DST_R, DST_W + bge start_of_loop_label +.endm + +/* + * Registers are allocated in the following way by default: + * v0, v1, v2, v3 - reserved for loading source pixel data + * v4, v5, v6, v7 - reserved for loading destination pixel data + * v24, v25, v26, v27 - reserved for loading mask pixel data + * v28, v29, v30, v31 - final destination pixel data for writeback to memory + */ +.macro generate_composite_function fname, \ + src_bpp_, \ + mask_bpp_, \ + dst_w_bpp_, \ + flags, \ + pixblock_size_, \ + prefetch_distance, \ + init, \ + cleanup, \ + process_pixblock_head, \ + process_pixblock_tail, \ + process_pixblock_tail_head, \ + dst_w_basereg_ = 28, \ + dst_r_basereg_ = 4, \ + src_basereg_ = 0, \ + mask_basereg_ = 24 + + pixman_asm_function fname + stp x29, x30, [sp, -16]! + mov x29, sp + sub sp, sp, 232 /* push all registers */ + sub x29, x29, 64 + st1 {v8.8b, v9.8b, v10.8b, v11.8b}, [x29], #32 + st1 {v12.8b, v13.8b, v14.8b, v15.8b}, [x29], #32 + stp x8, x9, [x29, -80] + stp x10, x11, [x29, -96] + stp x12, x13, [x29, -112] + stp x14, x15, [x29, -128] + stp x16, x17, [x29, -144] + stp x18, x19, [x29, -160] + stp x20, x21, [x29, -176] + stp x22, x23, [x29, -192] + stp x24, x25, [x29, -208] + stp x26, x27, [x29, -224] + str x28, [x29, -232] + +/* + * Select prefetch type for this function. If prefetch distance is + * set to 0 or one of the color formats is 24bpp, SIMPLE prefetch + * has to be used instead of ADVANCED. + */ + .set PREFETCH_TYPE_CURRENT, PREFETCH_TYPE_DEFAULT +.if prefetch_distance == 0 + .set PREFETCH_TYPE_CURRENT, PREFETCH_TYPE_NONE +.elseif (PREFETCH_TYPE_CURRENT > PREFETCH_TYPE_SIMPLE) && \ + ((src_bpp_ == 24) || (mask_bpp_ == 24) || (dst_w_bpp_ == 24)) + .set PREFETCH_TYPE_CURRENT, PREFETCH_TYPE_SIMPLE +.endif + +/* + * Make some macro arguments globally visible and accessible + * from other macros + */ + .set src_bpp, src_bpp_ + .set mask_bpp, mask_bpp_ + .set dst_w_bpp, dst_w_bpp_ + .set pixblock_size, pixblock_size_ + .set dst_w_basereg, dst_w_basereg_ + .set dst_r_basereg, dst_r_basereg_ + .set src_basereg, src_basereg_ + .set mask_basereg, mask_basereg_ + + .macro pixld_src x:vararg + pixld x + .endm + .macro fetch_src_pixblock + pixld_src pixblock_size, src_bpp, \ + (src_basereg - pixblock_size * src_bpp / 64), SRC + .endm +/* + * Assign symbolic names to registers + */ + W .req x0 /* width (is updated during processing) */ + H .req x1 /* height (is updated during processing) */ + DST_W .req x2 /* destination buffer pointer for writes */ + DST_STRIDE .req x3 /* destination image stride */ + SRC .req x4 /* source buffer pointer */ + SRC_STRIDE .req x5 /* source image stride */ + MASK .req x6 /* mask pointer */ + MASK_STRIDE .req x7 /* mask stride */ + + DST_R .req x8 /* destination buffer pointer for reads */ + + PF_CTL .req x9 /* combined lines counter and prefetch */ + /* distance increment counter */ + PF_X .req x10 /* pixel index in a scanline for current */ + /* pretetch position */ + PF_SRC .req x11 /* pointer to source scanline start */ + /* for prefetch purposes */ + PF_DST .req x12 /* pointer to destination scanline start */ + /* for prefetch purposes */ + PF_MASK .req x13 /* pointer to mask scanline start */ + /* for prefetch purposes */ + + ORIG_W .req x14 /* saved original width */ + DUMMY .req x15 /* temporary register */ + + sxtw x0, w0 + sxtw x1, w1 + sxtw x3, w3 + sxtw x5, w5 + sxtw x7, w7 + + .set mask_bpp_shift, -1 +.if src_bpp == 32 + .set src_bpp_shift, 2 +.elseif src_bpp == 24 + .set src_bpp_shift, 0 +.elseif src_bpp == 16 + .set src_bpp_shift, 1 +.elseif src_bpp == 8 + .set src_bpp_shift, 0 +.elseif src_bpp == 0 + .set src_bpp_shift, -1 +.else + .error "requested src bpp (src_bpp) is not supported" +.endif +.if mask_bpp == 32 + .set mask_bpp_shift, 2 +.elseif mask_bpp == 24 + .set mask_bpp_shift, 0 +.elseif mask_bpp == 8 + .set mask_bpp_shift, 0 +.elseif mask_bpp == 0 + .set mask_bpp_shift, -1 +.else + .error "requested mask bpp (mask_bpp) is not supported" +.endif +.if dst_w_bpp == 32 + .set dst_bpp_shift, 2 +.elseif dst_w_bpp == 24 + .set dst_bpp_shift, 0 +.elseif dst_w_bpp == 16 + .set dst_bpp_shift, 1 +.elseif dst_w_bpp == 8 + .set dst_bpp_shift, 0 +.else + .error "requested dst bpp (dst_w_bpp) is not supported" +.endif + +.if (((flags) & FLAG_DST_READWRITE) != 0) + .set dst_r_bpp, dst_w_bpp +.else + .set dst_r_bpp, 0 +.endif +.if (((flags) & FLAG_DEINTERLEAVE_32BPP) != 0) + .set DEINTERLEAVE_32BPP_ENABLED, 1 +.else + .set DEINTERLEAVE_32BPP_ENABLED, 0 +.endif + +.if prefetch_distance < 0 || prefetch_distance > 15 + .error "invalid prefetch distance (prefetch_distance)" +.endif + + PF mov PF_X, #0 + mov DST_R, DST_W + +.if src_bpp == 24 + sub SRC_STRIDE, SRC_STRIDE, W + sub SRC_STRIDE, SRC_STRIDE, W, lsl #1 +.endif +.if mask_bpp == 24 + sub MASK_STRIDE, MASK_STRIDE, W + sub MASK_STRIDE, MASK_STRIDE, W, lsl #1 +.endif +.if dst_w_bpp == 24 + sub DST_STRIDE, DST_STRIDE, W + sub DST_STRIDE, DST_STRIDE, W, lsl #1 +.endif + +/* + * Setup advanced prefetcher initial state + */ + PF mov PF_SRC, SRC + PF mov PF_DST, DST_R + PF mov PF_MASK, MASK + /* PF_CTL = prefetch_distance | ((h - 1) << 4) */ + PF lsl DUMMY, H, #4 + PF mov PF_CTL, DUMMY + PF add PF_CTL, PF_CTL, #(prefetch_distance - 0x10) + + init + subs H, H, #1 + mov ORIG_W, W + blt 9f + cmp W, #(pixblock_size * 2) + blt 800f +/* + * This is the start of the pipelined loop, which if optimized for + * long scanlines + */ +0: + ensure_destination_ptr_alignment process_pixblock_head, \ + process_pixblock_tail, \ + process_pixblock_tail_head + + /* Implement "head (tail_head) ... (tail_head) tail" loop pattern */ + pixld_a pixblock_size, dst_r_bpp, \ + (dst_r_basereg - pixblock_size * dst_r_bpp / 64), DST_R + fetch_src_pixblock + pixld pixblock_size, mask_bpp, \ + (mask_basereg - pixblock_size * mask_bpp / 64), MASK + PF add PF_X, PF_X, #pixblock_size + process_pixblock_head + cache_preload 0, pixblock_size + cache_preload_simple + subs W, W, #(pixblock_size * 2) + blt 200f + +100: + process_pixblock_tail_head + cache_preload_simple + subs W, W, #pixblock_size + bge 100b + +200: + process_pixblock_tail + pixst_a pixblock_size, dst_w_bpp, \ + (dst_w_basereg - pixblock_size * dst_w_bpp / 64), DST_W + + /* Process the remaining trailing pixels in the scanline */ + process_trailing_pixels 1, 1, \ + process_pixblock_head, \ + process_pixblock_tail, \ + process_pixblock_tail_head + advance_to_next_scanline 0b + + cleanup +1000: + /* pop all registers */ + sub x29, x29, 64 + ld1 {v8.8b, v9.8b, v10.8b, v11.8b}, [x29], 32 + ld1 {v12.8b, v13.8b, v14.8b, v15.8b}, [x29], 32 + ldp x8, x9, [x29, -80] + ldp x10, x11, [x29, -96] + ldp x12, x13, [x29, -112] + ldp x14, x15, [x29, -128] + ldp x16, x17, [x29, -144] + ldp x18, x19, [x29, -160] + ldp x20, x21, [x29, -176] + ldp x22, x23, [x29, -192] + ldp x24, x25, [x29, -208] + ldp x26, x27, [x29, -224] + ldr x28, [x29, -232] + mov sp, x29 + ldp x29, x30, [sp], 16 + ret /* exit */ +/* + * This is the start of the loop, designed to process images with small width + * (less than pixblock_size * 2 pixels). In this case neither pipelining + * nor prefetch are used. + */ +800: +.if src_bpp_shift >= 0 + PF lsl DUMMY, SRC_STRIDE, #src_bpp_shift + PF prfm PREFETCH_MODE, [SRC, DUMMY] +.endif +.if dst_r_bpp != 0 + PF lsl DUMMY, DST_STRIDE, #dst_bpp_shift + PF prfm PREFETCH_MODE, [DST_R, DUMMY] +.endif +.if mask_bpp_shift >= 0 + PF lsl DUMMY, MASK_STRIDE, #mask_bpp_shift + PF prfm PREFETCH_MODE, [MASK, DUMMY] +.endif + /* Process exactly pixblock_size pixels if needed */ + tst W, #pixblock_size + beq 100f + pixld pixblock_size, dst_r_bpp, \ + (dst_r_basereg - pixblock_size * dst_r_bpp / 64), DST_R + fetch_src_pixblock + pixld pixblock_size, mask_bpp, \ + (mask_basereg - pixblock_size * mask_bpp / 64), MASK + process_pixblock_head + process_pixblock_tail + pixst pixblock_size, dst_w_bpp, \ + (dst_w_basereg - pixblock_size * dst_w_bpp / 64), DST_W +100: + /* Process the remaining trailing pixels in the scanline */ + process_trailing_pixels 0, 0, \ + process_pixblock_head, \ + process_pixblock_tail, \ + process_pixblock_tail_head + advance_to_next_scanline 800b +9: + cleanup + /* pop all registers */ + sub x29, x29, 64 + ld1 {v8.8b, v9.8b, v10.8b, v11.8b}, [x29], 32 + ld1 {v12.8b, v13.8b, v14.8b, v15.8b}, [x29], 32 + ldp x8, x9, [x29, -80] + ldp x10, x11, [x29, -96] + ldp x12, x13, [x29, -112] + ldp x14, x15, [x29, -128] + ldp x16, x17, [x29, -144] + ldp x18, x19, [x29, -160] + ldp x20, x21, [x29, -176] + ldp x22, x23, [x29, -192] + ldp x24, x25, [x29, -208] + ldp x26, x27, [x29, -224] + ldr x28, [x29, -232] + mov sp, x29 + ldp x29, x30, [sp], 16 + ret /* exit */ + + .purgem fetch_src_pixblock + .purgem pixld_src + + .unreq SRC + .unreq MASK + .unreq DST_R + .unreq DST_W + .unreq ORIG_W + .unreq W + .unreq H + .unreq SRC_STRIDE + .unreq DST_STRIDE + .unreq MASK_STRIDE + .unreq PF_CTL + .unreq PF_X + .unreq PF_SRC + .unreq PF_DST + .unreq PF_MASK + .unreq DUMMY + .endfunc +.endm + +/* + * A simplified variant of function generation template for a single + * scanline processing (for implementing pixman combine functions) + */ +.macro generate_composite_function_scanline use_nearest_scaling, \ + fname, \ + src_bpp_, \ + mask_bpp_, \ + dst_w_bpp_, \ + flags, \ + pixblock_size_, \ + init, \ + cleanup, \ + process_pixblock_head, \ + process_pixblock_tail, \ + process_pixblock_tail_head, \ + dst_w_basereg_ = 28, \ + dst_r_basereg_ = 4, \ + src_basereg_ = 0, \ + mask_basereg_ = 24 + + pixman_asm_function fname + .set PREFETCH_TYPE_CURRENT, PREFETCH_TYPE_NONE + +/* + * Make some macro arguments globally visible and accessible + * from other macros + */ + .set src_bpp, src_bpp_ + .set mask_bpp, mask_bpp_ + .set dst_w_bpp, dst_w_bpp_ + .set pixblock_size, pixblock_size_ + .set dst_w_basereg, dst_w_basereg_ + .set dst_r_basereg, dst_r_basereg_ + .set src_basereg, src_basereg_ + .set mask_basereg, mask_basereg_ + +.if use_nearest_scaling != 0 + /* + * Assign symbolic names to registers for nearest scaling + */ + W .req x0 + DST_W .req x1 + SRC .req x2 + VX .req x3 + UNIT_X .req x4 + SRC_WIDTH_FIXED .req x5 + MASK .req x6 + TMP1 .req x8 + TMP2 .req x9 + DST_R .req x10 + DUMMY .req x30 + + .macro pixld_src x:vararg + pixld_s x + .endm + + sxtw x0, w0 + sxtw x3, w3 + sxtw x4, w4 + sxtw x5, w5 + + stp x29, x30, [sp, -16]! + mov x29, sp + sub sp, sp, 88 + sub x29, x29, 64 + st1 {v8.8b, v9.8b, v10.8b, v11.8b}, [x29], 32 + st1 {v12.8b, v13.8b, v14.8b, v15.8b}, [x29], 32 + stp x8, x9, [x29, -80] + str x10, [x29, -88] +.else + /* + * Assign symbolic names to registers + */ + W .req x0 /* width (is updated during processing) */ + DST_W .req x1 /* destination buffer pointer for writes */ + SRC .req x2 /* source buffer pointer */ + MASK .req x3 /* mask pointer */ + DST_R .req x4 /* destination buffer pointer for reads */ + DUMMY .req x30 + + .macro pixld_src x:vararg + pixld x + .endm + + sxtw x0, w0 + + stp x29, x30, [sp, -16]! + mov x29, sp + sub sp, sp, 64 + sub x29, x29, 64 + st1 {v8.8b, v9.8b, v10.8b, v11.8b}, [x29], 32 + st1 {v12.8b, v13.8b, v14.8b, v15.8b}, [x29], 32 +.endif + +.if (((flags) & FLAG_DST_READWRITE) != 0) + .set dst_r_bpp, dst_w_bpp +.else + .set dst_r_bpp, 0 +.endif +.if (((flags) & FLAG_DEINTERLEAVE_32BPP) != 0) + .set DEINTERLEAVE_32BPP_ENABLED, 1 +.else + .set DEINTERLEAVE_32BPP_ENABLED, 0 +.endif + + .macro fetch_src_pixblock + pixld_src pixblock_size, src_bpp, \ + (src_basereg - pixblock_size * src_bpp / 64), SRC + .endm + + init + mov DST_R, DST_W + + cmp W, #pixblock_size + blt 800f + + ensure_destination_ptr_alignment process_pixblock_head, \ + process_pixblock_tail, \ + process_pixblock_tail_head + + subs W, W, #pixblock_size + blt 700f + + /* Implement "head (tail_head) ... (tail_head) tail" loop pattern */ + pixld_a pixblock_size, dst_r_bpp, \ + (dst_r_basereg - pixblock_size * dst_r_bpp / 64), DST_R + fetch_src_pixblock + pixld pixblock_size, mask_bpp, \ + (mask_basereg - pixblock_size * mask_bpp / 64), MASK + process_pixblock_head + subs W, W, #pixblock_size + blt 200f +100: + process_pixblock_tail_head + subs W, W, #pixblock_size + bge 100b +200: + process_pixblock_tail + pixst_a pixblock_size, dst_w_bpp, \ + (dst_w_basereg - pixblock_size * dst_w_bpp / 64), DST_W +700: + /* Process the remaining trailing pixels in the scanline (dst aligned) */ + process_trailing_pixels 0, 1, \ + process_pixblock_head, \ + process_pixblock_tail, \ + process_pixblock_tail_head + + cleanup +.if use_nearest_scaling != 0 + sub x29, x29, 64 + ld1 {v8.8b, v9.8b, v10.8b, v11.8b}, [x29], 32 + ld1 {v12.8b, v13.8b, v14.8b, v15.8b}, [x29], 32 + ldp x8, x9, [x29, -80] + ldr x10, [x29, -96] + mov sp, x29 + ldp x29, x30, [sp], 16 + ret /* exit */ +.else + sub x29, x29, 64 + ld1 {v8.8b, v9.8b, v10.8b, v11.8b}, [x29], 32 + ld1 {v12.8b, v13.8b, v14.8b, v15.8b}, [x29], 32 + mov sp, x29 + ldp x29, x30, [sp], 16 + ret /* exit */ +.endif +800: + /* Process the remaining trailing pixels in the scanline (dst unaligned) */ + process_trailing_pixels 0, 0, \ + process_pixblock_head, \ + process_pixblock_tail, \ + process_pixblock_tail_head + + cleanup +.if use_nearest_scaling != 0 + sub x29, x29, 64 + ld1 {v8.8b, v9.8b, v10.8b, v11.8b}, [x29], 32 + ld1 {v12.8b, v13.8b, v14.8b, v15.8b}, [x29], 32 + ldp x8, x9, [x29, -80] + ldr x10, [x29, -88] + mov sp, x29 + ldp x29, x30, [sp], 16 + ret /* exit */ + + .unreq DUMMY + .unreq DST_R + .unreq SRC + .unreq W + .unreq VX + .unreq UNIT_X + .unreq TMP1 + .unreq TMP2 + .unreq DST_W + .unreq MASK + .unreq SRC_WIDTH_FIXED + +.else + sub x29, x29, 64 + ld1 {v8.8b, v9.8b, v10.8b, v11.8b}, [x29], 32 + ld1 {v12.8b, v13.8b, v14.8b, v15.8b}, [x29], 32 + mov sp, x29 + ldp x29, x30, [sp], 16 + ret /* exit */ + + .unreq DUMMY + .unreq SRC + .unreq MASK + .unreq DST_R + .unreq DST_W + .unreq W +.endif + + .purgem fetch_src_pixblock + .purgem pixld_src + + .endfunc +.endm + +.macro generate_composite_function_single_scanline x:vararg + generate_composite_function_scanline 0, x +.endm + +.macro generate_composite_function_nearest_scanline x:vararg + generate_composite_function_scanline 1, x +.endm + +/* Default prologue/epilogue, nothing special needs to be done */ + +.macro default_init +.endm + +.macro default_cleanup +.endm + +/* + * Prologue/epilogue variant which additionally saves/restores v8-v15 + * registers (they need to be saved/restored by callee according to ABI). + * This is required if the code needs to use all the NEON registers. + */ + +.macro default_init_need_all_regs +.endm + +.macro default_cleanup_need_all_regs +.endm + +/******************************************************************************/ + +/* + * Conversion of 8 r5g6b6 pixels packed in 128-bit register (in) + * into a planar a8r8g8b8 format (with a, r, g, b color components + * stored into 64-bit registers out_a, out_r, out_g, out_b respectively). + * + * Warning: the conversion is destructive and the original + * value (in) is lost. + */ +.macro convert_0565_to_8888 in, out_a, out_r, out_g, out_b + shrn &out_r&.8b, &in&.8h, #8 + shrn &out_g&.8b, &in&.8h, #3 + sli &in&.8h, &in&.8h, #5 + movi &out_a&.8b, #255 + sri &out_r&.8b, &out_r&.8b, #5 + sri &out_g&.8b, &out_g&.8b, #6 + shrn &out_b&.8b, &in&.8h, #2 +.endm + +.macro convert_0565_to_x888 in, out_r, out_g, out_b + shrn &out_r&.8b, &in&.8h, #8 + shrn &out_g&.8b, &in&.8h, #3 + sli &in&.8h, &in&.8h, #5 + sri &out_r&.8b, &out_r&.8b, #5 + sri &out_g&.8b, &out_g&.8b, #6 + shrn &out_b&.8b, &in&.8h, #2 +.endm + +/* + * Conversion from planar a8r8g8b8 format (with a, r, g, b color components + * in 64-bit registers in_a, in_r, in_g, in_b respectively) into 8 r5g6b6 + * pixels packed in 128-bit register (out). Requires two temporary 128-bit + * registers (tmp1, tmp2) + */ +.macro convert_8888_to_0565 in_r, in_g, in_b, out, tmp1, tmp2 + ushll &tmp1&.8h, &in_g&.8b, #7 + shl &tmp1&.8h, &tmp1&.8h, #1 + ushll &out&.8h, &in_r&.8b, #7 + shl &out&.8h, &out&.8h, #1 + ushll &tmp2&.8h, &in_b&.8b, #7 + shl &tmp2&.8h, &tmp2&.8h, #1 + sri &out&.8h, &tmp1&.8h, #5 + sri &out&.8h, &tmp2&.8h, #11 +.endm + +/* + * Conversion of four r5g6b5 pixels (in) to four x8r8g8b8 pixels + * returned in (out0, out1) registers pair. Requires one temporary + * 64-bit register (tmp). 'out1' and 'in' may overlap, the original + * value from 'in' is lost + */ +.macro convert_four_0565_to_x888_packed in, out0, out1, tmp + shl &out0&.4h, &in&.4h, #5 /* G top 6 bits */ + shl &tmp&.4h, &in&.4h, #11 /* B top 5 bits */ + sri &in&.4h, &in&.4h, #5 /* R is ready in top bits */ + sri &out0&.4h, &out0&.4h, #6 /* G is ready in top bits */ + sri &tmp&.4h, &tmp&.4h, #5 /* B is ready in top bits */ + ushr &out1&.4h, &in&.4h, #8 /* R is in place */ + sri &out0&.4h, &tmp&.4h, #8 /* G & B is in place */ + zip1 &tmp&.4h, &out0&.4h, &out1&.4h /* everything is in place */ + zip2 &out1&.4h, &out0&.4h, &out1&.4h + mov &out0&.d[0], &tmp&.d[0] +.endm diff --git a/pixman/pixman-private.h b/pixman/pixman-private.h index 73a5414..22c8ccc --- a/pixman/pixman-private.h +++ b/pixman/pixman-private.h @@ -607,6 +607,11 @@ pixman_implementation_t * _pixman_implementation_create_arm_neon (pixman_implementation_t *fallback); #endif +#ifdef USE_ARM_A64_NEON +pixman_implementation_t * +_pixman_implementation_create_arm_neon (pixman_implementation_t *fallback); +#endif + #ifdef USE_MIPS_DSPR2 pixman_implementation_t * _pixman_implementation_create_mips_dspr2 (pixman_implementation_t *fallback); -- 2.8.0