Attachment 122816 Details for Bug 94758 – Proposed patch v3

[patch] Proposed patch v3

0001-PATCH-pixman-Add-support-for-aarch64-neon-optimizati.patch (text/plain), 191.71 KB, created by Mizuki Asakura on 2016-04-08 11:33:35 UTC

(hide)

Description:

Filename:

MIME Type:

Creator: Mizuki Asakura

Created: 2016-04-08 11:33:35 UTC

Size: 191.71 KB

patch

obsolete

>ï»¿From a211409e1195549af76371f2967385a19df0aecf Mon Sep 17 00:00:00 2001
>From: Mizuki Asakura <ed6e117f@gmail.com>
>Date: Fri, 8 Apr 2016 19:40:05 +0900
>Subject: [PATCH] pixman: Add support for aarch64 neon optimization (ver.3)
>MIME-Version: 1.0
>Content-Type: text/plain; charset=UTF-8
>Content-Transfer-Encoding: 8bit
>
>Since aarch64 has different neon syntax from aarch32 and has no
>support for (older) arm-simd,
>there are no SIMD accelerations for pixman on aarch64.
>
>We need new implementations.
>
>This patch only contains FAST_PATH codes, not bilinear optimizations codes.
>After completing optimization this patch, bilinear related codes should be done.
>
>Added: https://bugs.freedesktop.org/show_bug.cgi?id=94758
>Signed-off-by: Mizuki Asakura <ed6e117f@gmail.com>
>---
> configure.ac                    |   36 +-
> pixman/Makefile.am              |   16 +-
> pixman/pixman-arm-neon.c        |   12 +-
> pixman/pixman-arm.c             |    6 +
> pixman/pixman-arma64-neon-asm.S | 3708 +++++++++++++++++++++++++++++++++++++++
> pixman/pixman-arma64-neon-asm.h | 1300 ++++++++++++++
> pixman/pixman-private.h         |    7 +-
> 7 files changed, 5080 insertions(+), 5 deletions(-)
> create mode 100644 pixman/pixman-arma64-neon-asm.S
> create mode 100644 pixman/pixman-arma64-neon-asm.h
>
>diff --git a/configure.ac b/configure.ac
>index 6b2134e..4ddcf73 100644
>--- a/configure.ac
>+++ b/configure.ac
>@@ -667,6 +667,40 @@ if test $enable_arm_neon = yes && test $have_arm_neon = no ; then
>    AC_MSG_ERROR([ARM NEON intrinsics not detected])
> fi
> 
>+dnl ==========================================================================
>+dnl Check if assembler is gas compatible and supports ARM-aarch64 NEON instructions
>+have_arm_a64_neon=no
>+AC_MSG_CHECKING(whether to use ARM A64 NEON assembler)
>+xserver_save_CFLAGS=$CFLAGS
>+CFLAGS="-x assembler-with-cpp $CFLAGS"
>+AC_COMPILE_IFELSE([AC_LANG_SOURCE([[
>+.text
>+.arch armv8-a
>+.altmacro
>+prfm pldl2strm, [x0]
>+xtn v0.8b, v0.8h]])], have_arm_a64_neon=yes)
>+CFLAGS=$xserver_save_CFLAGS
>+
>+AC_ARG_ENABLE(arm-a64-neon,
>+   [AC_HELP_STRING([--disable-arm-a64-neon],
>+                   [disable ARM A64 NEON fast paths])],
>+   [enable_arm_a64_neon=$enableval], [enable_arm_a64_neon=auto])
>+
>+if test $enable_arm_a64_neon = no ; then
>+   have_arm_a64_neon=disabled
>+fi
>+
>+if test $have_arm_a64_neon = yes ; then
>+   AC_DEFINE(USE_ARM_A64_NEON, 1, [use ARM A64_NEON assembly optimizations])
>+fi
>+
>+AM_CONDITIONAL(USE_ARM_A64_NEON, test $have_arm_a64_neon = yes)
>+
>+AC_MSG_RESULT($have_arm_a64_neon)
>+if test $enable_arm_a64_neon = yes && test $have_arm_a64_neon4 = no ; then
>+   AC_MSG_ERROR([ARM A64 NEON intrinsics not detected])
>+fi
>+
> dnl ===========================================================================
> dnl Check for IWMMXT
> 
>diff --git a/pixman/Makefile.am b/pixman/Makefile.am
>index 581b6f6..5185f0f 100644
>--- a/pixman/Makefile.am
>+++ b/pixman/Makefile.am
>@@ -94,6 +94,20 @@ libpixman_1_la_LIBADD += libpixman-arm-neon.la
> ASM_CFLAGS_arm_neon=
> endif
> 
>+# arm a64 neon code
>+if USE_ARM_A64_NEON
>+noinst_LTLIBRARIES += libpixman-arma64-neon.la
>+libpixman_arma64_neon_la_SOURCES = \
>+        pixman-arm-neon.c	 \
>+        pixman-arm-common.h	 \
>+        pixman-arma64-neon-asm.S \
>+        pixman-arm-asm.h	 \
>+        pixman-arma64-neon-asm.h
>+libpixman_1_la_LIBADD += libpixman-arma64-neon.la
>+
>+ASM_CFLAGS_arm_neon=
>+endif
>+
> # iwmmxt code
> if USE_ARM_IWMMXT
> libpixman_iwmmxt_la_SOURCES = pixman-mmx.c
>diff --git a/pixman/pixman-arm-neon.c b/pixman/pixman-arm-neon.c
>index be761c9..0dc9f05 100644
>--- a/pixman/pixman-arm-neon.c
>+++ b/pixman/pixman-arm-neon.c
>@@ -143,6 +143,7 @@ PIXMAN_ARM_BIND_SCALED_BILINEAR_SRC_DST (0, neon, 0565_x888, SRC,
>                                          uint16_t, uint32_t)
> PIXMAN_ARM_BIND_SCALED_BILINEAR_SRC_DST (0, neon, 0565_0565, SRC,
>                                          uint16_t, uint16_t)
>+#ifndef __aarch64__
> PIXMAN_ARM_BIND_SCALED_BILINEAR_SRC_DST (SKIP_ZERO_SRC, neon, 8888_8888, OVER,
>                                          uint32_t, uint32_t)
> PIXMAN_ARM_BIND_SCALED_BILINEAR_SRC_DST (SKIP_ZERO_SRC, neon, 8888_8888, ADD,
>@@ -160,6 +161,7 @@ PIXMAN_ARM_BIND_SCALED_BILINEAR_SRC_A8_DST (SKIP_ZERO_SRC, neon, 8888_8_8888, OV
>                                             uint32_t, uint32_t)
> PIXMAN_ARM_BIND_SCALED_BILINEAR_SRC_A8_DST (SKIP_ZERO_SRC, neon, 8888_8_8888, ADD,
>                                             uint32_t, uint32_t)
>+#endif
> 
> void
> pixman_composite_src_n_8_asm_neon (int32_t   w,
>@@ -194,7 +196,7 @@ arm_neon_fill (pixman_implementation_t *imp,
> 	       uint32_t                 _xor)
> {
>     /* stride is always multiple of 32bit units in pixman */
>-    uint32_t byte_stride = stride * sizeof(uint32_t);
>+    int32_t byte_stride = stride * sizeof(uint32_t);
> 
>     switch (bpp)
>     {
>@@ -399,6 +401,7 @@ static const pixman_fast_path_t arm_neon_fast_paths[] =
>     SIMPLE_BILINEAR_FAST_PATH (SRC, r5g6b5, x8r8g8b8, neon_0565_x888),
>     SIMPLE_BILINEAR_FAST_PATH (SRC, r5g6b5, r5g6b5, neon_0565_0565),
> 
>+#ifndef __aarch64__
>     SIMPLE_BILINEAR_FAST_PATH (OVER, a8r8g8b8, a8r8g8b8, neon_8888_8888),
>     SIMPLE_BILINEAR_FAST_PATH (OVER, a8r8g8b8, x8r8g8b8, neon_8888_8888),
> 
>@@ -420,10 +423,12 @@ static const pixman_fast_path_t arm_neon_fast_paths[] =
> 
>     SIMPLE_BILINEAR_A8_MASK_FAST_PATH (ADD, a8r8g8b8, a8r8g8b8, neon_8888_8_8888),
>     SIMPLE_BILINEAR_A8_MASK_FAST_PATH (ADD, a8r8g8b8, x8r8g8b8, neon_8888_8_8888),
>+#endif
> 
>     { PIXMAN_OP_NONE },
> };
> 
>+#ifndef __aarch64__
> #define BIND_COMBINE_U(name)                                             \
> void                                                                     \
> pixman_composite_scanline_##name##_mask_asm_neon (int32_t         w,     \
>@@ -454,6 +459,7 @@ neon_combine_##name##_u (pixman_implementation_t *imp,                   \
> BIND_COMBINE_U (over)
> BIND_COMBINE_U (add)
> BIND_COMBINE_U (out_reverse)
>+#endif
> 
> pixman_implementation_t *
> _pixman_implementation_create_arm_neon (pixman_implementation_t *fallback)
>@@ -461,9 +467,11 @@ _pixman_implementation_create_arm_neon (pixman_implementation_t *fallback)
>     pixman_implementation_t *imp =
> 	_pixman_implementation_create (fallback, arm_neon_fast_paths);
> 
>+#ifndef __aarch64__
>     imp->combine_32[PIXMAN_OP_OVER] = neon_combine_over_u;
>     imp->combine_32[PIXMAN_OP_ADD] = neon_combine_add_u;
>     imp->combine_32[PIXMAN_OP_OUT_REVERSE] = neon_combine_out_reverse_u;
>+#endif
> 
>     imp->blt = arm_neon_blt;
>     imp->fill = arm_neon_fill;
>diff --git a/pixman/pixman-arm.c b/pixman/pixman-arm.c
>index 23374e4..734cbea 100644
>--- a/pixman/pixman-arm.c
>+++ b/pixman/pixman-arm.c
>@@ -221,5 +221,11 @@ _pixman_arm_get_implementations (pixman_implementation_t *imp)
> 	imp = _pixman_implementation_create_arm_neon (imp);
> #endif
> 
>+#ifdef USE_ARM_A64_NEON
>+    /* neon is a part of aarch64 */
>+    if (!_pixman_disabled ("arm-neon"))
>+        imp = _pixman_implementation_create_arm_neon (imp);
>+#endif
>+
>     return imp;
> }
>diff --git a/pixman/pixman-arma64-neon-asm.S b/pixman/pixman-arma64-neon-asm.S
>new file mode 100644
>index 0000000..6e1fee5
>--- /dev/null
>+++ b/pixman/pixman-arma64-neon-asm.S
>@@ -0,0 +1,3708 @@
>+/*
>+ * Copyright Â© 2009 Nokia Corporation
>+ *
>+ * Permission is hereby granted, free of charge, to any person obtaining a
>+ * copy of this software and associated documentation files (the "Software"),
>+ * to deal in the Software without restriction, including without limitation
>+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
>+ * and/or sell copies of the Software, and to permit persons to whom the
>+ * Software is furnished to do so, subject to the following conditions:
>+ *
>+ * The above copyright notice and this permission notice (including the next
>+ * paragraph) shall be included in all copies or substantial portions of the
>+ * Software.
>+ *
>+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
>+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
>+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
>+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
>+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
>+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
>+ * DEALINGS IN THE SOFTWARE.
>+ *
>+ * Author:  Siarhei Siamashka (siarhei.siamashka@nokia.com)
>+ */
>+
>+/*
>+ * This file contains implementations of NEON optimized pixel processing
>+ * functions. There is no full and detailed tutorial, but some functions
>+ * (those which are exposing some new or interesting features) are
>+ * extensively commented and can be used as examples.
>+ *
>+ * You may want to have a look at the comments for following functions:
>+ *  - pixman_composite_over_8888_0565_asm_neon
>+ *  - pixman_composite_over_n_8_0565_asm_neon
>+ */
>+
>+/* Prevent the stack from becoming executable for no reason... */
>+#if defined(__linux__) && defined(__ELF__)
>+.section .note.GNU-stack,"",%progbits
>+#endif
>+
>+.text
>+.arch armv8-a
>+
>+.altmacro
>+.p2align 2
>+
>+#include "pixman-private.h"
>+#include "pixman-arm-asm.h"
>+#include "pixman-arma64-neon-asm.h"
>+
>+/* Global configuration options and preferences */
>+
>+/*
>+ * The code can optionally make use of unaligned memory accesses to improve
>+ * performance of handling leading/trailing pixels for each scanline.
>+ * Configuration variable RESPECT_STRICT_ALIGNMENT can be set to 0 for
>+ * example in linux if unaligned memory accesses are not configured to
>+ * generate.exceptions.
>+ */
>+.set RESPECT_STRICT_ALIGNMENT, 1
>+
>+/*
>+ * Set default prefetch type. There is a choice between the following options:
>+ *
>+ * PREFETCH_TYPE_NONE (may be useful for the ARM cores where PLD is set to work
>+ * as NOP to workaround some HW bugs or for whatever other reason)
>+ *
>+ * PREFETCH_TYPE_SIMPLE (may be useful for simple single-issue ARM cores where
>+ * advanced prefetch intruduces heavy overhead)
>+ *
>+ * PREFETCH_TYPE_ADVANCED (useful for superscalar cores such as ARM Cortex-A8
>+ * which can run ARM and NEON instructions simultaneously so that extra ARM
>+ * instructions do not add (many) extra cycles, but improve prefetch efficiency)
>+ *
>+ * Note: some types of function can't support advanced prefetch and fallback
>+ *       to simple one (those which handle 24bpp pixels)
>+ */
>+.set PREFETCH_TYPE_DEFAULT, PREFETCH_TYPE_ADVANCED
>+
>+/* Prefetch distance in pixels for simple prefetch */
>+.set PREFETCH_DISTANCE_SIMPLE, 64
>+
>+/*
>+ * Implementation of pixman_composite_over_8888_0565_asm_neon
>+ *
>+ * This function takes a8r8g8b8 source buffer, r5g6b5 destination buffer and
>+ * performs OVER compositing operation. Function fast_composite_over_8888_0565
>+ * from pixman-fast-path.c does the same in C and can be used as a reference.
>+ *
>+ * First we need to have some NEON assembly code which can do the actual
>+ * operation on the pixels and provide it to the template macro.
>+ *
>+ * Template macro quite conveniently takes care of emitting all the necessary
>+ * code for memory reading and writing (including quite tricky cases of
>+ * handling unaligned leading/trailing pixels), so we only need to deal with
>+ * the data in NEON registers.
>+ *
>+ * NEON registers allocation in general is recommented to be the following:
>+ * v0,  v1,  v2,  v3  - contain loaded source pixel data
>+ * v4,  v5,  v6,  v7  - contain loaded destination pixels (if they are needed)
>+ * v24, v25, v26, v27 - contain loading mask pixel data (if mask is used)
>+ * v28, v29, v30, v31 - place for storing the result (destination pixels)
>+ *
>+ * As can be seen above, four 64-bit NEON registers are used for keeping
>+ * intermediate pixel data and up to 8 pixels can be processed in one step
>+ * for 32bpp formats (16 pixels for 16bpp, 32 pixels for 8bpp).
>+ *
>+ * This particular function uses the following registers allocation:
>+ * v0,  v1,  v2,  v3  - contain loaded source pixel data
>+ * v4,  v5            - contain loaded destination pixels (they are needed)
>+ * v28, v29           - place for storing the result (destination pixels)
>+ */
>+
>+/*
>+ * Step one. We need to have some code to do some arithmetics on pixel data.
>+ * This is implemented as a pair of macros: '*_head' and '*_tail'. When used
>+ * back-to-back, they take pixel data from {v0, v1, v2, v3} and {v4, v5},
>+ * perform all the needed calculations and write the result to {v28, v29}.
>+ * The rationale for having two macros and not just one will be explained
>+ * later. In practice, any single monolitic function which does the work can
>+ * be split into two parts in any arbitrary way without affecting correctness.
>+ *
>+ * There is one special trick here too. Common template macro can optionally
>+ * make our life a bit easier by doing R, G, B, A color components
>+ * deinterleaving for 32bpp pixel formats (and this feature is used in
>+ * 'pixman_composite_over_8888_0565_asm_neon' function). So it means that
>+ * instead of having 8 packed pixels in {v0, v1, v2, v3} registers, we
>+ * actually use v0 register for blue channel (a vector of eight 8-bit
>+ * values), v1 register for green, v2 for red and v3 for alpha. This
>+ * simple conversion can be also done with a few NEON instructions:
>+ *
>+ * Packed to planar conversion: // vuzp8 is a wrapper macro
>+ *  vuzp8 v0, v1
>+ *  vuzp8 v2, v3
>+ *  vuzp8 v1, v3
>+ *  vuzp8 v0, v2
>+ *
>+ * Planar to packed conversion: // vzip8 is a wrapper macro
>+ *  vzip8 v0, v2
>+ *  vzip8 v1, v3
>+ *  vzip8 v2, v3
>+ *  vzip8 v0, v1
>+ *
>+ * But pixel can be loaded directly in planar format using LD4 / b NEON
>+ * instruction. It is 1 cycle slower than LD1 / s, so this is not always
>+ * desirable, that's why deinterleaving is optional.
>+ *
>+ * But anyway, here is the code:
>+ */
>+
>+.macro pixman_composite_over_8888_0565_process_pixblock_head
>+    /* convert 8 r5g6b5 pixel data from {v4} to planar 8-bit format
>+       and put data into v6 - red, v7 - green, v30 - blue */
>+    mov         v4.d[1], v5.d[0]
>+    shrn        v6.8b, v4.8h, #8
>+    shrn        v7.8b, v4.8h, #3
>+    sli         v4.8h, v4.8h, #5
>+    sri         v6.8b, v6.8b, #5
>+    mvn         v3.8b, v3.8b      /* invert source alpha */
>+    sri         v7.8b, v7.8b, #6
>+    shrn        v30.8b, v4.8h, #2
>+    /* now do alpha blending, storing results in 8-bit planar format
>+       into v20 - red, v23 - green, v22 - blue */
>+    umull       v10.8h, v3.8b, v6.8b
>+    umull       v11.8h, v3.8b, v7.8b
>+    umull       v12.8h, v3.8b, v30.8b
>+    urshr       v17.8h, v10.8h, #8
>+    urshr       v18.8h, v11.8h, #8
>+    urshr       v19.8h, v12.8h, #8
>+    raddhn      v20.8b, v10.8h, v17.8h
>+    raddhn      v23.8b, v11.8h, v18.8h
>+    raddhn      v22.8b, v12.8h, v19.8h
>+.endm
>+
>+.macro pixman_composite_over_8888_0565_process_pixblock_tail
>+    /* ... continue alpha blending */
>+    uqadd       v17.8b, v2.8b, v20.8b
>+    uqadd       v18.8b, v0.8b, v22.8b
>+    uqadd       v19.8b, v1.8b, v23.8b
>+    /* convert the result to r5g6b5 and store it into {v14} */
>+    ushll       v14.8h, v17.8b, #7
>+    sli         v14.8h, v14.8h, #1
>+    ushll       v8.8h, v19.8b, #7
>+    sli         v8.8h, v8.8h, #1
>+    ushll       v9.8h, v18.8b, #7
>+    sli         v9.8h, v9.8h, #1
>+    sri         v14.8h, v8.8h, #5
>+    sri         v14.8h, v9.8h, #11
>+    mov         v28.d[0], v14.d[0]
>+    mov         v29.d[0], v14.d[1]
>+.endm
>+
>+/*
>+ * OK, now we got almost everything that we need. Using the above two
>+ * macros, the work can be done right. But now we want to optimize
>+ * it a bit. ARM Cortex-A8 is an in-order core, and benefits really
>+ * a lot from good code scheduling and software pipelining.
>+ *
>+ * Let's construct some code, which will run in the core main loop.
>+ * Some pseudo-code of the main loop will look like this:
>+ *   head
>+ *   while (...) {
>+ *     tail
>+ *     head
>+ *   }
>+ *   tail
>+ *
>+ * It may look a bit weird, but this setup allows to hide instruction
>+ * latencies better and also utilize dual-issue capability more
>+ * efficiently (make pairs of load-store and ALU instructions).
>+ *
>+ * So what we need now is a '*_tail_head' macro, which will be used
>+ * in the core main loop. A trivial straightforward implementation
>+ * of this macro would look like this:
>+ *
>+ *   pixman_composite_over_8888_0565_process_pixblock_tail
>+ *   st1         {v28.4h, v29.4h}, [DST_W], #32
>+ *   ld1         {v4.4h, v5.4h}, [DST_R], #16
>+ *   ld4         {v0.2s, v1.2s, v2.2s, v3.2s}, [SRC], #32
>+ *   pixman_composite_over_8888_0565_process_pixblock_head
>+ *   cache_preload 8, 8
>+ *
>+ * Now it also got some VLD/VST instructions. We simply can't move from
>+ * processing one block of pixels to the other one with just arithmetics.
>+ * The previously processed data needs to be written to memory and new
>+ * data needs to be fetched. Fortunately, this main loop does not deal
>+ * with partial leading/trailing pixels and can load/store a full block
>+ * of pixels in a bulk. Additionally, destination buffer is already
>+ * 16 bytes aligned here (which is good for performance).
>+ *
>+ * New things here are DST_R, DST_W, SRC and MASK identifiers. These
>+ * are the aliases for ARM registers which are used as pointers for
>+ * accessing data. We maintain separate pointers for reading and writing
>+ * destination buffer (DST_R and DST_W).
>+ *
>+ * Another new thing is 'cache_preload' macro. It is used for prefetching
>+ * data into CPU L2 cache and improve performance when dealing with large
>+ * images which are far larger than cache size. It uses one argument
>+ * (actually two, but they need to be the same here) - number of pixels
>+ * in a block. Looking into 'pixman-arm-neon-asm.h' can provide some
>+ * details about this macro. Moreover, if good performance is needed
>+ * the code from this macro needs to be copied into '*_tail_head' macro
>+ * and mixed with the rest of code for optimal instructions scheduling.
>+ * We are actually doing it below.
>+ *
>+ * Now after all the explanations, here is the optimized code.
>+ * Different instruction streams (originaling from '*_head', '*_tail'
>+ * and 'cache_preload' macro) use different indentation levels for
>+ * better readability. Actually taking the code from one of these
>+ * indentation levels and ignoring a few LD/ST instructions would
>+ * result in exactly the code from '*_head', '*_tail' or 'cache_preload'
>+ * macro!
>+ */
>+
>+#if 1
>+
>+.macro pixman_composite_over_8888_0565_process_pixblock_tail_head
>+        uqadd       v17.8b, v2.8b, v20.8b
>+    ld1         {v4.4h, v5.4h}, [DST_R], #16
>+    mov         v4.d[1], v5.d[0]
>+        uqadd       v18.8b, v0.8b, v22.8b
>+        uqadd       v19.8b, v1.8b, v23.8b
>+    shrn        v6.8b, v4.8h, #8
>+    fetch_src_pixblock
>+    shrn        v7.8b, v4.8h, #3
>+    sli         v4.8h, v4.8h, #5
>+        ushll       v14.8h, v17.8b, #7
>+        sli         v14.8h, v14.8h, #1
>+                                    PF add PF_X, PF_X, #8
>+        ushll       v8.8h, v19.8b, #7
>+        sli         v8.8h, v8.8h,  #1
>+                                    PF tst PF_CTL, #0xF
>+    sri         v6.8b, v6.8b, #5
>+                                    PF beq 10f
>+                                    PF add PF_X, PF_X, #8
>+10:
>+    mvn         v3.8b, v3.8b
>+                                    PF beq 10f
>+                                    PF sub PF_CTL, PF_CTL, #1
>+10:
>+    sri         v7.8b, v7.8b, #6
>+    shrn        v30.8b, v4.8h, #2
>+    umull       v10.8h, v3.8b, v6.8b
>+                                    PF lsl DUMMY, PF_X, #src_bpp_shift
>+                                    PF prfm PREFETCH_MODE, [PF_SRC, DUMMY]
>+    umull       v11.8h, v3.8b, v7.8b
>+    umull       v12.8h, v3.8b, v30.8b
>+                                    PF lsl DUMMY, PF_X, #dst_bpp_shift
>+                                    PF prfm PREFETCH_MODE, [PF_DST, DUMMY]
>+        sri         v14.8h, v8.8h, #5
>+                                    PF cmp PF_X, ORIG_W
>+        ushll       v9.8h, v18.8b, #7
>+        sli         v9.8h, v9.8h, #1
>+    urshr       v17.8h, v10.8h, #8
>+                                    PF ble 10f
>+                                    PF sub PF_X, PF_X, ORIG_W
>+10:
>+    urshr       v19.8h, v11.8h, #8
>+    urshr       v18.8h, v12.8h, #8
>+                                    PF ble 10f
>+                                    PF subs PF_CTL, PF_CTL, #0x10
>+10:
>+        sri         v14.8h, v9.8h, #11
>+        mov         v28.d[0], v14.d[0]
>+        mov         v29.d[0], v14.d[1]
>+                                    PF ble 10f
>+                                    PF lsl DUMMY, SRC_STRIDE, #src_bpp_shift
>+                                    PF ldrsb DUMMY, [PF_SRC, DUMMY]
>+                                    PF add PF_SRC, PF_SRC, #1
>+10:
>+    raddhn      v20.8b, v10.8h, v17.8h
>+    raddhn      v23.8b, v11.8h, v19.8h
>+                                    PF ble 10f
>+                                    PF lsl DUMMY, DST_STRIDE, #dst_bpp_shift
>+                                    PF ldrsb DUMMY, [PF_DST, DUMMY]
>+                                    PF add PF_DST, PF_SRC, #1
>+10:
>+    raddhn      v22.8b, v12.8h, v18.8h
>+        st1         {v14.8h}, [DST_W], #16
>+.endm
>+
>+#else
>+
>+/* If we did not care much about the performance, we would just use this... */
>+.macro pixman_composite_over_8888_0565_process_pixblock_tail_head
>+    pixman_composite_over_8888_0565_process_pixblock_tail
>+    st1         {v14.8h}, [DST_W], #16
>+    ld1         {v4.4h, v4.5h}, [DST_R], #16
>+    fetch_src_pixblock
>+    pixman_composite_over_8888_0565_process_pixblock_head
>+    cache_preload 8, 8
>+.endm
>+
>+#endif
>+
>+/*
>+ * And now the final part. We are using 'generate_composite_function' macro
>+ * to put all the stuff together. We are specifying the name of the function
>+ * which we want to get, number of bits per pixel for the source, mask and
>+ * destination (0 if unused, like mask in this case). Next come some bit
>+ * flags:
>+ *   FLAG_DST_READWRITE      - tells that the destination buffer is both read
>+ *                             and written, for write-only buffer we would use
>+ *                             FLAG_DST_WRITEONLY flag instead
>+ *   FLAG_DEINTERLEAVE_32BPP - tells that we prefer to work with planar data
>+ *                             and separate color channels for 32bpp format.
>+ * The next things are:
>+ *  - the number of pixels processed per iteration (8 in this case, because
>+ *    that's the maximum what can fit into four 64-bit NEON registers).
>+ *  - prefetch distance, measured in pixel blocks. In this case it is 5 times
>+ *    by 8 pixels. That would be 40 pixels, or up to 160 bytes. Optimal
>+ *    prefetch distance can be selected by running some benchmarks.
>+ *
>+ * After that we specify some macros, these are 'default_init',
>+ * 'default_cleanup' here which are empty (but it is possible to have custom
>+ * init/cleanup macros to be able to save/restore some extra NEON registers
>+ * like d8-d15 or do anything else) followed by
>+ * 'pixman_composite_over_8888_0565_process_pixblock_head',
>+ * 'pixman_composite_over_8888_0565_process_pixblock_tail' and
>+ * 'pixman_composite_over_8888_0565_process_pixblock_tail_head'
>+ * which we got implemented above.
>+ *
>+ * The last part is the NEON registers allocation scheme.
>+ */
>+generate_composite_function \
>+    pixman_composite_over_8888_0565_asm_neon, 32, 0, 16, \
>+    FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
>+    8, /* number of pixels, processed in a single block */ \
>+    5, /* prefetch distance */ \
>+    default_init, \
>+    default_cleanup, \
>+    pixman_composite_over_8888_0565_process_pixblock_head, \
>+    pixman_composite_over_8888_0565_process_pixblock_tail, \
>+    pixman_composite_over_8888_0565_process_pixblock_tail_head, \
>+    28, /* dst_w_basereg */ \
>+    4,  /* dst_r_basereg */ \
>+    0,  /* src_basereg   */ \
>+    24  /* mask_basereg  */
>+
>+/******************************************************************************/
>+
>+.macro pixman_composite_over_n_0565_process_pixblock_head
>+    /* convert 8 r5g6b5 pixel data from {v4} to planar 8-bit format
>+       and put data into v6 - red, v7 - green, v30 - blue */
>+    mov         v4.d[1], v5.d[0]
>+    shrn        v6.8b, v4.8h, #8
>+    shrn        v7.8b, v4.8h, #3
>+    sli         v4.8h, v4.8h, #5
>+    sri         v6.8b, v6.8b, #5
>+    sri         v7.8b, v7.8b, #6
>+    shrn        v30.8b, v4.8h, #2
>+    /* now do alpha blending, storing results in 8-bit planar format
>+       into v20 - red, v23 - green, v22 - blue */
>+    umull       v10.8h, v3.8b, v6.8b
>+    umull       v11.8h, v3.8b, v7.8b
>+    umull       v12.8h, v3.8b, v30.8b
>+    urshr       v13.8h, v10.8h, #8
>+    urshr       v14.8h, v11.8h, #8
>+    urshr       v15.8h, v12.8h, #8
>+    raddhn      v20.8b, v10.8h, v13.8h
>+    raddhn      v23.8b, v11.8h, v14.8h
>+    raddhn      v22.8b, v12.8h, v15.8h
>+.endm
>+
>+.macro pixman_composite_over_n_0565_process_pixblock_tail
>+    /* ... continue alpha blending */
>+    uqadd       v17.8b, v2.8b, v20.8b
>+    uqadd       v18.8b, v0.8b, v22.8b
>+    uqadd       v19.8b, v1.8b, v23.8b
>+    /* convert the result to r5g6b5 and store it into {v14} */
>+    ushll       v14.8h, v17.8b, #7
>+    sli         v14.8h, v14.8h, #1
>+    ushll       v8.8h, v19.8b, #7
>+    sli         v8.8h, v8.8h, #1
>+    ushll       v9.8h, v18.8b, #7
>+    sli         v9.8h, v9.8h, #1
>+    sri         v14.8h, v8.8h, #5
>+    sri         v14.8h, v9.8h, #11
>+    mov         v28.d[0], v14.d[0]
>+    mov         v29.d[0], v14.d[1]
>+.endm
>+
>+/* TODO: expand macros and do better instructions scheduling */
>+.macro pixman_composite_over_n_0565_process_pixblock_tail_head
>+    pixman_composite_over_n_0565_process_pixblock_tail
>+    ld1         {v4.4h, v5.4h}, [DST_R], #16
>+    st1         {v14.8h}, [DST_W], #16
>+    pixman_composite_over_n_0565_process_pixblock_head
>+    cache_preload 8, 8
>+.endm
>+
>+.macro pixman_composite_over_n_0565_init
>+    mov         v3.s[0], w4
>+    dup         v0.8b, v3.b[0]
>+    dup         v1.8b, v3.b[1]
>+    dup         v2.8b, v3.b[2]
>+    dup         v3.8b, v3.b[3]
>+    mvn         v3.8b, v3.8b      /* invert source alpha */
>+.endm
>+
>+generate_composite_function \
>+    pixman_composite_over_n_0565_asm_neon, 0, 0, 16, \
>+    FLAG_DST_READWRITE, \
>+    8, /* number of pixels, processed in a single block */ \
>+    5, /* prefetch distance */ \
>+    pixman_composite_over_n_0565_init, \
>+    default_cleanup, \
>+    pixman_composite_over_n_0565_process_pixblock_head, \
>+    pixman_composite_over_n_0565_process_pixblock_tail, \
>+    pixman_composite_over_n_0565_process_pixblock_tail_head, \
>+    28, /* dst_w_basereg */ \
>+    4,  /* dst_r_basereg */ \
>+    0,  /* src_basereg   */ \
>+    24  /* mask_basereg  */
>+
>+/******************************************************************************/
>+
>+.macro pixman_composite_src_8888_0565_process_pixblock_head
>+    ushll       v8.8h,  v1.8b,  #7
>+    sli         v8.8h,  v8.8h,  #1
>+    ushll       v14.8h, v2.8b,  #7
>+    sli         v14.8h, v14.8h, #1
>+    ushll       v9.8h,  v0.8b,  #7
>+    sli         v9.8h,  v9.8h,  #1
>+.endm
>+
>+.macro pixman_composite_src_8888_0565_process_pixblock_tail
>+    sri         v14.8h, v8.8h, #5
>+    sri         v14.8h, v9.8h, #11
>+    mov         v28.d[0], v14.d[0]
>+    mov         v29.d[0], v14.d[1]
>+.endm
>+
>+.macro pixman_composite_src_8888_0565_process_pixblock_tail_head
>+        sri         v14.8h, v8.8h, #5
>+                                    PF add PF_X, PF_X, #8
>+                                    PF tst PF_CTL, #0xF
>+    fetch_src_pixblock
>+                                    PF beq 10f
>+                                    PF add PF_X, PF_X, #8
>+                                    PF sub PF_CTL, PF_CTL, #1
>+10:
>+        sri         v14.8h, v9.8h, #11
>+        mov         v28.d[0], v14.d[0]
>+        mov         v29.d[0], v14.d[1]
>+                                    PF cmp PF_X, ORIG_W
>+                                    PF lsl DUMMY, PF_X, #src_bpp_shift
>+                                    PF prfm PREFETCH_MODE, [PF_SRC, DUMMY]
>+    ushll       v8.8h, v1.8b, #7
>+    sli         v8.8h, v8.8h, #1
>+        st1        {v14.8h}, [DST_W], #16
>+                                    PF ble 10f
>+                                    PF sub PF_X, PF_X, ORIG_W
>+                                    PF subs PF_CTL, PF_CTL, #0x10
>+10:
>+    ushll       v14.8h, v2.8b, #7
>+    sli         v14.8h, v14.8h, #1
>+                                    PF ble 10f
>+                                    PF lsl DUMMY, SRC_STRIDE, #src_bpp_shift
>+                                    PF ldrsb DUMMY, [PF_SRC, DUMMY]
>+                                    PF add PF_SRC, PF_SRC, #1
>+10:
>+    ushll       v9.8h, v0.8b, #7
>+    sli         v9.8h, v9.8h, #1
>+.endm
>+
>+generate_composite_function \
>+    pixman_composite_src_8888_0565_asm_neon, 32, 0, 16, \
>+    FLAG_DST_WRITEONLY | FLAG_DEINTERLEAVE_32BPP, \
>+    8, /* number of pixels, processed in a single block */ \
>+    10, /* prefetch distance */ \
>+    default_init, \
>+    default_cleanup, \
>+    pixman_composite_src_8888_0565_process_pixblock_head, \
>+    pixman_composite_src_8888_0565_process_pixblock_tail, \
>+    pixman_composite_src_8888_0565_process_pixblock_tail_head
>+
>+/******************************************************************************/
>+
>+.macro pixman_composite_src_0565_8888_process_pixblock_head
>+    mov         v0.d[1], v1.d[0]
>+    shrn        v30.8b, v0.8h, #8
>+    shrn        v29.8b, v0.8h, #3
>+    sli         v0.8h,  v0.8h, #5
>+    movi        v31.8b, #255
>+    sri         v30.8b, v30.8b, #5
>+    sri         v29.8b, v29.8b, #6
>+    shrn        v28.8b, v0.8h, #2
>+.endm
>+
>+.macro pixman_composite_src_0565_8888_process_pixblock_tail
>+.endm
>+
>+/* TODO: expand macros and do better instructions scheduling */
>+.macro pixman_composite_src_0565_8888_process_pixblock_tail_head
>+    pixman_composite_src_0565_8888_process_pixblock_tail
>+    st4         {v28.8b, v29.8b, v30.8b, v31.8b}, [DST_W], #32
>+    fetch_src_pixblock
>+    pixman_composite_src_0565_8888_process_pixblock_head
>+    cache_preload 8, 8
>+.endm
>+
>+generate_composite_function \
>+    pixman_composite_src_0565_8888_asm_neon, 16, 0, 32, \
>+    FLAG_DST_WRITEONLY | FLAG_DEINTERLEAVE_32BPP, \
>+    8, /* number of pixels, processed in a single block */ \
>+    10, /* prefetch distance */ \
>+    default_init, \
>+    default_cleanup, \
>+    pixman_composite_src_0565_8888_process_pixblock_head, \
>+    pixman_composite_src_0565_8888_process_pixblock_tail, \
>+    pixman_composite_src_0565_8888_process_pixblock_tail_head
>+
>+/******************************************************************************/
>+
>+.macro pixman_composite_add_8_8_process_pixblock_head
>+    uqadd       v28.8b, v0.8b, v4.8b
>+    uqadd       v29.8b, v1.8b, v5.8b
>+    uqadd       v30.8b, v2.8b, v6.8b
>+    uqadd       v31.8b, v3.8b, v7.8b
>+.endm
>+
>+.macro pixman_composite_add_8_8_process_pixblock_tail
>+.endm
>+
>+.macro pixman_composite_add_8_8_process_pixblock_tail_head
>+    fetch_src_pixblock
>+                                    PF add PF_X, PF_X, #32
>+                                    PF tst PF_CTL, #0xF
>+    ld1         {v4.8b, v5.8b, v6.8b, v7.8b}, [DST_R], #32
>+                                    PF beq 10f
>+                                    PF add PF_X, PF_X, #32
>+                                    PF sub PF_CTL, PF_CTL, #1
>+10:
>+        st1     {v28.8b, v29.8b, v30.8b, v31.8b}, [DST_W], #32
>+                                    PF cmp PF_X, ORIG_W
>+                                    PF lsl DUMMY, PF_X, #src_bpp_shift
>+                                    PF prfm PREFETCH_MODE, [PF_SRC, DUMMY]
>+                                    PF lsl DUMMY, PF_X, #dst_bpp_shift
>+                                    PF prfm PREFETCH_MODE, [PF_DST, DUMMY]
>+                                    PF ble 10f
>+                                    PF sub PF_X, PF_X, ORIG_W
>+                                    PF subs PF_CTL, PF_CTL, #0x10
>+10:
>+    uqadd       v28.8b, v0.8b, v4.8b
>+                                    PF ble 10f
>+                                    PF lsl DUMMY, SRC_STRIDE, #src_bpp_shift
>+                                    PF ldrsb DUMMY, [PF_SRC, DUMMY]
>+                                    PF add PF_SRC, PF_SRC, #1
>+                                    PF lsl DUMMY, DST_STRIDE, #dst_bpp_shift
>+                                    PF ldrsb DUMMY, [PF_DST, DUMMY]
>+                                    PF add PF_DST, PF_DST, #1
>+10:
>+    uqadd       v29.8b, v1.8b, v5.8b
>+    uqadd       v30.8b, v2.8b, v6.8b
>+    uqadd       v31.8b, v3.8b, v7.8b
>+.endm
>+
>+generate_composite_function \
>+    pixman_composite_add_8_8_asm_neon, 8, 0, 8, \
>+    FLAG_DST_READWRITE, \
>+    32, /* number of pixels, processed in a single block */ \
>+    10, /* prefetch distance */ \
>+    default_init, \
>+    default_cleanup, \
>+    pixman_composite_add_8_8_process_pixblock_head, \
>+    pixman_composite_add_8_8_process_pixblock_tail, \
>+    pixman_composite_add_8_8_process_pixblock_tail_head
>+
>+/******************************************************************************/
>+
>+.macro pixman_composite_add_8888_8888_process_pixblock_tail_head
>+    fetch_src_pixblock
>+                                    PF add PF_X, PF_X, #8
>+                                    PF tst PF_CTL, #0xF
>+    ld1         {v4.8b, v5.8b, v6.8b, v7.8b}, [DST_R], #32
>+                                    PF beq 10f
>+                                    PF add PF_X, PF_X, #8
>+                                    PF sub PF_CTL, PF_CTL, #1
>+10:
>+        st1     {v28.8b, v29.8b, v30.8b, v31.8b}, [DST_W], #32
>+                                    PF cmp PF_X, ORIG_W
>+                                    PF lsl DUMMY, PF_X, #src_bpp_shift
>+                                    PF prfm PREFETCH_MODE, [PF_SRC, DUMMY]
>+                                    PF lsl DUMMY, PF_X, #dst_bpp_shift
>+                                    PF prfm PREFETCH_MODE, [PF_DST, DUMMY]
>+                                    PF ble 10f
>+                                    PF sub PF_X, PF_X, ORIG_W
>+                                    PF subs PF_CTL, PF_CTL, #0x10
>+10:
>+    uqadd       v28.8b, v0.8b, v4.8b
>+                                    PF ble 10f
>+                                    PF lsl DUMMY, SRC_STRIDE, #src_bpp_shift
>+                                    PF ldrsb DUMMY, [PF_SRC, DUMMY]
>+                                    PF add PF_SRC, PF_SRC, #1
>+                                    PF lsl DUMMY, DST_STRIDE, #dst_bpp_shift
>+                                    PF ldrsb DUMMY, [PF_DST, DUMMY]
>+                                    PF add PF_DST, PF_DST, #1
>+10:
>+    uqadd       v29.8b, v1.8b, v5.8b
>+    uqadd       v30.8b, v2.8b, v6.8b
>+    uqadd       v31.8b, v3.8b, v7.8b
>+.endm
>+
>+generate_composite_function \
>+    pixman_composite_add_8888_8888_asm_neon, 32, 0, 32, \
>+    FLAG_DST_READWRITE, \
>+    8, /* number of pixels, processed in a single block */ \
>+    10, /* prefetch distance */ \
>+    default_init, \
>+    default_cleanup, \
>+    pixman_composite_add_8_8_process_pixblock_head, \
>+    pixman_composite_add_8_8_process_pixblock_tail, \
>+    pixman_composite_add_8888_8888_process_pixblock_tail_head
>+
>+generate_composite_function_single_scanline \
>+    pixman_composite_scanline_add_asm_neon, 32, 0, 32, \
>+    FLAG_DST_READWRITE, \
>+    8, /* number of pixels, processed in a single block */ \
>+    default_init, \
>+    default_cleanup, \
>+    pixman_composite_add_8_8_process_pixblock_head, \
>+    pixman_composite_add_8_8_process_pixblock_tail, \
>+    pixman_composite_add_8888_8888_process_pixblock_tail_head
>+
>+/******************************************************************************/
>+
>+.macro pixman_composite_out_reverse_8888_8888_process_pixblock_head
>+    mvn         v24.8b, v3.8b  /* get inverted alpha */
>+    /* do alpha blending */
>+    umull       v8.8h, v24.8b, v4.8b
>+    umull       v9.8h, v24.8b, v5.8b
>+    umull       v10.8h, v24.8b, v6.8b
>+    umull       v11.8h, v24.8b, v7.8b
>+.endm
>+
>+.macro pixman_composite_out_reverse_8888_8888_process_pixblock_tail
>+    urshr       v14.8h, v8.8h, #8
>+    urshr       v15.8h, v9.8h, #8
>+    urshr       v16.8h, v10.8h, #8
>+    urshr       v17.8h, v11.8h, #8
>+    raddhn      v28.8b, v14.8h, v8.8h
>+    raddhn      v29.8b, v15.8h, v9.8h
>+    raddhn      v30.8b, v16.8h, v10.8h
>+    raddhn      v31.8b, v17.8h, v11.8h
>+.endm
>+
>+.macro pixman_composite_out_reverse_8888_8888_process_pixblock_tail_head
>+     ld4        {v4.8b, v5.8b, v6.8b, v7.8b}, [DST_R], #32
>+        urshr       v14.8h, v8.8h, #8
>+                                    PF add PF_X, PF_X, #8
>+                                    PF tst PF_CTL, #0xF
>+        urshr       v15.8h, v9.8h, #8
>+        urshr       v16.8h, v10.8h, #8
>+        urshr       v17.8h, v11.8h, #8
>+                                    PF beq 10f
>+                                    PF add PF_X, PF_X, #8
>+                                    PF sub PF_CTL, PF_CTL, #1
>+10:
>+        raddhn      v28.8b, v14.8h, v8.8h
>+        raddhn      v29.8b, v15.8h, v9.8h
>+                                    PF cmp PF_X, ORIG_W
>+        raddhn      v30.8b, v16.8h, v10.8h
>+        raddhn      v31.8b, v17.8h, v11.8h
>+    fetch_src_pixblock
>+                                    PF lsl DUMMY, PF_X, #src_bpp_shift
>+                                    PF prfm PREFETCH_MODE, [PF_SRC, DUMMY]
>+    mvn         v22.8b, v3.8b
>+                                    PF lsl DUMMY, PF_X, #dst_bpp_shift
>+                                    PF prfm PREFETCH_MODE, [PF_DST, DUMMY]
>+         st4        {v28.8b, v29.8b, v30.8b, v31.8b}, [DST_W], #32
>+                                    PF ble 10f
>+                                    PF sub PF_X, PF_X, ORIG_W
>+10:
>+    umull      v8.8h, v22.8b, v4.8b
>+                                    PF ble 10f
>+                                    PF subs PF_CTL, PF_CTL, #0x10
>+10:
>+    umull      v9.8h, v22.8b, v5.8b
>+                                    PF ble 10f
>+                                    PF lsl DUMMY, SRC_STRIDE, #src_bpp_shift
>+                                    PF ldrsb DUMMY, [PF_SRC, DUMMY]
>+                                    PF add PF_SRC, PF_SRC, #1
>+10:
>+    umull      v10.8h, v22.8b, v6.8b
>+                                    PF ble 10f
>+                                    PF lsl DUMMY, DST_STRIDE, #dst_bpp_shift
>+                                    PF ldrsb DUMMY, [PF_DST, DUMMY]
>+                                    PF add PF_DST, PF_DST, #1
>+10:
>+     umull     v11.8h, v22.8b, v7.8b
>+.endm
>+
>+generate_composite_function_single_scanline \
>+    pixman_composite_scanline_out_reverse_asm_neon, 32, 0, 32, \
>+    FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
>+    8, /* number of pixels, processed in a single block */ \
>+    default_init, \
>+    default_cleanup, \
>+    pixman_composite_out_reverse_8888_8888_process_pixblock_head, \
>+    pixman_composite_out_reverse_8888_8888_process_pixblock_tail, \
>+    pixman_composite_out_reverse_8888_8888_process_pixblock_tail_head
>+
>+/******************************************************************************/
>+
>+.macro pixman_composite_over_8888_8888_process_pixblock_head
>+    pixman_composite_out_reverse_8888_8888_process_pixblock_head
>+.endm
>+
>+.macro pixman_composite_over_8888_8888_process_pixblock_tail
>+    pixman_composite_out_reverse_8888_8888_process_pixblock_tail
>+    uqadd       v28.8b, v0.8b, v28.8b
>+    uqadd       v29.8b, v1.8b, v29.8b
>+    uqadd       v30.8b, v2.8b, v30.8b
>+    uqadd       v31.8b, v3.8b, v31.8b
>+.endm
>+
>+.macro pixman_composite_over_8888_8888_process_pixblock_tail_head
>+     ld4        {v4.8b, v5.8b, v6.8b, v7.8b}, [DST_R], #32
>+        urshr       v14.8h, v8.8h, #8
>+                                    PF add PF_X, PF_X, #8
>+                                    PF tst PF_CTL, #0xF
>+        urshr       v15.8h, v9.8h, #8
>+        urshr       v16.8h, v10.8h, #8
>+        urshr       v17.8h, v11.8h, #8
>+                                    PF beq 10f
>+                                    PF add PF_X, PF_X, #8
>+                                    PF sub PF_CTL, PF_CTL, #1
>+10:
>+        raddhn      v28.8b, v14.8h, v8.8h
>+        raddhn      v29.8b, v15.8h, v9.8h
>+                                    PF cmp PF_X, ORIG_W
>+        raddhn      v30.8b, v16.8h, v10.8h
>+        raddhn      v31.8b, v17.8h, v11.8h
>+        uqadd       v28.8b, v0.8b, v28.8b
>+        uqadd       v29.8b, v1.8b, v29.8b
>+        uqadd       v30.8b, v2.8b, v30.8b
>+        uqadd       v31.8b, v3.8b, v31.8b
>+    fetch_src_pixblock
>+                                    PF lsl DUMMY, PF_X, #src_bpp_shift
>+                                    PF prfm PREFETCH_MODE, [PF_SRC, DUMMY]
>+    mvn        v22.8b, v3.8b
>+                                    PF lsl DUMMY, PF_X, #dst_bpp_shift
>+                                    PF prfm PREFETCH_MODE, [PF_DST, DUMMY]
>+         st4        {v28.8b, v29.8b, v30.8b, v31.8b}, [DST_W], #32
>+                                    PF ble 10f
>+                                    PF sub PF_X, PF_X, ORIG_W
>+10:
>+    umull      v8.8h, v22.8b, v4.8b
>+                                    PF ble 10f
>+                                    PF subs PF_CTL, PF_CTL, #0x10
>+10:
>+    umull      v9.8h, v22.8b, v5.8b
>+                                    PF ble 10f
>+                                    PF lsl DUMMY, SRC_STRIDE, #src_bpp_shift
>+                                    PF ldrsb DUMMY, [PF_SRC, DUMMY]
>+                                    PF add PF_SRC, PF_SRC, #1
>+10:
>+    umull      v10.8h, v22.8b, v6.8b
>+                                    PF ble 10f
>+                                    PF lsl DUMMY, DST_STRIDE, #dst_bpp_shift
>+                                    PF ldrsb DUMMY, [PF_DST, DUMMY]
>+                                    PF add PF_DST, PF_DST, #1
>+10:
>+    umull      v11.8h, v22.8b, v7.8b
>+.endm
>+
>+generate_composite_function \
>+    pixman_composite_over_8888_8888_asm_neon, 32, 0, 32, \
>+    FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
>+    8, /* number of pixels, processed in a single block */ \
>+    5, /* prefetch distance */ \
>+    default_init, \
>+    default_cleanup, \
>+    pixman_composite_over_8888_8888_process_pixblock_head, \
>+    pixman_composite_over_8888_8888_process_pixblock_tail, \
>+    pixman_composite_over_8888_8888_process_pixblock_tail_head
>+
>+generate_composite_function_single_scanline \
>+    pixman_composite_scanline_over_asm_neon, 32, 0, 32, \
>+    FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
>+    8, /* number of pixels, processed in a single block */ \
>+    default_init, \
>+    default_cleanup, \
>+    pixman_composite_over_8888_8888_process_pixblock_head, \
>+    pixman_composite_over_8888_8888_process_pixblock_tail, \
>+    pixman_composite_over_8888_8888_process_pixblock_tail_head
>+
>+/******************************************************************************/
>+
>+.macro pixman_composite_over_n_8888_process_pixblock_head
>+    /* deinterleaved source pixels in {v0, v1, v2, v3} */
>+    /* inverted alpha in {v24} */
>+    /* destination pixels in {v4, v5, v6, v7} */
>+    umull       v8.8h, v24.8b, v4.8b
>+    umull       v9.8h, v24.8b, v5.8b
>+    umull       v10.8h, v24.8b, v6.8b
>+    umull       v11.8h, v24.8b, v7.8b
>+.endm
>+
>+.macro pixman_composite_over_n_8888_process_pixblock_tail
>+    urshr       v14.8h, v8.8h, #8
>+    urshr       v15.8h, v9.8h, #8
>+    urshr       v16.8h, v10.8h, #8
>+    urshr       v17.8h, v11.8h, #8
>+    raddhn      v28.8b, v14.8h, v8.8h
>+    raddhn      v29.8b, v15.8h, v9.8h
>+    raddhn      v30.8b, v16.8h, v10.8h
>+    raddhn      v31.8b, v17.8h, v11.8h
>+    uqadd       v28.8b, v0.8b, v28.8b
>+    uqadd       v29.8b, v1.8b, v29.8b
>+    uqadd       v30.8b, v2.8b, v30.8b
>+    uqadd       v31.8b, v3.8b, v31.8b
>+.endm
>+
>+.macro pixman_composite_over_n_8888_process_pixblock_tail_head
>+        urshr       v14.8h, v8.8h, #8
>+        urshr       v15.8h, v9.8h, #8
>+        urshr       v16.8h, v10.8h, #8
>+        urshr       v17.8h, v11.8h, #8
>+        raddhn      v28.8b, v14.8h, v8.8h
>+        raddhn      v29.8b, v15.8h, v9.8h
>+        raddhn      v30.8b, v16.8h, v10.8h
>+        raddhn      v31.8b, v17.8h, v11.8h
>+    ld4         {v4.8b, v5.8b, v6.8b, v7.8b}, [DST_R], #32
>+        uqadd       v28.8b, v0.8b, v28.8b
>+                                    PF add PF_X, PF_X, #8
>+                                    PF tst PF_CTL, #0x0F
>+                                    PF beq 10f
>+                                    PF add PF_X, PF_X, #8
>+                                    PF sub PF_CTL, PF_CTL, #1
>+10:
>+        uqadd       v29.8b, v1.8b, v29.8b
>+        uqadd       v30.8b, v2.8b, v30.8b
>+        uqadd       v31.8b, v3.8b, v31.8b
>+                                    PF cmp PF_X, ORIG_W
>+    umull       v8.8h, v24.8b, v4.8b
>+                                    PF lsl DUMMY, PF_X, #dst_bpp_shift
>+                                    PF prfm PREFETCH_MODE, [PF_DST, DUMMY]
>+    umull       v9.8h, v24.8b, v5.8b
>+                                    PF ble 10f
>+                                    PF sub PF_X, PF_X, ORIG_W
>+10:
>+    umull       v10.8h, v24.8b, v6.8b
>+                                    PF subs PF_CTL, PF_CTL, #0x10
>+    umull       v11.8h, v24.8b, v7.8b
>+                                    PF ble 10f
>+                                    PF lsl DUMMY, DST_STRIDE, #dst_bpp_shift
>+                                    PF ldrsb DUMMY, [PF_DST, DUMMY]
>+                                    PF add PF_DST, PF_DST, #1
>+10:
>+        st4         {v28.8b, v29.8b, v30.8b, v31.8b}, [DST_W], #32
>+.endm
>+
>+.macro pixman_composite_over_n_8888_init
>+    mov         v3.s[0], w4
>+    dup         v0.8b, v3.b[0]
>+    dup         v1.8b, v3.b[1]
>+    dup         v2.8b, v3.b[2]
>+    dup         v3.8b, v3.b[3]
>+    mvn         v24.8b, v3.8b  /* get inverted alpha */
>+.endm
>+
>+generate_composite_function \
>+    pixman_composite_over_n_8888_asm_neon, 0, 0, 32, \
>+    FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
>+    8, /* number of pixels, processed in a single block */ \
>+    5, /* prefetch distance */ \
>+    pixman_composite_over_n_8888_init, \
>+    default_cleanup, \
>+    pixman_composite_over_8888_8888_process_pixblock_head, \
>+    pixman_composite_over_8888_8888_process_pixblock_tail, \
>+    pixman_composite_over_n_8888_process_pixblock_tail_head
>+
>+/******************************************************************************/
>+
>+.macro pixman_composite_over_reverse_n_8888_process_pixblock_tail_head
>+        urshr       v14.8h, v8.8h, #8
>+                                    PF add PF_X, PF_X, #8
>+                                    PF tst PF_CTL, #0xF
>+        urshr       v15.8h, v9.8h, #8
>+        urshr       v12.8h, v10.8h, #8
>+        urshr       v13.8h, v11.8h, #8
>+                                    PF beq 10f
>+                                    PF add PF_X, PF_X, #8
>+                                    PF sub PF_CTL, PF_CTL, #1
>+10:
>+        raddhn      v28.8b, v14.8h, v8.8h
>+        raddhn      v29.8b, v15.8h, v9.8h
>+                                    PF cmp PF_X, ORIG_W
>+        raddhn      v30.8b, v12.8h, v10.8h
>+        raddhn      v31.8b, v13.8h, v11.8h
>+        uqadd       v28.8b, v0.8b, v28.8b
>+        uqadd       v29.8b, v1.8b, v29.8b
>+        uqadd       v30.8b, v2.8b, v30.8b
>+        uqadd       v31.8b, v3.8b, v31.8b
>+    ld4         {v0.8b, v1.8b, v2.8b, v3.8b}, [DST_R], #32
>+    mvn         v22.8b, v3.8b
>+                                    PF lsl DUMMY, PF_X, #dst_bpp_shift
>+                                    PF prfm PREFETCH_MODE, [PF_DST, DUMMY]
>+        st4         {v28.8b, v29.8b, v30.8b, v31.8b}, [DST_W], #32
>+                                    PF blt 10f
>+                                    PF sub PF_X, PF_X, ORIG_W
>+10:
>+    umull       v8.8h, v22.8b, v4.8b
>+                                    PF blt 10f
>+                                    PF subs PF_CTL, PF_CTL, #0x10
>+10:
>+    umull       v9.8h, v22.8b, v5.8b
>+    umull       v10.8h, v22.8b, v6.8b
>+                                    PF blt 10f
>+                                    PF lsl DUMMY, DST_STRIDE, #dst_bpp_shift
>+                                    PF ldrsb DUMMY, [PF_DST, DUMMY]
>+                                    PF add PF_DST, PF_DST, #1
>+10:
>+    umull       v11.8h, v22.8b, v7.8b
>+.endm
>+
>+.macro pixman_composite_over_reverse_n_8888_init
>+    mov         v7.s[0], w4
>+    dup         v4.8b, v7.b[0]
>+    dup         v5.8b, v7.b[1]
>+    dup         v6.8b, v7.b[2]
>+    dup         v7.8b, v7.b[3]
>+.endm
>+
>+generate_composite_function \
>+    pixman_composite_over_reverse_n_8888_asm_neon, 0, 0, 32, \
>+    FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
>+    8, /* number of pixels, processed in a single block */ \
>+    5, /* prefetch distance */ \
>+    pixman_composite_over_reverse_n_8888_init, \
>+    default_cleanup, \
>+    pixman_composite_over_8888_8888_process_pixblock_head, \
>+    pixman_composite_over_8888_8888_process_pixblock_tail, \
>+    pixman_composite_over_reverse_n_8888_process_pixblock_tail_head, \
>+    28, /* dst_w_basereg */ \
>+    0,  /* dst_r_basereg */ \
>+    4,  /* src_basereg   */ \
>+    24  /* mask_basereg  */
>+
>+/******************************************************************************/
>+
>+.macro pixman_composite_over_8888_8_0565_process_pixblock_head
>+    umull       v0.8h,  v24.8b, v8.8b    /* IN for SRC pixels (part1) */
>+    umull       v1.8h,  v24.8b, v9.8b
>+    umull       v2.8h,  v24.8b, v10.8b
>+    umull       v3.8h,  v24.8b, v11.8b
>+        mov         v4.d[1], v5.d[0]
>+        shrn        v25.8b,  v4.8h, #8 /* convert DST_R data to 32-bpp (part1) */
>+        shrn        v26.8b,  v4.8h, #3
>+        sli         v4.8h,   v4.8h, #5
>+    urshr       v17.8h, v0.8h,  #8    /* IN for SRC pixels (part2) */
>+    urshr       v18.8h, v1.8h,  #8
>+    urshr       v19.8h, v2.8h,  #8
>+    urshr       v20.8h, v3.8h,  #8
>+    raddhn      v0.8b,  v0.8h,  v17.8h
>+    raddhn      v1.8b,  v1.8h,  v18.8h
>+    raddhn      v2.8b,  v2.8h,  v19.8h
>+    raddhn      v3.8b,  v3.8h,  v20.8h
>+        sri         v25.8b, v25.8b, #5 /* convert DST_R data to 32-bpp (part2) */
>+        sri         v26.8b, v26.8b, #6
>+    mvn         v3.8b,  v3.8b
>+        shrn        v30.8b, v4.8h,  #2
>+    umull       v18.8h, v3.8b, v25.8b     /* now do alpha blending */
>+    umull       v19.8h, v3.8b, v26.8b
>+    umull       v20.8h, v3.8b, v30.8b
>+.endm
>+
>+.macro pixman_composite_over_8888_8_0565_process_pixblock_tail
>+    /* 3 cycle bubble (after vmull.u8) */
>+    urshr       v5.8h, v18.8h, #8
>+    urshr       v6.8h, v19.8h, #8
>+    urshr       v7.8h, v20.8h, #8
>+    raddhn      v17.8b, v18.8h, v5.8h
>+    raddhn      v19.8b, v19.8h, v6.8h
>+    raddhn      v18.8b, v20.8h, v7.8h
>+    uqadd       v5.8b, v2.8b,  v17.8b
>+    /* 1 cycle bubble */
>+    uqadd       v6.8b, v0.8b,  v18.8b
>+    uqadd       v7.8b, v1.8b,  v19.8b
>+    ushll       v14.8h, v5.8b, #7    /* convert to 16bpp */
>+    sli         v14.8h, v14.8h, #1
>+    ushll       v18.8h, v7.8b, #7
>+    sli         v18.8h, v18.8h, #1
>+    ushll       v19.8h, v6.8b, #7
>+    sli         v19.8h, v19.8h, #1
>+    sri         v14.8h, v18.8h, #5
>+    /* 1 cycle bubble */
>+    sri         v14.8h, v19.8h, #11
>+    mov         v28.d[0], v14.d[0]
>+    mov         v29.d[0], v14.d[1]
>+.endm
>+
>+.macro pixman_composite_over_8888_8_0565_process_pixblock_tail_head
>+#if 0
>+    ld1         {v4.8h}, [DST_R], #16
>+    shrn        v25.8b,  v4.8h,  #8
>+    fetch_mask_pixblock
>+    shrn        v26.8b,  v4.8h,  #3
>+    fetch_src_pixblock
>+    umull       v22.8h,  v24.8b, v10.8b
>+        urshr       v13.8h, v18.8h, #8
>+        urshr       v11.8h, v19.8h, #8
>+        urshr       v15.8h, v20.8h, #8
>+        raddhn      v17.8b, v18.8h, v13.8h
>+        raddhn      v19.8b, v19.8h, v11.8h
>+        raddhn      v18.8b, v20.8h, v15.8h
>+        uqadd       v17.8b, v2.8b, v17.8b
>+    umull       v21.8h,  v24.8b, v9.8b
>+        uqadd       v18.8b, v0.8b, v18.8b
>+        uqadd       v19.8b, v1.8b, v19.8b
>+        ushll       v14.8h, v17.8b, #7
>+        sli         v14.8h, v14.8h, #1
>+    umull       v20.8h,  v24.8b, v8.8b
>+        ushll       v18.8h,  v18.8b, #7
>+        sli         v18.8h,  v18.8h, #1
>+        ushll       v19.8h,  v19.8b, #7
>+        sli         v19.8h,  v19.8h, #1
>+        sri         v14.8h,  v18.8h, #5
>+    umull       v23.8h,  v24.8b, v11.8b
>+        sri         v14.8h,  v19.8h, #11
>+        mov         v28.d[0], v14.d[0]
>+        mov         v29.d[0], v14.d[1]
>+
>+    cache_preload 8, 8
>+    
>+    sli         v4.8h,  v4.8h,   #5
>+    urshr       v16.8h, v20.8h,  #8
>+    urshr       v17.8h, v21.8h,  #8
>+    urshr       v18.8h, v22.8h,  #8
>+    urshr       v19.8h, v23.8h,  #8
>+    raddhn      v0.8b,  v20.8h, v16.8h
>+    raddhn      v1.8b,  v21.8h, v17.8h
>+    raddhn      v2.8b,  v22.8h, v18.8h
>+    raddhn      v3.8b,  v23.8h, v19.8h
>+    sri         v25.8b,  v25.8b,  #5
>+    sri         v26.8b,  v26.8b,  #6
>+    mvn         v3.8b,  v3.8b
>+    shrn        v30.8b, v4.8h,  #2
>+    st1         {v14.8h}, [DST_W], #16
>+    umull       v18.8h, v3.8b, v25.8b
>+    umull       v19.8h, v3.8b, v26.8b
>+    umull       v20.8h, v3.8b, v30.8b
>+#else
>+    pixman_composite_over_8888_8_0565_process_pixblock_tail
>+    st1         {v28.4h, v29.4h}, [DST_W], #16
>+    ld1         {v4.4h, v5.4h}, [DST_R], #16
>+    fetch_mask_pixblock
>+    fetch_src_pixblock
>+    pixman_composite_over_8888_8_0565_process_pixblock_head
>+#endif
>+.endm
>+
>+generate_composite_function \
>+    pixman_composite_over_8888_8_0565_asm_neon, 32, 8, 16, \
>+    FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
>+    8, /* number of pixels, processed in a single block */ \
>+    5, /* prefetch distance */ \
>+    default_init_need_all_regs, \
>+    default_cleanup_need_all_regs, \
>+    pixman_composite_over_8888_8_0565_process_pixblock_head, \
>+    pixman_composite_over_8888_8_0565_process_pixblock_tail, \
>+    pixman_composite_over_8888_8_0565_process_pixblock_tail_head, \
>+    28, /* dst_w_basereg */ \
>+    4,  /* dst_r_basereg */ \
>+    8,  /* src_basereg   */ \
>+    24  /* mask_basereg  */
>+
>+/******************************************************************************/
>+
>+/*
>+ * This function needs a special initialization of solid mask.
>+ * Solid source pixel data is fetched from stack at ARGS_STACK_OFFSET
>+ * offset, split into color components and replicated in d8-d11
>+ * registers. Additionally, this function needs all the NEON registers,
>+ * so it has to save d8-d15 registers which are callee saved according
>+ * to ABI. These registers are restored from 'cleanup' macro. All the
>+ * other NEON registers are caller saved, so can be clobbered freely
>+ * without introducing any problems.
>+ */
>+.macro pixman_composite_over_n_8_0565_init
>+    mov         v11.s[0], w4
>+    dup         v8.8b, v11.b[0]
>+    dup         v9.8b, v11.b[1]
>+    dup         v10.8b, v11.b[2]
>+    dup         v11.8b, v11.b[3]
>+.endm
>+
>+.macro pixman_composite_over_n_8_0565_cleanup
>+.endm
>+
>+generate_composite_function \
>+    pixman_composite_over_n_8_0565_asm_neon, 0, 8, 16, \
>+    FLAG_DST_READWRITE, \
>+    8, /* number of pixels, processed in a single block */ \
>+    5, /* prefetch distance */ \
>+    pixman_composite_over_n_8_0565_init, \
>+    pixman_composite_over_n_8_0565_cleanup, \
>+    pixman_composite_over_8888_8_0565_process_pixblock_head, \
>+    pixman_composite_over_8888_8_0565_process_pixblock_tail, \
>+    pixman_composite_over_8888_8_0565_process_pixblock_tail_head, \
>+    28, /* dst_w_basereg */ \
>+    4,  /* dst_r_basereg */ \
>+    8,  /* src_basereg   */ \
>+    24  /* mask_basereg  */
>+
>+/******************************************************************************/
>+
>+.macro pixman_composite_over_8888_n_0565_init
>+    mov         v24.s[0], w6
>+    dup         v24.8b, v24.b[3]
>+.endm
>+
>+.macro pixman_composite_over_8888_n_0565_cleanup
>+.endm
>+
>+generate_composite_function \
>+    pixman_composite_over_8888_n_0565_asm_neon, 32, 0, 16, \
>+    FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
>+    8, /* number of pixels, processed in a single block */ \
>+    5, /* prefetch distance */ \
>+    pixman_composite_over_8888_n_0565_init, \
>+    pixman_composite_over_8888_n_0565_cleanup, \
>+    pixman_composite_over_8888_8_0565_process_pixblock_head, \
>+    pixman_composite_over_8888_8_0565_process_pixblock_tail, \
>+    pixman_composite_over_8888_8_0565_process_pixblock_tail_head, \
>+    28, /* dst_w_basereg */ \
>+    4,  /* dst_r_basereg */ \
>+    8,  /* src_basereg   */ \
>+    24  /* mask_basereg  */
>+
>+/******************************************************************************/
>+
>+.macro pixman_composite_src_0565_0565_process_pixblock_head
>+.endm
>+
>+.macro pixman_composite_src_0565_0565_process_pixblock_tail
>+.endm
>+
>+.macro pixman_composite_src_0565_0565_process_pixblock_tail_head
>+    st1     {v0.4h, v1.4h, v2.4h, v3.4h}, [DST_W], #32
>+    fetch_src_pixblock
>+    cache_preload 16, 16
>+.endm
>+
>+generate_composite_function \
>+    pixman_composite_src_0565_0565_asm_neon, 16, 0, 16, \
>+    FLAG_DST_WRITEONLY, \
>+    16, /* number of pixels, processed in a single block */ \
>+    10, /* prefetch distance */ \
>+    default_init, \
>+    default_cleanup, \
>+    pixman_composite_src_0565_0565_process_pixblock_head, \
>+    pixman_composite_src_0565_0565_process_pixblock_tail, \
>+    pixman_composite_src_0565_0565_process_pixblock_tail_head, \
>+    0, /* dst_w_basereg */ \
>+    0, /* dst_r_basereg */ \
>+    0, /* src_basereg   */ \
>+    0  /* mask_basereg  */
>+
>+/******************************************************************************/
>+
>+.macro pixman_composite_src_n_8_process_pixblock_head
>+.endm
>+
>+.macro pixman_composite_src_n_8_process_pixblock_tail
>+.endm
>+
>+.macro pixman_composite_src_n_8_process_pixblock_tail_head
>+    st1         {v0.8b, v1.8b, v2.8b, v3.8b}, [DST_W], 32
>+.endm
>+
>+.macro pixman_composite_src_n_8_init
>+    mov         v0.s[0], w4
>+    dup         v3.8b, v0.b[0]
>+    dup         v2.8b, v0.b[0]
>+    dup         v1.8b, v0.b[0]
>+    dup         v0.8b, v0.b[0]
>+.endm
>+
>+.macro pixman_composite_src_n_8_cleanup
>+.endm
>+
>+generate_composite_function \
>+    pixman_composite_src_n_8_asm_neon, 0, 0, 8, \
>+    FLAG_DST_WRITEONLY, \
>+    32, /* number of pixels, processed in a single block */ \
>+    0,  /* prefetch distance */ \
>+    pixman_composite_src_n_8_init, \
>+    pixman_composite_src_n_8_cleanup, \
>+    pixman_composite_src_n_8_process_pixblock_head, \
>+    pixman_composite_src_n_8_process_pixblock_tail, \
>+    pixman_composite_src_n_8_process_pixblock_tail_head, \
>+    0, /* dst_w_basereg */ \
>+    0, /* dst_r_basereg */ \
>+    0, /* src_basereg   */ \
>+    0  /* mask_basereg  */
>+
>+/******************************************************************************/
>+
>+.macro pixman_composite_src_n_0565_process_pixblock_head
>+.endm
>+
>+.macro pixman_composite_src_n_0565_process_pixblock_tail
>+.endm
>+
>+.macro pixman_composite_src_n_0565_process_pixblock_tail_head
>+    st1     {v0.4h, v1.4h, v2.4h, v3.4h}, [DST_W], #32
>+.endm
>+
>+.macro pixman_composite_src_n_0565_init
>+    mov         v0.s[0], w4
>+    dup         v3.4h, v0.h[0]
>+    dup         v2.4h, v0.h[0]
>+    dup         v1.4h, v0.h[0]
>+    dup         v0.4h, v0.h[0]
>+.endm
>+
>+.macro pixman_composite_src_n_0565_cleanup
>+.endm
>+
>+generate_composite_function \
>+    pixman_composite_src_n_0565_asm_neon, 0, 0, 16, \
>+    FLAG_DST_WRITEONLY, \
>+    16, /* number of pixels, processed in a single block */ \
>+    0,  /* prefetch distance */ \
>+    pixman_composite_src_n_0565_init, \
>+    pixman_composite_src_n_0565_cleanup, \
>+    pixman_composite_src_n_0565_process_pixblock_head, \
>+    pixman_composite_src_n_0565_process_pixblock_tail, \
>+    pixman_composite_src_n_0565_process_pixblock_tail_head, \
>+    0, /* dst_w_basereg */ \
>+    0, /* dst_r_basereg */ \
>+    0, /* src_basereg   */ \
>+    0  /* mask_basereg  */
>+
>+/******************************************************************************/
>+
>+.macro pixman_composite_src_n_8888_process_pixblock_head
>+.endm
>+
>+.macro pixman_composite_src_n_8888_process_pixblock_tail
>+.endm
>+
>+.macro pixman_composite_src_n_8888_process_pixblock_tail_head
>+    st1         {v0.2s, v1.2s, v2.2s, v3.2s}, [DST_W], #32
>+.endm
>+
>+.macro pixman_composite_src_n_8888_init
>+    mov         v0.s[0], w4
>+    dup         v3.2s, v0.s[0]
>+    dup         v2.2s, v0.s[0]
>+    dup         v1.2s, v0.s[0]
>+    dup         v0.2s, v0.s[0]
>+.endm
>+
>+.macro pixman_composite_src_n_8888_cleanup
>+.endm
>+
>+generate_composite_function \
>+    pixman_composite_src_n_8888_asm_neon, 0, 0, 32, \
>+    FLAG_DST_WRITEONLY, \
>+    8, /* number of pixels, processed in a single block */ \
>+    0, /* prefetch distance */ \
>+    pixman_composite_src_n_8888_init, \
>+    pixman_composite_src_n_8888_cleanup, \
>+    pixman_composite_src_n_8888_process_pixblock_head, \
>+    pixman_composite_src_n_8888_process_pixblock_tail, \
>+    pixman_composite_src_n_8888_process_pixblock_tail_head, \
>+    0, /* dst_w_basereg */ \
>+    0, /* dst_r_basereg */ \
>+    0, /* src_basereg   */ \
>+    0  /* mask_basereg  */
>+
>+/******************************************************************************/
>+
>+.macro pixman_composite_src_8888_8888_process_pixblock_head
>+.endm
>+
>+.macro pixman_composite_src_8888_8888_process_pixblock_tail
>+.endm
>+
>+.macro pixman_composite_src_8888_8888_process_pixblock_tail_head
>+    st1  {v0.2s, v1.2s, v2.2s, v3.2s}, [DST_W], #32
>+    fetch_src_pixblock
>+    cache_preload 8, 8
>+.endm
>+
>+generate_composite_function \
>+    pixman_composite_src_8888_8888_asm_neon, 32, 0, 32, \
>+    FLAG_DST_WRITEONLY, \
>+    8, /* number of pixels, processed in a single block */ \
>+    10, /* prefetch distance */ \
>+    default_init, \
>+    default_cleanup, \
>+    pixman_composite_src_8888_8888_process_pixblock_head, \
>+    pixman_composite_src_8888_8888_process_pixblock_tail, \
>+    pixman_composite_src_8888_8888_process_pixblock_tail_head, \
>+    0, /* dst_w_basereg */ \
>+    0, /* dst_r_basereg */ \
>+    0, /* src_basereg   */ \
>+    0  /* mask_basereg  */
>+
>+/******************************************************************************/
>+
>+.macro pixman_composite_src_x888_8888_process_pixblock_head
>+    orr      v0.8b, v0.8b, v4.8b
>+    orr      v1.8b, v1.8b, v4.8b
>+    orr      v2.8b, v2.8b, v4.8b
>+    orr      v3.8b, v3.8b, v4.8b
>+.endm
>+
>+.macro pixman_composite_src_x888_8888_process_pixblock_tail
>+.endm
>+
>+.macro pixman_composite_src_x888_8888_process_pixblock_tail_head
>+    st1      {v0.2s, v1.2s, v2.2s, v3.2s}, [DST_W], #32
>+    fetch_src_pixblock
>+    orr      v0.8b, v0.8b, v4.8b
>+    orr      v1.8b, v1.8b, v4.8b
>+    orr      v2.8b, v2.8b, v4.8b
>+    orr      v3.8b, v3.8b, v4.8b
>+    cache_preload 8, 8
>+.endm
>+
>+.macro pixman_composite_src_x888_8888_init
>+    mov     w20, #0xFF
>+    dup     v4.8b, w20
>+    shl     v4.2s, v4.2s, #24
>+.endm
>+
>+generate_composite_function \
>+    pixman_composite_src_x888_8888_asm_neon, 32, 0, 32, \
>+    FLAG_DST_WRITEONLY, \
>+    8, /* number of pixels, processed in a single block */ \
>+    10, /* prefetch distance */ \
>+    pixman_composite_src_x888_8888_init, \
>+    default_cleanup, \
>+    pixman_composite_src_x888_8888_process_pixblock_head, \
>+    pixman_composite_src_x888_8888_process_pixblock_tail, \
>+    pixman_composite_src_x888_8888_process_pixblock_tail_head, \
>+    0, /* dst_w_basereg */ \
>+    0, /* dst_r_basereg */ \
>+    0, /* src_basereg   */ \
>+    0  /* mask_basereg  */
>+
>+/******************************************************************************/
>+
>+.macro pixman_composite_src_n_8_8888_process_pixblock_head
>+    /* expecting solid source in {v0, v1, v2, v3} */
>+    /* mask is in v24 (v25, v26, v27 are unused) */
>+
>+    /* in */
>+    umull       v8.8h,  v24.8b, v0.8b
>+    umull       v9.8h,  v24.8b, v1.8b
>+    umull       v10.8h, v24.8b, v2.8b
>+    umull       v11.8h, v24.8b, v3.8b
>+    ursra       v8.8h,  v8.8h, #8
>+    ursra       v9.8h,  v9.8h, #8
>+    ursra       v10.8h, v10.8h, #8
>+    ursra       v11.8h, v11.8h, #8
>+.endm
>+
>+.macro pixman_composite_src_n_8_8888_process_pixblock_tail
>+    rshrn       v28.8b, v8.8h, #8
>+    rshrn       v29.8b, v9.8h, #8
>+    rshrn       v30.8b, v10.8h, #8
>+    rshrn       v31.8b, v11.8h, #8
>+.endm
>+
>+.macro pixman_composite_src_n_8_8888_process_pixblock_tail_head
>+    fetch_mask_pixblock
>+                                    PF add PF_X, PF_X, #8
>+        rshrn       v28.8b, v8.8h, #8
>+                                    PF tst PF_CTL, #0x0F
>+        rshrn       v29.8b, v9.8h, #8
>+                                    PF beq 10f
>+                                    PF add PF_X, PF_X, #8
>+10:
>+        rshrn      v30.8b, v10.8h, #8
>+                                    PF beq 10f
>+                                    PF sub PF_CTL, PF_CTL, #1
>+10:
>+        rshrn      v31.8b, v11.8h, #8
>+                                    PF cmp PF_X, ORIG_W
>+    umull          v8.8h, v24.8b, v0.8b
>+                                    PF lsl DUMMY, PF_X, #mask_bpp_shift
>+                                    PF prfm PREFETCH_MODE, [PF_MASK, DUMMY]
>+    umull          v9.8h, v24.8b, v1.8b
>+                                    PF ble 10f
>+                                    PF sub PF_X, PF_X, ORIG_W
>+10:
>+    umull          v10.8h, v24.8b, v2.8b
>+                                    PF ble 10f
>+                                    PF subs PF_CTL, PF_CTL, #0x10
>+10:
>+    umull          v11.8h, v24.8b, v3.8b
>+                                    PF ble 10f
>+                                    PF lsl DUMMY, MASK_STRIDE, #mask_bpp_shift
>+                                    PF ldrsb DUMMY, [PF_MASK, DUMMY]
>+                                    PF add PF_MASK, PF_MASK, #1
>+10:
>+        st4        {v28.8b, v29.8b, v30.8b, v31.8b}, [DST_W], #32
>+    ursra       v8.8h, v8.8h, #8
>+    ursra       v9.8h, v9.8h, #8
>+    ursra       v10.8h, v10.8h, #8
>+    ursra       v11.8h, v11.8h, #8
>+.endm
>+
>+.macro pixman_composite_src_n_8_8888_init
>+    mov         v3.s[0], w4
>+    dup         v0.8b, v3.b[0]
>+    dup         v1.8b, v3.b[1]
>+    dup         v2.8b, v3.b[2]
>+    dup         v3.8b, v3.b[3]
>+.endm
>+
>+.macro pixman_composite_src_n_8_8888_cleanup
>+.endm
>+
>+generate_composite_function \
>+    pixman_composite_src_n_8_8888_asm_neon, 0, 8, 32, \
>+    FLAG_DST_WRITEONLY | FLAG_DEINTERLEAVE_32BPP, \
>+    8, /* number of pixels, processed in a single block */ \
>+    5, /* prefetch distance */ \
>+    pixman_composite_src_n_8_8888_init, \
>+    pixman_composite_src_n_8_8888_cleanup, \
>+    pixman_composite_src_n_8_8888_process_pixblock_head, \
>+    pixman_composite_src_n_8_8888_process_pixblock_tail, \
>+    pixman_composite_src_n_8_8888_process_pixblock_tail_head, \
>+
>+/******************************************************************************/
>+
>+.macro pixman_composite_src_n_8_8_process_pixblock_head
>+    umull       v0.8h, v24.8b, v16.8b
>+    umull       v1.8h, v25.8b, v16.8b
>+    umull       v2.8h, v26.8b, v16.8b
>+    umull       v3.8h, v27.8b, v16.8b
>+    ursra       v0.8h, v0.8h,  #8
>+    ursra       v1.8h, v1.8h,  #8
>+    ursra       v2.8h, v2.8h,  #8
>+    ursra       v3.8h, v3.8h,  #8
>+.endm
>+
>+.macro pixman_composite_src_n_8_8_process_pixblock_tail
>+    rshrn       v28.8b, v0.8h, #8
>+    rshrn       v29.8b, v1.8h, #8
>+    rshrn       v30.8b, v2.8h, #8
>+    rshrn       v31.8b, v3.8h, #8
>+.endm
>+
>+.macro pixman_composite_src_n_8_8_process_pixblock_tail_head
>+    fetch_mask_pixblock
>+                                    PF add PF_X, PF_X, #8
>+        rshrn       v28.8b, v0.8h, #8
>+                                    PF tst PF_CTL, #0x0F
>+        rshrn       v29.8b, v1.8h, #8
>+                                    PF beq 10f
>+                                    PF add PF_X, PF_X, #8
>+10:
>+        rshrn       v30.8b, v2.8h, #8
>+                                    PF beq 10f
>+                                    PF sub PF_CTL, PF_CTL, #1
>+10:
>+        rshrn       v31.8b, v3.8h, #8
>+                                    PF cmp PF_X, ORIG_W
>+    umull       v0.8h,  v24.8b, v16.8b
>+                                    PF lsl DUMMY, PF_X, mask_bpp_shift
>+                                    PF prfm PREFETCH_MODE, [PF_MASK, DUMMY]
>+    umull       v1.8h,  v25.8b, v16.8b
>+                                    PF ble 10f
>+                                    PF sub PF_X, PF_X, ORIG_W
>+10:
>+    umull       v2.8h,  v26.8b, v16.8b
>+                                    PF ble 10f
>+                                    PF subs PF_CTL, PF_CTL, #0x10
>+10:
>+    umull       v3.8h,  v27.8b, v16.8b
>+                                    PF ble 10f
>+                                    PF lsl DUMMY, MASK_STRIDE, #mask_bpp_shift
>+                                    PF ldrsb DUMMY, [PF_MASK, DUMMY]
>+                                    PF add PF_MASK, PF_MASK, #1
>+10:
>+        st1         {v28.8b, v29.8b, v30.8b, v31.8b}, [DST_W], #32
>+    ursra       v0.8h, v0.8h,  #8
>+    ursra       v1.8h, v1.8h,  #8
>+    ursra       v2.8h, v2.8h,  #8
>+    ursra       v3.8h, v3.8h,  #8
>+.endm
>+
>+.macro pixman_composite_src_n_8_8_init
>+    mov         v16.s[0], w4
>+    dup         v16.8b, v16.b[3]
>+.endm
>+
>+.macro pixman_composite_src_n_8_8_cleanup
>+.endm
>+
>+generate_composite_function \
>+    pixman_composite_src_n_8_8_asm_neon, 0, 8, 8, \
>+    FLAG_DST_WRITEONLY, \
>+    32, /* number of pixels, processed in a single block */ \
>+    5, /* prefetch distance */ \
>+    pixman_composite_src_n_8_8_init, \
>+    pixman_composite_src_n_8_8_cleanup, \
>+    pixman_composite_src_n_8_8_process_pixblock_head, \
>+    pixman_composite_src_n_8_8_process_pixblock_tail, \
>+    pixman_composite_src_n_8_8_process_pixblock_tail_head
>+
>+/******************************************************************************/
>+
>+.macro pixman_composite_over_n_8_8888_process_pixblock_head
>+    /* expecting deinterleaved source data in {v8, v9, v10, v11} */
>+    /* v8 - blue, v9 - green, v10 - red, v11 - alpha */
>+    /* and destination data in {v4, v5, v6, v7} */
>+    /* mask is in v24 (v25, v26, v27 are unused) */
>+
>+    /* in */
>+    umull       v12.8h, v24.8b, v8.8b
>+    umull       v13.8h, v24.8b, v9.8b
>+    umull       v14.8h, v24.8b, v10.8b
>+    umull       v15.8h, v24.8b, v11.8b
>+    urshr       v16.8h, v12.8h, #8
>+    urshr       v17.8h, v13.8h, #8
>+    urshr       v18.8h, v14.8h, #8
>+    urshr       v19.8h, v15.8h, #8
>+    raddhn      v0.8b, v12.8h, v16.8h
>+    raddhn      v1.8b, v13.8h, v17.8h
>+    raddhn      v2.8b, v14.8h, v18.8h
>+    raddhn      v3.8b, v15.8h, v19.8h
>+    mvn         v25.8b, v3.8b  /* get inverted alpha */
>+    /* source:      v0 - blue, v1 - green, v2 - red, v3 - alpha */
>+    /* destination: v4 - blue, v5 - green, v6 - red, v7 - alpha */
>+    /* now do alpha blending */
>+    umull       v12.8h, v25.8b, v4.8b
>+    umull       v13.8h, v25.8b, v5.8b
>+    umull       v14.8h, v25.8b, v6.8b
>+    umull       v15.8h, v25.8b, v7.8b
>+.endm
>+
>+.macro pixman_composite_over_n_8_8888_process_pixblock_tail
>+    urshr       v16.8h, v12.8h, #8
>+    urshr       v17.8h, v13.8h, #8
>+    urshr       v18.8h, v14.8h, #8
>+    urshr       v19.8h, v15.8h, #8
>+    raddhn      v28.8b, v16.8h, v12.8h
>+    raddhn      v29.8b, v17.8h, v13.8h
>+    raddhn      v30.8b, v18.8h, v14.8h
>+    raddhn      v31.8b, v19.8h, v15.8h
>+    uqadd       v28.8b, v0.8b, v28.8b
>+    uqadd       v29.8b, v1.8b, v29.8b
>+    uqadd       v30.8b, v2.8b, v30.8b
>+    uqadd       v31.8b, v3.8b, v31.8b
>+.endm
>+
>+.macro pixman_composite_over_n_8_8888_process_pixblock_tail_head
>+        urshr       v16.8h, v12.8h, #8
>+     ld4        {v4.8b, v5.8b, v6.8b, v7.8b}, [DST_R], #32
>+        urshr       v17.8h, v13.8h, #8
>+    fetch_mask_pixblock
>+        urshr       v18.8h, v14.8h, #8
>+                                    PF add PF_X, PF_X, #8
>+        urshr       v19.8h, v15.8h, #8
>+                                    PF tst PF_CTL, #0x0F
>+        raddhn      v28.8b, v16.8h, v12.8h
>+                                    PF beq 10f
>+                                    PF add PF_X, PF_X, #8
>+10:
>+        raddhn      v29.8b, v17.8h, v13.8h
>+                                    PF beq 10f
>+                                    PF sub PF_CTL, PF_CTL, #1
>+10:
>+        raddhn      v30.8b, v18.8h, v14.8h
>+                                    PF cmp PF_X, ORIG_W
>+        raddhn      v31.8b, v19.8h, v15.8h
>+                                    PF lsl DUMMY, PF_X, #dst_bpp_shift
>+                                    PF prfm PREFETCH_MODE, [PF_DST, DUMMY]
>+    umull       v16.8h, v24.8b, v8.8b
>+                                    PF lsl DUMMY, PF_X, #mask_bpp_shift
>+                                    PF prfm PREFETCH_MODE, [PF_MASK, DUMMY]
>+    umull       v17.8h, v24.8b, v9.8b
>+                                    PF ble 10f
>+                                    PF sub PF_X, PF_X, ORIG_W
>+10:
>+    umull       v18.8h, v24.8b, v10.8b
>+                                    PF ble 10f
>+                                    PF subs PF_CTL, PF_CTL, #0x10
>+10:
>+    umull       v19.8h, v24.8b, v11.8b
>+                                    PF ble 10f
>+                                    PF lsl DUMMY, DST_STRIDE, #dst_bpp_shift
>+                                    PF ldrsb DUMMY, [PF_DST, DUMMY]
>+                                    PF add PF_DST, PF_DST, #1
>+10:
>+        uqadd       v28.8b, v0.8b, v28.8b
>+                                    PF ble 10f
>+                                    PF lsl DUMMY, MASK_STRIDE, #mask_bpp_shift
>+                                    PF ldrsb DUMMY, [PF_MASK, DUMMY]
>+                                    PF add PF_MASK, PF_MASK, #1
>+10:
>+        uqadd        v29.8b, v1.8b, v29.8b
>+        uqadd        v30.8b, v2.8b, v30.8b
>+        uqadd        v31.8b, v3.8b, v31.8b
>+    urshr       v12.8h, v16.8h, #8
>+    urshr       v13.8h, v17.8h, #8
>+    urshr       v14.8h, v18.8h, #8
>+    urshr       v15.8h, v19.8h, #8
>+    raddhn      v0.8b, v16.8h, v12.8h
>+    raddhn      v1.8b, v17.8h, v13.8h
>+    raddhn      v2.8b, v18.8h, v14.8h
>+    raddhn      v3.8b, v19.8h, v15.8h
>+        st4          {v28.8b, v29.8b, v30.8b, v31.8b}, [DST_W], #32
>+    mvn         v25.8b, v3.8b
>+    umull       v12.8h, v25.8b, v4.8b
>+    umull       v13.8h, v25.8b, v5.8b
>+    umull       v14.8h, v25.8b, v6.8b
>+    umull       v15.8h, v25.8b, v7.8b
>+.endm
>+
>+.macro pixman_composite_over_n_8_8888_init
>+    mov         v11.s[0], w4
>+    dup         v8.8b, v11.b[0]
>+    dup         v9.8b, v11.b[1]
>+    dup         v10.8b, v11.b[2]
>+    dup         v11.8b, v11.b[3]
>+.endm
>+
>+.macro pixman_composite_over_n_8_8888_cleanup
>+.endm
>+
>+generate_composite_function \
>+    pixman_composite_over_n_8_8888_asm_neon, 0, 8, 32, \
>+    FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
>+    8, /* number of pixels, processed in a single block */ \
>+    5, /* prefetch distance */ \
>+    pixman_composite_over_n_8_8888_init, \
>+    pixman_composite_over_n_8_8888_cleanup, \
>+    pixman_composite_over_n_8_8888_process_pixblock_head, \
>+    pixman_composite_over_n_8_8888_process_pixblock_tail, \
>+    pixman_composite_over_n_8_8888_process_pixblock_tail_head
>+
>+/******************************************************************************/
>+
>+.macro pixman_composite_over_n_8_8_process_pixblock_head
>+    umull       v0.8h,  v24.8b, v8.8b
>+    umull       v1.8h,  v25.8b, v8.8b
>+    umull       v2.8h,  v26.8b, v8.8b
>+    umull       v3.8h,  v27.8b, v8.8b
>+    urshr       v10.8h, v0.8h,  #8
>+    urshr       v11.8h, v1.8h,  #8
>+    urshr       v12.8h, v2.8h,  #8
>+    urshr       v13.8h, v3.8h,  #8
>+    raddhn      v0.8b,  v0.8h,  v10.8h
>+    raddhn      v1.8b,  v1.8h,  v11.8h
>+    raddhn      v2.8b,  v2.8h,  v12.8h
>+    raddhn      v3.8b,  v3.8h,  v13.8h
>+    mvn         v24.8b, v0.8b
>+    mvn         v25.8b, v1.8b
>+    mvn         v26.8b, v2.8b
>+    mvn         v27.8b, v3.8b
>+    umull       v10.8h, v24.8b, v4.8b
>+    umull       v11.8h, v25.8b, v5.8b
>+    umull       v12.8h, v26.8b, v6.8b
>+    umull       v13.8h, v27.8b, v7.8b
>+.endm
>+
>+.macro pixman_composite_over_n_8_8_process_pixblock_tail
>+    urshr       v14.8h, v10.8h,  #8
>+    urshr       v15.8h, v11.8h,  #8
>+    urshr       v16.8h, v12.8h, #8
>+    urshr       v17.8h, v13.8h, #8
>+    raddhn      v28.8b, v14.8h, v10.8h
>+    raddhn      v29.8b, v15.8h, v11.8h
>+    raddhn      v30.8b, v16.8h, v12.8h
>+    raddhn      v31.8b, v17.8h, v13.8h
>+    uqadd       v28.8b, v0.8b,  v28.8b
>+    uqadd       v29.8b, v1.8b,  v29.8b
>+    uqadd       v30.8b, v2.8b,  v30.8b
>+    uqadd       v31.8b, v3.8b,  v31.8b
>+.endm
>+
>+/* TODO: expand macros and do better instructions scheduling */
>+.macro pixman_composite_over_n_8_8_process_pixblock_tail_head
>+    ld1         {v4.8b, v5.8b, v6.8b, v7.8b}, [DST_R], #32
>+    pixman_composite_over_n_8_8_process_pixblock_tail
>+    fetch_mask_pixblock
>+    cache_preload 32, 32
>+    st1         {v28.8b, v29.8b, v30.8b, v31.8b}, [DST_W], #32
>+    pixman_composite_over_n_8_8_process_pixblock_head
>+.endm
>+
>+.macro pixman_composite_over_n_8_8_init
>+    mov         v8.s[0], w4
>+    dup         v8.8b, v8.b[3]
>+.endm
>+
>+.macro pixman_composite_over_n_8_8_cleanup
>+.endm
>+
>+generate_composite_function \
>+    pixman_composite_over_n_8_8_asm_neon, 0, 8, 8, \
>+    FLAG_DST_READWRITE, \
>+    32, /* number of pixels, processed in a single block */ \
>+    5, /* prefetch distance */ \
>+    pixman_composite_over_n_8_8_init, \
>+    pixman_composite_over_n_8_8_cleanup, \
>+    pixman_composite_over_n_8_8_process_pixblock_head, \
>+    pixman_composite_over_n_8_8_process_pixblock_tail, \
>+    pixman_composite_over_n_8_8_process_pixblock_tail_head
>+
>+/******************************************************************************/
>+
>+.macro pixman_composite_over_n_8888_8888_ca_process_pixblock_head
>+    /*
>+     * 'combine_mask_ca' replacement
>+     *
>+     * input:  solid src (n) in {v8,  v9,  v10, v11}
>+     *         dest in          {v4,  v5,  v6,  v7 }
>+     *         mask in          {v24, v25, v26, v27}
>+     * output: updated src in   {v0,  v1,  v2,  v3 }
>+     *         updated mask in  {v24, v25, v26, v3 }
>+     */
>+    umull       v0.8h,  v24.8b, v8.8b
>+    umull       v1.8h,  v25.8b, v9.8b
>+    umull       v2.8h,  v26.8b, v10.8b
>+    umull       v3.8h,  v27.8b, v11.8b
>+    umull       v12.8h, v11.8b, v25.8b
>+    umull       v13.8h, v11.8b, v24.8b
>+    umull       v14.8h, v11.8b, v26.8b
>+    urshr       v15.8h, v0.8h,  #8
>+    urshr       v16.8h, v1.8h,  #8
>+    urshr       v17.8h, v2.8h,  #8
>+    raddhn      v0.8b,  v0.8h,  v15.8h
>+    raddhn      v1.8b,  v1.8h,  v16.8h
>+    raddhn      v2.8b,  v2.8h,  v17.8h
>+    urshr       v15.8h, v13.8h, #8
>+    urshr       v16.8h, v12.8h, #8
>+    urshr       v17.8h, v14.8h, #8
>+    urshr       v18.8h, v3.8h,  #8
>+    raddhn      v24.8b, v13.8h, v15.8h
>+    raddhn      v25.8b, v12.8h, v16.8h
>+    raddhn      v26.8b, v14.8h, v17.8h
>+    raddhn      v3.8b,  v3.8h,  v18.8h
>+    /*
>+     * 'combine_over_ca' replacement
>+     *
>+     * output: updated dest in {v28, v29, v30, v31}
>+     */
>+    mvn         v24.8b, v24.8b
>+    mvn         v25.8b, v25.8b
>+    mvn         v26.8b, v26.8b
>+    mvn         v27.8b, v3.8b
>+    umull       v12.8h, v24.8b, v4.8b
>+    umull       v13.8h, v25.8b, v5.8b
>+    umull       v14.8h, v26.8b, v6.8b
>+    umull       v15.8h, v27.8b, v7.8b
>+.endm
>+
>+.macro pixman_composite_over_n_8888_8888_ca_process_pixblock_tail
>+    /* ... continue 'combine_over_ca' replacement */
>+    urshr       v16.8h, v12.8h, #8
>+    urshr       v17.8h, v13.8h, #8
>+    urshr       v18.8h, v14.8h, #8
>+    urshr       v19.8h, v15.8h, #8
>+    raddhn      v28.8b, v16.8h, v12.8h
>+    raddhn      v29.8b, v17.8h, v13.8h
>+    raddhn      v30.8b, v18.8h, v14.8h
>+    raddhn      v31.8b, v19.8h, v15.8h
>+    uqadd       v28.8b, v0.8b,  v28.8b
>+    uqadd       v29.8b, v1.8b,  v29.8b
>+    uqadd       v30.8b, v2.8b,  v30.8b
>+    uqadd       v31.8b, v3.8b,  v31.8b
>+.endm
>+
>+.macro pixman_composite_over_n_8888_8888_ca_process_pixblock_tail_head
>+        urshr       v16.8h, v12.8h, #8
>+        urshr       v17.8h, v13.8h, #8
>+    ld4         {v4.8b, v5.8b, v6.8b, v7.8b}, [DST_R], #32
>+        urshr       v18.8h, v14.8h, #8
>+        urshr       v19.8h, v15.8h, #8
>+        raddhn      v28.8b, v16.8h, v12.8h
>+        raddhn      v29.8b, v17.8h, v13.8h
>+        raddhn      v30.8b, v18.8h, v14.8h
>+        raddhn      v31.8b, v19.8h, v15.8h
>+    fetch_mask_pixblock
>+        uqadd       v28.8b, v0.8b, v28.8b
>+        uqadd       v29.8b, v1.8b, v29.8b
>+        uqadd       v30.8b, v2.8b, v30.8b
>+        uqadd       v31.8b, v3.8b, v31.8b
>+    cache_preload 8, 8
>+    pixman_composite_over_n_8888_8888_ca_process_pixblock_head
>+    st4         {v28.8b, v29.8b, v30.8b, v31.8b}, [DST_W], #32
>+.endm
>+
>+.macro pixman_composite_over_n_8888_8888_ca_init
>+    mov         v13.s[0], w4
>+    dup         v8.8b, v13.b[0]
>+    dup         v9.8b, v13.b[1]
>+    dup         v10.8b, v13.b[2]
>+    dup         v11.8b, v13.b[3]
>+.endm
>+
>+.macro pixman_composite_over_n_8888_8888_ca_cleanup
>+.endm
>+
>+generate_composite_function \
>+    pixman_composite_over_n_8888_8888_ca_asm_neon, 0, 32, 32, \
>+    FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
>+    8, /* number of pixels, processed in a single block */ \
>+    5, /* prefetch distance */ \
>+    pixman_composite_over_n_8888_8888_ca_init, \
>+    pixman_composite_over_n_8888_8888_ca_cleanup, \
>+    pixman_composite_over_n_8888_8888_ca_process_pixblock_head, \
>+    pixman_composite_over_n_8888_8888_ca_process_pixblock_tail, \
>+    pixman_composite_over_n_8888_8888_ca_process_pixblock_tail_head
>+
>+/******************************************************************************/
>+
>+.macro pixman_composite_over_n_8888_0565_ca_process_pixblock_head
>+    /*
>+     * 'combine_mask_ca' replacement
>+     *
>+     * input:  solid src (n) in {v8,  v9,  v10, v11}  [B, G, R, A]
>+     *         mask in          {v24, v25, v26}       [B, G, R]
>+     * output: updated src in   {v0,  v1,  v2 }       [B, G, R]
>+     *         updated mask in  {v24, v25, v26}       [B, G, R]
>+     */
>+    umull       v0.8h,  v24.8b, v8.8b
>+    umull       v1.8h,  v25.8b, v9.8b
>+    umull       v2.8h,  v26.8b, v10.8b
>+    umull       v12.8h, v11.8b, v24.8b
>+    umull       v13.8h, v11.8b, v25.8b
>+    umull       v14.8h, v11.8b, v26.8b
>+    urshr       v15.8h, v0.8h,  #8
>+    urshr       v16.8h, v1.8h,  #8
>+    urshr       v17.8h, v2.8h,  #8
>+    raddhn      v0.8b,  v0.8h,  v15.8h
>+    raddhn      v1.8b,  v1.8h,  v16.8h
>+    raddhn      v2.8b,  v2.8h,  v17.8h
>+    urshr       v19.8h, v12.8h, #8
>+    urshr       v20.8h, v13.8h, #8
>+    urshr       v21.8h, v14.8h, #8
>+    raddhn      v24.8b, v12.8h, v19.8h
>+    raddhn      v25.8b, v13.8h, v20.8h
>+    /*
>+     * convert 8 r5g6b5 pixel data from {v4} to planar 8-bit format
>+     * and put data into v16 - blue, v17 - green, v18 - red
>+     */
>+       mov         v4.d[1], v5.d[0]
>+       shrn        v17.8b, v4.8h,  #3
>+       shrn        v18.8b, v4.8h,  #8
>+    raddhn      v26.8b, v14.8h, v21.8h
>+       sli         v4.8h,  v4.8h,  #5
>+       sri         v18.8b, v18.8b, #5
>+       sri         v17.8b, v17.8b, #6
>+    /*
>+     * 'combine_over_ca' replacement
>+     *
>+     * output: updated dest in v16 - blue, v17 - green, v18 - red
>+     */
>+    mvn         v24.8b, v24.8b
>+    mvn         v25.8b, v25.8b
>+       shrn       v16.8b, v4.8h,  #2
>+    mvn         v26.8b, v26.8b
>+    umull       v5.8h, v16.8b, v24.8b
>+    umull       v6.8h, v17.8b, v25.8b
>+    umull       v7.8h, v18.8b, v26.8b
>+.endm
>+
>+.macro pixman_composite_over_n_8888_0565_ca_process_pixblock_tail
>+    /* ... continue 'combine_over_ca' replacement */
>+    urshr       v13.8h, v5.8h, #8
>+    urshr       v14.8h, v6.8h, #8
>+    urshr       v15.8h, v7.8h, #8
>+    raddhn      v16.8b, v13.8h, v5.8h
>+    raddhn      v17.8b, v14.8h, v6.8h
>+    raddhn      v18.8b, v15.8h, v7.8h
>+    uqadd       v16.8b, v0.8b, v16.8b
>+    uqadd       v17.8b, v1.8b, v17.8b
>+    uqadd       v18.8b, v2.8b, v18.8b
>+    /*
>+     * convert the results in v16, v17, v18 to r5g6b5 and store
>+     * them into {v14}
>+     */
>+    ushll       v14.8h, v18.8b, #7
>+    sli         v14.8h, v14.8h, #1
>+    ushll       v12.8h, v17.8b, #7
>+    sli         v12.8h, v12.8h, #1
>+    ushll       v13.8h, v16.8b, #7
>+    sli         v13.8h, v13.8h, #1
>+    sri         v14.8h, v12.8h, #5
>+    sri         v14.8h, v13.8h, #11
>+    mov         v28.d[0], v14.d[0]
>+    mov         v29.d[0], v14.d[1]
>+.endm
>+
>+.macro pixman_composite_over_n_8888_0565_ca_process_pixblock_tail_head
>+    fetch_mask_pixblock
>+        urshr       v13.8h, v5.8h, #8
>+        urshr       v14.8h, v6.8h, #8
>+    ld1         {v4.8h}, [DST_R], #16
>+        urshr       v15.8h, v7.8h, #8
>+        raddhn      v16.8b, v13.8h, v5.8h
>+        raddhn      v17.8b, v14.8h, v6.8h
>+        raddhn      v18.8b, v15.8h, v7.8h
>+    mov         v5.d[0], v4.d[1]
>+            /* process_pixblock_head */
>+            /*
>+             * 'combine_mask_ca' replacement
>+             *
>+             * input:  solid src (n) in {v8,  v9,  v10, v11}  [B, G, R, A]
>+             *         mask in          {v24, v25, v26}       [B, G, R]
>+             * output: updated src in   {v0,  v1,  v2 }       [B, G, R]
>+             *         updated mask in  {v24, v25, v26}       [B, G, R]
>+             */
>+        uqadd       v16.8b, v0.8b, v16.8b
>+        uqadd       v17.8b, v1.8b, v17.8b
>+        uqadd       v18.8b, v2.8b, v18.8b
>+            umull       v0.8h,  v24.8b, v8.8b
>+            umull       v1.8h,  v25.8b, v9.8b
>+            umull       v2.8h,  v26.8b, v10.8b
>+        /*
>+         * convert the result in v16, v17, v18 to r5g6b5 and store
>+         * it into {v14}
>+         */
>+        ushll       v14.8h, v18.8b, #7
>+        sli         v14.8h, v14.8h, #1
>+        ushll       v18.8h, v16.8b, #7
>+        sli         v18.8h, v18.8h, #1
>+        ushll       v19.8h, v17.8b, #7
>+        sli         v19.8h, v19.8h, #1
>+            umull       v12.8h, v11.8b, v24.8b
>+        sri         v14.8h, v19.8h, #5
>+            umull       v13.8h, v11.8b, v25.8b
>+            umull       v15.8h, v11.8b, v26.8b
>+        sri         v14.8h, v18.8h, #11
>+        mov         v28.d[0], v14.d[0]
>+        mov         v29.d[0], v14.d[1]
>+    cache_preload 8, 8
>+            urshr       v16.8h, v0.8h,  #8
>+            urshr       v17.8h, v1.8h,  #8
>+            urshr       v18.8h, v2.8h,  #8
>+            raddhn      v0.8b,  v0.8h,  v16.8h
>+            raddhn      v1.8b,  v1.8h,  v17.8h
>+            raddhn      v2.8b,  v2.8h,  v18.8h
>+            urshr       v19.8h, v12.8h, #8
>+            urshr       v20.8h, v13.8h, #8
>+            urshr       v21.8h, v15.8h, #8
>+            raddhn      v24.8b, v12.8h, v19.8h
>+            raddhn      v25.8b, v13.8h, v20.8h
>+                /*
>+                 * convert 8 r5g6b5 pixel data from {v4, v5} to planar
>+             * 8-bit format and put data into v16 - blue, v17 - green,
>+             * v18 - red
>+                 */
>+		mov         v4.d[1], v5.d[0]
>+                shrn        v17.8b, v4.8h,  #3
>+                shrn        v18.8b, v4.8h,  #8
>+            raddhn      v26.8b, v15.8h, v21.8h
>+                sli         v4.8h,  v4.8h,  #5
>+                sri         v17.8b, v17.8b, #6
>+                sri         v18.8b, v18.8b, #5
>+            /*
>+             * 'combine_over_ca' replacement
>+             *
>+             * output: updated dest in v16 - blue, v17 - green, v18 - red
>+             */
>+            mvn         v24.8b, v24.8b
>+            mvn         v25.8b, v25.8b
>+                shrn        v16.8b, v4.8h,  #2
>+            mvn         v26.8b, v26.8b
>+            umull       v5.8h, v16.8b, v24.8b
>+            umull       v6.8h, v17.8b, v25.8b
>+            umull       v7.8h, v18.8b, v26.8b
>+    st1         {v14.8h}, [DST_W], #16
>+.endm
>+
>+.macro pixman_composite_over_n_8888_0565_ca_init
>+    mov         v13.s[0], w4
>+    dup         v8.8b, v13.b[0]
>+    dup         v9.8b, v13.b[1]
>+    dup         v10.8b, v13.b[2]
>+    dup         v11.8b, v13.b[3]
>+.endm
>+
>+.macro pixman_composite_over_n_8888_0565_ca_cleanup
>+.endm
>+
>+generate_composite_function \
>+    pixman_composite_over_n_8888_0565_ca_asm_neon, 0, 32, 16, \
>+    FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
>+    8, /* number of pixels, processed in a single block */ \
>+    5, /* prefetch distance */ \
>+    pixman_composite_over_n_8888_0565_ca_init, \
>+    pixman_composite_over_n_8888_0565_ca_cleanup, \
>+    pixman_composite_over_n_8888_0565_ca_process_pixblock_head, \
>+    pixman_composite_over_n_8888_0565_ca_process_pixblock_tail, \
>+    pixman_composite_over_n_8888_0565_ca_process_pixblock_tail_head
>+
>+/******************************************************************************/
>+
>+.macro pixman_composite_in_n_8_process_pixblock_head
>+    /* expecting source data in {v0, v1, v2, v3} */
>+    /* and destination data in {v4, v5, v6, v7} */
>+    umull       v8.8h,  v4.8b,  v3.8b
>+    umull       v9.8h,  v5.8b,  v3.8b
>+    umull       v10.8h, v6.8b,  v3.8b
>+    umull       v11.8h, v7.8b,  v3.8b
>+.endm
>+
>+.macro pixman_composite_in_n_8_process_pixblock_tail
>+    urshr       v14.8h, v8.8h,  #8
>+    urshr       v15.8h, v9.8h,  #8
>+    urshr       v12.8h, v10.8h, #8
>+    urshr       v13.8h, v11.8h, #8
>+    raddhn      v28.8b, v8.8h,  v14.8h
>+    raddhn      v29.8b, v9.8h,  v15.8h
>+    raddhn      v30.8b, v10.8h, v12.8h
>+    raddhn      v31.8b, v11.8h, v13.8h
>+.endm
>+
>+.macro pixman_composite_in_n_8_process_pixblock_tail_head
>+    pixman_composite_in_n_8_process_pixblock_tail
>+    ld1         {v4.8b, v5.8b, v6.8b, v7.8b}, [DST_R], #32
>+    cache_preload 32, 32
>+    pixman_composite_in_n_8_process_pixblock_head
>+    st1         {v28.8b, v29.8b, v30.8b, v31.8b}, [DST_W], #32
>+.endm
>+
>+.macro pixman_composite_in_n_8_init
>+    mov         v3.s[0], w4
>+    dup         v3.8b, v3.b[3]
>+.endm
>+
>+.macro pixman_composite_in_n_8_cleanup
>+.endm
>+
>+generate_composite_function \
>+    pixman_composite_in_n_8_asm_neon, 0, 0, 8, \
>+    FLAG_DST_READWRITE, \
>+    32, /* number of pixels, processed in a single block */ \
>+    5, /* prefetch distance */ \
>+    pixman_composite_in_n_8_init, \
>+    pixman_composite_in_n_8_cleanup, \
>+    pixman_composite_in_n_8_process_pixblock_head, \
>+    pixman_composite_in_n_8_process_pixblock_tail, \
>+    pixman_composite_in_n_8_process_pixblock_tail_head, \
>+    28, /* dst_w_basereg */ \
>+    4,  /* dst_r_basereg */ \
>+    0,  /* src_basereg   */ \
>+    24  /* mask_basereg  */
>+
>+.macro pixman_composite_add_n_8_8_process_pixblock_head
>+    /* expecting source data in {v8, v9, v10, v11} */
>+    /* v8 - blue, v9 - green, v10 - red, v11 - alpha */
>+    /* and destination data in {v4, v5, v6, v7} */
>+    /* mask is in v24, v25, v26, v27 */
>+    umull       v0.8h, v24.8b, v11.8b
>+    umull       v1.8h, v25.8b, v11.8b
>+    umull       v2.8h, v26.8b, v11.8b
>+    umull       v3.8h, v27.8b, v11.8b
>+    urshr       v12.8h, v0.8h, #8
>+    urshr       v13.8h, v1.8h, #8
>+    urshr       v14.8h, v2.8h, #8
>+    urshr       v15.8h, v3.8h, #8
>+    raddhn      v0.8b, v0.8h, v12.8h
>+    raddhn      v1.8b, v1.8h, v13.8h
>+    raddhn      v2.8b, v2.8h, v14.8h
>+    raddhn      v3.8b, v3.8h, v15.8h
>+    uqadd       v28.8b, v0.8b, v4.8b
>+    uqadd       v29.8b, v1.8b, v5.8b
>+    uqadd       v30.8b, v2.8b, v6.8b
>+    uqadd       v31.8b, v3.8b, v7.8b
>+.endm
>+
>+.macro pixman_composite_add_n_8_8_process_pixblock_tail
>+.endm
>+
>+/* TODO: expand macros and do better instructions scheduling */
>+.macro pixman_composite_add_n_8_8_process_pixblock_tail_head
>+    pixman_composite_add_n_8_8_process_pixblock_tail
>+    st1         {v28.8b, v29.8b, v30.8b, v31.8b}, [DST_W], #32
>+    ld1         {v4.8b, v5.8b, v6.8b, v7.8b}, [DST_R], #32
>+    fetch_mask_pixblock
>+    cache_preload 32, 32
>+    pixman_composite_add_n_8_8_process_pixblock_head
>+.endm
>+
>+.macro pixman_composite_add_n_8_8_init
>+    mov         v11.s[0], w4
>+    dup         v11.8b, v11.b[3]
>+.endm
>+
>+.macro pixman_composite_add_n_8_8_cleanup
>+.endm
>+
>+generate_composite_function \
>+    pixman_composite_add_n_8_8_asm_neon, 0, 8, 8, \
>+    FLAG_DST_READWRITE, \
>+    32, /* number of pixels, processed in a single block */ \
>+    5, /* prefetch distance */ \
>+    pixman_composite_add_n_8_8_init, \
>+    pixman_composite_add_n_8_8_cleanup, \
>+    pixman_composite_add_n_8_8_process_pixblock_head, \
>+    pixman_composite_add_n_8_8_process_pixblock_tail, \
>+    pixman_composite_add_n_8_8_process_pixblock_tail_head
>+
>+/******************************************************************************/
>+
>+.macro pixman_composite_add_8_8_8_process_pixblock_head
>+    /* expecting source data in {v0, v1, v2, v3} */
>+    /* destination data in {v4, v5, v6, v7} */
>+    /* mask in {v24, v25, v26, v27} */
>+    umull       v8.8h, v24.8b, v0.8b
>+    umull       v9.8h, v25.8b, v1.8b
>+    umull       v10.8h, v26.8b, v2.8b
>+    umull       v11.8h, v27.8b, v3.8b
>+    urshr       v0.8h, v8.8h, #8
>+    urshr       v1.8h, v9.8h, #8
>+    urshr       v12.8h, v10.8h, #8
>+    urshr       v13.8h, v11.8h, #8
>+    raddhn      v0.8b, v0.8h, v8.8h
>+    raddhn      v1.8b, v1.8h, v9.8h
>+    raddhn      v2.8b, v12.8h, v10.8h
>+    raddhn      v3.8b, v13.8h, v11.8h
>+    uqadd       v28.8b, v0.8b, v4.8b
>+    uqadd       v29.8b, v1.8b, v5.8b
>+    uqadd       v30.8b, v2.8b, v6.8b
>+    uqadd       v31.8b, v3.8b, v7.8b
>+.endm
>+
>+.macro pixman_composite_add_8_8_8_process_pixblock_tail
>+.endm
>+
>+/* TODO: expand macros and do better instructions scheduling */
>+.macro pixman_composite_add_8_8_8_process_pixblock_tail_head
>+    pixman_composite_add_8_8_8_process_pixblock_tail
>+    st1         {v28.8b, v29.8b, v30.8b, v31.8b}, [DST_W], #32
>+    ld1         {v4.8b, v5.8b, v6.8b, v7.8b}, [DST_R], #32
>+    fetch_mask_pixblock
>+    fetch_src_pixblock
>+    cache_preload 32, 32
>+    pixman_composite_add_8_8_8_process_pixblock_head
>+.endm
>+
>+.macro pixman_composite_add_8_8_8_init
>+.endm
>+
>+.macro pixman_composite_add_8_8_8_cleanup
>+.endm
>+
>+generate_composite_function \
>+    pixman_composite_add_8_8_8_asm_neon, 8, 8, 8, \
>+    FLAG_DST_READWRITE, \
>+    32, /* number of pixels, processed in a single block */ \
>+    5, /* prefetch distance */ \
>+    pixman_composite_add_8_8_8_init, \
>+    pixman_composite_add_8_8_8_cleanup, \
>+    pixman_composite_add_8_8_8_process_pixblock_head, \
>+    pixman_composite_add_8_8_8_process_pixblock_tail, \
>+    pixman_composite_add_8_8_8_process_pixblock_tail_head
>+
>+/******************************************************************************/
>+
>+.macro pixman_composite_add_8888_8888_8888_process_pixblock_head
>+    /* expecting source data in {v0, v1, v2, v3} */
>+    /* destination data in {v4, v5, v6, v7} */
>+    /* mask in {v24, v25, v26, v27} */
>+    umull       v8.8h,  v27.8b, v0.8b
>+    umull       v9.8h,  v27.8b, v1.8b
>+    umull       v10.8h, v27.8b, v2.8b
>+    umull       v11.8h, v27.8b, v3.8b
>+    /* 1 cycle bubble */
>+    ursra       v8.8h,  v8.8h,  #8
>+    ursra       v9.8h,  v9.8h,  #8
>+    ursra       v10.8h, v10.8h, #8
>+    ursra       v11.8h, v11.8h, #8
>+.endm
>+
>+.macro pixman_composite_add_8888_8888_8888_process_pixblock_tail
>+    /* 2 cycle bubble */
>+    rshrn       v28.8b, v8.8h,  #8
>+    rshrn       v29.8b, v9.8h,  #8
>+    rshrn       v30.8b, v10.8h, #8
>+    rshrn       v31.8b, v11.8h, #8
>+    uqadd       v28.8b, v4.8b,  v28.8b
>+    uqadd       v29.8b, v5.8b,  v29.8b
>+    uqadd       v30.8b, v6.8b,  v30.8b
>+    uqadd       v31.8b, v7.8b,  v31.8b
>+.endm
>+
>+.macro pixman_composite_add_8888_8888_8888_process_pixblock_tail_head
>+    fetch_src_pixblock
>+        rshrn       v28.8b, v8.8h,  #8
>+    fetch_mask_pixblock
>+        rshrn       v29.8b, v9.8h,  #8
>+    umull       v8.8h,  v27.8b, v0.8b
>+        rshrn       v30.8b, v10.8h, #8
>+    umull       v9.8h,  v27.8b, v1.8b
>+        rshrn       v31.8b, v11.8h, #8
>+    umull       v10.8h, v27.8b, v2.8b
>+    umull       v11.8h, v27.8b, v3.8b
>+        uqadd       v28.8b, v4.8b,  v28.8b
>+        uqadd       v29.8b, v5.8b,  v29.8b
>+        uqadd       v30.8b, v6.8b,  v30.8b
>+        uqadd       v31.8b, v7.8b,  v31.8b
>+    ursra       v8.8h,  v8.8h,  #8
>+    ld4         {v4.8b, v5.8b, v6.8b, v7.8b}, [DST_R], #32
>+    ursra       v9.8h,  v9.8h,  #8
>+        st4         {v28.8b, v29.8b, v30.8b, v31.8b}, [DST_W], #32
>+    ursra       v10.8h, v10.8h, #8
>+
>+    cache_preload 8, 8
>+
>+    ursra       v11.8h, v11.8h, #8
>+.endm
>+
>+generate_composite_function \
>+    pixman_composite_add_8888_8888_8888_asm_neon, 32, 32, 32, \
>+    FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
>+    8, /* number of pixels, processed in a single block */ \
>+    10, /* prefetch distance */ \
>+    default_init, \
>+    default_cleanup, \
>+    pixman_composite_add_8888_8888_8888_process_pixblock_head, \
>+    pixman_composite_add_8888_8888_8888_process_pixblock_tail, \
>+    pixman_composite_add_8888_8888_8888_process_pixblock_tail_head, \
>+    28, /* dst_w_basereg */ \
>+    4,  /* dst_r_basereg */ \
>+    0,  /* src_basereg   */ \
>+    24  /* mask_basereg  */
>+
>+generate_composite_function_single_scanline \
>+    pixman_composite_scanline_add_mask_asm_neon, 32, 32, 32, \
>+    FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
>+    8, /* number of pixels, processed in a single block */ \
>+    default_init, \
>+    default_cleanup, \
>+    pixman_composite_add_8888_8888_8888_process_pixblock_head, \
>+    pixman_composite_add_8888_8888_8888_process_pixblock_tail, \
>+    pixman_composite_add_8888_8888_8888_process_pixblock_tail_head, \
>+    28, /* dst_w_basereg */ \
>+    4,  /* dst_r_basereg */ \
>+    0,  /* src_basereg   */ \
>+    24  /* mask_basereg  */
>+
>+/******************************************************************************/
>+
>+generate_composite_function \
>+    pixman_composite_add_8888_8_8888_asm_neon, 32, 8, 32, \
>+    FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
>+    8, /* number of pixels, processed in a single block */ \
>+    5, /* prefetch distance */ \
>+    default_init, \
>+    default_cleanup, \
>+    pixman_composite_add_8888_8888_8888_process_pixblock_head, \
>+    pixman_composite_add_8888_8888_8888_process_pixblock_tail, \
>+    pixman_composite_add_8888_8888_8888_process_pixblock_tail_head, \
>+    28, /* dst_w_basereg */ \
>+    4,  /* dst_r_basereg */ \
>+    0,  /* src_basereg   */ \
>+    27  /* mask_basereg  */
>+
>+/******************************************************************************/
>+
>+.macro pixman_composite_add_n_8_8888_init
>+    mov         v3.s[0], w4
>+    dup         v0.8b, v3.b[0]
>+    dup         v1.8b, v3.b[1]
>+    dup         v2.8b, v3.b[2]
>+    dup         v3.8b, v3.b[3]
>+.endm
>+
>+.macro pixman_composite_add_n_8_8888_cleanup
>+.endm
>+
>+generate_composite_function \
>+    pixman_composite_add_n_8_8888_asm_neon, 0, 8, 32, \
>+    FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
>+    8, /* number of pixels, processed in a single block */ \
>+    5, /* prefetch distance */ \
>+    pixman_composite_add_n_8_8888_init, \
>+    pixman_composite_add_n_8_8888_cleanup, \
>+    pixman_composite_add_8888_8888_8888_process_pixblock_head, \
>+    pixman_composite_add_8888_8888_8888_process_pixblock_tail, \
>+    pixman_composite_add_8888_8888_8888_process_pixblock_tail_head, \
>+    28, /* dst_w_basereg */ \
>+    4,  /* dst_r_basereg */ \
>+    0,  /* src_basereg   */ \
>+    27  /* mask_basereg  */
>+
>+/******************************************************************************/
>+
>+.macro pixman_composite_add_8888_n_8888_init
>+    mov         v27.s[0], w6
>+    dup         v27.8b, v27.b[3]
>+.endm
>+
>+.macro pixman_composite_add_8888_n_8888_cleanup
>+.endm
>+
>+generate_composite_function \
>+    pixman_composite_add_8888_n_8888_asm_neon, 32, 0, 32, \
>+    FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
>+    8, /* number of pixels, processed in a single block */ \
>+    5, /* prefetch distance */ \
>+    pixman_composite_add_8888_n_8888_init, \
>+    pixman_composite_add_8888_n_8888_cleanup, \
>+    pixman_composite_add_8888_8888_8888_process_pixblock_head, \
>+    pixman_composite_add_8888_8888_8888_process_pixblock_tail, \
>+    pixman_composite_add_8888_8888_8888_process_pixblock_tail_head, \
>+    28, /* dst_w_basereg */ \
>+    4,  /* dst_r_basereg */ \
>+    0,  /* src_basereg   */ \
>+    27  /* mask_basereg  */
>+
>+/******************************************************************************/
>+
>+.macro pixman_composite_out_reverse_8888_n_8888_process_pixblock_head
>+    /* expecting source data in {v0, v1, v2, v3} */
>+    /* destination data in {v4, v5, v6, v7} */
>+    /* solid mask is in v15 */
>+
>+    /* 'in' */
>+    umull       v11.8h, v15.8b, v3.8b
>+    umull       v10.8h, v15.8b, v2.8b
>+    umull       v9.8h,  v15.8b, v1.8b
>+    umull       v8.8h,  v15.8b, v0.8b
>+    urshr       v16.8h, v11.8h, #8
>+    urshr       v14.8h, v10.8h, #8
>+    urshr       v13.8h,  v9.8h, #8
>+    urshr       v12.8h,  v8.8h, #8
>+    raddhn      v3.8b, v11.8h, v16.8h
>+    raddhn      v2.8b, v10.8h, v14.8h
>+    raddhn      v1.8b,  v9.8h, v13.8h
>+    raddhn      v0.8b,  v8.8h, v12.8h
>+    mvn         v24.8b, v3.8b  /* get inverted alpha */
>+    /* now do alpha blending */
>+    umull       v8.8h, v24.8b, v4.8b
>+    umull       v9.8h, v24.8b, v5.8b
>+    umull       v10.8h, v24.8b, v6.8b
>+    umull       v11.8h, v24.8b, v7.8b
>+.endm
>+
>+.macro pixman_composite_out_reverse_8888_n_8888_process_pixblock_tail
>+    urshr       v16.8h, v8.8h, #8
>+    urshr       v17.8h, v9.8h, #8
>+    urshr       v18.8h, v10.8h, #8
>+    urshr       v19.8h, v11.8h, #8
>+    raddhn      v28.8b, v16.8h, v8.8h
>+    raddhn      v29.8b, v17.8h, v9.8h
>+    raddhn      v30.8b, v18.8h, v10.8h
>+    raddhn      v31.8b, v19.8h, v11.8h
>+.endm
>+
>+/* TODO: expand macros and do better instructions scheduling */
>+.macro pixman_composite_out_reverse_8888_8888_8888_process_pixblock_tail_head
>+    ld4        {v4.8b, v5.8b, v6.8b, v7.8b}, [DST_R], #32
>+    pixman_composite_out_reverse_8888_n_8888_process_pixblock_tail
>+    fetch_src_pixblock
>+    cache_preload 8, 8
>+    fetch_mask_pixblock
>+    pixman_composite_out_reverse_8888_n_8888_process_pixblock_head
>+    st4        {v28.8b, v29.8b, v30.8b, v31.8b}, [DST_W], #32
>+.endm
>+
>+generate_composite_function_single_scanline \
>+    pixman_composite_scanline_out_reverse_mask_asm_neon, 32, 32, 32, \
>+    FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
>+    8, /* number of pixels, processed in a single block */ \
>+    default_init_need_all_regs, \
>+    default_cleanup_need_all_regs, \
>+    pixman_composite_out_reverse_8888_n_8888_process_pixblock_head, \
>+    pixman_composite_out_reverse_8888_n_8888_process_pixblock_tail, \
>+    pixman_composite_out_reverse_8888_8888_8888_process_pixblock_tail_head \
>+    28, /* dst_w_basereg */ \
>+    4,  /* dst_r_basereg */ \
>+    0,  /* src_basereg   */ \
>+    12  /* mask_basereg  */
>+
>+/******************************************************************************/
>+
>+.macro pixman_composite_over_8888_n_8888_process_pixblock_head
>+    pixman_composite_out_reverse_8888_n_8888_process_pixblock_head
>+.endm
>+
>+.macro pixman_composite_over_8888_n_8888_process_pixblock_tail
>+    pixman_composite_out_reverse_8888_n_8888_process_pixblock_tail
>+    uqadd       v28.8b, v0.8b, v28.8b
>+    uqadd       v29.8b, v1.8b, v29.8b
>+    uqadd       v30.8b, v2.8b, v30.8b
>+    uqadd       v31.8b, v3.8b, v31.8b
>+.endm
>+
>+/* TODO: expand macros and do better instructions scheduling */
>+.macro pixman_composite_over_8888_n_8888_process_pixblock_tail_head
>+    ld4        {v4.8b, v5.8b, v6.8b, v7.8b}, [DST_R], #32
>+    pixman_composite_over_8888_n_8888_process_pixblock_tail
>+    fetch_src_pixblock
>+    cache_preload 8, 8
>+    pixman_composite_over_8888_n_8888_process_pixblock_head
>+    st4        {v28.8b, v29.8b, v30.8b, v31.8b}, [DST_W], #32
>+.endm
>+
>+.macro pixman_composite_over_8888_n_8888_init
>+    mov         v15.s[0], w6
>+    dup         v15.8b, v15.b[3]
>+.endm
>+
>+.macro pixman_composite_over_8888_n_8888_cleanup
>+.endm
>+
>+generate_composite_function \
>+    pixman_composite_over_8888_n_8888_asm_neon, 32, 0, 32, \
>+    FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
>+    8, /* number of pixels, processed in a single block */ \
>+    5, /* prefetch distance */ \
>+    pixman_composite_over_8888_n_8888_init, \
>+    pixman_composite_over_8888_n_8888_cleanup, \
>+    pixman_composite_over_8888_n_8888_process_pixblock_head, \
>+    pixman_composite_over_8888_n_8888_process_pixblock_tail, \
>+    pixman_composite_over_8888_n_8888_process_pixblock_tail_head, \
>+    28, /* dst_w_basereg */ \
>+    4,  /* dst_r_basereg */ \
>+    0,  /* src_basereg   */ \
>+    12  /* mask_basereg  */
>+
>+/******************************************************************************/
>+
>+/* TODO: expand macros and do better instructions scheduling */
>+.macro pixman_composite_over_8888_8888_8888_process_pixblock_tail_head
>+    ld4        {v4.8b, v5.8b, v6.8b, v7.8b}, [DST_R], #32
>+    pixman_composite_over_8888_n_8888_process_pixblock_tail
>+    fetch_src_pixblock
>+    cache_preload 8, 8
>+    fetch_mask_pixblock
>+    pixman_composite_over_8888_n_8888_process_pixblock_head
>+    st4        {v28.8b, v29.8b, v30.8b, v31.8b}, [DST_W], #32
>+.endm
>+
>+generate_composite_function \
>+    pixman_composite_over_8888_8888_8888_asm_neon, 32, 32, 32, \
>+    FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
>+    8, /* number of pixels, processed in a single block */ \
>+    5, /* prefetch distance */ \
>+    default_init_need_all_regs, \
>+    default_cleanup_need_all_regs, \
>+    pixman_composite_over_8888_n_8888_process_pixblock_head, \
>+    pixman_composite_over_8888_n_8888_process_pixblock_tail, \
>+    pixman_composite_over_8888_8888_8888_process_pixblock_tail_head \
>+    28, /* dst_w_basereg */ \
>+    4,  /* dst_r_basereg */ \
>+    0,  /* src_basereg   */ \
>+    12  /* mask_basereg  */
>+
>+generate_composite_function_single_scanline \
>+    pixman_composite_scanline_over_mask_asm_neon, 32, 32, 32, \
>+    FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
>+    8, /* number of pixels, processed in a single block */ \
>+    default_init_need_all_regs, \
>+    default_cleanup_need_all_regs, \
>+    pixman_composite_over_8888_n_8888_process_pixblock_head, \
>+    pixman_composite_over_8888_n_8888_process_pixblock_tail, \
>+    pixman_composite_over_8888_8888_8888_process_pixblock_tail_head \
>+    28, /* dst_w_basereg */ \
>+    4,  /* dst_r_basereg */ \
>+    0,  /* src_basereg   */ \
>+    12  /* mask_basereg  */
>+
>+/******************************************************************************/
>+
>+/* TODO: expand macros and do better instructions scheduling */
>+.macro pixman_composite_over_8888_8_8888_process_pixblock_tail_head
>+    ld4        {v4.8b, v5.8b, v6.8b, v7.8b}, [DST_R], #32
>+    pixman_composite_over_8888_n_8888_process_pixblock_tail
>+    fetch_src_pixblock
>+    cache_preload 8, 8
>+    fetch_mask_pixblock
>+    pixman_composite_over_8888_n_8888_process_pixblock_head
>+    st4        {v28.8b, v29.8b, v30.8b, v31.8b}, [DST_W], #32
>+.endm
>+
>+generate_composite_function \
>+    pixman_composite_over_8888_8_8888_asm_neon, 32, 8, 32, \
>+    FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
>+    8, /* number of pixels, processed in a single block */ \
>+    5, /* prefetch distance */ \
>+    default_init_need_all_regs, \
>+    default_cleanup_need_all_regs, \
>+    pixman_composite_over_8888_n_8888_process_pixblock_head, \
>+    pixman_composite_over_8888_n_8888_process_pixblock_tail, \
>+    pixman_composite_over_8888_8_8888_process_pixblock_tail_head \
>+    28, /* dst_w_basereg */ \
>+    4,  /* dst_r_basereg */ \
>+    0,  /* src_basereg   */ \
>+    15  /* mask_basereg  */
>+
>+/******************************************************************************/
>+
>+.macro pixman_composite_src_0888_0888_process_pixblock_head
>+.endm
>+
>+.macro pixman_composite_src_0888_0888_process_pixblock_tail
>+.endm
>+
>+.macro pixman_composite_src_0888_0888_process_pixblock_tail_head
>+    st3     {v0.8b, v1.8b, v2.8b}, [DST_W], #24
>+    fetch_src_pixblock
>+    cache_preload 8, 8
>+.endm
>+
>+generate_composite_function \
>+    pixman_composite_src_0888_0888_asm_neon, 24, 0, 24, \
>+    FLAG_DST_WRITEONLY, \
>+    8, /* number of pixels, processed in a single block */ \
>+    10, /* prefetch distance */ \
>+    default_init, \
>+    default_cleanup, \
>+    pixman_composite_src_0888_0888_process_pixblock_head, \
>+    pixman_composite_src_0888_0888_process_pixblock_tail, \
>+    pixman_composite_src_0888_0888_process_pixblock_tail_head, \
>+    0, /* dst_w_basereg */ \
>+    0, /* dst_r_basereg */ \
>+    0, /* src_basereg   */ \
>+    0  /* mask_basereg  */
>+
>+/******************************************************************************/
>+
>+.macro pixman_composite_src_0888_8888_rev_process_pixblock_head
>+    mov    v31.8b, v2.8b
>+    mov    v2.8b, v0.8b
>+    mov    v0.8b, v31.8b
>+.endm
>+
>+.macro pixman_composite_src_0888_8888_rev_process_pixblock_tail
>+.endm
>+
>+.macro pixman_composite_src_0888_8888_rev_process_pixblock_tail_head
>+    st4    {v0.8b, v1.8b, v2.8b, v3.8b}, [DST_W], #32
>+    fetch_src_pixblock
>+    mov    v31.8b, v2.8b
>+    mov    v2.8b, v0.8b
>+    mov    v0.8b, v31.8b
>+    cache_preload 8, 8
>+.endm
>+
>+.macro pixman_composite_src_0888_8888_rev_init
>+    eor    v3.8b, v3.8b, v3.8b
>+.endm
>+
>+generate_composite_function \
>+    pixman_composite_src_0888_8888_rev_asm_neon, 24, 0, 32, \
>+    FLAG_DST_WRITEONLY | FLAG_DEINTERLEAVE_32BPP, \
>+    8, /* number of pixels, processed in a single block */ \
>+    10, /* prefetch distance */ \
>+    pixman_composite_src_0888_8888_rev_init, \
>+    default_cleanup, \
>+    pixman_composite_src_0888_8888_rev_process_pixblock_head, \
>+    pixman_composite_src_0888_8888_rev_process_pixblock_tail, \
>+    pixman_composite_src_0888_8888_rev_process_pixblock_tail_head, \
>+    0, /* dst_w_basereg */ \
>+    0, /* dst_r_basereg */ \
>+    0, /* src_basereg   */ \
>+    0  /* mask_basereg  */
>+
>+/******************************************************************************/
>+
>+.macro pixman_composite_src_0888_0565_rev_process_pixblock_head
>+    ushll       v8.8h, v1.8b, #7
>+    sli         v8.8h, v8.8h, #1
>+    ushll       v9.8h, v2.8b, #7
>+    sli         v9.8h, v9.8h, #1
>+.endm
>+
>+.macro pixman_composite_src_0888_0565_rev_process_pixblock_tail
>+    ushll       v14.8h, v0.8b, #7
>+    sli         v14.8h, v14.8h, #1
>+    sri         v14.8h, v8.8h, #5
>+    sri         v14.8h, v9.8h, #11
>+    mov         v28.d[0], v14.d[0]
>+    mov         v29.d[0], v14.d[1]
>+.endm
>+
>+.macro pixman_composite_src_0888_0565_rev_process_pixblock_tail_head
>+        ushll       v14.8h, v0.8b, #7
>+        sli         v14.8h, v14.8h, #1
>+    fetch_src_pixblock
>+        sri         v14.8h, v8.8h, #5
>+        sri         v14.8h, v9.8h, #11
>+        mov         v28.d[0], v14.d[0]
>+        mov         v29.d[0], v14.d[1]
>+    ushll       v8.8h, v1.8b, #7
>+    sli         v8.8h, v8.8h, #1
>+        st1     {v14.8h}, [DST_W], #16
>+    ushll       v9.8h, v2.8b, #7
>+    sli         v9.8h, v9.8h, #1
>+.endm
>+
>+generate_composite_function \
>+    pixman_composite_src_0888_0565_rev_asm_neon, 24, 0, 16, \
>+    FLAG_DST_WRITEONLY, \
>+    8, /* number of pixels, processed in a single block */ \
>+    10, /* prefetch distance */ \
>+    default_init, \
>+    default_cleanup, \
>+    pixman_composite_src_0888_0565_rev_process_pixblock_head, \
>+    pixman_composite_src_0888_0565_rev_process_pixblock_tail, \
>+    pixman_composite_src_0888_0565_rev_process_pixblock_tail_head, \
>+    28, /* dst_w_basereg */ \
>+    0, /* dst_r_basereg */ \
>+    0, /* src_basereg   */ \
>+    0  /* mask_basereg  */
>+
>+/******************************************************************************/
>+
>+.macro pixman_composite_src_pixbuf_8888_process_pixblock_head
>+    umull       v8.8h, v3.8b, v0.8b
>+    umull       v9.8h, v3.8b, v1.8b
>+    umull       v10.8h, v3.8b, v2.8b
>+.endm
>+
>+.macro pixman_composite_src_pixbuf_8888_process_pixblock_tail
>+    urshr       v11.8h, v8.8h, #8
>+    mov         v30.8b, v31.8b
>+    mov         v31.8b, v3.8b
>+    mov         v3.8b, v30.8b
>+    urshr       v12.8h, v9.8h, #8
>+    urshr       v13.8h, v10.8h, #8
>+    raddhn      v30.8b, v11.8h, v8.8h
>+    raddhn      v29.8b, v12.8h, v9.8h
>+    raddhn      v28.8b, v13.8h, v10.8h
>+.endm
>+
>+.macro pixman_composite_src_pixbuf_8888_process_pixblock_tail_head
>+        urshr       v11.8h, v8.8h, #8
>+        mov         v30.8b, v31.8b
>+        mov         v31.8b, v3.8b
>+        mov         v3.8b, v31.8b
>+        urshr       v12.8h, v9.8h, #8
>+        urshr       v13.8h, v10.8h, #8
>+    fetch_src_pixblock
>+        raddhn      v30.8b, v11.8h, v8.8h
>+                                    PF add PF_X, PF_X, #8
>+                                    PF tst PF_CTL, #0xF
>+                                    PF beq 10f
>+                                    PF add PF_X, PF_X, #8
>+                                    PF sub PF_CTL, PF_CTL, #1
>+10:
>+        raddhn      v29.8b, v12.8h, v9.8h
>+        raddhn      v28.8b, v13.8h, v10.8h
>+    umull       v8.8h, v3.8b, v0.8b
>+    umull       v9.8h, v3.8b, v1.8b
>+    umull       v10.8h, v3.8b, v2.8b
>+         st4    {v28.8b, v29.8b, v30.8b, v31.8b}, [DST_W], #32
>+                                    PF cmp PF_X, ORIG_W
>+                                    PF lsl DUMMY, PF_X, src_bpp_shift
>+                                    PF prfm PREFETCH_MODE, [PF_SRC, DUMMY]
>+                                    PF ble 10f
>+                                    PF sub PF_X, PF_X, ORIG_W
>+                                    PF subs PF_CTL, PF_CTL, #0x10
>+                                    PF ble 10f
>+                                    PF lsl DUMMY, SRC_STRIDE, #src_bpp_shift
>+                                    PF ldrsb DUMMY, [PF_SRC, DUMMY]
>+                                    PF add PF_SRC, PF_SRC, #1
>+10:
>+.endm
>+
>+generate_composite_function \
>+    pixman_composite_src_pixbuf_8888_asm_neon, 32, 0, 32, \
>+    FLAG_DST_WRITEONLY | FLAG_DEINTERLEAVE_32BPP, \
>+    8, /* number of pixels, processed in a single block */ \
>+    10, /* prefetch distance */ \
>+    default_init, \
>+    default_cleanup, \
>+    pixman_composite_src_pixbuf_8888_process_pixblock_head, \
>+    pixman_composite_src_pixbuf_8888_process_pixblock_tail, \
>+    pixman_composite_src_pixbuf_8888_process_pixblock_tail_head, \
>+    28, /* dst_w_basereg */ \
>+    0, /* dst_r_basereg */ \
>+    0, /* src_basereg   */ \
>+    0  /* mask_basereg  */
>+
>+/******************************************************************************/
>+
>+.macro pixman_composite_src_rpixbuf_8888_process_pixblock_head
>+    umull       v8.8h, v3.8b, v0.8b
>+    umull       v9.8h, v3.8b, v1.8b
>+    umull       v10.8h, v3.8b, v2.8b
>+.endm
>+
>+.macro pixman_composite_src_rpixbuf_8888_process_pixblock_tail
>+    urshr       v11.8h, v8.8h, #8
>+    mov         v30.8b, v31.8b
>+    mov         v31.8b, v3.8b
>+    mov         v3.8b, v30.8b
>+    urshr       v12.8h, v9.8h, #8
>+    urshr       v13.8h, v10.8h, #8
>+    raddhn      v28.8b, v11.8h, v8.8h
>+    raddhn      v29.8b, v12.8h, v9.8h
>+    raddhn      v30.8b, v13.8h, v10.8h
>+.endm
>+
>+.macro pixman_composite_src_rpixbuf_8888_process_pixblock_tail_head
>+        urshr       v11.8h, v8.8h, #8
>+        mov         v30.8b, v31.8b
>+        mov         v31.8b, v3.8b
>+        mov         v3.8b, v30.8b
>+        urshr       v12.8h, v9.8h, #8
>+        urshr       v13.8h, v10.8h, #8
>+    fetch_src_pixblock
>+        raddhn      v28.8b, v11.8h, v8.8h
>+                                    PF add PF_X, PF_X, #8
>+                                    PF tst PF_CTL, #0xF
>+                                    PF beq 10f
>+                                    PF add PF_X, PF_X, #8
>+                                    PF sub PF_CTL, PF_CTL, #1
>+10:
>+        raddhn      v29.8b, v12.8h, v9.8h
>+        raddhn      v30.8b, v13.8h, v10.8h
>+    umull       v8.8h, v3.8b, v0.8b
>+    umull       v9.8h, v3.8b, v1.8b
>+    umull       v10.8h, v3.8b, v2.8b
>+         st4    {v28.8b, v29.8b, v30.8b, v31.8b}, [DST_W], #32
>+                                    PF cmp PF_X, ORIG_W
>+                                    PF lsl DUMMY, PF_X, src_bpp_shift
>+                                    PF prfm PREFETCH_MODE, [PF_SRC, DUMMY]
>+                                    PF ble 10f
>+                                    PF sub PF_X, PF_X, ORIG_W
>+                                    PF subs PF_CTL, PF_CTL, #0x10
>+                                    PF ble 10f
>+                                    PF lsl DUMMY, SRC_STRIDE, #src_bpp_shift
>+                                    PF ldrsb DUMMY, [PF_SRC, DUMMY]
>+                                    PF add PF_SRC, PF_SRC, #1
>+10:
>+.endm
>+
>+generate_composite_function \
>+    pixman_composite_src_rpixbuf_8888_asm_neon, 32, 0, 32, \
>+    FLAG_DST_WRITEONLY | FLAG_DEINTERLEAVE_32BPP, \
>+    8, /* number of pixels, processed in a single block */ \
>+    10, /* prefetch distance */ \
>+    default_init, \
>+    default_cleanup, \
>+    pixman_composite_src_rpixbuf_8888_process_pixblock_head, \
>+    pixman_composite_src_rpixbuf_8888_process_pixblock_tail, \
>+    pixman_composite_src_rpixbuf_8888_process_pixblock_tail_head, \
>+    28, /* dst_w_basereg */ \
>+    0, /* dst_r_basereg */ \
>+    0, /* src_basereg   */ \
>+    0  /* mask_basereg  */
>+
>+/******************************************************************************/
>+
>+.macro pixman_composite_over_0565_8_0565_process_pixblock_head
>+    /* mask is in v15 */
>+    mov         v4.d[0], v8.d[0]
>+    mov         v4.d[1], v9.d[0]
>+    mov         v13.d[0], v10.d[0]
>+    mov         v13.d[1], v11.d[0]
>+    convert_0565_to_x888 v4, v2, v1, v0
>+    convert_0565_to_x888 v13, v6, v5, v4
>+    /* source pixel data is in      {v0, v1, v2, XX} */
>+    /* destination pixel data is in {v4, v5, v6, XX} */
>+    mvn         v7.8b,  v15.8b
>+    umull       v10.8h, v15.8b, v2.8b
>+    umull       v9.8h,  v15.8b, v1.8b
>+    umull       v8.8h,  v15.8b, v0.8b
>+    umull       v11.8h, v7.8b,  v4.8b
>+    umull       v12.8h, v7.8b,  v5.8b
>+    umull       v13.8h, v7.8b,  v6.8b
>+    urshr       v19.8h, v10.8h, #8
>+    urshr       v18.8h, v9.8h,  #8
>+    urshr       v17.8h, v8.8h,  #8
>+    raddhn      v2.8b,  v10.8h, v19.8h
>+    raddhn      v1.8b,  v9.8h,  v18.8h
>+    raddhn      v0.8b,  v8.8h,  v17.8h
>+.endm
>+
>+.macro pixman_composite_over_0565_8_0565_process_pixblock_tail
>+    urshr       v17.8h, v11.8h,  #8
>+    urshr       v18.8h, v12.8h,  #8
>+    urshr       v19.8h, v13.8h,  #8
>+    raddhn      v28.8b, v17.8h, v11.8h
>+    raddhn      v29.8b, v18.8h, v12.8h
>+    raddhn      v30.8b, v19.8h, v13.8h
>+    uqadd       v0.8b,  v0.8b,  v28.8b
>+    uqadd       v1.8b,  v1.8b,  v29.8b
>+    uqadd       v2.8b,  v2.8b,  v30.8b
>+    /* 32bpp result is in {v0, v1, v2, XX} */
>+    convert_8888_to_0565 v2, v1, v0, v14, v30, v13
>+    mov         v28.d[0], v14.d[0]
>+    mov         v29.d[0], v14.d[1]
>+.endm
>+
>+/* TODO: expand macros and do better instructions scheduling */
>+.macro pixman_composite_over_0565_8_0565_process_pixblock_tail_head
>+    fetch_mask_pixblock
>+    pixman_composite_over_0565_8_0565_process_pixblock_tail
>+    fetch_src_pixblock
>+    ld1        {v10.4h, v11.4h}, [DST_R], #16
>+    cache_preload 8, 8
>+    pixman_composite_over_0565_8_0565_process_pixblock_head
>+    st1        {v14.8h}, [DST_W], #16
>+.endm
>+
>+generate_composite_function \
>+    pixman_composite_over_0565_8_0565_asm_neon, 16, 8, 16, \
>+    FLAG_DST_READWRITE, \
>+    8, /* number of pixels, processed in a single block */ \
>+    5, /* prefetch distance */ \
>+    default_init_need_all_regs, \
>+    default_cleanup_need_all_regs, \
>+    pixman_composite_over_0565_8_0565_process_pixblock_head, \
>+    pixman_composite_over_0565_8_0565_process_pixblock_tail, \
>+    pixman_composite_over_0565_8_0565_process_pixblock_tail_head, \
>+    28, /* dst_w_basereg */ \
>+    10,  /* dst_r_basereg */ \
>+    8,  /* src_basereg   */ \
>+    15  /* mask_basereg  */
>+
>+/******************************************************************************/
>+
>+.macro pixman_composite_over_0565_n_0565_init
>+    mov         v15.s[0], w6
>+    dup         v15.8b, v15.b[3]
>+.endm
>+
>+.macro pixman_composite_over_0565_n_0565_cleanup
>+.endm
>+
>+generate_composite_function \
>+    pixman_composite_over_0565_n_0565_asm_neon, 16, 0, 16, \
>+    FLAG_DST_READWRITE, \
>+    8, /* number of pixels, processed in a single block */ \
>+    5, /* prefetch distance */ \
>+    pixman_composite_over_0565_n_0565_init, \
>+    pixman_composite_over_0565_n_0565_cleanup, \
>+    pixman_composite_over_0565_8_0565_process_pixblock_head, \
>+    pixman_composite_over_0565_8_0565_process_pixblock_tail, \
>+    pixman_composite_over_0565_8_0565_process_pixblock_tail_head, \
>+    28, /* dst_w_basereg */ \
>+    10,  /* dst_r_basereg */ \
>+    8,  /* src_basereg   */ \
>+    15  /* mask_basereg  */
>+
>+/******************************************************************************/
>+
>+.macro pixman_composite_add_0565_8_0565_process_pixblock_head
>+    /* mask is in v15 */
>+    mov         v4.d[0], v8.d[0]
>+    mov         v4.d[1], v9.d[0]
>+    mov         v13.d[0], v10.d[0]
>+    mov         v13.d[1], v11.d[0]
>+    convert_0565_to_x888 v4,  v2, v1, v0
>+    convert_0565_to_x888 v13, v6, v5, v4
>+    /* source pixel data is in      {v0, v1, v2, XX} */
>+    /* destination pixel data is in {v4, v5, v6, XX} */
>+    umull       v9.8h,  v15.8b, v2.8b
>+    umull       v8.8h,  v15.8b, v1.8b
>+    umull       v7.8h,  v15.8b, v0.8b
>+    urshr       v12.8h, v9.8h,  #8
>+    urshr       v11.8h, v8.8h,  #8
>+    urshr       v10.8h, v7.8h,  #8
>+    raddhn      v2.8b,  v9.8h,  v12.8h
>+    raddhn      v1.8b,  v8.8h,  v11.8h
>+    raddhn      v0.8b,  v7.8h,  v10.8h
>+.endm
>+
>+.macro pixman_composite_add_0565_8_0565_process_pixblock_tail
>+    uqadd       v0.8b,  v0.8b,  v4.8b
>+    uqadd       v1.8b,  v1.8b,  v5.8b
>+    uqadd       v2.8b,  v2.8b,  v6.8b
>+    /* 32bpp result is in {v0, v1, v2, XX} */
>+    convert_8888_to_0565 v2, v1, v0, v14, v30, v13
>+    mov         v28.d[0], v14.d[0]
>+    mov         v29.d[0], v14.d[1]
>+.endm
>+
>+/* TODO: expand macros and do better instructions scheduling */
>+.macro pixman_composite_add_0565_8_0565_process_pixblock_tail_head
>+    fetch_mask_pixblock
>+    pixman_composite_add_0565_8_0565_process_pixblock_tail
>+    fetch_src_pixblock
>+    ld1        {v10.4h, v11.4h}, [DST_R], #16
>+    cache_preload 8, 8
>+    pixman_composite_add_0565_8_0565_process_pixblock_head
>+    st1        {v14.8h}, [DST_W], #16
>+.endm
>+
>+generate_composite_function \
>+    pixman_composite_add_0565_8_0565_asm_neon, 16, 8, 16, \
>+    FLAG_DST_READWRITE, \
>+    8, /* number of pixels, processed in a single block */ \
>+    5, /* prefetch distance */ \
>+    default_init_need_all_regs, \
>+    default_cleanup_need_all_regs, \
>+    pixman_composite_add_0565_8_0565_process_pixblock_head, \
>+    pixman_composite_add_0565_8_0565_process_pixblock_tail, \
>+    pixman_composite_add_0565_8_0565_process_pixblock_tail_head, \
>+    28, /* dst_w_basereg */ \
>+    10, /* dst_r_basereg */ \
>+    8,  /* src_basereg   */ \
>+    15  /* mask_basereg  */
>+
>+/******************************************************************************/
>+
>+.macro pixman_composite_out_reverse_8_0565_process_pixblock_head
>+    /* mask is in v15 */
>+    mov         v12.d[0], v10.d[0]
>+    mov         v12.d[1], v11.d[0]
>+    convert_0565_to_x888 v12, v6, v5, v4
>+    /* destination pixel data is in {v4, v5, v6, xx} */
>+    mvn         v24.8b, v15.8b /* get inverted alpha */
>+    /* now do alpha blending */
>+    umull       v8.8h,  v24.8b, v4.8b
>+    umull       v9.8h,  v24.8b, v5.8b
>+    umull       v10.8h, v24.8b, v6.8b
>+.endm
>+
>+.macro pixman_composite_out_reverse_8_0565_process_pixblock_tail
>+    urshr       v11.8h, v8.8h, #8
>+    urshr       v12.8h, v9.8h, #8
>+    urshr       v13.8h, v10.8h, #8
>+    raddhn      v0.8b, v11.8h, v8.8h
>+    raddhn      v1.8b, v12.8h, v9.8h
>+    raddhn      v2.8b, v13.8h, v10.8h
>+    /* 32bpp result is in {v0, v1, v2, XX} */
>+    convert_8888_to_0565 v2, v1, v0, v14, v12, v3
>+    mov         v28.d[0], v14.d[0]
>+    mov         v29.d[0], v14.d[1]
>+.endm
>+
>+/* TODO: expand macros and do better instructions scheduling */
>+.macro pixman_composite_out_reverse_8_0565_process_pixblock_tail_head
>+    fetch_src_pixblock
>+    pixman_composite_out_reverse_8_0565_process_pixblock_tail
>+    ld1        {v10.4h, v11.4h}, [DST_R], #16
>+    cache_preload 8, 8
>+    pixman_composite_out_reverse_8_0565_process_pixblock_head
>+    st1        {v14.8h}, [DST_W], #16
>+.endm
>+
>+generate_composite_function \
>+    pixman_composite_out_reverse_8_0565_asm_neon, 8, 0, 16, \
>+    FLAG_DST_READWRITE, \
>+    8, /* number of pixels, processed in a single block */ \
>+    5, /* prefetch distance */ \
>+    default_init_need_all_regs, \
>+    default_cleanup_need_all_regs, \
>+    pixman_composite_out_reverse_8_0565_process_pixblock_head, \
>+    pixman_composite_out_reverse_8_0565_process_pixblock_tail, \
>+    pixman_composite_out_reverse_8_0565_process_pixblock_tail_head, \
>+    28, /* dst_w_basereg */ \
>+    10,  /* dst_r_basereg */ \
>+    15, /* src_basereg   */ \
>+    0   /* mask_basereg  */
>+
>+/******************************************************************************/
>+
>+.macro pixman_composite_out_reverse_8_8888_process_pixblock_head
>+    /* src is in v0 */
>+    /* destination pixel data is in {v4, v5, v6, v7} */
>+    mvn         v1.8b, v0.8b /* get inverted alpha */
>+    /* now do alpha blending */
>+    umull       v8.8h, v1.8b, v4.8b
>+    umull       v9.8h, v1.8b, v5.8b
>+    umull       v10.8h, v1.8b, v6.8b
>+    umull       v11.8h, v1.8b, v7.8b
>+.endm
>+
>+.macro pixman_composite_out_reverse_8_8888_process_pixblock_tail
>+    urshr       v14.8h, v8.8h, #8
>+    urshr       v15.8h, v9.8h, #8
>+    urshr       v12.8h, v10.8h, #8
>+    urshr       v13.8h, v11.8h, #8
>+    raddhn      v28.8b, v14.8h, v8.8h
>+    raddhn      v29.8b, v15.8h, v9.8h
>+    raddhn      v30.8b, v12.8h, v10.8h
>+    raddhn      v31.8b, v13.8h, v11.8h
>+    /* 32bpp result is in {v28, v29, v30, v31} */
>+.endm
>+
>+/* TODO: expand macros and do better instructions scheduling */
>+.macro pixman_composite_out_reverse_8_8888_process_pixblock_tail_head
>+    fetch_src_pixblock
>+    pixman_composite_out_reverse_8_8888_process_pixblock_tail
>+    ld4       {v4.8b, v5.8b, v6.8b, v7.8b}, [DST_R], #32
>+    cache_preload 8, 8
>+    pixman_composite_out_reverse_8_8888_process_pixblock_head
>+    st4       {v28.8b, v29.8b, v30.8b, v31.8b}, [DST_W], #32
>+.endm
>+
>+generate_composite_function \
>+    pixman_composite_out_reverse_8_8888_asm_neon, 8, 0, 32, \
>+    FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
>+    8, /* number of pixels, processed in a single block */ \
>+    5, /* prefetch distance */ \
>+    default_init, \
>+    default_cleanup, \
>+    pixman_composite_out_reverse_8_8888_process_pixblock_head, \
>+    pixman_composite_out_reverse_8_8888_process_pixblock_tail, \
>+    pixman_composite_out_reverse_8_8888_process_pixblock_tail_head, \
>+    28, /* dst_w_basereg */ \
>+    4, /* dst_r_basereg */ \
>+    0, /* src_basereg   */ \
>+    0   /* mask_basereg  */
>+    
>+/******************************************************************************/
>+
>+generate_composite_function_nearest_scanline \
>+    pixman_scaled_nearest_scanline_8888_8888_OVER_asm_neon, 32, 0, 32, \
>+    FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
>+    8, /* number of pixels, processed in a single block */ \
>+    default_init, \
>+    default_cleanup, \
>+    pixman_composite_over_8888_8888_process_pixblock_head, \
>+    pixman_composite_over_8888_8888_process_pixblock_tail, \
>+    pixman_composite_over_8888_8888_process_pixblock_tail_head
>+
>+generate_composite_function_nearest_scanline \
>+    pixman_scaled_nearest_scanline_8888_0565_OVER_asm_neon, 32, 0, 16, \
>+    FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
>+    8, /* number of pixels, processed in a single block */ \
>+    default_init, \
>+    default_cleanup, \
>+    pixman_composite_over_8888_0565_process_pixblock_head, \
>+    pixman_composite_over_8888_0565_process_pixblock_tail, \
>+    pixman_composite_over_8888_0565_process_pixblock_tail_head, \
>+    28, /* dst_w_basereg */ \
>+    4,  /* dst_r_basereg */ \
>+    0,  /* src_basereg   */ \
>+    24  /* mask_basereg  */
>+
>+generate_composite_function_nearest_scanline \
>+    pixman_scaled_nearest_scanline_8888_0565_SRC_asm_neon, 32, 0, 16, \
>+    FLAG_DST_WRITEONLY | FLAG_DEINTERLEAVE_32BPP, \
>+    8, /* number of pixels, processed in a single block */ \
>+    default_init, \
>+    default_cleanup, \
>+    pixman_composite_src_8888_0565_process_pixblock_head, \
>+    pixman_composite_src_8888_0565_process_pixblock_tail, \
>+    pixman_composite_src_8888_0565_process_pixblock_tail_head, \
>+    
>+generate_composite_function_nearest_scanline \
>+    pixman_scaled_nearest_scanline_0565_8888_SRC_asm_neon, 16, 0, 32, \
>+    FLAG_DST_WRITEONLY | FLAG_DEINTERLEAVE_32BPP, \
>+    8, /* number of pixels, processed in a single block */ \
>+    default_init, \
>+    default_cleanup, \
>+    pixman_composite_src_0565_8888_process_pixblock_head, \
>+    pixman_composite_src_0565_8888_process_pixblock_tail, \
>+    pixman_composite_src_0565_8888_process_pixblock_tail_head
>+
>+generate_composite_function_nearest_scanline \
>+    pixman_scaled_nearest_scanline_8888_8_0565_OVER_asm_neon, 32, 8, 16, \
>+    FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
>+    8, /* number of pixels, processed in a single block */ \
>+    default_init_need_all_regs, \
>+    default_cleanup_need_all_regs, \
>+    pixman_composite_over_8888_8_0565_process_pixblock_head, \
>+    pixman_composite_over_8888_8_0565_process_pixblock_tail, \
>+    pixman_composite_over_8888_8_0565_process_pixblock_tail_head, \
>+    28, /* dst_w_basereg */ \
>+    4,  /* dst_r_basereg */ \
>+    8,  /* src_basereg   */ \
>+    24  /* mask_basereg  */
>+
>+generate_composite_function_nearest_scanline \
>+    pixman_scaled_nearest_scanline_0565_8_0565_OVER_asm_neon, 16, 8, 16, \
>+    FLAG_DST_READWRITE, \
>+    8, /* number of pixels, processed in a single block */ \
>+    default_init_need_all_regs, \
>+    default_cleanup_need_all_regs, \
>+    pixman_composite_over_0565_8_0565_process_pixblock_head, \
>+    pixman_composite_over_0565_8_0565_process_pixblock_tail, \
>+    pixman_composite_over_0565_8_0565_process_pixblock_tail_head, \
>+    28, /* dst_w_basereg */ \
>+    10,  /* dst_r_basereg */ \
>+    8,  /* src_basereg   */ \
>+    15  /* mask_basereg  */
>+
>+/******************************************************************************/
>+
>+/*
>+ * Bilinear scaling support code which tries to provide pixel fetching, color
>+ * format conversion, and interpolation as separate macros which can be used
>+ * as the basic building blocks for constructing bilinear scanline functions.
>+ */
>+
>+.macro bilinear_load_8888 reg1, reg2, tmp
>+    asr       TMP1, X, #16
>+    add       X, X, UX
>+    lsl       TMP2, TMP1, #2
>+    add       TMP1, TOP, TMP2
>+    ld1       {&reg1&.2s}, [TMP1], STRIDE
>+    ld1       {&reg2&.2s}, [TMP1]
>+.endm
>+
>+.macro bilinear_load_0565 reg1, reg2, tmp
>+    asr       TMP1, X, #16
>+    add       X, X, UX
>+    lsl       TMP2, TMP1, #1
>+    add       TMP1, TOP, TMP2
>+    ld1       {&reg2&.s}[0], [TMP1], STRIDE
>+    ld1       {&reg2&.s}[1], [TMP1]
>+    convert_four_0565_to_x888_packed reg2, reg1, reg2, tmp
>+.endm
>+
>+.macro bilinear_load_and_vertical_interpolate_two_8888 \
>+                    acc1, acc2, reg1, reg2, reg3, reg4, tmp1, tmp2
>+
>+    bilinear_load_8888 reg1, reg2, tmp1
>+    umull     &acc1&.8h, &reg1&.8b, v28.8b
>+    umlal     &acc1&.8h, &reg2&.8b, v29.8b
>+    bilinear_load_8888 reg3, reg4, tmp2
>+    umull     &acc2&.8h, &reg3&.8b, v28.8b
>+    umlal     &acc2&.8h, &reg4&.8b, v29.8b
>+.endm
>+
>+.macro bilinear_load_and_vertical_interpolate_four_8888 \
>+                xacc1, xacc2, xreg1, xreg2, xreg3, xreg4, xacc2lo, xacc2hi \
>+                yacc1, yacc2, yreg1, yreg2, yreg3, yreg4, yacc2lo, yacc2hi
>+
>+    bilinear_load_and_vertical_interpolate_two_8888 \
>+                xacc1, xacc2, xreg1, xreg2, xreg3, xreg4, xacc2lo, xacc2hi
>+    bilinear_load_and_vertical_interpolate_two_8888 \
>+                yacc1, yacc2, yreg1, yreg2, yreg3, yreg4, yacc2lo, yacc2hi
>+.endm
>+
>+.macro vzip reg1, reg2
>+    umov      TMP4, v31.d[0]
>+    zip1      v31.8b, reg1, reg2
>+    zip2      reg2,   reg1, reg2
>+    mov       reg1,   v31.8b
>+    mov       v31.d[0], TMP4
>+.endm
>+
>+.macro vuzp reg1, reg2
>+    umov      TMP4, v31.d[0]
>+    uzp1      v31.8b, reg1, reg2
>+    uzp2      reg2,   reg1, reg2
>+    mov       reg1,   v31.8b
>+    mov       v31.d[0], TMP4
>+.endm
>+
>+.macro bilinear_load_and_vertical_interpolate_two_0565 \
>+                acc1, acc2, reg1, reg2, reg3, reg4, acc2lo, acc2hi
>+    asr       TMP1, X, #16
>+    add       X, X, UX
>+    add       TMP1, TOP, TMP1, lsl #1
>+    asr       TMP2, X, #16
>+    add       X, X, UX
>+    add       TMP2, TOP, TMP2, lsl #1
>+    ld1       {&acc2&.s}[0], [TMP1], STRIDE
>+    ld1       {&acc2&.s}[2], [TMP2], STRIDE
>+    ld1       {&acc2&.s}[1], [TMP1]
>+    ld1       {&acc2&.s}[3], [TMP2]
>+    convert_0565_to_x888 acc2, reg3, reg2, reg1
>+    vzip      &reg1&.8b, &reg3&.8b
>+    vzip      &reg2&.8b, &reg4&.8b
>+    vzip      &reg3&.8b, &reg4&.8b
>+    vzip      &reg1&.8b, &reg2&.8b
>+    umull     &acc1&.8h, &reg1&.8b, v28.8b
>+    umlal     &acc1&.8h, &reg2&.8b, v29.8b
>+    umull     &acc2&.8h, &reg3&.8b, v28.8b
>+    umlal     &acc2&.8h, &reg4&.8b, v29.8b
>+.endm
>+
>+.macro bilinear_load_and_vertical_interpolate_four_0565 \
>+                xacc1, xacc2, xreg1, xreg2, xreg3, xreg4, xacc2lo, xacc2hi \
>+                yacc1, yacc2, yreg1, yreg2, yreg3, yreg4, yacc2lo, yacc2hi
>+    asr       TMP1, X, #16
>+    add       X, X, UX
>+    add       TMP1, TOP, TMP1, lsl #1
>+    asr       TMP2, X, #16
>+    add       X, X, UX
>+    add       TMP2, TOP, TMP2, lsl #1
>+    ld1       {&xacc2&.s}[0], [TMP1], STRIDE
>+    ld1       {&xacc2&.s}[2], [TMP2], STRIDE
>+    ld1       {&xacc2&.s}[1], [TMP1]
>+    ld1       {&xacc2&.s}[3], [TMP2]
>+    convert_0565_to_x888 xacc2, xreg3, xreg2, xreg1
>+    asr       TMP1, X, #16
>+    add       X, X, UX
>+    add       TMP1, TOP, TMP1, lsl #1
>+    asr       TMP2, X, #16
>+    add       X, X, UX
>+    add       TMP2, TOP, TMP2, lsl #1
>+    ld1       {&yacc2&.s}[0], [TMP1], STRIDE
>+    vzip      &xreg1&.8b, &xreg3&.8b
>+    ld1       {&yacc2&.s}[2], [TMP2], STRIDE
>+    vzip      &xreg2&.8b, &xreg4&.8b
>+    ld1       {&yacc2&.s}[1], [TMP1]
>+    vzip      &xreg3&.8b, &xreg4&.8b
>+    ld1       {&yacc2&.s}[3], [TMP2]
>+    vzip      &xreg1&.8b, &xreg2&.8b
>+    convert_0565_to_x888 yacc2, yreg3, yreg2, yreg1
>+    umull     &xacc1&.8h, &xreg1&.8b, v28.8b
>+    vzip      &yreg1&.8b, &yreg3&.8b
>+    umlal     &xacc1&.8h, &xreg2&.8b, v29.8b
>+    vzip      &yreg2&.8b, &yreg4&.8b
>+    umull     &xacc2&.8h, &xreg3&.8b, v28.8b
>+    vzip      &yreg3&.8b, &yreg4&.8b
>+    umlal     &xacc2&.8h, &xreg4&.8b, v29.8b
>+    vzip      &yreg1&.8b, &yreg2&.8b
>+    umull     &yacc1&.8h, &yreg1&.8b, v28.8b
>+    umlal     &yacc1&.8h, &yreg2&.8b, v29.8b
>+    umull     &yacc2&.8h, &yreg3&.8b, v28.8b
>+    umlal     &yacc2&.8h, &yreg4&.8b, v29.8b
>+.endm
>+
>+.macro bilinear_store_8888 numpix, tmp1, tmp2
>+.if numpix == 4
>+    st1       {v0.2s, v1.2s}, [OUT], #16
>+.elseif numpix == 2
>+    st1       {v0.2s}, [OUT], #8
>+.elseif numpix == 1
>+    st1       {v0.s}[0], [OUT], #4
>+.else
>+    .error bilinear_store_8888 numpix is unsupported
>+.endif
>+.endm
>+
>+.macro bilinear_store_0565 numpix, tmp1, tmp2
>+    vuzp      v0.8b, v1.8b
>+    vuzp      v2.8b, v3.8b
>+    vuzp      v1.8b, v3.8b
>+    vuzp      v0.8b, v2.8b
>+    convert_8888_to_0565 v2, v1, v0, v1, tmp1, tmp2
>+.if numpix == 4
>+    st1       {v1.4h}, [OUT], #8
>+.elseif numpix == 2
>+    st1       {v1.s}[0], [OUT], #4
>+.elseif numpix == 1
>+    st1       {v1.h}[0], [OUT], #2
>+.else
>+    .error bilinear_store_0565 numpix is unsupported
>+.endif
>+.endm
>+
>+.macro bilinear_interpolate_last_pixel src_fmt, dst_fmt
>+    bilinear_load_&src_fmt v0, v1, v2
>+    umull     v2.8h, v0.8b, v28.8b
>+    umlal     v2.8h, v1.8b, v29.8b
>+    /* 5 cycles bubble */
>+    ushll     v0.4s, v2.4h, #BILINEAR_INTERPOLATION_BITS
>+    umlsl     v0.4s, v2.4h, v15.h[0]
>+    umlal2    v0.4s, v2.8h, v15.h[0]
>+    /* 5 cycles bubble */
>+    shrn      v0.4h, v0.4s, #(2 * BILINEAR_INTERPOLATION_BITS)
>+    /* 3 cycles bubble */
>+    xtn       v0.8b, v0.8h
>+    /* 1 cycle bubble */
>+    bilinear_store_&dst_fmt 1, v3, v4
>+.endm
>+
>+.macro bilinear_interpolate_two_pixels src_fmt, dst_fmt
>+    bilinear_load_and_vertical_interpolate_two_&src_fmt \
>+                v1, v11, v2, v3, v20, v21, v22, v23
>+    ushll     v0.4s, v1.4h, #BILINEAR_INTERPOLATION_BITS
>+    umlsl     v0.4s, v1.4h, v15.h[0]
>+    umlal2    v0.4s, v1.8h, v15.h[0]
>+    ushll     v10.4s, v11.4h, #BILINEAR_INTERPOLATION_BITS
>+    umlsl     v10.4s, v11.4h, v15.h[4]
>+    umlal2    v10.4s, v11.8h, v15.h[4]
>+    shrn      v0.4h, v0.4s, #(2 * BILINEAR_INTERPOLATION_BITS)
>+    shrn2     v0.8h, v10.4s, #(2 * BILINEAR_INTERPOLATION_BITS)
>+    ushr      v15.8h, v12.8h, #(16 - BILINEAR_INTERPOLATION_BITS)
>+    add       v12.8h, v12.8h, v13.8h
>+    xtn       v0.8b, v0.8h
>+    bilinear_store_&dst_fmt 2, v3, v4
>+.endm
>+
>+.macro bilinear_interpolate_four_pixels src_fmt, dst_fmt
>+    bilinear_load_and_vertical_interpolate_four_&src_fmt \
>+                v1, v11, v14, v20, v16, v17, v22, v23 \
>+                v3, v9,  v24, v25, v26, v27, v18, v19
>+    prfm      PREFETCH_MODE, [TMP1, PF_OFFS]
>+    sub       TMP1, TMP1, STRIDE
>+    ushll     v0.4s, v1.4h, #BILINEAR_INTERPOLATION_BITS
>+    umlsl     v0.4s, v1.4h, v15.h[0]
>+    umlal2    v0.4s, v1.8h, v15.h[0]
>+    ushll     v10.4s, v11.4h, #BILINEAR_INTERPOLATION_BITS
>+    umlsl     v10.4s, v11.4h, v15.h[4]
>+    umlal2    v10.4s, v11.8h, v15.h[4]
>+    ushr      v15.8h, v12.8h, #(16 - BILINEAR_INTERPOLATION_BITS)
>+    ushll     v2.4s, v3.4h, #BILINEAR_INTERPOLATION_BITS
>+    umlsl     v2.4s, v3.4h, v15.h[0]
>+    umlal2    v2.4s, v3.8h, v15.h[0]
>+    ushll     v8.4s, v9.4h, #BILINEAR_INTERPOLATION_BITS
>+    prfm      PREFETCH_MODE, [TMP2, PF_OFFS]
>+    umlsl     v8.4s, v9.4h, v15.h[4]
>+    umlal2    v8.4s, v9.8h, v15.h[4]
>+    add       v12.8h, v12.8h, v13.8h
>+    shrn      v0.4h, v0.4s, #(2 * BILINEAR_INTERPOLATION_BITS)
>+    shrn2     v0.8h, v10.4s, #(2 * BILINEAR_INTERPOLATION_BITS)
>+    shrn      v2.4h, v2.4s, #(2 * BILINEAR_INTERPOLATION_BITS)
>+    shrn2     v2.8h, v8.4s, #(2 * BILINEAR_INTERPOLATION_BITS)
>+    ushr      v15.8h, v12.8h, #(16 - BILINEAR_INTERPOLATION_BITS)
>+    xtn       v0.8b, v0.8h
>+    xtn       v1.8b, v2.8h
>+    add       v12.8h, v12.8h, v13.8h
>+    bilinear_store_&dst_fmt 4, v3, v4
>+.endm
>+
>+.macro bilinear_interpolate_four_pixels_head src_fmt, dst_fmt
>+.ifdef have_bilinear_interpolate_four_pixels_&src_fmt&_&dst_fmt
>+    bilinear_interpolate_four_pixels_&src_fmt&_&dst_fmt&_head
>+.else
>+    bilinear_interpolate_four_pixels src_fmt, dst_fmt
>+.endif
>+.endm
>+
>+.macro bilinear_interpolate_four_pixels_tail src_fmt, dst_fmt
>+.ifdef have_bilinear_interpolate_four_pixels_&src_fmt&_&dst_fmt
>+    bilinear_interpolate_four_pixels_&src_fmt&_&dst_fmt&_tail
>+.endif
>+.endm
>+
>+.macro bilinear_interpolate_four_pixels_tail_head src_fmt, dst_fmt
>+.ifdef have_bilinear_interpolate_four_pixels_&src_fmt&_&dst_fmt
>+    bilinear_interpolate_four_pixels_&src_fmt&_&dst_fmt&_tail_head
>+.else
>+    bilinear_interpolate_four_pixels src_fmt, dst_fmt
>+.endif
>+.endm
>+
>+.macro bilinear_interpolate_eight_pixels_head src_fmt, dst_fmt
>+.ifdef have_bilinear_interpolate_eight_pixels_&src_fmt&_&dst_fmt
>+    bilinear_interpolate_eight_pixels_&src_fmt&_&dst_fmt&_head
>+.else
>+    bilinear_interpolate_four_pixels_head src_fmt, dst_fmt
>+    bilinear_interpolate_four_pixels_tail_head src_fmt, dst_fmt
>+.endif
>+.endm
>+
>+.macro bilinear_interpolate_eight_pixels_tail src_fmt, dst_fmt
>+.ifdef have_bilinear_interpolate_eight_pixels_&src_fmt&_&dst_fmt
>+    bilinear_interpolate_eight_pixels_&src_fmt&_&dst_fmt&_tail
>+.else
>+    bilinear_interpolate_four_pixels_tail src_fmt, dst_fmt
>+.endif
>+.endm
>+
>+.macro bilinear_interpolate_eight_pixels_tail_head src_fmt, dst_fmt
>+.ifdef have_bilinear_interpolate_eight_pixels_&src_fmt&_&dst_fmt
>+    bilinear_interpolate_eight_pixels_&src_fmt&_&dst_fmt&_tail_head
>+.else
>+    bilinear_interpolate_four_pixels_tail_head src_fmt, dst_fmt
>+    bilinear_interpolate_four_pixels_tail_head src_fmt, dst_fmt
>+.endif
>+.endm
>+
>+.set BILINEAR_FLAG_UNROLL_4,          0
>+.set BILINEAR_FLAG_UNROLL_8,          1
>+.set BILINEAR_FLAG_USE_ALL_NEON_REGS, 2
>+
>+/*
>+ * Main template macro for generating NEON optimized bilinear scanline
>+ * functions.
>+ *
>+ * Bilinear scanline scaler macro template uses the following arguments:
>+ *  fname             - name of the function to generate
>+ *  src_fmt           - source color format (8888 or 0565)
>+ *  dst_fmt           - destination color format (8888 or 0565)
>+ *  bpp_shift         - (1 << bpp_shift) is the size of source pixel in bytes
>+ *  prefetch_distance - prefetch in the source image by that many
>+ *                      pixels ahead
>+ */
>+
>+.macro generate_bilinear_scanline_func fname, src_fmt, dst_fmt, \
>+                                       src_bpp_shift, dst_bpp_shift, \
>+                                       prefetch_distance, flags
>+
>+pixman_asm_function fname
>+    OUT       .req      x0
>+    TOP       .req      x1
>+    BOTTOM    .req      x2
>+    WT        .req      x3
>+    WB        .req      x4
>+    X         .req      x5
>+    UX        .req      x6
>+    WIDTH     .req      x7
>+    TMP1      .req      x8
>+    TMP2      .req      x9
>+    PF_OFFS   .req      x10
>+    TMP3      .req      x11
>+    TMP4      .req      x12
>+    STRIDE    .req      x13
>+    
>+    sxtw      x3, w3
>+    sxtw      x4, w4
>+    sxtw      x5, w5
>+    sxtw      x6, w6
>+    sxtw      x7, w7
>+
>+    stp       x29, x30, [sp, -16]!
>+    mov       x29, sp
>+    sub       sp,  sp, 112  /* push all registers */
>+    sub       x29, x29, 64
>+    st1       {v8.8b, v9.8b, v10.8b, v11.8b}, [x29], #32
>+    st1       {v12.8b, v13.8b, v14.8b, v15.8b}, [x29], #32
>+    stp        x8,  x9, [x29, -80]
>+    stp       x10, x11, [x29, -96]
>+    stp       x12, x13, [x29, -112]
>+
>+    mov       PF_OFFS, #prefetch_distance
>+    mul       PF_OFFS, PF_OFFS, UX
>+
>+    subs      STRIDE, BOTTOM, TOP
>+    .unreq    BOTTOM
>+
>+    cmp       WIDTH, #0
>+    ble       300f
>+
>+    dup       v12.8h, w5
>+    dup       v13.8h, w6
>+    dup       v28.8b, w3
>+    dup       v29.8b, w4
>+    mov       v25.d[0], v12.d[1]
>+    mov       v26.d[0], v13.d[0]
>+    add       v25.4h, v25.4h, v26.4h
>+    mov       v12.d[1], v25.d[0]
>+
>+    /* ensure good destination alignment  */
>+    cmp       WIDTH, #1
>+    blt       100f
>+    tst       OUT, #(1 << dst_bpp_shift)
>+    beq       100f
>+    ushr      v15.8h, v12.8h, #(16 - BILINEAR_INTERPOLATION_BITS)
>+    add       v12.8h, v12.8h, v13.8h
>+    bilinear_interpolate_last_pixel src_fmt, dst_fmt
>+    sub       WIDTH, WIDTH, #1
>+100:
>+    add       v13.8h, v13.8h, v13.8h
>+    ushr      v15.8h, v12.8h, #(16 - BILINEAR_INTERPOLATION_BITS)
>+    add       v12.8h, v12.8h, v13.8h
>+
>+    cmp       WIDTH, #2
>+    blt       100f
>+    tst       OUT, #(1 << (dst_bpp_shift + 1))
>+    beq       100f
>+    bilinear_interpolate_two_pixels src_fmt, dst_fmt
>+    sub       WIDTH, WIDTH, #2
>+100:
>+.if ((flags) & BILINEAR_FLAG_UNROLL_8) != 0
>+/*********** 8 pixels per iteration *****************/
>+    cmp       WIDTH, #4
>+    blt       100f
>+    tst       OUT, #(1 << (dst_bpp_shift + 2))
>+    beq       100f
>+    bilinear_interpolate_four_pixels src_fmt, dst_fmt
>+    sub       WIDTH, WIDTH, #4
>+100:
>+    subs      WIDTH, WIDTH, #8
>+    blt       100f
>+    asr       PF_OFFS, PF_OFFS, #(16 - src_bpp_shift)
>+    bilinear_interpolate_eight_pixels_head src_fmt, dst_fmt
>+    subs      WIDTH, WIDTH, #8
>+    blt       500f
>+1000:
>+    bilinear_interpolate_eight_pixels_tail_head src_fmt, dst_fmt
>+    subs      WIDTH, WIDTH, #8
>+    bge       1000b
>+500:
>+    bilinear_interpolate_eight_pixels_tail src_fmt, dst_fmt
>+100:
>+    tst       WIDTH, #4
>+    beq       200f
>+    bilinear_interpolate_four_pixels src_fmt, dst_fmt
>+200:
>+.else
>+/*********** 4 pixels per iteration *****************/
>+    subs      WIDTH, WIDTH, #4
>+    blt       100f
>+    asr       PF_OFFS, PF_OFFS, #(16 - src_bpp_shift)
>+    bilinear_interpolate_four_pixels_head src_fmt, dst_fmt
>+    subs      WIDTH, WIDTH, #4
>+    blt       500f
>+1000:
>+    bilinear_interpolate_four_pixels_tail_head src_fmt, dst_fmt
>+    subs      WIDTH, WIDTH, #4
>+    bge       1000b
>+500:
>+    bilinear_interpolate_four_pixels_tail src_fmt, dst_fmt
>+100:
>+/****************************************************/
>+.endif
>+    /* handle the remaining trailing pixels */
>+    tst       WIDTH, #2
>+    beq       200f
>+    bilinear_interpolate_two_pixels src_fmt, dst_fmt
>+200:
>+    tst       WIDTH, #1
>+    beq       300f
>+    bilinear_interpolate_last_pixel src_fmt, dst_fmt
>+300:
>+    sub       x29, x29, 64
>+    ld1       {v8.8b, v9.8b, v10.8b, v11.8b}, [x29], #32
>+    ld1       {v12.8b, v13.8b, v14.8b, v15.8b}, [x29], #32
>+    ldp        x8,  x9, [x29, -80]
>+    ldp       x10, x11, [x29, -96]
>+    ldp       x12, x13, [x29, -104]
>+    mov       sp, x29
>+    ldp       x29, x30, [sp], 16 
>+    ret
>+
>+    .unreq    OUT
>+    .unreq    TOP
>+    .unreq    WT
>+    .unreq    WB
>+    .unreq    X
>+    .unreq    UX
>+    .unreq    WIDTH
>+    .unreq    TMP1
>+    .unreq    TMP2
>+    .unreq    PF_OFFS
>+    .unreq    TMP3
>+    .unreq    TMP4
>+    .unreq    STRIDE
>+.endfunc
>+
>+.endm
>+
>+/*****************************************************************************/
>+
>+.set have_bilinear_interpolate_four_pixels_8888_8888, 1
>+
>+.macro bilinear_interpolate_four_pixels_8888_8888_head
>+    asr       TMP1, X, #16
>+    add       X, X, UX
>+    add       TMP1, TOP, TMP1, lsl #2
>+    asr       TMP2, X, #16
>+    add       X, X, UX
>+    add       TMP2, TOP, TMP2, lsl #2
>+
>+    ld1       {v22.2s}, [TMP1], STRIDE
>+    ld1       {v23.2s}, [TMP1]
>+    asr       TMP3, X, #16
>+    add       X, X, UX
>+    add       TMP3, TOP, TMP3, lsl #2
>+    umull     v8.8h, v22.8b, v28.8b
>+    umlal     v8.8h, v23.8b, v29.8b
>+
>+    ld1       {v22.2s}, [TMP2], STRIDE
>+    ld1       {v23.2s}, [TMP2]
>+    asr       TMP4, X, #16
>+    add       X, X, UX
>+    add       TMP4, TOP, TMP4, lsl #2
>+    umull     v9.8h, v22.8b, v28.8b
>+    umlal     v9.8h, v23.8b, v29.8b
>+
>+    ld1       {v22.2s}, [TMP3], STRIDE
>+    ld1       {v23.2s}, [TMP3]
>+    umull     v10.8h, v22.8b, v28.8b
>+    umlal     v10.8h, v23.8b, v29.8b
>+
>+    ushll     v0.4s, v8.4h, #BILINEAR_INTERPOLATION_BITS
>+    umlsl     v0.4s, v8.4h, v15.h[0]
>+    umlal2    v0.4s, v8.8h, v15.h[0]
>+
>+    prfm      PREFETCH_MODE, [TMP4, PF_OFFS]
>+    ld1       {v16.2s}, [TMP4], STRIDE
>+    ld1       {v17.2s}, [TMP4]
>+    prfm      PREFETCH_MODE, [TMP4, PF_OFFS]
>+    umull     v11.8h, v16.8b, v28.8b
>+    umlal     v11.8h, v17.8b, v29.8b
>+
>+    ushll     v1.4s, v9.4h, #BILINEAR_INTERPOLATION_BITS
>+    umlsl     v1.4s, v9.4h, v15.h[4]
>+.endm
>+
>+.macro bilinear_interpolate_four_pixels_8888_8888_tail
>+    umlal2    v1.4s, v9.8h, v15.h[4]
>+    ushr      v15.8h, v12.8h, #(16 - BILINEAR_INTERPOLATION_BITS)
>+    ushll     v2.4s, v10.4h, #BILINEAR_INTERPOLATION_BITS
>+    umlsl     v2.4s, v10.4h, v15.h[0]
>+    umlal2    v2.4s, v10.8h, v15.h[0]
>+    ushll     v3.4s, v11.4h, #BILINEAR_INTERPOLATION_BITS
>+    umlsl     v3.4s, v11.4h, v15.h[4]
>+    umlal2    v3.4s, v11.8h, v15.h[4]
>+    add       v12.8h, v12.8h, v13.8h
>+    shrn      v0.4h, v0.4s, #(2 * BILINEAR_INTERPOLATION_BITS)
>+    shrn2     v0.8h, v1.4s, #(2 * BILINEAR_INTERPOLATION_BITS)
>+    shrn      v2.4h, v2.4s, #(2 * BILINEAR_INTERPOLATION_BITS)
>+    ushr      v15.8h, v12.8h, #(16 - BILINEAR_INTERPOLATION_BITS)
>+    shrn2     v2.8h, v3.4s, #(2 * BILINEAR_INTERPOLATION_BITS)
>+    xtn       v6.8b, v0.8h
>+    xtn       v7.8b, v2.8h
>+    add       v12.8h, v12.8h, v13.8h
>+    st1       {v6.2s, v7.2s}, [OUT], #16
>+.endm
>+
>+.macro bilinear_interpolate_four_pixels_8888_8888_tail_head
>+    asr       TMP1, X, #16
>+    add       X, X, UX
>+    add       TMP1, TOP, TMP1, lsl #2
>+    asr       TMP2, X, #16
>+    add       X, X, UX
>+    add       TMP2, TOP, TMP2, lsl #2
>+        umlal2    v1.4s, v9.8h, v15.h[4]
>+        ushr      v15.8h, v12.8h, #(16 - BILINEAR_INTERPOLATION_BITS)
>+        ushll     v2.4s, v10.4h, #BILINEAR_INTERPOLATION_BITS
>+        umlsl     v2.4s, v10.4h, v15.h[0]
>+        umlal2    v2.4s, v10.8h, v15.h[0]
>+        ushll     v3.4s, v11.4h, #BILINEAR_INTERPOLATION_BITS
>+    ld1       {v20.2s}, [TMP1], STRIDE
>+        umlsl     v3.4s, v11.4h, v15.h[4]
>+        umlal2    v3.4s, v11.8h, v15.h[4]
>+    ld1       {v21.2s}, [TMP1]
>+    umull     v8.8h, v20.8b, v28.8b
>+    umlal     v8.8h, v21.8b, v29.8b
>+        shrn      v0.4h, v0.4s, #(2 * BILINEAR_INTERPOLATION_BITS)
>+        shrn2     v0.8h, v1.4s, #(2 * BILINEAR_INTERPOLATION_BITS)
>+        shrn      v4.4h, v2.4s, #(2 * BILINEAR_INTERPOLATION_BITS)
>+    ld1       {v22.2s}, [TMP2], STRIDE
>+        shrn2     v4.8h, v3.4s, #(2 * BILINEAR_INTERPOLATION_BITS)
>+        add       v12.8h, v12.8h, v13.8h
>+    ld1       {v23.2s}, [TMP2]
>+    umull     v9.8h, v22.8b, v28.8b
>+    asr       TMP3, X, #16
>+    add       X, X, UX
>+    add       TMP3, TOP, TMP3, lsl #2
>+    asr       TMP4, X, #16
>+    add       X, X, UX
>+    add       TMP4, TOP, TMP4, lsl #2
>+    umlal     v9.8h, v23.8b, v29.8b
>+    ld1       {v22.2s}, [TMP3], STRIDE
>+        ushr      v15.8h, v12.8h, #(16 - BILINEAR_INTERPOLATION_BITS)
>+    ld1       {v23.2s}, [TMP3]
>+    umull     v10.8h, v22.8b, v28.8b
>+    umlal     v10.8h, v23.8b, v29.8b
>+        xtn       v6.8b, v0.8h
>+    ushll     v0.4s, v8.4h, #BILINEAR_INTERPOLATION_BITS
>+        xtn       v7.8b, v4.8h
>+    umlsl     v0.4s, v8.4h, v15.h[0]
>+    umlal2    v0.4s, v8.8h, v15.h[0]
>+    prfm      PREFETCH_MODE, [TMP4, PF_OFFS]
>+    ld1       {v16.2s}, [TMP4], STRIDE
>+        add       v12.8h, v12.8h, v13.8h
>+    ld1       {v17.2s}, [TMP4]
>+    prfm      PREFETCH_MODE, [TMP4, PF_OFFS]
>+    umull     v11.8h, v16.8b, v28.8b
>+    umlal     v11.8h, v17.8b, v29.8b
>+        st1       {v6.2s, v7.2s}, [OUT], #16
>+    ushll     v1.4s, v9.4h, #BILINEAR_INTERPOLATION_BITS
>+    umlsl     v1.4s, v9.4h, v15.h[4]
>+.endm
>+
>+/*****************************************************************************/
>+
>+generate_bilinear_scanline_func \
>+    pixman_scaled_bilinear_scanline_8888_8888_SRC_asm_neon, 8888, 8888, \
>+    2, 2, 28, BILINEAR_FLAG_UNROLL_4
>+
>+generate_bilinear_scanline_func \
>+    pixman_scaled_bilinear_scanline_8888_0565_SRC_asm_neon, 8888, 0565, \
>+    2, 1, 28, BILINEAR_FLAG_UNROLL_8 | BILINEAR_FLAG_USE_ALL_NEON_REGS
>+
>+generate_bilinear_scanline_func \
>+    pixman_scaled_bilinear_scanline_0565_x888_SRC_asm_neon, 0565, 8888, \
>+    1, 2, 28, BILINEAR_FLAG_UNROLL_4
>+
>+generate_bilinear_scanline_func \
>+    pixman_scaled_bilinear_scanline_0565_0565_SRC_asm_neon, 0565, 0565, \
>+    1, 1, 28, BILINEAR_FLAG_UNROLL_4
>diff --git a/pixman/pixman-arma64-neon-asm.h b/pixman/pixman-arma64-neon-asm.h
>new file mode 100644
>index 0000000..8c864ca
>--- /dev/null
>+++ b/pixman/pixman-arma64-neon-asm.h
>@@ -0,0 +1,1300 @@
>+/*
>+ * Copyright Â© 2009 Nokia Corporation
>+ *
>+ * Permission is hereby granted, free of charge, to any person obtaining a
>+ * copy of this software and associated documentation files (the "Software"),
>+ * to deal in the Software without restriction, including without limitation
>+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
>+ * and/or sell copies of the Software, and to permit persons to whom the
>+ * Software is furnished to do so, subject to the following conditions:
>+ *
>+ * The above copyright notice and this permission notice (including the next
>+ * paragraph) shall be included in all copies or substantial portions of the
>+ * Software.
>+ *
>+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
>+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
>+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
>+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
>+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
>+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
>+ * DEALINGS IN THE SOFTWARE.
>+ *
>+ * Author:  Siarhei Siamashka (siarhei.siamashka@nokia.com)
>+ */
>+
>+/*
>+ * This file contains a macro ('generate_composite_function') which can
>+ * construct 2D image processing functions, based on a common template.
>+ * Any combinations of source, destination and mask images with 8bpp,
>+ * 16bpp, 24bpp, 32bpp color formats are supported.
>+ *
>+ * This macro takes care of:
>+ *  - handling of leading and trailing unaligned pixels
>+ *  - doing most of the work related to L2 cache preload
>+ *  - encourages the use of software pipelining for better instructions
>+ *    scheduling
>+ *
>+ * The user of this macro has to provide some configuration parameters
>+ * (bit depths for the images, prefetch distance, etc.) and a set of
>+ * macros, which should implement basic code chunks responsible for
>+ * pixels processing. See 'pixman-armv8-neon-asm.S' file for the usage
>+ * examples.
>+ *
>+ * TODO:
>+ *  - try overlapped pixel method (from Ian Rickards) when processing
>+ *    exactly two blocks of pixels
>+ *  - maybe add an option to do reverse scanline processing
>+ */
>+
>+/*
>+ * Bit flags for 'generate_composite_function' macro which are used
>+ * to tune generated functions behavior.
>+ */
>+.set FLAG_DST_WRITEONLY,       0
>+.set FLAG_DST_READWRITE,       1
>+.set FLAG_DEINTERLEAVE_32BPP,  2
>+
>+/*
>+ * Constants for selecting preferable prefetch type.
>+ */
>+.set PREFETCH_TYPE_NONE,       0 /* No prefetch at all */
>+.set PREFETCH_TYPE_SIMPLE,     1 /* A simple, fixed-distance-ahead prefetch */
>+.set PREFETCH_TYPE_ADVANCED,   2 /* Advanced fine-grained prefetch */
>+
>+/*
>+ * prefetch mode
>+ * available modes are:
>+ * pldl1keep
>+ * pldl1strm
>+ * pldl2keep
>+ * pldl2strm
>+ * pldl3keep
>+ * pldl3strm
>+ */
>+#define PREFETCH_MODE pldl1keep
>+
>+/*
>+ * Definitions of supplementary pixld/pixst macros (for partial load/store of
>+ * pixel data).
>+ */
>+
>+.macro pixldst1 op, elem_size, reg1, mem_operand, abits
>+    op {v&reg1&.&elem_size}, [&mem_operand&], #8
>+.endm
>+
>+.macro pixldst2 op, elem_size, reg1, reg2, mem_operand, abits
>+    op {v&reg1&.&elem_size, v&reg2&.&elem_size}, [&mem_operand&], #16
>+.endm
>+
>+.macro pixldst4 op, elem_size, reg1, reg2, reg3, reg4, mem_operand, abits
>+    op {v&reg1&.&elem_size, v&reg2&.&elem_size, v&reg3&.&elem_size, v&reg4&.&elem_size}, [&mem_operand&], #32
>+.endm
>+
>+.macro pixldst0 op, elem_size, reg1, idx, mem_operand, abits, bytes
>+    op {v&reg1&.&elem_size}[idx], [&mem_operand&], #&bytes&
>+.endm
>+
>+.macro pixldst3 op, elem_size, reg1, reg2, reg3, mem_operand
>+    op {v&reg1&.&elem_size, v&reg2&.&elem_size, v&reg3&.&elem_size}, [&mem_operand&], #24
>+.endm
>+
>+.macro pixldst30 op, elem_size, reg1, reg2, reg3, idx, mem_operand
>+    op {v&reg1&.&elem_size, v&reg2&.&elem_size, v&reg3&.&elem_size}[idx], [&mem_operand&], #3
>+.endm
>+
>+.macro pixldst numbytes, op, elem_size, basereg, mem_operand, abits
>+.if numbytes == 32
>+    .if elem_size==32
>+        pixldst4 op, 2s, %(basereg+4), %(basereg+5), \
>+                              %(basereg+6), %(basereg+7), mem_operand, abits
>+    .elseif elem_size==16
>+        pixldst4 op, 4h, %(basereg+4), %(basereg+5), \
>+                              %(basereg+6), %(basereg+7), mem_operand, abits
>+    .else
>+        pixldst4 op, 8b, %(basereg+4), %(basereg+5), \
>+                              %(basereg+6), %(basereg+7), mem_operand, abits
>+    .endif
>+.elseif numbytes == 16
>+    .if elem_size==32
>+          pixldst2 op, 2s, %(basereg+2), %(basereg+3), mem_operand, abits
>+    .elseif elem_size==16
>+          pixldst2 op, 4h, %(basereg+2), %(basereg+3), mem_operand, abits
>+    .else
>+          pixldst2 op, 8b, %(basereg+2), %(basereg+3), mem_operand, abits
>+    .endif
>+.elseif numbytes == 8
>+    .if elem_size==32
>+        pixldst1 op, 2s, %(basereg+1), mem_operand, abits
>+    .elseif elem_size==16
>+        pixldst1 op, 4h, %(basereg+1), mem_operand, abits
>+    .else
>+        pixldst1 op, 8b, %(basereg+1), mem_operand, abits
>+    .endif
>+.elseif numbytes == 4
>+    .if !RESPECT_STRICT_ALIGNMENT || (elem_size == 32)
>+        pixldst0 op, s, %(basereg+0), 1, mem_operand, abits, 4
>+    .elseif elem_size == 16
>+        pixldst0 op, h, %(basereg+0), 2, mem_operand, abits, 2
>+        pixldst0 op, h, %(basereg+0), 3, mem_operand, abits, 2
>+    .else
>+        pixldst0 op, b, %(basereg+0), 4, mem_operand, abits, 1
>+        pixldst0 op, b, %(basereg+0), 5, mem_operand, abits, 1
>+        pixldst0 op, b, %(basereg+0), 6, mem_operand, abits, 1
>+        pixldst0 op, b, %(basereg+0), 7, mem_operand, abits, 1
>+    .endif
>+.elseif numbytes == 2
>+    .if !RESPECT_STRICT_ALIGNMENT || (elem_size == 16)
>+        pixldst0 op, h, %(basereg+0), 1, mem_operand, abits, 2
>+    .else
>+        pixldst0 op, b, %(basereg+0), 2, mem_operand, abits, 1
>+        pixldst0 op, b, %(basereg+0), 3, mem_operand, abits, 1
>+    .endif
>+.elseif numbytes == 1
>+        pixldst0 op, b, %(basereg+0), 1, mem_operand, abits, 1
>+.else
>+    .error "unsupported size: numbytes"
>+.endif
>+.endm
>+
>+.macro pixld numpix, bpp, basereg, mem_operand, abits=0
>+.if bpp > 0
>+.if (bpp == 32) && (numpix == 8) && (DEINTERLEAVE_32BPP_ENABLED != 0)
>+    pixldst4 ld4, 8b, %(basereg+4), %(basereg+5), \
>+                      %(basereg+6), %(basereg+7), mem_operand, abits
>+.elseif (bpp == 24) && (numpix == 8)
>+    pixldst3 ld3, 8b, %(basereg+3), %(basereg+4), %(basereg+5), mem_operand
>+.elseif (bpp == 24) && (numpix == 4)
>+    pixldst30 ld3, b, %(basereg+0), %(basereg+1), %(basereg+2), 4, mem_operand
>+    pixldst30 ld3, b, %(basereg+0), %(basereg+1), %(basereg+2), 5, mem_operand
>+    pixldst30 ld3, b, %(basereg+0), %(basereg+1), %(basereg+2), 6, mem_operand
>+    pixldst30 ld3, b, %(basereg+0), %(basereg+1), %(basereg+2), 7, mem_operand
>+.elseif (bpp == 24) && (numpix == 2)
>+    pixldst30 ld3, b, %(basereg+0), %(basereg+1), %(basereg+2), 2, mem_operand
>+    pixldst30 ld3, b, %(basereg+0), %(basereg+1), %(basereg+2), 3, mem_operand
>+.elseif (bpp == 24) && (numpix == 1)
>+    pixldst30 ld3, b, %(basereg+0), %(basereg+1), %(basereg+2), 1, mem_operand
>+.else
>+    pixldst %(numpix * bpp / 8), ld1, %(bpp), basereg, mem_operand, abits
>+.endif
>+.endif
>+.endm
>+
>+.macro pixst numpix, bpp, basereg, mem_operand, abits=0
>+.if bpp > 0
>+.if (bpp == 32) && (numpix == 8) && (DEINTERLEAVE_32BPP_ENABLED != 0)
>+    pixldst4 st4, 8b, %(basereg+4), %(basereg+5), \
>+                      %(basereg+6), %(basereg+7), mem_operand, abits
>+.elseif (bpp == 24) && (numpix == 8)
>+    pixldst3 st3, 8b, %(basereg+3), %(basereg+4), %(basereg+5), mem_operand
>+.elseif (bpp == 24) && (numpix == 4)
>+    pixldst30 st3, b, %(basereg+0), %(basereg+1), %(basereg+2), 4, mem_operand
>+    pixldst30 st3, b, %(basereg+0), %(basereg+1), %(basereg+2), 5, mem_operand
>+    pixldst30 st3, b, %(basereg+0), %(basereg+1), %(basereg+2), 6, mem_operand
>+    pixldst30 st3, b, %(basereg+0), %(basereg+1), %(basereg+2), 7, mem_operand
>+.elseif (bpp == 24) && (numpix == 2)
>+    pixldst30 st3, b, %(basereg+0), %(basereg+1), %(basereg+2), 2, mem_operand
>+    pixldst30 st3, b, %(basereg+0), %(basereg+1), %(basereg+2), 3, mem_operand
>+.elseif (bpp == 24) && (numpix == 1)
>+    pixldst30 st3, b, %(basereg+0), %(basereg+1), %(basereg+2), 1, mem_operand
>+.else
>+    pixldst %(numpix * bpp / 8), st1, %(bpp), basereg, mem_operand, abits
>+.endif
>+.endif
>+.endm
>+
>+.macro pixld_a numpix, bpp, basereg, mem_operand
>+.if (bpp * numpix) <= 128
>+    pixld numpix, bpp, basereg, mem_operand, %(bpp * numpix)
>+.else
>+    pixld numpix, bpp, basereg, mem_operand, 128
>+.endif
>+.endm
>+
>+.macro pixst_a numpix, bpp, basereg, mem_operand
>+.if (bpp * numpix) <= 128
>+    pixst numpix, bpp, basereg, mem_operand, %(bpp * numpix)
>+.else
>+    pixst numpix, bpp, basereg, mem_operand, 128
>+.endif
>+.endm
>+
>+/*
>+ * Pixel fetcher for nearest scaling (needs TMP1, TMP2, VX, UNIT_X register
>+ * aliases to be defined)
>+ */
>+.macro pixld1_s elem_size, reg1, mem_operand
>+.if elem_size == 16
>+    asr     TMP1, VX, #16
>+    adds    VX, VX, UNIT_X
>+    bmi     55f
>+5:  subs    VX, VX, SRC_WIDTH_FIXED
>+    bpl     5b
>+55:
>+    lsl     DUMMY, TMP1, #1
>+    add     TMP1, mem_operand, DUMMY
>+    asr     TMP2, VX, #16
>+    adds    VX, VX, UNIT_X
>+    bmi     55f
>+5:  subs    VX, VX, SRC_WIDTH_FIXED
>+    bpl     5b
>+55:
>+    lsl     DUMMY, TMP2, #1
>+    add     TMP2, mem_operand, DUMMY
>+    ld1     {v&reg1&.h}[0], [TMP1]
>+    asr     TMP1, VX, #16
>+    adds    VX, VX, UNIT_X
>+    bmi     55f
>+5:  subs    VX, VX, SRC_WIDTH_FIXED
>+    bpl     5b
>+55:
>+    lsl     DUMMY, TMP1, #1
>+    add     TMP1, mem_operand, DUMMY
>+    ld1     {v&reg1&.h}[1], [TMP2]
>+    asr     TMP2, VX, #16
>+    adds    VX, VX, UNIT_X
>+    bmi     55f
>+5:  subs    VX, VX, SRC_WIDTH_FIXED
>+    bpl     5b
>+55:
>+    lsl     DUMMY, TMP2, #1
>+    add     TMP2, mem_operand, DUMMY
>+    ld1     {v&reg1&.h}[2], [TMP1]
>+    ld1     {v&reg1&.h}[3], [TMP2]
>+.elseif elem_size == 32
>+    asr     TMP1, VX, #16
>+    adds    VX, VX, UNIT_X
>+    bmi     55f
>+5:  subs    VX, VX, SRC_WIDTH_FIXED
>+    bpl     5b
>+55:
>+    lsl     DUMMY, TMP1, #2
>+    add     TMP1, mem_operand, DUMMY
>+    asr     TMP2, VX, #16
>+    adds    VX, VX, UNIT_X
>+    bmi     55f
>+5:  subs    VX, VX, SRC_WIDTH_FIXED
>+    bpl     5b
>+55:
>+    lsl     DUMMY, TMP2, #2
>+    add     TMP2, mem_operand, DUMMY
>+    ld1     {v&reg1&.s}[0], [TMP1]
>+    ld1     {v&reg1&.s}[1], [TMP2]
>+.else
>+    .error "unsupported"
>+.endif
>+.endm
>+
>+.macro pixld2_s elem_size, reg1, reg2, mem_operand
>+.if 0 /* elem_size == 32 */
>+    mov     TMP1, VX, asr #16
>+    add     VX, VX, UNIT_X, asl #1
>+    add     TMP1, mem_operand, TMP1, asl #2
>+    mov     TMP2, VX, asr #16
>+    sub     VX, VX, UNIT_X
>+    add     TMP2, mem_operand, TMP2, asl #2
>+    ld1     {v&reg1&.s}[0], [TMP1]
>+    mov     TMP1, VX, asr #16
>+    add     VX, VX, UNIT_X, asl #1
>+    add     TMP1, mem_operand, TMP1, asl #2
>+    ld1     {v&reg2&.s}[0], [TMP2, :32]
>+    mov     TMP2, VX, asr #16
>+    add     VX, VX, UNIT_X
>+    add     TMP2, mem_operand, TMP2, asl #2
>+    ld1     {v&reg1&.s}[1], [TMP1]
>+    ld1     {v&reg2&.s}[1], [TMP2]
>+.else
>+    pixld1_s elem_size, reg1, mem_operand
>+    pixld1_s elem_size, reg2, mem_operand
>+.endif
>+.endm
>+
>+.macro pixld0_s elem_size, reg1, idx, mem_operand
>+.if elem_size == 16
>+    asr     TMP1, VX, #16
>+    adds    VX, VX, UNIT_X
>+    bmi     55f
>+5:  subs    VX, VX, SRC_WIDTH_FIXED
>+    bpl     5b
>+55:
>+    lsl     DUMMY, TMP1, #1
>+    add     TMP1, mem_operand, DUMMY
>+    ld1     {v&reg1&.h}[idx], [TMP1]
>+.elseif elem_size == 32
>+    asr     DUMMY, VX, #16
>+    mov     TMP1, DUMMY
>+    adds    VX, VX, UNIT_X
>+    bmi     55f
>+5:  subs    VX, VX, SRC_WIDTH_FIXED
>+    bpl     5b
>+55:
>+    lsl     DUMMY, TMP1, #2
>+    add     TMP1, mem_operand, DUMMY
>+    ld1     {v&reg1&.s}[idx], [TMP1]
>+.endif
>+.endm
>+
>+.macro pixld_s_internal numbytes, elem_size, basereg, mem_operand
>+.if numbytes == 32
>+    pixld2_s elem_size, %(basereg+4), %(basereg+5), mem_operand
>+    pixld2_s elem_size, %(basereg+6), %(basereg+7), mem_operand
>+    pixdeinterleave elem_size, %(basereg+4)
>+.elseif numbytes == 16
>+    pixld2_s elem_size, %(basereg+2), %(basereg+3), mem_operand
>+.elseif numbytes == 8
>+    pixld1_s elem_size, %(basereg+1), mem_operand
>+.elseif numbytes == 4
>+    .if elem_size == 32
>+        pixld0_s elem_size, %(basereg+0), 1, mem_operand
>+    .elseif elem_size == 16
>+        pixld0_s elem_size, %(basereg+0), 2, mem_operand
>+        pixld0_s elem_size, %(basereg+0), 3, mem_operand
>+    .else
>+        pixld0_s elem_size, %(basereg+0), 4, mem_operand
>+        pixld0_s elem_size, %(basereg+0), 5, mem_operand
>+        pixld0_s elem_size, %(basereg+0), 6, mem_operand
>+        pixld0_s elem_size, %(basereg+0), 7, mem_operand
>+    .endif
>+.elseif numbytes == 2
>+    .if elem_size == 16
>+        pixld0_s elem_size, %(basereg+0), 1, mem_operand
>+    .else
>+        pixld0_s elem_size, %(basereg+0), 2, mem_operand
>+        pixld0_s elem_size, %(basereg+0), 3, mem_operand
>+    .endif
>+.elseif numbytes == 1
>+    pixld0_s elem_size, %(basereg+0), 1, mem_operand
>+.else
>+    .error "unsupported size: numbytes"
>+.endif
>+.endm
>+
>+.macro pixld_s numpix, bpp, basereg, mem_operand
>+.if bpp > 0
>+    pixld_s_internal %(numpix * bpp / 8), %(bpp), basereg, mem_operand
>+.endif
>+.endm
>+
>+.macro vuzp8 reg1, reg2
>+    umov DUMMY, v16.d[0]
>+    uzp1 v16.8b,     v&reg1&.8b, v&reg2&.8b
>+    uzp2 v&reg2&.8b, v&reg1&.8b, v&reg2&.8b
>+    mov  v&reg1&.8b, v16.8b
>+    mov  v16.d[0], DUMMY
>+.endm
>+
>+.macro vzip8 reg1, reg2
>+    umov DUMMY, v16.d[0]
>+    zip1 v16.8b,     v&reg1&.8b, v&reg2&.8b
>+    zip2 v&reg2&.8b, v&reg1&.8b, v&reg2&.8b
>+    mov  v&reg1&.8b, v16.8b
>+    mov  v16.d[0], DUMMY
>+.endm
>+
>+/* deinterleave B, G, R, A channels for eight 32bpp pixels in 4 registers */
>+.macro pixdeinterleave bpp, basereg
>+.if (bpp == 32) && (DEINTERLEAVE_32BPP_ENABLED != 0)
>+    vuzp8 %(basereg+0), %(basereg+1)
>+    vuzp8 %(basereg+2), %(basereg+3)
>+    vuzp8 %(basereg+1), %(basereg+3)
>+    vuzp8 %(basereg+0), %(basereg+2)
>+.endif
>+.endm
>+
>+/* interleave B, G, R, A channels for eight 32bpp pixels in 4 registers */
>+.macro pixinterleave bpp, basereg
>+.if (bpp == 32) && (DEINTERLEAVE_32BPP_ENABLED != 0)
>+    vzip8 %(basereg+0), %(basereg+2)
>+    vzip8 %(basereg+1), %(basereg+3)
>+    vzip8 %(basereg+2), %(basereg+3)
>+    vzip8 %(basereg+0), %(basereg+1)
>+.endif
>+.endm
>+
>+/*
>+ * This is a macro for implementing cache preload. The main idea is that
>+ * cache preload logic is mostly independent from the rest of pixels
>+ * processing code. It starts at the top left pixel and moves forward
>+ * across pixels and can jump across scanlines. Prefetch distance is
>+ * handled in an 'incremental' way: it starts from 0 and advances to the
>+ * optimal distance over time. After reaching optimal prefetch distance,
>+ * it is kept constant. There are some checks which prevent prefetching
>+ * unneeded pixel lines below the image (but it still can prefetch a bit
>+ * more data on the right side of the image - not a big issue and may
>+ * be actually helpful when rendering text glyphs). Additional trick is
>+ * the use of LDR instruction for prefetch instead of PLD when moving to
>+ * the next line, the point is that we have a high chance of getting TLB
>+ * miss in this case, and PLD would be useless.
>+ *
>+ * This sounds like it may introduce a noticeable overhead (when working with
>+ * fully cached data). But in reality, due to having a separate pipeline and
>+ * instruction queue for NEON unit in ARM Cortex-A8, normal ARM code can
>+ * execute simultaneously with NEON and be completely shadowed by it. Thus
>+ * we get no performance overhead at all (*). This looks like a very nice
>+ * feature of Cortex-A8, if used wisely. We don't have a hardware prefetcher,
>+ * but still can implement some rather advanced prefetch logic in software
>+ * for almost zero cost!
>+ *
>+ * (*) The overhead of the prefetcher is visible when running some trivial
>+ * pixels processing like simple copy. Anyway, having prefetch is a must
>+ * when working with the graphics data.
>+ */
>+.macro PF a, x:vararg
>+.if (PREFETCH_TYPE_CURRENT == PREFETCH_TYPE_ADVANCED)
>+    a x
>+.endif
>+.endm
>+
>+.macro cache_preload std_increment, boost_increment
>+.if (src_bpp_shift >= 0) || (dst_r_bpp != 0) || (mask_bpp_shift >= 0)
>+.if std_increment != 0
>+    PF add PF_X, PF_X, #std_increment
>+.endif
>+    PF tst PF_CTL, #0xF
>+    PF beq 71f
>+    PF add PF_X, PF_X, #boost_increment
>+    PF sub PF_CTL, PF_CTL, #1
>+71:
>+    PF cmp PF_X, ORIG_W
>+.if src_bpp_shift >= 0
>+    PF lsl DUMMY, PF_X, #src_bpp_shift
>+    PF prfm PREFETCH_MODE, [PF_SRC, DUMMY]
>+.endif
>+.if dst_r_bpp != 0
>+    PF lsl DUMMY, PF_X, #dst_bpp_shift
>+    PF prfm PREFETCH_MODE, [PF_DST, DUMMY]
>+.endif
>+.if mask_bpp_shift >= 0
>+    PF lsl DUMMY, PF_X, #mask_bpp_shift
>+    PF prfm PREFETCH_MODE, [PF_MASK, DUMMY]
>+.endif
>+    PF ble 71f
>+    PF sub PF_X, PF_X, ORIG_W
>+    PF subs PF_CTL, PF_CTL, #0x10
>+71:
>+    PF ble 72f
>+.if src_bpp_shift >= 0
>+    PF lsl DUMMY, SRC_STRIDE, #src_bpp_shift
>+    PF ldrsb DUMMY, [PF_SRC, DUMMY]
>+    PF add PF_SRC, PF_SRC, #1
>+.endif
>+.if dst_r_bpp != 0
>+    PF lsl DUMMY, DST_STRIDE, #dst_bpp_shift
>+    PF ldrsb DUMMY, [PF_DST, DUMMY]
>+    PF add PF_DST, PF_DST, #1
>+.endif
>+.if mask_bpp_shift >= 0
>+    PF lsl DUMMY, MASK_STRIDE, #mask_bpp_shift
>+    PF ldrsb DUMMY, [PF_MASK, DUMMY]
>+    PF add PF_MASK, PF_MASK, #1
>+.endif
>+72:
>+.endif
>+.endm
>+
>+.macro cache_preload_simple
>+.if (PREFETCH_TYPE_CURRENT == PREFETCH_TYPE_SIMPLE)
>+.if src_bpp > 0
>+    prfm PREFETCH_MODE, [SRC, #(PREFETCH_DISTANCE_SIMPLE * src_bpp / 8)]
>+.endif
>+.if dst_r_bpp > 0
>+    prfm PREFETCH_MODE, [DST_R, #(PREFETCH_DISTANCE_SIMPLE * dst_r_bpp / 8)]
>+.endif
>+.if mask_bpp > 0
>+    prfm PREFETCH_MODE, [MASK, #(PREFETCH_DISTANCE_SIMPLE * mask_bpp / 8)]
>+.endif
>+.endif
>+.endm
>+
>+.macro fetch_mask_pixblock
>+    pixld       pixblock_size, mask_bpp, \
>+                (mask_basereg - pixblock_size * mask_bpp / 64), MASK
>+.endm
>+
>+/*
>+ * Macro which is used to process leading pixels until destination
>+ * pointer is properly aligned (at 16 bytes boundary). When destination
>+ * buffer uses 16bpp format, this is unnecessary, or even pointless.
>+ */
>+.macro ensure_destination_ptr_alignment process_pixblock_head, \
>+                                        process_pixblock_tail, \
>+                                        process_pixblock_tail_head
>+.if dst_w_bpp != 24
>+    tst         DST_R, #0xF
>+    beq         52f
>+.irp lowbit, 1, 2, 4, 8, 16
>+local skip1
>+.if (dst_w_bpp <= (lowbit * 8)) && ((lowbit * 8) < (pixblock_size * dst_w_bpp))
>+.if lowbit < 16 /* we don't need more than 16-byte alignment */
>+    tst         DST_R, #lowbit
>+    beq         51f
>+.endif
>+    pixld_src   (lowbit * 8 / dst_w_bpp), src_bpp, src_basereg, SRC
>+    pixld       (lowbit * 8 / dst_w_bpp), mask_bpp, mask_basereg, MASK
>+.if dst_r_bpp > 0
>+    pixld_a     (lowbit * 8 / dst_r_bpp), dst_r_bpp, dst_r_basereg, DST_R
>+.else
>+    add         DST_R, DST_R, #lowbit
>+.endif
>+    PF add      PF_X, PF_X, #(lowbit * 8 / dst_w_bpp)
>+    sub         W, W, #(lowbit * 8 / dst_w_bpp)
>+51:
>+.endif
>+.endr
>+    pixdeinterleave src_bpp, src_basereg
>+    pixdeinterleave mask_bpp, mask_basereg
>+    pixdeinterleave dst_r_bpp, dst_r_basereg
>+
>+    process_pixblock_head
>+    cache_preload 0, pixblock_size
>+    cache_preload_simple
>+    process_pixblock_tail
>+
>+    pixinterleave dst_w_bpp, dst_w_basereg
>+
>+.irp lowbit, 1, 2, 4, 8, 16
>+.if (dst_w_bpp <= (lowbit * 8)) && ((lowbit * 8) < (pixblock_size * dst_w_bpp))
>+.if lowbit < 16 /* we don't need more than 16-byte alignment */
>+    tst         DST_W, #lowbit
>+    beq         51f
>+.endif
>+    pixst_a     (lowbit * 8 / dst_w_bpp), dst_w_bpp, dst_w_basereg, DST_W
>+51:
>+.endif
>+.endr
>+.endif
>+52:
>+.endm
>+
>+/*
>+ * Special code for processing up to (pixblock_size - 1) remaining
>+ * trailing pixels. As SIMD processing performs operation on
>+ * pixblock_size pixels, anything smaller than this has to be loaded
>+ * and stored in a special way. Loading and storing of pixel data is
>+ * performed in such a way that we fill some 'slots' in the NEON
>+ * registers (some slots naturally are unused), then perform compositing
>+ * operation as usual. In the end, the data is taken from these 'slots'
>+ * and saved to memory.
>+ *
>+ * cache_preload_flag - allows to suppress prefetch if
>+ *                      set to 0
>+ * dst_aligned_flag   - selects whether destination buffer
>+ *                      is aligned
>+ */
>+.macro process_trailing_pixels cache_preload_flag, \
>+                               dst_aligned_flag, \
>+                               process_pixblock_head, \
>+                               process_pixblock_tail, \
>+                               process_pixblock_tail_head
>+    tst         W, #(pixblock_size - 1)
>+    beq         52f
>+.irp chunk_size, 16, 8, 4, 2, 1
>+.if pixblock_size > chunk_size
>+    tst         W, #chunk_size
>+    beq         51f
>+    pixld_src   chunk_size, src_bpp, src_basereg, SRC
>+    pixld       chunk_size, mask_bpp, mask_basereg, MASK
>+.if dst_aligned_flag != 0
>+    pixld_a     chunk_size, dst_r_bpp, dst_r_basereg, DST_R
>+.else
>+    pixld       chunk_size, dst_r_bpp, dst_r_basereg, DST_R
>+.endif
>+.if cache_preload_flag != 0
>+    PF add      PF_X, PF_X, #chunk_size
>+.endif
>+51:
>+.endif
>+.endr
>+    pixdeinterleave src_bpp, src_basereg
>+    pixdeinterleave mask_bpp, mask_basereg
>+    pixdeinterleave dst_r_bpp, dst_r_basereg
>+
>+    process_pixblock_head
>+.if cache_preload_flag != 0
>+    cache_preload 0, pixblock_size
>+    cache_preload_simple
>+.endif
>+    process_pixblock_tail
>+    pixinterleave dst_w_bpp, dst_w_basereg
>+.irp chunk_size, 16, 8, 4, 2, 1
>+.if pixblock_size > chunk_size
>+    tst         W, #chunk_size
>+    beq         51f
>+.if dst_aligned_flag != 0
>+    pixst_a     chunk_size, dst_w_bpp, dst_w_basereg, DST_W
>+.else
>+    pixst       chunk_size, dst_w_bpp, dst_w_basereg, DST_W
>+.endif
>+51:
>+.endif
>+.endr
>+52:
>+.endm
>+
>+/*
>+ * Macro, which performs all the needed operations to switch to the next
>+ * scanline and start the next loop iteration unless all the scanlines
>+ * are already processed.
>+ */
>+.macro advance_to_next_scanline start_of_loop_label
>+    mov         W, ORIG_W
>+    lsl         DUMMY, DST_STRIDE, #dst_bpp_shift
>+    add         DST_W, DST_W, DUMMY
>+.if src_bpp != 0
>+    lsl         DUMMY, SRC_STRIDE, #src_bpp_shift
>+    add         SRC, SRC, DUMMY
>+.endif
>+.if mask_bpp != 0
>+    lsl         DUMMY, MASK_STRIDE, #mask_bpp_shift
>+    add         MASK, MASK, DUMMY
>+.endif
>+.if (dst_w_bpp != 24)
>+    lsl         DUMMY, W, #dst_bpp_shift
>+    sub         DST_W, DST_W, DUMMY
>+.endif
>+.if (src_bpp != 24) && (src_bpp != 0)
>+    lsl         DUMMY, W, #src_bpp_shift
>+    sub         SRC, SRC, DUMMY
>+.endif
>+.if (mask_bpp != 24) && (mask_bpp != 0)
>+    lsl         DUMMY, W, #mask_bpp_shift
>+    sub         MASK, MASK, DUMMY
>+.endif
>+    subs        H, H, #1
>+    mov         DST_R, DST_W
>+    bge         start_of_loop_label
>+.endm
>+
>+/*
>+ * Registers are allocated in the following way by default:
>+ * v0, v1, v2, v3     - reserved for loading source pixel data
>+ * v4, v5, v6, v7     - reserved for loading destination pixel data
>+ * v24, v25, v26, v27 - reserved for loading mask pixel data
>+ * v28, v29, v30, v31 - final destination pixel data for writeback to memory
>+ */
>+.macro generate_composite_function fname, \
>+                                   src_bpp_, \
>+                                   mask_bpp_, \
>+                                   dst_w_bpp_, \
>+                                   flags, \
>+                                   pixblock_size_, \
>+                                   prefetch_distance, \
>+                                   init, \
>+                                   cleanup, \
>+                                   process_pixblock_head, \
>+                                   process_pixblock_tail, \
>+                                   process_pixblock_tail_head, \
>+                                   dst_w_basereg_ = 28, \
>+                                   dst_r_basereg_ = 4, \
>+                                   src_basereg_   = 0, \
>+                                   mask_basereg_  = 24
>+
>+    pixman_asm_function fname
>+    stp         x29, x30, [sp, -16]!
>+    mov         x29, sp
>+    sub         sp,   sp, 232  /* push all registers */
>+    sub         x29, x29, 64
>+    st1         {v8.8b, v9.8b, v10.8b, v11.8b}, [x29], #32
>+    st1         {v12.8b, v13.8b, v14.8b, v15.8b}, [x29], #32
>+    stp          x8,   x9, [x29, -80]
>+    stp         x10,  x11, [x29, -96]
>+    stp         x12,  x13, [x29, -112]
>+    stp         x14,  x15, [x29, -128]
>+    stp         x16,  x17, [x29, -144]
>+    stp         x18,  x19, [x29, -160]
>+    stp         x20,  x21, [x29, -176]
>+    stp         x22,  x23, [x29, -192]
>+    stp         x24,  x25, [x29, -208]
>+    stp         x26,  x27, [x29, -224]
>+    str         x28, [x29, -232]
>+
>+/*
>+ * Select prefetch type for this function. If prefetch distance is
>+ * set to 0 or one of the color formats is 24bpp, SIMPLE prefetch
>+ * has to be used instead of ADVANCED.
>+ */
>+    .set PREFETCH_TYPE_CURRENT, PREFETCH_TYPE_DEFAULT
>+.if prefetch_distance == 0
>+    .set PREFETCH_TYPE_CURRENT, PREFETCH_TYPE_NONE
>+.elseif (PREFETCH_TYPE_CURRENT > PREFETCH_TYPE_SIMPLE) && \
>+        ((src_bpp_ == 24) || (mask_bpp_ == 24) || (dst_w_bpp_ == 24))
>+    .set PREFETCH_TYPE_CURRENT, PREFETCH_TYPE_SIMPLE
>+.endif
>+
>+/*
>+ * Make some macro arguments globally visible and accessible
>+ * from other macros
>+ */
>+    .set src_bpp, src_bpp_
>+    .set mask_bpp, mask_bpp_
>+    .set dst_w_bpp, dst_w_bpp_
>+    .set pixblock_size, pixblock_size_
>+    .set dst_w_basereg, dst_w_basereg_
>+    .set dst_r_basereg, dst_r_basereg_
>+    .set src_basereg, src_basereg_
>+    .set mask_basereg, mask_basereg_
>+
>+    .macro pixld_src x:vararg
>+        pixld x
>+    .endm
>+    .macro fetch_src_pixblock
>+        pixld_src   pixblock_size, src_bpp, \
>+                    (src_basereg - pixblock_size * src_bpp / 64), SRC
>+    .endm
>+/*
>+ * Assign symbolic names to registers
>+ */
>+    W           .req       x0      /* width (is updated during processing) */
>+    H           .req       x1      /* height (is updated during processing) */
>+    DST_W       .req       x2      /* destination buffer pointer for writes */
>+    DST_STRIDE  .req       x3      /* destination image stride */
>+    SRC         .req       x4      /* source buffer pointer */
>+    SRC_STRIDE  .req       x5      /* source image stride */
>+    MASK        .req       x6      /* mask pointer */
>+    MASK_STRIDE .req       x7      /* mask stride */
>+
>+    DST_R       .req       x8      /* destination buffer pointer for reads */
>+
>+    PF_CTL      .req       x9      /* combined lines counter and prefetch */
>+                                    /* distance increment counter */
>+    PF_X        .req       x10     /* pixel index in a scanline for current */
>+                                    /* pretetch position */
>+    PF_SRC      .req       x11     /* pointer to source scanline start */
>+                                    /* for prefetch purposes */
>+    PF_DST      .req       x12     /* pointer to destination scanline start */
>+                                    /* for prefetch purposes */
>+    PF_MASK     .req       x13     /* pointer to mask scanline start */
>+                                    /* for prefetch purposes */
>+
>+    ORIG_W      .req       x14     /* saved original width */
>+    DUMMY       .req       x15     /* temporary register */
>+
>+    sxtw        x0, w0
>+    sxtw        x1, w1
>+    sxtw        x3, w3
>+    sxtw        x5, w5
>+    sxtw        x7, w7
>+
>+    .set mask_bpp_shift, -1
>+.if src_bpp == 32
>+    .set src_bpp_shift, 2
>+.elseif src_bpp == 24
>+    .set src_bpp_shift, 0
>+.elseif src_bpp == 16
>+    .set src_bpp_shift, 1
>+.elseif src_bpp == 8
>+    .set src_bpp_shift, 0
>+.elseif src_bpp == 0
>+    .set src_bpp_shift, -1
>+.else
>+    .error "requested src bpp (src_bpp) is not supported"
>+.endif
>+.if mask_bpp == 32
>+    .set mask_bpp_shift, 2
>+.elseif mask_bpp == 24
>+    .set mask_bpp_shift, 0
>+.elseif mask_bpp == 8
>+    .set mask_bpp_shift, 0
>+.elseif mask_bpp == 0
>+    .set mask_bpp_shift, -1
>+.else
>+    .error "requested mask bpp (mask_bpp) is not supported"
>+.endif
>+.if dst_w_bpp == 32
>+    .set dst_bpp_shift, 2
>+.elseif dst_w_bpp == 24
>+    .set dst_bpp_shift, 0
>+.elseif dst_w_bpp == 16
>+    .set dst_bpp_shift, 1
>+.elseif dst_w_bpp == 8
>+    .set dst_bpp_shift, 0
>+.else
>+    .error "requested dst bpp (dst_w_bpp) is not supported"
>+.endif
>+
>+.if (((flags) & FLAG_DST_READWRITE) != 0)
>+    .set dst_r_bpp, dst_w_bpp
>+.else
>+    .set dst_r_bpp, 0
>+.endif
>+.if (((flags) & FLAG_DEINTERLEAVE_32BPP) != 0)
>+    .set DEINTERLEAVE_32BPP_ENABLED, 1
>+.else
>+    .set DEINTERLEAVE_32BPP_ENABLED, 0
>+.endif
>+
>+.if prefetch_distance < 0 || prefetch_distance > 15
>+    .error "invalid prefetch distance (prefetch_distance)"
>+.endif
>+
>+    PF mov      PF_X, #0
>+    mov         DST_R, DST_W
>+
>+.if src_bpp == 24
>+    sub         SRC_STRIDE, SRC_STRIDE, W
>+    sub         SRC_STRIDE, SRC_STRIDE, W, lsl #1
>+.endif
>+.if mask_bpp == 24
>+    sub         MASK_STRIDE, MASK_STRIDE, W
>+    sub         MASK_STRIDE, MASK_STRIDE, W, lsl #1
>+.endif
>+.if dst_w_bpp == 24
>+    sub         DST_STRIDE, DST_STRIDE, W
>+    sub         DST_STRIDE, DST_STRIDE, W, lsl #1
>+.endif
>+
>+/*
>+ * Setup advanced prefetcher initial state
>+ */
>+    PF mov      PF_SRC, SRC
>+    PF mov      PF_DST, DST_R
>+    PF mov      PF_MASK, MASK
>+    /* PF_CTL = prefetch_distance | ((h - 1) << 4) */
>+    PF lsl      DUMMY, H, #4
>+    PF mov      PF_CTL, DUMMY
>+    PF add      PF_CTL, PF_CTL, #(prefetch_distance - 0x10)
>+
>+    init
>+    subs        H, H, #1
>+    mov         ORIG_W, W
>+    blt         9f
>+    cmp         W, #(pixblock_size * 2)
>+    blt         800f
>+/*
>+ * This is the start of the pipelined loop, which if optimized for
>+ * long scanlines
>+ */
>+0:
>+    ensure_destination_ptr_alignment process_pixblock_head, \
>+                                     process_pixblock_tail, \
>+                                     process_pixblock_tail_head
>+
>+    /* Implement "head (tail_head) ... (tail_head) tail" loop pattern */
>+    pixld_a     pixblock_size, dst_r_bpp, \
>+                (dst_r_basereg - pixblock_size * dst_r_bpp / 64), DST_R
>+    fetch_src_pixblock
>+    pixld       pixblock_size, mask_bpp, \
>+                (mask_basereg - pixblock_size * mask_bpp / 64), MASK
>+    PF add      PF_X, PF_X, #pixblock_size
>+    process_pixblock_head
>+    cache_preload 0, pixblock_size
>+    cache_preload_simple
>+    subs        W, W, #(pixblock_size * 2)
>+    blt         200f
>+
>+100:
>+    process_pixblock_tail_head
>+    cache_preload_simple
>+    subs        W, W, #pixblock_size
>+    bge         100b
>+
>+200:
>+    process_pixblock_tail
>+    pixst_a     pixblock_size, dst_w_bpp, \
>+                (dst_w_basereg - pixblock_size * dst_w_bpp / 64), DST_W
>+
>+    /* Process the remaining trailing pixels in the scanline */
>+    process_trailing_pixels 1, 1, \
>+                            process_pixblock_head, \
>+                            process_pixblock_tail, \
>+                            process_pixblock_tail_head
>+    advance_to_next_scanline 0b
>+
>+    cleanup
>+1000:
>+    /* pop all registers */
>+    sub         x29, x29, 64
>+    ld1         {v8.8b, v9.8b, v10.8b, v11.8b}, [x29], 32
>+    ld1         {v12.8b, v13.8b, v14.8b, v15.8b}, [x29], 32
>+    ldp          x8,   x9, [x29, -80]
>+    ldp         x10,  x11, [x29, -96]
>+    ldp         x12,  x13, [x29, -112]
>+    ldp         x14,  x15, [x29, -128]
>+    ldp         x16,  x17, [x29, -144]
>+    ldp         x18,  x19, [x29, -160]
>+    ldp         x20,  x21, [x29, -176]
>+    ldp         x22,  x23, [x29, -192]
>+    ldp         x24,  x25, [x29, -208]
>+    ldp         x26,  x27, [x29, -224]
>+    ldr         x28, [x29, -232]
>+    mov         sp, x29
>+    ldp         x29, x30, [sp], 16 
>+    ret  /* exit */
>+/*
>+ * This is the start of the loop, designed to process images with small width
>+ * (less than pixblock_size * 2 pixels). In this case neither pipelining
>+ * nor prefetch are used.
>+ */
>+800:
>+    /* Process exactly pixblock_size pixels if needed */
>+    tst         W, #pixblock_size
>+    beq         100f
>+    pixld       pixblock_size, dst_r_bpp, \
>+                (dst_r_basereg - pixblock_size * dst_r_bpp / 64), DST_R
>+    fetch_src_pixblock
>+    pixld       pixblock_size, mask_bpp, \
>+                (mask_basereg - pixblock_size * mask_bpp / 64), MASK
>+    process_pixblock_head
>+    process_pixblock_tail
>+    pixst       pixblock_size, dst_w_bpp, \
>+                (dst_w_basereg - pixblock_size * dst_w_bpp / 64), DST_W
>+100:
>+    /* Process the remaining trailing pixels in the scanline */
>+    process_trailing_pixels 0, 0, \
>+                            process_pixblock_head, \
>+                            process_pixblock_tail, \
>+                            process_pixblock_tail_head
>+    advance_to_next_scanline 800b
>+9:
>+    cleanup
>+    /* pop all registers */
>+    sub         x29, x29, 64
>+    ld1         {v8.8b, v9.8b, v10.8b, v11.8b}, [x29], 32
>+    ld1         {v12.8b, v13.8b, v14.8b, v15.8b}, [x29], 32
>+    ldp          x8,   x9, [x29, -80]
>+    ldp         x10,  x11, [x29, -96]
>+    ldp         x12,  x13, [x29, -112]
>+    ldp         x14,  x15, [x29, -128]
>+    ldp         x16,  x17, [x29, -144]
>+    ldp         x18,  x19, [x29, -160]
>+    ldp         x20,  x21, [x29, -176]
>+    ldp         x22,  x23, [x29, -192]
>+    ldp         x24,  x25, [x29, -208]
>+    ldp         x26,  x27, [x29, -224]
>+    ldr         x28, [x29, -232]
>+    mov         sp, x29
>+    ldp         x29, x30, [sp], 16 
>+    ret  /* exit */
>+
>+    .purgem     fetch_src_pixblock
>+    .purgem     pixld_src
>+
>+    .unreq      SRC
>+    .unreq      MASK
>+    .unreq      DST_R
>+    .unreq      DST_W
>+    .unreq      ORIG_W
>+    .unreq      W
>+    .unreq      H
>+    .unreq      SRC_STRIDE
>+    .unreq      DST_STRIDE
>+    .unreq      MASK_STRIDE
>+    .unreq      PF_CTL
>+    .unreq      PF_X
>+    .unreq      PF_SRC
>+    .unreq      PF_DST
>+    .unreq      PF_MASK
>+    .unreq      DUMMY
>+    .endfunc
>+.endm
>+
>+/*
>+ * A simplified variant of function generation template for a single
>+ * scanline processing (for implementing pixman combine functions)
>+ */
>+.macro generate_composite_function_scanline        use_nearest_scaling, \
>+                                                   fname, \
>+                                                   src_bpp_, \
>+                                                   mask_bpp_, \
>+                                                   dst_w_bpp_, \
>+                                                   flags, \
>+                                                   pixblock_size_, \
>+                                                   init, \
>+                                                   cleanup, \
>+                                                   process_pixblock_head, \
>+                                                   process_pixblock_tail, \
>+                                                   process_pixblock_tail_head, \
>+                                                   dst_w_basereg_ = 28, \
>+                                                   dst_r_basereg_ = 4, \
>+                                                   src_basereg_   = 0, \
>+                                                   mask_basereg_  = 24
>+
>+    pixman_asm_function fname
>+    .set PREFETCH_TYPE_CURRENT, PREFETCH_TYPE_NONE
>+
>+/*
>+ * Make some macro arguments globally visible and accessible
>+ * from other macros
>+ */
>+    .set src_bpp, src_bpp_
>+    .set mask_bpp, mask_bpp_
>+    .set dst_w_bpp, dst_w_bpp_
>+    .set pixblock_size, pixblock_size_
>+    .set dst_w_basereg, dst_w_basereg_
>+    .set dst_r_basereg, dst_r_basereg_
>+    .set src_basereg, src_basereg_
>+    .set mask_basereg, mask_basereg_
>+    
>+.if use_nearest_scaling != 0
>+    /*
>+     * Assign symbolic names to registers for nearest scaling
>+     */
>+    W           .req        x0
>+    DST_W       .req        x1
>+    SRC         .req        x2
>+    VX          .req        x3
>+    UNIT_X      .req        x4
>+    SRC_WIDTH_FIXED .req    x5
>+    MASK        .req        x6
>+    TMP1        .req        x8
>+    TMP2        .req        x9
>+    DST_R       .req        x10
>+    DUMMY       .req        x30
>+
>+    .macro pixld_src x:vararg
>+        pixld_s x
>+    .endm
>+
>+    sxtw        x0, w0
>+    sxtw        x3, w3
>+    sxtw        x4, w4
>+    sxtw        x5, w5
>+
>+    stp         x29, x30, [sp, -16]!
>+    mov         x29, sp
>+    sub         sp, sp, 88
>+    sub         x29, x29, 64
>+    st1         {v8.8b, v9.8b, v10.8b, v11.8b}, [x29], 32
>+    st1         {v12.8b, v13.8b, v14.8b, v15.8b}, [x29], 32
>+    stp         x8, x9, [x29, -80]
>+    str         x10, [x29, -88]
>+.else
>+    /*
>+     * Assign symbolic names to registers
>+     */
>+    W           .req        x0      /* width (is updated during processing) */
>+    DST_W       .req        x1      /* destination buffer pointer for writes */
>+    SRC         .req        x2      /* source buffer pointer */
>+    MASK        .req        x3      /* mask pointer */
>+    DST_R       .req        x4      /* destination buffer pointer for reads */
>+    DUMMY       .req        x30
>+
>+    .macro pixld_src x:vararg
>+        pixld x
>+    .endm
>+
>+    sxtw        x0, w0
>+
>+    stp         x29, x30, [sp, -16]!
>+    mov         x29, sp
>+    sub         sp, sp, 64
>+    sub         x29, x29, 64
>+    st1         {v8.8b, v9.8b, v10.8b, v11.8b}, [x29], 32
>+    st1         {v12.8b, v13.8b, v14.8b, v15.8b}, [x29], 32
>+.endif
>+
>+.if (((flags) & FLAG_DST_READWRITE) != 0)
>+    .set dst_r_bpp, dst_w_bpp
>+.else
>+    .set dst_r_bpp, 0
>+.endif
>+.if (((flags) & FLAG_DEINTERLEAVE_32BPP) != 0)
>+    .set DEINTERLEAVE_32BPP_ENABLED, 1
>+.else
>+    .set DEINTERLEAVE_32BPP_ENABLED, 0
>+.endif
>+
>+    .macro fetch_src_pixblock
>+        pixld_src   pixblock_size, src_bpp, \
>+                    (src_basereg - pixblock_size * src_bpp / 64), SRC
>+    .endm
>+
>+    init
>+    mov         DST_R, DST_W
>+
>+    cmp         W, #pixblock_size
>+    blt         800f
>+
>+    ensure_destination_ptr_alignment process_pixblock_head, \
>+                                     process_pixblock_tail, \
>+                                     process_pixblock_tail_head
>+
>+    subs        W, W, #pixblock_size
>+    blt         700f
>+
>+    /* Implement "head (tail_head) ... (tail_head) tail" loop pattern */
>+    pixld_a     pixblock_size, dst_r_bpp, \
>+                (dst_r_basereg - pixblock_size * dst_r_bpp / 64), DST_R
>+    fetch_src_pixblock
>+    pixld       pixblock_size, mask_bpp, \
>+                (mask_basereg - pixblock_size * mask_bpp / 64), MASK
>+    process_pixblock_head
>+    subs        W, W, #pixblock_size
>+    blt         200f
>+100:
>+    process_pixblock_tail_head
>+    subs        W, W, #pixblock_size
>+    bge         100b
>+200:
>+    process_pixblock_tail
>+    pixst_a     pixblock_size, dst_w_bpp, \
>+                (dst_w_basereg - pixblock_size * dst_w_bpp / 64), DST_W
>+700:
>+    /* Process the remaining trailing pixels in the scanline (dst aligned) */
>+    process_trailing_pixels 0, 1, \
>+                            process_pixblock_head, \
>+                            process_pixblock_tail, \
>+                            process_pixblock_tail_head
>+
>+    cleanup
>+.if use_nearest_scaling != 0
>+    sub         x29, x29, 64
>+    ld1         {v8.8b, v9.8b, v10.8b, v11.8b}, [x29], 32
>+    ld1         {v12.8b, v13.8b, v14.8b, v15.8b}, [x29], 32
>+    ldp         x8, x9, [x29, -80]
>+    ldr         x10, [x29, -96]
>+    mov         sp, x29
>+    ldp         x29, x30, [sp], 16 
>+    ret  /* exit */
>+.else
>+    sub         x29, x29, 64
>+    ld1         {v8.8b, v9.8b, v10.8b, v11.8b}, [x29], 32
>+    ld1         {v12.8b, v13.8b, v14.8b, v15.8b}, [x29], 32
>+    mov         sp, x29
>+    ldp         x29, x30, [sp], 16 
>+    ret  /* exit */
>+.endif
>+800:
>+    /* Process the remaining trailing pixels in the scanline (dst unaligned) */
>+    process_trailing_pixels 0, 0, \
>+                            process_pixblock_head, \
>+                            process_pixblock_tail, \
>+                            process_pixblock_tail_head
>+
>+    cleanup
>+.if use_nearest_scaling != 0
>+    sub         x29, x29, 64
>+    ld1         {v8.8b, v9.8b, v10.8b, v11.8b}, [x29], 32
>+    ld1         {v12.8b, v13.8b, v14.8b, v15.8b}, [x29], 32
>+    ldp         x8, x9, [x29, -80]
>+    ldr         x10, [x29, -88]
>+    mov         sp, x29
>+    ldp         x29, x30, [sp], 16 
>+    ret  /* exit */
>+
>+    .unreq      DUMMY
>+    .unreq      DST_R
>+    .unreq      SRC
>+    .unreq      W
>+    .unreq      VX
>+    .unreq      UNIT_X
>+    .unreq      TMP1
>+    .unreq      TMP2
>+    .unreq      DST_W
>+    .unreq      MASK
>+    .unreq      SRC_WIDTH_FIXED
>+
>+.else
>+    sub         x29, x29, 64
>+    ld1         {v8.8b, v9.8b, v10.8b, v11.8b}, [x29], 32
>+    ld1         {v12.8b, v13.8b, v14.8b, v15.8b}, [x29], 32
>+    mov          sp, x29
>+    ldp          x29, x30, [sp], 16 
>+    ret  /* exit */
>+
>+    .unreq      DUMMY
>+    .unreq      SRC
>+    .unreq      MASK
>+    .unreq      DST_R
>+    .unreq      DST_W
>+    .unreq      W
>+.endif
>+
>+    .purgem     fetch_src_pixblock
>+    .purgem     pixld_src
>+
>+    .endfunc
>+.endm
>+
>+.macro generate_composite_function_single_scanline x:vararg
>+    generate_composite_function_scanline 0, x
>+.endm
>+
>+.macro generate_composite_function_nearest_scanline x:vararg
>+    generate_composite_function_scanline 1, x
>+.endm
>+
>+/* Default prologue/epilogue, nothing special needs to be done */
>+
>+.macro default_init
>+.endm
>+
>+.macro default_cleanup
>+.endm
>+
>+/*
>+ * Prologue/epilogue variant which additionally saves/restores v8-v15
>+ * registers (they need to be saved/restored by callee according to ABI).
>+ * This is required if the code needs to use all the NEON registers.
>+ */
>+
>+.macro default_init_need_all_regs
>+.endm
>+
>+.macro default_cleanup_need_all_regs
>+.endm
>+
>+/******************************************************************************/
>+
>+/*
>+ * Conversion of 8 r5g6b6 pixels packed in 128-bit register (in)
>+ * into a planar a8r8g8b8 format (with a, r, g, b color components
>+ * stored into 64-bit registers out_a, out_r, out_g, out_b respectively).
>+ *
>+ * Warning: the conversion is destructive and the original
>+ *          value (in) is lost.
>+ */
>+.macro convert_0565_to_8888 in, out_a, out_r, out_g, out_b
>+    shrn        &out_r&.8b, &in&.8h,    #8
>+    shrn        &out_g&.8b, &in&.8h,    #3
>+    sli         &in&.8h,    &in&.8h,    #5
>+    movi        &out_a&.8b, #255
>+    sri         &out_r&.8b, &out_r&.8b, #5
>+    sri         &out_g&.8b, &out_g&.8b, #6
>+    shrn        &out_b&.8b, &in&.8h,    #2
>+.endm
>+
>+.macro convert_0565_to_x888 in, out_r, out_g, out_b
>+    shrn        &out_r&.8b, &in&.8h,    #8
>+    shrn        &out_g&.8b, &in&.8h,    #3
>+    sli         &in&.8h,    &in&.8h,    #5
>+    sri         &out_r&.8b, &out_r&.8b, #5
>+    sri         &out_g&.8b, &out_g&.8b, #6
>+    shrn        &out_b&.8b, &in&.8h,    #2
>+.endm
>+
>+/*
>+ * Conversion from planar a8r8g8b8 format (with a, r, g, b color components
>+ * in 64-bit registers in_a, in_r, in_g, in_b respectively) into 8 r5g6b6
>+ * pixels packed in 128-bit register (out). Requires two temporary 128-bit
>+ * registers (tmp1, tmp2)
>+ */
>+.macro convert_8888_to_0565 in_r, in_g, in_b, out, tmp1, tmp2
>+    ushll       &tmp1&.8h, &in_g&.8b, #7
>+    shl         &tmp1&.8h, &tmp1&.8h, #1
>+    ushll       &out&.8h,  &in_r&.8b, #7
>+    shl         &out&.8h,  &out&.8h,  #1
>+    ushll       &tmp2&.8h, &in_b&.8b, #7
>+    shl         &tmp2&.8h, &tmp2&.8h, #1
>+    sri         &out&.8h, &tmp1&.8h, #5
>+    sri         &out&.8h, &tmp2&.8h, #11
>+.endm
>+
>+/*
>+ * Conversion of four r5g6b5 pixels (in) to four x8r8g8b8 pixels
>+ * returned in (out0, out1) registers pair. Requires one temporary
>+ * 64-bit register (tmp). 'out1' and 'in' may overlap, the original
>+ * value from 'in' is lost
>+ */
>+.macro convert_four_0565_to_x888_packed in, out0, out1, tmp
>+    shl         &out0&.4h, &in&.4h,   #5  /* G top 6 bits */
>+    shl         &tmp&.4h,  &in&.4h,   #11 /* B top 5 bits */
>+    sri         &in&.4h,   &in&.4h,   #5  /* R is ready in top bits */
>+    sri         &out0&.4h, &out0&.4h, #6  /* G is ready in top bits */
>+    sri         &tmp&.4h,  &tmp&.4h,  #5  /* B is ready in top bits */
>+    ushr        &out1&.4h, &in&.4h,   #8  /* R is in place */
>+    sri         &out0&.4h, &tmp&.4h,  #8  /* G & B is in place */
>+    zip1        &tmp&.4h,  &out0&.4h, &out1&.4h  /* everything is in place */
>+    zip2        &out1&.4h, &out0&.4h, &out1&.4h
>+    mov         &out0&.d[0], &tmp&.d[0]
>+.endm
>diff --git a/pixman/pixman-private.h b/pixman/pixman-private.h
>index 73a5414..22c8ccc 100644
>--- a/pixman/pixman-private.h
>+++ b/pixman/pixman-private.h
>@@ -607,6 +607,11 @@ pixman_implementation_t *
> _pixman_implementation_create_arm_neon (pixman_implementation_t *fallback);
> #endif
> 
>+#ifdef USE_ARM_A64_NEON
>+pixman_implementation_t *
>+_pixman_implementation_create_arm_neon (pixman_implementation_t *fallback);
>+#endif
>+
> #ifdef USE_MIPS_DSPR2
> pixman_implementation_t *
> _pixman_implementation_create_mips_dspr2 (pixman_implementation_t *fallback);
>-- 
>2.7.4
>

Actions: View | Splinter Review

Attachments on bug 94758: 122634 | 122635 | 122636 | 122695 | 122816 | 122937 | 123008 | 123009 | 123010

Use of freedesktop.org services, including Bugzilla, is subject to our Code of Conduct. How we collect and use information is described in our Privacy Policy.