;*****************************************************************************
;* deblock-a.asm: x86 deblocking
;*****************************************************************************
;* Copyright (C) 2005-2016 x264 project
;*
;* Authors: Loren Merritt <lorenm@u.washington.edu>
;* Fiona Glaser <fiona@x264.com>
;* Oskar Arvidsson <oskar@irock.se>
;*
;* This program is free software; you can redistribute it and/or modify
;* it under the terms of the GNU General Public License as published by
;* the Free Software Foundation; either version 2 of the License, or
;* (at your option) any later version.
;*
;* This program is distributed in the hope that it will be useful,
;* but WITHOUT ANY WARRANTY; without even the implied warranty of
;* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
;* GNU General Public License for more details.
;*
;* You should have received a copy of the GNU General Public License
;* along with this program; if not, write to the Free Software
;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111, USA.
;*
;* This program is also available under a commercial proprietary license.
;* For more information, contact us at licensing@x264.com.
;*****************************************************************************
%include "x86inc.asm"
%include "x86util.asm"
SECTION_RODATA 32
load_bytes_shuf: times 2 db 3,4,5,6,11,12,13,14,4,5,6,7,12,13,14,15
insert_top_shuf: dd 0,1,4,5,7,2,3,6
transpose_shuf: db 0,4,8,12,1,5,9,13,2,6,10,14,3,7,11,15
SECTION .text
cextern pb_0
cextern pb_1
cextern pb_3
cextern pb_a1
cextern pw_2
cextern pw_4
cextern pw_00ff
cextern pw_pixel_max
cextern pb_unpackbd1
%if HIGH_BIT_DEPTH
; out: %4 = |%1-%2|-%3
; clobbers: %5
%macro ABS_SUB 5
psubusw %5, %2, %1
psubusw %4, %1, %2
por %4, %5
psubw %4, %3
%endmacro
; out: %4 = |%1-%2|<%3
%macro DIFF_LT 5
psubusw %4, %2, %1
psubusw %5, %1, %2
por %5, %4 ; |%1-%2|
pxor %4, %4
psubw %5, %3 ; |%1-%2|-%3
pcmpgtw %4, %5 ; 0 > |%1-%2|-%3
%endmacro
%macro LOAD_AB 4
movd %1, %3
movd %2, %4
SPLATW %1, %1
SPLATW %2, %2
%endmacro
; in: %2=tc reg
; out: %1=splatted tc
%macro LOAD_TC 2
%if mmsize == 8
pshufw %1, [%2-1], 0
%else
movd %1, [%2]
punpcklbw %1, %1
pshuflw %1, %1, q1100
pshufd %1, %1, q1100
%endif
psraw %1, 8
%endmacro
; in: %1=p1, %2=p0, %3=q0, %4=q1
; %5=alpha, %6=beta, %7-%9=tmp
; out: %7=mask
%macro LOAD_MASK 9
ABS_SUB %2, %3, %5, %8, %7 ; |p0-q0| - alpha
ABS_SUB %1, %2, %6, %9, %7 ; |p1-p0| - beta
pand %8, %9
ABS_SUB %3, %4, %6, %9, %7 ; |q1-q0| - beta
pxor %7, %7
pand %8, %9
pcmpgtw %7, %8
%endmacro
; in: %1=p0, %2=q0, %3=p1, %4=q1, %5=mask, %6=tmp, %7=tmp
; out: %1=p0', m2=q0'
%macro DEBLOCK_P0_Q0 7
psubw %3, %4
pxor %7, %7
paddw %3, [pw_4]
psubw %7, %5
psubw %6, %2, %1
psllw %6, 2
paddw %3, %6
psraw %3, 3
mova %6, [pw_pixel_max]
CLIPW %3, %7, %5
pxor %7, %7
paddw %1, %3
psubw %2, %3
CLIPW %1, %7, %6
CLIPW %2, %7, %6
%endmacro
; in: %1=x2, %2=x1, %3=p0, %4=q0 %5=mask&tc, %6=tmp
%macro LUMA_Q1 6
pavgw %6, %3, %4 ; (p0+q0+1)>>1
paddw %1, %6
pxor %6, %6
psraw %1, 1
psubw %6, %5
psubw %1, %2
CLIPW %1, %6, %5
paddw %1, %2
%endmacro
%macro LUMA_DEBLOCK_ONE 3
DIFF_LT m5, %1, bm, m4, m6
pxor m6, m6
mova %3, m4
pcmpgtw m6, tcm
pand m4, tcm
pandn m6, m7
pand m4, m6
LUMA_Q1 m5, %2, m1, m2, m4, m6
%endmacro
%macro LUMA_H_STORE 2
%if mmsize == 8
movq [r0-4], m0
movq [r0+r1-4], m1
movq [r0+r1*2-4], m2
movq [r0+%2-4], m3
%else
movq [r0-4], m0
movhps [r0+r1-4], m0
movq [r0+r1*2-4], m1
movhps [%1-4], m1
movq [%1+r1-4], m2
movhps [%1+r1*2-4], m2
movq [%1+%2-4], m3
movhps [%1+r1*4-4], m3
%endif
%endmacro
%macro DEBLOCK_LUMA 0
;-----------------------------------------------------------------------------
; void deblock_v_luma( uint16_t *pix, intptr_t stride, int alpha, int beta, int8_t *tc0 )
;-----------------------------------------------------------------------------
cglobal deblock_v_luma, 5,5,8,0-5*mmsize
%define tcm [rsp]
%define ms1 [rsp+mmsize]
%define ms2 [rsp+mmsize*2]
%define am [rsp+mmsize*3]
%define bm [rsp+mmsize*4]
add r1, r1
LOAD_AB m4, m5, r2d, r3d
mov r3, 32/mmsize
mov r2, r0
sub r0, r1
mova am, m4
sub r0, r1
mova bm, m5
sub r0, r1
.loop:
mova m0, [r0+r1]
mova m1, [r0+r1*2]
mova m2, [r2]
mova m3, [r2+r1]
LOAD_MASK m0, m1, m2, m3, am, bm, m7, m4, m6
LOAD_TC m6, r4
mova tcm, m6
mova m5, [r0]
LUMA_DEBLOCK_ONE m1, m0, ms1
mova [r0+r1], m5
mova m5, [r2+r1*2]
LUMA_DEBLOCK_ONE m2, m3, ms2
mova [r2+r1], m5
pxor m5, m5
mova m6, tcm
pcmpgtw m5, tcm
psubw m6, ms1
pandn m5, m7
psubw m6, ms2
pand m5, m6
DEBLOCK_P0_Q0 m1, m2, m0, m3, m5, m7, m6
mova [r0+r1*2], m1
mova [r2], m2
add r0, mmsize
add r2, mmsize
add r4, mmsize/8
dec r3
jg .loop
RET
cglobal deblock_h_luma, 5,6,8,0-7*mmsize
%define tcm [rsp]
%define ms1 [rsp+mmsize]
%define ms2 [rsp+mmsize*2]
%define p1m [rsp+mmsize*3]
%define p2m [rsp+mmsize*4]
%define am [rsp+mmsize*5]
%define bm [rsp+mmsize*6]
add r1, r1
LOAD_AB m4, m5, r2d, r3d
mov r3, r1
mova am, m4
add r3, r1
mov r5, 32/mmsize
mova bm, m5
add r3, r1
%if mmsize == 16
mov r2, r0
add r2, r3
%endif
.loop:
%if mmsize == 8
movq m2, [r0-8] ; y q2 q1 q0
movq m7, [r0+0]
movq m5, [r0+r1-8]
movq m3, [r0+r1+0]
movq m0, [r0+r1*2-8]
movq m6, [r0+r1*2+0]
movq m1, [r0+r3-8]
TRANSPOSE4x4W 2, 5, 0, 1, 4
SWAP 2, 7
movq m7, [r0+r3]
TRANSPOSE4x4W 2, 3, 6, 7, 4
%else
movu m5, [r0-8] ; y q2 q1 q0 p0 p1 p2 x
movu m0, [r0+r1-8]
movu m2, [r0+r1*2-8]
movu m3, [r2-8]
TRANSPOSE4x4W 5, 0, 2, 3, 6
mova tcm, m3
movu m4, [r2+r1-8]
movu m1, [r2+r1*2-8]
movu m3, [r2+r3-8]
movu m7, [r2+r1*4-8]
TRANSPOSE4x4W 4, 1, 3, 7, 6
mova m6, tcm
punpcklqdq m6, m7
punpckhqdq m5, m4
SBUTTERFLY qdq, 0, 1, 7
SBUTTERFLY qdq, 2, 3, 7
%endif
mova p2m, m6
LOAD_MASK m0, m1, m2, m3, am, bm, m7, m4, m6
LOAD_TC m6, r4
mova tcm, m6
LUMA_DEBLOCK_ONE m1, m0, ms1
mova p1m, m5
mova m5, p2m
LUMA_DEBLOCK_ONE m2, m3, ms2
mova p2m, m5
pxor m5, m5
mova m6, tcm
pcmpgtw m5, tcm
psubw m6, ms1
pandn m5, m7
psubw m6, ms2
pand m5, m6
DEBLOCK_P0_Q0 m1, m2, m0, m3, m5, m7, m6
mova m0, p1m
mova m3, p2m
TRANSPOSE4x4W 0, 1, 2, 3, 4
LUMA_H_STORE r2, r3
add r4, mmsize/8
lea r0, [r0+r1*(mmsize/2)]
lea r2, [r2+r1*(mmsize/2)]
dec r5
jg .loop
RET
%endmacro
%if ARCH_X86_64
; in: m0=p1, m1=p0, m2=q0, m3=q1, m8=p2, m9=q2
; m12=alpha, m13=beta
; out: m0=p1', m3=q1', m1=p0', m2=q0'
; clobbers: m4, m5, m6, m7, m10, m11, m14
%macro DEBLOCK_LUMA_INTER_SSE2 0
LOAD_MASK m0, m1, m2, m3, m12, m13, m7, m4, m6
LOAD_TC m6, r4
DIFF_LT m8, m1, m13, m10, m4
DIFF_LT m9, m2, m13, m11, m4
pand m6, m7
mova m14, m6
pxor m4, m4
pcmpgtw m6, m4
pand m6, m14
mova m5, m10
pand m5, m6
LUMA_Q1 m8, m0, m1, m2, m5, m4
mova m5, m11
pand m5, m6
LUMA_Q1 m9, m3, m1, m2, m5, m4
pxor m4, m4
psubw m6, m10
pcmpgtw m4, m14
pandn m4, m7
psubw m6, m11
pand m4, m6
DEBLOCK_P0_Q0 m1, m2, m0, m3, m4, m5, m6
SWAP 0, 8
SWAP 3, 9
%endmacro
%macro DEBLOCK_LUMA_64 0
cglobal deblock_v_luma, 5,5,15
%define p2 m8
%define p1 m0
%define p0 m1
%define q0 m2
%define q1 m3
%define q2 m9
%define mask0 m7
%define mask1 m10
%define mask2 m11
add r1, r1
LOAD_AB m12, m13, r2d, r3d
mov r2, r0
sub r0, r1
sub r0, r1
sub r0, r1
mov r3, 2
.loop:
mova p2, [r0]
mova p1, [r0+r1]
mova p0, [r0+r1*2]
mova q0, [r2]
mova q1, [r2+r1]
mova q2, [r2+r1*2]
DEBLOCK_LUMA_INTER_SSE2
mova [r0+r1], p1
mova [r0+r1*2], p0
mova [r2], q0
mova [r2+r1], q1
add r0, mmsize
add r2, mmsize
add r4, 2
dec r3
jg .loop
RET
cglobal deblock_h_luma, 5,7,15
add r1, r1
LOAD_AB m12, m13, r2d, r3d
mov r2, r1
add r2, r1
add r2, r1
mov r5, r0
add r5, r2
mov r6, 2
.loop:
movu m8, [r0-8] ; y q2 q1 q0 p0 p1 p2 x
movu m0, [r0+r1-8]
movu m2, [r0+r1*2-8]
movu m9, [r5-8]
movu m5, [r5+r1-8]
movu m1, [r5+r1*2-8]
movu m3, [r5+r2-8]
movu m7, [r5+r1*4-8]
TRANSPOSE4x4W 8, 0, 2, 9, 10
TRANSPOSE4x4W 5, 1, 3, 7, 10
punpckhqdq m8, m5
SBUTTERFLY qdq, 0, 1, 10
SBUTTERFLY qdq, 2, 3, 10
punpcklqdq m9, m7
DEBLOCK_LUMA_INTER_SSE2
TRANSPOSE4x4W 0, 1, 2, 3, 4
LUMA_H_STORE r5, r2
add r4, 2
lea r0, [r0+r1*8]
lea r5, [r5+r1*8]
dec r6
jg .loop
RET
%endmacro
INIT_XMM sse2
DEBLOCK_LUMA_64
INIT_XMM avx
DEBLOCK_LUMA_64
%endif
%macro SWAPMOVA 2
%ifid %1
SWAP %1, %2
%else
mova %1, %2
%endif
%endmacro
; in: t0-t2: tmp registers
; %1=p0 %2=p1 %3=p2 %4=p3 %5=q0 %6=q1 %7=mask0
; %8=mask1p %9=2 %10=p0' %11=p1' %12=p2'
%macro LUMA_INTRA_P012 12 ; p0..p3 in memory
%if ARCH_X86_64
paddw t0, %3, %2
mova t2, %4
paddw t2, %3
%else
mova t0, %3
mova t2, %4
paddw t0, %2
paddw t2, %3
%endif
paddw t0, %1
paddw t2, t2
paddw t0, %5
paddw t2, %9
paddw t0, %9 ; (p2 + p1 + p0 + q0 + 2)
paddw t2, t0 ; (2*p3 + 3*p2 + p1 + p0 + q0 + 4)
psrlw t2, 3
psrlw t1, t0, 2
psubw t2, %3
psubw t1, %2
pand t2, %8
pand t1, %8
paddw t2, %3
paddw t1, %2
SWAPMOVA %11, t1
psubw t1, t0, %3
paddw t0, t0
psubw t1, %5
psubw t0, %3
paddw t1, %6
paddw t1, %2
paddw t0, %6
psrlw t1, 2 ; (2*p1 + p0 + q1 + 2)/4
psrlw t0, 3 ; (p2 + 2*p1 + 2*p0 + 2*q0 + q1 + 4)>>3
pxor t0, t1
pxor t1, %1
pand t0, %8
pand t1, %7
pxor t0, t1
pxor t0, %1
SWAPMOVA %10, t0
SWAPMOVA %12, t2
%endmacro
%macro LUMA_INTRA_INIT 1
%define t0 m4
%define t1 m5
%define t2 m6
%define t3 m7
%assign i 4
%rep %1
CAT_XDEFINE t, i, [rsp+mmsize*(i-4)]
%assign i i+1
%endrep
add r1, r1
%endmacro
; in: %1-%3=tmp, %4=p2, %5=q2
%macro LUMA_INTRA_INTER 5
LOAD_AB t0, t1, r2d, r3d
mova %1, t0
LOAD_MASK m0, m1, m2, m3, %1, t1, t0, t2, t3
%if ARCH_X86_64
mova %2, t0 ; mask0
psrlw t3, %1, 2
%else
mova t3, %1
mova %2, t0 ; mask0
psrlw t3, 2
%endif
paddw t3, [pw_2] ; alpha/4+2
DIFF_LT m1, m2, t3, t2, t0 ; t2 = |p0-q0| < alpha/4+2
pand t2, %2
mova t3, %5 ; q2
mova %1, t2 ; mask1
DIFF_LT t3, m2, t1, t2, t0 ; t2 = |q2-q0| < beta
pand t2, %1
mova t3, %4 ; p2
mova %3, t2 ; mask1q
DIFF_LT t3, m1, t1, t2, t0 ; t2 = |p2-p0| < beta
pand t2, %1
mova %1, t2 ; mask1p
%endmacro
%macro LUMA_H_INTRA_LOAD 0
%if mmsize == 8
movu t0, [r0-8]
movu t1, [r0+r1-8]
movu m0, [r0+r1*2-8]
movu m1, [r0+r4-8]
TRANSPOSE4x4W 4, 5, 0, 1, 2
mova t4, t0 ; p3
mova t5, t1 ; p2
movu m2, [r0]
movu m3, [r0+r1]
movu t0, [r0+r1*2]
movu t1, [r0+r4]
TRANSPOSE4x4W 2, 3, 4, 5, 6
mova t6, t0 ; q2
mova t7, t1 ; q3
%else
movu t0, [r0-8]
movu t1, [r0+r1-8]
movu m0, [r0+r1*2-8]
movu m1, [r0+r5-8]
movu m2, [r4-8]
movu m3, [r4+r1-8]
movu t2, [r4+r1*2-8]
movu t3, [r4+r5-8]
TRANSPOSE8x8W 4, 5, 0, 1, 2, 3, 6, 7, t4, t5
mova t4, t0 ; p3
mova t5, t1 ; p2
mova t6, t2 ; q2
mova t7, t3 ; q3
%endif
%endmacro
; in: %1=q3 %2=q2' %3=q1' %4=q0' %5=p0' %6=p1' %7=p2' %8=p3 %9=tmp
%macro LUMA_H_INTRA_STORE 9
%if mmsize == 8
TRANSPOSE4x4W %1, %2, %3, %4, %9
movq [r0-8], m%1
movq [r0+r1-8], m%2
movq [r0+r1*2-8], m%3
movq [r0+r4-8], m%4
movq m%1, %8
TRANSPOSE4x4W %5, %6, %7, %1, %9
movq [r0], m%5
movq [r0+r1], m%6
movq [r0+r1*2], m%7
movq [r0+r4], m%1
%else
TRANSPOSE2x4x4W %1, %2, %3, %4, %9
movq [r0-8], m%1
movq [r0+r1-8], m%2
movq [r0+r1*2-8], m%3
movq [r0+r5-8], m%4
movhps [r4-8], m%1
movhps [r4+r1-8], m%2
movhps [r4+r1*2-8], m%3
movhps [r4+r5-8], m%4
%ifnum %8
SWAP %1, %8
%else
mova m%1, %8
%endif
TRANSPOSE2x4x4W %5, %6, %7, %1, %9
movq [r0], m%5
movq [r0+r1], m%6
movq [r0+r1*2], m%7
movq [r0+r5], m%1
movhps [r4], m%5
movhps [r4+r1], m%6
movhps [r4+r1*2], m%7
movhps [r4+r5], m%1
%endif
%endmacro
%if ARCH_X86_64
;-----------------------------------------------------------------------------
; void deblock_v_luma_intra( uint16_t *pix, intptr_t stride, int alpha, int beta )
;-----------------------------------------------------------------------------
%macro DEBLOCK_LUMA_INTRA_64 0
cglobal deblock_v_luma_intra, 4,7,16
%define t0 m1
%define t1 m2
%define t2 m4
%define p2 m8
%define p1 m9
%define p0 m10
%define q0 m11
%define q1 m12
%define q2 m13
%define aa m5
%define bb m14
add r1, r1
lea r4, [r1*4]
lea r5, [r1*3] ; 3*stride
neg r4
add r4, r0 ; pix-4*stride
mov r6, 2
mova m0, [pw_2]
LOAD_AB aa, bb, r2d, r3d
.loop:
mova p2, [r4+r1]
mova p1, [r4+2*r1]
mova p0, [r4+r5]
mova q0, [r0]
mova q1, [r0+r1]
mova q2, [r0+2*r1]
LOAD_MASK p1, p0, q0, q1, aa, bb, m3, t0, t1
mova t2, aa
psrlw t2, 2
paddw t2, m0 ; alpha/4+2
DIFF_LT p0, q0, t2, m6, t0 ; m6 = |p0-q0| < alpha/4+2
DIFF_LT p2, p0, bb, t1, t0 ; m7 = |p2-p0| < beta
DIFF_LT q2, q0, bb, m7, t0 ; t1 = |q2-q0| < beta
pand m6, m3
pand m7, m6
pand m6, t1
LUMA_INTRA_P012 p0, p1, p2, [r4], q0, q1, m3, m6, m0, [r4+r5], [r4+2*r1], [r4+r1]
LUMA_INTRA_P012 q0, q1, q2, [r0+r5], p0, p1, m3, m7, m0, [r0], [r0+r1], [r0+2*r1]
add r0, mmsize
add r4, mmsize
dec r6
jg .loop
RET
;-----------------------------------------------------------------------------
; void deblock_h_luma_intra( uint16_t *pix, intptr_t stride, int alpha, int beta )
;-----------------------------------------------------------------------------
cglobal deblock_h_luma_intra, 4,7,16
%define t0 m15
%define t1 m14
%define t2 m2
%define q3 m5
%define q2 m8
%define q1 m9
%define q0 m10
%define p0 m11
%define p1 m12
%define p2 m13
%define p3 m4
%define spill [rsp]
%assign pad 24-(stack_offset&15)
SUB rsp, pad
add r1, r1
lea r4, [r1*4]
lea r5, [r1*3] ; 3*stride
add r4, r0 ; pix+4*stride
mov r6, 2
mova m0, [pw_2]
.loop:
movu q3, [r0-8]
movu q2, [r0+r1-8]
movu q1, [r0+r1*2-8]
movu q0, [r0+r5-8]
movu p0, [r4-8]
movu p1, [r4+r1-8]
movu p2, [r4+r1*2-8]
movu p3, [r4+r5-8]
TRANSPOSE8x8W 5, 8, 9, 10, 11, 12, 13, 4, 1
LOAD_AB m1, m2, r2d, r3d
LOAD_MASK q1, q0, p0, p1, m1, m2, m3, t0, t1
psrlw m1, 2
paddw m1, m0 ; alpha/4+2
DIFF_LT p0, q0, m1, m6, t0 ; m6 = |p0-q0| < alpha/4+2
DIFF_LT q2, q0, m2, t1, t0 ; t1 = |q2-q0| < beta
DIFF_LT p0, p2, m2, m7, t0 ; m7 = |p2-p0| < beta
pand m6, m3
pand m7, m6
pand m6, t1
mova spill, q3
LUMA_INTRA_P012 q0, q1, q2, q3, p0, p1, m3, m6, m0, m5, m1, q2
LUMA_INTRA_P012 p0, p1, p2, p3, q0, q1, m3, m7, m0, p0, m6, p2
mova m7, spill
LUMA_H_INTRA_STORE 7, 8, 1, 5, 11, 6, 13, 4, 14
lea r0, [r0+r1*8]
lea r4, [r4+r1*8]
dec r6
jg .loop
ADD rsp, pad
RET
%endmacro
INIT_XMM sse2
DEBLOCK_LUMA_INTRA_64
INIT_XMM avx
DEBLOCK_LUMA_INTRA_64
%endif
%macro DEBLOCK_LUMA_INTRA 0
;-----------------------------------------------------------------------------
; void deblock_v_luma_intra( uint16_t *pix, intptr_t stride, int alpha, int beta )
;-----------------------------------------------------------------------------
cglobal deblock_v_luma_intra, 4,7,8,0-3*mmsize
LUMA_INTRA_INIT 3
lea r4, [r1*4]
lea r5, [r1*3]
neg r4
add r4, r0
mov r6, 32/mmsize
.loop:
mova m0, [r4+r1*2] ; p1
mova m1, [r4+r5] ; p0
mova m2, [r0] ; q0
mova m3, [r0+r1] ; q1
LUMA_INTRA_INTER t4, t5, t6, [r4+r1], [r0+r1*2]
LUMA_INTRA_P012 m1, m0, t3, [r4], m2, m3, t5, t4, [pw_2], [r4+r5], [r4+2*r1], [r4+r1]
mova t3, [r0+r1*2] ; q2
LUMA_INTRA_P012 m2, m3, t3, [r0+r5], m1, m0, t5, t6, [pw_2], [r0], [r0+r1], [r0+2*r1]
add r0, mmsize
add r4, mmsize
dec r6
jg .loop
RET
;-----------------------------------------------------------------------------
; void deblock_h_luma_intra( uint16_t *pix, intptr_t stride, int alpha, int beta )
;-----------------------------------------------------------------------------
cglobal deblock_h_luma_intra, 4,7,8,0-8*mmsize
LUMA_INTRA_INIT 8
%if mmsize == 8
lea r4, [r1*3]
mov r5, 32/mmsize
%else
lea r4, [r1*4]
lea r5, [r1*3] ; 3*stride
add r4, r0 ; pix+4*stride
mov r6, 32/mmsize
%endif
.loop:
LUMA_H_INTRA_LOAD
LUMA_INTRA_INTER t8, t9, t10, t5, t6
LUMA_INTRA_P012 m1, m0, t3, t4, m2, m3, t9, t8, [pw_2], t8, t5, t11
mova t3, t6 ; q2
LUMA_INTRA_P012 m2, m3, t3, t7, m1, m0, t9, t10, [pw_2], m4, t6, m5
mova m2, t4
mova m0, t11
mova m1, t5
mova m3, t8
mova m6, t6
LUMA_H_INTRA_STORE 2, 0, 1, 3, 4, 6, 5, t7, 7
lea r0, [r0+r1*(mmsize/2)]
%if mmsize == 8
dec r5
%else
lea r4, [r4+r1*(mmsize/2)]
dec r6
%endif
jg .loop
RET
%endmacro
%if ARCH_X86_64 == 0
INIT_MMX mmx2
DEBLOCK_LUMA
DEBLOCK_LUMA_INTRA
INIT_XMM sse2
DEBLOCK_LUMA
DEBLOCK_LUMA_INTRA
INIT_XMM avx
DEBLOCK_LUMA
DEBLOCK_LUMA_INTRA
%endif
%endif ; HIGH_BIT_DEPTH
%if HIGH_BIT_DEPTH == 0
; expands to [base],...,[base+7*stride]
%define PASS8ROWS(base, base3, stride, stride3) \
[base], [base+stride], [base+stride*2], [base3], \
[base3+stride], [base3+stride*2], [base3+stride3], [base3+stride*4]
%define PASS8ROWS(base, base3, stride, stride3, offset) \
PASS8ROWS(base+offset, base3+offset, stride, stride3)
; in: 4 rows of 8 bytes in m0..m3
; out: 8 rows of 4 bytes in %1..%8
%macro TRANSPOSE8x4B_STORE 8
punpckhdq m4, m0, m0
punpckhdq m5, m1, m1
punpckhdq m6, m2, m2
punpcklbw m0, m1
punpcklbw m2, m3
punpcklwd m1, m0, m2
punpckhwd m0, m2
movd %1, m1
punpckhdq m1, m1
movd %2, m1
movd %3, m0
punpckhdq m0, m0
movd %4, m0
punpckhdq m3, m3
punpcklbw m4, m5
punpcklbw m6, m3
punpcklwd m5, m4, m6
punpckhwd m4, m6
movd %5, m5
punpckhdq m5, m5
movd %6, m5
movd %7, m4
punpckhdq m4, m4
movd %8, m4
%endmacro
; in: 8 rows of 4 bytes in %9..%10
; out: 8 rows of 4 bytes in %1..%8
%macro STORE_8x4B 10
movd %1, %9
pextrd %2, %9, 1
pextrd %3, %9, 2
pextrd %4, %9, 3
movd %5, %10
pextrd %6, %10, 1
pextrd %7, %10, 2
pextrd %8, %10, 3
%endmacro
; in: 4 rows of 4 words in %1..%4
; out: 4 rows of 4 word in m0..m3
; clobbers: m4
%macro TRANSPOSE4x4W_LOAD 4-8
%if mmsize==8
SWAP 1, 4, 2, 3
movq m0, %1
movq m1, %2
movq m2, %3
movq m3, %4
TRANSPOSE4x4W 0, 1, 2, 3, 4
%else
movq m0, %1
movq m2, %2
movq m1, %3
movq m3, %4
punpcklwd m0, m2
punpcklwd m1, m3
mova m2, m0
punpckldq m0, m1
punpckhdq m2, m1
MOVHL m1, m0
MOVHL m3, m2
%endif
%endmacro
; in: 2 rows of 4 words in m1..m2
; out: 4 rows of 2 words in %1..%4
; clobbers: m0, m1
%macro TRANSPOSE4x2W_STORE 4-8
%if mmsize==8
punpckhwd m0, m1, m2
punpcklwd m1, m2
%else
punpcklwd m1, m2
MOVHL m0, m1
%endif
movd %3, m0
movd %1, m1
psrlq m1, 32
psrlq m0, 32
movd %2, m1
movd %4, m0
%endmacro
; in: 4/8 rows of 4 words in %1..%8
; out: 4 rows of 4/8 word in m0..m3
; clobbers: m4, m5, m6, m7
%macro TRANSPOSE4x8W_LOAD 8
%if mmsize==8
TRANSPOSE4x4W_LOAD %1, %2, %3, %4
%else
movq m0, %1
movq m2, %2
movq m1, %3
movq m3, %4
punpcklwd m0, m2
punpcklwd m1, m3
mova m2, m0
punpckldq m0, m1
punpckhdq m2, m1
movq m4, %5
movq m6, %6
movq m5, %7
movq m7, %8
punpcklwd m4, m6
punpcklwd m5, m7
mova m6, m4
punpckldq m4, m5
punpckhdq m6, m5
punpckhqdq m1, m0, m4
punpckhqdq m3, m2, m6
punpcklqdq m0, m4
punpcklqdq m2, m6
%endif
%endmacro
; in: 2 rows of 4/8 words in m1..m2
; out: 4/8 rows of 2 words in %1..%8
; clobbers: m0, m1
%macro TRANSPOSE8x2W_STORE 8
%if mmsize==8
TRANSPOSE4x2W_STORE %1, %2, %3, %4
%else
punpckhwd m0, m1, m2
punpcklwd m1, m2
movd %5, m0
movd %1, m1
psrldq m1, 4
psrldq m0, 4
movd %2, m1
movd %6, m0
psrldq m1, 4
psrldq m0, 4
movd %3, m1
movd %7, m0
psrldq m1, 4
psrldq m0, 4
movd %4, m1
movd %8, m0
%endif
%endmacro
%macro SBUTTERFLY3 4
punpckh%1 %4, %2, %3
punpckl%1 %2, %3
%endmacro
; in: 8 rows of 8 (only the middle 6 pels are used) in %1..%8
; out: 6 rows of 8 in [%9+0*16] .. [%9+5*16]
%macro TRANSPOSE6x8_MEM 9
RESET_MM_PERMUTATION
%if cpuflag(avx)
; input:
; _ABCDEF_
; _GHIJKL_
; _MNOPQR_
; _STUVWX_
; _YZabcd_
; _efghij_
; _klmnop_
; _qrstuv_
movh m0, %1
movh m2, %2
movh m1, %3
movh m3, %4
punpcklbw m0, m2 ; __ AG BH CI DJ EK FL __
punpcklbw m1, m3 ; __ MS NT OU PV QW RX __
movh m2, %5
movh m3, %6
punpcklbw m2, m3 ; __ Ye Zf ag bh ci dj __
movh m3, %7
movh m4, %8
punpcklbw m3, m4 ; __ kq lr ms nt ou pv __
SBUTTERFLY wd, 0, 1, 4 ; __ __ AG MS BH NT CI OU
; DJ PV EK QW FL RX __ __
SBUTTERFLY wd, 2, 3, 4 ; __ __ Ye kq Zf lr ag ms
; bh nt ci ou dj pv __ __
SBUTTERFLY dq, 0, 2, 4 ; __ __ __ __ AG MS Ye kq
; BH NT Zf lr CI FL OU RX
SBUTTERFLY dq, 1, 3, 4 ; DJ PV bh nt EK QW Zf lr
; FL RX dj pv __ __ __ __
movhps [%9+0x00], m0
movh [%9+0x10], m2
movhps [%9+0x20], m2
movh [%9+0x30], m1
movhps [%9+0x40], m1
movh [%9+0x50], m3
%else
movq m0, %1
movq m1, %2
movq m2, %3
movq m3, %4
movq m4, %5
movq m5, %6
movq m6, %7
SBUTTERFLY bw, 0, 1, 7
SBUTTERFLY bw, 2, 3, 7
SBUTTERFLY bw, 4, 5, 7
movq [%9+0x10], m3
SBUTTERFLY3 bw, m6, %8, m7
SBUTTERFLY wd, 0, 2, 3
SBUTTERFLY wd, 4, 6, 3
punpckhdq m0, m4
movq [%9+0x00], m0
SBUTTERFLY3 wd, m1, [%9+0x10], m3
SBUTTERFLY wd, 5, 7, 0
SBUTTERFLY dq, 1, 5, 0
SBUTTERFLY dq, 2, 6, 0
punpckldq m3, m7
movq [%9+0x10], m2
movq [%9+0x20], m6
movq [%9+0x30], m1
movq [%9+0x40], m5
movq [%9+0x50], m3
%endif
RESET_MM_PERMUTATION
%endmacro
; in: 8 rows of 8 in %1..%8
; out: 8 rows of 8 in %9..%16
%macro TRANSPOSE8x8_MEM 16
RESET_MM_PERMUTATION
%if cpuflag(avx)
movh m0, %1
movh m4, %2
movh m1, %3
movh m5, %4
movh m2, %5
movh m3, %7
punpcklbw m0, m4
punpcklbw m1, m5
movh m4, %6
movh m5, %8
punpcklbw m2, m4
punpcklbw m3, m5
SBUTTERFLY wd, 0, 1, 4
SBUTTERFLY wd, 2, 3, 4
SBUTTERFLY dq, 0, 2, 4
SBUTTERFLY dq, 1, 3, 4
movh %9, m0
movhps %10, m0
movh %11, m2
movhps %12, m2
movh %13, m1
movhps %14, m1
movh %15, m3
movhps %16, m3
%else
movq m0, %1
movq m1, %2
movq m2, %3
movq m3, %4
movq m4, %5
movq m5, %6
movq m6, %7
SBUTTERFLY bw, 0, 1, 7
SBUTTERFLY bw, 2, 3, 7
SBUTTERFLY bw, 4, 5, 7
SBUTTERFLY3 bw, m6, %8, m7
movq %9, m5
SBUTTERFLY wd, 0, 2, 5
SBUTTERFLY wd, 4, 6, 5
SBUTTERFLY wd, 1, 3, 5
movq %11, m6
movq m6, %9
SBUTTERFLY wd, 6, 7, 5
SBUTTERFLY dq, 0, 4, 5
SBUTTERFLY dq, 1, 6, 5
movq %9, m0
movq %10, m4
movq %13, m1
movq %14, m6
SBUTTERFLY3 dq, m2, %11, m0
SBUTTERFLY dq, 3, 7, 4
movq %11, m2
movq %12, m0
movq %15, m3
movq %16, m7
%endif
RESET_MM_PERMUTATION
%endmacro
; out: %4 = |%1-%2|>%3
; clobbers: %5
%macro DIFF_GT 5
%if avx_enabled == 0
mova %5, %2
mova %4, %1
psubusb %5, %1
psubusb %4, %2
%else
psubusb %5, %2, %1
psubusb %4, %1, %2
%endif
por %4, %5
psubusb %4, %3
%endmacro
; out: %4 = |%1-%2|>%3
; clobbers: %5
%macro DIFF_GT2 5-6
%if %0<6
psubusb %4, %1, %2
psubusb %5, %2, %1
%else
mova %4, %1
mova %5, %2
psubusb %4, %2
psubusb %5, %1
%endif
psubusb %5, %3
psubusb %4, %3
pcmpeqb %4, %5
%endmacro
; in: m0=p1 m1=p0 m2=q0 m3=q1 %1=alpha %2=beta
; out: m5=beta-1, m7=mask, %3=alpha-1
; clobbers: m4,m6
%macro LOAD_MASK 2-3
%if cpuflag(ssse3)
movd m4, %1
movd m5, %2
pxor m6, m6
pshufb m4, m6
pshufb m5, m6
%else
movd m4, %1
movd m5, %2
punpcklbw m4, m4
punpcklbw m5, m5
SPLATW m4, m4
SPLATW m5, m5
%endif
mova m6, [pb_1]
psubusb m4, m6 ; alpha - 1
psubusb m5, m6 ; beta - 1
%if %0>2
mova %3, m4
%endif
DIFF_GT m1, m2, m4, m7, m6 ; |p0-q0| > alpha-1
DIFF_GT m0, m1, m5, m4, m6 ; |p1-p0| > beta-1
por m7, m4
DIFF_GT m3, m2, m5, m4, m6 ; |q1-q0| > beta-1
por m7, m4
pxor m6, m6
pcmpeqb m7, m6
%endmacro
; in: m0=p1 m1=p0 m2=q0 m3=q1 m7=(tc&mask)
; out: m1=p0' m2=q0'
; clobbers: m0,3-6
%macro DEBLOCK_P0_Q0 0
pxor m5, m1, m2 ; p0^q0
pand m5, [pb_1] ; (p0^q0)&1
pcmpeqb m4, m4
pxor m3, m4
pavgb m3, m0 ; (p1 - q1 + 256)>>1
pavgb m3, [pb_3] ; (((p1 - q1 + 256)>>1)+4)>>1 = 64+2+(p1-q1)>>2
pxor m4, m1
pavgb m4, m2 ; (q0 - p0 + 256)>>1
pavgb m3, m5
paddusb m3, m4 ; d+128+33
mova m6, [pb_a1]
psubusb m6, m3
psubusb m3, [pb_a1]
pminub m6, m7
pminub m3, m7
psubusb m1, m6
psubusb m2, m3
paddusb m1, m3
paddusb m2, m6
%endmacro
; in: m1=p0 m2=q0
; %1=p1 %2=q2 %3=[q2] %4=[q1] %5=tc0 %6=tmp
; out: [q1] = clip( (q2+((p0+q0+1)>>1))>>1, q1-tc0, q1+tc0 )
; clobbers: q2, tmp, tc0
%macro LUMA_Q1 6
pavgb %6, m1, m2
pavgb %2, %6 ; avg(p2,avg(p0,q0))
pxor %6, %3
pand %6, [pb_1] ; (p2^avg(p0,q0))&1
psubusb %2, %6 ; (p2+((p0+q0+1)>>1))>>1
psubusb %6, %1, %5
paddusb %5, %1
pmaxub %2, %6
pminub %2, %5
mova %4, %2
%endmacro
%if ARCH_X86_64
;-----------------------------------------------------------------------------
; void deblock_v_luma( uint8_t *pix, intptr_t stride, int alpha, int beta, int8_t *tc0 )
;-----------------------------------------------------------------------------
%macro DEBLOCK_LUMA 0
cglobal deblock_v_luma, 5,5,10
movd m8, [r4] ; tc0
lea r4, [r1*3]
neg r4
add r4, r0 ; pix-3*stride
mova m0, [r4+r1] ; p1
mova m1, [r4+2*r1] ; p0
mova m2, [r0] ; q0
mova m3, [r0+r1] ; q1
LOAD_MASK r2d, r3d
%if cpuflag(avx)
pshufb m8, [pb_unpackbd1]
pblendvb m9, m7, m6, m8
%else
punpcklbw m8, m8
punpcklbw m8, m8 ; tc = 4x tc0[3], 4x tc0[2], 4x tc0[1], 4x tc0[0]
pcmpeqb m9, m9
pcmpeqb m9, m8
pandn m9, m7
%endif
pand m8, m9
mova m3, [r4] ; p2
DIFF_GT2 m1, m3, m5, m6, m7 ; |p2-p0| > beta-1
pand m6, m9
psubb m7, m8, m6 ; tc++
pand m6, m8
LUMA_Q1 m0, m3, [r4], [r4+r1], m6, m4
mova m4, [r0+2*r1] ; q2
DIFF_GT2 m2, m4, m5, m6, m3 ; |q2-q0| > beta-1
pand m6, m9
pand m8, m6
psubb m7, m6
mova m3, [r0+r1]
LUMA_Q1 m3, m4, [r0+2*r1], [r0+r1], m8, m6
DEBLOCK_P0_Q0
mova [r4+2*r1], m1
mova [r0], m2
RET
;-----------------------------------------------------------------------------
; void deblock_h_luma( uint8_t *pix, intptr_t stride, int alpha, int beta, int8_t *tc0 )
;-----------------------------------------------------------------------------
%if cpuflag(avx)
INIT_XMM cpuname
%else
INIT_MMX cpuname
%endif
cglobal deblock_h_luma, 5,9,0,0x60+16*WIN64
lea r8, [r1*3]
lea r6, [r0-4]
lea r5, [r0-4+r8]
%xdefine pix_tmp rsp+0x30*WIN64 ; shadow space + r4
; transpose 6x16 -> tmp space
TRANSPOSE6x8_MEM PASS8ROWS(r6, r5, r1, r8), pix_tmp
lea r6, [r6+r1*8]
lea r5, [r5+r1*8]
TRANSPOSE6x8_MEM PASS8ROWS(r6, r5, r1, r8), pix_tmp+8
; vertical filter
; alpha, beta, tc0 are still in r2d, r3d, r4
; don't backup r6, r5, r7, r8 because deblock_v_luma_sse2 doesn't use them
mov r7, r1
lea r0, [pix_tmp+0x30]
mov r1d, 0x10
%if WIN64
mov [rsp+0x20], r4
%endif
call deblock_v_luma
; transpose 16x4 -> original space (only the middle 4 rows were changed by the filter)
add r6, 2
add r5, 2
%if cpuflag(sse4)
mova m0, [pix_tmp+0x10]
mova m1, [pix_tmp+0x20]
mova m2, [pix_tmp+0x30]
mova m3, [pix_tmp+0x40]
SBUTTERFLY bw, 0, 1, 4
SBUTTERFLY bw, 2, 3, 4
SBUTTERFLY wd, 0, 2, 4
SBUTTERFLY wd, 1, 3, 4
STORE_8x4B PASS8ROWS(r6, r5, r7, r8), m1, m3
shl r7, 3
sub r6, r7
sub r5, r7
shr r7, 3
STORE_8x4B PASS8ROWS(r6, r5, r7, r8), m0, m2
%else
movq m0, [pix_tmp+0x18]
movq m1, [pix_tmp+0x28]
movq m2, [pix_tmp+0x38]
movq m3, [pix_tmp+0x48]
TRANSPOSE8x4B_STORE PASS8ROWS(r6, r5, r7, r8)
shl r7, 3
sub r6, r7
sub r5, r7
shr r7, 3
movq m0, [pix_tmp+0x10]
movq m1, [pix_tmp+0x20]
movq m2, [pix_tmp+0x30]
movq m3, [pix_tmp+0x40]
TRANSPOSE8x4B_STORE PASS8ROWS(r6, r5, r7, r8)
%endif
RET
%endmacro
INIT_XMM sse2
DEBLOCK_LUMA
INIT_XMM avx
DEBLOCK_LUMA
%else
%macro DEBLOCK_LUMA 2
;-----------------------------------------------------------------------------
; void deblock_v8_luma( uint8_t *pix, intptr_t stride, int alpha, int beta, int8_t *tc0 )
;-----------------------------------------------------------------------------
cglobal deblock_%1_luma, 5,5,8,2*%2
lea r4, [r1*3]
neg r4
add r4, r0 ; pix-3*stride
mova m0, [r4+r1] ; p1
mova m1, [r4+2*r1] ; p0
mova m2, [r0] ; q0
mova m3, [r0+r1] ; q1
LOAD_MASK r2d, r3d
mov r3, r4mp
movd m4, [r3] ; tc0
%if cpuflag(avx)
pshufb m4, [pb_unpackbd1]
mova [esp+%2], m4 ; tc
pblendvb m4, m7, m6, m4
%else
punpcklbw m4, m4
punpcklbw m4, m4 ; tc = 4x tc0[3], 4x tc0[2], 4x tc0[1], 4x tc0[0]
mova [esp+%2], m4 ; tc
pcmpeqb m3, m3
pcmpgtb m4, m3
pand m4, m7
%endif
mova [esp], m4 ; mask
mova m3, [r4] ; p2
DIFF_GT2 m1, m3, m5, m6, m7 ; |p2-p0| > beta-1
pand m6, m4
pand m4, [esp+%2] ; tc
psubb m7, m4, m6
pand m6, m4
LUMA_Q1 m0, m3, [r4], [r4+r1], m6, m4
mova m4, [r0+2*r1] ; q2
DIFF_GT2 m2, m4, m5, m6, m3 ; |q2-q0| > beta-1
mova m5, [esp] ; mask
pand m6, m5
mova m5, [esp+%2] ; tc
pand m5, m6
psubb m7, m6
mova m3, [r0+r1]
LUMA_Q1 m3, m4, [r0+2*r1], [r0+r1], m5, m6
DEBLOCK_P0_Q0
mova [r4+2*r1], m1
mova [r0], m2
RET
;-----------------------------------------------------------------------------
; void deblock_h_luma( uint8_t *pix, intptr_t stride, int alpha, int beta, int8_t *tc0 )
;-----------------------------------------------------------------------------
%if cpuflag(avx)
INIT_XMM cpuname
%else
INIT_MMX cpuname
%endif
cglobal deblock_h_luma, 1,5,8,0x60+12
mov r3, r1m
lea r4, [r3*3]
sub r0, 4
lea r1, [r0+r4]
%define pix_tmp esp+12
; esp is intentionally misaligned to make it aligned after pushing the arguments for deblock_%1_luma.
; transpose 6x16 -> tmp space
TRANSPOSE6x8_MEM PASS8ROWS(r0, r1, r3, r4), pix_tmp
lea r0, [r0+r3*8]
lea r1, [r1+r3*8]
TRANSPOSE6x8_MEM PASS8ROWS(r0, r1, r3, r4), pix_tmp+8
; vertical filter
lea r0, [pix_tmp+0x30]
PUSH dword r4m
PUSH dword r3m
PUSH dword r2m
PUSH dword 16
PUSH dword r0
call deblock_%1_luma
%ifidn %1, v8
add dword [esp ], 8 ; pix_tmp+0x38
add dword [esp+16], 2 ; tc0+2
call deblock_%1_luma
%endif
ADD esp, 20
; transpose 16x4 -> original space (only the middle 4 rows were changed by the filter)
mov r0, r0mp
sub r0, 2
lea r1, [r0+r4]
%if cpuflag(avx)
mova m0, [pix_tmp+0x10]
mova m1, [pix_tmp+0x20]
mova m2, [pix_tmp+0x30]
mova m3, [pix_tmp+0x40]
SBUTTERFLY bw, 0, 1, 4
SBUTTERFLY bw, 2, 3, 4
SBUTTERFLY wd, 0, 2, 4
SBUTTERFLY wd, 1, 3, 4
STORE_8x4B PASS8ROWS(r0, r1, r3, r4), m0, m2
lea r0, [r0+r3*8]
lea r1, [r1+r3*8]
STORE_8x4B PASS8ROWS(r0, r1, r3, r4), m1, m3
%else
movq m0, [pix_tmp+0x10]
movq m1, [pix_tmp+0x20]
movq m2, [pix_tmp+0x30]
movq m3, [pix_tmp+0x40]
TRANSPOSE8x4B_STORE PASS8ROWS(r0, r1, r3, r4)
lea r0, [r0+r3*8]
lea r1, [r1+r3*8]
movq m0, [pix_tmp+0x18]
movq m1, [pix_tmp+0x28]
movq m2, [pix_tmp+0x38]
movq m3, [pix_tmp+0x48]
TRANSPOSE8x4B_STORE PASS8ROWS(r0, r1, r3, r4)
%endif
RET
%endmacro ; DEBLOCK_LUMA
INIT_MMX mmx2
DEBLOCK_LUMA v8, 8
INIT_XMM sse2
DEBLOCK_LUMA v, 16
INIT_XMM avx
DEBLOCK_LUMA v, 16
%endif ; ARCH
%macro LUMA_INTRA_P012 4 ; p0..p3 in memory
%if ARCH_X86_64
pavgb t0, p2, p1
pavgb t1, p0, q0
%else
mova t0, p2
mova t1, p0
pavgb t0, p1
pavgb t1, q0
%endif
pavgb t0, t1 ; ((p2+p1+1)/2 + (p0+q0+1)/2 + 1)/2
mova t5, t1
%if ARCH_X86_64
paddb t2, p2, p1
paddb t3, p0, q0
%else
mova t2, p2
mova t3, p0
paddb t2, p1
paddb t3, q0
%endif
paddb t2, t3
mova t3, t2
mova t4, t2
psrlw t2, 1
pavgb t2, mpb_0
pxor t2, t0
pand t2, mpb_1
psubb t0, t2 ; p1' = (p2+p1+p0+q0+2)/4;
%if ARCH_X86_64
pavgb t1, p2, q1
psubb t2, p2, q1
%else
mova t1, p2
mova t2, p2
pavgb t1, q1
psubb t2, q1
%endif
paddb t3, t3
psubb t3, t2 ; p2+2*p1+2*p0+2*q0+q1
pand t2, mpb_1
psubb t1, t2
pavgb t1, p1
pavgb t1, t5 ; (((p2+q1)/2 + p1+1)/2 + (p0+q0+1)/2 + 1)/2
psrlw t3, 2
pavgb t3, mpb_0
pxor t3, t1
pand t3, mpb_1
psubb t1, t3 ; p0'a = (p2+2*p1+2*p0+2*q0+q1+4)/8
pxor t3, p0, q1
pavgb t2, p0, q1
pand t3, mpb_1
psubb t2, t3
pavgb t2, p1 ; p0'b = (2*p1+p0+q0+2)/4
pxor t1, t2
pxor t2, p0
pand t1, mask1p
pand t2, mask0
pxor t1, t2
pxor t1, p0
mova %1, t1 ; store p0
mova t1, %4 ; p3
paddb t2, t1, p2
pavgb t1, p2
pavgb t1, t0 ; (p3+p2+1)/2 + (p2+p1+p0+q0+2)/4
paddb t2, t2
paddb t2, t4 ; 2*p3+3*p2+p1+p0+q0
psrlw t2, 2
pavgb t2, mpb_0
pxor t2, t1
pand t2, mpb_1
psubb t1, t2 ; p2' = (2*p3+3*p2+p1+p0+q0+4)/8
pxor t0, p1
pxor t1, p2
pand t0, mask1p
pand t1, mask1p
pxor t0, p1
pxor t1, p2
mova %2, t0 ; store p1
mova %3, t1 ; store p2
%endmacro
%macro LUMA_INTRA_SWAP_PQ 0
%define q1 m0
%define q0 m1
%define p0 m2
%define p1 m3
%define p2 q2
%define mask1p mask1q
%endmacro
%macro DEBLOCK_LUMA_INTRA 1
%define p1 m0
%define p0 m1
%define q0 m2
%define q1 m3
%define t0 m4
%define t1 m5
%define t2 m6
%define t3 m7
%if ARCH_X86_64
%define p2 m8
%define q2 m9
%define t4 m10
%define t5 m11
%define mask0 m12
%define mask1p m13
%if WIN64
%define mask1q [rsp]
%else
%define mask1q [rsp-24]
%endif
%define mpb_0 m14
%define mpb_1 m15
%else
%define spill(x) [esp+16*x]
%define p2 [r4+r1]
%define q2 [r0+2*r1]
%define t4 spill(0)
%define t5 spill(1)
%define mask0 spill(2)
%define mask1p spill(3)
%define mask1q spill(4)
%define mpb_0 [pb_0]
%define mpb_1 [pb_1]
%endif
;-----------------------------------------------------------------------------
; void deblock_v_luma_intra( uint8_t *pix, intptr_t stride, int alpha, int beta )
;-----------------------------------------------------------------------------
cglobal deblock_%1_luma_intra, 4,6,16,0-(1-ARCH_X86_64)*0x50-WIN64*0x10
lea r4, [r1*4]
lea r5, [r1*3] ; 3*stride
neg r4
add r4, r0 ; pix-4*stride
mova p1, [r4+2*r1]
mova p0, [r4+r5]
mova q0, [r0]
mova q1, [r0+r1]
%if ARCH_X86_64
pxor mpb_0, mpb_0
mova mpb_1, [pb_1]
LOAD_MASK r2d, r3d, t5 ; m5=beta-1, t5=alpha-1, m7=mask0
SWAP 7, 12 ; m12=mask0
pavgb t5, mpb_0
pavgb t5, mpb_1 ; alpha/4+1
movdqa p2, [r4+r1]
movdqa q2, [r0+2*r1]
DIFF_GT2 p0, q0, t5, t0, t3 ; t0 = |p0-q0| > alpha/4+1
DIFF_GT2 p0, p2, m5, t2, t5, 1 ; mask1 = |p2-p0| > beta-1
DIFF_GT2 q0, q2, m5, t4, t5, 1 ; t4 = |q2-q0| > beta-1
pand t0, mask0
pand t4, t0
pand t2, t0
mova mask1q, t4
mova mask1p, t2
%else
LOAD_MASK r2d, r3d, t5 ; m5=beta-1, t5=alpha-1, m7=mask0
mova m4, t5
mova mask0, m7
pavgb m4, [pb_0]
pavgb m4, [pb_1] ; alpha/4+1
DIFF_GT2 p0, q0, m4, m6, m7 ; m6 = |p0-q0| > alpha/4+1
pand m6, mask0
DIFF_GT2 p0, p2, m5, m4, m7, 1 ; m4 = |p2-p0| > beta-1
pand m4, m6
mova mask1p, m4
DIFF_GT2 q0, q2, m5, m4, m7, 1 ; m4 = |q2-q0| > beta-1
pand m4, m6
mova mask1q, m4
%endif
LUMA_INTRA_P012 [r4+r5], [r4+2*r1], [r4+r1], [r4]
LUMA_INTRA_SWAP_PQ
LUMA_INTRA_P012 [r0], [r0+r1], [r0+2*r1], [r0+r5]
.end:
REP_RET
%if cpuflag(avx)
INIT_XMM cpuname
%else
INIT_MMX cpuname
%endif
%if ARCH_X86_64
;-----------------------------------------------------------------------------
; void deblock_h_luma_intra( uint8_t *pix, intptr_t stride, int alpha, int beta )
;-----------------------------------------------------------------------------
cglobal deblock_h_luma_intra, 4,9,0,0x80
lea r8, [r1*3]
lea r6, [r0-4]
lea r5, [r0-4+r8]
%if WIN64
%define pix_tmp rsp+0x20 ; shadow space
%else
%define pix_tmp rsp
%endif
; transpose 8x16 -> tmp space
TRANSPOSE8x8_MEM PASS8ROWS(r6, r5, r1, r8), PASS8ROWS(pix_tmp, pix_tmp+0x30, 0x10, 0x30)
lea r6, [r6+r1*8]
lea r5, [r5+r1*8]
TRANSPOSE8x8_MEM PASS8ROWS(r6, r5, r1, r8), PASS8ROWS(pix_tmp+8, pix_tmp+0x38, 0x10, 0x30)
mov r7, r1
lea r0, [pix_tmp+0x40]
mov r1, 0x10
call deblock_v_luma_intra
; transpose 16x6 -> original space (but we can't write only 6 pixels, so really 16x8)
lea r5, [r6+r8]
TRANSPOSE8x8_MEM PASS8ROWS(pix_tmp+8, pix_tmp+0x38, 0x10, 0x30), PASS8ROWS(r6, r5, r7, r8)
shl r7, 3
sub r6, r7
sub r5, r7
shr r7, 3
TRANSPOSE8x8_MEM PASS8ROWS(pix_tmp, pix_tmp+0x30, 0x10, 0x30), PASS8ROWS(r6, r5, r7, r8)
RET
%else
cglobal deblock_h_luma_intra, 2,4,8,0x80
lea r3, [r1*3]
sub r0, 4
lea r2, [r0+r3]
%define pix_tmp rsp
; transpose 8x16 -> tmp space
TRANSPOSE8x8_MEM PASS8ROWS(r0, r2, r1, r3), PASS8ROWS(pix_tmp, pix_tmp+0x30, 0x10, 0x30)
lea r0, [r0+r1*8]
lea r2, [r2+r1*8]
TRANSPOSE8x8_MEM PASS8ROWS(r0, r2, r1, r3), PASS8ROWS(pix_tmp+8, pix_tmp+0x38, 0x10, 0x30)
lea r0, [pix_tmp+0x40]
PUSH dword r3m
PUSH dword r2m
PUSH dword 16
PUSH r0
call deblock_%1_luma_intra
%ifidn %1, v8
add dword [rsp], 8 ; pix_tmp+8
call deblock_%1_luma_intra
%endif
ADD esp, 16
mov r1, r1m
mov r0, r0mp
lea r3, [r1*3]
sub r0, 4
lea r2, [r0+r3]
; transpose 16x6 -> original space (but we can't write only 6 pixels, so really 16x8)
TRANSPOSE8x8_MEM PASS8ROWS(pix_tmp, pix_tmp+0x30, 0x10, 0x30), PASS8ROWS(r0, r2, r1, r3)
lea r0, [r0+r1*8]
lea r2, [r2+r1*8]
TRANSPOSE8x8_MEM PASS8ROWS(pix_tmp+8, pix_tmp+0x38, 0x10, 0x30), PASS8ROWS(r0, r2, r1, r3)
RET
%endif ; ARCH_X86_64
%endmacro ; DEBLOCK_LUMA_INTRA
INIT_XMM sse2
DEBLOCK_LUMA_INTRA v
INIT_XMM avx
DEBLOCK_LUMA_INTRA v
%if ARCH_X86_64 == 0
INIT_MMX mmx2
DEBLOCK_LUMA_INTRA v8
%endif
%endif ; !HIGH_BIT_DEPTH
%if HIGH_BIT_DEPTH
; in: %1=p0, %2=q0, %3=p1, %4=q1, %5=mask, %6=tmp, %7=tmp
; out: %1=p0', %2=q0'
%macro CHROMA_DEBLOCK_P0_Q0_INTRA 7
mova %6, [pw_2]
paddw %6, %3
paddw %6, %4
paddw %7, %6, %2
paddw %6, %1
paddw %6, %3
paddw %7, %4
psraw %6, 2
psraw %7, 2
psubw %6, %1
psubw %7, %2
pand %6, %5
pand %7, %5
paddw %1, %6
paddw %2, %7
%endmacro
; out: m0-m3
; clobbers: m4-m7
%macro CHROMA_H_LOAD 0-1
movq m0, [r0-8] ; p1 p1 p0 p0
movq m2, [r0] ; q0 q0 q1 q1
movq m5, [r0+r1-8]
movq m7, [r0+r1]
%if mmsize == 8
mova m1, m0
mova m3, m2
punpckldq m0, m5 ; p1
punpckhdq m1, m5 ; p0
punpckldq m2, m7 ; q0
punpckhdq m3, m7 ; q1
%else
movq m4, [r0+r1*2-8]
movq m6, [r0+r1*2]
movq m1, [r0+%1-8]
movq m3, [r0+%1]
punpckldq m0, m5 ; p1 ... p0 ...
punpckldq m2, m7 ; q0 ... q1 ...
punpckldq m4, m1
punpckldq m6, m3
punpckhqdq m1, m0, m4 ; p0
punpcklqdq m0, m4 ; p1
punpckhqdq m3, m2, m6 ; q1
punpcklqdq m2, m6 ; q0
%endif
%endmacro
%macro CHROMA_V_LOAD 1
mova m0, [r0] ; p1
mova m1, [r0+r1] ; p0
mova m2, [%1] ; q0
mova m3, [%1+r1] ; q1
%endmacro
; clobbers: m1, m2, m3
%macro CHROMA_H_STORE 0-1
SBUTTERFLY dq, 1, 2, 3
%if mmsize == 8
movq [r0-4], m1
movq [r0+r1-4], m2
%else
movq [r0-4], m1
movq [r0+r1*2-4], m2
movhps [r0+r1-4], m1
movhps [r0+%1-4], m2
%endif
%endmacro
%macro CHROMA_V_STORE 0
mova [r0+1*r1], m1
mova [r0+2*r1], m2
%endmacro
%macro DEBLOCK_CHROMA 0
cglobal deblock_inter_body
LOAD_AB m4, m5, r2d, r3d
LOAD_MASK m0, m1, m2, m3, m4, m5, m7, m6, m4
pxor m4, m4
LOAD_TC m6, r4
pmaxsw m6, m4
pand m7, m6
DEBLOCK_P0_Q0 m1, m2, m0, m3, m7, m5, m6
ret
;-----------------------------------------------------------------------------
; void deblock_v_chroma( uint16_t *pix, intptr_t stride, int alpha, int beta, int8_t *tc0 )
;-----------------------------------------------------------------------------
cglobal deblock_v_chroma, 5,7,8
FIX_STRIDES r1
mov r5, r0
sub r0, r1
sub r0, r1
mov r6, 32/mmsize
.loop:
CHROMA_V_LOAD r5
call deblock_inter_body
CHROMA_V_STORE
add r0, mmsize
add r5, mmsize
add r4, mmsize/8
dec r6
jg .loop
RET
;-----------------------------------------------------------------------------
; void deblock_h_chroma( uint16_t *pix, intptr_t stride, int alpha, int beta, int8_t *tc0 )
;-----------------------------------------------------------------------------
cglobal deblock_h_chroma, 5,7,8
add r1, r1
mov r5, 32/mmsize
%if mmsize == 16
lea r6, [r1*3]
%endif
.loop:
CHROMA_H_LOAD r6
call deblock_inter_body
CHROMA_H_STORE r6
lea r0, [r0+r1*(mmsize/4)]
add r4, mmsize/8
dec r5
jg .loop
RET
cglobal deblock_intra_body
LOAD_AB m4, m5, r2d, r3d
LOAD_MASK m0, m1, m2, m3, m4, m5, m7, m6, m4
CHROMA_DEBLOCK_P0_Q0_INTRA m1, m2, m0, m3, m7, m5, m6
ret
;-----------------------------------------------------------------------------
; void deblock_v_chroma_intra( uint16_t *pix, intptr_t stride, int alpha, int beta )
;-----------------------------------------------------------------------------
cglobal deblock_v_chroma_intra, 4,6,8
add r1, r1
mov r5, 32/mmsize
movd m5, r3d
mov r4, r0
sub r0, r1
sub r0, r1
SPLATW m5, m5
.loop:
CHROMA_V_LOAD r4
call deblock_intra_body
CHROMA_V_STORE
add r0, mmsize
add r4, mmsize
dec r5
jg .loop
RET
;-----------------------------------------------------------------------------
; void deblock_h_chroma_intra( uint16_t *pix, intptr_t stride, int alpha, int beta )
;-----------------------------------------------------------------------------
cglobal deblock_h_chroma_intra, 4,6,8
add r1, r1
mov r4, 32/mmsize
%if mmsize == 16
lea r5, [r1*3]
%endif
.loop:
CHROMA_H_LOAD r5
call deblock_intra_body
CHROMA_H_STORE r5
lea r0, [r0+r1*(mmsize/4)]
dec r4
jg .loop
RET
;-----------------------------------------------------------------------------
; void deblock_h_chroma_intra_mbaff( uint16_t *pix, intptr_t stride, int alpha, int beta )
;-----------------------------------------------------------------------------
cglobal deblock_h_chroma_intra_mbaff, 4,6,8
add r1, r1
%if mmsize == 8
mov r4, 16/mmsize
.loop:
%else
lea r5, [r1*3]
%endif
CHROMA_H_LOAD r5
LOAD_AB m4, m5, r2d, r3d
LOAD_MASK m0, m1, m2, m3, m4, m5, m7, m6, m4
CHROMA_DEBLOCK_P0_Q0_INTRA m1, m2, m0, m3, m7, m5, m6
CHROMA_H_STORE r5
%if mmsize == 8
lea r0, [r0+r1*(mmsize/4)]
dec r4
jg .loop
%endif
RET
;-----------------------------------------------------------------------------
; void deblock_h_chroma_mbaff( uint16_t *pix, intptr_t stride, int alpha, int beta, int8_t *tc0 )
;-----------------------------------------------------------------------------
cglobal deblock_h_chroma_mbaff, 5,7,8
add r1, r1
lea r6, [r1*3]
%if mmsize == 8
mov r5, 16/mmsize
.loop:
%endif
CHROMA_H_LOAD r6
LOAD_AB m4, m5, r2d, r3d
LOAD_MASK m0, m1, m2, m3, m4, m5, m7, m6, m4
movd m6, [r4]
punpcklbw m6, m6
psraw m6, 8
punpcklwd m6, m6
pand m7, m6
DEBLOCK_P0_Q0 m1, m2, m0, m3, m7, m5, m6
CHROMA_H_STORE r6
%if mmsize == 8
lea r0, [r0+r1*(mmsize/4)]
add r4, mmsize/4
dec r5
jg .loop
%endif
RET
;-----------------------------------------------------------------------------
; void deblock_h_chroma_422_intra( uint16_t *pix, intptr_t stride, int alpha, int beta )
;-----------------------------------------------------------------------------
cglobal deblock_h_chroma_422_intra, 4,6,8
add r1, r1
mov r4, 64/mmsize
%if mmsize == 16
lea r5, [r1*3]
%endif
.loop:
CHROMA_H_LOAD r5
call deblock_intra_body
CHROMA_H_STORE r5
lea r0, [r0+r1*(mmsize/4)]
dec r4
jg .loop
RET
;-----------------------------------------------------------------------------
; void deblock_h_chroma_422( uint16_t *pix, intptr_t stride, int alpha, int beta, int8_t *tc0 )
;-----------------------------------------------------------------------------
cglobal deblock_h_chroma_422, 5,7,8
add r1, r1
mov r5, 64/mmsize
lea r6, [r1*3]
.loop:
CHROMA_H_LOAD r6
LOAD_AB m4, m5, r2m, r3d
LOAD_MASK m0, m1, m2, m3, m4, m5, m7, m6, m4
pxor m4, m4
movd m6, [r4-1]
psraw m6, 8
SPLATW m6, m6
pmaxsw m6, m4
pand m7, m6
DEBLOCK_P0_Q0 m1, m2, m0, m3, m7, m5, m6
CHROMA_H_STORE r6
lea r0, [r0+r1*(mmsize/4)]
%if mmsize == 16
inc r4
%else
mov r2, r5
and r2, 1
add r4, r2 ; increment once every 2 iterations
%endif
dec r5
jg .loop
RET
%endmacro ; DEBLOCK_CHROMA
%if ARCH_X86_64 == 0
INIT_MMX mmx2
DEBLOCK_CHROMA
%endif
INIT_XMM sse2
DEBLOCK_CHROMA
INIT_XMM avx
DEBLOCK_CHROMA
%endif ; HIGH_BIT_DEPTH
%if HIGH_BIT_DEPTH == 0
%macro CHROMA_V_START 0
mov t5, r0
sub t5, r1
sub t5, r1
%if mmsize==8
mov dword r0m, 2
.loop:
%endif
%endmacro
%macro CHROMA_H_START 0
sub r0, 4
lea t6, [r1*3]
mov t5, r0
add r0, t6
%endmacro
%macro CHROMA_V_LOOP 1
%if mmsize==8
add r0, 8
add t5, 8
%if %1
add r4, 2
%endif
dec dword r0m
jg .loop
%endif
%endmacro
%macro CHROMA_H_LOOP 1
%if mmsize==8
lea r0, [r0+r1*4]
lea t5, [t5+r1*4]
%if %1
add r4, 2
%endif
dec dword r0m
jg .loop
%endif
%endmacro
%define t5 r5
%define t6 r6
%macro DEBLOCK_CHROMA 0
cglobal chroma_inter_body
LOAD_MASK r2d, r3d
movd m6, [r4] ; tc0
punpcklbw m6, m6
punpcklbw m6, m6
pand m7, m6
DEBLOCK_P0_Q0
ret
;-----------------------------------------------------------------------------
; void deblock_v_chroma( uint8_t *pix, intptr_t stride, int alpha, int beta, int8_t *tc0 )
;-----------------------------------------------------------------------------
cglobal deblock_v_chroma, 5,6,8
CHROMA_V_START
mova m0, [t5]
mova m1, [t5+r1]
mova m2, [r0]
mova m3, [r0+r1]
call chroma_inter_body
mova [t5+r1], m1
mova [r0], m2
CHROMA_V_LOOP 1
RET
;-----------------------------------------------------------------------------
; void deblock_h_chroma( uint8_t *pix, intptr_t stride, int alpha, int beta, int8_t *tc0 )
;-----------------------------------------------------------------------------
cglobal deblock_h_chroma, 5,7,8
CHROMA_H_START
%if mmsize==8
mov dword r0m, 2
.loop:
%endif
TRANSPOSE4x8W_LOAD PASS8ROWS(t5, r0, r1, t6)
call chroma_inter_body
TRANSPOSE8x2W_STORE PASS8ROWS(t5, r0, r1, t6, 2)
CHROMA_H_LOOP 1
RET
%endmacro ; DEBLOCK_CHROMA
INIT_XMM sse2
DEBLOCK_CHROMA
INIT_XMM avx
DEBLOCK_CHROMA
%if ARCH_X86_64 == 0
INIT_MMX mmx2
DEBLOCK_CHROMA
%endif
;-----------------------------------------------------------------------------
; void deblock_h_chroma_mbaff( uint8_t *pix, intptr_t stride, int alpha, int beta, int8_t *tc0 )
;-----------------------------------------------------------------------------
%macro DEBLOCK_H_CHROMA_420_MBAFF 0
cglobal deblock_h_chroma_mbaff, 5,7,8
CHROMA_H_START
TRANSPOSE4x4W_LOAD PASS8ROWS(t5, r0, r1, t6)
LOAD_MASK r2d, r3d
movd m6, [r4] ; tc0
punpcklbw m6, m6
pand m7, m6
DEBLOCK_P0_Q0
TRANSPOSE4x2W_STORE PASS8ROWS(t5, r0, r1, t6, 2)
RET
%endmacro
INIT_XMM sse2
DEBLOCK_H_CHROMA_420_MBAFF
%if ARCH_X86_64 == 0
INIT_MMX mmx2
DEBLOCK_H_CHROMA_420_MBAFF
%endif
%macro DEBLOCK_H_CHROMA_422 0
cglobal deblock_h_chroma_422, 5,8,8
%if ARCH_X86_64
%define cntr r7
%else
%define cntr dword r0m
%endif
CHROMA_H_START
mov cntr, 32/mmsize
.loop:
TRANSPOSE4x8W_LOAD PASS8ROWS(t5, r0, r1, t6)
LOAD_MASK r2d, r3d
movd m6, [r4] ; tc0
punpcklbw m6, m6
%if mmsize == 16
punpcklbw m6, m6
punpcklbw m6, m6
%else
pshufw m6, m6, q0000
%endif
pand m7, m6
DEBLOCK_P0_Q0
TRANSPOSE8x2W_STORE PASS8ROWS(t5, r0, r1, t6, 2)
lea r0, [r0+r1*(mmsize/2)]
lea t5, [t5+r1*(mmsize/2)]
add r4, mmsize/8
dec cntr
jg .loop
RET
%endmacro
INIT_MMX mmx2
DEBLOCK_H_CHROMA_422
INIT_XMM sse2
DEBLOCK_H_CHROMA_422
INIT_XMM avx
DEBLOCK_H_CHROMA_422
; in: %1=p0 %2=p1 %3=q1
; out: p0 = (p0 + q1 + 2*p1 + 2) >> 2
%macro CHROMA_INTRA_P0 3
pxor m4, %1, %3
pand m4, [pb_1] ; m4 = (p0^q1)&1
pavgb %1, %3
psubusb %1, m4
pavgb %1, %2 ; dst = avg(p1, avg(p0,q1) - ((p0^q1)&1))
%endmacro
%define t5 r4
%define t6 r5
%macro DEBLOCK_CHROMA_INTRA_BODY 0
cglobal chroma_intra_body
LOAD_MASK r2d, r3d
mova m5, m1
mova m6, m2
CHROMA_INTRA_P0 m1, m0, m3
CHROMA_INTRA_P0 m2, m3, m0
psubb m1, m5
psubb m2, m6
pand m1, m7
pand m2, m7
paddb m1, m5
paddb m2, m6
ret
%endmacro
%macro DEBLOCK_CHROMA_INTRA 0
;-----------------------------------------------------------------------------
; void deblock_v_chroma_intra( uint8_t *pix, intptr_t stride, int alpha, int beta )
;-----------------------------------------------------------------------------
cglobal deblock_v_chroma_intra, 4,5,8
CHROMA_V_START
mova m0, [t5]
mova m1, [t5+r1]
mova m2, [r0]
mova m3, [r0+r1]
call chroma_intra_body
mova [t5+r1], m1
mova [r0], m2
CHROMA_V_LOOP 0
RET
;-----------------------------------------------------------------------------
; void deblock_h_chroma_intra( uint8_t *pix, intptr_t stride, int alpha, int beta )
;-----------------------------------------------------------------------------
cglobal deblock_h_chroma_intra, 4,6,8
CHROMA_H_START
%if mmsize==8
mov dword r0m, 2
.loop:
%endif
TRANSPOSE4x8W_LOAD PASS8ROWS(t5, r0, r1, t6)
call chroma_intra_body
TRANSPOSE8x2W_STORE PASS8ROWS(t5, r0, r1, t6, 2)
CHROMA_H_LOOP 0
RET
cglobal deblock_h_chroma_422_intra, 4,7,8
CHROMA_H_START
mov r6d, 32/mmsize
.loop:
TRANSPOSE4x8W_LOAD PASS8ROWS(t5, r0, r1, t6)
call chroma_intra_body
TRANSPOSE8x2W_STORE PASS8ROWS(t5, r0, r1, t6, 2)
lea r0, [r0+r1*(mmsize/2)]
lea t5, [t5+r1*(mmsize/2)]
dec r6d
jg .loop
RET
%endmacro ; DEBLOCK_CHROMA_INTRA
INIT_XMM sse2
DEBLOCK_CHROMA_INTRA_BODY
DEBLOCK_CHROMA_INTRA
INIT_XMM avx
DEBLOCK_CHROMA_INTRA_BODY
DEBLOCK_CHROMA_INTRA
INIT_MMX mmx2
DEBLOCK_CHROMA_INTRA_BODY
%if ARCH_X86_64 == 0
DEBLOCK_CHROMA_INTRA
%endif
;-----------------------------------------------------------------------------
; void deblock_h_chroma_intra_mbaff( uint8_t *pix, intptr_t stride, int alpha, int beta )
;-----------------------------------------------------------------------------
INIT_MMX mmx2
cglobal deblock_h_chroma_intra_mbaff, 4,6,8
CHROMA_H_START
TRANSPOSE4x4W_LOAD PASS8ROWS(t5, r0, r1, t6)
call chroma_intra_body
TRANSPOSE4x2W_STORE PASS8ROWS(t5, r0, r1, t6, 2)
RET
%endif ; !HIGH_BIT_DEPTH
;-----------------------------------------------------------------------------
; static void deblock_strength( uint8_t nnz[48], int8_t ref[2][40], int16_t mv[2][40][2],
; uint8_t bs[2][4][4], int mvy_limit, int bframe )
;-----------------------------------------------------------------------------
%define scan8start (4+1*8)
%define nnz r0+scan8start
%define ref r1+scan8start
%define mv r2+scan8start*4
%define bs0 r3
%define bs1 r3+32
%macro LOAD_BYTES_MMX 1
movd m2, [%1+8*0-1]
movd m0, [%1+8*0]
movd m3, [%1+8*2-1]
movd m1, [%1+8*2]
punpckldq m2, [%1+8*1-1]
punpckldq m0, [%1+8*1]
punpckldq m3, [%1+8*3-1]
punpckldq m1, [%1+8*3]
%endmacro
%macro DEBLOCK_STRENGTH_REFS_MMX 0
LOAD_BYTES_MMX ref
pxor m2, m0
pxor m3, m1
por m2, [bs0+0]
por m3, [bs0+8]
movq [bs0+0], m2
movq [bs0+8], m3
movd m2, [ref-8*1]
movd m3, [ref+8*1]
punpckldq m2, m0 ; row -1, row 0
punpckldq m3, m1 ; row 1, row 2
pxor m0, m2
pxor m1, m3
por m0, [bs1+0]
por m1, [bs1+8]
movq [bs1+0], m0
movq [bs1+8], m1
%endmacro
%macro DEBLOCK_STRENGTH_MVS_MMX 2
mova m0, [mv-%2]
mova m1, [mv-%2+8]
psubw m0, [mv]
psubw m1, [mv+8]
packsswb m0, m1
ABSB m0, m1
psubusb m0, m7
packsswb m0, m0
por m0, [%1]
movd [%1], m0
%endmacro
%macro DEBLOCK_STRENGTH_NNZ_MMX 1
por m2, m0
por m3, m1
mova m4, [%1]
mova m5, [%1+8]
pminub m2, m6
pminub m3, m6
pminub m4, m6 ; mv ? 1 : 0
pminub m5, m6
paddb m2, m2 ; nnz ? 2 : 0
paddb m3, m3
pmaxub m2, m4
pmaxub m3, m5
%endmacro
%macro LOAD_BYTES_XMM 1
movu m2, [%1-4] ; FIXME could be aligned if we changed nnz's allocation
movu m1, [%1+12]
pslldq m0, m2, 1
shufps m2, m1, q3131 ; cur nnz, all rows
pslldq m1, 1
shufps m0, m1, q3131 ; left neighbors
pslldq m1, m2, 4
movd m3, [%1-8] ; could be palignr if nnz was aligned
por m1, m3 ; top neighbors
%endmacro
INIT_MMX mmx2
cglobal deblock_strength, 6,6
; Prepare mv comparison register
shl r4d, 8
add r4d, 3 - (1<<8)
movd m7, r4d
SPLATW m7, m7
mova m6, [pb_1]
pxor m0, m0
mova [bs0+0], m0
mova [bs0+8], m0
mova [bs1+0], m0
mova [bs1+8], m0
.lists:
DEBLOCK_STRENGTH_REFS_MMX
mov r4d, 4
.mvs:
DEBLOCK_STRENGTH_MVS_MMX bs0, 4
DEBLOCK_STRENGTH_MVS_MMX bs1, 4*8
add r2, 4*8
add r3, 4
dec r4d
jg .mvs
add r1, 40
add r2, 4*8
sub r3, 16
dec r5d
jge .lists
; Check nnz
LOAD_BYTES_MMX nnz
DEBLOCK_STRENGTH_NNZ_MMX bs0
; Transpose column output
SBUTTERFLY bw, 2, 3, 4
SBUTTERFLY bw, 2, 3, 4
mova [bs0+0], m2
mova [bs0+8], m3
movd m2, [nnz-8*1]
movd m3, [nnz+8*1]
punpckldq m2, m0 ; row -1, row 0
punpckldq m3, m1 ; row 1, row 2
DEBLOCK_STRENGTH_NNZ_MMX bs1
mova [bs1+0], m2
mova [bs1+8], m3
RET
%macro DEBLOCK_STRENGTH_XMM 0
cglobal deblock_strength, 6,6,7
; Prepare mv comparison register
shl r4d, 8
add r4d, 3 - (1<<8)
movd m6, r4d
SPLATW m6, m6
pxor m4, m4 ; bs0
pxor m5, m5 ; bs1
.lists:
; Check refs
LOAD_BYTES_XMM ref
pxor m0, m2
pxor m1, m2
por m4, m0
por m5, m1
; Check mvs
%if cpuflag(ssse3)
mova m0, [mv+4*8*0]
mova m1, [mv+4*8*1]
palignr m3, m0, [mv+4*8*0-16], 12
palignr m2, m1, [mv+4*8*1-16], 12
psubw m0, m3
psubw m1, m2
packsswb m0, m1
mova m2, [mv+4*8*2]
mova m1, [mv+4*8*3]
palignr m3, m2, [mv+4*8*2-16], 12
psubw m2, m3
palignr m3, m1, [mv+4*8*3-16], 12
psubw m1, m3
packsswb m2, m1
%else
movu m0, [mv-4+4*8*0]
movu m1, [mv-4+4*8*1]
movu m2, [mv-4+4*8*2]
movu m3, [mv-4+4*8*3]
psubw m0, [mv+4*8*0]
psubw m1, [mv+4*8*1]
psubw m2, [mv+4*8*2]
psubw m3, [mv+4*8*3]
packsswb m0, m1
packsswb m2, m3
%endif
ABSB m0, m1
ABSB m2, m3
psubusb m0, m6
psubusb m2, m6
packsswb m0, m2
por m4, m0
mova m0, [mv+4*8*-1]
mova m1, [mv+4*8* 0]
mova m2, [mv+4*8* 1]
mova m3, [mv+4*8* 2]
psubw m0, m1
psubw m1, m2
psubw m2, m3
psubw m3, [mv+4*8* 3]
packsswb m0, m1
packsswb m2, m3
ABSB m0, m1
ABSB m2, m3
psubusb m0, m6
psubusb m2, m6
packsswb m0, m2
por m5, m0
add r1, 40
add r2, 4*8*5
dec r5d
jge .lists
; Check nnz
LOAD_BYTES_XMM nnz
por m0, m2
por m1, m2
mova m6, [pb_1]
pminub m0, m6
pminub m1, m6
pminub m4, m6 ; mv ? 1 : 0
pminub m5, m6
paddb m0, m0 ; nnz ? 2 : 0
paddb m1, m1
pmaxub m4, m0
pmaxub m5, m1
%if cpuflag(ssse3)
pshufb m4, [transpose_shuf]
%else
movhlps m3, m4
punpcklbw m4, m3
movhlps m3, m4
punpcklbw m4, m3
%endif
mova [bs1], m5
mova [bs0], m4
RET
%endmacro
INIT_XMM sse2
DEBLOCK_STRENGTH_XMM
INIT_XMM ssse3
DEBLOCK_STRENGTH_XMM
INIT_XMM avx
DEBLOCK_STRENGTH_XMM
%macro LOAD_BYTES_YMM 1
movu m0, [%1-4] ; ___E FGHI ___J KLMN ___O PQRS ___T UVWX
pshufb m0, [load_bytes_shuf] ; EFGH JKLM FGHI KLMN OPQR TUVW PQRS UVWX
mova m2, [insert_top_shuf]
vpermq m1, m0, q3131 ; FGHI KLMN PQRS UVWX x2
vpermd m0, m2, m0 ; EFGH JKLM OPQR TUVW ____ FGHI KLMN PQRS
vpbroadcastd m2, [%1-8] ; ABCD ....
vpblendd m0, m0, m2, 00010000b ; EFGH JKLM OPQR TUVW ABCD FGHI KLMN PQRS
%endmacro
INIT_YMM avx2
cglobal deblock_strength, 6,6,7
; Prepare mv comparison register
shl r4d, 8
add r4d, 3 - (1<<8)
movd xm6, r4d
vpbroadcastw m6, xm6
pxor m5, m5 ; bs0,bs1
.lists:
; Check refs
LOAD_BYTES_YMM ref
pxor m0, m1
por m5, m0
; Check mvs
movu xm0, [mv-4+4*8*0]
vinserti128 m0, m0, [mv+4*8*-1], 1
vbroadcasti128 m2, [mv+4*8* 0]
vinserti128 m1, m2, [mv-4+4*8*1], 0
vbroadcasti128 m3, [mv+4*8* 1]
psubw m0, m2
psubw m1, m3
vinserti128 m2, m3, [mv-4+4*8*2], 0
vbroadcasti128 m4, [mv+4*8* 2]
vinserti128 m3, m4, [mv-4+4*8*3], 0
psubw m2, m4
vbroadcasti128 m4, [mv+4*8* 3]
psubw m3, m4
packsswb m0, m1
packsswb m2, m3
pabsb m0, m0
pabsb m2, m2
psubusb m0, m6
psubusb m2, m6
packsswb m0, m2
por m5, m0
add r1, 40
add r2, 4*8*5
dec r5d
jge .lists
; Check nnz
LOAD_BYTES_YMM nnz
por m0, m1
mova m6, [pb_1]
pminub m0, m6
pminub m5, m6 ; mv ? 1 : 0
paddb m0, m0 ; nnz ? 2 : 0
pmaxub m5, m0
vextracti128 [bs1], m5, 1
pshufb xm5, [transpose_shuf]
mova [bs0], xm5
RET
