CoCalc -- macroblock.c

05. Matplotlib / ffmpeg-3.0 / libx264 / common / macroblock.c
⁵²⁸⁶⁶ views
1
/*****************************************************************************
2
 * macroblock.c: macroblock common functions
3
 *****************************************************************************
4
 * Copyright (C) 2003-2016 x264 project
5
 *
6
 * Authors: Fiona Glaser <[email protected]>
7
 *          Laurent Aimar <[email protected]>
8
 *          Loren Merritt <[email protected]>
9
 *          Henrik Gramner <[email protected]>
10
 *
11
 * This program is free software; you can redistribute it and/or modify
12
 * it under the terms of the GNU General Public License as published by
13
 * the Free Software Foundation; either version 2 of the License, or
14
 * (at your option) any later version.
15
 *
16
 * This program is distributed in the hope that it will be useful,
17
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
18
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
19
 * GNU General Public License for more details.
20
 *
21
 * You should have received a copy of the GNU General Public License
22
 * along with this program; if not, write to the Free Software
23
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02111, USA.
24
 *
25
 * This program is also available under a commercial proprietary license.
26
 * For more information, contact us at [email protected].
27
 *****************************************************************************/
28

29
#include "common.h"
30
#include "encoder/me.h"
31

32
#define MC_LUMA(list,p) \
33
    h->mc.mc_luma( &h->mb.pic.p_fdec[p][4*y*FDEC_STRIDE+4*x], FDEC_STRIDE, \
34
                   &h->mb.pic.p_fref[list][i_ref][p*4], h->mb.pic.i_stride[p], \
35
                   mvx, mvy, 4*width, 4*height, \
36
                   list ? x264_weight_none : &h->sh.weight[i_ref][p] );
37

38
static NOINLINE void x264_mb_mc_0xywh( x264_t *h, int x, int y, int width, int height )
39
{
40
    int i8    = x264_scan8[0]+x+8*y;
41
    int i_ref = h->mb.cache.ref[0][i8];
42
    int mvx   = x264_clip3( h->mb.cache.mv[0][i8][0], h->mb.mv_min[0], h->mb.mv_max[0] ) + 4*4*x;
43
    int mvy   = x264_clip3( h->mb.cache.mv[0][i8][1], h->mb.mv_min[1], h->mb.mv_max[1] ) + 4*4*y;
44

45
    MC_LUMA( 0, 0 );
46

47
    if( CHROMA444 )
48
    {
49
        MC_LUMA( 0, 1 );
50
        MC_LUMA( 0, 2 );
51
    }
52
    else
53
    {
54
        int v_shift = CHROMA_V_SHIFT;
55
        // Chroma in 4:2:0 is offset if MCing from a field of opposite parity
56
        if( v_shift & MB_INTERLACED & i_ref )
57
            mvy += (h->mb.i_mb_y & 1)*4 - 2;
58

59
        int offset = (4*FDEC_STRIDE>>v_shift)*y + 2*x;
60
        height = 4*height >> v_shift;
61

62
        h->mc.mc_chroma( &h->mb.pic.p_fdec[1][offset],
63
                         &h->mb.pic.p_fdec[2][offset], FDEC_STRIDE,
64
                         h->mb.pic.p_fref[0][i_ref][4], h->mb.pic.i_stride[1],
65
                         mvx, 2*mvy>>v_shift, 2*width, height );
66

67
        if( h->sh.weight[i_ref][1].weightfn )
68
            h->sh.weight[i_ref][1].weightfn[width>>1]( &h->mb.pic.p_fdec[1][offset], FDEC_STRIDE,
69
                                                       &h->mb.pic.p_fdec[1][offset], FDEC_STRIDE,
70
                                                       &h->sh.weight[i_ref][1], height );
71
        if( h->sh.weight[i_ref][2].weightfn )
72
            h->sh.weight[i_ref][2].weightfn[width>>1]( &h->mb.pic.p_fdec[2][offset], FDEC_STRIDE,
73
                                                       &h->mb.pic.p_fdec[2][offset], FDEC_STRIDE,
74
                                                       &h->sh.weight[i_ref][2], height );
75
    }
76
}
77
static NOINLINE void x264_mb_mc_1xywh( x264_t *h, int x, int y, int width, int height )
78
{
79
    int i8    = x264_scan8[0]+x+8*y;
80
    int i_ref = h->mb.cache.ref[1][i8];
81
    int mvx   = x264_clip3( h->mb.cache.mv[1][i8][0], h->mb.mv_min[0], h->mb.mv_max[0] ) + 4*4*x;
82
    int mvy   = x264_clip3( h->mb.cache.mv[1][i8][1], h->mb.mv_min[1], h->mb.mv_max[1] ) + 4*4*y;
83

84
    MC_LUMA( 1, 0 );
85

86
    if( CHROMA444 )
87
    {
88
        MC_LUMA( 1, 1 );
89
        MC_LUMA( 1, 2 );
90
    }
91
    else
92
    {
93
        int v_shift = CHROMA_V_SHIFT;
94
        if( v_shift & MB_INTERLACED & i_ref )
95
            mvy += (h->mb.i_mb_y & 1)*4 - 2;
96

97
        int offset = (4*FDEC_STRIDE>>v_shift)*y + 2*x;
98
        h->mc.mc_chroma( &h->mb.pic.p_fdec[1][offset],
99
                         &h->mb.pic.p_fdec[2][offset], FDEC_STRIDE,
100
                         h->mb.pic.p_fref[1][i_ref][4], h->mb.pic.i_stride[1],
101
                         mvx, 2*mvy>>v_shift, 2*width, 4*height>>v_shift );
102
    }
103
}
104

105
#define MC_LUMA_BI(p) \
106
    src0 = h->mc.get_ref( tmp0, &i_stride0, &h->mb.pic.p_fref[0][i_ref0][p*4], h->mb.pic.i_stride[p], \
107
                          mvx0, mvy0, 4*width, 4*height, x264_weight_none ); \
108
    src1 = h->mc.get_ref( tmp1, &i_stride1, &h->mb.pic.p_fref[1][i_ref1][p*4], h->mb.pic.i_stride[p], \
109
                          mvx1, mvy1, 4*width, 4*height, x264_weight_none ); \
110
    h->mc.avg[i_mode]( &h->mb.pic.p_fdec[p][4*y*FDEC_STRIDE+4*x], FDEC_STRIDE, \
111
                       src0, i_stride0, src1, i_stride1, weight );
112

113
static NOINLINE void x264_mb_mc_01xywh( x264_t *h, int x, int y, int width, int height )
114
{
115
    int i8 = x264_scan8[0]+x+8*y;
116
    int i_ref0 = h->mb.cache.ref[0][i8];
117
    int i_ref1 = h->mb.cache.ref[1][i8];
118
    int weight = h->mb.bipred_weight[i_ref0][i_ref1];
119
    int mvx0   = x264_clip3( h->mb.cache.mv[0][i8][0], h->mb.mv_min[0], h->mb.mv_max[0] ) + 4*4*x;
120
    int mvx1   = x264_clip3( h->mb.cache.mv[1][i8][0], h->mb.mv_min[0], h->mb.mv_max[0] ) + 4*4*x;
121
    int mvy0   = x264_clip3( h->mb.cache.mv[0][i8][1], h->mb.mv_min[1], h->mb.mv_max[1] ) + 4*4*y;
122
    int mvy1   = x264_clip3( h->mb.cache.mv[1][i8][1], h->mb.mv_min[1], h->mb.mv_max[1] ) + 4*4*y;
123
    int i_mode = x264_size2pixel[height][width];
124
    intptr_t i_stride0 = 16, i_stride1 = 16;
125
    ALIGNED_ARRAY_N( pixel, tmp0,[16*16] );
126
    ALIGNED_ARRAY_N( pixel, tmp1,[16*16] );
127
    pixel *src0, *src1;
128

129
    MC_LUMA_BI( 0 );
130

131
    if( CHROMA444 )
132
    {
133
        MC_LUMA_BI( 1 );
134
        MC_LUMA_BI( 2 );
135
    }
136
    else
137
    {
138
        int v_shift = CHROMA_V_SHIFT;
139
        if( v_shift & MB_INTERLACED & i_ref0 )
140
            mvy0 += (h->mb.i_mb_y & 1)*4 - 2;
141
        if( v_shift & MB_INTERLACED & i_ref1 )
142
            mvy1 += (h->mb.i_mb_y & 1)*4 - 2;
143

144
        h->mc.mc_chroma( tmp0, tmp0+8, 16, h->mb.pic.p_fref[0][i_ref0][4], h->mb.pic.i_stride[1],
145
                         mvx0, 2*mvy0>>v_shift, 2*width, 4*height>>v_shift );
146
        h->mc.mc_chroma( tmp1, tmp1+8, 16, h->mb.pic.p_fref[1][i_ref1][4], h->mb.pic.i_stride[1],
147
                         mvx1, 2*mvy1>>v_shift, 2*width, 4*height>>v_shift );
148

149
        int chromapix = h->luma2chroma_pixel[i_mode];
150
        int offset = (4*FDEC_STRIDE>>v_shift)*y + 2*x;
151
        h->mc.avg[chromapix]( &h->mb.pic.p_fdec[1][offset], FDEC_STRIDE, tmp0,   16, tmp1,   16, weight );
152
        h->mc.avg[chromapix]( &h->mb.pic.p_fdec[2][offset], FDEC_STRIDE, tmp0+8, 16, tmp1+8, 16, weight );
153
    }
154
}
155

156
#undef MC_LUMA
157
#undef MC_LUMA_BI
158

159
void x264_mb_mc_8x8( x264_t *h, int i8 )
160
{
161
    int x = 2*(i8&1);
162
    int y = 2*(i8>>1);
163

164
    if( h->sh.i_type == SLICE_TYPE_P )
165
    {
166
        switch( h->mb.i_sub_partition[i8] )
167
        {
168
            case D_L0_8x8:
169
                x264_mb_mc_0xywh( h, x, y, 2, 2 );
170
                break;
171
            case D_L0_8x4:
172
                x264_mb_mc_0xywh( h, x, y+0, 2, 1 );
173
                x264_mb_mc_0xywh( h, x, y+1, 2, 1 );
174
                break;
175
            case D_L0_4x8:
176
                x264_mb_mc_0xywh( h, x+0, y, 1, 2 );
177
                x264_mb_mc_0xywh( h, x+1, y, 1, 2 );
178
                break;
179
            case D_L0_4x4:
180
                x264_mb_mc_0xywh( h, x+0, y+0, 1, 1 );
181
                x264_mb_mc_0xywh( h, x+1, y+0, 1, 1 );
182
                x264_mb_mc_0xywh( h, x+0, y+1, 1, 1 );
183
                x264_mb_mc_0xywh( h, x+1, y+1, 1, 1 );
184
                break;
185
        }
186
    }
187
    else
188
    {
189
        int scan8 = x264_scan8[0] + x + 8*y;
190

191
        if( h->mb.cache.ref[0][scan8] >= 0 )
192
            if( h->mb.cache.ref[1][scan8] >= 0 )
193
                x264_mb_mc_01xywh( h, x, y, 2, 2 );
194
            else
195
                x264_mb_mc_0xywh( h, x, y, 2, 2 );
196
        else
197
            x264_mb_mc_1xywh( h, x, y, 2, 2 );
198
    }
199
}
200

201
void x264_mb_mc( x264_t *h )
202
{
203
    if( h->mb.i_partition == D_8x8 )
204
    {
205
        for( int i = 0; i < 4; i++ )
206
            x264_mb_mc_8x8( h, i );
207
    }
208
    else
209
    {
210
        int ref0a = h->mb.cache.ref[0][x264_scan8[ 0]];
211
        int ref0b = h->mb.cache.ref[0][x264_scan8[12]];
212
        int ref1a = h->mb.cache.ref[1][x264_scan8[ 0]];
213
        int ref1b = h->mb.cache.ref[1][x264_scan8[12]];
214

215
        if( h->mb.i_partition == D_16x16 )
216
        {
217
            if( ref0a >= 0 )
218
                if( ref1a >= 0 ) x264_mb_mc_01xywh( h, 0, 0, 4, 4 );
219
                else             x264_mb_mc_0xywh ( h, 0, 0, 4, 4 );
220
            else                 x264_mb_mc_1xywh ( h, 0, 0, 4, 4 );
221
        }
222
        else if( h->mb.i_partition == D_16x8 )
223
        {
224
            if( ref0a >= 0 )
225
                if( ref1a >= 0 ) x264_mb_mc_01xywh( h, 0, 0, 4, 2 );
226
                else             x264_mb_mc_0xywh ( h, 0, 0, 4, 2 );
227
            else                 x264_mb_mc_1xywh ( h, 0, 0, 4, 2 );
228

229
            if( ref0b >= 0 )
230
                if( ref1b >= 0 ) x264_mb_mc_01xywh( h, 0, 2, 4, 2 );
231
                else             x264_mb_mc_0xywh ( h, 0, 2, 4, 2 );
232
            else                 x264_mb_mc_1xywh ( h, 0, 2, 4, 2 );
233
        }
234
        else if( h->mb.i_partition == D_8x16 )
235
        {
236
            if( ref0a >= 0 )
237
                if( ref1a >= 0 ) x264_mb_mc_01xywh( h, 0, 0, 2, 4 );
238
                else             x264_mb_mc_0xywh ( h, 0, 0, 2, 4 );
239
            else                 x264_mb_mc_1xywh ( h, 0, 0, 2, 4 );
240

241
            if( ref0b >= 0 )
242
                if( ref1b >= 0 ) x264_mb_mc_01xywh( h, 2, 0, 2, 4 );
243
                else             x264_mb_mc_0xywh ( h, 2, 0, 2, 4 );
244
            else                 x264_mb_mc_1xywh ( h, 2, 0, 2, 4 );
245
        }
246
    }
247
}
248

249
int x264_macroblock_cache_allocate( x264_t *h )
250
{
251
    int i_mb_count = h->mb.i_mb_count;
252

253
    h->mb.i_mb_stride = h->mb.i_mb_width;
254
    h->mb.i_b8_stride = h->mb.i_mb_width * 2;
255
    h->mb.i_b4_stride = h->mb.i_mb_width * 4;
256

257
    h->mb.b_interlaced = PARAM_INTERLACED;
258

259
    PREALLOC_INIT
260

261
    PREALLOC( h->mb.qp, i_mb_count * sizeof(int8_t) );
262
    PREALLOC( h->mb.cbp, i_mb_count * sizeof(int16_t) );
263
    PREALLOC( h->mb.mb_transform_size, i_mb_count * sizeof(int8_t) );
264
    PREALLOC( h->mb.slice_table, i_mb_count * sizeof(uint16_t) );
265

266
    /* 0 -> 3 top(4), 4 -> 6 : left(3) */
267
    PREALLOC( h->mb.intra4x4_pred_mode, i_mb_count * 8 * sizeof(int8_t) );
268

269
    /* all coeffs */
270
    PREALLOC( h->mb.non_zero_count, i_mb_count * 48 * sizeof(uint8_t) );
271

272
    if( h->param.b_cabac )
273
    {
274
        PREALLOC( h->mb.skipbp, i_mb_count * sizeof(int8_t) );
275
        PREALLOC( h->mb.chroma_pred_mode, i_mb_count * sizeof(int8_t) );
276
        PREALLOC( h->mb.mvd[0], i_mb_count * sizeof( **h->mb.mvd ) );
277
        if( h->param.i_bframe )
278
            PREALLOC( h->mb.mvd[1], i_mb_count * sizeof( **h->mb.mvd ) );
279
    }
280

281
    for( int i = 0; i < 2; i++ )
282
    {
283
        int i_refs = X264_MIN(X264_REF_MAX, (i ? 1 + !!h->param.i_bframe_pyramid : h->param.i_frame_reference) ) << PARAM_INTERLACED;
284
        if( h->param.analyse.i_weighted_pred == X264_WEIGHTP_SMART )
285
            i_refs = X264_MIN(X264_REF_MAX, i_refs + 1 + (BIT_DEPTH == 8)); //smart weights add two duplicate frames, one in >8-bit
286

287
        for( int j = !i; j < i_refs; j++ )
288
            PREALLOC( h->mb.mvr[i][j], 2 * (i_mb_count + 1) * sizeof(int16_t) );
289
    }
290

291
    if( h->param.analyse.i_weighted_pred )
292
    {
293
        int i_padv = PADV << PARAM_INTERLACED;
294
        int luma_plane_size = 0;
295
        int numweightbuf;
296

297
        if( h->param.analyse.i_weighted_pred == X264_WEIGHTP_FAKE )
298
        {
299
            // only need buffer for lookahead
300
            if( !h->param.i_sync_lookahead || h == h->thread[h->param.i_threads] )
301
            {
302
                // Fake analysis only works on lowres
303
                luma_plane_size = h->fdec->i_stride_lowres * (h->mb.i_mb_height*8+2*i_padv);
304
                // Only need 1 buffer for analysis
305
                numweightbuf = 1;
306
            }
307
            else
308
                numweightbuf = 0;
309
        }
310
        else
311
        {
312
            /* Both ref and fenc is stored for 4:2:0 and 4:2:2 which means that 4:2:0 and 4:4:4
313
             * needs the same amount of space and 4:2:2 needs twice that much */
314
            luma_plane_size = h->fdec->i_stride[0] * (h->mb.i_mb_height*(16<<(CHROMA_FORMAT==CHROMA_422))+2*i_padv);
315

316
            if( h->param.analyse.i_weighted_pred == X264_WEIGHTP_SMART )
317
                //smart can weight one ref and one offset -1 in 8-bit
318
                numweightbuf = 1 + (BIT_DEPTH == 8);
319
            else
320
                //simple only has one weighted ref
321
                numweightbuf = 1;
322
        }
323

324
        for( int i = 0; i < numweightbuf; i++ )
325
            PREALLOC( h->mb.p_weight_buf[i], luma_plane_size * sizeof(pixel) );
326
    }
327

328
    PREALLOC_END( h->mb.base );
329

330
    memset( h->mb.slice_table, -1, i_mb_count * sizeof(uint16_t) );
331

332
    for( int i = 0; i < 2; i++ )
333
    {
334
        int i_refs = X264_MIN(X264_REF_MAX, (i ? 1 + !!h->param.i_bframe_pyramid : h->param.i_frame_reference) ) << PARAM_INTERLACED;
335
        if( h->param.analyse.i_weighted_pred == X264_WEIGHTP_SMART )
336
            i_refs = X264_MIN(X264_REF_MAX, i_refs + 1 + (BIT_DEPTH == 8)); //smart weights add two duplicate frames, one in >8-bit
337

338
        for( int j = !i; j < i_refs; j++ )
339
        {
340
            M32( h->mb.mvr[i][j][0] ) = 0;
341
            h->mb.mvr[i][j]++;
342
        }
343
    }
344

345
    return 0;
346
fail:
347
    return -1;
348
}
349
void x264_macroblock_cache_free( x264_t *h )
350
{
351
    x264_free( h->mb.base );
352
}
353

354
int x264_macroblock_thread_allocate( x264_t *h, int b_lookahead )
355
{
356
    if( !b_lookahead )
357
    {
358
        for( int i = 0; i < (PARAM_INTERLACED ? 5 : 2); i++ )
359
            for( int j = 0; j < (CHROMA444 ? 3 : 2); j++ )
360
            {
361
                CHECKED_MALLOC( h->intra_border_backup[i][j], (h->sps->i_mb_width*16+32) * sizeof(pixel) );
362
                h->intra_border_backup[i][j] += 16;
363
            }
364
        for( int i = 0; i <= PARAM_INTERLACED; i++ )
365
        {
366
            if( h->param.b_sliced_threads )
367
            {
368
                /* Only allocate the first one, and allocate it for the whole frame, because we
369
                 * won't be deblocking until after the frame is fully encoded. */
370
                if( h == h->thread[0] && !i )
371
                    CHECKED_MALLOC( h->deblock_strength[0], sizeof(**h->deblock_strength) * h->mb.i_mb_count );
372
                else
373
                    h->deblock_strength[i] = h->thread[0]->deblock_strength[0];
374
            }
375
            else
376
                CHECKED_MALLOC( h->deblock_strength[i], sizeof(**h->deblock_strength) * h->mb.i_mb_width );
377
            h->deblock_strength[1] = h->deblock_strength[i];
378
        }
379
    }
380

381
    /* Allocate scratch buffer */
382
    int scratch_size = 0;
383
    if( !b_lookahead )
384
    {
385
        int buf_hpel = (h->thread[0]->fdec->i_width[0]+48+32) * sizeof(int16_t);
386
        int buf_ssim = h->param.analyse.b_ssim * 8 * (h->param.i_width/4+3) * sizeof(int);
387
        int me_range = X264_MIN(h->param.analyse.i_me_range, h->param.analyse.i_mv_range);
388
        int buf_tesa = (h->param.analyse.i_me_method >= X264_ME_ESA) *
389
            ((me_range*2+24) * sizeof(int16_t) + (me_range+4) * (me_range+1) * 4 * sizeof(mvsad_t));
390
        scratch_size = X264_MAX3( buf_hpel, buf_ssim, buf_tesa );
391
    }
392
    int buf_mbtree = h->param.rc.b_mb_tree * ((h->mb.i_mb_width+7)&~7) * sizeof(int16_t);
393
    scratch_size = X264_MAX( scratch_size, buf_mbtree );
394
    if( scratch_size )
395
        CHECKED_MALLOC( h->scratch_buffer, scratch_size );
396
    else
397
        h->scratch_buffer = NULL;
398

399
    int buf_lookahead_threads = (h->mb.i_mb_height + (4 + 32) * h->param.i_lookahead_threads) * sizeof(int) * 2;
400
    int buf_mbtree2 = buf_mbtree * 12; /* size of the internal propagate_list asm buffer */
401
    scratch_size = X264_MAX( buf_lookahead_threads, buf_mbtree2 );
402
    CHECKED_MALLOC( h->scratch_buffer2, scratch_size );
403

404
    return 0;
405
fail:
406
    return -1;
407
}
408

409
void x264_macroblock_thread_free( x264_t *h, int b_lookahead )
410
{
411
    if( !b_lookahead )
412
    {
413
        for( int i = 0; i <= PARAM_INTERLACED; i++ )
414
            if( !h->param.b_sliced_threads || (h == h->thread[0] && !i) )
415
                x264_free( h->deblock_strength[i] );
416
        for( int i = 0; i < (PARAM_INTERLACED ? 5 : 2); i++ )
417
            for( int j = 0; j < (CHROMA444 ? 3 : 2); j++ )
418
                x264_free( h->intra_border_backup[i][j] - 16 );
419
    }
420
    x264_free( h->scratch_buffer );
421
    x264_free( h->scratch_buffer2 );
422
}
423

424
void x264_macroblock_slice_init( x264_t *h )
425
{
426
    h->mb.mv[0] = h->fdec->mv[0];
427
    h->mb.mv[1] = h->fdec->mv[1];
428
    h->mb.mvr[0][0] = h->fdec->mv16x16;
429
    h->mb.ref[0] = h->fdec->ref[0];
430
    h->mb.ref[1] = h->fdec->ref[1];
431
    h->mb.type = h->fdec->mb_type;
432
    h->mb.partition = h->fdec->mb_partition;
433
    h->mb.field = h->fdec->field;
434

435
    h->fdec->i_ref[0] = h->i_ref[0];
436
    h->fdec->i_ref[1] = h->i_ref[1];
437
    for( int i = 0; i < h->i_ref[0]; i++ )
438
        h->fdec->ref_poc[0][i] = h->fref[0][i]->i_poc;
439
    if( h->sh.i_type == SLICE_TYPE_B )
440
    {
441
        for( int i = 0; i < h->i_ref[1]; i++ )
442
            h->fdec->ref_poc[1][i] = h->fref[1][i]->i_poc;
443

444
        map_col_to_list0(-1) = -1;
445
        map_col_to_list0(-2) = -2;
446
        for( int i = 0; i < h->fref[1][0]->i_ref[0]; i++ )
447
        {
448
            int poc = h->fref[1][0]->ref_poc[0][i];
449
            map_col_to_list0(i) = -2;
450
            for( int j = 0; j < h->i_ref[0]; j++ )
451
                if( h->fref[0][j]->i_poc == poc )
452
                {
453
                    map_col_to_list0(i) = j;
454
                    break;
455
                }
456
        }
457
    }
458
    else if( h->sh.i_type == SLICE_TYPE_P )
459
    {
460
        if( h->sh.i_disable_deblocking_filter_idc != 1 && h->param.analyse.i_weighted_pred == X264_WEIGHTP_SMART )
461
        {
462
            deblock_ref_table(-2) = -2;
463
            deblock_ref_table(-1) = -1;
464
            for( int i = 0; i < h->i_ref[0] << SLICE_MBAFF; i++ )
465
            {
466
                /* Mask off high bits to avoid frame num collisions with -1/-2.
467
                 * In current x264 frame num values don't cover a range of more
468
                 * than 32, so 6 bits is enough for uniqueness. */
469
                if( !MB_INTERLACED )
470
                    deblock_ref_table(i) = h->fref[0][i]->i_frame_num&63;
471
                else
472
                    deblock_ref_table(i) = ((h->fref[0][i>>1]->i_frame_num&63)<<1) + (i&1);
473
            }
474
        }
475
    }
476

477
    /* init with not available (for top right idx=7,15) */
478
    memset( h->mb.cache.ref, -2, sizeof( h->mb.cache.ref ) );
479

480
    if( h->i_ref[0] > 0 )
481
        for( int field = 0; field <= SLICE_MBAFF; field++ )
482
        {
483
            int curpoc = h->fdec->i_poc + h->fdec->i_delta_poc[field];
484
            int refpoc = h->fref[0][0]->i_poc + h->fref[0][0]->i_delta_poc[field];
485
            int delta = curpoc - refpoc;
486

487
            h->fdec->inv_ref_poc[field] = (256 + delta/2) / delta;
488
        }
489

490
    h->mb.i_neighbour4[6] =
491
    h->mb.i_neighbour4[9] =
492
    h->mb.i_neighbour4[12] =
493
    h->mb.i_neighbour4[14] = MB_LEFT|MB_TOP|MB_TOPLEFT|MB_TOPRIGHT;
494
    h->mb.i_neighbour4[3] =
495
    h->mb.i_neighbour4[7] =
496
    h->mb.i_neighbour4[11] =
497
    h->mb.i_neighbour4[13] =
498
    h->mb.i_neighbour4[15] =
499
    h->mb.i_neighbour8[3] = MB_LEFT|MB_TOP|MB_TOPLEFT;
500
}
501

502
void x264_macroblock_thread_init( x264_t *h )
503
{
504
    h->mb.i_me_method = h->param.analyse.i_me_method;
505
    h->mb.i_subpel_refine = h->param.analyse.i_subpel_refine;
506
    if( h->sh.i_type == SLICE_TYPE_B && (h->mb.i_subpel_refine == 6 || h->mb.i_subpel_refine == 8) )
507
        h->mb.i_subpel_refine--;
508
    h->mb.b_chroma_me = h->param.analyse.b_chroma_me &&
509
                        ((h->sh.i_type == SLICE_TYPE_P && h->mb.i_subpel_refine >= 5) ||
510
                         (h->sh.i_type == SLICE_TYPE_B && h->mb.i_subpel_refine >= 9));
511
    h->mb.b_dct_decimate = h->sh.i_type == SLICE_TYPE_B ||
512
                          (h->param.analyse.b_dct_decimate && h->sh.i_type != SLICE_TYPE_I);
513
    h->mb.i_mb_prev_xy = -1;
514

515
    /*          4:2:0                      4:2:2                      4:4:4
516
     * fdec            fenc       fdec            fenc       fdec            fenc
517
     * y y y y y y y   Y Y Y Y    y y y y y y y   Y Y Y Y    y y y y y y y   Y Y Y Y
518
     * y Y Y Y Y       Y Y Y Y    y Y Y Y Y       Y Y Y Y    y Y Y Y Y       Y Y Y Y
519
     * y Y Y Y Y       Y Y Y Y    y Y Y Y Y       Y Y Y Y    y Y Y Y Y       Y Y Y Y
520
     * y Y Y Y Y       Y Y Y Y    y Y Y Y Y       Y Y Y Y    y Y Y Y Y       Y Y Y Y
521
     * y Y Y Y Y       U U V V    y Y Y Y Y       U U V V    y Y Y Y Y       U U U U
522
     * u u u   v v v   U U V V    u u u   v v v   U U V V    u u u u u u u   U U U U
523
     * u U U   v V V              u U U   v V V   U U V V    u U U U U       U U U U
524
     * u U U   v V V              u U U   v V V   U U V V    u U U U U       U U U U
525
     *                            u U U   v V V              u U U U U       V V V V
526
     *                            u U U   v V V              u U U U U       V V V V
527
     *                                                       v v v v v v v   V V V V
528
     *                                                       v V V V V       V V V V
529
     *                                                       v V V V V
530
     *                                                       v V V V V
531
     *                                                       v V V V V
532
     */
533
    h->mb.pic.p_fenc[0] = h->mb.pic.fenc_buf;
534
    h->mb.pic.p_fdec[0] = h->mb.pic.fdec_buf + 2*FDEC_STRIDE;
535
    h->mb.pic.p_fenc[1] = h->mb.pic.fenc_buf + 16*FENC_STRIDE;
536
    h->mb.pic.p_fdec[1] = h->mb.pic.fdec_buf + 19*FDEC_STRIDE;
537
    if( CHROMA444 )
538
    {
539
        h->mb.pic.p_fenc[2] = h->mb.pic.fenc_buf + 32*FENC_STRIDE;
540
        h->mb.pic.p_fdec[2] = h->mb.pic.fdec_buf + 36*FDEC_STRIDE;
541
    }
542
    else
543
    {
544
        h->mb.pic.p_fenc[2] = h->mb.pic.fenc_buf + 16*FENC_STRIDE + 8;
545
        h->mb.pic.p_fdec[2] = h->mb.pic.fdec_buf + 19*FDEC_STRIDE + 16;
546
    }
547
}
548

549
void x264_prefetch_fenc( x264_t *h, x264_frame_t *fenc, int i_mb_x, int i_mb_y )
550
{
551
    int stride_y  = fenc->i_stride[0];
552
    int stride_uv = fenc->i_stride[1];
553
    int off_y  = 16 * i_mb_x + 16 * i_mb_y * stride_y;
554
    int off_uv = 16 * i_mb_x + (16 * i_mb_y * stride_uv >> CHROMA_V_SHIFT);
555
    h->mc.prefetch_fenc( fenc->plane[0]+off_y, stride_y,
556
                         fenc->plane[1]+off_uv, stride_uv, i_mb_x );
557
}
558

559
NOINLINE void x264_copy_column8( pixel *dst, pixel *src )
560
{
561
    // input pointers are offset by 4 rows because that's faster (smaller instruction size on x86)
562
    for( int i = -4; i < 4; i++ )
563
        dst[i*FDEC_STRIDE] = src[i*FDEC_STRIDE];
564
}
565

566
static void ALWAYS_INLINE x264_macroblock_load_pic_pointers( x264_t *h, int mb_x, int mb_y, int i, int b_chroma, int b_mbaff )
567
{
568
    int mb_interlaced = b_mbaff && MB_INTERLACED;
569
    int height = b_chroma ? 16 >> CHROMA_V_SHIFT : 16;
570
    int i_stride = h->fdec->i_stride[i];
571
    int i_stride2 = i_stride << mb_interlaced;
572
    int i_pix_offset = mb_interlaced
573
                     ? 16 * mb_x + height * (mb_y&~1) * i_stride + (mb_y&1) * i_stride
574
                     : 16 * mb_x + height * mb_y * i_stride;
575
    pixel *plane_fdec = &h->fdec->plane[i][i_pix_offset];
576
    int fdec_idx = b_mbaff ? (mb_interlaced ? (3 + (mb_y&1)) : (mb_y&1) ? 2 : 4) : !(mb_y&1);
577
    pixel *intra_fdec = &h->intra_border_backup[fdec_idx][i][mb_x*16];
578
    int ref_pix_offset[2] = { i_pix_offset, i_pix_offset };
579
    /* ref_pix_offset[0] references the current field and [1] the opposite field. */
580
    if( mb_interlaced )
581
        ref_pix_offset[1] += (1-2*(mb_y&1)) * i_stride;
582
    h->mb.pic.i_stride[i] = i_stride2;
583
    h->mb.pic.p_fenc_plane[i] = &h->fenc->plane[i][i_pix_offset];
584
    if( b_chroma )
585
    {
586
        h->mc.load_deinterleave_chroma_fenc( h->mb.pic.p_fenc[1], h->mb.pic.p_fenc_plane[1], i_stride2, height );
587
        memcpy( h->mb.pic.p_fdec[1]-FDEC_STRIDE, intra_fdec, 8*sizeof(pixel) );
588
        memcpy( h->mb.pic.p_fdec[2]-FDEC_STRIDE, intra_fdec+8, 8*sizeof(pixel) );
589
        h->mb.pic.p_fdec[1][-FDEC_STRIDE-1] = intra_fdec[-1-8];
590
        h->mb.pic.p_fdec[2][-FDEC_STRIDE-1] = intra_fdec[-1];
591
    }
592
    else
593
    {
594
        h->mc.copy[PIXEL_16x16]( h->mb.pic.p_fenc[i], FENC_STRIDE, h->mb.pic.p_fenc_plane[i], i_stride2, 16 );
595
        memcpy( h->mb.pic.p_fdec[i]-FDEC_STRIDE, intra_fdec, 24*sizeof(pixel) );
596
        h->mb.pic.p_fdec[i][-FDEC_STRIDE-1] = intra_fdec[-1];
597
    }
598
    if( b_mbaff || h->mb.b_reencode_mb )
599
    {
600
        for( int j = 0; j < height; j++ )
601
            if( b_chroma )
602
            {
603
                h->mb.pic.p_fdec[1][-1+j*FDEC_STRIDE] = plane_fdec[-2+j*i_stride2];
604
                h->mb.pic.p_fdec[2][-1+j*FDEC_STRIDE] = plane_fdec[-1+j*i_stride2];
605
            }
606
            else
607
                h->mb.pic.p_fdec[i][-1+j*FDEC_STRIDE] = plane_fdec[-1+j*i_stride2];
608
    }
609
    pixel *plane_src, **filtered_src;
610
    for( int j = 0; j < h->mb.pic.i_fref[0]; j++ )
611
    {
612
        // Interpolate between pixels in same field.
613
        if( mb_interlaced )
614
        {
615
            plane_src = h->fref[0][j>>1]->plane_fld[i];
616
            filtered_src = h->fref[0][j>>1]->filtered_fld[i];
617
        }
618
        else
619
        {
620
            plane_src = h->fref[0][j]->plane[i];
621
            filtered_src = h->fref[0][j]->filtered[i];
622
        }
623
        h->mb.pic.p_fref[0][j][i*4] = plane_src + ref_pix_offset[j&1];
624

625
        if( !b_chroma )
626
        {
627
            for( int k = 1; k < 4; k++ )
628
                h->mb.pic.p_fref[0][j][i*4+k] = filtered_src[k] + ref_pix_offset[j&1];
629
            if( !i )
630
            {
631
                if( h->sh.weight[j][0].weightfn )
632
                    h->mb.pic.p_fref_w[j] = &h->fenc->weighted[j >> mb_interlaced][ref_pix_offset[j&1]];
633
                else
634
                    h->mb.pic.p_fref_w[j] = h->mb.pic.p_fref[0][j][0];
635
            }
636
        }
637
    }
638
    if( h->sh.i_type == SLICE_TYPE_B )
639
        for( int j = 0; j < h->mb.pic.i_fref[1]; j++ )
640
        {
641
            if( mb_interlaced )
642
            {
643
                plane_src = h->fref[1][j>>1]->plane_fld[i];
644
                filtered_src = h->fref[1][j>>1]->filtered_fld[i];
645
            }
646
            else
647
            {
648
                plane_src = h->fref[1][j]->plane[i];
649
                filtered_src = h->fref[1][j]->filtered[i];
650
            }
651
            h->mb.pic.p_fref[1][j][i*4] = plane_src + ref_pix_offset[j&1];
652

653
            if( !b_chroma )
654
                for( int k = 1; k < 4; k++ )
655
                    h->mb.pic.p_fref[1][j][i*4+k] = filtered_src[k] + ref_pix_offset[j&1];
656
        }
657
}
658

659
static const x264_left_table_t left_indices[4] =
660
{
661
    /* Current is progressive */
662
    {{ 4, 4, 5, 5}, { 3,  3,  7,  7}, {16+1, 16+1, 32+1, 32+1}, {0, 0, 1, 1}, {0, 0, 0, 0}},
663
    {{ 6, 6, 3, 3}, {11, 11, 15, 15}, {16+5, 16+5, 32+5, 32+5}, {2, 2, 3, 3}, {1, 1, 1, 1}},
664
    /* Current is interlaced */
665
    {{ 4, 6, 4, 6}, { 3, 11,  3, 11}, {16+1, 16+1, 32+1, 32+1}, {0, 2, 0, 2}, {0, 1, 0, 1}},
666
    /* Both same */
667
    {{ 4, 5, 6, 3}, { 3,  7, 11, 15}, {16+1, 16+5, 32+1, 32+5}, {0, 1, 2, 3}, {0, 0, 1, 1}}
668
};
669

670
static void ALWAYS_INLINE x264_macroblock_cache_load_neighbours( x264_t *h, int mb_x, int mb_y, int b_interlaced )
671
{
672
    const int mb_interlaced = b_interlaced && MB_INTERLACED;
673
    int top_y = mb_y - (1 << mb_interlaced);
674
    int top = top_y * h->mb.i_mb_stride + mb_x;
675

676
    h->mb.i_mb_x = mb_x;
677
    h->mb.i_mb_y = mb_y;
678
    h->mb.i_mb_xy = mb_y * h->mb.i_mb_stride + mb_x;
679
    h->mb.i_b8_xy = 2*(mb_y * h->mb.i_b8_stride + mb_x);
680
    h->mb.i_b4_xy = 4*(mb_y * h->mb.i_b4_stride + mb_x);
681
    h->mb.left_b8[0] =
682
    h->mb.left_b8[1] = -1;
683
    h->mb.left_b4[0] =
684
    h->mb.left_b4[1] = -1;
685
    h->mb.i_neighbour = 0;
686
    h->mb.i_neighbour_intra = 0;
687
    h->mb.i_neighbour_frame = 0;
688
    h->mb.i_mb_top_xy = -1;
689
    h->mb.i_mb_top_y = -1;
690
    h->mb.i_mb_left_xy[0] = h->mb.i_mb_left_xy[1] = -1;
691
    h->mb.i_mb_topleft_xy = -1;
692
    h->mb.i_mb_topright_xy = -1;
693
    h->mb.i_mb_type_top = -1;
694
    h->mb.i_mb_type_left[0] = h->mb.i_mb_type_left[1] = -1;
695
    h->mb.i_mb_type_topleft = -1;
696
    h->mb.i_mb_type_topright = -1;
697
    h->mb.left_index_table = &left_indices[3];
698
    h->mb.topleft_partition = 0;
699

700
    int topleft_y = top_y;
701
    int topright_y = top_y;
702
    int left[2];
703

704
    left[0] = left[1] = h->mb.i_mb_xy - 1;
705
    h->mb.left_b8[0] = h->mb.left_b8[1] = h->mb.i_b8_xy - 2;
706
    h->mb.left_b4[0] = h->mb.left_b4[1] = h->mb.i_b4_xy - 4;
707

708
    if( b_interlaced )
709
    {
710
        h->mb.i_mb_top_mbpair_xy = h->mb.i_mb_xy - 2*h->mb.i_mb_stride;
711
        h->mb.i_mb_topleft_y = -1;
712
        h->mb.i_mb_topright_y = -1;
713

714
        if( mb_y&1 )
715
        {
716
            if( mb_x && mb_interlaced != h->mb.field[h->mb.i_mb_xy-1] )
717
            {
718
                left[0] = left[1] = h->mb.i_mb_xy - 1 - h->mb.i_mb_stride;
719
                h->mb.left_b8[0] = h->mb.left_b8[1] = h->mb.i_b8_xy - 2 - 2*h->mb.i_b8_stride;
720
                h->mb.left_b4[0] = h->mb.left_b4[1] = h->mb.i_b4_xy - 4 - 4*h->mb.i_b4_stride;
721

722
                if( mb_interlaced )
723
                {
724
                    h->mb.left_index_table = &left_indices[2];
725
                    left[1] += h->mb.i_mb_stride;
726
                    h->mb.left_b8[1] += 2*h->mb.i_b8_stride;
727
                    h->mb.left_b4[1] += 4*h->mb.i_b4_stride;
728
                }
729
                else
730
                {
731
                    h->mb.left_index_table = &left_indices[1];
732
                    topleft_y++;
733
                    h->mb.topleft_partition = 1;
734
                }
735
            }
736
            if( !mb_interlaced )
737
                topright_y = -1;
738
        }
739
        else
740
        {
741
            if( mb_interlaced && top >= 0 )
742
            {
743
                if( !h->mb.field[top] )
744
                {
745
                    top += h->mb.i_mb_stride;
746
                    top_y++;
747
                }
748
                if( mb_x )
749
                    topleft_y += !h->mb.field[h->mb.i_mb_stride*topleft_y + mb_x - 1];
750
                if( mb_x < h->mb.i_mb_width-1 )
751
                    topright_y += !h->mb.field[h->mb.i_mb_stride*topright_y + mb_x + 1];
752
            }
753
            if( mb_x && mb_interlaced != h->mb.field[h->mb.i_mb_xy-1] )
754
            {
755
                if( mb_interlaced )
756
                {
757
                    h->mb.left_index_table = &left_indices[2];
758
                    left[1] += h->mb.i_mb_stride;
759
                    h->mb.left_b8[1] += 2*h->mb.i_b8_stride;
760
                    h->mb.left_b4[1] += 4*h->mb.i_b4_stride;
761
                }
762
                else
763
                    h->mb.left_index_table = &left_indices[0];
764
            }
765
        }
766
    }
767

768
    if( mb_x > 0 )
769
    {
770
        h->mb.i_neighbour_frame |= MB_LEFT;
771
        h->mb.i_mb_left_xy[0] = left[0];
772
        h->mb.i_mb_left_xy[1] = left[1];
773
        h->mb.i_mb_type_left[0] = h->mb.type[h->mb.i_mb_left_xy[0]];
774
        h->mb.i_mb_type_left[1] = h->mb.type[h->mb.i_mb_left_xy[1]];
775
        if( h->mb.slice_table[left[0]] == h->sh.i_first_mb )
776
        {
777
            h->mb.i_neighbour |= MB_LEFT;
778

779
            // FIXME: We don't currently support constrained intra + mbaff.
780
            if( !h->param.b_constrained_intra || IS_INTRA( h->mb.i_mb_type_left[0] ) )
781
                h->mb.i_neighbour_intra |= MB_LEFT;
782
        }
783
    }
784

785
    /* We can't predict from the previous threadslice since it hasn't been encoded yet. */
786
    if( (h->i_threadslice_start >> mb_interlaced) != (mb_y >> mb_interlaced) )
787
    {
788
        if( top >= 0 )
789
        {
790
            h->mb.i_neighbour_frame |= MB_TOP;
791
            h->mb.i_mb_top_xy = top;
792
            h->mb.i_mb_top_y = top_y;
793
            h->mb.i_mb_type_top = h->mb.type[h->mb.i_mb_top_xy];
794
            if( h->mb.slice_table[top] == h->sh.i_first_mb )
795
            {
796
                h->mb.i_neighbour |= MB_TOP;
797

798
                if( !h->param.b_constrained_intra || IS_INTRA( h->mb.i_mb_type_top ) )
799
                    h->mb.i_neighbour_intra |= MB_TOP;
800

801
                /* We only need to prefetch the top blocks because the left was just written
802
                 * to as part of the previous cache_save.  Since most target CPUs use write-allocate
803
                 * caches, left blocks are near-guaranteed to be in L1 cache.  Top--not so much. */
804
                x264_prefetch( &h->mb.cbp[top] );
805
                x264_prefetch( h->mb.intra4x4_pred_mode[top] );
806
                x264_prefetch( &h->mb.non_zero_count[top][12] );
807
                /* These aren't always allocated, but prefetching an invalid address can't hurt. */
808
                x264_prefetch( &h->mb.mb_transform_size[top] );
809
                x264_prefetch( &h->mb.skipbp[top] );
810
            }
811
        }
812

813
        if( mb_x > 0 && topleft_y >= 0  )
814
        {
815
            h->mb.i_neighbour_frame |= MB_TOPLEFT;
816
            h->mb.i_mb_topleft_xy = h->mb.i_mb_stride*topleft_y + mb_x - 1;
817
            h->mb.i_mb_topleft_y = topleft_y;
818
            h->mb.i_mb_type_topleft = h->mb.type[h->mb.i_mb_topleft_xy];
819
            if( h->mb.slice_table[h->mb.i_mb_topleft_xy] == h->sh.i_first_mb )
820
            {
821
                h->mb.i_neighbour |= MB_TOPLEFT;
822

823
                if( !h->param.b_constrained_intra || IS_INTRA( h->mb.i_mb_type_topleft ) )
824
                    h->mb.i_neighbour_intra |= MB_TOPLEFT;
825
            }
826
        }
827

828
        if( mb_x < h->mb.i_mb_width - 1 && topright_y >= 0 )
829
        {
830
            h->mb.i_neighbour_frame |= MB_TOPRIGHT;
831
            h->mb.i_mb_topright_xy = h->mb.i_mb_stride*topright_y + mb_x + 1;
832
            h->mb.i_mb_topright_y = topright_y;
833
            h->mb.i_mb_type_topright = h->mb.type[h->mb.i_mb_topright_xy];
834
            if( h->mb.slice_table[h->mb.i_mb_topright_xy] == h->sh.i_first_mb )
835
            {
836
                h->mb.i_neighbour |= MB_TOPRIGHT;
837

838
                if( !h->param.b_constrained_intra || IS_INTRA( h->mb.i_mb_type_topright ) )
839
                    h->mb.i_neighbour_intra |= MB_TOPRIGHT;
840
            }
841
        }
842
    }
843
}
844

845
#define LTOP 0
846
#if HAVE_INTERLACED
847
#   define LBOT 1
848
#else
849
#   define LBOT 0
850
#endif
851

852
static void ALWAYS_INLINE x264_macroblock_cache_load( x264_t *h, int mb_x, int mb_y, int b_mbaff )
853
{
854
    x264_macroblock_cache_load_neighbours( h, mb_x, mb_y, b_mbaff );
855

856
    int *left = h->mb.i_mb_left_xy;
857
    int top  = h->mb.i_mb_top_xy;
858
    int top_y = h->mb.i_mb_top_y;
859
    int s8x8 = h->mb.i_b8_stride;
860
    int s4x4 = h->mb.i_b4_stride;
861
    int top_8x8 = (2*top_y+1) * s8x8 + 2*mb_x;
862
    int top_4x4 = (4*top_y+3) * s4x4 + 4*mb_x;
863
    int lists = (1 << h->sh.i_type) & 3;
864

865
    /* GCC pessimizes direct loads from heap-allocated arrays due to aliasing. */
866
    /* By only dereferencing them once, we avoid this issue. */
867
    int8_t (*i4x4)[8] = h->mb.intra4x4_pred_mode;
868
    uint8_t (*nnz)[48] = h->mb.non_zero_count;
869
    int16_t *cbp = h->mb.cbp;
870

871
    const x264_left_table_t *left_index_table = h->mb.left_index_table;
872

873
    h->mb.cache.deblock_strength = h->deblock_strength[mb_y&1][h->param.b_sliced_threads?h->mb.i_mb_xy:mb_x];
874

875
    /* load cache */
876
    if( h->mb.i_neighbour & MB_TOP )
877
    {
878
        h->mb.cache.i_cbp_top = cbp[top];
879
        /* load intra4x4 */
880
        CP32( &h->mb.cache.intra4x4_pred_mode[x264_scan8[0] - 8], &i4x4[top][0] );
881

882
        /* load non_zero_count */
883
        CP32( &h->mb.cache.non_zero_count[x264_scan8[ 0] - 8], &nnz[top][12] );
884
        CP32( &h->mb.cache.non_zero_count[x264_scan8[16] - 8], &nnz[top][16-4 + (16>>CHROMA_V_SHIFT)] );
885
        CP32( &h->mb.cache.non_zero_count[x264_scan8[32] - 8], &nnz[top][32-4 + (16>>CHROMA_V_SHIFT)] );
886

887
        /* Finish the prefetching */
888
        for( int l = 0; l < lists; l++ )
889
        {
890
            x264_prefetch( &h->mb.mv[l][top_4x4-1] );
891
            /* Top right being not in the same cacheline as top left will happen
892
             * once every 4 MBs, so one extra prefetch is worthwhile */
893
            x264_prefetch( &h->mb.mv[l][top_4x4+4] );
894
            x264_prefetch( &h->mb.ref[l][top_8x8-1] );
895
            x264_prefetch( &h->mb.mvd[l][top] );
896
        }
897
    }
898
    else
899
    {
900
        h->mb.cache.i_cbp_top = -1;
901

902
        /* load intra4x4 */
903
        M32( &h->mb.cache.intra4x4_pred_mode[x264_scan8[0] - 8] ) = 0xFFFFFFFFU;
904

905
        /* load non_zero_count */
906
        M32( &h->mb.cache.non_zero_count[x264_scan8[ 0] - 8] ) = 0x80808080U;
907
        M32( &h->mb.cache.non_zero_count[x264_scan8[16] - 8] ) = 0x80808080U;
908
        M32( &h->mb.cache.non_zero_count[x264_scan8[32] - 8] ) = 0x80808080U;
909
    }
910

911
    if( h->mb.i_neighbour & MB_LEFT )
912
    {
913
        int ltop = left[LTOP];
914
        int lbot = b_mbaff ? left[LBOT] : ltop;
915
        if( b_mbaff )
916
        {
917
            const int16_t top_luma = (cbp[ltop] >> (left_index_table->mv[0]&(~1))) & 2;
918
            const int16_t bot_luma = (cbp[lbot] >> (left_index_table->mv[2]&(~1))) & 2;
919
            h->mb.cache.i_cbp_left = (cbp[ltop] & 0xfff0) | (bot_luma<<2) | top_luma;
920
        }
921
        else
922
            h->mb.cache.i_cbp_left = cbp[ltop];
923

924
        /* load intra4x4 */
925
        h->mb.cache.intra4x4_pred_mode[x264_scan8[ 0] - 1] = i4x4[ltop][left_index_table->intra[0]];
926
        h->mb.cache.intra4x4_pred_mode[x264_scan8[ 2] - 1] = i4x4[ltop][left_index_table->intra[1]];
927
        h->mb.cache.intra4x4_pred_mode[x264_scan8[ 8] - 1] = i4x4[lbot][left_index_table->intra[2]];
928
        h->mb.cache.intra4x4_pred_mode[x264_scan8[10] - 1] = i4x4[lbot][left_index_table->intra[3]];
929

930
        /* load non_zero_count */
931
        h->mb.cache.non_zero_count[x264_scan8[ 0] - 1] = nnz[ltop][left_index_table->nnz[0]];
932
        h->mb.cache.non_zero_count[x264_scan8[ 2] - 1] = nnz[ltop][left_index_table->nnz[1]];
933
        h->mb.cache.non_zero_count[x264_scan8[ 8] - 1] = nnz[lbot][left_index_table->nnz[2]];
934
        h->mb.cache.non_zero_count[x264_scan8[10] - 1] = nnz[lbot][left_index_table->nnz[3]];
935

936
        if( CHROMA_FORMAT >= CHROMA_422 )
937
        {
938
            int offset = (4>>CHROMA_H_SHIFT) - 4;
939
            h->mb.cache.non_zero_count[x264_scan8[16+ 0] - 1] = nnz[ltop][left_index_table->nnz[0]+16+offset];
940
            h->mb.cache.non_zero_count[x264_scan8[16+ 2] - 1] = nnz[ltop][left_index_table->nnz[1]+16+offset];
941
            h->mb.cache.non_zero_count[x264_scan8[16+ 8] - 1] = nnz[lbot][left_index_table->nnz[2]+16+offset];
942
            h->mb.cache.non_zero_count[x264_scan8[16+10] - 1] = nnz[lbot][left_index_table->nnz[3]+16+offset];
943
            h->mb.cache.non_zero_count[x264_scan8[32+ 0] - 1] = nnz[ltop][left_index_table->nnz[0]+32+offset];
944
            h->mb.cache.non_zero_count[x264_scan8[32+ 2] - 1] = nnz[ltop][left_index_table->nnz[1]+32+offset];
945
            h->mb.cache.non_zero_count[x264_scan8[32+ 8] - 1] = nnz[lbot][left_index_table->nnz[2]+32+offset];
946
            h->mb.cache.non_zero_count[x264_scan8[32+10] - 1] = nnz[lbot][left_index_table->nnz[3]+32+offset];
947
        }
948
        else
949
        {
950
            h->mb.cache.non_zero_count[x264_scan8[16+ 0] - 1] = nnz[ltop][left_index_table->nnz_chroma[0]];
951
            h->mb.cache.non_zero_count[x264_scan8[16+ 2] - 1] = nnz[lbot][left_index_table->nnz_chroma[1]];
952
            h->mb.cache.non_zero_count[x264_scan8[32+ 0] - 1] = nnz[ltop][left_index_table->nnz_chroma[2]];
953
            h->mb.cache.non_zero_count[x264_scan8[32+ 2] - 1] = nnz[lbot][left_index_table->nnz_chroma[3]];
954
        }
955
    }
956
    else
957
    {
958
        h->mb.cache.i_cbp_left = -1;
959

960
        h->mb.cache.intra4x4_pred_mode[x264_scan8[ 0] - 1] =
961
        h->mb.cache.intra4x4_pred_mode[x264_scan8[ 2] - 1] =
962
        h->mb.cache.intra4x4_pred_mode[x264_scan8[ 8] - 1] =
963
        h->mb.cache.intra4x4_pred_mode[x264_scan8[10] - 1] = -1;
964

965
        /* load non_zero_count */
966
        h->mb.cache.non_zero_count[x264_scan8[ 0] - 1] =
967
        h->mb.cache.non_zero_count[x264_scan8[ 2] - 1] =
968
        h->mb.cache.non_zero_count[x264_scan8[ 8] - 1] =
969
        h->mb.cache.non_zero_count[x264_scan8[10] - 1] =
970
        h->mb.cache.non_zero_count[x264_scan8[16+ 0] - 1] =
971
        h->mb.cache.non_zero_count[x264_scan8[16+ 2] - 1] =
972
        h->mb.cache.non_zero_count[x264_scan8[32+ 0] - 1] =
973
        h->mb.cache.non_zero_count[x264_scan8[32+ 2] - 1] = 0x80;
974
        if( CHROMA_FORMAT >= CHROMA_422 )
975
        {
976
            h->mb.cache.non_zero_count[x264_scan8[16+ 8] - 1] =
977
            h->mb.cache.non_zero_count[x264_scan8[16+10] - 1] =
978
            h->mb.cache.non_zero_count[x264_scan8[32+ 8] - 1] =
979
            h->mb.cache.non_zero_count[x264_scan8[32+10] - 1] = 0x80;
980
        }
981
    }
982

983
    if( h->pps->b_transform_8x8_mode )
984
    {
985
        h->mb.cache.i_neighbour_transform_size =
986
            ( (h->mb.i_neighbour & MB_LEFT) && h->mb.mb_transform_size[left[0]] )
987
          + ( (h->mb.i_neighbour & MB_TOP) && h->mb.mb_transform_size[top]  );
988
    }
989

990
    if( b_mbaff )
991
    {
992
        h->mb.pic.i_fref[0] = h->i_ref[0] << MB_INTERLACED;
993
        h->mb.pic.i_fref[1] = h->i_ref[1] << MB_INTERLACED;
994
    }
995

996
    if( !b_mbaff )
997
    {
998
        x264_copy_column8( h->mb.pic.p_fdec[0]-1+ 4*FDEC_STRIDE, h->mb.pic.p_fdec[0]+15+ 4*FDEC_STRIDE );
999
        x264_copy_column8( h->mb.pic.p_fdec[0]-1+12*FDEC_STRIDE, h->mb.pic.p_fdec[0]+15+12*FDEC_STRIDE );
1000
        x264_macroblock_load_pic_pointers( h, mb_x, mb_y, 0, 0, 0 );
1001
        if( CHROMA444 )
1002
        {
1003
            x264_copy_column8( h->mb.pic.p_fdec[1]-1+ 4*FDEC_STRIDE, h->mb.pic.p_fdec[1]+15+ 4*FDEC_STRIDE );
1004
            x264_copy_column8( h->mb.pic.p_fdec[1]-1+12*FDEC_STRIDE, h->mb.pic.p_fdec[1]+15+12*FDEC_STRIDE );
1005
            x264_copy_column8( h->mb.pic.p_fdec[2]-1+ 4*FDEC_STRIDE, h->mb.pic.p_fdec[2]+15+ 4*FDEC_STRIDE );
1006
            x264_copy_column8( h->mb.pic.p_fdec[2]-1+12*FDEC_STRIDE, h->mb.pic.p_fdec[2]+15+12*FDEC_STRIDE );
1007
            x264_macroblock_load_pic_pointers( h, mb_x, mb_y, 1, 0, 0 );
1008
            x264_macroblock_load_pic_pointers( h, mb_x, mb_y, 2, 0, 0 );
1009
        }
1010
        else
1011
        {
1012
            x264_copy_column8( h->mb.pic.p_fdec[1]-1+ 4*FDEC_STRIDE, h->mb.pic.p_fdec[1]+ 7+ 4*FDEC_STRIDE );
1013
            x264_copy_column8( h->mb.pic.p_fdec[2]-1+ 4*FDEC_STRIDE, h->mb.pic.p_fdec[2]+ 7+ 4*FDEC_STRIDE );
1014
            if( CHROMA_FORMAT == CHROMA_422 )
1015
            {
1016
                x264_copy_column8( h->mb.pic.p_fdec[1]-1+12*FDEC_STRIDE, h->mb.pic.p_fdec[1]+ 7+12*FDEC_STRIDE );
1017
                x264_copy_column8( h->mb.pic.p_fdec[2]-1+12*FDEC_STRIDE, h->mb.pic.p_fdec[2]+ 7+12*FDEC_STRIDE );
1018
            }
1019
            x264_macroblock_load_pic_pointers( h, mb_x, mb_y, 1, 1, 0 );
1020
        }
1021
    }
1022
    else
1023
    {
1024
        x264_macroblock_load_pic_pointers( h, mb_x, mb_y, 0, 0, 1 );
1025
        if( CHROMA444 )
1026
        {
1027
            x264_macroblock_load_pic_pointers( h, mb_x, mb_y, 1, 0, 1 );
1028
            x264_macroblock_load_pic_pointers( h, mb_x, mb_y, 2, 0, 1 );
1029
        }
1030
        else
1031
            x264_macroblock_load_pic_pointers( h, mb_x, mb_y, 1, 1, 1 );
1032
    }
1033

1034
    if( h->fdec->integral )
1035
    {
1036
        int offset = 16 * (mb_x + mb_y * h->fdec->i_stride[0]);
1037
        for( int list = 0; list < 2; list++ )
1038
            for( int i = 0; i < h->mb.pic.i_fref[list]; i++ )
1039
                h->mb.pic.p_integral[list][i] = &h->fref[list][i]->integral[offset];
1040
    }
1041

1042
    x264_prefetch_fenc( h, h->fenc, mb_x, mb_y );
1043

1044
    /* load ref/mv/mvd */
1045
    for( int l = 0; l < lists; l++ )
1046
    {
1047
        int16_t (*mv)[2] = h->mb.mv[l];
1048
        int8_t *ref = h->mb.ref[l];
1049

1050
        int i8 = x264_scan8[0] - 1 - 1*8;
1051
        if( h->mb.i_neighbour & MB_TOPLEFT )
1052
        {
1053
            int ir = b_mbaff ? 2*(s8x8*h->mb.i_mb_topleft_y + mb_x-1)+1+s8x8 : top_8x8 - 1;
1054
            int iv = b_mbaff ? 4*(s4x4*h->mb.i_mb_topleft_y + mb_x-1)+3+3*s4x4 : top_4x4 - 1;
1055
            if( b_mbaff && h->mb.topleft_partition )
1056
            {
1057
                /* Take motion vector from the middle of macroblock instead of
1058
                 * the bottom right as usual. */
1059
                iv -= 2*s4x4;
1060
                ir -= s8x8;
1061
            }
1062
            h->mb.cache.ref[l][i8] = ref[ir];
1063
            CP32( h->mb.cache.mv[l][i8], mv[iv] );
1064
        }
1065
        else
1066
        {
1067
            h->mb.cache.ref[l][i8] = -2;
1068
            M32( h->mb.cache.mv[l][i8] ) = 0;
1069
        }
1070

1071
        i8 = x264_scan8[0] - 8;
1072
        if( h->mb.i_neighbour & MB_TOP )
1073
        {
1074
            h->mb.cache.ref[l][i8+0] =
1075
            h->mb.cache.ref[l][i8+1] = ref[top_8x8 + 0];
1076
            h->mb.cache.ref[l][i8+2] =
1077
            h->mb.cache.ref[l][i8+3] = ref[top_8x8 + 1];
1078
            CP128( h->mb.cache.mv[l][i8], mv[top_4x4] );
1079
        }
1080
        else
1081
        {
1082
            M128( h->mb.cache.mv[l][i8] ) = M128_ZERO;
1083
            M32( &h->mb.cache.ref[l][i8] ) = (uint8_t)(-2) * 0x01010101U;
1084
        }
1085

1086
        i8 = x264_scan8[0] + 4 - 1*8;
1087
        if( h->mb.i_neighbour & MB_TOPRIGHT )
1088
        {
1089
            int ir = b_mbaff ? 2*(s8x8*h->mb.i_mb_topright_y + (mb_x+1))+s8x8 : top_8x8 + 2;
1090
            int iv = b_mbaff ? 4*(s4x4*h->mb.i_mb_topright_y + (mb_x+1))+3*s4x4 : top_4x4 + 4;
1091
            h->mb.cache.ref[l][i8] = ref[ir];
1092
            CP32( h->mb.cache.mv[l][i8], mv[iv] );
1093
        }
1094
        else
1095
             h->mb.cache.ref[l][i8] = -2;
1096

1097
        i8 = x264_scan8[0] - 1;
1098
        if( h->mb.i_neighbour & MB_LEFT )
1099
        {
1100
            if( b_mbaff )
1101
            {
1102
                h->mb.cache.ref[l][i8+0*8] = ref[h->mb.left_b8[LTOP] + 1 + s8x8*left_index_table->ref[0]];
1103
                h->mb.cache.ref[l][i8+1*8] = ref[h->mb.left_b8[LTOP] + 1 + s8x8*left_index_table->ref[1]];
1104
                h->mb.cache.ref[l][i8+2*8] = ref[h->mb.left_b8[LBOT] + 1 + s8x8*left_index_table->ref[2]];
1105
                h->mb.cache.ref[l][i8+3*8] = ref[h->mb.left_b8[LBOT] + 1 + s8x8*left_index_table->ref[3]];
1106

1107
                CP32( h->mb.cache.mv[l][i8+0*8], mv[h->mb.left_b4[LTOP] + 3 + s4x4*left_index_table->mv[0]] );
1108
                CP32( h->mb.cache.mv[l][i8+1*8], mv[h->mb.left_b4[LTOP] + 3 + s4x4*left_index_table->mv[1]] );
1109
                CP32( h->mb.cache.mv[l][i8+2*8], mv[h->mb.left_b4[LBOT] + 3 + s4x4*left_index_table->mv[2]] );
1110
                CP32( h->mb.cache.mv[l][i8+3*8], mv[h->mb.left_b4[LBOT] + 3 + s4x4*left_index_table->mv[3]] );
1111
            }
1112
            else
1113
            {
1114
                const int ir = h->mb.i_b8_xy - 1;
1115
                const int iv = h->mb.i_b4_xy - 1;
1116
                h->mb.cache.ref[l][i8+0*8] =
1117
                h->mb.cache.ref[l][i8+1*8] = ref[ir + 0*s8x8];
1118
                h->mb.cache.ref[l][i8+2*8] =
1119
                h->mb.cache.ref[l][i8+3*8] = ref[ir + 1*s8x8];
1120

1121
                CP32( h->mb.cache.mv[l][i8+0*8], mv[iv + 0*s4x4] );
1122
                CP32( h->mb.cache.mv[l][i8+1*8], mv[iv + 1*s4x4] );
1123
                CP32( h->mb.cache.mv[l][i8+2*8], mv[iv + 2*s4x4] );
1124
                CP32( h->mb.cache.mv[l][i8+3*8], mv[iv + 3*s4x4] );
1125
            }
1126
        }
1127
        else
1128
        {
1129
            for( int i = 0; i < 4; i++ )
1130
            {
1131
                h->mb.cache.ref[l][i8+i*8] = -2;
1132
                M32( h->mb.cache.mv[l][i8+i*8] ) = 0;
1133
            }
1134
        }
1135

1136
        /* Extra logic for top right mv in mbaff.
1137
         * . . . d  . . a .
1138
         * . . . e  . . . .
1139
         * . . . f  b . c .
1140
         * . . . .  . . . .
1141
         *
1142
         * If the top right of the 4x4 partitions labeled a, b and c in the
1143
         * above diagram do not exist, but the entries d, e and f exist (in
1144
         * the macroblock to the left) then use those instead.
1145
         */
1146
        if( b_mbaff && (h->mb.i_neighbour & MB_LEFT) )
1147
        {
1148
            if( MB_INTERLACED && !h->mb.field[h->mb.i_mb_xy-1] )
1149
            {
1150
                h->mb.cache.topright_ref[l][0] = ref[h->mb.left_b8[0] + 1 + s8x8*0];
1151
                h->mb.cache.topright_ref[l][1] = ref[h->mb.left_b8[0] + 1 + s8x8*1];
1152
                h->mb.cache.topright_ref[l][2] = ref[h->mb.left_b8[1] + 1 + s8x8*0];
1153
                CP32( h->mb.cache.topright_mv[l][0], mv[h->mb.left_b4[0] + 3 + s4x4*(left_index_table->mv[0]+1)] );
1154
                CP32( h->mb.cache.topright_mv[l][1], mv[h->mb.left_b4[0] + 3 + s4x4*(left_index_table->mv[1]+1)] );
1155
                CP32( h->mb.cache.topright_mv[l][2], mv[h->mb.left_b4[1] + 3 + s4x4*(left_index_table->mv[2]+1)] );
1156
            }
1157
            else if( !MB_INTERLACED && h->mb.field[h->mb.i_mb_xy-1] )
1158
            {
1159
                // Looking at the bottom field so always take the bottom macroblock of the pair.
1160
                h->mb.cache.topright_ref[l][0] = ref[h->mb.left_b8[0] + 1 + s8x8*2 + s8x8*left_index_table->ref[0]];
1161
                h->mb.cache.topright_ref[l][1] = ref[h->mb.left_b8[0] + 1 + s8x8*2 + s8x8*left_index_table->ref[1]];
1162
                h->mb.cache.topright_ref[l][2] = ref[h->mb.left_b8[0] + 1 + s8x8*2 + s8x8*left_index_table->ref[2]];
1163
                CP32( h->mb.cache.topright_mv[l][0], mv[h->mb.left_b4[0] + 3 + s4x4*4 + s4x4*left_index_table->mv[0]] );
1164
                CP32( h->mb.cache.topright_mv[l][1], mv[h->mb.left_b4[0] + 3 + s4x4*4 + s4x4*left_index_table->mv[1]] );
1165
                CP32( h->mb.cache.topright_mv[l][2], mv[h->mb.left_b4[0] + 3 + s4x4*4 + s4x4*left_index_table->mv[2]] );
1166
            }
1167
        }
1168

1169
        if( h->param.b_cabac )
1170
        {
1171
            uint8_t (*mvd)[8][2] = h->mb.mvd[l];
1172
            if( h->mb.i_neighbour & MB_TOP )
1173
                CP64( h->mb.cache.mvd[l][x264_scan8[0] - 8], mvd[top][0] );
1174
            else
1175
                M64( h->mb.cache.mvd[l][x264_scan8[0] - 8] ) = 0;
1176

1177
            if( h->mb.i_neighbour & MB_LEFT && (!b_mbaff || h->mb.cache.ref[l][x264_scan8[0]-1] >= 0) )
1178
            {
1179
                CP16( h->mb.cache.mvd[l][x264_scan8[0 ] - 1], mvd[left[LTOP]][left_index_table->intra[0]] );
1180
                CP16( h->mb.cache.mvd[l][x264_scan8[2 ] - 1], mvd[left[LTOP]][left_index_table->intra[1]] );
1181
            }
1182
            else
1183
            {
1184
                M16( h->mb.cache.mvd[l][x264_scan8[0]-1+0*8] ) = 0;
1185
                M16( h->mb.cache.mvd[l][x264_scan8[0]-1+1*8] ) = 0;
1186
            }
1187
            if( h->mb.i_neighbour & MB_LEFT && (!b_mbaff || h->mb.cache.ref[l][x264_scan8[0]-1+2*8] >=0) )
1188
            {
1189
                CP16( h->mb.cache.mvd[l][x264_scan8[8 ] - 1], mvd[left[LBOT]][left_index_table->intra[2]] );
1190
                CP16( h->mb.cache.mvd[l][x264_scan8[10] - 1], mvd[left[LBOT]][left_index_table->intra[3]] );
1191
            }
1192
            else
1193
            {
1194
                M16( h->mb.cache.mvd[l][x264_scan8[0]-1+2*8] ) = 0;
1195
                M16( h->mb.cache.mvd[l][x264_scan8[0]-1+3*8] ) = 0;
1196
            }
1197
        }
1198

1199
        /* If motion vectors are cached from frame macroblocks but this
1200
         * macroblock is a field macroblock then the motion vector must be
1201
         * halved. Similarly, motion vectors from field macroblocks are doubled. */
1202
        if( b_mbaff )
1203
        {
1204
#define MAP_MVS\
1205
                if( FIELD_DIFFERENT(h->mb.i_mb_topleft_xy) )\
1206
                    MAP_F2F(mv, ref, x264_scan8[0] - 1 - 1*8)\
1207
                if( FIELD_DIFFERENT(top) )\
1208
                {\
1209
                    MAP_F2F(mv, ref, x264_scan8[0] + 0 - 1*8)\
1210
                    MAP_F2F(mv, ref, x264_scan8[0] + 1 - 1*8)\
1211
                    MAP_F2F(mv, ref, x264_scan8[0] + 2 - 1*8)\
1212
                    MAP_F2F(mv, ref, x264_scan8[0] + 3 - 1*8)\
1213
                }\
1214
                if( FIELD_DIFFERENT(h->mb.i_mb_topright_xy) )\
1215
                    MAP_F2F(mv, ref, x264_scan8[0] + 4 - 1*8)\
1216
                if( FIELD_DIFFERENT(left[0]) )\
1217
                {\
1218
                    MAP_F2F(mv, ref, x264_scan8[0] - 1 + 0*8)\
1219
                    MAP_F2F(mv, ref, x264_scan8[0] - 1 + 1*8)\
1220
                    MAP_F2F(mv, ref, x264_scan8[0] - 1 + 2*8)\
1221
                    MAP_F2F(mv, ref, x264_scan8[0] - 1 + 3*8)\
1222
                    MAP_F2F(topright_mv, topright_ref, 0)\
1223
                    MAP_F2F(topright_mv, topright_ref, 1)\
1224
                    MAP_F2F(topright_mv, topright_ref, 2)\
1225
                }
1226

1227
            if( MB_INTERLACED )
1228
            {
1229
#define FIELD_DIFFERENT(macroblock) (macroblock >= 0 && !h->mb.field[macroblock])
1230
#define MAP_F2F(varmv, varref, index)\
1231
                if( h->mb.cache.varref[l][index] >= 0 )\
1232
                {\
1233
                    h->mb.cache.varref[l][index] <<= 1;\
1234
                    h->mb.cache.varmv[l][index][1] /= 2;\
1235
                    h->mb.cache.mvd[l][index][1] >>= 1;\
1236
                }
1237
                MAP_MVS
1238
#undef MAP_F2F
1239
#undef FIELD_DIFFERENT
1240
            }
1241
            else
1242
            {
1243
#define FIELD_DIFFERENT(macroblock) (macroblock >= 0 && h->mb.field[macroblock])
1244
#define MAP_F2F(varmv, varref, index)\
1245
                if( h->mb.cache.varref[l][index] >= 0 )\
1246
                {\
1247
                    h->mb.cache.varref[l][index] >>= 1;\
1248
                    h->mb.cache.varmv[l][index][1] <<= 1;\
1249
                    h->mb.cache.mvd[l][index][1] <<= 1;\
1250
                }
1251
                MAP_MVS
1252
#undef MAP_F2F
1253
#undef FIELD_DIFFERENT
1254
            }
1255
        }
1256
    }
1257

1258
    if( b_mbaff && mb_x == 0 && !(mb_y&1) )
1259
    {
1260
        if( h->mb.i_mb_top_xy >= h->sh.i_first_mb )
1261
            h->mb.field_decoding_flag = h->mb.field[h->mb.i_mb_top_xy];
1262
        else
1263
            h->mb.field_decoding_flag = 0;
1264
    }
1265

1266
    /* Check whether skip here would cause decoder to predict interlace mode incorrectly.
1267
     * FIXME: It might be better to change the interlace type rather than forcing a skip to be non-skip. */
1268
    h->mb.b_allow_skip = 1;
1269
    if( b_mbaff )
1270
    {
1271
        if( MB_INTERLACED != h->mb.field_decoding_flag &&
1272
            (mb_y&1) && IS_SKIP(h->mb.type[h->mb.i_mb_xy - h->mb.i_mb_stride]) )
1273
            h->mb.b_allow_skip = 0;
1274
    }
1275

1276
    if( h->param.b_cabac )
1277
    {
1278
        if( b_mbaff )
1279
        {
1280
            int left_xy, top_xy;
1281
            /* Neighbours here are calculated based on field_decoding_flag */
1282
            int mb_xy = mb_x + (mb_y&~1)*h->mb.i_mb_stride;
1283
            left_xy = mb_xy - 1;
1284
            if( (mb_y&1) && mb_x > 0 && h->mb.field_decoding_flag == h->mb.field[left_xy] )
1285
                left_xy += h->mb.i_mb_stride;
1286
            if( h->mb.field_decoding_flag )
1287
            {
1288
                top_xy = mb_xy - h->mb.i_mb_stride;
1289
                if( !(mb_y&1) && top_xy >= 0 && h->mb.slice_table[top_xy] == h->sh.i_first_mb && h->mb.field[top_xy] )
1290
                    top_xy -= h->mb.i_mb_stride;
1291
            }
1292
            else
1293
                top_xy = mb_x + (mb_y-1)*h->mb.i_mb_stride;
1294

1295
            h->mb.cache.i_neighbour_skip =   (mb_x >  0 && h->mb.slice_table[left_xy] == h->sh.i_first_mb && !IS_SKIP( h->mb.type[left_xy] ))
1296
                                         + (top_xy >= 0 && h->mb.slice_table[top_xy]  == h->sh.i_first_mb && !IS_SKIP( h->mb.type[top_xy] ));
1297
        }
1298
        else
1299
        {
1300
            h->mb.cache.i_neighbour_skip = ((h->mb.i_neighbour & MB_LEFT) && !IS_SKIP( h->mb.i_mb_type_left[0] ))
1301
                                         + ((h->mb.i_neighbour & MB_TOP)  && !IS_SKIP( h->mb.i_mb_type_top ));
1302
        }
1303
    }
1304

1305
    /* load skip */
1306
    if( h->sh.i_type == SLICE_TYPE_B )
1307
    {
1308
        h->mb.bipred_weight = h->mb.bipred_weight_buf[MB_INTERLACED][MB_INTERLACED&(mb_y&1)];
1309
        h->mb.dist_scale_factor = h->mb.dist_scale_factor_buf[MB_INTERLACED][MB_INTERLACED&(mb_y&1)];
1310
        if( h->param.b_cabac )
1311
        {
1312
            uint8_t skipbp;
1313
            x264_macroblock_cache_skip( h, 0, 0, 4, 4, 0 );
1314
            if( b_mbaff )
1315
            {
1316
                skipbp = (h->mb.i_neighbour & MB_LEFT) ? h->mb.skipbp[left[LTOP]] : 0;
1317
                h->mb.cache.skip[x264_scan8[0] - 1] = (skipbp >> (1+(left_index_table->mv[0]&~1))) & 1;
1318
                skipbp = (h->mb.i_neighbour & MB_LEFT) ? h->mb.skipbp[left[LBOT]] : 0;
1319
                h->mb.cache.skip[x264_scan8[8] - 1] = (skipbp >> (1+(left_index_table->mv[2]&~1))) & 1;
1320
            }
1321
            else
1322
            {
1323
                skipbp = (h->mb.i_neighbour & MB_LEFT) ? h->mb.skipbp[left[0]] : 0;
1324
                h->mb.cache.skip[x264_scan8[0] - 1] = skipbp & 0x2;
1325
                h->mb.cache.skip[x264_scan8[8] - 1] = skipbp & 0x8;
1326
            }
1327
            skipbp = (h->mb.i_neighbour & MB_TOP) ? h->mb.skipbp[top] : 0;
1328
            h->mb.cache.skip[x264_scan8[0] - 8] = skipbp & 0x4;
1329
            h->mb.cache.skip[x264_scan8[4] - 8] = skipbp & 0x8;
1330
        }
1331
    }
1332

1333
    if( h->sh.i_type == SLICE_TYPE_P )
1334
        x264_mb_predict_mv_pskip( h, h->mb.cache.pskip_mv );
1335

1336
    h->mb.i_neighbour4[0] =
1337
    h->mb.i_neighbour8[0] = (h->mb.i_neighbour_intra & (MB_TOP|MB_LEFT|MB_TOPLEFT))
1338
                            | ((h->mb.i_neighbour_intra & MB_TOP) ? MB_TOPRIGHT : 0);
1339
    h->mb.i_neighbour4[4] =
1340
    h->mb.i_neighbour4[1] = MB_LEFT | ((h->mb.i_neighbour_intra & MB_TOP) ? (MB_TOP|MB_TOPLEFT|MB_TOPRIGHT) : 0);
1341
    h->mb.i_neighbour4[2] =
1342
    h->mb.i_neighbour4[8] =
1343
    h->mb.i_neighbour4[10] =
1344
    h->mb.i_neighbour8[2] = MB_TOP|MB_TOPRIGHT | ((h->mb.i_neighbour_intra & MB_LEFT) ? (MB_LEFT|MB_TOPLEFT) : 0);
1345
    h->mb.i_neighbour4[5] =
1346
    h->mb.i_neighbour8[1] = MB_LEFT | (h->mb.i_neighbour_intra & MB_TOPRIGHT)
1347
                            | ((h->mb.i_neighbour_intra & MB_TOP) ? MB_TOP|MB_TOPLEFT : 0);
1348
}
1349

1350
void x264_macroblock_cache_load_progressive( x264_t *h, int mb_x, int mb_y )
1351
{
1352
    x264_macroblock_cache_load( h, mb_x, mb_y, 0 );
1353
}
1354

1355
void x264_macroblock_cache_load_interlaced( x264_t *h, int mb_x, int mb_y )
1356
{
1357
    x264_macroblock_cache_load( h, mb_x, mb_y, 1 );
1358
}
1359

1360
static void x264_macroblock_deblock_strength_mbaff( x264_t *h, uint8_t (*bs)[8][4] )
1361
{
1362
    if( (h->mb.i_neighbour & MB_LEFT) && h->mb.field[h->mb.i_mb_left_xy[0]] != MB_INTERLACED )
1363
    {
1364
        static const uint8_t offset[2][2][8] =
1365
        {   {   { 0, 0, 0, 0, 1, 1, 1, 1 },
1366
                { 2, 2, 2, 2, 3, 3, 3, 3 }, },
1367
            {   { 0, 1, 2, 3, 0, 1, 2, 3 },
1368
                { 0, 1, 2, 3, 0, 1, 2, 3 }, }
1369
        };
1370
        ALIGNED_ARRAY_8( uint8_t, tmpbs, [8] );
1371

1372
        const uint8_t *off = offset[MB_INTERLACED][h->mb.i_mb_y&1];
1373
        uint8_t (*nnz)[48] = h->mb.non_zero_count;
1374

1375
        for( int i = 0; i < 8; i++ )
1376
        {
1377
            int left = h->mb.i_mb_left_xy[MB_INTERLACED ? i>>2 : i&1];
1378
            int nnz_this = h->mb.cache.non_zero_count[x264_scan8[0]+8*(i>>1)];
1379
            int nnz_left = nnz[left][3 + 4*off[i]];
1380
            if( !h->param.b_cabac && h->pps->b_transform_8x8_mode )
1381
            {
1382
                int j = off[i]&~1;
1383
                if( h->mb.mb_transform_size[left] )
1384
                    nnz_left = !!(M16( &nnz[left][2+4*j] ) | M16( &nnz[left][2+4*(1+j)] ));
1385
            }
1386
            tmpbs[i] = (nnz_left || nnz_this) ? 2 : 1;
1387
        }
1388

1389
        if( MB_INTERLACED )
1390
        {
1391
            CP32( bs[0][0], &tmpbs[0] );
1392
            CP32( bs[0][4], &tmpbs[4] );
1393
        }
1394
        else
1395
        {
1396
            for( int i = 0; i < 4; i++ ) bs[0][0][i] = tmpbs[2*i];
1397
            for( int i = 0; i < 4; i++ ) bs[0][4][i] = tmpbs[1+2*i];
1398
        }
1399
    }
1400

1401
    if( (h->mb.i_neighbour & MB_TOP) && MB_INTERLACED != h->mb.field[h->mb.i_mb_top_xy] )
1402
    {
1403
        if( !(h->mb.i_mb_y&1) && !MB_INTERLACED )
1404
        {
1405
            /* Need to filter both fields (even for frame macroblocks).
1406
             * Filter top two rows using the top macroblock of the above
1407
             * pair and then the bottom one. */
1408
            int mbn_xy = h->mb.i_mb_xy - 2 * h->mb.i_mb_stride;
1409
            uint8_t *nnz_cur = &h->mb.cache.non_zero_count[x264_scan8[0]];
1410

1411
            for( int j = 0; j < 2; j++, mbn_xy += h->mb.i_mb_stride )
1412
            {
1413
                uint8_t (*nnz)[48] = h->mb.non_zero_count;
1414

1415
                ALIGNED_4( uint8_t nnz_top[4] );
1416
                CP32( nnz_top, &nnz[mbn_xy][3*4] );
1417

1418
                if( !h->param.b_cabac && h->pps->b_transform_8x8_mode && h->mb.mb_transform_size[mbn_xy] )
1419
                {
1420
                    nnz_top[0] = nnz_top[1] = M16( &nnz[mbn_xy][ 8] ) || M16( &nnz[mbn_xy][12] );
1421
                    nnz_top[2] = nnz_top[3] = M16( &nnz[mbn_xy][10] ) || M16( &nnz[mbn_xy][14] );
1422
                }
1423

1424
                for( int i = 0; i < 4; i++ )
1425
                    bs[1][4*j][i] = (nnz_cur[i] || nnz_top[i]) ? 2 : 1;
1426
            }
1427
        }
1428
        else
1429
            for( int i = 0; i < 4; i++ )
1430
                bs[1][0][i] = X264_MAX( bs[1][0][i], 1 );
1431
    }
1432
}
1433

1434
void x264_macroblock_deblock_strength( x264_t *h )
1435
{
1436
    uint8_t (*bs)[8][4] = h->mb.cache.deblock_strength;
1437
    if( IS_INTRA( h->mb.i_type ) )
1438
    {
1439
        M32( bs[0][1] ) = 0x03030303;
1440
        M64( bs[0][2] ) = 0x0303030303030303ULL;
1441
        M32( bs[1][1] ) = 0x03030303;
1442
        M64( bs[1][2] ) = 0x0303030303030303ULL;
1443
        return;
1444
    }
1445

1446
    /* Early termination: in this case, nnz guarantees all edges use strength 2.*/
1447
    if( h->mb.b_transform_8x8 && !CHROMA444 )
1448
    {
1449
        int cbp_mask = 0xf >> CHROMA_V_SHIFT;
1450
        if( (h->mb.i_cbp_luma&cbp_mask) == cbp_mask )
1451
        {
1452
            M32( bs[0][0] ) = 0x02020202;
1453
            M32( bs[0][2] ) = 0x02020202;
1454
            M32( bs[0][4] ) = 0x02020202;
1455
            M64( bs[1][0] ) = 0x0202020202020202ULL; /* [1][1] and [1][3] has to be set for 4:2:2 */
1456
            M64( bs[1][2] ) = 0x0202020202020202ULL;
1457
            M32( bs[1][4] ) = 0x02020202;
1458
            return;
1459
        }
1460
    }
1461

1462
    int neighbour_changed = 0;
1463
    if( h->sh.i_disable_deblocking_filter_idc != 2 )
1464
    {
1465
        neighbour_changed = h->mb.i_neighbour_frame&~h->mb.i_neighbour;
1466
        h->mb.i_neighbour = h->mb.i_neighbour_frame;
1467
    }
1468

1469
    /* MBAFF deblock uses different left neighbors from encoding */
1470
    if( SLICE_MBAFF && (h->mb.i_neighbour & MB_LEFT) && (h->mb.field[h->mb.i_mb_xy - 1] != MB_INTERLACED) )
1471
    {
1472
        h->mb.i_mb_left_xy[1] =
1473
        h->mb.i_mb_left_xy[0] = h->mb.i_mb_xy - 1;
1474
        if( h->mb.i_mb_y&1 )
1475
            h->mb.i_mb_left_xy[0] -= h->mb.i_mb_stride;
1476
        else
1477
            h->mb.i_mb_left_xy[1] += h->mb.i_mb_stride;
1478
    }
1479

1480
    /* If we have multiple slices and we're deblocking on slice edges, we
1481
     * have to reload neighbour data. */
1482
    if( neighbour_changed )
1483
    {
1484
        int top_y = h->mb.i_mb_top_y;
1485
        int top_8x8 = (2*top_y+1) * h->mb.i_b8_stride + 2*h->mb.i_mb_x;
1486
        int top_4x4 = (4*top_y+3) * h->mb.i_b4_stride + 4*h->mb.i_mb_x;
1487
        int s8x8 = h->mb.i_b8_stride;
1488
        int s4x4 = h->mb.i_b4_stride;
1489

1490
        uint8_t (*nnz)[48] = h->mb.non_zero_count;
1491
        const x264_left_table_t *left_index_table = SLICE_MBAFF ? h->mb.left_index_table : &left_indices[3];
1492

1493
        if( neighbour_changed & MB_TOP )
1494
            CP32( &h->mb.cache.non_zero_count[x264_scan8[0] - 8], &nnz[h->mb.i_mb_top_xy][12] );
1495

1496
        if( neighbour_changed & MB_LEFT )
1497
        {
1498
            int *left = h->mb.i_mb_left_xy;
1499
            h->mb.cache.non_zero_count[x264_scan8[0 ] - 1] = nnz[left[0]][left_index_table->nnz[0]];
1500
            h->mb.cache.non_zero_count[x264_scan8[2 ] - 1] = nnz[left[0]][left_index_table->nnz[1]];
1501
            h->mb.cache.non_zero_count[x264_scan8[8 ] - 1] = nnz[left[1]][left_index_table->nnz[2]];
1502
            h->mb.cache.non_zero_count[x264_scan8[10] - 1] = nnz[left[1]][left_index_table->nnz[3]];
1503
        }
1504

1505
        for( int l = 0; l <= (h->sh.i_type == SLICE_TYPE_B); l++ )
1506
        {
1507
            int16_t (*mv)[2] = h->mb.mv[l];
1508
            int8_t *ref = h->mb.ref[l];
1509

1510
            int i8 = x264_scan8[0] - 8;
1511
            if( neighbour_changed & MB_TOP )
1512
            {
1513
                h->mb.cache.ref[l][i8+0] =
1514
                h->mb.cache.ref[l][i8+1] = ref[top_8x8 + 0];
1515
                h->mb.cache.ref[l][i8+2] =
1516
                h->mb.cache.ref[l][i8+3] = ref[top_8x8 + 1];
1517
                CP128( h->mb.cache.mv[l][i8], mv[top_4x4] );
1518
            }
1519

1520
            i8 = x264_scan8[0] - 1;
1521
            if( neighbour_changed & MB_LEFT )
1522
            {
1523
                h->mb.cache.ref[l][i8+0*8] =
1524
                h->mb.cache.ref[l][i8+1*8] = ref[h->mb.left_b8[0] + 1 + s8x8*left_index_table->ref[0]];
1525
                h->mb.cache.ref[l][i8+2*8] =
1526
                h->mb.cache.ref[l][i8+3*8] = ref[h->mb.left_b8[1] + 1 + s8x8*left_index_table->ref[2]];
1527

1528
                CP32( h->mb.cache.mv[l][i8+0*8], mv[h->mb.left_b4[0] + 3 + s4x4*left_index_table->mv[0]] );
1529
                CP32( h->mb.cache.mv[l][i8+1*8], mv[h->mb.left_b4[0] + 3 + s4x4*left_index_table->mv[1]] );
1530
                CP32( h->mb.cache.mv[l][i8+2*8], mv[h->mb.left_b4[1] + 3 + s4x4*left_index_table->mv[2]] );
1531
                CP32( h->mb.cache.mv[l][i8+3*8], mv[h->mb.left_b4[1] + 3 + s4x4*left_index_table->mv[3]] );
1532
            }
1533
        }
1534
    }
1535

1536
    if( h->param.analyse.i_weighted_pred == X264_WEIGHTP_SMART && h->sh.i_type == SLICE_TYPE_P )
1537
    {
1538
        /* Handle reference frame duplicates */
1539
        int i8 = x264_scan8[0] - 8;
1540
        h->mb.cache.ref[0][i8+0] =
1541
        h->mb.cache.ref[0][i8+1] = deblock_ref_table(h->mb.cache.ref[0][i8+0]);
1542
        h->mb.cache.ref[0][i8+2] =
1543
        h->mb.cache.ref[0][i8+3] = deblock_ref_table(h->mb.cache.ref[0][i8+2]);
1544

1545
        i8 = x264_scan8[0] - 1;
1546
        h->mb.cache.ref[0][i8+0*8] =
1547
        h->mb.cache.ref[0][i8+1*8] = deblock_ref_table(h->mb.cache.ref[0][i8+0*8]);
1548
        h->mb.cache.ref[0][i8+2*8] =
1549
        h->mb.cache.ref[0][i8+3*8] = deblock_ref_table(h->mb.cache.ref[0][i8+2*8]);
1550

1551
        int ref0 = deblock_ref_table(h->mb.cache.ref[0][x264_scan8[ 0]]);
1552
        int ref1 = deblock_ref_table(h->mb.cache.ref[0][x264_scan8[ 4]]);
1553
        int ref2 = deblock_ref_table(h->mb.cache.ref[0][x264_scan8[ 8]]);
1554
        int ref3 = deblock_ref_table(h->mb.cache.ref[0][x264_scan8[12]]);
1555
        uint32_t reftop = pack16to32( (uint8_t)ref0, (uint8_t)ref1 ) * 0x0101;
1556
        uint32_t refbot = pack16to32( (uint8_t)ref2, (uint8_t)ref3 ) * 0x0101;
1557

1558
        M32( &h->mb.cache.ref[0][x264_scan8[0]+8*0] ) = reftop;
1559
        M32( &h->mb.cache.ref[0][x264_scan8[0]+8*1] ) = reftop;
1560
        M32( &h->mb.cache.ref[0][x264_scan8[0]+8*2] ) = refbot;
1561
        M32( &h->mb.cache.ref[0][x264_scan8[0]+8*3] ) = refbot;
1562
    }
1563

1564
    /* Munge NNZ for cavlc + 8x8dct */
1565
    if( !h->param.b_cabac && h->pps->b_transform_8x8_mode )
1566
    {
1567
        uint8_t (*nnz)[48] = h->mb.non_zero_count;
1568
        int top = h->mb.i_mb_top_xy;
1569
        int *left = h->mb.i_mb_left_xy;
1570

1571
        if( (h->mb.i_neighbour & MB_TOP) && h->mb.mb_transform_size[top] )
1572
        {
1573
            int i8 = x264_scan8[0] - 8;
1574
            int nnz_top0 = M16( &nnz[top][8] ) | M16( &nnz[top][12] );
1575
            int nnz_top1 = M16( &nnz[top][10] ) | M16( &nnz[top][14] );
1576
            M16( &h->mb.cache.non_zero_count[i8+0] ) = nnz_top0 ? 0x0101 : 0;
1577
            M16( &h->mb.cache.non_zero_count[i8+2] ) = nnz_top1 ? 0x0101 : 0;
1578
        }
1579

1580
        if( h->mb.i_neighbour & MB_LEFT )
1581
        {
1582
            int i8 = x264_scan8[0] - 1;
1583
            if( h->mb.mb_transform_size[left[0]] )
1584
            {
1585
                int nnz_left0 = M16( &nnz[left[0]][2] ) | M16( &nnz[left[0]][6] );
1586
                h->mb.cache.non_zero_count[i8+8*0] = !!nnz_left0;
1587
                h->mb.cache.non_zero_count[i8+8*1] = !!nnz_left0;
1588
            }
1589
            if( h->mb.mb_transform_size[left[1]] )
1590
            {
1591
                int nnz_left1 = M16( &nnz[left[1]][10] ) | M16( &nnz[left[1]][14] );
1592
                h->mb.cache.non_zero_count[i8+8*2] = !!nnz_left1;
1593
                h->mb.cache.non_zero_count[i8+8*3] = !!nnz_left1;
1594
            }
1595
        }
1596

1597
        if( h->mb.b_transform_8x8 )
1598
        {
1599
            int nnz0 = M16( &h->mb.cache.non_zero_count[x264_scan8[ 0]] ) | M16( &h->mb.cache.non_zero_count[x264_scan8[ 2]] );
1600
            int nnz1 = M16( &h->mb.cache.non_zero_count[x264_scan8[ 4]] ) | M16( &h->mb.cache.non_zero_count[x264_scan8[ 6]] );
1601
            int nnz2 = M16( &h->mb.cache.non_zero_count[x264_scan8[ 8]] ) | M16( &h->mb.cache.non_zero_count[x264_scan8[10]] );
1602
            int nnz3 = M16( &h->mb.cache.non_zero_count[x264_scan8[12]] ) | M16( &h->mb.cache.non_zero_count[x264_scan8[14]] );
1603
            uint32_t nnztop = pack16to32( !!nnz0, !!nnz1 ) * 0x0101;
1604
            uint32_t nnzbot = pack16to32( !!nnz2, !!nnz3 ) * 0x0101;
1605

1606
            M32( &h->mb.cache.non_zero_count[x264_scan8[0]+8*0] ) = nnztop;
1607
            M32( &h->mb.cache.non_zero_count[x264_scan8[0]+8*1] ) = nnztop;
1608
            M32( &h->mb.cache.non_zero_count[x264_scan8[0]+8*2] ) = nnzbot;
1609
            M32( &h->mb.cache.non_zero_count[x264_scan8[0]+8*3] ) = nnzbot;
1610
        }
1611
    }
1612

1613
    h->loopf.deblock_strength( h->mb.cache.non_zero_count, h->mb.cache.ref, h->mb.cache.mv,
1614
                               bs, 4 >> MB_INTERLACED, h->sh.i_type == SLICE_TYPE_B );
1615

1616
    if( SLICE_MBAFF )
1617
        x264_macroblock_deblock_strength_mbaff( h, bs );
1618
}
1619

1620
static void ALWAYS_INLINE x264_macroblock_store_pic( x264_t *h, int mb_x, int mb_y, int i, int b_chroma, int b_mbaff )
1621
{
1622
    int height = b_chroma ? 16>>CHROMA_V_SHIFT : 16;
1623
    int i_stride = h->fdec->i_stride[i];
1624
    int i_stride2 = i_stride << (b_mbaff && MB_INTERLACED);
1625
    int i_pix_offset = (b_mbaff && MB_INTERLACED)
1626
                     ? 16 * mb_x + height * (mb_y&~1) * i_stride + (mb_y&1) * i_stride
1627
                     : 16 * mb_x + height * mb_y * i_stride;
1628
    if( b_chroma )
1629
        h->mc.store_interleave_chroma( &h->fdec->plane[1][i_pix_offset], i_stride2, h->mb.pic.p_fdec[1], h->mb.pic.p_fdec[2], height );
1630
    else
1631
        h->mc.copy[PIXEL_16x16]( &h->fdec->plane[i][i_pix_offset], i_stride2, h->mb.pic.p_fdec[i], FDEC_STRIDE, 16 );
1632
}
1633

1634
static void ALWAYS_INLINE x264_macroblock_backup_intra( x264_t *h, int mb_x, int mb_y, int b_mbaff )
1635
{
1636
    /* In MBAFF we store the last two rows in intra_border_backup[0] and [1].
1637
     * For progressive mbs this is the bottom two rows, and for interlaced the
1638
     * bottom row of each field. We also store samples needed for the next
1639
     * mbpair in intra_border_backup[2]. */
1640
    int backup_dst = !b_mbaff ? (mb_y&1) : (mb_y&1) ? 1 : MB_INTERLACED ? 0 : 2;
1641
    memcpy( &h->intra_border_backup[backup_dst][0][mb_x*16  ], h->mb.pic.p_fdec[0]+FDEC_STRIDE*15, 16*sizeof(pixel) );
1642
    if( CHROMA444 )
1643
    {
1644
        memcpy( &h->intra_border_backup[backup_dst][1][mb_x*16  ], h->mb.pic.p_fdec[1]+FDEC_STRIDE*15, 16*sizeof(pixel) );
1645
        memcpy( &h->intra_border_backup[backup_dst][2][mb_x*16  ], h->mb.pic.p_fdec[2]+FDEC_STRIDE*15, 16*sizeof(pixel) );
1646
    }
1647
    else
1648
    {
1649
        int backup_src = (15>>CHROMA_V_SHIFT) * FDEC_STRIDE;
1650
        memcpy( &h->intra_border_backup[backup_dst][1][mb_x*16  ], h->mb.pic.p_fdec[1]+backup_src, 8*sizeof(pixel) );
1651
        memcpy( &h->intra_border_backup[backup_dst][1][mb_x*16+8], h->mb.pic.p_fdec[2]+backup_src, 8*sizeof(pixel) );
1652
    }
1653
    if( b_mbaff )
1654
    {
1655
        if( mb_y&1 )
1656
        {
1657
            int backup_src = (MB_INTERLACED ? 7 : 14) * FDEC_STRIDE;
1658
            backup_dst = MB_INTERLACED ? 2 : 0;
1659
            memcpy( &h->intra_border_backup[backup_dst][0][mb_x*16  ], h->mb.pic.p_fdec[0]+backup_src, 16*sizeof(pixel) );
1660
            if( CHROMA444 )
1661
            {
1662
                memcpy( &h->intra_border_backup[backup_dst][1][mb_x*16  ], h->mb.pic.p_fdec[1]+backup_src, 16*sizeof(pixel) );
1663
                memcpy( &h->intra_border_backup[backup_dst][2][mb_x*16  ], h->mb.pic.p_fdec[2]+backup_src, 16*sizeof(pixel) );
1664
            }
1665
            else
1666
            {
1667
                if( CHROMA_FORMAT == CHROMA_420 )
1668
                    backup_src = (MB_INTERLACED ? 3 : 6) * FDEC_STRIDE;
1669
                memcpy( &h->intra_border_backup[backup_dst][1][mb_x*16  ], h->mb.pic.p_fdec[1]+backup_src,  8*sizeof(pixel) );
1670
                memcpy( &h->intra_border_backup[backup_dst][1][mb_x*16+8], h->mb.pic.p_fdec[2]+backup_src,  8*sizeof(pixel) );
1671
            }
1672
        }
1673
    }
1674
}
1675

1676
void x264_macroblock_cache_save( x264_t *h )
1677
{
1678
    const int i_mb_xy = h->mb.i_mb_xy;
1679
    const int i_mb_type = x264_mb_type_fix[h->mb.i_type];
1680
    const int s8x8 = h->mb.i_b8_stride;
1681
    const int s4x4 = h->mb.i_b4_stride;
1682
    const int i_mb_4x4 = h->mb.i_b4_xy;
1683
    const int i_mb_8x8 = h->mb.i_b8_xy;
1684

1685
    /* GCC pessimizes direct stores to heap-allocated arrays due to aliasing. */
1686
    /* By only dereferencing them once, we avoid this issue. */
1687
    int8_t *i4x4 = h->mb.intra4x4_pred_mode[i_mb_xy];
1688
    uint8_t *nnz = h->mb.non_zero_count[i_mb_xy];
1689

1690
    if( SLICE_MBAFF )
1691
    {
1692
        x264_macroblock_backup_intra( h, h->mb.i_mb_x, h->mb.i_mb_y, 1 );
1693
        x264_macroblock_store_pic( h, h->mb.i_mb_x, h->mb.i_mb_y, 0, 0, 1 );
1694
        if( CHROMA444 )
1695
        {
1696
            x264_macroblock_store_pic( h, h->mb.i_mb_x, h->mb.i_mb_y, 1, 0, 1 );
1697
            x264_macroblock_store_pic( h, h->mb.i_mb_x, h->mb.i_mb_y, 2, 0, 1 );
1698
        }
1699
        else
1700
            x264_macroblock_store_pic( h, h->mb.i_mb_x, h->mb.i_mb_y, 1, 1, 1 );
1701
    }
1702
    else
1703
    {
1704
        x264_macroblock_backup_intra( h, h->mb.i_mb_x, h->mb.i_mb_y, 0 );
1705
        x264_macroblock_store_pic( h, h->mb.i_mb_x, h->mb.i_mb_y, 0, 0, 0 );
1706
        if( CHROMA444 )
1707
        {
1708
            x264_macroblock_store_pic( h, h->mb.i_mb_x, h->mb.i_mb_y, 1, 0, 0 );
1709
            x264_macroblock_store_pic( h, h->mb.i_mb_x, h->mb.i_mb_y, 2, 0, 0 );
1710
        }
1711
        else
1712
            x264_macroblock_store_pic( h, h->mb.i_mb_x, h->mb.i_mb_y, 1, 1, 0 );
1713
    }
1714

1715
    x264_prefetch_fenc( h, h->fdec, h->mb.i_mb_x, h->mb.i_mb_y );
1716

1717
    h->mb.type[i_mb_xy] = i_mb_type;
1718
    h->mb.slice_table[i_mb_xy] = h->sh.i_first_mb;
1719
    h->mb.partition[i_mb_xy] = IS_INTRA( i_mb_type ) ? D_16x16 : h->mb.i_partition;
1720
    h->mb.i_mb_prev_xy = i_mb_xy;
1721

1722
    /* save intra4x4 */
1723
    if( i_mb_type == I_4x4 )
1724
    {
1725
        CP32( &i4x4[0], &h->mb.cache.intra4x4_pred_mode[x264_scan8[10]] );
1726
        M32( &i4x4[4] ) = pack8to32( h->mb.cache.intra4x4_pred_mode[x264_scan8[5] ],
1727
                                     h->mb.cache.intra4x4_pred_mode[x264_scan8[7] ],
1728
                                     h->mb.cache.intra4x4_pred_mode[x264_scan8[13] ], 0);
1729
    }
1730
    else if( !h->param.b_constrained_intra || IS_INTRA(i_mb_type) )
1731
        M64( i4x4 ) = I_PRED_4x4_DC * 0x0101010101010101ULL;
1732
    else
1733
        M64( i4x4 ) = (uint8_t)(-1) * 0x0101010101010101ULL;
1734

1735

1736
    if( i_mb_type == I_PCM )
1737
    {
1738
        h->mb.qp[i_mb_xy] = 0;
1739
        h->mb.i_last_dqp = 0;
1740
        h->mb.i_cbp_chroma = CHROMA444 ? 0 : 2;
1741
        h->mb.i_cbp_luma = 0xf;
1742
        h->mb.cbp[i_mb_xy] = (h->mb.i_cbp_chroma << 4) | h->mb.i_cbp_luma | 0x700;
1743
        h->mb.b_transform_8x8 = 0;
1744
        for( int i = 0; i < 48; i++ )
1745
            h->mb.cache.non_zero_count[x264_scan8[i]] = h->param.b_cabac ? 1 : 16;
1746
    }
1747
    else
1748
    {
1749
        if( h->mb.i_type != I_16x16 && h->mb.i_cbp_luma == 0 && h->mb.i_cbp_chroma == 0 )
1750
            h->mb.i_qp = h->mb.i_last_qp;
1751
        h->mb.qp[i_mb_xy] = h->mb.i_qp;
1752
        h->mb.i_last_dqp = h->mb.i_qp - h->mb.i_last_qp;
1753
        h->mb.i_last_qp = h->mb.i_qp;
1754
    }
1755

1756
    /* save non zero count */
1757
    CP32( &nnz[ 0+0*4], &h->mb.cache.non_zero_count[x264_scan8[ 0]] );
1758
    CP32( &nnz[ 0+1*4], &h->mb.cache.non_zero_count[x264_scan8[ 2]] );
1759
    CP32( &nnz[ 0+2*4], &h->mb.cache.non_zero_count[x264_scan8[ 8]] );
1760
    CP32( &nnz[ 0+3*4], &h->mb.cache.non_zero_count[x264_scan8[10]] );
1761
    CP32( &nnz[16+0*4], &h->mb.cache.non_zero_count[x264_scan8[16+0]] );
1762
    CP32( &nnz[16+1*4], &h->mb.cache.non_zero_count[x264_scan8[16+2]] );
1763
    CP32( &nnz[32+0*4], &h->mb.cache.non_zero_count[x264_scan8[32+0]] );
1764
    CP32( &nnz[32+1*4], &h->mb.cache.non_zero_count[x264_scan8[32+2]] );
1765
    if( CHROMA_FORMAT >= CHROMA_422 )
1766
    {
1767
        CP32( &nnz[16+2*4], &h->mb.cache.non_zero_count[x264_scan8[16+ 8]] );
1768
        CP32( &nnz[16+3*4], &h->mb.cache.non_zero_count[x264_scan8[16+10]] );
1769
        CP32( &nnz[32+2*4], &h->mb.cache.non_zero_count[x264_scan8[32+ 8]] );
1770
        CP32( &nnz[32+3*4], &h->mb.cache.non_zero_count[x264_scan8[32+10]] );
1771
    }
1772

1773
    if( h->mb.i_cbp_luma == 0 && h->mb.i_type != I_8x8 )
1774
        h->mb.b_transform_8x8 = 0;
1775
    h->mb.mb_transform_size[i_mb_xy] = h->mb.b_transform_8x8;
1776

1777
    if( h->sh.i_type != SLICE_TYPE_I )
1778
    {
1779
        int16_t (*mv0)[2] = &h->mb.mv[0][i_mb_4x4];
1780
        int16_t (*mv1)[2] = &h->mb.mv[1][i_mb_4x4];
1781
        int8_t *ref0 = &h->mb.ref[0][i_mb_8x8];
1782
        int8_t *ref1 = &h->mb.ref[1][i_mb_8x8];
1783
        if( !IS_INTRA( i_mb_type ) )
1784
        {
1785
            ref0[0+0*s8x8] = h->mb.cache.ref[0][x264_scan8[0]];
1786
            ref0[1+0*s8x8] = h->mb.cache.ref[0][x264_scan8[4]];
1787
            ref0[0+1*s8x8] = h->mb.cache.ref[0][x264_scan8[8]];
1788
            ref0[1+1*s8x8] = h->mb.cache.ref[0][x264_scan8[12]];
1789
            CP128( &mv0[0*s4x4], h->mb.cache.mv[0][x264_scan8[0]+8*0] );
1790
            CP128( &mv0[1*s4x4], h->mb.cache.mv[0][x264_scan8[0]+8*1] );
1791
            CP128( &mv0[2*s4x4], h->mb.cache.mv[0][x264_scan8[0]+8*2] );
1792
            CP128( &mv0[3*s4x4], h->mb.cache.mv[0][x264_scan8[0]+8*3] );
1793
            if( h->sh.i_type == SLICE_TYPE_B )
1794
            {
1795
                ref1[0+0*s8x8] = h->mb.cache.ref[1][x264_scan8[0]];
1796
                ref1[1+0*s8x8] = h->mb.cache.ref[1][x264_scan8[4]];
1797
                ref1[0+1*s8x8] = h->mb.cache.ref[1][x264_scan8[8]];
1798
                ref1[1+1*s8x8] = h->mb.cache.ref[1][x264_scan8[12]];
1799
                CP128( &mv1[0*s4x4], h->mb.cache.mv[1][x264_scan8[0]+8*0] );
1800
                CP128( &mv1[1*s4x4], h->mb.cache.mv[1][x264_scan8[0]+8*1] );
1801
                CP128( &mv1[2*s4x4], h->mb.cache.mv[1][x264_scan8[0]+8*2] );
1802
                CP128( &mv1[3*s4x4], h->mb.cache.mv[1][x264_scan8[0]+8*3] );
1803
            }
1804
        }
1805
        else
1806
        {
1807
            M16( &ref0[0*s8x8] ) = (uint8_t)(-1) * 0x0101;
1808
            M16( &ref0[1*s8x8] ) = (uint8_t)(-1) * 0x0101;
1809
            M128( &mv0[0*s4x4] ) = M128_ZERO;
1810
            M128( &mv0[1*s4x4] ) = M128_ZERO;
1811
            M128( &mv0[2*s4x4] ) = M128_ZERO;
1812
            M128( &mv0[3*s4x4] ) = M128_ZERO;
1813
            if( h->sh.i_type == SLICE_TYPE_B )
1814
            {
1815
                M16( &ref1[0*s8x8] ) = (uint8_t)(-1) * 0x0101;
1816
                M16( &ref1[1*s8x8] ) = (uint8_t)(-1) * 0x0101;
1817
                M128( &mv1[0*s4x4] ) = M128_ZERO;
1818
                M128( &mv1[1*s4x4] ) = M128_ZERO;
1819
                M128( &mv1[2*s4x4] ) = M128_ZERO;
1820
                M128( &mv1[3*s4x4] ) = M128_ZERO;
1821
            }
1822
        }
1823
    }
1824

1825
    if( h->param.b_cabac )
1826
    {
1827
        uint8_t (*mvd0)[2] = h->mb.mvd[0][i_mb_xy];
1828
        uint8_t (*mvd1)[2] = h->mb.mvd[1][i_mb_xy];
1829
        if( IS_INTRA(i_mb_type) && i_mb_type != I_PCM )
1830
            h->mb.chroma_pred_mode[i_mb_xy] = x264_mb_chroma_pred_mode_fix[h->mb.i_chroma_pred_mode];
1831
        else
1832
            h->mb.chroma_pred_mode[i_mb_xy] = I_PRED_CHROMA_DC;
1833

1834
        if( (0x3FF30 >> i_mb_type) & 1 ) /* !INTRA && !SKIP && !DIRECT */
1835
        {
1836
            CP64( mvd0[0], h->mb.cache.mvd[0][x264_scan8[10]] );
1837
            CP16( mvd0[4], h->mb.cache.mvd[0][x264_scan8[5 ]] );
1838
            CP16( mvd0[5], h->mb.cache.mvd[0][x264_scan8[7 ]] );
1839
            CP16( mvd0[6], h->mb.cache.mvd[0][x264_scan8[13]] );
1840
            if( h->sh.i_type == SLICE_TYPE_B )
1841
            {
1842
                CP64( mvd1[0], h->mb.cache.mvd[1][x264_scan8[10]] );
1843
                CP16( mvd1[4], h->mb.cache.mvd[1][x264_scan8[5 ]] );
1844
                CP16( mvd1[5], h->mb.cache.mvd[1][x264_scan8[7 ]] );
1845
                CP16( mvd1[6], h->mb.cache.mvd[1][x264_scan8[13]] );
1846
            }
1847
        }
1848
        else
1849
        {
1850
            M128( mvd0[0] ) = M128_ZERO;
1851
            if( h->sh.i_type == SLICE_TYPE_B )
1852
                M128( mvd1[0] ) = M128_ZERO;
1853
        }
1854

1855
        if( h->sh.i_type == SLICE_TYPE_B )
1856
        {
1857
            if( i_mb_type == B_SKIP || i_mb_type == B_DIRECT )
1858
                h->mb.skipbp[i_mb_xy] = 0xf;
1859
            else if( i_mb_type == B_8x8 )
1860
            {
1861
                int skipbp = ( h->mb.i_sub_partition[0] == D_DIRECT_8x8 ) << 0;
1862
                skipbp    |= ( h->mb.i_sub_partition[1] == D_DIRECT_8x8 ) << 1;
1863
                skipbp    |= ( h->mb.i_sub_partition[2] == D_DIRECT_8x8 ) << 2;
1864
                skipbp    |= ( h->mb.i_sub_partition[3] == D_DIRECT_8x8 ) << 3;
1865
                h->mb.skipbp[i_mb_xy] = skipbp;
1866
            }
1867
            else
1868
                h->mb.skipbp[i_mb_xy] = 0;
1869
        }
1870
    }
1871
}
1872

1873

1874
void x264_macroblock_bipred_init( x264_t *h )
1875
{
1876
    for( int mbfield = 0; mbfield <= SLICE_MBAFF; mbfield++ )
1877
        for( int field = 0; field <= SLICE_MBAFF; field++ )
1878
            for( int i_ref0 = 0; i_ref0 < (h->i_ref[0]<<mbfield); i_ref0++ )
1879
            {
1880
                x264_frame_t *l0 = h->fref[0][i_ref0>>mbfield];
1881
                int poc0 = l0->i_poc + mbfield*l0->i_delta_poc[field^(i_ref0&1)];
1882
                for( int i_ref1 = 0; i_ref1 < (h->i_ref[1]<<mbfield); i_ref1++ )
1883
                {
1884
                    int dist_scale_factor;
1885
                    x264_frame_t *l1 = h->fref[1][i_ref1>>mbfield];
1886
                    int cur_poc = h->fdec->i_poc + mbfield*h->fdec->i_delta_poc[field];
1887
                    int poc1 = l1->i_poc + mbfield*l1->i_delta_poc[field^(i_ref1&1)];
1888
                    int td = x264_clip3( poc1 - poc0, -128, 127 );
1889
                    if( td == 0 /* || pic0 is a long-term ref */ )
1890
                        dist_scale_factor = 256;
1891
                    else
1892
                    {
1893
                        int tb = x264_clip3( cur_poc - poc0, -128, 127 );
1894
                        int tx = (16384 + (abs(td) >> 1)) / td;
1895
                        dist_scale_factor = x264_clip3( (tb * tx + 32) >> 6, -1024, 1023 );
1896
                    }
1897

1898
                    h->mb.dist_scale_factor_buf[mbfield][field][i_ref0][i_ref1] = dist_scale_factor;
1899

1900
                    dist_scale_factor >>= 2;
1901
                    if( h->param.analyse.b_weighted_bipred
1902
                          && dist_scale_factor >= -64
1903
                          && dist_scale_factor <= 128 )
1904
                    {
1905
                        h->mb.bipred_weight_buf[mbfield][field][i_ref0][i_ref1] = 64 - dist_scale_factor;
1906
                        // ssse3 implementation of biweight doesn't support the extrema.
1907
                        // if we ever generate them, we'll have to drop that optimization.
1908
                        assert( dist_scale_factor >= -63 && dist_scale_factor <= 127 );
1909
                    }
1910
                    else
1911
                        h->mb.bipred_weight_buf[mbfield][field][i_ref0][i_ref1] = 32;
1912
                }
1913
            }
1914
}
1915

1916

1917
Product

Resources

Company
