Browse code

avformat/flvenc: add add_keyframe_index option

Add keyframe index metadata
Used to facilitate seeking; particularly for HTTP pseudo streaming.
1. read live streaming or file by sequence
2. if use add_keyframe_index option, add a mark flag at the position,
use to insert new context at the last step.
3. add the keyframes *offset* and *timestamp* into a list
4. if use add_keyframe_index option, shift the metadata data from
mark flag offset
5. insert the keyframes *offset* and *timestamp* from the list by
sequence
6. free the list
7. end.

Add FATE test case;

Reviewed-by: Lou Logan <lou@lrcd.com>
Signed-off-by: Steven Liu <liuqi@gosun.com>
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

Steven Liu authored on 2016/11/08 20:27:21
Showing 6 changed files
... ...
@@ -147,6 +147,9 @@ Place AAC sequence header based on audio stream data.
147 147
 
148 148
 @item no_sequence_end
149 149
 Disable sequence end tag.
150
+
151
+@item add_keyframe_index
152
+Used to facilitate seeking; particularly for HTTP pseudo streaming.
150 153
 @end table
151 154
 @end table
152 155
 
... ...
@@ -24,6 +24,8 @@
24 24
 #include "libavutil/intfloat.h"
25 25
 #include "libavutil/avassert.h"
26 26
 #include "libavutil/mathematics.h"
27
+#include "avio_internal.h"
28
+#include "avio.h"
27 29
 #include "avc.h"
28 30
 #include "avformat.h"
29 31
 #include "flv.h"
... ...
@@ -64,8 +66,15 @@ static const AVCodecTag flv_audio_codec_ids[] = {
64 64
 typedef enum {
65 65
     FLV_AAC_SEQ_HEADER_DETECT = (1 << 0),
66 66
     FLV_NO_SEQUENCE_END = (1 << 1),
67
+    FLV_ADD_KEYFRAME_INDEX = (1 << 2),
67 68
 } FLVFlags;
68 69
 
70
+typedef struct FLVFileposition {
71
+    int64_t keyframe_position;
72
+    double keyframe_timestamp;
73
+    struct FLVFileposition *next;
74
+} FLVFileposition;
75
+
69 76
 typedef struct FLVContext {
70 77
     AVClass *av_class;
71 78
     int     reserved;
... ...
@@ -74,6 +83,33 @@ typedef struct FLVContext {
74 74
     int64_t duration;
75 75
     int64_t delay;      ///< first dts delay (needed for AVC & Speex)
76 76
 
77
+    int64_t datastart_offset;
78
+    int64_t datasize_offset;
79
+    int64_t datasize;
80
+    int64_t videosize_offset;
81
+    int64_t videosize;
82
+    int64_t audiosize_offset;
83
+    int64_t audiosize;
84
+
85
+    int64_t metadata_size_pos;
86
+    int64_t metadata_totalsize_pos;
87
+    int64_t metadata_totalsize;
88
+    int64_t keyframe_index_size;
89
+
90
+    int64_t lasttimestamp_offset;
91
+    double lasttimestamp;
92
+    int64_t lastkeyframetimestamp_offset;
93
+    double lastkeyframetimestamp;
94
+    int64_t lastkeyframelocation_offset;
95
+    int64_t lastkeyframelocation;
96
+
97
+    int acurframeindex;
98
+    int64_t keyframes_info_offset;
99
+
100
+    int64_t filepositions_count;
101
+    FLVFileposition *filepositions;
102
+    FLVFileposition *head_filepositions;
103
+
77 104
     AVCodecParameters *audio_par;
78 105
     AVCodecParameters *video_par;
79 106
     double framerate;
... ...
@@ -211,6 +247,17 @@ static void put_amf_double(AVIOContext *pb, double d)
211 211
     avio_wb64(pb, av_double2int(d));
212 212
 }
213 213
 
214
+static void put_amf_byte(AVIOContext *pb, unsigned char abyte)
215
+{
216
+    avio_w8(pb, abyte);
217
+}
218
+
219
+static void put_amf_dword_array(AVIOContext *pb, uint32_t dw)
220
+{
221
+    avio_w8(pb, AMF_DATA_TYPE_ARRAY);
222
+    avio_wb32(pb, dw);
223
+}
224
+
214 225
 static void put_amf_bool(AVIOContext *pb, int b)
215 226
 {
216 227
     avio_w8(pb, AMF_DATA_TYPE_BOOL);
... ...
@@ -222,12 +269,12 @@ static void write_metadata(AVFormatContext *s, unsigned int ts)
222 222
     AVIOContext *pb = s->pb;
223 223
     FLVContext *flv = s->priv_data;
224 224
     int metadata_count = 0;
225
-    int64_t metadata_size_pos, data_size, metadata_count_pos;
225
+    int64_t metadata_count_pos;
226 226
     AVDictionaryEntry *tag = NULL;
227 227
 
228 228
     /* write meta_tag */
229
-    avio_w8(pb, 18);            // tag type META
230
-    metadata_size_pos = avio_tell(pb);
229
+    avio_w8(pb, FLV_TAG_TYPE_META);            // tag type META
230
+    flv->metadata_size_pos = avio_tell(pb);
231 231
     avio_wb24(pb, 0);           // size of data part (sum of all parts below)
232 232
     avio_wb24(pb, ts);          // timestamp
233 233
     avio_wb32(pb, 0);           // reserved
... ...
@@ -336,19 +383,87 @@ static void write_metadata(AVFormatContext *s, unsigned int ts)
336 336
         put_amf_double(pb, 0); // delayed write
337 337
     }
338 338
 
339
+    if (flv->flags & FLV_ADD_KEYFRAME_INDEX) {
340
+        flv->acurframeindex = 0;
341
+        flv->keyframe_index_size = 0;
342
+
343
+        put_amf_string(pb, "hasVideo");
344
+        put_amf_bool(pb, !!flv->video_par);
345
+        metadata_count++;
346
+
347
+        put_amf_string(pb, "hasKeyframes");
348
+        put_amf_bool(pb, 1);
349
+        metadata_count++;
350
+
351
+        put_amf_string(pb, "hasAudio");
352
+        put_amf_bool(pb, !!flv->audio_par);
353
+        metadata_count++;
354
+
355
+        put_amf_string(pb, "hasMetadata");
356
+        put_amf_bool(pb, 1);
357
+        metadata_count++;
358
+
359
+        put_amf_string(pb, "canSeekToEnd");
360
+        put_amf_bool(pb, 1);
361
+        metadata_count++;
362
+
363
+        put_amf_string(pb, "datasize");
364
+        flv->datasize_offset = avio_tell(pb);
365
+        flv->datasize = 0;
366
+        put_amf_double(pb, flv->datasize);
367
+        metadata_count++;
368
+
369
+        put_amf_string(pb, "videosize");
370
+        flv->videosize_offset = avio_tell(pb);
371
+        flv->videosize = 0;
372
+        put_amf_double(pb, flv->videosize);
373
+        metadata_count++;
374
+
375
+        put_amf_string(pb, "audiosize");
376
+        flv->audiosize_offset = avio_tell(pb);
377
+        flv->audiosize = 0;
378
+        put_amf_double(pb, flv->audiosize);
379
+        metadata_count++;
380
+
381
+        put_amf_string(pb, "lasttimestamp");
382
+        flv->lasttimestamp_offset = avio_tell(pb);
383
+        flv->lasttimestamp = 0;
384
+        put_amf_double(pb, 0);
385
+        metadata_count++;
386
+
387
+        put_amf_string(pb, "lastkeyframetimestamp");
388
+        flv->lastkeyframetimestamp_offset = avio_tell(pb);
389
+        flv->lastkeyframetimestamp = 0;
390
+        put_amf_double(pb, 0);
391
+        metadata_count++;
392
+
393
+        put_amf_string(pb, "lastkeyframelocation");
394
+        flv->lastkeyframelocation_offset = avio_tell(pb);
395
+        flv->lastkeyframelocation = 0;
396
+        put_amf_double(pb, 0);
397
+        metadata_count++;
398
+
399
+        put_amf_string(pb, "keyframes");
400
+        put_amf_byte(pb, AMF_DATA_TYPE_OBJECT);
401
+        metadata_count++;
402
+
403
+        flv->keyframes_info_offset = avio_tell(pb);
404
+    }
405
+
339 406
     put_amf_string(pb, "");
340 407
     avio_w8(pb, AMF_END_OF_OBJECT);
341 408
 
342 409
     /* write total size of tag */
343
-    data_size = avio_tell(pb) - metadata_size_pos - 10;
410
+    flv->metadata_totalsize = avio_tell(pb) - flv->metadata_size_pos - 10;
344 411
 
345 412
     avio_seek(pb, metadata_count_pos, SEEK_SET);
346 413
     avio_wb32(pb, metadata_count);
347 414
 
348
-    avio_seek(pb, metadata_size_pos, SEEK_SET);
349
-    avio_wb24(pb, data_size);
350
-    avio_skip(pb, data_size + 10 - 3);
351
-    avio_wb32(pb, data_size + 11);
415
+    avio_seek(pb, flv->metadata_size_pos, SEEK_SET);
416
+    avio_wb24(pb, flv->metadata_totalsize);
417
+    avio_skip(pb, flv->metadata_totalsize + 10 - 3);
418
+    flv->metadata_totalsize_pos = avio_tell(pb);
419
+    avio_wb32(pb, flv->metadata_totalsize + 11);
352 420
 }
353 421
 
354 422
 static int unsupported_codec(AVFormatContext *s,
... ...
@@ -425,6 +540,111 @@ static void flv_write_codec_header(AVFormatContext* s, AVCodecParameters* par) {
425 425
     }
426 426
 }
427 427
 
428
+static int flv_append_keyframe_info(AVFormatContext *s, FLVContext *flv, double ts, int64_t pos)
429
+{
430
+    FLVFileposition *position = av_malloc(sizeof(FLVFileposition));
431
+
432
+    if (!position) {
433
+        av_log(s, AV_LOG_WARNING, "no mem for add keyframe index!\n");
434
+        return AVERROR(ENOMEM);
435
+    }
436
+
437
+    position->keyframe_timestamp = ts;
438
+    position->keyframe_position = pos;
439
+
440
+    if (!flv->filepositions_count) {
441
+        flv->filepositions = position;
442
+        flv->head_filepositions = flv->filepositions;
443
+        position->next = NULL;
444
+    } else {
445
+        flv->filepositions->next = position;
446
+        position->next = NULL;
447
+        flv->filepositions = flv->filepositions->next;
448
+    }
449
+
450
+    flv->filepositions_count++;
451
+
452
+    return 0;
453
+}
454
+
455
+static int shift_data(AVFormatContext *s)
456
+{
457
+    int ret = 0;
458
+    int n = 0;
459
+    int64_t metadata_size = 0;
460
+    FLVContext *flv = s->priv_data;
461
+    int64_t pos, pos_end = avio_tell(s->pb);
462
+    uint8_t *buf, *read_buf[2];
463
+    int read_buf_id = 0;
464
+    int read_size[2];
465
+    AVIOContext *read_pb;
466
+
467
+    metadata_size = flv->filepositions_count * 9 * 2 + 10; /* filepositions and times value */
468
+    metadata_size += 2 + 13; /* filepositions String */
469
+    metadata_size += 2 + 5; /* times String */
470
+    metadata_size += 3; /* Object end */
471
+
472
+    flv->keyframe_index_size = metadata_size;
473
+
474
+    if (metadata_size < 0)
475
+        return metadata_size;
476
+
477
+    buf = av_malloc_array(metadata_size, 2);
478
+    if (!buf) {
479
+        return AVERROR(ENOMEM);
480
+    }
481
+    read_buf[0] = buf;
482
+    read_buf[1] = buf + metadata_size;
483
+
484
+    avio_seek(s->pb, flv->metadata_size_pos, SEEK_SET);
485
+    avio_wb24(s->pb, flv->metadata_totalsize + metadata_size);
486
+
487
+    avio_seek(s->pb, flv->metadata_totalsize_pos, SEEK_SET);
488
+    avio_wb32(s->pb, flv->metadata_totalsize + 11 + metadata_size);
489
+    avio_seek(s->pb, pos_end, SEEK_SET);
490
+
491
+    /* Shift the data: the AVIO context of the output can only be used for
492
+     * writing, so we re-open the same output, but for reading. It also avoids
493
+     * a read/seek/write/seek back and forth. */
494
+    avio_flush(s->pb);
495
+    ret = s->io_open(s, &read_pb, s->filename, AVIO_FLAG_READ, NULL);
496
+    if (ret < 0) {
497
+        av_log(s, AV_LOG_ERROR, "Unable to re-open %s output file for "
498
+               "the second pass (add_keyframe_index)\n", s->filename);
499
+        goto end;
500
+    }
501
+
502
+    /* mark the end of the shift to up to the last data we wrote, and get ready
503
+     * for writing */
504
+    pos_end = avio_tell(s->pb);
505
+    avio_seek(s->pb, flv->keyframes_info_offset + metadata_size, SEEK_SET);
506
+
507
+    /* start reading at where the keyframe index information will be placed */
508
+    avio_seek(read_pb, flv->keyframes_info_offset, SEEK_SET);
509
+    pos = avio_tell(read_pb);
510
+
511
+    /* shift data by chunk of at most keyframe *filepositions* and *times* size */
512
+    read_size[read_buf_id] = avio_read(read_pb, read_buf[read_buf_id], metadata_size);  \
513
+    read_buf_id ^= 1;
514
+    do {
515
+
516
+        read_size[read_buf_id] = avio_read(read_pb, read_buf[read_buf_id], metadata_size);  \
517
+        read_buf_id ^= 1;
518
+        n = read_size[read_buf_id];
519
+        if (n < 0)
520
+            break;
521
+        avio_write(s->pb, read_buf[read_buf_id], n);
522
+        pos += n;
523
+    } while (pos <= pos_end);
524
+
525
+    ff_format_io_close(s, &read_pb);
526
+
527
+end:
528
+    av_free(buf);
529
+    return ret;
530
+}
531
+
532
+
428 533
 static int flv_write_header(AVFormatContext *s)
429 534
 {
430 535
     int i;
... ...
@@ -530,17 +750,75 @@ static int flv_write_header(AVFormatContext *s)
530 530
         flv_write_codec_header(s, s->streams[i]->codecpar);
531 531
     }
532 532
 
533
+    flv->datastart_offset = avio_tell(pb);
533 534
     return 0;
534 535
 }
535 536
 
536 537
 static int flv_write_trailer(AVFormatContext *s)
537 538
 {
538 539
     int64_t file_size;
539
-
540 540
     AVIOContext *pb = s->pb;
541 541
     FLVContext *flv = s->priv_data;
542
-    int i;
542
+    int build_keyframes_idx = flv->flags & FLV_ADD_KEYFRAME_INDEX;
543
+    int i, res;
544
+    int64_t cur_pos = avio_tell(s->pb);
545
+
546
+    if (build_keyframes_idx) {
547
+        FLVFileposition *newflv_posinfo, *p;
548
+
549
+        avio_seek(pb, flv->videosize_offset, SEEK_SET);
550
+        put_amf_double(pb, flv->videosize);
551
+
552
+        avio_seek(pb, flv->audiosize_offset, SEEK_SET);
553
+        put_amf_double(pb, flv->audiosize);
554
+
555
+        avio_seek(pb, flv->lasttimestamp_offset, SEEK_SET);
556
+        put_amf_double(pb, flv->lasttimestamp);
557
+
558
+        avio_seek(pb, flv->lastkeyframetimestamp_offset, SEEK_SET);
559
+        put_amf_double(pb, flv->lastkeyframetimestamp);
543 560
 
561
+        avio_seek(pb, flv->lastkeyframelocation_offset, SEEK_SET);
562
+        put_amf_double(pb, flv->lastkeyframelocation + flv->keyframe_index_size);
563
+        avio_seek(pb, cur_pos, SEEK_SET);
564
+
565
+        res = shift_data(s);
566
+        if (res < 0) {
567
+             goto end;
568
+        }
569
+        avio_seek(pb, flv->keyframes_info_offset, SEEK_SET);
570
+        put_amf_string(pb, "filepositions");
571
+        put_amf_dword_array(pb, flv->filepositions_count);
572
+        for (newflv_posinfo = flv->head_filepositions; newflv_posinfo; newflv_posinfo = newflv_posinfo->next) {
573
+            put_amf_double(pb, newflv_posinfo->keyframe_position + flv->keyframe_index_size);
574
+        }
575
+
576
+        put_amf_string(pb, "times");
577
+        put_amf_dword_array(pb, flv->filepositions_count);
578
+        for (newflv_posinfo = flv->head_filepositions; newflv_posinfo; newflv_posinfo = newflv_posinfo->next) {
579
+            put_amf_double(pb, newflv_posinfo->keyframe_timestamp);
580
+        }
581
+
582
+        newflv_posinfo = flv->head_filepositions;
583
+        while (newflv_posinfo) {
584
+            p = newflv_posinfo->next;
585
+            if (p) {
586
+                newflv_posinfo->next = p->next;
587
+                av_free(p);
588
+                p = NULL;
589
+            } else {
590
+                av_free(newflv_posinfo);
591
+                newflv_posinfo = NULL;
592
+            }
593
+        }
594
+
595
+        put_amf_string(pb, "");
596
+        avio_w8(pb, AMF_END_OF_OBJECT);
597
+
598
+        avio_seek(pb, cur_pos + flv->keyframe_index_size, SEEK_SET);
599
+    }
600
+
601
+end:
544 602
     if (flv->flags & FLV_NO_SEQUENCE_END) {
545 603
         av_log(s, AV_LOG_DEBUG, "FLV no sequence end mode open\n");
546 604
     } else {
... ...
@@ -556,6 +834,11 @@ static int flv_write_trailer(AVFormatContext *s)
556 556
 
557 557
     file_size = avio_tell(pb);
558 558
 
559
+    if (build_keyframes_idx) {
560
+        flv->datasize = file_size - flv->datastart_offset;
561
+        avio_seek(pb, flv->datasize_offset, SEEK_SET);
562
+        put_amf_double(pb, flv->datasize);
563
+    }
559 564
     if (pb->seekable) {
560 565
         /* update information */
561 566
         if (avio_seek(pb, flv->duration_offset, SEEK_SET) < 0) {
... ...
@@ -583,6 +866,7 @@ static int flv_write_packet(AVFormatContext *s, AVPacket *pkt)
583 583
     int size = pkt->size;
584 584
     uint8_t *data = NULL;
585 585
     int flags = -1, flags_size, ret;
586
+    int64_t cur_offset = avio_tell(pb);
586 587
 
587 588
     if (par->codec_id == AV_CODEC_ID_VP6F || par->codec_id == AV_CODEC_ID_VP6A ||
588 589
         par->codec_id == AV_CODEC_ID_VP6  || par->codec_id == AV_CODEC_ID_AAC)
... ...
@@ -736,6 +1020,32 @@ static int flv_write_packet(AVFormatContext *s, AVPacket *pkt)
736 736
                               pkt->pts + flv->delay + pkt->duration);
737 737
     }
738 738
 
739
+    if (flv->flags & FLV_ADD_KEYFRAME_INDEX) {
740
+        switch (par->codec_type) {
741
+            case AVMEDIA_TYPE_VIDEO:
742
+                flv->videosize += (avio_tell(pb) - cur_offset);
743
+                flv->lasttimestamp = flv->acurframeindex / flv->framerate;
744
+                if (pkt->flags & AV_PKT_FLAG_KEY) {
745
+                    double ts = flv->acurframeindex / flv->framerate;
746
+                    int64_t pos = cur_offset;
747
+
748
+                    flv->lastkeyframetimestamp = flv->acurframeindex / flv->framerate;
749
+                    flv->lastkeyframelocation = pos;
750
+                    flv_append_keyframe_info(s, flv, ts, pos);
751
+                }
752
+                flv->acurframeindex++;
753
+                break;
754
+
755
+            case AVMEDIA_TYPE_AUDIO:
756
+                flv->audiosize += (avio_tell(pb) - cur_offset);
757
+                break;
758
+
759
+            default:
760
+                av_log(s, AV_LOG_WARNING, "par->codec_type is type = [%d]\n", par->codec_type);
761
+                break;
762
+        }
763
+    }
764
+
739 765
     av_free(data);
740 766
 
741 767
     return pb->error;
... ...
@@ -745,6 +1055,7 @@ static const AVOption options[] = {
745 745
     { "flvflags", "FLV muxer flags", offsetof(FLVContext, flags), AV_OPT_TYPE_FLAGS, {.i64 = 0}, INT_MIN, INT_MAX, AV_OPT_FLAG_ENCODING_PARAM, "flvflags" },
746 746
     { "aac_seq_header_detect", "Put AAC sequence header based on stream data", 0, AV_OPT_TYPE_CONST, {.i64 = FLV_AAC_SEQ_HEADER_DETECT}, INT_MIN, INT_MAX, AV_OPT_FLAG_ENCODING_PARAM, "flvflags" },
747 747
     { "no_sequence_end", "disable sequence end for FLV", 0, AV_OPT_TYPE_CONST, {.i64 = FLV_NO_SEQUENCE_END}, INT_MIN, INT_MAX, AV_OPT_FLAG_ENCODING_PARAM, "flvflags" },
748
+    { "add_keyframe_index", "Add keyframe index metadata", 0, AV_OPT_TYPE_CONST, {.i64 = FLV_ADD_KEYFRAME_INDEX}, INT_MIN, INT_MAX, AV_OPT_FLAG_ENCODING_PARAM, "flvflags" },
748 749
     { NULL },
749 750
 };
750 751
 
... ...
@@ -132,6 +132,7 @@ include $(SRC_PATH)/tests/fate/fifo-muxer.mak
132 132
 include $(SRC_PATH)/tests/fate/filter-audio.mak
133 133
 include $(SRC_PATH)/tests/fate/filter-video.mak
134 134
 include $(SRC_PATH)/tests/fate/flac.mak
135
+include $(SRC_PATH)/tests/fate/flvenc.mak
135 136
 include $(SRC_PATH)/tests/fate/gapless.mak
136 137
 include $(SRC_PATH)/tests/fate/gif.mak
137 138
 include $(SRC_PATH)/tests/fate/h264.mak
... ...
@@ -129,6 +129,10 @@ framecrc(){
129 129
     ffmpeg "$@" -flags +bitexact -fflags +bitexact -f framecrc -
130 130
 }
131 131
 
132
+ffmetadata(){
133
+    ffmpeg "$@" -flags +bitexact -fflags +bitexact -f ffmetadata -
134
+}
135
+
132 136
 framemd5(){
133 137
     ffmpeg "$@" -flags +bitexact -fflags +bitexact -f framemd5 -
134 138
 }
135 139
new file mode 100644
... ...
@@ -0,0 +1,11 @@
0
+tests/data/add_keyframe_index.flv: TAG = GEN
1
+tests/data/add_keyframe_index.flv: ffmpeg$(PROGSSUF)$(EXESUF) | tests/data
2
+	$(M)$(TARGET_EXEC) $(TARGET_PATH)/$< \
3
+		-f lavfi -i "sws_flags=+accurate_rnd+bitexact;testsrc=r=7:n=2:d=20" -sws_flags '+accurate_rnd+bitexact' -metadata "encoder=Lavf" -pix_fmt yuv420p -c:v flv1 -g 7 -f flv -flags +bitexact \
4
+		-flvflags add_keyframe_index -idct simple -dct int -y $(TARGET_PATH)/tests/data/add_keyframe_index.flv 2> /dev/null;
5
+
6
+FATE_AFILTER-$(call ALLYES, FLV_MUXER FLV_DEMUXER AVDEVICE TESTSRC_FILTER LAVFI_INDEV FLV_ENCODER) += fate-flv-add_keyframe_index
7
+fate-flv-add_keyframe_index: tests/data/add_keyframe_index.flv
8
+fate-flv-add_keyframe_index: CMD = ffmetadata -flags +bitexact -i $(TARGET_PATH)/tests/data/add_keyframe_index.flv
9
+
10
+
0 11
new file mode 100644
... ...
@@ -0,0 +1,12 @@
0
+;FFMETADATA1
1
+hasVideo=true
2
+hasKeyframes=true
3
+hasAudio=false
4
+hasMetadata=true
5
+canSeekToEnd=true
6
+datasize=629776
7
+videosize=629381
8
+audiosize=0
9
+lasttimestamp=20
10
+lastkeyframetimestamp=19
11
+lastkeyframelocation=597963