Browse code

avformat/hlsenc: size and duration in segment filenames

1st:
This patch makes it possible to put actual segment file size (measured
in bytes) and/or duration (calculated in microseconds) into segment
filenames. This feature is useful when post-processing live streaming
access log files. New behaviour works only when -use_localtime option
is set and second_level_segment_size or/and
second_level_segment_duration new hls_flags are specified. %%s is the
placeholder for size and %%t for duration in hls_segment_filename
option. Fix sized trailing zeropadding also works eg. %%09s or %%023t.

A command to test new features:
./ffmpeg -loglevel info -y -f lavfi -i color=c=red:size=640x480:r=25 -f
lavfi -i sine=f=440:b=4:r=44100 -c:v mpeg2video -g 25 -acodec aac
-cutoff 20000 -ac 2 -ar 44100 -ab 192k -f hls -hls_time 3 -hls_list_size
5 -hls_flags
second_level_segment_index+second_level_segment_size+second_level_segment_duration
-use_localtime 1 -use_localtime_mkdir 1 -hls_segment_filename
"segment_%Y%m%d%H%M%S_%%04d_%%08s_%%013t.ts" stream.m3u8

2nd:
doc/muxers: beside second_level_segment_duration and second_level_segment_size,
added some more details and example to hls_segment_filename,
use_localtime, use_localtime_mkdir, hls_flags. hls_flags option list
reformatted to table

Signed-off-by: Bela Bodecs <bodecsb@vivanet.hu>
Signed-off-by: Steven Liu <lq@chinaffmpeg.org>

Bela Bodecs authored on 2017/01/03 23:57:51
Showing 2 changed files
... ...
@@ -441,8 +441,15 @@ ffmpeg -i in.nut -hls_segment_filename 'file%03d.ts' out.m3u8
441 441
 This example will produce the playlist, @file{out.m3u8}, and segment files:
442 442
 @file{file000.ts}, @file{file001.ts}, @file{file002.ts}, etc.
443 443
 
444
+@var{filename} may contain full path or relative path specification,
445
+but only the file name part without any path info will be contained in the m3u8 segment list.
446
+Should a relative path be specified, the path of the created segment
447
+files will be relative to the current working directory.
448
+When use_localtime_mkdir is set, the whole expanded value of @var{filename} will be written into the m3u8 segment list.
449
+
450
+
444 451
 @item use_localtime
445
-Use strftime on @var{filename} to expand the segment filename with localtime.
452
+Use strftime() on @var{filename} to expand the segment filename with localtime.
446 453
 The segment number is also available in this mode, but to use it, you need to specify second_level_segment_index
447 454
 hls_flag and %%d will be the specifier.
448 455
 @example
... ...
@@ -450,6 +457,8 @@ ffmpeg -i in.nut -use_localtime 1 -hls_segment_filename 'file-%Y%m%d-%s.ts' out.
450 450
 @end example
451 451
 This example will produce the playlist, @file{out.m3u8}, and segment files:
452 452
 @file{file-20160215-1455569023.ts}, @file{file-20160215-1455569024.ts}, etc.
453
+Note: On some systems/environments, the @code{%s} specifier is not available. See
454
+  @code{strftime()} documentation.
453 455
 @example
454 456
 ffmpeg -i in.nut -use_localtime 1 -hls_flags second_level_segment_index -hls_segment_filename 'file-%Y%m%d-%%04d.ts' out.m3u8
455 457
 @end example
... ...
@@ -457,14 +466,21 @@ This example will produce the playlist, @file{out.m3u8}, and segment files:
457 457
 @file{file-20160215-0001.ts}, @file{file-20160215-0002.ts}, etc.
458 458
 
459 459
 @item use_localtime_mkdir
460
-Used together with -use_localtime, it will create up to one subdirectory which
460
+Used together with -use_localtime, it will create all subdirectories which
461 461
 is expanded in @var{filename}.
462 462
 @example
463 463
 ffmpeg -i in.nut -use_localtime 1 -use_localtime_mkdir 1 -hls_segment_filename '%Y%m%d/file-%Y%m%d-%s.ts' out.m3u8
464 464
 @end example
465 465
 This example will create a directory 201560215 (if it does not exist), and then
466 466
 produce the playlist, @file{out.m3u8}, and segment files:
467
-@file{201560215/file-20160215-1455569023.ts}, @file{201560215/file-20160215-1455569024.ts}, etc.
467
+@file{20160215/file-20160215-1455569023.ts}, @file{20160215/file-20160215-1455569024.ts}, etc.
468
+
469
+@example
470
+ffmpeg -i in.nut -use_localtime 1 -use_localtime_mkdir 1 -hls_segment_filename '%Y/%m/%d/file-%Y%m%d-%s.ts' out.m3u8
471
+@end example
472
+This example will create a directory hierarchy 2016/02/15 (if any of them do not exist), and then
473
+produce the playlist, @file{out.m3u8}, and segment files:
474
+@file{2016/02/15/file-20160215-1455569023.ts}, @file{2016/02/15/file-20160215-1455569024.ts}, etc.
468 475
 
469 476
 
470 477
 @item hls_key_info_file @var{key_info_file}
... ...
@@ -523,7 +539,12 @@ ffmpeg -f lavfi -re -i testsrc -c:v h264 -hls_flags delete_segments \
523 523
   -hls_key_info_file file.keyinfo out.m3u8
524 524
 @end example
525 525
 
526
-@item hls_flags single_file
526
+
527
+@item hls_flags @var{flags}
528
+Possible values:
529
+
530
+@table @samp
531
+@item single_file
527 532
 If this flag is set, the muxer will store all segments in a single MPEG-TS
528 533
 file, and will use byte ranges in the playlist. HLS playlists generated with
529 534
 this way will have the version number 4.
... ...
@@ -534,36 +555,60 @@ ffmpeg -i in.nut -hls_flags single_file out.m3u8
534 534
 Will produce the playlist, @file{out.m3u8}, and a single segment file,
535 535
 @file{out.ts}.
536 536
 
537
-@item hls_flags delete_segments
537
+@item delete_segments
538 538
 Segment files removed from the playlist are deleted after a period of time
539 539
 equal to the duration of the segment plus the duration of the playlist.
540 540
 
541
-@item hls_flags append_list
541
+@item append_list
542 542
 Append new segments into the end of old segment list,
543 543
 and remove the @code{#EXT-X-ENDLIST} from the old segment list.
544 544
 
545
-@item hls_flags round_durations
545
+@item round_durations
546 546
 Round the duration info in the playlist file segment info to integer
547 547
 values, instead of using floating point.
548 548
 
549
-@item hls_flags discont_starts
549
+@item discont_starts
550 550
 Add the @code{#EXT-X-DISCONTINUITY} tag to the playlist, before the
551 551
 first segment's information.
552 552
 
553
-@item hls_flags omit_endlist
553
+@item omit_endlist
554 554
 Do not append the @code{EXT-X-ENDLIST} tag at the end of the playlist.
555 555
 
556
-@item hls_flags split_by_time
556
+@item split_by_time
557 557
 Allow segments to start on frames other than keyframes. This improves
558 558
 behavior on some players when the time between keyframes is inconsistent,
559 559
 but may make things worse on others, and can cause some oddities during
560 560
 seeking. This flag should be used with the @code{hls_time} option.
561 561
 
562
-@item hls_flags program_date_time
562
+@item program_date_time
563 563
 Generate @code{EXT-X-PROGRAM-DATE-TIME} tags.
564 564
 
565
-@item hls_flags second_level_segment_index
566
-Makes it possible to use segment indexes as %%d besides date/time values when use_localtime is on.
565
+@item second_level_segment_index
566
+Makes it possible to use segment indexes as %%d in hls_segment_filename expression
567
+besides date/time values when use_localtime is on.
568
+To get fixed width numbers with trailing zeroes, %%0xd format is available where x is the required width.
569
+
570
+@item second_level_segment_size
571
+Makes it possible to use segment sizes (counted in bytes) as %%s in hls_segment_filename
572
+expression besides date/time values when use_localtime is on.
573
+To get fixed width numbers with trailing zeroes, %%0xs format is available where x is the required width.
574
+
575
+@item second_level_segment_duration
576
+Makes it possible to use segment duration (calculated  in microseconds) as %%t in hls_segment_filename
577
+expression besides date/time values when use_localtime is on.
578
+To get fixed width numbers with trailing zeroes, %%0xt format is available where x is the required width.
579
+
580
+@example
581
+ffmpeg -i sample.mpeg \
582
+   -f hls -hls_time 3 -hls_list_size 5 \
583
+   -hls_flags second_level_segment_index+second_level_segment_size+second_level_segment_duration \
584
+   -use_localtime 1 -use_localtime_mkdir 1 -hls_segment_filename "segment_%Y%m%d%H%M%S_%%04d_%%08s_%%013t.ts" stream.m3u8
585
+@end example
586
+This will produce segments like this:
587
+@file{segment_20170102194334_0003_00122200_0000003000000.ts}, @file{segment_20170102194334_0004_00120072_0000003000000.ts} etc.
588
+
589
+
590
+@end table
567 591
 
568 592
 @item hls_playlist_type event
569 593
 Emit @code{#EXT-X-PLAYLIST-TYPE:EVENT} in the m3u8 header. Forces
... ...
@@ -67,6 +67,8 @@ typedef enum HLSFlags {
67 67
     HLS_APPEND_LIST = (1 << 6),
68 68
     HLS_PROGRAM_DATE_TIME = (1 << 7),
69 69
     HLS_SECOND_LEVEL_SEGMENT_INDEX = (1 << 8), // include segment index in segment filenames when use_localtime  e.g.: %%03d
70
+    HLS_SECOND_LEVEL_SEGMENT_DURATION = (1 << 9), // include segment duration (microsec) in segment filenames when use_localtime  e.g.: %%09t
71
+    HLS_SECOND_LEVEL_SEGMENT_SIZE = (1 << 10), // include segment size (bytes) in segment filenames when use_localtime  e.g.: %%014s
70 72
 } HLSFlags;
71 73
 
72 74
 typedef enum {
... ...
@@ -134,6 +136,7 @@ typedef struct HLSContext {
134 134
     char *method;
135 135
 
136 136
     double initial_prog_date_time;
137
+    char current_segment_final_filename_fmt[1024]; // when renaming segments
137 138
 } HLSContext;
138 139
 
139 140
 static int mkdir_p(const char *path) {
... ...
@@ -169,6 +172,58 @@ static int mkdir_p(const char *path) {
169 169
     return ret;
170 170
 }
171 171
 
172
+static int replace_int_data_in_filename(char *buf, int buf_size, const char *filename, char placeholder, int64_t number)
173
+{
174
+    const char *p;
175
+    char *q, buf1[20], c;
176
+    int nd, len, addchar_count;
177
+    int found_count = 0;
178
+
179
+    q = buf;
180
+    p = filename;
181
+    for (;;) {
182
+        c = *p;
183
+        if (c == '\0')
184
+            break;
185
+        if (c == '%' && *(p+1) == '%')  // %%
186
+            addchar_count = 2;
187
+        else if (c == '%' && (av_isdigit(*(p+1)) || *(p+1) == placeholder)) {
188
+            nd = 0;
189
+            addchar_count = 1;
190
+            while (av_isdigit(*(p + addchar_count))) {
191
+                nd = nd * 10 + *(p + addchar_count) - '0';
192
+                addchar_count++;
193
+            }
194
+
195
+            if (*(p + addchar_count) == placeholder) {
196
+                len = snprintf(buf1, sizeof(buf1), "%0*"PRId64, (number < 0) ? nd : nd++, number);
197
+                if (len < 1)  // returned error or empty buf1
198
+                    goto fail;
199
+                if ((q - buf + len) > buf_size - 1)
200
+                    goto fail;
201
+                memcpy(q, buf1, len);
202
+                q += len;
203
+                p += (addchar_count + 1);
204
+                addchar_count = 0;
205
+                found_count++;
206
+            }
207
+
208
+        } else
209
+            addchar_count = 1;
210
+
211
+        while (addchar_count--)
212
+            if ((q - buf) < buf_size - 1)
213
+                *q++ = *p++;
214
+            else
215
+                goto fail;
216
+    }
217
+    *q = '\0';
218
+    return found_count;
219
+fail:
220
+    *q = '\0';
221
+    return -1;
222
+}
223
+
172 224
 static int hls_delete_old_segments(HLSContext *hls) {
173 225
 
174 226
     HLSSegment *segment, *previous_segment = NULL;
... ...
@@ -388,6 +443,47 @@ static int hls_append_segment(struct AVFormatContext *s, HLSContext *hls, double
388 388
     if (!en)
389 389
         return AVERROR(ENOMEM);
390 390
 
391
+    if ((hls->flags & (HLS_SECOND_LEVEL_SEGMENT_SIZE | HLS_SECOND_LEVEL_SEGMENT_DURATION)) &&
392
+        strlen(hls->current_segment_final_filename_fmt)) {
393
+        char * old_filename = av_strdup(hls->avf->filename);  // %%s will be %s after strftime
394
+        av_strlcpy(hls->avf->filename, hls->current_segment_final_filename_fmt, sizeof(hls->avf->filename));
395
+        if (hls->flags & HLS_SECOND_LEVEL_SEGMENT_SIZE) {
396
+            char * filename = av_strdup(hls->avf->filename);  // %%s will be %s after strftime
397
+            if (!filename)
398
+                return AVERROR(ENOMEM);
399
+            if (replace_int_data_in_filename(hls->avf->filename, sizeof(hls->avf->filename),
400
+                filename, 's', pos + size) < 1) {
401
+                av_log(hls, AV_LOG_ERROR,
402
+                       "Invalid second level segment filename template '%s', "
403
+                        "you can try to remove second_level_segment_size flag\n",
404
+                       filename);
405
+                av_free(filename);
406
+                av_free(old_filename);
407
+                return AVERROR(EINVAL);
408
+            }
409
+            av_free(filename);
410
+        }
411
+        if (hls->flags & HLS_SECOND_LEVEL_SEGMENT_DURATION) {
412
+            char * filename = av_strdup(hls->avf->filename);  // %%t will be %t after strftime
413
+            if (!filename)
414
+                return AVERROR(ENOMEM);
415
+            if (replace_int_data_in_filename(hls->avf->filename, sizeof(hls->avf->filename),
416
+                filename, 't',  (int64_t)round(1000000 * duration)) < 1) {
417
+                av_log(hls, AV_LOG_ERROR,
418
+                       "Invalid second level segment filename template '%s', "
419
+                        "you can try to remove second_level_segment_time flag\n",
420
+                       filename);
421
+                av_free(filename);
422
+                av_free(old_filename);
423
+                return AVERROR(EINVAL);
424
+            }
425
+            av_free(filename);
426
+        }
427
+        ff_rename(old_filename, hls->avf->filename, hls);
428
+        av_free(old_filename);
429
+    }
430
+
431
+
391 432
     filename = av_basename(hls->avf->filename);
392 433
 
393 434
     if (hls->use_localtime_mkdir) {
... ...
@@ -709,15 +805,49 @@ static int hls_start(AVFormatContext *s)
709 709
                 char * filename = av_strdup(oc->filename);  // %%d will be %d after strftime
710 710
                 if (!filename)
711 711
                     return AVERROR(ENOMEM);
712
-                if (av_get_frame_filename2(oc->filename, sizeof(oc->filename),
713
-                    filename, c->wrap ? c->sequence % c->wrap : c->sequence,
714
-                    AV_FRAME_FILENAME_FLAGS_MULTIPLE) < 0) {
715
-                    av_log(c, AV_LOG_ERROR, "Invalid second level segment filename template '%s', you can try to remove second_level_segment_index flag\n", filename);
712
+                if (replace_int_data_in_filename(oc->filename, sizeof(oc->filename),
713
+                    filename, 'd', c->wrap ? c->sequence % c->wrap : c->sequence) < 1) {
714
+                    av_log(c, AV_LOG_ERROR,
715
+                           "Invalid second level segment filename template '%s', "
716
+                            "you can try to remove second_level_segment_index flag\n",
717
+                           filename);
716 718
                     av_free(filename);
717 719
                     return AVERROR(EINVAL);
718 720
                 }
719 721
                 av_free(filename);
720 722
             }
723
+            if (c->flags & (HLS_SECOND_LEVEL_SEGMENT_SIZE | HLS_SECOND_LEVEL_SEGMENT_DURATION)) {
724
+                av_strlcpy(c->current_segment_final_filename_fmt, oc->filename,
725
+                           sizeof(c->current_segment_final_filename_fmt));
726
+                if (c->flags & HLS_SECOND_LEVEL_SEGMENT_SIZE) {
727
+                    char * filename = av_strdup(oc->filename);  // %%s will be %s after strftime
728
+                    if (!filename)
729
+                        return AVERROR(ENOMEM);
730
+                    if (replace_int_data_in_filename(oc->filename, sizeof(oc->filename), filename, 's', 0) < 1) {
731
+                        av_log(c, AV_LOG_ERROR,
732
+                               "Invalid second level segment filename template '%s', "
733
+                                "you can try to remove second_level_segment_size flag\n",
734
+                               filename);
735
+                        av_free(filename);
736
+                        return AVERROR(EINVAL);
737
+                    }
738
+                    av_free(filename);
739
+                }
740
+                if (c->flags & HLS_SECOND_LEVEL_SEGMENT_DURATION) {
741
+                    char * filename = av_strdup(oc->filename);  // %%t will be %t after strftime
742
+                    if (!filename)
743
+                        return AVERROR(ENOMEM);
744
+                    if (replace_int_data_in_filename(oc->filename, sizeof(oc->filename), filename, 't', 0) < 1) {
745
+                        av_log(c, AV_LOG_ERROR,
746
+                               "Invalid second level segment filename template '%s', "
747
+                                "you can try to remove second_level_segment_time flag\n",
748
+                               filename);
749
+                        av_free(filename);
750
+                        return AVERROR(EINVAL);
751
+                    }
752
+                    av_free(filename);
753
+                }
754
+            }
721 755
             if (c->use_localtime_mkdir) {
722 756
                 const char *dir;
723 757
                 char *fn_copy = av_strdup(oc->filename);
... ...
@@ -832,6 +962,7 @@ static int hls_write_header(AVFormatContext *s)
832 832
     hls->sequence       = hls->start_sequence;
833 833
     hls->recording_time = (hls->init_time ? hls->init_time : hls->time) * AV_TIME_BASE;
834 834
     hls->start_pts      = AV_NOPTS_VALUE;
835
+    hls->current_segment_final_filename_fmt[0] = '\0';
835 836
 
836 837
     if (hls->flags & HLS_PROGRAM_DATE_TIME) {
837 838
         time_t now0;
... ...
@@ -906,10 +1037,41 @@ static int hls_write_header(AVFormatContext *s)
906 906
             av_strlcat(hls->basename, pattern, basename_size);
907 907
         }
908 908
     }
909
-    if (!hls->use_localtime && (hls->flags & HLS_SECOND_LEVEL_SEGMENT_INDEX)) {
910
-        av_log(hls, AV_LOG_ERROR, "second_level_segment_index hls_flag requires use_localtime to be true\n");
911
-        ret = AVERROR(EINVAL);
912
-        goto fail;
909
+    if (!hls->use_localtime) {
910
+        if (hls->flags & HLS_SECOND_LEVEL_SEGMENT_DURATION) {
911
+             av_log(hls, AV_LOG_ERROR,
912
+                    "second_level_segment_duration hls_flag requires use_localtime to be true\n");
913
+             ret = AVERROR(EINVAL);
914
+             goto fail;
915
+        }
916
+        if (hls->flags & HLS_SECOND_LEVEL_SEGMENT_SIZE) {
917
+             av_log(hls, AV_LOG_ERROR,
918
+                    "second_level_segment_size hls_flag requires use_localtime to be true\n");
919
+             ret = AVERROR(EINVAL);
920
+             goto fail;
921
+        }
922
+        if (hls->flags & HLS_SECOND_LEVEL_SEGMENT_INDEX) {
923
+            av_log(hls, AV_LOG_ERROR,
924
+                   "second_level_segment_index hls_flag requires use_localtime to be true\n");
925
+            ret = AVERROR(EINVAL);
926
+            goto fail;
927
+        }
928
+    } else {
929
+        const char *proto = avio_find_protocol_name(hls->basename);
930
+        int segment_renaming_ok = proto && !strcmp(proto, "file");
931
+
932
+        if ((hls->flags & HLS_SECOND_LEVEL_SEGMENT_DURATION) && !segment_renaming_ok) {
933
+             av_log(hls, AV_LOG_ERROR,
934
+                    "second_level_segment_duration hls_flag works only with file protocol segment names\n");
935
+             ret = AVERROR(EINVAL);
936
+             goto fail;
937
+        }
938
+        if ((hls->flags & HLS_SECOND_LEVEL_SEGMENT_SIZE) && !segment_renaming_ok) {
939
+             av_log(hls, AV_LOG_ERROR,
940
+                    "second_level_segment_size hls_flag works only with file protocol segment names\n");
941
+             ret = AVERROR(EINVAL);
942
+             goto fail;
943
+        }
913 944
     }
914 945
     if(hls->has_subtitle) {
915 946
 
... ...
@@ -1167,6 +1329,8 @@ static const AVOption options[] = {
1167 1167
     {"append_list", "append the new segments into old hls segment list", 0, AV_OPT_TYPE_CONST, {.i64 = HLS_APPEND_LIST }, 0, UINT_MAX,   E, "flags"},
1168 1168
     {"program_date_time", "add EXT-X-PROGRAM-DATE-TIME", 0, AV_OPT_TYPE_CONST, {.i64 = HLS_PROGRAM_DATE_TIME }, 0, UINT_MAX,   E, "flags"},
1169 1169
     {"second_level_segment_index", "include segment index in segment filenames when use_localtime", 0, AV_OPT_TYPE_CONST, {.i64 = HLS_SECOND_LEVEL_SEGMENT_INDEX }, 0, UINT_MAX,   E, "flags"},
1170
+    {"second_level_segment_duration", "include segment duration in segment filenames when use_localtime", 0, AV_OPT_TYPE_CONST, {.i64 = HLS_SECOND_LEVEL_SEGMENT_DURATION }, 0, UINT_MAX,   E, "flags"},
1171
+    {"second_level_segment_size", "include segment size in segment filenames when use_localtime", 0, AV_OPT_TYPE_CONST, {.i64 = HLS_SECOND_LEVEL_SEGMENT_SIZE }, 0, UINT_MAX,   E, "flags"},
1170 1172
     {"use_localtime", "set filename expansion with strftime at segment creation", OFFSET(use_localtime), AV_OPT_TYPE_BOOL, {.i64 = 0 }, 0, 1, E },
1171 1173
     {"use_localtime_mkdir", "create last directory component in strftime-generated filename", OFFSET(use_localtime_mkdir), AV_OPT_TYPE_BOOL, {.i64 = 0 }, 0, 1, E },
1172 1174
     {"hls_playlist_type", "set the HLS playlist type", OFFSET(pl_type), AV_OPT_TYPE_INT, {.i64 = PLAYLIST_TYPE_NONE }, 0, PLAYLIST_TYPE_NB-1, E, "pl_type" },