1. ffmpeg
  2. Commit 5e3a418b6047acd848698c4bb4bf0c1b73526744
Browse code

add signature filter for MPEG7 video signature

This filter does not implement all features of MPEG7. Missing features:
- compression of signature files
- work only on (cropped) parts of the video

Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

Gerion Entrup authored on 2017/01/02 10:08:57
Showing 9 changed files
Changelog
History View file @ 5e3a418
... ... 
@@ -30,6 +30,7 @@ version <next>:
30 30 
 - Support MOV with multiple sample description tables
31 31 
 - XPM decoder
32 32 
 - Removed the legacy X11 screen grabber, use XCB instead
33 
+- MPEG-7 Video Signature filter
33 34
34 35
35 36 
 version 3.2:
configure
History View file @ 5e3a418
... ... 
@@ -3139,6 +3139,7 @@ showspectrum_filter_deps="avcodec"
3139 3139 
 showspectrum_filter_select="fft"
3140 3140 
 showspectrumpic_filter_deps="avcodec"
3141 3141 
 showspectrumpic_filter_select="fft"
3142 
+signature_filter_deps="gpl avcodec avformat"
3142 3143 
 smartblur_filter_deps="gpl swscale"
3143 3144 
 sofalizer_filter_deps="netcdf avcodec"
3144 3145 
 sofalizer_filter_select="fft"
doc/filters.texi
History View file @ 5e3a418
... ... 
@@ -12660,6 +12660,95 @@ saturation maximum: %@{metadata:lavfi.signalstats.SATMAX@}
12660 12660 
 @end example
12661 12661 
 @end itemize
12662 12662
12663 
+@anchor{signature}
12664 
+@section signature
12665 
+
12666 
+Calculates the MPEG-7 Video Signature. The filter can handle more than one
12667 
+input. In this case the matching between the inputs can be calculated additionally.
12668 
+The filter always passes through the first input. The signature of each stream can
12669 
+be written into a file.
12670 
+
12671 
+It accepts the following options:
12672 
+
12673 
+@table @option
12674 
+@item detectmode
12675 
+Enable or disable the matching process.
12676 
+
12677 
+Available values are:
12678 
+
12679 
+@table @samp
12680 
+@item off
12681 
+Disable the calculation of a matching (default).
12682 
+@item full
12683 
+Calculate the matching for the whole video and output whether the whole video
12684 
+matches or only parts.
12685 
+@item fast
12686 
+Calculate only until a matching is found or the video ends. Should be faster in
12687 
+some cases.
12688 
+@end table
12689 
+
12690 
+@item nb_inputs
12691 
+Set the number of inputs. The option value must be a non negative integer.
12692 
+Default value is 1.
12693 
+
12694 
+@item filename
12695 
+Set the path to which the output is written. If there is more than one input,
12696 
+the path must be a prototype, i.e. must contain %d or %0nd (where n is a positive
12697 
+integer), that will be replaced with the input number. If no filename is
12698 
+specified, no output will be written. This is the default.
12699 
+
12700 
+@item format
12701 
+Choose the output format.
12702 
+
12703 
+Available values are:
12704 
+
12705 
+@table @samp
12706 
+@item binary
12707 
+Use the specified binary representation (default).
12708 
+@item xml
12709 
+Use the specified xml representation.
12710 
+@end table
12711 
+
12712 
+@item th_d
12713 
+Set threshold to detect one word as similar. The option value must be an integer
12714 
+greater than zero. The default value is 9000.
12715 
+
12716 
+@item th_dc
12717 
+Set threshold to detect all words as similar. The option value must be an integer
12718 
+greater than zero. The default value is 60000.
12719 
+
12720 
+@item th_xh
12721 
+Set threshold to detect frames as similar. The option value must be an integer
12722 
+greater than zero. The default value is 116.
12723 
+
12724 
+@item th_di
12725 
+Set the minimum length of a sequence in frames to recognize it as matching
12726 
+sequence. The option value must be a non negative integer value.
12727 
+The default value is 0.
12728 
+
12729 
+@item th_it
12730 
+Set the minimum relation, that matching frames to all frames must have.
12731 
+The option value must be a double value between 0 and 1. The default value is 0.5.
12732 
+@end table
12733 
+
12734 
+@subsection Examples
12735 
+
12736 
+@itemize
12737 
+@item
12738 
+To calculate the signature of an input video and store it in signature.bin:
12739 
+@example
12740 
+ffmpeg -i input.mkv -vf signature=filename=signature.bin -map 0:v -f null -
12741 
+@end example
12742 
+
12743 
+@item
12744 
+To detect whether two videos match and store the signatures in XML format in
12745 
+signature0.xml and signature1.xml:
12746 
+@example
12747 
+ffmpeg -i input1.mkv -i input2.mkv -filter_complex "[0:v][1:v] signature=nb_inputs=2:detectmode=full:format=xml:filename=signature%d.xml" -map :v -f null -
12748 
+@end example
12749 
+
12750 
+@end itemize
12751 
+
12663 12752 
 @anchor{smartblur}
12664 12753 
 @section smartblur
12665 12754
libavfilter/Makefile
History View file @ 5e3a418
... ... 
@@ -280,6 +280,7 @@ OBJS-$(CONFIG_SHUFFLEFRAMES_FILTER)          += vf_shuffleframes.o
280 280 
 OBJS-$(CONFIG_SHUFFLEPLANES_FILTER)          += vf_shuffleplanes.o
281 281 
 OBJS-$(CONFIG_SIDEDATA_FILTER)               += f_sidedata.o
282 282 
 OBJS-$(CONFIG_SIGNALSTATS_FILTER)            += vf_signalstats.o
283 
+OBJS-$(CONFIG_SIGNATURE_FILTER)              += vf_signature.o
283 284 
 OBJS-$(CONFIG_SMARTBLUR_FILTER)              += vf_smartblur.o
284 285 
 OBJS-$(CONFIG_SOBEL_FILTER)                  += vf_convolution.o
285 286 
 OBJS-$(CONFIG_SPLIT_FILTER)                  += split.o
libavfilter/allfilters.c
History View file @ 5e3a418
... ... 
@@ -290,6 +290,7 @@ static void register_all(void)
290 290 
     REGISTER_FILTER(SHUFFLEPLANES,  shuffleplanes,  vf);
291 291 
     REGISTER_FILTER(SIDEDATA,       sidedata,       vf);
292 292 
     REGISTER_FILTER(SIGNALSTATS,    signalstats,    vf);
293 
+    REGISTER_FILTER(SIGNATURE,      signature,      vf);
293 294 
     REGISTER_FILTER(SMARTBLUR,      smartblur,      vf);
294 295 
     REGISTER_FILTER(SOBEL,          sobel,          vf);
295 296 
     REGISTER_FILTER(SPLIT,          split,          vf);
libavfilter/signature.h
History View file @ 5e3a418
296 297 
new file mode 100644
... ... 
@@ -0,0 +1,569 @@
0 
+/*
1 
+ * Copyright (c) 2017 Gerion Entrup
2 
+ *
3 
+ * This file is part of FFmpeg.
4 
+ *
5 
+ * FFmpeg is free software; you can redistribute it and/or modify
6 
+ * it under the terms of the GNU General Public License as published by
7 
+ * the Free Software Foundation; either version 2 of the License, or
8 
+ * (at your option) any later version.
9 
+ *
10 
+ * FFmpeg is distributed in the hope that it will be useful,
11 
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13 
+ * GNU General Public License for more details.
14 
+ *
15 
+ * You should have received a copy of the GNU General Public License along
16 
+ * with FFmpeg; if not, write to the Free Software Foundation, Inc.,
17 
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
18 
+ */
19 
+
20 
+/**
21 
+ * @file
22 
+ * MPEG-7 video signature calculation and lookup filter
23 
+ */
24 
+
25 
+#ifndef AVFILTER_SIGNATURE_H
26 
+#define AVFILTER_SIGNATURE_H
27 
+
28 
+#include <float.h>
29 
+#include "libavutil/common.h"
30 
+#include "libavutil/opt.h"
31 
+#include "libavutil/timestamp.h"
32 
+#include "avfilter.h"
33 
+#include "internal.h"
34 
+
35 
+#define ELEMENT_COUNT 10
36 
+#define SIGELEM_SIZE 380
37 
+#define DIFFELEM_SIZE 348 /* SIGELEM_SIZE - elem_a1 - elem_a2 */
38 
+#define COARSE_SIZE 90
39 
+
40 
+enum lookup_mode {
41 
+    MODE_OFF,
42 
+    MODE_FULL,
43 
+    MODE_FAST,
44 
+    NB_LOOKUP_MODE
45 
+};
46 
+
47 
+enum formats {
48 
+    FORMAT_BINARY,
49 
+    FORMAT_XML,
50 
+    NB_FORMATS
51 
+};
52 
+
53 
+typedef struct {
54 
+    uint8_t x;
55 
+    uint8_t y;
56 
+} Point;
57 
+
58 
+typedef struct {
59 
+    Point up;
60 
+    Point to;
61 
+} Block;
62 
+
63 
+typedef struct {
64 
+    int av_elem; /* average element category */
65 
+    short left_count; /* count of blocks that will be added together */
66 
+    short block_count; /* count of blocks per element */
67 
+    short elem_count;
68 
+    const Block* blocks;
69 
+} ElemCat;
70 
+
71 
+typedef struct FineSignature {
72 
+    struct FineSignature* next;
73 
+    struct FineSignature* prev;
74 
+    uint64_t pts;
75 
+    uint32_t index; /* needed for xmlexport */
76 
+    uint8_t confidence;
77 
+    uint8_t words[5];
78 
+    uint8_t framesig[SIGELEM_SIZE/5];
79 
+} FineSignature;
80 
+
81 
+typedef struct CoarseSignature {
82 
+    uint8_t data[5][31]; /* 5 words with min. 243 bit */
83 
+    struct FineSignature* first; /* associated Finesignatures */
84 
+    struct FineSignature* last;
85 
+    struct CoarseSignature* next;
86 
+} CoarseSignature;
87 
+
88 
+/* lookup types */
89 
+typedef struct MatchingInfo {
90 
+    double meandist;
91 
+    double framerateratio; /* second/first */
92 
+    int score;
93 
+    int offset;
94 
+    int matchframes; /* number of matching frames */
95 
+    int whole;
96 
+    struct FineSignature* first;
97 
+    struct FineSignature* second;
98 
+    struct MatchingInfo* next;
99 
+} MatchingInfo;
100 
+
101 
+typedef struct {
102 
+    AVRational time_base;
103 
+    /* needed for xml_export */
104 
+    int w; /* height */
105 
+    int h; /* width */
106 
+
107 
+    /* overflow protection */
108 
+    int divide;
109 
+
110 
+    FineSignature* finesiglist;
111 
+    FineSignature* curfinesig;
112 
+
113 
+    CoarseSignature* coarsesiglist;
114 
+    CoarseSignature* coarseend; /* needed for xml export */
115 
+    /* helpers to store the alternating signatures */
116 
+    CoarseSignature* curcoarsesig1;
117 
+    CoarseSignature* curcoarsesig2;
118 
+
119 
+    int coarsecount; /* counter from 0 to 89 */
120 
+    int midcoarse;   /* whether it is a coarsesignature beginning from 45 + i * 90 */
121 
+    uint32_t lastindex; /* helper to store amount of frames */
122 
+
123 
+    int exported; /* boolean whether stream already exported */
124 
+} StreamContext;
125 
+
126 
+typedef struct {
127 
+    const AVClass *class;
128 
+    /* input parameters */
129 
+    int mode;
130 
+    int nb_inputs;
131 
+    char *filename;
132 
+    int format;
133 
+    int thworddist;
134 
+    int thcomposdist;
135 
+    int thl1;
136 
+    int thdi;
137 
+    int thit;
138 
+    /* end input parameters */
139 
+
140 
+    uint8_t l1distlut[243*242/2]; /* 243 + 242 + 241 ... */
141 
+    StreamContext* streamcontexts;
142 
+} SignatureContext;
143 
+
144 
+
145 
+static const Block elem_a1_data[] = {
146 
+    {{ 0, 0},{ 7, 7}},
147 
+    {{ 8, 0},{15, 7}},
148 
+    {{ 0, 8},{ 7,15}},
149 
+    {{ 8, 8},{15,15}},
150 
+    {{16, 0},{23, 7}},
151 
+    {{24, 0},{31, 7}},
152 
+    {{16, 8},{23,15}},
153 
+    {{24, 8},{31,15}},
154 
+    {{ 0,16},{ 7,23}},
155 
+    {{ 8,16},{15,23}},
156 
+    {{ 0,24},{ 7,31}},
157 
+    {{ 8,24},{15,31}},
158 
+    {{16,16},{23,23}},
159 
+    {{24,16},{31,23}},
160 
+    {{16,24},{23,31}},
161 
+    {{24,24},{31,31}},
162 
+    {{ 0, 0},{15,15}},
163 
+    {{16, 0},{31,15}},
164 
+    {{ 0,16},{15,31}},
165 
+    {{16,16},{31,31}}
166 
+};
167 
+static const ElemCat elem_a1 = { 1, 1, 1, 20, elem_a1_data };
168 
+
169 
+static const Block elem_a2_data[] = {
170 
+    {{ 2, 2},{ 9, 9}},
171 
+    {{12, 2},{19, 9}},
172 
+    {{22, 2},{29, 9}},
173 
+    {{ 2,12},{ 9,19}},
174 
+    {{12,12},{19,19}},
175 
+    {{22,12},{29,19}},
176 
+    {{ 2,22},{ 9,29}},
177 
+    {{12,22},{19,29}},
178 
+    {{22,22},{29,29}},
179 
+    {{ 9, 9},{22,22}},
180 
+    {{ 6, 6},{25,25}},
181 
+    {{ 3, 3},{28,28}}
182 
+};
183 
+static const ElemCat elem_a2 = { 1, 1, 1, 12, elem_a2_data };
184 
+
185 
+static const Block elem_d1_data[] = {
186 
+    {{ 0, 0},{ 1, 3}},{{ 2, 0},{ 3, 3}},
187 
+    {{ 4, 0},{ 7, 1}},{{ 4, 2},{ 7, 3}},
188 
+    {{ 0, 6},{ 3, 7}},{{ 0, 4},{ 3, 5}},
189 
+    {{ 6, 4},{ 7, 7}},{{ 4, 4},{ 5, 7}},
190 
+    {{ 8, 0},{ 9, 3}},{{10, 0},{11, 3}},
191 
+    {{12, 0},{15, 1}},{{12, 2},{15, 3}},
192 
+    {{ 8, 6},{11, 7}},{{ 8, 4},{11, 5}},
193 
+    {{14, 4},{15, 7}},{{12, 4},{13, 7}},
194 
+    {{ 0, 8},{ 1,11}},{{ 2, 8},{ 3,11}},
195 
+    {{ 4, 8},{ 7, 9}},{{ 4,10},{ 7,11}},
196 
+    {{ 0,14},{ 3,15}},{{ 0,12},{ 3,13}},
197 
+    {{ 6,12},{ 7,15}},{{ 4,12},{ 5,15}},
198 
+    {{ 8, 8},{ 9,11}},{{10, 8},{11,11}},
199 
+    {{12, 8},{15, 9}},{{12,10},{15,11}},
200 
+    {{ 8,14},{11,15}},{{ 8,12},{11,13}},
201 
+    {{14,12},{15,15}},{{12,12},{13,15}},
202 
+    {{16, 0},{19, 1}},{{16, 2},{19, 3}},
203 
+    {{22, 0},{23, 3}},{{20, 0},{21, 3}},
204 
+    {{16, 4},{17, 7}},{{18, 4},{19, 7}},
205 
+    {{20, 6},{23, 7}},{{20, 4},{23, 5}},
206 
+    {{24, 0},{27, 1}},{{24, 2},{27, 3}},
207 
+    {{30, 0},{31, 3}},{{28, 0},{29, 3}},
208 
+    {{24, 4},{25, 7}},{{26, 4},{27, 7}},
209 
+    {{28, 6},{31, 7}},{{28, 4},{31, 5}},
210 
+    {{16, 8},{19, 9}},{{16,10},{19,11}},
211 
+    {{22, 8},{23,11}},{{20, 8},{21,11}},
212 
+    {{16,12},{17,15}},{{18,12},{19,15}},
213 
+    {{20,14},{23,15}},{{20,12},{23,13}},
214 
+    {{24, 8},{27, 9}},{{24,10},{27,11}},
215 
+    {{30, 8},{31,11}},{{28, 8},{29,11}},
216 
+    {{24,12},{25,15}},{{26,12},{27,15}},
217 
+    {{28,14},{31,15}},{{28,12},{31,13}},
218 
+    {{ 0,16},{ 3,17}},{{ 0,18},{ 3,19}},
219 
+    {{ 6,16},{ 7,19}},{{ 4,16},{ 5,19}},
220 
+    {{ 0,20},{ 1,23}},{{ 2,20},{ 3,23}},
221 
+    {{ 4,22},{ 7,23}},{{ 4,20},{ 7,21}},
222 
+    {{ 8,16},{11,17}},{{ 8,18},{11,19}},
223 
+    {{14,16},{15,19}},{{12,16},{13,19}},
224 
+    {{ 8,20},{ 9,23}},{{10,20},{11,23}},
225 
+    {{12,22},{15,23}},{{12,20},{15,21}},
226 
+    {{ 0,24},{ 3,25}},{{ 0,26},{ 3,27}},
227 
+    {{ 6,24},{ 7,27}},{{ 4,24},{ 5,27}},
228 
+    {{ 0,28},{ 1,31}},{{ 2,28},{ 3,31}},
229 
+    {{ 4,30},{ 7,31}},{{ 4,28},{ 7,29}},
230 
+    {{ 8,24},{11,25}},{{ 8,26},{11,27}},
231 
+    {{14,24},{15,27}},{{12,24},{13,27}},
232 
+    {{ 8,28},{ 9,31}},{{10,28},{11,31}},
233 
+    {{12,30},{15,31}},{{12,28},{15,29}},
234 
+    {{16,16},{17,19}},{{18,16},{19,19}},
235 
+    {{20,16},{23,17}},{{20,18},{23,19}},
236 
+    {{16,22},{19,23}},{{16,20},{19,21}},
237 
+    {{22,20},{23,23}},{{20,20},{21,23}},
238 
+    {{24,16},{25,19}},{{26,16},{27,19}},
239 
+    {{28,16},{31,17}},{{28,18},{31,19}},
240 
+    {{24,22},{27,23}},{{24,20},{27,21}},
241 
+    {{30,20},{31,23}},{{28,20},{29,23}},
242 
+    {{16,24},{17,27}},{{18,24},{19,27}},
243 
+    {{20,24},{23,25}},{{20,26},{23,27}},
244 
+    {{16,30},{19,31}},{{16,28},{19,29}},
245 
+    {{22,28},{23,31}},{{20,28},{21,31}},
246 
+    {{24,24},{25,27}},{{26,24},{27,27}},
247 
+    {{28,24},{31,25}},{{28,26},{31,27}},
248 
+    {{24,30},{27,31}},{{24,28},{27,29}},
249 
+    {{30,28},{31,31}},{{28,28},{29,31}},
250 
+    {{ 2, 2},{ 3, 5}},{{ 4, 2},{ 5, 5}},
251 
+    {{ 6, 2},{ 9, 3}},{{ 6, 4},{ 9, 5}},
252 
+    {{ 2, 8},{ 5, 9}},{{ 2, 6},{ 5, 7}},
253 
+    {{ 8, 6},{ 9, 9}},{{ 6, 6},{ 7, 9}},
254 
+    {{12, 2},{13, 5}},{{14, 2},{15, 5}},
255 
+    {{16, 2},{19, 3}},{{16, 4},{19, 5}},
256 
+    {{12, 8},{15, 9}},{{12, 6},{15, 7}},
257 
+    {{18, 6},{19, 9}},{{16, 6},{17, 9}},
258 
+    {{22, 2},{23, 5}},{{24, 2},{25, 5}},
259 
+    {{26, 2},{29, 3}},{{26, 4},{29, 5}},
260 
+    {{22, 8},{25, 9}},{{22, 6},{25, 7}},
261 
+    {{28, 6},{29, 9}},{{26, 6},{27, 9}},
262 
+    {{ 2,12},{ 3,15}},{{ 4,12},{ 5,15}},
263 
+    {{ 6,12},{ 9,13}},{{ 6,14},{ 9,15}},
264 
+    {{ 2,18},{ 5,19}},{{ 2,16},{ 5,17}},
265 
+    {{ 8,16},{ 9,19}},{{ 6,16},{ 7,19}},
266 
+    {{12,12},{15,13}},{{12,14},{15,15}},
267 
+    {{16,12},{19,13}},{{16,14},{19,15}},
268 
+    {{12,18},{15,19}},{{12,16},{15,17}},
269 
+    {{16,18},{19,19}},{{16,16},{19,17}},
270 
+    {{22,12},{23,15}},{{24,12},{25,15}},
271 
+    {{26,12},{29,13}},{{26,14},{29,15}},
272 
+    {{22,18},{25,19}},{{22,16},{25,17}},
273 
+    {{28,16},{29,19}},{{26,16},{27,19}},
274 
+    {{ 2,22},{ 3,25}},{{ 4,22},{ 5,25}},
275 
+    {{ 6,22},{ 9,23}},{{ 6,24},{ 9,25}},
276 
+    {{ 2,28},{ 5,29}},{{ 2,26},{ 5,27}},
277 
+    {{ 8,26},{ 9,29}},{{ 6,26},{ 7,29}},
278 
+    {{12,22},{13,25}},{{14,22},{15,25}},
279 
+    {{16,22},{19,23}},{{16,24},{19,25}},
280 
+    {{12,28},{15,29}},{{12,26},{15,27}},
281 
+    {{18,26},{19,29}},{{16,26},{17,29}},
282 
+    {{22,22},{23,25}},{{24,22},{25,25}},
283 
+    {{26,22},{29,23}},{{26,24},{29,25}},
284 
+    {{22,28},{25,29}},{{22,26},{25,27}},
285 
+    {{28,26},{29,29}},{{26,26},{27,29}},
286 
+    {{ 7, 7},{10, 8}},{{ 7, 9},{10,10}},
287 
+    {{11, 7},{12,10}},{{13, 7},{14,10}},
288 
+    {{ 7,11},{ 8,14}},{{ 9,11},{10,14}},
289 
+    {{11,11},{14,12}},{{11,13},{14,14}},
290 
+    {{17, 7},{20, 8}},{{17, 9},{20,10}},
291 
+    {{21, 7},{22,10}},{{23, 7},{24,10}},
292 
+    {{17,11},{18,14}},{{19,11},{20,14}},
293 
+    {{21,11},{24,12}},{{21,13},{24,14}},
294 
+    {{ 7,17},{10,18}},{{ 7,19},{10,20}},
295 
+    {{11,17},{12,20}},{{13,17},{14,20}},
296 
+    {{ 7,21},{ 8,24}},{{ 9,21},{10,24}},
297 
+    {{11,21},{14,22}},{{11,23},{14,24}},
298 
+    {{17,17},{20,18}},{{17,19},{20,20}},
299 
+    {{21,17},{22,20}},{{23,17},{24,20}},
300 
+    {{17,21},{18,24}},{{19,21},{20,24}},
301 
+    {{21,21},{24,22}},{{21,23},{24,24}}
302 
+};
303 
+static const ElemCat elem_d1 = { 0, 1, 2, 116, elem_d1_data };
304 
+
305 
+static const Block elem_d2_data[] = {
306 
+    {{ 0, 0},{ 3, 3}},{{ 4, 4},{ 7, 7}},{{ 4, 0},{ 7, 3}},{{ 0, 4},{ 3, 7}},
307 
+    {{ 8, 0},{11, 3}},{{12, 4},{15, 7}},{{12, 0},{15, 3}},{{ 8, 4},{11, 7}},
308 
+    {{16, 0},{19, 3}},{{20, 4},{23, 7}},{{20, 0},{23, 3}},{{16, 4},{19, 7}},
309 
+    {{24, 0},{27, 3}},{{28, 4},{31, 7}},{{28, 0},{31, 3}},{{24, 4},{27, 7}},
310 
+    {{ 0, 8},{ 3,11}},{{ 4,12},{ 7,15}},{{ 4, 8},{ 7,11}},{{ 0,12},{ 3,15}},
311 
+    {{ 8, 8},{11,11}},{{12,12},{15,15}},{{12, 8},{15,11}},{{ 8,12},{11,15}},
312 
+    {{16, 8},{19,11}},{{20,12},{23,15}},{{20, 8},{23,11}},{{16,12},{19,15}},
313 
+    {{24, 8},{27,11}},{{28,12},{31,15}},{{28, 8},{31,11}},{{24,12},{27,15}},
314 
+    {{ 0,16},{ 3,19}},{{ 4,20},{ 7,23}},{{ 4,16},{ 7,19}},{{ 0,20},{ 3,23}},
315 
+    {{ 8,16},{11,19}},{{12,20},{15,23}},{{12,16},{15,19}},{{ 8,20},{11,23}},
316 
+    {{16,16},{19,19}},{{20,20},{23,23}},{{20,16},{23,19}},{{16,20},{19,23}},
317 
+    {{24,16},{27,19}},{{28,20},{31,23}},{{28,16},{31,19}},{{24,20},{27,23}},
318 
+    {{ 0,24},{ 3,27}},{{ 4,28},{ 7,31}},{{ 4,24},{ 7,27}},{{ 0,28},{ 3,31}},
319 
+    {{ 8,24},{11,27}},{{12,28},{15,31}},{{12,24},{15,27}},{{ 8,28},{11,31}},
320 
+    {{16,24},{19,27}},{{20,28},{23,31}},{{20,24},{23,27}},{{16,28},{19,31}},
321 
+    {{24,24},{27,27}},{{28,28},{31,31}},{{28,24},{31,27}},{{24,28},{27,31}},
322 
+    {{ 4, 4},{ 7, 7}},{{ 8, 8},{11,11}},{{ 8, 4},{11, 7}},{{ 4, 8},{ 7,11}},
323 
+    {{12, 4},{15, 7}},{{16, 8},{19,11}},{{16, 4},{19, 7}},{{12, 8},{15,11}},
324 
+    {{20, 4},{23, 7}},{{24, 8},{27,11}},{{24, 4},{27, 7}},{{20, 8},{23,11}},
325 
+    {{ 4,12},{ 7,15}},{{ 8,16},{11,19}},{{ 8,12},{11,15}},{{ 4,16},{ 7,19}},
326 
+    {{12,12},{15,15}},{{16,16},{19,19}},{{16,12},{19,15}},{{12,16},{15,19}},
327 
+    {{20,12},{23,15}},{{24,16},{27,19}},{{24,12},{27,15}},{{20,16},{23,19}},
328 
+    {{ 4,20},{ 7,23}},{{ 8,24},{11,27}},{{ 8,20},{11,23}},{{ 4,24},{ 7,27}},
329 
+    {{12,20},{15,23}},{{16,24},{19,27}},{{16,20},{19,23}},{{12,24},{15,27}},
330 
+    {{20,20},{23,23}},{{24,24},{27,27}},{{24,20},{27,23}},{{20,24},{23,27}}
331 
+};
332 
+static const ElemCat elem_d2 = { 0, 2, 4, 25, elem_d2_data };
333 
+
334 
+static const Block elem_d3_data[] = {
335 
+    {{ 1, 1},{10,10}},{{11, 1},{20,10}},
336 
+    {{ 1, 1},{10,10}},{{21, 1},{30,10}},
337 
+    {{ 1, 1},{10,10}},{{ 1,11},{10,20}},
338 
+    {{ 1, 1},{10,10}},{{11,11},{20,20}},
339 
+    {{ 1, 1},{10,10}},{{21,11},{30,20}},
340 
+    {{ 1, 1},{10,10}},{{ 1,21},{10,30}},
341 
+    {{ 1, 1},{10,10}},{{11,21},{20,30}},
342 
+    {{ 1, 1},{10,10}},{{21,21},{30,30}},
343 
+    {{11, 1},{20,10}},{{21, 1},{30,10}},
344 
+    {{11, 1},{20,10}},{{ 1,11},{10,20}},
345 
+    {{11, 1},{20,10}},{{11,11},{20,20}},
346 
+    {{11, 1},{20,10}},{{21,11},{30,20}},
347 
+    {{11, 1},{20,10}},{{ 1,21},{10,30}},
348 
+    {{11, 1},{20,10}},{{11,21},{20,30}},
349 
+    {{11, 1},{20,10}},{{21,21},{30,30}},
350 
+    {{21, 1},{30,10}},{{ 1,11},{10,20}},
351 
+    {{21, 1},{30,10}},{{11,11},{20,20}},
352 
+    {{21, 1},{30,10}},{{21,11},{30,20}},
353 
+    {{21, 1},{30,10}},{{ 1,21},{10,30}},
354 
+    {{21, 1},{30,10}},{{11,21},{20,30}},
355 
+    {{21, 1},{30,10}},{{21,21},{30,30}},
356 
+    {{ 1,11},{10,20}},{{11,11},{20,20}},
357 
+    {{ 1,11},{10,20}},{{21,11},{30,20}},
358 
+    {{ 1,11},{10,20}},{{ 1,21},{10,30}},
359 
+    {{ 1,11},{10,20}},{{11,21},{20,30}},
360 
+    {{ 1,11},{10,20}},{{21,21},{30,30}},
361 
+    {{11,11},{20,20}},{{21,11},{30,20}},
362 
+    {{11,11},{20,20}},{{ 1,21},{10,30}},
363 
+    {{11,11},{20,20}},{{11,21},{20,30}},
364 
+    {{11,11},{20,20}},{{21,21},{30,30}},
365 
+    {{21,11},{30,20}},{{ 1,21},{10,30}},
366 
+    {{21,11},{30,20}},{{11,21},{20,30}},
367 
+    {{21,11},{30,20}},{{21,21},{30,30}},
368 
+    {{ 1,21},{10,30}},{{11,21},{20,30}},
369 
+    {{ 1,21},{10,30}},{{21,21},{30,30}},
370 
+    {{11,21},{20,30}},{{21,21},{30,30}}
371 
+};
372 
+static const ElemCat elem_d3 = { 0, 1, 2, 36, elem_d3_data };
373 
+
374 
+static const Block elem_d4_data[] = {
375 
+    {{ 7,13},{12,18}},{{19,13},{24,18}},
376 
+    {{13, 7},{18,12}},{{13,19},{18,24}},
377 
+    {{ 7, 7},{12,12}},{{19,19},{24,24}},
378 
+    {{19, 7},{24,12}},{{ 7,19},{12,24}},
379 
+    {{13, 7},{18,12}},{{19,13},{24,18}},
380 
+    {{19,13},{24,18}},{{13,19},{18,24}},
381 
+    {{13,19},{18,24}},{{ 7,13},{12,18}},
382 
+    {{ 7,13},{12,18}},{{13, 7},{18,12}},
383 
+    {{ 7, 7},{12,12}},{{19, 7},{24,12}},
384 
+    {{19, 7},{24,12}},{{19,19},{24,24}},
385 
+    {{19,19},{24,24}},{{ 7,19},{12,24}},
386 
+    {{ 7,19},{12,24}},{{ 7, 7},{12,12}},
387 
+    {{13,13},{18,18}},{{13, 1},{18, 6}},
388 
+    {{13,13},{18,18}},{{25,13},{30,18}},
389 
+    {{13,13},{18,18}},{{13,25},{18,30}},
390 
+    {{13,13},{18,18}},{{ 1,13},{ 6,18}},
391 
+    {{13, 1},{18, 6}},{{13,25},{18,30}},
392 
+    {{ 1,13},{ 6,18}},{{25,13},{30,18}},
393 
+    {{ 7, 1},{12, 6}},{{19, 1},{24, 6}},
394 
+    {{ 7,25},{12,30}},{{19,25},{24,30}},
395 
+    {{ 1, 7},{ 6,12}},{{ 1,19},{ 6,24}},
396 
+    {{25, 7},{30,12}},{{25,19},{30,24}},
397 
+    {{ 7, 1},{12, 6}},{{ 1, 7},{ 6,12}},
398 
+    {{19, 1},{24, 6}},{{25, 7},{30,12}},
399 
+    {{25,19},{30,24}},{{19,25},{24,30}},
400 
+    {{ 1,19},{ 6,24}},{{ 7,25},{12,30}},
401 
+    {{ 1, 1},{ 6, 6}},{{25, 1},{30, 6}},
402 
+    {{25, 1},{30, 6}},{{25,25},{30,30}},
403 
+    {{25,25},{30,30}},{{ 1,25},{ 6,30}},
404 
+    {{ 1,25},{ 6,30}},{{ 1, 1},{ 6, 6}}
405 
+};
406 
+static const ElemCat elem_d4 = { 0, 1, 2, 30, elem_d4_data };
407 
+
408 
+static const Block elem_d5_data[] = {
409 
+    {{ 1, 1},{10, 3}},{{ 1, 4},{ 3, 7}},{{ 8, 4},{10, 7}},{{ 1, 8},{10,10}},{{ 4, 4},{ 7, 7}},
410 
+    {{11, 1},{20, 3}},{{11, 4},{13, 7}},{{18, 4},{20, 7}},{{11, 8},{20,10}},{{14, 4},{17, 7}},
411 
+    {{21, 1},{30, 3}},{{21, 4},{23, 7}},{{28, 4},{30, 7}},{{21, 8},{30,10}},{{24, 4},{27, 7}},
412 
+    {{ 1,11},{10,13}},{{ 1,14},{ 3,17}},{{ 8,14},{10,17}},{{ 1,18},{10,20}},{{ 4,14},{ 7,17}},
413 
+    {{11,11},{20,13}},{{11,14},{13,17}},{{18,14},{20,17}},{{11,18},{20,20}},{{14,14},{17,17}},
414 
+    {{21,11},{30,13}},{{21,14},{23,17}},{{28,14},{30,17}},{{21,18},{30,20}},{{24,14},{27,17}},
415 
+    {{ 1,21},{10,23}},{{ 1,24},{ 3,27}},{{ 8,24},{10,27}},{{ 1,28},{10,30}},{{ 4,24},{ 7,27}},
416 
+    {{11,21},{20,23}},{{11,24},{13,27}},{{18,24},{20,27}},{{11,28},{20,30}},{{14,24},{17,27}},
417 
+    {{21,21},{30,23}},{{21,24},{23,27}},{{28,24},{30,27}},{{21,28},{30,30}},{{24,24},{27,27}},
418 
+    {{ 6, 6},{15, 8}},{{ 6, 9},{ 8,12}},{{13, 9},{15,12}},{{ 6,13},{15,15}},{{ 9, 9},{12,12}},
419 
+    {{16, 6},{25, 8}},{{16, 9},{18,12}},{{23, 9},{25,12}},{{16,13},{25,15}},{{19, 9},{22,12}},
420 
+    {{ 6,16},{15,18}},{{ 6,19},{ 8,22}},{{13,19},{15,22}},{{ 6,23},{15,25}},{{ 9,19},{12,22}},
421 
+    {{16,16},{25,18}},{{16,19},{18,22}},{{23,19},{25,22}},{{16,23},{25,25}},{{19,19},{22,22}},
422 
+    {{ 6, 1},{15, 3}},{{ 6, 4},{ 8, 7}},{{13, 4},{15, 7}},{{ 6, 8},{15,10}},{{ 9, 4},{12, 7}},
423 
+    {{16, 1},{25, 3}},{{16, 4},{18, 7}},{{23, 4},{25, 7}},{{16, 8},{25,10}},{{19, 4},{22, 7}},
424 
+    {{ 1, 6},{10, 8}},{{ 1, 9},{ 3,12}},{{ 8, 9},{10,12}},{{ 1,13},{10,15}},{{ 4, 9},{ 7,12}},
425 
+    {{11, 6},{20, 8}},{{11, 9},{13,12}},{{18, 9},{20,12}},{{11,13},{20,15}},{{14, 9},{17,12}},
426 
+    {{21, 6},{30, 8}},{{21, 9},{23,12}},{{28, 9},{30,12}},{{21,13},{30,15}},{{24, 9},{27,12}},
427 
+    {{ 6,11},{15,13}},{{ 6,14},{ 8,17}},{{13,14},{15,17}},{{ 6,18},{15,20}},{{ 9,14},{12,17}},
428 
+    {{16,11},{25,13}},{{16,14},{18,17}},{{23,14},{25,17}},{{16,18},{25,20}},{{19,14},{22,17}},
429 
+    {{ 1,16},{10,18}},{{ 1,19},{ 3,22}},{{ 8,19},{10,22}},{{ 1,23},{10,25}},{{ 4,19},{ 7,22}},
430 
+    {{11,16},{20,18}},{{11,19},{13,22}},{{18,19},{20,22}},{{11,23},{20,25}},{{14,19},{17,22}},
431 
+    {{21,16},{30,18}},{{21,19},{23,22}},{{28,19},{30,22}},{{21,23},{30,25}},{{24,19},{27,22}},
432 
+    {{ 6,21},{15,23}},{{ 6,24},{ 8,27}},{{13,24},{15,27}},{{ 6,28},{15,30}},{{ 9,24},{12,27}},
433 
+    {{16,21},{25,23}},{{16,24},{18,27}},{{23,24},{25,27}},{{16,28},{25,30}},{{19,24},{22,27}},
434 
+    {{ 2, 2},{14, 6}},{{ 2, 7},{ 6, 9}},{{10, 7},{14, 9}},{{ 2,10},{14,14}},{{ 7, 7},{ 9, 9}},
435 
+    {{ 7, 2},{19, 6}},{{ 7, 7},{11, 9}},{{15, 7},{19, 9}},{{ 7,10},{19,14}},{{12, 7},{14, 9}},
436 
+    {{12, 2},{24, 6}},{{12, 7},{16, 9}},{{20, 7},{24, 9}},{{12,10},{24,14}},{{17, 7},{19, 9}},
437 
+    {{17, 2},{29, 6}},{{17, 7},{21, 9}},{{25, 7},{29, 9}},{{17,10},{29,14}},{{22, 7},{24, 9}},
438 
+    {{ 2, 7},{14,11}},{{ 2,12},{ 6,14}},{{10,12},{14,14}},{{ 2,15},{14,19}},{{ 7,12},{ 9,14}},
439 
+    {{ 7, 7},{19,11}},{{ 7,12},{11,14}},{{15,12},{19,14}},{{ 7,15},{19,19}},{{12,12},{14,14}},
440 
+    {{12, 7},{24,11}},{{12,12},{16,14}},{{20,12},{24,14}},{{12,15},{24,19}},{{17,12},{19,14}},
441 
+    {{17, 7},{29,11}},{{17,12},{21,14}},{{25,12},{29,14}},{{17,15},{29,19}},{{22,12},{24,14}},
442 
+    {{ 2,12},{14,16}},{{ 2,17},{ 6,19}},{{10,17},{14,19}},{{ 2,20},{14,24}},{{ 7,17},{ 9,19}},
443 
+    {{ 7,12},{19,16}},{{ 7,17},{11,19}},{{15,17},{19,19}},{{ 7,20},{19,24}},{{12,17},{14,19}},
444 
+    {{12,12},{24,16}},{{12,17},{16,19}},{{20,17},{24,19}},{{12,20},{24,24}},{{17,17},{19,19}},
445 
+    {{17,12},{29,16}},{{17,17},{21,19}},{{25,17},{29,19}},{{17,20},{29,24}},{{22,17},{24,19}},
446 
+    {{ 2,17},{14,21}},{{ 2,22},{ 6,24}},{{10,22},{14,24}},{{ 2,25},{14,29}},{{ 7,22},{ 9,24}},
447 
+    {{ 7,17},{19,21}},{{ 7,22},{11,24}},{{15,22},{19,24}},{{ 7,25},{19,29}},{{12,22},{14,24}},
448 
+    {{12,17},{24,21}},{{12,22},{16,24}},{{20,22},{24,24}},{{12,25},{24,29}},{{17,22},{19,24}},
449 
+    {{17,17},{29,21}},{{17,22},{21,24}},{{25,22},{29,24}},{{17,25},{29,29}},{{22,22},{24,24}},
450 
+    {{ 8, 3},{13, 4}},{{ 8, 5},{ 9, 6}},{{12, 5},{13, 6}},{{ 8, 7},{13, 8}},{{10, 5},{11, 6}},
451 
+    {{13, 3},{18, 4}},{{13, 5},{14, 6}},{{17, 5},{18, 6}},{{13, 7},{18, 8}},{{15, 5},{16, 6}},
452 
+    {{18, 3},{23, 4}},{{18, 5},{19, 6}},{{22, 5},{23, 6}},{{18, 7},{23, 8}},{{20, 5},{21, 6}},
453 
+    {{ 3, 8},{ 8, 9}},{{ 3,10},{ 4,11}},{{ 7,10},{ 8,11}},{{ 3,12},{ 8,13}},{{ 5,10},{ 6,11}},
454 
+    {{ 8, 8},{13, 9}},{{ 8,10},{ 9,11}},{{12,10},{13,11}},{{ 8,12},{13,13}},{{10,10},{11,11}},
455 
+    {{13, 8},{18, 9}},{{13,10},{14,11}},{{17,10},{18,11}},{{13,12},{18,13}},{{15,10},{16,11}},
456 
+    {{18, 8},{23, 9}},{{18,10},{19,11}},{{22,10},{23,11}},{{18,12},{23,13}},{{20,10},{21,11}},
457 
+    {{23, 8},{28, 9}},{{23,10},{24,11}},{{27,10},{28,11}},{{23,12},{28,13}},{{25,10},{26,11}},
458 
+    {{ 3,13},{ 8,14}},{{ 3,15},{ 4,16}},{{ 7,15},{ 8,16}},{{ 3,17},{ 8,18}},{{ 5,15},{ 6,16}},
459 
+    {{ 8,13},{13,14}},{{ 8,15},{ 9,16}},{{12,15},{13,16}},{{ 8,17},{13,18}},{{10,15},{11,16}},
460 
+    {{13,13},{18,14}},{{13,15},{14,16}},{{17,15},{18,16}},{{13,17},{18,18}},{{15,15},{16,16}},
461 
+    {{18,13},{23,14}},{{18,15},{19,16}},{{22,15},{23,16}},{{18,17},{23,18}},{{20,15},{21,16}},
462 
+    {{23,13},{28,14}},{{23,15},{24,16}},{{27,15},{28,16}},{{23,17},{28,18}},{{25,15},{26,16}},
463 
+    {{ 3,18},{ 8,19}},{{ 3,20},{ 4,21}},{{ 7,20},{ 8,21}},{{ 3,22},{ 8,23}},{{ 5,20},{ 6,21}},
464 
+    {{ 8,18},{13,19}},{{ 8,20},{ 9,21}},{{12,20},{13,21}},{{ 8,22},{13,23}},{{10,20},{11,21}},
465 
+    {{13,18},{18,19}},{{13,20},{14,21}},{{17,20},{18,21}},{{13,22},{18,23}},{{15,20},{16,21}},
466 
+    {{18,18},{23,19}},{{18,20},{19,21}},{{22,20},{23,21}},{{18,22},{23,23}},{{20,20},{21,21}},
467 
+    {{23,18},{28,19}},{{23,20},{24,21}},{{27,20},{28,21}},{{23,22},{28,23}},{{25,20},{26,21}},
468 
+    {{ 8,23},{13,24}},{{ 8,25},{ 9,26}},{{12,25},{13,26}},{{ 8,27},{13,28}},{{10,25},{11,26}},
469 
+    {{13,23},{18,24}},{{13,25},{14,26}},{{17,25},{18,26}},{{13,27},{18,28}},{{15,25},{16,26}},
470 
+    {{18,23},{23,24}},{{18,25},{19,26}},{{22,25},{23,26}},{{18,27},{23,28}},{{20,25},{21,26}}
471 
+};
472 
+static const ElemCat elem_d5 = { 0, 4, 5, 62, elem_d5_data };
473 
+
474 
+static const Block elem_d6_data[] = {
475 
+    {{ 3, 5},{12,10}},{{ 5, 3},{10,12}},
476 
+    {{11, 5},{20,10}},{{13, 3},{18,12}},
477 
+    {{19, 5},{28,10}},{{21, 3},{26,12}},
478 
+    {{ 3,13},{12,18}},{{ 5,11},{10,20}},
479 
+    {{11,13},{20,18}},{{13,11},{18,20}},
480 
+    {{19,13},{28,18}},{{21,11},{26,20}},
481 
+    {{ 3,21},{12,26}},{{ 5,19},{10,28}},
482 
+    {{11,21},{20,26}},{{13,19},{18,28}},
483 
+    {{19,21},{28,26}},{{21,19},{26,28}}
484 
+};
485 
+static const ElemCat elem_d6 = { 0, 1, 2, 9, elem_d6_data };
486 
+
487 
+static const Block elem_d7_data[] = {
488 
+    {{ 0, 4},{ 3, 7}},{{ 8, 4},{11, 7}},{{ 4, 4},{ 7, 7}},
489 
+    {{ 4, 0},{ 7, 3}},{{ 4, 8},{ 7,11}},{{ 4, 4},{ 7, 7}},
490 
+    {{ 5, 4},{ 8, 7}},{{13, 4},{16, 7}},{{ 9, 4},{12, 7}},
491 
+    {{ 9, 0},{12, 3}},{{ 9, 8},{12,11}},{{ 9, 4},{12, 7}},
492 
+    {{10, 4},{13, 7}},{{18, 4},{21, 7}},{{14, 4},{17, 7}},
493 
+    {{14, 0},{17, 3}},{{14, 8},{17,11}},{{14, 4},{17, 7}},
494 
+    {{15, 4},{18, 7}},{{23, 4},{26, 7}},{{19, 4},{22, 7}},
495 
+    {{19, 0},{22, 3}},{{19, 8},{22,11}},{{19, 4},{22, 7}},
496 
+    {{20, 4},{23, 7}},{{28, 4},{31, 7}},{{24, 4},{27, 7}},
497 
+    {{24, 0},{27, 3}},{{24, 8},{27,11}},{{24, 4},{27, 7}},
498 
+    {{ 0, 9},{ 3,12}},{{ 8, 9},{11,12}},{{ 4, 9},{ 7,12}},
499 
+    {{ 4, 5},{ 7, 8}},{{ 4,13},{ 7,16}},{{ 4, 9},{ 7,12}},
500 
+    {{ 5, 9},{ 8,12}},{{13, 9},{16,12}},{{ 9, 9},{12,12}},
501 
+    {{ 9, 5},{12, 8}},{{ 9,13},{12,16}},{{ 9, 9},{12,12}},
502 
+    {{10, 9},{13,12}},{{18, 9},{21,12}},{{14, 9},{17,12}},
503 
+    {{14, 5},{17, 8}},{{14,13},{17,16}},{{14, 9},{17,12}},
504 
+    {{15, 9},{18,12}},{{23, 9},{26,12}},{{19, 9},{22,12}},
505 
+    {{19, 5},{22, 8}},{{19,13},{22,16}},{{19, 9},{22,12}},
506 
+    {{20, 9},{23,12}},{{28, 9},{31,12}},{{24, 9},{27,12}},
507 
+    {{24, 5},{27, 8}},{{24,13},{27,16}},{{24, 9},{27,12}},
508 
+    {{ 0,14},{ 3,17}},{{ 8,14},{11,17}},{{ 4,14},{ 7,17}},
509 
+    {{ 4,10},{ 7,13}},{{ 4,18},{ 7,21}},{{ 4,14},{ 7,17}},
510 
+    {{ 5,14},{ 8,17}},{{13,14},{16,17}},{{ 9,14},{12,17}},
511 
+    {{ 9,10},{12,13}},{{ 9,18},{12,21}},{{ 9,14},{12,17}},
512 
+    {{10,14},{13,17}},{{18,14},{21,17}},{{14,14},{17,17}},
513 
+    {{14,10},{17,13}},{{14,18},{17,21}},{{14,14},{17,17}},
514 
+    {{15,14},{18,17}},{{23,14},{26,17}},{{19,14},{22,17}},
515 
+    {{19,10},{22,13}},{{19,18},{22,21}},{{19,14},{22,17}},
516 
+    {{20,14},{23,17}},{{28,14},{31,17}},{{24,14},{27,17}},
517 
+    {{24,10},{27,13}},{{24,18},{27,21}},{{24,14},{27,17}},
518 
+    {{ 0,19},{ 3,22}},{{ 8,19},{11,22}},{{ 4,19},{ 7,22}},
519 
+    {{ 4,15},{ 7,18}},{{ 4,23},{ 7,26}},{{ 4,19},{ 7,22}},
520 
+    {{ 5,19},{ 8,22}},{{13,19},{16,22}},{{ 9,19},{12,22}},
521 
+    {{ 9,15},{12,18}},{{ 9,23},{12,26}},{{ 9,19},{12,22}},
522 
+    {{10,19},{13,22}},{{18,19},{21,22}},{{14,19},{17,22}},
523 
+    {{14,15},{17,18}},{{14,23},{17,26}},{{14,19},{17,22}},
524 
+    {{15,19},{18,22}},{{23,19},{26,22}},{{19,19},{22,22}},
525 
+    {{19,15},{22,18}},{{19,23},{22,26}},{{19,19},{22,22}},
526 
+    {{20,19},{23,22}},{{28,19},{31,22}},{{24,19},{27,22}},
527 
+    {{24,15},{27,18}},{{24,23},{27,26}},{{24,19},{27,22}},
528 
+    {{ 0,24},{ 3,27}},{{ 8,24},{11,27}},{{ 4,24},{ 7,27}},
529 
+    {{ 4,20},{ 7,23}},{{ 4,28},{ 7,31}},{{ 4,24},{ 7,27}},
530 
+    {{ 5,24},{ 8,27}},{{13,24},{16,27}},{{ 9,24},{12,27}},
531 
+    {{ 9,20},{12,23}},{{ 9,28},{12,31}},{{ 9,24},{12,27}},
532 
+    {{10,24},{13,27}},{{18,24},{21,27}},{{14,24},{17,27}},
533 
+    {{14,20},{17,23}},{{14,28},{17,31}},{{14,24},{17,27}},
534 
+    {{15,24},{18,27}},{{23,24},{26,27}},{{19,24},{22,27}},
535 
+    {{19,20},{22,23}},{{19,28},{22,31}},{{19,24},{22,27}},
536 
+    {{20,24},{23,27}},{{28,24},{31,27}},{{24,24},{27,27}},
537 
+    {{24,20},{27,23}},{{24,28},{27,31}},{{24,24},{27,27}}
538 
+};
539 
+static const ElemCat elem_d7 = { 0, 2, 3, 50, elem_d7_data };
540 
+
541 
+static const Block elem_d8_data[] = {
542 
+    {{ 0, 0},{ 7, 3}},{{ 0, 4},{ 7, 7}},
543 
+    {{ 8, 0},{11, 7}},{{12, 0},{15, 7}},
544 
+    {{ 0, 8},{ 3,15}},{{ 4, 8},{ 7,15}},
545 
+    {{ 8, 8},{15,11}},{{ 8,12},{15,15}},
546 
+    {{16, 0},{19, 7}},{{20, 0},{23, 7}},
547 
+    {{24, 0},{31, 3}},{{24, 4},{31, 7}},
548 
+    {{16, 8},{23,11}},{{16,12},{23,15}},
549 
+    {{24, 8},{27,15}},{{28, 8},{31,15}},
550 
+    {{ 0,16},{ 3,23}},{{ 4,16},{ 7,23}},
551 
+    {{ 8,16},{15,19}},{{ 8,20},{15,23}},
552 
+    {{ 0,24},{ 7,27}},{{ 0,28},{ 7,31}},
553 
+    {{ 8,24},{11,31}},{{12,24},{15,31}},
554 
+    {{16,16},{23,19}},{{16,20},{23,23}},
555 
+    {{24,16},{27,23}},{{28,16},{31,23}},
556 
+    {{16,24},{19,31}},{{20,24},{23,31}},
557 
+    {{24,24},{31,27}},{{24,28},{31,31}},
558 
+    {{ 0, 0},{ 7,15}},{{ 8, 0},{15,15}},
559 
+    {{16, 0},{31, 7}},{{16, 8},{31,15}},
560 
+    {{ 0,16},{15,23}},{{ 0,24},{15,31}},
561 
+    {{16,16},{23,31}},{{24,16},{31,31}}
562 
+};
563 
+static const ElemCat elem_d8 = { 0, 1, 2, 20, elem_d8_data };
564 
+
565 
+static const ElemCat* elements[ELEMENT_COUNT] = { &elem_a1, &elem_a2,
566 
+                                                  &elem_d1, &elem_d2, &elem_d3, &elem_d4,
567 
+                                                  &elem_d5, &elem_d6, &elem_d7, &elem_d8 };
568 
+#endif /* AVFILTER_SIGNATURE_H */
libavfilter/signature_lookup.c
History View file @ 5e3a418
0 569 
new file mode 100644
... ... 
@@ -0,0 +1,573 @@
0 
+/*
1 
+ * Copyright (c) 2017 Gerion Entrup
2 
+ *
3 
+ * This file is part of FFmpeg.
4 
+ *
5 
+ * FFmpeg is free software; you can redistribute it and/or modify
6 
+ * it under the terms of the GNU General Public License as published by
7 
+ * the Free Software Foundation; either version 2 of the License, or
8 
+ * (at your option) any later version.
9 
+ *
10 
+ * FFmpeg is distributed in the hope that it will be useful,
11 
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13 
+ * GNU General Public License for more details.
14 
+ *
15 
+ * You should have received a copy of the GNU General Public License along
16 
+ * with FFmpeg; if not, write to the Free Software Foundation, Inc.,
17 
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
18 
+ */
19 
+
20 
+/**
21 
+ * @file
22 
+ * MPEG-7 video signature calculation and lookup filter
23 
+ */
24 
+
25 
+#include "signature.h"
26 
+
27 
+#define HOUGH_MAX_OFFSET 90
28 
+#define MAX_FRAMERATE 60
29 
+
30 
+#define DIR_PREV 0
31 
+#define DIR_NEXT 1
32 
+#define DIR_PREV_END 2
33 
+#define DIR_NEXT_END 3
34 
+
35 
+#define STATUS_NULL 0
36 
+#define STATUS_END_REACHED 1
37 
+#define STATUS_BEGIN_REACHED 2
38 
+
39 
+static void fill_l1distlut(uint8_t lut[])
40 
+{
41 
+    int i, j, tmp_i, tmp_j,count;
42 
+    uint8_t dist;
43 
+
44 
+    for (i = 0, count = 0; i < 242; i++) {
45 
+        for (j = i + 1; j < 243; j++, count++) {
46 
+            /* ternary distance between i and j */
47 
+            dist = 0;
48 
+            tmp_i = i; tmp_j = j;
49 
+            do {
50 
+                dist += FFABS((tmp_j % 3) - (tmp_i % 3));
51 
+                tmp_j /= 3;
52 
+                tmp_i /= 3;
53 
+            } while (tmp_i > 0 || tmp_j > 0);
54 
+            lut[count] = dist;
55 
+        }
56 
+    }
57 
+}
58 
+
59 
+static unsigned int intersection_word(const uint8_t *first, const uint8_t *second)
60 
+{
61 
+    unsigned int val=0,i;
62 
+    for (i = 0; i < 28; i += 4) {
63 
+        val += av_popcount( (first[i]   & second[i]  ) << 24 |
64 
+                            (first[i+1] & second[i+1]) << 16 |
65 
+                            (first[i+2] & second[i+2]) << 8  |
66 
+                            (first[i+3] & second[i+3]) );
67 
+    }
68 
+    val += av_popcount( (first[28] & second[28]) << 16 |
69 
+                        (first[29] & second[29]) << 8  |
70 
+                        (first[30] & second[30]) );
71 
+    return val;
72 
+}
73 
+
74 
+static unsigned int union_word(const uint8_t *first, const uint8_t *second)
75 
+{
76 
+    unsigned int val=0,i;
77 
+    for (i = 0; i < 28; i += 4) {
78 
+        val += av_popcount( (first[i]   | second[i]  ) << 24 |
79 
+                            (first[i+1] | second[i+1]) << 16 |
80 
+                            (first[i+2] | second[i+2]) << 8  |
81 
+                            (first[i+3] | second[i+3]) );
82 
+    }
83 
+    val += av_popcount( (first[28] | second[28]) << 16 |
84 
+                        (first[29] | second[29]) << 8  |
85 
+                        (first[30] | second[30]) );
86 
+    return val;
87 
+}
88 
+
89 
+static unsigned int get_l1dist(AVFilterContext *ctx, SignatureContext *sc, const uint8_t *first, const uint8_t *second)
90 
+{
91 
+    unsigned int i;
92 
+    unsigned int dist = 0;
93 
+    uint8_t f, s;
94 
+
95 
+    for (i = 0; i < SIGELEM_SIZE/5; i++) {
96 
+        if (first[i] != second[i]) {
97 
+            f = first[i];
98 
+            s = second[i];
99 
+            if (f > s) {
100 
+                /* little variation of gauss sum formula */
101 
+                dist += sc->l1distlut[243*242/2 - (243-s)*(242-s)/2 + f - s - 1];
102 
+            } else {
103 
+                dist += sc->l1distlut[243*242/2 - (243-f)*(242-f)/2 + s - f - 1];
104 
+            }
105 
+        }
106 
+    }
107 
+    return dist;
108 
+}
109 
+
110 
+/**
111 
+ * calculates the jaccard distance and evaluates a pair of coarse signatures as good
112 
+ * @return 0 if pair is bad, 1 otherwise
113 
+ */
114 
+static int get_jaccarddist(SignatureContext *sc, CoarseSignature *first, CoarseSignature *second)
115 
+{
116 
+    int jaccarddist, i, composdist = 0, cwthcount = 0;
117 
+    for (i = 0; i < 5; i++) {
118 
+        if ((jaccarddist = intersection_word(first->data[i], second->data[i])) > 0) {
119 
+            jaccarddist /= union_word(first->data[i], second->data[i]);
120 
+        }
121 
+        if (jaccarddist >= sc->thworddist) {
122 
+            if (++cwthcount > 2) {
123 
+                /* more than half (5/2) of distances are too wide */
124 
+                return 0;
125 
+            }
126 
+        }
127 
+        composdist += jaccarddist;
128 
+        if (composdist > sc->thcomposdist) {
129 
+            return 0;
130 
+        }
131 
+    }
132 
+    return 1;
133 
+}
134 
+
135 
+/**
136 
+ * step through the coarsesignatures as long as a good candidate is found
137 
+ * @return 0 if no candidate is found, 1 otherwise
138 
+ */
139 
+static int find_next_coarsecandidate(SignatureContext *sc, CoarseSignature *secondstart, CoarseSignature **first, CoarseSignature **second, int start)
140 
+{
141 
+    /* go one coarsesignature foreword */
142 
+    if (!start) {
143 
+        if ((*second)->next) {
144 
+            *second = (*second)->next;
145 
+        } else if ((*first)->next) {
146 
+            *second = secondstart;
147 
+            *first = (*first)->next;
148 
+        } else {
149 
+            return 0;
150 
+        }
151 
+    }
152 
+
153 
+    while (1) {
154 
+        if (get_jaccarddist(sc, *first, *second))
155 
+            return 1;
156 
+
157 
+        /* next signature */
158 
+        if ((*second)->next) {
159 
+            *second = (*second)->next;
160 
+        } else if ((*first)->next) {
161 
+            *second = secondstart;
162 
+            *first = (*first)->next;
163 
+        } else {
164 
+            return 0;
165 
+        }
166 
+    }
167 
+}
168 
+
169 
+/**
170 
+ * compares framesignatures and sorts out signatures with a l1 distance above a given threshold.
171 
+ * Then tries to find out offset and differences between framerates with a hough transformation
172 
+ */
173 
+static MatchingInfo* get_matching_parameters(AVFilterContext *ctx, SignatureContext *sc, FineSignature *first, FineSignature *second)
174 
+{
175 
+    FineSignature *f, *s;
176 
+    size_t i, j, k, l, hmax = 0, score;
177 
+    int framerate, offset, l1dist;
178 
+    double m;
179 
+    MatchingInfo *cands = NULL, *c = NULL;
180 
+
181 
+    struct {
182 
+        uint8_t size;
183 
+        unsigned int dist;
184 
+        FineSignature *a;
185 
+        uint8_t b_pos[COARSE_SIZE];
186 
+        FineSignature *b[COARSE_SIZE];
187 
+    } pairs[COARSE_SIZE];
188 
+
189 
+    typedef struct {
190 
+        int dist;
191 
+        size_t score;
192 
+        FineSignature *a;
193 
+        FineSignature *b;
194 
+    } hspace_elem;
195 
+
196 
+    /* houghspace */
197 
+    hspace_elem** hspace = av_malloc_array(MAX_FRAMERATE, sizeof(hspace_elem *));
198 
+
199 
+    /* initialize houghspace */
200 
+    for (i = 0; i < MAX_FRAMERATE; i++) {
201 
+        hspace[i] = av_malloc_array(2 * HOUGH_MAX_OFFSET + 1, sizeof(hspace_elem));
202 
+        for (j = 0; j < HOUGH_MAX_OFFSET; j++) {
203 
+            hspace[i][j].score = 0;
204 
+            hspace[i][j].dist = 99999;
205 
+        }
206 
+    }
207 
+
208 
+    /* l1 distances */
209 
+    for (i = 0, f = first; i < COARSE_SIZE && f->next; i++, f = f->next) {
210 
+        pairs[i].size = 0;
211 
+        pairs[i].dist = 99999;
212 
+        pairs[i].a = f;
213 
+        for (j = 0, s = second; j < COARSE_SIZE && s->next; j++, s = s->next) {
214 
+            /* l1 distance of finesignature */
215 
+            l1dist = get_l1dist(ctx, sc, f->framesig, s->framesig);
216 
+            if (l1dist < sc->thl1) {
217 
+                if (l1dist < pairs[i].dist) {
218 
+                    pairs[i].size = 1;
219 
+                    pairs[i].dist = l1dist;
220 
+                    pairs[i].b_pos[0] = j;
221 
+                    pairs[i].b[0] = s;
222 
+                } else if (l1dist == pairs[i].dist) {
223 
+                    pairs[i].b[pairs[i].size] = s;
224 
+                    pairs[i].b_pos[pairs[i].size] = j;
225 
+                    pairs[i].size++;
226 
+                }
227 
+            }
228 
+        }
229 
+    }
230 
+    /* last incomplete coarsesignature */
231 
+    if (f->next == NULL) {
232 
+        for (; i < COARSE_SIZE; i++) {
233 
+            pairs[i].size = 0;
234 
+            pairs[i].dist = 99999;
235 
+        }
236 
+    }
237 
+
238 
+    /* hough transformation */
239 
+    for (i = 0; i < COARSE_SIZE; i++) {
240 
+        for (j = 0; j < pairs[i].size; j++) {
241 
+            for (k = i + 1; k < COARSE_SIZE; k++) {
242 
+                for (l = 0; l < pairs[k].size; l++) {
243 
+                    if (pairs[i].b[j] != pairs[k].b[l]) {
244 
+                        /* linear regression */
245 
+                        m = (pairs[k].b_pos[l]-pairs[i].b_pos[j]) / (k-i); /* good value between 0.0 - 2.0 */
246 
+                        framerate = (int) m*30 + 0.5; /* round up to 0 - 60 */
247 
+                        if (framerate>0 && framerate <= MAX_FRAMERATE) {
248 
+                            offset = pairs[i].b_pos[j] - ((int) m*i + 0.5); /* only second part has to be rounded up */
249 
+                            if (offset > -HOUGH_MAX_OFFSET && offset < HOUGH_MAX_OFFSET) {
250 
+                                if (pairs[i].dist < pairs[k].dist) {
251 
+                                    if (pairs[i].dist < hspace[framerate-1][offset+HOUGH_MAX_OFFSET].dist) {
252 
+                                        hspace[framerate-1][offset+HOUGH_MAX_OFFSET].dist = pairs[i].dist;
253 
+                                        hspace[framerate-1][offset+HOUGH_MAX_OFFSET].a = pairs[i].a;
254 
+                                        hspace[framerate-1][offset+HOUGH_MAX_OFFSET].b = pairs[i].b[j];
255 
+                                    }
256 
+                                } else {
257 
+                                    if (pairs[k].dist < hspace[framerate-1][offset+HOUGH_MAX_OFFSET].dist) {
258 
+                                        hspace[framerate-1][offset+HOUGH_MAX_OFFSET].dist = pairs[k].dist;
259 
+                                        hspace[framerate-1][offset+HOUGH_MAX_OFFSET].a = pairs[k].a;
260 
+                                        hspace[framerate-1][offset+HOUGH_MAX_OFFSET].b = pairs[k].b[l];
261 
+                                    }
262 
+                                }
263 
+
264 
+                                score = hspace[framerate-1][offset+HOUGH_MAX_OFFSET].score + 1;
265 
+                                if (score > hmax )
266 
+                                    hmax = score;
267 
+                                hspace[framerate-1][offset+HOUGH_MAX_OFFSET].score = score;
268 
+                            }
269 
+                        }
270 
+                    }
271 
+                }
272 
+            }
273 
+        }
274 
+    }
275 
+
276 
+    if (hmax > 0) {
277 
+        hmax = (int) (0.7*hmax);
278 
+        for (i = 0; i < MAX_FRAMERATE; i++) {
279 
+            for (j = 0; j < HOUGH_MAX_OFFSET; j++) {
280 
+                if (hmax < hspace[i][j].score) {
281 
+                    if (c == NULL) {
282 
+                        c = av_malloc(sizeof(MatchingInfo));
283 
+                        if (!c)
284 
+                            av_log(ctx, AV_LOG_FATAL, "Could not allocate memory");
285 
+                        cands = c;
286 
+                    } else {
287 
+                        c->next = av_malloc(sizeof(MatchingInfo));
288 
+                        if (!c->next)
289 
+                            av_log(ctx, AV_LOG_FATAL, "Could not allocate memory");
290 
+                        c = c->next;
291 
+                    }
292 
+                    c->framerateratio = (i+1.0) / 30;
293 
+                    c->score = hspace[i][j].score;
294 
+                    c->offset = j-90;
295 
+                    c->first = hspace[i][j].a;
296 
+                    c->second = hspace[i][j].b;
297 
+                    c->next = NULL;
298 
+
299 
+                    /* not used */
300 
+                    c->meandist = 0;
301 
+                    c->matchframes = 0;
302 
+                    c->whole = 0;
303 
+                }
304 
+            }
305 
+        }
306 
+    }
307 
+    for (i = 0; i < MAX_FRAMERATE; i++) {
308 
+        av_freep(&hspace[i]);
309 
+    }
310 
+    av_freep(&hspace);
311 
+    return cands;
312 
+}
313 
+
314 
+static int iterate_frame(double frr, FineSignature **a, FineSignature **b, int fcount, int *bcount, int dir)
315 
+{
316 
+    int step;
317 
+
318 
+    /* between 1 and 2, because frr is between 1 and 2 */
319 
+    step = ((int) 0.5 + fcount     * frr) /* current frame */
320 
+          -((int) 0.5 + (fcount-1) * frr);/* last frame */
321 
+
322 
+    if (dir == DIR_NEXT) {
323 
+        if (frr >= 1.0) {
324 
+            if ((*a)->next) {
325 
+                *a = (*a)->next;
326 
+            } else {
327 
+                return DIR_NEXT_END;
328 
+            }
329 
+
330 
+            if (step == 1) {
331 
+                if ((*b)->next) {
332 
+                    *b = (*b)->next;
333 
+                    (*bcount)++;
334 
+                } else {
335 
+                    return DIR_NEXT_END;
336 
+                }
337 
+            } else {
338 
+                if ((*b)->next && (*b)->next->next) {
339 
+                    *b = (*b)->next->next;
340 
+                    (*bcount)++;
341 
+                } else {
342 
+                    return DIR_NEXT_END;
343 
+                }
344 
+            }
345 
+        } else {
346 
+            if ((*b)->next) {
347 
+                *b = (*b)->next;
348 
+                (*bcount)++;
349 
+            } else {
350 
+                return DIR_NEXT_END;
351 
+            }
352 
+
353 
+            if (step == 1) {
354 
+                if ((*a)->next) {
355 
+                    *a = (*a)->next;
356 
+                } else {
357 
+                    return DIR_NEXT_END;
358 
+                }
359 
+            } else {
360 
+                if ((*a)->next && (*a)->next->next) {
361 
+                    *a = (*a)->next->next;
362 
+                } else {
363 
+                    return DIR_NEXT_END;
364 
+                }
365 
+            }
366 
+        }
367 
+        return DIR_NEXT;
368 
+    } else {
369 
+        if (frr >= 1.0) {
370 
+            if ((*a)->prev) {
371 
+                *a = (*a)->prev;
372 
+            } else {
373 
+                return DIR_PREV_END;
374 
+            }
375 
+
376 
+            if (step == 1) {
377 
+                if ((*b)->prev) {
378 
+                    *b = (*b)->prev;
379 
+                    (*bcount)++;
380 
+                } else {
381 
+                    return DIR_PREV_END;
382 
+                }
383 
+            } else {
384 
+                if ((*b)->prev && (*b)->prev->prev) {
385 
+                    *b = (*b)->prev->prev;
386 
+                    (*bcount)++;
387 
+                } else {
388 
+                    return DIR_PREV_END;
389 
+                }
390 
+            }
391 
+        } else {
392 
+            if ((*b)->prev) {
393 
+                *b = (*b)->prev;
394 
+                (*bcount)++;
395 
+            } else {
396 
+                return DIR_PREV_END;
397 
+            }
398 
+
399 
+            if (step == 1) {
400 
+                if ((*a)->prev) {
401 
+                    *a = (*a)->prev;
402 
+                } else {
403 
+                    return DIR_PREV_END;
404 
+                }
405 
+            } else {
406 
+                if ((*a)->prev && (*a)->prev->prev) {
407 
+                    *a = (*a)->prev->prev;
408 
+                } else {
409 
+                    return DIR_PREV_END;
410 
+                }
411 
+            }
412 
+        }
413 
+        return DIR_PREV;
414 
+    }
415 
+}
416 
+
417 
+static MatchingInfo evaluate_parameters(AVFilterContext *ctx, SignatureContext *sc, MatchingInfo *infos, MatchingInfo bestmatch, int mode)
418 
+{
419 
+    int dist, distsum = 0, bcount = 1, dir = DIR_NEXT;
420 
+    int fcount = 0, goodfcount = 0, gooda = 0, goodb = 0;
421 
+    double meandist, minmeandist = bestmatch.meandist;
422 
+    int tolerancecount = 0;
423 
+    FineSignature *a, *b, *aprev, *bprev;
424 
+    int status = STATUS_NULL;
425 
+
426 
+    for (; infos != NULL; infos = infos->next) {
427 
+        a = infos->first;
428 
+        b = infos->second;
429 
+        while (1) {
430 
+            dist = get_l1dist(ctx, sc, a->framesig, b->framesig);
431 
+
432 
+            if (dist > sc->thl1) {
433 
+                if (a->confidence >= 1 || b->confidence >= 1) {
434 
+                    /* bad frame (because high different information) */
435 
+                    tolerancecount++;
436 
+                }
437 
+
438 
+                if (tolerancecount > 2) {
439 
+                    a = aprev;
440 
+                    b = bprev;
441 
+                    if (dir == DIR_NEXT) {
442 
+                        /* turn around */
443 
+                        a = infos->first;
444 
+                        b = infos->second;
445 
+                        dir = DIR_PREV;
446 
+                    } else {
447 
+                        break;
448 
+                    }
449 
+                }
450 
+            } else {
451 
+                /* good frame */
452 
+                distsum += dist;
453 
+                goodfcount++;
454 
+                tolerancecount=0;
455 
+
456 
+                aprev = a;
457 
+                bprev = b;
458 
+
459 
+                if (a->confidence < 1) gooda++;
460 
+                if (b->confidence < 1) goodb++;
461 
+            }
462 
+
463 
+            fcount++;
464 
+
465 
+            dir = iterate_frame(infos->framerateratio, &a, &b, fcount, &bcount, dir);
466 
+            if (dir == DIR_NEXT_END) {
467 
+                status = STATUS_END_REACHED;
468 
+                a = infos->first;
469 
+                b = infos->second;
470 
+                dir = iterate_frame(infos->framerateratio, &a, &b, fcount, &bcount, DIR_PREV);
471 
+            }
472 
+
473 
+            if (dir == DIR_PREV_END) {
474 
+                status |= STATUS_BEGIN_REACHED;
475 
+                break;
476 
+            }
477 
+
478 
+            if (sc->thdi != 0 && bcount >= sc->thdi) {
479 
+                break; /* enough frames found */
480 
+            }
481 
+        }
482 
+
483 
+        if (bcount < sc->thdi)
484 
+            continue; /* matching sequence is too short */
485 
+        if ((double) goodfcount / (double) fcount < sc->thit)
486 
+            continue;
487 
+        if ((double) goodfcount*0.5 < FFMAX(gooda, goodb))
488 
+            continue;
489 
+
490 
+        meandist = (double) goodfcount / (double) distsum;
491 
+
492 
+        if (meandist < minmeandist ||
493 
+                status == STATUS_END_REACHED | STATUS_BEGIN_REACHED ||
494 
+                mode == MODE_FAST){
495 
+            minmeandist = meandist;
496 
+            /* bestcandidate in this iteration */
497 
+            bestmatch.meandist = meandist;
498 
+            bestmatch.matchframes = bcount;
499 
+            bestmatch.framerateratio = infos->framerateratio;
500 
+            bestmatch.score = infos->score;
501 
+            bestmatch.offset = infos->offset;
502 
+            bestmatch.first = infos->first;
503 
+            bestmatch.second = infos->second;
504 
+            bestmatch.whole = 0; /* will be set to true later */
505 
+            bestmatch.next = NULL;
506 
+        }
507 
+
508 
+        /* whole sequence is automatically best match */
509 
+        if (status == (STATUS_END_REACHED | STATUS_BEGIN_REACHED)) {
510 
+            bestmatch.whole = 1;
511 
+            break;
512 
+        }
513 
+
514 
+        /* first matching sequence is enough, finding the best one is not necessary */
515 
+        if (mode == MODE_FAST) {
516 
+            break;
517 
+        }
518 
+    }
519 
+    return bestmatch;
520 
+}
521 
+
522 
+static void sll_free(MatchingInfo *sll)
523 
+{
524 
+    void *tmp;
525 
+    while (sll) {
526 
+        tmp = sll;
527 
+        sll = sll->next;
528 
+        av_freep(&tmp);
529 
+    }
530 
+}
531 
+
532 
+static MatchingInfo lookup_signatures(AVFilterContext *ctx, SignatureContext *sc, StreamContext *first, StreamContext *second, int mode)
533 
+{
534 
+    CoarseSignature *cs, *cs2;
535 
+    MatchingInfo *infos;
536 
+    MatchingInfo bestmatch;
537 
+    MatchingInfo *i;
538 
+
539 
+    cs = first->coarsesiglist;
540 
+    cs2 = second->coarsesiglist;
541 
+
542 
+    /* score of bestmatch is 0, if no match is found */
543 
+    bestmatch.score = 0;
544 
+    bestmatch.meandist = 99999;
545 
+    bestmatch.whole = 0;
546 
+
547 
+    fill_l1distlut(sc->l1distlut);
548 
+
549 
+    /* stage 1: coarsesignature matching */
550 
+    if (find_next_coarsecandidate(sc, second->coarsesiglist, &cs, &cs2, 1) == 0)
551 
+        return bestmatch; /* no candidate found */
552 
+    do {
553 
+        av_log(ctx, AV_LOG_DEBUG, "Stage 1: got coarsesignature pair. indices of first frame: %d and %d\n", cs->first->index, cs2->first->index);
554 
+        /* stage 2: l1-distance and hough-transform */
555 
+        av_log(ctx, AV_LOG_DEBUG, "Stage 2: calculate matching parameters\n");
556 
+        infos = get_matching_parameters(ctx, sc, cs->first, cs2->first);
557 
+        if (av_log_get_level() == AV_LOG_DEBUG) {
558 
+            for (i = infos; i != NULL; i = i->next) {
559 
+                av_log(ctx, AV_LOG_DEBUG, "Stage 2: matching pair at %d and %d, ratio %f, offset %d\n", i->first->index, i->second->index, i->framerateratio, i->offset);
560 
+            }
561 
+        }
562 
+        /* stage 3: evaluation */
563 
+        av_log(ctx, AV_LOG_DEBUG, "Stage 3: evaluate\n");
564 
+        if (infos) {
565 
+            bestmatch = evaluate_parameters(ctx, sc, infos, bestmatch, mode);
566 
+            av_log(ctx, AV_LOG_DEBUG, "Stage 3: best matching pair at %d and %d, ratio %f, offset %d, score %d, %d frames matching\n", bestmatch.first->index, bestmatch.second->index, bestmatch.framerateratio, bestmatch.offset, bestmatch.score, bestmatch.matchframes);
567 
+            sll_free(infos);
568 
+        }
569 
+    } while (find_next_coarsecandidate(sc, second->coarsesiglist, &cs, &cs2, 0) && !bestmatch.whole);
570 
+    return bestmatch;
571 
+
572 
+}
libavfilter/version.h
History View file @ 5e3a418
... ... 
@@ -30,7 +30,7 @@
30 30 
 #include "libavutil/version.h"
31 31
32 32 
 #define LIBAVFILTER_VERSION_MAJOR   6
33 
-#define LIBAVFILTER_VERSION_MINOR  77
33 
+#define LIBAVFILTER_VERSION_MINOR  78
34 34 
 #define LIBAVFILTER_VERSION_MICRO 100
35 35
36 36 
 #define LIBAVFILTER_VERSION_INT AV_VERSION_INT(LIBAVFILTER_VERSION_MAJOR, \
libavfilter/vf_signature.c
History View file @ 5e3a418
37 37 
new file mode 100644
... ... 
@@ -0,0 +1,767 @@
0 
+/*
1 
+ * Copyright (c) 2017 Gerion Entrup
2 
+ *
3 
+ * This file is part of FFmpeg.
4 
+ *
5 
+ * FFmpeg is free software; you can redistribute it and/or modify
6 
+ * it under the terms of the GNU General Public License as published by
7 
+ * the Free Software Foundation; either version 2 of the License, or
8 
+ * (at your option) any later version.
9 
+ *
10 
+ * FFmpeg is distributed in the hope that it will be useful,
11 
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13 
+ * GNU General Public License for more details.
14 
+ *
15 
+ * You should have received a copy of the GNU General Public License along
16 
+ * with FFmpeg; if not, write to the Free Software Foundation, Inc.,
17 
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
18 
+ */
19 
+
20 
+/**
21 
+ * @file
22 
+ * MPEG-7 video signature calculation and lookup filter
23 
+ * @see http://epubs.surrey.ac.uk/531590/1/MPEG-7%20Video%20Signature%20Author%27s%20Copy.pdf
24 
+ */
25 
+
26 
+#include <float.h>
27 
+#include "libavcodec/put_bits.h"
28 
+#include "libavformat/avformat.h"
29 
+#include "libavutil/opt.h"
30 
+#include "libavutil/avstring.h"
31 
+#include "libavutil/intreadwrite.h"
32 
+#include "libavutil/timestamp.h"
33 
+#include "avfilter.h"
34 
+#include "internal.h"
35 
+#include "signature.h"
36 
+#include "signature_lookup.c"
37 
+
38 
+#define OFFSET(x) offsetof(SignatureContext, x)
39 
+#define FLAGS AV_OPT_FLAG_FILTERING_PARAM | AV_OPT_FLAG_VIDEO_PARAM
40 
+#define BLOCK_LCM (int64_t) 476985600
41 
+
42 
+static const AVOption signature_options[] = {
43 
+    { "detectmode", "set the detectmode",
44 
+        OFFSET(mode),         AV_OPT_TYPE_INT,    {.i64 = MODE_OFF}, 0, NB_LOOKUP_MODE-1, FLAGS, "mode" },
45 
+        { "off",  NULL, 0, AV_OPT_TYPE_CONST, {.i64 = MODE_OFF},  0, 0, .flags = FLAGS, "mode" },
46 
+        { "full", NULL, 0, AV_OPT_TYPE_CONST, {.i64 = MODE_FULL}, 0, 0, .flags = FLAGS, "mode" },
47 
+        { "fast", NULL, 0, AV_OPT_TYPE_CONST, {.i64 = MODE_FAST}, 0, 0, .flags = FLAGS, "mode" },
48 
+    { "nb_inputs",  "number of inputs",
49 
+        OFFSET(nb_inputs),    AV_OPT_TYPE_INT,    {.i64 = 1},        1, INT_MAX,          FLAGS },
50 
+    { "filename",   "filename for output files",
51 
+        OFFSET(filename),     AV_OPT_TYPE_STRING, {.str = ""},       0, NB_FORMATS-1,     FLAGS },
52 
+    { "format",     "set output format",
53 
+        OFFSET(format),       AV_OPT_TYPE_INT,    {.i64 = FORMAT_BINARY}, 0, 1,           FLAGS , "format" },
54 
+        { "binary", 0, 0, AV_OPT_TYPE_CONST, {.i64=FORMAT_BINARY}, 0, 0, FLAGS, "format" },
55 
+        { "xml",    0, 0, AV_OPT_TYPE_CONST, {.i64=FORMAT_XML},    0, 0, FLAGS, "format" },
56 
+    { "th_d",       "threshold to detect one word as similar",
57 
+        OFFSET(thworddist),   AV_OPT_TYPE_INT,    {.i64 = 9000},     1, INT_MAX,          FLAGS },
58 
+    { "th_dc",      "threshold to detect all words as similar",
59 
+        OFFSET(thcomposdist), AV_OPT_TYPE_INT,    {.i64 = 60000},    1, INT_MAX,          FLAGS },
60 
+    { "th_xh",      "threshold to detect frames as similar",
61 
+        OFFSET(thl1),         AV_OPT_TYPE_INT,    {.i64 = 116},      1, INT_MAX,          FLAGS },
62 
+    { "th_di",      "minimum length of matching sequence in frames",
63 
+        OFFSET(thdi),         AV_OPT_TYPE_INT,    {.i64 = 0},        0, INT_MAX,          FLAGS },
64 
+    { "th_it",      "threshold for relation of good to all frames",
65 
+        OFFSET(thit),         AV_OPT_TYPE_DOUBLE, {.dbl = 0.5},    0.0, 1.0,              FLAGS },
66 
+    { NULL }
67 
+};
68 
+
69 
+AVFILTER_DEFINE_CLASS(signature);
70 
+
71 
+static int query_formats(AVFilterContext *ctx)
72 
+{
73 
+    /* all formats with a seperate gray value */
74 
+    static const enum AVPixelFormat pix_fmts[] = {
75 
+        AV_PIX_FMT_GRAY8,
76 
+        AV_PIX_FMT_YUV410P, AV_PIX_FMT_YUV411P,
77 
+        AV_PIX_FMT_YUV420P, AV_PIX_FMT_YUV422P,
78 
+        AV_PIX_FMT_YUV440P, AV_PIX_FMT_YUV444P,
79 
+        AV_PIX_FMT_YUVJ411P, AV_PIX_FMT_YUVJ420P,
80 
+        AV_PIX_FMT_YUVJ422P, AV_PIX_FMT_YUVJ444P,
81 
+        AV_PIX_FMT_YUVJ440P,
82 
+        AV_PIX_FMT_NV12, AV_PIX_FMT_NV21,
83 
+        AV_PIX_FMT_NONE
84 
+    };
85 
+
86 
+    return ff_set_common_formats(ctx, ff_make_format_list(pix_fmts));
87 
+}
88 
+
89 
+static int config_input(AVFilterLink *inlink)
90 
+{
91 
+    AVFilterContext *ctx = inlink->dst;
92 
+    SignatureContext *sic = ctx->priv;
93 
+    StreamContext *sc = &(sic->streamcontexts[FF_INLINK_IDX(inlink)]);
94 
+
95 
+    sc->time_base = inlink->time_base;
96 
+    /* test for overflow */
97 
+    sc->divide = (((uint64_t) inlink->w/32) * (inlink->w/32 + 1) * (inlink->h/32 * inlink->h/32 + 1) > INT64_MAX / (BLOCK_LCM * 255));
98 
+    if (sc->divide) {
99 
+        av_log(ctx, AV_LOG_WARNING, "Input dimension too high for precise calculation, numbers will be rounded.\n");
100 
+    }
101 
+    sc->w = inlink->w;
102 
+    sc->h = inlink->h;
103 
+    return 0;
104 
+}
105 
+
106 
+static int get_block_size(const Block *b)
107 
+{
108 
+    return (b->to.y - b->up.y + 1) * (b->to.x - b->up.x + 1);
109 
+}
110 
+
111 
+static uint64_t get_block_sum(StreamContext *sc, uint64_t intpic[32][32], const Block *b)
112 
+{
113 
+    uint64_t sum = 0;
114 
+
115 
+    int x0, y0, x1, y1;
116 
+
117 
+    x0 = b->up.x;
118 
+    y0 = b->up.y;
119 
+    x1 = b->to.x;
120 
+    y1 = b->to.y;
121 
+
122 
+    if (x0-1 >= 0 && y0-1 >= 0) {
123 
+        sum = intpic[y1][x1] + intpic[y0-1][x0-1] - intpic[y1][x0-1] - intpic[y0-1][x1];
124 
+    } else if (x0-1 >= 0) {
125 
+        sum = intpic[y1][x1] - intpic[y1][x0-1];
126 
+    } else if (y0-1 >= 0) {
127 
+        sum = intpic[y1][x1] - intpic[y0-1][x1];
128 
+    } else {
129 
+        sum = intpic[y1][x1];
130 
+    }
131 
+    return sum;
132 
+}
133 
+
134 
+static int cmp(const uint64_t *a, const uint64_t *b)
135 
+{
136 
+    return *a < *b ? -1 : ( *a > *b ? 1 : 0 );
137 
+}
138 
+
139 
+/**
140 
+ * sets the bit at position pos to 1 in data
141 
+ */
142 
+static void set_bit(uint8_t* data, size_t pos)
143 
+{
144 
+    uint8_t mask = 1 << 7-(pos%8);
145 
+    data[pos/8] |= mask;
146 
+}
147 
+
148 
+static int filter_frame(AVFilterLink *inlink, AVFrame *picref)
149 
+{
150 
+    AVFilterContext *ctx = inlink->dst;
151 
+    SignatureContext *sic = ctx->priv;
152 
+    StreamContext *sc = &(sic->streamcontexts[FF_INLINK_IDX(inlink)]);
153 
+    FineSignature* fs;
154 
+
155 
+    static const uint8_t pot3[5] = { 3*3*3*3, 3*3*3, 3*3, 3, 1 };
156 
+    /* indexes of words : 210,217,219,274,334  44,175,233,270,273  57,70,103,237,269  100,285,295,337,354  101,102,111,275,296
157 
+    s2usw = sorted to unsorted wordvec: 44 is at index 5, 57 at index 10...
158 
+    */
159 
+    static const unsigned int wordvec[25] = {44,57,70,100,101,102,103,111,175,210,217,219,233,237,269,270,273,274,275,285,295,296,334,337,354};
160 
+    static const uint8_t      s2usw[25]   = { 5,10,11, 15, 20, 21, 12, 22,  6,  0,  1,  2,  7, 13, 14,  8,  9,  3, 23, 16, 17, 24,  4, 18, 19};
161 
+
162 
+    uint8_t wordt2b[5] = { 0, 0, 0, 0, 0 }; /* word ternary to binary */
163 
+    uint64_t intpic[32][32];
164 
+    uint64_t rowcount;
165 
+    uint8_t *p = picref->data[0];
166 
+    int inti, intj;
167 
+    int *intjlut;
168 
+
169 
+    uint64_t conflist[DIFFELEM_SIZE];
170 
+    int f = 0, g = 0, w = 0;
171 
+    int32_t dh1 = 1, dh2 = 1, dw1 = 1, dw2 = 1, a, b;
172 
+    int64_t denom;
173 
+    int i, j, k, ternary;
174 
+    uint64_t blocksum;
175 
+    int blocksize;
176 
+    int64_t th; /* threshold */
177 
+    int64_t sum;
178 
+
179 
+    int64_t precfactor = (sc->divide) ? 65536 : BLOCK_LCM;
180 
+
181 
+    /* initialize fs */
182 
+    if (sc->curfinesig) {
183 
+        fs = av_mallocz(sizeof(FineSignature));
184 
+        if (!fs)
185 
+            return AVERROR(ENOMEM);
186 
+        sc->curfinesig->next = fs;
187 
+        fs->prev = sc->curfinesig;
188 
+        sc->curfinesig = fs;
189 
+    } else {
190 
+        fs = sc->curfinesig = sc->finesiglist;
191 
+        sc->curcoarsesig1->first = fs;
192 
+    }
193 
+
194 
+    fs->pts = picref->pts;
195 
+    fs->index = sc->lastindex++;
196 
+
197 
+    memset(intpic, 0, sizeof(uint64_t)*32*32);
198 
+    intjlut = av_malloc_array(inlink->w, sizeof(int));
199 
+    if (!intjlut)
200 
+        return AVERROR(ENOMEM);
201 
+    for (i = 0; i < inlink->w; i++) {
202 
+        intjlut[i] = (i*32)/inlink->w;
203 
+    }
204 
+
205 
+    for (i = 0; i < inlink->h; i++) {
206 
+        inti = (i*32)/inlink->h;
207 
+        for (j = 0; j < inlink->w; j++) {
208 
+            intj = intjlut[j];
209 
+            intpic[inti][intj] += p[j];
210 
+        }
211 
+        p += picref->linesize[0];
212 
+    }
213 
+    av_freep(&intjlut);
214 
+
215 
+    /* The following calculates a summed area table (intpic) and brings the numbers
216 
+     * in intpic to the same denominator.
217 
+     * So you only have to handle the numinator in the following sections.
218 
+     */
219 
+    dh1 = inlink->h / 32;
220 
+    if (inlink->h % 32)
221 
+        dh2 = dh1 + 1;
222 
+    dw1 = inlink->w / 32;
223 
+    if (inlink->w % 32)
224 
+        dw2 = dw1 + 1;
225 
+    denom = (sc->divide) ? dh1 * dh2 * dw1 * dw2 : 1;
226 
+
227 
+    for (i = 0; i < 32; i++) {
228 
+        rowcount = 0;
229 
+        a = 1;
230 
+        if (dh2 > 1) {
231 
+            a = ((inlink->h*(i+1))%32 == 0) ? (inlink->h*(i+1))/32 - 1 : (inlink->h*(i+1))/32;
232 
+            a -= ((inlink->h*i)%32 == 0) ? (inlink->h*i)/32 - 1 : (inlink->h*i)/32;
233 
+            a = (a == dh1)? dh2 : dh1;
234 
+        }
235 
+        for (j = 0; j < 32; j++) {
236 
+            b = 1;
237 
+            if (dw2 > 1) {
238 
+                b = ((inlink->w*(j+1))%32 == 0) ? (inlink->w*(j+1))/32 - 1 : (inlink->w*(j+1))/32;
239 
+                b -= ((inlink->w*j)%32 == 0) ? (inlink->w*j)/32 - 1 : (inlink->w*j)/32;
240 
+                b = (b == dw1)? dw2 : dw1;
241 
+            }
242 
+            rowcount += intpic[i][j] * a * b * precfactor / denom;
243 
+            if (i > 0) {
244 
+                intpic[i][j] = intpic[i-1][j] + rowcount;
245 
+            } else {
246 
+                intpic[i][j] = rowcount;
247 
+            }
248 
+        }
249 
+    }
250 
+
251 
+    denom = (sc->divide) ? 1 : dh1 * dh2 * dw1 * dw2;
252 
+
253 
+    for (i = 0; i < ELEMENT_COUNT; i++) {
254 
+        const ElemCat* elemcat = elements[i];
255 
+        int64_t* elemsignature;
256 
+        uint64_t* sortsignature;
257 
+
258 
+        elemsignature = av_malloc_array(elemcat->elem_count, sizeof(int64_t));
259 
+        if (!elemsignature)
260 
+            return AVERROR(ENOMEM);
261 
+        sortsignature = av_malloc_array(elemcat->elem_count, sizeof(int64_t));
262 
+        if (!sortsignature)
263 
+            return AVERROR(ENOMEM);
264 
+
265 
+        for (j = 0; j < elemcat->elem_count; j++) {
266 
+            blocksum = 0;
267 
+            blocksize = 0;
268 
+            for (k = 0; k < elemcat->left_count; k++) {
269 
+                blocksum += get_block_sum(sc, intpic, &elemcat->blocks[j*elemcat->block_count+k]);
270 
+                blocksize += get_block_size(&elemcat->blocks[j*elemcat->block_count+k]);
271 
+            }
272 
+            sum = blocksum / blocksize;
273 
+            if (elemcat->av_elem) {
274 
+                sum -= 128 * precfactor * denom;
275 
+            } else {
276 
+                blocksum = 0;
277 
+                blocksize = 0;
278 
+                for (; k < elemcat->block_count; k++) {
279 
+                    blocksum += get_block_sum(sc, intpic, &elemcat->blocks[j*elemcat->block_count+k]);
280 
+                    blocksize += get_block_size(&elemcat->blocks[j*elemcat->block_count+k]);
281 
+                }
282 
+                sum -= blocksum / blocksize;
283 
+                conflist[g++] = FFABS(sum * 8 / (precfactor * denom));
284 
+            }
285 
+
286 
+            elemsignature[j] = sum;
287 
+            sortsignature[j] = FFABS(sum);
288 
+        }
289 
+
290 
+        /* get threshold */
291 
+        qsort(sortsignature, elemcat->elem_count, sizeof(uint64_t), (void*) cmp);
292 
+        th = sortsignature[(int) (elemcat->elem_count*0.333)];
293 
+
294 
+        /* ternarize */
295 
+        for (j = 0; j < elemcat->elem_count; j++) {
296 
+            if (elemsignature[j] < -th) {
297 
+                ternary = 0;
298 
+            } else if (elemsignature[j] <= th) {
299 
+                ternary = 1;
300 
+            } else {
301 
+                ternary = 2;
302 
+            }
303 
+            fs->framesig[f/5] += ternary * pot3[f%5];
304 
+
305 
+            if (f == wordvec[w]) {
306 
+                fs->words[s2usw[w]/5] += ternary * pot3[wordt2b[s2usw[w]/5]++];
307 
+                if (w < 24)
308 
+                    w++;
309 
+            }
310 
+            f++;
311 
+        }
312 
+        av_freep(&elemsignature);
313 
+        av_freep(&sortsignature);
314 
+    }
315 
+
316 
+    /* confidence */
317 
+    qsort(conflist, DIFFELEM_SIZE, sizeof(uint64_t), (void*) cmp);
318 
+    fs->confidence = FFMIN(conflist[DIFFELEM_SIZE/2], 255);
319 
+
320 
+    /* coarsesignature */
321 
+    if (sc->coarsecount == 0) {
322 
+        if (sc->curcoarsesig2) {
323 
+            sc->curcoarsesig1 = av_mallocz(sizeof(CoarseSignature));
324 
+            if (!sc->curcoarsesig1)
325 
+                return AVERROR(ENOMEM);
326 
+            sc->curcoarsesig1->first = fs;
327 
+            sc->curcoarsesig2->next = sc->curcoarsesig1;
328 
+            sc->coarseend = sc->curcoarsesig1;
329 
+        }
330 
+    }
331 
+    if (sc->coarsecount == 45) {
332 
+        sc->midcoarse = 1;
333 
+        sc->curcoarsesig2 = av_mallocz(sizeof(CoarseSignature));
334 
+        if (!sc->curcoarsesig2)
335 
+            return AVERROR(ENOMEM);
336 
+        sc->curcoarsesig2->first = fs;
337 
+        sc->curcoarsesig1->next = sc->curcoarsesig2;
338 
+        sc->coarseend = sc->curcoarsesig2;
339 
+    }
340 
+    for (i = 0; i < 5; i++) {
341 
+        set_bit(sc->curcoarsesig1->data[i], fs->words[i]);
342 
+    }
343 
+    /* assuming the actual frame is the last */
344 
+    sc->curcoarsesig1->last = fs;
345 
+    if (sc->midcoarse) {
346 
+        for (i = 0; i < 5; i++) {
347 
+            set_bit(sc->curcoarsesig2->data[i], fs->words[i]);
348 
+        }
349 
+        sc->curcoarsesig2->last = fs;
350 
+    }
351 
+
352 
+    sc->coarsecount = (sc->coarsecount+1)%90;
353 
+
354 
+    /* debug printing finesignature */
355 
+    if (av_log_get_level() == AV_LOG_DEBUG) {
356 
+        av_log(ctx, AV_LOG_DEBUG, "input %d, confidence: %d\n", FF_INLINK_IDX(inlink), fs->confidence);
357 
+
358 
+        av_log(ctx, AV_LOG_DEBUG, "words:");
359 
+        for (i = 0; i < 5; i++) {
360 
+            av_log(ctx, AV_LOG_DEBUG, " %d:", fs->words[i] );
361 
+            av_log(ctx, AV_LOG_DEBUG, " %d", fs->words[i] / pot3[0] );
362 
+            for (j = 1; j < 5; j++)
363 
+                av_log(ctx, AV_LOG_DEBUG, ",%d", fs->words[i] % pot3[j-1] / pot3[j] );
364 
+            av_log(ctx, AV_LOG_DEBUG, ";");
365 
+        }
366 
+        av_log(ctx, AV_LOG_DEBUG, "\n");
367 
+
368 
+        av_log(ctx, AV_LOG_DEBUG, "framesignature:");
369 
+        for (i = 0; i < SIGELEM_SIZE/5; i++) {
370 
+            av_log(ctx, AV_LOG_DEBUG, " %d", fs->framesig[i] / pot3[0] );
371 
+            for (j = 1; j < 5; j++)
372 
+                av_log(ctx, AV_LOG_DEBUG, ",%d", fs->framesig[i] % pot3[j-1] / pot3[j] );
373 
+        }
374 
+        av_log(ctx, AV_LOG_DEBUG, "\n");
375 
+    }
376 
+
377 
+    if (FF_INLINK_IDX(inlink) == 0)
378 
+        return ff_filter_frame(inlink->dst->outputs[0], picref);
379 
+    return 1;
380 
+}
381 
+
382 
+static int xml_export(AVFilterContext *ctx, StreamContext *sc, const char* filename)
383 
+{
384 
+    FineSignature* fs;
385 
+    CoarseSignature* cs;
386 
+    int i, j;
387 
+    FILE* f;
388 
+    unsigned int pot3[5] = { 3*3*3*3, 3*3*3, 3*3, 3, 1 };
389 
+
390 
+    f = fopen(filename, "w");
391 
+    if (!f) {
392 
+        int err = AVERROR(EINVAL);
393 
+        char buf[128];
394 
+        av_strerror(err, buf, sizeof(buf));
395 
+        av_log(ctx, AV_LOG_ERROR, "cannot open xml file %s: %s\n", filename, buf);
396 
+        return err;
397 
+    }
398 
+
399 
+    /* header */
400 
+    fprintf(f, "<?xml version='1.0' encoding='ASCII' ?>\n");
401 
+    fprintf(f, "<Mpeg7 xmlns=\"urn:mpeg:mpeg7:schema:2001\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"urn:mpeg:mpeg7:schema:2001 schema/Mpeg7-2001.xsd\">\n");
402 
+    fprintf(f, "  <DescriptionUnit xsi:type=\"DescriptorCollectionType\">\n");
403 
+    fprintf(f, "    <Descriptor xsi:type=\"VideoSignatureType\">\n");
404 
+    fprintf(f, "      <VideoSignatureRegion>\n");
405 
+    fprintf(f, "        <VideoSignatureSpatialRegion>\n");
406 
+    fprintf(f, "          <Pixel>0 0 </Pixel>\n");
407 
+    fprintf(f, "          <Pixel>%d %d </Pixel>\n", sc->w - 1, sc->h - 1);
408 
+    fprintf(f, "        </VideoSignatureSpatialRegion>\n");
409 
+    fprintf(f, "        <StartFrameOfSpatialRegion>0</StartFrameOfSpatialRegion>\n");
410 
+    /* hoping num is 1, other values are vague */
411 
+    fprintf(f, "        <MediaTimeUnit>%d</MediaTimeUnit>\n", sc->time_base.den / sc->time_base.num);
412 
+    fprintf(f, "        <MediaTimeOfSpatialRegion>\n");
413 
+    fprintf(f, "          <StartMediaTimeOfSpatialRegion>0</StartMediaTimeOfSpatialRegion>\n");
414 
+    fprintf(f, "          <EndMediaTimeOfSpatialRegion>%" PRIu64 "</EndMediaTimeOfSpatialRegion>\n", sc->coarseend->last->pts);
415 
+    fprintf(f, "        </MediaTimeOfSpatialRegion>\n");
416 
+
417 
+    /* coarsesignatures */
418 
+    for (cs = sc->coarsesiglist; cs; cs = cs->next) {
419 
+        fprintf(f, "        <VSVideoSegment>\n");
420 
+        fprintf(f, "          <StartFrameOfSegment>%" PRIu32 "</StartFrameOfSegment>\n", cs->first->index);
421 
+        fprintf(f, "          <EndFrameOfSegment>%" PRIu32 "</EndFrameOfSegment>\n", cs->last->index);
422 
+        fprintf(f, "          <MediaTimeOfSegment>\n");
423 
+        fprintf(f, "            <StartMediaTimeOfSegment>%" PRIu64 "</StartMediaTimeOfSegment>\n", cs->first->pts);
424 
+        fprintf(f, "            <EndMediaTimeOfSegment>%" PRIu64 "</EndMediaTimeOfSegment>\n", cs->last->pts);
425 
+        fprintf(f, "          </MediaTimeOfSegment>\n");
426 
+        for (i = 0; i < 5; i++) {
427 
+            fprintf(f, "          <BagOfWords>");
428 
+            for (j = 0; j < 31; j++) {
429 
+                uint8_t n = cs->data[i][j];
430 
+                if (j < 30) {
431 
+                    fprintf(f, "%d  %d  %d  %d  %d  %d  %d  %d  ", (n & 0x80) >> 7,
432 
+                                                                   (n & 0x40) >> 6,
433 
+                                                                   (n & 0x20) >> 5,
434 
+                                                                   (n & 0x10) >> 4,
435 
+                                                                   (n & 0x08) >> 3,
436 
+                                                                   (n & 0x04) >> 2,
437 
+                                                                   (n & 0x02) >> 1,
438 
+                                                                   (n & 0x01));
439 
+                } else {
440 
+                    /* print only 3 bit in last byte */
441 
+                    fprintf(f, "%d  %d  %d ", (n & 0x80) >> 7,
442 
+                                              (n & 0x40) >> 6,
443 
+                                              (n & 0x20) >> 5);
444 
+                }
445 
+            }
446 
+            fprintf(f, "</BagOfWords>\n");
447 
+        }
448 
+        fprintf(f, "        </VSVideoSegment>\n");
449 
+    }
450 
+
451 
+    /* finesignatures */
452 
+    for (fs = sc->finesiglist; fs; fs = fs->next) {
453 
+        fprintf(f, "        <VideoFrame>\n");
454 
+        fprintf(f, "          <MediaTimeOfFrame>%" PRIu64 "</MediaTimeOfFrame>\n", fs->pts);
455 
+        /* confidence */
456 
+        fprintf(f, "          <FrameConfidence>%d</FrameConfidence>\n", fs->confidence);
457 
+        /* words */
458 
+        fprintf(f, "          <Word>");
459 
+        for (i = 0; i < 5; i++) {
460 
+            fprintf(f, "%d ", fs->words[i]);
461 
+            if (i < 4) {
462 
+                fprintf(f, " ");
463 
+            }
464 
+        }
465 
+        fprintf(f, "</Word>\n");
466 
+        /* framesignature */
467 
+        fprintf(f, "          <FrameSignature>");
468 
+        for (i = 0; i< SIGELEM_SIZE/5; i++) {
469 
+            if (i > 0) {
470 
+                fprintf(f, " ");
471 
+            }
472 
+            fprintf(f, "%d ", fs->framesig[i] / pot3[0]);
473 
+            for (j = 1; j < 5; j++)
474 
+                fprintf(f, " %d ", fs->framesig[i] % pot3[j-1] / pot3[j] );
475 
+        }
476 
+        fprintf(f, "</FrameSignature>\n");
477 
+        fprintf(f, "        </VideoFrame>\n");
478 
+    }
479 
+    fprintf(f, "      </VideoSignatureRegion>\n");
480 
+    fprintf(f, "    </Descriptor>\n");
481 
+    fprintf(f, "  </DescriptionUnit>\n");
482 
+    fprintf(f, "</Mpeg7>\n");
483 
+
484 
+    fclose(f);
485 
+    return 0;
486 
+}
487 
+
488 
+static int binary_export(AVFilterContext *ctx, StreamContext *sc, const char* filename)
489 
+{
490 
+    FILE* f;
491 
+    FineSignature* fs;
492 
+    CoarseSignature* cs;
493 
+    uint32_t numofsegments = (sc->lastindex + 44)/45;
494 
+    int i, j;
495 
+    PutBitContext buf;
496 
+    /* buffer + header + coarsesignatures + finesignature */
497 
+    int len = (512 + 6 * 32 + 3*16 + 2 +
498 
+        numofsegments * (4*32 + 1 + 5*243) +
499 
+        sc->lastindex * (2 + 32 + 6*8 + 608)) / 8;
500 
+    uint8_t* buffer = av_malloc_array(len, sizeof(uint8_t));
501 
+    if (!buffer)
502 
+        return AVERROR(ENOMEM);
503 
+
504 
+    f = fopen(filename, "wb");
505 
+    if (!f) {
506 
+        int err = AVERROR(EINVAL);
507 
+        char buf[128];
508 
+        av_strerror(err, buf, sizeof(buf));
509 
+        av_log(ctx, AV_LOG_ERROR, "cannot open file %s: %s\n", filename, buf);
510 
+        return err;
511 
+    }
512 
+    init_put_bits(&buf, buffer, len);
513 
+
514 
+    put_bits32(&buf, 1); /* NumOfSpatial Regions, only 1 supported */
515 
+    put_bits(&buf, 1, 1); /* SpatialLocationFlag, always the whole image */
516 
+    put_bits32(&buf, 0); /* PixelX,1 PixelY,1, 0,0 */
517 
+    put_bits(&buf, 16, sc->w-1 & 0xFFFF); /* PixelX,2 */
518 
+    put_bits(&buf, 16, sc->h-1 & 0xFFFF); /* PixelY,2 */
519 
+    put_bits32(&buf, 0); /* StartFrameOfSpatialRegion */
520 
+    put_bits32(&buf, sc->lastindex); /* NumOfFrames */
521 
+    /* hoping num is 1, other values are vague */
522 
+    /* den/num might be greater than 16 bit, so cutting it */
523 
+    put_bits(&buf, 16, 0xFFFF & (sc->time_base.den / sc->time_base.num)); /* MediaTimeUnit */
524 
+    put_bits(&buf, 1, 1); /* MediaTimeFlagOfSpatialRegion */
525 
+    put_bits32(&buf, 0); /* StartMediaTimeOfSpatialRegion */
526 
+    put_bits32(&buf, 0xFFFFFFFF & sc->coarseend->last->pts); /* EndMediaTimeOfSpatialRegion */
527 
+    put_bits32(&buf, numofsegments); /* NumOfSegments */
528 
+    /* coarsesignatures */
529 
+    for (cs = sc->coarsesiglist; cs; cs = cs->next) {
530 
+        put_bits32(&buf, cs->first->index); /* StartFrameOfSegment */
531 
+        put_bits32(&buf, cs->last->index); /* EndFrameOfSegment */
532 
+        put_bits(&buf, 1, 1); /* MediaTimeFlagOfSegment */
533 
+        put_bits32(&buf, 0xFFFFFFFF & cs->first->pts); /* StartMediaTimeOfSegment */
534 
+        put_bits32(&buf, 0xFFFFFFFF & cs->last->pts); /* EndMediaTimeOfSegment */
535 
+        for (i = 0; i < 5; i++) {
536 
+            /* put 243 bits ( = 7 * 32 + 19 = 8 * 28 + 19) into buffer */
537 
+            for (j = 0; j < 30; j++) {
538 
+                put_bits(&buf, 8, cs->data[i][j]);
539 
+            }
540 
+            put_bits(&buf, 3, cs->data[i][30] >> 5);
541 
+        }
542 
+    }
543 
+    /* finesignatures */
544 
+    put_bits(&buf, 1, 0); /* CompressionFlag, only 0 supported */
545 
+    for (fs = sc->finesiglist; fs; fs = fs->next) {
546 
+        put_bits(&buf, 1, 1); /* MediaTimeFlagOfFrame */
547 
+        put_bits32(&buf, 0xFFFFFFFF & fs->pts); /* MediaTimeOfFrame */
548 
+        put_bits(&buf, 8, fs->confidence); /* FrameConfidence */
549 
+        for (i = 0; i < 5; i++) {
550 
+            put_bits(&buf, 8, fs->words[i]); /* Words */
551 
+        }
552 
+        /* framesignature */
553 
+        for (i = 0; i < SIGELEM_SIZE/5; i++) {
554 
+            put_bits(&buf, 8, fs->framesig[i]);
555 
+        }
556 
+    }
557 
+
558 
+    avpriv_align_put_bits(&buf);
559 
+    flush_put_bits(&buf);
560 
+    fwrite(buffer, 1, put_bits_count(&buf)/8, f);
561 
+    fclose(f);
562 
+    av_freep(&buffer);
563 
+    return 0;
564 
+}
565 
+
566 
+static int export(AVFilterContext *ctx, StreamContext *sc, int input)
567 
+{
568 
+    SignatureContext* sic = ctx->priv;
569 
+    char filename[1024];
570 
+
571 
+    if (sic->nb_inputs > 1) {
572 
+        /* error already handled */
573 
+        av_assert0(av_get_frame_filename(filename, sizeof(filename), sic->filename, input) == 0);
574 
+    } else {
575 
+        strcpy(filename, sic->filename);
576 
+    }
577 
+    if (sic->format == FORMAT_XML) {
578 
+        return xml_export(ctx, sc, filename);
579 
+    } else {
580 
+        return binary_export(ctx, sc, filename);
581 
+    }
582 
+}
583 
+
584 
+static int request_frame(AVFilterLink *outlink)
585 
+{
586 
+    AVFilterContext *ctx = outlink->src;
587 
+    SignatureContext *sic = ctx->priv;
588 
+    StreamContext *sc, *sc2;
589 
+    MatchingInfo match;
590 
+    int i, j, ret;
591 
+    int lookup = 1; /* indicates wheather EOF of all files is reached */
592 
+
593 
+    /* process all inputs */
594 
+    for (i = 0; i < sic->nb_inputs; i++){
595 
+        sc = &(sic->streamcontexts[i]);
596 
+
597 
+        ret = ff_request_frame(ctx->inputs[i]);
598 
+
599 
+        /* return if unexpected error occurs in input stream */
600 
+        if (ret < 0 && ret != AVERROR_EOF)
601 
+            return ret;
602 
+
603 
+        /* export signature at EOF */
604 
+        if (ret == AVERROR_EOF && !sc->exported) {
605 
+            /* export if wanted */
606 
+            if (strlen(sic->filename) > 0) {
607 
+                if (export(ctx, sc, i) < 0)
608 
+                    return ret;
609 
+            }
610 
+            sc->exported = 1;
611 
+        }
612 
+        lookup &= sc->exported;
613 
+    }
614 
+
615 
+    /* signature lookup */
616 
+    if (lookup && sic->mode != MODE_OFF) {
617 
+        /* iterate over every pair */
618 
+        for (i = 0; i < sic->nb_inputs; i++) {
619 
+            sc = &(sic->streamcontexts[i]);
620 
+            for (j = i+1; j < sic->nb_inputs; j++) {
621 
+                sc2 = &(sic->streamcontexts[j]);
622 
+                match = lookup_signatures(ctx, sic, sc, sc2, sic->mode);
623 
+                if (match.score != 0) {
624 
+                    av_log(ctx, AV_LOG_INFO, "matching of video %d at %f and %d at %f, %d frames matching\n",
625 
+                            i, ((double) match.first->pts * sc->time_base.num) / sc->time_base.den,
626 
+                            j, ((double) match.second->pts * sc2->time_base.num) / sc2->time_base.den,
627 
+                            match.matchframes);
628 
+                    if (match.whole)
629 
+                        av_log(ctx, AV_LOG_INFO, "whole video matching\n");
630 
+                } else {
631 
+                    av_log(ctx, AV_LOG_INFO, "no matching of video %d and %d\n", i, j);
632 
+                }
633 
+            }
634 
+        }
635 
+    }
636 
+
637 
+    return ret;
638 
+}
639 
+
640 
+static av_cold int init(AVFilterContext *ctx)
641 
+{
642 
+
643 
+    SignatureContext *sic = ctx->priv;
644 
+    StreamContext *sc;
645 
+    int i, ret;
646 
+    char tmp[1024];
647 
+
648 
+    sic->streamcontexts = av_mallocz(sic->nb_inputs * sizeof(StreamContext));
649 
+    if (!sic->streamcontexts)
650 
+        return AVERROR(ENOMEM);
651 
+
652 
+    for (i = 0; i < sic->nb_inputs; i++) {
653 
+        AVFilterPad pad = {
654 
+            .type = AVMEDIA_TYPE_VIDEO,
655 
+            .name = av_asprintf("in%d", i),
656 
+            .config_props = config_input,
657 
+            .filter_frame = filter_frame,
658 
+        };
659 
+
660 
+        if (!pad.name)
661 
+            return AVERROR(ENOMEM);
662 
+
663 
+        sc = &(sic->streamcontexts[i]);
664 
+
665 
+        sc->lastindex = 0;
666 
+        sc->finesiglist = av_mallocz(sizeof(FineSignature));
667 
+        if (!sc->finesiglist)
668 
+            return AVERROR(ENOMEM);
669 
+        sc->curfinesig = NULL;
670 
+
671 
+        sc->coarsesiglist = av_mallocz(sizeof(CoarseSignature));
672 
+        if (!sc->coarsesiglist)
673 
+            return AVERROR(ENOMEM);
674 
+        sc->curcoarsesig1 = sc->coarsesiglist;
675 
+        sc->coarseend = sc->coarsesiglist;
676 
+        sc->coarsecount = 0;
677 
+        sc->midcoarse = 0;
678 
+
679 
+        if ((ret = ff_insert_inpad(ctx, i, &pad)) < 0) {
680 
+            av_freep(&pad.name);
681 
+            return ret;
682 
+        }
683 
+    }
684 
+
685 
+    /* check filename */
686 
+    if (sic->nb_inputs > 1 && strlen(sic->filename) > 0 && av_get_frame_filename(tmp, sizeof(tmp), sic->filename, 0) == -1) {
687 
+        av_log(ctx, AV_LOG_ERROR, "The filename must contain %%d or %%0nd, if you have more than one input.\n");
688 
+        return AVERROR(EINVAL);
689 
+    }
690 
+
691 
+    return 0;
692 
+}
693 
+
694 
+
695 
+
696 
+static av_cold void uninit(AVFilterContext *ctx)
697 
+{
698 
+    SignatureContext *sic = ctx->priv;
699 
+    StreamContext *sc;
700 
+    void* tmp;
701 
+    FineSignature* finsig;
702 
+    CoarseSignature* cousig;
703 
+    int i;
704 
+
705 
+
706 
+    /* free the lists */
707 
+    if (sic->streamcontexts != NULL) {
708 
+        for (i = 0; i < sic->nb_inputs; i++) {
709 
+            sc = &(sic->streamcontexts[i]);
710 
+            finsig = sc->finesiglist;
711 
+            cousig = sc->coarsesiglist;
712 
+
713 
+            while (finsig) {
714 
+                tmp = finsig;
715 
+                finsig = finsig->next;
716 
+                av_freep(&tmp);
717 
+            }
718 
+            sc->finesiglist = NULL;
719 
+
720 
+            while (cousig) {
721 
+                tmp = cousig;
722 
+                cousig = cousig->next;
723 
+                av_freep(&tmp);
724 
+            }
725 
+            sc->coarsesiglist = NULL;
726 
+        }
727 
+        av_freep(&sic->streamcontexts);
728 
+    }
729 
+}
730 
+
731 
+static int config_output(AVFilterLink *outlink)
732 
+{
733 
+    AVFilterContext *ctx = outlink->src;
734 
+    AVFilterLink *inlink = ctx->inputs[0];
735 
+
736 
+    outlink->time_base = inlink->time_base;
737 
+    outlink->frame_rate = inlink->frame_rate;
738 
+    outlink->sample_aspect_ratio = inlink->sample_aspect_ratio;
739 
+    outlink->w = inlink->w;
740 
+    outlink->h = inlink->h;
741 
+
742 
+    return 0;
743 
+}
744 
+
745 
+static const AVFilterPad signature_outputs[] = {
746 
+    {
747 
+        .name          = "default",
748 
+        .type          = AVMEDIA_TYPE_VIDEO,
749 
+        .request_frame = request_frame,
750 
+        .config_props  = config_output,
751 
+    },
752 
+    { NULL }
753 
+};
754 
+
755 
+AVFilter ff_vf_signature = {
756 
+    .name          = "signature",
757 
+    .description   = NULL_IF_CONFIG_SMALL("Calculate the MPEG-7 video signature"),
758 
+    .priv_size     = sizeof(SignatureContext),
759 
+    .priv_class    = &signature_class,
760 
+    .init          = init,
761 
+    .uninit        = uninit,
762 
+    .query_formats = query_formats,
763 
+    .outputs       = signature_outputs,
764 
+    .inputs        = NULL,
765 
+    .flags         = AVFILTER_FLAG_DYNAMIC_INPUTS,
766 
+};