digiKam
Loading...
Searching...
No Matches
opencvdnnfacerecognizer_p.h
Go to the documentation of this file.
1/* ============================================================
2 *
3 * This file is a part of digiKam
4 *
5 * Date : 2020-05-22
6 * Description : Wrapper of face recognition using OpenFace.
7 *
8 * SPDX-FileCopyrightText: 2019 by Thanh Trung Dinh <dinhthanhtrung1996 at gmail dot com>
9 * SPDX-FileCopyrightText: 2020-2024 by Gilles Caulier <caulier dot gilles at gmail dot com>
10 * SPDX-FileCopyrightText: 2020 by Nghia Duong <minhnghiaduong997 at gmail dot com>
11 * SPDX-FileCopyrightText: 2024 by Michael Miller <michael underscore miller at msn dot com>
12 *
13 * SPDX-License-Identifier: GPL-2.0-or-later
14 *
15 * ============================================================ */
16
17#pragma once
18
20
21// C++ includes
22
23#include <iostream>
24
25// Qt includes
26
27#include <QElapsedTimer>
28
29// Local includes
30
31#include "digikam_debug.h"
33#include "dnnsfaceextractor.h"
34#include "facedbaccess.h"
35#include "facedb.h"
36#include "kd_treebase.h"
37
38namespace Digikam
39{
40
41class Q_DECL_HIDDEN OpenCVDNNFaceRecognizer::Private
42{
43public:
44
46 : method (mthd),
47 recognizeModel(recModel)
48 {
49 ref = 1;
50
51 for (int i = 0 ; i < 1 ; ++i)
52 {
53 switch (recognizeModel)
54 {
55 case FaceScanSettings::FaceRecognitionModel::OpenFace:
56 {
57 extractors << new DNNOpenFaceExtractor;
58 break;
59 }
60
61 case FaceScanSettings::FaceRecognitionModel::SFace:
62 {
63 extractors << new DNNSFaceExtractor;
64 break;
65 }
66
67 default:
68 {
69 qCritical(DIGIKAM_DPLUGIN_GENERIC_LOG) << "OpenCVDNNFaceRecognizer::Private() Unknown recognition model specified" << Qt::endl;
70 break;
71 }
72 }
73 }
74
75 switch (method)
76 {
77 case SVM:
78 {
79 svm = cv::ml::SVM::create();
80 svm->setKernel(cv::ml::SVM::LINEAR);
81 break;
82 }
83
84 case OpenCV_KNN:
85 {
86 knn = cv::ml::KNearest::create();
87 knn->setAlgorithmType(cv::ml::KNearest::BRUTE_FORCE);
88 knn->setIsClassifier(true);
89 break;
90 }
91
92 case Tree:
93 {
94 if (tree)
95 {
96 delete tree;
97 tree = nullptr; // safety in case reconstructTree fails
98 }
99
100 tree = FaceDbAccess().db()->reconstructTree(recognizeModel);
101 break;
102 }
103
104 case DB:
105 {
106 break;
107 }
108
109 default:
110 {
111 qFatal("Invalid classifier");
112 }
113 }
114 }
115
117 {
118 QVector<DNNFaceExtractorBase*>::iterator extractor = extractors.begin();
119
120 while (extractor != extractors.end())
121 {
122 delete *extractor;
123 extractor = extractors.erase(extractor);
124 }
125
126 delete tree;
127 }
128
129public:
130
131 bool trainSVM();
132 bool trainKNN();
133
134 int predictSVM(const cv::Mat& faceEmbedding);
135 int predictKNN(const cv::Mat& faceEmbedding);
136
137 int predictKDTree(const cv::Mat& faceEmbedding) const;
138 int predictDb(const cv::Mat& faceEmbedding) const;
139 int predictSFace(const cv::Mat& faceEmbedding) const;
140
141 bool insertData(const cv::Mat& position, const int label, const QString& context = QString());
142
143public:
144
145 int ref = 1;
146
148
149 QVector<DNNFaceExtractorBase*> extractors;
150 cv::Ptr<cv::ml::SVM> svm;
151 cv::Ptr<cv::ml::KNearest> knn;
152
153 KDTreeBase* tree = nullptr;
154 int kNeighbors = 5;
155/*
156 float threshold = 0.4F;
157*/
159
160 bool newDataAdded = true;
161
162 FaceScanSettings::FaceRecognitionModel recognizeModel = FaceScanSettings::FaceRecognitionModel::SFace;
163
164public:
165
166 class ParallelRecognizer;
167 class ParallelTrainer;
168};
169
171{
172public:
173
175 const QList<QPair<QImage*, QString> >& images,
176 QVector<int>& ids)
177 : images (images),
178 ids (ids),
179 d (d)
180 {
181 ids.resize(images.size());
182 }
183
184 void operator()(const cv::Range& range) const override
185 {
186 for(int i = range.start ; i < range.end ; ++i)
187 {
188 int id = -1;
189
190 cv::Mat faceEmbedding = d->extractors[i % (d->extractors.size())]->
191 getFaceEmbedding(OpenCVDNNFaceRecognizer::prepareForRecognition(*(images[i].first)));
192
193 switch (d->method)
194 {
195 case SVM:
196 {
197 id = d->predictSVM(faceEmbedding);
198 break;
199 }
200
201 case OpenCV_KNN:
202 {
203 id = d->predictKNN(faceEmbedding);
204 break;
205 }
206
207 case Tree:
208 {
209 id = d->predictKDTree(faceEmbedding);
210 break;
211 }
212
213 case DB:
214 {
215 id = d->predictDb(faceEmbedding);
216 break;
217 }
218
219 default:
220 {
221 qCWarning(DIGIKAM_FACEDB_LOG) << "Not recognized classifying method";
222 break;
223 }
224 }
225
226 ids[i] = id;
227 }
228 }
229
230private:
231
232 const QList<QPair<QImage*, QString> >& images;
233 QVector<int>& ids;
234
235 OpenCVDNNFaceRecognizer::Private* const d = nullptr;
236
237private:
238
239 Q_DISABLE_COPY(ParallelRecognizer)
240};
241
243{
244public:
245
247 const QList<QPair<QImage*,
248 QString> >& images,
249 const int& id)
250 : images (images),
251 id (id),
252 d (d)
253 {
254 }
255
256 void operator()(const cv::Range& range) const override
257 {
258 for(int i = range.start ; i < range.end ; ++i)
259 {
260 cv::Mat faceEmbedding = d->extractors[i % (d->extractors.size())]->
261 getFaceEmbedding(OpenCVDNNFaceRecognizer::prepareForRecognition(*(images[i].first)));
262
263 if (!d->insertData(faceEmbedding, id, images[i].second))
264 {
265 qCWarning(DIGIKAM_FACEDB_LOG) << "Fail to register a face of identity" << id;
266 }
267 }
268 }
269
270private:
271
272 const QList<QPair<QImage*, QString> >& images;
273 const int& id;
274
275 OpenCVDNNFaceRecognizer::Private* const d = nullptr;
276
277private:
278
279 Q_DISABLE_COPY(ParallelTrainer)
280};
281
283{
284 QElapsedTimer timer;
285 timer.start();
286
287 svm->train(FaceDbAccess().db()->trainData());
288
289 qCDebug(DIGIKAM_FACEDB_LOG) << "Support vector machine trains in" << timer.elapsed() << "ms";
290
291 return (svm->isTrained());
292}
293
295{
296 QElapsedTimer timer;
297 timer.start();
298
299 knn->train(FaceDbAccess().db()->trainData());
300
301 qCDebug(DIGIKAM_FACEDB_LOG) << "KNN trains in" << timer.elapsed() << "ms";
302
303 return (knn->isTrained());
304}
305
306int OpenCVDNNFaceRecognizer::Private::predictSVM(const cv::Mat& faceEmbedding)
307{
308 if (newDataAdded)
309 {
310 if (!trainSVM())
311 {
312 return -1;
313 }
314
315 newDataAdded = false;
316 }
317
318 return (int(svm->predict(faceEmbedding)));
319}
320
321int OpenCVDNNFaceRecognizer::Private::predictKNN(const cv::Mat& faceEmbedding)
322{
323 if (newDataAdded)
324 {
325 if (!trainKNN())
326 {
327 return -1;
328 }
329
330 newDataAdded = false;
331 }
332
333 cv::Mat output;
334 knn->findNearest(faceEmbedding, kNeighbors, output);
335
336 return (int(output.at<float>(0)));
337}
338
339int OpenCVDNNFaceRecognizer::Private::predictKDTree(const cv::Mat& faceEmbedding) const
340{
341 if (!tree)
342 {
343 return -1;
344 }
345
346 double threshold;
347
349 {
350 threshold = DNNModelManager::instance()->getModel(QLatin1String("SFace"),
351 DNNModelUsage::DNNUsageFaceRecognition)->getThreshold(uiThreshold);
352 }
353 else
354 {
355 threshold = DNNModelManager::instance()->getModel(QLatin1String("OpenFace"),
356 DNNModelUsage::DNNUsageFaceRecognition)->getThreshold(uiThreshold);
357 }
358
359 // Look for K-nearest neighbor which have the cosine distance greater than the threshold.
360
361 QMap<double, QVector<int> > closestNeighbors = tree->getClosestNeighbors(faceEmbedding, threshold, kNeighbors);
362
363 QMap<int, QVector<double> > votingGroups;
364
365 for (QMap<double, QVector<int> >::const_iterator iter = closestNeighbors.cbegin();
366 iter != closestNeighbors.cend();
367 ++iter)
368 {
369 for (QVector<int>::const_iterator node = iter.value().cbegin();
370 node != iter.value().cend();
371 ++node)
372 {
373 int label = (*node);
374
375 votingGroups[label].append(iter.key());
376 }
377 }
378
379 double maxScore = 0.0;
380 int prediction = -1;
381
382 for (QMap<int, QVector<double> >::const_iterator group = votingGroups.cbegin();
383 group != votingGroups.cend();
384 ++group)
385 {
386 double score = 0.0;
387
388 for (int i = 0 ; i < group.value().size() ; ++i)
389 {
390 score += (threshold - group.value()[i]);
391 }
392
393 if (score > maxScore)
394 {
395 maxScore = score;
396 prediction = group.key();
397 }
398 }
399
400 return prediction;
401}
402
403int OpenCVDNNFaceRecognizer::Private::predictDb(const cv::Mat& faceEmbedding) const
404{
405 double threshold;
406
408 {
409 threshold = DNNModelManager::instance()->getModel(QLatin1String("SFace"),
410 DNNModelUsage::DNNUsageFaceRecognition)->getThreshold(uiThreshold);
411 }
412 else
413 {
414 threshold = DNNModelManager::instance()->getModel(QLatin1String("OpenFace"),
415 DNNModelUsage::DNNUsageFaceRecognition)->getThreshold(uiThreshold);
416 }
417
418 QMap<double, QVector<int> > closestNeighbors = FaceDbAccess().db()->getClosestNeighborsTreeDb(faceEmbedding, threshold, 0.8, kNeighbors);
419
420 QMap<int, QVector<double> > votingGroups;
421
422 for (QMap<double, QVector<int> >::const_iterator iter = closestNeighbors.cbegin();
423 iter != closestNeighbors.cend();
424 ++iter)
425 {
426 for (int i = 0 ; i < iter.value().size() ; ++i)
427 {
428 votingGroups[iter.value()[i]].append(iter.key());
429 }
430 }
431
432 double maxScore = 0.0;
433 int prediction = -1;
434
435 for (QMap<int, QVector<double> >::const_iterator group = votingGroups.cbegin();
436 group != votingGroups.cend();
437 ++group)
438 {
439 double score = 0.0;
440
441 for (int i = 0 ; i < group.value().size() ; ++i)
442 {
443 score += (threshold - group.value()[i]);
444 }
445
446 if (score > maxScore)
447 {
448 maxScore = score;
449 prediction = group.key();
450 }
451 }
452
453 return prediction;
454}
455
456bool OpenCVDNNFaceRecognizer::Private::insertData(const cv::Mat& nodePos, const int label, const QString& hash)
457{
458 if (nodePos.rows != 1)
459 {
460 qCWarning(DIGIKAM_FACEDB_LOG) << "Error face embedding not valid";
461
462 return false;
463 }
464
465 int nodeId = FaceDbAccess().db()->insertFaceVector(nodePos, label, hash);
466
467 if (nodeId <= 0)
468 {
469 qCWarning(DIGIKAM_FACEDB_LOG) << "Error inserting face embedding to database";
470 }
471
472 if (method == DB)
473 {
474 if (!FaceDbAccess().db()->insertToTreeDb(nodeId, nodePos))
475 {
476 qCWarning(DIGIKAM_FACEDB_LOG) << "Error insert face embedding";
477
478 return false;
479 }
480 }
481 else if (method == Tree)
482 {
483 KDNodeBase* const newNode = tree->add(nodePos, label);
484
485 if (newNode)
486 {
487 newNode->setNodeId(nodeId);
488 }
489 else
490 {
491 qCWarning(DIGIKAM_FACEDB_LOG) << "Error insert new node" << nodeId;
492
493 return false;
494 }
495 }
496
497 return true;
498}
499
500} // namespace Digikam
float getThreshold(int uiThreshold=DNN_MODEL_THRESHOLD_NOT_SET) const
Definition dnnmodelbase.cpp:137
static DNNModelManager * instance()
Definition dnnmodelmanager.cpp:77
DNNModelBase * getModel(const QString &modelName, DNNModelUsage usage) const
Definition dnnmodelmanager.cpp:99
Definition dnnopenfaceextractor.h:30
Definition dnnsfaceextractor.h:31
Definition facedbaccess.h:35
FaceDb * db() const
Definition facedbaccess.cpp:131
KDTreeBase * reconstructTree(FaceScanSettings::FaceRecognitionModel recModel)
reconstructTree: reconstruct KD-Tree from data in the database.
Definition facedb_dnn.cpp:79
QMap< double, QVector< int > > getClosestNeighborsTreeDb(const cv::Mat &position, float sqRange, float cosThreshold, int maxNbNeighbors) const
getClosestNeighborsTreeDb: return a list of closest neighbor, limited by maxNbNeighbors and sqRange.
Definition facedb_dnn_spatial.cpp:106
int insertFaceVector(const cv::Mat &faceEmbedding, const int label, const QString &context) const
insertFaceVector: insert a new face embedding to database.
Definition facedb_dnn.cpp:28
FaceRecognitionModel
Definition facescansettings.h:89
@ SFace
SFace pre-trained neural network model [https://github.com/opencv/opencv_zoo/blob/main/models/face_re...
Definition facescansettings.h:91
Definition kd_nodebase.h:34
void setNodeId(int id)
Definition kd_nodebase.cpp:189
Definition kd_treebase.h:44
Definition opencvdnnfacerecognizer_p.h:171
void operator()(const cv::Range &range) const override
Definition opencvdnnfacerecognizer_p.h:184
ParallelRecognizer(OpenCVDNNFaceRecognizer::Private *const d, const QList< QPair< QImage *, QString > > &images, QVector< int > &ids)
Definition opencvdnnfacerecognizer_p.h:174
Definition opencvdnnfacerecognizer_p.h:243
ParallelTrainer(OpenCVDNNFaceRecognizer::Private *const d, const QList< QPair< QImage *, QString > > &images, const int &id)
Definition opencvdnnfacerecognizer_p.h:246
void operator()(const cv::Range &range) const override
Definition opencvdnnfacerecognizer_p.h:256
Definition opencvdnnfacerecognizer_p.h:42
int predictKDTree(const cv::Mat &faceEmbedding) const
Definition opencvdnnfacerecognizer_p.h:339
Classifier method
Definition opencvdnnfacerecognizer_p.h:147
QVector< DNNFaceExtractorBase * > extractors
Definition opencvdnnfacerecognizer_p.h:149
int predictSFace(const cv::Mat &faceEmbedding) const
~Private()
Definition opencvdnnfacerecognizer_p.h:116
cv::Ptr< cv::ml::SVM > svm
Definition opencvdnnfacerecognizer_p.h:150
int predictDb(const cv::Mat &faceEmbedding) const
Definition opencvdnnfacerecognizer_p.h:403
cv::Ptr< cv::ml::KNearest > knn
Definition opencvdnnfacerecognizer_p.h:151
int predictKNN(const cv::Mat &faceEmbedding)
Definition opencvdnnfacerecognizer_p.h:321
int predictSVM(const cv::Mat &faceEmbedding)
Definition opencvdnnfacerecognizer_p.h:306
bool insertData(const cv::Mat &position, const int label, const QString &context=QString())
Definition opencvdnnfacerecognizer_p.h:456
bool trainSVM()
Definition opencvdnnfacerecognizer_p.h:282
bool trainKNN()
Definition opencvdnnfacerecognizer_p.h:294
Private(Classifier mthd, FaceScanSettings::FaceRecognitionModel recModel)
Definition opencvdnnfacerecognizer_p.h:45
Definition opencvdnnfacerecognizer.h:34
static cv::Mat prepareForRecognition(QImage &inputImage)
Definition opencvdnnfacerecognizer.cpp:57
Classifier
Definition opencvdnnfacerecognizer.h:38
@ OpenCV_KNN
K-Nearest Neighbors (https://docs.opencv.org/4.x/dc/dd6/ml_intro.html#ml_intro_knn)
Definition opencvdnnfacerecognizer.h:40
@ DB
Closest Neighbors Tree from the database.
Definition opencvdnnfacerecognizer.h:42
@ Tree
K-Nearest Neighbors Tree (https://en.wikipedia.org/wiki/K-nearest_neighbors_algorithm)
Definition opencvdnnfacerecognizer.h:41
@ SVM
Support Vector Machines (https://docs.opencv.org/4.x/dc/dd6/ml_intro.html#ml_intro_svm)
Definition opencvdnnfacerecognizer.h:39
Definition datefolderview.cpp:34
const int DNN_MODEL_THRESHOLD_NOT_SET
Definition dnnmodeldefinitions.h:43