|
|
#include "alg_jd.h"
|
|
|
#include <direct.h> //ËùÐèµÄ¿âÎļþ
|
|
|
// Remove the bounding boxes with low confidence using non-maxima suppression
|
|
|
static void post_process(cv::Mat& frame, std::vector<cv::Mat>& outs, std::vector<std::pair<int, cv::Rect> > &results);
|
|
|
|
|
|
static void post_process_batch(std::vector<cv::Mat>& vec_frame, std::vector<cv::Mat>& outs, std::vector<std::vector<std::pair<int, cv::Rect> > > &vec_results);
|
|
|
|
|
|
// Get the names of the output layers
|
|
|
static std::vector<cv::String> getOutputsNames(const cv::dnn::Net& net);
|
|
|
|
|
|
// Draw the predicted bounding box
|
|
|
static void drawPred(int classId, float conf, int left, int top, int right, int bottom, cv::Mat& frame);
|
|
|
|
|
|
// Initialize the parameters
|
|
|
static float confThreshold = 0.1; // Confidence threshold
|
|
|
static float nmsThreshold = 0.4; // Non-maximum suppression threshold
|
|
|
static int inpWidth = 416; // Width of network's input image
|
|
|
static int inpHeight = 416; // Height of network's input image
|
|
|
|
|
|
static std::vector<std::string> classes;
|
|
|
|
|
|
bool AlgJd::init(QString model_path, QString jpg_path)
|
|
|
{
|
|
|
// Load names of classes
|
|
|
std::string classesFile = "../model/jd.names";
|
|
|
|
|
|
// Give the configuration and weight files for the model
|
|
|
cv::String modelConfiguration = "../model/jd.cfg";
|
|
|
cv::String modelWeights;
|
|
|
|
|
|
if (model_path.length() > 0) {
|
|
|
modelWeights = model_path.toStdString();
|
|
|
}
|
|
|
else {
|
|
|
modelWeights = "../model/jd.weights";
|
|
|
}
|
|
|
|
|
|
std::ifstream classNamesFile(classesFile.c_str());
|
|
|
if (classNamesFile.is_open())
|
|
|
{
|
|
|
std::string className = "";
|
|
|
while (std::getline(classNamesFile, className))
|
|
|
classes.push_back(className);
|
|
|
}
|
|
|
else{
|
|
|
return false;
|
|
|
}
|
|
|
|
|
|
// Load the network
|
|
|
net = cv::dnn::readNetFromDarknet(modelConfiguration, modelWeights);
|
|
|
|
|
|
net.setPreferableBackend(cv::dnn::DNN_BACKEND_CUDA);
|
|
|
net.setPreferableTarget(cv::dnn::DNN_TARGET_CUDA);
|
|
|
|
|
|
//cv::Mat image = cv::imread("alg_jd.jpg");
|
|
|
cv::Mat image;
|
|
|
if (jpg_path.length() > 0) {
|
|
|
image = cv::imread(jpg_path.toStdString());
|
|
|
}
|
|
|
else {
|
|
|
image = cv::imread("alg_jd.jpg");
|
|
|
}
|
|
|
//ʶ±ðÒ»ÕÅͼ£¬²âÊÔÄ£ÐÍÊÇ·ñÕýÈ·£¬²¢ÇÒÍê³ÉGPUÊý¾Ý¼ÓÔØ
|
|
|
if (!image.data) return false; //ÅжϲâÊÔͼƬÊÇ·ñÕý³£¶ÁÈ¡
|
|
|
std::vector<std::pair<int, cv::Rect> > results;
|
|
|
detect(image, image, results);
|
|
|
if (results.size() > 0)
|
|
|
return true; //¼ì²âµ½Ä¿±ê£¬Ôò³õʼ»¯³É¹¦
|
|
|
else
|
|
|
return false; //·ñÔò³õʼ»¯Ê§°Ü
|
|
|
}
|
|
|
|
|
|
bool AlgJd::test_detect()
|
|
|
{
|
|
|
cv::Mat m1;
|
|
|
m1 = cv::Mat(544, 728, CV_8UC3, cv::Scalar(0, 0, 0));
|
|
|
std::vector<std::pair<int, cv::Rect> > results;
|
|
|
double t = (double)cv::getTickCount();
|
|
|
detect(m1, m1, results);
|
|
|
t = ((double)cv::getTickCount() - t) / cv::getTickFrequency();
|
|
|
DEBUG(" test_detect time process:%f\n", t);
|
|
|
return true;
|
|
|
}
|
|
|
|
|
|
bool AlgJd::test_detect_batcht(int shoot)
|
|
|
{
|
|
|
std::vector<cv::Mat> vec_in;
|
|
|
std::vector<cv::Mat> vec_out;
|
|
|
std::vector<std::vector<std::pair<int, cv::Rect> > > vec_results;
|
|
|
cv::Mat m1, m2, m3;
|
|
|
m1 = cv::Mat(544, 728, CV_8UC3, cv::Scalar(0, 0, 0));
|
|
|
m2 = cv::Mat(544, 728, CV_8UC3, cv::Scalar(0, 0, 0));
|
|
|
m3 = cv::Mat(544, 728, CV_8UC3, cv::Scalar(0, 0, 0));
|
|
|
|
|
|
double t = (double)cv::getTickCount();
|
|
|
switch(shoot){
|
|
|
case 1:{
|
|
|
std::vector<std::pair<int, cv::Rect> > results;
|
|
|
detect(m1, m1, results);
|
|
|
break;
|
|
|
}
|
|
|
case 3:vec_in.push_back(m3);
|
|
|
case 2:vec_in.push_back(m2);
|
|
|
default:{
|
|
|
vec_in.push_back(m1);
|
|
|
detect_batch(vec_in, vec_out, vec_results);
|
|
|
}
|
|
|
}
|
|
|
t = ((double)cv::getTickCount() - t) / cv::getTickFrequency();
|
|
|
DEBUG(" test_detect_batcht time process:%f\n", t);
|
|
|
return true;
|
|
|
}
|
|
|
|
|
|
int AlgJd::detect(cv::Mat& _frame, cv::Mat &out, std::vector<std::pair<int, cv::Rect> > &results)
|
|
|
{
|
|
|
cv::Mat frame = _frame.clone();
|
|
|
cv::Mat image_clone=frame.clone();
|
|
|
// Process frames.
|
|
|
// Create a 4D blob from a frame.
|
|
|
cv::Mat blob;
|
|
|
cv::dnn::blobFromImage(frame, blob, 1/255.0, cv::Size(inpWidth, inpHeight), cv::Scalar(0,0,0), true, false);
|
|
|
|
|
|
//Sets the input to the network
|
|
|
net.setInput(blob);
|
|
|
|
|
|
|
|
|
// Runs the forward pass to get output of the output layers
|
|
|
std::vector<cv::Mat> outs;
|
|
|
net.forward(outs, getOutputsNames(net));
|
|
|
|
|
|
|
|
|
// Remove the bounding boxes with low confidence
|
|
|
post_process(frame, outs, results);
|
|
|
|
|
|
|
|
|
// Write the frame with the detection boxes
|
|
|
cv::Mat detectedFrame;
|
|
|
frame.convertTo(detectedFrame, CV_8U);
|
|
|
//show detectedFrame
|
|
|
out=detectedFrame.clone();
|
|
|
return results.size();
|
|
|
}
|
|
|
|
|
|
// Get the names of the output layers
|
|
|
static std::vector<cv::String> getOutputsNames(const cv::dnn::Net& net)
|
|
|
{
|
|
|
static std::vector<cv::String> names;
|
|
|
if (names.empty())
|
|
|
{
|
|
|
//Get the indices of the output layers, i.e. the layers with unconnected outputs
|
|
|
std::vector<int> outLayers = net.getUnconnectedOutLayers();
|
|
|
|
|
|
//get the names of all the layers in the network
|
|
|
std::vector<cv::String> layersNames = net.getLayerNames();
|
|
|
|
|
|
// Get the names of the output layers in names
|
|
|
names.resize(outLayers.size());
|
|
|
for (size_t i = 0; i < outLayers.size(); ++i)
|
|
|
names[i] = layersNames[outLayers[i] - 1];
|
|
|
}
|
|
|
return names;
|
|
|
}
|
|
|
|
|
|
static void post_process(cv::Mat& frame, std::vector<cv::Mat>& outs, std::vector<std::pair<int, cv::Rect> > &results)
|
|
|
{
|
|
|
std::vector<int> classIds;
|
|
|
std::vector<float> confidences;
|
|
|
std::vector<cv::Rect> boxes;
|
|
|
|
|
|
for (size_t i = 0; i < outs.size(); ++i)
|
|
|
{
|
|
|
// Scan through all the bounding boxes output from the network and keep only the
|
|
|
// ones with high confidence scores. Assign the box's class label as the class
|
|
|
// with the highest score for the box.
|
|
|
float* data = (float*)outs[i].data;
|
|
|
for (int j = 0; j < outs[i].rows; ++j, data += outs[i].cols)
|
|
|
{
|
|
|
cv::Mat scores = outs[i].row(j).colRange(5, outs[i].cols);
|
|
|
cv::Point classIdPoint;
|
|
|
double confidence;
|
|
|
// Get the value and location of the maximum score
|
|
|
cv::minMaxLoc(scores, 0, &confidence, 0, &classIdPoint);
|
|
|
|
|
|
if (confidence > confThreshold)
|
|
|
{
|
|
|
int centerX = (int)(data[0] * frame.cols);
|
|
|
int centerY = (int)(data[1] * frame.rows);
|
|
|
int width = (int)(data[2] * frame.cols);
|
|
|
int height = (int)(data[3] * frame.rows);
|
|
|
int left = centerX - width / 2;
|
|
|
int top = centerY - height / 2;
|
|
|
|
|
|
classIds.push_back(classIdPoint.x);
|
|
|
confidences.push_back((float)confidence);
|
|
|
boxes.push_back(cv::Rect(left, top, width, height));
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
|
|
|
|
|
|
// Perform non maximum suppression to eliminate redundant overlapping boxes with
|
|
|
// lower confidences
|
|
|
std::vector<int> indices;
|
|
|
cv::dnn::NMSBoxes(boxes, confidences, confThreshold, nmsThreshold, indices);
|
|
|
|
|
|
for (size_t i = 0; i < indices.size(); ++i)
|
|
|
{
|
|
|
int idx = indices[i];
|
|
|
cv::Rect box = boxes[idx];
|
|
|
drawPred(classIds[idx], confidences[idx], box.x, box.y,
|
|
|
box.x + box.width, box.y + box.height, frame);
|
|
|
results.push_back(std::make_pair(classIds[idx], box));
|
|
|
}
|
|
|
}
|
|
|
|
|
|
// Draw the predicted bounding box
|
|
|
static void drawPred(int classId, float conf, int left, int top, int right, int bottom, cv::Mat& frame)
|
|
|
{
|
|
|
cv::Scalar color;
|
|
|
if(classId==0)
|
|
|
color = cv::Scalar(0,255,0);
|
|
|
else if(classId == 1)
|
|
|
color = cv::Scalar(255,0,0);
|
|
|
else
|
|
|
color = cv::Scalar(0, 255, 0);
|
|
|
//Draw a rectangle displaying the bounding box
|
|
|
cv::rectangle(frame, cv::Point(left, top), cv::Point(right, bottom), color, 4);
|
|
|
|
|
|
//Get the label for the class name and its confidence
|
|
|
std::string label = cv::format("%.2f%%", (conf*100));///
|
|
|
if (!classes.empty())
|
|
|
{
|
|
|
CV_Assert(classId < (int)classes.size());
|
|
|
label = classes[classId] + ": " + label;
|
|
|
}
|
|
|
else
|
|
|
{
|
|
|
std::cout<<"classes is empty..."<<std::endl;
|
|
|
}
|
|
|
|
|
|
//Display the label at the top of the bounding box
|
|
|
int baseLine;
|
|
|
cv::Size labelSize = cv::getTextSize(label, cv::FONT_HERSHEY_SIMPLEX, 1, 1, &baseLine);
|
|
|
top = std::max(top, labelSize.height);
|
|
|
cv::putText(frame, label, cv::Point(left, top-5), cv::FONT_HERSHEY_SIMPLEX, 0.8, color, 1);
|
|
|
}
|
|
|
|
|
|
void AlgJd::detect_batch(std::vector<cv::Mat>& vec_in, std::vector<cv::Mat> &vec_out, std::vector<std::vector<std::pair<int, cv::Rect> > > &vec_results)
|
|
|
{
|
|
|
cv::Mat blobs;
|
|
|
std::vector<cv::Mat> image;
|
|
|
|
|
|
cv::dnn::blobFromImages(vec_in, blobs, 1 / 255.0, cv::Size(inpWidth, inpHeight), cv::Scalar(0, 0, 0), true, false);
|
|
|
|
|
|
//Sets the input to the network
|
|
|
net.setInput(blobs);
|
|
|
|
|
|
// Runs the forward pass to get output of the output layers
|
|
|
std::vector<cv::Mat> outs;
|
|
|
net.forward(outs, getOutputsNames(net));
|
|
|
|
|
|
for (int i = 0; i < vec_in.size(); i++)
|
|
|
{
|
|
|
image.push_back(vec_in[i].clone());
|
|
|
}
|
|
|
|
|
|
// Remove the bounding boxes with low confidence
|
|
|
post_process_batch(image, outs, vec_results);
|
|
|
|
|
|
|
|
|
// Write the frame with the detection boxes
|
|
|
for (int i = 0; i < vec_in.size(); i++)
|
|
|
{
|
|
|
cv::Mat detectedFrame, out;
|
|
|
image[i].convertTo(detectedFrame, CV_8U);
|
|
|
out = detectedFrame.clone();
|
|
|
vec_out.push_back(out);
|
|
|
}
|
|
|
}
|
|
|
|
|
|
static void post_process_batch(std::vector<cv::Mat>& vec_frame, std::vector<cv::Mat>& outs, std::vector<std::vector<std::pair<int, cv::Rect> > > &vec_results)
|
|
|
{
|
|
|
int batch = vec_frame.size();
|
|
|
for (int k = 0; k < batch; k++)
|
|
|
{
|
|
|
std::vector<int> classIds;
|
|
|
std::vector<float> confidences;
|
|
|
std::vector<cv::Rect> boxes;
|
|
|
//std::cout << "outs.size()\t" << outs.size() << std::endl;
|
|
|
//std::cout << "Type\t" << outs[0].type() << std::endl;
|
|
|
|
|
|
for (size_t i = 0; i < outs.size(); ++i)
|
|
|
{
|
|
|
//std::cout << "outs.dims\t" << outs[i].dims << std::endl;
|
|
|
//std::cout << "outs[" << i << "].rows\t" << outs[i].size[0] << std::endl;
|
|
|
//std::cout << "outs[" << i << "].cols\t" << outs[i].size[1] << std::endl;
|
|
|
//std::cout << "outs[" << i << "].cols\t" << outs[i].size[2] << std::endl;
|
|
|
// Scan through all the bounding boxes output from the network and keep only the
|
|
|
// ones with high confidence scores. Assign the box's class label as the class
|
|
|
//
|
|
|
cv::Mat m0(outs[i].size[1], outs[i].size[2], CV_32F, outs[i].data + outs[i].step[0] * k);
|
|
|
// with the highest score for the box.
|
|
|
float* data = (float*)m0.data;
|
|
|
for (int j = 0; j < m0.rows; ++j, data += m0.cols)
|
|
|
{
|
|
|
cv::Mat scores = m0.row(j).colRange(5, m0.cols);
|
|
|
cv::Point classIdPoint;
|
|
|
double confidence;
|
|
|
// Get the value and location of the maximum score
|
|
|
cv::minMaxLoc(scores, 0, &confidence, 0, &classIdPoint);
|
|
|
|
|
|
if (confidence > confThreshold)
|
|
|
{
|
|
|
int centerX = (int)(data[0] * vec_frame[k].cols);
|
|
|
int centerY = (int)(data[1] * vec_frame[k].rows);
|
|
|
int width = (int)(data[2] * vec_frame[k].cols);
|
|
|
int height = (int)(data[3] * vec_frame[k].rows);
|
|
|
int left = centerX - width / 2;
|
|
|
int top = centerY - height / 2;
|
|
|
|
|
|
classIds.push_back(classIdPoint.x);
|
|
|
confidences.push_back((float)confidence);
|
|
|
boxes.push_back(cv::Rect(left, top, width, height));
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
|
|
|
// Perform non maximum suppression to eliminate redundant overlapping boxes with
|
|
|
// lower confidences
|
|
|
std::vector<int> indices;
|
|
|
cv::dnn::NMSBoxes(boxes, confidences, confThreshold, nmsThreshold, indices);
|
|
|
|
|
|
std::vector<std::pair<int, cv::Rect> > results;
|
|
|
for (size_t i = 0; i < indices.size(); ++i)
|
|
|
{
|
|
|
int idx = indices[i];
|
|
|
cv::Rect box = boxes[idx];
|
|
|
//ʶ±ð¿ò¿í¶È´óÓÚ15ÏÔʾʶ±ð£¬Ð¡ÓÚÈÏΪÎÞ½ºµã£¬NG´¦Àí
|
|
|
if (box.width > 15)
|
|
|
{
|
|
|
drawPred(classIds[idx], confidences[idx], box.x, box.y,
|
|
|
box.x + box.width, box.y + box.height, vec_frame[k]);
|
|
|
results.push_back(std::make_pair(classIds[idx], box));
|
|
|
}
|
|
|
|
|
|
}
|
|
|
vec_results.push_back(results);
|
|
|
}
|
|
|
}
|