gitee/face-search
1
0
Fork 0
mirror of https://gitee.com/open-visual/face-search.git synced 2025-07-25 19:41:42 +08:00
Code Issues Packages Projects Releases Wiki Activity

update:修改PCN的网络推理代码，修复存在的可能内存泄露

Browse Source
This commit is contained in:
divenswu 2022-11-29 22:51:45 +08:00
parent 5837776bb1
commit 940fb3c8b2
5 changed files with 465 additions and 273 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
face-search-core/src/main/java/com/visual/face/search/core/domain/ImageMat.java
View File

@ -681,8 +681,12 @@ public class ImageMat implements Serializable {
*/
public void release(){
if(this.mat != null){
this.mat.release();
this.mat = null;
try {
this.mat.release();
this.mat = null;
}catch (Exception e){
e.printStackTrace();
}
}
}
}

44
face-search-core/src/main/java/com/visual/face/search/core/domain/Mats.java Normal file
View File

@ -0,0 +1,44 @@
package com.visual.face.search.core.domain;
import org.opencv.core.Mat;
import java.util.ArrayList;
public class Mats extends ArrayList<Mat> {
public static Mats build(){
return new Mats();
}
public Mats append(Mat mat){
this.add(mat);
return this;
}
public Mats clone(){
Mats mats = new Mats();
for(Mat mat : this){
mats.add(mat.clone());
}
return mats;
}
public void release(){
if(this.isEmpty()){
return;
}
for(Mat mat : this){
if(null != mat){
try {
mat.release();
}catch (Exception e){
e.printStackTrace();
}finally {
mat = null;
}
}
}
this.clear();
}
}

588
face-search-core/src/main/java/com/visual/face/search/core/models/PcnNetworkFaceDetection.java
View File

@ -8,6 +8,8 @@ import com.visual.face.search.core.base.BaseOnnxInfer;
import com.visual.face.search.core.base.FaceDetection;
import com.visual.face.search.core.domain.FaceInfo;
import com.visual.face.search.core.domain.ImageMat;
import com.visual.face.search.core.domain.Mats;
import com.visual.face.search.core.utils.ReleaseUtil;
import org.opencv.core.*;
import org.opencv.imgproc.Imgproc;
@ -60,114 +62,116 @@ public class PcnNetworkFaceDetection extends BaseOnnxInfer implements FaceDetect
public List<FaceInfo> inference(ImageMat image, float scoreTh, float iouTh, Map<String, Object> params) {
Mat mat = null;
Mat imgPad = null;
Mat resizeMat = null;
ImageMat imageMat = image.clone();
try {
mat = imageMat.toCvMat();
imgPad = pad_img_not_release_mat(mat);
float scale = (float) ((mat.size().height > mat.size().width) ? mat.size().width/512: mat.size().height/512);
resizeMat = resize_img_release_mat(mat.clone(), scale);
imgPad = pad_img_release_mat(resizeMat.clone());
float[] iouThs = iouTh <= 0 ? defIouThs : new float[]{iouTh, iouTh, 0.3f};
float[] scoreThs = scoreTh <= 0 ? defScoreThs : new float[]{0.375f * scoreTh, 0.5f * scoreTh, scoreTh};
List<Window2> willis = detect(this.getSessions(), mat, imgPad, scoreThs, iouThs);
return trans_window(mat, imgPad, willis);
List<Window2> willis = detect_release_mat(this.getSessions(), resizeMat.clone(), imgPad.clone(), scoreThs, iouThs);
return trans_window(resizeMat, imgPad, willis, scale);
} catch (Exception e) {
throw new RuntimeException(e);
}finally {
if(null != mat){
mat.release();
}
if(null != imgPad){
imgPad.release();
}
if(null != imageMat){
imageMat.release();
}
ReleaseUtil.release(imgPad, resizeMat, mat);
ReleaseUtil.release(imageMat);
}
}
/********************************分割线*************************************/
private static Mat pad_img_not_release_mat(Mat mat){
int row = Math.min((int)(mat.size().height * 0.2), 100);
int col = Math.min((int)(mat.size().width * 0.2), 100);
Mat dst = new Mat();
Core.copyMakeBorder(mat, dst, row, row, col, col, Core.BORDER_CONSTANT);
return dst;
private static Mat pad_img_release_mat(Mat mat){
try {
int row = Math.min((int)(mat.size().height * 0.2), 100);
int col = Math.min((int)(mat.size().width * 0.2), 100);
Mat dst = new Mat();
Core.copyMakeBorder(mat, dst, row, row, col, col, Core.BORDER_CONSTANT);
return dst;
}finally {
ReleaseUtil.release(mat);
}
}
private static Mat resize_img(Mat mat, float scale){
double h = mat.size().height;
double w = mat.size().width;
int h_ = (int) (h / scale);
int w_ = (int) (w / scale);
Mat matF32 = new Mat();
if(mat.type() != CvType.CV_32FC3){
mat.convertTo(matF32, CvType.CV_32FC3);
}else{
mat.copyTo(matF32);
private static Mat resize_img_release_mat(Mat mat, float scale){
Mat matF32 = null;
try{
double h = mat.size().height;
double w = mat.size().width;
int h_ = (int) (h / scale);
int w_ = (int) (w / scale);
matF32 = new Mat();
if(mat.type() != CvType.CV_32FC3){
mat.convertTo(matF32, CvType.CV_32FC3);
}else{
mat.copyTo(matF32);
}
Mat dst = new Mat();
Imgproc.resize(matF32, dst, new Size(w_, h_), 0,0, Imgproc.INTER_NEAREST);
return dst;
}finally {
ReleaseUtil.release(matF32, mat);
}
Mat dst = new Mat();
Imgproc.resize(matF32, dst, new Size(w_, h_), 0,0, Imgproc.INTER_NEAREST);
mat.release();
matF32.release();
return dst;
}
private static Mat preprocess_img(Mat mat, int dim){
Mat matTmp = new Mat();
if(dim > 0){
Imgproc.resize(mat, matTmp, new Size(dim, dim), 0, 0, Imgproc.INTER_NEAREST);
}else{
mat.copyTo(matTmp);
private static Mat preprocess_img_release_mat(Mat mat, int dim){
Mat matTmp = null;
Mat matF32 = null;
try {
//resize
matTmp = new Mat();
if(dim > 0){
Imgproc.resize(mat, matTmp, new Size(dim, dim), 0, 0, Imgproc.INTER_NEAREST);
}else{
mat.copyTo(matTmp);
}
//格式转化
matF32 = new Mat();
if(mat.type() != CvType.CV_32FC3){
matTmp.convertTo(matF32, CvType.CV_32FC3);
}else{
matTmp.copyTo(matF32);
}
//减法运算
Mat dst = new Mat();
Core.subtract(matF32, new Scalar(104, 117, 123), dst);
return dst;
}finally {
ReleaseUtil.release(matF32, matTmp, mat);
}
//格式转化
Mat matF32 = new Mat();
if(mat.type() != CvType.CV_32FC3){
matTmp.convertTo(matF32, CvType.CV_32FC3);
}else{
matTmp.copyTo(matF32);
}
Mat dst = new Mat();
Core.subtract(matF32, new Scalar(104, 117, 123), dst);
mat.release();
matTmp.release();
matF32.release();
return dst;
}
private static OnnxTensor set_input(Mat mat){
private static OnnxTensor set_input_release_mat(Mat mat){
Mat dst = null;
try {
dst = new Mat();
mat.copyTo(dst);
return ImageMat.fromCVMat(dst).to4dFloatOnnxTensorAndDoReleaseMat(true);
}finally {
if(null != dst){
dst.release();
}
ReleaseUtil.release(dst, mat);
}
}
private static OnnxTensor set_input(List<Mat> mats){
float[][][][] arrays = new float[mats.size()][][][];
for(int i=0; i< mats.size(); i++){
Mat dst = new Mat();
mats.get(i).copyTo(dst);
float[][][][] array = ImageMat.fromCVMat(dst).to4dFloatArrayAndDoReleaseMat(true);
arrays[i] = array[0];
dst.release();
}
private static OnnxTensor set_input_release_mat(Mats mats){
try {
float[][][][] arrays = new float[mats.size()][][][];
for(int i=0; i< mats.size(); i++){
float[][][][] array = ImageMat.fromCVMat(mats.get(i)).to4dFloatArrayAndDoReleaseMat(true);
arrays[i] = array[0];
}
return OnnxTensor.createTensor(OrtEnvironment.getEnvironment(), arrays);
}catch (Exception e){
throw new RuntimeException(e);
}finally {
ReleaseUtil.release(mats);
}
}
private static boolean legal(int x, int y, Mat mat){
if(0 <= x && x < mat.size().width && 0 <= y && y< mat.size().height){
private static boolean legal(int x, int y, Size size){
if(0 <= x && x < size.width && 0 <= y && y< size.height){
return true;
}else{
return false;
@ -191,87 +195,106 @@ public class PcnNetworkFaceDetection extends BaseOnnxInfer implements FaceDetect
}
private static List<Window2> NMS(List<Window2> winlist, boolean local, float threshold){
if(winlist==null || winlist.isEmpty()){
return new ArrayList<>();
}
//排序
Collections.sort(winlist);
int [] flag = new int[winlist.size()];
for(int i=0; i< winlist.size(); i++){
if(flag[i] > 0){
continue;
try{
if(winlist==null || winlist.isEmpty()){
return new ArrayList<>();
}
for(int j=i+1; j<winlist.size(); j++){
if(local && Math.abs(winlist.get(i).scale - winlist.get(j).scale) > EPS){
//排序
Collections.sort(winlist);
int [] flag = new int[winlist.size()];
for(int i=0; i< winlist.size(); i++){
if(flag[i] > 0){
continue;
}
if(IoU(winlist.get(i), winlist.get(j)) > threshold){
flag[j] = 1;
for(int j=i+1; j<winlist.size(); j++){
if(local && Math.abs(winlist.get(i).scale - winlist.get(j).scale) > EPS){
continue;
}
if(IoU(winlist.get(i), winlist.get(j)) > threshold){
flag[j] = 1;
}
}
}
}
List<Window2> list = new ArrayList<>();
for(int i=0; i< flag.length; i++){
if(flag[i] == 0){
list.add(winlist.get(i));
List<Window2> list = new ArrayList<>();
for(int i=0; i< flag.length; i++){
if(flag[i] == 0){
list.add(winlist.get(i));
}
}
return list;
}finally {
if(null != winlist && !winlist.isEmpty()){
winlist.clear();
}
}
return list;
}
private static List<Window2> deleteFP(List<Window2> winlist){
if (winlist == null || winlist.isEmpty()) {
return new ArrayList<>();
}
//排序
Collections.sort(winlist);
int [] flag = new int[winlist.size()];
for(int i=0; i< winlist.size(); i++){
if(flag[i] > 0){
continue;
try {
if(null == winlist || winlist.isEmpty()) {
return new ArrayList<>();
}
for(int j=i+1; j<winlist.size(); j++){
Window2 win = winlist.get(j);
if(inside(win.x, win.y, winlist.get(i)) && inside(win.x + win.w - 1, win.y + win.h - 1, winlist.get(i))){
flag[j] = 1;
//排序
Collections.sort(winlist);
int [] flag = new int[winlist.size()];
for(int i=0; i< winlist.size(); i++){
if(flag[i] > 0){
continue;
}
for(int j=i+1; j<winlist.size(); j++){
Window2 win = winlist.get(j);
if(inside(win.x, win.y, winlist.get(i)) && inside(win.x + win.w - 1, win.y + win.h - 1, winlist.get(i))){
flag[j] = 1;
}
}
}
}
List<Window2> list = new ArrayList<>();
for(int i=0; i< flag.length; i++){
if(flag[i] == 0){
list.add(winlist.get(i));
List<Window2> list = new ArrayList<>();
for(int i=0; i< flag.length; i++){
if(flag[i] == 0){
list.add(winlist.get(i));
}
}
return list;
}finally {
if(null != winlist && !winlist.isEmpty()){
winlist.clear();
}
}
return list;
}
private static List<FaceInfo> trans_window(Mat img, Mat imgPad, List<Window2> winlist){
int row = (imgPad.size(0) - img.size(0)) / 2;
int col = (imgPad.size(1) - img.size(1)) / 2;
private static List<FaceInfo> trans_window(Mat img, Mat imgPad, List<Window2> winlist, float scale){
List<FaceInfo> ret = new ArrayList<>();
for(Window2 win : winlist){
if( win.w > 0 && win.h > 0){
int x1 = win.x - col;
int y1 = win.y - row;
int x2 = win.x - col + win.w;
int y2 = win.y - row + win.w;
int angle = (win.angle + 360) % 360;
//扩展人脸高度
float rw = 0f;
float rh = 0.1f;
int w = Math.abs(x2 - x1);
int h = Math.abs(y2 - y1);
x1 = Math.max(x1 - (int)(w * rw), 1);
y1 = Math.max(y1 - (int)(h * rh), 1);
x2 = Math.min(x2 + (int)(w * rw), img.size(1)-1);
y2 = Math.min(y2 + (int)(h * rh), img.size(0)-1);
//构建人脸信息
FaceInfo faceInfo = FaceInfo.build(win.conf, angle, FaceInfo.FaceBox.build(x1, y1, x2, y2), FaceInfo.Points.build());
ret.add(faceInfo);
try {
int row = (imgPad.size(0) - img.size(0)) / 2;
int col = (imgPad.size(1) - img.size(1)) / 2;
for(Window2 win : winlist){
if( win.w > 0 && win.h > 0){
int x1 = win.x - col;
int y1 = win.y - row;
int x2 = win.x - col + win.w;
int y2 = win.y - row + win.w;
int angle = (win.angle + 360) % 360;
//扩展人脸高度
float rw = 0f;
float rh = 0.1f;
int w = Math.abs(x2 - x1);
int h = Math.abs(y2 - y1);
x1 = Math.max(Float.valueOf((x1 - (int)(w * rw)) * scale).intValue(), 1);
y1 = Math.max(Float.valueOf((y1 - (int)(h * rh)) * scale).intValue(), 1);
x2 = Math.min(Float.valueOf((x2 + (int)(w * rw)) * scale).intValue(), Float.valueOf((img.size(1)) * scale).intValue()-1);
y2 = Math.min(Float.valueOf((y2 + (int)(h * rh)) * scale).intValue(), Float.valueOf((img.size(0)) * scale).intValue()-1);
//构建人脸信息
FaceInfo faceInfo = FaceInfo.build(win.conf, angle, FaceInfo.FaceBox.build(x1, y1, x2, y2), FaceInfo.Points.build());
ret.add(faceInfo);
}
}
return ret;
}finally {
if(null != winlist && !winlist.isEmpty()){
winlist.clear();
}
}
return ret;
}
/**
@ -284,34 +307,30 @@ public class PcnNetworkFaceDetection extends BaseOnnxInfer implements FaceDetect
* @throws OrtException
* @throws IOException
*/
private static List<Window2> stage1(Mat img, Mat imgPad, OrtSession net, float thres) throws RuntimeException {
int netSize = 24;
float curScale = minFace_ / netSize;
int row = (int) ((imgPad.size().height - img.size().height) / 2);
int col = (int) ((imgPad.size().width - img.size().width) / 2);
private static List<Window2> stage1_release_mat(Mat img, Mat imgPad, OrtSession net, float thres) throws RuntimeException {
Mat img_resized = null;
OnnxTensor net_input1 = null;
OrtSession.Result output = null;
List<Window2> winlist = new ArrayList<>();
try {
img_resized = resize_img(img.clone(), curScale);
//获取必要参数
int netSize = 24;
float curScale = minFace_ / netSize;
int row = (int) ((imgPad.size().height - img.size().height) / 2);
int col = (int) ((imgPad.size().width - img.size().width) / 2);
img_resized = resize_img_release_mat(img.clone(), curScale);
//循环处理
while(Math.min(img_resized.size().height, img_resized.size().width) >= netSize){
img_resized = preprocess_img(img_resized, 0);
net_input1 = set_input(img_resized);
img_resized = preprocess_img_release_mat(img_resized, 0);
net_input1 = set_input_release_mat(img_resized.clone());
//推理网络
output = net.run(Collections.singletonMap(net.getInputNames().iterator().next(), net_input1));
float[][][][] cls_prob = (float[][][][]) output.get(0).getValue();
float[][][][] rotate = (float[][][][]) output.get(1).getValue();
float[][][][] bbox = (float[][][][]) output.get(2).getValue();
//关闭对象
if(null != net_input1){
net_input1.close();
net_input1 = null;
}
if(null != output){
output.close();
output = null;
}
ReleaseUtil.release(output); output = null;
ReleaseUtil.release(net_input1); net_input1 = null;
//计算业务逻辑
float w = netSize * curScale;
for(int i=0; i< cls_prob[0][0].length; i++){
@ -323,7 +342,7 @@ public class PcnNetworkFaceDetection extends BaseOnnxInfer implements FaceDetect
int rx = (int)(j * curScale * stride_ - 0.5 * sn * w + sn * xn * w + 0.5 * w) + col;
int ry = (int)(i * curScale * stride_ - 0.5 * sn * w + sn * yn * w + 0.5 * w) + row;
int rw = (int)(w * sn);
if (legal(rx, ry, imgPad) && legal(rx + rw - 1, ry + rw - 1, imgPad)){
if (legal(rx, ry, imgPad.size()) && legal(rx + rw - 1, ry + rw - 1, imgPad.size())){
if (rotate[0][1][i][j] > 0.5){
winlist.add(new Window2(rx, ry, rw, rw, 0, curScale, cls_prob[0][1][i][j]));
}else{
@ -333,21 +352,15 @@ public class PcnNetworkFaceDetection extends BaseOnnxInfer implements FaceDetect
}
}
}
img_resized = resize_img(img_resized, scale_);
img_resized = resize_img_release_mat(img_resized, scale_);
curScale = (float) (img.size().height / img_resized.size().height);
}
}catch (Exception e){
throw new RuntimeException(e);
}finally {
if(null != net_input1){
net_input1.close();
}
if(null != output){
output.close();
}
if(null != img_resized){
img_resized.release();
}
ReleaseUtil.release(output);
ReleaseUtil.release(net_input1);
ReleaseUtil.release(img_resized, imgPad, img);
}
//返回
return winlist;
@ -364,56 +377,58 @@ public class PcnNetworkFaceDetection extends BaseOnnxInfer implements FaceDetect
* @return
* @throws OrtException
*/
private static List<Window2> stage2(Mat img, Mat img180, OrtSession net, float thres, int dim, List<Window2> winlist) throws OrtException {
private static List<Window2> stage2_release_mat(Mat img, Mat img180, OrtSession net, float thres, int dim, List<Window2> winlist) throws OrtException {
if(winlist==null || winlist.isEmpty()){
return new ArrayList<>();
}
//逻辑处理
float[][] bbox;
float[][] rotate;
float[][] cls_prob;
Mats datalist = null;
OnnxTensor input = null;
OrtSession.Result output = null;
Size size = img.size();
int height = img.size(0);
List<Mat> datalist = new ArrayList<>();
for(Window2 win : winlist){
if(Math.abs(win.angle) < EPS){
Mat cloneMat = img.clone();
Mat corp = new Mat(cloneMat, new Rect(win.x, win.y, win.w, win.h));
Mat corp1 = preprocess_img(corp, dim);
datalist.add(corp1);
if(null != corp){
corp.release();
}
if(null != cloneMat){
cloneMat.release();
}
}else{
int y2 = win.y + win.h - 1;
int y = height - 1 - y2;
Mat cloneMat = img180.clone();
Mat corp = new Mat(cloneMat, new Rect(win.x, y, win.w, win.h));
Mat corp1 = preprocess_img(corp, dim);
datalist.add(corp1);
if(null != corp){
corp.release();
}
if(null != cloneMat){
cloneMat.release();
try {
datalist = Mats.build();
for(Window2 win : winlist){
if(Math.abs(win.angle) < EPS){
Mat corp = null;
try {
corp = new Mat(img, new Rect(win.x, win.y, win.w, win.h));
Mat preprocess = preprocess_img_release_mat(corp.clone(), dim);
datalist.add(preprocess);
}finally {
ReleaseUtil.release(corp);
}
}else{
Mat corp = null;
try {
int y2 = win.y + win.h - 1;
int y = height - 1 - y2;
corp = new Mat(img180, new Rect(win.x, y, win.w, win.h));
Mat preprocess = preprocess_img_release_mat(corp.clone(), dim);
datalist.add(preprocess);
}finally {
ReleaseUtil.release(corp);
}
}
}
//模型推理
input = set_input_release_mat(datalist.clone());
output = net.run(Collections.singletonMap(net.getInputNames().iterator().next(), input));
cls_prob = (float[][]) output.get(0).getValue();
rotate = (float[][]) output.get(1).getValue();
bbox = (float[][]) output.get(2).getValue();
}finally {
ReleaseUtil.release(datalist);
ReleaseUtil.release(output);
ReleaseUtil.release(input);
ReleaseUtil.release(img180, img);
output = null; input = null;
}
OnnxTensor net_input = set_input(datalist);
OrtSession.Result output = net.run(Collections.singletonMap(net.getInputNames().iterator().next(), net_input));
float[][] cls_prob = (float[][]) output.get(0).getValue();
float[][] rotate = (float[][]) output.get(1).getValue();
float[][] bbox = (float[][]) output.get(2).getValue();
//关闭对象
for(Mat mat : datalist){
mat.release();
}
if(null != net_input){
net_input.close();
}
if(null != output){
output.close();
}
//再次后处理数据
List<Window2> ret = new ArrayList<>();
for(int i=0; i<winlist.size(); i++){
if(cls_prob[i][1] > thres){
@ -439,7 +454,7 @@ public class PcnNetworkFaceDetection extends BaseOnnxInfer implements FaceDetect
}
}
if(legal(x, y, img) && legal(x + w - 1, y + w - 1, img)){
if(legal(x, y, size) && legal(x + w - 1, y + w - 1, size)){
int angle = 0;
if(Math.abs(winlist.get(i).angle) < EPS){
if(maxRotateIndex == 0){
@ -479,49 +494,79 @@ public class PcnNetworkFaceDetection extends BaseOnnxInfer implements FaceDetect
* @return
* @throws OrtException
*/
private static List<Window2> stage3(Mat imgPad, Mat img180, Mat img90, Mat imgNeg90, OrtSession net, float thres, int dim, List<Window2> winlist) throws OrtException {
private static List<Window2> stage3_release_mat(Mat imgPad, Mat img180, Mat img90, Mat imgNeg90, OrtSession net, float thres, int dim, List<Window2> winlist) throws OrtException {
if (winlist == null || winlist.isEmpty()) {
return new ArrayList<>();
}
//逻辑处理
float[][] bbox;
float[][] rotate;
float[][] cls_prob;
Mats datalist = null;
OnnxTensor input = null;
OrtSession.Result output = null;
int height = imgPad.size(0);
int width = imgPad.size(1);
List<Mat> datalist = new ArrayList<>();
for(Window2 win : winlist){
if(Math.abs(win.angle) < EPS){
Mat corp = new Mat(imgPad, new Rect(win.x, win.y, win.w, win.h));
datalist.add(preprocess_img(corp, dim));
}else if(Math.abs(win.angle - 90) < EPS){
Mat corp = new Mat(img90, new Rect(win.y, win.x, win.h, win.w));
datalist.add(preprocess_img(corp, dim));
}else if(Math.abs(win.angle + 90) < EPS){
int x = win.y;
int y = width - 1 - (win.x + win.w - 1);
Mat corp = new Mat(imgNeg90, new Rect(x, y, win.w, win.h));
datalist.add(preprocess_img(corp, dim));
}else{
int y2 = win.y + win.h - 1;
int y = height - 1 - y2;
Mat corp = new Mat(img180, new Rect(win.x, y, win.w, win.h));
datalist.add(preprocess_img(corp, dim));
Size imgPadSize = imgPad.size();
Size img180Size = img180.size();
Size img90Size = img90.size();
Size imgNeg90Size = imgNeg90.size();
try {
datalist = Mats.build();
for(Window2 win : winlist){
if(Math.abs(win.angle) < EPS){
Mat corp = null;
try {
corp = new Mat(imgPad, new Rect(win.x, win.y, win.w, win.h));
datalist.add(preprocess_img_release_mat(corp.clone(), dim));
}finally {
ReleaseUtil.release(corp);
}
}else if(Math.abs(win.angle - 90) < EPS){
Mat corp = null;
try {
corp = new Mat(img90, new Rect(win.y, win.x, win.h, win.w));
datalist.add(preprocess_img_release_mat(corp.clone(), dim));
}finally {
ReleaseUtil.release(corp);
}
}else if(Math.abs(win.angle + 90) < EPS){
Mat corp = null;
try {
int x = win.y;
int y = width - 1 - (win.x + win.w - 1);
corp = new Mat(imgNeg90, new Rect(x, y, win.w, win.h));
datalist.add(preprocess_img_release_mat(corp.clone(), dim));
}finally {
ReleaseUtil.release(corp);
}
}else{
Mat corp = null;
try {
int y2 = win.y + win.h - 1;
int y = height - 1 - y2;
corp = new Mat(img180, new Rect(win.x, y, win.w, win.h));
datalist.add(preprocess_img_release_mat(corp.clone(), dim));
}finally {
ReleaseUtil.release(corp);
}
}
}
input = set_input_release_mat(datalist);
output = net.run(Collections.singletonMap(net.getInputNames().iterator().next(), input));
cls_prob = (float[][]) output.get(0).getValue();
rotate = (float[][]) output.get(1).getValue();
bbox = (float[][]) output.get(2).getValue();
}finally {
ReleaseUtil.release(datalist);
ReleaseUtil.release(output);
ReleaseUtil.release(input);
ReleaseUtil.release(imgPad, img180, img90, imgNeg90);
output = null; input = null;
}
OnnxTensor net_input = set_input(datalist);
OrtSession.Result output = net.run(Collections.singletonMap(net.getInputNames().iterator().next(), net_input));
float[][] cls_prob = (float[][]) output.get(0).getValue();
float[][] rotate = (float[][]) output.get(1).getValue();
float[][] bbox = (float[][]) output.get(2).getValue();
//关闭对象
for(Mat mat : datalist){
mat.release();
}
if(null != net_input){
net_input.close();
}
if(null != output){
output.close();
}
//模型后处理
List<Window2> ret = new ArrayList<>();
for(int i=0; i<winlist.size(); i++) {
if (cls_prob[i][1] > thres) {
@ -531,24 +576,24 @@ public class PcnNetworkFaceDetection extends BaseOnnxInfer implements FaceDetect
float cropX = winlist.get(i).x;
float cropY = winlist.get(i).y;
float cropW = winlist.get(i).w;
Mat img_tmp = imgPad;
Size img_tmp_size = imgPadSize;
if (Math.abs(winlist.get(i).angle - 180) < EPS) {
cropY = height - 1 - (cropY + cropW - 1);
img_tmp = img180;
img_tmp_size = img180Size;
}else if (Math.abs(winlist.get(i).angle - 90) < EPS) {
cropX = winlist.get(i).y;
cropY = winlist.get(i).x;
img_tmp = img90;
img_tmp_size = img90Size;
}else if (Math.abs(winlist.get(i).angle + 90) < EPS) {
cropX = winlist.get(i).y;
cropY = width - 1 - (winlist.get(i).x + winlist.get(i).w - 1);
img_tmp = imgNeg90;
img_tmp_size = imgNeg90Size;
}
int w = (int) (sn * cropW);
int x = (int) (cropX - 0.5 * sn * cropW + cropW * sn * xn + 0.5 * cropW);
int y = (int) (cropY - 0.5 * sn * cropW + cropW * sn * yn + 0.5 * cropW);
int angle = (int)(angleRange_ * rotate[i][0]);
if(legal(x, y, img_tmp) && legal(x + w - 1, y + w - 1, img_tmp)){
if(legal(x, y, img_tmp_size) && legal(x + w - 1, y + w - 1, img_tmp_size)){
if(Math.abs(winlist.get(i).angle) < EPS){
ret.add(new Window2(x, y, w, w, angle, winlist.get(i).scale, cls_prob[i][1]));
}else if(Math.abs(winlist.get(i).angle - 180) < EPS){
@ -573,32 +618,33 @@ public class PcnNetworkFaceDetection extends BaseOnnxInfer implements FaceDetect
* @throws OrtException
* @throws IOException
*/
private static List<Window2> detect(OrtSession[] sessions, Mat img, Mat imgPad, float[] scoreThs, float iouThs[]) throws OrtException, IOException {
Mat img180 = new Mat();
Core.flip(imgPad, img180, 0);
Mat img90 = new Mat();
Core.transpose(imgPad, img90);
Mat imgNeg90 = new Mat();
Core.flip(img90, imgNeg90, 0);
List<Window2> winlist = stage1(img, imgPad, sessions[0], scoreThs[0]);
winlist = NMS(winlist, true, iouThs[0]);
winlist = stage2(imgPad, img180, sessions[1], scoreThs[1], 24, winlist);
winlist = NMS(winlist, true, iouThs[1]);
winlist = stage3(imgPad, img180, img90, imgNeg90, sessions[2], scoreThs[2], 48, winlist);
winlist = NMS(winlist, false, iouThs[2]);
winlist = deleteFP(winlist);
img90.release();
img180.release();
imgNeg90.release();
return winlist;
private static List<Window2> detect_release_mat(OrtSession[] sessions, Mat img, Mat imgPad, float[] scoreThs, float iouThs[]) throws OrtException, IOException {
Mat img180 = null;
Mat img90 = null;
Mat imgNeg90 = null;
try {
//前置处理
img180 = new Mat();
Core.flip(imgPad, img180, 0);
img90 = new Mat();
Core.transpose(imgPad, img90);
imgNeg90 = new Mat();
Core.flip(img90, imgNeg90, 0);
//第一步
List<Window2> winlist = stage1_release_mat(img.clone(), imgPad.clone(), sessions[0], scoreThs[0]);
winlist = NMS(winlist, true, iouThs[0]);
//第二步
winlist = stage2_release_mat(imgPad.clone(), img180.clone(), sessions[1], scoreThs[1], 24, winlist);
winlist = NMS(winlist, true, iouThs[1]);
//第三步
winlist = stage3_release_mat(imgPad.clone(), img180.clone(), img90.clone(), imgNeg90.clone(), sessions[2], scoreThs[2], 48, winlist);
winlist = NMS(winlist, false, iouThs[2]);
//后处理
winlist = deleteFP(winlist);
return winlist;
}finally {
ReleaseUtil.release(imgNeg90, img90, img180, imgPad, img);
}
}
/**
@ -642,7 +688,7 @@ public class PcnNetworkFaceDetection extends BaseOnnxInfer implements FaceDetect
", angle=" + angle +
", scale=" + scale +
", conf=" + conf +
'}' +"\n";
"}" +"\n";
}
}
}

98
face-search-core/src/main/java/com/visual/face/search/core/utils/ReleaseUtil.java Normal file
View File

@ -0,0 +1,98 @@
package com.visual.face.search.core.utils;
import ai.onnxruntime.OnnxTensor;
import ai.onnxruntime.OrtSession;
import com.visual.face.search.core.domain.ImageMat;
import com.visual.face.search.core.domain.Mats;
import org.opencv.core.Mat;
public class ReleaseUtil {
public static void release(Mat ...mats){
for(Mat mat : mats){
if(null != mat){
try {
mat.release();
}catch (Exception e){
e.printStackTrace();
}finally {
mat = null;
}
}
}
}
public static void release(Mats mats){
if(null == mats || mats.isEmpty()){
return;
}
try {
mats.release();
}catch (Exception e){
e.printStackTrace();
}finally {
mats = null;
}
}
public static void release(ImageMat ...imageMats){
for(ImageMat imageMat : imageMats){
if(null != imageMat){
try {
imageMat.release();
}catch (Exception e){
e.printStackTrace();
}finally {
imageMat = null;
}
}
}
}
public static void release(OnnxTensor ...tensors){
if(null == tensors || tensors.length == 0){
return;
}
try {
for(OnnxTensor tensor : tensors){
try {
if(null != tensor){
tensor.close();
}
}catch (Exception e) {
e.printStackTrace();
}finally {
tensor = null;
}
}
}catch (Exception e){
e.printStackTrace();
}finally {
tensors = null;
}
}
public static void release(OrtSession.Result ...results){
if(null == results || results.length == 0){
return;
}
try {
for(OrtSession.Result result : results){
try {
if(null != result){
result.close();
}
}catch (Exception e) {
e.printStackTrace();
}finally {
result = null;
}
}
}catch (Exception e){
e.printStackTrace();
}finally {
results = null;
}
}
}

BIN
face-search-core/src/test/resources/images/faces/debug/debug_0005.jpg Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 19 KiB