diff --git a/face-search-core/src/main/java/com/visual/face/search/core/domain/ImageMat.java b/face-search-core/src/main/java/com/visual/face/search/core/domain/ImageMat.java index 2588dad..b61f459 100755 --- a/face-search-core/src/main/java/com/visual/face/search/core/domain/ImageMat.java +++ b/face-search-core/src/main/java/com/visual/face/search/core/domain/ImageMat.java @@ -681,8 +681,12 @@ public class ImageMat implements Serializable { */ public void release(){ if(this.mat != null){ - this.mat.release(); - this.mat = null; + try { + this.mat.release(); + this.mat = null; + }catch (Exception e){ + e.printStackTrace(); + } } } } diff --git a/face-search-core/src/main/java/com/visual/face/search/core/domain/Mats.java b/face-search-core/src/main/java/com/visual/face/search/core/domain/Mats.java new file mode 100644 index 0000000..eeaf94a --- /dev/null +++ b/face-search-core/src/main/java/com/visual/face/search/core/domain/Mats.java @@ -0,0 +1,44 @@ +package com.visual.face.search.core.domain; + +import org.opencv.core.Mat; + +import java.util.ArrayList; + +public class Mats extends ArrayList { + + public static Mats build(){ + return new Mats(); + } + + public Mats append(Mat mat){ + this.add(mat); + return this; + } + + public Mats clone(){ + Mats mats = new Mats(); + for(Mat mat : this){ + mats.add(mat.clone()); + } + return mats; + } + + public void release(){ + if(this.isEmpty()){ + return; + } + for(Mat mat : this){ + if(null != mat){ + try { + mat.release(); + }catch (Exception e){ + e.printStackTrace(); + }finally { + mat = null; + } + } + } + this.clear(); + } + +} diff --git a/face-search-core/src/main/java/com/visual/face/search/core/models/PcnNetworkFaceDetection.java b/face-search-core/src/main/java/com/visual/face/search/core/models/PcnNetworkFaceDetection.java index 9b6c328..44e3e9a 100755 --- a/face-search-core/src/main/java/com/visual/face/search/core/models/PcnNetworkFaceDetection.java +++ b/face-search-core/src/main/java/com/visual/face/search/core/models/PcnNetworkFaceDetection.java @@ -8,6 +8,8 @@ import com.visual.face.search.core.base.BaseOnnxInfer; import com.visual.face.search.core.base.FaceDetection; import com.visual.face.search.core.domain.FaceInfo; import com.visual.face.search.core.domain.ImageMat; +import com.visual.face.search.core.domain.Mats; +import com.visual.face.search.core.utils.ReleaseUtil; import org.opencv.core.*; import org.opencv.imgproc.Imgproc; @@ -60,114 +62,116 @@ public class PcnNetworkFaceDetection extends BaseOnnxInfer implements FaceDetect public List inference(ImageMat image, float scoreTh, float iouTh, Map params) { Mat mat = null; Mat imgPad = null; + Mat resizeMat = null; ImageMat imageMat = image.clone(); try { mat = imageMat.toCvMat(); - imgPad = pad_img_not_release_mat(mat); + float scale = (float) ((mat.size().height > mat.size().width) ? mat.size().width/512: mat.size().height/512); + resizeMat = resize_img_release_mat(mat.clone(), scale); + imgPad = pad_img_release_mat(resizeMat.clone()); float[] iouThs = iouTh <= 0 ? defIouThs : new float[]{iouTh, iouTh, 0.3f}; float[] scoreThs = scoreTh <= 0 ? defScoreThs : new float[]{0.375f * scoreTh, 0.5f * scoreTh, scoreTh}; - List willis = detect(this.getSessions(), mat, imgPad, scoreThs, iouThs); - return trans_window(mat, imgPad, willis); + List willis = detect_release_mat(this.getSessions(), resizeMat.clone(), imgPad.clone(), scoreThs, iouThs); + return trans_window(resizeMat, imgPad, willis, scale); } catch (Exception e) { throw new RuntimeException(e); }finally { - if(null != mat){ - mat.release(); - } - if(null != imgPad){ - imgPad.release(); - } - if(null != imageMat){ - imageMat.release(); - } + ReleaseUtil.release(imgPad, resizeMat, mat); + ReleaseUtil.release(imageMat); } } /********************************分割线*************************************/ - private static Mat pad_img_not_release_mat(Mat mat){ - int row = Math.min((int)(mat.size().height * 0.2), 100); - int col = Math.min((int)(mat.size().width * 0.2), 100); - Mat dst = new Mat(); - Core.copyMakeBorder(mat, dst, row, row, col, col, Core.BORDER_CONSTANT); - return dst; + private static Mat pad_img_release_mat(Mat mat){ + try { + int row = Math.min((int)(mat.size().height * 0.2), 100); + int col = Math.min((int)(mat.size().width * 0.2), 100); + Mat dst = new Mat(); + Core.copyMakeBorder(mat, dst, row, row, col, col, Core.BORDER_CONSTANT); + return dst; + }finally { + ReleaseUtil.release(mat); + } } - private static Mat resize_img(Mat mat, float scale){ - double h = mat.size().height; - double w = mat.size().width; - int h_ = (int) (h / scale); - int w_ = (int) (w / scale); - Mat matF32 = new Mat(); - if(mat.type() != CvType.CV_32FC3){ - mat.convertTo(matF32, CvType.CV_32FC3); - }else{ - mat.copyTo(matF32); + private static Mat resize_img_release_mat(Mat mat, float scale){ + Mat matF32 = null; + try{ + double h = mat.size().height; + double w = mat.size().width; + int h_ = (int) (h / scale); + int w_ = (int) (w / scale); + matF32 = new Mat(); + if(mat.type() != CvType.CV_32FC3){ + mat.convertTo(matF32, CvType.CV_32FC3); + }else{ + mat.copyTo(matF32); + } + Mat dst = new Mat(); + Imgproc.resize(matF32, dst, new Size(w_, h_), 0,0, Imgproc.INTER_NEAREST); + return dst; + }finally { + ReleaseUtil.release(matF32, mat); } - Mat dst = new Mat(); - Imgproc.resize(matF32, dst, new Size(w_, h_), 0,0, Imgproc.INTER_NEAREST); - mat.release(); - matF32.release(); - return dst; } - private static Mat preprocess_img(Mat mat, int dim){ - Mat matTmp = new Mat(); - if(dim > 0){ - Imgproc.resize(mat, matTmp, new Size(dim, dim), 0, 0, Imgproc.INTER_NEAREST); - }else{ - mat.copyTo(matTmp); + private static Mat preprocess_img_release_mat(Mat mat, int dim){ + Mat matTmp = null; + Mat matF32 = null; + try { + //resize + matTmp = new Mat(); + if(dim > 0){ + Imgproc.resize(mat, matTmp, new Size(dim, dim), 0, 0, Imgproc.INTER_NEAREST); + }else{ + mat.copyTo(matTmp); + } + //格式转化 + matF32 = new Mat(); + if(mat.type() != CvType.CV_32FC3){ + matTmp.convertTo(matF32, CvType.CV_32FC3); + }else{ + matTmp.copyTo(matF32); + } + //减法运算 + Mat dst = new Mat(); + Core.subtract(matF32, new Scalar(104, 117, 123), dst); + return dst; + }finally { + ReleaseUtil.release(matF32, matTmp, mat); } - //格式转化 - Mat matF32 = new Mat(); - if(mat.type() != CvType.CV_32FC3){ - matTmp.convertTo(matF32, CvType.CV_32FC3); - }else{ - matTmp.copyTo(matF32); - } - - Mat dst = new Mat(); - Core.subtract(matF32, new Scalar(104, 117, 123), dst); - - mat.release(); - matTmp.release(); - matF32.release(); - - return dst; } - private static OnnxTensor set_input(Mat mat){ + private static OnnxTensor set_input_release_mat(Mat mat){ Mat dst = null; try { dst = new Mat(); mat.copyTo(dst); return ImageMat.fromCVMat(dst).to4dFloatOnnxTensorAndDoReleaseMat(true); }finally { - if(null != dst){ - dst.release(); - } + ReleaseUtil.release(dst, mat); } } - private static OnnxTensor set_input(List mats){ - float[][][][] arrays = new float[mats.size()][][][]; - for(int i=0; i< mats.size(); i++){ - Mat dst = new Mat(); - mats.get(i).copyTo(dst); - float[][][][] array = ImageMat.fromCVMat(dst).to4dFloatArrayAndDoReleaseMat(true); - arrays[i] = array[0]; - dst.release(); - } + private static OnnxTensor set_input_release_mat(Mats mats){ try { + float[][][][] arrays = new float[mats.size()][][][]; + for(int i=0; i< mats.size(); i++){ + float[][][][] array = ImageMat.fromCVMat(mats.get(i)).to4dFloatArrayAndDoReleaseMat(true); + arrays[i] = array[0]; + } return OnnxTensor.createTensor(OrtEnvironment.getEnvironment(), arrays); }catch (Exception e){ throw new RuntimeException(e); + }finally { + ReleaseUtil.release(mats); } } - private static boolean legal(int x, int y, Mat mat){ - if(0 <= x && x < mat.size().width && 0 <= y && y< mat.size().height){ + private static boolean legal(int x, int y, Size size){ + if(0 <= x && x < size.width && 0 <= y && y< size.height){ return true; }else{ return false; @@ -191,87 +195,106 @@ public class PcnNetworkFaceDetection extends BaseOnnxInfer implements FaceDetect } private static List NMS(List winlist, boolean local, float threshold){ - if(winlist==null || winlist.isEmpty()){ - return new ArrayList<>(); - } - //排序 - Collections.sort(winlist); - int [] flag = new int[winlist.size()]; - for(int i=0; i< winlist.size(); i++){ - if(flag[i] > 0){ - continue; + try{ + if(winlist==null || winlist.isEmpty()){ + return new ArrayList<>(); } - for(int j=i+1; j EPS){ + //排序 + Collections.sort(winlist); + int [] flag = new int[winlist.size()]; + for(int i=0; i< winlist.size(); i++){ + if(flag[i] > 0){ continue; } - if(IoU(winlist.get(i), winlist.get(j)) > threshold){ - flag[j] = 1; + for(int j=i+1; j EPS){ + continue; + } + if(IoU(winlist.get(i), winlist.get(j)) > threshold){ + flag[j] = 1; + } } } - } - List list = new ArrayList<>(); - for(int i=0; i< flag.length; i++){ - if(flag[i] == 0){ - list.add(winlist.get(i)); + List list = new ArrayList<>(); + for(int i=0; i< flag.length; i++){ + if(flag[i] == 0){ + list.add(winlist.get(i)); + } + } + return list; + }finally { + if(null != winlist && !winlist.isEmpty()){ + winlist.clear(); } } - return list; } private static List deleteFP(List winlist){ - if (winlist == null || winlist.isEmpty()) { - return new ArrayList<>(); - } - //排序 - Collections.sort(winlist); - int [] flag = new int[winlist.size()]; - for(int i=0; i< winlist.size(); i++){ - if(flag[i] > 0){ - continue; + try { + if(null == winlist || winlist.isEmpty()) { + return new ArrayList<>(); } - for(int j=i+1; j 0){ + continue; + } + for(int j=i+1; j list = new ArrayList<>(); - for(int i=0; i< flag.length; i++){ - if(flag[i] == 0){ - list.add(winlist.get(i)); + List list = new ArrayList<>(); + for(int i=0; i< flag.length; i++){ + if(flag[i] == 0){ + list.add(winlist.get(i)); + } + } + return list; + }finally { + if(null != winlist && !winlist.isEmpty()){ + winlist.clear(); } } - return list; } - private static List trans_window(Mat img, Mat imgPad, List winlist){ - int row = (imgPad.size(0) - img.size(0)) / 2; - int col = (imgPad.size(1) - img.size(1)) / 2; + private static List trans_window(Mat img, Mat imgPad, List winlist, float scale){ List ret = new ArrayList<>(); - for(Window2 win : winlist){ - if( win.w > 0 && win.h > 0){ - int x1 = win.x - col; - int y1 = win.y - row; - int x2 = win.x - col + win.w; - int y2 = win.y - row + win.w; - int angle = (win.angle + 360) % 360; - //扩展人脸高度 - float rw = 0f; - float rh = 0.1f; - int w = Math.abs(x2 - x1); - int h = Math.abs(y2 - y1); - x1 = Math.max(x1 - (int)(w * rw), 1); - y1 = Math.max(y1 - (int)(h * rh), 1); - x2 = Math.min(x2 + (int)(w * rw), img.size(1)-1); - y2 = Math.min(y2 + (int)(h * rh), img.size(0)-1); - //构建人脸信息 - FaceInfo faceInfo = FaceInfo.build(win.conf, angle, FaceInfo.FaceBox.build(x1, y1, x2, y2), FaceInfo.Points.build()); - ret.add(faceInfo); + try { + int row = (imgPad.size(0) - img.size(0)) / 2; + int col = (imgPad.size(1) - img.size(1)) / 2; + for(Window2 win : winlist){ + if( win.w > 0 && win.h > 0){ + int x1 = win.x - col; + int y1 = win.y - row; + int x2 = win.x - col + win.w; + int y2 = win.y - row + win.w; + int angle = (win.angle + 360) % 360; + //扩展人脸高度 + float rw = 0f; + float rh = 0.1f; + int w = Math.abs(x2 - x1); + int h = Math.abs(y2 - y1); + x1 = Math.max(Float.valueOf((x1 - (int)(w * rw)) * scale).intValue(), 1); + y1 = Math.max(Float.valueOf((y1 - (int)(h * rh)) * scale).intValue(), 1); + x2 = Math.min(Float.valueOf((x2 + (int)(w * rw)) * scale).intValue(), Float.valueOf((img.size(1)) * scale).intValue()-1); + y2 = Math.min(Float.valueOf((y2 + (int)(h * rh)) * scale).intValue(), Float.valueOf((img.size(0)) * scale).intValue()-1); + //构建人脸信息 + FaceInfo faceInfo = FaceInfo.build(win.conf, angle, FaceInfo.FaceBox.build(x1, y1, x2, y2), FaceInfo.Points.build()); + ret.add(faceInfo); + } + } + return ret; + }finally { + if(null != winlist && !winlist.isEmpty()){ + winlist.clear(); } } - return ret; + } /** @@ -284,34 +307,30 @@ public class PcnNetworkFaceDetection extends BaseOnnxInfer implements FaceDetect * @throws OrtException * @throws IOException */ - private static List stage1(Mat img, Mat imgPad, OrtSession net, float thres) throws RuntimeException { - int netSize = 24; - float curScale = minFace_ / netSize; - int row = (int) ((imgPad.size().height - img.size().height) / 2); - int col = (int) ((imgPad.size().width - img.size().width) / 2); - + private static List stage1_release_mat(Mat img, Mat imgPad, OrtSession net, float thres) throws RuntimeException { Mat img_resized = null; OnnxTensor net_input1 = null; OrtSession.Result output = null; List winlist = new ArrayList<>(); try { - img_resized = resize_img(img.clone(), curScale); + //获取必要参数 + int netSize = 24; + float curScale = minFace_ / netSize; + int row = (int) ((imgPad.size().height - img.size().height) / 2); + int col = (int) ((imgPad.size().width - img.size().width) / 2); + img_resized = resize_img_release_mat(img.clone(), curScale); + //循环处理 while(Math.min(img_resized.size().height, img_resized.size().width) >= netSize){ - img_resized = preprocess_img(img_resized, 0); - net_input1 = set_input(img_resized); + img_resized = preprocess_img_release_mat(img_resized, 0); + net_input1 = set_input_release_mat(img_resized.clone()); + //推理网络 output = net.run(Collections.singletonMap(net.getInputNames().iterator().next(), net_input1)); float[][][][] cls_prob = (float[][][][]) output.get(0).getValue(); float[][][][] rotate = (float[][][][]) output.get(1).getValue(); float[][][][] bbox = (float[][][][]) output.get(2).getValue(); //关闭对象 - if(null != net_input1){ - net_input1.close(); - net_input1 = null; - } - if(null != output){ - output.close(); - output = null; - } + ReleaseUtil.release(output); output = null; + ReleaseUtil.release(net_input1); net_input1 = null; //计算业务逻辑 float w = netSize * curScale; for(int i=0; i< cls_prob[0][0].length; i++){ @@ -323,7 +342,7 @@ public class PcnNetworkFaceDetection extends BaseOnnxInfer implements FaceDetect int rx = (int)(j * curScale * stride_ - 0.5 * sn * w + sn * xn * w + 0.5 * w) + col; int ry = (int)(i * curScale * stride_ - 0.5 * sn * w + sn * yn * w + 0.5 * w) + row; int rw = (int)(w * sn); - if (legal(rx, ry, imgPad) && legal(rx + rw - 1, ry + rw - 1, imgPad)){ + if (legal(rx, ry, imgPad.size()) && legal(rx + rw - 1, ry + rw - 1, imgPad.size())){ if (rotate[0][1][i][j] > 0.5){ winlist.add(new Window2(rx, ry, rw, rw, 0, curScale, cls_prob[0][1][i][j])); }else{ @@ -333,21 +352,15 @@ public class PcnNetworkFaceDetection extends BaseOnnxInfer implements FaceDetect } } } - img_resized = resize_img(img_resized, scale_); + img_resized = resize_img_release_mat(img_resized, scale_); curScale = (float) (img.size().height / img_resized.size().height); } }catch (Exception e){ throw new RuntimeException(e); }finally { - if(null != net_input1){ - net_input1.close(); - } - if(null != output){ - output.close(); - } - if(null != img_resized){ - img_resized.release(); - } + ReleaseUtil.release(output); + ReleaseUtil.release(net_input1); + ReleaseUtil.release(img_resized, imgPad, img); } //返回 return winlist; @@ -364,56 +377,58 @@ public class PcnNetworkFaceDetection extends BaseOnnxInfer implements FaceDetect * @return * @throws OrtException */ - private static List stage2(Mat img, Mat img180, OrtSession net, float thres, int dim, List winlist) throws OrtException { + private static List stage2_release_mat(Mat img, Mat img180, OrtSession net, float thres, int dim, List winlist) throws OrtException { if(winlist==null || winlist.isEmpty()){ return new ArrayList<>(); } + //逻辑处理 + float[][] bbox; + float[][] rotate; + float[][] cls_prob; + Mats datalist = null; + OnnxTensor input = null; + OrtSession.Result output = null; + Size size = img.size(); int height = img.size(0); - List datalist = new ArrayList<>(); - for(Window2 win : winlist){ - if(Math.abs(win.angle) < EPS){ - Mat cloneMat = img.clone(); - Mat corp = new Mat(cloneMat, new Rect(win.x, win.y, win.w, win.h)); - Mat corp1 = preprocess_img(corp, dim); - datalist.add(corp1); - if(null != corp){ - corp.release(); - } - if(null != cloneMat){ - cloneMat.release(); - } - }else{ - int y2 = win.y + win.h - 1; - int y = height - 1 - y2; - - Mat cloneMat = img180.clone(); - Mat corp = new Mat(cloneMat, new Rect(win.x, y, win.w, win.h)); - Mat corp1 = preprocess_img(corp, dim); - datalist.add(corp1); - if(null != corp){ - corp.release(); - } - if(null != cloneMat){ - cloneMat.release(); + try { + datalist = Mats.build(); + for(Window2 win : winlist){ + if(Math.abs(win.angle) < EPS){ + Mat corp = null; + try { + corp = new Mat(img, new Rect(win.x, win.y, win.w, win.h)); + Mat preprocess = preprocess_img_release_mat(corp.clone(), dim); + datalist.add(preprocess); + }finally { + ReleaseUtil.release(corp); + } + }else{ + Mat corp = null; + try { + int y2 = win.y + win.h - 1; + int y = height - 1 - y2; + corp = new Mat(img180, new Rect(win.x, y, win.w, win.h)); + Mat preprocess = preprocess_img_release_mat(corp.clone(), dim); + datalist.add(preprocess); + }finally { + ReleaseUtil.release(corp); + } } } + //模型推理 + input = set_input_release_mat(datalist.clone()); + output = net.run(Collections.singletonMap(net.getInputNames().iterator().next(), input)); + cls_prob = (float[][]) output.get(0).getValue(); + rotate = (float[][]) output.get(1).getValue(); + bbox = (float[][]) output.get(2).getValue(); + }finally { + ReleaseUtil.release(datalist); + ReleaseUtil.release(output); + ReleaseUtil.release(input); + ReleaseUtil.release(img180, img); + output = null; input = null; } - - OnnxTensor net_input = set_input(datalist); - OrtSession.Result output = net.run(Collections.singletonMap(net.getInputNames().iterator().next(), net_input)); - float[][] cls_prob = (float[][]) output.get(0).getValue(); - float[][] rotate = (float[][]) output.get(1).getValue(); - float[][] bbox = (float[][]) output.get(2).getValue(); - //关闭对象 - for(Mat mat : datalist){ - mat.release(); - } - if(null != net_input){ - net_input.close(); - } - if(null != output){ - output.close(); - } + //再次后处理数据 List ret = new ArrayList<>(); for(int i=0; i thres){ @@ -439,7 +454,7 @@ public class PcnNetworkFaceDetection extends BaseOnnxInfer implements FaceDetect } } - if(legal(x, y, img) && legal(x + w - 1, y + w - 1, img)){ + if(legal(x, y, size) && legal(x + w - 1, y + w - 1, size)){ int angle = 0; if(Math.abs(winlist.get(i).angle) < EPS){ if(maxRotateIndex == 0){ @@ -479,49 +494,79 @@ public class PcnNetworkFaceDetection extends BaseOnnxInfer implements FaceDetect * @return * @throws OrtException */ - private static List stage3(Mat imgPad, Mat img180, Mat img90, Mat imgNeg90, OrtSession net, float thres, int dim, List winlist) throws OrtException { + private static List stage3_release_mat(Mat imgPad, Mat img180, Mat img90, Mat imgNeg90, OrtSession net, float thres, int dim, List winlist) throws OrtException { if (winlist == null || winlist.isEmpty()) { return new ArrayList<>(); } + //逻辑处理 + float[][] bbox; + float[][] rotate; + float[][] cls_prob; + Mats datalist = null; + OnnxTensor input = null; + OrtSession.Result output = null; + int height = imgPad.size(0); int width = imgPad.size(1); - List datalist = new ArrayList<>(); - for(Window2 win : winlist){ - if(Math.abs(win.angle) < EPS){ - Mat corp = new Mat(imgPad, new Rect(win.x, win.y, win.w, win.h)); - datalist.add(preprocess_img(corp, dim)); - }else if(Math.abs(win.angle - 90) < EPS){ - Mat corp = new Mat(img90, new Rect(win.y, win.x, win.h, win.w)); - datalist.add(preprocess_img(corp, dim)); - }else if(Math.abs(win.angle + 90) < EPS){ - int x = win.y; - int y = width - 1 - (win.x + win.w - 1); - Mat corp = new Mat(imgNeg90, new Rect(x, y, win.w, win.h)); - datalist.add(preprocess_img(corp, dim)); - }else{ - int y2 = win.y + win.h - 1; - int y = height - 1 - y2; - Mat corp = new Mat(img180, new Rect(win.x, y, win.w, win.h)); - datalist.add(preprocess_img(corp, dim)); + Size imgPadSize = imgPad.size(); + Size img180Size = img180.size(); + Size img90Size = img90.size(); + Size imgNeg90Size = imgNeg90.size(); + + try { + datalist = Mats.build(); + for(Window2 win : winlist){ + if(Math.abs(win.angle) < EPS){ + Mat corp = null; + try { + corp = new Mat(imgPad, new Rect(win.x, win.y, win.w, win.h)); + datalist.add(preprocess_img_release_mat(corp.clone(), dim)); + }finally { + ReleaseUtil.release(corp); + } + }else if(Math.abs(win.angle - 90) < EPS){ + Mat corp = null; + try { + corp = new Mat(img90, new Rect(win.y, win.x, win.h, win.w)); + datalist.add(preprocess_img_release_mat(corp.clone(), dim)); + }finally { + ReleaseUtil.release(corp); + } + }else if(Math.abs(win.angle + 90) < EPS){ + Mat corp = null; + try { + int x = win.y; + int y = width - 1 - (win.x + win.w - 1); + corp = new Mat(imgNeg90, new Rect(x, y, win.w, win.h)); + datalist.add(preprocess_img_release_mat(corp.clone(), dim)); + }finally { + ReleaseUtil.release(corp); + } + }else{ + Mat corp = null; + try { + int y2 = win.y + win.h - 1; + int y = height - 1 - y2; + corp = new Mat(img180, new Rect(win.x, y, win.w, win.h)); + datalist.add(preprocess_img_release_mat(corp.clone(), dim)); + }finally { + ReleaseUtil.release(corp); + } + } } + input = set_input_release_mat(datalist); + output = net.run(Collections.singletonMap(net.getInputNames().iterator().next(), input)); + cls_prob = (float[][]) output.get(0).getValue(); + rotate = (float[][]) output.get(1).getValue(); + bbox = (float[][]) output.get(2).getValue(); + }finally { + ReleaseUtil.release(datalist); + ReleaseUtil.release(output); + ReleaseUtil.release(input); + ReleaseUtil.release(imgPad, img180, img90, imgNeg90); + output = null; input = null; } - OnnxTensor net_input = set_input(datalist); - OrtSession.Result output = net.run(Collections.singletonMap(net.getInputNames().iterator().next(), net_input)); - float[][] cls_prob = (float[][]) output.get(0).getValue(); - float[][] rotate = (float[][]) output.get(1).getValue(); - float[][] bbox = (float[][]) output.get(2).getValue(); - - //关闭对象 - for(Mat mat : datalist){ - mat.release(); - } - if(null != net_input){ - net_input.close(); - } - if(null != output){ - output.close(); - } - + //模型后处理 List ret = new ArrayList<>(); for(int i=0; i thres) { @@ -531,24 +576,24 @@ public class PcnNetworkFaceDetection extends BaseOnnxInfer implements FaceDetect float cropX = winlist.get(i).x; float cropY = winlist.get(i).y; float cropW = winlist.get(i).w; - Mat img_tmp = imgPad; + Size img_tmp_size = imgPadSize; if (Math.abs(winlist.get(i).angle - 180) < EPS) { cropY = height - 1 - (cropY + cropW - 1); - img_tmp = img180; + img_tmp_size = img180Size; }else if (Math.abs(winlist.get(i).angle - 90) < EPS) { cropX = winlist.get(i).y; cropY = winlist.get(i).x; - img_tmp = img90; + img_tmp_size = img90Size; }else if (Math.abs(winlist.get(i).angle + 90) < EPS) { cropX = winlist.get(i).y; cropY = width - 1 - (winlist.get(i).x + winlist.get(i).w - 1); - img_tmp = imgNeg90; + img_tmp_size = imgNeg90Size; } int w = (int) (sn * cropW); int x = (int) (cropX - 0.5 * sn * cropW + cropW * sn * xn + 0.5 * cropW); int y = (int) (cropY - 0.5 * sn * cropW + cropW * sn * yn + 0.5 * cropW); int angle = (int)(angleRange_ * rotate[i][0]); - if(legal(x, y, img_tmp) && legal(x + w - 1, y + w - 1, img_tmp)){ + if(legal(x, y, img_tmp_size) && legal(x + w - 1, y + w - 1, img_tmp_size)){ if(Math.abs(winlist.get(i).angle) < EPS){ ret.add(new Window2(x, y, w, w, angle, winlist.get(i).scale, cls_prob[i][1])); }else if(Math.abs(winlist.get(i).angle - 180) < EPS){ @@ -573,32 +618,33 @@ public class PcnNetworkFaceDetection extends BaseOnnxInfer implements FaceDetect * @throws OrtException * @throws IOException */ - private static List detect(OrtSession[] sessions, Mat img, Mat imgPad, float[] scoreThs, float iouThs[]) throws OrtException, IOException { - Mat img180 = new Mat(); - Core.flip(imgPad, img180, 0); - - Mat img90 = new Mat(); - Core.transpose(imgPad, img90); - - Mat imgNeg90 = new Mat(); - Core.flip(img90, imgNeg90, 0); - - List winlist = stage1(img, imgPad, sessions[0], scoreThs[0]); - winlist = NMS(winlist, true, iouThs[0]); - - winlist = stage2(imgPad, img180, sessions[1], scoreThs[1], 24, winlist); - winlist = NMS(winlist, true, iouThs[1]); - - winlist = stage3(imgPad, img180, img90, imgNeg90, sessions[2], scoreThs[2], 48, winlist); - winlist = NMS(winlist, false, iouThs[2]); - - winlist = deleteFP(winlist); - - img90.release(); - img180.release(); - imgNeg90.release(); - - return winlist; + private static List detect_release_mat(OrtSession[] sessions, Mat img, Mat imgPad, float[] scoreThs, float iouThs[]) throws OrtException, IOException { + Mat img180 = null; + Mat img90 = null; + Mat imgNeg90 = null; + try { + //前置处理 + img180 = new Mat(); + Core.flip(imgPad, img180, 0); + img90 = new Mat(); + Core.transpose(imgPad, img90); + imgNeg90 = new Mat(); + Core.flip(img90, imgNeg90, 0); + //第一步 + List winlist = stage1_release_mat(img.clone(), imgPad.clone(), sessions[0], scoreThs[0]); + winlist = NMS(winlist, true, iouThs[0]); + //第二步 + winlist = stage2_release_mat(imgPad.clone(), img180.clone(), sessions[1], scoreThs[1], 24, winlist); + winlist = NMS(winlist, true, iouThs[1]); + //第三步 + winlist = stage3_release_mat(imgPad.clone(), img180.clone(), img90.clone(), imgNeg90.clone(), sessions[2], scoreThs[2], 48, winlist); + winlist = NMS(winlist, false, iouThs[2]); + //后处理 + winlist = deleteFP(winlist); + return winlist; + }finally { + ReleaseUtil.release(imgNeg90, img90, img180, imgPad, img); + } } /** @@ -642,7 +688,7 @@ public class PcnNetworkFaceDetection extends BaseOnnxInfer implements FaceDetect ", angle=" + angle + ", scale=" + scale + ", conf=" + conf + - '}' +"\n"; + "}" +"\n"; } } } diff --git a/face-search-core/src/main/java/com/visual/face/search/core/utils/ReleaseUtil.java b/face-search-core/src/main/java/com/visual/face/search/core/utils/ReleaseUtil.java new file mode 100644 index 0000000..29fe0e2 --- /dev/null +++ b/face-search-core/src/main/java/com/visual/face/search/core/utils/ReleaseUtil.java @@ -0,0 +1,98 @@ +package com.visual.face.search.core.utils; + +import ai.onnxruntime.OnnxTensor; +import ai.onnxruntime.OrtSession; +import com.visual.face.search.core.domain.ImageMat; +import com.visual.face.search.core.domain.Mats; +import org.opencv.core.Mat; + +public class ReleaseUtil { + + public static void release(Mat ...mats){ + for(Mat mat : mats){ + if(null != mat){ + try { + mat.release(); + }catch (Exception e){ + e.printStackTrace(); + }finally { + mat = null; + } + } + } + } + + public static void release(Mats mats){ + if(null == mats || mats.isEmpty()){ + return; + } + try { + mats.release(); + }catch (Exception e){ + e.printStackTrace(); + }finally { + mats = null; + } + } + + public static void release(ImageMat ...imageMats){ + for(ImageMat imageMat : imageMats){ + if(null != imageMat){ + try { + imageMat.release(); + }catch (Exception e){ + e.printStackTrace(); + }finally { + imageMat = null; + } + } + } + } + + public static void release(OnnxTensor ...tensors){ + if(null == tensors || tensors.length == 0){ + return; + } + try { + for(OnnxTensor tensor : tensors){ + try { + if(null != tensor){ + tensor.close(); + } + }catch (Exception e) { + e.printStackTrace(); + }finally { + tensor = null; + } + } + }catch (Exception e){ + e.printStackTrace(); + }finally { + tensors = null; + } + } + + public static void release(OrtSession.Result ...results){ + if(null == results || results.length == 0){ + return; + } + try { + for(OrtSession.Result result : results){ + try { + if(null != result){ + result.close(); + } + }catch (Exception e) { + e.printStackTrace(); + }finally { + result = null; + } + } + }catch (Exception e){ + e.printStackTrace(); + }finally { + results = null; + } + } + +} diff --git a/face-search-core/src/test/resources/images/faces/debug/debug_0005.jpg b/face-search-core/src/test/resources/images/faces/debug/debug_0005.jpg new file mode 100644 index 0000000..31cdb61 Binary files /dev/null and b/face-search-core/src/test/resources/images/faces/debug/debug_0005.jpg differ