1.Android 腾讯NCNN3分钟实现摄像头检测模型加载人体部位识别 (人工智能)

深南大盗

5124人浏览 · 2020-07-05 16:06:13

深南大盗 · 2020-07-05 16:06:13 发布

NCNN

ncnn 是腾讯提供的移动端框架非常时候在手机玩

ncnn 是一个为手机端极致优化的高性能神经网络前向计算框架。ncnn 从设计之初深刻考虑手机端的部署和使用。无第三方依赖，跨平台，手机端 cpu 的速度快于目前所有已知的开源框架。基于 ncnn，开发者能够将深度学习算法轻松移植到手机端高效执行，开发出人工智能 APP，将 AI 带到你的指尖。ncnn 目前已在腾讯多款应用中使用，如 QQ，Qzone，微信，天天P图等。

功能概述

支持卷积神经网络，支持多输入和多分支结构，可计算部分分支
无任何第三方库依赖，不依赖 BLAS/NNPACK 等计算框架
纯 C++ 实现，跨平台，支持 android ios 等
ARM NEON 汇编级良心优化，计算速度极快
精细的内存管理和数据结构设计，内存占用极低
支持多核并行计算加速，ARM big.LITTLE cpu 调度优化
支持基于全新低消耗的 vulkan api GPU 加速
整体库体积小于 700K，并可轻松精简到小于 300K
可扩展的模型设计，支持 8bit 量化和半精度浮点存储，可导入 caffe/pytorch/mxnet/onnx/darknet 模型
支持直接内存零拷贝引用加载网络模型
可注册自定义层实现并扩展
恩，很强就是了，不怕被塞卷 QvQ

demo功能演示

1.人脸识别

2.身份证，银行卡识别

3.语音识别

4.车票识别

5.人体部位识别

DEMO解决了几个问题：

1.替换了模型，检测不出结果的问题

2.模型加载不了的问题或者奔溃

3.模型加载耗时6s。问题修复

4.识别内存溢出问题修复

demo地址：模型加载和自动识别

https://github.com/nihui/ncnn-android-mobilenetssd

3分钟实现demo：

step1

https://github.com/Tencent/ncnn/releases

download ncnn-android-vulkan-lib.zip or build ncnn for android yourself

下载库ncnn-android-vulkan-lib：

step2

extract ncnn-android-vulkan-lib.zip into app/src/main/jni or change the ncnn path to yours in app/src/main/jni/CMakeLists.txt

配置cmake

android ios 预编译库 20200616 622879a

编译版本，默认配置，android-ndk-r21d，cctools-port 895 + ld64-274.2 + ios 10.2 sdk libc++
ncnn-android-lib 是 android 的静态库(armeabi-v7a + arm64-v8a + x86 + x86_64)
ncnn-android-vulkan-lib 是 android 的静态库(armeabi-v7a + arm64-v8a + x86 + x86_64，包含vulkan支持)
ncnn.framework.zip 是 ios 的静态库(armv7 + arm64 + i386 + x86_64，bitcode)
ncnn-vulkan.framework.zip 是 ios 的静态库(arm64 + x86_64，bitcode，包含vulkan支持，MoltenVK-1.1.82.0)
openmp.framework.zip 是 ios ncnn openmp 运行时静态库(armv7 + arm64 + i386 + x86_64，bitcode)

adreno gpu image存储+fp16p/fp16s/fp16pa/fp16sa优化，在qcom855之前的高通芯片上默认启用，包括全部gpu shader
新增darknet转换器，支持yolov4和efficientnetb0-yolov3(by zhiliu6)
新增simplestl，可替代std::string/std::vector，默认不启用(by scarsty)
新增NCNN_LOGE宏，android自动在adb logcat输出信息(by maxint)
运行时生成spirv，大幅减小gpu库体积
新增python绑定链接
新增查询当前可用gpu显存接口
gpu fp16/fp32转换，buffer/image

源码分析：

第一步：加载模型。量化模型

// init param
{
    int ret = mobilenetssd.load_param(mgr, "mobilenet_ssd_voc_ncnn.param");
    if (ret != 0)
    {
        __android_log_print(ANDROID_LOG_DEBUG, "MobilenetSSDNcnn", "load_param failed");
        return JNI_FALSE;
    }
    __android_log_print(ANDROID_LOG_DEBUG, "MobilenetSSDNcnn", "load_param success");
}

// init bin
{
    int ret = mobilenetssd.load_model(mgr, "mobilenet_ssd_voc_ncnn.bin");
    if (ret != 0)
    {
        __android_log_print(ANDROID_LOG_DEBUG, "MobilenetSSDNcnn", "load_model failed");
        return JNI_FALSE;
    }

    __android_log_print(ANDROID_LOG_DEBUG, "MobilenetSSDNcnn", "load_model success");
}

第二步：初始化java需要的一些参数

// init jni glue
jclass localObjCls = env->FindClass("com/tencent/mobilenetssdncnn/MobilenetSSDNcnn$Obj");
objCls = reinterpret_cast<jclass>(env->NewGlobalRef(localObjCls));

__android_log_print(ANDROID_LOG_DEBUG, "MobilenetSSDNcnn", "start");

constructortorId = env->GetMethodID(objCls, "<init>", "(Lcom/tencent/mobilenetssdncnn/MobilenetSSDNcnn;)V");

xId = env->GetFieldID(objCls, "x", "F");
yId = env->GetFieldID(objCls, "y", "F");
wId = env->GetFieldID(objCls, "w", "F");
hId = env->GetFieldID(objCls, "h", "F");
labelId = env->GetFieldID(objCls, "label", "Ljava/lang/String;");
probId = env->GetFieldID(objCls, "prob", "F");

__android_log_print(ANDROID_LOG_DEBUG, "MobilenetSSDNcnn", "end");

第三步：识别模型

}

double start_time = ncnn::get_current_time();

AndroidBitmapInfo info;
AndroidBitmap_getInfo(env, bitmap, &info);
int width = info.width;
int height = info.height;
if (info.format != ANDROID_BITMAP_FORMAT_RGBA_8888)
    return NULL;

// ncnn from bitmap
ncnn::Mat in = ncnn::Mat::from_android_bitmap_resize(env, bitmap, ncnn::Mat::PIXEL_BGR, 300, 300);

// mobilenetssd
std::vector<Object> objects;
{
    const float mean_vals[3] = { 98.1f, 106.0f, 133.1f };
    const float norm_vals[3] = { 0.2207f, 0.2281f, 0.2640f };
    in.substract_mean_normalize(mean_vals, norm_vals);

    ncnn::Extractor ex = mobilenetssd.create_extractor();

    ex.set_vulkan_compute(use_gpu);

    ex.input("data", in);

    ncnn::Mat out;
    ex.extract("detection_out", out);

    for (int i=0; i<out.h; i++)
    {
        const float* values = out.row(i);
        if(values[0]==1&&values[1]>0.7){
            Object object;
            object.label = values[0];
            object.prob = values[1];
            object.x = values[2] * width;
            object.y = values[3] * height;
            object.w = values[4] * width - object.x;
            object.h = values[5] * height - object.y;

            objects.push_back(object);
        } else{
            continue;
        }

    }
}

第4步：根据坐标进行车子的绘制

    for (int i = 0; i < objects.length; i++)

    {

        Log.d("peng","objects"+objects[i].toString());
        canvas.drawRect(objects[i].x, objects[i].y, objects[i].x + objects[i].w, objects[i].y + objects[i].h, paint);

        // draw filled text inside image
        {
            String text = objects[i].label + " = " + String.format("%.1f", objects[i].prob * 100) + "%";

            float text_width = textpaint.measureText(text);
            float text_height = - textpaint.ascent() + textpaint.descent();

            float x = objects[i].x;
            float y = objects[i].y - text_height;
            if (y < 0)
                y = 0;
            if (x + text_width > rgba.getWidth())
                x = rgba.getWidth() - text_width;

            canvas.drawRect(x, y, x + text_width, y + text_height, textbgpaint);

            canvas.drawText(text, x, y - textpaint.ascent(), textpaint);
        }
    }

    imageView.setImageBitmap(rgba);
}

#include <android/asset_manager_jni.h>
#include <android/bitmap.h>
#include <android/log.h>

#include <jni.h>

#include <string>
#include <vector>
#include <gpu.h>

// ncnn
#include "net.h"
#include "benchmark.h"

static ncnn::UnlockedPoolAllocator g_blob_pool_allocator;
static ncnn::PoolAllocator g_workspace_pool_allocator;

static ncnn::Net mobilenetssd;

struct Object
{
    float x;
    float y;
    float w;
    float h;
    int label;
    float prob;
};

extern "C" {

// FIXME DeleteGlobalRef is missing for objCls
static jclass objCls = NULL;
static jmethodID constructortorId;
static jfieldID xId;
static jfieldID yId;
static jfieldID wId;
static jfieldID hId;
static jfieldID labelId;
static jfieldID probId;

JNIEXPORT jint JNI_OnLoad(JavaVM* vm, void* reserved)
{
    __android_log_print(ANDROID_LOG_DEBUG, "MobilenetSSDNcnn", "JNI_OnLoad");

    ncnn::create_gpu_instance();

    return JNI_VERSION_1_4;
}

JNIEXPORT void JNI_OnUnload(JavaVM* vm, void* reserved)
{
    __android_log_print(ANDROID_LOG_DEBUG, "MobilenetSSDNcnn", "JNI_OnUnload");

    ncnn::destroy_gpu_instance();
}

// public native boolean Init(AssetManager mgr);
JNIEXPORT jboolean JNICALL Java_com_tencent_mobilenetssdncnn_MobilenetSSDNcnn_Init(JNIEnv* env, jobject thiz, jobject assetManager)
{
    ncnn::Option opt;
    opt.lightmode = true;
    opt.num_threads = 4;
    opt.blob_allocator = &g_blob_pool_allocator;
    opt.workspace_allocator = &g_workspace_pool_allocator;
    opt.use_packing_layout = true;

    // use vulkan compute
    if (ncnn::get_gpu_count() != 0)
        opt.use_vulkan_compute = true;

    AAssetManager* mgr = AAssetManager_fromJava(env, assetManager);

    mobilenetssd.opt = opt;

    // init param
    {
        int ret = mobilenetssd.load_param(mgr, "mobilenet_ssd_voc_ncnn.param");
        if (ret != 0)
        {
            __android_log_print(ANDROID_LOG_DEBUG, "MobilenetSSDNcnn", "load_param failed");
            return JNI_FALSE;
        }
        __android_log_print(ANDROID_LOG_DEBUG, "MobilenetSSDNcnn", "load_param success");
    }

    // init bin
    {
        int ret = mobilenetssd.load_model(mgr, "mobilenet_ssd_voc_ncnn.bin");
        if (ret != 0)
        {
            __android_log_print(ANDROID_LOG_DEBUG, "MobilenetSSDNcnn", "load_model failed");
            return JNI_FALSE;
        }

        __android_log_print(ANDROID_LOG_DEBUG, "MobilenetSSDNcnn", "load_model success");
    }

    // init jni glue
    jclass localObjCls = env->FindClass("com/tencent/mobilenetssdncnn/MobilenetSSDNcnn$Obj");
    objCls = reinterpret_cast<jclass>(env->NewGlobalRef(localObjCls));

    __android_log_print(ANDROID_LOG_DEBUG, "MobilenetSSDNcnn", "start");

    constructortorId = env->GetMethodID(objCls, "<init>", "(Lcom/tencent/mobilenetssdncnn/MobilenetSSDNcnn;)V");

    xId = env->GetFieldID(objCls, "x", "F");
    yId = env->GetFieldID(objCls, "y", "F");
    wId = env->GetFieldID(objCls, "w", "F");
    hId = env->GetFieldID(objCls, "h", "F");
    labelId = env->GetFieldID(objCls, "label", "Ljava/lang/String;");
    probId = env->GetFieldID(objCls, "prob", "F");

    __android_log_print(ANDROID_LOG_DEBUG, "MobilenetSSDNcnn", "end");

    return JNI_TRUE;
}

// public native Obj[] Detect(Bitmap bitmap, boolean use_gpu);
JNIEXPORT jobjectArray JNICALL Java_com_tencent_mobilenetssdncnn_MobilenetSSDNcnn_Detect(JNIEnv* env, jobject thiz, jobject bitmap, jboolean use_gpu)
{
    if (use_gpu == JNI_TRUE && ncnn::get_gpu_count() == 0)
    {
        return NULL;
        //return env->NewStringUTF("no vulkan capable gpu");
    }

    double start_time = ncnn::get_current_time();

    AndroidBitmapInfo info;
    AndroidBitmap_getInfo(env, bitmap, &info);
    int width = info.width;
    int height = info.height;
    if (info.format != ANDROID_BITMAP_FORMAT_RGBA_8888)
        return NULL;

    // ncnn from bitmap
    ncnn::Mat in = ncnn::Mat::from_android_bitmap_resize(env, bitmap, ncnn::Mat::PIXEL_BGR, 300, 300);

    // mobilenetssd
    std::vector<Object> objects;
    {
        const float mean_vals[3] = { 98.1f, 106.0f, 133.1f };
        const float norm_vals[3] = { 0.2207f, 0.2281f, 0.2640f };
        in.substract_mean_normalize(mean_vals, norm_vals);

        ncnn::Extractor ex = mobilenetssd.create_extractor();

        ex.set_vulkan_compute(use_gpu);

        ex.input("data", in);

        ncnn::Mat out;
        ex.extract("detection_out", out);

        for (int i=0; i<out.h; i++)
        {
            const float* values = out.row(i);
            if(values[0]==1&&values[1]>0.7){
                Object object;
                object.label = values[0];
                object.prob = values[1];
                object.x = values[2] * width;
                object.y = values[3] * height;
                object.w = values[4] * width - object.x;
                object.h = values[5] * height - object.y;

                objects.push_back(object);
            } else{
                continue;
            }

        }
    }

    // objects to Obj[]
    static const char* class_names[] = {"background",
        "tongue", "furred-tongue", "tooth-mark", "crack"};

    jobjectArray jObjArray = env->NewObjectArray(objects.size(), objCls, NULL);

    for (size_t i=0; i<objects.size(); i++)
    {
        jobject jObj = env->NewObject(objCls, constructortorId, thiz);

        env->SetFloatField(jObj, xId, objects[i].x);
        env->SetFloatField(jObj, yId, objects[i].y);
        env->SetFloatField(jObj, wId, objects[i].w);
        env->SetFloatField(jObj, hId, objects[i].h);
        env->SetObjectField(jObj, labelId, env->NewStringUTF(class_names[objects[i].label]));
        env->SetFloatField(jObj, probId, objects[i].prob);

        env->SetObjectArrayElement(jObjArray, i, jObj);
    }

    double elasped = ncnn::get_current_time() - start_time;
    __android_log_print(ANDROID_LOG_DEBUG, "MobilenetSSDNcnn", "%.2fms   detect", elasped);

    return jObjArray;
}

}

GitCode 开源社区

旨在为数千万中国开发者提供一个无缝且高效的云端环境，以支持学习、使用和贡献开源项目。

更多推荐

[转载]在Windows环境下安装GNU Radio

转自：在Windows环境下安装GNURadio_恐弱智_新浪博客GNU Radio是用Python开发的，大部分开源的工程能够在Linux环境下运行良好，而Windows下却运行的很勉强，而且安装配置都很复杂。GNU Radio算是个例外了，不光提供了Windows的二进制安装，还有比较详细的说明。我是Python小白，所以折腾了好久才弄好，特意记录下来，免得以后再装还折腾。GNU Radio的

GitCode 开源社区

centOS 8 使用dnf安装Docker

DNF是什么？CentOS 8使用YUM软件包管理器版本v4.0.4。现在，该版本使用DNF(已删除YUM)。DNF是软件包管理器。它会在Linux发行版上安装，执行更新并删除软件包。使用DNF安装Docker跳过具有损坏依赖性的程序包一个有效的解决方案是使您的CentOS 8系统使用以下--nobest命令安装最符合条件的版本：sudo dnf install docker...

GitCode 开源社区

定时同步数据库表(mysql+linux+crontab)

sync.sh里面的参数需要改变，ip/username/password/database/tablesync.sh#!/bin/sh# Please change the IP and password of the data source db.# Then change the table name.filename=/home/nington/db/$(date +%Y-%m