/** @brief Source file of vendor ai net sample code. @file ai_net_with_buf.c @ingroup ai_net_sample @note Nothing. Copyright Novatek Microelectronics Corp. 2020. All rights reserved. */ /*-----------------------------------------------------------------------------*/ /* Including Files */ /*-----------------------------------------------------------------------------*/ #include #include #include #include #include #include "hdal.h" #include "hd_type.h" #include "hd_debug.h" #include "vendor_ai.h" #include "vendor_ai_cpu/vendor_ai_cpu.h" #include "vendor_ai_cpu_postproc.h" #include "vendor_ai_util.h" #include #include #include "limit_fdet_lib.h" #include "pvdcnn_lib.h" // platform dependent #if defined(__LINUX) #include //for pthread API #define MAIN(argc, argv) int main(int argc, char** argv) #define GETCHAR() getchar() #else #include #include //for pthread API #include //for sleep API #define sleep(x) vos_util_delay_ms(1000*(x)) #define msleep(x) vos_util_delay_ms(x) #define usleep(x) vos_util_delay_us(x) #include //for MAIN(), GETCHAR() API #define MAIN(argc, argv) EXAMFUNC_ENTRY(ai_net_with_buf, argc, argv) #endif /////////////////////////////////////////////////////////////////////////////// #define VENDOR_AI_CFG 0x000f0000 //vendor ai config #define AI_RGB_BUFSIZE(w, h) (ALIGN_CEIL_4((w) * HD_VIDEO_PXLFMT_BPP(HD_VIDEO_PXLFMT_RGB888_PLANAR) / 8) * (h)) #define LIMIT_PVD 1 #define max_distance_mode 0 #define debuginf 1 #define MAX_FRAME_WIDTH 2880 //1920 #define MAX_FRAME_HEIGHT 1620 //1080 #define SCALE_BUF_SIZE (MAX_FRAME_WIDTH * MAX_FRAME_HEIGHT * 3) #define SAVE_SCALE 0 /////////////////////////////////////////////////////////////////////////////// BOOL is_net_exit = FALSE; /*-----------------------------------------------------------------------------*/ /* Type Definitions */ /*-----------------------------------------------------------------------------*/ typedef struct _MEM_PARM { UINT32 pa; UINT32 va; UINT32 size; UINT32 blk; } MEM_PARM; typedef struct _NET_PROC { CHAR model_filename[256]; INT32 binsize; int job_method; int job_wait_ms; int buf_method; MEM_PARM proc_mem; UINT32 proc_id; MEM_PARM io_mem; } NET_PROC; typedef struct _NET_IN { CHAR input_filename[256]; UINT32 w; UINT32 h; UINT32 c; UINT32 loff; UINT32 fmt; VENDOR_AI_BUF src_img; MEM_PARM input_mem; } NET_IN; /*-----------------------------------------------------------------------------*/ /* Global Functions */ /*-----------------------------------------------------------------------------*/ BOOL need_ise_resize(NET_IN nn_in, INT32 *size) { if (nn_in.w < (UINT32)size[0] || nn_in.h < (UINT32)size[1]) { return TRUE; } else if (nn_in.w > MAX_FRAME_WIDTH || nn_in.h > MAX_FRAME_HEIGHT) { return TRUE; } else { return FALSE; } } static HD_RESULT model_hdal_mem_get(MEM_PARM *mem_parm, UINT32 size) { HD_RESULT ret = HD_OK; UINT32 pa = 0; void *va = NULL; HD_COMMON_MEM_VB_BLK blk; /*ret = hd_common_mem_init(NULL); if (HD_OK != ret) { printf("hd_common_mem_init err: %d\r\n", ret); return ret; } // Allocate parameter buffer if (mem_parm->va != 0) { printf("err: mem has already been inited\r\n"); return -1; }*/ blk = hd_common_mem_get_block(HD_COMMON_MEM_USER_DEFINIED_POOL, size, DDR_ID0); //HD_COMMON_MEM_USER_DEFINIED_POOL + id HD_COMMON_MEM_CNN_POOL if (HD_COMMON_MEM_VB_INVALID_BLK == blk) { printf("hd_common_mem_get_block fail\r\n"); return HD_ERR_NG; } pa = hd_common_mem_blk2pa(blk); if (pa == 0) { printf("not get buffer, pa=%08x\r\n", (int)pa); return HD_ERR_NOMEM; } va = hd_common_mem_mmap(HD_COMMON_MEM_MEM_TYPE_CACHE, pa, size); /* Release buffer */ if (va == 0) { ret = hd_common_mem_munmap(va, size); if (ret != HD_OK) { printf("mem unmap fail\r\n"); return ret; } } mem_parm->pa = pa; mem_parm->va = (UINT32)va; mem_parm->size = size; mem_parm->blk = blk; return HD_OK; } static HD_RESULT model_hdal_mem_release(MEM_PARM *mem_parm) { HD_RESULT ret = HD_OK; /* Release in buffer */ if (mem_parm->va) { ret = hd_common_mem_munmap((void *)mem_parm->va, mem_parm->size); if (ret != HD_OK) { printf("mem_uninit : (g_mem.va)hd_common_mem_munmap fail.\r\n"); return ret; } } ret = hd_common_mem_release_block(mem_parm->blk); if (ret != HD_OK) { printf("mem_uninit : (g_mem.pa)hd_common_mem_release_block fail.\r\n"); return ret; } mem_parm->pa = 0; mem_parm->va = 0; mem_parm->size = 0; mem_parm->blk = (UINT32) - 1; return HD_OK; } static HD_RESULT hdal_mem_alloc(MEM_PARM *mem_parm, CHAR *name, UINT32 size) { HD_RESULT ret = HD_OK; UINT32 pa = 0; void *va = NULL; //alloc private pool ret = hd_common_mem_alloc(name, &pa, (void **)&va, size, DDR_ID0); if (ret != HD_OK) { return ret; } mem_parm->pa = pa; mem_parm->va = (UINT32)va; mem_parm->size = size; mem_parm->blk = (UINT32) - 1; return HD_OK; } static HD_RESULT hdal_mem_free(MEM_PARM *mem_parm) { HD_RESULT ret = HD_OK; //free private pool ret = hd_common_mem_free(mem_parm->pa, (void *)mem_parm->va); if (ret != HD_OK) { return ret; } //printf("1free pa = 0x%x, va = 0x%x\r\n", mem_parm->pa, mem_parm->va); mem_parm->pa = 0; mem_parm->va = 0; mem_parm->size = 0; mem_parm->blk = (UINT32) - 1; //printf("2free pa = 0x%x, va = 0x%x\r\n", mem_parm->pa, mem_parm->va); return HD_OK; } static INT32 _getsize_model(char *filename) { FILE *bin_fd; UINT32 bin_size = 0; bin_fd = fopen(filename, "rb"); if (!bin_fd) { printf("get bin(%s) size fail\n", filename); return (-1); } fseek(bin_fd, 0, SEEK_END); bin_size = ftell(bin_fd); fseek(bin_fd, 0, SEEK_SET); fclose(bin_fd); return bin_size; } static INT32 load_bin_file_to_mem(const CHAR *filename, MEM_PARM *mem_parm) { FILE *fd; INT32 size = 0; fd = fopen(filename, "rb"); if (!fd) { printf("cannot read %s\r\n", filename); return -1; } fseek(fd, 0, SEEK_END); size = ftell(fd); fseek(fd, 0, SEEK_SET); INT32 ww = (INT32)fread((VOID *)mem_parm->va, 1, size, fd); //printf("size: %ld, ww:%ld\n", size, ww); if (size < 0) { printf("getting %s size failed\r\n", filename); //} else if ((INT32)fread((VOID *)mem_parm->va, 1, size, fd) != size) { } else if (ww != size) { printf("read size < %ld\r\n", size); size = -1; } //mem_parm->size = size; if (fd) { fclose(fd); } return size; } /*-----------------------------------------------------------------------------*/ /* Input Functions */ /*-----------------------------------------------------------------------------*/ /////////////////////////////////////////////////////////////////////////////// static HD_RESULT input_open(NET_IN *p_nn_in) { HD_RESULT ret = HD_OK; p_nn_in->src_img.size = p_nn_in->w * p_nn_in->h * 3 / 2; //ret = hdal_mem_alloc(&p_nn_in->input_mem, "ai_in_buf", p_nn_in->src_img.size); ret = hdal_mem_alloc(&p_nn_in->input_mem, "ai_in_buf", AI_RGB_BUFSIZE(p_nn_in->w, p_nn_in->h)); if (ret != HD_OK) { printf("alloc ai_in_buf fail\r\n"); return HD_ERR_FAIL; } UINT32 file_len = load_bin_file_to_mem(p_nn_in->input_filename, &(p_nn_in->input_mem)); if (file_len < 0) { printf("load buf(%s) fail\r\n", p_nn_in->input_filename); return HD_ERR_NG; } hd_common_mem_flush_cache((VOID *)p_nn_in->input_mem.va, file_len); p_nn_in->src_img.width = p_nn_in->w; p_nn_in->src_img.height = p_nn_in->h; p_nn_in->src_img.channel = p_nn_in->c; p_nn_in->src_img.line_ofs = p_nn_in->loff; p_nn_in->src_img.fmt = HD_VIDEO_PXLFMT_YUV420; //p_nn_in->src_img.fmt = HD_VIDEO_PXLFMT_Y8; //bin�n�� p_nn_in->src_img.pa = p_nn_in->input_mem.pa; p_nn_in->src_img.va = p_nn_in->input_mem.va; p_nn_in->src_img.sign = MAKEFOURCC('A', 'B', 'U', 'F'); //p_nn_in->src_img.size = p_nn_in->loff * p_nn_in->h * 3 / 2; return ret; } static HD_RESULT input_close(NET_IN *p_nn_in) { HD_RESULT ret = HD_OK; if (p_nn_in->src_img.pa && p_nn_in->src_img.va) { MEM_PARM input_mem; input_mem.pa = p_nn_in->src_img.pa; input_mem.va = p_nn_in->src_img.va; hdal_mem_free(&input_mem); } return ret; } /*-----------------------------------------------------------------------------*/ /* Network Functions */ /*-----------------------------------------------------------------------------*/ static HD_RESULT network_alloc_io_buf(NET_PROC *p_net) { HD_RESULT ret = HD_OK; VENDOR_AI_NET_CFG_WORKBUF wbuf = {0}; printf("network_alloc_io_buf, procid: %ld\n", p_net->proc_id); ret = vendor_ai_net_get(p_net->proc_id, VENDOR_AI_NET_PARAM_CFG_WORKBUF, &wbuf); printf("after vendor_ai_net_get\n"); if (ret != HD_OK) { printf("proc_id(%lu) get VENDOR_AI_NET_PARAM_CFG_WORKBUF fail\r\n", p_net->proc_id); return HD_ERR_FAIL; } ret = hdal_mem_alloc(&p_net->io_mem, "ai_io_buf", wbuf.size); if (ret != HD_OK) { printf("proc_id(%lu) alloc ai_io_buf fail\r\n", p_net->proc_id); return HD_ERR_FAIL; } wbuf.pa = p_net->io_mem.pa; wbuf.va = p_net->io_mem.va; wbuf.size = p_net->io_mem.size; ret = vendor_ai_net_set(p_net->proc_id, VENDOR_AI_NET_PARAM_CFG_WORKBUF, &wbuf); if (ret != HD_OK) { printf("proc_id(%lu) set VENDOR_AI_NET_PARAM_CFG_WORKBUF fail\r\n", p_net->proc_id); return HD_ERR_FAIL; } //printf("alloc_io_buf: work buf, pa = %#lx, va = %#lx, size = %lu\r\n", wbuf.pa, wbuf.va, wbuf.size); //printf("alloc_io_buf: work buf size = %lu\r\n", wbuf.size); return ret; } static HD_RESULT network_free_io_buf(NET_PROC *p_net) { HD_RESULT ret = HD_OK; if (p_net->io_mem.pa && p_net->io_mem.va) { hdal_mem_free(&p_net->io_mem); } return ret; } static HD_RESULT network_open(NET_PROC *p_net, INT32 cls) { HD_RESULT ret = HD_OK; UINT32 loadsize = 0; // set buf opt /*printf("set buf opt\n"); VENDOR_AI_NET_CFG_BUF_OPT cfg_buf_opt = {0}; cfg_buf_opt.method = VENDOR_AI_NET_BUF_OPT_NONE; cfg_buf_opt.ddr_id = DDR_ID0; vendor_ai_net_set(proc_id, VENDOR_AI_NET_PARAM_CFG_BUF_OPT, &cfg_buf_opt); // set job option VENDOR_AI_NET_CFG_JOB_OPT cfg_job_opt = {0}; cfg_job_opt.method = VENDOR_AI_NET_JOB_OPT_LINEAR_O1; cfg_job_opt.wait_ms = 0; cfg_job_opt.schd_parm = VENDOR_AI_FAIR_CORE_ALL; //FAIR dispatch to ALL core vendor_ai_net_set(proc_id, VENDOR_AI_NET_PARAM_CFG_JOB_OPT, &cfg_job_opt);*/ VENDOR_AI_NET_CFG_BUF_OPT cfg_buf_opt = {0}; cfg_buf_opt.method = VENDOR_AI_NET_BUF_OPT_NONE;//p_net->buf_method; cfg_buf_opt.ddr_id = DDR_ID0; vendor_ai_net_set(p_net->proc_id, VENDOR_AI_NET_PARAM_CFG_BUF_OPT, &cfg_buf_opt); // set job opt VENDOR_AI_NET_CFG_JOB_OPT cfg_job_opt = {0}; cfg_job_opt.method = VENDOR_AI_NET_JOB_OPT_LINEAR_O1;//p_net->job_method;//0 cfg_job_opt.wait_ms = 0;//p_net->job_wait_ms;//-1 cfg_job_opt.schd_parm = VENDOR_AI_FAIR_CORE_ALL; //FAIR dispatch to ALL core //#define VENDOR_AI_FAIR_CORE_ALL 0x000000ff ///< fair: all core //#define VENDOR_AI_FAIR_CORE(ch) ((ch) & 0xff) ///< fair: select core (default 0) vendor_ai_net_set(p_net->proc_id, VENDOR_AI_NET_PARAM_CFG_JOB_OPT, &cfg_job_opt); if (strlen(p_net->model_filename) == 0) { printf("proc_id(%u) model is null\r\n", p_net->proc_id); return 0; } if (cls > 0) { /* MEM_PARM input_mem_parm; UINT32 input_mem_size = (UINT32)p_net->binsize; ret = hdal_mem_alloc(&input_mem_parm, "cls_model_mem", input_mem_size); if (HD_OK != ret) { printf("cls model mem allocate!!\n"); return 0; }*/ //p_net->proc_mem.va = buf->va; //p_net->proc_mem.pa = buf->pa; //p_net->proc_mem.size = buf->size; //p_net->proc_mem.blk = buf->blk; printf("before hdal_mem_alloc cls\n"); ret = hdal_mem_alloc(&(p_net->proc_mem), "cls_model_mem", p_net->binsize); printf("after hdal_mem_alloc cls\n"); loadsize = load_bin_file_to_mem(p_net->model_filename, &(p_net->proc_mem)); //printf("line328 load_bin_size: %d\r\n", loadsize); printf("loadsize: %ld, binsize: %ld\r\n", loadsize, p_net->binsize); printf("model load : %s\r\n", p_net->model_filename); printf("proc_id(%u)\r\n", p_net->proc_id); if (loadsize <= 0) { printf("proc_id(%u) model load fail: %s\r\n", p_net->proc_id, p_net->model_filename); return 0; } printf("cls:%ld, va:%ld, size:%ld\n", cls, p_net->proc_mem.va, p_net->proc_mem.size); VENDOR_AIS_FLOW_MEM_PARM version_check; version_check.va = p_net->proc_mem.va; version_check.pa = p_net->proc_mem.pa; version_check.size = p_net->proc_mem.size; ret = limit_fdet_version_check(&version_check); p_net->proc_mem.va = version_check.va; p_net->proc_mem.pa = version_check.pa; p_net->proc_mem.size = version_check.size; if (ret != HD_OK) { printf("ERR: limit_fdet version check fail (%d)!!\r\n", ret); return ret; } } else { // printf("before model_1hdal_mem_get\n"); ret = model_hdal_mem_get(&p_net->proc_mem, p_net->binsize); // printf("after model_1hdal_mem_get\n"); //load file //p_net->proc_mem.va = buf->va; //p_net->proc_mem.pa = buf->pa; //p_net->proc_mem.size = buf->size; //p_net->proc_mem.blk = buf->blk; loadsize = load_bin_file_to_mem(p_net->model_filename, &(p_net->proc_mem)); //printf("line328 load_bin_size: %d\r\n", loadsize); printf("loadsize: %ld, binsize: %ld\r\n", loadsize, p_net->binsize); printf("model load : %s\r\n", p_net->model_filename); printf("proc_id(%u)\r\n", p_net->proc_id); if (loadsize <= 0) { printf("proc_id(%u) model load fail: %s\r\n", p_net->proc_id, p_net->model_filename); return 0; } VENDOR_AIS_FLOW_MEM_PARM version_pvd_check; version_pvd_check.va = p_net->proc_mem.va; version_pvd_check.pa = p_net->proc_mem.pa; version_pvd_check.size = p_net->proc_mem.size; ret = pvdcnn_version_check(&version_pvd_check); p_net->proc_mem.va = version_pvd_check.va; p_net->proc_mem.pa = version_pvd_check.pa; p_net->proc_mem.size = version_pvd_check.size; if (ret != HD_OK) { printf("ERR: limit_fdet version check fail (%d)!!\r\n", ret); return ret; } printf("clsornot:%ld, va:%ld, size:%ld\n", cls, p_net->proc_mem.va, p_net->proc_mem.size); } // set model vendor_ai_net_set(p_net->proc_id, VENDOR_AI_NET_PARAM_CFG_MODEL, (VENDOR_AI_NET_CFG_MODEL *)&p_net->proc_mem); // open printf("before vendor_ai_net_open\n"); vendor_ai_net_open(p_net->proc_id); printf("after vendor_ai_net_open\n"); if ((ret = network_alloc_io_buf(p_net)) != HD_OK) { return ret; } printf("after network_alloc_io_buf\n\n"); return ret; } static HD_RESULT network_close(NET_PROC *p_net) { HD_RESULT ret = HD_OK; if ((ret = network_free_io_buf(p_net)) != HD_OK) { return ret; } // close ret = vendor_ai_net_close(p_net->proc_id); model_hdal_mem_release(&p_net->proc_mem); return ret; } static HD_RESULT network_get_ai_inputlayer_info(INT32 proc_id) { HD_RESULT ret = HD_OK; VENDOR_AI_BUF p_inbuf = {0}; VENDOR_AI_BUF p_outbuf = {0}; // get layer0 in buf ret = vendor_ai_net_get(proc_id, VENDOR_AI_NET_PARAM_IN(0, 0), &p_inbuf); if (HD_OK != ret) { printf("proc_id(%u) get layer0 inbuf fail !!\n", proc_id); return ret; } // get layer0 in buf ret = vendor_ai_net_get(proc_id, VENDOR_AI_NET_PARAM_OUT(0, 0), &p_outbuf); if (HD_OK != ret) { printf("proc_id(%u) get layer0 outbuf fail !!\n", proc_id); return ret; } /* printf("dump layer0 info:\n"); printf(" channel(%lu)\n", p_inbuf.channel); printf(" fmt(0x%lx)\n", p_inbuf.fmt); printf(" width(%lu)\n", p_outbuf.width); printf(" height(%lu)\n", p_outbuf.height); printf(" channel(%lu)\n", p_outbuf.channel); printf(" batch_num(%lu)\n", p_outbuf.batch_num); printf(" fmt(0x%lx)\n", p_outbuf.fmt); printf("\n"); */ return ret; } static HD_RESULT network_get_ai_outlayer_list(UINT32 proc_id, UINT32 *outlayer_num, UINT32 *outlayer_path_list) { HD_RESULT ret = HD_OK; VENDOR_AI_NET_INFO net_info = {0}; // get output layer number ret = vendor_ai_net_get(proc_id, VENDOR_AI_NET_PARAM_INFO, &net_info); if (HD_OK != ret) { printf("proc_id(%lu) get info fail !!\n", proc_id); return ret; } *outlayer_num = net_info.out_buf_cnt; // get path_list ret = vendor_ai_net_get(proc_id, VENDOR_AI_NET_PARAM_OUT_PATH_LIST, outlayer_path_list); if (HD_OK != ret) { printf("proc_id(%u) get outlayer_path_list fail !!\n", proc_id); } return ret; } static HD_RESULT network_get_ai_outlayer_by_path_id(INT32 proc_id, UINT32 path_id, VENDOR_AI_BUF *p_outbuf) { HD_RESULT ret = HD_OK; // get out buf by path_id ret = vendor_ai_net_get(proc_id, path_id, p_outbuf); if (HD_OK != ret) { printf("proc_id(%u) get AI_OUTBUF fail !!\n", proc_id); return ret; } /* INT8 bitdepth = HD_VIDEO_PXLFMT_BITS(p_outbuf->fmt); INT8 sign_bits = HD_VIDEO_PXLFMT_SIGN(p_outbuf->fmt); INT8 int_bits = HD_VIDEO_PXLFMT_INT(p_outbuf->fmt); INT8 frac_bits = HD_VIDEO_PXLFMT_FRAC(p_outbuf->fmt); printf("dump ai_outbuf:\n"); printf(" name (%s)\n", p_outbuf->name); printf(" pa (0x%lx)\n", p_outbuf->pa); printf(" va (0x%lx)\n", p_outbuf->va); printf(" size (%lu)\n", p_outbuf->size); printf(" width (%lu)\n", p_outbuf->width); printf(" height (%lu)\n", p_outbuf->height); printf(" channel (%lu)\n", p_outbuf->channel); printf(" layout (%s)\n", p_outbuf->layout); printf(" bitdepth (%d)\n", bitdepth); printf(" sign_bits (%d)\n", sign_bits); printf(" int_bits (%d)\n", int_bits); printf(" frac_bits (%d)\n", frac_bits); printf(" scale_ratio (%f)\n", p_outbuf->scale_ratio); printf("\n"); */ return ret; } #if 0 static HD_RESULT network_get_ai_outlayer_by_name(INT32 proc_id, VENDOR_AI_BUF *p_outbuf, char *p_buf_name) { HD_RESULT ret = HD_OK; VENDOR_AI_BUF_NAME buf_name = {0}; sprintf(buf_name.name, p_buf_name); // get out_path by name ret = vendor_ai_net_get(proc_id, VENDOR_AI_NET_PARAM_OUT_PATH_BY_NAME, &buf_name); if (HD_OK != ret) { printf("proc_id(%u) get OUT_PATH_BY_NAME fail !!\n", proc_id); return ret; } // get out buf by path_id ret = vendor_ai_net_get(proc_id, buf_name.path_id, p_outbuf); if (HD_OK != ret) { printf("proc_id(%u) get AI_OUTBUF fail !!\n", proc_id); return ret; } /*INT8 bitdepth = HD_VIDEO_PXLFMT_BITS(p_outbuf->fmt); INT8 sign_bits = HD_VIDEO_PXLFMT_SIGN(p_outbuf->fmt); INT8 int_bits = HD_VIDEO_PXLFMT_INT(p_outbuf->fmt); INT8 frac_bits = HD_VIDEO_PXLFMT_FRAC(p_outbuf->fmt); printf("dump ai_outbuf:\n"); printf(" name (%s)\n", p_outbuf->name); printf(" pa (0x%lx)\n", p_outbuf->pa); printf(" va (0x%lx)\n", p_outbuf->va); printf(" size (%lu)\n", p_outbuf->size); printf(" width (%lu)\n", p_outbuf->width); printf(" height (%lu)\n", p_outbuf->height); printf(" channel (%lu)\n", p_outbuf->channel); printf(" layout (%s)\n", p_outbuf->layout); printf(" bitdepth (%d)\n", bitdepth); printf(" sign_bits (%d)\n", sign_bits); printf(" int_bits (%d)\n", int_bits); printf(" frac_bits (%d)\n", frac_bits); printf(" scale_ratio (%f)\n", p_outbuf->scale_ratio); printf("\n");*/ return ret; } #endif VOID pd_gfx_img_to_vendor(VENDOR_AI_BUF *out_img, HD_GFX_IMG_BUF *in_img, UINT32 va) { //VENDOR_AIS_IMG_PARM* p_src_img; out_img->fmt = in_img->format; out_img->channel = 2; out_img->pa = in_img->p_phy_addr[0]; out_img->va = va; out_img->line_ofs = in_img->lineoffset[0]; out_img->width = in_img->dim.w; out_img->height = in_img->dim.h; out_img->sign = MAKEFOURCC('A', 'B', 'U', 'F'); out_img->size = in_img->dim.w * in_img->dim.h * 3 / 2; //return p_src_img; } HD_RESULT pdcnn_crop_img(HD_GFX_IMG_BUF *dst_img, VENDOR_AI_BUF *src_img, HD_GFX_SCALE_QUALITY method, PVD_IRECT *roi) { HD_RESULT ret; HD_GFX_SCALE param; memset(¶m, 0, sizeof(HD_GFX_SCALE)); UINT32 dif = 0; param.src_img.dim.w = src_img->width; param.src_img.dim.h = src_img->height; param.src_img.format = HD_VIDEO_PXLFMT_YUV420; param.src_img.p_phy_addr[0] = src_img->pa; param.src_img.p_phy_addr[1] = src_img->pa + src_img->width * src_img->height; param.src_img.lineoffset[0] = src_img->line_ofs; param.src_img.lineoffset[1] = src_img->line_ofs; param.dst_img.dim.w = dst_img->dim.w; param.dst_img.dim.h = dst_img->dim.h; param.dst_img.format = HD_VIDEO_PXLFMT_YUV420; param.dst_img.p_phy_addr[0] = dst_img->p_phy_addr[0]; param.dst_img.p_phy_addr[1] = dst_img->p_phy_addr[0] + dst_img->dim.w * dst_img->dim.h; param.dst_img.lineoffset[0] = dst_img->dim.w; param.dst_img.lineoffset[1] = dst_img->dim.w; param.dst_region.x = 0; param.dst_region.y = 0; param.dst_region.w = dst_img->dim.w; param.dst_region.h = dst_img->dim.h; param.src_region.w = ALIGN_CEIL_4((UINT32)(roi->x2 - roi->x1 + 0.5)); param.src_region.h = ALIGN_CEIL_4((UINT32)(roi->y2 - roi->y1 + 0.5)); if (param.src_region.w < 16) { dif = 16 - param.src_region.w; param.src_region.w = 16; param.src_region.x = ALIGN_CEIL_4((UINT32)(roi->x1 - dif / 2)); } else { param.src_region.x = ALIGN_CEIL_4((UINT32)(roi->x1 + 0.5)); } if (param.src_region.h < 16) { dif = 16 - param.src_region.h; param.src_region.h = 16; param.src_region.y = ALIGN_CEIL_4((UINT32)(roi->y1 - dif / 2)); } else { param.src_region.y = ALIGN_CEIL_4((UINT32)(roi->y1 + 0.5)); } param.quality = method; ret = hd_gfx_scale(¶m); if (ret != HD_OK) { printf("hd_gfx_scale fail=%d\n", ret); } return ret; } // static BOOL pdcnn_check_overlap(PVDCNN_RESULT *p_box1, PVDCNN_RESULT *p_box2, FLOAT ratio, INT32 method) // { // FLOAT left_x, right_x, up_y, down_y; // FLOAT inter_value, union_value, min_value; // // caculate intersect position // left_x = MAX(p_box1->x1, p_box2->x1); // right_x = MIN(p_box1->x2, p_box2->x2); // if (left_x >= right_x) return FALSE; // up_y = MAX(p_box1->y1, p_box2->y1); // down_y = MIN(p_box1->y2, p_box2->y2); // if (up_y >= down_y) return FALSE; // // caculate Intersection o Union // inter_value = (right_x - left_x) * (down_y - up_y); // if (1 == method) // { // union_value = (p_box1->x2 - p_box1->x1) * (p_box1->y2 - p_box1->y1) + (p_box2->x2 - p_box2->x1) * (p_box2->y2 - p_box2->y1) - inter_value; // if (inter_value >= union_value * ratio) return TRUE; // else return FALSE; // } // else if (2 == method) // { // min_value = MIN((p_box1->x2 - p_box1->x1) * (p_box1->y2 - p_box1->y1), (p_box2->x2 - p_box2->x1) * (p_box2->y2 - p_box2->y1)); // if (inter_value >= min_value * ratio) return TRUE; // else return FALSE; // } // else // { // printf("Error: Only support IOU and MIN method!\r\n"); // return FALSE; // } // } VOID pdcnn_quick_sort(PVDCNN_RESULT *persons, INT32 left, INT32 right) { if (left >= right) { return; } INT32 i = left; INT32 j = right; PVDCNN_RESULT key = persons[left]; while (i < j) { while (i < j && key.score >= persons[j].score) { j--; } persons[i] = persons[j]; while (i < j && key.score <= persons[i].score) { i++; } persons[j] = persons[i]; } persons[i] = key; pdcnn_quick_sort(persons, left, i - 1); pdcnn_quick_sort(persons, i + 1, right); } HD_RESULT ai_crop_img(HD_GFX_IMG_BUF *dst_img, VENDOR_AI_BUF src_img, HD_GFX_SCALE_QUALITY method) { HD_RESULT ret; HD_GFX_SCALE param; memset(¶m, 0, sizeof(HD_GFX_SCALE)); param.src_img.dim.w = src_img.width; param.src_img.dim.h = src_img.height; param.src_img.format = HD_VIDEO_PXLFMT_YUV420; param.src_img.p_phy_addr[0] = src_img.pa; param.src_img.p_phy_addr[1] = src_img.pa + src_img.width * src_img.height; param.src_img.lineoffset[0] = src_img.line_ofs; param.src_img.lineoffset[1] = src_img.line_ofs; param.dst_img.dim.w = dst_img->dim.w; param.dst_img.dim.h = dst_img->dim.h; param.dst_img.format = HD_VIDEO_PXLFMT_YUV420; param.dst_img.p_phy_addr[0] = dst_img->p_phy_addr[0]; param.dst_img.p_phy_addr[1] = dst_img->p_phy_addr[0] + dst_img->dim.w * dst_img->dim.h; param.dst_img.lineoffset[0] = dst_img->dim.w; param.dst_img.lineoffset[1] = dst_img->dim.w; param.src_region.x = 0;//fd_result->x_min; param.src_region.y = 0;//fd_result->y_min; param.src_region.w = src_img.width;//fd_result->w; param.src_region.h = src_img.height;//fd_result->h; param.dst_region.x = 0; param.dst_region.y = 0; param.dst_region.w = dst_img->dim.w; param.dst_region.h = dst_img->dim.h; param.quality = method; ret = hd_gfx_scale(¶m); if (ret != HD_OK) { printf("hd_gfx_scale fail=%d\n", ret); } return ret; } /*-----------------------------------------------------------------------------*/ /* Interface Functions */ /*-----------------------------------------------------------------------------*/ static VOID *nn_thread_api(VOID *arg) { HD_RESULT ret; HD_GFX_IMG_BUF gfx_img = {0}; VENDOR_AI_BUF p_src_img; //NET_IN* p_nn_in = (NET_IN*)arg; HD_COMMON_MEM_INIT_CONFIG mem_cfg = {0}; static struct timeval tstart, tend; UINT32 proc_id = 0; // config extend engine plugin, process scheduler /*typedef enum { VENDOR_AI_PROC_SCHD_FAIR = 0, ///< overlapping with fair core (default) VENDOR_AI_PROC_SCHD_CAPACITY = 1, ///< overlapping with max rate (TODO) VENDOR_AI_PROC_SCHD_FIFO = 2, ///< first in first out } VENDOR_AI_PROC_SCHD; */ // get vendor ai version VENDOR_AI_NET_CFG_IMPL_VERSION ai_version; ret = vendor_ai_cfg_get(VENDOR_AI_CFG_IMPL_VERSION, &ai_version); if (ret != HD_OK) { printf("get vendor ai sdk fail=%d\n", ret); goto exit_thread; } // get max proc_id number and each proc_id status VENDOR_AI_NET_CFG_PROC_COUNT proc_max_count; ret = vendor_ai_cfg_get(VENDOR_AI_CFG_PROC_COUNT, &proc_max_count); if (ret != HD_OK) { printf("get max proc_id number fail=%d\n", ret); goto exit_thread; } for (UINT32 i = 0; i < proc_max_count.max_proc_count; i++) { UINT32 state = 0; //1=ready, 2=open, 3=start vendor_ai_net_get(i, VENDOR_AI_NET_PARAM_STATE, &state); } NET_PROC net_info; net_info.proc_id = proc_id; #if PRUNE37PVD sprintf(net_info.model_filename, "/mnt/sd/CNNLib/para/pvdcnn/nvt_model_prune37.bin"); #else sprintf(net_info.model_filename, "/mnt/sd/CNNLib/para/pvdcnn/nvt_model_origin.bin"); #endif net_info.binsize = _getsize_model(net_info.model_filename); //net_info.job_method = VENDOR_AI_NET_JOB_OPT_LINEAR_O1; //net_info.job_wait_ms = 0; //net_info.buf_method = VENDOR_AI_NET_BUF_OPT_NONE; //job=1 net_info.job_method = VENDOR_AI_NET_JOB_OPT_GRAPH_O1; net_info.job_wait_ms = 0; net_info.buf_method = VENDOR_AI_NET_BUF_OPT_SHRINK_O2; if (net_info.binsize <= 0) { printf("proc_id(%u) model is not exist?\r\n", proc_id); goto exit_thread; } //printf("proc_id(%u) set net_info: model-file(%s), binsize=%d\r\n", proc_id, net_info.model_filename, net_info.binsize); // config hdal common pool (in) mem_cfg.pool_info[0].type = HD_COMMON_MEM_USER_DEFINIED_POOL; mem_cfg.pool_info[0].blk_size = net_info.binsize; mem_cfg.pool_info[0].blk_cnt = 1; mem_cfg.pool_info[0].ddr_id = DDR_ID0; // scale img buf mem_cfg.pool_info[1].type = HD_COMMON_MEM_USER_DEFINIED_POOL; mem_cfg.pool_info[1].blk_size = SCALE_BUF_SIZE; mem_cfg.pool_info[1].blk_cnt = 1; mem_cfg.pool_info[1].ddr_id = DDR_ID0; //---------------------------------------------------------------------------------------------------------------------- #if LIMIT_PVD LIMIT_FDET_PARAM limit_fdet_params = { 0 }; // limit_fdet_mem = { 0 }; limit_fdet_params.run_id = 1; NET_PROC net_info1; //UINT32 proc_id1 = 1; net_info1.proc_id = limit_fdet_params.run_id; sprintf(net_info1.model_filename, "/mnt/sd/CNNLib/para/public/nvt_model.bin"); net_info1.binsize = _getsize_model(net_info1.model_filename); //net_info.job_method = VENDOR_AI_NET_JOB_OPT_LINEAR_O1; //net_info.job_wait_ms = 0; //net_info.buf_method = VENDOR_AI_NET_BUF_OPT_NONE; //job=1 net_info1.job_method = VENDOR_AI_NET_JOB_OPT_GRAPH_O1; net_info1.job_wait_ms = 0; net_info1.buf_method = VENDOR_AI_NET_BUF_OPT_SHRINK_O2; if (net_info1.binsize <= 0) { printf("limit_fdet_params.run_id(%u) cls model is not exist?\r\n", limit_fdet_params.run_id); goto exit_thread; } // config hdal common pool (in) mem_cfg.pool_info[2].type = HD_COMMON_MEM_USER_DEFINIED_POOL; mem_cfg.pool_info[2].blk_size = net_info1.binsize; mem_cfg.pool_info[2].blk_cnt = 1; mem_cfg.pool_info[2].ddr_id = DDR_ID0; #endif ret = hd_common_mem_init(&mem_cfg); if (HD_OK != ret) { printf("hd_common_mem_init err: %d\r\n", ret); goto exit_thread; } INT32 cls = -1000; if ((ret = network_open(&net_info, cls)) != HD_OK) { printf("proc_id(%u nn open fail !!\n", proc_id); goto exit_thread; } printf("load pvd model\n"); // open network(model) #if LIMIT_PVD cls = 1000; if ((ret = network_open(&net_info1, cls)) != HD_OK) { printf("limit_fdet_params.run_id(%u cls nn open fail !!\n", limit_fdet_params.run_id); goto exit_thread; } printf("load cls model\n"); #endif for (UINT32 i = 0; i < proc_max_count.max_proc_count; i++) { UINT32 state = 0; //1=ready, 2=open, 3=start vendor_ai_net_get(i, VENDOR_AI_NET_PARAM_STATE, &state); } // start network proc_id ret = vendor_ai_net_start(proc_id); if (HD_OK != ret) { printf("proc_id(%u) nn start fail !!\n", proc_id); goto exit_thread; } #if LIMIT_PVD // start network limit_fdet_params.run_id ret = vendor_ai_net_start(limit_fdet_params.run_id); if (HD_OK != ret) { printf("limit_fdet_params.run_id(%u) nn start fail !!\n", limit_fdet_params.run_id); goto exit_thread; } #endif //get nn layer 0 info network_get_ai_inputlayer_info(proc_id); //get output layer info list CHAR out_layer_name[2][128] = {"layer115-conv", "layer125-conv"}; UINT32 outlayer_num = 0; UINT32 outlayer_path_list[256] = {0}; UINT32 outlayer_path_list_tmp[256] = {0}; network_get_ai_outlayer_list(proc_id, &outlayer_num, outlayer_path_list_tmp); memcpy(outlayer_path_list, outlayer_path_list_tmp, sizeof(UINT32) * outlayer_num); for (UINT32 i = 0; i < outlayer_num; i++) { VENDOR_AI_BUF layer_tmp; network_get_ai_outlayer_by_path_id(proc_id, outlayer_path_list_tmp[i], &layer_tmp); //printf("%d %hd %s\r\n", i, outlayer_path_list_tmp[i], layer_tmp.name); for (UINT32 k = 0; k < outlayer_num; k++) { if (strncmp(out_layer_name[k], layer_tmp.name, strlen(out_layer_name[k])) == 0) { outlayer_path_list[k] = outlayer_path_list_tmp[i]; //printf(" --> %d %s \r\n", k, conf_loc_layer_name[k]); break; } } } INT32 max_map[2] = {0}; VENDOR_AI_BUF *layer_buffer = (VENDOR_AI_BUF *)malloc(outlayer_num * sizeof(VENDOR_AI_BUF)); FLOAT **out_layer_float = (FLOAT **)malloc(outlayer_num * sizeof(FLOAT)); for (UINT32 i = 0; i < outlayer_num; i++) { network_get_ai_outlayer_by_path_id(proc_id, outlayer_path_list[i], &(layer_buffer[i])); /* INT8 bitdepth = HD_VIDEO_PXLFMT_BITS(layer_buffer[i].fmt); INT8 sign_bits = HD_VIDEO_PXLFMT_SIGN(layer_buffer[i].fmt); INT8 int_bits = HD_VIDEO_PXLFMT_INT(layer_buffer[i].fmt); INT8 frac_bits = HD_VIDEO_PXLFMT_FRAC(layer_buffer[i].fmt); printf(" layer (%s)\n", layer_buffer[i].name); printf(" pa (0x%lx)\n", layer_buffer[i].pa); printf(" va (0x%lx)\n", layer_buffer[i].va); printf(" size (%lu)\n", layer_buffer[i].size); printf(" width (%lu)\n", layer_buffer[i].width); printf(" height (%lu)\n", layer_buffer[i].height); printf(" channel (%lu)\n", layer_buffer[i].channel); printf(" layout (%s)\n", layer_buffer[i].layout); printf(" bitdepth (%d)\n", bitdepth); printf(" sign_bits (%d)\n", sign_bits); printf(" int_bits (%d)\n", int_bits); printf(" frac_bits (%d)\n", frac_bits); printf(" scale_ratio (%f)\n", layer_buffer[i].scale_ratio); */ UINT32 length = layer_buffer[i].width * layer_buffer[i].height * layer_buffer[i].channel * layer_buffer[i].batch_num; out_layer_float[i] = (FLOAT *) malloc(sizeof(FLOAT) * length); if (i == outlayer_num - 1) { max_map[0] = (INT32)layer_buffer[i].width; max_map[1] = (INT32)layer_buffer[i].height; } } // PVD detection out layer include params! default PVD_LAYER_PARAM PVD_params; PVD_params.nms_threshold = 0.5; PVD_params.confindence_threshold = 0.38; PVD_params.pd_limit_param.limit_module = 2; PVD_params.vd_limit_param.limit_module = 2; PVD_params.pd_limit_param.sm_thr_num = 2; PVD_params.vd_limit_param.sm_thr_num = 2; PVD_params.cls_score_thresh = 2;//0.986; char paramsfile[256] = "/mnt/sd/CNNLib/para/pvdcnn/params.txt"; pvdcnn_getparam(&PVD_params, max_map, paramsfile); printf("%f, %f, %f, %d %d %d %d\n", PVD_params.nms_threshold, PVD_params.confindence_threshold, PVD_params.cls_score_thresh, PVD_params.pd_limit_param.limit_module, PVD_params.vd_limit_param.limit_module, PVD_params.pd_limit_param.sm_thr_num, PVD_params.vd_limit_param.sm_thr_num); #if(LIMIT_PVD) FLOAT cls_det_threshold = 0.98; FLOAT cls_threshold = 0.7; INT32 cls_debug_print = 0; char para_file[256]; snprintf(para_file, 256, "/mnt/sd/CNNLib/para/public/para.txt"); limit_fdet_read_para(&cls_det_threshold, &cls_threshold, &cls_debug_print, para_file); printf("det_threshold:%f, cls:thresh: %f, debug_print:%d\n", cls_det_threshold, cls_threshold, cls_debug_print); limit_fdet_params.cls_det_threshold = cls_det_threshold; limit_fdet_params.cls_threshold = cls_threshold; limit_fdet_params.cls_debug_print = cls_debug_print; #endif INT8 pvd_light_night = 9; // >0 light; <0 night pdcnn_para_init(&PVD_params, pvd_light_night); vdcnn_para_init(&PVD_params, pvd_light_night); if (pvd_light_night < 0) { //night PVD_params.cls_score_thresh = PVD_params.cls_score_thresh - 0.2; printf("PVD is in night model now\n"); } PVD_LAYER_MEM PVD_det_mem; PVD_det_mem.swap_data = (FLOAT *)malloc(PVD_params.size_swap_data); PVD_det_mem.class_score = (FLOAT *)malloc(PVD_params.size_class_score); PVD_det_mem.predicts_init = (PVD_PREDICT *)malloc(PVD_params.size_predicts_init); PVD_det_mem.predicts_initVD = (PVD_PREDICT *)malloc(PVD_params.size_predicts_init); // PVD_det_mem.final_pred = (PVDCNN_RESULT*)malloc(size_final_pred); VENDOR_AIS_FLOW_MEM_PARM tmp; MEM_PARM input_mem_parm2; ret = hdal_mem_alloc(&input_mem_parm2, "input_mem_parm", PVD_params.size_final_pred); if (HD_OK != ret) { printf("PVD_det_mem.final_pred_mem allocate!!\n"); return 0; } tmp.pa = input_mem_parm2.pa; tmp.va = input_mem_parm2.va; tmp.size = input_mem_parm2.size; PVD_det_mem.final_pred = (PVDCNN_RESULT *)tmp.va; #if LIMIT_PVD VENDOR_AIS_FLOW_MEM_PARM tmp2; MEM_PARM input_mem_parm3; ret = hdal_mem_alloc(&input_mem_parm3, "input_mem_parm", PVD_params.size_final_pred); if (HD_OK != ret) { printf("PVD_det_mem.final_pred_mem allocate!!\n"); return 0; } tmp2.pa = input_mem_parm3.pa; tmp2.va = input_mem_parm3.va; tmp2.size = input_mem_parm3.size; PVDCNN_RESULT *out_pred = (PVDCNN_RESULT *)tmp2.va; // memset(out_pred, 0, size_final_pred); // CHAR SAVE_TXT2[256]; // FILE *fs2; // sprintf(SAVE_TXT2, "/mnt/sd/det_results/cls_debug.txt"); // fs2 = fopen(SAVE_TXT2, "w+"); #endif memset(PVD_det_mem.swap_data, 0, PVD_params.size_swap_data); memset(PVD_det_mem.class_score, 0, PVD_params.size_class_score); memset(PVD_det_mem.predicts_init, 0, PVD_params.size_predicts_init); memset(PVD_det_mem.predicts_initVD, 0, PVD_params.size_predicts_init); // memset(PVD_det_mem.final_pred, 0, size_final_pred); //scale buf MEM_PARM scale_buf = {0}; ret = model_hdal_mem_get(&scale_buf, (UINT32)SCALE_BUF_SIZE); if (ret != HD_OK) { printf("scale mem get fail (%d)!!\r\n", ret); goto exit_thread; } NET_IN nn_in; //batch test UINT32 net_all_time = 0, post_all_time = 0; UINT32 *posttime; posttime = (UINT32 *)malloc(sizeof(posttime)); *posttime = 0; UINT32 *nmspost; nmspost = (UINT32 *)malloc(sizeof(nmspost)); *nmspost = 0; INT32 img_num = 0; INT32 instances = 0; #if SAVE_SCALE CHAR BMP_FILE[256]; #endif CHAR IMG_PATH[256]; //CHAR IMG_FILE[512]; CHAR SAVE_TXT1[256]; //CHAR SAVE_TXT2[256]; //CHAR SAVE_BIN[256]; CHAR IMG_LIST[256]; CHAR list_infor[256]; CHAR *line_infor; BOOL INPUT_STATE = TRUE; sprintf(IMG_LIST, "/mnt/sd/jpg/pvdlist.txt"); sprintf(IMG_PATH, "/mnt/sd/jpg/PVD"); FILE *fr, *fs1; sprintf(SAVE_TXT1, "/mnt/sd/det_results/PVDresults.txt"); fr = fopen(IMG_LIST, "r"); //fp = fopen(SAVE_BIN, "wb+"); fs1 = fopen(SAVE_TXT1, "w"); INT32 len = 0; CHAR img_name[256] = {0}; CHAR *token; INT32 sl = 0; BOOL iseresize = FALSE; //input image parameters //nn_in.src_img.sign = MAKEFOURCC('A','B','U','F'); //nn_in.src_img.channel = 2; //nn_in.src_img.fmt = HD_VIDEO_PXLFMT_YUV420; //printf("nn_in.src_img_size: %d %d %d\r\n", nn_in.w, nn_in.h, nn_in.src_img.size); INT32 numimg = 0; char imgname[300]; if (NULL == fr) { printf("Failed to open img_list!\r\n"); } while (fgets(list_infor, 256, fr) != NULL) { len = strlen(list_infor); list_infor[len - 1] = '\0'; sl = 0; line_infor = list_infor; while ((token = strtok(line_infor, " ")) != NULL) { if (sl > 2) { break; } if (sl == 0) { strcpy(img_name, token); sprintf(nn_in.input_filename, "%s/%s", IMG_PATH, token); sprintf(imgname, "%s", token); // printf("%s ", token); } if (sl == 1) { nn_in.w = atoi(token); //nn_in.src_img.width = nn_in.w; nn_in.loff = ALIGN_CEIL_4(nn_in.w); //nn_in.loff = 8; // //bin // nn_in.loff = nn_in.w; //nn_in.src_img.line_ofs = nn_in.loff; // printf("%d ",nn_in.loff); // printf("%s ", token); } if (sl == 2) { nn_in.h = atoi(token); //nn_in.src_img.height = nn_in.h; // printf("%s\r\n", token); } line_infor = NULL; sl++; } numimg++; nn_in.c = 2; // printf("nn_in.c = %d \n",nn_in.c); if ((ret = input_open(&nn_in)) != HD_OK) { printf("proc_id(%u) input image open fail !!\n", proc_id); goto exit_thread; } UINT32 img_width_ori = nn_in.w; UINT32 img_height_ori = nn_in.h; iseresize = need_ise_resize(nn_in, PVD_params.input_size); if (debuginf) { printf("img_name: %s, img_num: %d, w: %ld, h: %ld, isresize: %ld\n", img_name, img_num, img_width_ori, img_height_ori, iseresize); } if (iseresize) { gfx_img.dim.w = PVD_params.input_size[0]; gfx_img.dim.h = PVD_params.input_size[1]; gfx_img.format = nn_in.src_img.fmt; gfx_img.lineoffset[0] = ALIGN_CEIL_4(PVD_params.input_size[0]); gfx_img.lineoffset[1] = ALIGN_CEIL_4(PVD_params.input_size[0]); gfx_img.p_phy_addr[0] = scale_buf.pa; gfx_img.p_phy_addr[1] = scale_buf.pa + PVD_params.input_size[0] * PVD_params.input_size[1]; //printf("p_phy_addr[0] = 0x%x, p_phy_addr[1] = 0x%x\r\n", gfx_img.p_phy_addr[0], gfx_img.p_phy_addr[1]); ret = ai_crop_img(&gfx_img, nn_in.src_img, HD_GFX_SCALE_QUALITY_BILINEAR); if (ret != HD_OK) { printf("ai_crop_img fail=%d\n", ret); } gfx_img_to_vendor(&p_src_img, &gfx_img, (UINT32)scale_buf.va); #if SAVE_SCALE FILE *fb; sprintf(BMP_FILE, "/mnt/sd/save_bmp/%s_scale.bin", img_name); fb = fopen(BMP_FILE, "wb+"); fwrite((UINT32 *)scale_buf.va, sizeof(UINT32), (gfx_img.dim.h * gfx_img.dim.w + gfx_img.dim.h * gfx_img.dim.w / 2), fb); fclose(fb); #endif } else { p_src_img = nn_in.src_img; p_src_img.width = nn_in.w; p_src_img.height = nn_in.h; p_src_img.channel = nn_in.c; p_src_img.line_ofs = nn_in.loff; p_src_img.fmt = HD_VIDEO_PXLFMT_YUV420; p_src_img.pa = nn_in.input_mem.pa; p_src_img.va = nn_in.input_mem.va; p_src_img.sign = MAKEFOURCC('A', 'B', 'U', 'F'); p_src_img.size = nn_in.src_img.size; } // printf("input w:%ld, h:%ld\n", p_src_img.width, p_src_img.height); // set input image ret = vendor_ai_net_set(proc_id, VENDOR_AI_NET_PARAM_IN(0, 0), &p_src_img); if (HD_OK != ret) { printf("proc_id(%u) push input fail !!\n", proc_id); continue; } // do net proc gettimeofday(&tstart, NULL); ret = vendor_ai_net_proc(proc_id); if (HD_OK != ret) { printf("proc_id(%u) do net proc fail !!\n", proc_id); continue; } gettimeofday(&tend, NULL); // printf("after vendor_ai_net_proc \n"); net_all_time += (tend.tv_sec - tstart.tv_sec) * 1000000 + (tend.tv_usec - tstart.tv_usec); // do post proc gettimeofday(&tstart, NULL); for (UINT32 i = 0; i < outlayer_num; i++) { hd_common_mem_flush_cache((VOID *)layer_buffer[i].va, layer_buffer[i].size); UINT32 length = layer_buffer[i].width * layer_buffer[i].height * layer_buffer[i].channel * layer_buffer[i].batch_num; ret = vendor_ai_cpu_util_fixed2float((VOID *)layer_buffer[i].va, layer_buffer[i].fmt, out_layer_float[i], layer_buffer[i].scale_ratio, length); } INT32 obj_num = pvdcnn_detectionsample_armneon(layer_buffer, &PVD_params, out_layer_float, &PVD_det_mem, 0, 1, 1, posttime, nmspost); gettimeofday(&tend, NULL); post_all_time += (tend.tv_sec - tstart.tv_sec) * 1000000 + (tend.tv_usec - tstart.tv_usec); PVDCNN_RESULT *objs_info = PVD_det_mem.final_pred; INT32 prenum = obj_num; for (INT32 num = 0; num < obj_num; num++) { // printf("(img_width_ori: %d img_height_ori: %d [%f %f %f %f])\r\n", img_width_ori, img_height_ori, objs_info[num].xmin, objs_info[num].ymin, objs_info[num].xmax, objs_info[num].ymax); FLOAT x1 = objs_info[num].x1 * img_width_ori; FLOAT y1 = objs_info[num].y1 * img_height_ori; FLOAT x2 = objs_info[num].x2 * img_width_ori; FLOAT y2 = objs_info[num].y2 * img_height_ori; INT32 category = objs_info[num].category; FLOAT score = objs_info[num].score; objs_info[num].x1 = x1; objs_info[num].y1 = y1; objs_info[num].x2 = x2; objs_info[num].y2 = y2; objs_info[num].category = category; objs_info[num].score = score; // printf("obj information is: (classtype: %d socre: %f [%f %f %f %f])\r\n", category, score, x1, y1, x2, y); //fprintf(fs1, "%s %d %f %f %f %f %f\r\n", img_name, category, score, x1, y1, x2, y2); } // printf("") // max_distance_mode 0; if (max_distance_mode) { INT32 input_size[] = { 416, 416 }; MEM_PARM scale_buf_roi = {0}; ret = hdal_mem_alloc(&scale_buf_roi, "scale_buf_roi", SCALE_BUF_SIZE); if (HD_OK != ret) { printf("scale_buf_roi get fail (%d)!!\r\n", ret); goto exit_thread; } // printf("before ROI_MULTI scale_buf_roi\n"); // ret = model_hdal_mem_get(&scale_buf_roi, (UINT32)SCALE_BUF_SIZE); // printf("after ROI_MULTI scale_buf_roi\n"); HD_GFX_IMG_BUF scale_img = { 0 }; scale_img.dim.w = input_size[0]; scale_img.dim.h = input_size[1]; scale_img.format = p_src_img.fmt; scale_img.lineoffset[0] = ALIGN_CEIL_4(input_size[0]); scale_img.lineoffset[1] = ALIGN_CEIL_4(input_size[1]); scale_img.p_phy_addr[0] = scale_buf_roi.pa; scale_img.p_phy_addr[1] = scale_buf_roi.pa + input_size[0] * input_size[1]; VENDOR_AI_BUF p_img = { 0 }; FLOAT roi_center_value[4][2] = {{0.3046875, 0.3125}, {0.6953125, 0.3125}, {0.3046875, 0.6875}, {0.6953125, 0.6875}}; //(cx, cy) FLOAT roisize[2] = { PVDROI_MODE3_WIDTH, PVDROI_MODE3_HEIGHT }; INT32 roi_num = 0; if (max_distance_mode == 1) { roi_num = 1; roi_center_value[0][0] = 0.5; roi_center_value[0][1] = 0.466; roisize[0] = PVDROI_MODE1_WIDTH; roisize[1] = PVDROI_MODE1_HEIGHT; } else if (max_distance_mode == 2) { roi_num = 2; roi_center_value[0][0] = 0.32421875; roi_center_value[0][1] = 0.46527778; roi_center_value[1][0] = 0.67578125; roi_center_value[1][1] = 0.46527778; roisize[0] = PVDROI_MODE2_WIDTH; roisize[1] = PVDROI_MODE2_HEIGHT; } else if (max_distance_mode == 3) { roi_num = 4; } else { printf("ERR: Long distance mode only supports 1, 2 or 3, please reset!!\r\n"); exit(0); } PVD_IRECT roi = { 0, 0, (FLOAT)input_size[0], (FLOAT)input_size[1] }; for (INT32 n = 0; n < roi_num; n++) { roi.x1 = (FLOAT)(roi_center_value[n][0] - (FLOAT)roisize[0] / 2.0) * img_width_ori; roi.y1 = (FLOAT)(roi_center_value[n][1] - (FLOAT)roisize[1] / 2.0) * img_height_ori; roi.x2 = (FLOAT)(roi_center_value[n][0] + (FLOAT)roisize[0] / 2.0) * img_width_ori; roi.y2 = (FLOAT)(roi_center_value[n][1] + (FLOAT)roisize[1] / 2.0) * img_height_ori; ret = pdcnn_crop_img(&scale_img, &p_src_img, HD_GFX_SCALE_QUALITY_NULL, &roi); if (ret != HD_OK) { printf("pdcnn_crop_img fail=%d\n", ret); } pd_gfx_img_to_vendor(&p_img, &scale_img, scale_buf_roi.va); #if 0 CHAR YUVFILE[256]; FILE *fb; sprintf(YUVFILE, "/mnt/sd/det_results/AI/yuv/%s_%09ld_%ld.bin", imgname, numimg, n); fb = fopen(YUVFILE, "wb+"); fwrite((UINT8 *)scale_buf_roi.va, sizeof(UINT8), (416 * 416 * 3 / 2), fb); fclose(fb); #endif ret = vendor_ai_net_set(proc_id, VENDOR_AI_NET_PARAM_IN(0, 0), &p_img); if (HD_OK != ret) { printf("proc_id(%u) push input fail !!\n", proc_id); continue; } //printf("set input image\n"); // do net proc // static struct timeval tstart, tend; // gettimeofday(&tstart, NULL); ret = vendor_ai_net_proc(proc_id); if (HD_OK != ret) { printf("proc_id(%u) do net proc fail !!\n", proc_id); continue; } // gettimeofday(&tend, NULL); //printf("after vendor_ai_net_proc \n"); // (*net_all_time) += (tend.tv_sec - tstart.tv_sec) * 1000000 + (tend.tv_usec - tstart.tv_usec); // do post proc // gettimeofday(&tstart, NULL); for (UINT32 i = 0; i < outlayer_num; i++) { hd_common_mem_flush_cache((VOID *)layer_buffer[i].va, layer_buffer[i].size); UINT32 length = layer_buffer[i].width * layer_buffer[i].height * layer_buffer[i].channel * layer_buffer[i].batch_num; ret = vendor_ai_cpu_util_fixed2float((VOID *)layer_buffer[i].va, layer_buffer[i].fmt, out_layer_float[i], layer_buffer[i].scale_ratio, length); } //printf("PVD_detection_out\n"); INT32 obj_num = pvdcnn_detectionsample_armneon(layer_buffer, &PVD_params, out_layer_float, &PVD_det_mem, prenum, 1, 1, posttime, nmspost); // gettimeofday(&tend, NULL); // (*post_all_time) += (tend.tv_sec - tstart.tv_sec) * 1000000 + (tend.tv_usec - tstart.tv_usec); for (INT32 num = 0; num < obj_num; num++) { FLOAT x1 = objs_info[prenum + num].x1 * roisize[0] * img_width_ori; //p_img.width; FLOAT y1 = objs_info[prenum + num].y1 * roisize[1] * img_height_ori; //p_img.height; FLOAT x2 = objs_info[prenum + num].x2 * roisize[0] * img_width_ori; //p_img.width; FLOAT y2 = objs_info[prenum + num].y2 * roisize[1] * img_height_ori; //p_img.height; INT32 category = objs_info[prenum + num].category; FLOAT score = objs_info[prenum + num].score; //printf("max_distance_mode [%f %f %f %f], roi.x1: %f, roi.y1: %f\n", x1, y1, x2, y2, roi.x1, roi.y1); objs_info[prenum + num].x1 = x1 + roi.x1; objs_info[prenum + num].y1 = y1 + roi.y1; objs_info[prenum + num].x2 = x2 + roi.x1; objs_info[prenum + num].y2 = y2 + roi.y1; objs_info[prenum + num].category = category; objs_info[prenum + num].score = score; printf("max_distance_mode prenum: %ld, obj_num:%ld, obj information is: (category: %d socre: %f [%f %f %f %f])\r\n", prenum, obj_num, category, score, x1 + roi.x1, y1 + roi.y1, x2 + roi.x1, y2 + roi.y1); } prenum += (INT32)obj_num; } obj_num = pvdcnn_nonmax_suppress(objs_info, prenum, PVD_params.nms_threshold, 1, 30); //printf("after nms, prenum: %ld, limit_para->limit_fdet: %ld\n", prenum, limit_para->limit_fdet); hdal_mem_free(&scale_buf_roi); } UINT8 pvd_cls; #if LIMIT_PVD pvd_cls = 9; memset(out_pred, 0, PVD_params.size_final_pred); //printf("LIMIT_PVD\n"); limit_fdet_params.ratiow = 1.0; //nn_in.src_img.width / (FLOAT)PD_YUV_WIDTH; limit_fdet_params.ratioh = 1.0; //nn_in.src_img.height / (FLOAT)PD_YUV_HEIGHT; limit_fdet_params.cls_id = 0; limit_fdet_params.pvdcls = 1; limit_fdet_params.proc_num = obj_num; VENDOR_AIS_FLOW_MEM_PARM input_mem; MEM_PARM input_mem_parm; UINT32 input_mem_size = LIMIT_FDET_WIDTH * LIMIT_FDET_WIDTH * 3 / 2; ret = hdal_mem_alloc(&input_mem_parm, "input_mem_parm", input_mem_size); if (HD_OK != ret) { printf("cls input_mem allocate!!\n"); continue; } input_mem.pa = input_mem_parm.pa; input_mem.va = input_mem_parm.va; input_mem.size = input_mem_parm.size; // fprintf(fs2, "%s \t\n", img_name); LIMIT_FDET_MEM limit_fdet_mem = { 0 }; limit_fdet_mem.det_in_mem = tmp; //pdcnn_mem.out_result; limit_fdet_mem.det_out_mem = tmp2; limit_fdet_mem.input_mem = input_mem; ret = limit_fdet_process(&limit_fdet_params, &limit_fdet_mem, &nn_in.src_img); // ret = limit_pvdet_process(&limit_fdet_params, &limit_pvdet_mem, &nn_in.src_img, limit_fdet_params.score_thres); //fprintf(fs2, "\n"); // printf("after limit_fdet_process\n"); if (ret != HD_OK) { printf("ERR: pd limit_fdet_process fail (%ld)!\r\n", ret); goto exit_thread; } //objs_info = out_pred; obj_num = limit_fdet_params.proc_num; hdal_mem_free(&input_mem_parm); if (debuginf) { printf("before cls limit %d\n", obj_num); } if (PVD_params.pd_limit_param.limit_module != 0) obj_num = dynamic_limit(obj_num, &PVD_params.pd_limit_param, PVD_params.cls_score_thresh, pvd_cls, out_pred, 1, img_width_ori, img_height_ori); if (PVD_params.vd_limit_param.limit_module != 0) obj_num = dynamic_limit(obj_num, &PVD_params.vd_limit_param, PVD_params.cls_score_thresh, pvd_cls, out_pred, 2, img_width_ori, img_height_ori); if (debuginf) { printf("after cls limit %d\n", obj_num); } for (INT32 num = 0; num < obj_num; num++) { FLOAT xmin = out_pred[num].x1; FLOAT ymin = out_pred[num].y1; FLOAT xmax = out_pred[num].x2; FLOAT ymax = out_pred[num].y2; INT32 category = out_pred[num].category; FLOAT score = out_pred[num].score; if (debuginf) { printf("finally cls num: %ld obj information is: (category: %d socre: %f [%f %f %f %f])\r\n", obj_num, category, score, xmin, ymin, xmax, ymax); } fprintf(fs1, "%s %d %f %f %f %f %f\r\n", img_name, category, score, xmin, ymin, xmax, ymax); } #else //img_width_ori, img_height_ori // printf("before limit %d\n", obj_num); pvd_cls = -9; if (PVD_params.pd_limit_param.limit_module != 0) obj_num = dynamic_limit(obj_num, &PVD_params.pd_limit_param, PVD_params.cls_score_thresh, pvd_cls, objs_info, 1, img_width_ori, img_height_ori); if (PVD_params.vd_limit_param.limit_module != 0) obj_num = dynamic_limit(obj_num, &PVD_params.vd_limit_param, PVD_params.cls_score_thresh, pvd_cls, objs_info, 2, img_width_ori, img_height_ori); // printf("after limit %d\n", obj_num); for (INT32 num = 0; num < obj_num; num++) { FLOAT xmin = objs_info[num].x1; FLOAT ymin = objs_info[num].y1; FLOAT xmax = objs_info[num].x2; FLOAT ymax = objs_info[num].y2; INT32 category = objs_info[num].category; FLOAT score = objs_info[num].score; if (debuginf) { printf("finally num: %ld obj information is: (category: %d socre: %f [%f %f %f %f])\r\n", obj_num, category, score, xmin, ymin, xmax, ymax); } fprintf(fs1, "%s %d %f %f %f %f %f\r\n", img_name, category, score, xmin, ymin, xmax, ymax); } #endif instances += obj_num; // free buf every time if ((ret = input_close(&nn_in)) != HD_OK) { printf("proc_id(%u) input image close fail !!\n", proc_id); } img_num++; if (instances == 0) { instances = 1; } if (debuginf) printf("img_num: %d, net mean_time: %d, post mean_time: %d, posttime_means: %d, posttime_eachinstance: %d, nmspost_m: %d, nmspost_ecisce: %d\n", img_num, net_all_time / img_num, post_all_time / img_num, (*posttime) / img_num, (*posttime) / instances, (*nmspost) / img_num, (*nmspost) / instances); } printf("tests are done!\n"); if (INPUT_STATE == TRUE) { printf("all test img number is: %d\r\n", img_num); } fclose(fs1); //fclose(fs2); fclose(fr); printf("closed output stream\n"); // #if LIMIT_PVD // fclose(fs2); // #endif printf("net mean_time: %d, post mean_time: %d, posttime_means: %d, posttime_eachinstance: %d, nmspost_m: %d, nmspost_ecisce: %d\n", net_all_time / img_num, post_all_time / img_num, (*posttime) / img_num, (*posttime) / instances, (*nmspost) / img_num, (*nmspost) / instances); free(layer_buffer); for (UINT32 i = 0; i < outlayer_num; i++) { free(out_layer_float[i]); } free(out_layer_float); free(PVD_det_mem.swap_data); free(PVD_det_mem.class_score); free(PVD_det_mem.predicts_init); free(PVD_det_mem.predicts_initVD); free(posttime); free(nmspost); hdal_mem_free(&input_mem_parm2); hdal_mem_free(&nn_in.input_mem); hdal_mem_free(&net_info.io_mem); hdal_mem_free(&net_info.proc_mem); #if LIMIT_PVD hdal_mem_free(&net_info1.io_mem); hdal_mem_free(&net_info1.proc_mem); #endif exit_thread: // stop ret = vendor_ai_net_stop(proc_id); if (HD_OK != ret) { printf("proc_id(%u) nn stop fail !!\n", proc_id); } #if LIMIT_PVD hdal_mem_free(&input_mem_parm3); ret = vendor_ai_net_stop(limit_fdet_params.run_id); if (HD_OK != ret) { printf("proc_id(%u) nn stop fail !!\n", limit_fdet_params.run_id); } if ((ret = network_close(&net_info1)) != HD_OK) { printf("proc_id(%u) nn close fail !!\n", limit_fdet_params.run_id); } #endif if ((ret = input_close(&nn_in)) != HD_OK) { printf("proc_id(%u) input image close fail !!\n", proc_id); } // close network modules if ((ret = network_close(&net_info)) != HD_OK) { printf("proc_id(%u) nn close fail !!\n", proc_id); } //need release scale_buf if ((ret = model_hdal_mem_release(&scale_buf)) != HD_OK) { //printf("ret: %ld\n", ret); //printf("HD_ok: %ld\n", HD_OK); printf("proc_id(%u) mem release fail !!\n", proc_id); } // uninit network modules ret = vendor_ai_uninit(); if (ret != HD_OK) { printf("vendor_ai_uninit fail=%d\n", ret); } return 0; } int main(int argc, char *argv[]) { HD_RESULT ret; //INT32 key; NET_IN nn_in; pthread_t nn_thread_id; // init hdal ret = hd_common_init(0); if (ret != HD_OK) { printf("hd_common_init fail=%d\n", ret); goto exit; } // set project config for AI hd_common_sysconfig(0, (1 << 16), 0, VENDOR_AI_CFG); //enable AI engine ret = hd_gfx_init(); if (ret != HD_OK) { printf("hd_gfx_init fail\r\n"); goto exit; } UINT32 schd = VENDOR_AI_PROC_SCHD_FAIR; vendor_ai_cfg_set(VENDOR_AI_CFG_PLUGIN_ENGINE, vendor_ai_cpu1_get_engine()); vendor_ai_cfg_set(VENDOR_AI_CFG_PROC_SCHD, &schd); ret = vendor_ai_init(); if (ret != HD_OK) { printf("vendor_ai_init fail=%d\n", ret); goto exit; } // create nn_proc_thread ret = pthread_create(&nn_thread_id, NULL, nn_thread_api, (VOID *)(&nn_in)); if (ret < 0) { printf("create encode thread failed"); goto exit; } /*do { printf("usage:\n"); printf(" enter q: exit\n"); printf(" enter r: run engine\n"); key = getchar(); if (key == 'q' || key == 0x3) { is_net_proc = FALSE; is_net_exit = TRUE; break; } else if (key == 'r') { is_net_proc = TRUE; printf("[ai sample] nn proc done!\n"); usleep(500000); continue; }else if (key == 'd'){ //dkn 20210120 //enter debug menu hd_debug_run_menu(); printf("\r\nEnter q to exit, Enter d to debug\r\n"); } } while(1);*/ // wait encode thread destroyed pthread_join(nn_thread_id, NULL); exit: ret = hd_gfx_uninit(); if (ret != HD_OK) { printf("hd_gfx_uninit fail\r\n"); } // uninit memory ret = hd_common_mem_uninit(); if (ret != HD_OK) { printf("mem fail=%d\n", ret); } // uninit hdal ret = hd_common_uninit(); if (ret != HD_OK) { printf("common fail=%d\n", ret); } return ret; }