diff --git a/gst-plugin/src/ascii.h b/gst-plugin/src/ascii.h index d7a5d40..1c016a5 100755 --- a/gst-plugin/src/ascii.h +++ b/gst-plugin/src/ascii.h @@ -26,7 +26,9 @@ #ifndef ASCII_H #define ASCII_H -const char g_ascii_table[][6] = +#include + +const static std::vector> g_ascii_table = { {0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, /* ' '*/ {0x00, 0x00, 0x79, 0x00, 0x00, 0x00}, /* ! */ diff --git a/gst-plugin/src/box.cpp b/gst-plugin/src/box.cpp index cc4eb53..5a971b7 100644 --- a/gst-plugin/src/box.cpp +++ b/gst-plugin/src/box.cpp @@ -100,18 +100,14 @@ float box_iou(Box a, Box b) * th_nms = threshold for nms * Return value : - ******************************************/ -void filter_boxes_nms(std::vector &det, int32_t size, float th_nms) +void filter_boxes_nms(std::vector &det, float th_nms) { - int32_t count = size; - int32_t i = 0; - int32_t j = 0; - Box a; - Box b; - float b_intersection = 0; - for (i = 0; i < count; i++) + std::size_t count = det.size(); + + for (std::size_t i = 0; i < count; i++) { - a = det[i].bbox; - for (j = 0; j < count; j++) + Box a = det[i].bbox; + for (std::size_t j = 0; j < count; j++) { if (i == j) { @@ -121,8 +117,8 @@ void filter_boxes_nms(std::vector &det, int32_t size, float th_nms) { continue; } - b = det[j].bbox; - b_intersection = box_intersection(a, b); + Box b = det[j].bbox; + float b_intersection = box_intersection(a, b); if ((box_iou(a, b)>th_nms) || (b_intersection >= a.h * a.w - 1) || (b_intersection >= b.h * b.w - 1)) { if (det[i].prob > det[j].prob) @@ -136,5 +132,4 @@ void filter_boxes_nms(std::vector &det, int32_t size, float th_nms) } } } - return; } diff --git a/gst-plugin/src/box.h b/gst-plugin/src/box.h index 6a234ab..6ea18d1 100644 --- a/gst-plugin/src/box.h +++ b/gst-plugin/src/box.h @@ -26,9 +26,9 @@ #define BOX_H #include -#include -#include -#include +#include +#include +#include /***************************************** * Box : Bounding box coordinates and its size @@ -55,6 +55,6 @@ float box_iou(Box a, Box b); float overlap(float x1, float w1, float x2, float w2); float box_intersection(Box a, Box b); float box_union(Box a, Box b); -void filter_boxes_nms(std::vector &det, int32_t size, float th_nms); +void filter_boxes_nms(std::vector &det, float th_nms); #endif diff --git a/gst-plugin/src/define.h b/gst-plugin/src/define.h index 83d57ac..324d217 100755 --- a/gst-plugin/src/define.h +++ b/gst-plugin/src/define.h @@ -28,29 +28,22 @@ /***************************************** * includes ******************************************/ -#include -#include +#include +#include #include #include #include #include #include -#include +#include #include +#include #include #include #include #include -#include -#include - -/***************************************** -* Common Static Variables -* - input_img = input image to DRP-AI (size and format are determined in DRP-AI Translator) -* - output_img = output image from the application with bounding box -******************************************/ -const static std::string input_img = "sample.bmp"; -const static std::string output_img = "sample_output.bmp"; +#include +#include /***************************************** * Static Variables and Macro for each YOLO model @@ -107,11 +100,11 @@ const static std::vector label_file_map = { "aeroplane", "bicycle", /* Number of output layers. This value MUST match with the length of num_grids[] below */ #define NUM_INF_OUT_LAYER (1) /* Number of grids in the image. The length of this array MUST match with the NUM_INF_OUT_LAYER */ -const static uint8_t num_grids[] = { 13 }; +const static std::array num_grids = { 13 }; /* Number of DRP-AI output */ const static uint32_t num_inf_out = (NUM_CLASS + 5)* NUM_BB * num_grids[0] * num_grids[0]; /* Anchor box information */ -const static double anchors[] = +const static std::vector anchors = { 1.3221, 1.73145, 3.19275, 4.00944, diff --git a/gst-plugin/src/drpai.cpp b/gst-plugin/src/drpai.cpp index ab65cde..07ee960 100644 --- a/gst-plugin/src/drpai.cpp +++ b/gst-plugin/src/drpai.cpp @@ -5,6 +5,8 @@ #include #include "drpai.h" +#define NOW std::chrono::steady_clock::now() + /***************************************** * Function Name : read_addrmap_txt * Description : Loads address and size of DRP-AI Object files into struct addr. @@ -14,11 +16,6 @@ ******************************************/ int8_t DRPAI::read_addrmap_txt(const std::string& addr_file) { - std::string str; - uint32_t l_addr; - uint32_t l_size; - std::string element, a, s; - std::ifstream ifs(addr_file); if (ifs.fail()) { @@ -26,12 +23,14 @@ int8_t DRPAI::read_addrmap_txt(const std::string& addr_file) return -1; } + std::string str; while (getline(ifs, str)) { std::istringstream iss(str); + std::string element, a, s; iss >> element >> a >> s; - l_addr = strtol(a.c_str(), NULL, 16); - l_size = strtol(s.c_str(), NULL, 16); + uint32_t l_addr = std::stol(a, nullptr, 16); + uint32_t l_size = std::stol(s, nullptr, 16); if ("drp_config" == element) { @@ -100,11 +99,9 @@ int8_t DRPAI::read_addrmap_txt(const std::string& addr_file) int8_t DRPAI::load_data_to_mem(const std::string& data, uint32_t from, uint32_t size) { int8_t ret_load_data = 0; - int8_t obj_fd; + int32_t obj_fd; uint8_t drpai_buf[BUF_SIZE]; drpai_data_t drpai_data; - size_t ret = 0; - int32_t i = 0; printf("Loading : %s\n", data.c_str()); errno = 0; @@ -120,26 +117,23 @@ int8_t DRPAI::load_data_to_mem(const std::string& data, uint32_t from, uint32_t drpai_data.size = size; errno = 0; - ret = ioctl(drpai_fd, DRPAI_ASSIGN, &drpai_data); - if ( -1 == ret ) + if ( ioctl(drpai_fd, DRPAI_ASSIGN, &drpai_data) == -1 ) { fprintf(stderr, "[ERROR] Failed to run DRPAI_ASSIGN: errno=%d\n", errno); ret_load_data = -1; goto end; } - for (i = 0; i < (drpai_data.size / BUF_SIZE); i++) + for (uint32_t i = 0; i < (drpai_data.size / BUF_SIZE); i++) { errno = 0; - ret = read(obj_fd, drpai_buf, BUF_SIZE); - if ( 0 > ret ) + if ( read(obj_fd, drpai_buf, BUF_SIZE) < 0 ) { fprintf(stderr, "[ERROR] Failed to read: %s errno=%d\n", data.c_str(), errno); ret_load_data = -1; goto end; } - ret = write(drpai_fd, drpai_buf, BUF_SIZE); - if ( -1 == ret ) + if ( write(drpai_fd, drpai_buf, BUF_SIZE) == -1 ) { fprintf(stderr, "[ERROR] Failed to write via DRP-AI Driver: errno=%d\n", errno); ret_load_data = -1; @@ -149,15 +143,13 @@ int8_t DRPAI::load_data_to_mem(const std::string& data, uint32_t from, uint32_t if ( 0 != (drpai_data.size % BUF_SIZE)) { errno = 0; - ret = read(obj_fd, drpai_buf, (drpai_data.size % BUF_SIZE)); - if ( 0 > ret ) + if ( read(obj_fd, drpai_buf, (drpai_data.size % BUF_SIZE)) < 0 ) { fprintf(stderr, "[ERROR] Failed to read: %s errno=%d\n", data.c_str(), errno); ret_load_data = -1; goto end; } - ret = write(drpai_fd, drpai_buf, (drpai_data.size % BUF_SIZE)); - if ( -1 == ret ) + if ( write(drpai_fd, drpai_buf, (drpai_data.size % BUF_SIZE)) == -1 ) { fprintf(stderr, "[ERROR] Failed to write via DRP-AI Driver: errno=%d\n", errno); ret_load_data = -1; @@ -185,9 +177,7 @@ int8_t DRPAI::load_drpai_data() { uint32_t addr = 0; uint32_t size = 0; - int32_t i = 0; - size_t ret = 0; - for ( i = 0; i < 5; i++ ) + for (int32_t i = 0; i < 5; i++ ) { switch (i) { @@ -215,8 +205,7 @@ int8_t DRPAI::load_drpai_data() break; } - ret = load_data_to_mem(drpai_file_path[i], addr, size); - if (0 != ret) + if ( load_data_to_mem(drpai_file_path[i], addr, size) != 0 ) { fprintf(stderr,"[ERROR] Failed to load data from memory: %s\n",drpai_file_path[i].c_str()); return -1; @@ -243,7 +232,7 @@ std::vector DRPAI::load_label_file(const std::string& label_file_na return list; } - std::string line = ""; + std::string line; while (getline(infile,line)) { list.push_back(line); @@ -271,43 +260,38 @@ int8_t DRPAI::get_result(uint32_t output_addr, uint32_t output_size) float drpai_buf[BUF_SIZE]; drpai_data.address = output_addr; drpai_data.size = output_size; - int32_t i = 0; - int8_t ret = 0; errno = 0; /* Assign the memory address and size to be read */ - ret = ioctl(drpai_fd, DRPAI_ASSIGN, &drpai_data); - if (-1 == ret) + if ( ioctl(drpai_fd, DRPAI_ASSIGN, &drpai_data) == -1 ) { fprintf(stderr, "[ERROR] Failed to run DRPAI_ASSIGN: errno=%d\n", errno); return -1; } /* Read the memory via DRP-AI Driver and store the output to buffer */ - for (i = 0; i < (drpai_data.size / BUF_SIZE); i++) + for (uint32_t i = 0; i < (drpai_data.size / BUF_SIZE); i++) { errno = 0; - ret = read(drpai_fd, drpai_buf, BUF_SIZE); - if ( -1 == ret ) + if ( read(drpai_fd, drpai_buf, BUF_SIZE) == -1 ) { fprintf(stderr, "[ERROR] Failed to read via DRP-AI Driver: errno=%d\n", errno); return -1; } - memcpy(&drpai_output_buf[BUF_SIZE/sizeof(float)*i], drpai_buf, BUF_SIZE); + std::memcpy(&drpai_output_buf[BUF_SIZE/sizeof(float)*i], drpai_buf, BUF_SIZE); } if ( 0 != (drpai_data.size % BUF_SIZE)) { errno = 0; - ret = read(drpai_fd, drpai_buf, (drpai_data.size % BUF_SIZE)); - if ( -1 == ret) + if ( read(drpai_fd, drpai_buf, (drpai_data.size % BUF_SIZE)) == -1 ) { fprintf(stderr, "[ERROR] Failed to read via DRP-AI Driver: errno=%d\n", errno); return -1; } - memcpy(&drpai_output_buf[(drpai_data.size - (drpai_data.size%BUF_SIZE))/sizeof(float)], drpai_buf, (drpai_data.size % BUF_SIZE)); + std::memcpy(&drpai_output_buf[(drpai_data.size - (drpai_data.size%BUF_SIZE))/sizeof(float)], drpai_buf, (drpai_data.size % BUF_SIZE)); } return 0; } @@ -318,9 +302,9 @@ int8_t DRPAI::get_result(uint32_t output_addr, uint32_t output_size) * Arguments : x = input argument for the calculation * Return value : sigmoid result of input x ******************************************/ -double sigmoid(double x) +float sigmoid(float x) { - return 1.0/(1.0 + exp(-x)); + return 1.0f/(1.0f + std::exp(-x)); } /***************************************** @@ -329,19 +313,19 @@ double sigmoid(double x) * Arguments : val[] = array to be computed softmax * Return value : - ******************************************/ -void softmax(float val[NUM_CLASS]) +void softmax(std::array& val) { float max_num = -FLT_MAX; float sum = 0; int32_t i; for ( i = 0 ; i classes {}; /* Get the class prediction */ - for (i = 0;i < NUM_CLASS;i++) + for (uint32_t i = 0;i < NUM_CLASS;i++) { #if defined(YOLOV3) || defined(TINYYOLOV3) classes[i] = sigmoid(drpai_output_buf[yolo_index(n, offs, 5+i)]); @@ -515,9 +476,9 @@ int8_t DRPAI::print_result_yolo() #if defined(YOLOV2) || defined(TINYYOLOV2) softmax(classes); #endif - max_pred = 0; - pred_class = -1; - for (i = 0; i < NUM_CLASS; i++) + float max_pred = 0; + int32_t pred_class = -1; + for (int32_t i = 0; i < NUM_CLASS; i++) { if (classes[i] > max_pred) { @@ -527,10 +488,10 @@ int8_t DRPAI::print_result_yolo() } /* Store the result into the list if the probability is more than the threshold */ - probability = max_pred * objectness; + float probability = max_pred * objectness; if (probability > TH_PROB) { - d = {bb, pred_class, probability}; + detection d = {bb, pred_class, probability}; det.push_back(d); } } @@ -538,50 +499,50 @@ int8_t DRPAI::print_result_yolo() } } /* Non-Maximum Supression filter */ - filter_boxes_nms(det, det.size(), TH_NMS); + filter_boxes_nms(det, TH_NMS); /* Render boxes on image and print their details */ - n = 1; - printf("DRP-AI detected items: "); - for (const auto& detection: det) - { - /* Skip the overlapped bounding boxes */ - if (detection.prob == 0) continue; - - /* Print the box details on console */ - //print_box(det[i], n++); - printf("%s (%.1f %%)\t", label_file_map[detection.c].c_str(), detection.prob*100); + if(log_detects) { + printf("DRP-AI detected items: "); + for (const auto &detection: det) { + /* Skip the overlapped bounding boxes */ + if (detection.prob == 0) continue; + + /* Print the box details on console */ + //print_box(detection, n++); + printf("%s (%.1f %%)\t", label_file_map[detection.c].c_str(), detection.prob * 100); + } + printf("\n"); } - printf("\n"); return 0; } -int8_t DRPAI::initialize() { +int DRPAI::open_resources() { printf("RZ/V2L DRP-AI Plugin\n"); printf("Model : Darknet YOLO | %s\n", drpai_prefix.c_str()); - int32_t ret = 0; + frame_time = NOW; + if (multithread) + process_thread = new std::thread(&DRPAI::thread_function_loop, this); + else + thread_state = Ready; /* Obtain udmabuf memory area starting address */ - uint64_t udmabuf_address = 0; - int8_t fd = 0; char addr[1024]; - int32_t read_ret = 0; errno = 0; - fd = open("/sys/class/u-dma-buf/udmabuf0/phys_addr", O_RDONLY); + auto fd = open("/sys/class/u-dma-buf/udmabuf0/phys_addr", O_RDONLY); if (0 > fd) { fprintf(stderr, "[ERROR] Failed to open udmabuf0/phys_addr : errno=%d\n", errno); return -1; } - read_ret = read(fd, addr, 1024); - if (0 > read_ret) + if ( read(fd, addr, 1024) < 0 ) { fprintf(stderr, "[ERROR] Failed to read udmabuf0/phys_addr : errno=%d\n", errno); close(fd); return -1; } - sscanf(addr, "%lx", &udmabuf_address); + uint64_t udmabuf_address = std::strtoul(addr, nullptr, 16); close(fd); /* Filter the bit higher than 32 bit */ udmabuf_address &=0xFFFFFFFF; @@ -591,8 +552,7 @@ int8_t DRPAI::initialize() { /**********************************************************************/ /* Read DRP-AI Object files address and size */ - ret = read_addrmap_txt(drpai_address_file); - if (0 != ret) + if ( read_addrmap_txt(drpai_address_file) != 0 ) { fprintf(stderr, "[ERROR] Failed to read addressmap text file: %s\n", drpai_address_file.c_str()); return -1; @@ -618,8 +578,7 @@ int8_t DRPAI::initialize() { } /* Load DRP-AI Data from Filesystem to Memory via DRP-AI Driver */ - ret = load_drpai_data(); - if (0 > ret) + if ( load_drpai_data() < 0) { fprintf(stderr, "[ERROR] Failed to load DRP-AI Object files.\n"); return -1; @@ -641,17 +600,17 @@ int8_t DRPAI::initialize() { proc[DRPAI_INDEX_OUTPUT].address = drpai_address.data_out_addr; proc[DRPAI_INDEX_OUTPUT].size = drpai_address.data_out_size; - if (multithread) - process_thread = new std::thread(&DRPAI::thread_function_loop, this); + if (image_mapped_udma.map_udmabuf() < 0) { + fprintf(stderr, "[ERROR] Failed to map Image buffer to UDMA.\n"); + return -1; + } printf("DRP-AI Ready!\n"); return 0; } -int8_t DRPAI::process(uint8_t* img_data) { - +int DRPAI::process_image(uint8_t* img_data) { { - std::unique_lock state_lock(state_mutex); switch (thread_state) { case Failed: case Unknown: @@ -661,9 +620,11 @@ int8_t DRPAI::process(uint8_t* img_data) { break; case Ready: - Image img (DRPAI_IN_WIDTH, DRPAI_IN_HEIGHT, DRPAI_IN_CHANNEL_BGR); - img.map_udmabuf(); - std::memcpy(img.img_buffer, img_data, img.get_size()); + if(image_mapped_udma.img_buffer) + std::memcpy(image_mapped_udma.img_buffer, img_data, image_mapped_udma.get_size()); + //std::this_thread::sleep_for(std::chrono::milliseconds(50)); + if(show_fps) + drpai_frame_count++; thread_state = Processing; if (multithread) v.notify_one(); @@ -677,6 +638,22 @@ int8_t DRPAI::process(uint8_t* img_data) { Image img (DRPAI_IN_WIDTH, DRPAI_IN_HEIGHT, DRPAI_IN_CHANNEL_BGR); img.img_buffer = img_data; + if(show_fps) { + video_frame_count++; + if (std::chrono::duration(NOW - frame_time).count() >= 1) { + frame_time = NOW; + video_rate = video_frame_count; + drpai_rate = drpai_frame_count; + video_frame_count = 0; + drpai_frame_count = 0; + } + + auto rate_str = "Video Rate: " + std::to_string(int(video_rate)) + " fps"; + img.write_string(rate_str, 0, 0, WHITE_DATA, BLACK_DATA, 5); + rate_str = "DRPAI Rate: " + (drpai_fd ? std::to_string(int(drpai_rate)) + " fps" : "N/A"); + img.write_string(rate_str, 0, 15, WHITE_DATA, BLACK_DATA, 5); + } + /* Compute the result, draw the result on img and display it on console */ const std::unique_lock lock(output_mutex); for (const auto& detection: last_det) @@ -694,7 +671,7 @@ int8_t DRPAI::process(uint8_t* img_data) { return 0; } -int8_t DRPAI::release() { +int DRPAI::release_resources() { if(process_thread) { { std::unique_lock state_lock(state_mutex); @@ -725,90 +702,85 @@ void DRPAI::thread_function_loop() { int8_t DRPAI::thread_function_single() { { std::unique_lock lock(state_mutex); + if (thread_state == Closing) + return -1; + thread_state = Ready; if(multithread) { v.wait(lock, [&] { return thread_state != Ready; }); - if (thread_state == Closing) - return -1; } } + //std::this_thread::sleep_for(std::chrono::milliseconds(50)); - /********************************************************************** - * START Inference - **********************************************************************/ + if(drpai_fd) { + /********************************************************************** + * START Inference + **********************************************************************/ // printf("[START] DRP-AI\n"); - errno = 0; - int ret = ioctl(drpai_fd, DRPAI_START, &proc[0]); - if (0 != ret) { - fprintf(stderr, "[ERROR] Failed to run DRPAI_START: errno=%d\n", errno); - thread_state = Failed; - return -1; - } + errno = 0; + int ret = ioctl(drpai_fd, DRPAI_START, &proc[0]); + if (0 != ret) { + fprintf(stderr, "[ERROR] Failed to run DRPAI_START: errno=%d\n", errno); + thread_state = Failed; + return -1; + } - /********************************************************************** - * Wait until the DRP-AI finish (Thread will sleep) - **********************************************************************/ - fd_set rfds; - int8_t ret_drpai; - drpai_status_t drpai_status; + /********************************************************************** + * Wait until the DRP-AI finish (Thread will sleep) + **********************************************************************/ + fd_set rfds; + drpai_status_t drpai_status; - FD_ZERO(&rfds); - FD_SET(drpai_fd, &rfds); - timeval tv { DRPAI_TIMEOUT, 0 }; + FD_ZERO(&rfds); + FD_SET(drpai_fd, &rfds); + timeval tv{DRPAI_TIMEOUT, 0}; - ret_drpai = select(drpai_fd + 1, &rfds, NULL, NULL, &tv); - if (0 == ret_drpai) { - fprintf(stderr, "[ERROR] DRP-AI select() Timeout : errno=%d\n", errno); - thread_state = Failed; - return -1; - } else if (-1 == ret_drpai) { - fprintf(stderr, "[ERROR] DRP-AI select() Error : errno=%d\n", errno); - ret_drpai = ioctl(drpai_fd, DRPAI_GET_STATUS, &drpai_status); - if (-1 == ret_drpai) { - fprintf(stderr, "[ERROR] Failed to run DRPAI_GET_STATUS : errno=%d\n", errno); + switch (select(drpai_fd + 1, &rfds, nullptr, nullptr, &tv)) { + case 0: + fprintf(stderr, "[ERROR] DRP-AI select() Timeout : errno=%d\n", errno); + thread_state = Failed; + return -1; + case -1: + fprintf(stderr, "[ERROR] DRP-AI select() Error : errno=%d\n", errno); + if (ioctl(drpai_fd, DRPAI_GET_STATUS, &drpai_status) == -1) { + fprintf(stderr, "[ERROR] Failed to run DRPAI_GET_STATUS : errno=%d\n", errno); + } + thread_state = Failed; + return -1; } - thread_state = Failed; - return -1; - } else { - /*Do nothing*/ - } - if (FD_ISSET(drpai_fd, &rfds)) { - errno = 0; - ret_drpai = ioctl(drpai_fd, DRPAI_GET_STATUS, &drpai_status); - if (-1 == ret_drpai) { - fprintf(stderr, "[ERROR] Failed to run DRPAI_GET_STATUS : errno=%d\n", errno); - thread_state = Failed; - return -1; - } + if (FD_ISSET(drpai_fd, &rfds)) { + errno = 0; + if (ioctl(drpai_fd, DRPAI_GET_STATUS, &drpai_status) == -1) { + fprintf(stderr, "[ERROR] Failed to run DRPAI_GET_STATUS : errno=%d\n", errno); + thread_state = Failed; + return -1; + } // printf("[END] DRP-AI\n"); - } + } - /********************************************************************** - * CPU Post-processing - **********************************************************************/ + /********************************************************************** + * CPU Post-processing + **********************************************************************/ - /* Get the output data from memory */ - ret = get_result(drpai_address.data_out_addr, drpai_address.data_out_size); - if (0 != ret) - { - fprintf(stderr, "[ERROR] Failed to get result from memory.\n"); - thread_state = Failed; - return -1; - } + /* Get the output data from memory */ + if (get_result(drpai_address.data_out_addr, drpai_address.data_out_size) != 0) { + fprintf(stderr, "[ERROR] Failed to get result from memory.\n"); + thread_state = Failed; + return -1; + } - ret = print_result_yolo(); - if (0 != ret) - { - fprintf(stderr, "[ERROR] Failed to run CPU Post Processing.\n"); - thread_state = Failed; - return -1; - } + if (print_result_yolo() != 0) { + fprintf(stderr, "[ERROR] Failed to run CPU Post Processing.\n"); + thread_state = Failed; + return -1; + } - { - const std::unique_lock lock(output_mutex); - last_det = det; + { + const std::unique_lock lock(output_mutex); + last_det = det; + } } return 0; } diff --git a/gst-plugin/src/drpai.h b/gst-plugin/src/drpai.h index cb34201..4f8a42c 100644 --- a/gst-plugin/src/drpai.h +++ b/gst-plugin/src/drpai.h @@ -13,40 +13,50 @@ #include "image.h" #include #include +#include #include #include #include +#include class DRPAI { - enum ThreadState { Unknown, Ready, Processing, Failed, Closing }; - public: - DRPAI(bool multithread): - multithread(multithread) {}; - int8_t initialize(); - int8_t process(uint8_t* img_data); - int8_t release(); + explicit DRPAI(): + image_mapped_udma(DRPAI_IN_WIDTH, DRPAI_IN_HEIGHT, DRPAI_IN_CHANNEL_BGR) {}; + + bool multithread = true; + bool log_detects = false; + bool show_fps = false; + int open_resources(); + int process_image(uint8_t* img_data); + int release_resources(); private: - int8_t drpai_fd = 0; + int32_t drpai_fd = 0; st_addr_t drpai_address{}; - float drpai_output_buf[num_inf_out]{}; - std::vector det; - std::vector last_det; std::vector labels; - drpai_data_t proc[DRPAI_INDEX_NUM]{}; - + std::array proc {}; + Image image_mapped_udma; int8_t read_addrmap_txt(const std::string& addr_file); int8_t load_drpai_data(); int8_t load_data_to_mem(const std::string& data, uint32_t from, uint32_t size); + std::vector load_label_file(const std::string& label_file_name); + + /* Output Section */ + std::array drpai_output_buf {}; + std::vector det{}; + std::vector last_det{}; int8_t get_result(uint32_t output_addr, uint32_t output_size); int8_t print_result_yolo(); - std::vector load_label_file(const std::string& label_file_name); - bool multithread; + /* FPS Section */ + std::chrono::time_point frame_time; + int8_t video_frame_count = 0, drpai_frame_count = 0; + double video_rate=0, drpai_rate=0; /* Thread Section */ + enum ThreadState { Unknown, Ready, Processing, Failed, Closing }; ThreadState thread_state = Unknown; std::thread* process_thread = nullptr; std::mutex output_mutex; diff --git a/gst-plugin/src/gstdrpai.cpp b/gst-plugin/src/gstdrpai.cpp index c5b7e9d..4145a3f 100644 --- a/gst-plugin/src/gstdrpai.cpp +++ b/gst-plugin/src/gstdrpai.cpp @@ -75,19 +75,21 @@ enum { enum { PROP_0, - PROP_SILENT, PROP_MULTITHREAD, + PROP_SHOW_FPS, + PROP_LOG_DETECTS, + PROP_STOP_ERROR, }; /* the capabilities of the inputs and outputs. * * describe the real formats here. */ +auto pad_caps = "video/x-raw, width = (int) 640, height = (int) 480, format = (string) BGR"; static GstStaticPadTemplate sink_factory = - GST_STATIC_PAD_TEMPLATE("sink", GST_PAD_SINK, GST_PAD_ALWAYS, GST_STATIC_CAPS("ANY")); - + GST_STATIC_PAD_TEMPLATE("sink", GST_PAD_SINK, GST_PAD_ALWAYS, GST_STATIC_CAPS(pad_caps)); static GstStaticPadTemplate src_factory = - GST_STATIC_PAD_TEMPLATE ("src", GST_PAD_SRC, GST_PAD_ALWAYS, GST_STATIC_CAPS("ANY")); + GST_STATIC_PAD_TEMPLATE ("src", GST_PAD_SRC, GST_PAD_ALWAYS, GST_STATIC_CAPS(pad_caps)); #define gst_drpai_parent_class parent_class @@ -119,11 +121,17 @@ gst_drpai_class_init(GstDRPAIClass *klass) { gstelement_class->change_state = gst_drpai_change_state; - g_object_class_install_property(gobject_class, PROP_SILENT, - g_param_spec_boolean("silent", "Silent", "Produce verbose output ?", - FALSE, G_PARAM_READWRITE)); g_object_class_install_property(gobject_class, PROP_MULTITHREAD, - g_param_spec_boolean("multithread", "MultiThread", "Use MultiThreading", + g_param_spec_boolean("multithread", "MultiThread", "Use a separate thread for object detection.", + TRUE, G_PARAM_READWRITE)); + g_object_class_install_property(gobject_class, PROP_LOG_DETECTS, + g_param_spec_boolean("log_detects", "Log Detects", "Print detected objects in standard output.", + FALSE, G_PARAM_READWRITE)); + g_object_class_install_property(gobject_class, PROP_SHOW_FPS, + g_param_spec_boolean("show_fps", "Show Frame Rates", "Render video and object detection frame rates at the corner of the video.", + FALSE, G_PARAM_READWRITE)); + g_object_class_install_property(gobject_class, PROP_STOP_ERROR, + g_param_spec_boolean("stop_error", "Stop On Errors", "Stop the gstreamer if kernel modules fail to open.", TRUE, G_PARAM_READWRITE)); gst_element_class_set_details_simple(gstelement_class, @@ -143,21 +151,21 @@ gst_drpai_class_init(GstDRPAIClass *klass) { * initialize instance structure */ static void -gst_drpai_init(GstDRPAI *filter) { - filter->sinkpad = gst_pad_new_from_static_template(&sink_factory, "sink"); - gst_pad_set_event_function (filter->sinkpad, +gst_drpai_init(GstDRPAI *obj) { + obj->sinkpad = gst_pad_new_from_static_template(&sink_factory, "sink"); + gst_pad_set_event_function (obj->sinkpad, GST_DEBUG_FUNCPTR(gst_drpai_sink_event)); - gst_pad_set_chain_function (filter->sinkpad, + gst_pad_set_chain_function (obj->sinkpad, GST_DEBUG_FUNCPTR(gst_drpai_chain)); - GST_PAD_SET_PROXY_CAPS (filter->sinkpad); - gst_element_add_pad(GST_ELEMENT (filter), filter->sinkpad); + GST_PAD_SET_PROXY_CAPS (obj->sinkpad); + gst_element_add_pad(GST_ELEMENT (obj), obj->sinkpad); - filter->srcpad = gst_pad_new_from_static_template(&src_factory, "src"); - GST_PAD_SET_PROXY_CAPS (filter->srcpad); - gst_element_add_pad(GST_ELEMENT (filter), filter->srcpad); + obj->srcpad = gst_pad_new_from_static_template(&src_factory, "src"); + GST_PAD_SET_PROXY_CAPS (obj->srcpad); + gst_element_add_pad(GST_ELEMENT (obj), obj->srcpad); - filter->silent = FALSE; - filter->multithread = TRUE; + obj->drpai = new DRPAI(); + obj->stop_error = TRUE; } static GstStateChangeReturn @@ -168,9 +176,9 @@ gst_drpai_change_state (GstElement * element, GstStateChange transition) { switch (transition) { case GST_STATE_CHANGE_NULL_TO_READY: /* open the device */ - obj->drpai = new DRPAI(obj->multithread); - if(obj->drpai->initialize() == -1) - return GST_STATE_CHANGE_FAILURE; + if ( obj->drpai->open_resources() == -1 ) + if (obj->stop_error) + return GST_STATE_CHANGE_FAILURE; break; default: break; @@ -178,11 +186,11 @@ gst_drpai_change_state (GstElement * element, GstStateChange transition) { state_change_ret = GST_ELEMENT_CLASS (parent_class)->change_state (element, transition); - int8_t ret; + int ret; switch (transition) { case GST_STATE_CHANGE_READY_TO_NULL: /* close the device */ - ret = obj->drpai->release(); + ret = obj->drpai->release_resources(); delete obj->drpai; if (ret == -1) return GST_STATE_CHANGE_FAILURE; @@ -197,14 +205,20 @@ gst_drpai_change_state (GstElement * element, GstStateChange transition) { static void gst_drpai_set_property(GObject *object, guint prop_id, const GValue *value, GParamSpec *pspec) { - GstDRPAI *filter = GST_PLUGIN_DRPAI(object); + GstDRPAI *obj = GST_PLUGIN_DRPAI(object); switch (prop_id) { - case PROP_SILENT: - filter->silent = g_value_get_boolean(value); - break; case PROP_MULTITHREAD: - filter->multithread = g_value_get_boolean(value); + obj->drpai->multithread = g_value_get_boolean(value); + break; + case PROP_LOG_DETECTS: + obj->drpai->log_detects = g_value_get_boolean(value); + break; + case PROP_SHOW_FPS: + obj->drpai->show_fps = g_value_get_boolean(value); + break; + case PROP_STOP_ERROR: + obj->stop_error = g_value_get_boolean(value); break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); @@ -215,14 +229,20 @@ gst_drpai_set_property(GObject *object, guint prop_id, static void gst_drpai_get_property(GObject *object, guint prop_id, GValue *value, GParamSpec *pspec) { - GstDRPAI *filter = GST_PLUGIN_DRPAI(object); + GstDRPAI *obj = GST_PLUGIN_DRPAI(object); switch (prop_id) { - case PROP_SILENT: - g_value_set_boolean(value, filter->silent); - break; case PROP_MULTITHREAD: - g_value_set_boolean(value, filter->multithread); + g_value_set_boolean(value, obj->drpai->multithread); + break; + case PROP_LOG_DETECTS: + g_value_set_boolean(value, obj->drpai->log_detects); + break; + case PROP_SHOW_FPS: + g_value_set_boolean(value, obj->drpai->show_fps); + break; + case PROP_STOP_ERROR: + g_value_set_boolean(value, obj->stop_error); break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); @@ -236,12 +256,12 @@ gst_drpai_get_property(GObject *object, guint prop_id, static gboolean gst_drpai_sink_event(GstPad *pad, GstObject *parent, GstEvent *event) { - GstDRPAI *filter; + GstDRPAI *obj; gboolean ret; - filter = GST_PLUGIN_DRPAI(parent); + obj = GST_PLUGIN_DRPAI(parent); - GST_LOG_OBJECT (filter, "Received %s event: %" GST_PTR_FORMAT, + GST_LOG_OBJECT (obj, "Received %s event: %" GST_PTR_FORMAT, GST_EVENT_TYPE_NAME(event), event); switch (GST_EVENT_TYPE (event)) { @@ -267,22 +287,24 @@ gst_drpai_sink_event(GstPad *pad, GstObject *parent, */ static GstFlowReturn gst_drpai_chain(GstPad *pad, GstObject *parent, GstBuffer *buf) { - GstDRPAI *filter; + GstDRPAI *obj; - filter = GST_PLUGIN_DRPAI(parent); + obj = GST_PLUGIN_DRPAI(parent); GstMapInfo info; gst_buffer_map(buf, &info, GST_MAP_READWRITE); - if (filter->drpai->process(info.data) == -1) { - gst_buffer_unref (buf); - return GST_FLOW_ERROR; + if (obj->drpai->process_image(info.data) == -1) { + if(obj->stop_error) { + gst_buffer_unref (buf); + return GST_FLOW_ERROR; + } } gst_buffer_unmap(buf, &info); /* just push out the incoming buffer without touching it */ - return gst_pad_push(filter->srcpad, buf); + return gst_pad_push(obj->srcpad, buf); } diff --git a/gst-plugin/src/gstdrpai.h b/gst-plugin/src/gstdrpai.h index 46358a8..e6b50ce 100644 --- a/gst-plugin/src/gstdrpai.h +++ b/gst-plugin/src/gstdrpai.h @@ -62,8 +62,7 @@ struct _GstDRPAI GstPad *sinkpad, *srcpad; - gboolean silent; - gboolean multithread; + gboolean stop_error; DRPAI *drpai; }; diff --git a/gst-plugin/src/image.cpp b/gst-plugin/src/image.cpp index 1d5133b..9d63824 100755 --- a/gst-plugin/src/image.cpp +++ b/gst-plugin/src/image.cpp @@ -49,13 +49,12 @@ Image::~Image() ******************************************/ uint8_t Image::map_udmabuf() { - int32_t i = 0; udmabuf_fd = open("/dev/udmabuf0", O_RDWR ); if (udmabuf_fd < 0) { return -1; } - img_buffer =(uint8_t*) mmap(NULL, size ,PROT_READ|PROT_WRITE, MAP_SHARED, udmabuf_fd, 0); + img_buffer =(uint8_t*) mmap(nullptr, size ,PROT_READ|PROT_WRITE, MAP_SHARED, udmabuf_fd, 0); if (img_buffer == MAP_FAILED) { @@ -65,7 +64,7 @@ uint8_t Image::map_udmabuf() * Note: Do not use memset() for this. * Because it does not work as expected. */ { - for (i = 0 ; i < size; i++) + for (int32_t i = 0 ; i < size; i++) { img_buffer[i] = 0; } @@ -82,28 +81,27 @@ uint8_t Image::map_udmabuf() * Return value : 0 if succeeded * not 0 otherwise ******************************************/ -uint8_t Image::read_bmp(std::string filename) +uint8_t Image::read_bmp(const std::string& filename) { uint32_t width = img_w; uint32_t height = img_h; uint32_t channel = img_c; - int32_t i = 0; - FILE *fp = NULL; + FILE *fp = nullptr; size_t ret = 0; uint8_t * bmp_line_data; /* Number of byte in single row */ /* NOTE: Number of byte in single row of Windows Bitmap image must be aligned to 4 bytes. */ - int32_t line_width = width * channel + width % 4; + uint32_t line_width = width * channel + width % 4; /* Read header for Windows Bitmap v3 file. */ fp = fopen(filename.c_str(), "rb"); - if (NULL == fp) + if (nullptr == fp) { return -1; } /* Read all header */ - ret = fread(bmp_header, sizeof(uint8_t), header_size, fp); + ret = fread(bmp_header.data(), sizeof(uint8_t), header_size, fp); if (!ret) { fclose(fp); @@ -111,14 +109,14 @@ uint8_t Image::read_bmp(std::string filename) } /* Single row image data */ bmp_line_data = (uint8_t *) malloc(sizeof(uint8_t) * line_width); - if (NULL == bmp_line_data) + if (nullptr == bmp_line_data) { free(bmp_line_data); fclose(fp); return -1; } - for (i = height-1; i >= 0; i--) + for (int32_t i = (int32_t)height-1; i >= 0; i--) { ret = fread(bmp_line_data, sizeof(uint8_t), line_width, fp); if (!ret) @@ -127,7 +125,7 @@ uint8_t Image::read_bmp(std::string filename) fclose(fp); return -1; } - memcpy(img_buffer+i*width*channel, bmp_line_data, sizeof(uint8_t)*width*channel); + std::memcpy(img_buffer+i*width*channel, bmp_line_data, sizeof(uint8_t)*width*channel); } free(bmp_line_data); @@ -145,10 +143,9 @@ uint8_t Image::read_bmp(std::string filename) * Return value : 0 if suceeded * not 0 otherwise ******************************************/ -uint8_t Image::save_bmp(std::string filename) +uint8_t Image::save_bmp(const std::string& filename) const { - int32_t i = 0; - FILE * fp = NULL; + FILE * fp = nullptr; uint8_t * bmp_line_data; uint32_t width = img_w; uint32_t height = img_h; @@ -156,26 +153,26 @@ uint8_t Image::save_bmp(std::string filename) size_t ret = 0; /* Number of byte in single row */ - int32_t line_width = width * channel + width % 4; + uint32_t line_width = width * channel + width % 4; fp = fopen(filename.c_str(), "wb"); - if (NULL == fp) + if (nullptr == fp) { return -1; } /* Write header for Windows Bitmap v3 file. */ - fwrite(bmp_header, sizeof(uint8_t), header_size, fp); + fwrite(bmp_header.data(), sizeof(uint8_t), header_size, fp); bmp_line_data = (uint8_t *) malloc(sizeof(uint8_t) * line_width); - if (NULL == bmp_line_data) + if (nullptr == bmp_line_data) { free(bmp_line_data); fclose(fp); return -1; } - for (i = height - 1; i >= 0; i--) + for (int32_t i = (int32_t)height - 1; i >= 0; i--) { memcpy(bmp_line_data, img_buffer + i*width*channel, sizeof(uint8_t)*width*channel); ret = fwrite(bmp_line_data, sizeof(uint8_t), line_width, fp); @@ -246,16 +243,27 @@ void Image::write_char(char code, int32_t x, int32_t y, int32_t color, int32_t b * backcolor = character background color * Return Value : - ******************************************/ -void Image::write_string(const std::string& pcode, int32_t x, int32_t y, int32_t color, int32_t backcolor) +void Image::write_string(const std::string& pcode, int32_t x, int32_t y, + int32_t color, int32_t backcolor, int8_t margin) { - size_t i = 0; + const auto str_size = (int32_t)pcode.size(); + if(margin > 0) { + int32_t right = margin*2 + str_size * font_w - 1; + int32_t bottom = margin*2 + font_h - 1; + draw_line(x, y, x + right, y, backcolor); + draw_line(x, y+bottom, x+right, y+bottom, backcolor); + draw_line(x, y, x, y+bottom, backcolor); + draw_line(x+right, y, x+right, y+bottom, backcolor); + write_string(pcode, x+1, y+1, color, backcolor, int8_t(margin-1)); + return; + } - x = x < 0 ? 2 : x; - x = (x > img_w - (i * font_w)) ? img_w - (i * font_w)-2 : x; - y = y < 0 ? 2 : y; - y = (y > img_h - font_h) ? img_h - font_h - 2 : y; + x = std::max(0, x); + x = std::min(img_w - str_size*font_w - 2, x); + y = std::max(0, y); + y = std::min(img_h - font_h - 2, y); - for (i = 0; i < pcode.length(); i++) + for (int32_t i = 0; i < str_size; i++) { write_char(pcode[i], (x + (i * font_w)), y, color, backcolor); } @@ -313,13 +321,13 @@ void Image::draw_line(int32_t x0, int32_t y0, int32_t x1, int32_t y1, int32_t co if (dx > dy) { /* Horizontal Line */ - for (i = dx, de = i / 2; i; i--) + for (i = dx, de = (float)i / 2; i; i--) { x0 += sx; - de += dy; - if (de > dx) + de += (float)dy; + if (de > (float)dx) { - de -= dx; + de -= (float)dx; y0 += sy; } draw_point(x0, y0, color); @@ -328,19 +336,18 @@ void Image::draw_line(int32_t x0, int32_t y0, int32_t x1, int32_t y1, int32_t co else { /* Vertical Line */ - for (i = dy, de = i / 2; i; i--) + for (i = dy, de = (float)i / 2; i; i--) { y0 += sy; - de += dx; - if (de > dy) + de += (float)dx; + if (de > (float)dy) { - de -= dy; + de -= (float)dy; x0 += sx; } draw_point(x0, y0, color); } } - return; } /***************************************** @@ -355,10 +362,10 @@ void Image::draw_line(int32_t x0, int32_t y0, int32_t x1, int32_t y1, int32_t co ******************************************/ void Image::draw_rect(int32_t x, int32_t y, int32_t w, int32_t h, const std::string& str) { - int32_t x_min = x - round(w / 2.); - int32_t y_min = y - round(h / 2.); - int32_t x_max = x + round(w / 2.) - 1; - int32_t y_max = y + round(h / 2.) - 1; + int32_t x_min = x - (int32_t)round(w / 2.); + int32_t y_min = y - (int32_t)round(h / 2.); + int32_t x_max = x + (int32_t)round(w / 2.) - 1; + int32_t y_max = y + (int32_t)round(h / 2.) - 1; /* Check the bounding box is in the image range */ x_min = x_min < 1 ? 1 : x_min; x_max = ((img_w - 2) < x_max) ? (img_w - 2) : x_max; @@ -366,7 +373,7 @@ void Image::draw_rect(int32_t x, int32_t y, int32_t w, int32_t h, const std::str y_max = ((img_h - 2) < y_max) ? (img_h - 2) : y_max; /* Draw the class and probability */ - write_string(str, x_min + 1, y_min + 1, back_color, front_color); + write_string(str, x_min + 1, y_min + 1, back_color, front_color, 5); /* Draw the bounding box */ draw_line(x_min, y_min, x_max, y_min, front_color); draw_line(x_max, y_min, x_max, y_max, front_color); @@ -386,7 +393,7 @@ void Image::draw_rect(int32_t x, int32_t y, int32_t w, int32_t h, const std::str * Arguments : a = index of img_buffer * Return Value : value of img_buffer at index a ******************************************/ -uint8_t Image::at(int32_t a) +uint8_t Image::at(int32_t a) const { return img_buffer[a]; } diff --git a/gst-plugin/src/image.h b/gst-plugin/src/image.h index 43a577b..a4d9c17 100755 --- a/gst-plugin/src/image.h +++ b/gst-plugin/src/image.h @@ -34,19 +34,21 @@ class Image img_w(w), img_h(h), img_c(c), size(w * h * c) {}; ~Image(); - uint8_t * img_buffer{}; - int32_t get_size() const { return size; } - uint8_t at(int32_t a); + uint8_t* img_buffer = nullptr; + [[nodiscard]] std::size_t get_size() const { return size; } + [[nodiscard]] uint8_t at(int32_t a) const; void set(int32_t a, uint8_t val); uint8_t map_udmabuf(); - uint8_t read_bmp(std::string filename); - uint8_t save_bmp(std::string filename); + uint8_t read_bmp(const std::string& filename); + uint8_t save_bmp(const std::string& filename) const; void draw_rect(int32_t x, int32_t y, int32_t w, int32_t h, const std::string& str); + void write_string(const std::string& pcode, int32_t x, int32_t y, + int32_t color, int32_t backcolor, int8_t margin=0); private: uint8_t header_size = FILEHEADERSIZE+INFOHEADERSIZE_W_V3; - uint8_t bmp_header[FILEHEADERSIZE+INFOHEADERSIZE_W_V3]{}; + std::array bmp_header {}; uint8_t udmabuf_fd = 0; int32_t img_w; int32_t img_h; @@ -60,8 +62,6 @@ class Image void draw_point(int32_t x, int32_t y, int32_t color); void draw_line(int32_t x0, int32_t y0, int32_t x1, int32_t y1, int32_t color); void write_char(char code, int32_t x, int32_t y, int32_t color, int32_t backcolor); - void write_string(const std::string& pcode, int32_t x, int32_t y, int32_t color, int32_t backcolor); - }; #endif