Merge branch 'feature/face_recognition' into 'v2'

Feature/face recognition

See merge request face-recognition-framework/esp-who!68

components/esp-dl

@@ -1 +1 @@
-Subproject commit bad67b57f8123e76c740ba64e0abe39aff71509d
+Subproject commit c4735f45b64dd529006db116c0bf1d695e34de63

components/fb_gfx/CMakeLists.txt

@@ -1,5 +1,5 @@
-set(COMPONENT_SRCS "fb_gfx.c")
+set(src_dirs .)
-set(COMPONENT_ADD_INCLUDEDIRS "include")
+set(include_dirs .
-set(COMPONENT_PRIV_INCLUDEDIRS "")
+                ./include)
-set(COMPONENT_PRIV_REQUIRES newlib)
+set(requires  esp32-camera)
-register_component()
+idf_component_register(SRC_DIRS ${src_dirs} INCLUDE_DIRS ${include_dirs} REQUIRES ${requires})

components/fb_gfx/fb_gfx.c

@@ -38,10 +38,24 @@ typedef struct
 #include "FreeMonoBold12pt7b.h" //14x24
 #define gfxFont ((GFXfont *)(&FreeMonoBold12pt7b))
 
-void fb_gfx_fillRect(fb_data_t *fb, int32_t x, int32_t y, int32_t w, int32_t h, uint32_t color)
+void fb_gfx_fillRect(camera_fb_t *fb, int32_t x, int32_t y, int32_t w, int32_t h, uint32_t color)
 {
+    int bytes_per_pixel = 0;
+    switch (fb->format)
+    {
+    case PIXFORMAT_GRAYSCALE:
+        bytes_per_pixel = 1;
+        break;
+    case PIXFORMAT_RGB565:
+        bytes_per_pixel = 2;
+        break;
+    case PIXFORMAT_RGB888:
+        bytes_per_pixel = 3;
+    default:
+        break;
+    }
     int32_t line_step = (fb->width - w) * 3;
-    uint8_t *data = fb->data + ((x + (y * fb->width)) * 3);
+    uint8_t *data = fb->buf + ((x + (y * fb->width)) * bytes_per_pixel);
     uint8_t c0 = color >> 16;
     uint8_t c1 = color >> 8;
     uint8_t c2 = color;
@@ -49,26 +63,41 @@ void fb_gfx_fillRect(fb_data_t *fb, int32_t x, int32_t y, int32_t w, int32_t h,
     {
         for (int j = 0; j < w; j++)
         {
+            switch (bytes_per_pixel)
+            {
+            case 1:
+                data[0] = c2;
+                data++;
+                break;
+            case 2:
+                data[0] = c1;
+                data[1] = c2;
+                data += 2;
+                break;
+            case 3:
                 data[0] = c0;
                 data[1] = c1;
                 data[2] = c2;
                 data += 3;
+            default:
+                break;
+            }
         }
         data += line_step;
     }
 }
 
-void fb_gfx_drawFastHLine(fb_data_t *fb, int32_t x, int32_t y, int32_t w, uint32_t color)
+void fb_gfx_drawFastHLine(camera_fb_t *fb, int32_t x, int32_t y, int32_t w, uint32_t color)
 {
     fb_gfx_fillRect(fb, x, y, w, 1, color);
 }
 
-void fb_gfx_drawFastVLine(fb_data_t *fb, int32_t x, int32_t y, int32_t h, uint32_t color)
+void fb_gfx_drawFastVLine(camera_fb_t *fb, int32_t x, int32_t y, int32_t h, uint32_t color)
 {
     fb_gfx_fillRect(fb, x, y, 1, h, color);
 }
 
-uint8_t fb_gfx_putc(fb_data_t *fb, int32_t x, int32_t y, uint32_t color, unsigned char c)
+uint8_t fb_gfx_putc(camera_fb_t *fb, int32_t x, int32_t y, uint32_t color, unsigned char c)
 {
     uint16_t line_width;
     uint8_t xa = 0, bit = 0, bits = 0, xx, yy;
@@ -120,7 +149,7 @@ uint8_t fb_gfx_putc(fb_data_t *fb, int32_t x, int32_t y, uint32_t color, unsigne
     return xa;
 }
 
-uint32_t fb_gfx_print(fb_data_t *fb, int x, int y, uint32_t color, const char *str)
+uint32_t fb_gfx_print(camera_fb_t *fb, int x, int y, uint32_t color, const char *str)
 {
     uint32_t l = 0;
     int xc = x, yc = y, lc = fb->width - gfxFont->glyph[0].xAdvance;
@@ -151,7 +180,7 @@ uint32_t fb_gfx_print(fb_data_t *fb, int x, int y, uint32_t color, const char *s
     return l;
 }
 
-uint32_t fb_gfx_printf(fb_data_t *fb, int32_t x, int32_t y, uint32_t color, const char *format, ...)
+uint32_t fb_gfx_printf(camera_fb_t *fb, int32_t x, int32_t y, uint32_t color, const char *format, ...)
 {
     char loc_buf[64];
     char *temp = loc_buf;

components/fb_gfx/include/fb_gfx.h

@@ -13,29 +13,30 @@
 // limitations under the License.
 #ifndef _FB_GFX_H_
 #define _FB_GFX_H_
+#include "esp_camera.h"
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-    typedef enum {
+    // typedef enum {
-        FB_RGB888, FB_BGR888, FB_RGB565, FB_BGR565
+    //     FB_RGB888, FB_BGR888, FB_RGB565, FB_BGR565
-    } fb_format_t;
+    // } fb_format_t;
 
-    typedef struct {
+    // typedef struct {
-            int width;
+    //         int width;
-            int height;
+    //         int height;
-            int bytes_per_pixel;
+    //         int bytes_per_pixel;
-            fb_format_t format;
+    //         fb_format_t format;
-            uint8_t * data;
+    //         uint8_t * data;
-    } fb_data_t;
+    // } fb_data_t;
 
-    void     fb_gfx_fillRect     (fb_data_t *fb, int32_t x, int32_t y, int32_t w, int32_t h, uint32_t color);
+    void     fb_gfx_fillRect     (camera_fb_t *fb, int32_t x, int32_t y, int32_t w, int32_t h, uint32_t color);
-    void     fb_gfx_drawFastHLine(fb_data_t *fb, int32_t x, int32_t y, int32_t w, uint32_t color);
+    void     fb_gfx_drawFastHLine(camera_fb_t *fb, int32_t x, int32_t y, int32_t w, uint32_t color);
-    void     fb_gfx_drawFastVLine(fb_data_t *fb, int32_t x, int32_t y, int32_t h, uint32_t color);
+    void     fb_gfx_drawFastVLine(camera_fb_t *fb, int32_t x, int32_t y, int32_t h, uint32_t color);
-    uint8_t  fb_gfx_putc         (fb_data_t *fb, int32_t x, int32_t y, uint32_t color, unsigned char c);
+    uint8_t  fb_gfx_putc         (camera_fb_t *fb, int32_t x, int32_t y, uint32_t color, unsigned char c);
-    uint32_t fb_gfx_print        (fb_data_t *fb, int32_t x, int32_t y, uint32_t color, const char * str);
+    uint32_t fb_gfx_print        (camera_fb_t *fb, int32_t x, int32_t y, uint32_t color, const char * str);
-    uint32_t fb_gfx_printf       (fb_data_t *fb, int32_t x, int32_t y, uint32_t color, const char *format, ...);
+    uint32_t fb_gfx_printf       (camera_fb_t *fb, int32_t x, int32_t y, uint32_t color, const char *format, ...);
 
 #ifdef __cplusplus
 }

components/modules/CMakeLists.txt

@@ -8,6 +8,7 @@ set(src_dirs
                 camera
                 lcd
                 led
+                button
                 web
                 trace)
 
@@ -16,6 +17,7 @@ set(include_dirs
                     camera
                     lcd
                     led
+                    button
                     web
                     trace)
 
@@ -26,6 +28,7 @@ set(requires    esp32-camera
                 esp_http_server
                 nvs_flash
                 mdns
+                esp_adc_cal
                 fb_gfx)
 
 idf_component_register(SRC_DIRS ${src_dirs} INCLUDE_DIRS ${include_dirs} REQUIRES ${requires} EMBED_FILES ${embed_files})

components/modules/Kconfig

@@ -215,4 +215,34 @@ menu "ESP-WHO Configuration"
                 Select Camera Y9 pin.
     endmenu
 
+
+    menu "Model Configuration"
+        menu "Face Recognition"
+            choice FACE_RECOGNITION_MODEL
+                bool "Face Recognition Model"
+                default MFN_V1
+                help
+                    Select Face Recognition Model.
+
+                config MFN_V1
+                    bool "mfn v1"
+    
+            endchoice
+            
+            choice QUANTIZATION
+                bool "Quantization"
+                default S8
+                help
+                    Select Face Recognition Model.
+
+                config S8
+                    bool "8-bit"
+                config S16
+                    bool "16-bit"
+    
+            endchoice
+        endmenu
+
+    endmenu
+
 endmenu

components/modules/ai/who_human_face_recognition.cpp

@@ -0,0 +1,239 @@
+#include "who_human_face_recognition.hpp"
+
+#include "esp_log.h"
+#include "esp_camera.h"
+
+#include "dl_image.hpp"
+#include "fb_gfx.h"
+
+#include "human_face_detect_msr01.hpp"
+#include "human_face_detect_mnp01.hpp"
+#include "face_recognition_tool.hpp"
+
+#if CONFIG_MFN_V1
+#if CONFIG_S8
+#include "face_recognition_112_v1_s8.hpp"
+#elif CONFIG_S16
+#include "face_recognition_112_v1_s16.hpp"
+#endif
+#endif
+
+#include "who_ai_utils.hpp"
+
+using namespace std;
+using namespace dl;
+
+static const char *TAG = "human_face_detection";
+
+static QueueHandle_t xQueueFrameI = NULL;
+static QueueHandle_t xQueueEvent = NULL;
+static QueueHandle_t xQueueFrameO = NULL;
+static QueueHandle_t xQueueResult = NULL;
+
+static recognizer_state_t gEvent = DETECT;
+static bool gReturnFB = true;
+static face_info_t recognize_result;
+
+SemaphoreHandle_t xMutex;
+
+typedef enum
+{
+    SHOW_STATE_IDLE,
+    SHOW_STATE_DELETE,
+    SHOW_STATE_RECOGNIZE,
+    SHOW_STATE_ENROLL,
+} show_state_t;
+
+#define RGB565_MASK_RED 0xF800
+#define RGB565_MASK_GREEN 0x07E0
+#define RGB565_MASK_BLUE 0x001F
+#define FRAME_DELAY_NUM 16
+
+static void rgb_print(camera_fb_t *fb, uint32_t color, const char *str)
+{
+    fb_gfx_print(fb, (fb->width - (strlen(str) * 14)) / 2, 10, color, str);
+}
+
+static int rgb_printf(camera_fb_t *fb, uint32_t color, const char *format, ...)
+{
+    char loc_buf[64];
+    char *temp = loc_buf;
+    int len;
+    va_list arg;
+    va_list copy;
+    va_start(arg, format);
+    va_copy(copy, arg);
+    len = vsnprintf(loc_buf, sizeof(loc_buf), format, arg);
+    va_end(copy);
+    if (len >= sizeof(loc_buf))
+    {
+        temp = (char *)malloc(len + 1);
+        if (temp == NULL)
+        {
+            return 0;
+        }
+    }
+    vsnprintf(temp, len + 1, format, arg);
+    va_end(arg);
+    rgb_print(fb, color, temp);
+    if (len > 64)
+    {
+        free(temp);
+    }
+    return len;
+}
+
+static void task_process_handler(void *arg)
+{
+    camera_fb_t *frame = NULL;
+    HumanFaceDetectMSR01 detector(0.3F, 0.3F, 10, 0.3F);
+    HumanFaceDetectMNP01 detector2(0.4F, 0.3F, 10);
+
+#if CONFIG_MFN_V1
+#if CONFIG_S8
+    FaceRecognition112V1S8 *recognizer = new FaceRecognition112V1S8();
+#elif CONFIG_S16
+    FaceRecognition112V1S16 *recognizer = new FaceRecognition112V1S16();
+#endif
+#endif
+    show_state_t frame_show_state = SHOW_STATE_IDLE;
+    recognizer_state_t _gEvent;
+
+    while (true)
+    {
+        xSemaphoreTake(xMutex, portMAX_DELAY);
+        _gEvent = gEvent;
+        gEvent = DETECT;
+        xSemaphoreGive(xMutex);
+
+        if (_gEvent)
+        {
+            bool is_detected = false;
+
+            if (xQueueReceive(xQueueFrameI, &frame, portMAX_DELAY))
+            {
+                std::list<dl::detect::result_t> &detect_candidates = detector.infer((uint16_t *)frame->buf, {(int)frame->height, (int)frame->width, 3});
+                std::list<dl::detect::result_t> &detect_results = detector2.infer((uint16_t *)frame->buf, {(int)frame->height, (int)frame->width, 3}, detect_candidates);
+
+                if (detect_results.size() == 1)
+                    is_detected = true;
+
+                if (is_detected)
+                {
+                    switch (_gEvent)
+                    {
+                    case ENROLL:
+                        recognizer->enroll_id((uint16_t *)frame->buf, {(int)frame->height, (int)frame->width, 3}, detect_results.front().keypoint);
+                        ESP_LOGW("ENROLL", "ID %d is enrolled", recognizer->get_enrolled_ids().back().id);
+                        frame_show_state = SHOW_STATE_ENROLL;
+                        break;
+
+                    case RECOGNIZE:
+                        recognize_result = recognizer->recognize((uint16_t *)frame->buf, {(int)frame->height, (int)frame->width, 3}, detect_results.front().keypoint);
+                        print_detection_result(detect_results);
+                        if (recognize_result.id > 0)
+                            ESP_LOGI("RECOGNIZE", "Similarity: %f, Match ID: %d", recognize_result.similarity, recognize_result.id);
+                        else
+                            ESP_LOGE("RECOGNIZE", "Similarity: %f, Match ID: %d", recognize_result.similarity, recognize_result.id);
+                        frame_show_state = SHOW_STATE_RECOGNIZE;
+                        break;
+
+                    case DELETE:
+                        recognizer->delete_id();
+                        ESP_LOGE("DELETE", "% d IDs left", recognizer->get_enrolled_id_num());
+                        frame_show_state = SHOW_STATE_DELETE;
+                        break;
+
+                    default:
+                        break;
+                    }
+                }
+
+                if (frame_show_state != SHOW_STATE_IDLE)
+                {
+                    static int frame_count = 0;
+                    switch (frame_show_state)
+                    {
+                    case SHOW_STATE_DELETE:
+                        rgb_printf(frame, RGB565_MASK_RED, "%d IDs left", recognizer->get_enrolled_id_num());
+                        break;
+
+                    case SHOW_STATE_RECOGNIZE:
+                        if (recognize_result.id > 0)
+                            rgb_printf(frame, RGB565_MASK_GREEN, "ID %d", recognize_result.id);
+                        else
+                            rgb_print(frame, RGB565_MASK_RED, "who ?");
+                        break;
+
+                    case SHOW_STATE_ENROLL:
+                        rgb_printf(frame, RGB565_MASK_BLUE, "Enroll: ID %d", recognizer->get_enrolled_ids().back().id);
+                        break;
+
+                    default:
+                        break;
+                    }
+
+                    if (++frame_count > FRAME_DELAY_NUM)
+                    {
+                        frame_count = 0;
+                        frame_show_state = SHOW_STATE_IDLE;
+                    }
+                }
+
+                if (detect_results.size())
+                {
+                    draw_detection_result((uint16_t *)frame->buf, frame->height, frame->width, detect_results);
+                }
+            }
+
+            if (xQueueFrameO)
+            {
+
+                xQueueSend(xQueueFrameO, &frame, portMAX_DELAY);
+            }
+            else if (gReturnFB)
+            {
+                esp_camera_fb_return(frame);
+            }
+            else
+            {
+                free(frame);
+            }
+
+            if (xQueueResult && is_detected)
+            {
+                xQueueSend(xQueueResult, &recognize_result, portMAX_DELAY);
+            }
+        }
+    }
+}
+
+static void task_event_handler(void *arg)
+{
+    recognizer_state_t _gEvent;
+    while (true)
+    {
+        xQueueReceive(xQueueEvent, &(_gEvent), portMAX_DELAY);
+        xSemaphoreTake(xMutex, portMAX_DELAY);
+        gEvent = _gEvent;
+        xSemaphoreGive(xMutex);
+    }
+}
+
+void register_human_face_recognition(const QueueHandle_t frame_i,
+                                     const QueueHandle_t event,
+                                     const QueueHandle_t result,
+                                     const QueueHandle_t frame_o,
+                                     const bool camera_fb_return)
+{
+    xQueueFrameI = frame_i;
+    xQueueFrameO = frame_o;
+    xQueueEvent = event;
+    xQueueResult = result;
+    gReturnFB = camera_fb_return;
+    xMutex = xSemaphoreCreateMutex();
+
+    xTaskCreatePinnedToCore(task_process_handler, TAG, 4 * 1024, NULL, 5, NULL, 0);
+    if (xQueueEvent)
+        xTaskCreatePinnedToCore(task_event_handler, TAG, 4 * 1024, NULL, 5, NULL, 1);
+}

components/modules/ai/who_human_face_recognition.hpp

@@ -0,0 +1,21 @@
+#pragma once
+
+#include "freertos/FreeRTOS.h"
+#include "freertos/queue.h"
+#include "freertos/task.h"
+#include "freertos/semphr.h"
+
+typedef enum
+{
+    IDLE = 0,
+    DETECT,
+    ENROLL,
+    RECOGNIZE,
+    DELETE,
+} recognizer_state_t;
+
+void register_human_face_recognition(QueueHandle_t frame_i,
+                                   QueueHandle_t event,
+                                   QueueHandle_t result,
+                                   QueueHandle_t frame_o = NULL,
+                                   const bool camera_fb_return = false);

components/modules/button/who_adc_button.c

@@ -0,0 +1,110 @@
+/* ADC1 Example
+
+   This example code is in the Public Domain (or CC0 licensed, at your option.)
+
+   Unless required by applicable law or agreed to in writing, this
+   software is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+   CONDITIONS OF ANY KIND, either express or implied.
+*/
+#include <stdio.h>
+#include <stdlib.h>
+#include "esp_log.h"
+#include "freertos/FreeRTOS.h"
+#include "freertos/task.h"
+#include "driver/adc_common.h"
+#include "esp_adc_cal.h"
+#include "who_adc_button.h"
+
+//ADC Channels
+#define ADC1_EXAMPLE_CHAN0 ADC1_CHANNEL_0
+//ADC Attenuation
+#define ADC_EXAMPLE_ATTEN ADC_ATTEN_DB_11
+//ADC Calibration
+#if CONFIG_IDF_TARGET_ESP32
+#define ADC_EXAMPLE_CALI_SCHEME ESP_ADC_CAL_VAL_EFUSE_VREF
+#elif CONFIG_IDF_TARGET_ESP32S2
+#define ADC_EXAMPLE_CALI_SCHEME ESP_ADC_CAL_VAL_EFUSE_TP
+#elif CONFIG_IDF_TARGET_ESP32C3
+#define ADC_EXAMPLE_CALI_SCHEME ESP_ADC_CAL_VAL_EFUSE_TP
+#elif CONFIG_IDF_TARGET_ESP32S3
+#define ADC_EXAMPLE_CALI_SCHEME ESP_ADC_CAL_VAL_EFUSE_TP_FIT
+#endif
+
+#define PRESS_INTERVAL 500000
+
+static uint32_t voltage = 0;
+
+static const char *TAG = "ADC SINGLE";
+
+static esp_adc_cal_characteristics_t adc1_chars;
+
+button_adc_config_t *adc_buttons;
+int adc_button_num;
+static QueueHandle_t xQueueKeyStateO = NULL;
+
+static bool adc_calibration_init(void)
+{
+    esp_err_t ret;
+    bool cali_enable = false;
+
+    ret = esp_adc_cal_check_efuse(ADC_EXAMPLE_CALI_SCHEME);
+    if (ret == ESP_ERR_NOT_SUPPORTED)
+    {
+        ESP_LOGW(TAG, "Calibration scheme not supported, skip software calibration");
+    }
+    else if (ret == ESP_ERR_INVALID_VERSION)
+    {
+        ESP_LOGW(TAG, "eFuse not burnt, skip software calibration");
+    }
+    else if (ret == ESP_OK)
+    {
+        cali_enable = true;
+        esp_adc_cal_characterize(ADC_UNIT_1, ADC_EXAMPLE_ATTEN, ADC_WIDTH_BIT_DEFAULT, 0, &adc1_chars);
+    }
+    else
+    {
+        ESP_LOGE(TAG, "Invalid arg");
+    }
+    return cali_enable;
+}
+
+void adc_button_task(void *arg)
+{
+    int last_button_pressed = -1;
+    int button_pressed = -1;
+    int64_t backup_time = esp_timer_get_time();
+    int64_t last_time = esp_timer_get_time();
+
+    //ADC1 config
+    ESP_ERROR_CHECK(adc1_config_width(ADC_WIDTH_BIT_DEFAULT));
+    ESP_ERROR_CHECK(adc1_config_channel_atten(ADC1_EXAMPLE_CHAN0, ADC_EXAMPLE_ATTEN));
+
+    while (1)
+    {
+        voltage = adc1_get_raw(ADC1_EXAMPLE_CHAN0);
+        backup_time = esp_timer_get_time();
+        for (int i = 0; i < adc_button_num; ++i)
+        {
+            if ((voltage >= adc_buttons[i].min) && (voltage <= adc_buttons[i].max))
+            {
+                button_pressed = adc_buttons[i].button_index;
+                if ((button_pressed != last_button_pressed) || ((backup_time - last_time) > PRESS_INTERVAL))
+                {
+                    last_button_pressed = button_pressed;
+                    last_time = backup_time;
+                    xQueueOverwrite(xQueueKeyStateO, &button_pressed);
+                    break;
+                }
+            }
+        }
+        vTaskDelay(pdMS_TO_TICKS(10));
+    }
+}
+
+void register_adc_button(button_adc_config_t *buttons_ptr, int button_num, const QueueHandle_t key_state_o)
+{
+    xQueueKeyStateO = key_state_o;
+    adc_buttons = buttons_ptr;
+    adc_button_num = button_num;
+    xTaskCreatePinnedToCore(adc_button_task, "adc_button_scan_task", 1024, NULL, 5, NULL, 0);
+}

components/modules/button/who_adc_button.h

@@ -0,0 +1,30 @@
+#pragma once
+#include "esp_event_loop.h"
+#include "soc/system_reg.h"
+#include "driver/gpio.h"
+#include "esp_log.h"
+
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+
+    typedef struct
+    {
+        int button_index;       /**< button index on the channel */
+        int min;               /**< min voltage in mv corresponding to the button */
+        int max;               /**< max voltage in mv corresponding to the button */
+    } button_adc_config_t;
+
+    /**
+     * @brief initialize adc button
+     * 
+     * @param buttons_ptr the pointer of adc button configuration
+     * @param button_num the numbers of adc buttons
+     * @param key_state_o the queue to send which button is pressed
+     */
+    void register_adc_button(button_adc_config_t *buttons_ptr, int button_num, const QueueHandle_t key_state_o);
+
+#ifdef __cplusplus
+}
+#endif

components/modules/button/who_button.c

@@ -0,0 +1,100 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include "who_button.h"
+
+typedef struct
+{
+    gpio_num_t io_num;
+    key_state_t state;
+} key_scan_state_t;
+
+#define LONG_PRESS_THRESH 700000
+#define DOUBLE_CLICK_THRESH 300000
+
+static xQueueHandle gpio_evt_queue = NULL;
+static QueueHandle_t xQueueKeyStateO = NULL;
+
+static void IRAM_ATTR gpio_isr_handler_key(void *arg)
+{
+    uint32_t gpio_num = (uint32_t)arg;
+    xQueueSendFromISR(gpio_evt_queue, &gpio_num, NULL);
+}
+
+int key_scan(TickType_t ticks_to_wait)
+{
+    gpio_num_t io_num;
+    BaseType_t press_key = pdFALSE;
+    BaseType_t lift_key = pdFALSE;
+    int64_t backup_time = 0;
+    int64_t interval_time = 0;
+    static int64_t last_time = 0;
+
+    for (;;)
+    {
+        xQueueReceive(gpio_evt_queue, &io_num, ticks_to_wait);
+
+        if (gpio_get_level(io_num) == 0)
+        {
+            press_key = pdTRUE;
+            backup_time = esp_timer_get_time();
+            interval_time = backup_time - last_time;
+        }
+        else if (press_key)
+        {
+            lift_key = pdTRUE;
+            last_time = esp_timer_get_time();
+            backup_time = last_time - backup_time;
+        }
+
+        if (press_key & lift_key)
+        {
+            press_key = pdFALSE;
+            lift_key = pdFALSE;
+
+            if (backup_time > LONG_PRESS_THRESH)
+            {
+                return KEY_LONG_PRESS;
+            }
+            else
+            {
+                if ((interval_time < DOUBLE_CLICK_THRESH) && (interval_time > 0))
+                    return KEY_DOUBLE_CLICK;
+                else
+                    return KEY_SHORT_PRESS;
+            }
+        }
+    }
+}
+
+void key_trigger(void *arg)
+{
+    int ret = 0;
+
+    while (1)
+    {
+        ret = key_scan(portMAX_DELAY);
+        xQueueOverwrite(xQueueKeyStateO, &ret);
+    }
+
+    vTaskDelete(NULL);
+}
+
+void key_init(gpio_num_t gpio_num)
+{
+    gpio_config_t io_conf = {0};
+    io_conf.mode = GPIO_MODE_INPUT;
+    io_conf.intr_type = GPIO_INTR_ANYEDGE;
+    io_conf.pin_bit_mask = 1LL << gpio_num;
+    io_conf.pull_up_en = GPIO_PULLUP_ENABLE;
+    gpio_config(&io_conf);
+    gpio_evt_queue = xQueueCreate(5, sizeof(uint32_t));
+    gpio_install_isr_service(0);
+    gpio_isr_handler_add(gpio_num, gpio_isr_handler_key, (void *)gpio_num);
+}
+
+void register_button(const gpio_num_t key_io_num, const QueueHandle_t key_state_o)
+{
+    xQueueKeyStateO = key_state_o;
+    key_init(key_io_num);
+    xTaskCreatePinnedToCore(key_trigger, "key_scan_task", 1024, NULL, 5, NULL, 0);
+}

components/modules/button/who_button.h

@@ -0,0 +1,29 @@
+#pragma once
+#include "esp_event_loop.h"
+#include "soc/system_reg.h"
+#include "driver/gpio.h"
+#include "esp_log.h"
+
+typedef enum
+{
+    KEY_SHORT_PRESS = 1,
+    KEY_LONG_PRESS,
+    KEY_DOUBLE_CLICK,
+} key_state_t;
+
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+
+/**
+ * @brief initialize gpio button
+ * 
+ * @param key_io_num the gpio number of the button
+ * @param key_state_o the queue to send the button state
+ */
+void register_button(const gpio_num_t key_io_num, const QueueHandle_t key_state_o);
+
+#ifdef __cplusplus
+}
+#endif

components/modules/camera/who_camera.h

@@ -192,7 +192,7 @@
 #define CAMERA_PIN_PCLK CONFIG_CAMERA_PIN_PCLK
 #endif
 
-#define XCLK_FREQ_HZ 20000000
+#define XCLK_FREQ_HZ 16000000
 
 #ifdef __cplusplus
 extern "C"

examples/human_face_recognition/README.md

@@ -0,0 +1,18 @@
+# Human Face Recognition Example [[中文]](./README_ZH.md)
+
+The following is the special configuration of face recognition examples. For general configuration, please refer to [this](../../README.md)
+
+
+## Configure the face recognition model
+
+Enter `idf.py menuconfig` in the terminal and click (Top) -> Component config -> ESP-WHO Configuration -> Model Configuration -> Face Recognition to enter the face recognition model configuration interface, as shown below:
+![](../../img/face_recognition_model_config.png)
+You can configure the version and quantification type of the model here.
+
+## How to Use Example
+
+- The interactive button is the Boot button.
+- Short press the button: recognize the face captured by the camera at this time.
+- Long press the button: enroll the face captured by the camera at this time.
+- Double click the button: delete the last enrolled face.
+

examples/human_face_recognition/README_CN.md

@@ -0,0 +1,18 @@
+# Human Face Recognition Example [[English]](./README.md)
+
+以下为使用人脸识别示例的特殊配置，若要进行通用配置可参考此[说明](../../README_ZH.md)。
+
+## 人脸识别模型配置
+
+在终端输入 `idf.py menuconfig` ，依次 (Top) -> Component config -> ESP-WHO Configuration -> Model Configuration -> Face Recognition 可进入人脸识别模型配置界面，如下图所示：
+
+![](../../img/face_recognition_model_config.png)
+
+您可以在这里配置模型的版本和量化方式。
+
+## 如何使用示例
+
+- 交互按键为Boot键。
+- 短按按键：识别此时摄像头拍到的人脸。
+- 长按按键：录入此时摄像头拍到的人脸。
+- 双击按键：删除最后一个被录入的人脸。

examples/human_face_recognition/lcd/CMakeLists.txt

@@ -0,0 +1,8 @@
+# The following lines of boilerplate have to be in your project's
+# CMakeLists in this exact order for cmake to work correctly
+cmake_minimum_required(VERSION 3.5)
+
+set(EXTRA_COMPONENT_DIRS ../../../components)
+add_compile_options(-fdiagnostics-color=always)
+include($ENV{IDF_PATH}/tools/cmake/project.cmake)
+project(human_face_recognition_lcd)

examples/human_face_recognition/lcd/main/CMakeLists.txt

@@ -0,0 +1,5 @@
+set(src_dirs        .)
+
+set(include_dirs    .)
+
+idf_component_register(SRC_DIRS ${src_dirs} INCLUDE_DIRS ${include_dirs})

examples/human_face_recognition/lcd/main/app_main.cpp

@@ -0,0 +1,30 @@
+#include "who_camera.h"
+#include "who_human_face_recognition.hpp"
+#include "who_lcd.h"
+#include "who_button.h"
+#include "event_logic.hpp"
+#include "who_adc_button.h"
+
+static QueueHandle_t xQueueAIFrame = NULL;
+static QueueHandle_t xQueueLCDFrame = NULL;
+static QueueHandle_t xQueueKeyState = NULL;
+static QueueHandle_t xQueueEventLogic = NULL;
+static button_adc_config_t buttons[4] = {{1, 2800, 3000}, {2, 2250, 2450}, {3, 300, 500}, {4, 850, 1050}};
+
+#define GPIO_BOOT GPIO_NUM_0
+
+extern "C" void app_main()
+{
+    xQueueAIFrame = xQueueCreate(2, sizeof(camera_fb_t *));
+    xQueueLCDFrame = xQueueCreate(2, sizeof(camera_fb_t *));
+    xQueueKeyState = xQueueCreate(1, sizeof(int *));
+    xQueueEventLogic = xQueueCreate(1, sizeof(int *));
+
+    register_camera(PIXFORMAT_RGB565, FRAMESIZE_240X240, 2, xQueueAIFrame);
+    register_button(GPIO_BOOT, xQueueKeyState);
+    // register_adc_button(buttons, 4, xQueueKeyState);
+    register_event(xQueueKeyState, xQueueEventLogic);
+    register_human_face_recognition(xQueueAIFrame, xQueueEventLogic, NULL, xQueueLCDFrame, false);
+    register_lcd(xQueueLCDFrame, NULL, true);
+
+}

examples/human_face_recognition/lcd/main/event_logic.cpp

@@ -0,0 +1,84 @@
+#include <stdio.h>
+#include "event_logic.hpp"
+#include "who_button.h"
+#include "who_human_face_recognition.hpp"
+
+typedef enum
+{
+    MENU = 1,
+    PLAY,
+    UP,
+    DOWN
+}key_name_t;
+
+static QueueHandle_t xQueueKeyStateI = NULL;
+static QueueHandle_t xQueueEventO = NULL;
+static key_state_t key_state;
+static key_name_t adc_button_name;
+static recognizer_state_t recognizer_state;
+
+void event_generate(void *arg)
+{
+    while (1)
+    {
+        xQueueReceive(xQueueKeyStateI, &key_state, portMAX_DELAY);
+        switch (key_state)
+        {
+        case KEY_SHORT_PRESS:
+            recognizer_state = RECOGNIZE;
+            break;
+
+        case KEY_LONG_PRESS:
+            recognizer_state = ENROLL;
+            break;
+
+        case KEY_DOUBLE_CLICK:
+            recognizer_state = DELETE;
+            break;
+
+        default:
+            recognizer_state = DETECT;
+            break;
+        }
+        xQueueSend(xQueueEventO, &recognizer_state, portMAX_DELAY);
+    }
+}
+
+
+void event_generate_from_adc_button(void *arg)
+{
+    while (1)
+    {
+        xQueueReceive(xQueueKeyStateI, &adc_button_name, portMAX_DELAY);
+        switch (adc_button_name)
+        {
+        case MENU:
+            recognizer_state = ENROLL;
+            break;
+
+        case PLAY:
+            recognizer_state = DELETE;
+            break;
+
+        case UP:
+            recognizer_state = RECOGNIZE;
+            break;
+
+        case DOWN:
+            recognizer_state = RECOGNIZE;
+            break;
+
+        default:
+            recognizer_state = DETECT;
+            break;
+        }
+        xQueueSend(xQueueEventO, &recognizer_state, portMAX_DELAY);
+    }
+}
+
+void register_event(const QueueHandle_t key_state_i, const QueueHandle_t event_o)
+{
+    xQueueKeyStateI = key_state_i;
+    xQueueEventO = event_o;
+    xTaskCreatePinnedToCore(event_generate, "event_logic_task", 1024, NULL, 5, NULL, 0);
+}

examples/human_face_recognition/lcd/main/event_logic.hpp

@@ -0,0 +1,13 @@
+#pragma once
+
+#include "freertos/FreeRTOS.h"
+#include "freertos/queue.h"
+#include "freertos/task.h"
+
+/**
+ * @brief 
+ * 
+ * @param key_state_i 
+ * @param event_o 
+ */
+void register_event(const QueueHandle_t key_state_i, const QueueHandle_t event_o);

examples/human_face_recognition/lcd/partitions.csv

@@ -0,0 +1,5 @@
+# Name,   Type, SubType, Offset,  Size, Flags
+# Note: if you change the phy_init or app partition offset, make sure to change the offset in Kconfig.projbuild
+factory, app,  factory, 0x010000, 3840K
+nvs,     data, nvs,     0x3D0000, 16K
+fr,      32,   32,      0x3E0000, 128K

examples/human_face_recognition/lcd/sdkconfig.defaults

@@ -0,0 +1,12 @@
+CONFIG_ESPTOOLPY_FLASHSIZE_4MB=y
+
+CONFIG_ESPTOOLPY_FLASHFREQ_80M=y
+CONFIG_ESPTOOLPY_FLASHMODE_QIO=y
+
+CONFIG_SPIRAM_SPEED_80M=y
+CONFIG_SPIRAM_USE_CAPS_ALLOC=y
+
+CONFIG_PARTITION_TABLE_CUSTOM=y
+CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partitions.csv"
+
+CONFIG_S16=y

examples/human_face_recognition/lcd/sdkconfig.defaults.esp32s3

@@ -0,0 +1,15 @@
+CONFIG_ESP32S3_DEFAULT_CPU_FREQ_240=y
+CONFIG_ESP32S3_SPIRAM_SUPPORT=y
+
+CONFIG_ESP32S3_DATA_CACHE_64KB=y
+CONFIG_ESP32S3_DATA_CACHE_8WAYS=y
+CONFIG_ESP32S3_DATA_CACHE_LINE_64B=y
+
+CONFIG_CAMERA_MODULE_ESP_S3_EYE=y
+CONFIG_LCD_DRIVER_SCREEN_CONTROLLER_ST7789=y
+CONFIG_ESPTOOLPY_NO_STUB=y
+CONFIG_ESP_CONSOLE_USB_SERIAL_JTAG=y
+
+CONFIG_SPIRAM_MODE_OCT=y
+
+CONFIG_S8=y

examples/human_face_recognition/terminal/CMakeLists.txt

@@ -0,0 +1,8 @@
+# The following lines of boilerplate have to be in your project's
+# CMakeLists in this exact order for cmake to work correctly
+cmake_minimum_required(VERSION 3.5)
+
+set(EXTRA_COMPONENT_DIRS ../../../components)
+add_compile_options(-fdiagnostics-color=always)
+include($ENV{IDF_PATH}/tools/cmake/project.cmake)
+project(human_face_recognition_terminal)

examples/human_face_recognition/terminal/main/CMakeLists.txt

@@ -0,0 +1,5 @@
+set(src_dirs        .)
+
+set(include_dirs    .)
+
+idf_component_register(SRC_DIRS ${src_dirs} INCLUDE_DIRS ${include_dirs})

examples/human_face_recognition/terminal/main/app_main.cpp

@@ -0,0 +1,25 @@
+#include "who_camera.h"
+#include "who_human_face_recognition.hpp"
+#include "who_button.h"
+#include "event_logic.hpp"
+#include "who_adc_button.h"
+
+static QueueHandle_t xQueueAIFrame = NULL;
+static QueueHandle_t xQueueKeyState = NULL;
+static QueueHandle_t xQueueEventLogic = NULL;
+static button_adc_config_t buttons[4] = {{1, 2800, 3000}, {2, 2250, 2450}, {3, 300, 500}, {4, 850, 1050}};
+
+#define GPIO_BOOT GPIO_NUM_0
+
+extern "C" void app_main()
+{
+    xQueueAIFrame = xQueueCreate(2, sizeof(camera_fb_t *));
+    xQueueKeyState = xQueueCreate(1, sizeof(int *));
+    xQueueEventLogic = xQueueCreate(1, sizeof(int *));
+
+    register_camera(PIXFORMAT_RGB565, FRAMESIZE_240X240, 2, xQueueAIFrame);
+    register_button(GPIO_BOOT, xQueueKeyState);
+    register_event(xQueueKeyState, xQueueEventLogic);
+    register_human_face_recognition(xQueueAIFrame, xQueueEventLogic, NULL, NULL, true);
+
+}

examples/human_face_recognition/terminal/main/event_logic.cpp

@@ -0,0 +1,84 @@
+#include <stdio.h>
+#include "event_logic.hpp"
+#include "who_button.h"
+#include "who_human_face_recognition.hpp"
+
+typedef enum
+{
+    MENU = 1,
+    PLAY,
+    UP,
+    DOWN
+}key_name_t;
+
+static QueueHandle_t xQueueKeyStateI = NULL;
+static QueueHandle_t xQueueEventO = NULL;
+static key_state_t key_state;
+static key_name_t adc_button_name;
+static recognizer_state_t recognizer_state;
+
+void event_generate(void *arg)
+{
+    while (1)
+    {
+        xQueueReceive(xQueueKeyStateI, &key_state, portMAX_DELAY);
+        switch (key_state)
+        {
+        case KEY_SHORT_PRESS:
+            recognizer_state = RECOGNIZE;
+            break;
+
+        case KEY_LONG_PRESS:
+            recognizer_state = ENROLL;
+            break;
+
+        case KEY_DOUBLE_CLICK:
+            recognizer_state = DELETE;
+            break;
+
+        default:
+            recognizer_state = DETECT;
+            break;
+        }
+        xQueueSend(xQueueEventO, &recognizer_state, portMAX_DELAY);
+    }
+}
+
+
+void event_generate_from_adc_button(void *arg)
+{
+    while (1)
+    {
+        xQueueReceive(xQueueKeyStateI, &adc_button_name, portMAX_DELAY);
+        switch (adc_button_name)
+        {
+        case MENU:
+            recognizer_state = ENROLL;
+            break;
+
+        case PLAY:
+            recognizer_state = DELETE;
+            break;
+
+        case UP:
+            recognizer_state = RECOGNIZE;
+            break;
+
+        case DOWN:
+            recognizer_state = RECOGNIZE;
+            break;
+
+        default:
+            recognizer_state = DETECT;
+            break;
+        }
+        xQueueSend(xQueueEventO, &recognizer_state, portMAX_DELAY);
+    }
+}
+
+void register_event(const QueueHandle_t key_state_i, const QueueHandle_t event_o)
+{
+    xQueueKeyStateI = key_state_i;
+    xQueueEventO = event_o;
+    xTaskCreatePinnedToCore(event_generate, "event_logic_task", 1024, NULL, 5, NULL, 0);
+}

examples/human_face_recognition/terminal/main/event_logic.hpp

@@ -0,0 +1,13 @@
+#pragma once
+
+#include "freertos/FreeRTOS.h"
+#include "freertos/queue.h"
+#include "freertos/task.h"
+
+/**
+ * @brief 
+ * 
+ * @param key_state_i 
+ * @param event_o 
+ */
+void register_event(const QueueHandle_t key_state_i, const QueueHandle_t event_o);

examples/human_face_recognition/terminal/partitions.csv

@@ -0,0 +1,5 @@
+# Name,   Type, SubType, Offset,  Size, Flags
+# Note: if you change the phy_init or app partition offset, make sure to change the offset in Kconfig.projbuild
+factory, app,  factory, 0x010000, 3840K
+nvs,     data, nvs,     0x3D0000, 16K
+fr,      32,   32,      0x3E0000, 128K

examples/human_face_recognition/terminal/sdkconfig.defaults

@@ -0,0 +1,12 @@
+CONFIG_ESPTOOLPY_FLASHSIZE_4MB=y
+
+CONFIG_ESPTOOLPY_FLASHFREQ_80M=y
+CONFIG_ESPTOOLPY_FLASHMODE_QIO=y
+
+CONFIG_SPIRAM_SPEED_80M=y
+CONFIG_SPIRAM_USE_CAPS_ALLOC=y
+
+CONFIG_PARTITION_TABLE_CUSTOM=y
+CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partitions.csv"
+
+CONFIG_S16=y

examples/human_face_recognition/terminal/sdkconfig.defaults.esp32

@@ -0,0 +1,4 @@
+CONFIG_ESP32_DEFAULT_CPU_FREQ_240=y
+CONFIG_ESP32_SPIRAM_SUPPORT=y
+
+CONFIG_CAMERA_MODULE_ESP_EYE=y

examples/human_face_recognition/terminal/sdkconfig.defaults.esp32s2

@@ -0,0 +1,7 @@
+CONFIG_ESP32S2_DEFAULT_CPU_FREQ_240=y
+CONFIG_ESP32S2_SPIRAM_SUPPORT=y
+
+CONFIG_ESP32S2_DATA_CACHE_16KB=y
+ESP32S2_DATA_CACHE_LINE_32B=y
+
+CONFIG_CAMERA_MODULE_ESP_S2_KALUGA=y

examples/human_face_recognition/terminal/sdkconfig.defaults.esp32s3

@@ -0,0 +1,15 @@
+CONFIG_ESP32S3_DEFAULT_CPU_FREQ_240=y
+CONFIG_ESP32S3_SPIRAM_SUPPORT=y
+
+CONFIG_ESP32S3_DATA_CACHE_64KB=y
+CONFIG_ESP32S3_DATA_CACHE_8WAYS=y
+CONFIG_ESP32S3_DATA_CACHE_LINE_64B=y
+
+CONFIG_CAMERA_MODULE_ESP_S3_EYE=y
+CONFIG_LCD_DRIVER_SCREEN_CONTROLLER_ST7789=y
+CONFIG_ESPTOOLPY_NO_STUB=y
+CONFIG_ESP_CONSOLE_USB_SERIAL_JTAG=y
+
+CONFIG_SPIRAM_MODE_OCT=y
+
+CONFIG_S8=y