zhengxinandCursor
851e347cf5
纳入固件源码、屏幕组件、nand_xfer 工具和文档。忽略本机构建产物、sdkconfig，以及含本机路径的依赖锁。 Co-authored-by: Cursor <[email protected]>
474 lines
18 KiB
C
474 lines
18 KiB
C
#include "speech_app.h"
|
||
#include "speech_init.h"
|
||
#include "LVGL/lvgl_init.h"
|
||
#include <math.h>
|
||
static const char *TAG_LVGL = "sr";
|
||
// 音频设备句柄
|
||
esp_codec_dev_handle_t record_dev = NULL;
|
||
const char *cmd_phoneme[] =
|
||
{
|
||
"ni hao xiao qi",
|
||
"xiao qi zai jian", // 小奇，再见
|
||
"xiao qi tan ge zi ta", // 小奇，弹个吉他
|
||
"xiao qi bi ge xin", // 小奇，比个心
|
||
"xiao qi si kao yi xia zhe ge wen ti", // 小奇，思考一下这个问题
|
||
};
|
||
|
||
const uint32_t cmd_phoneme_number = sizeof(cmd_phoneme)/sizeof(cmd_phoneme[0]);
|
||
|
||
typedef struct __attribute__((packed)) {
|
||
char chunkID[4];
|
||
uint32_t chunkSize;
|
||
char format[4];
|
||
char subchunk1ID[4];
|
||
uint32_t subchunk1Size;
|
||
uint16_t audioFormat;
|
||
uint16_t numChannels;
|
||
uint32_t sampleRate;
|
||
uint32_t byteRate;
|
||
uint16_t blockAlign;
|
||
uint16_t bitsPerSample;
|
||
char subchunk2ID[4];
|
||
uint32_t subchunk2Size;
|
||
} wav_header_t;
|
||
void write_wav_header(FILE* fp, uint32_t total_data_bytes) {
|
||
wav_header_t wav = {
|
||
.chunkID = {'R','I','F','F'},
|
||
.format = {'W','A','V','E'},
|
||
.subchunk1ID = {'f','m','t',' '},
|
||
.subchunk1Size = 16,
|
||
.audioFormat = 1,
|
||
.numChannels = 2, // 你的参数是resdata
|
||
.sampleRate = 16000, // 你的参数是16kHz
|
||
.bitsPerSample = 16, // 你的参数是16-bit
|
||
.subchunk2ID = {'d','a','t','a'},
|
||
.subchunk2Size = total_data_bytes,
|
||
};
|
||
// 计算其他依赖的字段
|
||
wav.byteRate = wav.sampleRate * wav.numChannels * wav.bitsPerSample / 8;
|
||
wav.blockAlign = wav.numChannels * wav.bitsPerSample / 8;
|
||
wav.chunkSize = 36 + wav.subchunk2Size; // 文件总大小 - 8
|
||
|
||
// 写入文件头
|
||
fwrite(&wav, sizeof(wav), 1, fp);
|
||
}
|
||
int slience_count = 0;
|
||
#define GAIN 1.5f // 例如增益 1.5（约 +3.5 dB）
|
||
#define MAX_VAL 32767
|
||
#define MIN_VAL -32768
|
||
/**
|
||
* @brief 音频数据采集任务
|
||
* 从I2S接口读取音频数据并喂给音频前端处理
|
||
*
|
||
* @param arg 音频前端数据句柄
|
||
*/
|
||
void feed_Task(void *arg)
|
||
{
|
||
ESP_LOGI(TAG_LVGL, "Starting feed task");
|
||
|
||
esp_afe_sr_data_t *afe_data_ = arg;
|
||
|
||
// 获取音频前端处理的块大小和通道数
|
||
/*
|
||
int feed_chunksize = afe_handle->get_feed_chunksize(afe_data_);
|
||
int feed_nch = afe_handle->get_feed_channel_num(afe_data_);
|
||
ESP_LOGE(TAG_LVGL, "feed_chunksize: %d, feed_nch: %d", feed_chunksize, feed_nch);
|
||
*/
|
||
|
||
/*
|
||
FILE* fp = fopen("/extflash/audio1.wav", "wb"); // 或 "/extflash/audio.wav"
|
||
if (!fp) {
|
||
ESP_LOGW(TAG_MAIN, "WAV文件打开失败，无法写入");
|
||
}
|
||
uint32_t sampleRate = 16000;
|
||
uint16_t numChannels = 2;
|
||
uint16_t bitsPerSample = 16;
|
||
uint32_t totalSamples = sampleRate * 2; // 2秒总采样数（每声道）
|
||
uint32_t totalDataBytes = totalSamples * numChannels * (bitsPerSample / 8);
|
||
write_wav_header(fp, totalDataBytes);
|
||
int counter = 0;
|
||
*/
|
||
while (1)
|
||
{
|
||
size_t bytesIn = 0;
|
||
// 从I2S接口读取音频数据
|
||
//int ret=esp_codec_dev_read(record_dev, feed_buff, feed_chunksize * feed_nch * sizeof(int16_t)); // 读取音频数据（录音）
|
||
|
||
|
||
// 将左右声道数据求和取半，转为单声道数据
|
||
//for (int i = 0; i < feed_chunksize; i++) {
|
||
// int left = feed_buff[2*i];
|
||
// int right = feed_buff[2*i + 1];
|
||
// 求和并取平均，防止数值溢出
|
||
// mono_buff[i] = (left + right) >> 1;
|
||
//}
|
||
//ESP_LOGW(TAG_LVGL, "codec read error: %d", ret);
|
||
/*
|
||
if(counter < 33) // 采集100次后关闭文件（约10秒音频）
|
||
{
|
||
fwrite(feed_buff, sizeof(int16_t), feed_chunksize * feed_nch, fp);
|
||
counter++;
|
||
}
|
||
if(counter == 33) // 采集100次后关闭文件（约10秒音频）
|
||
{
|
||
fclose(fp);
|
||
counter++;
|
||
FILE* fp = fopen("/extflash/audio2.wav", "wb");
|
||
write_wav_header(fp, totalDataBytes);
|
||
ESP_LOGW(TAG_MAIN,"WAV 文件已保存：/extflash/audio1.wav\n");
|
||
ESP_LOGW(TAG_MAIN,"开始写入第二个文件：/extflash/audio2.wav\n");
|
||
}
|
||
if(counter > 33 && counter < 66)
|
||
{
|
||
fwrite(feed_buff, sizeof(int16_t), feed_chunksize * feed_nch, fp);
|
||
counter++;
|
||
}
|
||
if(counter == 66) // 采集100次后关闭文件（约10秒音频）
|
||
{
|
||
fclose(fp);
|
||
ESP_LOGW(TAG_MAIN,"第二个 WAV 文件已保存：/extflash/audio2.wav\n");
|
||
counter++;
|
||
FILE* fp = fopen("/extflash/audio3.wav", "wb");
|
||
write_wav_header(fp, totalDataBytes);
|
||
ESP_LOGW(TAG_MAIN,"开始写入第三个文件：/extflash/audio3.wav\n");
|
||
}
|
||
if(counter > 66 && counter < 99)
|
||
{
|
||
fwrite(feed_buff, sizeof(int16_t), feed_chunksize * feed_nch, fp);
|
||
counter++;
|
||
}
|
||
if(counter == 99) // 采集100次后关闭文件（约10秒音频）
|
||
{
|
||
fclose(fp);
|
||
ESP_LOGW(TAG_MAIN,"第三个 WAV 文件已保存：/extflash/audio3.wav\n");
|
||
counter++;
|
||
break; // 结束循环，停止采集
|
||
}
|
||
*/
|
||
// 将音频数据喂给音频前端处理
|
||
//ESP_LOGI("I2S RAW", "L:%d, R:%d, L:%d, R:%d,L:%d, R:%d,L:%d, R:%d", feed_buff[0], feed_buff[1], feed_buff[2], feed_buff[3], feed_buff[4], feed_buff[5], feed_buff[6], feed_buff[7]);
|
||
//int num_samples = feed_chunksize * feed_nch;
|
||
/*
|
||
long long sum_squares = 0;
|
||
for (int i = 0; i < num_samples; i++) {
|
||
sum_squares += (long long)feed_buff[i] * feed_buff[i];
|
||
}
|
||
double rms = sqrt((double)sum_squares / num_samples);
|
||
#define REFERENCE_RMS 750
|
||
double db = 20.0 * log10(rms / REFERENCE_RMS);
|
||
//ESP_LOGI("AUDIO", "RMS: %.2f, dB: %.2f", rms, db);
|
||
*/
|
||
//afe_handle->feed(afe_data_, feed_buff);
|
||
vTaskDelay(pdMS_TO_TICKS(1));
|
||
//ESP_LOGW(TAG_LVGL,"feed_task");
|
||
}
|
||
// 任务退出时释放资源
|
||
|
||
vTaskDelete(NULL);
|
||
}
|
||
|
||
extern bool is_ai_recording; // AI语音交互状态标志（外部声明）
|
||
extern bool is_ai_create_pcmfile; // AI语音交互生成pcm文件状态标志（外部声明）
|
||
extern bool is_ai_end_recording; // AI语音交互结束录音状态标志（外部声明）
|
||
void get_file_size(const char *path) {
|
||
struct stat st;
|
||
if (stat(path, &st) != 0) {
|
||
ESP_LOGW(TAG_MAIN,"stat failed for %s\n", path);
|
||
return;
|
||
}
|
||
float size_kb = st.st_size / 1024.0;
|
||
ESP_LOGI(TAG_MAIN, "File %s size: %.2f KB", path, size_kb);
|
||
}
|
||
/**
|
||
* @brief 语音检测任务
|
||
* 从音频前端获取处理后的数据并进行语音命令识别
|
||
*
|
||
* @param arg 音频前端数据句柄
|
||
*/
|
||
extern int64_t init_start_us;
|
||
int silence_count = 0;
|
||
void detect_Task(void *arg)
|
||
{
|
||
ESP_LOGI(TAG_LVGL, "Starting detect task");
|
||
int feed_chunksize=512;
|
||
int feed_nch=2;
|
||
// 分配音频数据缓冲区
|
||
int16_t *feed_buff = (int16_t *)heap_caps_malloc(feed_chunksize * feed_nch * sizeof(int16_t), MALLOC_CAP_SPIRAM);
|
||
assert(feed_buff); // 确保内存分配成功
|
||
int16_t *mono_buff = (int16_t *)malloc(feed_chunksize * sizeof(int16_t));
|
||
if (mono_buff == NULL) {
|
||
ESP_LOGE(TAG_LVGL, "Failed to allocate mono buffer");
|
||
return;
|
||
}
|
||
record_dev = ES7243E_Init();
|
||
//esp_afe_sr_data_t *afe_data_ = arg;
|
||
// 获取获取数据的块大小并分配缓冲区
|
||
//int afe_chunksize = afe_handle->get_fetch_chunksize(afe_data_);
|
||
//ESP_LOGW(TAG_LVGL, "afe_chunksize: %d", afe_chunksize);
|
||
/*
|
||
FILE* fp = fopen("/extflash/audio1res.wav", "wb");
|
||
if (!fp) {
|
||
ESP_LOGW(TAG_MAIN, "WAV文件打开失败，无法写入");
|
||
}
|
||
uint32_t sampleRate = 16000;
|
||
uint16_t numChannels = 1;
|
||
uint16_t bitsPerSample = 16;
|
||
uint32_t totalSamples = sampleRate * 3.2; // 3.2秒总采样数（每声道）
|
||
uint32_t totalDataBytes = totalSamples * numChannels * (bitsPerSample / 8);
|
||
write_wav_header(fp, totalDataBytes);
|
||
int counter = 0;
|
||
*/
|
||
ESP_LOGW(TAG_LVGL, "开始识别");
|
||
const int64_t init_elapsed_us = esp_timer_get_time() - init_start_us;
|
||
ESP_LOGI(TAG_MAIN, "语音识别初始化耗时: %" PRId64 " ms (%" PRId64 " us)",
|
||
init_elapsed_us / 1000, init_elapsed_us);
|
||
|
||
while (1)
|
||
{
|
||
|
||
//对话开启，启动PCM文件创建
|
||
if(is_ai_create_pcmfile == true){
|
||
FILE* fp = fopen("/extflash/audio_ai.pcm", "wb");
|
||
if (!fp) {
|
||
ESP_LOGW(TAG_MAIN, "PCM文件打开失败，无法写入，语音对话阶段一失败");
|
||
}
|
||
uint32_t sampleRate = 16000;
|
||
uint16_t numChannels = 2;
|
||
uint16_t bitsPerSample = 16;
|
||
uint32_t totalSamples = sampleRate * 2; // 2秒总采样数（每声道）
|
||
uint32_t totalDataBytes = 0; // PCM文件不需要头部，数据大小可以动态增长，需要最后填入
|
||
write_wav_header(fp, totalDataBytes);
|
||
is_ai_create_pcmfile = false;
|
||
ESP_LOGW(TAG_LVGL, "创建PCM基础格式成功");
|
||
is_ai_recording = true;//进入录音状态
|
||
fclose(fp);
|
||
}
|
||
|
||
int ret=esp_codec_dev_read(record_dev, feed_buff, feed_chunksize * feed_nch * sizeof(int16_t)); // 读取音频数据（录音）
|
||
//分贝检测
|
||
|
||
//音频录音
|
||
uint32_t totalDataBytes = 0;
|
||
if(is_ai_recording == true){
|
||
FILE* fp = fopen("/extflash/audio_ai.pcm", "ab");
|
||
if (!fp) {
|
||
ESP_LOGW(TAG_MAIN, "PCM文件录音失败,无法写入,语音对话阶段二失败");
|
||
}
|
||
fwrite(feed_buff, sizeof(int16_t), feed_chunksize * feed_nch, fp);
|
||
totalDataBytes += feed_chunksize * feed_nch * sizeof(int16_t);
|
||
//结束录音状态
|
||
fclose(fp);
|
||
|
||
//RMS对话结束检测函数
|
||
int num_samples = feed_chunksize * feed_nch;
|
||
long long sum_squares = 0;
|
||
for (int i = 0; i < num_samples; i++) {
|
||
sum_squares += (long long)feed_buff[i] * feed_buff[i];
|
||
}
|
||
double rms = sqrt((double)sum_squares / num_samples);
|
||
#define REFERENCE_RMS 750
|
||
double db = 20.0 * log10(rms / REFERENCE_RMS);
|
||
ESP_LOGI("AUDIO", "RMS: %.2f, dB: %.2f", rms, db);
|
||
if(rms<400){
|
||
silence_count++;
|
||
}
|
||
if(silence_count==53){
|
||
is_ai_recording = false;
|
||
is_ai_end_recording = true;
|
||
silence_count = 0;
|
||
}
|
||
}
|
||
|
||
// 录制结束，更新 PCM 文件头中的数据大小
|
||
if(is_ai_end_recording == true){
|
||
FILE* fp = fopen("/extflash/audio_ai.pcm", "ab");
|
||
if (!fp) {
|
||
ESP_LOGW(TAG_MAIN, "PCM文件录音结束失败,语音对话阶段三失败");
|
||
}
|
||
fseek(fp, 0, SEEK_SET); // 定位到文件开头
|
||
write_wav_header(fp, totalDataBytes); // 更新 WAV 头部中的数据
|
||
fclose(fp);
|
||
get_file_size("/extflash/audio_ai.pcm");
|
||
ESP_LOGW(TAG_LVGL, "PCM文件录制完成并更新头部成功");
|
||
change_task_post_action(0X000C);
|
||
is_ai_end_recording = false; //重置结束录音状态
|
||
}
|
||
|
||
|
||
|
||
// 将左右声道数据求和取半，转为单声道数据
|
||
for (int i = 0; i < feed_chunksize; i++) {
|
||
int left = feed_buff[2*i];
|
||
int right = feed_buff[2*i + 1];
|
||
// 求和并取平均，防止数值溢出
|
||
int avg = (left + right) >> 1; // 取平均
|
||
|
||
mono_buff[i] = (int)avg;
|
||
}
|
||
ESP_LOGI("I2S RAW", "L:%d, R:%d, L:%d, R:%d,L:%d, R:%d,L:%d, R:%d", feed_buff[0], feed_buff[1], feed_buff[2], feed_buff[3], feed_buff[4], feed_buff[5], feed_buff[6], feed_buff[7]);
|
||
ESP_LOGI("I2S RAW", "1:%d, 2:%d, 3:%d, 4:%d,5:%d, 6:%d,7:%d, 8:%d", mono_buff[0], mono_buff[1], mono_buff[2], mono_buff[3], mono_buff[4], mono_buff[5], mono_buff[6], mono_buff[7]);
|
||
// 从音频前端获取处理后的数据
|
||
//ESP_LOGW(TAG_LVGL, "detect_task");
|
||
//afe_fetch_result_t *res = afe_handle->fetch(afe_data_);
|
||
//int sample_count = res->data_size / sizeof(int16_t);
|
||
// 检查获取结果是否有效
|
||
//if (!res || res->ret_value == ESP_FAIL)
|
||
//{
|
||
// ESP_LOGE(TAG_LVGL, "获取音频数据失败!");
|
||
// res = afe_handle->fetch(afe_data_);
|
||
// break;
|
||
//}
|
||
/*
|
||
if(counter < 100) // 采集100次后关闭文件（约10秒音频）
|
||
{
|
||
fwrite(res->data, sizeof(int16_t), sample_count, fp);
|
||
counter++;
|
||
}
|
||
if(counter == 100) // 采集100次后关闭文件（约10秒音频）
|
||
{
|
||
fclose(fp);
|
||
counter++;
|
||
FILE* fp = fopen("/extflash/audio2res.wav", "wb");
|
||
write_wav_header(fp, totalDataBytes);
|
||
ESP_LOGW(TAG_MAIN,"WAV 文件已保存：/extflash/audio1res.wav\n");
|
||
ESP_LOGW(TAG_MAIN,"开始写入第二个文件：/extflash/audio2res.wav\n");
|
||
}
|
||
if(counter > 100 && counter < 200)
|
||
{
|
||
fwrite(res->data, sizeof(int16_t), sample_count, fp);
|
||
counter++;
|
||
}
|
||
if(counter == 200) // 采集100次后关闭文件（约10秒音频）
|
||
{
|
||
fclose(fp);
|
||
ESP_LOGW(TAG_MAIN,"第二个 WAV 文件已保存：/extflash/audio2res.wav\n");
|
||
counter++;
|
||
FILE* fp = fopen("/extflash/audio3res.wav", "wb");
|
||
write_wav_header(fp, totalDataBytes);
|
||
ESP_LOGW(TAG_MAIN,"开始写入第三个文件：/extflash/audio3res.wav\n");
|
||
}
|
||
if(counter > 200 && counter < 300)
|
||
{
|
||
fwrite(res->data, sizeof(int16_t), sample_count, fp);
|
||
counter++;
|
||
}
|
||
if(counter == 300) // 采集100次后关闭文件（约10秒音频）
|
||
{
|
||
fclose(fp);
|
||
ESP_LOGW(TAG_MAIN,"第三个 WAV 文件已保存：/extflash/audio3res.wav\n");
|
||
counter++;
|
||
break; // 结束循环，停止采集
|
||
}
|
||
// 获取并处理 VAD 状态
|
||
if (res->vad_state == 1) {
|
||
// 检测到语音活动
|
||
ESP_LOGI(TAG_LVGL, "VAD: Speech detected.");
|
||
} else if (res->vad_state == 0) {
|
||
// 当前为静音或噪声
|
||
ESP_LOGE(TAG_LVGL, "VAD: Silence.");
|
||
}
|
||
*/
|
||
|
||
/*
|
||
int sample_count = res->data_size / sizeof(int16_t);
|
||
int16_t *samples = res->data;
|
||
ESP_LOGI(TAG_LVGL, "data[0~5] = %d, %d, %d, %d, %d, %d",
|
||
samples[0], samples[1], samples[2],
|
||
samples[3], samples[4], samples[5]);
|
||
int16_t *samples2 = res->raw_data;
|
||
ESP_LOGI(TAG_LVGL, "raw_data[0~5] = %d, %d, %d, %d, %d, %d",
|
||
samples2[0], samples2[1], samples2[2],
|
||
samples2[3], samples2[4], samples2[5]);
|
||
*/
|
||
//ESP_LOGI(TAG_LVGL, "data_volume = %.2f dB, ringbuff_free_pct = %.2f%%",
|
||
// res->data_volume, res->ringbuff_free_pct * 100);*/
|
||
|
||
// 直接进行语音命令识别，跳过唤醒词检测
|
||
|
||
// 使用多网络模型进行语音命令检测
|
||
|
||
esp_mn_state_t mn_state = ESP_MN_STATE_DETECTING;
|
||
mn_state = multinet->detect(model_data, mono_buff);
|
||
// 如果仍在检测中，继续循环
|
||
if (ESP_MN_STATE_DETECTING == mn_state)
|
||
{
|
||
//ESP_LOGI(TAG_LVGL, "语音识别中...");
|
||
continue;
|
||
}
|
||
// 处理超时情况
|
||
if (ESP_MN_STATE_TIMEOUT == mn_state)
|
||
{
|
||
ESP_LOGW(TAG_LVGL, "语音识别超时");
|
||
sr_result_t result =
|
||
{
|
||
.state = mn_state,
|
||
.command_id = 0,
|
||
};
|
||
// xQueueSend(g_result_que, &result, 10);
|
||
continue;
|
||
}
|
||
// 处理识别到命令的情况
|
||
if (ESP_MN_STATE_DETECTED == mn_state)
|
||
{
|
||
// 获取识别结果
|
||
esp_mn_results_t *mn_result = multinet->get_results(model_data);
|
||
for (int i = 0; i < mn_result->num; i++)
|
||
{
|
||
int prob_permille = (int)(mn_result->prob[i] * 1000.0f);
|
||
ESP_LOGI(TAG_LVGL, "TOP %d, command_id: %d, phrase_id: %d, prob: %d.%03d",
|
||
i + 1, mn_result->command_id[i], mn_result->phrase_id[i],
|
||
prob_permille / 1000, abs(prob_permille % 1000));
|
||
}
|
||
int sr_command_id = mn_result->command_id[0];
|
||
|
||
// 在这里添加指令对应的中文打印
|
||
switch (sr_command_id)
|
||
{
|
||
case 0: // "ni hao xiao qi"
|
||
ESP_LOGE(TAG_LVGL, "识别到指令：你好小奇");
|
||
change_task_post_action(0X000B);
|
||
is_ai_create_pcmfile = true;
|
||
//ret =lv_obj_send_event(ui_Screen_Lottie, LV_EVENT_KEY, (void *)LV_KEY_ENTER);
|
||
ESP_LOGW(TAG_LVGL, "RET: %d", ret);
|
||
break;
|
||
case 1: // "huan ge feng ge"
|
||
ESP_LOGE(TAG_LVGL, "识别到指令：小奇，再见");
|
||
change_task_post_action(0X000C);
|
||
//ret =lv_obj_send_event(ui_Screen_Lottie, LV_EVENT_KEY, (void *)LV_KEY_NEXT);
|
||
ESP_LOGW(TAG_LVGL, "RET: %d", ret);
|
||
break;
|
||
case 2: // "yao tou huang nao"
|
||
ESP_LOGE(TAG_LVGL, "识别到指令：小奇，弹个吉他");
|
||
break;
|
||
case 3: // "yao tou huang nao"
|
||
ESP_LOGE(TAG_LVGL, "识别到指令：小奇，比个心");
|
||
break;
|
||
case 4: // "yao tou huang nao"
|
||
ESP_LOGE(TAG_LVGL, "识别到指令：小奇，思考一下这个问题");
|
||
break;
|
||
default:
|
||
ESP_LOGE(TAG_LVGL, "识别到未知指令：%d", sr_command_id);
|
||
break;
|
||
}
|
||
// 组织识别结果
|
||
sr_result_t result = {
|
||
.state = mn_state,
|
||
.command_id = sr_command_id,
|
||
};
|
||
}
|
||
vTaskDelay(pdMS_TO_TICKS(1));
|
||
}
|
||
if (feed_buff)
|
||
{
|
||
free(feed_buff);
|
||
feed_buff = NULL;
|
||
}
|
||
if (mono_buff)
|
||
{
|
||
free(mono_buff);
|
||
mono_buff = NULL;
|
||
}
|
||
vTaskDelete(NULL);
|
||
}
|
||
|