重大进展 RetroArch 专用 AI 服务接口已经写出来了

blindholmes · 发表于 2025-1-19 00:11

ラングリッサー发表于 2025-1-18 22:16
ra哪个接口？

这个服务是 RA 请求我的接口，我返回数据给它
数据定义在 `tasks\task_translation.c` 这个文件里，从 200 行开始

/* Parse JSON body for the image and sound data */
for (;;)
{
static const char *keys[] = { "image", "sound", "text", "error", "auto", "press" };
const char *str = NULL;
size_t str_len = 0;
enum rjson_type json_type = rjson_next(json);
if (json_type == RJSON_DONE || json_type == RJSON_ERROR)
break;
if (json_type != RJSON_STRING)
continue;
if (rjson_get_context_type(json) != RJSON_OBJECT)
continue;
str = rjson_get_string(json, &str_len);
if ((rjson_get_context_count(json) & 1) == 1)
{
int i;
json_current_key = -1;
for (i = 0; i < (int)ARRAY_SIZE(keys); i++)
{
if (string_is_equal(str, keys[i]))
{
json_current_key = i;
break;
}
}
}
else
{
switch (json_current_key)
{
case 0: /* image */
raw_image_file_data = (char*)unbase64(str,
(int)str_len, &new_image_size);
break;
#ifdef HAVE_AUDIOMIXER
case 1: /* sound */
raw_sound_data = (void*)unbase64(str,
(int)str_len, &new_sound_size);
break;
#endif
case 2: /* text */
txt_str = strdup(str);
break;
case 3: /* error */
err_str = strdup(str);
break;
case 4: /* auto */
auto_str = strdup(str);
break;
case 5: /* press */
key_str = strdup(str);
break;
}
json_current_key = -1;
}
}
if (string_is_equal(err_str, "No text found."))
{
#ifdef DEBUG
RARCH_LOG("No text found...\n");
#endif
if (txt_str)
{
free(txt_str);
txt_str = NULL;
}
txt_str = (char*)malloc(15);
strlcpy(txt_str, err_str, 15);
#ifdef HAVE_GFX_WIDGETS
if (gfx_widgets_paused)
{
/* In this case we have to unpause and then repause for a frame */
p_dispwidget->ai_service_overlay_state = 2;
command_event(CMD_EVENT_UNPAUSE, NULL);
}
#endif
}
if ( !raw_image_file_data
&& !raw_sound_data
&& !txt_str
&& !key_str
&& (access_st->ai_service_auto != 2))
{
error = "Invalid JSON body.";
goto finish;
}
if (raw_image_file_data)
{
unsigned image_width, image_height;
/* Get the video frame dimensions reference */
const void *dummy_data = video_st->frame_cache_data;
unsigned width = video_st->frame_cache_width;
unsigned height = video_st->frame_cache_height;
/* try two different modes for text display *
* In the first mode, we use display widget overlays, but they require
* the video poke interface to be able to load image buffers.
*
* The other method is to draw to the video buffer directly, which needs
* a software core to be running. */
#ifdef HAVE_GFX_WIDGETS
if ( video_st->poke
&& video_st->poke->load_texture
&& video_st->poke->unload_texture)
{
enum image_type_enum image_type;
/* Write to overlay */
if ( raw_image_file_data[0] == 'B'
&& raw_image_file_data[1] == 'M')
image_type = IMAGE_TYPE_BMP;
else if ( raw_image_file_data[1] == 'P'
&& raw_image_file_data[2] == 'N'
&& raw_image_file_data[3] == 'G')
image_type = IMAGE_TYPE_PNG;
else
{
/* TODO/FIXME - localize */
RARCH_LOG("Invalid image type returned from server.\n");
goto finish;
}
if (!gfx_widgets_ai_service_overlay_load(
raw_image_file_data, (unsigned)new_image_size,
image_type))
{
/* TODO/FIXME - localize */
const char *_msg = "Video driver not supported.";
RARCH_LOG("Video driver not supported for AI Service.");
runloop_msg_queue_push(_msg, strlen(_msg), 1, 180, true, NULL,
MESSAGE_QUEUE_ICON_DEFAULT, MESSAGE_QUEUE_CATEGORY_INFO);
}
else if (gfx_widgets_paused)
{
/* In this case we have to unpause and then repause for a frame */
/* Unpausing state */
p_dispwidget->ai_service_overlay_state = 2;
command_event(CMD_EVENT_UNPAUSE, NULL);
}
}
else
#endif
/* Can't use display widget overlays, so try writing to video buffer */
{
size_t pitch;
/* Write to video buffer directly (software cores only) */
/* This is a BMP file coming back. */
if ( raw_image_file_data[0] == 'B'
&& raw_image_file_data[1] == 'M')
{
/* Get image data (24 bit), and convert to the emulated pixel format */
image_width =
((uint32_t) ((uint8_t)raw_image_file_data[21]) << 24) +
((uint32_t) ((uint8_t)raw_image_file_data[20]) << 16) +
((uint32_t) ((uint8_t)raw_image_file_data[19]) << 8) +
((uint32_t) ((uint8_t)raw_image_file_data[18]) << 0);
image_height =
((uint32_t) ((uint8_t)raw_image_file_data[25]) << 24) +
((uint32_t) ((uint8_t)raw_image_file_data[24]) << 16) +
((uint32_t) ((uint8_t)raw_image_file_data[23]) << 8) +
((uint32_t) ((uint8_t)raw_image_file_data[22]) << 0);
raw_image_data = (void*)malloc(image_width * image_height * 3 * sizeof(uint8_t));
if (raw_image_data)
memcpy(raw_image_data,
raw_image_file_data + 54 * sizeof(uint8_t),
image_width * image_height * 3 * sizeof(uint8_t));
}
/* PNG coming back from the url */
else if (raw_image_file_data[1] == 'P'
&& raw_image_file_data[2] == 'N'
&& raw_image_file_data[3] == 'G')
{
int retval = 0;
rpng_t *rpng = NULL;
image_width =
((uint32_t) ((uint8_t)raw_image_file_data[16]) << 24)+
((uint32_t) ((uint8_t)raw_image_file_data[17]) << 16)+
((uint32_t) ((uint8_t)raw_image_file_data[18]) << 8)+
((uint32_t) ((uint8_t)raw_image_file_data[19]) << 0);
image_height =
((uint32_t) ((uint8_t)raw_image_file_data[20]) << 24)+
((uint32_t) ((uint8_t)raw_image_file_data[21]) << 16)+
((uint32_t) ((uint8_t)raw_image_file_data[22]) << 8)+
((uint32_t) ((uint8_t)raw_image_file_data[23]) << 0);
if (!(rpng = rpng_alloc()))
{
error = "Can't allocate memory.";
goto finish;
}
rpng_set_buf_ptr(rpng, raw_image_file_data, (size_t)new_image_size);
rpng_start(rpng);
while (rpng_iterate_image(rpng));
do
{
retval = rpng_process_image(rpng, &raw_image_data_alpha,
(size_t)new_image_size, &image_width, &image_height);
} while (retval == IMAGE_PROCESS_NEXT);
/* Returned output from the png processor is an upside down RGBA
* image, so we have to change that to RGB first. This should
* probably be replaced with a scaler call.*/
{
unsigned ui;
int tw, th, tc;
int d = 0;
raw_image_data = (void*)malloc(image_width*image_height*3*sizeof(uint8_t));
for (ui = 0; ui < image_width * image_height * 4; ui++)
{
if (ui % 4 != 3)
{
tc = d % 3;
th = image_height-d / (image_width * 3) - 1;
tw = (d % (image_width * 3)) / 3;
((uint8_t*) raw_image_data)[tw * 3 + th * 3 * image_width + tc] = ((uint8_t *)raw_image_data_alpha)[ui];
d += 1;
}
}
}
rpng_free(rpng);
}
else
{
RARCH_LOG("Output from URL not a valid file type, or is not supported.\n");
goto finish;
}
if (!(scaler = (struct scaler_ctx*)calloc(1, sizeof(struct scaler_ctx))))
goto finish;
if (dummy_data == RETRO_HW_FRAME_BUFFER_VALID)
{
/*
In this case, we used the viewport to grab the image
and translate it, and we have the translated image in
the raw_image_data buffer.
*/
RARCH_LOG("Hardware frame buffer core, but selected video driver isn't supported.\n");
goto finish;
}
/* The assigned pitch may not be reliable. The width of
the video frame can change during run-time, but the
pitch may not, so we just assign it as the width
times the byte depth.
*/
if (video_driver_pix_fmt == RETRO_PIXEL_FORMAT_XRGB8888)
{
raw_output_data = (uint8_t*)malloc(width * height * 4 * sizeof(uint8_t));
scaler->out_fmt = SCALER_FMT_ARGB8888;
pitch = width * 4;
scaler->out_stride = (int)pitch;
}
else
{
raw_output_data = (uint8_t*)malloc(width * height * 2 * sizeof(uint8_t));
scaler->out_fmt = SCALER_FMT_RGB565;
pitch = width * 2;
scaler->out_stride = width;
}
if (!raw_output_data)
goto finish;
scaler->in_fmt = SCALER_FMT_BGR24;
scaler->in_width = image_width;
scaler->in_height = image_height;
scaler->out_width = width;
scaler->out_height = height;
scaler->scaler_type = SCALER_TYPE_POINT;
scaler_ctx_gen_filter(scaler);
scaler->in_stride = -1 * width * 3;
scaler_ctx_scale_direct(scaler, raw_output_data,
(uint8_t*)raw_image_data + (image_height - 1) * width * 3);
video_driver_frame(raw_output_data, image_width, image_height, pitch);
}
}
#ifdef HAVE_AUDIOMIXER
if (raw_sound_data)
{
audio_mixer_stream_params_t params;
params.volume = 1.0f;
params.slot_selection_type = AUDIO_MIXER_SLOT_SELECTION_MANUAL; /* user->slot_selection_type; */
params.slot_selection_idx = 10;
params.stream_type = AUDIO_STREAM_TYPE_SYSTEM; /* user->stream_type; */
params.type = AUDIO_MIXER_TYPE_WAV;
params.state = AUDIO_STREAM_STATE_PLAYING;
params.buf = raw_sound_data;
params.bufsize = new_sound_size;
params.cb = NULL;
params.basename = NULL;
audio_driver_mixer_add_stream(&params);
if (raw_sound_data)
{
free(raw_sound_data);
raw_sound_data = NULL;
}
}

复制代码

ラングリッサー · 发表于 2025-1-19 09:05

看来这个API要wav 不要PCM
https://jun711.github.io/aws/convert-aws-polly-synthesized-speech-from-pcm-to-wav-format/
试试这个Python？

blindholmes · 发表于 2025-1-19 18:56

卧槽 PCM 和 wav 还不一样？

账号		自动登录	找回密码
密码			注册

[原创] 重大进展 RetroArch 专用 AI 服务接口已经写出来了

浏览过的版块