34 lines
1.2 KiB
C++
34 lines
1.2 KiB
C++
#include "transcript_cleaner.h"
|
|
|
|
#include <cctype>
|
|
#include <string_view>
|
|
|
|
std::string clean_transcript(std::string transcript) {
|
|
// Qwen3-ASR prefixes raw transcripts with the detected language and a text marker.
|
|
constexpr std::string_view kLanguagePrefix = "language ";
|
|
constexpr std::string_view kMarkers[] = {"<asr_text>", "<astext>"};
|
|
|
|
size_t prefix = 0;
|
|
if (transcript.compare(0, 3, "\xef\xbb\xbf") == 0) prefix = 3;
|
|
while (prefix < transcript.size() &&
|
|
std::isspace(static_cast<unsigned char>(transcript[prefix]))) {
|
|
prefix++;
|
|
}
|
|
if (transcript.compare(prefix, kLanguagePrefix.size(), kLanguagePrefix) != 0) {
|
|
return transcript;
|
|
}
|
|
|
|
size_t marker_position = std::string::npos;
|
|
std::string_view matched_marker;
|
|
for (std::string_view marker : kMarkers) {
|
|
const size_t position = transcript.find(marker, prefix + kLanguagePrefix.size());
|
|
if (position != std::string::npos &&
|
|
(marker_position == std::string::npos || position < marker_position)) {
|
|
marker_position = position;
|
|
matched_marker = marker;
|
|
}
|
|
}
|
|
if (marker_position == std::string::npos) return transcript;
|
|
return transcript.substr(marker_position + matched_marker.size());
|
|
}
|