add-text-preprocessing

src/App.tsx

@@ -3,6 +3,7 @@ import { Zap, AlignLeft, Quote, Type, FileText, Check, X, Dices } from "lucide-r
 import { useTTS } from "./components/TTSContext";
 import { TTSProvider } from "./components/TTSProvider";
 import { streamTTS, createAudioBlob } from "./tts";
+import { preprocessText } from "./text-preprocessor";
 import { SAMPLE_RATE, EXAMPLE_SENTENCES } from "./constants";
 import { AudioResult } from "./components/AudioResult";
 import { Controls } from "./components/Controls";
@@ -155,7 +156,8 @@ const AppContent = () => {
       if (!tts.current || !speakerEmbeddings.current) throw new Error("TTS pipeline not ready");
       const selectedEmbedding = speakerEmbeddings.current[voice];
 
-      for await (const result of streamTTS(text, tts.current, selectedEmbedding, quality, speed)) {
+      const preprocessedText = preprocessText(text);
+      for await (const result of streamTTS(preprocessedText, tts.current, selectedEmbedding, quality, speed)) {
         if (stopGenerationRef.current) {
           break;
         }

src/text-preprocessor.ts

@@ -0,0 +1,85 @@
+/**
+ * Preprocesses text for TTS by normalizing, cleaning, and correcting punctuation.
+ * Based on: https://github.com/supertone-inc/supertonic/blob/main/csharp/Helper.cs
+ * @param text The input text to preprocess.
+ * @returns The cleaned and preprocessed text.
+ */
+export function preprocessText(text: string): string {
+  // Normalize to NFKD form to separate characters from their diacritics.
+  text = text.normalize("NFKD");
+
+  // Remove emojis.
+  text = text.replace(
+    /([\u2600-\u27BF]|[\uE000-\uF8FF]|\uD83C[\uDC00-\uDFFF]|\uD83D[\uDC00-\uDFFF]|\uD83E[\uDD10-\uDDFF])/g,
+    "",
+  );
+
+  // Define character replacements.
+  const replacements: Record<string, string> = {
+    "—": "-", // em dash
+    "–": "-", // en dash
+    "‑": "-", // non-breaking hyphen
+    "¯": " ", // macron
+    "_": " ", // underscore
+    "\u201C": '"', // left double quote
+    "\u201D": '"', // right double quote
+    "\u2018": "'", // left single quote
+    "\u2019": "'", // right single quote
+    "´": "'", // acute accent
+    "`": "'", // grave accent
+    "[": " ",
+    "]": " ",
+    "|": " ",
+    "/": " ",
+    "#": " ",
+    "→": " ",
+    "←": " ",
+  };
+
+  // Apply character replacements.
+  for (const [key, value] of Object.entries(replacements)) {
+    text = text.replace(new RegExp(`\\${key}`, "g"), value);
+  }
+
+  // Remove combining diacritical marks.
+  text = text.replace(/[\u0300-\u036f]/g, "");
+
+  // Remove special symbols that are not handled by emoji removal.
+  text = text.replace(/[♥☆♡©\\]/g, "");
+
+  // Replace known expressions.
+  const exprReplacements: Record<string, string> = {
+    "@": " at ",
+    " e.g.": " for example, ",
+    " i.e.": " that is, ",
+  };
+
+  for (const [key, value] of Object.entries(exprReplacements)) {
+    text = text.replace(new RegExp(key, "g"), value);
+  }
+
+  // Fix spacing around punctuation.
+  text = text
+    .replace(/ ,/g, ",")
+    .replace(/ \./g, ".")
+    .replace(/ !/g, "!")
+    .replace(/ \?/g, "?")
+    .replace(/ ;/g, ";")
+    .replace(/ :/g, ":")
+    .replace(/ '/g, "'");
+
+  // Remove duplicate quotes.
+  text = text.replace(/""+/g, '"');
+  text = text.replace(/''+/g, "'");
+  text = text.replace(/``+/g, "`");
+
+  // Remove extra spaces.
+  text = text.replace(/\s+/g, " ").trim();
+
+  // If text doesn't end with punctuation, quotes, or closing brackets, add a period.
+  if (!/[.!?;:,'")\]}…。」』】〉》›»]$/.test(text)) {
+    text += ".";
+  }
+
+  return text;
+}