Shaman
61 lines
2.0 KiB
Python
61 lines
2.0 KiB
Python
import re
|
|
from dataclasses import dataclass
|
|
|
|
from razdel import sentenize
|
|
|
|
from app.normalize import normalize_common, normalize_language
|
|
from app.ssml import DEFAULT_PAUSE_MS, explicit_pause_char
|
|
from app.translit import transliterate_latin
|
|
|
|
_CYRILIC_RE = re.compile(r'[\u0400-\u04FF]')
|
|
_PARAGRAPH_RE = re.compile(r"\n[ \t]*\n")
|
|
# Явная отметка паузы во входном тексте: `|||` (пауза по умолчанию) или
|
|
# `|||1500` (миллисекунды). Извлекается ДО нормализации, длительность
|
|
# ограничивается здесь, чтобы цифры не попадали в rutextnorm.
|
|
_EXPLICIT_PAUSE_RE = re.compile(r"\|\|\|(?:(\d+))?")
|
|
|
|
|
|
@dataclass
|
|
class Segment:
|
|
text: str
|
|
lang: str
|
|
para: int = 0
|
|
|
|
|
|
def split_sentences(text: str) -> list[str]:
|
|
return [s.text for s in sentenize(text)]
|
|
|
|
|
|
def split_paragraphs(text: str) -> list[str]:
|
|
return [p for p in (block.strip() for block in _PARAGRAPH_RE.split(text)) if p]
|
|
|
|
|
|
def extract_explicit_pauses(text: str) -> str:
|
|
def repl(m: re.Match) -> str:
|
|
ms = int(m.group(1)) if m.group(1) else DEFAULT_PAUSE_MS
|
|
return explicit_pause_char(ms)
|
|
return _EXPLICIT_PAUSE_RE.sub(repl, text)
|
|
|
|
|
|
def has_cyrillic(text: str) -> bool:
|
|
return bool(_CYRILIC_RE.search(text))
|
|
|
|
|
|
def detect_language(text: str) -> str:
|
|
return "ru" if has_cyrillic(text) else "en"
|
|
|
|
|
|
def prepare_segments(text: str) -> list[Segment]:
|
|
lang = detect_language(text)
|
|
text = extract_explicit_pauses(text)
|
|
text = normalize_common(text)
|
|
all_segments: list[Segment] = []
|
|
for para_idx, para in enumerate(split_paragraphs(text)):
|
|
for sent in split_sentences(para):
|
|
if lang == "ru":
|
|
sent = transliterate_latin(sent)
|
|
seg_text = normalize_language(sent, lang)
|
|
if seg_text.strip():
|
|
all_segments.append(Segment(text=seg_text, lang=lang, para=para_idx))
|
|
return all_segments
|