import re from contextlib import contextmanager from functools import lru_cache from typing import NamedTuple import numpy as np from scipy.io import wavfile from scipy.signal import butter, sosfilt SR = 44100 VIOLET, INDIGO, BLUE, CYAN, GREEN, YELLOW, ORANGE, RED = 2200, 1850, 1500, 1150, 800, 550, 380, 250 def env(sig, attack=0.1, release=0.3): n = len(sig) e = np.ones(n) a, r = int(n * attack), int(n * release) if a > 0: e[:a] = np.linspace(0, 1, a) if r > 0: e[-r:] *= np.linspace(1, 0, r) return sig * e @lru_cache(maxsize=512) def _sos(btype, wn, order): return butter(order, wn, btype=btype, output='sos') def bandpass(sig, low, high, order=2): low, high = max(20, low), min(SR / 2 - 100, high) return sosfilt(_sos('band', (low / (SR / 2), high / (SR / 2)), order), sig) def lowpass(sig, cutoff, order=2): return sosfilt(_sos('low', cutoff / (SR / 2), order), sig) def highpass(sig, cutoff, order=2): return sosfilt(_sos('high', cutoff / (SR / 2), order), sig) def silence(dur): return np.zeros(int(SR * dur)) def _noise(n): return np.random.uniform(-1, 1, n) def _tone(freq, n): return np.sin(2 * np.pi * freq * np.arange(n) / SR) _quiet = False def _warn(msg): if not _quiet: print(f"[warning] {msg}") _VOICE = {'pitch': 1.0, 'formant': 1.0} VOICE_PRESETS = { 'neu': (1.00, 1.00), 'masc': (0.78, 0.90), 'fem': (1.28, 1.10), } def set_voice(voice=None, pitch=None, formant=None): if voice is not None: _VOICE['pitch'], _VOICE['formant'] = VOICE_PRESETS[voice] if isinstance(voice, str) else voice if pitch is not None: _VOICE['pitch'] = pitch if formant is not None: _VOICE['formant'] = formant def _fs(freq): return freq * _VOICE['formant'] def pulse(freq, coherent=True, dur=0.09): freq = _fs(freq) noise = _noise(int(SR * dur)) return env(bandpass(noise, freq * 0.9, freq * 1.1, order=4) if coherent else noise, 0.02, 0.5) def _stop(freq, coherent=True, dm=1.0): base = 0.09 closure = silence(base * (dm - 1)) if dm > 1.0 else np.zeros(0) return np.concatenate([closure, pulse(freq, coherent, base * min(dm, 1.0))]) def turbulence(freq, coherent=True, dur=0.16): freq = _fs(freq) n = int(SR * dur) shaped = lowpass(_noise(n), freq * 1.3) sig = shaped * 0.8 + _tone(freq, n) * 0.3 if coherent else shaped return env(sig, 0.1, 0.25) def sibilant(freq, coherent=True, dur=0.14): freq = _fs(freq) n = int(SR * dur) sig = highpass(_noise(n), 1800 + freq * 0.3) if coherent: sig = sig * 0.7 + bandpass(_noise(n), freq * 1.5, freq * 2.0, order=4) * 0.5 return env(sig, 0.03, 0.2) def hum(freq, coherent=True, dur=0.18): freq = _fs(freq) n = int(SR * dur) tone = (_tone(freq, n) * 0.7 + _tone(freq * 2, n) * 0.25 + _tone(freq * 0.5, n) * 0.2 + _tone(freq * 1.005, n) * 0.4) sig = tone if coherent else lowpass(_noise(n), freq * 1.4) * 0.85 + tone * 0.15 return env(sig, 0.1, 0.35) def flutter(freq, coherent=True, dur=0.22, rate=26): freq = _fs(freq) n = int(SR * dur) phase = (np.arange(n) / SR * rate) % 1.0 gate = np.clip(1 - 10 * np.abs(phase - 0.2), 0, 1) carrier = _tone(freq, n) if coherent else lowpass(_noise(n), freq * 1.4) return env(carrier * gate, 0.05, 0.15) def wave(freq, coherent=True, dur=0.15): freq = _fs(freq) n = int(SR * dur) sig = _tone(freq, n) if coherent else lowpass(_noise(n), freq * 1.2) * 0.85 + _tone(freq, n) * 0.15 return env(sig, 0.35, 0.4) def splitwave(freq, coherent=True, dur=0.16): freq = _fs(freq) n = int(SR * dur) if coherent: sig = 0.5 * _tone(freq * 0.85, n) + 0.5 * _tone(freq * 1.18, n) else: ch1 = lowpass(_noise(n), freq * 0.85 * 1.3) ch2 = lowpass(_noise(n), freq * 1.18 * 1.3) sig = 0.5 * ch1 + 0.5 * ch2 return env(sig, 0.2, 0.35) def splitturbulence(freq, coherent=True, dur=0.16): freq = _fs(freq) n = int(SR * dur) ch1 = bandpass(_noise(n), freq * 0.45, freq * 0.75, order=6) ch2 = bandpass(_noise(n), freq * 1.25, freq * 1.65, order=6) shaped = 0.5 * ch1 + 0.5 * ch2 if coherent: shaped = shaped * 0.8 + _tone(freq, n) * 0.3 return env(shaped, 0.1, 0.25) def affricate(freq, coherent=True, dm=1.0): return np.concatenate([pulse(freq, coherent, 0.05 * dm), turbulence(freq, coherent, 0.1 * dm)]) def aspiration(dur=0.05): return turbulence(GREEN, False, dur) * 0.6 def labialization(dur=0.045): return wave(VIOLET, dur=dur) * 0.6 def palatalization(dur=0.045): return wave(BLUE, dur=dur) * 0.6 def _build_f0_signal(f0, n): if f0 is None: return None if isinstance(f0, (list, tuple)) and len(f0) > 1: bounds = np.linspace(0, n, len(f0)).astype(int) curve = np.zeros(n) for i in range(len(f0) - 1): s, e = bounds[i], bounds[i + 1] curve[s:e] = np.linspace(f0[i], f0[i + 1], e - s, endpoint=False) curve[-1] = f0[-1] return np.sin(np.cumsum(2 * np.pi * curve / SR)) return _tone(f0[0] if isinstance(f0, (list, tuple)) else f0, n) def _finish_vowel(sig, n, f0): f0_sig = _build_f0_signal(f0, n) if f0_sig is not None: sig = sig * 0.7 + f0_sig * 0.35 return env(sig, 0.15, 0.3) def vowel_glide(freq_pairs, total_dur, f0=None): n = int(SR * total_dur) if n <= 0: return np.zeros(0) if len(freq_pairs) == 1: f1, f2 = freq_pairs[0] return _finish_vowel(0.5 * _tone(f1, n) + 0.5 * _tone(f2, n), n, f0) bounds = np.linspace(0, n, len(freq_pairs)).astype(int) curves = np.zeros((2, n)) for i in range(len(freq_pairs) - 1): s, e = bounds[i], bounds[i + 1] for k in (0, 1): curves[k, s:e] = np.linspace(freq_pairs[i][k], freq_pairs[i + 1][k], e - s, endpoint=False) curves[:, -1] = freq_pairs[-1] phases = np.cumsum(2 * np.pi * curves / SR, axis=1) return _finish_vowel(0.5 * np.sin(phases[0]) + 0.5 * np.sin(phases[1]), n, f0) def _overlap_add(parts, overlap_sec=0.02): if not parts: return np.zeros(0) k0 = int(SR * overlap_sec) out = parts[0] for p in parts[1:]: k = min(k0, len(out), len(p)) if k > 0: out = np.concatenate([out[:-k], out[-k:] + p[:k], p[k:]]) else: out = np.concatenate([out, p]) return out TONES = {'`': 260, "'": 220, '-': 185, ',': 150, '.': 115} _SHAPES = { 'stop': lambda f, c, dm: _stop(f, c, dm), 'glstop': lambda f, c, dm: pulse(f, c, 0.09 * dm), 'fric': lambda f, c, dm: turbulence(f, c, 0.16 * dm), 'appr': lambda f, c, dm: wave(f, c, 0.15 * dm), 'lat': lambda f, c, dm: splitwave(f, c, 0.16 * dm), 'tap': lambda f, c, dm: pulse(f, c, 0.04 * dm), 'trill': lambda f, c, dm: flutter(f, c, 0.22 * dm), 'nasal': lambda f, c, dm: hum(f, c, 0.18 * dm), 'affr': lambda f, c, dm: affricate(f, c, dm), 'sib': lambda f, c, dm: sibilant(f, c, 0.14 * dm), 'latfric': lambda f, c, dm: splitturbulence(f, c, 0.16 * dm), } _COLUMNS = [('stop', False), ('stop', True), ('fric', False), ('fric', True), ('appr', True), ('lat', True), ('tap', False), ('tap', True), ('trill', True), ('nasal', True), ('affr', False), ('affr', True), ('sib', False), ('sib', True), ('latfric', False), ('latfric', True)] _PLACES = [ (VIOLET, 'p b ɸ β w ⱳ ᴘ ⱱ ʙ m'), (INDIGO, 't d θ ð ɹ l ɍ ɾ r n ʦ ʣ s z ɬ ɮ'), (BLUE, 'c ɟ ç ʝ j ʎ ꞔ ƴ ɏ ɲ ʨ ʥ ɕ ʑ ꜿ ỿ'), (CYAN, 'q ɢ χ ʁ ɰ ʟ ꝗ ꞡ ʀ ɴ'), (GREEN, 'ʔ 7 h ɦ ʕ ꝇ ƕ ʮ ʢ ƞ'), (YELLOW, 'ფ ბ ს ზ ჯ ლ ტ წ რ ნ'), (ORANGE, 'փ բ ֆ վ ր լ չ ջ ռ մ'), (RED, 'Ⱂ Ⰱ Ⱄ Ⰸ Ⰻ Ⰾ Ⱌ Ⰷ Ⱃ Ⰿ'), ] def _shape(manner, freq, coherent): fn = _SHAPES['glstop' if manner == 'stop' and freq == GREEN else manner] return lambda dm=1.0: fn(freq, coherent, dm) CONSONANTS = {} SONORANT_VOICELESS = {} for _freq, _row in _PLACES: for _sym, (_manner, _voiced) in zip(_row.split(), _COLUMNS): CONSONANTS[_sym] = _shape(_manner, _freq, _voiced) if _manner in ('appr', 'lat', 'trill', 'nasal'): SONORANT_VOICELESS[_sym] = _shape(_manner, _freq, False) VOWELS = { 'i': (280, 2250), 'y': (280, 1900), 'ɯ': (280, 1200), 'u': (280, 650), 'ɪ': (350, 2000), 'ɵ': (420, 1400), 'ʊ': (380, 850), 'e': (400, 2100), 'ø': (400, 1700), 'ɤ': (400, 1300), 'o': (400, 700), 'ə': (500, 1500), 'ɛ': (570, 1900), 'œ': (570, 1600), 'ʌ': (600, 1100), 'ɔ': (550, 850), 'æ': (700, 1750), 'ɐ': (700, 1400), 'ɑ': (750, 1050), 'a': (850, 1600), } ASCII = { 'p': 'p', 'b': 'b', 't': 't', 'd': 'd', 'k': 'c', 'g': 'ɟ', 'q': 'q', 'gq': 'ɢ', '?': 'ʔ', '7': '7', 'uvt': 'ფ', 'uvd': 'ბ', 'xrt': 'փ', 'xrd': 'բ', 'gmt': 'Ⱂ', 'gmd': 'Ⰱ', 'f': 'ɸ', 'v': 'β', 'th': 'θ', 'dh': 'ð', 'kh': 'ç', 'gh': 'ʝ', 'x': 'χ', 'xg': 'ʁ', 'h': 'h', 'hg': 'ɦ', 'uvs': 'ს', 'uvz': 'ზ', 'xrs': 'ֆ', 'xrz': 'վ', 'gms': 'Ⱄ', 'gmz': 'Ⰸ', 'w': 'w', 'wl': 'ⱳ', 'rz': 'ɹ', 'l': 'l', 'j': 'j', 'lj': 'ʎ', 'jq': 'ɰ', 'lq': 'ʟ', 'jh': 'ʕ', 'lh': 'ꝇ', 'uvj': 'ჯ', 'uvl': 'ლ', 'xrj': 'ր', 'xrl': 'լ', 'gmj': 'Ⰻ', 'gml': 'Ⰾ', 'rp': 'ᴘ', 'rb': 'ⱱ', 'rh': 'ɍ', 'r': 'ɾ', 'rk': 'ꞔ', 'rj': 'ƴ', 'rq': 'ꝗ', 'rg': 'ꞡ', 'hr': 'ƕ', 'hgr': 'ʮ', 'uvrh': 'ტ', 'uvr': 'წ', 'xrrh': 'չ', 'xrr': 'ջ', 'gmrh': 'Ⱌ', 'gmr': 'Ⰷ', 'bb': 'ʙ', 'rr': 'r', 'jj': 'ɏ', 'gg': 'ʀ', 'hh': 'ʢ', 'uvrr': 'რ', 'xrrr': 'ռ', 'gmrr': 'Ⱃ', 'm': 'm', 'n': 'n', 'nj': 'ɲ', 'ng': 'ɴ', 'nh': 'ƞ', 'uvn': 'ნ', 'xrn': 'մ', 'gmn': 'Ⰿ', 'ts': 'ʦ', 'dz': 'ʣ', 'tsh': 'ʨ', 'dzh': 'ʥ', 's': 's', 'z': 'z', 'sh': 'ɕ', 'zh': 'ʑ', 'ls': 'ɬ', 'lz': 'ɮ', 'lsh': 'ꜿ', 'lzh': 'ỿ', 'i': 'i', 'y': 'y', 'eu': 'ɯ', 'u': 'u', 'ie': 'ɪ', 'yo': 'ɵ', 'uo': 'ʊ', 'e': 'e', 'oe': 'ø', 'eo': 'ɤ', 'o': 'o', 'ea': 'ə', 'ee': 'ɛ', 'oee': 'œ', 'eoo': 'ʌ', 'oo': 'ɔ', 'ae': 'æ', 'eaa': 'ɐ', 'aa': 'ɑ', 'a': 'a', } SECONDARY = {'>h': aspiration, '>w': labialization, '>j': palatalization} PLACE_TRANSITION = { VIOLET: ('locus', 700), INDIGO: ('locus', 1800), BLUE: ('locus', 2400), CYAN: ('perturb', (120, -350)), } TRANSITION_PLACE = { 'p': VIOLET, 'b': VIOLET, 't': INDIGO, 'd': INDIGO, 'c': BLUE, 'ɟ': BLUE, 'q': CYAN, 'ɢ': CYAN, 'χ': CYAN, 'ʁ': CYAN, } def _place_vowel_transition(place_color, vowel_freqs, dur=0.055): entry = PLACE_TRANSITION.get(place_color) if entry is None or entry[1] is None: return None kind, param = entry f1, f2 = vowel_freqs if kind == 'perturb': n = int(SR * 0.07) d1, d2 = _fs(param[0]), _fs(param[1]) f1_curve = np.linspace(f1 + d1, f1, n) f2_curve = np.linspace(max(200, f2 + d2), f2, n) else: n = int(SR * dur) if n <= 0: return None f1_curve = np.full(n, f1) f2_curve = np.linspace(_fs(param), f2, n) sig = 0.5 * np.sin(np.cumsum(2 * np.pi * f1_curve / SR)) + 0.5 * np.sin(np.cumsum(2 * np.pi * f2_curve / SR)) return env(sig, 0.08, 0.08) * 1.6 LONG_STEP = 1.8 BRIEF_STEP = 1.8 class Phone(NamedTuple): core: str secondary: list tone: object dur: float devoiced: bool def _strip_marks(phon): core, secondary = phon, [] while core[-2:] in SECONDARY: secondary.insert(0, core[-2:]) core = core[:-2] devoiced = core.endswith('*') if devoiced: core = core[:-1] stripped = core.rstrip(''.join(TONES)) tones = tuple(TONES[c] for c in core[len(stripped):]) core = stripped tone_f0 = tones[0] if len(tones) == 1 else (tones or None) stripped = core.rstrip(':') if len(stripped) != len(core): dur_mult, core = LONG_STEP ** (len(core) - len(stripped)), stripped else: stripped = core.rstrip('+') dur_mult = BRIEF_STEP ** -(len(core) - len(stripped)) core = stripped return core, secondary, tone_f0, dur_mult, devoiced def _parse(phones, translate=None): parsed = [] for p in phones: core, secondary, tone, dur, devoiced = _strip_marks(p) if translate: core = translate.get(core, core) parsed.append(Phone(core, secondary, tone, dur, devoiced)) return parsed def _group_phones(parsed): groups, i = [], 0 while i < len(parsed): if parsed[i].core in VOWELS: j = i + 1 while j < len(parsed) and parsed[j].core in VOWELS: j += 1 groups.append(('v', list(range(i, j)))) i = j else: groups.append(('c', i)) i += 1 return groups def _consonant_audio(ph, text, unknown_dur): if ph.core not in CONSONANTS: _warn(f"unknown phoneme '{text}', inserting silence") return silence(unknown_dur) if ph.tone is not None: _warn(f"tone mark ignored on consonant '{text}'") if ph.devoiced and ph.core in SONORANT_VOICELESS: audio = SONORANT_VOICELESS[ph.core](ph.dur) else: if ph.devoiced: _warn(f"no voiceless form for consonant '{text}', ignoring '*'") audio = CONSONANTS[ph.core](ph.dur) if ph.secondary: audio = np.concatenate([audio] + [SECONDARY[m]() for m in ph.secondary]) return audio def _formants(parsed, run): return [(_fs(VOWELS[parsed[i].core][0]), _fs(VOWELS[parsed[i].core][1])) for i in run] def _onset(parsed, groups, gi, first_pair): if gi > 0 and groups[gi - 1][0] == 'c': place = TRANSITION_PLACE.get(parsed[groups[gi - 1][1]].core) if place is not None: return _place_vowel_transition(place, first_pair) return None def _tone_contour(tones): if all(t is None for t in tones): return None filled, last = [], None for t in tones: if t is None: filled.append(185 if last is None else last) else: filled.extend(t if isinstance(t, tuple) else (t,)) last = t[-1] if isinstance(t, tuple) else t filled = [_VOICE['pitch'] * f for f in filled] return filled if len(filled) > 1 else filled[0] def _render_phones(phones, seg_gap=0.02, vowel_gap=0.004, translate=None): parsed = _parse(phones, translate) groups = _group_phones(parsed) chunks, is_vowel = [], [] for gi, (kind, ref) in enumerate(groups): if kind == 'c': chunks.append(_consonant_audio(parsed[ref], phones[ref], 0.01)) else: pairs = _formants(parsed, ref) audio = vowel_glide(pairs, sum(0.22 * parsed[i].dur for i in ref), f0=_tone_contour([parsed[i].tone for i in ref])) trans = _onset(parsed, groups, gi, pairs[0]) if trans is not None: audio = _overlap_add([trans, audio], 0.015) chunks.append(audio) is_vowel.append(kind == 'v') out = [] for gi, chunk in enumerate(chunks): out.append(chunk) if gi < len(chunks) - 1: out.append(silence(vowel_gap if is_vowel[gi + 1] else seg_gap)) return out NOTE_SEMITONE = {'C': -9, 'C#': -8, 'Db': -8, 'D': -7, 'D#': -6, 'Eb': -6, 'E': -5, 'F': -4, 'F#': -3, 'Gb': -3, 'G': -2, 'G#': -1, 'Ab': -1, 'A': 0, 'A#': 1, 'Bb': 1, 'B': 2} def note_to_hz(note): if isinstance(note, (int, float)): return float(note) m = re.match(r'([A-Ga-g][#b]?)(-?\d+)', note) if not m: raise ValueError(f"unrecognized note name: {note!r}") n = NOTE_SEMITONE[m.group(1).capitalize()] + (int(m.group(2)) - 4) * 12 return 440.0 * 2 ** (n / 12) def sleep(beats): return (None, None, beats) def render_note_syllable(phoneme_str, note, dur_sec, min_vowel_dur=0.05, xfade=0.02, translate=ASCII): hz = note_to_hz(note) phones = phoneme_str.split() parsed = _parse(phones, translate) groups = _group_phones(parsed) cons = {ref: _consonant_audio(parsed[ref], phones[ref], 0.03) for kind, ref in groups if kind == 'c'} slots = sum(len(ref) for kind, ref in groups if kind == 'v') per_vowel = 0.0 if slots: free = dur_sec - sum(len(a) for a in cons.values()) / SR per_vowel = max(min_vowel_dur * slots, free) / slots chunks = [] for gi, (kind, ref) in enumerate(groups): if kind == 'c': chunks.append(cons[ref]) else: pairs = _formants(parsed, ref) trans = _onset(parsed, groups, gi, pairs[0]) if trans is not None: chunks.append(trans) chunks.append(vowel_glide(pairs, per_vowel * len(ref), f0=hz)) return _overlap_add(chunks, xfade) @lru_cache(maxsize=256) def _note_audio(phoneme_str, note, n, pitch, formant): audio = render_note_syllable(phoneme_str, note, n / SR) audio.flags.writeable = False return audio def _fit_length(audio, n): if len(audio) < n: return np.concatenate([audio, np.zeros(n - len(audio))]) if len(audio) > n: out = audio[:n].copy() k = min(int(0.01 * SR), n) out[-k:] *= np.linspace(1, 0, k) return out return audio STRESS_BOOST = 1.35 def _words(seq, translate, seg_gap, word_gap): words = seq.split('|') out = [] for wi, group in enumerate(words): phones = group.split() stressed = bool(phones) and phones[-1] == '!' if stressed: phones.pop() parts = _render_phones(phones, seg_gap, translate=translate) audio = np.concatenate(parts) if parts else silence(0) out.append(audio * STRESS_BOOST if stressed else audio) if wi < len(words) - 1: out.append(silence(word_gap)) return out def _notes(seq, bpm, note_gap): out = [] for phoneme_str, note, beats in seq: slot = max(1, int(round(beats * 60.0 / bpm * SR))) if phoneme_str is None: out.append(np.zeros(slot)) continue stressed = phoneme_str.endswith('!') ps = phoneme_str[:-1] if stressed else phoneme_str gap = min(int(note_gap * SR), slot // 2) audio = _note_audio(ps, note, slot - gap, _VOICE['pitch'], _VOICE['formant']) if stressed: audio = audio * STRESS_BOOST out += [_fit_length(audio, slot - gap), np.zeros(gap)] return out @contextmanager def _settings(voice, quiet): global _quiet prev_voice, prev_quiet = dict(_VOICE), _quiet try: if voice is not None: set_voice(voice) _quiet = quiet yield finally: _VOICE.update(prev_voice) _quiet = prev_quiet def synthesize(seq, filename='output.wav', mode='ipa', seg_gap=0.02, word_gap=0.15, bpm=100, note_gap=0.01, voice=None, quiet=False): if mode not in ('ipa', 'ascii', 'vocaloid'): raise ValueError("mode must be 'ipa', 'ascii', or 'vocaloid'") with _settings(voice, quiet): if mode == 'vocaloid': segments = _notes(seq, bpm, note_gap) else: segments = _words(seq, ASCII if mode == 'ascii' else None, seg_gap, word_gap) audio = np.concatenate(segments) if segments else np.zeros(1) peak = np.max(np.abs(audio)) if peak > 0: audio = audio / peak * 0.85 wavfile.write(filename, SR, (audio * 32767).astype(np.int16)) if not quiet: print(f"wrote {filename}") return len(audio) / SR if __name__ == '__main__': synthesize("o uo | n o uo", "demo.wav", mode='ascii')