Luigi commited on
Commit
55bcc77
·
1 Parent(s): 06a6552

Add MOSS-TTS-Nano 2nd backend (rs.cpp + codec.cpp, ORT-free) with voice clone

Browse files

Second TTS engine alongside the MB-iSTFT-VITS: MOSS-TTS-Nano-100M autoregressive
LLM-TTS, fully ggml (no onnxruntime). Pipeline: SentencePiece tokenizer +
manifest-driven prompt + rs.cpp moss-gen (AR, 16 codebooks) + codec.cpp
(moss_audio codec) -> 48kHz. User can upload a reference clip to clone its voice.

- moss_backend.py: self-contained streaming backend (graceful-degrades to VITS-only)
- server.py: /has_moss, /moss_ref (ref upload), /moss_ws (48kHz stream); startup
asset download (gguf + tokenizer + manifest + hans00 codec gguf)
- static/index.html: engine selector + ref-audio upload + dynamic playback SR
- Dockerfile: builds moss-gen + codec-cli (shared ggml, CPU) in-image
- requirements: sentencepiece, librosa, gguf

RTF ~0.7 on 2 vCPU = real-time on the free CPU tier.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Files changed (6) hide show
  1. Dockerfile +20 -3
  2. MOSS_DEPLOY.md +68 -0
  3. moss_backend.py +136 -0
  4. requirements.txt +3 -0
  5. server.py +97 -1
  6. static/index.html +45 -7
Dockerfile CHANGED
@@ -1,13 +1,30 @@
1
- # PrimeTTS real-time streaming demo — FastAPI + WebSocket + Web Audio (ONNX, torch-free).
 
2
  FROM python:3.10-slim
3
  ENV DEBIAN_FRONTEND=noninteractive
4
  RUN apt-get update && apt-get install -y --no-install-recommends \
5
- ca-certificates libgomp1 && rm -rf /var/lib/apt/lists/*
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6
  COPY requirements.txt /app/requirements.txt
7
  RUN pip install --no-cache-dir -r /app/requirements.txt
8
  RUN useradd -m -u 1000 user
9
  ENV HOME=/home/user HF_HOME=/home/user/.cache/huggingface \
10
- G2PW_CACHE=/home/user/.cache/g2pw NUMBA_CACHE_DIR=/tmp/numba
 
 
11
  WORKDIR /app
12
  COPY . /app
13
  RUN cd /app && python -c "from g2pw_min.api import download_model; download_model('G2PWModel')" && \
 
1
+ # PrimeTTS real-time streaming demo — FastAPI + WebSocket + Web Audio (torch-free).
2
+ # Two TTS backends: MB-iSTFT-VITS (ONNX phoneme) + MOSS-TTS-Nano (rs.cpp+codec.cpp, ggml).
3
  FROM python:3.10-slim
4
  ENV DEBIAN_FRONTEND=noninteractive
5
  RUN apt-get update && apt-get install -y --no-install-recommends \
6
+ ca-certificates libgomp1 build-essential cmake git && rm -rf /var/lib/apt/lists/*
7
+
8
+ # ---- MOSS ggml binaries: codec-cli (codec.cpp) + moss-gen (fork tool), sharing one ggml ----
9
+ RUN git clone --recursive --depth 1 https://github.com/mybigday/codec.cpp /tmp/cc && \
10
+ cd /tmp/cc && cmake -B build -DGGML_CUDA=OFF -DCMAKE_BUILD_TYPE=Release && \
11
+ cmake --build build --target codec-cli -j"$(nproc)" && \
12
+ mkdir -p /app/bin && cp build/codec-cli /app/bin/ && \
13
+ (cp build/libcodec.so /app/bin/ 2>/dev/null || true) && \
14
+ find build -name 'libggml*.so' -exec cp {} /app/bin/ \;
15
+ # moss-gen: compile the fork's standalone tool against codec.cpp's ggml (no rapidspeech-core)
16
+ RUN git clone --depth 1 -b jetson-nano-gen1 https://github.com/vieenrose/RapidSpeech.cpp /tmp/rs && \
17
+ g++ -O3 -std=c++17 -I /tmp/cc/ggml/include /tmp/rs/tools/moss_gen.cpp -o /app/bin/moss-gen \
18
+ -L /app/bin -lggml -lggml-base -lggml-cpu && \
19
+ rm -rf /tmp/cc /tmp/rs
20
+
21
  COPY requirements.txt /app/requirements.txt
22
  RUN pip install --no-cache-dir -r /app/requirements.txt
23
  RUN useradd -m -u 1000 user
24
  ENV HOME=/home/user HF_HOME=/home/user/.cache/huggingface \
25
+ G2PW_CACHE=/home/user/.cache/g2pw NUMBA_CACHE_DIR=/tmp/numba \
26
+ LD_LIBRARY_PATH=/app/bin MOSS_GEN=/app/bin/moss-gen MOSS_CODEC_CLI=/app/bin/codec-cli \
27
+ MOSS_DIR=/home/user/moss MOSS_THREADS=2
28
  WORKDIR /app
29
  COPY . /app
30
  RUN cd /app && python -c "from g2pw_min.api import download_model; download_model('G2PWModel')" && \
MOSS_DEPLOY.md ADDED
@@ -0,0 +1,68 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Deploying the MOSS-TTS-Nano 2nd backend to the Space
2
+
3
+ The MOSS backend adds a text→codec autoregressive TTS (rs.cpp CPU, 48 kHz, voice
4
+ clone) alongside the phoneme VITS. It **graceful-degrades**: if any asset is
5
+ missing, `MOSS.available = False` and the demo serves VITS only — so this is
6
+ safe to ship incrementally.
7
+
8
+ ## 1. Binary — build in-Space (glibc safety)
9
+ The dev-box binary is glibc 2.39; HF Spaces are older. **Build `moss-gen` in the
10
+ Space's own image** (Docker Space) so glibc matches. `Dockerfile`:
11
+
12
+ ```dockerfile
13
+ FROM python:3.11-slim
14
+ RUN apt-get update && apt-get install -y build-essential cmake git && rm -rf /var/lib/apt/lists/*
15
+ # build moss-gen + ggml (CPU) from the fork
16
+ RUN git clone --depth 1 -b jetson-nano-gen1 https://github.com/vieenrose/RapidSpeech.cpp /tmp/rs && \
17
+ cd /tmp/rs && cmake -B build -DGGML_CUDA=OFF -DCMAKE_BUILD_TYPE=Release && \
18
+ cmake --build build --target moss-gen -j && \
19
+ mkdir -p /app/bin && cp build/moss-gen /app/bin/ && cp build/ggml/src/libggml*.so /app/bin/
20
+ # codec.cpp (ggml codec-cli) — replaces the codec ONNX (ORT-free)
21
+ RUN git clone --recursive --depth 1 https://github.com/mybigday/codec.cpp /tmp/cc && \
22
+ cd /tmp/cc && cmake -B build -DGGML_CUDA=OFF -DCMAKE_BUILD_TYPE=Release && \
23
+ cmake --build build --target codec-cli -j && cp build/codec-cli /app/bin/ && cp build/libcodec.so /app/bin/ 2>/dev/null || true
24
+ ENV LD_LIBRARY_PATH=/app/bin
25
+ WORKDIR /app
26
+ COPY requirements.txt .
27
+ RUN pip install --no-cache-dir -r requirements.txt
28
+ COPY . .
29
+ ENV MOSS_GEN=/app/bin/moss-gen
30
+ CMD ["uvicorn", "server:app", "--host", "0.0.0.0", "--port", "7860"]
31
+ ```
32
+
33
+ (If the fork target `moss-gen` isn't in the default CMake, it is guarded by the
34
+ same `tools/` block as the other moss tools — ensure `-DRS_BUILD_TOOLS=ON` or the
35
+ equivalent, or compile `tools/moss_gen.cpp` standalone against the built libs.)
36
+
37
+ ## 2. Assets — host in an HF model repo, download at startup
38
+ Per repo policy, weights live in a **model repo**, not the Space. Already hosted:
39
+ `Luigi/MOSS-TTS-Nano-100M-GGUF` → `moss_nano_full.gguf`. Also needed (add to that
40
+ repo or a `*-demo-assets` repo):
41
+ - codec gguf: `hans00/MOSS-TTS-Nano-GGUF` `codec-q8_0.gguf` (used by codec.cpp — no ONNX)
42
+ - MOSS **tokenizer**: `tokenizer.model`, `tokenization_moss_tts_nano.py`, `config.json`,
43
+ `special_tokens_map.json`, `tokenizer_config.json`
44
+ - (embedding tables are **derived from the gguf at startup** — no hosting needed)
45
+
46
+ `server.py` downloads them at startup into `MOSS_DIR` (mirror the existing
47
+ `hf_hub_download` pattern used for the VITS/LLM weights).
48
+
49
+ ## 3. Support files
50
+ None — `moss_backend.py` is self-contained (SentencePiece tokenizer + manifest-driven
51
+ prompt build + `moss-gen` + `codec-cli`). No `NanoRuntime` / `ort_cpu_runtime` / ONNX.
52
+
53
+ ## 4. requirements.txt — add
54
+ ```
55
+ sentencepiece
56
+ soundfile
57
+ librosa
58
+ gguf
59
+ ```
60
+
61
+ ## 5. Verify before pushing
62
+ ```
63
+ MOSS_DIR=... MOSS_GEN=.../moss-gen python -c "import server" # MOSS ready?
64
+ ```
65
+ Check `[moss] backend ready` in logs, then exercise `/has_moss`, `/moss_ref`
66
+ (upload a clip), `/moss_ws` (stream). RTF ~0.72 on 2 vCPU = real-time.
67
+ ```
68
+ ```
moss_backend.py ADDED
@@ -0,0 +1,136 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """MOSS-TTS-Nano-100M streaming backend — fully rs.cpp/ggml (ORT-free).
2
+
3
+ Pipeline (no onnxruntime anywhere):
4
+ tokenizer : SentencePiece (tokenizer.model)
5
+ prompt : build_voice_clone_request_rows (replicated from the manifest config)
6
+ AR gen : RapidSpeech.cpp `moss-gen` (ggml) — text+ref -> 16-codebook frames
7
+ codec : codec.cpp `codec-cli` (ggml, moss_audio gguf) — ref encode + frame decode
8
+
9
+ Assets (model repo): moss_nano_full.gguf (AR), codec gguf, tokenizer.model,
10
+ browser_poc_manifest.json. Graceful: `available=False` if a binary/asset is missing.
11
+ """
12
+ import os, json, subprocess, tempfile, threading
13
+ import numpy as np
14
+
15
+ SR_OUT = 48000
16
+
17
+ class MossBackend:
18
+ def __init__(self, moss_dir, gguf, moss_gen, codec_cli, codec_gguf,
19
+ chunk_ms=240, threads=2):
20
+ self.available = False
21
+ self._lock = threading.Lock()
22
+ self.ref_codes = {} # session -> [T,16] frame-major ref codes
23
+ self.chunk_bytes = int(SR_OUT * chunk_ms / 1000) * 2
24
+ try:
25
+ import sentencepiece as spm
26
+ self.moss_dir = os.path.abspath(moss_dir)
27
+ self.gguf = gguf if os.path.isabs(gguf) else os.path.join(self.moss_dir, gguf)
28
+ self.moss_gen, self.codec_cli, self.codec_gguf = moss_gen, codec_cli, codec_gguf
29
+ self.threads = threads
30
+ self.man = json.load(open(os.path.join(self.moss_dir, "browser_poc_manifest.json")))
31
+ self.tc = self.man["tts_config"]; self.pt = self.man["prompt_templates"]
32
+ self.NVQ = int(self.tc["n_vq"]); self.PAD = int(self.tc["audio_pad_token_id"])
33
+ self.sp = spm.SentencePieceProcessor(model_file=os.path.join(self.moss_dir, "tokenizer.model"))
34
+ self.H = 768
35
+ self.TOK, self.AUD = self._embed_tables()
36
+ for p in (self.moss_gen, self.codec_cli, self.codec_gguf):
37
+ if not os.path.exists(p): raise FileNotFoundError(p)
38
+ self.available = True
39
+ print("[moss] backend ready (rs.cpp+codec.cpp, ORT-free, 48kHz AR-TTS + clone)")
40
+ except Exception as e:
41
+ print(f"[moss] backend unavailable ({e}); VITS-only")
42
+
43
+ def _embed_tables(self):
44
+ tb = os.path.join(self.moss_dir, "gg_tok_embd.bin"); ab = os.path.join(self.moss_dir, "gg_audio_embd.bin")
45
+ if os.path.exists(tb) and os.path.exists(ab):
46
+ return (np.fromfile(tb, np.float32).reshape(-1, self.H),
47
+ np.fromfile(ab, np.float32).reshape(16, 1024, self.H))
48
+ from gguf import GGUFReader, GGMLQuantizationType
49
+ from gguf.quants import dequantize
50
+ r = GGUFReader(self.gguf)
51
+ def T(n):
52
+ t = next(x for x in r.tensors if x.name == n); a = np.array(t.data)
53
+ if t.tensor_type != GGMLQuantizationType.F32: a = dequantize(a, t.tensor_type)
54
+ return a.reshape(t.shape[::-1])
55
+ return (T("token_embd.weight").astype(np.float32),
56
+ np.stack([T(f"audio_embd.{k}.weight") for k in range(16)]).astype(np.float32))
57
+
58
+ # ---- prompt build (replicated from OrtCpuRuntime, config-driven, no ONNX) ----
59
+ def _text_rows(self, ids):
60
+ w = self.NVQ + 1
61
+ return [[t] + [self.PAD] * self.NVQ for t in ids] # row[0]=tok, rest pad
62
+ def _audio_rows(self, codes, slot):
63
+ rows = []
64
+ for cr in codes:
65
+ row = [self.PAD] * (self.NVQ + 1); row[0] = slot
66
+ for i in range(min(len(cr), self.NVQ)): row[i + 1] = int(cr[i])
67
+ rows.append(row)
68
+ return rows
69
+ def _prompt17(self, ref_codes, text_ids):
70
+ prefix = [*self.pt["user_prompt_prefix_token_ids"], int(self.tc["audio_start_token_id"])]
71
+ suffix = [int(self.tc["audio_end_token_id"]), *self.pt["user_prompt_after_reference_token_ids"],
72
+ *text_ids, *self.pt["assistant_prompt_prefix_token_ids"], int(self.tc["audio_start_token_id"])]
73
+ return (self._text_rows(prefix) + self._audio_rows(ref_codes, int(self.tc["audio_user_slot_token_id"]))
74
+ + self._text_rows(suffix))
75
+
76
+ def _prompt_embed(self, p17):
77
+ L = len(p17); emb = np.zeros((L, self.H), np.float32)
78
+ for p in range(L):
79
+ emb[p] = self.TOK[p17[p][0]]
80
+ for k in range(self.NVQ):
81
+ c = p17[p][k + 1]
82
+ if c != self.PAD and c < 1024: emb[p] += self.AUD[k][c]
83
+ return emb
84
+
85
+ def _codec_encode(self, wav_f32_48k):
86
+ import soundfile as sf
87
+ st = np.stack([wav_f32_48k, wav_f32_48k], axis=1).astype(np.float32) # stereo
88
+ with tempfile.TemporaryDirectory() as td:
89
+ w = os.path.join(td, "r.wav"); c = os.path.join(td, "c.npy")
90
+ sf.write(w, st, SR_OUT)
91
+ subprocess.run([self.codec_cli, "encode", "--model", self.codec_gguf, "--in", w,
92
+ "--out", c, "--nq", str(self.NVQ), "--threads", str(self.threads)],
93
+ check=True, stdout=subprocess.DEVNULL)
94
+ return np.load(c) # [16, T]
95
+
96
+ def _codec_decode(self, frames_N16):
97
+ import soundfile as sf
98
+ codes = np.ascontiguousarray(frames_N16.T.astype(np.int32)) # [16, N] C-order
99
+ with tempfile.TemporaryDirectory() as td:
100
+ c = os.path.join(td, "c.npy"); w = os.path.join(td, "o.wav")
101
+ np.save(c, codes)
102
+ subprocess.run([self.codec_cli, "decode", "--model", self.codec_gguf, "--codes", c,
103
+ "--out", w, "--nq", str(self.NVQ), "--threads", str(self.threads)],
104
+ check=True, stdout=subprocess.DEVNULL)
105
+ a, _ = sf.read(w)
106
+ return (a.mean(1) if getattr(a, "ndim", 1) > 1 else a).astype(np.float32)
107
+
108
+ # ---- voice clone ----
109
+ def set_reference(self, session, wav_f32_48k):
110
+ codes = self._codec_encode(wav_f32_48k) # [16,T]
111
+ self.ref_codes[session] = codes.T.tolist() # [T,16] frame-major
112
+ return len(self.ref_codes[session])
113
+ def clear_reference(self, session):
114
+ self.ref_codes.pop(session, None)
115
+
116
+ # ---- streaming synth (clause granularity: generate -> codec decode -> chunked emit) ----
117
+ def stream(self, text, emit, session_id=None, seed=7, sample=True, max_frames=400):
118
+ if not self.available: return
119
+ with self._lock:
120
+ ids = self.sp.encode(text)
121
+ ref = self.ref_codes.get(session_id) or [[self.PAD] * self.NVQ]
122
+ p17 = self._prompt17(ref, ids); L = len(p17)
123
+ emb = self._prompt_embed(p17)
124
+ with tempfile.TemporaryDirectory() as td:
125
+ pe = os.path.join(td, "pe.bin"); out = os.path.join(td, "fr.bin")
126
+ emb.astype(np.float32).tofile(pe)
127
+ subprocess.run([self.moss_gen, self.gguf, str(max_frames), str(seed),
128
+ "sample" if sample else "greedy", pe, str(L), out],
129
+ check=True, env=dict(os.environ, MOSS_DIR=self.moss_dir),
130
+ stdout=subprocess.DEVNULL)
131
+ frames = np.fromfile(out, np.int32).reshape(-1, 16)
132
+ if not len(frames): return
133
+ audio = self._codec_decode(frames)
134
+ pcm = (np.clip(audio, -1, 1) * 32767).astype("<i2").tobytes()
135
+ for i in range(0, len(pcm), self.chunk_bytes): # sub-chunk for smooth playback
136
+ if emit(pcm[i:i + self.chunk_bytes]) is False: break
requirements.txt CHANGED
@@ -13,3 +13,6 @@ inflect
13
  fastapi
14
  uvicorn[standard]
15
  llama-cpp-python
 
 
 
 
13
  fastapi
14
  uvicorn[standard]
15
  llama-cpp-python
16
+ sentencepiece
17
+ librosa
18
+ gguf
server.py CHANGED
@@ -11,7 +11,7 @@ falls back to the pure-onnxruntime split runner (bit-exact) otherwise.
11
  import os, json, struct, asyncio
12
  from pathlib import Path
13
 
14
- from fastapi import FastAPI, WebSocket, WebSocketDisconnect
15
  from fastapi.responses import HTMLResponse
16
  from fastapi.staticfiles import StaticFiles
17
  import numpy as np
@@ -111,6 +111,40 @@ def _pcm16(x):
111
 
112
  BACKEND = Backend()
113
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
114
  # ---- two preloaded gguf models: story (llama2.c-zh 15M) + chat (SmolLM-Chinese-180M) ----
115
  import time, random, textgen
116
  import opencc
@@ -221,6 +255,68 @@ async def healthz():
221
  return {"backend": BACKEND.kind, "sr": SR, "chunk_ms": CHUNK * HOP // (SR // 1000)}
222
 
223
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
224
  @app.websocket("/ws")
225
  async def ws(sock: WebSocket):
226
  await sock.accept()
 
11
  import os, json, struct, asyncio
12
  from pathlib import Path
13
 
14
+ from fastapi import FastAPI, WebSocket, WebSocketDisconnect, UploadFile, File
15
  from fastapi.responses import HTMLResponse
16
  from fastapi.staticfiles import StaticFiles
17
  import numpy as np
 
111
 
112
  BACKEND = Backend()
113
 
114
+ # ---- 2nd backend: MOSS-TTS-Nano-100M (rs.cpp CPU, 48kHz AR-TTS + voice clone) ----
115
+ # Assets (gguf + codec ONNX + tokenizer) live in the model repo, downloaded at startup.
116
+ MOSS_DIR = os.environ.get("MOSS_DIR", str(HERE / "moss"))
117
+ def _fetch_moss_assets():
118
+ if os.environ.get("MOSS_GEN") is None and not (HERE / "bin" / "moss-gen").exists():
119
+ return False # no binary -> skip download, backend will report unavailable
120
+ try:
121
+ from huggingface_hub import snapshot_download
122
+ snapshot_download(os.environ.get("MOSS_REPO", "Luigi/MOSS-TTS-Nano-100M-GGUF"),
123
+ local_dir=MOSS_DIR,
124
+ allow_patterns=["moss_nano_full.gguf", "tokenizer/*", "browser_poc_manifest.json"])
125
+ from huggingface_hub import hf_hub_download
126
+ os.environ.setdefault("MOSS_CODEC_GGUF",
127
+ hf_hub_download(os.environ.get("MOSS_CODEC_REPO", "hans00/MOSS-TTS-Nano-GGUF"),
128
+ os.environ.get("MOSS_CODEC_FILE", "codec-q8_0.gguf")))
129
+ # tokenizer files live under tokenizer/ in the repo — flatten into MOSS_DIR
130
+ import shutil
131
+ for f in ("tokenizer.model", "browser_poc_manifest.json"):
132
+ src = os.path.join(MOSS_DIR, "tokenizer", f)
133
+ if os.path.exists(src) and not os.path.exists(os.path.join(MOSS_DIR, f)):
134
+ shutil.copy(src, os.path.join(MOSS_DIR, f))
135
+ return True
136
+ except Exception as e:
137
+ print(f"[moss] asset download failed ({e})"); return False
138
+ _fetch_moss_assets()
139
+ from moss_backend import MossBackend, SR_OUT as MOSS_SR
140
+ MOSS = MossBackend(
141
+ moss_dir=MOSS_DIR,
142
+ gguf=os.environ.get("MOSS_GGUF", "moss_nano_full.gguf"),
143
+ moss_gen=os.environ.get("MOSS_GEN", str(HERE / "bin" / "moss-gen")),
144
+ codec_cli=os.environ.get("MOSS_CODEC_CLI", str(HERE / "bin" / "codec-cli")),
145
+ codec_gguf=os.environ.get("MOSS_CODEC_GGUF", str(MOSS_DIR + "/codec-q8_0.gguf")),
146
+ threads=int(os.environ.get("MOSS_THREADS", "2")))
147
+
148
  # ---- two preloaded gguf models: story (llama2.c-zh 15M) + chat (SmolLM-Chinese-180M) ----
149
  import time, random, textgen
150
  import opencc
 
255
  return {"backend": BACKEND.kind, "sr": SR, "chunk_ms": CHUNK * HOP // (SR // 1000)}
256
 
257
 
258
+ @app.get("/has_moss")
259
+ async def has_moss():
260
+ return {"available": MOSS.available, "sr": MOSS_SR}
261
+
262
+ @app.post("/moss_ref")
263
+ async def moss_ref(session: str = "", clear: int = 0, file: UploadFile = File(None)):
264
+ """Upload a reference clip to clone its voice (or clear=1 to reset)."""
265
+ if not MOSS.available:
266
+ return {"ok": False, "error": "moss backend unavailable"}
267
+ if clear or file is None:
268
+ MOSS.clear_reference(session)
269
+ return {"ok": True, "cleared": True}
270
+ import io, soundfile as sf
271
+ data = await file.read()
272
+ w, sr = sf.read(io.BytesIO(data))
273
+ if getattr(w, "ndim", 1) > 1:
274
+ w = w.mean(1)
275
+ w = w.astype(np.float32)
276
+ if sr != MOSS_SR:
277
+ import librosa
278
+ w = librosa.resample(w, orig_sr=sr, target_sr=MOSS_SR)
279
+ n = MOSS.set_reference(session, w)
280
+ return {"ok": True, "ref_frames": n}
281
+
282
+ @app.websocket("/moss_ws")
283
+ async def moss_ws(sock: WebSocket):
284
+ """Stream MOSS-Nano: client sends {text, session}. 48 kHz PCM chunks (~80 ms/frame)."""
285
+ await sock.accept()
286
+ loop = asyncio.get_event_loop()
287
+ try:
288
+ while True:
289
+ msg = json.loads(await sock.receive_text())
290
+ text = (msg.get("text") or "").strip()
291
+ if not text:
292
+ continue
293
+ session = str(msg.get("session", ""))
294
+ if not MOSS.available:
295
+ await sock.send_text(json.dumps({"type": "error", "msg": "moss unavailable"}))
296
+ continue
297
+ await sock.send_text(json.dumps({"type": "start", "sr": MOSS_SR}))
298
+ q: asyncio.Queue = asyncio.Queue()
299
+
300
+ def emit(pcm):
301
+ loop.call_soon_threadsafe(q.put_nowait, pcm)
302
+ return True
303
+
304
+ def work():
305
+ for cl in _split_clauses(text):
306
+ MOSS.stream(cl, emit, session_id=session)
307
+ loop.call_soon_threadsafe(q.put_nowait, None)
308
+
309
+ fut = loop.run_in_executor(None, work)
310
+ while True:
311
+ pcm = await q.get()
312
+ if pcm is None:
313
+ break
314
+ await sock.send_bytes(pcm)
315
+ await fut
316
+ await sock.send_text(json.dumps({"type": "end"}))
317
+ except WebSocketDisconnect:
318
+ pass
319
+
320
  @app.websocket("/ws")
321
  async def ws(sock: WebSocket):
322
  await sock.accept()
static/index.html CHANGED
@@ -47,9 +47,19 @@
47
  <div class="chips" id="chips"></div>
48
  <div class="row">
49
  <button id="go">▶ Speak (stream)</button>
50
- <label class="opt">voice
 
 
 
 
 
 
51
  <select id="voice" style="background:#0b0f16;color:var(--fg);border:1px solid #2a3342;border-radius:8px;padding:5px 8px;font-size:13px;"></select>
52
  </label>
 
 
 
 
53
  <label class="opt"><input type="checkbox" id="onenter" checked> stream on Enter</label>
54
  <span class="stat">first-audio <b id="first">–</b> · chunks <b id="nc">0</b> · <b id="audio">0.0s</b></span>
55
  <a id="dl" download="primetts.wav" style="display:none;color:var(--acc);font-size:13px;text-decoration:none;border:1px solid #2a3342;border-radius:8px;padding:5px 10px;">⬇ WAV</a>
@@ -98,15 +108,40 @@ fetch('/voices').then(r=>r.json()).then(j=>{
98
  }).catch(()=>{});
99
  const getSid = () => parseInt(($('voice')||{}).value || '0', 10);
100
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
101
  let ac, nextStart = 0;
102
  let curWs = null, srcs = []; // active socket + scheduled audio sources (for Stop)
103
- function ensureCtx(){ if(!ac) ac = new (window.AudioContext||window.webkitAudioContext)({sampleRate:SR}); if(ac.state==='suspended') ac.resume(); }
 
104
 
105
  function playPCM(buf){
106
  const i16 = new Int16Array(buf);
107
  const f32 = new Float32Array(i16.length);
108
  for (let i=0;i<i16.length;i++) f32[i] = i16[i] / 32768;
109
- const ab = ac.createBuffer(1, f32.length, SR);
110
  ab.copyToChannel(f32, 0);
111
  const src = ac.createBufferSource(); src.buffer = ab; src.connect(ac.destination);
112
  const now = ac.currentTime;
@@ -126,7 +161,7 @@ function capFinish(anchor){
126
  const wr = (o,s)=>{ for(let i=0;i<s.length;i++) dv.setUint8(o+i, s.charCodeAt(i)); };
127
  wr(0,'RIFF'); dv.setUint32(4, 36+capBytes, true); wr(8,'WAVE'); wr(12,'fmt ');
128
  dv.setUint32(16,16,true); dv.setUint16(20,1,true); dv.setUint16(22,1,true);
129
- dv.setUint32(24,SR,true); dv.setUint32(28,SR*2,true); dv.setUint16(32,2,true); dv.setUint16(34,16,true);
130
  wr(36,'data'); dv.setUint32(40, capBytes, true);
131
  let off=44; for(const c of capChunks){ wav.set(c, off); off+=c.length; }
132
  const url = URL.createObjectURL(new Blob([wav], {type:'audio/wav'}));
@@ -152,19 +187,22 @@ function speak(){
152
  capReset($('dl'));
153
  const t0 = performance.now(); let first=null, n=0, samples=0;
154
  const proto = location.protocol==='https:' ? 'wss':'ws';
155
- const ws = new WebSocket(`${proto}://${location.host}/ws`); curWs = ws;
 
 
156
  ws.binaryType = 'arraybuffer';
157
  const done = () => { setBtn($('go'), false, '▶ Speak (stream)'); if(curWs===ws) curWs=null; };
158
- ws.onopen = () => ws.send(JSON.stringify({text, sid: getSid()}));
159
  ws.onmessage = ev => {
160
  if (typeof ev.data === 'string'){ const m=JSON.parse(ev.data);
 
161
  if(m.type==='end'){ ws.close(); }
162
  if(m.type==='error'){ $('first').textContent='—'; ws.close(); }
163
  return; }
164
  if(first===null){ first=performance.now()-t0; $('first').textContent=first.toFixed(0)+' ms'; }
165
  playPCM(ev.data); capPush(ev.data);
166
  n++; samples += ev.data.byteLength/2;
167
- $('nc').textContent=n; $('audio').textContent=(samples/SR).toFixed(1)+'s';
168
  $('bar').style.width = Math.min(100, n*8)+'%';
169
  };
170
  ws.onclose = () => { done(); $('bar').style.width='100%'; capFinish($('dl')); };
 
47
  <div class="chips" id="chips"></div>
48
  <div class="row">
49
  <button id="go">▶ Speak (stream)</button>
50
+ <label class="opt">engine
51
+ <select id="engine" style="background:#0b0f16;color:var(--fg);border:1px solid #2a3342;border-radius:8px;padding:5px 8px;font-size:13px;">
52
+ <option value="vits">PrimeTTS (VITS 16k)</option>
53
+ <option value="moss">MOSS-Nano (clone 48k)</option>
54
+ </select>
55
+ </label>
56
+ <label class="opt" id="voicewrap">voice
57
  <select id="voice" style="background:#0b0f16;color:var(--fg);border:1px solid #2a3342;border-radius:8px;padding:5px 8px;font-size:13px;"></select>
58
  </label>
59
+ <label class="opt" id="refwrap" style="display:none;">🎙 ref
60
+ <input type="file" id="ref" accept="audio/*" style="font-size:12px;max-width:150px;">
61
+ <span id="refstat" style="font-size:12px;color:var(--acc);"></span>
62
+ </label>
63
  <label class="opt"><input type="checkbox" id="onenter" checked> stream on Enter</label>
64
  <span class="stat">first-audio <b id="first">–</b> · chunks <b id="nc">0</b> · <b id="audio">0.0s</b></span>
65
  <a id="dl" download="primetts.wav" style="display:none;color:var(--acc);font-size:13px;text-decoration:none;border:1px solid #2a3342;border-radius:8px;padding:5px 10px;">⬇ WAV</a>
 
108
  }).catch(()=>{});
109
  const getSid = () => parseInt(($('voice')||{}).value || '0', 10);
110
 
111
+ // ---- engine selector (VITS / MOSS-Nano voice clone) ----
112
+ const SESSION = 'moss-' + Math.random().toString(36).slice(2);
113
+ fetch('/has_moss').then(r=>r.json()).then(j=>{
114
+ if(!j || !j.available){ const e=$('engine'); if(e){ [...e.options].forEach(o=>{ if(o.value==='moss') o.disabled=true; }); } }
115
+ }).catch(()=>{});
116
+ function syncEngine(){
117
+ const moss = ($('engine')||{}).value === 'moss';
118
+ if($('refwrap')) $('refwrap').style.display = moss ? '' : 'none';
119
+ if($('voicewrap')) $('voicewrap').style.display = moss ? 'none' : '';
120
+ }
121
+ if($('engine')) $('engine').addEventListener('change', syncEngine);
122
+ syncEngine();
123
+ // upload a reference clip -> clone its voice for this session
124
+ if($('ref')) $('ref').addEventListener('change', async e=>{
125
+ const f = e.target.files[0]; if(!f) return;
126
+ $('refstat').textContent = 'uploading…';
127
+ const fd = new FormData(); fd.append('file', f);
128
+ try{
129
+ const r = await fetch('/moss_ref?session='+encodeURIComponent(SESSION), {method:'POST', body:fd});
130
+ const j = await r.json();
131
+ $('refstat').textContent = j.ok ? `✓ cloned (${j.ref_frames} frames)` : ('✗ ' + (j.error||'failed'));
132
+ }catch(err){ $('refstat').textContent = '✗ error'; }
133
+ });
134
+
135
  let ac, nextStart = 0;
136
  let curWs = null, srcs = []; // active socket + scheduled audio sources (for Stop)
137
+ let curSR = SR; // playback SR of the active stream (16k VITS / 48k MOSS)
138
+ function ensureCtx(){ if(!ac) ac = new (window.AudioContext||window.webkitAudioContext)(); if(ac.state==='suspended') ac.resume(); }
139
 
140
  function playPCM(buf){
141
  const i16 = new Int16Array(buf);
142
  const f32 = new Float32Array(i16.length);
143
  for (let i=0;i<i16.length;i++) f32[i] = i16[i] / 32768;
144
+ const ab = ac.createBuffer(1, f32.length, curSR); // browser resamples to ctx rate
145
  ab.copyToChannel(f32, 0);
146
  const src = ac.createBufferSource(); src.buffer = ab; src.connect(ac.destination);
147
  const now = ac.currentTime;
 
161
  const wr = (o,s)=>{ for(let i=0;i<s.length;i++) dv.setUint8(o+i, s.charCodeAt(i)); };
162
  wr(0,'RIFF'); dv.setUint32(4, 36+capBytes, true); wr(8,'WAVE'); wr(12,'fmt ');
163
  dv.setUint32(16,16,true); dv.setUint16(20,1,true); dv.setUint16(22,1,true);
164
+ dv.setUint32(24,curSR,true); dv.setUint32(28,curSR*2,true); dv.setUint16(32,2,true); dv.setUint16(34,16,true);
165
  wr(36,'data'); dv.setUint32(40, capBytes, true);
166
  let off=44; for(const c of capChunks){ wav.set(c, off); off+=c.length; }
167
  const url = URL.createObjectURL(new Blob([wav], {type:'audio/wav'}));
 
187
  capReset($('dl'));
188
  const t0 = performance.now(); let first=null, n=0, samples=0;
189
  const proto = location.protocol==='https:' ? 'wss':'ws';
190
+ const moss = ($('engine')||{}).value === 'moss';
191
+ curSR = moss ? 48000 : SR; // provisional; start msg confirms
192
+ const ws = new WebSocket(`${proto}://${location.host}/${moss ? 'moss_ws' : 'ws'}`); curWs = ws;
193
  ws.binaryType = 'arraybuffer';
194
  const done = () => { setBtn($('go'), false, '▶ Speak (stream)'); if(curWs===ws) curWs=null; };
195
+ ws.onopen = () => ws.send(JSON.stringify(moss ? {text, session: SESSION} : {text, sid: getSid()}));
196
  ws.onmessage = ev => {
197
  if (typeof ev.data === 'string'){ const m=JSON.parse(ev.data);
198
+ if(m.type==='start' && m.sr){ curSR = m.sr; }
199
  if(m.type==='end'){ ws.close(); }
200
  if(m.type==='error'){ $('first').textContent='—'; ws.close(); }
201
  return; }
202
  if(first===null){ first=performance.now()-t0; $('first').textContent=first.toFixed(0)+' ms'; }
203
  playPCM(ev.data); capPush(ev.data);
204
  n++; samples += ev.data.byteLength/2;
205
+ $('nc').textContent=n; $('audio').textContent=(samples/curSR).toFixed(1)+'s';
206
  $('bar').style.width = Math.min(100, n*8)+'%';
207
  };
208
  ws.onclose = () => { done(); $('bar').style.width='100%'; capFinish($('dl')); };