Music & Speech Detection

curl -sS -X POST https://platform.modulate.ai/api/velma-2-music-detection-batch \
  -H "X-API-Key: $MODULATE_API_KEY" \
  -F "upload_file=@audio.mp3" \
| jq -r 'if .detail then "error: \(.detail)" else
    "\(.primary_label): music \(.music_pct)%, speech \(.speech_pct)%",
    "\(.duration_s)s, \(.frames | length) frames of 192 ms",
    (.frames[:3][]
      | "  \(.start_time_ms)-\(.end_time_ms)ms  music=\(.music_prob)  speech=\(.speech_prob)")
  end'

import os, requests

response = requests.post(
    "https://platform.modulate.ai/api/velma-2-music-detection-batch",
    headers={"X-API-Key": os.environ["MODULATE_API_KEY"]},
    files={"upload_file": open("audio.mp3", "rb")},
)
response.raise_for_status()
result = response.json()

print(f"{result['primary_label']}: music {result['music_pct']}%, speech {result['speech_pct']}%")
print(f"{result['duration_s']}s, {len(result['frames'])} frames of 192 ms")

# frames covers the whole clip; the first three are enough to see the shape
for f in result["frames"][:3]:
    print(f"  {f['start_time_ms']}-{f['end_time_ms']}ms  "
          f"music={f['music_prob']}  speech={f['speech_prob']}")

import fs from "fs";
import FormData from "form-data";

const form = new FormData();
form.append("upload_file", fs.createReadStream("audio.mp3"), { filename: "audio.mp3" });

const response = await fetch(
  "https://platform.modulate.ai/api/velma-2-music-detection-batch",
  {
    method: "POST",
    headers: { "X-API-Key": process.env.MODULATE_API_KEY, ...form.getHeaders() },
    body: form,
  }
);

const result = await response.json();

if (!response.ok) {
  console.error(`error: ${result.detail}`);
  process.exit(1);
}

console.log(`${result.primary_label}: music ${result.music_pct}%, speech ${result.speech_pct}%`);
console.log(`${result.duration_s}s, ${result.frames.length} frames of 192 ms`);

// frames covers the whole clip; the first three are enough to see the shape
for (const f of result.frames.slice(0, 3)) {
  console.log(`  ${f.start_time_ms}-${f.end_time_ms}ms  music=${f.music_prob}  speech=${f.speech_prob}`);
}

import os, asyncio, json, websockets

API_KEY = os.environ["MODULATE_API_KEY"]
AUDIO_FILE = "audio.mp3"
CHUNK_SIZE = 65536

async def stream():
    url = (
        f"wss://platform.modulate.ai/api/velma-2-music-detection-streaming"
        f"?api_key={API_KEY}&audio_format=mp3"
    )
    async with websockets.connect(url) as ws:
        async def send():
            with open(AUDIO_FILE, "rb") as f:
                while chunk := f.read(CHUNK_SIZE):
                    await ws.send(chunk)
            await ws.send("")  # end of stream

        async def receive():
            async for message in ws:
                msg = json.loads(message)
                if msg["type"] == "frame":
                    f = msg["frame"]
                    print(f"  {f['start_time_ms']}-{f['end_time_ms']}ms  "
                          f"music={f['music_prob']}  speech={f['speech_prob']}")
                elif msg["type"] == "done":
                    print(f"done: {msg['duration_ms']}ms of audio, {msg['frame_count']} frames")
                    print(f"{msg['primary_label']}: music {msg['music_pct']}%, "
                          f"speech {msg['speech_pct']}%")
                    break
                elif msg["type"] == "error":
                    raise RuntimeError(msg["error"])

        await asyncio.gather(send(), receive())

asyncio.run(stream())

import { WebSocket } from "ws";
import { createReadStream } from "fs";

const url =
  `wss://platform.modulate.ai/api/velma-2-music-detection-streaming` +
  `?api_key=${process.env.MODULATE_API_KEY}&audio_format=mp3`;

const ws = new WebSocket(url);

ws.on("open", () => {
  const stream = createReadStream("audio.mp3", { highWaterMark: 65536 });
  stream.on("data", (chunk) => ws.send(chunk));
  stream.on("end", () => ws.send(""));
});

ws.on("message", (data) => {
  const msg = JSON.parse(data);
  if (msg.type === "frame") {
    const f = msg.frame;
    console.log(`  ${f.start_time_ms}-${f.end_time_ms}ms  music=${f.music_prob}  speech=${f.speech_prob}`);
  } else if (msg.type === "done") {
    console.log(`done: ${msg.duration_ms}ms of audio, ${msg.frame_count} frames`);
    console.log(`${msg.primary_label}: music ${msg.music_pct}%, speech ${msg.speech_pct}%`);
    ws.close();
  } else if (msg.type === "error") {
    console.error(`error: ${msg.error}`);
  }
});

// A rejected handshake closes without sending an error frame first.
ws.on("close", (code) => {
  if (code !== 1000) console.error(`error: connection closed ${code}`);
});
ws.on("error", (err) => console.error(`error: ${err.message}`));