phonon:40m-multilingual

10 2 hours ago

Phonon turns written text into spoken audio

... /
json
e312ab694d5e ยท 4.7kB
{
"languages": [
"en",
"fr",
"de",
"pt",
"es"
],
"embeddings_ext": [
"ba7a5a45@300"
],
"flow_lm": {
"d_model": 512,
"num_heads": 8,
"num_layers": 9,
"dim_feedforward": 2048,
"max_period": 10000.0,
"n_bins": 4000,
"lut_dim": 512,
"flow_dim": 512,
"flow_depth": 6,
"ldim": 32,
"time_rms_norm": "rms",
"gelu": "erf"
},
"mimi": {
"channels": 1,
"sample_rate": 24000,
"frame_rate": 12.5,
"dimension": 512,
"quantizer_dimension": 32,
"quantizer_output_dimension": 512,
"n_filters": 64,
"n_residual_layers": 1,
"ratios": [
6,
5,
4
],
"kernel_size": 7,
"last_kernel_size": 3,
"residual_kernel_size": 3,
"dilation_base": 2,
"compress": 2,
"transformer_d_model": 512,
"transformer_num_heads": 8,
"transformer_num_layers": 2,
"transformer_layer_scale": 0.01,
"transformer_context": 250,
"transformer_max_period": 10000.0,
"transformer_dim_feedforward": 2048,
"gelu": "erf"
},
"temp": 0.8,
"lsd_decode_steps": 1,
"eos_threshold": 4.0,
"model_id": {
"sig": "796254a3",
"epoch": 100,
"mimi_sig": "b77c7ffa",
"mimi_epoch": 150
},
"weights_name": "model.safetensors",
"tokenizer_name": "tokenizer.model",
"conditioners": [
{
"type": "lut",
"lut": {
"n_bins": 31,
"dim": 16,
"tokenizer": "noop",
"possible_values": [
"0",
"1",
"2"
]
},
"name": "num_speakers"
},
{
"type": "continuous",
"continuous": {
"scale_factor": 1000.0,
"dim": 128,
"max_period": 10000.0
},
"name": "duration_delta"
}
],
"fuser": {
"cross_attention_pos_emb": false,
"cross_attention_pos_emb_scale": 1,
"sum": [
"num_speakers",
"duration_delta"
],
"streaming_sum": [],
"prepend": [
"transcript_in_segment",
"speaker_wavs"
],
"cross": [],
"pad_prepend": false
},
"speaker_mimi": {
"prefix": "mimi_speaker",
"mimi": {
"channels": 1,
"sample_rate": 24000,
"frame_rate": 12.5,
"dimension": 512,
"quantizer_dimension": 256,
"quantizer_output_dimension": 512,
"n_filters": 64,
"n_residual_layers": 1,
"ratios": [
8,
6,
5,
4
],
"kernel_size": 7,
"last_kernel_size": 3,
"residual_kernel_size": 3,
"dilation_base": 2,
"compress": 2,
"transformer_d_model": 512,
"transformer_num_heads": 8,
"transformer_num_layers": 8,
"transformer_layer_scale": 0.01,
"transformer_context": 250,
"transformer_max_period": 10000.0,
"transformer_dim_feedforward": 2048,
"gelu": "erf"
}
},
"audio_prompt_min_duration": 2.5,
"architectures": [
"GradiumPhononTTSForConditionalGeneration"
],
"attention_bias": false,
"attention_dropout": 0.0,
"audio_bos_token_id": 4,
"audio_latent_token_id": 6,
"audio_prompt_dim": 512,
"audio_prompt_token_id": 5,
"bos_token_id": null,
"dtype": "bfloat16",
"duration_delta": 0.0,
"duration_delta_embedding_dim": 128,
"duration_delta_max_period": 10000.0,
"duration_delta_scale": 1000.0,
"eos_token_id": null,
"flow_head_config": {
"_name_or_path": "",
"architectures": null,
"chunk_size_feed_forward": 0,
"condition_dim": 512,
"dtype": null,
"flow_type": "lsd",
"frequency_embedding_size": 256,
"hidden_act": "silu",
"hidden_size": 512,
"id2label": {
"0": "LABEL_0",
"1": "LABEL_1"
},
"is_encoder_decoder": false,
"label2id": {
"LABEL_0": 0,
"LABEL_1": 1
},
"latent_dim": 32,
"layer_norm_eps": 1e-06,
"max_period": 10000.0,
"model_type": "",
"num_hidden_layers": 6,
"output_attentions": false,
"output_hidden_states": false,
"problem_type": null,
"return_dict": true,
"rms_norm_eps": 1e-05
},
"frame_rate": 12.5,
"head_dim": 64,
"hidden_act": "gelu",
"hidden_size": 512,
"initializer_range": 0.02,
"intermediate_size": 2048,
"latent_dim": 32,
"layer_norm_eps": 1e-05,
"max_position_embeddings": 8192,
"model_type": "gradium_phonon_tts",
"num_attention_heads": 8,
"num_hidden_layers": 9,
"num_key_value_heads": 8,
"num_speakers": 1,
"num_speakers_embedding_dim": 16,
"num_speakers_vocab_size": 32,
"pad_token_id": 3,
"rope_parameters": {
"rope_theta": 10000.0,
"rope_type": "default"
},
"transformers_version": "5.21.0.dev0",
"use_cache": true,
"vocab_size": 4001
}