embeddinggemma-2:latest

4 12 hours ago

EmbeddingGemma 2 is a multimodal embedding model from Google built on the Gemma 4 architecture.

vision embedding audio 270m 440m 570m 740m
... /
json
f440a5e27d63 · 5.3kB
{
"architectures": [
"EmbeddingGemma2Model"
],
"audio_config": {
"_name_or_path": "",
"architectures": null,
"attention_chunk_size": 12,
"attention_context_left": 13,
"attention_context_right": 0,
"attention_invalid_logits_value": -1000000000.0,
"attention_logit_cap": 50.0,
"chunk_size_feed_forward": 0,
"conv_kernel_size": 5,
"dtype": "bfloat16",
"gradient_clipping": 10000000000.0,
"hidden_act": "silu",
"hidden_size": 1024,
"id2label": {
"0": "LABEL_0",
"1": "LABEL_1"
},
"initializer_range": 0.02,
"is_encoder_decoder": false,
"label2id": {
"LABEL_0": 0,
"LABEL_1": 1
},
"model_type": "gemma4_audio",
"num_attention_heads": 8,
"num_hidden_layers": 12,
"output_attentions": false,
"output_hidden_states": false,
"output_proj_dims": 1536,
"problem_type": null,
"residual_weight": 0.5,
"return_dict": true,
"rms_norm_eps": 1e-06,
"subsampling_conv_channels": [
128,
32
],
"use_clipped_linears": true
},
"audio_token_id": 258881,
"boa_token_id": 256000,
"boi_token_id": 255999,
"dtype": "bfloat16",
"eoa_token_index": 258883,
"eoi_token_id": 258882,
"image_token_id": 258880,
"initializer_range": 0.02,
"model_type": "embedding_gemma2",
"text_config": {
"attention_bias": false,
"attention_dropout": 0.0,
"bos_token_id": 2,
"dtype": "bfloat16",
"embedding_dim": 768,
"eos_token_id": 1,
"head_dim": 256,
"hidden_activation": "gelu_pytorch_tanh",
"hidden_size": 512,
"hidden_size_per_layer_input": 512,
"initializer_range": 0.02,
"intermediate_size": 2048,
"layer_types": [
"sliding_attention",
"sliding_attention",
"sliding_attention",
"sliding_attention",
"sliding_attention",
"full_attention",
"sliding_attention",
"sliding_attention",
"sliding_attention",
"sliding_attention",
"sliding_attention",
"full_attention",
"sliding_attention",
"sliding_attention",
"sliding_attention",
"sliding_attention",
"sliding_attention",
"full_attention",
"sliding_attention",
"sliding_attention",
"sliding_attention",
"sliding_attention",
"sliding_attention",
"full_attention"
],
"max_position_embeddings": 262144,
"model_type": "embedding_gemma2_text",
"num_attention_heads": 4,
"num_hidden_layers": 24,
"num_key_value_heads": 2,
"pad_token_id": 0,
"per_layer_config": {
"05": {
"head_dim": 512,
"num_key_value_heads": 1
},
"11": {
"head_dim": 512,
"num_key_value_heads": 1
},
"17": {
"head_dim": 512,
"num_key_value_heads": 1
},
"23": {
"head_dim": 512,
"num_key_value_heads": 1
}
},
"rms_norm_eps": 1e-06,
"rope_parameters": {
"full_attention": {
"rope_theta": 1000000.0,
"rope_type": "default"
},
"sliding_attention": {
"rope_theta": 10000.0,
"rope_type": "default"
}
},
"sliding_window": 512,
"vocab_size": 262144
},
"transformers_version": "5.18.0.dev0",
"video_token_id": 258884,
"vision_config": {
"_name_or_path": "",
"architectures": null,
"attention_bias": false,
"attention_dropout": 0.0,
"chunk_size_feed_forward": 0,
"default_output_length": 280,
"dtype": "bfloat16",
"global_head_dim": 64,
"head_dim": 64,
"hidden_activation": "gelu_pytorch_tanh",
"hidden_size": 768,
"id2label": {
"0": "LABEL_0",
"1": "LABEL_1"
},
"initializer_range": 0.02,
"intermediate_size": 3072,
"is_encoder_decoder": false,
"label2id": {
"LABEL_0": 0,
"LABEL_1": 1
},
"max_position_embeddings": 131072,
"model_type": "gemma4_vision",
"num_attention_heads": 12,
"num_hidden_layers": 16,
"num_key_value_heads": 12,
"output_attentions": false,
"output_hidden_states": false,
"patch_size": 16,
"pooling_kernel_size": 3,
"position_embedding_size": 10240,
"problem_type": null,
"return_dict": true,
"rms_norm_eps": 1e-06,
"rope_parameters": {
"rope_theta": 100.0,
"rope_type": "axial"
},
"standardize": false,
"use_clipped_linears": false
},
"vision_soft_tokens_per_image": 280,
"quantization_config": {
"config_groups": {
"group_0": {
"input_activations": {
"dynamic": false,
"num_bits": 4,
"type": "float",
"group_size": 16
},
"weights": {
"dynamic": false,
"num_bits": 4,
"type": "float",
"group_size": 16
},
"targets": [
"Linear"
]
}
},
"ignore": [
"audio_tower*",
"embed_audio*",
"embed_vision*",
"language_model.embedding_projection",
"vision_tower*"
],
"quant_algo": "NVFP4",
"kv_cache_scheme": {
"dynamic": false,
"num_bits": 8,
"type": "float"
},
"producer": {
"name": "modelopt",
"version": "0.0.1.dev1+ge5b63320a"
},
"quant_method": "modelopt"
}
}