bert

type Attention struct {
	Query     *nn.Linear    `gguf:"attn_q"`
	QueryNorm *nn.LayerNorm `gguf:"attn_q_norm"`

	Key     *nn.Linear    `gguf:"attn_k"`
	KeyNorm *nn.LayerNorm `gguf:"attn_k_norm"`

	Value *nn.Linear `gguf:"attn_v"`

	Output *nn.Linear `gguf:"attn_output"`
}

type EncoderLayer struct {
	*Attention
	AttentionNorm *nn.LayerNorm `gguf:"attn_output_norm"`

	*MLP
	MLPNorm *nn.LayerNorm `gguf:"layer_output_norm"`
}

type MLP struct {
	Up   *nn.Linear `gguf:"ffn_up"`
	Down *nn.Linear `gguf:"ffn_down"`
}

type Model struct {
	model.Base
	model.TextProcessor

	TokenEmbedding     *nn.Embedding `gguf:"token_embd"`
	TypeEmbedding      *nn.Embedding `gguf:"token_types"`
	PositionEmbedding  *nn.Embedding `gguf:"position_embd"`
	TokenEmbeddingNorm *nn.LayerNorm `gguf:"token_embd_norm"`

	Layers []EncoderLayer `gguf:"blk"`

	Options
}

type Options struct {
	// contains filtered or unexported fields
}