sleepy/llama.cpp
1
0
Fork 0
Code Issues 12 Pull Requests Actions 217 Packages Projects Releases Wiki Activity

models : fix qwen3.5 beta/gate shapes (#19730)

Browse Source 
* models : fix qwen3.5 beta/gate shapes

* cont : avoid extra reshapes
This commit is contained in:
Georgi Gerganov
2026-02-19 15:19:53 +02:00
committed by GitHub
parent e6267a9359
commit da348c9dfb
3 changed files with 11 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/models/qwen35moe.cpp
+3 -1
View File
@@ -216,7 +216,7 @@ ggml_tensor * llm_build_qwen35moe ::build_layer_attn_linear(
ggml_tensor * z = qkvz.second;
ggml_tensor * beta = build_lora_mm(model.layers[il].ssm_beta, cur);
beta = ggml_reshape_4d(ctx0, beta, num_v_heads, 1, n_seq_tokens, n_seqs);
beta = ggml_reshape_4d(ctx0, beta, 1, num_v_heads, n_seq_tokens, n_seqs);
cb(beta, "beta", il);
beta = ggml_sigmoid(ctx0, beta);
@@ -232,6 +232,8 @@ ggml_tensor * llm_build_qwen35moe ::build_layer_attn_linear(
ggml_tensor * gate = ggml_mul(ctx0, alpha_softplus, model.layers[il].ssm_a); // -A_log.exp() * softplus
cb(gate, "gate", il);
gate = ggml_reshape_4d(ctx0, gate, 1, num_v_heads, n_seq_tokens, n_seqs);
// Get convolution states from cache
ggml_tensor * conv_states_all = mctx_cur->get_r_l(il);
ggml_tensor * ssm_states_all = mctx_cur->get_s_l(il);