sleepy/llama.cpp
1
0
Fork 0
Code Issues 11 Pull Requests Actions 109 Packages Projects Releases Wiki Activity

[metal] extend bin op fusion to MUL/SUB/DIV chains (#28) #38

Merged
sleepy merged 1 commits from fix/28-bin-op-fusion into master 2026-04-30 21:03:15 +02:00
Conversation 1 Commits 1 Files Changed 2
+25 -19
2 changed files with 25 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ggml/src/ggml-metal/ggml-metal-common.cpp
+12 -2
View File
@@ -394,20 +394,30 @@ void ggml_graph_optimize(ggml_cgraph * gf) {
// fuse only ops that start with these operations
// can be expanded when needed
if (node.op() == GGML_OP_ADD ||
node.op() == GGML_OP_SUB ||
node.op() == GGML_OP_MUL ||
node.op() == GGML_OP_DIV ||
node.op() == GGML_OP_NORM ||
node.op() == GGML_OP_RMS_NORM) {
ops[0] = node.op();
int f = i + 1;
while (f < n && f < i + MAX_FUSE) {
// conservatively allow fusing only these ops
// can be expanded when needed
// bin ops (ADD/SUB/MUL/DIV) must be same type to fuse
// NORM/RMS_NORM can chain with MUL/ADD
if (node.op() == GGML_OP_ADD ||
node.op() == GGML_OP_SUB ||
node.op() == GGML_OP_MUL ||
node.op() == GGML_OP_DIV) {
if (gf->nodes[f]->op != node.op()) break;
} else {
if (gf->nodes[f]->op != GGML_OP_ADD &&
gf->nodes[f]->op != GGML_OP_MUL &&
gf->nodes[f]->op != GGML_OP_NORM &&
gf->nodes[f]->op != GGML_OP_RMS_NORM) {
break;
}
}
ops[f - i] = gf->nodes[f]->op;
f++;
}
ggml/src/ggml-metal/ggml-metal-ops.cpp
+8 -12
View File
@@ -3118,19 +3118,15 @@ int ggml_metal_op_bin(ggml_metal_op_t ctx, int idx) {
int n_fuse = 1;
// c[0] = add(a, b[0])
// c[1] = add(c[0], b[1])
// c[2] = add(c[1], b[2])
// c[0] = op(a, b[0])
// c[1] = op(c[0], b[1])
// c[2] = op(c[1], b[2])
// ...
if (use_fusion) {
fops[0] = GGML_OP_ADD;
fops[1] = GGML_OP_ADD;
fops[2] = GGML_OP_ADD;
fops[3] = GGML_OP_ADD;
fops[4] = GGML_OP_ADD;
fops[5] = GGML_OP_ADD;
fops[6] = GGML_OP_ADD;
fops[7] = GGML_OP_ADD;
ggml_op cur_op = op->op;
for (int i = 0; i < 8; ++i) {
fops[i] = cur_op;
}
// note: in metal, we sometimes encode the graph in parallel so we have to avoid fusing ops
// across splits. idx_end indicates the last node in the current split
@@ -3165,7 +3161,7 @@ int ggml_metal_op_bin(ggml_metal_op_t ctx, int idx) {
++n_fuse;
if (debug_fusion > 1 && n_fuse > 1) {
GGML_LOG_DEBUG("%s: fuse: ADD x %d\n", __func__, n_fuse);
GGML_LOG_DEBUG("%s: fuse: %s x %d\n", __func__, ggml_op_name(cur_op), n_fuse);
}
}