Stephen Cox
547765a93e
* mtmd: add Gemma 4 audio conformer encoder support Add audio processing for Gemma 4 E2B/E4B via a USM-style Conformer. Architecture: - 12-layer Conformer: FFN → Self-Attention → Causal Conv1D → FFN → Norm - Subsampling Conv Projection: 2x Conv2D(stride=2) with LayerNorm - Full self-attention with sinusoidal RPE and sliding window mask (24) - Logit softcapping at 50.0, ClippableLinear clamping - Output: 1024 → 1536 → RMSNorm → multimodal embedder Mel preprocessing (dedicated mtmd_audio_preprocessor_gemma4a): - HTK mel scale, 128 bins, magnitude STFT, mel_floor=1e-3 - Standard periodic Hann window (320 samples), zero-padded to FFT size - Semicausal left-padding (frame_length/2 samples) - Frame count matched to PyTorch (unfold formula) - No pre-emphasis, no Whisper-style normalization - Mel cosine similarity vs PyTorch: 0.9998 Key fixes: - Tensor loading dedup: prevent get_tensor() from creating duplicate entries in ctx_data. Fixed with std::set guard. - ClippableLinear clamp_info loading moved after per-layer tensors. - Sliding window mask (24 positions) matching PyTorch context_size. - Skip Whisper normalization for Gemma4 mel output. Tested on E2B and E4B with CPU and Vulkan backends. Transcribes: "Glad to see things are going well and business is starting to pick up" (matching ground truth). Ref: #21325
124 lines
3.7 KiB
C++
124 lines
3.7 KiB
C++
#pragma once
|
||
|
||
#include "ggml.h"
|
||
#include "clip-model.h"
|
||
|
||
#include <cstdint>
|
||
#include <vector>
|
||
#include <string>
|
||
|
||
#define MTMD_INTERNAL_HEADER
|
||
|
||
struct mtmd_audio_mel {
|
||
int n_len;
|
||
int n_len_org;
|
||
int n_mel;
|
||
|
||
std::vector<float> data;
|
||
};
|
||
|
||
struct mtmd_audio_mel_filters {
|
||
int32_t n_mel;
|
||
int32_t n_fft;
|
||
|
||
std::vector<float> data;
|
||
};
|
||
|
||
// cache for audio processing, each processor instance owns its own cache
|
||
struct mtmd_audio_cache {
|
||
std::vector<float> sin_vals;
|
||
std::vector<float> cos_vals;
|
||
|
||
std::vector<float> hann_window;
|
||
|
||
mtmd_audio_mel_filters filters;
|
||
|
||
void fill_sin_cos_table(uint32_t n);
|
||
|
||
void fill_hann_window(uint32_t length, bool periodic);
|
||
|
||
// Build mel filterbank matrix [n_mel × n_fft_bins] at runtime.
|
||
// n_fft_bins must be (N_fft / 2 + 1). Example: if N_fft=512 -> n_fft_bins=257.
|
||
void fill_mel_filterbank_matrix(int n_mel,
|
||
int n_fft,
|
||
int sample_rate, // e.g. 16000
|
||
float fmin = 0.0f, // e.g. 0.0
|
||
float fmax = -1.0f, // e.g. sr/2; pass -1 for auto
|
||
bool slaney_area_norm = true,
|
||
float scale = 1.0f,
|
||
bool use_htk = false
|
||
);
|
||
};
|
||
|
||
struct mtmd_audio_preprocessor {
|
||
const clip_hparams & hparams;
|
||
|
||
mtmd_audio_preprocessor(const clip_ctx * ctx): hparams(*clip_get_hparams(ctx)) {}
|
||
|
||
virtual ~mtmd_audio_preprocessor() = default;
|
||
virtual void initialize() = 0; // NOT thread-safe
|
||
virtual bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) = 0;
|
||
};
|
||
|
||
struct mtmd_audio_preprocessor_whisper : mtmd_audio_preprocessor {
|
||
mtmd_audio_preprocessor_whisper(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}
|
||
void initialize() override;
|
||
bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) override;
|
||
|
||
private:
|
||
mtmd_audio_cache cache;
|
||
};
|
||
|
||
struct mtmd_audio_preprocessor_conformer : mtmd_audio_preprocessor {
|
||
mtmd_audio_preprocessor_conformer(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}
|
||
void initialize() override;
|
||
bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) override;
|
||
|
||
private:
|
||
mtmd_audio_cache cache;
|
||
};
|
||
|
||
struct mtmd_audio_preprocessor_gemma4a : mtmd_audio_preprocessor {
|
||
mtmd_audio_preprocessor_gemma4a(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}
|
||
void initialize() override;
|
||
bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) override;
|
||
|
||
private:
|
||
mtmd_audio_cache cache;
|
||
};
|
||
|
||
//
|
||
// streaming ISTFT - converts spectrogram frames back to audio one frame at a time
|
||
//
|
||
struct mtmd_audio_streaming_istft {
|
||
mtmd_audio_streaming_istft(int n_fft, int hop_length);
|
||
|
||
// reset streaming state
|
||
void reset();
|
||
|
||
// process a single STFT frame (streaming)
|
||
// frame_spectrum: [n_fft_bins x 2] interleaved real/imag
|
||
// returns: up to hop_length samples
|
||
std::vector<float> process_frame(const float * frame_spectrum);
|
||
|
||
// flush remaining samples at end of stream
|
||
std::vector<float> flush();
|
||
|
||
private:
|
||
int n_fft;
|
||
int hop_length;
|
||
int n_fft_bins;
|
||
|
||
// Own cache for output processing
|
||
mtmd_audio_cache cache;
|
||
|
||
// Streaming state
|
||
std::vector<float> overlap_buffer;
|
||
std::vector<float> window_sum_buffer;
|
||
int padding_to_remove;
|
||
|
||
// Working buffers for IFFT
|
||
std::vector<float> ifft_in;
|
||
std::vector<float> ifft_out;
|
||
};
|