pub enum Launch {
Show 29 variants
Gather {
table: String,
index: String,
out: String,
cols: usize,
rows: usize,
format: Option<GgmlType>,
scale: Option<f64>,
},
View {
src: String,
offset: usize,
out: String,
},
Norm {Show 13 fields
x: String,
part: Option<Part>,
add: Option<String>,
pre: Option<PreNorm>,
sum: Option<String>,
w: Option<String>,
out: String,
into: Option<Dst>,
f32: bool,
quant: bool,
rows: usize,
cols: usize,
eps: Epsilon,
},
Quant {
x: String,
n: usize,
},
Gemv {Show 13 fields
x: String,
w: String,
format: GgmlType,
bias: Option<String>,
y: Dst,
k: usize,
n: usize,
m: usize,
per_warp: usize,
f32: bool,
quant: bool,
tail: Option<(usize, usize)>,
batch: bool,
},
GemvF32 {
x: String,
w: String,
bias: Option<String>,
y: Dst,
k: usize,
n: usize,
m: usize,
split: usize,
batch: bool,
},
GemmParts {
x: String,
format: GgmlType,
k: usize,
m: usize,
parts: Vec<GemmPart>,
tail: Option<(usize, usize)>,
per_warp: usize,
batch: bool,
},
Binary {
op: ElementwiseOp,
a: String,
b: String,
gate: Option<Activation>,
row: Option<usize>,
out: String,
f32: bool,
quant: bool,
n: usize,
},
Scale {
x: String,
y: String,
n: usize,
factor: f64,
},
SwiGlu {
g: String,
u: String,
out: String,
f32: bool,
quant: bool,
n: usize,
},
Unary {
act: Activation,
x: String,
y: String,
n: usize,
},
Rope {
x: String,
part: Option<Part>,
norm: Option<(String, Epsilon)>,
cos: String,
sin: String,
pos: Option<String>,
y: Dst,
heads: usize,
per_row: usize,
hd: usize,
half: usize,
},
RopeParts {
sides: Vec<RopeSide>,
cos: String,
sin: String,
pos: Option<String>,
hd: usize,
half: usize,
seqs: bool,
},
SetRow {
table: String,
index: RowSel,
src: String,
cols: usize,
count: usize,
seq: Option<usize>,
f32: bool,
quant: bool,
},
Attention {Show 26 fields
q: String,
kc: String,
vc: String,
pos: Option<String>,
sinks: Option<String>,
out: String,
heads: usize,
group: usize,
hd: usize,
vd: usize,
len: usize,
rows: usize,
window: Option<usize>,
vstride: usize,
voff: usize,
scale: f64,
per_block: usize,
splits: usize,
parallel: bool,
split: Option<Split>,
shared: Option<usize>,
seq: Option<usize>,
f32: bool,
quant: bool,
gate: Option<(String, Activation)>,
goff: usize,
},
ArgMax {
x: String,
out: String,
n: usize,
advance: Option<Advance>,
rows: usize,
},
Conv {
x: String,
w: String,
ring: String,
y: String,
chan: usize,
taps: usize,
rows: usize,
silu: bool,
l2: Option<(usize, Epsilon)>,
},
ConvParts {
parts: Vec<ConvPart>,
taps: usize,
block: usize,
eps: Option<Epsilon>,
seqs: usize,
},
L2Norm {
x: String,
y: String,
rows: usize,
cols: usize,
eps: Epsilon,
},
Decay {
ar: String,
dt: String,
al: String,
y: String,
n: usize,
heads: usize,
},
Gates {
br: String,
ar: String,
dt: String,
al: String,
beta: String,
decay: String,
n: usize,
heads: usize,
},
GateProj {
x: String,
wb: String,
wa: String,
dt: String,
al: String,
beta: String,
decay: String,
k: usize,
heads: usize,
parts: usize,
},
Delta {Show 17 fields
q: String,
k: String,
v: String,
z: String,
b: String,
a: String,
st: String,
gn: String,
out: String,
f32: bool,
quant: bool,
hv: usize,
hk: usize,
d: usize,
rows: usize,
eps: Epsilon,
seqs: usize,
},
Moe {Show 21 fields
x: String,
wr: String,
wg: String,
wu: String,
wd: String,
formats: [GgmlType; 3],
y: Dst,
d: usize,
dff: usize,
experts: usize,
k: usize,
router: Router,
rb: Option<String>,
biases: [Option<String>; 3],
renormalize: bool,
act: MoeAct,
shared: Option<SharedExperts>,
per_warp: usize,
rows: usize,
router_split: usize,
batch: bool,
},
Slice {
x: String,
y: String,
groups: usize,
every: usize,
from: usize,
width: usize,
},
Join {
a: String,
b: String,
y: Dst,
rows: usize,
heads: usize,
ha: usize,
hb: usize,
yz: usize,
},
HeadGemv {Show 13 fields
x: String,
part: Option<Part>,
w: String,
format: GgmlType,
y: Dst,
m: usize,
heads: usize,
a: usize,
b: usize,
rows_out: bool,
shared: bool,
upto: Option<(String, usize)>,
batch: bool,
},
GemvGlu {Show 15 fields
x: String,
wg: String,
wu: String,
formats: [GgmlType; 2],
biases: [Option<String>; 2],
act: Option<Activation>,
out: String,
f32: bool,
quant: bool,
k: usize,
n: usize,
m: usize,
per_warp: usize,
tail: Option<(usize, usize)>,
batch: bool,
},
Fold {
first: Box<Launch>,
then: Box<Launch>,
},
}Expand description
One launch of a library kernel. A producer names the tensor it computes (out); f32 when it writes the tensor’s
f32 buffer, quant when it writes its quantized form. Row counts are the block’s (1 for a decode step). A part
is where a launch reads its input x unit by unit: a part of every unit of a wider tensor (none, the whole of x).
Variants§
Gather
Fields
View
Norm
into: where its f32 result lands when not in out’s own buffer.
Fields
Quant
Gemv
A projection of m rows, per_warp output rows a warp; one of four rows a warp quantizes its output in its
epilogue (quant, keeping the f32 result when f32). On the tensor cores, tail is how its last wave is
filled (kernels::gemm_blk_tail): the tiles a block each, and the parts every later tile is cut into.
Fields
GemvF32
An f32 projection of m rows; split is the parts it cuts k into: each tile’s in tiles
(Launch::multiplies_in_tiles, kernels::gemm_f32_splits), each output row’s over a block’s warps otherwise
(kernels::gemv_f32_parts).
Fields
GemmParts
Projections of one activation’s m rows by matrices in one format, each into a buffer of its own or rows of a
table, as one launch over their outputs side by side: a tensor-core GEMM above the GEMV’s rows
(merged_projections; tail as a projection’s, over the parts’ tiles together), the GEMV in parts at or below
them (gemv_parts; per_warp output rows a warp, as each part’s own GEMV took them).
Fields
Binary
Scale
SwiGlu
Unary
Rope
Fields
RopeParts
Rotations of a step’s one row by one table at one position — a layer’s query heads and key heads, each normed
first where the model norms them — as one launch over their heads (rope_parts), each head computed as its own
rotation’s launch computes it.
seqs: a batch’s rotations over its sequences (sequences), a side’s row r sequence r, its position item
r of pos, its destination row in its own table RopeSide::yz values on from the last row’s.
Fields
SetRow
seq: a batch’s rows set over its sequences (sequences): row r of src into its own table, seq values on
from the last row’s, at its own index, item r of the index.
f32 and quant: the rows’ values are set, and their quantized form too, into the table’s — a batch’s
sequence’s row, quantized where its producer stored it (row_write_quant, PARITY-BATCH-ROW-QUANT), its rows whole
32-value blocks, its values set only where something reads the table f32.
Fields
Attention
vd the value head width; the value rows vstride apart, the heads vd wide from voff; scale the scores’.
A block takes per_block query heads of one group together; a row’s span is cut into at most splits chunks,
a block each when parallel, all in its row’s block otherwise. With a split, each query and key is read in
two parts. shared: the most chunks each row’s span is cut into when every head reads its one key/value group’s
rows together, each cached row read once for all of them (lk_attention_shared). seq: a batch’s attention over
its sequences (sequences), row r sequence r, its caches seq values on from the last row’s, its position
item r of pos.
quant: the attention quantizes its output in its own store for the projection that reads it (keeping it f32
too when f32), as lk_quant would — the decode step’s and a short chunk’s kernels (lk_attention,
lk_attention_shared and its fold), never those in tiles.
gate: the attention multiplies each output by its activation of the gate’s value at the same place as it stores
it, before it quantizes it (attn_output_gate) — the gate’s values read from goff on: a batch’s sequence’s row
of the batch’s gate, 0 otherwise.
ArgMax
advance: what the arg-max that closes the step moves on besides its result (argmax_advance).
rows: a batch’s arg-maxes over its rows sequences (sequences), each row’s logits, result and advance its
own; 1 otherwise.
Conv
Fields
ConvParts
Causal convolutions of a step’s one row through the SiLU, each over its own input, weights and ring, as one launch
(conv_parts): a block block channels of one part, the parts’ blocks one after another; a normed part’s heads
block wide, each normed to unit length with eps.
seqs: a batch’s convolutions over its seqs sequences (sequences), each part’s input row, ring and output
the sequence’s part of one array of them all; 1 otherwise.
L2Norm
Decay
Gates
GateProj
A step’s two Gated DeltaNet gates from its one activation row (gate_projections): each head’s write strength
sigmoid(x·wb) and forget gate (1 + e^(x·wa + dt))^al, the head’s two f32 dots summed as its own projections
sum them (parts warps a head, kernels::gemv_f32_parts) and its gates as Gates computes them, in one launch.
Fields
Delta
seqs: a batch’s delta rules over its seqs sequences, a row and a memory each (sequences), each input,
gate, memory and output the sequence’s part of one array of them all; 1 otherwise.
Fields
Moe
A mixture of experts experts of dff keeping k, and the shared experts every token takes, summed after —
its gate and up rows per_warp a warp — over rows rows of x routed each on its own, every expert reading its
weights once for all the rows that chose it. router_split: the parts an f32 router’s logits GEMM cuts each
tile’s k into (kernels::gemm_f32_splits) on the tensor cores.
Fields
Slice
y[g·width + i] = x[g·every + from + i] over groups groups.
Join
Each of the rows · heads units’ ha values of a, then its hb values of b. yz: a batch’s join into its
sequences’ caches (sequences), row r of rows into its own table, yz values on from the last’s, at its own
position; 0 otherwise.
HeadGemv
The per-head matmul over m rows and heads heads (a inputs, b outputs a head): rows_out when the matrix’s
rows are the heads’ outputs, its rows its inputs otherwise. shared: every head reads its row’s one unit of x
(the part of each row). upto: the rows multiplied are those before the position the parameter names plus
the count given, at most m.
Fields
GemvGlu
Two projections of x through a gate: out = act(x·wg + bg)·(x·wu + bu), SwiGLU when act is none — each
matrix n rows of k, m activation rows, per_warp output rows a warp. On the tensor cores, one GEMM of the
gate’s and up’s tiles side by side (lk_gemm_blk’s G 5), tail the gate’s own GEMM’s over twice its tiles.
Fields
act: Option<Activation>Fold
One launch running first, then then over what first wrote, in the blocks that finish what then reads: a
projection with the residual add and the next norm in its epilogue (gemv_norm_epilogue), run by its last block;
projections of one activation in parts with their heads’ norms and rotations (qkv_head_epilogue), each head
turned by the block that writes its last rows; the embedding’s gather with the first norm (embed_norm), each
row normed by the block that gathers it; or a mixture — alone, or folded with its sum with another array — with
the residual add and the next norm (moe_deferred_finalize), run by the decode step’s last down block or by a
chunk’s combine, a block a row. A projection’s blocks count in as they finish, and the finishing block reads their
results back through L2; every block runs then’s work as then’s own launch would, so each computes what it
computes apart.
Implementations§
Source§impl Launch
impl Launch
Sourcepub fn reads(&self) -> Vec<&str>
pub fn reads(&self) -> Vec<&str>
The tensors the launch reads (a parameter naming a row included).
Sourcepub fn typed(&self, numerics: Q8MatKind) -> Vec<&str>
pub fn typed(&self, numerics: Q8MatKind) -> Vec<&str>
The arrays the launch’s kernels take in any storage type, reading each value widened exactly and writing it
rounded to the nearest: an attention’s keys, values and split keys’ rest, a delta rule’s memory, a slice’s source,
a tiled per-head matmul’s activation, and every destination a producer writes into a table’s rows (lk_out).
A view touches nothing itself.
Sourcepub fn writes(&self) -> Vec<&str>
pub fn writes(&self) -> Vec<&str>
The tensors the launch writes (a cache it appends to, a memory it updates, included).
Sourcepub fn touches(
&self,
numerics: Q8MatKind,
) -> (Vec<(&str, Touch)>, Vec<(&str, Touch)>)
pub fn touches( &self, numerics: Q8MatKind, ) -> (Vec<(&str, Touch)>, Vec<(&str, Touch)>)
The tensors the launch reads and writes, each in the form it touches: a projection reads its activation
quantized unless the numerics keep it f32; a producer writes its result f32, quantized, or both. A view
touches nothing: what its readers touch of it, they touch of its tensor.
Sourcepub fn on_tensor_cores(&self, numerics: Q8MatKind) -> bool
pub fn on_tensor_cores(&self, numerics: Q8MatKind) -> bool
Whether the launch multiplies on the int8 tensor cores under numerics: a quantized projection of more rows than
the GEMV takes, its activation quantized, by lk_gemm_blk in any format, Q8_0’s split arrays included — several
side by side in parts, a gated pair as one paired GEMM — and an expert mixture over as many rows, each expert’s
matrices by the grouped lk_gemm_blk.
Sourcepub fn batch_on_tensor_cores(&self, numerics: Q8MatKind) -> bool
pub fn batch_on_tensor_cores(&self, numerics: Q8MatKind) -> bool
Whether a batch’s projection runs on the tensor cores in its rows’ own order (lk_gemv_mma): a projection, its
parts or its gated pair over Q8_0 matrices of a batch’s GEMV_MMA_MIN_ROWS or more rows, under the quantized
numerics, every part a whole number of 8-output tiles and k a whole number of 32-value blocks.
Sourcepub fn is_batch(&self) -> bool
pub fn is_batch(&self) -> bool
Whether the launch is a batch’s projection (batch): its rows are the batch’s sequences, each summed as the
decode step sums its one row — row by row whatever their count, never in tiles.
Sourcepub fn gemm_cols(&self) -> usize
pub fn gemm_cols(&self) -> usize
A projection’s outputs a row: a GEMM in parts’ over its parts together; none for any other launch.
Sourcepub fn parts(&self) -> Vec<&Launch>
pub fn parts(&self) -> Vec<&Launch>
The launches this launch runs the work of: a fold’s first’s and then’s, any other launch itself — what a question about one kernel’s reads or kind asks of each.
Sourcepub fn mixture_mut(&mut self) -> Option<&mut Launch>
pub fn mixture_mut(&mut self) -> Option<&mut Launch>
The mixture a fold finishes with the next norm, for the schedule to set how it runs: the launch itself where it is a mixture.
Sourcepub fn finished_mixture(
&self,
) -> Option<(&Launch, &Launch, Option<(&str, &str)>)>
pub fn finished_mixture( &self, ) -> Option<(&Launch, &Launch, Option<(&str, &str)>)>
The mixture a fold finishes with the next norm, its norm, and — where the norm adds the mixture’s sum with another array — that array and the sum.
Sourcepub fn gemm_rows(&self, numerics: Q8MatKind) -> usize
pub fn gemm_rows(&self, numerics: Q8MatKind) -> usize
The activation rows a block of a dense tensor-core GEMM takes under numerics (kernels::gemm_blk_rows).
Sourcepub fn gemm_tiles_wide(&self) -> usize
pub fn gemm_tiles_wide(&self) -> usize
The tiles a tensor-core GEMM’s outputs a row take: of GEMM_TILE_N outputs, a gated pair’s of half as many
hidden values (its gate’s and up’s side by side).
Sourcepub fn multiplies_in_tiles(&self, numerics: Q8MatKind) -> bool
pub fn multiplies_in_tiles(&self, numerics: Q8MatKind) -> bool
Whether an f32 projection multiplies its rows as one f32 GEMM of 64 × 64 tiles under numerics
(lk_gemm_f32): a chunk of more rows than the GEMV takes under the quantized numerics — the precise numerics sum
each row’s dots in f64 — its rows whole float4s, its result its own buffer.
Sourcepub fn pairs_gate_and_up(&self, numerics: Q8MatKind) -> bool
pub fn pairs_gate_and_up(&self, numerics: Q8MatKind) -> bool
Whether an expert mixture on the tensor cores multiplies each expert’s gate and up together (lk_gemm_blk’s
paired mode, the gated activation and its quantizer in the GEMM’s epilogue): its gate and up in one format.
Sourcepub fn delta_in_chunks(&self, numerics: Q8MatKind) -> bool
pub fn delta_in_chunks(&self, numerics: Q8MatKind) -> bool
Whether the launch walks its delta rule a chunk of rows at a time under numerics (lk_delta_prep, then
lk_delta_state): a delta rule over a chunk of more rows than the GEMV takes, under the quantized numerics, its
heads a width the chunked rule takes (kernels::delta_chunks_take).
Sourcepub fn attends_on_tensor_cores(&self, numerics: Q8MatKind) -> bool
pub fn attends_on_tensor_cores(&self, numerics: Q8MatKind) -> bool
Whether the launch attends on the f16 tensor cores under numerics (lk_attention_mma): it attends in tiles
under the one-word numerics — llama.cpp’s precision, whose attention rounds its operands to f16 — its heads and
values whole multiples of 16 and at most 256 wide.
Sourcepub fn attends_in_two_words(&self, numerics: Q8MatKind) -> bool
pub fn attends_in_two_words(&self, numerics: Q8MatKind) -> bool
Whether the launch attends on the int8 tensor cores in two words under numerics (lk_attention_i8, its keys and
values taken into two words once by lk_kv_two_words): it attends in tiles under the two-word numerics, its heads
and values whole multiples of 32 and at most 256 wide.
Sourcepub fn attends_flash(&self, numerics: Q8MatKind) -> bool
pub fn attends_flash(&self, numerics: Q8MatKind) -> bool
Whether the launch attends in tiles in flash style under numerics (lk_attention_flash): it attends in tiles
and on neither the f16 nor the int8 tensor cores, its heads and values at most 256 wide; wider ones take
lk_attention_tiled.
Sourcepub fn attends_in_tiles(&self, numerics: Q8MatKind) -> bool
pub fn attends_in_tiles(&self, numerics: Q8MatKind) -> bool
Whether the launch attends in tiles under numerics (lk_attention_flash, or lk_attention_tiled for wider
heads): an attention over a chunk of more rows than the GEMV takes, under the quantized numerics, its keys,
values and outputs read and written four at a time and its values at most 512 wide.
Sourcepub fn heads_in_tiles(&self, numerics: Q8MatKind) -> bool
pub fn heads_in_tiles(&self, numerics: Q8MatKind) -> bool
Whether the launch is a per-head matmul run in tiles under numerics (lk_head_tiled): over a chunk of more rows
than the GEMV takes, under the quantized numerics, its inputs read four words at a time.
Sourcepub fn writes_whole(&self, t: &str, touch: Touch, len: usize) -> bool
pub fn writes_whole(&self, t: &str, touch: Touch, len: usize) -> bool
Whether the launch writes all len elements of tensor t’s array touch — a result in its own buffer, every
element of it — and not rows of a table, a part of every unit of a wider tensor, or carried state it updates.
Sourcepub fn writes_part(&self, t: &str, len: usize) -> Option<(usize, usize, usize)>
pub fn writes_part(&self, t: &str, len: usize) -> Option<(usize, usize, usize)>
The part of every unit of tensor t (len elements) the launch writes, (pitch, from, width): a result written
in place as a part of every unit of a wider tensor (join_in_place), all of its units.
Sourcepub fn projected_input(&self) -> Option<&str>
pub fn projected_input(&self) -> Option<&str>
The activation a product by checkpoint rows reads quantized, under every numerics but the precise one. A
per-head matmul — a latent’s expansion half — reads its activation f32 under every numerics
(kernels::head_mode).
Trait Implementations§
impl StructuralPartialEq for Launch
Auto Trait Implementations§
impl Freeze for Launch
impl RefUnwindSafe for Launch
impl Send for Launch
impl Sync for Launch
impl Unpin for Launch
impl UnsafeUnpin for Launch
impl UnwindSafe for Launch
Blanket Implementations§
Source§impl<T> BorrowMut<T> for Twhere
T: ?Sized,
impl<T> BorrowMut<T> for Twhere
T: ?Sized,
Source§fn borrow_mut(&mut self) -> &mut T
fn borrow_mut(&mut self) -> &mut T
Source§impl<T> CloneToUninit for Twhere
T: Clone,
impl<T> CloneToUninit for Twhere
T: Clone,
§impl<T> Downcast for Twhere
T: Any,
impl<T> Downcast for Twhere
T: Any,
§fn into_any(self: Box<T>) -> Box<dyn Any>
fn into_any(self: Box<T>) -> Box<dyn Any>
Box<dyn Trait> (where Trait: Downcast) to Box<dyn Any>. Box<dyn Any> can
then be further downcast into Box<ConcreteType> where ConcreteType implements Trait.§fn into_any_rc(self: Rc<T>) -> Rc<dyn Any>
fn into_any_rc(self: Rc<T>) -> Rc<dyn Any>
Rc<Trait> (where Trait: Downcast) to Rc<Any>. Rc<Any> can then be
further downcast into Rc<ConcreteType> where ConcreteType implements Trait.§fn as_any(&self) -> &(dyn Any + 'static)
fn as_any(&self) -> &(dyn Any + 'static)
&Trait (where Trait: Downcast) to &Any. This is needed since Rust cannot
generate &Any’s vtable from &Trait’s.§fn as_any_mut(&mut self) -> &mut (dyn Any + 'static)
fn as_any_mut(&mut self) -> &mut (dyn Any + 'static)
&mut Trait (where Trait: Downcast) to &Any. This is needed since Rust cannot
generate &mut Any’s vtable from &mut Trait’s.