1pub fn fp32_verilog() -> &'static str {
19 FP32_VERILOG
20}
21
22pub const FMUL_LATENCY: usize = 3;
24
25pub const FADD_LATENCY: usize = 4;
27
28pub const TREE16_LATENCY: usize = 4 * FADD_LATENCY;
30
31pub const F32_TO_F16_LATENCY: usize = 3;
34
35const FP32_VERILOG: &str = r#"// The engine's f32 multiply and add: round to nearest even, denormals flushed, one NaN.
36module logos_fmul(input [31:0] a, input [31:0] b, output reg [31:0] y);
37 wire sa = a[31], sb = b[31], s = sa ^ sb;
38 wire [7:0] ea = a[30:23], eb = b[30:23];
39 wire [22:0] ma = a[22:0], mb = b[22:0];
40 wire za = (ea == 8'd0), zb = (eb == 8'd0);
41 wire ia = (ea == 8'hff) && (ma == 23'd0), ib = (eb == 8'hff) && (mb == 23'd0);
42 wire na = (ea == 8'hff) && (ma != 23'd0), nb = (eb == 8'hff) && (mb != 23'd0);
43 wire [47:0] p = {1'b1, ma} * {1'b1, mb};
44 wire top = p[47];
45 // The product normalized to a leading one at bit 47, and the biased exponent of that one.
46 wire [47:0] pn = top ? p : {p[46:0], 1'b0};
47 wire signed [11:0] e = $signed({4'd0, ea}) + $signed({4'd0, eb}) - 12'sd127 + (top ? 12'sd1 : 12'sd0);
48 // Below the normals the significand is rounded at the denormals' place, and survives only by rounding up to the
49 // least normal: one place below (e = 0) it is shifted a place first, and lower (e < 0) its 24 bits are under 2^22, so
50 // no rounding reaches 2^23 and it is flushed as a zero significand is.
51 wire below = (e == 12'sd0), gone = (e < 12'sd0);
52 wire [23:0] keep = gone ? 24'd0 : below ? {1'b0, pn[47:25]} : pn[47:24];
53 wire guard = !gone && (below ? pn[24] : pn[23]);
54 wire sticky = !gone && (below ? |pn[23:0] : |pn[22:0]);
55 wire up = guard && (sticky || keep[0]);
56 wire [24:0] r = {1'b0, keep} + {24'd0, up};
57 // A carry out of 24 bits renormalizes; below the normals a carry into bit 23 reaches the least normal.
58 wire carry = r[24];
59 wire signed [11:0] ef = (e < 12'sd1) ? (r[23] ? 12'sd1 : 12'sd0) : (carry ? e + 12'sd1 : e);
60 wire [22:0] mf = carry ? r[23:1] : r[22:0];
61 always @* begin
62 if (na || nb || (ia && zb) || (ib && za)) y = 32'h7fc00000;
63 else if (ia || ib) y = {s, 8'hff, 23'd0};
64 else if (za || zb) y = {s, 31'd0};
65 else if (ef >= 12'sd255) y = {s, 8'hff, 23'd0};
66 else if (ef <= 12'sd0) y = {s, 31'd0};
67 else y = {s, ef[7:0], mf};
68 end
69endmodule
70
71module logos_fadd(input [31:0] a, input [31:0] b, output reg [31:0] y);
72 wire sa = a[31], sb = b[31];
73 wire [7:0] ea = a[30:23], eb = b[30:23];
74 wire [22:0] ma = a[22:0], mb = b[22:0];
75 wire za = (ea == 8'd0), zb = (eb == 8'd0);
76 wire ia = (ea == 8'hff) && (ma == 23'd0), ib = (eb == 8'hff) && (mb == 23'd0);
77 wire na = (ea == 8'hff) && (ma != 23'd0), nb = (eb == 8'hff) && (mb != 23'd0);
78 // The larger magnitude first; a flushed operand is a zero.
79 wire swap = {ea, ma} < {eb, mb};
80 wire s1 = swap ? sb : sa, s2 = swap ? sa : sb;
81 wire [7:0] e1 = swap ? eb : ea, e2 = swap ? ea : eb;
82 wire z2 = swap ? za : zb;
83 wire [23:0] f1 = {1'b1, swap ? mb : ma};
84 wire [23:0] f2 = z2 ? 24'd0 : {1'b1, swap ? ma : mb};
85 wire [7:0] d = e1 - e2;
86 // Three places below the significand: guard, round, and a sticky of everything shifted past them.
87 wire [50:0] aligned = {f2, 27'd0} >> ((d > 8'd50) ? 8'd50 : d);
88 wire [26:0] g2 = {aligned[50:25], |aligned[24:0]};
89 wire [26:0] g1 = {f1, 3'd0};
90 wire sub = s1 ^ s2;
91 wire [27:0] sum = sub ? {1'b0, g1} - {1'b0, g2} : {1'b0, g1} + {1'b0, g2};
92 // The leading one of the sum: at bit 27 after a carry, else at 26 or below after a cancellation.
93 reg [4:0] lz;
94 integer i;
95 always @* begin
96 lz = 5'd27;
97 for (i = 0; i <= 26; i = i + 1) if (sum[i]) lz = 5'd26 - i[4:0];
98 end
99 wire c = sum[27];
100 wire signed [10:0] en = c ? $signed({3'd0, e1}) + 11'sd1 : $signed({3'd0, e1}) - $signed({6'd0, lz});
101 // Normalized with the leading one at bit 26, the sticky kept through a right shift.
102 wire [26:0] norm = c ? {sum[27:2], sum[1] | sum[0]} : (sum[26:0] << lz);
103 // Two normals are whole multiples of the least denormal, and so is their sum: one below the normals is exact at the
104 // denormals' precision, rounds to nothing above them, and is flushed.
105 wire [23:0] keep = norm[26:3];
106 wire guard = norm[2];
107 wire sticky = |norm[1:0];
108 wire up = guard && (sticky || keep[0]);
109 wire [24:0] r = {1'b0, keep} + {24'd0, up};
110 wire carry = r[24];
111 wire signed [10:0] ef = (en < 11'sd1) ? 11'sd0 : (carry ? en + 11'sd1 : en);
112 wire [22:0] mf = carry ? r[23:1] : r[22:0];
113 wire zero = (sum == 28'd0);
114 always @* begin
115 if (na || nb || (ia && ib && (sa != sb))) y = 32'h7fc00000;
116 else if (ia) y = {sa, 8'hff, 23'd0};
117 else if (ib) y = {sb, 8'hff, 23'd0};
118 else if (za && zb) y = {sa & sb, 31'd0};
119 else if (za) y = b;
120 else if (zb) y = a;
121 else if (zero) y = 32'd0;
122 else if (ef >= 11'sd255) y = {s1, 8'hff, 23'd0};
123 else if (ef <= 11'sd0) y = {s1, 31'd0};
124 else y = {s1, ef[7:0], mf};
125 end
126endmodule
127
128// The pairwise tree over sixteen lanes: adjacent pairs, then pairs of pairs.
129// The engine's f32 multiply, three registered stages: y is logos_fmul(a, b) of the operands three clocks before.
130// Stage 1 classifies the operands and multiplies their significands, stage 2 normalizes the product and places it at
131// the denormals' precision when it is below the normals, stage 3 rounds and assembles; each stage's logic is
132// logos_fmul's, in its order.
133module logos_fmul_p(input clk, input [31:0] a, input [31:0] b, output reg [31:0] y);
134 // Stage 1.
135 wire sa = a[31], sb = b[31];
136 wire [7:0] ea = a[30:23], eb = b[30:23];
137 wire [22:0] ma = a[22:0], mb = b[22:0];
138 reg s1, nan1, inf1, zero1;
139 reg [47:0] p1;
140 reg signed [11:0] e1;
141 always @(posedge clk) begin
142 s1 <= sa ^ sb;
143 nan1 <= ((ea == 8'hff) && (ma != 23'd0)) || ((eb == 8'hff) && (mb != 23'd0))
144 || ((ea == 8'hff) && (ma == 23'd0) && (eb == 8'd0)) || ((eb == 8'hff) && (mb == 23'd0) && (ea == 8'd0));
145 inf1 <= ((ea == 8'hff) && (ma == 23'd0)) || ((eb == 8'hff) && (mb == 23'd0));
146 zero1 <= (ea == 8'd0) || (eb == 8'd0);
147 p1 <= {1'b1, ma} * {1'b1, mb};
148 e1 <= $signed({4'd0, ea}) + $signed({4'd0, eb}) - 12'sd127;
149 end
150 // Stage 2.
151 wire top = p1[47];
152 wire [47:0] pn = top ? p1 : {p1[46:0], 1'b0};
153 wire signed [11:0] e = e1 + (top ? 12'sd1 : 12'sd0);
154 wire below = (e == 12'sd0), gone = (e < 12'sd0);
155 reg s2, nan2, inf2, zero2, guard2, sticky2;
156 reg [23:0] keep2;
157 reg signed [11:0] e2;
158 always @(posedge clk) begin
159 s2 <= s1; nan2 <= nan1; inf2 <= inf1; zero2 <= zero1;
160 keep2 <= gone ? 24'd0 : below ? {1'b0, pn[47:25]} : pn[47:24];
161 guard2 <= !gone && (below ? pn[24] : pn[23]);
162 sticky2 <= !gone && (below ? |pn[23:0] : |pn[22:0]);
163 e2 <= e;
164 end
165 // Stage 3.
166 wire up = guard2 && (sticky2 || keep2[0]);
167 wire [24:0] r = {1'b0, keep2} + {24'd0, up};
168 wire carry = r[24];
169 wire signed [11:0] ef = (e2 < 12'sd1) ? (r[23] ? 12'sd1 : 12'sd0) : (carry ? e2 + 12'sd1 : e2);
170 wire [22:0] mf = carry ? r[23:1] : r[22:0];
171 always @(posedge clk) begin
172 if (nan2) y <= 32'h7fc00000;
173 else if (inf2) y <= {s2, 8'hff, 23'd0};
174 else if (zero2) y <= {s2, 31'd0};
175 else if (ef >= 12'sd255) y <= {s2, 8'hff, 23'd0};
176 else if (ef <= 12'sd0) y <= {s2, 31'd0};
177 else y <= {s2, ef[7:0], mf};
178 end
179endmodule
180
181// The engine's f32 add, four registered stages: y is logos_fadd(a, b) of the operands four clocks before. Stage 1
182// orders the operands by magnitude, stage 2 aligns the smaller and adds, stage 3 finds the sum's leading one and
183// normalizes, stage 4 rounds and assembles, a sum below the normals flushed; each stage's logic is logos_fadd's.
184module logos_fadd_p(input clk, input [31:0] a, input [31:0] b, output reg [31:0] y);
185 // Stage 1.
186 wire sa = a[31], sb = b[31];
187 wire [7:0] ea = a[30:23], eb = b[30:23];
188 wire [22:0] ma = a[22:0], mb = b[22:0];
189 wire za = (ea == 8'd0), zb = (eb == 8'd0);
190 wire ia = (ea == 8'hff) && (ma == 23'd0), ib = (eb == 8'hff) && (mb == 23'd0);
191 wire na = (ea == 8'hff) && (ma != 23'd0), nb = (eb == 8'hff) && (mb != 23'd0);
192 wire swap = {ea, ma} < {eb, mb};
193 wire [7:0] e1w = swap ? eb : ea, e2w = swap ? ea : eb;
194 wire z2 = swap ? za : zb;
195 wire [7:0] d = e1w - e2w;
196 // The special cases decided here and carried: a NaN, an infinity's sign, a zero operand's result.
197 reg [2:0] kind1; // 0 the sum, 1 NaN, 2 +-inf of sa, 3 +-inf of sb, 4 both zero, 5 b, 6 a
198 reg sa1, sb1, s11;
199 reg [31:0] a1, b1;
200 reg [7:0] e11;
201 reg [23:0] f11, f21;
202 reg [5:0] d1;
203 reg sub1;
204 always @(posedge clk) begin
205 kind1 <= (na || nb || (ia && ib && (sa != sb))) ? 3'd1 : ia ? 3'd2 : ib ? 3'd3 : (za && zb) ? 3'd4 : za ? 3'd5 : zb ? 3'd6 : 3'd0;
206 sa1 <= sa; sb1 <= sb; a1 <= a; b1 <= b;
207 s11 <= swap ? sb : sa;
208 e11 <= e1w;
209 f11 <= {1'b1, swap ? mb : ma};
210 f21 <= z2 ? 24'd0 : {1'b1, swap ? ma : mb};
211 d1 <= (d > 8'd50) ? 6'd50 : d[5:0];
212 sub1 <= sa ^ sb;
213 end
214 // Stage 2.
215 wire [50:0] aligned = {f21, 27'd0} >> d1;
216 wire [26:0] g1 = {f11, 3'd0}, g2 = {aligned[50:25], |aligned[24:0]};
217 wire [27:0] sum = sub1 ? {1'b0, g1} - {1'b0, g2} : {1'b0, g1} + {1'b0, g2};
218 reg [2:0] kind2;
219 reg sa2, sb2, s12;
220 reg [31:0] a2, b2;
221 reg [7:0] e12;
222 reg [27:0] sum2;
223 always @(posedge clk) begin
224 kind2 <= kind1; sa2 <= sa1; sb2 <= sb1; a2 <= a1; b2 <= b1; s12 <= s11; e12 <= e11;
225 sum2 <= sum;
226 end
227 // Stage 3.
228 reg [4:0] lz;
229 integer i;
230 always @* begin
231 lz = 5'd27;
232 for (i = 0; i <= 26; i = i + 1) if (sum2[i]) lz = 5'd26 - i[4:0];
233 end
234 wire c = sum2[27];
235 wire signed [10:0] en = c ? $signed({3'd0, e12}) + 11'sd1 : $signed({3'd0, e12}) - $signed({6'd0, lz});
236 wire [26:0] norm = c ? {sum2[27:2], sum2[1] | sum2[0]} : (sum2[26:0] << lz);
237 reg [2:0] kind3;
238 reg sa3, sb3, s13, zero3, guard3, sticky3;
239 reg [31:0] a3, b3;
240 reg [23:0] keep3;
241 reg signed [10:0] en3;
242 always @(posedge clk) begin
243 kind3 <= kind2; sa3 <= sa2; sb3 <= sb2; a3 <= a2; b3 <= b2; s13 <= s12;
244 zero3 <= (sum2 == 28'd0);
245 keep3 <= norm[26:3];
246 guard3 <= norm[2];
247 sticky3 <= |norm[1:0];
248 en3 <= en;
249 end
250 // Stage 4.
251 wire up = guard3 && (sticky3 || keep3[0]);
252 wire [24:0] r = {1'b0, keep3} + {24'd0, up};
253 wire carry = r[24];
254 wire signed [10:0] ef = (en3 < 11'sd1) ? 11'sd0 : (carry ? en3 + 11'sd1 : en3);
255 wire [22:0] mf = carry ? r[23:1] : r[22:0];
256 always @(posedge clk) begin
257 case (kind3)
258 3'd1: y <= 32'h7fc00000;
259 3'd2: y <= {sa3, 8'hff, 23'd0};
260 3'd3: y <= {sb3, 8'hff, 23'd0};
261 3'd4: y <= {sa3 & sb3, 31'd0};
262 3'd5: y <= b3;
263 3'd6: y <= a3;
264 default:
265 if (zero3) y <= 32'd0;
266 else if (ef >= 11'sd255) y <= {s13, 8'hff, 23'd0};
267 else if (ef <= 11'sd0) y <= {s13, 31'd0};
268 else y <= {s13, ef[7:0], mf};
269 endcase
270 end
271endmodule
272
273module logos_tree16(input [511:0] lanes, output [31:0] sum);
274 wire [31:0] l1 [0:7];
275 wire [31:0] l2 [0:3];
276 wire [31:0] l3 [0:1];
277 genvar i;
278 generate
279 for (i = 0; i < 8; i = i + 1) begin : t1
280 logos_fadd a(.a(lanes[64*i +: 32]), .b(lanes[64*i + 32 +: 32]), .y(l1[i]));
281 end
282 for (i = 0; i < 4; i = i + 1) begin : t2
283 logos_fadd a(.a(l1[2*i]), .b(l1[2*i + 1]), .y(l2[i]));
284 end
285 for (i = 0; i < 2; i = i + 1) begin : t3
286 logos_fadd a(.a(l2[2*i]), .b(l2[2*i + 1]), .y(l3[i]));
287 end
288 endgenerate
289 logos_fadd t4(.a(l3[0]), .b(l3[1]), .y(sum));
290endmodule
291
292// The pairwise tree of sixteen lanes on the pipelined add: sum is logos_tree16 of the lanes sixteen clocks before, four
293// levels of logos_fadd_p, sixteen lanes a clock.
294module logos_tree16_p(input clk, input [511:0] lanes, output [31:0] sum);
295 wire [31:0] l1 [0:7];
296 wire [31:0] l2 [0:3];
297 wire [31:0] l3 [0:1];
298 genvar i;
299 generate
300 for (i = 0; i < 8; i = i + 1) begin : t1
301 logos_fadd_p a(.clk(clk), .a(lanes[64*i +: 32]), .b(lanes[64*i + 32 +: 32]), .y(l1[i]));
302 end
303 for (i = 0; i < 4; i = i + 1) begin : t2
304 logos_fadd_p a(.clk(clk), .a(l1[2*i]), .b(l1[2*i + 1]), .y(l2[i]));
305 end
306 for (i = 0; i < 2; i = i + 1) begin : t3
307 logos_fadd_p a(.clk(clk), .a(l2[2*i]), .b(l2[2*i + 1]), .y(l3[i]));
308 end
309 endgenerate
310 logos_fadd_p t4(.clk(clk), .a(l3[0]), .b(l3[1]), .y(sum));
311endmodule
312
313// The binary16 codec.
314module logos_f32_to_f16(input [31:0] x, output reg [15:0] h);
315 wire s = x[31];
316 wire [7:0] ex = x[30:23];
317 wire [22:0] mant = x[22:0];
318 wire signed [9:0] e = $signed({2'd0, ex}) - 10'sd127;
319 wire [9:0] m = mant[22:13];
320 wire [12:0] rem = mant[12:0];
321 wire [15:0] normal = {1'b0, e[4:0] + 5'd15, m};
322 wire up = (rem > 13'h1000) || (rem == 13'h1000 && m[0]);
323 // A binary16 denormal counts units of 2^-24: the 24-bit significand shifted right by -(e + 1).
324 wire [23:0] sig = {1'b1, mant};
325 wire signed [9:0] sh = -(e + 10'sd1);
326 wire [23:0] n = sig >> sh;
327 wire [23:0] dr = sig & ((24'd1 << sh) - 24'd1);
328 wire [23:0] dh = 24'd1 << (sh - 10'sd1);
329 wire dup = (dr > dh) || (dr == dh && n[0]);
330 always @* begin
331 if (ex == 8'hff) h = {s, 5'h1f, (mant != 23'd0) ? 10'h200 : 10'd0};
332 else if (ex == 8'd0) h = {s, 15'd0};
333 else if (e > 10'sd15) h = {s, 15'h7c00};
334 else if (e >= -10'sd14) h = {s, 15'd0} | (normal + {15'd0, up});
335 else if (sh > 10'sd24) h = {s, 15'd0};
336 else h = {s, 15'd0} | (n[15:0] + {15'd0, dup});
337 end
338endmodule
339
340// logos_f32_to_f16 for the user clock, an operand a clock: h is logos_f32_to_f16 of the operand two clocks before, its
341// normal and denormal candidates with their round bits and its class registered, then the rounding and the choice.
342module logos_f32_to_f16_p(input clk, input [31:0] x, output reg [15:0] h);
343 // Stage 1: the class, the normal's rounding, and the denormal's shift.
344 wire s = x[31];
345 wire [7:0] ex = x[30:23];
346 wire [22:0] mant = x[22:0];
347 wire signed [9:0] e = $signed({2'd0, ex}) - 10'sd127;
348 wire [9:0] m = mant[22:13];
349 wire [12:0] rem = mant[12:0];
350 wire [15:0] normal = {1'b0, e[4:0] + 5'd15, m};
351 wire up = (rem > 13'h1000) || (rem == 13'h1000 && m[0]);
352 wire signed [9:0] sh = -(e + 10'sd1);
353 // The class, in logos_f32_to_f16's order: NaN or infinity, zero or denormal, past the top, normal, below the
354 // least denormal, denormal.
355 wire [2:0] cls_c = (ex == 8'hff) ? 3'd0 : (ex == 8'd0) ? 3'd1 : (e > 10'sd15) ? 3'd2 : (e >= -10'sd14) ? 3'd3
356 : (sh > 10'sd24) ? 3'd4 : 3'd5;
357 reg s1, up1, nan1;
358 reg [15:0] normal1;
359 reg [23:0] sig1;
360 reg [4:0] sh1;
361 reg [2:0] cls1;
362 always @(posedge clk) begin
363 s1 <= s; up1 <= up; nan1 <= mant != 23'd0; normal1 <= normal; sig1 <= {1'b1, mant}; sh1 <= sh[4:0]; cls1 <= cls_c;
364 end
365 // Stage 2: the denormal's significand and its rounding, nearest even.
366 wire [23:0] n = sig1 >> sh1;
367 wire [23:0] dr = sig1 & ((24'd1 << sh1) - 24'd1);
368 wire [23:0] dh = 24'd1 << (sh1 - 5'd1);
369 wire dup = (dr > dh) || (dr == dh && n[0]);
370 reg s2, up2, dup2, nan2;
371 reg [15:0] normal2, n2;
372 reg [2:0] cls;
373 always @(posedge clk) begin
374 s2 <= s1; up2 <= up1; dup2 <= dup; nan2 <= nan1; normal2 <= normal1; n2 <= n[15:0]; cls <= cls1;
375 end
376 // Stage 3: the binary16.
377 always @(posedge clk)
378 case (cls)
379 3'd0: h <= {s2, 5'h1f, nan2 ? 10'h200 : 10'd0};
380 3'd1: h <= {s2, 15'd0};
381 3'd2: h <= {s2, 15'h7c00};
382 3'd3: h <= {s2, 15'd0} | (normal2 + {15'd0, up2});
383 3'd4: h <= {s2, 15'd0};
384 default: h <= {s2, 15'd0} | (n2 + {15'd0, dup2});
385 endcase
386endmodule
387
388// A subnormal's leading one at bit p gives its significand m << (10 - p) and exponent 103 + p, each candidate chosen by
389// the one-hot of p (lead), so the normalization is an and-or of ten candidates rather than a search and a shift.
390module logos_f16_to_f32(input [15:0] h, output [31:0] x);
391 wire s = h[15];
392 wire [4:0] e = h[14:10];
393 wire [9:0] m = h[9:0];
394 wire [9:0] lead;
395 genvar p;
396 generate
397 for (p = 0; p < 10; p = p + 1) begin : one
398 assign lead[p] = m[p] && (m >> (p + 1)) == 10'd0;
399 end
400 endgenerate
401 reg [9:0] sub_m;
402 reg [7:0] sub_e;
403 integer q;
404 always @* begin
405 sub_m = 10'd0;
406 sub_e = 8'd0;
407 for (q = 0; q < 10; q = q + 1) begin
408 sub_m = sub_m | ({10{lead[q]}} & (m << (10 - q)));
409 sub_e = sub_e | ({8{lead[q]}} & (8'd103 + q[7:0]));
410 end
411 end
412 assign x = e == 5'd0 ? (m == 10'd0 ? {s, 31'd0} : {s, sub_e, sub_m, 13'd0})
413 : e == 5'h1f ? (m == 10'd0 ? {s, 8'hff, 23'd0} : {s, 8'hff, 1'b1, m[8:0], 13'd0})
414 : {s, {3'd0, e} + 8'd112, m, 13'd0};
415endmodule
416"#;