Hugging Face's logo

Spaces:
UlrickBL
/
transformer_parameters
Running

App Files Community
UlrickBL commited on
Commit
97c0098
·
verified ·
1 Parent(s): b8fdafb

Update index.html

Browse files
Files changed (1) hide show
  1. index.html +78 -15
index.html CHANGED
@@ -3,7 +3,7 @@
3
  <head>
4
  <meta charset="UTF-8">
5
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
6
- <title>Transformers Total Parameters Calculator</title>
7
  <style>
8
  body {
9
  font-family: Arial, sans-serif;
@@ -22,11 +22,14 @@
22
  border-right: 2px solid #ee4a4f;
23
  }
24
 
 
 
 
 
25
  .output {
26
  width: 80%;
27
  padding: 20px;
28
  overflow-y: auto;
29
- font-size : 14px;
30
  }
31
 
32
  table {
@@ -99,12 +102,21 @@
99
  button:hover {
100
  background-color: #e60000;
101
  }
102
-
103
- th, td {
104
- font-size: 14px;
105
- }
106
  </style>
107
  <script>
 
 
 
 
 
 
 
 
 
 
 
 
 
108
  function calculateParameters() {
109
  const hiddenSize = parseInt(document.getElementById('hidden_size').value);
110
  const num_kv_heads = parseInt(document.getElementById('num_key_value_heads').value);
@@ -112,7 +124,10 @@
112
  const intermediateSize = parseInt(document.getElementById('intermediate_size').value);
113
  const vocabSize = parseInt(document.getElementById('vocab_size').value);
114
  const numHiddenLayers = parseInt(document.getElementById('num_hidden_layers').value);
 
 
115
  const includeBias = document.getElementById('include_bias').value === 'yes';
 
116
 
117
  const calcTotal = (input, output) => input * output;
118
  const calcKVdim = (hiddenSize, num_heads,num_kv_heads) => hiddenSize * num_kv_heads / num_heads;
@@ -138,25 +153,49 @@
138
  const luB = includeBias ? intermediateSize : 0;
139
  const projW = calcTotal(intermediateSize, hiddenSize);
140
  const projB = includeBias ? hiddenSize : 0;
 
 
 
 
141
 
142
- const feedForwardTotal = switchW + switchB + luW + luB + projW + projB;
143
 
 
 
 
 
 
144
  // Embedding
145
  const embeddingTotal = calcTotal(vocabSize, hiddenSize);
146
 
147
  // 1 layer parameters
148
  const oneLayerParams = attentionTotal + feedForwardTotal;
149
 
 
 
 
 
 
150
  // Full layers parameters
151
  const fullLayersParams = oneLayerParams * numHiddenLayers;
152
 
 
 
 
 
 
153
  // Full size (includes embedding)
154
  const fullSize = fullLayersParams + embeddingTotal;
155
 
 
 
 
 
 
156
  // Display results
157
  const outputDiv = document.getElementById('output');
158
  outputDiv.innerHTML = `
159
- <h1>Transformer total number of parameters Calculator</h1>
160
  <div class="section">
161
  <h3>Attention</h3>
162
  <table>
@@ -198,13 +237,13 @@
198
  <th>Total (input*output)</th>
199
  </tr>
200
  <tr>
201
- <td>Swish/Gelu - W</td>
202
  <td>${hiddenSize.toLocaleString()}</td>
203
  <td>${intermediateSize.toLocaleString()}</td>
204
  <td>${switchW.toLocaleString()}</td>
205
  </tr>
206
  ${includeBias ? `<tr>
207
- <td>Swish/Gelu - b</td>
208
  <td>-</td>
209
  <td>${intermediateSize.toLocaleString()}</td>
210
  <td>${switchB.toLocaleString()}</td>
@@ -237,11 +276,20 @@
237
  <th colspan="3">Total Feed Forward parameters</th>
238
  <td>${feedForwardTotal.toLocaleString()}</td>
239
  </tr>
 
 
 
 
 
240
  </table>
241
  </div>
242
  <div class="highlight">
243
- <strong>1 Layer Parameters (sum of attention and FFN):</strong> ${oneLayerParams.toLocaleString()}<br>
244
- <strong>Full Layers Parameters (1 layer parameters * num layers):</strong> ${fullLayersParams.toLocaleString()}<br>
 
 
 
 
245
  </div>
246
  <div class="section">
247
  <h3>Embedding</h3>
@@ -261,7 +309,8 @@
261
  </table>
262
  </div>
263
  <div class="highlight">
264
- <strong>Complete Model Parmeters (embedding size + full layers size):</strong> ${fullSize.toLocaleString()}
 
265
  </div>
266
  `;
267
  }
@@ -270,7 +319,7 @@
270
  <body>
271
 
272
  <div class="console">
273
- <h3>Input Config</h3>
274
  <label for="hidden_size">Hidden size:</label><br>
275
  <input type="number" id="hidden_size" value="896"><br><br>
276
 
@@ -295,12 +344,26 @@
295
  <option value="yes">Yes</option>
296
  </select><br><br>
297
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
298
  <button onclick="calculateParameters()">Calculate</button>
299
  </div>
300
 
301
  <div class="output" id="output">
302
  <h1>Transformer total number of parameters Calculator</h1>
303
- <h3>Enter model hyperparameters in the console and press calculate (curently working for classic transformer/LLM architecture with GQA and GLU - MOE mode incoming ...)</h3>
304
  </div>
305
 
306
  </body>
 
3
  <head>
4
  <meta charset="UTF-8">
5
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
6
+ <title>Model Parameter Calculator</title>
7
  <style>
8
  body {
9
  font-family: Arial, sans-serif;
 
22
  border-right: 2px solid #ee4a4f;
23
  }
24
 
25
+ #additionalFieldsMOE {
26
+ display: none;
27
+ }
28
+
29
  .output {
30
  width: 80%;
31
  padding: 20px;
32
  overflow-y: auto;
 
33
  }
34
 
35
  table {
 
102
  button:hover {
103
  background-color: #e60000;
104
  }
 
 
 
 
105
  </style>
106
  <script>
107
+ function handleSelectChange() {
108
+ // Récupère la valeur sélectionnée
109
+ const isMoe = document.getElementById('is_moe').value;
110
+ const additionalFieldsMOE = document.getElementById('additionalFieldsMOE');
111
+
112
+ // Affiche ou cache les champs supplémentaires
113
+ if (isMoe === 'yes') {
114
+ additionalFieldsMOE.style.display = 'block';
115
+ } else {
116
+ additionalFieldsMOE.style.display = 'none';
117
+ }
118
+ }
119
+
120
  function calculateParameters() {
121
  const hiddenSize = parseInt(document.getElementById('hidden_size').value);
122
  const num_kv_heads = parseInt(document.getElementById('num_key_value_heads').value);
 
124
  const intermediateSize = parseInt(document.getElementById('intermediate_size').value);
125
  const vocabSize = parseInt(document.getElementById('vocab_size').value);
126
  const numHiddenLayers = parseInt(document.getElementById('num_hidden_layers').value);
127
+ const active_expert_number = parseInt(document.getElementById('active_expert_number').value);
128
+ const expert_number = parseInt(document.getElementById('expert_number').value);
129
  const includeBias = document.getElementById('include_bias').value === 'yes';
130
+ const isMoe = document.getElementById('is_moe').value === 'yes';
131
 
132
  const calcTotal = (input, output) => input * output;
133
  const calcKVdim = (hiddenSize, num_heads,num_kv_heads) => hiddenSize * num_kv_heads / num_heads;
 
153
  const luB = includeBias ? intermediateSize : 0;
154
  const projW = calcTotal(intermediateSize, hiddenSize);
155
  const projB = includeBias ? hiddenSize : 0;
156
+ let feedForwardTotal;
157
+
158
+ feedForwardTotal = switchW + switchB + luW + luB + projW + projB;
159
+ const feedForwardTotalCache = feedForwardTotal
160
 
161
+ let feedForwardTotalActive;
162
 
163
+ if (isMoe) {
164
+ feedForwardTotalActive = feedForwardTotalCache * active_expert_number;
165
+ feedForwardTotal = feedForwardTotalCache * expert_number;
166
+ console.log(feedForwardTotal)
167
+ }
168
  // Embedding
169
  const embeddingTotal = calcTotal(vocabSize, hiddenSize);
170
 
171
  // 1 layer parameters
172
  const oneLayerParams = attentionTotal + feedForwardTotal;
173
 
174
+ let oneLayerParamsActive;
175
+ if (isMoe) {
176
+ oneLayerParamsActive = attentionTotal + feedForwardTotalActive;
177
+ }
178
+
179
  // Full layers parameters
180
  const fullLayersParams = oneLayerParams * numHiddenLayers;
181
 
182
+ let fullLayersParamsMOEActive;
183
+ if (isMoe) {
184
+ fullLayersParamsMOEActive = oneLayerParamsActive * numHiddenLayers;
185
+ }
186
+
187
  // Full size (includes embedding)
188
  const fullSize = fullLayersParams + embeddingTotal;
189
 
190
+ let fullSizeActive;
191
+ if (isMoe) {
192
+ fullSizeActive = fullLayersParamsMOEActive + embeddingTotal;
193
+ }
194
+
195
  // Display results
196
  const outputDiv = document.getElementById('output');
197
  outputDiv.innerHTML = `
198
+ <h1>Model Parameter Calculator</h1>
199
  <div class="section">
200
  <h3>Attention</h3>
201
  <table>
 
237
  <th>Total (input*output)</th>
238
  </tr>
239
  <tr>
240
+ <td>Swish - W</td>
241
  <td>${hiddenSize.toLocaleString()}</td>
242
  <td>${intermediateSize.toLocaleString()}</td>
243
  <td>${switchW.toLocaleString()}</td>
244
  </tr>
245
  ${includeBias ? `<tr>
246
+ <td>Swish - b</td>
247
  <td>-</td>
248
  <td>${intermediateSize.toLocaleString()}</td>
249
  <td>${switchB.toLocaleString()}</td>
 
276
  <th colspan="3">Total Feed Forward parameters</th>
277
  <td>${feedForwardTotal.toLocaleString()}</td>
278
  </tr>
279
+ ${isMoe ?`
280
+ <tr>
281
+ <th colspan="3">Total active Feed Forward parameters</th>
282
+ <td>${feedForwardTotalActive.toLocaleString()}</td>
283
+ </tr>` : ''}
284
  </table>
285
  </div>
286
  <div class="highlight">
287
+ <strong>1 Layer Parameters (Attention + FFN):</strong> ${oneLayerParams.toLocaleString()}<br>
288
+ <strong>Full Layers Parameters (1 layer parameters * num layers):</strong> ${fullLayersParams.toLocaleString()}<br><br>
289
+ ${isMoe ? `<tr>
290
+ <strong>1 Layer Parameters Active(Attention + FFN):</strong> ${oneLayerParamsActive.toLocaleString()}<br>
291
+ <strong>Full Layers Parameters Active(1 layer active parameters * num layers):</strong> ${fullLayersParamsMOEActive.toLocaleString()}<br>
292
+ </tr>` : ''}
293
  </div>
294
  <div class="section">
295
  <h3>Embedding</h3>
 
309
  </table>
310
  </div>
311
  <div class="highlight">
312
+ <strong>Complete Model Parmeters (embedding size + full layers size):</strong> ${fullSize.toLocaleString()}<br><br>
313
+ ${isMoe ? `<strong>Complete Model Parmeters Active:</strong> ${fullSizeActive.toLocaleString()}` : ''}
314
  </div>
315
  `;
316
  }
 
319
  <body>
320
 
321
  <div class="console">
322
+ <h3>Input Parameters</h3>
323
  <label for="hidden_size">Hidden size:</label><br>
324
  <input type="number" id="hidden_size" value="896"><br><br>
325
 
 
344
  <option value="yes">Yes</option>
345
  </select><br><br>
346
 
347
+ <label for="is_moe">Is MOE ?</label><br>
348
+ <select id="is_moe" onchange="handleSelectChange()">
349
+ <option value="no">No</option>
350
+ <option value="yes">Yes</option>
351
+ </select><br><br>
352
+
353
+ <div id="additionalFieldsMOE">
354
+ <label for="expert_number">Total expert number :</label><br>
355
+ <input type="number" id="expert_number" name="expert_number"><br><br>
356
+
357
+ <label for="active_expert_number">Total active experts (shared + specifics):</label><br>
358
+ <input type="number" id="active_expert_number" name="active_expert_number"><br><br>
359
+ </div>
360
+
361
  <button onclick="calculateParameters()">Calculate</button>
362
  </div>
363
 
364
  <div class="output" id="output">
365
  <h1>Transformer total number of parameters Calculator</h1>
366
+ <h3>Enter model hyperparameters in the console and press calculate (curently working for classic transformer/LLM architecture with GQA and GLU)</h3>
367
  </div>
368
 
369
  </body>