|
<!DOCTYPE html> |
|
<html lang="en"> |
|
<head> |
|
<meta charset="UTF-8"> |
|
<meta name="viewport" content="width=device-width, initial-scale=1.0"> |
|
<title>Model Parameter Calculator</title> |
|
<style> |
|
body { |
|
font-family: Arial, sans-serif; |
|
margin: 0; |
|
padding: 0; |
|
display: flex; |
|
flex-direction: row; |
|
height: 100vh; |
|
} |
|
|
|
.console { |
|
width: 30%; |
|
padding: 20px; |
|
background-color: #f4f4f4; |
|
overflow-y: auto; |
|
} |
|
|
|
.output { |
|
width: 70%; |
|
padding: 20px; |
|
overflow-y: auto; |
|
} |
|
|
|
table { |
|
width: 100%; |
|
border-collapse: collapse; |
|
} |
|
|
|
th, td { |
|
border: 1px solid #ddd; |
|
padding: 8px; |
|
text-align: left; |
|
} |
|
|
|
th { |
|
background-color: #f2f2f2; |
|
} |
|
|
|
.highlight { |
|
background-color: #d4edda; |
|
color: #155724; |
|
padding: 10px; |
|
margin: 20px 0; |
|
border: 1px solid #c3e6cb; |
|
border-radius: 4px; |
|
} |
|
</style> |
|
<script> |
|
function calculateParameters() { |
|
const hiddenSize = parseInt(document.getElementById('hidden_size').value); |
|
const intermediateSize = parseInt(document.getElementById('intermediate_size').value); |
|
const vocabSize = parseInt(document.getElementById('vocab_size').value); |
|
const numKeyValueHeads = parseInt(document.getElementById('num_key_value_heads').value); |
|
const numAttentionHeads = parseInt(document.getElementById('num_attention_heads').value); |
|
const numHiddenLayers = parseInt(document.getElementById('num_hidden_layers').value); |
|
const includeBias = document.getElementById('include_bias').value === 'yes'; |
|
|
|
// Attention calculations |
|
const attentionQ = hiddenSize * hiddenSize; |
|
const attentionV = hiddenSize * numKeyValueHeads; |
|
const attentionBq = includeBias ? hiddenSize : 0; |
|
const attentionBv = includeBias ? numKeyValueHeads : 0; |
|
const attentionTotal = 2 * (attentionQ + attentionBq) + 2 * (attentionV + attentionBv); |
|
|
|
// Feed Forward calculations |
|
const switchW = hiddenSize * intermediateSize; |
|
const switchB = includeBias ? intermediateSize : 0; |
|
const luW = hiddenSize * intermediateSize; |
|
const luB = includeBias ? intermediateSize : 0; |
|
const projW = intermediateSize * hiddenSize; |
|
const projB = includeBias ? hiddenSize : 0; |
|
const feedForwardTotal = switchW + switchB + luW + luB + projW + projB; |
|
|
|
// Embedding |
|
const embeddingTotal = vocabSize * hiddenSize; |
|
|
|
// 1 layer parameters |
|
const oneLayerParams = attentionTotal + feedForwardTotal; |
|
|
|
// Full layers parameters |
|
const fullLayersParams = oneLayerParams * numHiddenLayers; |
|
|
|
// Full size (includes embedding) |
|
const fullSize = fullLayersParams + embeddingTotal; |
|
|
|
// Display results |
|
const outputTable = document.getElementById('output_table'); |
|
outputTable.innerHTML = ` |
|
<tr> |
|
<th>Section</th> |
|
<th>Parameter</th> |
|
<th>Value</th> |
|
</tr> |
|
<tr> |
|
<td>Attention</td> |
|
<td>Q</td> |
|
<td>${attentionQ.toLocaleString()}</td> |
|
</tr> |
|
<tr> |
|
<td>Attention</td> |
|
<td>Bq</td> |
|
<td>${attentionBq.toLocaleString()}</td> |
|
</tr> |
|
<tr> |
|
<td>Attention</td> |
|
<td>V</td> |
|
<td>${attentionV.toLocaleString()}</td> |
|
</tr> |
|
<tr> |
|
<td>Attention</td> |
|
<td>Bv</td> |
|
<td>${attentionBv.toLocaleString()}</td> |
|
</tr> |
|
<tr> |
|
<td>Attention</td> |
|
<td>Total</td> |
|
<td>${attentionTotal.toLocaleString()}</td> |
|
</tr> |
|
<tr> |
|
<td>Feed Forward</td> |
|
<td>Switch W</td> |
|
<td>${switchW.toLocaleString()}</td> |
|
</tr> |
|
<tr> |
|
<td>Feed Forward</td> |
|
<td>Switch b</td> |
|
<td>${switchB.toLocaleString()}</td> |
|
</tr> |
|
<tr> |
|
<td>Feed Forward</td> |
|
<td>LU W</td> |
|
<td>${luW.toLocaleString()}</td> |
|
</tr> |
|
<tr> |
|
<td>Feed Forward</td> |
|
<td>LU b</td> |
|
<td>${luB.toLocaleString()}</td> |
|
</tr> |
|
<tr> |
|
<td>Feed Forward</td> |
|
<td>Proj W</td> |
|
<td>${projW.toLocaleString()}</td> |
|
</tr> |
|
<tr> |
|
<td>Feed Forward</td> |
|
<td>Proj b</td> |
|
<td>${projB.toLocaleString()}</td> |
|
</tr> |
|
<tr> |
|
<td>Feed Forward</td> |
|
<td>Total</td> |
|
<td>${feedForwardTotal.toLocaleString()}</td> |
|
</tr> |
|
<tr> |
|
<td>Embedding</td> |
|
<td>Embedding</td> |
|
<td>${embeddingTotal.toLocaleString()}</td> |
|
</tr> |
|
`; |
|
|
|
document.getElementById('one_layer_params').innerText = oneLayerParams.toLocaleString(); |
|
document.getElementById('full_layers_params').innerText = fullLayersParams.toLocaleString(); |
|
document.getElementById('full_size').innerText = fullSize.toLocaleString(); |
|
} |
|
</script> |
|
</head> |
|
<body> |
|
|
|
<div class="console"> |
|
<h3>Input Parameters</h3> |
|
<label for="hidden_size">Hidden size:</label><br> |
|
<input type="number" id="hidden_size" value="2048"><br><br> |
|
|
|
<label for="intermediate_size">Intermediate size:</label><br> |
|
<input type="number" id="intermediate_size" value="16384"><br><br> |
|
|
|
<label for="vocab_size">Vocab size:</label><br> |
|
<input type="number" id="vocab_size" value="64000"><br><br> |
|
|
|
<label for="num_key_value_heads">Number of key-value heads:</label><br> |
|
<input type="number" id="num_key_value_heads" value="80"><br><br> |
|
|
|
<label for="num_attention_heads">Number of attention heads:</label><br> |
|
<input type="number" id="num_attention_heads" value="8"><br><br> |
|
|
|
<label for="num_hidden_layers">Number of hidden layers:</label><br> |
|
<input type="number" id="num_hidden_layers" value="64"><br><br> |
|
|
|
<label for="include_bias">Include bias?</label><br> |
|
<select id="include_bias"> |
|
<option value="yes">Yes</option> |
|
<option value="no">No</option> |
|
</select><br><br> |
|
|
|
<button onclick="calculateParameters()">Calculate</button> |
|
</div> |
|
|
|
<div class="output"> |
|
<h3>Model Parameter Results</h3> |
|
|
|
<table id="output_table"> |
|
<tr> |
|
<th>Section</th> |
|
<th>Parameter</th> |
|
<th>Value</th> |
|
</tr> |
|
</table> |
|
|
|
<div class="highlight"> |
|
<strong>1 Layer Parameters:</strong> <span id="one_layer_params">0</span><br> |
|
<strong>Full Layers Parameters:</strong> <span id="full_layers_params">0</span><br> |
|
<strong>Complete Model Size:</strong> <span id="full_size">0</span> |
|
</div> |
|
</div> |
|
|
|
</body> |
|
</html> |
|
|
