|
<!DOCTYPE html> |
|
<html lang="en"> |
|
<head> |
|
<meta charset="UTF-8"> |
|
<meta name="viewport" content="width=device-width, initial-scale=1.0"> |
|
<title>Model Parameter Calculator</title> |
|
<style> |
|
body { |
|
font-family: Arial, sans-serif; |
|
margin: 0; |
|
padding: 0; |
|
display: flex; |
|
flex-direction: row; |
|
height: 100vh; |
|
} |
|
|
|
.console { |
|
width: 20%; |
|
padding: 20px; |
|
background-color: #f6b5b5; |
|
overflow-y: auto; |
|
border-right: 2px solid #ee4a4f; |
|
} |
|
|
|
.output { |
|
width: 80%; |
|
padding: 20px; |
|
overflow-y: auto; |
|
} |
|
|
|
table { |
|
width: 100%; |
|
border-collapse: collapse; |
|
margin-bottom: 20px; |
|
background-color: #fff7f9; |
|
border: 1px solid #f6b5b5; |
|
} |
|
|
|
th, td { |
|
border: 1px solid #f6b5b5; |
|
padding: 8px; |
|
text-align: center; |
|
} |
|
|
|
th { |
|
background-color: #f6d4d4; |
|
} |
|
|
|
.highlight { |
|
background-color: #ffe8e8; |
|
color: #ad0d0d; |
|
padding: 10px; |
|
margin: 20px 0; |
|
border: 1px solid #f6b5b5; |
|
border-radius: 4px; |
|
text-align: center; |
|
} |
|
|
|
h3 { |
|
color: #ad0d0d; |
|
} |
|
|
|
.section { |
|
margin-bottom: 20px; |
|
} |
|
label { |
|
font-weight: bold; |
|
font-size: 0.9rem; |
|
} |
|
|
|
input[type="number"], select { |
|
width: 100%; |
|
padding: 8px; |
|
border: 1px solid #ddd; |
|
border-radius: 5px; |
|
box-sizing: border-box; |
|
font-size: 0.9rem; |
|
} |
|
|
|
input[type="number"]:focus, select:focus { |
|
border-color: #ff6666; |
|
outline: none; |
|
box-shadow: 0px 0px 5px rgba(255, 102, 102, 0.5); |
|
} |
|
|
|
button { |
|
background-color: #ff6666; |
|
color: white; |
|
border: none; |
|
border-radius: 5px; |
|
padding: 5px; |
|
width: 100%; |
|
font-size: 1rem; |
|
cursor: pointer; |
|
transition: background-color 0.3s ease; |
|
} |
|
|
|
button:hover { |
|
background-color: #e60000; |
|
} |
|
</style> |
|
<script> |
|
function calculateParameters() { |
|
const hiddenSize = parseInt(document.getElementById('hidden_size').value); |
|
const num_kv_heads = parseInt(document.getElementById('num_key_value_heads').value); |
|
const num_heads = parseInt(document.getElementById('num_heads').value); |
|
const intermediateSize = parseInt(document.getElementById('intermediate_size').value); |
|
const vocabSize = parseInt(document.getElementById('vocab_size').value); |
|
const numHiddenLayers = parseInt(document.getElementById('num_hidden_layers').value); |
|
const includeBias = document.getElementById('include_bias').value === 'yes'; |
|
|
|
const calcTotal = (input, output) => input * output; |
|
const calcKVdim = (hiddenSize, num_heads,num_kv_heads) => hiddenSize * num_kv_heads / num_heads; |
|
|
|
// Attention calculations |
|
const attention = [ |
|
{ name: 'Query', input: hiddenSize, output: hiddenSize }, |
|
{ name: 'Key', input: hiddenSize, output: calcKVdim(hiddenSize, num_heads,num_kv_heads) }, |
|
{ name: 'Value', input: hiddenSize, output: calcKVdim(hiddenSize, num_heads,num_kv_heads) }, |
|
{ name: 'Projection', input: hiddenSize, output: hiddenSize } |
|
].map(entry => ({ |
|
...entry, |
|
wTotal: calcTotal(entry.input, entry.output), |
|
bTotal: includeBias ? entry.output : 0 |
|
})); |
|
|
|
const attentionTotal = attention.reduce((sum, entry) => sum + entry.wTotal + entry.bTotal, 0); |
|
|
|
// Feed Forward calculations |
|
const switchW = calcTotal(hiddenSize, intermediateSize); |
|
const switchB = includeBias ? intermediateSize : 0; |
|
const luW = calcTotal(hiddenSize, intermediateSize); |
|
const luB = includeBias ? intermediateSize : 0; |
|
const projW = calcTotal(intermediateSize, hiddenSize); |
|
const projB = includeBias ? hiddenSize : 0; |
|
|
|
const feedForwardTotal = switchW + switchB + luW + luB + projW + projB; |
|
|
|
// Embedding |
|
const embeddingTotal = calcTotal(vocabSize, hiddenSize); |
|
|
|
// 1 layer parameters |
|
const oneLayerParams = attentionTotal + feedForwardTotal; |
|
|
|
// Full layers parameters |
|
const fullLayersParams = oneLayerParams * numHiddenLayers; |
|
|
|
// Full size (includes embedding) |
|
const fullSize = fullLayersParams + embeddingTotal; |
|
|
|
// Display results |
|
const outputDiv = document.getElementById('output'); |
|
outputDiv.innerHTML = ` |
|
<h1>Model Parameter Calculator</h1> |
|
<div class="section"> |
|
<h3>Attention</h3> |
|
<table> |
|
<tr> |
|
<th>Parameter</th> |
|
<th>Input Size</th> |
|
<th>Output Size</th> |
|
<th>Total parameters (input*ouput)</th> |
|
</tr> |
|
${attention.map(entry => ` |
|
<tr> |
|
<td>${entry.name} - W</td> |
|
<td>${entry.input.toLocaleString()}</td> |
|
<td>${entry.output.toLocaleString()}</td> |
|
<td>${entry.wTotal.toLocaleString()}</td> |
|
</tr> |
|
${includeBias ? ` |
|
<tr> |
|
<td>${entry.name} - b</td> |
|
<td>-</td> |
|
<td>${entry.output.toLocaleString()}</td> |
|
<td>${entry.bTotal.toLocaleString()}</td> |
|
</tr> |
|
` : ''} |
|
`).join('')} |
|
<tr> |
|
<th colspan="3">Total attention parameters</th> |
|
<td colspan="2">${attentionTotal.toLocaleString()}</td> |
|
</tr> |
|
</table> |
|
</div> |
|
<div class="section"> |
|
<h3>Feed Forward</h3> |
|
<table> |
|
<tr> |
|
<th>Parameter</th> |
|
<th>Input Size</th> |
|
<th>Output Size</th> |
|
<th>Total (input*output)</th> |
|
</tr> |
|
<tr> |
|
<td>Swish - W</td> |
|
<td>${hiddenSize.toLocaleString()}</td> |
|
<td>${intermediateSize.toLocaleString()}</td> |
|
<td>${switchW.toLocaleString()}</td> |
|
</tr> |
|
${includeBias ? `<tr> |
|
<td>Swish - b</td> |
|
<td>-</td> |
|
<td>${intermediateSize.toLocaleString()}</td> |
|
<td>${switchB.toLocaleString()}</td> |
|
</tr>` : ''} |
|
<tr> |
|
<td>GLU W</td> |
|
<td>${hiddenSize.toLocaleString()}</td> |
|
<td>${intermediateSize.toLocaleString()}</td> |
|
<td>${luW.toLocaleString()}</td> |
|
</tr> |
|
${includeBias ? `<tr> |
|
<td>GLU - b</td> |
|
<td>-</td> |
|
<td>${intermediateSize.toLocaleString()}</td> |
|
<td>${luB.toLocaleString()}</td> |
|
</tr>` : ''} |
|
<tr> |
|
<td>Projection - W</td> |
|
<td>${intermediateSize.toLocaleString()}</td> |
|
<td>${hiddenSize.toLocaleString()}</td> |
|
<td>${projW.toLocaleString()}</td> |
|
</tr> |
|
${includeBias ? `<tr> |
|
<td>Projection - b</td> |
|
<td>-</td> |
|
<td>${hiddenSize.toLocaleString()}</td> |
|
<td>${projB.toLocaleString()}</td> |
|
</tr>` : ''} |
|
<tr> |
|
<th colspan="3">Total Feed Forward parameters</th> |
|
<td>${feedForwardTotal.toLocaleString()}</td> |
|
</tr> |
|
</table> |
|
</div> |
|
<div class="highlight"> |
|
<strong>1 Layer Parameters (sum of attention and FFN):</strong> ${oneLayerParams.toLocaleString()}<br> |
|
<strong>Full Layers Parameters (1 layer parameters * num layers):</strong> ${fullLayersParams.toLocaleString()}<br> |
|
</div> |
|
<div class="section"> |
|
<h3>Embedding</h3> |
|
<table> |
|
<tr> |
|
<th>Parameter</th> |
|
<th>Vocab Size</th> |
|
<th>Hidden Size</th> |
|
<th>Total (vocab * hidden)</th> |
|
</tr> |
|
<tr> |
|
<td>Embedding</td> |
|
<td>${vocabSize.toLocaleString()}</td> |
|
<td>${hiddenSize.toLocaleString()}</td> |
|
<td>${embeddingTotal.toLocaleString()}</td> |
|
</tr> |
|
</table> |
|
</div> |
|
<div class="highlight"> |
|
<strong>Complete Model Parmeters (embedding size + full layers size):</strong> ${fullSize.toLocaleString()} |
|
</div> |
|
`; |
|
} |
|
</script> |
|
</head> |
|
<body> |
|
|
|
<div class="console"> |
|
<h3>Input Parameters</h3> |
|
<label for="hidden_size">Hidden size:</label><br> |
|
<input type="number" id="hidden_size" value="896"><br><br> |
|
|
|
<label for="intermediate_size">Intermediate size:</label><br> |
|
<input type="number" id="intermediate_size" value="4864"><br><br> |
|
|
|
<label for="vocab_size">Vocab size:</label><br> |
|
<input type="number" id="vocab_size" value="151646"><br><br> |
|
|
|
<label for="num_key_value_heads">Number of key-value heads:</label><br> |
|
<input type="number" id="num_key_value_heads" value="2"><br><br> |
|
|
|
<label for="num_heads">Number of attention (query) heads:</label><br> |
|
<input type="number" id="num_heads" value="14"><br><br> |
|
|
|
<label for="num_hidden_layers">Number of hidden layers:</label><br> |
|
<input type="number" id="num_hidden_layers" value="24"><br><br> |
|
|
|
<label for="include_bias">Include bias?</label><br> |
|
<select id="include_bias"> |
|
<option value="no">No</option> |
|
<option value="yes">Yes</option> |
|
</select><br><br> |
|
|
|
<button onclick="calculateParameters()">Calculate</button> |
|
</div> |
|
|
|
<div class="output" id="output"> |
|
<h1>Model Parameter Calculator</h1> |
|
<h3>Enter model parameters in the console and press calculate</h3> |
|
</div> |
|
|
|
</body> |
|
</html> |
|
|
