|
<!DOCTYPE html> |
|
<html lang="en"> |
|
<head> |
|
<meta charset="UTF-8"> |
|
<meta name="viewport" content="width=device-width, initial-scale=1.0"> |
|
<title>Model Parameter Calculator</title> |
|
<style> |
|
body { |
|
font-family: Arial, sans-serif; |
|
margin: 0; |
|
padding: 0; |
|
display: flex; |
|
flex-direction: row; |
|
height: 100vh; |
|
background: linear-gradient(120deg, #f8b7d4, #d1c1f2); |
|
color: #4a154b; |
|
} |
|
|
|
.console { |
|
width: 30%; |
|
padding: 20px; |
|
background-color: #fef4fc; |
|
border-right: 2px solid #d1c1f2; |
|
overflow-y: auto; |
|
} |
|
|
|
.output { |
|
width: 70%; |
|
padding: 20px; |
|
overflow-y: auto; |
|
} |
|
|
|
table { |
|
width: 100%; |
|
border-collapse: collapse; |
|
margin-bottom: 20px; |
|
} |
|
|
|
th, td { |
|
border: 1px solid #ddd; |
|
padding: 8px; |
|
text-align: left; |
|
} |
|
|
|
th { |
|
background-color: #d1c1f2; |
|
color: #4a154b; |
|
} |
|
|
|
.highlight { |
|
background-color: #f8d7f1; |
|
color: #4a154b; |
|
padding: 10px; |
|
margin: 20px 0; |
|
border: 1px solid #d1c1f2; |
|
border-radius: 4px; |
|
} |
|
|
|
h3 { |
|
color: #4a154b; |
|
} |
|
|
|
label, input, select { |
|
display: block; |
|
margin-bottom: 10px; |
|
width: 100%; |
|
} |
|
|
|
input, select { |
|
padding: 8px; |
|
border: 1px solid #d1c1f2; |
|
border-radius: 4px; |
|
background: #fef4fc; |
|
color: #4a154b; |
|
} |
|
|
|
button { |
|
background-color: #d1c1f2; |
|
color: white; |
|
padding: 10px 20px; |
|
border: none; |
|
border-radius: 4px; |
|
cursor: pointer; |
|
} |
|
|
|
button:hover { |
|
background-color: #bfa8e3; |
|
} |
|
|
|
.section-title { |
|
background-color: #e9d8f5; |
|
color: #4a154b; |
|
text-align: center; |
|
padding: 5px 0; |
|
margin-bottom: 5px; |
|
} |
|
|
|
.sub-table { |
|
margin-top: 10px; |
|
} |
|
|
|
.no-bias { |
|
display: none; |
|
} |
|
</style> |
|
<script> |
|
function calculateParameters() { |
|
const hiddenSize = parseInt(document.getElementById('hidden_size').value); |
|
const intermediateSize = parseInt(document.getElementById('intermediate_size').value); |
|
const vocabSize = parseInt(document.getElementById('vocab_size').value); |
|
const numKeyValueHeads = parseInt(document.getElementById('num_key_value_heads').value); |
|
const numAttentionHeads = parseInt(document.getElementById('num_attention_heads').value); |
|
const numHiddenLayers = parseInt(document.getElementById('num_hidden_layers').value); |
|
const includeBias = document.getElementById('include_bias').value === 'yes'; |
|
|
|
// Attention calculations |
|
const attentionQ = hiddenSize * hiddenSize; |
|
const attentionK = hiddenSize * numKeyValueHeads; |
|
const attentionV = hiddenSize * numKeyValueHeads; |
|
const attentionBq = includeBias ? hiddenSize : 0; |
|
const attentionBk = includeBias ? numKeyValueHeads : 0; |
|
const attentionBv = includeBias ? numKeyValueHeads : 0; |
|
const attentionQOutput = hiddenSize; |
|
const attentionKOutput = numKeyValueHeads; |
|
const attentionVOutput = numKeyValueHeads; |
|
const attentionTotal = 2 * (attentionQ + attentionBq) + 2 * (attentionK + attentionBk) + 2 * (attentionV + attentionBv); |
|
|
|
// Feed Forward calculations |
|
const switchW = hiddenSize * intermediateSize; |
|
const switchB = includeBias ? intermediateSize : 0; |
|
const luW = hiddenSize * intermediateSize; |
|
const luB = includeBias ? intermediateSize : 0; |
|
const projW = intermediateSize * hiddenSize; |
|
const projB = includeBias ? hiddenSize : 0; |
|
const feedForwardTotal = switchW + switchB + luW + luB + projW + projB; |
|
|
|
// Embedding |
|
const embeddingTotal = vocabSize * hiddenSize; |
|
|
|
// 1 layer parameters |
|
const oneLayerParams = attentionTotal + feedForwardTotal; |
|
|
|
// Full layers parameters |
|
const fullLayersParams = oneLayerParams * numHiddenLayers; |
|
|
|
// Full size (includes embedding) |
|
const fullSize = fullLayersParams + embeddingTotal; |
|
|
|
// Display results |
|
const attentionTable = document.getElementById('attention_table'); |
|
attentionTable.innerHTML = ` |
|
<tr><th>Type</th><th>Input Size</th><th>Output Size</th><th>Total Size</th></tr> |
|
<tr><td>Q</td><td>${hiddenSize.toLocaleString()}</td><td>${attentionQOutput.toLocaleString()}</td><td>${(hiddenSize * attentionQOutput).toLocaleString()}</td></tr> |
|
<tr><td>K</td><td>${hiddenSize.toLocaleString()}</td><td>${attentionKOutput.toLocaleString()}</td><td>${(hiddenSize * attentionKOutput).toLocaleString()}</td></tr> |
|
<tr><td>V</td><td>${hiddenSize.toLocaleString()}</td><td>${attentionVOutput.toLocaleString()}</td><td>${(hiddenSize * attentionVOutput).toLocaleString()}</td></tr> |
|
${includeBias ? `<tr class="bias-row"><td>Bias</td><td>${attentionBq.toLocaleString()}</td><td>${attentionBk.toLocaleString()}</td><td>${attentionBv.toLocaleString()}</td><td></td></tr>` : ''} |
|
`; |
|
|
|
const feedForwardTable = document.getElementById('feed_forward_table'); |
|
feedForwardTable.innerHTML = ` |
|
<tr><th>Type</th><th>Input Size</th><th>Output Size</th><th>Total Size</th></tr> |
|
<tr><td>Switch W</td><td>${hiddenSize.toLocaleString()}</td><td>${intermediateSize.toLocaleString()}</td><td>${(hiddenSize * intermediateSize).toLocaleString()}</td></tr> |
|
<tr><td>LU W</td><td>${hiddenSize.toLocaleString()}</td><td>${intermediateSize.toLocaleString()}</td><td>${(hiddenSize * intermediateSize).toLocaleString()}</td></tr> |
|
<tr><td>Proj W</td><td>${intermediateSize.toLocaleString()}</td><td>${hiddenSize.toLocaleString()}</td><td>${(intermediateSize * hiddenSize).toLocaleString()}</td></tr> |
|
${includeBias ? `<tr class="bias-row"><td>Bias</td><td>${switchB.toLocaleString()}</td><td>${luB.toLocaleString()}</td><td>${projB.toLocaleString()}</td><td></td></tr>` : ''} |
|
`; |
|
|
|
const embeddingTable = document.getElementById('embedding_table'); |
|
embeddingTable.innerHTML = ` |
|
<tr><th>Type</th><th>Input Size</th><th>Output Size</th><th>Total Size</th></tr> |
|
<tr><td>Embedding</td><td>${vocabSize.toLocaleString()}</td><td>${hiddenSize.toLocaleString()}</td><td>${embeddingTotal.toLocaleString()}</td></tr> |
|
`; |
|
|
|
document.getElementById('one_layer_params').innerText = oneLayerParams.toLocaleString(); |
|
document.getElementById('full_layers_params').innerText = fullLayersParams.toLocaleString(); |
|
document.getElementById('full_size').innerText = fullSize.toLocaleString(); |
|
} |
|
</script> |
|
</head> |
|
<body> |
|
|
|
<div class="console"> |
|
<h3>Input Parameters</h3> |
|
<label for="hidden_size">Hidden size:</label> |
|
<input type="number" id="hidden_size" value="2048"> |
|
|
|
<label for="intermediate_size">Intermediate size:</label> |
|
<input type="number" id="intermediate_size" value="16384"> |
|
|
|
<label for="vocab_size">Vocab size:</label> |
|
<input type="number" id="vocab_size" value="64000"> |
|
|
|
<label for="num_key_value_heads">Number of key-value heads:</label> |
|
<input type="number" id="num_key_value_heads" value="80"> |
|
|
|
<label for="num_attention_heads">Number of attention heads:</label> |
|
<input type="number" id="num_attention_heads" value="8"> |
|
|
|
<label for="num_hidden_layers">Number of hidden layers:</label> |
|
<input type="number" id="num_hidden_layers" value="64"> |
|
|
|
<label for="include_bias">Include bias?</label> |
|
<select id="include_bias"> |
|
<option value="yes">Yes</option> |
|
<option value="no">No</option> |
|
</select> |
|
|
|
<button onclick="calculateParameters()">Calculate</button> |
|
</div> |
|
|
|
<div class="output"> |
|
<h3>Model Parameter Results</h3> |
|
|
|
<div class="section-title">1 Layer Parameters</div> |
|
<div><strong>1 Layer Parameters:</strong> <span id="one_layer_params">0</span></div> |
|
<div><strong>Full Layers Parameters:</strong> <span id="full_layers_params">0</span></div> |
|
|
|
<div class="section-title">Attention Parameters</div> |
|
<table id="attention_table" class="sub-table"></table> |
|
|
|
<div class="section-title">Feed Forward Parameters</div> |
|
<table id="feed_forward_table" class="sub-table"></table> |
|
|
|
<div class="section-title">Embedding Parameters</div> |
|
<table id="embedding_table" class="sub-table"></table> |
|
|
|
<div class="highlight"> |
|
<strong>Complete Model Size:</strong> <span id="full_size">0</span> |
|
</div> |
|
</div> |
|
|
|
</body> |
|
</html> |
|
|
