Spaces:

UlrickBL
/

transformer_parameters

Running

File size: 14,822 Bytes

adfaaab
 
 
 
 
97c0098
adfaaab
 
 
 
 
 
 
 
 
 
 
01bae5c
adfaaab
01bae5c
adfaaab
01bae5c
adfaaab
 
97c0098
 
 
 
adfaaab
01bae5c
adfaaab
 
 
 
 
 
 
721a254
01bae5c
 
adfaaab
 
 
01bae5c
adfaaab
f23606c
adfaaab
 
 
01bae5c
adfaaab
 
 
01bae5c
 
adfaaab
 
01bae5c
 
 
 
 
 
 
721a254
 
01bae5c
 
 
 
f23606c
01bae5c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
721a254
adfaaab
 
97c0098
 
 
 
 
 
 
 
 
 
 
 
 
adfaaab
 
01bae5c
 
adfaaab
 
 
97c0098
 
adfaaab
97c0098
adfaaab
01bae5c
 
 
adfaaab
01bae5c
 
 
 
 
 
 
 
 
 
 
 
adfaaab
 
01bae5c
 
 
 
 
 
97c0098
 
 
 
01bae5c
97c0098
adfaaab
97c0098
 
 
 
 
adfaaab
01bae5c
adfaaab
 
 
 
97c0098
 
 
 
 
adfaaab
 
 
97c0098
 
 
 
 
adfaaab
01bae5c
 
97c0098
 
 
 
 
01bae5c
 
 
97c0098
2464571
01bae5c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
97c0098
01bae5c
 
 
 
 
97c0098
01bae5c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
97c0098
 
 
 
 
01bae5c
 
f23606c
97c0098
 
 
 
 
 
01bae5c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
f23606c
 
97c0098
 
f23606c
01bae5c
adfaaab
 
 
 
 
 
97c0098
f23606c
01bae5c
adfaaab
f23606c
01bae5c
adfaaab
f23606c
01bae5c
adfaaab
f23606c
01bae5c
 
 
 
adfaaab
f23606c
01bae5c
adfaaab
f23606c
adfaaab
 
01bae5c
f23606c
adfaaab
97c0098
 
 
 
 
 
 
 
 
 
 
 
 
 
adfaaab
 
 
f23606c
38cb18d
97c0098
adfaaab
 
 
124eaa7

<!DOCTYPE html>
<html lang="en">
<head>
    <meta charset="UTF-8">
    <meta name="viewport" content="width=device-width, initial-scale=1.0">
    <title>Model Parameter Calculator</title>
    <style>
        body {
            font-family: Arial, sans-serif;
            margin: 0;
            padding: 0;
            display: flex;
            flex-direction: row;
            height: 100vh;
        }

        .console {
            width: 20%;
            padding: 20px;
            background-color: #f6b5b5;
            overflow-y: auto;
            border-right: 2px solid #ee4a4f;
        }

        #additionalFieldsMOE {
            display: none;
        }

        .output {
            width: 80%;
            padding: 20px;
            overflow-y: auto;
        }

        table {
            width: 100%;
            border-collapse: collapse;
            margin-bottom: 20px;
            background-color: #fff7f9;
            border: 1px solid #f6b5b5;
        }

        th, td {
            border: 1px solid #f6b5b5;
            padding: 8px;
            text-align: center;
        }

        th {
            background-color: #f6d4d4;
        }

        .highlight {
            background-color: #ffe8e8;
            color: #ad0d0d;
            padding: 10px;
            margin: 20px 0;
            border: 1px solid #f6b5b5;
            border-radius: 4px;
            text-align: center;
        }

        h3 {
            color: #ad0d0d;
        }

        .section {
            margin-bottom: 20px;
        }
        label {
            font-weight: bold;
            font-size: 0.9rem;
        }

        input[type="number"], select {
            width: 100%;
            padding: 8px;
            border: 1px solid #ddd;
            border-radius: 5px;
            box-sizing: border-box;
            font-size: 0.9rem;
        }

        input[type="number"]:focus, select:focus {
            border-color: #ff6666;
            outline: none;
            box-shadow: 0px 0px 5px rgba(255, 102, 102, 0.5);
        }

        button {
            background-color: #ff6666;
            color: white;
            border: none;
            border-radius: 5px;
            padding: 5px;
            width: 100%;
            font-size: 1rem;
            cursor: pointer;
            transition: background-color 0.3s ease;
        }

        button:hover {
            background-color: #e60000;
        }
    </style>
    <script>
        function handleSelectChange() {
            // Récupère la valeur sélectionnée
            const isMoe = document.getElementById('is_moe').value;
            const additionalFieldsMOE = document.getElementById('additionalFieldsMOE');
            
            // Affiche ou cache les champs supplémentaires
            if (isMoe === 'yes') {
                additionalFieldsMOE.style.display = 'block';
            } else {
                additionalFieldsMOE.style.display = 'none';
            }
        }

        function calculateParameters() {
            const hiddenSize = parseInt(document.getElementById('hidden_size').value);
            const num_kv_heads = parseInt(document.getElementById('num_key_value_heads').value);
            const num_heads = parseInt(document.getElementById('num_heads').value);
            const intermediateSize = parseInt(document.getElementById('intermediate_size').value);
            const vocabSize = parseInt(document.getElementById('vocab_size').value);
            const numHiddenLayers = parseInt(document.getElementById('num_hidden_layers').value);
            const active_expert_number = parseInt(document.getElementById('active_expert_number').value);
            const expert_number = parseInt(document.getElementById('expert_number').value);
            const includeBias = document.getElementById('include_bias').value === 'yes';
            const isMoe = document.getElementById('is_moe').value === 'yes';

            const calcTotal = (input, output) => input * output;
            const calcKVdim = (hiddenSize, num_heads,num_kv_heads) => hiddenSize * num_kv_heads / num_heads;

            // Attention calculations
            const attention = [
                { name: 'Query', input: hiddenSize, output: hiddenSize },
                { name: 'Key', input: hiddenSize, output: calcKVdim(hiddenSize, num_heads,num_kv_heads) },
                { name: 'Value', input: hiddenSize, output: calcKVdim(hiddenSize, num_heads,num_kv_heads) },
                { name: 'Projection', input: hiddenSize, output: hiddenSize }
            ].map(entry => ({
                ...entry,
                wTotal: calcTotal(entry.input, entry.output),
                bTotal: includeBias ? entry.output : 0
            }));

            const attentionTotal = attention.reduce((sum, entry) => sum + entry.wTotal + entry.bTotal, 0);

            // Feed Forward calculations
            const switchW = calcTotal(hiddenSize, intermediateSize);
            const switchB = includeBias ? intermediateSize : 0;
            const luW = calcTotal(hiddenSize, intermediateSize);
            const luB = includeBias ? intermediateSize : 0;
            const projW = calcTotal(intermediateSize, hiddenSize);
            const projB = includeBias ? hiddenSize : 0;
            let feedForwardTotal;

            feedForwardTotal = switchW + switchB + luW + luB + projW + projB;
            const feedForwardTotalCache = feedForwardTotal

            let feedForwardTotalActive;

            if (isMoe) {
                feedForwardTotalActive = feedForwardTotalCache * active_expert_number;
                feedForwardTotal = feedForwardTotalCache * expert_number;
                console.log(feedForwardTotal)
            }
            // Embedding
            const embeddingTotal = calcTotal(vocabSize, hiddenSize);

            // 1 layer parameters
            const oneLayerParams = attentionTotal + feedForwardTotal;

            let oneLayerParamsActive;
            if (isMoe) {
                oneLayerParamsActive = attentionTotal + feedForwardTotalActive;
            }

            // Full layers parameters
            const fullLayersParams = oneLayerParams * numHiddenLayers;

            let fullLayersParamsMOEActive;
            if (isMoe) {
                fullLayersParamsMOEActive = oneLayerParamsActive * numHiddenLayers;
            }

            // Full size (includes embedding)
            const fullSize = fullLayersParams + embeddingTotal;

            let fullSizeActive;
            if (isMoe) {
                fullSizeActive = fullLayersParamsMOEActive + embeddingTotal;
            }
            
            // Display results
            const outputDiv = document.getElementById('output');
            outputDiv.innerHTML = `
                <h1>Model Parameter Calculator</h1>
                <div class="section">
                    <h3>Attention</h3>
                    <table>
                        <tr>
                            <th>Parameter</th>
                            <th>Input Size</th>
                            <th>Output Size</th>
                            <th>Total parameters (input*ouput)</th>
                        </tr>
                        ${attention.map(entry => `
                            <tr>
                                <td>${entry.name} - W</td>
                                <td>${entry.input.toLocaleString()}</td>
                                <td>${entry.output.toLocaleString()}</td>
                                <td>${entry.wTotal.toLocaleString()}</td>
                            </tr>
                            ${includeBias ? `
                            <tr>
                                <td>${entry.name} - b</td>
                                <td>-</td>
                                <td>${entry.output.toLocaleString()}</td>
                                <td>${entry.bTotal.toLocaleString()}</td>
                            </tr>
                            ` : ''}
                        `).join('')}
                        <tr>
                            <th colspan="3">Total attention parameters</th>
                            <td colspan="2">${attentionTotal.toLocaleString()}</td>
                        </tr>
                    </table>
                </div>
                <div class="section">
                    <h3>Feed Forward</h3>
                    <table>
                        <tr>
                            <th>Parameter</th>
                            <th>Input Size</th>
                            <th>Output Size</th>
                            <th>Total (input*output)</th>
                        </tr>
                        <tr>
                            <td>Swish - W</td>
                            <td>${hiddenSize.toLocaleString()}</td>
                            <td>${intermediateSize.toLocaleString()}</td>
                            <td>${switchW.toLocaleString()}</td>
                        </tr>
                        ${includeBias ? `<tr>
                            <td>Swish - b</td>
                            <td>-</td>
                            <td>${intermediateSize.toLocaleString()}</td>
                            <td>${switchB.toLocaleString()}</td>
                        </tr>` : ''}
                        <tr>
                            <td>GLU W</td>
                            <td>${hiddenSize.toLocaleString()}</td>
                            <td>${intermediateSize.toLocaleString()}</td>
                            <td>${luW.toLocaleString()}</td>
                        </tr>
                        ${includeBias ? `<tr>
                            <td>GLU - b</td>
                            <td>-</td>
                            <td>${intermediateSize.toLocaleString()}</td>
                            <td>${luB.toLocaleString()}</td>
                        </tr>` : ''}
                        <tr>
                            <td>Projection - W</td>
                            <td>${intermediateSize.toLocaleString()}</td>
                            <td>${hiddenSize.toLocaleString()}</td>
                            <td>${projW.toLocaleString()}</td>
                        </tr>
                        ${includeBias ? `<tr>
                            <td>Projection - b</td>
                            <td>-</td>
                            <td>${hiddenSize.toLocaleString()}</td>
                            <td>${projB.toLocaleString()}</td>
                        </tr>` : ''}
                        <tr>
                            <th colspan="3">Total Feed Forward parameters</th>
                            <td>${feedForwardTotal.toLocaleString()}</td>
                        </tr>
                        ${isMoe ?`
                        <tr>
                            <th colspan="3">Total active Feed Forward parameters</th>
                            <td>${feedForwardTotalActive.toLocaleString()}</td>
                        </tr>` : ''}
                    </table>
                </div>
                <div class="highlight">
                    <strong>1 Layer Parameters (Attention + FFN):</strong> ${oneLayerParams.toLocaleString()}<br>
                    <strong>Full Layers Parameters (1 layer parameters * num layers):</strong> ${fullLayersParams.toLocaleString()}<br><br>
                    ${isMoe ? `<tr>
                            <strong>1 Layer Parameters Active(Attention + FFN):</strong> ${oneLayerParamsActive.toLocaleString()}<br>
                            <strong>Full Layers Parameters Active(1 layer active parameters * num layers):</strong> ${fullLayersParamsMOEActive.toLocaleString()}<br>
                        </tr>` : ''}
                </div>
                <div class="section">
                    <h3>Embedding</h3>
                    <table>
                        <tr>
                            <th>Parameter</th>
                            <th>Vocab Size</th>
                            <th>Hidden Size</th>
                            <th>Total (vocab * hidden)</th>
                        </tr>
                        <tr>
                            <td>Embedding</td>
                            <td>${vocabSize.toLocaleString()}</td>
                            <td>${hiddenSize.toLocaleString()}</td>
                            <td>${embeddingTotal.toLocaleString()}</td>
                        </tr>
                    </table>
                </div>
                <div class="highlight">
                    <strong>Complete Model Parmeters (embedding size + full layers size):</strong> ${fullSize.toLocaleString()}<br><br>
                    ${isMoe ? `<strong>Complete Model Parmeters Active:</strong> ${fullSizeActive.toLocaleString()}` : ''}
                </div>
            `;
        }
    </script>
</head>
<body>

<div class="console">
    <h3>Input Parameters</h3>
    <label for="hidden_size">Hidden size:</label><br>
    <input type="number" id="hidden_size" value="896"><br><br>

    <label for="intermediate_size">Intermediate size:</label><br>
    <input type="number" id="intermediate_size" value="4864"><br><br>

    <label for="vocab_size">Vocab size:</label><br>
    <input type="number" id="vocab_size" value="151646"><br><br>

    <label for="num_key_value_heads">Number of key-value heads:</label><br>
    <input type="number" id="num_key_value_heads" value="2"><br><br>

    <label for="num_heads">Number of attention (query) heads:</label><br>
    <input type="number" id="num_heads" value="14"><br><br>

    <label for="num_hidden_layers">Number of hidden layers:</label><br>
    <input type="number" id="num_hidden_layers" value="24"><br><br>

    <label for="include_bias">Include bias?</label><br>
    <select id="include_bias">
        <option value="no">No</option>
        <option value="yes">Yes</option>
    </select><br><br>

    <label for="is_moe">Is MOE ?</label><br>
    <select id="is_moe" onchange="handleSelectChange()">
        <option value="no">No</option>
        <option value="yes">Yes</option>
    </select><br><br>

    <div id="additionalFieldsMOE">
        <label for="expert_number">Total expert number :</label><br>
        <input type="number" id="expert_number" name="expert_number"><br><br>

        <label for="active_expert_number">Total active experts (shared + specifics):</label><br>
        <input type="number" id="active_expert_number" name="active_expert_number"><br><br>
    </div>

    <button onclick="calculateParameters()">Calculate</button>
</div>

<div class="output" id="output">
    <h1>Transformer total number of parameters Calculator</h1>
    <h3>Enter model hyperparameters in the console and press calculate (curently working for classic transformer/LLM architecture with GQA and GLU)</h3>
</div>

</body>
</html>