Spaces:

transformers-community
/

Transformers-tenets

Running

App Files Files Community

Molbap HF Staff commited on Sep 26

Commit

8caaff8

1 Parent(s): 68997c6

better frags

Browse files

Files changed (2) hide show

src/fragments/tp-plan.html +27 -0
src/fragments/warmup_demo.html +421 -0

src/fragments/tp-plan.html ADDED Viewed

	@@ -0,0 +1,27 @@

+    <pre><code class="language-python"># In the model's config (example: ERNIE 4.5-style decoder blocks)
+  base_model_tp_plan = {
+      "layers.*.self_attn.q_proj": "colwise",
+      "layers.*.self_attn.k_proj": "colwise",
+      "layers.*.self_attn.v_proj": "colwise",
+      "layers.*.self_attn.o_proj": "rowwise",
+      "layers.*.mlp.gate_proj": "colwise",
+      "layers.*.mlp.up_proj":   "colwise",
+      "layers.*.mlp.down_proj": "rowwise",
+  }
+  # Runtime
+  import torch
+  from transformers import AutoModelForCausalLM, AutoTokenizer
+  model_id = "your/model-or-local-checkpoint"
+  model = AutoModelForCausalLM.from_pretrained(
+      model_id,
+      dtype=torch.bfloat16,
+      tp_plan=base_model_tp_plan,   # <-- plan defined above
+  )
+  tok = AutoTokenizer.from_pretrained(model_id)
+  inputs = tok("Hello", return_tensors="pt").to(model.device)
+  out = model(**inputs)</code></pre>

src/fragments/warmup_demo.html ADDED Viewed

	@@ -0,0 +1,421 @@

+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <style>
+        body {
+            font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;
+            margin: 0;
+            padding: 20px;
+            background-color: #f5f5f5;
+        }
+        .container {
+            max-width: 1200px;
+            margin: 0 auto;
+            background: white;
+            border-radius: 12px;
+            padding: 30px;
+            box-shadow: 0 4px 6px rgba(0, 0, 0, 0.1);
+        }
+        h1 {
+            text-align: center;
+            color: #333;
+            margin-bottom: 10px;
+        }
+        .subtitle {
+            text-align: center;
+            color: #666;
+            margin-bottom: 30px;
+            font-size: 16px;
+        }
+        .demo-container {
+            display: flex;
+            gap: 40px;
+            margin-bottom: 30px;
+        }
+        .side {
+            flex: 1;
+            border: 2px solid #ddd;
+            border-radius: 8px;
+            padding: 20px;
+            background: #fafafa;
+        }
+        .side h2 {
+            text-align: center;
+            margin-top: 0;
+            color: #333;
+        }
+        .no-warmup h2 {
+            color: #d63384;
+        }
+        .with-warmup h2 {
+            color: #198754;
+        }
+        .memory-area {
+            height: 400px;
+            border: 2px dashed #ccc;
+            border-radius: 6px;
+            padding: 10px;
+            margin: 20px 0;
+            background: #fff;
+            position: relative;
+            overflow: hidden;
+        }
+        .layer-box {
+            width: 80px;
+            height: 30px;
+            border: 2px solid #666;
+            border-radius: 4px;
+            margin: 3px;
+            display: inline-block;
+            position: relative;
+            background: #fff;
+            transition: all 0.3s ease;
+        }
+        .layer-box.allocating {
+            background: #e9ecef;
+            border-color: #adb5bd;
+        }
+        .layer-box.allocating::after {
+            content: "malloc";
+            position: absolute;
+            top: 50%;
+            left: 50%;
+            transform: translate(-50%, -50%);
+            font-size: 10px;
+            color: #666;
+            font-weight: bold;
+        }
+        .layer-box.loaded {
+            background: #d1e7dd;
+            border-color: #198754;
+        }
+        .layer-box.loaded::after {
+            content: "data";
+            position: absolute;
+            top: 50%;
+            left: 50%;
+            transform: translate(-50%, -50%);
+            font-size: 10px;
+            color: #198754;
+            font-weight: bold;
+        }
+        .warmup-container {
+            width: 100%;
+            height: 60px;
+            border: 3px solid #666;
+            border-radius: 6px;
+            margin-bottom: 20px;
+            background: #fff;
+            position: relative;
+            overflow: hidden;
+        }
+        .warmup-container.allocated {
+            border-color: #0d6efd;
+            background: #e7f1ff;
+        }
+        .warmup-container::before {
+            content: "Pre-allocated Memory Pool";
+            position: absolute;
+            top: 50%;
+            left: 50%;
+            transform: translate(-50%, -50%);
+            font-size: 14px;
+            color: #666;
+            font-weight: bold;
+            z-index: 1;
+        }
+        .warmup-container.allocated::before {
+            color: #0d6efd;
+        }
+        .warmup-fill {
+            height: 100%;
+            background: linear-gradient(90deg, #198754, #20c997);
+            width: 0%;
+            transition: width 0.5s ease;
+            border-radius: 3px;
+            position: relative;
+            z-index: 2;
+        }
+        .warmup-fill::after {
+            content: "Layer Data Loading";
+            position: absolute;
+            top: 50%;
+            left: 50%;
+            transform: translate(-50%, -50%);
+            font-size: 12px;
+            color: white;
+            font-weight: bold;
+            white-space: nowrap;
+        }
+        .timing {
+            text-align: center;
+            font-size: 24px;
+            font-weight: bold;
+            margin: 15px 0;
+            min-height: 30px;
+        }
+        .no-warmup .timing {
+            color: #d63384;
+        }
+        .with-warmup .timing {
+            color: #198754;
+        }
+        .controls {
+            text-align: center;
+            margin: 30px 0;
+        }
+        .btn {
+            background: #0d6efd;
+            color: white;
+            border: none;
+            padding: 12px 24px;
+            border-radius: 6px;
+            font-size: 16px;
+            cursor: pointer;
+            margin: 0 10px;
+            transition: background 0.3s ease;
+        }
+        .btn:hover {
+            background: #0b5ed7;
+        }
+        .btn:disabled {
+            background: #6c757d;
+            cursor: not-allowed;
+        }
+        .description {
+            background: #f8f9fa;
+            padding: 15px;
+            border-radius: 6px;
+            margin-top: 15px;
+            font-size: 14px;
+            line-height: 1.5;
+        }
+        .phase-indicator {
+            font-size: 14px;
+            color: #666;
+            text-align: center;
+            margin-top: 10px;
+            min-height: 20px;
+        }
+        .layer-counter {
+            text-align: center;
+            font-size: 16px;
+            color: #495057;
+            margin: 10px 0;
+        }
+    </style>
+</head>
+<body>
+    <div class="container">
+        <h1>CUDA Caching Allocator Warmup Visualization</h1>
+        <p class="subtitle">Demonstrating memory allocation patterns during model loading</p>
+        <div class="controls">
+            <button class="btn" id="startBtn" onclick="startDemo()">Start Animation</button>
+            <button class="btn" id="resetBtn" onclick="resetDemo()">Reset</button>
+        </div>
+        <div class="demo-container">
+            <div class="side no-warmup">
+                <h2 data-no-toc>❌ Without Warmup</h2>
+                <div class="timing" id="noWarmupTime">0.00s</div>
+                <div class="layer-counter" id="noWarmupCounter">Layers loaded: 0/10</div>
+                <div class="phase-indicator" id="noWarmupPhase"></div>
+                <div class="memory-area" id="noWarmupArea"></div>
+                <div class="description">
+                    <strong>Individual Allocations:</strong><br>
+                    Each model layer triggers a separate cudaMalloc() call, creating memory fragmentation and allocation overhead.
+                    <br><br>
+                    📦 <strong>Grey "malloc"</strong> = Memory allocation overhead<br>
+                    ✅ <strong>Green "data"</strong> = Actual layer data loading
+                </div>
+            </div>
+            <div class="side with-warmup">
+                <h2 data-no-toc>✅ With Warmup</h2>
+                <div class="timing" id="warmupTime">0.00s</div>
+                <div class="layer-counter" id="warmupCounter">Layers loaded: 0/10</div>
+                <div class="phase-indicator" id="warmupPhase"></div>
+                <div class="memory-area" id="warmupArea">
+                    <div class="warmup-container" id="warmupContainer">
+                        <div class="warmup-fill" id="warmupFill"></div>
+                    </div>
+                    <div id="warmupLayers"></div>
+                </div>
+                <div class="description">
+                    <strong>Pre-allocated Pool:</strong><br>
+                    The warmup function calculates total memory needed and makes ONE large allocation. Subsequent layers load directly into this pool, eliminating malloc overhead.
+                    <br><br>
+                    🔵 <strong>Blue container</strong> = Single large malloc (warmup)<br>
+                    🟢 <strong>Green progress bar</strong> = Layer data loading (no malloc needed)
+                </div>
+            </div>
+        </div>
+    </div>
+    <script>
+        let animationSpeed = 1 / 2.4; // Slower animation
+        let isRunning = false;
+        const totalLayers = 10;
+        function startDemo() {
+            if (isRunning) return;
+            isRunning = true;
+            document.getElementById('startBtn').disabled = true;
+            document.getElementById('resetBtn').disabled = true;
+            // Start both animations simultaneously
+            Promise.all([
+                animateNoWarmup(),
+                animateWithWarmup()
+            ]).then(() => {
+                isRunning = false;
+                document.getElementById('startBtn').disabled = false;
+                document.getElementById('resetBtn').disabled = false;
+            });
+        }
+        function resetDemo() {
+            if (isRunning) return;
+            // Clear containers
+            document.getElementById('noWarmupArea').innerHTML = '';
+            document.getElementById('warmupLayers').innerHTML = '';
+            document.getElementById('warmupFill').style.width = '0%';
+            document.getElementById('warmupContainer').classList.remove('allocated');
+            // Reset timers
+            document.getElementById('noWarmupTime').textContent = '0.00s';
+            document.getElementById('warmupTime').textContent = '0.00s';
+            // Reset counters
+            document.getElementById('noWarmupCounter').textContent = 'Layers loaded: 0/10';
+            document.getElementById('warmupCounter').textContent = 'Layers loaded: 0/10';
+            // Reset phase indicators
+            document.getElementById('noWarmupPhase').textContent = '';
+            document.getElementById('warmupPhase').textContent = '';
+        }
+        async function animateNoWarmup() {
+            const container = document.getElementById('noWarmupArea');
+            const timeEl = document.getElementById('noWarmupTime');
+            const counterEl = document.getElementById('noWarmupCounter');
+            const phaseEl = document.getElementById('noWarmupPhase');
+            let currentTime = 0;
+            const baseDelay = 200 / animationSpeed; // Base delay between operations
+            phaseEl.textContent = 'Loading model layers...';
+            for (let i = 0; i < totalLayers; i++) {
+                // Create layer box
+                const layerBox = document.createElement('div');
+                layerBox.className = 'layer-box';
+                container.appendChild(layerBox);
+                // Malloc phase (grey)
+                await sleep(baseDelay * 0.3);
+                layerBox.classList.add('allocating');
+                currentTime += 0.08; // malloc overhead
+                timeEl.textContent = currentTime.toFixed(2) + 's';
+                // Data loading phase (green)
+                await sleep(baseDelay * 0.7);
+                layerBox.classList.remove('allocating');
+                layerBox.classList.add('loaded');
+                currentTime += 0.12; // data transfer time
+                timeEl.textContent = currentTime.toFixed(2) + 's';
+                // Update counter
+                counterEl.textContent = `Layers loaded: ${i + 1}/${totalLayers}`;
+            }
+            phaseEl.textContent = 'Complete!';
+        }
+        async function animateWithWarmup() {
+            const container = document.getElementById('warmupLayers');
+            const timeEl = document.getElementById('warmupTime');
+            const counterEl = document.getElementById('warmupCounter');
+            const phaseEl = document.getElementById('warmupPhase');
+            const warmupContainer = document.getElementById('warmupContainer');
+            const warmupFill = document.getElementById('warmupFill');
+            let currentTime = 0;
+            const baseDelay = 200 / animationSpeed;
+            // Warmup phase
+            phaseEl.textContent = 'Warming up allocator...';
+            await sleep(baseDelay * 2);
+            warmupContainer.classList.add('allocated');
+            currentTime += 0.3; // warmup overhead
+            timeEl.textContent = currentTime.toFixed(2) + 's';
+            phaseEl.textContent = 'Loading model layers...';
+            // Load layers directly into pre-allocated memory
+            for (let i = 0; i < totalLayers; i++) {
+                // Create layer indicator (much smaller since no malloc)
+                const layerBox = document.createElement('div');
+                layerBox.className = 'layer-box loaded';
+                layerBox.style.width = '40px';
+                layerBox.style.height = '20px';
+                container.appendChild(layerBox);
+                // Update progress bar
+                const progress = ((i + 1) / totalLayers) * 100;
+                warmupFill.style.width = progress + '%';
+                // Only data transfer time (no malloc overhead)
+                await sleep(baseDelay * 0.5);
+                currentTime += 0.08; // faster due to no malloc
+                timeEl.textContent = currentTime.toFixed(2) + 's';
+                // Update counter
+                counterEl.textContent = `Layers loaded: ${i + 1}/${totalLayers}`;
+            }
+            phaseEl.textContent = 'Complete!';
+        }
+        function sleep(ms) {
+            return new Promise(resolve => setTimeout(resolve, ms));
+        }
+    </script>
+</body>
+</html>