Upload folder using huggingface_hub

README.md

@@ -0,0 +1,82 @@
+---
+tags:
+- model_hub_mixin
+- pytorch_model_hub_mixin
+- audio
+- rhythm-game
+- music
+---
+
+# GameChartEvaluator (GCE4)
+
+A neural network model for evaluating the quality of rhythm game charts relative to their corresponding music. The model predicts a quality score (0-1) indicating how well a chart synchronizes with the music.
+
+## Model Architecture
+
+The model uses an early fusion approach with dilated convolutions for temporal analysis:
+
+1. **Early Fusion**: Concatenates music and chart mel spectrograms along the channel dimension (80 + 80 = 160 channels)
+2. **Dilated Residual Encoder**: 4 residual blocks with increasing dilation rates (1, 2, 4, 8) to capture multi-scale temporal context while preserving 11ms frame resolution
+3. **Error-Sensitive Scoring Head**: Combines average local scores with the worst 10% of scores using a learnable mixing parameter
+
+```
+Input: (B, 80, T) music_mels + (B, 80, T) chart_mels
+  ↓ Concatenate
+(B, 160, T)
+  ↓ Conv1D Projection
+(B, 128, T)
+  ↓ Dilated ResBlocks × 4
+(B, 128, T)
+  ↓ Linear → Sigmoid (per-frame scores)
+(B, T, 1)
+  ↓ Error-Sensitive Pooling
+(B,) final score
+```
+
+## Usage
+
+```python
+import torch
+from gce4 import GameChartEvaluator
+
+model = GameChartEvaluator.from_pretrained("JacobLinCool/gce4")
+model.eval()
+
+# Input: 80-band mel spectrograms
+music_mels = torch.randn(1, 80, 1000)  # (batch, freq, time)
+chart_mels = torch.randn(1, 80, 1000)
+
+# Get overall quality score (0-1)
+with torch.no_grad():
+    score = model(music_mels, chart_mels)
+    print(f"Quality Score: {score.item():.3f}")
+
+# Get per-frame quality trace for explainability
+with torch.no_grad():
+    trace = model.predict_trace(music_mels, chart_mels)
+    # trace shape: (batch, time)
+```
+
+## Input Specifications
+
+- **music_mels**: `(Batch, 80, Time)` - Mel spectrogram of the music
+- **chart_mels**: `(Batch, 80, Time)` - Mel spectrogram of synthesized chart audio (click sounds at note positions)
+
+Both inputs should be normalized and have the same temporal dimensions.
+
+## Output
+
+- **forward()**: `(Batch,)` - Single quality score per sample in range [0, 1]
+- **predict_trace()**: `(Batch, Time)` - Per-frame quality scores for interpretability
+
+## Model Configuration
+
+| Parameter | Default | Description |
+|-----------|---------|-------------|
+| `input_dim` | 80 | Mel spectrogram frequency bins |
+| `d_model` | 128 | Hidden dimension |
+| `n_layers` | 4 | Number of residual blocks |
+
+## Training
+
+The model was trained to detect misaligned or poorly-synchronized rhythm game charts by comparing music-chart pairs with various synthetic corruptions (time shifts, random note placement, etc).

config.json

@@ -0,0 +1,5 @@
+{
+  "d_model": 128,
+  "input_dim": 80,
+  "n_layers": 4
+}

gce4.py

@@ -0,0 +1,138 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import math
+from huggingface_hub import PyTorchModelHubMixin
+
+
+class ResBlock1D(nn.Module):
+    """
+    Residual Block for extracting rhythmic features from audio spectrograms.
+    Maintains temporal resolution while increasing receptive field.
+    """
+
+    def __init__(self, channels, kernel_size=3, dilation=1):
+        super().__init__()
+        padding = (kernel_size - 1) * dilation // 2
+        self.conv1 = nn.Conv1d(
+            channels, channels, kernel_size, padding=padding, dilation=dilation
+        )
+        self.bn1 = nn.BatchNorm1d(channels)
+        self.conv2 = nn.Conv1d(
+            channels, channels, kernel_size, padding=padding, dilation=dilation
+        )
+        self.bn2 = nn.BatchNorm1d(channels)
+
+    def forward(self, x):
+        res = x
+        x = F.gelu(self.bn1(self.conv1(x)))
+        x = self.bn2(self.conv2(x))
+        return F.gelu(x + res)
+
+
+class GameChartEvaluator(nn.Module, PyTorchModelHubMixin):
+    def __init__(self, input_dim=80, d_model=128, n_layers=4):
+        super().__init__()
+
+        # --- Early Fusion ---
+        # Input is (Batch, 80 * 2, Time)
+        # We stack Music (80) + Chart (80) = 160 channels
+        self.input_proj = nn.Conv1d(
+            input_dim * 2, d_model, kernel_size=3, stride=1, padding=1
+        )
+
+        # --- STRICT TEMPORAL ENCODER ---
+        # No Pooling (stride=1) to preserve 11ms resolution
+        # Dilations allow seeing context without losing resolution
+        self.encoder = nn.Sequential(
+            ResBlock1D(d_model, kernel_size=3, dilation=1),
+            ResBlock1D(d_model, kernel_size=3, dilation=2),
+            ResBlock1D(d_model, kernel_size=3, dilation=4),
+            ResBlock1D(d_model, kernel_size=3, dilation=8),
+            # Add more layers if you need wider context (e.g. 16, 32)
+        )
+
+        # --- SCORING HEAD ---
+        # Simple projection to scalar
+        self.quality_proj = nn.Linear(d_model, 1)
+
+        # Learnable Mixing
+        self.raw_severity = nn.Parameter(torch.tensor(0.0))
+
+    def forward(self, music_mels, chart_mels):
+        """
+        music_mels: (Batch, 80, Time)
+        chart_mels: (Batch, 80, Time)
+        """
+        # 1. Early Fusion: Concatenate along Channel dimension
+        # Shape becomes (Batch, 160, Time)
+        x = torch.cat([music_mels, chart_mels], dim=1)
+
+        # 2. Extract Features (Strictly Local + Context)
+        x = F.gelu(self.input_proj(x))
+        x = self.encoder(x)
+
+        # 3. Predict Score per Frame
+        # (Batch, Dim, Time) -> (Batch, Time, Dim)
+        x = x.permute(0, 2, 1)
+        local_scores = torch.sigmoid(self.quality_proj(x))  # (Batch, Time, 1)
+
+        # 4. Error-Sensitive Pooling
+        avg_score = local_scores.mean(dim=1)
+
+        k = max(1, int(local_scores.size(1) * 0.1))
+        min_vals, _ = torch.topk(local_scores, k, dim=1, largest=False)
+        worst_score = min_vals.mean(dim=1)
+
+        alpha = torch.sigmoid(self.raw_severity)
+        final_score = (alpha * worst_score) + ((1 - alpha) * avg_score)
+
+        return final_score.squeeze(1)
+
+    def predict_trace(self, music_mels, chart_mels):
+        """
+        Explainability Method: Returns the second-by-second quality curve.
+
+        Returns:
+            local_scores: (Batch, Time) - The quality score at every timestep.
+        """
+        with torch.no_grad():
+            # 1. Early Fusion: Concatenate along Channel dimension
+            # Shape becomes (Batch, 160, Time)
+            x = torch.cat([music_mels, chart_mels], dim=1)
+
+            # 2. Extract Features (Strictly Local + Context)
+            x = F.gelu(self.input_proj(x))
+            x = self.encoder(x)
+
+            # 3. Predict Score per Frame
+            # (Batch, Dim, Time) -> (Batch, Time, Dim)
+            x = x.permute(0, 2, 1)
+            local_scores = torch.sigmoid(self.quality_proj(x))  # (Batch, Time, 1)
+        return local_scores.squeeze(2)
+
+
+if __name__ == "__main__":
+    # Sanity Check
+    from torchinfo import summary
+
+    model = GameChartEvaluator()
+    print(
+        f"Model initialized. Learnable Severity: {torch.sigmoid(model.raw_severity).item():.2f}"
+    )
+
+    # Dummy data (Batch=2, Freq=80, Time=1000)
+    m = torch.randn(2, 80, 1000)
+    c = torch.randn(2, 80, 1000)
+
+    output = model(m, c)
+    print(f"Output shape: {output.shape}")  # Should be torch.Size([2])
+    print(f"Scores: {output}")
+
+    # Trace check
+    trace = model.predict_trace(m, c)
+    print(
+        f"Trace shape: {trace.shape}"
+    )  # Should be torch.Size([2, 500]) (due to MaxPool1d(2))
+
+    summary(model, input_data=[m, c])

logs/events.out.tfevents.1765826528.msiit232.2878790.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4ffaff1b4785cecde5e9ec55c6ccb9c5bd95ccfc05e4b441020e64cbfe2f38ae
+size 81061

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:14cdf0af591718d42744931c641f91756ac720513d3125c860548f56cca9f59d
+size 1845464